aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorAlexey Bogolyubskiy <i@bogolyubskiyalexey.ru>2022-02-10 16:51:14 +0300
committerDaniil Cherednik <dcherednik@yandex-team.ru>2022-02-10 16:51:14 +0300
commit49023e629b5849b5f604b412acaf64558162d1d3 (patch)
tree5d5cb817648f650d76cf1076100726fd9b8448e8
parent4d3f24f89a157c192be964fae784c6597dbbeb36 (diff)
downloadydb-49023e629b5849b5f604b412acaf64558162d1d3.tar.gz
Restoring authorship annotation for Alexey Bogolyubskiy <i@bogolyubskiyalexey.ru>. Commit 2 of 2.
-rw-r--r--build/rules/kikimr.policy6
-rw-r--r--ydb/core/base/appdata.h4
-rw-r--r--ydb/core/base/grpc_service_factory.h104
-rw-r--r--ydb/core/client/server/msgbus_server.cpp30
-rw-r--r--ydb/core/client/server/msgbus_server.h26
-rw-r--r--ydb/core/client/server/msgbus_server_persqueue.cpp74
-rw-r--r--ydb/core/client/server/msgbus_server_persqueue.h22
-rw-r--r--ydb/core/client/server/msgbus_server_pq_metacache.cpp40
-rw-r--r--ydb/core/client/server/msgbus_server_pq_metarequest.cpp22
-rw-r--r--ydb/core/client/server/msgbus_server_pq_metarequest.h12
-rw-r--r--ydb/core/client/server/msgbus_server_pq_metarequest_ut.cpp14
-rw-r--r--ydb/core/client/server/msgbus_server_proxy.cpp10
-rw-r--r--ydb/core/client/server/msgbus_server_proxy.h14
-rw-r--r--ydb/core/client/server/msgbus_server_scheme_request.cpp2
-rw-r--r--ydb/core/client/server/msgbus_server_tracer.cpp44
-rw-r--r--ydb/core/client/server/msgbus_server_tracer.h14
-rw-r--r--ydb/core/client/server/ya.make4
-rw-r--r--ydb/core/cms/console/net_classifier_updater.cpp78
-rw-r--r--ydb/core/cms/console/net_classifier_updater_ut.cpp186
-rw-r--r--ydb/core/driver_lib/run/factories.h6
-rw-r--r--ydb/core/driver_lib/run/kikimr_services_initializers.cpp54
-rw-r--r--ydb/core/driver_lib/run/kikimr_services_initializers.h18
-rw-r--r--ydb/core/driver_lib/run/run.cpp46
-rw-r--r--ydb/core/driver_lib/run/run.h8
-rw-r--r--ydb/core/persqueue/actor_persqueue_client_iface.h140
-rw-r--r--ydb/core/persqueue/blob.cpp38
-rw-r--r--ydb/core/persqueue/blob.h16
-rw-r--r--ydb/core/persqueue/cache_eviction.h2
-rw-r--r--ydb/core/persqueue/cluster_tracker.cpp4
-rw-r--r--ydb/core/persqueue/config/ya.make2
-rw-r--r--ydb/core/persqueue/event_helpers.cpp78
-rw-r--r--ydb/core/persqueue/event_helpers.h44
-rw-r--r--ydb/core/persqueue/events/events.cpp4
-rw-r--r--ydb/core/persqueue/events/global.h14
-rw-r--r--ydb/core/persqueue/events/internal.h104
-rw-r--r--ydb/core/persqueue/events/ya.make32
-rw-r--r--ydb/core/persqueue/internals_ut.cpp30
-rw-r--r--ydb/core/persqueue/key.h4
-rw-r--r--ydb/core/persqueue/mirrorer.cpp1058
-rw-r--r--ydb/core/persqueue/mirrorer.h372
-rw-r--r--ydb/core/persqueue/ownerinfo.h4
-rw-r--r--ydb/core/persqueue/partition.cpp984
-rw-r--r--ydb/core/persqueue/partition.h94
-rw-r--r--ydb/core/persqueue/pq.h2
-rw-r--r--ydb/core/persqueue/pq_impl.cpp92
-rw-r--r--ydb/core/persqueue/pq_impl.h2
-rw-r--r--ydb/core/persqueue/pq_l2_cache.h6
-rw-r--r--ydb/core/persqueue/pq_l2_service.h2
-rw-r--r--ydb/core/persqueue/pq_ut.cpp30
-rw-r--r--ydb/core/persqueue/pq_ut.h8
-rw-r--r--ydb/core/persqueue/read.h2
-rw-r--r--ydb/core/persqueue/read_balancer.cpp32
-rw-r--r--ydb/core/persqueue/read_balancer.h4
-rw-r--r--ydb/core/persqueue/read_speed_limiter.cpp356
-rw-r--r--ydb/core/persqueue/read_speed_limiter.h270
-rw-r--r--ydb/core/persqueue/subscriber.cpp8
-rw-r--r--ydb/core/persqueue/subscriber.h80
-rw-r--r--ydb/core/persqueue/user_info.cpp318
-rw-r--r--ydb/core/persqueue/user_info.h232
-rw-r--r--ydb/core/persqueue/user_info_ut.cpp70
-rw-r--r--ydb/core/persqueue/ut/ya.make12
-rw-r--r--ydb/core/persqueue/ut_slow/ya.make8
-rw-r--r--ydb/core/persqueue/working_time_counter.h18
-rw-r--r--ydb/core/persqueue/write_meta.cpp104
-rw-r--r--ydb/core/persqueue/write_meta.h142
-rw-r--r--ydb/core/persqueue/writer/source_id_encoding.cpp44
-rw-r--r--ydb/core/persqueue/writer/source_id_encoding.h26
-rw-r--r--ydb/core/persqueue/ya.make14
-rw-r--r--ydb/core/protos/config.proto4
-rw-r--r--ydb/core/protos/counters_pq.proto2
-rw-r--r--ydb/core/protos/msgbus_pq.proto2
-rw-r--r--ydb/core/protos/netclassifier.proto2
-rw-r--r--ydb/core/protos/out/out.cpp216
-rw-r--r--ydb/core/protos/out/ya.make22
-rw-r--r--ydb/core/protos/pqconfig.proto138
-rw-r--r--ydb/core/protos/services.proto4
-rw-r--r--ydb/core/tablet/tablet_counters_aggregator.cpp4
-rw-r--r--ydb/core/testlib/tablet_helpers.cpp4
-rw-r--r--ydb/core/testlib/test_client.cpp28
-rw-r--r--ydb/core/testlib/test_client.h24
-rw-r--r--ydb/core/testlib/test_pq_client.h306
-rw-r--r--ydb/core/tx/scheme_board/cache.cpp16
-rw-r--r--ydb/core/tx/scheme_cache/scheme_cache.h6
-rw-r--r--ydb/core/viewer/json_pqconsumerinfo.h4
-rw-r--r--ydb/core/ymq/actor/metering.cpp182
-rw-r--r--ydb/core/ymq/actor/metering.h64
-rw-r--r--ydb/core/ymq/actor/queue_schema.cpp402
-rw-r--r--ydb/core/ymq/actor/queue_schema.h18
-rw-r--r--ydb/core/ymq/actor/ut/metering_ut.cpp692
-rw-r--r--ydb/core/ymq/actor/ut/ya.make2
-rw-r--r--ydb/core/ymq/base/counters.cpp18
-rw-r--r--ydb/core/ymq/base/counters.h40
-rw-r--r--ydb/core/ymq/base/ut/counters_ut.cpp54
-rw-r--r--ydb/library/persqueue/counter_time_keeper/counter_time_keeper.cpp2
-rw-r--r--ydb/library/persqueue/counter_time_keeper/counter_time_keeper.h42
-rw-r--r--ydb/library/persqueue/counter_time_keeper/ya.make24
-rw-r--r--ydb/library/persqueue/obfuscate/obfuscate.cpp30
-rw-r--r--ydb/library/persqueue/obfuscate/obfuscate.h18
-rw-r--r--ydb/library/persqueue/obfuscate/ya.make18
-rw-r--r--ydb/library/persqueue/topic_parser/ya.make30
-rw-r--r--ydb/library/persqueue/ya.make12
-rw-r--r--ydb/library/pretty_types_print/protobuf/out.cpp18
-rw-r--r--ydb/library/pretty_types_print/protobuf/ya.make26
-rw-r--r--ydb/library/pretty_types_print/wilson/out.cpp12
-rw-r--r--ydb/library/pretty_types_print/wilson/ya.make24
-rw-r--r--ydb/library/pretty_types_print/ya.make8
-rw-r--r--ydb/public/api/protos/draft/persqueue_common.proto82
-rw-r--r--ydb/public/api/protos/out/out.cpp20
-rw-r--r--ydb/public/api/protos/out/ya.make24
-rw-r--r--ydb/public/api/protos/ya.make4
-rw-r--r--ydb/public/api/protos/ydb_persqueue_v1.proto88
-rw-r--r--ydb/public/lib/base/msgbus.h30
-rw-r--r--ydb/public/lib/experimental/ydb_clickhouse_internal.cpp6
-rw-r--r--ydb/public/lib/experimental/ydb_experimental.cpp8
-rw-r--r--ydb/public/lib/experimental/ydb_s3_internal.cpp6
-rw-r--r--ydb/public/lib/idx_test/idx_test_data_provider.cpp4
-rw-r--r--ydb/public/sdk/cpp/client/draft/ut/ya.make44
-rw-r--r--ydb/public/sdk/cpp/client/draft/ya.make2
-rw-r--r--ydb/public/sdk/cpp/client/draft/ydb_scripting.cpp10
-rw-r--r--ydb/public/sdk/cpp/client/extensions/discovery_mutator/discovery_mutator_ut.cpp2
-rw-r--r--ydb/public/sdk/cpp/client/extensions/discovery_mutator/ut/ya.make50
-rw-r--r--ydb/public/sdk/cpp/client/extensions/discovery_mutator/ya.make2
-rw-r--r--ydb/public/sdk/cpp/client/extensions/solomon_stats/README.md2
-rw-r--r--ydb/public/sdk/cpp/client/extensions/solomon_stats/pull_client.h6
-rw-r--r--ydb/public/sdk/cpp/client/extensions/ya.make12
-rw-r--r--ydb/public/sdk/cpp/client/impl/ydb_endpoints/endpoints.cpp12
-rw-r--r--ydb/public/sdk/cpp/client/impl/ydb_endpoints/endpoints_ut.cpp6
-rw-r--r--ydb/public/sdk/cpp/client/impl/ydb_endpoints/ut/ya.make44
-rw-r--r--ydb/public/sdk/cpp/client/impl/ydb_endpoints/ya.make4
-rw-r--r--ydb/public/sdk/cpp/client/impl/ydb_internal/common/types.h38
-rw-r--r--ydb/public/sdk/cpp/client/impl/ydb_internal/common/ya.make14
-rw-r--r--ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/authenticator.cpp66
-rw-r--r--ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/authenticator.h52
-rw-r--r--ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/endpoint_pool.cpp222
-rw-r--r--ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/endpoint_pool.h108
-rw-r--r--ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/state.cpp388
-rw-r--r--ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/state.h120
-rw-r--r--ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/ya.make30
-rw-r--r--ydb/public/sdk/cpp/client/impl/ydb_internal/driver/constants.h24
-rw-r--r--ydb/public/sdk/cpp/client/impl/ydb_internal/driver/interface.h22
-rw-r--r--ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/actions.cpp218
-rw-r--r--ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/actions.h418
-rw-r--r--ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/grpc_connections.cpp750
-rw-r--r--ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/grpc_connections.h1190
-rw-r--r--ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/params.h48
-rw-r--r--ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/ya.make24
-rw-r--r--ydb/public/sdk/cpp/client/impl/ydb_internal/internal_client/client.h58
-rw-r--r--ydb/public/sdk/cpp/client/impl/ydb_internal/internal_header.h22
-rw-r--r--ydb/public/sdk/cpp/client/impl/ydb_internal/logger/log.cpp90
-rw-r--r--ydb/public/sdk/cpp/client/impl/ydb_internal/logger/log.h20
-rw-r--r--ydb/public/sdk/cpp/client/impl/ydb_internal/logger/ya.make24
-rw-r--r--ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.cpp24
-rw-r--r--ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.h78
-rw-r--r--ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/ya.make24
-rw-r--r--ydb/public/sdk/cpp/client/impl/ydb_internal/plain_status/status.cpp116
-rw-r--r--ydb/public/sdk/cpp/client/impl/ydb_internal/plain_status/status.h94
-rw-r--r--ydb/public/sdk/cpp/client/impl/ydb_internal/plain_status/ya.make22
-rw-r--r--ydb/public/sdk/cpp/client/impl/ydb_internal/rpc_request_settings/settings.h34
-rw-r--r--ydb/public/sdk/cpp/client/impl/ydb_internal/stats_extractor/extractor.h82
-rw-r--r--ydb/public/sdk/cpp/client/impl/ydb_internal/table_helpers/helpers.h68
-rw-r--r--ydb/public/sdk/cpp/client/impl/ydb_internal/thread_pool/pool.cpp4
-rw-r--r--ydb/public/sdk/cpp/client/impl/ydb_internal/thread_pool/pool.h36
-rw-r--r--ydb/public/sdk/cpp/client/impl/ydb_internal/thread_pool/ya.make16
-rw-r--r--ydb/public/sdk/cpp/client/impl/ydb_internal/value_helpers/helpers.cpp190
-rw-r--r--ydb/public/sdk/cpp/client/impl/ydb_internal/value_helpers/helpers.h18
-rw-r--r--ydb/public/sdk/cpp/client/impl/ydb_internal/value_helpers/ya.make22
-rw-r--r--ydb/public/sdk/cpp/client/impl/ydb_stats/stats.cpp2
-rw-r--r--ydb/public/sdk/cpp/client/impl/ydb_stats/stats.h6
-rw-r--r--ydb/public/sdk/cpp/client/impl/ydb_stats/ya.make34
-rw-r--r--ydb/public/sdk/cpp/client/ydb_common_client/impl/client.cpp2
-rw-r--r--ydb/public/sdk/cpp/client/ydb_common_client/impl/client.h6
-rw-r--r--ydb/public/sdk/cpp/client/ydb_common_client/impl/ya.make30
-rw-r--r--ydb/public/sdk/cpp/client/ydb_common_client/settings.cpp16
-rw-r--r--ydb/public/sdk/cpp/client/ydb_common_client/settings.h54
-rw-r--r--ydb/public/sdk/cpp/client/ydb_common_client/ya.make30
-rw-r--r--ydb/public/sdk/cpp/client/ydb_coordination/coordination.cpp10
-rw-r--r--ydb/public/sdk/cpp/client/ydb_coordination/coordination.h10
-rw-r--r--ydb/public/sdk/cpp/client/ydb_coordination/ut/ya.make50
-rw-r--r--ydb/public/sdk/cpp/client/ydb_coordination/ya.make36
-rw-r--r--ydb/public/sdk/cpp/client/ydb_datastreams/ya.make26
-rw-r--r--ydb/public/sdk/cpp/client/ydb_discovery/discovery.cpp4
-rw-r--r--ydb/public/sdk/cpp/client/ydb_discovery/discovery.h2
-rw-r--r--ydb/public/sdk/cpp/client/ydb_discovery/ya.make30
-rw-r--r--ydb/public/sdk/cpp/client/ydb_driver/driver.cpp4
-rw-r--r--ydb/public/sdk/cpp/client/ydb_driver/driver_ut.cpp148
-rw-r--r--ydb/public/sdk/cpp/client/ydb_driver/ut/ya.make50
-rw-r--r--ydb/public/sdk/cpp/client/ydb_driver/ya.make32
-rw-r--r--ydb/public/sdk/cpp/client/ydb_export/export.cpp16
-rw-r--r--ydb/public/sdk/cpp/client/ydb_export/ya.make34
-rw-r--r--ydb/public/sdk/cpp/client/ydb_extension/extension.cpp2
-rw-r--r--ydb/public/sdk/cpp/client/ydb_extension/extension.h36
-rw-r--r--ydb/public/sdk/cpp/client/ydb_extension/ya.make32
-rw-r--r--ydb/public/sdk/cpp/client/ydb_import/import.cpp8
-rw-r--r--ydb/public/sdk/cpp/client/ydb_import/ya.make34
-rw-r--r--ydb/public/sdk/cpp/client/ydb_operation/operation.cpp8
-rw-r--r--ydb/public/sdk/cpp/client/ydb_operation/ya.make30
-rw-r--r--ydb/public/sdk/cpp/client/ydb_params/impl.cpp6
-rw-r--r--ydb/public/sdk/cpp/client/ydb_params/impl.h4
-rw-r--r--ydb/public/sdk/cpp/client/ydb_params/params.cpp20
-rw-r--r--ydb/public/sdk/cpp/client/ydb_params/params.h10
-rw-r--r--ydb/public/sdk/cpp/client/ydb_params/params_ut.cpp4
-rw-r--r--ydb/public/sdk/cpp/client/ydb_params/ut/ya.make2
-rw-r--r--ydb/public/sdk/cpp/client/ydb_params/ya.make34
-rw-r--r--ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/common.cpp22
-rw-r--r--ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/common.h6
-rw-r--r--ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue.cpp2
-rw-r--r--ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue_impl.cpp4
-rw-r--r--ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue_impl.h12
-rw-r--r--ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/read_session.cpp210
-rw-r--r--ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/read_session.h58
-rw-r--r--ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/read_session_messages.cpp476
-rw-r--r--ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/write_session.cpp2
-rw-r--r--ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/write_session.h6
-rw-r--r--ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/ya.make42
-rw-r--r--ydb/public/sdk/cpp/client/ydb_persqueue_core/persqueue.h328
-rw-r--r--ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/basic_usage_ut.cpp6
-rw-r--r--ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/compress_executor_ut.cpp2
-rw-r--r--ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/read_session_ut.cpp10
-rw-r--r--ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/retry_policy_ut.cpp4
-rw-r--r--ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/data_plane_helpers.cpp2
-rw-r--r--ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/sdk_test_setup.h6
-rw-r--r--ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/ya.make2
-rw-r--r--ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/with_offset_ranges_mode_ut/ya.make2
-rw-r--r--ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ya.make4
-rw-r--r--ydb/public/sdk/cpp/client/ydb_persqueue_core/ya.make36
-rw-r--r--ydb/public/sdk/cpp/client/ydb_persqueue_public/codecs/codecs.cpp2
-rw-r--r--ydb/public/sdk/cpp/client/ydb_proto/accessor.cpp4
-rw-r--r--ydb/public/sdk/cpp/client/ydb_proto/accessor.h20
-rw-r--r--ydb/public/sdk/cpp/client/ydb_proto/ya.make30
-rw-r--r--ydb/public/sdk/cpp/client/ydb_rate_limiter/rate_limiter.cpp10
-rw-r--r--ydb/public/sdk/cpp/client/ydb_rate_limiter/rate_limiter.h2
-rw-r--r--ydb/public/sdk/cpp/client/ydb_rate_limiter/ya.make30
-rw-r--r--ydb/public/sdk/cpp/client/ydb_result/proto_accessor.cpp20
-rw-r--r--ydb/public/sdk/cpp/client/ydb_result/result.cpp6
-rw-r--r--ydb/public/sdk/cpp/client/ydb_result/result_ut.cpp388
-rw-r--r--ydb/public/sdk/cpp/client/ydb_result/ut/ya.make44
-rw-r--r--ydb/public/sdk/cpp/client/ydb_result/ya.make34
-rw-r--r--ydb/public/sdk/cpp/client/ydb_scheme/scheme.cpp10
-rw-r--r--ydb/public/sdk/cpp/client/ydb_scheme/ya.make34
-rw-r--r--ydb/public/sdk/cpp/client/ydb_table/impl/client_session.cpp372
-rw-r--r--ydb/public/sdk/cpp/client/ydb_table/impl/client_session.h4
-rw-r--r--ydb/public/sdk/cpp/client/ydb_table/impl/data_query.cpp122
-rw-r--r--ydb/public/sdk/cpp/client/ydb_table/impl/data_query.h68
-rw-r--r--ydb/public/sdk/cpp/client/ydb_table/impl/request_migrator.cpp2
-rw-r--r--ydb/public/sdk/cpp/client/ydb_table/impl/request_migrator.h4
-rw-r--r--ydb/public/sdk/cpp/client/ydb_table/impl/ut/ya.make50
-rw-r--r--ydb/public/sdk/cpp/client/ydb_table/impl/ya.make38
-rw-r--r--ydb/public/sdk/cpp/client/ydb_table/proto_accessor.cpp44
-rw-r--r--ydb/public/sdk/cpp/client/ydb_table/query_stats/stats.cpp6
-rw-r--r--ydb/public/sdk/cpp/client/ydb_table/query_stats/stats.h6
-rw-r--r--ydb/public/sdk/cpp/client/ydb_table/query_stats/ya.make32
-rw-r--r--ydb/public/sdk/cpp/client/ydb_table/table.cpp20
-rw-r--r--ydb/public/sdk/cpp/client/ydb_table/table.h14
-rw-r--r--ydb/public/sdk/cpp/client/ydb_table/ya.make36
-rw-r--r--ydb/public/sdk/cpp/client/ydb_types/credentials/credentials.cpp132
-rw-r--r--ydb/public/sdk/cpp/client/ydb_types/credentials/credentials.h38
-rw-r--r--ydb/public/sdk/cpp/client/ydb_types/credentials/ya.make16
-rw-r--r--ydb/public/sdk/cpp/client/ydb_types/exceptions/exceptions.cpp2
-rw-r--r--ydb/public/sdk/cpp/client/ydb_types/exceptions/ya.make16
-rw-r--r--ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers/handlers.cpp14
-rw-r--r--ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers/handlers.h20
-rw-r--r--ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers/ya.make30
-rw-r--r--ydb/public/sdk/cpp/client/ydb_types/fluent_settings_helpers.h88
-rw-r--r--ydb/public/sdk/cpp/client/ydb_types/operation/operation.cpp174
-rw-r--r--ydb/public/sdk/cpp/client/ydb_types/operation/operation.h86
-rw-r--r--ydb/public/sdk/cpp/client/ydb_types/operation/ya.make26
-rw-r--r--ydb/public/sdk/cpp/client/ydb_types/request_settings.h80
-rw-r--r--ydb/public/sdk/cpp/client/ydb_types/status/status.cpp110
-rw-r--r--ydb/public/sdk/cpp/client/ydb_types/status/status.h56
-rw-r--r--ydb/public/sdk/cpp/client/ydb_types/status/ya.make22
-rw-r--r--ydb/public/sdk/cpp/client/ydb_types/ya.make22
-rw-r--r--ydb/public/sdk/cpp/client/ydb_types/ydb.h58
-rw-r--r--ydb/public/sdk/cpp/client/ydb_value/ut/ya.make50
-rw-r--r--ydb/public/sdk/cpp/client/ydb_value/value.cpp10
-rw-r--r--ydb/public/sdk/cpp/client/ydb_value/value.h6
-rw-r--r--ydb/public/sdk/cpp/client/ydb_value/value_ut.cpp6
-rw-r--r--ydb/public/sdk/cpp/client/ydb_value/ya.make38
-rw-r--r--ydb/public/sdk/python/ydb/resolver.py2
-rw-r--r--ydb/public/tools/lib/cmds/ya.make26
-rw-r--r--ydb/services/datastreams/datastreams_proxy.cpp114
-rw-r--r--ydb/services/datastreams/datastreams_ut.cpp10
-rw-r--r--ydb/services/lib/actors/pq_schema_actor.cpp144
-rw-r--r--ydb/services/lib/actors/pq_schema_actor.h104
-rw-r--r--ydb/services/persqueue_cluster_discovery/cluster_ordering/ut/ya.make2
-rw-r--r--ydb/services/persqueue_cluster_discovery/cluster_ordering/ya.make2
-rw-r--r--ydb/services/persqueue_cluster_discovery/ut/ya.make8
-rw-r--r--ydb/services/persqueue_cluster_discovery/ya.make2
-rw-r--r--ydb/services/persqueue_v1/grpc_pq_actor.h12
-rw-r--r--ydb/services/persqueue_v1/grpc_pq_read_actor.cpp32
-rw-r--r--ydb/services/persqueue_v1/grpc_pq_schema.cpp94
-rw-r--r--ydb/services/persqueue_v1/grpc_pq_write_actor.cpp96
-rw-r--r--ydb/services/persqueue_v1/persqueue_common_ut.cpp2
-rw-r--r--ydb/services/persqueue_v1/persqueue_ut.cpp1666
-rw-r--r--ydb/services/persqueue_v1/ut/new_schemecache_ut/ya.make2
-rw-r--r--ydb/services/persqueue_v1/ut/rate_limiter_test_setup.cpp120
-rw-r--r--ydb/services/persqueue_v1/ut/rate_limiter_test_setup.h32
-rw-r--r--ydb/services/persqueue_v1/ut/ya.make10
-rw-r--r--ydb/services/persqueue_v1/ya.make6
-rw-r--r--ydb/services/rate_limiter/rate_limiter_ut.cpp4
-rw-r--r--ydb/services/ydb/ydb_stats_ut.cpp12
-rw-r--r--ydb/services/ydb/ydb_table_ut.cpp6
-rw-r--r--ydb/tests/functional/sqs/sqs_test_base.py4
-rw-r--r--ydb/tests/functional/sqs/test_multiplexing_tables_format.py112
-rw-r--r--ydb/tests/functional/sqs/ya.make4
-rw-r--r--ydb/tests/library/harness/kikimr_client.py8
-rw-r--r--ydb/tests/library/ya.make2
306 files changed, 10399 insertions, 10399 deletions
diff --git a/build/rules/kikimr.policy b/build/rules/kikimr.policy
index 114dcf93c6..5bb6426e30 100644
--- a/build/rules/kikimr.policy
+++ b/build/rules/kikimr.policy
@@ -69,8 +69,8 @@ ALLOW .* -> kikimr/public/lib/json_value
ALLOW .* -> kikimr/public/lib/yson_value
ALLOW .* -> kikimr/public/lib/ydb_cli
-ALLOW .* -> kikimr/yndx/api
-
+ALLOW .* -> kikimr/yndx/api
+
ALLOW cloud/blockstore -> kikimr/blockstore
ALLOW cloud/blockstore -> kikimr/core
ALLOW cloud/blockstore -> kikimr/driver_lib/run
@@ -119,7 +119,7 @@ ALLOW logbroker/config-manager -> kikimr/core/util
ALLOW logbroker/config-manager -> kikimr/library/persqueue/topic_parser
ALLOW logbroker/config-manager -> kikimr/public/lib/deprecated/kicli
ALLOW logbroker/config-manager -> kikimr/library/security
-ALLOW logbroker/config-manager -> kikimr/yndx/security
+ALLOW logbroker/config-manager -> kikimr/yndx/security
ALLOW logbroker/config-manager -> kikimr/core/testlib
ALLOW logbroker/push-agent/ut -> kikimr/core/testlib
diff --git a/ydb/core/base/appdata.h b/ydb/core/base/appdata.h
index dc3e2ccd8e..c666f7468c 100644
--- a/ydb/core/base/appdata.h
+++ b/ydb/core/base/appdata.h
@@ -58,7 +58,7 @@ namespace NSQS {
}
namespace NPQ {
- class IPersQueueMirrorReaderFactory;
+ class IPersQueueMirrorReaderFactory;
}
class TFormatFactory;
@@ -83,7 +83,7 @@ struct TAppData {
IActor*(*FolderServiceFactory)(const NKikimrProto::NFolderService::TFolderServiceConfig&);
- const NPQ::IPersQueueMirrorReaderFactory* PersQueueMirrorReaderFactory = nullptr;
+ const NPQ::IPersQueueMirrorReaderFactory* PersQueueMirrorReaderFactory = nullptr;
NYdb::TDriver* YdbDriver = nullptr;
const NPDisk::IIoContextFactory* IoContextFactory = nullptr;
diff --git a/ydb/core/base/grpc_service_factory.h b/ydb/core/base/grpc_service_factory.h
index bdba3349f9..0ae4fd9dd8 100644
--- a/ydb/core/base/grpc_service_factory.h
+++ b/ydb/core/base/grpc_service_factory.h
@@ -8,66 +8,66 @@ namespace NKikimr {
class TGrpcServiceFactory {
private:
using TServicePtr = TIntrusivePtr<NGrpc::IGRpcService>;
- using TFactoryMethod = std::function<
- TServicePtr(
- NActors::TActorSystem*,
- TIntrusivePtr<NMonitoring::TDynamicCounters>,
- NActors::TActorId
- )
- >;
-
- struct TServiceParams {
- TServiceParams(TFactoryMethod method, bool enableByDefault, std::optional<NActors::TActorId> proxyId)
- : Method(method)
- , EnableByDefault(enableByDefault)
- , GrpcRequestProxyId(proxyId)
- {}
-
- TFactoryMethod Method;
- bool EnableByDefault;
- std::optional<NActors::TActorId> GrpcRequestProxyId;
- };
-
-private:
- std::unordered_map<TString, std::vector<TServiceParams>> Registry;
-
+ using TFactoryMethod = std::function<
+ TServicePtr(
+ NActors::TActorSystem*,
+ TIntrusivePtr<NMonitoring::TDynamicCounters>,
+ NActors::TActorId
+ )
+ >;
+
+ struct TServiceParams {
+ TServiceParams(TFactoryMethod method, bool enableByDefault, std::optional<NActors::TActorId> proxyId)
+ : Method(method)
+ , EnableByDefault(enableByDefault)
+ , GrpcRequestProxyId(proxyId)
+ {}
+
+ TFactoryMethod Method;
+ bool EnableByDefault;
+ std::optional<NActors::TActorId> GrpcRequestProxyId;
+ };
+
+private:
+ std::unordered_map<TString, std::vector<TServiceParams>> Registry;
+
public:
template <class TService>
- void Register(
- const TString& name,
- bool enableByDefault = false,
- std::optional<NActors::TActorId> grpcRequestProxyIdForService = std::nullopt
- ) {
- auto method = [](
- NActors::TActorSystem* actorSystem,
- TIntrusivePtr<NMonitoring::TDynamicCounters> counters,
- NActors::TActorId grpcRequestProxyId
- ) {
+ void Register(
+ const TString& name,
+ bool enableByDefault = false,
+ std::optional<NActors::TActorId> grpcRequestProxyIdForService = std::nullopt
+ ) {
+ auto method = [](
+ NActors::TActorSystem* actorSystem,
+ TIntrusivePtr<NMonitoring::TDynamicCounters> counters,
+ NActors::TActorId grpcRequestProxyId
+ ) {
return TServicePtr(new TService(actorSystem, counters, grpcRequestProxyId));
- };
- Registry[name].emplace_back(
- method,
- enableByDefault,
- grpcRequestProxyIdForService
- );
+ };
+ Registry[name].emplace_back(
+ method,
+ enableByDefault,
+ grpcRequestProxyIdForService
+ );
}
- std::vector<TServicePtr> Create(
- const std::unordered_set<TString>& enabled,
- const std::unordered_set<TString>& disabled,
- NActors::TActorSystem* actorSystem,
- TIntrusivePtr<NMonitoring::TDynamicCounters> counters,
- NActors::TActorId grpcRequestProxyId
- ) {
+ std::vector<TServicePtr> Create(
+ const std::unordered_set<TString>& enabled,
+ const std::unordered_set<TString>& disabled,
+ NActors::TActorSystem* actorSystem,
+ TIntrusivePtr<NMonitoring::TDynamicCounters> counters,
+ NActors::TActorId grpcRequestProxyId
+ ) {
std::vector<TServicePtr> services;
for (const auto& [name, methods] : Registry) {
- for (const auto& [method, enableByDefault, grpcRequestProxyIdForService] : methods) {
+ for (const auto& [method, enableByDefault, grpcRequestProxyIdForService] : methods) {
if (!disabled.count(name) && (enabled.count(name) || enableByDefault)) {
- services.emplace_back(method(
- actorSystem,
- counters,
- grpcRequestProxyIdForService.value_or(grpcRequestProxyId)
- ));
+ services.emplace_back(method(
+ actorSystem,
+ counters,
+ grpcRequestProxyIdForService.value_or(grpcRequestProxyId)
+ ));
}
}
}
diff --git a/ydb/core/client/server/msgbus_server.cpp b/ydb/core/client/server/msgbus_server.cpp
index 58afb3a0e6..8a4d8a3bbd 100644
--- a/ydb/core/client/server/msgbus_server.cpp
+++ b/ydb/core/client/server/msgbus_server.cpp
@@ -435,15 +435,15 @@ protected:
const NBus::TBusServerSessionConfig &SessionConfig;
};
-TMessageBusServer::TMessageBusServer(
- const NBus::TBusServerSessionConfig &sessionConfig,
- NBus::TBusMessageQueue *busQueue,
- ui32 bindPort,
- std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory
-)
+TMessageBusServer::TMessageBusServer(
+ const NBus::TBusServerSessionConfig &sessionConfig,
+ NBus::TBusMessageQueue *busQueue,
+ ui32 bindPort,
+ std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory
+)
: SessionConfig(sessionConfig)
, BusQueue(busQueue)
- , PQReadSessionsInfoWorkerFactory(pqReadSessionsInfoWorkerFactory)
+ , PQReadSessionsInfoWorkerFactory(pqReadSessionsInfoWorkerFactory)
, Protocol(bindPort)
{}
@@ -599,20 +599,20 @@ void TMessageBusServer::UnknownMessage(TBusMessageContext &msg) {
}
IActor* TMessageBusServer::CreateProxy() {
- return CreateMessageBusServerProxy(this, PQReadSessionsInfoWorkerFactory);
+ return CreateMessageBusServerProxy(this, PQReadSessionsInfoWorkerFactory);
}
IActor* TMessageBusServer::CreateMessageBusTraceService() {
return nullptr;
}
-IMessageBusServer* CreateMsgBusServer(
- NBus::TBusMessageQueue *queue,
- const NBus::TBusServerSessionConfig &config,
- std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory,
- ui32 bindPort
-) {
- return new TMessageBusServer(config, queue, bindPort, pqReadSessionsInfoWorkerFactory);
+IMessageBusServer* CreateMsgBusServer(
+ NBus::TBusMessageQueue *queue,
+ const NBus::TBusServerSessionConfig &config,
+ std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory,
+ ui32 bindPort
+) {
+ return new TMessageBusServer(config, queue, bindPort, pqReadSessionsInfoWorkerFactory);
}
}
diff --git a/ydb/core/client/server/msgbus_server.h b/ydb/core/client/server/msgbus_server.h
index 2416df0981..3d40fc9001 100644
--- a/ydb/core/client/server/msgbus_server.h
+++ b/ydb/core/client/server/msgbus_server.h
@@ -136,18 +136,18 @@ class TMessageBusServer : public IMessageBusServer, public NBus::IBusServerHandl
NBus::TBusServerSessionPtr Session;
TActorId Proxy;
TActorId Monitor;
- std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> PQReadSessionsInfoWorkerFactory;
+ std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> PQReadSessionsInfoWorkerFactory;
protected:
TProtocol Protocol;
TActorSystem *ActorSystem = nullptr;
TIntrusivePtr<IMessageBusHttpServer> HttpServer;
public:
- TMessageBusServer(
- const NBus::TBusServerSessionConfig &sessionConfig,
- NBus::TBusMessageQueue *busQueue,
- ui32 bindPort,
- std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory
- );
+ TMessageBusServer(
+ const NBus::TBusServerSessionConfig &sessionConfig,
+ NBus::TBusMessageQueue *busQueue,
+ ui32 bindPort,
+ std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory
+ );
~TMessageBusServer();
void InitSession(TActorSystem *actorSystem, const TActorId &proxy);
@@ -272,13 +272,13 @@ void CopyProtobufsByFieldName(TProtoTo& protoTo, const TProtoFrom& protoFrom) {
}
}
-class IPersQueueGetReadSessionsInfoWorkerFactory;
+class IPersQueueGetReadSessionsInfoWorkerFactory;
+
+IActor* CreateMessageBusServerProxy(
+ TMessageBusServer *server,
+ std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory
+);
-IActor* CreateMessageBusServerProxy(
- TMessageBusServer *server,
- std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory
-);
-
//IActor* CreateMessageBusDatashardSetConfig(TBusMessageContext &msg);
IActor* CreateMessageBusTabletCountersRequest(TBusMessageContext &msg);
IActor* CreateMessageBusLocalMKQL(TBusMessageContext &msg);
diff --git a/ydb/core/client/server/msgbus_server_persqueue.cpp b/ydb/core/client/server/msgbus_server_persqueue.cpp
index d796fa65e6..78b83ef3a3 100644
--- a/ydb/core/client/server/msgbus_server_persqueue.cpp
+++ b/ydb/core/client/server/msgbus_server_persqueue.cpp
@@ -1388,12 +1388,12 @@ public:
TString RequestId;
};
-template <template <class TImpl, class... TArgs> class TSenderImpl, class... T>
-IActor* CreatePersQueueRequestProcessor(
- const NKikimrClient::TPersQueueRequest& request,
- std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory,
- T&&... constructorParams
-) {
+template <template <class TImpl, class... TArgs> class TSenderImpl, class... T>
+IActor* CreatePersQueueRequestProcessor(
+ const NKikimrClient::TPersQueueRequest& request,
+ std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory,
+ T&&... constructorParams
+) {
try {
if (request.HasMetaRequest() + request.HasPartitionRequest() + request.HasFetchRequest() > 1) {
throw std::runtime_error("only one from meta partition or fetch requests must be filled");
@@ -1414,10 +1414,10 @@ IActor* CreatePersQueueRequestProcessor(
} else if (meta.HasCmdGetPartitionStatus()) {
return new TSenderImpl<TPersQueueGetPartitionStatusProcessor>(std::forward<T>(constructorParams)...);
} else if (meta.HasCmdGetReadSessionsInfo()) {
- return new TSenderImpl<TPersQueueGetReadSessionsInfoProcessor>(
- std::forward<T>(constructorParams)...,
- pqReadSessionsInfoWorkerFactory
- );
+ return new TSenderImpl<TPersQueueGetReadSessionsInfoProcessor>(
+ std::forward<T>(constructorParams)...,
+ pqReadSessionsInfoWorkerFactory
+ );
} else {
throw std::runtime_error("Not implemented yet");
}
@@ -1439,9 +1439,9 @@ IActor* CreatePersQueueRequestProcessor(
template <class TImplActor>
class TMessageBusServerPersQueue : public TImplActor, TMessageBusSessionIdentHolder {
public:
- template <class... T>
- TMessageBusServerPersQueue(TBusMessageContext& msg, T&&... constructorParams)
- : TImplActor(static_cast<TBusPersQueue*>(msg.GetMessage())->Record, std::forward<T>(constructorParams)...)
+ template <class... T>
+ TMessageBusServerPersQueue(TBusMessageContext& msg, T&&... constructorParams)
+ : TImplActor(static_cast<TBusPersQueue*>(msg.GetMessage())->Record, std::forward<T>(constructorParams)...)
, TMessageBusSessionIdentHolder(msg)
{}
@@ -1459,33 +1459,33 @@ public:
};
-IActor* CreateMessageBusServerPersQueue(
- TBusMessageContext& msg,
- const TActorId& schemeCache,
- std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory
-) {
+IActor* CreateMessageBusServerPersQueue(
+ TBusMessageContext& msg,
+ const TActorId& schemeCache,
+ std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory
+) {
const NKikimrClient::TPersQueueRequest& request = static_cast<TBusPersQueue*>(msg.GetMessage())->Record;
- return CreatePersQueueRequestProcessor<TMessageBusServerPersQueue>(
- request,
- pqReadSessionsInfoWorkerFactory,
- msg,
- schemeCache
- );
+ return CreatePersQueueRequestProcessor<TMessageBusServerPersQueue>(
+ request,
+ pqReadSessionsInfoWorkerFactory,
+ msg,
+ schemeCache
+ );
}
-IActor* CreateActorServerPersQueue(
- const TActorId& parentId,
- const NKikimrClient::TPersQueueRequest& request,
- const TActorId& schemeCache,
- std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory
-) {
- return CreatePersQueueRequestProcessor<TReplierToParent>(
- request,
- pqReadSessionsInfoWorkerFactory,
- parentId,
- request,
- schemeCache
- );
+IActor* CreateActorServerPersQueue(
+ const TActorId& parentId,
+ const NKikimrClient::TPersQueueRequest& request,
+ const TActorId& schemeCache,
+ std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory
+) {
+ return CreatePersQueueRequestProcessor<TReplierToParent>(
+ request,
+ pqReadSessionsInfoWorkerFactory,
+ parentId,
+ request,
+ schemeCache
+ );
}
}
diff --git a/ydb/core/client/server/msgbus_server_persqueue.h b/ydb/core/client/server/msgbus_server_persqueue.h
index 3cecd75360..52c67832e4 100644
--- a/ydb/core/client/server/msgbus_server_persqueue.h
+++ b/ydb/core/client/server/msgbus_server_persqueue.h
@@ -28,17 +28,17 @@ TProcessingResult ProcessMetaCacheAllTopicsResponse(NPqMetaCacheV2::TEvPqNewMeta
TProcessingResult ProcessMetaCacheSingleTopicsResponse(const NSchemeCache::TSchemeCacheNavigate::TEntry& entry);
// Worker actor creation
-IActor* CreateMessageBusServerPersQueue(
- TBusMessageContext& msg,
- const TActorId& schemeCache,
- std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory = nullptr
-);
-IActor* CreateActorServerPersQueue(
- const TActorId& parentId,
- const NKikimrClient::TPersQueueRequest& request,
- const TActorId& schemeCache,
- std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory = nullptr
-);
+IActor* CreateMessageBusServerPersQueue(
+ TBusMessageContext& msg,
+ const TActorId& schemeCache,
+ std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory = nullptr
+);
+IActor* CreateActorServerPersQueue(
+ const TActorId& parentId,
+ const NKikimrClient::TPersQueueRequest& request,
+ const TActorId& schemeCache,
+ std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory = nullptr
+);
NKikimrClient::TResponse CreateErrorReply(EResponseStatus status, NPersQueue::NErrorCode::EErrorCode code, const TString& errorReason);
diff --git a/ydb/core/client/server/msgbus_server_pq_metacache.cpp b/ydb/core/client/server/msgbus_server_pq_metacache.cpp
index c8f6247718..f44ab6f480 100644
--- a/ydb/core/client/server/msgbus_server_pq_metacache.cpp
+++ b/ydb/core/client/server/msgbus_server_pq_metacache.cpp
@@ -35,7 +35,7 @@ public:
: Counters(counters)
, ClientWrapper(std::move(std::make_unique<TClientWrapper>(grpcPort)))
, VersionCheckInterval(versionCheckInterval)
- , Generation(std::make_shared<TAtomicCounter>())
+ , Generation(std::make_shared<TAtomicCounter>())
{
}
@@ -43,7 +43,7 @@ public:
const TDuration& versionCheckInterval)
: Counters(counters)
, VersionCheckInterval(versionCheckInterval)
- , Generation(std::make_shared<TAtomicCounter>())
+ , Generation(std::make_shared<TAtomicCounter>())
{
}
@@ -86,22 +86,22 @@ public:
}
private:
- template<class TEventType, class TFutureType, class... TArgs>
- void SubscribeEvent(const TFutureType& future, TArgs... args) {
- std::weak_ptr<TAtomicCounter> weakGeneration(Generation);
+ template<class TEventType, class TFutureType, class... TArgs>
+ void SubscribeEvent(const TFutureType& future, TArgs... args) {
+ std::weak_ptr<TAtomicCounter> weakGeneration(Generation);
future.Subscribe(
- [
- id = SelfId(),
- originalGen = Generation->Val(),
- weakGen = weakGeneration,
- as = ActorSystem,
- ... args = std::forward<TArgs>(args)
- ](const auto&) mutable {
- auto currentGen = weakGen.lock();
- if (currentGen && originalGen == currentGen->Val()) {
- as->Send(id, new TEventType(args...));
+ [
+ id = SelfId(),
+ originalGen = Generation->Val(),
+ weakGen = weakGeneration,
+ as = ActorSystem,
+ ... args = std::forward<TArgs>(args)
+ ](const auto&) mutable {
+ auto currentGen = weakGen.lock();
+ if (currentGen && originalGen == currentGen->Val()) {
+ as->Send(id, new TEventType(args...));
}
- }
+ }
);
}
@@ -119,7 +119,7 @@ private:
void StartSession(const TActorContext& ctx) {
LOG_DEBUG_S(ctx, NKikimrServices::PQ_METACACHE, "Start new session");
SessionFuture = ClientWrapper->GetClient()->GetSession();
- SubscribeEvent<TEvPqNewMetaCache::TEvSessionStarted>(SessionFuture);
+ SubscribeEvent<TEvPqNewMetaCache::TEvSessionStarted>(SessionFuture);
}
void HandleSessionStarted(const TActorContext& ctx) {
@@ -136,7 +136,7 @@ private:
void PrepareTopicsQuery(const TActorContext& ctx) {
Y_UNUSED(ctx);
TopicsQueryFuture = YdbSession->PrepareDataQuery(TopicsQuery);
- SubscribeEvent<TEvPqNewMetaCache::TEvQueryPrepared>(TopicsQueryFuture);
+ SubscribeEvent<TEvPqNewMetaCache::TEvQueryPrepared>(TopicsQueryFuture);
}
void HandleTopicsQueryPrepared(const TActorContext& ctx) {
@@ -176,7 +176,7 @@ private:
}
AsyncQueryResult = PreparedTopicsQuery->Execute(TTxControl::BeginTx().CommitTx(), builder.Build());
}
- SubscribeEvent<TEvPqNewMetaCache::TEvQueryComplete>(AsyncQueryResult, type);
+ SubscribeEvent<TEvPqNewMetaCache::TEvQueryComplete>(AsyncQueryResult, type);
}
void HandleCheckVersionResult(const TActorContext& ctx) {
@@ -522,7 +522,7 @@ private:
NActors::TActorId SchemeCacheId;
TString PathPrefix;
TVector<TString> PathPrefixParts;
- std::shared_ptr<TAtomicCounter> Generation;
+ std::shared_ptr<TAtomicCounter> Generation;
bool SkipVersionCheck = false;
};
diff --git a/ydb/core/client/server/msgbus_server_pq_metarequest.cpp b/ydb/core/client/server/msgbus_server_pq_metarequest.cpp
index a32f14495b..32f37251cc 100644
--- a/ydb/core/client/server/msgbus_server_pq_metarequest.cpp
+++ b/ydb/core/client/server/msgbus_server_pq_metarequest.cpp
@@ -1,5 +1,5 @@
#include "msgbus_server_pq_metarequest.h"
-#include "msgbus_server_pq_read_session_info.h"
+#include "msgbus_server_pq_read_session_info.h"
namespace NKikimr {
namespace NMsgBusProxy {
@@ -439,12 +439,12 @@ void TPersQueueGetPartitionLocationsTopicWorker::Answer(
//
TPersQueueGetReadSessionsInfoProcessor::TPersQueueGetReadSessionsInfoProcessor(
- const NKikimrClient::TPersQueueRequest& request,
- const TActorId& schemeCache,
- std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory
+ const NKikimrClient::TPersQueueRequest& request,
+ const TActorId& schemeCache,
+ std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory
)
: TPersQueueBaseRequestProcessor(request, schemeCache, true)
- , PQReadSessionsInfoWorkerFactory(pqReadSessionsInfoWorkerFactory)
+ , PQReadSessionsInfoWorkerFactory(pqReadSessionsInfoWorkerFactory)
{
const auto& cmd = RequestProto->GetMetaRequest().GetCmdGetReadSessionsInfo();
const auto& topics = cmd.GetTopic();
@@ -531,13 +531,13 @@ bool TPersQueueGetReadSessionsInfoTopicWorker::WaitAllPipeEvents(const TActorCon
return false;
}
-THolder<IActor> TPersQueueGetReadSessionsInfoProcessor::CreateSessionsSubactor(
- const THashMap<TString, TActorId>&& readSessions
-) {
- if (PQReadSessionsInfoWorkerFactory) {
- return PQReadSessionsInfoWorkerFactory->Create(SelfId(), std::move(readSessions), NodesInfo);
+THolder<IActor> TPersQueueGetReadSessionsInfoProcessor::CreateSessionsSubactor(
+ const THashMap<TString, TActorId>&& readSessions
+) {
+ if (PQReadSessionsInfoWorkerFactory) {
+ return PQReadSessionsInfoWorkerFactory->Create(SelfId(), std::move(readSessions), NodesInfo);
}
- return MakeHolder<TPersQueueGetReadSessionsInfoWorker>(SelfId(), std::move(readSessions), NodesInfo);
+ return MakeHolder<TPersQueueGetReadSessionsInfoWorker>(SelfId(), std::move(readSessions), NodesInfo);
}
STFUNC(TPersQueueGetReadSessionsInfoTopicWorker::WaitAllPipeEventsStateFunc) {
diff --git a/ydb/core/client/server/msgbus_server_pq_metarequest.h b/ydb/core/client/server/msgbus_server_pq_metarequest.h
index 32cf7fddd1..1d56c28dc0 100644
--- a/ydb/core/client/server/msgbus_server_pq_metarequest.h
+++ b/ydb/core/client/server/msgbus_server_pq_metarequest.h
@@ -131,11 +131,11 @@ private:
class TPersQueueGetReadSessionsInfoProcessor : public TPersQueueBaseRequestProcessor {
public:
- TPersQueueGetReadSessionsInfoProcessor(
- const NKikimrClient::TPersQueueRequest& request,
- const TActorId& schemeCache,
- std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory
- );
+ TPersQueueGetReadSessionsInfoProcessor(
+ const NKikimrClient::TPersQueueRequest& request,
+ const TActorId& schemeCache,
+ std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory
+ );
bool ReadyForAnswer(const TActorContext& ctx) override {
if (TPersQueueBaseRequestProcessor::ReadyForAnswer(ctx)) {
@@ -172,7 +172,7 @@ private:
THolder<IActor> CreateTopicSubactor(const TSchemeEntry& topicEntry, const TString& name) override;
THolder<IActor> CreateSessionsSubactor(const THashMap<TString, TActorId>&& readSessions);
- std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> PQReadSessionsInfoWorkerFactory;
+ std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> PQReadSessionsInfoWorkerFactory;
mutable bool HasSessionsRequest = false;
THashMap<TString, TActorId> ReadSessions;
};
diff --git a/ydb/core/client/server/msgbus_server_pq_metarequest_ut.cpp b/ydb/core/client/server/msgbus_server_pq_metarequest_ut.cpp
index 67eb9645c9..237bba147b 100644
--- a/ydb/core/client/server/msgbus_server_pq_metarequest_ut.cpp
+++ b/ydb/core/client/server/msgbus_server_pq_metarequest_ut.cpp
@@ -1,5 +1,5 @@
#include "msgbus_server_persqueue.h"
-#include "msgbus_server_pq_read_session_info.h"
+#include "msgbus_server_pq_read_session_info.h"
#include <ydb/core/base/tabletid.h>
#include <ydb/core/engine/minikql/flat_local_tx_factory.h>
@@ -225,12 +225,12 @@ protected:
// Register tested actor
void RegisterActor(const NKikimrClient::TPersQueueRequest& request) {
- Actor = CreateActorServerPersQueue(
- EdgeActorId,
- request,
- GetMockPQMetaCache().SelfId(),
- nullptr
- );
+ Actor = CreateActorServerPersQueue(
+ EdgeActorId,
+ request,
+ GetMockPQMetaCache().SelfId(),
+ nullptr
+ );
TestMainActorId = Runtime->Register(Actor);
TestActors.insert(TestMainActorId);
}
diff --git a/ydb/core/client/server/msgbus_server_proxy.cpp b/ydb/core/client/server/msgbus_server_proxy.cpp
index 4701cea25f..7f3146d21a 100644
--- a/ydb/core/client/server/msgbus_server_proxy.cpp
+++ b/ydb/core/client/server/msgbus_server_proxy.cpp
@@ -100,11 +100,11 @@ public:
}
};
-IActor* CreateMessageBusServerProxy(
- TMessageBusServer* server,
- std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory
-) {
- return new TMessageBusServerProxy(server, pqReadSessionsInfoWorkerFactory);
+IActor* CreateMessageBusServerProxy(
+ TMessageBusServer* server,
+ std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory
+) {
+ return new TMessageBusServerProxy(server, pqReadSessionsInfoWorkerFactory);
}
TBusResponse* ProposeTransactionStatusToResponse(EResponseStatus status,
diff --git a/ydb/core/client/server/msgbus_server_proxy.h b/ydb/core/client/server/msgbus_server_proxy.h
index 13b62aad64..cb9f9c19d3 100644
--- a/ydb/core/client/server/msgbus_server_proxy.h
+++ b/ydb/core/client/server/msgbus_server_proxy.h
@@ -31,8 +31,8 @@ struct TMessageBusDbOpsCounters : TAtomicRefCount<TMessageBusDbOpsCounters> {
};
class TMessageBusServerProxy : public TActorBootstrapped<TMessageBusServerProxy> {
- TMessageBusServer* const Server;
- std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> PQReadSessionsInfoWorkerFactory;
+ TMessageBusServer* const Server;
+ std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> PQReadSessionsInfoWorkerFactory;
TIntrusivePtr<NMonitoring::TDynamicCounters> SchemeCacheCounters;
@@ -61,12 +61,12 @@ public:
return NKikimrServices::TActivity::MSGBUS_PROXY_ACTOR;
}
- TMessageBusServerProxy(
- TMessageBusServer* server,
- std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory
- )
+ TMessageBusServerProxy(
+ TMessageBusServer* server,
+ std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory
+ )
: Server(server)
- , PQReadSessionsInfoWorkerFactory(pqReadSessionsInfoWorkerFactory)
+ , PQReadSessionsInfoWorkerFactory(pqReadSessionsInfoWorkerFactory)
{
}
diff --git a/ydb/core/client/server/msgbus_server_scheme_request.cpp b/ydb/core/client/server/msgbus_server_scheme_request.cpp
index 4b19b9e349..7df5e262d0 100644
--- a/ydb/core/client/server/msgbus_server_scheme_request.cpp
+++ b/ydb/core/client/server/msgbus_server_scheme_request.cpp
@@ -218,7 +218,7 @@ void TMessageBusServerProxy::Handle(TEvBusProxy::TEvPersQueue::TPtr& ev, const T
ctx.Register(new TMessageBusServerSchemeRequest<TBusPersQueue>(ev->Get()), TMailboxType::HTSwap, AppData()->UserPoolId);
return;
}
- ctx.Register(CreateMessageBusServerPersQueue(msg->MsgContext, PqMetaCache, PQReadSessionsInfoWorkerFactory));
+ ctx.Register(CreateMessageBusServerPersQueue(msg->MsgContext, PqMetaCache, PQReadSessionsInfoWorkerFactory));
}
void TMessageBusServerProxy::Handle(TEvBusProxy::TEvFlatTxRequest::TPtr& ev, const TActorContext& ctx) {
diff --git a/ydb/core/client/server/msgbus_server_tracer.cpp b/ydb/core/client/server/msgbus_server_tracer.cpp
index 68098aa184..ff93f8d1cb 100644
--- a/ydb/core/client/server/msgbus_server_tracer.cpp
+++ b/ydb/core/client/server/msgbus_server_tracer.cpp
@@ -6,14 +6,14 @@ namespace NKikimr {
namespace NMessageBusTracer {
-TMessageBusTracingServer::TMessageBusTracingServer(
- const NBus::TBusServerSessionConfig &sessionConfig,
- NBus::TBusMessageQueue *busQueue,
- const TString& tracePath,
- ui32 bindPort,
- std::shared_ptr<NMsgBusProxy::IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory
-)
- : NKikimr::NMsgBusProxy::TMessageBusServer(sessionConfig, busQueue, bindPort, pqReadSessionsInfoWorkerFactory)
+TMessageBusTracingServer::TMessageBusTracingServer(
+ const NBus::TBusServerSessionConfig &sessionConfig,
+ NBus::TBusMessageQueue *busQueue,
+ const TString& tracePath,
+ ui32 bindPort,
+ std::shared_ptr<NMsgBusProxy::IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory
+)
+ : NKikimr::NMsgBusProxy::TMessageBusServer(sessionConfig, busQueue, bindPort, pqReadSessionsInfoWorkerFactory)
, MessageBusTracerActorID(MakeMessageBusTraceServiceID())
, TraceActive(false)
, TracePath(tracePath)
@@ -239,20 +239,20 @@ IActor* CreateMessageBusTracerStopTrace(NMsgBusProxy::TBusMessageContext &msg) {
namespace NMsgBusProxy {
-IMessageBusServer* CreateMsgBusTracingServer(
- NBus::TBusMessageQueue *queue,
- const NBus::TBusServerSessionConfig &config,
- const TString &tracePath,
- std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory,
- ui32 bindPort
-) {
- return new NMessageBusTracer::TMessageBusTracingServer(
- config,
- queue,
- tracePath,
- bindPort,
- pqReadSessionsInfoWorkerFactory
- );
+IMessageBusServer* CreateMsgBusTracingServer(
+ NBus::TBusMessageQueue *queue,
+ const NBus::TBusServerSessionConfig &config,
+ const TString &tracePath,
+ std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory,
+ ui32 bindPort
+) {
+ return new NMessageBusTracer::TMessageBusTracingServer(
+ config,
+ queue,
+ tracePath,
+ bindPort,
+ pqReadSessionsInfoWorkerFactory
+ );
}
}
diff --git a/ydb/core/client/server/msgbus_server_tracer.h b/ydb/core/client/server/msgbus_server_tracer.h
index 487af92085..57a3812f5d 100644
--- a/ydb/core/client/server/msgbus_server_tracer.h
+++ b/ydb/core/client/server/msgbus_server_tracer.h
@@ -17,13 +17,13 @@ namespace NMessageBusTracer {
class TMessageBusTracingServer : public NKikimr::NMsgBusProxy::TMessageBusServer, public NMsgBusProxy::IMessageWatcher {
public:
using TBusKey = NBus::TBusKey;
- TMessageBusTracingServer(
- const NBus::TBusServerSessionConfig &sessionConfig,
- NBus::TBusMessageQueue *busQueue,
- const TString &tracePath,
- ui32 bindPort,
- std::shared_ptr<NMsgBusProxy::IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory
- );
+ TMessageBusTracingServer(
+ const NBus::TBusServerSessionConfig &sessionConfig,
+ NBus::TBusMessageQueue *busQueue,
+ const TString &tracePath,
+ ui32 bindPort,
+ std::shared_ptr<NMsgBusProxy::IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory
+ );
IActor* CreateMessageBusTraceService() override;
protected:
TActorId MessageBusTracerActorID;
diff --git a/ydb/core/client/server/ya.make b/ydb/core/client/server/ya.make
index 309164a89e..8cbeec21b7 100644
--- a/ydb/core/client/server/ya.make
+++ b/ydb/core/client/server/ya.make
@@ -3,7 +3,7 @@ LIBRARY()
OWNER(
ddoarn
g:kikimr
- g:logbroker
+ g:logbroker
)
SRCS(
@@ -31,7 +31,7 @@ SRCS(
msgbus_server_pq_metacache.cpp
msgbus_server_pq_metarequest.h
msgbus_server_pq_metarequest.cpp
- msgbus_server_pq_read_session_info.cpp
+ msgbus_server_pq_read_session_info.cpp
msgbus_server_resolve_node.cpp
msgbus_server_ic_debug.cpp
msgbus_server_load.cpp
diff --git a/ydb/core/cms/console/net_classifier_updater.cpp b/ydb/core/cms/console/net_classifier_updater.cpp
index 5667e9c376..121a65efc3 100644
--- a/ydb/core/cms/console/net_classifier_updater.cpp
+++ b/ydb/core/cms/console/net_classifier_updater.cpp
@@ -175,7 +175,7 @@ private:
auto FormNetDataFromJson(TStringBuf jsonData) const {
NKikimrNetClassifier::TNetData netData;
- TVector<TString> tagsToFilter(UpdaterConfig().GetNetBoxTags().begin(), UpdaterConfig().GetNetBoxTags().end());
+ TVector<TString> tagsToFilter(UpdaterConfig().GetNetBoxTags().begin(), UpdaterConfig().GetNetBoxTags().end());
NJson::TJsonValue value;
bool res = NJson::ReadJsonTree(jsonData, &value);
if (!res)
@@ -186,25 +186,25 @@ private:
if (!v["prefix"].IsString())
return NKikimrNetClassifier::TNetData{};
TString mask = v["prefix"].GetString();
-
+
if (!v["tags"].IsArray() || v["tags"].GetArray().size() == 0)
return NKikimrNetClassifier::TNetData{};
- const auto& tags = v["tags"].GetArray();
- TString label;
- for (auto& tag : tags) {
- if (!tag.IsString())
- return NKikimrNetClassifier::TNetData{};
- if (std::count(tagsToFilter.begin(), tagsToFilter.end(), tag.GetString())) {
- label = tag.GetString();
- break;
- }
- }
- if (tagsToFilter.empty()) {
- label = tags.front().GetString();
- }
- if (!label) {
- continue;
- }
+ const auto& tags = v["tags"].GetArray();
+ TString label;
+ for (auto& tag : tags) {
+ if (!tag.IsString())
+ return NKikimrNetClassifier::TNetData{};
+ if (std::count(tagsToFilter.begin(), tagsToFilter.end(), tag.GetString())) {
+ label = tag.GetString();
+ break;
+ }
+ }
+ if (tagsToFilter.empty()) {
+ label = tags.front().GetString();
+ }
+ if (!label) {
+ continue;
+ }
auto& subnet = *netData.AddSubnets();
subnet.SetMask(mask);
subnet.SetLabel(label);
@@ -241,28 +241,28 @@ private:
}
void HandleWhileWorking(NHttp::TEvHttpProxy::TEvHttpIncomingResponse::TPtr& ev) {
- if (ev->Get()->Error.empty()) {
- if (ev->Get()->Response->Status == "200") {
- const auto netData = UpdaterConfig().GetFormat() == NKikimrNetClassifier::TNetClassifierUpdaterConfig::TSV
- ? FormNetData(ev->Get()->Response->Body)
- : FormNetDataFromJson(ev->Get()->Response->Body);
- if (netData.SubnetsSize() != 0) {
- PackedNetData = PackAcquiredSubnets(netData);
-
- LastUpdateTimestamp = TActivationContext::Now();
- LastUpdateDatetimeUTC = LastUpdateTimestamp.ToRfc822String(); // for viewer
-
- // To modify the config it's essential to find the current id and generation
- RequestCurrentConfigViaCookie();
- return;
- } else {
- BLOG_ERROR("NetClassifierUpdater failed to get subnets: got empty subnets list");
- }
- } else {
- BLOG_ERROR("NetClassifierUpdater failed to get subnets: http_status=" <<ev->Get()->Response->Status);
+ if (ev->Get()->Error.empty()) {
+ if (ev->Get()->Response->Status == "200") {
+ const auto netData = UpdaterConfig().GetFormat() == NKikimrNetClassifier::TNetClassifierUpdaterConfig::TSV
+ ? FormNetData(ev->Get()->Response->Body)
+ : FormNetDataFromJson(ev->Get()->Response->Body);
+ if (netData.SubnetsSize() != 0) {
+ PackedNetData = PackAcquiredSubnets(netData);
+
+ LastUpdateTimestamp = TActivationContext::Now();
+ LastUpdateDatetimeUTC = LastUpdateTimestamp.ToRfc822String(); // for viewer
+
+ // To modify the config it's essential to find the current id and generation
+ RequestCurrentConfigViaCookie();
+ return;
+ } else {
+ BLOG_ERROR("NetClassifierUpdater failed to get subnets: got empty subnets list");
+ }
+ } else {
+ BLOG_ERROR("NetClassifierUpdater failed to get subnets: http_status=" <<ev->Get()->Response->Status);
}
- } else {
- BLOG_ERROR("NetClassifierUpdater failed to get subnets: " << ev->Get()->Error);
+ } else {
+ BLOG_ERROR("NetClassifierUpdater failed to get subnets: " << ev->Get()->Error);
}
InitializeAgain();
}
diff --git a/ydb/core/cms/console/net_classifier_updater_ut.cpp b/ydb/core/cms/console/net_classifier_updater_ut.cpp
index 4b50f496b4..da41ee51f2 100644
--- a/ydb/core/cms/console/net_classifier_updater_ut.cpp
+++ b/ydb/core/cms/console/net_classifier_updater_ut.cpp
@@ -19,9 +19,9 @@ namespace NKikimr::NNetClassifierUpdaterTests {
using namespace NConsole;
using namespace Tests;
-using TNetClassifierUpdaterConfig = NKikimrNetClassifier::TNetClassifierUpdaterConfig;
-const TString NETWORKS_URI = "/fancy_path/networks.tsv";
-
+using TNetClassifierUpdaterConfig = NKikimrNetClassifier::TNetClassifierUpdaterConfig;
+const TString NETWORKS_URI = "/fancy_path/networks.tsv";
+
static NHttp::TEvHttpProxy::TEvHttpOutgoingResponse* MakeHttpResponse(NHttp::TEvHttpProxy::TEvHttpIncomingRequest* request, const TString& netData) {
const TString content = TStringBuilder() << "HTTP/1.1 200 OK\r\nConnection: Close\r\nContent-Type: application/octet-stream\r\nContent-Length: "
<< netData.size() << "\r\n\r\n" << netData;
@@ -113,26 +113,26 @@ static TString ConvertToJson(const NKikimrNetClassifier::TNetData& netData) {
return res;
}
-NKikimrNetClassifier::TNetClassifierUpdaterConfig CreateUpdaterConfig(
- ui16 netDataSourcePort,
- TNetClassifierUpdaterConfig::EFormat format,
- const TVector<TString>& netBoxTags = {}
-) {
- const TString url = TStringBuilder() << "http://[::1]:" << netDataSourcePort << NETWORKS_URI;
- NKikimrNetClassifier::TNetClassifierUpdaterConfig updaterConfig;
- updaterConfig.SetNetDataSourceUrl(url);
- updaterConfig.SetFormat(format);
- *updaterConfig.MutableNetBoxTags() = {netBoxTags.begin(), netBoxTags.end()};
- return updaterConfig;
-}
+NKikimrNetClassifier::TNetClassifierUpdaterConfig CreateUpdaterConfig(
+ ui16 netDataSourcePort,
+ TNetClassifierUpdaterConfig::EFormat format,
+ const TVector<TString>& netBoxTags = {}
+) {
+ const TString url = TStringBuilder() << "http://[::1]:" << netDataSourcePort << NETWORKS_URI;
+ NKikimrNetClassifier::TNetClassifierUpdaterConfig updaterConfig;
+ updaterConfig.SetNetDataSourceUrl(url);
+ updaterConfig.SetFormat(format);
+ *updaterConfig.MutableNetBoxTags() = {netBoxTags.begin(), netBoxTags.end()};
+ return updaterConfig;
+}
Y_UNIT_TEST_SUITE(TNetClassifierUpdaterTest) {
- void TestGetUpdatesFromHttpServer(
- const TString& sourceResponce,
- const NKikimrNetClassifier::TNetData& expectedNetData,
- TNetClassifierUpdaterConfig::EFormat format = TNetClassifierUpdaterConfig::TSV,
- const TVector<TString>& netBoxTags = {}
- ) {
+ void TestGetUpdatesFromHttpServer(
+ const TString& sourceResponce,
+ const NKikimrNetClassifier::TNetData& expectedNetData,
+ TNetClassifierUpdaterConfig::EFormat format = TNetClassifierUpdaterConfig::TSV,
+ const TVector<TString>& netBoxTags = {}
+ ) {
NMonitoring::TMetricRegistry sensors;
TPortManager pm;
@@ -140,7 +140,7 @@ Y_UNIT_TEST_SUITE(TNetClassifierUpdaterTest) {
const ui64 netDataSourcePort = pm.GetPort(13334);
TServerSettings settings(port);
auto& updaterConfig = *settings.NetClassifierConfig.MutableUpdaterConfig();
- updaterConfig = CreateUpdaterConfig(netDataSourcePort, format, netBoxTags);
+ updaterConfig = CreateUpdaterConfig(netDataSourcePort, format, netBoxTags);
TServer cleverServer = TServer(settings);
auto& actorSystem = *cleverServer.GetRuntime();
@@ -153,13 +153,13 @@ Y_UNIT_TEST_SUITE(TNetClassifierUpdaterTest) {
NActors::TActorId serverId = actorSystem.AllocateEdgeActor();
- actorSystem.Send(new NActors::IEventHandle(proxyId, serverId, new NHttp::TEvHttpProxy::TEvRegisterHandler(NETWORKS_URI, serverId)), 0, true);
-
+ actorSystem.Send(new NActors::IEventHandle(proxyId, serverId, new NHttp::TEvHttpProxy::TEvRegisterHandler(NETWORKS_URI, serverId)), 0, true);
+
TAutoPtr<NActors::IEventHandle> handle;
NHttp::TEvHttpProxy::TEvHttpIncomingRequest* request = actorSystem.GrabEdgeEvent<NHttp::TEvHttpProxy::TEvHttpIncomingRequest>(handle);
- UNIT_ASSERT_EQUAL(request->Request->URL, NETWORKS_URI);
+ UNIT_ASSERT_EQUAL(request->Request->URL, NETWORKS_URI);
- actorSystem.Send(new NActors::IEventHandle(handle->Sender, serverId, MakeHttpResponse(request, sourceResponce)), 0, true);
+ actorSystem.Send(new NActors::IEventHandle(handle->Sender, serverId, MakeHttpResponse(request, sourceResponce)), 0, true);
const TActorId sender = actorSystem.AllocateEdgeActor();
size_t iterations = 0;
@@ -174,7 +174,7 @@ Y_UNIT_TEST_SUITE(TNetClassifierUpdaterTest) {
const auto event = cleverServer.GetRuntime()->GrabEdgeEvent<TEvConfigsDispatcher::TEvGetConfigResponse>(handle);
- if (CheckDistributableConfig(event->Config->GetNetClassifierDistributableConfig(), expectedNetData)) {
+ if (CheckDistributableConfig(event->Config->GetNetClassifierDistributableConfig(), expectedNetData)) {
break;
}
@@ -184,74 +184,74 @@ Y_UNIT_TEST_SUITE(TNetClassifierUpdaterTest) {
}
Y_UNIT_TEST(TestGetUpdatesFromHttpServer) {
- auto netData = FormNetData();
- TestGetUpdatesFromHttpServer(ConvertToTsv(netData), netData);
- TestGetUpdatesFromHttpServer(ConvertToJson(netData), netData, TNetClassifierUpdaterConfig::NETBOX);
+ auto netData = FormNetData();
+ TestGetUpdatesFromHttpServer(ConvertToTsv(netData), netData);
+ TestGetUpdatesFromHttpServer(ConvertToJson(netData), netData, TNetClassifierUpdaterConfig::NETBOX);
}
- Y_UNIT_TEST(TestFiltrationByNetboxTags) {
- const TString netboxResponce = "{ \
- \"count\": 5, \
- \"results\": [ \
- {\"prefix\": \"5.45.192.0/18\", \"tags\": [\"asd\", \"zxcv\"]}, \
- {\"prefix\": \"5.255.192.0/18\", \"tags\": [\"zxcv\", \"asd\"]}, \
- {\"prefix\": \"37.9.64.0/18\", \"tags\": [\"zxcv\"]}, \
- {\"prefix\": \"95.108.128.0/17\", \"tags\": [\"asd\"]}, \
- {\"prefix\": \"172.24.0.0/13\", \"tags\": [\"qwerty\"]} \
- ]}";
-
- auto addMask = [](NKikimrNetClassifier::TNetData& data, const TString& mask, const TString& label) {
- auto& subnet = *data.AddSubnets();
- subnet.SetMask(mask);
- subnet.SetLabel(label);
- };
-
- {
- NKikimrNetClassifier::TNetData data;
- addMask(data, "5.45.192.0/18", "asd");
- addMask(data, "5.255.192.0/18", "zxcv");
- addMask(data, "37.9.64.0/18", "zxcv");
- addMask(data, "95.108.128.0/17", "asd");
- addMask(data, "172.24.0.0/13", "qwerty");
- TestGetUpdatesFromHttpServer(netboxResponce, data, TNetClassifierUpdaterConfig::NETBOX);
- }
- {
- NKikimrNetClassifier::TNetData data;
- addMask(data, "5.45.192.0/18", "asd");
- addMask(data, "5.255.192.0/18", "zxcv");
- addMask(data, "37.9.64.0/18", "zxcv");
- addMask(data, "95.108.128.0/17", "asd");
- addMask(data, "172.24.0.0/13", "qwerty");
- TestGetUpdatesFromHttpServer(netboxResponce, data, TNetClassifierUpdaterConfig::NETBOX, {"asd", "zxcv", "qwerty", "faketag"});
- }
- {
- NKikimrNetClassifier::TNetData data;
- addMask(data, "5.45.192.0/18", "asd");
- addMask(data, "5.255.192.0/18", "zxcv");
- addMask(data, "37.9.64.0/18", "zxcv");
- addMask(data, "95.108.128.0/17", "asd");
- TestGetUpdatesFromHttpServer(netboxResponce, data, TNetClassifierUpdaterConfig::NETBOX, {"zxcv", "asd"});
- }
- {
- NKikimrNetClassifier::TNetData data;
- addMask(data, "5.45.192.0/18", "zxcv");
- addMask(data, "5.255.192.0/18", "zxcv");
- addMask(data, "37.9.64.0/18", "zxcv");
- TestGetUpdatesFromHttpServer(netboxResponce, data, TNetClassifierUpdaterConfig::NETBOX, {"zxcv"});
- }
- {
- NKikimrNetClassifier::TNetData data;
- addMask(data, "5.45.192.0/18", "asd");
- addMask(data, "5.255.192.0/18", "asd");
- addMask(data, "95.108.128.0/17", "asd");
- TestGetUpdatesFromHttpServer(netboxResponce, data, TNetClassifierUpdaterConfig::NETBOX, {"asd"});
- }
- {
- NKikimrNetClassifier::TNetData data;
- addMask(data, "172.24.0.0/13", "qwerty");
- TestGetUpdatesFromHttpServer(netboxResponce, data, TNetClassifierUpdaterConfig::NETBOX, {"qwerty"});
- }
- }
+ Y_UNIT_TEST(TestFiltrationByNetboxTags) {
+ const TString netboxResponce = "{ \
+ \"count\": 5, \
+ \"results\": [ \
+ {\"prefix\": \"5.45.192.0/18\", \"tags\": [\"asd\", \"zxcv\"]}, \
+ {\"prefix\": \"5.255.192.0/18\", \"tags\": [\"zxcv\", \"asd\"]}, \
+ {\"prefix\": \"37.9.64.0/18\", \"tags\": [\"zxcv\"]}, \
+ {\"prefix\": \"95.108.128.0/17\", \"tags\": [\"asd\"]}, \
+ {\"prefix\": \"172.24.0.0/13\", \"tags\": [\"qwerty\"]} \
+ ]}";
+
+ auto addMask = [](NKikimrNetClassifier::TNetData& data, const TString& mask, const TString& label) {
+ auto& subnet = *data.AddSubnets();
+ subnet.SetMask(mask);
+ subnet.SetLabel(label);
+ };
+
+ {
+ NKikimrNetClassifier::TNetData data;
+ addMask(data, "5.45.192.0/18", "asd");
+ addMask(data, "5.255.192.0/18", "zxcv");
+ addMask(data, "37.9.64.0/18", "zxcv");
+ addMask(data, "95.108.128.0/17", "asd");
+ addMask(data, "172.24.0.0/13", "qwerty");
+ TestGetUpdatesFromHttpServer(netboxResponce, data, TNetClassifierUpdaterConfig::NETBOX);
+ }
+ {
+ NKikimrNetClassifier::TNetData data;
+ addMask(data, "5.45.192.0/18", "asd");
+ addMask(data, "5.255.192.0/18", "zxcv");
+ addMask(data, "37.9.64.0/18", "zxcv");
+ addMask(data, "95.108.128.0/17", "asd");
+ addMask(data, "172.24.0.0/13", "qwerty");
+ TestGetUpdatesFromHttpServer(netboxResponce, data, TNetClassifierUpdaterConfig::NETBOX, {"asd", "zxcv", "qwerty", "faketag"});
+ }
+ {
+ NKikimrNetClassifier::TNetData data;
+ addMask(data, "5.45.192.0/18", "asd");
+ addMask(data, "5.255.192.0/18", "zxcv");
+ addMask(data, "37.9.64.0/18", "zxcv");
+ addMask(data, "95.108.128.0/17", "asd");
+ TestGetUpdatesFromHttpServer(netboxResponce, data, TNetClassifierUpdaterConfig::NETBOX, {"zxcv", "asd"});
+ }
+ {
+ NKikimrNetClassifier::TNetData data;
+ addMask(data, "5.45.192.0/18", "zxcv");
+ addMask(data, "5.255.192.0/18", "zxcv");
+ addMask(data, "37.9.64.0/18", "zxcv");
+ TestGetUpdatesFromHttpServer(netboxResponce, data, TNetClassifierUpdaterConfig::NETBOX, {"zxcv"});
+ }
+ {
+ NKikimrNetClassifier::TNetData data;
+ addMask(data, "5.45.192.0/18", "asd");
+ addMask(data, "5.255.192.0/18", "asd");
+ addMask(data, "95.108.128.0/17", "asd");
+ TestGetUpdatesFromHttpServer(netboxResponce, data, TNetClassifierUpdaterConfig::NETBOX, {"asd"});
+ }
+ {
+ NKikimrNetClassifier::TNetData data;
+ addMask(data, "172.24.0.0/13", "qwerty");
+ TestGetUpdatesFromHttpServer(netboxResponce, data, TNetClassifierUpdaterConfig::NETBOX, {"qwerty"});
+ }
+ }
}
} // namespace NKikimr::NNetClassifierUpdaterTests
diff --git a/ydb/core/driver_lib/run/factories.h b/ydb/core/driver_lib/run/factories.h
index eb35a94f2b..41a45b44e0 100644
--- a/ydb/core/driver_lib/run/factories.h
+++ b/ydb/core/driver_lib/run/factories.h
@@ -33,8 +33,8 @@ struct TModuleFactories {
std::shared_ptr<ILogBackendFactory> LogBackendFactory;
// A backend factory for Query Replay
std::shared_ptr<NKqp::IQueryReplayBackendFactory> QueryReplayBackendFactory;
- //
- std::shared_ptr<NMsgBusProxy::IPersQueueGetReadSessionsInfoWorkerFactory> PQReadSessionsInfoWorkerFactory;
+ //
+ std::shared_ptr<NMsgBusProxy::IPersQueueGetReadSessionsInfoWorkerFactory> PQReadSessionsInfoWorkerFactory;
// Can be nullptr. In that case there would be no ability to work with Yandex Logbroker in Yandex Query.
NPq::NConfigurationManager::IConnections::TPtr PqCmConnections;
// Export implementation for Data Shards
@@ -50,7 +50,7 @@ struct TModuleFactories {
// Factory for grpc services
TGrpcServiceFactory GrpcServiceFactory;
- std::shared_ptr<NPQ::IPersQueueMirrorReaderFactory> PersQueueMirrorReaderFactory;
+ std::shared_ptr<NPQ::IPersQueueMirrorReaderFactory> PersQueueMirrorReaderFactory;
/// Factory for pdisk's aio engines
std::shared_ptr<NPDisk::IIoContextFactory> IoContextFactory;
diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp
index d8086f97d1..819c1478d1 100644
--- a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp
+++ b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp
@@ -1471,12 +1471,12 @@ void TSecurityServicesInitializer::InitializeServices(NActors::TActorSystemSetup
// TGRpcServicesInitializer
-TGRpcServicesInitializer::TGRpcServicesInitializer(
- const TKikimrRunConfig& runConfig,
- std::shared_ptr<TModuleFactories> factories
-)
+TGRpcServicesInitializer::TGRpcServicesInitializer(
+ const TKikimrRunConfig& runConfig,
+ std::shared_ptr<TModuleFactories> factories
+)
: IKikimrServicesInitializer(runConfig)
- , Factories(factories)
+ , Factories(factories)
{}
void TGRpcServicesInitializer::InitializeServices(NActors::TActorSystemSetup* setup,
@@ -1484,10 +1484,10 @@ void TGRpcServicesInitializer::InitializeServices(NActors::TActorSystemSetup* se
{
if (!IsServiceInitialized(setup, NMsgBusProxy::CreateMsgBusProxyId())
&& Config.HasGRpcConfig() && Config.GetGRpcConfig().GetStartGRpcProxy()) {
- IActor * proxy = NMsgBusProxy::CreateMessageBusServerProxy(
- nullptr,
- Factories ? Factories->PQReadSessionsInfoWorkerFactory : nullptr
- );
+ IActor * proxy = NMsgBusProxy::CreateMessageBusServerProxy(
+ nullptr,
+ Factories ? Factories->PQReadSessionsInfoWorkerFactory : nullptr
+ );
Y_VERIFY(proxy);
setup->LocalServices.emplace_back(
NMsgBusProxy::CreateMsgBusProxyId(),
@@ -1799,11 +1799,11 @@ TPersQueueL2CacheInitializer::TPersQueueL2CacheInitializer(const TKikimrRunConfi
void TPersQueueL2CacheInitializer::InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) {
static const ui64 DEFAULT_PQ_L2_MAX_SIZE_MB = 8 * 1024;
- static const TDuration DEFAULT_PQ_L2_KEEP_TIMEOUT = TDuration::Seconds(10);
+ static const TDuration DEFAULT_PQ_L2_KEEP_TIMEOUT = TDuration::Seconds(10);
NPQ::TCacheL2Parameters params;
params.MaxSizeMB = DEFAULT_PQ_L2_MAX_SIZE_MB;
- params.KeepTime = DEFAULT_PQ_L2_KEEP_TIMEOUT;
+ params.KeepTime = DEFAULT_PQ_L2_KEEP_TIMEOUT;
if (Config.HasBootstrapConfig() && Config.GetBootstrapConfig().HasNodeLimits()) {
auto nodeLimits = Config.GetBootstrapConfig().GetNodeLimits();
@@ -1812,7 +1812,7 @@ void TPersQueueL2CacheInitializer::InitializeServices(NActors::TActorSystemSetup
if (cfg.HasSharedCacheSizeMb())
params.MaxSizeMB = cfg.GetSharedCacheSizeMb();
if (cfg.HasCacheKeepTimeSec())
- params.KeepTime = TDuration::Seconds(cfg.GetCacheKeepTimeSec());
+ params.KeepTime = TDuration::Seconds(cfg.GetCacheKeepTimeSec());
}
}
@@ -1860,24 +1860,24 @@ void TPersQueueClusterTrackerInitializer::InitializeServices(NActors::TActorSyst
TActorSetupCmd(actor, TMailboxType::HTSwap, appData->UserPoolId)));
}
-// TPersQueueLibSharedInstanceInitializer
+// TPersQueueLibSharedInstanceInitializer
+
+TPersQueueLibSharedInstanceInitializer::TPersQueueLibSharedInstanceInitializer(const TKikimrRunConfig& runConfig)
+ : IKikimrServicesInitializer(runConfig)
+{}
-TPersQueueLibSharedInstanceInitializer::TPersQueueLibSharedInstanceInitializer(const TKikimrRunConfig& runConfig)
- : IKikimrServicesInitializer(runConfig)
-{}
-
void TPersQueueLibSharedInstanceInitializer::InitializeServices(NActors::TActorSystemSetup*, const NKikimr::TAppData* appData) {
- if (Config.HasPQConfig() && Config.GetPQConfig().GetEnabled()) {
- if (Config.GetPQConfig().GetMirrorConfig().GetEnabled()) {
- if (appData->PersQueueMirrorReaderFactory) {
- appData->PersQueueMirrorReaderFactory->Initialize(Config.GetPQConfig().GetMirrorConfig().GetPQLibSettings());
+ if (Config.HasPQConfig() && Config.GetPQConfig().GetEnabled()) {
+ if (Config.GetPQConfig().GetMirrorConfig().GetEnabled()) {
+ if (appData->PersQueueMirrorReaderFactory) {
+ appData->PersQueueMirrorReaderFactory->Initialize(Config.GetPQConfig().GetMirrorConfig().GetPQLibSettings());
}
- }
- }
-}
-
-// TMemProfMonitorInitializer
-
+ }
+ }
+}
+
+// TMemProfMonitorInitializer
+
TMemProfMonitorInitializer::TMemProfMonitorInitializer(const TKikimrRunConfig& runConfig)
: IKikimrServicesInitializer(runConfig)
{}
diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.h b/ydb/core/driver_lib/run/kikimr_services_initializers.h
index 331859b8eb..407ce1bb7b 100644
--- a/ydb/core/driver_lib/run/kikimr_services_initializers.h
+++ b/ydb/core/driver_lib/run/kikimr_services_initializers.h
@@ -254,11 +254,11 @@ public:
// grpc_proxy
class TGRpcServicesInitializer : public IKikimrServicesInitializer {
-private:
- std::shared_ptr<TModuleFactories> Factories;
-
+private:
+ std::shared_ptr<TModuleFactories> Factories;
+
public:
- TGRpcServicesInitializer(const TKikimrRunConfig& runConfig, std::shared_ptr<TModuleFactories> factories);
+ TGRpcServicesInitializer(const TKikimrRunConfig& runConfig, std::shared_ptr<TModuleFactories> factories);
void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override;
};
@@ -350,13 +350,13 @@ public:
void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override;
};
-class TPersQueueLibSharedInstanceInitializer : public IKikimrServicesInitializer {
-public:
- TPersQueueLibSharedInstanceInitializer(const TKikimrRunConfig& runConfig);
+class TPersQueueLibSharedInstanceInitializer : public IKikimrServicesInitializer {
+public:
+ TPersQueueLibSharedInstanceInitializer(const TKikimrRunConfig& runConfig);
void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override;
-};
-
+};
+
class TMemProfMonitorInitializer : public IKikimrServicesInitializer {
public:
TMemProfMonitorInitializer(const TKikimrRunConfig& runConfig);
diff --git a/ydb/core/driver_lib/run/run.cpp b/ydb/core/driver_lib/run/run.cpp
index 86a905d5b3..a4f74aa4e0 100644
--- a/ydb/core/driver_lib/run/run.cpp
+++ b/ydb/core/driver_lib/run/run.cpp
@@ -398,10 +398,10 @@ void TKikimrRunner::InitializeControlBoard(const TKikimrRunConfig& runConfig)
}
}
-void TKikimrRunner::InitializeMessageBus(
- const TKikimrRunConfig& runConfig,
- std::shared_ptr<TModuleFactories> factories
-) {
+void TKikimrRunner::InitializeMessageBus(
+ const TKikimrRunConfig& runConfig,
+ std::shared_ptr<TModuleFactories> factories
+) {
if (runConfig.AppConfig.HasMessageBusConfig() && runConfig.AppConfig.GetMessageBusConfig().GetStartBusProxy()) {
const auto& msgbusConfig = runConfig.AppConfig.GetMessageBusConfig();
@@ -440,21 +440,21 @@ void TKikimrRunner::InitializeMessageBus(
Bus.Reset(NBus::CreateMessageQueue(ProxyBusQueueConfig, "server proxy"));
if (msgbusConfig.GetStartTracingBusProxy()) {
- BusServer.Reset(NMsgBusProxy::CreateMsgBusTracingServer(
- Bus.Get(),
- ProxyBusSessionConfig,
- msgbusConfig.GetTracePath(),
- factories ? factories->PQReadSessionsInfoWorkerFactory : nullptr,
- msgbusConfig.GetBusProxyPort())
- );
+ BusServer.Reset(NMsgBusProxy::CreateMsgBusTracingServer(
+ Bus.Get(),
+ ProxyBusSessionConfig,
+ msgbusConfig.GetTracePath(),
+ factories ? factories->PQReadSessionsInfoWorkerFactory : nullptr,
+ msgbusConfig.GetBusProxyPort())
+ );
}
else {
- BusServer.Reset(NMsgBusProxy::CreateMsgBusServer(
- Bus.Get(),
- ProxyBusSessionConfig,
- factories ? factories->PQReadSessionsInfoWorkerFactory : nullptr,
- msgbusConfig.GetBusProxyPort()
- ));
+ BusServer.Reset(NMsgBusProxy::CreateMsgBusServer(
+ Bus.Get(),
+ ProxyBusSessionConfig,
+ factories ? factories->PQReadSessionsInfoWorkerFactory : nullptr,
+ msgbusConfig.GetBusProxyPort()
+ ));
}
if (Monitoring) {
@@ -852,7 +852,7 @@ void TKikimrRunner::InitializeAppData(const TKikimrRunConfig& runConfig)
&KikimrShouldContinue));
AppData->DataShardExportFactory = ModuleFactories ? ModuleFactories->DataShardExportFactory.get() : nullptr;
AppData->SqsEventsWriterFactory = ModuleFactories ? ModuleFactories->SqsEventsWriterFactory.get() : nullptr;
- AppData->PersQueueMirrorReaderFactory = ModuleFactories ? ModuleFactories->PersQueueMirrorReaderFactory.get() : nullptr;
+ AppData->PersQueueMirrorReaderFactory = ModuleFactories ? ModuleFactories->PersQueueMirrorReaderFactory.get() : nullptr;
AppData->IoContextFactory = ModuleFactories ? ModuleFactories->IoContextFactory.get() : nullptr;
AppData->SqsAuthFactory = ModuleFactories
@@ -1223,7 +1223,7 @@ TIntrusivePtr<TServiceInitializersList> TKikimrRunner::CreateServiceInitializers
}
if (serviceMask.EnableGRpcService) {
- sil->AddServiceInitializer(new TGRpcServicesInitializer(runConfig, ModuleFactories));
+ sil->AddServiceInitializer(new TGRpcServicesInitializer(runConfig, ModuleFactories));
}
#ifdef ACTORSLIB_COLLECT_EXEC_STATS
@@ -1260,8 +1260,8 @@ TIntrusivePtr<TServiceInitializersList> TKikimrRunner::CreateServiceInitializers
sil->AddServiceInitializer(new TPersQueueClusterTrackerInitializer(runConfig));
}
- sil->AddServiceInitializer(new TPersQueueLibSharedInstanceInitializer(runConfig));
-
+ sil->AddServiceInitializer(new TPersQueueLibSharedInstanceInitializer(runConfig));
+
sil->AddServiceInitializer(new TMemProfMonitorInitializer(runConfig));
#if defined(ENABLE_MEMORY_TRACKING)
@@ -1559,12 +1559,12 @@ TIntrusivePtr<TKikimrRunner> TKikimrRunner::CreateKikimrRunner(
std::shared_ptr<TModuleFactories> factories) {
TBasicKikimrServicesMask servicesMask; // all services enabled by default
- TIntrusivePtr<TKikimrRunner> runner(new TKikimrRunner(factories));
+ TIntrusivePtr<TKikimrRunner> runner(new TKikimrRunner(factories));
runner->InitializeAllocator(runConfig);
runner->InitializeRegistries(runConfig);
runner->InitializeMonitoring(runConfig);
runner->InitializeControlBoard(runConfig);
- runner->InitializeMessageBus(runConfig, factories);
+ runner->InitializeMessageBus(runConfig, factories);
runner->InitializeAppData(runConfig);
runner->InitializeLogSettings(runConfig);
TIntrusivePtr<TServiceInitializersList> sil(runner->CreateServiceInitializersList(runConfig, servicesMask));
diff --git a/ydb/core/driver_lib/run/run.h b/ydb/core/driver_lib/run/run.h
index 1e1c9c2c29..dfd93a24e7 100644
--- a/ydb/core/driver_lib/run/run.h
+++ b/ydb/core/driver_lib/run/run.h
@@ -81,10 +81,10 @@ protected:
void InitializeMonitoringLogin(const TKikimrRunConfig& runConfig);
- void InitializeMessageBus(
- const TKikimrRunConfig& runConfig,
- std::shared_ptr<TModuleFactories> factories = nullptr
- );
+ void InitializeMessageBus(
+ const TKikimrRunConfig& runConfig,
+ std::shared_ptr<TModuleFactories> factories = nullptr
+ );
void InitializeGRpc(const TKikimrRunConfig& runConfig);
diff --git a/ydb/core/persqueue/actor_persqueue_client_iface.h b/ydb/core/persqueue/actor_persqueue_client_iface.h
index 4312d21e52..2f8a8d4ab8 100644
--- a/ydb/core/persqueue/actor_persqueue_client_iface.h
+++ b/ydb/core/persqueue/actor_persqueue_client_iface.h
@@ -3,88 +3,88 @@
#include <ydb/core/protos/pqconfig.pb.h>
#include <ydb/public/sdk/cpp/client/ydb_driver/driver.h>
#include <ydb/public/sdk/cpp/client/ydb_persqueue_core/persqueue.h>
-
+
#include <library/cpp/actors/core/actor.h>
-#include <library/cpp/logger/log.h>
+#include <library/cpp/logger/log.h>
+
+#include <util/datetime/base.h>
-#include <util/datetime/base.h>
-
namespace NKikimr::NPQ {
-class IPersQueueMirrorReaderFactory {
+class IPersQueueMirrorReaderFactory {
public:
- virtual void Initialize(const NKikimrPQ::TPQConfig::TPQLibSettings& settings) const = 0;
+ virtual void Initialize(const NKikimrPQ::TPQConfig::TPQLibSettings& settings) const = 0;
- virtual std::shared_ptr<NYdb::ICredentialsProviderFactory> GetCredentialsProvider(
- const NKikimrPQ::TMirrorPartitionConfig::TCredentials& cred
- ) const = 0;
+ virtual std::shared_ptr<NYdb::ICredentialsProviderFactory> GetCredentialsProvider(
+ const NKikimrPQ::TMirrorPartitionConfig::TCredentials& cred
+ ) const = 0;
- virtual std::shared_ptr<NYdb::NPersQueue::IReadSession> GetReadSession(
- const NKikimrPQ::TMirrorPartitionConfig& config,
- ui32 partition,
- std::shared_ptr<NYdb::ICredentialsProviderFactory> credentialsProviderFactory,
- ui64 maxMemoryUsageBytes
- ) const = 0;
+ virtual std::shared_ptr<NYdb::NPersQueue::IReadSession> GetReadSession(
+ const NKikimrPQ::TMirrorPartitionConfig& config,
+ ui32 partition,
+ std::shared_ptr<NYdb::ICredentialsProviderFactory> credentialsProviderFactory,
+ ui64 maxMemoryUsageBytes
+ ) const = 0;
- virtual ~IPersQueueMirrorReaderFactory() = default;
+ virtual ~IPersQueueMirrorReaderFactory() = default;
};
-class TPersQueueMirrorReaderFactory : public IPersQueueMirrorReaderFactory {
+class TPersQueueMirrorReaderFactory : public IPersQueueMirrorReaderFactory {
public:
- void Initialize(const NKikimrPQ::TPQConfig::TPQLibSettings& settings) const override {
- auto driverConfig = NYdb::TDriverConfig()
- .SetNetworkThreadsNum(settings.GetThreadsCount());
- Driver = std::make_shared<NYdb::TDriver>(driverConfig);
- }
-
- std::shared_ptr<NYdb::ICredentialsProviderFactory> GetCredentialsProvider(
- const NKikimrPQ::TMirrorPartitionConfig::TCredentials& cred
- ) const override {
- switch (cred.GetCredentialsCase()) {
- case NKikimrPQ::TMirrorPartitionConfig::TCredentials::CREDENTIALS_NOT_SET: {
- return NYdb::CreateInsecureCredentialsProviderFactory();
- }
- default: {
- ythrow yexception() << "unsupported credentials type " << ui64(cred.GetCredentialsCase());
- }
- }
- }
-
- std::shared_ptr<NYdb::NPersQueue::IReadSession> GetReadSession(
- const NKikimrPQ::TMirrorPartitionConfig& config,
- ui32 partition,
- std::shared_ptr<NYdb::ICredentialsProviderFactory> credentialsProviderFactory,
- ui64 maxMemoryUsageBytes
- ) const override {
- NYdb::NPersQueue::TPersQueueClientSettings clientSettings = NYdb::NPersQueue::TPersQueueClientSettings()
- .DiscoveryEndpoint(TStringBuilder() << config.GetEndpoint() << ":" << config.GetEndpointPort())
- .CredentialsProviderFactory(credentialsProviderFactory)
- .EnableSsl(config.GetUseSecureConnection());
- if (config.HasDatabase()) {
- clientSettings.Database(config.GetDatabase());
- }
-
- NYdb::NPersQueue::TReadSessionSettings settings = NYdb::NPersQueue::TReadSessionSettings()
- .ConsumerName(config.GetConsumer())
- .MaxMemoryUsageBytes(maxMemoryUsageBytes)
- .Decompress(false)
- .DisableClusterDiscovery(true)
- .ReadOnlyOriginal(true)
- .RetryPolicy(NYdb::NPersQueue::IRetryPolicy::GetNoRetryPolicy());
- if (config.HasReadFromTimestampsMs()) {
- settings.StartingMessageTimestamp(TInstant::MilliSeconds(config.GetReadFromTimestampsMs()));
- }
- NYdb::NPersQueue::TTopicReadSettings topicSettings(config.GetTopic());
- topicSettings.AppendPartitionGroupIds({partition + 1});
- settings.AppendTopics(topicSettings);
-
- NYdb::NPersQueue::TPersQueueClient persQueueClient(*Driver, clientSettings);
- return persQueueClient.CreateReadSession(settings);
- }
-
-private:
- mutable std::shared_ptr<NYdb::TDriver> Driver;
+ void Initialize(const NKikimrPQ::TPQConfig::TPQLibSettings& settings) const override {
+ auto driverConfig = NYdb::TDriverConfig()
+ .SetNetworkThreadsNum(settings.GetThreadsCount());
+ Driver = std::make_shared<NYdb::TDriver>(driverConfig);
+ }
+
+ std::shared_ptr<NYdb::ICredentialsProviderFactory> GetCredentialsProvider(
+ const NKikimrPQ::TMirrorPartitionConfig::TCredentials& cred
+ ) const override {
+ switch (cred.GetCredentialsCase()) {
+ case NKikimrPQ::TMirrorPartitionConfig::TCredentials::CREDENTIALS_NOT_SET: {
+ return NYdb::CreateInsecureCredentialsProviderFactory();
+ }
+ default: {
+ ythrow yexception() << "unsupported credentials type " << ui64(cred.GetCredentialsCase());
+ }
+ }
+ }
+
+ std::shared_ptr<NYdb::NPersQueue::IReadSession> GetReadSession(
+ const NKikimrPQ::TMirrorPartitionConfig& config,
+ ui32 partition,
+ std::shared_ptr<NYdb::ICredentialsProviderFactory> credentialsProviderFactory,
+ ui64 maxMemoryUsageBytes
+ ) const override {
+ NYdb::NPersQueue::TPersQueueClientSettings clientSettings = NYdb::NPersQueue::TPersQueueClientSettings()
+ .DiscoveryEndpoint(TStringBuilder() << config.GetEndpoint() << ":" << config.GetEndpointPort())
+ .CredentialsProviderFactory(credentialsProviderFactory)
+ .EnableSsl(config.GetUseSecureConnection());
+ if (config.HasDatabase()) {
+ clientSettings.Database(config.GetDatabase());
+ }
+
+ NYdb::NPersQueue::TReadSessionSettings settings = NYdb::NPersQueue::TReadSessionSettings()
+ .ConsumerName(config.GetConsumer())
+ .MaxMemoryUsageBytes(maxMemoryUsageBytes)
+ .Decompress(false)
+ .DisableClusterDiscovery(true)
+ .ReadOnlyOriginal(true)
+ .RetryPolicy(NYdb::NPersQueue::IRetryPolicy::GetNoRetryPolicy());
+ if (config.HasReadFromTimestampsMs()) {
+ settings.StartingMessageTimestamp(TInstant::MilliSeconds(config.GetReadFromTimestampsMs()));
+ }
+ NYdb::NPersQueue::TTopicReadSettings topicSettings(config.GetTopic());
+ topicSettings.AppendPartitionGroupIds({partition + 1});
+ settings.AppendTopics(topicSettings);
+
+ NYdb::NPersQueue::TPersQueueClient persQueueClient(*Driver, clientSettings);
+ return persQueueClient.CreateReadSession(settings);
+ }
+
+private:
+ mutable std::shared_ptr<NYdb::TDriver> Driver;
};
} // namespace NKikimr::NSQS
diff --git a/ydb/core/persqueue/blob.cpp b/ydb/core/persqueue/blob.cpp
index 044f02a32a..7727f12a13 100644
--- a/ydb/core/persqueue/blob.cpp
+++ b/ydb/core/persqueue/blob.cpp
@@ -1,7 +1,7 @@
#include "blob.h"
#include "type_codecs.h"
-#include <util/string/builder.h>
+#include <util/string/builder.h>
#include <util/string/escape.h>
#include <util/system/unaligned_mem.h>
@@ -97,10 +97,10 @@ void TClientBlob::Serialize(TBuffer& res) const
res.Append(ExplicitHashKey.data(), ExplicitHashKey.size());
}
- ui64 writeTimestampMs = WriteTimestamp.MilliSeconds();
- ui64 createTimestampMs = CreateTimestamp.MilliSeconds();
- res.Append((const char*)&writeTimestampMs, sizeof(ui64));
- res.Append((const char*)&createTimestampMs, sizeof(ui64));
+ ui64 writeTimestampMs = WriteTimestamp.MilliSeconds();
+ ui64 createTimestampMs = CreateTimestamp.MilliSeconds();
+ res.Append((const char*)&writeTimestampMs, sizeof(ui64));
+ res.Append((const char*)&createTimestampMs, sizeof(ui64));
if (outputUncompressedSize)
res.Append((const char*)&(UncompressedSize), sizeof(ui32));
@@ -153,15 +153,15 @@ TClientBlob TClientBlob::Deserialize(const char* data, ui32 size)
data += explicitHashKey.Size();
}
- TInstant writeTimestamp;
- TInstant createTimestamp;
+ TInstant writeTimestamp;
+ TInstant createTimestamp;
ui32 us = 0;
if (hasTS) {
- writeTimestamp = TInstant::MilliSeconds(ReadUnaligned<ui64>(data));
+ writeTimestamp = TInstant::MilliSeconds(ReadUnaligned<ui64>(data));
data += sizeof(ui64);
}
if (hasTS2) {
- createTimestamp = TInstant::MilliSeconds(ReadUnaligned<ui64>(data));
+ createTimestamp = TInstant::MilliSeconds(ReadUnaligned<ui64>(data));
data += sizeof(ui64);
}
if (hasUS) {
@@ -317,8 +317,8 @@ void TBatch::Pack() {
{
auto chunk = MakeChunk<NScheme::TDeltaVarIntCodec<ui64, false>>(output);
for (ui32 i = 0; i < Blobs.size(); ++i) {
- ui64 writeTimestampMs = Blobs[i].WriteTimestamp.MilliSeconds();
- chunk->AddData((const char*)&writeTimestampMs, sizeof(ui64));
+ ui64 writeTimestampMs = Blobs[i].WriteTimestamp.MilliSeconds();
+ chunk->AddData((const char*)&writeTimestampMs, sizeof(ui64));
}
OutputChunk(chunk, output, res);
}
@@ -345,8 +345,8 @@ void TBatch::Pack() {
{
auto chunk = MakeChunk<NScheme::TDeltaVarIntCodec<ui64, false>>(output);
for (ui32 i = 0; i < Blobs.size(); ++i) {
- ui64 createTimestampMs = Blobs[i].CreateTimestamp.MilliSeconds();
- chunk->AddData((const char*)&createTimestampMs, sizeof(ui64));
+ ui64 createTimestampMs = Blobs[i].CreateTimestamp.MilliSeconds();
+ chunk->AddData((const char*)&createTimestampMs, sizeof(ui64));
}
OutputChunk(chunk, output, res);
}
@@ -495,17 +495,17 @@ void TBatch::UnpackToType1(TVector<TClientBlob> *blobs) {
partData.insert(std::make_pair(ps, TPartData(partNo, totalParts, totalSize)));
}
}
- TVector<TInstant> wtime;
+ TVector<TInstant> wtime;
wtime.reserve(totalBlobs);
{
auto chunk = NScheme::IChunkDecoder::ReadChunk(GetChunk(data, dataEnd), &ui64Codecs);
auto iter = chunk->MakeIterator();
for (ui32 i = 0; i < totalBlobs; ++i) {
- ui64 timestampMs = *(ui64*)iter->Next().Data();
- wtime.push_back(TInstant::MilliSeconds(timestampMs));
+ ui64 timestampMs = *(ui64*)iter->Next().Data();
+ wtime.push_back(TInstant::MilliSeconds(timestampMs));
}
}
- TVector<TInstant> ctime;
+ TVector<TInstant> ctime;
ctime.reserve(totalBlobs);
TVector<TString> partitionKey;
@@ -540,8 +540,8 @@ void TBatch::UnpackToType1(TVector<TClientBlob> *blobs) {
auto chunk = NScheme::IChunkDecoder::ReadChunk(GetChunk(data, dataEnd), &ui64Codecs);
auto iter = chunk->MakeIterator();
for (ui32 i = 0; i < totalBlobs; ++i) {
- ui64 timestampMs = *(ui64*)iter->Next().Data();
- ctime.push_back(TInstant::MilliSeconds(timestampMs));
+ ui64 timestampMs = *(ui64*)iter->Next().Data();
+ ctime.push_back(TInstant::MilliSeconds(timestampMs));
}
}
} else {
diff --git a/ydb/core/persqueue/blob.h b/ydb/core/persqueue/blob.h
index c483b4aebf..633ec2499a 100644
--- a/ydb/core/persqueue/blob.h
+++ b/ydb/core/persqueue/blob.h
@@ -2,11 +2,11 @@
#include "header.h"
#include "key.h"
-#include <util/datetime/base.h>
-#include <util/generic/maybe.h>
-#include <util/generic/vector.h>
-
-#include <deque>
+#include <util/datetime/base.h>
+#include <util/generic/maybe.h>
+#include <util/generic/vector.h>
+
+#include <deque>
namespace NKikimr {
namespace NPQ {
@@ -38,8 +38,8 @@ struct TClientBlob {
ui64 SeqNo;
TString Data;
TMaybe<TPartData> PartData;
- TInstant WriteTimestamp;
- TInstant CreateTimestamp;
+ TInstant WriteTimestamp;
+ TInstant CreateTimestamp;
ui32 UncompressedSize;
TString PartitionKey;
TString ExplicitHashKey;
@@ -49,7 +49,7 @@ struct TClientBlob {
, UncompressedSize(0)
{}
- TClientBlob(const TString& sourceId, const ui64 seqNo, const TString& data, TMaybe<TPartData> &&partData, TInstant writeTimestamp, TInstant createTimestamp,
+ TClientBlob(const TString& sourceId, const ui64 seqNo, const TString& data, TMaybe<TPartData> &&partData, TInstant writeTimestamp, TInstant createTimestamp,
const ui64 uncompressedSize, const TString& partitionKey, const TString& explicitHashKey)
: SourceId(sourceId)
, SeqNo(seqNo)
diff --git a/ydb/core/persqueue/cache_eviction.h b/ydb/core/persqueue/cache_eviction.h
index afe070a2d4..07ae210f85 100644
--- a/ydb/core/persqueue/cache_eviction.h
+++ b/ydb/core/persqueue/cache_eviction.h
@@ -1,5 +1,5 @@
#pragma once
-
+
#include "blob.h"
#include "pq_l2_service.h"
diff --git a/ydb/core/persqueue/cluster_tracker.cpp b/ydb/core/persqueue/cluster_tracker.cpp
index 60f4750c21..59fdea5c2b 100644
--- a/ydb/core/persqueue/cluster_tracker.cpp
+++ b/ydb/core/persqueue/cluster_tracker.cpp
@@ -174,10 +174,10 @@ private:
cluster.IsLocal = t.GetList(i).GetStruct(2).GetOptional().GetBool();
cluster.IsEnabled = t.GetList(i).GetStruct(3).GetOptional().GetBool();
- cluster.Weight = t.GetList(i).GetStruct(4).GetOptional().GetUint64();
+ cluster.Weight = t.GetList(i).GetStruct(4).GetOptional().GetUint64();
}
- clustersList->Version = t.GetList(0).GetStruct(5).GetOptional().GetInt64();
+ clustersList->Version = t.GetList(0).GetStruct(5).GetOptional().GetInt64();
ClustersList = std::move(clustersList);
ClustersListUpdateTimestamp = Ctx().Now();
diff --git a/ydb/core/persqueue/config/ya.make b/ydb/core/persqueue/config/ya.make
index 8ace455da0..08ff7c6de1 100644
--- a/ydb/core/persqueue/config/ya.make
+++ b/ydb/core/persqueue/config/ya.make
@@ -3,7 +3,7 @@ LIBRARY()
OWNER(
alexnick
g:kikimr
- g:logbroker
+ g:logbroker
)
SRCS(
diff --git a/ydb/core/persqueue/event_helpers.cpp b/ydb/core/persqueue/event_helpers.cpp
index 392dc5e601..320002a936 100644
--- a/ydb/core/persqueue/event_helpers.cpp
+++ b/ydb/core/persqueue/event_helpers.cpp
@@ -1,48 +1,48 @@
-#include "event_helpers.h"
-
+#include "event_helpers.h"
+
#include <ydb/core/persqueue/events/internal.h>
#include <ydb/core/protos/counters_pq.pb.h>
-
-#include <library/cpp/actors/core/log.h>
-
-#include <util/string/builder.h>
-
-namespace NKikimr {
-namespace NPQ {
-
-void ReplyPersQueueError(
- TActorId dstActor,
- const TActorContext& ctx,
- ui64 tabletId,
- const TString& topicName,
- TMaybe<ui32> partition,
- NKikimr::TTabletCountersBase& counters,
- NKikimrServices::EServiceKikimr service,
- const ui64 responseCookie,
- NPersQueue::NErrorCode::EErrorCode errorCode,
+
+#include <library/cpp/actors/core/log.h>
+
+#include <util/string/builder.h>
+
+namespace NKikimr {
+namespace NPQ {
+
+void ReplyPersQueueError(
+ TActorId dstActor,
+ const TActorContext& ctx,
+ ui64 tabletId,
+ const TString& topicName,
+ TMaybe<ui32> partition,
+ NKikimr::TTabletCountersBase& counters,
+ NKikimrServices::EServiceKikimr service,
+ const ui64 responseCookie,
+ NPersQueue::NErrorCode::EErrorCode errorCode,
const TString& error,
bool logDebug
-) {
- if (errorCode == NPersQueue::NErrorCode::BAD_REQUEST) {
- counters.Cumulative()[COUNTER_PQ_BAD_REQUEST].Increment(1);
- } else if (errorCode == NPersQueue::NErrorCode::INITIALIZING) {
- counters.Cumulative()[COUNTER_PQ_INITIALIZING].Increment(1);
- }
-
- TStringBuilder logStr;
- logStr << "tablet " << tabletId << " topic '" << topicName;
- if (partition) {
- logStr << "' partition " << partition;
- }
- logStr << " error: " << error;
-
+) {
+ if (errorCode == NPersQueue::NErrorCode::BAD_REQUEST) {
+ counters.Cumulative()[COUNTER_PQ_BAD_REQUEST].Increment(1);
+ } else if (errorCode == NPersQueue::NErrorCode::INITIALIZING) {
+ counters.Cumulative()[COUNTER_PQ_INITIALIZING].Increment(1);
+ }
+
+ TStringBuilder logStr;
+ logStr << "tablet " << tabletId << " topic '" << topicName;
+ if (partition) {
+ logStr << "' partition " << partition;
+ }
+ logStr << " error: " << error;
+
if (logDebug) {
LOG_DEBUG_S(ctx, service, logStr);
} else {
LOG_WARN_S(ctx, service, logStr);
}
- ctx.Send(dstActor, new TEvPQ::TEvError(errorCode, error, responseCookie));
-}
-
-}// NPQ
-}// NKikimr
+ ctx.Send(dstActor, new TEvPQ::TEvError(errorCode, error, responseCookie));
+}
+
+}// NPQ
+}// NKikimr
diff --git a/ydb/core/persqueue/event_helpers.h b/ydb/core/persqueue/event_helpers.h
index 9b855b9637..51bb814c92 100644
--- a/ydb/core/persqueue/event_helpers.h
+++ b/ydb/core/persqueue/event_helpers.h
@@ -1,27 +1,27 @@
-#pragma once
-
+#pragma once
+
#include <ydb/core/tablet/tablet_counters.h>
#include <ydb/core/protos/services.pb.h>
#include <ydb/public/api/protos/draft/persqueue_error_codes.pb.h>
-
-#include <library/cpp/actors/core/actor.h>
-
-namespace NKikimr {
-namespace NPQ {
-
-void ReplyPersQueueError(
- TActorId dstActor,
- const TActorContext& ctx,
- ui64 tabletId,
- const TString& topicName,
- TMaybe<ui32> partition,
- NKikimr::TTabletCountersBase& counters,
- NKikimrServices::EServiceKikimr service,
- const ui64 responseCookie,
- NPersQueue::NErrorCode::EErrorCode errorCode,
+
+#include <library/cpp/actors/core/actor.h>
+
+namespace NKikimr {
+namespace NPQ {
+
+void ReplyPersQueueError(
+ TActorId dstActor,
+ const TActorContext& ctx,
+ ui64 tabletId,
+ const TString& topicName,
+ TMaybe<ui32> partition,
+ NKikimr::TTabletCountersBase& counters,
+ NKikimrServices::EServiceKikimr service,
+ const ui64 responseCookie,
+ NPersQueue::NErrorCode::EErrorCode errorCode,
const TString& error,
bool logDebug = false
-);
-
-}// NPQ
-}// NKikimr
+);
+
+}// NPQ
+}// NKikimr
diff --git a/ydb/core/persqueue/events/events.cpp b/ydb/core/persqueue/events/events.cpp
index 97707bebad..2b31588b56 100644
--- a/ydb/core/persqueue/events/events.cpp
+++ b/ydb/core/persqueue/events/events.cpp
@@ -1,2 +1,2 @@
-#include "global.h"
-#include "internal.h"
+#include "global.h"
+#include "internal.h"
diff --git a/ydb/core/persqueue/events/global.h b/ydb/core/persqueue/events/global.h
index c1f623dce5..e1957e7a16 100644
--- a/ydb/core/persqueue/events/global.h
+++ b/ydb/core/persqueue/events/global.h
@@ -42,7 +42,7 @@ struct TEvPersQueue {
EvError,
EvGetPartitionIdForWrite,
EvGetPartitionIdForWriteResponse,
- EvReportPartitionError,
+ EvReportPartitionError,
EvResponse = EvRequest + 256,
EvInternalEvents = EvResponse + 256,
EvEnd
@@ -199,7 +199,7 @@ struct TEvPersQueue {
};
struct TEvError : public TEventPB<TEvError,
- NPersQueueCommon::TError, EvError> {
+ NPersQueueCommon::TError, EvError> {
TEvError() {}
};
@@ -213,10 +213,10 @@ struct TEvPersQueue {
{};
};
- struct TEvReportPartitionError : public TEventPB<TEvReportPartitionError, NKikimrPQ::TStatusResponse::TErrorMessage, EvReportPartitionError> {
- TEvReportPartitionError()
- {}
- };
-
+ struct TEvReportPartitionError : public TEventPB<TEvReportPartitionError, NKikimrPQ::TStatusResponse::TErrorMessage, EvReportPartitionError> {
+ TEvReportPartitionError()
+ {}
+ };
+
};
} //NKikimr
diff --git a/ydb/core/persqueue/events/internal.h b/ydb/core/persqueue/events/internal.h
index f47b60d47a..bc892988b4 100644
--- a/ydb/core/persqueue/events/internal.h
+++ b/ydb/core/persqueue/events/internal.h
@@ -1,10 +1,10 @@
#pragma once
-#include "global.h"
-
+#include "global.h"
+
#include <ydb/core/protos/pqconfig.pb.h>
#include <ydb/core/tablet/tablet_counters.h>
-
+
#include <library/cpp/actors/core/event_local.h>
#include <library/cpp/actors/core/actorid.h>
@@ -97,17 +97,17 @@ struct TEvPQ {
EvRegisterMessageGroup,
EvDeregisterMessageGroup,
EvSplitMessageGroup,
- EvUpdateCounters,
- EvMirrorerCounters,
- EvReadLimiterRequest,
- EvReadLimiterResponse,
- EvReadLimiterConsumed,
- EvReadLimiterCounters,
- EvRetryWrite,
- EvInitCredentials,
- EvCreateConsumer,
- EvRequestPartitionStatus,
- EvReaderEventArrived,
+ EvUpdateCounters,
+ EvMirrorerCounters,
+ EvReadLimiterRequest,
+ EvReadLimiterResponse,
+ EvReadLimiterConsumed,
+ EvReadLimiterCounters,
+ EvRetryWrite,
+ EvInitCredentials,
+ EvCreateConsumer,
+ EvRequestPartitionStatus,
+ EvReaderEventArrived,
EvEnd
};
@@ -580,44 +580,44 @@ struct TEvPQ {
TVector<TDeregisterBody> Deregistrations;
TVector<TRegisterBody> Registrations;
};
-
- struct TEvUpdateCounters : public TEventLocal<TEvUpdateCounters, EvUpdateCounters> {
- TEvUpdateCounters()
- {}
- };
-
- struct TEvMirrorerCounters : public TEventLocal<TEvMirrorerCounters, EvMirrorerCounters> {
- TEvMirrorerCounters(const NKikimr::TTabletCountersBase& counters) {
- Counters.Populate(counters);
- }
-
- NKikimr::TTabletCountersBase Counters;
- };
-
- struct TEvRetryWrite : public TEventLocal<TEvRetryWrite, EvRetryWrite> {
- TEvRetryWrite()
- {}
- };
-
- struct TEvInitCredentials : public TEventLocal<TEvInitCredentials, EvInitCredentials> {
- TEvInitCredentials()
- {}
- };
-
- struct TEvCreateConsumer : public TEventLocal<TEvCreateConsumer, EvCreateConsumer> {
- TEvCreateConsumer()
- {}
- };
-
- struct TEvRequestPartitionStatus : public TEventLocal<TEvRequestPartitionStatus, EvRequestPartitionStatus> {
- TEvRequestPartitionStatus()
- {}
- };
-
- struct TEvReaderEventArrived : public TEventLocal<TEvReaderEventArrived, EvReaderEventArrived> {
- TEvReaderEventArrived()
- {}
- };
+
+ struct TEvUpdateCounters : public TEventLocal<TEvUpdateCounters, EvUpdateCounters> {
+ TEvUpdateCounters()
+ {}
+ };
+
+ struct TEvMirrorerCounters : public TEventLocal<TEvMirrorerCounters, EvMirrorerCounters> {
+ TEvMirrorerCounters(const NKikimr::TTabletCountersBase& counters) {
+ Counters.Populate(counters);
+ }
+
+ NKikimr::TTabletCountersBase Counters;
+ };
+
+ struct TEvRetryWrite : public TEventLocal<TEvRetryWrite, EvRetryWrite> {
+ TEvRetryWrite()
+ {}
+ };
+
+ struct TEvInitCredentials : public TEventLocal<TEvInitCredentials, EvInitCredentials> {
+ TEvInitCredentials()
+ {}
+ };
+
+ struct TEvCreateConsumer : public TEventLocal<TEvCreateConsumer, EvCreateConsumer> {
+ TEvCreateConsumer()
+ {}
+ };
+
+ struct TEvRequestPartitionStatus : public TEventLocal<TEvRequestPartitionStatus, EvRequestPartitionStatus> {
+ TEvRequestPartitionStatus()
+ {}
+ };
+
+ struct TEvReaderEventArrived : public TEventLocal<TEvReaderEventArrived, EvReaderEventArrived> {
+ TEvReaderEventArrived()
+ {}
+ };
};
} //NKikimr
diff --git a/ydb/core/persqueue/events/ya.make b/ydb/core/persqueue/events/ya.make
index acc4439f81..2fd0e80ca4 100644
--- a/ydb/core/persqueue/events/ya.make
+++ b/ydb/core/persqueue/events/ya.make
@@ -1,21 +1,21 @@
-LIBRARY()
-
-OWNER(
- alexnick
- g:kikimr
-)
-
-SRCS(
- events.cpp
-)
-
-PEERDIR(
- library/cpp/actors/core
+LIBRARY()
+
+OWNER(
+ alexnick
+ g:kikimr
+)
+
+SRCS(
+ events.cpp
+)
+
+PEERDIR(
+ library/cpp/actors/core
ydb/core/base
ydb/core/keyvalue
ydb/core/protos
ydb/core/tablet
ydb/public/api/protos
-)
-
-END()
+)
+
+END()
diff --git a/ydb/core/persqueue/internals_ut.cpp b/ydb/core/persqueue/internals_ut.cpp
index 24f6a1533f..d72ffe5e11 100644
--- a/ydb/core/persqueue/internals_ut.cpp
+++ b/ydb/core/persqueue/internals_ut.cpp
@@ -37,10 +37,10 @@ void Test(bool headCompacted, ui32 parts, ui32 partSize, ui32 leftInHead)
TString value(102400, 'a');
head.Batches.push_back(TBatch(head.Offset, 0, TVector<TClientBlob>()));
for (ui32 i = 0; i < 50; ++i) {
- head.Batches.back().AddBlob(TClientBlob(
- "sourceId" + TString(1,'a' + rand() % 26), i + 1, value, TMaybe<TPartData>(),
+ head.Batches.back().AddBlob(TClientBlob(
+ "sourceId" + TString(1,'a' + rand() % 26), i + 1, value, TMaybe<TPartData>(),
TInstant::MilliSeconds(i + 1), TInstant::MilliSeconds(i + 1), 1, "", ""
- ));
+ ));
if (!headCompacted)
all.push_back(head.Batches.back().Blobs.back());
}
@@ -59,10 +59,10 @@ void Test(bool headCompacted, ui32 parts, ui32 partSize, ui32 leftInHead)
newHead.Offset = head.GetNextOffset();
newHead.Batches.push_back(TBatch(newHead.Offset, 0, TVector<TClientBlob>()));
for (ui32 i = 0; i < 10; ++i) {
- newHead.Batches.back().AddBlob(TClientBlob(
- "sourceId2", i + 1, value, TMaybe<TPartData>(),
+ newHead.Batches.back().AddBlob(TClientBlob(
+ "sourceId2", i + 1, value, TMaybe<TPartData>(),
TInstant::MilliSeconds(i + 1000), TInstant::MilliSeconds(i + 1000), 1, "", ""
- ));
+ ));
all.push_back(newHead.Batches.back().Blobs.back()); //newHead always glued
}
newHead.PackedSize = newHead.Batches.back().GetUnpackedSize();
@@ -77,10 +77,10 @@ void Test(bool headCompacted, ui32 parts, ui32 partSize, ui32 leftInHead)
UNIT_ASSERT(!blob.IsComplete());
UNIT_ASSERT(blob.IsNextPart("sourceId3", 1, i, &error));
TMaybe<TPartData> partData = TPartData(i, parts, value2.size());
- TClientBlob clientBlob(
- "soruceId3", 1, value2, std::move(partData),
+ TClientBlob clientBlob(
+ "soruceId3", 1, value2, std::move(partData),
TInstant::MilliSeconds(1), TInstant::MilliSeconds(1), 1, "", ""
- );
+ );
all.push_back(clientBlob);
auto res = blob.Add(std::move(clientBlob));
if (!res.second.empty())
@@ -171,10 +171,10 @@ Y_UNIT_TEST(TestBatchPacking) {
TString value(10, 'a');
TBatch batch;
for (ui32 i = 0; i < 100; ++i) {
- batch.AddBlob(TClientBlob(
- "sourceId1", i + 1, value, TMaybe<TPartData>(),
+ batch.AddBlob(TClientBlob(
+ "sourceId1", i + 1, value, TMaybe<TPartData>(),
TInstant::MilliSeconds(1), TInstant::MilliSeconds(1), 0, "", ""
- ));
+ ));
}
batch.Pack();
TString s = batch.PackedData;
@@ -189,10 +189,10 @@ Y_UNIT_TEST(TestBatchPacking) {
Y_VERIFY(batch2.Blobs.size() == 100);
TBatch batch3;
- batch3.AddBlob(TClientBlob(
- "sourceId", 999999999999999ll, "abacaba", TPartData{33,66,4000000000u},
+ batch3.AddBlob(TClientBlob(
+ "sourceId", 999999999999999ll, "abacaba", TPartData{33,66,4000000000u},
TInstant::MilliSeconds(999999999999ll), TInstant::MilliSeconds(1000), 0, "", ""
- ));
+ ));
batch3.Pack();
UNIT_ASSERT(batch3.Header.GetFormat() == NKikimrPQ::TBatchHeader::EUncompressed);
batch3.Unpack();
diff --git a/ydb/core/persqueue/key.h b/ydb/core/persqueue/key.h
index ec3fe5ad85..098c67cf7d 100644
--- a/ydb/core/persqueue/key.h
+++ b/ydb/core/persqueue/key.h
@@ -1,8 +1,8 @@
#pragma once
#include <util/generic/buffer.h>
-#include <util/string/cast.h>
-#include <util/string/printf.h>
+#include <util/string/cast.h>
+#include <util/string/printf.h>
namespace NKikimr {
namespace NPQ {
diff --git a/ydb/core/persqueue/mirrorer.cpp b/ydb/core/persqueue/mirrorer.cpp
index c5b720146a..37aea95ad1 100644
--- a/ydb/core/persqueue/mirrorer.cpp
+++ b/ydb/core/persqueue/mirrorer.cpp
@@ -1,557 +1,557 @@
-#include "mirrorer.h"
-#include "write_meta.h"
-
+#include "mirrorer.h"
+#include "write_meta.h"
+
#include <ydb/core/persqueue/writer/source_id_encoding.h>
#include <ydb/core/protos/grpc_pq_old.pb.h>
#include <ydb/library/persqueue/topic_parser/topic_parser.h>
#include <ydb/core/base/counters.h>
#include <ydb/core/util/yverify_stream.h>
-
-#include <library/cpp/time_provider/time_provider.h>
-
+
+#include <library/cpp/time_provider/time_provider.h>
+
#include <google/protobuf/util/message_differencer.h>
-
-#include <util/string/join.h>
-
-
-namespace NKikimr {
-namespace NPQ {
-
-using TPersQueueReadEvent = NYdb::NPersQueue::TReadSessionEvent;
-
-constexpr NKikimrServices::TActivity::EType TMirrorer::ActorActivityType() {
- return NKikimrServices::TActivity::PERSQUEUE_MIRRORER;
-}
-
-TMirrorer::TMirrorer(
- TActorId tabletActor,
+
+#include <util/string/join.h>
+
+
+namespace NKikimr {
+namespace NPQ {
+
+using TPersQueueReadEvent = NYdb::NPersQueue::TReadSessionEvent;
+
+constexpr NKikimrServices::TActivity::EType TMirrorer::ActorActivityType() {
+ return NKikimrServices::TActivity::PERSQUEUE_MIRRORER;
+}
+
+TMirrorer::TMirrorer(
+ TActorId tabletActor,
TActorId partitionActor,
- const TString& topicName,
- ui32 partition,
- bool localDC,
- ui64 endOffset,
- const NKikimrPQ::TMirrorPartitionConfig& config,
- const TTabletCountersBase& counters
-)
- : TabletActor(tabletActor)
- , PartitionActor(partitionActor)
- , TopicName(topicName)
- , Partition(partition)
- , LocalDC(localDC)
- , EndOffset(endOffset)
- , OffsetToRead(endOffset)
- , Config(config)
-{
- Counters.Populate(counters);
-}
-
-void TMirrorer::Bootstrap(const TActorContext& ctx) {
- Become(&TThis::StateInitConsumer);
-
- StartInit(ctx);
- ctx.Schedule(UPDATE_COUNTERS_INTERVAL, new TEvPQ::TEvUpdateCounters);
-
- if (AppData(ctx)->Counters) {
- if (TopicName.find("--") != TString::npos) {
- TVector<std::pair<ui64, TString>> lagsIntervals{{100, "100ms"}, {200, "200ms"}, {500, "500ms"},
- {1000, "1000ms"}, {2000, "2000ms"}, {5000, "5000ms"}, {10000, "10000ms"},
- {30000, "30000ms"}, {60000, "60000ms"}, {180000,"180000ms"}, {9999999, "999999ms"}};
-
- auto counters = AppData(ctx)->Counters;
- TString suffix = LocalDC ? "Remote" : "Internal";
- MirrorerErrors = NKikimr::NPQ::TMultiCounter(
+ const TString& topicName,
+ ui32 partition,
+ bool localDC,
+ ui64 endOffset,
+ const NKikimrPQ::TMirrorPartitionConfig& config,
+ const TTabletCountersBase& counters
+)
+ : TabletActor(tabletActor)
+ , PartitionActor(partitionActor)
+ , TopicName(topicName)
+ , Partition(partition)
+ , LocalDC(localDC)
+ , EndOffset(endOffset)
+ , OffsetToRead(endOffset)
+ , Config(config)
+{
+ Counters.Populate(counters);
+}
+
+void TMirrorer::Bootstrap(const TActorContext& ctx) {
+ Become(&TThis::StateInitConsumer);
+
+ StartInit(ctx);
+ ctx.Schedule(UPDATE_COUNTERS_INTERVAL, new TEvPQ::TEvUpdateCounters);
+
+ if (AppData(ctx)->Counters) {
+ if (TopicName.find("--") != TString::npos) {
+ TVector<std::pair<ui64, TString>> lagsIntervals{{100, "100ms"}, {200, "200ms"}, {500, "500ms"},
+ {1000, "1000ms"}, {2000, "2000ms"}, {5000, "5000ms"}, {10000, "10000ms"},
+ {30000, "30000ms"}, {60000, "60000ms"}, {180000,"180000ms"}, {9999999, "999999ms"}};
+
+ auto counters = AppData(ctx)->Counters;
+ TString suffix = LocalDC ? "Remote" : "Internal";
+ MirrorerErrors = NKikimr::NPQ::TMultiCounter(
GetServiceCounters(counters, "pqproxy|writeSession"),
GetLabels(TopicName), {}, {"MirrorerErrors" + suffix}, true
- );
- MirrorerTimeLags = THolder<TPercentileCounter>(new TPercentileCounter(
+ );
+ MirrorerTimeLags = THolder<TPercentileCounter>(new TPercentileCounter(
GetServiceCounters(counters, "pqproxy|mirrorWriteTimeLag"),
- GetLabels(TopicName),
- {{"sensor", "TimeLags" + suffix}},
- "Interval", lagsIntervals, true
- ));
- InitTimeoutCounter = NKikimr::NPQ::TMultiCounter(
- GetServiceCounters(counters, "pqproxy|writeSession"),
+ GetLabels(TopicName),
+ {{"sensor", "TimeLags" + suffix}},
+ "Interval", lagsIntervals, true
+ ));
+ InitTimeoutCounter = NKikimr::NPQ::TMultiCounter(
+ GetServiceCounters(counters, "pqproxy|writeSession"),
GetLabels(TopicName), {}, {"MirrorerInitTimeout" + suffix}, true
- );
- WriteTimeoutCounter = NKikimr::NPQ::TMultiCounter(
- GetServiceCounters(counters, "pqproxy|writeSession"),
+ );
+ WriteTimeoutCounter = NKikimr::NPQ::TMultiCounter(
+ GetServiceCounters(counters, "pqproxy|writeSession"),
{}, {}, {"MirrorerWriteTimeout"}, true, "sensor", false
- );
- }
- }
-}
-
-void TMirrorer::StartInit(const TActorContext& ctx) {
- Become(&TThis::StateInitConsumer);
- LastInitStageTimestamp = ctx.Now();
- ctx.Send(SelfId(), new TEvPQ::TEvInitCredentials);
-}
-
-void TMirrorer::Handle(TEvents::TEvPoisonPill::TPtr&, const TActorContext& ctx) {
- LOG_NOTICE_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription() << " killed");
- ReadSession = nullptr;
- PartitionStream = nullptr;
+ );
+ }
+ }
+}
+
+void TMirrorer::StartInit(const TActorContext& ctx) {
+ Become(&TThis::StateInitConsumer);
+ LastInitStageTimestamp = ctx.Now();
+ ctx.Send(SelfId(), new TEvPQ::TEvInitCredentials);
+}
+
+void TMirrorer::Handle(TEvents::TEvPoisonPill::TPtr&, const TActorContext& ctx) {
+ LOG_NOTICE_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription() << " killed");
+ ReadSession = nullptr;
+ PartitionStream = nullptr;
CredentialsProvider = nullptr;
- Die(ctx);
-}
-
-bool TMirrorer::AddToWriteRequest(
- NKikimrClient::TPersQueuePartitionRequest& request,
- TPersQueueReadEvent::TDataReceivedEvent::TCompressedMessage& message
-) {
- if (!request.HasCmdWriteOffset()) {
- Y_VERIFY(request.CmdWriteSize() == 0);
- request.SetCmdWriteOffset(message.GetOffset(0));
- }
- if (request.GetCmdWriteOffset() + request.CmdWriteSize() != message.GetOffset(0)) {
- return false;
- }
-
- auto write = request.AddCmdWrite();
- write->SetData(GetSerializedData(message));
- write->SetSourceId(NSourceIdEncoding::EncodeSimple(message.GetMessageGroupId(0)));
- write->SetSeqNo(message.GetSeqNo(0));
- write->SetCreateTimeMS(message.GetCreateTime(0).MilliSeconds());
- if (Config.GetSyncWriteTime()) {
- write->SetWriteTimeMS(message.GetWriteTime(0).MilliSeconds());
- }
- write->SetDisableDeduplication(true);
- write->SetUncompressedSize(message.GetUncompressedSize(0));
- return true;
-}
-
-void TMirrorer::ProcessError(const TActorContext& ctx, const TString& msg) {
- if (MirrorerErrors) {
- MirrorerErrors.Inc(1);
- }
-
+ Die(ctx);
+}
+
+bool TMirrorer::AddToWriteRequest(
+ NKikimrClient::TPersQueuePartitionRequest& request,
+ TPersQueueReadEvent::TDataReceivedEvent::TCompressedMessage& message
+) {
+ if (!request.HasCmdWriteOffset()) {
+ Y_VERIFY(request.CmdWriteSize() == 0);
+ request.SetCmdWriteOffset(message.GetOffset(0));
+ }
+ if (request.GetCmdWriteOffset() + request.CmdWriteSize() != message.GetOffset(0)) {
+ return false;
+ }
+
+ auto write = request.AddCmdWrite();
+ write->SetData(GetSerializedData(message));
+ write->SetSourceId(NSourceIdEncoding::EncodeSimple(message.GetMessageGroupId(0)));
+ write->SetSeqNo(message.GetSeqNo(0));
+ write->SetCreateTimeMS(message.GetCreateTime(0).MilliSeconds());
+ if (Config.GetSyncWriteTime()) {
+ write->SetWriteTimeMS(message.GetWriteTime(0).MilliSeconds());
+ }
+ write->SetDisableDeduplication(true);
+ write->SetUncompressedSize(message.GetUncompressedSize(0));
+ return true;
+}
+
+void TMirrorer::ProcessError(const TActorContext& ctx, const TString& msg) {
+ if (MirrorerErrors) {
+ MirrorerErrors.Inc(1);
+ }
+
THolder<TEvPersQueue::TEvReportPartitionError> request = MakeHolder<TEvPersQueue::TEvReportPartitionError>();
- auto& record = request->Record;
- record.SetTimestamp(ctx.Now().Seconds());
- record.SetService(NKikimrServices::PQ_MIRRORER);
- record.SetMessage(TStringBuilder() << MirrorerDescription() << ": " << msg);
-
- Send(PartitionActor, request.Release());
-}
-
-void TMirrorer::ProcessError(const TActorContext& ctx, const TString& msg, const NKikimrClient::TResponse& response) {
- ProcessError(ctx, msg);
-
- if (response.HasPartitionResponse()) {
- bool isWriteRequest = response.GetPartitionResponse().HasCookie() && response.GetPartitionResponse().GetCookie() == EEventCookie::WRITE_REQUEST_COOKIE;
- if (isWriteRequest) {
- ScheduleWithIncreasingTimeout<TEvPQ::TEvRetryWrite>(SelfId(), WriteRetryTimeout, WRITE_RETRY_TIMEOUT_MAX, ctx);
- }
- }
-}
-
-void TMirrorer::AfterSuccesWrite(const TActorContext& ctx) {
- Y_VERIFY(WriteInFlight.empty());
- Y_VERIFY(WriteRequestInFlight);
- LOG_INFO_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription()
- << " written " << WriteRequestInFlight.value().CmdWriteSize()
- << " messages with first offset=" << WriteRequestInFlight.value().GetCmdWriteOffset()
- << ", current queue size: " << Queue.size() << "(" << BytesInFlight << "bytes)");
-
- WriteRequestInFlight.reset();
- Send(SelfId(), new TEvents::TEvWakeup());
-
- WriteRetryTimeout = WRITE_RETRY_TIMEOUT_START;
-
- TryUpdateWriteTimetsamp(ctx);
-}
-
-void TMirrorer::ProcessWriteResponse(
- const TActorContext& ctx,
- const NKikimrClient::TPersQueuePartitionResponse& response
-) {
- Y_VERIFY_S(response.CmdWriteResultSize() == WriteInFlight.size(), MirrorerDescription()
- << "CmdWriteResultSize=" << response.CmdWriteResultSize() << ", but expected=" << WriteInFlight.size()
- << ". First expected offset= " << (WriteInFlight.empty() ? -1 : WriteInFlight.front().Offset)
- << " response: " << response);
-
-
- NYdb::NPersQueue::TDeferredCommit deferredCommit;
- for (auto& result : response.GetCmdWriteResult()) {
- if (result.GetAlreadyWritten()) {
- Y_VERIFY_S(
- !WasSuccessfulRecording,
- MirrorerDescription() << "got write result 'already written',"
- <<" but we have successful recording before: " << result
- );
- ProcessError(ctx, TStringBuilder() << "got write result 'already written': " << result);
- } else {
- WasSuccessfulRecording = true;
- }
- auto& writtenMessageInfo = WriteInFlight.front();
- if (MirrorerTimeLags) {
- TDuration lag = TInstant::MilliSeconds(result.GetWriteTimestampMS()) - writtenMessageInfo.WriteTime;
- MirrorerTimeLags->IncFor(lag.MilliSeconds(), 1);
- }
- ui64 offset = writtenMessageInfo.Offset;
- Y_VERIFY((ui64)result.GetOffset() == offset);
- Y_VERIFY_S(EndOffset <= offset, MirrorerDescription()
- << "end offset more the written " << EndOffset << ">" << offset);
- EndOffset = offset + 1;
- BytesInFlight -= writtenMessageInfo.Size;
-
- if (PartitionStream) {
- deferredCommit.Add(PartitionStream, offset);
- }
- WriteInFlight.pop_front();
- }
- deferredCommit.Commit();
- AfterSuccesWrite(ctx);
-}
-
-void TMirrorer::Handle(TEvPersQueue::TEvResponse::TPtr& ev, const TActorContext& ctx) {
- auto& response = ev->Get()->Record;
- if (response.GetStatus() != NMsgBusProxy::MSTATUS_OK) {
- ProcessError(ctx, "status is not ok: " + response.GetErrorReason(), response);
- return;
- }
- if (response.GetErrorCode() != NPersQueue::NErrorCode::OK) {
- ProcessError(ctx, response.GetErrorReason(), response);
- return;
- }
- if (response.HasPartitionResponse()) {
- const auto& partitionResponse = response.GetPartitionResponse();
- switch(partitionResponse.GetCookie()) {
- case EEventCookie::WRITE_REQUEST_COOKIE: {
- ProcessWriteResponse(ctx, partitionResponse);
- return;
- }
- case EEventCookie::UPDATE_WRITE_TIMESTAMP: {
- LOG_DEBUG_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription()
- << " got response to update write timestamp request: " << partitionResponse);
- return;
- }
- default: {
- ProcessError(ctx, TStringBuilder() << "unexpected partition response: " << response);
- return;
- }
- }
- }
- ProcessError(ctx, TStringBuilder() << "unexpected response: " << response);
-}
-
-void TMirrorer::Handle(TEvPQ::TEvUpdateCounters::TPtr& /*ev*/, const TActorContext& ctx) {
- ctx.Schedule(UPDATE_COUNTERS_INTERVAL, new TEvPQ::TEvUpdateCounters);
- ctx.Send(PartitionActor, new TEvPQ::TEvMirrorerCounters(Counters));
-
- if (ctx.Now() - LastStateLogTimestamp > LOG_STATE_INTERVAL) {
- LastStateLogTimestamp = ctx.Now();
- LOG_NOTICE_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription()
- << "[STATE] current state=" << GetCurrentState()
- << ", read session=" << bool(ReadSession) << ", credentials provider=" << bool(CredentialsProvider));
- if (ReadSession) {
- LOG_NOTICE_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription()
- << "[STATE] read session id " << ReadSession->GetSessionId());
- }
- if (PartitionStream) {
- LOG_NOTICE_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription()
+ auto& record = request->Record;
+ record.SetTimestamp(ctx.Now().Seconds());
+ record.SetService(NKikimrServices::PQ_MIRRORER);
+ record.SetMessage(TStringBuilder() << MirrorerDescription() << ": " << msg);
+
+ Send(PartitionActor, request.Release());
+}
+
+void TMirrorer::ProcessError(const TActorContext& ctx, const TString& msg, const NKikimrClient::TResponse& response) {
+ ProcessError(ctx, msg);
+
+ if (response.HasPartitionResponse()) {
+ bool isWriteRequest = response.GetPartitionResponse().HasCookie() && response.GetPartitionResponse().GetCookie() == EEventCookie::WRITE_REQUEST_COOKIE;
+ if (isWriteRequest) {
+ ScheduleWithIncreasingTimeout<TEvPQ::TEvRetryWrite>(SelfId(), WriteRetryTimeout, WRITE_RETRY_TIMEOUT_MAX, ctx);
+ }
+ }
+}
+
+void TMirrorer::AfterSuccesWrite(const TActorContext& ctx) {
+ Y_VERIFY(WriteInFlight.empty());
+ Y_VERIFY(WriteRequestInFlight);
+ LOG_INFO_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription()
+ << " written " << WriteRequestInFlight.value().CmdWriteSize()
+ << " messages with first offset=" << WriteRequestInFlight.value().GetCmdWriteOffset()
+ << ", current queue size: " << Queue.size() << "(" << BytesInFlight << "bytes)");
+
+ WriteRequestInFlight.reset();
+ Send(SelfId(), new TEvents::TEvWakeup());
+
+ WriteRetryTimeout = WRITE_RETRY_TIMEOUT_START;
+
+ TryUpdateWriteTimetsamp(ctx);
+}
+
+void TMirrorer::ProcessWriteResponse(
+ const TActorContext& ctx,
+ const NKikimrClient::TPersQueuePartitionResponse& response
+) {
+ Y_VERIFY_S(response.CmdWriteResultSize() == WriteInFlight.size(), MirrorerDescription()
+ << "CmdWriteResultSize=" << response.CmdWriteResultSize() << ", but expected=" << WriteInFlight.size()
+ << ". First expected offset= " << (WriteInFlight.empty() ? -1 : WriteInFlight.front().Offset)
+ << " response: " << response);
+
+
+ NYdb::NPersQueue::TDeferredCommit deferredCommit;
+ for (auto& result : response.GetCmdWriteResult()) {
+ if (result.GetAlreadyWritten()) {
+ Y_VERIFY_S(
+ !WasSuccessfulRecording,
+ MirrorerDescription() << "got write result 'already written',"
+ <<" but we have successful recording before: " << result
+ );
+ ProcessError(ctx, TStringBuilder() << "got write result 'already written': " << result);
+ } else {
+ WasSuccessfulRecording = true;
+ }
+ auto& writtenMessageInfo = WriteInFlight.front();
+ if (MirrorerTimeLags) {
+ TDuration lag = TInstant::MilliSeconds(result.GetWriteTimestampMS()) - writtenMessageInfo.WriteTime;
+ MirrorerTimeLags->IncFor(lag.MilliSeconds(), 1);
+ }
+ ui64 offset = writtenMessageInfo.Offset;
+ Y_VERIFY((ui64)result.GetOffset() == offset);
+ Y_VERIFY_S(EndOffset <= offset, MirrorerDescription()
+ << "end offset more the written " << EndOffset << ">" << offset);
+ EndOffset = offset + 1;
+ BytesInFlight -= writtenMessageInfo.Size;
+
+ if (PartitionStream) {
+ deferredCommit.Add(PartitionStream, offset);
+ }
+ WriteInFlight.pop_front();
+ }
+ deferredCommit.Commit();
+ AfterSuccesWrite(ctx);
+}
+
+void TMirrorer::Handle(TEvPersQueue::TEvResponse::TPtr& ev, const TActorContext& ctx) {
+ auto& response = ev->Get()->Record;
+ if (response.GetStatus() != NMsgBusProxy::MSTATUS_OK) {
+ ProcessError(ctx, "status is not ok: " + response.GetErrorReason(), response);
+ return;
+ }
+ if (response.GetErrorCode() != NPersQueue::NErrorCode::OK) {
+ ProcessError(ctx, response.GetErrorReason(), response);
+ return;
+ }
+ if (response.HasPartitionResponse()) {
+ const auto& partitionResponse = response.GetPartitionResponse();
+ switch(partitionResponse.GetCookie()) {
+ case EEventCookie::WRITE_REQUEST_COOKIE: {
+ ProcessWriteResponse(ctx, partitionResponse);
+ return;
+ }
+ case EEventCookie::UPDATE_WRITE_TIMESTAMP: {
+ LOG_DEBUG_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription()
+ << " got response to update write timestamp request: " << partitionResponse);
+ return;
+ }
+ default: {
+ ProcessError(ctx, TStringBuilder() << "unexpected partition response: " << response);
+ return;
+ }
+ }
+ }
+ ProcessError(ctx, TStringBuilder() << "unexpected response: " << response);
+}
+
+void TMirrorer::Handle(TEvPQ::TEvUpdateCounters::TPtr& /*ev*/, const TActorContext& ctx) {
+ ctx.Schedule(UPDATE_COUNTERS_INTERVAL, new TEvPQ::TEvUpdateCounters);
+ ctx.Send(PartitionActor, new TEvPQ::TEvMirrorerCounters(Counters));
+
+ if (ctx.Now() - LastStateLogTimestamp > LOG_STATE_INTERVAL) {
+ LastStateLogTimestamp = ctx.Now();
+ LOG_NOTICE_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription()
+ << "[STATE] current state=" << GetCurrentState()
+ << ", read session=" << bool(ReadSession) << ", credentials provider=" << bool(CredentialsProvider));
+ if (ReadSession) {
+ LOG_NOTICE_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription()
+ << "[STATE] read session id " << ReadSession->GetSessionId());
+ }
+ if (PartitionStream) {
+ LOG_NOTICE_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription()
<< "[STATE] has partition stream " << PartitionStream->GetTopicPath()
- << ":" << PartitionStream->GetPartitionId()
- << " with id " << PartitionStream->GetPartitionStreamId());
- } else {
- LOG_NOTICE_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription() << "[STATE] hasn't partition stream");
- }
- if (StreamStatus) {
- LOG_NOTICE_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription()
- << "[STATE] last source partition status: " << StreamStatus->DebugString());
- }
- LOG_NOTICE_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription()
- << "[STATE] next offset to read " << OffsetToRead << ", current end offset " << EndOffset);
- LOG_NOTICE_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription()
- << "[STATE] bytes in flight " << BytesInFlight
- << ", messages in write request " << WriteInFlight.size()
- << ", queue to write: " << Queue.size());
- }
- if (!ReadSession && LastInitStageTimestamp + INIT_TIMEOUT < ctx.Now()) {
- ProcessError(ctx, TStringBuilder() << "read session was not created, the last stage of initialization was at "
- << LastInitStageTimestamp.Seconds());
- if (InitTimeoutCounter) {
- InitTimeoutCounter.Inc(1);
- }
- StartInit(ctx);
- }
- if (WriteRequestInFlight && WriteRequestTimestamp + WRITE_TIMEOUT < ctx.Now()) {
- LOG_ERROR_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription() << " write request was sent at "
- << WriteRequestTimestamp.Seconds() << ", but no response has been received yet. Tablet will be killed.");
- if (WriteTimeoutCounter) {
- WriteTimeoutCounter.Inc(1);
- }
- ctx.Send(TabletActor, new TEvents::TEvPoisonPill());
- }
-}
-
-void TMirrorer::HandleChangeConfig(TEvPQ::TEvChangeConfig::TPtr& ev, const TActorContext& ctx) {
- bool equalConfigs = google::protobuf::util::MessageDifferencer::Equals(
- Config,
- ev->Get()->Config.GetPartitionConfig().GetMirrorFrom()
- );
- LOG_NOTICE_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription() << " got new config, equal with previous: " << equalConfigs);
- if (!equalConfigs) {
- Config = ev->Get()->Config.GetPartitionConfig().GetMirrorFrom();
- LOG_NOTICE_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription() << " changing config");
-
- Become(&TThis::StateInitConsumer);
- ctx.Send(SelfId(), new TEvPQ::TEvInitCredentials);
- }
-}
-
-void TMirrorer::TryToRead(const TActorContext& ctx) {
- if (BytesInFlight < MAX_BYTES_IN_FLIGHT && ReadSession) {
- StartWaitNextReaderEvent(ctx);
- }
-}
-
-void TMirrorer::TryToWrite(const TActorContext& ctx) {
- if (WriteRequestInFlight || Queue.empty()) {
- return;
- }
-
+ << ":" << PartitionStream->GetPartitionId()
+ << " with id " << PartitionStream->GetPartitionStreamId());
+ } else {
+ LOG_NOTICE_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription() << "[STATE] hasn't partition stream");
+ }
+ if (StreamStatus) {
+ LOG_NOTICE_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription()
+ << "[STATE] last source partition status: " << StreamStatus->DebugString());
+ }
+ LOG_NOTICE_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription()
+ << "[STATE] next offset to read " << OffsetToRead << ", current end offset " << EndOffset);
+ LOG_NOTICE_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription()
+ << "[STATE] bytes in flight " << BytesInFlight
+ << ", messages in write request " << WriteInFlight.size()
+ << ", queue to write: " << Queue.size());
+ }
+ if (!ReadSession && LastInitStageTimestamp + INIT_TIMEOUT < ctx.Now()) {
+ ProcessError(ctx, TStringBuilder() << "read session was not created, the last stage of initialization was at "
+ << LastInitStageTimestamp.Seconds());
+ if (InitTimeoutCounter) {
+ InitTimeoutCounter.Inc(1);
+ }
+ StartInit(ctx);
+ }
+ if (WriteRequestInFlight && WriteRequestTimestamp + WRITE_TIMEOUT < ctx.Now()) {
+ LOG_ERROR_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription() << " write request was sent at "
+ << WriteRequestTimestamp.Seconds() << ", but no response has been received yet. Tablet will be killed.");
+ if (WriteTimeoutCounter) {
+ WriteTimeoutCounter.Inc(1);
+ }
+ ctx.Send(TabletActor, new TEvents::TEvPoisonPill());
+ }
+}
+
+void TMirrorer::HandleChangeConfig(TEvPQ::TEvChangeConfig::TPtr& ev, const TActorContext& ctx) {
+ bool equalConfigs = google::protobuf::util::MessageDifferencer::Equals(
+ Config,
+ ev->Get()->Config.GetPartitionConfig().GetMirrorFrom()
+ );
+ LOG_NOTICE_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription() << " got new config, equal with previous: " << equalConfigs);
+ if (!equalConfigs) {
+ Config = ev->Get()->Config.GetPartitionConfig().GetMirrorFrom();
+ LOG_NOTICE_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription() << " changing config");
+
+ Become(&TThis::StateInitConsumer);
+ ctx.Send(SelfId(), new TEvPQ::TEvInitCredentials);
+ }
+}
+
+void TMirrorer::TryToRead(const TActorContext& ctx) {
+ if (BytesInFlight < MAX_BYTES_IN_FLIGHT && ReadSession) {
+ StartWaitNextReaderEvent(ctx);
+ }
+}
+
+void TMirrorer::TryToWrite(const TActorContext& ctx) {
+ if (WriteRequestInFlight || Queue.empty()) {
+ return;
+ }
+
THolder<TEvPersQueue::TEvRequest> request = MakeHolder<TEvPersQueue::TEvRequest>();
- auto req = request->Record.MutablePartitionRequest();
- req->SetTopic(TopicName);
- req->SetPartition(Partition);
- req->SetMessageNo(0);
- req->SetCookie(WRITE_REQUEST_COOKIE);
-
- while (!Queue.empty() && AddToWriteRequest(*req, Queue.front())) {
- auto& message = Queue.front();
- ui64 dataSize = message.GetData().size();
- WriteInFlight.emplace_back(message.GetOffset(0), dataSize, message.GetWriteTime(0));
- Queue.pop_front();
- }
-
- WriteRequestInFlight = request->Record.GetPartitionRequest();
-
- Send(TabletActor, request.Release());
- WriteRequestTimestamp = ctx.Now();
-}
-
-
-void TMirrorer::HandleInitCredentials(TEvPQ::TEvInitCredentials::TPtr& /*ev*/, const TActorContext& ctx) {
- LastInitStageTimestamp = ctx.Now();
- try {
+ auto req = request->Record.MutablePartitionRequest();
+ req->SetTopic(TopicName);
+ req->SetPartition(Partition);
+ req->SetMessageNo(0);
+ req->SetCookie(WRITE_REQUEST_COOKIE);
+
+ while (!Queue.empty() && AddToWriteRequest(*req, Queue.front())) {
+ auto& message = Queue.front();
+ ui64 dataSize = message.GetData().size();
+ WriteInFlight.emplace_back(message.GetOffset(0), dataSize, message.GetWriteTime(0));
+ Queue.pop_front();
+ }
+
+ WriteRequestInFlight = request->Record.GetPartitionRequest();
+
+ Send(TabletActor, request.Release());
+ WriteRequestTimestamp = ctx.Now();
+}
+
+
+void TMirrorer::HandleInitCredentials(TEvPQ::TEvInitCredentials::TPtr& /*ev*/, const TActorContext& ctx) {
+ LastInitStageTimestamp = ctx.Now();
+ try {
RecreateCredentialsProvider(ctx);
- } catch(...) {
- ProcessError(ctx, "cannot initialize credentials provider: " + CurrentExceptionMessage());
- ScheduleWithIncreasingTimeout<TEvPQ::TEvInitCredentials>(SelfId(), ConsumerInitInterval, CONSUMER_INIT_INTERVAL_MAX, ctx);
- return;
- }
- LOG_NOTICE_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription() << " credentials provider created");
- ConsumerInitInterval = CONSUMER_INIT_INTERVAL_START;
- ScheduleConsumerCreation(ctx);
-}
-
-void TMirrorer::RetryWrite(const TActorContext& ctx) {
- Y_VERIFY(WriteRequestInFlight);
-
+ } catch(...) {
+ ProcessError(ctx, "cannot initialize credentials provider: " + CurrentExceptionMessage());
+ ScheduleWithIncreasingTimeout<TEvPQ::TEvInitCredentials>(SelfId(), ConsumerInitInterval, CONSUMER_INIT_INTERVAL_MAX, ctx);
+ return;
+ }
+ LOG_NOTICE_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription() << " credentials provider created");
+ ConsumerInitInterval = CONSUMER_INIT_INTERVAL_START;
+ ScheduleConsumerCreation(ctx);
+}
+
+void TMirrorer::RetryWrite(const TActorContext& ctx) {
+ Y_VERIFY(WriteRequestInFlight);
+
THolder<TEvPersQueue::TEvRequest> request = MakeHolder<TEvPersQueue::TEvRequest>();
- auto req = request->Record.MutablePartitionRequest();
- req->CopyFrom(WriteRequestInFlight.value());
-
- Send(TabletActor, request.Release());
- WriteRequestTimestamp = ctx.Now();
-}
-
-void TMirrorer::HandleRetryWrite(TEvPQ::TEvRetryWrite::TPtr& /*ev*/, const TActorContext& ctx) {
- RetryWrite(ctx);
-}
-
-void TMirrorer::HandleWakeup(const TActorContext& ctx) {
- TryToRead(ctx);
- TryToWrite(ctx);
-}
-
-void TMirrorer::CreateConsumer(TEvPQ::TEvCreateConsumer::TPtr&, const TActorContext& ctx) {
- LastInitStageTimestamp = ctx.Now();
- LOG_NOTICE_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription() << " creating new read session");
-
- if (!Queue.empty()) {
- OffsetToRead = Queue.front().GetOffset(0);
- while (!Queue.empty()) {
- ui64 dataSize = Queue.back().GetData().size();
- Y_VERIFY(BytesInFlight >= dataSize);
- BytesInFlight -= dataSize;
- Queue.pop_back();
- }
- }
-
- ReadSession.reset();
- PartitionStream.Reset();
-
- auto factory = AppData(ctx)->PersQueueMirrorReaderFactory;
+ auto req = request->Record.MutablePartitionRequest();
+ req->CopyFrom(WriteRequestInFlight.value());
+
+ Send(TabletActor, request.Release());
+ WriteRequestTimestamp = ctx.Now();
+}
+
+void TMirrorer::HandleRetryWrite(TEvPQ::TEvRetryWrite::TPtr& /*ev*/, const TActorContext& ctx) {
+ RetryWrite(ctx);
+}
+
+void TMirrorer::HandleWakeup(const TActorContext& ctx) {
+ TryToRead(ctx);
+ TryToWrite(ctx);
+}
+
+void TMirrorer::CreateConsumer(TEvPQ::TEvCreateConsumer::TPtr&, const TActorContext& ctx) {
+ LastInitStageTimestamp = ctx.Now();
+ LOG_NOTICE_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription() << " creating new read session");
+
+ if (!Queue.empty()) {
+ OffsetToRead = Queue.front().GetOffset(0);
+ while (!Queue.empty()) {
+ ui64 dataSize = Queue.back().GetData().size();
+ Y_VERIFY(BytesInFlight >= dataSize);
+ BytesInFlight -= dataSize;
+ Queue.pop_back();
+ }
+ }
+
+ ReadSession.reset();
+ PartitionStream.Reset();
+
+ auto factory = AppData(ctx)->PersQueueMirrorReaderFactory;
Y_VERIFY(factory);
-
- ReadSession = factory->GetReadSession(Config, Partition, CredentialsProvider, MAX_BYTES_IN_FLIGHT);
-
- LOG_NOTICE_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription()
- << " read session created: " << ReadSession->GetSessionId());
-
- Send(SelfId(), new TEvents::TEvWakeup());
- Become(&TThis::StateWork);
-}
-
-void TMirrorer::RequestSourcePartitionStatus(TEvPQ::TEvRequestPartitionStatus::TPtr&, const TActorContext&) {
- RequestSourcePartitionStatus();
-}
-
-void TMirrorer::RequestSourcePartitionStatus() {
- if (Config.GetSyncWriteTime() && PartitionStream && ReadSession) {
- PartitionStream->RequestStatus();
- }
-}
-
-void TMirrorer::TryUpdateWriteTimetsamp(const TActorContext &ctx) {
- if (!WriteRequestInFlight && StreamStatus && EndOffset == StreamStatus->GetEndOffset()) {
- LOG_INFO_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription()
- << " update write timestamp from original topic: " << StreamStatus->DebugString());
- THolder<TEvPersQueue::TEvRequest> request = MakeHolder<TEvPersQueue::TEvRequest>();
- auto req = request->Record.MutablePartitionRequest();
- req->SetTopic(TopicName);
- req->SetPartition(Partition);
- req->SetCookie(UPDATE_WRITE_TIMESTAMP);
- req->MutableCmdUpdateWriteTimestamp()->SetWriteTimeMS(StreamStatus->GetWriteWatermark().MilliSeconds());
- ctx.Send(TabletActor, request.Release());
- }
-}
-
-void TMirrorer::AddMessagesToQueue(TVector<TPersQueueReadEvent::TDataReceivedEvent::TCompressedMessage>&& messages) {
- for (auto& msg : messages) {
- Y_VERIFY(msg.GetBlocksCount() == 1); // TODO support several compressed messages
- ui64 offset = msg.GetOffset(0);
- Y_VERIFY(OffsetToRead <= offset);
- ui64 messageSize = msg.GetData().size();
-
- Counters.Cumulative()[COUNTER_PQ_TABLET_NETWORK_BYTES_USAGE].Increment(messageSize);
- BytesInFlight += messageSize;
-
- OffsetToRead = offset + 1;
- Queue.emplace_back(std::move(msg));
- }
-}
-
+
+ ReadSession = factory->GetReadSession(Config, Partition, CredentialsProvider, MAX_BYTES_IN_FLIGHT);
+
+ LOG_NOTICE_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription()
+ << " read session created: " << ReadSession->GetSessionId());
+
+ Send(SelfId(), new TEvents::TEvWakeup());
+ Become(&TThis::StateWork);
+}
+
+void TMirrorer::RequestSourcePartitionStatus(TEvPQ::TEvRequestPartitionStatus::TPtr&, const TActorContext&) {
+ RequestSourcePartitionStatus();
+}
+
+void TMirrorer::RequestSourcePartitionStatus() {
+ if (Config.GetSyncWriteTime() && PartitionStream && ReadSession) {
+ PartitionStream->RequestStatus();
+ }
+}
+
+void TMirrorer::TryUpdateWriteTimetsamp(const TActorContext &ctx) {
+ if (!WriteRequestInFlight && StreamStatus && EndOffset == StreamStatus->GetEndOffset()) {
+ LOG_INFO_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription()
+ << " update write timestamp from original topic: " << StreamStatus->DebugString());
+ THolder<TEvPersQueue::TEvRequest> request = MakeHolder<TEvPersQueue::TEvRequest>();
+ auto req = request->Record.MutablePartitionRequest();
+ req->SetTopic(TopicName);
+ req->SetPartition(Partition);
+ req->SetCookie(UPDATE_WRITE_TIMESTAMP);
+ req->MutableCmdUpdateWriteTimestamp()->SetWriteTimeMS(StreamStatus->GetWriteWatermark().MilliSeconds());
+ ctx.Send(TabletActor, request.Release());
+ }
+}
+
+void TMirrorer::AddMessagesToQueue(TVector<TPersQueueReadEvent::TDataReceivedEvent::TCompressedMessage>&& messages) {
+ for (auto& msg : messages) {
+ Y_VERIFY(msg.GetBlocksCount() == 1); // TODO support several compressed messages
+ ui64 offset = msg.GetOffset(0);
+ Y_VERIFY(OffsetToRead <= offset);
+ ui64 messageSize = msg.GetData().size();
+
+ Counters.Cumulative()[COUNTER_PQ_TABLET_NETWORK_BYTES_USAGE].Increment(messageSize);
+ BytesInFlight += messageSize;
+
+ OffsetToRead = offset + 1;
+ Queue.emplace_back(std::move(msg));
+ }
+}
+
void TMirrorer::ScheduleConsumerCreation(const TActorContext& ctx) {
- LastInitStageTimestamp = ctx.Now();
- ReadSession = nullptr;
- PartitionStream = nullptr;
- Become(&TThis::StateInitConsumer);
-
- LOG_NOTICE_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription() << " schedule consumer creation");
- ScheduleWithIncreasingTimeout<TEvPQ::TEvCreateConsumer>(SelfId(), ConsumerInitInterval, CONSUMER_INIT_INTERVAL_MAX, ctx);
-}
-
+ LastInitStageTimestamp = ctx.Now();
+ ReadSession = nullptr;
+ PartitionStream = nullptr;
+ Become(&TThis::StateInitConsumer);
+
+ LOG_NOTICE_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription() << " schedule consumer creation");
+ ScheduleWithIncreasingTimeout<TEvPQ::TEvCreateConsumer>(SelfId(), ConsumerInitInterval, CONSUMER_INIT_INTERVAL_MAX, ctx);
+}
+
void TMirrorer::RecreateCredentialsProvider(const TActorContext& ctx) {
CredentialsProvider = nullptr;
- auto factory = AppData(ctx)->PersQueueMirrorReaderFactory;
- Y_VERIFY(factory);
- CredentialsProvider = factory->GetCredentialsProvider(Config.GetCredentials());
-}
-
-TString TMirrorer::MirrorerDescription() const {
- return TStringBuilder() << "[mirrorer for " << TopicName << ':' << Partition << ']';
-}
-
-TString TMirrorer::GetCurrentState() const {
- if (CurrentStateFunc() == &TThis::StateInitConsumer) {
- return "StateInitConsumer";
- } else if (CurrentStateFunc() == &TThis::StateWork) {
- return "StateWork";
- }
- return "UNKNOWN";
-}
-
-void TMirrorer::StartWaitNextReaderEvent(const TActorContext& ctx) {
- if (WaitNextReaderEventInFlight) {
- return;
- }
- auto future = ReadSession->WaitEvent();
- future.Subscribe(
- [
- actorSystem = ctx.ExecutorThread.ActorSystem,
- selfId = SelfId()
- ](const NThreading::TFuture<void>&) {
- actorSystem->Send(new NActors::IEventHandle(selfId, selfId, new TEvPQ::TEvReaderEventArrived()));
- }
- );
-}
-
-void TMirrorer::ProcessNextReaderEvent(TEvPQ::TEvReaderEventArrived::TPtr&, const TActorContext& ctx) {
- TMaybe<NYdb::NPersQueue::TReadSessionEvent::TEvent> event = ReadSession->GetEvent(false);
-
- WaitNextReaderEventInFlight = false;
- if (!event) {
- StartWaitNextReaderEvent(ctx);
- return;
- }
-
- if (auto* dataEvent = std::get_if<TPersQueueReadEvent::TDataReceivedEvent>(&event.GetRef())) {
- AddMessagesToQueue(std::move(dataEvent->GetCompressedMessages()));
- } else if (auto* createStream = std::get_if<TPersQueueReadEvent::TCreatePartitionStreamEvent>(&event.GetRef())) {
- Y_VERIFY_S(
+ auto factory = AppData(ctx)->PersQueueMirrorReaderFactory;
+ Y_VERIFY(factory);
+ CredentialsProvider = factory->GetCredentialsProvider(Config.GetCredentials());
+}
+
+TString TMirrorer::MirrorerDescription() const {
+ return TStringBuilder() << "[mirrorer for " << TopicName << ':' << Partition << ']';
+}
+
+TString TMirrorer::GetCurrentState() const {
+ if (CurrentStateFunc() == &TThis::StateInitConsumer) {
+ return "StateInitConsumer";
+ } else if (CurrentStateFunc() == &TThis::StateWork) {
+ return "StateWork";
+ }
+ return "UNKNOWN";
+}
+
+void TMirrorer::StartWaitNextReaderEvent(const TActorContext& ctx) {
+ if (WaitNextReaderEventInFlight) {
+ return;
+ }
+ auto future = ReadSession->WaitEvent();
+ future.Subscribe(
+ [
+ actorSystem = ctx.ExecutorThread.ActorSystem,
+ selfId = SelfId()
+ ](const NThreading::TFuture<void>&) {
+ actorSystem->Send(new NActors::IEventHandle(selfId, selfId, new TEvPQ::TEvReaderEventArrived()));
+ }
+ );
+}
+
+void TMirrorer::ProcessNextReaderEvent(TEvPQ::TEvReaderEventArrived::TPtr&, const TActorContext& ctx) {
+ TMaybe<NYdb::NPersQueue::TReadSessionEvent::TEvent> event = ReadSession->GetEvent(false);
+
+ WaitNextReaderEventInFlight = false;
+ if (!event) {
+ StartWaitNextReaderEvent(ctx);
+ return;
+ }
+
+ if (auto* dataEvent = std::get_if<TPersQueueReadEvent::TDataReceivedEvent>(&event.GetRef())) {
+ AddMessagesToQueue(std::move(dataEvent->GetCompressedMessages()));
+ } else if (auto* createStream = std::get_if<TPersQueueReadEvent::TCreatePartitionStreamEvent>(&event.GetRef())) {
+ Y_VERIFY_S(
!PartitionStream,
MirrorerDescription()
- << " already has stream " << PartitionStream->GetPartitionStreamId()
- << ", new stream " << createStream->GetPartitionStream()->GetPartitionStreamId()
- );
- PartitionStream = createStream->GetPartitionStream();
- Y_VERIFY_S(
- Partition == PartitionStream->GetPartitionId(),
- MirrorerDescription()
+ << " already has stream " << PartitionStream->GetPartitionStreamId()
+ << ", new stream " << createStream->GetPartitionStream()->GetPartitionStreamId()
+ );
+ PartitionStream = createStream->GetPartitionStream();
+ Y_VERIFY_S(
+ Partition == PartitionStream->GetPartitionId(),
+ MirrorerDescription()
<< " got stream for incorrect partition, stream: topic=" << PartitionStream->GetTopicPath()
- << " partition=" << PartitionStream->GetPartitionId()
- );
- if (OffsetToRead < createStream->GetCommittedOffset()) {
+ << " partition=" << PartitionStream->GetPartitionId()
+ );
+ if (OffsetToRead < createStream->GetCommittedOffset()) {
ProcessError(ctx, TStringBuilder() << "stream has commit offset more then partition end offset,"
- << "gap will be created [" << OffsetToRead << ";" << createStream->GetCommittedOffset() << ")"
- );
- }
- LOG_INFO_S(ctx, NKikimrServices::PQ_MIRRORER,
+ << "gap will be created [" << OffsetToRead << ";" << createStream->GetCommittedOffset() << ")"
+ );
+ }
+ LOG_INFO_S(ctx, NKikimrServices::PQ_MIRRORER,
MirrorerDescription() << " got create stream event for '" << createStream->DebugString()
- << " and will set offset=" << OffsetToRead);
-
- createStream->Confirm(OffsetToRead, OffsetToRead);
- RequestSourcePartitionStatus();
- } else if (auto* destroyStream = std::get_if<TPersQueueReadEvent::TDestroyPartitionStreamEvent>(&event.GetRef())) {
- destroyStream->Confirm();
-
- PartitionStream.Reset();
- LOG_INFO_S(ctx, NKikimrServices::PQ_MIRRORER,
- MirrorerDescription()
- << " got destroy stream event for partition stream id: "
- << destroyStream->GetPartitionStream()->GetPartitionStreamId());
- } else if (auto* streamClosed = std::get_if<TPersQueueReadEvent::TPartitionStreamClosedEvent>(&event.GetRef())) {
- PartitionStream.Reset();
- LOG_INFO_S(ctx, NKikimrServices::PQ_MIRRORER,
- MirrorerDescription()
- << " got stream closed event for partition stream id: "
- << streamClosed->GetPartitionStream()->GetPartitionStreamId()
- << " reason: " << streamClosed->GetReason());
- } else if (auto* streamStatus = std::get_if<TPersQueueReadEvent::TPartitionStreamStatusEvent >(&event.GetRef())) {
- if (PartitionStream
- && PartitionStream->GetPartitionStreamId() == streamStatus->GetPartitionStream()->GetPartitionStreamId()
- ) {
- StreamStatus = MakeHolder<TPersQueueReadEvent::TPartitionStreamStatusEvent>(*streamStatus);
-
- ctx.Schedule(TDuration::Seconds(1), new TEvPQ::TEvRequestPartitionStatus);
- TryUpdateWriteTimetsamp(ctx);
- }
- } else if (auto* commitAck = std::get_if<TPersQueueReadEvent::TCommitAcknowledgementEvent>(&event.GetRef())) {
- LOG_INFO_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription()
- << " got commit responce, commited offset: " << commitAck->GetCommittedOffset());
- } else if (auto* closeSessionEvent = std::get_if<NYdb::NPersQueue::TSessionClosedEvent>(&event.GetRef())) {
- ProcessError(ctx, TStringBuilder() << " read session closed: " << closeSessionEvent->DebugString());
- ScheduleConsumerCreation(ctx);
- return;
- }
-
- Send(SelfId(), new TEvents::TEvWakeup());
- ConsumerInitInterval = CONSUMER_INIT_INTERVAL_START;
-}
-
-}// NPQ
-}// NKikimr
+ << " and will set offset=" << OffsetToRead);
+
+ createStream->Confirm(OffsetToRead, OffsetToRead);
+ RequestSourcePartitionStatus();
+ } else if (auto* destroyStream = std::get_if<TPersQueueReadEvent::TDestroyPartitionStreamEvent>(&event.GetRef())) {
+ destroyStream->Confirm();
+
+ PartitionStream.Reset();
+ LOG_INFO_S(ctx, NKikimrServices::PQ_MIRRORER,
+ MirrorerDescription()
+ << " got destroy stream event for partition stream id: "
+ << destroyStream->GetPartitionStream()->GetPartitionStreamId());
+ } else if (auto* streamClosed = std::get_if<TPersQueueReadEvent::TPartitionStreamClosedEvent>(&event.GetRef())) {
+ PartitionStream.Reset();
+ LOG_INFO_S(ctx, NKikimrServices::PQ_MIRRORER,
+ MirrorerDescription()
+ << " got stream closed event for partition stream id: "
+ << streamClosed->GetPartitionStream()->GetPartitionStreamId()
+ << " reason: " << streamClosed->GetReason());
+ } else if (auto* streamStatus = std::get_if<TPersQueueReadEvent::TPartitionStreamStatusEvent >(&event.GetRef())) {
+ if (PartitionStream
+ && PartitionStream->GetPartitionStreamId() == streamStatus->GetPartitionStream()->GetPartitionStreamId()
+ ) {
+ StreamStatus = MakeHolder<TPersQueueReadEvent::TPartitionStreamStatusEvent>(*streamStatus);
+
+ ctx.Schedule(TDuration::Seconds(1), new TEvPQ::TEvRequestPartitionStatus);
+ TryUpdateWriteTimetsamp(ctx);
+ }
+ } else if (auto* commitAck = std::get_if<TPersQueueReadEvent::TCommitAcknowledgementEvent>(&event.GetRef())) {
+ LOG_INFO_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription()
+ << " got commit responce, commited offset: " << commitAck->GetCommittedOffset());
+ } else if (auto* closeSessionEvent = std::get_if<NYdb::NPersQueue::TSessionClosedEvent>(&event.GetRef())) {
+ ProcessError(ctx, TStringBuilder() << " read session closed: " << closeSessionEvent->DebugString());
+ ScheduleConsumerCreation(ctx);
+ return;
+ }
+
+ Send(SelfId(), new TEvents::TEvWakeup());
+ ConsumerInitInterval = CONSUMER_INIT_INTERVAL_START;
+}
+
+}// NPQ
+}// NKikimr
diff --git a/ydb/core/persqueue/mirrorer.h b/ydb/core/persqueue/mirrorer.h
index 0e7586fe04..6819891f6c 100644
--- a/ydb/core/persqueue/mirrorer.h
+++ b/ydb/core/persqueue/mirrorer.h
@@ -1,9 +1,9 @@
-#pragma once
-
+#pragma once
+
#include "actor_persqueue_client_iface.h"
-
-#include <library/cpp/actors/core/hfunc.h>
-#include <library/cpp/actors/core/log.h>
+
+#include <library/cpp/actors/core/hfunc.h>
+#include <library/cpp/actors/core/log.h>
#include <ydb/core/base/appdata.h>
#include <ydb/core/persqueue/percentile_counter.h>
#include <ydb/core/protos/counters_pq.pb.h>
@@ -12,188 +12,188 @@
#include <ydb/core/persqueue/events/internal.h>
#include <ydb/library/persqueue/counter_time_keeper/counter_time_keeper.h>
#include <ydb/public/sdk/cpp/client/ydb_persqueue_core/persqueue.h>
-
-
-namespace NKikimr {
-namespace NPQ {
-
-class TMirrorer : public TActorBootstrapped<TMirrorer> {
-private:
- const ui64 MAX_BYTES_IN_FLIGHT = 8 * 1024 * 1024;
- const TDuration WRITE_RETRY_TIMEOUT_MAX = TDuration::Seconds(1);
- const TDuration WRITE_RETRY_TIMEOUT_START = TDuration::MilliSeconds(1);
-
- const TDuration CONSUMER_INIT_TIMEOUT_MAX = TDuration::Seconds(60);
- const TDuration CONSUMER_INIT_TIMEOUT_START = TDuration::Seconds(5);
-
- const TDuration CONSUMER_INIT_INTERVAL_MAX = TDuration::Seconds(60);
- const TDuration CONSUMER_INIT_INTERVAL_START = TDuration::Seconds(1);
-
- const TDuration READ_RETRY_TIMEOUT_MAX = TDuration::Seconds(1);
- const TDuration READ_RETRY_TIMEOUT_START = TDuration::MilliSeconds(1);
-
- const TDuration UPDATE_COUNTERS_INTERVAL = TDuration::Seconds(5);
-
- const TDuration LOG_STATE_INTERVAL = TDuration::Minutes(1);
- const TDuration INIT_TIMEOUT = TDuration::Minutes(1);
- const TDuration WRITE_TIMEOUT = TDuration::Minutes(10);
-
-
-private:
- enum EEventCookie : ui64 {
- WRITE_REQUEST_COOKIE = 1,
- UPDATE_WRITE_TIMESTAMP = 2
- };
-
- struct TMessageInfo {
- ui64 Offset;
- ui64 Size;
- TInstant WriteTime;
-
- TMessageInfo(ui64 offset, ui64 size, TInstant writeTime)
- : Offset(offset)
- , Size(size)
- , WriteTime(writeTime)
- {}
- };
-
-private:
-
- STFUNC(StateInitConsumer)
- {
- NPersQueue::TCounterTimeKeeper keeper(Counters.Cumulative()[COUNTER_PQ_TABLET_CPU_USAGE]);
-
- TRACE_EVENT(NKikimrServices::PQ_MIRRORER);
- switch (ev->GetTypeRewrite()) {
- HFuncTraced(TEvPQ::TEvInitCredentials, HandleInitCredentials);
- HFuncTraced(TEvPQ::TEvChangeConfig, HandleChangeConfig);
- HFuncTraced(TEvPQ::TEvCreateConsumer, CreateConsumer);
- HFuncTraced(TEvPQ::TEvRetryWrite, HandleRetryWrite);
- HFuncTraced(TEvPersQueue::TEvResponse, Handle);
- HFuncTraced(TEvPQ::TEvUpdateCounters, Handle);
- HFuncTraced(TEvents::TEvPoisonPill, Handle);
- default:
- break;
- };
- }
-
- STFUNC(StateWork)
- {
- NPersQueue::TCounterTimeKeeper keeper(Counters.Cumulative()[COUNTER_PQ_TABLET_CPU_USAGE]);
-
- TRACE_EVENT(NKikimrServices::PQ_MIRRORER);
- switch (ev->GetTypeRewrite()) {
- HFuncTraced(TEvPQ::TEvChangeConfig, HandleChangeConfig);
- CFunc(TEvents::TSystem::Wakeup, HandleWakeup);
- HFuncTraced(TEvPQ::TEvRequestPartitionStatus, RequestSourcePartitionStatus);
- HFuncTraced(TEvPQ::TEvRetryWrite, HandleRetryWrite);
- HFuncTraced(TEvPersQueue::TEvResponse, Handle);
- HFuncTraced(TEvPQ::TEvUpdateCounters, Handle);
- HFuncTraced(TEvPQ::TEvReaderEventArrived, ProcessNextReaderEvent);
- HFuncTraced(TEvents::TEvPoisonPill, Handle);
- default:
- break;
- };
- }
-
-private:
- template<class TEvent>
+
+
+namespace NKikimr {
+namespace NPQ {
+
+class TMirrorer : public TActorBootstrapped<TMirrorer> {
+private:
+ const ui64 MAX_BYTES_IN_FLIGHT = 8 * 1024 * 1024;
+ const TDuration WRITE_RETRY_TIMEOUT_MAX = TDuration::Seconds(1);
+ const TDuration WRITE_RETRY_TIMEOUT_START = TDuration::MilliSeconds(1);
+
+ const TDuration CONSUMER_INIT_TIMEOUT_MAX = TDuration::Seconds(60);
+ const TDuration CONSUMER_INIT_TIMEOUT_START = TDuration::Seconds(5);
+
+ const TDuration CONSUMER_INIT_INTERVAL_MAX = TDuration::Seconds(60);
+ const TDuration CONSUMER_INIT_INTERVAL_START = TDuration::Seconds(1);
+
+ const TDuration READ_RETRY_TIMEOUT_MAX = TDuration::Seconds(1);
+ const TDuration READ_RETRY_TIMEOUT_START = TDuration::MilliSeconds(1);
+
+ const TDuration UPDATE_COUNTERS_INTERVAL = TDuration::Seconds(5);
+
+ const TDuration LOG_STATE_INTERVAL = TDuration::Minutes(1);
+ const TDuration INIT_TIMEOUT = TDuration::Minutes(1);
+ const TDuration WRITE_TIMEOUT = TDuration::Minutes(10);
+
+
+private:
+ enum EEventCookie : ui64 {
+ WRITE_REQUEST_COOKIE = 1,
+ UPDATE_WRITE_TIMESTAMP = 2
+ };
+
+ struct TMessageInfo {
+ ui64 Offset;
+ ui64 Size;
+ TInstant WriteTime;
+
+ TMessageInfo(ui64 offset, ui64 size, TInstant writeTime)
+ : Offset(offset)
+ , Size(size)
+ , WriteTime(writeTime)
+ {}
+ };
+
+private:
+
+ STFUNC(StateInitConsumer)
+ {
+ NPersQueue::TCounterTimeKeeper keeper(Counters.Cumulative()[COUNTER_PQ_TABLET_CPU_USAGE]);
+
+ TRACE_EVENT(NKikimrServices::PQ_MIRRORER);
+ switch (ev->GetTypeRewrite()) {
+ HFuncTraced(TEvPQ::TEvInitCredentials, HandleInitCredentials);
+ HFuncTraced(TEvPQ::TEvChangeConfig, HandleChangeConfig);
+ HFuncTraced(TEvPQ::TEvCreateConsumer, CreateConsumer);
+ HFuncTraced(TEvPQ::TEvRetryWrite, HandleRetryWrite);
+ HFuncTraced(TEvPersQueue::TEvResponse, Handle);
+ HFuncTraced(TEvPQ::TEvUpdateCounters, Handle);
+ HFuncTraced(TEvents::TEvPoisonPill, Handle);
+ default:
+ break;
+ };
+ }
+
+ STFUNC(StateWork)
+ {
+ NPersQueue::TCounterTimeKeeper keeper(Counters.Cumulative()[COUNTER_PQ_TABLET_CPU_USAGE]);
+
+ TRACE_EVENT(NKikimrServices::PQ_MIRRORER);
+ switch (ev->GetTypeRewrite()) {
+ HFuncTraced(TEvPQ::TEvChangeConfig, HandleChangeConfig);
+ CFunc(TEvents::TSystem::Wakeup, HandleWakeup);
+ HFuncTraced(TEvPQ::TEvRequestPartitionStatus, RequestSourcePartitionStatus);
+ HFuncTraced(TEvPQ::TEvRetryWrite, HandleRetryWrite);
+ HFuncTraced(TEvPersQueue::TEvResponse, Handle);
+ HFuncTraced(TEvPQ::TEvUpdateCounters, Handle);
+ HFuncTraced(TEvPQ::TEvReaderEventArrived, ProcessNextReaderEvent);
+ HFuncTraced(TEvents::TEvPoisonPill, Handle);
+ default:
+ break;
+ };
+ }
+
+private:
+ template<class TEvent>
void ScheduleWithIncreasingTimeout(const TActorId& recipient, TDuration& timeout, const TDuration& maxTimeout, const TActorContext &ctx) {
- ctx.ExecutorThread.ActorSystem->Schedule(timeout, new IEventHandle(recipient, SelfId(), new TEvent()));
- timeout = Min(timeout * 2, maxTimeout);
- }
-
- bool AddToWriteRequest(
- NKikimrClient::TPersQueuePartitionRequest& request,
- NYdb::NPersQueue::TReadSessionEvent::TDataReceivedEvent::TCompressedMessage& message
- );
- void ProcessError(const TActorContext& ctx, const TString& msg);
- void ProcessError(const TActorContext& ctx, const TString& msg, const NKikimrClient::TResponse& response);
- void AfterSuccesWrite(const TActorContext& ctx);
- void ProcessWriteResponse(
- const TActorContext& ctx,
- const NKikimrClient::TPersQueuePartitionResponse& response
- );
+ ctx.ExecutorThread.ActorSystem->Schedule(timeout, new IEventHandle(recipient, SelfId(), new TEvent()));
+ timeout = Min(timeout * 2, maxTimeout);
+ }
+
+ bool AddToWriteRequest(
+ NKikimrClient::TPersQueuePartitionRequest& request,
+ NYdb::NPersQueue::TReadSessionEvent::TDataReceivedEvent::TCompressedMessage& message
+ );
+ void ProcessError(const TActorContext& ctx, const TString& msg);
+ void ProcessError(const TActorContext& ctx, const TString& msg, const NKikimrClient::TResponse& response);
+ void AfterSuccesWrite(const TActorContext& ctx);
+ void ProcessWriteResponse(
+ const TActorContext& ctx,
+ const NKikimrClient::TPersQueuePartitionResponse& response
+ );
void ScheduleConsumerCreation(const TActorContext& ctx);
void RecreateCredentialsProvider(const TActorContext& ctx);
- void StartInit(const TActorContext& ctx);
- void RetryWrite(const TActorContext& ctx);
-
- void ProcessNextReaderEvent(TEvPQ::TEvReaderEventArrived::TPtr& ev, const TActorContext& ctx);
-
- TString MirrorerDescription() const;
-
- TString GetCurrentState() const;
-
-public:
- static constexpr NKikimrServices::TActivity::EType ActorActivityType();
- TMirrorer(
- TActorId tabletActor,
- TActorId partitionActor,
- const TString& topicName,
- ui32 partition,
- bool localDC,
- ui64 endOffset,
- const NKikimrPQ::TMirrorPartitionConfig& config,
- const TTabletCountersBase& counters
- );
- void Bootstrap(const TActorContext& ctx);
- void Handle(TEvents::TEvPoisonPill::TPtr& ev, const TActorContext& ctx);
- void Handle(TEvPersQueue::TEvResponse::TPtr& ev, const TActorContext& ctx);
- void Handle(TEvPQ::TEvUpdateCounters::TPtr& ev, const TActorContext& ctx);
- void HandleChangeConfig(TEvPQ::TEvChangeConfig::TPtr& ev, const TActorContext& ctx);
- void TryToRead(const TActorContext& ctx);
- void TryToWrite(const TActorContext& ctx);
- void HandleInitCredentials(TEvPQ::TEvInitCredentials::TPtr& ev, const TActorContext& ctx);
- void HandleRetryWrite(TEvPQ::TEvRetryWrite::TPtr& ev, const TActorContext& ctx);
- void HandleWakeup(const TActorContext& ctx);
- void CreateConsumer(TEvPQ::TEvCreateConsumer::TPtr& ev, const TActorContext& ctx);
- void RequestSourcePartitionStatus(TEvPQ::TEvRequestPartitionStatus::TPtr& ev, const TActorContext& ctx);
- void RequestSourcePartitionStatus();
- void TryUpdateWriteTimetsamp(const TActorContext &ctx);
- void AddMessagesToQueue(
- TVector<NYdb::NPersQueue::TReadSessionEvent::TDataReceivedEvent::TCompressedMessage>&& messages
- );
- void StartWaitNextReaderEvent(const TActorContext& ctx);
-
-private:
- TActorId TabletActor;
+ void StartInit(const TActorContext& ctx);
+ void RetryWrite(const TActorContext& ctx);
+
+ void ProcessNextReaderEvent(TEvPQ::TEvReaderEventArrived::TPtr& ev, const TActorContext& ctx);
+
+ TString MirrorerDescription() const;
+
+ TString GetCurrentState() const;
+
+public:
+ static constexpr NKikimrServices::TActivity::EType ActorActivityType();
+ TMirrorer(
+ TActorId tabletActor,
+ TActorId partitionActor,
+ const TString& topicName,
+ ui32 partition,
+ bool localDC,
+ ui64 endOffset,
+ const NKikimrPQ::TMirrorPartitionConfig& config,
+ const TTabletCountersBase& counters
+ );
+ void Bootstrap(const TActorContext& ctx);
+ void Handle(TEvents::TEvPoisonPill::TPtr& ev, const TActorContext& ctx);
+ void Handle(TEvPersQueue::TEvResponse::TPtr& ev, const TActorContext& ctx);
+ void Handle(TEvPQ::TEvUpdateCounters::TPtr& ev, const TActorContext& ctx);
+ void HandleChangeConfig(TEvPQ::TEvChangeConfig::TPtr& ev, const TActorContext& ctx);
+ void TryToRead(const TActorContext& ctx);
+ void TryToWrite(const TActorContext& ctx);
+ void HandleInitCredentials(TEvPQ::TEvInitCredentials::TPtr& ev, const TActorContext& ctx);
+ void HandleRetryWrite(TEvPQ::TEvRetryWrite::TPtr& ev, const TActorContext& ctx);
+ void HandleWakeup(const TActorContext& ctx);
+ void CreateConsumer(TEvPQ::TEvCreateConsumer::TPtr& ev, const TActorContext& ctx);
+ void RequestSourcePartitionStatus(TEvPQ::TEvRequestPartitionStatus::TPtr& ev, const TActorContext& ctx);
+ void RequestSourcePartitionStatus();
+ void TryUpdateWriteTimetsamp(const TActorContext &ctx);
+ void AddMessagesToQueue(
+ TVector<NYdb::NPersQueue::TReadSessionEvent::TDataReceivedEvent::TCompressedMessage>&& messages
+ );
+ void StartWaitNextReaderEvent(const TActorContext& ctx);
+
+private:
+ TActorId TabletActor;
TActorId PartitionActor;
- TString TopicName;
- ui32 Partition;
- bool LocalDC;
- ui64 EndOffset;
- ui64 OffsetToRead;
- NKikimrPQ::TMirrorPartitionConfig Config;
-
- TDeque<NYdb::NPersQueue::TReadSessionEvent::TDataReceivedEvent::TCompressedMessage> Queue;
- TDeque<TMessageInfo> WriteInFlight;
- ui64 BytesInFlight = 0;
- std::optional<NKikimrClient::TPersQueuePartitionRequest> WriteRequestInFlight;
- TDuration WriteRetryTimeout = WRITE_RETRY_TIMEOUT_START;
- TInstant WriteRequestTimestamp;
- std::shared_ptr<NYdb::ICredentialsProviderFactory> CredentialsProvider;
- std::shared_ptr<NYdb::NPersQueue::IReadSession> ReadSession;
- NYdb::NPersQueue::TPartitionStream::TPtr PartitionStream;
- THolder<NYdb::NPersQueue::TReadSessionEvent::TPartitionStreamStatusEvent> StreamStatus;
- TInstant LastInitStageTimestamp;
-
- TDuration ConsumerInitTimeout = CONSUMER_INIT_TIMEOUT_START;
- TDuration ConsumerInitInterval = CONSUMER_INIT_INTERVAL_START;
- TDuration ReadRetryTimeout = READ_RETRY_TIMEOUT_START;
-
- TTabletCountersBase Counters;
-
- bool WaitNextReaderEventInFlight = false;
-
- bool WasSuccessfulRecording = false;
- TInstant LastStateLogTimestamp;
-
- TMultiCounter MirrorerErrors;
- TMultiCounter InitTimeoutCounter;
- TMultiCounter WriteTimeoutCounter;
- THolder<TPercentileCounter> MirrorerTimeLags;
-};
-
-}// NPQ
-}// NKikimr
+ TString TopicName;
+ ui32 Partition;
+ bool LocalDC;
+ ui64 EndOffset;
+ ui64 OffsetToRead;
+ NKikimrPQ::TMirrorPartitionConfig Config;
+
+ TDeque<NYdb::NPersQueue::TReadSessionEvent::TDataReceivedEvent::TCompressedMessage> Queue;
+ TDeque<TMessageInfo> WriteInFlight;
+ ui64 BytesInFlight = 0;
+ std::optional<NKikimrClient::TPersQueuePartitionRequest> WriteRequestInFlight;
+ TDuration WriteRetryTimeout = WRITE_RETRY_TIMEOUT_START;
+ TInstant WriteRequestTimestamp;
+ std::shared_ptr<NYdb::ICredentialsProviderFactory> CredentialsProvider;
+ std::shared_ptr<NYdb::NPersQueue::IReadSession> ReadSession;
+ NYdb::NPersQueue::TPartitionStream::TPtr PartitionStream;
+ THolder<NYdb::NPersQueue::TReadSessionEvent::TPartitionStreamStatusEvent> StreamStatus;
+ TInstant LastInitStageTimestamp;
+
+ TDuration ConsumerInitTimeout = CONSUMER_INIT_TIMEOUT_START;
+ TDuration ConsumerInitInterval = CONSUMER_INIT_INTERVAL_START;
+ TDuration ReadRetryTimeout = READ_RETRY_TIMEOUT_START;
+
+ TTabletCountersBase Counters;
+
+ bool WaitNextReaderEventInFlight = false;
+
+ bool WasSuccessfulRecording = false;
+ TInstant LastStateLogTimestamp;
+
+ TMultiCounter MirrorerErrors;
+ TMultiCounter InitTimeoutCounter;
+ TMultiCounter WriteTimeoutCounter;
+ THolder<TPercentileCounter> MirrorerTimeLags;
+};
+
+}// NPQ
+}// NKikimr
diff --git a/ydb/core/persqueue/ownerinfo.h b/ydb/core/persqueue/ownerinfo.h
index ed7c31be04..a6f5aef716 100644
--- a/ydb/core/persqueue/ownerinfo.h
+++ b/ydb/core/persqueue/ownerinfo.h
@@ -1,10 +1,10 @@
#pragma once
#include <ydb/core/keyvalue/keyvalue_events.h>
#include <ydb/core/persqueue/events/internal.h>
-
+
#include <library/cpp/actors/core/actor.h>
-
+
namespace NKikimr {
namespace NPQ {
diff --git a/ydb/core/persqueue/partition.cpp b/ydb/core/persqueue/partition.cpp
index 739b3a3465..cc8e2419cf 100644
--- a/ydb/core/persqueue/partition.cpp
+++ b/ydb/core/persqueue/partition.cpp
@@ -1,10 +1,10 @@
#include "partition.h"
-#include "event_helpers.h"
+#include "event_helpers.h"
#include "read.h"
#include "sourceid.h"
#include "ownerinfo.h"
-#include "mirrorer.h"
-
+#include "mirrorer.h"
+
#include <ydb/core/base/appdata.h>
#include <ydb/core/base/blobstorage.h>
#include <ydb/core/base/counters.h>
@@ -28,19 +28,19 @@
namespace NKikimr {
namespace NPQ {
-static const ui32 BATCH_UNPACK_SIZE_BORDER = 500 * 1024; //500kb
-
+static const ui32 BATCH_UNPACK_SIZE_BORDER = 500 * 1024; //500kb
+
static const ui32 MAX_WRITE_CYCLE_SIZE = 16 << 20; //16MB
static const ui32 MAX_USER_ACTS = 1000;
-static const TDuration WAKE_TIMEOUT = TDuration::Seconds(5);
+static const TDuration WAKE_TIMEOUT = TDuration::Seconds(5);
static const ui32 MAX_INLINE_SIZE = 1000;
static const ui32 LEVEL0 = 32;
-static const TDuration UPDATE_AVAIL_SIZE_INTERVAL = TDuration::MilliSeconds(100);
+static const TDuration UPDATE_AVAIL_SIZE_INTERVAL = TDuration::MilliSeconds(100);
static const TString WRITE_QUOTA_ROOT_PATH = "write-quota";
@@ -58,7 +58,7 @@ struct TPartition::THasDataReq {
};
struct TPartition::THasDataDeadline {
- TInstant Deadline;
+ TInstant Deadline;
TPartition::THasDataReq Request;
bool operator < (const THasDataDeadline& dl) const
@@ -215,12 +215,12 @@ IOutputStream& operator <<(IOutputStream& out, const TKeyLevel& value) {
}
-ui64 GetOffsetEstimate(const std::deque<TDataKey>& container, TInstant timestamp, ui64 offset) {
+ui64 GetOffsetEstimate(const std::deque<TDataKey>& container, TInstant timestamp, ui64 offset) {
if (container.empty()) {
return offset;
}
auto it = std::lower_bound(container.begin(), container.end(), timestamp,
- [](const TDataKey& p, const TInstant timestamp) { return timestamp > p.Timestamp; });
+ [](const TDataKey& p, const TInstant timestamp) { return timestamp > p.Timestamp; });
if (it == container.end()) {
return offset;
} else {
@@ -228,23 +228,23 @@ ui64 GetOffsetEstimate(const std::deque<TDataKey>& container, TInstant timestamp
}
}
-struct TMirrorerInfo {
- TMirrorerInfo(const TActorId& actor, const TTabletCountersBase& baseline)
- : Actor(actor)
- {
- Baseline.Populate(baseline);
- }
-
- TActorId Actor;
- TTabletCountersBase Baseline;
-};
-
+struct TMirrorerInfo {
+ TMirrorerInfo(const TActorId& actor, const TTabletCountersBase& baseline)
+ : Actor(actor)
+ {
+ Baseline.Populate(baseline);
+ }
+
+ TActorId Actor;
+ TTabletCountersBase Baseline;
+};
+
void TPartition::ReplyError(const TActorContext& ctx, const ui64 dst, NPersQueue::NErrorCode::EErrorCode errorCode, const TString& error)
{
- ReplyPersQueueError(
- dst == 0 ? ctx.SelfID : Tablet, ctx, TabletID, TopicName, Partition, Counters, NKikimrServices::PERSQUEUE,
+ ReplyPersQueueError(
+ dst == 0 ? ctx.SelfID : Tablet, ctx, TabletID, TopicName, Partition, Counters, NKikimrServices::PERSQUEUE,
dst, errorCode, error, true
- );
+ );
}
void TPartition::ReplyOk(const TActorContext& ctx, const ui64 dst)
@@ -266,10 +266,10 @@ void TPartition::ReplyOwnerOk(const TActorContext& ctx, const ui64 dst, const TS
ctx.Send(Tablet, response.Release());
}
-void TPartition::ReplyWrite(
- const TActorContext& ctx, const ui64 dst, const TString& sourceId, const ui64 seqNo, const ui16 partNo, const ui16 totalParts,
- const ui64 offset, const TInstant writeTimestamp, bool already, const ui64 maxSeqNo,
- const ui64 partitionQuotedTime, const TDuration topicQuotedTime, const ui64 queueTime, const ui64 writeTime)
+void TPartition::ReplyWrite(
+ const TActorContext& ctx, const ui64 dst, const TString& sourceId, const ui64 seqNo, const ui16 partNo, const ui16 totalParts,
+ const ui64 offset, const TInstant writeTimestamp, bool already, const ui64 maxSeqNo,
+ const ui64 partitionQuotedTime, const TDuration topicQuotedTime, const ui64 queueTime, const ui64 writeTime)
{
Y_VERIFY(offset <= (ui64)Max<i64>(), "Offset is too big: %" PRIu64, offset);
Y_VERIFY(seqNo <= (ui64)Max<i64>(), "SeqNo is too big: %" PRIu64, seqNo);
@@ -281,7 +281,7 @@ void TPartition::ReplyWrite(
auto write = resp.MutablePartitionResponse()->AddCmdWriteResult();
write->SetSourceId(sourceId);
write->SetSeqNo(seqNo);
- write->SetWriteTimestampMS(writeTimestamp.MilliSeconds());
+ write->SetWriteTimestampMS(writeTimestamp.MilliSeconds());
if (totalParts > 1)
write->SetPartNo(partNo);
write->SetAlreadyWritten(already);
@@ -290,7 +290,7 @@ void TPartition::ReplyWrite(
write->SetOffset(offset);
write->SetPartitionQuotedTimeMs(partitionQuotedTime);
- write->SetTopicQuotedTimeMs(topicQuotedTime.MilliSeconds());
+ write->SetTopicQuotedTimeMs(topicQuotedTime.MilliSeconds());
write->SetTotalTimeInPartitionQueueMs(queueTime);
write->SetWriteTimeMs(writeTime);
@@ -299,7 +299,7 @@ void TPartition::ReplyWrite(
void TPartition::ReplyGetClientOffsetOk(const TActorContext& ctx, const ui64 dst, const i64 offset,
- const TInstant writeTimestamp, const TInstant createTimestamp)
+ const TInstant writeTimestamp, const TInstant createTimestamp)
{
THolder<TEvPQ::TEvProxyResponse> response = MakeHolder<TEvPQ::TEvProxyResponse>(dst);
NKikimrClient::TResponse& resp = response->Response;
@@ -309,11 +309,11 @@ void TPartition::ReplyGetClientOffsetOk(const TActorContext& ctx, const ui64 dst
auto user = resp.MutablePartitionResponse()->MutableCmdGetClientOffsetResult();
if (offset > -1)
user->SetOffset(offset);
- if (writeTimestamp)
- user->SetWriteTimestampMS(writeTimestamp.MilliSeconds());
- if (createTimestamp) {
- Y_VERIFY(writeTimestamp);
- user->SetCreateTimestampMS(createTimestamp.MilliSeconds());
+ if (writeTimestamp)
+ user->SetWriteTimestampMS(writeTimestamp.MilliSeconds());
+ if (createTimestamp) {
+ Y_VERIFY(writeTimestamp);
+ user->SetCreateTimestampMS(createTimestamp.MilliSeconds());
}
user->SetEndOffset(EndOffset);
user->SetSizeLag(GetSizeLag(offset));
@@ -412,17 +412,17 @@ void RequestDataRange(const TActorContext& ctx, const TActorId& dst, ui32 partit
void TPartition::FillReadFromTimestamps(const NKikimrPQ::TPQTabletConfig& config, const TActorContext& ctx) {
TSet<TString> hasReadRule;
- for (auto& userInfo : UsersInfoStorage.GetAll()) {
- userInfo.second.ReadFromTimestamp = TInstant::Zero();
- if (userInfo.second.HasReadRule) {
- userInfo.second.HasReadRule = false;
- hasReadRule.insert(userInfo.first);
+ for (auto& userInfo : UsersInfoStorage.GetAll()) {
+ userInfo.second.ReadFromTimestamp = TInstant::Zero();
+ if (userInfo.second.HasReadRule) {
+ userInfo.second.HasReadRule = false;
+ hasReadRule.insert(userInfo.first);
}
}
for (ui32 i = 0; i < config.ReadRulesSize(); ++i) {
const auto& consumer = config.GetReadRules(i);
- auto& userInfo = UsersInfoStorage.GetOrCreate(consumer, ctx);
- userInfo.HasReadRule = true;
+ auto& userInfo = UsersInfoStorage.GetOrCreate(consumer, ctx);
+ userInfo.HasReadRule = true;
ui64 rrGen = i < config.ReadRuleGenerationsSize() ? config.GetReadRuleGenerations(i) : 0;
if (userInfo.ReadRuleGeneration != rrGen) {
THolder<TEvPQ::TEvSetClientInfo> event = MakeHolder<TEvPQ::TEvSetClientInfo>(0, consumer, 0, "", 0, 0,
@@ -433,15 +433,15 @@ void TPartition::FillReadFromTimestamps(const NKikimrPQ::TPQTabletConfig& config
userInfo.Step = userInfo.Generation = 0;
}
hasReadRule.erase(consumer);
- TInstant ts = i < config.ReadFromTimestampsMsSize() ? TInstant::MilliSeconds(config.GetReadFromTimestampsMs(i)) : TInstant::Zero();
- if (!ts) ts += TDuration::MilliSeconds(1);
- if (!userInfo.ReadFromTimestamp|| userInfo.ReadFromTimestamp > ts)
- userInfo.ReadFromTimestamp = ts;
+ TInstant ts = i < config.ReadFromTimestampsMsSize() ? TInstant::MilliSeconds(config.GetReadFromTimestampsMs(i)) : TInstant::Zero();
+ if (!ts) ts += TDuration::MilliSeconds(1);
+ if (!userInfo.ReadFromTimestamp|| userInfo.ReadFromTimestamp > ts)
+ userInfo.ReadFromTimestamp = ts;
}
for (auto& consumer : hasReadRule) {
- auto& userInfo = UsersInfoStorage.GetOrCreate(consumer, ctx);
- if (!userInfo.Important) {
- ctx.Send(Tablet, new TEvPQ::TEvPartitionLabeledCountersDrop(Partition, userInfo.LabeledCounters.GetGroup()));
+ auto& userInfo = UsersInfoStorage.GetOrCreate(consumer, ctx);
+ if (!userInfo.Important) {
+ ctx.Send(Tablet, new TEvPQ::TEvPartitionLabeledCountersDrop(Partition, userInfo.LabeledCounters.GetGroup()));
}
THolder<TEvPQ::TEvSetClientInfo> event = MakeHolder<TEvPQ::TEvSetClientInfo>(0, consumer,
0, "", 0, 0, TEvPQ::TEvSetClientInfo::ESCI_DROP_READ_RULE, 0);
@@ -470,7 +470,7 @@ TPartition::TPartition(ui64 tabletId, ui32 partition, const TActorId& tablet, co
, BlobCache(blobCache)
, InitState(WaitDiskStatus)
, PartitionedBlob(partition, 0, 0, 0, 0, 0, Head, NewHead, true, false, 8 << 20)
- , NewHeadKey{TKey{}, 0, TInstant::Zero(), 0}
+ , NewHeadKey{TKey{}, 0, TInstant::Zero(), 0}
, BodySize(0)
, MaxWriteResponsesSize(0)
, GapSize(0)
@@ -508,13 +508,13 @@ TPartition::TPartition(ui64 tabletId, ui32 partition, const TActorId& tablet, co
{
if (Config.GetPartitionConfig().HasMirrorFrom()) {
ManageWriteTimestampEstimate = !Config.GetPartitionConfig().GetMirrorFrom().GetSyncWriteTime();
- } else {
- ManageWriteTimestampEstimate = LocalDC;
- }
+ } else {
+ ManageWriteTimestampEstimate = LocalDC;
+ }
- WriteTimestampEstimate = ManageWriteTimestampEstimate ? ctx.Now() : TInstant::Zero();
-
+ WriteTimestampEstimate = ManageWriteTimestampEstimate ? ctx.Now() : TInstant::Zero();
+
CalcTopicWriteQuotaParams();
@@ -588,7 +588,7 @@ void TPartition::HandleMonitoring(TEvPQ::TEvMonRequest::TPtr& ev, const TActorCo
TABLER() {
TABLED() {out << "DataBody";}
TABLED() {out << i++;}
- TABLED() {out << ToStringLocalTimeUpToSeconds(d.Timestamp);}
+ TABLED() {out << ToStringLocalTimeUpToSeconds(d.Timestamp);}
TABLED() {out << d.Key.GetOffset();}
TABLED() {out << d.Key.GetPartNo();}
TABLED() {out << d.Key.GetCount();}
@@ -605,7 +605,7 @@ void TPartition::HandleMonitoring(TEvPQ::TEvMonRequest::TPtr& ev, const TActorCo
TABLER() {
TABLED() {out << "DataHead[" << currentLevel << "]";}
TABLED() {out << i++;}
- TABLED() {out << ToStringLocalTimeUpToSeconds(HeadKeys[p].Timestamp);}
+ TABLED() {out << ToStringLocalTimeUpToSeconds(HeadKeys[p].Timestamp);}
TABLED() {out << HeadKeys[p].Key.GetOffset();}
TABLED() {out << HeadKeys[p].Key.GetPartNo();}
TABLED() {out << HeadKeys[p].Key.GetCount();}
@@ -692,17 +692,17 @@ void TPartition::HandleMonitoring(TEvPQ::TEvMonRequest::TPtr& ev, const TActorCo
}
}
TABLEBODY() {
- for (auto& d: UsersInfoStorage.GetAll()) {
+ for (auto& d: UsersInfoStorage.GetAll()) {
TABLER() {
TABLED() {out << EncodeHtmlPcdata(d.first);}
TABLED() {out << d.second.Offset;}
TABLED() {out << (EndOffset - d.second.Offset);}
- TABLED() {out << ToStringLocalTimeUpToSeconds(d.second.ReadFromTimestamp);}
- TABLED() {out << ToStringLocalTimeUpToSeconds(d.second.WriteTimestamp);}
- TABLED() {out << ToStringLocalTimeUpToSeconds(d.second.CreateTimestamp);}
+ TABLED() {out << ToStringLocalTimeUpToSeconds(d.second.ReadFromTimestamp);}
+ TABLED() {out << ToStringLocalTimeUpToSeconds(d.second.WriteTimestamp);}
+ TABLED() {out << ToStringLocalTimeUpToSeconds(d.second.CreateTimestamp);}
TABLED() {out << (d.second.GetReadOffset());}
- TABLED() {out << ToStringLocalTimeUpToSeconds(d.second.GetReadWriteTimestamp());}
- TABLED() {out << ToStringLocalTimeUpToSeconds(d.second.GetReadCreateTimestamp());}
+ TABLED() {out << ToStringLocalTimeUpToSeconds(d.second.GetReadWriteTimestamp());}
+ TABLED() {out << ToStringLocalTimeUpToSeconds(d.second.GetReadCreateTimestamp());}
TABLED() {out << (d.second.ReadOffsetRewindSum);}
TABLED() {out << d.second.ActiveReads;}
TABLED() {out << d.second.Subscriptions;}
@@ -720,8 +720,8 @@ void TPartition::HandleMonitoring(TEvPQ::TEvMonRequest::TPtr& ev, const TActorCo
void TPartition::Bootstrap(const TActorContext& ctx)
{
- UsersInfoStorage.Init(Tablet, SelfId());
-
+ UsersInfoStorage.Init(Tablet, SelfId());
+
Y_VERIFY(AppData(ctx)->PQConfig.GetMaxBlobsPerLevel() > 0);
ui32 border = LEVEL0;
MaxSizeCheck = 0;
@@ -744,8 +744,8 @@ void TPartition::Bootstrap(const TActorContext& ctx)
}
for (const auto& readQuota : Config.GetPartitionConfig().GetReadQuota()) {
- auto &userInfo = UsersInfoStorage.GetOrCreate(readQuota.GetClientId(), ctx);
- userInfo.ReadQuota.UpdateConfig(readQuota.GetBurstSize(), readQuota.GetSpeedInBytesPerSecond());
+ auto &userInfo = UsersInfoStorage.GetOrCreate(readQuota.GetClientId(), ctx);
+ userInfo.ReadQuota.UpdateConfig(readQuota.GetBurstSize(), readQuota.GetSpeedInBytesPerSecond());
}
LOG_INFO_S(ctx, NKikimrServices::PERSQUEUE, "boostrapping " << Partition << " " << ctx.SelfID);
@@ -930,8 +930,8 @@ void TPartition::ProcessHasDataRequests(const TActorContext& ctx) {
res->Record.SetCookie(*(it->Cookie));
ctx.Send(it->Sender, res.Release());
if (!it->ClientId.empty()) {
- auto& userInfo = UsersInfoStorage.GetOrCreate(it->ClientId, ctx);
- userInfo.ForgetSubscription(ctx.Now());
+ auto& userInfo = UsersInfoStorage.GetOrCreate(it->ClientId, ctx);
+ userInfo.ForgetSubscription(ctx.Now());
}
it = HasDataRequests.erase(it);
} else {
@@ -939,7 +939,7 @@ void TPartition::ProcessHasDataRequests(const TActorContext& ctx) {
}
}
for (auto it = HasDataDeadlines.begin(); it != HasDataDeadlines.end();) {
- if (it->Deadline <= ctx.Now()) {
+ if (it->Deadline <= ctx.Now()) {
auto jt = HasDataRequests.find(it->Request);
if (jt != HasDataRequests.end()) {
TAutoPtr<TEvPersQueue::TEvHasDataInfoResponse> res(new TEvPersQueue::TEvHasDataInfoResponse());
@@ -950,8 +950,8 @@ void TPartition::ProcessHasDataRequests(const TActorContext& ctx) {
res->Record.SetCookie(*(it->Request.Cookie));
ctx.Send(it->Request.Sender, res.Release());
if (!it->Request.ClientId.empty()) {
- auto& userInfo = UsersInfoStorage.GetOrCreate(it->Request.ClientId, ctx);
- userInfo.ForgetSubscription(ctx.Now());
+ auto& userInfo = UsersInfoStorage.GetOrCreate(it->Request.ClientId, ctx);
+ userInfo.ForgetSubscription(ctx.Now());
}
HasDataRequests.erase(jt);
}
@@ -969,7 +969,7 @@ void TPartition::UpdateAvailableSize(const TActorContext& ctx) {
auto now = ctx.Now();
WriteQuota.Update(now);
- for (auto &c : UsersInfoStorage.GetAll()) {
+ for (auto &c : UsersInfoStorage.GetAll()) {
while (true) {
c.second.ReadQuota.Update(now);
if (!c.second.ReadQuota.CanExaust() && !c.second.ReadRequests.empty()) {
@@ -1001,7 +1001,7 @@ void TPartition::HandleOnWrite(TEvPQ::TEvUpdateAvailableSize::TPtr&, const TActo
void TPartition::HandleWakeup(const TActorContext& ctx) {
FilterDeadlinedWrites(ctx);
- ctx.Schedule(WAKE_TIMEOUT, new TEvents::TEvWakeup());
+ ctx.Schedule(WAKE_TIMEOUT, new TEvents::TEvWakeup());
ctx.Send(Tablet, new TEvPQ::TEvPartitionCounters(Partition, Counters));
ReportLabeledCounters(ctx);
@@ -1009,9 +1009,9 @@ void TPartition::HandleWakeup(const TActorContext& ctx) {
ProcessHasDataRequests(ctx);
auto now = ctx.Now();
- for (auto& userInfo : UsersInfoStorage.GetAll()) {
- userInfo.second.UpdateReadingTimeAndState(now);
- for (auto& avg : userInfo.second.AvgReadBytes) {
+ for (auto& userInfo : UsersInfoStorage.GetAll()) {
+ userInfo.second.UpdateReadingTimeAndState(now);
+ for (auto& avg : userInfo.second.AvgReadBytes) {
avg.Update(now);
}
}
@@ -1032,7 +1032,7 @@ void TPartition::HandleWakeup(const TActorContext& ctx) {
}
Y_VERIFY(CurrentStateFunc() == &TThis::StateIdle);
- if (ManageWriteTimestampEstimate)
+ if (ManageWriteTimestampEstimate)
WriteTimestampEstimate = now;
THolder <TEvKeyValue::TEvRequest> request = MakeHolder<TEvKeyValue::TEvRequest>();
@@ -1094,10 +1094,10 @@ bool TPartition::DropOldData(TEvKeyValue::TEvRequest *request, bool hasWrites, c
ui64 minOffset = EndOffset;
for (const auto& importantClientId : Config.GetPartitionConfig().GetImportantClientId()) {
- TUserInfo* userInfo = UsersInfoStorage.GetIfExists(importantClientId);
+ TUserInfo* userInfo = UsersInfoStorage.GetIfExists(importantClientId);
ui64 curOffset = StartOffset;
- if (userInfo && userInfo->Offset >= 0) //-1 means no offset
- curOffset = userInfo->Offset;
+ if (userInfo && userInfo->Offset >= 0) //-1 means no offset
+ curOffset = userInfo->Offset;
minOffset = Min<ui64>(minOffset, curOffset);
}
@@ -1192,39 +1192,39 @@ void TPartition::Handle(TEvPersQueue::TEvHasDataInfo::TPtr& ev, const TActorCont
} else {
THasDataReq req{++HasDataReqNum, (ui64)record.GetOffset(), sender, record.HasCookie() ? TMaybe<ui64>(record.GetCookie()) : TMaybe<ui64>(),
record.HasClientId() && InitDone ? record.GetClientId() : ""};
- THasDataDeadline dl{TInstant::MilliSeconds(record.GetDeadline()), req};
+ THasDataDeadline dl{TInstant::MilliSeconds(record.GetDeadline()), req};
auto res = HasDataRequests.insert(req);
HasDataDeadlines.insert(dl);
Y_VERIFY(res.second);
if (InitDone && record.HasClientId() && !record.GetClientId().empty()) {
- auto& userInfo = UsersInfoStorage.GetOrCreate(record.GetClientId(), ctx);
- ++userInfo.Subscriptions;
- userInfo.UpdateReadOffset((i64)EndOffset - 1, ctx.Now(), ctx.Now(), ctx.Now());
- userInfo.UpdateReadingTimeAndState(ctx.Now());
- }
- }
-}
-
-void TPartition::Handle(TEvPQ::TEvMirrorerCounters::TPtr& ev, const TActorContext& /*ctx*/) {
- if (Mirrorer) {
- auto diff = ev->Get()->Counters.MakeDiffForAggr(Mirrorer->Baseline);
- Counters.Populate(*diff.Get());
- ev->Get()->Counters.RememberCurrentStateAsBaseline(Mirrorer->Baseline);
- }
-}
-
-void TPartition::Handle(NReadSpeedLimiterEvents::TEvCounters::TPtr& ev, const TActorContext& /*ctx*/) {
- auto userInfo = UsersInfoStorage.GetIfExists(ev->Get()->User);
- Y_VERIFY(userInfo);
- if (userInfo) {
- Y_VERIFY(userInfo->ReadSpeedLimiter);
- auto diff = ev->Get()->Counters.MakeDiffForAggr(userInfo->ReadSpeedLimiter->Baseline);
- Counters.Populate(*diff.Get());
- ev->Get()->Counters.RememberCurrentStateAsBaseline(userInfo->ReadSpeedLimiter->Baseline);
- }
-}
-
+ auto& userInfo = UsersInfoStorage.GetOrCreate(record.GetClientId(), ctx);
+ ++userInfo.Subscriptions;
+ userInfo.UpdateReadOffset((i64)EndOffset - 1, ctx.Now(), ctx.Now(), ctx.Now());
+ userInfo.UpdateReadingTimeAndState(ctx.Now());
+ }
+ }
+}
+
+void TPartition::Handle(TEvPQ::TEvMirrorerCounters::TPtr& ev, const TActorContext& /*ctx*/) {
+ if (Mirrorer) {
+ auto diff = ev->Get()->Counters.MakeDiffForAggr(Mirrorer->Baseline);
+ Counters.Populate(*diff.Get());
+ ev->Get()->Counters.RememberCurrentStateAsBaseline(Mirrorer->Baseline);
+ }
+}
+
+void TPartition::Handle(NReadSpeedLimiterEvents::TEvCounters::TPtr& ev, const TActorContext& /*ctx*/) {
+ auto userInfo = UsersInfoStorage.GetIfExists(ev->Get()->User);
+ Y_VERIFY(userInfo);
+ if (userInfo) {
+ Y_VERIFY(userInfo->ReadSpeedLimiter);
+ auto diff = ev->Get()->Counters.MakeDiffForAggr(userInfo->ReadSpeedLimiter->Baseline);
+ Counters.Populate(*diff.Get());
+ ev->Get()->Counters.RememberCurrentStateAsBaseline(userInfo->ReadSpeedLimiter->Baseline);
+ }
+}
+
void TPartition::Handle(TEvents::TEvPoisonPill::TPtr&, const TActorContext& ctx) {
// Reply to all outstanding requests in order to destroy corresponding actors
@@ -1248,11 +1248,11 @@ void TPartition::Handle(TEvents::TEvPoisonPill::TPtr&, const TActorContext& ctx)
TStringBuilder() << ss << " (ReadInfo) cookie " << ri.first);
}
- if (Mirrorer) {
- Send(Mirrorer->Actor, new TEvents::TEvPoisonPill());
- }
- UsersInfoStorage.Clear(ctx);
-
+ if (Mirrorer) {
+ Send(Mirrorer->Actor, new TEvents::TEvPoisonPill());
+ }
+ UsersInfoStorage.Clear(ctx);
+
Die(ctx);
}
@@ -1331,9 +1331,9 @@ void TPartition::HandleGetDiskStatus(const NKikimrClient::TResponse& response, c
diskIsOk = diskIsOk && CheckDiskStatus(res.GetStatusFlags());
}
DiskIsFull = !diskIsOk;
- if (DiskIsFull) {
- LogAndCollectError(NKikimrServices::PERSQUEUE, "disk is full", ctx);
- }
+ if (DiskIsFull) {
+ LogAndCollectError(NKikimrServices::PERSQUEUE, "disk is full", ctx);
+ }
InitState = WaitMetaRead;
RequestMetaRead(ctx, Tablet, Partition);
@@ -1399,9 +1399,9 @@ void TPartition::HandleInfoRangeRead(const NKikimrClient::TKeyValueResponse::TRe
} else if ((*key)[TKeyPrefix::MarkPosition()] == TKeyPrefix::MarkProtoSourceId) {
SourceIdStorage.LoadSourceIdInfo(*key, pair.GetValue(), ctx.Now());
} else if ((*key)[TKeyPrefix::MarkPosition()] == TKeyPrefix::MarkUser) {
- UsersInfoStorage.Parse(*key, pair.GetValue(), ctx);
+ UsersInfoStorage.Parse(*key, pair.GetValue(), ctx);
} else if ((*key)[TKeyPrefix::MarkPosition()] == TKeyPrefix::MarkUserDeprecated) {
- UsersInfoStorage.ParseDeprecated(*key, pair.GetValue(), ctx);
+ UsersInfoStorage.ParseDeprecated(*key, pair.GetValue(), ctx);
}
}
//make next step
@@ -1453,7 +1453,7 @@ void TPartition::FillBlobsMetaData(const NKikimrClient::TKeyValueResponse::TRead
LOG_DEBUG_S(ctx, NKikimrServices::PERSQUEUE, "Got data topic " << TopicName << " partition " << k.GetPartition() << " offset " << k.GetOffset()
<< " count " << k.GetCount() << " size " << pair.GetValueSize() << " so " << StartOffset << " eo " << EndOffset << " " << pair.GetKey());
- DataKeysBody.push_back({k, pair.GetValueSize(), TInstant::Seconds(pair.GetCreationUnixTime()), DataKeysBody.empty() ? 0 : DataKeysBody.back().CumulativeSize + DataKeysBody.back().Size});
+ DataKeysBody.push_back({k, pair.GetValueSize(), TInstant::Seconds(pair.GetCreationUnixTime()), DataKeysBody.empty() ? 0 : DataKeysBody.back().CumulativeSize + DataKeysBody.back().Size});
}
Y_VERIFY(EndOffset >= StartOffset);
@@ -1682,9 +1682,9 @@ void TPartition::InitComplete(const TActorContext& ctx) {
InitUserInfoForImportantClients(ctx);
- for (auto& userInfoPair : UsersInfoStorage.GetAll()) {
- Y_VERIFY(userInfoPair.second.Offset >= 0);
- ReadTimestampForOffset(userInfoPair.first, userInfoPair.second, ctx);
+ for (auto& userInfoPair : UsersInfoStorage.GetAll()) {
+ Y_VERIFY(userInfoPair.second.Offset >= 0);
+ ReadTimestampForOffset(userInfoPair.first, userInfoPair.second, ctx);
}
PartitionLabeledCounters.GetCounters()[METRIC_INIT_TIME] = InitDuration.MilliSeconds();
@@ -1695,17 +1695,17 @@ void TPartition::InitComplete(const TActorContext& ctx) {
UpdateUserInfoEndOffset(ctx.Now());
ScheduleUpdateAvailableSize(ctx);
-
+
if (Config.GetPartitionConfig().HasMirrorFrom()) {
- CreateMirrorerActor();
- }
+ CreateMirrorerActor();
+ }
}
void TPartition::UpdateUserInfoEndOffset(const TInstant& now) {
- for (auto& userInfo : UsersInfoStorage.GetAll()) {
- userInfo.second.EndOffset = (i64)EndOffset;
- userInfo.second.UpdateReadingTimeAndState(now);
+ for (auto& userInfo : UsersInfoStorage.GetAll()) {
+ userInfo.second.EndOffset = (i64)EndOffset;
+ userInfo.second.UpdateReadingTimeAndState(now);
}
}
@@ -1755,27 +1755,27 @@ THashMap<TString, NKikimr::NPQ::TOwnerInfo>::iterator TPartition::DropOwner(THas
void TPartition::InitUserInfoForImportantClients(const TActorContext& ctx) {
TSet<TString> important;
for (const auto& importantUser : Config.GetPartitionConfig().GetImportantClientId()) {
- important.insert(importantUser);
- TUserInfo* userInfo = UsersInfoStorage.GetIfExists(importantUser);
- if (userInfo && !userInfo->Important) {
- ctx.Send(Tablet, new TEvPQ::TEvPartitionLabeledCountersDrop(Partition, userInfo->LabeledCounters.GetGroup()));
- userInfo->SetImportant(true);
+ important.insert(importantUser);
+ TUserInfo* userInfo = UsersInfoStorage.GetIfExists(importantUser);
+ if (userInfo && !userInfo->Important) {
+ ctx.Send(Tablet, new TEvPQ::TEvPartitionLabeledCountersDrop(Partition, userInfo->LabeledCounters.GetGroup()));
+ userInfo->SetImportant(true);
continue;
}
- if (!userInfo) {
+ if (!userInfo) {
userInfo = &UsersInfoStorage.Create(ctx, importantUser, 0, true, "", 0, 0, 0, 0, TInstant::Zero());
- }
- if (userInfo->Offset < (i64)StartOffset)
- userInfo->Offset = StartOffset;
- ReadTimestampForOffset(importantUser, *userInfo, ctx);
- }
- for (auto& userInfoPair : UsersInfoStorage.GetAll()) {
- if (!important.contains(userInfoPair.first) && userInfoPair.second.Important) {
- ctx.Send(
- Tablet,
- new TEvPQ::TEvPartitionLabeledCountersDrop(Partition, userInfoPair.second.LabeledCounters.GetGroup())
- );
- userInfoPair.second.SetImportant(false);
+ }
+ if (userInfo->Offset < (i64)StartOffset)
+ userInfo->Offset = StartOffset;
+ ReadTimestampForOffset(importantUser, *userInfo, ctx);
+ }
+ for (auto& userInfoPair : UsersInfoStorage.GetAll()) {
+ if (!important.contains(userInfoPair.first) && userInfoPair.second.Important) {
+ ctx.Send(
+ Tablet,
+ new TEvPQ::TEvPartitionLabeledCountersDrop(Partition, userInfoPair.second.LabeledCounters.GetGroup())
+ );
+ userInfoPair.second.SetImportant(false);
}
}
}
@@ -1798,27 +1798,27 @@ void TPartition::Handle(TEvPQ::TEvChangeConfig::TPtr& ev, const TActorContext& c
}
for (const auto& readQuota : Config.GetPartitionConfig().GetReadQuota()) {
- auto& userInfo = UsersInfoStorage.GetOrCreate(readQuota.GetClientId(), ctx);
- userInfo.ReadQuota.UpdateConfig(readQuota.GetBurstSize(), readQuota.GetSpeedInBytesPerSecond());
+ auto& userInfo = UsersInfoStorage.GetOrCreate(readQuota.GetClientId(), ctx);
+ userInfo.ReadQuota.UpdateConfig(readQuota.GetBurstSize(), readQuota.GetSpeedInBytesPerSecond());
}
if (CurrentStateFunc() != &TThis::StateInit) {
InitUserInfoForImportantClients(ctx);
FillReadFromTimestamps(Config, ctx);
}
-
+
if (Config.GetPartitionConfig().HasMirrorFrom()) {
- if (Mirrorer) {
- ctx.Send(ev->Forward(Mirrorer->Actor));
- } else {
- CreateMirrorerActor();
- }
- } else {
- if (Mirrorer) {
- ctx.Send(Mirrorer->Actor, new TEvents::TEvPoisonPill());
- Mirrorer.Reset();
- }
- }
+ if (Mirrorer) {
+ ctx.Send(ev->Forward(Mirrorer->Actor));
+ } else {
+ CreateMirrorerActor();
+ }
+ } else {
+ if (Mirrorer) {
+ ctx.Send(Mirrorer->Actor, new TEvents::TEvPoisonPill());
+ Mirrorer.Reset();
+ }
+ }
}
@@ -1924,17 +1924,17 @@ void TPartition::Handle(TEvPQ::TEvPartitionOffsets::TPtr& ev, const TActorContex
result.SetWriteTimestampEstimateMS(WriteTimestampEstimate.MilliSeconds());
if (!ev->Get()->ClientId.empty()) {
- TUserInfo* userInfo = UsersInfoStorage.GetIfExists(ev->Get()->ClientId);
- if (userInfo) {
- i64 offset = Max<i64>(userInfo->Offset, 0);
- result.SetClientOffset(userInfo->Offset);
- TInstant tmp = userInfo->GetWriteTimestamp() ? userInfo->GetWriteTimestamp() : GetWriteTimeEstimate(offset);
- result.SetWriteTimestampMS(tmp.MilliSeconds());
- result.SetCreateTimestampMS(userInfo->GetCreateTimestamp().MilliSeconds());
- result.SetClientReadOffset(userInfo->GetReadOffset());
- tmp = userInfo->GetReadWriteTimestamp() ? userInfo->GetReadWriteTimestamp() : GetWriteTimeEstimate(userInfo->GetReadOffset());
- result.SetReadWriteTimestampMS(tmp.MilliSeconds());
- result.SetReadCreateTimestampMS(userInfo->GetReadCreateTimestamp().MilliSeconds());
+ TUserInfo* userInfo = UsersInfoStorage.GetIfExists(ev->Get()->ClientId);
+ if (userInfo) {
+ i64 offset = Max<i64>(userInfo->Offset, 0);
+ result.SetClientOffset(userInfo->Offset);
+ TInstant tmp = userInfo->GetWriteTimestamp() ? userInfo->GetWriteTimestamp() : GetWriteTimeEstimate(offset);
+ result.SetWriteTimestampMS(tmp.MilliSeconds());
+ result.SetCreateTimestampMS(userInfo->GetCreateTimestamp().MilliSeconds());
+ result.SetClientReadOffset(userInfo->GetReadOffset());
+ tmp = userInfo->GetReadWriteTimestamp() ? userInfo->GetReadWriteTimestamp() : GetWriteTimeEstimate(userInfo->GetReadOffset());
+ result.SetReadWriteTimestampMS(tmp.MilliSeconds());
+ result.SetReadCreateTimestampMS(userInfo->GetReadCreateTimestamp().MilliSeconds());
}
}
ctx.Send(ev->Get()->Sender, new TEvPQ::TEvPartitionOffsetsResponse(result));
@@ -1979,43 +1979,43 @@ void TPartition::Handle(TEvPQ::TEvPartitionStatus::TPtr& ev, const TActorContext
result.SetAvgQuotaSpeedPerDay(AvgQuotaBytes[3].GetValue());
result.SetSourceIdCount(SourceIdStorage.GetInMemorySourceIds().size());
- result.SetSourceIdRetentionPeriodSec((ctx.Now() - SourceIdStorage.MinAvailableTimestamp(ctx.Now())).Seconds());
+ result.SetSourceIdRetentionPeriodSec((ctx.Now() - SourceIdStorage.MinAvailableTimestamp(ctx.Now())).Seconds());
result.SetWriteBytesQuota(WriteQuota.GetTotalSpeed());
TVector<ui64> resSpeed;
resSpeed.resize(4);
ui64 maxQuota = 0;
- for (auto& userInfoPair : UsersInfoStorage.GetAll()) {
- auto& userInfo = userInfoPair.second;
- if (ev->Get()->ClientId.empty() || ev->Get()->ClientId == userInfo.User) {
- Y_VERIFY(userInfo.AvgReadBytes.size() == 4);
+ for (auto& userInfoPair : UsersInfoStorage.GetAll()) {
+ auto& userInfo = userInfoPair.second;
+ if (ev->Get()->ClientId.empty() || ev->Get()->ClientId == userInfo.User) {
+ Y_VERIFY(userInfo.AvgReadBytes.size() == 4);
for (ui32 i = 0; i < 4; ++i) {
- resSpeed[i] += userInfo.AvgReadBytes[i].GetValue();
+ resSpeed[i] += userInfo.AvgReadBytes[i].GetValue();
}
- maxQuota += userInfo.ReadQuota.GetTotalSpeed();
+ maxQuota += userInfo.ReadQuota.GetTotalSpeed();
}
- if (ev->Get()->ClientId == userInfo.User) { //fill lags
+ if (ev->Get()->ClientId == userInfo.User) { //fill lags
NKikimrPQ::TClientInfo* clientInfo = result.MutableLagsInfo();
- clientInfo->SetClientId(userInfo.User);
+ clientInfo->SetClientId(userInfo.User);
auto write = clientInfo->MutableWritePosition();
write->SetOffset(userInfo.Offset);
userInfo.EndOffset = EndOffset;
- write->SetWriteTimestamp((userInfo.GetWriteTimestamp() ? userInfo.GetWriteTimestamp() : GetWriteTimeEstimate(userInfo.Offset)).MilliSeconds());
- write->SetCreateTimestamp(userInfo.GetCreateTimestamp().MilliSeconds());
+ write->SetWriteTimestamp((userInfo.GetWriteTimestamp() ? userInfo.GetWriteTimestamp() : GetWriteTimeEstimate(userInfo.Offset)).MilliSeconds());
+ write->SetCreateTimestamp(userInfo.GetCreateTimestamp().MilliSeconds());
auto read = clientInfo->MutableReadPosition();
read->SetOffset(userInfo.GetReadOffset());
- read->SetWriteTimestamp((userInfo.GetReadWriteTimestamp() ? userInfo.GetReadWriteTimestamp() : GetWriteTimeEstimate(userInfo.GetReadOffset())).MilliSeconds());
- read->SetCreateTimestamp(userInfo.GetReadCreateTimestamp().MilliSeconds());
+ read->SetWriteTimestamp((userInfo.GetReadWriteTimestamp() ? userInfo.GetReadWriteTimestamp() : GetWriteTimeEstimate(userInfo.GetReadOffset())).MilliSeconds());
+ read->SetCreateTimestamp(userInfo.GetReadCreateTimestamp().MilliSeconds());
write->SetSize(GetSizeLag(userInfo.Offset));
read->SetSize(GetSizeLag(userInfo.GetReadOffset()));
clientInfo->SetReadLagMs(userInfo.GetReadOffset() < (i64)EndOffset
- ? (userInfo.GetReadTimestamp() - TInstant::MilliSeconds(read->GetWriteTimestamp())).MilliSeconds()
+ ? (userInfo.GetReadTimestamp() - TInstant::MilliSeconds(read->GetWriteTimestamp())).MilliSeconds()
: 0);
- clientInfo->SetLastReadTimestampMs(userInfo.GetReadTimestamp().MilliSeconds());
+ clientInfo->SetLastReadTimestampMs(userInfo.GetReadTimestamp().MilliSeconds());
clientInfo->SetWriteLagMs(userInfo.GetWriteLagMs());
- ui64 totalLag = clientInfo->GetReadLagMs() + userInfo.GetWriteLagMs() + (ctx.Now() - userInfo.GetReadTimestamp()).MilliSeconds();
+ ui64 totalLag = clientInfo->GetReadLagMs() + userInfo.GetWriteLagMs() + (ctx.Now() - userInfo.GetReadTimestamp()).MilliSeconds();
clientInfo->SetTotalLagMs(totalLag);
}
}
@@ -2033,8 +2033,8 @@ void TPartition::Handle(TEvPQ::TEvPartitionStatus::TPtr& ev, const TActorContext
result.SetLastWriteTimestampMs(WriteTimestamp.MilliSeconds());
result.SetWriteLagMs(WriteLagMs.GetValue());
- *result.MutableErrors() = {Errors.begin(), Errors.end()};
-
+ *result.MutableErrors() = {Errors.begin(), Errors.end()};
+
ctx.Send(ev->Get()->Sender, new TEvPQ::TEvPartitionStatusResponse(result));
}
@@ -2055,57 +2055,57 @@ void TPartition::Handle(TEvPQ::TEvGetPartitionClientInfo::TPtr& ev, const TActor
result.SetStartOffset(StartOffset);
result.SetEndOffset(EndOffset);
result.SetResponseTimestamp(ctx.Now().MilliSeconds());
- for (auto& pr : UsersInfoStorage.GetAll()) {
+ for (auto& pr : UsersInfoStorage.GetAll()) {
TUserInfo& userInfo(pr.second);
NKikimrPQ::TClientInfo& clientInfo = *result.AddClientInfo();
clientInfo.SetClientId(pr.first);
auto& write = *clientInfo.MutableWritePosition();
write.SetOffset(userInfo.Offset);
userInfo.EndOffset = EndOffset;
- write.SetWriteTimestamp((userInfo.GetWriteTimestamp() ? userInfo.GetWriteTimestamp() : GetWriteTimeEstimate(userInfo.Offset)).MilliSeconds());
- write.SetCreateTimestamp(userInfo.GetCreateTimestamp().MilliSeconds());
+ write.SetWriteTimestamp((userInfo.GetWriteTimestamp() ? userInfo.GetWriteTimestamp() : GetWriteTimeEstimate(userInfo.Offset)).MilliSeconds());
+ write.SetCreateTimestamp(userInfo.GetCreateTimestamp().MilliSeconds());
auto& read = *clientInfo.MutableReadPosition();
read.SetOffset(userInfo.GetReadOffset());
- read.SetWriteTimestamp((userInfo.GetReadWriteTimestamp() ? userInfo.GetReadWriteTimestamp() : GetWriteTimeEstimate(userInfo.GetReadOffset())).MilliSeconds());
- read.SetCreateTimestamp(userInfo.GetReadCreateTimestamp().MilliSeconds());
+ read.SetWriteTimestamp((userInfo.GetReadWriteTimestamp() ? userInfo.GetReadWriteTimestamp() : GetWriteTimeEstimate(userInfo.GetReadOffset())).MilliSeconds());
+ read.SetCreateTimestamp(userInfo.GetReadCreateTimestamp().MilliSeconds());
write.SetSize(GetSizeLag(userInfo.Offset));
read.SetSize(GetSizeLag(userInfo.GetReadOffset()));
}
ctx.Send(ev->Get()->Sender, response.Release(), 0, ev->Cookie);
}
-void TPartition::Handle(TEvPersQueue::TEvReportPartitionError::TPtr& ev, const TActorContext& ctx) {
- LogAndCollectError(ev->Get()->Record, ctx);
-}
-
-void TPartition::LogAndCollectError(const NKikimrPQ::TStatusResponse::TErrorMessage& error, const TActorContext& ctx) {
- if (Errors.size() == MAX_ERRORS_COUNT_TO_STORE) {
- Errors.pop_front();
- }
- Errors.push_back(error);
- LOG_ERROR_S(ctx, error.GetService(), error.GetMessage());
-}
-
-void TPartition::LogAndCollectError(NKikimrServices::EServiceKikimr service, const TString& msg, const TActorContext& ctx) {
- NKikimrPQ::TStatusResponse::TErrorMessage error;
- error.SetTimestamp(ctx.Now().Seconds());
- error.SetService(service);
- error.SetMessage(TStringBuilder() << "topic '" << TopicName << "' partition " << Partition << " got error: " << msg);
- LogAndCollectError(error, ctx);
-}
-
-std::pair<TInstant, TInstant> TPartition::GetTime(const TUserInfo& userInfo, ui64 offset) const
+void TPartition::Handle(TEvPersQueue::TEvReportPartitionError::TPtr& ev, const TActorContext& ctx) {
+ LogAndCollectError(ev->Get()->Record, ctx);
+}
+
+void TPartition::LogAndCollectError(const NKikimrPQ::TStatusResponse::TErrorMessage& error, const TActorContext& ctx) {
+ if (Errors.size() == MAX_ERRORS_COUNT_TO_STORE) {
+ Errors.pop_front();
+ }
+ Errors.push_back(error);
+ LOG_ERROR_S(ctx, error.GetService(), error.GetMessage());
+}
+
+void TPartition::LogAndCollectError(NKikimrServices::EServiceKikimr service, const TString& msg, const TActorContext& ctx) {
+ NKikimrPQ::TStatusResponse::TErrorMessage error;
+ error.SetTimestamp(ctx.Now().Seconds());
+ error.SetService(service);
+ error.SetMessage(TStringBuilder() << "topic '" << TopicName << "' partition " << Partition << " got error: " << msg);
+ LogAndCollectError(error, ctx);
+}
+
+std::pair<TInstant, TInstant> TPartition::GetTime(const TUserInfo& userInfo, ui64 offset) const
{
- TInstant wtime = userInfo.WriteTimestamp > TInstant::Zero() ? userInfo.WriteTimestamp : GetWriteTimeEstimate(offset);
+ TInstant wtime = userInfo.WriteTimestamp > TInstant::Zero() ? userInfo.WriteTimestamp : GetWriteTimeEstimate(offset);
return std::make_pair(wtime, userInfo.CreateTimestamp);
}
//zero means no such record
-TInstant TPartition::GetWriteTimeEstimate(ui64 offset) const
+TInstant TPartition::GetWriteTimeEstimate(ui64 offset) const
{
if (offset < StartOffset) offset = StartOffset;
if (offset >= EndOffset)
- return TInstant::Zero();
+ return TInstant::Zero();
const std::deque<TDataKey>& container = (offset < Head.Offset || offset == Head.Offset && Head.PartNo > 0) ? DataKeysBody : HeadKeys;
Y_VERIFY(!container.empty());
auto it = std::upper_bound(container.begin(), container.end(), offset,
@@ -2116,17 +2116,17 @@ TInstant TPartition::GetWriteTimeEstimate(ui64 offset) const
--it;
if (it != container.begin())
--it;
- return it->Timestamp;
+ return it->Timestamp;
}
void TPartition::Handle(TEvPQ::TEvGetClientOffset::TPtr& ev, const TActorContext& ctx) {
- auto& userInfo = UsersInfoStorage.GetOrCreate(ev->Get()->ClientId, ctx);
- Y_VERIFY(userInfo.Offset >= -1, "Unexpected Offset: %" PRIi64, userInfo.Offset);
- ui64 offset = Max<i64>(userInfo.Offset, 0);
- auto ts = GetTime(userInfo, offset);
+ auto& userInfo = UsersInfoStorage.GetOrCreate(ev->Get()->ClientId, ctx);
+ Y_VERIFY(userInfo.Offset >= -1, "Unexpected Offset: %" PRIi64, userInfo.Offset);
+ ui64 offset = Max<i64>(userInfo.Offset, 0);
+ auto ts = GetTime(userInfo, offset);
Counters.Cumulative()[COUNTER_PQ_GET_CLIENT_OFFSET_OK].Increment(1);
- ReplyGetClientOffsetOk(ctx, ev->Get()->Cookie, userInfo.Offset, ts.first, ts.second);
+ ReplyGetClientOffsetOk(ctx, ev->Get()->Cookie, userInfo.Offset, ts.first, ts.second);
}
void TPartition::Handle(TEvPQ::TEvUpdateWriteTimestamp::TPtr& ev, const TActorContext& ctx) {
@@ -2143,12 +2143,12 @@ void TPartition::Handle(TEvPQ::TEvUpdateWriteTimestamp::TPtr& ev, const TActorCo
void TPartition::Handle(TEvPQ::TEvSetClientInfo::TPtr& ev, const TActorContext& ctx) {
const TString& user = ev->Get()->ClientId;
- auto& userInfo = UsersInfoStorage.GetOrCreate(user, ctx);
+ auto& userInfo = UsersInfoStorage.GetOrCreate(user, ctx);
- if (userInfo.UserActs.size() > MAX_USER_ACTS) {
+ if (userInfo.UserActs.size() > MAX_USER_ACTS) {
Counters.Cumulative()[COUNTER_PQ_SET_CLIENT_OFFSET_ERROR].Increment(1);
ReplyError(ctx, ev->Get()->Cookie, NPersQueue::NErrorCode::OVERLOAD,
- TStringBuilder() << "too big inflight: " << userInfo.UserActs.size());
+ TStringBuilder() << "too big inflight: " << userInfo.UserActs.size());
return;
}
@@ -2156,7 +2156,7 @@ void TPartition::Handle(TEvPQ::TEvSetClientInfo::TPtr& ev, const TActorContext&
const ui64& offset = ev->Get()->Offset;
Y_VERIFY(offset <= (ui64)Max<i64>(), "Unexpected Offset: %" PRIu64, offset);
- userInfo.UserActs.push_back(ev->Release());
+ userInfo.UserActs.push_back(ev->Release());
ProcessUserActs(userInfo, ctx);
}
@@ -2208,33 +2208,33 @@ void TPartition::Handle(TEvPQ::TEvBlobResponse::TPtr& ev, const TActorContext& c
auto it = ReadInfo.find(cookie);
Y_VERIFY(it != ReadInfo.end());
- TReadInfo info = std::move(it->second);
- ReadInfo.erase(it);
+ TReadInfo info = std::move(it->second);
+ ReadInfo.erase(it);
//make readinfo class
- TReadAnswer answer(info.FormAnswer(
- ctx, *ev->Get(), EndOffset, Partition, &UsersInfoStorage.GetOrCreate(info.User, ctx),
- info.Destination, GetSizeLag(info.Offset)
- ));
+ TReadAnswer answer(info.FormAnswer(
+ ctx, *ev->Get(), EndOffset, Partition, &UsersInfoStorage.GetOrCreate(info.User, ctx),
+ info.Destination, GetSizeLag(info.Offset)
+ ));
if (HasError(*ev->Get())) {
- if (info.IsSubscription) {
+ if (info.IsSubscription) {
Counters.Cumulative()[COUNTER_PQ_READ_SUBSCRIPTION_ERROR].Increment(1);
}
Counters.Cumulative()[COUNTER_PQ_READ_ERROR].Increment(1);
- Counters.Percentile()[COUNTER_LATENCY_PQ_READ_ERROR].IncrementFor((ctx.Now() - info.Timestamp).MilliSeconds());
+ Counters.Percentile()[COUNTER_LATENCY_PQ_READ_ERROR].IncrementFor((ctx.Now() - info.Timestamp).MilliSeconds());
} else {
- if (info.IsSubscription) {
+ if (info.IsSubscription) {
Counters.Cumulative()[COUNTER_PQ_READ_SUBSCRIPTION_OK].Increment(1);
}
- const auto& resp = dynamic_cast<TEvPQ::TEvProxyResponse*>(answer.Event.Get())->Response;
+ const auto& resp = dynamic_cast<TEvPQ::TEvProxyResponse*>(answer.Event.Get())->Response;
Counters.Cumulative()[COUNTER_PQ_READ_OK].Increment(1);
- Counters.Percentile()[COUNTER_LATENCY_PQ_READ_OK].IncrementFor((ctx.Now() - info.Timestamp).MilliSeconds());
+ Counters.Percentile()[COUNTER_LATENCY_PQ_READ_OK].IncrementFor((ctx.Now() - info.Timestamp).MilliSeconds());
Counters.Cumulative()[COUNTER_PQ_READ_BYTES].Increment(resp.ByteSize());
}
- ctx.Send(info.Destination != 0 ? Tablet : ctx.SelfID, answer.Event.Release());
+ ctx.Send(info.Destination != 0 ? Tablet : ctx.SelfID, answer.Event.Release());
ReportLabeledCounters(ctx);
- OnReadRequestFinished(std::move(info), answer.Size);
+ OnReadRequestFinished(std::move(info), answer.Size);
}
@@ -2246,8 +2246,8 @@ static void AddResultBlob(T* read, const TClientBlob& blob, ui64 offset)
cc->SetData(blob.Data);
cc->SetSourceId(blob.SourceId);
cc->SetSeqNo(blob.SeqNo);
- cc->SetWriteTimestampMS(blob.WriteTimestamp.MilliSeconds());
- cc->SetCreateTimestampMS(blob.CreateTimestamp.MilliSeconds());
+ cc->SetWriteTimestampMS(blob.WriteTimestamp.MilliSeconds());
+ cc->SetCreateTimestampMS(blob.CreateTimestamp.MilliSeconds());
cc->SetUncompressedSize(blob.UncompressedSize);
cc->SetPartitionKey(blob.PartitionKey);
cc->SetExplicitHash(blob.ExplicitHashKey);
@@ -2280,31 +2280,31 @@ static void AddResultDebugInfo(const TEvPQ::TEvBlobResponse* response, T* readRe
readResult->SetBlobsFromDisk(diskBlobs);
}
-TReadAnswer TReadInfo::FormAnswer(
- const TActorContext& ctx,
- const TEvPQ::TEvBlobResponse& blobResponse,
- const ui64 endOffset,
- const ui32 partition,
- TUserInfo* userInfo,
- const ui64 cookie,
- const ui64 sizeLag
-) {
+TReadAnswer TReadInfo::FormAnswer(
+ const TActorContext& ctx,
+ const TEvPQ::TEvBlobResponse& blobResponse,
+ const ui64 endOffset,
+ const ui32 partition,
+ TUserInfo* userInfo,
+ const ui64 cookie,
+ const ui64 sizeLag
+) {
Y_UNUSED(partition);
THolder<TEvPQ::TEvProxyResponse> answer = MakeHolder<TEvPQ::TEvProxyResponse>(cookie);
NKikimrClient::TResponse& res = answer->Response;
const TEvPQ::TEvBlobResponse* response = &blobResponse;
if (HasError(blobResponse)) {
- return TReadAnswer{
- blobResponse.Error.ErrorStr.size(),
+ return TReadAnswer{
+ blobResponse.Error.ErrorStr.size(),
MakeHolder<TEvPQ::TEvError>(blobResponse.Error.ErrorCode, blobResponse.Error.ErrorStr, cookie)
- };
+ };
}
res.SetStatus(NMsgBusProxy::MSTATUS_OK);
res.SetErrorCode(NPersQueue::NErrorCode::OK);
auto readResult = res.MutablePartitionResponse()->MutableCmdReadResult();
- readResult->SetWaitQuotaTimeMs(WaitQuotaTime.MilliSeconds());
+ readResult->SetWaitQuotaTimeMs(WaitQuotaTime.MilliSeconds());
readResult->SetMaxOffset(endOffset);
readResult->SetRealReadOffset(Offset);
Y_VERIFY(endOffset <= (ui64)Max<i64>(), "Max offset is too big: %" PRIu64, endOffset);
@@ -2332,8 +2332,8 @@ TReadAnswer TReadInfo::FormAnswer(
if (blobValue.empty()) { // this is ok. Means that someone requested too much data
LOG_DEBUG(ctx, NKikimrServices::PERSQUEUE, "Not full answer here!");
- ui64 answerSize = answer->Response.ByteSize();
- if (userInfo && Destination != 0) {
+ ui64 answerSize = answer->Response.ByteSize();
+ if (userInfo && Destination != 0) {
userInfo->ReadDone(ctx, ctx.Now(), answerSize, cnt, ClientDC);
}
readResult->SetSizeLag(sizeLag - size);
@@ -2393,7 +2393,7 @@ TReadAnswer TReadInfo::FormAnswer(
if (res.IsLastPart()) {
++cnt;
}
- }
+ }
if (res.IsLastPart()) {
PartNo = 0;
@@ -2426,12 +2426,12 @@ TReadAnswer TReadInfo::FormAnswer(
readResult->SetBlobsCachedSize(readResult->GetBlobsCachedSize() + writeBlob.GetBlobSize());
size += writeBlob.GetBlobSize();
- if (userInfo) {
- userInfo->AddTimestampToCache(
- Offset, writeBlob.WriteTimestamp, writeBlob.CreateTimestamp,
- Destination != 0, ctx.Now()
- );
- }
+ if (userInfo) {
+ userInfo->AddTimestampToCache(
+ Offset, writeBlob.WriteTimestamp, writeBlob.CreateTimestamp,
+ Destination != 0, ctx.Now()
+ );
+ }
AddResultBlob(readResult, writeBlob, Offset);
if (writeBlob.IsLastPart()) {
++Offset;
@@ -2442,8 +2442,8 @@ TReadAnswer TReadInfo::FormAnswer(
Y_VERIFY(pcnt <= Count && psize <= Size);
Y_VERIFY(pcnt <= cnt && psize <= size);
Y_VERIFY(Offset <= (ui64)Max<i64>(), "Offset is too big: %" PRIu64, Offset);
- ui64 answerSize = answer->Response.ByteSize();
- if (userInfo && Destination != 0) {
+ ui64 answerSize = answer->Response.ByteSize();
+ if (userInfo && Destination != 0) {
userInfo->ReadDone(ctx, ctx.Now(), answerSize, cnt, ClientDC);
}
readResult->SetSizeLag(sizeLag - size);
@@ -2458,17 +2458,17 @@ void TPartition::HandleOnIdle(TEvPQ::TEvWrite::TPtr& ev, const TActorContext& ct
void TPartition::Handle(TEvPQ::TEvReadTimeout::TPtr& ev, const TActorContext& ctx) {
- auto res = Subscriber.OnTimeout(ev);
+ auto res = Subscriber.OnTimeout(ev);
if (!res)
return;
- TReadAnswer answer(res->FormAnswer(ctx, res->Offset, Partition, nullptr, res->Destination, 0));
- ctx.Send(Tablet, answer.Event.Release());
- LOG_DEBUG_S(ctx, NKikimrServices::PERSQUEUE, " waiting read cookie " << ev->Get()->Cookie
- << " partition " << Partition << " read timeout for " << res->User << " offset " << res->Offset);
- auto& userInfo = UsersInfoStorage.GetOrCreate(res->User, ctx);
-
- userInfo.ForgetSubscription(ctx.Now());
- OnReadRequestFinished(std::move(res.GetRef()), answer.Size);
+ TReadAnswer answer(res->FormAnswer(ctx, res->Offset, Partition, nullptr, res->Destination, 0));
+ ctx.Send(Tablet, answer.Event.Release());
+ LOG_DEBUG_S(ctx, NKikimrServices::PERSQUEUE, " waiting read cookie " << ev->Get()->Cookie
+ << " partition " << Partition << " read timeout for " << res->User << " offset " << res->Offset);
+ auto& userInfo = UsersInfoStorage.GetOrCreate(res->User, ctx);
+
+ userInfo.ForgetSubscription(ctx.Now());
+ OnReadRequestFinished(std::move(res.GetRef()), answer.Size);
}
@@ -2605,10 +2605,10 @@ void TPartition::Handle(TEvPQ::TEvRead::TPtr& ev, const TActorContext& ctx) {
Y_VERIFY(read->Offset <= EndOffset);
- auto& userInfo = UsersInfoStorage.GetOrCreate(user, ctx);
+ auto& userInfo = UsersInfoStorage.GetOrCreate(user, ctx);
if (!read->SessionId.empty()) {
- if (userInfo.Session != read->SessionId) {
+ if (userInfo.Session != read->SessionId) {
Counters.Cumulative()[COUNTER_PQ_READ_ERROR_NO_SESSION].Increment(1);
Counters.Percentile()[COUNTER_LATENCY_PQ_READ_ERROR].IncrementFor(0);
ReplyError(ctx, read->Cookie, NPersQueue::NErrorCode::READ_ERROR_NO_SESSION,
@@ -2617,32 +2617,32 @@ void TPartition::Handle(TEvPQ::TEvRead::TPtr& ev, const TActorContext& ctx) {
}
}
- if (userInfo.ReadSpeedLimiter) {
- Send(userInfo.ReadSpeedLimiter->Actor, new NReadSpeedLimiterEvents::TEvRequest(ev.Release()));
- } else {
- DoRead(ev.Release(), TDuration::Zero(), ctx);
- }
-}
-
-void TPartition::Handle(NReadSpeedLimiterEvents::TEvResponse::TPtr& ev, const TActorContext& ctx) {
- DoRead(ev->Get()->ReadRequest.Release(), ev->Get()->WaitTime, ctx);
-}
-
-void TPartition::DoRead(TEvPQ::TEvRead::TPtr ev, TDuration waitQuotaTime, const TActorContext& ctx) {
- auto read = ev->Get();
- const TString& user = read->ClientId;
- auto& userInfo = UsersInfoStorage.GetOrCreate(user, ctx);
-
+ if (userInfo.ReadSpeedLimiter) {
+ Send(userInfo.ReadSpeedLimiter->Actor, new NReadSpeedLimiterEvents::TEvRequest(ev.Release()));
+ } else {
+ DoRead(ev.Release(), TDuration::Zero(), ctx);
+ }
+}
+
+void TPartition::Handle(NReadSpeedLimiterEvents::TEvResponse::TPtr& ev, const TActorContext& ctx) {
+ DoRead(ev->Get()->ReadRequest.Release(), ev->Get()->WaitTime, ctx);
+}
+
+void TPartition::DoRead(TEvPQ::TEvRead::TPtr ev, TDuration waitQuotaTime, const TActorContext& ctx) {
+ auto read = ev->Get();
+ const TString& user = read->ClientId;
+ auto& userInfo = UsersInfoStorage.GetOrCreate(user, ctx);
+
ui64 offset = read->Offset;
- if (read->MaxTimeLagMs > 0 || read->ReadTimestampMs > 0 || userInfo.ReadFromTimestamp > TInstant::MilliSeconds(1)) {
- TInstant timestamp = read->MaxTimeLagMs > 0 ? ctx.Now() - TDuration::MilliSeconds(read->MaxTimeLagMs) : TInstant::Zero();
- timestamp = Max(timestamp, TInstant::MilliSeconds(read->ReadTimestampMs));
- timestamp = Max(timestamp, userInfo.ReadFromTimestamp);
+ if (read->MaxTimeLagMs > 0 || read->ReadTimestampMs > 0 || userInfo.ReadFromTimestamp > TInstant::MilliSeconds(1)) {
+ TInstant timestamp = read->MaxTimeLagMs > 0 ? ctx.Now() - TDuration::MilliSeconds(read->MaxTimeLagMs) : TInstant::Zero();
+ timestamp = Max(timestamp, TInstant::MilliSeconds(read->ReadTimestampMs));
+ timestamp = Max(timestamp, userInfo.ReadFromTimestamp);
offset = Max(GetOffsetEstimate(DataKeysBody, timestamp, Min(Head.Offset, EndOffset - 1)), offset);
- userInfo.ReadOffsetRewindSum += offset - read->Offset;
+ userInfo.ReadOffsetRewindSum += offset - read->Offset;
}
- TReadInfo info(user, read->ClientDC, offset, read->PartNo, read->Count, read->Size, read->Cookie, read->ReadTimestampMs, waitQuotaTime);
+ TReadInfo info(user, read->ClientDC, offset, read->PartNo, read->Count, read->Size, read->Cookie, read->ReadTimestampMs, waitQuotaTime);
ui64 cookie = Cookie++;
@@ -2659,8 +2659,8 @@ void TPartition::DoRead(TEvPQ::TEvRead::TPtr ev, TDuration waitQuotaTime, const
read->Timeout = 30000;
}
Subscriber.AddSubscription(std::move(info), read->Timeout, cookie, ctx);
- ++userInfo.Subscriptions;
- userInfo.UpdateReadOffset((i64)offset - 1, userInfo.WriteTimestamp, userInfo.CreateTimestamp, ctx.Now());
+ ++userInfo.Subscriptions;
+ userInfo.UpdateReadOffset((i64)offset - 1, userInfo.WriteTimestamp, userInfo.CreateTimestamp, ctx.Now());
return;
}
@@ -2670,18 +2670,18 @@ void TPartition::DoRead(TEvPQ::TEvRead::TPtr ev, TDuration waitQuotaTime, const
ProcessRead(ctx, std::move(info), cookie, false);
}
-void TPartition::OnReadRequestFinished(TReadInfo&& info, ui64 answerSize) {
- auto userInfo = UsersInfoStorage.GetIfExists(info.User);
- Y_VERIFY(userInfo);
+void TPartition::OnReadRequestFinished(TReadInfo&& info, ui64 answerSize) {
+ auto userInfo = UsersInfoStorage.GetIfExists(info.User);
+ Y_VERIFY(userInfo);
+
+ if (userInfo->ReadSpeedLimiter) {
+ Send(
+ userInfo->ReadSpeedLimiter->Actor,
+ new NReadSpeedLimiterEvents::TEvConsumed(answerSize, info.Destination)
+ );
+ }
+}
- if (userInfo->ReadSpeedLimiter) {
- Send(
- userInfo->ReadSpeedLimiter->Actor,
- new NReadSpeedLimiterEvents::TEvConsumed(answerSize, info.Destination)
- );
- }
-}
-
void TPartition::AnswerCurrentWrites(const TActorContext& ctx)
{
ui64 offset = EndOffset;
@@ -2729,11 +2729,11 @@ void TPartition::AnswerCurrentWrites(const TActorContext& ctx)
Counters.Cumulative()[COUNTER_PQ_WRITE_OK].Increment(1);
}
- ReplyWrite(
- ctx, writeResponse.Cookie, s, seqNo, partNo, totalParts,
- already ? maxOffset : offset, CurrentTimestamp, already, maxSeqNo,
- quotedTime, TopicQuotaWaitTimeForCurrentBlob, queueTime, writeTime
- );
+ ReplyWrite(
+ ctx, writeResponse.Cookie, s, seqNo, partNo, totalParts,
+ already ? maxOffset : offset, CurrentTimestamp, already, maxSeqNo,
+ quotedTime, TopicQuotaWaitTimeForCurrentBlob, queueTime, writeTime
+ );
LOG_DEBUG_S(
ctx,
NKikimrServices::PERSQUEUE,
@@ -2797,28 +2797,28 @@ void TPartition::AnswerCurrentWrites(const TActorContext& ctx)
}
-void TPartition::ReadTimestampForOffset(const TString& user, TUserInfo& userInfo, const TActorContext& ctx)
+void TPartition::ReadTimestampForOffset(const TString& user, TUserInfo& userInfo, const TActorContext& ctx)
{
- if (userInfo.ReadScheduled)
+ if (userInfo.ReadScheduled)
return;
- userInfo.ReadScheduled = true;
+ userInfo.ReadScheduled = true;
LOG_DEBUG_S(ctx, NKikimrServices::PERSQUEUE, "Topic '" << TopicName << "' partition " << Partition
- << " user " << user << " readTimeStamp for offset " << userInfo.Offset << " initiated " << " queuesize " << UpdateUserInfoTimestamp.size() << " startOffset " << StartOffset << " ReadingTimestamp " << ReadingTimestamp);
+ << " user " << user << " readTimeStamp for offset " << userInfo.Offset << " initiated " << " queuesize " << UpdateUserInfoTimestamp.size() << " startOffset " << StartOffset << " ReadingTimestamp " << ReadingTimestamp);
if (ReadingTimestamp) {
UpdateUserInfoTimestamp.push_back(user);
return;
}
- if (userInfo.Offset < (i64)StartOffset) {
- userInfo.ReadScheduled = false;
- auto now = ctx.Now();
+ if (userInfo.Offset < (i64)StartOffset) {
+ userInfo.ReadScheduled = false;
+ auto now = ctx.Now();
userInfo.CreateTimestamp = now - TDuration::Seconds(Max(86400, Config.GetPartitionConfig().GetLifetimeSeconds()));
userInfo.WriteTimestamp = now - TDuration::Seconds(Max(86400, Config.GetPartitionConfig().GetLifetimeSeconds()));
- userInfo.ActualTimestamps = true;
- if (userInfo.ReadOffset + 1 < userInfo.Offset) {
- userInfo.ReadOffset = userInfo.Offset - 1;
- userInfo.ReadCreateTimestamp = userInfo.CreateTimestamp;
- userInfo.ReadWriteTimestamp = userInfo.WriteTimestamp;
+ userInfo.ActualTimestamps = true;
+ if (userInfo.ReadOffset + 1 < userInfo.Offset) {
+ userInfo.ReadOffset = userInfo.Offset - 1;
+ userInfo.ReadCreateTimestamp = userInfo.CreateTimestamp;
+ userInfo.ReadWriteTimestamp = userInfo.WriteTimestamp;
}
Counters.Cumulative()[COUNTER_PQ_WRITE_TIMESTAMP_OFFSET_IS_LOST].Increment(1);
@@ -2827,7 +2827,7 @@ void TPartition::ReadTimestampForOffset(const TString& user, TUserInfo& userInfo
}
if (userInfo.Offset >= (i64)EndOffset || StartOffset == EndOffset) {
- userInfo.ReadScheduled = false;
+ userInfo.ReadScheduled = false;
ReportLabeledCounters(ctx);
return;
}
@@ -2836,7 +2836,7 @@ void TPartition::ReadTimestampForOffset(const TString& user, TUserInfo& userInfo
ReadingTimestamp = true;
ReadingForUser = user;
- ReadingForOffset = userInfo.Offset;
+ ReadingForOffset = userInfo.Offset;
ReadingForUserReadRuleGeneration = userInfo.ReadRuleGeneration;
for (const auto& user : UpdateUserInfoTimestamp) {
@@ -2844,7 +2844,7 @@ void TPartition::ReadTimestampForOffset(const TString& user, TUserInfo& userInfo
}
LOG_DEBUG_S(ctx, NKikimrServices::PERSQUEUE, "Topic '" << TopicName << "' partition " << Partition
- << " user " << user << " send read request for offset " << userInfo.Offset << " initiated " << " queuesize " << UpdateUserInfoTimestamp.size() << " startOffset " << StartOffset << " ReadingTimestamp " << ReadingTimestamp);
+ << " user " << user << " send read request for offset " << userInfo.Offset << " initiated " << " queuesize " << UpdateUserInfoTimestamp.size() << " startOffset " << StartOffset << " ReadingTimestamp " << ReadingTimestamp);
THolder<TEvPQ::TEvRead> event = MakeHolder<TEvPQ::TEvRead>(0, userInfo.Offset, 0, 1, "",
@@ -2856,9 +2856,9 @@ void TPartition::ReadTimestampForOffset(const TString& user, TUserInfo& userInfo
void TPartition::ProcessTimestampsForNewData(const ui64 prevEndOffset, const TActorContext& ctx) {
- for (auto& userInfoPair : UsersInfoStorage.GetAll()) {
- if (userInfoPair.second.Offset >= (i64)prevEndOffset && userInfoPair.second.Offset < (i64)EndOffset) {
- ReadTimestampForOffset(userInfoPair.first, userInfoPair.second, ctx);
+ for (auto& userInfoPair : UsersInfoStorage.GetAll()) {
+ if (userInfoPair.second.Offset >= (i64)prevEndOffset && userInfoPair.second.Offset < (i64)EndOffset) {
+ ReadTimestampForOffset(userInfoPair.first, userInfoPair.second, ctx);
}
}
}
@@ -2880,7 +2880,7 @@ void TPartition::Handle(TEvPQ::TEvProxyResponse::TPtr& ev, const TActorContext&
<< " queuesize " << UpdateUserInfoTimestamp.size() << " startOffset " << StartOffset);
if (!userInfo->ActualTimestamps) {
- LOG_INFO_S(
+ LOG_INFO_S(
ctx,
NKikimrServices::PERSQUEUE,
"Reading Timestamp failed for offset " << ReadingForOffset << " ( "<< userInfo->Offset << " ) " << ev->Get()->Response.DebugString()
@@ -2992,7 +2992,7 @@ void TPartition::SyncMemoryStateWithKVState(const TActorContext& ctx)
HeadKeys.pop_back();
}
HeadKeys.push_back(NewHeadKey);
- NewHeadKey = TDataKey{TKey{}, 0, TInstant::Zero(), 0};
+ NewHeadKey = TDataKey{TKey{}, 0, TInstant::Zero(), 0};
}
if (CompactedKeys.empty() && NewHead.PackedSize == 0) { //Nothing writed at all
@@ -3022,7 +3022,7 @@ void TPartition::SyncMemoryStateWithKVState(const TActorContext& ctx)
GapSize += ck.first.GetOffset() - lastOffset;
}
}
- DataKeysBody.push_back({ck.first, ck.second, ctx.Now(), DataKeysBody.empty() ? 0 : DataKeysBody.back().CumulativeSize + DataKeysBody.back().Size});
+ DataKeysBody.push_back({ck.first, ck.second, ctx.Now(), DataKeysBody.empty() ? 0 : DataKeysBody.back().CumulativeSize + DataKeysBody.back().Size});
CompactedKeys.pop_front();
} // head cleared, all data moved to body
@@ -3072,108 +3072,108 @@ void TPartition::ReportLabeledCounters(const TActorContext& ctx)
{
//per client counters
const auto now = ctx.Now();
- for (auto& userInfoPair : UsersInfoStorage.GetAll()) {
- auto& userInfo = userInfoPair.second;
- if (!userInfo.HasReadRule && !userInfo.Important)
+ for (auto& userInfoPair : UsersInfoStorage.GetAll()) {
+ auto& userInfo = userInfoPair.second;
+ if (!userInfo.HasReadRule && !userInfo.Important)
continue;
bool haveChanges = false;
- userInfo.EndOffset = EndOffset;
- userInfo.UpdateReadingTimeAndState(now);
- ui64 ts = userInfo.GetWriteTimestamp().MilliSeconds();
+ userInfo.EndOffset = EndOffset;
+ userInfo.UpdateReadingTimeAndState(now);
+ ui64 ts = userInfo.GetWriteTimestamp().MilliSeconds();
if (ts < MIN_TIMESTAMP_MS) ts = Max<i64>();
- if (userInfo.LabeledCounters.GetCounters()[METRIC_COMMIT_WRITE_TIME].Get() != ts) {
+ if (userInfo.LabeledCounters.GetCounters()[METRIC_COMMIT_WRITE_TIME].Get() != ts) {
haveChanges = true;
- userInfo.LabeledCounters.GetCounters()[METRIC_COMMIT_WRITE_TIME].Set(ts);
+ userInfo.LabeledCounters.GetCounters()[METRIC_COMMIT_WRITE_TIME].Set(ts);
}
- ts = userInfo.GetCreateTimestamp().MilliSeconds();
+ ts = userInfo.GetCreateTimestamp().MilliSeconds();
if (ts < MIN_TIMESTAMP_MS) ts = Max<i64>();
- if (userInfo.LabeledCounters.GetCounters()[METRIC_COMMIT_CREATE_TIME].Get() != ts) {
+ if (userInfo.LabeledCounters.GetCounters()[METRIC_COMMIT_CREATE_TIME].Get() != ts) {
haveChanges = true;
- userInfo.LabeledCounters.GetCounters()[METRIC_COMMIT_CREATE_TIME].Set(ts);
+ userInfo.LabeledCounters.GetCounters()[METRIC_COMMIT_CREATE_TIME].Set(ts);
}
- ts = userInfo.GetReadWriteTimestamp().MilliSeconds();
- if (userInfo.LabeledCounters.GetCounters()[METRIC_READ_WRITE_TIME].Get() != ts) {
+ ts = userInfo.GetReadWriteTimestamp().MilliSeconds();
+ if (userInfo.LabeledCounters.GetCounters()[METRIC_READ_WRITE_TIME].Get() != ts) {
haveChanges = true;
- userInfo.LabeledCounters.GetCounters()[METRIC_READ_WRITE_TIME].Set(ts);
+ userInfo.LabeledCounters.GetCounters()[METRIC_READ_WRITE_TIME].Set(ts);
}
- i64 off = userInfo.GetReadOffset(); //we want to track first not-readed offset
- TInstant wts = userInfo.GetReadWriteTimestamp() ? userInfo.GetReadWriteTimestamp() : GetWriteTimeEstimate(userInfo.GetReadOffset());
- TInstant readTimestamp = userInfo.GetReadTimestamp();
- ui64 readTimeLag = off >= (i64)EndOffset ? 0 : (readTimestamp - wts).MilliSeconds();
- ui64 totalLag = userInfo.GetWriteLagMs() + readTimeLag + (now - readTimestamp).MilliSeconds();
+ i64 off = userInfo.GetReadOffset(); //we want to track first not-readed offset
+ TInstant wts = userInfo.GetReadWriteTimestamp() ? userInfo.GetReadWriteTimestamp() : GetWriteTimeEstimate(userInfo.GetReadOffset());
+ TInstant readTimestamp = userInfo.GetReadTimestamp();
+ ui64 readTimeLag = off >= (i64)EndOffset ? 0 : (readTimestamp - wts).MilliSeconds();
+ ui64 totalLag = userInfo.GetWriteLagMs() + readTimeLag + (now - readTimestamp).MilliSeconds();
- if (userInfo.LabeledCounters.GetCounters()[METRIC_READ_TOTAL_TIME].Get() != totalLag) {
+ if (userInfo.LabeledCounters.GetCounters()[METRIC_READ_TOTAL_TIME].Get() != totalLag) {
haveChanges = true;
- userInfo.LabeledCounters.GetCounters()[METRIC_READ_TOTAL_TIME].Set(totalLag);
+ userInfo.LabeledCounters.GetCounters()[METRIC_READ_TOTAL_TIME].Set(totalLag);
}
- ts = readTimestamp.MilliSeconds();
- if (userInfo.LabeledCounters.GetCounters()[METRIC_LAST_READ_TIME].Get() != ts) {
+ ts = readTimestamp.MilliSeconds();
+ if (userInfo.LabeledCounters.GetCounters()[METRIC_LAST_READ_TIME].Get() != ts) {
haveChanges = true;
- userInfo.LabeledCounters.GetCounters()[METRIC_LAST_READ_TIME].Set(ts);
+ userInfo.LabeledCounters.GetCounters()[METRIC_LAST_READ_TIME].Set(ts);
}
- ui64 timeLag = userInfo.GetWriteLagMs();
- if (userInfo.LabeledCounters.GetCounters()[METRIC_WRITE_TIME_LAG].Get() != timeLag) {
+ ui64 timeLag = userInfo.GetWriteLagMs();
+ if (userInfo.LabeledCounters.GetCounters()[METRIC_WRITE_TIME_LAG].Get() != timeLag) {
haveChanges = true;
- userInfo.LabeledCounters.GetCounters()[METRIC_WRITE_TIME_LAG].Set(timeLag);
+ userInfo.LabeledCounters.GetCounters()[METRIC_WRITE_TIME_LAG].Set(timeLag);
}
- if (userInfo.LabeledCounters.GetCounters()[METRIC_READ_TIME_LAG].Get() != readTimeLag) {
+ if (userInfo.LabeledCounters.GetCounters()[METRIC_READ_TIME_LAG].Get() != readTimeLag) {
haveChanges = true;
- userInfo.LabeledCounters.GetCounters()[METRIC_READ_TIME_LAG].Set(readTimeLag);
+ userInfo.LabeledCounters.GetCounters()[METRIC_READ_TIME_LAG].Set(readTimeLag);
}
- if (userInfo.LabeledCounters.GetCounters()[METRIC_COMMIT_MESSAGE_LAG].Get() != EndOffset - userInfo.Offset) {
+ if (userInfo.LabeledCounters.GetCounters()[METRIC_COMMIT_MESSAGE_LAG].Get() != EndOffset - userInfo.Offset) {
haveChanges = true;
- userInfo.LabeledCounters.GetCounters()[METRIC_COMMIT_MESSAGE_LAG].Set(EndOffset - userInfo.Offset);
+ userInfo.LabeledCounters.GetCounters()[METRIC_COMMIT_MESSAGE_LAG].Set(EndOffset - userInfo.Offset);
}
- if (userInfo.LabeledCounters.GetCounters()[METRIC_READ_MESSAGE_LAG].Get() != EndOffset - off) {
+ if (userInfo.LabeledCounters.GetCounters()[METRIC_READ_MESSAGE_LAG].Get() != EndOffset - off) {
haveChanges = true;
- userInfo.LabeledCounters.GetCounters()[METRIC_READ_MESSAGE_LAG].Set(EndOffset - off);
- userInfo.LabeledCounters.GetCounters()[METRIC_READ_TOTAL_MESSAGE_LAG].Set(EndOffset - off);
+ userInfo.LabeledCounters.GetCounters()[METRIC_READ_MESSAGE_LAG].Set(EndOffset - off);
+ userInfo.LabeledCounters.GetCounters()[METRIC_READ_TOTAL_MESSAGE_LAG].Set(EndOffset - off);
}
- ui64 sizeLag = GetSizeLag(userInfo.Offset);
- ui64 sizeLagRead = GetSizeLag(userInfo.ReadOffset);
- if (userInfo.LabeledCounters.GetCounters()[METRIC_COMMIT_SIZE_LAG].Get() != sizeLag) {
+ ui64 sizeLag = GetSizeLag(userInfo.Offset);
+ ui64 sizeLagRead = GetSizeLag(userInfo.ReadOffset);
+ if (userInfo.LabeledCounters.GetCounters()[METRIC_COMMIT_SIZE_LAG].Get() != sizeLag) {
haveChanges = true;
- userInfo.LabeledCounters.GetCounters()[METRIC_COMMIT_SIZE_LAG].Set(sizeLag);
+ userInfo.LabeledCounters.GetCounters()[METRIC_COMMIT_SIZE_LAG].Set(sizeLag);
}
- if (userInfo.LabeledCounters.GetCounters()[METRIC_READ_SIZE_LAG].Get() != sizeLagRead) {
+ if (userInfo.LabeledCounters.GetCounters()[METRIC_READ_SIZE_LAG].Get() != sizeLagRead) {
haveChanges = true;
- userInfo.LabeledCounters.GetCounters()[METRIC_READ_SIZE_LAG].Set(sizeLagRead);
- userInfo.LabeledCounters.GetCounters()[METRIC_READ_TOTAL_SIZE_LAG].Set(sizeLag);
+ userInfo.LabeledCounters.GetCounters()[METRIC_READ_SIZE_LAG].Set(sizeLagRead);
+ userInfo.LabeledCounters.GetCounters()[METRIC_READ_TOTAL_SIZE_LAG].Set(sizeLag);
}
- if (userInfo.LabeledCounters.GetCounters()[METRIC_USER_PARTITIONS].Get() == 0) {
+ if (userInfo.LabeledCounters.GetCounters()[METRIC_USER_PARTITIONS].Get() == 0) {
haveChanges = true;
- userInfo.LabeledCounters.GetCounters()[METRIC_USER_PARTITIONS].Set(1);
+ userInfo.LabeledCounters.GetCounters()[METRIC_USER_PARTITIONS].Set(1);
}
- ui64 speed = userInfo.ReadQuota.GetTotalSpeed();
- if (speed != userInfo.LabeledCounters.GetCounters()[METRIC_READ_QUOTA_BYTES].Get()) {
+ ui64 speed = userInfo.ReadQuota.GetTotalSpeed();
+ if (speed != userInfo.LabeledCounters.GetCounters()[METRIC_READ_QUOTA_BYTES].Get()) {
haveChanges = true;
- userInfo.LabeledCounters.GetCounters()[METRIC_READ_QUOTA_BYTES].Set(speed);
+ userInfo.LabeledCounters.GetCounters()[METRIC_READ_QUOTA_BYTES].Set(speed);
}
- ui64 availSec = userInfo.ReadQuota.GetAvailableAvgSec(ctx.Now());
- if (availSec != userInfo.LabeledCounters.GetCounters()[METRIC_MIN_READ_QUOTA_BYTES_AVAIL_SEC].Get()) {
+ ui64 availSec = userInfo.ReadQuota.GetAvailableAvgSec(ctx.Now());
+ if (availSec != userInfo.LabeledCounters.GetCounters()[METRIC_MIN_READ_QUOTA_BYTES_AVAIL_SEC].Get()) {
haveChanges = true;
- userInfo.LabeledCounters.GetCounters()[METRIC_MIN_READ_QUOTA_BYTES_AVAIL_SEC].Set(availSec);
+ userInfo.LabeledCounters.GetCounters()[METRIC_MIN_READ_QUOTA_BYTES_AVAIL_SEC].Set(availSec);
}
- ui64 availMin = userInfo.ReadQuota.GetAvailableAvgMin(ctx.Now());
- if (availMin != userInfo.LabeledCounters.GetCounters()[METRIC_MIN_READ_QUOTA_BYTES_AVAIL_MIN].Get()) {
+ ui64 availMin = userInfo.ReadQuota.GetAvailableAvgMin(ctx.Now());
+ if (availMin != userInfo.LabeledCounters.GetCounters()[METRIC_MIN_READ_QUOTA_BYTES_AVAIL_MIN].Get()) {
haveChanges = true;
- userInfo.LabeledCounters.GetCounters()[METRIC_MIN_READ_QUOTA_BYTES_AVAIL_MIN].Set(availMin);
+ userInfo.LabeledCounters.GetCounters()[METRIC_MIN_READ_QUOTA_BYTES_AVAIL_MIN].Set(availMin);
}
- ui64 readOffsetRewindSum = userInfo.ReadOffsetRewindSum;
- if (readOffsetRewindSum != userInfo.LabeledCounters.GetCounters()[METRIC_READ_OFFSET_REWIND_SUM].Get()) {
+ ui64 readOffsetRewindSum = userInfo.ReadOffsetRewindSum;
+ if (readOffsetRewindSum != userInfo.LabeledCounters.GetCounters()[METRIC_READ_OFFSET_REWIND_SUM].Get()) {
haveChanges = true;
- userInfo.LabeledCounters.GetCounters()[METRIC_READ_OFFSET_REWIND_SUM].Set(readOffsetRewindSum);
+ userInfo.LabeledCounters.GetCounters()[METRIC_READ_OFFSET_REWIND_SUM].Set(readOffsetRewindSum);
}
if (readOffsetRewindSum != userInfo.LabeledCounters.GetCounters()[METRIC_READ_OFFSET_REWIND_TOTAL].Get()) {
haveChanges = true;
@@ -3181,25 +3181,25 @@ void TPartition::ReportLabeledCounters(const TActorContext& ctx)
}
ui32 id = METRIC_TOTAL_READ_SPEED_1;
- for (ui32 i = 0; i < userInfo.AvgReadBytes.size(); ++i) {
- ui64 avg = userInfo.AvgReadBytes[i].GetValue();
- if (avg != userInfo.LabeledCounters.GetCounters()[id].Get()) {
+ for (ui32 i = 0; i < userInfo.AvgReadBytes.size(); ++i) {
+ ui64 avg = userInfo.AvgReadBytes[i].GetValue();
+ if (avg != userInfo.LabeledCounters.GetCounters()[id].Get()) {
haveChanges = true;
- userInfo.LabeledCounters.GetCounters()[id].Set(avg); //total
- userInfo.LabeledCounters.GetCounters()[id + 1].Set(avg); //max
+ userInfo.LabeledCounters.GetCounters()[id].Set(avg); //total
+ userInfo.LabeledCounters.GetCounters()[id + 1].Set(avg); //max
}
id += 2;
}
Y_VERIFY(id == METRIC_MAX_READ_SPEED_4 + 1);
- if (userInfo.ReadQuota.GetTotalSpeed()) {
- ui64 quotaUsage = ui64(userInfo.AvgReadBytes[1].GetValue()) * 1000000 / userInfo.ReadQuota.GetTotalSpeed() / 60;
- if (quotaUsage != userInfo.LabeledCounters.GetCounters()[METRIC_READ_QUOTA_USAGE].Get()) {
+ if (userInfo.ReadQuota.GetTotalSpeed()) {
+ ui64 quotaUsage = ui64(userInfo.AvgReadBytes[1].GetValue()) * 1000000 / userInfo.ReadQuota.GetTotalSpeed() / 60;
+ if (quotaUsage != userInfo.LabeledCounters.GetCounters()[METRIC_READ_QUOTA_USAGE].Get()) {
haveChanges = true;
- userInfo.LabeledCounters.GetCounters()[METRIC_READ_QUOTA_USAGE].Set(quotaUsage);
+ userInfo.LabeledCounters.GetCounters()[METRIC_READ_QUOTA_USAGE].Set(quotaUsage);
}
}
if (haveChanges) {
- ctx.Send(Tablet, new TEvPQ::TEvPartitionLabeledCounters(Partition, userInfo.LabeledCounters));
+ ctx.Send(Tablet, new TEvPQ::TEvPartitionLabeledCounters(Partition, userInfo.LabeledCounters));
}
}
//Partition counters
@@ -3210,10 +3210,10 @@ void TPartition::ReportLabeledCounters(const TActorContext& ctx)
PartitionLabeledCounters.GetCounters()[METRIC_NUM_SIDS].Set(SourceIdStorage.GetInMemorySourceIds().size());
}
- TDuration lifetimeNow = ctx.Now() - SourceIdStorage.MinAvailableTimestamp(ctx.Now());
- if (lifetimeNow.MilliSeconds() != PartitionLabeledCounters.GetCounters()[METRIC_MIN_SID_LIFETIME].Get()) {
+ TDuration lifetimeNow = ctx.Now() - SourceIdStorage.MinAvailableTimestamp(ctx.Now());
+ if (lifetimeNow.MilliSeconds() != PartitionLabeledCounters.GetCounters()[METRIC_MIN_SID_LIFETIME].Get()) {
haveChanges = true;
- PartitionLabeledCounters.GetCounters()[METRIC_MIN_SID_LIFETIME].Set(lifetimeNow.MilliSeconds());
+ PartitionLabeledCounters.GetCounters()[METRIC_MIN_SID_LIFETIME].Set(lifetimeNow.MilliSeconds());
}
ui64 headGapSize = DataKeysBody.empty() ? 0 : (Head.Offset - (DataKeysBody.back().Key.GetOffset() + DataKeysBody.back().Key.GetCount()));
@@ -3381,8 +3381,8 @@ void TPartition::HandleSetOffsetResponse(NKikimrClient::TResponse& response, con
TString user = it->second;
CookieToUser.erase(it);
- TUserInfo* userInfo = UsersInfoStorage.GetIfExists(user);
- Y_VERIFY(userInfo);
+ TUserInfo* userInfo = UsersInfoStorage.GetIfExists(user);
+ Y_VERIFY(userInfo);
Y_VERIFY(!userInfo->UserActs.empty());
@@ -3419,7 +3419,7 @@ void TPartition::HandleSetOffsetResponse(NKikimrClient::TResponse& response, con
LOG_DEBUG_S(ctx, NKikimrServices::PERSQUEUE, "Topic '" << TopicName << "' partition " << Partition
<< " user " << user << " reinit with generation " << readRuleGeneration << " done");
userInfo->ReadRuleGeneration = readRuleGeneration;
- userInfo->Session = "";
+ userInfo->Session = "";
userInfo->Generation = userInfo->Step = 0;
userInfo->Offset = 0;
} else {
@@ -3463,7 +3463,7 @@ void TPartition::HandleSetOffsetResponse(NKikimrClient::TResponse& response, con
void TPartition::ScheduleUpdateAvailableSize(const TActorContext& ctx)
{
- ctx.Schedule(UPDATE_AVAIL_SIZE_INTERVAL, new TEvPQ::TEvUpdateAvailableSize());
+ ctx.Schedule(UPDATE_AVAIL_SIZE_INTERVAL, new TEvPQ::TEvUpdateAvailableSize());
}
@@ -3556,25 +3556,25 @@ void TPartition::HandleOnWrite(TEvPQ::TEvWrite::TPtr& ev, const TActorContext& c
});
bool mirroredPartition = Config.GetPartitionConfig().HasMirrorFrom();
- if (mirroredPartition && !ev->Get()->OwnerCookie.empty()) {
+ if (mirroredPartition && !ev->Get()->OwnerCookie.empty()) {
ReplyError(ctx, ev->Get()->Cookie, NPersQueue::NErrorCode::BAD_REQUEST,
- TStringBuilder() << "Write to mirrored topic is forbiden ");
+ TStringBuilder() << "Write to mirrored topic is forbiden ");
return;
}
- ui64 decReservedSize = 0;
+ ui64 decReservedSize = 0;
TStringBuf owner;
if (!mirroredPartition && !ev->Get()->IsDirectWrite) {
owner = TOwnerInfo::GetOwnerFromOwnerCookie(ev->Get()->OwnerCookie);
- auto it = Owners.find(owner);
-
- if (it == Owners.end() || it->second.NeedResetOwner) {
- ReplyError(ctx, ev->Get()->Cookie, NPersQueue::NErrorCode::WRONG_COOKIE,
- TStringBuilder() << "new GetOwnership request needed for owner " << owner);
- return;
- }
-
+ auto it = Owners.find(owner);
+
+ if (it == Owners.end() || it->second.NeedResetOwner) {
+ ReplyError(ctx, ev->Get()->Cookie, NPersQueue::NErrorCode::WRONG_COOKIE,
+ TStringBuilder() << "new GetOwnership request needed for owner " << owner);
+ return;
+ }
+
if (it->second.SourceIdDeleted) {
ReplyError(ctx, ev->Get()->Cookie, NPersQueue::NErrorCode::SOURCEID_DELETED,
TStringBuilder() << "Yours maximum written sequence number for session was deleted, need to recreate session. "
@@ -3584,21 +3584,21 @@ void TPartition::HandleOnWrite(TEvPQ::TEvWrite::TPtr& ev, const TActorContext& c
return;
}
- if (it->second.OwnerCookie != ev->Get()->OwnerCookie) {
- ReplyError(ctx, ev->Get()->Cookie, NPersQueue::NErrorCode::WRONG_COOKIE,
- TStringBuilder() << "incorrect ownerCookie " << ev->Get()->OwnerCookie << ", must be " << it->second.OwnerCookie);
- return;
- }
-
- if (ev->Get()->MessageNo != it->second.NextMessageNo) {
- ReplyError(ctx, ev->Get()->Cookie, NPersQueue::NErrorCode::BAD_REQUEST,
- TStringBuilder() << "reorder in requests, waiting " << it->second.NextMessageNo << ", but got " << ev->Get()->MessageNo);
- DropOwner(it, ctx);
- return;
- }
-
- ++it->second.NextMessageNo;
- decReservedSize = it->second.DecReservedSize();
+ if (it->second.OwnerCookie != ev->Get()->OwnerCookie) {
+ ReplyError(ctx, ev->Get()->Cookie, NPersQueue::NErrorCode::WRONG_COOKIE,
+ TStringBuilder() << "incorrect ownerCookie " << ev->Get()->OwnerCookie << ", must be " << it->second.OwnerCookie);
+ return;
+ }
+
+ if (ev->Get()->MessageNo != it->second.NextMessageNo) {
+ ReplyError(ctx, ev->Get()->Cookie, NPersQueue::NErrorCode::BAD_REQUEST,
+ TStringBuilder() << "reorder in requests, waiting " << it->second.NextMessageNo << ", but got " << ev->Get()->MessageNo);
+ DropOwner(it, ctx);
+ return;
+ }
+
+ ++it->second.NextMessageNo;
+ decReservedSize = it->second.DecReservedSize();
}
TMaybe<ui64> offset = ev->Get()->Offset;
@@ -3661,9 +3661,9 @@ void TPartition::HandleOnWrite(TEvPQ::TEvWrite::TPtr& ev, const TActorContext& c
++*offset;
}
WriteInflightSize += size;
-
- ReservedSize -= decReservedSize;
- Y_VERIFY(size <= decReservedSize || decReservedSize == 0); //TODO: remove decReservedSize == 0
+
+ ReservedSize -= decReservedSize;
+ Y_VERIFY(size <= decReservedSize || decReservedSize == 0); //TODO: remove decReservedSize == 0
Counters.Simple()[COUNTER_PQ_TABLET_RESERVED_BYTES_SIZE].Set(ReservedSize);
UpdateWriteBufferIsFullState(ctx.Now());
}
@@ -3808,13 +3808,13 @@ void TPartition::BecomeIdle(const TActorContext&)
}
-void TPartition::WriteClientInfo(const ui64 cookie, TUserInfo& userInfo, const TActorContext& ctx) {
+void TPartition::WriteClientInfo(const ui64 cookie, TUserInfo& userInfo, const TActorContext& ctx) {
THolder<TEvKeyValue::TEvRequest> request(new TEvKeyValue::TEvRequest);
Y_VERIFY(!userInfo.WriteInProgress);
- Y_VERIFY(!userInfo.UserActs.empty());
- while (!userInfo.UserActs.empty()) {
- const auto ev = userInfo.UserActs.front().Get();
+ Y_VERIFY(!userInfo.UserActs.empty());
+ while (!userInfo.UserActs.empty()) {
+ const auto ev = userInfo.UserActs.front().Get();
TKeyPrefix ikey(TKeyPrefix::TypeInfo, Partition, TKeyPrefix::MarkUser);
ikey.Append(ev->ClientId.c_str(), ev->ClientId.size());
@@ -3839,9 +3839,9 @@ void TPartition::WriteClientInfo(const ui64 cookie, TUserInfo& userInfo, const T
}
if (ev->Type == TEvPQ::TEvSetClientInfo::ESCI_CREATE_SESSION && ev->SessionId == userInfo.Session) { //this is retry of current request, answer ok
- auto ts = GetTime(userInfo, userInfo.Offset);
- ReplyGetClientOffsetOk(ctx, ev->Cookie, userInfo.Offset, ts.first, ts.second);
- userInfo.UserActs.pop_front();
+ auto ts = GetTime(userInfo, userInfo.Offset);
+ ReplyGetClientOffsetOk(ctx, ev->Cookie, userInfo.Offset, ts.first, ts.second);
+ userInfo.UserActs.pop_front();
continue;
}
@@ -3849,17 +3849,17 @@ void TPartition::WriteClientInfo(const ui64 cookie, TUserInfo& userInfo, const T
&& !ev->SessionId.empty() && userInfo.Session != ev->SessionId //request to wrong session
&& (ev->Type != TEvPQ::TEvSetClientInfo::ESCI_DROP_SESSION || !userInfo.Session.empty()) //but allow DropSession request when session is already dropped - for idempotence
|| (ev->Type == TEvPQ::TEvSetClientInfo::ESCI_CREATE_SESSION && !userInfo.Session.empty()
- && (ev->Generation < userInfo.Generation || ev->Generation == userInfo.Generation && ev->Step <= userInfo.Step))) { //old generation request
+ && (ev->Generation < userInfo.Generation || ev->Generation == userInfo.Generation && ev->Step <= userInfo.Step))) { //old generation request
Counters.Cumulative()[COUNTER_PQ_SET_CLIENT_OFFSET_ERROR].Increment(1);
ReplyError(ctx, ev->Cookie, NPersQueue::NErrorCode::WRONG_COOKIE,
- TStringBuilder() << "set offset in already dead session " << ev->SessionId << " actual is " << userInfo.Session);
- userInfo.UserActs.pop_front();
+ TStringBuilder() << "set offset in already dead session " << ev->SessionId << " actual is " << userInfo.Session);
+ userInfo.UserActs.pop_front();
continue;
}
- if (!ev->SessionId.empty() && ev->Type == TEvPQ::TEvSetClientInfo::ESCI_OFFSET && (i64)ev->Offset <= userInfo.Offset) { //this is stale request, answer ok for it
+ if (!ev->SessionId.empty() && ev->Type == TEvPQ::TEvSetClientInfo::ESCI_OFFSET && (i64)ev->Offset <= userInfo.Offset) { //this is stale request, answer ok for it
ReplyOk(ctx, ev->Cookie);
- userInfo.UserActs.pop_front();
+ userInfo.UserActs.pop_front();
continue;
}
@@ -3868,7 +3868,7 @@ void TPartition::WriteClientInfo(const ui64 cookie, TUserInfo& userInfo, const T
TString session = (ev->Type == TEvPQ::TEvSetClientInfo::ESCI_CREATE_SESSION ? ev->SessionId : (ev->Type == TEvPQ::TEvSetClientInfo::ESCI_DROP_SESSION ? "" : userInfo.Session));
ui32 gen = (ev->Type == TEvPQ::TEvSetClientInfo::ESCI_CREATE_SESSION ? ev->Generation : (ev->Type == TEvPQ::TEvSetClientInfo::ESCI_DROP_SESSION ? 0 : userInfo.Generation));
ui32 step = (ev->Type == TEvPQ::TEvSetClientInfo::ESCI_CREATE_SESSION ? ev->Step : (ev->Type == TEvPQ::TEvSetClientInfo::ESCI_DROP_SESSION ? 0 : userInfo.Step));
- ui64 offset = (ev->Type == TEvPQ::TEvSetClientInfo::ESCI_OFFSET ? ev->Offset : userInfo.Offset);
+ ui64 offset = (ev->Type == TEvPQ::TEvSetClientInfo::ESCI_OFFSET ? ev->Offset : userInfo.Offset);
ui64 readRuleGeneration = userInfo.ReadRuleGeneration;
if (ev->Type == TEvPQ::TEvSetClientInfo::ESCI_INIT_READ_RULE) {
readRuleGeneration = ev->ReadRuleGeneration;
@@ -3889,24 +3889,24 @@ void TPartition::WriteClientInfo(const ui64 cookie, TUserInfo& userInfo, const T
/* Counters.Cumulative()[COUNTER_PQ_SET_CLIENT_OFFSET_ERROR].Increment(1);
ReplyError(ctx, ev->Cookie, NPersQueue::NErrorCode::SET_OFFSET_ERROR_COMMIT_TO_FUTURE,
TStringBuilder() << "can't commit to future. Offset " << offset << " EndOffset " << EndOffset);
- userInfo.UserActrs.pop_front();
+ userInfo.UserActrs.pop_front();
continue;*/
}
TBuffer idata;
- {
- NKikimrPQ::TUserInfo userData;
- userData.SetOffset(offset);
- userData.SetGeneration(gen);
- userData.SetStep(step);
- userData.SetSession(session);
- userData.SetOffsetRewindSum(userInfo.ReadOffsetRewindSum);
+ {
+ NKikimrPQ::TUserInfo userData;
+ userData.SetOffset(offset);
+ userData.SetGeneration(gen);
+ userData.SetStep(step);
+ userData.SetSession(session);
+ userData.SetOffsetRewindSum(userInfo.ReadOffsetRewindSum);
userData.SetReadRuleGeneration(readRuleGeneration);
- TString out;
+ TString out;
Y_PROTOBUF_SUPPRESS_NODISCARD userData.SerializeToString(&out);
- idata.Append(out.c_str(), out.size());
- }
+ idata.Append(out.c_str(), out.size());
+ }
TBuffer idataDeprecated = NDeprecatedUserData::Serialize(offset, gen, step, session);
auto write = request->Record.AddCmdWrite();
@@ -3968,7 +3968,7 @@ bool TPartition::AppendHeadWithNewWrites(TEvKeyValue::TEvRequest* request, const
WriteNewSizeUncompressed = 0;
WriteNewMessages = 0;
UpdateWriteBufferIsFullState(ctx.Now());
- CurrentTimestamp = ctx.Now();
+ CurrentTimestamp = ctx.Now();
NewHead.Offset = EndOffset;
NewHead.PartNo = 0;
@@ -4143,14 +4143,14 @@ bool TPartition::AppendHeadWithNewWrites(TEvKeyValue::TEvRequest* request, const
}
WriteTimestamp = ctx.Now();
WriteTimestampEstimate = p.Msg.WriteTimestamp > 0 ? TInstant::MilliSeconds(p.Msg.WriteTimestamp) : WriteTimestamp;
- TClientBlob blob(p.Msg.SourceId, p.Msg.SeqNo, p.Msg.Data, std::move(partData), WriteTimestampEstimate,
+ TClientBlob blob(p.Msg.SourceId, p.Msg.SeqNo, p.Msg.Data, std::move(partData), WriteTimestampEstimate,
TInstant::MilliSeconds(p.Msg.CreateTimestamp == 0 ? curOffset : p.Msg.CreateTimestamp),
p.Msg.UncompressedSize, p.Msg.PartitionKey, p.Msg.ExplicitHashKey); //remove curOffset when LB will report CTime
- ui64 writeLagMs = (WriteTimestamp - TInstant::MilliSeconds(p.Msg.CreateTimestamp)).MilliSeconds();
- WriteLagMs.Update(writeLagMs, WriteTimestamp);
+ ui64 writeLagMs = (WriteTimestamp - TInstant::MilliSeconds(p.Msg.CreateTimestamp)).MilliSeconds();
+ WriteLagMs.Update(writeLagMs, WriteTimestamp);
if (InputTimeLag) {
- InputTimeLag->IncFor(writeLagMs, 1);
+ InputTimeLag->IncFor(writeLagMs, 1);
if (p.Msg.PartNo == 0) {
MessageSize->IncFor(p.Msg.TotalSize + p.Msg.SourceId.size(), 1);
}
@@ -4403,7 +4403,7 @@ void TPartition::AddNewWriteBlob(std::pair<TKey, ui32>& res, TEvKeyValue::TEvReq
NewHead.PartNo = 0;
} else {
Y_VERIFY(NewHeadKey.Size == 0);
- NewHeadKey = {key, res.second, CurrentTimestamp, 0};
+ NewHeadKey = {key, res.second, CurrentTimestamp, 0};
}
WriteCycleSize += write->GetValue().size();
UpdateWriteBufferIsFullState(ctx.Now());
@@ -4567,15 +4567,15 @@ void TPartition::ProcessRead(const TActorContext& ctx, TReadInfo&& info, const u
ui32 size = 0;
Y_VERIFY(!info.User.empty());
- auto& userInfo = UsersInfoStorage.GetOrCreate(info.User, ctx);
+ auto& userInfo = UsersInfoStorage.GetOrCreate(info.User, ctx);
if (subscription) {
- userInfo.ForgetSubscription(ctx.Now());
+ userInfo.ForgetSubscription(ctx.Now());
}
- if (!userInfo.ReadQuota.CanExaust()) {
- userInfo.ReadRequests.push_back({std::move(info), cookie});
- userInfo.UpdateReadingTimeAndState(ctx.Now());
+ if (!userInfo.ReadQuota.CanExaust()) {
+ userInfo.ReadRequests.push_back({std::move(info), cookie});
+ userInfo.UpdateReadingTimeAndState(ctx.Now());
return;
}
TVector<TRequestedBlob> blobs = GetReadRequestFromBody(info.Offset, info.PartNo, info.Count, info.Size, &count, &size);
@@ -4589,27 +4589,27 @@ void TPartition::ProcessRead(const TActorContext& ctx, TReadInfo&& info, const u
info.CachedOffset = Head.Offset > 0 ? Head.Offset : insideHeadOffset;
if (info.Destination != 0) {
- ++userInfo.ActiveReads;
- userInfo.UpdateReadingTimeAndState(ctx.Now());
+ ++userInfo.ActiveReads;
+ userInfo.UpdateReadingTimeAndState(ctx.Now());
}
if (info.Blobs.empty()) { //all from head, answer right now
LOG_DEBUG_S(ctx, NKikimrServices::PERSQUEUE, "Reading cookie " << cookie << ". All data is from uncompacted head.");
- TReadAnswer answer(info.FormAnswer(
- ctx, EndOffset, Partition, &UsersInfoStorage.GetOrCreate(info.User, ctx),
- info.Destination, GetSizeLag(info.Offset)
- ));
- const auto& resp = dynamic_cast<TEvPQ::TEvProxyResponse*>(answer.Event.Get())->Response;
+ TReadAnswer answer(info.FormAnswer(
+ ctx, EndOffset, Partition, &UsersInfoStorage.GetOrCreate(info.User, ctx),
+ info.Destination, GetSizeLag(info.Offset)
+ ));
+ const auto& resp = dynamic_cast<TEvPQ::TEvProxyResponse*>(answer.Event.Get())->Response;
if (info.IsSubscription) {
Counters.Cumulative()[COUNTER_PQ_READ_SUBSCRIPTION_OK].Increment(1);
}
Counters.Cumulative()[COUNTER_PQ_READ_HEAD_ONLY_OK].Increment(1);
Counters.Percentile()[COUNTER_LATENCY_PQ_READ_HEAD_ONLY].IncrementFor((ctx.Now() - info.Timestamp).MilliSeconds());
Counters.Cumulative()[COUNTER_PQ_READ_BYTES].Increment(resp.ByteSize());
- ctx.Send(info.Destination != 0 ? Tablet : ctx.SelfID, answer.Event.Release());
+ ctx.Send(info.Destination != 0 ? Tablet : ctx.SelfID, answer.Event.Release());
ReportLabeledCounters(ctx);
- OnReadRequestFinished(std::move(info), answer.Size);
+ OnReadRequestFinished(std::move(info), answer.Size);
return;
}
@@ -4740,13 +4740,13 @@ void TPartition::CalcTopicWriteQuotaParams()
}
}
-void TPartition::CreateMirrorerActor() {
- Mirrorer = MakeHolder<TMirrorerInfo>(
+void TPartition::CreateMirrorerActor() {
+ Mirrorer = MakeHolder<TMirrorerInfo>(
Register(new TMirrorer(Tablet, SelfId(), TopicName, Partition, LocalDC, EndOffset, Config.GetPartitionConfig().GetMirrorFrom(), Counters)),
- Counters
- );
-}
-
+ Counters
+ );
+}
+
bool TPartition::IsQuotingEnabled() const {
const auto& pqConfig = AppData()->PQConfig;
const auto& quotingConfig = pqConfig.GetQuotingConfig();
diff --git a/ydb/core/persqueue/partition.h b/ydb/core/persqueue/partition.h
index d6fe2ea41b..2eec26f251 100644
--- a/ydb/core/persqueue/partition.h
+++ b/ydb/core/persqueue/partition.h
@@ -11,7 +11,7 @@
#include <ydb/core/protos/pqconfig.pb.h>
#include <ydb/core/persqueue/events/internal.h>
#include <ydb/library/persqueue/counter_time_keeper/counter_time_keeper.h>
-
+
#include "key.h"
#include "blob.h"
#include "subscriber.h"
@@ -35,19 +35,19 @@ typedef TProtobufTabletLabeledCounters<EPartitionLabeledCounters_descriptor> TPa
struct TDataKey {
TKey Key;
ui32 Size;
- TInstant Timestamp;
+ TInstant Timestamp;
ui64 CumulativeSize;
};
-ui64 GetOffsetEstimate(const std::deque<TDataKey>& container, TInstant timestamp, ui64 headOffset);
+ui64 GetOffsetEstimate(const std::deque<TDataKey>& container, TInstant timestamp, ui64 headOffset);
+
+struct TMirrorerInfo;
-struct TMirrorerInfo;
-
class TPartition : public TActorBootstrapped<TPartition> {
-private:
- static constexpr ui32 MAX_ERRORS_COUNT_TO_STORE = 10;
+private:
+ static constexpr ui32 MAX_ERRORS_COUNT_TO_STORE = 10;
-private:
+private:
struct THasDataReq;
struct THasDataDeadline;
@@ -60,20 +60,20 @@ private:
void ReplyError(const TActorContext& ctx, const ui64 dst, NPersQueue::NErrorCode::EErrorCode errorCode, const TString& error);
void ReplyErrorForStoredWrites(const TActorContext& ctx);
void ReplyOk(const TActorContext& ctx, const ui64 dst);
- void ReplyWrite(
- const TActorContext& ctx, ui64 dst, const TString& sourceId, ui64 seqNo, ui16 partNo, ui16 totalParts,
- ui64 offset, TInstant writeTimestamp, bool already, ui64 maxSeqNo,
- ui64 partitionQuotedTime, TDuration topicQuotedTime, ui64 queueTime, ui64 writeTime);
+ void ReplyWrite(
+ const TActorContext& ctx, ui64 dst, const TString& sourceId, ui64 seqNo, ui16 partNo, ui16 totalParts,
+ ui64 offset, TInstant writeTimestamp, bool already, ui64 maxSeqNo,
+ ui64 partitionQuotedTime, TDuration topicQuotedTime, ui64 queueTime, ui64 writeTime);
void ReplyGetClientOffsetOk(const TActorContext& ctx, const ui64 dst, const i64 offset,
- const TInstant writeTimestamp, const TInstant createTimestamp);
+ const TInstant writeTimestamp, const TInstant createTimestamp);
void ReplyOwnerOk(const TActorContext& ctx, const ui64 dst, const TString& ownerCookie);
void Handle(TEvPersQueue::TEvHasDataInfo::TPtr& ev, const TActorContext& ctx);
- void Handle(TEvPQ::TEvMirrorerCounters::TPtr& ev, const TActorContext& ctx);
- void Handle(NReadSpeedLimiterEvents::TEvCounters::TPtr& ev, const TActorContext& ctx);
-
+ void Handle(TEvPQ::TEvMirrorerCounters::TPtr& ev, const TActorContext& ctx);
+ void Handle(NReadSpeedLimiterEvents::TEvCounters::TPtr& ev, const TActorContext& ctx);
+
//answer for reads for Timestamps
void Handle(TEvPQ::TEvProxyResponse::TPtr& ev, const TActorContext& ctx);
void Handle(TEvPQ::TEvError::TPtr& ev, const TActorContext& ctx);
@@ -117,10 +117,10 @@ private:
void Handle(TEvPQ::TEvPartitionStatus::TPtr& ev, const TActorContext& ctx);
void Handle(TEvPQ::TEvGetPartitionClientInfo::TPtr& ev, const TActorContext& ctx);
- void Handle(TEvPersQueue::TEvReportPartitionError::TPtr& ev, const TActorContext& ctx);
- void LogAndCollectError(const NKikimrPQ::TStatusResponse::TErrorMessage& error, const TActorContext& ctx);
- void LogAndCollectError(NKikimrServices::EServiceKikimr service, const TString& msg, const TActorContext& ctx);
-
+ void Handle(TEvPersQueue::TEvReportPartitionError::TPtr& ev, const TActorContext& ctx);
+ void LogAndCollectError(const NKikimrPQ::TStatusResponse::TErrorMessage& error, const TActorContext& ctx);
+ void LogAndCollectError(NKikimrServices::EServiceKikimr service, const TString& msg, const TActorContext& ctx);
+
void HandleOnIdle(TEvPQ::TEvUpdateAvailableSize::TPtr& ev, const TActorContext& ctx);
void HandleOnWrite(TEvPQ::TEvUpdateAvailableSize::TPtr& ev, const TActorContext& ctx);
@@ -136,9 +136,9 @@ private:
void Handle(TEvents::TEvPoisonPill::TPtr& ev, const TActorContext& ctx);
void Handle(TEvPQ::TEvRead::TPtr& ev, const TActorContext& ctx);
- void Handle(NReadSpeedLimiterEvents::TEvResponse::TPtr& ev, const TActorContext& ctx);
- void DoRead(TEvPQ::TEvRead::TPtr ev, TDuration waitQuotaTime, const TActorContext& ctx);
- void OnReadRequestFinished(TReadInfo&& info, ui64 answerSize);
+ void Handle(NReadSpeedLimiterEvents::TEvResponse::TPtr& ev, const TActorContext& ctx);
+ void DoRead(TEvPQ::TEvRead::TPtr ev, TDuration waitQuotaTime, const TActorContext& ctx);
+ void OnReadRequestFinished(TReadInfo&& info, ui64 answerSize);
// will return rcount and rsize also
TVector<TRequestedBlob> GetReadRequestFromBody(const ui64 startOffset, const ui16 partNo, const ui32 maxCount, const ui32 maxSize, ui32* rcount, ui32* rsize);
@@ -207,7 +207,7 @@ private:
void Handle(TEvPQ::TEvReserveBytes::TPtr& ev, const TActorContext& ctx);
void ProcessReserveRequests(const TActorContext& ctx);
- void CreateMirrorerActor();
+ void CreateMirrorerActor();
bool IsQuotingEnabled() const;
void SetupTopicCounters(const TActorContext& ctx);
@@ -252,7 +252,7 @@ private:
STFUNC(StateInit)
{
- NPersQueue::TCounterTimeKeeper keeper(Counters.Cumulative()[COUNTER_PQ_TABLET_CPU_USAGE]);
+ NPersQueue::TCounterTimeKeeper keeper(Counters.Cumulative()[COUNTER_PQ_TABLET_CPU_USAGE]);
LOG_TRACE_S(ctx, NKikimrServices::PERSQUEUE, EventStr("StateInit", ev));
@@ -265,10 +265,10 @@ private:
HFuncTraced(TEvPQ::TEvChangeConfig, Handle);
HFuncTraced(TEvPQ::TEvPartitionOffsets, HandleOnInit);
HFuncTraced(TEvPQ::TEvPartitionStatus, HandleOnInit);
- HFuncTraced(TEvPersQueue::TEvReportPartitionError, Handle);
+ HFuncTraced(TEvPersQueue::TEvReportPartitionError, Handle);
HFuncTraced(TEvPersQueue::TEvHasDataInfo, Handle);
- HFuncTraced(TEvPQ::TEvMirrorerCounters, Handle);
- HFuncTraced(NReadSpeedLimiterEvents::TEvCounters, Handle);
+ HFuncTraced(TEvPQ::TEvMirrorerCounters, Handle);
+ HFuncTraced(NReadSpeedLimiterEvents::TEvCounters, Handle);
HFuncTraced(TEvPQ::TEvGetPartitionClientInfo, Handle);
default:
LOG_ERROR_S(ctx, NKikimrServices::PERSQUEUE, "Unexpected " << EventStr("StateInit", ev));
@@ -278,7 +278,7 @@ private:
STFUNC(StateIdle)
{
- NPersQueue::TCounterTimeKeeper keeper(Counters.Cumulative()[COUNTER_PQ_TABLET_CPU_USAGE]);
+ NPersQueue::TCounterTimeKeeper keeper(Counters.Cumulative()[COUNTER_PQ_TABLET_CPU_USAGE]);
LOG_TRACE_S(ctx, NKikimrServices::PERSQUEUE, EventStr("StateIdle", ev));
@@ -289,7 +289,7 @@ private:
HFuncTraced(TEvPQ::TEvBlobResponse, Handle);
HFuncTraced(TEvPQ::TEvWrite, HandleOnIdle);
HFuncTraced(TEvPQ::TEvRead, Handle);
- HFuncTraced(NReadSpeedLimiterEvents::TEvResponse, Handle);
+ HFuncTraced(NReadSpeedLimiterEvents::TEvResponse, Handle);
HFuncTraced(TEvPQ::TEvReadTimeout, Handle);
HFuncTraced(TEvents::TEvPoisonPill, Handle);
HFuncTraced(TEvPQ::TEvMonRequest, HandleMonitoring);
@@ -300,11 +300,11 @@ private:
HFuncTraced(TEvPQ::TEvSetClientInfo, Handle);
HFuncTraced(TEvPQ::TEvPartitionOffsets, Handle);
HFuncTraced(TEvPQ::TEvPartitionStatus, Handle);
- HFuncTraced(TEvPersQueue::TEvReportPartitionError, Handle);
+ HFuncTraced(TEvPersQueue::TEvReportPartitionError, Handle);
HFuncTraced(TEvPQ::TEvChangeOwner, Handle);
HFuncTraced(TEvPersQueue::TEvHasDataInfo, Handle);
- HFuncTraced(TEvPQ::TEvMirrorerCounters, Handle);
- HFuncTraced(NReadSpeedLimiterEvents::TEvCounters, Handle);
+ HFuncTraced(TEvPQ::TEvMirrorerCounters, Handle);
+ HFuncTraced(NReadSpeedLimiterEvents::TEvCounters, Handle);
HFuncTraced(TEvPQ::TEvProxyResponse, Handle);
HFuncTraced(TEvPQ::TEvError, Handle);
HFuncTraced(TEvPQ::TEvGetPartitionClientInfo, Handle);
@@ -325,7 +325,7 @@ private:
STFUNC(StateWrite)
{
- NPersQueue::TCounterTimeKeeper keeper(Counters.Cumulative()[COUNTER_PQ_TABLET_CPU_USAGE]);
+ NPersQueue::TCounterTimeKeeper keeper(Counters.Cumulative()[COUNTER_PQ_TABLET_CPU_USAGE]);
LOG_TRACE_S(ctx, NKikimrServices::PERSQUEUE, EventStr("StateWrite", ev));
@@ -337,7 +337,7 @@ private:
HFuncTraced(TEvPQ::TEvBlobResponse, Handle);
HFuncTraced(TEvPQ::TEvWrite, HandleOnWrite);
HFuncTraced(TEvPQ::TEvRead, Handle);
- HFuncTraced(NReadSpeedLimiterEvents::TEvResponse, Handle);
+ HFuncTraced(NReadSpeedLimiterEvents::TEvResponse, Handle);
HFuncTraced(TEvPQ::TEvReadTimeout, Handle);
HFuncTraced(TEvents::TEvPoisonPill, Handle);
HFuncTraced(TEvPQ::TEvMonRequest, HandleMonitoring);
@@ -347,12 +347,12 @@ private:
HFuncTraced(TEvPQ::TEvSetClientInfo, Handle);
HFuncTraced(TEvPQ::TEvPartitionOffsets, Handle);
HFuncTraced(TEvPQ::TEvPartitionStatus, Handle);
- HFuncTraced(TEvPersQueue::TEvReportPartitionError, Handle);
+ HFuncTraced(TEvPersQueue::TEvReportPartitionError, Handle);
HFuncTraced(TEvPQ::TEvChangeOwner, Handle);
HFuncTraced(TEvPQ::TEvChangeConfig, Handle);
HFuncTraced(TEvPersQueue::TEvHasDataInfo, Handle);
- HFuncTraced(TEvPQ::TEvMirrorerCounters, Handle);
- HFuncTraced(NReadSpeedLimiterEvents::TEvCounters, Handle);
+ HFuncTraced(TEvPQ::TEvMirrorerCounters, Handle);
+ HFuncTraced(NReadSpeedLimiterEvents::TEvCounters, Handle);
HFuncTraced(TEvPQ::TEvProxyResponse, Handle);
HFuncTraced(TEvPQ::TEvError, Handle);
HFuncTraced(TEvPQ::TEvReserveBytes, Handle);
@@ -379,8 +379,8 @@ private:
void CheckHeadConsistency() const;
- std::pair<TInstant, TInstant> GetTime(const TUserInfo& userInfo, ui64 offset) const;
- TInstant GetWriteTimeEstimate(ui64 offset) const;
+ std::pair<TInstant, TInstant> GetTime(const TUserInfo& userInfo, ui64 offset) const;
+ TInstant GetWriteTimeEstimate(ui64 offset) const;
ui32 NextChannel(bool isHead, ui32 blobSize);
@@ -568,7 +568,7 @@ private:
TString DbId;
TString FolderId;
- TUsersInfoStorage UsersInfoStorage;
+ TUsersInfoStorage UsersInfoStorage;
std::deque<TString> UpdateUserInfoTimestamp;
bool ReadingTimestamp;
@@ -606,7 +606,7 @@ private:
ui32 WriteNewMessages;
ui32 WriteNewMessagesInternal;
- TInstant CurrentTimestamp;
+ TInstant CurrentTimestamp;
bool DiskIsFull;
@@ -632,7 +632,7 @@ private:
TInstant WriteTimestamp;
TInstant WriteTimestampEstimate;
- bool ManageWriteTimestampEstimate = true;
+ bool ManageWriteTimestampEstimate = true;
NSlidingWindow::TSlidingWindow<NSlidingWindow::TMaxOperation<ui64>> WriteLagMs;
THolder<TPercentileCounter> InputTimeLag;
THolder<TPercentileCounter> MessageSize;
@@ -656,10 +656,10 @@ private:
TString TopicWriteQuoterPath;
TString TopicWriteQuotaResourcePath;
ui64 NextTopicWriteQuotaRequestCookie = 1;
-
- TDeque<NKikimrPQ::TStatusResponse::TErrorMessage> Errors;
-
- THolder<TMirrorerInfo> Mirrorer;
+
+ TDeque<NKikimrPQ::TStatusResponse::TErrorMessage> Errors;
+
+ THolder<TMirrorerInfo> Mirrorer;
};
}// NPQ
diff --git a/ydb/core/persqueue/pq.h b/ydb/core/persqueue/pq.h
index 8f4bdadd5e..e5f3309ba9 100644
--- a/ydb/core/persqueue/pq.h
+++ b/ydb/core/persqueue/pq.h
@@ -2,7 +2,7 @@
#include <ydb/core/base/blobstorage.h>
-#include <library/cpp/actors/core/actorid.h>
+#include <library/cpp/actors/core/actorid.h>
namespace NKikimr {
diff --git a/ydb/core/persqueue/pq_impl.cpp b/ydb/core/persqueue/pq_impl.cpp
index 7fffb27cb7..9937dbe561 100644
--- a/ydb/core/persqueue/pq_impl.cpp
+++ b/ydb/core/persqueue/pq_impl.cpp
@@ -1,6 +1,6 @@
-
+
#include "pq_impl.h"
-#include "event_helpers.h"
+#include "event_helpers.h"
#include "partition.h"
#include "read.h"
#include <ydb/core/persqueue/config/config.h>
@@ -25,7 +25,7 @@ namespace NPQ {
const TString TMP_REQUEST_MARKER = "__TMP__REQUEST__MARKER__";
const ui32 CACHE_SIZE = 100 << 20; //100mb per tablet by default
const ui32 MAX_BYTES = 25 * 1024 * 1024;
-const TDuration TOTAL_TIMEOUT = TDuration::Seconds(120);
+const TDuration TOTAL_TIMEOUT = TDuration::Seconds(120);
static constexpr ui32 MAX_SOURCE_ID_LENGTH = 10240;
struct TPartitionInfo {
@@ -34,17 +34,17 @@ struct TPartitionInfo {
: Actor(actor)
, KeyRange(std::move(keyRange))
, InitDone(initDone)
- {
- Baseline.Populate(baseline);
- }
+ {
+ Baseline.Populate(baseline);
+ }
TPartitionInfo(const TPartitionInfo& info)
: Actor(info.Actor)
, KeyRange(info.KeyRange)
, InitDone(info.InitDone)
- {
- Baseline.Populate(info.Baseline);
- }
+ {
+ Baseline.Populate(info.Baseline);
+ }
TActorId Actor;
TMaybe<TPartitionKeyRange> KeyRange;
@@ -143,9 +143,9 @@ private:
auto partResp = Response->Record.MutablePartitionResponse()->MutableCmdReadResult();
- partResp->SetMaxOffset(res.GetMaxOffset());
- partResp->SetSizeLag(res.GetSizeLag());
- partResp->SetWaitQuotaTimeMs(partResp->GetWaitQuotaTimeMs() + res.GetWaitQuotaTimeMs());
+ partResp->SetMaxOffset(res.GetMaxOffset());
+ partResp->SetSizeLag(res.GetSizeLag());
+ partResp->SetWaitQuotaTimeMs(partResp->GetWaitQuotaTimeMs() + res.GetWaitQuotaTimeMs());
for (ui32 i = 0; i < res.ResultSize(); ++i) {
if (!res.GetResult(i).HasPartNo() || res.GetResult(i).GetPartNo() == 0) {
@@ -343,7 +343,7 @@ public:
AnswerAndDie(ctx);
return;
}
- ctx.Schedule(TOTAL_TIMEOUT, new TEvents::TEvWakeup());
+ ctx.Schedule(TOTAL_TIMEOUT, new TEvents::TEvWakeup());
}
private:
@@ -538,10 +538,10 @@ private:
void TPersQueue::ReplyError(const TActorContext& ctx, const ui64 responseCookie, NPersQueue::NErrorCode::EErrorCode errorCode, const TString& error)
{
- ReplyPersQueueError(
- ctx.SelfID, ctx, TabletID(), TopicName, Nothing(), *Counters, NKikimrServices::PERSQUEUE,
- responseCookie, errorCode, error
- );
+ ReplyPersQueueError(
+ ctx.SelfID, ctx, TabletID(), TopicName, Nothing(), *Counters, NKikimrServices::PERSQUEUE,
+ responseCookie, errorCode, error
+ );
}
void TPersQueue::FillMeteringParams(const TActorContext& ctx)
@@ -621,8 +621,8 @@ void TPersQueue::ApplyNewConfigAndReply(const TActorContext& ctx)
Partitions.emplace(partitionId, TPartitionInfo(
ctx.Register(new TPartition(TabletID(), partitionId, ctx.SelfID, CacheActor, TopicName, TopicPath, LocalDC, DCId, Config, *Counters, ctx, true)),
GetPartitionKeyRange(partition),
- true,
- *Counters
+ true,
+ *Counters
));
// InitCompleted is true because this partition is empty
@@ -729,8 +729,8 @@ void TPersQueue::ReadConfig(const NKikimrClient::TKeyValueResponse::TReadResult&
Partitions.emplace(partitionId, TPartitionInfo(
ctx.Register(new TPartition(TabletID(), partitionId, ctx.SelfID, CacheActor, TopicName, TopicPath, LocalDC, DCId, Config, *Counters, ctx, false)),
GetPartitionKeyRange(partition),
- false,
- *Counters
+ false,
+ *Counters
));
}
ConfigInited = true;
@@ -849,19 +849,19 @@ void TPersQueue::Handle(TEvPQ::TEvPartitionCounters::TPtr& ev, const TActorConte
Y_VERIFY(it != Partitions.end());
auto diff = ev->Get()->Counters.MakeDiffForAggr(it->second.Baseline);
ui64 cpuUsage = diff->Cumulative()[COUNTER_PQ_TABLET_CPU_USAGE].Get();
- ui64 networkBytesUsage = diff->Cumulative()[COUNTER_PQ_TABLET_NETWORK_BYTES_USAGE].Get();
- if (ResourceMetrics) {
- if (cpuUsage > 0) {
+ ui64 networkBytesUsage = diff->Cumulative()[COUNTER_PQ_TABLET_NETWORK_BYTES_USAGE].Get();
+ if (ResourceMetrics) {
+ if (cpuUsage > 0) {
ResourceMetrics->CPU.Increment(cpuUsage);
- }
- if (networkBytesUsage > 0) {
- ResourceMetrics->Network.Increment(networkBytesUsage);
- }
- if (cpuUsage > 0 || networkBytesUsage > 0) {
+ }
+ if (networkBytesUsage > 0) {
+ ResourceMetrics->Network.Increment(networkBytesUsage);
+ }
+ if (cpuUsage > 0 || networkBytesUsage > 0) {
ResourceMetrics->TryUpdate(ctx);
- }
+ }
}
-
+
Counters->Populate(*diff.Get());
ev->Get()->Counters.RememberCurrentStateAsBaseline(it->second.Baseline);
@@ -1413,8 +1413,8 @@ void TPersQueue::HandleWriteRequest(const ui64 responseCookie, const TActorId& p
TVector <TEvPQ::TEvWrite::TMsg> msgs;
- bool mirroredPartition = Config.GetPartitionConfig().HasMirrorFrom();
-
+ bool mirroredPartition = Config.GetPartitionConfig().HasMirrorFrom();
+
if (!req.GetIsDirectWrite()) {
if (!req.HasMessageNo()) {
ReplyError(ctx, responseCookie, NPersQueue::NErrorCode::BAD_REQUEST, "MessageNo must be set for writes");
@@ -1427,7 +1427,7 @@ void TPersQueue::HandleWriteRequest(const ui64 responseCookie, const TActorId& p
return;
}
}
-
+
if (req.HasCmdWriteOffset() && req.GetCmdWriteOffset() < 0) {
ReplyError(ctx, responseCookie, NPersQueue::NErrorCode::BAD_REQUEST, "CmdWriteOffset can't be negative");
return;
@@ -1479,14 +1479,14 @@ void TPersQueue::HandleWriteRequest(const ui64 responseCookie, const TActorId& p
errorStr = "TotalSize is incorrect";
} else if (cmd.GetSourceId().size() > MAX_SOURCE_ID_LENGTH) {
errorStr = "Too big SourceId";
- } else if (mirroredPartition && !cmd.GetDisableDeduplication()) {
- errorStr = "Write to mirrored topic is forbiden";
+ } else if (mirroredPartition && !cmd.GetDisableDeduplication()) {
+ errorStr = "Write to mirrored topic is forbiden";
}
- ui64 createTimestampMs = 0, writeTimestampMs = 0;
+ ui64 createTimestampMs = 0, writeTimestampMs = 0;
if (cmd.HasCreateTimeMS() && cmd.GetCreateTimeMS() >= 0)
- createTimestampMs = cmd.GetCreateTimeMS();
+ createTimestampMs = cmd.GetCreateTimeMS();
if (cmd.HasWriteTimeMS() && cmd.GetWriteTimeMS() > 0) {
- writeTimestampMs = cmd.GetWriteTimeMS();
+ writeTimestampMs = cmd.GetWriteTimeMS();
if (!cmd.GetDisableDeduplication()) {
errorStr = "WriteTimestamp avail only without deduplication";
}
@@ -1498,7 +1498,7 @@ void TPersQueue::HandleWriteRequest(const ui64 responseCookie, const TActorId& p
}
ui32 mSize = MAX_BLOB_PART_SIZE - cmd.GetSourceId().size() - sizeof(ui32) - TClientBlob::OVERHEAD; //megaqc - remove this
Y_VERIFY(mSize > 204800);
- ui64 receiveTimestampMs = TAppData::TimeProvider->Now().MilliSeconds();
+ ui64 receiveTimestampMs = TAppData::TimeProvider->Now().MilliSeconds();
bool disableDeduplication = cmd.GetDisableDeduplication();
if (cmd.GetData().size() > mSize) {
if (cmd.HasPartNo()) {
@@ -1525,10 +1525,10 @@ void TPersQueue::HandleWriteRequest(const ui64 responseCookie, const TActorId& p
pos += mSize - diff;
diff = 0;
msgs.push_back({cmd.GetSourceId(), static_cast<ui64>(cmd.GetSeqNo()), partNo,
- totalParts, totalSize, createTimestampMs, receiveTimestampMs,
+ totalParts, totalSize, createTimestampMs, receiveTimestampMs,
disableDeduplication, writeTimestampMs, data, uncompressedSize,
cmd.GetPartitionKey(), cmd.GetExplicitHash(), cmd.GetExternalOperation()
- });
+ });
partNo++;
uncompressedSize = 0;
LOG_DEBUG_S(ctx, NKikimrServices::PERSQUEUE, "got client PART message topic: " << TopicName << " partition: " << req.GetPartition()
@@ -1539,12 +1539,12 @@ void TPersQueue::HandleWriteRequest(const ui64 responseCookie, const TActorId& p
Y_VERIFY(partNo == totalParts);
} else {
msgs.push_back({cmd.GetSourceId(), static_cast<ui64>(cmd.GetSeqNo()), static_cast<ui16>(cmd.HasPartNo() ? cmd.GetPartNo() : 0),
- static_cast<ui16>(cmd.HasPartNo() ? cmd.GetTotalParts() : 1),
- static_cast<ui32>(cmd.HasTotalSize() ? cmd.GetTotalSize() : cmd.GetData().Size()),
- createTimestampMs, receiveTimestampMs, disableDeduplication, writeTimestampMs, cmd.GetData(),
+ static_cast<ui16>(cmd.HasPartNo() ? cmd.GetTotalParts() : 1),
+ static_cast<ui32>(cmd.HasTotalSize() ? cmd.GetTotalSize() : cmd.GetData().Size()),
+ createTimestampMs, receiveTimestampMs, disableDeduplication, writeTimestampMs, cmd.GetData(),
cmd.HasUncompressedSize() ? cmd.GetUncompressedSize() : 0u, cmd.GetPartitionKey(), cmd.GetExplicitHash(),
cmd.GetExternalOperation()
- });
+ });
}
LOG_DEBUG_S(ctx, NKikimrServices::PERSQUEUE, "got client message topic: " << TopicName <<
" partition: " << req.GetPartition() <<
diff --git a/ydb/core/persqueue/pq_impl.h b/ydb/core/persqueue/pq_impl.h
index 60e04cf9bb..bc3bfb10ba 100644
--- a/ydb/core/persqueue/pq_impl.h
+++ b/ydb/core/persqueue/pq_impl.h
@@ -1,5 +1,5 @@
#pragma once
-
+
#include "percentile_counter.h"
#include <ydb/core/keyvalue/keyvalue_flat_impl.h>
#include <ydb/core/tablet/tablet_counters.h>
diff --git a/ydb/core/persqueue/pq_l2_cache.h b/ydb/core/persqueue/pq_l2_cache.h
index c672b190ab..f9fcccbc8e 100644
--- a/ydb/core/persqueue/pq_l2_cache.h
+++ b/ydb/core/persqueue/pq_l2_cache.h
@@ -1,5 +1,5 @@
#pragma once
-
+
#include "read.h"
#include "pq_l2_service.h"
@@ -81,8 +81,8 @@ public:
: Cache(SizeInBytes(1024*1024)/MAX_BLOB_SIZE) // It's some "much bigger then we need" size here.
, MaxSize(SizeInBytes(params.MaxSizeMB))
, CurrentSize(0)
- , KeepTime(params.KeepTime)
- , RetentionTime(TDuration::Zero())
+ , KeepTime(params.KeepTime)
+ , RetentionTime(TDuration::Zero())
, Counters(countersGroup)
{}
diff --git a/ydb/core/persqueue/pq_l2_service.h b/ydb/core/persqueue/pq_l2_service.h
index c71418c34d..a9c5eefd16 100644
--- a/ydb/core/persqueue/pq_l2_service.h
+++ b/ydb/core/persqueue/pq_l2_service.h
@@ -16,7 +16,7 @@ inline TActorId MakePersQueueL2CacheID() {
struct TCacheL2Parameters {
ui32 MaxSizeMB;
- TDuration KeepTime;
+ TDuration KeepTime;
};
IActor* CreateNodePersQueueL2Cache(const TCacheL2Parameters&, TIntrusivePtr<NMonitoring::TDynamicCounters>);
diff --git a/ydb/core/persqueue/pq_ut.cpp b/ydb/core/persqueue/pq_ut.cpp
index 5d7a2d05c5..13a6122910 100644
--- a/ydb/core/persqueue/pq_ut.cpp
+++ b/ydb/core/persqueue/pq_ut.cpp
@@ -1905,22 +1905,22 @@ Y_UNIT_TEST(TestPQCacheSizeManagement) {
Y_UNIT_TEST(TestOffsetEstimation) {
std::deque<NPQ::TDataKey> container = {
- {NPQ::TKey(NPQ::TKeyPrefix::EType::TypeNone, 0, 1, 0, 0, 0), 0, TInstant::Seconds(1), 10},
- {NPQ::TKey(NPQ::TKeyPrefix::EType::TypeNone, 0, 2, 0, 0, 0), 0, TInstant::Seconds(1), 10},
- {NPQ::TKey(NPQ::TKeyPrefix::EType::TypeNone, 0, 3, 0, 0, 0), 0, TInstant::Seconds(2), 10},
- {NPQ::TKey(NPQ::TKeyPrefix::EType::TypeNone, 0, 4, 0, 0, 0), 0, TInstant::Seconds(2), 10},
- {NPQ::TKey(NPQ::TKeyPrefix::EType::TypeNone, 0, 5, 0, 0, 0), 0, TInstant::Seconds(3), 10},
- {NPQ::TKey(NPQ::TKeyPrefix::EType::TypeNone, 0, 6, 0, 0, 0), 0, TInstant::Seconds(3), 10},
+ {NPQ::TKey(NPQ::TKeyPrefix::EType::TypeNone, 0, 1, 0, 0, 0), 0, TInstant::Seconds(1), 10},
+ {NPQ::TKey(NPQ::TKeyPrefix::EType::TypeNone, 0, 2, 0, 0, 0), 0, TInstant::Seconds(1), 10},
+ {NPQ::TKey(NPQ::TKeyPrefix::EType::TypeNone, 0, 3, 0, 0, 0), 0, TInstant::Seconds(2), 10},
+ {NPQ::TKey(NPQ::TKeyPrefix::EType::TypeNone, 0, 4, 0, 0, 0), 0, TInstant::Seconds(2), 10},
+ {NPQ::TKey(NPQ::TKeyPrefix::EType::TypeNone, 0, 5, 0, 0, 0), 0, TInstant::Seconds(3), 10},
+ {NPQ::TKey(NPQ::TKeyPrefix::EType::TypeNone, 0, 6, 0, 0, 0), 0, TInstant::Seconds(3), 10},
};
- UNIT_ASSERT_EQUAL(NPQ::GetOffsetEstimate({}, TInstant::MilliSeconds(0), 9999), 9999);
- UNIT_ASSERT_EQUAL(NPQ::GetOffsetEstimate(container, TInstant::MilliSeconds(0), 9999), 1);
- UNIT_ASSERT_EQUAL(NPQ::GetOffsetEstimate(container, TInstant::MilliSeconds(500), 9999), 1);
- UNIT_ASSERT_EQUAL(NPQ::GetOffsetEstimate(container, TInstant::MilliSeconds(1000), 9999), 1);
- UNIT_ASSERT_EQUAL(NPQ::GetOffsetEstimate(container, TInstant::MilliSeconds(1500), 9999), 3);
- UNIT_ASSERT_EQUAL(NPQ::GetOffsetEstimate(container, TInstant::MilliSeconds(2000), 9999), 3);
- UNIT_ASSERT_EQUAL(NPQ::GetOffsetEstimate(container, TInstant::MilliSeconds(2500), 9999), 5);
- UNIT_ASSERT_EQUAL(NPQ::GetOffsetEstimate(container, TInstant::MilliSeconds(3000), 9999), 5);
- UNIT_ASSERT_EQUAL(NPQ::GetOffsetEstimate(container, TInstant::MilliSeconds(3500), 9999), 9999);
+ UNIT_ASSERT_EQUAL(NPQ::GetOffsetEstimate({}, TInstant::MilliSeconds(0), 9999), 9999);
+ UNIT_ASSERT_EQUAL(NPQ::GetOffsetEstimate(container, TInstant::MilliSeconds(0), 9999), 1);
+ UNIT_ASSERT_EQUAL(NPQ::GetOffsetEstimate(container, TInstant::MilliSeconds(500), 9999), 1);
+ UNIT_ASSERT_EQUAL(NPQ::GetOffsetEstimate(container, TInstant::MilliSeconds(1000), 9999), 1);
+ UNIT_ASSERT_EQUAL(NPQ::GetOffsetEstimate(container, TInstant::MilliSeconds(1500), 9999), 3);
+ UNIT_ASSERT_EQUAL(NPQ::GetOffsetEstimate(container, TInstant::MilliSeconds(2000), 9999), 3);
+ UNIT_ASSERT_EQUAL(NPQ::GetOffsetEstimate(container, TInstant::MilliSeconds(2500), 9999), 5);
+ UNIT_ASSERT_EQUAL(NPQ::GetOffsetEstimate(container, TInstant::MilliSeconds(3000), 9999), 5);
+ UNIT_ASSERT_EQUAL(NPQ::GetOffsetEstimate(container, TInstant::MilliSeconds(3500), 9999), 9999);
}
Y_UNIT_TEST(TestMaxTimeLagRewind) {
diff --git a/ydb/core/persqueue/pq_ut.h b/ydb/core/persqueue/pq_ut.h
index e21ba73316..3db93bc104 100644
--- a/ydb/core/persqueue/pq_ut.h
+++ b/ydb/core/persqueue/pq_ut.h
@@ -1,7 +1,7 @@
#pragma once
-
-#include "pq.h"
-#include "user_info.h"
+
+#include "pq.h"
+#include "user_info.h"
#include <ydb/core/testlib/basics/runtime.h>
#include <ydb/core/tablet_flat/tablet_flat_executed.h>
@@ -1093,7 +1093,7 @@ void FillDeprecatedUserInfo(NKikimrClient::TKeyValueRequest_TCmdWrite* write, TS
NPQ::TKeyPrefix ikeyDeprecated(NPQ::TKeyPrefix::TypeInfo, partition, NPQ::TKeyPrefix::MarkUserDeprecated);
ikeyDeprecated.Append(client.c_str(), client.size());
- TBuffer idataDeprecated = NPQ::NDeprecatedUserData::Serialize(offset, gen, step, session);
+ TBuffer idataDeprecated = NPQ::NDeprecatedUserData::Serialize(offset, gen, step, session);
write->SetKey(ikeyDeprecated.Data(), ikeyDeprecated.Size());
write->SetValue(idataDeprecated.Data(), idataDeprecated.Size());
}
diff --git a/ydb/core/persqueue/read.h b/ydb/core/persqueue/read.h
index 483add46e8..d0bc79c692 100644
--- a/ydb/core/persqueue/read.h
+++ b/ydb/core/persqueue/read.h
@@ -1,5 +1,5 @@
#pragma once
-
+
#include "partition.h"
#include "pq_l2_service.h"
#include "cache_eviction.h"
diff --git a/ydb/core/persqueue/read_balancer.cpp b/ydb/core/persqueue/read_balancer.cpp
index 665c854cb2..4eb9c7206d 100644
--- a/ydb/core/persqueue/read_balancer.cpp
+++ b/ydb/core/persqueue/read_balancer.cpp
@@ -9,9 +9,9 @@ namespace NPQ {
using namespace NTabletFlatExecutor;
-static constexpr TDuration ACL_SUCCESS_RETRY_TIMEOUT = TDuration::Seconds(30);
-static constexpr TDuration ACL_ERROR_RETRY_TIMEOUT = TDuration::Seconds(5);
-static constexpr TDuration ACL_EXPIRATION_TIMEOUT = TDuration::Minutes(5);
+static constexpr TDuration ACL_SUCCESS_RETRY_TIMEOUT = TDuration::Seconds(30);
+static constexpr TDuration ACL_ERROR_RETRY_TIMEOUT = TDuration::Seconds(5);
+static constexpr TDuration ACL_EXPIRATION_TIMEOUT = TDuration::Minutes(5);
bool TPersQueueReadBalancer::TTxPreInit::Execute(TTransactionContext& txc, const TActorContext& ctx) {
Y_UNUSED(ctx);
@@ -319,7 +319,7 @@ void TPersQueueReadBalancer::Handle(TEvPersQueue::TEvCheckACL::TPtr &ev, const T
return;
}
- if (ctx.Now() > LastACLUpdate + ACL_EXPIRATION_TIMEOUT || Topic.empty()) { //Topic.empty is only for tests
+ if (ctx.Now() > LastACLUpdate + ACL_EXPIRATION_TIMEOUT || Topic.empty()) { //Topic.empty is only for tests
WaitingACLRequests.push_back(ev);
return;
}
@@ -394,7 +394,7 @@ void TPersQueueReadBalancer::Handle(TEvPersQueue::TEvWakeupClient::TPtr &ev, con
}
void TPersQueueReadBalancer::Handle(TEvPersQueue::TEvDescribe::TPtr &ev, const TActorContext& ctx) {
- if (ctx.Now() > LastACLUpdate + ACL_EXPIRATION_TIMEOUT || Topic.empty()) { //Topic.empty is only for tests
+ if (ctx.Now() > LastACLUpdate + ACL_EXPIRATION_TIMEOUT || Topic.empty()) { //Topic.empty is only for tests
WaitingDescribeRequests.push_back(ev);
return;
} else {
@@ -685,7 +685,7 @@ void TPersQueueReadBalancer::Handle(NSchemeShard::TEvSchemeShard::TEvDescribeSch
AnswerWaitingRequests(ctx);
} else {
LOG_DEBUG_S(ctx, NKikimrServices::PERSQUEUE_READ_BALANCER, GetPrefix() << "couldn't receive ACL due to " << record.GetStatus());
- ctx.Schedule(ACL_ERROR_RETRY_TIMEOUT, new TEvPersQueue::TEvUpdateACL());
+ ctx.Schedule(ACL_ERROR_RETRY_TIMEOUT, new TEvPersQueue::TEvUpdateACL());
}
}
@@ -715,7 +715,7 @@ void TPersQueueReadBalancer::GetACL(const TActorContext& ctx) {
if (WaitingForACL) // if there is request infly
return;
if (SchemeShardId == 0) {
- ctx.Schedule(ACL_SUCCESS_RETRY_TIMEOUT, new TEvPersQueue::TEvUpdateACL());
+ ctx.Schedule(ACL_SUCCESS_RETRY_TIMEOUT, new TEvPersQueue::TEvUpdateACL());
} else {
WaitingForACL = true;
RequestTabletIfNeeded(SchemeShardId, ctx);
@@ -771,14 +771,14 @@ void TPersQueueReadBalancer::TClientInfo::AddSession(const ui32 group, const THa
}
auto it = ClientGroupsInfo.find(group);
- it->second.SessionsInfo.insert({
- std::make_pair(pipe, it->second.RandomNumber),
- TClientGroupInfo::TSessionInfo(
- record.GetSession(), sender,
- record.HasClientNode() ? record.GetClientNode() : "none",
- sender.NodeId(), TAppData::TimeProvider->Now()
- )
- });
+ it->second.SessionsInfo.insert({
+ std::make_pair(pipe, it->second.RandomNumber),
+ TClientGroupInfo::TSessionInfo(
+ record.GetSession(), sender,
+ record.HasClientNode() ? record.GetClientNode() : "none",
+ sender.NodeId(), TAppData::TimeProvider->Now()
+ )
+ });
}
@@ -888,7 +888,7 @@ void TPersQueueReadBalancer::Handle(TEvPersQueue::TEvGetReadSessionsInfo::TPtr&
pi->SetClientNode(jt->second.ClientNode);
pi->SetProxyNodeId(jt->second.ProxyNodeId);
pi->SetSession(jt->second.Session);
- pi->SetTimestamp(jt->second.Timestamp.Seconds());
+ pi->SetTimestamp(jt->second.Timestamp.Seconds());
} else {
pi->SetClientNode("");
pi->SetProxyNodeId(0);
diff --git a/ydb/core/persqueue/read_balancer.h b/ydb/core/persqueue/read_balancer.h
index 1fefe3f49f..6a07edcb78 100644
--- a/ydb/core/persqueue/read_balancer.h
+++ b/ydb/core/persqueue/read_balancer.h
@@ -334,7 +334,7 @@ class TPersQueueReadBalancer : public TActor<TPersQueueReadBalancer>, public TTa
struct TClientGroupInfo {
struct TSessionInfo {
- TSessionInfo(const TString& session, const TActorId sender, const TString& clientNode, ui32 proxyNodeId, TInstant ts)
+ TSessionInfo(const TString& session, const TActorId sender, const TString& clientNode, ui32 proxyNodeId, TInstant ts)
: Session(session)
, Sender(sender)
, NumSuspended(0)
@@ -351,7 +351,7 @@ class TPersQueueReadBalancer : public TActor<TPersQueueReadBalancer>, public TTa
TString ClientNode;
ui32 ProxyNodeId;
- TInstant Timestamp;
+ TInstant Timestamp;
};
TString ClientId;
diff --git a/ydb/core/persqueue/read_speed_limiter.cpp b/ydb/core/persqueue/read_speed_limiter.cpp
index 47be44036a..793338420d 100644
--- a/ydb/core/persqueue/read_speed_limiter.cpp
+++ b/ydb/core/persqueue/read_speed_limiter.cpp
@@ -1,184 +1,184 @@
-#include "read_speed_limiter.h"
-#include "event_helpers.h"
-
+#include "read_speed_limiter.h"
+#include "event_helpers.h"
+
#include <ydb/core/base/appdata.h>
#include <ydb/core/base/counters.h>
#include <ydb/core/persqueue/percentile_counter.h>
#include <ydb/library/persqueue/topic_parser/topic_parser.h>
-
-#include <library/cpp/actors/core/log.h>
-
-#include <util/string/join.h>
-#include <util/string/vector.h>
-
-namespace NKikimr {
-namespace NPQ {
-
-const TDuration UPDATE_COUNTERS_INTERVAL = TDuration::Seconds(5);
-const TDuration DO_NOT_QUOTE_AFTER_ERROR_PERIOD = TDuration::Seconds(5);
-
-const TString TReadSpeedLimiter::READ_QUOTA_ROOT_PATH = "read-quota";
-
-constexpr NKikimrServices::TActivity::EType TReadSpeedLimiter::ActorActivityType() {
- return NKikimrServices::TActivity::PERSQUEUE_READ_SPEED_LIMITER;
-}
-
-TReadSpeedLimiter::TReadSpeedLimiter(
- TActorId tabletActor,
- TActorId partitionActor,
- ui64 tabletId,
- const TString& topicName,
- ui32 partition,
- const TString& user,
- const TTabletCountersBase& counters
-)
- : TabletActor(tabletActor)
- , PartitionActor(partitionActor)
- , TabletId(tabletId)
- , TopicName(topicName)
- , Partition(partition)
- , User(user)
- , ConsumerPath(NPersQueue::ConvertOldConsumerName(user))
- , ReadCreditBytes(AppData()->PQConfig.GetQuotingConfig().GetReadCreditBytes())
-{
- Counters.Populate(counters);
-
- auto userParts = SplitString(ConsumerPath, "/"); // account/folder/topic // account is first element
-
- const TString account = userParts[0];
- userParts[0] = READ_QUOTA_ROOT_PATH; // read-quota/folder/topic
-
- const auto& quotingConfig = AppData()->PQConfig.GetQuotingConfig();
- KesusPath = Join("/", quotingConfig.GetQuotersDirectoryPath(), account);
- QuotaResourcePath = JoinSeq("/", userParts);
- LOG_INFO_S(TActivationContext::AsActorContext(), NKikimrServices::PQ_READ_SPEED_LIMITER,
- LimiterDescription() <<" kesus=" << KesusPath << " resource_path=" << QuotaResourcePath);
-}
-
-
-void TReadSpeedLimiter::Bootstrap(const TActorContext& ctx) {
- Become(&TThis::StateWork);
- ctx.Schedule(UPDATE_COUNTERS_INTERVAL, new TEvPQ::TEvUpdateCounters);
-
- auto counters = AppData()->Counters;
- if (counters && TopicName.Contains("--")) {
+
+#include <library/cpp/actors/core/log.h>
+
+#include <util/string/join.h>
+#include <util/string/vector.h>
+
+namespace NKikimr {
+namespace NPQ {
+
+const TDuration UPDATE_COUNTERS_INTERVAL = TDuration::Seconds(5);
+const TDuration DO_NOT_QUOTE_AFTER_ERROR_PERIOD = TDuration::Seconds(5);
+
+const TString TReadSpeedLimiter::READ_QUOTA_ROOT_PATH = "read-quota";
+
+constexpr NKikimrServices::TActivity::EType TReadSpeedLimiter::ActorActivityType() {
+ return NKikimrServices::TActivity::PERSQUEUE_READ_SPEED_LIMITER;
+}
+
+TReadSpeedLimiter::TReadSpeedLimiter(
+ TActorId tabletActor,
+ TActorId partitionActor,
+ ui64 tabletId,
+ const TString& topicName,
+ ui32 partition,
+ const TString& user,
+ const TTabletCountersBase& counters
+)
+ : TabletActor(tabletActor)
+ , PartitionActor(partitionActor)
+ , TabletId(tabletId)
+ , TopicName(topicName)
+ , Partition(partition)
+ , User(user)
+ , ConsumerPath(NPersQueue::ConvertOldConsumerName(user))
+ , ReadCreditBytes(AppData()->PQConfig.GetQuotingConfig().GetReadCreditBytes())
+{
+ Counters.Populate(counters);
+
+ auto userParts = SplitString(ConsumerPath, "/"); // account/folder/topic // account is first element
+
+ const TString account = userParts[0];
+ userParts[0] = READ_QUOTA_ROOT_PATH; // read-quota/folder/topic
+
+ const auto& quotingConfig = AppData()->PQConfig.GetQuotingConfig();
+ KesusPath = Join("/", quotingConfig.GetQuotersDirectoryPath(), account);
+ QuotaResourcePath = JoinSeq("/", userParts);
+ LOG_INFO_S(TActivationContext::AsActorContext(), NKikimrServices::PQ_READ_SPEED_LIMITER,
+ LimiterDescription() <<" kesus=" << KesusPath << " resource_path=" << QuotaResourcePath);
+}
+
+
+void TReadSpeedLimiter::Bootstrap(const TActorContext& ctx) {
+ Become(&TThis::StateWork);
+ ctx.Schedule(UPDATE_COUNTERS_INTERVAL, new TEvPQ::TEvUpdateCounters);
+
+ auto counters = AppData()->Counters;
+ if (counters && TopicName.Contains("--")) {
QuotaWaitCounter.Reset(new TPercentileCounter(
- GetServiceCounters(counters, "pqproxy|consumerReadQuotaWait"),
- GetLabels(TopicName),
- {
- {"Client", User},
- {"ConsumerPath", ConsumerPath},
- {"sensor", "ConsumerReadQuotaWait"}
- },
- "Interval",
- TVector<std::pair<ui64, TString>>{{0, "0ms"}, {1, "1ms"}, {5, "5ms"}, {10, "10ms"},
- {20, "20ms"}, {50, "50ms"}, {100, "100ms"}, {500, "500ms"},
- {1000, "1000ms"}, {2500, "2500ms"}, {5000, "5000ms"}, {10000, "10000ms"}, {9999999, "999999ms"}},
- true
+ GetServiceCounters(counters, "pqproxy|consumerReadQuotaWait"),
+ GetLabels(TopicName),
+ {
+ {"Client", User},
+ {"ConsumerPath", ConsumerPath},
+ {"sensor", "ConsumerReadQuotaWait"}
+ },
+ "Interval",
+ TVector<std::pair<ui64, TString>>{{0, "0ms"}, {1, "1ms"}, {5, "5ms"}, {10, "10ms"},
+ {20, "20ms"}, {50, "50ms"}, {100, "100ms"}, {500, "500ms"},
+ {1000, "1000ms"}, {2500, "2500ms"}, {5000, "5000ms"}, {10000, "10000ms"}, {9999999, "999999ms"}},
+ true
));
- }
-}
-
-void TReadSpeedLimiter::Handle(TEvents::TEvPoisonPill::TPtr&, const TActorContext& ctx) {
- LOG_INFO_S(ctx, NKikimrServices::PQ_READ_SPEED_LIMITER, LimiterDescription() << " killed");
- for (const auto& event : Queue) {
- auto cookie = event.Event->Get()->Cookie;
- ReplyPersQueueError(
- TabletActor, ctx, TabletId, TopicName, Partition, Counters, NKikimrServices::PQ_READ_SPEED_LIMITER,
- cookie, NPersQueue::NErrorCode::INITIALIZING,
- TStringBuilder() << "Tablet is restarting, topic " << TopicName << " (ReadInfo) cookie " << cookie
- );
- }
- Die(ctx);
-}
-
-void TReadSpeedLimiter::HandleUpdateCounters(TEvPQ::TEvUpdateCounters::TPtr&, const TActorContext& ctx) {
- ctx.Schedule(UPDATE_COUNTERS_INTERVAL, new TEvPQ::TEvUpdateCounters);
- ctx.Send(PartitionActor, new NReadSpeedLimiterEvents::TEvCounters(Counters, User));
-}
-
-void TReadSpeedLimiter::HandleReadQuotaRequest(NReadSpeedLimiterEvents::TEvRequest::TPtr& ev, const TActorContext& ctx) {
- LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_SPEED_LIMITER,
- LimiterDescription() << " quota required for cookie=" << ev->Get()->ReadRequest->Get()->Cookie
- );
- bool hasActualErrors = ctx.Now() - LastReportedErrorTime <= DO_NOT_QUOTE_AFTER_ERROR_PERIOD;
- if ((QuotaRequestInFlight || !InProcessReadRequestCookies.empty()) && !hasActualErrors) {
- Queue.emplace_back(std::move(ev->Get()->ReadRequest), ctx.Now());
- } else {
- ApproveRead(ev->Get()->ReadRequest, ctx.Now(), ctx);
- }
-}
-
-void TReadSpeedLimiter::HandleReadQuotaConsumed(NReadSpeedLimiterEvents::TEvConsumed::TPtr& ev, const TActorContext& ctx) {
- ConsumedBytesInCredit += ev->Get()->ReadBytes;
- LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_SPEED_LIMITER, LimiterDescription()
- << "consumed read quota " << ev->Get()->ReadBytes
- << " bytes by cookie=" << ev->Get()->ReadRequestCookie
- << ", consumed in credit " << ConsumedBytesInCredit << "/" << ReadCreditBytes
- );
- auto it = InProcessReadRequestCookies.find(ev->Get()->ReadRequestCookie);
- Y_VERIFY(it != InProcessReadRequestCookies.end());
- InProcessReadRequestCookies.erase(it);
-
- if (!QuotaRequestInFlight) {
- if (ConsumedBytesInCredit >= ReadCreditBytes) {
- Send(MakeQuoterServiceID(),
- new TEvQuota::TEvRequest(
- TEvQuota::EResourceOperator::And,
- { TEvQuota::TResourceLeaf(KesusPath, QuotaResourcePath, ConsumedBytesInCredit) },
- TDuration::Max()),
- 0,
- ++CurrentQuotaRequestCookie
- );
- QuotaRequestInFlight = true;
- ConsumedBytesInCredit = 0;
- } else if (!Queue.empty()){
- ApproveRead(std::move(Queue.front().Event), Queue.front().StartWait, ctx);
- Queue.pop_front();
- }
- }
-}
-
-void TReadSpeedLimiter::HandleClearance(TEvQuota::TEvClearance::TPtr& ev, const TActorContext& ctx) {
- QuotaRequestInFlight = false;
- const ui64 cookie = ev->Cookie;
- LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_SPEED_LIMITER,
- LimiterDescription() << "Got read quota:" << ev->Get()->Result << ". Cookie: " << cookie
- );
-
- Y_VERIFY(CurrentQuotaRequestCookie == cookie);
- if (!Queue.empty()) {
- ApproveRead(std::move(Queue.front().Event), Queue.front().StartWait, ctx);
- Queue.pop_front();
- }
-
- if (Y_UNLIKELY(ev->Get()->Result != TEvQuota::TEvClearance::EResult::Success)) {
- Y_VERIFY(ev->Get()->Result != TEvQuota::TEvClearance::EResult::Deadline); // We set deadline == inf in quota request.
- if (ctx.Now() - LastReportedErrorTime > TDuration::Minutes(1)) {
- LOG_ERROR_S(ctx, NKikimrServices::PQ_READ_SPEED_LIMITER, LimiterDescription() << "Got read quota error: " << ev->Get()->Result);
- LastReportedErrorTime = ctx.Now();
- }
- return;
- }
-}
-
-void TReadSpeedLimiter::TReadSpeedLimiter::ApproveRead(TEvPQ::TEvRead::TPtr ev, TInstant startWait, const TActorContext& ctx) {
- LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_SPEED_LIMITER,
- LimiterDescription() << " approve read for cookie=" << ev->Get()->Cookie
- );
- InProcessReadRequestCookies.insert(ev->Get()->Cookie);
-
- auto waitTime = ctx.Now() - startWait;
- Send(PartitionActor, new NReadSpeedLimiterEvents::TEvResponse(ev.Release(), waitTime));
-
- if (QuotaWaitCounter) {
- QuotaWaitCounter->IncFor(waitTime.MilliSeconds());
- }
-}
-
-TString TReadSpeedLimiter::LimiterDescription() const {
- return TStringBuilder() << "topic=" << TopicName << ":" << Partition << " user=" << User << ": ";
-}
-
-}// NPQ
-}// NKikimr
+ }
+}
+
+void TReadSpeedLimiter::Handle(TEvents::TEvPoisonPill::TPtr&, const TActorContext& ctx) {
+ LOG_INFO_S(ctx, NKikimrServices::PQ_READ_SPEED_LIMITER, LimiterDescription() << " killed");
+ for (const auto& event : Queue) {
+ auto cookie = event.Event->Get()->Cookie;
+ ReplyPersQueueError(
+ TabletActor, ctx, TabletId, TopicName, Partition, Counters, NKikimrServices::PQ_READ_SPEED_LIMITER,
+ cookie, NPersQueue::NErrorCode::INITIALIZING,
+ TStringBuilder() << "Tablet is restarting, topic " << TopicName << " (ReadInfo) cookie " << cookie
+ );
+ }
+ Die(ctx);
+}
+
+void TReadSpeedLimiter::HandleUpdateCounters(TEvPQ::TEvUpdateCounters::TPtr&, const TActorContext& ctx) {
+ ctx.Schedule(UPDATE_COUNTERS_INTERVAL, new TEvPQ::TEvUpdateCounters);
+ ctx.Send(PartitionActor, new NReadSpeedLimiterEvents::TEvCounters(Counters, User));
+}
+
+void TReadSpeedLimiter::HandleReadQuotaRequest(NReadSpeedLimiterEvents::TEvRequest::TPtr& ev, const TActorContext& ctx) {
+ LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_SPEED_LIMITER,
+ LimiterDescription() << " quota required for cookie=" << ev->Get()->ReadRequest->Get()->Cookie
+ );
+ bool hasActualErrors = ctx.Now() - LastReportedErrorTime <= DO_NOT_QUOTE_AFTER_ERROR_PERIOD;
+ if ((QuotaRequestInFlight || !InProcessReadRequestCookies.empty()) && !hasActualErrors) {
+ Queue.emplace_back(std::move(ev->Get()->ReadRequest), ctx.Now());
+ } else {
+ ApproveRead(ev->Get()->ReadRequest, ctx.Now(), ctx);
+ }
+}
+
+void TReadSpeedLimiter::HandleReadQuotaConsumed(NReadSpeedLimiterEvents::TEvConsumed::TPtr& ev, const TActorContext& ctx) {
+ ConsumedBytesInCredit += ev->Get()->ReadBytes;
+ LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_SPEED_LIMITER, LimiterDescription()
+ << "consumed read quota " << ev->Get()->ReadBytes
+ << " bytes by cookie=" << ev->Get()->ReadRequestCookie
+ << ", consumed in credit " << ConsumedBytesInCredit << "/" << ReadCreditBytes
+ );
+ auto it = InProcessReadRequestCookies.find(ev->Get()->ReadRequestCookie);
+ Y_VERIFY(it != InProcessReadRequestCookies.end());
+ InProcessReadRequestCookies.erase(it);
+
+ if (!QuotaRequestInFlight) {
+ if (ConsumedBytesInCredit >= ReadCreditBytes) {
+ Send(MakeQuoterServiceID(),
+ new TEvQuota::TEvRequest(
+ TEvQuota::EResourceOperator::And,
+ { TEvQuota::TResourceLeaf(KesusPath, QuotaResourcePath, ConsumedBytesInCredit) },
+ TDuration::Max()),
+ 0,
+ ++CurrentQuotaRequestCookie
+ );
+ QuotaRequestInFlight = true;
+ ConsumedBytesInCredit = 0;
+ } else if (!Queue.empty()){
+ ApproveRead(std::move(Queue.front().Event), Queue.front().StartWait, ctx);
+ Queue.pop_front();
+ }
+ }
+}
+
+void TReadSpeedLimiter::HandleClearance(TEvQuota::TEvClearance::TPtr& ev, const TActorContext& ctx) {
+ QuotaRequestInFlight = false;
+ const ui64 cookie = ev->Cookie;
+ LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_SPEED_LIMITER,
+ LimiterDescription() << "Got read quota:" << ev->Get()->Result << ". Cookie: " << cookie
+ );
+
+ Y_VERIFY(CurrentQuotaRequestCookie == cookie);
+ if (!Queue.empty()) {
+ ApproveRead(std::move(Queue.front().Event), Queue.front().StartWait, ctx);
+ Queue.pop_front();
+ }
+
+ if (Y_UNLIKELY(ev->Get()->Result != TEvQuota::TEvClearance::EResult::Success)) {
+ Y_VERIFY(ev->Get()->Result != TEvQuota::TEvClearance::EResult::Deadline); // We set deadline == inf in quota request.
+ if (ctx.Now() - LastReportedErrorTime > TDuration::Minutes(1)) {
+ LOG_ERROR_S(ctx, NKikimrServices::PQ_READ_SPEED_LIMITER, LimiterDescription() << "Got read quota error: " << ev->Get()->Result);
+ LastReportedErrorTime = ctx.Now();
+ }
+ return;
+ }
+}
+
+void TReadSpeedLimiter::TReadSpeedLimiter::ApproveRead(TEvPQ::TEvRead::TPtr ev, TInstant startWait, const TActorContext& ctx) {
+ LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_SPEED_LIMITER,
+ LimiterDescription() << " approve read for cookie=" << ev->Get()->Cookie
+ );
+ InProcessReadRequestCookies.insert(ev->Get()->Cookie);
+
+ auto waitTime = ctx.Now() - startWait;
+ Send(PartitionActor, new NReadSpeedLimiterEvents::TEvResponse(ev.Release(), waitTime));
+
+ if (QuotaWaitCounter) {
+ QuotaWaitCounter->IncFor(waitTime.MilliSeconds());
+ }
+}
+
+TString TReadSpeedLimiter::LimiterDescription() const {
+ return TStringBuilder() << "topic=" << TopicName << ":" << Partition << " user=" << User << ": ";
+}
+
+}// NPQ
+}// NKikimr
diff --git a/ydb/core/persqueue/read_speed_limiter.h b/ydb/core/persqueue/read_speed_limiter.h
index 9504946722..23ca8dad95 100644
--- a/ydb/core/persqueue/read_speed_limiter.h
+++ b/ydb/core/persqueue/read_speed_limiter.h
@@ -1,137 +1,137 @@
-#pragma once
-
+#pragma once
+
#include <ydb/core/base/quoter.h>
#include <ydb/core/persqueue/events/internal.h>
-
-#include <library/cpp/actors/core/hfunc.h>
-
-
-namespace NKikimr {
-namespace NPQ {
-
-class TPercentileCounter;
-
-namespace NReadSpeedLimiterEvents {
- struct TEvRequest : public TEventLocal<TEvRequest, TEvPQ::EvReadLimiterRequest> {
- TEvRequest(TEvPQ::TEvRead::TPtr readRequest)
- : ReadRequest(std::move(readRequest))
- {}
-
- TEvPQ::TEvRead::TPtr ReadRequest;
- };
-
- struct TEvResponse : public TEventLocal<TEvResponse, TEvPQ::EvReadLimiterResponse> {
- TEvResponse(TEvPQ::TEvRead::TPtr readRequest, TDuration waitTime)
- : ReadRequest(std::move(readRequest))
- , WaitTime(waitTime)
- {}
-
- TEvPQ::TEvRead::TPtr ReadRequest;
- TDuration WaitTime;
- };
-
- struct TEvConsumed : public TEventLocal<TEvConsumed, TEvPQ::EvReadLimiterConsumed> {
- TEvConsumed(ui64 readBytes, ui64 readRequestCookie)
- : ReadBytes(readBytes)
- , ReadRequestCookie(readRequestCookie)
- {}
-
- ui64 ReadBytes;
- ui64 ReadRequestCookie;
- };
-
- struct TEvCounters : public TEventLocal<TEvCounters, TEvPQ::EvReadLimiterCounters> {
- TEvCounters(const NKikimr::TTabletCountersBase& counters, const TString& user)
- : User(user)
- {
- Counters.Populate(counters);
- }
-
- NKikimr::TTabletCountersBase Counters;
- const TString User;
- };
-}
-
-class TReadSpeedLimiter : public TActorBootstrapped<TReadSpeedLimiter> {
-private:
- static const TString READ_QUOTA_ROOT_PATH;
-
- struct TQueueEvent {
- TQueueEvent(TEvPQ::TEvRead::TPtr&& event, TInstant startWait)
- : Event(event)
- , StartWait(startWait)
- {}
-
- TEvPQ::TEvRead::TPtr Event;
- TInstant StartWait;
- };
-
-private:
- STFUNC(StateWork)
- {
- TRACE_EVENT(NKikimrServices::PQ_READ_SPEED_LIMITER);
- switch (ev->GetTypeRewrite()) {
- HFuncTraced(TEvPQ::TEvUpdateCounters, HandleUpdateCounters);
- HFuncTraced(NReadSpeedLimiterEvents::TEvRequest, HandleReadQuotaRequest);
- HFuncTraced(NReadSpeedLimiterEvents::TEvConsumed, HandleReadQuotaConsumed);
- HFuncTraced(TEvQuota::TEvClearance, HandleClearance);
- HFuncTraced(TEvents::TEvPoisonPill, Handle);
- default:
- break;
- };
- }
-
-public:
- static constexpr NKikimrServices::TActivity::EType ActorActivityType();
-
- TReadSpeedLimiter(
- TActorId tabletActor,
- TActorId partitionActor,
- ui64 tabletId,
- const TString& topicName,
- ui32 partition,
- const TString& user,
- const TTabletCountersBase& counters
- );
-
- void Bootstrap(const TActorContext& ctx);
- void Handle(TEvents::TEvPoisonPill::TPtr& ev, const TActorContext& ctx);
- void HandleUpdateCounters(TEvPQ::TEvUpdateCounters::TPtr& ev, const TActorContext& ctx);
- void HandleReadQuotaRequest(NReadSpeedLimiterEvents::TEvRequest::TPtr& ev, const TActorContext& ctx);
- void HandleReadQuotaConsumed(NReadSpeedLimiterEvents::TEvConsumed::TPtr& ev, const TActorContext& ctx);
- void HandleClearance(TEvQuota::TEvClearance::TPtr& ev, const TActorContext& ctx);
-
- void ApproveRead(TEvPQ::TEvRead::TPtr ev, TInstant startWait, const TActorContext& ctx);
-
-private:
- TString LimiterDescription() const;
-
-private:
- const TActorId TabletActor;
- const TActorId PartitionActor;
- const ui64 TabletId;
- const TString TopicName;
- const ui32 Partition;
- const TString User;
- const TString ConsumerPath;
- const ui64 ReadCreditBytes;
-
- ui64 ConsumedBytesInCredit = 0;
-
- TString KesusPath;
- TString QuotaResourcePath;
-
- TDeque<TQueueEvent> Queue;
-
- bool QuotaRequestInFlight = false;
- ui64 CurrentQuotaRequestCookie = 0;
- THashSet<ui64> InProcessReadRequestCookies;
-
-
- TTabletCountersBase Counters;
- THolder<TPercentileCounter> QuotaWaitCounter;
- TInstant LastReportedErrorTime;
-};
-
-}// NPQ
-}// NKikimr
+
+#include <library/cpp/actors/core/hfunc.h>
+
+
+namespace NKikimr {
+namespace NPQ {
+
+class TPercentileCounter;
+
+namespace NReadSpeedLimiterEvents {
+ struct TEvRequest : public TEventLocal<TEvRequest, TEvPQ::EvReadLimiterRequest> {
+ TEvRequest(TEvPQ::TEvRead::TPtr readRequest)
+ : ReadRequest(std::move(readRequest))
+ {}
+
+ TEvPQ::TEvRead::TPtr ReadRequest;
+ };
+
+ struct TEvResponse : public TEventLocal<TEvResponse, TEvPQ::EvReadLimiterResponse> {
+ TEvResponse(TEvPQ::TEvRead::TPtr readRequest, TDuration waitTime)
+ : ReadRequest(std::move(readRequest))
+ , WaitTime(waitTime)
+ {}
+
+ TEvPQ::TEvRead::TPtr ReadRequest;
+ TDuration WaitTime;
+ };
+
+ struct TEvConsumed : public TEventLocal<TEvConsumed, TEvPQ::EvReadLimiterConsumed> {
+ TEvConsumed(ui64 readBytes, ui64 readRequestCookie)
+ : ReadBytes(readBytes)
+ , ReadRequestCookie(readRequestCookie)
+ {}
+
+ ui64 ReadBytes;
+ ui64 ReadRequestCookie;
+ };
+
+ struct TEvCounters : public TEventLocal<TEvCounters, TEvPQ::EvReadLimiterCounters> {
+ TEvCounters(const NKikimr::TTabletCountersBase& counters, const TString& user)
+ : User(user)
+ {
+ Counters.Populate(counters);
+ }
+
+ NKikimr::TTabletCountersBase Counters;
+ const TString User;
+ };
+}
+
+class TReadSpeedLimiter : public TActorBootstrapped<TReadSpeedLimiter> {
+private:
+ static const TString READ_QUOTA_ROOT_PATH;
+
+ struct TQueueEvent {
+ TQueueEvent(TEvPQ::TEvRead::TPtr&& event, TInstant startWait)
+ : Event(event)
+ , StartWait(startWait)
+ {}
+
+ TEvPQ::TEvRead::TPtr Event;
+ TInstant StartWait;
+ };
+
+private:
+ STFUNC(StateWork)
+ {
+ TRACE_EVENT(NKikimrServices::PQ_READ_SPEED_LIMITER);
+ switch (ev->GetTypeRewrite()) {
+ HFuncTraced(TEvPQ::TEvUpdateCounters, HandleUpdateCounters);
+ HFuncTraced(NReadSpeedLimiterEvents::TEvRequest, HandleReadQuotaRequest);
+ HFuncTraced(NReadSpeedLimiterEvents::TEvConsumed, HandleReadQuotaConsumed);
+ HFuncTraced(TEvQuota::TEvClearance, HandleClearance);
+ HFuncTraced(TEvents::TEvPoisonPill, Handle);
+ default:
+ break;
+ };
+ }
+
+public:
+ static constexpr NKikimrServices::TActivity::EType ActorActivityType();
+
+ TReadSpeedLimiter(
+ TActorId tabletActor,
+ TActorId partitionActor,
+ ui64 tabletId,
+ const TString& topicName,
+ ui32 partition,
+ const TString& user,
+ const TTabletCountersBase& counters
+ );
+
+ void Bootstrap(const TActorContext& ctx);
+ void Handle(TEvents::TEvPoisonPill::TPtr& ev, const TActorContext& ctx);
+ void HandleUpdateCounters(TEvPQ::TEvUpdateCounters::TPtr& ev, const TActorContext& ctx);
+ void HandleReadQuotaRequest(NReadSpeedLimiterEvents::TEvRequest::TPtr& ev, const TActorContext& ctx);
+ void HandleReadQuotaConsumed(NReadSpeedLimiterEvents::TEvConsumed::TPtr& ev, const TActorContext& ctx);
+ void HandleClearance(TEvQuota::TEvClearance::TPtr& ev, const TActorContext& ctx);
+
+ void ApproveRead(TEvPQ::TEvRead::TPtr ev, TInstant startWait, const TActorContext& ctx);
+
+private:
+ TString LimiterDescription() const;
+
+private:
+ const TActorId TabletActor;
+ const TActorId PartitionActor;
+ const ui64 TabletId;
+ const TString TopicName;
+ const ui32 Partition;
+ const TString User;
+ const TString ConsumerPath;
+ const ui64 ReadCreditBytes;
+
+ ui64 ConsumedBytesInCredit = 0;
+
+ TString KesusPath;
+ TString QuotaResourcePath;
+
+ TDeque<TQueueEvent> Queue;
+
+ bool QuotaRequestInFlight = false;
+ ui64 CurrentQuotaRequestCookie = 0;
+ THashSet<ui64> InProcessReadRequestCookies;
+
+
+ TTabletCountersBase Counters;
+ THolder<TPercentileCounter> QuotaWaitCounter;
+ TInstant LastReportedErrorTime;
+};
+
+}// NPQ
+}// NKikimr
diff --git a/ydb/core/persqueue/subscriber.cpp b/ydb/core/persqueue/subscriber.cpp
index 0d5fa02f79..641c566a1c 100644
--- a/ydb/core/persqueue/subscriber.cpp
+++ b/ydb/core/persqueue/subscriber.cpp
@@ -55,11 +55,11 @@ TSubscriber::TSubscriber(const ui32 partition, TTabletCountersBase& counters, co
, Tablet(tablet)
{}
-TMaybe<TReadInfo> TSubscriber::OnTimeout(TEvPQ::TEvReadTimeout::TPtr& ev) {
+TMaybe<TReadInfo> TSubscriber::OnTimeout(TEvPQ::TEvReadTimeout::TPtr& ev) {
TMaybe<TReadInfo> res = Subscriber.ForgetSubscription(ev->Get()->Cookie);
- if (res) {
- Counters.Cumulative()[COUNTER_PQ_READ_SUBSCRIPTION_TIMEOUT].Increment(1);
- }
+ if (res) {
+ Counters.Cumulative()[COUNTER_PQ_READ_SUBSCRIPTION_TIMEOUT].Increment(1);
+ }
return res;
}
diff --git a/ydb/core/persqueue/subscriber.h b/ydb/core/persqueue/subscriber.h
index 13c9fd08c6..7bbfe41e1f 100644
--- a/ydb/core/persqueue/subscriber.h
+++ b/ydb/core/persqueue/subscriber.h
@@ -1,8 +1,8 @@
#pragma once
-
+
#include "header.h"
#include "blob.h"
-
+
#include <ydb/core/tablet/tablet_counters.h>
#include <ydb/core/base/appdata.h>
#include <ydb/core/persqueue/events/internal.h>
@@ -12,11 +12,11 @@ namespace NPQ {
struct TUserInfo;
-struct TReadAnswer {
- ui64 Size;
- THolder<IEventBase> Event;
-};
-
+struct TReadAnswer {
+ ui64 Size;
+ THolder<IEventBase> Event;
+};
+
struct TReadInfo {
TString User;
TString ClientDC;
@@ -27,7 +27,7 @@ struct TReadInfo {
ui64 Destination;
TInstant Timestamp;
ui64 ReadTimestampMs;
- TDuration WaitQuotaTime;
+ TDuration WaitQuotaTime;
bool IsSubscription;
@@ -36,17 +36,17 @@ struct TReadInfo {
TVector<TClientBlob> Cached; //records from head
TReadInfo() = delete;
- TReadInfo(
- const TString& user,
- const TString& clientDC,
- const ui64 offset,
- const ui16 partNo,
- const ui64 count,
- const ui32 size,
- const ui64 dst,
- ui64 readTimestampMs,
- TDuration waitQuotaTime
- )
+ TReadInfo(
+ const TString& user,
+ const TString& clientDC,
+ const ui64 offset,
+ const ui16 partNo,
+ const ui64 count,
+ const ui32 size,
+ const ui64 dst,
+ ui64 readTimestampMs,
+ TDuration waitQuotaTime
+ )
: User(user)
, ClientDC(clientDC)
, Offset(offset)
@@ -55,30 +55,30 @@ struct TReadInfo {
, Size(size)
, Destination(dst)
, Timestamp(TAppData::TimeProvider->Now())
- , ReadTimestampMs(readTimestampMs)
- , WaitQuotaTime(waitQuotaTime)
+ , ReadTimestampMs(readTimestampMs)
+ , WaitQuotaTime(waitQuotaTime)
, IsSubscription(false)
, CachedOffset(0)
{}
- TReadAnswer FormAnswer(
- const TActorContext& ctx,
- const TEvPQ::TEvBlobResponse& response,
- const ui64 endOffset,
- const ui32 partition,
- TUserInfo* ui,
- const ui64 dst,
- const ui64 sizeLag
- );
-
- TReadAnswer FormAnswer(
- const TActorContext& ctx,
- const ui64 endOffset,
- const ui32 partition,
- TUserInfo* ui,
- const ui64 dst,
- const ui64 sizeLag
- ) {
+ TReadAnswer FormAnswer(
+ const TActorContext& ctx,
+ const TEvPQ::TEvBlobResponse& response,
+ const ui64 endOffset,
+ const ui32 partition,
+ TUserInfo* ui,
+ const ui64 dst,
+ const ui64 sizeLag
+ );
+
+ TReadAnswer FormAnswer(
+ const TActorContext& ctx,
+ const ui64 endOffset,
+ const ui32 partition,
+ TUserInfo* ui,
+ const ui64 dst,
+ const ui64 sizeLag
+ ) {
TEvPQ::TEvBlobResponse response(0, TVector<TRequestedBlob>());
return FormAnswer(ctx, response, endOffset, partition, ui, dst, sizeLag);
}
@@ -116,7 +116,7 @@ public:
void AddSubscription(TReadInfo&& info, const ui32 timeout, const ui64 cookie, const TActorContext& ctx);
//handle of timeout for some read
- TMaybe<TReadInfo> OnTimeout(TEvPQ::TEvReadTimeout::TPtr& ev);
+ TMaybe<TReadInfo> OnTimeout(TEvPQ::TEvReadTimeout::TPtr& ev);
//get completed subscriptions
TVector<std::pair<TReadInfo, ui64>> GetReads(const ui64 endOffsets);
diff --git a/ydb/core/persqueue/user_info.cpp b/ydb/core/persqueue/user_info.cpp
index 5b987c6b3a..26811765e2 100644
--- a/ydb/core/persqueue/user_info.cpp
+++ b/ydb/core/persqueue/user_info.cpp
@@ -1,129 +1,129 @@
-#include "user_info.h"
-
-namespace NKikimr {
-namespace NPQ {
-
-namespace NDeprecatedUserData {
- TBuffer Serialize(ui64 offset, ui32 gen, ui32 step, const TString& session) {
- TBuffer data;
- data.Resize(sizeof(ui64) + sizeof(ui32) * 2 + session.size());
- memcpy(data.Data(), &offset, sizeof(ui64));
- memcpy(data.Data() + sizeof(ui64), &gen, sizeof(ui32));
- memcpy(data.Data() + sizeof(ui64) + sizeof(ui32), &step, sizeof(ui32));
- memcpy(data.Data() + sizeof(ui64) + 2 * sizeof(ui32), session.data(), session.size());
- return data;
- }
-
- void Parse(const TString& data, ui64& offset, ui32& gen, ui32& step, TString& session) {
- Y_VERIFY(sizeof(ui64) <= data.size());
-
- offset = *reinterpret_cast<const ui64*>(data.c_str());
- gen = 0;
- step = 0;
- if (data.size() > sizeof(ui64)) {
- gen = reinterpret_cast<const ui32*>(data.c_str() + sizeof(ui64))[0];
- step = reinterpret_cast<const ui32*>(data.c_str() + sizeof(ui64))[1];
- session = data.substr(sizeof(ui64) + 2 * sizeof(ui32));
- }
- }
-} // NDeprecatedUserData
-
-TUsersInfoStorage::TUsersInfoStorage(
+#include "user_info.h"
+
+namespace NKikimr {
+namespace NPQ {
+
+namespace NDeprecatedUserData {
+ TBuffer Serialize(ui64 offset, ui32 gen, ui32 step, const TString& session) {
+ TBuffer data;
+ data.Resize(sizeof(ui64) + sizeof(ui32) * 2 + session.size());
+ memcpy(data.Data(), &offset, sizeof(ui64));
+ memcpy(data.Data() + sizeof(ui64), &gen, sizeof(ui32));
+ memcpy(data.Data() + sizeof(ui64) + sizeof(ui32), &step, sizeof(ui32));
+ memcpy(data.Data() + sizeof(ui64) + 2 * sizeof(ui32), session.data(), session.size());
+ return data;
+ }
+
+ void Parse(const TString& data, ui64& offset, ui32& gen, ui32& step, TString& session) {
+ Y_VERIFY(sizeof(ui64) <= data.size());
+
+ offset = *reinterpret_cast<const ui64*>(data.c_str());
+ gen = 0;
+ step = 0;
+ if (data.size() > sizeof(ui64)) {
+ gen = reinterpret_cast<const ui32*>(data.c_str() + sizeof(ui64))[0];
+ step = reinterpret_cast<const ui32*>(data.c_str() + sizeof(ui64))[1];
+ session = data.substr(sizeof(ui64) + 2 * sizeof(ui32));
+ }
+ }
+} // NDeprecatedUserData
+
+TUsersInfoStorage::TUsersInfoStorage(
TString dcId,
- ui64 tabletId,
- const TString& topicName,
- ui32 partition,
+ ui64 tabletId,
+ const TString& topicName,
+ ui32 partition,
const TTabletCountersBase& counters,
const NKikimrPQ::TPQTabletConfig& config,
const TString& cloudId,
const TString& dbId,
const TString& folderId
-)
+)
: DCId(std::move(dcId))
- , TabletId(tabletId)
- , TopicName(topicName)
- , Partition(partition)
+ , TabletId(tabletId)
+ , TopicName(topicName)
+ , Partition(partition)
, Config(config)
, CloudId(cloudId)
, DbId(dbId)
, FolderId(folderId)
-{
- Counters.Populate(counters);
-}
-
-void TUsersInfoStorage::Init(TActorId tabletActor, TActorId partitionActor) {
- Y_VERIFY(!TabletActor);
- Y_VERIFY(!PartitionActor);
- TabletActor = tabletActor;
- PartitionActor = partitionActor;
-
- for (auto& userInfoPair : UsersInfo) {
- auto& userInfo = userInfoPair.second;
- Y_VERIFY(!userInfo.ReadSpeedLimiter);
- userInfo.ReadSpeedLimiter = CreateReadSpeedLimiter(userInfo.User);
- }
-}
-
-void TUsersInfoStorage::ParseDeprecated(const TString& key, const TString& data, const TActorContext& ctx) {
- Y_VERIFY(key.size() >= TKeyPrefix::MarkedSize());
- Y_VERIFY(key[TKeyPrefix::MarkPosition()] == TKeyPrefix::MarkUserDeprecated);
- TString user = key.substr(TKeyPrefix::MarkedSize());
-
+{
+ Counters.Populate(counters);
+}
+
+void TUsersInfoStorage::Init(TActorId tabletActor, TActorId partitionActor) {
+ Y_VERIFY(!TabletActor);
+ Y_VERIFY(!PartitionActor);
+ TabletActor = tabletActor;
+ PartitionActor = partitionActor;
+
+ for (auto& userInfoPair : UsersInfo) {
+ auto& userInfo = userInfoPair.second;
+ Y_VERIFY(!userInfo.ReadSpeedLimiter);
+ userInfo.ReadSpeedLimiter = CreateReadSpeedLimiter(userInfo.User);
+ }
+}
+
+void TUsersInfoStorage::ParseDeprecated(const TString& key, const TString& data, const TActorContext& ctx) {
+ Y_VERIFY(key.size() >= TKeyPrefix::MarkedSize());
+ Y_VERIFY(key[TKeyPrefix::MarkPosition()] == TKeyPrefix::MarkUserDeprecated);
+ TString user = key.substr(TKeyPrefix::MarkedSize());
+
TUserInfo* userInfo = GetIfExists(user);
if (userInfo && userInfo->Parsed) {
return;
}
- ui64 offset = 0;
- ui32 gen = 0;
- ui32 step = 0;
- TString session;
- NDeprecatedUserData::Parse(data, offset, gen, step, session);
- Y_VERIFY(offset <= (ui64)Max<i64>(), "Offset is too big: %" PRIu64, offset);
-
- if (!userInfo) {
+ ui64 offset = 0;
+ ui32 gen = 0;
+ ui32 step = 0;
+ TString session;
+ NDeprecatedUserData::Parse(data, offset, gen, step, session);
+ Y_VERIFY(offset <= (ui64)Max<i64>(), "Offset is too big: %" PRIu64, offset);
+
+ if (!userInfo) {
Create(ctx, user, 0, false, session, gen, step, static_cast<i64>(offset), 0, TInstant::Zero());
- } else {
- userInfo->Session = session;
- userInfo->Generation = gen;
- userInfo->Step = step;
- userInfo->Offset = static_cast<i64>(offset);
- }
-}
-
-void TUsersInfoStorage::Parse(const TString& key, const TString& data, const TActorContext& ctx) {
- Y_VERIFY(key.size() >= TKeyPrefix::MarkedSize());
- Y_VERIFY(key[TKeyPrefix::MarkPosition()] == TKeyPrefix::MarkUser);
- TString user = key.substr(TKeyPrefix::MarkedSize());
-
- Y_VERIFY(sizeof(ui64) <= data.size());
-
- NKikimrPQ::TUserInfo userData;
- bool res = userData.ParseFromString(data);
- Y_VERIFY(res);
-
- Y_VERIFY(userData.GetOffset() <= (ui64)Max<i64>(), "Offset is too big: %" PRIu64, userData.GetOffset());
- i64 offset = static_cast<i64>(userData.GetOffset());
-
- TUserInfo* userInfo = GetIfExists(user);
- if (!userInfo) {
- Create(
+ } else {
+ userInfo->Session = session;
+ userInfo->Generation = gen;
+ userInfo->Step = step;
+ userInfo->Offset = static_cast<i64>(offset);
+ }
+}
+
+void TUsersInfoStorage::Parse(const TString& key, const TString& data, const TActorContext& ctx) {
+ Y_VERIFY(key.size() >= TKeyPrefix::MarkedSize());
+ Y_VERIFY(key[TKeyPrefix::MarkPosition()] == TKeyPrefix::MarkUser);
+ TString user = key.substr(TKeyPrefix::MarkedSize());
+
+ Y_VERIFY(sizeof(ui64) <= data.size());
+
+ NKikimrPQ::TUserInfo userData;
+ bool res = userData.ParseFromString(data);
+ Y_VERIFY(res);
+
+ Y_VERIFY(userData.GetOffset() <= (ui64)Max<i64>(), "Offset is too big: %" PRIu64, userData.GetOffset());
+ i64 offset = static_cast<i64>(userData.GetOffset());
+
+ TUserInfo* userInfo = GetIfExists(user);
+ if (!userInfo) {
+ Create(
ctx, user, userData.GetReadRuleGeneration(), false, userData.GetSession(),
- userData.GetGeneration(), userData.GetStep(), offset, userData.GetOffsetRewindSum(), TInstant::Zero()
- );
- } else {
- userInfo->Session = userData.GetSession();
- userInfo->Generation = userData.GetGeneration();
- userInfo->Step = userData.GetStep();
- userInfo->Offset = offset;
- userInfo->ReadOffsetRewindSum = userData.GetOffsetRewindSum();
+ userData.GetGeneration(), userData.GetStep(), offset, userData.GetOffsetRewindSum(), TInstant::Zero()
+ );
+ } else {
+ userInfo->Session = userData.GetSession();
+ userInfo->Generation = userData.GetGeneration();
+ userInfo->Step = userData.GetStep();
+ userInfo->Offset = offset;
+ userInfo->ReadOffsetRewindSum = userData.GetOffsetRewindSum();
userInfo->ReadRuleGeneration = userData.GetReadRuleGeneration();
- }
+ }
userInfo = GetIfExists(user);
Y_VERIFY(userInfo);
userInfo->Parsed = true;
-}
-
+}
+
void TUsersInfoStorage::Remove(const TString& user, const TActorContext& ctx) {
auto it = UsersInfo.find(user);
Y_VERIFY(it != UsersInfo.end());
@@ -131,76 +131,76 @@ void TUsersInfoStorage::Remove(const TString& user, const TActorContext& ctx) {
UsersInfo.erase(it);
}
-TUserInfo& TUsersInfoStorage::GetOrCreate(const TString& user, const TActorContext& ctx) {
- Y_VERIFY(!user.empty());
- auto it = UsersInfo.find(user);
- if (it == UsersInfo.end()) {
+TUserInfo& TUsersInfoStorage::GetOrCreate(const TString& user, const TActorContext& ctx) {
+ Y_VERIFY(!user.empty());
+ auto it = UsersInfo.find(user);
+ if (it == UsersInfo.end()) {
return Create(ctx, user, 0, false, "", 0, 0, 0, 0, TInstant::Zero());
- }
- return it->second;
-}
-
-TUserInfo* TUsersInfoStorage::GetIfExists(const TString& user) {
- auto it = UsersInfo.find(user);
- return it != UsersInfo.end() ? &it->second : nullptr;
-}
-
-THashMap<TString, TUserInfo>& TUsersInfoStorage::GetAll() {
- return UsersInfo;
-}
-
-TUserInfo& TUsersInfoStorage::Create(
+ }
+ return it->second;
+}
+
+TUserInfo* TUsersInfoStorage::GetIfExists(const TString& user) {
+ auto it = UsersInfo.find(user);
+ return it != UsersInfo.end() ? &it->second : nullptr;
+}
+
+THashMap<TString, TUserInfo>& TUsersInfoStorage::GetAll() {
+ return UsersInfo;
+}
+
+TUserInfo& TUsersInfoStorage::Create(
const TActorContext& ctx, const TString& user, const ui64 readRuleGeneration, bool important, const TString& session,
- ui32 gen, ui32 step, i64 offset, ui64 readOffsetRewindSum, TInstant readFromTimestamp
-) {
+ ui32 gen, ui32 step, i64 offset, ui64 readOffsetRewindSum, TInstant readFromTimestamp
+) {
ui64 burst = 1'000'000'000, speed = 1'000'000'000;
if (AppData(ctx)->PQConfig.GetQuotingConfig().GetPartitionReadQuotaIsTwiceWriteQuota()) {
burst = Config.GetPartitionConfig().GetBurstSize() * 2;
speed = Config.GetPartitionConfig().GetWriteSpeedInBytesPerSecond() * 2;
}
- auto result = UsersInfo.emplace(
- std::piecewise_construct,
- std::forward_as_tuple(user),
+ auto result = UsersInfo.emplace(
+ std::piecewise_construct,
+ std::forward_as_tuple(user),
std::forward_as_tuple(ctx, CreateReadSpeedLimiter(user), user, readRuleGeneration, important, TopicName, Partition, session,
gen, step, offset, readOffsetRewindSum, DCId, readFromTimestamp, CloudId, DbId, FolderId, burst, speed)
- );
- Y_VERIFY(result.second);
- return result.first->second;
-}
-
-void TUsersInfoStorage::Clear(const TActorContext& ctx) {
- for (auto& userInfoPair : UsersInfo) {
+ );
+ Y_VERIFY(result.second);
+ return result.first->second;
+}
+
+void TUsersInfoStorage::Clear(const TActorContext& ctx) {
+ for (auto& userInfoPair : UsersInfo) {
userInfoPair.second.Clear(ctx);
- }
+ }
UsersInfo.clear();
-}
-
+}
+
void TUserInfo::Clear(const TActorContext& ctx) {
if (ReadSpeedLimiter) {
ctx.Send(ReadSpeedLimiter->Actor, new TEvents::TEvPoisonPill());
}
}
-THolder<TReadSpeedLimiterHolder> TUsersInfoStorage::CreateReadSpeedLimiter(const TString& user) const {
- const auto& quotingConfig = AppData()->PQConfig.GetQuotingConfig();
- if (TabletActor && quotingConfig.GetEnableQuoting() && quotingConfig.GetEnableReadQuoting()) {
- TActorId actorId = TActivationContext::Register(
- new TReadSpeedLimiter(
- TabletActor.GetRef(),
- PartitionActor.GetRef(),
- TabletId,
- TopicName,
- Partition,
- user,
- Counters
- ),
- PartitionActor.GetRef()
- );
- return MakeHolder<TReadSpeedLimiterHolder>(actorId, Counters);
- }
- return nullptr;
-}
-
-} //NPQ
-} //NKikimr
+THolder<TReadSpeedLimiterHolder> TUsersInfoStorage::CreateReadSpeedLimiter(const TString& user) const {
+ const auto& quotingConfig = AppData()->PQConfig.GetQuotingConfig();
+ if (TabletActor && quotingConfig.GetEnableQuoting() && quotingConfig.GetEnableReadQuoting()) {
+ TActorId actorId = TActivationContext::Register(
+ new TReadSpeedLimiter(
+ TabletActor.GetRef(),
+ PartitionActor.GetRef(),
+ TabletId,
+ TopicName,
+ Partition,
+ user,
+ Counters
+ ),
+ PartitionActor.GetRef()
+ );
+ return MakeHolder<TReadSpeedLimiterHolder>(actorId, Counters);
+ }
+ return nullptr;
+}
+
+} //NPQ
+} //NKikimr
diff --git a/ydb/core/persqueue/user_info.h b/ydb/core/persqueue/user_info.h
index 04b6d4d443..258e947c46 100644
--- a/ydb/core/persqueue/user_info.h
+++ b/ydb/core/persqueue/user_info.h
@@ -3,8 +3,8 @@
#include "working_time_counter.h"
#include "subscriber.h"
#include "percentile_counter.h"
-#include "read_speed_limiter.h"
-
+#include "read_speed_limiter.h"
+
#include <ydb/core/base/counters.h>
#include <ydb/core/protos/counters_pq.pb.h>
#include <ydb/core/protos/pqconfig.pb.h>
@@ -13,17 +13,17 @@
#include <library/cpp/sliding_window/sliding_window.h>
-#include <util/generic/set.h>
-
+#include <util/generic/set.h>
+
namespace NKikimr {
namespace NPQ {
-namespace NDeprecatedUserData {
- // [offset:64bit][generation:32bit][step:32bit][session:other bytes]
- TBuffer Serialize(ui64 offset, ui32 gen, ui32 step, const TString& session);
- void Parse(const TString& data, ui64& offset, ui32& gen, ui32& step, TString& session);
-} // NDeprecatedUserInfo
-
+namespace NDeprecatedUserData {
+ // [offset:64bit][generation:32bit][step:32bit][session:other bytes]
+ TBuffer Serialize(ui64 offset, ui32 gen, ui32 step, const TString& session);
+ void Parse(const TString& data, ui64& offset, ui32& gen, ui32& step, TString& session);
+} // NDeprecatedUserInfo
+
static const ui32 MAX_USER_TS_CACHE_SIZE = 10'000;
static const ui64 MIN_TIMESTAMP_MS = 1'000'000'000'000ll; // around 2002 year
static const TString CLIENTID_TO_READ_INTERNALLY = "$without_consumer";
@@ -137,42 +137,42 @@ private:
ui64 QuotedTime;
};
-struct TReadSpeedLimiterHolder {
- TReadSpeedLimiterHolder(const TActorId& actor, const TTabletCountersBase& baseline)
- : Actor(actor)
- {
- Baseline.Populate(baseline);
- }
-
- TActorId Actor;
- TTabletCountersBase Baseline;
-};
-
+struct TReadSpeedLimiterHolder {
+ TReadSpeedLimiterHolder(const TActorId& actor, const TTabletCountersBase& baseline)
+ : Actor(actor)
+ {
+ Baseline.Populate(baseline);
+ }
+
+ TActorId Actor;
+ TTabletCountersBase Baseline;
+};
+
struct TUserInfo {
- THolder<TReadSpeedLimiterHolder> ReadSpeedLimiter;
-
+ THolder<TReadSpeedLimiterHolder> ReadSpeedLimiter;
+
TString Session = "";
ui32 Generation = 0;
ui32 Step = 0;
i64 Offset = 0;
- TInstant WriteTimestamp;
- TInstant CreateTimestamp;
- TInstant ReadTimestamp;
+ TInstant WriteTimestamp;
+ TInstant CreateTimestamp;
+ TInstant ReadTimestamp;
bool ActualTimestamps = false;
i64 ReadOffset = -1;
- TInstant ReadWriteTimestamp;
- TInstant ReadCreateTimestamp;
+ TInstant ReadWriteTimestamp;
+ TInstant ReadCreateTimestamp;
ui64 ReadOffsetRewindSum = 0;
bool ReadScheduled = false;
//cache is used for storing WriteTime;CreateTime for offsets.
//When client will commit to new position, timestamps for this offset could be in cache - not insane client should read data before commit
- std::deque<std::pair<ui64, std::pair<TInstant, TInstant>>> Cache;
+ std::deque<std::pair<ui64, std::pair<TInstant, TInstant>>> Cache;
bool Important = false;
- TInstant ReadFromTimestamp;
+ TInstant ReadFromTimestamp;
bool HasReadRule = false;
TUserLabeledCounters LabeledCounters;
TString User;
@@ -209,23 +209,23 @@ struct TUserInfo {
const ui64 readRuleGeneration, bool important, const TString& topic,
const ui32 partition, bool doExternalRead,
ui64 burst = 1'000'000'000, ui64 speed = 1'000'000'000)
- : ReadSpeedLimiter(std::move(readSpeedLimiter))
- , Important(important)
- , LabeledCounters(user + "/" + (important ? "1" : "0") + "/" + topic, partition)
- , User(user)
+ : ReadSpeedLimiter(std::move(readSpeedLimiter))
+ , Important(important)
+ , LabeledCounters(user + "/" + (important ? "1" : "0") + "/" + topic, partition)
+ , User(user)
, ReadRuleGeneration(readRuleGeneration)
- , Topic(topic)
+ , Topic(topic)
, ReadQuota(burst, speed, TAppData::TimeProvider->Now())
- , Counter(nullptr)
- , BytesRead()
- , MsgsRead()
- , ActiveReads(0)
- , Subscriptions(0)
- , EndOffset(0)
- , WriteLagMs(TDuration::Minutes(1), 100)
+ , Counter(nullptr)
+ , BytesRead()
+ , MsgsRead()
+ , ActiveReads(0)
+ , Subscriptions(0)
+ , EndOffset(0)
+ , WriteLagMs(TDuration::Minutes(1), 100)
, DoExternalRead(doExternalRead)
- {
- }
+ {
+ }
void ForgetSubscription(const TInstant& now) {
if(Subscriptions > 0)
@@ -237,7 +237,7 @@ struct TUserInfo {
Counter.UpdateState(Subscriptions > 0 || ActiveReads > 0 || ReadRequests.size() > 0); //no data for read or got read requests from client
}
- void UpdateReadingTimeAndState(TInstant now) {
+ void UpdateReadingTimeAndState(TInstant now) {
Counter.UpdateWorkingTime(now);
UpdateReadingState();
@@ -245,7 +245,7 @@ struct TUserInfo {
WriteLagMs.Update(0, now);
}
if (Subscriptions > 0) {
- ReadTimestamp = now;
+ ReadTimestamp = now;
}
}
@@ -282,32 +282,32 @@ struct TUserInfo {
Y_VERIFY(ActiveReads > 0);
--ActiveReads;
UpdateReadingTimeAndState(now);
- ReadTimestamp = now;
+ ReadTimestamp = now;
}
- TUserInfo(
+ TUserInfo(
const TActorContext& ctx, THolder<TReadSpeedLimiterHolder> readSpeedLimiter, const TString& user,
const ui64 readRuleGeneration, const bool important, const TString& topic, const ui32 partition, const TString &session,
ui32 gen, ui32 step, i64 offset, const ui64 readOffsetRewindSum, const TString& dcId,
TInstant readFromTimestamp, const TString& cloudId, const TString& dbId, const TString& folderId,
ui64 burst = 1'000'000'000, ui64 speed = 1'000'000'000
- )
- : ReadSpeedLimiter(std::move(readSpeedLimiter))
- , Session(session)
+ )
+ : ReadSpeedLimiter(std::move(readSpeedLimiter))
+ , Session(session)
, Generation(gen)
, Step(step)
, Offset(offset)
- , WriteTimestamp(TAppData::TimeProvider->Now())
- , CreateTimestamp(TAppData::TimeProvider->Now())
- , ReadTimestamp(TAppData::TimeProvider->Now())
+ , WriteTimestamp(TAppData::TimeProvider->Now())
+ , CreateTimestamp(TAppData::TimeProvider->Now())
+ , ReadTimestamp(TAppData::TimeProvider->Now())
, ActualTimestamps(false)
, ReadOffset(-1)
- , ReadWriteTimestamp(TAppData::TimeProvider->Now())
- , ReadCreateTimestamp(TAppData::TimeProvider->Now())
+ , ReadWriteTimestamp(TAppData::TimeProvider->Now())
+ , ReadCreateTimestamp(TAppData::TimeProvider->Now())
, ReadOffsetRewindSum(readOffsetRewindSum)
, ReadScheduled(false)
, Important(important)
- , ReadFromTimestamp(readFromTimestamp)
+ , ReadFromTimestamp(readFromTimestamp)
, HasReadRule(false)
, LabeledCounters(user + "/" +(important ? "1" : "0") + "/" + topic, partition)
, User(user)
@@ -415,35 +415,35 @@ struct TUserInfo {
void Clear(const TActorContext& ctx);
- void UpdateReadOffset(const i64 offset, TInstant writeTimestamp, TInstant createTimestamp, TInstant now) {
+ void UpdateReadOffset(const i64 offset, TInstant writeTimestamp, TInstant createTimestamp, TInstant now) {
ReadOffset = offset;
- ReadWriteTimestamp = writeTimestamp;
- ReadCreateTimestamp = createTimestamp;
- WriteLagMs.Update((ReadWriteTimestamp - ReadCreateTimestamp).MilliSeconds(), ReadWriteTimestamp);
+ ReadWriteTimestamp = writeTimestamp;
+ ReadCreateTimestamp = createTimestamp;
+ WriteLagMs.Update((ReadWriteTimestamp - ReadCreateTimestamp).MilliSeconds(), ReadWriteTimestamp);
if (Subscriptions > 0) {
- ReadTimestamp = now;
+ ReadTimestamp = now;
}
}
- void AddTimestampToCache(const ui64 offset, TInstant writeTimestamp, TInstant createTimestamp, bool isUserRead, TInstant now)
+ void AddTimestampToCache(const ui64 offset, TInstant writeTimestamp, TInstant createTimestamp, bool isUserRead, TInstant now)
{
if ((ui64)Max<i64>(Offset, 0) == offset) {
- WriteTimestamp = writeTimestamp;
- CreateTimestamp = createTimestamp;
+ WriteTimestamp = writeTimestamp;
+ CreateTimestamp = createTimestamp;
ActualTimestamps = true;
if (ReadOffset == -1) {
- UpdateReadOffset(offset, writeTimestamp, createTimestamp, now);
+ UpdateReadOffset(offset, writeTimestamp, createTimestamp, now);
}
}
if (isUserRead) {
- UpdateReadOffset(offset, writeTimestamp, createTimestamp, now);
+ UpdateReadOffset(offset, writeTimestamp, createTimestamp, now);
if (ReadTimeLag) {
- ReadTimeLag->IncFor((now - createTimestamp).MilliSeconds(), 1);
+ ReadTimeLag->IncFor((now - createTimestamp).MilliSeconds(), 1);
}
}
if (!Cache.empty() && Cache.back().first >= offset) //already got data in cache
return;
- Cache.push_back(std::make_pair(offset, std::make_pair(writeTimestamp, createTimestamp)));
+ Cache.push_back(std::make_pair(offset, std::make_pair(writeTimestamp, createTimestamp)));
if (Cache.size() > MAX_USER_TS_CACHE_SIZE)
Cache.pop_front();
}
@@ -454,11 +454,11 @@ struct TUserInfo {
Cache.pop_front();
}
if (!Cache.empty() && Cache.front().first == (ui64)Max<i64>(Offset, 0)) {
- WriteTimestamp = Cache.front().second.first;
- CreateTimestamp = Cache.front().second.second;
+ WriteTimestamp = Cache.front().second.first;
+ CreateTimestamp = Cache.front().second.second;
ActualTimestamps = true;
if (ReadOffset == -1) {
- UpdateReadOffset(Offset - 1, Cache.front().second.first, Cache.front().second.second, TAppData::TimeProvider->Now());
+ UpdateReadOffset(Offset - 1, Cache.front().second.first, Cache.front().second.second, TAppData::TimeProvider->Now());
}
return true;
}
@@ -475,21 +475,21 @@ struct TUserInfo {
return ReadOffset == -1 ? Offset : (ReadOffset + 1); //+1 because we want to track first not readed offset
}
- TInstant GetReadTimestamp() const {
+ TInstant GetReadTimestamp() const {
return ReadTimestamp;
}
- TInstant GetWriteTimestamp() const {
- return Offset == EndOffset ? TAppData::TimeProvider->Now() : WriteTimestamp;
+ TInstant GetWriteTimestamp() const {
+ return Offset == EndOffset ? TAppData::TimeProvider->Now() : WriteTimestamp;
}
- TInstant GetCreateTimestamp() const {
- return Offset == EndOffset ? TAppData::TimeProvider->Now() : CreateTimestamp;
+ TInstant GetCreateTimestamp() const {
+ return Offset == EndOffset ? TAppData::TimeProvider->Now() : CreateTimestamp;
}
- TInstant GetReadWriteTimestamp() const {
- TInstant ts = ReadOffset == -1 ? WriteTimestamp : ReadWriteTimestamp;
- ts = GetReadOffset() >= EndOffset ? TAppData::TimeProvider->Now() : ts;
+ TInstant GetReadWriteTimestamp() const {
+ TInstant ts = ReadOffset == -1 ? WriteTimestamp : ReadWriteTimestamp;
+ ts = GetReadOffset() >= EndOffset ? TAppData::TimeProvider->Now() : ts;
return ts;
}
@@ -497,63 +497,63 @@ struct TUserInfo {
return WriteLagMs.GetValue();
}
- TInstant GetReadCreateTimestamp() const {
- TInstant ts = ReadOffset == -1 ? CreateTimestamp : ReadCreateTimestamp;
- ts = GetReadOffset() >= EndOffset ? TAppData::TimeProvider->Now() : ts;
+ TInstant GetReadCreateTimestamp() const {
+ TInstant ts = ReadOffset == -1 ? CreateTimestamp : ReadCreateTimestamp;
+ ts = GetReadOffset() >= EndOffset ? TAppData::TimeProvider->Now() : ts;
return ts;
}
};
-class TUsersInfoStorage {
-public:
+class TUsersInfoStorage {
+public:
TUsersInfoStorage(TString dcId, ui64 tabletId, const TString& topicName, ui32 partition,
const TTabletCountersBase& counters, const NKikimrPQ::TPQTabletConfig& config,
const TString& CloudId, const TString& DbId, const TString& FolderId);
-
- void Init(TActorId tabletActor, TActorId partitionActor);
-
- void ParseDeprecated(const TString& key, const TString& data, const TActorContext& ctx);
- void Parse(const TString& key, const TString& data, const TActorContext& ctx);
-
- TUserInfo& GetOrCreate(const TString& user, const TActorContext& ctx);
- TUserInfo* GetIfExists(const TString& user);
-
+
+ void Init(TActorId tabletActor, TActorId partitionActor);
+
+ void ParseDeprecated(const TString& key, const TString& data, const TActorContext& ctx);
+ void Parse(const TString& key, const TString& data, const TActorContext& ctx);
+
+ TUserInfo& GetOrCreate(const TString& user, const TActorContext& ctx);
+ TUserInfo* GetIfExists(const TString& user);
+
void UpdateConfig(const NKikimrPQ::TPQTabletConfig& config) {
Config = config;
}
- THashMap<TString, TUserInfo>& GetAll();
-
- TUserInfo& Create(
+ THashMap<TString, TUserInfo>& GetAll();
+
+ TUserInfo& Create(
const TActorContext& ctx, const TString& user, const ui64 readRuleGeneration, bool important, const TString &session,
- ui32 gen, ui32 step, i64 offset, ui64 readOffsetRewindSum, TInstant readFromTimestamp
- );
+ ui32 gen, ui32 step, i64 offset, ui64 readOffsetRewindSum, TInstant readFromTimestamp
+ );
+
+ void Clear(const TActorContext& ctx);
- void Clear(const TActorContext& ctx);
-
void Remove(const TString& user, const TActorContext& ctx);
-private:
- THolder<TReadSpeedLimiterHolder> CreateReadSpeedLimiter(const TString& user) const;
-
-private:
- THashMap<TString, TUserInfo> UsersInfo;
-
+private:
+ THolder<TReadSpeedLimiterHolder> CreateReadSpeedLimiter(const TString& user) const;
+
+private:
+ THashMap<TString, TUserInfo> UsersInfo;
+
const TString DCId;
- ui64 TabletId;
- const TString TopicName;
- const ui32 Partition;
- TTabletCountersBase Counters;
-
- TMaybe<TActorId> TabletActor;
- TMaybe<TActorId> PartitionActor;
+ ui64 TabletId;
+ const TString TopicName;
+ const ui32 Partition;
+ TTabletCountersBase Counters;
+
+ TMaybe<TActorId> TabletActor;
+ TMaybe<TActorId> PartitionActor;
NKikimrPQ::TPQTabletConfig Config;
TString CloudId;
TString DbId;
TString FolderId;
-};
-
+};
+
} //NPQ
} //NKikimr
diff --git a/ydb/core/persqueue/user_info_ut.cpp b/ydb/core/persqueue/user_info_ut.cpp
index c48148a21a..8f8ec0aa58 100644
--- a/ydb/core/persqueue/user_info_ut.cpp
+++ b/ydb/core/persqueue/user_info_ut.cpp
@@ -1,35 +1,35 @@
-#include "user_info.h"
-
-#include <library/cpp/testing/unittest/registar.h>
-
-namespace NKikimr::NPQ {
-
- Y_UNIT_TEST_SUITE(TPQUserInfoTest) {
- Y_UNIT_TEST(UserDataDeprecatedSerializaion) {
- ui64 offset = 1234556789012345678ULL;
- ui32 gen = 987654321;
- ui32 step = 1234567890;
- const TString session = "some_source_id";
- TBuffer buffer = NDeprecatedUserData::Serialize(offset, gen, step, session);
- TString data(buffer.data(), buffer.size());
- {
- UNIT_ASSERT_EQUAL(data.size(), sizeof(offset) + sizeof(gen) + sizeof(step) + session.size());
- UNIT_ASSERT_EQUAL(offset, *reinterpret_cast<const ui64*>(data.c_str()));
- UNIT_ASSERT_EQUAL(gen, reinterpret_cast<const ui32*>(data.c_str() + sizeof(ui64))[0]);
- UNIT_ASSERT_EQUAL(step, reinterpret_cast<const ui32*>(data.c_str() + sizeof(ui64))[1]);
- UNIT_ASSERT_EQUAL(session, data.substr(sizeof(ui64) + 2 * sizeof(ui32)));
- }
- {
- ui64 parsedOffset = 0;
- ui32 parsedGen = 0;
- ui32 parsedStep = 0;
- TString parsedSession;
- NDeprecatedUserData::Parse(data, parsedOffset, parsedGen, parsedStep, parsedSession);
- UNIT_ASSERT_EQUAL(offset, parsedOffset);
- UNIT_ASSERT_EQUAL(gen, parsedGen);
- UNIT_ASSERT_EQUAL(step, parsedStep);
- UNIT_ASSERT_EQUAL(session, parsedSession);
- }
- }
- }
-}
+#include "user_info.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+
+namespace NKikimr::NPQ {
+
+ Y_UNIT_TEST_SUITE(TPQUserInfoTest) {
+ Y_UNIT_TEST(UserDataDeprecatedSerializaion) {
+ ui64 offset = 1234556789012345678ULL;
+ ui32 gen = 987654321;
+ ui32 step = 1234567890;
+ const TString session = "some_source_id";
+ TBuffer buffer = NDeprecatedUserData::Serialize(offset, gen, step, session);
+ TString data(buffer.data(), buffer.size());
+ {
+ UNIT_ASSERT_EQUAL(data.size(), sizeof(offset) + sizeof(gen) + sizeof(step) + session.size());
+ UNIT_ASSERT_EQUAL(offset, *reinterpret_cast<const ui64*>(data.c_str()));
+ UNIT_ASSERT_EQUAL(gen, reinterpret_cast<const ui32*>(data.c_str() + sizeof(ui64))[0]);
+ UNIT_ASSERT_EQUAL(step, reinterpret_cast<const ui32*>(data.c_str() + sizeof(ui64))[1]);
+ UNIT_ASSERT_EQUAL(session, data.substr(sizeof(ui64) + 2 * sizeof(ui32)));
+ }
+ {
+ ui64 parsedOffset = 0;
+ ui32 parsedGen = 0;
+ ui32 parsedStep = 0;
+ TString parsedSession;
+ NDeprecatedUserData::Parse(data, parsedOffset, parsedGen, parsedStep, parsedSession);
+ UNIT_ASSERT_EQUAL(offset, parsedOffset);
+ UNIT_ASSERT_EQUAL(gen, parsedGen);
+ UNIT_ASSERT_EQUAL(step, parsedStep);
+ UNIT_ASSERT_EQUAL(session, parsedSession);
+ }
+ }
+ }
+}
diff --git a/ydb/core/persqueue/ut/ya.make b/ydb/core/persqueue/ut/ya.make
index 39578c05aa..7cbb0b9496 100644
--- a/ydb/core/persqueue/ut/ya.make
+++ b/ydb/core/persqueue/ut/ya.make
@@ -1,10 +1,10 @@
UNITTEST_FOR(ydb/core/persqueue)
-OWNER(
- alexnick
+OWNER(
+ alexnick
g:kikimr
- g:logbroker
-)
+ g:logbroker
+)
FORK_SUBTESTS()
@@ -31,12 +31,12 @@ YQL_LAST_ABI_VERSION()
SRCS(
internals_ut.cpp
- mirrorer_ut.cpp
+ mirrorer_ut.cpp
pq_ut.cpp
type_codecs_ut.cpp
pq_ut.h
sourceid_ut.cpp
- user_info_ut.cpp
+ user_info_ut.cpp
)
END()
diff --git a/ydb/core/persqueue/ut_slow/ya.make b/ydb/core/persqueue/ut_slow/ya.make
index c7fd14c5eb..9e018d205b 100644
--- a/ydb/core/persqueue/ut_slow/ya.make
+++ b/ydb/core/persqueue/ut_slow/ya.make
@@ -1,10 +1,10 @@
UNITTEST_FOR(ydb/core/persqueue)
-OWNER(
- alexnick
+OWNER(
+ alexnick
g:kikimr
- g:logbroker
-)
+ g:logbroker
+)
FORK_SUBTESTS()
diff --git a/ydb/core/persqueue/working_time_counter.h b/ydb/core/persqueue/working_time_counter.h
index 5e0cfb1481..4f4d81d263 100644
--- a/ydb/core/persqueue/working_time_counter.h
+++ b/ydb/core/persqueue/working_time_counter.h
@@ -2,8 +2,8 @@
#include <ydb/core/protos/counters_pq.pb.h>
-#include <library/cpp/monlib/dynamic_counters/counters.h>
-
+#include <library/cpp/monlib/dynamic_counters/counters.h>
+
namespace NKikimr {
namespace NPQ {
@@ -11,26 +11,26 @@ namespace NPQ {
class TWorkingTimeCounter {
private:
bool IsInWorkingState;
- NMonitoring::TDynamicCounters::TCounterPtr WorkingTimeMicroSec;
+ NMonitoring::TDynamicCounters::TCounterPtr WorkingTimeMicroSec;
TInstant LastUpdateTimestamp;
public:
TWorkingTimeCounter(NMonitoring::TDynamicCounters::TCounterPtr counter)
: IsInWorkingState(false)
- , WorkingTimeMicroSec(counter)
+ , WorkingTimeMicroSec(counter)
{}
void UpdateState(bool state) {
IsInWorkingState = state;
}
- void UpdateWorkingTime(const TInstant now) {
- if (!WorkingTimeMicroSec) //no counter
+ void UpdateWorkingTime(const TInstant now) {
+ if (!WorkingTimeMicroSec) //no counter
return;
if (IsInWorkingState && LastUpdateTimestamp > TInstant::Zero()) {
- TDuration res = now - LastUpdateTimestamp;
- (*WorkingTimeMicroSec) += res.MicroSeconds();
- LastUpdateTimestamp += res;
+ TDuration res = now - LastUpdateTimestamp;
+ (*WorkingTimeMicroSec) += res.MicroSeconds();
+ LastUpdateTimestamp += res;
} else {
LastUpdateTimestamp = now;
}
diff --git a/ydb/core/persqueue/write_meta.cpp b/ydb/core/persqueue/write_meta.cpp
index 26d090260c..58bed2badf 100644
--- a/ydb/core/persqueue/write_meta.cpp
+++ b/ydb/core/persqueue/write_meta.cpp
@@ -1,53 +1,53 @@
-#include "write_meta.h"
-
+#include "write_meta.h"
+
#include <ydb/core/persqueue/codecs/pqv1.h>
-
-
-namespace NKikimr {
-
-
-void SetMetaField(NKikimrPQClient::TDataChunk& proto, const TString& key, const TString& value) {
- if (key == "server") {
- proto.MutableMeta()->SetServer(value);
- } else if (key == "ident") {
- proto.MutableMeta()->SetIdent(value);
- } else if (key == "logtype") {
- proto.MutableMeta()->SetLogType(value);
- } else if (key == "file") {
- proto.MutableMeta()->SetFile(value);
- } else {
- auto res = proto.MutableExtraFields()->AddItems();
- res->SetKey(key);
- res->SetValue(value);
- }
-}
-
-TString GetSerializedData(const NYdb::NPersQueue::TReadSessionEvent::TDataReceivedEvent::TCompressedMessage& message) {
- NKikimrPQClient::TDataChunk proto;
- for (const auto& item : message.GetMeta(0)->Fields) {
- SetMetaField(proto, item.first, item.second);
- }
- proto.SetIp(message.GetIp(0));
- proto.SetSeqNo(message.GetSeqNo(0));
- proto.SetCreateTime(message.GetCreateTime(0).MilliSeconds());
- auto codec = NPQ::FromV1Codec(message.GetCodec());
- Y_VERIFY(codec);
- proto.SetCodec(codec.value());
- proto.SetData(message.GetData());
-
- TString str;
- bool res = proto.SerializeToString(&str);
- Y_VERIFY(res);
- return str;
-}
-
-NKikimrPQClient::TDataChunk GetDeserializedData(const TString& string) {
- NKikimrPQClient::TDataChunk proto;
- bool res = proto.ParseFromString(string);
- Y_UNUSED(res);
- //TODO: check errors of parsing
- return proto;
-}
-
-} // NKikimr
-
+
+
+namespace NKikimr {
+
+
+void SetMetaField(NKikimrPQClient::TDataChunk& proto, const TString& key, const TString& value) {
+ if (key == "server") {
+ proto.MutableMeta()->SetServer(value);
+ } else if (key == "ident") {
+ proto.MutableMeta()->SetIdent(value);
+ } else if (key == "logtype") {
+ proto.MutableMeta()->SetLogType(value);
+ } else if (key == "file") {
+ proto.MutableMeta()->SetFile(value);
+ } else {
+ auto res = proto.MutableExtraFields()->AddItems();
+ res->SetKey(key);
+ res->SetValue(value);
+ }
+}
+
+TString GetSerializedData(const NYdb::NPersQueue::TReadSessionEvent::TDataReceivedEvent::TCompressedMessage& message) {
+ NKikimrPQClient::TDataChunk proto;
+ for (const auto& item : message.GetMeta(0)->Fields) {
+ SetMetaField(proto, item.first, item.second);
+ }
+ proto.SetIp(message.GetIp(0));
+ proto.SetSeqNo(message.GetSeqNo(0));
+ proto.SetCreateTime(message.GetCreateTime(0).MilliSeconds());
+ auto codec = NPQ::FromV1Codec(message.GetCodec());
+ Y_VERIFY(codec);
+ proto.SetCodec(codec.value());
+ proto.SetData(message.GetData());
+
+ TString str;
+ bool res = proto.SerializeToString(&str);
+ Y_VERIFY(res);
+ return str;
+}
+
+NKikimrPQClient::TDataChunk GetDeserializedData(const TString& string) {
+ NKikimrPQClient::TDataChunk proto;
+ bool res = proto.ParseFromString(string);
+ Y_UNUSED(res);
+ //TODO: check errors of parsing
+ return proto;
+}
+
+} // NKikimr
+
diff --git a/ydb/core/persqueue/write_meta.h b/ydb/core/persqueue/write_meta.h
index fe7c4f349e..0fba086633 100644
--- a/ydb/core/persqueue/write_meta.h
+++ b/ydb/core/persqueue/write_meta.h
@@ -1,73 +1,73 @@
-#pragma once
-
+#pragma once
+
#include <ydb/core/protos/grpc_pq_old.pb.h>
#include <ydb/public/sdk/cpp/client/ydb_persqueue_core/persqueue.h>
-
-#include <util/string/vector.h>
-
-namespace NKikimr {
-
-template <class TWriteRequestInit>
-void FillExtraFieldsForDataChunk(
- const TWriteRequestInit& init, NKikimrPQClient::TDataChunk& data, TString& server, TString& ident, TString& logType, TString& file);
-
-
-template <class TWriteRequestInit>
-NKikimrPQClient::TDataChunk GetInitialDataChunk(const TWriteRequestInit& init, const TString& topic, const TString& peerName) {
- NKikimrPQClient::TDataChunk data;
- TString server, ident, logType, file;
-
- FillExtraFieldsForDataChunk(init, data, server, ident, logType, file);
-
- if (server.empty()) {
- server = peerName;
- }
- if (ident.empty()) {
- auto p = SplitString(topic, "--");
- if (p.size() == 3)
- ident = p[1];
- else
- ident = "unknown";
- }
- if (logType.empty()) {
- auto p = SplitString(topic, "--");
- if (p.size() == 3)
- logType = p.back();
- else
- logType = "unknown";
- }
-
- data.MutableMeta()->SetServer(server);
- data.MutableMeta()->SetIdent(ident);
- data.MutableMeta()->SetLogType(logType);
- if (!file.empty())
- data.MutableMeta()->SetFile(file);
-
- data.SetIp(peerName);
- return data;
-}
-
-template <class TData>
-void FillChunkDataFromReq(NKikimrPQClient::TDataChunk& proto, const TData& data, int);
-
-template <class TData>
-void FillChunkDataFromReq(NKikimrPQClient::TDataChunk& proto, const TData& data);
-
-template <typename... TArgs>
-TString GetSerializedData(const NKikimrPQClient::TDataChunk& init, TArgs&...args) {
- NKikimrPQClient::TDataChunk proto;
- proto.CopyFrom(init);
- FillChunkDataFromReq(proto, std::forward<TArgs>(args)...);
-
- TString str;
- bool res = proto.SerializeToString(&str);
- Y_VERIFY(res);
- return str;
-}
-
-TString GetSerializedData(const NYdb::NPersQueue::TReadSessionEvent::TDataReceivedEvent::TCompressedMessage& message);
-
-NKikimrPQClient::TDataChunk GetDeserializedData(const TString& string);
-
-} // NKikimr
-
+
+#include <util/string/vector.h>
+
+namespace NKikimr {
+
+template <class TWriteRequestInit>
+void FillExtraFieldsForDataChunk(
+ const TWriteRequestInit& init, NKikimrPQClient::TDataChunk& data, TString& server, TString& ident, TString& logType, TString& file);
+
+
+template <class TWriteRequestInit>
+NKikimrPQClient::TDataChunk GetInitialDataChunk(const TWriteRequestInit& init, const TString& topic, const TString& peerName) {
+ NKikimrPQClient::TDataChunk data;
+ TString server, ident, logType, file;
+
+ FillExtraFieldsForDataChunk(init, data, server, ident, logType, file);
+
+ if (server.empty()) {
+ server = peerName;
+ }
+ if (ident.empty()) {
+ auto p = SplitString(topic, "--");
+ if (p.size() == 3)
+ ident = p[1];
+ else
+ ident = "unknown";
+ }
+ if (logType.empty()) {
+ auto p = SplitString(topic, "--");
+ if (p.size() == 3)
+ logType = p.back();
+ else
+ logType = "unknown";
+ }
+
+ data.MutableMeta()->SetServer(server);
+ data.MutableMeta()->SetIdent(ident);
+ data.MutableMeta()->SetLogType(logType);
+ if (!file.empty())
+ data.MutableMeta()->SetFile(file);
+
+ data.SetIp(peerName);
+ return data;
+}
+
+template <class TData>
+void FillChunkDataFromReq(NKikimrPQClient::TDataChunk& proto, const TData& data, int);
+
+template <class TData>
+void FillChunkDataFromReq(NKikimrPQClient::TDataChunk& proto, const TData& data);
+
+template <typename... TArgs>
+TString GetSerializedData(const NKikimrPQClient::TDataChunk& init, TArgs&...args) {
+ NKikimrPQClient::TDataChunk proto;
+ proto.CopyFrom(init);
+ FillChunkDataFromReq(proto, std::forward<TArgs>(args)...);
+
+ TString str;
+ bool res = proto.SerializeToString(&str);
+ Y_VERIFY(res);
+ return str;
+}
+
+TString GetSerializedData(const NYdb::NPersQueue::TReadSessionEvent::TDataReceivedEvent::TCompressedMessage& message);
+
+NKikimrPQClient::TDataChunk GetDeserializedData(const TString& string);
+
+} // NKikimr
+
diff --git a/ydb/core/persqueue/writer/source_id_encoding.cpp b/ydb/core/persqueue/writer/source_id_encoding.cpp
index 81ede360c9..13a37c2c61 100644
--- a/ydb/core/persqueue/writer/source_id_encoding.cpp
+++ b/ydb/core/persqueue/writer/source_id_encoding.cpp
@@ -1,14 +1,14 @@
-#include "source_id_encoding.h"
-
-#include <library/cpp/string_utils/base64/base64.h>
-
-#include <util/generic/yexception.h>
-#include <util/string/strip.h>
-
-namespace NKikimr {
+#include "source_id_encoding.h"
+
+#include <library/cpp/string_utils/base64/base64.h>
+
+#include <util/generic/yexception.h>
+#include <util/string/strip.h>
+
+namespace NKikimr {
namespace NPQ {
namespace NSourceIdEncoding {
-
+
struct TTags {
static constexpr char Simple = 0;
static constexpr char Base64 = 1;
@@ -16,10 +16,10 @@ struct TTags {
static constexpr TStringBuf Base64Prefix = "base64:";
-TString EncodeSimple(const TString& sourceId) {
+TString EncodeSimple(const TString& sourceId) {
return TString(1, TTags::Simple) + sourceId;
-}
-
+}
+
TString DecodeSimple(const TString& sourceId) {
Y_VERIFY(!sourceId.empty() && sourceId[0] == TTags::Simple);
return sourceId.substr(1);
@@ -49,14 +49,14 @@ TString DecodeBase64(const TString& sourceId) {
return Base64Prefix + StripStringRight(Base64EncodeUrl(sourceId.substr(1)), EqualsStripAdapter(','));
}
-TString Encode(const TString& sourceId) {
+TString Encode(const TString& sourceId) {
if (sourceId.StartsWith(Base64Prefix)) {
return EncodeBase64(sourceId);
- } else {
+ } else {
return EncodeSimple(sourceId);
- }
-}
-
+ }
+}
+
TString Decode(const TString& sourceId) {
Y_VERIFY(!sourceId.empty());
@@ -67,9 +67,9 @@ TString Decode(const TString& sourceId) {
return DecodeBase64(sourceId);
default:
return sourceId;
- }
-}
-
+ }
+}
+
bool IsValidEncoded(const TString& sourceId) {
if (sourceId.empty()) {
return false;
@@ -82,8 +82,8 @@ bool IsValidEncoded(const TString& sourceId) {
default:
return false;
}
-}
-
+}
+
} // NSourceIdEncoding
} // NPQ
} // NKikimr
diff --git a/ydb/core/persqueue/writer/source_id_encoding.h b/ydb/core/persqueue/writer/source_id_encoding.h
index bfd6727cfc..6472d8b4ac 100644
--- a/ydb/core/persqueue/writer/source_id_encoding.h
+++ b/ydb/core/persqueue/writer/source_id_encoding.h
@@ -1,18 +1,18 @@
-#pragma once
-
-#include <util/generic/fwd.h>
-
-namespace NKikimr {
+#pragma once
+
+#include <util/generic/fwd.h>
+
+namespace NKikimr {
namespace NPQ {
namespace NSourceIdEncoding {
-
-TString EncodeSimple(const TString& sourceId);
-TString Encode(const TString& sourceId);
-
-TString Decode(const TString& encodedSourceId);
-
-bool IsValidEncoded(const TString& encodedSourceId);
-
+
+TString EncodeSimple(const TString& sourceId);
+TString Encode(const TString& sourceId);
+
+TString Decode(const TString& encodedSourceId);
+
+bool IsValidEncoded(const TString& encodedSourceId);
+
} // NSourceIdEncoding
} // NPQ
} // NKikimr
diff --git a/ydb/core/persqueue/ya.make b/ydb/core/persqueue/ya.make
index 0adff6544e..50e89676ef 100644
--- a/ydb/core/persqueue/ya.make
+++ b/ydb/core/persqueue/ya.make
@@ -3,30 +3,30 @@ LIBRARY()
OWNER(
alexnick
g:kikimr
- g:logbroker
+ g:logbroker
)
SRCS(
cluster_tracker.cpp
blob.cpp
- event_helpers.cpp
+ event_helpers.cpp
header.cpp
percentile_counter.cpp
pq.cpp
pq_database.cpp
pq_impl.cpp
sourceid.cpp
- mirrorer.cpp
- mirrorer.h
+ mirrorer.cpp
+ mirrorer.h
ownerinfo.cpp
partition.cpp
pq_l2_cache.cpp
read_balancer.cpp
- read_speed_limiter.cpp
+ read_speed_limiter.cpp
subscriber.cpp
type_codecs_defs.cpp
- user_info.cpp
- write_meta.cpp
+ user_info.cpp
+ write_meta.cpp
actor_persqueue_client_iface.h
)
diff --git a/ydb/core/protos/config.proto b/ydb/core/protos/config.proto
index 1c5711a15d..d64169d4fc 100644
--- a/ydb/core/protos/config.proto
+++ b/ydb/core/protos/config.proto
@@ -839,8 +839,8 @@ message TSqsConfig {
optional uint64 AddMessagesToInflyMinCheckAttempts = 63 [default = 10];
optional uint64 MinimumGarbageAgeSeconds = 64 [default = 3600];
-
- optional bool MeteringByNetClassifierOnly = 65 [default = false];
+
+ optional bool MeteringByNetClassifierOnly = 65 [default = false];
message TYcSearchEventsConfig {
optional bool EnableYcSearch = 1 [default = false];
diff --git a/ydb/core/protos/counters_pq.proto b/ydb/core/protos/counters_pq.proto
index e2aa4c8d8c..7233758135 100644
--- a/ydb/core/protos/counters_pq.proto
+++ b/ydb/core/protos/counters_pq.proto
@@ -55,7 +55,7 @@ enum ECumulativeCounters {
COUNTER_PQ_WRITE_SMALL_OFFSET = 34 [(CounterOpts) = {Name: "WriteSmallOffset"}];
COUNTER_PQ_TABLET_CPU_USAGE = 35 [(CounterOpts) = {Name: "CPUUsage"}];
- COUNTER_PQ_TABLET_NETWORK_BYTES_USAGE = 36 [(CounterOpts) = {Name: "NetworkUsage"}];
+ COUNTER_PQ_TABLET_NETWORK_BYTES_USAGE = 36 [(CounterOpts) = {Name: "NetworkUsage"}];
}
enum ESimpleCounters {
diff --git a/ydb/core/protos/msgbus_pq.proto b/ydb/core/protos/msgbus_pq.proto
index 2fc298d01e..849b292383 100644
--- a/ydb/core/protos/msgbus_pq.proto
+++ b/ydb/core/protos/msgbus_pq.proto
@@ -378,7 +378,7 @@ message TCmdReadResult {
optional uint64 BlobsCachedSize = 8;
optional uint64 SizeLag = 9;
optional uint64 RealReadOffset = 10;
- optional uint64 WaitQuotaTimeMs = 11;
+ optional uint64 WaitQuotaTimeMs = 11;
}
diff --git a/ydb/core/protos/netclassifier.proto b/ydb/core/protos/netclassifier.proto
index 7f76031514..0c71eae7bf 100644
--- a/ydb/core/protos/netclassifier.proto
+++ b/ydb/core/protos/netclassifier.proto
@@ -12,7 +12,7 @@ message TNetClassifierUpdaterConfig {
optional uint32 RetryIntervalSeconds = 2;
optional uint32 NetDataUpdateIntervalSeconds = 3;
optional EFormat Format = 4 [default = TSV];
- repeated string NetBoxTags = 5;
+ repeated string NetBoxTags = 5;
}
message TSubnet {
diff --git a/ydb/core/protos/out/out.cpp b/ydb/core/protos/out/out.cpp
index 2609cf635c..28c49a5c36 100644
--- a/ydb/core/protos/out/out.cpp
+++ b/ydb/core/protos/out/out.cpp
@@ -10,65 +10,65 @@
#include <ydb/core/protos/resource_broker.pb.h>
#include <ydb/core/protos/tenant_pool.pb.h>
#include <ydb/core/protos/tx_datashard.pb.h>
-
-#include <util/stream/output.h>
-
-Y_DECLARE_OUT_SPEC(, NKikimrBlobStorage::EPutHandleClass, stream, value) {
- stream << NKikimrBlobStorage::EPutHandleClass_Name(value);
-}
-
+
+#include <util/stream/output.h>
+
+Y_DECLARE_OUT_SPEC(, NKikimrBlobStorage::EPutHandleClass, stream, value) {
+ stream << NKikimrBlobStorage::EPutHandleClass_Name(value);
+}
+
Y_DECLARE_OUT_SPEC(, NKikimrBlobStorage::EGetHandleClass, stream, value) {
stream << NKikimrBlobStorage::EGetHandleClass_Name(value);
}
-Y_DECLARE_OUT_SPEC(, NKikimrBlobStorage::TVDiskKind::EVDiskKind, stream, value) {
- stream << NKikimrBlobStorage::TVDiskKind::EVDiskKind_Name(value);
-}
-
-Y_DECLARE_OUT_SPEC(, NKikimrVDiskData::TSyncerVDiskEntry::ESyncStatus, stream, value) {
- stream << NKikimrVDiskData::TSyncerVDiskEntry::ESyncStatus_Name(value);
-}
-
-Y_DECLARE_OUT_SPEC(, NKikimrBlobStorage::TSyncGuidInfo::EState, stream, value) {
- stream << NKikimrBlobStorage::TSyncGuidInfo::EState_Name(value);
-}
-
-Y_DECLARE_OUT_SPEC(, NKikimrBlobStorage::TLocalGuidInfo::EState, stream, value) {
- stream << NKikimrBlobStorage::TLocalGuidInfo::EState_Name(value);
-}
-
-Y_DECLARE_OUT_SPEC(, NKikimrBlobStorage::TSyncerStatus::EPhase, stream, value) {
- stream << NKikimrBlobStorage::TSyncerStatus::EPhase_Name(value);
-}
-
-Y_DECLARE_OUT_SPEC(, NKikimrBlobStorage::EVDiskQueueId, stream, value) {
- stream << NKikimrBlobStorage::EVDiskQueueId_Name(value);
-}
-
-Y_DECLARE_OUT_SPEC(, NKikimrBlobStorage::EVDiskInternalQueueId, stream, value) {
- stream << NKikimrBlobStorage::EVDiskInternalQueueId_Name(value);
-}
-
-Y_DECLARE_OUT_SPEC(, NKikimrBlobStorage::ESyncFullStage, stream, value) {
- stream << NKikimrBlobStorage::ESyncFullStage_Name(value);
-}
-
-Y_DECLARE_OUT_SPEC(, NKikimrProto::EReplyStatus, stream, value) {
- stream << NKikimrProto::EReplyStatus_Name(value);
-}
-
-Y_DECLARE_OUT_SPEC(, NKikimrCms::TStatus::ECode, stream, value) {
- stream << NKikimrCms::TStatus::ECode_Name(value);
-}
-
-Y_DECLARE_OUT_SPEC(, NKikimrCms::EState, stream, value) {
- stream << NKikimrCms::EState_Name(value);
-}
-
-Y_DECLARE_OUT_SPEC(, NKikimrBlobStorage::EDriveStatus, stream, value) {
- stream << NKikimrBlobStorage::EDriveStatus_Name(value);
-}
-
+Y_DECLARE_OUT_SPEC(, NKikimrBlobStorage::TVDiskKind::EVDiskKind, stream, value) {
+ stream << NKikimrBlobStorage::TVDiskKind::EVDiskKind_Name(value);
+}
+
+Y_DECLARE_OUT_SPEC(, NKikimrVDiskData::TSyncerVDiskEntry::ESyncStatus, stream, value) {
+ stream << NKikimrVDiskData::TSyncerVDiskEntry::ESyncStatus_Name(value);
+}
+
+Y_DECLARE_OUT_SPEC(, NKikimrBlobStorage::TSyncGuidInfo::EState, stream, value) {
+ stream << NKikimrBlobStorage::TSyncGuidInfo::EState_Name(value);
+}
+
+Y_DECLARE_OUT_SPEC(, NKikimrBlobStorage::TLocalGuidInfo::EState, stream, value) {
+ stream << NKikimrBlobStorage::TLocalGuidInfo::EState_Name(value);
+}
+
+Y_DECLARE_OUT_SPEC(, NKikimrBlobStorage::TSyncerStatus::EPhase, stream, value) {
+ stream << NKikimrBlobStorage::TSyncerStatus::EPhase_Name(value);
+}
+
+Y_DECLARE_OUT_SPEC(, NKikimrBlobStorage::EVDiskQueueId, stream, value) {
+ stream << NKikimrBlobStorage::EVDiskQueueId_Name(value);
+}
+
+Y_DECLARE_OUT_SPEC(, NKikimrBlobStorage::EVDiskInternalQueueId, stream, value) {
+ stream << NKikimrBlobStorage::EVDiskInternalQueueId_Name(value);
+}
+
+Y_DECLARE_OUT_SPEC(, NKikimrBlobStorage::ESyncFullStage, stream, value) {
+ stream << NKikimrBlobStorage::ESyncFullStage_Name(value);
+}
+
+Y_DECLARE_OUT_SPEC(, NKikimrProto::EReplyStatus, stream, value) {
+ stream << NKikimrProto::EReplyStatus_Name(value);
+}
+
+Y_DECLARE_OUT_SPEC(, NKikimrCms::TStatus::ECode, stream, value) {
+ stream << NKikimrCms::TStatus::ECode_Name(value);
+}
+
+Y_DECLARE_OUT_SPEC(, NKikimrCms::EState, stream, value) {
+ stream << NKikimrCms::EState_Name(value);
+}
+
+Y_DECLARE_OUT_SPEC(, NKikimrBlobStorage::EDriveStatus, stream, value) {
+ stream << NKikimrBlobStorage::EDriveStatus_Name(value);
+}
+
Y_DECLARE_OUT_SPEC(, NKikimrBlobStorage::TGroupStatus::E, stream, value) {
stream << NKikimrBlobStorage::TGroupStatus::E_Name(value);
}
@@ -81,65 +81,65 @@ Y_DECLARE_OUT_SPEC(, NKikimrBlobStorage::TSerialManagementStage::E, stream, valu
stream << NKikimrBlobStorage::TSerialManagementStage::E_Name(value);
}
-Y_DECLARE_OUT_SPEC(, NKikimrResourceBroker::EResourceType, stream, value) {
- stream << NKikimrResourceBroker::EResourceType_Name(value);
-}
+Y_DECLARE_OUT_SPEC(, NKikimrResourceBroker::EResourceType, stream, value) {
+ stream << NKikimrResourceBroker::EResourceType_Name(value);
+}
/* FIXME
-Y_DECLARE_OUT_SPEC(, Ydb::Cms::GetDatabaseStatusResult::State, stream, value) {
- stream << Ydb::Cms::GetDatabaseStatusResult::State_Name(value);
-}
-
-Y_DECLARE_OUT_SPEC(, Ydb::StatusIds::StatusCode, stream, value) {
- stream << Ydb::StatusIds::StatusCode_Name(value);
-}
+Y_DECLARE_OUT_SPEC(, Ydb::Cms::GetDatabaseStatusResult::State, stream, value) {
+ stream << Ydb::Cms::GetDatabaseStatusResult::State_Name(value);
+}
+
+Y_DECLARE_OUT_SPEC(, Ydb::StatusIds::StatusCode, stream, value) {
+ stream << Ydb::StatusIds::StatusCode_Name(value);
+}
*/
-Y_DECLARE_OUT_SPEC(, NKikimrConsole::TConfigItem::EKind, stream, value) {
- stream << NKikimrConsole::TConfigItem::EKind_Name(value);
-}
-
-Y_DECLARE_OUT_SPEC(, NKikimrTenantPool::EStatus, stream, value) {
- stream << NKikimrTenantPool::EStatus_Name(value);
-}
-
-Y_DECLARE_OUT_SPEC(, NKikimrBlobStorage::EPDiskType, stream, value) {
- stream << NKikimrBlobStorage::EPDiskType_Name(value);
-}
-
+Y_DECLARE_OUT_SPEC(, NKikimrConsole::TConfigItem::EKind, stream, value) {
+ stream << NKikimrConsole::TConfigItem::EKind_Name(value);
+}
+
+Y_DECLARE_OUT_SPEC(, NKikimrTenantPool::EStatus, stream, value) {
+ stream << NKikimrTenantPool::EStatus_Name(value);
+}
+
+Y_DECLARE_OUT_SPEC(, NKikimrBlobStorage::EPDiskType, stream, value) {
+ stream << NKikimrBlobStorage::EPDiskType_Name(value);
+}
+
Y_DECLARE_OUT_SPEC(, NKikimrBlobStorage::EVDiskStatus, stream, value) {
stream << NKikimrBlobStorage::EVDiskStatus_Name(value);
}
-Y_DECLARE_OUT_SPEC(, NKikimrNodeBroker::TStatus::ECode, stream, value) {
- stream << NKikimrNodeBroker::TStatus::ECode_Name(value);
-}
-
-Y_DECLARE_OUT_SPEC(, NKikimrCms::EMarker, stream, value) {
- stream << NKikimrCms::EMarker_Name(value);
-}
-
-Y_DECLARE_OUT_SPEC(, NKikimrTxDataShard::ETransactionKind, stream, value) {
- stream << NKikimrTxDataShard::ETransactionKind_Name(value);
-}
-
-Y_DECLARE_OUT_SPEC(, NKikimrCms::ETextFormat, stream, value) {
- stream << NKikimrCms::ETextFormat_Name(value);
-}
-
-Y_DECLARE_OUT_SPEC(, NKikimrCms::TLogRecordData::EType, stream, value) {
- stream << NKikimrCms::TLogRecordData::EType_Name(value);
-}
-
-Y_DECLARE_OUT_SPEC(, NKikimrConfig::TBootstrap::ETabletType, stream, value) {
- stream << NKikimrConfig::TBootstrap::ETabletType_Name(value);
-}
-
-Y_DECLARE_OUT_SPEC(, NKikimrTxDataShard::TEvProposeTransactionResult::EStatus, stream, value) {
- stream << NKikimrTxDataShard::TEvProposeTransactionResult::EStatus_Name(value);
-}
-
-Y_DECLARE_OUT_SPEC(, NKikimrTxDataShard::EDatashardState, stream, value) {
- stream << NKikimrTxDataShard::EDatashardState_Name(value);
-}
+Y_DECLARE_OUT_SPEC(, NKikimrNodeBroker::TStatus::ECode, stream, value) {
+ stream << NKikimrNodeBroker::TStatus::ECode_Name(value);
+}
+
+Y_DECLARE_OUT_SPEC(, NKikimrCms::EMarker, stream, value) {
+ stream << NKikimrCms::EMarker_Name(value);
+}
+
+Y_DECLARE_OUT_SPEC(, NKikimrTxDataShard::ETransactionKind, stream, value) {
+ stream << NKikimrTxDataShard::ETransactionKind_Name(value);
+}
+
+Y_DECLARE_OUT_SPEC(, NKikimrCms::ETextFormat, stream, value) {
+ stream << NKikimrCms::ETextFormat_Name(value);
+}
+
+Y_DECLARE_OUT_SPEC(, NKikimrCms::TLogRecordData::EType, stream, value) {
+ stream << NKikimrCms::TLogRecordData::EType_Name(value);
+}
+
+Y_DECLARE_OUT_SPEC(, NKikimrConfig::TBootstrap::ETabletType, stream, value) {
+ stream << NKikimrConfig::TBootstrap::ETabletType_Name(value);
+}
+
+Y_DECLARE_OUT_SPEC(, NKikimrTxDataShard::TEvProposeTransactionResult::EStatus, stream, value) {
+ stream << NKikimrTxDataShard::TEvProposeTransactionResult::EStatus_Name(value);
+}
+
+Y_DECLARE_OUT_SPEC(, NKikimrTxDataShard::EDatashardState, stream, value) {
+ stream << NKikimrTxDataShard::EDatashardState_Name(value);
+}
Y_DECLARE_OUT_SPEC(, NKikimrBlobStorage::TPDiskState::E, stream, value) {
stream << NKikimrBlobStorage::TPDiskState::E_Name(value);
diff --git a/ydb/core/protos/out/ya.make b/ydb/core/protos/out/ya.make
index d56ff87819..bb7ca44717 100644
--- a/ydb/core/protos/out/ya.make
+++ b/ydb/core/protos/out/ya.make
@@ -1,14 +1,14 @@
-LIBRARY()
+LIBRARY()
-OWNER(g:kikimr)
-
-SRCS(
- out.cpp
+OWNER(g:kikimr)
+
+SRCS(
+ out.cpp
out_sequenceshard.cpp
-)
-
-PEERDIR(
+)
+
+PEERDIR(
ydb/core/protos
-)
-
-END()
+)
+
+END()
diff --git a/ydb/core/protos/pqconfig.proto b/ydb/core/protos/pqconfig.proto
index 14b11d3678..7c85927449 100644
--- a/ydb/core/protos/pqconfig.proto
+++ b/ydb/core/protos/pqconfig.proto
@@ -67,8 +67,8 @@ message TPQConfig {
WRITTEN_BLOB_SIZE = 1; // Written blob size, including additional data for compaction.
USER_PAYLOAD_SIZE = 2; // Message + source id size.
}
- optional bool EnableReadQuoting = 4 [default = false];
- optional uint64 ReadCreditBytes = 5 [default = 100000];
+ optional bool EnableReadQuoting = 4 [default = false];
+ optional uint64 ReadCreditBytes = 5 [default = 100000];
optional uint64 QuotaWaitDurationMs = 6 [default = 0]; // 0 means infinity
@@ -85,18 +85,18 @@ message TPQConfig {
optional string Root = 27 [default = "/Root/PQ"];
- message TPQLibSettings {
- optional uint32 ThreadsCount = 1 [default = 1];
- optional uint32 CompressionPoolThreads = 2 [default = 1];
- optional uint32 GRpcThreads = 3 [default = 1];
- }
-
- message TMirrorConfig {
- optional bool Enabled = 1 [default = true];
- optional TPQLibSettings PQLibSettings = 2;
- }
-
- optional TMirrorConfig MirrorConfig = 29;
+ message TPQLibSettings {
+ optional uint32 ThreadsCount = 1 [default = 1];
+ optional uint32 CompressionPoolThreads = 2 [default = 1];
+ optional uint32 GRpcThreads = 3 [default = 1];
+ }
+
+ message TMirrorConfig {
+ optional bool Enabled = 1 [default = true];
+ optional TPQLibSettings PQLibSettings = 2;
+ }
+
+ optional TMirrorConfig MirrorConfig = 29;
optional uint64 MinWriteLatencyMs = 30 [default = 0];
@@ -108,20 +108,20 @@ message TPQConfig {
repeated uint32 ValidWriteSpeedLimitsKbPerSec = 34;
- message TReadMeteringConfig {
- message TPricing {
- optional string Name = 1 [default = "unknown"];
- }
- repeated TPricing Pricing = 2;
- optional TPricing UnknownPricing = 3;
- }
-
+ message TReadMeteringConfig {
+ message TPricing {
+ optional string Name = 1 [default = "unknown"];
+ }
+ repeated TPricing Pricing = 2;
+ optional TPricing UnknownPricing = 3;
+ }
+
message TBillingMeteringConfig {
optional bool Enabled = 1 [default = false];
optional uint64 FlushIntervalSec = 2 [default = 30];
- optional TReadMeteringConfig Read = 3;
+ optional TReadMeteringConfig Read = 3;
}
-
+
optional TBillingMeteringConfig BillingMeteringConfig = 35;
optional NKikimrNodeLimits.TNodeLimitsConfig.TPersQueueNodeConfig PersQueueNodeConfig = 36;
@@ -129,16 +129,16 @@ message TPQConfig {
optional bool EnableProtoSourceIdInfo = 37 [default = false];
optional string Database = 38;
-
- message TClientServiceType {
- optional string Name = 1 [default = "data-transfer"];
- optional string ReadPricingName = 2;
+
+ message TClientServiceType {
+ optional string Name = 1 [default = "data-transfer"];
+ optional string ReadPricingName = 2;
optional uint32 MaxReadRulesCountPerTopic = 3 [default = 0]; // 0 means no limit. If you want to allow only zero count, then do not specify ClientServiceType.
- }
-
- repeated TClientServiceType ClientServiceType = 39;
- optional TClientServiceType DefaultClientServiceType = 40;
- optional bool DisallowDefaultClientServiceType = 43 [default = false];
+ }
+
+ repeated TClientServiceType ClientServiceType = 39;
+ optional TClientServiceType DefaultClientServiceType = 40;
+ optional bool DisallowDefaultClientServiceType = 43 [default = false];
optional uint32 MetaCacheRefreshIntervalMilliSeconds = 41 [default = 10000];
optional bool MetaCacheSkipVersionCheck = 42 [default = false];
@@ -153,30 +153,30 @@ message TChannelProfile {
optional double WriteBandwidth = 6; // required bandwidth (in bytes/sec) for write requests
}
-message TMirrorPartitionConfig {
- message TCredentials {
- message IamCredentials {
- optional string Endpoint = 1;
- optional string ServiceAccountKey = 2;
- }
- oneof Credentials {
- string OauthToken = 1;
- string JwtParams = 2;
- IamCredentials Iam = 3;
- }
- }
-
- optional string Endpoint = 1;
- optional uint32 EndpointPort = 6;
- optional string Topic = 2;
- optional string Consumer = 3;
- optional uint64 ReadFromTimestampsMs = 5;
- optional TCredentials Credentials = 7;
- optional string Database = 8;
- optional bool UseSecureConnection = 9 [default = false];
- optional bool SyncWriteTime = 10 [default = false];
-}
-
+message TMirrorPartitionConfig {
+ message TCredentials {
+ message IamCredentials {
+ optional string Endpoint = 1;
+ optional string ServiceAccountKey = 2;
+ }
+ oneof Credentials {
+ string OauthToken = 1;
+ string JwtParams = 2;
+ IamCredentials Iam = 3;
+ }
+ }
+
+ optional string Endpoint = 1;
+ optional uint32 EndpointPort = 6;
+ optional string Topic = 2;
+ optional string Consumer = 3;
+ optional uint64 ReadFromTimestampsMs = 5;
+ optional TCredentials Credentials = 7;
+ optional string Database = 8;
+ optional bool UseSecureConnection = 9 [default = false];
+ optional bool SyncWriteTime = 10 [default = false];
+}
+
message TPartitionConfig {
optional int32 MaxCountInPartition = 1 [default = 10000000];
optional int64 MaxSizeInPartition = 2 [default = 1099511627776];
@@ -206,8 +206,8 @@ message TPartitionConfig {
optional uint32 TotalPartitions = 13 [default = 1];
repeated TChannelProfile ExplicitChannelProfiles = 14;
-
- optional TMirrorPartitionConfig MirrorFrom = 15;
+
+ optional TMirrorPartitionConfig MirrorFrom = 15;
};
message TPartitionKeyRange {
@@ -247,7 +247,7 @@ message TPQTabletConfig {
repeated string Codecs = 2;
}
repeated TCodecs ConsumerCodecs = 16;
- repeated string ReadRuleServiceTypes = 17;
+ repeated string ReadRuleServiceTypes = 17;
optional uint64 FormatVersion = 20;
optional TCodecs Codecs = 21;
@@ -357,7 +357,7 @@ message TDescribeResponse {
message TCheckACL {
optional EOperation Operation = 2;
- optional NPersQueueCommon.Credentials Auth = 4; //leaved for compatibility
+ optional NPersQueueCommon.Credentials Auth = 4; //leaved for compatibility
optional string User = 5;
optional bytes Token = 6;
}
@@ -571,12 +571,12 @@ message TStatusResponse {
STATUS_DISK_IS_FULL = 4;
}
- message TErrorMessage {
- optional uint64 Timestamp = 1;
- optional NKikimrServices.EServiceKikimr Service = 2;
- optional string Message = 3;
- }
-
+ message TErrorMessage {
+ optional uint64 Timestamp = 1;
+ optional NKikimrServices.EServiceKikimr Service = 2;
+ optional string Message = 3;
+ }
+
message TPartResult {
optional int32 Partition = 1; // mandatory
optional EStatus Status = 2;
@@ -615,8 +615,8 @@ message TStatusResponse {
optional int64 SourceIdCount = 27;
optional int64 SourceIdRetentionPeriodSec = 28;
-
- repeated TErrorMessage Errors = 29;
+
+ repeated TErrorMessage Errors = 29;
}
optional uint64 TabletId = 1;
diff --git a/ydb/core/protos/services.proto b/ydb/core/protos/services.proto
index f2d7b219ea..c17c8a7dc3 100644
--- a/ydb/core/protos/services.proto
+++ b/ydb/core/protos/services.proto
@@ -164,8 +164,8 @@ enum EServiceKikimr {
PQ_METACACHE = 441;
PQ_READ_PROXY = 442;
PQ_WRITE_PROXY = 443;
- PQ_MIRRORER = 446;
- PQ_READ_SPEED_LIMITER = 447;
+ PQ_MIRRORER = 446;
+ PQ_READ_SPEED_LIMITER = 447;
PERSQUEUE_READ_BALANCER = 448;
CHOOSE_PROXY = 444;
diff --git a/ydb/core/tablet/tablet_counters_aggregator.cpp b/ydb/core/tablet/tablet_counters_aggregator.cpp
index 18a589e6f0..1885b26e09 100644
--- a/ydb/core/tablet/tablet_counters_aggregator.cpp
+++ b/ydb/core/tablet/tablet_counters_aggregator.cpp
@@ -15,10 +15,10 @@
#include <ydb/core/tx/scheme_cache/scheme_cache.h>
#include <ydb/core/util/wildcard.h>
#include <ydb/library/persqueue/topic_parser/topic_parser.h>
-
+
#include <library/cpp/monlib/service/pages/templates.h>
#include <library/cpp/monlib/dynamic_counters/encode.h>
-
+
#include <util/generic/xrange.h>
#include <util/string/vector.h>
#include <util/string/split.h>
diff --git a/ydb/core/testlib/tablet_helpers.cpp b/ydb/core/testlib/tablet_helpers.cpp
index eed935e589..a853040b93 100644
--- a/ydb/core/testlib/tablet_helpers.cpp
+++ b/ydb/core/testlib/tablet_helpers.cpp
@@ -53,7 +53,7 @@ const bool TRACE_DELAY_TIMING = true;
const bool SUPPRESS_DELAYS = false;
const bool VARIATE_RANDOM_SEED = false;
const ui64 PQ_CACHE_MAX_SIZE_MB = 32;
-const TDuration PQ_CACHE_KEEP_TIMEOUT = TDuration::Seconds(10);
+const TDuration PQ_CACHE_KEEP_TIMEOUT = TDuration::Seconds(10);
static NActors::TTestActorRuntime& AsKikimrRuntime(NActors::TTestActorRuntimeBase& r) {
try {
@@ -113,7 +113,7 @@ namespace NKikimr {
void SetupPQNodeCache(TTestActorRuntime& runtime, ui32 nodeIndex)
{
- struct NPQ::TCacheL2Parameters l2Params = {PQ_CACHE_MAX_SIZE_MB, PQ_CACHE_KEEP_TIMEOUT};
+ struct NPQ::TCacheL2Parameters l2Params = {PQ_CACHE_MAX_SIZE_MB, PQ_CACHE_KEEP_TIMEOUT};
runtime.AddLocalService(NPQ::MakePersQueueL2CacheID(),
TActorSetupCmd(
NPQ::CreateNodePersQueueL2Cache(l2Params, runtime.GetDynamicCounters(0)),
diff --git a/ydb/core/testlib/test_client.cpp b/ydb/core/testlib/test_client.cpp
index 13171086ec..d4907c26f1 100644
--- a/ydb/core/testlib/test_client.cpp
+++ b/ydb/core/testlib/test_client.cpp
@@ -226,20 +226,20 @@ namespace Tests {
if (port) {
Bus = NBus::CreateMessageQueue(NBus::TBusQueueConfig());
if (tracePath) {
- BusServer.Reset(NMsgBusProxy::CreateMsgBusTracingServer(
- Bus.Get(),
- BusServerSessionConfig,
- tracePath,
- Settings->PersQueueGetReadSessionsInfoWorkerFactory,
- port
- ));
+ BusServer.Reset(NMsgBusProxy::CreateMsgBusTracingServer(
+ Bus.Get(),
+ BusServerSessionConfig,
+ tracePath,
+ Settings->PersQueueGetReadSessionsInfoWorkerFactory,
+ port
+ ));
} else {
- BusServer.Reset(NMsgBusProxy::CreateMsgBusServer(
- Bus.Get(),
- BusServerSessionConfig,
- Settings->PersQueueGetReadSessionsInfoWorkerFactory,
- port
- ));
+ BusServer.Reset(NMsgBusProxy::CreateMsgBusServer(
+ Bus.Get(),
+ BusServerSessionConfig,
+ Settings->PersQueueGetReadSessionsInfoWorkerFactory,
+ port
+ ));
}
}
}
@@ -704,7 +704,7 @@ namespace Tests {
TActorId netClassifierId = Runtime->Register(netClassifier, nodeIdx);
Runtime->RegisterService(NNetClassifier::MakeNetClassifierID(), netClassifierId, nodeIdx);
}
-
+
if (Settings->EnableYq) {
NYq::NConfig::TConfig protoConfig;
protoConfig.SetEnabled(true);
diff --git a/ydb/core/testlib/test_client.h b/ydb/core/testlib/test_client.h
index 39f644357e..2064752ab4 100644
--- a/ydb/core/testlib/test_client.h
+++ b/ydb/core/testlib/test_client.h
@@ -120,7 +120,7 @@ namespace Tests {
NKikimrConfig::TCompactionConfig CompactionConfig;
TMap<ui32, TString> NodeKeys;
ui64 DomainPlanResolution = 0;
- std::shared_ptr<NKikimr::NMsgBusProxy::IPersQueueGetReadSessionsInfoWorkerFactory> PersQueueGetReadSessionsInfoWorkerFactory;
+ std::shared_ptr<NKikimr::NMsgBusProxy::IPersQueueGetReadSessionsInfoWorkerFactory> PersQueueGetReadSessionsInfoWorkerFactory;
bool EnableMetering = false;
TString MeteringFilePath;
@@ -159,23 +159,23 @@ namespace Tests {
TServerSettings& SetChangesQueueItemsLimit(ui64 value) { ChangesQueueItemsLimit = value; return *this; }
TServerSettings& SetChangesQueueBytesLimit(ui64 value) { ChangesQueueBytesLimit = value; return *this; }
TServerSettings& SetMeteringFilePath(const TString& path) { EnableMetering = true; MeteringFilePath = path; return *this; }
- TServerSettings& SetPersQueueGetReadSessionsInfoWorkerFactory(
- std::shared_ptr<NKikimr::NMsgBusProxy::IPersQueueGetReadSessionsInfoWorkerFactory> factory
- ) {
- PersQueueGetReadSessionsInfoWorkerFactory = factory;
- return *this;
- }
+ TServerSettings& SetPersQueueGetReadSessionsInfoWorkerFactory(
+ std::shared_ptr<NKikimr::NMsgBusProxy::IPersQueueGetReadSessionsInfoWorkerFactory> factory
+ ) {
+ PersQueueGetReadSessionsInfoWorkerFactory = factory;
+ return *this;
+ }
// Add additional grpc services
template <typename TService>
- TServerSettings& RegisterGrpcService(
- const TString& name,
- std::optional<NActors::TActorId> proxyId = std::nullopt
- ) {
+ TServerSettings& RegisterGrpcService(
+ const TString& name,
+ std::optional<NActors::TActorId> proxyId = std::nullopt
+ ) {
if (!GrpcServiceFactory) {
GrpcServiceFactory = std::make_shared<TGrpcServiceFactory>();
}
- GrpcServiceFactory->Register<TService>(name, true, proxyId);
+ GrpcServiceFactory->Register<TService>(name, true, proxyId);
return *this;
}
diff --git a/ydb/core/testlib/test_pq_client.h b/ydb/core/testlib/test_pq_client.h
index 67e08f9809..f059923250 100644
--- a/ydb/core/testlib/test_pq_client.h
+++ b/ydb/core/testlib/test_pq_client.h
@@ -74,54 +74,54 @@ inline Tests::TServerSettings PQSettings(ui16 port, ui32 nodesCount = 2, bool ro
const TString TopicPrefix = "/Root/PQ/";
const static TString DEFAULT_SRC_IDS_PATH = "/Root/PQ/SourceIdMeta2";
-
+
struct TRequestCreatePQ {
- TRequestCreatePQ(
- const TString& topic,
- ui32 numParts,
- ui32 cacheSize = 0,
- ui64 lifetimeS = 86400,
- ui32 lowWatermark = 8 * 1024 * 1024,
- ui64 writeSpeed = 20000000,
- const TString& user = "",
- ui64 readSpeed = 20000000,
- const TVector<TString>& readRules = {},
- const TVector<TString>& important = {},
- std::optional<NKikimrPQ::TMirrorPartitionConfig> mirrorFrom = {},
- ui64 sourceIdMaxCount = 6000000,
- ui64 sourceIdLifetime = 86400
- )
- : Topic(topic)
- , NumParts(numParts)
- , CacheSize(cacheSize)
- , LifetimeS(lifetimeS)
- , LowWatermark(lowWatermark)
- , WriteSpeed(writeSpeed)
- , User(user)
- , ReadSpeed(readSpeed)
- , ReadRules(readRules)
- , Important(important)
- , MirrorFrom(mirrorFrom)
- , SourceIdMaxCount(sourceIdMaxCount)
- , SourceIdLifetime(sourceIdLifetime)
- {}
-
+ TRequestCreatePQ(
+ const TString& topic,
+ ui32 numParts,
+ ui32 cacheSize = 0,
+ ui64 lifetimeS = 86400,
+ ui32 lowWatermark = 8 * 1024 * 1024,
+ ui64 writeSpeed = 20000000,
+ const TString& user = "",
+ ui64 readSpeed = 20000000,
+ const TVector<TString>& readRules = {},
+ const TVector<TString>& important = {},
+ std::optional<NKikimrPQ::TMirrorPartitionConfig> mirrorFrom = {},
+ ui64 sourceIdMaxCount = 6000000,
+ ui64 sourceIdLifetime = 86400
+ )
+ : Topic(topic)
+ , NumParts(numParts)
+ , CacheSize(cacheSize)
+ , LifetimeS(lifetimeS)
+ , LowWatermark(lowWatermark)
+ , WriteSpeed(writeSpeed)
+ , User(user)
+ , ReadSpeed(readSpeed)
+ , ReadRules(readRules)
+ , Important(important)
+ , MirrorFrom(mirrorFrom)
+ , SourceIdMaxCount(sourceIdMaxCount)
+ , SourceIdLifetime(sourceIdLifetime)
+ {}
+
TString Topic;
ui32 NumParts;
ui32 CacheSize;
- ui64 LifetimeS;
- ui32 LowWatermark;
+ ui64 LifetimeS;
+ ui32 LowWatermark;
- ui64 WriteSpeed;
+ ui64 WriteSpeed;
TString User;
- ui64 ReadSpeed;
+ ui64 ReadSpeed;
TVector<TString> ReadRules;
TVector<TString> Important;
- std::optional<NKikimrPQ::TMirrorPartitionConfig> MirrorFrom;
-
+ std::optional<NKikimrPQ::TMirrorPartitionConfig> MirrorFrom;
+
ui64 SourceIdMaxCount;
ui64 SourceIdLifetime;
@@ -156,10 +156,10 @@ struct TRequestCreatePQ {
config->MutablePartitionConfig()->SetBurstSize(WriteSpeed);
for (auto& rr : ReadRules) {
config->AddReadRules(rr);
- config->AddReadFromTimestampsMs(0);
- config->AddConsumerFormatVersions(0);
- config->AddReadRuleVersions(0);
- config->AddConsumerCodecs()->AddIds(0);
+ config->AddReadFromTimestampsMs(0);
+ config->AddConsumerFormatVersions(0);
+ config->AddReadRuleVersions(0);
+ config->AddConsumerCodecs()->AddIds(0);
}
if (!ReadRules.empty()) {
config->SetRequireAuthRead(true);
@@ -171,38 +171,38 @@ struct TRequestCreatePQ {
rq->SetClientId(User);
}
- if (MirrorFrom) {
- auto mirrorFromConfig = config->MutablePartitionConfig()->MutableMirrorFrom();
- mirrorFromConfig->CopyFrom(MirrorFrom.value());
- }
+ if (MirrorFrom) {
+ auto mirrorFromConfig = config->MutablePartitionConfig()->MutableMirrorFrom();
+ mirrorFromConfig->CopyFrom(MirrorFrom.value());
+ }
return request;
}
};
struct TRequestAlterPQ {
- TRequestAlterPQ(
- const TString& topic,
- ui32 numParts,
- ui64 cacheSize = 0,
- ui64 lifetimeS = 86400,
- bool fillPartitionConfig = false,
- std::optional<NKikimrPQ::TMirrorPartitionConfig> mirrorFrom = {}
- )
- : Topic(topic)
- , NumParts(numParts)
- , CacheSize(cacheSize)
- , LifetimeS(lifetimeS)
- , FillPartitionConfig(fillPartitionConfig)
- , MirrorFrom(mirrorFrom)
- {}
-
+ TRequestAlterPQ(
+ const TString& topic,
+ ui32 numParts,
+ ui64 cacheSize = 0,
+ ui64 lifetimeS = 86400,
+ bool fillPartitionConfig = false,
+ std::optional<NKikimrPQ::TMirrorPartitionConfig> mirrorFrom = {}
+ )
+ : Topic(topic)
+ , NumParts(numParts)
+ , CacheSize(cacheSize)
+ , LifetimeS(lifetimeS)
+ , FillPartitionConfig(fillPartitionConfig)
+ , MirrorFrom(mirrorFrom)
+ {}
+
TString Topic;
ui32 NumParts;
ui64 CacheSize;
ui64 LifetimeS;
- bool FillPartitionConfig;
- std::optional<NKikimrPQ::TMirrorPartitionConfig> MirrorFrom;
+ bool FillPartitionConfig;
+ std::optional<NKikimrPQ::TMirrorPartitionConfig> MirrorFrom;
THolder<NMsgBusProxy::TBusPersQueue> GetRequest() {
THolder<NMsgBusProxy::TBusPersQueue> request(new NMsgBusProxy::TBusPersQueue);
@@ -213,21 +213,21 @@ struct TRequestAlterPQ {
auto config = req->MutableConfig();
config->SetCacheSize(CacheSize);
}
- if (FillPartitionConfig) {
- req->MutableConfig()->MutablePartitionConfig()->SetLifetimeSeconds(LifetimeS);
- if (MirrorFrom) {
- req->MutableConfig()->MutablePartitionConfig()->MutableMirrorFrom()->CopyFrom(MirrorFrom.value());
- }
- }
+ if (FillPartitionConfig) {
+ req->MutableConfig()->MutablePartitionConfig()->SetLifetimeSeconds(LifetimeS);
+ if (MirrorFrom) {
+ req->MutableConfig()->MutablePartitionConfig()->MutableMirrorFrom()->CopyFrom(MirrorFrom.value());
+ }
+ }
return request;
}
};
struct TRequestDeletePQ {
- TRequestDeletePQ(const TString& topic)
- : Topic(topic)
- {}
-
+ TRequestDeletePQ(const TString& topic)
+ : Topic(topic)
+ {}
+
TString Topic;
THolder<NMsgBusProxy::TBusPersQueue> GetRequest() {
@@ -239,11 +239,11 @@ struct TRequestDeletePQ {
};
struct TRequestGetOwnership {
- TRequestGetOwnership(const TString& topic, ui32 partition)
- : Topic(topic)
- , Partition(partition)
- {}
-
+ TRequestGetOwnership(const TString& topic, ui32 partition)
+ : Topic(topic)
+ , Partition(partition)
+ {}
+
TString Topic;
ui32 Partition;
@@ -259,13 +259,13 @@ struct TRequestGetOwnership {
struct TRequestWritePQ {
- TRequestWritePQ(const TString& topic, ui32 partition, const TString& sourceId, ui64 seqNo)
- : Topic(topic)
- , Partition(partition)
- , SourceId(sourceId)
- , SeqNo(seqNo)
- {}
-
+ TRequestWritePQ(const TString& topic, ui32 partition, const TString& sourceId, ui64 seqNo)
+ : Topic(topic)
+ , Partition(partition)
+ , SourceId(sourceId)
+ , SeqNo(seqNo)
+ {}
+
TString Topic;
ui32 Partition;
TString SourceId;
@@ -287,20 +287,20 @@ struct TRequestWritePQ {
};
struct TRequestReadPQ {
- TRequestReadPQ(
- const TString& topic,
- ui32 partition,
- ui64 startOffset,
- ui32 count,
- const TString& user
- )
- : Topic(topic)
- , Partition(partition)
- , StartOffset(startOffset)
- , Count(count)
- , User(user)
- {}
-
+ TRequestReadPQ(
+ const TString& topic,
+ ui32 partition,
+ ui64 startOffset,
+ ui32 count,
+ const TString& user
+ )
+ : Topic(topic)
+ , Partition(partition)
+ , StartOffset(startOffset)
+ , Count(count)
+ , User(user)
+ {}
+
TString Topic;
ui32 Partition;
ui64 StartOffset;
@@ -321,18 +321,18 @@ struct TRequestReadPQ {
};
struct TRequestSetClientOffsetPQ {
- TRequestSetClientOffsetPQ(
- const TString& topic,
- ui32 partition,
- ui64 offset,
- const TString& user
- )
- : Topic(topic)
- , Partition(partition)
- , Offset(offset)
- , User(user)
- {}
-
+ TRequestSetClientOffsetPQ(
+ const TString& topic,
+ ui32 partition,
+ ui64 offset,
+ const TString& user
+ )
+ : Topic(topic)
+ , Partition(partition)
+ , Offset(offset)
+ , User(user)
+ {}
+
TString Topic;
ui32 Partition;
ui64 Offset;
@@ -459,7 +459,7 @@ enum class ETransport {
struct TPQTestClusterInfo {
TString Balancer;
bool Enabled;
- ui64 Weight = 1000;
+ ui64 Weight = 1000;
};
static THashMap<TString, TPQTestClusterInfo> DEFAULT_CLUSTERS_LIST = {
@@ -615,7 +615,7 @@ public:
balancer Utf8,
local Bool,
enabled Bool,
- weight Uint64,
+ weight Uint64,
PRIMARY KEY (name)
);
CREATE TABLE [/Root/PQ/Config/V2/Topics] (
@@ -634,14 +634,14 @@ public:
)___");
TStringBuilder upsertClusters;
- upsertClusters << "UPSERT INTO [/Root/PQ/Config/V2/Cluster] (name, balancer, local, enabled, weight) VALUES ";
+ upsertClusters << "UPSERT INTO [/Root/PQ/Config/V2/Cluster] (name, balancer, local, enabled, weight) VALUES ";
bool first = true;
for (auto& [cluster, info] : clusters) {
bool isLocal = localCluster.empty() ? first : localCluster == cluster;
if (!first)
upsertClusters << ", ";
upsertClusters << "(\"" << cluster << "\", \"" << info.Balancer << "\", " << (isLocal ? "true" : "false")
- << ", " << (info.Enabled ? "true" : "false") << ", " << info.Weight << ")";
+ << ", " << (info.Enabled ? "true" : "false") << ", " << info.Weight << ")";
first = false;
}
upsertClusters << ";\n";
@@ -661,21 +661,21 @@ public:
RunYqlDataQuery(query);
}
- TPQTestClusterInfo GetDcInfo(const TString& name) {
- TStringBuilder query;
- query << "SELECT balancer, enabled, weight FROM [/Root/PQ/Config/V2/Cluster] where name = \"" << name << "\";";
- auto result = RunYqlDataQuery(query);
- NYdb::TResultSetParser parser(*result);
- UNIT_ASSERT_VALUES_EQUAL(parser.RowsCount(), 1);
- parser.TryNextRow();
-
- TPQTestClusterInfo info;
- info.Balancer = *parser.ColumnParser("balancer").GetOptionalUtf8();
- info.Enabled = *parser.ColumnParser("enabled").GetOptionalBool();
- info.Weight = *parser.ColumnParser("weight").GetOptionalUint64();
- return info;
- }
-
+ TPQTestClusterInfo GetDcInfo(const TString& name) {
+ TStringBuilder query;
+ query << "SELECT balancer, enabled, weight FROM [/Root/PQ/Config/V2/Cluster] where name = \"" << name << "\";";
+ auto result = RunYqlDataQuery(query);
+ NYdb::TResultSetParser parser(*result);
+ UNIT_ASSERT_VALUES_EQUAL(parser.RowsCount(), 1);
+ parser.TryNextRow();
+
+ TPQTestClusterInfo info;
+ info.Balancer = *parser.ColumnParser("balancer").GetOptionalUtf8();
+ info.Enabled = *parser.ColumnParser("enabled").GetOptionalBool();
+ info.Weight = *parser.ColumnParser("weight").GetOptionalUint64();
+ return info;
+ }
+
void InitUserRegistry() {
MkDir("/Root/PQ", "Config");
MkDir("/Root/PQ/Config", "V2");
@@ -915,27 +915,27 @@ public:
}
}
- void CreateTopic(
- const TString& name,
- ui32 nParts,
- ui32 lowWatermark = 8*1024*1024,
- ui64 lifetimeS = 86400,
- ui64 writeSpeed = 20000000,
- TString user = "",
- ui64 readSpeed = 200000000,
- TVector<TString> rr = {},
- TVector<TString> important = {},
+ void CreateTopic(
+ const TString& name,
+ ui32 nParts,
+ ui32 lowWatermark = 8*1024*1024,
+ ui64 lifetimeS = 86400,
+ ui64 writeSpeed = 20000000,
+ TString user = "",
+ ui64 readSpeed = 200000000,
+ TVector<TString> rr = {},
+ TVector<TString> important = {},
std::optional<NKikimrPQ::TMirrorPartitionConfig> mirrorFrom = {},
ui64 sourceIdMaxCount = 6000000,
ui64 sourceIdLifetime = 86400
- ) {
+ ) {
Y_VERIFY(name.StartsWith("rt3."));
Cerr << "PQ Client: create topic: " << name << " with " << nParts << " partitions" << Endl;
- auto request = TRequestCreatePQ(
+ auto request = TRequestCreatePQ(
name, nParts, 0, lifetimeS, lowWatermark, writeSpeed, user, readSpeed, rr, important, mirrorFrom,
sourceIdMaxCount, sourceIdLifetime
- );
+ );
return CreateTopic(request);
}
@@ -955,17 +955,17 @@ public:
Cerr << "Alter topic (" << path << ") response: " << res.GetValue().IsSuccess() << " " << res.GetValue().GetIssues().ToString() << Endl;
}
- void AlterTopic(
- const TString& name,
- ui32 nParts,
- ui32 cacheSize = 0,
- ui64 lifetimeS = 86400,
- bool fillPartitionConfig = false,
- std::optional<NKikimrPQ::TMirrorPartitionConfig> mirrorFrom = {}
- ) {
+ void AlterTopic(
+ const TString& name,
+ ui32 nParts,
+ ui32 cacheSize = 0,
+ ui64 lifetimeS = 86400,
+ bool fillPartitionConfig = false,
+ std::optional<NKikimrPQ::TMirrorPartitionConfig> mirrorFrom = {}
+ ) {
Y_VERIFY(name.StartsWith("rt3."));
- TRequestAlterPQ requestDescr(name, nParts, cacheSize, lifetimeS, fillPartitionConfig, mirrorFrom);
- THolder<NMsgBusProxy::TBusPersQueue> request = requestDescr.GetRequest();
+ TRequestAlterPQ requestDescr(name, nParts, cacheSize, lifetimeS, fillPartitionConfig, mirrorFrom);
+ THolder<NMsgBusProxy::TBusPersQueue> request = requestDescr.GetRequest();
ui32 prevVersion = TopicCreated(name);
diff --git a/ydb/core/tx/scheme_board/cache.cpp b/ydb/core/tx/scheme_board/cache.cpp
index 75bc4425e4..8e778a4a1d 100644
--- a/ydb/core/tx/scheme_board/cache.cpp
+++ b/ydb/core/tx/scheme_board/cache.cpp
@@ -190,7 +190,7 @@ namespace {
break;
}
- entry.Self.Drop();
+ entry.Self.Drop();
entry.SecurityObject.Drop();
entry.DomainInfo.Drop();
entry.Kind = TNavigate::KindUnknown;
@@ -697,8 +697,8 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> {
CdcStreams.clear();
Partitioning.clear();
- Self.Drop();
-
+ Self.Drop();
+
DomainDescription.Drop();
RtmrVolumeInfo.Drop();
KesusInfo.Drop();
@@ -1317,8 +1317,8 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> {
Y_VERIFY(pathDesc.HasSelf());
const auto& entryDesc = pathDesc.GetSelf();
- Self = new TNavigate::TDirEntryInfo();
- Self->Info.CopyFrom(entryDesc);
+ Self = new TNavigate::TDirEntryInfo();
+ Self->Info.CopyFrom(entryDesc);
Created = entryDesc.HasCreateFinished() && entryDesc.GetCreateFinished();
CreateStep = entryDesc.GetCreateStep();
@@ -1665,7 +1665,7 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> {
}
// specific (it's safe to fill them all)
- entry.Self = Self;
+ entry.Self = Self;
entry.Columns = Columns;
entry.NotNullColumns = NotNullColumns;
entry.Indexes = Indexes;
@@ -1916,8 +1916,8 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> {
TVector<NKikimrSchemeOp::TCdcStreamDescription> CdcStreams;
TVector<TKeyDesc::TPartitionInfo> Partitioning;
- TIntrusivePtr<TNavigate::TDirEntryInfo> Self;
-
+ TIntrusivePtr<TNavigate::TDirEntryInfo> Self;
+
// RTMR specific
TIntrusivePtr<TNavigate::TRtmrVolumeInfo> RtmrVolumeInfo;
diff --git a/ydb/core/tx/scheme_cache/scheme_cache.h b/ydb/core/tx/scheme_cache/scheme_cache.h
index ff1759b08a..26a07897a1 100644
--- a/ydb/core/tx/scheme_cache/scheme_cache.h
+++ b/ydb/core/tx/scheme_cache/scheme_cache.h
@@ -145,10 +145,10 @@ struct TSchemeCacheNavigate {
TVector<TChild> Children;
};
- struct TDirEntryInfo : public TAtomicRefCount<TDirEntryInfo>{
+ struct TDirEntryInfo : public TAtomicRefCount<TDirEntryInfo>{
NKikimrSchemeOp::TDirEntry Info;
- };
-
+ };
+
struct TDomainDescription : public TAtomicRefCount<TDomainDescription> {
EKind Kind = KindUnknown;
NKikimrSubDomains::TDomainDescription Description;
diff --git a/ydb/core/viewer/json_pqconsumerinfo.h b/ydb/core/viewer/json_pqconsumerinfo.h
index c4d55aad7b..b98dfc2632 100644
--- a/ydb/core/viewer/json_pqconsumerinfo.h
+++ b/ydb/core/viewer/json_pqconsumerinfo.h
@@ -61,14 +61,14 @@ public:
NKikimrClient::TPersQueueRequest request;
request.MutableMetaRequest()->MutableCmdGetPartitionStatus()->SetClientId(Client);
request.MutableMetaRequest()->MutableCmdGetPartitionStatus()->AddTopicRequest()->SetTopic(Topic);
- ctx.Register(NMsgBusProxy::CreateActorServerPersQueue(ctx.SelfID, request, NMsgBusProxy::CreatePersQueueMetaCacheV2Id(), nullptr));
+ ctx.Register(NMsgBusProxy::CreateActorServerPersQueue(ctx.SelfID, request, NMsgBusProxy::CreatePersQueueMetaCacheV2Id(), nullptr));
++Requests;
}
{
NKikimrClient::TPersQueueRequest request;
request.MutableMetaRequest()->MutableCmdGetReadSessionsInfo()->SetClientId(Client);
request.MutableMetaRequest()->MutableCmdGetReadSessionsInfo()->AddTopic(Topic);
- ctx.Register(NMsgBusProxy::CreateActorServerPersQueue(ctx.SelfID, request, NMsgBusProxy::CreatePersQueueMetaCacheV2Id(), nullptr));
+ ctx.Register(NMsgBusProxy::CreateActorServerPersQueue(ctx.SelfID, request, NMsgBusProxy::CreatePersQueueMetaCacheV2Id(), nullptr));
++Requests;
}
Become(&TThis::StateRequestedTopicInfo, ctx, TDuration::MilliSeconds(Timeout), new TEvents::TEvWakeup());
diff --git a/ydb/core/ymq/actor/metering.cpp b/ydb/core/ymq/actor/metering.cpp
index 144325410e..94ccef7eb0 100644
--- a/ydb/core/ymq/actor/metering.cpp
+++ b/ydb/core/ymq/actor/metering.cpp
@@ -5,7 +5,7 @@
#include "proxy_actor.h"
#include <ydb/core/mind/address_classification/net_classifier.h>
-
+
#include <ydb/core/ymq/base/action.h>
#include <ydb/core/ymq/base/counters.h>
#include <ydb/core/ymq/base/processed_request_attributes.h>
@@ -14,7 +14,7 @@
#include <library/cpp/logger/record.h>
#include <util/generic/guid.h>
-#include <util/generic/serialized_enum.h>
+#include <util/generic/serialized_enum.h>
#include <util/string/builder.h>
#include <util/system/hostname.h>
@@ -24,28 +24,28 @@ namespace NKikimr::NSQS {
using namespace NAddressClassifier;
-const THashMap<TString, TString> TProcessedRequestsAggregator::LabelTransformation = {
- {"yacloud", ToString(ENetworkClass::cloud)}
-};
-
-TProcessedRequestsAggregator::TProcessedRequestsAggregator(const NKikimrConfig::TSqsConfig& config)
- : NetClassifierOnly(config.GetMeteringByNetClassifierOnly())
-{
- auto enumValues = GetEnumNames<ENetworkClass>();
- TVector<TString> labels(enumValues.size());
- for (auto enumItem : enumValues) {
- Y_VERIFY(enumItem.first < labels.size());
- labels[enumItem.first] = enumItem.second;
- }
- Counters = new TMeteringCounters(config, GetSqsServiceCounters(AppData()->Counters, "metering"), labels);
- if (!NetClassifierOnly) {
- InitAddressClassifier(config, std::move(labels));
- }
-}
+const THashMap<TString, TString> TProcessedRequestsAggregator::LabelTransformation = {
+ {"yacloud", ToString(ENetworkClass::cloud)}
+};
+
+TProcessedRequestsAggregator::TProcessedRequestsAggregator(const NKikimrConfig::TSqsConfig& config)
+ : NetClassifierOnly(config.GetMeteringByNetClassifierOnly())
+{
+ auto enumValues = GetEnumNames<ENetworkClass>();
+ TVector<TString> labels(enumValues.size());
+ for (auto enumItem : enumValues) {
+ Y_VERIFY(enumItem.first < labels.size());
+ labels[enumItem.first] = enumItem.second;
+ }
+ Counters = new TMeteringCounters(config, GetSqsServiceCounters(AppData()->Counters, "metering"), labels);
+ if (!NetClassifierOnly) {
+ InitAddressClassifier(config, std::move(labels));
+ }
+}
bool TProcessedRequestsAggregator::TReportedTrafficKey::operator<(const TReportedTrafficKey& rhs) const {
- return std::tie(ResourceId, TrafficType, NetworkClassLabel) <
- std::tie(rhs.ResourceId, rhs.TrafficType, rhs.NetworkClassLabel);
+ return std::tie(ResourceId, TrafficType, NetworkClassLabel) <
+ std::tie(rhs.ResourceId, rhs.TrafficType, rhs.NetworkClassLabel);
}
bool TProcessedRequestsAggregator::TReportedRequestsKey::operator<(const TReportedRequestsKey& rhs) const {
@@ -53,33 +53,33 @@ bool TProcessedRequestsAggregator::TReportedRequestsKey::operator<(const TReport
std::tie(rhs.ResourceId, rhs.QueueType);
}
-TString TProcessedRequestsAggregator::ClassifyNetwork(NAddressClassifier::TLabeledAddressClassifier::TConstPtr classifier, const TString& address) {
- if (classifier) {
- auto result = classifier->ClassifyAddress(address);
- if (!result) {
- return ToString(ENetworkClass::inet);
- }
- auto it = LabelTransformation.find(result.GetRef());
- if (it != LabelTransformation.end()) {
- return it->second;
- }
- return result.GetRef();
- }
- return ToString(ENetworkClass::unknown);
-}
-
-TString TProcessedRequestsAggregator::ClassifyNetwork(const TString& address) const {
- TString netClassifierResult = ClassifyNetwork(NetClassifier, address);
-
- TString result;
- if (NetClassifierOnly) {
- result = std::move(netClassifierResult);
- } else {
- result = ClassifyNetwork(AddressClassifier, address);
- INC_COUNTER(Counters, IdleClassifierRequestsResults[netClassifierResult]);
+TString TProcessedRequestsAggregator::ClassifyNetwork(NAddressClassifier::TLabeledAddressClassifier::TConstPtr classifier, const TString& address) {
+ if (classifier) {
+ auto result = classifier->ClassifyAddress(address);
+ if (!result) {
+ return ToString(ENetworkClass::inet);
+ }
+ auto it = LabelTransformation.find(result.GetRef());
+ if (it != LabelTransformation.end()) {
+ return it->second;
+ }
+ return result.GetRef();
}
- INC_COUNTER(Counters, ClassifierRequestsResults[result]);
- return result;
+ return ToString(ENetworkClass::unknown);
+}
+
+TString TProcessedRequestsAggregator::ClassifyNetwork(const TString& address) const {
+ TString netClassifierResult = ClassifyNetwork(NetClassifier, address);
+
+ TString result;
+ if (NetClassifierOnly) {
+ result = std::move(netClassifierResult);
+ } else {
+ result = ClassifyNetwork(AddressClassifier, address);
+ INC_COUNTER(Counters, IdleClassifierRequestsResults[netClassifierResult]);
+ }
+ INC_COUNTER(Counters, ClassifierRequestsResults[result]);
+ return result;
}
ui64 TProcessedRequestsAggregator::CountBlocks(const ui64 bytes, const ui64 blockSize) const {
@@ -97,13 +97,13 @@ bool TProcessedRequestsAggregator::Add(const TProcessedRequestAttributes& attrs)
queueType = EQueueType::other;
resourceId = "";
}
- const auto networkClassLabel = ClassifyNetwork(attrs.SourceAddress);
+ const auto networkClassLabel = ClassifyNetwork(attrs.SourceAddress);
Y_VERIFY(attrs.RequestSizeInBytes);
Y_VERIFY(attrs.ResponseSizeInBytes);
- ReportedTraffic[attrs.FolderId][{resourceId, ETrafficType::ingress, networkClassLabel}] += attrs.RequestSizeInBytes;
- ReportedTraffic[attrs.FolderId][{resourceId, ETrafficType::egress, networkClassLabel}] += attrs.ResponseSizeInBytes;
+ ReportedTraffic[attrs.FolderId][{resourceId, ETrafficType::ingress, networkClassLabel}] += attrs.RequestSizeInBytes;
+ ReportedTraffic[attrs.FolderId][{resourceId, ETrafficType::egress, networkClassLabel}] += attrs.ResponseSizeInBytes;
static const ui64 defaultRequestBlockSize = 64 * 1024; // SQS-22
ReportedRequests[attrs.FolderId][{resourceId, queueType}] += CountBlocks(attrs.RequestSizeInBytes + attrs.ResponseSizeInBytes,
@@ -112,16 +112,16 @@ bool TProcessedRequestsAggregator::Add(const TProcessedRequestAttributes& attrs)
return true;
}
-NSc::TValue CreateMeteringBillingRecord(
- const TString& folderId,
- const TString& resourceId,
- const TString& schema,
- const TString& fqdn,
- const TInstant& now,
- const ui64 quantity,
- const TString& unit,
- const NSc::TValue& tags
-) {
+NSc::TValue CreateMeteringBillingRecord(
+ const TString& folderId,
+ const TString& resourceId,
+ const TString& schema,
+ const TString& fqdn,
+ const TInstant& now,
+ const ui64 quantity,
+ const TString& unit,
+ const NSc::TValue& tags
+) {
const TString& billingRecordVersion = "v1";
const ui64 utcSeconds = now.Seconds();
@@ -157,10 +157,10 @@ TVector<NSc::TValue> TProcessedRequestsAggregator::DumpReportedTrafficAsJsonArra
for (const auto& [trafficKey, trafficValue] : perFolderData.second) {
NSc::TValue tags;
tags.SetDict();
- tags["type"] = trafficKey.NetworkClassLabel;
+ tags["type"] = trafficKey.NetworkClassLabel;
tags["direction"] = ToString(trafficKey.TrafficType);
- result.push_back(CreateMeteringBillingRecord(folderId,
+ result.push_back(CreateMeteringBillingRecord(folderId,
trafficKey.ResourceId,
"ymq.traffic.v1",
fqdn,
@@ -184,7 +184,7 @@ TVector<NSc::TValue> TProcessedRequestsAggregator::DumpReportedRequestsAsJsonArr
tags.SetDict();
tags["queue_type"] = ToString(requestsKey.QueueType);
- result.push_back(CreateMeteringBillingRecord(folderId,
+ result.push_back(CreateMeteringBillingRecord(folderId,
requestsKey.ResourceId,
"ymq.requests.v1",
fqdn,
@@ -203,23 +203,23 @@ void TProcessedRequestsAggregator::ResetReportsStorage() {
ReportedRequests.clear();
}
-void TProcessedRequestsAggregator::UpdateNetClassifier(NAddressClassifier::TLabeledAddressClassifier::TConstPtr classifier, TMaybe<TInstant> /*netDataUpdateTimestamp*/) {
- NetClassifier = classifier;
-}
-
-void TProcessedRequestsAggregator::InitAddressClassifier(const NKikimrConfig::TSqsConfig& config, TVector<TString>&& labels) {
- TAddressClassifier addressClassifier;
- for (size_t i = 0; i < config.MeteringCloudNetCidrSize(); ++i) {
- addressClassifier.AddNetByCidrAndLabel(config.GetMeteringCloudNetCidr(i) , ENetworkClass::cloud);
- }
-
- for (size_t i = 0; i < config.MeteringYandexNetCidrSize(); ++i) {
- addressClassifier.AddNetByCidrAndLabel(config.GetMeteringYandexNetCidr(i) , ENetworkClass::yandex);
- }
-
- AddressClassifier = TLabeledAddressClassifier::MakeLabeledAddressClassifier(std::move(addressClassifier), std::move(labels));
-}
-
+void TProcessedRequestsAggregator::UpdateNetClassifier(NAddressClassifier::TLabeledAddressClassifier::TConstPtr classifier, TMaybe<TInstant> /*netDataUpdateTimestamp*/) {
+ NetClassifier = classifier;
+}
+
+void TProcessedRequestsAggregator::InitAddressClassifier(const NKikimrConfig::TSqsConfig& config, TVector<TString>&& labels) {
+ TAddressClassifier addressClassifier;
+ for (size_t i = 0; i < config.MeteringCloudNetCidrSize(); ++i) {
+ addressClassifier.AddNetByCidrAndLabel(config.GetMeteringCloudNetCidr(i) , ENetworkClass::cloud);
+ }
+
+ for (size_t i = 0; i < config.MeteringYandexNetCidrSize(); ++i) {
+ addressClassifier.AddNetByCidrAndLabel(config.GetMeteringYandexNetCidr(i) , ENetworkClass::yandex);
+ }
+
+ AddressClassifier = TLabeledAddressClassifier::MakeLabeledAddressClassifier(std::move(addressClassifier), std::move(labels));
+}
+
class TMeteringActor
: public TActorBootstrapped<TMeteringActor>
{
@@ -229,11 +229,11 @@ public:
{
}
- void Bootstrap(const TActorContext& ctx) {
+ void Bootstrap(const TActorContext& ctx) {
Become(&TThis::Work);
- ctx.Send(NNetClassifier::MakeNetClassifierID(), new NNetClassifier::TEvNetClassifier::TEvSubscribe);
-
+ ctx.Send(NNetClassifier::MakeNetClassifierID(), new NNetClassifier::TEvNetClassifier::TEvSubscribe);
+
Aggregator = MakeHolder<TProcessedRequestsAggregator>(Cfg());
FlushProcessedRequestsAttributes();
@@ -286,17 +286,17 @@ public:
STATEFN(Work) {
switch (ev->GetTypeRewrite()) {
hFunc(TEvWakeup, HandleWakeup);
- hFunc(NNetClassifier::TEvNetClassifier::TEvClassifierUpdate, HandleNetClassifierUpdate);
+ hFunc(NNetClassifier::TEvNetClassifier::TEvClassifierUpdate, HandleNetClassifierUpdate);
hFunc(TSqsEvents::TEvReportProcessedRequestAttributes, HandleReportProcessedRequestAttributes);
}
}
-
+
+private:
+ void HandleNetClassifierUpdate(NNetClassifier::TEvNetClassifier::TEvClassifierUpdate::TPtr& ev) {
+ Aggregator->UpdateNetClassifier(ev->Get()->Classifier, ev->Get()->NetDataUpdateTimestamp);
+ }
+
private:
- void HandleNetClassifierUpdate(NNetClassifier::TEvNetClassifier::TEvClassifierUpdate::TPtr& ev) {
- Aggregator->UpdateNetClassifier(ev->Get()->Classifier, ev->Get()->NetDataUpdateTimestamp);
- }
-
-private:
const TString HostFQDN;
THolder<TProcessedRequestsAggregator> Aggregator;
};
diff --git a/ydb/core/ymq/actor/metering.h b/ydb/core/ymq/actor/metering.h
index 3e4fb93c03..c9d5a58991 100644
--- a/ydb/core/ymq/actor/metering.h
+++ b/ydb/core/ymq/actor/metering.h
@@ -15,23 +15,23 @@
namespace NKikimr::NSQS {
-struct TMeteringCounters;
-
-NSc::TValue CreateMeteringBillingRecord(
- const TString& folderId,
- const TString& resourceId,
- const TString& schema,
- const TString& fqdn,
- const TInstant& now,
- const ui64 quantity,
- const TString& unit,
- const NSc::TValue& tags
-);
-
+struct TMeteringCounters;
+
+NSc::TValue CreateMeteringBillingRecord(
+ const TString& folderId,
+ const TString& resourceId,
+ const TString& schema,
+ const TString& fqdn,
+ const TInstant& now,
+ const ui64 quantity,
+ const TString& unit,
+ const NSc::TValue& tags
+);
+
class TProcessedRequestsAggregator {
public:
- static const THashMap<TString, TString> LabelTransformation;
-public:
+ static const THashMap<TString, TString> LabelTransformation;
+public:
TProcessedRequestsAggregator(const NKikimrConfig::TSqsConfig& config);
// Do not rename enum members in the enums below (SQS-22)
@@ -43,8 +43,8 @@ public:
enum ENetworkClass {
inet = 0,
cloud,
- yandex,
- unknown
+ yandex,
+ unknown
};
enum EQueueType {
@@ -56,7 +56,7 @@ public:
struct TReportedTrafficKey {
TString ResourceId;
ETrafficType TrafficType;
- TString NetworkClassLabel;
+ TString NetworkClassLabel;
bool operator<(const TReportedTrafficKey& rhs) const;
};
@@ -68,7 +68,7 @@ public:
bool operator<(const TReportedRequestsKey& rhs) const;
};
- TString ClassifyNetwork(const TString& address) const;
+ TString ClassifyNetwork(const TString& address) const;
ui64 CountBlocks(const ui64 bytes, const ui64 blockSize) const;
@@ -79,22 +79,22 @@ public:
void ResetReportsStorage();
- void UpdateNetClassifier(NAddressClassifier::TLabeledAddressClassifier::TConstPtr classifier, TMaybe<TInstant> netDataUpdateTimestamp);
-
+ void UpdateNetClassifier(NAddressClassifier::TLabeledAddressClassifier::TConstPtr classifier, TMaybe<TInstant> netDataUpdateTimestamp);
+
+private:
+ static TString ClassifyNetwork(NAddressClassifier::TLabeledAddressClassifier::TConstPtr classifier, const TString& address);
+
+ void InitAddressClassifier(const NKikimrConfig::TSqsConfig& config, TVector<TString>&& labels);
+
private:
- static TString ClassifyNetwork(NAddressClassifier::TLabeledAddressClassifier::TConstPtr classifier, const TString& address);
-
- void InitAddressClassifier(const NKikimrConfig::TSqsConfig& config, TVector<TString>&& labels);
-
-private:
- bool NetClassifierOnly;
-
+ bool NetClassifierOnly;
+
THashMap<TString, TMap<TReportedTrafficKey, ui64>> ReportedTraffic;
THashMap<TString, TMap<TReportedRequestsKey, ui64>> ReportedRequests;
- NAddressClassifier::TLabeledAddressClassifier::TConstPtr AddressClassifier;
- NAddressClassifier::TLabeledAddressClassifier::TConstPtr NetClassifier;
-
- TIntrusivePtr<TMeteringCounters> Counters;
+ NAddressClassifier::TLabeledAddressClassifier::TConstPtr AddressClassifier;
+ NAddressClassifier::TLabeledAddressClassifier::TConstPtr NetClassifier;
+
+ TIntrusivePtr<TMeteringCounters> Counters;
};
} // namespace NKikimr::NSQS
diff --git a/ydb/core/ymq/actor/queue_schema.cpp b/ydb/core/ymq/actor/queue_schema.cpp
index 629af7ddea..91d3c5d3df 100644
--- a/ydb/core/ymq/actor/queue_schema.cpp
+++ b/ydb/core/ymq/actor/queue_schema.cpp
@@ -9,22 +9,22 @@
#include <ydb/core/ymq/queues/fifo/schema.h>
#include <ydb/core/ymq/queues/std/schema.h>
-#include <util/digest/city.h>
+#include <util/digest/city.h>
#include <util/generic/utility.h>
-#include <util/string/join.h>
+#include <util/string/join.h>
using NKikimr::NClient::TValue;
namespace NKikimr::NSQS {
-constexpr TStringBuf FIFO_TABLES_DIR = ".FIFO";
-constexpr TStringBuf STD_TABLES_DIR = ".STD";
-
-template <class... TArgs>
-ui64 GetHash(TArgs&&... args) {
- return CityHash64(Join("#", args...));
-}
-
+constexpr TStringBuf FIFO_TABLES_DIR = ".FIFO";
+constexpr TStringBuf STD_TABLES_DIR = ".STD";
+
+template <class... TArgs>
+ui64 GetHash(TArgs&&... args) {
+ return CityHash64(Join("#", args...));
+}
+
static bool IsGoodStatusCode(ui32 code) {
switch (NTxProxy::TResultStatus::EStatus(code)) {
case NTxProxy::TResultStatus::EStatus::ExecComplete:
@@ -411,40 +411,40 @@ void TCreateQueueSchemaActorV2::OnAtomicCounterIncrement(TSqsEvents::TEvAtomicCo
PassAway();
}
-static const char* const GetTablesFormatQuery = R"__(
- (
- (let defaultTablesFormat (Parameter 'DEFAULT_TABLES_FORMAT (DataType 'Uint32)))
- (let userName (Parameter 'USER_NAME (DataType 'Utf8String)))
- (let settingsTable '%1$s/.Settings)
- (let tablesFormatSettingRow '(
- '('Account userName)
- '('Name (Utf8String '"CreateQueuesWithTabletFormat"))))
- (let tablesFormatSettingSelect '('Value))
- (let tablesFormatSettingRead (SelectRow settingsTable tablesFormatSettingRow tablesFormatSettingSelect))
- (let tablesFormatSetting
- (If (Exists tablesFormatSettingRead)
- (Cast (Member tablesFormatSettingRead 'Value) 'Uint32)
- defaultTablesFormat
- )
- )
-
- (return (AsList
- (SetResult 'tablesFormat tablesFormatSetting)
- )
- )
- )
-)__";
-
-void TCreateQueueSchemaActorV2::RequestTablesFormatSettings(const TString& accountName) {
- auto ev = MakeExecuteEvent(Sprintf(GetTablesFormatQuery, Cfg().GetRoot().c_str()));
- auto* trans = ev->Record.MutableTransaction()->MutableMiniKQLTransaction();
- TParameters(trans->MutableParams()->MutableProto())
- .Utf8("USER_NAME", accountName)
- .Uint32("DEFAULT_TABLES_FORMAT", 0);
-
- Register(new TMiniKqlExecutionActor(SelfId(), RequestId_, std::move(ev), false, QueuePath_, GetTransactionCounters(UserCounters_)));
-}
-
+static const char* const GetTablesFormatQuery = R"__(
+ (
+ (let defaultTablesFormat (Parameter 'DEFAULT_TABLES_FORMAT (DataType 'Uint32)))
+ (let userName (Parameter 'USER_NAME (DataType 'Utf8String)))
+ (let settingsTable '%1$s/.Settings)
+ (let tablesFormatSettingRow '(
+ '('Account userName)
+ '('Name (Utf8String '"CreateQueuesWithTabletFormat"))))
+ (let tablesFormatSettingSelect '('Value))
+ (let tablesFormatSettingRead (SelectRow settingsTable tablesFormatSettingRow tablesFormatSettingSelect))
+ (let tablesFormatSetting
+ (If (Exists tablesFormatSettingRead)
+ (Cast (Member tablesFormatSettingRead 'Value) 'Uint32)
+ defaultTablesFormat
+ )
+ )
+
+ (return (AsList
+ (SetResult 'tablesFormat tablesFormatSetting)
+ )
+ )
+ )
+)__";
+
+void TCreateQueueSchemaActorV2::RequestTablesFormatSettings(const TString& accountName) {
+ auto ev = MakeExecuteEvent(Sprintf(GetTablesFormatQuery, Cfg().GetRoot().c_str()));
+ auto* trans = ev->Record.MutableTransaction()->MutableMiniKQLTransaction();
+ TParameters(trans->MutableParams()->MutableProto())
+ .Utf8("USER_NAME", accountName)
+ .Uint32("DEFAULT_TABLES_FORMAT", 0);
+
+ Register(new TMiniKqlExecutionActor(SelfId(), RequestId_, std::move(ev), false, QueuePath_, GetTransactionCounters(UserCounters_)));
+}
+
void TCreateQueueSchemaActorV2::RegisterMakeDirActor(const TString& workingDir, const TString& dirName) {
auto ev = MakeHolder<TEvTxUserProxy::TEvProposeTransaction>();
auto* trans = ev->Record.MutableTransaction()->MutableModifyScheme();
@@ -466,10 +466,10 @@ void TCreateQueueSchemaActorV2::RequestLeaderTabletId() {
void TCreateQueueSchemaActorV2::CreateComponents() {
switch (CurrentCreationStep_) {
- case ECreateComponentsStep::GetTablesFormatSetting: {
- RequestTablesFormatSettings(QueuePath_.UserName);
- break;
- }
+ case ECreateComponentsStep::GetTablesFormatSetting: {
+ RequestTablesFormatSettings(QueuePath_.UserName);
+ break;
+ }
case ECreateComponentsStep::MakeQueueDir: {
RegisterMakeDirActor(QueuePath_.GetUserPath(), QueuePath_.QueueName);
break;
@@ -500,10 +500,10 @@ void TCreateQueueSchemaActorV2::CreateComponents() {
break;
}
- case ECreateComponentsStep::DescribeTableForSetSchemeShardId: {
- SendDescribeTable();
- break;
- }
+ case ECreateComponentsStep::DescribeTableForSetSchemeShardId: {
+ SendDescribeTable();
+ break;
+ }
case ECreateComponentsStep::DiscoverLeaderTabletId: {
RequestLeaderTabletId();
break;
@@ -519,7 +519,7 @@ STATEFN(TCreateQueueSchemaActorV2::CreateComponentsState) {
switch (ev->GetTypeRewrite()) {
hFunc(TSqsEvents::TEvExecuted, OnExecuted);
hFunc(NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult, OnDescribeSchemeResult);
- hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, HandleTableDescription);
+ hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, HandleTableDescription);
hFunc(NKesus::TEvKesus::TEvAddQuoterResourceResult, HandleAddQuoterResource);
cFunc(TEvPoisonPill::EventType, PassAway);
}
@@ -528,16 +528,16 @@ STATEFN(TCreateQueueSchemaActorV2::CreateComponentsState) {
void TCreateQueueSchemaActorV2::Step() {
RLOG_SQS_TRACE("Next step. Step: " << (int)CurrentCreationStep_);
switch (CurrentCreationStep_) {
- case ECreateComponentsStep::GetTablesFormatSetting: {
- CurrentCreationStep_ = ECreateComponentsStep::MakeQueueDir;
- break;
- }
+ case ECreateComponentsStep::GetTablesFormatSetting: {
+ CurrentCreationStep_ = ECreateComponentsStep::MakeQueueDir;
+ break;
+ }
case ECreateComponentsStep::MakeQueueDir: {
- if (TablesFormat_ == 0) {
- CurrentCreationStep_ = ECreateComponentsStep::MakeQueueVersionDir;
- } else {
- CurrentCreationStep_ = ECreateComponentsStep::DescribeTableForSetSchemeShardId;
- }
+ if (TablesFormat_ == 0) {
+ CurrentCreationStep_ = ECreateComponentsStep::MakeQueueVersionDir;
+ } else {
+ CurrentCreationStep_ = ECreateComponentsStep::DescribeTableForSetSchemeShardId;
+ }
break;
}
case ECreateComponentsStep::MakeQueueVersionDir: {
@@ -566,12 +566,12 @@ void TCreateQueueSchemaActorV2::Step() {
CurrentCreationStep_ = ECreateComponentsStep::DiscoverLeaderTabletId;
break;
}
- case ECreateComponentsStep::DescribeTableForSetSchemeShardId: {
- Y_VERIFY(TablesFormat_ == 1);
- Y_VERIFY(TableWithLeaderPathId_.first && TableWithLeaderPathId_.second);
- CurrentCreationStep_ = ECreateComponentsStep::DiscoverLeaderTabletId;
- break;
- }
+ case ECreateComponentsStep::DescribeTableForSetSchemeShardId: {
+ Y_VERIFY(TablesFormat_ == 1);
+ Y_VERIFY(TableWithLeaderPathId_.first && TableWithLeaderPathId_.second);
+ CurrentCreationStep_ = ECreateComponentsStep::DiscoverLeaderTabletId;
+ break;
+ }
case ECreateComponentsStep::DiscoverLeaderTabletId: {
Y_VERIFY(Cfg().GetQuotingConfig().GetEnableQuoting() && Cfg().GetQuotingConfig().HasKesusQuoterConfig());
CurrentCreationStep_ = ECreateComponentsStep::AddQuoterResource;
@@ -601,28 +601,28 @@ void TCreateQueueSchemaActorV2::OnExecuted(TSqsEvents::TEvExecuted::TPtr& ev) {
// SS finishes transaction immediately if the specified path already exists
// DO NOT add any special logic based on the result type (except for an error)
if (IsGoodStatusCode(status)) {
- if (CurrentCreationStep_ == ECreateComponentsStep::GetTablesFormatSetting) {
- const TValue value(TValue::Create(record.GetExecutionEngineEvaluatedResponse()));
- const TValue formatValue = value["tablesFormat"];
- if (formatValue.HaveValue()) {
- TablesFormat_ = static_cast<ui32>(formatValue);
- }
- if (!formatValue.HaveValue() || TablesFormat_ > 1) {
- RLOG_SQS_WARN("Incorrect TablesFormat settings for account "
- << QueuePath_.UserName << ", responce:" << record);
-
- auto resp = MakeErrorResponse(NErrors::INTERNAL_FAILURE);
- resp->State = EQueueState::Creating;
- resp->Error = "Incorrect TablesFormat settings for account";
-
- Send(Sender_, std::move(resp));
- PassAway();
- return;
- }
- RLOG_SQS_DEBUG("Got table format '" << TablesFormat_ << "' for "
- << QueuePath_.UserName << record);
- }
-
+ if (CurrentCreationStep_ == ECreateComponentsStep::GetTablesFormatSetting) {
+ const TValue value(TValue::Create(record.GetExecutionEngineEvaluatedResponse()));
+ const TValue formatValue = value["tablesFormat"];
+ if (formatValue.HaveValue()) {
+ TablesFormat_ = static_cast<ui32>(formatValue);
+ }
+ if (!formatValue.HaveValue() || TablesFormat_ > 1) {
+ RLOG_SQS_WARN("Incorrect TablesFormat settings for account "
+ << QueuePath_.UserName << ", responce:" << record);
+
+ auto resp = MakeErrorResponse(NErrors::INTERNAL_FAILURE);
+ resp->State = EQueueState::Creating;
+ resp->Error = "Incorrect TablesFormat settings for account";
+
+ Send(Sender_, std::move(resp));
+ PassAway();
+ return;
+ }
+ RLOG_SQS_DEBUG("Got table format '" << TablesFormat_ << "' for "
+ << QueuePath_.UserName << record);
+ }
+
Step();
} else {
RLOG_SQS_WARN("Component creation request execution error: " << record);
@@ -667,30 +667,30 @@ void TCreateQueueSchemaActorV2::OnDescribeSchemeResult(NSchemeShard::TEvSchemeSh
}
}
-void TCreateQueueSchemaActorV2::SendDescribeTable() {
- auto navigateRequest = std::make_unique<NSchemeCache::TSchemeCacheNavigate>();
-
- NSchemeCache::TSchemeCacheNavigate::TEntry entry;
- entry.Path = NKikimr::SplitPath(Cfg().GetRoot() + "/.Queues");
- entry.Operation = NSchemeCache::TSchemeCacheNavigate::OpTable;
- navigateRequest->ResultSet.emplace_back(entry);
-
- Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvNavigateKeySet(navigateRequest.release()));
-}
-
-void TCreateQueueSchemaActorV2::HandleTableDescription(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) {
- const NSchemeCache::TSchemeCacheNavigate* result = ev->Get()->Request.Get();
- Y_VERIFY(result->ResultSet.size() == 1);
- const auto& response = result->ResultSet.front();
-
- TableWithLeaderPathId_ = std::make_pair(
- response.TableId.PathId.OwnerId,
- response.TableId.PathId.LocalPathId
- );
-
- Step();
-}
-
+void TCreateQueueSchemaActorV2::SendDescribeTable() {
+ auto navigateRequest = std::make_unique<NSchemeCache::TSchemeCacheNavigate>();
+
+ NSchemeCache::TSchemeCacheNavigate::TEntry entry;
+ entry.Path = NKikimr::SplitPath(Cfg().GetRoot() + "/.Queues");
+ entry.Operation = NSchemeCache::TSchemeCacheNavigate::OpTable;
+ navigateRequest->ResultSet.emplace_back(entry);
+
+ Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvNavigateKeySet(navigateRequest.release()));
+}
+
+void TCreateQueueSchemaActorV2::HandleTableDescription(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) {
+ const NSchemeCache::TSchemeCacheNavigate* result = ev->Get()->Request.Get();
+ Y_VERIFY(result->ResultSet.size() == 1);
+ const auto& response = result->ResultSet.front();
+
+ TableWithLeaderPathId_ = std::make_pair(
+ response.TableId.PathId.OwnerId,
+ response.TableId.PathId.LocalPathId
+ );
+
+ Step();
+}
+
void TCreateQueueSchemaActorV2::AddRPSQuota() {
NKikimrKesus::TEvAddQuoterResource cmd;
auto& res = *cmd.MutableResource();
@@ -730,9 +730,9 @@ static const char* const CommitQueueParamsQuery = R"__(
(let shards (Parameter 'SHARDS (DataType 'Uint64)))
(let partitions (Parameter 'PARTITIONS (DataType 'Uint64)))
(let masterTabletId (Parameter 'MASTER_TABLET_ID (DataType 'Uint64)))
- (let tablesFormat (Parameter 'TABLES_FORMAT (DataType 'Uint32)))
+ (let tablesFormat (Parameter 'TABLES_FORMAT (DataType 'Uint32)))
(let version (Parameter 'VERSION (DataType 'Uint64)))
- (let queueIdNumberHash (Parameter 'QUEUE_ID_NUMBER_HASH (DataType 'Uint64)))
+ (let queueIdNumberHash (Parameter 'QUEUE_ID_NUMBER_HASH (DataType 'Uint64)))
(let maxSize (Parameter 'MAX_SIZE (DataType 'Uint64)))
(let delay (Parameter 'DELAY (DataType 'Uint64)))
(let visibility (Parameter 'VISIBILITY (DataType 'Uint64)))
@@ -744,11 +744,11 @@ static const char* const CommitQueueParamsQuery = R"__(
(let defaultMaxQueuesCount (Parameter 'DEFAULT_MAX_QUEUES_COUNT (DataType 'Uint64)))
(let userName (Parameter 'USER_NAME (DataType 'Utf8String)))
- (let attrsTable '%1$s/Attributes)
- (let stateTable '%1$s/State)
- (let settingsTable '%2$s/.Settings)
- (let queuesTable '%2$s/.Queues)
- (let eventsTable '%2$s/.Events)
+ (let attrsTable '%1$s/Attributes)
+ (let stateTable '%1$s/State)
+ (let settingsTable '%2$s/.Settings)
+ (let queuesTable '%2$s/.Queues)
+ (let eventsTable '%2$s/.Events)
(let maxQueuesCountSettingRow '(
'('Account userName)
@@ -833,16 +833,16 @@ static const char* const CommitQueueParamsQuery = R"__(
'('Partitions partitions)
'('Version version)
'('DlqName dlqName)
- '('MasterTabletId masterTabletId)
- '('TablesFormat tablesFormat)))
+ '('MasterTabletId masterTabletId)
+ '('TablesFormat tablesFormat)))
(let eventsUpdate '(
'('CustomQueueName customName)
'('EventTimestamp now)
'('FolderId folderId)))
- (let attrRow '(%3$s))
-
+ (let attrRow '(%3$s))
+
(let attrUpdate '(
'('ContentBasedDeduplication contentBasedDeduplication)
'('DelaySeconds delay)
@@ -860,19 +860,19 @@ static const char* const CommitQueueParamsQuery = R"__(
(Not queueExists)
(Not overLimit)))
- (let stateUpdate '(
- '('CleanupTimestamp now)
- '('CreatedTimestamp now)
- '('LastModifiedTimestamp now)
- '('InflyCount (Int64 '0))
- '('MessageCount (Int64 '0))
- '('RetentionBoundary (Uint64 '0))
- '('ReadOffset (Uint64 '0))
- '('WriteOffset (Uint64 '0))
- '('CleanupVersion (Uint64 '0))))
-
- (let queueIdNumberAndShardHashes (AsList %4$s))
-
+ (let stateUpdate '(
+ '('CleanupTimestamp now)
+ '('CreatedTimestamp now)
+ '('LastModifiedTimestamp now)
+ '('InflyCount (Int64 '0))
+ '('MessageCount (Int64 '0))
+ '('RetentionBoundary (Uint64 '0))
+ '('ReadOffset (Uint64 '0))
+ '('WriteOffset (Uint64 '0))
+ '('CleanupVersion (Uint64 '0))))
+
+ (let queueIdNumberAndShardHashes (AsList %4$s))
+
(return (Extend
(AsList
(SetResult 'exists queueExists)
@@ -886,14 +886,14 @@ static const char* const CommitQueueParamsQuery = R"__(
(ListIf willCommit (UpdateRow queuesTable queuesRow queuesUpdate))
(ListIf willCommit (UpdateRow eventsTable eventsRow eventsUpdate))
(ListIf willCommit (UpdateRow attrsTable attrRow attrUpdate))
-
- (If (Not willCommit) (AsList (Void))
- (Map (Enumerate queueIdNumberAndShardHashes) (lambda '(item) (block '(
- (let shardOriginal (Nth item '0))
- (let shard (Cast shardOriginal 'Uint32))
- (let queueIdNumberAndShardHash (Nth item '1))
-
- (let row '(%5$s))
+
+ (If (Not willCommit) (AsList (Void))
+ (Map (Enumerate queueIdNumberAndShardHashes) (lambda '(item) (block '(
+ (let shardOriginal (Nth item '0))
+ (let shard (Cast shardOriginal 'Uint32))
+ (let queueIdNumberAndShardHash (Nth item '1))
+
+ (let row '(%5$s))
(let update '(
'('CleanupTimestamp now)
'('CreatedTimestamp now)
@@ -909,62 +909,62 @@ static const char* const CommitQueueParamsQuery = R"__(
)
)__";
-TString GetStateTableKeys(ui32 tablesFormat, bool isFifo) {
- if (tablesFormat == 1) {
- if (isFifo) {
- return R"__(
- '('QueueIdNumberHash queueIdNumberHash)
- '('QueueIdNumber version)
- )__";
- }
- return R"__(
- '('QueueIdNumberAndShardHash queueIdNumberAndShardHash)
- '('QueueIdNumber version)
- '('Shard shard)
- )__";
-
- }
- return "'('State shardOriginal)";
-}
-
-TString GetAttrTableKeys(ui32 tablesFormat) {
- if (tablesFormat == 1) {
- return R"__(
- '('QueueIdNumberHash queueIdNumberHash)
- '('QueueIdNumber version)
- )__";
- }
- return "'('State (Uint64 '0))";
-}
-
-TString GetQueueIdAndShardHashesList(ui64 version, ui32 shards) {
- TStringBuilder hashes;
- for (ui32 i = 0; i < shards; ++i) {
- hashes << "(Uint64 '" << GetHash(version, i) << ") ";
- }
- return hashes;
-}
-
+TString GetStateTableKeys(ui32 tablesFormat, bool isFifo) {
+ if (tablesFormat == 1) {
+ if (isFifo) {
+ return R"__(
+ '('QueueIdNumberHash queueIdNumberHash)
+ '('QueueIdNumber version)
+ )__";
+ }
+ return R"__(
+ '('QueueIdNumberAndShardHash queueIdNumberAndShardHash)
+ '('QueueIdNumber version)
+ '('Shard shard)
+ )__";
+
+ }
+ return "'('State shardOriginal)";
+}
+
+TString GetAttrTableKeys(ui32 tablesFormat) {
+ if (tablesFormat == 1) {
+ return R"__(
+ '('QueueIdNumberHash queueIdNumberHash)
+ '('QueueIdNumber version)
+ )__";
+ }
+ return "'('State (Uint64 '0))";
+}
+
+TString GetQueueIdAndShardHashesList(ui64 version, ui32 shards) {
+ TStringBuilder hashes;
+ for (ui32 i = 0; i < shards; ++i) {
+ hashes << "(Uint64 '" << GetHash(version, i) << ") ";
+ }
+ return hashes;
+}
+
void TCreateQueueSchemaActorV2::CommitNewVersion() {
Become(&TCreateQueueSchemaActorV2::FinalizeAndCommit);
- TString queuePath;
- if (TablesFormat_ == 0) {
- queuePath = Join("/", QueuePath_.GetUserPath(), QueuePath_.QueueName, VersionName_);
- } else {
- queuePath = Join("/", Cfg().GetRoot(), IsFifo_ ? FIFO_TABLES_DIR : STD_TABLES_DIR);
- }
-
- TString query = Sprintf(
- CommitQueueParamsQuery,
- queuePath.c_str(),
- Cfg().GetRoot().c_str(),
- GetAttrTableKeys(TablesFormat_).c_str(),
- GetQueueIdAndShardHashesList(Version_, RequiredShardsCount_).c_str(),
- GetStateTableKeys(TablesFormat_, IsFifo_).c_str()
- );
-
- auto ev = MakeExecuteEvent(query);
+ TString queuePath;
+ if (TablesFormat_ == 0) {
+ queuePath = Join("/", QueuePath_.GetUserPath(), QueuePath_.QueueName, VersionName_);
+ } else {
+ queuePath = Join("/", Cfg().GetRoot(), IsFifo_ ? FIFO_TABLES_DIR : STD_TABLES_DIR);
+ }
+
+ TString query = Sprintf(
+ CommitQueueParamsQuery,
+ queuePath.c_str(),
+ Cfg().GetRoot().c_str(),
+ GetAttrTableKeys(TablesFormat_).c_str(),
+ GetQueueIdAndShardHashesList(Version_, RequiredShardsCount_).c_str(),
+ GetStateTableKeys(TablesFormat_, IsFifo_).c_str()
+ );
+
+ auto ev = MakeExecuteEvent(query);
auto* trans = ev->Record.MutableTransaction()->MutableMiniKQLTransaction();
Y_VERIFY(LeaderTabletId_ != 0);
TParameters(trans->MutableParams()->MutableProto())
@@ -978,9 +978,9 @@ void TCreateQueueSchemaActorV2::CommitNewVersion() {
.Uint64("SHARDS", RequiredShardsCount_)
.Uint64("PARTITIONS", Request_.GetPartitions())
.Uint64("MASTER_TABLET_ID", LeaderTabletId_)
- .Uint32("TABLES_FORMAT", TablesFormat_)
+ .Uint32("TABLES_FORMAT", TablesFormat_)
.Uint64("VERSION", Version_)
- .Uint64("QUEUE_ID_NUMBER_HASH", GetHash(Version_))
+ .Uint64("QUEUE_ID_NUMBER_HASH", GetHash(Version_))
.Uint64("MAX_SIZE", *ValidatedAttributes_.MaximumMessageSize)
.Uint64("DELAY", SecondsToMs(*ValidatedAttributes_.DelaySeconds))
.Uint64("VISIBILITY", SecondsToMs(*ValidatedAttributes_.VisibilityTimeout))
diff --git a/ydb/core/ymq/actor/queue_schema.h b/ydb/core/ymq/actor/queue_schema.h
index 8cbdf86420..793ba1c06a 100644
--- a/ydb/core/ymq/actor/queue_schema.h
+++ b/ydb/core/ymq/actor/queue_schema.h
@@ -47,7 +47,7 @@ public:
void RequestCreateQueueQuota();
void OnCreateQueueQuota(TEvQuota::TEvClearance::TPtr& ev);
- void RequestTablesFormatSettings(const TString& accountName);
+ void RequestTablesFormatSettings(const TString& accountName);
void RegisterMakeDirActor(const TString& workingDir, const TString& dirName);
void RequestLeaderTabletId();
@@ -62,9 +62,9 @@ public:
void OnDescribeSchemeResult(NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult::TPtr& ev);
- void SendDescribeTable();
- void HandleTableDescription(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev);
-
+ void SendDescribeTable();
+ void HandleTableDescription(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev);
+
template <typename T>
T GetAttribute(const TStringBuf name, const T& defaultValue) const;
@@ -92,12 +92,12 @@ public:
private:
enum class ECreateComponentsStep : ui32 {
- GetTablesFormatSetting,
- MakeQueueDir,
+ GetTablesFormatSetting,
+ MakeQueueDir,
MakeQueueVersionDir,
MakeShards,
MakeTables,
- DescribeTableForSetSchemeShardId,
+ DescribeTableForSetSchemeShardId,
DiscoverLeaderTabletId,
AddQuoterResource,
};
@@ -115,7 +115,7 @@ private:
bool IsCloudMode_ = false;
bool EnableQueueAttributesValidation_ = true;
- ui32 TablesFormat_ = 0;
+ ui32 TablesFormat_ = 0;
ui64 Version_ = 0;
TString VersionName_;
@@ -134,7 +134,7 @@ private:
ui64 CreateTableWithLeaderTabletTxId_ = 0;
std::pair<ui64, ui64> TableWithLeaderPathId_ = std::make_pair(0, 0); // (scheme shard, path id) are required for describing table
- ECreateComponentsStep CurrentCreationStep_ = ECreateComponentsStep::GetTablesFormatSetting;
+ ECreateComponentsStep CurrentCreationStep_ = ECreateComponentsStep::GetTablesFormatSetting;
TActorId AddQuoterResourceActor_;
};
diff --git a/ydb/core/ymq/actor/ut/metering_ut.cpp b/ydb/core/ymq/actor/ut/metering_ut.cpp
index 4fc150676b..bb6cbc6458 100644
--- a/ydb/core/ymq/actor/ut/metering_ut.cpp
+++ b/ydb/core/ymq/actor/ut/metering_ut.cpp
@@ -1,352 +1,352 @@
#include <ydb/core/ymq/actor/serviceid.h>
-
+
#include <ydb/core/ymq/http/types.h>
#include <ydb/core/ymq/actor/events.h>
#include <ydb/core/ymq/actor/metering.h>
-
+
#include <ydb/core/mind/address_classification/net_classifier.h>
#include <ydb/core/testlib/test_client.h>
-
-#include <library/cpp/testing/unittest/registar.h>
-#include <library/cpp/logger/global/global.h>
-
-#include <util/generic/serialized_enum.h>
-#include <util/stream/file.h>
-
-namespace NKikimr::NSQS {
-
-using namespace Tests;
-
-bool IsSameRecords(const NSc::TValue& a, const NSc::TValue& b) {
- auto cmp = [&](const TString& key, const TString& subkey = "") {
- auto& va = a.Get(key);
- auto& vb = b.Get(key);
- if (subkey) {
- return NSc::TValue::Equal(va.Get(subkey), vb.Get(subkey));
- }
- return NSc::TValue::Equal(va, vb);
- };
- return cmp("folder_id") && cmp("resource_id") && cmp("schema") && cmp("version")
- && cmp("usage", "type") && cmp("usage", "unit") && cmp("usage", "quantity")
- && cmp("tags", "direction") && cmp("tags", "type");
-}
-
-TVector<NSc::TValue>::const_iterator MatchBillingRecord(const TVector<NSc::TValue>& records, const NSc::TValue& expectedRecord) {
- for (auto it = records.begin(); it != records.end(); ++it) {
- if (IsSameRecords(*it, expectedRecord)) {
- return it;
- }
- }
- return records.end();
-}
-
-void CheckBillingRecord(const TVector<NSc::TValue>& records, const TVector<NSc::TValue>& expectedRecords) {
- UNIT_ASSERT_VALUES_EQUAL(records.size(), expectedRecords.size());
- TVector<NSc::TValue> unmatchedRecords = records;
- for (auto& record : expectedRecords) {
- auto it = MatchBillingRecord(unmatchedRecords, record);
- UNIT_ASSERT(unmatchedRecords.end() != it);
- unmatchedRecords.erase(it);
- }
-}
-
-TVector<NSc::TValue> LoadBillingRecords(const TString& filepath) {
- TString data = TFileInput(filepath).ReadAll();
- auto rawRecords = SplitString(data, "\n");
- TVector<NSc::TValue> records;
- for (auto& record : rawRecords) {
- records.push_back(NSc::TValue::FromJson(record));
- }
- return records;
-}
-
-void AddRequestToQueue(
- TList<THolder<TSqsEvents::TEvReportProcessedRequestAttributes>>& requests,
- const TString sourceAddress,
- ui32 requestSizeInBytes,
- ui32 responseSizeInBytes,
- int statusCode,
- const TString& folderId = "",
- const TString& resourceId = "",
- bool isFifo = false
-) {
- auto request = MakeHolder<TSqsEvents::TEvReportProcessedRequestAttributes>();
- TProcessedRequestAttributes& requestAttributes = request->Data;
- requestAttributes.HttpStatusCode = statusCode;
- requestAttributes.IsFifo = isFifo;
- requestAttributes.FolderId = folderId;
- requestAttributes.RequestSizeInBytes = requestSizeInBytes;
- requestAttributes.ResponseSizeInBytes = responseSizeInBytes;
- requestAttributes.SourceAddress = sourceAddress;
- requestAttributes.ResourceId = resourceId;
- requestAttributes.Action = EAction::ModifyPermissions;
-
- requests.push_back(std::move(request));
-}
-
-auto AddToExpectedRecords(TVector<NSc::TValue>& expectedRecords, const TProcessedRequestAttributes& attrs, TProcessedRequestsAggregator::ENetworkClass networkClass) {
- {
- NSc::TValue tags;
- tags.SetDict();
- tags["type"] = ToString(networkClass);
- tags["direction"] = ToString(TProcessedRequestsAggregator::ETrafficType::ingress);
- expectedRecords.push_back(CreateMeteringBillingRecord(
- attrs.FolderId, attrs.ResourceId,
- "ymq.traffic.v1", "" /*fqdn*/, TInstant::Now(), attrs.RequestSizeInBytes, "byte", tags
- ));
- }
- {
- NSc::TValue tags;
- tags.SetDict();
- tags["type"] = ToString(networkClass);
- tags["direction"] = ToString(TProcessedRequestsAggregator::ETrafficType::egress);
- expectedRecords.push_back(CreateMeteringBillingRecord(
- attrs.FolderId, attrs.ResourceId,
- "ymq.traffic.v1", "" /*fqdn*/, TInstant::Now(), attrs.ResponseSizeInBytes, "byte", tags
- ));
- }
- {
- NSc::TValue tags;
- tags.SetDict();
- tags["queue_type"] = ToString(TProcessedRequestsAggregator::EQueueType::other);
- expectedRecords.push_back(CreateMeteringBillingRecord(
- attrs.FolderId, attrs.ResourceId,
- "ymq.requests.v1", "" /*fqdn*/, TInstant::Now(), attrs.RequestSizeInBytes, "request", tags
- ));
- }
-}
-
-Y_UNIT_TEST_SUITE(Metering) {
- Y_UNIT_TEST(BillingRecords) {
- TServerSettings serverSettings(0);
- TServer server = TServer(serverSettings, true);
- TTestActorRuntime* runtime = server.GetRuntime();
-
- auto& sqsConfig = runtime->GetAppData().SqsConfig;
-
- sqsConfig.SetMeteringFlushingIntervalMs(100);
- sqsConfig.SetMeteringLogFilePath("sqs_meterig.log");
- TFsPath(sqsConfig.GetMeteringLogFilePath()).DeleteIfExists();
-
- sqsConfig.AddMeteringCloudNetCidr("5.45.196.0/24");
- sqsConfig.AddMeteringCloudNetCidr("2a0d:d6c0::/29");
- sqsConfig.AddMeteringYandexNetCidr("127.0.0.0/8");
- sqsConfig.AddMeteringYandexNetCidr("5.45.217.0/24");
-
- DoInitGlobalLog(CreateOwningThreadedLogBackend(sqsConfig.GetMeteringLogFilePath(), 0));
-
- runtime->RegisterService(
- MakeSqsMeteringServiceID(),
- runtime->Register(NSQS::CreateSqsMeteringService())
- );
-
- TList<THolder<TSqsEvents::TEvReportProcessedRequestAttributes>> requests;
- TVector<NSc::TValue> expectedRecords;
-
-
-
- AddRequestToQueue(requests, "5.45.196.0", 1, 2, 200, "folder1");
- AddToExpectedRecords(expectedRecords, requests.back()->Data, TProcessedRequestsAggregator::ENetworkClass::cloud);
- AddRequestToQueue(requests, "5.45.196.0", 10, 20, 500, "folder1");
- AddRequestToQueue(requests, "5.45.196.0", 100, 200, 200);
-
- AddRequestToQueue(requests, "5.45.217.0", 1, 2, 200, "folder2");
- AddToExpectedRecords(expectedRecords, requests.back()->Data, TProcessedRequestsAggregator::ENetworkClass::yandex);
- AddRequestToQueue(requests, "5.45.217.0", 10, 20, 500, "folder2");
- AddRequestToQueue(requests, "5.45.217.0", 100, 200, 200);
-
- AddRequestToQueue(requests, "5.45.215.192", 1, 2, 200, "folder3");
- AddToExpectedRecords(expectedRecords, requests.back()->Data, TProcessedRequestsAggregator::ENetworkClass::inet);
- AddRequestToQueue(requests, "5.45.215.192", 10, 20, 500, "folder3");
- AddRequestToQueue(requests, "5.45.215.192", 100, 200, 200);
-
- for (auto& req : requests) {
- runtime->Send(new IEventHandle(
- MakeSqsMeteringServiceID(), runtime->AllocateEdgeActor(), req.Release()
- ));
- }
-
- Sleep(TDuration::Seconds(5));
- TVector<NSc::TValue> records = LoadBillingRecords(sqsConfig.GetMeteringLogFilePath());
- CheckBillingRecord(records, expectedRecords);
- }
-
- Y_UNIT_TEST(MockedNetClassifierOnly) {
- TServerSettings serverSettings(0);
- TServer server = TServer(serverSettings, true);
- TTestActorRuntime* runtime = server.GetRuntime();
-
- auto& sqsConfig = runtime->GetAppData().SqsConfig;
-
- sqsConfig.SetMeteringByNetClassifierOnly(true);
- sqsConfig.SetMeteringFlushingIntervalMs(100);
- sqsConfig.SetMeteringLogFilePath("sqs_meterig.log");
- TFsPath(sqsConfig.GetMeteringLogFilePath()).DeleteIfExists();
-
- sqsConfig.AddMeteringCloudNetCidr("5.45.196.0/24");
- sqsConfig.AddMeteringYandexNetCidr("5.45.217.0/24");
-
- DoInitGlobalLog(CreateOwningThreadedLogBackend(sqsConfig.GetMeteringLogFilePath(), 0));
-
- runtime->RegisterService(
- MakeSqsMeteringServiceID(),
- runtime->Register(NSQS::CreateSqsMeteringService())
- );
-
- TList<THolder<TSqsEvents::TEvReportProcessedRequestAttributes>> requests;
- TVector<NSc::TValue> expectedRecords;
-
- {
- AddRequestToQueue(requests, "5.45.196.0", 1, 2, 200, "folder1");
- AddToExpectedRecords(expectedRecords, requests.back()->Data, TProcessedRequestsAggregator::ENetworkClass::unknown);
- AddRequestToQueue(requests, "5.45.196.0", 10, 20, 500, "folder1");
- AddRequestToQueue(requests, "5.45.196.0", 100, 200, 200);
-
- AddRequestToQueue(requests, "5.45.217.0", 1, 2, 200, "folder2");
- AddToExpectedRecords(expectedRecords, requests.back()->Data, TProcessedRequestsAggregator::ENetworkClass::unknown);
- AddRequestToQueue(requests, "5.45.217.0", 10, 20, 500, "folder2");
- AddRequestToQueue(requests, "5.45.217.0", 100, 200, 200);
-
- AddRequestToQueue(requests, "5.45.215.192", 1, 2, 200, "folder3");
- AddToExpectedRecords(expectedRecords, requests.back()->Data, TProcessedRequestsAggregator::ENetworkClass::unknown);
- AddRequestToQueue(requests, "5.45.215.192", 10, 20, 500, "folder3");
- AddRequestToQueue(requests, "5.45.215.192", 100, 200, 200);
- }
-
- for (auto& req : requests) {
- runtime->Send(new IEventHandle(
- MakeSqsMeteringServiceID(), runtime->AllocateEdgeActor(), req.Release()
- ));
- }
-
- Sleep(TDuration::Seconds(5));
- TVector<NSc::TValue> records = LoadBillingRecords(sqsConfig.GetMeteringLogFilePath());
- CheckBillingRecord(records, expectedRecords);
-
- {
- NAddressClassifier::TAddressClassifier rawClassifier;
- rawClassifier.AddNetByCidrAndLabel("127.0.0.0/8" , TProcessedRequestsAggregator::ENetworkClass::cloud);
- rawClassifier.AddNetByCidrAndLabel("2a0d:d6c0::/29" , TProcessedRequestsAggregator::ENetworkClass::yandex);
-
- auto enumValues = GetEnumNames<TProcessedRequestsAggregator::ENetworkClass>();
- TVector<TString> labels(enumValues.size());
- for (auto enumItem : enumValues) {
- Y_VERIFY(enumItem.first < labels.size());
- labels[enumItem.first] = enumItem.second;
- }
- auto classifier = NAddressClassifier::TLabeledAddressClassifier::MakeLabeledAddressClassifier(std::move(rawClassifier), std::move(labels));
-
-
- auto classifierUpdateRequest = MakeHolder<NNetClassifier::TEvNetClassifier::TEvClassifierUpdate>();
- classifierUpdateRequest->Classifier = classifier;
- classifierUpdateRequest->NetDataUpdateTimestamp = TInstant::Now();
- runtime->Send(new IEventHandle(
- MakeSqsMeteringServiceID(), runtime->AllocateEdgeActor(), classifierUpdateRequest.Release()
- ));
- }
- Sleep(TDuration::MilliSeconds(100));
- {
- requests.clear();
- AddRequestToQueue(requests, "5.45.196.0", 1, 2, 200, "folder4");
- AddToExpectedRecords(expectedRecords, requests.back()->Data, TProcessedRequestsAggregator::ENetworkClass::inet);
- AddRequestToQueue(requests, "5.45.196.0", 10, 20, 500, "folder4");
- AddRequestToQueue(requests, "5.45.196.0", 100, 200, 200);
-
- AddRequestToQueue(requests, "5.45.217.0", 1, 2, 200, "folder5");
- AddToExpectedRecords(expectedRecords, requests.back()->Data, TProcessedRequestsAggregator::ENetworkClass::inet);
- AddRequestToQueue(requests, "5.45.217.0", 10, 20, 500, "folder5");
- AddRequestToQueue(requests, "5.45.217.0", 100, 200, 200);
-
- AddRequestToQueue(requests, "5.45.215.192", 1, 2, 200, "folder6");
- AddToExpectedRecords(expectedRecords, requests.back()->Data, TProcessedRequestsAggregator::ENetworkClass::inet);
- AddRequestToQueue(requests, "5.45.215.192", 10, 20, 500, "folder6");
- AddRequestToQueue(requests, "5.45.215.192", 100, 200, 200);
-
- AddRequestToQueue(requests, "127.255.255.255", 1, 2, 200, "folder7");
- AddToExpectedRecords(expectedRecords, requests.back()->Data, TProcessedRequestsAggregator::ENetworkClass::cloud);
- AddRequestToQueue(requests, "127.255.255.255", 10, 20, 500, "folder7");
- AddRequestToQueue(requests, "127.255.255.255", 100, 200, 200);
-
- AddRequestToQueue(requests, "2a0d:d6c0:bbbb:bbbb:bbbb:bbbb:bbbb:bbbb", 1, 2, 200, "folder8");
- AddToExpectedRecords(expectedRecords, requests.back()->Data, TProcessedRequestsAggregator::ENetworkClass::yandex);
- AddRequestToQueue(requests, "2a0d:d6c0:bbbb:bbbb:bbbb:bbbb:bbbb:bbbb", 10, 20, 500, "folder8");
- AddRequestToQueue(requests, "2a0d:d6c0:bbbb:bbbb:bbbb:bbbb:bbbb:bbbb", 100, 200, 200);
- }
- for (auto& req : requests) {
- runtime->Send(new IEventHandle(
- MakeSqsMeteringServiceID(), runtime->AllocateEdgeActor(), req.Release()
- ));
- }
-
- Sleep(TDuration::Seconds(5));
- records = LoadBillingRecords(sqsConfig.GetMeteringLogFilePath());
- CheckBillingRecord(records, expectedRecords);
- }
-
- Y_UNIT_TEST(MockedNetClassifierLabelTransformation) {
- TServerSettings serverSettings(0);
- TServer server = TServer(serverSettings, true);
- TTestActorRuntime* runtime = server.GetRuntime();
-
- auto& sqsConfig = runtime->GetAppData().SqsConfig;
-
- sqsConfig.SetMeteringByNetClassifierOnly(true);
- sqsConfig.SetMeteringFlushingIntervalMs(100);
- sqsConfig.SetMeteringLogFilePath("sqs_meterig.log");
- TFsPath(sqsConfig.GetMeteringLogFilePath()).DeleteIfExists();
-
- DoInitGlobalLog(CreateOwningThreadedLogBackend(sqsConfig.GetMeteringLogFilePath(), 0));
-
- runtime->RegisterService(
- MakeSqsMeteringServiceID(),
- runtime->Register(NSQS::CreateSqsMeteringService())
- );
- Sleep(TDuration::MilliSeconds(500)); // waiting to override classifier from real NetClassifier service
- {
- NAddressClassifier::TAddressClassifier rawClassifier;
- TVector<std::pair<TString, TString>> cidrWithLabel = {
- {"127.0.0.0/8", ToString(TProcessedRequestsAggregator::ENetworkClass::yandex)},
- {"2a0d:d6c0::/29", ToString(TProcessedRequestsAggregator::ENetworkClass::cloud)},
- {"5.45.196.0/24", "yacloud"}
- };
- TVector<TString> labels(cidrWithLabel.size());
- for (ui32 index = 0; index < cidrWithLabel.size(); ++index) {
- auto& [cidr, label] = cidrWithLabel[index];
- rawClassifier.AddNetByCidrAndLabel(cidr , index);
- labels[index] = label;
- }
- auto classifier = NAddressClassifier::TLabeledAddressClassifier::MakeLabeledAddressClassifier(std::move(rawClassifier), std::move(labels));
-
- auto classifierUpdateRequest = MakeHolder<NNetClassifier::TEvNetClassifier::TEvClassifierUpdate>();
- classifierUpdateRequest->Classifier = classifier;
- classifierUpdateRequest->NetDataUpdateTimestamp = TInstant::Now();
- runtime->Send(new IEventHandle(
- MakeSqsMeteringServiceID(), runtime->AllocateEdgeActor(), classifierUpdateRequest.Release()
- ));
- }
- Sleep(TDuration::MilliSeconds(100));
- TList<THolder<TSqsEvents::TEvReportProcessedRequestAttributes>> requests;
- TVector<NSc::TValue> expectedRecords;
- {
- AddRequestToQueue(requests, "127.255.255.255", 1, 2, 200, "folder1");
- AddToExpectedRecords(expectedRecords, requests.back()->Data, TProcessedRequestsAggregator::ENetworkClass::yandex);
-
- AddRequestToQueue(requests, "2a0d:d6c0:bbbb:bbbb:bbbb:bbbb:bbbb:bbbb", 1, 2, 200, "folder2");
- AddToExpectedRecords(expectedRecords, requests.back()->Data, TProcessedRequestsAggregator::ENetworkClass::cloud);
-
- AddRequestToQueue(requests, "5.45.196.0", 1, 2, 200, "folder3");
- AddToExpectedRecords(expectedRecords, requests.back()->Data, TProcessedRequestsAggregator::ENetworkClass::cloud);
- }
- for (auto& req : requests) {
- runtime->Send(new IEventHandle(
- MakeSqsMeteringServiceID(), runtime->AllocateEdgeActor(), req.Release()
- ));
- }
-
- Sleep(TDuration::Seconds(5));
- auto records = LoadBillingRecords(sqsConfig.GetMeteringLogFilePath());
- CheckBillingRecord(records, expectedRecords);
- }
-}
-
-} // NKikimr::NSQS
+
+#include <library/cpp/testing/unittest/registar.h>
+#include <library/cpp/logger/global/global.h>
+
+#include <util/generic/serialized_enum.h>
+#include <util/stream/file.h>
+
+namespace NKikimr::NSQS {
+
+using namespace Tests;
+
+bool IsSameRecords(const NSc::TValue& a, const NSc::TValue& b) {
+ auto cmp = [&](const TString& key, const TString& subkey = "") {
+ auto& va = a.Get(key);
+ auto& vb = b.Get(key);
+ if (subkey) {
+ return NSc::TValue::Equal(va.Get(subkey), vb.Get(subkey));
+ }
+ return NSc::TValue::Equal(va, vb);
+ };
+ return cmp("folder_id") && cmp("resource_id") && cmp("schema") && cmp("version")
+ && cmp("usage", "type") && cmp("usage", "unit") && cmp("usage", "quantity")
+ && cmp("tags", "direction") && cmp("tags", "type");
+}
+
+TVector<NSc::TValue>::const_iterator MatchBillingRecord(const TVector<NSc::TValue>& records, const NSc::TValue& expectedRecord) {
+ for (auto it = records.begin(); it != records.end(); ++it) {
+ if (IsSameRecords(*it, expectedRecord)) {
+ return it;
+ }
+ }
+ return records.end();
+}
+
+void CheckBillingRecord(const TVector<NSc::TValue>& records, const TVector<NSc::TValue>& expectedRecords) {
+ UNIT_ASSERT_VALUES_EQUAL(records.size(), expectedRecords.size());
+ TVector<NSc::TValue> unmatchedRecords = records;
+ for (auto& record : expectedRecords) {
+ auto it = MatchBillingRecord(unmatchedRecords, record);
+ UNIT_ASSERT(unmatchedRecords.end() != it);
+ unmatchedRecords.erase(it);
+ }
+}
+
+TVector<NSc::TValue> LoadBillingRecords(const TString& filepath) {
+ TString data = TFileInput(filepath).ReadAll();
+ auto rawRecords = SplitString(data, "\n");
+ TVector<NSc::TValue> records;
+ for (auto& record : rawRecords) {
+ records.push_back(NSc::TValue::FromJson(record));
+ }
+ return records;
+}
+
+void AddRequestToQueue(
+ TList<THolder<TSqsEvents::TEvReportProcessedRequestAttributes>>& requests,
+ const TString sourceAddress,
+ ui32 requestSizeInBytes,
+ ui32 responseSizeInBytes,
+ int statusCode,
+ const TString& folderId = "",
+ const TString& resourceId = "",
+ bool isFifo = false
+) {
+ auto request = MakeHolder<TSqsEvents::TEvReportProcessedRequestAttributes>();
+ TProcessedRequestAttributes& requestAttributes = request->Data;
+ requestAttributes.HttpStatusCode = statusCode;
+ requestAttributes.IsFifo = isFifo;
+ requestAttributes.FolderId = folderId;
+ requestAttributes.RequestSizeInBytes = requestSizeInBytes;
+ requestAttributes.ResponseSizeInBytes = responseSizeInBytes;
+ requestAttributes.SourceAddress = sourceAddress;
+ requestAttributes.ResourceId = resourceId;
+ requestAttributes.Action = EAction::ModifyPermissions;
+
+ requests.push_back(std::move(request));
+}
+
+auto AddToExpectedRecords(TVector<NSc::TValue>& expectedRecords, const TProcessedRequestAttributes& attrs, TProcessedRequestsAggregator::ENetworkClass networkClass) {
+ {
+ NSc::TValue tags;
+ tags.SetDict();
+ tags["type"] = ToString(networkClass);
+ tags["direction"] = ToString(TProcessedRequestsAggregator::ETrafficType::ingress);
+ expectedRecords.push_back(CreateMeteringBillingRecord(
+ attrs.FolderId, attrs.ResourceId,
+ "ymq.traffic.v1", "" /*fqdn*/, TInstant::Now(), attrs.RequestSizeInBytes, "byte", tags
+ ));
+ }
+ {
+ NSc::TValue tags;
+ tags.SetDict();
+ tags["type"] = ToString(networkClass);
+ tags["direction"] = ToString(TProcessedRequestsAggregator::ETrafficType::egress);
+ expectedRecords.push_back(CreateMeteringBillingRecord(
+ attrs.FolderId, attrs.ResourceId,
+ "ymq.traffic.v1", "" /*fqdn*/, TInstant::Now(), attrs.ResponseSizeInBytes, "byte", tags
+ ));
+ }
+ {
+ NSc::TValue tags;
+ tags.SetDict();
+ tags["queue_type"] = ToString(TProcessedRequestsAggregator::EQueueType::other);
+ expectedRecords.push_back(CreateMeteringBillingRecord(
+ attrs.FolderId, attrs.ResourceId,
+ "ymq.requests.v1", "" /*fqdn*/, TInstant::Now(), attrs.RequestSizeInBytes, "request", tags
+ ));
+ }
+}
+
+Y_UNIT_TEST_SUITE(Metering) {
+ Y_UNIT_TEST(BillingRecords) {
+ TServerSettings serverSettings(0);
+ TServer server = TServer(serverSettings, true);
+ TTestActorRuntime* runtime = server.GetRuntime();
+
+ auto& sqsConfig = runtime->GetAppData().SqsConfig;
+
+ sqsConfig.SetMeteringFlushingIntervalMs(100);
+ sqsConfig.SetMeteringLogFilePath("sqs_meterig.log");
+ TFsPath(sqsConfig.GetMeteringLogFilePath()).DeleteIfExists();
+
+ sqsConfig.AddMeteringCloudNetCidr("5.45.196.0/24");
+ sqsConfig.AddMeteringCloudNetCidr("2a0d:d6c0::/29");
+ sqsConfig.AddMeteringYandexNetCidr("127.0.0.0/8");
+ sqsConfig.AddMeteringYandexNetCidr("5.45.217.0/24");
+
+ DoInitGlobalLog(CreateOwningThreadedLogBackend(sqsConfig.GetMeteringLogFilePath(), 0));
+
+ runtime->RegisterService(
+ MakeSqsMeteringServiceID(),
+ runtime->Register(NSQS::CreateSqsMeteringService())
+ );
+
+ TList<THolder<TSqsEvents::TEvReportProcessedRequestAttributes>> requests;
+ TVector<NSc::TValue> expectedRecords;
+
+
+
+ AddRequestToQueue(requests, "5.45.196.0", 1, 2, 200, "folder1");
+ AddToExpectedRecords(expectedRecords, requests.back()->Data, TProcessedRequestsAggregator::ENetworkClass::cloud);
+ AddRequestToQueue(requests, "5.45.196.0", 10, 20, 500, "folder1");
+ AddRequestToQueue(requests, "5.45.196.0", 100, 200, 200);
+
+ AddRequestToQueue(requests, "5.45.217.0", 1, 2, 200, "folder2");
+ AddToExpectedRecords(expectedRecords, requests.back()->Data, TProcessedRequestsAggregator::ENetworkClass::yandex);
+ AddRequestToQueue(requests, "5.45.217.0", 10, 20, 500, "folder2");
+ AddRequestToQueue(requests, "5.45.217.0", 100, 200, 200);
+
+ AddRequestToQueue(requests, "5.45.215.192", 1, 2, 200, "folder3");
+ AddToExpectedRecords(expectedRecords, requests.back()->Data, TProcessedRequestsAggregator::ENetworkClass::inet);
+ AddRequestToQueue(requests, "5.45.215.192", 10, 20, 500, "folder3");
+ AddRequestToQueue(requests, "5.45.215.192", 100, 200, 200);
+
+ for (auto& req : requests) {
+ runtime->Send(new IEventHandle(
+ MakeSqsMeteringServiceID(), runtime->AllocateEdgeActor(), req.Release()
+ ));
+ }
+
+ Sleep(TDuration::Seconds(5));
+ TVector<NSc::TValue> records = LoadBillingRecords(sqsConfig.GetMeteringLogFilePath());
+ CheckBillingRecord(records, expectedRecords);
+ }
+
+ Y_UNIT_TEST(MockedNetClassifierOnly) {
+ TServerSettings serverSettings(0);
+ TServer server = TServer(serverSettings, true);
+ TTestActorRuntime* runtime = server.GetRuntime();
+
+ auto& sqsConfig = runtime->GetAppData().SqsConfig;
+
+ sqsConfig.SetMeteringByNetClassifierOnly(true);
+ sqsConfig.SetMeteringFlushingIntervalMs(100);
+ sqsConfig.SetMeteringLogFilePath("sqs_meterig.log");
+ TFsPath(sqsConfig.GetMeteringLogFilePath()).DeleteIfExists();
+
+ sqsConfig.AddMeteringCloudNetCidr("5.45.196.0/24");
+ sqsConfig.AddMeteringYandexNetCidr("5.45.217.0/24");
+
+ DoInitGlobalLog(CreateOwningThreadedLogBackend(sqsConfig.GetMeteringLogFilePath(), 0));
+
+ runtime->RegisterService(
+ MakeSqsMeteringServiceID(),
+ runtime->Register(NSQS::CreateSqsMeteringService())
+ );
+
+ TList<THolder<TSqsEvents::TEvReportProcessedRequestAttributes>> requests;
+ TVector<NSc::TValue> expectedRecords;
+
+ {
+ AddRequestToQueue(requests, "5.45.196.0", 1, 2, 200, "folder1");
+ AddToExpectedRecords(expectedRecords, requests.back()->Data, TProcessedRequestsAggregator::ENetworkClass::unknown);
+ AddRequestToQueue(requests, "5.45.196.0", 10, 20, 500, "folder1");
+ AddRequestToQueue(requests, "5.45.196.0", 100, 200, 200);
+
+ AddRequestToQueue(requests, "5.45.217.0", 1, 2, 200, "folder2");
+ AddToExpectedRecords(expectedRecords, requests.back()->Data, TProcessedRequestsAggregator::ENetworkClass::unknown);
+ AddRequestToQueue(requests, "5.45.217.0", 10, 20, 500, "folder2");
+ AddRequestToQueue(requests, "5.45.217.0", 100, 200, 200);
+
+ AddRequestToQueue(requests, "5.45.215.192", 1, 2, 200, "folder3");
+ AddToExpectedRecords(expectedRecords, requests.back()->Data, TProcessedRequestsAggregator::ENetworkClass::unknown);
+ AddRequestToQueue(requests, "5.45.215.192", 10, 20, 500, "folder3");
+ AddRequestToQueue(requests, "5.45.215.192", 100, 200, 200);
+ }
+
+ for (auto& req : requests) {
+ runtime->Send(new IEventHandle(
+ MakeSqsMeteringServiceID(), runtime->AllocateEdgeActor(), req.Release()
+ ));
+ }
+
+ Sleep(TDuration::Seconds(5));
+ TVector<NSc::TValue> records = LoadBillingRecords(sqsConfig.GetMeteringLogFilePath());
+ CheckBillingRecord(records, expectedRecords);
+
+ {
+ NAddressClassifier::TAddressClassifier rawClassifier;
+ rawClassifier.AddNetByCidrAndLabel("127.0.0.0/8" , TProcessedRequestsAggregator::ENetworkClass::cloud);
+ rawClassifier.AddNetByCidrAndLabel("2a0d:d6c0::/29" , TProcessedRequestsAggregator::ENetworkClass::yandex);
+
+ auto enumValues = GetEnumNames<TProcessedRequestsAggregator::ENetworkClass>();
+ TVector<TString> labels(enumValues.size());
+ for (auto enumItem : enumValues) {
+ Y_VERIFY(enumItem.first < labels.size());
+ labels[enumItem.first] = enumItem.second;
+ }
+ auto classifier = NAddressClassifier::TLabeledAddressClassifier::MakeLabeledAddressClassifier(std::move(rawClassifier), std::move(labels));
+
+
+ auto classifierUpdateRequest = MakeHolder<NNetClassifier::TEvNetClassifier::TEvClassifierUpdate>();
+ classifierUpdateRequest->Classifier = classifier;
+ classifierUpdateRequest->NetDataUpdateTimestamp = TInstant::Now();
+ runtime->Send(new IEventHandle(
+ MakeSqsMeteringServiceID(), runtime->AllocateEdgeActor(), classifierUpdateRequest.Release()
+ ));
+ }
+ Sleep(TDuration::MilliSeconds(100));
+ {
+ requests.clear();
+ AddRequestToQueue(requests, "5.45.196.0", 1, 2, 200, "folder4");
+ AddToExpectedRecords(expectedRecords, requests.back()->Data, TProcessedRequestsAggregator::ENetworkClass::inet);
+ AddRequestToQueue(requests, "5.45.196.0", 10, 20, 500, "folder4");
+ AddRequestToQueue(requests, "5.45.196.0", 100, 200, 200);
+
+ AddRequestToQueue(requests, "5.45.217.0", 1, 2, 200, "folder5");
+ AddToExpectedRecords(expectedRecords, requests.back()->Data, TProcessedRequestsAggregator::ENetworkClass::inet);
+ AddRequestToQueue(requests, "5.45.217.0", 10, 20, 500, "folder5");
+ AddRequestToQueue(requests, "5.45.217.0", 100, 200, 200);
+
+ AddRequestToQueue(requests, "5.45.215.192", 1, 2, 200, "folder6");
+ AddToExpectedRecords(expectedRecords, requests.back()->Data, TProcessedRequestsAggregator::ENetworkClass::inet);
+ AddRequestToQueue(requests, "5.45.215.192", 10, 20, 500, "folder6");
+ AddRequestToQueue(requests, "5.45.215.192", 100, 200, 200);
+
+ AddRequestToQueue(requests, "127.255.255.255", 1, 2, 200, "folder7");
+ AddToExpectedRecords(expectedRecords, requests.back()->Data, TProcessedRequestsAggregator::ENetworkClass::cloud);
+ AddRequestToQueue(requests, "127.255.255.255", 10, 20, 500, "folder7");
+ AddRequestToQueue(requests, "127.255.255.255", 100, 200, 200);
+
+ AddRequestToQueue(requests, "2a0d:d6c0:bbbb:bbbb:bbbb:bbbb:bbbb:bbbb", 1, 2, 200, "folder8");
+ AddToExpectedRecords(expectedRecords, requests.back()->Data, TProcessedRequestsAggregator::ENetworkClass::yandex);
+ AddRequestToQueue(requests, "2a0d:d6c0:bbbb:bbbb:bbbb:bbbb:bbbb:bbbb", 10, 20, 500, "folder8");
+ AddRequestToQueue(requests, "2a0d:d6c0:bbbb:bbbb:bbbb:bbbb:bbbb:bbbb", 100, 200, 200);
+ }
+ for (auto& req : requests) {
+ runtime->Send(new IEventHandle(
+ MakeSqsMeteringServiceID(), runtime->AllocateEdgeActor(), req.Release()
+ ));
+ }
+
+ Sleep(TDuration::Seconds(5));
+ records = LoadBillingRecords(sqsConfig.GetMeteringLogFilePath());
+ CheckBillingRecord(records, expectedRecords);
+ }
+
+ Y_UNIT_TEST(MockedNetClassifierLabelTransformation) {
+ TServerSettings serverSettings(0);
+ TServer server = TServer(serverSettings, true);
+ TTestActorRuntime* runtime = server.GetRuntime();
+
+ auto& sqsConfig = runtime->GetAppData().SqsConfig;
+
+ sqsConfig.SetMeteringByNetClassifierOnly(true);
+ sqsConfig.SetMeteringFlushingIntervalMs(100);
+ sqsConfig.SetMeteringLogFilePath("sqs_meterig.log");
+ TFsPath(sqsConfig.GetMeteringLogFilePath()).DeleteIfExists();
+
+ DoInitGlobalLog(CreateOwningThreadedLogBackend(sqsConfig.GetMeteringLogFilePath(), 0));
+
+ runtime->RegisterService(
+ MakeSqsMeteringServiceID(),
+ runtime->Register(NSQS::CreateSqsMeteringService())
+ );
+ Sleep(TDuration::MilliSeconds(500)); // waiting to override classifier from real NetClassifier service
+ {
+ NAddressClassifier::TAddressClassifier rawClassifier;
+ TVector<std::pair<TString, TString>> cidrWithLabel = {
+ {"127.0.0.0/8", ToString(TProcessedRequestsAggregator::ENetworkClass::yandex)},
+ {"2a0d:d6c0::/29", ToString(TProcessedRequestsAggregator::ENetworkClass::cloud)},
+ {"5.45.196.0/24", "yacloud"}
+ };
+ TVector<TString> labels(cidrWithLabel.size());
+ for (ui32 index = 0; index < cidrWithLabel.size(); ++index) {
+ auto& [cidr, label] = cidrWithLabel[index];
+ rawClassifier.AddNetByCidrAndLabel(cidr , index);
+ labels[index] = label;
+ }
+ auto classifier = NAddressClassifier::TLabeledAddressClassifier::MakeLabeledAddressClassifier(std::move(rawClassifier), std::move(labels));
+
+ auto classifierUpdateRequest = MakeHolder<NNetClassifier::TEvNetClassifier::TEvClassifierUpdate>();
+ classifierUpdateRequest->Classifier = classifier;
+ classifierUpdateRequest->NetDataUpdateTimestamp = TInstant::Now();
+ runtime->Send(new IEventHandle(
+ MakeSqsMeteringServiceID(), runtime->AllocateEdgeActor(), classifierUpdateRequest.Release()
+ ));
+ }
+ Sleep(TDuration::MilliSeconds(100));
+ TList<THolder<TSqsEvents::TEvReportProcessedRequestAttributes>> requests;
+ TVector<NSc::TValue> expectedRecords;
+ {
+ AddRequestToQueue(requests, "127.255.255.255", 1, 2, 200, "folder1");
+ AddToExpectedRecords(expectedRecords, requests.back()->Data, TProcessedRequestsAggregator::ENetworkClass::yandex);
+
+ AddRequestToQueue(requests, "2a0d:d6c0:bbbb:bbbb:bbbb:bbbb:bbbb:bbbb", 1, 2, 200, "folder2");
+ AddToExpectedRecords(expectedRecords, requests.back()->Data, TProcessedRequestsAggregator::ENetworkClass::cloud);
+
+ AddRequestToQueue(requests, "5.45.196.0", 1, 2, 200, "folder3");
+ AddToExpectedRecords(expectedRecords, requests.back()->Data, TProcessedRequestsAggregator::ENetworkClass::cloud);
+ }
+ for (auto& req : requests) {
+ runtime->Send(new IEventHandle(
+ MakeSqsMeteringServiceID(), runtime->AllocateEdgeActor(), req.Release()
+ ));
+ }
+
+ Sleep(TDuration::Seconds(5));
+ auto records = LoadBillingRecords(sqsConfig.GetMeteringLogFilePath());
+ CheckBillingRecord(records, expectedRecords);
+ }
+}
+
+} // NKikimr::NSQS
diff --git a/ydb/core/ymq/actor/ut/ya.make b/ydb/core/ymq/actor/ut/ya.make
index 1f41f730aa..a3b4f59f37 100644
--- a/ydb/core/ymq/actor/ut/ya.make
+++ b/ydb/core/ymq/actor/ut/ya.make
@@ -20,7 +20,7 @@ SRCS(
infly_ut.cpp
message_delay_stats_ut.cpp
sha256_ut.cpp
- metering_ut.cpp
+ metering_ut.cpp
)
YQL_LAST_ABI_VERSION()
diff --git a/ydb/core/ymq/base/counters.cpp b/ydb/core/ymq/base/counters.cpp
index 77f203d97c..90a9f5000f 100644
--- a/ydb/core/ymq/base/counters.cpp
+++ b/ydb/core/ymq/base/counters.cpp
@@ -938,13 +938,13 @@ void TCloudAuthCounters::InitCounters(TIntrusivePtr<NMonitoring::TDynamicCounter
INIT_HISTOGRAM_COUNTER(cloudAuthCounters, GetFolderIdDuration, ELifetime::Persistent, DurationBucketsMs, Lazy(*Cfg));
}
-void TMeteringCounters::InitCounters(const TVector<TString>& classifierLabels) {
- const TString classifierRequestsLabel("ClassifierRequests_");
- const TString idleClassifierRequestsLabel("IdleClassifierRequests_");
- for (auto label : classifierLabels) {
- INIT_COUNTER_WITH_NAME(SqsMeteringCounters, ClassifierRequestsResults[label], TStringBuilder() << classifierRequestsLabel << label, ELifetime::Persistent, EValueType::Derivative, Lazy(Config));
- INIT_COUNTER_WITH_NAME(SqsMeteringCounters, IdleClassifierRequestsResults[label], TStringBuilder() << idleClassifierRequestsLabel << label, ELifetime::Persistent, EValueType::Derivative, Lazy(Config));
- }
-}
-
+void TMeteringCounters::InitCounters(const TVector<TString>& classifierLabels) {
+ const TString classifierRequestsLabel("ClassifierRequests_");
+ const TString idleClassifierRequestsLabel("IdleClassifierRequests_");
+ for (auto label : classifierLabels) {
+ INIT_COUNTER_WITH_NAME(SqsMeteringCounters, ClassifierRequestsResults[label], TStringBuilder() << classifierRequestsLabel << label, ELifetime::Persistent, EValueType::Derivative, Lazy(Config));
+ INIT_COUNTER_WITH_NAME(SqsMeteringCounters, IdleClassifierRequestsResults[label], TStringBuilder() << idleClassifierRequestsLabel << label, ELifetime::Persistent, EValueType::Derivative, Lazy(Config));
+ }
+}
+
} // namespace NKikimr::NSQS
diff --git a/ydb/core/ymq/base/counters.h b/ydb/core/ymq/base/counters.h
index b246fe4b90..3c2667b241 100644
--- a/ydb/core/ymq/base/counters.h
+++ b/ydb/core/ymq/base/counters.h
@@ -788,24 +788,24 @@ private:
[GRPC_STATUSES_COUNT]; // 18 types.
};
-// Metering counters in SQS core subsystem.
-struct TMeteringCounters : public TAtomicRefCount<TMeteringCounters> {
- THashMap<TString, TLazyCachedCounter> ClassifierRequestsResults;
- THashMap<TString, TLazyCachedCounter> IdleClassifierRequestsResults;
-
- TMeteringCounters(const NKikimrConfig::TSqsConfig& config, const TIntrusivePtr<NMonitoring::TDynamicCounters>& sqsMeteringCounters, const TVector<TString>& classifierLabels)
- : SqsMeteringCounters(sqsMeteringCounters)
- , Config(config)
- {
- InitCounters(classifierLabels);
- }
-
-private:
- void InitCounters(const TVector<TString>& classifierLabels);
-
-private:
- TIntrusivePtr<NMonitoring::TDynamicCounters> SqsMeteringCounters;
- const NKikimrConfig::TSqsConfig& Config;
-};
-
+// Metering counters in SQS core subsystem.
+struct TMeteringCounters : public TAtomicRefCount<TMeteringCounters> {
+ THashMap<TString, TLazyCachedCounter> ClassifierRequestsResults;
+ THashMap<TString, TLazyCachedCounter> IdleClassifierRequestsResults;
+
+ TMeteringCounters(const NKikimrConfig::TSqsConfig& config, const TIntrusivePtr<NMonitoring::TDynamicCounters>& sqsMeteringCounters, const TVector<TString>& classifierLabels)
+ : SqsMeteringCounters(sqsMeteringCounters)
+ , Config(config)
+ {
+ InitCounters(classifierLabels);
+ }
+
+private:
+ void InitCounters(const TVector<TString>& classifierLabels);
+
+private:
+ TIntrusivePtr<NMonitoring::TDynamicCounters> SqsMeteringCounters;
+ const NKikimrConfig::TSqsConfig& Config;
+};
+
} // namespace NKikimr::NSQS
diff --git a/ydb/core/ymq/base/ut/counters_ut.cpp b/ydb/core/ymq/base/ut/counters_ut.cpp
index b82cd7cff2..5ff3d26e36 100644
--- a/ydb/core/ymq/base/ut/counters_ut.cpp
+++ b/ydb/core/ymq/base/ut/counters_ut.cpp
@@ -500,31 +500,31 @@ Y_UNIT_TEST_SUITE(HttpCountersTest) {
}
}
-Y_UNIT_TEST_SUITE(MeteringCountersTest) {
- Y_UNIT_TEST(CountersAggregationTest) {
- NKikimrConfig::TSqsConfig config;
- config.SetCreateLazyCounters(false);
- TIntrusivePtr<NMonitoring::TDynamicCounters> metering = new NMonitoring::TDynamicCounters();
- TIntrusivePtr<TMeteringCounters> counters = new TMeteringCounters(config, metering, {"inet", "yandex", "unknown", "cloud"});
- *counters->ClassifierRequestsResults["inet"] = 33;
- *counters->ClassifierRequestsResults["yandex"] = 42;
- *counters->ClassifierRequestsResults["unknown"] = 113;
- *counters->IdleClassifierRequestsResults["cloud"] = 128;
-
- AssertCounterValues(metering,
- {
- { "ConnectionsCount", 0 },
- { "ClassifierRequests_cloud", 0},
- { "ClassifierRequests_inet", 33},
- { "ClassifierRequests_unknown", 113},
- { "ClassifierRequests_yandex", 42},
- { "IdleClassifierRequests_cloud", 128},
- { "IdleClassifierRequests_inet", 0},
- { "IdleClassifierRequests_unknown", 0},
- { "IdleClassifierRequests_yandex", 0},
- { "RequestExceptions", 0}
- });
- }
-}
-
+Y_UNIT_TEST_SUITE(MeteringCountersTest) {
+ Y_UNIT_TEST(CountersAggregationTest) {
+ NKikimrConfig::TSqsConfig config;
+ config.SetCreateLazyCounters(false);
+ TIntrusivePtr<NMonitoring::TDynamicCounters> metering = new NMonitoring::TDynamicCounters();
+ TIntrusivePtr<TMeteringCounters> counters = new TMeteringCounters(config, metering, {"inet", "yandex", "unknown", "cloud"});
+ *counters->ClassifierRequestsResults["inet"] = 33;
+ *counters->ClassifierRequestsResults["yandex"] = 42;
+ *counters->ClassifierRequestsResults["unknown"] = 113;
+ *counters->IdleClassifierRequestsResults["cloud"] = 128;
+
+ AssertCounterValues(metering,
+ {
+ { "ConnectionsCount", 0 },
+ { "ClassifierRequests_cloud", 0},
+ { "ClassifierRequests_inet", 33},
+ { "ClassifierRequests_unknown", 113},
+ { "ClassifierRequests_yandex", 42},
+ { "IdleClassifierRequests_cloud", 128},
+ { "IdleClassifierRequests_inet", 0},
+ { "IdleClassifierRequests_unknown", 0},
+ { "IdleClassifierRequests_yandex", 0},
+ { "RequestExceptions", 0}
+ });
+ }
+}
+
} // namespace NKikimr::NSQS
diff --git a/ydb/library/persqueue/counter_time_keeper/counter_time_keeper.cpp b/ydb/library/persqueue/counter_time_keeper/counter_time_keeper.cpp
index 72867be7d6..64a6d6c5e2 100644
--- a/ydb/library/persqueue/counter_time_keeper/counter_time_keeper.cpp
+++ b/ydb/library/persqueue/counter_time_keeper/counter_time_keeper.cpp
@@ -1 +1 @@
-#include "counter_time_keeper.h"
+#include "counter_time_keeper.h"
diff --git a/ydb/library/persqueue/counter_time_keeper/counter_time_keeper.h b/ydb/library/persqueue/counter_time_keeper/counter_time_keeper.h
index 0d4cf81479..313de598d3 100644
--- a/ydb/library/persqueue/counter_time_keeper/counter_time_keeper.h
+++ b/ydb/library/persqueue/counter_time_keeper/counter_time_keeper.h
@@ -1,22 +1,22 @@
-#pragma once
-
+#pragma once
+
#include <ydb/core/tablet/tablet_counters.h>
-
-#include <util/system/hp_timer.h>
-
-namespace NPersQueue {
-
-class TCounterTimeKeeper {
-public:
- TCounterTimeKeeper (NKikimr::TTabletCumulativeCounter& counter)
- : Counter(counter)
- {}
- ~TCounterTimeKeeper() {
- Counter += ui64(CpuTimer.PassedReset() * 1000000.);
- }
-private:
- NKikimr::TTabletCumulativeCounter& Counter;
- THPTimer CpuTimer;
-};
-
-} // namespace NPersQueue
+
+#include <util/system/hp_timer.h>
+
+namespace NPersQueue {
+
+class TCounterTimeKeeper {
+public:
+ TCounterTimeKeeper (NKikimr::TTabletCumulativeCounter& counter)
+ : Counter(counter)
+ {}
+ ~TCounterTimeKeeper() {
+ Counter += ui64(CpuTimer.PassedReset() * 1000000.);
+ }
+private:
+ NKikimr::TTabletCumulativeCounter& Counter;
+ THPTimer CpuTimer;
+};
+
+} // namespace NPersQueue
diff --git a/ydb/library/persqueue/counter_time_keeper/ya.make b/ydb/library/persqueue/counter_time_keeper/ya.make
index 2980fbcada..106ad89431 100644
--- a/ydb/library/persqueue/counter_time_keeper/ya.make
+++ b/ydb/library/persqueue/counter_time_keeper/ya.make
@@ -1,15 +1,15 @@
OWNER(g:logbroker)
-
-LIBRARY()
-
-PEERDIR(
+
+LIBRARY()
+
+PEERDIR(
library/cpp/actors/protos
ydb/core/protos
-)
-
-SRCS(
- counter_time_keeper.h
- counter_time_keeper.cpp
-)
-
-END()
+)
+
+SRCS(
+ counter_time_keeper.h
+ counter_time_keeper.cpp
+)
+
+END()
diff --git a/ydb/library/persqueue/obfuscate/obfuscate.cpp b/ydb/library/persqueue/obfuscate/obfuscate.cpp
index 30f9bcc068..c1a9b0b1a7 100644
--- a/ydb/library/persqueue/obfuscate/obfuscate.cpp
+++ b/ydb/library/persqueue/obfuscate/obfuscate.cpp
@@ -1,15 +1,15 @@
-#include "obfuscate.h"
-
-#include <util/generic/string.h>
-
-namespace NPersQueue {
-
-TString ObfuscateString(TString str) {
- ui32 publicPartSize = Min<ui32>(4, str.size() / 4);
- for (ui32 i = publicPartSize; i < str.size() - publicPartSize; ++i) {
- str[i] = '*';
- }
- return str;
-}
-
-} // namespace NPersQueue
+#include "obfuscate.h"
+
+#include <util/generic/string.h>
+
+namespace NPersQueue {
+
+TString ObfuscateString(TString str) {
+ ui32 publicPartSize = Min<ui32>(4, str.size() / 4);
+ for (ui32 i = publicPartSize; i < str.size() - publicPartSize; ++i) {
+ str[i] = '*';
+ }
+ return str;
+}
+
+} // namespace NPersQueue
diff --git a/ydb/library/persqueue/obfuscate/obfuscate.h b/ydb/library/persqueue/obfuscate/obfuscate.h
index 8fd8bf2ff2..5eb7302fe1 100644
--- a/ydb/library/persqueue/obfuscate/obfuscate.h
+++ b/ydb/library/persqueue/obfuscate/obfuscate.h
@@ -1,9 +1,9 @@
-#pragma once
-
-#include <util/generic/fwd.h>
-
-namespace NPersQueue {
-
-TString ObfuscateString(TString str);
-
-} // namespace NPersQueue
+#pragma once
+
+#include <util/generic/fwd.h>
+
+namespace NPersQueue {
+
+TString ObfuscateString(TString str);
+
+} // namespace NPersQueue
diff --git a/ydb/library/persqueue/obfuscate/ya.make b/ydb/library/persqueue/obfuscate/ya.make
index f361e8ddf9..88df0eac37 100644
--- a/ydb/library/persqueue/obfuscate/ya.make
+++ b/ydb/library/persqueue/obfuscate/ya.make
@@ -1,10 +1,10 @@
OWNER(g:logbroker)
-
-LIBRARY()
-
-SRCS(
- obfuscate.h
- obfuscate.cpp
-)
-
-END()
+
+LIBRARY()
+
+SRCS(
+ obfuscate.h
+ obfuscate.cpp
+)
+
+END()
diff --git a/ydb/library/persqueue/topic_parser/ya.make b/ydb/library/persqueue/topic_parser/ya.make
index 200d6cde9a..2b4f126a59 100644
--- a/ydb/library/persqueue/topic_parser/ya.make
+++ b/ydb/library/persqueue/topic_parser/ya.make
@@ -1,23 +1,23 @@
-OWNER(
+OWNER(
komels
- g:kikimr
- g:logbroker
-)
-
-LIBRARY()
-
-PEERDIR(
+ g:kikimr
+ g:logbroker
+)
+
+LIBRARY()
+
+PEERDIR(
ydb/core/base
ydb/library/persqueue/topic_parser_public
ydb/public/api/protos
-)
-
-SRCS(
- topic_parser.h
- topic_parser.cpp
-)
+)
+
+SRCS(
+ topic_parser.h
+ topic_parser.cpp
+)
-END()
+END()
RECURSE_FOR_TESTS(
ut
diff --git a/ydb/library/persqueue/ya.make b/ydb/library/persqueue/ya.make
index 6dc12584fe..d28331e52e 100644
--- a/ydb/library/persqueue/ya.make
+++ b/ydb/library/persqueue/ya.make
@@ -1,9 +1,9 @@
-OWNER(g:logbroker)
-
-RECURSE(
- counter_time_keeper
- obfuscate
+OWNER(g:logbroker)
+
+RECURSE(
+ counter_time_keeper
+ obfuscate
tests
- topic_parser
+ topic_parser
topic_parser_public
)
diff --git a/ydb/library/pretty_types_print/protobuf/out.cpp b/ydb/library/pretty_types_print/protobuf/out.cpp
index 624ae35086..1013108dd2 100644
--- a/ydb/library/pretty_types_print/protobuf/out.cpp
+++ b/ydb/library/pretty_types_print/protobuf/out.cpp
@@ -1,9 +1,9 @@
-#include <util/stream/output.h>
-#include <util/string/join.h>
-
-#include <google/protobuf/repeated_field.h>
-
-Y_DECLARE_OUT_SPEC(, google::protobuf::RepeatedPtrField<TString>, stream, value)
-{
- stream << JoinSeq(", ", value);
-}
+#include <util/stream/output.h>
+#include <util/string/join.h>
+
+#include <google/protobuf/repeated_field.h>
+
+Y_DECLARE_OUT_SPEC(, google::protobuf::RepeatedPtrField<TString>, stream, value)
+{
+ stream << JoinSeq(", ", value);
+}
diff --git a/ydb/library/pretty_types_print/protobuf/ya.make b/ydb/library/pretty_types_print/protobuf/ya.make
index 1c95e9e9e1..e41c037f03 100644
--- a/ydb/library/pretty_types_print/protobuf/ya.make
+++ b/ydb/library/pretty_types_print/protobuf/ya.make
@@ -1,13 +1,13 @@
-LIBRARY()
-
-OWNER(g:kikimr)
-
-SRCS(
- out.cpp
-)
-
-PEERDIR(
- contrib/libs/protobuf
-)
-
-END()
+LIBRARY()
+
+OWNER(g:kikimr)
+
+SRCS(
+ out.cpp
+)
+
+PEERDIR(
+ contrib/libs/protobuf
+)
+
+END()
diff --git a/ydb/library/pretty_types_print/wilson/out.cpp b/ydb/library/pretty_types_print/wilson/out.cpp
index a1e88f0591..9b8c60f981 100644
--- a/ydb/library/pretty_types_print/wilson/out.cpp
+++ b/ydb/library/pretty_types_print/wilson/out.cpp
@@ -1,7 +1,7 @@
#include <ydb/library/wilson/wilson_trace.h>
-
-#include <util/stream/output.h>
-
-Y_DECLARE_OUT_SPEC(, NWilson::TTraceId, stream, value) {
- value.OutputSpanId(stream);
-}
+
+#include <util/stream/output.h>
+
+Y_DECLARE_OUT_SPEC(, NWilson::TTraceId, stream, value) {
+ value.OutputSpanId(stream);
+}
diff --git a/ydb/library/pretty_types_print/wilson/ya.make b/ydb/library/pretty_types_print/wilson/ya.make
index 99e27e74c4..6dba9499b1 100644
--- a/ydb/library/pretty_types_print/wilson/ya.make
+++ b/ydb/library/pretty_types_print/wilson/ya.make
@@ -1,13 +1,13 @@
-LIBRARY()
-
-OWNER(g:kikimr)
-
-SRCS(
- out.cpp
-)
-
-PEERDIR(
+LIBRARY()
+
+OWNER(g:kikimr)
+
+SRCS(
+ out.cpp
+)
+
+PEERDIR(
ydb/library/wilson
-)
-
-END()
+)
+
+END()
diff --git a/ydb/library/pretty_types_print/ya.make b/ydb/library/pretty_types_print/ya.make
index 191cca6931..957b484f05 100644
--- a/ydb/library/pretty_types_print/ya.make
+++ b/ydb/library/pretty_types_print/ya.make
@@ -1,4 +1,4 @@
-RECURSE(
- protobuf
- wilson
-)
+RECURSE(
+ protobuf
+ wilson
+)
diff --git a/ydb/public/api/protos/draft/persqueue_common.proto b/ydb/public/api/protos/draft/persqueue_common.proto
index 3a0afbf2ca..87bb3f236d 100644
--- a/ydb/public/api/protos/draft/persqueue_common.proto
+++ b/ydb/public/api/protos/draft/persqueue_common.proto
@@ -1,42 +1,42 @@
-syntax = "proto3";
-import "google/protobuf/descriptor.proto";
+syntax = "proto3";
+import "google/protobuf/descriptor.proto";
import "ydb/public/api/protos/draft/persqueue_error_codes.proto";
-
-package NPersQueueCommon;
-
-option java_package = "com.yandex.ydb.persqueue";
-option cc_enable_arenas = true;
-
-extend google.protobuf.FileOptions {
- bool GenerateYaStyle = 66678;
-}
-
-// TODO: Change to IssueMessage
-message Error {
- NPersQueue.NErrorCode.EErrorCode code = 1;
- string description = 2;
-}
-
-enum ECodec {
- RAW = 0;
- GZIP = 1;
- LZOP = 2;
- ZSTD = 3;
- DEFAULT = 100; // TODO: move to pqlib
-}
-
-/**
- * Structure for storing client credentials.
- * Client must provide this structre in first request (init of write\read) and in case of changing,
- * e.g. client gets new TVM Service ticket once per hour. Not all requests (read/write) could contain TCredentials - server will take already known credentials.
- * Provided auth credentials will be checked once per some time at serverside. This means that if TVM ticket is too old, then read\write session will die.
- * If no auth provided at all - client will be using guest rights. So, reading/writing is still possible without credentials.
- */
-
-message Credentials {
- oneof credentials {
- bytes tvm_service_ticket = 1;
- bytes oauth_token = 2;
- }
-}
-
+
+package NPersQueueCommon;
+
+option java_package = "com.yandex.ydb.persqueue";
+option cc_enable_arenas = true;
+
+extend google.protobuf.FileOptions {
+ bool GenerateYaStyle = 66678;
+}
+
+// TODO: Change to IssueMessage
+message Error {
+ NPersQueue.NErrorCode.EErrorCode code = 1;
+ string description = 2;
+}
+
+enum ECodec {
+ RAW = 0;
+ GZIP = 1;
+ LZOP = 2;
+ ZSTD = 3;
+ DEFAULT = 100; // TODO: move to pqlib
+}
+
+/**
+ * Structure for storing client credentials.
+ * Client must provide this structre in first request (init of write\read) and in case of changing,
+ * e.g. client gets new TVM Service ticket once per hour. Not all requests (read/write) could contain TCredentials - server will take already known credentials.
+ * Provided auth credentials will be checked once per some time at serverside. This means that if TVM ticket is too old, then read\write session will die.
+ * If no auth provided at all - client will be using guest rights. So, reading/writing is still possible without credentials.
+ */
+
+message Credentials {
+ oneof credentials {
+ bytes tvm_service_ticket = 1;
+ bytes oauth_token = 2;
+ }
+}
+
diff --git a/ydb/public/api/protos/out/out.cpp b/ydb/public/api/protos/out/out.cpp
index 850f902e77..9e8b7263ee 100644
--- a/ydb/public/api/protos/out/out.cpp
+++ b/ydb/public/api/protos/out/out.cpp
@@ -1,12 +1,12 @@
#include <ydb/public/api/protos/ydb_cms.pb.h>
#include <ydb/public/api/protos/ydb_status_codes.pb.h>
-
-#include <util/stream/output.h>
-
-Y_DECLARE_OUT_SPEC(, Ydb::Cms::GetDatabaseStatusResult::State, stream, value) {
- stream << Ydb::Cms::GetDatabaseStatusResult::State_Name(value);
-}
-
-Y_DECLARE_OUT_SPEC(, Ydb::StatusIds::StatusCode, stream, value) {
- stream << Ydb::StatusIds::StatusCode_Name(value);
-}
+
+#include <util/stream/output.h>
+
+Y_DECLARE_OUT_SPEC(, Ydb::Cms::GetDatabaseStatusResult::State, stream, value) {
+ stream << Ydb::Cms::GetDatabaseStatusResult::State_Name(value);
+}
+
+Y_DECLARE_OUT_SPEC(, Ydb::StatusIds::StatusCode, stream, value) {
+ stream << Ydb::StatusIds::StatusCode_Name(value);
+}
diff --git a/ydb/public/api/protos/out/ya.make b/ydb/public/api/protos/out/ya.make
index 82ce9b754e..c984157c89 100644
--- a/ydb/public/api/protos/out/ya.make
+++ b/ydb/public/api/protos/out/ya.make
@@ -1,13 +1,13 @@
-LIBRARY()
-
-OWNER(g:kikimr)
-
-SRCS(
- out.cpp
-)
-
-PEERDIR(
+LIBRARY()
+
+OWNER(g:kikimr)
+
+SRCS(
+ out.cpp
+)
+
+PEERDIR(
ydb/public/api/protos
-)
-
-END()
+)
+
+END()
diff --git a/ydb/public/api/protos/ya.make b/ydb/public/api/protos/ya.make
index b6e03aa5cb..8a156403a3 100644
--- a/ydb/public/api/protos/ya.make
+++ b/ydb/public/api/protos/ya.make
@@ -15,7 +15,7 @@ PEERDIR(
SRCS(
draft/datastreams.proto
- draft/persqueue_common.proto
+ draft/persqueue_common.proto
draft/persqueue_error_codes.proto
draft/ydb_long_tx.proto
draft/ydb_logstore.proto
@@ -51,7 +51,7 @@ CPP_PROTO_PLUGIN0(validation ydb/core/grpc_services/validation)
# .pb.h are only available in C++ variant of PROTO_LIBRARY
IF (MODULE_TAG == "CPP_PROTO")
- GENERATE_ENUM_SERIALIZATION(draft/persqueue_common.pb.h)
+ GENERATE_ENUM_SERIALIZATION(draft/persqueue_common.pb.h)
GENERATE_ENUM_SERIALIZATION(ydb_persqueue_cluster_discovery.pb.h)
GENERATE_ENUM_SERIALIZATION(draft/datastreams.pb.h)
ENDIF()
diff --git a/ydb/public/api/protos/ydb_persqueue_v1.proto b/ydb/public/api/protos/ydb_persqueue_v1.proto
index 6c33f2afa0..93a7fb6c79 100644
--- a/ydb/public/api/protos/ydb_persqueue_v1.proto
+++ b/ydb/public/api/protos/ydb_persqueue_v1.proto
@@ -1108,22 +1108,22 @@ message DropTopicResult {
}
/**
- * Credentials settings
- */
-
-message Credentials {
- message Iam {
- string endpoint = 1;
- string service_account_key = 2;
- }
- oneof credentials {
- string oauth_token = 1;
- string jwt_params = 2;
- Iam iam = 3;
- }
-}
-
-/**
+ * Credentials settings
+ */
+
+message Credentials {
+ message Iam {
+ string endpoint = 1;
+ string service_account_key = 2;
+ }
+ oneof credentials {
+ string oauth_token = 1;
+ string jwt_params = 2;
+ Iam iam = 3;
+ }
+}
+
+/**
* Message for describing topic internals.
*/
@@ -1134,9 +1134,9 @@ message TopicSettings {
}
// How many partitions in topic. Must less than database limit. Default limit - 10.
- int32 partitions_count = 1 [(value) = "> 0"];
+ int32 partitions_count = 1 [(value) = "> 0"];
// How long data in partition should be stored. Must be greater than 0 and less than limit for this database. Default limit - 36 hours.
- int64 retention_period_ms = 2 [(value) = "> 0"];
+ int64 retention_period_ms = 2 [(value) = "> 0"];
// How long last written seqno for message group should be stored. Must be greater then retention_period_ms and less then limit for this database. Default limit - 16 days.
int64 message_group_seqno_retention_period_ms = 12 [(value) = ">= 0"];
// How many last written seqno for various message groups should be stored per partition. Must be less than limit for this database. Default limit - 6*10^6 values.
@@ -1148,11 +1148,11 @@ message TopicSettings {
// Writes with codec not from this list are forbiden.
repeated Codec supported_codecs = 4 [(size).le = 100];
// Max storage usage for each topic's partition. Must be less than database limit. Default limit - 130 GB.
- int64 max_partition_storage_size = 5 [(value) = ">= 0"];
+ int64 max_partition_storage_size = 5 [(value) = ">= 0"];
// Partition write speed in bytes per second. Must be less than database limit. Default limit - 1 MB/s.
- int64 max_partition_write_speed = 6 [(value) = ">= 0"];
+ int64 max_partition_write_speed = 6 [(value) = ">= 0"];
// Burst size for write in partition, in bytes. Must be less than database limit. Default limit - 1 MB.
- int64 max_partition_write_burst = 7 [(value) = ">= 0"];
+ int64 max_partition_write_burst = 7 [(value) = ">= 0"];
// Disallows client writes. Used for mirrored topics in federation.
bool client_write_disabled = 8;
@@ -1164,7 +1164,7 @@ message TopicSettings {
// Flag that this consumer is important.
bool important = 2;
// All messages with smaller timestamp of write will be skipped.
- int64 starting_message_timestamp_ms = 3 [(value) = ">= 0"];
+ int64 starting_message_timestamp_ms = 3 [(value) = ">= 0"];
// Max format version that is supported by this consumer.
// supported_format on topic must not be greater.
Format supported_format = 4;
@@ -1173,10 +1173,10 @@ message TopicSettings {
repeated Codec supported_codecs = 5 [(size).le = 100];
// Read rule version. Any non-negative integer.
- int64 version = 6 [(value) = ">= 0"];
-
- // Client service type.
- string service_type = 7;
+ int64 version = 6 [(value) = ">= 0"];
+
+ // Client service type.
+ string service_type = 7;
}
// List of consumer read rules for this topic.
@@ -1184,24 +1184,24 @@ message TopicSettings {
// User and server attributes of topic. Server attributes starts from "_" and will be validated by server.
map<string, string> attributes = 10;
-
- // Message for remote mirror rule description.
- message RemoteMirrorRule {
- // Source cluster endpoint in format server:port.
- string endpoint = 1;
- // Source topic that we want to mirror.
- string topic_path = 2;
- // Source consumer for reading source topic.
- string consumer_name = 3;
- // Credentials for reading source topic by source consumer.
- Credentials credentials = 4;
- // All messages with smaller timestamp of write will be skipped.
- int64 starting_message_timestamp_ms = 5 [(value) = ">= 0"];
- // Database
- string database = 6;
- }
- // remote mirror rule for this topic.
- RemoteMirrorRule remote_mirror_rule = 11;
+
+ // Message for remote mirror rule description.
+ message RemoteMirrorRule {
+ // Source cluster endpoint in format server:port.
+ string endpoint = 1;
+ // Source topic that we want to mirror.
+ string topic_path = 2;
+ // Source consumer for reading source topic.
+ string consumer_name = 3;
+ // Credentials for reading source topic by source consumer.
+ Credentials credentials = 4;
+ // All messages with smaller timestamp of write will be skipped.
+ int64 starting_message_timestamp_ms = 5 [(value) = ">= 0"];
+ // Database
+ string database = 6;
+ }
+ // remote mirror rule for this topic.
+ RemoteMirrorRule remote_mirror_rule = 11;
}
/**
diff --git a/ydb/public/lib/base/msgbus.h b/ydb/public/lib/base/msgbus.h
index d97e36e880..f11bd9771f 100644
--- a/ydb/public/lib/base/msgbus.h
+++ b/ydb/public/lib/base/msgbus.h
@@ -258,22 +258,22 @@ public:
virtual ~IMessageBusServer() {}
};
-class IPersQueueGetReadSessionsInfoWorkerFactory;
+class IPersQueueGetReadSessionsInfoWorkerFactory;
+
+IMessageBusServer* CreateMsgBusServer(
+ NBus::TBusMessageQueue *queue,
+ const NBus::TBusServerSessionConfig &config,
+ std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory,
+ ui32 bindPort = TProtocol::DefaultPort
+);
+IMessageBusServer* CreateMsgBusTracingServer(
+ NBus::TBusMessageQueue *queue,
+ const NBus::TBusServerSessionConfig &config,
+ const TString &tracePath,
+ std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory,
+ ui32 bindPort = TProtocol::DefaultPort
+);
-IMessageBusServer* CreateMsgBusServer(
- NBus::TBusMessageQueue *queue,
- const NBus::TBusServerSessionConfig &config,
- std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory,
- ui32 bindPort = TProtocol::DefaultPort
-);
-IMessageBusServer* CreateMsgBusTracingServer(
- NBus::TBusMessageQueue *queue,
- const NBus::TBusServerSessionConfig &config,
- const TString &tracePath,
- std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory,
- ui32 bindPort = TProtocol::DefaultPort
-);
-
inline NActors::TActorId CreateMsgBusProxyId() {
return NActors::TActorId(0, "MsgBusProxy");
}
diff --git a/ydb/public/lib/experimental/ydb_clickhouse_internal.cpp b/ydb/public/lib/experimental/ydb_clickhouse_internal.cpp
index 179d8b9791..2984f680b7 100644
--- a/ydb/public/lib/experimental/ydb_clickhouse_internal.cpp
+++ b/ydb/public/lib/experimental/ydb_clickhouse_internal.cpp
@@ -1,9 +1,9 @@
#include "ydb_clickhouse_internal.h"
-#define INCLUDE_YDB_INTERNAL_H
+#define INCLUDE_YDB_INTERNAL_H
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.h>
-#undef INCLUDE_YDB_INTERNAL_H
-
+#undef INCLUDE_YDB_INTERNAL_H
+
#include <ydb/public/api/grpc/draft/ydb_clickhouse_internal_v1.grpc.pb.h>
#include <library/cpp/grpc/client/grpc_client_low.h>
#include <ydb/public/sdk/cpp/client/ydb_common_client/impl/client.h>
diff --git a/ydb/public/lib/experimental/ydb_experimental.cpp b/ydb/public/lib/experimental/ydb_experimental.cpp
index 939468e0be..8765c0c750 100644
--- a/ydb/public/lib/experimental/ydb_experimental.cpp
+++ b/ydb/public/lib/experimental/ydb_experimental.cpp
@@ -1,9 +1,9 @@
#include "ydb_experimental.h"
-#define INCLUDE_YDB_INTERNAL_H
+#define INCLUDE_YDB_INTERNAL_H
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.h>
-#undef INCLUDE_YDB_INTERNAL_H
-
+#undef INCLUDE_YDB_INTERNAL_H
+
#include <ydb/public/api/grpc/draft/ydb_experimental_v1.grpc.pb.h>
#include <library/cpp/grpc/client/grpc_client_low.h>
#include <ydb/public/sdk/cpp/client/ydb_proto/accessor.h>
@@ -112,7 +112,7 @@ public:
auto request = MakeRequest<Ydb::Experimental::ExecuteStreamQueryRequest>();
request.set_yql_text(query);
if (params) {
- *request.mutable_parameters() = params->GetProtoMap();
+ *request.mutable_parameters() = params->GetProtoMap();
}
switch (settings.ProfileMode_) {
diff --git a/ydb/public/lib/experimental/ydb_s3_internal.cpp b/ydb/public/lib/experimental/ydb_s3_internal.cpp
index c9486a6e1a..ac1c5025ed 100644
--- a/ydb/public/lib/experimental/ydb_s3_internal.cpp
+++ b/ydb/public/lib/experimental/ydb_s3_internal.cpp
@@ -1,9 +1,9 @@
#include "ydb_s3_internal.h"
-#define INCLUDE_YDB_INTERNAL_H
+#define INCLUDE_YDB_INTERNAL_H
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.h>
-#undef INCLUDE_YDB_INTERNAL_H
-
+#undef INCLUDE_YDB_INTERNAL_H
+
#include <ydb/public/api/grpc/draft/ydb_s3_internal_v1.grpc.pb.h>
#include <ydb/library/yql/public/issue/yql_issue.h>
diff --git a/ydb/public/lib/idx_test/idx_test_data_provider.cpp b/ydb/public/lib/idx_test/idx_test_data_provider.cpp
index 2d173baf13..0006551f29 100644
--- a/ydb/public/lib/idx_test/idx_test_data_provider.cpp
+++ b/ydb/public/lib/idx_test/idx_test_data_provider.cpp
@@ -4,8 +4,8 @@
#include <library/cpp/string_utils/base64/base64.h>
-#include <util/string/builder.h>
-
+#include <util/string/builder.h>
+
using namespace NYdb;
using namespace NYdb::NTable;
diff --git a/ydb/public/sdk/cpp/client/draft/ut/ya.make b/ydb/public/sdk/cpp/client/draft/ut/ya.make
index 9e40de167b..6e210f3fe3 100644
--- a/ydb/public/sdk/cpp/client/draft/ut/ya.make
+++ b/ydb/public/sdk/cpp/client/draft/ut/ya.make
@@ -1,23 +1,23 @@
UNITTEST_FOR(ydb/public/sdk/cpp/client/draft)
-
-OWNER(
- dcherednik
- g:kikimr
-)
-
-IF (SANITIZER_TYPE)
- TIMEOUT(1200)
- SIZE(LARGE)
- TAG(ya:fat)
-ELSE()
- TIMEOUT(600)
- SIZE(MEDIUM)
-ENDIF()
-
-FORK_SUBTESTS()
-
-SRCS(
- ydb_scripting_response_headers_ut.cpp
-)
-
-END()
+
+OWNER(
+ dcherednik
+ g:kikimr
+)
+
+IF (SANITIZER_TYPE)
+ TIMEOUT(1200)
+ SIZE(LARGE)
+ TAG(ya:fat)
+ELSE()
+ TIMEOUT(600)
+ SIZE(MEDIUM)
+ENDIF()
+
+FORK_SUBTESTS()
+
+SRCS(
+ ydb_scripting_response_headers_ut.cpp
+)
+
+END()
diff --git a/ydb/public/sdk/cpp/client/draft/ya.make b/ydb/public/sdk/cpp/client/draft/ya.make
index b2e982a502..a5f5fbf597 100644
--- a/ydb/public/sdk/cpp/client/draft/ya.make
+++ b/ydb/public/sdk/cpp/client/draft/ya.make
@@ -18,7 +18,7 @@ PEERDIR(
)
END()
-
+
RECURSE_FOR_TESTS(
ut
)
diff --git a/ydb/public/sdk/cpp/client/draft/ydb_scripting.cpp b/ydb/public/sdk/cpp/client/draft/ydb_scripting.cpp
index 6e39163d4d..38b8c8123d 100644
--- a/ydb/public/sdk/cpp/client/draft/ydb_scripting.cpp
+++ b/ydb/public/sdk/cpp/client/draft/ydb_scripting.cpp
@@ -1,10 +1,10 @@
#include "ydb_scripting.h"
-#define INCLUDE_YDB_INTERNAL_H
+#define INCLUDE_YDB_INTERNAL_H
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.h>
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/table_helpers/helpers.h>
-#undef INCLUDE_YDB_INTERNAL_H
-
+#undef INCLUDE_YDB_INTERNAL_H
+
#include <ydb/public/api/grpc/ydb_scripting_v1.grpc.pb.h>
#include <ydb/public/api/protos/ydb_scripting.pb.h>
#include <ydb/public/sdk/cpp/client/ydb_common_client/impl/client.h>
@@ -322,7 +322,7 @@ TParamsBuilder TScriptingClient::GetParamsBuilder() {
TAsyncExecuteYqlResult TScriptingClient::ExecuteYqlScript(const TString &query, NYdb::TParams&& params,
const TExecuteYqlRequestSettings &settings)
{
- auto paramsPtr = params.Empty() ? nullptr : params.GetProtoMapPtr();
+ auto paramsPtr = params.Empty() ? nullptr : params.GetProtoMapPtr();
return Impl_->ExecuteYqlScript(query, paramsPtr, settings);
}
@@ -338,7 +338,7 @@ TAsyncExecuteYqlResult TScriptingClient::ExecuteYqlScript(const TString &query,
using TProtoParamsType = const ::google::protobuf::Map<TString, Ydb::TypedValue>;
return Impl_->ExecuteYqlScript<TProtoParamsType&>(
query,
- params.GetProtoMap(),
+ params.GetProtoMap(),
settings);
}
}
diff --git a/ydb/public/sdk/cpp/client/extensions/discovery_mutator/discovery_mutator_ut.cpp b/ydb/public/sdk/cpp/client/extensions/discovery_mutator/discovery_mutator_ut.cpp
index a9db12c0df..7a5fa43cc3 100644
--- a/ydb/public/sdk/cpp/client/extensions/discovery_mutator/discovery_mutator_ut.cpp
+++ b/ydb/public/sdk/cpp/client/extensions/discovery_mutator/discovery_mutator_ut.cpp
@@ -2,7 +2,7 @@
#include <ydb/public/sdk/cpp/client/extensions/discovery_mutator/discovery_mutator.h>
#include <ydb/public/sdk/cpp/client/ydb_extension/extension.h>
#include <ydb/public/sdk/cpp/client/ydb_table/table.h>
-
+
#include <library/cpp/testing/unittest/registar.h>
#include <library/cpp/testing/unittest/tests_data.h>
diff --git a/ydb/public/sdk/cpp/client/extensions/discovery_mutator/ut/ya.make b/ydb/public/sdk/cpp/client/extensions/discovery_mutator/ut/ya.make
index 249e4409f8..d6688b8e0a 100644
--- a/ydb/public/sdk/cpp/client/extensions/discovery_mutator/ut/ya.make
+++ b/ydb/public/sdk/cpp/client/extensions/discovery_mutator/ut/ya.make
@@ -1,27 +1,27 @@
UNITTEST_FOR(ydb/public/sdk/cpp/client/extensions/discovery_mutator)
-
-OWNER(
- dcherednik
- g:kikimr
-)
-
-IF (SANITIZER_TYPE)
- TIMEOUT(1200)
- SIZE(LARGE)
- TAG(ya:fat)
-ELSE()
- TIMEOUT(600)
- SIZE(MEDIUM)
-ENDIF()
-
-FORK_SUBTESTS()
-
-PEERDIR(
+
+OWNER(
+ dcherednik
+ g:kikimr
+)
+
+IF (SANITIZER_TYPE)
+ TIMEOUT(1200)
+ SIZE(LARGE)
+ TAG(ya:fat)
+ELSE()
+ TIMEOUT(600)
+ SIZE(MEDIUM)
+ENDIF()
+
+FORK_SUBTESTS()
+
+PEERDIR(
ydb/public/sdk/cpp/client/ydb_table
-)
-
-SRCS(
- discovery_mutator_ut.cpp
-)
-
-END()
+)
+
+SRCS(
+ discovery_mutator_ut.cpp
+)
+
+END()
diff --git a/ydb/public/sdk/cpp/client/extensions/discovery_mutator/ya.make b/ydb/public/sdk/cpp/client/extensions/discovery_mutator/ya.make
index a1ec29b162..7d4997e94c 100644
--- a/ydb/public/sdk/cpp/client/extensions/discovery_mutator/ya.make
+++ b/ydb/public/sdk/cpp/client/extensions/discovery_mutator/ya.make
@@ -14,7 +14,7 @@ PEERDIR(
)
END()
-
+
RECURSE_FOR_TESTS(
ut
)
diff --git a/ydb/public/sdk/cpp/client/extensions/solomon_stats/README.md b/ydb/public/sdk/cpp/client/extensions/solomon_stats/README.md
index e934f99a89..869678d8dd 100644
--- a/ydb/public/sdk/cpp/client/extensions/solomon_stats/README.md
+++ b/ydb/public/sdk/cpp/client/extensions/solomon_stats/README.md
@@ -38,7 +38,7 @@ After creating NYdb::TDriver you need to add Solomon Monitoring extension. If yo
...
{
- auto config = NYdb::TDriverConfig();
+ auto config = NYdb::TDriverConfig();
NYdb::TDriver driver(config);
try {
diff --git a/ydb/public/sdk/cpp/client/extensions/solomon_stats/pull_client.h b/ydb/public/sdk/cpp/client/extensions/solomon_stats/pull_client.h
index 5ef7363256..8d410c2fa2 100644
--- a/ydb/public/sdk/cpp/client/extensions/solomon_stats/pull_client.h
+++ b/ydb/public/sdk/cpp/client/extensions/solomon_stats/pull_client.h
@@ -1,7 +1,7 @@
#pragma once
#include <ydb/public/sdk/cpp/client/ydb_extension/extension.h>
-
+
#include <library/cpp/http/server/response.h>
#include <library/cpp/monlib/metrics/metric_consumer.h>
#include <library/cpp/monlib/encode/json/json.h>
@@ -9,8 +9,8 @@
#include <library/cpp/monlib/service/pages/mon_page.h>
#include <library/cpp/monlib/service/monservice.h>
-#include <util/generic/vector.h>
-
+#include <util/generic/vector.h>
+
namespace NSolomonStatExtension {
class TSolomonStatPullExtension: public NYdb::IExtension {
diff --git a/ydb/public/sdk/cpp/client/extensions/ya.make b/ydb/public/sdk/cpp/client/extensions/ya.make
index 6b76506504..c6b7c59c93 100644
--- a/ydb/public/sdk/cpp/client/extensions/ya.make
+++ b/ydb/public/sdk/cpp/client/extensions/ya.make
@@ -1,6 +1,6 @@
-OWNER(g:kikimr)
-
-RECURSE(
- solomon_stats
- discovery_mutator
-)
+OWNER(g:kikimr)
+
+RECURSE(
+ solomon_stats
+ discovery_mutator
+)
diff --git a/ydb/public/sdk/cpp/client/impl/ydb_endpoints/endpoints.cpp b/ydb/public/sdk/cpp/client/impl/ydb_endpoints/endpoints.cpp
index 76284cd53a..b6d03cc50a 100644
--- a/ydb/public/sdk/cpp/client/impl/ydb_endpoints/endpoints.cpp
+++ b/ydb/public/sdk/cpp/client/impl/ydb_endpoints/endpoints.cpp
@@ -1,13 +1,13 @@
-#include "endpoints.h"
+#include "endpoints.h"
#include <library/cpp/monlib/metrics/metric_registry.h>
#include <library/cpp/string_utils/quote/quote.h>
-
-#include <util/random/random.h>
-#include <set>
-#include <unordered_set>
-
+#include <util/random/random.h>
+
+#include <set>
+#include <unordered_set>
+
namespace NYdb {
class TEndpointElectorSafe::TObjRegistry : public IObjRegistryHandle {
diff --git a/ydb/public/sdk/cpp/client/impl/ydb_endpoints/endpoints_ut.cpp b/ydb/public/sdk/cpp/client/impl/ydb_endpoints/endpoints_ut.cpp
index eef0a8671a..7823fb6460 100644
--- a/ydb/public/sdk/cpp/client/impl/ydb_endpoints/endpoints_ut.cpp
+++ b/ydb/public/sdk/cpp/client/impl/ydb_endpoints/endpoints_ut.cpp
@@ -1,13 +1,13 @@
#include <ydb/public/sdk/cpp/client/impl/ydb_endpoints/endpoints.h>
-
+
#include <library/cpp/testing/unittest/registar.h>
#include <library/cpp/testing/unittest/tests_data.h>
-#include <library/cpp/threading/future/core/future.h>
+#include <library/cpp/threading/future/core/future.h>
#include <util/system/thread.h>
#include <util/random/random.h>
-#include <unordered_set>
+#include <unordered_set>
using namespace NYdb;
diff --git a/ydb/public/sdk/cpp/client/impl/ydb_endpoints/ut/ya.make b/ydb/public/sdk/cpp/client/impl/ydb_endpoints/ut/ya.make
index 4300971cb9..1f9dfbbe1d 100644
--- a/ydb/public/sdk/cpp/client/impl/ydb_endpoints/ut/ya.make
+++ b/ydb/public/sdk/cpp/client/impl/ydb_endpoints/ut/ya.make
@@ -1,23 +1,23 @@
UNITTEST_FOR(ydb/public/sdk/cpp/client/impl/ydb_endpoints)
-
-OWNER(
- dcherednik
- g:kikimr
-)
-
-IF (SANITIZER_TYPE)
- TIMEOUT(1200)
- SIZE(LARGE)
- TAG(ya:fat)
-ELSE()
- TIMEOUT(600)
- SIZE(MEDIUM)
-ENDIF()
-
-FORK_SUBTESTS()
-
-SRCS(
- endpoints_ut.cpp
-)
-
-END()
+
+OWNER(
+ dcherednik
+ g:kikimr
+)
+
+IF (SANITIZER_TYPE)
+ TIMEOUT(1200)
+ SIZE(LARGE)
+ TAG(ya:fat)
+ELSE()
+ TIMEOUT(600)
+ SIZE(MEDIUM)
+ENDIF()
+
+FORK_SUBTESTS()
+
+SRCS(
+ endpoints_ut.cpp
+)
+
+END()
diff --git a/ydb/public/sdk/cpp/client/impl/ydb_endpoints/ya.make b/ydb/public/sdk/cpp/client/impl/ydb_endpoints/ya.make
index 85e1fd466b..da6a27f707 100644
--- a/ydb/public/sdk/cpp/client/impl/ydb_endpoints/ya.make
+++ b/ydb/public/sdk/cpp/client/impl/ydb_endpoints/ya.make
@@ -6,7 +6,7 @@ OWNER(
)
SRCS(
- endpoints.cpp
+ endpoints.cpp
)
PEERDIR(
@@ -15,7 +15,7 @@ PEERDIR(
)
END()
-
+
RECURSE_FOR_TESTS(
ut
)
diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/common/types.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/common/types.h
index 8048dd830f..c789a4b5e2 100644
--- a/ydb/public/sdk/cpp/client/impl/ydb_internal/common/types.h
+++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/common/types.h
@@ -1,26 +1,26 @@
-#pragma once
-
+#pragma once
+
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/internal_header.h>
-
+
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/common/type_switcher.h>
#include <ydb/public/sdk/cpp/client/ydb_types/status_codes.h>
#include <ydb/public/sdk/cpp/client/ydb_types/ydb.h>
-
+
#include <ydb/library/yql/public/issue/yql_issue.h>
-
-#include <library/cpp/grpc/client/grpc_client_low.h>
-
-
-namespace NYdb {
-
-// Other callbacks
-using TSimpleCb = std::function<void()>;
-using TErrorCb = std::function<void(NGrpc::TGrpcStatus&)>;
-
-struct TBalancingSettings {
- EBalancingPolicy Policy;
+
+#include <library/cpp/grpc/client/grpc_client_low.h>
+
+
+namespace NYdb {
+
+// Other callbacks
+using TSimpleCb = std::function<void()>;
+using TErrorCb = std::function<void(NGrpc::TGrpcStatus&)>;
+
+struct TBalancingSettings {
+ EBalancingPolicy Policy;
TStringType PolicyParams;
-};
-
-} // namespace NYdb
+};
+
+} // namespace NYdb
diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/common/ya.make b/ydb/public/sdk/cpp/client/impl/ydb_internal/common/ya.make
index 5072fa2259..db5fd8bf4e 100644
--- a/ydb/public/sdk/cpp/client/impl/ydb_internal/common/ya.make
+++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/common/ya.make
@@ -1,16 +1,16 @@
-LIBRARY()
-
+LIBRARY()
+
OWNER(g:kikimr)
-
+
SRCS(
parser.cpp
getenv.cpp
string_helpers.cpp
)
-PEERDIR(
+PEERDIR(
library/cpp/grpc/client
ydb/library/yql/public/issue
-)
-
-END()
+)
+
+END()
diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/authenticator.cpp b/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/authenticator.cpp
index 3ff50c3fbd..f8862d4c71 100644
--- a/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/authenticator.cpp
+++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/authenticator.cpp
@@ -1,34 +1,34 @@
-#define INCLUDE_YDB_INTERNAL_H
-#include "authenticator.h"
-
+#define INCLUDE_YDB_INTERNAL_H
+#include "authenticator.h"
+
#include <ydb/public/sdk/cpp/client/resources/ydb_resources.h>
-
-namespace NYdb {
-
-TYdbAuthenticator::TYdbAuthenticator(std::shared_ptr<ICredentialsProvider> credentialsProvider)
- : CredentialsProvider_(std::move(credentialsProvider))
-{}
-
-grpc::Status TYdbAuthenticator::GetMetadata(
- grpc::string_ref,
- grpc::string_ref,
- const grpc::AuthContext&,
- std::multimap<grpc::string, grpc::string>* metadata
-) {
- try {
- metadata->insert(std::make_pair(YDB_AUTH_TICKET_HEADER, CredentialsProvider_->GetAuthInfo()));
- } catch (const yexception& e) {
- return grpc::Status(
- grpc::StatusCode::UNAUTHENTICATED,
- "Can't get Authentication info from CredentialsProvider",
- e.what()
- );
- }
- return grpc::Status::OK;
-}
-
-bool TYdbAuthenticator::IsBlocking() const {
- return false;
-}
-
-} // namespace NYdb
+
+namespace NYdb {
+
+TYdbAuthenticator::TYdbAuthenticator(std::shared_ptr<ICredentialsProvider> credentialsProvider)
+ : CredentialsProvider_(std::move(credentialsProvider))
+{}
+
+grpc::Status TYdbAuthenticator::GetMetadata(
+ grpc::string_ref,
+ grpc::string_ref,
+ const grpc::AuthContext&,
+ std::multimap<grpc::string, grpc::string>* metadata
+) {
+ try {
+ metadata->insert(std::make_pair(YDB_AUTH_TICKET_HEADER, CredentialsProvider_->GetAuthInfo()));
+ } catch (const yexception& e) {
+ return grpc::Status(
+ grpc::StatusCode::UNAUTHENTICATED,
+ "Can't get Authentication info from CredentialsProvider",
+ e.what()
+ );
+ }
+ return grpc::Status::OK;
+}
+
+bool TYdbAuthenticator::IsBlocking() const {
+ return false;
+}
+
+} // namespace NYdb
diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/authenticator.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/authenticator.h
index 7744d6f9cc..e5e21bbb04 100644
--- a/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/authenticator.h
+++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/authenticator.h
@@ -1,28 +1,28 @@
-#pragma once
-
+#pragma once
+
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/internal_header.h>
-
+
#include <ydb/public/sdk/cpp/client/ydb_types/credentials/credentials.h>
-
-#include <library/cpp/grpc/client/grpc_client_low.h>
-
-namespace NYdb {
-
-class TYdbAuthenticator : public grpc::MetadataCredentialsPlugin {
-public:
- TYdbAuthenticator(std::shared_ptr<ICredentialsProvider> credentialsProvider);
-
- grpc::Status GetMetadata(
- grpc::string_ref,
- grpc::string_ref,
- const grpc::AuthContext&,
- std::multimap<grpc::string, grpc::string>* metadata
- ) override;
-
- bool IsBlocking() const override;
-
-private:
- std::shared_ptr<ICredentialsProvider> CredentialsProvider_;
-};
-
-} // namespace NYdb
+
+#include <library/cpp/grpc/client/grpc_client_low.h>
+
+namespace NYdb {
+
+class TYdbAuthenticator : public grpc::MetadataCredentialsPlugin {
+public:
+ TYdbAuthenticator(std::shared_ptr<ICredentialsProvider> credentialsProvider);
+
+ grpc::Status GetMetadata(
+ grpc::string_ref,
+ grpc::string_ref,
+ const grpc::AuthContext&,
+ std::multimap<grpc::string, grpc::string>* metadata
+ ) override;
+
+ bool IsBlocking() const override;
+
+private:
+ std::shared_ptr<ICredentialsProvider> CredentialsProvider_;
+};
+
+} // namespace NYdb
diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/endpoint_pool.cpp b/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/endpoint_pool.cpp
index 62253ced79..7b7947578d 100644
--- a/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/endpoint_pool.cpp
+++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/endpoint_pool.cpp
@@ -1,14 +1,14 @@
-#define INCLUDE_YDB_INTERNAL_H
-#include "endpoint_pool.h"
-
-namespace NYdb {
-TEndpointPool::TEndpointPool(TListEndpointsResultProvider&& provider, const IInternalClient* client)
- : Provider_(provider)
- , LastUpdateTime_(TInstant::Zero().MicroSeconds())
- , BalancingSettings_(client->GetBalancingSettings())
-{}
-
-TEndpointPool::~TEndpointPool() {
+#define INCLUDE_YDB_INTERNAL_H
+#include "endpoint_pool.h"
+
+namespace NYdb {
+TEndpointPool::TEndpointPool(TListEndpointsResultProvider&& provider, const IInternalClient* client)
+ : Provider_(provider)
+ , LastUpdateTime_(TInstant::Zero().MicroSeconds())
+ , BalancingSettings_(client->GetBalancingSettings())
+{}
+
+TEndpointPool::~TEndpointPool() {
try {
NThreading::TFuture<TEndpointUpdateResult> future;
{
@@ -16,43 +16,43 @@ TEndpointPool::~TEndpointPool() {
if (DiscoveryPromise_.Initialized()) {
future = DiscoveryPromise_.GetFuture();
}
- }
+ }
if (future.Initialized()) {
future.Wait();
}
} catch (...) {
Y_FAIL("Unexpected exception from endpoint pool dtor");
- }
-}
-
-std::pair<NThreading::TFuture<TEndpointUpdateResult>, bool> TEndpointPool::UpdateAsync() {
- NThreading::TFuture<TEndpointUpdateResult> future;
+ }
+}
+
+std::pair<NThreading::TFuture<TEndpointUpdateResult>, bool> TEndpointPool::UpdateAsync() {
+ NThreading::TFuture<TEndpointUpdateResult> future;
{
std::lock_guard guard(Mutex_);
- if (DiscoveryPromise_.Initialized()) {
- return {DiscoveryPromise_.GetFuture(), false};
- } else {
- DiscoveryPromise_ = NThreading::NewPromise<TEndpointUpdateResult>();
- future = DiscoveryPromise_.GetFuture();
- }
- }
- auto handler = [this](const TAsyncListEndpointsResult& future) {
- TListEndpointsResult result = future.GetValue();
+ if (DiscoveryPromise_.Initialized()) {
+ return {DiscoveryPromise_.GetFuture(), false};
+ } else {
+ DiscoveryPromise_ = NThreading::NewPromise<TEndpointUpdateResult>();
+ future = DiscoveryPromise_.GetFuture();
+ }
+ }
+ auto handler = [this](const TAsyncListEndpointsResult& future) {
+ TListEndpointsResult result = future.GetValue();
std::vector<TStringType> removed;
- if (result.DiscoveryStatus.Status == EStatus::SUCCESS) {
+ if (result.DiscoveryStatus.Status == EStatus::SUCCESS) {
std::vector<TEndpointRecord> records;
- // Is used to convert float to integer load factor
- // same integer values will be selected randomly.
- const float multiplicator = 10.0;
- const auto& preferedLocation = GetPreferedLocation(result.Result.self_location());
- for (const auto& endpoint : result.Result.endpoints()) {
- i32 loadFactor = (i32)(multiplicator * Min(LoadMax, Max(LoadMin, endpoint.load_factor())));
- if (BalancingSettings_.Policy != EBalancingPolicy::UseAllNodes) {
- if (endpoint.location() != preferedLocation) {
- // Location missmatch, shift this endpoint
- loadFactor += GetLocalityShift();
- }
- }
+ // Is used to convert float to integer load factor
+ // same integer values will be selected randomly.
+ const float multiplicator = 10.0;
+ const auto& preferedLocation = GetPreferedLocation(result.Result.self_location());
+ for (const auto& endpoint : result.Result.endpoints()) {
+ i32 loadFactor = (i32)(multiplicator * Min(LoadMax, Max(LoadMin, endpoint.load_factor())));
+ if (BalancingSettings_.Policy != EBalancingPolicy::UseAllNodes) {
+ if (endpoint.location() != preferedLocation) {
+ // Location missmatch, shift this endpoint
+ loadFactor += GetLocalityShift();
+ }
+ }
TStringType sslTargetNameOverride = endpoint.ssl_target_name_override();
auto getIpSslTargetNameOverride = [&]() -> TStringType {
@@ -105,82 +105,82 @@ std::pair<NThreading::TFuture<TEndpointUpdateResult>, bool> TEndpointPool::Updat
<< endpoint.port();
records.emplace_back(std::move(endpointString), loadFactor, std::move(sslTargetNameOverride));
}
- }
- LastUpdateTime_ = TInstant::Now().MicroSeconds();
- removed = Elector_.SetNewState(std::move(records));
- }
- NThreading::TPromise<TEndpointUpdateResult> promise;
+ }
+ LastUpdateTime_ = TInstant::Now().MicroSeconds();
+ removed = Elector_.SetNewState(std::move(records));
+ }
+ NThreading::TPromise<TEndpointUpdateResult> promise;
{
std::lock_guard guard(Mutex_);
- DiscoveryPromise_.Swap(promise);
- }
- promise.SetValue({std::move(removed), result.DiscoveryStatus});
- };
-
- Provider_().Subscribe(handler);
- return {future, true};
-}
-
+ DiscoveryPromise_.Swap(promise);
+ }
+ promise.SetValue({std::move(removed), result.DiscoveryStatus});
+ };
+
+ Provider_().Subscribe(handler);
+ return {future, true};
+}
+
TEndpointRecord TEndpointPool::GetEndpoint(const TStringType& preferredEndpoint) const {
- return Elector_.GetEndpoint(preferredEndpoint);
-}
-
-TDuration TEndpointPool::TimeSinceLastUpdate() const {
- auto now = TInstant::Now().MicroSeconds();
+ return Elector_.GetEndpoint(preferredEndpoint);
+}
+
+TDuration TEndpointPool::TimeSinceLastUpdate() const {
+ auto now = TInstant::Now().MicroSeconds();
return TDuration::MicroSeconds(now - LastUpdateTime_.load());
-}
-
+}
+
void TEndpointPool::BanEndpoint(const TStringType& endpoint) {
- Elector_.PessimizeEndpoint(endpoint);
-}
-
-int TEndpointPool::GetPessimizationRatio() {
- return Elector_.GetPessimizationRatio();
-}
-
+ Elector_.PessimizeEndpoint(endpoint);
+}
+
+int TEndpointPool::GetPessimizationRatio() {
+ return Elector_.GetPessimizationRatio();
+}
+
bool TEndpointPool::LinkObjToEndpoint(const TStringType& endpoint, TEndpointObj* obj, const void* tag) {
- return Elector_.LinkObjToEndpoint(endpoint, obj, tag);
-}
-
-void TEndpointPool::ForEachEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const {
- return Elector_.ForEachEndpoint(cb, 0, Max<i32>(), tag);
-}
-
-void TEndpointPool::ForEachLocalEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const {
- return Elector_.ForEachEndpoint(cb, 0, GetLocalityShift() - 1, tag);
-}
-
-void TEndpointPool::ForEachForeignEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const {
- return Elector_.ForEachEndpoint(cb, GetLocalityShift(), Max<i32>() - 1, tag);
-}
-
-EBalancingPolicy TEndpointPool::GetBalancingPolicy() const {
- return BalancingSettings_.Policy;
-}
-
-void TEndpointPool::SetStatCollector(NSdkStats::TStatCollector& statCollector) {
- if (!statCollector.IsCollecting())
- return;
- Elector_.SetStatCollector(statCollector.GetEndpointElectorStatCollector());
- StatCollector_ = &statCollector;
-}
-
-constexpr i32 TEndpointPool::GetLocalityShift() {
- return LoadMax * Multiplicator;
-}
-
+ return Elector_.LinkObjToEndpoint(endpoint, obj, tag);
+}
+
+void TEndpointPool::ForEachEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const {
+ return Elector_.ForEachEndpoint(cb, 0, Max<i32>(), tag);
+}
+
+void TEndpointPool::ForEachLocalEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const {
+ return Elector_.ForEachEndpoint(cb, 0, GetLocalityShift() - 1, tag);
+}
+
+void TEndpointPool::ForEachForeignEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const {
+ return Elector_.ForEachEndpoint(cb, GetLocalityShift(), Max<i32>() - 1, tag);
+}
+
+EBalancingPolicy TEndpointPool::GetBalancingPolicy() const {
+ return BalancingSettings_.Policy;
+}
+
+void TEndpointPool::SetStatCollector(NSdkStats::TStatCollector& statCollector) {
+ if (!statCollector.IsCollecting())
+ return;
+ Elector_.SetStatCollector(statCollector.GetEndpointElectorStatCollector());
+ StatCollector_ = &statCollector;
+}
+
+constexpr i32 TEndpointPool::GetLocalityShift() {
+ return LoadMax * Multiplicator;
+}
+
TStringType TEndpointPool::GetPreferedLocation(const TStringType& selfLocation) {
- switch (BalancingSettings_.Policy) {
- case EBalancingPolicy::UseAllNodes:
- return {};
- case EBalancingPolicy::UsePreferableLocation:
- if (BalancingSettings_.PolicyParams.empty()) {
- return selfLocation;
- } else {
- return BalancingSettings_.PolicyParams;
- }
- }
- return {};
-}
-
-} // namespace NYdb
+ switch (BalancingSettings_.Policy) {
+ case EBalancingPolicy::UseAllNodes:
+ return {};
+ case EBalancingPolicy::UsePreferableLocation:
+ if (BalancingSettings_.PolicyParams.empty()) {
+ return selfLocation;
+ } else {
+ return BalancingSettings_.PolicyParams;
+ }
+ }
+ return {};
+}
+
+} // namespace NYdb
diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/endpoint_pool.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/endpoint_pool.h
index ad519257b1..2c81ee79ef 100644
--- a/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/endpoint_pool.h
+++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/endpoint_pool.h
@@ -1,68 +1,68 @@
-#pragma once
-
+#pragma once
+
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/internal_header.h>
-
+
#include <ydb/public/api/protos/ydb_discovery.pb.h>
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/internal_client/client.h>
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/plain_status/status.h>
#include <ydb/public/sdk/cpp/client/impl/ydb_endpoints/endpoints.h>
-
-#include <library/cpp/threading/future/future.h>
-
+
+#include <library/cpp/threading/future/future.h>
+
#include <mutex>
-namespace NYdb {
-
-struct TListEndpointsResult {
- Ydb::Discovery::ListEndpointsResult Result;
- TPlainStatus DiscoveryStatus;
-};
-
-using TAsyncListEndpointsResult = NThreading::TFuture<TListEndpointsResult>;
-using TListEndpointsResultProvider = std::function<TAsyncListEndpointsResult()>;
-
-struct TEndpointUpdateResult {
+namespace NYdb {
+
+struct TListEndpointsResult {
+ Ydb::Discovery::ListEndpointsResult Result;
+ TPlainStatus DiscoveryStatus;
+};
+
+using TAsyncListEndpointsResult = NThreading::TFuture<TListEndpointsResult>;
+using TListEndpointsResultProvider = std::function<TAsyncListEndpointsResult()>;
+
+struct TEndpointUpdateResult {
std::vector<TStringType> Removed;
- TPlainStatus DiscoveryStatus;
-};
-
-class TEndpointPool {
-public:
- TEndpointPool(TListEndpointsResultProvider&& provider, const IInternalClient* client);
- ~TEndpointPool();
- std::pair<NThreading::TFuture<TEndpointUpdateResult>, bool> UpdateAsync();
+ TPlainStatus DiscoveryStatus;
+};
+
+class TEndpointPool {
+public:
+ TEndpointPool(TListEndpointsResultProvider&& provider, const IInternalClient* client);
+ ~TEndpointPool();
+ std::pair<NThreading::TFuture<TEndpointUpdateResult>, bool> UpdateAsync();
TEndpointRecord GetEndpoint(const TStringType& preferredEndpoint) const;
- TDuration TimeSinceLastUpdate() const;
+ TDuration TimeSinceLastUpdate() const;
void BanEndpoint(const TStringType& endpoint);
- int GetPessimizationRatio();
+ int GetPessimizationRatio();
bool LinkObjToEndpoint(const TStringType& endpoint, TEndpointObj* obj, const void* tag);
- void ForEachEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const;
- void ForEachLocalEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const;
- void ForEachForeignEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const;
- EBalancingPolicy GetBalancingPolicy() const;
- // TODO: Remove this mess
- void SetStatCollector(NSdkStats::TStatCollector& statCollector);
- static constexpr i32 GetLocalityShift();
-
-private:
+ void ForEachEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const;
+ void ForEachLocalEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const;
+ void ForEachForeignEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const;
+ EBalancingPolicy GetBalancingPolicy() const;
+ // TODO: Remove this mess
+ void SetStatCollector(NSdkStats::TStatCollector& statCollector);
+ static constexpr i32 GetLocalityShift();
+
+private:
TStringType GetPreferedLocation(const TStringType& selfLocation);
-
-private:
- TListEndpointsResultProvider Provider_;
+
+private:
+ TListEndpointsResultProvider Provider_;
std::mutex Mutex_;
- TEndpointElectorSafe Elector_;
- NThreading::TPromise<TEndpointUpdateResult> DiscoveryPromise_;
+ TEndpointElectorSafe Elector_;
+ NThreading::TPromise<TEndpointUpdateResult> DiscoveryPromise_;
std::atomic_uint64_t LastUpdateTime_;
- const TBalancingSettings BalancingSettings_;
-
- NSdkStats::TStatCollector* StatCollector_ = nullptr;
-
- // Max, min load factor returned by discovery service
- static constexpr float LoadMax = 100.0;
- static constexpr float LoadMin = -100.0;
- // Is used to convert float to integer load factor
- // same integer values will be selected randomly.
- static constexpr float Multiplicator = 10.0;
-};
-
-} // namespace NYdb
+ const TBalancingSettings BalancingSettings_;
+
+ NSdkStats::TStatCollector* StatCollector_ = nullptr;
+
+ // Max, min load factor returned by discovery service
+ static constexpr float LoadMax = 100.0;
+ static constexpr float LoadMin = -100.0;
+ // Is used to convert float to integer load factor
+ // same integer values will be selected randomly.
+ static constexpr float Multiplicator = 10.0;
+};
+
+} // namespace NYdb
diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/state.cpp b/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/state.cpp
index 70c1842dbf..7364c34524 100644
--- a/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/state.cpp
+++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/state.cpp
@@ -1,201 +1,201 @@
-#define INCLUDE_YDB_INTERNAL_H
-#include "state.h"
-
+#define INCLUDE_YDB_INTERNAL_H
+#include "state.h"
+
#include <ydb/public/sdk/cpp/client/ydb_types/credentials/credentials.h>
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/logger/log.h>
-
-#include <library/cpp/string_utils/quote/quote.h>
-
+
+#include <library/cpp/string_utils/quote/quote.h>
+
#include <thread>
#include <unordered_map>
-namespace NYdb {
-
-constexpr int PESSIMIZATION_DISCOVERY_THRESHOLD = 50; // percent of endpoints pessimized by transport error to start recheck
-constexpr TDuration ENDPOINT_UPDATE_PERIOD = TDuration::Minutes(1); // period to perform endpoints update in "normal" case
-constexpr TDuration DISCOVERY_RECHECK_PERIOD = TDuration::Seconds(5); // period to run periodic discovery task
-
-TDbDriverState::TDbDriverState(
+namespace NYdb {
+
+constexpr int PESSIMIZATION_DISCOVERY_THRESHOLD = 50; // percent of endpoints pessimized by transport error to start recheck
+constexpr TDuration ENDPOINT_UPDATE_PERIOD = TDuration::Minutes(1); // period to perform endpoints update in "normal" case
+constexpr TDuration DISCOVERY_RECHECK_PERIOD = TDuration::Seconds(5); // period to run periodic discovery task
+
+TDbDriverState::TDbDriverState(
const TStringType& database,
const TStringType& discoveryEndpoint,
- EDiscoveryMode discoveryMode,
+ EDiscoveryMode discoveryMode,
bool enableSsl,
- IInternalClient* client
-)
- : Database(database)
- , DiscoveryEndpoint(discoveryEndpoint)
- , DiscoveryMode(discoveryMode)
+ IInternalClient* client
+)
+ : Database(database)
+ , DiscoveryEndpoint(discoveryEndpoint)
+ , DiscoveryMode(discoveryMode)
, EnableSsl(enableSsl)
- , Client(client)
- , EndpointPool([this, client]() mutable {
- // this callback will be called just after shared_ptr initialization
- // so this call is safe
- auto self = shared_from_this();
- return client->GetEndpoints(self);
- }, client)
- , StatCollector(database, client->GetMetricRegistry())
- , Log(Client->GetLog())
-{
+ , Client(client)
+ , EndpointPool([this, client]() mutable {
+ // this callback will be called just after shared_ptr initialization
+ // so this call is safe
+ auto self = shared_from_this();
+ return client->GetEndpoints(self);
+ }, client)
+ , StatCollector(database, client->GetMetricRegistry())
+ , Log(Client->GetLog())
+{
EndpointPool.SetStatCollector(StatCollector);
Log.SetFormatter(GetPrefixLogFormatter(GetDatabaseLogPrefix(Database)));
}
void TDbDriverState::SetCredentialsProvider(std::shared_ptr<ICredentialsProvider> credentialsProvider) {
CredentialsProvider = std::move(credentialsProvider);
-#ifndef YDB_GRPC_UNSECURE_AUTH
- CallCredentials = grpc::MetadataCredentialsFromPlugin(
- std::unique_ptr<grpc::MetadataCredentialsPlugin>(new TYdbAuthenticator(CredentialsProvider)));
-#endif
-}
-
-void TDbDriverState::AddCb(TCb&& cb, ENotifyType type) {
+#ifndef YDB_GRPC_UNSECURE_AUTH
+ CallCredentials = grpc::MetadataCredentialsFromPlugin(
+ std::unique_ptr<grpc::MetadataCredentialsPlugin>(new TYdbAuthenticator(CredentialsProvider)));
+#endif
+}
+
+void TDbDriverState::AddCb(TCb&& cb, ENotifyType type) {
std::lock_guard lock(NotifyCbsLock);
NotifyCbs[static_cast<size_t>(type)].emplace_back(std::move(cb));
-}
-
-void TDbDriverState::ForEachEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const {
- EndpointPool.ForEachEndpoint(cb, tag);
-}
-
-void TDbDriverState::ForEachLocalEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const {
- EndpointPool.ForEachLocalEndpoint(cb, tag);
-}
-
-void TDbDriverState::ForEachForeignEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const {
- EndpointPool.ForEachForeignEndpoint(cb, tag);
-}
-
-EBalancingPolicy TDbDriverState::GetBalancingPolicy() const {
- return EndpointPool.GetBalancingPolicy();
-}
-
+}
+
+void TDbDriverState::ForEachEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const {
+ EndpointPool.ForEachEndpoint(cb, tag);
+}
+
+void TDbDriverState::ForEachLocalEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const {
+ EndpointPool.ForEachLocalEndpoint(cb, tag);
+}
+
+void TDbDriverState::ForEachForeignEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const {
+ EndpointPool.ForEachForeignEndpoint(cb, tag);
+}
+
+EBalancingPolicy TDbDriverState::GetBalancingPolicy() const {
+ return EndpointPool.GetBalancingPolicy();
+}
+
TStringType TDbDriverState::GetEndpoint() const {
return EndpointPool.GetEndpoint(TStringType()).Endpoint;
-}
-
-TPeriodicCb CreatePeriodicDiscoveryTask(TDbDriverState::TPtr driverState) {
- auto weak = std::weak_ptr<TDbDriverState>(driverState);
- return [weak](NYql::TIssues&&, EStatus status) {
- if (status != EStatus::SUCCESS) {
- return false;
- }
-
- TDbDriverState::TPtr strong = weak.lock();
- if (!strong) {
- return false;
- } else {
-
- bool pessThreshold = strong->EndpointPool.GetPessimizationRatio() > PESSIMIZATION_DISCOVERY_THRESHOLD;
- bool expiration = strong->EndpointPool.TimeSinceLastUpdate() > ENDPOINT_UPDATE_PERIOD;
-
- if (pessThreshold) {
- strong->StatCollector.IncDiscoveryDuePessimization();
- }
- if (expiration) {
- strong->StatCollector.IncDiscoveryDueExpiration();
- }
-
- if (pessThreshold || expiration) {
- auto asyncResult = strong->EndpointPool.UpdateAsync();
- // true - we were first who run UpdateAsync
- if (asyncResult.second) {
- auto cb = [strong](const NThreading::TFuture<TEndpointUpdateResult>& future) {
- const auto& updateResult = future.GetValue();
-#ifndef YDB_GRPC_BYPASS_CHANNEL_POOL
- strong->Client->DeleteChannels(updateResult.Removed);
-#endif
- if (strong->DiscoveryMode == EDiscoveryMode::Sync) {
+}
+
+TPeriodicCb CreatePeriodicDiscoveryTask(TDbDriverState::TPtr driverState) {
+ auto weak = std::weak_ptr<TDbDriverState>(driverState);
+ return [weak](NYql::TIssues&&, EStatus status) {
+ if (status != EStatus::SUCCESS) {
+ return false;
+ }
+
+ TDbDriverState::TPtr strong = weak.lock();
+ if (!strong) {
+ return false;
+ } else {
+
+ bool pessThreshold = strong->EndpointPool.GetPessimizationRatio() > PESSIMIZATION_DISCOVERY_THRESHOLD;
+ bool expiration = strong->EndpointPool.TimeSinceLastUpdate() > ENDPOINT_UPDATE_PERIOD;
+
+ if (pessThreshold) {
+ strong->StatCollector.IncDiscoveryDuePessimization();
+ }
+ if (expiration) {
+ strong->StatCollector.IncDiscoveryDueExpiration();
+ }
+
+ if (pessThreshold || expiration) {
+ auto asyncResult = strong->EndpointPool.UpdateAsync();
+ // true - we were first who run UpdateAsync
+ if (asyncResult.second) {
+ auto cb = [strong](const NThreading::TFuture<TEndpointUpdateResult>& future) {
+ const auto& updateResult = future.GetValue();
+#ifndef YDB_GRPC_BYPASS_CHANNEL_POOL
+ strong->Client->DeleteChannels(updateResult.Removed);
+#endif
+ if (strong->DiscoveryMode == EDiscoveryMode::Sync) {
std::unique_lock guard(strong->LastDiscoveryStatusRWLock);
- strong->LastDiscoveryStatus = updateResult.DiscoveryStatus;
- }
- };
- asyncResult.first.Subscribe(std::move(cb));
- }
- }
- }
- return true;
- };
-}
-
-TDbDriverStateTracker::TDbDriverStateTracker(IInternalClient* client)
- : DiscoveryClient_(client)
-{}
-
-TDbDriverStatePtr TDbDriverStateTracker::GetDriverState(
+ strong->LastDiscoveryStatus = updateResult.DiscoveryStatus;
+ }
+ };
+ asyncResult.first.Subscribe(std::move(cb));
+ }
+ }
+ }
+ return true;
+ };
+}
+
+TDbDriverStateTracker::TDbDriverStateTracker(IInternalClient* client)
+ : DiscoveryClient_(client)
+{}
+
+TDbDriverStatePtr TDbDriverStateTracker::GetDriverState(
TStringType database,
TStringType discoveryEndpoint,
- EDiscoveryMode discoveryMode,
+ EDiscoveryMode discoveryMode,
bool enableSsl,
- std::shared_ptr<ICredentialsProviderFactory> credentialsProviderFactory
-) {
+ std::shared_ptr<ICredentialsProviderFactory> credentialsProviderFactory
+) {
TStringType clientIdentity;
- if (credentialsProviderFactory) {
- clientIdentity = credentialsProviderFactory->GetClientIdentity();
- }
- Quote(database);
+ if (credentialsProviderFactory) {
+ clientIdentity = credentialsProviderFactory->GetClientIdentity();
+ }
+ Quote(database);
const TStateKey key{database, discoveryEndpoint, clientIdentity, discoveryMode, enableSsl};
- {
+ {
std::shared_lock lock(Lock_);
auto state = States_.find(key);
if (state != States_.end()) {
auto strong = state->second.lock();
- if (strong) {
- return strong;
- }
- // If we can't promote to shared see bellow
- }
- }
- TDbDriverStatePtr strongState;
- for (;;) {
+ if (strong) {
+ return strong;
+ }
+ // If we can't promote to shared see bellow
+ }
+ }
+ TDbDriverStatePtr strongState;
+ for (;;) {
std::unique_lock lock(Lock_);
- {
+ {
auto state = States_.find(key);
if (state != States_.end()) {
auto strong = state->second.lock();
- if (strong) {
- return strong;
- } else {
- // We could find state record, but couldn't promote weak to shared
- // this means weak ptr already expired but dtor hasn't been
- // called yet. Likely other thread now is waiting on mutex to
- // remove expired record from hashmap. So give him chance
- // to do it after that we will be able to create new state
+ if (strong) {
+ return strong;
+ } else {
+ // We could find state record, but couldn't promote weak to shared
+ // this means weak ptr already expired but dtor hasn't been
+ // called yet. Likely other thread now is waiting on mutex to
+ // remove expired record from hashmap. So give him chance
+ // to do it after that we will be able to create new state
lock.unlock();
std::this_thread::yield();
- continue;
- }
- }
- }
- {
- auto deleter = [this, key](TDbDriverState* p) {
- {
+ continue;
+ }
+ }
+ }
+ {
+ auto deleter = [this, key](TDbDriverState* p) {
+ {
std::unique_lock lock(Lock_);
- States_.erase(key);
- }
- delete p;
- };
- strongState = std::shared_ptr<TDbDriverState>(
- new TDbDriverState(
- database,
- discoveryEndpoint,
- discoveryMode,
+ States_.erase(key);
+ }
+ delete p;
+ };
+ strongState = std::shared_ptr<TDbDriverState>(
+ new TDbDriverState(
+ database,
+ discoveryEndpoint,
+ discoveryMode,
enableSsl,
- DiscoveryClient_),
- deleter);
+ DiscoveryClient_),
+ deleter);
strongState->SetCredentialsProvider(
credentialsProviderFactory
? credentialsProviderFactory->CreateProvider(strongState)
: CreateInsecureCredentialsProviderFactory()->CreateProvider(strongState));
- DiscoveryClient_->AddPeriodicTask(CreatePeriodicDiscoveryTask(strongState), DISCOVERY_RECHECK_PERIOD);
- Y_VERIFY(States_.emplace(key, strongState).second);
- break;
- }
- }
- auto updateResult = strongState->EndpointPool.UpdateAsync();
+ DiscoveryClient_->AddPeriodicTask(CreatePeriodicDiscoveryTask(strongState), DISCOVERY_RECHECK_PERIOD);
+ Y_VERIFY(States_.emplace(key, strongState).second);
+ break;
+ }
+ }
+ auto updateResult = strongState->EndpointPool.UpdateAsync();
- if (strongState->DiscoveryMode == EDiscoveryMode::Sync) {
+ if (strongState->DiscoveryMode == EDiscoveryMode::Sync) {
const auto& discoveryStatus = updateResult.first.GetValueSync().DiscoveryStatus;
// Almost always true, except the situation when the current thread was
// preempted just before UpdateAsync call and other one get
@@ -204,61 +204,61 @@ TDbDriverStatePtr TDbDriverStateTracker::GetDriverState(
std::unique_lock guard(strongState->LastDiscoveryStatusRWLock);
strongState->LastDiscoveryStatus = discoveryStatus;
}
- }
+ }
+
+ return strongState;
+}
- return strongState;
-}
-
void TDbDriverState::AddPeriodicTask(TPeriodicCb&& cb, TDuration period) {
Client->AddPeriodicTask(std::move(cb), period);
}
-NThreading::TFuture<void> TDbDriverStateTracker::SendNotification(
+NThreading::TFuture<void> TDbDriverStateTracker::SendNotification(
TDbDriverState::ENotifyType type
-) {
+) {
std::vector<std::weak_ptr<TDbDriverState>> states;
- {
+ {
std::shared_lock lock(Lock_);
- states.reserve(States_.size());
- for (auto& weak : States_) {
- states.push_back(weak.second);
- }
- }
+ states.reserve(States_.size());
+ for (auto& weak : States_) {
+ states.push_back(weak.second);
+ }
+ }
std::vector<NThreading::TFuture<void>> results;
- for (auto& state : states) {
- auto strong = state.lock();
- if (strong) {
+ for (auto& state : states) {
+ auto strong = state.lock();
+ if (strong) {
std::lock_guard lock(strong->NotifyCbsLock);
for (auto& cb : strong->NotifyCbs[static_cast<size_t>(type)]) {
if (cb) {
auto future = cb();
if (!future.HasException()) {
results.push_back(future);
- }
+ }
//TODO: Add loger
- }
- }
- }
- }
- return NThreading::WaitExceptionOrAll(results);
-}
-
-void TDbDriverStateTracker::SetMetricRegistry(NMonitoring::TMetricRegistry *sensorsRegistry) {
+ }
+ }
+ }
+ }
+ return NThreading::WaitExceptionOrAll(results);
+}
+
+void TDbDriverStateTracker::SetMetricRegistry(NMonitoring::TMetricRegistry *sensorsRegistry) {
std::vector<std::weak_ptr<TDbDriverState>> states;
- {
+ {
std::shared_lock lock(Lock_);
- states.reserve(States_.size());
- for (auto& weak : States_) {
- states.push_back(weak.second);
- }
- }
-
- for (auto& weak : states) {
- if (auto strong = weak.lock()) {
- strong->StatCollector.SetMetricRegistry(sensorsRegistry);
- strong->EndpointPool.SetStatCollector(strong->StatCollector);
- }
- }
-}
-
-} // namespace NYdb
+ states.reserve(States_.size());
+ for (auto& weak : States_) {
+ states.push_back(weak.second);
+ }
+ }
+
+ for (auto& weak : states) {
+ if (auto strong = weak.lock()) {
+ strong->StatCollector.SetMetricRegistry(sensorsRegistry);
+ strong->EndpointPool.SetStatCollector(strong->StatCollector);
+ }
+ }
+}
+
+} // namespace NYdb
diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/state.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/state.h
index 6df22f2d86..e0ce63e4b8 100644
--- a/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/state.h
+++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/state.h
@@ -1,71 +1,71 @@
-#pragma once
-
-#include "endpoint_pool.h"
-
+#pragma once
+
+#include "endpoint_pool.h"
+
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/internal_header.h>
-
+
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/internal_client/client.h>
#include <ydb/public/sdk/cpp/client/ydb_types/core_facility/core_facility.h>
-
-namespace NYdb {
-class ICredentialsProvider;
-class ICredentialsProviderFactory;
-
-// Represents state of driver for one particular database
+namespace NYdb {
+
+class ICredentialsProvider;
+class ICredentialsProviderFactory;
+
+// Represents state of driver for one particular database
class TDbDriverState
: public std::enable_shared_from_this<TDbDriverState>
, public ICoreFacility
{
public:
- enum class ENotifyType : size_t {
- STOP = 0,
- COUNT = 1 // types count
- };
-
+ enum class ENotifyType : size_t {
+ STOP = 0,
+ COUNT = 1 // types count
+ };
+
using TCb = std::function<NThreading::TFuture<void>()>;
- using TPtr = std::shared_ptr<TDbDriverState>;
-
- TDbDriverState(
+ using TPtr = std::shared_ptr<TDbDriverState>;
+
+ TDbDriverState(
const TStringType& database,
const TStringType& discoveryEndpoint,
- EDiscoveryMode discoveryMode,
+ EDiscoveryMode discoveryMode,
bool enableSsl,
- IInternalClient* client
- );
-
+ IInternalClient* client
+ );
+
void AddPeriodicTask(TPeriodicCb&& cb, TDuration period) override;
- void AddCb(TCb&& cb, ENotifyType type);
- void ForEachEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const;
- void ForEachLocalEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const;
- void ForEachForeignEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const;
- EBalancingPolicy GetBalancingPolicy() const;
+ void AddCb(TCb&& cb, ENotifyType type);
+ void ForEachEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const;
+ void ForEachLocalEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const;
+ void ForEachForeignEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const;
+ EBalancingPolicy GetBalancingPolicy() const;
TStringType GetEndpoint() const;
void SetCredentialsProvider(std::shared_ptr<ICredentialsProvider> credentialsProvider);
-
+
const TStringType Database;
const TStringType DiscoveryEndpoint;
- const EDiscoveryMode DiscoveryMode;
+ const EDiscoveryMode DiscoveryMode;
const bool EnableSsl;
- std::shared_ptr<ICredentialsProvider> CredentialsProvider;
- IInternalClient* Client;
- TEndpointPool EndpointPool;
- // StopCb allow client to subscribe for notifications from lower layer
+ std::shared_ptr<ICredentialsProvider> CredentialsProvider;
+ IInternalClient* Client;
+ TEndpointPool EndpointPool;
+ // StopCb allow client to subscribe for notifications from lower layer
std::mutex NotifyCbsLock;
std::array<std::vector<TCb>, static_cast<size_t>(ENotifyType::COUNT)> NotifyCbs;
-#ifndef YDB_GRPC_UNSECURE_AUTH
- std::shared_ptr<grpc::CallCredentials> CallCredentials;
-#endif
- // Status of last discovery call, used in sync mode, coresponding mutex
+#ifndef YDB_GRPC_UNSECURE_AUTH
+ std::shared_ptr<grpc::CallCredentials> CallCredentials;
+#endif
+ // Status of last discovery call, used in sync mode, coresponding mutex
std::shared_mutex LastDiscoveryStatusRWLock;
- TPlainStatus LastDiscoveryStatus;
- NSdkStats::TStatCollector StatCollector;
- TLog Log;
-};
-
-// Tracker allows to get driver state by database and credentials
-class TDbDriverStateTracker {
+ TPlainStatus LastDiscoveryStatus;
+ NSdkStats::TStatCollector StatCollector;
+ TLog Log;
+};
+
+// Tracker allows to get driver state by database and credentials
+class TDbDriverStateTracker {
using TStateKey = std::tuple<TStringType, TStringType, TStringType, EDiscoveryMode, bool>;
struct TStateKeyHash {
size_t operator()(const TStateKey& k) const noexcept {
@@ -77,24 +77,24 @@ class TDbDriverStateTracker {
return (h0 ^ h1 ^ h2 ^ h3);
}
};
-public:
- TDbDriverStateTracker(IInternalClient* client);
- TDbDriverState::TPtr GetDriverState(
+public:
+ TDbDriverStateTracker(IInternalClient* client);
+ TDbDriverState::TPtr GetDriverState(
TStringType database,
TStringType DiscoveryEndpoint,
- EDiscoveryMode discoveryMode,
+ EDiscoveryMode discoveryMode,
bool enableSsl,
- std::shared_ptr<ICredentialsProviderFactory> credentialsProviderFactory
- );
- NThreading::TFuture<void> SendNotification(
+ std::shared_ptr<ICredentialsProviderFactory> credentialsProviderFactory
+ );
+ NThreading::TFuture<void> SendNotification(
TDbDriverState::ENotifyType type);
- void SetMetricRegistry(NMonitoring::TMetricRegistry *sensorsRegistry);
-private:
- IInternalClient* DiscoveryClient_;
+ void SetMetricRegistry(NMonitoring::TMetricRegistry *sensorsRegistry);
+private:
+ IInternalClient* DiscoveryClient_;
std::unordered_map<TStateKey, std::weak_ptr<TDbDriverState>, TStateKeyHash> States_;
std::shared_mutex Lock_;
-};
-
-using TDbDriverStatePtr = TDbDriverState::TPtr;
-
-} // namespace NYdb
+};
+
+using TDbDriverStatePtr = TDbDriverState::TPtr;
+
+} // namespace NYdb
diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/ya.make b/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/ya.make
index eac291f6c8..b9d984f4ba 100644
--- a/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/ya.make
+++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/ya.make
@@ -1,20 +1,20 @@
-LIBRARY()
-
+LIBRARY()
+
OWNER(g:kikimr)
-
-SRCS(
- authenticator.cpp
- endpoint_pool.cpp
- state.cpp
-)
-
-PEERDIR(
- library/cpp/string_utils/quote
- library/cpp/threading/future
+
+SRCS(
+ authenticator.cpp
+ endpoint_pool.cpp
+ state.cpp
+)
+
+PEERDIR(
+ library/cpp/string_utils/quote
+ library/cpp/threading/future
ydb/public/sdk/cpp/client/impl/ydb_endpoints
ydb/public/sdk/cpp/client/impl/ydb_internal/logger
ydb/public/sdk/cpp/client/impl/ydb_internal/plain_status
ydb/public/sdk/cpp/client/ydb_types/credentials
-)
-
-END()
+)
+
+END()
diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/driver/constants.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/driver/constants.h
index 96e5586d37..3aa91d560d 100644
--- a/ydb/public/sdk/cpp/client/impl/ydb_internal/driver/constants.h
+++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/driver/constants.h
@@ -1,13 +1,13 @@
-#pragma once
-
+#pragma once
+
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/internal_header.h>
-
-namespace NYdb {
-
-constexpr ui64 TCP_KEEPALIVE_IDLE = 30; // The time the connection needs to remain idle
- // before TCP starts sending keepalive probes, seconds
-constexpr ui64 TCP_KEEPALIVE_COUNT = 5; // The maximum number of keepalive probes TCP should send before
- // dropping the connection
-constexpr ui64 TCP_KEEPALIVE_INTERVAL = 10; // The time between individual keepalive probes, seconds
-
-} // namespace NYdb
+
+namespace NYdb {
+
+constexpr ui64 TCP_KEEPALIVE_IDLE = 30; // The time the connection needs to remain idle
+ // before TCP starts sending keepalive probes, seconds
+constexpr ui64 TCP_KEEPALIVE_COUNT = 5; // The maximum number of keepalive probes TCP should send before
+ // dropping the connection
+constexpr ui64 TCP_KEEPALIVE_INTERVAL = 10; // The time between individual keepalive probes, seconds
+
+} // namespace NYdb
diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/driver/interface.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/driver/interface.h
index bafeca0eb1..d49b69d6e1 100644
--- a/ydb/public/sdk/cpp/client/impl/ydb_internal/driver/interface.h
+++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/driver/interface.h
@@ -1,12 +1,12 @@
-#pragma once
-
+#pragma once
+
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/internal_header.h>
-
-namespace NYdb {
-
-class TDriver;
-class TGRpcConnectionsImpl;
-
-std::shared_ptr<TGRpcConnectionsImpl> CreateInternalInterface(const TDriver connection);
-
-} // namespace NYdb
+
+namespace NYdb {
+
+class TDriver;
+class TGRpcConnectionsImpl;
+
+std::shared_ptr<TGRpcConnectionsImpl> CreateInternalInterface(const TDriver connection);
+
+} // namespace NYdb
diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/actions.cpp b/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/actions.cpp
index f6596583ad..f2db8c9416 100644
--- a/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/actions.cpp
+++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/actions.cpp
@@ -1,112 +1,112 @@
-#define INCLUDE_YDB_INTERNAL_H
-#include "actions.h"
-#include "grpc_connections.h"
-
+#define INCLUDE_YDB_INTERNAL_H
+#include "actions.h"
+#include "grpc_connections.h"
+
#include <ydb/public/api/grpc/ydb_operation_v1.grpc.pb.h>
-
-namespace NYdb {
-
-constexpr TDuration MAX_DEFERRED_CALL_DELAY = TDuration::Seconds(10); // The max delay between GetOperation calls for one operation
-
-TSimpleCbResult::TSimpleCbResult(
- TSimpleCb&& cb,
- TGRpcConnectionsImpl* connections,
- std::shared_ptr<IQueueClientContext> context)
- : TGenericCbHolder<TSimpleCb>(std::move(cb), connections, std::move(context))
-{ }
-
-void TSimpleCbResult::Process(void*) {
- UserResponseCb_();
- delete this;
-}
-
+
+namespace NYdb {
+
+constexpr TDuration MAX_DEFERRED_CALL_DELAY = TDuration::Seconds(10); // The max delay between GetOperation calls for one operation
+
+TSimpleCbResult::TSimpleCbResult(
+ TSimpleCb&& cb,
+ TGRpcConnectionsImpl* connections,
+ std::shared_ptr<IQueueClientContext> context)
+ : TGenericCbHolder<TSimpleCb>(std::move(cb), connections, std::move(context))
+{ }
+
+void TSimpleCbResult::Process(void*) {
+ UserResponseCb_();
+ delete this;
+}
+
TDeferredAction::TDeferredAction(const TStringType& operationId,
- TDeferredOperationCb&& userCb,
- TGRpcConnectionsImpl* connection,
- std::shared_ptr<IQueueClientContext> context,
- TDuration delay,
- TDbDriverStatePtr dbState,
+ TDeferredOperationCb&& userCb,
+ TGRpcConnectionsImpl* connection,
+ std::shared_ptr<IQueueClientContext> context,
+ TDuration delay,
+ TDbDriverStatePtr dbState,
const TStringType& endpoint)
- : TAlarmActionBase(std::move(userCb), connection, std::move(context))
- , NextDelay_(Min(delay * 2, MAX_DEFERRED_CALL_DELAY))
- , DbDriverState_(dbState)
- , OperationId_(operationId)
- , Endpoint_(endpoint)
-{
- Deadline_ = gpr_time_add(
- gpr_now(GPR_CLOCK_MONOTONIC),
- gpr_time_from_micros(delay.MicroSeconds(), GPR_TIMESPAN));
-}
-
-void TDeferredAction::OnAlarm() {
- Y_VERIFY(Connection_);
-
- Ydb::Operations::GetOperationRequest getOperationRequest;
- getOperationRequest.set_id(OperationId_);
-
- Connection_->RunDeferred<Ydb::Operation::V1::OperationService, Ydb::Operations::GetOperationRequest, Ydb::Operations::GetOperationResponse>(
- std::move(getOperationRequest),
- std::move(UserResponseCb_),
- &Ydb::Operation::V1::OperationService::Stub::AsyncGetOperation,
- DbDriverState_,
- NextDelay_,
- {},
- TDuration::Zero(),
- true,
- Endpoint_,
- std::move(Context_));
-}
-
-void TDeferredAction::OnError() {
- Y_VERIFY(Connection_);
- NGrpc::TGrpcStatus status = {"Deferred timer interrupted", -1, true};
- DbDriverState_->StatCollector.IncDiscoveryFailDueTransportError();
-
- auto resp = new TGRpcErrorResponse<Ydb::Operations::Operation>(
- std::move(status),
- std::move(UserResponseCb_),
- Connection_,
- std::move(Context_),
- Endpoint_);
- Connection_->EnqueueResponse(resp);
-}
-
-TPeriodicAction::TPeriodicAction(
- TPeriodicCb&& userCb,
- TGRpcConnectionsImpl* connection,
- std::shared_ptr<NGrpc::IQueueClientContext> context,
- TDuration period)
- : TAlarmActionBase(std::move(userCb), connection, std::move(context))
- , Period_(period)
-{
- Deadline_ = gpr_time_add(
- gpr_now(GPR_CLOCK_MONOTONIC),
- gpr_time_from_micros(Period_.MicroSeconds(), GPR_TIMESPAN));
-}
-
-void TPeriodicAction::OnAlarm() {
- NYql::TIssues issues;
- if (!UserResponseCb_(std::move(issues), EStatus::SUCCESS)) {
- return;
- }
-
- auto ctx = Connection_->CreateContext();
- if (!ctx)
- return;
- Context_ = ctx;
-
- auto action = MakeIntrusive<TPeriodicAction>(
- std::move(UserResponseCb_),
- Connection_,
- Context_,
- Period_);
- action->Start();
-}
-
-void TPeriodicAction::OnError() {
- NYql::TIssues issues;
- issues.AddIssue(NYql::TIssue("Deferred timer interrupted"));
- UserResponseCb_(std::move(issues), EStatus::CLIENT_INTERNAL_ERROR);
-}
-
-} // namespace NYdb
+ : TAlarmActionBase(std::move(userCb), connection, std::move(context))
+ , NextDelay_(Min(delay * 2, MAX_DEFERRED_CALL_DELAY))
+ , DbDriverState_(dbState)
+ , OperationId_(operationId)
+ , Endpoint_(endpoint)
+{
+ Deadline_ = gpr_time_add(
+ gpr_now(GPR_CLOCK_MONOTONIC),
+ gpr_time_from_micros(delay.MicroSeconds(), GPR_TIMESPAN));
+}
+
+void TDeferredAction::OnAlarm() {
+ Y_VERIFY(Connection_);
+
+ Ydb::Operations::GetOperationRequest getOperationRequest;
+ getOperationRequest.set_id(OperationId_);
+
+ Connection_->RunDeferred<Ydb::Operation::V1::OperationService, Ydb::Operations::GetOperationRequest, Ydb::Operations::GetOperationResponse>(
+ std::move(getOperationRequest),
+ std::move(UserResponseCb_),
+ &Ydb::Operation::V1::OperationService::Stub::AsyncGetOperation,
+ DbDriverState_,
+ NextDelay_,
+ {},
+ TDuration::Zero(),
+ true,
+ Endpoint_,
+ std::move(Context_));
+}
+
+void TDeferredAction::OnError() {
+ Y_VERIFY(Connection_);
+ NGrpc::TGrpcStatus status = {"Deferred timer interrupted", -1, true};
+ DbDriverState_->StatCollector.IncDiscoveryFailDueTransportError();
+
+ auto resp = new TGRpcErrorResponse<Ydb::Operations::Operation>(
+ std::move(status),
+ std::move(UserResponseCb_),
+ Connection_,
+ std::move(Context_),
+ Endpoint_);
+ Connection_->EnqueueResponse(resp);
+}
+
+TPeriodicAction::TPeriodicAction(
+ TPeriodicCb&& userCb,
+ TGRpcConnectionsImpl* connection,
+ std::shared_ptr<NGrpc::IQueueClientContext> context,
+ TDuration period)
+ : TAlarmActionBase(std::move(userCb), connection, std::move(context))
+ , Period_(period)
+{
+ Deadline_ = gpr_time_add(
+ gpr_now(GPR_CLOCK_MONOTONIC),
+ gpr_time_from_micros(Period_.MicroSeconds(), GPR_TIMESPAN));
+}
+
+void TPeriodicAction::OnAlarm() {
+ NYql::TIssues issues;
+ if (!UserResponseCb_(std::move(issues), EStatus::SUCCESS)) {
+ return;
+ }
+
+ auto ctx = Connection_->CreateContext();
+ if (!ctx)
+ return;
+ Context_ = ctx;
+
+ auto action = MakeIntrusive<TPeriodicAction>(
+ std::move(UserResponseCb_),
+ Connection_,
+ Context_,
+ Period_);
+ action->Start();
+}
+
+void TPeriodicAction::OnError() {
+ NYql::TIssues issues;
+ issues.AddIssue(NYql::TIssue("Deferred timer interrupted"));
+ UserResponseCb_(std::move(issues), EStatus::CLIENT_INTERNAL_ERROR);
+}
+
+} // namespace NYdb
diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/actions.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/actions.h
index 7be068bf50..ebd16c5a83 100644
--- a/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/actions.h
+++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/actions.h
@@ -1,232 +1,232 @@
-#pragma once
-
+#pragma once
+
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/internal_header.h>
-
+
#include <ydb/public/api/protos/ydb_operation.pb.h>
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/common/types.h>
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/state.h>
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/plain_status/status.h>
-
-#include <library/cpp/grpc/client/grpc_client_low.h>
-
-#include <util/thread/pool.h>
-
-#include <grpc++/alarm.h>
-
-namespace NYdb {
-
-using NGrpc::IQueueClientContext;
-using NGrpc::IQueueClientEvent;
-
-class TGRpcConnectionsImpl;
-struct TPlainStatus;
-
-
-template<typename TResponse>
-using TResponseCb = std::function<void(TResponse*, TPlainStatus status)>;
-using TDeferredOperationCb = std::function<void(Ydb::Operations::Operation*, TPlainStatus status)>;
-
-template<typename TCb>
-class TGenericCbHolder {
-protected:
- TGenericCbHolder(
- TCb&& userCb,
- TGRpcConnectionsImpl* connections,
- std::shared_ptr<IQueueClientContext> context)
- : UserResponseCb_(std::move(userCb))
- , Connection_(connections)
- , Context_(std::move(context))
- {}
-
- TCb UserResponseCb_;
- TGRpcConnectionsImpl* Connection_;
- std::shared_ptr<IQueueClientContext> Context_;
-};
-
-template<typename TCb>
-class TAlarmActionBase
- : public TThrRefBase
- , public TGenericCbHolder<TCb>
- , private IQueueClientEvent
-{
-public:
- using TPtr = TIntrusivePtr<TAlarmActionBase<TCb>>;
- using TGenericCbHolder<TCb>::TGenericCbHolder;
-
- virtual void OnAlarm() = 0;
- virtual void OnError() = 0;
-
- void Start() {
- Y_VERIFY(this->Context_, "Missing shared context");
- auto context = this->Context_->CreateContext();
+
+#include <library/cpp/grpc/client/grpc_client_low.h>
+
+#include <util/thread/pool.h>
+
+#include <grpc++/alarm.h>
+
+namespace NYdb {
+
+using NGrpc::IQueueClientContext;
+using NGrpc::IQueueClientEvent;
+
+class TGRpcConnectionsImpl;
+struct TPlainStatus;
+
+
+template<typename TResponse>
+using TResponseCb = std::function<void(TResponse*, TPlainStatus status)>;
+using TDeferredOperationCb = std::function<void(Ydb::Operations::Operation*, TPlainStatus status)>;
+
+template<typename TCb>
+class TGenericCbHolder {
+protected:
+ TGenericCbHolder(
+ TCb&& userCb,
+ TGRpcConnectionsImpl* connections,
+ std::shared_ptr<IQueueClientContext> context)
+ : UserResponseCb_(std::move(userCb))
+ , Connection_(connections)
+ , Context_(std::move(context))
+ {}
+
+ TCb UserResponseCb_;
+ TGRpcConnectionsImpl* Connection_;
+ std::shared_ptr<IQueueClientContext> Context_;
+};
+
+template<typename TCb>
+class TAlarmActionBase
+ : public TThrRefBase
+ , public TGenericCbHolder<TCb>
+ , private IQueueClientEvent
+{
+public:
+ using TPtr = TIntrusivePtr<TAlarmActionBase<TCb>>;
+ using TGenericCbHolder<TCb>::TGenericCbHolder;
+
+ virtual void OnAlarm() = 0;
+ virtual void OnError() = 0;
+
+ void Start() {
+ Y_VERIFY(this->Context_, "Missing shared context");
+ auto context = this->Context_->CreateContext();
{
std::lock_guard lock(Mutex_);
- LocalContext_ = context;
- Alarm_.Set(this->Context_->CompletionQueue(), Deadline_, PrepareTag());
- }
- context->SubscribeStop([self = TPtr(this)] {
- self->Stop();
- });
- }
-
- void Stop() {
- Alarm_.Cancel();
- }
-
-private:
- IQueueClientEvent* PrepareTag() {
- Ref();
- return this;
- }
-
- bool Execute(bool ok) override {
+ LocalContext_ = context;
+ Alarm_.Set(this->Context_->CompletionQueue(), Deadline_, PrepareTag());
+ }
+ context->SubscribeStop([self = TPtr(this)] {
+ self->Stop();
+ });
+ }
+
+ void Stop() {
+ Alarm_.Cancel();
+ }
+
+private:
+ IQueueClientEvent* PrepareTag() {
+ Ref();
+ return this;
+ }
+
+ bool Execute(bool ok) override {
{
std::lock_guard lock(Mutex_);
- LocalContext_.reset();
- }
-
- if (ok) {
- OnAlarm();
- } else {
- OnError();
- }
-
- return false;
- }
-
- void Destroy() override {
- UnRef();
- }
-
-protected:
+ LocalContext_.reset();
+ }
+
+ if (ok) {
+ OnAlarm();
+ } else {
+ OnError();
+ }
+
+ return false;
+ }
+
+ void Destroy() override {
+ UnRef();
+ }
+
+protected:
gpr_timespec Deadline_ = {};
-
-private:
+
+private:
std::mutex Mutex_;
- grpc::Alarm Alarm_;
- std::shared_ptr<IQueueClientContext> LocalContext_;
-};
-
-template<typename TResponse>
-class TGRpcErrorResponse
- : public TGenericCbHolder<TResponseCb<TResponse>>
- , public IObjectInQueue
-{
-public:
- TGRpcErrorResponse(
- NGrpc::TGrpcStatus&& status,
- TResponseCb<TResponse>&& userCb,
- TGRpcConnectionsImpl* connections,
- std::shared_ptr<IQueueClientContext> context,
+ grpc::Alarm Alarm_;
+ std::shared_ptr<IQueueClientContext> LocalContext_;
+};
+
+template<typename TResponse>
+class TGRpcErrorResponse
+ : public TGenericCbHolder<TResponseCb<TResponse>>
+ , public IObjectInQueue
+{
+public:
+ TGRpcErrorResponse(
+ NGrpc::TGrpcStatus&& status,
+ TResponseCb<TResponse>&& userCb,
+ TGRpcConnectionsImpl* connections,
+ std::shared_ptr<IQueueClientContext> context,
const TStringType& endpoint)
- : TGenericCbHolder<TResponseCb<TResponse>>(std::move(userCb), connections, std::move(context))
- , GRpcStatus_(std::move(status))
- , Endpoint_(endpoint)
- { }
-
- void Process(void*) override {
- TPlainStatus status(GRpcStatus_, Endpoint_, {});
-
+ : TGenericCbHolder<TResponseCb<TResponse>>(std::move(userCb), connections, std::move(context))
+ , GRpcStatus_(std::move(status))
+ , Endpoint_(endpoint)
+ { }
+
+ void Process(void*) override {
+ TPlainStatus status(GRpcStatus_, Endpoint_, {});
+
if (!Endpoint_.empty()) {
TStringType msg = "Grpc error response on endpoint ";
- msg += Endpoint_;
- status.Issues.AddIssue(NYql::TIssue(msg));
- }
-
- this->UserResponseCb_(nullptr, status);
- delete this;
- }
-
-private:
- NGrpc::TGrpcStatus GRpcStatus_;
+ msg += Endpoint_;
+ status.Issues.AddIssue(NYql::TIssue(msg));
+ }
+
+ this->UserResponseCb_(nullptr, status);
+ delete this;
+ }
+
+private:
+ NGrpc::TGrpcStatus GRpcStatus_;
TStringType Endpoint_;
-};
-
-template<typename TResponse>
-class TResult
- : public TGenericCbHolder<TResponseCb<TResponse>>
- , public IObjectInQueue
-{
-public:
- TResult(
- TResponse&& response,
- NGrpc::TGrpcStatus&& status,
- TResponseCb<TResponse>&& userCb,
- TGRpcConnectionsImpl* connections,
- std::shared_ptr<IQueueClientContext> context,
+};
+
+template<typename TResponse>
+class TResult
+ : public TGenericCbHolder<TResponseCb<TResponse>>
+ , public IObjectInQueue
+{
+public:
+ TResult(
+ TResponse&& response,
+ NGrpc::TGrpcStatus&& status,
+ TResponseCb<TResponse>&& userCb,
+ TGRpcConnectionsImpl* connections,
+ std::shared_ptr<IQueueClientContext> context,
const TStringType& endpoint,
std::multimap<TStringType, TStringType>&& metadata)
- : TGenericCbHolder<TResponseCb<TResponse>>(std::move(userCb), connections, std::move(context))
- , Response_(std::move(response))
- , GRpcStatus_(std::move(status))
- , Endpoint_(endpoint)
- , Metadata_(std::move(metadata)) {}
-
- void Process(void*) override {
- this->UserResponseCb_(&Response_, TPlainStatus{GRpcStatus_, Endpoint_, std::move(Metadata_)});
- delete this;
- }
-
-private:
- TResponse Response_;
- NGrpc::TGrpcStatus GRpcStatus_;
+ : TGenericCbHolder<TResponseCb<TResponse>>(std::move(userCb), connections, std::move(context))
+ , Response_(std::move(response))
+ , GRpcStatus_(std::move(status))
+ , Endpoint_(endpoint)
+ , Metadata_(std::move(metadata)) {}
+
+ void Process(void*) override {
+ this->UserResponseCb_(&Response_, TPlainStatus{GRpcStatus_, Endpoint_, std::move(Metadata_)});
+ delete this;
+ }
+
+private:
+ TResponse Response_;
+ NGrpc::TGrpcStatus GRpcStatus_;
const TStringType Endpoint_;
std::multimap<TStringType, TStringType> Metadata_;
-};
-
-class TSimpleCbResult
- : public TGenericCbHolder<TSimpleCb>
- , public IObjectInQueue
-{
-public:
- TSimpleCbResult(
- TSimpleCb&& cb,
- TGRpcConnectionsImpl* connections,
- std::shared_ptr<IQueueClientContext> context);
- void Process(void*) override;
-};
-
-////////////////////////////////////////////////////////////////////////////////
-
-class TDeferredAction
- : public TAlarmActionBase<TDeferredOperationCb>
-{
-public:
- using TPtr = TIntrusivePtr<TDeferredAction>;
-
- TDeferredAction(
+};
+
+class TSimpleCbResult
+ : public TGenericCbHolder<TSimpleCb>
+ , public IObjectInQueue
+{
+public:
+ TSimpleCbResult(
+ TSimpleCb&& cb,
+ TGRpcConnectionsImpl* connections,
+ std::shared_ptr<IQueueClientContext> context);
+ void Process(void*) override;
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TDeferredAction
+ : public TAlarmActionBase<TDeferredOperationCb>
+{
+public:
+ using TPtr = TIntrusivePtr<TDeferredAction>;
+
+ TDeferredAction(
const TStringType& operationId,
- TDeferredOperationCb&& userCb,
- TGRpcConnectionsImpl* connection,
- std::shared_ptr<IQueueClientContext> context,
- TDuration timeout,
- TDbDriverStatePtr dbState,
+ TDeferredOperationCb&& userCb,
+ TGRpcConnectionsImpl* connection,
+ std::shared_ptr<IQueueClientContext> context,
+ TDuration timeout,
+ TDbDriverStatePtr dbState,
const TStringType& endpoint);
-
- void OnAlarm() override;
- void OnError() override;
-
-private:
- TDuration NextDelay_;
- TDbDriverStatePtr DbDriverState_;
+
+ void OnAlarm() override;
+ void OnError() override;
+
+private:
+ TDuration NextDelay_;
+ TDbDriverStatePtr DbDriverState_;
const TStringType OperationId_;
const TStringType Endpoint_;
-};
-
-class TPeriodicAction
- : public TAlarmActionBase<TPeriodicCb>
-{
-public:
- TPeriodicAction(
- TPeriodicCb&& userCb,
- TGRpcConnectionsImpl* connection,
- std::shared_ptr<IQueueClientContext> context,
- TDuration period);
-
- void OnAlarm() override;
- void OnError() override;
-private:
- TDuration Period_;
-};
-
-} // namespace NYdb
+};
+
+class TPeriodicAction
+ : public TAlarmActionBase<TPeriodicCb>
+{
+public:
+ TPeriodicAction(
+ TPeriodicCb&& userCb,
+ TGRpcConnectionsImpl* connection,
+ std::shared_ptr<IQueueClientContext> context,
+ TDuration period);
+
+ void OnAlarm() override;
+ void OnError() override;
+private:
+ TDuration Period_;
+};
+
+} // namespace NYdb
diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/grpc_connections.cpp b/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/grpc_connections.cpp
index 9409c09870..8f2420043e 100644
--- a/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/grpc_connections.cpp
+++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/grpc_connections.cpp
@@ -1,162 +1,162 @@
-#define INCLUDE_YDB_INTERNAL_H
-#include "grpc_connections.h"
-
+#define INCLUDE_YDB_INTERNAL_H
+#include "grpc_connections.h"
+
#include <ydb/public/sdk/cpp/client/ydb_types/exceptions/exceptions.h>
-
-namespace NYdb {
-
+
+namespace NYdb {
+
bool IsTokenCorrect(const TStringType& in) {
- for (char c : in) {
- if (!(IsAsciiAlnum(c) || IsAsciiPunct(c) || c == ' '))
- return false;
- }
- return true;
-}
-
+ for (char c : in) {
+ if (!(IsAsciiAlnum(c) || IsAsciiPunct(c) || c == ' '))
+ return false;
+ }
+ return true;
+}
+
TStringType GetAuthInfo(TDbDriverStatePtr p) {
- auto token = p->CredentialsProvider->GetAuthInfo();
- if (!IsTokenCorrect(token)) {
- throw TContractViolation("token is incorrect, iligal characters found");
- }
- return token;
-}
-
+ auto token = p->CredentialsProvider->GetAuthInfo();
+ if (!IsTokenCorrect(token)) {
+ throw TContractViolation("token is incorrect, iligal characters found");
+ }
+ return token;
+}
+
void SetDatabaseHeader(TCallMeta& meta, const TStringType& database) {
- // See TDbDriverStateTracker::GetDriverState to find place where we do quote non ASCII characters
- meta.Aux.push_back({YDB_DATABASE_HEADER, database});
-}
-
+ // See TDbDriverStateTracker::GetDriverState to find place where we do quote non ASCII characters
+ meta.Aux.push_back({YDB_DATABASE_HEADER, database});
+}
+
TStringType CreateSDKBuildInfo() {
return TStringType("ydb-cpp-sdk/") + GetSdkSemver();
-}
-
-template<class TDerived>
-class TScheduledObject : public TThrRefBase {
- using TSelf = TScheduledObject<TDerived>;
- using TPtr = TIntrusivePtr<TSelf>;
-
- Y_FORCE_INLINE TDerived* Derived() {
- return static_cast<TDerived*>(this);
- }
-
-protected:
- TScheduledObject() { }
-
- void Start(TDuration timeout, IQueueClientContextProvider* provider) {
- auto context = provider->CreateContext();
- if (!context) {
- Derived()->OnComplete(false);
- return;
- }
-
- auto deadline = gpr_time_add(
- gpr_now(GPR_CLOCK_MONOTONIC),
- gpr_time_from_micros(timeout.MicroSeconds(), GPR_TIMESPAN));
-
+}
+
+template<class TDerived>
+class TScheduledObject : public TThrRefBase {
+ using TSelf = TScheduledObject<TDerived>;
+ using TPtr = TIntrusivePtr<TSelf>;
+
+ Y_FORCE_INLINE TDerived* Derived() {
+ return static_cast<TDerived*>(this);
+ }
+
+protected:
+ TScheduledObject() { }
+
+ void Start(TDuration timeout, IQueueClientContextProvider* provider) {
+ auto context = provider->CreateContext();
+ if (!context) {
+ Derived()->OnComplete(false);
+ return;
+ }
+
+ auto deadline = gpr_time_add(
+ gpr_now(GPR_CLOCK_MONOTONIC),
+ gpr_time_from_micros(timeout.MicroSeconds(), GPR_TIMESPAN));
+
{
std::lock_guard guard(Mutex);
- Context = context;
- Alarm.Set(context->CompletionQueue(), deadline, OnAlarmTag.Prepare());
- }
-
- context->SubscribeCancel([self = TPtr(this)] {
- self->Alarm.Cancel();
- });
- }
-
-private:
- void OnAlarm(bool ok) {
+ Context = context;
+ Alarm.Set(context->CompletionQueue(), deadline, OnAlarmTag.Prepare());
+ }
+
+ context->SubscribeCancel([self = TPtr(this)] {
+ self->Alarm.Cancel();
+ });
+ }
+
+private:
+ void OnAlarm(bool ok) {
{
std::lock_guard guard(Mutex);
- // Break circular dependencies
- Context.reset();
- }
-
- Derived()->OnComplete(ok);
- }
-
-private:
+ // Break circular dependencies
+ Context.reset();
+ }
+
+ Derived()->OnComplete(ok);
+ }
+
+private:
std::mutex Mutex;
- IQueueClientContextPtr Context;
- grpc::Alarm Alarm;
-
-private:
- using TFixedEvent = NGrpc::TQueueClientFixedEvent<TSelf>;
-
- TFixedEvent OnAlarmTag = { this, &TSelf::OnAlarm };
-};
-
-class TScheduledCallback : public TScheduledObject<TScheduledCallback> {
- using TBase = TScheduledObject<TScheduledCallback>;
-
-public:
- using TCallback = std::function<void(bool)>;
-
- TScheduledCallback(TCallback&& callback)
- : Callback(std::move(callback))
- { }
-
- void Start(TDuration timeout, IQueueClientContextProvider* provider) {
- TBase::Start(timeout, provider);
- }
-
- void OnComplete(bool ok) {
- Callback(ok);
- Callback = { };
- }
-
-private:
- TCallback Callback;
-};
-
-class TScheduledFuture : public TScheduledObject<TScheduledFuture> {
- using TBase = TScheduledObject<TScheduledFuture>;
-
-public:
- TScheduledFuture()
- : Promise(NThreading::NewPromise<bool>())
- { }
-
- NThreading::TFuture<bool> Start(TDuration timeout, IQueueClientContextProvider* provider) {
- auto future = Promise.GetFuture();
-
- TBase::Start(timeout, provider);
-
- return future;
- }
-
- void OnComplete(bool ok) {
- Promise.SetValue(ok);
- Promise = { };
- }
-
-private:
- NThreading::TPromise<bool> Promise;
-};
-
-TGRpcConnectionsImpl::TGRpcConnectionsImpl(std::shared_ptr<IConnectionsParams> params)
- : MetricRegistryPtr_(nullptr)
- , ResponseQueue_(CreateThreadPool(params->GetClientThreadsNum()))
- , DefaultDiscoveryEndpoint_(params->GetEndpoint())
- , EnableSsl_(params->IsSslEnabled())
- , CaCert_(params->GetCaCert())
- , DefaultDatabase_(params->GetDatabase())
- , DefaultCredentialsProviderFactory_(params->GetCredentialsProviderFactory())
- , StateTracker_(this)
- , DefaultDiscoveryMode_(params->GetDiscoveryMode())
- , MaxQueuedRequests_(params->GetMaxQueuedRequests())
- , DrainOnDtors_(params->GetDrinOnDtors())
- , BalancingSettings_(params->GetBalancingSettings())
- , GRpcKeepAliveTimeout_(params->GetGRpcKeepAliveTimeout())
- , GRpcKeepAlivePermitWithoutCalls_(params->GetGRpcKeepAlivePermitWithoutCalls())
+ IQueueClientContextPtr Context;
+ grpc::Alarm Alarm;
+
+private:
+ using TFixedEvent = NGrpc::TQueueClientFixedEvent<TSelf>;
+
+ TFixedEvent OnAlarmTag = { this, &TSelf::OnAlarm };
+};
+
+class TScheduledCallback : public TScheduledObject<TScheduledCallback> {
+ using TBase = TScheduledObject<TScheduledCallback>;
+
+public:
+ using TCallback = std::function<void(bool)>;
+
+ TScheduledCallback(TCallback&& callback)
+ : Callback(std::move(callback))
+ { }
+
+ void Start(TDuration timeout, IQueueClientContextProvider* provider) {
+ TBase::Start(timeout, provider);
+ }
+
+ void OnComplete(bool ok) {
+ Callback(ok);
+ Callback = { };
+ }
+
+private:
+ TCallback Callback;
+};
+
+class TScheduledFuture : public TScheduledObject<TScheduledFuture> {
+ using TBase = TScheduledObject<TScheduledFuture>;
+
+public:
+ TScheduledFuture()
+ : Promise(NThreading::NewPromise<bool>())
+ { }
+
+ NThreading::TFuture<bool> Start(TDuration timeout, IQueueClientContextProvider* provider) {
+ auto future = Promise.GetFuture();
+
+ TBase::Start(timeout, provider);
+
+ return future;
+ }
+
+ void OnComplete(bool ok) {
+ Promise.SetValue(ok);
+ Promise = { };
+ }
+
+private:
+ NThreading::TPromise<bool> Promise;
+};
+
+TGRpcConnectionsImpl::TGRpcConnectionsImpl(std::shared_ptr<IConnectionsParams> params)
+ : MetricRegistryPtr_(nullptr)
+ , ResponseQueue_(CreateThreadPool(params->GetClientThreadsNum()))
+ , DefaultDiscoveryEndpoint_(params->GetEndpoint())
+ , EnableSsl_(params->IsSslEnabled())
+ , CaCert_(params->GetCaCert())
+ , DefaultDatabase_(params->GetDatabase())
+ , DefaultCredentialsProviderFactory_(params->GetCredentialsProviderFactory())
+ , StateTracker_(this)
+ , DefaultDiscoveryMode_(params->GetDiscoveryMode())
+ , MaxQueuedRequests_(params->GetMaxQueuedRequests())
+ , DrainOnDtors_(params->GetDrinOnDtors())
+ , BalancingSettings_(params->GetBalancingSettings())
+ , GRpcKeepAliveTimeout_(params->GetGRpcKeepAliveTimeout())
+ , GRpcKeepAlivePermitWithoutCalls_(params->GetGRpcKeepAlivePermitWithoutCalls())
, MemoryQuota_(params->GetMemoryQuota())
- , QueuedRequests_(0)
-#ifndef YDB_GRPC_BYPASS_CHANNEL_POOL
+ , QueuedRequests_(0)
+#ifndef YDB_GRPC_BYPASS_CHANNEL_POOL
, ChannelPool_(params->GetTcpKeepAliveSettings(), params->GetSocketIdleTimeout())
-#endif
- , GRpcClientLow_(params->GetNetworkThreadsNum())
- , Log(params->GetLog())
-{
+#endif
+ , GRpcClientLow_(params->GetNetworkThreadsNum())
+ , Log(params->GetLog())
+{
#ifndef YDB_GRPC_BYPASS_CHANNEL_POOL
if (params->GetSocketIdleTimeout() != TDuration::Max()) {
auto channelPoolUpdateWrapper = [this]
@@ -172,247 +172,247 @@ TGRpcConnectionsImpl::TGRpcConnectionsImpl(std::shared_ptr<IConnectionsParams> p
AddPeriodicTask(channelPoolUpdateWrapper, params->GetSocketIdleTimeout() * 0.1);
}
#endif
- //TAdaptiveThreadPool ignores params
- ResponseQueue_->Start(params->GetClientThreadsNum(), params->GetMaxQueuedResponses());
+ //TAdaptiveThreadPool ignores params
+ ResponseQueue_->Start(params->GetClientThreadsNum(), params->GetMaxQueuedResponses());
if (!DefaultDatabase_.empty()) {
- DefaultState_ = StateTracker_.GetDriverState(
- DefaultDatabase_,
- DefaultDiscoveryEndpoint_,
- DefaultDiscoveryMode_,
+ DefaultState_ = StateTracker_.GetDriverState(
+ DefaultDatabase_,
+ DefaultDiscoveryEndpoint_,
+ DefaultDiscoveryMode_,
EnableSsl_,
- DefaultCredentialsProviderFactory_
- );
- }
-}
-
-TGRpcConnectionsImpl::~TGRpcConnectionsImpl() {
- GRpcClientLow_.Stop(true);
- ResponseQueue_->Stop();
-}
-
-void TGRpcConnectionsImpl::AddPeriodicTask(TPeriodicCb&& cb, TDuration period) {
- std::shared_ptr<IQueueClientContext> context;
- if (!TryCreateContext(context)) {
- NYql::TIssues issues;
- cb(std::move(issues), EStatus::CLIENT_INTERNAL_ERROR);
- } else {
- auto action = MakeIntrusive<TPeriodicAction>(
- std::move(cb),
- this,
- std::move(context),
- period);
- action->Start();
- }
-}
-
-void TGRpcConnectionsImpl::ScheduleOneTimeTask(TSimpleCb&& fn, TDuration timeout) {
- auto cbLow = [this, fn = std::move(fn)](NYql::TIssues&&, EStatus status) mutable {
- if (status != EStatus::SUCCESS) {
- return false;
- }
-
- std::shared_ptr<IQueueClientContext> context;
-
- if (!TryCreateContext(context)) {
- // Shutting down, fn must handle it
- fn();
- } else {
- // Enqueue to user pool
- auto resp = new TSimpleCbResult(
- std::move(fn),
- this,
- std::move(context));
- EnqueueResponse(resp);
- }
-
- return false;
- };
-
- if (timeout) {
- AddPeriodicTask(std::move(cbLow), timeout);
- } else {
- cbLow(NYql::TIssues(), EStatus::SUCCESS);
- }
-}
-
-NThreading::TFuture<bool> TGRpcConnectionsImpl::ScheduleFuture(
- TDuration timeout,
- IQueueClientContextPtr context)
-{
- IQueueClientContextProvider* provider = context.get();
- if (!provider) {
- provider = &GRpcClientLow_;
- }
-
- return MakeIntrusive<TScheduledFuture>()
- ->Start(timeout, provider);
-}
-
-void TGRpcConnectionsImpl::ScheduleCallback(
- TDuration timeout,
- std::function<void(bool)> callback,
- IQueueClientContextPtr context)
-{
- IQueueClientContextProvider* provider = context.get();
- if (!provider) {
- provider = &GRpcClientLow_;
- }
-
- return MakeIntrusive<TScheduledCallback>(std::move(callback))
- ->Start(timeout, provider);
-}
-
-TDbDriverStatePtr TGRpcConnectionsImpl::GetDriverState(
+ DefaultCredentialsProviderFactory_
+ );
+ }
+}
+
+TGRpcConnectionsImpl::~TGRpcConnectionsImpl() {
+ GRpcClientLow_.Stop(true);
+ ResponseQueue_->Stop();
+}
+
+void TGRpcConnectionsImpl::AddPeriodicTask(TPeriodicCb&& cb, TDuration period) {
+ std::shared_ptr<IQueueClientContext> context;
+ if (!TryCreateContext(context)) {
+ NYql::TIssues issues;
+ cb(std::move(issues), EStatus::CLIENT_INTERNAL_ERROR);
+ } else {
+ auto action = MakeIntrusive<TPeriodicAction>(
+ std::move(cb),
+ this,
+ std::move(context),
+ period);
+ action->Start();
+ }
+}
+
+void TGRpcConnectionsImpl::ScheduleOneTimeTask(TSimpleCb&& fn, TDuration timeout) {
+ auto cbLow = [this, fn = std::move(fn)](NYql::TIssues&&, EStatus status) mutable {
+ if (status != EStatus::SUCCESS) {
+ return false;
+ }
+
+ std::shared_ptr<IQueueClientContext> context;
+
+ if (!TryCreateContext(context)) {
+ // Shutting down, fn must handle it
+ fn();
+ } else {
+ // Enqueue to user pool
+ auto resp = new TSimpleCbResult(
+ std::move(fn),
+ this,
+ std::move(context));
+ EnqueueResponse(resp);
+ }
+
+ return false;
+ };
+
+ if (timeout) {
+ AddPeriodicTask(std::move(cbLow), timeout);
+ } else {
+ cbLow(NYql::TIssues(), EStatus::SUCCESS);
+ }
+}
+
+NThreading::TFuture<bool> TGRpcConnectionsImpl::ScheduleFuture(
+ TDuration timeout,
+ IQueueClientContextPtr context)
+{
+ IQueueClientContextProvider* provider = context.get();
+ if (!provider) {
+ provider = &GRpcClientLow_;
+ }
+
+ return MakeIntrusive<TScheduledFuture>()
+ ->Start(timeout, provider);
+}
+
+void TGRpcConnectionsImpl::ScheduleCallback(
+ TDuration timeout,
+ std::function<void(bool)> callback,
+ IQueueClientContextPtr context)
+{
+ IQueueClientContextProvider* provider = context.get();
+ if (!provider) {
+ provider = &GRpcClientLow_;
+ }
+
+ return MakeIntrusive<TScheduledCallback>(std::move(callback))
+ ->Start(timeout, provider);
+}
+
+TDbDriverStatePtr TGRpcConnectionsImpl::GetDriverState(
const TMaybe<TStringType>& database,
const TMaybe<TStringType>& discoveryEndpoint,
- const TMaybe<EDiscoveryMode>& discoveryMode,
+ const TMaybe<EDiscoveryMode>& discoveryMode,
const TMaybe<bool>& enableSsl,
- const TMaybe<std::shared_ptr<ICredentialsProviderFactory>>& credentialsProviderFactory
-) {
- return StateTracker_.GetDriverState(
- database ? database.GetRef() : DefaultDatabase_,
- discoveryEndpoint ? discoveryEndpoint.GetRef() : DefaultDiscoveryEndpoint_,
- discoveryMode ? discoveryMode.GetRef() : DefaultDiscoveryMode_,
+ const TMaybe<std::shared_ptr<ICredentialsProviderFactory>>& credentialsProviderFactory
+) {
+ return StateTracker_.GetDriverState(
+ database ? database.GetRef() : DefaultDatabase_,
+ discoveryEndpoint ? discoveryEndpoint.GetRef() : DefaultDiscoveryEndpoint_,
+ discoveryMode ? discoveryMode.GetRef() : DefaultDiscoveryMode_,
enableSsl ? enableSsl.GetRef() : EnableSsl_,
- credentialsProviderFactory ? credentialsProviderFactory.GetRef() : DefaultCredentialsProviderFactory_);
-}
-
-IQueueClientContextPtr TGRpcConnectionsImpl::CreateContext() {
- return GRpcClientLow_.CreateContext();
-}
-
-bool TGRpcConnectionsImpl::TryCreateContext(IQueueClientContextPtr& context) {
- if (!context) {
- // Keep CQ running until the request is complete
- context = CreateContext();
- if (!context) {
- return false;
- }
- }
- return true;
-}
-
-void TGRpcConnectionsImpl::WaitIdle() {
- GRpcClientLow_.WaitIdle();
-}
-
-void TGRpcConnectionsImpl::Stop(bool wait) {
+ credentialsProviderFactory ? credentialsProviderFactory.GetRef() : DefaultCredentialsProviderFactory_);
+}
+
+IQueueClientContextPtr TGRpcConnectionsImpl::CreateContext() {
+ return GRpcClientLow_.CreateContext();
+}
+
+bool TGRpcConnectionsImpl::TryCreateContext(IQueueClientContextPtr& context) {
+ if (!context) {
+ // Keep CQ running until the request is complete
+ context = CreateContext();
+ if (!context) {
+ return false;
+ }
+ }
+ return true;
+}
+
+void TGRpcConnectionsImpl::WaitIdle() {
+ GRpcClientLow_.WaitIdle();
+}
+
+void TGRpcConnectionsImpl::Stop(bool wait) {
StateTracker_.SendNotification(TDbDriverState::ENotifyType::STOP).Wait();
- GRpcClientLow_.Stop(wait);
-}
-
-void TGRpcConnectionsImpl::SetGrpcKeepAlive(NGrpc::TGRpcClientConfig& config, const TDuration& timeout, bool permitWithoutCalls) {
- ui64 timeoutMs = timeout.MilliSeconds();
- config.IntChannelParams[GRPC_ARG_KEEPALIVE_TIME_MS] = timeoutMs >> 3;
- config.IntChannelParams[GRPC_ARG_KEEPALIVE_TIMEOUT_MS] = timeoutMs;
- config.IntChannelParams[GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA] = 0;
- config.IntChannelParams[GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS] = permitWithoutCalls ? 1 : 0;
-}
-
-TAsyncListEndpointsResult TGRpcConnectionsImpl::GetEndpoints(TDbDriverStatePtr dbState) {
- Ydb::Discovery::ListEndpointsRequest request;
- request.set_database(dbState->Database);
-
- auto promise = NThreading::NewPromise<TListEndpointsResult>();
-
- auto extractor = [promise]
- (google::protobuf::Any* any, TPlainStatus status) mutable {
- Ydb::Discovery::ListEndpointsResult result;
- if (any) {
- any->UnpackTo(&result);
- }
- TListEndpointsResult val{result, status};
- promise.SetValue(std::move(val));
- };
-
- RunDeferred<Ydb::Discovery::V1::DiscoveryService, Ydb::Discovery::ListEndpointsRequest, Ydb::Discovery::ListEndpointsResponse>(
- std::move(request),
- extractor,
- &Ydb::Discovery::V1::DiscoveryService::Stub::AsyncListEndpoints,
- dbState->shared_from_this(),
- INITIAL_DEFERRED_CALL_DELAY,
- TRpcRequestSettings(),
- GET_ENDPOINTS_TIMEOUT);
-
- std::weak_ptr<TDbDriverState> weakState = dbState;
-
- return promise.GetFuture().Apply([this, weakState](NThreading::TFuture<TListEndpointsResult> future){
- auto strong = weakState.lock();
- auto result = future.ExtractValue();
- if (strong && result.DiscoveryStatus.IsTransportError()) {
- strong->StatCollector.IncDiscoveryFailDueTransportError();
- }
- return NThreading::MakeFuture<TListEndpointsResult>(MutateDiscovery(std::move(result), strong->Database));
- });
-}
-
+ GRpcClientLow_.Stop(wait);
+}
+
+void TGRpcConnectionsImpl::SetGrpcKeepAlive(NGrpc::TGRpcClientConfig& config, const TDuration& timeout, bool permitWithoutCalls) {
+ ui64 timeoutMs = timeout.MilliSeconds();
+ config.IntChannelParams[GRPC_ARG_KEEPALIVE_TIME_MS] = timeoutMs >> 3;
+ config.IntChannelParams[GRPC_ARG_KEEPALIVE_TIMEOUT_MS] = timeoutMs;
+ config.IntChannelParams[GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA] = 0;
+ config.IntChannelParams[GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS] = permitWithoutCalls ? 1 : 0;
+}
+
+TAsyncListEndpointsResult TGRpcConnectionsImpl::GetEndpoints(TDbDriverStatePtr dbState) {
+ Ydb::Discovery::ListEndpointsRequest request;
+ request.set_database(dbState->Database);
+
+ auto promise = NThreading::NewPromise<TListEndpointsResult>();
+
+ auto extractor = [promise]
+ (google::protobuf::Any* any, TPlainStatus status) mutable {
+ Ydb::Discovery::ListEndpointsResult result;
+ if (any) {
+ any->UnpackTo(&result);
+ }
+ TListEndpointsResult val{result, status};
+ promise.SetValue(std::move(val));
+ };
+
+ RunDeferred<Ydb::Discovery::V1::DiscoveryService, Ydb::Discovery::ListEndpointsRequest, Ydb::Discovery::ListEndpointsResponse>(
+ std::move(request),
+ extractor,
+ &Ydb::Discovery::V1::DiscoveryService::Stub::AsyncListEndpoints,
+ dbState->shared_from_this(),
+ INITIAL_DEFERRED_CALL_DELAY,
+ TRpcRequestSettings(),
+ GET_ENDPOINTS_TIMEOUT);
+
+ std::weak_ptr<TDbDriverState> weakState = dbState;
+
+ return promise.GetFuture().Apply([this, weakState](NThreading::TFuture<TListEndpointsResult> future){
+ auto strong = weakState.lock();
+ auto result = future.ExtractValue();
+ if (strong && result.DiscoveryStatus.IsTransportError()) {
+ strong->StatCollector.IncDiscoveryFailDueTransportError();
+ }
+ return NThreading::MakeFuture<TListEndpointsResult>(MutateDiscovery(std::move(result), strong->Database));
+ });
+}
+
TListEndpointsResult TGRpcConnectionsImpl::MutateDiscovery(TListEndpointsResult result, const TStringType& database) {
std::lock_guard lock(ExtensionsLock_);
- if (!DiscoveryMutatorCb)
- return result;
-
- auto endpoint = result.DiscoveryStatus.Endpoint;
+ if (!DiscoveryMutatorCb)
+ return result;
+
+ auto endpoint = result.DiscoveryStatus.Endpoint;
auto ydbStatus = NYdb::TStatus(std::move(result.DiscoveryStatus));
-
- ydbStatus = DiscoveryMutatorCb(&result.Result, std::move(ydbStatus), database);
-
- auto issues = ydbStatus.GetIssues();
-
- auto plainStatus = TPlainStatus(ydbStatus.GetStatus(), std::move(issues), endpoint, {});
- result.DiscoveryStatus = plainStatus;
- return result;
-}
-
-bool TGRpcConnectionsImpl::GetDrainOnDtors() const {
- return DrainOnDtors_;
-}
-
-TBalancingSettings TGRpcConnectionsImpl::GetBalancingSettings() const {
- return BalancingSettings_;
-}
-
-bool TGRpcConnectionsImpl::StartStatCollecting(NMonitoring::IMetricRegistry* sensorsRegistry) {
+
+ ydbStatus = DiscoveryMutatorCb(&result.Result, std::move(ydbStatus), database);
+
+ auto issues = ydbStatus.GetIssues();
+
+ auto plainStatus = TPlainStatus(ydbStatus.GetStatus(), std::move(issues), endpoint, {});
+ result.DiscoveryStatus = plainStatus;
+ return result;
+}
+
+bool TGRpcConnectionsImpl::GetDrainOnDtors() const {
+ return DrainOnDtors_;
+}
+
+TBalancingSettings TGRpcConnectionsImpl::GetBalancingSettings() const {
+ return BalancingSettings_;
+}
+
+bool TGRpcConnectionsImpl::StartStatCollecting(NMonitoring::IMetricRegistry* sensorsRegistry) {
{
std::lock_guard lock(ExtensionsLock_);
- if (MetricRegistryPtr_) {
- return false;
- }
- if (auto ptr = dynamic_cast<NMonitoring::TMetricRegistry*>(sensorsRegistry)) {
- MetricRegistryPtr_ = ptr;
- } else {
- Cerr << "Unknown IMetricRegistry impl" << Endl;
- return false;
- }
- }
-
- StateTracker_.SetMetricRegistry(MetricRegistryPtr_);
- return true;
-}
-
-NMonitoring::TMetricRegistry* TGRpcConnectionsImpl::GetMetricRegistry() {
+ if (MetricRegistryPtr_) {
+ return false;
+ }
+ if (auto ptr = dynamic_cast<NMonitoring::TMetricRegistry*>(sensorsRegistry)) {
+ MetricRegistryPtr_ = ptr;
+ } else {
+ Cerr << "Unknown IMetricRegistry impl" << Endl;
+ return false;
+ }
+ }
+
+ StateTracker_.SetMetricRegistry(MetricRegistryPtr_);
+ return true;
+}
+
+NMonitoring::TMetricRegistry* TGRpcConnectionsImpl::GetMetricRegistry() {
std::lock_guard lock(ExtensionsLock_);
return MetricRegistryPtr_;
-}
-
-void TGRpcConnectionsImpl::RegisterExtension(IExtension* extension) {
- Extensions_.emplace_back(extension);
-}
-
-void TGRpcConnectionsImpl::RegisterExtensionApi(IExtensionApi* api) {
- ExtensionApis_.emplace_back(api);
-}
-
-void TGRpcConnectionsImpl::SetDiscoveryMutator(IDiscoveryMutatorApi::TMutatorCb&& cb) {
+}
+
+void TGRpcConnectionsImpl::RegisterExtension(IExtension* extension) {
+ Extensions_.emplace_back(extension);
+}
+
+void TGRpcConnectionsImpl::RegisterExtensionApi(IExtensionApi* api) {
+ ExtensionApis_.emplace_back(api);
+}
+
+void TGRpcConnectionsImpl::SetDiscoveryMutator(IDiscoveryMutatorApi::TMutatorCb&& cb) {
std::lock_guard lock(ExtensionsLock_);
- DiscoveryMutatorCb = std::move(cb);
-}
-
-const TLog& TGRpcConnectionsImpl::GetLog() const {
- return Log;
-}
-
-void TGRpcConnectionsImpl::EnqueueResponse(IObjectInQueue* action) {
- Y_ENSURE(ResponseQueue_->Add(action));
-}
-
-} // namespace NYdb
+ DiscoveryMutatorCb = std::move(cb);
+}
+
+const TLog& TGRpcConnectionsImpl::GetLog() const {
+ return Log;
+}
+
+void TGRpcConnectionsImpl::EnqueueResponse(IObjectInQueue* action) {
+ Y_ENSURE(ResponseQueue_->Add(action));
+}
+
+} // namespace NYdb
diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/grpc_connections.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/grpc_connections.h
index 4aa2b8d0e4..bfc1b7ec38 100644
--- a/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/grpc_connections.h
+++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/grpc_connections.h
@@ -1,324 +1,324 @@
-#pragma once
-
+#pragma once
+
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/internal_header.h>
-
-#include "actions.h"
-#include "params.h"
-
+
+#include "actions.h"
+#include "params.h"
+
#include <ydb/public/api/grpc/ydb_discovery_v1.grpc.pb.h>
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/state.h>
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/rpc_request_settings/settings.h>
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/thread_pool/pool.h>
#include <ydb/public/sdk/cpp/client/resources/ydb_resources.h>
#include <ydb/public/sdk/cpp/client/ydb_extension/extension.h>
-
+
#include <ydb/library/yql/public/issue/yql_issue_message.h>
-
-namespace NYdb {
-
-constexpr TDuration GRPC_KEEP_ALIVE_TIMEOUT_FOR_DISCOVERY = TDuration::Seconds(10);
-constexpr TDuration INITIAL_DEFERRED_CALL_DELAY = TDuration::MilliSeconds(10); // The delay before first deferred service call
-constexpr TDuration GET_ENDPOINTS_TIMEOUT = TDuration::Seconds(10); // Time wait for ListEndpoints request, after this time we pass error to client
-
-using NGrpc::TCallMeta;
-using NGrpc::IQueueClientContextPtr;
-using NGrpc::IQueueClientContextProvider;
-
-class ICredentialsProvider;
-
-// Deferred callbacks
-using TDeferredResultCb = std::function<void(google::protobuf::Any*, TPlainStatus status)>;
-
+
+namespace NYdb {
+
+constexpr TDuration GRPC_KEEP_ALIVE_TIMEOUT_FOR_DISCOVERY = TDuration::Seconds(10);
+constexpr TDuration INITIAL_DEFERRED_CALL_DELAY = TDuration::MilliSeconds(10); // The delay before first deferred service call
+constexpr TDuration GET_ENDPOINTS_TIMEOUT = TDuration::Seconds(10); // Time wait for ListEndpoints request, after this time we pass error to client
+
+using NGrpc::TCallMeta;
+using NGrpc::IQueueClientContextPtr;
+using NGrpc::IQueueClientContextProvider;
+
+class ICredentialsProvider;
+
+// Deferred callbacks
+using TDeferredResultCb = std::function<void(google::protobuf::Any*, TPlainStatus status)>;
+
TStringType GetAuthInfo(TDbDriverStatePtr p);
void SetDatabaseHeader(TCallMeta& meta, const TStringType& database);
TStringType CreateSDKBuildInfo();
-
-class TGRpcConnectionsImpl
- : public IQueueClientContextProvider
- , public IInternalClient
-{
- friend class TDeferredAction;
-public:
- TGRpcConnectionsImpl(std::shared_ptr<IConnectionsParams> params);
- ~TGRpcConnectionsImpl();
-
- void AddPeriodicTask(TPeriodicCb&& cb, TDuration period) override;
- void ScheduleOneTimeTask(TSimpleCb&& fn, TDuration timeout);
- NThreading::TFuture<bool> ScheduleFuture(
- TDuration timeout,
- IQueueClientContextPtr token = nullptr
- );
- void ScheduleCallback(
- TDuration timeout,
- std::function<void(bool)> callback,
- IQueueClientContextPtr token = nullptr
- );
- // The idea is: sometimes we need to work with multiple databases simultaneously
- // This method returns DbDriverState (or just db state) for given database credentials pair
- // this state is used to keep data related to particular database.
- TDbDriverStatePtr GetDriverState(
+
+class TGRpcConnectionsImpl
+ : public IQueueClientContextProvider
+ , public IInternalClient
+{
+ friend class TDeferredAction;
+public:
+ TGRpcConnectionsImpl(std::shared_ptr<IConnectionsParams> params);
+ ~TGRpcConnectionsImpl();
+
+ void AddPeriodicTask(TPeriodicCb&& cb, TDuration period) override;
+ void ScheduleOneTimeTask(TSimpleCb&& fn, TDuration timeout);
+ NThreading::TFuture<bool> ScheduleFuture(
+ TDuration timeout,
+ IQueueClientContextPtr token = nullptr
+ );
+ void ScheduleCallback(
+ TDuration timeout,
+ std::function<void(bool)> callback,
+ IQueueClientContextPtr token = nullptr
+ );
+ // The idea is: sometimes we need to work with multiple databases simultaneously
+ // This method returns DbDriverState (or just db state) for given database credentials pair
+ // this state is used to keep data related to particular database.
+ TDbDriverStatePtr GetDriverState(
const TMaybe<TStringType>& database,
const TMaybe<TStringType>& discoveryEndpoint,
- const TMaybe<EDiscoveryMode>& discoveryMode,
+ const TMaybe<EDiscoveryMode>& discoveryMode,
const TMaybe<bool>& enableSsl,
- const TMaybe<std::shared_ptr<ICredentialsProviderFactory>>& credentialsProviderFactory
- );
+ const TMaybe<std::shared_ptr<ICredentialsProviderFactory>>& credentialsProviderFactory
+ );
IQueueClientContextPtr CreateContext() override;
- bool TryCreateContext(IQueueClientContextPtr& context);
- void WaitIdle();
- void Stop(bool wait = false);
-
- template<typename TService>
- using TServiceConnection = NGrpc::TServiceConnection<TService>;
-
- static void SetGrpcKeepAlive(NGrpc::TGRpcClientConfig& config, const TDuration& timeout, bool permitWithoutCalls);
-
- template<typename TService>
+ bool TryCreateContext(IQueueClientContextPtr& context);
+ void WaitIdle();
+ void Stop(bool wait = false);
+
+ template<typename TService>
+ using TServiceConnection = NGrpc::TServiceConnection<TService>;
+
+ static void SetGrpcKeepAlive(NGrpc::TGRpcClientConfig& config, const TDuration& timeout, bool permitWithoutCalls);
+
+ template<typename TService>
std::pair<std::unique_ptr<TServiceConnection<TService>>, TStringType> GetServiceConnection(
TDbDriverStatePtr dbState, const TStringType& preferredEndpoint,
TRpcRequestSettings::TEndpointPolicy endpointPolicy)
- {
- auto clientConfig = NGrpc::TGRpcClientConfig(dbState->DiscoveryEndpoint);
+ {
+ auto clientConfig = NGrpc::TGRpcClientConfig(dbState->DiscoveryEndpoint);
clientConfig.EnableSsl = dbState->EnableSsl;
- clientConfig.SslCaCert = CaCert_;
+ clientConfig.SslCaCert = CaCert_;
clientConfig.MemQuota = MemoryQuota_;
-
+
if (std::is_same<TService,Ydb::Discovery::V1::DiscoveryService>()
|| dbState->Database.empty()
|| endpointPolicy == TRpcRequestSettings::TEndpointPolicy::UseDiscoveryEndpoint)
{
SetGrpcKeepAlive(clientConfig, GRPC_KEEP_ALIVE_TIMEOUT_FOR_DISCOVERY, GRpcKeepAlivePermitWithoutCalls_);
} else {
- auto endpoint = dbState->EndpointPool.GetEndpoint(preferredEndpoint);
+ auto endpoint = dbState->EndpointPool.GetEndpoint(preferredEndpoint);
if (!endpoint) {
- return {nullptr, ""};
- }
+ return {nullptr, ""};
+ }
clientConfig.Locator = endpoint.Endpoint;
clientConfig.SslTargetNameOverride = endpoint.SslTargetNameOverride;
- if (GRpcKeepAliveTimeout_) {
- SetGrpcKeepAlive(clientConfig, GRpcKeepAliveTimeout_, GRpcKeepAlivePermitWithoutCalls_);
- }
- }
-
- std::unique_ptr<TServiceConnection<TService>> conn;
-#ifndef YDB_GRPC_BYPASS_CHANNEL_POOL
- ChannelPool_.GetStubsHolderLocked(
- clientConfig.Locator, clientConfig, [&conn, this](NGrpc::TStubsHolder& holder) mutable {
- conn.reset(GRpcClientLow_.CreateGRpcServiceConnection<TService>(holder).release());
- });
-#else
- conn = std::move(GRpcClientLow_.CreateGRpcServiceConnection<TService>(clientConfig));
-#endif
- return {std::move(conn), clientConfig.Locator};
- }
-
- template<class TService, class TRequest, class TResponse>
- using TSimpleRpc =
- typename NGrpc::TSimpleRequestProcessor<
- typename TService::Stub,
- TRequest,
- TResponse>::TAsyncRequest;
-
- template<typename TService, typename TRequest, typename TResponse>
- void Run(
- TRequest&& request,
- TResponseCb<TResponse>&& userResponseCb,
- TSimpleRpc<TService, TRequest, TResponse> rpc,
- TDbDriverStatePtr dbState,
- const TRpcRequestSettings& requestSettings,
- TDuration clientTimeout,
+ if (GRpcKeepAliveTimeout_) {
+ SetGrpcKeepAlive(clientConfig, GRpcKeepAliveTimeout_, GRpcKeepAlivePermitWithoutCalls_);
+ }
+ }
+
+ std::unique_ptr<TServiceConnection<TService>> conn;
+#ifndef YDB_GRPC_BYPASS_CHANNEL_POOL
+ ChannelPool_.GetStubsHolderLocked(
+ clientConfig.Locator, clientConfig, [&conn, this](NGrpc::TStubsHolder& holder) mutable {
+ conn.reset(GRpcClientLow_.CreateGRpcServiceConnection<TService>(holder).release());
+ });
+#else
+ conn = std::move(GRpcClientLow_.CreateGRpcServiceConnection<TService>(clientConfig));
+#endif
+ return {std::move(conn), clientConfig.Locator};
+ }
+
+ template<class TService, class TRequest, class TResponse>
+ using TSimpleRpc =
+ typename NGrpc::TSimpleRequestProcessor<
+ typename TService::Stub,
+ TRequest,
+ TResponse>::TAsyncRequest;
+
+ template<typename TService, typename TRequest, typename TResponse>
+ void Run(
+ TRequest&& request,
+ TResponseCb<TResponse>&& userResponseCb,
+ TSimpleRpc<TService, TRequest, TResponse> rpc,
+ TDbDriverStatePtr dbState,
+ const TRpcRequestSettings& requestSettings,
+ TDuration clientTimeout,
const TStringType& preferredEndpoint,
- std::shared_ptr<IQueueClientContext> context = nullptr)
- {
- using NGrpc::TGrpcStatus;
- using TConnection = std::unique_ptr<TServiceConnection<TService>>;
- Y_VERIFY(dbState);
-
- if (!TryCreateContext(context)) {
- TPlainStatus status(EStatus::CLIENT_CANCELLED, "Client is stopped");
- userResponseCb(nullptr, TPlainStatus{status.Status, std::move(status.Issues)});
- return;
- }
-
- if (dbState->StatCollector.IsCollecting()) {
- std::weak_ptr<TDbDriverState> weakState = dbState;
- const auto startTime = TInstant::Now();
- userResponseCb = std::move([cb = std::move(userResponseCb), weakState, startTime](TResponse* response, TPlainStatus status) {
- const auto resultSize = response ? response->ByteSizeLong() : 0;
- cb(response, status);
-
- if (auto state = weakState.lock()) {
- state->StatCollector.IncRequestLatency(TInstant::Now() - startTime);
- state->StatCollector.IncResultSize(resultSize);
- }
- });
- }
-
+ std::shared_ptr<IQueueClientContext> context = nullptr)
+ {
+ using NGrpc::TGrpcStatus;
+ using TConnection = std::unique_ptr<TServiceConnection<TService>>;
+ Y_VERIFY(dbState);
+
+ if (!TryCreateContext(context)) {
+ TPlainStatus status(EStatus::CLIENT_CANCELLED, "Client is stopped");
+ userResponseCb(nullptr, TPlainStatus{status.Status, std::move(status.Issues)});
+ return;
+ }
+
+ if (dbState->StatCollector.IsCollecting()) {
+ std::weak_ptr<TDbDriverState> weakState = dbState;
+ const auto startTime = TInstant::Now();
+ userResponseCb = std::move([cb = std::move(userResponseCb), weakState, startTime](TResponse* response, TPlainStatus status) {
+ const auto resultSize = response ? response->ByteSizeLong() : 0;
+ cb(response, status);
+
+ if (auto state = weakState.lock()) {
+ state->StatCollector.IncRequestLatency(TInstant::Now() - startTime);
+ state->StatCollector.IncResultSize(resultSize);
+ }
+ });
+ }
+
auto endpointPolicy = requestSettings.EndpointPolicy;
- WithServiceConnection<TService>(
- [this, request = std::move(request), userResponseCb = std::move(userResponseCb), rpc, requestSettings, context = std::move(context), clientTimeout, dbState]
+ WithServiceConnection<TService>(
+ [this, request = std::move(request), userResponseCb = std::move(userResponseCb), rpc, requestSettings, context = std::move(context), clientTimeout, dbState]
(TPlainStatus status, TConnection serviceConnection, TStringType endpoint) mutable -> void {
- if (!status.Ok()) {
- userResponseCb(
- nullptr,
- std::move(status));
- return;
- }
-
- TCallMeta meta;
- meta.Timeout = clientTimeout;
- #ifndef YDB_GRPC_UNSECURE_AUTH
- meta.CallCredentials = dbState->CallCredentials;
- #else
+ if (!status.Ok()) {
+ userResponseCb(
+ nullptr,
+ std::move(status));
+ return;
+ }
+
+ TCallMeta meta;
+ meta.Timeout = clientTimeout;
+ #ifndef YDB_GRPC_UNSECURE_AUTH
+ meta.CallCredentials = dbState->CallCredentials;
+ #else
if (requestSettings.UseAuth && dbState->CredentialsProvider && dbState->CredentialsProvider->IsValid()) {
- try {
- meta.Aux.push_back({ YDB_AUTH_TICKET_HEADER, GetAuthInfo(dbState) });
- } catch (const yexception& e) {
- userResponseCb(
- nullptr,
- TPlainStatus(
- EStatus::CLIENT_UNAUTHENTICATED,
- TStringBuilder() << "Can't get Authentication info from CredentialsProvider. " << e.what()
- )
- );
- return;
- }
- }
- #endif
- if (!requestSettings.TraceId.empty()) {
- meta.Aux.push_back({YDB_TRACE_ID_HEADER, requestSettings.TraceId});
- }
-
- if (!requestSettings.RequestType.empty()) {
- meta.Aux.push_back({YDB_REQUEST_TYPE_HEADER, requestSettings.RequestType});
- }
-
+ try {
+ meta.Aux.push_back({ YDB_AUTH_TICKET_HEADER, GetAuthInfo(dbState) });
+ } catch (const yexception& e) {
+ userResponseCb(
+ nullptr,
+ TPlainStatus(
+ EStatus::CLIENT_UNAUTHENTICATED,
+ TStringBuilder() << "Can't get Authentication info from CredentialsProvider. " << e.what()
+ )
+ );
+ return;
+ }
+ }
+ #endif
+ if (!requestSettings.TraceId.empty()) {
+ meta.Aux.push_back({YDB_TRACE_ID_HEADER, requestSettings.TraceId});
+ }
+
+ if (!requestSettings.RequestType.empty()) {
+ meta.Aux.push_back({YDB_REQUEST_TYPE_HEADER, requestSettings.RequestType});
+ }
+
if (!dbState->Database.empty()) {
- SetDatabaseHeader(meta, dbState->Database);
- }
-
- if (std::is_same<TService,Ydb::Discovery::V1::DiscoveryService>()) {
- meta.Aux.push_back({YDB_SDK_BUILD_INFO_HEADER, CreateSDKBuildInfo()});
- }
-
+ SetDatabaseHeader(meta, dbState->Database);
+ }
+
+ if (std::is_same<TService,Ydb::Discovery::V1::DiscoveryService>()) {
+ meta.Aux.push_back({YDB_SDK_BUILD_INFO_HEADER, CreateSDKBuildInfo()});
+ }
+
meta.Aux.insert(meta.Aux.end(), requestSettings.Header.begin(), requestSettings.Header.end());
- dbState->StatCollector.IncGRpcInFlight();
- dbState->StatCollector.IncGRpcInFlightByHost(endpoint);
-
- NGrpc::TAdvancedResponseCallback<TResponse> responseCbLow =
- [this, context, userResponseCb = std::move(userResponseCb), endpoint, dbState]
- (const grpc::ClientContext& ctx, TGrpcStatus&& grpcStatus, TResponse&& response) mutable -> void {
- dbState->StatCollector.DecGRpcInFlight();
- dbState->StatCollector.DecGRpcInFlightByHost(endpoint);
-
- if (NGrpc::IsGRpcStatusGood(grpcStatus)) {
+ dbState->StatCollector.IncGRpcInFlight();
+ dbState->StatCollector.IncGRpcInFlightByHost(endpoint);
+
+ NGrpc::TAdvancedResponseCallback<TResponse> responseCbLow =
+ [this, context, userResponseCb = std::move(userResponseCb), endpoint, dbState]
+ (const grpc::ClientContext& ctx, TGrpcStatus&& grpcStatus, TResponse&& response) mutable -> void {
+ dbState->StatCollector.DecGRpcInFlight();
+ dbState->StatCollector.DecGRpcInFlightByHost(endpoint);
+
+ if (NGrpc::IsGRpcStatusGood(grpcStatus)) {
std::multimap<TStringType, TStringType> metadata;
-
- for (const auto& [name, value] : ctx.GetServerInitialMetadata()) {
- metadata.emplace(
+
+ for (const auto& [name, value] : ctx.GetServerInitialMetadata()) {
+ metadata.emplace(
TStringType(name.begin(), name.end()),
TStringType(value.begin(), value.end()));
- }
- for (const auto& [name, value] : ctx.GetServerTrailingMetadata()) {
- metadata.emplace(
+ }
+ for (const auto& [name, value] : ctx.GetServerTrailingMetadata()) {
+ metadata.emplace(
TStringType(name.begin(), name.end()),
TStringType(value.begin(), value.end()));
- }
-
- auto resp = new TResult<TResponse>(
- std::move(response),
- std::move(grpcStatus),
- std::move(userResponseCb),
- this,
- std::move(context),
- endpoint,
- std::move(metadata));
-
- EnqueueResponse(resp);
- } else {
- dbState->StatCollector.IncReqFailDueTransportError();
- dbState->StatCollector.IncTransportErrorsByHost(endpoint);
-
- auto resp = new TGRpcErrorResponse<TResponse>(
- std::move(grpcStatus),
- std::move(userResponseCb),
- this,
- std::move(context),
- endpoint);
-
- dbState->EndpointPool.BanEndpoint(endpoint);
-
- EnqueueResponse(resp);
- }
- };
-
- serviceConnection->DoAdvancedRequest(std::move(request), std::move(responseCbLow), rpc, meta,
- context.get());
+ }
+
+ auto resp = new TResult<TResponse>(
+ std::move(response),
+ std::move(grpcStatus),
+ std::move(userResponseCb),
+ this,
+ std::move(context),
+ endpoint,
+ std::move(metadata));
+
+ EnqueueResponse(resp);
+ } else {
+ dbState->StatCollector.IncReqFailDueTransportError();
+ dbState->StatCollector.IncTransportErrorsByHost(endpoint);
+
+ auto resp = new TGRpcErrorResponse<TResponse>(
+ std::move(grpcStatus),
+ std::move(userResponseCb),
+ this,
+ std::move(context),
+ endpoint);
+
+ dbState->EndpointPool.BanEndpoint(endpoint);
+
+ EnqueueResponse(resp);
+ }
+ };
+
+ serviceConnection->DoAdvancedRequest(std::move(request), std::move(responseCbLow), rpc, meta,
+ context.get());
}, dbState, preferredEndpoint, endpointPolicy);
- }
-
- template<typename TService, typename TRequest, typename TResponse>
- void RunDeferred(
- TRequest&& request,
- TDeferredOperationCb&& userResponseCb,
- TSimpleRpc<TService, TRequest, TResponse> rpc,
- TDbDriverStatePtr dbState,
- TDuration deferredTimeout,
- const TRpcRequestSettings& requestSettings,
- TDuration clientTimeout,
- bool poll = false,
+ }
+
+ template<typename TService, typename TRequest, typename TResponse>
+ void RunDeferred(
+ TRequest&& request,
+ TDeferredOperationCb&& userResponseCb,
+ TSimpleRpc<TService, TRequest, TResponse> rpc,
+ TDbDriverStatePtr dbState,
+ TDuration deferredTimeout,
+ const TRpcRequestSettings& requestSettings,
+ TDuration clientTimeout,
+ bool poll = false,
const TStringType& preferredEndpoint = TStringType(),
- std::shared_ptr<IQueueClientContext> context = nullptr)
- {
- if (!TryCreateContext(context)) {
- TPlainStatus status(EStatus::CLIENT_CANCELLED, "Client is stopped");
- userResponseCb(nullptr, status);
- return;
- }
-
- auto responseCb = [this, userResponseCb = std::move(userResponseCb), dbState, deferredTimeout, poll, context]
- (TResponse* response, TPlainStatus status) mutable
- {
- if (response) {
- Ydb::Operations::Operation* operation = response->mutable_operation();
- if (!operation->ready() && poll) {
- auto action = MakeIntrusive<TDeferredAction>(
- operation->id(),
- std::move(userResponseCb),
- this,
- std::move(context),
- deferredTimeout,
- dbState,
- status.Endpoint);
-
- action->Start();
- } else {
- NYql::TIssues opIssues;
- NYql::IssuesFromMessage(operation->issues(), opIssues);
- userResponseCb(operation, TPlainStatus{static_cast<EStatus>(operation->status()), std::move(opIssues),
- status.Endpoint, std::move(status.Metadata)});
- }
- } else {
- userResponseCb(nullptr, status);
- }
- };
-
- Run<TService, TRequest, TResponse>(
- std::move(request),
- responseCb,
- rpc,
- dbState,
- requestSettings,
- clientTimeout,
- preferredEndpoint,
- std::move(context));
- }
-
+ std::shared_ptr<IQueueClientContext> context = nullptr)
+ {
+ if (!TryCreateContext(context)) {
+ TPlainStatus status(EStatus::CLIENT_CANCELLED, "Client is stopped");
+ userResponseCb(nullptr, status);
+ return;
+ }
+
+ auto responseCb = [this, userResponseCb = std::move(userResponseCb), dbState, deferredTimeout, poll, context]
+ (TResponse* response, TPlainStatus status) mutable
+ {
+ if (response) {
+ Ydb::Operations::Operation* operation = response->mutable_operation();
+ if (!operation->ready() && poll) {
+ auto action = MakeIntrusive<TDeferredAction>(
+ operation->id(),
+ std::move(userResponseCb),
+ this,
+ std::move(context),
+ deferredTimeout,
+ dbState,
+ status.Endpoint);
+
+ action->Start();
+ } else {
+ NYql::TIssues opIssues;
+ NYql::IssuesFromMessage(operation->issues(), opIssues);
+ userResponseCb(operation, TPlainStatus{static_cast<EStatus>(operation->status()), std::move(opIssues),
+ status.Endpoint, std::move(status.Metadata)});
+ }
+ } else {
+ userResponseCb(nullptr, status);
+ }
+ };
+
+ Run<TService, TRequest, TResponse>(
+ std::move(request),
+ responseCb,
+ rpc,
+ dbState,
+ requestSettings,
+ clientTimeout,
+ preferredEndpoint,
+ std::move(context));
+ }
+
// Run request using discovery endpoint.
// Mostly usefull to make calls from credential provider
- template<typename TService, typename TRequest, typename TResponse>
+ template<typename TService, typename TRequest, typename TResponse>
static void RunOnDiscoveryEndpoint(
std::shared_ptr<ICoreFacility> facility,
TRequest&& request,
@@ -346,281 +346,281 @@ public:
}
template<typename TService, typename TRequest, typename TResponse>
- void RunDeferred(
- TRequest&& request,
- TDeferredResultCb&& userResponseCb,
- TSimpleRpc<TService, TRequest, TResponse> rpc,
- TDbDriverStatePtr dbState,
- TDuration deferredTimeout,
- const TRpcRequestSettings& requestSettings,
- TDuration clientTimeout,
+ void RunDeferred(
+ TRequest&& request,
+ TDeferredResultCb&& userResponseCb,
+ TSimpleRpc<TService, TRequest, TResponse> rpc,
+ TDbDriverStatePtr dbState,
+ TDuration deferredTimeout,
+ const TRpcRequestSettings& requestSettings,
+ TDuration clientTimeout,
const TStringType& preferredEndpoint = TStringType(),
- std::shared_ptr<IQueueClientContext> context = nullptr)
- {
- auto operationCb = [userResponseCb = std::move(userResponseCb)](Ydb::Operations::Operation* operation, TPlainStatus status) mutable {
+ std::shared_ptr<IQueueClientContext> context = nullptr)
+ {
+ auto operationCb = [userResponseCb = std::move(userResponseCb)](Ydb::Operations::Operation* operation, TPlainStatus status) mutable {
if (operation) {
status.SetCostInfo(std::move(*operation->mutable_cost_info()));
userResponseCb(operation->mutable_result(), std::move(status));
} else {
userResponseCb(nullptr, std::move(status));
}
- };
-
- RunDeferred<TService, TRequest, TResponse>(
- std::move(request),
- operationCb,
- rpc,
- dbState,
- deferredTimeout,
- requestSettings,
- clientTimeout,
- true, // poll
- preferredEndpoint,
- context);
- }
-
- template<class TService, class TRequest, class TResponse, template<typename TA, typename TB, typename TC> class TStream>
- using TStreamRpc =
- typename TStream<
- typename TService::Stub,
- TRequest,
- TResponse>::TAsyncRequest;
-
- template<class TService, class TRequest, class TResponse, class TCallback>
- void StartReadStream(
- const TRequest& request,
- TCallback responseCb,
- TStreamRpc<TService, TRequest, TResponse, NGrpc::TStreamRequestReadProcessor> rpc,
- TDbDriverStatePtr dbState,
- const TRpcRequestSettings& requestSettings,
- std::shared_ptr<IQueueClientContext> context = nullptr)
- {
- using NGrpc::TGrpcStatus;
- using TConnection = std::unique_ptr<TServiceConnection<TService>>;
- using TProcessor = typename NGrpc::IStreamRequestReadProcessor<TResponse>::TPtr;
-
- if (!TryCreateContext(context)) {
- responseCb(TPlainStatus(EStatus::CLIENT_CANCELLED, "Client is stopped"), nullptr);
- return;
- }
-
+ };
+
+ RunDeferred<TService, TRequest, TResponse>(
+ std::move(request),
+ operationCb,
+ rpc,
+ dbState,
+ deferredTimeout,
+ requestSettings,
+ clientTimeout,
+ true, // poll
+ preferredEndpoint,
+ context);
+ }
+
+ template<class TService, class TRequest, class TResponse, template<typename TA, typename TB, typename TC> class TStream>
+ using TStreamRpc =
+ typename TStream<
+ typename TService::Stub,
+ TRequest,
+ TResponse>::TAsyncRequest;
+
+ template<class TService, class TRequest, class TResponse, class TCallback>
+ void StartReadStream(
+ const TRequest& request,
+ TCallback responseCb,
+ TStreamRpc<TService, TRequest, TResponse, NGrpc::TStreamRequestReadProcessor> rpc,
+ TDbDriverStatePtr dbState,
+ const TRpcRequestSettings& requestSettings,
+ std::shared_ptr<IQueueClientContext> context = nullptr)
+ {
+ using NGrpc::TGrpcStatus;
+ using TConnection = std::unique_ptr<TServiceConnection<TService>>;
+ using TProcessor = typename NGrpc::IStreamRequestReadProcessor<TResponse>::TPtr;
+
+ if (!TryCreateContext(context)) {
+ responseCb(TPlainStatus(EStatus::CLIENT_CANCELLED, "Client is stopped"), nullptr);
+ return;
+ }
+
auto endpointPolicy = requestSettings.EndpointPolicy;
- WithServiceConnection<TService>(
- [request, responseCb = std::move(responseCb), rpc, requestSettings, context = std::move(context), dbState]
+ WithServiceConnection<TService>(
+ [request, responseCb = std::move(responseCb), rpc, requestSettings, context = std::move(context), dbState]
(TPlainStatus status, TConnection serviceConnection, TStringType endpoint) mutable {
- if (!status.Ok()) {
- responseCb(std::move(status), nullptr);
- return;
- }
-
- TCallMeta meta;
-#ifndef YDB_GRPC_UNSECURE_AUTH
- meta.CallCredentials = dbState->CallCredentials;
-#else
+ if (!status.Ok()) {
+ responseCb(std::move(status), nullptr);
+ return;
+ }
+
+ TCallMeta meta;
+#ifndef YDB_GRPC_UNSECURE_AUTH
+ meta.CallCredentials = dbState->CallCredentials;
+#else
if (requestSettings.UseAuth && dbState->CredentialsProvider && dbState->CredentialsProvider->IsValid()) {
- try {
- meta.Aux.push_back({ YDB_AUTH_TICKET_HEADER, GetAuthInfo(dbState) });
- } catch (const yexception& e) {
- responseCb(
- TPlainStatus(
- EStatus::CLIENT_UNAUTHENTICATED,
- TStringBuilder() << "Can't get Authentication info from CredentialsProvider. " << e.what()
- ),
- nullptr
- );
- return;
- }
- }
-#endif
- if (!requestSettings.TraceId.empty()) {
- meta.Aux.push_back({YDB_TRACE_ID_HEADER, requestSettings.TraceId});
- }
-
- if (!requestSettings.RequestType.empty()) {
- meta.Aux.push_back({YDB_REQUEST_TYPE_HEADER, requestSettings.RequestType});
- }
-
+ try {
+ meta.Aux.push_back({ YDB_AUTH_TICKET_HEADER, GetAuthInfo(dbState) });
+ } catch (const yexception& e) {
+ responseCb(
+ TPlainStatus(
+ EStatus::CLIENT_UNAUTHENTICATED,
+ TStringBuilder() << "Can't get Authentication info from CredentialsProvider. " << e.what()
+ ),
+ nullptr
+ );
+ return;
+ }
+ }
+#endif
+ if (!requestSettings.TraceId.empty()) {
+ meta.Aux.push_back({YDB_TRACE_ID_HEADER, requestSettings.TraceId});
+ }
+
+ if (!requestSettings.RequestType.empty()) {
+ meta.Aux.push_back({YDB_REQUEST_TYPE_HEADER, requestSettings.RequestType});
+ }
+
if (!dbState->Database.empty()) {
- SetDatabaseHeader(meta, dbState->Database);
- }
-
- dbState->StatCollector.IncGRpcInFlight();
- dbState->StatCollector.IncGRpcInFlightByHost(endpoint);
-
- auto lowCallback = [responseCb = std::move(responseCb), dbState, endpoint]
- (TGrpcStatus grpcStatus, TProcessor processor) mutable {
- dbState->StatCollector.DecGRpcInFlight();
- dbState->StatCollector.DecGRpcInFlightByHost(endpoint);
-
- if (grpcStatus.Ok()) {
- Y_VERIFY(processor);
- auto finishedCallback = [dbState, endpoint] (TGrpcStatus grpcStatus) {
- if (!grpcStatus.Ok() && grpcStatus.GRpcStatusCode != grpc::StatusCode::CANCELLED) {
- dbState->EndpointPool.BanEndpoint(endpoint);
- }
- };
- processor->AddFinishedCallback(std::move(finishedCallback));
- // TODO: Add headers for streaming calls.
- TPlainStatus status(std::move(grpcStatus), endpoint, {});
- responseCb(std::move(status), std::move(processor));
- } else {
- dbState->StatCollector.IncReqFailDueTransportError();
- dbState->StatCollector.IncTransportErrorsByHost(endpoint);
- if (grpcStatus.GRpcStatusCode != grpc::StatusCode::CANCELLED) {
- dbState->EndpointPool.BanEndpoint(endpoint);
- }
- // TODO: Add headers for streaming calls.
- TPlainStatus status(std::move(grpcStatus), endpoint, {});
- responseCb(std::move(status), nullptr);
- }
- };
-
- serviceConnection->template DoStreamRequest<TRequest, TResponse>(
- request,
- std::move(lowCallback),
- std::move(rpc),
- std::move(meta),
- context.get());
+ SetDatabaseHeader(meta, dbState->Database);
+ }
+
+ dbState->StatCollector.IncGRpcInFlight();
+ dbState->StatCollector.IncGRpcInFlightByHost(endpoint);
+
+ auto lowCallback = [responseCb = std::move(responseCb), dbState, endpoint]
+ (TGrpcStatus grpcStatus, TProcessor processor) mutable {
+ dbState->StatCollector.DecGRpcInFlight();
+ dbState->StatCollector.DecGRpcInFlightByHost(endpoint);
+
+ if (grpcStatus.Ok()) {
+ Y_VERIFY(processor);
+ auto finishedCallback = [dbState, endpoint] (TGrpcStatus grpcStatus) {
+ if (!grpcStatus.Ok() && grpcStatus.GRpcStatusCode != grpc::StatusCode::CANCELLED) {
+ dbState->EndpointPool.BanEndpoint(endpoint);
+ }
+ };
+ processor->AddFinishedCallback(std::move(finishedCallback));
+ // TODO: Add headers for streaming calls.
+ TPlainStatus status(std::move(grpcStatus), endpoint, {});
+ responseCb(std::move(status), std::move(processor));
+ } else {
+ dbState->StatCollector.IncReqFailDueTransportError();
+ dbState->StatCollector.IncTransportErrorsByHost(endpoint);
+ if (grpcStatus.GRpcStatusCode != grpc::StatusCode::CANCELLED) {
+ dbState->EndpointPool.BanEndpoint(endpoint);
+ }
+ // TODO: Add headers for streaming calls.
+ TPlainStatus status(std::move(grpcStatus), endpoint, {});
+ responseCb(std::move(status), nullptr);
+ }
+ };
+
+ serviceConnection->template DoStreamRequest<TRequest, TResponse>(
+ request,
+ std::move(lowCallback),
+ std::move(rpc),
+ std::move(meta),
+ context.get());
}, dbState, TStringType(), endpointPolicy);
- }
-
- template<class TService, class TRequest, class TResponse, class TCallback>
- void StartBidirectionalStream(
- TCallback connectedCallback,
- TStreamRpc<TService, TRequest, TResponse, NGrpc::TStreamRequestReadWriteProcessor> rpc,
- TDbDriverStatePtr dbState,
- const TRpcRequestSettings& requestSettings,
- std::shared_ptr<IQueueClientContext> context = nullptr)
- {
- using NGrpc::TGrpcStatus;
- using TConnection = std::unique_ptr<TServiceConnection<TService>>;
- using TProcessor = typename NGrpc::IStreamRequestReadWriteProcessor<TRequest, TResponse>::TPtr;
-
- if (!TryCreateContext(context)) {
- connectedCallback(TPlainStatus(EStatus::CLIENT_CANCELLED, "Client is stopped"), nullptr);
- return;
- }
-
+ }
+
+ template<class TService, class TRequest, class TResponse, class TCallback>
+ void StartBidirectionalStream(
+ TCallback connectedCallback,
+ TStreamRpc<TService, TRequest, TResponse, NGrpc::TStreamRequestReadWriteProcessor> rpc,
+ TDbDriverStatePtr dbState,
+ const TRpcRequestSettings& requestSettings,
+ std::shared_ptr<IQueueClientContext> context = nullptr)
+ {
+ using NGrpc::TGrpcStatus;
+ using TConnection = std::unique_ptr<TServiceConnection<TService>>;
+ using TProcessor = typename NGrpc::IStreamRequestReadWriteProcessor<TRequest, TResponse>::TPtr;
+
+ if (!TryCreateContext(context)) {
+ connectedCallback(TPlainStatus(EStatus::CLIENT_CANCELLED, "Client is stopped"), nullptr);
+ return;
+ }
+
auto endpointPolicy = requestSettings.EndpointPolicy;
- WithServiceConnection<TService>(
- [connectedCallback = std::move(connectedCallback), rpc, requestSettings, context = std::move(context), dbState]
+ WithServiceConnection<TService>(
+ [connectedCallback = std::move(connectedCallback), rpc, requestSettings, context = std::move(context), dbState]
(TPlainStatus status, TConnection serviceConnection, TStringType endpoint) mutable {
- if (!status.Ok()) {
- connectedCallback(std::move(status), nullptr);
- return;
- }
-
- TCallMeta meta;
- #ifndef YDB_GRPC_UNSECURE_AUTH
- meta.CallCredentials = dbState->CallCredentials;
- #else
+ if (!status.Ok()) {
+ connectedCallback(std::move(status), nullptr);
+ return;
+ }
+
+ TCallMeta meta;
+ #ifndef YDB_GRPC_UNSECURE_AUTH
+ meta.CallCredentials = dbState->CallCredentials;
+ #else
if (requestSettings.UseAuth && dbState->CredentialsProvider && dbState->CredentialsProvider->IsValid()) {
- try {
- meta.Aux.push_back({ YDB_AUTH_TICKET_HEADER, GetAuthInfo(dbState) });
- } catch (const yexception& e) {
- connectedCallback(
- TPlainStatus(
- EStatus::CLIENT_UNAUTHENTICATED,
- TStringBuilder() << "Can't get Authentication info from CredentialsProvider. " << e.what()
- ),
- nullptr
- );
- return;
- }
- }
- #endif
- if (!requestSettings.TraceId.empty()) {
- meta.Aux.push_back({YDB_TRACE_ID_HEADER, requestSettings.TraceId});
- }
-
- if (!requestSettings.RequestType.empty()) {
- meta.Aux.push_back({YDB_REQUEST_TYPE_HEADER, requestSettings.RequestType});
- }
-
+ try {
+ meta.Aux.push_back({ YDB_AUTH_TICKET_HEADER, GetAuthInfo(dbState) });
+ } catch (const yexception& e) {
+ connectedCallback(
+ TPlainStatus(
+ EStatus::CLIENT_UNAUTHENTICATED,
+ TStringBuilder() << "Can't get Authentication info from CredentialsProvider. " << e.what()
+ ),
+ nullptr
+ );
+ return;
+ }
+ }
+ #endif
+ if (!requestSettings.TraceId.empty()) {
+ meta.Aux.push_back({YDB_TRACE_ID_HEADER, requestSettings.TraceId});
+ }
+
+ if (!requestSettings.RequestType.empty()) {
+ meta.Aux.push_back({YDB_REQUEST_TYPE_HEADER, requestSettings.RequestType});
+ }
+
if (!dbState->Database.empty()) {
- SetDatabaseHeader(meta, dbState->Database);
- }
-
- dbState->StatCollector.IncGRpcInFlight();
- dbState->StatCollector.IncGRpcInFlightByHost(endpoint);
-
- auto lowCallback = [connectedCallback = std::move(connectedCallback), dbState, endpoint]
- (TGrpcStatus grpcStatus, TProcessor processor) {
- dbState->StatCollector.DecGRpcInFlight();
- dbState->StatCollector.DecGRpcInFlightByHost(endpoint);
-
- if (grpcStatus.Ok()) {
- Y_VERIFY(processor);
- auto finishedCallback = [dbState, endpoint] (TGrpcStatus grpcStatus) {
- if (!grpcStatus.Ok() && grpcStatus.GRpcStatusCode != grpc::StatusCode::CANCELLED) {
- dbState->EndpointPool.BanEndpoint(endpoint);
- }
- };
- processor->AddFinishedCallback(std::move(finishedCallback));
- // TODO: Add headers for streaming calls.
- TPlainStatus status(std::move(grpcStatus), endpoint, {});
- connectedCallback(std::move(status), std::move(processor));
- } else {
- dbState->StatCollector.IncReqFailDueTransportError();
- dbState->StatCollector.IncTransportErrorsByHost(endpoint);
- if (grpcStatus.GRpcStatusCode != grpc::StatusCode::CANCELLED) {
- dbState->EndpointPool.BanEndpoint(endpoint);
- }
- // TODO: Add headers for streaming calls.
- TPlainStatus status(std::move(grpcStatus), endpoint, {});
- connectedCallback(std::move(status), nullptr);
- }
- };
-
- serviceConnection->template DoStreamRequest<TRequest, TResponse>(
- std::move(lowCallback),
- std::move(rpc),
- std::move(meta),
- context.get());
+ SetDatabaseHeader(meta, dbState->Database);
+ }
+
+ dbState->StatCollector.IncGRpcInFlight();
+ dbState->StatCollector.IncGRpcInFlightByHost(endpoint);
+
+ auto lowCallback = [connectedCallback = std::move(connectedCallback), dbState, endpoint]
+ (TGrpcStatus grpcStatus, TProcessor processor) {
+ dbState->StatCollector.DecGRpcInFlight();
+ dbState->StatCollector.DecGRpcInFlightByHost(endpoint);
+
+ if (grpcStatus.Ok()) {
+ Y_VERIFY(processor);
+ auto finishedCallback = [dbState, endpoint] (TGrpcStatus grpcStatus) {
+ if (!grpcStatus.Ok() && grpcStatus.GRpcStatusCode != grpc::StatusCode::CANCELLED) {
+ dbState->EndpointPool.BanEndpoint(endpoint);
+ }
+ };
+ processor->AddFinishedCallback(std::move(finishedCallback));
+ // TODO: Add headers for streaming calls.
+ TPlainStatus status(std::move(grpcStatus), endpoint, {});
+ connectedCallback(std::move(status), std::move(processor));
+ } else {
+ dbState->StatCollector.IncReqFailDueTransportError();
+ dbState->StatCollector.IncTransportErrorsByHost(endpoint);
+ if (grpcStatus.GRpcStatusCode != grpc::StatusCode::CANCELLED) {
+ dbState->EndpointPool.BanEndpoint(endpoint);
+ }
+ // TODO: Add headers for streaming calls.
+ TPlainStatus status(std::move(grpcStatus), endpoint, {});
+ connectedCallback(std::move(status), nullptr);
+ }
+ };
+
+ serviceConnection->template DoStreamRequest<TRequest, TResponse>(
+ std::move(lowCallback),
+ std::move(rpc),
+ std::move(meta),
+ context.get());
}, dbState, TStringType(), endpointPolicy);
- }
-
+ }
+
TAsyncListEndpointsResult GetEndpoints(TDbDriverStatePtr dbState) override;
TListEndpointsResult MutateDiscovery(TListEndpointsResult result, const TStringType& database);
-
-#ifndef YDB_GRPC_BYPASS_CHANNEL_POOL
+
+#ifndef YDB_GRPC_BYPASS_CHANNEL_POOL
void DeleteChannels(const std::vector<TStringType>& endpoints) override {
- for (const auto& endpoint : endpoints) {
- ChannelPool_.DeleteChannel(endpoint);
- }
- }
-#endif
-
- bool GetDrainOnDtors() const;
- TBalancingSettings GetBalancingSettings() const override;
- bool StartStatCollecting(NMonitoring::IMetricRegistry* sensorsRegistry) override;
- NMonitoring::TMetricRegistry* GetMetricRegistry() override;
- void RegisterExtension(IExtension* extension);
- void RegisterExtensionApi(IExtensionApi* api);
- void SetDiscoveryMutator(IDiscoveryMutatorApi::TMutatorCb&& cb);
- const TLog& GetLog() const override;
-
-private:
- template <typename TService, typename TCallback>
+ for (const auto& endpoint : endpoints) {
+ ChannelPool_.DeleteChannel(endpoint);
+ }
+ }
+#endif
+
+ bool GetDrainOnDtors() const;
+ TBalancingSettings GetBalancingSettings() const override;
+ bool StartStatCollecting(NMonitoring::IMetricRegistry* sensorsRegistry) override;
+ NMonitoring::TMetricRegistry* GetMetricRegistry() override;
+ void RegisterExtension(IExtension* extension);
+ void RegisterExtensionApi(IExtensionApi* api);
+ void SetDiscoveryMutator(IDiscoveryMutatorApi::TMutatorCb&& cb);
+ const TLog& GetLog() const override;
+
+private:
+ template <typename TService, typename TCallback>
void WithServiceConnection(TCallback callback, TDbDriverStatePtr dbState,
const TStringType& preferredEndpoint, TRpcRequestSettings::TEndpointPolicy endpointPolicy)
{
- using TConnection = std::unique_ptr<TServiceConnection<TService>>;
- TConnection serviceConnection;
+ using TConnection = std::unique_ptr<TServiceConnection<TService>>;
+ TConnection serviceConnection;
TStringType endpoint;
std::tie(serviceConnection, endpoint) = GetServiceConnection<TService>(dbState, preferredEndpoint, endpointPolicy);
- if (!serviceConnection) {
- if (dbState->DiscoveryMode == EDiscoveryMode::Sync) {
- TStringStream errString;
- errString << "Endpoint list is empty for database " << dbState->Database;
- errString << ", cluster endpoint " << dbState->DiscoveryEndpoint;
+ if (!serviceConnection) {
+ if (dbState->DiscoveryMode == EDiscoveryMode::Sync) {
+ TStringStream errString;
+ errString << "Endpoint list is empty for database " << dbState->Database;
+ errString << ", cluster endpoint " << dbState->DiscoveryEndpoint;
TPlainStatus discoveryStatus;
- {
+ {
std::shared_lock guard(dbState->LastDiscoveryStatusRWLock);
discoveryStatus = dbState->LastDiscoveryStatus;
- }
+ }
// Discovery returns success but no serviceConnection - in this case we unable to continue processing
if (discoveryStatus.Ok()) {
@@ -630,96 +630,96 @@ private:
errString <<".";
discoveryStatus.Issues.AddIssues({NYql::TIssue(errString.Str())});
}
- dbState->StatCollector.IncReqFailNoEndpoint();
- callback(
- discoveryStatus,
- TConnection{nullptr},
+ dbState->StatCollector.IncReqFailNoEndpoint();
+ callback(
+ discoveryStatus,
+ TConnection{nullptr},
TStringType{ });
- } else {
+ } else {
int64_t newVal;
int64_t val;
- do {
+ do {
val = QueuedRequests_.load();
- if (val >= MaxQueuedRequests_) {
- dbState->StatCollector.IncReqFailQueueOverflow();
- callback(
- TPlainStatus(EStatus::CLIENT_LIMITS_REACHED, "Requests queue limit reached"),
- TConnection{nullptr},
+ if (val >= MaxQueuedRequests_) {
+ dbState->StatCollector.IncReqFailQueueOverflow();
+ callback(
+ TPlainStatus(EStatus::CLIENT_LIMITS_REACHED, "Requests queue limit reached"),
+ TConnection{nullptr},
TStringType{ });
- return;
- }
- newVal = val + 1;
+ return;
+ }
+ newVal = val + 1;
} while (!QueuedRequests_.compare_exchange_weak(val, newVal));
-
- // UpdateAsync guarantee one update in progress for state
- auto asyncResult = dbState->EndpointPool.UpdateAsync();
- const bool needUpdateChannels = asyncResult.second;
+
+ // UpdateAsync guarantee one update in progress for state
+ auto asyncResult = dbState->EndpointPool.UpdateAsync();
+ const bool needUpdateChannels = asyncResult.second;
asyncResult.first.Subscribe([this, callback = std::move(callback), needUpdateChannels, dbState, preferredEndpoint, endpointPolicy]
- (const NThreading::TFuture<TEndpointUpdateResult>& future) mutable {
+ (const NThreading::TFuture<TEndpointUpdateResult>& future) mutable {
--QueuedRequests_;
- const auto& updateResult = future.GetValue();
- if (needUpdateChannels) {
-#ifndef YDB_GRPC_BYPASS_CHANNEL_POOL
- DeleteChannels(updateResult.Removed);
-#endif
- }
- auto discoveryStatus = updateResult.DiscoveryStatus;
- if (discoveryStatus.Status == EStatus::SUCCESS) {
+ const auto& updateResult = future.GetValue();
+ if (needUpdateChannels) {
+#ifndef YDB_GRPC_BYPASS_CHANNEL_POOL
+ DeleteChannels(updateResult.Removed);
+#endif
+ }
+ auto discoveryStatus = updateResult.DiscoveryStatus;
+ if (discoveryStatus.Status == EStatus::SUCCESS) {
WithServiceConnection<TService>(std::move(callback), dbState, preferredEndpoint, endpointPolicy);
- } else {
- callback(
- TPlainStatus(discoveryStatus.Status, std::move(discoveryStatus.Issues)),
- TConnection{nullptr},
+ } else {
+ callback(
+ TPlainStatus(discoveryStatus.Status, std::move(discoveryStatus.Issues)),
+ TConnection{nullptr},
TStringType{ });
- }
- });
- }
- return;
- }
-
- callback(
- TPlainStatus{ },
- std::move(serviceConnection),
- std::move(endpoint));
- }
-
- void EnqueueResponse(IObjectInQueue* action);
-
-private:
+ }
+ });
+ }
+ return;
+ }
+
+ callback(
+ TPlainStatus{ },
+ std::move(serviceConnection),
+ std::move(endpoint));
+ }
+
+ void EnqueueResponse(IObjectInQueue* action);
+
+private:
std::mutex ExtensionsLock_;
- NMonitoring::TMetricRegistry* MetricRegistryPtr_ = nullptr;
-
- std::unique_ptr<IThreadPool> ResponseQueue_;
-
+ NMonitoring::TMetricRegistry* MetricRegistryPtr_ = nullptr;
+
+ std::unique_ptr<IThreadPool> ResponseQueue_;
+
const TStringType DefaultDiscoveryEndpoint_;
- const bool EnableSsl_;
+ const bool EnableSsl_;
const TStringType CaCert_;
const TStringType DefaultDatabase_;
- std::shared_ptr<ICredentialsProviderFactory> DefaultCredentialsProviderFactory_;
- TDbDriverStateTracker StateTracker_;
- const EDiscoveryMode DefaultDiscoveryMode_;
- const i64 MaxQueuedRequests_;
- const bool DrainOnDtors_;
- const TBalancingSettings BalancingSettings_;
- const TDuration GRpcKeepAliveTimeout_;
- const bool GRpcKeepAlivePermitWithoutCalls_;
+ std::shared_ptr<ICredentialsProviderFactory> DefaultCredentialsProviderFactory_;
+ TDbDriverStateTracker StateTracker_;
+ const EDiscoveryMode DefaultDiscoveryMode_;
+ const i64 MaxQueuedRequests_;
+ const bool DrainOnDtors_;
+ const TBalancingSettings BalancingSettings_;
+ const TDuration GRpcKeepAliveTimeout_;
+ const bool GRpcKeepAlivePermitWithoutCalls_;
const ui64 MemoryQuota_;
-
+
std::atomic_int64_t QueuedRequests_;
-#ifndef YDB_GRPC_BYPASS_CHANNEL_POOL
- NGrpc::TChannelPool ChannelPool_;
-#endif
- // State for default database, token pair
- TDbDriverStatePtr DefaultState_;
-
+#ifndef YDB_GRPC_BYPASS_CHANNEL_POOL
+ NGrpc::TChannelPool ChannelPool_;
+#endif
+ // State for default database, token pair
+ TDbDriverStatePtr DefaultState_;
+
std::vector<std::unique_ptr<IExtension>> Extensions_;
std::vector<std::unique_ptr<IExtensionApi>> ExtensionApis_;
-
- IDiscoveryMutatorApi::TMutatorCb DiscoveryMutatorCb;
-
- // Must be the last member (first called destructor)
- NGrpc::TGRpcClientLow GRpcClientLow_;
- TLog Log;
-};
-
-} // namespace NYdb
+
+ IDiscoveryMutatorApi::TMutatorCb DiscoveryMutatorCb;
+
+ // Must be the last member (first called destructor)
+ NGrpc::TGRpcClientLow GRpcClientLow_;
+ TLog Log;
+};
+
+} // namespace NYdb
diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/params.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/params.h
index 2425d6883b..7833c60947 100644
--- a/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/params.h
+++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/params.h
@@ -1,30 +1,30 @@
-#pragma once
-
+#pragma once
+
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/internal_header.h>
-
-namespace NYdb {
-
-class IConnectionsParams {
-public:
- virtual ~IConnectionsParams() = default;
+
+namespace NYdb {
+
+class IConnectionsParams {
+public:
+ virtual ~IConnectionsParams() = default;
virtual TStringType GetEndpoint() const = 0;
- virtual size_t GetNetworkThreadsNum() const = 0;
- virtual size_t GetClientThreadsNum() const = 0;
- virtual size_t GetMaxQueuedResponses() const = 0;
- virtual bool IsSslEnabled() const = 0;
+ virtual size_t GetNetworkThreadsNum() const = 0;
+ virtual size_t GetClientThreadsNum() const = 0;
+ virtual size_t GetMaxQueuedResponses() const = 0;
+ virtual bool IsSslEnabled() const = 0;
virtual TStringType GetCaCert() const = 0;
virtual TStringType GetDatabase() const = 0;
- virtual std::shared_ptr<ICredentialsProviderFactory> GetCredentialsProviderFactory() const = 0;
- virtual EDiscoveryMode GetDiscoveryMode() const = 0;
- virtual size_t GetMaxQueuedRequests() const = 0;
- virtual NGrpc::TTcpKeepAliveSettings GetTcpKeepAliveSettings() const = 0;
- virtual bool GetDrinOnDtors() const = 0;
- virtual TBalancingSettings GetBalancingSettings() const = 0;
- virtual TDuration GetGRpcKeepAliveTimeout() const = 0;
- virtual bool GetGRpcKeepAlivePermitWithoutCalls() const = 0;
+ virtual std::shared_ptr<ICredentialsProviderFactory> GetCredentialsProviderFactory() const = 0;
+ virtual EDiscoveryMode GetDiscoveryMode() const = 0;
+ virtual size_t GetMaxQueuedRequests() const = 0;
+ virtual NGrpc::TTcpKeepAliveSettings GetTcpKeepAliveSettings() const = 0;
+ virtual bool GetDrinOnDtors() const = 0;
+ virtual TBalancingSettings GetBalancingSettings() const = 0;
+ virtual TDuration GetGRpcKeepAliveTimeout() const = 0;
+ virtual bool GetGRpcKeepAlivePermitWithoutCalls() const = 0;
virtual TDuration GetSocketIdleTimeout() const = 0;
- virtual const TLog& GetLog() const = 0;
+ virtual const TLog& GetLog() const = 0;
virtual ui64 GetMemoryQuota() const = 0;
-};
-
-} // namespace NYdb
+};
+
+} // namespace NYdb
diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/ya.make b/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/ya.make
index a23d6e5872..5d79153da3 100644
--- a/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/ya.make
+++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/ya.make
@@ -1,13 +1,13 @@
-LIBRARY()
-
+LIBRARY()
+
OWNER(g:kikimr)
-
-SRCS(
- actions.cpp
- grpc_connections.cpp
-)
-
-PEERDIR(
+
+SRCS(
+ actions.cpp
+ grpc_connections.cpp
+)
+
+PEERDIR(
ydb/public/api/grpc
ydb/public/api/protos
ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state
@@ -16,6 +16,6 @@ PEERDIR(
ydb/public/sdk/cpp/client/impl/ydb_stats
ydb/public/sdk/cpp/client/resources
ydb/public/sdk/cpp/client/ydb_types/exceptions
-)
-
-END()
+)
+
+END()
diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/internal_client/client.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/internal_client/client.h
index 87340dc021..07fbadf88b 100644
--- a/ydb/public/sdk/cpp/client/impl/ydb_internal/internal_client/client.h
+++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/internal_client/client.h
@@ -1,37 +1,37 @@
-#pragma once
-
+#pragma once
+
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/internal_header.h>
-
+
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/common/type_switcher.h>
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/common/types.h>
#include <ydb/public/sdk/cpp/client/ydb_types/ydb.h>
#include <ydb/public/sdk/cpp/client/ydb_types/core_facility/core_facility.h>
-
-#include <library/cpp/threading/future/future.h>
-#include <library/cpp/logger/log.h>
-
-namespace NMonitoring {
- class IMetricRegistry;
- class TMetricRegistry;
-}
-
-namespace NYdb {
-
+
+#include <library/cpp/threading/future/future.h>
+#include <library/cpp/logger/log.h>
+
+namespace NMonitoring {
+ class IMetricRegistry;
+ class TMetricRegistry;
+}
+
+namespace NYdb {
+
class TDbDriverState;
-struct TListEndpointsResult;
-
-class IInternalClient {
-public:
- virtual NThreading::TFuture<TListEndpointsResult> GetEndpoints(std::shared_ptr<TDbDriverState> dbState) = 0;
- virtual void AddPeriodicTask(TPeriodicCb&& cb, TDuration period) = 0;
-#ifndef YDB_GRPC_BYPASS_CHANNEL_POOL
+struct TListEndpointsResult;
+
+class IInternalClient {
+public:
+ virtual NThreading::TFuture<TListEndpointsResult> GetEndpoints(std::shared_ptr<TDbDriverState> dbState) = 0;
+ virtual void AddPeriodicTask(TPeriodicCb&& cb, TDuration period) = 0;
+#ifndef YDB_GRPC_BYPASS_CHANNEL_POOL
virtual void DeleteChannels(const std::vector<TStringType>& endpoints) = 0;
-#endif
- virtual TBalancingSettings GetBalancingSettings() const = 0;
- virtual bool StartStatCollecting(NMonitoring::IMetricRegistry* sensorsRegistry) = 0;
- virtual NMonitoring::TMetricRegistry* GetMetricRegistry() = 0;
- virtual const TLog& GetLog() const = 0;
-};
-
-} // namespace NYdb
+#endif
+ virtual TBalancingSettings GetBalancingSettings() const = 0;
+ virtual bool StartStatCollecting(NMonitoring::IMetricRegistry* sensorsRegistry) = 0;
+ virtual NMonitoring::TMetricRegistry* GetMetricRegistry() = 0;
+ virtual const TLog& GetLog() const = 0;
+};
+
+} // namespace NYdb
diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/internal_header.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/internal_header.h
index 159f034437..dce5d14301 100644
--- a/ydb/public/sdk/cpp/client/impl/ydb_internal/internal_header.h
+++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/internal_header.h
@@ -1,11 +1,11 @@
-#pragma once
-
-#if !defined(INCLUDE_YDB_INTERNAL_H)
-#error "you are trying to use internal ydb header"
-#endif // INCLUDE_YDB_INTERNAL_H
-
-// This macro allow to send credentials data via insecure channel
-#define YDB_GRPC_UNSECURE_AUTH
-
-// This macro is used for grpc debug purpose only
-//#define YDB_GRPC_BYPASS_CHANNEL_POOL
+#pragma once
+
+#if !defined(INCLUDE_YDB_INTERNAL_H)
+#error "you are trying to use internal ydb header"
+#endif // INCLUDE_YDB_INTERNAL_H
+
+// This macro allow to send credentials data via insecure channel
+#define YDB_GRPC_UNSECURE_AUTH
+
+// This macro is used for grpc debug purpose only
+//#define YDB_GRPC_BYPASS_CHANNEL_POOL
diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/logger/log.cpp b/ydb/public/sdk/cpp/client/impl/ydb_internal/logger/log.cpp
index 6e0f44a3a1..da3751e1f8 100644
--- a/ydb/public/sdk/cpp/client/impl/ydb_internal/logger/log.cpp
+++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/logger/log.cpp
@@ -1,47 +1,47 @@
-#define INCLUDE_YDB_INTERNAL_H
-#include "log.h"
-
-#include <util/datetime/base.h>
-#include <util/string/builder.h>
-
-namespace NYdb {
-
-static const TStringBuf LogPrioritiesStrings[] = {
- " :EMERG: ",
- " :ALERT: ",
- " :CRIT: ",
- " :ERROR: ",
- " :WARNING: ",
- " :NOTICE: ",
- " :INFO: ",
- " :DEBUG: ",
- " :TRACE: ",
-};
-
-TStringBuf LogPriorityToString(ELogPriority priority) {
- constexpr int maxPriority = static_cast<int>(std::size(LogPrioritiesStrings) - 1);
- const int index = static_cast<int>(priority);
- return LogPrioritiesStrings[ClampVal(index, 0, maxPriority)];
-}
-
-TLogFormatter GetPrefixLogFormatter(const TString& prefix) {
- return [prefix](ELogPriority priority, TStringBuf message) -> TString {
- constexpr size_t timeLen = 27; // 2020-10-08T20:31:11.202588Z
- constexpr size_t endlLen = 1;
-
- const TStringBuf priorityString = LogPriorityToString(priority);
- TStringBuilder result;
- const size_t toReserve = prefix.size() + message.Size() + timeLen + endlLen + priorityString.Size();
- result.reserve(toReserve);
-
- result << TInstant::Now() << priorityString << prefix << message << Endl;
- Y_ASSERT(result.size() == toReserve);
- return std::move(result);
- };
-}
-
+#define INCLUDE_YDB_INTERNAL_H
+#include "log.h"
+
+#include <util/datetime/base.h>
+#include <util/string/builder.h>
+
+namespace NYdb {
+
+static const TStringBuf LogPrioritiesStrings[] = {
+ " :EMERG: ",
+ " :ALERT: ",
+ " :CRIT: ",
+ " :ERROR: ",
+ " :WARNING: ",
+ " :NOTICE: ",
+ " :INFO: ",
+ " :DEBUG: ",
+ " :TRACE: ",
+};
+
+TStringBuf LogPriorityToString(ELogPriority priority) {
+ constexpr int maxPriority = static_cast<int>(std::size(LogPrioritiesStrings) - 1);
+ const int index = static_cast<int>(priority);
+ return LogPrioritiesStrings[ClampVal(index, 0, maxPriority)];
+}
+
+TLogFormatter GetPrefixLogFormatter(const TString& prefix) {
+ return [prefix](ELogPriority priority, TStringBuf message) -> TString {
+ constexpr size_t timeLen = 27; // 2020-10-08T20:31:11.202588Z
+ constexpr size_t endlLen = 1;
+
+ const TStringBuf priorityString = LogPriorityToString(priority);
+ TStringBuilder result;
+ const size_t toReserve = prefix.size() + message.Size() + timeLen + endlLen + priorityString.Size();
+ result.reserve(toReserve);
+
+ result << TInstant::Now() << priorityString << prefix << message << Endl;
+ Y_ASSERT(result.size() == toReserve);
+ return std::move(result);
+ };
+}
+
TStringType GetDatabaseLogPrefix(const TStringType& database) {
return "[" + database + "] ";
-}
-
-} // namespace NYdb
+}
+
+} // namespace NYdb
diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/logger/log.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/logger/log.h
index 7271ef4edb..dc1ede5d29 100644
--- a/ydb/public/sdk/cpp/client/impl/ydb_internal/logger/log.h
+++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/logger/log.h
@@ -1,13 +1,13 @@
-#pragma once
-
+#pragma once
+
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/internal_header.h>
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/common/type_switcher.h>
-
-#include <library/cpp/logger/log.h>
-
-namespace NYdb {
-
-TLogFormatter GetPrefixLogFormatter(const TString& prefix);
+
+#include <library/cpp/logger/log.h>
+
+namespace NYdb {
+
+TLogFormatter GetPrefixLogFormatter(const TString& prefix);
TStringType GetDatabaseLogPrefix(const TStringType& database);
-
-} // namespace NYdb
+
+} // namespace NYdb
diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/logger/ya.make b/ydb/public/sdk/cpp/client/impl/ydb_internal/logger/ya.make
index 54a1e21d05..89a802f98d 100644
--- a/ydb/public/sdk/cpp/client/impl/ydb_internal/logger/ya.make
+++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/logger/ya.make
@@ -1,13 +1,13 @@
-LIBRARY()
-
+LIBRARY()
+
OWNER(g:kikimr)
-
-SRCS(
- log.cpp
-)
-
-PEERDIR(
- library/cpp/logger
-)
-
-END()
+
+SRCS(
+ log.cpp
+)
+
+PEERDIR(
+ library/cpp/logger
+)
+
+END()
diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.cpp b/ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.cpp
index e4eaf8e7ab..bfbf465529 100644
--- a/ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.cpp
+++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.cpp
@@ -1,12 +1,12 @@
-#define INCLUDE_YDB_INTERNAL_H
-#include "make.h"
-
-
-namespace NYdb {
-
-void SetDuration(const TDuration& duration, google::protobuf::Duration& protoValue) {
- protoValue.set_seconds(duration.Seconds());
- protoValue.set_nanos(duration.NanoSecondsOfSecond());
-}
-
-} // namespace NYdb
+#define INCLUDE_YDB_INTERNAL_H
+#include "make.h"
+
+
+namespace NYdb {
+
+void SetDuration(const TDuration& duration, google::protobuf::Duration& protoValue) {
+ protoValue.set_seconds(duration.Seconds());
+ protoValue.set_nanos(duration.NanoSecondsOfSecond());
+}
+
+} // namespace NYdb
diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.h
index 3cf3a4cffc..c055ee9609 100644
--- a/ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.h
+++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.h
@@ -1,45 +1,45 @@
-#pragma once
-
+#pragma once
+
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/internal_header.h>
#include <ydb/public/api/protos/ydb_common.pb.h>
-
-#include <util/datetime/base.h>
-
-#include <google/protobuf/duration.pb.h>
-
-namespace NYdb {
-
-void SetDuration(const TDuration& duration, google::protobuf::Duration& protoValue);
-
-template <typename TRequestSettings, typename TProtoRequest>
-void FillOperationParams(const TRequestSettings& settings, TProtoRequest& params) {
- auto& operationParams = *params.mutable_operation_params();
-
- if (settings.CancelAfter_) {
- SetDuration(settings.CancelAfter_, *operationParams.mutable_cancel_after());
- }
-
- if (settings.OperationTimeout_) {
- SetDuration(settings.OperationTimeout_, *operationParams.mutable_operation_timeout());
- } else if (settings.ClientTimeout_ && settings.UseClientTimeoutForOperation_) {
- SetDuration(settings.ClientTimeout_, *operationParams.mutable_operation_timeout());
- }
+
+#include <util/datetime/base.h>
+
+#include <google/protobuf/duration.pb.h>
+
+namespace NYdb {
+
+void SetDuration(const TDuration& duration, google::protobuf::Duration& protoValue);
+
+template <typename TRequestSettings, typename TProtoRequest>
+void FillOperationParams(const TRequestSettings& settings, TProtoRequest& params) {
+ auto& operationParams = *params.mutable_operation_params();
+
+ if (settings.CancelAfter_) {
+ SetDuration(settings.CancelAfter_, *operationParams.mutable_cancel_after());
+ }
+
+ if (settings.OperationTimeout_) {
+ SetDuration(settings.OperationTimeout_, *operationParams.mutable_operation_timeout());
+ } else if (settings.ClientTimeout_ && settings.UseClientTimeoutForOperation_) {
+ SetDuration(settings.ClientTimeout_, *operationParams.mutable_operation_timeout());
+ }
if (settings.ReportCostInfo_) {
operationParams.set_report_cost_info(Ydb::FeatureFlag::ENABLED);
}
-}
-
-template <typename TProtoRequest>
-TProtoRequest MakeRequest() {
- return TProtoRequest();
-}
-
-template <typename TProtoRequest, typename TRequestSettings>
-TProtoRequest MakeOperationRequest(const TRequestSettings& settings) {
- auto request = MakeRequest<TProtoRequest>();
- FillOperationParams(settings, request);
- return request;
-}
-
-} // namespace NYdb
+}
+
+template <typename TProtoRequest>
+TProtoRequest MakeRequest() {
+ return TProtoRequest();
+}
+
+template <typename TProtoRequest, typename TRequestSettings>
+TProtoRequest MakeOperationRequest(const TRequestSettings& settings) {
+ auto request = MakeRequest<TProtoRequest>();
+ FillOperationParams(settings, request);
+ return request;
+}
+
+} // namespace NYdb
diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/ya.make b/ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/ya.make
index 46518f378a..87d7f3f272 100644
--- a/ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/ya.make
+++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/ya.make
@@ -1,14 +1,14 @@
-LIBRARY()
-
+LIBRARY()
+
OWNER(g:kikimr)
-
-SRCS(
- make.cpp
-)
-
-PEERDIR(
- contrib/libs/protobuf
+
+SRCS(
+ make.cpp
+)
+
+PEERDIR(
+ contrib/libs/protobuf
ydb/public/api/protos
-)
-
-END()
+)
+
+END()
diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/plain_status/status.cpp b/ydb/public/sdk/cpp/client/impl/ydb_internal/plain_status/status.cpp
index d86f3c1556..0449c78552 100644
--- a/ydb/public/sdk/cpp/client/impl/ydb_internal/plain_status/status.cpp
+++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/plain_status/status.cpp
@@ -1,63 +1,63 @@
-#define INCLUDE_YDB_INTERNAL_H
-#include "status.h"
-
-#include <util/string/builder.h>
-
-namespace NYdb {
-
-TPlainStatus::TPlainStatus(
- const NGrpc::TGrpcStatus& grpcStatus,
+#define INCLUDE_YDB_INTERNAL_H
+#include "status.h"
+
+#include <util/string/builder.h>
+
+namespace NYdb {
+
+TPlainStatus::TPlainStatus(
+ const NGrpc::TGrpcStatus& grpcStatus,
const TStringType& endpoint,
std::multimap<TStringType, TStringType>&& metadata)
- : Endpoint(endpoint)
- , Metadata(std::move(metadata))
-{
+ : Endpoint(endpoint)
+ , Metadata(std::move(metadata))
+{
TStringType msg;
- if (grpcStatus.InternalError) {
- Status = EStatus::CLIENT_INTERNAL_ERROR;
- if (grpcStatus.Msg) {
- msg = TStringBuilder() << "Internal client error: " << grpcStatus.Msg;
- } else {
- msg = "Unknown internal client error";
- }
- } else if (grpcStatus.GRpcStatusCode != grpc::StatusCode::OK) {
- switch (grpcStatus.GRpcStatusCode) {
- case grpc::StatusCode::UNAVAILABLE:
- Status = EStatus::TRANSPORT_UNAVAILABLE;
- break;
- case grpc::StatusCode::CANCELLED:
- Status = EStatus::CLIENT_CANCELLED;
- break;
- case grpc::StatusCode::UNAUTHENTICATED:
- Status = EStatus::CLIENT_UNAUTHENTICATED;
- break;
- case grpc::StatusCode::UNIMPLEMENTED:
- Status = EStatus::CLIENT_CALL_UNIMPLEMENTED;
- break;
- case grpc::StatusCode::RESOURCE_EXHAUSTED:
- Status = EStatus::CLIENT_RESOURCE_EXHAUSTED;
- break;
- case grpc::StatusCode::DEADLINE_EXCEEDED:
- Status = EStatus::CLIENT_DEADLINE_EXCEEDED;
- break;
- case grpc::StatusCode::OUT_OF_RANGE:
- Status = EStatus::CLIENT_OUT_OF_RANGE;
- break;
- default:
- Status = EStatus::CLIENT_INTERNAL_ERROR;
- break;
- }
- msg = TStringBuilder() << "GRpc error: (" << grpcStatus.GRpcStatusCode << "): " << grpcStatus.Msg;
- } else {
- Status = EStatus::SUCCESS;
- }
- if (msg) {
- Issues.AddIssue(NYql::TIssue(msg));
- }
-}
-
+ if (grpcStatus.InternalError) {
+ Status = EStatus::CLIENT_INTERNAL_ERROR;
+ if (grpcStatus.Msg) {
+ msg = TStringBuilder() << "Internal client error: " << grpcStatus.Msg;
+ } else {
+ msg = "Unknown internal client error";
+ }
+ } else if (grpcStatus.GRpcStatusCode != grpc::StatusCode::OK) {
+ switch (grpcStatus.GRpcStatusCode) {
+ case grpc::StatusCode::UNAVAILABLE:
+ Status = EStatus::TRANSPORT_UNAVAILABLE;
+ break;
+ case grpc::StatusCode::CANCELLED:
+ Status = EStatus::CLIENT_CANCELLED;
+ break;
+ case grpc::StatusCode::UNAUTHENTICATED:
+ Status = EStatus::CLIENT_UNAUTHENTICATED;
+ break;
+ case grpc::StatusCode::UNIMPLEMENTED:
+ Status = EStatus::CLIENT_CALL_UNIMPLEMENTED;
+ break;
+ case grpc::StatusCode::RESOURCE_EXHAUSTED:
+ Status = EStatus::CLIENT_RESOURCE_EXHAUSTED;
+ break;
+ case grpc::StatusCode::DEADLINE_EXCEEDED:
+ Status = EStatus::CLIENT_DEADLINE_EXCEEDED;
+ break;
+ case grpc::StatusCode::OUT_OF_RANGE:
+ Status = EStatus::CLIENT_OUT_OF_RANGE;
+ break;
+ default:
+ Status = EStatus::CLIENT_INTERNAL_ERROR;
+ break;
+ }
+ msg = TStringBuilder() << "GRpc error: (" << grpcStatus.GRpcStatusCode << "): " << grpcStatus.Msg;
+ } else {
+ Status = EStatus::SUCCESS;
+ }
+ if (msg) {
+ Issues.AddIssue(NYql::TIssue(msg));
+ }
+}
+
TPlainStatus TPlainStatus::Internal(const TStringType& message) {
return { EStatus::CLIENT_INTERNAL_ERROR, "Internal client error: " + message };
-}
-
-} // namespace NYdb
+}
+
+} // namespace NYdb
diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/plain_status/status.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/plain_status/status.h
index d1a5832566..084f667892 100644
--- a/ydb/public/sdk/cpp/client/impl/ydb_internal/plain_status/status.h
+++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/plain_status/status.h
@@ -1,68 +1,68 @@
-#pragma once
-
+#pragma once
+
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/internal_header.h>
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/common/type_switcher.h>
#include <ydb/public/sdk/cpp/client/ydb_types/status_codes.h>
-
+
#include <ydb/public/api/protos/ydb_operation.pb.h>
#include <ydb/library/yql/public/issue/yql_issue.h>
-
-#include <library/cpp/grpc/client/grpc_client_low.h>
-
-namespace NYdb {
-
-struct TPlainStatus {
- EStatus Status;
- NYql::TIssues Issues;
+
+#include <library/cpp/grpc/client/grpc_client_low.h>
+
+namespace NYdb {
+
+struct TPlainStatus {
+ EStatus Status;
+ NYql::TIssues Issues;
TStringType Endpoint;
std::multimap<TStringType, TStringType> Metadata;
Ydb::CostInfo ConstInfo;
-
- TPlainStatus()
- : Status(EStatus::SUCCESS)
- { }
-
- TPlainStatus(EStatus status, NYql::TIssues&& issues)
- : Status(status)
- , Issues(std::move(issues))
- { }
-
+
+ TPlainStatus()
+ : Status(EStatus::SUCCESS)
+ { }
+
+ TPlainStatus(EStatus status, NYql::TIssues&& issues)
+ : Status(status)
+ , Issues(std::move(issues))
+ { }
+
TPlainStatus(EStatus status, NYql::TIssues&& issues, const TStringType& endpoint,
std::multimap<TStringType, TStringType>&& metadata)
- : Status(status)
- , Issues(std::move(issues))
- , Endpoint(endpoint)
- , Metadata(std::move(metadata))
- { }
-
+ : Status(status)
+ , Issues(std::move(issues))
+ , Endpoint(endpoint)
+ , Metadata(std::move(metadata))
+ { }
+
TPlainStatus(EStatus status, const TStringType& message)
- : Status(status)
- {
+ : Status(status)
+ {
if (!message.empty()) {
- Issues.AddIssue(NYql::TIssue(message));
- }
- }
-
- TPlainStatus(
+ Issues.AddIssue(NYql::TIssue(message));
+ }
+ }
+
+ TPlainStatus(
const NGrpc::TGrpcStatus& grpcStatus, const TStringType& endpoint = TStringType(),
std::multimap<TStringType, TStringType>&& metadata = {});
-
+
template<class T>
void SetCostInfo(T&& costInfo) {
ConstInfo = std::forward<T>(costInfo);
}
- bool Ok() const {
- return Status == EStatus::SUCCESS;
- }
-
+ bool Ok() const {
+ return Status == EStatus::SUCCESS;
+ }
+
static TPlainStatus Internal(const TStringType& message);
-
- bool IsTransportError() const {
- auto status = static_cast<size_t>(Status);
- return TRANSPORT_STATUSES_FIRST <= status && status <= TRANSPORT_STATUSES_LAST;
- }
-};
-
-} // namespace NYdb
+
+ bool IsTransportError() const {
+ auto status = static_cast<size_t>(Status);
+ return TRANSPORT_STATUSES_FIRST <= status && status <= TRANSPORT_STATUSES_LAST;
+ }
+};
+
+} // namespace NYdb
diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/plain_status/ya.make b/ydb/public/sdk/cpp/client/impl/ydb_internal/plain_status/ya.make
index 683105e939..93157bcf39 100644
--- a/ydb/public/sdk/cpp/client/impl/ydb_internal/plain_status/ya.make
+++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/plain_status/ya.make
@@ -1,15 +1,15 @@
-LIBRARY()
-
+LIBRARY()
+
OWNER(g:kikimr)
-
-SRCS(
- status.cpp
-)
-
-PEERDIR(
+
+SRCS(
+ status.cpp
+)
+
+PEERDIR(
contrib/libs/protobuf
library/cpp/grpc/client
ydb/library/yql/public/issue
-)
-
-END()
+)
+
+END()
diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/rpc_request_settings/settings.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/rpc_request_settings/settings.h
index dda9b5302f..2a2611794d 100644
--- a/ydb/public/sdk/cpp/client/impl/ydb_internal/rpc_request_settings/settings.h
+++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/rpc_request_settings/settings.h
@@ -1,11 +1,11 @@
-#pragma once
-
+#pragma once
+
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/internal_header.h>
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/common/type_switcher.h>
-
-namespace NYdb {
-
-struct TRpcRequestSettings {
+
+namespace NYdb {
+
+struct TRpcRequestSettings {
TStringType TraceId;
TStringType RequestType;
std::vector<std::pair<TStringType, TStringType>> Header;
@@ -14,17 +14,17 @@ struct TRpcRequestSettings {
UseDiscoveryEndpoint // Use single discovery endpoint for request
} EndpointPolicy = TEndpointPolicy::UsePreferedEndpoint;
bool UseAuth = true;
-
- template<typename TRequestSettings>
- static TRpcRequestSettings Make(const TRequestSettings& settings) {
- TRpcRequestSettings rpcSettings;
- rpcSettings.TraceId = settings.TraceId_;
- rpcSettings.RequestType = settings.RequestType_;
+
+ template<typename TRequestSettings>
+ static TRpcRequestSettings Make(const TRequestSettings& settings) {
+ TRpcRequestSettings rpcSettings;
+ rpcSettings.TraceId = settings.TraceId_;
+ rpcSettings.RequestType = settings.RequestType_;
rpcSettings.Header = settings.Header_;
rpcSettings.EndpointPolicy = TEndpointPolicy::UsePreferedEndpoint;
rpcSettings.UseAuth = true;
- return rpcSettings;
- }
-};
-
-} // namespace NYdb
+ return rpcSettings;
+ }
+};
+
+} // namespace NYdb
diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/stats_extractor/extractor.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/stats_extractor/extractor.h
index 7ca0517f34..2a2186a0dc 100644
--- a/ydb/public/sdk/cpp/client/impl/ydb_internal/stats_extractor/extractor.h
+++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/stats_extractor/extractor.h
@@ -1,44 +1,44 @@
-#pragma once
-
+#pragma once
+
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/internal_header.h>
-
+
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/internal_client/client.h>
-
+
#include <ydb/public/sdk/cpp/client/ydb_extension/extension.h>
-
-#include <library/cpp/monlib/metrics/metric_registry.h>
-
-namespace NYdb {
-
-class TStatsExtractor: public NSdkStats::IStatApi {
-public:
-
- TStatsExtractor(std::shared_ptr<IInternalClient> client)
- : Client_(client)
- { }
-
- virtual void SetMetricRegistry(NMonitoring::IMetricRegistry* sensorsRegistry) override {
- auto strong = Client_.lock();
- if (strong) {
- strong->StartStatCollecting(sensorsRegistry);
- } else {
- return;
- }
- }
-
- void Accept(NMonitoring::IMetricConsumer* consumer) const override {
-
- auto strong = Client_.lock();
- if (strong) {
- auto sensorsRegistry = strong->GetMetricRegistry();
- Y_VERIFY(sensorsRegistry, "TMetricRegistry is null in Stats Extractor");
- sensorsRegistry->Accept(TInstant::Zero(), consumer);
- } else {
- throw NSdkStats::DestroyedClientException();
- }
- }
-private:
- std::weak_ptr<IInternalClient> Client_;
-};
-
-} // namespace NYdb
+
+#include <library/cpp/monlib/metrics/metric_registry.h>
+
+namespace NYdb {
+
+class TStatsExtractor: public NSdkStats::IStatApi {
+public:
+
+ TStatsExtractor(std::shared_ptr<IInternalClient> client)
+ : Client_(client)
+ { }
+
+ virtual void SetMetricRegistry(NMonitoring::IMetricRegistry* sensorsRegistry) override {
+ auto strong = Client_.lock();
+ if (strong) {
+ strong->StartStatCollecting(sensorsRegistry);
+ } else {
+ return;
+ }
+ }
+
+ void Accept(NMonitoring::IMetricConsumer* consumer) const override {
+
+ auto strong = Client_.lock();
+ if (strong) {
+ auto sensorsRegistry = strong->GetMetricRegistry();
+ Y_VERIFY(sensorsRegistry, "TMetricRegistry is null in Stats Extractor");
+ sensorsRegistry->Accept(TInstant::Zero(), consumer);
+ } else {
+ throw NSdkStats::DestroyedClientException();
+ }
+ }
+private:
+ std::weak_ptr<IInternalClient> Client_;
+};
+
+} // namespace NYdb
diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/table_helpers/helpers.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/table_helpers/helpers.h
index 14bf8a2fb7..2ab4c68dcc 100644
--- a/ydb/public/sdk/cpp/client/impl/ydb_internal/table_helpers/helpers.h
+++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/table_helpers/helpers.h
@@ -1,38 +1,38 @@
-#pragma once
-
+#pragma once
+
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/internal_header.h>
-
+
#include <ydb/public/api/protos/ydb_table.pb.h>
#include <ydb/public/sdk/cpp/client/ydb_scheme/scheme.h>
#include <ydb/public/sdk/cpp/client/ydb_table/query_stats/stats.h>
-
-namespace NYdb {
-
-template<typename TFrom>
-inline void PermissionToSchemeEntry(const TFrom& from, TVector<NScheme::TPermissions>* to) {
- for (const auto& effPerm : from) {
- to->push_back(NScheme::TPermissions{effPerm.subject(), TVector<TString>()});
- for (const auto& permName : effPerm.permission_names()) {
- to->back().PermissionNames.push_back(permName);
- }
- }
-}
-
-inline Ydb::Table::QueryStatsCollection::Mode GetStatsCollectionMode(TMaybe<NTable::ECollectQueryStatsMode> mode) {
- if (mode) {
- switch (*mode) {
- case NTable::ECollectQueryStatsMode::None:
- return Ydb::Table::QueryStatsCollection::STATS_COLLECTION_NONE;
- case NTable::ECollectQueryStatsMode::Basic:
- return Ydb::Table::QueryStatsCollection::STATS_COLLECTION_BASIC;
- case NTable::ECollectQueryStatsMode::Full:
- return Ydb::Table::QueryStatsCollection::STATS_COLLECTION_FULL;
- default:
- break;
- }
- }
-
- return Ydb::Table::QueryStatsCollection::STATS_COLLECTION_UNSPECIFIED;
-}
-
-} // namespace NYdb
+
+namespace NYdb {
+
+template<typename TFrom>
+inline void PermissionToSchemeEntry(const TFrom& from, TVector<NScheme::TPermissions>* to) {
+ for (const auto& effPerm : from) {
+ to->push_back(NScheme::TPermissions{effPerm.subject(), TVector<TString>()});
+ for (const auto& permName : effPerm.permission_names()) {
+ to->back().PermissionNames.push_back(permName);
+ }
+ }
+}
+
+inline Ydb::Table::QueryStatsCollection::Mode GetStatsCollectionMode(TMaybe<NTable::ECollectQueryStatsMode> mode) {
+ if (mode) {
+ switch (*mode) {
+ case NTable::ECollectQueryStatsMode::None:
+ return Ydb::Table::QueryStatsCollection::STATS_COLLECTION_NONE;
+ case NTable::ECollectQueryStatsMode::Basic:
+ return Ydb::Table::QueryStatsCollection::STATS_COLLECTION_BASIC;
+ case NTable::ECollectQueryStatsMode::Full:
+ return Ydb::Table::QueryStatsCollection::STATS_COLLECTION_FULL;
+ default:
+ break;
+ }
+ }
+
+ return Ydb::Table::QueryStatsCollection::STATS_COLLECTION_UNSPECIFIED;
+}
+
+} // namespace NYdb
diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/thread_pool/pool.cpp b/ydb/public/sdk/cpp/client/impl/ydb_internal/thread_pool/pool.cpp
index 459573259b..7fbf33277f 100644
--- a/ydb/public/sdk/cpp/client/impl/ydb_internal/thread_pool/pool.cpp
+++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/thread_pool/pool.cpp
@@ -1,2 +1,2 @@
-#define INCLUDE_YDB_INTERNAL_H
-#include "pool.h"
+#define INCLUDE_YDB_INTERNAL_H
+#include "pool.h"
diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/thread_pool/pool.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/thread_pool/pool.h
index 2480ce5a56..8917c83f25 100644
--- a/ydb/public/sdk/cpp/client/impl/ydb_internal/thread_pool/pool.h
+++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/thread_pool/pool.h
@@ -1,21 +1,21 @@
-#pragma once
-
+#pragma once
+
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/internal_header.h>
-
-#include <util/thread/pool.h>
-
+
+#include <util/thread/pool.h>
+
#include <memory>
-namespace NYdb {
-
-inline std::unique_ptr<IThreadPool> CreateThreadPool(size_t threads) {
- std::unique_ptr<IThreadPool> queue;
- if (threads) {
- queue.reset(new TThreadPool(TThreadPool::TParams().SetBlocking(true).SetCatching(false)));
- } else {
- queue.reset(new TAdaptiveThreadPool());
- }
- return queue;
-}
-
-} // namespace NYdb
+namespace NYdb {
+
+inline std::unique_ptr<IThreadPool> CreateThreadPool(size_t threads) {
+ std::unique_ptr<IThreadPool> queue;
+ if (threads) {
+ queue.reset(new TThreadPool(TThreadPool::TParams().SetBlocking(true).SetCatching(false)));
+ } else {
+ queue.reset(new TAdaptiveThreadPool());
+ }
+ return queue;
+}
+
+} // namespace NYdb
diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/thread_pool/ya.make b/ydb/public/sdk/cpp/client/impl/ydb_internal/thread_pool/ya.make
index fb23507c6f..cefbaddc23 100644
--- a/ydb/public/sdk/cpp/client/impl/ydb_internal/thread_pool/ya.make
+++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/thread_pool/ya.make
@@ -1,9 +1,9 @@
-LIBRARY()
-
+LIBRARY()
+
OWNER(g:kikimr)
-
-SRCS(
- pool.cpp
-)
-
-END()
+
+SRCS(
+ pool.cpp
+)
+
+END()
diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/value_helpers/helpers.cpp b/ydb/public/sdk/cpp/client/impl/ydb_internal/value_helpers/helpers.cpp
index 4cb281383b..0048cd3ddf 100644
--- a/ydb/public/sdk/cpp/client/impl/ydb_internal/value_helpers/helpers.cpp
+++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/value_helpers/helpers.cpp
@@ -1,95 +1,95 @@
-#define INCLUDE_YDB_INTERNAL_H
-#include "helpers.h"
-
-namespace NYdb {
-
-bool TypesEqual(const Ydb::Type& t1, const Ydb::Type& t2) {
- if (t1.type_case() != t2.type_case()) {
- return false;
- }
-
- switch (t1.type_case()) {
- case Ydb::Type::kTypeId:
- return t1.type_id() == t2.type_id();
- case Ydb::Type::kDecimalType:
- return t1.decimal_type().precision() == t2.decimal_type().precision()
- && t1.decimal_type().scale() == t2.decimal_type().scale();
- case Ydb::Type::kOptionalType:
- return TypesEqual(t1.optional_type().item(), t2.optional_type().item());
- case Ydb::Type::kListType:
- return TypesEqual(t1.list_type().item(), t2.list_type().item());
- case Ydb::Type::kTupleType:
- if (t1.tuple_type().elements_size() != t2.tuple_type().elements_size()) {
- return false;
- }
- for (size_t i = 0; i < (size_t)t1.tuple_type().elements_size(); ++i) {
- if (!TypesEqual(t1.tuple_type().elements(i), t2.tuple_type().elements(i))) {
- return false;
- }
- }
- return true;
- case Ydb::Type::kStructType:
- if (t1.struct_type().members_size() != t2.struct_type().members_size()) {
- return false;
- }
- for (size_t i = 0; i < (size_t)t1.struct_type().members_size(); ++i) {
- auto& m1 = t1.struct_type().members(i);
- auto& m2 = t2.struct_type().members(i);
- if (m1.name() != m2.name()) {
- return false;
- }
-
- if (!TypesEqual(m1.type(), m2.type())) {
- return false;
- }
- }
- return true;
- case Ydb::Type::kDictType:
- return TypesEqual(t1.dict_type().key(), t2.dict_type().key())
- && TypesEqual(t1.dict_type().payload(), t2.dict_type().payload());
- case Ydb::Type::kVariantType: {
- const auto& v1 = t1.variant_type();
- const auto& v2 = t2.variant_type();
- if (v1.type_case() != v2.type_case()) {
- return false;
- }
- switch (v1.type_case()) {
- case Ydb::VariantType::kTupleItems:
- if (v1.tuple_items().elements_size() != v2.tuple_items().elements_size()) {
- return false;
- }
- for (size_t i = 0; i < (size_t)v1.tuple_items().elements_size(); ++i) {
- if (!TypesEqual(v1.tuple_items().elements(i), v2.tuple_items().elements(i))) {
- return false;
- }
- }
- return true;
- case Ydb::VariantType::kStructItems:
- if (v1.struct_items().members_size() != v2.struct_items().members_size()) {
- return false;
- }
- for (size_t i = 0; i < (size_t)v1.struct_items().members_size(); ++i) {
- auto& m1 = v1.struct_items().members(i);
- auto& m2 = v2.struct_items().members(i);
- if (m1.name() != m2.name()) {
- return false;
- }
-
- if (!TypesEqual(m1.type(), m2.type())) {
- return false;
- }
- }
- return true;
- default:
- return false;
- }
- return false;
- }
- case Ydb::Type::kVoidType:
- return true;
- default:
- return false;
- }
-}
-
-} // namespace NYdb
+#define INCLUDE_YDB_INTERNAL_H
+#include "helpers.h"
+
+namespace NYdb {
+
+bool TypesEqual(const Ydb::Type& t1, const Ydb::Type& t2) {
+ if (t1.type_case() != t2.type_case()) {
+ return false;
+ }
+
+ switch (t1.type_case()) {
+ case Ydb::Type::kTypeId:
+ return t1.type_id() == t2.type_id();
+ case Ydb::Type::kDecimalType:
+ return t1.decimal_type().precision() == t2.decimal_type().precision()
+ && t1.decimal_type().scale() == t2.decimal_type().scale();
+ case Ydb::Type::kOptionalType:
+ return TypesEqual(t1.optional_type().item(), t2.optional_type().item());
+ case Ydb::Type::kListType:
+ return TypesEqual(t1.list_type().item(), t2.list_type().item());
+ case Ydb::Type::kTupleType:
+ if (t1.tuple_type().elements_size() != t2.tuple_type().elements_size()) {
+ return false;
+ }
+ for (size_t i = 0; i < (size_t)t1.tuple_type().elements_size(); ++i) {
+ if (!TypesEqual(t1.tuple_type().elements(i), t2.tuple_type().elements(i))) {
+ return false;
+ }
+ }
+ return true;
+ case Ydb::Type::kStructType:
+ if (t1.struct_type().members_size() != t2.struct_type().members_size()) {
+ return false;
+ }
+ for (size_t i = 0; i < (size_t)t1.struct_type().members_size(); ++i) {
+ auto& m1 = t1.struct_type().members(i);
+ auto& m2 = t2.struct_type().members(i);
+ if (m1.name() != m2.name()) {
+ return false;
+ }
+
+ if (!TypesEqual(m1.type(), m2.type())) {
+ return false;
+ }
+ }
+ return true;
+ case Ydb::Type::kDictType:
+ return TypesEqual(t1.dict_type().key(), t2.dict_type().key())
+ && TypesEqual(t1.dict_type().payload(), t2.dict_type().payload());
+ case Ydb::Type::kVariantType: {
+ const auto& v1 = t1.variant_type();
+ const auto& v2 = t2.variant_type();
+ if (v1.type_case() != v2.type_case()) {
+ return false;
+ }
+ switch (v1.type_case()) {
+ case Ydb::VariantType::kTupleItems:
+ if (v1.tuple_items().elements_size() != v2.tuple_items().elements_size()) {
+ return false;
+ }
+ for (size_t i = 0; i < (size_t)v1.tuple_items().elements_size(); ++i) {
+ if (!TypesEqual(v1.tuple_items().elements(i), v2.tuple_items().elements(i))) {
+ return false;
+ }
+ }
+ return true;
+ case Ydb::VariantType::kStructItems:
+ if (v1.struct_items().members_size() != v2.struct_items().members_size()) {
+ return false;
+ }
+ for (size_t i = 0; i < (size_t)v1.struct_items().members_size(); ++i) {
+ auto& m1 = v1.struct_items().members(i);
+ auto& m2 = v2.struct_items().members(i);
+ if (m1.name() != m2.name()) {
+ return false;
+ }
+
+ if (!TypesEqual(m1.type(), m2.type())) {
+ return false;
+ }
+ }
+ return true;
+ default:
+ return false;
+ }
+ return false;
+ }
+ case Ydb::Type::kVoidType:
+ return true;
+ default:
+ return false;
+ }
+}
+
+} // namespace NYdb
diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/value_helpers/helpers.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/value_helpers/helpers.h
index 8a1ee45884..9b6fd6a275 100644
--- a/ydb/public/sdk/cpp/client/impl/ydb_internal/value_helpers/helpers.h
+++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/value_helpers/helpers.h
@@ -1,11 +1,11 @@
-#pragma once
-
+#pragma once
+
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/internal_header.h>
-
+
#include <ydb/public/api/protos/ydb_value.pb.h>
-
-namespace NYdb {
-
-bool TypesEqual(const Ydb::Type& t1, const Ydb::Type& t2);
-
-} // namespace NYdb
+
+namespace NYdb {
+
+bool TypesEqual(const Ydb::Type& t1, const Ydb::Type& t2);
+
+} // namespace NYdb
diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/value_helpers/ya.make b/ydb/public/sdk/cpp/client/impl/ydb_internal/value_helpers/ya.make
index cdbf8d5039..a2054b8511 100644
--- a/ydb/public/sdk/cpp/client/impl/ydb_internal/value_helpers/ya.make
+++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/value_helpers/ya.make
@@ -1,13 +1,13 @@
-LIBRARY()
-
+LIBRARY()
+
OWNER(g:kikimr)
-
-SRCS(
- helpers.cpp
-)
-
-PEERDIR(
+
+SRCS(
+ helpers.cpp
+)
+
+PEERDIR(
ydb/public/api/protos
-)
-
-END()
+)
+
+END()
diff --git a/ydb/public/sdk/cpp/client/impl/ydb_stats/stats.cpp b/ydb/public/sdk/cpp/client/impl/ydb_stats/stats.cpp
index 4ef6f437b4..21a58b33cb 100644
--- a/ydb/public/sdk/cpp/client/impl/ydb_stats/stats.cpp
+++ b/ydb/public/sdk/cpp/client/impl/ydb_stats/stats.cpp
@@ -1,4 +1,4 @@
-#include "stats.h"
+#include "stats.h"
namespace NYdb {
namespace NSdkStats {
diff --git a/ydb/public/sdk/cpp/client/impl/ydb_stats/stats.h b/ydb/public/sdk/cpp/client/impl/ydb_stats/stats.h
index 62c4033535..cae4099682 100644
--- a/ydb/public/sdk/cpp/client/impl/ydb_stats/stats.h
+++ b/ydb/public/sdk/cpp/client/impl/ydb_stats/stats.h
@@ -1,12 +1,12 @@
#pragma once
-
+
#include <ydb/public/sdk/cpp/client/ydb_types/status_codes.h>
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/common/type_switcher.h>
-
+
#include <library/cpp/grpc/client/grpc_client_low.h>
#include <library/cpp/monlib/metrics/metric_registry.h>
#include <library/cpp/monlib/metrics/histogram_collector.h>
-
+
#include <util/string/builder.h>
#include <atomic>
diff --git a/ydb/public/sdk/cpp/client/impl/ydb_stats/ya.make b/ydb/public/sdk/cpp/client/impl/ydb_stats/ya.make
index a805a72700..06e178b8c7 100644
--- a/ydb/public/sdk/cpp/client/impl/ydb_stats/ya.make
+++ b/ydb/public/sdk/cpp/client/impl/ydb_stats/ya.make
@@ -1,17 +1,17 @@
-LIBRARY()
-
-OWNER(
- dcherednik
- g:kikimr
-)
-
-SRCS(
- stats.cpp
-)
-
-PEERDIR(
- library/cpp/grpc/client
- library/cpp/monlib/metrics
-)
-
-END()
+LIBRARY()
+
+OWNER(
+ dcherednik
+ g:kikimr
+)
+
+SRCS(
+ stats.cpp
+)
+
+PEERDIR(
+ library/cpp/grpc/client
+ library/cpp/monlib/metrics
+)
+
+END()
diff --git a/ydb/public/sdk/cpp/client/ydb_common_client/impl/client.cpp b/ydb/public/sdk/cpp/client/ydb_common_client/impl/client.cpp
index 18c4c8c3ef..f679c0d5ec 100644
--- a/ydb/public/sdk/cpp/client/ydb_common_client/impl/client.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_common_client/impl/client.cpp
@@ -1 +1 @@
-#include "client.h"
+#include "client.h"
diff --git a/ydb/public/sdk/cpp/client/ydb_common_client/impl/client.h b/ydb/public/sdk/cpp/client/ydb_common_client/impl/client.h
index 0689504f1c..c43e9aab69 100644
--- a/ydb/public/sdk/cpp/client/ydb_common_client/impl/client.h
+++ b/ydb/public/sdk/cpp/client/ydb_common_client/impl/client.h
@@ -5,9 +5,9 @@
#undef INCLUDE_YDB_INTERNAL_H
#include <ydb/public/sdk/cpp/client/ydb_types/exceptions/exceptions.h>
-
-#include <memory>
-
+
+#include <memory>
+
namespace NYdb {
template<typename T>
diff --git a/ydb/public/sdk/cpp/client/ydb_common_client/impl/ya.make b/ydb/public/sdk/cpp/client/ydb_common_client/impl/ya.make
index 9b7651f575..1908d735a0 100644
--- a/ydb/public/sdk/cpp/client/ydb_common_client/impl/ya.make
+++ b/ydb/public/sdk/cpp/client/ydb_common_client/impl/ya.make
@@ -1,16 +1,16 @@
-LIBRARY()
-
-OWNER(
- dcherednik
- g:kikimr
-)
-
-SRCS(
- client.cpp
-)
-
-PEERDIR(
+LIBRARY()
+
+OWNER(
+ dcherednik
+ g:kikimr
+)
+
+SRCS(
+ client.cpp
+)
+
+PEERDIR(
ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections
-)
-
-END()
+)
+
+END()
diff --git a/ydb/public/sdk/cpp/client/ydb_common_client/settings.cpp b/ydb/public/sdk/cpp/client/ydb_common_client/settings.cpp
index 37eac01bad..8bffcd7a5a 100644
--- a/ydb/public/sdk/cpp/client/ydb_common_client/settings.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_common_client/settings.cpp
@@ -1,13 +1,13 @@
-#include "settings.h"
-
+#include "settings.h"
+
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/common/parser.h>
-namespace NYdb {
-
+namespace NYdb {
+
TCommonClientSettings& TCommonClientSettings::AuthToken(const TMaybe<TStringType>& token) {
- return CredentialsProviderFactory(CreateOAuthCredentialsProviderFactory(token.GetRef()));
-}
-
+ return CredentialsProviderFactory(CreateOAuthCredentialsProviderFactory(token.GetRef()));
+}
+
TCommonClientSettings GetClientSettingsFromConnectionString(const TStringType& connectionString) {
TCommonClientSettings settings;
auto connectionInfo = ParseConnectionString(connectionString);
@@ -17,4 +17,4 @@ TCommonClientSettings GetClientSettingsFromConnectionString(const TStringType& c
return settings;
}
-} // namespace NYdb
+} // namespace NYdb
diff --git a/ydb/public/sdk/cpp/client/ydb_common_client/settings.h b/ydb/public/sdk/cpp/client/ydb_common_client/settings.h
index a0b234c524..4ada7c09d0 100644
--- a/ydb/public/sdk/cpp/client/ydb_common_client/settings.h
+++ b/ydb/public/sdk/cpp/client/ydb_common_client/settings.h
@@ -1,38 +1,38 @@
-#pragma once
-
+#pragma once
+
#include <ydb/public/sdk/cpp/client/ydb_types/credentials/credentials.h>
#include <ydb/public/sdk/cpp/client/ydb_types/fluent_settings_helpers.h>
#include <ydb/public/sdk/cpp/client/ydb_types/ydb.h>
-
+
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/common/type_switcher.h>
-#include <functional>
-
-namespace NYdb {
-
-struct TCommonClientSettings {
- using TSelf = TCommonClientSettings;
-
- //! NOTE: Options below are discovery state specific options.
- //! If client overrides it and no state in cache (no client with same settings)
- //! ctor may blocks during discovery (according to DiscoveryMode settings
- //! The key of state cache is <Database, Credentials, DiscoveryEndpoint, DiscoveryMode> tuple
- //! For performance reason (to avoid extra discovery requests) it is
- //! always better to keep instance of client.
-
- //! Allows to override current database for client
+#include <functional>
+
+namespace NYdb {
+
+struct TCommonClientSettings {
+ using TSelf = TCommonClientSettings;
+
+ //! NOTE: Options below are discovery state specific options.
+ //! If client overrides it and no state in cache (no client with same settings)
+ //! ctor may blocks during discovery (according to DiscoveryMode settings
+ //! The key of state cache is <Database, Credentials, DiscoveryEndpoint, DiscoveryMode> tuple
+ //! For performance reason (to avoid extra discovery requests) it is
+ //! always better to keep instance of client.
+
+ //! Allows to override current database for client
FLUENT_SETTING_OPTIONAL(TStringType, Database);
- //! Allows to override current discovery endpoint
+ //! Allows to override current discovery endpoint
FLUENT_SETTING_OPTIONAL(TStringType, DiscoveryEndpoint);
- //! Allows to override current token for client
+ //! Allows to override current token for client
TSelf& AuthToken(const TMaybe<TStringType>& token);
- //! Allows to override current credentials provider
- FLUENT_SETTING_OPTIONAL(std::shared_ptr<ICredentialsProviderFactory>, CredentialsProviderFactory);
- //! Allows to override discovery mode
- FLUENT_SETTING_OPTIONAL(EDiscoveryMode, DiscoveryMode);
+ //! Allows to override current credentials provider
+ FLUENT_SETTING_OPTIONAL(std::shared_ptr<ICredentialsProviderFactory>, CredentialsProviderFactory);
+ //! Allows to override discovery mode
+ FLUENT_SETTING_OPTIONAL(EDiscoveryMode, DiscoveryMode);
//! Allows to override current Ssl mode
FLUENT_SETTING_OPTIONAL(bool, EnableSsl);
-};
-
+};
+
template<class TDerived>
struct TCommonClientSettingsBase : public TCommonClientSettings {
@@ -55,4 +55,4 @@ struct TCommonClientSettingsBase : public TCommonClientSettings {
TCommonClientSettings GetClientSettingsFromConnectionString(const TStringType& connectionString);
-} // namespace NYdb
+} // namespace NYdb
diff --git a/ydb/public/sdk/cpp/client/ydb_common_client/ya.make b/ydb/public/sdk/cpp/client/ydb_common_client/ya.make
index b0f4d9275b..9667fbd3ec 100644
--- a/ydb/public/sdk/cpp/client/ydb_common_client/ya.make
+++ b/ydb/public/sdk/cpp/client/ydb_common_client/ya.make
@@ -1,17 +1,17 @@
-LIBRARY()
-
-OWNER(
- dcherednik
- g:kikimr
-)
-
-SRCS(
- settings.cpp
-)
-
-PEERDIR(
+LIBRARY()
+
+OWNER(
+ dcherednik
+ g:kikimr
+)
+
+SRCS(
+ settings.cpp
+)
+
+PEERDIR(
ydb/public/sdk/cpp/client/impl/ydb_internal/common
ydb/public/sdk/cpp/client/ydb_types/credentials
-)
-
-END()
+)
+
+END()
diff --git a/ydb/public/sdk/cpp/client/ydb_coordination/coordination.cpp b/ydb/public/sdk/cpp/client/ydb_coordination/coordination.cpp
index 402652d605..7a99c95214 100644
--- a/ydb/public/sdk/cpp/client/ydb_coordination/coordination.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_coordination/coordination.cpp
@@ -1,10 +1,10 @@
-#include "coordination.h"
-
-#define INCLUDE_YDB_INTERNAL_H
+#include "coordination.h"
+
+#define INCLUDE_YDB_INTERNAL_H
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.h>
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/table_helpers/helpers.h>
-#undef INCLUDE_YDB_INTERNAL_H
-
+#undef INCLUDE_YDB_INTERNAL_H
+
#include <ydb/public/api/grpc/ydb_coordination_v1.grpc.pb.h>
#include <ydb/public/sdk/cpp/client/ydb_common_client/impl/client.h>
diff --git a/ydb/public/sdk/cpp/client/ydb_coordination/coordination.h b/ydb/public/sdk/cpp/client/ydb_coordination/coordination.h
index a7b9b48a8d..fa8ab572d0 100644
--- a/ydb/public/sdk/cpp/client/ydb_coordination/coordination.h
+++ b/ydb/public/sdk/cpp/client/ydb_coordination/coordination.h
@@ -13,11 +13,11 @@ namespace Coordination {
}
namespace NYdb {
-
-namespace NScheme {
-struct TPermissions;
-}
-
+
+namespace NScheme {
+struct TPermissions;
+}
+
namespace NCoordination {
//! Represents result of a call with status
diff --git a/ydb/public/sdk/cpp/client/ydb_coordination/ut/ya.make b/ydb/public/sdk/cpp/client/ydb_coordination/ut/ya.make
index 2617467d1a..23e9ae6a1b 100644
--- a/ydb/public/sdk/cpp/client/ydb_coordination/ut/ya.make
+++ b/ydb/public/sdk/cpp/client/ydb_coordination/ut/ya.make
@@ -1,27 +1,27 @@
UNITTEST_FOR(ydb/public/sdk/cpp/client/ydb_coordination)
-
-OWNER(
- dcherednik
- g:kikimr
-)
-
-IF (SANITIZER_TYPE)
- TIMEOUT(1200)
- SIZE(LARGE)
- TAG(ya:fat)
-ELSE()
- TIMEOUT(600)
- SIZE(MEDIUM)
-ENDIF()
-
-FORK_SUBTESTS()
-
-PEERDIR(
+
+OWNER(
+ dcherednik
+ g:kikimr
+)
+
+IF (SANITIZER_TYPE)
+ TIMEOUT(1200)
+ SIZE(LARGE)
+ TAG(ya:fat)
+ELSE()
+ TIMEOUT(600)
+ SIZE(MEDIUM)
+ENDIF()
+
+FORK_SUBTESTS()
+
+PEERDIR(
ydb/public/api/grpc
-)
-
-SRCS(
- coordination_ut.cpp
-)
-
-END()
+)
+
+SRCS(
+ coordination_ut.cpp
+)
+
+END()
diff --git a/ydb/public/sdk/cpp/client/ydb_coordination/ya.make b/ydb/public/sdk/cpp/client/ydb_coordination/ya.make
index c499d066ff..ac7b926a07 100644
--- a/ydb/public/sdk/cpp/client/ydb_coordination/ya.make
+++ b/ydb/public/sdk/cpp/client/ydb_coordination/ya.make
@@ -1,17 +1,17 @@
-LIBRARY()
-
-OWNER(
- dcherednik
- g:kikimr
-)
-
-SRCS(
- coordination.cpp
-)
-
-GENERATE_ENUM_SERIALIZATION(coordination.h)
-
-PEERDIR(
+LIBRARY()
+
+OWNER(
+ dcherednik
+ g:kikimr
+)
+
+SRCS(
+ coordination.cpp
+)
+
+GENERATE_ENUM_SERIALIZATION(coordination.h)
+
+PEERDIR(
ydb/public/api/grpc
ydb/public/sdk/cpp/client/impl/ydb_internal/make_request
ydb/public/sdk/cpp/client/ydb_common_client
@@ -19,10 +19,10 @@ PEERDIR(
ydb/public/sdk/cpp/client/ydb_driver
ydb/public/sdk/cpp/client/ydb_types
ydb/public/sdk/cpp/client/ydb_types/status
-)
-
-END()
-
+)
+
+END()
+
RECURSE_FOR_TESTS(
ut
)
diff --git a/ydb/public/sdk/cpp/client/ydb_datastreams/ya.make b/ydb/public/sdk/cpp/client/ydb_datastreams/ya.make
index f79749e18e..3e94f96757 100644
--- a/ydb/public/sdk/cpp/client/ydb_datastreams/ya.make
+++ b/ydb/public/sdk/cpp/client/ydb_datastreams/ya.make
@@ -1,18 +1,18 @@
-LIBRARY()
-
-OWNER(g:kikimr)
-
-SRCS(
+LIBRARY()
+
+OWNER(g:kikimr)
+
+SRCS(
datastreams.cpp
-)
-
-PEERDIR(
- library/cpp/grpc/client
- library/cpp/string_utils/url
+)
+
+PEERDIR(
+ library/cpp/grpc/client
+ library/cpp/string_utils/url
ydb/public/api/grpc/draft
ydb/public/lib/operation_id
ydb/public/sdk/cpp/client/impl/ydb_internal/make_request
ydb/public/sdk/cpp/client/ydb_driver
-)
-
-END()
+)
+
+END()
diff --git a/ydb/public/sdk/cpp/client/ydb_discovery/discovery.cpp b/ydb/public/sdk/cpp/client/ydb_discovery/discovery.cpp
index ba4de61513..6957a558ec 100644
--- a/ydb/public/sdk/cpp/client/ydb_discovery/discovery.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_discovery/discovery.cpp
@@ -1,7 +1,7 @@
-#include "discovery.h"
+#include "discovery.h"
#include <ydb/public/sdk/cpp/client/ydb_common_client/impl/client.h>
-
+
namespace NYdb {
namespace NDiscovery {
diff --git a/ydb/public/sdk/cpp/client/ydb_discovery/discovery.h b/ydb/public/sdk/cpp/client/ydb_discovery/discovery.h
index 8162884a98..d81275b7f6 100644
--- a/ydb/public/sdk/cpp/client/ydb_discovery/discovery.h
+++ b/ydb/public/sdk/cpp/client/ydb_discovery/discovery.h
@@ -5,7 +5,7 @@
namespace Ydb {
namespace Discovery {
class ListEndpointsResult;
- class WhoAmIResult;
+ class WhoAmIResult;
} // namespace Discovery
} // namespace Ydb
diff --git a/ydb/public/sdk/cpp/client/ydb_discovery/ya.make b/ydb/public/sdk/cpp/client/ydb_discovery/ya.make
index 7ca2f01b71..2cf49a926e 100644
--- a/ydb/public/sdk/cpp/client/ydb_discovery/ya.make
+++ b/ydb/public/sdk/cpp/client/ydb_discovery/ya.make
@@ -1,17 +1,17 @@
-LIBRARY()
-
-OWNER(
- dcherednik
- g:kikimr
-)
-
-SRCS(
- discovery.cpp
-)
-
-PEERDIR(
+LIBRARY()
+
+OWNER(
+ dcherednik
+ g:kikimr
+)
+
+SRCS(
+ discovery.cpp
+)
+
+PEERDIR(
ydb/public/sdk/cpp/client/ydb_common_client/impl
ydb/public/sdk/cpp/client/ydb_driver
-)
-
-END()
+)
+
+END()
diff --git a/ydb/public/sdk/cpp/client/ydb_driver/driver.cpp b/ydb/public/sdk/cpp/client/ydb_driver/driver.cpp
index 601ef3fbf7..de17c6e679 100644
--- a/ydb/public/sdk/cpp/client/ydb_driver/driver.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_driver/driver.cpp
@@ -1,5 +1,5 @@
-#include "driver.h"
-
+#include "driver.h"
+
#define INCLUDE_YDB_INTERNAL_H
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/driver/constants.h>
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/grpc_connections.h>
diff --git a/ydb/public/sdk/cpp/client/ydb_driver/driver_ut.cpp b/ydb/public/sdk/cpp/client/ydb_driver/driver_ut.cpp
index 7c67478c15..08172ee80a 100644
--- a/ydb/public/sdk/cpp/client/ydb_driver/driver_ut.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_driver/driver_ut.cpp
@@ -1,6 +1,6 @@
#include <ydb/public/sdk/cpp/client/ydb_table/table.h>
#include <ydb/public/sdk/cpp/client/ydb_types/exceptions/exceptions.h>
-
+
#include <ydb/public/api/grpc/ydb_discovery_v1.grpc.pb.h>
#include <ydb/public/api/grpc/ydb_table_v1.grpc.pb.h>
@@ -8,16 +8,16 @@
#include <grpcpp/server_builder.h>
#include <grpcpp/server_context.h>
-#include <library/cpp/testing/unittest/registar.h>
-#include <library/cpp/testing/unittest/tests_data.h>
-
+#include <library/cpp/testing/unittest/registar.h>
+#include <library/cpp/testing/unittest/tests_data.h>
+
#include <atomic>
-
-#include <google/protobuf/text_format.h>
-
-using namespace NYdb;
-using namespace NYdb::NTable;
-
+
+#include <google/protobuf/text_format.h>
+
+using namespace NYdb;
+using namespace NYdb::NTable;
+
namespace {
class TMockDiscoveryService : public Ydb::Discovery::V1::DiscoveryService::Service {
@@ -77,72 +77,72 @@ namespace {
} // namespace
-Y_UNIT_TEST_SUITE(CppGrpcClientSimpleTest) {
- Y_UNIT_TEST(ConnectWrongPort) {
- auto driver = TDriver(
- TDriverConfig()
- .SetEndpoint("localhost:100"));
- auto client = NTable::TTableClient(driver);
- auto sessionFuture = client.CreateSession();
-
- UNIT_ASSERT(sessionFuture.Wait(TDuration::Seconds(10)));
- }
-
- Y_UNIT_TEST(ConnectWrongPortRetry) {
- auto driver = TDriver(
- TDriverConfig()
- .SetEndpoint("localhost:100"));
- auto client = NTable::TTableClient(driver);
-
+Y_UNIT_TEST_SUITE(CppGrpcClientSimpleTest) {
+ Y_UNIT_TEST(ConnectWrongPort) {
+ auto driver = TDriver(
+ TDriverConfig()
+ .SetEndpoint("localhost:100"));
+ auto client = NTable::TTableClient(driver);
+ auto sessionFuture = client.CreateSession();
+
+ UNIT_ASSERT(sessionFuture.Wait(TDuration::Seconds(10)));
+ }
+
+ Y_UNIT_TEST(ConnectWrongPortRetry) {
+ auto driver = TDriver(
+ TDriverConfig()
+ .SetEndpoint("localhost:100"));
+ auto client = NTable::TTableClient(driver);
+
std::atomic_int counter = 0;
- std::function<void(const NTable::TAsyncCreateSessionResult& future)> handler =
- [&handler, &counter, client] (const NTable::TAsyncCreateSessionResult& future) mutable {
- UNIT_ASSERT_EQUAL(future.GetValue().GetStatus(), EStatus::TRANSPORT_UNAVAILABLE);
- UNIT_ASSERT_EXCEPTION(future.GetValue().GetSession(), NYdb::TContractViolation);
+ std::function<void(const NTable::TAsyncCreateSessionResult& future)> handler =
+ [&handler, &counter, client] (const NTable::TAsyncCreateSessionResult& future) mutable {
+ UNIT_ASSERT_EQUAL(future.GetValue().GetStatus(), EStatus::TRANSPORT_UNAVAILABLE);
+ UNIT_ASSERT_EXCEPTION(future.GetValue().GetSession(), NYdb::TContractViolation);
++counter;
if (counter.load() > 4) {
- return;
- }
-
- auto f = client.CreateSession();
-
- f.Apply(handler).GetValueSync();
- };
-
- client.CreateSession().Apply(handler).GetValueSync();
- UNIT_ASSERT_EQUAL(counter, 5);
- }
-
- Y_UNIT_TEST(TokenCharacters) {
- auto checkToken = [](const TString& token) {
- auto driver = TDriver(
- TDriverConfig()
- .SetEndpoint("localhost:100")
- .SetAuthToken(token));
- auto client = NTable::TTableClient(driver);
-
- auto result = client.CreateSession().GetValueSync();
-
- return result.GetStatus();
- };
-
- TVector<TString> InvalidTokens = {
- TString('\t'),
- TString('\n'),
- TString('\r')
- };
- for (auto& t : InvalidTokens) {
- UNIT_ASSERT_EQUAL(checkToken(t), EStatus::CLIENT_UNAUTHENTICATED);
- }
-
- TVector<TString> ValidTokens = {
- TString("qwerty 1234 <>,.?/:;\"'\\|}{~`!@#$%^&*()_+=-"),
- TString()
- };
- for (auto& t : ValidTokens) {
- UNIT_ASSERT_EQUAL(checkToken(t), EStatus::TRANSPORT_UNAVAILABLE);
- }
- }
+ return;
+ }
+
+ auto f = client.CreateSession();
+
+ f.Apply(handler).GetValueSync();
+ };
+
+ client.CreateSession().Apply(handler).GetValueSync();
+ UNIT_ASSERT_EQUAL(counter, 5);
+ }
+
+ Y_UNIT_TEST(TokenCharacters) {
+ auto checkToken = [](const TString& token) {
+ auto driver = TDriver(
+ TDriverConfig()
+ .SetEndpoint("localhost:100")
+ .SetAuthToken(token));
+ auto client = NTable::TTableClient(driver);
+
+ auto result = client.CreateSession().GetValueSync();
+
+ return result.GetStatus();
+ };
+
+ TVector<TString> InvalidTokens = {
+ TString('\t'),
+ TString('\n'),
+ TString('\r')
+ };
+ for (auto& t : InvalidTokens) {
+ UNIT_ASSERT_EQUAL(checkToken(t), EStatus::CLIENT_UNAUTHENTICATED);
+ }
+
+ TVector<TString> ValidTokens = {
+ TString("qwerty 1234 <>,.?/:;\"'\\|}{~`!@#$%^&*()_+=-"),
+ TString()
+ };
+ for (auto& t : ValidTokens) {
+ UNIT_ASSERT_EQUAL(checkToken(t), EStatus::TRANSPORT_UNAVAILABLE);
+ }
+ }
Y_UNIT_TEST(UsingIpAddresses) {
TPortManager pm;
@@ -181,4 +181,4 @@ Y_UNIT_TEST_SUITE(CppGrpcClientSimpleTest) {
auto session = sessionResult.GetSession();
UNIT_ASSERT_VALUES_EQUAL(session.GetId(), "my-session-id");
}
-}
+}
diff --git a/ydb/public/sdk/cpp/client/ydb_driver/ut/ya.make b/ydb/public/sdk/cpp/client/ydb_driver/ut/ya.make
index ef65759572..0cdbb9d72b 100644
--- a/ydb/public/sdk/cpp/client/ydb_driver/ut/ya.make
+++ b/ydb/public/sdk/cpp/client/ydb_driver/ut/ya.make
@@ -1,27 +1,27 @@
UNITTEST_FOR(ydb/public/sdk/cpp/client/ydb_driver)
-
-OWNER(
- dcherednik
- g:kikimr
-)
-
-IF (SANITIZER_TYPE)
- TIMEOUT(1200)
- SIZE(LARGE)
- TAG(ya:fat)
-ELSE()
- TIMEOUT(600)
- SIZE(MEDIUM)
-ENDIF()
-
-FORK_SUBTESTS()
-
-PEERDIR(
+
+OWNER(
+ dcherednik
+ g:kikimr
+)
+
+IF (SANITIZER_TYPE)
+ TIMEOUT(1200)
+ SIZE(LARGE)
+ TAG(ya:fat)
+ELSE()
+ TIMEOUT(600)
+ SIZE(MEDIUM)
+ENDIF()
+
+FORK_SUBTESTS()
+
+PEERDIR(
ydb/public/sdk/cpp/client/ydb_table
-)
-
-SRCS(
- driver_ut.cpp
-)
-
-END()
+)
+
+SRCS(
+ driver_ut.cpp
+)
+
+END()
diff --git a/ydb/public/sdk/cpp/client/ydb_driver/ya.make b/ydb/public/sdk/cpp/client/ydb_driver/ya.make
index e3338852eb..ff187205bb 100644
--- a/ydb/public/sdk/cpp/client/ydb_driver/ya.make
+++ b/ydb/public/sdk/cpp/client/ydb_driver/ya.make
@@ -1,24 +1,24 @@
-LIBRARY()
-
-OWNER(
- dcherednik
- g:kikimr
-)
-
-SRCS(
- driver.cpp
-)
-
-PEERDIR(
+LIBRARY()
+
+OWNER(
+ dcherednik
+ g:kikimr
+)
+
+SRCS(
+ driver.cpp
+)
+
+PEERDIR(
ydb/public/sdk/cpp/client/impl/ydb_internal/common
ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections
ydb/public/sdk/cpp/client/resources
ydb/public/sdk/cpp/client/ydb_common_client
ydb/public/sdk/cpp/client/ydb_types/status
-)
-
-END()
-
+)
+
+END()
+
RECURSE_FOR_TESTS(
ut
)
diff --git a/ydb/public/sdk/cpp/client/ydb_export/export.cpp b/ydb/public/sdk/cpp/client/ydb_export/export.cpp
index b967b6467a..c391c033e1 100644
--- a/ydb/public/sdk/cpp/client/ydb_export/export.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_export/export.cpp
@@ -1,18 +1,18 @@
-#include "export.h"
+#include "export.h"
-#define INCLUDE_YDB_INTERNAL_H
+#define INCLUDE_YDB_INTERNAL_H
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.h>
-#undef INCLUDE_YDB_INTERNAL_H
-
+#undef INCLUDE_YDB_INTERNAL_H
+
#include <ydb/public/api/grpc/ydb_discovery_v1.grpc.pb.h>
#include <ydb/public/api/grpc/ydb_export_v1.grpc.pb.h>
#include <ydb/public/api/protos/ydb_export.pb.h>
#include <ydb/public/sdk/cpp/client/ydb_common_client/impl/client.h>
#include <ydb/public/sdk/cpp/client/ydb_proto/accessor.h>
-
-#include <google/protobuf/repeated_field.h>
-#include <google/protobuf/timestamp.pb.h>
-
+
+#include <google/protobuf/repeated_field.h>
+#include <google/protobuf/timestamp.pb.h>
+
namespace NYdb {
namespace NExport {
diff --git a/ydb/public/sdk/cpp/client/ydb_export/ya.make b/ydb/public/sdk/cpp/client/ydb_export/ya.make
index 9a084f23b4..67680a6f9d 100644
--- a/ydb/public/sdk/cpp/client/ydb_export/ya.make
+++ b/ydb/public/sdk/cpp/client/ydb_export/ya.make
@@ -1,23 +1,23 @@
-LIBRARY()
-
-OWNER(
- dcherednik
- g:kikimr
-)
-
-SRCS(
- export.cpp
-)
-
-GENERATE_ENUM_SERIALIZATION(export.h)
-
-PEERDIR(
+LIBRARY()
+
+OWNER(
+ dcherednik
+ g:kikimr
+)
+
+SRCS(
+ export.cpp
+)
+
+GENERATE_ENUM_SERIALIZATION(export.h)
+
+PEERDIR(
ydb/public/api/grpc
ydb/public/api/protos
ydb/public/sdk/cpp/client/ydb_common_client/impl
ydb/public/sdk/cpp/client/ydb_driver
ydb/public/sdk/cpp/client/ydb_proto
ydb/public/sdk/cpp/client/ydb_types/operation
-)
-
-END()
+)
+
+END()
diff --git a/ydb/public/sdk/cpp/client/ydb_extension/extension.cpp b/ydb/public/sdk/cpp/client/ydb_extension/extension.cpp
index f03aacc007..4cb966a5ce 100644
--- a/ydb/public/sdk/cpp/client/ydb_extension/extension.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_extension/extension.cpp
@@ -1,4 +1,4 @@
-#include "extension.h"
+#include "extension.h"
#define INCLUDE_YDB_INTERNAL_H
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/grpc_connections.h>
diff --git a/ydb/public/sdk/cpp/client/ydb_extension/extension.h b/ydb/public/sdk/cpp/client/ydb_extension/extension.h
index 8551322710..3371bffddc 100644
--- a/ydb/public/sdk/cpp/client/ydb_extension/extension.h
+++ b/ydb/public/sdk/cpp/client/ydb_extension/extension.h
@@ -1,7 +1,7 @@
#pragma once
#include <ydb/public/sdk/cpp/client/ydb_driver/driver.h>
-
+
#include <library/cpp/monlib/metrics/metric_registry.h>
namespace Ydb {
@@ -14,23 +14,23 @@ class ListEndpointsResult;
namespace NYdb {
-class IExtensionApi {
-public:
- friend class TDriver;
-public:
- virtual ~IExtensionApi() = default;
-private:
- void SelfRegister(TDriver driver);
-};
-
-class IExtension {
- friend class TDriver;
-public:
- virtual ~IExtension() = default;
-private:
- void SelfRegister(TDriver driver);
-};
-
+class IExtensionApi {
+public:
+ friend class TDriver;
+public:
+ virtual ~IExtensionApi() = default;
+private:
+ void SelfRegister(TDriver driver);
+};
+
+class IExtension {
+ friend class TDriver;
+public:
+ virtual ~IExtension() = default;
+private:
+ void SelfRegister(TDriver driver);
+};
+
namespace NSdkStats {
class IStatApi: public IExtensionApi {
diff --git a/ydb/public/sdk/cpp/client/ydb_extension/ya.make b/ydb/public/sdk/cpp/client/ydb_extension/ya.make
index 842bb6c8d2..31d367b6bf 100644
--- a/ydb/public/sdk/cpp/client/ydb_extension/ya.make
+++ b/ydb/public/sdk/cpp/client/ydb_extension/ya.make
@@ -1,17 +1,17 @@
-LIBRARY()
-
-OWNER(
- dcherednik
- g:kikimr
-)
-
-SRCS(
- extension.cpp
-)
-
-PEERDIR(
- library/cpp/monlib/metrics
+LIBRARY()
+
+OWNER(
+ dcherednik
+ g:kikimr
+)
+
+SRCS(
+ extension.cpp
+)
+
+PEERDIR(
+ library/cpp/monlib/metrics
ydb/public/sdk/cpp/client/ydb_driver
-)
-
-END()
+)
+
+END()
diff --git a/ydb/public/sdk/cpp/client/ydb_import/import.cpp b/ydb/public/sdk/cpp/client/ydb_import/import.cpp
index 7e2829486f..f0a55678ef 100644
--- a/ydb/public/sdk/cpp/client/ydb_import/import.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_import/import.cpp
@@ -1,9 +1,9 @@
-#include "import.h"
+#include "import.h"
-#define INCLUDE_YDB_INTERNAL_H
+#define INCLUDE_YDB_INTERNAL_H
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.h>
-#undef INCLUDE_YDB_INTERNAL_H
-
+#undef INCLUDE_YDB_INTERNAL_H
+
#include <ydb/public/api/grpc/ydb_discovery_v1.grpc.pb.h>
#include <ydb/public/api/grpc/ydb_import_v1.grpc.pb.h>
#include <ydb/public/api/protos/ydb_import.pb.h>
diff --git a/ydb/public/sdk/cpp/client/ydb_import/ya.make b/ydb/public/sdk/cpp/client/ydb_import/ya.make
index 943f8275b9..8e1ac730c8 100644
--- a/ydb/public/sdk/cpp/client/ydb_import/ya.make
+++ b/ydb/public/sdk/cpp/client/ydb_import/ya.make
@@ -1,23 +1,23 @@
-LIBRARY()
-
-OWNER(
- dcherednik
- g:kikimr
-)
-
-SRCS(
- import.cpp
-)
-
-GENERATE_ENUM_SERIALIZATION(import.h)
-
-PEERDIR(
+LIBRARY()
+
+OWNER(
+ dcherednik
+ g:kikimr
+)
+
+SRCS(
+ import.cpp
+)
+
+GENERATE_ENUM_SERIALIZATION(import.h)
+
+PEERDIR(
ydb/public/api/grpc
ydb/public/api/protos
ydb/public/sdk/cpp/client/ydb_common_client/impl
ydb/public/sdk/cpp/client/ydb_driver
ydb/public/sdk/cpp/client/ydb_proto
ydb/public/sdk/cpp/client/ydb_types/operation
-)
-
-END()
+)
+
+END()
diff --git a/ydb/public/sdk/cpp/client/ydb_operation/operation.cpp b/ydb/public/sdk/cpp/client/ydb_operation/operation.cpp
index 64d78d13d6..fac441c505 100644
--- a/ydb/public/sdk/cpp/client/ydb_operation/operation.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_operation/operation.cpp
@@ -1,9 +1,9 @@
-#include "operation.h"
+#include "operation.h"
-#define INCLUDE_YDB_INTERNAL_H
+#define INCLUDE_YDB_INTERNAL_H
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.h>
-#undef INCLUDE_YDB_INTERNAL_H
-
+#undef INCLUDE_YDB_INTERNAL_H
+
/* Headers below used to instantiate concrete 'Get' & 'List' methods */
#include <ydb/public/sdk/cpp/client/ydb_export/export.h>
#include <ydb/public/sdk/cpp/client/ydb_import/import.h>
diff --git a/ydb/public/sdk/cpp/client/ydb_operation/ya.make b/ydb/public/sdk/cpp/client/ydb_operation/ya.make
index b33455bdf9..b07cd710ab 100644
--- a/ydb/public/sdk/cpp/client/ydb_operation/ya.make
+++ b/ydb/public/sdk/cpp/client/ydb_operation/ya.make
@@ -1,15 +1,15 @@
-LIBRARY()
-
-OWNER(
- dcherednik
- g:kikimr
-)
-
-SRCS(
- operation.cpp
-)
-
-PEERDIR(
+LIBRARY()
+
+OWNER(
+ dcherednik
+ g:kikimr
+)
+
+SRCS(
+ operation.cpp
+)
+
+PEERDIR(
ydb/public/api/grpc
ydb/public/lib/operation_id
ydb/public/sdk/cpp/client/ydb_common_client/impl
@@ -17,6 +17,6 @@ PEERDIR(
ydb/public/sdk/cpp/client/ydb_export
ydb/public/sdk/cpp/client/ydb_import
ydb/public/sdk/cpp/client/ydb_types/operation
-)
-
-END()
+)
+
+END()
diff --git a/ydb/public/sdk/cpp/client/ydb_params/impl.cpp b/ydb/public/sdk/cpp/client/ydb_params/impl.cpp
index b1bef036f5..9e8c5e1e7d 100644
--- a/ydb/public/sdk/cpp/client/ydb_params/impl.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_params/impl.cpp
@@ -1,5 +1,5 @@
-#include "impl.h"
-
+#include "impl.h"
+
#include <ydb/public/api/protos/ydb_scheme.pb.h>
#include <ydb/public/api/protos/ydb_value.pb.h>
@@ -42,7 +42,7 @@ TMaybe<TValue> TParams::TImpl::GetValue(const TString& name) const {
return &ParamsMap_;
}
-const ::google::protobuf::Map<TString, Ydb::TypedValue>& TParams::TImpl::GetProtoMap() const {
+const ::google::protobuf::Map<TString, Ydb::TypedValue>& TParams::TImpl::GetProtoMap() const {
return ParamsMap_;
}
diff --git a/ydb/public/sdk/cpp/client/ydb_params/impl.h b/ydb/public/sdk/cpp/client/ydb_params/impl.h
index 8c58fb7d72..9e38d2463d 100644
--- a/ydb/public/sdk/cpp/client/ydb_params/impl.h
+++ b/ydb/public/sdk/cpp/client/ydb_params/impl.h
@@ -1,6 +1,6 @@
#pragma once
-#include "params.h"
+#include "params.h"
namespace NYdb {
@@ -12,7 +12,7 @@ public:
TMap<TString, TValue> GetValues() const;
TMaybe<TValue> GetValue(const TString& name) const;
::google::protobuf::Map<TString, Ydb::TypedValue>* GetProtoMapPtr();
- const ::google::protobuf::Map<TString, Ydb::TypedValue>& GetProtoMap() const;
+ const ::google::protobuf::Map<TString, Ydb::TypedValue>& GetProtoMap() const;
private:
::google::protobuf::Map<TString, Ydb::TypedValue> ParamsMap_;
diff --git a/ydb/public/sdk/cpp/client/ydb_params/params.cpp b/ydb/public/sdk/cpp/client/ydb_params/params.cpp
index 91b03f6b50..a69d29b2f6 100644
--- a/ydb/public/sdk/cpp/client/ydb_params/params.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_params/params.cpp
@@ -1,5 +1,5 @@
-#include "params.h"
-#include "impl.h"
+#include "params.h"
+#include "impl.h"
#include <ydb/public/sdk/cpp/client/ydb_proto/accessor.h>
#include <ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers/handlers.h>
@@ -14,14 +14,14 @@ namespace NYdb {
TParams::TParams(::google::protobuf::Map<TString, Ydb::TypedValue>&& protoMap)
: Impl_(new TImpl(std::move(protoMap))) {}
-::google::protobuf::Map<TString, Ydb::TypedValue>* TParams::GetProtoMapPtr() {
- return Impl_->GetProtoMapPtr();
-}
-
-const ::google::protobuf::Map<TString, Ydb::TypedValue>& TParams::GetProtoMap() const {
- return Impl_->GetProtoMap();
-}
-
+::google::protobuf::Map<TString, Ydb::TypedValue>* TParams::GetProtoMapPtr() {
+ return Impl_->GetProtoMapPtr();
+}
+
+const ::google::protobuf::Map<TString, Ydb::TypedValue>& TParams::GetProtoMap() const {
+ return Impl_->GetProtoMap();
+}
+
bool TParams::Empty() const {
return Impl_->Empty();
}
diff --git a/ydb/public/sdk/cpp/client/ydb_params/params.h b/ydb/public/sdk/cpp/client/ydb_params/params.h
index c9ccb8071f..0f85f18329 100644
--- a/ydb/public/sdk/cpp/client/ydb_params/params.h
+++ b/ydb/public/sdk/cpp/client/ydb_params/params.h
@@ -2,8 +2,8 @@
#include <ydb/public/sdk/cpp/client/ydb_value/value.h>
-#include <google/protobuf/map.h>
-
+#include <google/protobuf/map.h>
+
namespace Ydb {
class TypedValue;
}
@@ -42,9 +42,9 @@ public:
private:
TParams(::google::protobuf::Map<TString, Ydb::TypedValue>&& protoMap);
- ::google::protobuf::Map<TString, Ydb::TypedValue>* GetProtoMapPtr();
- const ::google::protobuf::Map<TString, Ydb::TypedValue>& GetProtoMap() const;
-
+ ::google::protobuf::Map<TString, Ydb::TypedValue>* GetProtoMapPtr();
+ const ::google::protobuf::Map<TString, Ydb::TypedValue>& GetProtoMap() const;
+
class TImpl;
std::shared_ptr<TImpl> Impl_;
};
diff --git a/ydb/public/sdk/cpp/client/ydb_params/params_ut.cpp b/ydb/public/sdk/cpp/client/ydb_params/params_ut.cpp
index ce33b8be96..1b516de577 100644
--- a/ydb/public/sdk/cpp/client/ydb_params/params_ut.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_params/params_ut.cpp
@@ -1,8 +1,8 @@
#include <ydb/public/lib/yson_value/ydb_yson_value.h>
#include <ydb/public/sdk/cpp/client/ydb_params/params.h>
-#include <library/cpp/testing/unittest/registar.h>
-#include <library/cpp/testing/unittest/tests_data.h>
+#include <library/cpp/testing/unittest/registar.h>
+#include <library/cpp/testing/unittest/tests_data.h>
using namespace NYdb;
diff --git a/ydb/public/sdk/cpp/client/ydb_params/ut/ya.make b/ydb/public/sdk/cpp/client/ydb_params/ut/ya.make
index 05a87bb748..cd5ee478c9 100644
--- a/ydb/public/sdk/cpp/client/ydb_params/ut/ya.make
+++ b/ydb/public/sdk/cpp/client/ydb_params/ut/ya.make
@@ -21,7 +21,7 @@ PEERDIR(
)
SRCS(
- params_ut.cpp
+ params_ut.cpp
)
END()
diff --git a/ydb/public/sdk/cpp/client/ydb_params/ya.make b/ydb/public/sdk/cpp/client/ydb_params/ya.make
index 5ee6777789..b9f225d275 100644
--- a/ydb/public/sdk/cpp/client/ydb_params/ya.make
+++ b/ydb/public/sdk/cpp/client/ydb_params/ya.make
@@ -1,22 +1,22 @@
-LIBRARY()
-
-OWNER(
- dcherednik
- g:kikimr
-)
-
-SRCS(
- params.cpp
- impl.cpp
-)
-
-PEERDIR(
+LIBRARY()
+
+OWNER(
+ dcherednik
+ g:kikimr
+)
+
+SRCS(
+ params.cpp
+ impl.cpp
+)
+
+PEERDIR(
ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers
ydb/public/sdk/cpp/client/ydb_value
-)
-
-END()
-
+)
+
+END()
+
RECURSE_FOR_TESTS(
ut
)
diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/common.cpp b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/common.cpp
index e82e48eb2a..dd42c8c4ed 100644
--- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/common.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/common.cpp
@@ -1,4 +1,4 @@
-#include "common.h"
+#include "common.h"
#include <util/charset/unidata.h>
@@ -73,16 +73,16 @@ NYql::TIssues MakeIssueWithSubIssues(const TString& description, const NYql::TIs
size_t CalcDataSize(const TReadSessionEvent::TEvent& event) {
if (const TReadSessionEvent::TDataReceivedEvent* dataEvent = std::get_if<TReadSessionEvent::TDataReceivedEvent>(&event)) {
size_t len = 0;
- if (dataEvent->IsCompressedMessages()) {
- for (const auto& msg : dataEvent->GetCompressedMessages()) {
- len += msg.GetData().size();
- }
- } else {
- for (const auto& msg : dataEvent->GetMessages()) {
- if (!msg.HasException()) {
- len += msg.GetData().size();
- }
- }
+ if (dataEvent->IsCompressedMessages()) {
+ for (const auto& msg : dataEvent->GetCompressedMessages()) {
+ len += msg.GetData().size();
+ }
+ } else {
+ for (const auto& msg : dataEvent->GetMessages()) {
+ if (!msg.HasException()) {
+ len += msg.GetData().size();
+ }
+ }
}
return len;
} else {
diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/common.h b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/common.h
index 51f924c1a1..7287b69894 100644
--- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/common.h
+++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/common.h
@@ -3,7 +3,7 @@
#include <ydb/public/sdk/cpp/client/ydb_persqueue_core/persqueue.h>
#include <ydb/public/sdk/cpp/client/ydb_common_client/impl/client.h>
-#include <util/generic/queue.h>
+#include <util/generic/queue.h>
#include <util/system/condvar.h>
#include <util/thread/pool.h>
@@ -20,8 +20,8 @@ NYql::TIssues MakeIssueWithSubIssues(const TString& description, const NYql::TIs
TString IssuesSingleLineString(const NYql::TIssues& issues);
-size_t CalcDataSize(const TReadSessionEvent::TEvent& event);
-
+size_t CalcDataSize(const TReadSessionEvent::TEvent& event);
+
template <class TMessage>
bool IsErrorMessage(const TMessage& serverMessage) {
const Ydb::StatusIds::StatusCode status = serverMessage.status();
diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue.cpp b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue.cpp
index 2e7dcb640d..a22ed23cea 100644
--- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue.cpp
@@ -126,7 +126,7 @@ TDescribeTopicResult::TTopicSettings::TReadRule::TReadRule(const Ydb::PersQueue:
SupportedCodecs_.push_back(static_cast<ECodec>(codec));
}
Version_ = settings.version();
- ServiceType_ = settings.service_type();
+ ServiceType_ = settings.service_type();
}
TDescribeTopicResult::TTopicSettings::TRemoteMirrorRule::TRemoteMirrorRule(const Ydb::PersQueue::V1::TopicSettings::RemoteMirrorRule& settings)
diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue_impl.cpp b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue_impl.cpp
index f86e6cd599..bd466fd6f3 100644
--- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue_impl.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue_impl.cpp
@@ -1,6 +1,6 @@
#include "persqueue_impl.h"
-#include "read_session.h"
-#include "write_session.h"
+#include "read_session.h"
+#include "write_session.h"
namespace NYdb::NPersQueue {
diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue_impl.h b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue_impl.h
index f4dfb247fe..57e9dd43bf 100644
--- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue_impl.h
+++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue_impl.h
@@ -1,11 +1,11 @@
#pragma once
-#include "common.h"
-
-#define INCLUDE_YDB_INTERNAL_H
+#include "common.h"
+
+#define INCLUDE_YDB_INTERNAL_H
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.h>
-#undef INCLUDE_YDB_INTERNAL_H
-
+#undef INCLUDE_YDB_INTERNAL_H
+
#include <ydb/public/api/grpc/draft/ydb_persqueue_v1.grpc.pb.h>
#include <ydb/public/sdk/cpp/client/ydb_persqueue_core/persqueue.h>
@@ -40,7 +40,7 @@ public:
for (const auto& codec : readRule.SupportedCodecs_) {
rrProps.add_supported_codecs((static_cast<Ydb::PersQueue::V1::Codec>(codec)));
}
- rrProps.set_service_type(readRule.ServiceType_);
+ rrProps.set_service_type(readRule.ServiceType_);
}
template <class TRequest, class TSettings>
diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/read_session.cpp b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/read_session.cpp
index 2a43f71a99..e7dd0a87e5 100644
--- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/read_session.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/read_session.cpp
@@ -1,11 +1,11 @@
#include "persqueue_impl.h"
-#include "read_session.h"
-#include "common.h"
+#include "read_session.h"
+#include "common.h"
-#define INCLUDE_YDB_INTERNAL_H
+#define INCLUDE_YDB_INTERNAL_H
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/logger/log.h>
-#undef INCLUDE_YDB_INTERNAL_H
-
+#undef INCLUDE_YDB_INTERNAL_H
+
#include <library/cpp/containers/disjoint_interval_tree/disjoint_interval_tree.h>
#include <util/generic/guid.h>
#include <util/generic/size_literals.h>
@@ -22,15 +22,15 @@ static const TString DRIVER_IS_STOPPING_DESCRIPTION = "Driver is stopping";
static const bool RangesMode = !GetEnv("PQ_OFFSET_RANGES_MODE").empty();
-std::pair<ui64, ui64> GetMessageOffsetRange(const TReadSessionEvent::TDataReceivedEvent& dataReceivedEvent, ui64 index) {
- if (dataReceivedEvent.IsCompressedMessages()) {
- const auto& msg = dataReceivedEvent.GetCompressedMessages()[index];
- return {msg.GetOffset(0), msg.GetOffset(msg.GetBlocksCount() - 1) + 1};
- }
- const auto& msg = dataReceivedEvent.GetMessages()[index];
- return {msg.GetOffset(), msg.GetOffset() + 1};
-}
-
+std::pair<ui64, ui64> GetMessageOffsetRange(const TReadSessionEvent::TDataReceivedEvent& dataReceivedEvent, ui64 index) {
+ if (dataReceivedEvent.IsCompressedMessages()) {
+ const auto& msg = dataReceivedEvent.GetCompressedMessages()[index];
+ return {msg.GetOffset(0), msg.GetOffset(msg.GetBlocksCount() - 1) + 1};
+ }
+ const auto& msg = dataReceivedEvent.GetMessages()[index];
+ return {msg.GetOffset(), msg.GetOffset() + 1};
+}
+
TString IssuesSingleLineString(const NYql::TIssues& issues) {
return SubstGlobalCopy(issues.ToString(), '\n', ' ');
}
@@ -1023,14 +1023,14 @@ void TSingleClusterReadSessionImpl::RequestPartitionStreamStatus(const TPartitio
void TSingleClusterReadSessionImpl::OnUserRetrievedEvent(const TReadSessionEvent::TEvent& event) {
Log << TLOG_DEBUG << "Read session event " << DebugString(event);
- const i64 bytesCount = static_cast<i64>(CalcDataSize(event));
+ const i64 bytesCount = static_cast<i64>(CalcDataSize(event));
Y_ASSERT(bytesCount >= 0);
if (!std::get_if<TReadSessionEvent::TDataReceivedEvent>(&event)) { // Event is not data event.
return;
}
- *Settings.Counters_->MessagesInflight -= std::get<TReadSessionEvent::TDataReceivedEvent>(event).GetMessagesCount();
+ *Settings.Counters_->MessagesInflight -= std::get<TReadSessionEvent::TDataReceivedEvent>(event).GetMessagesCount();
*Settings.Counters_->BytesInflightTotal -= bytesCount;
*Settings.Counters_->BytesInflightUncompressed -= bytesCount;
@@ -1580,18 +1580,18 @@ TReadSessionEvent::TPartitionStreamClosedEvent::TPartitionStreamClosedEvent(TPar
}
TReadSessionEvent::TDataReceivedEvent::TDataReceivedEvent(TVector<TMessage> messages,
- TVector<TCompressedMessage> compressedMessages,
+ TVector<TCompressedMessage> compressedMessages,
TPartitionStream::TPtr partitionStream)
: Messages(std::move(messages))
- , CompressedMessages(std::move(compressedMessages))
+ , CompressedMessages(std::move(compressedMessages))
, PartitionStream(std::move(partitionStream))
{
- for (size_t i = 0; i < GetMessagesCount(); ++i) {
- auto [from, to] = GetMessageOffsetRange(*this, i);
- if (OffsetRanges.empty() || OffsetRanges.back().second != from) {
- OffsetRanges.emplace_back(from, to);
- } else {
- OffsetRanges.back().second = to;
+ for (size_t i = 0; i < GetMessagesCount(); ++i) {
+ auto [from, to] = GetMessageOffsetRange(*this, i);
+ if (OffsetRanges.empty() || OffsetRanges.back().second != from) {
+ OffsetRanges.emplace_back(from, to);
+ } else {
+ OffsetRanges.back().second = to;
}
}
}
@@ -1616,14 +1616,14 @@ TString TReadSessionEvent::TDataReceivedEvent::DebugString(bool printData) const
TStringBuilder ret;
ret << "DataReceived { PartitionStreamId: " << GetPartitionStream()->GetPartitionStreamId()
<< " PartitionId: " << GetPartitionStream()->GetPartitionId();
- for (const auto& message : Messages) {
+ for (const auto& message : Messages) {
+ ret << " ";
+ message.DebugString(ret, printData);
+ }
+ for (const auto& message : CompressedMessages) {
ret << " ";
- message.DebugString(ret, printData);
+ message.DebugString(ret, printData);
}
- for (const auto& message : CompressedMessages) {
- ret << " ";
- message.DebugString(ret, printData);
- }
ret << " }";
return std::move(ret);
}
@@ -1696,16 +1696,16 @@ TReadSessionEventInfo::TReadSessionEventInfo(TIntrusivePtr<TPartitionStreamImpl>
TReadSessionEventInfo::TReadSessionEventInfo(TIntrusivePtr<TPartitionStreamImpl> partitionStream,
std::weak_ptr<IUserRetrievedEventCallback> session,
- TVector<TReadSessionEvent::TDataReceivedEvent::TMessage> messages,
- TVector<TReadSessionEvent::TDataReceivedEvent::TCompressedMessage> compressedMessages)
+ TVector<TReadSessionEvent::TDataReceivedEvent::TMessage> messages,
+ TVector<TReadSessionEvent::TDataReceivedEvent::TCompressedMessage> compressedMessages)
: PartitionStream(std::move(partitionStream))
- , Event(
- NMaybe::TInPlace(),
- std::in_place_type_t<TReadSessionEvent::TDataReceivedEvent>(),
- std::move(messages),
- std::move(compressedMessages),
- PartitionStream
- )
+ , Event(
+ NMaybe::TInPlace(),
+ std::in_place_type_t<TReadSessionEvent::TDataReceivedEvent>(),
+ std::move(messages),
+ std::move(compressedMessages),
+ PartitionStream
+ )
, Session(std::move(session))
{
}
@@ -1745,11 +1745,11 @@ void TReadSessionEventInfo::OnUserRetrievedEvent() {
}
}
-bool TReadSessionEventInfo::TakeData(TVector<TReadSessionEvent::TDataReceivedEvent::TMessage>* messages,
- TVector<TReadSessionEvent::TDataReceivedEvent::TCompressedMessage>* compressedMessages,
- size_t* maxByteSize)
-{
- return PartitionStream->TopEvent().GetData().TakeData(PartitionStream, messages, compressedMessages, maxByteSize);
+bool TReadSessionEventInfo::TakeData(TVector<TReadSessionEvent::TDataReceivedEvent::TMessage>* messages,
+ TVector<TReadSessionEvent::TDataReceivedEvent::TCompressedMessage>* compressedMessages,
+ size_t* maxByteSize)
+{
+ return PartitionStream->TopEvent().GetData().TakeData(PartitionStream, messages, compressedMessages, maxByteSize);
}
TReadSessionEventsQueue::TReadSessionEventsQueue(const TSettings& settings, std::weak_ptr<IUserRetrievedEventCallback> session)
@@ -1799,17 +1799,17 @@ TDataDecompressionInfo* TReadSessionEventsQueue::PushDataEvent(TIntrusivePtr<TPa
}
with_lock (Mutex) {
- return &partitionStream->InsertDataEvent(std::move(msg), Settings.Decompress_);
+ return &partitionStream->InsertDataEvent(std::move(msg), Settings.Decompress_);
}
}
TMaybe<TReadSessionEventsQueue::TEventInfo> TReadSessionEventsQueue::GetDataEventImpl(TEventInfo& srcDataEventInfo, size_t* maxByteSize) { // Assumes that we're under lock.
TVector<TReadSessionEvent::TDataReceivedEvent::TMessage> messages;
- TVector<TReadSessionEvent::TDataReceivedEvent::TCompressedMessage> compressedMessages;
+ TVector<TReadSessionEvent::TDataReceivedEvent::TCompressedMessage> compressedMessages;
TIntrusivePtr<TPartitionStreamImpl> partitionStream = srcDataEventInfo.PartitionStream;
bool messageExtracted = false;
while (srcDataEventInfo.HasReadyUnreadData() && *maxByteSize > 0) {
- const bool hasMoreUnpackedData = srcDataEventInfo.TakeData(&messages, &compressedMessages, maxByteSize);
+ const bool hasMoreUnpackedData = srcDataEventInfo.TakeData(&messages, &compressedMessages, maxByteSize);
if (!hasMoreUnpackedData) {
const bool messageIsFullyRead = !srcDataEventInfo.HasMoreData();
if (messageIsFullyRead) {
@@ -1823,10 +1823,10 @@ TMaybe<TReadSessionEventsQueue::TEventInfo> TReadSessionEventsQueue::GetDataEven
partitionStream->TopEvent().Signalled = false;
}
- if (messages.empty() && compressedMessages.empty()) {
+ if (messages.empty() && compressedMessages.empty()) {
return Nothing();
}
- return TEventInfo(partitionStream, partitionStream->GetSession(), std::move(messages), std::move(compressedMessages));
+ return TEventInfo(partitionStream, partitionStream->GetSession(), std::move(messages), std::move(compressedMessages));
}
void TReadSessionEventsQueue::SignalReadyEvents(TPartitionStreamImpl* partitionStream) {
@@ -1938,14 +1938,14 @@ void TReadSessionEventsQueue::ClearAllEvents() {
}
}
-TDataDecompressionInfo::TDataDecompressionInfo(
- Ydb::PersQueue::V1::MigrationStreamingReadServerMessage::DataBatch::PartitionData&& msg,
- std::weak_ptr<TSingleClusterReadSessionImpl> session,
- bool doDecompress
-)
+TDataDecompressionInfo::TDataDecompressionInfo(
+ Ydb::PersQueue::V1::MigrationStreamingReadServerMessage::DataBatch::PartitionData&& msg,
+ std::weak_ptr<TSingleClusterReadSessionImpl> session,
+ bool doDecompress
+)
: ServerMessage(std::move(msg))
, Session(std::move(session))
- , DoDecompress(doDecompress)
+ , DoDecompress(doDecompress)
{
for (const Ydb::PersQueue::V1::MigrationStreamingReadServerMessage::DataBatch::Batch& batch : ServerMessage.batches()) {
for (const Ydb::PersQueue::V1::MigrationStreamingReadServerMessage::DataBatch::MessageData& messageData : batch.message_data()) {
@@ -2035,11 +2035,11 @@ i64 TDataDecompressionInfo::StartDecompressionTasks(const IExecutor::TPtr& execu
return used;
}
-bool TDataDecompressionInfo::TakeData(const TIntrusivePtr<TPartitionStreamImpl>& partitionStream,
- TVector<TReadSessionEvent::TDataReceivedEvent::TMessage>* messages,
- TVector<TReadSessionEvent::TDataReceivedEvent::TCompressedMessage>* compressedMessages,
- size_t* maxByteSize)
-{
+bool TDataDecompressionInfo::TakeData(const TIntrusivePtr<TPartitionStreamImpl>& partitionStream,
+ TVector<TReadSessionEvent::TDataReceivedEvent::TMessage>* messages,
+ TVector<TReadSessionEvent::TDataReceivedEvent::TCompressedMessage>* compressedMessages,
+ size_t* maxByteSize)
+{
TMaybe<std::pair<size_t, size_t>> readyThreshold = GetReadyThreshold();
Y_ASSERT(readyThreshold);
auto& msg = GetServerMessage();
@@ -2054,35 +2054,35 @@ bool TDataDecompressionInfo::TakeData(const TIntrusivePtr<TPartitionStreamImpl>&
auto& messageData = *batch.mutable_message_data(CurrentReadingMessage.second);
minOffset = Min(minOffset, messageData.offset());
maxOffset = Max(maxOffset, messageData.offset());
- TReadSessionEvent::TDataReceivedEvent::TMessageInformation messageInfo(
+ TReadSessionEvent::TDataReceivedEvent::TMessageInformation messageInfo(
messageData.offset(),
batch.source_id(),
messageData.seq_no(),
TInstant::MilliSeconds(messageData.create_timestamp_ms()),
batchWriteTimestamp,
batch.ip(),
- meta,
- messageData.uncompressed_size()
+ meta,
+ messageData.uncompressed_size()
);
- if (DoDecompress) {
- messages->emplace_back(
- messageData.data(),
- GetDecompressionError(CurrentReadingMessage.first, CurrentReadingMessage.second),
- messageInfo,
- partitionStream,
- messageData.partition_key(),
- messageData.explicit_hash()
- );
- } else {
- compressedMessages->emplace_back(
- static_cast<ECodec>(messageData.codec()),
- messageData.data(),
- TVector<TReadSessionEvent::TDataReceivedEvent::TMessageInformation>{messageInfo},
- partitionStream,
- messageData.partition_key(),
- messageData.explicit_hash()
- );
- }
+ if (DoDecompress) {
+ messages->emplace_back(
+ messageData.data(),
+ GetDecompressionError(CurrentReadingMessage.first, CurrentReadingMessage.second),
+ messageInfo,
+ partitionStream,
+ messageData.partition_key(),
+ messageData.explicit_hash()
+ );
+ } else {
+ compressedMessages->emplace_back(
+ static_cast<ECodec>(messageData.codec()),
+ messageData.data(),
+ TVector<TReadSessionEvent::TDataReceivedEvent::TMessageInformation>{messageInfo},
+ partitionStream,
+ messageData.partition_key(),
+ messageData.explicit_hash()
+ );
+ }
*maxByteSize -= Min(*maxByteSize, messageData.data().size());
// Clear data to free internal session's memory.
@@ -2154,15 +2154,15 @@ void TDataDecompressionInfo::TDecompressionTask::operator()() {
maxOffset = Max(maxOffset, data.offset());
try {
- if (Parent->DoDecompress
- && data.codec() != Ydb::PersQueue::V1::CODEC_RAW
- && data.codec() != Ydb::PersQueue::V1::CODEC_UNSPECIFIED
- ) {
- TString decompressed = NCompressionDetails::Decompress(data);
- data.set_data(decompressed);
- data.set_codec(Ydb::PersQueue::V1::CODEC_RAW);
- }
- DecompressedSize += data.data().size();
+ if (Parent->DoDecompress
+ && data.codec() != Ydb::PersQueue::V1::CODEC_RAW
+ && data.codec() != Ydb::PersQueue::V1::CODEC_UNSPECIFIED
+ ) {
+ TString decompressed = NCompressionDetails::Decompress(data);
+ data.set_data(decompressed);
+ data.set_codec(Ydb::PersQueue::V1::CODEC_RAW);
+ }
+ DecompressedSize += data.data().size();
} catch (...) {
Parent->PutDecompressionError(std::current_exception(), messages.Batch, i);
data.clear_data(); // Free memory, because we don't count it.
@@ -2303,17 +2303,17 @@ public:
}
void OnDataReceived(TReadSessionEvent::TDataReceivedEvent& event) {
- Y_ASSERT(event.GetMessagesCount());
+ Y_ASSERT(event.GetMessagesCount());
TDeferredCommit deferredCommit;
with_lock (Lock) {
auto& offsetSet = PartitionStreamToUncommittedOffsets[event.GetPartitionStream()->GetPartitionStreamId()];
// Messages could contain holes in offset, but later commit ack will tell us right border.
// So we can easily insert the whole interval with holes included.
// It will be removed from set by specifying proper right border.
- auto firstMessageOffsets = GetMessageOffsetRange(event, 0);
- auto lastMessageOffsets = GetMessageOffsetRange(event, event.GetMessagesCount() - 1);
+ auto firstMessageOffsets = GetMessageOffsetRange(event, 0);
+ auto lastMessageOffsets = GetMessageOffsetRange(event, event.GetMessagesCount() - 1);
- offsetSet.InsertInterval(firstMessageOffsets.first, lastMessageOffsets.second);
+ offsetSet.InsertInterval(firstMessageOffsets.first, lastMessageOffsets.second);
if (CommitAfterProcessing) {
deferredCommit.Add(event);
@@ -2457,7 +2457,7 @@ TDeferredCommit::~TDeferredCommit() {
Impl = MakeHolder<TImpl>(); \
} \
Impl
-
+
void TDeferredCommit::Add(const TPartitionStream::TPtr& partitionStream, ui64 startOffset, ui64 endOffset) {
GET_IMPL()->Add(partitionStream, startOffset, endOffset);
}
@@ -2485,8 +2485,8 @@ void TDeferredCommit::Commit() {
void TDeferredCommit::TImpl::Add(const TReadSessionEvent::TDataReceivedEvent::TMessage& message) {
Y_ASSERT(message.GetPartitionStream());
Add(message.GetPartitionStream(), message.GetOffset());
-}
-
+}
+
void TDeferredCommit::TImpl::Add(const TPartitionStream::TPtr& partitionStream, TDisjointIntervalTree<ui64>& offsetSet, ui64 startOffset, ui64 endOffset) {
if (offsetSet.Intersects(startOffset, endOffset)) {
ThrowFatalError(TStringBuilder() << "Commit set already has some offsets from half-interval ["
@@ -2517,15 +2517,15 @@ void TDeferredCommit::TImpl::Add(const TReadSessionEvent::TDataReceivedEvent& da
const TPartitionStream::TPtr& partitionStream = dataReceivedEvent.GetPartitionStream();
Y_ASSERT(partitionStream);
auto& offsetSet = Offsets[partitionStream];
- auto [startOffset, endOffset] = GetMessageOffsetRange(dataReceivedEvent, 0);
- for (size_t i = 1; i < dataReceivedEvent.GetMessagesCount(); ++i) {
- auto msgOffsetRange = GetMessageOffsetRange(dataReceivedEvent, i);
- if (msgOffsetRange.first == endOffset) {
- endOffset= msgOffsetRange.second;
+ auto [startOffset, endOffset] = GetMessageOffsetRange(dataReceivedEvent, 0);
+ for (size_t i = 1; i < dataReceivedEvent.GetMessagesCount(); ++i) {
+ auto msgOffsetRange = GetMessageOffsetRange(dataReceivedEvent, i);
+ if (msgOffsetRange.first == endOffset) {
+ endOffset= msgOffsetRange.second;
} else {
Add(partitionStream, offsetSet, startOffset, endOffset);
- startOffset = msgOffsetRange.first;
- endOffset = msgOffsetRange.second;
+ startOffset = msgOffsetRange.first;
+ endOffset = msgOffsetRange.second;
}
}
Add(partitionStream, offsetSet, startOffset, endOffset);
diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/read_session.h b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/read_session.h
index 8d3dfde9a6..b5b874954f 100644
--- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/read_session.h
+++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/read_session.h
@@ -1,7 +1,7 @@
#pragma once
-
-#include "common.h"
-
+
+#include "common.h"
+
#include <ydb/public/sdk/cpp/client/ydb_common_client/impl/client.h>
#include <ydb/public/api/grpc/draft/ydb_persqueue_v1.grpc.pb.h>
@@ -98,11 +98,11 @@ class TDataDecompressionInfo {
public:
TDataDecompressionInfo(const TDataDecompressionInfo&) = default;
TDataDecompressionInfo(TDataDecompressionInfo&&) = default;
- TDataDecompressionInfo(
- Ydb::PersQueue::V1::MigrationStreamingReadServerMessage::DataBatch::PartitionData&& msg,
- std::weak_ptr<TSingleClusterReadSessionImpl> session,
- bool doDecompress
- );
+ TDataDecompressionInfo(
+ Ydb::PersQueue::V1::MigrationStreamingReadServerMessage::DataBatch::PartitionData&& msg,
+ std::weak_ptr<TSingleClusterReadSessionImpl> session,
+ bool doDecompress
+ );
i64 StartDecompressionTasks(const IExecutor::TPtr& executor,
i64 availableMemory,
@@ -155,10 +155,10 @@ public:
}
// Takes data. Returns true if event has more unpacked data.
- bool TakeData(const TIntrusivePtr<TPartitionStreamImpl>& partitionStream,
- TVector<TReadSessionEvent::TDataReceivedEvent::TMessage>* messages,
- TVector<TReadSessionEvent::TDataReceivedEvent::TCompressedMessage>* compressedMessages,
- size_t* maxByteSize);
+ bool TakeData(const TIntrusivePtr<TPartitionStreamImpl>& partitionStream,
+ TVector<TReadSessionEvent::TDataReceivedEvent::TMessage>* messages,
+ TVector<TReadSessionEvent::TDataReceivedEvent::TCompressedMessage>* compressedMessages,
+ size_t* maxByteSize);
bool HasMoreData() const {
return CurrentReadingMessage.first < static_cast<size_t>(GetServerMessage().batches_size());
@@ -212,7 +212,7 @@ private:
Ydb::PersQueue::V1::MigrationStreamingReadServerMessage::DataBatch::PartitionData ServerMessage;
std::vector<TWriteSessionMeta::TPtr> BatchesMeta;
std::weak_ptr<TSingleClusterReadSessionImpl> Session;
- bool DoDecompress;
+ bool DoDecompress;
i64 CompressedDataSize = 0;
std::atomic<i64> SourceDataNotProcessed = 0;
std::pair<size_t, size_t> CurrentDecompressingMessage = {0, 0}; // (Batch, Message)
@@ -256,16 +256,16 @@ struct TReadSessionEventInfo {
TReadSessionEventInfo(TIntrusivePtr<TPartitionStreamImpl> partitionStream,
std::weak_ptr<IUserRetrievedEventCallback> session,
- TVector<TReadSessionEvent::TDataReceivedEvent::TMessage> messages,
- TVector<TReadSessionEvent::TDataReceivedEvent::TCompressedMessage> compressedMessages);
+ TVector<TReadSessionEvent::TDataReceivedEvent::TMessage> messages,
+ TVector<TReadSessionEvent::TDataReceivedEvent::TCompressedMessage> compressedMessages);
bool IsEmpty() const;
bool IsDataEvent() const;
// Takes data. Returns true if event has more unpacked data.
- bool TakeData(TVector<TReadSessionEvent::TDataReceivedEvent::TMessage>* messages,
- TVector<TReadSessionEvent::TDataReceivedEvent::TCompressedMessage>* comressedMessages,
- size_t* maxByteSize);
+ bool TakeData(TVector<TReadSessionEvent::TDataReceivedEvent::TMessage>* messages,
+ TVector<TReadSessionEvent::TDataReceivedEvent::TCompressedMessage>* comressedMessages,
+ size_t* maxByteSize);
TEvent& GetEvent() {
Y_ASSERT(Event);
@@ -295,12 +295,12 @@ struct TRawPartitionStreamEvent {
TRawPartitionStreamEvent(const TRawPartitionStreamEvent&) = default;
TRawPartitionStreamEvent(TRawPartitionStreamEvent&&) = default;
- TRawPartitionStreamEvent(
- Ydb::PersQueue::V1::MigrationStreamingReadServerMessage::DataBatch::PartitionData&& msg,
- std::weak_ptr<TSingleClusterReadSessionImpl> session,
- bool doDecompress
- )
- : Event(std::in_place_type_t<TDataDecompressionInfo>(), std::move(msg), std::move(session), doDecompress)
+ TRawPartitionStreamEvent(
+ Ydb::PersQueue::V1::MigrationStreamingReadServerMessage::DataBatch::PartitionData&& msg,
+ std::weak_ptr<TSingleClusterReadSessionImpl> session,
+ bool doDecompress
+ )
+ : Event(std::in_place_type_t<TDataDecompressionInfo>(), std::move(msg), std::move(session), doDecompress)
{
}
@@ -413,12 +413,12 @@ public:
EventsQueue.emplace_back(std::forward<T>(event));
}
- TDataDecompressionInfo& InsertDataEvent(
- Ydb::PersQueue::V1::MigrationStreamingReadServerMessage::DataBatch::PartitionData&& msg,
- bool doDecompress
- ) {
+ TDataDecompressionInfo& InsertDataEvent(
+ Ydb::PersQueue::V1::MigrationStreamingReadServerMessage::DataBatch::PartitionData&& msg,
+ bool doDecompress
+ ) {
++DataDecompressionEventsCount;
- return EventsQueue.emplace_back(std::move(msg), Session, doDecompress).GetData();
+ return EventsQueue.emplace_back(std::move(msg), Session, doDecompress).GetData();
}
bool IsWaitingForDataDecompression() const {
diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/read_session_messages.cpp b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/read_session_messages.cpp
index f8636286ec..2f0fd93a8f 100644
--- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/read_session_messages.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/read_session_messages.cpp
@@ -1,242 +1,242 @@
-#include "read_session.h"
-
+#include "read_session.h"
+
#include <ydb/public/sdk/cpp/client/ydb_persqueue_core/persqueue.h>
-
-namespace NYdb::NPersQueue {
-
-TReadSessionEvent::TDataReceivedEvent::TMessageInformation::TMessageInformation(
- ui64 offset,
- TString messageGroupId,
- ui64 seqNo,
- TInstant createTime,
- TInstant writeTime,
- TString ip,
- TWriteSessionMeta::TPtr meta,
- ui64 uncompressedSize
-)
- : Offset(offset)
- , MessageGroupId(messageGroupId)
- , SeqNo(seqNo)
- , CreateTime(createTime)
- , WriteTime(writeTime)
- , Ip(ip)
- , Meta(meta)
+
+namespace NYdb::NPersQueue {
+
+TReadSessionEvent::TDataReceivedEvent::TMessageInformation::TMessageInformation(
+ ui64 offset,
+ TString messageGroupId,
+ ui64 seqNo,
+ TInstant createTime,
+ TInstant writeTime,
+ TString ip,
+ TWriteSessionMeta::TPtr meta,
+ ui64 uncompressedSize
+)
+ : Offset(offset)
+ , MessageGroupId(messageGroupId)
+ , SeqNo(seqNo)
+ , CreateTime(createTime)
+ , WriteTime(writeTime)
+ , Ip(ip)
+ , Meta(meta)
, UncompressedSize(uncompressedSize)
-{}
-
-static void DebugStringImpl(const TReadSessionEvent::TDataReceivedEvent::TMessageInformation& info, TStringBuilder& ret) {
+{}
+
+static void DebugStringImpl(const TReadSessionEvent::TDataReceivedEvent::TMessageInformation& info, TStringBuilder& ret) {
ret << " Information: {"
- << " Offset: " << info.Offset
- << " SeqNo: " << info.SeqNo
- << " MessageGroupId: \"" << info.MessageGroupId << "\""
- << " CreateTime: " << info.CreateTime
- << " WriteTime: " << info.WriteTime
- << " Ip: \"" << info.Ip << "\""
- << " UncompressedSize: " << info.UncompressedSize;
- ret << " Meta: {";
- bool firstKey = true;
- for (const auto& [k, v] : info.Meta->Fields) {
- ret << (firstKey ? " \"" : ", \"") << k << "\": \"" << v << "\"";
- firstKey = false;
- }
- ret << " } }";
-}
-
-template <class TSerializeInformationFunc>
-static void DebugStringImpl(TStringBuilder& ret,
- const TString& name,
- const TReadSessionEvent::TDataReceivedEvent::IMessage& msg,
- bool printData,
- TSerializeInformationFunc serializeInformationFunc,
- std::optional<ECodec> codec = std::nullopt)
-{
- ret << name << " {";
- try {
- const TString& data = msg.GetData();
- if (printData) {
- ret << " Data: \"" << data << "\"";
- } else {
- ret << " Data: .." << data.size() << " bytes..";
- }
- } catch (...) {
- ret << " DataDecompressionError: \"" << CurrentExceptionMessage() << "\"";
- }
- auto partitionStream = msg.GetPartitionStream();
- ret << " Partition stream id: " << partitionStream->GetPartitionStreamId()
- << " Cluster: \"" << partitionStream->GetCluster() << "\". Topic: \"" << partitionStream->GetTopicPath() << "\""
- << " Partition: " << partitionStream->GetPartitionId()
- << " PartitionKey: \"" << msg.GetPartitionKey() << "\"";
- if (codec) {
- ret << " Codec: " << codec.value();
- }
+ << " Offset: " << info.Offset
+ << " SeqNo: " << info.SeqNo
+ << " MessageGroupId: \"" << info.MessageGroupId << "\""
+ << " CreateTime: " << info.CreateTime
+ << " WriteTime: " << info.WriteTime
+ << " Ip: \"" << info.Ip << "\""
+ << " UncompressedSize: " << info.UncompressedSize;
+ ret << " Meta: {";
+ bool firstKey = true;
+ for (const auto& [k, v] : info.Meta->Fields) {
+ ret << (firstKey ? " \"" : ", \"") << k << "\": \"" << v << "\"";
+ firstKey = false;
+ }
+ ret << " } }";
+}
+
+template <class TSerializeInformationFunc>
+static void DebugStringImpl(TStringBuilder& ret,
+ const TString& name,
+ const TReadSessionEvent::TDataReceivedEvent::IMessage& msg,
+ bool printData,
+ TSerializeInformationFunc serializeInformationFunc,
+ std::optional<ECodec> codec = std::nullopt)
+{
+ ret << name << " {";
+ try {
+ const TString& data = msg.GetData();
+ if (printData) {
+ ret << " Data: \"" << data << "\"";
+ } else {
+ ret << " Data: .." << data.size() << " bytes..";
+ }
+ } catch (...) {
+ ret << " DataDecompressionError: \"" << CurrentExceptionMessage() << "\"";
+ }
+ auto partitionStream = msg.GetPartitionStream();
+ ret << " Partition stream id: " << partitionStream->GetPartitionStreamId()
+ << " Cluster: \"" << partitionStream->GetCluster() << "\". Topic: \"" << partitionStream->GetTopicPath() << "\""
+ << " Partition: " << partitionStream->GetPartitionId()
+ << " PartitionKey: \"" << msg.GetPartitionKey() << "\"";
+ if (codec) {
+ ret << " Codec: " << codec.value();
+ }
serializeInformationFunc(ret);
- ret << " }";
-}
-
-const TString& TReadSessionEvent::TDataReceivedEvent::IMessage::GetData() const {
- return Data;
-}
-
-const TPartitionStream::TPtr& TReadSessionEvent::TDataReceivedEvent::IMessage::GetPartitionStream() const {
- return PartitionStream;
-}
-
-const TString& TReadSessionEvent::TDataReceivedEvent::IMessage::GetPartitionKey() const {
- return PartitionKey;
-}
-
-const TString TReadSessionEvent::TDataReceivedEvent::IMessage::GetExplicitHash() const {
- return ExplicitHash;
-}
-
-TString TReadSessionEvent::TDataReceivedEvent::IMessage::DebugString(bool printData) const {
- TStringBuilder ret;
- DebugString(ret, printData);
- return std::move(ret);
-}
-
-TReadSessionEvent::TDataReceivedEvent::IMessage::IMessage(const TString& data,
- TPartitionStream::TPtr partitionStream,
- const TString& partitionKey,
- const TString& explicitHash)
- : Data(data)
- , PartitionStream(partitionStream)
- , PartitionKey(partitionKey)
- , ExplicitHash(explicitHash)
-{}
-
-const TString& TReadSessionEvent::TDataReceivedEvent::TMessage::GetData() const {
- if (DecompressionException) {
- std::rethrow_exception(DecompressionException);
- }
- return IMessage::GetData();
-}
-
-bool TReadSessionEvent::TDataReceivedEvent::TMessage::HasException() const {
- return DecompressionException != nullptr;
-}
-
-ui64 TReadSessionEvent::TDataReceivedEvent::TMessage::GetOffset() const {
- return Information.Offset;
-}
-
-const TString& TReadSessionEvent::TDataReceivedEvent::TMessage::GetMessageGroupId() const {
- return Information.MessageGroupId;
-}
-
-ui64 TReadSessionEvent::TDataReceivedEvent::TMessage::GetSeqNo() const {
- return Information.SeqNo;
-}
-
-TInstant TReadSessionEvent::TDataReceivedEvent::TMessage::GetCreateTime() const {
- return Information.CreateTime;
-}
-
-TInstant TReadSessionEvent::TDataReceivedEvent::TMessage::GetWriteTime() const {
- return Information.WriteTime;
-}
-
-const TString& TReadSessionEvent::TDataReceivedEvent::TMessage::GetIp() const {
- return Information.Ip;
-}
-
-const TWriteSessionMeta::TPtr& TReadSessionEvent::TDataReceivedEvent::TMessage::GetMeta() const {
- return Information.Meta;
-}
-
-void TReadSessionEvent::TDataReceivedEvent::TMessage::DebugString(TStringBuilder& ret, bool printData) const {
- DebugStringImpl(ret, "Message", *this, printData, [this](TStringBuilder& ret) {
- DebugStringImpl(this->Information, ret);
- });
-}
-
-TReadSessionEvent::TDataReceivedEvent::TMessage::TMessage(const TString& data,
- std::exception_ptr decompressionException,
- const TMessageInformation& information,
- TPartitionStream::TPtr partitionStream,
- const TString& partitionKey,
- const TString& explicitHash)
- : IMessage(data, partitionStream, partitionKey, explicitHash)
- , DecompressionException(std::move(decompressionException))
- , Information(information)
-{
-}
-
-void TReadSessionEvent::TDataReceivedEvent::TMessage::Commit() {
- static_cast<TPartitionStreamImpl*>(PartitionStream.Get())->Commit(Information.Offset, Information.Offset + 1);
-}
-
-ui64 TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::GetBlocksCount() const {
- return Information.size();
-}
-
-ECodec TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::GetCodec() const {
- return Codec;
-}
-
-ui64 TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::GetOffset(ui64 index) const {
- return Information.at(index).Offset;
-}
-
-const TString& TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::GetMessageGroupId(ui64 index) const {
- return Information.at(index).MessageGroupId;
-}
-
-ui64 TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::GetSeqNo(ui64 index) const {
- return Information.at(index).SeqNo;
-}
-
-TInstant TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::GetCreateTime(ui64 index) const {
- return Information.at(index).CreateTime;
-}
-
-TInstant TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::GetWriteTime(ui64 index) const {
- return Information.at(index).WriteTime;
-}
-
-const TString& TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::GetIp(ui64 index) const {
- return Information.at(index).Ip;
-}
-
-const TWriteSessionMeta::TPtr& TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::GetMeta(ui64 index) const {
- return Information.at(index).Meta;
-}
-
-ui64 TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::GetUncompressedSize(ui64 index) const {
- return Information.at(index).UncompressedSize;
-}
-
-void TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::DebugString(TStringBuilder& ret, bool printData) const {
- DebugStringImpl(
- ret,
- "CompressedMessage",
- *this,
- printData,
- [this](TStringBuilder& ret) {
- for (auto& info : this->Information) {
- DebugStringImpl(info, ret);
- }
- },
- Codec
- );
-}
-
-TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::TCompressedMessage(ECodec codec,
- const TString& data,
- const TVector<TMessageInformation>& information,
- TPartitionStream::TPtr partitionStream,
- const TString& partitionKey,
- const TString& explicitHash)
- : IMessage(data, partitionStream, partitionKey, explicitHash)
- , Codec(codec)
- , Information(information)
-{}
-
-void TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::Commit() {
- static_cast<TPartitionStreamImpl*>(PartitionStream.Get())->Commit(
- Information.front().Offset,
- Information.back().Offset + 1
- );
-}
-
-} // namespace NYdb::NPersQueue
+ ret << " }";
+}
+
+const TString& TReadSessionEvent::TDataReceivedEvent::IMessage::GetData() const {
+ return Data;
+}
+
+const TPartitionStream::TPtr& TReadSessionEvent::TDataReceivedEvent::IMessage::GetPartitionStream() const {
+ return PartitionStream;
+}
+
+const TString& TReadSessionEvent::TDataReceivedEvent::IMessage::GetPartitionKey() const {
+ return PartitionKey;
+}
+
+const TString TReadSessionEvent::TDataReceivedEvent::IMessage::GetExplicitHash() const {
+ return ExplicitHash;
+}
+
+TString TReadSessionEvent::TDataReceivedEvent::IMessage::DebugString(bool printData) const {
+ TStringBuilder ret;
+ DebugString(ret, printData);
+ return std::move(ret);
+}
+
+TReadSessionEvent::TDataReceivedEvent::IMessage::IMessage(const TString& data,
+ TPartitionStream::TPtr partitionStream,
+ const TString& partitionKey,
+ const TString& explicitHash)
+ : Data(data)
+ , PartitionStream(partitionStream)
+ , PartitionKey(partitionKey)
+ , ExplicitHash(explicitHash)
+{}
+
+const TString& TReadSessionEvent::TDataReceivedEvent::TMessage::GetData() const {
+ if (DecompressionException) {
+ std::rethrow_exception(DecompressionException);
+ }
+ return IMessage::GetData();
+}
+
+bool TReadSessionEvent::TDataReceivedEvent::TMessage::HasException() const {
+ return DecompressionException != nullptr;
+}
+
+ui64 TReadSessionEvent::TDataReceivedEvent::TMessage::GetOffset() const {
+ return Information.Offset;
+}
+
+const TString& TReadSessionEvent::TDataReceivedEvent::TMessage::GetMessageGroupId() const {
+ return Information.MessageGroupId;
+}
+
+ui64 TReadSessionEvent::TDataReceivedEvent::TMessage::GetSeqNo() const {
+ return Information.SeqNo;
+}
+
+TInstant TReadSessionEvent::TDataReceivedEvent::TMessage::GetCreateTime() const {
+ return Information.CreateTime;
+}
+
+TInstant TReadSessionEvent::TDataReceivedEvent::TMessage::GetWriteTime() const {
+ return Information.WriteTime;
+}
+
+const TString& TReadSessionEvent::TDataReceivedEvent::TMessage::GetIp() const {
+ return Information.Ip;
+}
+
+const TWriteSessionMeta::TPtr& TReadSessionEvent::TDataReceivedEvent::TMessage::GetMeta() const {
+ return Information.Meta;
+}
+
+void TReadSessionEvent::TDataReceivedEvent::TMessage::DebugString(TStringBuilder& ret, bool printData) const {
+ DebugStringImpl(ret, "Message", *this, printData, [this](TStringBuilder& ret) {
+ DebugStringImpl(this->Information, ret);
+ });
+}
+
+TReadSessionEvent::TDataReceivedEvent::TMessage::TMessage(const TString& data,
+ std::exception_ptr decompressionException,
+ const TMessageInformation& information,
+ TPartitionStream::TPtr partitionStream,
+ const TString& partitionKey,
+ const TString& explicitHash)
+ : IMessage(data, partitionStream, partitionKey, explicitHash)
+ , DecompressionException(std::move(decompressionException))
+ , Information(information)
+{
+}
+
+void TReadSessionEvent::TDataReceivedEvent::TMessage::Commit() {
+ static_cast<TPartitionStreamImpl*>(PartitionStream.Get())->Commit(Information.Offset, Information.Offset + 1);
+}
+
+ui64 TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::GetBlocksCount() const {
+ return Information.size();
+}
+
+ECodec TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::GetCodec() const {
+ return Codec;
+}
+
+ui64 TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::GetOffset(ui64 index) const {
+ return Information.at(index).Offset;
+}
+
+const TString& TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::GetMessageGroupId(ui64 index) const {
+ return Information.at(index).MessageGroupId;
+}
+
+ui64 TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::GetSeqNo(ui64 index) const {
+ return Information.at(index).SeqNo;
+}
+
+TInstant TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::GetCreateTime(ui64 index) const {
+ return Information.at(index).CreateTime;
+}
+
+TInstant TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::GetWriteTime(ui64 index) const {
+ return Information.at(index).WriteTime;
+}
+
+const TString& TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::GetIp(ui64 index) const {
+ return Information.at(index).Ip;
+}
+
+const TWriteSessionMeta::TPtr& TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::GetMeta(ui64 index) const {
+ return Information.at(index).Meta;
+}
+
+ui64 TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::GetUncompressedSize(ui64 index) const {
+ return Information.at(index).UncompressedSize;
+}
+
+void TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::DebugString(TStringBuilder& ret, bool printData) const {
+ DebugStringImpl(
+ ret,
+ "CompressedMessage",
+ *this,
+ printData,
+ [this](TStringBuilder& ret) {
+ for (auto& info : this->Information) {
+ DebugStringImpl(info, ret);
+ }
+ },
+ Codec
+ );
+}
+
+TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::TCompressedMessage(ECodec codec,
+ const TString& data,
+ const TVector<TMessageInformation>& information,
+ TPartitionStream::TPtr partitionStream,
+ const TString& partitionKey,
+ const TString& explicitHash)
+ : IMessage(data, partitionStream, partitionKey, explicitHash)
+ , Codec(codec)
+ , Information(information)
+{}
+
+void TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::Commit() {
+ static_cast<TPartitionStreamImpl*>(PartitionStream.Get())->Commit(
+ Information.front().Offset,
+ Information.back().Offset + 1
+ );
+}
+
+} // namespace NYdb::NPersQueue
diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/write_session.cpp b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/write_session.cpp
index 0b91e6acaf..af4e08b979 100644
--- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/write_session.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/write_session.cpp
@@ -1,4 +1,4 @@
-#include "write_session.h"
+#include "write_session.h"
#include <ydb/public/sdk/cpp/client/ydb_persqueue_core/persqueue.h>
#include <library/cpp/string_utils/url/url.h>
diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/write_session.h b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/write_session.h
index 37529f48a2..324550bb02 100644
--- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/write_session.h
+++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/write_session.h
@@ -1,10 +1,10 @@
#pragma once
-#include "common.h"
+#include "common.h"
#include "persqueue_impl.h"
-
+
#include <ydb/public/sdk/cpp/client/ydb_persqueue_core/persqueue.h>
-
+
#include <util/generic/buffer.h>
diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/ya.make b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/ya.make
index f6e6e8a5e8..21c92fa949 100644
--- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/ya.make
+++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/ya.make
@@ -1,29 +1,29 @@
-LIBRARY()
-
-OWNER(
+LIBRARY()
+
+OWNER(
g:kikimr
- g:logbroker
-)
-
-SRCS(
- read_session_messages.cpp
- common.cpp
- write_session.cpp
- read_session.cpp
- persqueue.cpp
+ g:logbroker
+)
+
+SRCS(
+ read_session_messages.cpp
+ common.cpp
+ write_session.cpp
+ read_session.cpp
+ persqueue.cpp
persqueue_impl.cpp
-)
-
-PEERDIR(
- library/cpp/containers/disjoint_interval_tree
- library/cpp/grpc/client
+)
+
+PEERDIR(
+ library/cpp/containers/disjoint_interval_tree
+ library/cpp/grpc/client
library/cpp/monlib/dynamic_counters
- library/cpp/string_utils/url
+ library/cpp/string_utils/url
ydb/library/persqueue/obfuscate
ydb/public/api/grpc/draft
ydb/public/sdk/cpp/client/impl/ydb_internal/make_request
ydb/public/sdk/cpp/client/ydb_common_client/impl
ydb/public/sdk/cpp/client/ydb_driver
-)
-
-END()
+)
+
+END()
diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/persqueue.h b/ydb/public/sdk/cpp/client/ydb_persqueue_core/persqueue.h
index 407eaf4999..ee9aa75a0a 100644
--- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/persqueue.h
+++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/persqueue.h
@@ -9,7 +9,7 @@
#include <util/generic/hash.h>
#include <util/generic/maybe.h>
#include <util/generic/ptr.h>
-#include <util/string/builder.h>
+#include <util/string/builder.h>
#include <util/thread/pool.h>
#include <exception>
@@ -77,7 +77,7 @@ struct TDescribeTopicResult : public TStatus {
return SupportedCodecs_;
}
GETTER(ui32, Version);
- GETTER(TString, ServiceType);
+ GETTER(TString, ServiceType);
private:
TString ConsumerName_;
@@ -86,7 +86,7 @@ struct TDescribeTopicResult : public TStatus {
EFormat SupportedFormat_;
TVector<ECodec> SupportedCodecs_;
ui32 Version_;
- TString ServiceType_;
+ TString ServiceType_;
};
struct TRemoteMirrorRule {
@@ -180,7 +180,7 @@ struct TReadRuleSettings {
FLUENT_SETTING_DEFAULT(TVector<ECodec>, SupportedCodecs, GetDefaultCodecs());
FLUENT_SETTING_DEFAULT(ui32, Version, 0);
- FLUENT_SETTING(TString, ServiceType);
+ FLUENT_SETTING(TString, ServiceType);
TReadRuleSettings& SetSettings(const TDescribeTopicResult::TTopicSettings::TReadRule& settings) {
ConsumerName_ = settings.ConsumerName();
@@ -192,7 +192,7 @@ struct TReadRuleSettings {
SupportedCodecs_.push_back(codec);
}
Version_ = settings.Version();
- ServiceType_ = settings.ServiceType();
+ ServiceType_ = settings.ServiceType();
return *this;
}
@@ -476,210 +476,210 @@ struct TReadSessionEvent {
//! Contains batch of messages from single partition stream.
struct TDataReceivedEvent {
- struct TMessageInformation {
- TMessageInformation(ui64 offset,
- TString messageGroupId,
- ui64 seqNo,
- TInstant createTime,
- TInstant writeTime,
- TString ip,
- TWriteSessionMeta::TPtr meta,
- ui64 uncompressedSize);
- ui64 Offset;
- TString MessageGroupId;
- ui64 SeqNo;
- TInstant CreateTime;
- TInstant WriteTime;
- TString Ip;
- TWriteSessionMeta::TPtr Meta;
- ui64 UncompressedSize;
- };
-
- class IMessage {
- public:
- virtual const TString& GetData() const;
-
- //! Partition stream. Same as in batch.
- const TPartitionStream::TPtr& GetPartitionStream() const;
-
- const TString& GetPartitionKey() const;
-
- const TString GetExplicitHash() const;
-
- virtual void Commit() = 0;
-
- TString DebugString(bool printData = false) const;
- virtual void DebugString(TStringBuilder& ret, bool printData = false) const = 0;
-
- IMessage(const TString& data,
- TPartitionStream::TPtr partitionStream,
- const TString& partitionKey,
- const TString& explicitHash);
-
+ struct TMessageInformation {
+ TMessageInformation(ui64 offset,
+ TString messageGroupId,
+ ui64 seqNo,
+ TInstant createTime,
+ TInstant writeTime,
+ TString ip,
+ TWriteSessionMeta::TPtr meta,
+ ui64 uncompressedSize);
+ ui64 Offset;
+ TString MessageGroupId;
+ ui64 SeqNo;
+ TInstant CreateTime;
+ TInstant WriteTime;
+ TString Ip;
+ TWriteSessionMeta::TPtr Meta;
+ ui64 UncompressedSize;
+ };
+
+ class IMessage {
+ public:
+ virtual const TString& GetData() const;
+
+ //! Partition stream. Same as in batch.
+ const TPartitionStream::TPtr& GetPartitionStream() const;
+
+ const TString& GetPartitionKey() const;
+
+ const TString GetExplicitHash() const;
+
+ virtual void Commit() = 0;
+
+ TString DebugString(bool printData = false) const;
+ virtual void DebugString(TStringBuilder& ret, bool printData = false) const = 0;
+
+ IMessage(const TString& data,
+ TPartitionStream::TPtr partitionStream,
+ const TString& partitionKey,
+ const TString& explicitHash);
+
virtual ~IMessage() = default;
- protected:
- TString Data;
-
- TPartitionStream::TPtr PartitionStream;
- TString PartitionKey;
- TString ExplicitHash;
- };
-
+ protected:
+ TString Data;
+
+ TPartitionStream::TPtr PartitionStream;
+ TString PartitionKey;
+ TString ExplicitHash;
+ };
+
//! Single message.
- struct TMessage : public IMessage {
+ struct TMessage : public IMessage {
//! User data.
//! Throws decompressor exception if decompression failed.
- const TString& GetData() const override;
+ const TString& GetData() const override;
- bool HasException() const;
+ bool HasException() const;
//! Message offset.
- ui64 GetOffset() const;
+ ui64 GetOffset() const;
//! Message group id.
- const TString& GetMessageGroupId() const;
+ const TString& GetMessageGroupId() const;
//! Sequence number.
- ui64 GetSeqNo() const;
+ ui64 GetSeqNo() const;
//! Message creation timestamp.
- TInstant GetCreateTime() const;
+ TInstant GetCreateTime() const;
//! Message write timestamp.
- TInstant GetWriteTime() const;
+ TInstant GetWriteTime() const;
//! Ip address of message source host.
- const TString& GetIp() const;
+ const TString& GetIp() const;
//! Metainfo.
- const TWriteSessionMeta::TPtr& GetMeta() const;
+ const TWriteSessionMeta::TPtr& GetMeta() const;
TMessage(const TString& data,
std::exception_ptr decompressionException,
- const TMessageInformation& information,
+ const TMessageInformation& information,
TPartitionStream::TPtr partitionStream,
const TString& partitionKey,
const TString& explicitHash);
- //! Commits single message.
- void Commit() override;
-
- using IMessage::DebugString;
- void DebugString(TStringBuilder& ret, bool printData = false) const override;
-
+ //! Commits single message.
+ void Commit() override;
+
+ using IMessage::DebugString;
+ void DebugString(TStringBuilder& ret, bool printData = false) const override;
+
private:
std::exception_ptr DecompressionException;
- TMessageInformation Information;
+ TMessageInformation Information;
+ };
+
+ struct TCompressedMessage : public IMessage {
+ //! Messages count in compressed data
+ ui64 GetBlocksCount() const;
+
+ //! Message codec
+ ECodec GetCodec() const;
+
+ //! Message offset.
+ ui64 GetOffset(ui64 index) const;
+
+ //! Message group id.
+ const TString& GetMessageGroupId(ui64 index) const;
+
+ //! Sequence number.
+ ui64 GetSeqNo(ui64 index) const;
+
+ //! Message creation timestamp.
+ TInstant GetCreateTime(ui64 index) const;
+
+ //! Message write timestamp.
+ TInstant GetWriteTime(ui64 index) const;
+
+ //! Ip address of message source host.
+ const TString& GetIp(ui64 index) const;
+
+ //! Metainfo.
+ const TWriteSessionMeta::TPtr& GetMeta(ui64 index) const;
+
+ //! Uncompressed block size.
+ ui64 GetUncompressedSize(ui64 index) const;
+
+ virtual ~TCompressedMessage() {}
+ TCompressedMessage(ECodec codec,
+ const TString& data,
+ const TVector<TMessageInformation>& information,
+ TPartitionStream::TPtr partitionStream,
+ const TString& partitionKey,
+ const TString& explicitHash);
+
+ //! Commits all offsets in compressed message.
+ void Commit() override;
+
+ using IMessage::DebugString;
+ void DebugString(TStringBuilder& ret, bool printData = false) const override;
+
+ private:
+ ECodec Codec;
+ TVector<TMessageInformation> Information;
};
- struct TCompressedMessage : public IMessage {
- //! Messages count in compressed data
- ui64 GetBlocksCount() const;
-
- //! Message codec
- ECodec GetCodec() const;
-
- //! Message offset.
- ui64 GetOffset(ui64 index) const;
-
- //! Message group id.
- const TString& GetMessageGroupId(ui64 index) const;
-
- //! Sequence number.
- ui64 GetSeqNo(ui64 index) const;
-
- //! Message creation timestamp.
- TInstant GetCreateTime(ui64 index) const;
-
- //! Message write timestamp.
- TInstant GetWriteTime(ui64 index) const;
-
- //! Ip address of message source host.
- const TString& GetIp(ui64 index) const;
-
- //! Metainfo.
- const TWriteSessionMeta::TPtr& GetMeta(ui64 index) const;
-
- //! Uncompressed block size.
- ui64 GetUncompressedSize(ui64 index) const;
-
- virtual ~TCompressedMessage() {}
- TCompressedMessage(ECodec codec,
- const TString& data,
- const TVector<TMessageInformation>& information,
- TPartitionStream::TPtr partitionStream,
- const TString& partitionKey,
- const TString& explicitHash);
-
- //! Commits all offsets in compressed message.
- void Commit() override;
-
- using IMessage::DebugString;
- void DebugString(TStringBuilder& ret, bool printData = false) const override;
-
- private:
- ECodec Codec;
- TVector<TMessageInformation> Information;
- };
-
//! Partition stream.
const TPartitionStream::TPtr& GetPartitionStream() const {
return PartitionStream;
}
- bool IsCompressedMessages() const {
- return !CompressedMessages.empty();
- }
-
- size_t GetMessagesCount() const {
- return Messages.size() + CompressedMessages.size();
- }
-
+ bool IsCompressedMessages() const {
+ return !CompressedMessages.empty();
+ }
+
+ size_t GetMessagesCount() const {
+ return Messages.size() + CompressedMessages.size();
+ }
+
//! Get messages.
TVector<TMessage>& GetMessages() {
- CheckMessagesFilled(false);
+ CheckMessagesFilled(false);
return Messages;
}
const TVector<TMessage>& GetMessages() const {
- CheckMessagesFilled(false);
+ CheckMessagesFilled(false);
return Messages;
}
- //! Get compressed messages.
- TVector<TCompressedMessage>& GetCompressedMessages() {
- CheckMessagesFilled(true);
- return CompressedMessages;
- }
-
- const TVector<TCompressedMessage>& GetCompressedMessages() const {
- CheckMessagesFilled(true);
- return CompressedMessages;
- }
-
+ //! Get compressed messages.
+ TVector<TCompressedMessage>& GetCompressedMessages() {
+ CheckMessagesFilled(true);
+ return CompressedMessages;
+ }
+
+ const TVector<TCompressedMessage>& GetCompressedMessages() const {
+ CheckMessagesFilled(true);
+ return CompressedMessages;
+ }
+
//! Commits all messages in batch.
void Commit();
TString DebugString(bool printData = false) const;
TDataReceivedEvent(TVector<TMessage> messages,
- TVector<TCompressedMessage> compressedMessages,
+ TVector<TCompressedMessage> compressedMessages,
TPartitionStream::TPtr partitionStream);
private:
- void CheckMessagesFilled(bool compressed) const {
- Y_VERIFY(!Messages.empty() || !CompressedMessages.empty());
- if (compressed && CompressedMessages.empty()) {
- ythrow yexception() << "cannot get compressed messages, parameter decompress=true for read session";
- }
- if (!compressed && Messages.empty()) {
- ythrow yexception() << "cannot get decompressed messages, parameter decompress=false for read session";
- }
- }
-
- private:
+ void CheckMessagesFilled(bool compressed) const {
+ Y_VERIFY(!Messages.empty() || !CompressedMessages.empty());
+ if (compressed && CompressedMessages.empty()) {
+ ythrow yexception() << "cannot get compressed messages, parameter decompress=true for read session";
+ }
+ if (!compressed && Messages.empty()) {
+ ythrow yexception() << "cannot get decompressed messages, parameter decompress=false for read session";
+ }
+ }
+
+ private:
TVector<TMessage> Messages;
- TVector<TCompressedMessage> CompressedMessages;
+ TVector<TCompressedMessage> CompressedMessages;
TPartitionStream::TPtr PartitionStream;
std::vector<std::pair<ui64, ui64>> OffsetRanges;
};
@@ -853,12 +853,12 @@ public:
//! Add all messages from dataReceivedEvent to set.
void Add(const TReadSessionEvent::TDataReceivedEvent& dataReceivedEvent);
- //! Add offsets range to set.
- void Add(const TPartitionStream::TPtr& partitionStream, ui64 startOffset, ui64 endOffset);
-
- //! Add offset to set.
- void Add(const TPartitionStream::TPtr& partitionStream, ui64 offset);
-
+ //! Add offsets range to set.
+ void Add(const TPartitionStream::TPtr& partitionStream, ui64 startOffset, ui64 endOffset);
+
+ //! Add offset to set.
+ void Add(const TPartitionStream::TPtr& partitionStream, ui64 offset);
+
//! Commit all added offsets.
void Commit();
@@ -1296,9 +1296,9 @@ struct TReadSessionSettings : public TRequestSettings<TReadSessionSettings> {
//! See description in TEventHandlers class.
FLUENT_SETTING(TEventHandlers, EventHandlers);
- //! Decompress messages
- FLUENT_SETTING_DEFAULT(bool, Decompress, true);
-
+ //! Decompress messages
+ FLUENT_SETTING_DEFAULT(bool, Decompress, true);
+
//! Executor for decompression tasks.
//! If not set, default executor will be used.
FLUENT_SETTING(IExecutor::TPtr, DecompressionExecutor);
diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/basic_usage_ut.cpp b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/basic_usage_ut.cpp
index d26bc6948c..7e0ac17404 100644
--- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/basic_usage_ut.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/basic_usage_ut.cpp
@@ -3,15 +3,15 @@
#include <library/cpp/testing/unittest/registar.h>
#include <library/cpp/threading/future/future.h>
#include <library/cpp/threading/future/async.h>
-#include <library/cpp/threading/chunk_queue/queue.h>
-
+#include <library/cpp/threading/chunk_queue/queue.h>
+
#include <util/generic/ptr.h>
#include <util/generic/queue.h>
#include <util/string/join.h>
#include <util/system/event.h>
#include <util/stream/zlib.h>
#include <util/stream/str.h>
-
+
#include <atomic>
using namespace NThreading;
diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/compress_executor_ut.cpp b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/compress_executor_ut.cpp
index 081f08da4a..04abfab569 100644
--- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/compress_executor_ut.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/compress_executor_ut.cpp
@@ -1,4 +1,4 @@
-#include "ut_utils.h"
+#include "ut_utils.h"
namespace NYdb::NPersQueue::NTests {
diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/read_session_ut.cpp b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/read_session_ut.cpp
index bf780ce78f..9b2367e3f3 100644
--- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/read_session_ut.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/read_session_ut.cpp
@@ -1,12 +1,12 @@
-#include "ut_utils.h"
+#include "ut_utils.h"
-#define INCLUDE_YDB_INTERNAL_H
+#define INCLUDE_YDB_INTERNAL_H
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/logger/log.h>
-#undef INCLUDE_YDB_INTERNAL_H
-
+#undef INCLUDE_YDB_INTERNAL_H
+
#include <ydb/public/sdk/cpp/client/ydb_persqueue_core/persqueue.h>
#include <ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/read_session.h>
-
+
#include <library/cpp/streams/zstd/zstd.h>
#include <library/cpp/testing/gmock_in_unittest/gmock.h>
#include <library/cpp/testing/unittest/registar.h>
diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/retry_policy_ut.cpp b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/retry_policy_ut.cpp
index 74ae538f06..50d050e973 100644
--- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/retry_policy_ut.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/retry_policy_ut.cpp
@@ -1,5 +1,5 @@
-#include "ut_utils.h"
-
+#include "ut_utils.h"
+
#include <library/cpp/threading/future/future.h>
#include <library/cpp/testing/unittest/registar.h>
diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/data_plane_helpers.cpp b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/data_plane_helpers.cpp
index 0c9cd6d70e..3cf6e7f8b8 100644
--- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/data_plane_helpers.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/data_plane_helpers.cpp
@@ -1,4 +1,4 @@
-#include "data_plane_helpers.h"
+#include "data_plane_helpers.h"
namespace NKikimr::NPersQueueTests {
diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/sdk_test_setup.h b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/sdk_test_setup.h
index 3fe421d05b..677eb8c03d 100644
--- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/sdk_test_setup.h
+++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/sdk_test_setup.h
@@ -87,9 +87,9 @@ public:
return LocalDC;
}
- ui16 GetGrpcPort() const {
- return Server.GrpcPort;
- }
+ ui16 GetGrpcPort() const {
+ return Server.GrpcPort;
+ }
NGrpc::TServerOptions& GetGrpcServerOptions() {
return Server.GrpcServerOptions;
diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/ya.make b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/ya.make
index 93643a245b..b5f051599c 100644
--- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/ya.make
+++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/ya.make
@@ -3,7 +3,7 @@ LIBRARY()
OWNER(g:logbroker)
SRCS(
- data_plane_helpers.cpp
+ data_plane_helpers.cpp
sdk_test_setup.h
test_utils.h
test_server.h
diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/with_offset_ranges_mode_ut/ya.make b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/with_offset_ranges_mode_ut/ya.make
index 7d02dd4644..fc27259e6e 100644
--- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/with_offset_ranges_mode_ut/ya.make
+++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/with_offset_ranges_mode_ut/ya.make
@@ -2,7 +2,7 @@ UNITTEST()
OWNER(
g:kikimr
- g:logbroker
+ g:logbroker
)
IF (SANITIZER_TYPE)
diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ya.make b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ya.make
index 2ddf7a1b78..a95f5de06f 100644
--- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ya.make
+++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ya.make
@@ -2,7 +2,7 @@ UNITTEST_FOR(ydb/public/sdk/cpp/client/ydb_persqueue_core)
OWNER(
g:kikimr
- g:logbroker
+ g:logbroker
)
IF (SANITIZER_TYPE)
@@ -35,7 +35,7 @@ SRCS(
read_session_ut.cpp
basic_usage_ut.cpp
compress_executor_ut.cpp
- compression_ut.cpp
+ compression_ut.cpp
retry_policy_ut.cpp
ut_utils.cpp
)
diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ya.make b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ya.make
index 60235628ba..d19d4f6b2e 100644
--- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ya.make
+++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ya.make
@@ -1,24 +1,24 @@
-LIBRARY()
-
-OWNER(
+LIBRARY()
+
+OWNER(
g:kikimr
- g:logbroker
-)
-
+ g:logbroker
+)
+
GENERATE_ENUM_SERIALIZATION(ydb/public/sdk/cpp/client/ydb_persqueue_core/persqueue.h)
-
-SRCS(
+
+SRCS(
persqueue.h
proto_accessor.cpp
-)
-
-PEERDIR(
+)
+
+PEERDIR(
ydb/public/sdk/cpp/client/ydb_persqueue_core/impl
ydb/public/sdk/cpp/client/ydb_proto
-)
-
-END()
-
-RECURSE_FOR_TESTS(
- ut
-)
+)
+
+END()
+
+RECURSE_FOR_TESTS(
+ ut
+)
diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_public/codecs/codecs.cpp b/ydb/public/sdk/cpp/client/ydb_persqueue_public/codecs/codecs.cpp
index a33205780b..9c3bcbd25c 100644
--- a/ydb/public/sdk/cpp/client/ydb_persqueue_public/codecs/codecs.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_persqueue_public/codecs/codecs.cpp
@@ -21,7 +21,7 @@ IInputStream* CreateDecompressorStream(TInputStreamVariant& inputStreamStorage,
default:
//case Ydb::PersQueue::V1::CODEC_RAW:
//case Ydb::PersQueue::V1::CODEC_UNSPECIFIED:
- throw yexception() << "unsupported codec value : " << ui64(codec);
+ throw yexception() << "unsupported codec value : " << ui64(codec);
}
}
diff --git a/ydb/public/sdk/cpp/client/ydb_proto/accessor.cpp b/ydb/public/sdk/cpp/client/ydb_proto/accessor.cpp
index a50b398b0f..dd02fb89db 100644
--- a/ydb/public/sdk/cpp/client/ydb_proto/accessor.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_proto/accessor.cpp
@@ -1,7 +1,7 @@
-#include "accessor.h"
+#include "accessor.h"
#include <ydb/public/sdk/cpp/client/ydb_value/value.h>
-
+
namespace NYdb {
const Ydb::Type& TProtoAccessor::GetProto(const TType& type) {
diff --git a/ydb/public/sdk/cpp/client/ydb_proto/accessor.h b/ydb/public/sdk/cpp/client/ydb_proto/accessor.h
index 907e3ccc66..be0165d74b 100644
--- a/ydb/public/sdk/cpp/client/ydb_proto/accessor.h
+++ b/ydb/public/sdk/cpp/client/ydb_proto/accessor.h
@@ -11,19 +11,19 @@
#include <ydb/public/sdk/cpp/client/ydb_import/import.h>
#include <ydb/public/sdk/cpp/client/ydb_table/table.h>
#include <ydb/public/sdk/cpp/client/ydb_persqueue_public/persqueue.h>
-
+
namespace NYdb {
-class TResultSet;
-class TValue;
-class TType;
-
-namespace NTable {
-class TQueryStats;
-class TTableDescription;
+class TResultSet;
+class TValue;
+class TType;
+
+namespace NTable {
+class TQueryStats;
+class TTableDescription;
class TIndexDescription;
-}
-
+}
+
//! Provides access to raw protobuf values of YDB API entities. It is not recommended to use this
//! class in client applications as it add dependency on API protobuf format which is subject to
//! change. Use functionality provided by YDB SDK classes.
diff --git a/ydb/public/sdk/cpp/client/ydb_proto/ya.make b/ydb/public/sdk/cpp/client/ydb_proto/ya.make
index 02f26c7674..68f3d76432 100644
--- a/ydb/public/sdk/cpp/client/ydb_proto/ya.make
+++ b/ydb/public/sdk/cpp/client/ydb_proto/ya.make
@@ -1,19 +1,19 @@
-LIBRARY()
-
-OWNER(
- dcherednik
- g:kikimr
-)
-
-SRCS(
- accessor.cpp
-)
-
-PEERDIR(
+LIBRARY()
+
+OWNER(
+ dcherednik
+ g:kikimr
+)
+
+SRCS(
+ accessor.cpp
+)
+
+PEERDIR(
ydb/public/api/grpc/draft
ydb/public/api/protos
ydb/public/lib/operation_id/protos
ydb/library/yql/public/issue/protos
-)
-
-END()
+)
+
+END()
diff --git a/ydb/public/sdk/cpp/client/ydb_rate_limiter/rate_limiter.cpp b/ydb/public/sdk/cpp/client/ydb_rate_limiter/rate_limiter.cpp
index 0946263a56..bea557812c 100644
--- a/ydb/public/sdk/cpp/client/ydb_rate_limiter/rate_limiter.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_rate_limiter/rate_limiter.cpp
@@ -1,9 +1,9 @@
-#include "rate_limiter.h"
+#include "rate_limiter.h"
-#define INCLUDE_YDB_INTERNAL_H
+#define INCLUDE_YDB_INTERNAL_H
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.h>
-#undef INCLUDE_YDB_INTERNAL_H
-
+#undef INCLUDE_YDB_INTERNAL_H
+
#include <ydb/public/api/grpc/ydb_rate_limiter_v1.grpc.pb.h>
#include <ydb/public/sdk/cpp/client/ydb_common_client/impl/client.h>
@@ -186,7 +186,7 @@ public:
&Ydb::RateLimiter::V1::RateLimiterService::Stub::AsyncAcquireResource,
TRpcRequestSettings::Make(settings),
settings.ClientTimeout_);
- }
+ }
};
TRateLimiterClient::TRateLimiterClient(const TDriver& driver, const TCommonClientSettings& settings)
diff --git a/ydb/public/sdk/cpp/client/ydb_rate_limiter/rate_limiter.h b/ydb/public/sdk/cpp/client/ydb_rate_limiter/rate_limiter.h
index 4f91206c79..dca90fbe63 100644
--- a/ydb/public/sdk/cpp/client/ydb_rate_limiter/rate_limiter.h
+++ b/ydb/public/sdk/cpp/client/ydb_rate_limiter/rate_limiter.h
@@ -161,7 +161,7 @@ public:
TAsyncDescribeResourceResult DescribeResource(const TString& coordinationNodePath, const TString& resourcePath, const TDescribeResourceSettings& = {});
// Acquire resources's units inside a coordination node.
- TAsyncStatus AcquireResource(const TString& coordinationNodePath, const TString& resourcePath, const TAcquireResourceSettings& = {});
+ TAsyncStatus AcquireResource(const TString& coordinationNodePath, const TString& resourcePath, const TAcquireResourceSettings& = {});
private:
class TImpl;
diff --git a/ydb/public/sdk/cpp/client/ydb_rate_limiter/ya.make b/ydb/public/sdk/cpp/client/ydb_rate_limiter/ya.make
index 4c02b74c17..45f1d6d1f1 100644
--- a/ydb/public/sdk/cpp/client/ydb_rate_limiter/ya.make
+++ b/ydb/public/sdk/cpp/client/ydb_rate_limiter/ya.make
@@ -1,18 +1,18 @@
-LIBRARY()
-
-OWNER(
- dcherednik
- g:kikimr
-)
-
-SRCS(
- rate_limiter.cpp
-)
-
-PEERDIR(
+LIBRARY()
+
+OWNER(
+ dcherednik
+ g:kikimr
+)
+
+SRCS(
+ rate_limiter.cpp
+)
+
+PEERDIR(
ydb/public/api/grpc
ydb/public/sdk/cpp/client/ydb_common_client/impl
ydb/public/sdk/cpp/client/ydb_driver
-)
-
-END()
+)
+
+END()
diff --git a/ydb/public/sdk/cpp/client/ydb_result/proto_accessor.cpp b/ydb/public/sdk/cpp/client/ydb_result/proto_accessor.cpp
index 3b85ce84f5..3ff390d4ce 100644
--- a/ydb/public/sdk/cpp/client/ydb_result/proto_accessor.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_result/proto_accessor.cpp
@@ -1,11 +1,11 @@
-#include "result.h"
-
+#include "result.h"
+
#include <ydb/public/sdk/cpp/client/ydb_proto/accessor.h>
-
-namespace NYdb {
-
-const Ydb::ResultSet& TProtoAccessor::GetProto(const TResultSet& resultSet) {
- return resultSet.GetProto();
-}
-
-} // namespace NYdb
+
+namespace NYdb {
+
+const Ydb::ResultSet& TProtoAccessor::GetProto(const TResultSet& resultSet) {
+ return resultSet.GetProto();
+}
+
+} // namespace NYdb
diff --git a/ydb/public/sdk/cpp/client/ydb_result/result.cpp b/ydb/public/sdk/cpp/client/ydb_result/result.cpp
index 0b1420a10f..dc7eb09c0a 100644
--- a/ydb/public/sdk/cpp/client/ydb_result/result.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_result/result.cpp
@@ -1,11 +1,11 @@
-#include "result.h"
+#include "result.h"
#include <ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers/handlers.h>
-
+
#include <ydb/public/api/protos/ydb_common.pb.h>
#include <ydb/public/api/protos/ydb_value.pb.h>
-#include <util/generic/map.h>
+#include <util/generic/map.h>
#include <util/string/builder.h>
#include <google/protobuf/text_format.h>
diff --git a/ydb/public/sdk/cpp/client/ydb_result/result_ut.cpp b/ydb/public/sdk/cpp/client/ydb_result/result_ut.cpp
index 3c26543938..16c5fa5db2 100644
--- a/ydb/public/sdk/cpp/client/ydb_result/result_ut.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_result/result_ut.cpp
@@ -1,196 +1,196 @@
#include <ydb/public/api/protos/ydb_value.pb.h>
#include <ydb/public/sdk/cpp/client/ydb_result/result.h>
-
-#include <library/cpp/testing/unittest/registar.h>
-#include <library/cpp/testing/unittest/tests_data.h>
-
-#include <google/protobuf/text_format.h>
-
-using namespace NYdb;
-
-Y_UNIT_TEST_SUITE(CppGrpcClientResultSetTest) {
- Y_UNIT_TEST(ListResultSet) {
- const TString resultSetString =
- "columns {\n"
- " name: \"colName\"\n"
- " type {\n"
- " list_type {\n"
- " item {\n"
- " type_id: INT32\n"
- " }\n"
- " }\n"
- " }\n"
- "}\n"
- "rows {\n"
- " items {\n"
- " items {\n"
- " int32_value: 42\n"
- " }\n"
- " items {\n"
- " int32_value: 43\n"
- " }\n"
- " items {\n"
- " int32_value: 44\n"
- " }\n"
- " }\n"
- "}\n"
- "rows {\n"
- " items {\n"
- " items {\n"
- " int32_value: 45\n"
- " }\n"
- " items {\n"
- " int32_value: 46\n"
- " }\n"
- " items {\n"
- " int32_value: 47\n"
- " }\n"
- " }\n"
- "}\n";
- Ydb::ResultSet rsProto;
- google::protobuf::TextFormat::ParseFromString(resultSetString, &rsProto);
-
- NYdb::TResultSet rs(std::move(rsProto));
- NYdb::TResultSetParser rsParser(rs);
- UNIT_ASSERT_EQUAL(rsParser.ColumnsCount(), 1);
- UNIT_ASSERT_EQUAL(rsParser.RowsCount(), 2);
- UNIT_ASSERT_EQUAL(rsParser.ColumnIndex("colName"), 0);
- UNIT_ASSERT_EQUAL(rsParser.ColumnIndex("otherName"), -1);
- int expectedVal = 42;
- auto& column0 = rsParser.ColumnParser(0);
- while (rsParser.TryNextRow()) {
- column0.OpenList();
- while (column0.TryNextListItem()) {
- UNIT_ASSERT_EQUAL(column0.GetInt32(), expectedVal++);
- }
- }
- }
-
- Y_UNIT_TEST(OptionalDictResultSet) {
- const TString resultSetString =
- "columns {\n"
- " name: \"colName\"\n"
- " type {\n"
- " optional_type {\n"
- " item {\n"
- " dict_type {\n"
- " key {\n"
- " type_id: INT32\n"
- " }\n"
- " payload {\n"
- " type_id: STRING\n"
- " }\n"
- " }\n"
- " }\n"
- " }\n"
- " }\n"
- "}\n"
- "rows {\n"
- " items {\n"
- " pairs {\n"
- " key {\n"
- " int32_value: 42\n"
- " }\n"
- " payload {\n"
- " bytes_value: \"abc\"\n"
- " }\n"
- " }\n"
- " }\n"
- "}\n"
- "rows {\n"
- " items {\n"
- " pairs {\n"
- " key {\n"
- " int32_value: 43\n"
- " }\n"
- " payload {\n"
- " bytes_value: \"zxc\"\n"
- " }\n"
- " }\n"
- " }\n"
- "}\n";
- Ydb::ResultSet rsProto;
- google::protobuf::TextFormat::ParseFromString(resultSetString, &rsProto);
-
- NYdb::TResultSet rs(std::move(rsProto));
- NYdb::TResultSetParser rsParser(rs);
- UNIT_ASSERT_EQUAL(rsParser.ColumnsCount(), 1);
- UNIT_ASSERT_EQUAL(rsParser.RowsCount(), 2);
- UNIT_ASSERT_EQUAL(rsParser.ColumnIndex("colName"), 0);
- UNIT_ASSERT_EQUAL(rsParser.ColumnIndex("otherName"), -1);
- auto& column0 = rsParser.ColumnParser(0);
-
- UNIT_ASSERT(rsParser.TryNextRow());
- column0.OpenOptional();
- UNIT_ASSERT(!column0.IsNull());
- column0.OpenDict();
- UNIT_ASSERT(column0.TryNextDictItem());
- column0.DictKey();
- UNIT_ASSERT_EQUAL(column0.GetInt32(), 42);
- column0.DictPayload();
- UNIT_ASSERT_EQUAL(column0.GetString(), "abc");
-
- UNIT_ASSERT(rsParser.TryNextRow());
- column0.OpenOptional();
- UNIT_ASSERT(!column0.IsNull());
- column0.OpenDict();
- UNIT_ASSERT(column0.TryNextDictItem());
- column0.DictKey();
- UNIT_ASSERT_EQUAL(column0.GetInt32(), 43);
- column0.DictPayload();
- UNIT_ASSERT_EQUAL(column0.GetString(), "zxc");
- column0.DictKey();
- column0.DictKey();
- UNIT_ASSERT_EQUAL(column0.GetInt32(), 43);
- UNIT_ASSERT_EQUAL(column0.GetInt32(), 43);
- column0.DictPayload();
- UNIT_ASSERT_EQUAL(column0.GetString(), "zxc");
- UNIT_ASSERT_EQUAL(column0.GetString(), "zxc");
- }
-
- Y_UNIT_TEST(Utf8OptionalResultSet) {
- const TString resultSetString =
- "columns {\n"
- " name: \"colName\"\n"
- " type {\n"
- " optional_type {\n"
- " item {\n"
- " type_id: UTF8\n"
- " }\n"
- " }\n"
- " }\n"
- "}\n"
- "rows {\n"
- " items {\n"
- " text_value: \"йцукен\"\n"
- " }\n"
- "}\n"
- "rows {\n"
- " items {\n"
- " null_flag_value: NULL_VALUE\n"
- " }\n"
- "}\n";
- Ydb::ResultSet rsProto;
- google::protobuf::TextFormat::ParseFromString(resultSetString, &rsProto);
-
- NYdb::TResultSet rs(std::move(rsProto));
- NYdb::TResultSetParser rsParser(rs);
- UNIT_ASSERT_EQUAL(rsParser.ColumnsCount(), 1);
- UNIT_ASSERT_EQUAL(rsParser.RowsCount(), 2);
- auto& column0 = rsParser.ColumnParser(0);
-
- int row = 0;
- while (rsParser.TryNextRow()) {
- column0.OpenOptional();
-
- if (row == 0) {
- UNIT_ASSERT(!column0.IsNull());
- UNIT_ASSERT_EQUAL(column0.GetUtf8(), "йцукен");
- } else {
- UNIT_ASSERT(column0.IsNull());
- }
-
- row++;
- }
- }
-}
+
+#include <library/cpp/testing/unittest/registar.h>
+#include <library/cpp/testing/unittest/tests_data.h>
+
+#include <google/protobuf/text_format.h>
+
+using namespace NYdb;
+
+Y_UNIT_TEST_SUITE(CppGrpcClientResultSetTest) {
+ Y_UNIT_TEST(ListResultSet) {
+ const TString resultSetString =
+ "columns {\n"
+ " name: \"colName\"\n"
+ " type {\n"
+ " list_type {\n"
+ " item {\n"
+ " type_id: INT32\n"
+ " }\n"
+ " }\n"
+ " }\n"
+ "}\n"
+ "rows {\n"
+ " items {\n"
+ " items {\n"
+ " int32_value: 42\n"
+ " }\n"
+ " items {\n"
+ " int32_value: 43\n"
+ " }\n"
+ " items {\n"
+ " int32_value: 44\n"
+ " }\n"
+ " }\n"
+ "}\n"
+ "rows {\n"
+ " items {\n"
+ " items {\n"
+ " int32_value: 45\n"
+ " }\n"
+ " items {\n"
+ " int32_value: 46\n"
+ " }\n"
+ " items {\n"
+ " int32_value: 47\n"
+ " }\n"
+ " }\n"
+ "}\n";
+ Ydb::ResultSet rsProto;
+ google::protobuf::TextFormat::ParseFromString(resultSetString, &rsProto);
+
+ NYdb::TResultSet rs(std::move(rsProto));
+ NYdb::TResultSetParser rsParser(rs);
+ UNIT_ASSERT_EQUAL(rsParser.ColumnsCount(), 1);
+ UNIT_ASSERT_EQUAL(rsParser.RowsCount(), 2);
+ UNIT_ASSERT_EQUAL(rsParser.ColumnIndex("colName"), 0);
+ UNIT_ASSERT_EQUAL(rsParser.ColumnIndex("otherName"), -1);
+ int expectedVal = 42;
+ auto& column0 = rsParser.ColumnParser(0);
+ while (rsParser.TryNextRow()) {
+ column0.OpenList();
+ while (column0.TryNextListItem()) {
+ UNIT_ASSERT_EQUAL(column0.GetInt32(), expectedVal++);
+ }
+ }
+ }
+
+ Y_UNIT_TEST(OptionalDictResultSet) {
+ const TString resultSetString =
+ "columns {\n"
+ " name: \"colName\"\n"
+ " type {\n"
+ " optional_type {\n"
+ " item {\n"
+ " dict_type {\n"
+ " key {\n"
+ " type_id: INT32\n"
+ " }\n"
+ " payload {\n"
+ " type_id: STRING\n"
+ " }\n"
+ " }\n"
+ " }\n"
+ " }\n"
+ " }\n"
+ "}\n"
+ "rows {\n"
+ " items {\n"
+ " pairs {\n"
+ " key {\n"
+ " int32_value: 42\n"
+ " }\n"
+ " payload {\n"
+ " bytes_value: \"abc\"\n"
+ " }\n"
+ " }\n"
+ " }\n"
+ "}\n"
+ "rows {\n"
+ " items {\n"
+ " pairs {\n"
+ " key {\n"
+ " int32_value: 43\n"
+ " }\n"
+ " payload {\n"
+ " bytes_value: \"zxc\"\n"
+ " }\n"
+ " }\n"
+ " }\n"
+ "}\n";
+ Ydb::ResultSet rsProto;
+ google::protobuf::TextFormat::ParseFromString(resultSetString, &rsProto);
+
+ NYdb::TResultSet rs(std::move(rsProto));
+ NYdb::TResultSetParser rsParser(rs);
+ UNIT_ASSERT_EQUAL(rsParser.ColumnsCount(), 1);
+ UNIT_ASSERT_EQUAL(rsParser.RowsCount(), 2);
+ UNIT_ASSERT_EQUAL(rsParser.ColumnIndex("colName"), 0);
+ UNIT_ASSERT_EQUAL(rsParser.ColumnIndex("otherName"), -1);
+ auto& column0 = rsParser.ColumnParser(0);
+
+ UNIT_ASSERT(rsParser.TryNextRow());
+ column0.OpenOptional();
+ UNIT_ASSERT(!column0.IsNull());
+ column0.OpenDict();
+ UNIT_ASSERT(column0.TryNextDictItem());
+ column0.DictKey();
+ UNIT_ASSERT_EQUAL(column0.GetInt32(), 42);
+ column0.DictPayload();
+ UNIT_ASSERT_EQUAL(column0.GetString(), "abc");
+
+ UNIT_ASSERT(rsParser.TryNextRow());
+ column0.OpenOptional();
+ UNIT_ASSERT(!column0.IsNull());
+ column0.OpenDict();
+ UNIT_ASSERT(column0.TryNextDictItem());
+ column0.DictKey();
+ UNIT_ASSERT_EQUAL(column0.GetInt32(), 43);
+ column0.DictPayload();
+ UNIT_ASSERT_EQUAL(column0.GetString(), "zxc");
+ column0.DictKey();
+ column0.DictKey();
+ UNIT_ASSERT_EQUAL(column0.GetInt32(), 43);
+ UNIT_ASSERT_EQUAL(column0.GetInt32(), 43);
+ column0.DictPayload();
+ UNIT_ASSERT_EQUAL(column0.GetString(), "zxc");
+ UNIT_ASSERT_EQUAL(column0.GetString(), "zxc");
+ }
+
+ Y_UNIT_TEST(Utf8OptionalResultSet) {
+ const TString resultSetString =
+ "columns {\n"
+ " name: \"colName\"\n"
+ " type {\n"
+ " optional_type {\n"
+ " item {\n"
+ " type_id: UTF8\n"
+ " }\n"
+ " }\n"
+ " }\n"
+ "}\n"
+ "rows {\n"
+ " items {\n"
+ " text_value: \"йцукен\"\n"
+ " }\n"
+ "}\n"
+ "rows {\n"
+ " items {\n"
+ " null_flag_value: NULL_VALUE\n"
+ " }\n"
+ "}\n";
+ Ydb::ResultSet rsProto;
+ google::protobuf::TextFormat::ParseFromString(resultSetString, &rsProto);
+
+ NYdb::TResultSet rs(std::move(rsProto));
+ NYdb::TResultSetParser rsParser(rs);
+ UNIT_ASSERT_EQUAL(rsParser.ColumnsCount(), 1);
+ UNIT_ASSERT_EQUAL(rsParser.RowsCount(), 2);
+ auto& column0 = rsParser.ColumnParser(0);
+
+ int row = 0;
+ while (rsParser.TryNextRow()) {
+ column0.OpenOptional();
+
+ if (row == 0) {
+ UNIT_ASSERT(!column0.IsNull());
+ UNIT_ASSERT_EQUAL(column0.GetUtf8(), "йцукен");
+ } else {
+ UNIT_ASSERT(column0.IsNull());
+ }
+
+ row++;
+ }
+ }
+}
diff --git a/ydb/public/sdk/cpp/client/ydb_result/ut/ya.make b/ydb/public/sdk/cpp/client/ydb_result/ut/ya.make
index e9c668d542..d99231cee7 100644
--- a/ydb/public/sdk/cpp/client/ydb_result/ut/ya.make
+++ b/ydb/public/sdk/cpp/client/ydb_result/ut/ya.make
@@ -1,23 +1,23 @@
UNITTEST_FOR(ydb/public/sdk/cpp/client/ydb_result)
-
-OWNER(
- dcherednik
- g:kikimr
-)
-
-IF (SANITIZER_TYPE)
- TIMEOUT(1200)
- SIZE(LARGE)
- TAG(ya:fat)
-ELSE()
- TIMEOUT(600)
- SIZE(MEDIUM)
-ENDIF()
-
-FORK_SUBTESTS()
-
-SRCS(
- result_ut.cpp
-)
-
-END()
+
+OWNER(
+ dcherednik
+ g:kikimr
+)
+
+IF (SANITIZER_TYPE)
+ TIMEOUT(1200)
+ SIZE(LARGE)
+ TAG(ya:fat)
+ELSE()
+ TIMEOUT(600)
+ SIZE(MEDIUM)
+ENDIF()
+
+FORK_SUBTESTS()
+
+SRCS(
+ result_ut.cpp
+)
+
+END()
diff --git a/ydb/public/sdk/cpp/client/ydb_result/ya.make b/ydb/public/sdk/cpp/client/ydb_result/ya.make
index b67b5e0d55..5bccb8e3f4 100644
--- a/ydb/public/sdk/cpp/client/ydb_result/ya.make
+++ b/ydb/public/sdk/cpp/client/ydb_result/ya.make
@@ -1,23 +1,23 @@
-LIBRARY()
-
-OWNER(
- dcherednik
- g:kikimr
-)
-
-SRCS(
- proto_accessor.cpp
- result.cpp
-)
-
-PEERDIR(
+LIBRARY()
+
+OWNER(
+ dcherednik
+ g:kikimr
+)
+
+SRCS(
+ proto_accessor.cpp
+ result.cpp
+)
+
+PEERDIR(
ydb/public/api/protos
ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers
ydb/public/sdk/cpp/client/ydb_value
-)
-
-END()
-
+)
+
+END()
+
RECURSE_FOR_TESTS(
ut
)
diff --git a/ydb/public/sdk/cpp/client/ydb_scheme/scheme.cpp b/ydb/public/sdk/cpp/client/ydb_scheme/scheme.cpp
index 6b82d5a243..31f5535b96 100644
--- a/ydb/public/sdk/cpp/client/ydb_scheme/scheme.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_scheme/scheme.cpp
@@ -1,14 +1,14 @@
-#include "scheme.h"
+#include "scheme.h"
-#define INCLUDE_YDB_INTERNAL_H
+#define INCLUDE_YDB_INTERNAL_H
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.h>
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/table_helpers/helpers.h>
-#undef INCLUDE_YDB_INTERNAL_H
-
+#undef INCLUDE_YDB_INTERNAL_H
+
#include <ydb/public/api/grpc/ydb_scheme_v1.grpc.pb.h>
#include <ydb/public/api/protos/ydb_scheme.pb.h>
#include <ydb/public/sdk/cpp/client/ydb_common_client/impl/client.h>
-
+
namespace NYdb {
namespace NScheme {
diff --git a/ydb/public/sdk/cpp/client/ydb_scheme/ya.make b/ydb/public/sdk/cpp/client/ydb_scheme/ya.make
index c351b5ac71..fe5b7b5ae5 100644
--- a/ydb/public/sdk/cpp/client/ydb_scheme/ya.make
+++ b/ydb/public/sdk/cpp/client/ydb_scheme/ya.make
@@ -1,20 +1,20 @@
-LIBRARY()
-
-OWNER(
- dcherednik
- g:kikimr
-)
-
-SRCS(
- scheme.cpp
-)
-
-GENERATE_ENUM_SERIALIZATION(scheme.h)
-
-PEERDIR(
+LIBRARY()
+
+OWNER(
+ dcherednik
+ g:kikimr
+)
+
+SRCS(
+ scheme.cpp
+)
+
+GENERATE_ENUM_SERIALIZATION(scheme.h)
+
+PEERDIR(
ydb/public/sdk/cpp/client/impl/ydb_internal/make_request
ydb/public/sdk/cpp/client/ydb_common_client/impl
ydb/public/sdk/cpp/client/ydb_driver
-)
-
-END()
+)
+
+END()
diff --git a/ydb/public/sdk/cpp/client/ydb_table/impl/client_session.cpp b/ydb/public/sdk/cpp/client/ydb_table/impl/client_session.cpp
index 091b325094..edfee60847 100644
--- a/ydb/public/sdk/cpp/client/ydb_table/impl/client_session.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_table/impl/client_session.cpp
@@ -1,43 +1,43 @@
-#include "client_session.h"
-#include "data_query.h"
-
-namespace NYdb {
-namespace NTable {
-
-TSession::TImpl::TImpl(const TString& sessionId, const TString& endpoint, bool useQueryCache, ui32 queryCacheSize)
- : SessionId_(sessionId)
- , Endpoint_(endpoint)
- , State_(S_STANDALONE)
- , UseQueryCache_(useQueryCache)
- , QueryCache_(queryCacheSize)
- , Lock_()
- , TimeToTouch_(TInstant::Now())
- , TimeInPast_(TInstant::Now())
- , NeedUpdateActiveCounter_(false)
-{}
-
-TSession::TImpl::~TImpl() {
- Unlink();
-}
-
-const TString& TSession::TImpl::GetId() const {
- return SessionId_;
-}
-
-const TString& TSession::TImpl::GetEndpoint() const {
- return Endpoint_;
-}
-
-// Can be called from interceptor, need lock
-void TSession::TImpl::MarkBroken() {
- with_lock(Lock_) {
- if (State_ == EState::S_ACTIVE) {
- NeedUpdateActiveCounter_ = true;
- }
- State_ = EState::S_BROKEN;
- }
-}
-
+#include "client_session.h"
+#include "data_query.h"
+
+namespace NYdb {
+namespace NTable {
+
+TSession::TImpl::TImpl(const TString& sessionId, const TString& endpoint, bool useQueryCache, ui32 queryCacheSize)
+ : SessionId_(sessionId)
+ , Endpoint_(endpoint)
+ , State_(S_STANDALONE)
+ , UseQueryCache_(useQueryCache)
+ , QueryCache_(queryCacheSize)
+ , Lock_()
+ , TimeToTouch_(TInstant::Now())
+ , TimeInPast_(TInstant::Now())
+ , NeedUpdateActiveCounter_(false)
+{}
+
+TSession::TImpl::~TImpl() {
+ Unlink();
+}
+
+const TString& TSession::TImpl::GetId() const {
+ return SessionId_;
+}
+
+const TString& TSession::TImpl::GetEndpoint() const {
+ return Endpoint_;
+}
+
+// Can be called from interceptor, need lock
+void TSession::TImpl::MarkBroken() {
+ with_lock(Lock_) {
+ if (State_ == EState::S_ACTIVE) {
+ NeedUpdateActiveCounter_ = true;
+ }
+ State_ = EState::S_BROKEN;
+ }
+}
+
void TSession::TImpl::MarkAsClosing() {
with_lock(Lock_) {
if (State_ == EState::S_ACTIVE) {
@@ -48,149 +48,149 @@ void TSession::TImpl::MarkAsClosing() {
}
}
-void TSession::TImpl::MarkStandalone() {
- State_ = EState::S_STANDALONE;
- NeedUpdateActiveCounter_ = false;
-}
-
-void TSession::TImpl::MarkActive() {
- State_ = EState::S_ACTIVE;
- NeedUpdateActiveCounter_ = false;
-}
-
-void TSession::TImpl::MarkIdle() {
- State_ = EState::S_IDLE;
- NeedUpdateActiveCounter_ = false;
-}
-
-// Can be called from interceptor, need lock
-void TSession::TImpl::MarkDisconnected() {
- with_lock(Lock_) {
- if (State_ == EState::S_ACTIVE) {
- NeedUpdateActiveCounter_ = true;
- }
- State_ = EState::S_DISCONNECTED;
- }
-}
-
-TSession::TImpl::EState TSession::TImpl::GetState() const {
- // See comments in InjectSessionStatusInterception about lock
- with_lock(Lock_) {
- return State_;
- }
-}
-
-void TSession::TImpl::SetNeedUpdateActiveCounter(bool flag) {
- NeedUpdateActiveCounter_ = flag;
-}
-
-bool TSession::TImpl::NeedUpdateActiveCounter() const {
- return NeedUpdateActiveCounter_;
-}
-
-void TSession::TImpl::InvalidateQueryInCache(const TString& key) {
- if (!UseQueryCache_) {
- return;
- }
-
- with_lock(Lock_) {
- auto it = QueryCache_.Find(key);
- if (it != QueryCache_.End()) {
- QueryCache_.Erase(it);
- }
- }
-}
-
-void TSession::TImpl::InvalidateQueryCache() {
- if (!UseQueryCache_) {
- return;
- }
-
- with_lock(Lock_) {
- QueryCache_.Clear();
- }
-}
-
-TMaybe<TSession::TImpl::TDataQueryInfo> TSession::TImpl::GetQueryFromCache(const TString& query, bool allowMigration) {
- if (!UseQueryCache_) {
- return {};
- }
-
- auto key = EncodeQuery(query, allowMigration);
-
- with_lock(Lock_) {
- auto it = QueryCache_.Find(key);
- if (it != QueryCache_.End()) {
- return *it;
- }
- }
-
- return Nothing();
-}
-
-void TSession::TImpl::AddQueryToCache(const TDataQuery& query) {
- if (!UseQueryCache_) {
- return;
- }
-
- const auto& id = query.Impl_->GetId();
- if (id.empty()) {
- return;
- }
-
- auto key = query.Impl_->GetTextHash();
- TDataQueryInfo queryInfo(id, query.Impl_->GetParameterTypes());
-
- with_lock(Lock_) {
- auto it = QueryCache_.Find(key);
- if (it != QueryCache_.End()) {
- *it = queryInfo;
- } else {
- QueryCache_.Insert(key, queryInfo);
- }
- }
-}
-
-// We need lock here because this method can be called from different thread if client
-// makes simultaneous calls on one session. It should be possible to rewrite this part.
-void TSession::TImpl::ScheduleTimeToTouch(TDuration interval, bool updateTimeInPast) {
- auto now = TInstant::Now();
- with_lock(Lock_) {
- if (updateTimeInPast) {
- TimeInPast_ = now;
- }
- TimeToTouch_ = now + interval;
- }
-}
-
-void TSession::TImpl::ScheduleTimeToTouchFast(TDuration interval, bool updateTimeInPast) {
- auto now = TInstant::Now();
- if (updateTimeInPast) {
- TimeInPast_ = now;
- }
- TimeToTouch_ = now + interval;
-}
-
-TInstant TSession::TImpl::GetTimeToTouchFast() const {
- return TimeToTouch_;
-}
-
-TInstant TSession::TImpl::GetTimeInPastFast() const {
- return TimeInPast_;
-}
-
-// SetTimeInterval/GetTimeInterval, are not atomic!
-void TSession::TImpl::SetTimeInterval(TDuration interval) {
- TimeInterval_ = interval;
-}
-
-TDuration TSession::TImpl::GetTimeInterval() const {
- return TimeInterval_;
-}
-
-const TLRUCache<TString, TSession::TImpl::TDataQueryInfo>& TSession::TImpl::GetQueryCacheUnsafe() const {
- return QueryCache_;
-}
-
-} // namespace NTable
-} // namespace NYdb
+void TSession::TImpl::MarkStandalone() {
+ State_ = EState::S_STANDALONE;
+ NeedUpdateActiveCounter_ = false;
+}
+
+void TSession::TImpl::MarkActive() {
+ State_ = EState::S_ACTIVE;
+ NeedUpdateActiveCounter_ = false;
+}
+
+void TSession::TImpl::MarkIdle() {
+ State_ = EState::S_IDLE;
+ NeedUpdateActiveCounter_ = false;
+}
+
+// Can be called from interceptor, need lock
+void TSession::TImpl::MarkDisconnected() {
+ with_lock(Lock_) {
+ if (State_ == EState::S_ACTIVE) {
+ NeedUpdateActiveCounter_ = true;
+ }
+ State_ = EState::S_DISCONNECTED;
+ }
+}
+
+TSession::TImpl::EState TSession::TImpl::GetState() const {
+ // See comments in InjectSessionStatusInterception about lock
+ with_lock(Lock_) {
+ return State_;
+ }
+}
+
+void TSession::TImpl::SetNeedUpdateActiveCounter(bool flag) {
+ NeedUpdateActiveCounter_ = flag;
+}
+
+bool TSession::TImpl::NeedUpdateActiveCounter() const {
+ return NeedUpdateActiveCounter_;
+}
+
+void TSession::TImpl::InvalidateQueryInCache(const TString& key) {
+ if (!UseQueryCache_) {
+ return;
+ }
+
+ with_lock(Lock_) {
+ auto it = QueryCache_.Find(key);
+ if (it != QueryCache_.End()) {
+ QueryCache_.Erase(it);
+ }
+ }
+}
+
+void TSession::TImpl::InvalidateQueryCache() {
+ if (!UseQueryCache_) {
+ return;
+ }
+
+ with_lock(Lock_) {
+ QueryCache_.Clear();
+ }
+}
+
+TMaybe<TSession::TImpl::TDataQueryInfo> TSession::TImpl::GetQueryFromCache(const TString& query, bool allowMigration) {
+ if (!UseQueryCache_) {
+ return {};
+ }
+
+ auto key = EncodeQuery(query, allowMigration);
+
+ with_lock(Lock_) {
+ auto it = QueryCache_.Find(key);
+ if (it != QueryCache_.End()) {
+ return *it;
+ }
+ }
+
+ return Nothing();
+}
+
+void TSession::TImpl::AddQueryToCache(const TDataQuery& query) {
+ if (!UseQueryCache_) {
+ return;
+ }
+
+ const auto& id = query.Impl_->GetId();
+ if (id.empty()) {
+ return;
+ }
+
+ auto key = query.Impl_->GetTextHash();
+ TDataQueryInfo queryInfo(id, query.Impl_->GetParameterTypes());
+
+ with_lock(Lock_) {
+ auto it = QueryCache_.Find(key);
+ if (it != QueryCache_.End()) {
+ *it = queryInfo;
+ } else {
+ QueryCache_.Insert(key, queryInfo);
+ }
+ }
+}
+
+// We need lock here because this method can be called from different thread if client
+// makes simultaneous calls on one session. It should be possible to rewrite this part.
+void TSession::TImpl::ScheduleTimeToTouch(TDuration interval, bool updateTimeInPast) {
+ auto now = TInstant::Now();
+ with_lock(Lock_) {
+ if (updateTimeInPast) {
+ TimeInPast_ = now;
+ }
+ TimeToTouch_ = now + interval;
+ }
+}
+
+void TSession::TImpl::ScheduleTimeToTouchFast(TDuration interval, bool updateTimeInPast) {
+ auto now = TInstant::Now();
+ if (updateTimeInPast) {
+ TimeInPast_ = now;
+ }
+ TimeToTouch_ = now + interval;
+}
+
+TInstant TSession::TImpl::GetTimeToTouchFast() const {
+ return TimeToTouch_;
+}
+
+TInstant TSession::TImpl::GetTimeInPastFast() const {
+ return TimeInPast_;
+}
+
+// SetTimeInterval/GetTimeInterval, are not atomic!
+void TSession::TImpl::SetTimeInterval(TDuration interval) {
+ TimeInterval_ = interval;
+}
+
+TDuration TSession::TImpl::GetTimeInterval() const {
+ return TimeInterval_;
+}
+
+const TLRUCache<TString, TSession::TImpl::TDataQueryInfo>& TSession::TImpl::GetQueryCacheUnsafe() const {
+ return QueryCache_;
+}
+
+} // namespace NTable
+} // namespace NYdb
diff --git a/ydb/public/sdk/cpp/client/ydb_table/impl/client_session.h b/ydb/public/sdk/cpp/client/ydb_table/impl/client_session.h
index ef12876a30..1c33edc333 100644
--- a/ydb/public/sdk/cpp/client/ydb_table/impl/client_session.h
+++ b/ydb/public/sdk/cpp/client/ydb_table/impl/client_session.h
@@ -2,11 +2,11 @@
#include <ydb/public/sdk/cpp/client/ydb_table/table.h>
#include <ydb/public/sdk/cpp/client/impl/ydb_endpoints/endpoints.h>
-
+
#include <ydb/public/api/protos/ydb_table.pb.h>
#include <library/cpp/cache/cache.h>
-
+
#include <util/datetime/base.h>
#include <functional>
diff --git a/ydb/public/sdk/cpp/client/ydb_table/impl/data_query.cpp b/ydb/public/sdk/cpp/client/ydb_table/impl/data_query.cpp
index 7d71abf864..ce6f4b5efb 100644
--- a/ydb/public/sdk/cpp/client/ydb_table/impl/data_query.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_table/impl/data_query.cpp
@@ -1,61 +1,61 @@
-#include "data_query.h"
-
-#include <contrib/libs/openssl/include/openssl/sha.h>
-
-namespace NYdb {
-namespace NTable {
-
-TString GetQueryHash(const TString& text) {
- SHA256_CTX sha;
- SHA256_Init(&sha);
- SHA256_Update(&sha, text.data(), text.size());
- unsigned char hash[SHA256_DIGEST_LENGTH];
- SHA256_Final(hash, &sha);
- return TString(reinterpret_cast<char*>(hash), sizeof(hash));
-}
-
-TString EncodeQuery(const TString& text, bool reversible) {
- if (reversible) {
- //TODO: may be compress and decompress this query
- return text;
- } else {
- return GetQueryHash(text);
- }
-}
-
-////////////////////////////////////////////////////////////////////////////////
-
-TDataQuery::TImpl::TImpl(const TSession& session, const TString& text, bool keepText, const TString& id, bool allowMigration)
- : Session_(session)
- , Id_(id)
- , TextHash_(EncodeQuery(text, allowMigration))
- , Text_(keepText ? text : TMaybe<TString>())
-{}
-
-TDataQuery::TImpl::TImpl(const TSession& session, const TString& text, bool keepText, const TString& id, bool allowMigration,
- const ::google::protobuf::Map<TString, Ydb::Type>& types)
- : Session_(session)
- , Id_(id)
- , ParameterTypes_(types)
- , TextHash_(EncodeQuery(text, allowMigration))
- , Text_(keepText ? text : TMaybe<TString>())
-{}
-
-const TString& TDataQuery::TImpl::GetId() const {
- return Id_;
-}
-
-const ::google::protobuf::Map<TString, Ydb::Type>& TDataQuery::TImpl::GetParameterTypes() const {
- return ParameterTypes_;
-}
-
-const TString& TDataQuery::TImpl::GetTextHash() const {
- return TextHash_;
-}
-
-const TMaybe<TString>& TDataQuery::TImpl::GetText() const {
- return Text_;
-}
-
-} // namespace NTable
-} // namespace NYdb
+#include "data_query.h"
+
+#include <contrib/libs/openssl/include/openssl/sha.h>
+
+namespace NYdb {
+namespace NTable {
+
+TString GetQueryHash(const TString& text) {
+ SHA256_CTX sha;
+ SHA256_Init(&sha);
+ SHA256_Update(&sha, text.data(), text.size());
+ unsigned char hash[SHA256_DIGEST_LENGTH];
+ SHA256_Final(hash, &sha);
+ return TString(reinterpret_cast<char*>(hash), sizeof(hash));
+}
+
+TString EncodeQuery(const TString& text, bool reversible) {
+ if (reversible) {
+ //TODO: may be compress and decompress this query
+ return text;
+ } else {
+ return GetQueryHash(text);
+ }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+TDataQuery::TImpl::TImpl(const TSession& session, const TString& text, bool keepText, const TString& id, bool allowMigration)
+ : Session_(session)
+ , Id_(id)
+ , TextHash_(EncodeQuery(text, allowMigration))
+ , Text_(keepText ? text : TMaybe<TString>())
+{}
+
+TDataQuery::TImpl::TImpl(const TSession& session, const TString& text, bool keepText, const TString& id, bool allowMigration,
+ const ::google::protobuf::Map<TString, Ydb::Type>& types)
+ : Session_(session)
+ , Id_(id)
+ , ParameterTypes_(types)
+ , TextHash_(EncodeQuery(text, allowMigration))
+ , Text_(keepText ? text : TMaybe<TString>())
+{}
+
+const TString& TDataQuery::TImpl::GetId() const {
+ return Id_;
+}
+
+const ::google::protobuf::Map<TString, Ydb::Type>& TDataQuery::TImpl::GetParameterTypes() const {
+ return ParameterTypes_;
+}
+
+const TString& TDataQuery::TImpl::GetTextHash() const {
+ return TextHash_;
+}
+
+const TMaybe<TString>& TDataQuery::TImpl::GetText() const {
+ return Text_;
+}
+
+} // namespace NTable
+} // namespace NYdb
diff --git a/ydb/public/sdk/cpp/client/ydb_table/impl/data_query.h b/ydb/public/sdk/cpp/client/ydb_table/impl/data_query.h
index 462d356702..2ebbd5296e 100644
--- a/ydb/public/sdk/cpp/client/ydb_table/impl/data_query.h
+++ b/ydb/public/sdk/cpp/client/ydb_table/impl/data_query.h
@@ -1,36 +1,36 @@
-#pragma once
-
+#pragma once
+
#include <ydb/public/sdk/cpp/client/ydb_table/table.h>
#include <ydb/public/api/protos/ydb_table.pb.h>
-
-namespace NYdb {
-namespace NTable {
-
-TString EncodeQuery(const TString& text, bool reversible);
-
-////////////////////////////////////////////////////////////////////////////////
-
-class TDataQuery::TImpl {
- friend class TDataQuery;
-
-public:
- TImpl(const TSession& session, const TString& text, bool keepText, const TString& id, bool allowMigration);
-
- TImpl(const TSession& session, const TString& text, bool keepText, const TString& id, bool allowMigration,
- const ::google::protobuf::Map<TString, Ydb::Type>& types);
-
- const TString& GetId() const;
- const ::google::protobuf::Map<TString, Ydb::Type>& GetParameterTypes() const;
- const TString& GetTextHash() const;
- const TMaybe<TString>& GetText() const;
-
-private:
- NTable::TSession Session_;
- TString Id_;
- ::google::protobuf::Map<TString, Ydb::Type> ParameterTypes_;
- TString TextHash_;
- TMaybe<TString> Text_;
-};
-
-} // namespace NTable
-} // namespace NYdb
+
+namespace NYdb {
+namespace NTable {
+
+TString EncodeQuery(const TString& text, bool reversible);
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TDataQuery::TImpl {
+ friend class TDataQuery;
+
+public:
+ TImpl(const TSession& session, const TString& text, bool keepText, const TString& id, bool allowMigration);
+
+ TImpl(const TSession& session, const TString& text, bool keepText, const TString& id, bool allowMigration,
+ const ::google::protobuf::Map<TString, Ydb::Type>& types);
+
+ const TString& GetId() const;
+ const ::google::protobuf::Map<TString, Ydb::Type>& GetParameterTypes() const;
+ const TString& GetTextHash() const;
+ const TMaybe<TString>& GetText() const;
+
+private:
+ NTable::TSession Session_;
+ TString Id_;
+ ::google::protobuf::Map<TString, Ydb::Type> ParameterTypes_;
+ TString TextHash_;
+ TMaybe<TString> Text_;
+};
+
+} // namespace NTable
+} // namespace NYdb
diff --git a/ydb/public/sdk/cpp/client/ydb_table/impl/request_migrator.cpp b/ydb/public/sdk/cpp/client/ydb_table/impl/request_migrator.cpp
index 6c1c6b0381..1484553c85 100644
--- a/ydb/public/sdk/cpp/client/ydb_table/impl/request_migrator.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_table/impl/request_migrator.cpp
@@ -1,4 +1,4 @@
-#include "request_migrator.h"
+#include "request_migrator.h"
#include <vector>
#include <numeric>
diff --git a/ydb/public/sdk/cpp/client/ydb_table/impl/request_migrator.h b/ydb/public/sdk/cpp/client/ydb_table/impl/request_migrator.h
index ace3e0e04d..81b89392be 100644
--- a/ydb/public/sdk/cpp/client/ydb_table/impl/request_migrator.h
+++ b/ydb/public/sdk/cpp/client/ydb_table/impl/request_migrator.h
@@ -1,9 +1,9 @@
#pragma once
-#include "client_session.h"
+#include "client_session.h"
#include <ydb/public/sdk/cpp/client/ydb_table/table.h>
-
+
#include <library/cpp/threading/future/future.h>
#include <util/generic/string.h>
diff --git a/ydb/public/sdk/cpp/client/ydb_table/impl/ut/ya.make b/ydb/public/sdk/cpp/client/ydb_table/impl/ut/ya.make
index e139fec547..dc192877ea 100644
--- a/ydb/public/sdk/cpp/client/ydb_table/impl/ut/ya.make
+++ b/ydb/public/sdk/cpp/client/ydb_table/impl/ut/ya.make
@@ -1,27 +1,27 @@
UNITTEST_FOR(ydb/public/sdk/cpp/client/ydb_table/impl)
-
-OWNER(
- dcherednik
- g:kikimr
-)
-
-IF (SANITIZER_TYPE)
- TIMEOUT(1200)
- SIZE(LARGE)
- TAG(ya:fat)
-ELSE()
- TIMEOUT(600)
- SIZE(MEDIUM)
-ENDIF()
-
-FORK_SUBTESTS()
-
-PEERDIR(
+
+OWNER(
+ dcherednik
+ g:kikimr
+)
+
+IF (SANITIZER_TYPE)
+ TIMEOUT(1200)
+ SIZE(LARGE)
+ TAG(ya:fat)
+ELSE()
+ TIMEOUT(600)
+ SIZE(MEDIUM)
+ENDIF()
+
+FORK_SUBTESTS()
+
+PEERDIR(
ydb/public/sdk/cpp/client/ydb_table
-)
-
-SRCS(
- request_migrator_ut.cpp
-)
-
-END()
+)
+
+SRCS(
+ request_migrator_ut.cpp
+)
+
+END()
diff --git a/ydb/public/sdk/cpp/client/ydb_table/impl/ya.make b/ydb/public/sdk/cpp/client/ydb_table/impl/ya.make
index 0a76173697..25daf2f20d 100644
--- a/ydb/public/sdk/cpp/client/ydb_table/impl/ya.make
+++ b/ydb/public/sdk/cpp/client/ydb_table/impl/ya.make
@@ -1,26 +1,26 @@
-LIBRARY()
-
-OWNER(
- dcherednik
- g:kikimr
-)
-
-SRCS(
- client_session.cpp
- data_query.cpp
- request_migrator.cpp
-)
-
-PEERDIR(
- library/cpp/threading/future
+LIBRARY()
+
+OWNER(
+ dcherednik
+ g:kikimr
+)
+
+SRCS(
+ client_session.cpp
+ data_query.cpp
+ request_migrator.cpp
+)
+
+PEERDIR(
+ library/cpp/threading/future
ydb/public/api/protos
ydb/public/lib/operation_id/protos
ydb/public/sdk/cpp/client/impl/ydb_endpoints
ydb/library/yql/public/issue/protos
-)
-
-END()
-
+)
+
+END()
+
RECURSE_FOR_TESTS(
ut
)
diff --git a/ydb/public/sdk/cpp/client/ydb_table/proto_accessor.cpp b/ydb/public/sdk/cpp/client/ydb_table/proto_accessor.cpp
index 3ebae56f2e..40bfe74f67 100644
--- a/ydb/public/sdk/cpp/client/ydb_table/proto_accessor.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_table/proto_accessor.cpp
@@ -1,28 +1,28 @@
#include <ydb/public/sdk/cpp/client/ydb_proto/accessor.h>
-
-#include "table.h"
-
-namespace NYdb {
-
-const Ydb::TableStats::QueryStats& TProtoAccessor::GetProto(const NTable::TQueryStats& queryStats) {
- return queryStats.GetProto();
-}
-
-const Ydb::Table::DescribeTableResult& TProtoAccessor::GetProto(const NTable::TTableDescription& tableDescription) {
- return tableDescription.GetProto();
-}
-
-NTable::TQueryStats TProtoAccessor::FromProto(const Ydb::TableStats::QueryStats& queryStats) {
- return NTable::TQueryStats(queryStats);
-}
-
-NTable::TTableDescription TProtoAccessor::FromProto(const Ydb::Table::CreateTableRequest& request) {
- return NTable::TTableDescription(request);
-}
-
+
+#include "table.h"
+
+namespace NYdb {
+
+const Ydb::TableStats::QueryStats& TProtoAccessor::GetProto(const NTable::TQueryStats& queryStats) {
+ return queryStats.GetProto();
+}
+
+const Ydb::Table::DescribeTableResult& TProtoAccessor::GetProto(const NTable::TTableDescription& tableDescription) {
+ return tableDescription.GetProto();
+}
+
+NTable::TQueryStats TProtoAccessor::FromProto(const Ydb::TableStats::QueryStats& queryStats) {
+ return NTable::TQueryStats(queryStats);
+}
+
+NTable::TTableDescription TProtoAccessor::FromProto(const Ydb::Table::CreateTableRequest& request) {
+ return NTable::TTableDescription(request);
+}
+
NTable::TIndexDescription TProtoAccessor::FromProto(const Ydb::Table::TableIndex& tableIndex) {
return NTable::TIndexDescription(tableIndex);
-}
+}
NTable::TIndexDescription TProtoAccessor::FromProto(const Ydb::Table::TableIndexDescription& tableIndexDesc) {
return NTable::TIndexDescription(tableIndexDesc);
diff --git a/ydb/public/sdk/cpp/client/ydb_table/query_stats/stats.cpp b/ydb/public/sdk/cpp/client/ydb_table/query_stats/stats.cpp
index 3df4277638..106b848c83 100644
--- a/ydb/public/sdk/cpp/client/ydb_table/query_stats/stats.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_table/query_stats/stats.cpp
@@ -1,11 +1,11 @@
-#include "stats.h"
+#include "stats.h"
#include <ydb/public/api/protos/ydb_table.pb.h>
#include <util/datetime/base.h>
-#include <google/protobuf/text_format.h>
-
+#include <google/protobuf/text_format.h>
+
namespace NYdb {
namespace NTable {
diff --git a/ydb/public/sdk/cpp/client/ydb_table/query_stats/stats.h b/ydb/public/sdk/cpp/client/ydb_table/query_stats/stats.h
index a8b1128069..355c271dc2 100644
--- a/ydb/public/sdk/cpp/client/ydb_table/query_stats/stats.h
+++ b/ydb/public/sdk/cpp/client/ydb_table/query_stats/stats.h
@@ -1,10 +1,10 @@
#pragma once
#include <util/generic/maybe.h>
-#include <util/generic/string.h>
+#include <util/generic/string.h>
+
+#include <memory>
-#include <memory>
-
class TDuration;
namespace Ydb {
diff --git a/ydb/public/sdk/cpp/client/ydb_table/query_stats/ya.make b/ydb/public/sdk/cpp/client/ydb_table/query_stats/ya.make
index 28f9270eb8..b0f884e23a 100644
--- a/ydb/public/sdk/cpp/client/ydb_table/query_stats/ya.make
+++ b/ydb/public/sdk/cpp/client/ydb_table/query_stats/ya.make
@@ -1,17 +1,17 @@
-LIBRARY()
-
-OWNER(
- dcherednik
- g:kikimr
-)
-
-SRCS(
- stats.cpp
-)
-
-PEERDIR(
- contrib/libs/protobuf
+LIBRARY()
+
+OWNER(
+ dcherednik
+ g:kikimr
+)
+
+SRCS(
+ stats.cpp
+)
+
+PEERDIR(
+ contrib/libs/protobuf
ydb/public/api/protos
-)
-
-END()
+)
+
+END()
diff --git a/ydb/public/sdk/cpp/client/ydb_table/table.cpp b/ydb/public/sdk/cpp/client/ydb_table/table.cpp
index 0fb997781f..610fec3884 100644
--- a/ydb/public/sdk/cpp/client/ydb_table/table.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_table/table.cpp
@@ -1,10 +1,10 @@
-#include "table.h"
+#include "table.h"
-#define INCLUDE_YDB_INTERNAL_H
+#define INCLUDE_YDB_INTERNAL_H
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/table_helpers/helpers.h>
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.h>
-#undef INCLUDE_YDB_INTERNAL_H
-
+#undef INCLUDE_YDB_INTERNAL_H
+
#include <ydb/public/api/grpc/ydb_table_v1.grpc.pb.h>
#include <ydb/public/api/protos/ydb_table.pb.h>
#include <ydb/public/sdk/cpp/client/impl/ydb_stats/stats.h>
@@ -14,7 +14,7 @@
#include <ydb/public/sdk/cpp/client/ydb_table/impl/client_session.h>
#include <ydb/public/sdk/cpp/client/ydb_table/impl/data_query.h>
#include <ydb/public/sdk/cpp/client/ydb_table/impl/request_migrator.h>
-
+
#include <library/cpp/cache/cache.h>
#include <util/generic/map.h>
@@ -3549,7 +3549,7 @@ TAsyncBulkUpsertResult TTableClient::BulkUpsert(const TString& table, EDataForma
TAsyncScanQueryPartIterator TTableClient::StreamExecuteScanQuery(const TString& query, const TParams& params,
const TStreamExecScanQuerySettings& settings)
{
- return Impl_->StreamExecuteScanQuery(query, &params.GetProtoMap(), settings);
+ return Impl_->StreamExecuteScanQuery(query, &params.GetProtoMap(), settings);
}
TAsyncScanQueryPartIterator TTableClient::StreamExecuteScanQuery(const TString& query,
@@ -3871,7 +3871,7 @@ TAsyncDataQueryResult TSession::ExecuteDataQuery(const TString& query, const TTx
TAsyncDataQueryResult TSession::ExecuteDataQuery(const TString& query, const TTxControl& txControl,
TParams&& params, const TExecDataQuerySettings& settings)
{
- auto paramsPtr = params.Empty() ? nullptr : params.GetProtoMapPtr();
+ auto paramsPtr = params.Empty() ? nullptr : params.GetProtoMapPtr();
return Client_->ExecuteDataQuery(*this, query, txControl, paramsPtr, settings);
}
@@ -3891,7 +3891,7 @@ TAsyncDataQueryResult TSession::ExecuteDataQuery(const TString& query, const TTx
*this,
query,
txControl,
- params.GetProtoMap(),
+ params.GetProtoMap(),
settings);
}
}
@@ -4048,7 +4048,7 @@ TAsyncDataQueryResult TDataQuery::Execute(const TTxControl& txControl,
TAsyncDataQueryResult TDataQuery::Execute(const TTxControl& txControl, TParams&& params,
const TExecDataQuerySettings& settings)
{
- auto paramsPtr = params.Empty() ? nullptr : params.GetProtoMapPtr();
+ auto paramsPtr = params.Empty() ? nullptr : params.GetProtoMapPtr();
return Impl_->Session_.Client_->ExecuteDataQuery(
Impl_->Session_,
*this,
@@ -4075,7 +4075,7 @@ TAsyncDataQueryResult TDataQuery::Execute(const TTxControl& txControl, const TPa
Impl_->Session_,
*this,
txControl,
- params.GetProtoMap(),
+ params.GetProtoMap(),
settings,
false);
}
diff --git a/ydb/public/sdk/cpp/client/ydb_table/table.h b/ydb/public/sdk/cpp/client/ydb_table/table.h
index 0a533b42b0..42d21d2fec 100644
--- a/ydb/public/sdk/cpp/client/ydb_table/table.h
+++ b/ydb/public/sdk/cpp/client/ydb_table/table.h
@@ -1,13 +1,13 @@
#pragma once
-#include "table_enum.h"
+#include "table_enum.h"
#include <ydb/public/sdk/cpp/client/ydb_driver/driver.h>
#include <ydb/public/sdk/cpp/client/ydb_result/result.h>
#include <ydb/public/sdk/cpp/client/ydb_table/query_stats/stats.h>
#include <ydb/public/sdk/cpp/client/ydb_params/params.h>
#include <ydb/public/sdk/cpp/client/ydb_types/operation/operation.h>
-
+
#include <util/generic/hash.h>
#include <util/generic/maybe.h>
#include <util/generic/variant.h>
@@ -30,11 +30,11 @@ class ValueSinceUnixEpochModeSettings;
}
namespace NYdb {
-
-namespace NScheme {
-struct TPermissions;
-}
-
+
+namespace NScheme {
+struct TPermissions;
+}
+
namespace NTable {
////////////////////////////////////////////////////////////////////////////////
diff --git a/ydb/public/sdk/cpp/client/ydb_table/ya.make b/ydb/public/sdk/cpp/client/ydb_table/ya.make
index b427661365..901908d243 100644
--- a/ydb/public/sdk/cpp/client/ydb_table/ya.make
+++ b/ydb/public/sdk/cpp/client/ydb_table/ya.make
@@ -1,18 +1,18 @@
-LIBRARY()
-
-OWNER(
- dcherednik
- g:kikimr
-)
-
-SRCS(
- table.cpp
- proto_accessor.cpp
-)
-
-GENERATE_ENUM_SERIALIZATION(table_enum.h)
-
-PEERDIR(
+LIBRARY()
+
+OWNER(
+ dcherednik
+ g:kikimr
+)
+
+SRCS(
+ table.cpp
+ proto_accessor.cpp
+)
+
+GENERATE_ENUM_SERIALIZATION(table_enum.h)
+
+PEERDIR(
ydb/public/api/protos
ydb/public/sdk/cpp/client/impl/ydb_internal/make_request
ydb/public/sdk/cpp/client/ydb_driver
@@ -23,6 +23,6 @@ PEERDIR(
ydb/public/sdk/cpp/client/ydb_table/query_stats
ydb/public/sdk/cpp/client/ydb_types/operation
ydb/public/sdk/cpp/client/ydb_value
-)
-
-END()
+)
+
+END()
diff --git a/ydb/public/sdk/cpp/client/ydb_types/credentials/credentials.cpp b/ydb/public/sdk/cpp/client/ydb_types/credentials/credentials.cpp
index ec00093baf..679634450f 100644
--- a/ydb/public/sdk/cpp/client/ydb_types/credentials/credentials.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_types/credentials/credentials.cpp
@@ -1,77 +1,77 @@
-#include "credentials.h"
-
-namespace NYdb {
-
-class TInsecureCredentialsProvider : public ICredentialsProvider {
-public:
- TInsecureCredentialsProvider()
- {}
-
+#include "credentials.h"
+
+namespace NYdb {
+
+class TInsecureCredentialsProvider : public ICredentialsProvider {
+public:
+ TInsecureCredentialsProvider()
+ {}
+
TStringType GetAuthInfo() const override {
return TStringType();
- }
-
- bool IsValid() const override {
- return false;
- }
-};
-
-class TInsecureCredentialsProviderFactory : public ICredentialsProviderFactory {
-public:
- TInsecureCredentialsProviderFactory()
- {}
-
- std::shared_ptr<ICredentialsProvider> CreateProvider() const override {
- return std::make_shared<TInsecureCredentialsProvider>();
- }
-
+ }
+
+ bool IsValid() const override {
+ return false;
+ }
+};
+
+class TInsecureCredentialsProviderFactory : public ICredentialsProviderFactory {
+public:
+ TInsecureCredentialsProviderFactory()
+ {}
+
+ std::shared_ptr<ICredentialsProvider> CreateProvider() const override {
+ return std::make_shared<TInsecureCredentialsProvider>();
+ }
+
TStringType GetClientIdentity() const override {
return TStringType();
- }
-};
-
-class TOAuthCredentialsProvider : public ICredentialsProvider {
-public:
+ }
+};
+
+class TOAuthCredentialsProvider : public ICredentialsProvider {
+public:
TOAuthCredentialsProvider(const TStringType& token)
- : Token(token)
- {}
-
+ : Token(token)
+ {}
+
TStringType GetAuthInfo() const override {
- return Token;
- }
-
- bool IsValid() const override {
- return !Token.empty();
- }
-
-private:
+ return Token;
+ }
+
+ bool IsValid() const override {
+ return !Token.empty();
+ }
+
+private:
TStringType Token;
-};
-
-class TOAuthCredentialsProviderFactory : public ICredentialsProviderFactory {
-public:
+};
+
+class TOAuthCredentialsProviderFactory : public ICredentialsProviderFactory {
+public:
TOAuthCredentialsProviderFactory(const TStringType& token)
- : Token(token)
- {}
-
- std::shared_ptr<ICredentialsProvider> CreateProvider() const override {
- return std::make_shared<TOAuthCredentialsProvider>(Token);
- }
-
+ : Token(token)
+ {}
+
+ std::shared_ptr<ICredentialsProvider> CreateProvider() const override {
+ return std::make_shared<TOAuthCredentialsProvider>(Token);
+ }
+
TStringType GetClientIdentity() const override {
- return Token;
- }
-
-private:
+ return Token;
+ }
+
+private:
TStringType Token;
-};
-
-std::shared_ptr<ICredentialsProviderFactory> CreateInsecureCredentialsProviderFactory() {
- return std::make_shared<TInsecureCredentialsProviderFactory>();
-}
+};
+
+std::shared_ptr<ICredentialsProviderFactory> CreateInsecureCredentialsProviderFactory() {
+ return std::make_shared<TInsecureCredentialsProviderFactory>();
+}
std::shared_ptr<ICredentialsProviderFactory> CreateOAuthCredentialsProviderFactory(const TStringType& token) {
- return std::make_shared<TOAuthCredentialsProviderFactory>(token);
-}
-
-} // namespace NYdb
-
+ return std::make_shared<TOAuthCredentialsProviderFactory>(token);
+}
+
+} // namespace NYdb
+
diff --git a/ydb/public/sdk/cpp/client/ydb_types/credentials/credentials.h b/ydb/public/sdk/cpp/client/ydb_types/credentials/credentials.h
index 17515b7ff1..75b86933ce 100644
--- a/ydb/public/sdk/cpp/client/ydb_types/credentials/credentials.h
+++ b/ydb/public/sdk/cpp/client/ydb_types/credentials/credentials.h
@@ -1,24 +1,24 @@
-#pragma once
-
+#pragma once
+
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/common/type_switcher.h>
-
+
#include <memory>
-namespace NYdb {
-
-class ICredentialsProvider {
-public:
- virtual ~ICredentialsProvider() = default;
+namespace NYdb {
+
+class ICredentialsProvider {
+public:
+ virtual ~ICredentialsProvider() = default;
virtual TStringType GetAuthInfo() const = 0;
- virtual bool IsValid() const = 0;
-};
-
+ virtual bool IsValid() const = 0;
+};
+
using TCredentialsProviderPtr = std::shared_ptr<ICredentialsProvider>;
class ICoreFacility;
-class ICredentialsProviderFactory {
-public:
- virtual ~ICredentialsProviderFactory() = default;
+class ICredentialsProviderFactory {
+public:
+ virtual ~ICredentialsProviderFactory() = default;
virtual TCredentialsProviderPtr CreateProvider() const = 0;
// !!!Experimental!!!
virtual TCredentialsProviderPtr CreateProvider(std::weak_ptr<ICoreFacility> facility) const {
@@ -26,13 +26,13 @@ public:
return CreateProvider();
}
virtual TStringType GetClientIdentity() const = 0;
-};
-
+};
+
using TCredentialsProviderFactoryPtr = std::shared_ptr<ICredentialsProviderFactory>;
-std::shared_ptr<ICredentialsProviderFactory> CreateInsecureCredentialsProviderFactory();
+std::shared_ptr<ICredentialsProviderFactory> CreateInsecureCredentialsProviderFactory();
std::shared_ptr<ICredentialsProviderFactory> CreateOAuthCredentialsProviderFactory(const TStringType& token);
-
+
struct TLoginCredentialsParams {
TString User;
TString Password;
@@ -40,4 +40,4 @@ struct TLoginCredentialsParams {
std::shared_ptr<ICredentialsProviderFactory> CreateLoginCredentialsProviderFactory(TLoginCredentialsParams params);
-} // namespace NYdb
+} // namespace NYdb
diff --git a/ydb/public/sdk/cpp/client/ydb_types/credentials/ya.make b/ydb/public/sdk/cpp/client/ydb_types/credentials/ya.make
index 4a296ea6d9..c58b2112f8 100644
--- a/ydb/public/sdk/cpp/client/ydb_types/credentials/ya.make
+++ b/ydb/public/sdk/cpp/client/ydb_types/credentials/ya.make
@@ -1,12 +1,12 @@
-LIBRARY()
-
+LIBRARY()
+
OWNER(g:kikimr)
-
-SRCS(
- credentials.cpp
+
+SRCS(
+ credentials.cpp
login.cpp
-)
-
+)
+
PEERDIR(
ydb/library/login
ydb/public/api/grpc
@@ -14,4 +14,4 @@ PEERDIR(
ydb/library/yql/public/issue
)
-END()
+END()
diff --git a/ydb/public/sdk/cpp/client/ydb_types/exceptions/exceptions.cpp b/ydb/public/sdk/cpp/client/ydb_types/exceptions/exceptions.cpp
index e0dfd27684..f103a88ecc 100644
--- a/ydb/public/sdk/cpp/client/ydb_types/exceptions/exceptions.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_types/exceptions/exceptions.cpp
@@ -1,4 +1,4 @@
-#include "exceptions.h"
+#include "exceptions.h"
namespace NYdb {
diff --git a/ydb/public/sdk/cpp/client/ydb_types/exceptions/ya.make b/ydb/public/sdk/cpp/client/ydb_types/exceptions/ya.make
index ae043eedd2..8e9a1705e7 100644
--- a/ydb/public/sdk/cpp/client/ydb_types/exceptions/ya.make
+++ b/ydb/public/sdk/cpp/client/ydb_types/exceptions/ya.make
@@ -1,9 +1,9 @@
-LIBRARY()
-
+LIBRARY()
+
OWNER(g:kikimr)
-
-SRCS(
- exceptions.cpp
-)
-
-END()
+
+SRCS(
+ exceptions.cpp
+)
+
+END()
diff --git a/ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers/handlers.cpp b/ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers/handlers.cpp
index 766f910058..69b94f9b40 100644
--- a/ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers/handlers.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers/handlers.cpp
@@ -1,10 +1,10 @@
-#include "handlers.h"
+#include "handlers.h"
#include <ydb/public/sdk/cpp/client/ydb_types/exceptions/exceptions.h>
-
-namespace NYdb {
-
+
+namespace NYdb {
+
void ThrowFatalError(const TString& str) {
throw TContractViolation(str);
-}
-
-} // namespace NYdb
+}
+
+} // namespace NYdb
diff --git a/ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers/handlers.h b/ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers/handlers.h
index 2ea990d417..6af54a52f2 100644
--- a/ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers/handlers.h
+++ b/ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers/handlers.h
@@ -1,11 +1,11 @@
-#pragma once
-
-#include <util/generic/string.h>
-
-#include <functional>
-
-namespace NYdb {
-
+#pragma once
+
+#include <util/generic/string.h>
+
+#include <functional>
+
+namespace NYdb {
+
void ThrowFatalError(const TString& str);
-
-} // namespace NYdb
+
+} // namespace NYdb
diff --git a/ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers/ya.make b/ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers/ya.make
index 7e9d79e7e7..423aa103de 100644
--- a/ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers/ya.make
+++ b/ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers/ya.make
@@ -1,16 +1,16 @@
-LIBRARY()
-
-OWNER(
- dcherednik
- g:kikimr
-)
-
-SRCS(
- handlers.cpp
-)
-
-PEERDIR(
+LIBRARY()
+
+OWNER(
+ dcherednik
+ g:kikimr
+)
+
+SRCS(
+ handlers.cpp
+)
+
+PEERDIR(
ydb/public/sdk/cpp/client/ydb_types/exceptions
-)
-
-END()
+)
+
+END()
diff --git a/ydb/public/sdk/cpp/client/ydb_types/fluent_settings_helpers.h b/ydb/public/sdk/cpp/client/ydb_types/fluent_settings_helpers.h
index a882840182..8747e049f5 100644
--- a/ydb/public/sdk/cpp/client/ydb_types/fluent_settings_helpers.h
+++ b/ydb/public/sdk/cpp/client/ydb_types/fluent_settings_helpers.h
@@ -1,44 +1,44 @@
-#pragma once
-
-#include <util/generic/maybe.h>
-
-#define FLUENT_SETTING(type, name) \
- type name##_; \
- TSelf& name(const type& value) { \
- name##_ = value; \
- return static_cast<TSelf&>(*this); \
- }
-
-#define FLUENT_SETTING_OPTIONAL(type, name) \
- TMaybe<type> name##_; \
- TSelf& name(const TMaybe<type>& value) { \
- name##_ = value; \
- return static_cast<TSelf&>(*this); \
- }
-
-#define FLUENT_SETTING_DEFAULT(type, name, defaultValue) \
- type name##_ = defaultValue; \
- TSelf& name(const type& value) { \
- name##_ = value; \
- return static_cast<TSelf&>(*this); \
- }
-
-#define FLUENT_SETTING_FLAG(name) \
- bool name##_ = false; \
- TSelf& name(bool value = true) { \
- name##_ = value; \
- return static_cast<TSelf&>(*this); \
- }
-
-#define FLUENT_SETTING_FLAG_ALIAS(name, other, value) \
- TSelf& name() { \
- other##_ = value; \
- return static_cast<TSelf&>(*this); \
- }
-
-#define FLUENT_SETTING_VECTOR(type, name) \
- TVector<type> name##_; \
- TSelf& Append##name(const type& value) { \
- name##_.push_back(value); \
- return static_cast<TSelf&>(*this); \
- }
+#pragma once
+
+#include <util/generic/maybe.h>
+
+#define FLUENT_SETTING(type, name) \
+ type name##_; \
+ TSelf& name(const type& value) { \
+ name##_ = value; \
+ return static_cast<TSelf&>(*this); \
+ }
+
+#define FLUENT_SETTING_OPTIONAL(type, name) \
+ TMaybe<type> name##_; \
+ TSelf& name(const TMaybe<type>& value) { \
+ name##_ = value; \
+ return static_cast<TSelf&>(*this); \
+ }
+
+#define FLUENT_SETTING_DEFAULT(type, name, defaultValue) \
+ type name##_ = defaultValue; \
+ TSelf& name(const type& value) { \
+ name##_ = value; \
+ return static_cast<TSelf&>(*this); \
+ }
+
+#define FLUENT_SETTING_FLAG(name) \
+ bool name##_ = false; \
+ TSelf& name(bool value = true) { \
+ name##_ = value; \
+ return static_cast<TSelf&>(*this); \
+ }
+
+#define FLUENT_SETTING_FLAG_ALIAS(name, other, value) \
+ TSelf& name() { \
+ other##_ = value; \
+ return static_cast<TSelf&>(*this); \
+ }
+
+#define FLUENT_SETTING_VECTOR(type, name) \
+ TVector<type> name##_; \
+ TSelf& Append##name(const type& value) { \
+ name##_.push_back(value); \
+ return static_cast<TSelf&>(*this); \
+ }
diff --git a/ydb/public/sdk/cpp/client/ydb_types/operation/operation.cpp b/ydb/public/sdk/cpp/client/ydb_types/operation/operation.cpp
index 4d37a150ea..aef87cc9cc 100644
--- a/ydb/public/sdk/cpp/client/ydb_types/operation/operation.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_types/operation/operation.cpp
@@ -1,91 +1,91 @@
-#include "operation.h"
-
+#include "operation.h"
+
#include <ydb/public/api/protos/ydb_operation.pb.h>
#include <ydb/public/sdk/cpp/client/ydb_types/status/status.h>
-
-namespace NYdb {
-
-
-class TOperation::TImpl {
-public:
- TImpl(TStatus&& status)
- : Status_(std::move(status))
- , Ready_(true)
- { }
-
- TImpl(TStatus&& status, Ydb::Operations::Operation&& operation)
- : Id_(operation.id(), true /* allowEmpty */)
- , Status_(std::move(status))
- , Ready_(operation.ready())
- , Operation_(std::move(operation))
- {
- }
-
- const TOperationId& Id() const {
- return Id_;
- }
-
- bool Ready() const {
- return Ready_;
- }
-
- const TStatus& Status() const {
- return Status_;
- }
-
- const Ydb::Operations::Operation& GetProto() const {
- return Operation_;
- }
-
-private:
- const TOperationId Id_;
- const TStatus Status_;
- const bool Ready_;
- const Ydb::Operations::Operation Operation_;
-};
-
-TOperation::TOperation(TStatus&& status)
- : Impl_(std::make_shared<TImpl>(std::move(status)))
-{ }
-
-TOperation::TOperation(TStatus&& status, Ydb::Operations::Operation&& operation)
- : Impl_(std::make_shared<TImpl>(std::move(status), std::move(operation)))
-{ }
-
-const TOperation::TOperationId& TOperation::Id() const {
- return Impl_->Id();
-}
-
-bool TOperation::Ready() const {
- return Impl_->Ready();
-}
-
-const TStatus& TOperation::Status() const {
- return Impl_->Status();
-}
-
-TString TOperation::ToString() const {
- TString result;
- TStringOutput out(result);
- Out(out);
- return result;
-}
-
-void TOperation::Out(IOutputStream& o) const {
- o << GetProto().DebugString();
-}
-
-TString TOperation::ToJsonString() const {
- using namespace google::protobuf::util;
-
- TString json;
+
+namespace NYdb {
+
+
+class TOperation::TImpl {
+public:
+ TImpl(TStatus&& status)
+ : Status_(std::move(status))
+ , Ready_(true)
+ { }
+
+ TImpl(TStatus&& status, Ydb::Operations::Operation&& operation)
+ : Id_(operation.id(), true /* allowEmpty */)
+ , Status_(std::move(status))
+ , Ready_(operation.ready())
+ , Operation_(std::move(operation))
+ {
+ }
+
+ const TOperationId& Id() const {
+ return Id_;
+ }
+
+ bool Ready() const {
+ return Ready_;
+ }
+
+ const TStatus& Status() const {
+ return Status_;
+ }
+
+ const Ydb::Operations::Operation& GetProto() const {
+ return Operation_;
+ }
+
+private:
+ const TOperationId Id_;
+ const TStatus Status_;
+ const bool Ready_;
+ const Ydb::Operations::Operation Operation_;
+};
+
+TOperation::TOperation(TStatus&& status)
+ : Impl_(std::make_shared<TImpl>(std::move(status)))
+{ }
+
+TOperation::TOperation(TStatus&& status, Ydb::Operations::Operation&& operation)
+ : Impl_(std::make_shared<TImpl>(std::move(status), std::move(operation)))
+{ }
+
+const TOperation::TOperationId& TOperation::Id() const {
+ return Impl_->Id();
+}
+
+bool TOperation::Ready() const {
+ return Impl_->Ready();
+}
+
+const TStatus& TOperation::Status() const {
+ return Impl_->Status();
+}
+
+TString TOperation::ToString() const {
+ TString result;
+ TStringOutput out(result);
+ Out(out);
+ return result;
+}
+
+void TOperation::Out(IOutputStream& o) const {
+ o << GetProto().DebugString();
+}
+
+TString TOperation::ToJsonString() const {
+ using namespace google::protobuf::util;
+
+ TString json;
auto status = MessageToJsonString(GetProto(), &json, JsonPrintOptions());
Y_VERIFY(status.ok());
- return json;
-}
-
-const Ydb::Operations::Operation& TOperation::GetProto() const {
- return Impl_->GetProto();
-}
-
-} // namespace NYdb
+ return json;
+}
+
+const Ydb::Operations::Operation& TOperation::GetProto() const {
+ return Impl_->GetProto();
+}
+
+} // namespace NYdb
diff --git a/ydb/public/sdk/cpp/client/ydb_types/operation/operation.h b/ydb/public/sdk/cpp/client/ydb_types/operation/operation.h
index 83263c34a2..bc5dce05eb 100644
--- a/ydb/public/sdk/cpp/client/ydb_types/operation/operation.h
+++ b/ydb/public/sdk/cpp/client/ydb_types/operation/operation.h
@@ -1,12 +1,12 @@
-#pragma once
-
+#pragma once
+
#include <ydb/public/lib/operation_id/operation_id.h>
-
-#include <library/cpp/threading/future/future.h>
-
-#include <google/protobuf/stubs/status.h>
-#include <google/protobuf/util/json_util.h>
-
+
+#include <library/cpp/threading/future/future.h>
+
+#include <google/protobuf/stubs/status.h>
+#include <google/protobuf/util/json_util.h>
+
namespace Ydb {
namespace Operations {
@@ -15,39 +15,39 @@ class Operation;
} // namespace Operations
} // namespace Ydb
-namespace NYdb {
-
-class TStatus;
-
-class TOperation {
-public:
- using TOperationId = NKikimr::NOperationId::TOperationId;
-
-public:
- TOperation(TStatus&& status);
- TOperation(TStatus&& status, Ydb::Operations::Operation&& operation);
- virtual ~TOperation() = default;
-
- const TOperationId& Id() const;
- bool Ready() const;
- const TStatus& Status() const;
-
- TString ToString() const;
- TString ToJsonString() const;
- void Out(IOutputStream& o) const;
-
-protected:
- const Ydb::Operations::Operation& GetProto() const;
-
-private:
- class TImpl;
- std::shared_ptr<TImpl> Impl_;
-};
-
-using TAsyncOperation = NThreading::TFuture<TOperation>;
-
-} // namespace NYdb
-
+namespace NYdb {
+
+class TStatus;
+
+class TOperation {
+public:
+ using TOperationId = NKikimr::NOperationId::TOperationId;
+
+public:
+ TOperation(TStatus&& status);
+ TOperation(TStatus&& status, Ydb::Operations::Operation&& operation);
+ virtual ~TOperation() = default;
+
+ const TOperationId& Id() const;
+ bool Ready() const;
+ const TStatus& Status() const;
+
+ TString ToString() const;
+ TString ToJsonString() const;
+ void Out(IOutputStream& o) const;
+
+protected:
+ const Ydb::Operations::Operation& GetProto() const;
+
+private:
+ class TImpl;
+ std::shared_ptr<TImpl> Impl_;
+};
+
+using TAsyncOperation = NThreading::TFuture<TOperation>;
+
+} // namespace NYdb
+
Y_DECLARE_OUT_SPEC(inline, NYdb::TOperation, o, x) {
- return x.Out(o);
-}
+ return x.Out(o);
+}
diff --git a/ydb/public/sdk/cpp/client/ydb_types/operation/ya.make b/ydb/public/sdk/cpp/client/ydb_types/operation/ya.make
index cae8f767c3..322463eca5 100644
--- a/ydb/public/sdk/cpp/client/ydb_types/operation/ya.make
+++ b/ydb/public/sdk/cpp/client/ydb_types/operation/ya.make
@@ -1,16 +1,16 @@
-LIBRARY()
-
+LIBRARY()
+
OWNER(g:kikimr)
-
-SRCS(
- operation.cpp
-)
-
-PEERDIR(
- contrib/libs/protobuf
- library/cpp/threading/future
+
+SRCS(
+ operation.cpp
+)
+
+PEERDIR(
+ contrib/libs/protobuf
+ library/cpp/threading/future
ydb/public/lib/operation_id
ydb/public/sdk/cpp/client/ydb_types
-)
-
-END()
+)
+
+END()
diff --git a/ydb/public/sdk/cpp/client/ydb_types/request_settings.h b/ydb/public/sdk/cpp/client/ydb_types/request_settings.h
index 3b7bf71285..fb8aa8ac68 100644
--- a/ydb/public/sdk/cpp/client/ydb_types/request_settings.h
+++ b/ydb/public/sdk/cpp/client/ydb_types/request_settings.h
@@ -1,21 +1,21 @@
-#pragma once
-
-#include "fluent_settings_helpers.h"
-
-#include <util/datetime/base.h>
-
+#pragma once
+
+#include "fluent_settings_helpers.h"
+
+#include <util/datetime/base.h>
+
#include <vector>
#include <utility>
-namespace NYdb {
-
-template<typename TDerived>
-struct TRequestSettings {
- using TSelf = TDerived;
+namespace NYdb {
+
+template<typename TDerived>
+struct TRequestSettings {
+ using TSelf = TDerived;
using THeader = std::vector<std::pair<TString, TString>>;
-
- FLUENT_SETTING(TString, TraceId);
- FLUENT_SETTING(TString, RequestType);
+
+ FLUENT_SETTING(TString, TraceId);
+ FLUENT_SETTING(TString, RequestType);
FLUENT_SETTING(THeader, Header);
TRequestSettings() = default;
@@ -26,13 +26,13 @@ struct TRequestSettings {
, RequestType_(other.RequestType_)
, Header_(other.Header_)
{}
-};
-
-template<typename TDerived>
-struct TSimpleRequestSettings : public TRequestSettings<TDerived> {
- using TSelf = TDerived;
-
- FLUENT_SETTING(TDuration, ClientTimeout);
+};
+
+template<typename TDerived>
+struct TSimpleRequestSettings : public TRequestSettings<TDerived> {
+ using TSelf = TDerived;
+
+ FLUENT_SETTING(TDuration, ClientTimeout);
TSimpleRequestSettings() = default;
@@ -41,18 +41,18 @@ struct TSimpleRequestSettings : public TRequestSettings<TDerived> {
: TRequestSettings<TDerived>(other)
, ClientTimeout_(other.ClientTimeout_)
{}
-};
-
-template<typename TDerived>
-struct TOperationRequestSettings : public TSimpleRequestSettings<TDerived> {
- using TSelf = TDerived;
-
- /* Cancel/timeout operation settings available from 18-8 YDB server version */
- FLUENT_SETTING(TDuration, OperationTimeout);
- FLUENT_SETTING(TDuration, CancelAfter);
- FLUENT_SETTING_DEFAULT(bool, UseClientTimeoutForOperation, true);
+};
+
+template<typename TDerived>
+struct TOperationRequestSettings : public TSimpleRequestSettings<TDerived> {
+ using TSelf = TDerived;
+
+ /* Cancel/timeout operation settings available from 18-8 YDB server version */
+ FLUENT_SETTING(TDuration, OperationTimeout);
+ FLUENT_SETTING(TDuration, CancelAfter);
+ FLUENT_SETTING_DEFAULT(bool, UseClientTimeoutForOperation, true);
FLUENT_SETTING_DEFAULT(bool, ReportCostInfo, false);
-
+
TOperationRequestSettings() = default;
template <typename T>
@@ -64,11 +64,11 @@ struct TOperationRequestSettings : public TSimpleRequestSettings<TDerived> {
, ReportCostInfo_(other.ReportCostInfo_)
{}
- TSelf& CancelAfterWithTimeout(const TDuration& cancelAfter, const TDuration& operationTimeout) {
- CancelAfter_ = cancelAfter;
- OperationTimeout_ = operationTimeout;
- return static_cast<TSelf&>(*this);
- }
-};
-
-} // namespace NYdb
+ TSelf& CancelAfterWithTimeout(const TDuration& cancelAfter, const TDuration& operationTimeout) {
+ CancelAfter_ = cancelAfter;
+ OperationTimeout_ = operationTimeout;
+ return static_cast<TSelf&>(*this);
+ }
+};
+
+} // namespace NYdb
diff --git a/ydb/public/sdk/cpp/client/ydb_types/status/status.cpp b/ydb/public/sdk/cpp/client/ydb_types/status/status.cpp
index eed05872bf..d3f2ca05af 100644
--- a/ydb/public/sdk/cpp/client/ydb_types/status/status.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_types/status/status.cpp
@@ -1,73 +1,73 @@
-#include "status.h"
-
-#define INCLUDE_YDB_INTERNAL_H
+#include "status.h"
+
+#define INCLUDE_YDB_INTERNAL_H
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/plain_status/status.h>
-#undef INCLUDE_YDB_INTERNAL_H
-
+#undef INCLUDE_YDB_INTERNAL_H
+
#include <ydb/public/sdk/cpp/client/ydb_types/exceptions/exceptions.h>
-namespace NYdb {
-
-class TStatus::TImpl {
-public:
- const TPlainStatus Status;
-
+namespace NYdb {
+
+class TStatus::TImpl {
+public:
+ const TPlainStatus Status;
+
TImpl(TPlainStatus&& status)
- : Status(std::move(status))
- { }
-
+ : Status(std::move(status))
+ { }
+
void CheckStatusOk(const TStringType& str) const {
- if (!Status.Ok()) {
+ if (!Status.Ok()) {
ThrowFatalError(TStringType("Attempt to use result with not successfull status. ") + str + "\n");
- }
- }
-
+ }
+ }
+
void RaiseError(const TStringType& str) const {
ythrow TContractViolation(str);
- }
-};
-
+ }
+};
+
TStatus::TStatus(EStatus statusCode, NYql::TIssues&& issues)
: Impl_(std::make_shared<TImpl>(TPlainStatus{statusCode, std::move(issues)}))
-{ }
-
+{ }
+
TStatus::TStatus(TPlainStatus&& plain)
: Impl_(std::make_shared<TImpl>(std::move(plain)))
-{ }
-
-const NYql::TIssues& TStatus::GetIssues() const {
- return Impl_->Status.Issues;
-}
-
-EStatus TStatus::GetStatus() const {
- return Impl_->Status.Status;
-}
-
-bool TStatus::IsSuccess() const {
- return Impl_->Status.Status == EStatus::SUCCESS;
-}
-
-bool TStatus::IsTransportError() const {
- return static_cast<size_t>(Impl_->Status.Status) >= TRANSPORT_STATUSES_FIRST
- && static_cast<size_t>(Impl_->Status.Status) <= TRANSPORT_STATUSES_LAST;
-}
-
+{ }
+
+const NYql::TIssues& TStatus::GetIssues() const {
+ return Impl_->Status.Issues;
+}
+
+EStatus TStatus::GetStatus() const {
+ return Impl_->Status.Status;
+}
+
+bool TStatus::IsSuccess() const {
+ return Impl_->Status.Status == EStatus::SUCCESS;
+}
+
+bool TStatus::IsTransportError() const {
+ return static_cast<size_t>(Impl_->Status.Status) >= TRANSPORT_STATUSES_FIRST
+ && static_cast<size_t>(Impl_->Status.Status) <= TRANSPORT_STATUSES_LAST;
+}
+
void TStatus::CheckStatusOk(const TStringType& str) const {
- Impl_->CheckStatusOk(str);
-}
-
+ Impl_->CheckStatusOk(str);
+}
+
void TStatus::RaiseError(const TStringType& str) const {
- Impl_->RaiseError(str);
-}
-
+ Impl_->RaiseError(str);
+}
+
const TStringType& TStatus::GetEndpoint() const {
- return Impl_->Status.Endpoint;
-}
-
+ return Impl_->Status.Endpoint;
+}
+
const std::multimap<TStringType, TStringType>& TStatus::GetResponseMetadata() const {
- return Impl_->Status.Metadata;
-}
-
+ return Impl_->Status.Metadata;
+}
+
float TStatus::GetConsumedRu() const {
return Impl_->Status.ConstInfo.consumed_units();
}
@@ -82,4 +82,4 @@ bool TStreamPartStatus::EOS() const {
return GetStatus() == EStatus::CLIENT_OUT_OF_RANGE;
}
-} // namespace NYdb
+} // namespace NYdb
diff --git a/ydb/public/sdk/cpp/client/ydb_types/status/status.h b/ydb/public/sdk/cpp/client/ydb_types/status/status.h
index 2f5354a106..7185a03964 100644
--- a/ydb/public/sdk/cpp/client/ydb_types/status/status.h
+++ b/ydb/public/sdk/cpp/client/ydb_types/status/status.h
@@ -1,46 +1,46 @@
-#pragma once
-
+#pragma once
+
#include <ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers/handlers.h>
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/common/type_switcher.h>
#include <ydb/public/sdk/cpp/client/ydb_types/ydb.h>
#include <ydb/library/yql/public/issue/yql_issue.h>
-
-#include <library/cpp/threading/future/future.h>
-
-namespace NYdb {
-
-//! Internal status representation
-struct TPlainStatus;
-
-//! Represents status of call
-class TStatus {
-public:
+
+#include <library/cpp/threading/future/future.h>
+
+namespace NYdb {
+
+//! Internal status representation
+struct TPlainStatus;
+
+//! Represents status of call
+class TStatus {
+public:
TStatus(EStatus statusCode, NYql::TIssues&& issues);
TStatus(TPlainStatus&& plain);
-
- EStatus GetStatus() const;
- const NYql::TIssues& GetIssues() const;
- bool IsSuccess() const;
- bool IsTransportError() const;
+
+ EStatus GetStatus() const;
+ const NYql::TIssues& GetIssues() const;
+ bool IsSuccess() const;
+ bool IsTransportError() const;
const TStringType& GetEndpoint() const;
const std::multimap<TStringType, TStringType>& GetResponseMetadata() const;
float GetConsumedRu() const;
-
-protected:
+
+protected:
void CheckStatusOk(const TStringType& str) const;
void RaiseError(const TStringType& str) const;
-private:
- class TImpl;
- std::shared_ptr<TImpl> Impl_;
-};
-
-using TAsyncStatus = NThreading::TFuture<TStatus>;
-
+private:
+ class TImpl;
+ std::shared_ptr<TImpl> Impl_;
+};
+
+using TAsyncStatus = NThreading::TFuture<TStatus>;
+
class TStreamPartStatus : public TStatus {
public:
TStreamPartStatus(TStatus&& status);
bool EOS() const;
};
-} // namespace NYdb
+} // namespace NYdb
diff --git a/ydb/public/sdk/cpp/client/ydb_types/status/ya.make b/ydb/public/sdk/cpp/client/ydb_types/status/ya.make
index 5a62725483..0a5bdd0da4 100644
--- a/ydb/public/sdk/cpp/client/ydb_types/status/ya.make
+++ b/ydb/public/sdk/cpp/client/ydb_types/status/ya.make
@@ -1,17 +1,17 @@
-LIBRARY()
-
+LIBRARY()
+
OWNER(g:kikimr)
-
-SRCS(
- status.cpp
-)
-
-PEERDIR(
+
+SRCS(
+ status.cpp
+)
+
+PEERDIR(
library/cpp/threading/future
ydb/public/sdk/cpp/client/impl/ydb_internal/plain_status
ydb/public/sdk/cpp/client/ydb_types
ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers
ydb/library/yql/public/issue
-)
-
-END()
+)
+
+END()
diff --git a/ydb/public/sdk/cpp/client/ydb_types/ya.make b/ydb/public/sdk/cpp/client/ydb_types/ya.make
index 3e3bf15f23..d1419ecd34 100644
--- a/ydb/public/sdk/cpp/client/ydb_types/ya.make
+++ b/ydb/public/sdk/cpp/client/ydb_types/ya.make
@@ -1,14 +1,14 @@
-LIBRARY()
-
+LIBRARY()
+
OWNER(g:kikimr)
-
-PEERDIR(
- contrib/libs/protobuf
- library/cpp/grpc/client
+
+PEERDIR(
+ contrib/libs/protobuf
+ library/cpp/grpc/client
ydb/library/yql/public/issue
-)
-
+)
+
GENERATE_ENUM_SERIALIZATION(s3_settings.h)
-GENERATE_ENUM_SERIALIZATION(status_codes.h)
-
-END()
+GENERATE_ENUM_SERIALIZATION(status_codes.h)
+
+END()
diff --git a/ydb/public/sdk/cpp/client/ydb_types/ydb.h b/ydb/public/sdk/cpp/client/ydb_types/ydb.h
index 284f158b13..2321d94c2a 100644
--- a/ydb/public/sdk/cpp/client/ydb_types/ydb.h
+++ b/ydb/public/sdk/cpp/client/ydb_types/ydb.h
@@ -1,29 +1,29 @@
-#pragma once
-
-#include "status_codes.h"
-
-namespace NYdb {
-
-enum class EDiscoveryMode {
- //! Block in ctor (driver or client if database and/or auth token is overridden by client settings)
- //! until we get list of endpoints, if list of endpoints become empty during executing requests
- //! corresponding error will be returned.
- //! Note: Even in Sync mode SDK will perform lazy async update of endpoints list
- Sync,
- //! Do not block in ctor to get endpoint list.
- //! If list of endpoints is empty in time of request (or becomes empty during execution) the request will be pending until
- //! we got endpoint list. The error will be returned if the endpoint list
- //! is empty and discovery failed
- //! This method is a bit more "user friendly" but can produce additional hidden latency
- Async
-};
-
-enum class EBalancingPolicy {
- //! Use all available cluster nodes regardless datacenter locality
- UseAllNodes,
- //! Use preferable location,
- //! params is a name of location (VLA, MAN), if params is empty local datacenter is used
- UsePreferableLocation
-};
-
-} // namespace NYdb
+#pragma once
+
+#include "status_codes.h"
+
+namespace NYdb {
+
+enum class EDiscoveryMode {
+ //! Block in ctor (driver or client if database and/or auth token is overridden by client settings)
+ //! until we get list of endpoints, if list of endpoints become empty during executing requests
+ //! corresponding error will be returned.
+ //! Note: Even in Sync mode SDK will perform lazy async update of endpoints list
+ Sync,
+ //! Do not block in ctor to get endpoint list.
+ //! If list of endpoints is empty in time of request (or becomes empty during execution) the request will be pending until
+ //! we got endpoint list. The error will be returned if the endpoint list
+ //! is empty and discovery failed
+ //! This method is a bit more "user friendly" but can produce additional hidden latency
+ Async
+};
+
+enum class EBalancingPolicy {
+ //! Use all available cluster nodes regardless datacenter locality
+ UseAllNodes,
+ //! Use preferable location,
+ //! params is a name of location (VLA, MAN), if params is empty local datacenter is used
+ UsePreferableLocation
+};
+
+} // namespace NYdb
diff --git a/ydb/public/sdk/cpp/client/ydb_value/ut/ya.make b/ydb/public/sdk/cpp/client/ydb_value/ut/ya.make
index 59696f2b9a..aaf085d2bd 100644
--- a/ydb/public/sdk/cpp/client/ydb_value/ut/ya.make
+++ b/ydb/public/sdk/cpp/client/ydb_value/ut/ya.make
@@ -1,28 +1,28 @@
UNITTEST_FOR(ydb/public/sdk/cpp/client/ydb_value)
-
-OWNER(
- dcherednik
- g:kikimr
-)
-
-IF (SANITIZER_TYPE)
- TIMEOUT(1200)
- SIZE(LARGE)
- TAG(ya:fat)
-ELSE()
- TIMEOUT(600)
- SIZE(MEDIUM)
-ENDIF()
-
-FORK_SUBTESTS()
-
-PEERDIR(
+
+OWNER(
+ dcherednik
+ g:kikimr
+)
+
+IF (SANITIZER_TYPE)
+ TIMEOUT(1200)
+ SIZE(LARGE)
+ TAG(ya:fat)
+ELSE()
+ TIMEOUT(600)
+ SIZE(MEDIUM)
+ENDIF()
+
+FORK_SUBTESTS()
+
+PEERDIR(
ydb/public/lib/json_value
ydb/public/lib/yson_value
-)
-
-SRCS(
- value_ut.cpp
-)
-
-END()
+)
+
+SRCS(
+ value_ut.cpp
+)
+
+END()
diff --git a/ydb/public/sdk/cpp/client/ydb_value/value.cpp b/ydb/public/sdk/cpp/client/ydb_value/value.cpp
index 83dacb79c5..8c18d4d50f 100644
--- a/ydb/public/sdk/cpp/client/ydb_value/value.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_value/value.cpp
@@ -1,5 +1,5 @@
-#include "value.h"
-
+#include "value.h"
+
#define INCLUDE_YDB_INTERNAL_H
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/value_helpers/helpers.h>
#undef INCLUDE_YDB_INTERNAL_H
@@ -7,15 +7,15 @@
#include <ydb/public/sdk/cpp/client/ydb_params/params.h>
#include <ydb/public/sdk/cpp/client/ydb_proto/accessor.h>
#include <ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers/handlers.h>
-
+
#include <ydb/public/api/protos/ydb_value.pb.h>
#include <library/cpp/containers/stack_vector/stack_vec.h>
#include <ydb/library/yql/public/decimal/yql_decimal.h>
-#include <util/generic/bitmap.h>
-#include <util/generic/map.h>
+#include <util/generic/bitmap.h>
+#include <util/generic/map.h>
#include <util/string/builder.h>
namespace NYdb {
diff --git a/ydb/public/sdk/cpp/client/ydb_value/value.h b/ydb/public/sdk/cpp/client/ydb_value/value.h
index c935af67ad..4a163b9220 100644
--- a/ydb/public/sdk/cpp/client/ydb_value/value.h
+++ b/ydb/public/sdk/cpp/client/ydb_value/value.h
@@ -1,10 +1,10 @@
#pragma once
-#include <util/datetime/base.h>
+#include <util/datetime/base.h>
#include <util/generic/maybe.h>
-#include <memory>
-
+#include <memory>
+
namespace Ydb {
class Type;
class Value;
diff --git a/ydb/public/sdk/cpp/client/ydb_value/value_ut.cpp b/ydb/public/sdk/cpp/client/ydb_value/value_ut.cpp
index 2895acb944..d7508a5f9b 100644
--- a/ydb/public/sdk/cpp/client/ydb_value/value_ut.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_value/value_ut.cpp
@@ -4,9 +4,9 @@
#include <ydb/public/lib/json_value/ydb_json_value.h>
#include <ydb/public/lib/yson_value/ydb_yson_value.h>
-#include <library/cpp/testing/unittest/registar.h>
-#include <library/cpp/testing/unittest/tests_data.h>
-
+#include <library/cpp/testing/unittest/registar.h>
+#include <library/cpp/testing/unittest/tests_data.h>
+
#include <google/protobuf/messagext.h>
namespace NYdb {
diff --git a/ydb/public/sdk/cpp/client/ydb_value/ya.make b/ydb/public/sdk/cpp/client/ydb_value/ya.make
index 7d8df655b3..c7db140950 100644
--- a/ydb/public/sdk/cpp/client/ydb_value/ya.make
+++ b/ydb/public/sdk/cpp/client/ydb_value/ya.make
@@ -1,27 +1,27 @@
-LIBRARY()
-
-OWNER(
- dcherednik
- g:kikimr
-)
-
-SRCS(
- value.cpp
-)
-
-GENERATE_ENUM_SERIALIZATION(value.h)
-
-PEERDIR(
- library/cpp/containers/stack_vector
+LIBRARY()
+
+OWNER(
+ dcherednik
+ g:kikimr
+)
+
+SRCS(
+ value.cpp
+)
+
+GENERATE_ENUM_SERIALIZATION(value.h)
+
+PEERDIR(
+ library/cpp/containers/stack_vector
ydb/public/api/protos
ydb/public/sdk/cpp/client/impl/ydb_internal/value_helpers
ydb/public/sdk/cpp/client/ydb_proto
ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers
ydb/library/yql/public/decimal
-)
-
-END()
-
+)
+
+END()
+
RECURSE_FOR_TESTS(
ut
)
diff --git a/ydb/public/sdk/python/ydb/resolver.py b/ydb/public/sdk/python/ydb/resolver.py
index 5170227269..54712c0085 100644
--- a/ydb/public/sdk/python/ydb/resolver.py
+++ b/ydb/public/sdk/python/ydb/resolver.py
@@ -137,7 +137,7 @@ class DiscoveryEndpointsResolver(object):
self._endpoints_iter = itertools.cycle(self._endpoints)
def _add_debug_details(self, message, *args):
- self.logger.debug(message, *args)
+ self.logger.debug(message, *args)
message = message % args
with self._lock:
self._debug_details_items.append(message)
diff --git a/ydb/public/tools/lib/cmds/ya.make b/ydb/public/tools/lib/cmds/ya.make
index 9e1ca7c4e7..61698522e9 100644
--- a/ydb/public/tools/lib/cmds/ya.make
+++ b/ydb/public/tools/lib/cmds/ya.make
@@ -1,13 +1,13 @@
-PY23_LIBRARY()
-OWNER(g:kikimr)
-
-PY_SRCS(
- __init__.py
-)
-
-PEERDIR(
- ydb/tests/library
- library/python/testing/recipe
-)
-
-END()
+PY23_LIBRARY()
+OWNER(g:kikimr)
+
+PY_SRCS(
+ __init__.py
+)
+
+PEERDIR(
+ ydb/tests/library
+ library/python/testing/recipe
+)
+
+END()
diff --git a/ydb/services/datastreams/datastreams_proxy.cpp b/ydb/services/datastreams/datastreams_proxy.cpp
index e6e4685af1..03a6d06213 100644
--- a/ydb/services/datastreams/datastreams_proxy.cpp
+++ b/ydb/services/datastreams/datastreams_proxy.cpp
@@ -22,7 +22,7 @@ using namespace NKikimrClient;
using grpc::Status;
namespace NKikimr::NDataStreams::V1 {
- const TString YDS_SERVICE_TYPE = "data-streams";
+ const TString YDS_SERVICE_TYPE = "data-streams";
using namespace NGRpcService;
using namespace NGRpcProxy::V1;
@@ -263,7 +263,7 @@ namespace NKikimr::NDataStreams::V1 {
}
void Bootstrap(const TActorContext& ctx);
- void ModifyPersqueueConfig(const TActorContext& ctx,
+ void ModifyPersqueueConfig(const TActorContext& ctx,
NKikimrSchemeOp::TPersQueueGroupDescription& groupConfig,
const NKikimrSchemeOp::TPersQueueGroupDescription& pqGroupDescription,
const NKikimrSchemeOp::TDirEntry& selfInfo);
@@ -275,14 +275,14 @@ namespace NKikimr::NDataStreams::V1 {
Become(&TBase::StateWork);
}
- void TUpdateShardCountActor::ModifyPersqueueConfig(
- const TActorContext& ctx,
+ void TUpdateShardCountActor::ModifyPersqueueConfig(
+ const TActorContext& ctx,
NKikimrSchemeOp::TPersQueueGroupDescription& groupConfig,
const NKikimrSchemeOp::TPersQueueGroupDescription& pqGroupDescription,
const NKikimrSchemeOp::TDirEntry& selfInfo
- ) {
- Y_UNUSED(pqGroupDescription);
- Y_UNUSED(selfInfo);
+ ) {
+ Y_UNUSED(pqGroupDescription);
+ Y_UNUSED(selfInfo);
TString error;
if (!ValidateShardsCount(*GetProtoRequest(), groupConfig, error)) {
@@ -304,7 +304,7 @@ namespace NKikimr::NDataStreams::V1 {
}
void Bootstrap(const TActorContext& ctx);
- void ModifyPersqueueConfig(const TActorContext& ctx,
+ void ModifyPersqueueConfig(const TActorContext& ctx,
NKikimrSchemeOp::TPersQueueGroupDescription& groupConfig,
const NKikimrSchemeOp::TPersQueueGroupDescription& pqGroupDescription,
const NKikimrSchemeOp::TDirEntry& selfInfo);
@@ -316,14 +316,14 @@ namespace NKikimr::NDataStreams::V1 {
Become(&TBase::StateWork);
}
- void TUpdateStreamActor::ModifyPersqueueConfig(
- const TActorContext& ctx,
+ void TUpdateStreamActor::ModifyPersqueueConfig(
+ const TActorContext& ctx,
NKikimrSchemeOp::TPersQueueGroupDescription& groupConfig,
const NKikimrSchemeOp::TPersQueueGroupDescription& pqGroupDescription,
const NKikimrSchemeOp::TDirEntry& selfInfo
- ) {
- Y_UNUSED(pqGroupDescription);
- Y_UNUSED(selfInfo);
+ ) {
+ Y_UNUSED(pqGroupDescription);
+ Y_UNUSED(selfInfo);
TString error;
if (!ValidateShardsCount(*GetProtoRequest(), groupConfig, error)
@@ -351,7 +351,7 @@ namespace NKikimr::NDataStreams::V1 {
}
void Bootstrap(const TActorContext& ctx);
- void ModifyPersqueueConfig(const TActorContext& ctx,
+ void ModifyPersqueueConfig(const TActorContext& ctx,
NKikimrSchemeOp::TPersQueueGroupDescription& groupConfig,
const NKikimrSchemeOp::TPersQueueGroupDescription& pqGroupDescription,
const NKikimrSchemeOp::TDirEntry& selfInfo);
@@ -363,14 +363,14 @@ namespace NKikimr::NDataStreams::V1 {
Become(&TBase::StateWork);
}
- void TSetWriteQuotaActor::ModifyPersqueueConfig(
- const TActorContext& ctx,
+ void TSetWriteQuotaActor::ModifyPersqueueConfig(
+ const TActorContext& ctx,
NKikimrSchemeOp::TPersQueueGroupDescription& groupConfig,
const NKikimrSchemeOp::TPersQueueGroupDescription& pqGroupDescription,
const NKikimrSchemeOp::TDirEntry& selfInfo
- ) {
- Y_UNUSED(pqGroupDescription);
- Y_UNUSED(selfInfo);
+ ) {
+ Y_UNUSED(pqGroupDescription);
+ Y_UNUSED(selfInfo);
TString error;
if (!ValidateWriteSpeedLimit(*GetProtoRequest(), error, ctx)) {
@@ -399,14 +399,14 @@ namespace NKikimr::NDataStreams::V1 {
TBase::Become(&TBase::StateWork);
}
- void ModifyPersqueueConfig(
- const TActorContext& ctx,
+ void ModifyPersqueueConfig(
+ const TActorContext& ctx,
NKikimrSchemeOp::TPersQueueGroupDescription& groupConfig,
const NKikimrSchemeOp::TPersQueueGroupDescription& pqGroupDescription,
const NKikimrSchemeOp::TDirEntry& selfInfo
- ) {
- Y_UNUSED(pqGroupDescription);
- Y_UNUSED(selfInfo);
+ ) {
+ Y_UNUSED(pqGroupDescription);
+ Y_UNUSED(selfInfo);
TString error;
if (!ValidateRetentionPeriod(*this->GetProtoRequest(), groupConfig, ShouldIncrease, error)) {
@@ -493,9 +493,9 @@ namespace NKikimr::NDataStreams::V1 {
}
void TDescribeStreamActor::HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) {
- const NSchemeCache::TSchemeCacheNavigate* result = ev->Get()->Request.Get();
- Y_VERIFY(result->ResultSet.size() == 1); // describe only one topic
- const auto& response = result->ResultSet.front();
+ const NSchemeCache::TSchemeCacheNavigate* result = ev->Get()->Request.Get();
+ Y_VERIFY(result->ResultSet.size() == 1); // describe only one topic
+ const auto& response = result->ResultSet.front();
const TString path = JoinSeq("/", response.Path);
if (ReplyIfNotTopic(ev, ctx)) {
@@ -535,29 +535,29 @@ namespace NKikimr::NDataStreams::V1 {
void TDescribeStreamActor::ReplyAndDie(const TActorContext& ctx) {
Ydb::DataStreams::V1::DescribeStreamResult result;
- auto& pqConfig = PQGroup.GetPQTabletConfig();
+ auto& pqConfig = PQGroup.GetPQTabletConfig();
ui32 writeSpeed = pqConfig.GetPartitionConfig().GetWriteSpeedInBytesPerSecond() / 1024;
auto& description = *result.mutable_stream_description();
description.set_stream_name(GetProtoRequest()->stream_name());
ui32 retentionPeriodHours = TInstant::Seconds(pqConfig.GetPartitionConfig().GetLifetimeSeconds()).Hours();
description.set_retention_period_hours(retentionPeriodHours);
description.set_write_quota_kb_per_sec(writeSpeed);
- if (SelfInfo.GetCreateFinished()) {
- description.set_stream_status(Ydb::DataStreams::V1::StreamDescription::ACTIVE);
- } else {
- description.set_stream_status(Ydb::DataStreams::V1::StreamDescription::CREATING);
+ if (SelfInfo.GetCreateFinished()) {
+ description.set_stream_status(Ydb::DataStreams::V1::StreamDescription::ACTIVE);
+ } else {
+ description.set_stream_status(Ydb::DataStreams::V1::StreamDescription::CREATING);
}
bool startShardFound = GetProtoRequest()->exclusive_start_shard_id().empty();
description.set_has_more_shards(false);
- description.set_owner(SelfInfo.GetOwner());
- description.set_stream_creation_timestamp(TInstant::MilliSeconds(SelfInfo.GetCreateStep()).Seconds());
+ description.set_owner(SelfInfo.GetOwner());
+ description.set_stream_creation_timestamp(TInstant::MilliSeconds(SelfInfo.GetCreateStep()).Seconds());
int limit = GetProtoRequest()->limit() == 0 ? 100 : GetProtoRequest()->limit();
- for (uint32_t i = 0; i < (uint32_t)PQGroup.GetPartitions().size(); ++i) {
- ui32 partitionId = PQGroup.GetPartitions(i).GetPartitionId();
+ for (uint32_t i = 0; i < (uint32_t)PQGroup.GetPartitions().size(); ++i) {
+ ui32 partitionId = PQGroup.GetPartitions(i).GetPartitionId();
TString shardName = GetShardName(partitionId);
if (shardName == GetProtoRequest()->exclusive_start_shard_id()) {
startShardFound = true;
@@ -569,7 +569,7 @@ namespace NKikimr::NDataStreams::V1 {
auto* shard = description.add_shards();
shard->set_shard_id(shardName);
auto* rangeProto = shard->mutable_hash_key_range();
- auto range = RangeFromShardNumber(partitionId, PQGroup.GetPartitions().size());
+ auto range = RangeFromShardNumber(partitionId, PQGroup.GetPartitions().size());
rangeProto->set_starting_hash_key(Uint128ToDecimalString(range.Start));
rangeProto->set_ending_hash_key(Uint128ToDecimalString(range.End));
auto it = StartEndOffsetsPerPartition.find(partitionId);
@@ -809,9 +809,9 @@ namespace NKikimr::NDataStreams::V1 {
}
void TListStreamConsumersActor::HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) {
- const NSchemeCache::TSchemeCacheNavigate* result = ev->Get()->Request.Get();
- Y_VERIFY(result->ResultSet.size() == 1); // describe only one topic
-
+ const NSchemeCache::TSchemeCacheNavigate* result = ev->Get()->Request.Get();
+ Y_VERIFY(result->ResultSet.size() == 1); // describe only one topic
+
if (ReplyIfNotTopic(ev, ctx)) {
return;
}
@@ -893,24 +893,24 @@ namespace NKikimr::NDataStreams::V1 {
Become(&TRegisterStreamConsumerActor::StateWork);
}
- void TRegisterStreamConsumerActor::ModifyPersqueueConfig(
- const TActorContext& ctx,
+ void TRegisterStreamConsumerActor::ModifyPersqueueConfig(
+ const TActorContext& ctx,
NKikimrSchemeOp::TPersQueueGroupDescription& groupConfig,
const NKikimrSchemeOp::TPersQueueGroupDescription& pqGroupDescription,
const NKikimrSchemeOp::TDirEntry& selfInfo
- ) {
- Y_UNUSED(pqGroupDescription);
-
+ ) {
+ Y_UNUSED(pqGroupDescription);
+
auto* pqConfig = groupConfig.MutablePQTabletConfig();
Ydb::PersQueue::V1::TopicSettings::ReadRule readRule;
readRule.set_consumer_name(ConsumerName);
readRule.set_supported_format(Ydb::PersQueue::V1::TopicSettings_Format_FORMAT_BASE);
readRule.set_starting_message_timestamp_ms(TInstant::Now().MilliSeconds());
readRule.set_important(false);
- readRule.set_service_type(YDS_SERVICE_TYPE);
+ readRule.set_service_type(YDS_SERVICE_TYPE);
if (readRule.version() == 0) {
- readRule.set_version(selfInfo.GetVersion().GetPQVersion());
+ readRule.set_version(selfInfo.GetVersion().GetPQVersion());
}
auto serviceTypes = GetSupportedClientServiceTypes(ctx);
TString error = AddReadRuleToConfig(pqConfig, readRule, serviceTypes, ctx);
@@ -967,19 +967,19 @@ namespace NKikimr::NDataStreams::V1 {
Become(&TDeregisterStreamConsumerActor::StateWork);
}
- void TDeregisterStreamConsumerActor::ModifyPersqueueConfig(
- const TActorContext& ctx,
+ void TDeregisterStreamConsumerActor::ModifyPersqueueConfig(
+ const TActorContext& ctx,
NKikimrSchemeOp::TPersQueueGroupDescription& groupConfig,
const NKikimrSchemeOp::TPersQueueGroupDescription& pqGroupDescription,
const NKikimrSchemeOp::TDirEntry& selfInfo
- ) {
- Y_UNUSED(selfInfo);
- auto error = RemoveReadRuleFromConfig(
- groupConfig.MutablePQTabletConfig(),
- pqGroupDescription.GetPQTabletConfig(),
- GetProtoRequest()->consumer_name(),
- ctx
- );
+ ) {
+ Y_UNUSED(selfInfo);
+ auto error = RemoveReadRuleFromConfig(
+ groupConfig.MutablePQTabletConfig(),
+ pqGroupDescription.GetPQTabletConfig(),
+ GetProtoRequest()->consumer_name(),
+ ctx
+ );
if (!error.Empty()) {
return ReplyWithError(Ydb::StatusIds::NOT_FOUND, Ydb::PersQueue::ErrorCode::BAD_REQUEST, error, ctx);
}
diff --git a/ydb/services/datastreams/datastreams_ut.cpp b/ydb/services/datastreams/datastreams_ut.cpp
index 4fedce232f..2ede2a2d05 100644
--- a/ydb/services/datastreams/datastreams_ut.cpp
+++ b/ydb/services/datastreams/datastreams_ut.cpp
@@ -42,7 +42,7 @@ public:
appConfig.MutablePQConfig()->MutableQuotingConfig()->SetPartitionReadQuotaIsTwiceWriteQuota(true);
appConfig.MutablePQConfig()->MutableBillingMeteringConfig()->SetEnabled(true);
appConfig.MutablePQConfig()->MutableBillingMeteringConfig()->SetFlushIntervalSec(1);
- appConfig.MutablePQConfig()->AddClientServiceType()->SetName("data-streams");
+ appConfig.MutablePQConfig()->AddClientServiceType()->SetName("data-streams");
MeteringFile = MakeHolder<TTempFileHandle>("meteringData.txt");
appConfig.MutableMeteringConfig()->SetMeteringFilePath(MeteringFile->Name());
@@ -157,13 +157,13 @@ Y_UNIT_TEST_SUITE(DataStreams) {
auto result = testServer.DataStreamsClient->DescribeStream(streamName).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
-
+
UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_description().stream_status(),
YDS_V1::StreamDescription::ACTIVE);
UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_description().stream_name(), streamName);
- UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_description().write_quota_kb_per_sec(), 1024);
- UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_description().retention_period_hours(), 24);
-
+ UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_description().write_quota_kb_per_sec(), 1024);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_description().retention_period_hours(), 24);
+
UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_description().shards().size(), 3);
UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_description().shards(0).sequence_number_range().starting_sequence_number(), "0");
UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_description().shards(0).hash_key_range().starting_hash_key(), "0");
diff --git a/ydb/services/lib/actors/pq_schema_actor.cpp b/ydb/services/lib/actors/pq_schema_actor.cpp
index ecbe921cae..f48e398488 100644
--- a/ydb/services/lib/actors/pq_schema_actor.cpp
+++ b/ydb/services/lib/actors/pq_schema_actor.cpp
@@ -22,54 +22,54 @@ namespace NKikimr::NGRpcProxy::V1 {
TClientServiceTypes GetSupportedClientServiceTypes(const TActorContext& ctx) {
TClientServiceTypes serviceTypes;
- const auto& pqConfig = AppData(ctx)->PQConfig;
+ const auto& pqConfig = AppData(ctx)->PQConfig;
ui32 count = pqConfig.GetDefaultClientServiceType().GetMaxReadRulesCountPerTopic();
if (count == 0) count = Max<ui32>();
TString name = pqConfig.GetDefaultClientServiceType().GetName();
serviceTypes.insert({name, {name, count}});
- for (const auto& serviceType : pqConfig.GetClientServiceType()) {
+ for (const auto& serviceType : pqConfig.GetClientServiceType()) {
ui32 count = serviceType.GetMaxReadRulesCountPerTopic();
if (count == 0) count = Max<ui32>();
TString name = serviceType.GetName();
serviceTypes.insert({name, {name, count}});
- }
- return serviceTypes;
- }
-
- TString ReadRuleServiceTypeMigration(NKikimrPQ::TPQTabletConfig *config, const TActorContext& ctx) {
- auto rrServiceTypes = config->MutableReadRuleServiceTypes();
- if (config->ReadRuleServiceTypesSize() > config->ReadRulesSize()) {
- rrServiceTypes->Clear();
- }
- if (config->ReadRuleServiceTypesSize() < config->ReadRulesSize()) {
- rrServiceTypes->Reserve(config->ReadRulesSize());
- const auto& pqConfig = AppData(ctx)->PQConfig;
- if (pqConfig.GetDisallowDefaultClientServiceType()) {
- return "service type must be set for all read rules";
- }
- for (ui64 i = rrServiceTypes->size(); i < config->ReadRulesSize(); ++i) {
- *rrServiceTypes->Add() = pqConfig.GetDefaultClientServiceType().GetName();
- }
- }
- return "";
- }
-
- TString AddReadRuleToConfig(
- NKikimrPQ::TPQTabletConfig* config,
- const Ydb::PersQueue::V1::TopicSettings::ReadRule& rr,
+ }
+ return serviceTypes;
+ }
+
+ TString ReadRuleServiceTypeMigration(NKikimrPQ::TPQTabletConfig *config, const TActorContext& ctx) {
+ auto rrServiceTypes = config->MutableReadRuleServiceTypes();
+ if (config->ReadRuleServiceTypesSize() > config->ReadRulesSize()) {
+ rrServiceTypes->Clear();
+ }
+ if (config->ReadRuleServiceTypesSize() < config->ReadRulesSize()) {
+ rrServiceTypes->Reserve(config->ReadRulesSize());
+ const auto& pqConfig = AppData(ctx)->PQConfig;
+ if (pqConfig.GetDisallowDefaultClientServiceType()) {
+ return "service type must be set for all read rules";
+ }
+ for (ui64 i = rrServiceTypes->size(); i < config->ReadRulesSize(); ++i) {
+ *rrServiceTypes->Add() = pqConfig.GetDefaultClientServiceType().GetName();
+ }
+ }
+ return "";
+ }
+
+ TString AddReadRuleToConfig(
+ NKikimrPQ::TPQTabletConfig* config,
+ const Ydb::PersQueue::V1::TopicSettings::ReadRule& rr,
const TClientServiceTypes& supportedClientServiceTypes,
- const TActorContext& ctx
- ) {
+ const TActorContext& ctx
+ ) {
auto consumerName = NPersQueue::ConvertNewConsumerName(rr.consumer_name(), ctx);
if(consumerName.find("/") != TString::npos || consumerName.find("|") != TString::npos) {
return TStringBuilder() << "consumer '" << rr.consumer_name() << "' has illegal symbols";
}
- {
- TString migrationError = ReadRuleServiceTypeMigration(config, ctx);
- if (migrationError) {
- return migrationError;
- }
- }
+ {
+ TString migrationError = ReadRuleServiceTypeMigration(config, ctx);
+ if (migrationError) {
+ return migrationError;
+ }
+ }
config->AddReadRules(consumerName);
@@ -102,29 +102,29 @@ namespace NKikimr::NGRpcProxy::V1 {
if (rr.important())
config->MutablePartitionConfig()->AddImportantClientId(consumerName);
- if (!rr.service_type().empty()) {
+ if (!rr.service_type().empty()) {
if (!supportedClientServiceTypes.contains(rr.service_type())) {
- return TStringBuilder() << "Unknown read rule service type '" << rr.service_type()
- << "' for consumer '" << rr.consumer_name() << "'";
- }
- config->AddReadRuleServiceTypes(rr.service_type());
- } else {
- const auto& pqConfig = AppData(ctx)->PQConfig;
- if (pqConfig.GetDisallowDefaultClientServiceType()) {
- return TStringBuilder() << "service type cannot be empty for consumer '" << rr.consumer_name() << "'";
- }
- const auto& defaultCientServiceType = pqConfig.GetDefaultClientServiceType().GetName();
- config->AddReadRuleServiceTypes(defaultCientServiceType);
- }
+ return TStringBuilder() << "Unknown read rule service type '" << rr.service_type()
+ << "' for consumer '" << rr.consumer_name() << "'";
+ }
+ config->AddReadRuleServiceTypes(rr.service_type());
+ } else {
+ const auto& pqConfig = AppData(ctx)->PQConfig;
+ if (pqConfig.GetDisallowDefaultClientServiceType()) {
+ return TStringBuilder() << "service type cannot be empty for consumer '" << rr.consumer_name() << "'";
+ }
+ const auto& defaultCientServiceType = pqConfig.GetDefaultClientServiceType().GetName();
+ config->AddReadRuleServiceTypes(defaultCientServiceType);
+ }
return "";
}
- TString RemoveReadRuleFromConfig(
- NKikimrPQ::TPQTabletConfig* config,
- const NKikimrPQ::TPQTabletConfig& originalConfig,
- const TString& consumerName,
- const TActorContext& ctx
- ) {
+ TString RemoveReadRuleFromConfig(
+ NKikimrPQ::TPQTabletConfig* config,
+ const NKikimrPQ::TPQTabletConfig& originalConfig,
+ const TString& consumerName,
+ const TActorContext& ctx
+ ) {
THashSet<TString> rulesToRemove;
rulesToRemove.insert(consumerName);
@@ -134,7 +134,7 @@ namespace NKikimr::NGRpcProxy::V1 {
config->ClearConsumerFormatVersions();
config->ClearConsumerCodecs();
config->MutablePartitionConfig()->ClearImportantClientId();
- config->ClearReadRuleServiceTypes();
+ config->ClearReadRuleServiceTypes();
for (const auto& importantConsumer : originalConfig.GetPartitionConfig().GetImportantClientId()) {
if (rulesToRemove.find(importantConsumer) == rulesToRemove.end()) {
@@ -142,7 +142,7 @@ namespace NKikimr::NGRpcProxy::V1 {
}
}
- const auto& pqConfig = AppData(ctx)->PQConfig;
+ const auto& pqConfig = AppData(ctx)->PQConfig;
for (size_t i = 0; i < originalConfig.ReadRulesSize(); i++) {
if (auto it = rulesToRemove.find(originalConfig.GetReadRules(i)); it != rulesToRemove.end()) {
rulesToRemove.erase(it);
@@ -158,15 +158,15 @@ namespace NKikimr::NGRpcProxy::V1 {
ct->AddCodecs(originalConfig.GetConsumerCodecs(i).GetCodecs(j));
ct->AddIds(originalConfig.GetConsumerCodecs(i).GetIds(j));
}
- if (i < originalConfig.ReadRuleServiceTypesSize()) {
- config->AddReadRuleServiceTypes(originalConfig.GetReadRuleServiceTypes(i));
- } else {
- if (pqConfig.GetDisallowDefaultClientServiceType()) {
+ if (i < originalConfig.ReadRuleServiceTypesSize()) {
+ config->AddReadRuleServiceTypes(originalConfig.GetReadRuleServiceTypes(i));
+ } else {
+ if (pqConfig.GetDisallowDefaultClientServiceType()) {
return TStringBuilder() << "service type cannot be empty for consumer '"
- << originalConfig.GetReadRules(i) << "'";
- }
- config->AddReadRuleServiceTypes(pqConfig.GetDefaultClientServiceType().GetName());
- }
+ << originalConfig.GetReadRules(i) << "'";
+ }
+ config->AddReadRuleServiceTypes(pqConfig.GetDefaultClientServiceType().GetName());
+ }
}
if (rulesToRemove.size() > 0) {
@@ -387,15 +387,15 @@ namespace NKikimr::NGRpcProxy::V1 {
return Ydb::StatusIds::BAD_REQUEST;
}
- {
- error = ReadRuleServiceTypeMigration(config, ctx);
- if (error) {
- return Ydb::StatusIds::INTERNAL_ERROR;
- }
- }
- const auto& supportedClientServiceTypes = GetSupportedClientServiceTypes(ctx);
+ {
+ error = ReadRuleServiceTypeMigration(config, ctx);
+ if (error) {
+ return Ydb::StatusIds::INTERNAL_ERROR;
+ }
+ }
+ const auto& supportedClientServiceTypes = GetSupportedClientServiceTypes(ctx);
for (const auto& rr : settings.read_rules()) {
- error = AddReadRuleToConfig(config, rr, supportedClientServiceTypes, ctx);
+ error = AddReadRuleToConfig(config, rr, supportedClientServiceTypes, ctx);
if (!error.Empty()) {
return Ydb::StatusIds::BAD_REQUEST;
}
diff --git a/ydb/services/lib/actors/pq_schema_actor.h b/ydb/services/lib/actors/pq_schema_actor.h
index 90761e978f..1be2e7fa24 100644
--- a/ydb/services/lib/actors/pq_schema_actor.h
+++ b/ydb/services/lib/actors/pq_schema_actor.h
@@ -12,37 +12,37 @@
namespace NKikimr::NGRpcProxy::V1 {
- Ydb::StatusIds::StatusCode FillProposeRequestImpl(
- const TString& name,
- const Ydb::PersQueue::V1::TopicSettings& settings,
+ Ydb::StatusIds::StatusCode FillProposeRequestImpl(
+ const TString& name,
+ const Ydb::PersQueue::V1::TopicSettings& settings,
NKikimrSchemeOp::TModifyScheme& modifyScheme,
- const TActorContext& ctx,
- bool alter,
- TString& error
- );
-
+ const TActorContext& ctx,
+ bool alter,
+ TString& error
+ );
+
struct TClientServiceType {
TString Name;
ui32 MaxCount;
};
typedef std::map<TString, TClientServiceType> TClientServiceTypes;
TClientServiceTypes GetSupportedClientServiceTypes(const TActorContext& ctx);
-
+
// Returns true if have duplicated read rules
bool CheckReadRulesConfig(const NKikimrPQ::TPQTabletConfig& config, const TClientServiceTypes& supportedReadRuleServiceTypes, TString& error);
-
- TString AddReadRuleToConfig(
- NKikimrPQ::TPQTabletConfig *config,
- const Ydb::PersQueue::V1::TopicSettings::ReadRule& rr,
+
+ TString AddReadRuleToConfig(
+ NKikimrPQ::TPQTabletConfig *config,
+ const Ydb::PersQueue::V1::TopicSettings::ReadRule& rr,
const TClientServiceTypes& supportedReadRuleServiceTypes,
- const TActorContext& ctx
- );
- TString RemoveReadRuleFromConfig(
- NKikimrPQ::TPQTabletConfig *config,
+ const TActorContext& ctx
+ );
+ TString RemoveReadRuleFromConfig(
+ NKikimrPQ::TPQTabletConfig *config,
const NKikimrPQ::TPQTabletConfig& originalConfig,
- const TString& consumerName,
- const TActorContext& ctx
- );
+ const TString& consumerName,
+ const TActorContext& ctx
+ );
NYql::TIssue FillIssue(const TString &errorReason, const Ydb::PersQueue::ErrorCode::ErrorCode errorCode);
@@ -105,14 +105,14 @@ namespace NKikimr::NGRpcProxy::V1 {
void SendDescribeProposeRequest(const NActors::TActorContext& ctx) {
PrepareTopicPath(ctx);
- auto navigateRequest = std::make_unique<NSchemeCache::TSchemeCacheNavigate>();
- navigateRequest->DatabaseName = CanonizePath(this->Request_->GetDatabaseName().GetOrElse(""));
+ auto navigateRequest = std::make_unique<NSchemeCache::TSchemeCacheNavigate>();
+ navigateRequest->DatabaseName = CanonizePath(this->Request_->GetDatabaseName().GetOrElse(""));
- NSchemeCache::TSchemeCacheNavigate::TEntry entry;
- entry.Path = NKikimr::SplitPath(TopicPath);
- entry.SyncVersion = true;
- entry.Operation = NSchemeCache::TSchemeCacheNavigate::OpTopic;
- navigateRequest->ResultSet.emplace_back(entry);
+ NSchemeCache::TSchemeCacheNavigate::TEntry entry;
+ entry.Path = NKikimr::SplitPath(TopicPath);
+ entry.SyncVersion = true;
+ entry.Operation = NSchemeCache::TSchemeCacheNavigate::OpTopic;
+ navigateRequest->ResultSet.emplace_back(entry);
if (this->Request_->GetInternalToken().empty()) {
if (AppData(ctx)->PQConfig.GetRequireCredentialsInNewProtocol()) {
@@ -120,24 +120,24 @@ namespace NKikimr::NGRpcProxy::V1 {
"Unauthenticated access is forbidden, please provide credentials", ctx);
}
} else {
- navigateRequest->UserToken = new NACLib::TUserToken(this->Request_->GetInternalToken());
+ navigateRequest->UserToken = new NACLib::TUserToken(this->Request_->GetInternalToken());
}
if (!IsDead) {
- ctx.Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvNavigateKeySet(navigateRequest.release()));
+ ctx.Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvNavigateKeySet(navigateRequest.release()));
}
}
bool ReplyIfNotTopic(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) {
- const NSchemeCache::TSchemeCacheNavigate* result = ev->Get()->Request.Get();
- Y_VERIFY(result->ResultSet.size() == 1);
- const auto& response = result->ResultSet.front();
- const TString path = JoinPath(response.Path);
+ const NSchemeCache::TSchemeCacheNavigate* result = ev->Get()->Request.Get();
+ Y_VERIFY(result->ResultSet.size() == 1);
+ const auto& response = result->ResultSet.front();
+ const TString path = JoinPath(response.Path);
if (ev->Get()->Request.Get()->ResultSet.size() != 1 ||
ev->Get()->Request.Get()->ResultSet.begin()->Kind !=
NSchemeCache::TSchemeCacheNavigate::KindTopic) {
- this->Request_->RaiseIssue(
- FillIssue(
+ this->Request_->RaiseIssue(
+ FillIssue(
TStringBuilder() << "path '" << path << "' is not a stream",
Ydb::PersQueue::ErrorCode::ERROR
)
@@ -165,10 +165,10 @@ namespace NKikimr::NGRpcProxy::V1 {
FillIssue(
TStringBuilder() << "path '" << path << "' does not exist or you " <<
"do not have access rights",
- Ydb::PersQueue::ErrorCode::ERROR
- )
- );
- return TBase::Reply(Ydb::StatusIds::SCHEME_ERROR, ctx);
+ Ydb::PersQueue::ErrorCode::ERROR
+ )
+ );
+ return TBase::Reply(Ydb::StatusIds::SCHEME_ERROR, ctx);
}
case NSchemeCache::TSchemeCacheNavigate::EStatus::TableCreationNotComplete: {
this->Request_->RaiseIssue(
@@ -256,28 +256,28 @@ namespace NKikimr::NGRpcProxy::V1 {
const TString& workingDir, const TString& name)
{
Y_UNUSED(name);
- const auto& response = DescribeSchemeResult->Get()->Request.Get()->ResultSet.front();
+ const auto& response = DescribeSchemeResult->Get()->Request.Get()->ResultSet.front();
NKikimrSchemeOp::TModifyScheme& modifyScheme(*proposal.Record.MutableTransaction()->MutableModifyScheme());
modifyScheme.SetOperationType(NKikimrSchemeOp::EOperationType::ESchemeOpAlterPersQueueGroup);
modifyScheme.SetWorkingDir(workingDir);
auto* config = modifyScheme.MutableAlterPersQueueGroup();
- Y_VERIFY(response.Self);
- Y_VERIFY(response.PQGroupInfo);
- config->CopyFrom(response.PQGroupInfo->Description);
+ Y_VERIFY(response.Self);
+ Y_VERIFY(response.PQGroupInfo);
+ config->CopyFrom(response.PQGroupInfo->Description);
{
auto applyIf = modifyScheme.AddApplyIf();
- applyIf->SetPathId(response.Self->Info.GetPathId());
- applyIf->SetPathVersion(response.Self->Info.GetPathVersion());
+ applyIf->SetPathId(response.Self->Info.GetPathId());
+ applyIf->SetPathVersion(response.Self->Info.GetPathVersion());
}
- static_cast<TDerived*>(this)->ModifyPersqueueConfig(
- ctx,
- *config,
- response.PQGroupInfo->Description,
- response.Self->Info
- );
+ static_cast<TDerived*>(this)->ModifyPersqueueConfig(
+ ctx,
+ *config,
+ response.PQGroupInfo->Description,
+ response.Self->Info
+ );
this->DescribeSchemeResult.Reset();
}
@@ -296,7 +296,7 @@ namespace NKikimr::NGRpcProxy::V1 {
}
private:
- THolder<NActors::TEventHandle<TEvTxProxySchemeCache::TEvNavigateKeySetResult>> DescribeSchemeResult;
+ THolder<NActors::TEventHandle<TEvTxProxySchemeCache::TEvNavigateKeySetResult>> DescribeSchemeResult;
};
}
diff --git a/ydb/services/persqueue_cluster_discovery/cluster_ordering/ut/ya.make b/ydb/services/persqueue_cluster_discovery/cluster_ordering/ut/ya.make
index fbcd07dfa8..c1e137b002 100644
--- a/ydb/services/persqueue_cluster_discovery/cluster_ordering/ut/ya.make
+++ b/ydb/services/persqueue_cluster_discovery/cluster_ordering/ut/ya.make
@@ -3,7 +3,7 @@ UNITTEST_FOR(ydb/services/persqueue_cluster_discovery/cluster_ordering)
OWNER(
radix
g:kikimr
- g:logbroker
+ g:logbroker
)
SRCS(
diff --git a/ydb/services/persqueue_cluster_discovery/cluster_ordering/ya.make b/ydb/services/persqueue_cluster_discovery/cluster_ordering/ya.make
index ca9ec26420..6978da1d9b 100644
--- a/ydb/services/persqueue_cluster_discovery/cluster_ordering/ya.make
+++ b/ydb/services/persqueue_cluster_discovery/cluster_ordering/ya.make
@@ -1,7 +1,7 @@
OWNER(
radix
g:kikimr
- g:logbroker
+ g:logbroker
)
LIBRARY()
diff --git a/ydb/services/persqueue_cluster_discovery/ut/ya.make b/ydb/services/persqueue_cluster_discovery/ut/ya.make
index 28f0cbb64c..352988d2fe 100644
--- a/ydb/services/persqueue_cluster_discovery/ut/ya.make
+++ b/ydb/services/persqueue_cluster_discovery/ut/ya.make
@@ -1,10 +1,10 @@
UNITTEST_FOR(ydb/services/persqueue_cluster_discovery)
-OWNER(
- radix
+OWNER(
+ radix
g:kikimr
- g:logbroker
-)
+ g:logbroker
+)
FORK_SUBTESTS()
diff --git a/ydb/services/persqueue_cluster_discovery/ya.make b/ydb/services/persqueue_cluster_discovery/ya.make
index 629439e884..23917ae376 100644
--- a/ydb/services/persqueue_cluster_discovery/ya.make
+++ b/ydb/services/persqueue_cluster_discovery/ya.make
@@ -3,7 +3,7 @@ LIBRARY()
OWNER(
radix
g:kikimr
- g:logbroker
+ g:logbroker
)
SRCS(
diff --git a/ydb/services/persqueue_v1/grpc_pq_actor.h b/ydb/services/persqueue_v1/grpc_pq_actor.h
index fd2c569730..65c8aca1db 100644
--- a/ydb/services/persqueue_v1/grpc_pq_actor.h
+++ b/ydb/services/persqueue_v1/grpc_pq_actor.h
@@ -280,17 +280,17 @@ struct TEvPQProxy {
};
struct TEvReadResponse : public NActors::TEventLocal<TEvReadResponse, EvReadResponse> {
- explicit TEvReadResponse(PersQueue::V1::MigrationStreamingReadServerMessage&& resp, ui64 nextReadOffset, bool fromDisk, TDuration waitQuotaTime)
+ explicit TEvReadResponse(PersQueue::V1::MigrationStreamingReadServerMessage&& resp, ui64 nextReadOffset, bool fromDisk, TDuration waitQuotaTime)
: Response(std::move(resp))
, NextReadOffset(nextReadOffset)
, FromDisk(fromDisk)
- , WaitQuotaTime(waitQuotaTime)
+ , WaitQuotaTime(waitQuotaTime)
{ }
PersQueue::V1::MigrationStreamingReadServerMessage Response;
ui64 NextReadOffset;
bool FromDisk;
- TDuration WaitQuotaTime;
+ TDuration WaitQuotaTime;
};
struct TCommitCookie {
@@ -1009,7 +1009,7 @@ private:
const TString Guid;
TInstant Start;
bool FromDisk;
- TDuration WaitQuotaTime;
+ TDuration WaitQuotaTime;
};
THashMap<TActorId, TFormedReadResponse::TPtr> PartitionToReadResponse; // Partition actor -> TFormedReadResponse answer that has this partition.
@@ -1163,7 +1163,7 @@ public:
TAddReadRuleActor(NKikimr::NGRpcService::TEvPQAddReadRuleRequest *request);
void Bootstrap(const NActors::TActorContext &ctx);
- void ModifyPersqueueConfig(const TActorContext& ctx,
+ void ModifyPersqueueConfig(const TActorContext& ctx,
NKikimrSchemeOp::TPersQueueGroupDescription& groupConfig,
const NKikimrSchemeOp::TPersQueueGroupDescription& pqGroupDescription,
const NKikimrSchemeOp::TDirEntry& selfInfo);
@@ -1176,7 +1176,7 @@ public:
TRemoveReadRuleActor(NKikimr::NGRpcService::TEvPQRemoveReadRuleRequest* request);
void Bootstrap(const NActors::TActorContext &ctx);
- void ModifyPersqueueConfig(const TActorContext& ctx,
+ void ModifyPersqueueConfig(const TActorContext& ctx,
NKikimrSchemeOp::TPersQueueGroupDescription& groupConfig,
const NKikimrSchemeOp::TPersQueueGroupDescription& pqGroupDescription,
const NKikimrSchemeOp::TDirEntry& selfInfo);
diff --git a/ydb/services/persqueue_v1/grpc_pq_read_actor.cpp b/ydb/services/persqueue_v1/grpc_pq_read_actor.cpp
index d597ec8396..654edfcfcd 100644
--- a/ydb/services/persqueue_v1/grpc_pq_read_actor.cpp
+++ b/ydb/services/persqueue_v1/grpc_pq_read_actor.cpp
@@ -1451,7 +1451,7 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvReadResponse::TPtr& ev, const TAct
if (event->FromDisk) {
formedResponse->FromDisk = true;
}
- formedResponse->WaitQuotaTime = Max(formedResponse->WaitQuotaTime, event->WaitQuotaTime);
+ formedResponse->WaitQuotaTime = Max(formedResponse->WaitQuotaTime, event->WaitQuotaTime);
--formedResponse->RequestsInfly;
BytesInflight_ += diff;
@@ -1472,7 +1472,7 @@ bool TReadSessionActor::WriteResponse(PersQueue::V1::MigrationStreamingReadServe
}
void TReadSessionActor::ProcessAnswer(const TActorContext& ctx, TFormedReadResponse::TPtr formedResponse) {
- ui32 readDurationMs = (ctx.Now() - formedResponse->Start - formedResponse->WaitQuotaTime).MilliSeconds();
+ ui32 readDurationMs = (ctx.Now() - formedResponse->Start - formedResponse->WaitQuotaTime).MilliSeconds();
if (formedResponse->FromDisk) {
ReadLatencyFromDisk.IncFor(readDurationMs, 1);
} else {
@@ -1988,7 +1988,7 @@ bool FillBatchedData(MigrationStreamingReadServerMessage::DataBatch * data, cons
message->set_partition_key(r.GetPartitionKey());
if (proto.HasCodec()) {
- message->set_codec(NPQ::ToV1Codec((NPersQueueCommon::ECodec)proto.GetCodec()));
+ message->set_codec(NPQ::ToV1Codec((NPersQueueCommon::ECodec)proto.GetCodec()));
}
message->set_uncompressed_size(r.GetUncompressedSize());
message->set_data(proto.GetData());
@@ -2166,13 +2166,13 @@ void TPartitionActor::Handle(TEvPersQueue::TEvResponse::TPtr& ev, const TActorCo
<< " EndOffset " << EndOffset << " ReadOffset " << ReadOffset << " ReadGuid " << ReadGuid << " has messages " << hasData);
ReadGuid = TString();
- auto readResponse = MakeHolder<TEvPQProxy::TEvReadResponse>(
- std::move(response),
- ReadOffset,
- res.GetBlobsFromDisk() > 0,
- TDuration::MilliSeconds(res.GetWaitQuotaTimeMs())
- );
- ctx.Send(ParentId, readResponse.Release());
+ auto readResponse = MakeHolder<TEvPQProxy::TEvReadResponse>(
+ std::move(response),
+ ReadOffset,
+ res.GetBlobsFromDisk() > 0,
+ TDuration::MilliSeconds(res.GetWaitQuotaTimeMs())
+ );
+ ctx.Send(ParentId, readResponse.Release());
CheckRelease(ctx);
PipeGeneration = 0; //reset tries counter - all ok
@@ -2858,12 +2858,12 @@ void TReadInfoActor::Handle(TEvPQProxy::TEvAuthResultOk::TPtr& ev, const TActorC
proto.MutableMetaRequest()->MutableCmdGetReadSessionsInfo()->AddTopic(t.TopicNameConverter->GetClientsideName());
}
- ctx.Register(NMsgBusProxy::CreateActorServerPersQueue(
- ctx.SelfID,
- proto,
- SchemeCache,
- std::make_shared<NMsgBusProxy::TPersQueueGetReadSessionsInfoWorkerFactory>()
- ));
+ ctx.Register(NMsgBusProxy::CreateActorServerPersQueue(
+ ctx.SelfID,
+ proto,
+ SchemeCache,
+ std::make_shared<NMsgBusProxy::TPersQueueGetReadSessionsInfoWorkerFactory>()
+ ));
}
diff --git a/ydb/services/persqueue_v1/grpc_pq_schema.cpp b/ydb/services/persqueue_v1/grpc_pq_schema.cpp
index f4b340de25..7756adc797 100644
--- a/ydb/services/persqueue_v1/grpc_pq_schema.cpp
+++ b/ydb/services/persqueue_v1/grpc_pq_schema.cpp
@@ -18,7 +18,7 @@ using grpc::Status;
namespace NKikimr::NGRpcProxy::V1 {
constexpr TStringBuf GRPCS_ENDPOINT_PREFIX = "grpcs://";
-
+
///////////////////////////////////////////////////////////////////////////////
using namespace PersQueue::V1;
@@ -122,13 +122,13 @@ void TDescribeTopicActor::StateWork(TAutoPtr<IEventHandle>& ev, const TActorCont
void TDescribeTopicActor::HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) {
- Y_VERIFY(ev->Get()->Request.Get()->ResultSet.size() == 1); // describe for only one topic
+ Y_VERIFY(ev->Get()->Request.Get()->ResultSet.size() == 1); // describe for only one topic
if (ReplyIfNotTopic(ev, ctx)) {
return;
}
- const auto& response = ev->Get()->Request.Get()->ResultSet.front();
-
+ const auto& response = ev->Get()->Request.Get()->ResultSet.front();
+
const TString path = JoinSeq("/", response.Path);
Ydb::PersQueue::V1::DescribeTopicResult result;
@@ -184,7 +184,7 @@ void TDescribeTopicActor::HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEv
settings->add_supported_codecs((Ydb::PersQueue::V1::Codec) (codec + 1));
}
- const auto& pqConfig = AppData(ctx)->PQConfig;
+ const auto& pqConfig = AppData(ctx)->PQConfig;
for (ui32 i = 0; i < config.ReadRulesSize(); ++i) {
auto rr = settings->add_read_rules();
auto consumerName = NPersQueue::ConvertOldConsumerName(config.GetReadRules(i), ctx);
@@ -204,20 +204,20 @@ void TDescribeTopicActor::HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEv
}
}
rr->set_important(important);
-
- if (i < config.ReadRuleServiceTypesSize()) {
- rr->set_service_type(config.GetReadRuleServiceTypes(i));
- } else {
- if (pqConfig.GetDisallowDefaultClientServiceType()) {
- this->Request_->RaiseIssue(FillIssue(
- "service type must be set for all read rules",
- Ydb::PersQueue::ErrorCode::ERROR
- ));
- Reply(Ydb::StatusIds::INTERNAL_ERROR, ctx);
- return;
- }
- rr->set_service_type(pqConfig.GetDefaultClientServiceType().GetName());
- }
+
+ if (i < config.ReadRuleServiceTypesSize()) {
+ rr->set_service_type(config.GetReadRuleServiceTypes(i));
+ } else {
+ if (pqConfig.GetDisallowDefaultClientServiceType()) {
+ this->Request_->RaiseIssue(FillIssue(
+ "service type must be set for all read rules",
+ Ydb::PersQueue::ErrorCode::ERROR
+ ));
+ Reply(Ydb::StatusIds::INTERNAL_ERROR, ctx);
+ return;
+ }
+ rr->set_service_type(pqConfig.GetDefaultClientServiceType().GetName());
+ }
}
if (partConfig.HasMirrorFrom()) {
auto rmr = settings->mutable_remote_mirror_rule();
@@ -250,7 +250,7 @@ void TDescribeTopicActor::HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEv
NPersQueue::ObfuscateString(
partConfig.GetMirrorFrom().GetCredentials().GetIam().GetServiceAccountKey())
);
- }
+ }
}
rmr->set_database(partConfig.GetMirrorFrom().GetDatabase());
}
@@ -279,19 +279,19 @@ void TAddReadRuleActor::Bootstrap(const NActors::TActorContext& ctx) {
Become(&TBase::StateWork);
}
-void TAddReadRuleActor::ModifyPersqueueConfig(
- const TActorContext& ctx,
+void TAddReadRuleActor::ModifyPersqueueConfig(
+ const TActorContext& ctx,
NKikimrSchemeOp::TPersQueueGroupDescription& groupConfig,
const NKikimrSchemeOp::TPersQueueGroupDescription& pqGroupDescription,
const NKikimrSchemeOp::TDirEntry& selfInfo
-) {
- Y_UNUSED(pqGroupDescription);
-
+) {
+ Y_UNUSED(pqGroupDescription);
+
auto* pqConfig = groupConfig.MutablePQTabletConfig();
auto rule = GetProtoRequest()->read_rule();
if (rule.version() == 0) {
- rule.set_version(selfInfo.GetVersion().GetPQVersion());
+ rule.set_version(selfInfo.GetVersion().GetPQVersion());
}
auto serviceTypes = GetSupportedClientServiceTypes(ctx);
TString error = AddReadRuleToConfig(pqConfig, rule, serviceTypes, ctx);
@@ -318,20 +318,20 @@ void TRemoveReadRuleActor::Bootstrap(const NActors::TActorContext& ctx) {
Become(&TBase::StateWork);
}
-void TRemoveReadRuleActor::ModifyPersqueueConfig(
- const TActorContext& ctx,
+void TRemoveReadRuleActor::ModifyPersqueueConfig(
+ const TActorContext& ctx,
NKikimrSchemeOp::TPersQueueGroupDescription& groupConfig,
const NKikimrSchemeOp::TPersQueueGroupDescription& pqGroupDescription,
const NKikimrSchemeOp::TDirEntry& selfInfo
-) {
- Y_UNUSED(selfInfo);
-
- auto error = RemoveReadRuleFromConfig(
- groupConfig.MutablePQTabletConfig(),
- pqGroupDescription.GetPQTabletConfig(),
- GetProtoRequest()->consumer_name(),
- ctx
- );
+) {
+ Y_UNUSED(selfInfo);
+
+ auto error = RemoveReadRuleFromConfig(
+ groupConfig.MutablePQTabletConfig(),
+ pqGroupDescription.GetPQTabletConfig(),
+ GetProtoRequest()->consumer_name(),
+ ctx
+ );
if (!error.Empty()) {
return ReplyWithError(Ydb::StatusIds::NOT_FOUND, Ydb::PersQueue::ErrorCode::BAD_REQUEST, error, ctx);
}
@@ -372,14 +372,14 @@ void TCreateTopicActor::FillProposeRequest(TEvTxUserProxy::TEvProposeTransaction
NKikimrSchemeOp::TModifyScheme& modifyScheme(*proposal.Record.MutableTransaction()->MutableModifyScheme());
modifyScheme.SetWorkingDir(workingDir);
- {
- TString error;
- auto status = FillProposeRequestImpl(name, GetProtoRequest()->settings(), modifyScheme, ctx, false, error);
- if (!error.empty()) {
- Request_->RaiseIssue(FillIssue(error, PersQueue::ErrorCode::BAD_REQUEST));
- return ReplyWithResult(status, ctx);
- }
- }
+ {
+ TString error;
+ auto status = FillProposeRequestImpl(name, GetProtoRequest()->settings(), modifyScheme, ctx, false, error);
+ if (!error.empty()) {
+ Request_->RaiseIssue(FillIssue(error, PersQueue::ErrorCode::BAD_REQUEST));
+ return ReplyWithResult(status, ctx);
+ }
+ }
const auto& pqDescr = modifyScheme.GetCreatePersQueueGroup();
const auto& config = pqDescr.GetPQTabletConfig();
@@ -389,8 +389,8 @@ void TCreateTopicActor::FillProposeRequest(TEvTxUserProxy::TEvProposeTransaction
<< "' instead of " << LocalCluster, PersQueue::ErrorCode::BAD_REQUEST));
return ReplyWithResult(Ydb::StatusIds::BAD_REQUEST, ctx);
}
- if (Count(Clusters, config.GetDC()) == 0 && !Clusters.empty()) {
- Request_->RaiseIssue(FillIssue(TStringBuilder() << "Unknown cluster '" << config.GetDC() << "'", PersQueue::ErrorCode::BAD_REQUEST));
+ if (Count(Clusters, config.GetDC()) == 0 && !Clusters.empty()) {
+ Request_->RaiseIssue(FillIssue(TStringBuilder() << "Unknown cluster '" << config.GetDC() << "'", PersQueue::ErrorCode::BAD_REQUEST));
return ReplyWithResult(Ydb::StatusIds::BAD_REQUEST, ctx);
}
}
diff --git a/ydb/services/persqueue_v1/grpc_pq_write_actor.cpp b/ydb/services/persqueue_v1/grpc_pq_write_actor.cpp
index 3a91a67af5..3038cc82a6 100644
--- a/ydb/services/persqueue_v1/grpc_pq_write_actor.cpp
+++ b/ydb/services/persqueue_v1/grpc_pq_write_actor.cpp
@@ -1,6 +1,6 @@
#include "grpc_pq_actor.h"
#include "grpc_pq_write.h"
-#include "grpc_pq_codecs.h"
+#include "grpc_pq_codecs.h"
#include <ydb/core/persqueue/pq_database.h>
#include <ydb/core/persqueue/write_meta.h>
@@ -19,8 +19,8 @@
using namespace NActors;
using namespace NKikimrClient;
-
-
+
+
namespace NKikimr {
using namespace NSchemeCache;
@@ -35,44 +35,44 @@ Ydb::PersQueue::V1::Codec CodecByName(const TString& codec) {
return codecIt != codecsByName.end() ? codecIt->second : Ydb::PersQueue::V1::CODEC_UNSPECIFIED;
}
-template <>
-void FillExtraFieldsForDataChunk(
- const Ydb::PersQueue::V1::StreamingWriteClientMessage::InitRequest& init,
- NKikimrPQClient::TDataChunk& data,
- TString& server,
- TString& ident,
- TString& logType,
- TString& file
-) {
- for (const auto& item : init.session_meta()) {
- if (item.first == "server") {
- server = item.second;
- } else if (item.first == "ident") {
- ident = item.second;
- } else if (item.first == "logtype") {
- logType = item.second;
- } else if (item.first == "file") {
- file = item.second;
- } else {
- auto res = data.MutableExtraFields()->AddItems();
- res->SetKey(item.first);
- res->SetValue(item.second);
- }
- }
-}
-
-template <>
-void FillChunkDataFromReq(
- NKikimrPQClient::TDataChunk& proto,
- const Ydb::PersQueue::V1::StreamingWriteClientMessage::WriteRequest& writeRequest,
- const i32 messageIndex
-) {
- proto.SetSeqNo(writeRequest.sequence_numbers(messageIndex));
- proto.SetCreateTime(writeRequest.created_at_ms(messageIndex));
- proto.SetCodec(writeRequest.blocks_headers(messageIndex).front());
- proto.SetData(writeRequest.blocks_data(messageIndex));
-}
-
+template <>
+void FillExtraFieldsForDataChunk(
+ const Ydb::PersQueue::V1::StreamingWriteClientMessage::InitRequest& init,
+ NKikimrPQClient::TDataChunk& data,
+ TString& server,
+ TString& ident,
+ TString& logType,
+ TString& file
+) {
+ for (const auto& item : init.session_meta()) {
+ if (item.first == "server") {
+ server = item.second;
+ } else if (item.first == "ident") {
+ ident = item.second;
+ } else if (item.first == "logtype") {
+ logType = item.second;
+ } else if (item.first == "file") {
+ file = item.second;
+ } else {
+ auto res = data.MutableExtraFields()->AddItems();
+ res->SetKey(item.first);
+ res->SetValue(item.second);
+ }
+ }
+}
+
+template <>
+void FillChunkDataFromReq(
+ NKikimrPQClient::TDataChunk& proto,
+ const Ydb::PersQueue::V1::StreamingWriteClientMessage::WriteRequest& writeRequest,
+ const i32 messageIndex
+) {
+ proto.SetSeqNo(writeRequest.sequence_numbers(messageIndex));
+ proto.SetCreateTime(writeRequest.created_at_ms(messageIndex));
+ proto.SetCodec(writeRequest.blocks_headers(messageIndex).front());
+ proto.SetData(writeRequest.blocks_data(messageIndex));
+}
+
namespace NGRpcProxy {
namespace V1 {
@@ -316,14 +316,14 @@ void TWriteSessionActor::Handle(TEvPQProxy::TEvWriteInit::TPtr& ev, const TActor
PeerName = event->PeerName;
SourceId = init.message_group_id();
- TString encodedSourceId;
- try {
+ TString encodedSourceId;
+ try {
encodedSourceId = NPQ::NSourceIdEncoding::Encode(SourceId);
- } catch (yexception& e) {
- CloseSession(TStringBuilder() << "incorrect sourceId \"" << SourceId << "\": " << e.what(), PersQueue::ErrorCode::BAD_REQUEST, ctx);
- return;
+ } catch (yexception& e) {
+ CloseSession(TStringBuilder() << "incorrect sourceId \"" << SourceId << "\": " << e.what(), PersQueue::ErrorCode::BAD_REQUEST, ctx);
+ return;
}
- EscapedSourceId = HexEncode(encodedSourceId);
+ EscapedSourceId = HexEncode(encodedSourceId);
TString s = TopicConverter->GetClientsideName() + encodedSourceId;
Hash = MurmurHash<ui32>(s.c_str(), s.size(), MURMUR_ARRAY_SEED);
@@ -950,7 +950,7 @@ void TWriteSessionActor::GenerateNextWriteRequest(const TActorContext& ctx) {
i64 diff = 0;
auto addData = [&](const StreamingWriteClientMessage::WriteRequest& writeRequest, const i32 messageIndex) {
auto w = request.MutablePartitionRequest()->AddCmdWrite();
- w->SetData(GetSerializedData(InitMeta, writeRequest, messageIndex));
+ w->SetData(GetSerializedData(InitMeta, writeRequest, messageIndex));
w->SetSeqNo(writeRequest.sequence_numbers(messageIndex));
w->SetSourceId(NPQ::NSourceIdEncoding::EncodeSimple(SourceId));
w->SetCreateTimeMS(writeRequest.created_at_ms(messageIndex));
diff --git a/ydb/services/persqueue_v1/persqueue_common_ut.cpp b/ydb/services/persqueue_v1/persqueue_common_ut.cpp
index d80eb1672d..9d706d0656 100644
--- a/ydb/services/persqueue_v1/persqueue_common_ut.cpp
+++ b/ydb/services/persqueue_v1/persqueue_common_ut.cpp
@@ -26,7 +26,7 @@
#include <ydb/public/api/protos/persqueue_error_codes_v1.pb.h>
#include <ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/data_plane_helpers.h>
-
+
namespace NKikimr::NPersQueueTests {
using namespace Tests;
diff --git a/ydb/services/persqueue_v1/persqueue_ut.cpp b/ydb/services/persqueue_v1/persqueue_ut.cpp
index 6176287f88..3bc9b21728 100644
--- a/ydb/services/persqueue_v1/persqueue_ut.cpp
+++ b/ydb/services/persqueue_v1/persqueue_ut.cpp
@@ -14,12 +14,12 @@
#include <ydb/library/aclib/aclib.h>
#include <ydb/library/persqueue/obfuscate/obfuscate.h>
-#include <ydb/library/persqueue/tests/counters.h>
+#include <ydb/library/persqueue/tests/counters.h>
#include <ydb/library/persqueue/topic_parser/topic_parser.h>
#include <library/cpp/testing/unittest/tests_data.h>
#include <library/cpp/testing/unittest/registar.h>
-#include <library/cpp/json/json_reader.h>
+#include <library/cpp/json/json_reader.h>
#include <library/cpp/monlib/dynamic_counters/encode.h>
#include <google/protobuf/text_format.h>
#include <google/protobuf/util/message_differencer.h>
@@ -433,9 +433,9 @@ namespace {
server.AnnoyingClient->CreateTopic("rt3.dc1--topic", 1);
auto driver = server.AnnoyingClient->GetDriver();
-
+
auto writer = CreateSimpleWriter(*driver, "topic", "test source ID");
-
+
bool res = true;
ui32 messageCount = 1000;
@@ -668,7 +668,7 @@ namespace {
server.EnableLogs({ NKikimrServices::PERSQUEUE });
- // empty data and sourceId
+ // empty data and sourceId
server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 1, "", 1, "", "", ETransport::MsgBus, NMsgBusProxy::MSTATUS_ERROR);
server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 1, "a", 1, "", "", ETransport::MsgBus, NMsgBusProxy::MSTATUS_ERROR);
server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 1, "", 1, "a", "", ETransport::MsgBus, NMsgBusProxy::MSTATUS_ERROR);
@@ -1623,7 +1623,7 @@ namespace {
auto pqLib = TPQLib::WithCerrLogger();
// Write nonempty data
- NKikimr::NPersQueueTests::TRequestWritePQ writeReq(DEFAULT_TOPIC_NAME, 0, "src", 4);
+ NKikimr::NPersQueueTests::TRequestWritePQ writeReq(DEFAULT_TOPIC_NAME, 0, "src", 4);
auto write = [&](const TString& data, bool empty = false) {
NKikimrPQClient::TDataChunk dataChunk;
@@ -1643,10 +1643,10 @@ namespace {
write("data3");
}
- ui32 maxCount = 1;
- bool unpack = false;
- ui32 maxInflyRequests = 1;
- ui32 maxMemoryUsage = 1;
+ ui32 maxCount = 1;
+ bool unpack = false;
+ ui32 maxInflyRequests = 1;
+ ui32 maxMemoryUsage = 1;
auto [consumer, ccResult] = CreateConsumer(
pqLib, server.GrpcPort, "user", {SHORT_TOPIC_NAME, {}},
maxCount, unpack, {}, maxInflyRequests, maxMemoryUsage
@@ -1707,10 +1707,10 @@ namespace {
auto f = producer->Write(1, TString(1024, 'a'));
f.Wait();
- ui32 maxCount = 1;
- bool unpack = false;
+ ui32 maxCount = 1;
+ bool unpack = false;
auto [consumer, ccResult] = CreateConsumer(pqLib, server.GrpcPort, "user", {SHORT_TOPIC_NAME, {}}, maxCount, unpack);
- Cerr << ccResult.Response << "\n";
+ Cerr << ccResult.Response << "\n";
auto msg = consumer->GetNextMessage();
msg.Wait();
@@ -1743,7 +1743,7 @@ namespace {
{
auto [producer, pcResult] = CreateProducer(pqLib, server.GrpcPort, "aaa/bbb/ccc/topic", "123");
- UNIT_ASSERT_C(pcResult.Response.server_message_case() == StreamingWriteServerMessage::kInitResponse, pcResult.Response);
+ UNIT_ASSERT_C(pcResult.Response.server_message_case() == StreamingWriteServerMessage::kInitResponse, pcResult.Response);
for (ui32 i = 1; i <= 11; ++i) {
auto f = producer->Write(i, TString(10, 'a'));
f.Wait();
@@ -1751,8 +1751,8 @@ namespace {
}
}
- ui32 maxCount = 1;
- bool unpack = false;
+ ui32 maxCount = 1;
+ bool unpack = false;
auto [consumer, ccResult] = CreateConsumer(pqLib, server.GrpcPort, "user", {"aaa/bbb/ccc/topic", {}}, maxCount, unpack);
UNIT_ASSERT_C(ccResult.Response.response_case() == MigrationStreamingReadServerMessage::kInitResponse, ccResult.Response);
@@ -1773,11 +1773,11 @@ namespace {
TPQDataWriter writer("source", server);
for (ui32 i = 1; i <= 3; ++i) {
- TString sourceId = "123" + ToString<int>(i);
- ui32 partitionGroup = i;
+ TString sourceId = "123" + ToString<int>(i);
+ ui32 partitionGroup = i;
auto [producer, pcResult] = CreateProducer(pqLib, server.GrpcPort, SHORT_TOPIC_NAME, sourceId, partitionGroup);
- UNIT_ASSERT(pcResult.Response.server_message_case() == StreamingWriteServerMessage::kInitResponse);
+ UNIT_ASSERT(pcResult.Response.server_message_case() == StreamingWriteServerMessage::kInitResponse);
auto f = producer->Write(i, TString(10, 'a'));
f.Wait();
}
@@ -1791,7 +1791,7 @@ namespace {
ss.Unpack = false;
auto [consumer, ccResult] = CreateConsumer(pqLib, ss);
- Cerr << ccResult.Response << "\n";
+ Cerr << ccResult.Response << "\n";
for (ui32 i = 1; i <= 3; ++i) {
auto msg = consumer->GetNextMessage();
@@ -1803,7 +1803,7 @@ namespace {
}
auto [consumer2, ccResult2] = CreateConsumer(pqLib, ss);
- Cerr << ccResult2.Response << "\n";
+ Cerr << ccResult2.Response << "\n";
auto msg = consumer->GetNextMessage();
auto msg2 = consumer2->GetNextMessage();
@@ -1838,12 +1838,12 @@ namespace {
TVector<std::pair<ui64, ui64>> cookies;
for (ui32 i = 1; i <= 3; ++i) {
- TString sourceId = "123" + ToString<int>(i);
- ui32 partitionGroup = i;
+ TString sourceId = "123" + ToString<int>(i);
+ ui32 partitionGroup = i;
auto [producer, pcResult] = CreateProducer(pqLib, server.GrpcPort, SHORT_TOPIC_NAME, sourceId, partitionGroup);
Cerr << "===Response: " << pcResult.Response << Endl;
- UNIT_ASSERT(pcResult.Response.server_message_case() == StreamingWriteServerMessage::kInitResponse);
+ UNIT_ASSERT(pcResult.Response.server_message_case() == StreamingWriteServerMessage::kInitResponse);
auto f = producer->Write(i, TString(10, 'a'));
f.Wait();
}
@@ -1857,7 +1857,7 @@ namespace {
ss.Unpack = false;
auto [consumer, ccResult] = CreateConsumer(pqLib, ss);
- Cerr << ccResult.Response << "\n";
+ Cerr << ccResult.Response << "\n";
for (ui32 i = 1; i <= 3; ++i) {
auto msg = GetNextMessageSkipAssignment(consumer);
@@ -1869,7 +1869,7 @@ namespace {
}
auto [consumer2, ccResult2] = CreateConsumer(pqLib, ss);
- Cerr << ccResult2.Response << "\n";
+ Cerr << ccResult2.Response << "\n";
auto msg = consumer->GetNextMessage();
auto msg2 = consumer2->GetNextMessage();
@@ -1910,7 +1910,7 @@ namespace {
ss.Unpack = false;
auto [consumer, ccResult] = CreateConsumer(pqLib, ss);
- Cerr << ccResult.Response << "\n";
+ Cerr << ccResult.Response << "\n";
auto msg = consumer->GetNextMessage();
msg.Wait();
@@ -1925,7 +1925,7 @@ namespace {
THolder<IConsumer> consumer2;
do {
std::tie(consumer2, ccResult) = CreateConsumer(pqLib, ss);
- Cerr << ccResult.Response << "\n";
+ Cerr << ccResult.Response << "\n";
} while(!msg.Wait(TDuration::Seconds(1)));
Cerr << msg.GetValue().Response << "\n";
@@ -2015,96 +2015,96 @@ namespace {
} while (count < 100);
}
- Y_UNIT_TEST(TestReadQuotasSimple) {
- TRateLimiterTestSetup setup(NKikimrPQ::TPQConfig::TQuotingConfig::USER_PAYLOAD_SIZE, 1000, 1000, true);
+ Y_UNIT_TEST(TestReadQuotasSimple) {
+ TRateLimiterTestSetup setup(NKikimrPQ::TPQConfig::TQuotingConfig::USER_PAYLOAD_SIZE, 1000, 1000, true);
+
+ const TString topicPath = "acc/topic1";
+ const TString consumerPath = "acc2/reader1";
+ setup.CreateTopic(topicPath);
+ setup.CreateConsumer(consumerPath);
+
+ THolder<IProducer> producer = setup.StartProducer(topicPath, true);
- const TString topicPath = "acc/topic1";
- const TString consumerPath = "acc2/reader1";
- setup.CreateTopic(topicPath);
- setup.CreateConsumer(consumerPath);
-
- THolder<IProducer> producer = setup.StartProducer(topicPath, true);
-
auto pqLib = TPQLib::WithCerrLogger();
auto [consumer, ccResult] = CreateConsumer(
pqLib, setup.GetGrpcPort(), consumerPath, {topicPath , {}}, 1000, false
);
- Cerr << ccResult.Response << "\n";
-
- {
- auto msg = consumer->GetNextMessage();
- msg.Wait();
+ Cerr << ccResult.Response << "\n";
+
+ {
+ auto msg = consumer->GetNextMessage();
+ msg.Wait();
Cerr << "consumer assign response: " << msg.GetValue().Response << "\n";
- UNIT_ASSERT(msg.GetValue().Type == EMT_ASSIGNED);
- msg.GetValue().StartRead.SetValue(TAssignInfo());
- }
-
- TVector<NThreading::TFuture<Ydb::PersQueue::TProducerCommitResponse>> writeResults;
- TVector<NThreading::TFuture<Ydb::PersQueue::TConsumerMessage>> readResults;
-
- for (ui32 readBatches = 0; readBatches < 10; ++readBatches) {
- auto msg = consumer->GetNextMessage();
- while (!msg.HasValue()) {
- producer->Write(TString(std::string(10000, 'A')));
- Sleep(TDuration::MilliSeconds(10));
- }
- const auto& response = msg.GetValue().Response;
- Cerr << "next read response: " << response << "\n";
-
- for (auto& data : response.data_batch().partition_data()) {
- for (auto& batch : data.batches()) {
- UNIT_ASSERT(batch.message_data_size() > 0);
- }
- }
- }
- }
-
- Y_UNIT_TEST(TestReadWithQuoterWithoutResources) {
+ UNIT_ASSERT(msg.GetValue().Type == EMT_ASSIGNED);
+ msg.GetValue().StartRead.SetValue(TAssignInfo());
+ }
+
+ TVector<NThreading::TFuture<Ydb::PersQueue::TProducerCommitResponse>> writeResults;
+ TVector<NThreading::TFuture<Ydb::PersQueue::TConsumerMessage>> readResults;
+
+ for (ui32 readBatches = 0; readBatches < 10; ++readBatches) {
+ auto msg = consumer->GetNextMessage();
+ while (!msg.HasValue()) {
+ producer->Write(TString(std::string(10000, 'A')));
+ Sleep(TDuration::MilliSeconds(10));
+ }
+ const auto& response = msg.GetValue().Response;
+ Cerr << "next read response: " << response << "\n";
+
+ for (auto& data : response.data_batch().partition_data()) {
+ for (auto& batch : data.batches()) {
+ UNIT_ASSERT(batch.message_data_size() > 0);
+ }
+ }
+ }
+ }
+
+ Y_UNIT_TEST(TestReadWithQuoterWithoutResources) {
if (NSan::ASanIsOn()) {
return;
}
- TRateLimiterTestSetup setup(NKikimrPQ::TPQConfig::TQuotingConfig::USER_PAYLOAD_SIZE, 1000, 1000, true);
-
- const TString topicPath = "acc/topic1";
- const TString consumerPath = "acc2/reader1"; // don't create kesus resources
- setup.CreateTopic(topicPath);
-
- THolder<IProducer> producer = setup.StartProducer(topicPath, true);
-
- TPQLibSettings pqLibSettings({ .DefaultLogger = new TCerrLogger(DEBUG_LOG_LEVEL) });
- TPQLib PQLib(pqLibSettings);
- auto [consumer, ccResult] = CreateConsumer(PQLib, setup.GetGrpcPort(), consumerPath, {topicPath , {}}, 1000, false);
- Cerr << ccResult.Response << "\n";
-
- {
- auto msg = consumer->GetNextMessage();
- msg.Wait();
- Cerr << "consumer assign response: " << msg.GetValue().Response << "\n";
- UNIT_ASSERT(msg.GetValue().Type == EMT_ASSIGNED);
- msg.GetValue().StartRead.SetValue(TAssignInfo());
- }
-
- TVector<NThreading::TFuture<Ydb::PersQueue::TProducerCommitResponse>> writeResults;
- TVector<NThreading::TFuture<Ydb::PersQueue::TConsumerMessage>> readResults;
-
- for (ui32 readBatches = 0; readBatches < 10; ++readBatches) {
- auto msg = consumer->GetNextMessage();
- while (!msg.HasValue()) {
- producer->Write(TString(std::string(10000, 'A')));
- Sleep(TDuration::MilliSeconds(10));
- }
- const auto& response = msg.GetValue().Response;
- Cerr << "next read response: " << response << "\n";
-
- for (auto& data : response.data_batch().partition_data()) {
- for (auto& batch : data.batches()) {
- UNIT_ASSERT(batch.message_data_size() > 0);
- }
- }
- }
- }
-
+ TRateLimiterTestSetup setup(NKikimrPQ::TPQConfig::TQuotingConfig::USER_PAYLOAD_SIZE, 1000, 1000, true);
+
+ const TString topicPath = "acc/topic1";
+ const TString consumerPath = "acc2/reader1"; // don't create kesus resources
+ setup.CreateTopic(topicPath);
+
+ THolder<IProducer> producer = setup.StartProducer(topicPath, true);
+
+ TPQLibSettings pqLibSettings({ .DefaultLogger = new TCerrLogger(DEBUG_LOG_LEVEL) });
+ TPQLib PQLib(pqLibSettings);
+ auto [consumer, ccResult] = CreateConsumer(PQLib, setup.GetGrpcPort(), consumerPath, {topicPath , {}}, 1000, false);
+ Cerr << ccResult.Response << "\n";
+
+ {
+ auto msg = consumer->GetNextMessage();
+ msg.Wait();
+ Cerr << "consumer assign response: " << msg.GetValue().Response << "\n";
+ UNIT_ASSERT(msg.GetValue().Type == EMT_ASSIGNED);
+ msg.GetValue().StartRead.SetValue(TAssignInfo());
+ }
+
+ TVector<NThreading::TFuture<Ydb::PersQueue::TProducerCommitResponse>> writeResults;
+ TVector<NThreading::TFuture<Ydb::PersQueue::TConsumerMessage>> readResults;
+
+ for (ui32 readBatches = 0; readBatches < 10; ++readBatches) {
+ auto msg = consumer->GetNextMessage();
+ while (!msg.HasValue()) {
+ producer->Write(TString(std::string(10000, 'A')));
+ Sleep(TDuration::MilliSeconds(10));
+ }
+ const auto& response = msg.GetValue().Response;
+ Cerr << "next read response: " << response << "\n";
+
+ for (auto& data : response.data_batch().partition_data()) {
+ for (auto& batch : data.batches()) {
+ UNIT_ASSERT(batch.message_data_size() > 0);
+ }
+ }
+ }
+ }
+
Y_UNIT_TEST(TestDeletionOfTopic) {
if (NSan::ASanIsOn()) {
return;
@@ -2118,8 +2118,8 @@ namespace {
server.AnnoyingClient->DeleteTopic2(DEFAULT_TOPIC_NAME, NPersQueue::NErrorCode::OK, false);
auto pqLib = TPQLib::WithCerrLogger();
- ui32 maxCount = 1;
- bool unpack = false;
+ ui32 maxCount = 1;
+ bool unpack = false;
auto [consumer, ccResult] = CreateConsumer(pqLib, server.GrpcPort, "user", {SHORT_TOPIC_NAME, {}}, maxCount, unpack);
Cerr << "Consumer create response: " << ccResult.Response << "\n";
@@ -2375,10 +2375,10 @@ namespace {
auto props = request.mutable_settings();
props->set_partitions_count(1);
props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE);
- props->set_retention_period_ms(TDuration::Days(1).MilliSeconds());
- props->set_max_partition_storage_size(1000);
- props->set_max_partition_write_speed(1000);
- props->set_max_partition_write_burst(1000);
+ props->set_retention_period_ms(TDuration::Days(1).MilliSeconds());
+ props->set_max_partition_storage_size(1000);
+ props->set_max_partition_write_speed(1000);
+ props->set_max_partition_write_burst(1000);
grpc::ClientContext rcontext;
rcontext.AddMetadata("x-ydb-auth-ticket", "user@" BUILTIN_ACL_DOMAIN);
@@ -2405,10 +2405,10 @@ namespace {
auto props = request.mutable_settings();
props->set_partitions_count(2);
props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE);
- props->set_retention_period_ms(TDuration::Days(1).MilliSeconds());
- props->set_max_partition_storage_size(1000);
- props->set_max_partition_write_speed(1000);
- props->set_max_partition_write_burst(1000);
+ props->set_retention_period_ms(TDuration::Days(1).MilliSeconds());
+ props->set_max_partition_storage_size(1000);
+ props->set_max_partition_write_speed(1000);
+ props->set_max_partition_write_burst(1000);
grpc::ClientContext rcontext;
@@ -2444,10 +2444,10 @@ namespace {
auto props = request.mutable_settings();
props->set_partitions_count(1);
props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE);
- props->set_retention_period_ms(TDuration::Days(1).MilliSeconds());
- props->set_max_partition_storage_size(1000);
- props->set_max_partition_write_speed(1000);
- props->set_max_partition_write_burst(1000);
+ props->set_retention_period_ms(TDuration::Days(1).MilliSeconds());
+ props->set_max_partition_storage_size(1000);
+ props->set_max_partition_write_speed(1000);
+ props->set_max_partition_write_burst(1000);
alter(request, Ydb::StatusIds::UNAUTHORIZED, true);
alter(request, Ydb::StatusIds::GENERIC_ERROR, false);
props->set_partitions_count(3);
@@ -2492,7 +2492,7 @@ namespace {
PartitionConfig {
MaxCountInPartition: 2147483647
MaxSizeInPartition: 234
- LifetimeSeconds: 86400
+ LifetimeSeconds: 86400
ImportantClientId: "consumer"
SourceIdLifetimeSeconds: 1382400
WriteSpeedInBytesPerSecond: 123
@@ -2553,7 +2553,7 @@ namespace {
Codecs: "lzop"
Codecs: "gzip"
}
- ReadRuleServiceTypes: "data-transfer"
+ ReadRuleServiceTypes: "data-transfer"
FormatVersion: 0
Codecs {
Ids: 0
@@ -2582,7 +2582,7 @@ namespace {
DescribeTopicResult res;
response.operation().result().UnpackTo(&res);
Cerr << response << "\n" << res << "\n";
- UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::SCHEME_ERROR); // muts be Ydb::StatusIds::UNAUTHORIZED);
+ UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::SCHEME_ERROR); // muts be Ydb::StatusIds::UNAUTHORIZED);
}
{
@@ -2684,226 +2684,226 @@ namespace {
}
-
- Y_UNIT_TEST(SchemeOperationsCheckPropValues) {
+
+ Y_UNIT_TEST(SchemeOperationsCheckPropValues) {
NPersQueue::TTestServer server;
server.EnableLogs({ NKikimrServices::PQ_READ_PROXY, NKikimrServices::BLACKBOX_VALIDATOR });
-
+
server.AnnoyingClient->CreateTopic("rt3.dc1--acc--topic1", 1);
server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1);
server.AnnoyingClient->CreateConsumer("user");
-
- std::shared_ptr<grpc::Channel> Channel_;
- std::unique_ptr<Ydb::PersQueue::V1::PersQueueService::Stub> StubP_;
-
- {
+
+ std::shared_ptr<grpc::Channel> Channel_;
+ std::unique_ptr<Ydb::PersQueue::V1::PersQueueService::Stub> StubP_;
+
+ {
Channel_ = grpc::CreateChannel("localhost:" + ToString(server.GrpcPort), grpc::InsecureChannelCredentials());
- StubP_ = Ydb::PersQueue::V1::PersQueueService::NewStub(Channel_);
- }
-
- {
- // zero value is forbidden for: partitions_count
- CreateTopicRequest request;
- CreateTopicResponse response;
- request.set_path("/Root/PQ/rt3.dc1--acc--topic1");
- auto props = request.mutable_settings();
- props->set_partitions_count(0);
- props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE);
- props->set_retention_period_ms(TDuration::Days(1).MilliSeconds());
-
- grpc::ClientContext rcontext;
- auto status = StubP_->CreateTopic(&rcontext, request, &response);
-
- UNIT_ASSERT(status.ok());
- CreateTopicResult res;
- response.operation().result().UnpackTo(&res);
- Cerr << response << "\n" << res << "\n";
- UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::BAD_REQUEST);
- }
- {
- // zero value is forbidden for: retention_period_ms
- CreateTopicRequest request;
- CreateTopicResponse response;
- request.set_path("/Root/PQ/rt3.dc1--acc--topic1");
- auto props = request.mutable_settings();
- props->set_partitions_count(1);
- props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE);
- props->set_retention_period_ms(0);
-
- grpc::ClientContext rcontext;
- auto status = StubP_->CreateTopic(&rcontext, request, &response);
-
- UNIT_ASSERT(status.ok());
- CreateTopicResult res;
- response.operation().result().UnpackTo(&res);
- Cerr << response << "\n" << res << "\n";
- UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::BAD_REQUEST);
- }
- {
- // zero value is allowed for: partition_storage_size, max_partition_write_speed, max_partition_write_burst
- CreateTopicRequest request;
- CreateTopicResponse response;
- request.set_path("/Root/PQ/rt3.dc1--acc--topic1");
- auto props = request.mutable_settings();
- props->set_partitions_count(1);
- props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE);
- props->set_retention_period_ms(TDuration::Days(1).MilliSeconds());
- props->set_max_partition_storage_size(0);
- props->set_max_partition_write_speed(0);
- props->set_max_partition_write_burst(0);
-
- grpc::ClientContext rcontext;
- auto status = StubP_->CreateTopic(&rcontext, request, &response);
-
- UNIT_ASSERT(status.ok());
- CreateTopicResult res;
- response.operation().result().UnpackTo(&res);
- Cerr << response << "\n" << res << "\n";
- UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::SUCCESS);
- }
- }
-
- Y_UNIT_TEST(ReadRuleServiceType) {
+ StubP_ = Ydb::PersQueue::V1::PersQueueService::NewStub(Channel_);
+ }
+
+ {
+ // zero value is forbidden for: partitions_count
+ CreateTopicRequest request;
+ CreateTopicResponse response;
+ request.set_path("/Root/PQ/rt3.dc1--acc--topic1");
+ auto props = request.mutable_settings();
+ props->set_partitions_count(0);
+ props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE);
+ props->set_retention_period_ms(TDuration::Days(1).MilliSeconds());
+
+ grpc::ClientContext rcontext;
+ auto status = StubP_->CreateTopic(&rcontext, request, &response);
+
+ UNIT_ASSERT(status.ok());
+ CreateTopicResult res;
+ response.operation().result().UnpackTo(&res);
+ Cerr << response << "\n" << res << "\n";
+ UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::BAD_REQUEST);
+ }
+ {
+ // zero value is forbidden for: retention_period_ms
+ CreateTopicRequest request;
+ CreateTopicResponse response;
+ request.set_path("/Root/PQ/rt3.dc1--acc--topic1");
+ auto props = request.mutable_settings();
+ props->set_partitions_count(1);
+ props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE);
+ props->set_retention_period_ms(0);
+
+ grpc::ClientContext rcontext;
+ auto status = StubP_->CreateTopic(&rcontext, request, &response);
+
+ UNIT_ASSERT(status.ok());
+ CreateTopicResult res;
+ response.operation().result().UnpackTo(&res);
+ Cerr << response << "\n" << res << "\n";
+ UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::BAD_REQUEST);
+ }
+ {
+ // zero value is allowed for: partition_storage_size, max_partition_write_speed, max_partition_write_burst
+ CreateTopicRequest request;
+ CreateTopicResponse response;
+ request.set_path("/Root/PQ/rt3.dc1--acc--topic1");
+ auto props = request.mutable_settings();
+ props->set_partitions_count(1);
+ props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE);
+ props->set_retention_period_ms(TDuration::Days(1).MilliSeconds());
+ props->set_max_partition_storage_size(0);
+ props->set_max_partition_write_speed(0);
+ props->set_max_partition_write_burst(0);
+
+ grpc::ClientContext rcontext;
+ auto status = StubP_->CreateTopic(&rcontext, request, &response);
+
+ UNIT_ASSERT(status.ok());
+ CreateTopicResult res;
+ response.operation().result().UnpackTo(&res);
+ Cerr << response << "\n" << res << "\n";
+ UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::SUCCESS);
+ }
+ }
+
+ Y_UNIT_TEST(ReadRuleServiceType) {
TServerSettings settings = PQSettings(0);
- {
- settings.PQConfig.AddClientServiceType()->SetName("MyGreatType");
- settings.PQConfig.AddClientServiceType()->SetName("AnotherType");
- settings.PQConfig.AddClientServiceType()->SetName("SecondType");
- }
+ {
+ settings.PQConfig.AddClientServiceType()->SetName("MyGreatType");
+ settings.PQConfig.AddClientServiceType()->SetName("AnotherType");
+ settings.PQConfig.AddClientServiceType()->SetName("SecondType");
+ }
NPersQueue::TTestServer server(settings);
server.EnableLogs({ NKikimrServices::PQ_READ_PROXY, NKikimrServices::BLACKBOX_VALIDATOR });
-
- std::unique_ptr<Ydb::PersQueue::V1::PersQueueService::Stub> pqStub;
-
- {
+
+ std::unique_ptr<Ydb::PersQueue::V1::PersQueueService::Stub> pqStub;
+
+ {
std::shared_ptr<grpc::Channel> channel = grpc::CreateChannel("localhost:" + ToString(server.GrpcPort), grpc::InsecureChannelCredentials());
- pqStub = Ydb::PersQueue::V1::PersQueueService::NewStub(channel);
- }
- auto checkDescribe = [&](const TVector<std::pair<TString, TString>>& readRules) {
- DescribeTopicRequest request;
- DescribeTopicResponse response;
- request.set_path("/Root/PQ/rt3.dc1--acc--some-topic");
- grpc::ClientContext rcontext;
-
- auto status = pqStub->DescribeTopic(&rcontext, request, &response);
- UNIT_ASSERT(status.ok());
- DescribeTopicResult res;
- response.operation().result().UnpackTo(&res);
- Cerr << response << "\n" << res << "\n";
- UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::SUCCESS);
-
- UNIT_ASSERT_VALUES_EQUAL(res.settings().read_rules().size(), readRules.size());
- for (ui64 i = 0; i < readRules.size(); ++i) {
- const auto& rr = res.settings().read_rules(i);
- UNIT_ASSERT_EQUAL(rr.consumer_name(), readRules[i].first);
- UNIT_ASSERT_EQUAL(rr.service_type(), readRules[i].second);
- }
- };
- {
- CreateTopicRequest request;
- CreateTopicResponse response;
- request.set_path("/Root/PQ/rt3.dc1--acc--some-topic");
- auto props = request.mutable_settings();
- props->set_partitions_count(1);
- props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE);
- props->set_retention_period_ms(TDuration::Days(1).MilliSeconds());
- {
- auto rr = props->add_read_rules();
- rr->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1));
- rr->set_consumer_name("acc/consumer1");
- }
- {
- auto rr = props->add_read_rules();
- rr->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1));
- rr->set_consumer_name("acc/consumer2");
- rr->set_service_type("MyGreatType");
- }
-
- grpc::ClientContext rcontext;
- auto status = pqStub->CreateTopic(&rcontext, request, &response);
-
- UNIT_ASSERT(status.ok());
- CreateTopicResult res;
- response.operation().result().UnpackTo(&res);
- Cerr << response << "\n" << res << "\n";
- UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::SUCCESS);
- }
- checkDescribe({
- {"acc/consumer1", "data-transfer"},
- {"acc/consumer2", "MyGreatType"}
- });
- {
- AlterTopicRequest request;
- AlterTopicResponse response;
- request.set_path("/Root/PQ/rt3.dc1--acc--some-topic");
- auto props = request.mutable_settings();
- props->set_partitions_count(1);
- props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE);
- props->set_retention_period_ms(TDuration::Days(1).MilliSeconds());
- {
- auto rr = props->add_read_rules();
- rr->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1));
- rr->set_consumer_name("acc/consumer1");
- }
- {
- auto rr = props->add_read_rules();
- rr->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1));
- rr->set_consumer_name("acc/consumer2");
- rr->set_service_type("AnotherType");
- }
- {
- auto rr = props->add_read_rules();
- rr->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1));
- rr->set_consumer_name("acc/consumer3");
- rr->set_service_type("SecondType");
- }
-
- grpc::ClientContext rcontext;
- auto status = pqStub->AlterTopic(&rcontext, request, &response);
-
- UNIT_ASSERT(status.ok());
- CreateTopicResult res;
- response.operation().result().UnpackTo(&res);
- Cerr << response << "\n" << res << "\n";
- UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::SUCCESS);
- }
- checkDescribe({
- {"acc/consumer1", "data-transfer"},
- {"acc/consumer2", "AnotherType"},
- {"acc/consumer3", "SecondType"}
- });
-
- {
- AlterTopicRequest request;
- AlterTopicResponse response;
- request.set_path("/Root/PQ/rt3.dc1--acc--some-topic");
- auto props = request.mutable_settings();
- props->set_partitions_count(1);
- props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE);
- props->set_retention_period_ms(TDuration::Days(1).MilliSeconds());
- {
- auto rr = props->add_read_rules();
- rr->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1));
- rr->set_consumer_name("acc/consumer1");
- rr->set_service_type("BadServiceType");
- }
-
- grpc::ClientContext rcontext;
- auto status = pqStub->AlterTopic(&rcontext, request, &response);
-
- UNIT_ASSERT(status.ok());
- CreateTopicResult res;
- response.operation().result().UnpackTo(&res);
- Cerr << response << "\n" << res << "\n";
- UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::BAD_REQUEST);
- }
- checkDescribe({
- {"acc/consumer1", "data-transfer"},
- {"acc/consumer2", "AnotherType"},
- {"acc/consumer3", "SecondType"}
- });
- }
-
+ pqStub = Ydb::PersQueue::V1::PersQueueService::NewStub(channel);
+ }
+ auto checkDescribe = [&](const TVector<std::pair<TString, TString>>& readRules) {
+ DescribeTopicRequest request;
+ DescribeTopicResponse response;
+ request.set_path("/Root/PQ/rt3.dc1--acc--some-topic");
+ grpc::ClientContext rcontext;
+
+ auto status = pqStub->DescribeTopic(&rcontext, request, &response);
+ UNIT_ASSERT(status.ok());
+ DescribeTopicResult res;
+ response.operation().result().UnpackTo(&res);
+ Cerr << response << "\n" << res << "\n";
+ UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::SUCCESS);
+
+ UNIT_ASSERT_VALUES_EQUAL(res.settings().read_rules().size(), readRules.size());
+ for (ui64 i = 0; i < readRules.size(); ++i) {
+ const auto& rr = res.settings().read_rules(i);
+ UNIT_ASSERT_EQUAL(rr.consumer_name(), readRules[i].first);
+ UNIT_ASSERT_EQUAL(rr.service_type(), readRules[i].second);
+ }
+ };
+ {
+ CreateTopicRequest request;
+ CreateTopicResponse response;
+ request.set_path("/Root/PQ/rt3.dc1--acc--some-topic");
+ auto props = request.mutable_settings();
+ props->set_partitions_count(1);
+ props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE);
+ props->set_retention_period_ms(TDuration::Days(1).MilliSeconds());
+ {
+ auto rr = props->add_read_rules();
+ rr->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1));
+ rr->set_consumer_name("acc/consumer1");
+ }
+ {
+ auto rr = props->add_read_rules();
+ rr->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1));
+ rr->set_consumer_name("acc/consumer2");
+ rr->set_service_type("MyGreatType");
+ }
+
+ grpc::ClientContext rcontext;
+ auto status = pqStub->CreateTopic(&rcontext, request, &response);
+
+ UNIT_ASSERT(status.ok());
+ CreateTopicResult res;
+ response.operation().result().UnpackTo(&res);
+ Cerr << response << "\n" << res << "\n";
+ UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::SUCCESS);
+ }
+ checkDescribe({
+ {"acc/consumer1", "data-transfer"},
+ {"acc/consumer2", "MyGreatType"}
+ });
+ {
+ AlterTopicRequest request;
+ AlterTopicResponse response;
+ request.set_path("/Root/PQ/rt3.dc1--acc--some-topic");
+ auto props = request.mutable_settings();
+ props->set_partitions_count(1);
+ props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE);
+ props->set_retention_period_ms(TDuration::Days(1).MilliSeconds());
+ {
+ auto rr = props->add_read_rules();
+ rr->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1));
+ rr->set_consumer_name("acc/consumer1");
+ }
+ {
+ auto rr = props->add_read_rules();
+ rr->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1));
+ rr->set_consumer_name("acc/consumer2");
+ rr->set_service_type("AnotherType");
+ }
+ {
+ auto rr = props->add_read_rules();
+ rr->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1));
+ rr->set_consumer_name("acc/consumer3");
+ rr->set_service_type("SecondType");
+ }
+
+ grpc::ClientContext rcontext;
+ auto status = pqStub->AlterTopic(&rcontext, request, &response);
+
+ UNIT_ASSERT(status.ok());
+ CreateTopicResult res;
+ response.operation().result().UnpackTo(&res);
+ Cerr << response << "\n" << res << "\n";
+ UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::SUCCESS);
+ }
+ checkDescribe({
+ {"acc/consumer1", "data-transfer"},
+ {"acc/consumer2", "AnotherType"},
+ {"acc/consumer3", "SecondType"}
+ });
+
+ {
+ AlterTopicRequest request;
+ AlterTopicResponse response;
+ request.set_path("/Root/PQ/rt3.dc1--acc--some-topic");
+ auto props = request.mutable_settings();
+ props->set_partitions_count(1);
+ props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE);
+ props->set_retention_period_ms(TDuration::Days(1).MilliSeconds());
+ {
+ auto rr = props->add_read_rules();
+ rr->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1));
+ rr->set_consumer_name("acc/consumer1");
+ rr->set_service_type("BadServiceType");
+ }
+
+ grpc::ClientContext rcontext;
+ auto status = pqStub->AlterTopic(&rcontext, request, &response);
+
+ UNIT_ASSERT(status.ok());
+ CreateTopicResult res;
+ response.operation().result().UnpackTo(&res);
+ Cerr << response << "\n" << res << "\n";
+ UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::BAD_REQUEST);
+ }
+ checkDescribe({
+ {"acc/consumer1", "data-transfer"},
+ {"acc/consumer2", "AnotherType"},
+ {"acc/consumer3", "SecondType"}
+ });
+ }
+
Y_UNIT_TEST(ReadRuleServiceTypeLimit) {
TServerSettings settings = PQSettings(0);
@@ -3023,514 +3023,514 @@ namespace {
}
- Y_UNIT_TEST(ReadRuleDisallowDefaultServiceType) {
- TServerSettings settings = PQSettings(0);
- {
- settings.PQConfig.AddClientServiceType()->SetName("MyGreatType");
- settings.PQConfig.AddClientServiceType()->SetName("AnotherType");
- settings.PQConfig.AddClientServiceType()->SetName("SecondType");
- settings.PQConfig.SetDisallowDefaultClientServiceType(true);
- }
- NPersQueue::TTestServer server(settings);
- server.EnableLogs({ NKikimrServices::PQ_READ_PROXY, NKikimrServices::BLACKBOX_VALIDATOR });
-
- std::unique_ptr<Ydb::PersQueue::V1::PersQueueService::Stub> pqStub;
-
- {
- std::shared_ptr<grpc::Channel> channel = grpc::CreateChannel("localhost:" + ToString(server.GrpcPort), grpc::InsecureChannelCredentials());
- pqStub = Ydb::PersQueue::V1::PersQueueService::NewStub(channel);
- }
- auto checkDescribe = [&](const TVector<std::pair<TString, TString>>& readRules) {
- DescribeTopicRequest request;
- DescribeTopicResponse response;
- request.set_path("/Root/PQ/rt3.dc1--acc--some-topic");
- grpc::ClientContext rcontext;
-
- auto status = pqStub->DescribeTopic(&rcontext, request, &response);
- UNIT_ASSERT(status.ok());
- DescribeTopicResult res;
- response.operation().result().UnpackTo(&res);
- Cerr << response << "\n" << res << "\n";
- UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::SUCCESS);
-
- UNIT_ASSERT_VALUES_EQUAL(res.settings().read_rules().size(), readRules.size());
- for (ui64 i = 0; i < readRules.size(); ++i) {
- const auto& rr = res.settings().read_rules(i);
- UNIT_ASSERT_EQUAL(rr.consumer_name(), readRules[i].first);
- UNIT_ASSERT_EQUAL(rr.service_type(), readRules[i].second);
- }
- };
- {
- CreateTopicRequest request;
- CreateTopicResponse response;
- request.set_path("/Root/PQ/rt3.dc1--acc--some-topic");
- auto props = request.mutable_settings();
- props->set_partitions_count(1);
- props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE);
- props->set_retention_period_ms(TDuration::Days(1).MilliSeconds());
- {
- auto rr = props->add_read_rules();
- rr->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1));
- rr->set_consumer_name("acc/consumer1");
- }
-
- grpc::ClientContext rcontext;
- auto status = pqStub->CreateTopic(&rcontext, request, &response);
-
- UNIT_ASSERT(status.ok());
- CreateTopicResult res;
- response.operation().result().UnpackTo(&res);
- Cerr << response << "\n" << res << "\n";
- UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::BAD_REQUEST);
- }
- {
- CreateTopicRequest request;
- CreateTopicResponse response;
- request.set_path("/Root/PQ/rt3.dc1--acc--some-topic");
- auto props = request.mutable_settings();
- props->set_partitions_count(1);
- props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE);
- props->set_retention_period_ms(TDuration::Days(1).MilliSeconds());
- {
- auto rr = props->add_read_rules();
- rr->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1));
- rr->set_consumer_name("acc/consumer1");
- rr->set_service_type("MyGreatType");
- }
-
- grpc::ClientContext rcontext;
- auto status = pqStub->CreateTopic(&rcontext, request, &response);
-
- UNIT_ASSERT(status.ok());
- CreateTopicResult res;
- response.operation().result().UnpackTo(&res);
- Cerr << response << "\n" << res << "\n";
- UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::SUCCESS);
- }
- checkDescribe({{"acc/consumer1", "MyGreatType"}});
- {
- AlterTopicRequest request;
- AlterTopicResponse response;
- request.set_path("/Root/PQ/rt3.dc1--acc--some-topic");
- auto props = request.mutable_settings();
- props->set_partitions_count(1);
- props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE);
- props->set_retention_period_ms(TDuration::Days(1).MilliSeconds());
- {
- auto rr = props->add_read_rules();
- rr->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1));
- rr->set_consumer_name("acc/consumer1");
- }
-
- grpc::ClientContext rcontext;
- auto status = pqStub->AlterTopic(&rcontext, request, &response);
-
- UNIT_ASSERT(status.ok());
- CreateTopicResult res;
- response.operation().result().UnpackTo(&res);
- Cerr << response << "\n" << res << "\n";
- UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::BAD_REQUEST);
- }
- checkDescribe({{"acc/consumer1", "MyGreatType"}});
-
- {
- AlterTopicRequest request;
- AlterTopicResponse response;
- request.set_path("/Root/PQ/rt3.dc1--acc--some-topic");
- auto props = request.mutable_settings();
- props->set_partitions_count(1);
- props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE);
- props->set_retention_period_ms(TDuration::Days(1).MilliSeconds());
- {
- auto rr = props->add_read_rules();
- rr->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1));
- rr->set_consumer_name("acc/consumer1");
- rr->set_service_type("AnotherType");
- }
- {
- auto rr = props->add_read_rules();
- rr->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1));
- rr->set_consumer_name("acc/consumer2");
- }
-
- grpc::ClientContext rcontext;
- auto status = pqStub->AlterTopic(&rcontext, request, &response);
-
- UNIT_ASSERT(status.ok());
- CreateTopicResult res;
- response.operation().result().UnpackTo(&res);
- Cerr << response << "\n" << res << "\n";
- UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::BAD_REQUEST);
- }
- checkDescribe({{"acc/consumer1", "MyGreatType"}});
- }
-
- Y_UNIT_TEST(ReadRuleServiceTypeMigration) {
+ Y_UNIT_TEST(ReadRuleDisallowDefaultServiceType) {
+ TServerSettings settings = PQSettings(0);
+ {
+ settings.PQConfig.AddClientServiceType()->SetName("MyGreatType");
+ settings.PQConfig.AddClientServiceType()->SetName("AnotherType");
+ settings.PQConfig.AddClientServiceType()->SetName("SecondType");
+ settings.PQConfig.SetDisallowDefaultClientServiceType(true);
+ }
+ NPersQueue::TTestServer server(settings);
+ server.EnableLogs({ NKikimrServices::PQ_READ_PROXY, NKikimrServices::BLACKBOX_VALIDATOR });
+
+ std::unique_ptr<Ydb::PersQueue::V1::PersQueueService::Stub> pqStub;
+
+ {
+ std::shared_ptr<grpc::Channel> channel = grpc::CreateChannel("localhost:" + ToString(server.GrpcPort), grpc::InsecureChannelCredentials());
+ pqStub = Ydb::PersQueue::V1::PersQueueService::NewStub(channel);
+ }
+ auto checkDescribe = [&](const TVector<std::pair<TString, TString>>& readRules) {
+ DescribeTopicRequest request;
+ DescribeTopicResponse response;
+ request.set_path("/Root/PQ/rt3.dc1--acc--some-topic");
+ grpc::ClientContext rcontext;
+
+ auto status = pqStub->DescribeTopic(&rcontext, request, &response);
+ UNIT_ASSERT(status.ok());
+ DescribeTopicResult res;
+ response.operation().result().UnpackTo(&res);
+ Cerr << response << "\n" << res << "\n";
+ UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::SUCCESS);
+
+ UNIT_ASSERT_VALUES_EQUAL(res.settings().read_rules().size(), readRules.size());
+ for (ui64 i = 0; i < readRules.size(); ++i) {
+ const auto& rr = res.settings().read_rules(i);
+ UNIT_ASSERT_EQUAL(rr.consumer_name(), readRules[i].first);
+ UNIT_ASSERT_EQUAL(rr.service_type(), readRules[i].second);
+ }
+ };
+ {
+ CreateTopicRequest request;
+ CreateTopicResponse response;
+ request.set_path("/Root/PQ/rt3.dc1--acc--some-topic");
+ auto props = request.mutable_settings();
+ props->set_partitions_count(1);
+ props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE);
+ props->set_retention_period_ms(TDuration::Days(1).MilliSeconds());
+ {
+ auto rr = props->add_read_rules();
+ rr->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1));
+ rr->set_consumer_name("acc/consumer1");
+ }
+
+ grpc::ClientContext rcontext;
+ auto status = pqStub->CreateTopic(&rcontext, request, &response);
+
+ UNIT_ASSERT(status.ok());
+ CreateTopicResult res;
+ response.operation().result().UnpackTo(&res);
+ Cerr << response << "\n" << res << "\n";
+ UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::BAD_REQUEST);
+ }
+ {
+ CreateTopicRequest request;
+ CreateTopicResponse response;
+ request.set_path("/Root/PQ/rt3.dc1--acc--some-topic");
+ auto props = request.mutable_settings();
+ props->set_partitions_count(1);
+ props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE);
+ props->set_retention_period_ms(TDuration::Days(1).MilliSeconds());
+ {
+ auto rr = props->add_read_rules();
+ rr->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1));
+ rr->set_consumer_name("acc/consumer1");
+ rr->set_service_type("MyGreatType");
+ }
+
+ grpc::ClientContext rcontext;
+ auto status = pqStub->CreateTopic(&rcontext, request, &response);
+
+ UNIT_ASSERT(status.ok());
+ CreateTopicResult res;
+ response.operation().result().UnpackTo(&res);
+ Cerr << response << "\n" << res << "\n";
+ UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::SUCCESS);
+ }
+ checkDescribe({{"acc/consumer1", "MyGreatType"}});
+ {
+ AlterTopicRequest request;
+ AlterTopicResponse response;
+ request.set_path("/Root/PQ/rt3.dc1--acc--some-topic");
+ auto props = request.mutable_settings();
+ props->set_partitions_count(1);
+ props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE);
+ props->set_retention_period_ms(TDuration::Days(1).MilliSeconds());
+ {
+ auto rr = props->add_read_rules();
+ rr->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1));
+ rr->set_consumer_name("acc/consumer1");
+ }
+
+ grpc::ClientContext rcontext;
+ auto status = pqStub->AlterTopic(&rcontext, request, &response);
+
+ UNIT_ASSERT(status.ok());
+ CreateTopicResult res;
+ response.operation().result().UnpackTo(&res);
+ Cerr << response << "\n" << res << "\n";
+ UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::BAD_REQUEST);
+ }
+ checkDescribe({{"acc/consumer1", "MyGreatType"}});
+
+ {
+ AlterTopicRequest request;
+ AlterTopicResponse response;
+ request.set_path("/Root/PQ/rt3.dc1--acc--some-topic");
+ auto props = request.mutable_settings();
+ props->set_partitions_count(1);
+ props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE);
+ props->set_retention_period_ms(TDuration::Days(1).MilliSeconds());
+ {
+ auto rr = props->add_read_rules();
+ rr->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1));
+ rr->set_consumer_name("acc/consumer1");
+ rr->set_service_type("AnotherType");
+ }
+ {
+ auto rr = props->add_read_rules();
+ rr->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1));
+ rr->set_consumer_name("acc/consumer2");
+ }
+
+ grpc::ClientContext rcontext;
+ auto status = pqStub->AlterTopic(&rcontext, request, &response);
+
+ UNIT_ASSERT(status.ok());
+ CreateTopicResult res;
+ response.operation().result().UnpackTo(&res);
+ Cerr << response << "\n" << res << "\n";
+ UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::BAD_REQUEST);
+ }
+ checkDescribe({{"acc/consumer1", "MyGreatType"}});
+ }
+
+ Y_UNIT_TEST(ReadRuleServiceTypeMigration) {
+ TServerSettings settings = PQSettings(0);
+ {
+ settings.PQConfig.MutableDefaultClientServiceType()->SetName("default_type");
+ settings.PQConfig.AddClientServiceType()->SetName("MyGreatType");
+ settings.PQConfig.AddClientServiceType()->SetName("AnotherType");
+ settings.PQConfig.AddClientServiceType()->SetName("SecondType");
+ }
+ NPersQueue::TTestServer server(settings);
+
+ server.EnableLogs({ NKikimrServices::PQ_READ_PROXY, NKikimrServices::BLACKBOX_VALIDATOR });
+
+ const ui32 topicsCount = 4;
+ for (ui32 i = 1; i <= topicsCount; ++i) {
+ TRequestCreatePQ createTopicRequest(TStringBuilder() << "rt3.dc1--topic_" << i, 1);
+ createTopicRequest.ReadRules.push_back("acc@user1");
+ createTopicRequest.ReadRules.push_back("acc@user2");
+ createTopicRequest.ReadRules.push_back("acc@user3");
+ server.AnnoyingClient->CreateTopic(createTopicRequest);
+ }
+
+ std::unique_ptr<Ydb::PersQueue::V1::PersQueueService::Stub> pqStub;
+ {
+ std::shared_ptr<grpc::Channel> channel = grpc::CreateChannel("localhost:" + ToString(server.GrpcPort), grpc::InsecureChannelCredentials());
+ pqStub = Ydb::PersQueue::V1::PersQueueService::NewStub(channel);
+ }
+ auto doAlter = [&](const TString& topic, const TVector<std::pair<TString, TString>>& readRules) {
+ AlterTopicRequest request;
+ AlterTopicResponse response;
+ request.set_path(topic);
+ auto props = request.mutable_settings();
+ props->set_partitions_count(1);
+ props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE);
+ props->set_retention_period_ms(TDuration::Days(1).MilliSeconds());
+ for (auto rrInfo : readRules) {
+ auto rr = props->add_read_rules();
+ rr->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1));
+ rr->set_consumer_name(rrInfo.first);
+ rr->set_service_type(rrInfo.second);
+ }
+
+ grpc::ClientContext rcontext;
+ auto status = pqStub->AlterTopic(&rcontext, request, &response);
+
+ UNIT_ASSERT(status.ok());
+ CreateTopicResult res;
+ response.operation().result().UnpackTo(&res);
+ Cerr << response << "\n" << res << "\n";
+ UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::SUCCESS);
+ };
+
+
+ auto checkDescribe = [&](const TString& topic, const TVector<std::pair<TString, TString>>& readRules) {
+ DescribeTopicRequest request;
+ DescribeTopicResponse response;
+ request.set_path(topic);
+ grpc::ClientContext rcontext;
+
+ auto status = pqStub->DescribeTopic(&rcontext, request, &response);
+ UNIT_ASSERT(status.ok());
+ DescribeTopicResult res;
+ response.operation().result().UnpackTo(&res);
+ Cerr << response << "\n" << res << "\n";
+ UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::SUCCESS);
+
+ UNIT_ASSERT_VALUES_EQUAL(res.settings().read_rules().size(), readRules.size());
+ for (ui64 i = 0; i < readRules.size(); ++i) {
+ const auto& rr = res.settings().read_rules(i);
+ UNIT_ASSERT_EQUAL(rr.consumer_name(), readRules[i].first);
+ UNIT_ASSERT_EQUAL(rr.service_type(), readRules[i].second);
+ }
+ };
+ checkDescribe(
+ "/Root/PQ/rt3.dc1--topic_1",
+ {
+ {"acc/user1", "default_type"},
+ {"acc/user2", "default_type"},
+ {"acc/user3", "default_type"}
+ }
+ );
+ {
+ doAlter(
+ "/Root/PQ/rt3.dc1--topic_2",
+ {
+ {"acc/user1", ""},
+ {"acc/new_user", "MyGreatType"},
+ {"acc/user2", "default_type"},
+ {"acc/user3", "default_type"},
+ {"acc/user4", "AnotherType"}
+ }
+ );
+ checkDescribe(
+ "/Root/PQ/rt3.dc1--topic_2",
+ {
+ {"acc/user1", "default_type"},
+ {"acc/new_user", "MyGreatType"},
+ {"acc/user2", "default_type"},
+ {"acc/user3", "default_type"},
+ {"acc/user4", "AnotherType"}
+ }
+ );
+ }
+ {
+ AddReadRuleRequest request;
+ AddReadRuleResponse response;
+ request.set_path("/Root/PQ/rt3.dc1--topic_3");
+ auto rr = request.mutable_read_rule();
+ rr->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1));
+ rr->set_consumer_name("acc/new_user");
+ rr->set_service_type("MyGreatType");
+
+ grpc::ClientContext rcontext;
+ auto status = pqStub->AddReadRule(&rcontext, request, &response);
+
+ UNIT_ASSERT(status.ok());
+ CreateTopicResult res;
+ response.operation().result().UnpackTo(&res);
+ Cerr << response << "\n" << res << "\n";
+ UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::SUCCESS);
+
+ checkDescribe(
+ "/Root/PQ/rt3.dc1--topic_3",
+ {
+ {"acc/user1", "default_type"},
+ {"acc/user2", "default_type"},
+ {"acc/user3", "default_type"},
+ {"acc/new_user", "MyGreatType"}
+ }
+ );
+ }
+
+ {
+ checkDescribe(
+ "/Root/PQ/rt3.dc1--topic_4",
+ {
+ {"acc/user1", "default_type"},
+ {"acc/user2", "default_type"},
+ {"acc/user3", "default_type"}
+ }
+ );
+
+ RemoveReadRuleRequest request;
+ RemoveReadRuleResponse response;
+ request.set_path("/Root/PQ/rt3.dc1--topic_4");
+ request.set_consumer_name("acc@user2");
+
+ grpc::ClientContext rcontext;
+ auto status = pqStub->RemoveReadRule(&rcontext, request, &response);
+
+ UNIT_ASSERT(status.ok());
+ CreateTopicResult res;
+ response.operation().result().UnpackTo(&res);
+ Cerr << response << "\n" << res << "\n";
+ UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::SUCCESS);
+
+ checkDescribe(
+ "/Root/PQ/rt3.dc1--topic_4",
+ {
+ {"acc/user1", "default_type"},
+ {"acc/user3", "default_type"}
+ }
+ );
+ }
+ }
+
+ Y_UNIT_TEST(ReadRuleServiceTypeMigrationWithDisallowDefault) {
TServerSettings settings = PQSettings(0);
- {
- settings.PQConfig.MutableDefaultClientServiceType()->SetName("default_type");
- settings.PQConfig.AddClientServiceType()->SetName("MyGreatType");
- settings.PQConfig.AddClientServiceType()->SetName("AnotherType");
- settings.PQConfig.AddClientServiceType()->SetName("SecondType");
- }
+ {
+ settings.PQConfig.MutableDefaultClientServiceType()->SetName("default_type");
+ settings.PQConfig.AddClientServiceType()->SetName("MyGreatType");
+ settings.PQConfig.AddClientServiceType()->SetName("AnotherType");
+ settings.PQConfig.AddClientServiceType()->SetName("SecondType");
+ settings.PQConfig.SetDisallowDefaultClientServiceType(true);
+ }
NPersQueue::TTestServer server(settings);
-
+
server.EnableLogs({ NKikimrServices::PQ_READ_PROXY, NKikimrServices::BLACKBOX_VALIDATOR });
-
- const ui32 topicsCount = 4;
- for (ui32 i = 1; i <= topicsCount; ++i) {
- TRequestCreatePQ createTopicRequest(TStringBuilder() << "rt3.dc1--topic_" << i, 1);
- createTopicRequest.ReadRules.push_back("acc@user1");
- createTopicRequest.ReadRules.push_back("acc@user2");
- createTopicRequest.ReadRules.push_back("acc@user3");
- server.AnnoyingClient->CreateTopic(createTopicRequest);
- }
-
- std::unique_ptr<Ydb::PersQueue::V1::PersQueueService::Stub> pqStub;
- {
+
+ const ui32 topicsCount = 4;
+ for (ui32 i = 1; i <= topicsCount; ++i) {
+ TRequestCreatePQ createTopicRequest(TStringBuilder() << "rt3.dc1--topic_" << i, 1);
+ createTopicRequest.ReadRules.push_back("acc@user1");
+ createTopicRequest.ReadRules.push_back("acc@user2");
+ createTopicRequest.ReadRules.push_back("acc@user3");
+ server.AnnoyingClient->CreateTopic(createTopicRequest);
+ }
+
+ std::unique_ptr<Ydb::PersQueue::V1::PersQueueService::Stub> pqStub;
+ {
std::shared_ptr<grpc::Channel> channel = grpc::CreateChannel("localhost:" + ToString(server.GrpcPort), grpc::InsecureChannelCredentials());
- pqStub = Ydb::PersQueue::V1::PersQueueService::NewStub(channel);
- }
- auto doAlter = [&](const TString& topic, const TVector<std::pair<TString, TString>>& readRules) {
- AlterTopicRequest request;
- AlterTopicResponse response;
- request.set_path(topic);
- auto props = request.mutable_settings();
- props->set_partitions_count(1);
- props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE);
- props->set_retention_period_ms(TDuration::Days(1).MilliSeconds());
- for (auto rrInfo : readRules) {
- auto rr = props->add_read_rules();
- rr->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1));
- rr->set_consumer_name(rrInfo.first);
- rr->set_service_type(rrInfo.second);
- }
-
- grpc::ClientContext rcontext;
- auto status = pqStub->AlterTopic(&rcontext, request, &response);
-
- UNIT_ASSERT(status.ok());
- CreateTopicResult res;
- response.operation().result().UnpackTo(&res);
- Cerr << response << "\n" << res << "\n";
- UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::SUCCESS);
- };
-
-
- auto checkDescribe = [&](const TString& topic, const TVector<std::pair<TString, TString>>& readRules) {
- DescribeTopicRequest request;
- DescribeTopicResponse response;
- request.set_path(topic);
- grpc::ClientContext rcontext;
-
- auto status = pqStub->DescribeTopic(&rcontext, request, &response);
- UNIT_ASSERT(status.ok());
- DescribeTopicResult res;
- response.operation().result().UnpackTo(&res);
- Cerr << response << "\n" << res << "\n";
- UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::SUCCESS);
-
- UNIT_ASSERT_VALUES_EQUAL(res.settings().read_rules().size(), readRules.size());
- for (ui64 i = 0; i < readRules.size(); ++i) {
- const auto& rr = res.settings().read_rules(i);
- UNIT_ASSERT_EQUAL(rr.consumer_name(), readRules[i].first);
- UNIT_ASSERT_EQUAL(rr.service_type(), readRules[i].second);
- }
- };
- checkDescribe(
- "/Root/PQ/rt3.dc1--topic_1",
- {
- {"acc/user1", "default_type"},
- {"acc/user2", "default_type"},
- {"acc/user3", "default_type"}
- }
- );
- {
- doAlter(
- "/Root/PQ/rt3.dc1--topic_2",
- {
- {"acc/user1", ""},
- {"acc/new_user", "MyGreatType"},
- {"acc/user2", "default_type"},
- {"acc/user3", "default_type"},
- {"acc/user4", "AnotherType"}
- }
- );
- checkDescribe(
- "/Root/PQ/rt3.dc1--topic_2",
- {
- {"acc/user1", "default_type"},
- {"acc/new_user", "MyGreatType"},
- {"acc/user2", "default_type"},
- {"acc/user3", "default_type"},
- {"acc/user4", "AnotherType"}
- }
- );
- }
- {
- AddReadRuleRequest request;
- AddReadRuleResponse response;
- request.set_path("/Root/PQ/rt3.dc1--topic_3");
- auto rr = request.mutable_read_rule();
- rr->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1));
- rr->set_consumer_name("acc/new_user");
- rr->set_service_type("MyGreatType");
-
- grpc::ClientContext rcontext;
- auto status = pqStub->AddReadRule(&rcontext, request, &response);
-
- UNIT_ASSERT(status.ok());
- CreateTopicResult res;
- response.operation().result().UnpackTo(&res);
- Cerr << response << "\n" << res << "\n";
- UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::SUCCESS);
-
- checkDescribe(
- "/Root/PQ/rt3.dc1--topic_3",
- {
- {"acc/user1", "default_type"},
- {"acc/user2", "default_type"},
- {"acc/user3", "default_type"},
- {"acc/new_user", "MyGreatType"}
- }
- );
- }
-
- {
- checkDescribe(
- "/Root/PQ/rt3.dc1--topic_4",
- {
- {"acc/user1", "default_type"},
- {"acc/user2", "default_type"},
- {"acc/user3", "default_type"}
- }
- );
-
- RemoveReadRuleRequest request;
- RemoveReadRuleResponse response;
- request.set_path("/Root/PQ/rt3.dc1--topic_4");
- request.set_consumer_name("acc@user2");
-
- grpc::ClientContext rcontext;
- auto status = pqStub->RemoveReadRule(&rcontext, request, &response);
-
- UNIT_ASSERT(status.ok());
- CreateTopicResult res;
- response.operation().result().UnpackTo(&res);
- Cerr << response << "\n" << res << "\n";
- UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::SUCCESS);
-
- checkDescribe(
- "/Root/PQ/rt3.dc1--topic_4",
- {
- {"acc/user1", "default_type"},
- {"acc/user3", "default_type"}
- }
- );
- }
- }
-
- Y_UNIT_TEST(ReadRuleServiceTypeMigrationWithDisallowDefault) {
- TServerSettings settings = PQSettings(0);
- {
- settings.PQConfig.MutableDefaultClientServiceType()->SetName("default_type");
- settings.PQConfig.AddClientServiceType()->SetName("MyGreatType");
- settings.PQConfig.AddClientServiceType()->SetName("AnotherType");
- settings.PQConfig.AddClientServiceType()->SetName("SecondType");
- settings.PQConfig.SetDisallowDefaultClientServiceType(true);
- }
- NPersQueue::TTestServer server(settings);
-
- server.EnableLogs({ NKikimrServices::PQ_READ_PROXY, NKikimrServices::BLACKBOX_VALIDATOR });
-
- const ui32 topicsCount = 4;
- for (ui32 i = 1; i <= topicsCount; ++i) {
- TRequestCreatePQ createTopicRequest(TStringBuilder() << "rt3.dc1--topic_" << i, 1);
- createTopicRequest.ReadRules.push_back("acc@user1");
- createTopicRequest.ReadRules.push_back("acc@user2");
- createTopicRequest.ReadRules.push_back("acc@user3");
- server.AnnoyingClient->CreateTopic(createTopicRequest);
- }
-
- std::unique_ptr<Ydb::PersQueue::V1::PersQueueService::Stub> pqStub;
- {
- std::shared_ptr<grpc::Channel> channel = grpc::CreateChannel("localhost:" + ToString(server.GrpcPort), grpc::InsecureChannelCredentials());
- pqStub = Ydb::PersQueue::V1::PersQueueService::NewStub(channel);
- }
-
- auto doAlter = [&](
- const TString& topic,
- const TVector<std::pair<TString, TString>>& readRules,
- Ydb::StatusIds::StatusCode statusCode = Ydb::StatusIds::SUCCESS
- ) {
- AlterTopicRequest request;
- AlterTopicResponse response;
- request.set_path(topic);
- auto props = request.mutable_settings();
- props->set_partitions_count(1);
- props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE);
- props->set_retention_period_ms(TDuration::Days(1).MilliSeconds());
- for (auto rrInfo : readRules) {
- auto rr = props->add_read_rules();
- rr->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1));
- rr->set_consumer_name(rrInfo.first);
- rr->set_service_type(rrInfo.second);
- }
-
- grpc::ClientContext rcontext;
- auto status = pqStub->AlterTopic(&rcontext, request, &response);
-
- UNIT_ASSERT(status.ok());
- CreateTopicResult res;
- response.operation().result().UnpackTo(&res);
- Cerr << response << "\n" << res << "\n";
- UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), statusCode);
- };
-
- auto checkDescribe = [&](
- const TString& topic,
- const TVector<std::pair<TString, TString>>& readRules,
- Ydb::StatusIds::StatusCode statusCode = Ydb::StatusIds::SUCCESS
- ) {
- DescribeTopicRequest request;
- DescribeTopicResponse response;
- request.set_path(topic);
- grpc::ClientContext rcontext;
-
- auto status = pqStub->DescribeTopic(&rcontext, request, &response);
- UNIT_ASSERT(status.ok());
- DescribeTopicResult res;
- response.operation().result().UnpackTo(&res);
- Cerr << response << "\n" << res << "\n";
- UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), statusCode);
- if (statusCode == Ydb::StatusIds::SUCCESS) {
- UNIT_ASSERT_VALUES_EQUAL(res.settings().read_rules().size(), readRules.size());
- for (ui64 i = 0; i < readRules.size(); ++i) {
- const auto& rr = res.settings().read_rules(i);
- UNIT_ASSERT_EQUAL(rr.consumer_name(), readRules[i].first);
- UNIT_ASSERT_EQUAL(rr.service_type(), readRules[i].second);
- }
- }
- };
- checkDescribe(
- "/Root/PQ/rt3.dc1--topic_1",
- {},
- Ydb::StatusIds::INTERNAL_ERROR
- );
- {
- doAlter(
- "/Root/PQ/rt3.dc1--topic_2",
- {
- {"acc/new_user", "MyGreatType"},
- {"acc/user2", "SecondType"},
- {"acc/user3", "AnotherType"},
- {"acc/user4", "AnotherType"}
- }
- );
- checkDescribe(
- "/Root/PQ/rt3.dc1--topic_2",
- {
- {"acc/new_user", "MyGreatType"},
- {"acc/user2", "SecondType"},
- {"acc/user3", "AnotherType"},
- {"acc/user4", "AnotherType"}
- }
- );
- }
- }
-
- Y_UNIT_TEST(TClusterTrackerTest) {
- APITestSetup setup{TEST_CASE_NAME};
- setup.GetPQConfig().SetClustersUpdateTimeoutSec(0);
- const auto edgeActorID = setup.GetServer().GetRuntime()->AllocateEdgeActor();
- THashMap<TString, TPQTestClusterInfo> clusters = DEFAULT_CLUSTERS_LIST;
-
- auto compareInfo = [](const TString& name, const TPQTestClusterInfo& info, const NPQ::NClusterTracker::TClustersList::TCluster& trackerInfo) {
- UNIT_ASSERT_EQUAL(name, trackerInfo.Name);
- UNIT_ASSERT_EQUAL(name, trackerInfo.Datacenter);
- UNIT_ASSERT_EQUAL(info.Balancer, trackerInfo.Balancer);
- UNIT_ASSERT_EQUAL(info.Enabled, trackerInfo.IsEnabled);
- UNIT_ASSERT_EQUAL(info.Weight, trackerInfo.Weight);
- };
-
- auto getClustersFromTracker = [&]() {
- setup.GetServer().GetRuntime()->Send(new IEventHandle(
- NPQ::NClusterTracker::MakeClusterTrackerID(),
- edgeActorID,
- new NPQ::NClusterTracker::TEvClusterTracker::TEvSubscribe
- ));
- return setup.GetServer().GetRuntime()->GrabEdgeEvent<NPQ::NClusterTracker::TEvClusterTracker::TEvClustersUpdate>();
- };
-
-
- {
- auto trackerResponce = getClustersFromTracker();
- for (auto& clusterInfo : trackerResponce->ClustersList->Clusters) {
- auto it = clusters.find(clusterInfo.Name);
- UNIT_ASSERT(it != clusters.end());
- compareInfo(it->first, it->second, clusterInfo);
- }
- }
-
- UNIT_ASSERT_EQUAL(clusters.count("dc1"), 1);
- UNIT_ASSERT_EQUAL(clusters.count("dc2"), 1);
- clusters["dc1"].Weight = 666;
- clusters["dc2"].Balancer = "newbalancer.net";
- setup.GetFlatMsgBusPQClient().InitDCs(clusters);
- TInstant updateTime = TInstant::Now();
-
- while (true) {
- auto trackerResponce = getClustersFromTracker();
- if (trackerResponce->ClustersListUpdateTimestamp) {
- if (trackerResponce->ClustersListUpdateTimestamp.GetRef() >= updateTime + TDuration::Seconds(5)) {
- for (auto& clusterInfo : trackerResponce->ClustersList->Clusters) {
- auto it = clusters.find(clusterInfo.Name);
- UNIT_ASSERT(it != clusters.end());
- compareInfo(it->first, it->second, clusterInfo);
- }
- break;
- }
- }
- Sleep(TDuration::MilliSeconds(100));
- }
- }
-
- Y_UNIT_TEST(TestReadPartitionByGroupId) {
- NPersQueue::TTestServer server;
-
- ui32 partitionsCount = 100;
+ pqStub = Ydb::PersQueue::V1::PersQueueService::NewStub(channel);
+ }
+
+ auto doAlter = [&](
+ const TString& topic,
+ const TVector<std::pair<TString, TString>>& readRules,
+ Ydb::StatusIds::StatusCode statusCode = Ydb::StatusIds::SUCCESS
+ ) {
+ AlterTopicRequest request;
+ AlterTopicResponse response;
+ request.set_path(topic);
+ auto props = request.mutable_settings();
+ props->set_partitions_count(1);
+ props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE);
+ props->set_retention_period_ms(TDuration::Days(1).MilliSeconds());
+ for (auto rrInfo : readRules) {
+ auto rr = props->add_read_rules();
+ rr->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1));
+ rr->set_consumer_name(rrInfo.first);
+ rr->set_service_type(rrInfo.second);
+ }
+
+ grpc::ClientContext rcontext;
+ auto status = pqStub->AlterTopic(&rcontext, request, &response);
+
+ UNIT_ASSERT(status.ok());
+ CreateTopicResult res;
+ response.operation().result().UnpackTo(&res);
+ Cerr << response << "\n" << res << "\n";
+ UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), statusCode);
+ };
+
+ auto checkDescribe = [&](
+ const TString& topic,
+ const TVector<std::pair<TString, TString>>& readRules,
+ Ydb::StatusIds::StatusCode statusCode = Ydb::StatusIds::SUCCESS
+ ) {
+ DescribeTopicRequest request;
+ DescribeTopicResponse response;
+ request.set_path(topic);
+ grpc::ClientContext rcontext;
+
+ auto status = pqStub->DescribeTopic(&rcontext, request, &response);
+ UNIT_ASSERT(status.ok());
+ DescribeTopicResult res;
+ response.operation().result().UnpackTo(&res);
+ Cerr << response << "\n" << res << "\n";
+ UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), statusCode);
+ if (statusCode == Ydb::StatusIds::SUCCESS) {
+ UNIT_ASSERT_VALUES_EQUAL(res.settings().read_rules().size(), readRules.size());
+ for (ui64 i = 0; i < readRules.size(); ++i) {
+ const auto& rr = res.settings().read_rules(i);
+ UNIT_ASSERT_EQUAL(rr.consumer_name(), readRules[i].first);
+ UNIT_ASSERT_EQUAL(rr.service_type(), readRules[i].second);
+ }
+ }
+ };
+ checkDescribe(
+ "/Root/PQ/rt3.dc1--topic_1",
+ {},
+ Ydb::StatusIds::INTERNAL_ERROR
+ );
+ {
+ doAlter(
+ "/Root/PQ/rt3.dc1--topic_2",
+ {
+ {"acc/new_user", "MyGreatType"},
+ {"acc/user2", "SecondType"},
+ {"acc/user3", "AnotherType"},
+ {"acc/user4", "AnotherType"}
+ }
+ );
+ checkDescribe(
+ "/Root/PQ/rt3.dc1--topic_2",
+ {
+ {"acc/new_user", "MyGreatType"},
+ {"acc/user2", "SecondType"},
+ {"acc/user3", "AnotherType"},
+ {"acc/user4", "AnotherType"}
+ }
+ );
+ }
+ }
+
+ Y_UNIT_TEST(TClusterTrackerTest) {
+ APITestSetup setup{TEST_CASE_NAME};
+ setup.GetPQConfig().SetClustersUpdateTimeoutSec(0);
+ const auto edgeActorID = setup.GetServer().GetRuntime()->AllocateEdgeActor();
+ THashMap<TString, TPQTestClusterInfo> clusters = DEFAULT_CLUSTERS_LIST;
+
+ auto compareInfo = [](const TString& name, const TPQTestClusterInfo& info, const NPQ::NClusterTracker::TClustersList::TCluster& trackerInfo) {
+ UNIT_ASSERT_EQUAL(name, trackerInfo.Name);
+ UNIT_ASSERT_EQUAL(name, trackerInfo.Datacenter);
+ UNIT_ASSERT_EQUAL(info.Balancer, trackerInfo.Balancer);
+ UNIT_ASSERT_EQUAL(info.Enabled, trackerInfo.IsEnabled);
+ UNIT_ASSERT_EQUAL(info.Weight, trackerInfo.Weight);
+ };
+
+ auto getClustersFromTracker = [&]() {
+ setup.GetServer().GetRuntime()->Send(new IEventHandle(
+ NPQ::NClusterTracker::MakeClusterTrackerID(),
+ edgeActorID,
+ new NPQ::NClusterTracker::TEvClusterTracker::TEvSubscribe
+ ));
+ return setup.GetServer().GetRuntime()->GrabEdgeEvent<NPQ::NClusterTracker::TEvClusterTracker::TEvClustersUpdate>();
+ };
+
+
+ {
+ auto trackerResponce = getClustersFromTracker();
+ for (auto& clusterInfo : trackerResponce->ClustersList->Clusters) {
+ auto it = clusters.find(clusterInfo.Name);
+ UNIT_ASSERT(it != clusters.end());
+ compareInfo(it->first, it->second, clusterInfo);
+ }
+ }
+
+ UNIT_ASSERT_EQUAL(clusters.count("dc1"), 1);
+ UNIT_ASSERT_EQUAL(clusters.count("dc2"), 1);
+ clusters["dc1"].Weight = 666;
+ clusters["dc2"].Balancer = "newbalancer.net";
+ setup.GetFlatMsgBusPQClient().InitDCs(clusters);
+ TInstant updateTime = TInstant::Now();
+
+ while (true) {
+ auto trackerResponce = getClustersFromTracker();
+ if (trackerResponce->ClustersListUpdateTimestamp) {
+ if (trackerResponce->ClustersListUpdateTimestamp.GetRef() >= updateTime + TDuration::Seconds(5)) {
+ for (auto& clusterInfo : trackerResponce->ClustersList->Clusters) {
+ auto it = clusters.find(clusterInfo.Name);
+ UNIT_ASSERT(it != clusters.end());
+ compareInfo(it->first, it->second, clusterInfo);
+ }
+ break;
+ }
+ }
+ Sleep(TDuration::MilliSeconds(100));
+ }
+ }
+
+ Y_UNIT_TEST(TestReadPartitionByGroupId) {
+ NPersQueue::TTestServer server;
+
+ ui32 partitionsCount = 100;
TString topic = "topic1";
TString topicFullName = "rt3.dc1--" + topic;
-
+
server.AnnoyingClient->CreateTopic(topicFullName, partitionsCount);
server.EnableLogs({ NKikimrServices::PQ_READ_PROXY});
-
- auto driver = server.AnnoyingClient->GetDriver();
-
- for (ui32 partition = 30; partition < partitionsCount; ++partition) {
- auto reader = CreateReader(
- *driver,
- NYdb::NPersQueue::TReadSessionSettings()
- .AppendTopics(
- NYdb::NPersQueue::TTopicReadSettings(topic)
- .AppendPartitionGroupIds(partition + 1)
- )
- .ConsumerName("shared/user")
- .ReadOnlyOriginal(true)
- );
-
- TMaybe<NYdb::NPersQueue::TReadSessionEvent::TEvent> event = reader->GetEvent(true, 1);
- auto createStream = std::get_if<NYdb::NPersQueue::TReadSessionEvent::TCreatePartitionStreamEvent>(&*event);
- UNIT_ASSERT(createStream);
+
+ auto driver = server.AnnoyingClient->GetDriver();
+
+ for (ui32 partition = 30; partition < partitionsCount; ++partition) {
+ auto reader = CreateReader(
+ *driver,
+ NYdb::NPersQueue::TReadSessionSettings()
+ .AppendTopics(
+ NYdb::NPersQueue::TTopicReadSettings(topic)
+ .AppendPartitionGroupIds(partition + 1)
+ )
+ .ConsumerName("shared/user")
+ .ReadOnlyOriginal(true)
+ );
+
+ TMaybe<NYdb::NPersQueue::TReadSessionEvent::TEvent> event = reader->GetEvent(true, 1);
+ auto createStream = std::get_if<NYdb::NPersQueue::TReadSessionEvent::TCreatePartitionStreamEvent>(&*event);
+ UNIT_ASSERT(createStream);
TString stepDescription = TStringBuilder() << "create stream for partition=" << partition
- << " : " << createStream->DebugString();
- Cerr << stepDescription << Endl;
- UNIT_ASSERT_EQUAL_C(
- partition,
- createStream->GetPartitionStream()->GetPartitionId(),
- stepDescription
+ << " : " << createStream->DebugString();
+ Cerr << stepDescription << Endl;
+ UNIT_ASSERT_EQUAL_C(
+ partition,
+ createStream->GetPartitionStream()->GetPartitionId(),
+ stepDescription
);
}
- }
+ }
}
}
diff --git a/ydb/services/persqueue_v1/ut/new_schemecache_ut/ya.make b/ydb/services/persqueue_v1/ut/new_schemecache_ut/ya.make
index 4cffa3015d..2a8fc402a9 100644
--- a/ydb/services/persqueue_v1/ut/new_schemecache_ut/ya.make
+++ b/ydb/services/persqueue_v1/ut/new_schemecache_ut/ya.make
@@ -3,7 +3,7 @@ UNITTEST_FOR(ydb/services/persqueue_v1)
OWNER(
zhenyok
g:kikimr
- g:logbroker
+ g:logbroker
)
CFLAGS(
diff --git a/ydb/services/persqueue_v1/ut/rate_limiter_test_setup.cpp b/ydb/services/persqueue_v1/ut/rate_limiter_test_setup.cpp
index c801eb7b4e..779d60e778 100644
--- a/ydb/services/persqueue_v1/ut/rate_limiter_test_setup.cpp
+++ b/ydb/services/persqueue_v1/ut/rate_limiter_test_setup.cpp
@@ -9,88 +9,88 @@ using namespace NKikimr::Tests;
namespace NKikimr::NPersQueueTests {
-TRateLimiterTestSetup::TRateLimiterTestSetup(
- NKikimrPQ::TPQConfig::TQuotingConfig::ELimitedEntity limitedEntity,
- double writeAccountQuota,
- double readAccountQuota,
- bool enableReadQuoting
-)
+TRateLimiterTestSetup::TRateLimiterTestSetup(
+ NKikimrPQ::TPQConfig::TQuotingConfig::ELimitedEntity limitedEntity,
+ double writeAccountQuota,
+ double readAccountQuota,
+ bool enableReadQuoting
+)
: Server(new NPersQueue::TTestServer(false))
, LimitedEntity(limitedEntity)
- , WriteAccountQuota(writeAccountQuota)
- , ReadAccountQuota(readAccountQuota)
+ , WriteAccountQuota(writeAccountQuota)
+ , ReadAccountQuota(readAccountQuota)
{
- Start(enableReadQuoting);
+ Start(enableReadQuoting);
}
-void TRateLimiterTestSetup::CreateTopic(const TString& path) {
+void TRateLimiterTestSetup::CreateTopic(const TString& path) {
const TString name = BuildFullTopicName(path, "dc1");
const TString account = GetAccount(name);
Cerr << "Creating topic \"" << name << "\"" << Endl;
Server->AnnoyingClient->CreateTopic(name, 1);
- CreateKesus(account);
- CreateQuotaResources(path, "write-quota", false);
- CreateQuotaResources(path, "read-quota", true);
-}
+ CreateKesus(account);
+ CreateQuotaResources(path, "write-quota", false);
+ CreateQuotaResources(path, "read-quota", true);
+}
-void TRateLimiterTestSetup::CreateConsumer(const TString& path) {
- const TString account = GetAccount(path);
+void TRateLimiterTestSetup::CreateConsumer(const TString& path) {
+ const TString account = GetAccount(path);
- Cerr << "Creating consumer \"" << path << "\"" << Endl;
+ Cerr << "Creating consumer \"" << path << "\"" << Endl;
Server->AnnoyingClient->CreateConsumer(path);
- CreateKesus(account);
- CreateQuotaResources(path, "write-quota", true);
- CreateQuotaResources(path, "read-quota", false);
-}
-
-void TRateLimiterTestSetup::CreateKesus(const TString& account) {
+ CreateKesus(account);
+ CreateQuotaResources(path, "write-quota", true);
+ CreateQuotaResources(path, "read-quota", false);
+}
+
+void TRateLimiterTestSetup::CreateKesus(const TString& account) {
const NMsgBusProxy::EResponseStatus createKesusResult = Server->AnnoyingClient->CreateKesus(QuotersRootPath, account);
- UNIT_ASSERT_C(createKesusResult == NMsgBusProxy::MSTATUS_OK, createKesusResult);
-
- const TString kesusPath = TStringBuilder() << QuotersRootPath << "/" << account;
+ UNIT_ASSERT_C(createKesusResult == NMsgBusProxy::MSTATUS_OK, createKesusResult);
- Cerr << "Creating kesus with path=" << kesusPath << Endl;
- auto setAccountQuota = [&](const TString& quotaPrefix, double value) {
+ const TString kesusPath = TStringBuilder() << QuotersRootPath << "/" << account;
+
+ Cerr << "Creating kesus with path=" << kesusPath << Endl;
+ auto setAccountQuota = [&](const TString& quotaPrefix, double value) {
Cerr << "Adding quota for account kesus=" << kesusPath << " quota-path=" << quotaPrefix << " value=" << value << Endl;
const auto statusCode = Server->AnnoyingClient->AddQuoterResource(
Server->CleverServer->GetRuntime(), kesusPath, quotaPrefix, value
);
- UNIT_ASSERT_C(
- statusCode == Ydb::StatusIds::SUCCESS || statusCode == Ydb::StatusIds::ALREADY_EXISTS,
- "Status: " << Ydb::StatusIds::StatusCode_Name(statusCode)
- );
- };
- setAccountQuota("write-quota", WriteAccountQuota);
- setAccountQuota("read-quota", ReadAccountQuota);
-}
-
-void TRateLimiterTestSetup::CreateQuotaResources(const TString& path, const TString& quotaPrefix, bool excludeLastComponent) {
+ UNIT_ASSERT_C(
+ statusCode == Ydb::StatusIds::SUCCESS || statusCode == Ydb::StatusIds::ALREADY_EXISTS,
+ "Status: " << Ydb::StatusIds::StatusCode_Name(statusCode)
+ );
+ };
+ setAccountQuota("write-quota", WriteAccountQuota);
+ setAccountQuota("read-quota", ReadAccountQuota);
+}
+
+void TRateLimiterTestSetup::CreateQuotaResources(const TString& path, const TString& quotaPrefix, bool excludeLastComponent) {
TVector<TString> pathComponents = SplitPath(path);
- if (pathComponents.size() <= 1) {
- return;
+ if (pathComponents.size() <= 1) {
+ return;
}
- TStringBuilder prefixPath;
- prefixPath << quotaPrefix;
-
- auto firstIt = pathComponents.begin() + 1; // resource path must be without account
- auto lastIt = pathComponents.end() - (excludeLastComponent ? 1 : 0);
-
- const TString account = GetAccount(path);
- const TString kesusPath = TStringBuilder() << QuotersRootPath << "/" << account;
- for (auto currentComponent = firstIt; currentComponent != lastIt; ++currentComponent) {
- prefixPath << "/" << *currentComponent;
- Cerr << "Adding quoter resource: \"" << prefixPath << "\"" << Endl;
+ TStringBuilder prefixPath;
+ prefixPath << quotaPrefix;
+
+ auto firstIt = pathComponents.begin() + 1; // resource path must be without account
+ auto lastIt = pathComponents.end() - (excludeLastComponent ? 1 : 0);
+
+ const TString account = GetAccount(path);
+ const TString kesusPath = TStringBuilder() << QuotersRootPath << "/" << account;
+ for (auto currentComponent = firstIt; currentComponent != lastIt; ++currentComponent) {
+ prefixPath << "/" << *currentComponent;
+ Cerr << "Adding quoter resource: \"" << prefixPath << "\"" << Endl;
const auto statusCode = Server->AnnoyingClient->AddQuoterResource(
Server->CleverServer->GetRuntime(), kesusPath, prefixPath
);
- UNIT_ASSERT_C(
- statusCode == Ydb::StatusIds::SUCCESS || statusCode == Ydb::StatusIds::ALREADY_EXISTS,
- "Status: " << Ydb::StatusIds::StatusCode_Name(statusCode)
- );
- }
+ UNIT_ASSERT_C(
+ statusCode == Ydb::StatusIds::SUCCESS || statusCode == Ydb::StatusIds::ALREADY_EXISTS,
+ "Status: " << Ydb::StatusIds::StatusCode_Name(statusCode)
+ );
+ }
}
/*
@@ -107,13 +107,13 @@ THolder<Ydb::PersQueue::IProducer> TRateLimiterTestSetup::StartProducer(const TS
}
*/
-void TRateLimiterTestSetup::Start(bool enableReadQuoting) {
- InitServer(enableReadQuoting);
+void TRateLimiterTestSetup::Start(bool enableReadQuoting) {
+ InitServer(enableReadQuoting);
InitQuoting();
WaitWritePQServiceInitialization();
}
-void TRateLimiterTestSetup::InitServer(bool enableReadQuoting) {
+void TRateLimiterTestSetup::InitServer(bool enableReadQuoting) {
auto& settings = Server->ServerSettings;
settings.PQConfig.MutableQuotingConfig()->SetEnableQuoting(true);
@@ -130,7 +130,7 @@ void TRateLimiterTestSetup::InitServer(bool enableReadQuoting) {
NKikimrServices::QUOTER_SERVICE,
NKikimrServices::QUOTER_PROXY,
NKikimrServices::KESUS_TABLET,
- NKikimrServices::PQ_READ_SPEED_LIMITER
+ NKikimrServices::PQ_READ_SPEED_LIMITER
},
NActors::NLog::PRI_TRACE
);
diff --git a/ydb/services/persqueue_v1/ut/rate_limiter_test_setup.h b/ydb/services/persqueue_v1/ut/rate_limiter_test_setup.h
index dbb25e3769..01529818e1 100644
--- a/ydb/services/persqueue_v1/ut/rate_limiter_test_setup.h
+++ b/ydb/services/persqueue_v1/ut/rate_limiter_test_setup.h
@@ -8,15 +8,15 @@ namespace NKikimr::NPersQueueTests {
class TRateLimiterTestSetup {
public:
- explicit TRateLimiterTestSetup(
- NKikimrPQ::TPQConfig::TQuotingConfig::ELimitedEntity limitedEntity,
- double writeAccountQuota = 1000.0,
- double readAccountQuota = 1000.0,
- bool enableReadQuoting = false
- );
+ explicit TRateLimiterTestSetup(
+ NKikimrPQ::TPQConfig::TQuotingConfig::ELimitedEntity limitedEntity,
+ double writeAccountQuota = 1000.0,
+ double readAccountQuota = 1000.0,
+ bool enableReadQuoting = false
+ );
- void CreateTopic(const TString& path);
- void CreateConsumer(const TString& path);
+ void CreateTopic(const TString& path);
+ void CreateConsumer(const TString& path);
// namespace NPersQueue = Ydb::PersQueue;
// THolder<Ydb::PersQueue::IProducer> StartProducer(const TString& topicPath, bool compress = false);
@@ -39,12 +39,12 @@ public:
}
private:
- void CreateKesus(const TString& account);
- void CreateQuotaResources(const TString& path, const TString& quotaPrefix, bool excludeLastComponent);
+ void CreateKesus(const TString& account);
+ void CreateQuotaResources(const TString& path, const TString& quotaPrefix, bool excludeLastComponent);
- void Start(bool enableReadQuoting);
-
- void InitServer(bool enableReadQuoting);
+ void Start(bool enableReadQuoting);
+
+ void InitServer(bool enableReadQuoting);
void InitQuoting();
void WaitWritePQServiceInitialization();
@@ -52,8 +52,8 @@ private:
THolder<NPersQueue::TTestServer> Server;
THolder<TPQDataWriter> PQDataWriter; // For waiting for grpc writer service initialization.
const NKikimrPQ::TPQConfig::TQuotingConfig::ELimitedEntity LimitedEntity;
- double WriteAccountQuota;
- double ReadAccountQuota;
- const TString QuotersRootPath = "/Root/PersQueue/System/Quoters";
+ double WriteAccountQuota;
+ double ReadAccountQuota;
+ const TString QuotersRootPath = "/Root/PersQueue/System/Quoters";
};
}
diff --git a/ydb/services/persqueue_v1/ut/ya.make b/ydb/services/persqueue_v1/ut/ya.make
index e349b2e85f..70c088383d 100644
--- a/ydb/services/persqueue_v1/ut/ya.make
+++ b/ydb/services/persqueue_v1/ut/ya.make
@@ -1,11 +1,11 @@
UNITTEST_FOR(ydb/services/persqueue_v1)
-OWNER(
- alexnick
+OWNER(
+ alexnick
g:kikimr
- g:logbroker
-)
-
+ g:logbroker
+)
+
CFLAGS(
-DACTORLIB_HUGE_PB_SIZE
)
diff --git a/ydb/services/persqueue_v1/ya.make b/ydb/services/persqueue_v1/ya.make
index 79179d7463..ade0406fcc 100644
--- a/ydb/services/persqueue_v1/ya.make
+++ b/ydb/services/persqueue_v1/ya.make
@@ -3,12 +3,12 @@ LIBRARY()
OWNER(
alexnick
g:kikimr
- g:logbroker
+ g:logbroker
)
SRCS(
grpc_pq_actor.h
- grpc_pq_codecs.cpp
+ grpc_pq_codecs.cpp
grpc_pq_read_actor.cpp
grpc_pq_read.cpp
grpc_pq_read.h
@@ -37,7 +37,7 @@ PEERDIR(
ydb/core/ydb_convert
ydb/library/aclib
ydb/library/persqueue/obfuscate
- ydb/library/persqueue/tests
+ ydb/library/persqueue/tests
ydb/library/persqueue/topic_parser
ydb/public/api/grpc/draft
ydb/public/api/protos
diff --git a/ydb/services/rate_limiter/rate_limiter_ut.cpp b/ydb/services/rate_limiter/rate_limiter_ut.cpp
index 1be91c9454..4ac05a571c 100644
--- a/ydb/services/rate_limiter/rate_limiter_ut.cpp
+++ b/ydb/services/rate_limiter/rate_limiter_ut.cpp
@@ -76,7 +76,7 @@ public:
void virtual CheckAcquireResource(const TString& coordinationNodePath, const TString& resourcePath, const NYdb::NRateLimiter::TAcquireResourceSettings& settings, NYdb::EStatus expected) {
const auto acquireResultFuture = RateLimiterClient.AcquireResource(coordinationNodePath, resourcePath, settings);
ASSERT_STATUS(acquireResultFuture, expected);
- }
+ }
static TString CoordinationNodePath;
@@ -345,7 +345,7 @@ Y_UNIT_TEST_SUITE(TGRpcRateLimiterTest) {
setup->CheckAcquireResource(TTestSetup::CoordinationNodePath, "res", TAcquireResourceSettings().Amount(1).OperationTimeout(TDuration::MilliSeconds(200)), NYdb::EStatus::TIMEOUT);
setup->CheckAcquireResource(TTestSetup::CoordinationNodePath, "res", TAcquireResourceSettings().Amount(1).IsUsedAmount(true).OperationTimeout(TDuration::MilliSeconds(200)), NYdb::EStatus::SUCCESS);
}
- }
+ }
Y_UNIT_TEST(AcquireResourceManyRequiredGrpcApi) {
AcquireResourceManyRequired(false);
diff --git a/ydb/services/ydb/ydb_stats_ut.cpp b/ydb/services/ydb/ydb_stats_ut.cpp
index dcb7029cfe..b40e22fc42 100644
--- a/ydb/services/ydb/ydb_stats_ut.cpp
+++ b/ydb/services/ydb/ydb_stats_ut.cpp
@@ -1,15 +1,15 @@
-#include "ydb_common_ut.h"
-
+#include "ydb_common_ut.h"
+
#include <ydb/public/sdk/cpp/client/extensions/solomon_stats/pull_connector.h>
#include <ydb/public/sdk/cpp/client/ydb_extension/extension.h>
#include <ydb/public/sdk/cpp/client/ydb_table/table.h>
-
+
#include <library/cpp/testing/unittest/registar.h>
#include <library/cpp/testing/unittest/tests_data.h>
-#include <library/cpp/monlib/encode/encoder.h>
+#include <library/cpp/monlib/encode/encoder.h>
#include <library/cpp/monlib/encode/json/json.h>
-
-#include <util/generic/ptr.h>
+
+#include <util/generic/ptr.h>
#include <util/system/valgrind.h>
struct TStatCounters {
diff --git a/ydb/services/ydb/ydb_table_ut.cpp b/ydb/services/ydb/ydb_table_ut.cpp
index e22d10131d..4861c06b92 100644
--- a/ydb/services/ydb/ydb_table_ut.cpp
+++ b/ydb/services/ydb/ydb_table_ut.cpp
@@ -1,4 +1,4 @@
-#include "ydb_common_ut.h"
+#include "ydb_common_ut.h"
#include <ydb/public/api/grpc/ydb_table_v1.grpc.pb.h>
#include <ydb/public/sdk/cpp/client/ydb_proto/accessor.h>
@@ -11,11 +11,11 @@
#include <ydb/public/sdk/cpp/client/ydb_operation/operation.h>
#include <ydb/public/sdk/cpp/client/resources/ydb_resources.h>
#include <ydb/public/lib/yson_value/ydb_yson_value.h>
-
+
#include <ydb/library/yql/public/issue/yql_issue.h>
#include <ydb/library/yql/public/issue/yql_issue_message.h>
-#include <library/cpp/grpc/client/grpc_client_low.h>
+#include <library/cpp/grpc/client/grpc_client_low.h>
#include <util/thread/factory.h>
diff --git a/ydb/tests/functional/sqs/sqs_test_base.py b/ydb/tests/functional/sqs/sqs_test_base.py
index 26ebcb3bec..05d5303f64 100644
--- a/ydb/tests/functional/sqs/sqs_test_base.py
+++ b/ydb/tests/functional/sqs/sqs_test_base.py
@@ -15,7 +15,7 @@ import ydb.tests.library.common.yatest_common as yatest_common
from ydb.tests.library.harness.kikimr_cluster import kikimr_cluster_factory
from ydb.tests.library.harness.kikimr_config import KikimrConfigGenerator
from ydb.tests.library.harness.util import LogLevels
-from ydb.tests.library.sqs.tables import create_all_tables as create_all_sqs_tables
+from ydb.tests.library.sqs.tables import create_all_tables as create_all_sqs_tables
from sqs_requests_client import SqsHttpApi
@@ -282,7 +282,7 @@ class KikimrSqsTestBase(object):
driver.wait()
with ydb.SessionPool(driver, size=1) as pool:
with pool.checkout() as session:
- create_all_sqs_tables(cls.sqs_root, driver, session)
+ create_all_sqs_tables(cls.sqs_root, driver, session)
cls.create_metauser(cluster, config_generator)
@classmethod
diff --git a/ydb/tests/functional/sqs/test_multiplexing_tables_format.py b/ydb/tests/functional/sqs/test_multiplexing_tables_format.py
index 44065f6e30..f5840333d3 100644
--- a/ydb/tests/functional/sqs/test_multiplexing_tables_format.py
+++ b/ydb/tests/functional/sqs/test_multiplexing_tables_format.py
@@ -1,56 +1,56 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-import pytest
-import yatest
-
-from sqs_test_base import KikimrSqsTestBase, get_test_with_sqs_installation_by_path, get_test_with_sqs_tenant_installation, IS_FIFO_PARAMS
-
-
-class MultiplexingTablesFormatTest(KikimrSqsTestBase):
- def _set_new_format_settings(self, username=None, tables_format=1):
- if username is None:
- username = self._username
- self._execute_yql_query(
- f'UPSERT INTO `{self.sqs_root}/.Settings` (Account, Name, Value) \
- VALUES ("{username}", "CreateQueuesWithTabletFormat", "{tables_format}")'
- )
-
- def create_queue(self, is_fifo):
- if is_fifo:
- self.queue_name = self.queue_name + '.fifo'
- self._create_queue_and_assert(self.queue_name, is_fifo=is_fifo)
-
- def create_queue_must_fail(self, is_fifo):
- try:
- self.create_queue(is_fifo)
- except yatest.common.process.ExecutionError:
- return
- assert(False)
-
- def create_queue_with_wrong_tables_format(self, tables_format):
- self._set_new_format_settings(tables_format='qwerty')
- self.create_queue_must_fail(True)
- self.create_queue_must_fail(False)
-
- @pytest.mark.parametrize(**IS_FIFO_PARAMS)
- def test_create_queue(self, is_fifo):
- self._set_new_format_settings()
- self.create_queue(is_fifo)
-
- def test_create_queue_with_incorrect_tables_format(self):
- self.create_queue_with_wrong_tables_format('qwerty')
-
- def test_create_queue_with_empty_tables_format(self):
- self.create_queue_with_wrong_tables_format('')
-
- def test_create_queue_with_unsupported_tables_format(self):
- self.create_queue_with_wrong_tables_format(2)
-
-
-class TestMultiplexingTablesFormatWithTenant(get_test_with_sqs_tenant_installation(MultiplexingTablesFormatTest)):
- pass
-
-
-class TestMultiplexingTablesFormatWithPath(get_test_with_sqs_installation_by_path(MultiplexingTablesFormatTest)):
- pass
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import pytest
+import yatest
+
+from sqs_test_base import KikimrSqsTestBase, get_test_with_sqs_installation_by_path, get_test_with_sqs_tenant_installation, IS_FIFO_PARAMS
+
+
+class MultiplexingTablesFormatTest(KikimrSqsTestBase):
+ def _set_new_format_settings(self, username=None, tables_format=1):
+ if username is None:
+ username = self._username
+ self._execute_yql_query(
+ f'UPSERT INTO `{self.sqs_root}/.Settings` (Account, Name, Value) \
+ VALUES ("{username}", "CreateQueuesWithTabletFormat", "{tables_format}")'
+ )
+
+ def create_queue(self, is_fifo):
+ if is_fifo:
+ self.queue_name = self.queue_name + '.fifo'
+ self._create_queue_and_assert(self.queue_name, is_fifo=is_fifo)
+
+ def create_queue_must_fail(self, is_fifo):
+ try:
+ self.create_queue(is_fifo)
+ except yatest.common.process.ExecutionError:
+ return
+ assert(False)
+
+ def create_queue_with_wrong_tables_format(self, tables_format):
+ self._set_new_format_settings(tables_format='qwerty')
+ self.create_queue_must_fail(True)
+ self.create_queue_must_fail(False)
+
+ @pytest.mark.parametrize(**IS_FIFO_PARAMS)
+ def test_create_queue(self, is_fifo):
+ self._set_new_format_settings()
+ self.create_queue(is_fifo)
+
+ def test_create_queue_with_incorrect_tables_format(self):
+ self.create_queue_with_wrong_tables_format('qwerty')
+
+ def test_create_queue_with_empty_tables_format(self):
+ self.create_queue_with_wrong_tables_format('')
+
+ def test_create_queue_with_unsupported_tables_format(self):
+ self.create_queue_with_wrong_tables_format(2)
+
+
+class TestMultiplexingTablesFormatWithTenant(get_test_with_sqs_tenant_installation(MultiplexingTablesFormatTest)):
+ pass
+
+
+class TestMultiplexingTablesFormatWithPath(get_test_with_sqs_installation_by_path(MultiplexingTablesFormatTest)):
+ pass
diff --git a/ydb/tests/functional/sqs/ya.make b/ydb/tests/functional/sqs/ya.make
index 8e8170d042..bbb0a57bb8 100644
--- a/ydb/tests/functional/sqs/ya.make
+++ b/ydb/tests/functional/sqs/ya.make
@@ -17,7 +17,7 @@ TEST_SRCS(
test_generic_messaging.py
test_fifo_messaging.py
test_multinode_cluster.py
- test_multiplexing_tables_format.py
+ test_multiplexing_tables_format.py
test_ping.py
test_polling.py
test_queue_attributes_validation.py
@@ -50,7 +50,7 @@ DEPENDS(
PEERDIR(
ydb/tests/library
- ydb/tests/library/sqs
+ ydb/tests/library/sqs
contrib/python/xmltodict
contrib/python/boto3
contrib/python/botocore
diff --git a/ydb/tests/library/harness/kikimr_client.py b/ydb/tests/library/harness/kikimr_client.py
index 225a50487f..d74383ae23 100644
--- a/ydb/tests/library/harness/kikimr_client.py
+++ b/ydb/tests/library/harness/kikimr_client.py
@@ -74,14 +74,14 @@ class KiKiMRMessageBusClient(object):
request.Options.ReturnPartitioningInfo = False
return self.send(request, 'SchemeDescribe')
- def _get_invoke_callee(self, method):
- return getattr(self._stub, method)
-
+ def _get_invoke_callee(self, method):
+ return getattr(self._stub, method)
+
def invoke(self, request, method):
retry = self.__retry_count
while True:
try:
- callee = self._get_invoke_callee(method)
+ callee = self._get_invoke_callee(method)
return callee(request)
except (RuntimeError, grpc.RpcError):
retry -= 1
diff --git a/ydb/tests/library/ya.make b/ydb/tests/library/ya.make
index 3ab351ae9b..33fbdd5d01 100644
--- a/ydb/tests/library/ya.make
+++ b/ydb/tests/library/ya.make
@@ -96,7 +96,7 @@ PEERDIR(
ydb/public/api/grpc
ydb/public/api/grpc/draft
ydb/public/api/protos
- ydb/tests/library/sqs
+ ydb/tests/library/sqs
)
END()