aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authordinmukhammed <dinmukhammed@yandex-team.ru>2022-02-10 16:51:14 +0300
committerDaniil Cherednik <dcherednik@yandex-team.ru>2022-02-10 16:51:14 +0300
commitf4f0dcffead22e9e9feebedc03057853a44ccc75 (patch)
tree5d5cb817648f650d76cf1076100726fd9b8448e8
parent8435bcab44202d0f1e0eacf7a0535e238b995b09 (diff)
downloadydb-f4f0dcffead22e9e9feebedc03057853a44ccc75.tar.gz
Restoring authorship annotation for <dinmukhammed@yandex-team.ru>. Commit 2 of 2.
-rw-r--r--build/rules/kikimr.policy4
-rw-r--r--ydb/core/base/events.h4
-rw-r--r--ydb/core/driver_lib/run/kikimr_services_initializers.cpp26
-rw-r--r--ydb/core/driver_lib/run/kikimr_services_initializers.h6
-rw-r--r--ydb/core/driver_lib/run/run.cpp6
-rw-r--r--ydb/core/grpc_services/base/base.h8
-rw-r--r--ydb/core/grpc_services/grpc_request_proxy.cpp10
-rw-r--r--ydb/core/grpc_services/rpc_analytics_internal.cpp202
-rw-r--r--ydb/core/grpc_services/service_analytics_internal.h8
-rw-r--r--ydb/core/grpc_services/ya.make2
-rw-r--r--ydb/core/protos/services.proto30
-rw-r--r--ydb/core/testlib/test_client.cpp16
-rw-r--r--ydb/core/yq/libs/actors/clusters_from_connections.cpp34
-rw-r--r--ydb/core/yq/libs/actors/clusters_from_connections.h10
-rw-r--r--ydb/core/yq/libs/actors/database_resolver.cpp186
-rw-r--r--ydb/core/yq/libs/actors/database_resolver.h10
-rw-r--r--ydb/core/yq/libs/actors/error.cpp4
-rw-r--r--ydb/core/yq/libs/actors/nodes_health_check.cpp246
-rw-r--r--ydb/core/yq/libs/actors/nodes_manager.cpp440
-rw-r--r--ydb/core/yq/libs/actors/nodes_manager.h48
-rw-r--r--ydb/core/yq/libs/actors/pending_fetcher.cpp216
-rw-r--r--ydb/core/yq/libs/actors/pinger.cpp38
-rw-r--r--ydb/core/yq/libs/actors/proxy.cpp4
-rw-r--r--ydb/core/yq/libs/actors/proxy.h26
-rw-r--r--ydb/core/yq/libs/actors/proxy_private.cpp198
-rw-r--r--ydb/core/yq/libs/actors/proxy_private.h98
-rw-r--r--ydb/core/yq/libs/actors/result_writer.cpp328
-rw-r--r--ydb/core/yq/libs/actors/run_actor.cpp264
-rw-r--r--ydb/core/yq/libs/actors/run_actor_params.cpp76
-rw-r--r--ydb/core/yq/libs/actors/run_actor_params.h86
-rw-r--r--ydb/core/yq/libs/actors/system_clusters.cpp10
-rw-r--r--ydb/core/yq/libs/actors/system_clusters.h10
-rw-r--r--ydb/core/yq/libs/actors/table_bindings_from_bindings.cpp14
-rw-r--r--ydb/core/yq/libs/actors/table_bindings_from_bindings.h8
-rw-r--r--ydb/core/yq/libs/actors/task_get.cpp400
-rw-r--r--ydb/core/yq/libs/actors/task_ping.cpp376
-rw-r--r--ydb/core/yq/libs/actors/task_result_write.cpp264
-rw-r--r--ydb/core/yq/libs/actors/ya.make10
-rw-r--r--ydb/core/yq/libs/checkpoint_storage/gc.cpp2
-rw-r--r--ydb/core/yq/libs/checkpoint_storage/storage_proxy.cpp2
-rw-r--r--ydb/core/yq/libs/checkpointing/checkpoint_coordinator.h4
-rw-r--r--ydb/core/yq/libs/common/cache.h4
-rw-r--r--ydb/core/yq/libs/common/cache_ut.cpp2
-rw-r--r--ydb/core/yq/libs/common/database_token_builder.cpp84
-rw-r--r--ydb/core/yq/libs/common/database_token_builder.h38
-rw-r--r--ydb/core/yq/libs/common/entity_id.cpp4
-rw-r--r--ydb/core/yq/libs/common/entity_id.h4
-rw-r--r--ydb/core/yq/libs/common/entity_id_ut.cpp2
-rw-r--r--ydb/core/yq/libs/common/rows_proto_splitter.cpp154
-rw-r--r--ydb/core/yq/libs/common/rows_proto_splitter.h74
-rw-r--r--ydb/core/yq/libs/common/rows_proto_splitter_ut.cpp226
-rw-r--r--ydb/core/yq/libs/common/ut/ya.make2
-rw-r--r--ydb/core/yq/libs/common/ya.make26
-rw-r--r--ydb/core/yq/libs/config/protos/common.proto2
-rw-r--r--ydb/core/yq/libs/config/protos/gateways.proto2
-rw-r--r--ydb/core/yq/libs/config/protos/read_actors_factory.proto34
-rw-r--r--ydb/core/yq/libs/config/protos/ya.make4
-rw-r--r--ydb/core/yq/libs/config/protos/yq_config.proto6
-rw-r--r--ydb/core/yq/libs/control_plane_proxy/control_plane_proxy.cpp4
-rw-r--r--ydb/core/yq/libs/control_plane_storage/control_plane_storage_counters.cpp114
-rw-r--r--ydb/core/yq/libs/control_plane_storage/control_plane_storage_counters.h78
-rw-r--r--ydb/core/yq/libs/control_plane_storage/events/events.h140
-rw-r--r--ydb/core/yq/libs/control_plane_storage/in_memory_control_plane_storage.cpp2
-rw-r--r--ydb/core/yq/libs/control_plane_storage/internal/nodes_health_check.cpp132
-rw-r--r--ydb/core/yq/libs/control_plane_storage/internal/response_tasks.cpp94
-rw-r--r--ydb/core/yq/libs/control_plane_storage/internal/response_tasks.h54
-rw-r--r--ydb/core/yq/libs/control_plane_storage/internal/task_get.cpp480
-rw-r--r--ydb/core/yq/libs/control_plane_storage/internal/task_ping.cpp526
-rw-r--r--ydb/core/yq/libs/control_plane_storage/internal/task_result_write.cpp134
-rw-r--r--ydb/core/yq/libs/control_plane_storage/internal/utils.cpp142
-rw-r--r--ydb/core/yq/libs/control_plane_storage/internal/utils.h42
-rw-r--r--ydb/core/yq/libs/control_plane_storage/internal/ya.make42
-rw-r--r--ydb/core/yq/libs/control_plane_storage/message_builders.h136
-rw-r--r--ydb/core/yq/libs/control_plane_storage/probes.h10
-rw-r--r--ydb/core/yq/libs/control_plane_storage/proto/yq_internal.proto2
-rw-r--r--ydb/core/yq/libs/control_plane_storage/schema.h14
-rw-r--r--ydb/core/yq/libs/control_plane_storage/util.cpp112
-rw-r--r--ydb/core/yq/libs/control_plane_storage/util.h12
-rw-r--r--ydb/core/yq/libs/control_plane_storage/ya.make2
-rw-r--r--ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage.cpp260
-rw-r--r--ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage_bindings.cpp12
-rw-r--r--ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage_connections.cpp28
-rw-r--r--ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage_impl.h132
-rw-r--r--ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage_queries.cpp236
-rw-r--r--ydb/core/yq/libs/db_resolver/db_async_resolver.h42
-rw-r--r--ydb/core/yq/libs/db_resolver/db_async_resolver_impl.cpp82
-rw-r--r--ydb/core/yq/libs/db_resolver/db_async_resolver_impl.h54
-rw-r--r--ydb/core/yq/libs/db_resolver/db_async_resolver_with_meta.cpp138
-rw-r--r--ydb/core/yq/libs/db_resolver/db_async_resolver_with_meta.h92
-rw-r--r--ydb/core/yq/libs/db_resolver/ya.make28
-rw-r--r--ydb/core/yq/libs/db_schema/db_schema.cpp196
-rw-r--r--ydb/core/yq/libs/db_schema/db_schema.h28
-rw-r--r--ydb/core/yq/libs/events/event_ids.h106
-rw-r--r--ydb/core/yq/libs/events/events.h158
-rw-r--r--ydb/core/yq/libs/events/ya.make2
-rw-r--r--ydb/core/yq/libs/init/init.cpp172
-rw-r--r--ydb/core/yq/libs/init/init.h8
-rw-r--r--ydb/core/yq/libs/init/ya.make2
-rw-r--r--ydb/core/yq/libs/logs/log.cpp4
-rw-r--r--ydb/core/yq/libs/mock/yql_mock.cpp6
-rw-r--r--ydb/core/yq/libs/private_client/private_client.cpp352
-rw-r--r--ydb/core/yq/libs/private_client/private_client.h154
-rw-r--r--ydb/core/yq/libs/private_client/utils.cpp104
-rw-r--r--ydb/core/yq/libs/private_client/utils.h14
-rw-r--r--ydb/core/yq/libs/private_client/ya.make8
-rw-r--r--ydb/core/yq/libs/read_rule/read_rule_creator.cpp12
-rw-r--r--ydb/core/yq/libs/read_rule/read_rule_deleter.cpp22
-rw-r--r--ydb/core/yq/libs/read_rule/read_rule_deleter.h2
-rw-r--r--ydb/core/yq/libs/result_formatter/result_formatter.cpp12
-rw-r--r--ydb/core/yq/libs/result_formatter/result_formatter.h4
-rw-r--r--ydb/core/yq/libs/result_formatter/result_formatter_ut.cpp2
-rw-r--r--ydb/core/yq/libs/result_formatter/ut/ya.make34
-rw-r--r--ydb/core/yq/libs/result_formatter/ya.make32
-rw-r--r--ydb/core/yq/libs/shared_resources/db_pool.cpp76
-rw-r--r--ydb/core/yq/libs/shared_resources/db_pool.h24
-rw-r--r--ydb/core/yq/libs/shared_resources/shared_resources.cpp18
-rw-r--r--ydb/core/yq/libs/shared_resources/shared_resources.h6
-rw-r--r--ydb/core/yq/libs/shared_resources/ya.make2
-rw-r--r--ydb/core/yq/libs/ya.make8
-rw-r--r--ydb/library/mkql_proto/mkql_proto.cpp22
-rw-r--r--ydb/library/yql/dq/actors/compute/dq_compute_actor_checkpoints.h4
-rw-r--r--ydb/library/yql/providers/clickhouse/provider/ya.make6
-rw-r--r--ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_datasource.cpp10
-rw-r--r--ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_io_discovery.cpp222
-rw-r--r--ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_provider.cpp10
-rw-r--r--ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_provider.h10
-rw-r--r--ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_provider_impl.h2
-rw-r--r--ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_settings.h52
-rw-r--r--ydb/library/yql/providers/common/http_gateway/ya.make6
-rw-r--r--ydb/library/yql/providers/common/http_gateway/yql_http_gateway.cpp78
-rw-r--r--ydb/library/yql/providers/common/http_gateway/yql_http_gateway.h12
-rw-r--r--ydb/library/yql/providers/common/proto/gateways_config.proto18
-rw-r--r--ydb/library/yql/providers/dq/actors/worker_actor.cpp2
-rw-r--r--ydb/library/yql/providers/dq/api/protos/dqs.proto2
-rw-r--r--ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp30
-rw-r--r--ydb/library/yql/providers/dq/provider/yql_dq_gateway.h2
-rw-r--r--ydb/library/yql/providers/pq/async_io/dq_pq_read_actor.cpp4
-rw-r--r--ydb/library/yql/providers/pq/async_io/dq_pq_write_actor.cpp4
-rw-r--r--ydb/library/yql/providers/pq/gateway/dummy/yql_pq_dummy_gateway.cpp24
-rw-r--r--ydb/library/yql/providers/pq/gateway/dummy/yql_pq_dummy_gateway.h12
-rw-r--r--ydb/library/yql/providers/pq/gateway/native/yql_pq_gateway.cpp44
-rw-r--r--ydb/library/yql/providers/pq/provider/ya.make2
-rw-r--r--ydb/library/yql/providers/pq/provider/yql_pq_datasink.cpp12
-rw-r--r--ydb/library/yql/providers/pq/provider/yql_pq_datasink_io_discovery.cpp134
-rw-r--r--ydb/library/yql/providers/pq/provider/yql_pq_gateway.h12
-rw-r--r--ydb/library/yql/providers/pq/provider/yql_pq_helpers.cpp154
-rw-r--r--ydb/library/yql/providers/pq/provider/yql_pq_helpers.h20
-rw-r--r--ydb/library/yql/providers/pq/provider/yql_pq_io_discovery.cpp56
-rw-r--r--ydb/library/yql/providers/pq/provider/yql_pq_provider.cpp14
-rw-r--r--ydb/library/yql/providers/pq/provider/yql_pq_provider.h14
-rw-r--r--ydb/library/yql/providers/pq/provider/yql_pq_provider_impl.h2
-rw-r--r--ydb/library/yql/providers/pq/provider/yql_pq_settings.cpp110
-rw-r--r--ydb/library/yql/providers/pq/provider/yql_pq_settings.h16
-rw-r--r--ydb/library/yql/providers/s3/actors/yql_s3_read_actor.cpp164
-rw-r--r--ydb/library/yql/providers/s3/actors/yql_s3_read_actor.h4
-rw-r--r--ydb/library/yql/providers/s3/actors/yql_s3_source_factory.cpp14
-rw-r--r--ydb/library/yql/providers/s3/actors/yql_s3_source_factory.h14
-rw-r--r--ydb/library/yql/providers/s3/proto/retry_config.proto20
-rw-r--r--ydb/library/yql/providers/s3/proto/ya.make2
-rw-r--r--ydb/library/yql/providers/s3/provider/yql_s3_io_discovery.cpp64
-rw-r--r--ydb/library/yql/providers/solomon/async_io/dq_solomon_write_actor.cpp4
-rw-r--r--ydb/library/yql/providers/ydb/actors/yql_ydb_read_actor.cpp6
-rw-r--r--ydb/library/yql/providers/ydb/provider/ya.make6
-rw-r--r--ydb/library/yql/providers/ydb/provider/yql_ydb_datasource.cpp24
-rw-r--r--ydb/library/yql/providers/ydb/provider/yql_ydb_io_discovery.cpp194
-rw-r--r--ydb/library/yql/providers/ydb/provider/yql_ydb_load_meta.cpp82
-rw-r--r--ydb/library/yql/providers/ydb/provider/yql_ydb_provider.cpp14
-rw-r--r--ydb/library/yql/providers/ydb/provider/yql_ydb_provider.h26
-rw-r--r--ydb/library/yql/providers/ydb/provider/yql_ydb_provider_impl.h2
-rw-r--r--ydb/library/yql/providers/ydb/provider/yql_ydb_settings.cpp32
-rw-r--r--ydb/library/yql/providers/ydb/provider/yql_ydb_settings.h20
-rw-r--r--ydb/library/yql/utils/actor_log/log.cpp20
-rw-r--r--ydb/library/yql/utils/actor_log/log.h42
-rw-r--r--ydb/public/api/grpc/draft/yql_db_v1.proto16
-rw-r--r--ydb/public/api/protos/draft/yq_private.proto150
-rw-r--r--ydb/public/api/protos/ya.make2
-rw-r--r--ydb/public/api/protos/yq.proto4
-rw-r--r--ydb/services/yq/private_grpc.cpp104
-rw-r--r--ydb/services/yq/private_grpc.h62
-rw-r--r--ydb/services/yq/ut_integration/ut_utils.cpp14
-rw-r--r--ydb/services/yq/ut_integration/ut_utils.h4
-rw-r--r--ydb/services/yq/ut_integration/ya.make4
-rw-r--r--ydb/services/yq/ut_integration/yq_ut.cpp1234
-rw-r--r--ydb/services/yq/ya.make4
184 files changed, 6410 insertions, 6410 deletions
diff --git a/build/rules/kikimr.policy b/build/rules/kikimr.policy
index 6cb346dc9b..5bb6426e30 100644
--- a/build/rules/kikimr.policy
+++ b/build/rules/kikimr.policy
@@ -146,10 +146,10 @@ ALLOW ydb/library/yql/providers/dq/worker_manager -> kikimr/core/kqp
ALLOW ydb/library/yql/providers/clickhouse/provider -> kikimr/yq/libs/db_resolver
ALLOW ydb/library/yql/providers/clickhouse/provider -> kikimr/yq/libs/common
-
+
ALLOW ydb/library/yql/providers/pq/provider -> kikimr/yq/libs/db_resolver
ALLOW ydb/library/yql/providers/pq/provider -> kikimr/yq/libs/common
-
+
ALLOW yql/tools/yqlworker/dq/worker_manager -> kikimr/core/kqp
ALLOW ydb/library/yql/dq/actors/compute -> kikimr/core/kqp/runtime
diff --git a/ydb/core/base/events.h b/ydb/core/base/events.h
index 10e6cfd323..f5fedfe19b 100644
--- a/ydb/core/base/events.h
+++ b/ydb/core/base/events.h
@@ -6,7 +6,7 @@
#include <ydb/library/yql/dq/actors/dq_events_ids.h>
#include <ydb/core/yq/libs/events/event_ids.h>
-
+
namespace NKikimr {
struct TKikimrEvents : TEvents {
@@ -126,7 +126,7 @@ struct TKikimrEvents : TEvents {
ES_TX_COLUMNSHARD,
ES_CROSSREF,
ES_SCHEME_BOARD_MON,
- ES_YQL_ANALYTICS_PROXY = NYq::TEventIds::ES_YQL_ANALYTICS_PROXY,
+ ES_YQL_ANALYTICS_PROXY = NYq::TEventIds::ES_YQL_ANALYTICS_PROXY,
ES_BLOB_CACHE,
ES_LONG_TX_SERVICE,
ES_TEST_SHARD,
diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp
index 5fffe37aa4..819c1478d1 100644
--- a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp
+++ b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp
@@ -190,8 +190,8 @@ namespace NKikimr {
namespace NKikimrServicesInitializers {
-ui32 TYandexQueryInitializer::IcPort = 0;
-
+ui32 TYandexQueryInitializer::IcPort = 0;
+
IKikimrServicesInitializer::IKikimrServicesInitializer(const TKikimrRunConfig& runConfig)
: Config(runConfig.AppConfig)
, NodeId(runConfig.NodeId)
@@ -594,7 +594,7 @@ void TBasicServicesInitializer::InitializeServices(NActors::TActorSystemSetup* s
case NKikimrConfig::TStaticNameserviceConfig::NS_EXTERNAL:
nameservice = NActors::CreateDynamicNameserver(table, TDuration::Seconds(3), appData->IOPoolId);
break;
- }
+ }
setup->LocalServices.emplace_back(
nameserviceId,
@@ -707,12 +707,12 @@ void TBasicServicesInitializer::InitializeServices(NActors::TActorSystemSetup* s
setup->Interconnect.ProxyActors.resize(maxNode + 1);
setup->Interconnect.ProxyWrapperFactory = CreateProxyWrapperFactory(icCommon, interconnectPoolId);
- std::unordered_set<ui32> staticIds;
-
- for (const auto& node : table->StaticNodeTable) {
+ std::unordered_set<ui32> staticIds;
+
+ for (const auto& node : table->StaticNodeTable) {
const ui32 destId = node.first;
if (destId != NodeId) {
- staticIds.insert(destId);
+ staticIds.insert(destId);
setup->Interconnect.ProxyActors[destId] = TActorSetupCmd(new TInterconnectProxyTCP(destId, icCommon),
TMailboxType::ReadAsFilled, interconnectPoolId);
} else {
@@ -2257,10 +2257,10 @@ TYandexQueryInitializer::TYandexQueryInitializer(const TKikimrRunConfig& runConf
{
}
-void TYandexQueryInitializer::SetIcPort(ui32 icPort) {
- IcPort = icPort;
-}
-
+void TYandexQueryInitializer::SetIcPort(ui32 icPort) {
+ IcPort = icPort;
+}
+
void TYandexQueryInitializer::InitializeServices(TActorSystemSetup* setup, const TAppData* appData) {
const auto& protoConfig = Config.GetYandexQueryConfig();
if (!protoConfig.GetEnabled()) {
@@ -2292,8 +2292,8 @@ void TYandexQueryInitializer::InitializeServices(TActorSystemSetup* setup, const
YqSharedResources,
Factories->FolderServiceFactory,
Factories->YqAuditServiceFactory,
- Factories->YdbCredentialProviderFactory,
- IcPort
+ Factories->YdbCredentialProviderFactory,
+ IcPort
);
}
diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.h b/ydb/core/driver_lib/run/kikimr_services_initializers.h
index d2362dc0f0..407ce1bb7b 100644
--- a/ydb/core/driver_lib/run/kikimr_services_initializers.h
+++ b/ydb/core/driver_lib/run/kikimr_services_initializers.h
@@ -491,12 +491,12 @@ public:
TYandexQueryInitializer(const TKikimrRunConfig& runConfig, std::shared_ptr<TModuleFactories> factories, NYq::IYqSharedResources::TPtr yqSharedResources);
void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override;
-
- static void SetIcPort(ui32 icPort);
+
+ static void SetIcPort(ui32 icPort);
private:
std::shared_ptr<TModuleFactories> Factories;
NYq::IYqSharedResources::TPtr YqSharedResources;
- static ui32 IcPort;
+ static ui32 IcPort;
};
} // namespace NKikimrServicesInitializers
diff --git a/ydb/core/driver_lib/run/run.cpp b/ydb/core/driver_lib/run/run.cpp
index c2405fc86d..a4f74aa4e0 100644
--- a/ydb/core/driver_lib/run/run.cpp
+++ b/ydb/core/driver_lib/run/run.cpp
@@ -715,7 +715,7 @@ void TKikimrRunner::InitializeGRpc(const TKikimrRunConfig& runConfig) {
if (hasYandexQuery) {
server.AddService(new NGRpcService::TGRpcYandexQueryService(ActorSystem.Get(), Counters, grpcRequestProxyId));
- server.AddService(new NGRpcService::TGRpcYqPrivateTaskService(ActorSystem.Get(), Counters, grpcRequestProxyId));
+ server.AddService(new NGRpcService::TGRpcYqPrivateTaskService(ActorSystem.Get(), Counters, grpcRequestProxyId));
}
if (hasLogStore) {
@@ -1316,8 +1316,8 @@ TIntrusivePtr<TServiceInitializersList> TKikimrRunner::CreateServiceInitializers
}
if (serviceMask.EnableYandexQuery && runConfig.AppConfig.GetYandexQueryConfig().GetEnabled()) {
- YqSharedResources = NYq::CreateYqSharedResources(runConfig.AppConfig.GetYandexQueryConfig(),
- ModuleFactories->YdbCredentialProviderFactory, Counters->GetSubgroup("counters", "yq"));
+ YqSharedResources = NYq::CreateYqSharedResources(runConfig.AppConfig.GetYandexQueryConfig(),
+ ModuleFactories->YdbCredentialProviderFactory, Counters->GetSubgroup("counters", "yq"));
sil->AddServiceInitializer(new TYandexQueryInitializer(runConfig, ModuleFactories, YqSharedResources));
}
diff --git a/ydb/core/grpc_services/base/base.h b/ydb/core/grpc_services/base/base.h
index dc7ca0bfd9..44b25c4a5f 100644
--- a/ydb/core/grpc_services/base/base.h
+++ b/ydb/core/grpc_services/base/base.h
@@ -198,10 +198,10 @@ struct TRpcServices {
EvDropLogTable,
EvAlterLogTable,
EvLogin,
- EvAnalyticsInternalPingTask,
- EvAnalyticsInternalGetTask,
- EvAnalyticsInternalWriteTaskResult,
- EvAnalyticsInternalNodesHealthCheck,
+ EvAnalyticsInternalPingTask,
+ EvAnalyticsInternalGetTask,
+ EvAnalyticsInternalWriteTaskResult,
+ EvAnalyticsInternalNodesHealthCheck,
EvCreateYndxRateLimiterResource,
EvAlterYndxRateLimiterResource,
EvDropYndxRateLimiterResource,
diff --git a/ydb/core/grpc_services/grpc_request_proxy.cpp b/ydb/core/grpc_services/grpc_request_proxy.cpp
index 2b5e5ba458..a1e7accb87 100644
--- a/ydb/core/grpc_services/grpc_request_proxy.cpp
+++ b/ydb/core/grpc_services/grpc_request_proxy.cpp
@@ -172,14 +172,14 @@ private:
Handle(event, ctx);
return;
}
-
+
auto state = requestBaseCtx->GetAuthState();
if (state.State == NGrpc::TAuthState::AS_FAIL) {
requestBaseCtx->ReplyUnauthenticated();
return;
}
-
+
if (state.State == NGrpc::TAuthState::AS_UNAVAILABLE) {
Counters->IncDatabaseUnavailableCounter();
const TString error = "Unable to resolve token";
@@ -317,7 +317,7 @@ void TGRpcRequestProxyImpl::Bootstrap(const TActorContext& ctx) {
}
Counters = MakeIntrusive<TGrpcProxyCounters>(AppData()->Counters);
-
+
RootDatabase = DatabaseFromDomain();
TDatabaseInfo& database = Databases[RootDatabase];
database.DatabaseType = TDatabaseInfo::TDatabaseType::Root;
@@ -334,7 +334,7 @@ void TGRpcRequestProxyImpl::ReplayEvents(const TString& databaseName, const TAct
std::deque<TEventReqHolder> deferredEvents;
std::swap(deferredEvents, queue); // we can put back event to DeferredEvents queue in StateFunc KIKIMR-12851
while (!deferredEvents.empty()) {
- StateFunc(deferredEvents.front().Ev, ctx);
+ StateFunc(deferredEvents.front().Ev, ctx);
deferredEvents.pop_front();
}
if (queue.empty()) {
@@ -552,7 +552,7 @@ void TGRpcRequestProxyImpl::StateFunc(TAutoPtr<IEventHandle>& ev, const TActorCo
hFunc(TEvents::TEvUndelivered, HandleUndelivery);
HFunc(TSchemeBoardEvents::TEvNotifyUpdate, HandleSchemeBoard);
hFunc(TSchemeBoardEvents::TEvNotifyDelete, HandleSchemeBoard);
-
+
default:
handled = false;
break;
diff --git a/ydb/core/grpc_services/rpc_analytics_internal.cpp b/ydb/core/grpc_services/rpc_analytics_internal.cpp
index 45cec13a64..48227a57fd 100644
--- a/ydb/core/grpc_services/rpc_analytics_internal.cpp
+++ b/ydb/core/grpc_services/rpc_analytics_internal.cpp
@@ -1,108 +1,108 @@
#include "service_analytics_internal.h"
-#include "rpc_common.h"
-#include "rpc_deferrable.h"
-
+#include "rpc_common.h"
+#include "rpc_deferrable.h"
+
#include <ydb/core/yq/libs/events/events.h>
#include <ydb/core/yq/libs/actors/proxy_private.h>
-
-#include <library/cpp/actors/core/hfunc.h>
-
+
+#include <library/cpp/actors/core/hfunc.h>
+
#include <ydb/core/grpc_services/base/base.h>
#include <ydb/public/api/protos/draft/yq_private.pb.h>
-namespace NKikimr {
-namespace NGRpcService {
-
-using TEvYqPrivatePingTaskRequest =
- TGrpcRequestOperationCall<Yq::Private::PingTaskRequest, Yq::Private::PingTaskResponse>;
-using TEvYqPrivateGetTaskRequest =
- TGrpcRequestOperationCall<Yq::Private::GetTaskRequest, Yq::Private::GetTaskResponse>;
-using TEvYqPrivateWriteTaskResultRequest =
- TGrpcRequestOperationCall<Yq::Private::WriteTaskResultRequest, Yq::Private::WriteTaskResultResponse>;
-using TEvYqPrivateNodesHealthCheckRequest =
- TGrpcRequestOperationCall<Yq::Private::NodesHealthCheckRequest, Yq::Private::NodesHealthCheckResponse>;
-
-namespace {
- template <typename TEv, typename TReq>
- void SendResponse(const TEv& ev, TReq& req) {
- if (!ev->Get()->Record) {
+namespace NKikimr {
+namespace NGRpcService {
+
+using TEvYqPrivatePingTaskRequest =
+ TGrpcRequestOperationCall<Yq::Private::PingTaskRequest, Yq::Private::PingTaskResponse>;
+using TEvYqPrivateGetTaskRequest =
+ TGrpcRequestOperationCall<Yq::Private::GetTaskRequest, Yq::Private::GetTaskResponse>;
+using TEvYqPrivateWriteTaskResultRequest =
+ TGrpcRequestOperationCall<Yq::Private::WriteTaskResultRequest, Yq::Private::WriteTaskResultResponse>;
+using TEvYqPrivateNodesHealthCheckRequest =
+ TGrpcRequestOperationCall<Yq::Private::NodesHealthCheckRequest, Yq::Private::NodesHealthCheckResponse>;
+
+namespace {
+ template <typename TEv, typename TReq>
+ void SendResponse(const TEv& ev, TReq& req) {
+ if (!ev->Get()->Record) {
req.RaiseIssues(ev->Get()->Issues);
- req.ReplyWithYdbStatus(ev->Get()->Status);
- } else {
- req.SendResult(*ev->Get()->Record, ev->Get()->Status);
- }
- }
-
-}
-
-template <typename RpcRequestType, typename EvRequestType, typename EvResponseType>
-class TYqPrivateRequestRPC : public TRpcOperationRequestActor<
- TYqPrivateRequestRPC<RpcRequestType,EvRequestType,EvResponseType>, RpcRequestType> {
-
- using TBase = TRpcOperationRequestActor<
- TYqPrivateRequestRPC<RpcRequestType,EvRequestType,EvResponseType>,
- RpcRequestType>;
-
-public:
- TYqPrivateRequestRPC(IRequestOpCtx* request) : TBase(request) {}
-
- void Bootstrap(const TActorContext& ctx) {
- Y_UNUSED(ctx);
- const auto req = this->GetProtoRequest();
- auto ev = MakeHolder<EvRequestType>();
- auto request = dynamic_cast<RpcRequestType*>(this->Request_.get());
- Y_VERIFY(request);
- ev->Record = *req;
- this->Send(NYq::MakeYqPrivateProxyId(), ev.Release());
- this->Become(&TYqPrivateRequestRPC<RpcRequestType, EvRequestType, EvResponseType>::StateFunc);
- }
-
-private:
- STRICT_STFUNC(StateFunc,
- HFunc(EvResponseType, Handle);
- )
-
- void Handle(typename EvResponseType::TPtr& ev, const TActorContext& ctx) {
- SendResponse(ev, *this->Request_);
- this->Die(ctx);
- }
-};
-
-using TYqPrivatePingTaskRPC = TYqPrivateRequestRPC<
- TEvYqPrivatePingTaskRequest,
- NYq::TEvents::TEvPingTaskRequest,
- NYq::TEvents::TEvPingTaskResponse>;
-
-using TYqPrivateGetTaskRPC = TYqPrivateRequestRPC<
- TEvYqPrivateGetTaskRequest,
- NYq::TEvents::TEvGetTaskRequest,
- NYq::TEvents::TEvGetTaskResponse>;
-
-using TYqPrivateWriteTaskResultRPC = TYqPrivateRequestRPC<
- TEvYqPrivateWriteTaskResultRequest,
- NYq::TEvents::TEvWriteTaskResultRequest,
- NYq::TEvents::TEvWriteTaskResultResponse>;
-
-using TYqPrivateNodesHealthCheckRPC = TYqPrivateRequestRPC<
- TEvYqPrivateNodesHealthCheckRequest,
- NYq::TEvents::TEvNodesHealthCheckRequest,
- NYq::TEvents::TEvNodesHealthCheckResponse>;
-
-void DoYqPrivatePingTaskRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&) {
- TActivationContext::AsActorContext().Register(new TYqPrivatePingTaskRPC(p.release()));
-}
-
-void DoYqPrivateGetTaskRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&) {
- TActivationContext::AsActorContext().Register(new TYqPrivateGetTaskRPC(p.release()));
-}
-
-void DoYqPrivateWriteTaskResultRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&) {
- TActivationContext::AsActorContext().Register(new TYqPrivateWriteTaskResultRPC(p.release()));
-}
-
-void DoYqPrivateNodesHealthCheckRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&) {
- TActivationContext::AsActorContext().Register(new TYqPrivateNodesHealthCheckRPC(p.release()));
-}
-
-} // namespace NGRpcService
-} // namespace NKikimr
+ req.ReplyWithYdbStatus(ev->Get()->Status);
+ } else {
+ req.SendResult(*ev->Get()->Record, ev->Get()->Status);
+ }
+ }
+
+}
+
+template <typename RpcRequestType, typename EvRequestType, typename EvResponseType>
+class TYqPrivateRequestRPC : public TRpcOperationRequestActor<
+ TYqPrivateRequestRPC<RpcRequestType,EvRequestType,EvResponseType>, RpcRequestType> {
+
+ using TBase = TRpcOperationRequestActor<
+ TYqPrivateRequestRPC<RpcRequestType,EvRequestType,EvResponseType>,
+ RpcRequestType>;
+
+public:
+ TYqPrivateRequestRPC(IRequestOpCtx* request) : TBase(request) {}
+
+ void Bootstrap(const TActorContext& ctx) {
+ Y_UNUSED(ctx);
+ const auto req = this->GetProtoRequest();
+ auto ev = MakeHolder<EvRequestType>();
+ auto request = dynamic_cast<RpcRequestType*>(this->Request_.get());
+ Y_VERIFY(request);
+ ev->Record = *req;
+ this->Send(NYq::MakeYqPrivateProxyId(), ev.Release());
+ this->Become(&TYqPrivateRequestRPC<RpcRequestType, EvRequestType, EvResponseType>::StateFunc);
+ }
+
+private:
+ STRICT_STFUNC(StateFunc,
+ HFunc(EvResponseType, Handle);
+ )
+
+ void Handle(typename EvResponseType::TPtr& ev, const TActorContext& ctx) {
+ SendResponse(ev, *this->Request_);
+ this->Die(ctx);
+ }
+};
+
+using TYqPrivatePingTaskRPC = TYqPrivateRequestRPC<
+ TEvYqPrivatePingTaskRequest,
+ NYq::TEvents::TEvPingTaskRequest,
+ NYq::TEvents::TEvPingTaskResponse>;
+
+using TYqPrivateGetTaskRPC = TYqPrivateRequestRPC<
+ TEvYqPrivateGetTaskRequest,
+ NYq::TEvents::TEvGetTaskRequest,
+ NYq::TEvents::TEvGetTaskResponse>;
+
+using TYqPrivateWriteTaskResultRPC = TYqPrivateRequestRPC<
+ TEvYqPrivateWriteTaskResultRequest,
+ NYq::TEvents::TEvWriteTaskResultRequest,
+ NYq::TEvents::TEvWriteTaskResultResponse>;
+
+using TYqPrivateNodesHealthCheckRPC = TYqPrivateRequestRPC<
+ TEvYqPrivateNodesHealthCheckRequest,
+ NYq::TEvents::TEvNodesHealthCheckRequest,
+ NYq::TEvents::TEvNodesHealthCheckResponse>;
+
+void DoYqPrivatePingTaskRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&) {
+ TActivationContext::AsActorContext().Register(new TYqPrivatePingTaskRPC(p.release()));
+}
+
+void DoYqPrivateGetTaskRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&) {
+ TActivationContext::AsActorContext().Register(new TYqPrivateGetTaskRPC(p.release()));
+}
+
+void DoYqPrivateWriteTaskResultRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&) {
+ TActivationContext::AsActorContext().Register(new TYqPrivateWriteTaskResultRPC(p.release()));
+}
+
+void DoYqPrivateNodesHealthCheckRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&) {
+ TActivationContext::AsActorContext().Register(new TYqPrivateNodesHealthCheckRPC(p.release()));
+}
+
+} // namespace NGRpcService
+} // namespace NKikimr
diff --git a/ydb/core/grpc_services/service_analytics_internal.h b/ydb/core/grpc_services/service_analytics_internal.h
index 5fee70f346..f3d1ed966d 100644
--- a/ydb/core/grpc_services/service_analytics_internal.h
+++ b/ydb/core/grpc_services/service_analytics_internal.h
@@ -8,10 +8,10 @@ namespace NGRpcService {
class IRequestOpCtx;
class IFacilityProvider;
-void DoYqPrivatePingTaskRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider& facility);
-void DoYqPrivateGetTaskRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider& facility);
-void DoYqPrivateWriteTaskResultRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider& facility);
-void DoYqPrivateNodesHealthCheckRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider& facility);
+void DoYqPrivatePingTaskRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider& facility);
+void DoYqPrivateGetTaskRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider& facility);
+void DoYqPrivateWriteTaskResultRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider& facility);
+void DoYqPrivateNodesHealthCheckRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider& facility);
} // namespace NGRpcService
} // namespace NKikimr
diff --git a/ydb/core/grpc_services/ya.make b/ydb/core/grpc_services/ya.make
index c364c80745..05b156bf04 100644
--- a/ydb/core/grpc_services/ya.make
+++ b/ydb/core/grpc_services/ya.make
@@ -72,7 +72,7 @@ SRCS(
rpc_yq.cpp
table_profiles.cpp
table_settings.cpp
- rpc_analytics_internal.cpp
+ rpc_analytics_internal.cpp
)
PEERDIR(
diff --git a/ydb/core/protos/services.proto b/ydb/core/protos/services.proto
index eb568d5e1f..c17c8a7dc3 100644
--- a/ydb/core/protos/services.proto
+++ b/ydb/core/protos/services.proto
@@ -262,10 +262,10 @@ enum EServiceKikimr {
YQL_PROXY = 713;
- YQL_PRIVATE_PROXY = 714;
-
- YQL_NODES_MANAGER = 715;
-
+ YQL_PRIVATE_PROXY = 714;
+
+ YQL_NODES_MANAGER = 715;
+
// Exports (& imports)
EXPORT = 800;
DATASHARD_RESTORE = 801;
@@ -829,9 +829,9 @@ message TActivity {
CHANGE_SENDER_ACTOR = 524;
CHANGE_SENDER_ASYNC_INDEX_ACTOR_MAIN = 525;
CHANGE_SENDER_ASYNC_INDEX_ACTOR_PARTITION = 526;
- YQL_GET_CONNECTIONS_REQUEST_ACTOR = 527;
- YQL_MODIFY_CONNECTIONS_REQUEST_ACTOR = 528;
- YQL_MODIFY_HISTORY_REQUEST_ACTOR = 529;
+ YQL_GET_CONNECTIONS_REQUEST_ACTOR = 527;
+ YQL_MODIFY_CONNECTIONS_REQUEST_ACTOR = 528;
+ YQL_MODIFY_HISTORY_REQUEST_ACTOR = 529;
YQL_WRITE_RESULT_DATA_ACTOR = 530;
BS_SCRUB_ACTOR = 531;
BS_BLOB_RECOVERY_ACTOR = 532;
@@ -839,19 +839,19 @@ message TActivity {
YQ_CONTROL_PLANE_STORAGE_ACTOR = 534;
YQ_CONTROL_PLANE_PROXY_ACTOR = 535;
DB_WATCHER_ACTOR = 495;
- YQL_PENDING_FETCHER_ACTOR = 536;
- YQL_PINGER_ACTOR = 537;
- YQL_PRIVATE_PROXY_ACTOR = 538;
- YQL_PRIVATE_PING_TASK_ACTOR = 539;
+ YQL_PENDING_FETCHER_ACTOR = 536;
+ YQL_PINGER_ACTOR = 537;
+ YQL_PRIVATE_PROXY_ACTOR = 538;
+ YQL_PRIVATE_PING_TASK_ACTOR = 539;
EXT_COUNTERS_UPDATER_ACTOR = 540;
- YQL_PRIVATE_GET_TASK_ACTOR = 541;
- YQL_PRIVATE_WRITE_TASK_ACTOR = 542;
- YQL_NODES_MANAGER_ACTOR = 543;
+ YQL_PRIVATE_GET_TASK_ACTOR = 541;
+ YQL_PRIVATE_WRITE_TASK_ACTOR = 542;
+ YQL_NODES_MANAGER_ACTOR = 543;
KESUS_ACCOUNTING_ACTOR = 544;
BLOCKSTORE_PARTITION_COMMON = 545;
INTERCONNECT_PROXY_WRAPPER = 546;
CLOUD_STORAGE_HIVE_PROXY = 547;
- YQL_PRIVATE_NODES_HEALTH_CHECK_ACTOR = 548;
+ YQL_PRIVATE_NODES_HEALTH_CHECK_ACTOR = 548;
CHANGE_SENDER_CDC_ACTOR_MAIN = 549;
KQP_TEST_WORKLOAD = 550;
PQ_PARTITION_WRITER_ACTOR = 551;
diff --git a/ydb/core/testlib/test_client.cpp b/ydb/core/testlib/test_client.cpp
index bdb3f4f7cc..d4907c26f1 100644
--- a/ydb/core/testlib/test_client.cpp
+++ b/ydb/core/testlib/test_client.cpp
@@ -319,7 +319,7 @@ namespace Tests {
GRpcServer->AddService(new NGRpcService::TGRpcDataStreamsService(system, counters, grpcRequestProxyId));
if (Settings->EnableYq) {
GRpcServer->AddService(new NGRpcService::TGRpcYandexQueryService(system, counters, grpcRequestProxyId));
- GRpcServer->AddService(new NGRpcService::TGRpcYqPrivateTaskService(system, counters, grpcRequestProxyId));
+ GRpcServer->AddService(new NGRpcService::TGRpcYqPrivateTaskService(system, counters, grpcRequestProxyId));
}
if (const auto& factory = Settings->GrpcServiceFactory) {
// All services enabled by default for ut
@@ -746,7 +746,7 @@ namespace Tests {
controlPlaneStorageConfig.AddAvailableConnection("OBJECT_STORAGE");
controlPlaneStorageConfig.AddAvailableConnection("MONITORING");
}
-
+
{
auto& commonConfig = *protoConfig.MutableCommon();
commonConfig.SetYdbMvpCloudEndpoint(ydbMvpEndpoint);
@@ -801,20 +801,20 @@ namespace Tests {
};
const auto ydbCredFactory = NKikimr::CreateYdbCredentialsProviderFactory;
- auto counters = MakeIntrusive<NMonitoring::TDynamicCounters>();
- auto yqSharedResources = NYq::CreateYqSharedResources(protoConfig, ydbCredFactory, counters);
+ auto counters = MakeIntrusive<NMonitoring::TDynamicCounters>();
+ auto yqSharedResources = NYq::CreateYqSharedResources(protoConfig, ydbCredFactory, counters);
NYq::Init(
protoConfig,
Runtime->GetNodeId(nodeIdx),
actorRegistrator,
- &appData,
+ &appData,
"TestTenant",
nullptr, // MakeIntrusive<NPq::NConfigurationManager::TConnections>(),
yqSharedResources,
NKikimr::NFolderService::CreateMockFolderServiceActor,
NYq::CreateMockYqAuditServiceActor,
- ydbCredFactory,
- /*IcPort = */0
+ ydbCredFactory,
+ /*IcPort = */0
);
NYq::InitTest(Runtime.Get(), port, Settings->GrpcPort, yqSharedResources);
}
@@ -843,7 +843,7 @@ namespace Tests {
//Runtime->SetLogPriority(NKikimrServices::SCHEME_BOARD_REPLICA, NActors::NLog::PRI_DEBUG);
//Runtime->SetLogPriority(NKikimrServices::SCHEME_BOARD_POPULATOR, NActors::NLog::PRI_DEBUG);
//Runtime->SetLogPriority(NKikimrServices::SCHEME_BOARD_SUBSCRIBER, NActors::NLog::PRI_TRACE);
- //Runtime->SetLogPriority(NKikimrServices::YQL_PROXY, NActors::NLog::PRI_DEBUG);
+ //Runtime->SetLogPriority(NKikimrServices::YQL_PROXY, NActors::NLog::PRI_DEBUG);
if (Settings->LoggerInitializer) {
Settings->LoggerInitializer(*Runtime);
diff --git a/ydb/core/yq/libs/actors/clusters_from_connections.cpp b/ydb/core/yq/libs/actors/clusters_from_connections.cpp
index c2927919b9..93e0471892 100644
--- a/ydb/core/yq/libs/actors/clusters_from_connections.cpp
+++ b/ydb/core/yq/libs/actors/clusters_from_connections.cpp
@@ -6,10 +6,10 @@
#include <util/string/builder.h>
#include <util/system/env.h>
-namespace NYq {
+namespace NYq {
+
+using namespace NYql;
-using namespace NYql;
-
namespace {
template <typename TClusterConfig>
@@ -30,8 +30,8 @@ void FillClusterAuth(TClusterConfig& clusterCfg, const YandexQuery::IamAuth& aut
}
}
-} //namespace
-
+} //namespace
+
void AddClustersFromConnections(const THashMap<TString, YandexQuery::Connection>& connections,
bool useBearerForYdb,
const TString& objectStorageEndpoint,
@@ -47,10 +47,10 @@ void AddClustersFromConnections(const THashMap<TString, YandexQuery::Connection>
auto* clusterCfg = gatewaysConfig.MutableYdb()->AddClusterMapping();
clusterCfg->SetName(connectionName);
clusterCfg->SetId(db.database_id());
- if (db.database())
- clusterCfg->SetDatabase(db.database());
- if (db.endpoint())
- clusterCfg->SetEndpoint(db.endpoint());
+ if (db.database())
+ clusterCfg->SetDatabase(db.database());
+ if (db.endpoint())
+ clusterCfg->SetEndpoint(db.endpoint());
clusterCfg->SetSecure(db.secure());
clusterCfg->SetAddBearerToToken(useBearerForYdb);
FillClusterAuth(*clusterCfg, db.auth(), authToken, accountIdSignatures);
@@ -61,11 +61,11 @@ void AddClustersFromConnections(const THashMap<TString, YandexQuery::Connection>
const auto& ch = conn.content().setting().clickhouse_cluster();
auto* clusterCfg = gatewaysConfig.MutableClickHouse()->AddClusterMapping();
clusterCfg->SetName(connectionName);
- clusterCfg->SetId(ch.database_id());
- if (ch.host())
- clusterCfg->SetCluster(ch.host());
- clusterCfg->SetNativeHostPort(9440);
- clusterCfg->SetNativeSecure(true);
+ clusterCfg->SetId(ch.database_id());
+ if (ch.host())
+ clusterCfg->SetCluster(ch.host());
+ clusterCfg->SetNativeHostPort(9440);
+ clusterCfg->SetNativeSecure(true);
clusterCfg->SetCHToken(TStringBuilder() << "basic#" << ch.login() << "#" << ch.password());
clusters.emplace(connectionName, ClickHouseProviderName);
break;
@@ -89,8 +89,8 @@ void AddClustersFromConnections(const THashMap<TString, YandexQuery::Connection>
const auto& ds = conn.content().setting().data_streams();
auto* clusterCfg = gatewaysConfig.MutablePq()->AddClusterMapping();
clusterCfg->SetName(connectionName);
- if (ds.endpoint())
- clusterCfg->SetEndpoint(ds.endpoint());
+ if (ds.endpoint())
+ clusterCfg->SetEndpoint(ds.endpoint());
clusterCfg->SetDatabase(ds.database());
clusterCfg->SetDatabaseId(ds.database_id());
clusterCfg->SetUseSsl(ds.secure());
@@ -135,4 +135,4 @@ void AddClustersFromConnections(const THashMap<TString, YandexQuery::Connection>
}
}
}
-} //NYq
+} //NYq
diff --git a/ydb/core/yq/libs/actors/clusters_from_connections.h b/ydb/core/yq/libs/actors/clusters_from_connections.h
index 0a3cc6783d..08db9704ea 100644
--- a/ydb/core/yq/libs/actors/clusters_from_connections.h
+++ b/ydb/core/yq/libs/actors/clusters_from_connections.h
@@ -3,14 +3,14 @@
#include <ydb/library/yql/providers/common/proto/gateways_config.pb.h>
#include <ydb/public/api/protos/yq.pb.h>
-namespace NYq {
-
+namespace NYq {
+
void AddClustersFromConnections(const THashMap<TString, YandexQuery::Connection>& connections,
bool useBearerForYdb,
const TString& objectStorageEndpoint,
const TString& authToken,
const THashMap<TString, TString>& accountIdSignatures,
- NYql::TGatewaysConfig& gatewaysConfig,
+ NYql::TGatewaysConfig& gatewaysConfig,
THashMap<TString, TString>& clusters);
-
-} //NYq
+
+} //NYq
diff --git a/ydb/core/yq/libs/actors/database_resolver.cpp b/ydb/core/yq/libs/actors/database_resolver.cpp
index 8aa27187c5..c784d80c00 100644
--- a/ydb/core/yq/libs/actors/database_resolver.cpp
+++ b/ydb/core/yq/libs/actors/database_resolver.cpp
@@ -15,93 +15,93 @@
#define LOG_D(stream) \
LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::YQL_PROXY, "DatabaseResolver - TraceId: " << TraceId << ": " << stream)
-namespace NYq {
+namespace NYq {
using namespace NActors;
-using namespace NYql;
-
+using namespace NYql;
+
using TEndpoint = TEvents::TEvEndpointResponse::TEndpoint;
using TParsers = THashMap<DatabaseType, std::function<TEndpoint(NJson::TJsonValue& body, bool)>>;
-using TCache = TTtlCache<std::tuple<TString, DatabaseType, TEvents::TDatabaseAuth>, std::variant<TEndpoint, TString>>;
+using TCache = TTtlCache<std::tuple<TString, DatabaseType, TEvents::TDatabaseAuth>, std::variant<TEndpoint, TString>>;
+
+TString TransformMdbHostToCorrectFormat(const TString& mdbHost) {
+ return mdbHost.substr(0, mdbHost.find('.')) + ".db.yandex.net:8443";
+}
-TString TransformMdbHostToCorrectFormat(const TString& mdbHost) {
- return mdbHost.substr(0, mdbHost.find('.')) + ".db.yandex.net:8443";
-}
-
-class TResponseProcessor : public TActorBootstrapped<TResponseProcessor>
+class TResponseProcessor : public TActorBootstrapped<TResponseProcessor>
{
public:
- enum EWakeUp {
- WU_Die_On_Ttl
- };
-
+ enum EWakeUp {
+ WU_Die_On_Ttl
+ };
+
TResponseProcessor(
- const TActorId sender,
+ const TActorId sender,
TCache& cache,
const THashMap<std::pair<TString, DatabaseType>, TEndpoint>& ready,
const THashMap<NHttp::THttpOutgoingRequestPtr, std::tuple<TString, DatabaseType, TEvents::TDatabaseAuth>>& requests,
- const TString& traceId,
+ const TString& traceId,
bool mdbTransformHost,
const TParsers& parsers)
- : Sender(sender)
+ : Sender(sender)
, Cache(cache)
, Requests(requests)
, TraceId(traceId)
- , MdbTransformHost(mdbTransformHost)
+ , MdbTransformHost(mdbTransformHost)
, DatabaseId2Endpoint(ready)
, Parsers(parsers)
{ }
- static constexpr char ActorName[] = "YQ_RESPONSE_PROCESSOR";
-
- void Bootstrap() {
- Schedule(ResolvingTtl, new NActors::TEvents::TEvWakeup(WU_Die_On_Ttl));
- Become(&TResponseProcessor::StateFunc);
- }
-
- STRICT_STFUNC(StateFunc, {
- hFunc(NHttp::TEvHttpProxy::TEvHttpIncomingResponse, Handle);
- hFunc(NActors::TEvents::TEvWakeup, HandleWakeup)
- cFunc(NActors::TEvents::TEvPoison::EventType, PassAway);
+ static constexpr char ActorName[] = "YQ_RESPONSE_PROCESSOR";
+
+ void Bootstrap() {
+ Schedule(ResolvingTtl, new NActors::TEvents::TEvWakeup(WU_Die_On_Ttl));
+ Become(&TResponseProcessor::StateFunc);
+ }
+
+ STRICT_STFUNC(StateFunc, {
+ hFunc(NHttp::TEvHttpProxy::TEvHttpIncomingResponse, Handle);
+ hFunc(NActors::TEvents::TEvWakeup, HandleWakeup)
+ cFunc(NActors::TEvents::TEvPoison::EventType, PassAway);
});
-private:
- void HandleWakeup(NActors::TEvents::TEvWakeup::TPtr& ev) {
- auto tag = ev->Get()->Tag;
- switch (tag) {
- case WU_Die_On_Ttl:
- DieOnTtl();
- break;
- }
- }
-
- void DieOnTtl() {
- Success = false;
-
- TString logMsg = "Could not resolve database ids: ";
- bool firstUnresolvedDbId = true;
- for (const auto& [_, info]: Requests) {
- const auto& dbId = std::get<0>(info);
- const auto& dbType = std::get<1>(info);
- if (const auto it = DatabaseId2Endpoint.find(std::make_pair(dbId, dbType)); it == DatabaseId2Endpoint.end()) {
- logMsg += (firstUnresolvedDbId ? TString{""} : TString{", "}) + dbId;
- if (firstUnresolvedDbId)
- firstUnresolvedDbId = false;
- }
- }
- logMsg += TStringBuilder() << " in " << ResolvingTtl << " seconds.";
- LOG_E(logMsg);
-
- SendResolvedEndpointsAndDie();
- }
-
- void SendResolvedEndpointsAndDie() {
- Send(Sender, new TEvents::TEvEndpointResponse(std::move(DatabaseId2Endpoint), Success));
- PassAway();
- }
-
- void Handle(NHttp::TEvHttpProxy::TEvHttpIncomingResponse::TPtr& ev)
+private:
+ void HandleWakeup(NActors::TEvents::TEvWakeup::TPtr& ev) {
+ auto tag = ev->Get()->Tag;
+ switch (tag) {
+ case WU_Die_On_Ttl:
+ DieOnTtl();
+ break;
+ }
+ }
+
+ void DieOnTtl() {
+ Success = false;
+
+ TString logMsg = "Could not resolve database ids: ";
+ bool firstUnresolvedDbId = true;
+ for (const auto& [_, info]: Requests) {
+ const auto& dbId = std::get<0>(info);
+ const auto& dbType = std::get<1>(info);
+ if (const auto it = DatabaseId2Endpoint.find(std::make_pair(dbId, dbType)); it == DatabaseId2Endpoint.end()) {
+ logMsg += (firstUnresolvedDbId ? TString{""} : TString{", "}) + dbId;
+ if (firstUnresolvedDbId)
+ firstUnresolvedDbId = false;
+ }
+ }
+ logMsg += TStringBuilder() << " in " << ResolvingTtl << " seconds.";
+ LOG_E(logMsg);
+
+ SendResolvedEndpointsAndDie();
+ }
+
+ void SendResolvedEndpointsAndDie() {
+ Send(Sender, new TEvents::TEvEndpointResponse(std::move(DatabaseId2Endpoint), Success));
+ PassAway();
+ }
+
+ void Handle(NHttp::TEvHttpProxy::TEvHttpIncomingResponse::TPtr& ev)
{
TString status;
TString errorMessage;
@@ -109,7 +109,7 @@ private:
DatabaseType databaseType = DatabaseType::Ydb;
TEvents::TDatabaseAuth info;
TMaybe<TEndpoint> result;
- HandledIds++;
+ HandledIds++;
if (ev->Get()->Error.empty() && (ev->Get()->Response && ((status = ev->Get()->Response->Status) == "200"))) {
NJson::TJsonReaderConfig jsonConfig;
NJson::TJsonValue databaseInfo;
@@ -134,20 +134,20 @@ private:
<< databaseType << " Id: "
<< databaseId << "\n"
<< CurrentExceptionMessage();
- }
+ }
} else {
- errorMessage = TStringBuilder() << "Unable to parse database information."
- << "Database Id: " << databaseId
+ errorMessage = TStringBuilder() << "Unable to parse database information."
+ << "Database Id: " << databaseId
<< ", Database Type: " << databaseType;
}
}
} else {
errorMessage = ev->Get()->Error;
- const TString error = "Cannot resolve databaseId (status = " + ToString(status) + ")";
- if (!errorMessage.empty()) {
- errorMessage += '\n';
+ const TString error = "Cannot resolve databaseId (status = " + ToString(status) + ")";
+ if (!errorMessage.empty()) {
+ errorMessage += '\n';
}
- errorMessage += error;
+ errorMessage += error;
}
if (errorMessage) {
@@ -157,18 +157,18 @@ private:
if (databaseId) {
auto key = std::make_tuple(databaseId, databaseType, info);
- if (errorMessage) {
- Cache.Put(key, errorMessage);
- } else {
- Cache.Put(key, result);
- }
+ if (errorMessage) {
+ Cache.Put(key, errorMessage);
+ } else {
+ Cache.Put(key, result);
+ }
}
- if (HandledIds == Requests.size()) {
- SendResolvedEndpointsAndDie();
+ if (HandledIds == Requests.size()) {
+ SendResolvedEndpointsAndDie();
}
- LOG_D(DatabaseId2Endpoint.size() << " of " << Requests.size() << " done");
+ LOG_D(DatabaseId2Endpoint.size() << " of " << Requests.size() << " done");
}
private:
@@ -176,12 +176,12 @@ private:
TCache& Cache;
const THashMap<NHttp::THttpOutgoingRequestPtr, std::tuple<TString, DatabaseType, TEvents::TDatabaseAuth>> Requests;
const TString TraceId;
- const bool MdbTransformHost;
+ const bool MdbTransformHost;
THashMap<std::pair<TString, DatabaseType>, TEvents::TEvEndpointResponse::TEndpoint> DatabaseId2Endpoint;
- size_t HandledIds = 0;
+ size_t HandledIds = 0;
bool Success = true;
const TParsers& Parsers;
- TDuration ResolvingTtl = TDuration::Seconds(30); //TODO: Use cfg
+ TDuration ResolvingTtl = TDuration::Seconds(30); //TODO: Use cfg
};
class TDatabaseResolver: public TActor<TDatabaseResolver>
@@ -248,8 +248,8 @@ public:
};
}
- static constexpr char ActorName[] = "YQ_DATABASE_RESOLVER";
-
+ static constexpr char ActorName[] = "YQ_DATABASE_RESOLVER";
+
private:
STRICT_STFUNC(State, {
HFunc(TEvents::TEvEndpointRequest, Handle);
@@ -264,13 +264,13 @@ private:
THashMap<std::pair<TString, DatabaseType>, TEndpoint> ready;
for (const auto& [p, info] : ev->Get()->DatabaseIds) {
const auto& [databaseId, type] = p;
- TMaybe<std::variant<TEndpoint, TString>> endpoint;
+ TMaybe<std::variant<TEndpoint, TString>> endpoint;
auto key = std::make_tuple(databaseId, type, info);
if (Cache.Get(key, &endpoint)) {
if (endpoint) {
- ready.insert(std::make_pair(p,
- (endpoint->index() == 0 ? std::get<0>(*endpoint) : TEndpoint{})
- ));
+ ready.insert(std::make_pair(p,
+ (endpoint->index() == 0 ? std::get<0>(*endpoint) : TEndpoint{})
+ ));
}
continue;
}
@@ -291,15 +291,15 @@ private:
requests[httpRequest] = key;
} catch (const std::exception& e) {
- const TString msg = TStringBuilder() << " Error while preparing to resolve databaseId " << databaseId << ", details: " << e.what();
+ const TString msg = TStringBuilder() << " Error while preparing to resolve databaseId " << databaseId << ", details: " << e.what();
LOG_E(msg);
Cache.Put(key, endpoint);
}
}
if (!requests.empty()) {
- auto helper = Register(
- new TResponseProcessor(ev->Sender, Cache, ready, requests, TraceId, ev->Get()->MdbTransformHost, Parsers));
+ auto helper = Register(
+ new TResponseProcessor(ev->Sender, Cache, ready, requests, TraceId, ev->Get()->MdbTransformHost, Parsers));
for (const auto& [request, _] : requests) {
ctx.Send(new IEventHandle(HttpProxy, helper, new NHttp::TEvHttpProxy::TEvHttpOutgoingRequest(request)));
@@ -321,4 +321,4 @@ NActors::IActor* CreateDatabaseResolver(NActors::TActorId httpProxy, ISecuredSer
return new TDatabaseResolver(httpProxy, credentialsFactory);
}
-} /* namespace NYq */
+} /* namespace NYq */
diff --git a/ydb/core/yq/libs/actors/database_resolver.h b/ydb/core/yq/libs/actors/database_resolver.h
index f7b98beec0..3e50f46bc4 100644
--- a/ydb/core/yq/libs/actors/database_resolver.h
+++ b/ydb/core/yq/libs/actors/database_resolver.h
@@ -3,10 +3,10 @@
#include <library/cpp/actors/core/actor.h>
#include <ydb/library/yql/providers/common/token_accessor/client/factory.h>
-namespace NYq {
+namespace NYq {
-TString TransformMdbHostToCorrectFormat(const TString& mdbHost);
-
-NActors::IActor* CreateDatabaseResolver(NActors::TActorId httpProxy, NYql::ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory);
+TString TransformMdbHostToCorrectFormat(const TString& mdbHost);
-} /* namespace NYq */
+NActors::IActor* CreateDatabaseResolver(NActors::TActorId httpProxy, NYql::ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory);
+
+} /* namespace NYq */
diff --git a/ydb/core/yq/libs/actors/error.cpp b/ydb/core/yq/libs/actors/error.cpp
index 22212eb4b3..7a847961c1 100644
--- a/ydb/core/yq/libs/actors/error.cpp
+++ b/ydb/core/yq/libs/actors/error.cpp
@@ -2,10 +2,10 @@
#include <util/system/hostname.h>
-namespace NYq {
+namespace NYq {
TString MakeInternalError(const TString& text) {
return TStringBuilder() << "Internal error (" << text << ", host: " << HostName() << ")";
}
-} // namespace NYq
+} // namespace NYq
diff --git a/ydb/core/yq/libs/actors/nodes_health_check.cpp b/ydb/core/yq/libs/actors/nodes_health_check.cpp
index 395b09cb89..96e7c1c1cd 100644
--- a/ydb/core/yq/libs/actors/nodes_health_check.cpp
+++ b/ydb/core/yq/libs/actors/nodes_health_check.cpp
@@ -1,132 +1,132 @@
-#include "proxy_private.h"
-
+#include "proxy_private.h"
+
#include <ydb/core/protos/services.pb.h>
#include <ydb/library/yql/public/issue/yql_issue_message.h>
-
-#include <library/cpp/yson/node/node_io.h>
-#include <library/cpp/actors/core/events.h>
-#include <library/cpp/actors/core/hfunc.h>
-#include <library/cpp/actors/core/actor_bootstrapped.h>
-#include <library/cpp/actors/core/log.h>
-
+
+#include <library/cpp/yson/node/node_io.h>
+#include <library/cpp/actors/core/events.h>
+#include <library/cpp/actors/core/hfunc.h>
+#include <library/cpp/actors/core/actor_bootstrapped.h>
+#include <library/cpp/actors/core/log.h>
+
#include <ydb/core/yq/libs/control_plane_storage/control_plane_storage.h>
#include <ydb/core/yq/libs/control_plane_storage/events/events.h>
-#include <google/protobuf/util/time_util.h>
-
+#include <google/protobuf/util/time_util.h>
+
#define LOG_E(stream) \
LOG_ERROR_S(*TlsActivationContext, NKikimrServices::YQL_NODES_MANAGER, stream)
#define LOG_D(stream) \
LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::YQL_NODES_MANAGER, stream)
-
-namespace NYq {
-
-using namespace NActors;
-using namespace NYql;
-using namespace NMonitoring;
-
-class TNodesHealthCheckActor
- : public NActors::TActorBootstrapped<TNodesHealthCheckActor>
-{
-public:
- TNodesHealthCheckActor(
- const NActors::TActorId& sender,
- TIntrusivePtr<ITimeProvider> timeProvider,
- TAutoPtr<TEvents::TEvNodesHealthCheckRequest> ev,
- TDynamicCounterPtr counters)
+
+namespace NYq {
+
+using namespace NActors;
+using namespace NYql;
+using namespace NMonitoring;
+
+class TNodesHealthCheckActor
+ : public NActors::TActorBootstrapped<TNodesHealthCheckActor>
+{
+public:
+ TNodesHealthCheckActor(
+ const NActors::TActorId& sender,
+ TIntrusivePtr<ITimeProvider> timeProvider,
+ TAutoPtr<TEvents::TEvNodesHealthCheckRequest> ev,
+ TDynamicCounterPtr counters)
: Sender(sender)
- , TimeProvider(timeProvider)
- , Ev(std::move(ev))
- , Counters(std::move(counters))
- , LifetimeDuration(Counters->GetSubgroup("subsystem", "private_api")->GetHistogram("NodesHealthCheckTask", ExponentialHistogram(10, 2, 50)))
- , StartTime(TInstant::Now())
- {}
-
- static constexpr char ActorName[] = "YQ_PRIVATE_NODES_HEALTH_CHECK";
-
- void OnUndelivered(NActors::TEvents::TEvUndelivered::TPtr& ev, const NActors::TActorContext& ctx) {
- LOG_E("TNodesHealthCheckActor::OnUndelivered");
- auto res = MakeHolder<TEvents::TEvNodesHealthCheckResponse>();
- res->Status = Ydb::StatusIds::GENERIC_ERROR;
- res->Issues.AddIssue("UNDELIVERED");
- ctx.Send(ev->Sender, res.Release());
- Die(ctx);
- }
-
- void PassAway() final {
- LifetimeDuration->Collect((TInstant::Now() - StartTime).MilliSeconds());
- NActors::IActor::PassAway();
- }
-
- void Fail(const TString& message, Ydb::StatusIds::StatusCode reqStatus = Ydb::StatusIds::INTERNAL_ERROR) {
- Issues.AddIssue(message);
- const auto codeStr = Ydb::StatusIds_StatusCode_Name(reqStatus);
- LOG_E(TStringBuilder()
- << "Failed with code: " << codeStr
- << " Details: " << Issues.ToString());
- auto res = MakeHolder<TEvents::TEvNodesHealthCheckResponse>();
- res->Status = reqStatus;
- res->Issues.AddIssues(Issues);
- Send(Sender, res.Release());
- PassAway();
- }
-
- void Bootstrap(const TActorContext&) {
- Become(&TNodesHealthCheckActor::StateFunc);
- auto& req = Ev->Record;
- Tenant = req.tenant();
-
- Send(NYq::ControlPlaneStorageServiceActorId(),
- new NYq::TEvControlPlaneStorage::TEvNodesHealthCheckRequest(std::move(req)));
- }
-
-private:
- STRICT_STFUNC(
- StateFunc,
- CFunc(NActors::TEvents::TEvPoison::EventType, Die)
- HFunc(NYq::TEvControlPlaneStorage::TEvNodesHealthCheckResponse, HandleResponse)
- HFunc(NActors::TEvents::TEvUndelivered, OnUndelivered)
- )
-
- void HandleResponse(NYq::TEvControlPlaneStorage::TEvNodesHealthCheckResponse::TPtr& ev, const TActorContext& ctx) {
- auto res = MakeHolder<TEvents::TEvNodesHealthCheckResponse>();
- try {
- const auto& issues = ev->Get()->Issues;
- if (issues) {
- ythrow yexception() << issues.ToString();
- }
- res->Record.ConstructInPlace();
- res->Status = Ydb::StatusIds::SUCCESS;
- res->Record = ev->Get()->Record;
- ctx.Send(Sender, res.Release());
- Die(ctx);
- } catch (...) {
- const auto msg = TStringBuilder() << "Can't do NodesHealthCheck: " << CurrentExceptionMessage();
- Fail(msg);
- }
- }
-
-private:
- const TActorId Sender;
- TIntrusivePtr<ITimeProvider> TimeProvider;
- TAutoPtr<TEvents::TEvNodesHealthCheckRequest> Ev;
- TDynamicCounterPtr Counters;
- const THistogramPtr LifetimeDuration;
- const TInstant StartTime;
-
- NYql::TIssues Issues;
- TString Tenant;
-};
-
-IActor* CreateNodesHealthCheckActor(
- const NActors::TActorId& sender,
- TIntrusivePtr<ITimeProvider> timeProvider,
- TAutoPtr<TEvents::TEvNodesHealthCheckRequest> ev,
- TDynamicCounterPtr counters) {
- return new TNodesHealthCheckActor(
- sender,
- timeProvider,
- std::move(ev),
- std::move(counters));
-}
-
-} /* NYq */
+ , TimeProvider(timeProvider)
+ , Ev(std::move(ev))
+ , Counters(std::move(counters))
+ , LifetimeDuration(Counters->GetSubgroup("subsystem", "private_api")->GetHistogram("NodesHealthCheckTask", ExponentialHistogram(10, 2, 50)))
+ , StartTime(TInstant::Now())
+ {}
+
+ static constexpr char ActorName[] = "YQ_PRIVATE_NODES_HEALTH_CHECK";
+
+ void OnUndelivered(NActors::TEvents::TEvUndelivered::TPtr& ev, const NActors::TActorContext& ctx) {
+ LOG_E("TNodesHealthCheckActor::OnUndelivered");
+ auto res = MakeHolder<TEvents::TEvNodesHealthCheckResponse>();
+ res->Status = Ydb::StatusIds::GENERIC_ERROR;
+ res->Issues.AddIssue("UNDELIVERED");
+ ctx.Send(ev->Sender, res.Release());
+ Die(ctx);
+ }
+
+ void PassAway() final {
+ LifetimeDuration->Collect((TInstant::Now() - StartTime).MilliSeconds());
+ NActors::IActor::PassAway();
+ }
+
+ void Fail(const TString& message, Ydb::StatusIds::StatusCode reqStatus = Ydb::StatusIds::INTERNAL_ERROR) {
+ Issues.AddIssue(message);
+ const auto codeStr = Ydb::StatusIds_StatusCode_Name(reqStatus);
+ LOG_E(TStringBuilder()
+ << "Failed with code: " << codeStr
+ << " Details: " << Issues.ToString());
+ auto res = MakeHolder<TEvents::TEvNodesHealthCheckResponse>();
+ res->Status = reqStatus;
+ res->Issues.AddIssues(Issues);
+ Send(Sender, res.Release());
+ PassAway();
+ }
+
+ void Bootstrap(const TActorContext&) {
+ Become(&TNodesHealthCheckActor::StateFunc);
+ auto& req = Ev->Record;
+ Tenant = req.tenant();
+
+ Send(NYq::ControlPlaneStorageServiceActorId(),
+ new NYq::TEvControlPlaneStorage::TEvNodesHealthCheckRequest(std::move(req)));
+ }
+
+private:
+ STRICT_STFUNC(
+ StateFunc,
+ CFunc(NActors::TEvents::TEvPoison::EventType, Die)
+ HFunc(NYq::TEvControlPlaneStorage::TEvNodesHealthCheckResponse, HandleResponse)
+ HFunc(NActors::TEvents::TEvUndelivered, OnUndelivered)
+ )
+
+ void HandleResponse(NYq::TEvControlPlaneStorage::TEvNodesHealthCheckResponse::TPtr& ev, const TActorContext& ctx) {
+ auto res = MakeHolder<TEvents::TEvNodesHealthCheckResponse>();
+ try {
+ const auto& issues = ev->Get()->Issues;
+ if (issues) {
+ ythrow yexception() << issues.ToString();
+ }
+ res->Record.ConstructInPlace();
+ res->Status = Ydb::StatusIds::SUCCESS;
+ res->Record = ev->Get()->Record;
+ ctx.Send(Sender, res.Release());
+ Die(ctx);
+ } catch (...) {
+ const auto msg = TStringBuilder() << "Can't do NodesHealthCheck: " << CurrentExceptionMessage();
+ Fail(msg);
+ }
+ }
+
+private:
+ const TActorId Sender;
+ TIntrusivePtr<ITimeProvider> TimeProvider;
+ TAutoPtr<TEvents::TEvNodesHealthCheckRequest> Ev;
+ TDynamicCounterPtr Counters;
+ const THistogramPtr LifetimeDuration;
+ const TInstant StartTime;
+
+ NYql::TIssues Issues;
+ TString Tenant;
+};
+
+IActor* CreateNodesHealthCheckActor(
+ const NActors::TActorId& sender,
+ TIntrusivePtr<ITimeProvider> timeProvider,
+ TAutoPtr<TEvents::TEvNodesHealthCheckRequest> ev,
+ TDynamicCounterPtr counters) {
+ return new TNodesHealthCheckActor(
+ sender,
+ timeProvider,
+ std::move(ev),
+ std::move(counters));
+}
+
+} /* NYq */
diff --git a/ydb/core/yq/libs/actors/nodes_manager.cpp b/ydb/core/yq/libs/actors/nodes_manager.cpp
index 62cf85e5c2..62d92aec0f 100644
--- a/ydb/core/yq/libs/actors/nodes_manager.cpp
+++ b/ydb/core/yq/libs/actors/nodes_manager.cpp
@@ -1,119 +1,119 @@
-#include "nodes_manager.h"
+#include "nodes_manager.h"
#include <ydb/core/yq/libs/config/protos/yq_config.pb.h>
-
-#include <library/cpp/actors/core/events.h>
-#include <library/cpp/actors/core/hfunc.h>
-#include <library/cpp/actors/core/actor_bootstrapped.h>
+
+#include <library/cpp/actors/core/events.h>
+#include <library/cpp/actors/core/hfunc.h>
+#include <library/cpp/actors/core/actor_bootstrapped.h>
#include <ydb/library/yql/providers/dq/worker_manager/interface/events.h>
#include <ydb/library/yql/public/issue/yql_issue_message.h>
#include <ydb/public/sdk/cpp/client/ydb_driver/driver.h>
#include <ydb/public/sdk/cpp/client/ydb_value/value.h>
#include <ydb/core/yq/libs/common/entity_id.h>
#include <ydb/core/yq/libs/private_client/private_client.h>
-#include <library/cpp/actors/core/log.h>
-#include <util/system/hostname.h>
+#include <library/cpp/actors/core/log.h>
+#include <util/system/hostname.h>
#include <ydb/core/protos/services.pb.h>
-
-
+
+
#define LOG_E(stream) \
LOG_ERROR_S(*TlsActivationContext, NKikimrServices::YQL_NODES_MANAGER, stream)
#define LOG_I(stream) \
LOG_INFO_S(*TlsActivationContext, NKikimrServices::YQL_NODES_MANAGER, stream)
#define LOG_D(stream) \
LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::YQL_NODES_MANAGER, stream)
-
-namespace NYq {
-
-using namespace NActors;
-using namespace NYql;
-using namespace NDqs;
-
-struct TEvHealthNodesResponse : public NActors::TEventLocal<TEvHealthNodesResponse, NActors::TEvents::TSystem::Completed>{
- bool Success;
- NYdb::EStatus Status;
- const NYql::TIssues Issues;
- Yq::Private::NodesHealthCheckResult Record;
-
- explicit TEvHealthNodesResponse(
- const bool success,
- const NYdb::EStatus& status,
- const TIssues& issues,
- const Yq::Private::NodesHealthCheckResult& record)
- : Success(success)
- , Status(status)
- , Issues(issues)
- , Record(record)
- { }
-};
-
-class TYqlNodesManagerActor : public NActors::TActorBootstrapped<TYqlNodesManagerActor> {
-public:
- enum EWakeUp {
- WU_NodesHealthCheck
- };
-
- TYqlNodesManagerActor(
+
+namespace NYq {
+
+using namespace NActors;
+using namespace NYql;
+using namespace NDqs;
+
+struct TEvHealthNodesResponse : public NActors::TEventLocal<TEvHealthNodesResponse, NActors::TEvents::TSystem::Completed>{
+ bool Success;
+ NYdb::EStatus Status;
+ const NYql::TIssues Issues;
+ Yq::Private::NodesHealthCheckResult Record;
+
+ explicit TEvHealthNodesResponse(
+ const bool success,
+ const NYdb::EStatus& status,
+ const TIssues& issues,
+ const Yq::Private::NodesHealthCheckResult& record)
+ : Success(success)
+ , Status(status)
+ , Issues(issues)
+ , Record(record)
+ { }
+};
+
+class TYqlNodesManagerActor : public NActors::TActorBootstrapped<TYqlNodesManagerActor> {
+public:
+ enum EWakeUp {
+ WU_NodesHealthCheck
+ };
+
+ TYqlNodesManagerActor(
const NYq::TYqSharedResources::TPtr& yqSharedResources,
- const NDqs::TWorkerManagerCounters& workerManagerCounters,
- TIntrusivePtr<ITimeProvider> timeProvider,
- TIntrusivePtr<IRandomProvider> randomProvider,
+ const NDqs::TWorkerManagerCounters& workerManagerCounters,
+ TIntrusivePtr<ITimeProvider> timeProvider,
+ TIntrusivePtr<IRandomProvider> randomProvider,
const ::NYq::NCommon::TServiceCounters& serviceCounters,
const NConfig::TPrivateApiConfig& privateApiConfig,
- const ui32& icPort,
- const TString& address,
+ const ui32& icPort,
+ const TString& address,
const TString& tenant,
- ui64 mkqlInitialMemoryLimit,
- const NMonitoring::TDynamicCounterPtr& clientCounters)
- : WorkerManagerCounters(workerManagerCounters)
- , TimeProvider(timeProvider)
- , RandomProvider(randomProvider)
+ ui64 mkqlInitialMemoryLimit,
+ const NMonitoring::TDynamicCounterPtr& clientCounters)
+ : WorkerManagerCounters(workerManagerCounters)
+ , TimeProvider(timeProvider)
+ , RandomProvider(randomProvider)
, ServiceCounters(serviceCounters, "node_manager")
, PrivateApiConfig(privateApiConfig)
- , Tenant(tenant)
+ , Tenant(tenant)
, MkqlInitialMemoryLimit(mkqlInitialMemoryLimit)
, YqSharedResources(yqSharedResources)
- , IcPort(icPort)
- , Address(address)
- , Client(
+ , IcPort(icPort)
+ , Address(address)
+ , Client(
YqSharedResources->YdbDriver,
- NYdb::TCommonClientSettings()
+ NYdb::TCommonClientSettings()
.DiscoveryEndpoint(PrivateApiConfig.GetTaskServiceEndpoint())
- .Database(PrivateApiConfig.GetTaskServiceDatabase() ? PrivateApiConfig.GetTaskServiceDatabase() : TMaybe<TString>()),
- clientCounters)
-
- {
- InstanceId = GetGuidAsString(RandomProvider->GenUuid4());
- }
-
- static constexpr char ActorName[] = "YQ_NODES_MANAGER";
-
- void PassAway() final {
+ .Database(PrivateApiConfig.GetTaskServiceDatabase() ? PrivateApiConfig.GetTaskServiceDatabase() : TMaybe<TString>()),
+ clientCounters)
+
+ {
+ InstanceId = GetGuidAsString(RandomProvider->GenUuid4());
+ }
+
+ static constexpr char ActorName[] = "YQ_NODES_MANAGER";
+
+ void PassAway() final {
LOG_I("PassAway STOPPED");
- NActors::IActor::PassAway();
- }
-
+ NActors::IActor::PassAway();
+ }
+
void Bootstrap(const TActorContext&) {
- Become(&TYqlNodesManagerActor::StateFunc);
+ Become(&TYqlNodesManagerActor::StateFunc);
ServiceCounters.Counters->GetCounter("EvBootstrap", true)->Inc();
LOG_I("Bootstrap STARTED");
NodesHealthCheck();
- }
-
-private:
+ }
+
+private:
void Handle(NDqs::TEvAllocateWorkersRequest::TPtr& ev) {
ServiceCounters.Counters->GetCounter("EvAllocateWorkersRequest", true)->Inc();
- const auto &rec = ev->Get()->Record;
- const auto count = rec.GetCount();
- Y_ASSERT(count != 0);
- auto resourceId = rec.GetResourceId();
- if (!resourceId) {
+ const auto &rec = ev->Get()->Record;
+ const auto count = rec.GetCount();
+ Y_ASSERT(count != 0);
+ auto resourceId = rec.GetResourceId();
+ if (!resourceId) {
resourceId = (ui64(++ResourceIdPart) << 32) | SelfId().NodeId();
- }
-
- TVector<TPeer> nodes;
- for (ui32 i = 0; i < count; ++i) {
+ }
+
+ TVector<TPeer> nodes;
+ for (ui32 i = 0; i < count; ++i) {
TPeer node = {SelfId().NodeId(), InstanceId + "," + HostName(), 0, 0, 0};
- if (!Peers.empty()) {
+ if (!Peers.empty()) {
auto FirstPeer = NextPeer;
while (true) {
if (NextPeer >= Peers.size()) {
@@ -132,57 +132,57 @@ private:
node = nextNode;
break;
}
- }
- }
- nodes.push_back(node);
- }
-
- auto req = MakeHolder<NDqs::TEvAllocateWorkersResponse>();
- req->Record.ClearError();
- auto& group = *req->Record.MutableNodes();
- group.SetResourceId(resourceId);
- for (const auto& node : nodes) {
- auto* worker = group.AddWorker();
- *worker->MutableGuid() = node.InstanceId;
- worker->SetNodeId(node.NodeId);
- }
+ }
+ }
+ nodes.push_back(node);
+ }
+
+ auto req = MakeHolder<NDqs::TEvAllocateWorkersResponse>();
+ req->Record.ClearError();
+ auto& group = *req->Record.MutableNodes();
+ group.SetResourceId(resourceId);
+ for (const auto& node : nodes) {
+ auto* worker = group.AddWorker();
+ *worker->MutableGuid() = node.InstanceId;
+ worker->SetNodeId(node.NodeId);
+ }
LOG_D("TEvAllocateWorkersResponse " << req->Record.DebugString());
-
- Send(ev->Sender, req.Release());
- }
-
+
+ Send(ev->Sender, req.Release());
+ }
+
void Handle(NDqs::TEvFreeWorkersNotify::TPtr&) {
ServiceCounters.Counters->GetCounter("EvFreeWorkersNotify", true)->Inc();
- }
-
- STRICT_STFUNC(
- StateFunc,
-
+ }
+
+ STRICT_STFUNC(
+ StateFunc,
+
hFunc(NActors::TEvents::TEvWakeup, HandleWakeup)
hFunc(NDqs::TEvAllocateWorkersRequest, Handle)
hFunc(NDqs::TEvFreeWorkersNotify, Handle)
hFunc(NActors::TEvents::TEvUndelivered, OnUndelivered)
- hFunc(TEvHealthNodesResponse, HandleResponse)
- )
-
+ hFunc(TEvHealthNodesResponse, HandleResponse)
+ )
+
void HandleWakeup(NActors::TEvents::TEvWakeup::TPtr& ev) {
ServiceCounters.Counters->GetCounter("EvWakeup", true)->Inc();
- auto tag = ev->Get()->Tag;
- switch (tag) {
- case WU_NodesHealthCheck:
+ auto tag = ev->Get()->Tag;
+ switch (tag) {
+ case WU_NodesHealthCheck:
NodesHealthCheck();
- break;
- }
- }
-
+ break;
+ }
+ }
+
void NodesHealthCheck() {
- const TDuration ttl = TDuration::Seconds(5);
+ const TDuration ttl = TDuration::Seconds(5);
Schedule(ttl, new NActors::TEvents::TEvWakeup(WU_NodesHealthCheck));
-
+
ServiceCounters.Counters->GetCounter("NodesHealthCheck", true)->Inc();
-
- Yq::Private::NodesHealthCheckRequest request;
- request.set_tenant(Tenant);
+
+ Yq::Private::NodesHealthCheckRequest request;
+ request.set_tenant(Tenant);
auto& node = *request.mutable_node();
node.set_node_id(SelfId().NodeId());
node.set_instance_id(InstanceId);
@@ -190,122 +190,122 @@ private:
node.set_active_workers(AtomicGet(WorkerManagerCounters.ActiveWorkers->GetAtomic()));
node.set_memory_limit(AtomicGet(WorkerManagerCounters.MkqlMemoryLimit->GetAtomic()));
node.set_memory_allocated(AtomicGet(WorkerManagerCounters.MkqlMemoryAllocated->GetAtomic()));
- node.set_interconnect_port(IcPort);
- node.set_node_address(Address);
- const auto actorSystem = NActors::TActivationContext::ActorSystem();
- const auto selfId = SelfId();
- Client
- .NodesHealthCheck(std::move(request))
- .Subscribe([actorSystem, selfId](const auto& future) {
- const auto& wrappedResult = future.GetValue();
- if (wrappedResult.IsResultSet()) {
- actorSystem->Send(selfId,
- new TEvHealthNodesResponse{wrappedResult.IsSuccess(), wrappedResult.GetStatus(),
- wrappedResult.GetIssues(), wrappedResult.GetResult()});
- }
- });
- }
-
+ node.set_interconnect_port(IcPort);
+ node.set_node_address(Address);
+ const auto actorSystem = NActors::TActivationContext::ActorSystem();
+ const auto selfId = SelfId();
+ Client
+ .NodesHealthCheck(std::move(request))
+ .Subscribe([actorSystem, selfId](const auto& future) {
+ const auto& wrappedResult = future.GetValue();
+ if (wrappedResult.IsResultSet()) {
+ actorSystem->Send(selfId,
+ new TEvHealthNodesResponse{wrappedResult.IsSuccess(), wrappedResult.GetStatus(),
+ wrappedResult.GetIssues(), wrappedResult.GetResult()});
+ }
+ });
+ }
+
void OnUndelivered(NActors::TEvents::TEvUndelivered::TPtr&) {
- LOG_E("TYqlNodesManagerActor::OnUndelivered");
+ LOG_E("TYqlNodesManagerActor::OnUndelivered");
ServiceCounters.Counters->GetCounter("OnUndelivered", true)->Inc();
- }
-
- void HandleResponse(TEvHealthNodesResponse::TPtr& ev) {
- try {
- const auto& status = ev->Get()->Status;
- THolder<TEvInterconnect::TEvNodesInfo> nameServiceUpdateReq(new TEvInterconnect::TEvNodesInfo());
- if (!ev->Get()->Success) {
- ythrow yexception() << status << '\n' << ev->Get()->Issues.ToString();
- }
- const auto& res = ev->Get()->Record;
-
- auto& nodesInfo = nameServiceUpdateReq->Nodes;
- nodesInfo.reserve(res.nodes().size());
-
- Peers.clear();
- for (const auto& node : res.nodes()) {
+ }
+
+ void HandleResponse(TEvHealthNodesResponse::TPtr& ev) {
+ try {
+ const auto& status = ev->Get()->Status;
+ THolder<TEvInterconnect::TEvNodesInfo> nameServiceUpdateReq(new TEvInterconnect::TEvNodesInfo());
+ if (!ev->Get()->Success) {
+ ythrow yexception() << status << '\n' << ev->Get()->Issues.ToString();
+ }
+ const auto& res = ev->Get()->Record;
+
+ auto& nodesInfo = nameServiceUpdateReq->Nodes;
+ nodesInfo.reserve(res.nodes().size());
+
+ Peers.clear();
+ for (const auto& node : res.nodes()) {
Peers.push_back({node.node_id(), node.instance_id() + "," + node.hostname(),
node.active_workers(), node.memory_limit(), node.memory_allocated()});
-
- if (node.interconnect_port()) {
- nodesInfo.emplace_back(TEvInterconnect::TNodeInfo{
- node.node_id(),
- node.node_address(),
- node.hostname(), // host
- node.hostname(), // resolveHost
- static_cast<ui16>(node.interconnect_port()),
- /* NodeLocation = */{}});
- }
- }
-
+
+ if (node.interconnect_port()) {
+ nodesInfo.emplace_back(TEvInterconnect::TNodeInfo{
+ node.node_id(),
+ node.node_address(),
+ node.hostname(), // host
+ node.hostname(), // resolveHost
+ static_cast<ui16>(node.interconnect_port()),
+ /* NodeLocation = */{}});
+ }
+ }
+
ServiceCounters.Counters->GetCounter("PeerCount", false)->Set(Peers.size());
ServiceCounters.Counters->GetCounter("NodesHealthCheckOk", true)->Inc();
-
- LOG_D("Send NodeInfo with size: " << nodesInfo.size() << " to DynamicNameserver");
- if (!nodesInfo.empty()) {
- Send(GetNameserviceActorId(), nameServiceUpdateReq.Release());
- }
- } catch (yexception &e) {
- LOG_E(e.what());
+
+ LOG_D("Send NodeInfo with size: " << nodesInfo.size() << " to DynamicNameserver");
+ if (!nodesInfo.empty()) {
+ Send(GetNameserviceActorId(), nameServiceUpdateReq.Release());
+ }
+ } catch (yexception &e) {
+ LOG_E(e.what());
ServiceCounters.Counters->GetCounter("NodesHealthCheckFail", true)->Inc();
- }
- }
-
-private:
- NDqs::TWorkerManagerCounters WorkerManagerCounters;
- TIntrusivePtr<ITimeProvider> TimeProvider;
- TIntrusivePtr<IRandomProvider> RandomProvider;
+ }
+ }
+
+private:
+ NDqs::TWorkerManagerCounters WorkerManagerCounters;
+ TIntrusivePtr<ITimeProvider> TimeProvider;
+ TIntrusivePtr<IRandomProvider> RandomProvider;
::NYq::NCommon::TServiceCounters ServiceCounters;
NConfig::TPrivateApiConfig PrivateApiConfig;
- TString Tenant;
+ TString Tenant;
ui64 MkqlInitialMemoryLimit;
-
+
NYq::TYqSharedResources::TPtr YqSharedResources;
- const ui32 IcPort; // Interconnect Port
- const TString Address;
-
- TPrivateClient Client;
-
- struct TPeer {
- ui32 NodeId;
- TString InstanceId;
+ const ui32 IcPort; // Interconnect Port
+ const TString Address;
+
+ TPrivateClient Client;
+
+ struct TPeer {
+ ui32 NodeId;
+ TString InstanceId;
ui64 ActiveWorkers;
ui64 MemoryLimit;
ui64 MemoryAllocated;
- };
- TVector<TPeer> Peers;
- ui32 ResourceIdPart = 0;
- ui32 NextPeer = 0;
- TString InstanceId;
-};
-
-TActorId MakeYqlNodesManagerId() {
- constexpr TStringBuf name = "YQLNODESCTRL";
- return NActors::TActorId(0, name);
-}
-
-TActorId MakeYqlNodesManagerHttpId() {
- constexpr TStringBuf name = "YQLHTTPNODES";
- return NActors::TActorId(0, name);
-}
-
-IActor* CreateYqlNodesManager(
- const NDqs::TWorkerManagerCounters& workerManagerCounters,
- TIntrusivePtr<ITimeProvider> timeProvider,
- TIntrusivePtr<IRandomProvider> randomProvider,
+ };
+ TVector<TPeer> Peers;
+ ui32 ResourceIdPart = 0;
+ ui32 NextPeer = 0;
+ TString InstanceId;
+};
+
+TActorId MakeYqlNodesManagerId() {
+ constexpr TStringBuf name = "YQLNODESCTRL";
+ return NActors::TActorId(0, name);
+}
+
+TActorId MakeYqlNodesManagerHttpId() {
+ constexpr TStringBuf name = "YQLHTTPNODES";
+ return NActors::TActorId(0, name);
+}
+
+IActor* CreateYqlNodesManager(
+ const NDqs::TWorkerManagerCounters& workerManagerCounters,
+ TIntrusivePtr<ITimeProvider> timeProvider,
+ TIntrusivePtr<IRandomProvider> randomProvider,
const ::NYq::NCommon::TServiceCounters& serviceCounters,
const NConfig::TPrivateApiConfig& privateApiConfig,
const NYq::TYqSharedResources::TPtr& yqSharedResources,
- const ui32& icPort,
- const TString& address,
+ const ui32& icPort,
+ const TString& address,
const TString& tenant,
- ui64 mkqlInitialMemoryLimit,
- const NMonitoring::TDynamicCounterPtr& clientCounters) {
+ ui64 mkqlInitialMemoryLimit,
+ const NMonitoring::TDynamicCounterPtr& clientCounters) {
return new TYqlNodesManagerActor(yqSharedResources, workerManagerCounters,
- timeProvider, randomProvider,
- serviceCounters, privateApiConfig, icPort, address, tenant, mkqlInitialMemoryLimit, clientCounters);
-}
-
-} // namespace NYq
+ timeProvider, randomProvider,
+ serviceCounters, privateApiConfig, icPort, address, tenant, mkqlInitialMemoryLimit, clientCounters);
+}
+
+} // namespace NYq
diff --git a/ydb/core/yq/libs/actors/nodes_manager.h b/ydb/core/yq/libs/actors/nodes_manager.h
index a6a365a682..7301219b58 100644
--- a/ydb/core/yq/libs/actors/nodes_manager.h
+++ b/ydb/core/yq/libs/actors/nodes_manager.h
@@ -1,5 +1,5 @@
-#pragma once
-
+#pragma once
+
#include <ydb/core/yq/libs/common/service_counters.h>
#include <ydb/core/yq/libs/events/events.h>
#include <ydb/core/yq/libs/shared_resources/shared_resources.h>
@@ -9,29 +9,29 @@
#include <ydb/library/yql/providers/dq/worker_manager/interface/counters.h>
#include <ydb/library/yql/providers/dq/actors/proto_builder.h>
#include <ydb/library/yql/providers/common/http_gateway/yql_http_gateway.h>
-
-#include <library/cpp/actors/core/actorsystem.h>
-#include <library/cpp/time_provider/time_provider.h>
-#include <library/cpp/random_provider/random_provider.h>
-
-namespace NYq {
-
-using namespace NActors;
-
-TActorId MakeYqlNodesManagerId();
-TActorId MakeYqlNodesManagerHttpId();
-
-IActor* CreateYqlNodesManager(
- const NYql::NDqs::TWorkerManagerCounters& workerManagerCounters,
- TIntrusivePtr<ITimeProvider> timeProvider,
- TIntrusivePtr<IRandomProvider> randomProvider,
+
+#include <library/cpp/actors/core/actorsystem.h>
+#include <library/cpp/time_provider/time_provider.h>
+#include <library/cpp/random_provider/random_provider.h>
+
+namespace NYq {
+
+using namespace NActors;
+
+TActorId MakeYqlNodesManagerId();
+TActorId MakeYqlNodesManagerHttpId();
+
+IActor* CreateYqlNodesManager(
+ const NYql::NDqs::TWorkerManagerCounters& workerManagerCounters,
+ TIntrusivePtr<ITimeProvider> timeProvider,
+ TIntrusivePtr<IRandomProvider> randomProvider,
const ::NYq::NCommon::TServiceCounters& serviceCounters,
const NConfig::TPrivateApiConfig& privateApiConfig,
const NYq::TYqSharedResources::TPtr& yqSharedResources,
- const ui32& icPort,
- const TString& address,
+ const ui32& icPort,
+ const TString& address,
const TString& tenant = "",
- ui64 mkqlInitialMemoryLimit = 0,
- const NMonitoring::TDynamicCounterPtr& clientCounters = MakeIntrusive<NMonitoring::TDynamicCounters>());
-
-}
+ ui64 mkqlInitialMemoryLimit = 0,
+ const NMonitoring::TDynamicCounterPtr& clientCounters = MakeIntrusive<NMonitoring::TDynamicCounters>());
+
+}
diff --git a/ydb/core/yq/libs/actors/pending_fetcher.cpp b/ydb/core/yq/libs/actors/pending_fetcher.cpp
index f7cd877d37..0095963c8f 100644
--- a/ydb/core/yq/libs/actors/pending_fetcher.cpp
+++ b/ydb/core/yq/libs/actors/pending_fetcher.cpp
@@ -1,7 +1,7 @@
#include <ydb/core/yq/libs/config/protos/pinger.pb.h>
#include <ydb/core/yq/libs/config/protos/yq_config.pb.h>
#include "proxy.h"
-#include "nodes_manager.h"
+#include "nodes_manager.h"
#include "database_resolver.h"
@@ -64,27 +64,27 @@
#define LOG_D(stream) \
LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::YQL_PROXY, "Fetcher: " << stream)
-namespace NYq {
+namespace NYq {
using namespace NActors;
namespace {
-struct TEvGetTaskInternalResponse : public NActors::TEventLocal<TEvGetTaskInternalResponse, NActors::TEvents::TSystem::Completed> {
- bool Success = false;
- const TIssues Issues;
- const Yq::Private::GetTaskResult Result;
-
- TEvGetTaskInternalResponse(
- bool success,
- const TIssues& issues,
- const Yq::Private::GetTaskResult& result)
- : Success(success)
- , Issues(issues)
- , Result(result)
- { }
-};
-
+struct TEvGetTaskInternalResponse : public NActors::TEventLocal<TEvGetTaskInternalResponse, NActors::TEvents::TSystem::Completed> {
+ bool Success = false;
+ const TIssues Issues;
+ const Yq::Private::GetTaskResult Result;
+
+ TEvGetTaskInternalResponse(
+ bool success,
+ const TIssues& issues,
+ const Yq::Private::GetTaskResult& result)
+ : Success(success)
+ , Issues(issues)
+ , Result(result)
+ { }
+};
+
template <class TElement>
TVector<TElement> VectorFromProto(const ::google::protobuf::RepeatedPtrField<TElement>& field) {
return { field.begin(), field.end() };
@@ -108,8 +108,8 @@ public:
const ::NYq::NCommon::TServiceCounters& serviceCounters,
ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory,
IHTTPGateway::TPtr s3Gateway,
- ::NPq::NConfigurationManager::IConnections::TPtr pqCmConnections,
- const NMonitoring::TDynamicCounterPtr& clientCounters
+ ::NPq::NConfigurationManager::IConnections::TPtr pqCmConnections,
+ const NMonitoring::TDynamicCounterPtr& clientCounters
)
: YqSharedResources(yqSharedResources)
, CommonConfig(commonConfig)
@@ -125,22 +125,22 @@ public:
, CredentialsFactory(credentialsFactory)
, S3Gateway(s3Gateway)
, PqCmConnections(std::move(pqCmConnections))
- , Guid(CreateGuidAsString())
- , ClientCounters(clientCounters)
- , Client(
+ , Guid(CreateGuidAsString())
+ , ClientCounters(clientCounters)
+ , Client(
YqSharedResources->YdbDriver,
- NYdb::TCommonClientSettings()
+ NYdb::TCommonClientSettings()
.DiscoveryEndpoint(PrivateApiConfig.GetTaskServiceEndpoint())
- .Database(PrivateApiConfig.GetTaskServiceDatabase() ? PrivateApiConfig.GetTaskServiceDatabase() : TMaybe<TString>()),
- ClientCounters)
+ .Database(PrivateApiConfig.GetTaskServiceDatabase() ? PrivateApiConfig.GetTaskServiceDatabase() : TMaybe<TString>()),
+ ClientCounters)
{
Y_ENSURE(GetYqlDefaultModuleResolverWithContext(ModuleResolver));
}
- static constexpr char ActorName[] = "YQ_PENDING_FETCHER";
+ static constexpr char ActorName[] = "YQ_PENDING_FETCHER";
void PassAway() final {
- LOG_D("Stop Fetcher");
+ LOG_D("Stop Fetcher");
Send(DatabaseResolver, new NActors::TEvents::TEvPoison());
NActors::IActor::PassAway();
}
@@ -151,10 +151,10 @@ public:
Y_UNUSED(ctx);
DatabaseResolver = Register(CreateDatabaseResolver(MakeYqlAnalyticsHttpProxyId(), CredentialsFactory));
- Send(SelfId(), new NActors::TEvents::TEvWakeup());
+ Send(SelfId(), new NActors::TEvents::TEvWakeup());
- LOG_I("STARTED");
- LogScope.ConstructInPlace(NActors::TActivationContext::ActorSystem(), NKikimrServices::YQL_PROXY, Guid);
+ LOG_I("STARTED");
+ LogScope.ConstructInPlace(NActors::TActivationContext::ActorSystem(), NKikimrServices::YQL_PROXY, Guid);
}
private:
@@ -173,66 +173,66 @@ private:
}
}
- void HandleResponse(TEvGetTaskInternalResponse::TPtr& ev) {
+ void HandleResponse(TEvGetTaskInternalResponse::TPtr& ev) {
HasRunningRequest = false;
- LOG_D("Got GetTask response from PrivateApi");
- if (!ev->Get()->Success) {
+ LOG_D("Got GetTask response from PrivateApi");
+ if (!ev->Get()->Success) {
LOG_E("Error with GetTask: "<< ev->Get()->Issues.ToString());
- return;
- }
-
- const auto& res = ev->Get()->Result;
+ return;
+ }
- LOG_D("Tasks count: " << res.tasks().size());
- if (!res.tasks().empty()) {
+ const auto& res = ev->Get()->Result;
+
+ LOG_D("Tasks count: " << res.tasks().size());
+ if (!res.tasks().empty()) {
ProcessTask(res);
- HasRunningRequest = true;
- GetPendingTask();
- }
- }
+ HasRunningRequest = true;
+ GetPendingTask();
+ }
+ }
- void GetPendingTask() {
+ void GetPendingTask() {
LOG_D("Request Private::GetTask" << ", Owner: " << Guid << ", Host: " << HostName());
- Yq::Private::GetTaskRequest request;
- request.set_owner_id(Guid);
- request.set_host(HostName());
- const auto actorSystem = NActors::TActivationContext::ActorSystem();
- const auto selfId = SelfId();
- Client
- .GetTask(std::move(request))
- .Subscribe([actorSystem, selfId](const NThreading::TFuture<TGetTaskResult>& future) {
- const auto& wrappedResult = future.GetValue();
- if (wrappedResult.IsResultSet()) {
- actorSystem->Send(selfId, new TEvGetTaskInternalResponse(
- wrappedResult.IsSuccess(), wrappedResult.GetIssues(), wrappedResult.GetResult())
- );
+ Yq::Private::GetTaskRequest request;
+ request.set_owner_id(Guid);
+ request.set_host(HostName());
+ const auto actorSystem = NActors::TActivationContext::ActorSystem();
+ const auto selfId = SelfId();
+ Client
+ .GetTask(std::move(request))
+ .Subscribe([actorSystem, selfId](const NThreading::TFuture<TGetTaskResult>& future) {
+ const auto& wrappedResult = future.GetValue();
+ if (wrappedResult.IsResultSet()) {
+ actorSystem->Send(selfId, new TEvGetTaskInternalResponse(
+ wrappedResult.IsSuccess(), wrappedResult.GetIssues(), wrappedResult.GetResult())
+ );
} else {
actorSystem->Send(selfId, new TEvGetTaskInternalResponse(
- false, TIssues{{TIssue{"grpc private api result is not set for get task call"}}}, Yq::Private::GetTaskResult{})
+ false, TIssues{{TIssue{"grpc private api result is not set for get task call"}}}, Yq::Private::GetTaskResult{})
);
}
- });
+ });
}
- void ProcessTask(const Yq::Private::GetTaskResult& result) {
- for (const auto& task : result.tasks()) {
- RunTask(task);
- }
-
- }
-
- void RunTask(const Yq::Private::GetTaskResult::Task& task) {
+ void ProcessTask(const Yq::Private::GetTaskResult& result) {
+ for (const auto& task : result.tasks()) {
+ RunTask(task);
+ }
+
+ }
+
+ void RunTask(const Yq::Private::GetTaskResult::Task& task) {
LOG_D("NewTask:"
- << " Scope: " << task.scope()
- << " Id: " << task.query_id().value()
- << " UserId: " << task.user_id()
- << " AuthToken: " << NKikimr::MaskTicket(task.user_token()));
-
- THashMap<TString, TString> serviceAccounts;
- for (const auto& identity : task.service_accounts()) {
- serviceAccounts[identity.value()] = identity.signature();
- }
-
+ << " Scope: " << task.scope()
+ << " Id: " << task.query_id().value()
+ << " UserId: " << task.user_id()
+ << " AuthToken: " << NKikimr::MaskTicket(task.user_token()));
+
+ THashMap<TString, TString> serviceAccounts;
+ for (const auto& identity : task.service_accounts()) {
+ serviceAccounts[identity.value()] = identity.signature();
+ }
+
TRunActorParams params(
YqSharedResources->YdbDriver, S3Gateway,
FunctionRegistry, RandomProvider,
@@ -240,31 +240,31 @@ private:
DqCompFactory, PqCmConnections,
CommonConfig, CheckpointCoordinatorConfig,
PrivateApiConfig, GatewaysConfig, PingerConfig,
- task.text(), task.scope(), task.user_token(),
- DatabaseResolver, task.query_id().value(),
- task.user_id(), Guid, task.generation(),
- VectorFromProto(task.connection()),
- VectorFromProto(task.binding()),
+ task.text(), task.scope(), task.user_token(),
+ DatabaseResolver, task.query_id().value(),
+ task.user_id(), Guid, task.generation(),
+ VectorFromProto(task.connection()),
+ VectorFromProto(task.binding()),
CredentialsFactory,
serviceAccounts,
- task.query_type(),
- task.execute_mode(),
+ task.query_type(),
+ task.execute_mode(),
GetEntityIdAsString(CommonConfig.GetIdsPrefix(), EEntityType::RESULT),
- task.state_load_mode(),
+ task.state_load_mode(),
task.disposition(),
- task.status(),
- task.sensor_labels().at("cloud_id"),
- VectorFromProto(task.result_set_meta()),
- VectorFromProto(task.dq_graph()),
- task.dq_graph_index(),
- VectorFromProto(task.created_topic_consumers()),
- task.automatic(),
+ task.status(),
+ task.sensor_labels().at("cloud_id"),
+ VectorFromProto(task.result_set_meta()),
+ VectorFromProto(task.dq_graph()),
+ task.dq_graph_index(),
+ VectorFromProto(task.created_topic_consumers()),
+ task.automatic(),
task.query_name(),
- NProtoInterop::CastFromProto(task.deadline()),
- ClientCounters);
-
+ NProtoInterop::CastFromProto(task.deadline()),
+ ClientCounters);
+
NDq::SetYqlLogLevels(NActors::NLog::PRI_TRACE);
- Register(CreateRunActor(ServiceCounters, std::move(params)));
+ Register(CreateRunActor(ServiceCounters, std::move(params)));
}
STRICT_STFUNC(
@@ -272,7 +272,7 @@ private:
HFunc(NActors::TEvents::TEvWakeup, HandleWakeup)
HFunc(NActors::TEvents::TEvUndelivered, OnUndelivered)
- hFunc(TEvGetTaskInternalResponse, HandleResponse)
+ hFunc(TEvGetTaskInternalResponse, HandleResponse)
);
NYq::TYqSharedResources::TPtr YqSharedResources;
@@ -299,12 +299,12 @@ private:
ISecuredServiceAccountCredentialsFactory::TPtr CredentialsFactory;
const IHTTPGateway::TPtr S3Gateway;
const ::NPq::NConfigurationManager::IConnections::TPtr PqCmConnections;
-
- const TString Guid; //OwnerId
- const NMonitoring::TDynamicCounterPtr ClientCounters;
- TPrivateClient Client;
-
- TMaybe<NYql::NLog::TScopedBackend<NYql::NDq::TYqlLogScope>> LogScope;
+
+ const TString Guid; //OwnerId
+ const NMonitoring::TDynamicCounterPtr ClientCounters;
+ TPrivateClient Client;
+
+ TMaybe<NYql::NLog::TScopedBackend<NYql::NDq::TYqlLogScope>> LogScope;
};
@@ -322,8 +322,8 @@ NActors::IActor* CreatePendingFetcher(
const ::NYq::NCommon::TServiceCounters& serviceCounters,
ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory,
IHTTPGateway::TPtr s3Gateway,
- ::NPq::NConfigurationManager::IConnections::TPtr pqCmConnections,
- const NMonitoring::TDynamicCounterPtr& clientCounters)
+ ::NPq::NConfigurationManager::IConnections::TPtr pqCmConnections,
+ const NMonitoring::TDynamicCounterPtr& clientCounters)
{
return new TYqlPendingFetcher(
yqSharedResources,
@@ -339,8 +339,8 @@ NActors::IActor* CreatePendingFetcher(
serviceCounters,
credentialsFactory,
s3Gateway,
- std::move(pqCmConnections),
- clientCounters);
+ std::move(pqCmConnections),
+ clientCounters);
}
TActorId MakeYqlAnalyticsFetcherId(ui32 nodeId) {
@@ -348,4 +348,4 @@ TActorId MakeYqlAnalyticsFetcherId(ui32 nodeId) {
return NActors::TActorId(nodeId, name);
}
-} /* NYq */
+} /* NYq */
diff --git a/ydb/core/yq/libs/actors/pinger.cpp b/ydb/core/yq/libs/actors/pinger.cpp
index 0978e42de6..55fc80d6a0 100644
--- a/ydb/core/yq/libs/actors/pinger.cpp
+++ b/ydb/core/yq/libs/actors/pinger.cpp
@@ -28,10 +28,10 @@
#define LOG_T(stream) \
LOG_TRACE_S(*TlsActivationContext, NKikimrServices::YQL_PROXY, "Pinger - " << "QueryId: " << Id << ", Owner: " << OwnerId << " " << stream)
-namespace NYq {
+namespace NYq {
using namespace NActors;
-using namespace NYql;
+using namespace NYql;
struct TEvPingResponse : public NActors::TEventLocal<TEvPingResponse, NActors::TEvents::TSystem::CallbackCompletion> {
TPingTaskResult Result;
@@ -59,12 +59,12 @@ private:
class TPingerActor : public NActors::TActorBootstrapped<TPingerActor> {
class TRetryState {
public:
- void Init(const TInstant& now, const TInstant& startLeaseTime, const TDuration& maxRetryTime) {
+ void Init(const TInstant& now, const TInstant& startLeaseTime, const TDuration& maxRetryTime) {
StartRequestTime = now;
StartLeaseTime = startLeaseTime;
Delay = TDuration::Zero();
RetriesCount = 0;
- MaxRetryTime = maxRetryTime;
+ MaxRetryTime = maxRetryTime;
}
void UpdateStartLeaseTime(TInstant startLeaseTime) {
@@ -111,7 +111,7 @@ class TPingerActor : public NActors::TActorBootstrapped<TPingerActor> {
TInstant StartRequestTime;
size_t RetriesCount = 0;
- TDuration MaxRetryTime;
+ TDuration MaxRetryTime;
static constexpr TDuration MaxDelay = TDuration::Seconds(5);
static constexpr TDuration MinDelay = TDuration::MilliSeconds(100); // from second retry
};
@@ -134,9 +134,9 @@ class TPingerActor : public NActors::TActorBootstrapped<TPingerActor> {
TConfig(const NConfig::TPingerConfig& config)
: Proto(config)
{
- if (Proto.GetPingPeriod()) {
- Y_VERIFY(TDuration::TryParse(Proto.GetPingPeriod(), PingPeriod));
- }
+ if (Proto.GetPingPeriod()) {
+ Y_VERIFY(TDuration::TryParse(Proto.GetPingPeriod(), PingPeriod));
+ }
}
};
@@ -161,7 +161,7 @@ public:
{
}
- static constexpr char ActorName[] = "YQ_PINGER";
+ static constexpr char ActorName[] = "YQ_PINGER";
void Bootstrap() {
LOG_D("Start Pinger");
@@ -175,7 +175,7 @@ private:
StateFunc,
cFunc(NActors::TEvents::TEvPoison::EventType, PassAway)
hFunc(NActors::TEvents::TEvWakeup, Wakeup)
- hFunc(TEvPingResponse, Handle)
+ hFunc(TEvPingResponse, Handle)
hFunc(TEvents::TEvForwardPingRequest, Handle)
)
@@ -187,7 +187,7 @@ private:
void ScheduleNextPing() {
if (!Finishing) {
SchedulerCookieHolder.Reset(ISchedulerCookie::Make2Way());
- Schedule(Config.PingPeriod, new NActors::TEvents::TEvWakeup(ContinueLeaseWakeupTag), SchedulerCookieHolder.Get());
+ Schedule(Config.PingPeriod, new NActors::TEvents::TEvWakeup(ContinueLeaseWakeupTag), SchedulerCookieHolder.Get());
}
}
@@ -275,7 +275,7 @@ private:
return false;
}
- void Handle(TEvPingResponse::TPtr& ev) {
+ void Handle(TEvPingResponse::TPtr& ev) {
if (FatalError) {
LOG_D("Got ping response after fatal error. Ignore");
return;
@@ -374,14 +374,14 @@ private:
if (!retry) {
RetryState.Init(TActivationContext::Now(), StartLeaseTime, Config.PingPeriod);
}
- Ping(Yq::Private::PingTaskRequest(), ContinueLeaseRequestCookie);
+ Ping(Yq::Private::PingTaskRequest(), ContinueLeaseRequestCookie);
}
- void Ping(Yq::Private::PingTaskRequest request, ui64 cookie) {
+ void Ping(Yq::Private::PingTaskRequest request, ui64 cookie) {
// Fill ids
- request.set_scope(Scope.ToString());
- request.set_owner_id(OwnerId);
- request.mutable_query_id()->set_value(Id);
+ request.set_scope(Scope.ToString());
+ request.set_owner_id(OwnerId);
+ request.mutable_query_id()->set_value(Id);
*request.mutable_deadline() = NProtoInterop::CastToProto(Deadline);
const auto* actorSystem = NActors::TActivationContext::ActorSystem();
@@ -400,7 +400,7 @@ private:
<< CurrentExceptionMessage());
}
actorSystem->Send(new IEventHandle(selfId, selfId, ev.release(), 0, cookie));
- }
+ }
);
}
@@ -454,4 +454,4 @@ IActor* CreatePingerActor(
deadline);
}
-} /* NYq */
+} /* NYq */
diff --git a/ydb/core/yq/libs/actors/proxy.cpp b/ydb/core/yq/libs/actors/proxy.cpp
index 76da02bcc2..f06f2f8475 100644
--- a/ydb/core/yq/libs/actors/proxy.cpp
+++ b/ydb/core/yq/libs/actors/proxy.cpp
@@ -1,6 +1,6 @@
#include "proxy.h"
-namespace NYq {
+namespace NYq {
using namespace NActors;
@@ -9,4 +9,4 @@ NActors::TActorId MakeYqlAnalyticsHttpProxyId() {
return NActors::TActorId(0, name);
}
-} // namespace NYq
+} // namespace NYq
diff --git a/ydb/core/yq/libs/actors/proxy.h b/ydb/core/yq/libs/actors/proxy.h
index 2e274ffed4..442d29ac44 100644
--- a/ydb/core/yq/libs/actors/proxy.h
+++ b/ydb/core/yq/libs/actors/proxy.h
@@ -1,6 +1,6 @@
#pragma once
#include <ydb/core/yq/libs/config/protos/pinger.pb.h>
-#include "run_actor_params.h"
+#include "run_actor_params.h"
#include <util/datetime/base.h>
#include <ydb/core/yq/libs/events/events.h>
@@ -28,7 +28,7 @@ namespace NKikimr {
}
}
-namespace NYq {
+namespace NYq {
NActors::TActorId MakeYqlAnalyticsHttpProxyId();
NActors::TActorId MakeYqlAnalyticsFetcherId(ui32 nodeId);
@@ -45,10 +45,10 @@ NActors::IActor* CreatePendingFetcher(
TIntrusivePtr<IRandomProvider> randomProvider,
NKikimr::NMiniKQL::TComputationNodeFactory dqCompFactory,
const ::NYq::NCommon::TServiceCounters& serviceCounters,
- NYql::ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory,
+ NYql::ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory,
NYql::IHTTPGateway::TPtr s3Gateway,
- ::NPq::NConfigurationManager::IConnections::TPtr pqCmConnections,
- const NMonitoring::TDynamicCounterPtr& clientCounters
+ ::NPq::NConfigurationManager::IConnections::TPtr pqCmConnections,
+ const NMonitoring::TDynamicCounterPtr& clientCounters
);
NActors::IActor* CreateRunActor(
@@ -59,8 +59,8 @@ NActors::IActor* CreateRunActor(
struct TResultId {
TString Id;
int SetId;
- TString HistoryId;
- TString Owner;
+ TString HistoryId;
+ TString Owner;
TString CloudId;
};
@@ -72,20 +72,20 @@ NActors::IActor* CreateResultWriter(
const TResultId& resultId,
const TVector<TString>& columns,
const TString& traceId,
- const TInstant& deadline,
- const NMonitoring::TDynamicCounterPtr& clientCounters
- );
+ const TInstant& deadline,
+ const NMonitoring::TDynamicCounterPtr& clientCounters
+ );
-NActors::IActor* CreatePingerActor(
+NActors::IActor* CreatePingerActor(
const TScope& scope,
const TString& userId,
const TString& id,
const TString& owner,
- const NYq::TPrivateClient& client,
+ const NYq::TPrivateClient& client,
const NActors::TActorId parent,
const NConfig::TPingerConfig& config,
const TInstant& deadline);
TString MakeInternalError(const TString& text);
-} /* NYq */
+} /* NYq */
diff --git a/ydb/core/yq/libs/actors/proxy_private.cpp b/ydb/core/yq/libs/actors/proxy_private.cpp
index 2e3d60a42a..f852894578 100644
--- a/ydb/core/yq/libs/actors/proxy_private.cpp
+++ b/ydb/core/yq/libs/actors/proxy_private.cpp
@@ -1,120 +1,120 @@
-#include "proxy_private.h"
+#include "proxy_private.h"
#include <ydb/core/yq/libs/config/protos/yq_config.pb.h>
-
+
#include <ydb/core/yq/libs/events/events.h>
#include <ydb/core/yq/libs/shared_resources/db_pool.h>
-#include <library/cpp/actors/core/events.h>
-#include <library/cpp/actors/core/hfunc.h>
-#include <library/cpp/actors/core/actor_bootstrapped.h>
+#include <library/cpp/actors/core/events.h>
+#include <library/cpp/actors/core/hfunc.h>
+#include <library/cpp/actors/core/actor_bootstrapped.h>
#include <ydb/core/protos/services.pb.h>
-
-#include <library/cpp/actors/core/log.h>
-
-#include <util/generic/deque.h>
-#include <util/generic/guid.h>
-#include <util/system/hostname.h>
-
+
+#include <library/cpp/actors/core/log.h>
+
+#include <util/generic/deque.h>
+#include <util/generic/guid.h>
+#include <util/system/hostname.h>
+
#define LOG_E(stream) \
LOG_ERROR_S(*TlsActivationContext, NKikimrServices::YQL_PRIVATE_PROXY, stream)
#define LOG_I(stream) \
LOG_INFO_S(*TlsActivationContext, NKikimrServices::YQL_PRIVATE_PROXY, stream)
#define LOG_D(stream) \
LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::YQL_PRIVATE_PROXY, stream)
-
-namespace NYq {
-
-using namespace NActors;
-using namespace NMonitoring;
-
-class TYqlAnalyticsPrivateProxy : public NActors::TActorBootstrapped<TYqlAnalyticsPrivateProxy> {
-public:
- TYqlAnalyticsPrivateProxy(
- TIntrusivePtr<ITimeProvider> timeProvider,
- TIntrusivePtr<IRandomProvider> randomProvider,
- NMonitoring::TDynamicCounterPtr counters,
+
+namespace NYq {
+
+using namespace NActors;
+using namespace NMonitoring;
+
+class TYqlAnalyticsPrivateProxy : public NActors::TActorBootstrapped<TYqlAnalyticsPrivateProxy> {
+public:
+ TYqlAnalyticsPrivateProxy(
+ TIntrusivePtr<ITimeProvider> timeProvider,
+ TIntrusivePtr<IRandomProvider> randomProvider,
+ NMonitoring::TDynamicCounterPtr counters,
const NConfig::TTokenAccessorConfig& tokenAccessorConfig)
: TokenAccessorConfig(tokenAccessorConfig)
, TimeProvider(timeProvider)
- , RandomProvider(randomProvider)
- , Counters(counters->GetSubgroup("subsystem", "private_api"))
- { }
-
- static constexpr char ActorName[] = "YQ_PRIVATE_PROXY";
-
- void PassAway() final {
- NActors::IActor::PassAway();
- }
-
- void Bootstrap(const TActorContext&) {
- Become(&TYqlAnalyticsPrivateProxy::StateFunc);
- Counters->GetCounter("EvBootstrap", true)->Inc();
- }
-
-private:
- void Handle(TEvents::TEvPingTaskRequest::TPtr& ev, const TActorContext& ctx) {
- Counters->GetCounter("EvPingTaskRequest", true)->Inc();
-
- Register(
+ , RandomProvider(randomProvider)
+ , Counters(counters->GetSubgroup("subsystem", "private_api"))
+ { }
+
+ static constexpr char ActorName[] = "YQ_PRIVATE_PROXY";
+
+ void PassAway() final {
+ NActors::IActor::PassAway();
+ }
+
+ void Bootstrap(const TActorContext&) {
+ Become(&TYqlAnalyticsPrivateProxy::StateFunc);
+ Counters->GetCounter("EvBootstrap", true)->Inc();
+ }
+
+private:
+ void Handle(TEvents::TEvPingTaskRequest::TPtr& ev, const TActorContext& ctx) {
+ Counters->GetCounter("EvPingTaskRequest", true)->Inc();
+
+ Register(
CreatePingTaskRequestActor(ev->Sender, TimeProvider, ev->Release(), Counters),
- NActors::TMailboxType::HTSwap, ctx.SelfID.PoolID());
- }
-
- void Handle(TEvents::TEvGetTaskRequest::TPtr& ev, const TActorContext& ctx) {
- Counters->GetCounter("EvGetTaskRequest", true)->Inc();
- Register(
+ NActors::TMailboxType::HTSwap, ctx.SelfID.PoolID());
+ }
+
+ void Handle(TEvents::TEvGetTaskRequest::TPtr& ev, const TActorContext& ctx) {
+ Counters->GetCounter("EvGetTaskRequest", true)->Inc();
+ Register(
CreateGetTaskRequestActor(ev->Sender, TokenAccessorConfig, TimeProvider, ev->Release(), Counters),
- NActors::TMailboxType::HTSwap, ctx.SelfID.PoolID());
- }
-
- void Handle(TEvents::TEvWriteTaskResultRequest::TPtr& ev, const TActorContext& ctx) {
- Counters->GetCounter("EvWriteTaskResultRequest", true)->Inc();
-
- Register(
+ NActors::TMailboxType::HTSwap, ctx.SelfID.PoolID());
+ }
+
+ void Handle(TEvents::TEvWriteTaskResultRequest::TPtr& ev, const TActorContext& ctx) {
+ Counters->GetCounter("EvWriteTaskResultRequest", true)->Inc();
+
+ Register(
CreateWriteTaskResultRequestActor(ev->Sender, TimeProvider, ev->Release(), Counters),
- NActors::TMailboxType::HTSwap, ctx.SelfID.PoolID());
- }
-
- void Handle(TEvents::TEvNodesHealthCheckRequest::TPtr& ev, const TActorContext& ctx) {
- Counters->GetCounter("EvNodesHealthCheckRequest", true)->Inc();
-
- Register(
+ NActors::TMailboxType::HTSwap, ctx.SelfID.PoolID());
+ }
+
+ void Handle(TEvents::TEvNodesHealthCheckRequest::TPtr& ev, const TActorContext& ctx) {
+ Counters->GetCounter("EvNodesHealthCheckRequest", true)->Inc();
+
+ Register(
CreateNodesHealthCheckActor(ev->Sender, TimeProvider, ev->Release(), Counters),
- NActors::TMailboxType::HTSwap, ctx.SelfID.PoolID());
- }
-
- STRICT_STFUNC(
- StateFunc,
- HFunc(NActors::TEvents::TEvUndelivered, OnUndelivered)
- HFunc(TEvents::TEvPingTaskRequest, Handle)
- HFunc(TEvents::TEvGetTaskRequest, Handle)
- HFunc(TEvents::TEvWriteTaskResultRequest, Handle)
- HFunc(TEvents::TEvNodesHealthCheckRequest, Handle)
- )
-
- void OnUndelivered(NActors::TEvents::TEvUndelivered::TPtr&, const NActors::TActorContext&) {
- LOG_E("TYqlAnalyticsPrivateProxy::OnUndelivered");
- Counters->GetCounter("OnUndelivered", true)->Inc();
- }
-
-private:
+ NActors::TMailboxType::HTSwap, ctx.SelfID.PoolID());
+ }
+
+ STRICT_STFUNC(
+ StateFunc,
+ HFunc(NActors::TEvents::TEvUndelivered, OnUndelivered)
+ HFunc(TEvents::TEvPingTaskRequest, Handle)
+ HFunc(TEvents::TEvGetTaskRequest, Handle)
+ HFunc(TEvents::TEvWriteTaskResultRequest, Handle)
+ HFunc(TEvents::TEvNodesHealthCheckRequest, Handle)
+ )
+
+ void OnUndelivered(NActors::TEvents::TEvUndelivered::TPtr&, const NActors::TActorContext&) {
+ LOG_E("TYqlAnalyticsPrivateProxy::OnUndelivered");
+ Counters->GetCounter("OnUndelivered", true)->Inc();
+ }
+
+private:
const NConfig::TTokenAccessorConfig TokenAccessorConfig;
- TIntrusivePtr<ITimeProvider> TimeProvider;
- TIntrusivePtr<IRandomProvider> RandomProvider;
- NMonitoring::TDynamicCounterPtr Counters;
-};
-
-TActorId MakeYqPrivateProxyId() {
- constexpr TStringBuf name = "YQPRIVPROXY";
- return NActors::TActorId(0, name);
-}
-
-IActor* CreateYqlAnalyticsPrivateProxy(
- TIntrusivePtr<ITimeProvider> timeProvider,
- TIntrusivePtr<IRandomProvider> randomProvider,
- NMonitoring::TDynamicCounterPtr counters,
+ TIntrusivePtr<ITimeProvider> TimeProvider;
+ TIntrusivePtr<IRandomProvider> RandomProvider;
+ NMonitoring::TDynamicCounterPtr Counters;
+};
+
+TActorId MakeYqPrivateProxyId() {
+ constexpr TStringBuf name = "YQPRIVPROXY";
+ return NActors::TActorId(0, name);
+}
+
+IActor* CreateYqlAnalyticsPrivateProxy(
+ TIntrusivePtr<ITimeProvider> timeProvider,
+ TIntrusivePtr<IRandomProvider> randomProvider,
+ NMonitoring::TDynamicCounterPtr counters,
const NConfig::TTokenAccessorConfig& tokenAccessorConfig) {
return new TYqlAnalyticsPrivateProxy(timeProvider, randomProvider, counters, tokenAccessorConfig);
-}
-
-} // namespace NYq
+}
+
+} // namespace NYq
diff --git a/ydb/core/yq/libs/actors/proxy_private.h b/ydb/core/yq/libs/actors/proxy_private.h
index fbd17a7aee..f5378ebab8 100644
--- a/ydb/core/yq/libs/actors/proxy_private.h
+++ b/ydb/core/yq/libs/actors/proxy_private.h
@@ -1,4 +1,4 @@
-#pragma once
+#pragma once
#include <ydb/core/yq/libs/shared_resources/db_pool.h>
#include <ydb/core/yq/libs/events/events.h>
@@ -6,56 +6,56 @@
#include <ydb/library/yql/providers/dq/provider/yql_dq_gateway.h>
#include <ydb/library/yql/providers/dq/worker_manager/interface/counters.h>
#include <ydb/library/yql/providers/dq/actors/proto_builder.h>
-
+
#include <library/cpp/actors/core/actorsystem.h>
#include <library/cpp/time_provider/time_provider.h>
#include <library/cpp/random_provider/random_provider.h>
-#include <library/cpp/monlib/metrics/histogram_collector.h>
-
-namespace NKikimr {
- namespace NMiniKQL {
- class IFunctionRegistry;
- }
-}
-
-namespace NYq {
-
-NActors::TActorId MakeYqPrivateProxyId();
-
-NActors::IActor* CreateYqlAnalyticsPrivateProxy(
- TIntrusivePtr<ITimeProvider> timeProvider,
- TIntrusivePtr<IRandomProvider> randomProvider,
- NMonitoring::TDynamicCounterPtr counters,
+#include <library/cpp/monlib/metrics/histogram_collector.h>
+
+namespace NKikimr {
+ namespace NMiniKQL {
+ class IFunctionRegistry;
+ }
+}
+
+namespace NYq {
+
+NActors::TActorId MakeYqPrivateProxyId();
+
+NActors::IActor* CreateYqlAnalyticsPrivateProxy(
+ TIntrusivePtr<ITimeProvider> timeProvider,
+ TIntrusivePtr<IRandomProvider> randomProvider,
+ NMonitoring::TDynamicCounterPtr counters,
const NConfig::TTokenAccessorConfig& tockenAccessorConfig
-);
-
-NActors::IActor* CreatePingTaskRequestActor(
- const NActors::TActorId& sender,
- TIntrusivePtr<ITimeProvider> timeProvider,
- TAutoPtr<TEvents::TEvPingTaskRequest> ev,
- NMonitoring::TDynamicCounterPtr counters
-);
-
-NActors::IActor* CreateGetTaskRequestActor(
- const NActors::TActorId& sender,
+);
+
+NActors::IActor* CreatePingTaskRequestActor(
+ const NActors::TActorId& sender,
+ TIntrusivePtr<ITimeProvider> timeProvider,
+ TAutoPtr<TEvents::TEvPingTaskRequest> ev,
+ NMonitoring::TDynamicCounterPtr counters
+);
+
+NActors::IActor* CreateGetTaskRequestActor(
+ const NActors::TActorId& sender,
const NConfig::TTokenAccessorConfig& tockenAccessorConfig,
- TIntrusivePtr<ITimeProvider> timeProvider,
- TAutoPtr<TEvents::TEvGetTaskRequest> ev,
- NMonitoring::TDynamicCounterPtr counters
-);
-
-NActors::IActor* CreateWriteTaskResultRequestActor(
- const NActors::TActorId& sender,
- TIntrusivePtr<ITimeProvider> timeProvider,
- TAutoPtr<TEvents::TEvWriteTaskResultRequest> ev,
- NMonitoring::TDynamicCounterPtr counters
-);
-
-NActors::IActor* CreateNodesHealthCheckActor(
- const NActors::TActorId& sender,
- TIntrusivePtr<ITimeProvider> timeProvider,
- TAutoPtr<TEvents::TEvNodesHealthCheckRequest> ev,
- NMonitoring::TDynamicCounterPtr counters
-);
-
-} /* NYq */
+ TIntrusivePtr<ITimeProvider> timeProvider,
+ TAutoPtr<TEvents::TEvGetTaskRequest> ev,
+ NMonitoring::TDynamicCounterPtr counters
+);
+
+NActors::IActor* CreateWriteTaskResultRequestActor(
+ const NActors::TActorId& sender,
+ TIntrusivePtr<ITimeProvider> timeProvider,
+ TAutoPtr<TEvents::TEvWriteTaskResultRequest> ev,
+ NMonitoring::TDynamicCounterPtr counters
+);
+
+NActors::IActor* CreateNodesHealthCheckActor(
+ const NActors::TActorId& sender,
+ TIntrusivePtr<ITimeProvider> timeProvider,
+ TAutoPtr<TEvents::TEvNodesHealthCheckRequest> ev,
+ NMonitoring::TDynamicCounterPtr counters
+);
+
+} /* NYq */
diff --git a/ydb/core/yq/libs/actors/result_writer.cpp b/ydb/core/yq/libs/actors/result_writer.cpp
index d1ef15c428..d51b8aae9f 100644
--- a/ydb/core/yq/libs/actors/result_writer.cpp
+++ b/ydb/core/yq/libs/actors/result_writer.cpp
@@ -27,7 +27,7 @@
#define LOG_D(stream) \
LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::YQL_PROXY, "Writer: " << TraceId << ": " << stream)
-namespace NYq {
+namespace NYq {
using namespace NActors;
using namespace NYql;
@@ -43,22 +43,22 @@ public:
const TResultId& resultId,
const TVector<TString>& columns,
const TString& traceId,
- const TInstant& deadline,
- const NMonitoring::TDynamicCounterPtr& clientCounters)
+ const TInstant& deadline,
+ const NMonitoring::TDynamicCounterPtr& clientCounters)
: ExecuterId(executerId)
, ResultBuilder(MakeHolder<TProtoBuilder>(resultType, columns))
, ResultId({resultId})
, TraceId(traceId)
, Deadline(deadline)
- , Client(
+ , Client(
driver,
- NYdb::TCommonClientSettings()
+ NYdb::TCommonClientSettings()
.DiscoveryEndpoint(privateApiConfig.GetTaskServiceEndpoint())
- .Database(privateApiConfig.GetTaskServiceDatabase() ? privateApiConfig.GetTaskServiceDatabase(): TMaybe<TString>()),
- clientCounters)
+ .Database(privateApiConfig.GetTaskServiceDatabase() ? privateApiConfig.GetTaskServiceDatabase(): TMaybe<TString>()),
+ clientCounters)
{ }
- static constexpr char ActorName[] = "YQ_RESULT_WRITER";
+ static constexpr char ActorName[] = "YQ_RESULT_WRITER";
void Bootstrap(const TActorContext&) {
LOG_I("Bootstrap");
@@ -84,20 +84,20 @@ private:
}
void OnUndelivered(NActors::TEvents::TEvUndelivered::TPtr&, const NActors::TActorContext& ) {
- auto req = MakeHolder<TEvDqFailure>(TIssue("Undelivered").SetCode(NYql::DEFAULT_ERROR, TSeverityIds::S_ERROR), /*retriable=*/ false, /*needFallback=*/false);
+ auto req = MakeHolder<TEvDqFailure>(TIssue("Undelivered").SetCode(NYql::DEFAULT_ERROR, TSeverityIds::S_ERROR), /*retriable=*/ false, /*needFallback=*/false);
Send(ExecuterId, req.Release());
HasError = true;
}
- void SendIssuesAndSetErrorFlag(const TIssues& issues) {
- LOG_E("ControlPlane WriteResult Issues: " << issues.ToString());
- Issues.AddIssues(issues);
- HasError = true;
- auto req = MakeHolder<TEvDqFailure>();
- IssuesToMessage(issues, req->Record.MutableIssues());
- Send(ExecuterId, req.Release());
- }
-
+ void SendIssuesAndSetErrorFlag(const TIssues& issues) {
+ LOG_E("ControlPlane WriteResult Issues: " << issues.ToString());
+ Issues.AddIssues(issues);
+ HasError = true;
+ auto req = MakeHolder<TEvDqFailure>();
+ IssuesToMessage(issues, req->Record.MutableIssues());
+ Send(ExecuterId, req.Release());
+ }
+
void MaybeFinish() {
if (Finished && Requests.empty()) {
Send(ExecuterId, new TEvGraphFinished());
@@ -106,7 +106,7 @@ private:
void OnQueryResult(TEvQueryResponse::TPtr& ev, const TActorContext&) {
Finished = true;
- NYql::NDqProto::TQueryResponse queryResult(ev->Get()->Record);
+ NYql::NDqProto::TQueryResponse queryResult(ev->Get()->Record);
*queryResult.MutableYson() = ResultBuilder->BuildYson(Head);
if (!Issues.Empty()) {
@@ -122,50 +122,50 @@ private:
void HandleResponse(NYq::TEvControlPlaneStorage::TEvWriteResultDataResponse::TPtr& ev) {
const auto& issues = ev->Get()->Issues;
- auto it = Requests.find(ev->Get()->RequestId);
- if (issues) {
- SendIssuesAndSetErrorFlag(issues);
- return;
- }
-
+ auto it = Requests.find(ev->Get()->RequestId);
+ if (issues) {
+ SendIssuesAndSetErrorFlag(issues);
+ return;
+ }
+
if (it == Requests.end()) {
HasError = true;
- auto req = MakeHolder<TEvDqFailure>(TIssue("Unknown RequestId").SetCode(NYql::DEFAULT_ERROR, TSeverityIds::S_ERROR), /*retriable=*/ false, /*needFallback=*/false);
+ auto req = MakeHolder<TEvDqFailure>(TIssue("Unknown RequestId").SetCode(NYql::DEFAULT_ERROR, TSeverityIds::S_ERROR), /*retriable=*/ false, /*needFallback=*/false);
Send(ExecuterId, req.Release());
return;
}
auto& request = it->second;
- if (request.MessagesNum > 0)
- --request.MessagesNum;
-
- --InflightCounter;
-
- if (request.MessagesNum == 0) {
- FreeSpace += request.Size;
-
- if (FreeSpace > 0) {
- auto res = MakeHolder<NDq::TEvDqCompute::TEvChannelDataAck>();
- res->Record.SetChannelId(request.ChannelId);
- res->Record.SetFreeSpace(FreeSpace);
- res->Record.SetSeqNo(request.SeqNo);
- res->Record.SetFinish(HasError);
- Send(request.Sender, res.Release());
-
- auto duration = (TInstant::Now()-StartTime);
-
- LOG_D("ChannelData, Records: " << RowIndex
- << " HasError: " << HasError
- << " Size: " << Size
- << " Rows: " << Rows
- << " FreeSpace: " << FreeSpace
- << " Duration: " << duration
- << " AvgSpeed: " << Size/(duration.Seconds()+1)/1024/1024);
- }
-
- Requests.erase(it);
- MaybeFinish();
+ if (request.MessagesNum > 0)
+ --request.MessagesNum;
+
+ --InflightCounter;
+
+ if (request.MessagesNum == 0) {
+ FreeSpace += request.Size;
+
+ if (FreeSpace > 0) {
+ auto res = MakeHolder<NDq::TEvDqCompute::TEvChannelDataAck>();
+ res->Record.SetChannelId(request.ChannelId);
+ res->Record.SetFreeSpace(FreeSpace);
+ res->Record.SetSeqNo(request.SeqNo);
+ res->Record.SetFinish(HasError);
+ Send(request.Sender, res.Release());
+
+ auto duration = (TInstant::Now()-StartTime);
+
+ LOG_D("ChannelData, Records: " << RowIndex
+ << " HasError: " << HasError
+ << " Size: " << Size
+ << " Rows: " << Rows
+ << " FreeSpace: " << FreeSpace
+ << " Duration: " << duration
+ << " AvgSpeed: " << Size/(duration.Seconds()+1)/1024/1024);
+ }
+
+ Requests.erase(it);
+ MaybeFinish();
}
- SendResult(); // Send remaining rows
+ SendResult(); // Send remaining rows
}
void StopChannel(NDq::TEvDqCompute::TEvChannelData::TPtr& ev) {
@@ -177,82 +177,82 @@ private:
Send(ev->Sender, res.Release());
}
- Yq::Private::WriteTaskResultRequest CreateProtoRequestWithoutResultSet(ui64 startRowIndex) {
- Yq::Private::WriteTaskResultRequest protoReq;
- protoReq.set_owner_id(ResultId.Owner);
- protoReq.mutable_result_id()->set_value(ResultId.Id);
- protoReq.set_offset(startRowIndex);
- protoReq.set_result_set_id(ResultId.SetId);
- protoReq.set_request_id(Cookie);
+ Yq::Private::WriteTaskResultRequest CreateProtoRequestWithoutResultSet(ui64 startRowIndex) {
+ Yq::Private::WriteTaskResultRequest protoReq;
+ protoReq.set_owner_id(ResultId.Owner);
+ protoReq.mutable_result_id()->set_value(ResultId.Id);
+ protoReq.set_offset(startRowIndex);
+ protoReq.set_result_set_id(ResultId.SetId);
+ protoReq.set_request_id(Cookie);
*protoReq.mutable_deadline() = NProtoInterop::CastToProto(Deadline);
- return protoReq;
- }
-
- void SendResult() {
- if (InflightCounter || CurChunkInd >= ResultChunks.size()) {
- return;
- }
- ++InflightCounter;
- const auto actorSystem = NActors::TActivationContext::ActorSystem();
- const auto selfId = SelfId();
- Client
- .WriteTaskResult(std::move(ResultChunks[CurChunkInd++]))
- .Subscribe([actorSystem, selfId](const auto& future) {
- try {
- const auto& wrappedResult = future.GetValue();
- if (wrappedResult.IsResultSet()) {
- actorSystem->Send(selfId,
- new NYq::TEvControlPlaneStorage::TEvWriteResultDataResponse(wrappedResult.GetIssues(), wrappedResult.GetResult().request_id()));
- } else {
- auto issues = wrappedResult.GetIssues();
- issues.AddIssue("Error on writing result");
- actorSystem->Send(selfId,
- new NYq::TEvControlPlaneStorage::TEvWriteResultDataResponse(issues, UINT64_MAX));
- }
- } catch(...) {
- TIssues issues;
- issues.AddIssue(CurrentExceptionMessage());
- actorSystem->Send(selfId,
- new NYq::TEvControlPlaneStorage::TEvWriteResultDataResponse(issues, UINT64_MAX));
- }
- });
- }
-
- void ConstructResults(const Ydb::ResultSet& resultSet, ui64 startRowIndex) {
- const size_t baseProtoByteSize = CreateProtoRequestWithoutResultSet(startRowIndex).ByteSizeLong();
-
- NYq::TRowsProtoSplitter rowsSplitter(resultSet, ProtoMessageLimit, baseProtoByteSize, MaxRowsCountPerChunk);
- const auto splittedResultSets = rowsSplitter.Split();
-
- if (!splittedResultSets.Success) {
- SendIssuesAndSetErrorFlag(splittedResultSets.Issues);
- return;
- }
-
- for (const auto& resultSet : splittedResultSets.ResultSets) {
- auto protoReq = CreateProtoRequestWithoutResultSet(startRowIndex);
- startRowIndex += resultSet.rows().size();
- *protoReq.mutable_result_set() = resultSet;
- ResultChunks.emplace_back(std::move(protoReq));
- }
- Requests[Cookie].MessagesNum = splittedResultSets.ResultSets.size();
- }
-
+ return protoReq;
+ }
+
+ void SendResult() {
+ if (InflightCounter || CurChunkInd >= ResultChunks.size()) {
+ return;
+ }
+ ++InflightCounter;
+ const auto actorSystem = NActors::TActivationContext::ActorSystem();
+ const auto selfId = SelfId();
+ Client
+ .WriteTaskResult(std::move(ResultChunks[CurChunkInd++]))
+ .Subscribe([actorSystem, selfId](const auto& future) {
+ try {
+ const auto& wrappedResult = future.GetValue();
+ if (wrappedResult.IsResultSet()) {
+ actorSystem->Send(selfId,
+ new NYq::TEvControlPlaneStorage::TEvWriteResultDataResponse(wrappedResult.GetIssues(), wrappedResult.GetResult().request_id()));
+ } else {
+ auto issues = wrappedResult.GetIssues();
+ issues.AddIssue("Error on writing result");
+ actorSystem->Send(selfId,
+ new NYq::TEvControlPlaneStorage::TEvWriteResultDataResponse(issues, UINT64_MAX));
+ }
+ } catch(...) {
+ TIssues issues;
+ issues.AddIssue(CurrentExceptionMessage());
+ actorSystem->Send(selfId,
+ new NYq::TEvControlPlaneStorage::TEvWriteResultDataResponse(issues, UINT64_MAX));
+ }
+ });
+ }
+
+ void ConstructResults(const Ydb::ResultSet& resultSet, ui64 startRowIndex) {
+ const size_t baseProtoByteSize = CreateProtoRequestWithoutResultSet(startRowIndex).ByteSizeLong();
+
+ NYq::TRowsProtoSplitter rowsSplitter(resultSet, ProtoMessageLimit, baseProtoByteSize, MaxRowsCountPerChunk);
+ const auto splittedResultSets = rowsSplitter.Split();
+
+ if (!splittedResultSets.Success) {
+ SendIssuesAndSetErrorFlag(splittedResultSets.Issues);
+ return;
+ }
+
+ for (const auto& resultSet : splittedResultSets.ResultSets) {
+ auto protoReq = CreateProtoRequestWithoutResultSet(startRowIndex);
+ startRowIndex += resultSet.rows().size();
+ *protoReq.mutable_result_set() = resultSet;
+ ResultChunks.emplace_back(std::move(protoReq));
+ }
+ Requests[Cookie].MessagesNum = splittedResultSets.ResultSets.size();
+ }
+
void ProcessData(NDq::TEvDqCompute::TEvChannelData::TPtr& ev) {
auto& data = ev->Get()->Record.GetChannelData().GetData();
auto resultSet = ResultBuilder->BuildResultSet({data});
FreeSpace -= data.GetRaw().size();
- OccupiedSpace += data.GetRaw().size();
-
- if (OccupiedSpace > SpaceLimitPerQuery) {
- TIssues issues;
- issues.AddIssue(TStringBuilder() << "Can not write results with size > " << SpaceLimitPerQuery / (1024 * 1024) << "_MB");
- SendIssuesAndSetErrorFlag(issues);
- return;
- }
-
+ OccupiedSpace += data.GetRaw().size();
+
+ if (OccupiedSpace > SpaceLimitPerQuery) {
+ TIssues issues;
+ issues.AddIssue(TStringBuilder() << "Can not write results with size > " << SpaceLimitPerQuery / (1024 * 1024) << "_MB");
+ SendIssuesAndSetErrorFlag(issues);
+ return;
+ }
+
ui64 startRowIndex = RowIndex;
- RowIndex += resultSet.rows().size();
+ RowIndex += resultSet.rows().size();
auto& request = Requests[Cookie];
request.Sender = ev->Sender;
@@ -260,9 +260,9 @@ private:
request.SeqNo = ev->Get()->Record.GetSeqNo();
request.Size = data.GetRaw().size();
- ConstructResults(resultSet, startRowIndex);
- SendResult();
-
+ ConstructResults(resultSet, startRowIndex);
+ SendResult();
+
if (!Truncated &&
(!AllResultsBytesLimit || Size + data.GetRaw().size() < *AllResultsBytesLimit)
&& (!RowsLimitPerWrite || Rows + data.GetRows() < *RowsLimitPerWrite)) {
@@ -287,27 +287,27 @@ private:
try {
if (ev->Get()->Record.GetChannelData().HasData()) {
ProcessData(ev);
- } else {
- auto res = MakeHolder<NDq::TEvDqCompute::TEvChannelDataAck>();
- res->Record.SetChannelId(ev->Get()->Record.GetChannelData().GetChannelId());
- res->Record.SetSeqNo(ev->Get()->Record.GetSeqNo());
- res->Record.SetFreeSpace(FreeSpace);
- res->Record.SetFinish(HasError);
- Send(ev->Sender, res.Release());
-
- auto duration = (TInstant::Now()-StartTime);
-
- LOG_D("ChannelData, Records: " << RowIndex
- << " HasError: " << HasError
- << " Size: " << Size
- << " Rows: " << Rows
- << " FreeSpace: " << FreeSpace
- << " Duration: " << duration
- << " AvgSpeed: " << Size/(duration.Seconds()+1)/1024/1024);
- }
+ } else {
+ auto res = MakeHolder<NDq::TEvDqCompute::TEvChannelDataAck>();
+ res->Record.SetChannelId(ev->Get()->Record.GetChannelData().GetChannelId());
+ res->Record.SetSeqNo(ev->Get()->Record.GetSeqNo());
+ res->Record.SetFreeSpace(FreeSpace);
+ res->Record.SetFinish(HasError);
+ Send(ev->Sender, res.Release());
+
+ auto duration = (TInstant::Now()-StartTime);
+
+ LOG_D("ChannelData, Records: " << RowIndex
+ << " HasError: " << HasError
+ << " Size: " << Size
+ << " Rows: " << Rows
+ << " FreeSpace: " << FreeSpace
+ << " Duration: " << duration
+ << " AvgSpeed: " << Size/(duration.Seconds()+1)/1024/1024);
+ }
} catch (...) {
LOG_E(CurrentExceptionMessage());
- auto req = MakeHolder<TEvDqFailure>(TIssue("Internal error on data write").SetCode(NYql::DEFAULT_ERROR, TSeverityIds::S_ERROR), /*retriable=*/ false,
+ auto req = MakeHolder<TEvDqFailure>(TIssue("Internal error on data write").SetCode(NYql::DEFAULT_ERROR, TSeverityIds::S_ERROR), /*retriable=*/ false,
/*needFallback=*/false);
Send(ExecuterId, req.Release());
HasError = true;
@@ -319,11 +319,11 @@ private:
ui64 ChannelId;
ui64 SeqNo;
i64 Size;
- size_t MessagesNum;
+ size_t MessagesNum;
};
THashMap<ui64, TRequest> Requests;
- TVector<NYql::NDqProto::TData> Head;
+ TVector<NYql::NDqProto::TData> Head;
bool Truncated = false;
TMaybe<ui64> AllResultsBytesLimit = 10000;
TMaybe<ui64> RowsLimitPerWrite = 1000;
@@ -335,23 +335,23 @@ private:
const TResultId ResultId;
const TString TraceId;
TInstant Deadline;
- TPrivateClient Client;
+ TPrivateClient Client;
const TInstant StartTime = TInstant::Now();
ui64 RowIndex = 0;
ui64 Cookie = 0;
i64 FreeSpace = 64_MB; // TODO: make global free space
- const size_t ProtoMessageLimit = 10_MB;
- const size_t MaxRowsCountPerChunk = 100'000;
+ const size_t ProtoMessageLimit = 10_MB;
+ const size_t MaxRowsCountPerChunk = 100'000;
bool HasError = false;
bool Finished = false;
- NYql::TIssues Issues;
- ui64 SpaceLimitPerQuery = 20_MB;
- ui64 OccupiedSpace = 0;
-
- TVector<Yq::Private::WriteTaskResultRequest> ResultChunks;
- size_t CurChunkInd = 0;
- ui32 InflightCounter = 0;
+ NYql::TIssues Issues;
+ ui64 SpaceLimitPerQuery = 20_MB;
+ ui64 OccupiedSpace = 0;
+
+ TVector<Yq::Private::WriteTaskResultRequest> ResultChunks;
+ size_t CurChunkInd = 0;
+ ui32 InflightCounter = 0;
};
NActors::IActor* CreateResultWriter(
@@ -362,10 +362,10 @@ NActors::IActor* CreateResultWriter(
const TResultId& resultId,
const TVector<TString>& columns,
const TString& traceId,
- const TInstant& deadline,
- const NMonitoring::TDynamicCounterPtr& clientCounters)
+ const TInstant& deadline,
+ const NMonitoring::TDynamicCounterPtr& clientCounters)
{
- return new TResultWriter(driver, executerId, resultType, privateApiConfig, resultId, columns, traceId, deadline, clientCounters);
+ return new TResultWriter(driver, executerId, resultType, privateApiConfig, resultId, columns, traceId, deadline, clientCounters);
}
-} // namespace NYq
+} // namespace NYq
diff --git a/ydb/core/yq/libs/actors/run_actor.cpp b/ydb/core/yq/libs/actors/run_actor.cpp
index 25cdacc16d..5549f8f254 100644
--- a/ydb/core/yq/libs/actors/run_actor.cpp
+++ b/ydb/core/yq/libs/actors/run_actor.cpp
@@ -58,11 +58,11 @@
#include <ydb/core/yq/libs/tasks_packer/tasks_packer.h>
#include <util/system/hostname.h>
-#include <library/cpp/json/yson/json2yson.h>
-
+#include <library/cpp/json/yson/json2yson.h>
+
#include <ydb/core/yq/libs/control_plane_storage/control_plane_storage.h>
#include <ydb/core/yq/libs/control_plane_storage/events/events.h>
-#include <google/protobuf/util/time_util.h>
+#include <google/protobuf/util/time_util.h>
#include <util/string/split.h>
#include <ydb/core/yq/libs/checkpointing/checkpoint_coordinator.h>
@@ -78,7 +78,7 @@
#define LOG_D(stream) \
LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::YQL_PROXY, Params.QueryId << " RunActor : " << stream)
-namespace NYq {
+namespace NYq {
using namespace NActors;
using namespace NYql;
@@ -96,8 +96,8 @@ public:
{
}
- static constexpr char ActorName[] = "YQ_DEFERRED_COUNTERS_CLEANUP";
-
+ static constexpr char ActorName[] = "YQ_DEFERRED_COUNTERS_CLEANUP";
+
void Bootstrap() {
Become(&TDeferredCountersCleanupActor::StateFunc, TDuration::Seconds(60), new NActors::TEvents::TEvWakeup());
}
@@ -122,8 +122,8 @@ private:
const TString QueryId;
};
-class TRunActor : public NActors::TActorBootstrapped<TRunActor> {
-public:
+class TRunActor : public NActors::TActorBootstrapped<TRunActor> {
+public:
explicit TRunActor(
const ::NYq::NCommon::TServiceCounters& serviceCounters
, TRunActorParams&& params)
@@ -132,11 +132,11 @@ public:
, ServiceCounters(serviceCounters)
, QueryCounters(serviceCounters)
, EnableCheckpointCoordinator(Params.QueryType == YandexQuery::QueryContent::STREAMING && Params.CheckpointCoordinatorConfig.GetEnabled())
- , MaxTasksPerOperation(Params.CommonConfig.GetMaxTasksPerOperation() ? Params.CommonConfig.GetMaxTasksPerOperation() : 40)
+ , MaxTasksPerOperation(Params.CommonConfig.GetMaxTasksPerOperation() ? Params.CommonConfig.GetMaxTasksPerOperation() : 40)
{
}
- static constexpr char ActorName[] = "YQ_RUN_ACTOR";
+ static constexpr char ActorName[] = "YQ_RUN_ACTOR";
void Bootstrap() {
LOG_D("Start run actor. Compute state: " << YandexQuery::QueryMeta::ComputeStatus_Name(Params.Status));
@@ -153,8 +153,8 @@ public:
.DiscoveryEndpoint(Params.PrivateApiConfig.GetTaskServiceEndpoint())
.EnableSsl(Params.PrivateApiConfig.GetSecureTaskService())
.AuthToken(Params.AuthToken)
- .Database(Params.PrivateApiConfig.GetTaskServiceDatabase() ? Params.PrivateApiConfig.GetTaskServiceDatabase() : TMaybe<TString>()),
- Params.ClientCounters),
+ .Database(Params.PrivateApiConfig.GetTaskServiceDatabase() ? Params.PrivateApiConfig.GetTaskServiceDatabase() : TMaybe<TString>()),
+ Params.ClientCounters),
SelfId(),
Params.PingerConfig,
Params.Deadline
@@ -165,9 +165,9 @@ public:
} catch (const std::exception&) {
FailOnException();
}
- }
+ }
-private:
+private:
template <void (TRunActor::* DelegatedStateFunc)(STFUNC_SIG)>
STFUNC(StateFuncWrapper) {
try {
@@ -181,7 +181,7 @@ private:
HFunc(TEvents::TEvAsyncContinue, Handle);
hFunc(NActors::TEvents::TEvUndelivered, Handle);
hFunc(NYq::TEvents::TEvGraphParams, Handle);
- hFunc(NYq::TEvents::TEvDataStreamsReadRulesCreationResult, Handle);
+ hFunc(NYq::TEvents::TEvDataStreamsReadRulesCreationResult, Handle);
hFunc(NYql::NDqs::TEvQueryResponse, Handle);
hFunc(TEvents::TEvQueryActionResult, Handle);
hFunc(TEvents::TEvForwardPingResponse, Handle);
@@ -189,7 +189,7 @@ private:
)
STRICT_STFUNC(FinishStateFunc,
- hFunc(NYq::TEvents::TEvDataStreamsReadRulesCreationResult, HandleFinish);
+ hFunc(NYq::TEvents::TEvDataStreamsReadRulesCreationResult, HandleFinish);
hFunc(TEvents::TEvDataStreamsReadRulesDeletionResult, HandleFinish);
hFunc(NYql::NDqs::TEvQueryResponse, HandleFinish);
hFunc(TEvents::TEvForwardPingResponse, HandleFinish);
@@ -260,8 +260,8 @@ private:
Finish(GetFinalStatusFromFinalizingStatus(Params.Status));
break;
case YandexQuery::QueryMeta::STARTING:
- HandleConnections();
- RunProgram();
+ HandleConnections();
+ RunProgram();
break;
case YandexQuery::QueryMeta::RESUMING:
case YandexQuery::QueryMeta::RUNNING:
@@ -273,22 +273,22 @@ private:
}
}
- void HandleConnections() {
+ void HandleConnections() {
LOG_D("HandleConnections");
- THashMap<std::pair<TString, DatabaseType>, TEvents::TDatabaseAuth> databaseIds;
+ THashMap<std::pair<TString, DatabaseType>, TEvents::TDatabaseAuth> databaseIds;
for (const auto& connection : Params.Connections) {
if (!connection.content().name()) {
LOG_D("Connection with empty name " << connection.meta().id());
continue;
}
- Connections[connection.content().name()] = connection; // Necessary for TDatabaseAsyncResolverWithMeta
+ Connections[connection.content().name()] = connection; // Necessary for TDatabaseAsyncResolverWithMeta
YqConnections.emplace(connection.meta().id(), connection);
}
}
void RunProgram() {
- LOG_D("RunProgram");
+ LOG_D("RunProgram");
if (!CompileQuery()) {
Abort("Failed to compile query", YandexQuery::QueryMeta::FAILED);
}
@@ -301,9 +301,9 @@ private:
void Fail(const TString& errorMessage) {
LOG_E("Fail for query " << Params.QueryId << ", finishing: " << Finishing << ", details: " << errorMessage);
- if (YqConnections.empty()) {
- Issues.AddIssue("YqConnections array is empty");
- }
+ if (YqConnections.empty()) {
+ Issues.AddIssue("YqConnections array is empty");
+ }
if (!Finishing) {
Abort("Internal Error", YandexQuery::QueryMeta::FAILED);
@@ -314,7 +314,7 @@ private:
Issues.AddIssue("Internal Error");
if (!ConsumersAreDeleted) {
- for (const Yq::Private::TopicConsumer& c : Params.CreatedTopicConsumers) {
+ for (const Yq::Private::TopicConsumer& c : Params.CreatedTopicConsumers) {
TransientIssues.AddIssue(TStringBuilder() << "Created read rule `" << c.consumer_name() << "` for topic `" << c.topic_path() << "` (database id " << c.database_id() << ") maybe was left undeleted: internal error occurred");
TransientIssues.back().Severity = NYql::TSeverityIds::S_WARNING;
}
@@ -405,8 +405,8 @@ private:
for (NYql::NDqProto::TTaskInput& taskInput : *task.MutableInputs()) {
if (taskInput.GetTypeCase() == NYql::NDqProto::TTaskInput::kSource && taskInput.GetSource().GetType() == "PqSource") {
google::protobuf::Any& settingsAny = *taskInput.MutableSource()->MutableSettings();
- YQL_ENSURE(settingsAny.Is<NYql::NPq::NProto::TDqPqTopicSource>());
- NYql::NPq::NProto::TDqPqTopicSource srcDesc;
+ YQL_ENSURE(settingsAny.Is<NYql::NPq::NProto::TDqPqTopicSource>());
+ NYql::NPq::NProto::TDqPqTopicSource srcDesc;
YQL_ENSURE(settingsAny.UnpackTo(&srcDesc));
if (!srcDesc.GetConsumerName()) {
@@ -475,29 +475,29 @@ private:
}
}
- TString CheckLimitsOfDqGraphs() {
- size_t dqTasks = 0;
- for (const auto& dqGraph : DqGraphParams) {
- dqTasks += dqGraph.TasksSize();
- }
- LOG_D("Overall dq tasks: " << dqTasks);
- if (dqTasks > MaxTasksPerOperation) {
- return TStringBuilder() << "Too many tasks per operation: " << dqTasks << ". Allowed: less than " << MaxTasksPerOperation;
- }
- return "";
- }
-
- bool AbortOnExceedingDqGraphsLimits() {
- TString errorMsg = CheckLimitsOfDqGraphs();
- if (errorMsg) {
- Abort(errorMsg, YandexQuery::QueryMeta::FAILED, Program->Issues());
- return true;
- }
- return false;
- }
-
+ TString CheckLimitsOfDqGraphs() {
+ size_t dqTasks = 0;
+ for (const auto& dqGraph : DqGraphParams) {
+ dqTasks += dqGraph.TasksSize();
+ }
+ LOG_D("Overall dq tasks: " << dqTasks);
+ if (dqTasks > MaxTasksPerOperation) {
+ return TStringBuilder() << "Too many tasks per operation: " << dqTasks << ". Allowed: less than " << MaxTasksPerOperation;
+ }
+ return "";
+ }
+
+ bool AbortOnExceedingDqGraphsLimits() {
+ TString errorMsg = CheckLimitsOfDqGraphs();
+ if (errorMsg) {
+ Abort(errorMsg, YandexQuery::QueryMeta::FAILED, Program->Issues());
+ return true;
+ }
+ return false;
+ }
+
void Handle(NYq::TEvents::TEvGraphParams::TPtr& ev) {
- LOG_D("Graph params with tasks: " << ev->Get()->GraphParams.TasksSize());
+ LOG_D("Graph params with tasks: " << ev->Get()->GraphParams.TasksSize());
DqGraphParams.push_back(ev->Get()->GraphParams);
}
@@ -516,10 +516,10 @@ private:
}
void PrepareGraphs() {
- if (AbortOnExceedingDqGraphsLimits()) {
- return;
- }
- Yq::Private::PingTaskRequest request;
+ if (AbortOnExceedingDqGraphsLimits()) {
+ return;
+ }
+ Yq::Private::PingTaskRequest request;
TStringStream exprOut;
TStringStream planOut;
@@ -544,7 +544,7 @@ private:
Params.CreatedTopicConsumers.clear();
Params.CreatedTopicConsumers.reserve(TopicsForConsumersCreation.size());
- for (const NYql::NPq::NProto::TDqPqTopicSource& src : TopicsForConsumersCreation) {
+ for (const NYql::NPq::NProto::TDqPqTopicSource& src : TopicsForConsumersCreation) {
auto& consumer = *request.add_created_topic_consumers();
consumer.set_database_id(src.GetDatabaseId());
consumer.set_database(src.GetDatabase());
@@ -733,7 +733,7 @@ private:
ContinueFinish();
}
- void Handle(NYq::TEvents::TEvDataStreamsReadRulesCreationResult::TPtr& ev) {
+ void Handle(NYq::TEvents::TEvDataStreamsReadRulesCreationResult::TPtr& ev) {
LOG_D("Read rules creation finished. Issues: " << ev->Get()->Issues.Size());
ReadRulesCreatorId = {};
if (ev->Get()->Issues) {
@@ -745,7 +745,7 @@ private:
}
}
- void HandleFinish(NYq::TEvents::TEvDataStreamsReadRulesCreationResult::TPtr& ev) {
+ void HandleFinish(NYq::TEvents::TEvDataStreamsReadRulesCreationResult::TPtr& ev) {
ReadRulesCreatorId = {};
if (ev->Get()->Issues) {
TransientIssues.AddIssues(ev->Get()->Issues);
@@ -779,7 +779,7 @@ private:
void RunReadRulesDeletionActor() {
TVector<std::shared_ptr<NYdb::ICredentialsProviderFactory>> credentials;
credentials.reserve(Params.CreatedTopicConsumers.size());
- for (const Yq::Private::TopicConsumer& c : Params.CreatedTopicConsumers) {
+ for (const Yq::Private::TopicConsumer& c : Params.CreatedTopicConsumers) {
if (const TString& tokenName = c.token_name()) {
credentials.emplace_back(
CreateCredentialsProviderFactoryForStructuredToken(Params.CredentialsFactory, FindTokenByName(tokenName), c.add_bearer_to_token()));
@@ -810,7 +810,7 @@ private:
{
Params.Status = YandexQuery::QueryMeta::RUNNING;
- Yq::Private::PingTaskRequest request;
+ Yq::Private::PingTaskRequest request;
request.set_status(YandexQuery::QueryMeta::RUNNING);
*request.mutable_started_at() = google::protobuf::util::TimeUtil::MillisecondsToTimestamp(Now().MilliSeconds());
Send(Pinger, new TEvents::TEvForwardPingRequest(request), 0, UpdateQueryInfoCookie);
@@ -818,7 +818,7 @@ private:
/*
{ // Failure test -- keep it until integrational tests work
- Yq::Private::PingTaskRequest request;
+ Yq::Private::PingTaskRequest request;
request.set_status(YandexQuery::QueryMeta::RUNNING);
request.set_resign_query(true);
Send(Pinger, new TEvents::TEvForwardPingRequest(request, true), 0, UpdateQueryInfoCookie);
@@ -862,7 +862,7 @@ private:
NActors::TActorId resultId;
if (dqGraphParams.GetResultType()) {
- TResultId writerResultId;
+ TResultId writerResultId;
{
writerResultId.HistoryId = Params.QueryId;
writerResultId.Id = Params.ResultId;
@@ -874,9 +874,9 @@ private:
columns.emplace_back(column);
}
resultId = NActors::TActivationContext::Register(
- CreateResultWriter(
- Params.Driver, ExecuterId, dqGraphParams.GetResultType(), Params.PrivateApiConfig,
- writerResultId, columns, dqGraphParams.GetSession(), Params.Deadline, Params.ClientCounters));
+ CreateResultWriter(
+ Params.Driver, ExecuterId, dqGraphParams.GetResultType(), Params.PrivateApiConfig,
+ writerResultId, columns, dqGraphParams.GetSession(), Params.Deadline, Params.ClientCounters));
} else {
LOG_D("ResultWriter was NOT CREATED since ResultType is empty");
resultId = ExecuterId;
@@ -915,7 +915,7 @@ private:
attr = dqSettings.Add();
attr->SetName("MaxTasksPerOperation");
- attr->SetValue(ToString(MaxTasksPerOperation));
+ attr->SetValue(ToString(MaxTasksPerOperation));
attr = dqSettings.Add();
attr->SetName("EnableComputeActor");
@@ -945,7 +945,7 @@ private:
}
}
- void AddClustersFromConfig(NYql::TGatewaysConfig& gatewaysConfig, THashMap<TString, TString>& clusters) const {
+ void AddClustersFromConfig(NYql::TGatewaysConfig& gatewaysConfig, THashMap<TString, TString>& clusters) const {
for (const auto& pq : Params.GatewaysConfig.GetPq().GetClusterMapping()) {
auto& clusterCfg = *gatewaysConfig.MutablePq()->AddClusterMapping();
clusterCfg = pq;
@@ -1036,12 +1036,12 @@ private:
}
void Finish(YandexQuery::QueryMeta::ComputeStatus status) {
- LOG_D("Is about to finish query with status " << YandexQuery::QueryMeta::ComputeStatus_Name(status));;
+ LOG_D("Is about to finish query with status " << YandexQuery::QueryMeta::ComputeStatus_Name(status));;
Finishing = true;
RetryNeeded = false;
FinalQueryStatus = status;
QueryStateUpdateRequest.set_status(FinalQueryStatus); // Can be changed later.
- *QueryStateUpdateRequest.mutable_finished_at() = google::protobuf::util::TimeUtil::MillisecondsToTimestamp(TInstant::Now().MilliSeconds());
+ *QueryStateUpdateRequest.mutable_finished_at() = google::protobuf::util::TimeUtil::MillisecondsToTimestamp(TInstant::Now().MilliSeconds());
Become(&TRunActor::StateFuncWrapper<&TRunActor::FinishStateFunc>);
if (!FinalizingStatusIsWritten) {
@@ -1088,18 +1088,18 @@ private:
Finish(status);
}
- void FillDqGraphParams() {
+ void FillDqGraphParams() {
for (const auto& s : Params.DqGraphs) {
NYq::NProto::TGraphParams dqGraphParams;
Y_VERIFY(dqGraphParams.ParseFromString(s));
DqGraphParams.emplace_back(std::move(dqGraphParams));
}
- }
-
- void ReRunQuery() {
- if (AbortOnExceedingDqGraphsLimits()) {
- return;
- }
+ }
+
+ void ReRunQuery() {
+ if (AbortOnExceedingDqGraphsLimits()) {
+ return;
+ }
for (const auto& m : Params.ResultSetMetas) {
*QueryStateUpdateRequest.add_result_set_meta() = m;
}
@@ -1110,8 +1110,8 @@ private:
}
bool CompileQuery() {
- LOG_D("Compiling query ...");
- NYql::TGatewaysConfig gatewaysConfig;
+ LOG_D("Compiling query ...");
+ NYql::TGatewaysConfig gatewaysConfig;
SetupDqSettings(*gatewaysConfig.MutableDq()->MutableDefaultSettings());
THashMap<TString, TString> clusters;
@@ -1128,24 +1128,24 @@ private:
gatewaysConfig,
clusters);
- TVector<TDataProviderInitializer> dataProvidersInit;
- const auto dbResolver = std::make_shared<TDatabaseAsyncResolverWithMeta>(TDatabaseAsyncResolverWithMeta(NActors::TActivationContext::ActorSystem(), Params.DatabaseResolver,
+ TVector<TDataProviderInitializer> dataProvidersInit;
+ const auto dbResolver = std::make_shared<TDatabaseAsyncResolverWithMeta>(TDatabaseAsyncResolverWithMeta(NActors::TActivationContext::ActorSystem(), Params.DatabaseResolver,
Params.CommonConfig.GetYdbMvpCloudEndpoint(), Params.CommonConfig.GetMdbGateway(), Params.CommonConfig.GetMdbTransformHost(), Params.QueryId, Params.AuthToken, Params.AccountIdSignatures, Connections));
- {
+ {
// TBD: move init to better place
QueryStateUpdateRequest.set_scope(Params.Scope.ToString());
QueryStateUpdateRequest.mutable_query_id()->set_value(Params.QueryId);
QueryStateUpdateRequest.set_owner_id(Params.Owner);
dataProvidersInit.push_back(GetDqDataProviderInitializer(&CreateInMemoryExecTransformer, NYq::CreateEmptyGateway(SelfId()), Params.DqCompFactory, {}, nullptr));
- }
+ }
- {
- dataProvidersInit.push_back(GetYdbDataProviderInitializer(Params.Driver, Params.CredentialsFactory, dbResolver));
- }
+ {
+ dataProvidersInit.push_back(GetYdbDataProviderInitializer(Params.Driver, Params.CredentialsFactory, dbResolver));
+ }
- {
+ {
dataProvidersInit.push_back(GetClickHouseDataProviderInitializer(Params.S3Gateway, dbResolver));
- }
+ }
{
dataProvidersInit.push_back(GetS3DataProviderInitializer(Params.S3Gateway, Params.CredentialsFactory));
@@ -1160,7 +1160,7 @@ private:
Params.FunctionRegistry
);
const auto pqGateway = NYql::CreatePqNativeGateway(pqServices);
- dataProvidersInit.push_back(GetPqDataProviderInitializer(pqGateway, false, dbResolver));
+ dataProvidersInit.push_back(GetPqDataProviderInitializer(pqGateway, false, dbResolver));
}
{
@@ -1170,24 +1170,24 @@ private:
}
TProgramFactory progFactory(false, Params.FunctionRegistry, Params.NextUniqueId, dataProvidersInit, "yq");
- progFactory.SetModules(Params.ModuleResolver);
- progFactory.SetUdfResolver(NYql::NCommon::CreateSimpleUdfResolver(Params.FunctionRegistry, nullptr));
- progFactory.SetGatewaysConfig(&gatewaysConfig);
+ progFactory.SetModules(Params.ModuleResolver);
+ progFactory.SetUdfResolver(NYql::NCommon::CreateSimpleUdfResolver(Params.FunctionRegistry, nullptr));
+ progFactory.SetGatewaysConfig(&gatewaysConfig);
SessionId = TStringBuilder()
<< Params.QueryId << '#'
<< Params.ResultId << '#'
- << Params.Scope.ToString() << '#'
+ << Params.Scope.ToString() << '#'
<< Params.Owner << '#'
<< Params.CloudId;
Program = progFactory.Create("-stdin-", Params.Sql, SessionId);
- Program->EnableResultPosition();
-
- NSQLTranslation::TTranslationSettings sqlSettings;
- sqlSettings.ClusterMapping = clusters;
- sqlSettings.SyntaxVersion = 1;
- sqlSettings.V0Behavior = NSQLTranslation::EV0Behavior::Disable;
+ Program->EnableResultPosition();
+
+ NSQLTranslation::TTranslationSettings sqlSettings;
+ sqlSettings.ClusterMapping = clusters;
+ sqlSettings.SyntaxVersion = 1;
+ sqlSettings.V0Behavior = NSQLTranslation::EV0Behavior::Disable;
sqlSettings.Flags.insert({ "DqEngineEnable", "DqEngineForce" });
try {
AddTableBindingsFromBindings(Params.Bindings, YqConnections, sqlSettings);
@@ -1235,45 +1235,45 @@ private:
break;
default:
Issues.AddIssue(TStringBuilder() << "Unexpected execute mode " << static_cast<int>(Params.ExecuteMode));
- return false;
- }
+ return false;
+ }
futureStatus.Subscribe([actorSystem = NActors::TActivationContext::ActorSystem(), selfId = SelfId()](const TProgram::TFutureStatus& f) {
- actorSystem->Send(selfId, new TEvents::TEvAsyncContinue(f));
- });
- return true;
- }
-
- void Handle(TEvents::TEvAsyncContinue::TPtr& ev, const TActorContext& ctx) {
- LOG_D("Compiling finished");
+ actorSystem->Send(selfId, new TEvents::TEvAsyncContinue(f));
+ });
+ return true;
+ }
+
+ void Handle(TEvents::TEvAsyncContinue::TPtr& ev, const TActorContext& ctx) {
+ LOG_D("Compiling finished");
NYql::TProgram::TStatus status = TProgram::TStatus::Error;
- const auto& f = ev->Get()->Future;
- try {
+ const auto& f = ev->Get()->Future;
+ try {
status = f.GetValue();
- if (status == TProgram::TStatus::Async) {
- auto futureStatus = Program->ContinueAsync();
- auto actorSystem = ctx.ActorSystem();
- auto selfId = ctx.SelfID;
- futureStatus.Subscribe([actorSystem, selfId](const TProgram::TFutureStatus& f) {
- actorSystem->Send(selfId, new TEvents::TEvAsyncContinue(f));
- });
- return;
- }
+ if (status == TProgram::TStatus::Async) {
+ auto futureStatus = Program->ContinueAsync();
+ auto actorSystem = ctx.ActorSystem();
+ auto selfId = ctx.SelfID;
+ futureStatus.Subscribe([actorSystem, selfId](const TProgram::TFutureStatus& f) {
+ actorSystem->Send(selfId, new TEvents::TEvAsyncContinue(f));
+ });
+ return;
+ }
} catch (const std::exception& err) {
Issues.AddIssue(ExceptionToIssue(err));
}
-
+
if (status == TProgram::TStatus::Ok || (DqGraphParams.size() > 0 && !DqGraphParams[0].GetResultType())) {
PrepareGraphs();
} else {
Abort(TStringBuilder() << "Run query failed: " << ToString(status), YandexQuery::QueryMeta::FAILED, Program->Issues());
- }
- }
+ }
+ }
void Handle(NActors::TEvents::TEvUndelivered::TPtr&) {
Fail("TRunActor::OnUndelivered");
- }
+ }
TString FindTokenByName(const TString& tokenName) const {
for (auto& graphParams : DqGraphParams) {
@@ -1319,11 +1319,11 @@ private:
TRunActorParams Params;
THashMap<TString, YandexQuery::Connection> YqConnections;
- TProgramPtr Program;
- TIssues Issues;
+ TProgramPtr Program;
+ TIssues Issues;
TIssues TransientIssues;
- TQueryResult QueryResult;
- TInstant Deadline;
+ TQueryResult QueryResult;
+ TInstant Deadline;
TActorId Pinger;
TInstant CreatedAt;
YandexQuery::QueryAction Action = YandexQuery::QueryAction::QUERY_ACTION_UNSPECIFIED;
@@ -1340,11 +1340,11 @@ private:
::NYq::NCommon::TServiceCounters QueryCounters;
bool EnableCheckpointCoordinator = false;
bool RetryNeeded = false;
- Yq::Private::PingTaskRequest QueryStateUpdateRequest;
+ Yq::Private::PingTaskRequest QueryStateUpdateRequest;
THashMap<TString, YandexQuery::Connection> Connections; // Necessary for DbAsyncResolver
- const ui64 MaxTasksPerOperation = 100;
-
+ const ui64 MaxTasksPerOperation = 100;
+
// Consumers creation
TVector<NYql::NPq::NProto::TDqPqTopicSource> TopicsForConsumersCreation;
TVector<std::shared_ptr<NYdb::ICredentialsProviderFactory>> CredentialsForConsumersCreation;
@@ -1365,9 +1365,9 @@ private:
SaveFinalizingStatusCookie,
SetLoadFromCheckpointModeCookie,
};
-};
-
-
+};
+
+
IActor* CreateRunActor(
const ::NYq::NCommon::TServiceCounters& serviceCounters,
TRunActorParams&& params
@@ -1375,4 +1375,4 @@ IActor* CreateRunActor(
return new TRunActor(serviceCounters, std::move(params));
}
-} /* NYq */
+} /* NYq */
diff --git a/ydb/core/yq/libs/actors/run_actor_params.cpp b/ydb/core/yq/libs/actors/run_actor_params.cpp
index 933c9ef631..1446cae1bb 100644
--- a/ydb/core/yq/libs/actors/run_actor_params.cpp
+++ b/ydb/core/yq/libs/actors/run_actor_params.cpp
@@ -1,26 +1,26 @@
-#include "run_actor_params.h"
-
-namespace NYq {
-
-using namespace NActors;
-
-TRunActorParams::TRunActorParams(
- NYdb::TDriver driver,
+#include "run_actor_params.h"
+
+namespace NYq {
+
+using namespace NActors;
+
+TRunActorParams::TRunActorParams(
+ NYdb::TDriver driver,
NYql::IHTTPGateway::TPtr s3Gateway,
- const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry,
- TIntrusivePtr<IRandomProvider> randomProvider,
- NYql::IModuleResolver::TPtr& moduleResolver,
- ui64 nextUniqueId,
- NKikimr::NMiniKQL::TComputationNodeFactory dqCompFactory,
+ const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry,
+ TIntrusivePtr<IRandomProvider> randomProvider,
+ NYql::IModuleResolver::TPtr& moduleResolver,
+ ui64 nextUniqueId,
+ NKikimr::NMiniKQL::TComputationNodeFactory dqCompFactory,
::NPq::NConfigurationManager::IConnections::TPtr pqCmConnections,
const ::NYq::NConfig::TCommonConfig& commonConfig,
const ::NYq::NConfig::TCheckpointCoordinatorConfig& checkpointCoordinatorConfig,
const ::NYq::NConfig::TPrivateApiConfig& privateApiConfig,
const ::NYq::NConfig::TGatewaysConfig& gatewaysConfig,
const ::NYq::NConfig::TPingerConfig& pingerConfig,
- const TString& sql,
+ const TString& sql,
const TScope& scope,
- const TString& authToken,
+ const TString& authToken,
const TActorId& databaseResolver,
const TString& queryId,
const TString& userId,
@@ -28,9 +28,9 @@ TRunActorParams::TRunActorParams(
const int64_t previousQueryRevision,
TVector<YandexQuery::Connection> connections,
TVector<YandexQuery::Binding> bindings,
- NYql::ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory,
+ NYql::ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory,
THashMap<TString, TString> accountIdSignatures,
- YandexQuery::QueryContent::QueryType queryType,
+ YandexQuery::QueryContent::QueryType queryType,
YandexQuery::ExecuteMode executeMode,
const TString& resultId,
const YandexQuery::StateLoadMode stateLoadMode,
@@ -40,28 +40,28 @@ TRunActorParams::TRunActorParams(
TVector<YandexQuery::ResultSetMeta> resultSetMetas,
TVector<TString> dqGraphs,
int32_t dqGraphIndex,
- TVector<Yq::Private::TopicConsumer> createdTopicConsumers,
- bool automatic,
+ TVector<Yq::Private::TopicConsumer> createdTopicConsumers,
+ bool automatic,
const TString& queryName,
- const TInstant& deadline,
- const NMonitoring::TDynamicCounterPtr& clientCounters
- )
- : Driver(driver)
+ const TInstant& deadline,
+ const NMonitoring::TDynamicCounterPtr& clientCounters
+ )
+ : Driver(driver)
, S3Gateway(s3Gateway)
- , FunctionRegistry(functionRegistry)
- , RandomProvider(randomProvider)
- , ModuleResolver(moduleResolver)
- , NextUniqueId(nextUniqueId)
- , DqCompFactory(dqCompFactory)
+ , FunctionRegistry(functionRegistry)
+ , RandomProvider(randomProvider)
+ , ModuleResolver(moduleResolver)
+ , NextUniqueId(nextUniqueId)
+ , DqCompFactory(dqCompFactory)
, PqCmConnections(std::move(pqCmConnections))
, CommonConfig(commonConfig)
, CheckpointCoordinatorConfig(checkpointCoordinatorConfig)
, PrivateApiConfig(privateApiConfig)
, GatewaysConfig(gatewaysConfig)
, PingerConfig(pingerConfig)
- , Sql(sql)
+ , Sql(sql)
, Scope(scope)
- , AuthToken(authToken)
+ , AuthToken(authToken)
, DatabaseResolver(databaseResolver)
, QueryId(queryId)
, UserId(userId)
@@ -73,7 +73,7 @@ TRunActorParams::TRunActorParams(
, AccountIdSignatures(std::move(accountIdSignatures))
, QueryType(queryType)
, ExecuteMode(executeMode)
- , ResultId(resultId)
+ , ResultId(resultId)
, StateLoadMode(stateLoadMode)
, StreamingDisposition(streamingDisposition)
, Status(status)
@@ -82,11 +82,11 @@ TRunActorParams::TRunActorParams(
, DqGraphs(std::move(dqGraphs))
, DqGraphIndex(dqGraphIndex)
, CreatedTopicConsumers(std::move(createdTopicConsumers))
- , Automatic(automatic)
- , QueryName(queryName)
+ , Automatic(automatic)
+ , QueryName(queryName)
, Deadline(deadline)
- , ClientCounters(clientCounters)
- {
- }
-
-} /* NYq */
+ , ClientCounters(clientCounters)
+ {
+ }
+
+} /* NYq */
diff --git a/ydb/core/yq/libs/actors/run_actor_params.h b/ydb/core/yq/libs/actors/run_actor_params.h
index c1d63bc1f8..bad5c3c32b 100644
--- a/ydb/core/yq/libs/actors/run_actor_params.h
+++ b/ydb/core/yq/libs/actors/run_actor_params.h
@@ -10,41 +10,41 @@
#include <ydb/library/yql/providers/dq/worker_manager/interface/counters.h>
#include <ydb/library/yql/providers/solomon/provider/yql_solomon_gateway.h>
#include <ydb/library/yql/providers/pq/cm_client/interface/client.h>
-
+
#include <library/cpp/actors/core/actorsystem.h>
#include <library/cpp/time_provider/time_provider.h>
#include <library/cpp/random_provider/random_provider.h>
-namespace NYq {
-
-struct TRunActorParams { // TODO2 : Change name
- TRunActorParams(
- NYdb::TDriver driver,
+namespace NYq {
+
+struct TRunActorParams { // TODO2 : Change name
+ TRunActorParams(
+ NYdb::TDriver driver,
NYql::IHTTPGateway::TPtr s3Gateway,
- const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry,
- TIntrusivePtr<IRandomProvider> randomProvider,
- NYql::IModuleResolver::TPtr& moduleResolver,
- ui64 nextUniqueId,
- NKikimr::NMiniKQL::TComputationNodeFactory dqCompFactory,
+ const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry,
+ TIntrusivePtr<IRandomProvider> randomProvider,
+ NYql::IModuleResolver::TPtr& moduleResolver,
+ ui64 nextUniqueId,
+ NKikimr::NMiniKQL::TComputationNodeFactory dqCompFactory,
::NPq::NConfigurationManager::IConnections::TPtr pqCmConnections,
const ::NYq::NConfig::TCommonConfig& commonConfig,
const ::NYq::NConfig::TCheckpointCoordinatorConfig& checkpointCoordinatorConfig,
const ::NYq::NConfig::TPrivateApiConfig& privateApiConfig,
const ::NYq::NConfig::TGatewaysConfig& gatewaysConfig,
const ::NYq::NConfig::TPingerConfig& pingerConfig,
- const TString& sql,
+ const TString& sql,
const TScope& scope,
- const TString& authToken,
- const NActors::TActorId& databaseResolver,
+ const TString& authToken,
+ const NActors::TActorId& databaseResolver,
const TString& queryId,
const TString& userId,
const TString& owner,
const int64_t previousQueryRevision,
TVector<YandexQuery::Connection> connections,
TVector<YandexQuery::Binding> bindings,
- NYql::ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory,
+ NYql::ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory,
THashMap<TString, TString> accountIdSignatures,
- YandexQuery::QueryContent::QueryType queryType,
+ YandexQuery::QueryContent::QueryType queryType,
YandexQuery::ExecuteMode executeMode,
const TString& resultId,
const YandexQuery::StateLoadMode stateLoadMode,
@@ -54,23 +54,23 @@ struct TRunActorParams { // TODO2 : Change name
TVector<YandexQuery::ResultSetMeta> resultSetMetas,
TVector<TString> dqGraphs,
int32_t dqGraphIndex,
- TVector<Yq::Private::TopicConsumer> createdTopicConsumers,
- bool automatic,
+ TVector<Yq::Private::TopicConsumer> createdTopicConsumers,
+ bool automatic,
const TString& queryName,
- const TInstant& deadline,
- const NMonitoring::TDynamicCounterPtr& clientCounters
- );
-
+ const TInstant& deadline,
+ const NMonitoring::TDynamicCounterPtr& clientCounters
+ );
+
TRunActorParams(const TRunActorParams& params) = default;
TRunActorParams(TRunActorParams&& params) = default;
-
- NYdb::TDriver Driver;
+
+ NYdb::TDriver Driver;
NYql::IHTTPGateway::TPtr S3Gateway;
- const NKikimr::NMiniKQL::IFunctionRegistry* FunctionRegistry;
- TIntrusivePtr<IRandomProvider> RandomProvider;
- NYql::IModuleResolver::TPtr ModuleResolver;
- ui64 NextUniqueId;
- NKikimr::NMiniKQL::TComputationNodeFactory DqCompFactory;
+ const NKikimr::NMiniKQL::IFunctionRegistry* FunctionRegistry;
+ TIntrusivePtr<IRandomProvider> RandomProvider;
+ NYql::IModuleResolver::TPtr ModuleResolver;
+ ui64 NextUniqueId;
+ NKikimr::NMiniKQL::TComputationNodeFactory DqCompFactory;
::NPq::NConfigurationManager::IConnections::TPtr PqCmConnections;
const ::NYq::NConfig::TCommonConfig CommonConfig;
@@ -78,21 +78,21 @@ struct TRunActorParams { // TODO2 : Change name
const ::NYq::NConfig::TPrivateApiConfig PrivateApiConfig;
const ::NYq::NConfig::TGatewaysConfig GatewaysConfig;
const ::NYq::NConfig::TPingerConfig PingerConfig;
- const TString Sql;
+ const TString Sql;
const TScope Scope;
- const TString AuthToken;
- const NActors::TActorId DatabaseResolver;
+ const TString AuthToken;
+ const NActors::TActorId DatabaseResolver;
const TString QueryId;
const TString UserId;
const TString Owner;
const int64_t PreviousQueryRevision;
const TVector<YandexQuery::Connection> Connections;
const TVector<YandexQuery::Binding> Bindings;
- const NYql::ISecuredServiceAccountCredentialsFactory::TPtr CredentialsFactory;
+ const NYql::ISecuredServiceAccountCredentialsFactory::TPtr CredentialsFactory;
const THashMap<TString, TString> AccountIdSignatures;
const YandexQuery::QueryContent::QueryType QueryType;
const YandexQuery::ExecuteMode ExecuteMode;
- const TString ResultId;
+ const TString ResultId;
const YandexQuery::StateLoadMode StateLoadMode;
const YandexQuery::StreamingDisposition StreamingDisposition;
YandexQuery::QueryMeta::ComputeStatus Status;
@@ -100,13 +100,13 @@ struct TRunActorParams { // TODO2 : Change name
const TVector<YandexQuery::ResultSetMeta> ResultSetMetas;
const TVector<TString> DqGraphs;
const int32_t DqGraphIndex;
- TVector<Yq::Private::TopicConsumer> CreatedTopicConsumers;
-
- bool Automatic = false;
- TString QueryName;
+ TVector<Yq::Private::TopicConsumer> CreatedTopicConsumers;
+
+ bool Automatic = false;
+ TString QueryName;
TInstant Deadline;
-
- const NMonitoring::TDynamicCounterPtr ClientCounters;
-};
-
-} /* NYq */
+
+ const NMonitoring::TDynamicCounterPtr ClientCounters;
+};
+
+} /* NYq */
diff --git a/ydb/core/yq/libs/actors/system_clusters.cpp b/ydb/core/yq/libs/actors/system_clusters.cpp
index 2cfb0ad41f..2968ae6487 100644
--- a/ydb/core/yq/libs/actors/system_clusters.cpp
+++ b/ydb/core/yq/libs/actors/system_clusters.cpp
@@ -3,10 +3,10 @@
#include <ydb/library/yql/providers/common/provider/yql_provider_names.h>
#include <util/generic/hash.h>
-namespace NYq {
+namespace NYq {
+
+using namespace NYql;
-using namespace NYql;
-
void AddSystemClusters(TGatewaysConfig& gatewaysConfig, THashMap<TString, TString>& clusters, const TString& authToken) {
{
const auto clusterCfg = gatewaysConfig.MutablePq()->AddClusterMapping();
@@ -117,5 +117,5 @@ void AddSystemClusters(TGatewaysConfig& gatewaysConfig, THashMap<TString, TStrin
clusters.emplace(clusterCfg->GetName(), SolomonProviderName);
}
}
-
-} //NYq
+
+} //NYq
diff --git a/ydb/core/yq/libs/actors/system_clusters.h b/ydb/core/yq/libs/actors/system_clusters.h
index 2c496d7b99..c842948fdc 100644
--- a/ydb/core/yq/libs/actors/system_clusters.h
+++ b/ydb/core/yq/libs/actors/system_clusters.h
@@ -2,8 +2,8 @@
#include <ydb/library/yql/providers/common/proto/gateways_config.pb.h>
-namespace NYq {
-
-void AddSystemClusters(NYql::TGatewaysConfig& gatewaysConfig, THashMap<TString, TString>& clusters, const TString& authToken);
-
-} //NYq
+namespace NYq {
+
+void AddSystemClusters(NYql::TGatewaysConfig& gatewaysConfig, THashMap<TString, TString>& clusters, const TString& authToken);
+
+} //NYq
diff --git a/ydb/core/yq/libs/actors/table_bindings_from_bindings.cpp b/ydb/core/yq/libs/actors/table_bindings_from_bindings.cpp
index 7be0c1b98c..e353a194a4 100644
--- a/ydb/core/yq/libs/actors/table_bindings_from_bindings.cpp
+++ b/ydb/core/yq/libs/actors/table_bindings_from_bindings.cpp
@@ -4,12 +4,12 @@
#include <ydb/core/yq/libs/result_formatter/result_formatter.h>
#include <util/generic/vector.h>
-namespace NYq {
+namespace NYq {
+
+using namespace NYql;
-using namespace NYql;
-
namespace {
-
+
void FillBinding(NSQLTranslation::TTranslationSettings& sqlSettings, const YandexQuery::Binding& binding, const THashMap<TString, YandexQuery::Connection>& connections) {
TString clusterType;
TString path;
@@ -67,12 +67,12 @@ void FillBinding(NSQLTranslation::TTranslationSettings& sqlSettings, const Yande
sqlSettings.PrivateBindings[binding.content().name()] = std::move(bindSettings);
}
-} //namespace
+} //namespace
void AddTableBindingsFromBindings(const TVector<YandexQuery::Binding>& bindings, const THashMap<TString, YandexQuery::Connection>& connections, NSQLTranslation::TTranslationSettings& sqlSettings) {
for (const auto& binding : bindings) {
FillBinding(sqlSettings, binding, connections);
}
}
-
-} //NYq
+
+} //NYq
diff --git a/ydb/core/yq/libs/actors/table_bindings_from_bindings.h b/ydb/core/yq/libs/actors/table_bindings_from_bindings.h
index afbbcf68f1..76343b06dc 100644
--- a/ydb/core/yq/libs/actors/table_bindings_from_bindings.h
+++ b/ydb/core/yq/libs/actors/table_bindings_from_bindings.h
@@ -3,8 +3,8 @@
#include <ydb/library/yql/sql/settings/translation_settings.h>
#include <ydb/public/api/protos/yq.pb.h>
-namespace NYq {
-
+namespace NYq {
+
void AddTableBindingsFromBindings(const TVector<YandexQuery::Binding>& bindings, const THashMap<TString, YandexQuery::Connection>& connections, NSQLTranslation::TTranslationSettings& sqlSettings);
-
-} //NYq
+
+} //NYq
diff --git a/ydb/core/yq/libs/actors/task_get.cpp b/ydb/core/yq/libs/actors/task_get.cpp
index c9081ad884..d31517aa9f 100644
--- a/ydb/core/yq/libs/actors/task_get.cpp
+++ b/ydb/core/yq/libs/actors/task_get.cpp
@@ -1,238 +1,238 @@
#include <ydb/core/yq/libs/config/protos/yq_config.pb.h>
-#include "proxy_private.h"
-#include "proxy.h"
-
+#include "proxy_private.h"
+#include "proxy.h"
+
#include <ydb/core/protos/services.pb.h>
#include <ydb/library/yql/public/issue/yql_issue_message.h>
-
-#include <library/cpp/yson/node/node_io.h>
-#include <library/cpp/actors/core/events.h>
-#include <library/cpp/actors/core/hfunc.h>
-#include <library/cpp/actors/core/actor_bootstrapped.h>
-#include <library/cpp/actors/core/log.h>
+
+#include <library/cpp/yson/node/node_io.h>
+#include <library/cpp/actors/core/events.h>
+#include <library/cpp/actors/core/hfunc.h>
+#include <library/cpp/actors/core/actor_bootstrapped.h>
+#include <library/cpp/actors/core/log.h>
#include <library/cpp/protobuf/interop/cast.h>
-
+
#include <ydb/core/yq/libs/common/entity_id.h>
-
+
#include <ydb/core/yq/libs/control_plane_storage/control_plane_storage.h>
#include <ydb/core/yq/libs/control_plane_storage/events/events.h>
#include <ydb/library/security/util.h>
-
+
#define LOG_E(stream) \
LOG_ERROR_S(*TlsActivationContext, NKikimrServices::YQL_PRIVATE_PROXY, "PrivateGetTask - Owner: " << OwnerId << ", " << "Host: " << Host << ", "<< stream)
#define LOG_D(stream) \
LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::YQL_PRIVATE_PROXY, "PrivateGetTask - Owner: " << OwnerId << ", " << "Host: " << Host << ", " << stream)
-
-namespace NYq {
-
-using namespace NActors;
-using namespace NMonitoring;
-
-class TGetTaskRequestActor
- : public NActors::TActorBootstrapped<TGetTaskRequestActor>
-{
-public:
- TGetTaskRequestActor(
- const NActors::TActorId& sender,
+
+namespace NYq {
+
+using namespace NActors;
+using namespace NMonitoring;
+
+class TGetTaskRequestActor
+ : public NActors::TActorBootstrapped<TGetTaskRequestActor>
+{
+public:
+ TGetTaskRequestActor(
+ const NActors::TActorId& sender,
const NConfig::TTokenAccessorConfig& tokenAccessorConfig,
- TIntrusivePtr<ITimeProvider> timeProvider,
- TAutoPtr<TEvents::TEvGetTaskRequest> ev,
- TDynamicCounterPtr counters)
+ TIntrusivePtr<ITimeProvider> timeProvider,
+ TAutoPtr<TEvents::TEvGetTaskRequest> ev,
+ TDynamicCounterPtr counters)
: TokenAccessorConfig(tokenAccessorConfig)
- , Sender(sender)
- , TimeProvider(timeProvider)
- , Ev(std::move(ev))
- , Counters(std::move(counters->GetSubgroup("subsystem", "private_api")->GetSubgroup("subcomponent", "GetTask")))
- , LifetimeDuration(Counters->GetHistogram("LifetimeDurationMs", ExponentialHistogram(10, 2, 50)))
- , RequestedMBytes(Counters->GetHistogram("RequestedMB", ExponentialHistogram(6, 2, 3)))
- , StartTime(TInstant::Now())
- {
+ , Sender(sender)
+ , TimeProvider(timeProvider)
+ , Ev(std::move(ev))
+ , Counters(std::move(counters->GetSubgroup("subsystem", "private_api")->GetSubgroup("subcomponent", "GetTask")))
+ , LifetimeDuration(Counters->GetHistogram("LifetimeDurationMs", ExponentialHistogram(10, 2, 50)))
+ , RequestedMBytes(Counters->GetHistogram("RequestedMB", ExponentialHistogram(6, 2, 3)))
+ , StartTime(TInstant::Now())
+ {
if (TokenAccessorConfig.GetHmacSecretFile()) {
Signer = ::NYq::CreateSignerFromFile(TokenAccessorConfig.GetHmacSecretFile());
- }
- }
-
- static constexpr char ActorName[] = "YQ_PRIVATE_GET_TASK";
-
- void OnUndelivered(NActors::TEvents::TEvUndelivered::TPtr& ev, const NActors::TActorContext& ctx) {
- LOG_E("TGetTaskRequestActor::OnUndelivered");
- auto Res = MakeHolder<TEvents::TEvGetTaskResponse>();
- Res->Status = Ydb::StatusIds::GENERIC_ERROR;
- Res->Issues.AddIssue("UNDELIVERED");
- ctx.Send(ev->Sender, Res.Release());
- Die(ctx);
- }
-
- void PassAway() final {
- LifetimeDuration->Collect((TInstant::Now() - StartTime).MilliSeconds());
- NActors::IActor::PassAway();
- }
-
- void Fail(const TString& message, Ydb::StatusIds::StatusCode reqStatus = Ydb::StatusIds::INTERNAL_ERROR) {
- Issues.AddIssue(message);
- const auto codeStr = Ydb::StatusIds_StatusCode_Name(reqStatus);
- LOG_E(TStringBuilder()
- << "Failed with code: " << codeStr
- << " Details: " << Issues.ToString());
- auto Res = MakeHolder<TEvents::TEvGetTaskResponse>();
- Res->Status = reqStatus;
- Res->Issues.AddIssues(Issues);
- Send(Sender, Res.Release());
- PassAway();
- }
-
- void Bootstrap(const TActorContext& ctx) {
- Become(&TGetTaskRequestActor::StateFunc);
- const auto& req = Ev->Record;
- OwnerId = req.owner_id();
- Host = req.host();
- LOG_D("Request CP::GetTask with size: " << req.ByteSize() << " bytes");
- RequestedMBytes->Collect(req.ByteSize() / 1024 / 1024);
- ctx.Send(NYq::ControlPlaneStorageServiceActorId(),
- new NYq::TEvControlPlaneStorage::TEvGetTaskRequest(OwnerId, Host));
- }
-
+ }
+ }
+
+ static constexpr char ActorName[] = "YQ_PRIVATE_GET_TASK";
+
+ void OnUndelivered(NActors::TEvents::TEvUndelivered::TPtr& ev, const NActors::TActorContext& ctx) {
+ LOG_E("TGetTaskRequestActor::OnUndelivered");
+ auto Res = MakeHolder<TEvents::TEvGetTaskResponse>();
+ Res->Status = Ydb::StatusIds::GENERIC_ERROR;
+ Res->Issues.AddIssue("UNDELIVERED");
+ ctx.Send(ev->Sender, Res.Release());
+ Die(ctx);
+ }
+
+ void PassAway() final {
+ LifetimeDuration->Collect((TInstant::Now() - StartTime).MilliSeconds());
+ NActors::IActor::PassAway();
+ }
+
+ void Fail(const TString& message, Ydb::StatusIds::StatusCode reqStatus = Ydb::StatusIds::INTERNAL_ERROR) {
+ Issues.AddIssue(message);
+ const auto codeStr = Ydb::StatusIds_StatusCode_Name(reqStatus);
+ LOG_E(TStringBuilder()
+ << "Failed with code: " << codeStr
+ << " Details: " << Issues.ToString());
+ auto Res = MakeHolder<TEvents::TEvGetTaskResponse>();
+ Res->Status = reqStatus;
+ Res->Issues.AddIssues(Issues);
+ Send(Sender, Res.Release());
+ PassAway();
+ }
+
+ void Bootstrap(const TActorContext& ctx) {
+ Become(&TGetTaskRequestActor::StateFunc);
+ const auto& req = Ev->Record;
+ OwnerId = req.owner_id();
+ Host = req.host();
+ LOG_D("Request CP::GetTask with size: " << req.ByteSize() << " bytes");
+ RequestedMBytes->Collect(req.ByteSize() / 1024 / 1024);
+ ctx.Send(NYq::ControlPlaneStorageServiceActorId(),
+ new NYq::TEvControlPlaneStorage::TEvGetTaskRequest(OwnerId, Host));
+ }
+
static TString GetServiceAccountId(const YandexQuery::IamAuth& auth) {
return auth.has_service_account()
? auth.service_account().id()
: TString{};
}
- static TString ExtractServiceAccountId(const YandexQuery::Connection& c) {
- switch (c.content().setting().connection_case()) {
- case YandexQuery::ConnectionSetting::kYdbDatabase: {
+ static TString ExtractServiceAccountId(const YandexQuery::Connection& c) {
+ switch (c.content().setting().connection_case()) {
+ case YandexQuery::ConnectionSetting::kYdbDatabase: {
return GetServiceAccountId(c.content().setting().ydb_database().auth());
- }
- case YandexQuery::ConnectionSetting::kDataStreams: {
+ }
+ case YandexQuery::ConnectionSetting::kDataStreams: {
return GetServiceAccountId(c.content().setting().data_streams().auth());
- }
- case YandexQuery::ConnectionSetting::kObjectStorage: {
+ }
+ case YandexQuery::ConnectionSetting::kObjectStorage: {
return GetServiceAccountId(c.content().setting().object_storage().auth());
- }
- case YandexQuery::ConnectionSetting::kMonitoring: {
+ }
+ case YandexQuery::ConnectionSetting::kMonitoring: {
return GetServiceAccountId(c.content().setting().monitoring().auth());
- }
+ }
case YandexQuery::ConnectionSetting::kClickhouseCluster: {
return GetServiceAccountId(c.content().setting().clickhouse_cluster().auth());
}
// Do not replace with default. Adding a new connection should cause a compilation error
case YandexQuery::ConnectionSetting::CONNECTION_NOT_SET:
break;
- }
- return {};
- }
-
-private:
- void HandleResponse(NYq::TEvControlPlaneStorage::TEvGetTaskResponse::TPtr& ev, const TActorContext& ctx) { // YQ
- LOG_D("Got CP::GetTask Response");
- const auto& tasks = ev->Get()->Tasks;
- Res->Record.ConstructInPlace();
- Res->Status = Ydb::StatusIds::SUCCESS;
- const auto& issues = ev->Get()->Issues;
- if (issues) {
- Issues.AddIssues(issues);
- Fail("ControlPlane::GetTaskError", Ydb::StatusIds::GENERIC_ERROR);
- return;
- }
-
- try {
- for (const auto& task : tasks) {
- const auto& queryType = task.Query.content().type();
- if (queryType != YandexQuery::QueryContent::ANALYTICS && queryType != YandexQuery::QueryContent::STREAMING) { //TODO: fix
- ythrow yexception()
- << "query type "
- << YandexQuery::QueryContent::QueryType_Name(queryType)
- << " unsupported";
- }
- auto* newTask = Res->Record->add_tasks();
- newTask->set_query_type(queryType);
- newTask->set_execute_mode(task.Query.meta().execute_mode());
- newTask->set_state_load_mode(task.Internal.state_load_mode());
- auto* queryId = newTask->mutable_query_id();
- queryId->set_value(task.Query.meta().common().id());
- newTask->set_streaming(queryType == YandexQuery::QueryContent::STREAMING);
- newTask->set_text(task.Query.content().text());
- *newTask->mutable_connection() = task.Internal.connection();
- *newTask->mutable_binding() = task.Internal.binding();
- newTask->set_user_token(task.Internal.token());
- newTask->set_user_id(task.Query.meta().common().created_by());
- newTask->set_generation(task.Generation);
- newTask->set_status(task.Query.meta().status());
- *newTask->mutable_created_topic_consumers() = task.Internal.created_topic_consumers();
- newTask->mutable_sensor_labels()->insert({"cloud_id", task.Internal.cloud_id()});
- newTask->set_automatic(task.Query.content().automatic());
- newTask->set_query_name(task.Query.content().name());
+ }
+ return {};
+ }
+
+private:
+ void HandleResponse(NYq::TEvControlPlaneStorage::TEvGetTaskResponse::TPtr& ev, const TActorContext& ctx) { // YQ
+ LOG_D("Got CP::GetTask Response");
+ const auto& tasks = ev->Get()->Tasks;
+ Res->Record.ConstructInPlace();
+ Res->Status = Ydb::StatusIds::SUCCESS;
+ const auto& issues = ev->Get()->Issues;
+ if (issues) {
+ Issues.AddIssues(issues);
+ Fail("ControlPlane::GetTaskError", Ydb::StatusIds::GENERIC_ERROR);
+ return;
+ }
+
+ try {
+ for (const auto& task : tasks) {
+ const auto& queryType = task.Query.content().type();
+ if (queryType != YandexQuery::QueryContent::ANALYTICS && queryType != YandexQuery::QueryContent::STREAMING) { //TODO: fix
+ ythrow yexception()
+ << "query type "
+ << YandexQuery::QueryContent::QueryType_Name(queryType)
+ << " unsupported";
+ }
+ auto* newTask = Res->Record->add_tasks();
+ newTask->set_query_type(queryType);
+ newTask->set_execute_mode(task.Query.meta().execute_mode());
+ newTask->set_state_load_mode(task.Internal.state_load_mode());
+ auto* queryId = newTask->mutable_query_id();
+ queryId->set_value(task.Query.meta().common().id());
+ newTask->set_streaming(queryType == YandexQuery::QueryContent::STREAMING);
+ newTask->set_text(task.Query.content().text());
+ *newTask->mutable_connection() = task.Internal.connection();
+ *newTask->mutable_binding() = task.Internal.binding();
+ newTask->set_user_token(task.Internal.token());
+ newTask->set_user_id(task.Query.meta().common().created_by());
+ newTask->set_generation(task.Generation);
+ newTask->set_status(task.Query.meta().status());
+ *newTask->mutable_created_topic_consumers() = task.Internal.created_topic_consumers();
+ newTask->mutable_sensor_labels()->insert({"cloud_id", task.Internal.cloud_id()});
+ newTask->set_automatic(task.Query.content().automatic());
+ newTask->set_query_name(task.Query.content().name());
*newTask->mutable_deadline() = NProtoInterop::CastToProto(task.Deadline);
newTask->mutable_disposition()->CopyFrom(task.Internal.disposition());
-
- THashMap<TString, TString> accountIdSignatures;
- for (const auto& connection: task.Internal.connection()) {
- const auto serviceAccountId = ExtractServiceAccountId(connection);
- if (!serviceAccountId) {
- continue;
- }
-
- auto& signature = accountIdSignatures[serviceAccountId];
- if (!signature && Signer) {
- signature = Signer->SignAccountId(serviceAccountId);
- }
- auto* account = newTask->add_service_accounts();
- account->set_value(serviceAccountId);
- account->set_signature(signature);
- }
-
- *newTask->mutable_dq_graph() = task.Internal.dq_graph();
- newTask->set_dq_graph_index(task.Internal.dq_graph_index());
-
- *newTask->mutable_result_set_meta() = task.Query.result_set_meta();
- newTask->set_scope(task.Scope);
- }
- ctx.Send(Sender, Res.Release());
- Die(ctx);
- } catch (...) {
- const auto msg = TStringBuilder() << "Can't do GetTask: " << CurrentExceptionMessage();
- Fail(msg);
- }
- }
-
-private:
- STRICT_STFUNC(
- StateFunc,
- CFunc(NActors::TEvents::TEvPoison::EventType, Die)
- HFunc(NActors::TEvents::TEvUndelivered, OnUndelivered)
- HFunc(NYq::TEvControlPlaneStorage::TEvGetTaskResponse, HandleResponse)
- )
-
+
+ THashMap<TString, TString> accountIdSignatures;
+ for (const auto& connection: task.Internal.connection()) {
+ const auto serviceAccountId = ExtractServiceAccountId(connection);
+ if (!serviceAccountId) {
+ continue;
+ }
+
+ auto& signature = accountIdSignatures[serviceAccountId];
+ if (!signature && Signer) {
+ signature = Signer->SignAccountId(serviceAccountId);
+ }
+ auto* account = newTask->add_service_accounts();
+ account->set_value(serviceAccountId);
+ account->set_signature(signature);
+ }
+
+ *newTask->mutable_dq_graph() = task.Internal.dq_graph();
+ newTask->set_dq_graph_index(task.Internal.dq_graph_index());
+
+ *newTask->mutable_result_set_meta() = task.Query.result_set_meta();
+ newTask->set_scope(task.Scope);
+ }
+ ctx.Send(Sender, Res.Release());
+ Die(ctx);
+ } catch (...) {
+ const auto msg = TStringBuilder() << "Can't do GetTask: " << CurrentExceptionMessage();
+ Fail(msg);
+ }
+ }
+
+private:
+ STRICT_STFUNC(
+ StateFunc,
+ CFunc(NActors::TEvents::TEvPoison::EventType, Die)
+ HFunc(NActors::TEvents::TEvUndelivered, OnUndelivered)
+ HFunc(NYq::TEvControlPlaneStorage::TEvGetTaskResponse, HandleResponse)
+ )
+
const NConfig::TTokenAccessorConfig TokenAccessorConfig;
- const TActorId Sender;
- TIntrusivePtr<ITimeProvider> TimeProvider;
- TAutoPtr<TEvents::TEvGetTaskRequest> Ev;
- TDynamicCounterPtr Counters;
- const THistogramPtr LifetimeDuration;
- const THistogramPtr RequestedMBytes;
- const TInstant StartTime;
-
+ const TActorId Sender;
+ TIntrusivePtr<ITimeProvider> TimeProvider;
+ TAutoPtr<TEvents::TEvGetTaskRequest> Ev;
+ TDynamicCounterPtr Counters;
+ const THistogramPtr LifetimeDuration;
+ const THistogramPtr RequestedMBytes;
+ const TInstant StartTime;
+
::NYq::TSigner::TPtr Signer;
-
- NYql::TIssues Issues;
- TString OwnerId;
- TString Host;
-
- THolder<TEvents::TEvGetTaskResponse> Res = MakeHolder<TEvents::TEvGetTaskResponse>();
-};
-
-IActor* CreateGetTaskRequestActor(
- const NActors::TActorId& sender,
+
+ NYql::TIssues Issues;
+ TString OwnerId;
+ TString Host;
+
+ THolder<TEvents::TEvGetTaskResponse> Res = MakeHolder<TEvents::TEvGetTaskResponse>();
+};
+
+IActor* CreateGetTaskRequestActor(
+ const NActors::TActorId& sender,
const NConfig::TTokenAccessorConfig& tokenAccessorConfig,
- TIntrusivePtr<ITimeProvider> timeProvider,
- TAutoPtr<TEvents::TEvGetTaskRequest> ev,
- TDynamicCounterPtr counters) {
- return new TGetTaskRequestActor(
- sender,
+ TIntrusivePtr<ITimeProvider> timeProvider,
+ TAutoPtr<TEvents::TEvGetTaskRequest> ev,
+ TDynamicCounterPtr counters) {
+ return new TGetTaskRequestActor(
+ sender,
tokenAccessorConfig,
- timeProvider,
- std::move(ev),
- counters);
-}
-
-} /* NYq */
+ timeProvider,
+ std::move(ev),
+ counters);
+}
+
+} /* NYq */
diff --git a/ydb/core/yq/libs/actors/task_ping.cpp b/ydb/core/yq/libs/actors/task_ping.cpp
index 95623f0d9d..29410dccd1 100644
--- a/ydb/core/yq/libs/actors/task_ping.cpp
+++ b/ydb/core/yq/libs/actors/task_ping.cpp
@@ -1,157 +1,157 @@
-#include "proxy_private.h"
+#include "proxy_private.h"
#include <util/datetime/base.h>
-
+
#include <ydb/core/protos/services.pb.h>
#include <ydb/library/yql/public/issue/yql_issue_message.h>
-
-#include <library/cpp/yson/node/node_io.h>
-#include <library/cpp/actors/core/events.h>
-#include <library/cpp/actors/core/hfunc.h>
-#include <library/cpp/actors/core/actor_bootstrapped.h>
-#include <library/cpp/actors/core/log.h>
+
+#include <library/cpp/yson/node/node_io.h>
+#include <library/cpp/actors/core/events.h>
+#include <library/cpp/actors/core/hfunc.h>
+#include <library/cpp/actors/core/actor_bootstrapped.h>
+#include <library/cpp/actors/core/log.h>
#include <library/cpp/protobuf/interop/cast.h>
-
+
#include <ydb/core/yq/libs/control_plane_storage/control_plane_storage.h>
#include <ydb/core/yq/libs/control_plane_storage/events/events.h>
-#include <google/protobuf/util/time_util.h>
-
+#include <google/protobuf/util/time_util.h>
+
#define LOG_E(stream) \
LOG_ERROR_S(*TlsActivationContext, NKikimrServices::YQL_PRIVATE_PROXY, "PrivatePingTask - QueryId: " << OperationId << ", Owner: " << OwnerId << ", " << stream)
#define LOG_D(stream) \
LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::YQL_PRIVATE_PROXY, "PrivatePingTask - QueryId: " << OperationId << ", Owner: " << OwnerId << ", "<< stream)
-
-namespace NYq {
-
-using namespace NActors;
-using namespace NMonitoring;
-
-class TTaskPingRequestActor
- : public NActors::TActorBootstrapped<TTaskPingRequestActor>
-{
-public:
- TTaskPingRequestActor(
- const NActors::TActorId& sender,
- TIntrusivePtr<ITimeProvider> timeProvider,
- TAutoPtr<TEvents::TEvPingTaskRequest> ev,
- TDynamicCounterPtr counters)
+
+namespace NYq {
+
+using namespace NActors;
+using namespace NMonitoring;
+
+class TTaskPingRequestActor
+ : public NActors::TActorBootstrapped<TTaskPingRequestActor>
+{
+public:
+ TTaskPingRequestActor(
+ const NActors::TActorId& sender,
+ TIntrusivePtr<ITimeProvider> timeProvider,
+ TAutoPtr<TEvents::TEvPingTaskRequest> ev,
+ TDynamicCounterPtr counters)
: Sender(sender)
- , TimeProvider(timeProvider)
- , Ev(std::move(ev))
- , Counters(std::move(counters->GetSubgroup("subsystem", "private_api")->GetSubgroup("subcomponent", "PingTask")))
- , LifetimeDuration(Counters->GetHistogram("LifetimeDurationMs", ExponentialHistogram(10, 2, 50)))
- , RequestedMBytes(Counters->GetHistogram("RequestedMB", ExponentialHistogram(6, 2, 3)))
- , StartTime(TInstant::Now())
- {}
-
- static constexpr char ActorName[] = "YQ_PRIVATE_PING_TASK";
-
- void OnUndelivered(NActors::TEvents::TEvUndelivered::TPtr& ev, const NActors::TActorContext& ctx) {
- LOG_E("TTaskPingRequestActor::OnUndelivered");
- auto res = MakeHolder<TEvents::TEvPingTaskResponse>();
- res->Status = Ydb::StatusIds::GENERIC_ERROR;
- res->Issues.AddIssue("UNDELIVERED");
- ctx.Send(ev->Sender, res.Release());
- Die(ctx);
- }
-
- void PassAway() final {
- LifetimeDuration->Collect((TInstant::Now() - StartTime).MilliSeconds());
- NActors::IActor::PassAway();
- }
-
- void Fail(const TString& message, Ydb::StatusIds::StatusCode reqStatus = Ydb::StatusIds::INTERNAL_ERROR) {
- Issues.AddIssue(message);
- const auto codeStr = Ydb::StatusIds_StatusCode_Name(reqStatus);
- LOG_E(TStringBuilder()
- << "Failed with code: " << codeStr
- << " Details: " << Issues.ToString());
- auto res = MakeHolder<TEvents::TEvPingTaskResponse>();
- res->Status = reqStatus;
- res->Issues.AddIssues(Issues);
- Send(Sender, res.Release());
- PassAway();
- }
-
- void Bootstrap(const TActorContext& ctx) {
- Y_UNUSED(ctx);
- Become(&TTaskPingRequestActor::StateFunc);
- const auto& req = Ev->Record;
- OperationId = req.query_id().value();
+ , TimeProvider(timeProvider)
+ , Ev(std::move(ev))
+ , Counters(std::move(counters->GetSubgroup("subsystem", "private_api")->GetSubgroup("subcomponent", "PingTask")))
+ , LifetimeDuration(Counters->GetHistogram("LifetimeDurationMs", ExponentialHistogram(10, 2, 50)))
+ , RequestedMBytes(Counters->GetHistogram("RequestedMB", ExponentialHistogram(6, 2, 3)))
+ , StartTime(TInstant::Now())
+ {}
+
+ static constexpr char ActorName[] = "YQ_PRIVATE_PING_TASK";
+
+ void OnUndelivered(NActors::TEvents::TEvUndelivered::TPtr& ev, const NActors::TActorContext& ctx) {
+ LOG_E("TTaskPingRequestActor::OnUndelivered");
+ auto res = MakeHolder<TEvents::TEvPingTaskResponse>();
+ res->Status = Ydb::StatusIds::GENERIC_ERROR;
+ res->Issues.AddIssue("UNDELIVERED");
+ ctx.Send(ev->Sender, res.Release());
+ Die(ctx);
+ }
+
+ void PassAway() final {
+ LifetimeDuration->Collect((TInstant::Now() - StartTime).MilliSeconds());
+ NActors::IActor::PassAway();
+ }
+
+ void Fail(const TString& message, Ydb::StatusIds::StatusCode reqStatus = Ydb::StatusIds::INTERNAL_ERROR) {
+ Issues.AddIssue(message);
+ const auto codeStr = Ydb::StatusIds_StatusCode_Name(reqStatus);
+ LOG_E(TStringBuilder()
+ << "Failed with code: " << codeStr
+ << " Details: " << Issues.ToString());
+ auto res = MakeHolder<TEvents::TEvPingTaskResponse>();
+ res->Status = reqStatus;
+ res->Issues.AddIssues(Issues);
+ Send(Sender, res.Release());
+ PassAway();
+ }
+
+ void Bootstrap(const TActorContext& ctx) {
+ Y_UNUSED(ctx);
+ Become(&TTaskPingRequestActor::StateFunc);
+ const auto& req = Ev->Record;
+ OperationId = req.query_id().value();
OwnerId = req.owner_id();
- Scope = req.scope();
+ Scope = req.scope();
Deadline = NProtoInterop::CastFromProto(req.deadline());
- LOG_D("Request CP::PingTask with size: " << req.ByteSize() << " bytes");
- RequestedMBytes->Collect(req.ByteSize() / 1024 / 1024);
- try {
- auto event = CreateControlPlaneEvent();
- Send(NYq::ControlPlaneStorageServiceActorId(), event.release());
- } catch (const std::exception& err) {
- const auto msg = TStringBuilder() << "PingTask Boostrap Error: " << CurrentExceptionMessage();
- Fail(msg);
- }
- }
-
-private:
- STRICT_STFUNC(
- StateFunc,
- CFunc(NActors::TEvents::TEvPoison::EventType, Die)
- HFunc(NYq::TEvControlPlaneStorage::TEvPingTaskResponse, HandleResponse)
- HFunc(NActors::TEvents::TEvUndelivered, OnUndelivered)
- )
-
- std::unique_ptr<NYq::TEvControlPlaneStorage::TEvPingTaskRequest> CreateControlPlaneEvent() {
+ LOG_D("Request CP::PingTask with size: " << req.ByteSize() << " bytes");
+ RequestedMBytes->Collect(req.ByteSize() / 1024 / 1024);
+ try {
+ auto event = CreateControlPlaneEvent();
+ Send(NYq::ControlPlaneStorageServiceActorId(), event.release());
+ } catch (const std::exception& err) {
+ const auto msg = TStringBuilder() << "PingTask Boostrap Error: " << CurrentExceptionMessage();
+ Fail(msg);
+ }
+ }
+
+private:
+ STRICT_STFUNC(
+ StateFunc,
+ CFunc(NActors::TEvents::TEvPoison::EventType, Die)
+ HFunc(NYq::TEvControlPlaneStorage::TEvPingTaskResponse, HandleResponse)
+ HFunc(NActors::TEvents::TEvUndelivered, OnUndelivered)
+ )
+
+ std::unique_ptr<NYq::TEvControlPlaneStorage::TEvPingTaskRequest> CreateControlPlaneEvent() {
auto event = std::make_unique<NYq::TEvControlPlaneStorage::TEvPingTaskRequest>(Scope, OperationId, OwnerId, Deadline);
- const auto& req = Ev->Record;
- ui64 issuesByteSize = 0;
- ui64 transientIssuesByteSize = 0;
- ui64 resultSetMetaByteSize = 0;
- ui64 dqGraphBytesSize = 0;
-
- //TODO use all fields
- if (req.status() != YandexQuery::QueryMeta::COMPUTE_STATUS_UNSPECIFIED) {
- event->Status = req.status();
- }
- if (!req.issues().empty()) {
- NYql::TIssues reqIssues;
- for (const auto& issue : req.issues()) {
- issuesByteSize += issue.ByteSize();
- }
- NYql::IssuesFromMessage(req.issues(), reqIssues);
- Issues.AddIssues(reqIssues);
- event->Issues = Issues;
- }
+ const auto& req = Ev->Record;
+ ui64 issuesByteSize = 0;
+ ui64 transientIssuesByteSize = 0;
+ ui64 resultSetMetaByteSize = 0;
+ ui64 dqGraphBytesSize = 0;
+
+ //TODO use all fields
+ if (req.status() != YandexQuery::QueryMeta::COMPUTE_STATUS_UNSPECIFIED) {
+ event->Status = req.status();
+ }
+ if (!req.issues().empty()) {
+ NYql::TIssues reqIssues;
+ for (const auto& issue : req.issues()) {
+ issuesByteSize += issue.ByteSize();
+ }
+ NYql::IssuesFromMessage(req.issues(), reqIssues);
+ Issues.AddIssues(reqIssues);
+ event->Issues = Issues;
+ }
if (!req.transient_issues().empty()) {
NYql::TIssues transientIssues;
- for (const auto& issue : req.transient_issues()) {
- transientIssuesByteSize += issue.ByteSize();
- }
+ for (const auto& issue : req.transient_issues()) {
+ transientIssuesByteSize += issue.ByteSize();
+ }
NYql::IssuesFromMessage(req.transient_issues(), transientIssues);
event->TransientIssues = transientIssues;
}
- if (req.statistics()) {
- event->Statistics = req.statistics();
- }
- if (req.ast()) {
- event->Ast = req.ast();
- }
- if (req.result_id().value()) {
- event->ResultId = req.result_id().value();
- }
- if (req.plan()) {
- event->Plan = req.plan();
- }
- if (!req.result_set_meta().empty()) {
- for (const auto& rsMeta : req.result_set_meta()) {
- resultSetMetaByteSize += rsMeta.ByteSize();
- }
+ if (req.statistics()) {
+ event->Statistics = req.statistics();
+ }
+ if (req.ast()) {
+ event->Ast = req.ast();
+ }
+ if (req.result_id().value()) {
+ event->ResultId = req.result_id().value();
+ }
+ if (req.plan()) {
+ event->Plan = req.plan();
+ }
+ if (!req.result_set_meta().empty()) {
+ for (const auto& rsMeta : req.result_set_meta()) {
+ resultSetMetaByteSize += rsMeta.ByteSize();
+ }
event->ResultSetMetas = {req.result_set_meta().begin(), req.result_set_meta().end()};
- }
- if (req.has_started_at()) {
+ }
+ if (req.has_started_at()) {
event->StartedAt = TInstant::FromValue(google::protobuf::util::TimeUtil::TimestampToMicroseconds(req.started_at()));
- }
- if (req.has_finished_at()) {
+ }
+ if (req.has_finished_at()) {
event->FinishedAt = TInstant::FromValue(google::protobuf::util::TimeUtil::TimestampToMicroseconds(req.finished_at()));
- }
+ }
event->ResignQuery = req.resign_query();
event->CreatedTopicConsumers.reserve(req.created_topic_consumers_size());
@@ -169,76 +169,76 @@ private:
event->DqGraphs.reserve(req.dq_graph_size());
for (const auto& g : req.dq_graph()) {
- dqGraphBytesSize += g.size();
+ dqGraphBytesSize += g.size();
event->DqGraphs.emplace_back(g);
}
if (req.state_load_mode()) {
event->StateLoadMode = req.state_load_mode();
}
-
+
if (req.has_disposition()) {
event->StreamingDisposition = req.disposition();
}
- LOG_D("Statistics length: " << req.statistics().size() << ", "
- << "Ast length: " << req.ast().size() << " bytes, "
- << "Plan length: " << req.plan().size() << " bytes, "
- << "Result set meta size: " << resultSetMetaByteSize << " bytes, "
- << "Topic consumers size: " << event->CreatedTopicConsumers.size() * sizeof(TEvControlPlaneStorage::TTopicConsumer) << " bytes, "
- << "Dq graphs size: " << dqGraphBytesSize << " bytes, "
- << "Issues size: " << issuesByteSize << " bytes, "
- << "Transient issues size: " << transientIssuesByteSize << " bytes");
-
+ LOG_D("Statistics length: " << req.statistics().size() << ", "
+ << "Ast length: " << req.ast().size() << " bytes, "
+ << "Plan length: " << req.plan().size() << " bytes, "
+ << "Result set meta size: " << resultSetMetaByteSize << " bytes, "
+ << "Topic consumers size: " << event->CreatedTopicConsumers.size() * sizeof(TEvControlPlaneStorage::TTopicConsumer) << " bytes, "
+ << "Dq graphs size: " << dqGraphBytesSize << " bytes, "
+ << "Issues size: " << issuesByteSize << " bytes, "
+ << "Transient issues size: " << transientIssuesByteSize << " bytes");
+
event->DqGraphIndex = req.dq_graph_index();
- return std::move(event);
- }
-
- void HandleResponse(NYq::TEvControlPlaneStorage::TEvPingTaskResponse::TPtr& ev, const TActorContext& ctx) {
- LOG_D("Got CP::PingTaskResponse");
- const auto& issues = ev->Get()->Issues;
- if (issues) {
- Issues.AddIssues(issues);
- Fail("ControlPlane PingTaskError", Ydb::StatusIds::GENERIC_ERROR);
- return;
- }
- auto res = MakeHolder<TEvents::TEvPingTaskResponse>();
- res->Status = Ydb::StatusIds::SUCCESS;
- Yq::Private::PingTaskResult result;
+ return std::move(event);
+ }
+
+ void HandleResponse(NYq::TEvControlPlaneStorage::TEvPingTaskResponse::TPtr& ev, const TActorContext& ctx) {
+ LOG_D("Got CP::PingTaskResponse");
+ const auto& issues = ev->Get()->Issues;
+ if (issues) {
+ Issues.AddIssues(issues);
+ Fail("ControlPlane PingTaskError", Ydb::StatusIds::GENERIC_ERROR);
+ return;
+ }
+ auto res = MakeHolder<TEvents::TEvPingTaskResponse>();
+ res->Status = Ydb::StatusIds::SUCCESS;
+ Yq::Private::PingTaskResult result;
result.set_action(ev->Get()->Action);
res->Record.ConstructInPlace(result);
- ctx.Send(Sender, res.Release());
- Die(ctx);
- }
-
-private:
- const TActorId Sender;
- TIntrusivePtr<ITimeProvider> TimeProvider;
- TAutoPtr<TEvents::TEvPingTaskRequest> Ev;
- TDynamicCounterPtr Counters;
- const THistogramPtr LifetimeDuration;
- const THistogramPtr RequestedMBytes;
- const TInstant StartTime;
-
- TString OperationId;
+ ctx.Send(Sender, res.Release());
+ Die(ctx);
+ }
+
+private:
+ const TActorId Sender;
+ TIntrusivePtr<ITimeProvider> TimeProvider;
+ TAutoPtr<TEvents::TEvPingTaskRequest> Ev;
+ TDynamicCounterPtr Counters;
+ const THistogramPtr LifetimeDuration;
+ const THistogramPtr RequestedMBytes;
+ const TInstant StartTime;
+
+ TString OperationId;
TString OwnerId;
- TString Scope;
+ TString Scope;
TInstant Deadline;
- NYql::TIssues Issues;
-};
-
-IActor* CreatePingTaskRequestActor(
- const NActors::TActorId& sender,
- TIntrusivePtr<ITimeProvider> timeProvider,
- TAutoPtr<TEvents::TEvPingTaskRequest> ev,
- TDynamicCounterPtr counters) {
- return new TTaskPingRequestActor(
- sender,
- timeProvider,
- std::move(ev),
- std::move(counters));
-}
-
-} /* NYq */
+ NYql::TIssues Issues;
+};
+
+IActor* CreatePingTaskRequestActor(
+ const NActors::TActorId& sender,
+ TIntrusivePtr<ITimeProvider> timeProvider,
+ TAutoPtr<TEvents::TEvPingTaskRequest> ev,
+ TDynamicCounterPtr counters) {
+ return new TTaskPingRequestActor(
+ sender,
+ timeProvider,
+ std::move(ev),
+ std::move(counters));
+}
+
+} /* NYq */
diff --git a/ydb/core/yq/libs/actors/task_result_write.cpp b/ydb/core/yq/libs/actors/task_result_write.cpp
index 760b1b56c4..5919ddbaab 100644
--- a/ydb/core/yq/libs/actors/task_result_write.cpp
+++ b/ydb/core/yq/libs/actors/task_result_write.cpp
@@ -1,144 +1,144 @@
-#include "proxy_private.h"
-
+#include "proxy_private.h"
+
#include <ydb/core/protos/services.pb.h>
-
+
#include <ydb/library/yql/public/issue/yql_issue_message.h>
-
-#include <library/cpp/yson/node/node_io.h>
-#include <library/cpp/actors/core/events.h>
-#include <library/cpp/actors/core/hfunc.h>
-#include <library/cpp/actors/core/actor_bootstrapped.h>
-#include <library/cpp/actors/core/log.h>
+
+#include <library/cpp/yson/node/node_io.h>
+#include <library/cpp/actors/core/events.h>
+#include <library/cpp/actors/core/hfunc.h>
+#include <library/cpp/actors/core/actor_bootstrapped.h>
+#include <library/cpp/actors/core/log.h>
#include <library/cpp/protobuf/interop/cast.h>
-
+
#include <ydb/core/yq/libs/control_plane_storage/events/events.h>
#include <ydb/core/yq/libs/control_plane_storage/control_plane_storage.h>
-
+
#define LOG_E(stream) \
LOG_ERROR_S(*TlsActivationContext, NKikimrServices::YQL_PRIVATE_PROXY, "PrivateWriteTask - ResultId: " << ResultId << ", RequestId: " << RequestId << ", " << stream)
#define LOG_D(stream) \
LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::YQL_PRIVATE_PROXY, "PrivateWriteTask - ResultId: " << ResultId << ", RequestId: " << RequestId << ", " << stream)
-
-namespace NYq {
-
-using namespace NActors;
-using namespace NMonitoring;
-
-
-class TWriteTaskRequestActor
- : public NActors::TActorBootstrapped<TWriteTaskRequestActor>
-{
-public:
- TWriteTaskRequestActor(
- const NActors::TActorId& sender,
- TIntrusivePtr<ITimeProvider> timeProvider,
- TAutoPtr<TEvents::TEvWriteTaskResultRequest> ev,
- TDynamicCounterPtr counters)
+
+namespace NYq {
+
+using namespace NActors;
+using namespace NMonitoring;
+
+
+class TWriteTaskRequestActor
+ : public NActors::TActorBootstrapped<TWriteTaskRequestActor>
+{
+public:
+ TWriteTaskRequestActor(
+ const NActors::TActorId& sender,
+ TIntrusivePtr<ITimeProvider> timeProvider,
+ TAutoPtr<TEvents::TEvWriteTaskResultRequest> ev,
+ TDynamicCounterPtr counters)
: Sender(sender)
- , TimeProvider(timeProvider)
- , Ev(std::move(ev))
- , Counters(std::move(counters->GetSubgroup("subsystem", "private_api")->GetSubgroup("subcomponent", "WriteTaskResult")))
- , LifetimeDuration(Counters->GetHistogram("LifetimeDurationMs", ExponentialHistogram(10, 2, 50)))
- , RequestedMBytes(Counters->GetHistogram("RequestedMB", ExponentialHistogram(6, 2, 3)))
- , StartTime(TInstant::Now())
-
- {}
-
- static constexpr char ActorName[] = "YQ_PRIVATE_WRITE_RESULT_TASK";
-
- void OnUndelivered(NActors::TEvents::TEvUndelivered::TPtr& ev, const NActors::TActorContext& ctx) {
- LOG_E("TWriteTaskRequestActor::OnUndelivered");
- Res->Status = Ydb::StatusIds::GENERIC_ERROR;
- Res->Issues.AddIssue("UNDELIVERED");
- ctx.Send(ev->Sender, Res.Release());
- Die(ctx);
- }
-
- void PassAway() final {
- LifetimeDuration->Collect((TInstant::Now() - StartTime).MilliSeconds());
- NActors::IActor::PassAway();
- }
-
- void Fail(const TString& message, Ydb::StatusIds::StatusCode reqStatus = Ydb::StatusIds::INTERNAL_ERROR) {
- Issues.AddIssue(message);
- const auto codeStr = Ydb::StatusIds_StatusCode_Name(reqStatus);
- LOG_E(TStringBuilder()
- << "Failed with code: " << codeStr
- << " Details: " << Issues.ToString());
- Res->Status = reqStatus;
- Res->Issues.AddIssues(Issues);
- Send(Sender, Res.Release());
- PassAway();
- }
-
+ , TimeProvider(timeProvider)
+ , Ev(std::move(ev))
+ , Counters(std::move(counters->GetSubgroup("subsystem", "private_api")->GetSubgroup("subcomponent", "WriteTaskResult")))
+ , LifetimeDuration(Counters->GetHistogram("LifetimeDurationMs", ExponentialHistogram(10, 2, 50)))
+ , RequestedMBytes(Counters->GetHistogram("RequestedMB", ExponentialHistogram(6, 2, 3)))
+ , StartTime(TInstant::Now())
+
+ {}
+
+ static constexpr char ActorName[] = "YQ_PRIVATE_WRITE_RESULT_TASK";
+
+ void OnUndelivered(NActors::TEvents::TEvUndelivered::TPtr& ev, const NActors::TActorContext& ctx) {
+ LOG_E("TWriteTaskRequestActor::OnUndelivered");
+ Res->Status = Ydb::StatusIds::GENERIC_ERROR;
+ Res->Issues.AddIssue("UNDELIVERED");
+ ctx.Send(ev->Sender, Res.Release());
+ Die(ctx);
+ }
+
+ void PassAway() final {
+ LifetimeDuration->Collect((TInstant::Now() - StartTime).MilliSeconds());
+ NActors::IActor::PassAway();
+ }
+
+ void Fail(const TString& message, Ydb::StatusIds::StatusCode reqStatus = Ydb::StatusIds::INTERNAL_ERROR) {
+ Issues.AddIssue(message);
+ const auto codeStr = Ydb::StatusIds_StatusCode_Name(reqStatus);
+ LOG_E(TStringBuilder()
+ << "Failed with code: " << codeStr
+ << " Details: " << Issues.ToString());
+ Res->Status = reqStatus;
+ Res->Issues.AddIssues(Issues);
+ Send(Sender, Res.Release());
+ PassAway();
+ }
+
void Bootstrap(const TActorContext&) {
- Become(&TWriteTaskRequestActor::StateFunc);
- const auto& req = Ev->Record;
-
+ Become(&TWriteTaskRequestActor::StateFunc);
+ const auto& req = Ev->Record;
+
Deadline = NProtoInterop::CastFromProto(req.deadline());
-
- const auto& resultSet = req.result_set();
- ResultId = req.result_id().value();
- const auto& resultSetId = req.result_set_id();
- const auto& startRowIndex = req.offset();
- RequestId = req.request_id();
- LOG_D("Request CP::WriteTaskResult with size: " << req.ByteSize() << " bytes");
- RequestedMBytes->Collect(req.ByteSize() / 1024 / 1024);
- Send(NYq::ControlPlaneStorageServiceActorId(),
- new NYq::TEvControlPlaneStorage::TEvWriteResultDataRequest(ResultId, resultSetId, startRowIndex, Deadline, resultSet), 0, RequestId);
- }
-
-private:
- STRICT_STFUNC(
- StateFunc,
- CFunc(NActors::TEvents::TEvPoison::EventType, Die)
- HFunc(NActors::TEvents::TEvUndelivered, OnUndelivered)
- HFunc(NYq::TEvControlPlaneStorage::TEvWriteResultDataResponse, HandleResponse);
- )
-
- void HandleResponse(NYq::TEvControlPlaneStorage::TEvWriteResultDataResponse::TPtr& ev, const TActorContext& ctx) {
- LOG_D("Got CP::WriteTaskResult Response");
- const auto& issues = ev->Get()->Issues;
- if (issues) {
- Issues.AddIssues(issues);
- Fail("ControlPlane WriteTaskResult Error", Ydb::StatusIds::GENERIC_ERROR);
- return;
- }
- Res->Record.ConstructInPlace();
- Res->Record->set_request_id(RequestId);
- Res->Issues.AddIssues(Issues);
- Res->Status = Ydb::StatusIds::SUCCESS;
- ctx.Send(Sender, Res.Release());
- Die(ctx);
- }
-
- const TActorId Sender;
- TIntrusivePtr<ITimeProvider> TimeProvider;
- TAutoPtr<TEvents::TEvWriteTaskResultRequest> Ev;
- TDynamicCounterPtr Counters;
- const THistogramPtr LifetimeDuration;
- const THistogramPtr RequestedMBytes;
- const TInstant StartTime;
-
- TString ResultId;
- ui64 RequestId = 0;
- TInstant Deadline;
-
- NYql::TIssues Issues;
-
- THolder<TEvents::TEvWriteTaskResultResponse> Res = MakeHolder<TEvents::TEvWriteTaskResultResponse>();
-};
-
-IActor* CreateWriteTaskResultRequestActor(
- const NActors::TActorId& sender,
- TIntrusivePtr<ITimeProvider> timeProvider,
- TAutoPtr<TEvents::TEvWriteTaskResultRequest> ev,
- TDynamicCounterPtr counters) {
- return new TWriteTaskRequestActor(
- sender,
- timeProvider,
- std::move(ev),
- std::move(counters));
-}
-
-} /* NYq */
+
+ const auto& resultSet = req.result_set();
+ ResultId = req.result_id().value();
+ const auto& resultSetId = req.result_set_id();
+ const auto& startRowIndex = req.offset();
+ RequestId = req.request_id();
+ LOG_D("Request CP::WriteTaskResult with size: " << req.ByteSize() << " bytes");
+ RequestedMBytes->Collect(req.ByteSize() / 1024 / 1024);
+ Send(NYq::ControlPlaneStorageServiceActorId(),
+ new NYq::TEvControlPlaneStorage::TEvWriteResultDataRequest(ResultId, resultSetId, startRowIndex, Deadline, resultSet), 0, RequestId);
+ }
+
+private:
+ STRICT_STFUNC(
+ StateFunc,
+ CFunc(NActors::TEvents::TEvPoison::EventType, Die)
+ HFunc(NActors::TEvents::TEvUndelivered, OnUndelivered)
+ HFunc(NYq::TEvControlPlaneStorage::TEvWriteResultDataResponse, HandleResponse);
+ )
+
+ void HandleResponse(NYq::TEvControlPlaneStorage::TEvWriteResultDataResponse::TPtr& ev, const TActorContext& ctx) {
+ LOG_D("Got CP::WriteTaskResult Response");
+ const auto& issues = ev->Get()->Issues;
+ if (issues) {
+ Issues.AddIssues(issues);
+ Fail("ControlPlane WriteTaskResult Error", Ydb::StatusIds::GENERIC_ERROR);
+ return;
+ }
+ Res->Record.ConstructInPlace();
+ Res->Record->set_request_id(RequestId);
+ Res->Issues.AddIssues(Issues);
+ Res->Status = Ydb::StatusIds::SUCCESS;
+ ctx.Send(Sender, Res.Release());
+ Die(ctx);
+ }
+
+ const TActorId Sender;
+ TIntrusivePtr<ITimeProvider> TimeProvider;
+ TAutoPtr<TEvents::TEvWriteTaskResultRequest> Ev;
+ TDynamicCounterPtr Counters;
+ const THistogramPtr LifetimeDuration;
+ const THistogramPtr RequestedMBytes;
+ const TInstant StartTime;
+
+ TString ResultId;
+ ui64 RequestId = 0;
+ TInstant Deadline;
+
+ NYql::TIssues Issues;
+
+ THolder<TEvents::TEvWriteTaskResultResponse> Res = MakeHolder<TEvents::TEvWriteTaskResultResponse>();
+};
+
+IActor* CreateWriteTaskResultRequestActor(
+ const NActors::TActorId& sender,
+ TIntrusivePtr<ITimeProvider> timeProvider,
+ TAutoPtr<TEvents::TEvWriteTaskResultRequest> ev,
+ TDynamicCounterPtr counters) {
+ return new TWriteTaskRequestActor(
+ sender,
+ timeProvider,
+ std::move(ev),
+ std::move(counters));
+}
+
+} /* NYq */
diff --git a/ydb/core/yq/libs/actors/ya.make b/ydb/core/yq/libs/actors/ya.make
index 347e1dbea7..3638593814 100644
--- a/ydb/core/yq/libs/actors/ya.make
+++ b/ydb/core/yq/libs/actors/ya.make
@@ -7,24 +7,24 @@ SRCS(
database_resolver.cpp
error.cpp
nodes_health_check.cpp
- nodes_manager.cpp
+ nodes_manager.cpp
pending_fetcher.cpp
pinger.cpp
proxy.cpp
proxy_private.cpp
result_writer.cpp
run_actor.cpp
- run_actor_params.cpp
+ run_actor_params.cpp
system_clusters.cpp
table_bindings_from_bindings.cpp
task_get.cpp
- task_ping.cpp
- task_result_write.cpp
+ task_ping.cpp
+ task_result_write.cpp
)
PEERDIR(
library/cpp/actors/core
- library/cpp/actors/interconnect
+ library/cpp/actors/interconnect
library/cpp/json/yson
library/cpp/monlib/dynamic_counters
library/cpp/random_provider
diff --git a/ydb/core/yq/libs/checkpoint_storage/gc.cpp b/ydb/core/yq/libs/checkpoint_storage/gc.cpp
index 88a141ee65..2a0e73ecb4 100644
--- a/ydb/core/yq/libs/checkpoint_storage/gc.cpp
+++ b/ydb/core/yq/libs/checkpoint_storage/gc.cpp
@@ -61,7 +61,7 @@ public:
void Bootstrap(const TActorContext& ctx);
static constexpr char ActorName[] = "YQ_GC_ACTOR";
-
+
private:
STRICT_STFUNC(StateFunc,
HFunc(TEvCheckpointStorage::TEvNewCheckpointSucceeded, Handle);
diff --git a/ydb/core/yq/libs/checkpoint_storage/storage_proxy.cpp b/ydb/core/yq/libs/checkpoint_storage/storage_proxy.cpp
index e92fc1b208..07c8d5101a 100644
--- a/ydb/core/yq/libs/checkpoint_storage/storage_proxy.cpp
+++ b/ydb/core/yq/libs/checkpoint_storage/storage_proxy.cpp
@@ -51,7 +51,7 @@ public:
void Bootstrap();
static constexpr char ActorName[] = "YQ_STORAGE_PROXY";
-
+
private:
STRICT_STFUNC(StateFunc,
hFunc(TEvCheckpointStorage::TEvRegisterCoordinatorRequest, Handle);
diff --git a/ydb/core/yq/libs/checkpointing/checkpoint_coordinator.h b/ydb/core/yq/libs/checkpointing/checkpoint_coordinator.h
index cd2b68a35e..0bb76685e1 100644
--- a/ydb/core/yq/libs/checkpointing/checkpoint_coordinator.h
+++ b/ydb/core/yq/libs/checkpointing/checkpoint_coordinator.h
@@ -76,8 +76,8 @@ public:
void Bootstrap();
- static constexpr char ActorName[] = "YQ_CHECKPOINT_COORDINATOR";
-
+ static constexpr char ActorName[] = "YQ_CHECKPOINT_COORDINATOR";
+
private:
void InitCheckpoint();
void InjectCheckpoint(const TCheckpointId& checkpointId);
diff --git a/ydb/core/yq/libs/common/cache.h b/ydb/core/yq/libs/common/cache.h
index 0a27831310..e79eb32839 100644
--- a/ydb/core/yq/libs/common/cache.h
+++ b/ydb/core/yq/libs/common/cache.h
@@ -4,7 +4,7 @@
#include <util/datetime/base.h>
#include <util/system/mutex.h>
-namespace NYq {
+namespace NYq {
struct TTtlCacheSettings {
TDuration Ttl = TDuration::Minutes(10);
@@ -113,4 +113,4 @@ private:
TTtlCacheSettings Config;
};
-} // namespace NYq
+} // namespace NYq
diff --git a/ydb/core/yq/libs/common/cache_ut.cpp b/ydb/core/yq/libs/common/cache_ut.cpp
index 005689a709..4e5ace033d 100644
--- a/ydb/core/yq/libs/common/cache_ut.cpp
+++ b/ydb/core/yq/libs/common/cache_ut.cpp
@@ -2,7 +2,7 @@
#include <ydb/services/ydb/ydb_common_ut.h>
-using namespace NYq;
+using namespace NYq;
using TCache = TTtlCache<int,int>;
diff --git a/ydb/core/yq/libs/common/database_token_builder.cpp b/ydb/core/yq/libs/common/database_token_builder.cpp
index 14991504bb..89d36cfba6 100644
--- a/ydb/core/yq/libs/common/database_token_builder.cpp
+++ b/ydb/core/yq/libs/common/database_token_builder.cpp
@@ -1,45 +1,45 @@
-#include "database_token_builder.h"
+#include "database_token_builder.h"
#include <ydb/library/yql/providers/common/structured_token/yql_token_builder.h>
-
-namespace NYq {
-
-using namespace NYql;
-
- TString BuildStructuredToken(const YandexQuery::IamAuth& auth, const TString& authToken, const THashMap<TString, TString>& accountIdSignatures) {
- TStructuredTokenBuilder result;
- switch (auth.identity_case()) {
- case YandexQuery::IamAuth::kCurrentIam:
- result.SetIAMToken(authToken);
- break;
- case YandexQuery::IamAuth::kServiceAccount: {
- const auto& signature = accountIdSignatures.at(auth.service_account().id());
- result.SetServiceAccountIdAuth(auth.service_account().id(), signature);
- break;
- }
- default:
- result.SetNoAuth();
- break;
- }
-
- return result.ToJson();
- }
-
- void TryAddDatabaseToResolve(
- const YandexQuery::IamAuth& auth,
- const TString& databaseId,
- DatabaseType type,
- const TString& authToken,
- const THashMap<TString, TString>& accountIdSignatures,
- THashMap<std::pair<TString, DatabaseType>, TEvents::TDatabaseAuth>& databaseIds) {
- if (!databaseId) {
- return;
- }
-
- TEvents::TDatabaseAuth info;
- info.StructuredToken = BuildStructuredToken(auth, authToken, accountIdSignatures);
- info.AddBearerToToken = true; // XXX
- databaseIds[std::make_pair(databaseId, type)] = info;
- }
-
+
+namespace NYq {
+
+using namespace NYql;
+
+ TString BuildStructuredToken(const YandexQuery::IamAuth& auth, const TString& authToken, const THashMap<TString, TString>& accountIdSignatures) {
+ TStructuredTokenBuilder result;
+ switch (auth.identity_case()) {
+ case YandexQuery::IamAuth::kCurrentIam:
+ result.SetIAMToken(authToken);
+ break;
+ case YandexQuery::IamAuth::kServiceAccount: {
+ const auto& signature = accountIdSignatures.at(auth.service_account().id());
+ result.SetServiceAccountIdAuth(auth.service_account().id(), signature);
+ break;
+ }
+ default:
+ result.SetNoAuth();
+ break;
+ }
+
+ return result.ToJson();
+ }
+
+ void TryAddDatabaseToResolve(
+ const YandexQuery::IamAuth& auth,
+ const TString& databaseId,
+ DatabaseType type,
+ const TString& authToken,
+ const THashMap<TString, TString>& accountIdSignatures,
+ THashMap<std::pair<TString, DatabaseType>, TEvents::TDatabaseAuth>& databaseIds) {
+ if (!databaseId) {
+ return;
+ }
+
+ TEvents::TDatabaseAuth info;
+ info.StructuredToken = BuildStructuredToken(auth, authToken, accountIdSignatures);
+ info.AddBearerToToken = true; // XXX
+ databaseIds[std::make_pair(databaseId, type)] = info;
+ }
+
} // NYq
diff --git a/ydb/core/yq/libs/common/database_token_builder.h b/ydb/core/yq/libs/common/database_token_builder.h
index c903b756ea..a3e00efab5 100644
--- a/ydb/core/yq/libs/common/database_token_builder.h
+++ b/ydb/core/yq/libs/common/database_token_builder.h
@@ -2,23 +2,23 @@
#include <ydb/core/yq/libs/control_plane_storage/events/events.h>
#include <ydb/core/yq/libs/events/events.h>
-
-namespace NYq {
-
-using namespace NActors;
-using namespace NYql;
-
- TString BuildStructuredToken(
- const YandexQuery::IamAuth& auth,
- const TString& authToken,
- const THashMap<TString, TString>& accountIdSignatures);
-
- void TryAddDatabaseToResolve(
- const YandexQuery::IamAuth& auth,
- const TString& databaseId,
- DatabaseType type,
- const TString& authToken,
- const THashMap<TString, TString>& accountIdSignatures,
- THashMap<std::pair<TString, DatabaseType>, TEvents::TDatabaseAuth>& databaseIds);
-
+
+namespace NYq {
+
+using namespace NActors;
+using namespace NYql;
+
+ TString BuildStructuredToken(
+ const YandexQuery::IamAuth& auth,
+ const TString& authToken,
+ const THashMap<TString, TString>& accountIdSignatures);
+
+ void TryAddDatabaseToResolve(
+ const YandexQuery::IamAuth& auth,
+ const TString& databaseId,
+ DatabaseType type,
+ const TString& authToken,
+ const THashMap<TString, TString>& accountIdSignatures,
+ THashMap<std::pair<TString, DatabaseType>, TEvents::TDatabaseAuth>& databaseIds);
+
} // NYq
diff --git a/ydb/core/yq/libs/common/entity_id.cpp b/ydb/core/yq/libs/common/entity_id.cpp
index ea3ea069cb..dab1d1abca 100644
--- a/ydb/core/yq/libs/common/entity_id.cpp
+++ b/ydb/core/yq/libs/common/entity_id.cpp
@@ -7,7 +7,7 @@
#include <util/stream/format.h>
#include <util/string/ascii.h>
-namespace NYq {
+namespace NYq {
// used ascii order: IntToChar[i] < IntToChar[i+1]
constexpr char IntToChar[] = {
@@ -84,4 +84,4 @@ IEntityIdGenerator::TPtr CreateEntityIdGenerator(const TString& prefix) {
return MakeIntrusive<TEntityIdGenerator>(prefix);
}
-} // namespace NYq
+} // namespace NYq
diff --git a/ydb/core/yq/libs/common/entity_id.h b/ydb/core/yq/libs/common/entity_id.h
index 5175a04c7b..d34dee0f55 100644
--- a/ydb/core/yq/libs/common/entity_id.h
+++ b/ydb/core/yq/libs/common/entity_id.h
@@ -3,7 +3,7 @@
#include <util/generic/string.h>
#include <util/datetime/base.h>
-namespace NYq {
+namespace NYq {
enum class EEntityType : char {
UNDEFINED = 'u',
@@ -26,4 +26,4 @@ struct IEntityIdGenerator : public TThrRefBase {
IEntityIdGenerator::TPtr CreateEntityIdGenerator(const TString& prefix);
-} // namespace NYq
+} // namespace NYq
diff --git a/ydb/core/yq/libs/common/entity_id_ut.cpp b/ydb/core/yq/libs/common/entity_id_ut.cpp
index 1d785c1865..8533bdcca0 100644
--- a/ydb/core/yq/libs/common/entity_id_ut.cpp
+++ b/ydb/core/yq/libs/common/entity_id_ut.cpp
@@ -3,7 +3,7 @@
#include <ydb/services/ydb/ydb_common_ut.h>
#include <limits>
-using namespace NYq;
+using namespace NYq;
Y_UNIT_TEST_SUITE(EntityId) {
Y_UNIT_TEST(Distinct) {
diff --git a/ydb/core/yq/libs/common/rows_proto_splitter.cpp b/ydb/core/yq/libs/common/rows_proto_splitter.cpp
index 38fbd99812..00132486dd 100644
--- a/ydb/core/yq/libs/common/rows_proto_splitter.cpp
+++ b/ydb/core/yq/libs/common/rows_proto_splitter.cpp
@@ -1,77 +1,77 @@
-#include "rows_proto_splitter.h"
-
-#include <util/string/builder.h>
-
-namespace NYq {
-
-TRowsProtoSplitter::TRowsProtoSplitter(
- const Ydb::ResultSet& resultSet,
- const ui64 chunkLimit,
- const ui64 headerProtoByteSize,
- const ui64 maxRowsCountPerChunk)
- : ResultSet(resultSet)
- , ChunkLimit(chunkLimit)
- , MaxRowsCountPerChunk(maxRowsCountPerChunk)
- {
- ui64 colsBytes = 0;
- for (const auto& column : resultSet.columns()) {
- colsBytes += column.ByteSizeLong();
- }
- BaseProtoBytesSize = colsBytes + headerProtoByteSize;
- }
-
-TSplittedResultSets TRowsProtoSplitter::MakeResultWithIssues(const TString& msg) {
- Issues.AddIssue(msg);
- Success = false;
- return TSplittedResultSets{{}, std::move(Issues), Success};
-}
-
-TString TRowsProtoSplitter::CheckLimits(ui64 curRowBytes, size_t rowInd) {
- TString issueMsg;
-
- if (curRowBytes + BaseProtoBytesSize > ChunkLimit) {
- issueMsg += TStringBuilder() << "Can not write Row["<< rowInd << "] with size: "
- << curRowBytes + BaseProtoBytesSize << " bytes (> " << ChunkLimit / (1024 * 1024) << "_MB)\n";
- }
-
- return issueMsg;
-}
-
-void TRowsProtoSplitter::MakeNewChunk(Ydb::ResultSet& resultSet, ui64& chunkRowsCounter, ui64& chunkSize) {
- resultSet.Clear();
- resultSet.mutable_columns()->CopyFrom(ResultSet.columns());
-
- chunkRowsCounter = 0;
-
- chunkSize = BaseProtoBytesSize;
-}
-
-TSplittedResultSets TRowsProtoSplitter::Split() {
- size_t rowInd = 0;
- size_t curChunkRowsCounter = 0;
-
- Ydb::ResultSet curResultSet;
- ui64 curChunkSize = 0;
- MakeNewChunk(curResultSet, curChunkRowsCounter, curChunkSize);
-
- for (const auto& row : ResultSet.rows()) {
- if (const auto issueMsg = CheckLimits(row.ByteSizeLong(), rowInd++)) {
- return MakeResultWithIssues(issueMsg);
- }
-
- if (row.ByteSizeLong() + curChunkSize <= ChunkLimit && ++curChunkRowsCounter <= MaxRowsCountPerChunk) {
- curChunkSize += row.ByteSizeLong();
- *curResultSet.add_rows() = row;
- continue;
- }
-
- SplittedResultSets.emplace_back(curResultSet);
- MakeNewChunk(curResultSet, curChunkRowsCounter, curChunkSize);
- *curResultSet.add_rows() = row;
-
- }
- SplittedResultSets.emplace_back(curResultSet); //last rs
- return TSplittedResultSets{std::move(SplittedResultSets), std::move(Issues), Success};
-}
-
-} //NYq
+#include "rows_proto_splitter.h"
+
+#include <util/string/builder.h>
+
+namespace NYq {
+
+TRowsProtoSplitter::TRowsProtoSplitter(
+ const Ydb::ResultSet& resultSet,
+ const ui64 chunkLimit,
+ const ui64 headerProtoByteSize,
+ const ui64 maxRowsCountPerChunk)
+ : ResultSet(resultSet)
+ , ChunkLimit(chunkLimit)
+ , MaxRowsCountPerChunk(maxRowsCountPerChunk)
+ {
+ ui64 colsBytes = 0;
+ for (const auto& column : resultSet.columns()) {
+ colsBytes += column.ByteSizeLong();
+ }
+ BaseProtoBytesSize = colsBytes + headerProtoByteSize;
+ }
+
+TSplittedResultSets TRowsProtoSplitter::MakeResultWithIssues(const TString& msg) {
+ Issues.AddIssue(msg);
+ Success = false;
+ return TSplittedResultSets{{}, std::move(Issues), Success};
+}
+
+TString TRowsProtoSplitter::CheckLimits(ui64 curRowBytes, size_t rowInd) {
+ TString issueMsg;
+
+ if (curRowBytes + BaseProtoBytesSize > ChunkLimit) {
+ issueMsg += TStringBuilder() << "Can not write Row["<< rowInd << "] with size: "
+ << curRowBytes + BaseProtoBytesSize << " bytes (> " << ChunkLimit / (1024 * 1024) << "_MB)\n";
+ }
+
+ return issueMsg;
+}
+
+void TRowsProtoSplitter::MakeNewChunk(Ydb::ResultSet& resultSet, ui64& chunkRowsCounter, ui64& chunkSize) {
+ resultSet.Clear();
+ resultSet.mutable_columns()->CopyFrom(ResultSet.columns());
+
+ chunkRowsCounter = 0;
+
+ chunkSize = BaseProtoBytesSize;
+}
+
+TSplittedResultSets TRowsProtoSplitter::Split() {
+ size_t rowInd = 0;
+ size_t curChunkRowsCounter = 0;
+
+ Ydb::ResultSet curResultSet;
+ ui64 curChunkSize = 0;
+ MakeNewChunk(curResultSet, curChunkRowsCounter, curChunkSize);
+
+ for (const auto& row : ResultSet.rows()) {
+ if (const auto issueMsg = CheckLimits(row.ByteSizeLong(), rowInd++)) {
+ return MakeResultWithIssues(issueMsg);
+ }
+
+ if (row.ByteSizeLong() + curChunkSize <= ChunkLimit && ++curChunkRowsCounter <= MaxRowsCountPerChunk) {
+ curChunkSize += row.ByteSizeLong();
+ *curResultSet.add_rows() = row;
+ continue;
+ }
+
+ SplittedResultSets.emplace_back(curResultSet);
+ MakeNewChunk(curResultSet, curChunkRowsCounter, curChunkSize);
+ *curResultSet.add_rows() = row;
+
+ }
+ SplittedResultSets.emplace_back(curResultSet); //last rs
+ return TSplittedResultSets{std::move(SplittedResultSets), std::move(Issues), Success};
+}
+
+} //NYq
diff --git a/ydb/core/yq/libs/common/rows_proto_splitter.h b/ydb/core/yq/libs/common/rows_proto_splitter.h
index a666e8e894..b9ef0a57a7 100644
--- a/ydb/core/yq/libs/common/rows_proto_splitter.h
+++ b/ydb/core/yq/libs/common/rows_proto_splitter.h
@@ -1,42 +1,42 @@
-#pragma once
+#pragma once
#include <ydb/public/api/protos/ydb_value.pb.h>
#include <ydb/library/yql/public/issue/yql_issue_message.h>
-
-namespace NYq {
-
-struct TSplittedResultSets {
- TVector<Ydb::ResultSet> ResultSets;
- NYql::TIssues Issues;
- bool Success = true;
-};
-
-class TRowsProtoSplitter {
-public:
- TRowsProtoSplitter(
- const Ydb::ResultSet& resultSet,
- const ui64 chunkLimit,
- const ui64 headerProtoByteSize = 0,
+
+namespace NYq {
+
+struct TSplittedResultSets {
+ TVector<Ydb::ResultSet> ResultSets;
+ NYql::TIssues Issues;
+ bool Success = true;
+};
+
+class TRowsProtoSplitter {
+public:
+ TRowsProtoSplitter(
+ const Ydb::ResultSet& resultSet,
+ const ui64 chunkLimit,
+ const ui64 headerProtoByteSize = 0,
const ui64 maxRowsCountPerChunk = 100'000
);
- void MakeNewChunk(Ydb::ResultSet& resultSet, ui64& chunkRowsCounter, ui64& chunkSize);
-
- TString CheckLimits(ui64 curRowBytes, size_t rowInd);
-
- TSplittedResultSets MakeResultWithIssues(const TString& msg);
-
- TSplittedResultSets Split();
-
-
-private:
- Ydb::ResultSet ResultSet;
- const ui64 ChunkLimit = 0;
- ui64 BaseProtoBytesSize = 0;
- const ui64 MaxRowsCountPerChunk = 0;
-
- TVector<Ydb::ResultSet> SplittedResultSets;
- NYql::TIssues Issues;
- bool Success = true;
-};
-
-} //NYq
+ void MakeNewChunk(Ydb::ResultSet& resultSet, ui64& chunkRowsCounter, ui64& chunkSize);
+
+ TString CheckLimits(ui64 curRowBytes, size_t rowInd);
+
+ TSplittedResultSets MakeResultWithIssues(const TString& msg);
+
+ TSplittedResultSets Split();
+
+
+private:
+ Ydb::ResultSet ResultSet;
+ const ui64 ChunkLimit = 0;
+ ui64 BaseProtoBytesSize = 0;
+ const ui64 MaxRowsCountPerChunk = 0;
+
+ TVector<Ydb::ResultSet> SplittedResultSets;
+ NYql::TIssues Issues;
+ bool Success = true;
+};
+
+} //NYq
diff --git a/ydb/core/yq/libs/common/rows_proto_splitter_ut.cpp b/ydb/core/yq/libs/common/rows_proto_splitter_ut.cpp
index bd8befadc2..ee7dd0dd4d 100644
--- a/ydb/core/yq/libs/common/rows_proto_splitter_ut.cpp
+++ b/ydb/core/yq/libs/common/rows_proto_splitter_ut.cpp
@@ -1,115 +1,115 @@
-#include "rows_proto_splitter.h"
-
+#include "rows_proto_splitter.h"
+
#include <ydb/public/sdk/cpp/client/ydb_result/result.h>
#include <ydb/services/ydb/ydb_common_ut.h>
-
-namespace NYq {
-
-namespace {
-
-Ydb::ResultSet MakeProtoResultSet() {
- const TString resultSetString =
- "columns {\n"
- " name: \"colName\"\n"
- " type {\n"
- " list_type {\n"
- " item {\n"
- " type_id: INT32\n"
- " }\n"
- " }\n"
- " }\n"
- "}\n"
- "rows {\n"
- " items {\n"
- " items {\n"
- " int32_value: 42\n"
- " }\n"
- " items {\n"
- " int32_value: 43\n"
- " }\n"
- " items {\n"
- " int32_value: 44\n"
- " }\n"
- " }\n"
- "}\n"
- "rows {\n"
- " items {\n"
- " items {\n"
- " int32_value: 45\n"
- " }\n"
- " items {\n"
- " int32_value: 46\n"
- " }\n"
- " items {\n"
- " int32_value: 47\n"
- " }\n"
- " }\n"
- "}\n";
- Ydb::ResultSet rsProto;
- google::protobuf::TextFormat::ParseFromString(resultSetString, &rsProto);
- return rsProto;
-}
-
-} //namespace
-
-Y_UNIT_TEST_SUITE(SplitterBasic) {
- Y_UNIT_TEST(EqualSplitByMaxBytesLimitPerChunk) {
- Ydb::ResultSet resultSet = MakeProtoResultSet();
- TRowsProtoSplitter splitter(resultSet, 41, 0);
- const auto& result = splitter.Split();
- UNIT_ASSERT_C(result.Success, result.Issues.ToString());
- UNIT_ASSERT_VALUES_EQUAL(result.ResultSets.size(), 2);
-
- int expectedVal = 42;
- for (const auto& rs : result.ResultSets) {
- NYdb::TResultSetParser rsParser(rs);
- UNIT_ASSERT_EQUAL(rsParser.ColumnsCount(), 1);
- UNIT_ASSERT_EQUAL(rsParser.RowsCount(), 1);
- UNIT_ASSERT_EQUAL(rsParser.ColumnIndex("colName"), 0);
- UNIT_ASSERT_EQUAL(rsParser.ColumnIndex("otherName"), -1);
- auto& column0 = rsParser.ColumnParser(0);
- while (rsParser.TryNextRow()) {
- column0.OpenList();
- while (column0.TryNextListItem()) {
- UNIT_ASSERT_EQUAL(column0.GetInt32(), expectedVal++);
- }
- }
- }
- }
-
- Y_UNIT_TEST(EqualSplitByMaxRowsLimitPerChunk) {
- Ydb::ResultSet resultSet = MakeProtoResultSet();
- TRowsProtoSplitter splitter(resultSet, 1024 * 1024, 0, 1);
- const auto& result = splitter.Split();
- UNIT_ASSERT_C(result.Success, result.Issues.ToString());
- UNIT_ASSERT_VALUES_EQUAL(result.ResultSets.size(), 2);
-
- int expectedVal = 42;
- for (const auto& rs : result.ResultSets) {
- NYdb::TResultSetParser rsParser(rs);
- UNIT_ASSERT_EQUAL(rsParser.ColumnsCount(), 1);
- UNIT_ASSERT_EQUAL_C(rsParser.RowsCount(), 1, rsParser.RowsCount());
- UNIT_ASSERT_EQUAL(rsParser.ColumnIndex("colName"), 0);
- UNIT_ASSERT_EQUAL(rsParser.ColumnIndex("otherName"), -1);
- auto& column0 = rsParser.ColumnParser(0);
- while (rsParser.TryNextRow()) {
- column0.OpenList();
- while (column0.TryNextListItem()) {
- UNIT_ASSERT_EQUAL(column0.GetInt32(), expectedVal++);
- }
- }
- }
- }
-
- Y_UNIT_TEST(LimitExceed) {
- Ydb::ResultSet resultSet = MakeProtoResultSet();
- TRowsProtoSplitter splitter(resultSet, 1, 0);
- const auto& result = splitter.Split();
- const auto& issuesStr = result.Issues.ToString();
- UNIT_ASSERT_C(!result.Success, issuesStr);
- UNIT_ASSERT_VALUES_EQUAL(result.ResultSets.size(), 0);
- UNIT_ASSERT_C(issuesStr.Contains("Can not write Row[0] with size: 41 bytes"), issuesStr);
- }
-}
-
-} // NYq
+
+namespace NYq {
+
+namespace {
+
+Ydb::ResultSet MakeProtoResultSet() {
+ const TString resultSetString =
+ "columns {\n"
+ " name: \"colName\"\n"
+ " type {\n"
+ " list_type {\n"
+ " item {\n"
+ " type_id: INT32\n"
+ " }\n"
+ " }\n"
+ " }\n"
+ "}\n"
+ "rows {\n"
+ " items {\n"
+ " items {\n"
+ " int32_value: 42\n"
+ " }\n"
+ " items {\n"
+ " int32_value: 43\n"
+ " }\n"
+ " items {\n"
+ " int32_value: 44\n"
+ " }\n"
+ " }\n"
+ "}\n"
+ "rows {\n"
+ " items {\n"
+ " items {\n"
+ " int32_value: 45\n"
+ " }\n"
+ " items {\n"
+ " int32_value: 46\n"
+ " }\n"
+ " items {\n"
+ " int32_value: 47\n"
+ " }\n"
+ " }\n"
+ "}\n";
+ Ydb::ResultSet rsProto;
+ google::protobuf::TextFormat::ParseFromString(resultSetString, &rsProto);
+ return rsProto;
+}
+
+} //namespace
+
+Y_UNIT_TEST_SUITE(SplitterBasic) {
+ Y_UNIT_TEST(EqualSplitByMaxBytesLimitPerChunk) {
+ Ydb::ResultSet resultSet = MakeProtoResultSet();
+ TRowsProtoSplitter splitter(resultSet, 41, 0);
+ const auto& result = splitter.Split();
+ UNIT_ASSERT_C(result.Success, result.Issues.ToString());
+ UNIT_ASSERT_VALUES_EQUAL(result.ResultSets.size(), 2);
+
+ int expectedVal = 42;
+ for (const auto& rs : result.ResultSets) {
+ NYdb::TResultSetParser rsParser(rs);
+ UNIT_ASSERT_EQUAL(rsParser.ColumnsCount(), 1);
+ UNIT_ASSERT_EQUAL(rsParser.RowsCount(), 1);
+ UNIT_ASSERT_EQUAL(rsParser.ColumnIndex("colName"), 0);
+ UNIT_ASSERT_EQUAL(rsParser.ColumnIndex("otherName"), -1);
+ auto& column0 = rsParser.ColumnParser(0);
+ while (rsParser.TryNextRow()) {
+ column0.OpenList();
+ while (column0.TryNextListItem()) {
+ UNIT_ASSERT_EQUAL(column0.GetInt32(), expectedVal++);
+ }
+ }
+ }
+ }
+
+ Y_UNIT_TEST(EqualSplitByMaxRowsLimitPerChunk) {
+ Ydb::ResultSet resultSet = MakeProtoResultSet();
+ TRowsProtoSplitter splitter(resultSet, 1024 * 1024, 0, 1);
+ const auto& result = splitter.Split();
+ UNIT_ASSERT_C(result.Success, result.Issues.ToString());
+ UNIT_ASSERT_VALUES_EQUAL(result.ResultSets.size(), 2);
+
+ int expectedVal = 42;
+ for (const auto& rs : result.ResultSets) {
+ NYdb::TResultSetParser rsParser(rs);
+ UNIT_ASSERT_EQUAL(rsParser.ColumnsCount(), 1);
+ UNIT_ASSERT_EQUAL_C(rsParser.RowsCount(), 1, rsParser.RowsCount());
+ UNIT_ASSERT_EQUAL(rsParser.ColumnIndex("colName"), 0);
+ UNIT_ASSERT_EQUAL(rsParser.ColumnIndex("otherName"), -1);
+ auto& column0 = rsParser.ColumnParser(0);
+ while (rsParser.TryNextRow()) {
+ column0.OpenList();
+ while (column0.TryNextListItem()) {
+ UNIT_ASSERT_EQUAL(column0.GetInt32(), expectedVal++);
+ }
+ }
+ }
+ }
+
+ Y_UNIT_TEST(LimitExceed) {
+ Ydb::ResultSet resultSet = MakeProtoResultSet();
+ TRowsProtoSplitter splitter(resultSet, 1, 0);
+ const auto& result = splitter.Split();
+ const auto& issuesStr = result.Issues.ToString();
+ UNIT_ASSERT_C(!result.Success, issuesStr);
+ UNIT_ASSERT_VALUES_EQUAL(result.ResultSets.size(), 0);
+ UNIT_ASSERT_C(issuesStr.Contains("Can not write Row[0] with size: 41 bytes"), issuesStr);
+ }
+}
+
+} // NYq
diff --git a/ydb/core/yq/libs/common/ut/ya.make b/ydb/core/yq/libs/common/ut/ya.make
index 21401fe7d1..320ef17e66 100644
--- a/ydb/core/yq/libs/common/ut/ya.make
+++ b/ydb/core/yq/libs/common/ut/ya.make
@@ -14,7 +14,7 @@ ENDIF()
SRCS(
cache_ut.cpp
entity_id_ut.cpp
- rows_proto_splitter_ut.cpp
+ rows_proto_splitter_ut.cpp
)
PEERDIR(
diff --git a/ydb/core/yq/libs/common/ya.make b/ydb/core/yq/libs/common/ya.make
index ab214224a1..4622695700 100644
--- a/ydb/core/yq/libs/common/ya.make
+++ b/ydb/core/yq/libs/common/ya.make
@@ -1,24 +1,24 @@
OWNER(g:yq)
-
-LIBRARY()
-
-SRCS(
- database_token_builder.cpp
+
+LIBRARY()
+
+SRCS(
+ database_token_builder.cpp
entity_id.cpp
- rows_proto_splitter.cpp
-)
-
-PEERDIR(
+ rows_proto_splitter.cpp
+)
+
+PEERDIR(
library/cpp/monlib/dynamic_counters
ydb/core/yq/libs/control_plane_storage/events
ydb/core/yq/libs/events
ydb/library/yql/providers/common/structured_token
ydb/library/yql/public/issue
ydb/public/api/protos
-)
-
-YQL_LAST_ABI_VERSION()
-
+)
+
+YQL_LAST_ABI_VERSION()
+
END()
RECURSE_FOR_TESTS(
diff --git a/ydb/core/yq/libs/config/protos/common.proto b/ydb/core/yq/libs/config/protos/common.proto
index 2b0b850e3e..e86548cf3c 100644
--- a/ydb/core/yq/libs/config/protos/common.proto
+++ b/ydb/core/yq/libs/config/protos/common.proto
@@ -20,5 +20,5 @@ message TCommonConfig {
bool MdbTransformHost = 5;
string ObjectStorageEndpoint = 6;
string IdsPrefix = 7;
- uint64 MaxTasksPerOperation = 8;
+ uint64 MaxTasksPerOperation = 8;
}
diff --git a/ydb/core/yq/libs/config/protos/gateways.proto b/ydb/core/yq/libs/config/protos/gateways.proto
index d416454619..b953143764 100644
--- a/ydb/core/yq/libs/config/protos/gateways.proto
+++ b/ydb/core/yq/libs/config/protos/gateways.proto
@@ -15,5 +15,5 @@ message TGatewaysConfig {
NYql.TYdbGatewayConfig Ydb = 4;
NYql.TS3GatewayConfig S3 = 5;
NYql.TSolomonGatewayConfig Solomon = 6;
- NYql.THttpGatewayConfig HttpGateway = 7;
+ NYql.THttpGatewayConfig HttpGateway = 7;
}
diff --git a/ydb/core/yq/libs/config/protos/read_actors_factory.proto b/ydb/core/yq/libs/config/protos/read_actors_factory.proto
index c30111c13c..6addaab49f 100644
--- a/ydb/core/yq/libs/config/protos/read_actors_factory.proto
+++ b/ydb/core/yq/libs/config/protos/read_actors_factory.proto
@@ -1,17 +1,17 @@
-syntax = "proto3";
-option cc_enable_arenas = true;
-
-package NYq.NConfig;
-option java_package = "ru.yandex.kikimr.proto";
-
-import "ydb/library/yql/providers/s3/proto/retry_config.proto";
-
-////////////////////////////////////////////////////////////
-
-message TS3ReadActorFactoryConfig {
- NYql.NS3.TRetryConfig RetryConfig = 1;
-}
-
-message TReadActorsFactoryConfig {
- TS3ReadActorFactoryConfig S3ReadActorFactoryConfig = 1;
-}
+syntax = "proto3";
+option cc_enable_arenas = true;
+
+package NYq.NConfig;
+option java_package = "ru.yandex.kikimr.proto";
+
+import "ydb/library/yql/providers/s3/proto/retry_config.proto";
+
+////////////////////////////////////////////////////////////
+
+message TS3ReadActorFactoryConfig {
+ NYql.NS3.TRetryConfig RetryConfig = 1;
+}
+
+message TReadActorsFactoryConfig {
+ TS3ReadActorFactoryConfig S3ReadActorFactoryConfig = 1;
+}
diff --git a/ydb/core/yq/libs/config/protos/ya.make b/ydb/core/yq/libs/config/protos/ya.make
index 4ee4b0cd1a..72000d86fa 100644
--- a/ydb/core/yq/libs/config/protos/ya.make
+++ b/ydb/core/yq/libs/config/protos/ya.make
@@ -16,7 +16,7 @@ SRCS(
pinger.proto
private_api.proto
private_proxy.proto
- read_actors_factory.proto
+ read_actors_factory.proto
resource_manager.proto
storage.proto
test_connection.proto
@@ -27,7 +27,7 @@ SRCS(
PEERDIR(
ydb/library/folder_service/proto
ydb/library/yql/providers/common/proto
- ydb/library/yql/providers/s3/proto
+ ydb/library/yql/providers/s3/proto
)
EXCLUDE_TAGS(GO_PROTO)
diff --git a/ydb/core/yq/libs/config/protos/yq_config.proto b/ydb/core/yq/libs/config/protos/yq_config.proto
index 8bee641c10..841b816821 100644
--- a/ydb/core/yq/libs/config/protos/yq_config.proto
+++ b/ydb/core/yq/libs/config/protos/yq_config.proto
@@ -16,7 +16,7 @@ import "ydb/core/yq/libs/config/protos/pending_fetcher.proto";
import "ydb/core/yq/libs/config/protos/pinger.proto";
import "ydb/core/yq/libs/config/protos/private_api.proto";
import "ydb/core/yq/libs/config/protos/private_proxy.proto";
-import "ydb/core/yq/libs/config/protos/read_actors_factory.proto";
+import "ydb/core/yq/libs/config/protos/read_actors_factory.proto";
import "ydb/core/yq/libs/config/protos/resource_manager.proto";
import "ydb/core/yq/libs/config/protos/test_connection.proto";
import "ydb/core/yq/libs/config/protos/token_accessor.proto";
@@ -41,7 +41,7 @@ message TConfig {
TNodesManagerConfig NodesManager = 14;
TPendingFetcherConfig PendingFetcher = 15;
TPingerConfig Pinger = 16;
- bool EnableDynamicNameservice = 17;
+ bool EnableDynamicNameservice = 17;
TTestConnectionConfig TestConnection = 18;
- TReadActorsFactoryConfig ReadActorsFactoryConfig = 19;
+ TReadActorsFactoryConfig ReadActorsFactoryConfig = 19;
}
diff --git a/ydb/core/yq/libs/control_plane_proxy/control_plane_proxy.cpp b/ydb/core/yq/libs/control_plane_proxy/control_plane_proxy.cpp
index 33d031643b..53838351b4 100644
--- a/ydb/core/yq/libs/control_plane_proxy/control_plane_proxy.cpp
+++ b/ydb/core/yq/libs/control_plane_proxy/control_plane_proxy.cpp
@@ -104,7 +104,7 @@ class TRequestActor : public NActors::TActorBootstrapped<TRequestActor<TRequestP
public:
static constexpr char ActorName[] = "YQ_CONTROL_PLANE_PROXY_REQUEST_ACTOR";
-
+
explicit TRequestActor(const NConfig::TControlPlaneProxyConfig& config,
TActorId sender, ui32 cookie,
const TString& scope, const TString& folderId, TRequestProto&& requestProto,
@@ -331,7 +331,7 @@ public:
{
}
- static constexpr char ActorName[] = "YQ_CONTROL_PLANE_PROXY";
+ static constexpr char ActorName[] = "YQ_CONTROL_PLANE_PROXY";
void Bootstrap() {
CPP_LOG_D("Starting yandex query control plane proxy. Actor id: " << SelfId());
diff --git a/ydb/core/yq/libs/control_plane_storage/control_plane_storage_counters.cpp b/ydb/core/yq/libs/control_plane_storage/control_plane_storage_counters.cpp
index 17ed9a8aa7..8274a88589 100644
--- a/ydb/core/yq/libs/control_plane_storage/control_plane_storage_counters.cpp
+++ b/ydb/core/yq/libs/control_plane_storage/control_plane_storage_counters.cpp
@@ -1,11 +1,11 @@
-#include "control_plane_storage_counters.h"
-
-namespace NYq {
-
-TRequestCounters::TRequestCounters(const TString& name)
- : Name(name) {}
-
-void TRequestCounters::Register(const NMonitoring::TDynamicCounterPtr& counters) {
+#include "control_plane_storage_counters.h"
+
+namespace NYq {
+
+TRequestCounters::TRequestCounters(const TString& name)
+ : Name(name) {}
+
+void TRequestCounters::Register(const NMonitoring::TDynamicCounterPtr& counters) {
auto requestCounters = counters->GetSubgroup("request", Name);
InFly = requestCounters->GetCounter("InFly", false);
Ok = requestCounters->GetCounter("Ok", true);
@@ -13,52 +13,52 @@ void TRequestCounters::Register(const NMonitoring::TDynamicCounterPtr& counters)
Retry = requestCounters->GetCounter("Retry", true);
LatencyMs = requestCounters->GetHistogram("LatencyMs", GetLatencyHistogramBuckets());
Issues = requestCounters->GetSubgroup("subcomponent", "Issues");
-}
-
-NMonitoring::IHistogramCollectorPtr TRequestCounters::GetLatencyHistogramBuckets() {
- return NMonitoring::ExplicitHistogram({0, 1, 2, 5, 10, 20, 50, 100, 500, 1000, 2000, 5000, 10000, 30000, 50000, 500000});
-}
-
-
-TFinalStatusCounters::TFinalStatusCounters(const NMonitoring::TDynamicCounterPtr& counters) {
- auto subgroup = counters->GetSubgroup("subcomponent", "FinalStatus");
- Completed = subgroup->GetCounter("COMPLETED", true);
- AbortedBySystem = subgroup->GetCounter("ABORTED_BY_SYSTEM", true);
- AbortedByUser = subgroup->GetCounter("ABORTED_BY_USER", true);
- Failed = subgroup->GetCounter("FAILED", true);
- Paused = subgroup->GetCounter("PAUSED", true);
-}
-
-void TFinalStatusCounters::IncByStatus(YandexQuery::QueryMeta::ComputeStatus finalStatus) {
- switch (finalStatus) {
- case YandexQuery::QueryMeta::COMPLETED:
- Completed->Inc();
- break;
- case YandexQuery::QueryMeta::FAILED:
- Failed->Inc();
- break;
- case YandexQuery::QueryMeta::ABORTED_BY_SYSTEM:
- AbortedBySystem->Inc();
- break;
- case YandexQuery::QueryMeta::ABORTED_BY_USER:
- AbortedByUser->Inc();
- break;
- case YandexQuery::QueryMeta::PAUSED:
- Paused->Inc();
- break;
- case YandexQuery::QueryMeta::COMPUTE_STATUS_UNSPECIFIED:
- case YandexQuery::QueryMeta::STARTING:
- case YandexQuery::QueryMeta::ABORTING_BY_USER:
- case YandexQuery::QueryMeta::ABORTING_BY_SYSTEM:
- case YandexQuery::QueryMeta::RESUMING:
- case YandexQuery::QueryMeta::RUNNING:
- case YandexQuery::QueryMeta::COMPLETING:
- case YandexQuery::QueryMeta::FAILING:
- case YandexQuery::QueryMeta::PAUSING:
- break;
- default:
- Y_ENSURE(true, "Unexpected status: " << YandexQuery::QueryMeta_ComputeStatus_Name(finalStatus));
- }
-}
-
-} // NYq
+}
+
+NMonitoring::IHistogramCollectorPtr TRequestCounters::GetLatencyHistogramBuckets() {
+ return NMonitoring::ExplicitHistogram({0, 1, 2, 5, 10, 20, 50, 100, 500, 1000, 2000, 5000, 10000, 30000, 50000, 500000});
+}
+
+
+TFinalStatusCounters::TFinalStatusCounters(const NMonitoring::TDynamicCounterPtr& counters) {
+ auto subgroup = counters->GetSubgroup("subcomponent", "FinalStatus");
+ Completed = subgroup->GetCounter("COMPLETED", true);
+ AbortedBySystem = subgroup->GetCounter("ABORTED_BY_SYSTEM", true);
+ AbortedByUser = subgroup->GetCounter("ABORTED_BY_USER", true);
+ Failed = subgroup->GetCounter("FAILED", true);
+ Paused = subgroup->GetCounter("PAUSED", true);
+}
+
+void TFinalStatusCounters::IncByStatus(YandexQuery::QueryMeta::ComputeStatus finalStatus) {
+ switch (finalStatus) {
+ case YandexQuery::QueryMeta::COMPLETED:
+ Completed->Inc();
+ break;
+ case YandexQuery::QueryMeta::FAILED:
+ Failed->Inc();
+ break;
+ case YandexQuery::QueryMeta::ABORTED_BY_SYSTEM:
+ AbortedBySystem->Inc();
+ break;
+ case YandexQuery::QueryMeta::ABORTED_BY_USER:
+ AbortedByUser->Inc();
+ break;
+ case YandexQuery::QueryMeta::PAUSED:
+ Paused->Inc();
+ break;
+ case YandexQuery::QueryMeta::COMPUTE_STATUS_UNSPECIFIED:
+ case YandexQuery::QueryMeta::STARTING:
+ case YandexQuery::QueryMeta::ABORTING_BY_USER:
+ case YandexQuery::QueryMeta::ABORTING_BY_SYSTEM:
+ case YandexQuery::QueryMeta::RESUMING:
+ case YandexQuery::QueryMeta::RUNNING:
+ case YandexQuery::QueryMeta::COMPLETING:
+ case YandexQuery::QueryMeta::FAILING:
+ case YandexQuery::QueryMeta::PAUSING:
+ break;
+ default:
+ Y_ENSURE(true, "Unexpected status: " << YandexQuery::QueryMeta_ComputeStatus_Name(finalStatus));
+ }
+}
+
+} // NYq
diff --git a/ydb/core/yq/libs/control_plane_storage/control_plane_storage_counters.h b/ydb/core/yq/libs/control_plane_storage/control_plane_storage_counters.h
index bec7d629bf..58d3c56b27 100644
--- a/ydb/core/yq/libs/control_plane_storage/control_plane_storage_counters.h
+++ b/ydb/core/yq/libs/control_plane_storage/control_plane_storage_counters.h
@@ -1,40 +1,40 @@
-#pragma once
-
-#include <library/cpp/monlib/dynamic_counters/counters.h>
-#include <ydb/public/api/protos/yq.pb.h>
-
-namespace NYq {
-
-class TRequestCounters: public virtual TThrRefBase {
-public:
- const TString Name;
-
- NMonitoring::TDynamicCounters::TCounterPtr InFly;
- NMonitoring::TDynamicCounters::TCounterPtr Ok;
- NMonitoring::TDynamicCounters::TCounterPtr Error;
- NMonitoring::TDynamicCounters::TCounterPtr Retry;
- NMonitoring::THistogramPtr LatencyMs;
+#pragma once
+
+#include <library/cpp/monlib/dynamic_counters/counters.h>
+#include <ydb/public/api/protos/yq.pb.h>
+
+namespace NYq {
+
+class TRequestCounters: public virtual TThrRefBase {
+public:
+ const TString Name;
+
+ NMonitoring::TDynamicCounters::TCounterPtr InFly;
+ NMonitoring::TDynamicCounters::TCounterPtr Ok;
+ NMonitoring::TDynamicCounters::TCounterPtr Error;
+ NMonitoring::TDynamicCounters::TCounterPtr Retry;
+ NMonitoring::THistogramPtr LatencyMs;
NMonitoring::TDynamicCounterPtr Issues;
-
- explicit TRequestCounters(const TString& name);
-
- void Register(const NMonitoring::TDynamicCounterPtr& counters);
-
-private:
- static NMonitoring::IHistogramCollectorPtr GetLatencyHistogramBuckets();
-};
-
-class TFinalStatusCounters: public virtual TThrRefBase {
- NMonitoring::TDynamicCounters::TCounterPtr Completed;
- NMonitoring::TDynamicCounters::TCounterPtr AbortedBySystem;
- NMonitoring::TDynamicCounters::TCounterPtr AbortedByUser;
- NMonitoring::TDynamicCounters::TCounterPtr Failed;
- NMonitoring::TDynamicCounters::TCounterPtr Paused;
-
-public:
- TFinalStatusCounters(const NMonitoring::TDynamicCounterPtr& counters);
-
- void IncByStatus(YandexQuery::QueryMeta::ComputeStatus finalStatus);
-};
-
-} // NYq
+
+ explicit TRequestCounters(const TString& name);
+
+ void Register(const NMonitoring::TDynamicCounterPtr& counters);
+
+private:
+ static NMonitoring::IHistogramCollectorPtr GetLatencyHistogramBuckets();
+};
+
+class TFinalStatusCounters: public virtual TThrRefBase {
+ NMonitoring::TDynamicCounters::TCounterPtr Completed;
+ NMonitoring::TDynamicCounters::TCounterPtr AbortedBySystem;
+ NMonitoring::TDynamicCounters::TCounterPtr AbortedByUser;
+ NMonitoring::TDynamicCounters::TCounterPtr Failed;
+ NMonitoring::TDynamicCounters::TCounterPtr Paused;
+
+public:
+ TFinalStatusCounters(const NMonitoring::TDynamicCounterPtr& counters);
+
+ void IncByStatus(YandexQuery::QueryMeta::ComputeStatus finalStatus);
+};
+
+} // NYq
diff --git a/ydb/core/yq/libs/control_plane_storage/events/events.h b/ydb/core/yq/libs/control_plane_storage/events/events.h
index 1ef2323a62..8c98e790d7 100644
--- a/ydb/core/yq/libs/control_plane_storage/events/events.h
+++ b/ydb/core/yq/libs/control_plane_storage/events/events.h
@@ -27,12 +27,12 @@ struct TAuditDetails {
TString CloudId;
};
-struct TNodeInfo {
- ui32 NodeId;
- TString InstanceId;
- TString HostName;
-};
-
+struct TNodeInfo {
+ ui32 NodeId;
+ TString InstanceId;
+ TString HostName;
+};
+
struct TDebugItem {
TString Query;
NYdb::TParams Params;
@@ -108,7 +108,7 @@ struct TEvControlPlaneStorage {
EvDeleteQueryRequest,
EvDeleteQueryResponse,
EvControlQueryRequest,
- EvControlQueryResponse,
+ EvControlQueryResponse,
EvGetResultDataRequest,
EvGetResultDataResponse,
EvListJobsRequest,
@@ -141,10 +141,10 @@ struct TEvControlPlaneStorage {
EvGetTaskResponse,
EvPingTaskRequest,
EvPingTaskResponse,
- EvNodesHealthCheckRequest,
- EvNodesHealthCheckResponse,
- EvGetHealthNodesRequest,
- EvGetHealthNodesResponse,
+ EvNodesHealthCheckRequest,
+ EvNodesHealthCheckResponse,
+ EvGetHealthNodesRequest,
+ EvGetHealthNodesResponse,
EvEnd,
};
@@ -931,12 +931,12 @@ struct TEvControlPlaneStorage {
// internal messages
struct TEvWriteResultDataRequest : NActors::TEventLocal<TEvWriteResultDataRequest, EvWriteResultDataRequest> {
- explicit TEvWriteResultDataRequest(const TString& resultId,
+ explicit TEvWriteResultDataRequest(const TString& resultId,
const int32_t resultSetId,
const int64_t startRowId,
const TInstant& deadline,
const Ydb::ResultSet& resultSet)
- : ResultId(resultId)
+ : ResultId(resultId)
, ResultSetId(resultSetId)
, StartRowId(startRowId)
, Deadline(deadline)
@@ -944,7 +944,7 @@ struct TEvControlPlaneStorage {
{
}
- TString ResultId;
+ TString ResultId;
int32_t ResultSetId = 0;
int64_t StartRowId = 0;
TInstant Deadline;
@@ -957,23 +957,23 @@ struct TEvControlPlaneStorage {
{
}
- explicit TEvWriteResultDataResponse(
- const NYql::TIssues& issues,
- const ui64 requestId)
- : Issues(issues)
- , RequestId(requestId)
- {
- }
-
+ explicit TEvWriteResultDataResponse(
+ const NYql::TIssues& issues,
+ const ui64 requestId)
+ : Issues(issues)
+ , RequestId(requestId)
+ {
+ }
+
NYql::TIssues Issues;
- const ui64 RequestId = 0;
+ const ui64 RequestId = 0;
TDebugInfoPtr DebugInfo;
};
struct TEvGetTaskRequest : NActors::TEventLocal<TEvGetTaskRequest, EvGetTaskRequest> {
- explicit TEvGetTaskRequest(
- const TString& owner,
- const TString& hostName)
+ explicit TEvGetTaskRequest(
+ const TString& owner,
+ const TString& hostName)
: Owner(owner)
, HostName(hostName)
{
@@ -983,19 +983,19 @@ struct TEvControlPlaneStorage {
TString HostName;
};
- struct TTask {
- TString Scope;
- TString QueryId;
- YandexQuery::Query Query;
- YandexQuery::Internal::QueryInternal Internal;
- ui64 Generation = 0;
+ struct TTask {
+ TString Scope;
+ TString QueryId;
+ YandexQuery::Query Query;
+ YandexQuery::Internal::QueryInternal Internal;
+ ui64 Generation = 0;
TInstant Deadline;
- };
-
+ };
+
struct TEvGetTaskResponse : NActors::TEventLocal<TEvGetTaskResponse, EvGetTaskResponse> {
- explicit TEvGetTaskResponse(const TVector<TTask>& tasks, const TString& owner)
- : Tasks(tasks)
- , Owner(owner)
+ explicit TEvGetTaskResponse(const TVector<TTask>& tasks, const TString& owner)
+ : Tasks(tasks)
+ , Owner(owner)
{
}
@@ -1005,8 +1005,8 @@ struct TEvControlPlaneStorage {
}
NYql::TIssues Issues;
- TVector<TTask> Tasks;
- TString Owner;
+ TVector<TTask> Tasks;
+ TString Owner;
TDebugInfoPtr DebugInfo;
};
@@ -1026,17 +1026,17 @@ struct TEvControlPlaneStorage {
explicit TEvPingTaskRequest(const TString& scope, const TString& queryId, const TString& owner, const TInstant& deadline, const TString& resultId = "")
: Scope(scope)
, QueryId(queryId)
- , Owner(owner)
+ , Owner(owner)
, Deadline(deadline)
- , ResultId(resultId)
+ , ResultId(resultId)
{
}
- const TString Scope;
- const TString QueryId;
- const TString Owner;
+ const TString Scope;
+ const TString QueryId;
+ const TString Owner;
const TInstant Deadline;
- TString ResultId;
+ TString ResultId;
TMaybe<YandexQuery::QueryMeta::ComputeStatus> Status;
TMaybe<NYql::TIssues> Issues;
TMaybe<NYql::TIssues> TransientIssues;
@@ -1069,32 +1069,32 @@ struct TEvControlPlaneStorage {
NYql::TIssues Issues;
TDebugInfoPtr DebugInfo;
};
-
- struct TEvNodesHealthCheckRequest : NActors::TEventLocal<TEvNodesHealthCheckRequest, EvNodesHealthCheckRequest> {
- explicit TEvNodesHealthCheckRequest(
- Yq::Private::NodesHealthCheckRequest&& request)
- : Request(std::move(request))
- {}
- Yq::Private::NodesHealthCheckRequest Request;
- };
-
- struct TEvNodesHealthCheckResponse : NActors::TEventLocal<TEvNodesHealthCheckResponse, EvNodesHealthCheckResponse> {
-
- explicit TEvNodesHealthCheckResponse(
- const Yq::Private::NodesHealthCheckResult& record)
- : Record(record)
- {}
-
- explicit TEvNodesHealthCheckResponse(
- const NYql::TIssues& issues)
- : Issues(issues)
- {}
-
- Yq::Private::NodesHealthCheckResult Record;
- NYql::TIssues Issues;
+
+ struct TEvNodesHealthCheckRequest : NActors::TEventLocal<TEvNodesHealthCheckRequest, EvNodesHealthCheckRequest> {
+ explicit TEvNodesHealthCheckRequest(
+ Yq::Private::NodesHealthCheckRequest&& request)
+ : Request(std::move(request))
+ {}
+ Yq::Private::NodesHealthCheckRequest Request;
+ };
+
+ struct TEvNodesHealthCheckResponse : NActors::TEventLocal<TEvNodesHealthCheckResponse, EvNodesHealthCheckResponse> {
+
+ explicit TEvNodesHealthCheckResponse(
+ const Yq::Private::NodesHealthCheckResult& record)
+ : Record(record)
+ {}
+
+ explicit TEvNodesHealthCheckResponse(
+ const NYql::TIssues& issues)
+ : Issues(issues)
+ {}
+
+ Yq::Private::NodesHealthCheckResult Record;
+ NYql::TIssues Issues;
TDebugInfoPtr DebugInfo;
- };
-
+ };
+
};
}
diff --git a/ydb/core/yq/libs/control_plane_storage/in_memory_control_plane_storage.cpp b/ydb/core/yq/libs/control_plane_storage/in_memory_control_plane_storage.cpp
index 8f4d4931be..783b924238 100644
--- a/ydb/core/yq/libs/control_plane_storage/in_memory_control_plane_storage.cpp
+++ b/ydb/core/yq/libs/control_plane_storage/in_memory_control_plane_storage.cpp
@@ -318,7 +318,7 @@ private:
{
SendEmptyResponse<
TEvControlPlaneStorage::TEvNodesHealthCheckRequest::TPtr,
- Yq::Private::NodesHealthCheckResult,
+ Yq::Private::NodesHealthCheckResult,
TEvControlPlaneStorage::TEvNodesHealthCheckResponse>(ev, "NodesHealthCheckRequest");
}
diff --git a/ydb/core/yq/libs/control_plane_storage/internal/nodes_health_check.cpp b/ydb/core/yq/libs/control_plane_storage/internal/nodes_health_check.cpp
index 701fa89447..eb25dac4b1 100644
--- a/ydb/core/yq/libs/control_plane_storage/internal/nodes_health_check.cpp
+++ b/ydb/core/yq/libs/control_plane_storage/internal/nodes_health_check.cpp
@@ -1,17 +1,17 @@
-#include "utils.h"
-
+#include "utils.h"
+
#include <ydb/core/yq/libs/db_schema/db_schema.h>
-namespace NYq {
-
-void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvNodesHealthCheckRequest::TPtr& ev)
-{
- TInstant startTime = TInstant::Now();
- TRequestCountersPtr requestCounters = Counters.Requests[RT_NODES_HEALTH_CHECK];
- requestCounters->InFly->Inc();
-
- const auto& request = ev->Get()->Request;
- const TString tenant = request.tenant();
+namespace NYq {
+
+void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvNodesHealthCheckRequest::TPtr& ev)
+{
+ TInstant startTime = TInstant::Now();
+ TRequestCountersPtr requestCounters = Counters.Requests[RT_NODES_HEALTH_CHECK];
+ requestCounters->InFly->Inc();
+
+ const auto& request = ev->Get()->Request;
+ const TString tenant = request.tenant();
const auto& node = request.node();
const ui32 nodeId = node.node_id();
const TString instanceId = node.instance_id();
@@ -19,23 +19,23 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvNodesHealth
const ui64 activeWorkers = node.active_workers();
const ui64 memoryLimit = node.memory_limit();
const ui64 memoryAllocated = node.memory_allocated();
- const ui32 icPort = node.interconnect_port();
- const TString nodeAddress = node.node_address();
- const auto ttl = TDuration::Seconds(5);
- const auto deadline = startTime + ttl * 3;
-
- CPS_LOG_T("NodesHealthCheck: " << tenant << " " << nodeId << " " << instanceId << " " << hostName << " " << node.node_address() << ":" << node.interconnect_port());
-
- NYql::TIssues issues = ValidateNodesHealthCheck(tenant, instanceId, hostName);
- if (issues) {
- CPS_LOG_D("NodesHealthCheckRequest, validation failed: " << issues.ToString());
- const TDuration delta = TInstant::Now() - startTime;
- SendResponseIssues<TEvControlPlaneStorage::TEvNodesHealthCheckResponse>(ev->Sender, issues, ev->Cookie, delta, requestCounters);
- LWPROBE(NodesHealthCheckRequest, "", 0, "", "", delta, false);
- return;
- }
-
- std::shared_ptr<Yq::Private::NodesHealthCheckResult> response = std::make_shared<Yq::Private::NodesHealthCheckResult>();
+ const ui32 icPort = node.interconnect_port();
+ const TString nodeAddress = node.node_address();
+ const auto ttl = TDuration::Seconds(5);
+ const auto deadline = startTime + ttl * 3;
+
+ CPS_LOG_T("NodesHealthCheck: " << tenant << " " << nodeId << " " << instanceId << " " << hostName << " " << node.node_address() << ":" << node.interconnect_port());
+
+ NYql::TIssues issues = ValidateNodesHealthCheck(tenant, instanceId, hostName);
+ if (issues) {
+ CPS_LOG_D("NodesHealthCheckRequest, validation failed: " << issues.ToString());
+ const TDuration delta = TInstant::Now() - startTime;
+ SendResponseIssues<TEvControlPlaneStorage::TEvNodesHealthCheckResponse>(ev->Sender, issues, ev->Cookie, delta, requestCounters);
+ LWPROBE(NodesHealthCheckRequest, "", 0, "", "", delta, false);
+ return;
+ }
+
+ std::shared_ptr<Yq::Private::NodesHealthCheckResult> response = std::make_shared<Yq::Private::NodesHealthCheckResult>();
{
auto* node = response->add_nodes();
node->set_node_id(nodeId);
@@ -44,22 +44,22 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvNodesHealth
node->set_active_workers(activeWorkers);
node->set_memory_limit(memoryLimit);
node->set_memory_allocated(memoryAllocated);
- node->set_node_address(nodeAddress);
+ node->set_node_address(nodeAddress);
}
-
- TSqlQueryBuilder readQueryBuilder(YdbConnection->TablePathPrefix, "NodesHealthCheck(read)");
+
+ TSqlQueryBuilder readQueryBuilder(YdbConnection->TablePathPrefix, "NodesHealthCheck(read)");
readQueryBuilder.AddTimestamp("now", TInstant::Now());
readQueryBuilder.AddString("tenant", tenant);
readQueryBuilder.AddText(
- "SELECT `" NODE_ID_COLUMN_NAME "`, `" INSTANCE_ID_COLUMN_NAME "`, `" HOST_NAME_COLUMN_NAME "`, `" ACTIVE_WORKERS_COLUMN_NAME"`, `" MEMORY_LIMIT_COLUMN_NAME"`, "
- "`" MEMORY_ALLOCATED_COLUMN_NAME"`, `" INTERCONNECT_PORT_COLUMN_NAME "`, `" NODE_ADDRESS_COLUMN_NAME "` FROM `" NODES_TABLE_NAME "`\n"
- "WHERE `" TENANT_COLUMN_NAME"` = $tenant AND `" EXPIRE_AT_COLUMN_NAME "` >= $now;\n"
+ "SELECT `" NODE_ID_COLUMN_NAME "`, `" INSTANCE_ID_COLUMN_NAME "`, `" HOST_NAME_COLUMN_NAME "`, `" ACTIVE_WORKERS_COLUMN_NAME"`, `" MEMORY_LIMIT_COLUMN_NAME"`, "
+ "`" MEMORY_ALLOCATED_COLUMN_NAME"`, `" INTERCONNECT_PORT_COLUMN_NAME "`, `" NODE_ADDRESS_COLUMN_NAME "` FROM `" NODES_TABLE_NAME "`\n"
+ "WHERE `" TENANT_COLUMN_NAME"` = $tenant AND `" EXPIRE_AT_COLUMN_NAME "` >= $now;\n"
);
-
- auto prepareParams = [=](const TVector<TResultSet>& resultSets) {
- for (const auto& resultSet : resultSets) {
- TResultSetParser parser(resultSet);
- while (parser.TryNextRow()) {
+
+ auto prepareParams = [=](const TVector<TResultSet>& resultSets) {
+ for (const auto& resultSet : resultSets) {
+ TResultSetParser parser(resultSet);
+ while (parser.TryNextRow()) {
auto nid = *parser.ColumnParser(NODE_ID_COLUMN_NAME).GetOptionalUint32();
if (nid != nodeId) {
auto* node = response->add_nodes();
@@ -69,13 +69,13 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvNodesHealth
node->set_active_workers(*parser.ColumnParser(ACTIVE_WORKERS_COLUMN_NAME).GetOptionalUint64());
node->set_memory_limit(*parser.ColumnParser(MEMORY_LIMIT_COLUMN_NAME).GetOptionalUint64());
node->set_memory_allocated(*parser.ColumnParser(MEMORY_ALLOCATED_COLUMN_NAME).GetOptionalUint64());
- node->set_interconnect_port(parser.ColumnParser(INTERCONNECT_PORT_COLUMN_NAME).GetOptionalUint32().GetOrElse(0));
- node->set_node_address(*parser.ColumnParser(NODE_ADDRESS_COLUMN_NAME).GetOptionalString());
+ node->set_interconnect_port(parser.ColumnParser(INTERCONNECT_PORT_COLUMN_NAME).GetOptionalUint32().GetOrElse(0));
+ node->set_node_address(*parser.ColumnParser(NODE_ADDRESS_COLUMN_NAME).GetOptionalString());
}
- }
- }
+ }
+ }
- TSqlQueryBuilder writeQueryBuilder(YdbConnection->TablePathPrefix, "NodesHealthCheck(write)");
+ TSqlQueryBuilder writeQueryBuilder(YdbConnection->TablePathPrefix, "NodesHealthCheck(write)");
writeQueryBuilder.AddString("tenant", tenant);
writeQueryBuilder.AddUint32("node_id", nodeId);
writeQueryBuilder.AddString("instance_id", instanceId);
@@ -94,27 +94,27 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvNodesHealth
);
const auto writeQuery = writeQueryBuilder.Build();
return std::make_pair(writeQuery.Sql, writeQuery.Params);
- };
-
+ };
+
const auto readQuery = readQueryBuilder.Build();
auto debugInfo = Config.Proto.GetEnableDebugMode() ? std::make_shared<TDebugInfo>() : TDebugInfoPtr{};
TAsyncStatus status = ReadModifyWrite(NActors::TActivationContext::ActorSystem(), readQuery.Sql, readQuery.Params, prepareParams, requestCounters, debugInfo);
- auto prepare = [response] { return *response; };
- auto success = SendResponse<TEvControlPlaneStorage::TEvNodesHealthCheckResponse, Yq::Private::NodesHealthCheckResult>(
- "NodesHealthCheckRequest",
- NActors::TActivationContext::ActorSystem(),
- status,
- SelfId(),
- ev,
- startTime,
- requestCounters,
- prepare,
- debugInfo);
-
- success.Apply([=](const auto& future) {
- TDuration delta = TInstant::Now() - startTime;
- LWPROBE(NodesHealthCheckRequest, tenant, nodeId, instanceId, hostName, delta, future.GetValue());
- });
-}
-
-} // NYq
+ auto prepare = [response] { return *response; };
+ auto success = SendResponse<TEvControlPlaneStorage::TEvNodesHealthCheckResponse, Yq::Private::NodesHealthCheckResult>(
+ "NodesHealthCheckRequest",
+ NActors::TActivationContext::ActorSystem(),
+ status,
+ SelfId(),
+ ev,
+ startTime,
+ requestCounters,
+ prepare,
+ debugInfo);
+
+ success.Apply([=](const auto& future) {
+ TDuration delta = TInstant::Now() - startTime;
+ LWPROBE(NodesHealthCheckRequest, tenant, nodeId, instanceId, hostName, delta, future.GetValue());
+ });
+}
+
+} // NYq
diff --git a/ydb/core/yq/libs/control_plane_storage/internal/response_tasks.cpp b/ydb/core/yq/libs/control_plane_storage/internal/response_tasks.cpp
index a3feec3252..1d64d9ed58 100644
--- a/ydb/core/yq/libs/control_plane_storage/internal/response_tasks.cpp
+++ b/ydb/core/yq/libs/control_plane_storage/internal/response_tasks.cpp
@@ -1,47 +1,47 @@
-#include "response_tasks.h"
-
-namespace NYq {
-
-void TResponseTasks::AddTaskNonBlocking(const TString& key, const TEvControlPlaneStorage::TTask& task) {
- Tasks[key] = task;
-}
-
-void TResponseTasks::AddTaskBlocking(const TString& key, const TEvControlPlaneStorage::TTask& task) {
- with_lock (Mutex) {
- Tasks[key] = task;
- }
-}
-
-void TResponseTasks::SafeEraseTaskNonBlocking(const TString& key) {
- if (auto it = Tasks.find(key); it != Tasks.end())
- Tasks.erase(it);
-}
-
-void TResponseTasks::SafeEraseTaskBlocking(const TString& key) {
- with_lock (Mutex) {
- if (auto it = Tasks.find(key); it != Tasks.end())
- Tasks.erase(it);
- }
-}
-
-bool TResponseTasks::EmptyNonBlocking() {
- return Tasks.empty();
-}
-
-bool TResponseTasks::EmptyBlocking() {
- with_lock (Mutex) {
- return Tasks.empty();
- }
-}
-
-const THashMap<TString, TEvControlPlaneStorage::TTask>& TResponseTasks::GetTasksNonBlocking() {
- return Tasks;
-}
-
-const THashMap<TString, TEvControlPlaneStorage::TTask>& TResponseTasks::GetTasksBlocking() {
- with_lock (Mutex) {
- return Tasks;
- }
-}
-
-} //NYq \ No newline at end of file
+#include "response_tasks.h"
+
+namespace NYq {
+
+void TResponseTasks::AddTaskNonBlocking(const TString& key, const TEvControlPlaneStorage::TTask& task) {
+ Tasks[key] = task;
+}
+
+void TResponseTasks::AddTaskBlocking(const TString& key, const TEvControlPlaneStorage::TTask& task) {
+ with_lock (Mutex) {
+ Tasks[key] = task;
+ }
+}
+
+void TResponseTasks::SafeEraseTaskNonBlocking(const TString& key) {
+ if (auto it = Tasks.find(key); it != Tasks.end())
+ Tasks.erase(it);
+}
+
+void TResponseTasks::SafeEraseTaskBlocking(const TString& key) {
+ with_lock (Mutex) {
+ if (auto it = Tasks.find(key); it != Tasks.end())
+ Tasks.erase(it);
+ }
+}
+
+bool TResponseTasks::EmptyNonBlocking() {
+ return Tasks.empty();
+}
+
+bool TResponseTasks::EmptyBlocking() {
+ with_lock (Mutex) {
+ return Tasks.empty();
+ }
+}
+
+const THashMap<TString, TEvControlPlaneStorage::TTask>& TResponseTasks::GetTasksNonBlocking() {
+ return Tasks;
+}
+
+const THashMap<TString, TEvControlPlaneStorage::TTask>& TResponseTasks::GetTasksBlocking() {
+ with_lock (Mutex) {
+ return Tasks;
+ }
+}
+
+} //NYq \ No newline at end of file
diff --git a/ydb/core/yq/libs/control_plane_storage/internal/response_tasks.h b/ydb/core/yq/libs/control_plane_storage/internal/response_tasks.h
index 538c333967..1ab660d43d 100644
--- a/ydb/core/yq/libs/control_plane_storage/internal/response_tasks.h
+++ b/ydb/core/yq/libs/control_plane_storage/internal/response_tasks.h
@@ -1,31 +1,31 @@
#pragma once
-#include <util/system/mutex.h>
-
+#include <util/system/mutex.h>
+
#include <ydb/core/yq/libs/control_plane_storage/events/events.h>
-namespace NYq {
-
-class TResponseTasks {
-public:
- void AddTaskNonBlocking(const TString& key, const TEvControlPlaneStorage::TTask& task);
-
- void AddTaskBlocking(const TString& key, const TEvControlPlaneStorage::TTask& task);
-
- void SafeEraseTaskNonBlocking(const TString& key);
-
- void SafeEraseTaskBlocking(const TString& key);
-
- bool EmptyNonBlocking();
-
- bool EmptyBlocking();
-
- const THashMap<TString, TEvControlPlaneStorage::TTask>& GetTasksNonBlocking();
-
- const THashMap<TString, TEvControlPlaneStorage::TTask>& GetTasksBlocking();
-
-private:
- TMutex Mutex;
- THashMap<TString, TEvControlPlaneStorage::TTask> Tasks;
-};
-
+namespace NYq {
+
+class TResponseTasks {
+public:
+ void AddTaskNonBlocking(const TString& key, const TEvControlPlaneStorage::TTask& task);
+
+ void AddTaskBlocking(const TString& key, const TEvControlPlaneStorage::TTask& task);
+
+ void SafeEraseTaskNonBlocking(const TString& key);
+
+ void SafeEraseTaskBlocking(const TString& key);
+
+ bool EmptyNonBlocking();
+
+ bool EmptyBlocking();
+
+ const THashMap<TString, TEvControlPlaneStorage::TTask>& GetTasksNonBlocking();
+
+ const THashMap<TString, TEvControlPlaneStorage::TTask>& GetTasksBlocking();
+
+private:
+ TMutex Mutex;
+ THashMap<TString, TEvControlPlaneStorage::TTask> Tasks;
+};
+
} //NYq
diff --git a/ydb/core/yq/libs/control_plane_storage/internal/task_get.cpp b/ydb/core/yq/libs/control_plane_storage/internal/task_get.cpp
index bad099564d..42b3c1a1ac 100644
--- a/ydb/core/yq/libs/control_plane_storage/internal/task_get.cpp
+++ b/ydb/core/yq/libs/control_plane_storage/internal/task_get.cpp
@@ -1,36 +1,36 @@
-#include "utils.h"
-
-#include <random>
-
+#include "utils.h"
+
+#include <random>
+
#include <util/datetime/base.h>
#include <ydb/core/yq/libs/control_plane_storage/schema.h>
#include <ydb/core/yq/libs/db_schema/db_schema.h>
-namespace NYq {
-
-namespace {
-
-struct TTaskInternal {
- TEvControlPlaneStorage::TTask Task;
- ui64 RetryCounter = 0;
- TInstant RetryCounterUpdatedAt = TInstant::Zero();
- bool ShouldAbortTask = false;
- TString TablePathPrefix;
- TString Owner;
- TString HostName;
- TMaybe<YandexQuery::Job> Job;
+namespace NYq {
+
+namespace {
+
+struct TTaskInternal {
+ TEvControlPlaneStorage::TTask Task;
+ ui64 RetryCounter = 0;
+ TInstant RetryCounterUpdatedAt = TInstant::Zero();
+ bool ShouldAbortTask = false;
+ TString TablePathPrefix;
+ TString Owner;
+ TString HostName;
+ TMaybe<YandexQuery::Job> Job;
TInstant Deadline;
-};
-
-std::pair<TString, NYdb::TParams> MakeSql(const TTaskInternal& taskInternal) {
- const auto& task = taskInternal.Task;
- const auto& retryCounter = taskInternal.RetryCounter;
- const auto& retryCounterUpdatedAt = taskInternal.RetryCounterUpdatedAt;
- const auto& hostName = taskInternal.HostName;
- const auto& owner = taskInternal.Owner;
-
- TSqlQueryBuilder queryBuilder(taskInternal.TablePathPrefix, "GetTask(write)");
+};
+
+std::pair<TString, NYdb::TParams> MakeSql(const TTaskInternal& taskInternal) {
+ const auto& task = taskInternal.Task;
+ const auto& retryCounter = taskInternal.RetryCounter;
+ const auto& retryCounterUpdatedAt = taskInternal.RetryCounterUpdatedAt;
+ const auto& hostName = taskInternal.HostName;
+ const auto& owner = taskInternal.Owner;
+
+ TSqlQueryBuilder queryBuilder(taskInternal.TablePathPrefix, "GetTask(write)");
queryBuilder.AddString("scope", task.Scope);
queryBuilder.AddString("query_id", task.QueryId);
queryBuilder.AddString("query", task.Query.SerializeAsString());
@@ -44,213 +44,213 @@ std::pair<TString, NYdb::TParams> MakeSql(const TTaskInternal& taskInternal) {
// update queries
queryBuilder.AddText(
- "UPDATE `" QUERIES_TABLE_NAME "` SET `" GENERATION_COLUMN_NAME "` = $generation, `" QUERY_COLUMN_NAME "` = $query, `" INTERNAL_COLUMN_NAME "` = $internal\n"
+ "UPDATE `" QUERIES_TABLE_NAME "` SET `" GENERATION_COLUMN_NAME "` = $generation, `" QUERY_COLUMN_NAME "` = $query, `" INTERNAL_COLUMN_NAME "` = $internal\n"
"WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id;\n"
);
// update pending small
queryBuilder.AddText(
- "UPDATE `" PENDING_SMALL_TABLE_NAME "` SET `" LAST_SEEN_AT_COLUMN_NAME "` = $now,\n"
- "`" RETRY_COUNTER_COLUMN_NAME "` = $retry_counter, `" RETRY_COUNTER_UPDATE_COLUMN_NAME "` = $retry_counter_update_time, `" IS_RESIGN_QUERY_COLUMN_NAME "` = false,\n"
- "`" HOST_NAME_COLUMN_NAME "` = $host, `" OWNER_COLUMN_NAME "` = $owner\n"
+ "UPDATE `" PENDING_SMALL_TABLE_NAME "` SET `" LAST_SEEN_AT_COLUMN_NAME "` = $now,\n"
+ "`" RETRY_COUNTER_COLUMN_NAME "` = $retry_counter, `" RETRY_COUNTER_UPDATE_COLUMN_NAME "` = $retry_counter_update_time, `" IS_RESIGN_QUERY_COLUMN_NAME "` = false,\n"
+ "`" HOST_NAME_COLUMN_NAME "` = $host, `" OWNER_COLUMN_NAME "` = $owner\n"
"WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id;\n"
);
-
+
const auto query = queryBuilder.Build();
return std::make_pair(query.Sql, query.Params);
-}
-
-} // namespace
-
-std::tuple<TString, NYdb::TParams, std::function<std::pair<TString, NYdb::TParams>(const TVector<NYdb::TResultSet>&)>> MakeGetTaskUpdateQuery(
- const TTaskInternal& taskInternal,
- const std::shared_ptr<TResponseTasks>& responseTasks,
+}
+
+} // namespace
+
+std::tuple<TString, NYdb::TParams, std::function<std::pair<TString, NYdb::TParams>(const TVector<NYdb::TResultSet>&)>> MakeGetTaskUpdateQuery(
+ const TTaskInternal& taskInternal,
+ const std::shared_ptr<TResponseTasks>& responseTasks,
const TInstant& taskLeaseTimestamp,
bool disableCurrentIam,
const TDuration& automaticQueriesTtl,
const TDuration& resultSetsTtl)
-{
- const auto& task = taskInternal.Task;
-
- TSqlQueryBuilder queryBuilder(taskInternal.TablePathPrefix, "GetTask(read)");
+{
+ const auto& task = taskInternal.Task;
+
+ TSqlQueryBuilder queryBuilder(taskInternal.TablePathPrefix, "GetTask(read)");
queryBuilder.AddString("scope", task.Scope);
queryBuilder.AddString("query_id", task.QueryId);
queryBuilder.AddTimestamp("from", taskLeaseTimestamp);
-
+
queryBuilder.AddText(
- "SELECT `" GENERATION_COLUMN_NAME "`, `" INTERNAL_COLUMN_NAME "`, `" QUERY_COLUMN_NAME "`\n"
- "FROM `" QUERIES_TABLE_NAME "`\n"
- "WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id;\n"
- "SELECT `" LAST_SEEN_AT_COLUMN_NAME "`\n"
- "FROM `" PENDING_SMALL_TABLE_NAME "`\n"
- "WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id AND `" LAST_SEEN_AT_COLUMN_NAME "` < $from;\n"
+ "SELECT `" GENERATION_COLUMN_NAME "`, `" INTERNAL_COLUMN_NAME "`, `" QUERY_COLUMN_NAME "`\n"
+ "FROM `" QUERIES_TABLE_NAME "`\n"
+ "WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id;\n"
+ "SELECT `" LAST_SEEN_AT_COLUMN_NAME "`\n"
+ "FROM `" PENDING_SMALL_TABLE_NAME "`\n"
+ "WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id AND `" LAST_SEEN_AT_COLUMN_NAME "` < $from;\n"
);
-
- auto prepareParams = [=, taskInternal=taskInternal, responseTasks=responseTasks](const TVector<TResultSet>& resultSets) mutable {
- auto& task = taskInternal.Task;
- const auto shouldAbortTask = taskInternal.ShouldAbortTask;
- constexpr size_t expectedResultSetsSize = 2;
-
- if (resultSets.size() != expectedResultSetsSize || !resultSets[1].RowsCount()) {
- return std::make_pair(TString{}, TParamsBuilder{}.Build());
- }
-
- {
- TResultSetParser parser(resultSets[0]);
- while (parser.TryNextRow()) {
- task.Generation = parser.ColumnParser(GENERATION_COLUMN_NAME).GetOptionalUint64().GetOrElse(0) + 1;
-
- if (!task.Query.ParseFromString(*parser.ColumnParser(QUERY_COLUMN_NAME).GetOptionalString())) {
- throw TControlPlaneStorageException(TIssuesIds::INTERNAL_ERROR) << "Error parsing proto message for query. Please contact internal support";
- }
+
+ auto prepareParams = [=, taskInternal=taskInternal, responseTasks=responseTasks](const TVector<TResultSet>& resultSets) mutable {
+ auto& task = taskInternal.Task;
+ const auto shouldAbortTask = taskInternal.ShouldAbortTask;
+ constexpr size_t expectedResultSetsSize = 2;
+
+ if (resultSets.size() != expectedResultSetsSize || !resultSets[1].RowsCount()) {
+ return std::make_pair(TString{}, TParamsBuilder{}.Build());
+ }
+
+ {
+ TResultSetParser parser(resultSets[0]);
+ while (parser.TryNextRow()) {
+ task.Generation = parser.ColumnParser(GENERATION_COLUMN_NAME).GetOptionalUint64().GetOrElse(0) + 1;
+
+ if (!task.Query.ParseFromString(*parser.ColumnParser(QUERY_COLUMN_NAME).GetOptionalString())) {
+ throw TControlPlaneStorageException(TIssuesIds::INTERNAL_ERROR) << "Error parsing proto message for query. Please contact internal support";
+ }
const TInstant deadline = TInstant::Now() + (task.Query.content().automatic() ? std::min(automaticQueriesTtl, resultSetsTtl) : resultSetsTtl);
task.Deadline = deadline;
- if (!task.Internal.ParseFromString(*parser.ColumnParser(INTERNAL_COLUMN_NAME).GetOptionalString())) {
- throw TControlPlaneStorageException(TIssuesIds::INTERNAL_ERROR) << "Error parsing proto message for query internal. Please contact internal support";
- }
-
+ if (!task.Internal.ParseFromString(*parser.ColumnParser(INTERNAL_COLUMN_NAME).GetOptionalString())) {
+ throw TControlPlaneStorageException(TIssuesIds::INTERNAL_ERROR) << "Error parsing proto message for query internal. Please contact internal support";
+ }
+
if (disableCurrentIam) {
task.Internal.clear_token();
}
- }
- }
-
- if (shouldAbortTask) {
- Ydb::Issue::IssueMessage abortIssueMsg;
- abortIssueMsg.set_message("Query was aborted by system due to high failure rate");
- abortIssueMsg.set_severity(NYql::TSeverityIds::S_ERROR);
- *task.Query.add_issue() = abortIssueMsg;
- task.Query.mutable_meta()->set_status(YandexQuery::QueryMeta::ABORTING_BY_SYSTEM);
- }
-
- responseTasks->AddTaskBlocking(task.QueryId, task);
-
- return MakeSql(taskInternal);
- };
-
+ }
+ }
+
+ if (shouldAbortTask) {
+ Ydb::Issue::IssueMessage abortIssueMsg;
+ abortIssueMsg.set_message("Query was aborted by system due to high failure rate");
+ abortIssueMsg.set_severity(NYql::TSeverityIds::S_ERROR);
+ *task.Query.add_issue() = abortIssueMsg;
+ task.Query.mutable_meta()->set_status(YandexQuery::QueryMeta::ABORTING_BY_SYSTEM);
+ }
+
+ responseTasks->AddTaskBlocking(task.QueryId, task);
+
+ return MakeSql(taskInternal);
+ };
+
const auto query = queryBuilder.Build();
return std::make_tuple(query.Sql, query.Params, prepareParams);
-}
-
-void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvGetTaskRequest::TPtr& ev)
-{
- TInstant startTime = TInstant::Now();
- TRequestCountersPtr requestCounters = Counters.Requests[RT_GET_TASK];
- requestCounters->InFly->Inc();
-
- TEvControlPlaneStorage::TEvGetTaskRequest& request = *ev->Get();
- const TString owner = request.Owner;
- const TString hostName = request.HostName;
+}
+
+void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvGetTaskRequest::TPtr& ev)
+{
+ TInstant startTime = TInstant::Now();
+ TRequestCountersPtr requestCounters = Counters.Requests[RT_GET_TASK];
+ requestCounters->InFly->Inc();
+
+ TEvControlPlaneStorage::TEvGetTaskRequest& request = *ev->Get();
+ const TString owner = request.Owner;
+ const TString hostName = request.HostName;
const ui64 tasksBatchSize = Config.Proto.GetTasksBatchSize();
const ui64 numTasksProportion = Config.Proto.GetNumTasksProportion();
-
- CPS_LOG_T("GetTaskRequest: " << owner << " " << hostName);
-
- NYql::TIssues issues = ValidateGetTask(owner, hostName);
- if (issues) {
- CPS_LOG_D("GetTaskRequest, validation failed: " << owner << " " << hostName << " " << issues.ToString());
- const TDuration delta = TInstant::Now() - startTime;
- SendResponseIssues<TEvControlPlaneStorage::TEvGetTaskResponse>(ev->Sender, issues, ev->Cookie, delta, requestCounters);
- LWPROBE(GetTaskRequest, owner, hostName, delta, false);
- return;
- }
-
+
+ CPS_LOG_T("GetTaskRequest: " << owner << " " << hostName);
+
+ NYql::TIssues issues = ValidateGetTask(owner, hostName);
+ if (issues) {
+ CPS_LOG_D("GetTaskRequest, validation failed: " << owner << " " << hostName << " " << issues.ToString());
+ const TDuration delta = TInstant::Now() - startTime;
+ SendResponseIssues<TEvControlPlaneStorage::TEvGetTaskResponse>(ev->Sender, issues, ev->Cookie, delta, requestCounters);
+ LWPROBE(GetTaskRequest, owner, hostName, delta, false);
+ return;
+ }
+
auto debugInfo = Config.Proto.GetEnableDebugMode() ? std::make_shared<TDebugInfo>() : TDebugInfoPtr{};
-
- auto response = std::make_shared<std::tuple<TVector<TEvControlPlaneStorage::TTask>, TString>>(); //tasks, owner
-
- TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "GetTask(read stale ro)");
+
+ auto response = std::make_shared<std::tuple<TVector<TEvControlPlaneStorage::TTask>, TString>>(); //tasks, owner
+
+ TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "GetTask(read stale ro)");
const auto taskLeaseTimestamp = TInstant::Now() - Config.TaskLeaseTtl;
queryBuilder.AddTimestamp("from", taskLeaseTimestamp);
queryBuilder.AddUint64("tasks_limit", tasksBatchSize);
queryBuilder.AddText(
- "SELECT `" SCOPE_COLUMN_NAME "`, `" QUERY_ID_COLUMN_NAME "`,\n"
- "`" RETRY_COUNTER_COLUMN_NAME "`, `" RETRY_COUNTER_UPDATE_COLUMN_NAME "`, `" QUERY_TYPE_COLUMN_NAME "`, `" IS_RESIGN_QUERY_COLUMN_NAME "`\n"
- "FROM `" PENDING_SMALL_TABLE_NAME "`\n"
- "WHERE `" LAST_SEEN_AT_COLUMN_NAME "` < $from ORDER BY `" QUERY_ID_COLUMN_NAME "` DESC LIMIT $tasks_limit;\n"
+ "SELECT `" SCOPE_COLUMN_NAME "`, `" QUERY_ID_COLUMN_NAME "`,\n"
+ "`" RETRY_COUNTER_COLUMN_NAME "`, `" RETRY_COUNTER_UPDATE_COLUMN_NAME "`, `" QUERY_TYPE_COLUMN_NAME "`, `" IS_RESIGN_QUERY_COLUMN_NAME "`\n"
+ "FROM `" PENDING_SMALL_TABLE_NAME "`\n"
+ "WHERE `" LAST_SEEN_AT_COLUMN_NAME "` < $from ORDER BY `" QUERY_ID_COLUMN_NAME "` DESC LIMIT $tasks_limit;\n"
);
-
- auto responseTasks = std::make_shared<TResponseTasks>();
- auto prepareParams = [=, responseTasks=responseTasks](const TVector<TResultSet>& resultSets) mutable {
- TVector<TTaskInternal> tasks;
- TVector<TPickTaskParams> pickTaskParams;
- const auto now = TInstant::Now();
- if (resultSets.empty() || !resultSets.back().RowsCount()) {
+
+ auto responseTasks = std::make_shared<TResponseTasks>();
+ auto prepareParams = [=, responseTasks=responseTasks](const TVector<TResultSet>& resultSets) mutable {
+ TVector<TTaskInternal> tasks;
+ TVector<TPickTaskParams> pickTaskParams;
+ const auto now = TInstant::Now();
+ if (resultSets.empty() || !resultSets.back().RowsCount()) {
return pickTaskParams;
- }
-
- TResultSetParser parser(resultSets.back());
- while (parser.TryNextRow()) {
- TTaskInternal& taskInternal = tasks.emplace_back();
- taskInternal.Owner = owner;
- taskInternal.HostName = hostName;
- taskInternal.TablePathPrefix = YdbConnection->TablePathPrefix;
-
- auto& task = taskInternal.Task;
-
- task.Scope = *parser.ColumnParser(SCOPE_COLUMN_NAME).GetOptionalString();
- task.QueryId = *parser.ColumnParser(QUERY_ID_COLUMN_NAME).GetOptionalString();
-
- bool isResignQuery = parser.ColumnParser(IS_RESIGN_QUERY_COLUMN_NAME).GetOptionalBool().GetOrElse(false);
- taskInternal.RetryCounter = parser.ColumnParser(RETRY_COUNTER_COLUMN_NAME).GetOptionalUint64().GetOrElse(0);
- taskInternal.RetryCounterUpdatedAt = parser.ColumnParser(RETRY_COUNTER_UPDATE_COLUMN_NAME).GetOptionalTimestamp().GetOrElse(TInstant::Zero());
-
- YandexQuery::QueryContent::QueryType queryType = static_cast<YandexQuery::QueryContent::QueryType>(parser.ColumnParser(QUERY_TYPE_COLUMN_NAME).GetOptionalInt64().GetOrElse(0));
-
+ }
+
+ TResultSetParser parser(resultSets.back());
+ while (parser.TryNextRow()) {
+ TTaskInternal& taskInternal = tasks.emplace_back();
+ taskInternal.Owner = owner;
+ taskInternal.HostName = hostName;
+ taskInternal.TablePathPrefix = YdbConnection->TablePathPrefix;
+
+ auto& task = taskInternal.Task;
+
+ task.Scope = *parser.ColumnParser(SCOPE_COLUMN_NAME).GetOptionalString();
+ task.QueryId = *parser.ColumnParser(QUERY_ID_COLUMN_NAME).GetOptionalString();
+
+ bool isResignQuery = parser.ColumnParser(IS_RESIGN_QUERY_COLUMN_NAME).GetOptionalBool().GetOrElse(false);
+ taskInternal.RetryCounter = parser.ColumnParser(RETRY_COUNTER_COLUMN_NAME).GetOptionalUint64().GetOrElse(0);
+ taskInternal.RetryCounterUpdatedAt = parser.ColumnParser(RETRY_COUNTER_UPDATE_COLUMN_NAME).GetOptionalTimestamp().GetOrElse(TInstant::Zero());
+
+ YandexQuery::QueryContent::QueryType queryType = static_cast<YandexQuery::QueryContent::QueryType>(parser.ColumnParser(QUERY_TYPE_COLUMN_NAME).GetOptionalInt64().GetOrElse(0));
+
const auto retryCounterLimit = queryType == YandexQuery::QueryContent::ANALYTICS ? Config.Proto.GetAnalyticsRetryCounterLimit() : Config.Proto.GetStreamingRetryCounterLimit();
const auto retryCounterUpdateTime = queryType == YandexQuery::QueryContent::ANALYTICS ? Config.AnalyticsRetryCounterUpdateTime : Config.StreamingRetryCounterUpdateTime;
-
- if (isResignQuery) {
- continue;
- }
-
- ++taskInternal.RetryCounter;
-
- if (now > taskInternal.RetryCounterUpdatedAt + retryCounterUpdateTime) {
- taskInternal.RetryCounter = 0;
- taskInternal.RetryCounterUpdatedAt = now;
- }
-
- if (taskInternal.RetryCounter >= retryCounterLimit) {
- taskInternal.ShouldAbortTask = true;
- }
- }
-
- std::shuffle(tasks.begin(), tasks.end(), std::default_random_engine());
- const size_t numTasks = (std::min(tasks.size(), tasksBatchSize) + numTasksProportion - 1) / numTasksProportion;
-
- for (size_t i = 0; i < numTasks; ++i) {
- auto tupleParams = MakeGetTaskUpdateQuery(tasks[i],
- responseTasks, taskLeaseTimestamp, Config.Proto.GetDisableCurrentIam(),
- Config.AutomaticQueriesTtl, Config.ResultSetsTtl); // using for win32 build
- auto readQuery = std::get<0>(tupleParams);
- auto readParams = std::get<1>(tupleParams);
- auto prepareParams = std::get<2>(tupleParams);
- pickTaskParams.emplace_back(TPickTaskParams{readQuery, readParams, prepareParams, tasks[i].Task.QueryId, tasks[i].ShouldAbortTask});
- }
- return pickTaskParams;
- };
-
+
+ if (isResignQuery) {
+ continue;
+ }
+
+ ++taskInternal.RetryCounter;
+
+ if (now > taskInternal.RetryCounterUpdatedAt + retryCounterUpdateTime) {
+ taskInternal.RetryCounter = 0;
+ taskInternal.RetryCounterUpdatedAt = now;
+ }
+
+ if (taskInternal.RetryCounter >= retryCounterLimit) {
+ taskInternal.ShouldAbortTask = true;
+ }
+ }
+
+ std::shuffle(tasks.begin(), tasks.end(), std::default_random_engine());
+ const size_t numTasks = (std::min(tasks.size(), tasksBatchSize) + numTasksProportion - 1) / numTasksProportion;
+
+ for (size_t i = 0; i < numTasks; ++i) {
+ auto tupleParams = MakeGetTaskUpdateQuery(tasks[i],
+ responseTasks, taskLeaseTimestamp, Config.Proto.GetDisableCurrentIam(),
+ Config.AutomaticQueriesTtl, Config.ResultSetsTtl); // using for win32 build
+ auto readQuery = std::get<0>(tupleParams);
+ auto readParams = std::get<1>(tupleParams);
+ auto prepareParams = std::get<2>(tupleParams);
+ pickTaskParams.emplace_back(TPickTaskParams{readQuery, readParams, prepareParams, tasks[i].Task.QueryId, tasks[i].ShouldAbortTask});
+ }
+ return pickTaskParams;
+ };
+
const auto query = queryBuilder.Build();
auto [readStatus, resultSets] = Read(query.Sql, query.Params, requestCounters, debugInfo, TTxSettings::StaleRO());
- auto result = readStatus.Apply(
- [=,
- resultSets=resultSets,
- requestCounters=requestCounters,
- debugInfo=debugInfo,
- responseTasks=responseTasks] (const auto& readFuture) mutable
- {
- try {
- if (!readFuture.GetValue().IsSuccess())
- return readFuture;
- } catch (...) {
- return readFuture;
- }
-
- auto pickTaskParams = prepareParams(*resultSets);
- if (pickTaskParams.empty())
- return readFuture;
-
+ auto result = readStatus.Apply(
+ [=,
+ resultSets=resultSets,
+ requestCounters=requestCounters,
+ debugInfo=debugInfo,
+ responseTasks=responseTasks] (const auto& readFuture) mutable
+ {
+ try {
+ if (!readFuture.GetValue().IsSuccess())
+ return readFuture;
+ } catch (...) {
+ return readFuture;
+ }
+
+ auto pickTaskParams = prepareParams(*resultSets);
+ if (pickTaskParams.empty())
+ return readFuture;
+
auto debugInfos = std::make_shared<TVector<TDebugInfoPtr>>(pickTaskParams.size());
if (Config.Proto.GetEnableDebugMode()) {
for (size_t i = 0; i < pickTaskParams.size(); i++) {
@@ -258,53 +258,53 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvGetTaskRequ
}
}
- TVector<TFuture<void>> futures;
- for (size_t i = 0; i < pickTaskParams.size(); ++i) {
+ TVector<TFuture<void>> futures;
+ for (size_t i = 0; i < pickTaskParams.size(); ++i) {
futures.emplace_back(PickTask(pickTaskParams[i], requestCounters, (*debugInfos)[i], responseTasks));
- }
-
- auto allFuture = NThreading::WaitExceptionOrAll(futures);
- return allFuture.Apply([=, responseTasks=responseTasks](const auto& future) mutable {
+ }
+
+ auto allFuture = NThreading::WaitExceptionOrAll(futures);
+ return allFuture.Apply([=, responseTasks=responseTasks](const auto& future) mutable {
if (debugInfo) {
for (const auto& info: *debugInfos) {
debugInfo->insert(debugInfo->end(), info->begin(), info->end());
}
}
- NYql::TIssues issues;
- auto status = MakeFuture(TStatus{EStatus::SUCCESS, std::move(issues)});
- try {
- future.GetValue();
- TVector<TEvControlPlaneStorage::TTask> tasks;
- for (const auto& [_, task] : responseTasks->GetTasksNonBlocking()) {
- tasks.emplace_back(task);
- }
- *response = std::make_tuple(tasks, owner);
- } catch (...) {
- issues.AddIssue(CurrentExceptionMessage());
- status = MakeFuture(TStatus{EStatus::GENERIC_ERROR, std::move(issues)});
- }
- return status;
- });
- });
-
- auto prepare = [response] { return *response; };
- auto success = SendResponseTuple
- <TEvControlPlaneStorage::TEvGetTaskResponse,
- std::tuple<TVector<TEvControlPlaneStorage::TTask>, TString>> //tasks, owner
- ("GetTaskRequest",
- NActors::TActivationContext::ActorSystem(),
- result,
- SelfId(),
- ev,
- startTime,
- requestCounters,
- prepare,
- debugInfo);
-
- success.Apply([=](const auto& future) {
- TDuration delta = TInstant::Now() - startTime;
- LWPROBE(GetTaskRequest, owner, hostName, delta, future.GetValue());
- });
-}
-
-} // NYq
+ NYql::TIssues issues;
+ auto status = MakeFuture(TStatus{EStatus::SUCCESS, std::move(issues)});
+ try {
+ future.GetValue();
+ TVector<TEvControlPlaneStorage::TTask> tasks;
+ for (const auto& [_, task] : responseTasks->GetTasksNonBlocking()) {
+ tasks.emplace_back(task);
+ }
+ *response = std::make_tuple(tasks, owner);
+ } catch (...) {
+ issues.AddIssue(CurrentExceptionMessage());
+ status = MakeFuture(TStatus{EStatus::GENERIC_ERROR, std::move(issues)});
+ }
+ return status;
+ });
+ });
+
+ auto prepare = [response] { return *response; };
+ auto success = SendResponseTuple
+ <TEvControlPlaneStorage::TEvGetTaskResponse,
+ std::tuple<TVector<TEvControlPlaneStorage::TTask>, TString>> //tasks, owner
+ ("GetTaskRequest",
+ NActors::TActivationContext::ActorSystem(),
+ result,
+ SelfId(),
+ ev,
+ startTime,
+ requestCounters,
+ prepare,
+ debugInfo);
+
+ success.Apply([=](const auto& future) {
+ TDuration delta = TInstant::Now() - startTime;
+ LWPROBE(GetTaskRequest, owner, hostName, delta, future.GetValue());
+ });
+}
+
+} // NYq
diff --git a/ydb/core/yq/libs/control_plane_storage/internal/task_ping.cpp b/ydb/core/yq/libs/control_plane_storage/internal/task_ping.cpp
index b3a81355af..834d40c0f7 100644
--- a/ydb/core/yq/libs/control_plane_storage/internal/task_ping.cpp
+++ b/ydb/core/yq/libs/control_plane_storage/internal/task_ping.cpp
@@ -1,11 +1,11 @@
-#include "utils.h"
-
+#include "utils.h"
+
#include <util/datetime/base.h>
#include <ydb/core/yq/libs/db_schema/db_schema.h>
-namespace NYq {
-
+namespace NYq {
+
namespace {
bool IsFinishedStatus(YandexQuery::QueryMeta::ComputeStatus status) {
@@ -17,129 +17,129 @@ bool IsFinishedStatus(YandexQuery::QueryMeta::ComputeStatus status) {
} // namespace
-std::tuple<TString, TParams, const std::function<std::pair<TString, NYdb::TParams>(const TVector<NYdb::TResultSet>&)>> ConstructHardPingTask(
- const TEvControlPlaneStorage::TEvPingTaskRequest* request, std::shared_ptr<YandexQuery::QueryAction> response,
+std::tuple<TString, TParams, const std::function<std::pair<TString, NYdb::TParams>(const TVector<NYdb::TResultSet>&)>> ConstructHardPingTask(
+ const TEvControlPlaneStorage::TEvPingTaskRequest* request, std::shared_ptr<YandexQuery::QueryAction> response,
const TString& tablePathPrefix, const TDuration& automaticQueriesTtl) {
-
- TSqlQueryBuilder readQueryBuilder(tablePathPrefix, "HardPingTask(read)");
+
+ TSqlQueryBuilder readQueryBuilder(tablePathPrefix, "HardPingTask(read)");
readQueryBuilder.AddString("scope", request->Scope);
readQueryBuilder.AddString("query_id", request->QueryId);
readQueryBuilder.AddText(
- "$last_job_id = SELECT `" LAST_JOB_ID_COLUMN_NAME "` FROM `" QUERIES_TABLE_NAME "`\n"
- " WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id;\n"
- "SELECT `" QUERY_COLUMN_NAME "`, `" INTERNAL_COLUMN_NAME "` FROM `" QUERIES_TABLE_NAME "`\n"
- " WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id;\n"
- "SELECT `" JOB_ID_COLUMN_NAME "`, `" JOB_COLUMN_NAME "` FROM `" JOBS_TABLE_NAME "`\n"
- " WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id AND `" JOB_ID_COLUMN_NAME "` = $last_job_id;\n"
- "SELECT `" OWNER_COLUMN_NAME "` FROM `" PENDING_SMALL_TABLE_NAME "`\n"
- " WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id;\n"
+ "$last_job_id = SELECT `" LAST_JOB_ID_COLUMN_NAME "` FROM `" QUERIES_TABLE_NAME "`\n"
+ " WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id;\n"
+ "SELECT `" QUERY_COLUMN_NAME "`, `" INTERNAL_COLUMN_NAME "` FROM `" QUERIES_TABLE_NAME "`\n"
+ " WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id;\n"
+ "SELECT `" JOB_ID_COLUMN_NAME "`, `" JOB_COLUMN_NAME "` FROM `" JOBS_TABLE_NAME "`\n"
+ " WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id AND `" JOB_ID_COLUMN_NAME "` = $last_job_id;\n"
+ "SELECT `" OWNER_COLUMN_NAME "` FROM `" PENDING_SMALL_TABLE_NAME "`\n"
+ " WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id;\n"
);
-
- auto prepareParams = [=](const TVector<TResultSet>& resultSets) {
- TString jobId;
- YandexQuery::Query query;
- YandexQuery::Internal::QueryInternal internal;
- YandexQuery::Job job;
- TString selectedOwner;
-
- if (resultSets.size() != 3) {
- ythrow TControlPlaneStorageException(TIssuesIds::INTERNAL_ERROR) << "Result set size is not equal to 3 but equal " << resultSets.size() << ". Please contact internal support";
- }
-
- {
- TResultSetParser parser(resultSets[0]);
- if (parser.TryNextRow()) {
- if (!query.ParseFromString(*parser.ColumnParser(QUERY_COLUMN_NAME).GetOptionalString())) {
- ythrow TControlPlaneStorageException(TIssuesIds::INTERNAL_ERROR) << "Error parsing proto message for query. Please contact internal support";
- }
- if (!internal.ParseFromString(*parser.ColumnParser(INTERNAL_COLUMN_NAME).GetOptionalString())) {
- ythrow TControlPlaneStorageException(TIssuesIds::INTERNAL_ERROR) << "Error parsing proto message for query internal. Please contact internal support";
- }
- }
- }
-
- {
- TResultSetParser parser(resultSets[1]);
- if (parser.TryNextRow()) {
- if (!job.ParseFromString(*parser.ColumnParser(JOB_COLUMN_NAME).GetOptionalString())) {
- ythrow TControlPlaneStorageException(TIssuesIds::INTERNAL_ERROR) << "Error parsing proto message for job. Please contact internal support";
- }
- jobId = *parser.ColumnParser(JOB_ID_COLUMN_NAME).GetOptionalString();
- }
- }
-
- {
- TResultSetParser parser(resultSets[2]);
- if (parser.TryNextRow()) {
- selectedOwner = *parser.ColumnParser(OWNER_COLUMN_NAME).GetOptionalString();
- }
- }
-
- if (selectedOwner != request->Owner) {
- ythrow TControlPlaneStorageException(TIssuesIds::BAD_REQUEST) << "Query with the specified Owner: \"" << request->Owner << "\" does not exist. Selected owner: \"" << selectedOwner << "\"";
- }
-
- if (request->Status) {
- query.mutable_meta()->set_status(*request->Status);
- job.mutable_query_meta()->set_status(*request->Status);
- }
-
- if (request->Issues) {
- NYql::IssuesToMessage(*request->Issues, query.mutable_issue());
- NYql::IssuesToMessage(*request->Issues, job.mutable_issue());
- }
-
- if (request->TransientIssues) {
- NYql::TIssues issues = *request->TransientIssues;
- for (const auto& issue: *query.mutable_transient_issue()) {
- issues.AddIssue(NYql::IssueFromMessage(issue));
- }
-
- NYql::TIssues newIssues;
- std::for_each_n(issues.begin(), std::min(static_cast<unsigned long long>(issues.Size()), 20ULL), [&](auto& issue){ newIssues.AddIssue(issue); });
-
- NYql::IssuesToMessage(newIssues, query.mutable_transient_issue());
- }
-
- if (request->Statistics) {
- *query.mutable_statistics()->mutable_json() = *request->Statistics;
- *job.mutable_statistics()->mutable_json() = *request->Statistics;
- }
-
- if (request->ResultSetMetas) {
- // we will overwrite result_set_meta's COMPLETELY
- query.clear_result_set_meta();
- job.clear_result_set_meta();
- for (const auto& resultSetMeta : *request->ResultSetMetas) {
- *query.add_result_set_meta() = resultSetMeta;
- *job.add_result_set_meta() = resultSetMeta;
- }
- }
-
- if (request->Ast) {
- query.mutable_ast()->set_data(*request->Ast);
- job.mutable_ast()->set_data(*request->Ast);
- }
-
- if (request->Plan) {
- query.mutable_plan()->set_json(*request->Plan);
- job.mutable_plan()->set_json(*request->Plan);
- }
-
- if (request->StartedAt) {
+
+ auto prepareParams = [=](const TVector<TResultSet>& resultSets) {
+ TString jobId;
+ YandexQuery::Query query;
+ YandexQuery::Internal::QueryInternal internal;
+ YandexQuery::Job job;
+ TString selectedOwner;
+
+ if (resultSets.size() != 3) {
+ ythrow TControlPlaneStorageException(TIssuesIds::INTERNAL_ERROR) << "Result set size is not equal to 3 but equal " << resultSets.size() << ". Please contact internal support";
+ }
+
+ {
+ TResultSetParser parser(resultSets[0]);
+ if (parser.TryNextRow()) {
+ if (!query.ParseFromString(*parser.ColumnParser(QUERY_COLUMN_NAME).GetOptionalString())) {
+ ythrow TControlPlaneStorageException(TIssuesIds::INTERNAL_ERROR) << "Error parsing proto message for query. Please contact internal support";
+ }
+ if (!internal.ParseFromString(*parser.ColumnParser(INTERNAL_COLUMN_NAME).GetOptionalString())) {
+ ythrow TControlPlaneStorageException(TIssuesIds::INTERNAL_ERROR) << "Error parsing proto message for query internal. Please contact internal support";
+ }
+ }
+ }
+
+ {
+ TResultSetParser parser(resultSets[1]);
+ if (parser.TryNextRow()) {
+ if (!job.ParseFromString(*parser.ColumnParser(JOB_COLUMN_NAME).GetOptionalString())) {
+ ythrow TControlPlaneStorageException(TIssuesIds::INTERNAL_ERROR) << "Error parsing proto message for job. Please contact internal support";
+ }
+ jobId = *parser.ColumnParser(JOB_ID_COLUMN_NAME).GetOptionalString();
+ }
+ }
+
+ {
+ TResultSetParser parser(resultSets[2]);
+ if (parser.TryNextRow()) {
+ selectedOwner = *parser.ColumnParser(OWNER_COLUMN_NAME).GetOptionalString();
+ }
+ }
+
+ if (selectedOwner != request->Owner) {
+ ythrow TControlPlaneStorageException(TIssuesIds::BAD_REQUEST) << "Query with the specified Owner: \"" << request->Owner << "\" does not exist. Selected owner: \"" << selectedOwner << "\"";
+ }
+
+ if (request->Status) {
+ query.mutable_meta()->set_status(*request->Status);
+ job.mutable_query_meta()->set_status(*request->Status);
+ }
+
+ if (request->Issues) {
+ NYql::IssuesToMessage(*request->Issues, query.mutable_issue());
+ NYql::IssuesToMessage(*request->Issues, job.mutable_issue());
+ }
+
+ if (request->TransientIssues) {
+ NYql::TIssues issues = *request->TransientIssues;
+ for (const auto& issue: *query.mutable_transient_issue()) {
+ issues.AddIssue(NYql::IssueFromMessage(issue));
+ }
+
+ NYql::TIssues newIssues;
+ std::for_each_n(issues.begin(), std::min(static_cast<unsigned long long>(issues.Size()), 20ULL), [&](auto& issue){ newIssues.AddIssue(issue); });
+
+ NYql::IssuesToMessage(newIssues, query.mutable_transient_issue());
+ }
+
+ if (request->Statistics) {
+ *query.mutable_statistics()->mutable_json() = *request->Statistics;
+ *job.mutable_statistics()->mutable_json() = *request->Statistics;
+ }
+
+ if (request->ResultSetMetas) {
+ // we will overwrite result_set_meta's COMPLETELY
+ query.clear_result_set_meta();
+ job.clear_result_set_meta();
+ for (const auto& resultSetMeta : *request->ResultSetMetas) {
+ *query.add_result_set_meta() = resultSetMeta;
+ *job.add_result_set_meta() = resultSetMeta;
+ }
+ }
+
+ if (request->Ast) {
+ query.mutable_ast()->set_data(*request->Ast);
+ job.mutable_ast()->set_data(*request->Ast);
+ }
+
+ if (request->Plan) {
+ query.mutable_plan()->set_json(*request->Plan);
+ job.mutable_plan()->set_json(*request->Plan);
+ }
+
+ if (request->StartedAt) {
*query.mutable_meta()->mutable_started_at() = NProtoInterop::CastToProto(*request->StartedAt);
*job.mutable_query_meta()->mutable_started_at() = NProtoInterop::CastToProto(*request->StartedAt);
- }
-
- if (request->FinishedAt) {
+ }
+
+ if (request->FinishedAt) {
*query.mutable_meta()->mutable_finished_at() = NProtoInterop::CastToProto(*request->FinishedAt);
*job.mutable_query_meta()->mutable_finished_at() = NProtoInterop::CastToProto(*request->FinishedAt);
- if (!query.meta().has_started_at()) {
+ if (!query.meta().has_started_at()) {
*query.mutable_meta()->mutable_started_at() = NProtoInterop::CastToProto(*request->FinishedAt);
*job.mutable_query_meta()->mutable_started_at() = NProtoInterop::CastToProto(*request->FinishedAt);
- }
- }
-
+ }
+ }
+
TInstant expireAt = TInstant::Now() + automaticQueriesTtl;
if (IsTerminalStatus(query.meta().status()) && query.content().automatic()) {
*query.mutable_meta()->mutable_expire_at() = NProtoInterop::CastToProto(expireAt);
@@ -168,42 +168,42 @@ std::tuple<TString, TParams, const std::function<std::pair<TString, NYdb::TParam
internal.clear_dq_graph_index();
}
- if (!request->CreatedTopicConsumers.empty()) {
- std::set<Yq::Private::TopicConsumer, TTopicConsumerLess> mergedConsumers;
- for (auto&& c : *internal.mutable_created_topic_consumers()) {
- mergedConsumers.emplace(std::move(c));
- }
-
- for (const auto& c : request->CreatedTopicConsumers) {
- Yq::Private::TopicConsumer proto;
- proto.set_database_id(c.DatabaseId);
- proto.set_database(c.Database);
- proto.set_topic_path(c.TopicPath);
- proto.set_consumer_name(c.ConsumerName);
- proto.set_cluster_endpoint(c.ClusterEndpoint);
- proto.set_use_ssl(c.UseSsl);
- proto.set_token_name(c.TokenName);
- proto.set_add_bearer_to_token(c.AddBearerToToken);
- mergedConsumers.emplace(std::move(proto));
- }
- internal.clear_created_topic_consumers();
- for (auto&& c : mergedConsumers) {
- *internal.add_created_topic_consumers() = std::move(c);
- }
- }
-
- if (!request->DqGraphs.empty()) {
- internal.clear_dq_graph();
- for (const auto& g : request->DqGraphs) {
- internal.add_dq_graph(g);
- }
- }
-
- if (request->DqGraphIndex) {
- internal.set_dq_graph_index(request->DqGraphIndex);
- }
-
- TSqlQueryBuilder writeQueryBuilder(tablePathPrefix, "HardPingTask(write)");
+ if (!request->CreatedTopicConsumers.empty()) {
+ std::set<Yq::Private::TopicConsumer, TTopicConsumerLess> mergedConsumers;
+ for (auto&& c : *internal.mutable_created_topic_consumers()) {
+ mergedConsumers.emplace(std::move(c));
+ }
+
+ for (const auto& c : request->CreatedTopicConsumers) {
+ Yq::Private::TopicConsumer proto;
+ proto.set_database_id(c.DatabaseId);
+ proto.set_database(c.Database);
+ proto.set_topic_path(c.TopicPath);
+ proto.set_consumer_name(c.ConsumerName);
+ proto.set_cluster_endpoint(c.ClusterEndpoint);
+ proto.set_use_ssl(c.UseSsl);
+ proto.set_token_name(c.TokenName);
+ proto.set_add_bearer_to_token(c.AddBearerToToken);
+ mergedConsumers.emplace(std::move(proto));
+ }
+ internal.clear_created_topic_consumers();
+ for (auto&& c : mergedConsumers) {
+ *internal.add_created_topic_consumers() = std::move(c);
+ }
+ }
+
+ if (!request->DqGraphs.empty()) {
+ internal.clear_dq_graph();
+ for (const auto& g : request->DqGraphs) {
+ internal.add_dq_graph(g);
+ }
+ }
+
+ if (request->DqGraphIndex) {
+ internal.set_dq_graph_index(request->DqGraphIndex);
+ }
+
+ TSqlQueryBuilder writeQueryBuilder(tablePathPrefix, "HardPingTask(write)");
writeQueryBuilder.AddString("scope", request->Scope);
writeQueryBuilder.AddString("job_id", jobId);
writeQueryBuilder.AddString("job", job.SerializeAsString());
@@ -212,7 +212,7 @@ std::tuple<TString, TParams, const std::function<std::pair<TString, NYdb::TParam
writeQueryBuilder.AddString("internal", internal.SerializeAsString());
writeQueryBuilder.AddString("result_id", request->ResultId);
writeQueryBuilder.AddString("query_id", request->QueryId);
-
+
if (IsTerminalStatus(query.meta().status())) {
// delete pending
writeQueryBuilder.AddText(
@@ -220,7 +220,7 @@ std::tuple<TString, TParams, const std::function<std::pair<TString, NYdb::TParam
"WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id;\n"
);
} else {
- // update pending small
+ // update pending small
writeQueryBuilder.AddTimestamp("now", request->ResignQuery ? TInstant::Zero() : TInstant::Now());
const TString updateResignQueryFlag = request->ResignQuery ? ", `" IS_RESIGN_QUERY_COLUMN_NAME "` = true" : "";
writeQueryBuilder.AddText(
@@ -228,7 +228,7 @@ std::tuple<TString, TParams, const std::function<std::pair<TString, NYdb::TParam
"WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id;\n"
);
}
-
+
if (IsTerminalStatus(query.meta().status()) && query.content().automatic()) {
// set jobs ttl
writeQueryBuilder.AddText(
@@ -242,7 +242,7 @@ std::tuple<TString, TParams, const std::function<std::pair<TString, NYdb::TParam
"WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id AND `" EXPIRE_AT_COLUMN_NAME "` IS NOT NULL;\n"
);
}
-
+
TString updateResultSetsExpire;
if (query.meta().status() == YandexQuery::QueryMeta::COMPLETED) {
writeQueryBuilder.AddTimestamp("result_sets_expire_at", request->Deadline);
@@ -250,7 +250,7 @@ std::tuple<TString, TParams, const std::function<std::pair<TString, NYdb::TParam
} else {
updateResultSetsExpire = "`" RESULT_SETS_EXPIRE_AT_COLUMN_NAME "` = NULL";
}
-
+
TString updateQueryTtl;
if (IsTerminalStatus(query.meta().status()) && query.content().automatic()) {
writeQueryBuilder.AddTimestamp("expire_at", expireAt);
@@ -260,134 +260,134 @@ std::tuple<TString, TParams, const std::function<std::pair<TString, NYdb::TParam
}
writeQueryBuilder.AddText(
- "UPSERT INTO `" JOBS_TABLE_NAME "` (`" SCOPE_COLUMN_NAME "`, `" QUERY_ID_COLUMN_NAME "`, `" JOB_ID_COLUMN_NAME "`, `" JOB_COLUMN_NAME "`) VALUES($scope, $query_id, $job_id, $job);\n"
+ "UPSERT INTO `" JOBS_TABLE_NAME "` (`" SCOPE_COLUMN_NAME "`, `" QUERY_ID_COLUMN_NAME "`, `" JOB_ID_COLUMN_NAME "`, `" JOB_COLUMN_NAME "`) VALUES($scope, $query_id, $job_id, $job);\n"
"UPDATE `" QUERIES_TABLE_NAME "` SET `" QUERY_COLUMN_NAME "` = $query, `" STATUS_COLUMN_NAME "` = $status, `" INTERNAL_COLUMN_NAME "` = $internal, `" RESULT_ID_COLUMN_NAME "` = $result_id, " + updateResultSetsExpire + ", " + updateQueryTtl + ", `" META_REVISION_COLUMN_NAME "` = `" META_REVISION_COLUMN_NAME "` + 1\n"
- "WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id;\n"
+ "WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id;\n"
);
-
- *response = internal.action();
+
+ *response = internal.action();
const auto writeQuery = writeQueryBuilder.Build();
return std::make_pair(writeQuery.Sql, writeQuery.Params);
- };
+ };
const auto readQuery = readQueryBuilder.Build();
return std::make_tuple(readQuery.Sql, readQuery.Params, prepareParams);
-}
-
-std::tuple<TString, TParams, const std::function<std::pair<TString, NYdb::TParams>(const TVector<NYdb::TResultSet>&)>> ConstructSoftPingTask(
- const TEvControlPlaneStorage::TEvPingTaskRequest* request, std::shared_ptr<YandexQuery::QueryAction> response,
- const TString& tablePathPrefix) {
- TSqlQueryBuilder readQueryBuilder(tablePathPrefix, "SoftPingTask(read)");
+}
+
+std::tuple<TString, TParams, const std::function<std::pair<TString, NYdb::TParams>(const TVector<NYdb::TResultSet>&)>> ConstructSoftPingTask(
+ const TEvControlPlaneStorage::TEvPingTaskRequest* request, std::shared_ptr<YandexQuery::QueryAction> response,
+ const TString& tablePathPrefix) {
+ TSqlQueryBuilder readQueryBuilder(tablePathPrefix, "SoftPingTask(read)");
readQueryBuilder.AddString("scope", request->Scope);
readQueryBuilder.AddString("query_id", request->QueryId);
readQueryBuilder.AddText(
- "SELECT `" INTERNAL_COLUMN_NAME "`\n"
- "FROM `" QUERIES_TABLE_NAME "` WHERE `" QUERY_ID_COLUMN_NAME "` = $query_id AND `" SCOPE_COLUMN_NAME "` = $scope;\n"
- "SELECT `" OWNER_COLUMN_NAME "`\n"
- "FROM `" PENDING_SMALL_TABLE_NAME "` WHERE `" QUERY_ID_COLUMN_NAME "` = $query_id AND `" SCOPE_COLUMN_NAME "` = $scope;\n"
+ "SELECT `" INTERNAL_COLUMN_NAME "`\n"
+ "FROM `" QUERIES_TABLE_NAME "` WHERE `" QUERY_ID_COLUMN_NAME "` = $query_id AND `" SCOPE_COLUMN_NAME "` = $scope;\n"
+ "SELECT `" OWNER_COLUMN_NAME "`\n"
+ "FROM `" PENDING_SMALL_TABLE_NAME "` WHERE `" QUERY_ID_COLUMN_NAME "` = $query_id AND `" SCOPE_COLUMN_NAME "` = $scope;\n"
);
-
- auto prepareParams = [=](const TVector<TResultSet>& resultSets) {
- TString selectedOwner;
- YandexQuery::Internal::QueryInternal internal;
-
- if (resultSets.size() != 2) {
- ythrow TControlPlaneStorageException(TIssuesIds::INTERNAL_ERROR) << "Result set size is not equal to 2 but equal " << resultSets.size() << ". Please contact internal support";
- }
-
- {
- TResultSetParser parser(resultSets[0]);
- if (parser.TryNextRow()) {
- if (!internal.ParseFromString(*parser.ColumnParser(INTERNAL_COLUMN_NAME).GetOptionalString())) {
- ythrow TControlPlaneStorageException(TIssuesIds::INTERNAL_ERROR) << "Error parsing proto message for pending internal query. Please contact internal support";
- }
- }
- }
-
- {
- TResultSetParser parser(resultSets[1]);
- if (parser.TryNextRow()) {
- selectedOwner = *parser.ColumnParser(OWNER_COLUMN_NAME).GetOptionalString();
- }
- }
- *response = internal.action();
-
- if (selectedOwner != request->Owner) {
- ythrow TControlPlaneStorageException(TIssuesIds::BAD_REQUEST) << "query with the specified Owner: \"" << request->Owner << "\" does not exist. Selected owner: \"" << selectedOwner << "\"";
- }
-
- TSqlQueryBuilder writeQueryBuilder(tablePathPrefix, "SoftPingTask(write)");
+
+ auto prepareParams = [=](const TVector<TResultSet>& resultSets) {
+ TString selectedOwner;
+ YandexQuery::Internal::QueryInternal internal;
+
+ if (resultSets.size() != 2) {
+ ythrow TControlPlaneStorageException(TIssuesIds::INTERNAL_ERROR) << "Result set size is not equal to 2 but equal " << resultSets.size() << ". Please contact internal support";
+ }
+
+ {
+ TResultSetParser parser(resultSets[0]);
+ if (parser.TryNextRow()) {
+ if (!internal.ParseFromString(*parser.ColumnParser(INTERNAL_COLUMN_NAME).GetOptionalString())) {
+ ythrow TControlPlaneStorageException(TIssuesIds::INTERNAL_ERROR) << "Error parsing proto message for pending internal query. Please contact internal support";
+ }
+ }
+ }
+
+ {
+ TResultSetParser parser(resultSets[1]);
+ if (parser.TryNextRow()) {
+ selectedOwner = *parser.ColumnParser(OWNER_COLUMN_NAME).GetOptionalString();
+ }
+ }
+ *response = internal.action();
+
+ if (selectedOwner != request->Owner) {
+ ythrow TControlPlaneStorageException(TIssuesIds::BAD_REQUEST) << "query with the specified Owner: \"" << request->Owner << "\" does not exist. Selected owner: \"" << selectedOwner << "\"";
+ }
+
+ TSqlQueryBuilder writeQueryBuilder(tablePathPrefix, "SoftPingTask(write)");
writeQueryBuilder.AddTimestamp("now", request->ResignQuery ? TInstant::Zero() : TInstant::Now());
writeQueryBuilder.AddString("scope", request->Scope);
writeQueryBuilder.AddString("query_id", request->QueryId);
writeQueryBuilder.AddString("owner", request->Owner);
-
- const TString updateResignQueryFlag = request->ResignQuery ? ", `" IS_RESIGN_QUERY_COLUMN_NAME "` = true" : "";
+
+ const TString updateResignQueryFlag = request->ResignQuery ? ", `" IS_RESIGN_QUERY_COLUMN_NAME "` = true" : "";
writeQueryBuilder.AddText(
"UPDATE `" PENDING_SMALL_TABLE_NAME "` SET `" LAST_SEEN_AT_COLUMN_NAME "` = $now " + updateResignQueryFlag + "\n"
- "WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id;\n"
- );
-
+ "WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id;\n"
+ );
+
const auto writeQuery = writeQueryBuilder.Build();
return std::make_pair(writeQuery.Sql, writeQuery.Params);
- };
+ };
const auto readQuery = readQueryBuilder.Build();
return std::make_tuple(readQuery.Sql, readQuery.Params, prepareParams);
-}
-
-void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvPingTaskRequest::TPtr& ev)
-{
- TInstant startTime = TInstant::Now();
- TRequestCountersPtr requestCounters = Counters.Requests[RT_PING_TASK];
- requestCounters->InFly->Inc();
-
- TEvControlPlaneStorage::TEvPingTaskRequest* request = ev->Get();
- const TString scope = request->Scope;
- const TString queryId = request->QueryId;
- const TString owner = request->Owner;
+}
+
+void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvPingTaskRequest::TPtr& ev)
+{
+ TInstant startTime = TInstant::Now();
+ TRequestCountersPtr requestCounters = Counters.Requests[RT_PING_TASK];
+ requestCounters->InFly->Inc();
+
+ TEvControlPlaneStorage::TEvPingTaskRequest* request = ev->Get();
+ const TString scope = request->Scope;
+ const TString queryId = request->QueryId;
+ const TString owner = request->Owner;
const TInstant deadline = request->Deadline;
-
- CPS_LOG_T("PingTaskRequest: " << scope << " " << queryId
- << " " << owner << " " << deadline << " "
- << (request->Status ? YandexQuery::QueryMeta_ComputeStatus_Name(*request->Status) : "no status"));
-
+
+ CPS_LOG_T("PingTaskRequest: " << scope << " " << queryId
+ << " " << owner << " " << deadline << " "
+ << (request->Status ? YandexQuery::QueryMeta_ComputeStatus_Name(*request->Status) : "no status"));
+
NYql::TIssues issues = ValidatePingTask(scope, queryId, owner, deadline, Config.ResultSetsTtl);
- if (issues) {
+ if (issues) {
CPS_LOG_D("PingTaskRequest, validation failed: " << scope << " " << queryId << " " << owner << " " << deadline << issues.ToString());
- const TDuration delta = TInstant::Now() - startTime;
- SendResponseIssues<TEvControlPlaneStorage::TEvPingTaskResponse>(ev->Sender, issues, ev->Cookie, delta, requestCounters);
- LWPROBE(PingTaskRequest, queryId, delta, false);
- return;
- }
-
- std::shared_ptr<YandexQuery::QueryAction> response = std::make_shared<YandexQuery::QueryAction>();
-
- if (request->Status)
- FinalStatusCounters.IncByStatus(*request->Status);
- auto pingTaskParams = DoesPingTaskUpdateQueriesTable(request) ?
+ const TDuration delta = TInstant::Now() - startTime;
+ SendResponseIssues<TEvControlPlaneStorage::TEvPingTaskResponse>(ev->Sender, issues, ev->Cookie, delta, requestCounters);
+ LWPROBE(PingTaskRequest, queryId, delta, false);
+ return;
+ }
+
+ std::shared_ptr<YandexQuery::QueryAction> response = std::make_shared<YandexQuery::QueryAction>();
+
+ if (request->Status)
+ FinalStatusCounters.IncByStatus(*request->Status);
+ auto pingTaskParams = DoesPingTaskUpdateQueriesTable(request) ?
ConstructHardPingTask(request, response, YdbConnection->TablePathPrefix, Config.AutomaticQueriesTtl) :
- ConstructSoftPingTask(request, response, YdbConnection->TablePathPrefix);
- auto readQuery = std::get<0>(pingTaskParams); // Use std::get for win compiler
- auto readParams = std::get<1>(pingTaskParams);
- auto prepareParams = std::get<2>(pingTaskParams);
-
+ ConstructSoftPingTask(request, response, YdbConnection->TablePathPrefix);
+ auto readQuery = std::get<0>(pingTaskParams); // Use std::get for win compiler
+ auto readParams = std::get<1>(pingTaskParams);
+ auto prepareParams = std::get<2>(pingTaskParams);
+
auto debugInfo = Config.Proto.GetEnableDebugMode() ? std::make_shared<TDebugInfo>() : TDebugInfoPtr{};
- auto result = ReadModifyWrite(NActors::TActivationContext::ActorSystem(), readQuery, readParams, prepareParams, requestCounters, debugInfo);
- auto prepare = [response] { return std::make_tuple(*response); };
- auto success = SendResponseTuple<TEvControlPlaneStorage::TEvPingTaskResponse, std::tuple<YandexQuery::QueryAction>>(
- "PingTaskRequest",
- NActors::TActivationContext::ActorSystem(),
- result,
- SelfId(),
- ev,
- startTime,
- requestCounters,
- prepare,
- debugInfo);
-
- success.Apply([=](const auto& future) {
- TDuration delta = TInstant::Now() - startTime;
- LWPROBE(PingTaskRequest, queryId, delta, future.GetValue());
- });
-}
-
-} // NYq
+ auto result = ReadModifyWrite(NActors::TActivationContext::ActorSystem(), readQuery, readParams, prepareParams, requestCounters, debugInfo);
+ auto prepare = [response] { return std::make_tuple(*response); };
+ auto success = SendResponseTuple<TEvControlPlaneStorage::TEvPingTaskResponse, std::tuple<YandexQuery::QueryAction>>(
+ "PingTaskRequest",
+ NActors::TActivationContext::ActorSystem(),
+ result,
+ SelfId(),
+ ev,
+ startTime,
+ requestCounters,
+ prepare,
+ debugInfo);
+
+ success.Apply([=](const auto& future) {
+ TDuration delta = TInstant::Now() - startTime;
+ LWPROBE(PingTaskRequest, queryId, delta, future.GetValue());
+ });
+}
+
+} // NYq
diff --git a/ydb/core/yq/libs/control_plane_storage/internal/task_result_write.cpp b/ydb/core/yq/libs/control_plane_storage/internal/task_result_write.cpp
index f6c16f084f..db945dcca9 100644
--- a/ydb/core/yq/libs/control_plane_storage/internal/task_result_write.cpp
+++ b/ydb/core/yq/libs/control_plane_storage/internal/task_result_write.cpp
@@ -1,83 +1,83 @@
-#include "utils.h"
-
-namespace NYq {
-
-void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvWriteResultDataRequest::TPtr& ev)
-{
- TInstant startTime = TInstant::Now();
- TRequestCountersPtr requestCounters = Counters.Requests[RT_WRITE_RESULT_DATA];
- requestCounters->InFly->Inc();
-
- TEvControlPlaneStorage::TEvWriteResultDataRequest& request = *ev->Get();
- const TString resultId = request.ResultId;
- const int32_t resultSetId = request.ResultSetId;
- const int64_t startRowId = request.StartRowId;
+#include "utils.h"
+
+namespace NYq {
+
+void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvWriteResultDataRequest::TPtr& ev)
+{
+ TInstant startTime = TInstant::Now();
+ TRequestCountersPtr requestCounters = Counters.Requests[RT_WRITE_RESULT_DATA];
+ requestCounters->InFly->Inc();
+
+ TEvControlPlaneStorage::TEvWriteResultDataRequest& request = *ev->Get();
+ const TString resultId = request.ResultId;
+ const int32_t resultSetId = request.ResultSetId;
+ const int64_t startRowId = request.StartRowId;
const TInstant deadline = request.Deadline;
- const Ydb::ResultSet& resultSet = request.ResultSet;
- const int byteSize = resultSet.ByteSize();
-
+ const Ydb::ResultSet& resultSet = request.ResultSet;
+ const int byteSize = resultSet.ByteSize();
+
CPS_LOG_T("WriteResultDataRequest: " << resultId << " " << resultSetId << " " << startRowId << " " << resultSet.ByteSize() << " " << deadline);
-
+
NYql::TIssues issues = ValidateWriteResultData(resultId, resultSet, deadline, Config.ResultSetsTtl);
- if (issues) {
+ if (issues) {
CPS_LOG_D("WriteResultDataRequest, validation failed: " << resultId << " " << resultSetId << " " << startRowId << " " << resultSet.DebugString() << " " << deadline << " error: " << issues.ToString());
- const TDuration delta = TInstant::Now() - startTime;
- SendResponseIssues<TEvControlPlaneStorage::TEvWriteResultDataResponse>(ev->Sender, issues, ev->Cookie, delta, requestCounters);
- LWPROBE(WriteResultDataRequest, resultId, resultSetId, startRowId, resultSet.rows().size(), delta, deadline, byteSize, false);
- return;
- }
-
- NYdb::TValueBuilder itemsAsList;
- itemsAsList.BeginList();
-
- int64_t rowId = startRowId;
- for (const auto& row : resultSet.rows()) {
- TString serializedRow;
- if (!row.SerializeToString(&serializedRow)) {
- ythrow TControlPlaneStorageException(TIssuesIds::INTERNAL_ERROR) << "Error serialize proto message for row. Please contact internal support";
- }
-
- itemsAsList.AddListItem()
- .BeginStruct()
- .AddMember("row_id").Int64(rowId)
+ const TDuration delta = TInstant::Now() - startTime;
+ SendResponseIssues<TEvControlPlaneStorage::TEvWriteResultDataResponse>(ev->Sender, issues, ev->Cookie, delta, requestCounters);
+ LWPROBE(WriteResultDataRequest, resultId, resultSetId, startRowId, resultSet.rows().size(), delta, deadline, byteSize, false);
+ return;
+ }
+
+ NYdb::TValueBuilder itemsAsList;
+ itemsAsList.BeginList();
+
+ int64_t rowId = startRowId;
+ for (const auto& row : resultSet.rows()) {
+ TString serializedRow;
+ if (!row.SerializeToString(&serializedRow)) {
+ ythrow TControlPlaneStorageException(TIssuesIds::INTERNAL_ERROR) << "Error serialize proto message for row. Please contact internal support";
+ }
+
+ itemsAsList.AddListItem()
+ .BeginStruct()
+ .AddMember("row_id").Int64(rowId)
.AddMember("result_set").String(row.SerializeAsString())
- .EndStruct();
- rowId++;
- }
-
- itemsAsList.EndList();
-
- TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "TaskResultWrite");
+ .EndStruct();
+ rowId++;
+ }
+
+ itemsAsList.EndList();
+
+ TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "TaskResultWrite");
queryBuilder.AddString("result_id", resultId);
queryBuilder.AddInt32("result_set_id", resultSetId);
queryBuilder.AddTimestamp("expire_at", deadline);
queryBuilder.AddValue("items", itemsAsList.Build());
-
+
queryBuilder.AddText(
- "UPSERT INTO `" RESULT_SETS_TABLE_NAME "`\n"
+ "UPSERT INTO `" RESULT_SETS_TABLE_NAME "`\n"
"SELECT $result_id as result_id, $result_set_id as result_set_id,\n"
" T.*, $expire_at as expire_at FROM as_table($items) AS T;\n"
);
-
+
const auto query = queryBuilder.Build();
auto debugInfo = Config.Proto.GetEnableDebugMode() ? std::make_shared<TDebugInfo>() : TDebugInfoPtr{};
TAsyncStatus result = Write(NActors::TActivationContext::ActorSystem(), query.Sql, query.Params, requestCounters, debugInfo);
- auto prepare = []() { return std::make_tuple<NYql::TIssues>(NYql::TIssues{}); };
- auto success = SendResponseTuple<TEvControlPlaneStorage::TEvWriteResultDataResponse, std::tuple<NYql::TIssues>>(
- "WriteResultDataRequest",
- NActors::TActivationContext::ActorSystem(),
- result,
- SelfId(),
- ev,
- startTime,
- requestCounters,
- prepare,
- debugInfo);
-
- success.Apply([=](const auto& future) {
- TDuration delta = TInstant::Now() - startTime;
- LWPROBE(WriteResultDataRequest, resultId, resultSetId, startRowId, resultSet.rows().size(), delta, deadline, byteSize, future.GetValue());
- });
-}
-
-} // NYq
+ auto prepare = []() { return std::make_tuple<NYql::TIssues>(NYql::TIssues{}); };
+ auto success = SendResponseTuple<TEvControlPlaneStorage::TEvWriteResultDataResponse, std::tuple<NYql::TIssues>>(
+ "WriteResultDataRequest",
+ NActors::TActivationContext::ActorSystem(),
+ result,
+ SelfId(),
+ ev,
+ startTime,
+ requestCounters,
+ prepare,
+ debugInfo);
+
+ success.Apply([=](const auto& future) {
+ TDuration delta = TInstant::Now() - startTime;
+ LWPROBE(WriteResultDataRequest, resultId, resultSetId, startRowId, resultSet.rows().size(), delta, deadline, byteSize, future.GetValue());
+ });
+}
+
+} // NYq
diff --git a/ydb/core/yq/libs/control_plane_storage/internal/utils.cpp b/ydb/core/yq/libs/control_plane_storage/internal/utils.cpp
index 8e3a78deb0..bca91f1cfb 100644
--- a/ydb/core/yq/libs/control_plane_storage/internal/utils.cpp
+++ b/ydb/core/yq/libs/control_plane_storage/internal/utils.cpp
@@ -1,83 +1,83 @@
-#include "utils.h"
-
-namespace NYq {
-
+#include "utils.h"
+
+namespace NYq {
+
NYql::TIssues ValidateWriteResultData(const TString& resultId, const Ydb::ResultSet& resultSet, const TInstant& deadline, const TDuration& ttl)
-{
- NYql::TIssues issues;
- if (!resultId) {
- issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "result id is not specified"));
- }
-
- if (resultSet.rows().size() == 0) {
- issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "result set rows count is empty"));
- }
-
+{
+ NYql::TIssues issues;
+ if (!resultId) {
+ issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "result id is not specified"));
+ }
+
+ if (resultSet.rows().size() == 0) {
+ issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "result set rows count is empty"));
+ }
+
const auto hardLimit = TInstant::Now() + ttl;
if (deadline > hardLimit) {
issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "deadline " + deadline.ToString() + " must be less than " + hardLimit.ToString()));
}
- return issues;
-}
-
-NYql::TIssues ValidateGetTask(const TString& owner, const TString& hostName)
-{
- NYql::TIssues issues;
- if (!owner) {
- issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "owner is not specified"));
- }
-
- if (!hostName) {
- issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "host name is not specified"));
- }
-
- return issues;
-}
-
+ return issues;
+}
+
+NYql::TIssues ValidateGetTask(const TString& owner, const TString& hostName)
+{
+ NYql::TIssues issues;
+ if (!owner) {
+ issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "owner is not specified"));
+ }
+
+ if (!hostName) {
+ issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "host name is not specified"));
+ }
+
+ return issues;
+}
+
NYql::TIssues ValidatePingTask(const TString& scope, const TString& queryId, const TString& owner, const TInstant& deadline, const TDuration& ttl)
-{
- NYql::TIssues issues;
- if (!scope) {
- issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "scope is not specified"));
- }
-
- if (!queryId) {
- issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "query id is not specified"));
- }
-
- if (!owner) {
- issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "owner is not specified"));
- }
-
+{
+ NYql::TIssues issues;
+ if (!scope) {
+ issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "scope is not specified"));
+ }
+
+ if (!queryId) {
+ issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "query id is not specified"));
+ }
+
+ if (!owner) {
+ issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "owner is not specified"));
+ }
+
const auto hardLimit = TInstant::Now() + ttl;
if (deadline > hardLimit) {
issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "deadline " + deadline.ToString() + " must be less than " + hardLimit.ToString()));
}
- return issues;
-}
-
-NYql::TIssues ValidateNodesHealthCheck(
- const TString& tenant,
- const TString& instanceId,
- const TString& hostName
- )
-{
- NYql::TIssues issues;
- if (!tenant) {
- issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "tenant is not specified"));
- }
-
- if (!instanceId) {
- issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "instance id is not specified"));
- }
-
- if (!hostName) {
- issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "hostName is not specified"));
- }
-
- return issues;
-}
-
-}; \ No newline at end of file
+ return issues;
+}
+
+NYql::TIssues ValidateNodesHealthCheck(
+ const TString& tenant,
+ const TString& instanceId,
+ const TString& hostName
+ )
+{
+ NYql::TIssues issues;
+ if (!tenant) {
+ issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "tenant is not specified"));
+ }
+
+ if (!instanceId) {
+ issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "instance id is not specified"));
+ }
+
+ if (!hostName) {
+ issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "hostName is not specified"));
+ }
+
+ return issues;
+}
+
+}; \ No newline at end of file
diff --git a/ydb/core/yq/libs/control_plane_storage/internal/utils.h b/ydb/core/yq/libs/control_plane_storage/internal/utils.h
index 58ffe6b340..2fd0f708d1 100644
--- a/ydb/core/yq/libs/control_plane_storage/internal/utils.h
+++ b/ydb/core/yq/libs/control_plane_storage/internal/utils.h
@@ -1,5 +1,5 @@
-#pragma once
-
+#pragma once
+
#include <tuple>
#include <ydb/public/sdk/cpp/client/ydb_value/value.h>
@@ -8,25 +8,25 @@
#include <ydb/core/yq/libs/config/protos/issue_id.pb.h>
#include <ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage_impl.h>
-
-namespace NYq {
-
-struct TTopicConsumerLess {
- bool operator()(const Yq::Private::TopicConsumer& c1, const Yq::Private::TopicConsumer& c2) const {
- // Cluster endpoint/use ssl are not in key
- return std::tie(c1.database_id(), c1.database(), c1.topic_path(), c1.consumer_name()) < std::tie(c2.database_id(), c2.database(), c2.topic_path(), c2.consumer_name());
- }
-};
-
+
+namespace NYq {
+
+struct TTopicConsumerLess {
+ bool operator()(const Yq::Private::TopicConsumer& c1, const Yq::Private::TopicConsumer& c2) const {
+ // Cluster endpoint/use ssl are not in key
+ return std::tie(c1.database_id(), c1.database(), c1.topic_path(), c1.consumer_name()) < std::tie(c2.database_id(), c2.database(), c2.topic_path(), c2.consumer_name());
+ }
+};
+
NYql::TIssues ValidateWriteResultData(const TString& resultId, const Ydb::ResultSet& resultSet, const TInstant& deadline, const TDuration& ttl);
-
-NYql::TIssues ValidateGetTask(const TString& owner, const TString& hostName);
-
+
+NYql::TIssues ValidateGetTask(const TString& owner, const TString& hostName);
+
NYql::TIssues ValidatePingTask(const TString& scope, const TString& queryId, const TString& owner, const TInstant& deadline, const TDuration& ttl);
-
-NYql::TIssues ValidateNodesHealthCheck(
- const TString& tenant,
- const TString& instanceId,
- const TString& hostName);
-
+
+NYql::TIssues ValidateNodesHealthCheck(
+ const TString& tenant,
+ const TString& instanceId,
+ const TString& hostName);
+
};
diff --git a/ydb/core/yq/libs/control_plane_storage/internal/ya.make b/ydb/core/yq/libs/control_plane_storage/internal/ya.make
index 2e268259eb..f38b3b070c 100644
--- a/ydb/core/yq/libs/control_plane_storage/internal/ya.make
+++ b/ydb/core/yq/libs/control_plane_storage/internal/ya.make
@@ -1,20 +1,20 @@
OWNER(g:yq)
-
-LIBRARY()
-
-SRCS(
- nodes_health_check.cpp
- response_tasks.cpp
- task_get.cpp
- task_ping.cpp
- task_result_write.cpp
- utils.cpp
-)
-
-PEERDIR(
- library/cpp/actors/core
- library/cpp/lwtrace/mon
- library/cpp/monlib/service/pages
+
+LIBRARY()
+
+SRCS(
+ nodes_health_check.cpp
+ response_tasks.cpp
+ task_get.cpp
+ task_ping.cpp
+ task_result_write.cpp
+ utils.cpp
+)
+
+PEERDIR(
+ library/cpp/actors/core
+ library/cpp/lwtrace/mon
+ library/cpp/monlib/service/pages
ydb/core/base
ydb/core/mon
ydb/core/yq/libs/common
@@ -25,8 +25,8 @@ PEERDIR(
ydb/public/sdk/cpp/client/ydb_scheme
ydb/public/sdk/cpp/client/ydb_value
ydb/library/yql/public/issue
-)
-
-YQL_LAST_ABI_VERSION()
-
-END()
+)
+
+YQL_LAST_ABI_VERSION()
+
+END()
diff --git a/ydb/core/yq/libs/control_plane_storage/message_builders.h b/ydb/core/yq/libs/control_plane_storage/message_builders.h
index d7ebdb7098..75ee6f1f9f 100644
--- a/ydb/core/yq/libs/control_plane_storage/message_builders.h
+++ b/ydb/core/yq/libs/control_plane_storage/message_builders.h
@@ -151,7 +151,7 @@ public:
Request.set_query_id(queryId);
return *this;
}
-
+
TDeleteQueryBuilder& SetIdempotencyKey(const TString& idempotencyKey)
{
Request.set_idempotency_key(idempotencyKey);
@@ -249,12 +249,12 @@ public:
return *this;
}
- TModifyQueryBuilder& SetDescription(const TString& description)
- {
- Request.mutable_content()->set_description(description);
- return *this;
- }
-
+ TModifyQueryBuilder& SetDescription(const TString& description)
+ {
+ Request.mutable_content()->set_description(description);
+ return *this;
+ }
+
const YandexQuery::ModifyQueryRequest& Build()
{
return Request;
@@ -281,7 +281,7 @@ public:
Request.set_query_id(queryId);
return *this;
}
-
+
TControlQueryBuilder& SetIdempotencyKey(const TString& idempotencyKey)
{
Request.set_idempotency_key(idempotencyKey);
@@ -625,7 +625,7 @@ public:
TModifyConnectionBuilder& SetDescription(const TString& description)
{
- Request.mutable_content()->set_description(description);
+ Request.mutable_content()->set_description(description);
return *this;
}
@@ -662,7 +662,7 @@ public:
Request.set_connection_id(connectionId);
return *this;
}
-
+
TDeleteConnectionBuilder& SetIdempotencyKey(const TString& idempotencyKey)
{
Request.set_idempotency_key(idempotencyKey);
@@ -885,7 +885,7 @@ public:
Request.set_binding_id(bindingId);
return *this;
}
-
+
TDeleteBindingBuilder& SetIdempotencyKey(const TString& idempotencyKey)
{
Request.set_idempotency_key(idempotencyKey);
@@ -907,7 +907,7 @@ public:
// internal
class TWriteResultDataBuilder {
- TString ResultId;
+ TString ResultId;
int32_t ResultSetId = 0;
int64_t StartRowId = 0;
TInstant Deadline;
@@ -923,9 +923,9 @@ public:
SetResultSet(resultSet);
}
- TWriteResultDataBuilder& SetResultId(const TString& resultId)
+ TWriteResultDataBuilder& SetResultId(const TString& resultId)
{
- ResultId = resultId;
+ ResultId = resultId;
return *this;
}
@@ -955,7 +955,7 @@ public:
std::unique_ptr<TEvControlPlaneStorage::TEvWriteResultDataRequest> Build()
{
- return std::make_unique<TEvControlPlaneStorage::TEvWriteResultDataRequest>(ResultId, ResultSetId, StartRowId, Deadline, ResultSet);
+ return std::make_unique<TEvControlPlaneStorage::TEvWriteResultDataRequest>(ResultId, ResultSetId, StartRowId, Deadline, ResultSet);
}
};
@@ -984,15 +984,15 @@ public:
std::unique_ptr<TEvControlPlaneStorage::TEvGetTaskRequest> Build()
{
- return std::make_unique<TEvControlPlaneStorage::TEvGetTaskRequest>(Owner, HostName);
+ return std::make_unique<TEvControlPlaneStorage::TEvGetTaskRequest>(Owner, HostName);
}
};
class TPingTaskBuilder {
TString Scope;
TString QueryId;
- TString ResultId;
- TString Owner;
+ TString ResultId;
+ TString Owner;
TInstant Deadline;
TMaybe<YandexQuery::QueryMeta::ComputeStatus> Status;
TMaybe<NYql::TIssues> Issues;
@@ -1026,15 +1026,15 @@ public:
return *this;
}
- TPingTaskBuilder& SetResultId(const TString& resultId)
- {
- ResultId = resultId;
- return *this;
- }
-
- TPingTaskBuilder& SetOwner(const TString& owner)
+ TPingTaskBuilder& SetResultId(const TString& resultId)
{
- Owner = owner;
+ ResultId = resultId;
+ return *this;
+ }
+
+ TPingTaskBuilder& SetOwner(const TString& owner)
+ {
+ Owner = owner;
return *this;
}
@@ -1142,31 +1142,31 @@ public:
}
};
-class TNodesHealthCheckBuilder {
- TString Tenant;
+class TNodesHealthCheckBuilder {
+ TString Tenant;
ui32 NodeId = 0;
- TString HostName;
- TString InstanceId;
+ TString HostName;
+ TString InstanceId;
ui64 ActiveWorkers = 0;
ui64 MemoryLimit = 0;
ui64 MemoryAllocated = 0;
-public:
- TNodesHealthCheckBuilder()
- {}
-
- TNodesHealthCheckBuilder& SetTenant(const TString& tenant)
- {
- Tenant = tenant;
- return *this;
- }
-
- TNodesHealthCheckBuilder& SetNodeId(const ui32& nodeId)
- {
- NodeId = nodeId;
- return *this;
- }
-
+public:
+ TNodesHealthCheckBuilder()
+ {}
+
+ TNodesHealthCheckBuilder& SetTenant(const TString& tenant)
+ {
+ Tenant = tenant;
+ return *this;
+ }
+
+ TNodesHealthCheckBuilder& SetNodeId(const ui32& nodeId)
+ {
+ NodeId = nodeId;
+ return *this;
+ }
+
TNodesHealthCheckBuilder& SetHostName(const TString& hostName)
{
HostName = hostName;
@@ -1179,28 +1179,28 @@ public:
return *this;
}
- TNodesHealthCheckBuilder& SetActiveWorkers(const ui64& activeWorkers)
- {
- ActiveWorkers = activeWorkers;
- return *this;
- }
-
+ TNodesHealthCheckBuilder& SetActiveWorkers(const ui64& activeWorkers)
+ {
+ ActiveWorkers = activeWorkers;
+ return *this;
+ }
+
TNodesHealthCheckBuilder& SetMemoryLimit(const ui64& memoryLimit)
- {
+ {
MemoryLimit = memoryLimit;
- return *this;
- }
-
+ return *this;
+ }
+
TNodesHealthCheckBuilder& SetMemoryAllocated(const ui64& memoryAllocated)
- {
+ {
MemoryAllocated = memoryAllocated;
- return *this;
- }
-
- std::unique_ptr<TEvControlPlaneStorage::TEvNodesHealthCheckRequest> Build()
- {
- Yq::Private::NodesHealthCheckRequest request;
- request.set_tenant(Tenant);
+ return *this;
+ }
+
+ std::unique_ptr<TEvControlPlaneStorage::TEvNodesHealthCheckRequest> Build()
+ {
+ Yq::Private::NodesHealthCheckRequest request;
+ request.set_tenant(Tenant);
auto& node = *request.mutable_node();
node.set_node_id(NodeId);
node.set_instance_id(InstanceId);
@@ -1208,8 +1208,8 @@ public:
node.set_active_workers(ActiveWorkers);
node.set_memory_limit(MemoryLimit);
node.set_memory_allocated(MemoryAllocated);
- return std::make_unique<TEvControlPlaneStorage::TEvNodesHealthCheckRequest>(std::move(request));
- }
-};
-
+ return std::make_unique<TEvControlPlaneStorage::TEvNodesHealthCheckRequest>(std::move(request));
+ }
+};
+
}
diff --git a/ydb/core/yq/libs/control_plane_storage/probes.h b/ydb/core/yq/libs/control_plane_storage/probes.h
index e4322110cc..c82c68e5ef 100644
--- a/ydb/core/yq/libs/control_plane_storage/probes.h
+++ b/ydb/core/yq/libs/control_plane_storage/probes.h
@@ -86,7 +86,7 @@
PROBE(WriteResultDataRequest, \
GROUPS(), \
TYPES(TString, i32, i64, i64, TDuration, TInstant, i64, bool), \
- NAMES("resultId", "resulSetId", "startRowId", "countRows", "latencyMs", "deadlineSec", "size", "success")) \
+ NAMES("resultId", "resulSetId", "startRowId", "countRows", "latencyMs", "deadlineSec", "size", "success")) \
PROBE(GetTaskRequest, \
GROUPS(), \
TYPES(TString, TString, TDuration, bool), \
@@ -95,10 +95,10 @@
GROUPS(), \
TYPES(TString, TDuration, bool), \
NAMES("queryId", "latencyMs", "success")) \
- PROBE(NodesHealthCheckRequest, \
- GROUPS(), \
- TYPES(TString, ui32, TString, TString, TDuration, bool), \
- NAMES("tenant", "nodeId", "instanceId", "hostName", "latencyMs", "success")) \
+ PROBE(NodesHealthCheckRequest, \
+ GROUPS(), \
+ TYPES(TString, ui32, TString, TString, TDuration, bool), \
+ NAMES("tenant", "nodeId", "instanceId", "hostName", "latencyMs", "success")) \
// YQ_CONTROL_PLANE_STORAGE_PROVIDER
diff --git a/ydb/core/yq/libs/control_plane_storage/proto/yq_internal.proto b/ydb/core/yq/libs/control_plane_storage/proto/yq_internal.proto
index c941f1e965..e6ebff0c84 100644
--- a/ydb/core/yq/libs/control_plane_storage/proto/yq_internal.proto
+++ b/ydb/core/yq/libs/control_plane_storage/proto/yq_internal.proto
@@ -26,7 +26,7 @@ message QueryInternal {
ExecuteMode execute_mode = 9;
StateLoadMode state_load_mode = 10;
string cloud_id = 11;
- repeated Yq.Private.TopicConsumer created_topic_consumers = 12;
+ repeated Yq.Private.TopicConsumer created_topic_consumers = 12;
repeated bytes dq_graph = 13;
int32 dq_graph_index = 14;
StreamingDisposition disposition = 15;
diff --git a/ydb/core/yq/libs/control_plane_storage/schema.h b/ydb/core/yq/libs/control_plane_storage/schema.h
index a1c6700df5..613eb7bcc1 100644
--- a/ydb/core/yq/libs/control_plane_storage/schema.h
+++ b/ydb/core/yq/libs/control_plane_storage/schema.h
@@ -4,7 +4,7 @@ namespace NYq {
// tables
#define QUERIES_TABLE_NAME "queries"
-#define PENDING_SMALL_TABLE_NAME "pending_small"
+#define PENDING_SMALL_TABLE_NAME "pending_small"
#define CONNECTIONS_TABLE_NAME "connections"
#define BINDINGS_TABLE_NAME "bindings"
#define RESULT_SETS_TABLE_NAME "result_sets"
@@ -21,10 +21,10 @@ namespace NYq {
#define META_REVISION_COLUMN_NAME "meta_revision"
#define INTERNAL_COLUMN_NAME "internal"
#define GENERATION_COLUMN_NAME "generation"
-#define RETRY_COUNTER_COLUMN_NAME "retry_counter"
-#define RETRY_COUNTER_UPDATE_COLUMN_NAME "retry_counter_updated_at"
-#define IS_RESIGN_QUERY_COLUMN_NAME "is_resign_query"
-
+#define RETRY_COUNTER_COLUMN_NAME "retry_counter"
+#define RETRY_COUNTER_UPDATE_COLUMN_NAME "retry_counter_updated_at"
+#define IS_RESIGN_QUERY_COLUMN_NAME "is_resign_query"
+
#define STATUS_COLUMN_NAME "status"
#define QUERY_TYPE_COLUMN_NAME "query_type"
#define EXECUTE_MODE_COLUMN_NAME "execute_mode"
@@ -60,8 +60,8 @@ namespace NYq {
#define ACTIVE_WORKERS_COLUMN_NAME "active_workers"
#define MEMORY_LIMIT_COLUMN_NAME "memory_limit"
#define MEMORY_ALLOCATED_COLUMN_NAME "memory_allocated"
-#define INTERCONNECT_PORT_COLUMN_NAME "interconnect_port"
-#define NODE_ADDRESS_COLUMN_NAME "node_address"
+#define INTERCONNECT_PORT_COLUMN_NAME "interconnect_port"
+#define NODE_ADDRESS_COLUMN_NAME "node_address"
#define HOST_NAME_COLUMN_NAME "hostname"
#define OWNER_COLUMN_NAME "owner"
diff --git a/ydb/core/yq/libs/control_plane_storage/util.cpp b/ydb/core/yq/libs/control_plane_storage/util.cpp
index ca9281b495..288f25a1cb 100644
--- a/ydb/core/yq/libs/control_plane_storage/util.cpp
+++ b/ydb/core/yq/libs/control_plane_storage/util.cpp
@@ -43,47 +43,47 @@ NConfig::TControlPlaneStorageConfig FillDefaultParameters(NConfig::TControlPlane
if (!config.GetMaxCountJobs()) {
config.SetMaxCountJobs(20);
}
-
- if (!config.GetTasksBatchSize()) {
- config.SetTasksBatchSize(100);
- }
-
- if (!config.GetNumTasksProportion()) {
- config.SetNumTasksProportion(4);
- }
-
- if (!config.GetNumTasksProportion()) {
- config.SetNumTasksProportion(4);
- }
-
- if (!config.GetAnalyticsRetryCounterLimit()) {
- config.SetAnalyticsRetryCounterLimit(20);
- }
-
- if (!config.GetStreamingRetryCounterLimit()) {
- config.SetStreamingRetryCounterLimit(20);
- }
-
+
+ if (!config.GetTasksBatchSize()) {
+ config.SetTasksBatchSize(100);
+ }
+
+ if (!config.GetNumTasksProportion()) {
+ config.SetNumTasksProportion(4);
+ }
+
+ if (!config.GetNumTasksProportion()) {
+ config.SetNumTasksProportion(4);
+ }
+
+ if (!config.GetAnalyticsRetryCounterLimit()) {
+ config.SetAnalyticsRetryCounterLimit(20);
+ }
+
+ if (!config.GetStreamingRetryCounterLimit()) {
+ config.SetStreamingRetryCounterLimit(20);
+ }
+
if (!config.GetAnalyticsRetryCounterUpdateTime()) {
config.SetAnalyticsRetryCounterUpdateTime("1d");
- }
-
+ }
+
if (!config.GetStreamingRetryCounterUpdateTime()) {
config.SetStreamingRetryCounterUpdateTime("1d");
- }
-
+ }
+
if (!config.GetAutomaticQueriesTtl()) {
config.SetAutomaticQueriesTtl("1d");
}
-
+
if (!config.GetTaskLeaseTtl()) {
config.SetTaskLeaseTtl("30s");
- }
+ }
if (!config.GetStorage().GetToken() && config.GetStorage().GetOAuthFile()) {
config.MutableStorage()->SetToken(StripString(TFileInput(config.GetStorage().GetOAuthFile()).ReadAll()));
}
-
+
if (!config.GetResultSetsTtl()) {
config.SetResultSetsTtl("1d");
}
@@ -92,39 +92,39 @@ NConfig::TControlPlaneStorageConfig FillDefaultParameters(NConfig::TControlPlane
}
bool DoesPingTaskUpdateQueriesTable(const TEvControlPlaneStorage::TEvPingTaskRequest* request) {
- if (!request) {
- return false;
- }
- return request->Status ||
- request->Issues ||
- request->TransientIssues ||
- request->Statistics ||
+ if (!request) {
+ return false;
+ }
+ return request->Status ||
+ request->Issues ||
+ request->TransientIssues ||
+ request->Statistics ||
request->ResultSetMetas ||
- request->Ast ||
- request->Plan ||
- request->StartedAt ||
+ request->Ast ||
+ request->Plan ||
+ request->StartedAt ||
request->FinishedAt ||
!request->CreatedTopicConsumers.empty() ||
!request->DqGraphs.empty() ||
request->DqGraphIndex ||
request->StateLoadMode ||
request->StreamingDisposition;
-}
-
-NYdb::TValue PackItemsToList(const TVector<NYdb::TValue>& items) {
- NYdb::TValueBuilder itemsAsList;
- itemsAsList.BeginList();
- for (const NYdb::TValue& item: items) {
- itemsAsList.AddListItem(item);
- }
- itemsAsList.EndList();
- return itemsAsList.Build();
-}
-
-std::pair<TString, TString> SplitId(const TString& id, char delim) {
- auto it = std::find(id.begin(), id.end(), delim);
- return std::make_pair(id.substr(0, it - id.begin()),
- (it != id.end() ? id.substr(it - id.begin() + 1) : TString{""}));
-}
-
+}
+
+NYdb::TValue PackItemsToList(const TVector<NYdb::TValue>& items) {
+ NYdb::TValueBuilder itemsAsList;
+ itemsAsList.BeginList();
+ for (const NYdb::TValue& item: items) {
+ itemsAsList.AddListItem(item);
+ }
+ itemsAsList.EndList();
+ return itemsAsList.Build();
+}
+
+std::pair<TString, TString> SplitId(const TString& id, char delim) {
+ auto it = std::find(id.begin(), id.end(), delim);
+ return std::make_pair(id.substr(0, it - id.begin()),
+ (it != id.end() ? id.substr(it - id.begin() + 1) : TString{""}));
+}
+
} //namespace NYq
diff --git a/ydb/core/yq/libs/control_plane_storage/util.h b/ydb/core/yq/libs/control_plane_storage/util.h
index 9edc20d84e..8deddc25ca 100644
--- a/ydb/core/yq/libs/control_plane_storage/util.h
+++ b/ydb/core/yq/libs/control_plane_storage/util.h
@@ -25,9 +25,9 @@ auto CreateArray(const T(&list)[N]) -> std::array<T, K> {
}
bool DoesPingTaskUpdateQueriesTable(const TEvControlPlaneStorage::TEvPingTaskRequest* request);
-
-NYdb::TValue PackItemsToList(const TVector<NYdb::TValue>& items);
-
-std::pair<TString, TString> SplitId(const TString& id, char delim = '-');
-
-} // namespace NYq
+
+NYdb::TValue PackItemsToList(const TVector<NYdb::TValue>& items);
+
+std::pair<TString, TString> SplitId(const TString& id, char delim = '-');
+
+} // namespace NYq
diff --git a/ydb/core/yq/libs/control_plane_storage/ya.make b/ydb/core/yq/libs/control_plane_storage/ya.make
index b8f9939079..8304e50db5 100644
--- a/ydb/core/yq/libs/control_plane_storage/ya.make
+++ b/ydb/core/yq/libs/control_plane_storage/ya.make
@@ -3,7 +3,7 @@ OWNER(g:yq)
LIBRARY()
SRCS(
- control_plane_storage_counters.cpp
+ control_plane_storage_counters.cpp
exceptions.cpp
in_memory_control_plane_storage.cpp
probes.cpp
diff --git a/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage.cpp b/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage.cpp
index 5cdddd2619..222ebb6e8e 100644
--- a/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage.cpp
+++ b/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage.cpp
@@ -37,18 +37,18 @@ void TYdbControlPlaneStorageActor::Bootstrap() {
CPS_LOG_I("Starting ydb control plane storage service. Actor id: " << SelfId());
NLwTraceMonPage::ProbeRegistry().AddProbesList(LWTRACE_GET_PROBES(YQ_CONTROL_PLANE_STORAGE_PROVIDER));
- DbPool = YqSharedResources->DbPoolHolder->GetOrCreate(EDbPoolId::MAIN, 10);
+ DbPool = YqSharedResources->DbPoolHolder->GetOrCreate(EDbPoolId::MAIN, 10);
YdbConnection = NewYdbConnection(Config.Proto.GetStorage(), CredProviderFactory);
auto as = NActors::TActivationContext::ActorSystem();
CreateDirectory(as);
CreateQueriesTable(as);
- CreatePendingSmallTable(as);
+ CreatePendingSmallTable(as);
CreateConnectionsTable(as);
CreateBindingsTable(as);
CreateIdempotencyKeysTable(as);
CreateResultSetsTable(as);
CreateJobsTable(as);
- CreateNodesTable(as);
+ CreateNodesTable(as);
Become(&TThis::StateFunc);
}
@@ -81,8 +81,8 @@ TAsyncStatus TYdbControlPlaneStorageActor::CreateQueriesTable(TActorSystem* as)
auto description = TTableBuilder()
.AddNullableColumn(SCOPE_COLUMN_NAME, EPrimitiveType::String)
.AddNullableColumn(QUERY_ID_COLUMN_NAME, EPrimitiveType::String)
- .AddNullableColumn(RESULT_ID_COLUMN_NAME, EPrimitiveType::String)
- .AddNullableColumn(GENERATION_COLUMN_NAME, EPrimitiveType::Uint64)
+ .AddNullableColumn(RESULT_ID_COLUMN_NAME, EPrimitiveType::String)
+ .AddNullableColumn(GENERATION_COLUMN_NAME, EPrimitiveType::Uint64)
.AddNullableColumn(NAME_COLUMN_NAME, EPrimitiveType::String)
.AddNullableColumn(USER_COLUMN_NAME, EPrimitiveType::String)
.AddNullableColumn(VISIBILITY_COLUMN_NAME, EPrimitiveType::Int64)
@@ -115,20 +115,20 @@ TAsyncStatus TYdbControlPlaneStorageActor::CreateQueriesTable(TActorSystem* as)
});
}
-TAsyncStatus TYdbControlPlaneStorageActor::CreatePendingSmallTable(TActorSystem* as)
-{
- auto tablePath = JoinPath(YdbConnection->TablePathPrefix, PENDING_SMALL_TABLE_NAME);
-
- auto description = TTableBuilder()
- .AddNullableColumn(SCOPE_COLUMN_NAME, EPrimitiveType::String)
- .AddNullableColumn(QUERY_ID_COLUMN_NAME, EPrimitiveType::String)
+TAsyncStatus TYdbControlPlaneStorageActor::CreatePendingSmallTable(TActorSystem* as)
+{
+ auto tablePath = JoinPath(YdbConnection->TablePathPrefix, PENDING_SMALL_TABLE_NAME);
+
+ auto description = TTableBuilder()
+ .AddNullableColumn(SCOPE_COLUMN_NAME, EPrimitiveType::String)
+ .AddNullableColumn(QUERY_ID_COLUMN_NAME, EPrimitiveType::String)
.AddNullableColumn(LAST_SEEN_AT_COLUMN_NAME, EPrimitiveType::Timestamp)
- .AddNullableColumn(RETRY_COUNTER_COLUMN_NAME, EPrimitiveType::Uint64)
- .AddNullableColumn(RETRY_COUNTER_UPDATE_COLUMN_NAME, EPrimitiveType::Timestamp)
- .AddNullableColumn(QUERY_TYPE_COLUMN_NAME, EPrimitiveType::Int64)
- .AddNullableColumn(IS_RESIGN_QUERY_COLUMN_NAME, EPrimitiveType::Bool)
- .AddNullableColumn(HOST_NAME_COLUMN_NAME, EPrimitiveType::String)
- .AddNullableColumn(OWNER_COLUMN_NAME, EPrimitiveType::String)
+ .AddNullableColumn(RETRY_COUNTER_COLUMN_NAME, EPrimitiveType::Uint64)
+ .AddNullableColumn(RETRY_COUNTER_UPDATE_COLUMN_NAME, EPrimitiveType::Timestamp)
+ .AddNullableColumn(QUERY_TYPE_COLUMN_NAME, EPrimitiveType::Int64)
+ .AddNullableColumn(IS_RESIGN_QUERY_COLUMN_NAME, EPrimitiveType::Bool)
+ .AddNullableColumn(HOST_NAME_COLUMN_NAME, EPrimitiveType::String)
+ .AddNullableColumn(OWNER_COLUMN_NAME, EPrimitiveType::String)
.SetPrimaryKeyColumns({SCOPE_COLUMN_NAME, QUERY_ID_COLUMN_NAME})
.Build();
@@ -140,7 +140,7 @@ TAsyncStatus TYdbControlPlaneStorageActor::CreatePendingSmallTable(TActorSystem*
auto status = future.GetValue();
if (!IsTableCreated(status)) {
CPS_LOG_AS_E(*as, "create pending table error: " << status.GetIssues().ToString());
- return CreatePendingSmallTable(as);
+ return CreatePendingSmallTable(as);
}
return future;
});
@@ -202,7 +202,7 @@ TAsyncStatus TYdbControlPlaneStorageActor::CreateJobsTable(TActorSystem* as)
.AddNullableColumn(USER_COLUMN_NAME, EPrimitiveType::String)
.AddNullableColumn(VISIBILITY_COLUMN_NAME, EPrimitiveType::Int64)
.AddNullableColumn(EXPIRE_AT_COLUMN_NAME, EPrimitiveType::Timestamp)
- .SetPrimaryKeyColumns({SCOPE_COLUMN_NAME, QUERY_ID_COLUMN_NAME, JOB_ID_COLUMN_NAME})
+ .SetPrimaryKeyColumns({SCOPE_COLUMN_NAME, QUERY_ID_COLUMN_NAME, JOB_ID_COLUMN_NAME})
.SetTtlSettings(EXPIRE_AT_COLUMN_NAME)
.Build();
@@ -220,39 +220,39 @@ TAsyncStatus TYdbControlPlaneStorageActor::CreateJobsTable(TActorSystem* as)
});
}
-TAsyncStatus TYdbControlPlaneStorageActor::CreateNodesTable(TActorSystem* as)
-{
+TAsyncStatus TYdbControlPlaneStorageActor::CreateNodesTable(TActorSystem* as)
+{
auto tablePath = JoinPath(YdbConnection->TablePathPrefix, NODES_TABLE_NAME);
-
- auto description = TTableBuilder()
- .AddNullableColumn(TENANT_COLUMN_NAME, EPrimitiveType::String)
- .AddNullableColumn(INSTANCE_ID_COLUMN_NAME, EPrimitiveType::String)
- .AddNullableColumn(NODE_ID_COLUMN_NAME, EPrimitiveType::Uint32)
- .AddNullableColumn(HOST_NAME_COLUMN_NAME, EPrimitiveType::String)
- .AddNullableColumn(ACTIVE_WORKERS_COLUMN_NAME, EPrimitiveType::Uint64)
+
+ auto description = TTableBuilder()
+ .AddNullableColumn(TENANT_COLUMN_NAME, EPrimitiveType::String)
+ .AddNullableColumn(INSTANCE_ID_COLUMN_NAME, EPrimitiveType::String)
+ .AddNullableColumn(NODE_ID_COLUMN_NAME, EPrimitiveType::Uint32)
+ .AddNullableColumn(HOST_NAME_COLUMN_NAME, EPrimitiveType::String)
+ .AddNullableColumn(ACTIVE_WORKERS_COLUMN_NAME, EPrimitiveType::Uint64)
.AddNullableColumn(MEMORY_LIMIT_COLUMN_NAME, EPrimitiveType::Uint64)
.AddNullableColumn(MEMORY_ALLOCATED_COLUMN_NAME, EPrimitiveType::Uint64)
- .AddNullableColumn(EXPIRE_AT_COLUMN_NAME, EPrimitiveType::Timestamp)
- .AddNullableColumn(INTERCONNECT_PORT_COLUMN_NAME, EPrimitiveType::Uint32)
- .AddNullableColumn(NODE_ADDRESS_COLUMN_NAME, EPrimitiveType::String)
- .SetTtlSettings(EXPIRE_AT_COLUMN_NAME)
+ .AddNullableColumn(EXPIRE_AT_COLUMN_NAME, EPrimitiveType::Timestamp)
+ .AddNullableColumn(INTERCONNECT_PORT_COLUMN_NAME, EPrimitiveType::Uint32)
+ .AddNullableColumn(NODE_ADDRESS_COLUMN_NAME, EPrimitiveType::String)
+ .SetTtlSettings(EXPIRE_AT_COLUMN_NAME)
.SetPrimaryKeyColumns({TENANT_COLUMN_NAME, NODE_ID_COLUMN_NAME})
- .Build();
-
- return YdbConnection->Client.RetryOperation(
- [tablePath = std::move(tablePath), description = std::move(description)] (TSession session) mutable {
- return session.CreateTable(tablePath, TTableDescription(description));
- })
- .Apply([=](const auto& future) {
- auto status = future.GetValue();
+ .Build();
+
+ return YdbConnection->Client.RetryOperation(
+ [tablePath = std::move(tablePath), description = std::move(description)] (TSession session) mutable {
+ return session.CreateTable(tablePath, TTableDescription(description));
+ })
+ .Apply([=](const auto& future) {
+ auto status = future.GetValue();
if (!IsTableCreated(status)) {
- CPS_LOG_AS_E(*as, "create nodes table error: " << status.GetIssues().ToString());
- return CreateNodesTable(as);
- }
- return future;
- });
-}
-
+ CPS_LOG_AS_E(*as, "create nodes table error: " << status.GetIssues().ToString());
+ return CreateNodesTable(as);
+ }
+ return future;
+ });
+}
+
TAsyncStatus TYdbControlPlaneStorageActor::CreateBindingsTable(TActorSystem* as)
{
auto tablePath = JoinPath(YdbConnection->TablePathPrefix, BINDINGS_TABLE_NAME);
@@ -286,7 +286,7 @@ TAsyncStatus TYdbControlPlaneStorageActor::CreateBindingsTable(TActorSystem* as)
TAsyncStatus TYdbControlPlaneStorageActor::CreateIdempotencyKeysTable(TActorSystem* as)
{
-
+
auto tablePath = JoinPath(YdbConnection->TablePathPrefix, IDEMPOTENCY_KEYS_TABLE_NAME);
auto description = TTableBuilder()
@@ -317,13 +317,13 @@ TAsyncStatus TYdbControlPlaneStorageActor::CreateResultSetsTable(TActorSystem* a
auto tablePath = JoinPath(YdbConnection->TablePathPrefix, RESULT_SETS_TABLE_NAME);
auto description = TTableBuilder()
- .AddNullableColumn(RESULT_ID_COLUMN_NAME, EPrimitiveType::String)
+ .AddNullableColumn(RESULT_ID_COLUMN_NAME, EPrimitiveType::String)
.AddNullableColumn(RESULT_SET_ID_COLUMN_NAME, EPrimitiveType::Int32)
.AddNullableColumn(ROW_ID_COLUMN_NAME, EPrimitiveType::Int64)
.AddNullableColumn(RESULT_SET_COLUMN_NAME, EPrimitiveType::String)
.AddNullableColumn(EXPIRE_AT_COLUMN_NAME, EPrimitiveType::Timestamp)
.SetTtlSettings(EXPIRE_AT_COLUMN_NAME)
- .SetPrimaryKeyColumns({RESULT_ID_COLUMN_NAME, RESULT_SET_ID_COLUMN_NAME, ROW_ID_COLUMN_NAME})
+ .SetPrimaryKeyColumns({RESULT_ID_COLUMN_NAME, RESULT_SET_ID_COLUMN_NAME, ROW_ID_COLUMN_NAME})
.Build();
return YdbConnection->Client.RetryOperation(
@@ -373,31 +373,31 @@ void TYdbControlPlaneStorageActor::ReadIdempotencyKeyQuery(TSqlQueryBuilder& bui
class TDbRequest: public NActors::TActorBootstrapped<TDbRequest> {
using TFunction = std::function<NYdb::TAsyncStatus(NYdb::NTable::TSession&)>;
- TDbPool::TPtr DbPool;
+ TDbPool::TPtr DbPool;
TPromise<NYdb::TStatus> Promise;
TFunction Handler;
public:
- TDbRequest(const TDbPool::TPtr& dbPool, const TPromise<NYdb::TStatus>& promise, const TFunction& handler)
+ TDbRequest(const TDbPool::TPtr& dbPool, const TPromise<NYdb::TStatus>& promise, const TFunction& handler)
: DbPool(dbPool)
, Promise(promise)
, Handler(handler)
{}
static constexpr char ActorName[] = "YQ_CONTROL_PLANE_STORAGE_DB_REQUEST";
-
+
void Bootstrap() {
CPS_LOG_T("DbRequest actor request. Actor id: " << SelfId());
Become(&TDbRequest::StateFunc);
- Send(DbPool->GetNextActor(), new TEvents::TEvDbFunctionRequest(Handler), IEventHandle::FlagTrackDelivery);
+ Send(DbPool->GetNextActor(), new TEvents::TEvDbFunctionRequest(Handler), IEventHandle::FlagTrackDelivery);
}
STRICT_STFUNC(StateFunc,
- hFunc(TEvents::TEvDbFunctionResponse, Handle);
+ hFunc(TEvents::TEvDbFunctionResponse, Handle);
hFunc(NActors::TEvents::TEvUndelivered, OnUndelivered)
)
- void Handle(TEvents::TEvDbFunctionResponse::TPtr& ev) {
+ void Handle(TEvents::TEvDbFunctionResponse::TPtr& ev) {
CPS_LOG_T("DbRequest actor response. Actor id: " << SelfId());
Promise.SetValue(ev->Get()->Status);
PassAway();
@@ -405,21 +405,21 @@ public:
void OnUndelivered(NActors::TEvents::TEvUndelivered::TPtr&) {
CPS_LOG_E("On delivered. Actor id: " << SelfId());
- Send(DbPool->GetNextActor(), new TEvents::TEvDbFunctionRequest(Handler), IEventHandle::FlagTrackDelivery);
+ Send(DbPool->GetNextActor(), new TEvents::TEvDbFunctionRequest(Handler), IEventHandle::FlagTrackDelivery);
}
};
-std::pair<TAsyncStatus, std::shared_ptr<TVector<NYdb::TResultSet>>> TYdbControlPlaneStorageActor::Read(
- const TString& query,
- const NYdb::TParams& params,
- const TRequestCountersPtr& requestCounters,
- TDebugInfoPtr debugInfo,
- TTxSettings transactionMode,
- bool retryOnTli)
+std::pair<TAsyncStatus, std::shared_ptr<TVector<NYdb::TResultSet>>> TYdbControlPlaneStorageActor::Read(
+ const TString& query,
+ const NYdb::TParams& params,
+ const TRequestCountersPtr& requestCounters,
+ TDebugInfoPtr debugInfo,
+ TTxSettings transactionMode,
+ bool retryOnTli)
{
auto resultSet = std::make_shared<TVector<NYdb::TResultSet>>();
-
+
std::shared_ptr<int> retryCount = std::make_shared<int>();
auto handler = [=](TSession& session) {
if (*retryCount != 0) {
@@ -427,17 +427,17 @@ std::pair<TAsyncStatus, std::shared_ptr<TVector<NYdb::TResultSet>>> TYdbControlP
}
++(*retryCount);
CollectDebugInfo(query, params, session, debugInfo);
- auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx(transactionMode).CommitTx(), params, NYdb::NTable::TExecDataQuerySettings().KeepInQueryCache(true));
- return result.Apply([retryOnTli, resultSet] (const TFuture<TDataQueryResult>& future) {
+ auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx(transactionMode).CommitTx(), params, NYdb::NTable::TExecDataQuerySettings().KeepInQueryCache(true));
+ return result.Apply([retryOnTli, resultSet] (const TFuture<TDataQueryResult>& future) {
NYdb::NTable::TDataQueryResult result = future.GetValue();
*resultSet = result.GetResultSets();
auto status = static_cast<TStatus>(result);
if (status.GetStatus() == EStatus::SCHEME_ERROR) { // retry if table does not exist
return TStatus{EStatus::UNAVAILABLE, NYql::TIssues{status.GetIssues()}};
}
- if (!retryOnTli && status.GetStatus() == EStatus::ABORTED) {
- return TStatus{EStatus::GENERIC_ERROR, NYql::TIssues{status.GetIssues()}};
- }
+ if (!retryOnTli && status.GetStatus() == EStatus::ABORTED) {
+ return TStatus{EStatus::GENERIC_ERROR, NYql::TIssues{status.GetIssues()}};
+ }
return status;
});
};
@@ -447,15 +447,15 @@ std::pair<TAsyncStatus, std::shared_ptr<TVector<NYdb::TResultSet>>> TYdbControlP
return {promise.GetFuture(), resultSet};
}
-TAsyncStatus TYdbControlPlaneStorageActor::Validate(
- std::shared_ptr<TMaybe<TTransaction>> transaction,
- size_t item,
- const TVector<TValidationQuery>& validators,
- TSession session,
- std::shared_ptr<bool> successFinish,
- TDebugInfoPtr debugInfo,
- TTxSettings transactionMode)
-{
+TAsyncStatus TYdbControlPlaneStorageActor::Validate(
+ std::shared_ptr<TMaybe<TTransaction>> transaction,
+ size_t item,
+ const TVector<TValidationQuery>& validators,
+ TSession session,
+ std::shared_ptr<bool> successFinish,
+ TDebugInfoPtr debugInfo,
+ TTxSettings transactionMode)
+{
if (item >= validators.size()) {
return MakeFuture(TStatus{EStatus::SUCCESS, NYql::TIssues{}});
}
@@ -481,30 +481,30 @@ TAsyncStatus TYdbControlPlaneStorageActor::Validate(
});
}
-TAsyncStatus TYdbControlPlaneStorageActor::Write(
- NActors::TActorSystem* actorSystem,
- const TString& query,
- const NYdb::TParams& params,
- const TRequestCountersPtr& requestCounters,
- TDebugInfoPtr debugInfo,
- const TVector<TValidationQuery>& validators,
- TTxSettings transactionMode,
- bool retryOnTli)
+TAsyncStatus TYdbControlPlaneStorageActor::Write(
+ NActors::TActorSystem* actorSystem,
+ const TString& query,
+ const NYdb::TParams& params,
+ const TRequestCountersPtr& requestCounters,
+ TDebugInfoPtr debugInfo,
+ const TVector<TValidationQuery>& validators,
+ TTxSettings transactionMode,
+ bool retryOnTli)
{
std::shared_ptr<int> retryCount = std::make_shared<int>();
auto transaction = std::make_shared<TMaybe<TTransaction>>();
- auto writeHandler = [=, retryOnTli=retryOnTli] (TSession session) {
+ auto writeHandler = [=, retryOnTli=retryOnTli] (TSession session) {
CollectDebugInfo(query, params, session, debugInfo);
- auto result = session.ExecuteDataQuery(query, validators ? TTxControl::Tx(**transaction).CommitTx() : TTxControl::BeginTx(transactionMode).CommitTx(), params, NYdb::NTable::TExecDataQuerySettings().KeepInQueryCache(true));
- return result.Apply([=] (const TFuture<TDataQueryResult>& future) {
+ auto result = session.ExecuteDataQuery(query, validators ? TTxControl::Tx(**transaction).CommitTx() : TTxControl::BeginTx(transactionMode).CommitTx(), params, NYdb::NTable::TExecDataQuerySettings().KeepInQueryCache(true));
+ return result.Apply([=] (const TFuture<TDataQueryResult>& future) {
NYdb::NTable::TDataQueryResult result = future.GetValue();
auto status = static_cast<TStatus>(result);
if (status.GetStatus() == EStatus::SCHEME_ERROR) { // retry if table does not exist
return TStatus{EStatus::UNAVAILABLE, NYql::TIssues{status.GetIssues()}};
}
- if (!retryOnTli && status.GetStatus() == EStatus::ABORTED) {
- return TStatus{EStatus::GENERIC_ERROR, NYql::TIssues{status.GetIssues()}};
- }
+ if (!retryOnTli && status.GetStatus() == EStatus::ABORTED) {
+ return TStatus{EStatus::GENERIC_ERROR, NYql::TIssues{status.GetIssues()}};
+ }
return status;
});
};
@@ -542,42 +542,42 @@ TAsyncStatus TYdbControlPlaneStorageActor::Write(
return promise.GetFuture();
}
-NThreading::TFuture<void> TYdbControlPlaneStorageActor::PickTask(
- const TPickTaskParams& taskParams,
- const TRequestCountersPtr& requestCounters,
- TDebugInfoPtr debugInfo,
- std::shared_ptr<TResponseTasks> responseTasks,
- const TVector<TValidationQuery>& validators,
- TTxSettings transactionMode)
+NThreading::TFuture<void> TYdbControlPlaneStorageActor::PickTask(
+ const TPickTaskParams& taskParams,
+ const TRequestCountersPtr& requestCounters,
+ TDebugInfoPtr debugInfo,
+ std::shared_ptr<TResponseTasks> responseTasks,
+ const TVector<TValidationQuery>& validators,
+ TTxSettings transactionMode)
+{
+ return ReadModifyWrite(NActors::TActivationContext::ActorSystem(), taskParams.ReadQuery, taskParams.ReadParams,
+ taskParams.PrepareParams, requestCounters, debugInfo, validators, transactionMode, taskParams.RetryOnTli)
+ .Apply([=, responseTasks=responseTasks, queryId = taskParams.QueryId](const auto& future) {
+ const auto status = future.GetValue();
+ if (responseTasks && status.GetStatus() == EStatus::GENERIC_ERROR) {
+ responseTasks->SafeEraseTaskBlocking(queryId);
+ }
+ });
+}
+
+TAsyncStatus TYdbControlPlaneStorageActor::ReadModifyWrite(
+ NActors::TActorSystem* actorSystem,
+ const TString& readQuery,
+ const NYdb::TParams& readParams,
+ const std::function<std::pair<TString, NYdb::TParams>(const TVector<NYdb::TResultSet>&)>& prepare,
+ const TRequestCountersPtr& requestCounters,
+ TDebugInfoPtr debugInfo,
+ const TVector<TValidationQuery>& validators,
+ TTxSettings transactionMode,
+ bool retryOnTli)
{
- return ReadModifyWrite(NActors::TActivationContext::ActorSystem(), taskParams.ReadQuery, taskParams.ReadParams,
- taskParams.PrepareParams, requestCounters, debugInfo, validators, transactionMode, taskParams.RetryOnTli)
- .Apply([=, responseTasks=responseTasks, queryId = taskParams.QueryId](const auto& future) {
- const auto status = future.GetValue();
- if (responseTasks && status.GetStatus() == EStatus::GENERIC_ERROR) {
- responseTasks->SafeEraseTaskBlocking(queryId);
- }
- });
-}
-
-TAsyncStatus TYdbControlPlaneStorageActor::ReadModifyWrite(
- NActors::TActorSystem* actorSystem,
- const TString& readQuery,
- const NYdb::TParams& readParams,
- const std::function<std::pair<TString, NYdb::TParams>(const TVector<NYdb::TResultSet>&)>& prepare,
- const TRequestCountersPtr& requestCounters,
- TDebugInfoPtr debugInfo,
- const TVector<TValidationQuery>& validators,
- TTxSettings transactionMode,
- bool retryOnTli)
-{
std::shared_ptr<int> retryCount = std::make_shared<int>();
auto resultSets = std::make_shared<TVector<NYdb::TResultSet>>();
auto transaction = std::make_shared<TMaybe<TTransaction>>();
auto readModifyWriteHandler = [=](TSession session) {
CollectDebugInfo(readQuery, readParams, session, debugInfo);
- auto readResult = session.ExecuteDataQuery(readQuery, validators ? TTxControl::Tx(**transaction) : TTxControl::BeginTx(transactionMode), readParams, NYdb::NTable::TExecDataQuerySettings().KeepInQueryCache(true));
+ auto readResult = session.ExecuteDataQuery(readQuery, validators ? TTxControl::Tx(**transaction) : TTxControl::BeginTx(transactionMode), readParams, NYdb::NTable::TExecDataQuerySettings().KeepInQueryCache(true));
auto readResultStatus = readResult.Apply([resultSets, transaction] (const TFuture<TDataQueryResult>& future) {
NYdb::NTable::TDataQueryResult result = future.GetValue();
*resultSets = result.GetResultSets();
@@ -597,7 +597,7 @@ TAsyncStatus TYdbControlPlaneStorageActor::ReadModifyWrite(
if (!readResultStatus.GetValue().IsSuccess()) {
return readResultStatus;
}
-
+
try {
auto [writeQuery, params] = future.GetValue();
if (!writeQuery) {
@@ -612,15 +612,15 @@ TAsyncStatus TYdbControlPlaneStorageActor::ReadModifyWrite(
}
CollectDebugInfo(writeQuery, params, session, debugInfo);
auto writeResult = session.ExecuteDataQuery(writeQuery, TTxControl::Tx(**transaction).CommitTx(), params, NYdb::NTable::TExecDataQuerySettings().KeepInQueryCache(true));
- return writeResult.Apply([retryOnTli] (const TFuture<TDataQueryResult>& future) {
+ return writeResult.Apply([retryOnTli] (const TFuture<TDataQueryResult>& future) {
NYdb::NTable::TDataQueryResult result = future.GetValue();
auto status = static_cast<TStatus>(result);
if (status.GetStatus() == EStatus::SCHEME_ERROR) { // retry if table does not exist
return TStatus{EStatus::UNAVAILABLE, NYql::TIssues{status.GetIssues()}};
}
- if (!retryOnTli && status.GetStatus() == EStatus::ABORTED) {
- return TStatus{EStatus::GENERIC_ERROR, NYql::TIssues{status.GetIssues()}};
- }
+ if (!retryOnTli && status.GetStatus() == EStatus::ABORTED) {
+ return TStatus{EStatus::GENERIC_ERROR, NYql::TIssues{status.GetIssues()}};
+ }
return status;
});
} catch (const TControlPlaneStorageException& exception) {
@@ -634,7 +634,7 @@ TAsyncStatus TYdbControlPlaneStorageActor::ReadModifyWrite(
return MakeFuture(TStatus{EStatus::GENERIC_ERROR, NYql::TIssues{NYql::TIssue{CurrentExceptionMessage()}}});
}
});
- };
+ };
auto handler = [=] (TSession session) {
if (*retryCount != 0) {
diff --git a/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage_bindings.cpp b/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage_bindings.cpp
index 4612cce5bf..fc617fe5f0 100644
--- a/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage_bindings.cpp
+++ b/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage_bindings.cpp
@@ -64,7 +64,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvCreateBindi
response->second.After.ConstructInPlace().CopyFrom(binding);
response->second.CloudId = cloudId;
- TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "CreateBinding");
+ TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "CreateBinding");
queryBuilder.AddString("scope", scope);
queryBuilder.AddString("binding_id", bindingId);
queryBuilder.AddString("connection_id", connectionId);
@@ -172,7 +172,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvListBinding
return;
}
- TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "ListBindings");
+ TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "ListBindings");
queryBuilder.AddString("scope", scope);
queryBuilder.AddString("last_binding", pageToken);
queryBuilder.AddUint64("limit", limit + 1);
@@ -307,7 +307,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvDescribeBin
return;
}
- TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "DescribeBinding");
+ TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "DescribeBinding");
queryBuilder.AddString("scope", scope);
queryBuilder.AddString("binding_id", bindingId);
queryBuilder.AddText(
@@ -395,7 +395,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvModifyBindi
return;
}
- TSqlQueryBuilder readQueryBuilder(YdbConnection->TablePathPrefix, "ModifyBinding(read)");
+ TSqlQueryBuilder readQueryBuilder(YdbConnection->TablePathPrefix, "ModifyBinding(read)");
readQueryBuilder.AddString("scope", scope);
readQueryBuilder.AddString("binding_id", bindingId);
readQueryBuilder.AddText(
@@ -451,7 +451,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvModifyBindi
response->second.After.ConstructInPlace().CopyFrom(binding);
response->second.CloudId = bindingInternal.cloud_id();
- TSqlQueryBuilder writeQueryBuilder(YdbConnection->TablePathPrefix, "ModifyBinding(write)");
+ TSqlQueryBuilder writeQueryBuilder(YdbConnection->TablePathPrefix, "ModifyBinding(write)");
writeQueryBuilder.AddString("scope", scope);
writeQueryBuilder.AddString("binding_id", bindingId);
writeQueryBuilder.AddInt64("visibility", YandexQuery::Acl::SCOPE); // TODO: fix me
@@ -572,7 +572,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvDeleteBindi
std::shared_ptr<std::pair<YandexQuery::DeleteBindingResult, TAuditDetails<YandexQuery::Binding>>> response = std::make_shared<std::pair<YandexQuery::DeleteBindingResult, TAuditDetails<YandexQuery::Binding>>>();
- TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "DeleteBinding");
+ TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "DeleteBinding");
queryBuilder.AddString("scope", scope);
queryBuilder.AddString("binding_id", bindingId);
diff --git a/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage_connections.cpp b/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage_connections.cpp
index a9b920ad9e..ddb15b16c4 100644
--- a/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage_connections.cpp
+++ b/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage_connections.cpp
@@ -49,7 +49,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvCreateConne
const TDuration delta = TInstant::Now() - startTime;
SendResponseIssues<TEvControlPlaneStorage::TEvCreateConnectionResponse>(ev->Sender, issues, ev->Cookie, delta, requestCounters);
LWPROBE(CreateConnectionRequest, scope, user, delta, byteSize, false);
- return;
+ return;
}
YandexQuery::Connection connection;
@@ -65,7 +65,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvCreateConne
response->second.After.ConstructInPlace().CopyFrom(connection);
response->second.CloudId = cloudId;
- TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "CreateConnection");
+ TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "CreateConnection");
queryBuilder.AddString("scope", scope);
queryBuilder.AddString("connection_id", connectionId);
queryBuilder.AddString("user", user);
@@ -161,10 +161,10 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvListConnect
const TDuration delta = TInstant::Now() - startTime;
SendResponseIssues<TEvControlPlaneStorage::TEvListConnectionsResponse>(ev->Sender, issues, ev->Cookie, delta, requestCounters);
LWPROBE(ListConnectionsRequest, scope, user, delta, byteSize, false);
- return;
+ return;
}
- TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "ListConnections");
+ TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "ListConnections");
queryBuilder.AddString("scope", scope);
queryBuilder.AddString("last_connection", pageToken);
queryBuilder.AddUint64("limit", limit + 1);
@@ -286,10 +286,10 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvDescribeCon
const TDuration delta = TInstant::Now() - startTime;
SendResponseIssues<TEvControlPlaneStorage::TEvDescribeConnectionResponse>(ev->Sender, issues, ev->Cookie, delta, requestCounters);
LWPROBE(DescribeConnectionRequest, scope, connectionId, user, delta, byteSize, false);
- return;
+ return;
}
- TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "DescribeConnection");
+ TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "DescribeConnection");
queryBuilder.AddString("scope", scope);
queryBuilder.AddString("connection_id", connectionId);
@@ -381,10 +381,10 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvModifyConne
const TDuration delta = TInstant::Now() - startTime;
SendResponseIssues<TEvControlPlaneStorage::TEvModifyConnectionResponse>(ev->Sender, issues, ev->Cookie, delta, requestCounters);
LWPROBE(ModifyConnectionRequest, scope, connectionId, user, delta, byteSize, false);
- return;
+ return;
}
-
- TSqlQueryBuilder readQueryBuilder(YdbConnection->TablePathPrefix, "ModifyConnection(read)");
+
+ TSqlQueryBuilder readQueryBuilder(YdbConnection->TablePathPrefix, "ModifyConnection(read)");
readQueryBuilder.AddString("scope", scope);
readQueryBuilder.AddString("connection_id", connectionId);
readQueryBuilder.AddText(
@@ -441,7 +441,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvModifyConne
response->second.After.ConstructInPlace().CopyFrom(connection);
response->second.CloudId = connectionInternal.cloud_id();
- TSqlQueryBuilder writeQueryBuilder(YdbConnection->TablePathPrefix, "ModifyConnection(write)");
+ TSqlQueryBuilder writeQueryBuilder(YdbConnection->TablePathPrefix, "ModifyConnection(write)");
writeQueryBuilder.AddString("scope", scope);
writeQueryBuilder.AddString("connection_id", connectionId);
writeQueryBuilder.AddInt64("visibility", connection.content().acl().visibility());
@@ -545,7 +545,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvDeleteConne
<< "DeleteConnectionRequest: "
<< NKikimr::MaskTicket(token) << " "
<< request.DebugString());
-
+
NYql::TIssues issues = ValidateEvent(ev);
if (issues) {
CPS_LOG_D(MakeLogPrefix(scope, user, connectionId)
@@ -556,12 +556,12 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvDeleteConne
const TDuration delta = TInstant::Now() - startTime;
SendResponseIssues<TEvControlPlaneStorage::TEvDeleteConnectionResponse>(ev->Sender, issues, ev->Cookie, delta, requestCounters);
LWPROBE(DeleteConnectionRequest, scope, connectionId, user, delta, byteSize, false);
- return;
+ return;
}
-
+
std::shared_ptr<std::pair<YandexQuery::DeleteConnectionResult, TAuditDetails<YandexQuery::Connection>>> response = std::make_shared<std::pair<YandexQuery::DeleteConnectionResult, TAuditDetails<YandexQuery::Connection>>>();
- TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "DeleteConnection");
+ TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "DeleteConnection");
queryBuilder.AddString("scope", scope);
queryBuilder.AddString("connection_id", connectionId);
diff --git a/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage_impl.h b/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage_impl.h
index f338968f79..1c8cda7ee7 100644
--- a/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage_impl.h
+++ b/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage_impl.h
@@ -1,7 +1,7 @@
#pragma once
#include "control_plane_storage.h"
-#include "control_plane_storage_counters.h"
+#include "control_plane_storage_counters.h"
#include "exceptions.h"
#include "extractors.h"
#include <ydb/core/yq/libs/control_plane_storage/internal/response_tasks.h>
@@ -113,7 +113,7 @@ class TYdbControlPlaneStorageActor : public NActors::TActorBootstrapped<TYdbCont
RT_WRITE_RESULT_DATA,
RT_GET_TASK,
RT_PING_TASK,
- RT_NODES_HEALTH_CHECK,
+ RT_NODES_HEALTH_CHECK,
RT_MAX,
};
@@ -142,7 +142,7 @@ class TYdbControlPlaneStorageActor : public NActors::TActorBootstrapped<TYdbCont
{ MakeIntrusive<TRequestCounters>("WriteResultData") },
{ MakeIntrusive<TRequestCounters>("GetTask") },
{ MakeIntrusive<TRequestCounters>("PingTask") },
- { MakeIntrusive<TRequestCounters>("NodesHealthCheck") },
+ { MakeIntrusive<TRequestCounters>("NodesHealthCheck") },
});
NMonitoring::TDynamicCounterPtr Counters;
@@ -172,17 +172,17 @@ class TYdbControlPlaneStorageActor : public NActors::TActorBootstrapped<TYdbCont
};
TCounters Counters;
- TFinalStatusCounters FinalStatusCounters;
+ TFinalStatusCounters FinalStatusCounters;
TConfig Config;
TYdbConnectionPtr YdbConnection;
::NYq::TYqSharedResources::TPtr YqSharedResources;
- TDbPool::TPtr DbPool;
+ TDbPool::TPtr DbPool;
static constexpr int64_t InitialRevision = 1;
-
+
NKikimr::TYdbCredentialsProviderFactory CredProviderFactory;
public:
@@ -193,15 +193,15 @@ public:
const ::NYq::TYqSharedResources::TPtr& yqSharedResources,
const NKikimr::TYdbCredentialsProviderFactory& credProviderFactory)
: Counters(counters)
- , FinalStatusCounters(counters)
+ , FinalStatusCounters(counters)
, Config(config, common)
, YqSharedResources(yqSharedResources)
, CredProviderFactory(credProviderFactory)
{
}
- static constexpr char ActorName[] = "YQ_CONTROL_PLANE_STORAGE";
-
+ static constexpr char ActorName[] = "YQ_CONTROL_PLANE_STORAGE";
+
void Bootstrap();
STRICT_STFUNC(StateFunc,
@@ -228,7 +228,7 @@ public:
hFunc(TEvControlPlaneStorage::TEvWriteResultDataRequest, Handle);
hFunc(TEvControlPlaneStorage::TEvGetTaskRequest, Handle);
hFunc(TEvControlPlaneStorage::TEvPingTaskRequest, Handle);
- hFunc(TEvControlPlaneStorage::TEvNodesHealthCheckRequest, Handle);
+ hFunc(TEvControlPlaneStorage::TEvNodesHealthCheckRequest, Handle);
hFunc(NMon::TEvHttpInfo, Handle);
)
@@ -259,8 +259,8 @@ public:
void Handle(TEvControlPlaneStorage::TEvGetTaskRequest::TPtr& ev);
void Handle(TEvControlPlaneStorage::TEvPingTaskRequest::TPtr& ev);
- void Handle(TEvControlPlaneStorage::TEvNodesHealthCheckRequest::TPtr& ev);
-
+ void Handle(TEvControlPlaneStorage::TEvNodesHealthCheckRequest::TPtr& ev);
+
template<typename T>
NYql::TIssues ValidateConnection(T& ev, bool clickHousePasswordRequire = true)
{
@@ -512,10 +512,10 @@ public:
TAsyncStatus CreateDirectory(TActorSystem* as);
TAsyncStatus CreateQueriesTable(TActorSystem* as);
TAsyncStatus CreatePendingTable(TActorSystem* as);
- TAsyncStatus CreatePendingSmallTable(TActorSystem* as);
+ TAsyncStatus CreatePendingSmallTable(TActorSystem* as);
TAsyncStatus CreateConnectionsTable(TActorSystem* as);
TAsyncStatus CreateJobsTable(TActorSystem* as);
- TAsyncStatus CreateNodesTable(TActorSystem* as);
+ TAsyncStatus CreateNodesTable(TActorSystem* as);
TAsyncStatus CreateBindingsTable(TActorSystem* as);
TAsyncStatus CreateIdempotencyKeysTable(TActorSystem* as);
TAsyncStatus CreateResultSetsTable(TActorSystem* as);
@@ -530,42 +530,42 @@ private:
void ReadIdempotencyKeyQuery(TSqlQueryBuilder& builder, const TString& scope, const TString& idempotencyKey);
- std::pair<TAsyncStatus, std::shared_ptr<TVector<NYdb::TResultSet>>> Read(
- const TString& query,
- const NYdb::TParams& params,
- const TRequestCountersPtr& requestCounters,
- TDebugInfoPtr debugInfo,
- TTxSettings transactionMode = TTxSettings::SerializableRW(),
- bool retryOnTli = true);
-
- TAsyncStatus Validate(
- std::shared_ptr<TMaybe<TTransaction>> transaction,
- size_t item, const TVector<TValidationQuery>& validators,
- TSession session,
- std::shared_ptr<bool> successFinish,
- TDebugInfoPtr debugInfo,
- TTxSettings transactionMode = TTxSettings::SerializableRW());
-
- TAsyncStatus Write(
- NActors::TActorSystem* actorSystem,
- const TString& query,
- const NYdb::TParams& params,
- const TRequestCountersPtr& requestCounters,
- TDebugInfoPtr debugInfo,
- const TVector<TValidationQuery>& validators = {},
- TTxSettings transactionMode = TTxSettings::SerializableRW(),
- bool retryTli = true);
-
- TAsyncStatus ReadModifyWrite(
- NActors::TActorSystem* actorSystem,
- const TString& readQuery,
- const NYdb::TParams& readParams,
- const std::function<std::pair<TString, NYdb::TParams>(const TVector<NYdb::TResultSet>&)>& prepare,
- const TRequestCountersPtr& requestCounters,
- TDebugInfoPtr debugInfo = {},
- const TVector<TValidationQuery>& validators = {},
- TTxSettings transactionMode = TTxSettings::SerializableRW(),
- bool retryOnTli = true);
+ std::pair<TAsyncStatus, std::shared_ptr<TVector<NYdb::TResultSet>>> Read(
+ const TString& query,
+ const NYdb::TParams& params,
+ const TRequestCountersPtr& requestCounters,
+ TDebugInfoPtr debugInfo,
+ TTxSettings transactionMode = TTxSettings::SerializableRW(),
+ bool retryOnTli = true);
+
+ TAsyncStatus Validate(
+ std::shared_ptr<TMaybe<TTransaction>> transaction,
+ size_t item, const TVector<TValidationQuery>& validators,
+ TSession session,
+ std::shared_ptr<bool> successFinish,
+ TDebugInfoPtr debugInfo,
+ TTxSettings transactionMode = TTxSettings::SerializableRW());
+
+ TAsyncStatus Write(
+ NActors::TActorSystem* actorSystem,
+ const TString& query,
+ const NYdb::TParams& params,
+ const TRequestCountersPtr& requestCounters,
+ TDebugInfoPtr debugInfo,
+ const TVector<TValidationQuery>& validators = {},
+ TTxSettings transactionMode = TTxSettings::SerializableRW(),
+ bool retryTli = true);
+
+ TAsyncStatus ReadModifyWrite(
+ NActors::TActorSystem* actorSystem,
+ const TString& readQuery,
+ const NYdb::TParams& readParams,
+ const std::function<std::pair<TString, NYdb::TParams>(const TVector<NYdb::TResultSet>&)>& prepare,
+ const TRequestCountersPtr& requestCounters,
+ TDebugInfoPtr debugInfo = {},
+ const TVector<TValidationQuery>& validators = {},
+ TTxSettings transactionMode = TTxSettings::SerializableRW(),
+ bool retryOnTli = true);
template<typename T>
THashMap<TString, T> GetEntitiesWithVisibilityPriority(const TResultSet& resultSet, const TString& columnName)
@@ -821,22 +821,22 @@ private:
static TString MakeLogPrefix(const TString& scope, const TString& user, const TString& id = "") {
return "[" + scope + ", " + user + (id ? ", " + id : "") + "] ";
}
-
- struct TPickTaskParams {
- TString ReadQuery;
- TParams ReadParams;
- std::function<std::pair<TString, NYdb::TParams>(const TVector<NYdb::TResultSet>&)> PrepareParams;
- TString QueryId;
- bool RetryOnTli = false;
- };
-
- NThreading::TFuture<void> PickTask(
- const TPickTaskParams& taskParams,
- const TRequestCountersPtr& requestCounters,
- TDebugInfoPtr debugInfo,
- std::shared_ptr<TResponseTasks> responseTasks,
- const TVector<TValidationQuery>& validators = {},
- TTxSettings transactionMode = TTxSettings::SerializableRW());
+
+ struct TPickTaskParams {
+ TString ReadQuery;
+ TParams ReadParams;
+ std::function<std::pair<TString, NYdb::TParams>(const TVector<NYdb::TResultSet>&)> PrepareParams;
+ TString QueryId;
+ bool RetryOnTli = false;
+ };
+
+ NThreading::TFuture<void> PickTask(
+ const TPickTaskParams& taskParams,
+ const TRequestCountersPtr& requestCounters,
+ TDebugInfoPtr debugInfo,
+ std::shared_ptr<TResponseTasks> responseTasks,
+ const TVector<TValidationQuery>& validators = {},
+ TTxSettings transactionMode = TTxSettings::SerializableRW());
};
}
diff --git a/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage_queries.cpp b/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage_queries.cpp
index df7b3c9dde..398725f7f4 100644
--- a/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage_queries.cpp
+++ b/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage_queries.cpp
@@ -84,7 +84,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvCreateQuery
const TDuration delta = TInstant::Now() - startTime;
SendResponseIssues<TEvControlPlaneStorage::TEvCreateQueryResponse>(ev->Sender, issues, ev->Cookie, delta, requestCounters);
LWPROBE(CreateQueryRequest, scope, user, delta, byteSize, false);
- return;
+ return;
}
const TString idempotencyKey = request.idempotency_key();
@@ -116,7 +116,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvCreateQuery
response->first.set_query_id(queryId);
response->second.CloudId = cloudId;
- TSqlQueryBuilder readQueryBuilder(YdbConnection->TablePathPrefix, "CreateQuery(read)");
+ TSqlQueryBuilder readQueryBuilder(YdbConnection->TablePathPrefix, "CreateQuery(read)");
ReadIdempotencyKeyQuery(readQueryBuilder, scope, idempotencyKey);
if (request.execute_mode() != YandexQuery::SAVE) {
@@ -200,7 +200,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvCreateQuery
response->second.After.ConstructInPlace().CopyFrom(query);
- TSqlQueryBuilder writeQueryBuilder(YdbConnection->TablePathPrefix, "CreateQuery(write)");
+ TSqlQueryBuilder writeQueryBuilder(YdbConnection->TablePathPrefix, "CreateQuery(write)");
writeQueryBuilder.AddString("scope", scope);
writeQueryBuilder.AddString("query_id", queryId);
writeQueryBuilder.AddString("name", query.content().name());
@@ -224,20 +224,20 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvCreateQuery
// insert job
writeQueryBuilder.AddText(
- "INSERT INTO `" JOBS_TABLE_NAME "`\n"
- "(`" SCOPE_COLUMN_NAME "`, `" QUERY_ID_COLUMN_NAME "`, `" JOB_ID_COLUMN_NAME "`, \n"
- "`" JOB_COLUMN_NAME "`, `" USER_COLUMN_NAME "`, `" VISIBILITY_COLUMN_NAME "`)\n"
- "VALUES\n"
- " ($scope, $query_id, $job_id, $job, $user, $visibility);"
+ "INSERT INTO `" JOBS_TABLE_NAME "`\n"
+ "(`" SCOPE_COLUMN_NAME "`, `" QUERY_ID_COLUMN_NAME "`, `" JOB_ID_COLUMN_NAME "`, \n"
+ "`" JOB_COLUMN_NAME "`, `" USER_COLUMN_NAME "`, `" VISIBILITY_COLUMN_NAME "`)\n"
+ "VALUES\n"
+ " ($scope, $query_id, $job_id, $job, $user, $visibility);"
);
// insert pending small
writeQueryBuilder.AddText(
- "INSERT INTO `" PENDING_SMALL_TABLE_NAME "`\n"
- "(`" SCOPE_COLUMN_NAME "`, `" QUERY_ID_COLUMN_NAME "`, `" QUERY_TYPE_COLUMN_NAME "`, `" LAST_SEEN_AT_COLUMN_NAME "`,\n"
- "`" RETRY_COUNTER_COLUMN_NAME "`, `" RETRY_COUNTER_UPDATE_COLUMN_NAME "`, `" HOST_NAME_COLUMN_NAME "`, `" OWNER_COLUMN_NAME "`)\n"
- "VALUES\n"
- " ($scope, $query_id, $query_type, $zero_timestamp, 0, $now, \"\", \"\");"
+ "INSERT INTO `" PENDING_SMALL_TABLE_NAME "`\n"
+ "(`" SCOPE_COLUMN_NAME "`, `" QUERY_ID_COLUMN_NAME "`, `" QUERY_TYPE_COLUMN_NAME "`, `" LAST_SEEN_AT_COLUMN_NAME "`,\n"
+ "`" RETRY_COUNTER_COLUMN_NAME "`, `" RETRY_COUNTER_UPDATE_COLUMN_NAME "`, `" HOST_NAME_COLUMN_NAME "`, `" OWNER_COLUMN_NAME "`)\n"
+ "VALUES\n"
+ " ($scope, $query_id, $query_type, $zero_timestamp, 0, $now, \"\", \"\");"
);
}
@@ -292,8 +292,8 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvListQueries
if (IsSuperUser(user)) {
permissions.SetAll();
}
-
- const TString pageToken = request.page_token();
+
+ const TString pageToken = request.page_token();
const int byteSize = request.ByteSize();
const int64_t limit = request.limit();
@@ -312,10 +312,10 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvListQueries
const TDuration delta = TInstant::Now() - startTime;
SendResponseIssues<TEvControlPlaneStorage::TEvListQueriesResponse>(ev->Sender, issues, ev->Cookie, delta, requestCounters);
LWPROBE(ListQueriesRequest, scope, user, delta, byteSize, false);
- return;
+ return;
}
- TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "ListQueries");
+ TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "ListQueries");
queryBuilder.AddString("scope", scope);
queryBuilder.AddString("last_query", pageToken);
queryBuilder.AddTimestamp("now", TInstant::Now());
@@ -412,8 +412,8 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvListQueries
ythrow TControlPlaneStorageException(TIssuesIds::INTERNAL_ERROR) << "Error parsing proto message for query. Please contact internal support";
}
YandexQuery::BriefQuery briefQuery;
- const auto lastJobId = query.meta().last_job_id();
- query.mutable_meta()->set_last_job_id(lastJobId + "-" + query.meta().common().id());
+ const auto lastJobId = query.meta().last_job_id();
+ query.mutable_meta()->set_last_job_id(lastJobId + "-" + query.meta().common().id());
*briefQuery.mutable_meta() = query.meta();
briefQuery.set_name(query.content().name());
briefQuery.set_type(query.content().type());
@@ -478,10 +478,10 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvDescribeQue
const TDuration delta = TInstant::Now() - startTime;
SendResponseIssues<TEvControlPlaneStorage::TEvDescribeQueryResponse>(ev->Sender, issues, ev->Cookie, delta, requestCounters);
LWPROBE(DescribeQueryRequest, scope, user, queryId, delta, byteSize, false);
- return;
+ return;
}
- TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "DescribeQuery");
+ TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "DescribeQuery");
queryBuilder.AddString("scope", scope);
queryBuilder.AddString("query_id", queryId);
queryBuilder.AddTimestamp("now", TInstant::Now());
@@ -507,9 +507,9 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvDescribeQue
ythrow TControlPlaneStorageException(TIssuesIds::INTERNAL_ERROR) << "Error parsing proto message for query. Please contact internal support";
}
- const auto lastJobId = result.query().meta().last_job_id();
- result.mutable_query()->mutable_meta()->set_last_job_id(lastJobId + "-" + result.query().meta().common().id());
-
+ const auto lastJobId = result.query().meta().last_job_id();
+ result.mutable_query()->mutable_meta()->set_last_job_id(lastJobId + "-" + result.query().meta().common().id());
+
const auto queryVisibility = result.query().content().acl().visibility();
const auto queryUser = result.query().meta().common().created_by();
const bool hasViewAccess = HasViewAccess(permissions, queryVisibility, queryUser, user);
@@ -576,7 +576,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvGetQuerySta
return;
}
- TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "GetQueryStatus");
+ TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "GetQueryStatus");
queryBuilder.AddString("scope", scope);
queryBuilder.AddString("query_id", queryId);
queryBuilder.AddTimestamp("now", TInstant::Now());
@@ -679,15 +679,15 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvModifyQuery
const TDuration delta = TInstant::Now() - startTime;
SendResponseIssues<TEvControlPlaneStorage::TEvModifyQueryResponse>(ev->Sender, issues, ev->Cookie, delta, requestCounters);
LWPROBE(ModifyQueryRequest, scope, user, queryId, delta, byteSize, false);
- return;
+ return;
}
const TString idempotencyKey = request.idempotency_key();
- std::shared_ptr<std::pair<YandexQuery::ModifyQueryResult, TAuditDetails<YandexQuery::Query>>> response =
- std::make_shared<std::pair<YandexQuery::ModifyQueryResult, TAuditDetails<YandexQuery::Query>>>();
+ std::shared_ptr<std::pair<YandexQuery::ModifyQueryResult, TAuditDetails<YandexQuery::Query>>> response =
+ std::make_shared<std::pair<YandexQuery::ModifyQueryResult, TAuditDetails<YandexQuery::Query>>>();
- TSqlQueryBuilder readQueryBuilder(YdbConnection->TablePathPrefix, "ModifyQuery(read)");
+ TSqlQueryBuilder readQueryBuilder(YdbConnection->TablePathPrefix, "ModifyQuery(read)");
readQueryBuilder.AddString("scope", scope);
readQueryBuilder.AddString("query_id", queryId);
readQueryBuilder.AddTimestamp("now", TInstant::Now());
@@ -700,7 +700,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvModifyQuery
"SELECT `" CONNECTION_ID_COLUMN_NAME "`, `" CONNECTION_COLUMN_NAME "` FROM `" CONNECTIONS_TABLE_NAME "`\n"
"WHERE `" SCOPE_COLUMN_NAME "` = $scope AND (`" VISIBILITY_COLUMN_NAME "` = $scope_visibility OR `" USER_COLUMN_NAME "` = $user);\n"
);
-
+
// user bindings
readQueryBuilder.AddText(
"SELECT `" BINDING_ID_COLUMN_NAME "`, `" BINDING_COLUMN_NAME "` FROM `" BINDINGS_TABLE_NAME "`\n"
@@ -709,7 +709,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvModifyQuery
}
readQueryBuilder.AddText(
- "SELECT `" QUERY_COLUMN_NAME "`, `" INTERNAL_COLUMN_NAME "`, `" RESULT_ID_COLUMN_NAME "` FROM `" QUERIES_TABLE_NAME "`\n"
+ "SELECT `" QUERY_COLUMN_NAME "`, `" INTERNAL_COLUMN_NAME "`, `" RESULT_ID_COLUMN_NAME "` FROM `" QUERIES_TABLE_NAME "`\n"
"WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id AND (`" EXPIRE_AT_COLUMN_NAME "` is NULL OR `" EXPIRE_AT_COLUMN_NAME "` > $now);"
);
@@ -736,8 +736,8 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvModifyQuery
ythrow TControlPlaneStorageException(TIssuesIds::INTERNAL_ERROR) << "Error parsing proto message for query internal. Please contact internal support";
}
- const TString resultId = request.execute_mode() == YandexQuery::SAVE ? parser.ColumnParser(RESULT_ID_COLUMN_NAME).GetOptionalString().GetOrElse("") : "";
-
+ const TString resultId = request.execute_mode() == YandexQuery::SAVE ? parser.ColumnParser(RESULT_ID_COLUMN_NAME).GetOptionalString().GetOrElse("") : "";
+
const auto queryVisibility = query.content().acl().visibility();
const auto queryUser = query.meta().common().created_by();
const bool hasManageAccess = HasManageAccess(permissions, queryVisibility, queryUser, user);
@@ -781,11 +781,11 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvModifyQuery
internal.set_token(token);
}
if (request.execute_mode() != YandexQuery::SAVE) {
- if (request.state_load_mode() != YandexQuery::StateLoadMode::STATE_LOAD_MODE_UNSPECIFIED) {
- internal.set_state_load_mode(request.state_load_mode());
- }
- internal.mutable_disposition()->CopyFrom(request.disposition());
-
+ if (request.state_load_mode() != YandexQuery::StateLoadMode::STATE_LOAD_MODE_UNSPECIFIED) {
+ internal.set_state_load_mode(request.state_load_mode());
+ }
+ internal.mutable_disposition()->CopyFrom(request.disposition());
+
internal.clear_binding();
internal.clear_connection();
// TODO: move to run actor priority selection
@@ -862,7 +862,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvModifyQuery
response->second.After.ConstructInPlace().CopyFrom(query);
response->second.CloudId = internal.cloud_id();
- TSqlQueryBuilder writeQueryBuilder(YdbConnection->TablePathPrefix, "ModifyQuery(write)");
+ TSqlQueryBuilder writeQueryBuilder(YdbConnection->TablePathPrefix, "ModifyQuery(write)");
writeQueryBuilder.AddString("scope", scope);
writeQueryBuilder.AddString("query_id", queryId);
writeQueryBuilder.AddUint64("max_count_jobs", Config.Proto.GetMaxCountJobs());
@@ -905,34 +905,34 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvModifyQuery
writeQueryBuilder.AddTimestamp("now", TInstant::Now());
// insert job
writeQueryBuilder.AddText(
- "UPSERT INTO `" JOBS_TABLE_NAME "` (`" SCOPE_COLUMN_NAME "`, `" QUERY_ID_COLUMN_NAME "`, `" JOB_ID_COLUMN_NAME "`, `" JOB_COLUMN_NAME "`, `" USER_COLUMN_NAME "`, `" VISIBILITY_COLUMN_NAME "`) VALUES\n"
- " ($scope, $query_id, $job_id, $job, $user, $visibility);\n"
+ "UPSERT INTO `" JOBS_TABLE_NAME "` (`" SCOPE_COLUMN_NAME "`, `" QUERY_ID_COLUMN_NAME "`, `" JOB_ID_COLUMN_NAME "`, `" JOB_COLUMN_NAME "`, `" USER_COLUMN_NAME "`, `" VISIBILITY_COLUMN_NAME "`) VALUES\n"
+ " ($scope, $query_id, $job_id, $job, $user, $visibility);\n"
);
// insert pending small
writeQueryBuilder.AddText(
"INSERT INTO `" PENDING_SMALL_TABLE_NAME "`\n"
- " (`" SCOPE_COLUMN_NAME "`, `" QUERY_ID_COLUMN_NAME "`, `" LAST_SEEN_AT_COLUMN_NAME "`, `" RETRY_COUNTER_COLUMN_NAME "`, \n"
- " `" RETRY_COUNTER_UPDATE_COLUMN_NAME "`, `" QUERY_TYPE_COLUMN_NAME "`, `" HOST_NAME_COLUMN_NAME "`, `" OWNER_COLUMN_NAME "`)\n"
+ " (`" SCOPE_COLUMN_NAME "`, `" QUERY_ID_COLUMN_NAME "`, `" LAST_SEEN_AT_COLUMN_NAME "`, `" RETRY_COUNTER_COLUMN_NAME "`, \n"
+ " `" RETRY_COUNTER_UPDATE_COLUMN_NAME "`, `" QUERY_TYPE_COLUMN_NAME "`, `" HOST_NAME_COLUMN_NAME "`, `" OWNER_COLUMN_NAME "`)\n"
"VALUES\n"
- " ($scope, $query_id, $zero_timestamp, 0, $now, $query_type, \"\", \"\");\n"
+ " ($scope, $query_id, $zero_timestamp, 0, $now, $query_type, \"\", \"\");\n"
);
}
writeQueryBuilder.AddText(
"DELETE FROM `" JOBS_TABLE_NAME "` ON\n"
"SELECT * FROM $to_delete;\n"
- "UPDATE `" QUERIES_TABLE_NAME "` SET \n"
- " `" LAST_JOB_ID_COLUMN_NAME "` = $job_id, `" VISIBILITY_COLUMN_NAME "` = $visibility, `" AUTOMATIC_COLUMN_NAME "` = $automatic,\n"
- " `" NAME_COLUMN_NAME "` = $name, `" EXECUTE_MODE_COLUMN_NAME "` = $execute_mode,\n"
- " `" REVISION_COLUMN_NAME "` = $revision, `" STATUS_COLUMN_NAME "` = $status, "
- );
- writeQueryBuilder.AddText(
- (request.execute_mode() != YandexQuery::SAVE ? "`" INTERNAL_COLUMN_NAME "` = $internal,\n" : TString{"\n"})
- );
- writeQueryBuilder.AddText(
- " `" QUERY_TYPE_COLUMN_NAME "` = $query_type, `" QUERY_COLUMN_NAME "` = $query,\n"
- " `" RESULT_ID_COLUMN_NAME "` = $result_id, `" META_REVISION_COLUMN_NAME "` = `" META_REVISION_COLUMN_NAME "` + 1\n"
+ "UPDATE `" QUERIES_TABLE_NAME "` SET \n"
+ " `" LAST_JOB_ID_COLUMN_NAME "` = $job_id, `" VISIBILITY_COLUMN_NAME "` = $visibility, `" AUTOMATIC_COLUMN_NAME "` = $automatic,\n"
+ " `" NAME_COLUMN_NAME "` = $name, `" EXECUTE_MODE_COLUMN_NAME "` = $execute_mode,\n"
+ " `" REVISION_COLUMN_NAME "` = $revision, `" STATUS_COLUMN_NAME "` = $status, "
+ );
+ writeQueryBuilder.AddText(
+ (request.execute_mode() != YandexQuery::SAVE ? "`" INTERNAL_COLUMN_NAME "` = $internal,\n" : TString{"\n"})
+ );
+ writeQueryBuilder.AddText(
+ " `" QUERY_TYPE_COLUMN_NAME "` = $query_type, `" QUERY_COLUMN_NAME "` = $query,\n"
+ " `" RESULT_ID_COLUMN_NAME "` = $result_id, `" META_REVISION_COLUMN_NAME "` = `" META_REVISION_COLUMN_NAME "` + 1\n"
"WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id;"
);
@@ -1033,10 +1033,10 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvDeleteQuery
const TDuration delta = TInstant::Now() - startTime;
SendResponseIssues<TEvControlPlaneStorage::TEvDeleteQueryResponse>(ev->Sender, issues, ev->Cookie, delta, requestCounters);
LWPROBE(DeleteQueryRequest, scope, queryId, user, delta, byteSize, false);
- return;
+ return;
}
std::shared_ptr<std::pair<YandexQuery::DeleteQueryResult, TAuditDetails<YandexQuery::Query>>> response = std::make_shared<std::pair<YandexQuery::DeleteQueryResult, TAuditDetails<YandexQuery::Query>>>();
- TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "DeleteQuery");
+ TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "DeleteQuery");
queryBuilder.AddString("scope", scope);
queryBuilder.AddString("query_id", queryId);
@@ -1044,8 +1044,8 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvDeleteQuery
queryBuilder.AddText(
"DELETE FROM `" JOBS_TABLE_NAME "`\n"
"WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id;\n"
- "DELETE FROM `" PENDING_SMALL_TABLE_NAME "`\n"
- "WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id;\n"
+ "DELETE FROM `" PENDING_SMALL_TABLE_NAME "`\n"
+ "WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id;\n"
"DELETE FROM `" QUERIES_TABLE_NAME "`\n"
"WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id;"
);
@@ -1134,7 +1134,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvControlQuer
<< "ControlQueryRequest: "
<< NKikimr::MaskTicket(token) << " "
<< request.DebugString());
-
+
NYql::TIssues issues = ValidateEvent(ev);
if (issues) {
CPS_LOG_D(MakeLogPrefix(scope, user, queryId)
@@ -1145,12 +1145,12 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvControlQuer
const TDuration delta = TInstant::Now() - startTime;
SendResponseIssues<TEvControlPlaneStorage::TEvControlQueryResponse>(ev->Sender, issues, ev->Cookie, delta, requestCounters);
LWPROBE(ControlQueryRequest, scope, user, queryId, delta, byteSize, false);
- return;
+ return;
}
std::shared_ptr<std::pair<YandexQuery::ControlQueryResult, TAuditDetails<YandexQuery::Query>>> response = std::make_shared<std::pair<YandexQuery::ControlQueryResult, TAuditDetails<YandexQuery::Query>>>();
- TSqlQueryBuilder readQueryBuilder(YdbConnection->TablePathPrefix, "ControlQuery(read)");
+ TSqlQueryBuilder readQueryBuilder(YdbConnection->TablePathPrefix, "ControlQuery(read)");
readQueryBuilder.AddString("scope", scope);
readQueryBuilder.AddString("query_id", queryId);
@@ -1160,11 +1160,11 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvControlQuer
"$job_id = SELECT `" LAST_JOB_ID_COLUMN_NAME "` FROM $selected;\n"
"SELECT * FROM $selected;"
"SELECT `" JOB_COLUMN_NAME "` FROM `" JOBS_TABLE_NAME "`\n"
- "WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id AND `" JOB_ID_COLUMN_NAME "` = $job_id;\n"
+ "WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id AND `" JOB_ID_COLUMN_NAME "` = $job_id;\n"
);
auto prepareParams = [=, config=Config](const TVector<TResultSet>& resultSets) {
- if (resultSets.size() != 2) {
+ if (resultSets.size() != 2) {
ythrow TControlPlaneStorageException(TIssuesIds::INTERNAL_ERROR) << "Result set size is not equal to 2 but equal " << resultSets.size() << ". Please contact internal support";
}
@@ -1172,7 +1172,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvControlQuer
YandexQuery::Query query;
YandexQuery::Internal::QueryInternal queryInternal;
{
- TResultSetParser parser(resultSets[0]);
+ TResultSetParser parser(resultSets[0]);
if (!parser.TryNextRow()) {
ythrow TControlPlaneStorageException(TIssuesIds::ACCESS_DENIED) << "Query does not exist or permission denied. Please check the id query or your access rights";
}
@@ -1189,7 +1189,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvControlQuer
YandexQuery::Job job;
TString jobId;
{
- TResultSetParser parser(resultSets[1]);
+ TResultSetParser parser(resultSets[1]);
if (!parser.TryNextRow()) {
ythrow TControlPlaneStorageException(TIssuesIds::ACCESS_DENIED) << "Job does not exist or permission denied. Please check the id query or your access rights";
}
@@ -1208,25 +1208,25 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvControlQuer
queryInternal.set_action(action);
- auto& metaQuery = *query.mutable_meta();
- auto& commonQuery = *metaQuery.mutable_common();
- commonQuery.set_revision(commonQuery.revision() + 1);
- commonQuery.set_modified_by(user);
+ auto& metaQuery = *query.mutable_meta();
+ auto& commonQuery = *metaQuery.mutable_common();
+ commonQuery.set_revision(commonQuery.revision() + 1);
+ commonQuery.set_modified_by(user);
*commonQuery.mutable_modified_at() = NProtoInterop::CastToProto(now);
-
+
if (action == YandexQuery::ABORT || action == YandexQuery::ABORT_GRACEFULLY) {
const bool isValidStatusForAbort = IsIn({
YandexQuery::QueryMeta::STARTING,
YandexQuery::QueryMeta::RESUMING,
YandexQuery::QueryMeta::RUNNING,
YandexQuery::QueryMeta::PAUSING
- }, metaQuery.status());
-
+ }, metaQuery.status());
+
if (isValidStatusForAbort) {
- metaQuery.set_status(YandexQuery::QueryMeta::ABORTING_BY_USER);
+ metaQuery.set_status(YandexQuery::QueryMeta::ABORTING_BY_USER);
metaQuery.set_aborted_by(user);
} else {
- ythrow TControlPlaneStorageException(TIssuesIds::INTERNAL_ERROR) << "Conversion from status " << YandexQuery::QueryMeta::ComputeStatus_Name(metaQuery.status()) << " to " << YandexQuery::QueryMeta::ComputeStatus_Name(YandexQuery::QueryMeta::ABORTING_BY_USER) << " is not possible. Please wait for the previous operation to be completed";
+ ythrow TControlPlaneStorageException(TIssuesIds::INTERNAL_ERROR) << "Conversion from status " << YandexQuery::QueryMeta::ComputeStatus_Name(metaQuery.status()) << " to " << YandexQuery::QueryMeta::ComputeStatus_Name(YandexQuery::QueryMeta::ABORTING_BY_USER) << " is not possible. Please wait for the previous operation to be completed";
}
}
@@ -1234,34 +1234,34 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvControlQuer
const bool isValidStatusForPause = IsIn({
YandexQuery::QueryMeta::RESUMING,
YandexQuery::QueryMeta::RUNNING
- }, metaQuery.status());
-
+ }, metaQuery.status());
+
if (isValidStatusForPause) {
- metaQuery.set_status(YandexQuery::QueryMeta::PAUSING);
+ metaQuery.set_status(YandexQuery::QueryMeta::PAUSING);
metaQuery.set_paused_by(user);
} else {
- ythrow TControlPlaneStorageException(TIssuesIds::BAD_REQUEST) << "Conversion from status " << YandexQuery::QueryMeta::ComputeStatus_Name(metaQuery.status()) << " to " << YandexQuery::QueryMeta::ComputeStatus_Name(YandexQuery::QueryMeta::PAUSING) << " is not possible. Please wait for the previous operation to be completed";
+ ythrow TControlPlaneStorageException(TIssuesIds::BAD_REQUEST) << "Conversion from status " << YandexQuery::QueryMeta::ComputeStatus_Name(metaQuery.status()) << " to " << YandexQuery::QueryMeta::ComputeStatus_Name(YandexQuery::QueryMeta::PAUSING) << " is not possible. Please wait for the previous operation to be completed";
}
}
if (action == YandexQuery::RESUME) {
- const bool isValidStatusForResume = metaQuery.status() == YandexQuery::QueryMeta::PAUSED;
+ const bool isValidStatusForResume = metaQuery.status() == YandexQuery::QueryMeta::PAUSED;
if (isValidStatusForResume) {
- metaQuery.set_status(YandexQuery::QueryMeta::RESUMING);
+ metaQuery.set_status(YandexQuery::QueryMeta::RESUMING);
} else {
- ythrow TControlPlaneStorageException(TIssuesIds::BAD_REQUEST) << "Conversion from status " << YandexQuery::QueryMeta::ComputeStatus_Name(metaQuery.status()) << " to " << YandexQuery::QueryMeta::ComputeStatus_Name(YandexQuery::QueryMeta::RESUMING) << " is not possible. Please wait for the previous operation to be completed";
+ ythrow TControlPlaneStorageException(TIssuesIds::BAD_REQUEST) << "Conversion from status " << YandexQuery::QueryMeta::ComputeStatus_Name(metaQuery.status()) << " to " << YandexQuery::QueryMeta::ComputeStatus_Name(YandexQuery::QueryMeta::RESUMING) << " is not possible. Please wait for the previous operation to be completed";
}
}
- job.mutable_query_meta()->CopyFrom(metaQuery);
+ job.mutable_query_meta()->CopyFrom(metaQuery);
job.mutable_meta()->set_revision(job.meta().revision() + 1);
job.mutable_meta()->set_modified_by(user);
*job.mutable_meta()->mutable_modified_at() = NProtoInterop::CastToProto(now);
-
+
response->second.After.ConstructInPlace().CopyFrom(query);
response->second.CloudId = queryInternal.cloud_id();
- TSqlQueryBuilder writeQueryBuilder(YdbConnection->TablePathPrefix, "ControlQuery(write)");
+ TSqlQueryBuilder writeQueryBuilder(YdbConnection->TablePathPrefix, "ControlQuery(write)");
writeQueryBuilder.AddString("scope", scope);
writeQueryBuilder.AddString("job", job.SerializeAsString());
writeQueryBuilder.AddString("job_id", jobId);
@@ -1274,10 +1274,10 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvControlQuer
InsertIdempotencyKey(writeQueryBuilder, scope, idempotencyKey, response->first.SerializeAsString(), now + Config.IdempotencyKeyTtl);
writeQueryBuilder.AddText(
"UPDATE `" JOBS_TABLE_NAME "` SET `" JOB_COLUMN_NAME "` = $job\n"
- "WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id AND `" JOB_ID_COLUMN_NAME "` = $job_id;\n"
+ "WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id AND `" JOB_ID_COLUMN_NAME "` = $job_id;\n"
"UPDATE `" QUERIES_TABLE_NAME "` SET `" QUERY_COLUMN_NAME "` = $query, `" REVISION_COLUMN_NAME "` = $revision, `" STATUS_COLUMN_NAME "` = $status, `" INTERNAL_COLUMN_NAME "` = $internal, `" META_REVISION_COLUMN_NAME "` = `" META_REVISION_COLUMN_NAME "` + 1\n"
"WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id;\n"
- "UPDATE `" PENDING_SMALL_TABLE_NAME "` SET `" HOST_NAME_COLUMN_NAME "` = \"\"\n"
+ "UPDATE `" PENDING_SMALL_TABLE_NAME "` SET `" HOST_NAME_COLUMN_NAME "` = \"\"\n"
"WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id;"
);
@@ -1373,7 +1373,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvGetResultDa
return;
}
- TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "GetResultData");
+ TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "GetResultData");
queryBuilder.AddString("scope", scope);
queryBuilder.AddString("query_id", queryId);
queryBuilder.AddTimestamp("now", TInstant::Now());
@@ -1388,7 +1388,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvGetResultDa
"$result_id = SELECT `" RESULT_ID_COLUMN_NAME "` FROM $query_info\n"
"WHERE `" RESULT_SETS_EXPIRE_AT_COLUMN_NAME "` >= $now;\n"
"SELECT `" RESULT_SET_ID_COLUMN_NAME "`, `" RESULT_SET_COLUMN_NAME "`, `" ROW_ID_COLUMN_NAME "` FROM `" RESULT_SETS_TABLE_NAME "`\n"
- "WHERE `" RESULT_ID_COLUMN_NAME "` = $result_id AND `" RESULT_SET_ID_COLUMN_NAME "` = $result_set_index AND `" ROW_ID_COLUMN_NAME "` >= $offset\n"
+ "WHERE `" RESULT_ID_COLUMN_NAME "` = $result_id AND `" RESULT_SET_ID_COLUMN_NAME "` = $result_set_index AND `" ROW_ID_COLUMN_NAME "` >= $offset\n"
"ORDER BY `" ROW_ID_COLUMN_NAME "`\n"
"LIMIT $limit;\n"
);
@@ -1435,7 +1435,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvGetResultDa
if (!resultSetsExpireAtParser) {
ythrow TControlPlaneStorageException(TIssuesIds::BAD_REQUEST) << "Result doesn't exist";
}
-
+
if (*resultSetsExpireAtParser < TInstant::Now()) {
ythrow TControlPlaneStorageException(TIssuesIds::EXPIRED) << "Result removed by TTL";
}
@@ -1485,11 +1485,11 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvListJobsReq
if (request.has_filter() && request.filter().query_id()) {
queryId = request.filter().query_id();
}
-
- auto splittedPageToken = SplitId(request.page_token());
- const auto jobIdToken = std::move(splittedPageToken.first);
- const auto queryIdToken = std::move(splittedPageToken.second);
-
+
+ auto splittedPageToken = SplitId(request.page_token());
+ const auto jobIdToken = std::move(splittedPageToken.first);
+ const auto queryIdToken = std::move(splittedPageToken.second);
+
const int byteSize = ev->Get()->Request.ByteSize();
const TString token = ev->Get()->Token;
TPermissions permissions = Config.Proto.GetEnablePermissions()
@@ -1514,19 +1514,19 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvListJobsReq
const TDuration delta = TInstant::Now() - startTime;
SendResponseIssues<TEvControlPlaneStorage::TEvListJobsResponse>(ev->Sender, issues, ev->Cookie, delta, requestCounters);
LWPROBE(ListJobsRequest, scope, user, queryId, delta, byteSize, false);
- return;
+ return;
}
- TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "ListJobs");
+ TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "ListJobs");
queryBuilder.AddString("scope", scope);
- queryBuilder.AddString("last_job", jobIdToken);
- queryBuilder.AddString("last_query", queryIdToken);
+ queryBuilder.AddString("last_job", jobIdToken);
+ queryBuilder.AddString("last_query", queryIdToken);
queryBuilder.AddTimestamp("now", TInstant::Now());
queryBuilder.AddUint64("limit", limit + 1);
queryBuilder.AddText(
"SELECT `" JOB_ID_COLUMN_NAME "`, `" JOB_COLUMN_NAME "` FROM `" JOBS_TABLE_NAME "`\n"
- "WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` >= $last_query\n"
- "AND `" JOB_ID_COLUMN_NAME "` >= $last_job AND (`" EXPIRE_AT_COLUMN_NAME "` is NULL OR `" EXPIRE_AT_COLUMN_NAME "` > $now) "
+ "WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` >= $last_query\n"
+ "AND `" JOB_ID_COLUMN_NAME "` >= $last_job AND (`" EXPIRE_AT_COLUMN_NAME "` is NULL OR `" EXPIRE_AT_COLUMN_NAME "` > $now) "
);
TString filter;
@@ -1571,9 +1571,9 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvListJobsReq
if (!job.ParseFromString(*parser.ColumnParser(JOB_COLUMN_NAME).GetOptionalString())) {
ythrow TControlPlaneStorageException(TIssuesIds::INTERNAL_ERROR) << "Error parsing proto message for job. Please contact internal support";
}
- const TString mergedId = job.meta().id() + "-" + job.query_meta().common().id();
- job.mutable_meta()->set_id(mergedId);
- job.mutable_query_meta()->set_last_job_id(mergedId);
+ const TString mergedId = job.meta().id() + "-" + job.query_meta().common().id();
+ job.mutable_meta()->set_id(mergedId);
+ job.mutable_query_meta()->set_last_job_id(mergedId);
YandexQuery::BriefJob briefJob;
*briefJob.mutable_meta() = job.meta();
*briefJob.mutable_query_meta() = job.query_meta();
@@ -1585,8 +1585,8 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvListJobsReq
}
if (result.job_size() == limit + 1) {
- result.set_next_page_token(
- result.job(result.job_size() - 1).meta().id() + "-" + result.job(result.job_size() - 1).query_meta().common().id());
+ result.set_next_page_token(
+ result.job(result.job_size() - 1).meta().id() + "-" + result.job(result.job_size() - 1).query_meta().common().id());
result.mutable_job()->RemoveLast();
}
return result;
@@ -1618,11 +1618,11 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvDescribeJob
const YandexQuery::DescribeJobRequest& request = ev->Get()->Request;
const TString scope = ev->Get()->Scope;
const TString user = ev->Get()->User;
-
- auto splittedId = SplitId(request.job_id());
- const auto jobId = std::move(splittedId.first);
- const auto queryId = std::move(splittedId.second);
-
+
+ auto splittedId = SplitId(request.job_id());
+ const auto jobId = std::move(splittedId.first);
+ const auto queryId = std::move(splittedId.second);
+
const int byteSize = ev->Get()->Request.ByteSize();
const TString token = ev->Get()->Token;
TPermissions permissions = Config.Proto.GetEnablePermissions()
@@ -1648,23 +1648,23 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvDescribeJob
return;
}
- TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "DescribeJob");
+ TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "DescribeJob");
queryBuilder.AddString("scope", scope);
queryBuilder.AddString("job_id", jobId);
- queryBuilder.AddString("query_id", queryId);
+ queryBuilder.AddString("query_id", queryId);
queryBuilder.AddTimestamp("now", TInstant::Now());
queryBuilder.AddText(
"SELECT `" JOB_COLUMN_NAME "`, `" VISIBILITY_COLUMN_NAME "` FROM `" JOBS_TABLE_NAME "`\n"
- "WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id\n"
- " AND `" JOB_ID_COLUMN_NAME "` = $job_id AND (`" EXPIRE_AT_COLUMN_NAME "` is NULL OR `" EXPIRE_AT_COLUMN_NAME "` > $now);\n"
+ "WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id\n"
+ " AND `" JOB_ID_COLUMN_NAME "` = $job_id AND (`" EXPIRE_AT_COLUMN_NAME "` is NULL OR `" EXPIRE_AT_COLUMN_NAME "` > $now);\n"
);
const auto query = queryBuilder.Build();
auto debugInfo = Config.Proto.GetEnableDebugMode() ? std::make_shared<TDebugInfo>() : TDebugInfoPtr{};
auto [result, resultSets] = Read(query.Sql, query.Params, requestCounters, debugInfo);
- auto prepare = [=, id=request.job_id(), resultSets=resultSets] {
+ auto prepare = [=, id=request.job_id(), resultSets=resultSets] {
if (resultSets->size() != 1) {
ythrow TControlPlaneStorageException(TIssuesIds::INTERNAL_ERROR) << "Result set size is not equal to 1 but equal " << resultSets->size() << ". Please contact internal support";
}
@@ -1678,7 +1678,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvDescribeJob
ythrow TControlPlaneStorageException(TIssuesIds::INTERNAL_ERROR) << "Error parsing proto message for job. Please contact internal support";
}
auto visibility = static_cast<YandexQuery::Acl::Visibility>(*parser.ColumnParser(VISIBILITY_COLUMN_NAME).GetOptionalInt64());
- result.mutable_job()->mutable_meta()->set_id(id);
+ result.mutable_job()->mutable_meta()->set_id(id);
bool hasViewAccces = HasViewAccess(permissions, visibility, result.job().meta().created_by(), user);
if (!hasViewAccces) {
@@ -1704,4 +1704,4 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvDescribeJob
});
}
-} // NYq
+} // NYq
diff --git a/ydb/core/yq/libs/db_resolver/db_async_resolver.h b/ydb/core/yq/libs/db_resolver/db_async_resolver.h
index 02f6f3681a..ae81326839 100644
--- a/ydb/core/yq/libs/db_resolver/db_async_resolver.h
+++ b/ydb/core/yq/libs/db_resolver/db_async_resolver.h
@@ -1,22 +1,22 @@
-#pragma once
-#include <library/cpp/threading/future/future.h>
+#pragma once
+#include <library/cpp/threading/future/future.h>
#include <ydb/core/yq/libs/events/events.h>
-
-
-namespace NYq {
-
-struct TResolveParams { //TODO: remove
- THashMap<std::pair<TString, DatabaseType>, TEvents::TDatabaseAuth> Ids;
- TString TraceId;
-};
-
-class IDatabaseAsyncResolver {
-public:
- using TEndpoint = TEvents::TEvEndpointResponse::TEndpoint;
-
- virtual NThreading::TFuture<THashMap<std::pair<TString, DatabaseType>, TEndpoint>> ResolveIds(const TResolveParams& params) const = 0;
-
- virtual ~IDatabaseAsyncResolver() = default;
-};
-
-} // NYq
+
+
+namespace NYq {
+
+struct TResolveParams { //TODO: remove
+ THashMap<std::pair<TString, DatabaseType>, TEvents::TDatabaseAuth> Ids;
+ TString TraceId;
+};
+
+class IDatabaseAsyncResolver {
+public:
+ using TEndpoint = TEvents::TEvEndpointResponse::TEndpoint;
+
+ virtual NThreading::TFuture<THashMap<std::pair<TString, DatabaseType>, TEndpoint>> ResolveIds(const TResolveParams& params) const = 0;
+
+ virtual ~IDatabaseAsyncResolver() = default;
+};
+
+} // NYq
diff --git a/ydb/core/yq/libs/db_resolver/db_async_resolver_impl.cpp b/ydb/core/yq/libs/db_resolver/db_async_resolver_impl.cpp
index c798cc0d0e..253c205cee 100644
--- a/ydb/core/yq/libs/db_resolver/db_async_resolver_impl.cpp
+++ b/ydb/core/yq/libs/db_resolver/db_async_resolver_impl.cpp
@@ -1,41 +1,41 @@
-#include "db_async_resolver_impl.h"
-
-namespace NYq {
-
-using TEndpoint = TEvents::TEvEndpointResponse::TEndpoint;
-
-TDatabaseAsyncResolver::TDatabaseAsyncResolver(
- NActors::TActorSystem* actorSystem,
- const NActors::TActorId& recipient,
- const TString& ydbMvpEndpoint,
- const TString& mdbGateway,
- const bool mdbTransformHost)
- : ActorSystem(actorSystem)
- , Recipient(recipient)
- , YdbMvpEndpoint(ydbMvpEndpoint)
- , MdbGateway(mdbGateway)
- , MdbTransformHost(mdbTransformHost)
-{}
-
-NThreading::TFuture<THashMap<std::pair<TString, DatabaseType>, TEndpoint>> TDatabaseAsyncResolver::ResolveIds(const TResolveParams& params) const {
- auto promise = NThreading::NewPromise<THashMap<std::pair<TString, DatabaseType>, TEndpoint>>();
- auto callback = MakeHolder<NYql::TRichActorFutureCallback<TEvents::TEvEndpointResponse>>(
- [promise] (TAutoPtr<NActors::TEventHandle<TEvents::TEvEndpointResponse>>& event) mutable {
- promise.SetValue(event->Get()->DatabaseId2Endpoint);
- },
- [promise] () mutable {
- //TODO add logs
- promise.SetException("Error occurred on resolving ids. Message was undelivered.");
- },
- TDuration::Seconds(10)
- );
-
- NActors::TActorId callbackId = ActorSystem->Register(callback.Release());
-
- ActorSystem->Send(new NActors::IEventHandle(Recipient, callbackId,
- new TEvents::TEvEndpointRequest(params.Ids, YdbMvpEndpoint, MdbGateway,
- params.TraceId, MdbTransformHost)));
- return promise.GetFuture();
-}
-
-} // NYq
+#include "db_async_resolver_impl.h"
+
+namespace NYq {
+
+using TEndpoint = TEvents::TEvEndpointResponse::TEndpoint;
+
+TDatabaseAsyncResolver::TDatabaseAsyncResolver(
+ NActors::TActorSystem* actorSystem,
+ const NActors::TActorId& recipient,
+ const TString& ydbMvpEndpoint,
+ const TString& mdbGateway,
+ const bool mdbTransformHost)
+ : ActorSystem(actorSystem)
+ , Recipient(recipient)
+ , YdbMvpEndpoint(ydbMvpEndpoint)
+ , MdbGateway(mdbGateway)
+ , MdbTransformHost(mdbTransformHost)
+{}
+
+NThreading::TFuture<THashMap<std::pair<TString, DatabaseType>, TEndpoint>> TDatabaseAsyncResolver::ResolveIds(const TResolveParams& params) const {
+ auto promise = NThreading::NewPromise<THashMap<std::pair<TString, DatabaseType>, TEndpoint>>();
+ auto callback = MakeHolder<NYql::TRichActorFutureCallback<TEvents::TEvEndpointResponse>>(
+ [promise] (TAutoPtr<NActors::TEventHandle<TEvents::TEvEndpointResponse>>& event) mutable {
+ promise.SetValue(event->Get()->DatabaseId2Endpoint);
+ },
+ [promise] () mutable {
+ //TODO add logs
+ promise.SetException("Error occurred on resolving ids. Message was undelivered.");
+ },
+ TDuration::Seconds(10)
+ );
+
+ NActors::TActorId callbackId = ActorSystem->Register(callback.Release());
+
+ ActorSystem->Send(new NActors::IEventHandle(Recipient, callbackId,
+ new TEvents::TEvEndpointRequest(params.Ids, YdbMvpEndpoint, MdbGateway,
+ params.TraceId, MdbTransformHost)));
+ return promise.GetFuture();
+}
+
+} // NYq
diff --git a/ydb/core/yq/libs/db_resolver/db_async_resolver_impl.h b/ydb/core/yq/libs/db_resolver/db_async_resolver_impl.h
index 1bfe0b5d76..cb13cde383 100644
--- a/ydb/core/yq/libs/db_resolver/db_async_resolver_impl.h
+++ b/ydb/core/yq/libs/db_resolver/db_async_resolver_impl.h
@@ -1,28 +1,28 @@
-#pragma once
-#include "db_async_resolver.h"
+#pragma once
+#include "db_async_resolver.h"
#include <ydb/library/yql/providers/dq/actors/actor_helpers.h>
-
-namespace NYq {
-
-using TEndpoint = TEvents::TEvEndpointResponse::TEndpoint;
-
-class TDatabaseAsyncResolver : public IDatabaseAsyncResolver {
-public:
- TDatabaseAsyncResolver(
- NActors::TActorSystem* actorSystem,
- const NActors::TActorId& recipient,
- const TString& ydbMvpEndpoint,
- const TString& mdbGateway,
- const bool mdbTransformHost
- );
-
- NThreading::TFuture<THashMap<std::pair<TString, DatabaseType>, TEndpoint>> ResolveIds(const TResolveParams& params) const override;
-private:
- NActors::TActorSystem* ActorSystem;
- const NActors::TActorId Recipient;
- const TString YdbMvpEndpoint;
- const TString MdbGateway;
- const bool MdbTransformHost = false;
-};
-
-} // NYq
+
+namespace NYq {
+
+using TEndpoint = TEvents::TEvEndpointResponse::TEndpoint;
+
+class TDatabaseAsyncResolver : public IDatabaseAsyncResolver {
+public:
+ TDatabaseAsyncResolver(
+ NActors::TActorSystem* actorSystem,
+ const NActors::TActorId& recipient,
+ const TString& ydbMvpEndpoint,
+ const TString& mdbGateway,
+ const bool mdbTransformHost
+ );
+
+ NThreading::TFuture<THashMap<std::pair<TString, DatabaseType>, TEndpoint>> ResolveIds(const TResolveParams& params) const override;
+private:
+ NActors::TActorSystem* ActorSystem;
+ const NActors::TActorId Recipient;
+ const TString YdbMvpEndpoint;
+ const TString MdbGateway;
+ const bool MdbTransformHost = false;
+};
+
+} // NYq
diff --git a/ydb/core/yq/libs/db_resolver/db_async_resolver_with_meta.cpp b/ydb/core/yq/libs/db_resolver/db_async_resolver_with_meta.cpp
index 373c3825f5..4d38454f62 100644
--- a/ydb/core/yq/libs/db_resolver/db_async_resolver_with_meta.cpp
+++ b/ydb/core/yq/libs/db_resolver/db_async_resolver_with_meta.cpp
@@ -1,72 +1,72 @@
-#include "db_async_resolver_with_meta.h"
+#include "db_async_resolver_with_meta.h"
#include <ydb/core/yq/libs/common/database_token_builder.h>
-
-namespace NYq {
-
- TDatabaseAsyncResolverWithMeta::TDatabaseAsyncResolverWithMeta(
- NActors::TActorSystem* actorSystem,
- const NActors::TActorId& recipient,
- const TString& ydbMvpEndpoint,
- const TString& mdbGateway,
- const bool mdbTransformHost,
- const TString& traceId,
- const TString& token,
- const THashMap<TString, TString>& accountIdSignatures,
- const THashMap<TString, YandexQuery::Connection>& connections)
- : DbResolver(actorSystem, recipient, ydbMvpEndpoint, mdbGateway, mdbTransformHost)
- , TraceId(traceId)
- , Token(token)
- , AccountIdSignatures(accountIdSignatures)
- , Connections(connections)
- {}
-
- NThreading::TFuture<THashMap<std::pair<TString, DatabaseType>, TEndpoint>> TDatabaseAsyncResolverWithMeta::ResolveIds(const TResolveParams& params) const {
- return DbResolver.ResolveIds(params);
- }
-
- TString TDatabaseAsyncResolverWithMeta::GetTraceId() const {
- return TraceId;
- }
-
- TString TDatabaseAsyncResolverWithMeta::GetToken() const {
- return Token;
- }
-
- const THashMap<TString, TString>& TDatabaseAsyncResolverWithMeta::GetAccountIdSignatures() const {
- return AccountIdSignatures;
- }
-
- void TDatabaseAsyncResolverWithMeta::TryAddDbIdToResolve(
- const bool isEndpoint,
- const TString& clusterName,
- const TString& dbId,
- const DatabaseType type,
- THashMap<std::pair<TString, DatabaseType>, TEvents::TDatabaseAuth>& databaseIds) const {
- if (isEndpoint) {
- return;
- }
- const auto iter = Connections.find(clusterName);
- if (iter == Connections.end()) {
- return;
- }
- const auto& conn = iter->second;
- const auto& setting = conn.content().setting();
- YandexQuery::IamAuth auth;
- switch (type) {
- case DatabaseType::Ydb:
- auth = setting.ydb_database().auth();
- break;
- case DatabaseType::ClickHouse:
- auth = setting.clickhouse_cluster().auth();
- break;
- case DatabaseType::DataStreams:
- auth = setting.data_streams().auth();
- break;
- default:
- return;
- }
- TryAddDatabaseToResolve(auth, dbId, type, Token, AccountIdSignatures, databaseIds);
- }
-
+
+namespace NYq {
+
+ TDatabaseAsyncResolverWithMeta::TDatabaseAsyncResolverWithMeta(
+ NActors::TActorSystem* actorSystem,
+ const NActors::TActorId& recipient,
+ const TString& ydbMvpEndpoint,
+ const TString& mdbGateway,
+ const bool mdbTransformHost,
+ const TString& traceId,
+ const TString& token,
+ const THashMap<TString, TString>& accountIdSignatures,
+ const THashMap<TString, YandexQuery::Connection>& connections)
+ : DbResolver(actorSystem, recipient, ydbMvpEndpoint, mdbGateway, mdbTransformHost)
+ , TraceId(traceId)
+ , Token(token)
+ , AccountIdSignatures(accountIdSignatures)
+ , Connections(connections)
+ {}
+
+ NThreading::TFuture<THashMap<std::pair<TString, DatabaseType>, TEndpoint>> TDatabaseAsyncResolverWithMeta::ResolveIds(const TResolveParams& params) const {
+ return DbResolver.ResolveIds(params);
+ }
+
+ TString TDatabaseAsyncResolverWithMeta::GetTraceId() const {
+ return TraceId;
+ }
+
+ TString TDatabaseAsyncResolverWithMeta::GetToken() const {
+ return Token;
+ }
+
+ const THashMap<TString, TString>& TDatabaseAsyncResolverWithMeta::GetAccountIdSignatures() const {
+ return AccountIdSignatures;
+ }
+
+ void TDatabaseAsyncResolverWithMeta::TryAddDbIdToResolve(
+ const bool isEndpoint,
+ const TString& clusterName,
+ const TString& dbId,
+ const DatabaseType type,
+ THashMap<std::pair<TString, DatabaseType>, TEvents::TDatabaseAuth>& databaseIds) const {
+ if (isEndpoint) {
+ return;
+ }
+ const auto iter = Connections.find(clusterName);
+ if (iter == Connections.end()) {
+ return;
+ }
+ const auto& conn = iter->second;
+ const auto& setting = conn.content().setting();
+ YandexQuery::IamAuth auth;
+ switch (type) {
+ case DatabaseType::Ydb:
+ auth = setting.ydb_database().auth();
+ break;
+ case DatabaseType::ClickHouse:
+ auth = setting.clickhouse_cluster().auth();
+ break;
+ case DatabaseType::DataStreams:
+ auth = setting.data_streams().auth();
+ break;
+ default:
+ return;
+ }
+ TryAddDatabaseToResolve(auth, dbId, type, Token, AccountIdSignatures, databaseIds);
+ }
+
} // NYq
diff --git a/ydb/core/yq/libs/db_resolver/db_async_resolver_with_meta.h b/ydb/core/yq/libs/db_resolver/db_async_resolver_with_meta.h
index 99f422b3b8..4e0f5ae468 100644
--- a/ydb/core/yq/libs/db_resolver/db_async_resolver_with_meta.h
+++ b/ydb/core/yq/libs/db_resolver/db_async_resolver_with_meta.h
@@ -1,46 +1,46 @@
-#pragma once
-#include "db_async_resolver_impl.h"
-
-namespace NYq {
-
-using TEndpoint = TEvents::TEvEndpointResponse::TEndpoint;
-
-class TDatabaseAsyncResolverWithMeta : public IDatabaseAsyncResolver {
-public:
- TDatabaseAsyncResolverWithMeta(
- NActors::TActorSystem* actorSystem,
- const NActors::TActorId& recipient,
- const TString& ydbMvpEndpoint,
- const TString& mdbGateway,
- const bool mdbTransformHost,
- const TString& traceId,
- const TString& token,
- const THashMap<TString, TString>& accountIdSignatures,
- const THashMap<TString, YandexQuery::Connection>& connections
- );
-
- NThreading::TFuture<THashMap<std::pair<TString, DatabaseType>, TEndpoint>> ResolveIds(const TResolveParams& params) const override;
-
- TString GetTraceId() const;
-
- TString GetToken() const;
-
- const THashMap<TString, TString>& GetAccountIdSignatures() const;
-
- void TryAddDbIdToResolve(
- const bool isEndpoint,
- const TString& clusterName,
- const TString& dbId,
- const DatabaseType type,
- THashMap<std::pair<TString, DatabaseType>, TEvents::TDatabaseAuth>& databaseIds
- ) const;
-
-private:
- TDatabaseAsyncResolver DbResolver;
- const TString TraceId;
- const TString Token;
- const THashMap<TString, TString> AccountIdSignatures;
- const THashMap<TString, YandexQuery::Connection> Connections;
-};
-
-} // NYq \ No newline at end of file
+#pragma once
+#include "db_async_resolver_impl.h"
+
+namespace NYq {
+
+using TEndpoint = TEvents::TEvEndpointResponse::TEndpoint;
+
+class TDatabaseAsyncResolverWithMeta : public IDatabaseAsyncResolver {
+public:
+ TDatabaseAsyncResolverWithMeta(
+ NActors::TActorSystem* actorSystem,
+ const NActors::TActorId& recipient,
+ const TString& ydbMvpEndpoint,
+ const TString& mdbGateway,
+ const bool mdbTransformHost,
+ const TString& traceId,
+ const TString& token,
+ const THashMap<TString, TString>& accountIdSignatures,
+ const THashMap<TString, YandexQuery::Connection>& connections
+ );
+
+ NThreading::TFuture<THashMap<std::pair<TString, DatabaseType>, TEndpoint>> ResolveIds(const TResolveParams& params) const override;
+
+ TString GetTraceId() const;
+
+ TString GetToken() const;
+
+ const THashMap<TString, TString>& GetAccountIdSignatures() const;
+
+ void TryAddDbIdToResolve(
+ const bool isEndpoint,
+ const TString& clusterName,
+ const TString& dbId,
+ const DatabaseType type,
+ THashMap<std::pair<TString, DatabaseType>, TEvents::TDatabaseAuth>& databaseIds
+ ) const;
+
+private:
+ TDatabaseAsyncResolver DbResolver;
+ const TString TraceId;
+ const TString Token;
+ const THashMap<TString, TString> AccountIdSignatures;
+ const THashMap<TString, YandexQuery::Connection> Connections;
+};
+
+} // NYq \ No newline at end of file
diff --git a/ydb/core/yq/libs/db_resolver/ya.make b/ydb/core/yq/libs/db_resolver/ya.make
index b613350874..2262759e36 100644
--- a/ydb/core/yq/libs/db_resolver/ya.make
+++ b/ydb/core/yq/libs/db_resolver/ya.make
@@ -1,17 +1,17 @@
OWNER(g:yq)
-
-LIBRARY()
-
-SRCS(
- db_async_resolver_impl.cpp
- db_async_resolver_with_meta.cpp
-)
-
-PEERDIR(
+
+LIBRARY()
+
+SRCS(
+ db_async_resolver_impl.cpp
+ db_async_resolver_with_meta.cpp
+)
+
+PEERDIR(
ydb/core/yq/libs/common
ydb/core/yq/libs/events
-)
-
-YQL_LAST_ABI_VERSION()
-
-END()
+)
+
+YQL_LAST_ABI_VERSION()
+
+END()
diff --git a/ydb/core/yq/libs/db_schema/db_schema.cpp b/ydb/core/yq/libs/db_schema/db_schema.cpp
index 44f57e171a..fdda16df40 100644
--- a/ydb/core/yq/libs/db_schema/db_schema.cpp
+++ b/ydb/core/yq/libs/db_schema/db_schema.cpp
@@ -3,13 +3,13 @@
#include <ydb/public/sdk/cpp/client/ydb_table/table.h>
#include <util/string/printf.h>
-namespace NYq {
+namespace NYq {
using namespace NYdb;
-TSqlQueryBuilder::TSqlQueryBuilder(const TString& tablePrefix, const TString& queryName)
+TSqlQueryBuilder::TSqlQueryBuilder(const TString& tablePrefix, const TString& queryName)
: TablePrefix(tablePrefix)
- , QueryName(queryName)
+ , QueryName(queryName)
{ }
TSqlQueryBuilder& TSqlQueryBuilder::PushPk() {
@@ -65,12 +65,12 @@ TSqlQueryBuilder& TSqlQueryBuilder::AddValue(const TString& name, const TValue&
return *this;
}
-TSqlQueryBuilder& TSqlQueryBuilder::AddColNames(TVector<TString>&& colNames)
-{
- ColNames = colNames;
- return *this;
-}
-
+TSqlQueryBuilder& TSqlQueryBuilder::AddColNames(TVector<TString>&& colNames)
+{
+ ColNames = colNames;
+ return *this;
+}
+
TSqlQueryBuilder& TSqlQueryBuilder::MaybeAddValue(const TString& name, const TMaybe<TValue>& value) {
if (value) {
return AddValue(name, *value, "==");
@@ -89,28 +89,28 @@ TSqlQueryBuilder& TSqlQueryBuilder::AddText(const TString& sql, const TString& t
return *this;
}
-TSqlQueryBuilder& TSqlQueryBuilder::AddBeforeQuery(const TString& sql, const TString& tableName ) {
- if (tableName) {
+TSqlQueryBuilder& TSqlQueryBuilder::AddBeforeQuery(const TString& sql, const TString& tableName ) {
+ if (tableName) {
BeforeQuery << SubstGlobalCopy(sql, TString("{TABLENAME}"), tableName);
- } else {
- BeforeQuery << sql;
- }
- return *this;
-
-}
-
+ } else {
+ BeforeQuery << sql;
+ }
+ return *this;
+
+}
+
TSqlQueryBuilder& TSqlQueryBuilder::Upsert(const TString& tableName) {
Op = "UPSERT";
Table = tableName;
return *this;
}
-TSqlQueryBuilder& TSqlQueryBuilder::Update(const TString& tableName) {
- Op = "UPDATE";
- Table = tableName;
- return *this;
-}
-
+TSqlQueryBuilder& TSqlQueryBuilder::Update(const TString& tableName) {
+ Op = "UPDATE";
+ Table = tableName;
+ return *this;
+}
+
TSqlQueryBuilder& TSqlQueryBuilder::Insert(const TString& tableName) {
Op = "INSERT";
Table = tableName;
@@ -125,47 +125,47 @@ TSqlQueryBuilder& TSqlQueryBuilder::Delete(const TString& tableName, int limit)
}
TSqlQueryBuilder& TSqlQueryBuilder::Get(const TString& tableName, int limit) {
- Op = "SELECT";
- Table = tableName;
+ Op = "SELECT";
+ Table = tableName;
Limit = limit;
- return *this;
-}
-
-TSqlQueryBuilder& TSqlQueryBuilder::GetCount(const TString& tableName) {
- Op = "SELECT COUNT(*)";
- Table = tableName;
- return *this;
-}
-
+ return *this;
+}
+
+TSqlQueryBuilder& TSqlQueryBuilder::GetCount(const TString& tableName) {
+ Op = "SELECT COUNT(*)";
+ Table = tableName;
+ return *this;
+}
+
TSqlQueryBuilder& TSqlQueryBuilder::GetLastDeleted() {
Op = "SELECT LAST DELETED";
return *this;
}
-void TSqlQueryBuilder::ConstructWhereFilter(TStringBuilder& text) {
- text << "WHERE ";
- size_t i = 0;
- for (const auto& pk : Pk) {
- text << "`" << pk << "` == " << "$" << pk;
- ++i;
- if (Fields || i != Pk.size()) {
- text << " and ";
- }
- }
- i = 0;
+void TSqlQueryBuilder::ConstructWhereFilter(TStringBuilder& text) {
+ text << "WHERE ";
+ size_t i = 0;
+ for (const auto& pk : Pk) {
+ text << "`" << pk << "` == " << "$" << pk;
+ ++i;
+ if (Fields || i != Pk.size()) {
+ text << " and ";
+ }
+ }
+ i = 0;
for (const auto& t : Fields) {
auto k = std::get<0>(t); // structured capture don't work on win32
auto v = std::get<1>(t); // structured capture don't work on win32
auto p = std::get<2>(t); // structured capture don't work on win32
text << "`" << k << "` " << p << " " << "$" << v;
- ++i;
- if (i != Fields.size()) {
- text << " and ";
- }
- }
-}
-
+ ++i;
+ if (i != Fields.size()) {
+ text << " and ";
+ }
+ }
+}
+
TSqlQueryBuilder& TSqlQueryBuilder::End()
{
if (!Op) {
@@ -191,22 +191,22 @@ TSqlQueryBuilder& TSqlQueryBuilder::End()
text << "DELETE FROM `" << Table << "`\n";
text << "ON (SELECT * from $todelete" << Counter << ");\n";
- } else if (Op == "SELECT") {
- text << Op << ' ';
- size_t i = 0;
- for (const auto& name : ColNames) {
- text << '`' << name << '`';
- ++i;
- if (i != ColNames.size()) {
- text << ", ";
- }
- }
+ } else if (Op == "SELECT") {
+ text << Op << ' ';
+ size_t i = 0;
+ for (const auto& name : ColNames) {
+ text << '`' << name << '`';
+ ++i;
+ if (i != ColNames.size()) {
+ text << ", ";
+ }
+ }
text << "\nFROM `" << Table << "`\n";
- ConstructWhereFilter(text);
+ ConstructWhereFilter(text);
if (Limit > 0) {
text << " LIMIT " << Limit;
}
- text << ";\n";
+ text << ";\n";
} else if (Op == "SELECT LAST DELETED") {
text << "SELECT ";
size_t i = 0;
@@ -218,48 +218,48 @@ TSqlQueryBuilder& TSqlQueryBuilder::End()
}
}
text << "\n FROM $todelete" << (Counter-1) << ";\n";
- } else if (Op == "SELECT COUNT(*)") {
- text << Op ;
+ } else if (Op == "SELECT COUNT(*)") {
+ text << Op ;
text << "\nFROM `" << Table << "`\n";
- ConstructWhereFilter(text);
- text << ";\n";
- } else if (Op == "UPDATE") {
+ ConstructWhereFilter(text);
+ text << ";\n";
+ } else if (Op == "UPDATE") {
text << Op << " `" << Table << "`\n";
- if (Pk.empty() && Fields.empty()) {
- ythrow yexception() << "dangerous sql query";
- }
- text << "SET\n";
- size_t i = 0;
- for (const auto& t : Fields) {
- auto k = std::get<0>(t); // structured capture don't work on win32
- auto v = std::get<1>(t); // structured capture don't work on win32
- auto p = std::get<2>(t); // structured capture don't work on win32
-
- text << "`" << k << "` " << " = " << "$" << v;
- ++i;
- if (i != Fields.size()) {
- text << ", ";
- }
- }
- text << "\nWHERE\n";
- for (const auto& pk : Pk) {
- text << "$" << pk << " == " << "`" << pk << "` and ";
- }
- text << " $count != 0;\n";
- BeforeQuery << text;
- text.clear();
+ if (Pk.empty() && Fields.empty()) {
+ ythrow yexception() << "dangerous sql query";
+ }
+ text << "SET\n";
+ size_t i = 0;
+ for (const auto& t : Fields) {
+ auto k = std::get<0>(t); // structured capture don't work on win32
+ auto v = std::get<1>(t); // structured capture don't work on win32
+ auto p = std::get<2>(t); // structured capture don't work on win32
+
+ text << "`" << k << "` " << " = " << "$" << v;
+ ++i;
+ if (i != Fields.size()) {
+ text << ", ";
+ }
+ }
+ text << "\nWHERE\n";
+ for (const auto& pk : Pk) {
+ text << "$" << pk << " == " << "`" << pk << "` and ";
+ }
+ text << " $count != 0;\n";
+ BeforeQuery << text;
+ text.clear();
} else {
text << Op << " INTO `" << Table << "`\n";
text << "(\n" << " ";
for (const auto& pk : Pk) {
text << "`" << pk << "`" << ",";
}
- size_t i = 0;
+ size_t i = 0;
for (const auto& t : Fields) {
auto field = std::get<0>(t); // structured capture don't work on win32
text << "`" << field << "`";
i++;
- if (i != Fields.size()) {
+ if (i != Fields.size()) {
text << ",";
}
}
@@ -274,7 +274,7 @@ TSqlQueryBuilder& TSqlQueryBuilder::End()
auto field = std::get<1>(t); // structured capture don't work on win32
text << "$" << field;
i++;
- if (i != Fields.size()) {
+ if (i != Fields.size()) {
text << ", ";
}
}
@@ -290,7 +290,7 @@ TSqlQueryBuilder::TResult TSqlQueryBuilder::Build()
{
TStringBuilder declr;
declr << "--!syntax_v1\n";
- declr << "-- Query name: " << QueryName << "\n";
+ declr << "-- Query name: " << QueryName << "\n";
declr << "PRAGMA TablePathPrefix(\"" << TablePrefix << "\");\n";
for (const auto& [k, v] : ParametersOrdered) {
declr << "DECLARE $" << k << " as " << v << ";\n";
@@ -301,4 +301,4 @@ TSqlQueryBuilder::TResult TSqlQueryBuilder::Build()
return {declr, ParamsBuilder.Build()};
}
-} // namespace NYq
+} // namespace NYq
diff --git a/ydb/core/yq/libs/db_schema/db_schema.h b/ydb/core/yq/libs/db_schema/db_schema.h
index 4359ad7f04..c14a7d21c0 100644
--- a/ydb/core/yq/libs/db_schema/db_schema.h
+++ b/ydb/core/yq/libs/db_schema/db_schema.h
@@ -6,11 +6,11 @@
#include <util/string/builder.h>
#include <util/generic/hash_set.h>
-namespace NYq {
+namespace NYq {
class TSqlQueryBuilder {
public:
- TSqlQueryBuilder(const TString& tablePrefix, const TString& queryName = "Unknown query name");
+ TSqlQueryBuilder(const TString& tablePrefix, const TString& queryName = "Unknown query name");
TSqlQueryBuilder& PushPk();
TSqlQueryBuilder& PopPk();
@@ -54,15 +54,15 @@ public:
TSqlQueryBuilder& AddValue(const TString& name, const NYdb::TValue& value, const TString& pred);
TSqlQueryBuilder& AddValue(const TString& name, const TString& value, const TString& pred) {
- return AddValue(name, NYdb::TValueBuilder().String(value).Build(), pred);
+ return AddValue(name, NYdb::TValueBuilder().String(value).Build(), pred);
}
TSqlQueryBuilder& AddValue(const TString& name, ui32 value, const TString& pred) {
- return AddValue(name, NYdb::TValueBuilder().Uint32(value).Build(), pred);
+ return AddValue(name, NYdb::TValueBuilder().Uint32(value).Build(), pred);
}
TSqlQueryBuilder& AddValue(const TString& name, ui64 value, const TString& pred) {
- return AddValue(name, NYdb::TValueBuilder().Uint64(value).Build(), pred);
+ return AddValue(name, NYdb::TValueBuilder().Uint64(value).Build(), pred);
}
TSqlQueryBuilder& AddValue(const TString& name, TInstant value, const TString& pred) {
@@ -79,17 +79,17 @@ public:
return *this;
}
- TSqlQueryBuilder& AddColNames(TVector<TString>&& colNames);
-
+ TSqlQueryBuilder& AddColNames(TVector<TString>&& colNames);
+
TSqlQueryBuilder& AddText(const TString& sql, const TString& tableName = "");
- TSqlQueryBuilder& AddBeforeQuery(const TString& sql, const TString& tableName = "");
+ TSqlQueryBuilder& AddBeforeQuery(const TString& sql, const TString& tableName = "");
TSqlQueryBuilder& Upsert(const TString& tableName);
- TSqlQueryBuilder& Update(const TString& tableName);
+ TSqlQueryBuilder& Update(const TString& tableName);
TSqlQueryBuilder& Insert(const TString& tableName);
TSqlQueryBuilder& Delete(const TString& tableName, int limit = 0);
TSqlQueryBuilder& Get(const TString& tableName, int limit = 0);
- TSqlQueryBuilder& GetCount(const TString& tableName);
+ TSqlQueryBuilder& GetCount(const TString& tableName);
TSqlQueryBuilder& GetLastDeleted();
TSqlQueryBuilder& End();
@@ -108,7 +108,7 @@ private:
void ConstructWhereFilter(TStringBuilder& text);
const TString TablePrefix;
- const TString QueryName;
+ const TString QueryName;
TString Table;
int Limit = 0;
TString Op;
@@ -123,8 +123,8 @@ private:
TStringBuilder BeforeQuery;
TStringBuilder Text;
int Counter = 0;
-
- TVector<TString> ColNames;
+
+ TVector<TString> ColNames;
};
-} // namespace NYq
+} // namespace NYq
diff --git a/ydb/core/yq/libs/events/event_ids.h b/ydb/core/yq/libs/events/event_ids.h
index 83ed11b390..56032d50ea 100644
--- a/ydb/core/yq/libs/events/event_ids.h
+++ b/ydb/core/yq/libs/events/event_ids.h
@@ -1,54 +1,54 @@
-#pragma once
-#include <library/cpp/actors/core/event_local.h>
-#include <library/cpp/actors/core/events.h>
-
-namespace NYq {
-
-struct TEventIds {
- enum EEventSpaceYqlProxy {
- ES_YQL_ANALYTICS_PROXY = 4205 //TKikimrEvents::ES_YQL_ANALYTICS_PROXY
- };
-
- static constexpr ui32 EventSpace = ES_YQL_ANALYTICS_PROXY;
-
- // Event ids.
- enum EEv : ui32 {
- EvBegin = EventSpaceBegin(EventSpace),
-
- //YQL Internal
- EvPingTaskRequest = EvBegin,
- EvPingTaskResponse,
- EvGetTaskRequest,
- EvGetTaskResponse,
- EvWriteTaskResultRequest,
- EvWriteTaskResultResponse,
-
- EvNodesHealthCheckRequest,
- EvNodesHealthCheckResponse,
-
- EvUpdateConfig,
-
- // Internal events
- EvAsyncContinue,
- EvDbRequest,
- EvDbResponse,
- EvDbFunctionRequest,
- EvDbFunctionResponse,
- EvEndpointRequest,
- EvEndpointResponse,
- EvDataStreamsReadRulesCreationResult,
- EvDataStreamsReadRulesDeletionResult,
- EvQueryActionResult,
- EvForwardPingRequest,
- EvForwardPingResponse,
+#pragma once
+#include <library/cpp/actors/core/event_local.h>
+#include <library/cpp/actors/core/events.h>
+
+namespace NYq {
+
+struct TEventIds {
+ enum EEventSpaceYqlProxy {
+ ES_YQL_ANALYTICS_PROXY = 4205 //TKikimrEvents::ES_YQL_ANALYTICS_PROXY
+ };
+
+ static constexpr ui32 EventSpace = ES_YQL_ANALYTICS_PROXY;
+
+ // Event ids.
+ enum EEv : ui32 {
+ EvBegin = EventSpaceBegin(EventSpace),
+
+ //YQL Internal
+ EvPingTaskRequest = EvBegin,
+ EvPingTaskResponse,
+ EvGetTaskRequest,
+ EvGetTaskResponse,
+ EvWriteTaskResultRequest,
+ EvWriteTaskResultResponse,
+
+ EvNodesHealthCheckRequest,
+ EvNodesHealthCheckResponse,
+
+ EvUpdateConfig,
+
+ // Internal events
+ EvAsyncContinue,
+ EvDbRequest,
+ EvDbResponse,
+ EvDbFunctionRequest,
+ EvDbFunctionResponse,
+ EvEndpointRequest,
+ EvEndpointResponse,
+ EvDataStreamsReadRulesCreationResult,
+ EvDataStreamsReadRulesDeletionResult,
+ EvQueryActionResult,
+ EvForwardPingRequest,
+ EvForwardPingResponse,
EvGraphParams,
-
- // Special events
- EvEnd
- };
-
- static_assert(EvEnd < EventSpaceEnd(EventSpace), "expect EvEnd < EventSpaceEnd(EventSpace)");
-
-};
-
-} // namespace NYq
+
+ // Special events
+ EvEnd
+ };
+
+ static_assert(EvEnd < EventSpaceEnd(EventSpace), "expect EvEnd < EventSpaceEnd(EventSpace)");
+
+};
+
+} // namespace NYq
diff --git a/ydb/core/yq/libs/events/events.h b/ydb/core/yq/libs/events/events.h
index eca0b2a543..b3c70e5c5f 100644
--- a/ydb/core/yq/libs/events/events.h
+++ b/ydb/core/yq/libs/events/events.h
@@ -1,5 +1,5 @@
#pragma once
-#include "event_ids.h"
+#include "event_ids.h"
#include <ydb/library/yql/core/facade/yql_facade.h>
#include <ydb/library/yql/providers/dq/provider/yql_dq_gateway.h>
@@ -14,16 +14,16 @@
#include <util/digest/multi.h>
-namespace NYq {
+namespace NYq {
using NYdb::NYq::TScope;
-enum class DatabaseType {
- Ydb,
+enum class DatabaseType {
+ Ydb,
ClickHouse,
DataStreams,
ObjectStorage
-};
+};
struct TQueryResult {
TVector<Ydb::ResultSet> Sets;
@@ -39,55 +39,55 @@ struct TEvents {
bool HasPerm = false;
};
- struct TEvPingTaskRequest : NActors::TEventLocal<TEvPingTaskRequest, TEventIds::EvPingTaskRequest>, TEvAnalyticsBase {
- Yq::Private::PingTaskRequest Record;
- };
-
- struct TEvPingTaskResponse : NActors::TEventLocal<TEvPingTaskResponse, TEventIds::EvPingTaskResponse> {
- Ydb::StatusIds::StatusCode Status;
- NYql::TIssues Issues;
- TMaybe<Yq::Private::PingTaskResult> Record;
- };
-
- struct TEvGetTaskRequest : NActors::TEventLocal<TEvGetTaskRequest, TEventIds::EvGetTaskRequest>, TEvAnalyticsBase {
- Yq::Private::GetTaskRequest Record;
- };
-
- struct TEvGetTaskResponse : NActors::TEventLocal<TEvGetTaskResponse, TEventIds::EvGetTaskResponse> {
- Ydb::StatusIds::StatusCode Status;
- NYql::TIssues Issues;
- TMaybe<Yq::Private::GetTaskResult> Record;
- };
-
- struct TEvWriteTaskResultRequest : NActors::TEventLocal<TEvWriteTaskResultRequest, TEventIds::EvWriteTaskResultRequest>, TEvAnalyticsBase {
- Yq::Private::WriteTaskResultRequest Record;
- };
-
- struct TEvWriteTaskResultResponse : NActors::TEventLocal<TEvWriteTaskResultResponse, TEventIds::EvWriteTaskResultResponse> {
- Ydb::StatusIds::StatusCode Status;
- NYql::TIssues Issues;
- TMaybe<Yq::Private::WriteTaskResultResult> Record;
- };
-
- struct TEvNodesHealthCheckRequest : NActors::TEventLocal<TEvNodesHealthCheckRequest, TEventIds::EvNodesHealthCheckRequest>{
- Yq::Private::NodesHealthCheckRequest Record;
- };
-
- struct TEvNodesHealthCheckResponse : NActors::TEventLocal<TEvNodesHealthCheckResponse, TEventIds::EvNodesHealthCheckResponse> {
- Ydb::StatusIds::StatusCode Status;
- NYql::TIssues Issues;
- TMaybe<Yq::Private::NodesHealthCheckResult> Record;
- };
-
- struct TEvAsyncContinue : NActors::TEventLocal<TEvAsyncContinue, TEventIds::EvAsyncContinue> {
- const NYql::TProgram::TFutureStatus Future;
-
- explicit TEvAsyncContinue(const NYql::TProgram::TFutureStatus& future)
+ struct TEvPingTaskRequest : NActors::TEventLocal<TEvPingTaskRequest, TEventIds::EvPingTaskRequest>, TEvAnalyticsBase {
+ Yq::Private::PingTaskRequest Record;
+ };
+
+ struct TEvPingTaskResponse : NActors::TEventLocal<TEvPingTaskResponse, TEventIds::EvPingTaskResponse> {
+ Ydb::StatusIds::StatusCode Status;
+ NYql::TIssues Issues;
+ TMaybe<Yq::Private::PingTaskResult> Record;
+ };
+
+ struct TEvGetTaskRequest : NActors::TEventLocal<TEvGetTaskRequest, TEventIds::EvGetTaskRequest>, TEvAnalyticsBase {
+ Yq::Private::GetTaskRequest Record;
+ };
+
+ struct TEvGetTaskResponse : NActors::TEventLocal<TEvGetTaskResponse, TEventIds::EvGetTaskResponse> {
+ Ydb::StatusIds::StatusCode Status;
+ NYql::TIssues Issues;
+ TMaybe<Yq::Private::GetTaskResult> Record;
+ };
+
+ struct TEvWriteTaskResultRequest : NActors::TEventLocal<TEvWriteTaskResultRequest, TEventIds::EvWriteTaskResultRequest>, TEvAnalyticsBase {
+ Yq::Private::WriteTaskResultRequest Record;
+ };
+
+ struct TEvWriteTaskResultResponse : NActors::TEventLocal<TEvWriteTaskResultResponse, TEventIds::EvWriteTaskResultResponse> {
+ Ydb::StatusIds::StatusCode Status;
+ NYql::TIssues Issues;
+ TMaybe<Yq::Private::WriteTaskResultResult> Record;
+ };
+
+ struct TEvNodesHealthCheckRequest : NActors::TEventLocal<TEvNodesHealthCheckRequest, TEventIds::EvNodesHealthCheckRequest>{
+ Yq::Private::NodesHealthCheckRequest Record;
+ };
+
+ struct TEvNodesHealthCheckResponse : NActors::TEventLocal<TEvNodesHealthCheckResponse, TEventIds::EvNodesHealthCheckResponse> {
+ Ydb::StatusIds::StatusCode Status;
+ NYql::TIssues Issues;
+ TMaybe<Yq::Private::NodesHealthCheckResult> Record;
+ };
+
+ struct TEvAsyncContinue : NActors::TEventLocal<TEvAsyncContinue, TEventIds::EvAsyncContinue> {
+ const NYql::TProgram::TFutureStatus Future;
+
+ explicit TEvAsyncContinue(const NYql::TProgram::TFutureStatus& future)
: Future(future)
{}
};
- struct TEvDbRequest : NActors::TEventLocal<TEvDbRequest, TEventIds::EvDbRequest> {
+ struct TEvDbRequest : NActors::TEventLocal<TEvDbRequest, TEventIds::EvDbRequest> {
TString Sql;
NYdb::TParams Params;
bool Idempotent;
@@ -99,7 +99,7 @@ struct TEvents {
{}
};
- struct TEvDbResponse : NActors::TEventLocal<TEvDbResponse, TEventIds::EvDbResponse> {
+ struct TEvDbResponse : NActors::TEventLocal<TEvDbResponse, TEventIds::EvDbResponse> {
NYdb::TStatus Status;
TVector<NYdb::TResultSet> ResultSets;
@@ -109,7 +109,7 @@ struct TEvents {
{}
};
- struct TEvDbFunctionRequest : NActors::TEventLocal<TEvDbFunctionRequest, TEventIds::EvDbFunctionRequest> {
+ struct TEvDbFunctionRequest : NActors::TEventLocal<TEvDbFunctionRequest, TEventIds::EvDbFunctionRequest> {
using TFunction = std::function<NYdb::TAsyncStatus(NYdb::NTable::TSession&)>;
TFunction Handler;
@@ -118,7 +118,7 @@ struct TEvents {
{}
};
- struct TEvDbFunctionResponse : NActors::TEventLocal<TEvDbFunctionResponse, TEventIds::EvDbFunctionResponse> {
+ struct TEvDbFunctionResponse : NActors::TEventLocal<TEvDbFunctionResponse, TEventIds::EvDbFunctionResponse> {
NYdb::TStatus Status;
explicit TEvDbFunctionResponse(NYdb::TStatus status)
@@ -126,21 +126,21 @@ struct TEvents {
{}
};
- struct TEvEndpointResponse : NActors::TEventLocal<TEvEndpointResponse, TEventIds::EvEndpointResponse> {
- struct TEndpoint {
- TString Endpoint;
- TString Database;
+ struct TEvEndpointResponse : NActors::TEventLocal<TEvEndpointResponse, TEventIds::EvEndpointResponse> {
+ struct TEndpoint {
+ TString Endpoint;
+ TString Database;
bool Secure = false;
- };
+ };
THashMap<std::pair<TString, DatabaseType>, TEndpoint> DatabaseId2Endpoint;
- bool Success;
-
+ bool Success;
+
TEvEndpointResponse(THashMap<std::pair<TString, DatabaseType>, TEndpoint>&& res, bool success)
: DatabaseId2Endpoint(std::move(res))
- , Success(success)
- { }
- };
-
+ , Success(success)
+ { }
+ };
+
struct TDatabaseAuth {
TString StructuredToken;
bool AddBearerToToken = false;
@@ -153,12 +153,12 @@ struct TEvents {
}
};
- struct TEvEndpointRequest : NActors::TEventLocal<TEvEndpointRequest, TEventIds::EvEndpointRequest> {
+ struct TEvEndpointRequest : NActors::TEventLocal<TEvEndpointRequest, TEventIds::EvEndpointRequest> {
THashMap<std::pair<TString, DatabaseType>, TDatabaseAuth> DatabaseIds; // DbId, DatabaseType => database auth
- TString YdbMvpEndpoint;
+ TString YdbMvpEndpoint;
TString MdbGateway;
TString TraceId;
- bool MdbTransformHost;
+ bool MdbTransformHost;
TEvEndpointRequest(
const THashMap<std::pair<TString, DatabaseType>, TDatabaseAuth>& databaseIds,
@@ -167,14 +167,14 @@ struct TEvents {
const TString& traceId,
bool mdbTransformHost)
: DatabaseIds(databaseIds)
- , YdbMvpEndpoint(ydbMvpEndpoint)
+ , YdbMvpEndpoint(ydbMvpEndpoint)
, MdbGateway(mdbGateway)
, TraceId(traceId)
- , MdbTransformHost(mdbTransformHost)
+ , MdbTransformHost(mdbTransformHost)
{ }
};
- struct TEvDataStreamsReadRulesCreationResult : NActors::TEventLocal<TEvDataStreamsReadRulesCreationResult, TEventIds::EvDataStreamsReadRulesCreationResult> {
+ struct TEvDataStreamsReadRulesCreationResult : NActors::TEventLocal<TEvDataStreamsReadRulesCreationResult, TEventIds::EvDataStreamsReadRulesCreationResult> {
explicit TEvDataStreamsReadRulesCreationResult(NYql::TIssues issues)
: Issues(std::move(issues))
{
@@ -183,7 +183,7 @@ struct TEvents {
NYql::TIssues Issues;
};
- struct TEvDataStreamsReadRulesDeletionResult : NActors::TEventLocal<TEvDataStreamsReadRulesDeletionResult, TEventIds::EvDataStreamsReadRulesDeletionResult> {
+ struct TEvDataStreamsReadRulesDeletionResult : NActors::TEventLocal<TEvDataStreamsReadRulesDeletionResult, TEventIds::EvDataStreamsReadRulesDeletionResult> {
explicit TEvDataStreamsReadRulesDeletionResult(NYql::TIssues transientIssues)
: TransientIssues(std::move(transientIssues))
{
@@ -192,7 +192,7 @@ struct TEvents {
NYql::TIssues TransientIssues;
};
- struct TEvQueryActionResult : NActors::TEventLocal<TEvQueryActionResult, TEventIds::EvQueryActionResult> {
+ struct TEvQueryActionResult : NActors::TEventLocal<TEvQueryActionResult, TEventIds::EvQueryActionResult> {
explicit TEvQueryActionResult(YandexQuery::QueryAction action)
: Action(action)
{
@@ -201,17 +201,17 @@ struct TEvents {
YandexQuery::QueryAction Action;
};
- struct TEvForwardPingRequest : NActors::TEventLocal<TEvForwardPingRequest, TEventIds::EvForwardPingRequest> {
- explicit TEvForwardPingRequest(const Yq::Private::PingTaskRequest& request, bool final = false)
+ struct TEvForwardPingRequest : NActors::TEventLocal<TEvForwardPingRequest, TEventIds::EvForwardPingRequest> {
+ explicit TEvForwardPingRequest(const Yq::Private::PingTaskRequest& request, bool final = false)
: Request(request)
, Final(final)
{ }
- Yq::Private::PingTaskRequest Request;
+ Yq::Private::PingTaskRequest Request;
bool Final; // Is this the last ping request.
};
- struct TEvForwardPingResponse : NActors::TEventLocal<TEvForwardPingResponse, TEventIds::EvForwardPingResponse> {
+ struct TEvForwardPingResponse : NActors::TEventLocal<TEvForwardPingResponse, TEventIds::EvForwardPingResponse> {
TEvForwardPingResponse(bool success, YandexQuery::QueryAction action)
: Success(success)
, Action(action)
@@ -235,11 +235,11 @@ struct TEvents {
};
};
-} // namespace NYq
+} // namespace NYq
template<>
-struct THash<NYq::TEvents::TDatabaseAuth> {
- inline ui64 operator()(const NYq::TEvents::TDatabaseAuth& x) const noexcept {
+struct THash<NYq::TEvents::TDatabaseAuth> {
+ inline ui64 operator()(const NYq::TEvents::TDatabaseAuth& x) const noexcept {
return MultiHash(x.StructuredToken, x.AddBearerToToken);
}
};
diff --git a/ydb/core/yq/libs/events/ya.make b/ydb/core/yq/libs/events/ya.make
index 1f91ea511b..c12bdd681f 100644
--- a/ydb/core/yq/libs/events/ya.make
+++ b/ydb/core/yq/libs/events/ya.make
@@ -4,7 +4,7 @@ LIBRARY()
GENERATE_ENUM_SERIALIZATION(events.h)
-GENERATE_ENUM_SERIALIZATION(event_ids.h)
+GENERATE_ENUM_SERIALIZATION(event_ids.h)
PEERDIR(
library/cpp/actors/core
diff --git a/ydb/core/yq/libs/init/init.cpp b/ydb/core/yq/libs/init/init.cpp
index 747d25f192..d66ac202bf 100644
--- a/ydb/core/yq/libs/init/init.cpp
+++ b/ydb/core/yq/libs/init/init.cpp
@@ -24,7 +24,7 @@
#include <ydb/library/yql/providers/dq/task_runner/tasks_runner_local.h>
#include <ydb/library/yql/providers/dq/worker_manager/local_worker_manager.h>
#include <ydb/library/yql/providers/s3/actors/yql_s3_source_factory.h>
-#include <ydb/library/yql/providers/s3/proto/retry_config.pb.h>
+#include <ydb/library/yql/providers/s3/proto/retry_config.pb.h>
#include <ydb/library/yql/providers/clickhouse/actors/yql_ch_source_factory.h>
#include <ydb/library/yql/providers/pq/async_io/dq_pq_read_actor.h>
#include <ydb/library/yql/providers/pq/async_io/dq_pq_write_actor.h>
@@ -35,62 +35,62 @@
#include <ydb/library/yql/providers/ydb/actors/yql_ydb_source_factory.h>
#include <util/stream/file.h>
-#include <util/system/hostname.h>
-
-namespace {
-
-std::tuple<TString, TString> GetLocalAddress(const TString* overrideHostname = nullptr) {
- constexpr auto MaxLocalHostNameLength = 4096;
- std::array<char, MaxLocalHostNameLength> buffer;
- buffer.fill(0);
- TString hostName;
- TString localAddress;
-
- int result = gethostname(buffer.data(), buffer.size() - 1);
- if (result != 0) {
- Cerr << "gethostname failed " << strerror(errno) << Endl;
- return std::make_tuple(hostName, localAddress);
- }
-
- if (overrideHostname) {
- memcpy(&buffer[0], overrideHostname->c_str(), Min<int>(
- overrideHostname->size()+1, buffer.size()-1
- ));
- }
-
- hostName = &buffer[0];
-
- addrinfo request;
- memset(&request, 0, sizeof(request));
- request.ai_family = AF_INET6;
- request.ai_socktype = SOCK_STREAM;
-
- addrinfo* response = nullptr;
- result = getaddrinfo(buffer.data(), nullptr, &request, &response);
- if (result != 0) {
- Cerr << "getaddrinfo failed " << gai_strerror(result) << Endl;
- return std::make_tuple(hostName, localAddress);
- }
-
- std::unique_ptr<addrinfo, void (*)(addrinfo*)> holder(response, &freeaddrinfo);
-
- if (!response->ai_addr) {
- Cerr << "getaddrinfo failed: no ai_addr" << Endl;
- return std::make_tuple(hostName, localAddress);
- }
-
- auto* sa = response->ai_addr;
- Y_VERIFY(sa->sa_family == AF_INET6);
- inet_ntop(AF_INET6, &(((struct sockaddr_in6*)sa)->sin6_addr),
- &buffer[0], buffer.size() - 1);
-
- localAddress = &buffer[0];
-
- return std::make_tuple(hostName, localAddress);
-}
-
-}
-
+#include <util/system/hostname.h>
+
+namespace {
+
+std::tuple<TString, TString> GetLocalAddress(const TString* overrideHostname = nullptr) {
+ constexpr auto MaxLocalHostNameLength = 4096;
+ std::array<char, MaxLocalHostNameLength> buffer;
+ buffer.fill(0);
+ TString hostName;
+ TString localAddress;
+
+ int result = gethostname(buffer.data(), buffer.size() - 1);
+ if (result != 0) {
+ Cerr << "gethostname failed " << strerror(errno) << Endl;
+ return std::make_tuple(hostName, localAddress);
+ }
+
+ if (overrideHostname) {
+ memcpy(&buffer[0], overrideHostname->c_str(), Min<int>(
+ overrideHostname->size()+1, buffer.size()-1
+ ));
+ }
+
+ hostName = &buffer[0];
+
+ addrinfo request;
+ memset(&request, 0, sizeof(request));
+ request.ai_family = AF_INET6;
+ request.ai_socktype = SOCK_STREAM;
+
+ addrinfo* response = nullptr;
+ result = getaddrinfo(buffer.data(), nullptr, &request, &response);
+ if (result != 0) {
+ Cerr << "getaddrinfo failed " << gai_strerror(result) << Endl;
+ return std::make_tuple(hostName, localAddress);
+ }
+
+ std::unique_ptr<addrinfo, void (*)(addrinfo*)> holder(response, &freeaddrinfo);
+
+ if (!response->ai_addr) {
+ Cerr << "getaddrinfo failed: no ai_addr" << Endl;
+ return std::make_tuple(hostName, localAddress);
+ }
+
+ auto* sa = response->ai_addr;
+ Y_VERIFY(sa->sa_family == AF_INET6);
+ inet_ntop(AF_INET6, &(((struct sockaddr_in6*)sa)->sin6_addr),
+ &buffer[0], buffer.size() - 1);
+
+ localAddress = &buffer[0];
+
+ return std::make_tuple(hostName, localAddress);
+}
+
+}
+
namespace NYq {
using namespace NKikimr;
@@ -100,18 +100,18 @@ void Init(
ui32 nodeId,
const TActorRegistrator& actorRegistrator,
const TAppData* appData,
- const TString& tenant,
+ const TString& tenant,
::NPq::NConfigurationManager::IConnections::TPtr pqCmConnections,
const IYqSharedResources::TPtr& iyqSharedResources,
const std::function<IActor*(const NKikimrProto::NFolderService::TFolderServiceConfig& authConfig)>& folderServiceFactory,
const std::function<IActor*(const NYq::NConfig::TAuditConfig& auditConfig)>& auditServiceFactory,
- const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory,
- const ui32& icPort
+ const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory,
+ const ui32& icPort
)
{
Y_VERIFY(iyqSharedResources, "No YQ shared resources created");
TYqSharedResources::TPtr yqSharedResources = TYqSharedResources::Cast(iyqSharedResources);
- const auto clientCounters = appData->Counters->GetSubgroup("counters", "yq")->GetSubgroup("subsystem", "ClientMetrics");
+ const auto clientCounters = appData->Counters->GetSubgroup("counters", "yq")->GetSubgroup("subsystem", "ClientMetrics");
if (protoConfig.GetControlPlaneStorage().GetEnabled()) {
auto controlPlaneStorage = protoConfig.GetControlPlaneStorage().GetUseInMemory()
@@ -154,8 +154,8 @@ void Init(
actorRegistrator(NYql::NDq::MakeCheckpointStorageID(), checkpointStorage.release());
}
- auto yqCounters = appData->Counters->GetSubgroup("counters", "yq");
- auto workerManagerCounters = NYql::NDqs::TWorkerManagerCounters(yqCounters->GetSubgroup("subsystem", "worker_manager"));
+ auto yqCounters = appData->Counters->GetSubgroup("counters", "yq");
+ auto workerManagerCounters = NYql::NDqs::TWorkerManagerCounters(yqCounters->GetSubgroup("subsystem", "worker_manager"));
NKikimr::NMiniKQL::TComputationNodeFactory dqCompFactory = NKikimr::NMiniKQL::GetCompositeWithBuiltinFactory({
NYql::GetCommonDqFactory(),
@@ -172,17 +172,17 @@ void Init(
auto sinkActorFactory = MakeIntrusive<NYql::NDq::TDqSinkFactory>();
NYql::ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory;
- const auto httpGateway = NYql::IHTTPGateway::Make(
- &protoConfig.GetGateways().GetHttpGateway(),
- yqCounters->GetSubgroup("subcomponent", "http_gateway"));
-
+ const auto httpGateway = NYql::IHTTPGateway::Make(
+ &protoConfig.GetGateways().GetHttpGateway(),
+ yqCounters->GetSubgroup("subcomponent", "http_gateway"));
+
if (protoConfig.GetTokenAccessor().GetEnabled()) {
credentialsFactory = NYql::CreateSecuredServiceAccountCredentialsOverTokenAccessorFactory(protoConfig.GetTokenAccessor().GetEndpoint(), protoConfig.GetTokenAccessor().GetUseSsl());
RegisterDqPqReadActorFactory(*sourceActorFactory, yqSharedResources->YdbDriver, credentialsFactory);
RegisterYdbReadActorFactory(*sourceActorFactory, yqSharedResources->YdbDriver, credentialsFactory);
- RegisterS3ReadActorFactory(*sourceActorFactory, credentialsFactory,
- httpGateway, std::make_shared<NYql::NS3::TRetryConfig>(protoConfig.GetReadActorsFactoryConfig().GetS3ReadActorFactoryConfig().GetRetryConfig()));
- RegisterClickHouseReadActorFactory(*sourceActorFactory, credentialsFactory, httpGateway);
+ RegisterS3ReadActorFactory(*sourceActorFactory, credentialsFactory,
+ httpGateway, std::make_shared<NYql::NS3::TRetryConfig>(protoConfig.GetReadActorsFactoryConfig().GetS3ReadActorFactoryConfig().GetRetryConfig()));
+ RegisterClickHouseReadActorFactory(*sourceActorFactory, credentialsFactory, httpGateway);
RegisterDqPqWriteActorFactory(*sinkActorFactory, yqSharedResources->YdbDriver, credentialsFactory);
RegisterDQSolomonWriteActorFactory(*sinkActorFactory, credentialsFactory);
@@ -209,7 +209,7 @@ void Init(
lwmOptions.MkqlInitialMemoryLimit = mkqlInitialMemoryLimit;
lwmOptions.MkqlTotalMemoryLimit = mkqlTotalMemoryLimit;
lwmOptions.MkqlMinAllocSize = mkqlAllocSize;
- auto resman = NYql::NDqs::CreateLocalWorkerManager(lwmOptions);
+ auto resman = NYql::NDqs::CreateLocalWorkerManager(lwmOptions);
actorRegistrator(NYql::NDqs::MakeWorkerManagerActorID(nodeId), resman);
}
@@ -217,7 +217,7 @@ void Init(
::NYq::NCommon::TServiceCounters serviceCounters(appData->Counters);
if (protoConfig.GetNodesManager().GetEnabled()) {
- const auto localAddr = GetLocalAddress(&HostName());
+ const auto localAddr = GetLocalAddress(&HostName());
auto nodesManager = CreateYqlNodesManager(
workerManagerCounters,
TAppData::TimeProvider,
@@ -225,17 +225,17 @@ void Init(
serviceCounters,
protoConfig.GetPrivateApi(),
yqSharedResources,
- icPort,
- std::get<1>(localAddr),
+ icPort,
+ std::get<1>(localAddr),
tenant,
- mkqlInitialMemoryLimit,
- clientCounters);
+ mkqlInitialMemoryLimit,
+ clientCounters);
actorRegistrator(MakeYqlNodesManagerId(), nodesManager);
}
auto httpProxy = NHttp::CreateHttpProxy(*NMonitoring::TMetricRegistry::Instance());
- actorRegistrator(MakeYqlAnalyticsHttpProxyId(), httpProxy);
+ actorRegistrator(MakeYqlAnalyticsHttpProxyId(), httpProxy);
if (protoConfig.GetPendingFetcher().GetEnabled()) {
auto fetcher = CreatePendingFetcher(
@@ -251,31 +251,31 @@ void Init(
dqCompFactory,
serviceCounters,
credentialsFactory,
- httpGateway,
- std::move(pqCmConnections),
- clientCounters
+ httpGateway,
+ std::move(pqCmConnections),
+ clientCounters
);
actorRegistrator(MakeYqlAnalyticsFetcherId(nodeId), fetcher);
}
-
+
if (protoConfig.GetPrivateProxy().GetEnabled()) {
auto proxyPrivate = CreateYqlAnalyticsPrivateProxy(
TAppData::TimeProvider,
TAppData::RandomProvider,
serviceCounters.Counters,
protoConfig.GetTokenAccessor());
-
- actorRegistrator(MakeYqPrivateProxyId(), proxyPrivate);
+
+ actorRegistrator(MakeYqPrivateProxyId(), proxyPrivate);
}
}
IYqSharedResources::TPtr CreateYqSharedResources(
const NYq::NConfig::TConfig& config,
- const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory,
- const NMonitoring::TDynamicCounterPtr& counters)
+ const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory,
+ const NMonitoring::TDynamicCounterPtr& counters)
{
- return CreateYqSharedResourcesImpl(config, credentialsProviderFactory, counters);
+ return CreateYqSharedResourcesImpl(config, credentialsProviderFactory, counters);
}
} // NYq
diff --git a/ydb/core/yq/libs/init/init.h b/ydb/core/yq/libs/init/init.h
index 5aee6d8878..fa26651f18 100644
--- a/ydb/core/yq/libs/init/init.h
+++ b/ydb/core/yq/libs/init/init.h
@@ -23,8 +23,8 @@ using TActorRegistrator = std::function<void(NActors::TActorId, NActors::IActor*
IYqSharedResources::TPtr CreateYqSharedResources(
const NYq::NConfig::TConfig& config,
- const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory,
- const NMonitoring::TDynamicCounterPtr& counters);
+ const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory,
+ const NMonitoring::TDynamicCounterPtr& counters);
void Init(
const NYq::NConfig::TConfig& config,
@@ -36,8 +36,8 @@ void Init(
const IYqSharedResources::TPtr& yqSharedResources,
const std::function<IActor*(const NKikimrProto::NFolderService::TFolderServiceConfig& authConfig)>& folderServiceFactory,
const std::function<IActor*(const NYq::NConfig::TAuditConfig& auditConfig)>& auditServiceFactory,
- const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory,
- const ui32& icPort
+ const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory,
+ const ui32& icPort
);
} // NYq
diff --git a/ydb/core/yq/libs/init/ya.make b/ydb/core/yq/libs/init/ya.make
index 330275a50b..129103b024 100644
--- a/ydb/core/yq/libs/init/ya.make
+++ b/ydb/core/yq/libs/init/ya.make
@@ -40,7 +40,7 @@ PEERDIR(
ydb/library/yql/providers/pq/gateway/native
ydb/library/yql/providers/pq/provider
ydb/library/yql/providers/s3/actors
- ydb/library/yql/providers/s3/proto
+ ydb/library/yql/providers/s3/proto
ydb/library/yql/providers/s3/provider
ydb/library/yql/providers/solomon/async_io
ydb/library/yql/providers/solomon/gateway
diff --git a/ydb/core/yq/libs/logs/log.cpp b/ydb/core/yq/libs/logs/log.cpp
index f26c6a23c1..91c4af73ca 100644
--- a/ydb/core/yq/libs/logs/log.cpp
+++ b/ydb/core/yq/libs/logs/log.cpp
@@ -13,8 +13,8 @@ using namespace NActors;
class TYqlLogsUpdater : public TActorBootstrapped<TYqlLogsUpdater> {
public:
- static constexpr char ActorName[] = "YQ_LOGS_UPDATER";
-
+ static constexpr char ActorName[] = "YQ_LOGS_UPDATER";
+
TYqlLogsUpdater(const NKikimrConfig::TLogConfig& logConfig)
: YqlLoggerScope(new NYql::NLog::TTlsLogBackend(new TNullLogBackend()))
, LogConfig(logConfig)
diff --git a/ydb/core/yq/libs/mock/yql_mock.cpp b/ydb/core/yq/libs/mock/yql_mock.cpp
index 2e2d412453..8fe694f35b 100644
--- a/ydb/core/yq/libs/mock/yql_mock.cpp
+++ b/ydb/core/yq/libs/mock/yql_mock.cpp
@@ -36,8 +36,8 @@ public:
Location.Endpoint = TStringBuilder() << "localhost:" << grpcPort;
}
- static constexpr char ActorName[] = "YQL_MOCK";
-
+ static constexpr char ActorName[] = "YQL_MOCK";
+
private:
STRICT_STFUNC(Handler, {
HFunc(NHttp::TEvHttpProxy::TEvHttpIncomingRequest, Handle);
@@ -76,7 +76,7 @@ void InitTest(NActors::TTestActorRuntime* runtime, int httpPort, int grpcPort, c
{
yqSharedResources->Init(runtime->GetAnyNodeActorSystem());
- auto httpProxyId = NYq::MakeYqlAnalyticsHttpProxyId();
+ auto httpProxyId = NYq::MakeYqlAnalyticsHttpProxyId();
TActorId mockActorId = runtime->Register(CreateYqlMockActor(grpcPort));
diff --git a/ydb/core/yq/libs/private_client/private_client.cpp b/ydb/core/yq/libs/private_client/private_client.cpp
index bce67cbd3b..bccb8ddd1d 100644
--- a/ydb/core/yq/libs/private_client/private_client.cpp
+++ b/ydb/core/yq/libs/private_client/private_client.cpp
@@ -1,183 +1,183 @@
-#include "private_client.h"
+#include "private_client.h"
#include <ydb/library/yql/public/issue/yql_issue.h>
#include <ydb/library/yql/public/issue/yql_issue_message.h>
#include <ydb/public/sdk/cpp/client/ydb_common_client/impl/client.h>
-
-namespace NYq {
-
-using namespace NYdb;
-
-class TPrivateClient::TImpl : public TClientImplCommon<TPrivateClient::TImpl> {
-public:
- TImpl(
- std::shared_ptr<TGRpcConnectionsImpl>&& connections,
- const TCommonClientSettings& settings,
- const NMonitoring::TDynamicCounterPtr& counters)
- : TClientImplCommon(std::move(connections), settings)
- , Counters(counters->GetSubgroup("subsystem", "private_api")->GetSubgroup("subcomponent", "ClientMetrics"))
- , PingTaskTime(Counters->GetHistogram("PingTaskMs", NMonitoring::ExponentialHistogram(10, 2, 50)))
- , GetTaskTime(Counters->GetHistogram("GetTaskMs", NMonitoring::ExponentialHistogram(10, 2, 50)))
- , WriteTaskResultTime(Counters->GetHistogram("WriteTaskResultMs", NMonitoring::ExponentialHistogram(10, 2, 50)))
- , NodesHealthCheckTime(Counters->GetHistogram("NodesHealthCheckMs", NMonitoring::ExponentialHistogram(10, 2, 50)))
- {}
-
- template<class TProtoResult, class TResultWrapper>
- auto MakeResultExtractor(NThreading::TPromise<TResultWrapper> promise, const NMonitoring::THistogramPtr& hist, TInstant startedAt) {
- return [=, promise = std::move(promise)]
- (google::protobuf::Any* any, TPlainStatus status) mutable {
- std::unique_ptr<TProtoResult> result;
- if (any) {
- result.reset(new TProtoResult);
- any->UnpackTo(result.get());
- }
-
- hist->Collect((TInstant::Now() - startedAt).MilliSeconds());
-
- promise.SetValue(
- TResultWrapper(
- TStatus(std::move(status)),
- std::move(result)));
- };
- }
-
- TAsyncPingTaskResult PingTask(
- Yq::Private::PingTaskRequest&& request,
- const TPingTaskSettings& settings) {
- const auto startedAt = TInstant::Now();
- auto promise = NThreading::NewPromise<TPingTaskResult>();
- auto future = promise.GetFuture();
-
- auto extractor = MakeResultExtractor<
- Yq::Private::PingTaskResult,
- TPingTaskResult>(std::move(promise), PingTaskTime, startedAt);
-
- Connections_->RunDeferred<
- Yq::Private::V1::YqPrivateTaskService,
- Yq::Private::PingTaskRequest,
- Yq::Private::PingTaskResponse>(
- std::move(request),
- std::move(extractor),
- &Yq::Private::V1::YqPrivateTaskService::Stub::AsyncPingTask,
- DbDriverState_,
- INITIAL_DEFERRED_CALL_DELAY,
- TRpcRequestSettings::Make(settings),
- settings.ClientTimeout_);
-
- return future;
- }
-
- TAsyncGetTaskResult GetTask(
- Yq::Private::GetTaskRequest&& request,
- const TGetTaskSettings& settings) {
- const auto startedAt = TInstant::Now();
- auto promise = NThreading::NewPromise<TGetTaskResult>();
- auto future = promise.GetFuture();
- auto extractor = MakeResultExtractor<
- Yq::Private::GetTaskResult,
- TGetTaskResult>(std::move(promise), GetTaskTime, startedAt);
-
- Connections_->RunDeferred<
- Yq::Private::V1::YqPrivateTaskService,
- Yq::Private::GetTaskRequest,
- Yq::Private::GetTaskResponse>(
- std::move(request),
- std::move(extractor),
- &Yq::Private::V1::YqPrivateTaskService::Stub::AsyncGetTask,
- DbDriverState_,
- INITIAL_DEFERRED_CALL_DELAY,
- TRpcRequestSettings::Make(settings),
- settings.ClientTimeout_);
-
- return future;
- }
-
- TAsyncWriteTaskResult WriteTaskResult(
- Yq::Private::WriteTaskResultRequest&& request,
- const TWriteTaskResultSettings& settings) {
- const auto startedAt = TInstant::Now();
- auto promise = NThreading::NewPromise<TWriteTaskResult>();
- auto future = promise.GetFuture();
- auto extractor = MakeResultExtractor<
- Yq::Private::WriteTaskResultResult,
- TWriteTaskResult>(std::move(promise), WriteTaskResultTime, startedAt);
-
- Connections_->RunDeferred<
- Yq::Private::V1::YqPrivateTaskService,
- Yq::Private::WriteTaskResultRequest,
- Yq::Private::WriteTaskResultResponse>(
- std::move(request),
- std::move(extractor),
- &Yq::Private::V1::YqPrivateTaskService::Stub::AsyncWriteTaskResult,
- DbDriverState_,
- INITIAL_DEFERRED_CALL_DELAY,
- TRpcRequestSettings::Make(settings),
- settings.ClientTimeout_);
-
- return future;
- }
-
- TAsyncNodesHealthCheckResult NodesHealthCheck(
- Yq::Private::NodesHealthCheckRequest&& request,
- const TNodesHealthCheckSettings& settings) {
- const auto startedAt = TInstant::Now();
- auto promise = NThreading::NewPromise<TNodesHealthCheckResult>();
- auto future = promise.GetFuture();
- auto extractor = MakeResultExtractor<
- Yq::Private::NodesHealthCheckResult,
- TNodesHealthCheckResult>(std::move(promise), NodesHealthCheckTime, startedAt);
-
- Connections_->RunDeferred<
- Yq::Private::V1::YqPrivateTaskService,
- Yq::Private::NodesHealthCheckRequest,
- Yq::Private::NodesHealthCheckResponse>(
- std::move(request),
- std::move(extractor),
- &Yq::Private::V1::YqPrivateTaskService::Stub::AsyncNodesHealthCheck,
- DbDriverState_,
- INITIAL_DEFERRED_CALL_DELAY,
- TRpcRequestSettings::Make(settings),
- settings.ClientTimeout_);
-
- return future;
- }
-private:
- const NMonitoring::TDynamicCounterPtr Counters;
- const NMonitoring::THistogramPtr PingTaskTime;
- const NMonitoring::THistogramPtr GetTaskTime;
- const NMonitoring::THistogramPtr WriteTaskResultTime;
- const NMonitoring::THistogramPtr NodesHealthCheckTime;
-};
-
-TPrivateClient::TPrivateClient(
- const TDriver& driver,
- const TCommonClientSettings& settings,
- const NMonitoring::TDynamicCounterPtr& counters)
- : Impl(new TImpl(CreateInternalInterface(driver), settings, counters))
-{}
-
-TAsyncPingTaskResult TPrivateClient::PingTask(
- Yq::Private::PingTaskRequest&& request,
- const TPingTaskSettings& settings) {
+
+namespace NYq {
+
+using namespace NYdb;
+
+class TPrivateClient::TImpl : public TClientImplCommon<TPrivateClient::TImpl> {
+public:
+ TImpl(
+ std::shared_ptr<TGRpcConnectionsImpl>&& connections,
+ const TCommonClientSettings& settings,
+ const NMonitoring::TDynamicCounterPtr& counters)
+ : TClientImplCommon(std::move(connections), settings)
+ , Counters(counters->GetSubgroup("subsystem", "private_api")->GetSubgroup("subcomponent", "ClientMetrics"))
+ , PingTaskTime(Counters->GetHistogram("PingTaskMs", NMonitoring::ExponentialHistogram(10, 2, 50)))
+ , GetTaskTime(Counters->GetHistogram("GetTaskMs", NMonitoring::ExponentialHistogram(10, 2, 50)))
+ , WriteTaskResultTime(Counters->GetHistogram("WriteTaskResultMs", NMonitoring::ExponentialHistogram(10, 2, 50)))
+ , NodesHealthCheckTime(Counters->GetHistogram("NodesHealthCheckMs", NMonitoring::ExponentialHistogram(10, 2, 50)))
+ {}
+
+ template<class TProtoResult, class TResultWrapper>
+ auto MakeResultExtractor(NThreading::TPromise<TResultWrapper> promise, const NMonitoring::THistogramPtr& hist, TInstant startedAt) {
+ return [=, promise = std::move(promise)]
+ (google::protobuf::Any* any, TPlainStatus status) mutable {
+ std::unique_ptr<TProtoResult> result;
+ if (any) {
+ result.reset(new TProtoResult);
+ any->UnpackTo(result.get());
+ }
+
+ hist->Collect((TInstant::Now() - startedAt).MilliSeconds());
+
+ promise.SetValue(
+ TResultWrapper(
+ TStatus(std::move(status)),
+ std::move(result)));
+ };
+ }
+
+ TAsyncPingTaskResult PingTask(
+ Yq::Private::PingTaskRequest&& request,
+ const TPingTaskSettings& settings) {
+ const auto startedAt = TInstant::Now();
+ auto promise = NThreading::NewPromise<TPingTaskResult>();
+ auto future = promise.GetFuture();
+
+ auto extractor = MakeResultExtractor<
+ Yq::Private::PingTaskResult,
+ TPingTaskResult>(std::move(promise), PingTaskTime, startedAt);
+
+ Connections_->RunDeferred<
+ Yq::Private::V1::YqPrivateTaskService,
+ Yq::Private::PingTaskRequest,
+ Yq::Private::PingTaskResponse>(
+ std::move(request),
+ std::move(extractor),
+ &Yq::Private::V1::YqPrivateTaskService::Stub::AsyncPingTask,
+ DbDriverState_,
+ INITIAL_DEFERRED_CALL_DELAY,
+ TRpcRequestSettings::Make(settings),
+ settings.ClientTimeout_);
+
+ return future;
+ }
+
+ TAsyncGetTaskResult GetTask(
+ Yq::Private::GetTaskRequest&& request,
+ const TGetTaskSettings& settings) {
+ const auto startedAt = TInstant::Now();
+ auto promise = NThreading::NewPromise<TGetTaskResult>();
+ auto future = promise.GetFuture();
+ auto extractor = MakeResultExtractor<
+ Yq::Private::GetTaskResult,
+ TGetTaskResult>(std::move(promise), GetTaskTime, startedAt);
+
+ Connections_->RunDeferred<
+ Yq::Private::V1::YqPrivateTaskService,
+ Yq::Private::GetTaskRequest,
+ Yq::Private::GetTaskResponse>(
+ std::move(request),
+ std::move(extractor),
+ &Yq::Private::V1::YqPrivateTaskService::Stub::AsyncGetTask,
+ DbDriverState_,
+ INITIAL_DEFERRED_CALL_DELAY,
+ TRpcRequestSettings::Make(settings),
+ settings.ClientTimeout_);
+
+ return future;
+ }
+
+ TAsyncWriteTaskResult WriteTaskResult(
+ Yq::Private::WriteTaskResultRequest&& request,
+ const TWriteTaskResultSettings& settings) {
+ const auto startedAt = TInstant::Now();
+ auto promise = NThreading::NewPromise<TWriteTaskResult>();
+ auto future = promise.GetFuture();
+ auto extractor = MakeResultExtractor<
+ Yq::Private::WriteTaskResultResult,
+ TWriteTaskResult>(std::move(promise), WriteTaskResultTime, startedAt);
+
+ Connections_->RunDeferred<
+ Yq::Private::V1::YqPrivateTaskService,
+ Yq::Private::WriteTaskResultRequest,
+ Yq::Private::WriteTaskResultResponse>(
+ std::move(request),
+ std::move(extractor),
+ &Yq::Private::V1::YqPrivateTaskService::Stub::AsyncWriteTaskResult,
+ DbDriverState_,
+ INITIAL_DEFERRED_CALL_DELAY,
+ TRpcRequestSettings::Make(settings),
+ settings.ClientTimeout_);
+
+ return future;
+ }
+
+ TAsyncNodesHealthCheckResult NodesHealthCheck(
+ Yq::Private::NodesHealthCheckRequest&& request,
+ const TNodesHealthCheckSettings& settings) {
+ const auto startedAt = TInstant::Now();
+ auto promise = NThreading::NewPromise<TNodesHealthCheckResult>();
+ auto future = promise.GetFuture();
+ auto extractor = MakeResultExtractor<
+ Yq::Private::NodesHealthCheckResult,
+ TNodesHealthCheckResult>(std::move(promise), NodesHealthCheckTime, startedAt);
+
+ Connections_->RunDeferred<
+ Yq::Private::V1::YqPrivateTaskService,
+ Yq::Private::NodesHealthCheckRequest,
+ Yq::Private::NodesHealthCheckResponse>(
+ std::move(request),
+ std::move(extractor),
+ &Yq::Private::V1::YqPrivateTaskService::Stub::AsyncNodesHealthCheck,
+ DbDriverState_,
+ INITIAL_DEFERRED_CALL_DELAY,
+ TRpcRequestSettings::Make(settings),
+ settings.ClientTimeout_);
+
+ return future;
+ }
+private:
+ const NMonitoring::TDynamicCounterPtr Counters;
+ const NMonitoring::THistogramPtr PingTaskTime;
+ const NMonitoring::THistogramPtr GetTaskTime;
+ const NMonitoring::THistogramPtr WriteTaskResultTime;
+ const NMonitoring::THistogramPtr NodesHealthCheckTime;
+};
+
+TPrivateClient::TPrivateClient(
+ const TDriver& driver,
+ const TCommonClientSettings& settings,
+ const NMonitoring::TDynamicCounterPtr& counters)
+ : Impl(new TImpl(CreateInternalInterface(driver), settings, counters))
+{}
+
+TAsyncPingTaskResult TPrivateClient::PingTask(
+ Yq::Private::PingTaskRequest&& request,
+ const TPingTaskSettings& settings) {
return Impl->PingTask(std::move(request), settings);
-}
-
-TAsyncGetTaskResult TPrivateClient::GetTask(
- Yq::Private::GetTaskRequest&& request,
- const TGetTaskSettings& settings) {
+}
+
+TAsyncGetTaskResult TPrivateClient::GetTask(
+ Yq::Private::GetTaskRequest&& request,
+ const TGetTaskSettings& settings) {
return Impl->GetTask(std::move(request), settings);
-}
-
-TAsyncWriteTaskResult TPrivateClient::WriteTaskResult(
- Yq::Private::WriteTaskResultRequest&& request,
- const TWriteTaskResultSettings& settings) {
+}
+
+TAsyncWriteTaskResult TPrivateClient::WriteTaskResult(
+ Yq::Private::WriteTaskResultRequest&& request,
+ const TWriteTaskResultSettings& settings) {
return Impl->WriteTaskResult(std::move(request), settings);
-}
-
-TAsyncNodesHealthCheckResult TPrivateClient::NodesHealthCheck(
- Yq::Private::NodesHealthCheckRequest&& request,
- const TNodesHealthCheckSettings& settings) {
+}
+
+TAsyncNodesHealthCheckResult TPrivateClient::NodesHealthCheck(
+ Yq::Private::NodesHealthCheckRequest&& request,
+ const TNodesHealthCheckSettings& settings) {
return Impl->NodesHealthCheck(std::move(request), settings);
-}
-
-} //NYq
-
+}
+
+} //NYq
+
diff --git a/ydb/core/yq/libs/private_client/private_client.h b/ydb/core/yq/libs/private_client/private_client.h
index 35c8e1c0a4..e63b67eddb 100644
--- a/ydb/core/yq/libs/private_client/private_client.h
+++ b/ydb/core/yq/libs/private_client/private_client.h
@@ -1,81 +1,81 @@
-#pragma once
-
-#include <library/cpp/monlib/dynamic_counters/counters.h>
-
+#pragma once
+
+#include <library/cpp/monlib/dynamic_counters/counters.h>
+
#include <ydb/public/sdk/cpp/client/ydb_table/table.h>
#include <ydb/public/api/grpc/draft/yql_db_v1.grpc.pb.h>
-
-namespace NYq {
-
-template<class TProtoResult>
-class TProtoResultInternalWrapper : public NYdb::TStatus {
- friend class TPrivateClient;
-
+
+namespace NYq {
+
+template<class TProtoResult>
+class TProtoResultInternalWrapper : public NYdb::TStatus {
+ friend class TPrivateClient;
+
public:
- TProtoResultInternalWrapper(
- NYdb::TStatus&& status,
- std::unique_ptr<TProtoResult> result)
- : TStatus(std::move(status))
- , Result(std::move(result))
- { }
-
-public:
- const TProtoResult& GetResult() const {
- Y_VERIFY(Result, "Uninitialized result");
- return *Result;
- }
-
- bool IsResultSet() const {
- return Result ? true : false;
- }
-
-private:
- std::unique_ptr<TProtoResult> Result;
-};
-
-
-using TGetTaskResult = TProtoResultInternalWrapper<Yq::Private::GetTaskResult>;
-using TPingTaskResult = TProtoResultInternalWrapper<Yq::Private::PingTaskResult>;
-using TWriteTaskResult = TProtoResultInternalWrapper<Yq::Private::WriteTaskResultResult>;
-using TNodesHealthCheckResult = TProtoResultInternalWrapper<Yq::Private::NodesHealthCheckResult>;
-
-using TAsyncGetTaskResult = NThreading::TFuture<TGetTaskResult>;
-using TAsyncPingTaskResult = NThreading::TFuture<TPingTaskResult>;
-using TAsyncWriteTaskResult = NThreading::TFuture<TWriteTaskResult>;
-using TAsyncNodesHealthCheckResult = NThreading::TFuture<TNodesHealthCheckResult>;
-
-struct TGetTaskSettings : public NYdb::TOperationRequestSettings<TGetTaskSettings> {};
-struct TPingTaskSettings : public NYdb::TOperationRequestSettings<TPingTaskSettings> {};
-struct TWriteTaskResultSettings : public NYdb::TOperationRequestSettings<TWriteTaskResultSettings> {};
-struct TNodesHealthCheckSettings : public NYdb::TOperationRequestSettings<TNodesHealthCheckSettings> {};
-
-class TPrivateClient {
- class TImpl;
-
-public:
- TPrivateClient(
- const NYdb::TDriver& driver,
- const NYdb::TCommonClientSettings& settings = NYdb::TCommonClientSettings(),
- const NMonitoring::TDynamicCounterPtr& counters = MakeIntrusive<NMonitoring::TDynamicCounters>());
-
- TAsyncGetTaskResult GetTask(
- Yq::Private::GetTaskRequest&& request,
- const TGetTaskSettings& settings = TGetTaskSettings());
-
- TAsyncPingTaskResult PingTask(
- Yq::Private::PingTaskRequest&& request,
- const TPingTaskSettings& settings = TPingTaskSettings());
-
- TAsyncWriteTaskResult WriteTaskResult(
- Yq::Private::WriteTaskResultRequest&& request,
- const TWriteTaskResultSettings& settings = TWriteTaskResultSettings());
-
- TAsyncNodesHealthCheckResult NodesHealthCheck(
- Yq::Private::NodesHealthCheckRequest&& request,
- const TNodesHealthCheckSettings& settings = TNodesHealthCheckSettings());
-
-private:
+ TProtoResultInternalWrapper(
+ NYdb::TStatus&& status,
+ std::unique_ptr<TProtoResult> result)
+ : TStatus(std::move(status))
+ , Result(std::move(result))
+ { }
+
+public:
+ const TProtoResult& GetResult() const {
+ Y_VERIFY(Result, "Uninitialized result");
+ return *Result;
+ }
+
+ bool IsResultSet() const {
+ return Result ? true : false;
+ }
+
+private:
+ std::unique_ptr<TProtoResult> Result;
+};
+
+
+using TGetTaskResult = TProtoResultInternalWrapper<Yq::Private::GetTaskResult>;
+using TPingTaskResult = TProtoResultInternalWrapper<Yq::Private::PingTaskResult>;
+using TWriteTaskResult = TProtoResultInternalWrapper<Yq::Private::WriteTaskResultResult>;
+using TNodesHealthCheckResult = TProtoResultInternalWrapper<Yq::Private::NodesHealthCheckResult>;
+
+using TAsyncGetTaskResult = NThreading::TFuture<TGetTaskResult>;
+using TAsyncPingTaskResult = NThreading::TFuture<TPingTaskResult>;
+using TAsyncWriteTaskResult = NThreading::TFuture<TWriteTaskResult>;
+using TAsyncNodesHealthCheckResult = NThreading::TFuture<TNodesHealthCheckResult>;
+
+struct TGetTaskSettings : public NYdb::TOperationRequestSettings<TGetTaskSettings> {};
+struct TPingTaskSettings : public NYdb::TOperationRequestSettings<TPingTaskSettings> {};
+struct TWriteTaskResultSettings : public NYdb::TOperationRequestSettings<TWriteTaskResultSettings> {};
+struct TNodesHealthCheckSettings : public NYdb::TOperationRequestSettings<TNodesHealthCheckSettings> {};
+
+class TPrivateClient {
+ class TImpl;
+
+public:
+ TPrivateClient(
+ const NYdb::TDriver& driver,
+ const NYdb::TCommonClientSettings& settings = NYdb::TCommonClientSettings(),
+ const NMonitoring::TDynamicCounterPtr& counters = MakeIntrusive<NMonitoring::TDynamicCounters>());
+
+ TAsyncGetTaskResult GetTask(
+ Yq::Private::GetTaskRequest&& request,
+ const TGetTaskSettings& settings = TGetTaskSettings());
+
+ TAsyncPingTaskResult PingTask(
+ Yq::Private::PingTaskRequest&& request,
+ const TPingTaskSettings& settings = TPingTaskSettings());
+
+ TAsyncWriteTaskResult WriteTaskResult(
+ Yq::Private::WriteTaskResultRequest&& request,
+ const TWriteTaskResultSettings& settings = TWriteTaskResultSettings());
+
+ TAsyncNodesHealthCheckResult NodesHealthCheck(
+ Yq::Private::NodesHealthCheckRequest&& request,
+ const TNodesHealthCheckSettings& settings = TNodesHealthCheckSettings());
+
+private:
std::shared_ptr<TImpl> Impl;
-};
-
-} // namespace NYq
+};
+
+} // namespace NYq
diff --git a/ydb/core/yq/libs/private_client/utils.cpp b/ydb/core/yq/libs/private_client/utils.cpp
index e90b6f4513..f1ba5c7e22 100644
--- a/ydb/core/yq/libs/private_client/utils.cpp
+++ b/ydb/core/yq/libs/private_client/utils.cpp
@@ -1,57 +1,57 @@
-#include "utils.h"
-#include <library/cpp/protobuf/json/json2proto.h>
-#include <library/cpp/protobuf/json/proto2json.h>
+#include "utils.h"
+#include <library/cpp/protobuf/json/json2proto.h>
+#include <library/cpp/protobuf/json/proto2json.h>
#include <ydb/public/lib/ydb_cli/commands/ydb_common.h>
-
-namespace NYq {
-
+
+namespace NYq {
+
using namespace NYdb::NYq;
-void UpdateConnections(
- TClient& client,
+void UpdateConnections(
+ TClient& client,
const TString& folderId,
const TString& connectionsStr) {
- NJson::TJsonValue value;
- TStringStream in(connectionsStr);
- NJson::ReadJsonTree(&in, &value);
- YandexQuery::ListConnectionsRequest listConsRequest;
- listConsRequest.set_limit(100);
- auto result = client
- .ListConnections(listConsRequest, CreateYqSettings<TListConnectionsSettings>(folderId))
- .ExtractValueSync();
- NYdb::NConsoleClient::ThrowOnError(result);
- THashMap<TString, YandexQuery::Connection> connections;
- for (const auto& connection : result.GetResult().connection()) {
- if (!connection.content().name()) {
- continue;
- }
- connections.emplace(connection.content().name(), connection);
- }
- for (auto conJson : value.GetArraySafe()) {
- const auto& content = conJson["content"];
- const TString conName = content["name"].GetString();
- auto it = connections.find(conName);
- if (it == connections.end()) {
- Cerr << " Add " << conName << Endl;
- YandexQuery::CreateConnectionRequest request;
- NProtobufJson::Json2Proto(conJson, request);
- const auto result = client
- .CreateConnection(request, CreateYqSettings<TCreateConnectionSettings>(folderId))
- .ExtractValueSync();
- result.GetIssues().PrintTo(Cerr);
- NYdb::NConsoleClient::ThrowOnError(result);
- } else {
- Cerr << " Update " << conName << Endl;
- YandexQuery::ModifyConnectionRequest request;
- conJson["connection_id"] = it->second.meta().id();
- NProtobufJson::Json2Proto(conJson, request);
- const auto result = client
- .ModifyConnection(request, CreateYqSettings<TModifyConnectionSettings>(folderId))
- .ExtractValueSync();
- result.GetIssues().PrintTo(Cerr);
- NYdb::NConsoleClient::ThrowOnError(result);
- }
- }
-}
-
-} //NYq
+ NJson::TJsonValue value;
+ TStringStream in(connectionsStr);
+ NJson::ReadJsonTree(&in, &value);
+ YandexQuery::ListConnectionsRequest listConsRequest;
+ listConsRequest.set_limit(100);
+ auto result = client
+ .ListConnections(listConsRequest, CreateYqSettings<TListConnectionsSettings>(folderId))
+ .ExtractValueSync();
+ NYdb::NConsoleClient::ThrowOnError(result);
+ THashMap<TString, YandexQuery::Connection> connections;
+ for (const auto& connection : result.GetResult().connection()) {
+ if (!connection.content().name()) {
+ continue;
+ }
+ connections.emplace(connection.content().name(), connection);
+ }
+ for (auto conJson : value.GetArraySafe()) {
+ const auto& content = conJson["content"];
+ const TString conName = content["name"].GetString();
+ auto it = connections.find(conName);
+ if (it == connections.end()) {
+ Cerr << " Add " << conName << Endl;
+ YandexQuery::CreateConnectionRequest request;
+ NProtobufJson::Json2Proto(conJson, request);
+ const auto result = client
+ .CreateConnection(request, CreateYqSettings<TCreateConnectionSettings>(folderId))
+ .ExtractValueSync();
+ result.GetIssues().PrintTo(Cerr);
+ NYdb::NConsoleClient::ThrowOnError(result);
+ } else {
+ Cerr << " Update " << conName << Endl;
+ YandexQuery::ModifyConnectionRequest request;
+ conJson["connection_id"] = it->second.meta().id();
+ NProtobufJson::Json2Proto(conJson, request);
+ const auto result = client
+ .ModifyConnection(request, CreateYqSettings<TModifyConnectionSettings>(folderId))
+ .ExtractValueSync();
+ result.GetIssues().PrintTo(Cerr);
+ NYdb::NConsoleClient::ThrowOnError(result);
+ }
+ }
+}
+
+} //NYq
diff --git a/ydb/core/yq/libs/private_client/utils.h b/ydb/core/yq/libs/private_client/utils.h
index b2131b2806..ff1b255c31 100644
--- a/ydb/core/yq/libs/private_client/utils.h
+++ b/ydb/core/yq/libs/private_client/utils.h
@@ -1,12 +1,12 @@
-#pragma once
+#pragma once
#include <ydb/public/lib/yq/yq.h>
#include <ydb/public/lib/yq/helpers.h>
-
-namespace NYq {
-
-void UpdateConnections(
+
+namespace NYq {
+
+void UpdateConnections(
NYdb::NYq::TClient& client,
const TString& folderId,
const TString& connectionsStr);
-
-} // NYq
+
+} // NYq
diff --git a/ydb/core/yq/libs/private_client/ya.make b/ydb/core/yq/libs/private_client/ya.make
index 50dd2d6cb6..247be520a7 100644
--- a/ydb/core/yq/libs/private_client/ya.make
+++ b/ydb/core/yq/libs/private_client/ya.make
@@ -3,13 +3,13 @@ LIBRARY()
OWNER(g:kikimr)
SRCS(
- utils.cpp
- private_client.cpp
+ utils.cpp
+ private_client.cpp
)
PEERDIR(
- library/cpp/monlib/dynamic_counters
- library/cpp/protobuf/json
+ library/cpp/monlib/dynamic_counters
+ library/cpp/protobuf/json
ydb/public/api/grpc/draft
ydb/public/sdk/cpp/client/ydb_table
)
diff --git a/ydb/core/yq/libs/read_rule/read_rule_creator.cpp b/ydb/core/yq/libs/read_rule/read_rule_creator.cpp
index 406cd3df9f..4ca1a5ca47 100644
--- a/ydb/core/yq/libs/read_rule/read_rule_creator.cpp
+++ b/ydb/core/yq/libs/read_rule/read_rule_creator.cpp
@@ -89,7 +89,7 @@ public:
}
static constexpr char ActorName[] = "YQ_SINGLE_READ_RULE_CREATOR";
-
+
TString GetTopicPath() const {
TStringBuilder ret;
ret << Topic.GetDatabase();
@@ -146,13 +146,13 @@ public:
PassAway();
} else {
if (!CheckFinish()) {
- Schedule(*nextRetryDelay, new NActors::TEvents::TEvWakeup());
+ Schedule(*nextRetryDelay, new NActors::TEvents::TEvWakeup());
}
}
}
}
- void Handle(NActors::TEvents::TEvWakeup::TPtr&) {
+ void Handle(NActors::TEvents::TEvWakeup::TPtr&) {
if (!CheckFinish()) {
StartRequest();
}
@@ -175,7 +175,7 @@ public:
STRICT_STFUNC(StateFunc,
hFunc(TEvPrivate::TEvAddReadRuleStatus, Handle);
- hFunc(NActors::TEvents::TEvWakeup, Handle);
+ hFunc(NActors::TEvents::TEvWakeup, Handle);
hFunc(NActors::TEvents::TEvPoison, Handle);
)
@@ -223,7 +223,7 @@ public:
}
static constexpr char ActorName[] = "YQ_READ_RULE_CREATOR";
-
+
void Bootstrap() {
Become(&TReadRuleCreator::StateFunc);
@@ -266,7 +266,7 @@ public:
}
issues.AddIssue(std::move(mainIssue));
}
- Send(Owner, MakeHolder<NYq::TEvents::TEvDataStreamsReadRulesCreationResult>(std::move(issues)));
+ Send(Owner, MakeHolder<NYq::TEvents::TEvDataStreamsReadRulesCreationResult>(std::move(issues)));
PassAway();
}
}
diff --git a/ydb/core/yq/libs/read_rule/read_rule_deleter.cpp b/ydb/core/yq/libs/read_rule/read_rule_deleter.cpp
index 71c5e8f8c1..b5c2e4d549 100644
--- a/ydb/core/yq/libs/read_rule/read_rule_deleter.cpp
+++ b/ydb/core/yq/libs/read_rule/read_rule_deleter.cpp
@@ -66,7 +66,7 @@ public:
NActors::TActorId owner,
TString queryId,
NYdb::TDriver ydbDriver,
- Yq::Private::TopicConsumer topic,
+ Yq::Private::TopicConsumer topic,
std::shared_ptr<NYdb::ICredentialsProviderFactory> credentialsProvider,
ui64 index,
size_t maxRetries
@@ -82,7 +82,7 @@ public:
}
static constexpr char ActorName[] = "YQ_SINGLE_READ_RULE_DELETER";
-
+
void Bootstrap() {
Become(&TSingleReadRuleDeleter::StateFunc);
StartRequest();
@@ -139,18 +139,18 @@ public:
Send(Owner, MakeHolder<TEvPrivate::TEvSingleReadRuleDeleterResult>(status.GetIssues()), 0, Index);
PassAway();
} else {
- Schedule(*nextRetryDelay, new NActors::TEvents::TEvWakeup());
+ Schedule(*nextRetryDelay, new NActors::TEvents::TEvWakeup());
}
}
}
- void Handle(NActors::TEvents::TEvWakeup::TPtr&) {
+ void Handle(NActors::TEvents::TEvWakeup::TPtr&) {
StartRequest();
}
STRICT_STFUNC(StateFunc,
hFunc(TEvPrivate::TEvRemoveReadRuleStatus, Handle);
- hFunc(NActors::TEvents::TEvWakeup, Handle);
+ hFunc(NActors::TEvents::TEvWakeup, Handle);
cFunc(NActors::TEvents::TEvPoison::EventType, PassAway);
)
@@ -168,7 +168,7 @@ private:
private:
const NActors::TActorId Owner;
const TString QueryId;
- const Yq::Private::TopicConsumer Topic;
+ const Yq::Private::TopicConsumer Topic;
NYdb::TDriver YdbDriver;
NYdb::NPersQueue::TPersQueueClient PqClient;
ui64 Index = 0;
@@ -183,7 +183,7 @@ public:
NActors::TActorId owner,
TString queryId,
NYdb::TDriver ydbDriver,
- TVector<Yq::Private::TopicConsumer> topics,
+ TVector<Yq::Private::TopicConsumer> topics,
TVector<std::shared_ptr<NYdb::ICredentialsProviderFactory>> credentials,
size_t maxRetries
)
@@ -210,7 +210,7 @@ public:
}
static constexpr char ActorName[] = "YQ_READ_RULE_DELETER";
-
+
void Handle(TEvPrivate::TEvSingleReadRuleDeleterResult::TPtr& ev) {
const ui64 index = ev->Cookie;
Y_VERIFY(!Results[index]);
@@ -242,7 +242,7 @@ public:
}
issues.AddIssue(std::move(mainIssue));
}
- Send(Owner, MakeHolder<TEvents::TEvDataStreamsReadRulesDeletionResult>(std::move(issues)));
+ Send(Owner, MakeHolder<TEvents::TEvDataStreamsReadRulesDeletionResult>(std::move(issues)));
PassAway();
}
}
@@ -256,7 +256,7 @@ private:
const NActors::TActorId Owner;
const TString QueryId;
NYdb::TDriver YdbDriver;
- const TVector<Yq::Private::TopicConsumer> Topics;
+ const TVector<Yq::Private::TopicConsumer> Topics;
const TVector<std::shared_ptr<NYdb::ICredentialsProviderFactory>> Credentials;
const size_t MaxRetries;
size_t ResultsGot = 0;
@@ -271,7 +271,7 @@ NActors::IActor* MakeReadRuleDeleterActor(
NActors::TActorId owner,
TString queryId,
NYdb::TDriver ydbDriver,
- TVector<Yq::Private::TopicConsumer> topics,
+ TVector<Yq::Private::TopicConsumer> topics,
TVector<std::shared_ptr<NYdb::ICredentialsProviderFactory>> credentials, // For each topic
size_t maxRetries
)
diff --git a/ydb/core/yq/libs/read_rule/read_rule_deleter.h b/ydb/core/yq/libs/read_rule/read_rule_deleter.h
index e76e17340f..b3a92745a0 100644
--- a/ydb/core/yq/libs/read_rule/read_rule_deleter.h
+++ b/ydb/core/yq/libs/read_rule/read_rule_deleter.h
@@ -11,7 +11,7 @@ NActors::IActor* MakeReadRuleDeleterActor(
NActors::TActorId owner,
TString queryId,
NYdb::TDriver ydbDriver,
- TVector<Yq::Private::TopicConsumer> topics,
+ TVector<Yq::Private::TopicConsumer> topics,
TVector<std::shared_ptr<NYdb::ICredentialsProviderFactory>> credentials, // For each topic
size_t maxRetries = 15
);
diff --git a/ydb/core/yq/libs/result_formatter/result_formatter.cpp b/ydb/core/yq/libs/result_formatter/result_formatter.cpp
index b4da61874d..394832b7e5 100644
--- a/ydb/core/yq/libs/result_formatter/result_formatter.cpp
+++ b/ydb/core/yq/libs/result_formatter/result_formatter.cpp
@@ -13,10 +13,10 @@
#include <library/cpp/json/yson/json2yson.h>
-namespace NYq {
+namespace NYq {
using namespace NKikimr::NMiniKQL;
-using NYql::NUdf::TUnboxedValuePod;
+using NYql::NUdf::TUnboxedValuePod;
namespace {
@@ -154,7 +154,7 @@ NKikimr::NMiniKQL::TType* MakeEmptyDictType(NKikimr::NMiniKQL::TTypeEnvironment&
return env.GetTypeOfEmptyDict();
}
-const NYql::TTypeAnnotationNode* MakeVariantType(const NYql::TTypeAnnotationNode* underlyingType, NYql::TExprContext& ctx)
+const NYql::TTypeAnnotationNode* MakeVariantType(const NYql::TTypeAnnotationNode* underlyingType, NYql::TExprContext& ctx)
{
return ctx.MakeType<NYql::TVariantExprType>(underlyingType);
}
@@ -164,7 +164,7 @@ NKikimr::NMiniKQL::TType* MakeVariantType(NKikimr::NMiniKQL::TType* underlyingTy
return TVariantType::Create(underlyingType, env);
}
-const NYql::TTypeAnnotationNode* MakeTaggedType(const TString& tag, const NYql::TTypeAnnotationNode* underlyingType, NYql::TExprContext& ctx)
+const NYql::TTypeAnnotationNode* MakeTaggedType(const TString& tag, const NYql::TTypeAnnotationNode* underlyingType, NYql::TExprContext& ctx)
{
return ctx.MakeType<NYql::TTaggedExprType>(underlyingType, tag);
}
@@ -323,7 +323,7 @@ void FormatColumnValue(
{
const Ydb::Value& rawProtoValue = NYdb::TProtoAccessor::GetProto(value);
- NYql::NUdf::TUnboxedValue unboxed = ImportValueFromProto(
+ NYql::NUdf::TUnboxedValue unboxed = ImportValueFromProto(
type,
rawProtoValue,
holderFactory);
@@ -389,4 +389,4 @@ void FormatResultSet(NJson::TJsonValue& root, const NYdb::TResultSet& resultSet)
}
}
-} // namespace NYq
+} // namespace NYq
diff --git a/ydb/core/yq/libs/result_formatter/result_formatter.h b/ydb/core/yq/libs/result_formatter/result_formatter.h
index bbe4724c41..031196e050 100644
--- a/ydb/core/yq/libs/result_formatter/result_formatter.h
+++ b/ydb/core/yq/libs/result_formatter/result_formatter.h
@@ -8,9 +8,9 @@
#include <library/cpp/json/json_writer.h>
-namespace NYq {
+namespace NYq {
void FormatResultSet(NJson::TJsonValue& root, const NYdb::TResultSet& resultSet);
TString FormatSchema(const YandexQuery::Schema& schema);
-} // namespace NYq
+} // namespace NYq
diff --git a/ydb/core/yq/libs/result_formatter/result_formatter_ut.cpp b/ydb/core/yq/libs/result_formatter/result_formatter_ut.cpp
index a7a49ebe00..bebbf68ab3 100644
--- a/ydb/core/yq/libs/result_formatter/result_formatter_ut.cpp
+++ b/ydb/core/yq/libs/result_formatter/result_formatter_ut.cpp
@@ -2,7 +2,7 @@
#include <ydb/services/ydb/ydb_common_ut.h>
-using namespace NYq;
+using namespace NYq;
Y_UNIT_TEST_SUITE(ResultFormatter) {
Y_UNIT_TEST(Primitive) {
diff --git a/ydb/core/yq/libs/result_formatter/ut/ya.make b/ydb/core/yq/libs/result_formatter/ut/ya.make
index b6cf21e339..371714623b 100644
--- a/ydb/core/yq/libs/result_formatter/ut/ya.make
+++ b/ydb/core/yq/libs/result_formatter/ut/ya.make
@@ -1,25 +1,25 @@
UNITTEST_FOR(ydb/core/yq/libs/result_formatter)
-
+
OWNER(
g:kikimr
g:yq
)
-FORK_SUBTESTS()
-
-IF (SANITIZER_TYPE OR WITH_VALGRIND)
- SIZE(MEDIUM)
-ENDIF()
-
-SRCS(
- result_formatter_ut.cpp
-)
-
-PEERDIR(
+FORK_SUBTESTS()
+
+IF (SANITIZER_TYPE OR WITH_VALGRIND)
+ SIZE(MEDIUM)
+ENDIF()
+
+SRCS(
+ result_formatter_ut.cpp
+)
+
+PEERDIR(
ydb/library/yql/public/udf/service/stub
ydb/services/ydb
-)
-
-YQL_LAST_ABI_VERSION()
-
-END()
+)
+
+YQL_LAST_ABI_VERSION()
+
+END()
diff --git a/ydb/core/yq/libs/result_formatter/ya.make b/ydb/core/yq/libs/result_formatter/ya.make
index 9a166bf337..16a912791d 100644
--- a/ydb/core/yq/libs/result_formatter/ya.make
+++ b/ydb/core/yq/libs/result_formatter/ya.make
@@ -1,14 +1,14 @@
OWNER(g:yq)
-
-LIBRARY()
-
-SRCS(
- result_formatter.cpp
-)
-
-PEERDIR(
- library/cpp/json
- library/cpp/json/yson
+
+LIBRARY()
+
+SRCS(
+ result_formatter.cpp
+)
+
+PEERDIR(
+ library/cpp/json
+ library/cpp/json/yson
ydb/core/engine
ydb/library/yql/ast
ydb/library/yql/minikql
@@ -21,12 +21,12 @@ PEERDIR(
ydb/library/yql/providers/common/codec
ydb/library/yql/providers/common/schema/expr
ydb/library/yql/providers/common/schema/mkql
-)
-
-YQL_LAST_ABI_VERSION()
-
-END()
-
+)
+
+YQL_LAST_ABI_VERSION()
+
+END()
+
RECURSE_FOR_TESTS(
ut
)
diff --git a/ydb/core/yq/libs/shared_resources/db_pool.cpp b/ydb/core/yq/libs/shared_resources/db_pool.cpp
index aec0866c7e..372dfd72cd 100644
--- a/ydb/core/yq/libs/shared_resources/db_pool.cpp
+++ b/ydb/core/yq/libs/shared_resources/db_pool.cpp
@@ -12,24 +12,24 @@
#define LOG_E(stream) \
LOG_ERROR_S(*TlsActivationContext, NKikimrServices::YQL_PROXY, stream)
-namespace NYq {
+namespace NYq {
using namespace NActors;
using namespace NYql;
class TDbPoolActor : public NActors::TActor<TDbPoolActor> {
public:
- TDbPoolActor(
- const NYdb::NTable::TTableClient& tableClient,
- const NMonitoring::TDynamicCounterPtr& counters)
+ TDbPoolActor(
+ const NYdb::NTable::TTableClient& tableClient,
+ const NMonitoring::TDynamicCounterPtr& counters)
: TActor(&TThis::WorkingState)
, TableClient(tableClient)
- , QueueSize(counters->GetSubgroup("subcomponent", "DbPool")->GetHistogram("InFlight", NMonitoring::ExponentialHistogram(10, 2, 10)))
- , TotalInFlight(counters->GetSubgroup("subcomponent", "DbPool")->GetCounter("TotalInflight"))
- , RequestsTime(counters->GetSubgroup("subcomponent", "DbPool")->GetHistogram("RequestTimeMs", NMonitoring::ExponentialHistogram(6, 3, 100)))
+ , QueueSize(counters->GetSubgroup("subcomponent", "DbPool")->GetHistogram("InFlight", NMonitoring::ExponentialHistogram(10, 2, 10)))
+ , TotalInFlight(counters->GetSubgroup("subcomponent", "DbPool")->GetCounter("TotalInflight"))
+ , RequestsTime(counters->GetSubgroup("subcomponent", "DbPool")->GetHistogram("RequestTimeMs", NMonitoring::ExponentialHistogram(6, 3, 100)))
{}
- static constexpr char ActorName[] = "YQ_DB_POOL";
+ static constexpr char ActorName[] = "YQ_DB_POOL";
STRICT_STFUNC(WorkingState,
CFunc(NActors::TEvents::TEvPoison::EventType, Die)
@@ -56,14 +56,14 @@ public:
}
void ProcessQueue(const TActorContext& ctx) {
- QueueSize->Collect(Requests.size());
+ QueueSize->Collect(Requests.size());
if (Requests.empty() || RequestInProgress) {
return;
}
- TotalInFlight->Inc();
+ TotalInFlight->Inc();
RequestInProgress = true;
- RequestInProgressTimestamp = TInstant::Now();
+ RequestInProgressTimestamp = TInstant::Now();
const auto& requestVariant = Requests.front();
if (auto pRequest = std::get_if<TRequest>(&requestVariant)) {
@@ -110,20 +110,20 @@ public:
ProcessQueue(ctx);
}
- void PopFromQueueAndProcess(const TActorContext& ctx) {
+ void PopFromQueueAndProcess(const TActorContext& ctx) {
RequestInProgress = false;
- RequestsTime->Collect(TInstant::Now().MilliSeconds() - RequestInProgressTimestamp.MilliSeconds());
+ RequestsTime->Collect(TInstant::Now().MilliSeconds() - RequestInProgressTimestamp.MilliSeconds());
Requests.pop_front();
- TotalInFlight->Dec();
+ TotalInFlight->Dec();
ProcessQueue(ctx);
}
- void HandleResponse(TEvents::TEvDbResponse::TPtr& ev, const TActorContext& ctx) {
- const auto& request = Requests.front();
- ctx.Send(ev->Forward(std::visit([](const auto& arg) { return arg.Sender; }, request)));
- PopFromQueueAndProcess(ctx);
- }
-
+ void HandleResponse(TEvents::TEvDbResponse::TPtr& ev, const TActorContext& ctx) {
+ const auto& request = Requests.front();
+ ctx.Send(ev->Forward(std::visit([](const auto& arg) { return arg.Sender; }, request)));
+ PopFromQueueAndProcess(ctx);
+ }
+
void HandleRequest(TEvents::TEvDbFunctionRequest::TPtr& ev, const TActorContext& ctx) {
auto request = ev->Get();
Requests.emplace_back(TFunctionRequest{ev->Sender, ev->Cookie, std::move(request->Handler)});
@@ -133,7 +133,7 @@ public:
void HandleResponse(TEvents::TEvDbFunctionResponse::TPtr& ev, const TActorContext& ctx) {
const auto& request = Requests.front();
ctx.Send(ev->Forward(std::visit([](const auto& arg) { return arg.Sender; }, request)));
- PopFromQueueAndProcess(ctx);
+ PopFromQueueAndProcess(ctx);
}
private:
@@ -171,17 +171,17 @@ private:
NYdb::NTable::TTableClient TableClient;
TDeque<std::variant<TRequest, TFunctionRequest>> Requests;
bool RequestInProgress = false;
- TInstant RequestInProgressTimestamp = TInstant::Now();
+ TInstant RequestInProgressTimestamp = TInstant::Now();
std::shared_ptr<int> State = std::make_shared<int>();
- const NMonitoring::THistogramPtr QueueSize;
- const NMonitoring::TDynamicCounters::TCounterPtr TotalInFlight;
- const NMonitoring::THistogramPtr RequestsTime;
+ const NMonitoring::THistogramPtr QueueSize;
+ const NMonitoring::TDynamicCounters::TCounterPtr TotalInFlight;
+ const NMonitoring::THistogramPtr RequestsTime;
};
-TDbPool::TDbPool(
- ui32 sessionsCount,
- const NYdb::NTable::TTableClient& tableClient,
- const NMonitoring::TDynamicCounterPtr& counters)
+TDbPool::TDbPool(
+ ui32 sessionsCount,
+ const NYdb::NTable::TTableClient& tableClient,
+ const NMonitoring::TDynamicCounterPtr& counters)
{
const auto& ctx = NActors::TActivationContext::AsActorContext();
auto parentId = ctx.SelfID;
@@ -189,7 +189,7 @@ TDbPool::TDbPool(
for (ui32 i = 0; i < sessionsCount; ++i) {
Actors.emplace_back(
NActors::TActivationContext::Register(
- new TDbPoolActor(tableClient, counters),
+ new TDbPoolActor(tableClient, counters),
parentId, NActors::TMailboxType::HTSwap, parentId.PoolID()));
}
}
@@ -224,12 +224,12 @@ static void PrepareConfig(NYq::NConfig::TDbPoolConfig& config) {
TDbPoolMap::TDbPoolMap(
const NYq::NConfig::TDbPoolConfig& config,
NYdb::TDriver driver,
- const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory,
- const NMonitoring::TDynamicCounterPtr& counters)
+ const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory,
+ const NMonitoring::TDynamicCounterPtr& counters)
: Config(config)
, Driver(driver)
, CredentialsProviderFactory(credentialsProviderFactory)
- , Counters(counters)
+ , Counters(counters)
{
PrepareConfig(Config);
}
@@ -237,10 +237,10 @@ TDbPoolMap::TDbPoolMap(
TDbPoolHolder::TDbPoolHolder(
const NYq::NConfig::TDbPoolConfig& config,
const NYdb::TDriver& driver,
- const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory,
- const NMonitoring::TDynamicCounterPtr& counters)
+ const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory,
+ const NMonitoring::TDynamicCounterPtr& counters)
: Driver(driver)
- , Pools(new TDbPoolMap(config, Driver, credentialsProviderFactory, counters))
+ , Pools(new TDbPoolMap(config, Driver, credentialsProviderFactory, counters))
{ }
TDbPoolHolder::~TDbPoolHolder()
@@ -297,7 +297,7 @@ TDbPool::TPtr TDbPoolMap::GetOrCreate(EDbPoolId dbPoolId, ui32 sessionsCount) {
TableClient = MakeHolder<NYdb::NTable::TTableClient>(Driver, clientSettings);
}
- TDbPool::TPtr dbPool = new TDbPool(sessionsCount, *TableClient, Counters);
+ TDbPool::TPtr dbPool = new TDbPool(sessionsCount, *TableClient, Counters);
Pools.emplace(dbPoolId, dbPool);
return dbPool;
}
@@ -306,4 +306,4 @@ NYdb::TDriver& TDbPoolHolder::GetDriver() {
return Driver;
}
-} /* namespace NYq */
+} /* namespace NYq */
diff --git a/ydb/core/yq/libs/shared_resources/db_pool.h b/ydb/core/yq/libs/shared_resources/db_pool.h
index 461726990f..68d9adc2fc 100644
--- a/ydb/core/yq/libs/shared_resources/db_pool.h
+++ b/ydb/core/yq/libs/shared_resources/db_pool.h
@@ -7,11 +7,11 @@
#include <ydb/library/security/ydb_credentials_provider_factory.h>
#include <library/cpp/actors/core/actor.h>
-#include <library/cpp/monlib/dynamic_counters/counters.h>
+#include <library/cpp/monlib/dynamic_counters/counters.h>
#include <util/system/mutex.h>
-namespace NYq {
+namespace NYq {
class TDbPool: public TThrRefBase {
public:
@@ -24,12 +24,12 @@ public:
private:
friend class TDbPoolMap;
- TDbPool(ui32 sessionsCount, const NYdb::NTable::TTableClient& tableClient, const NMonitoring::TDynamicCounterPtr& counters);
+ TDbPool(ui32 sessionsCount, const NYdb::NTable::TTableClient& tableClient, const NMonitoring::TDynamicCounterPtr& counters);
TMutex Mutex;
TVector<NActors::TActorId> Actors;
ui32 Index = 0;
- const NMonitoring::TDynamicCounterPtr Counters;
+ const NMonitoring::TDynamicCounterPtr Counters;
};
enum class EDbPoolId {
@@ -48,9 +48,9 @@ private:
TDbPoolMap(const NYq::NConfig::TDbPoolConfig& config,
NYdb::TDriver driver,
- const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory,
- const NMonitoring::TDynamicCounterPtr& counters);
-
+ const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory,
+ const NMonitoring::TDynamicCounterPtr& counters);
+
void Reset(const NYq::NConfig::TDbPoolConfig& config);
TMutex Mutex;
NYq::NConfig::TDbPoolConfig Config;
@@ -58,7 +58,7 @@ private:
THashMap<EDbPoolId, TDbPool::TPtr> Pools;
THolder<NYdb::NTable::TTableClient> TableClient;
NKikimr::TYdbCredentialsProviderFactory CredentialsProviderFactory;
- const NMonitoring::TDynamicCounterPtr Counters;
+ const NMonitoring::TDynamicCounterPtr Counters;
};
class TDbPoolHolder: public TThrRefBase {
@@ -67,9 +67,9 @@ public:
TDbPoolHolder(
const NYq::NConfig::TDbPoolConfig& config,
const NYdb::TDriver& driver,
- const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory,
- const NMonitoring::TDynamicCounterPtr& counters);
-
+ const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory,
+ const NMonitoring::TDynamicCounterPtr& counters);
+
~TDbPoolHolder();
void Reset(const NYq::NConfig::TDbPoolConfig& config);
@@ -82,4 +82,4 @@ public:
TDbPoolMap::TPtr Pools;
};
-} /* NYq */
+} /* NYq */
diff --git a/ydb/core/yq/libs/shared_resources/shared_resources.cpp b/ydb/core/yq/libs/shared_resources/shared_resources.cpp
index 2504a6abdf..3dfd566232 100644
--- a/ydb/core/yq/libs/shared_resources/shared_resources.cpp
+++ b/ydb/core/yq/libs/shared_resources/shared_resources.cpp
@@ -86,11 +86,11 @@ struct TActorSystemPtrMixin {
struct TYqSharedResourcesImpl : public TActorSystemPtrMixin, public TYqSharedResources {
explicit TYqSharedResourcesImpl(
const NYq::NConfig::TConfig& config,
- const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory,
- const NMonitoring::TDynamicCounterPtr& counters)
+ const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory,
+ const NMonitoring::TDynamicCounterPtr& counters)
: TYqSharedResources(NYdb::TDriver(GetYdbDriverConfig(config.GetCommon().GetYdbDriverConfig())))
{
- CreateDbPoolHolder(config.GetDbPool(), credentialsProviderFactory, counters);
+ CreateDbPoolHolder(config.GetDbPool(), credentialsProviderFactory, counters);
}
void Init(NActors::TActorSystem* actorSystem) override {
@@ -120,9 +120,9 @@ struct TYqSharedResourcesImpl : public TActorSystemPtrMixin, public TYqSharedRes
void CreateDbPoolHolder(
const NYq::NConfig::TDbPoolConfig& config,
- const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory,
- const NMonitoring::TDynamicCounterPtr& counters) {
- DbPoolHolder = MakeIntrusive<NYq::TDbPoolHolder>(config, YdbDriver, credentialsProviderFactory, counters);
+ const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory,
+ const NMonitoring::TDynamicCounterPtr& counters) {
+ DbPoolHolder = MakeIntrusive<NYq::TDbPoolHolder>(config, YdbDriver, credentialsProviderFactory, counters);
}
};
@@ -130,9 +130,9 @@ struct TYqSharedResourcesImpl : public TActorSystemPtrMixin, public TYqSharedRes
TYqSharedResources::TPtr CreateYqSharedResourcesImpl(
const NYq::NConfig::TConfig& config,
- const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory,
- const NMonitoring::TDynamicCounterPtr& counters) {
- return MakeIntrusive<TYqSharedResourcesImpl>(config, credentialsProviderFactory, counters);
+ const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory,
+ const NMonitoring::TDynamicCounterPtr& counters) {
+ return MakeIntrusive<TYqSharedResourcesImpl>(config, credentialsProviderFactory, counters);
}
TYqSharedResources::TYqSharedResources(NYdb::TDriver driver)
diff --git a/ydb/core/yq/libs/shared_resources/shared_resources.h b/ydb/core/yq/libs/shared_resources/shared_resources.h
index 2f24ab9258..bfd1ec6695 100644
--- a/ydb/core/yq/libs/shared_resources/shared_resources.h
+++ b/ydb/core/yq/libs/shared_resources/shared_resources.h
@@ -17,7 +17,7 @@ struct TYqSharedResources : public IYqSharedResources {
// Resources
NYdb::TDriver YdbDriver;
- TDbPoolHolder::TPtr DbPoolHolder;
+ TDbPoolHolder::TPtr DbPoolHolder;
protected:
explicit TYqSharedResources(NYdb::TDriver driver);
@@ -25,7 +25,7 @@ protected:
TYqSharedResources::TPtr CreateYqSharedResourcesImpl(
const NYq::NConfig::TConfig& config,
- const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory,
- const NMonitoring::TDynamicCounterPtr& counters);
+ const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory,
+ const NMonitoring::TDynamicCounterPtr& counters);
} // namespace NYq
diff --git a/ydb/core/yq/libs/shared_resources/ya.make b/ydb/core/yq/libs/shared_resources/ya.make
index fed68026c5..46c9a3bb2b 100644
--- a/ydb/core/yq/libs/shared_resources/ya.make
+++ b/ydb/core/yq/libs/shared_resources/ya.make
@@ -9,7 +9,7 @@ SRCS(
PEERDIR(
library/cpp/actors/core
- library/cpp/monlib/dynamic_counters
+ library/cpp/monlib/dynamic_counters
ydb/core/protos
ydb/core/yq/libs/events
ydb/core/yq/libs/shared_resources/interface
diff --git a/ydb/core/yq/libs/ya.make b/ydb/core/yq/libs/ya.make
index 613e2d963c..ccc88a55b6 100644
--- a/ydb/core/yq/libs/ya.make
+++ b/ydb/core/yq/libs/ya.make
@@ -6,11 +6,11 @@ RECURSE(
checkpoint_storage
checkpointing
checkpointing_common
- common
+ common
config
control_plane_proxy
control_plane_storage
- db_resolver
+ db_resolver
db_schema
events
gateway
@@ -20,9 +20,9 @@ RECURSE(
logs
mock
pretty_printers
- private_client
+ private_client
read_rule
- result_formatter
+ result_formatter
shared_resources
signer
tasks_packer
diff --git a/ydb/library/mkql_proto/mkql_proto.cpp b/ydb/library/mkql_proto/mkql_proto.cpp
index 52bef4723a..53d7722dee 100644
--- a/ydb/library/mkql_proto/mkql_proto.cpp
+++ b/ydb/library/mkql_proto/mkql_proto.cpp
@@ -425,8 +425,8 @@ Y_FORCE_INLINE void HandleKindDataExport(const TType* type, const NUdf::TUnboxed
void ExportValueToProtoImpl(TType* type, const NUdf::TUnboxedValuePod& value, NKikimrMiniKQL::TValue& res, const TVector<ui32>* columnOrder = nullptr) {
switch (type->GetKind()) {
case TType::EKind::Void:
- case TType::EKind::EmptyList:
- case TType::EKind::EmptyDict:
+ case TType::EKind::EmptyList:
+ case TType::EKind::EmptyDict:
break;
case TType::EKind::Null: {
@@ -507,12 +507,12 @@ void ExportValueToProtoImpl(TType* type, const NUdf::TUnboxedValuePod& value, NK
break;
}
- case TType::EKind::Tagged: {
- auto taggedType = static_cast<TTaggedType*>(type);
- ExportValueToProtoImpl(taggedType->GetBaseType(), value, res);
- break;
- }
-
+ case TType::EKind::Tagged: {
+ auto taggedType = static_cast<TTaggedType*>(type);
+ ExportValueToProtoImpl(taggedType->GetBaseType(), value, res);
+ break;
+ }
+
default:
MKQL_ENSURE(false, TStringBuilder() << "Unknown kind: " << type->GetKindAsStr());
}
@@ -521,9 +521,9 @@ void ExportValueToProtoImpl(TType* type, const NUdf::TUnboxedValuePod& value, NK
void ExportValueToProtoImpl(TType* type, const NUdf::TUnboxedValuePod& value, Ydb::Value& res, const TVector<ui32>* columnOrder = nullptr) {
switch (type->GetKind()) {
case TType::EKind::Void:
- case TType::EKind::EmptyList:
- case TType::EKind::EmptyDict:
- break;
+ case TType::EKind::EmptyList:
+ case TType::EKind::EmptyDict:
+ break;
case TType::EKind::Null: {
res.set_null_flag_value(::google::protobuf::NULL_VALUE);
diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_actor_checkpoints.h b/ydb/library/yql/dq/actors/compute/dq_compute_actor_checkpoints.h
index 5efedb370f..27fd851338 100644
--- a/ydb/library/yql/dq/actors/compute/dq_compute_actor_checkpoints.h
+++ b/ydb/library/yql/dq/actors/compute/dq_compute_actor_checkpoints.h
@@ -64,8 +64,8 @@ class TDqComputeActorCheckpoints : public NActors::TActor<TDqComputeActorCheckpo
};
public:
- static constexpr char ActorName[] = "DQ_COMPUTE_ACTOR_CHECKPOINTS";
-
+ static constexpr char ActorName[] = "DQ_COMPUTE_ACTOR_CHECKPOINTS";
+
struct ICallbacks {
[[nodiscard]]
virtual bool ReadyToCheckpoint() const = 0;
diff --git a/ydb/library/yql/providers/clickhouse/provider/ya.make b/ydb/library/yql/providers/clickhouse/provider/ya.make
index e2cfd1cb42..05effd38eb 100644
--- a/ydb/library/yql/providers/clickhouse/provider/ya.make
+++ b/ydb/library/yql/providers/clickhouse/provider/ya.make
@@ -9,7 +9,7 @@ SRCS(
yql_clickhouse_datasource.cpp
yql_clickhouse_datasource_type_ann.cpp
yql_clickhouse_dq_integration.cpp
- yql_clickhouse_io_discovery.cpp
+ yql_clickhouse_io_discovery.cpp
yql_clickhouse_load_meta.cpp
yql_clickhouse_logical_opt.cpp
yql_clickhouse_mkql_compiler.cpp
@@ -27,8 +27,8 @@ YQL_LAST_ABI_VERSION()
PEERDIR(
library/cpp/json
- library/cpp/random_provider
- library/cpp/time_provider
+ library/cpp/random_provider
+ library/cpp/time_provider
ydb/core/yq/libs/common
ydb/core/yq/libs/db_resolver
ydb/library/yql/ast
diff --git a/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_datasource.cpp b/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_datasource.cpp
index c371d97991..e23600f80f 100644
--- a/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_datasource.cpp
+++ b/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_datasource.cpp
@@ -19,7 +19,7 @@ class TClickHouseDataSource : public TDataProviderBase {
public:
TClickHouseDataSource(TClickHouseState::TPtr state, IHTTPGateway::TPtr gateway)
: State_(state)
- , IODiscoveryTransformer_(CreateClickHouseIODiscoveryTransformer(State_))
+ , IODiscoveryTransformer_(CreateClickHouseIODiscoveryTransformer(State_))
, LoadMetaDataTransformer_(CreateClickHouseLoadTableMetadataTransformer(State_, std::move(gateway)))
, TypeAnnotationTransformer_(CreateClickHouseDataSourceTypeAnnotationTransformer(State_))
, DqIntegration_(CreateClickHouseDqIntegration(State_))
@@ -53,10 +53,10 @@ public:
return TypeAnnotationTransformer_->CanParse(node);
}
- IGraphTransformer& GetIODiscoveryTransformer() override {
- return *IODiscoveryTransformer_;
- }
-
+ IGraphTransformer& GetIODiscoveryTransformer() override {
+ return *IODiscoveryTransformer_;
+ }
+
IGraphTransformer& GetLoadTableMetadataTransformer() override {
return *LoadMetaDataTransformer_;
}
diff --git a/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_io_discovery.cpp b/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_io_discovery.cpp
index 9bc674949f..193c1e7bae 100644
--- a/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_io_discovery.cpp
+++ b/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_io_discovery.cpp
@@ -1,118 +1,118 @@
-#include "yql_clickhouse_provider_impl.h"
+#include "yql_clickhouse_provider_impl.h"
#include <ydb/library/yql/providers/clickhouse/expr_nodes/yql_clickhouse_expr_nodes.h>
#include <ydb/library/yql/providers/common/provider/yql_provider_names.h>
#include <ydb/library/yql/core/expr_nodes/yql_expr_nodes.h>
#include <ydb/library/yql/core/yql_expr_optimize.h>
#include <ydb/library/yql/core/yql_graph_transformer.h>
#include <ydb/library/yql/ast/yql_expr.h>
-
+
#include <ydb/core/yq/libs/events/events.h>
-
-namespace NYql {
-
-namespace {
-
-using namespace NNodes;
-
-class TClickHouseIODiscoveryTransformer : public TGraphTransformerBase {
-
-using TDbId2Endpoint = THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TEvEndpointResponse::TEndpoint>;
-
-public:
- TClickHouseIODiscoveryTransformer(TClickHouseState::TPtr state)
- : State_(std::move(state))
- {}
-
- TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final {
- output = input;
-
- if (ctx.Step.IsDone(TExprStep::DiscoveryIO))
- return TStatus::Ok;
-
- if (!State_->DbResolver)
- return TStatus::Ok;
-
- THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TDatabaseAuth> ids;
- if (auto reads = FindNodes(input, [&](const TExprNode::TPtr& node) {
- const TExprBase nodeExpr(node);
- if (!nodeExpr.Maybe<TClRead>())
- return false;
-
- auto read = nodeExpr.Maybe<TClRead>().Cast();
- if (read.DataSource().Category().Value() != ClickHouseProviderName) {
- return false;
- }
- return true;
- }); !reads.empty()) {
- for (auto& node : reads) {
- const TClRead read(node);
- const auto cluster = read.DataSource().Cluster().StringValue();
- YQL_CLOG(DEBUG, ProviderClickHouse) << "Found cluster: " << cluster;
- auto dbId = State_->Configuration->Endpoints[cluster].first;
- dbId = dbId.substr(0, dbId.find(':'));
- YQL_CLOG(DEBUG, ProviderClickHouse) << "Found dbId: " << dbId;
- const auto idKey = std::make_pair(dbId, NYq::DatabaseType::ClickHouse);
- const auto iter = State_->DatabaseIds.find(idKey);
- if (iter != State_->DatabaseIds.end()) {
- YQL_CLOG(DEBUG, ProviderClickHouse) << "Resolve CH id: " << dbId;
- ids[idKey] = iter->second;
- }
- }
- }
- YQL_CLOG(DEBUG, ProviderClickHouse) << "Ids to resolve: " << ids.size();
- if (ids.empty()) {
- return TStatus::Ok;
- }
- AsyncFuture_ = State_->DbResolver->ResolveIds({ids, State_->DbResolver->GetTraceId()}).Apply([resolvedIds_ = ResolvedIds_](const auto& future) {
- *resolvedIds_ = future.GetValue();
- });
- return TStatus::Async;
- }
-
- NThreading::TFuture<void> DoGetAsyncFuture(const TExprNode&) final {
- return AsyncFuture_;
- }
-
- TStatus DoApplyAsyncChanges(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext&) final {
- output = input;
- AsyncFuture_.GetValue();
- FullResolvedIds_.insert(ResolvedIds_->begin(), ResolvedIds_->end());
- ResolvedIds_->clear();
- YQL_CLOG(DEBUG, ProviderClickHouse) << "ResolvedIds: " << FullResolvedIds_.size();
- auto& endpoints = State_->Configuration->Endpoints;
- const auto& id2Clusters = State_->Configuration->DbId2Clusters;
- for (const auto& [dbIdWithType, info] : FullResolvedIds_) {
- const auto& dbId = dbIdWithType.first;
- const auto iter = id2Clusters.find(dbId);
- if (iter == id2Clusters.end()) {
- continue;
- }
- for (const auto& clusterName : iter->second) {
- YQL_CLOG(DEBUG, ProviderClickHouse) << "Resolved endpoint: " << info.Endpoint << " for id: "<< dbId;
- auto& endpoint = endpoints[clusterName].first;
- auto& secure = endpoints[clusterName].second;
- if (const auto it = endpoint.find(':'); it != TString::npos) {
- secure = info.Secure;
- const auto port = endpoint.substr(it);
- endpoint = info.Endpoint;
- if (info.Endpoint.find(':') == TString::npos)
- endpoint += port;
- }
- }
- }
- return TStatus::Ok;
- }
-private:
- const TClickHouseState::TPtr State_;
-
- NThreading::TFuture<void> AsyncFuture_;
- TDbId2Endpoint FullResolvedIds_;
- std::shared_ptr<TDbId2Endpoint> ResolvedIds_ = std::make_shared<TDbId2Endpoint>();
-};
-}
-
-THolder<IGraphTransformer> CreateClickHouseIODiscoveryTransformer(TClickHouseState::TPtr state) {
- return THolder(new TClickHouseIODiscoveryTransformer(std::move(state)));
-}
-
-}
+
+namespace NYql {
+
+namespace {
+
+using namespace NNodes;
+
+class TClickHouseIODiscoveryTransformer : public TGraphTransformerBase {
+
+using TDbId2Endpoint = THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TEvEndpointResponse::TEndpoint>;
+
+public:
+ TClickHouseIODiscoveryTransformer(TClickHouseState::TPtr state)
+ : State_(std::move(state))
+ {}
+
+ TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final {
+ output = input;
+
+ if (ctx.Step.IsDone(TExprStep::DiscoveryIO))
+ return TStatus::Ok;
+
+ if (!State_->DbResolver)
+ return TStatus::Ok;
+
+ THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TDatabaseAuth> ids;
+ if (auto reads = FindNodes(input, [&](const TExprNode::TPtr& node) {
+ const TExprBase nodeExpr(node);
+ if (!nodeExpr.Maybe<TClRead>())
+ return false;
+
+ auto read = nodeExpr.Maybe<TClRead>().Cast();
+ if (read.DataSource().Category().Value() != ClickHouseProviderName) {
+ return false;
+ }
+ return true;
+ }); !reads.empty()) {
+ for (auto& node : reads) {
+ const TClRead read(node);
+ const auto cluster = read.DataSource().Cluster().StringValue();
+ YQL_CLOG(DEBUG, ProviderClickHouse) << "Found cluster: " << cluster;
+ auto dbId = State_->Configuration->Endpoints[cluster].first;
+ dbId = dbId.substr(0, dbId.find(':'));
+ YQL_CLOG(DEBUG, ProviderClickHouse) << "Found dbId: " << dbId;
+ const auto idKey = std::make_pair(dbId, NYq::DatabaseType::ClickHouse);
+ const auto iter = State_->DatabaseIds.find(idKey);
+ if (iter != State_->DatabaseIds.end()) {
+ YQL_CLOG(DEBUG, ProviderClickHouse) << "Resolve CH id: " << dbId;
+ ids[idKey] = iter->second;
+ }
+ }
+ }
+ YQL_CLOG(DEBUG, ProviderClickHouse) << "Ids to resolve: " << ids.size();
+ if (ids.empty()) {
+ return TStatus::Ok;
+ }
+ AsyncFuture_ = State_->DbResolver->ResolveIds({ids, State_->DbResolver->GetTraceId()}).Apply([resolvedIds_ = ResolvedIds_](const auto& future) {
+ *resolvedIds_ = future.GetValue();
+ });
+ return TStatus::Async;
+ }
+
+ NThreading::TFuture<void> DoGetAsyncFuture(const TExprNode&) final {
+ return AsyncFuture_;
+ }
+
+ TStatus DoApplyAsyncChanges(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext&) final {
+ output = input;
+ AsyncFuture_.GetValue();
+ FullResolvedIds_.insert(ResolvedIds_->begin(), ResolvedIds_->end());
+ ResolvedIds_->clear();
+ YQL_CLOG(DEBUG, ProviderClickHouse) << "ResolvedIds: " << FullResolvedIds_.size();
+ auto& endpoints = State_->Configuration->Endpoints;
+ const auto& id2Clusters = State_->Configuration->DbId2Clusters;
+ for (const auto& [dbIdWithType, info] : FullResolvedIds_) {
+ const auto& dbId = dbIdWithType.first;
+ const auto iter = id2Clusters.find(dbId);
+ if (iter == id2Clusters.end()) {
+ continue;
+ }
+ for (const auto& clusterName : iter->second) {
+ YQL_CLOG(DEBUG, ProviderClickHouse) << "Resolved endpoint: " << info.Endpoint << " for id: "<< dbId;
+ auto& endpoint = endpoints[clusterName].first;
+ auto& secure = endpoints[clusterName].second;
+ if (const auto it = endpoint.find(':'); it != TString::npos) {
+ secure = info.Secure;
+ const auto port = endpoint.substr(it);
+ endpoint = info.Endpoint;
+ if (info.Endpoint.find(':') == TString::npos)
+ endpoint += port;
+ }
+ }
+ }
+ return TStatus::Ok;
+ }
+private:
+ const TClickHouseState::TPtr State_;
+
+ NThreading::TFuture<void> AsyncFuture_;
+ TDbId2Endpoint FullResolvedIds_;
+ std::shared_ptr<TDbId2Endpoint> ResolvedIds_ = std::make_shared<TDbId2Endpoint>();
+};
+}
+
+THolder<IGraphTransformer> CreateClickHouseIODiscoveryTransformer(TClickHouseState::TPtr state) {
+ return THolder(new TClickHouseIODiscoveryTransformer(std::move(state)));
+}
+
+}
diff --git a/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_provider.cpp b/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_provider.cpp
index 7287848b5d..15c4d834e3 100644
--- a/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_provider.cpp
+++ b/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_provider.cpp
@@ -4,10 +4,10 @@
namespace NYql {
-TDataProviderInitializer GetClickHouseDataProviderInitializer(
+TDataProviderInitializer GetClickHouseDataProviderInitializer(
IHTTPGateway::TPtr gateway,
- const std::shared_ptr<NYq::TDatabaseAsyncResolverWithMeta> dbResolverWithMeta)
-{
+ const std::shared_ptr<NYq::TDatabaseAsyncResolverWithMeta> dbResolverWithMeta)
+{
return [gateway, dbResolverWithMeta] (
const TString& userName,
const TString& sessionId,
@@ -29,9 +29,9 @@ TDataProviderInitializer GetClickHouseDataProviderInitializer(
state->Types = typeCtx.Get();
state->FunctionRegistry = functionRegistry;
- state->DbResolver = dbResolverWithMeta;
+ state->DbResolver = dbResolverWithMeta;
if (gatewaysConfig) {
- state->Configuration->Init(gatewaysConfig->GetClickHouse(), state->DbResolver, state->DatabaseIds);
+ state->Configuration->Init(gatewaysConfig->GetClickHouse(), state->DbResolver, state->DatabaseIds);
}
TDataProviderInfo info;
diff --git a/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_provider.h b/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_provider.h
index 3b918ca877..7184fe1ba3 100644
--- a/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_provider.h
+++ b/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_provider.h
@@ -26,14 +26,14 @@ struct TClickHouseState : public TThrRefBase
TTypeAnnotationContext* Types = nullptr;
TClickHouseConfiguration::TPtr Configuration = MakeIntrusive<TClickHouseConfiguration>();
const NKikimr::NMiniKQL::IFunctionRegistry* FunctionRegistry = nullptr;
- THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TDatabaseAuth> DatabaseIds;
- std::shared_ptr<NYq::TDatabaseAsyncResolverWithMeta> DbResolver;
+ THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TDatabaseAuth> DatabaseIds;
+ std::shared_ptr<NYq::TDatabaseAsyncResolverWithMeta> DbResolver;
};
-TDataProviderInitializer GetClickHouseDataProviderInitializer(
+TDataProviderInitializer GetClickHouseDataProviderInitializer(
IHTTPGateway::TPtr gateway,
- std::shared_ptr<NYq::TDatabaseAsyncResolverWithMeta> dbResolverWithMeta = nullptr
-);
+ std::shared_ptr<NYq::TDatabaseAsyncResolverWithMeta> dbResolverWithMeta = nullptr
+);
TIntrusivePtr<IDataProvider> CreateClickHouseDataSource(TClickHouseState::TPtr state, IHTTPGateway::TPtr gateway);
TIntrusivePtr<IDataProvider> CreateClickHouseDataSink(TClickHouseState::TPtr state);
diff --git a/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_provider_impl.h b/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_provider_impl.h
index 026b613b1c..cdf5c20405 100644
--- a/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_provider_impl.h
+++ b/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_provider_impl.h
@@ -10,7 +10,7 @@
namespace NYql {
-THolder<IGraphTransformer> CreateClickHouseIODiscoveryTransformer(TClickHouseState::TPtr state);
+THolder<IGraphTransformer> CreateClickHouseIODiscoveryTransformer(TClickHouseState::TPtr state);
THolder<IGraphTransformer> CreateClickHouseLoadTableMetadataTransformer(TClickHouseState::TPtr state, IHTTPGateway::TPtr gateway);
THolder<TVisitorTransformerBase> CreateClickHouseDataSourceTypeAnnotationTransformer(TClickHouseState::TPtr state);
diff --git a/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_settings.h b/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_settings.h
index 5a29448cf3..8d0b094af4 100644
--- a/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_settings.h
+++ b/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_settings.h
@@ -7,7 +7,7 @@
#include <ydb/core/yq/libs/events/events.h>
#include <ydb/core/yq/libs/db_resolver/db_async_resolver_with_meta.h>
#include <ydb/core/yq/libs/common/database_token_builder.h>
-
+
namespace NYql {
struct TClickHouseSettings {
@@ -21,10 +21,10 @@ struct TClickHouseConfiguration : public TClickHouseSettings, public NCommon::TS
TClickHouseConfiguration(const TClickHouseConfiguration&) = delete;
template <typename TProtoConfig>
- void Init(
- const TProtoConfig& config,
- const std::shared_ptr<NYq::TDatabaseAsyncResolverWithMeta> dbResolver,
- THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TDatabaseAuth>& databaseIds)
+ void Init(
+ const TProtoConfig& config,
+ const std::shared_ptr<NYq::TDatabaseAsyncResolverWithMeta> dbResolver,
+ THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TDatabaseAuth>& databaseIds)
{
TVector<TString> clusters(Reserve(config.ClusterMappingSize()));
for (auto& cluster: config.GetClusterMapping()) {
@@ -36,28 +36,28 @@ struct TClickHouseConfiguration : public TClickHouseSettings, public NCommon::TS
this->Dispatch(config.GetDefaultSettings());
for (auto& cluster: config.GetClusterMapping()) {
this->Dispatch(cluster.GetName(), cluster.GetSettings());
-
- if (dbResolver) { //TODO: change log level
- YQL_CLOG(DEBUG, ProviderClickHouse) << "Settings: clusterName = " << cluster.GetName()
- << ", clusterDbId = " << cluster.GetId() << ", cluster.GetCluster(): " << cluster.GetCluster() << ", HasCluster: " << (cluster.HasCluster() ? "TRUE" : "FALSE") ;
- dbResolver->TryAddDbIdToResolve(cluster.HasCluster(), cluster.GetName(), cluster.GetId(), NYq::DatabaseType::ClickHouse, databaseIds);
- if (cluster.GetId()) {
- DbId2Clusters[cluster.GetId()].emplace_back(cluster.GetName());
- YQL_CLOG(DEBUG, ProviderClickHouse) << "Add dbId: " << cluster.GetId() << " to DbId2Clusters";
- }
- }
-
+
+ if (dbResolver) { //TODO: change log level
+ YQL_CLOG(DEBUG, ProviderClickHouse) << "Settings: clusterName = " << cluster.GetName()
+ << ", clusterDbId = " << cluster.GetId() << ", cluster.GetCluster(): " << cluster.GetCluster() << ", HasCluster: " << (cluster.HasCluster() ? "TRUE" : "FALSE") ;
+ dbResolver->TryAddDbIdToResolve(cluster.HasCluster(), cluster.GetName(), cluster.GetId(), NYq::DatabaseType::ClickHouse, databaseIds);
+ if (cluster.GetId()) {
+ DbId2Clusters[cluster.GetId()].emplace_back(cluster.GetName());
+ YQL_CLOG(DEBUG, ProviderClickHouse) << "Add dbId: " << cluster.GetId() << " to DbId2Clusters";
+ }
+ }
+
Tokens[cluster.GetName()] = cluster.GetCHToken();
// TODO: Drop later
- TString endpoint;
- if (cluster.HasCluster()) {
- endpoint = cluster.GetCluster();
- if (endpoint.StartsWith("https://")) {
- endpoint = endpoint.substr(8);
- }
- endpoint = endpoint.substr(0, endpoint.find(':'));
- } else {
- endpoint = cluster.GetId();
+ TString endpoint;
+ if (cluster.HasCluster()) {
+ endpoint = cluster.GetCluster();
+ if (endpoint.StartsWith("https://")) {
+ endpoint = endpoint.substr(8);
+ }
+ endpoint = endpoint.substr(0, endpoint.find(':'));
+ } else {
+ endpoint = cluster.GetId();
}
Endpoints[cluster.GetName()] = std::make_pair(
endpoint + ":" + ToString(cluster.GetNativeHostPort()), cluster.GetNativeSecure());
@@ -100,7 +100,7 @@ struct TClickHouseConfiguration : public TClickHouseSettings, public NCommon::TS
THashMap<TString, TString> Tokens;
THashMap<TString, std::tuple<TString, EHostScheme, ui16>> Urls;
THashMap<TString, std::pair<TString, bool>> Endpoints;
- THashMap<TString, TVector<TString>> DbId2Clusters; // DatabaseId -> ClusterNames
+ THashMap<TString, TVector<TString>> DbId2Clusters; // DatabaseId -> ClusterNames
};
} // NYql
diff --git a/ydb/library/yql/providers/common/http_gateway/ya.make b/ydb/library/yql/providers/common/http_gateway/ya.make
index 8fdcdfe9a3..f53489973e 100644
--- a/ydb/library/yql/providers/common/http_gateway/ya.make
+++ b/ydb/library/yql/providers/common/http_gateway/ya.make
@@ -10,9 +10,9 @@ SRCS(
)
PEERDIR(
- contrib/libs/curl
- library/cpp/monlib/dynamic_counters
- ydb/library/yql/providers/common/proto
+ contrib/libs/curl
+ library/cpp/monlib/dynamic_counters
+ ydb/library/yql/providers/common/proto
ydb/library/yql/public/issue
)
diff --git a/ydb/library/yql/providers/common/http_gateway/yql_http_gateway.cpp b/ydb/library/yql/providers/common/http_gateway/yql_http_gateway.cpp
index 10897fc634..5c2a1451b4 100644
--- a/ydb/library/yql/providers/common/http_gateway/yql_http_gateway.cpp
+++ b/ydb/library/yql/providers/common/http_gateway/yql_http_gateway.cpp
@@ -131,24 +131,24 @@ public:
using TPtr = std::shared_ptr<THTTPMultiGateway>;
using TWeakPtr = std::weak_ptr<THTTPMultiGateway>;
- explicit THTTPMultiGateway(
- const THttpGatewayConfig* httpGatewaysCfg,
- NMonitoring::TDynamicCounterPtr counters)
- : Counters(std::move(counters))
- , Rps(Counters->GetCounter("RPS", true))
- , InFlight(Counters->GetCounter("InFlight"))
- , AllocatedMemory(Counters->GetCounter("AllocatedMemory"))
- {
- if (!httpGatewaysCfg) {
- return;
- }
- if (httpGatewaysCfg->HasMaxInFlightCount()) {
- MaxHandlers = httpGatewaysCfg->GetMaxInFlightCount();
- }
- if (httpGatewaysCfg->HasMaxSimulatenousDownloadsSize()) {
- MaxSimulatenousDownloadsSize = httpGatewaysCfg->GetMaxSimulatenousDownloadsSize();
- }
- }
+ explicit THTTPMultiGateway(
+ const THttpGatewayConfig* httpGatewaysCfg,
+ NMonitoring::TDynamicCounterPtr counters)
+ : Counters(std::move(counters))
+ , Rps(Counters->GetCounter("RPS", true))
+ , InFlight(Counters->GetCounter("InFlight"))
+ , AllocatedMemory(Counters->GetCounter("AllocatedMemory"))
+ {
+ if (!httpGatewaysCfg) {
+ return;
+ }
+ if (httpGatewaysCfg->HasMaxInFlightCount()) {
+ MaxHandlers = httpGatewaysCfg->GetMaxInFlightCount();
+ }
+ if (httpGatewaysCfg->HasMaxSimulatenousDownloadsSize()) {
+ MaxSimulatenousDownloadsSize = httpGatewaysCfg->GetMaxSimulatenousDownloadsSize();
+ }
+ }
~THTTPMultiGateway() {
if (Handle)
@@ -156,17 +156,17 @@ public:
Thread.join();
}
private:
- std::size_t MaxHandlers = 1024U;
- std::size_t MaxSimulatenousDownloadsSize = 8_GB;
+ std::size_t MaxHandlers = 1024U;
+ std::size_t MaxSimulatenousDownloadsSize = 8_GB;
static void Perform(const TWeakPtr& weak) {
OutputSize.store(0ULL);
curl_global_init(CURL_GLOBAL_ALL);
if (const auto handle = curl_multi_init()) {
- if (const auto& initHandle = weak.lock()) {
- initHandle->Handle = handle;
- }
+ if (const auto& initHandle = weak.lock()) {
+ initHandle->Handle = handle;
+ }
for (size_t handlers = 0U;;) {
if (const auto& self = weak.lock())
@@ -218,7 +218,7 @@ private:
curl_multi_add_handle(Handle, handle);
}
- AllocatedMemory->Set(AllocatedSize);
+ AllocatedMemory->Set(AllocatedSize);
return Allocated.size();
}
@@ -235,10 +235,10 @@ private:
if (Await.empty() && Allocated.empty())
Requests.clear();
}
- if (easy) {
- InFlight->Dec();
+ if (easy) {
+ InFlight->Dec();
easy->Done(result);
- }
+ }
}
void Fail(CURLMcode result) {
@@ -258,7 +258,7 @@ private:
}
const TIssue error(curl_multi_strerror(result));
- InFlight->Sub(works.size());
+ InFlight->Sub(works.size());
while (!works.empty()) {
works.top()->Fail(error);
works.pop();
@@ -266,9 +266,9 @@ private:
}
void Download(TString url, THeaders headers, std::size_t expectedSize, TOnResult callback, TString data) final {
- Rps->Inc();
- InFlight->Inc();
-
+ Rps->Inc();
+ InFlight->Inc();
+
const std::unique_lock lock(Sync);
auto& entry = Requests[TKeyType(url, headers, data)];
if (const auto& easy = entry.lock())
@@ -298,11 +298,11 @@ private:
static std::mutex CreateSync;
static TWeakPtr Singleton;
-
- const NMonitoring::TDynamicCounterPtr Counters;
- const NMonitoring::TDynamicCounters::TCounterPtr Rps;
- const NMonitoring::TDynamicCounters::TCounterPtr InFlight;
- const NMonitoring::TDynamicCounters::TCounterPtr AllocatedMemory;
+
+ const NMonitoring::TDynamicCounterPtr Counters;
+ const NMonitoring::TDynamicCounters::TCounterPtr Rps;
+ const NMonitoring::TDynamicCounters::TCounterPtr InFlight;
+ const NMonitoring::TDynamicCounters::TCounterPtr AllocatedMemory;
};
std::atomic_size_t THTTPMultiGateway::OutputSize = 0ULL;
@@ -386,12 +386,12 @@ IHTTPGateway::TContent::~TContent()
template<>
IHTTPGateway::TPtr
-IHTTPGateway::Make<true>(const THttpGatewayConfig* httpGatewaysCfg, NMonitoring::TDynamicCounterPtr counters) {
+IHTTPGateway::Make<true>(const THttpGatewayConfig* httpGatewaysCfg, NMonitoring::TDynamicCounterPtr counters) {
const std::unique_lock lock(THTTPMultiGateway::CreateSync);
if (const auto g = THTTPMultiGateway::Singleton.lock())
return g;
- const auto gateway = std::make_shared<THTTPMultiGateway>(httpGatewaysCfg, std::move(counters));
+ const auto gateway = std::make_shared<THTTPMultiGateway>(httpGatewaysCfg, std::move(counters));
THTTPMultiGateway::Singleton = gateway;
gateway->Thread = std::thread(std::bind(&THTTPMultiGateway::Perform, THTTPMultiGateway::Singleton));
return gateway;
@@ -399,7 +399,7 @@ IHTTPGateway::Make<true>(const THttpGatewayConfig* httpGatewaysCfg, NMonitoring:
template<>
IHTTPGateway::TPtr
-IHTTPGateway::Make<false>(const THttpGatewayConfig*, NMonitoring::TDynamicCounterPtr) {
+IHTTPGateway::Make<false>(const THttpGatewayConfig*, NMonitoring::TDynamicCounterPtr) {
return std::make_shared<THTTPEasyGateway>();
}
diff --git a/ydb/library/yql/providers/common/http_gateway/yql_http_gateway.h b/ydb/library/yql/providers/common/http_gateway/yql_http_gateway.h
index a6ddfdcf27..4e15aa9c78 100644
--- a/ydb/library/yql/providers/common/http_gateway/yql_http_gateway.h
+++ b/ydb/library/yql/providers/common/http_gateway/yql_http_gateway.h
@@ -1,10 +1,10 @@
#pragma once
-#include <ydb/library/yql/providers/common/proto/gateways_config.pb.h>
-
+#include <ydb/library/yql/providers/common/proto/gateways_config.pb.h>
+
#include <ydb/library/yql/public/issue/yql_issue.h>
#include <library/cpp/containers/stack_vector/stack_vec.h>
-#include <library/cpp/monlib/dynamic_counters/counters.h>
+#include <library/cpp/monlib/dynamic_counters/counters.h>
#include <variant>
#include <functional>
@@ -19,9 +19,9 @@ public:
virtual ~IHTTPGateway() = default;
template<bool SingleThread = true>
- static TPtr Make(
- const THttpGatewayConfig* httpGatewaysCfg = nullptr,
- NMonitoring::TDynamicCounterPtr counters = MakeIntrusive<NMonitoring::TDynamicCounters>());
+ static TPtr Make(
+ const THttpGatewayConfig* httpGatewaysCfg = nullptr,
+ NMonitoring::TDynamicCounterPtr counters = MakeIntrusive<NMonitoring::TDynamicCounters>());
class TContent : private TString {
friend class TEasyCurl;
diff --git a/ydb/library/yql/providers/common/proto/gateways_config.proto b/ydb/library/yql/providers/common/proto/gateways_config.proto
index 42c53e1e8e..8561fe2536 100644
--- a/ydb/library/yql/providers/common/proto/gateways_config.proto
+++ b/ydb/library/yql/providers/common/proto/gateways_config.proto
@@ -50,13 +50,13 @@ enum ETokenType {
CREDENTIALS = 3;
};
-/////////////////////////////// HTTP GATEWAY //////////////////////
-
-message THttpGatewayConfig {
- optional uint32 MaxInFlightCount = 1;
- optional uint64 MaxSimulatenousDownloadsSize = 2;
-}
-
+/////////////////////////////// HTTP GATEWAY //////////////////////
+
+message THttpGatewayConfig {
+ optional uint32 MaxInFlightCount = 1;
+ optional uint64 MaxSimulatenousDownloadsSize = 2;
+}
+
/////////////////////////////// YT ///////////////////////////////
message TYtClusterConfig {
@@ -179,7 +179,7 @@ message TClickHouseClusterConfig {
optional NYql.NProto.TClickHouseRunnerConfig Runner = 10;
optional uint32 NativeHostPort = 11 [default = 9000];
optional bool NativeSecure = 12 [default = false];
- optional string Id = 13;
+ optional string Id = 13;
}
message TClickHouseGatewayConfig {
@@ -453,5 +453,5 @@ message TGatewaysConfig {
optional TYdbGatewayConfig Ydb = 15;
optional TPqGatewayConfig Pq = 16;
optional TS3GatewayConfig S3 = 17;
- optional THttpGatewayConfig HttpGateway = 18;
+ optional THttpGatewayConfig HttpGateway = 18;
}
diff --git a/ydb/library/yql/providers/dq/actors/worker_actor.cpp b/ydb/library/yql/providers/dq/actors/worker_actor.cpp
index a7f27f1d8d..f4503a60b2 100644
--- a/ydb/library/yql/providers/dq/actors/worker_actor.cpp
+++ b/ydb/library/yql/providers/dq/actors/worker_actor.cpp
@@ -71,7 +71,7 @@ class TDqWorker: public TRichActor<TDqWorker>
public:
static constexpr char ActorName[] = "YQL_DQ_WORKER";
-
+
explicit TDqWorker(
const ITaskRunnerActorFactory::TPtr& taskRunnerActorFactory,
const IDqSourceActorFactory::TPtr& sourceActorFactory,
diff --git a/ydb/library/yql/providers/dq/api/protos/dqs.proto b/ydb/library/yql/providers/dq/api/protos/dqs.proto
index 6d8929be78..09aecfcf92 100644
--- a/ydb/library/yql/providers/dq/api/protos/dqs.proto
+++ b/ydb/library/yql/providers/dq/api/protos/dqs.proto
@@ -176,7 +176,7 @@ message TQueryResponse {
bool NeedFallback = 6;
repeated TMetric Metric = 5;
bool Truncated = 7;
- uint64 RowsCount = 8;
+ uint64 RowsCount = 8;
}
message TDqFailure {
diff --git a/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp b/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp
index f33d6264e3..8759f44221 100644
--- a/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp
+++ b/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp
@@ -764,10 +764,10 @@ private:
}
auto raw = NYT::NodeToYsonString(item);
- const bool truncated = res.Truncated;
- const ui64 rowsCount = res.RowsCount;
+ const bool truncated = res.Truncated;
+ const ui64 rowsCount = res.RowsCount;
- if (truncated && item.IsList()) {
+ if (truncated && item.IsList()) {
ui64 bytes = 0;
ui64 rows = 0;
writer.OnBeginList();
@@ -804,11 +804,11 @@ private:
writer.OnRaw(raw);
}
- if (rowsCount) {
- writer.OnKeyedItem("RowsCount");
- writer.OnUint64Scalar(rowsCount);
- }
-
+ if (rowsCount) {
+ writer.OnKeyedItem("RowsCount");
+ writer.OnUint64Scalar(rowsCount);
+ }
+
writer.OnEndMap();
input->SetResult(ctx.NewAtom(input->Pos(), out.Str()));
@@ -1100,8 +1100,8 @@ private:
auto raw = NYT::NodeToYsonString(item);
TString trStr = "";
- const bool truncated = res.Truncated;
- const ui64 rowsCount = res.RowsCount;
+ const bool truncated = res.Truncated;
+ const ui64 rowsCount = res.RowsCount;
if (truncated && !state->TypeCtx->ForceDq && !enableFullResultWrite) {
auto issue = TIssue(ctx.GetPosition(input->Pos()), TStringBuilder() << "DQ cannot execute the query. Cause: " << "too big result " << trStr).SetCode(TIssuesIds::DQ_GATEWAY_NEED_FALLBACK_ERROR, TSeverityIds::S_INFO);
@@ -1158,11 +1158,11 @@ private:
writer.OnRaw(raw);
}
- if (rowsCount) {
- writer.OnKeyedItem("RowsCount");
- writer.OnUint64Scalar(rowsCount);
- }
-
+ if (rowsCount) {
+ writer.OnKeyedItem("RowsCount");
+ writer.OnUint64Scalar(rowsCount);
+ }
+
writer.OnEndMap();
ctx.IssueManager.RaiseIssues(res.Issues);
diff --git a/ydb/library/yql/providers/dq/provider/yql_dq_gateway.h b/ydb/library/yql/providers/dq/provider/yql_dq_gateway.h
index 64385ad351..d562776033 100644
--- a/ydb/library/yql/providers/dq/provider/yql_dq_gateway.h
+++ b/ydb/library/yql/providers/dq/provider/yql_dq_gateway.h
@@ -49,7 +49,7 @@ public:
bool ForceFallback = false;
bool Retriable = false;
bool Truncated = false;
- ui64 RowsCount = 0;
+ ui64 RowsCount = 0;
TIssues Issues;
TOperationStatistics Statistics;
diff --git a/ydb/library/yql/providers/pq/async_io/dq_pq_read_actor.cpp b/ydb/library/yql/providers/pq/async_io/dq_pq_read_actor.cpp
index a58129d428..e26956545e 100644
--- a/ydb/library/yql/providers/pq/async_io/dq_pq_read_actor.cpp
+++ b/ydb/library/yql/providers/pq/async_io/dq_pq_read_actor.cpp
@@ -126,8 +126,8 @@ public:
return opts;
}
- static constexpr char ActorName[] = "DQ_PQ_READ_ACTOR";
-
+ static constexpr char ActorName[] = "DQ_PQ_READ_ACTOR";
+
public:
void SaveState(const NDqProto::TCheckpoint& checkpoint, NDqProto::TSourceState& state) override {
NPq::NProto::TDqPqTopicSourceState stateProto;
diff --git a/ydb/library/yql/providers/pq/async_io/dq_pq_write_actor.cpp b/ydb/library/yql/providers/pq/async_io/dq_pq_write_actor.cpp
index 54fb25a147..85fbc91c01 100644
--- a/ydb/library/yql/providers/pq/async_io/dq_pq_write_actor.cpp
+++ b/ydb/library/yql/providers/pq/async_io/dq_pq_write_actor.cpp
@@ -109,8 +109,8 @@ public:
, PersQueueClient(Driver, GetPersQueueClientSettings())
{ }
- static constexpr char ActorName[] = "DQ_PQ_WRITE_ACTOR";
-
+ static constexpr char ActorName[] = "DQ_PQ_WRITE_ACTOR";
+
public:
void SendData(
NKikimr::NMiniKQL::TUnboxedValueVector&& batch,
diff --git a/ydb/library/yql/providers/pq/gateway/dummy/yql_pq_dummy_gateway.cpp b/ydb/library/yql/providers/pq/gateway/dummy/yql_pq_dummy_gateway.cpp
index 8a37bb133e..b1d64befb3 100644
--- a/ydb/library/yql/providers/pq/gateway/dummy/yql_pq_dummy_gateway.cpp
+++ b/ydb/library/yql/providers/pq/gateway/dummy/yql_pq_dummy_gateway.cpp
@@ -52,16 +52,16 @@ TDummyPqGateway& TDummyPqGateway::AddDummyTopic(const TDummyTopic& topic) {
}
}
-void TDummyPqGateway::UpdateClusterConfigs(
- const TString& clusterName,
- const TString& endpoint,
- const TString& database,
- bool secure)
-{
- Y_UNUSED(clusterName);
- Y_UNUSED(endpoint);
- Y_UNUSED(database);
- Y_UNUSED(secure);
-}
-
+void TDummyPqGateway::UpdateClusterConfigs(
+ const TString& clusterName,
+ const TString& endpoint,
+ const TString& database,
+ bool secure)
+{
+ Y_UNUSED(clusterName);
+ Y_UNUSED(endpoint);
+ Y_UNUSED(database);
+ Y_UNUSED(secure);
+}
+
} // namespace NYql
diff --git a/ydb/library/yql/providers/pq/gateway/dummy/yql_pq_dummy_gateway.h b/ydb/library/yql/providers/pq/gateway/dummy/yql_pq_dummy_gateway.h
index 78262dc0a1..753c849c20 100644
--- a/ydb/library/yql/providers/pq/gateway/dummy/yql_pq_dummy_gateway.h
+++ b/ydb/library/yql/providers/pq/gateway/dummy/yql_pq_dummy_gateway.h
@@ -36,12 +36,12 @@ public:
NPq::NConfigurationManager::TAsyncDescribePathResult DescribePath(const TString& sessionId, const TString& cluster, const TString& database, const TString& path, const TString& token) override;
- void UpdateClusterConfigs(
- const TString& clusterName,
- const TString& endpoint,
- const TString& database,
- bool secure) override;
-
+ void UpdateClusterConfigs(
+ const TString& clusterName,
+ const TString& endpoint,
+ const TString& database,
+ bool secure) override;
+
private:
mutable TMutex Mutex;
THashMap<std::pair<TString, TString>, TDummyTopic> Topics;
diff --git a/ydb/library/yql/providers/pq/gateway/native/yql_pq_gateway.cpp b/ydb/library/yql/providers/pq/gateway/native/yql_pq_gateway.cpp
index dafe21ea2f..bd89565050 100644
--- a/ydb/library/yql/providers/pq/gateway/native/yql_pq_gateway.cpp
+++ b/ydb/library/yql/providers/pq/gateway/native/yql_pq_gateway.cpp
@@ -26,12 +26,12 @@ public:
const TString& path,
const TString& token) override;
- void UpdateClusterConfigs(
- const TString& clusterName,
- const TString& endpoint,
- const TString& database,
- bool secure) override;
-
+ void UpdateClusterConfigs(
+ const TString& clusterName,
+ const TString& endpoint,
+ const TString& database,
+ bool secure) override;
+
private:
void InitClusterConfigs();
TPqSession::TPtr GetExistingSession(const TString& sessionId) const;
@@ -68,22 +68,22 @@ void TPqNativeGateway::InitClusterConfigs() {
}
}
-void TPqNativeGateway::UpdateClusterConfigs(
- const TString& clusterName,
- const TString& endpoint,
- const TString& database,
- bool secure)
-{
- with_lock (Mutex) {
- const auto foundCluster = ClusterConfigs->find(clusterName);
- Y_VERIFY(foundCluster != ClusterConfigs->end());
- auto& cluster = foundCluster->second;
- cluster.SetEndpoint(endpoint);
- cluster.SetDatabase(database);
- cluster.SetUseSsl(secure);
- }
-}
-
+void TPqNativeGateway::UpdateClusterConfigs(
+ const TString& clusterName,
+ const TString& endpoint,
+ const TString& database,
+ bool secure)
+{
+ with_lock (Mutex) {
+ const auto foundCluster = ClusterConfigs->find(clusterName);
+ Y_VERIFY(foundCluster != ClusterConfigs->end());
+ auto& cluster = foundCluster->second;
+ cluster.SetEndpoint(endpoint);
+ cluster.SetDatabase(database);
+ cluster.SetUseSsl(secure);
+ }
+}
+
NThreading::TFuture<void> TPqNativeGateway::OpenSession(const TString& sessionId, const TString& username) {
with_lock (Mutex) {
auto [sessionIt, isNewSession] = Sessions.emplace(sessionId,
diff --git a/ydb/library/yql/providers/pq/provider/ya.make b/ydb/library/yql/providers/pq/provider/ya.make
index ad331ccf71..ead04a48fc 100644
--- a/ydb/library/yql/providers/pq/provider/ya.make
+++ b/ydb/library/yql/providers/pq/provider/ya.make
@@ -9,7 +9,7 @@ OWNER(
SRCS(
yql_pq_datasink.cpp
yql_pq_datasink_execution.cpp
- yql_pq_datasink_io_discovery.cpp
+ yql_pq_datasink_io_discovery.cpp
yql_pq_datasink_type_ann.cpp
yql_pq_datasource.cpp
yql_pq_datasource_type_ann.cpp
diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_datasink.cpp b/ydb/library/yql/providers/pq/provider/yql_pq_datasink.cpp
index 24e01d3eae..380df33d52 100644
--- a/ydb/library/yql/providers/pq/provider/yql_pq_datasink.cpp
+++ b/ydb/library/yql/providers/pq/provider/yql_pq_datasink.cpp
@@ -33,7 +33,7 @@ public:
TPqDataSinkProvider(TPqState::TPtr state, IPqGateway::TPtr gateway)
: State_(state)
, Gateway_(gateway)
- , IODiscoveryTransformer_(CreatePqDataSinkIODiscoveryTransformer(State_))
+ , IODiscoveryTransformer_(CreatePqDataSinkIODiscoveryTransformer(State_))
, TypeAnnotationTransformer_(CreatePqDataSinkTypeAnnotationTransformer(State_))
, ExecutionTransformer_(CreatePqDataSinkExecTransformer(State_))
, LogicalOptProposalTransformer_(CreatePqLogicalOptProposalTransformer(State_))
@@ -53,10 +53,10 @@ public:
return TypeAnnotationTransformer_->CanParse(node);
}
- IGraphTransformer& GetIODiscoveryTransformer() override {
- return *IODiscoveryTransformer_;
- }
-
+ IGraphTransformer& GetIODiscoveryTransformer() override {
+ return *IODiscoveryTransformer_;
+ }
+
IGraphTransformer& GetTypeAnnotationTransformer(bool instantOnly) override {
Y_UNUSED(instantOnly);
return *TypeAnnotationTransformer_;
@@ -174,7 +174,7 @@ public:
private:
TPqState::TPtr State_;
IPqGateway::TPtr Gateway_;
- THolder<IGraphTransformer> IODiscoveryTransformer_;
+ THolder<IGraphTransformer> IODiscoveryTransformer_;
THolder<TVisitorTransformerBase> TypeAnnotationTransformer_;
THolder<TExecTransformerBase> ExecutionTransformer_;
THolder<IGraphTransformer> LogicalOptProposalTransformer_;
diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_datasink_io_discovery.cpp b/ydb/library/yql/providers/pq/provider/yql_pq_datasink_io_discovery.cpp
index 21abaccb95..847afd3b3d 100644
--- a/ydb/library/yql/providers/pq/provider/yql_pq_datasink_io_discovery.cpp
+++ b/ydb/library/yql/providers/pq/provider/yql_pq_datasink_io_discovery.cpp
@@ -1,70 +1,70 @@
-#include "yql_pq_provider_impl.h"
-#include "yql_pq_helpers.h"
-
+#include "yql_pq_provider_impl.h"
+#include "yql_pq_helpers.h"
+
#include <ydb/library/yql/core/yql_expr_optimize.h>
#include <ydb/library/yql/utils/log/log.h>
#include <ydb/library/yql/providers/pq/expr_nodes/yql_pq_expr_nodes.h>
-
-namespace NYql {
-
-namespace {
-
-using namespace NNodes;
-
-class TPqDataSinkIODiscoveryTransformer : public TGraphTransformerBase {
-
-using TDbId2Endpoint = THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TEvEndpointResponse::TEndpoint>;
-
-public:
- explicit TPqDataSinkIODiscoveryTransformer(TPqState::TPtr state)
- : State_(state)
- {
- }
-
- TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final {
- Y_UNUSED(ctx);
- output = input;
- if (ctx.Step.IsDone(TExprStep::DiscoveryIO))
- return TStatus::Ok;
-
- if (!State_->DbResolver)
- return TStatus::Ok;
-
- THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TDatabaseAuth> ids;
- FindYdsDbIdsForResolving(State_, input, ids);
-
- if (ids.empty())
- return TStatus::Ok;
-
- AsyncFuture_ = State_->DbResolver->ResolveIds({ids, State_->DbResolver->GetTraceId()}).Apply([resolvedIds_ = ResolvedIds_](const auto& future) {
- *resolvedIds_ = future.GetValue();
- });
- return TStatus::Async;
- }
-
- NThreading::TFuture<void> DoGetAsyncFuture(const TExprNode&) final {
- return AsyncFuture_;
- }
-
- TStatus DoApplyAsyncChanges(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext&) final {
- output = input;
- AsyncFuture_.GetValue();
- FullResolvedIds_.insert(ResolvedIds_->begin(), ResolvedIds_->end());
- ResolvedIds_->clear();
- FillSettingsWithResolvedYdsIds(State_, FullResolvedIds_);
- return TStatus::Ok;
- }
-
-private:
- const TPqState::TPtr State_;
- NThreading::TFuture<void> AsyncFuture_;
- TDbId2Endpoint FullResolvedIds_;
- std::shared_ptr<TDbId2Endpoint> ResolvedIds_ = std::make_shared<TDbId2Endpoint>();
-};
-}
-
-THolder<IGraphTransformer> CreatePqDataSinkIODiscoveryTransformer(TPqState::TPtr state) {
- return THolder(new TPqDataSinkIODiscoveryTransformer(state));
-}
-
-}
+
+namespace NYql {
+
+namespace {
+
+using namespace NNodes;
+
+class TPqDataSinkIODiscoveryTransformer : public TGraphTransformerBase {
+
+using TDbId2Endpoint = THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TEvEndpointResponse::TEndpoint>;
+
+public:
+ explicit TPqDataSinkIODiscoveryTransformer(TPqState::TPtr state)
+ : State_(state)
+ {
+ }
+
+ TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final {
+ Y_UNUSED(ctx);
+ output = input;
+ if (ctx.Step.IsDone(TExprStep::DiscoveryIO))
+ return TStatus::Ok;
+
+ if (!State_->DbResolver)
+ return TStatus::Ok;
+
+ THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TDatabaseAuth> ids;
+ FindYdsDbIdsForResolving(State_, input, ids);
+
+ if (ids.empty())
+ return TStatus::Ok;
+
+ AsyncFuture_ = State_->DbResolver->ResolveIds({ids, State_->DbResolver->GetTraceId()}).Apply([resolvedIds_ = ResolvedIds_](const auto& future) {
+ *resolvedIds_ = future.GetValue();
+ });
+ return TStatus::Async;
+ }
+
+ NThreading::TFuture<void> DoGetAsyncFuture(const TExprNode&) final {
+ return AsyncFuture_;
+ }
+
+ TStatus DoApplyAsyncChanges(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext&) final {
+ output = input;
+ AsyncFuture_.GetValue();
+ FullResolvedIds_.insert(ResolvedIds_->begin(), ResolvedIds_->end());
+ ResolvedIds_->clear();
+ FillSettingsWithResolvedYdsIds(State_, FullResolvedIds_);
+ return TStatus::Ok;
+ }
+
+private:
+ const TPqState::TPtr State_;
+ NThreading::TFuture<void> AsyncFuture_;
+ TDbId2Endpoint FullResolvedIds_;
+ std::shared_ptr<TDbId2Endpoint> ResolvedIds_ = std::make_shared<TDbId2Endpoint>();
+};
+}
+
+THolder<IGraphTransformer> CreatePqDataSinkIODiscoveryTransformer(TPqState::TPtr state) {
+ return THolder(new TPqDataSinkIODiscoveryTransformer(state));
+}
+
+}
diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_gateway.h b/ydb/library/yql/providers/pq/provider/yql_pq_gateway.h
index 4b27e0aa6c..118b37c012 100644
--- a/ydb/library/yql/providers/pq/provider/yql_pq_gateway.h
+++ b/ydb/library/yql/providers/pq/provider/yql_pq_gateway.h
@@ -17,12 +17,12 @@ struct IPqGateway : public TThrRefBase {
// CM API.
virtual ::NPq::NConfigurationManager::TAsyncDescribePathResult DescribePath(const TString& sessionId, const TString& cluster, const TString& database, const TString& path, const TString& token) = 0;
-
- virtual void UpdateClusterConfigs(
- const TString& clusterName,
- const TString& endpoint,
- const TString& database,
- bool secure) = 0;
+
+ virtual void UpdateClusterConfigs(
+ const TString& clusterName,
+ const TString& endpoint,
+ const TString& database,
+ bool secure) = 0;
};
} // namespace NYql
diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_helpers.cpp b/ydb/library/yql/providers/pq/provider/yql_pq_helpers.cpp
index e3a780aa79..47a2134147 100644
--- a/ydb/library/yql/providers/pq/provider/yql_pq_helpers.cpp
+++ b/ydb/library/yql/providers/pq/provider/yql_pq_helpers.cpp
@@ -1,6 +1,6 @@
#include "yql_pq_helpers.h"
-#include "yql_pq_provider_impl.h"
+#include "yql_pq_provider_impl.h"
#include <ydb/library/yql/core/yql_expr_optimize.h>
#include <ydb/library/yql/utils/log/log.h>
#include <ydb/library/yql/providers/pq/common/yql_names.h>
@@ -26,80 +26,80 @@ TCoNameValueTupleList BuildTopicPropsList(const TPqState::TTopicMeta& meta, TPos
.Done();
}
-void FindYdsDbIdsForResolving(
- const TPqState::TPtr& state,
- TExprNode::TPtr input,
- THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TDatabaseAuth>& ids)
-{
- if (auto pqNodes = FindNodes(input, [&](const TExprNode::TPtr& node) {
- if (auto maybePqRead = TMaybeNode<TPqRead>(node)) {
- TPqRead read = maybePqRead.Cast();
- if (read.DataSource().Category().Value() == PqProviderName) {
- return true;
- }
- } else if (auto maybePqWrite = TMaybeNode<TPqWrite>(node)) {
- TPqWrite write = maybePqWrite.Cast();
- if (write.DataSink().Category().Value() == PqProviderName) {
- return true;
- }
- }
- return false;
- }); !pqNodes.empty()) {
- TString cluster;
- for (auto& node : pqNodes) {
- if (auto maybePqRead = TMaybeNode<TPqRead>(node)) {
- TPqRead read = maybePqRead.Cast();
- cluster = read.DataSource().Cluster().StringValue();
- } else if (auto maybePqWrite = TMaybeNode<TPqWrite>(node)) {
- TPqWrite write = maybePqWrite.Cast();
- cluster = write.DataSink().Cluster().StringValue();
- } else {
- Y_FAIL("Unrecognized pq node");
- }
- YQL_CLOG(INFO, ProviderPq) << "Found cluster: " << cluster;
- const auto& clusterCfgSettings = state->Configuration->ClustersConfigurationSettings;
- const auto foundSetting = clusterCfgSettings.find(cluster);
- if (foundSetting == clusterCfgSettings.end()
- || foundSetting->second.ClusterType != NYql::TPqClusterConfig::CT_DATA_STREAMS
- || foundSetting->second.Endpoint)
- continue;
- YQL_CLOG(INFO, ProviderPq) << "Found dbId: " << foundSetting->second.DatabaseId;
- if (!foundSetting->second.DatabaseId)
- continue;
- YQL_CLOG(INFO, ProviderPq) << "Resolve YDS id: " << foundSetting->second.DatabaseId;
- const auto idKey = std::make_pair(foundSetting->second.DatabaseId, NYq::DatabaseType::DataStreams);
- const auto foundDbId = state->DatabaseIds.find(idKey);
- if (foundDbId != state->DatabaseIds.end()) {
- ids[idKey] = foundDbId->second;
- }
- }
- }
- YQL_CLOG(INFO, ProviderPq) << "Ids to resolve: " << ids.size();
-}
-
-void FillSettingsWithResolvedYdsIds(
- const TPqState::TPtr& state,
- const THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TEvEndpointResponse::TEndpoint>& fullResolvedIds)
-{
- YQL_CLOG(INFO, ProviderPq) << "FullResolvedIds size: " << fullResolvedIds.size();
- auto& clusters = state->Configuration->ClustersConfigurationSettings;
- const auto& id2Clusters = state->Configuration->DbId2Clusters;
- for (const auto& [dbIdWithType, info] : fullResolvedIds) {
- const auto& dbId = dbIdWithType.first;
- YQL_CLOG(INFO, ProviderPq) << "DbId = " << dbId;
- const auto iter = id2Clusters.find(dbId);
- if (iter == id2Clusters.end()) {
- continue;
- }
- for (const auto& clusterName : iter->second) {
- auto& setting = clusters[clusterName];
- setting.Endpoint = info.Endpoint;
- setting.Database = info.Database;
- setting.UseSsl = info.Secure;
- state->Gateway->UpdateClusterConfigs(clusterName, info.Endpoint, info.Database, info.Secure);
- }
- }
-}
-
-
+void FindYdsDbIdsForResolving(
+ const TPqState::TPtr& state,
+ TExprNode::TPtr input,
+ THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TDatabaseAuth>& ids)
+{
+ if (auto pqNodes = FindNodes(input, [&](const TExprNode::TPtr& node) {
+ if (auto maybePqRead = TMaybeNode<TPqRead>(node)) {
+ TPqRead read = maybePqRead.Cast();
+ if (read.DataSource().Category().Value() == PqProviderName) {
+ return true;
+ }
+ } else if (auto maybePqWrite = TMaybeNode<TPqWrite>(node)) {
+ TPqWrite write = maybePqWrite.Cast();
+ if (write.DataSink().Category().Value() == PqProviderName) {
+ return true;
+ }
+ }
+ return false;
+ }); !pqNodes.empty()) {
+ TString cluster;
+ for (auto& node : pqNodes) {
+ if (auto maybePqRead = TMaybeNode<TPqRead>(node)) {
+ TPqRead read = maybePqRead.Cast();
+ cluster = read.DataSource().Cluster().StringValue();
+ } else if (auto maybePqWrite = TMaybeNode<TPqWrite>(node)) {
+ TPqWrite write = maybePqWrite.Cast();
+ cluster = write.DataSink().Cluster().StringValue();
+ } else {
+ Y_FAIL("Unrecognized pq node");
+ }
+ YQL_CLOG(INFO, ProviderPq) << "Found cluster: " << cluster;
+ const auto& clusterCfgSettings = state->Configuration->ClustersConfigurationSettings;
+ const auto foundSetting = clusterCfgSettings.find(cluster);
+ if (foundSetting == clusterCfgSettings.end()
+ || foundSetting->second.ClusterType != NYql::TPqClusterConfig::CT_DATA_STREAMS
+ || foundSetting->second.Endpoint)
+ continue;
+ YQL_CLOG(INFO, ProviderPq) << "Found dbId: " << foundSetting->second.DatabaseId;
+ if (!foundSetting->second.DatabaseId)
+ continue;
+ YQL_CLOG(INFO, ProviderPq) << "Resolve YDS id: " << foundSetting->second.DatabaseId;
+ const auto idKey = std::make_pair(foundSetting->second.DatabaseId, NYq::DatabaseType::DataStreams);
+ const auto foundDbId = state->DatabaseIds.find(idKey);
+ if (foundDbId != state->DatabaseIds.end()) {
+ ids[idKey] = foundDbId->second;
+ }
+ }
+ }
+ YQL_CLOG(INFO, ProviderPq) << "Ids to resolve: " << ids.size();
+}
+
+void FillSettingsWithResolvedYdsIds(
+ const TPqState::TPtr& state,
+ const THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TEvEndpointResponse::TEndpoint>& fullResolvedIds)
+{
+ YQL_CLOG(INFO, ProviderPq) << "FullResolvedIds size: " << fullResolvedIds.size();
+ auto& clusters = state->Configuration->ClustersConfigurationSettings;
+ const auto& id2Clusters = state->Configuration->DbId2Clusters;
+ for (const auto& [dbIdWithType, info] : fullResolvedIds) {
+ const auto& dbId = dbIdWithType.first;
+ YQL_CLOG(INFO, ProviderPq) << "DbId = " << dbId;
+ const auto iter = id2Clusters.find(dbId);
+ if (iter == id2Clusters.end()) {
+ continue;
+ }
+ for (const auto& clusterName : iter->second) {
+ auto& setting = clusters[clusterName];
+ setting.Endpoint = info.Endpoint;
+ setting.Database = info.Database;
+ setting.UseSsl = info.Secure;
+ state->Gateway->UpdateClusterConfigs(clusterName, info.Endpoint, info.Database, info.Secure);
+ }
+ }
+}
+
+
} // namespace NYql
diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_helpers.h b/ydb/library/yql/providers/pq/provider/yql_pq_helpers.h
index 181a25cce8..628ab2f65d 100644
--- a/ydb/library/yql/providers/pq/provider/yql_pq_helpers.h
+++ b/ydb/library/yql/providers/pq/provider/yql_pq_helpers.h
@@ -1,6 +1,6 @@
#pragma once
-#include "yql_pq_provider_impl.h"
+#include "yql_pq_provider_impl.h"
#include <ydb/library/yql/ast/yql_expr.h>
#include <ydb/library/yql/ast/yql_pos_handle.h>
#include <ydb/library/yql/core/expr_nodes/yql_expr_nodes.h>
@@ -12,13 +12,13 @@ void Add(TVector<NNodes::TCoNameValueTuple>& settings, TStringBuf name, TStringB
NNodes::TCoNameValueTupleList BuildTopicPropsList(const TPqState::TTopicMeta& meta, TPositionHandle pos, TExprContext& ctx);
-void FindYdsDbIdsForResolving(
- const TPqState::TPtr& state,
- TExprNode::TPtr input,
- THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TDatabaseAuth>& ids);
-
-void FillSettingsWithResolvedYdsIds(
- const TPqState::TPtr& state,
- const THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TEvEndpointResponse::TEndpoint>& fullResolvedIds);
-
+void FindYdsDbIdsForResolving(
+ const TPqState::TPtr& state,
+ TExprNode::TPtr input,
+ THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TDatabaseAuth>& ids);
+
+void FillSettingsWithResolvedYdsIds(
+ const TPqState::TPtr& state,
+ const THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TEvEndpointResponse::TEndpoint>& fullResolvedIds);
+
} // namespace NYql
diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_io_discovery.cpp b/ydb/library/yql/providers/pq/provider/yql_pq_io_discovery.cpp
index d8cbcc896b..f2e347cf73 100644
--- a/ydb/library/yql/providers/pq/provider/yql_pq_io_discovery.cpp
+++ b/ydb/library/yql/providers/pq/provider/yql_pq_io_discovery.cpp
@@ -1,5 +1,5 @@
#include "yql_pq_provider_impl.h"
-#include "yql_pq_helpers.h"
+#include "yql_pq_helpers.h"
#include <ydb/library/yql/core/yql_expr_optimize.h>
#include <ydb/library/yql/utils/log/log.h>
@@ -12,9 +12,9 @@ namespace {
using namespace NNodes;
class TPqIODiscoveryTransformer : public TGraphTransformerBase {
-
-using TDbId2Endpoint = THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TEvEndpointResponse::TEndpoint>;
-
+
+using TDbId2Endpoint = THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TEvEndpointResponse::TEndpoint>;
+
public:
explicit TPqIODiscoveryTransformer(TPqState::TPtr state)
: State_(state)
@@ -24,42 +24,42 @@ public:
TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final {
Y_UNUSED(ctx);
output = input;
- if (ctx.Step.IsDone(TExprStep::DiscoveryIO))
- return TStatus::Ok;
-
- if (!State_->DbResolver)
- return TStatus::Ok;
-
- THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TDatabaseAuth> ids;
- FindYdsDbIdsForResolving(State_, input, ids);
-
- if (ids.empty())
- return TStatus::Ok;
-
- AsyncFuture_ = State_->DbResolver->ResolveIds({ids, State_->DbResolver->GetTraceId()}).Apply([resolvedIds_ = ResolvedIds_](const auto& future) {
- *resolvedIds_ = future.GetValue();
- });
- return TStatus::Async;
+ if (ctx.Step.IsDone(TExprStep::DiscoveryIO))
+ return TStatus::Ok;
+
+ if (!State_->DbResolver)
+ return TStatus::Ok;
+
+ THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TDatabaseAuth> ids;
+ FindYdsDbIdsForResolving(State_, input, ids);
+
+ if (ids.empty())
+ return TStatus::Ok;
+
+ AsyncFuture_ = State_->DbResolver->ResolveIds({ids, State_->DbResolver->GetTraceId()}).Apply([resolvedIds_ = ResolvedIds_](const auto& future) {
+ *resolvedIds_ = future.GetValue();
+ });
+ return TStatus::Async;
}
NThreading::TFuture<void> DoGetAsyncFuture(const TExprNode&) final {
- return AsyncFuture_;
+ return AsyncFuture_;
}
TStatus DoApplyAsyncChanges(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext&) final {
output = input;
- AsyncFuture_.GetValue();
- FullResolvedIds_.insert(ResolvedIds_->begin(), ResolvedIds_->end());
- ResolvedIds_->clear();
- FillSettingsWithResolvedYdsIds(State_, FullResolvedIds_);
+ AsyncFuture_.GetValue();
+ FullResolvedIds_.insert(ResolvedIds_->begin(), ResolvedIds_->end());
+ ResolvedIds_->clear();
+ FillSettingsWithResolvedYdsIds(State_, FullResolvedIds_);
return TStatus::Ok;
}
private:
const TPqState::TPtr State_;
- NThreading::TFuture<void> AsyncFuture_;
- TDbId2Endpoint FullResolvedIds_;
- std::shared_ptr<TDbId2Endpoint> ResolvedIds_ = std::make_shared<TDbId2Endpoint>();
+ NThreading::TFuture<void> AsyncFuture_;
+ TDbId2Endpoint FullResolvedIds_;
+ std::shared_ptr<TDbId2Endpoint> ResolvedIds_ = std::make_shared<TDbId2Endpoint>();
};
}
diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_provider.cpp b/ydb/library/yql/providers/pq/provider/yql_pq_provider.cpp
index 9dc52ed7b8..b3f700c147 100644
--- a/ydb/library/yql/providers/pq/provider/yql_pq_provider.cpp
+++ b/ydb/library/yql/providers/pq/provider/yql_pq_provider.cpp
@@ -9,11 +9,11 @@
namespace NYql {
-TDataProviderInitializer GetPqDataProviderInitializer(
- IPqGateway::TPtr gateway,
- bool supportRtmrMode,
- std::shared_ptr<NYq::TDatabaseAsyncResolverWithMeta> dbResolverWithMeta) {
- return [gateway, supportRtmrMode, dbResolverWithMeta] (
+TDataProviderInitializer GetPqDataProviderInitializer(
+ IPqGateway::TPtr gateway,
+ bool supportRtmrMode,
+ std::shared_ptr<NYq::TDatabaseAsyncResolverWithMeta> dbResolverWithMeta) {
+ return [gateway, supportRtmrMode, dbResolverWithMeta] (
const TString& userName,
const TString& sessionId,
const TGatewaysConfig* gatewaysConfig,
@@ -33,9 +33,9 @@ TDataProviderInitializer GetPqDataProviderInitializer(
state->SupportRtmrMode = supportRtmrMode;
state->Types = typeCtx.Get();
state->FunctionRegistry = functionRegistry;
- state->DbResolver = dbResolverWithMeta;
+ state->DbResolver = dbResolverWithMeta;
if (gatewaysConfig) {
- state->Configuration->Init(gatewaysConfig->GetPq(), typeCtx, dbResolverWithMeta, state->DatabaseIds);
+ state->Configuration->Init(gatewaysConfig->GetPq(), typeCtx, dbResolverWithMeta, state->DatabaseIds);
}
state->Gateway = gateway;
state->DqIntegration = CreatePqDqIntegration(state);
diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_provider.h b/ydb/library/yql/providers/pq/provider/yql_pq_provider.h
index 7aadfd904f..a890e32b69 100644
--- a/ydb/library/yql/providers/pq/provider/yql_pq_provider.h
+++ b/ydb/library/yql/providers/pq/provider/yql_pq_provider.h
@@ -52,14 +52,14 @@ public:
const NKikimr::NMiniKQL::IFunctionRegistry* FunctionRegistry = nullptr;
IPqGateway::TPtr Gateway;
THolder<IDqIntegration> DqIntegration;
- THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TDatabaseAuth> DatabaseIds;
- std::shared_ptr<NYq::TDatabaseAsyncResolverWithMeta> DbResolver;
+ THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TDatabaseAuth> DatabaseIds;
+ std::shared_ptr<NYq::TDatabaseAsyncResolverWithMeta> DbResolver;
};
-TDataProviderInitializer GetPqDataProviderInitializer(
- IPqGateway::TPtr gateway,
- bool supportRtmrMode = false,
- std::shared_ptr<NYq::TDatabaseAsyncResolverWithMeta> dbResolverWithMeta = nullptr
-);
+TDataProviderInitializer GetPqDataProviderInitializer(
+ IPqGateway::TPtr gateway,
+ bool supportRtmrMode = false,
+ std::shared_ptr<NYq::TDatabaseAsyncResolverWithMeta> dbResolverWithMeta = nullptr
+);
} // namespace NYql
diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_provider_impl.h b/ydb/library/yql/providers/pq/provider/yql_pq_provider_impl.h
index 50c620f10c..619eb468e2 100644
--- a/ydb/library/yql/providers/pq/provider/yql_pq_provider_impl.h
+++ b/ydb/library/yql/providers/pq/provider/yql_pq_provider_impl.h
@@ -15,7 +15,7 @@ TIntrusivePtr<IDataProvider> CreatePqDataSink(TPqState::TPtr state, IPqGateway::
THolder<IGraphTransformer> CreatePqLoadTopicMetadataTransformer(TPqState::TPtr state);
-THolder<IGraphTransformer> CreatePqDataSinkIODiscoveryTransformer(TPqState::TPtr state);
+THolder<IGraphTransformer> CreatePqDataSinkIODiscoveryTransformer(TPqState::TPtr state);
THolder<TVisitorTransformerBase> CreatePqDataSourceTypeAnnotationTransformer(TPqState::TPtr state);
THolder<TVisitorTransformerBase> CreatePqDataSinkTypeAnnotationTransformer(TPqState::TPtr state);
diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_settings.cpp b/ydb/library/yql/providers/pq/provider/yql_pq_settings.cpp
index e6fe651538..278db980f4 100644
--- a/ydb/library/yql/providers/pq/provider/yql_pq_settings.cpp
+++ b/ydb/library/yql/providers/pq/provider/yql_pq_settings.cpp
@@ -14,59 +14,59 @@ TPqSettings::TConstPtr TPqConfiguration::Snapshot() const {
return std::make_shared<const TPqSettings>(*this);
}
-void TPqConfiguration::Init(
- const TPqGatewayConfig& config,
- TIntrusivePtr<TTypeAnnotationContext> typeCtx,
- const std::shared_ptr<NYq::TDatabaseAsyncResolverWithMeta> dbResolver,
- THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TDatabaseAuth>& databaseIds)
-{
- TVector<TString> clusters(Reserve(config.ClusterMappingSize()));
- for (auto& cluster: config.GetClusterMapping()) {
- clusters.push_back(cluster.GetName());
- }
-
- SetValidClusters(clusters);
-
- Dispatch(config.GetDefaultSettings());
-
- for (auto& cluster: config.GetClusterMapping()) {
- Dispatch(cluster.GetName(), cluster.GetSettings());
- TPqClusterConfigurationSettings& clusterSettings = ClustersConfigurationSettings[cluster.GetName()];
-
- clusterSettings.ClusterName = cluster.GetName();
- clusterSettings.ClusterType = cluster.GetClusterType();
- clusterSettings.Endpoint = cluster.GetEndpoint();
- clusterSettings.ConfigManagerEndpoint = cluster.GetConfigManagerEndpoint();
- clusterSettings.Database = cluster.GetDatabase();
- clusterSettings.DatabaseId = cluster.GetDatabaseId();
- clusterSettings.TvmId = cluster.GetTvmId();
- clusterSettings.UseSsl = cluster.GetUseSsl();
- clusterSettings.AddBearerToToken = cluster.GetAddBearerToToken();
-
- if (dbResolver) {
- YQL_CLOG(DEBUG, ProviderPq) << "Settings: clusterName = " << cluster.GetName()
- << ", clusterDbId = " << cluster.GetDatabaseId() << ", cluster.GetEndpoint(): " << cluster.GetEndpoint() << ", HasEndpoint = " << (cluster.HasEndpoint() ? "TRUE" : "FALSE") ;
- dbResolver->TryAddDbIdToResolve(cluster.HasEndpoint(), cluster.GetName(), cluster.GetDatabaseId(), NYq::DatabaseType::DataStreams, databaseIds);
- if (cluster.GetDatabaseId()) {
- DbId2Clusters[cluster.GetDatabaseId()].emplace_back(cluster.GetName());
- YQL_CLOG(DEBUG, ProviderPq) << "Add dbId: " << cluster.GetDatabaseId() << " to DbId2Clusters";
- }
- }
-
- const TString authToken = typeCtx->FindCredentialContent("cluster:default_" + clusterSettings.ClusterName, "default_pq", cluster.GetToken());
- clusterSettings.AuthToken = authToken;
- Tokens[clusterSettings.ClusterName] = ComposeStructuredTokenJsonForServiceAccount(cluster.GetServiceAccountId(), cluster.GetServiceAccountIdSignature(), authToken);
- }
- FreezeDefaults();
-}
-
-TString TPqConfiguration::GetDatabaseForTopic(const TString& cluster) const {
- if (TMaybe<TString> explicitDb = Database.Get()) {
- return *explicitDb;
- }
- const auto clusterSetting = ClustersConfigurationSettings.FindPtr(cluster);
- YQL_ENSURE(clusterSetting, "Unknown cluster " << cluster);
- return clusterSetting->Database;
-}
-
+void TPqConfiguration::Init(
+ const TPqGatewayConfig& config,
+ TIntrusivePtr<TTypeAnnotationContext> typeCtx,
+ const std::shared_ptr<NYq::TDatabaseAsyncResolverWithMeta> dbResolver,
+ THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TDatabaseAuth>& databaseIds)
+{
+ TVector<TString> clusters(Reserve(config.ClusterMappingSize()));
+ for (auto& cluster: config.GetClusterMapping()) {
+ clusters.push_back(cluster.GetName());
+ }
+
+ SetValidClusters(clusters);
+
+ Dispatch(config.GetDefaultSettings());
+
+ for (auto& cluster: config.GetClusterMapping()) {
+ Dispatch(cluster.GetName(), cluster.GetSettings());
+ TPqClusterConfigurationSettings& clusterSettings = ClustersConfigurationSettings[cluster.GetName()];
+
+ clusterSettings.ClusterName = cluster.GetName();
+ clusterSettings.ClusterType = cluster.GetClusterType();
+ clusterSettings.Endpoint = cluster.GetEndpoint();
+ clusterSettings.ConfigManagerEndpoint = cluster.GetConfigManagerEndpoint();
+ clusterSettings.Database = cluster.GetDatabase();
+ clusterSettings.DatabaseId = cluster.GetDatabaseId();
+ clusterSettings.TvmId = cluster.GetTvmId();
+ clusterSettings.UseSsl = cluster.GetUseSsl();
+ clusterSettings.AddBearerToToken = cluster.GetAddBearerToToken();
+
+ if (dbResolver) {
+ YQL_CLOG(DEBUG, ProviderPq) << "Settings: clusterName = " << cluster.GetName()
+ << ", clusterDbId = " << cluster.GetDatabaseId() << ", cluster.GetEndpoint(): " << cluster.GetEndpoint() << ", HasEndpoint = " << (cluster.HasEndpoint() ? "TRUE" : "FALSE") ;
+ dbResolver->TryAddDbIdToResolve(cluster.HasEndpoint(), cluster.GetName(), cluster.GetDatabaseId(), NYq::DatabaseType::DataStreams, databaseIds);
+ if (cluster.GetDatabaseId()) {
+ DbId2Clusters[cluster.GetDatabaseId()].emplace_back(cluster.GetName());
+ YQL_CLOG(DEBUG, ProviderPq) << "Add dbId: " << cluster.GetDatabaseId() << " to DbId2Clusters";
+ }
+ }
+
+ const TString authToken = typeCtx->FindCredentialContent("cluster:default_" + clusterSettings.ClusterName, "default_pq", cluster.GetToken());
+ clusterSettings.AuthToken = authToken;
+ Tokens[clusterSettings.ClusterName] = ComposeStructuredTokenJsonForServiceAccount(cluster.GetServiceAccountId(), cluster.GetServiceAccountIdSignature(), authToken);
+ }
+ FreezeDefaults();
+}
+
+TString TPqConfiguration::GetDatabaseForTopic(const TString& cluster) const {
+ if (TMaybe<TString> explicitDb = Database.Get()) {
+ return *explicitDb;
+ }
+ const auto clusterSetting = ClustersConfigurationSettings.FindPtr(cluster);
+ YQL_ENSURE(clusterSetting, "Unknown cluster " << cluster);
+ return clusterSetting->Database;
+}
+
} // NYql
diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_settings.h b/ydb/library/yql/providers/pq/provider/yql_pq_settings.h
index 4bfd0abfaa..5ed40d07d9 100644
--- a/ydb/library/yql/providers/pq/provider/yql_pq_settings.h
+++ b/ydb/library/yql/providers/pq/provider/yql_pq_settings.h
@@ -9,7 +9,7 @@
#include <ydb/core/yq/libs/events/events.h>
#include <ydb/core/yq/libs/db_resolver/db_async_resolver_with_meta.h>
#include <ydb/core/yq/libs/common/database_token_builder.h>
-
+
namespace NYql {
struct TPqSettings {
@@ -39,18 +39,18 @@ struct TPqConfiguration : public TPqSettings, public NCommon::TSettingDispatcher
TPqConfiguration();
TPqConfiguration(const TPqConfiguration&) = delete;
- void Init(
- const TPqGatewayConfig& config,
- TIntrusivePtr<TTypeAnnotationContext> typeCtx,
- const std::shared_ptr<NYq::TDatabaseAsyncResolverWithMeta> dbResolver,
- THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TDatabaseAuth>& databaseIds);
+ void Init(
+ const TPqGatewayConfig& config,
+ TIntrusivePtr<TTypeAnnotationContext> typeCtx,
+ const std::shared_ptr<NYq::TDatabaseAsyncResolverWithMeta> dbResolver,
+ THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TDatabaseAuth>& databaseIds);
- TString GetDatabaseForTopic(const TString& cluster) const;
+ TString GetDatabaseForTopic(const TString& cluster) const;
TPqSettings::TConstPtr Snapshot() const;
THashMap<TString, TPqClusterConfigurationSettings> ClustersConfigurationSettings;
THashMap<TString, TString> Tokens;
- THashMap<TString, TVector<TString>> DbId2Clusters; // DatabaseId -> ClusterNames
+ THashMap<TString, TVector<TString>> DbId2Clusters; // DatabaseId -> ClusterNames
};
} // NYql
diff --git a/ydb/library/yql/providers/s3/actors/yql_s3_read_actor.cpp b/ydb/library/yql/providers/s3/actors/yql_s3_read_actor.cpp
index d1ec6141d6..eb5410a24e 100644
--- a/ydb/library/yql/providers/s3/actors/yql_s3_read_actor.cpp
+++ b/ydb/library/yql/providers/s3/actors/yql_s3_read_actor.cpp
@@ -25,7 +25,7 @@ struct TEvPrivate {
EvReadResult = EvBegin,
EvReadError,
- EvRetry,
+ EvRetry,
EvEnd
};
@@ -39,51 +39,51 @@ struct TEvPrivate {
};
struct TEvReadError : public TEventLocal<TEvReadError, EvReadError> {
- TEvReadError(TIssues&& error, size_t pathInd): Error(std::move(error)), PathIndex(pathInd) {}
+ TEvReadError(TIssues&& error, size_t pathInd): Error(std::move(error)), PathIndex(pathInd) {}
TIssues Error;
- size_t PathIndex = 0;
+ size_t PathIndex = 0;
+ };
+
+ struct TEvRetryEvent : public NActors::TEventLocal<TEvRetryEvent, EvRetry> {
+ explicit TEvRetryEvent(size_t pathIndex) : PathIndex(pathIndex) {}
+ size_t PathIndex = 0;
};
-
- struct TEvRetryEvent : public NActors::TEventLocal<TEvRetryEvent, EvRetry> {
- explicit TEvRetryEvent(size_t pathIndex) : PathIndex(pathIndex) {}
- size_t PathIndex = 0;
- };
};
} // namespace
class TS3ReadActor : public TActorBootstrapped<TS3ReadActor>, public IDqSourceActor {
-private:
- class TRetryParams {
- public:
- TRetryParams(const std::shared_ptr<NS3::TRetryConfig>& retryConfig) {
- if (retryConfig) {
- DelayMs = retryConfig->GetInitialDelayMs() ? TDuration::MilliSeconds(retryConfig->GetInitialDelayMs()) : DelayMs;
- Epsilon = retryConfig->GetEpsilon() ? retryConfig->GetEpsilon() : Epsilon;
- Y_VERIFY(0.0 < Epsilon && Epsilon < 1.0);
- }
- }
-
- TDuration GetNextDelay(ui32 maxRetries) {
- if (Retries > maxRetries) return TDuration::Zero();
- return DelayMs = GenerateNextDelay();
- };
-
- void IncRetries() {
- ++Retries;
- }
-
- private:
- TDuration GenerateNextDelay() {
- double low = 1 - Epsilon;
- auto jitter = low + std::rand() / (RAND_MAX / (2 * Epsilon));
- return DelayMs * jitter;
- }
-
- ui32 Retries = 0;
- TDuration DelayMs = TDuration::MilliSeconds(100);
- double Epsilon = 0.1;
- };
+private:
+ class TRetryParams {
+ public:
+ TRetryParams(const std::shared_ptr<NS3::TRetryConfig>& retryConfig) {
+ if (retryConfig) {
+ DelayMs = retryConfig->GetInitialDelayMs() ? TDuration::MilliSeconds(retryConfig->GetInitialDelayMs()) : DelayMs;
+ Epsilon = retryConfig->GetEpsilon() ? retryConfig->GetEpsilon() : Epsilon;
+ Y_VERIFY(0.0 < Epsilon && Epsilon < 1.0);
+ }
+ }
+
+ TDuration GetNextDelay(ui32 maxRetries) {
+ if (Retries > maxRetries) return TDuration::Zero();
+ return DelayMs = GenerateNextDelay();
+ };
+
+ void IncRetries() {
+ ++Retries;
+ }
+
+ private:
+ TDuration GenerateNextDelay() {
+ double low = 1 - Epsilon;
+ auto jitter = low + std::rand() / (RAND_MAX / (2 * Epsilon));
+ return DelayMs * jitter;
+ }
+
+ ui32 Retries = 0;
+ TDuration DelayMs = TDuration::MilliSeconds(100);
+ double Epsilon = 0.1;
+ };
public:
using TPath = std::tuple<TString, size_t>;
using TPathList = std::vector<TPath>;
@@ -93,8 +93,8 @@ using TPathList = std::vector<TPath>;
const TString& url,
const TString& token,
TPathList&& paths,
- ICallbacks* callbacks,
- const std::shared_ptr<NS3::TRetryConfig>& retryConfig
+ ICallbacks* callbacks,
+ const std::shared_ptr<NS3::TRetryConfig>& retryConfig
) : Gateway(std::move(gateway))
, InputIndex(inputIndex)
, Callbacks(callbacks)
@@ -102,26 +102,26 @@ using TPathList = std::vector<TPath>;
, Url(url)
, Headers(MakeHeader(token))
, Paths(std::move(paths))
- , RetryConfig(retryConfig)
- {
- if (RetryConfig) {
- MaxRetriesPerPath = RetryConfig->GetMaxRetriesPerPath() ? RetryConfig->GetMaxRetriesPerPath() : MaxRetriesPerPath;
- }
- }
+ , RetryConfig(retryConfig)
+ {
+ if (RetryConfig) {
+ MaxRetriesPerPath = RetryConfig->GetMaxRetriesPerPath() ? RetryConfig->GetMaxRetriesPerPath() : MaxRetriesPerPath;
+ }
+ }
void Bootstrap() {
Become(&TS3ReadActor::StateFunc);
- RetriesPerPath.resize(Paths.size(), TRetryParams(RetryConfig));
- for (size_t pathInd = 0; pathInd < Paths.size(); ++pathInd) {
- const TPath& path = Paths[pathInd];
- Gateway->Download(Url + std::get<TString>(path),
- Headers, std::get<size_t>(path),
- std::bind(&TS3ReadActor::OnDownloadFinished, ActorSystem, SelfId(), std::placeholders::_1, pathInd));
- };
+ RetriesPerPath.resize(Paths.size(), TRetryParams(RetryConfig));
+ for (size_t pathInd = 0; pathInd < Paths.size(); ++pathInd) {
+ const TPath& path = Paths[pathInd];
+ Gateway->Download(Url + std::get<TString>(path),
+ Headers, std::get<size_t>(path),
+ std::bind(&TS3ReadActor::OnDownloadFinished, ActorSystem, SelfId(), std::placeholders::_1, pathInd));
+ };
}
-
- static constexpr char ActorName[] = "S3_READ_ACTOR";
-
+
+ static constexpr char ActorName[] = "S3_READ_ACTOR";
+
private:
void SaveState(const NDqProto::TCheckpoint&, NDqProto::TSourceState&) final {}
void LoadState(const NDqProto::TSourceState&) final {}
@@ -131,16 +131,16 @@ private:
STRICT_STFUNC(StateFunc,
hFunc(TEvPrivate::TEvReadResult, Handle);
hFunc(TEvPrivate::TEvReadError, Handle);
- hFunc(TEvPrivate::TEvRetryEvent, HandleRetry);
+ hFunc(TEvPrivate::TEvRetryEvent, HandleRetry);
)
- static void OnDownloadFinished(TActorSystem* actorSystem, TActorId selfId, IHTTPGateway::TResult&& result, size_t pathInd) {
+ static void OnDownloadFinished(TActorSystem* actorSystem, TActorId selfId, IHTTPGateway::TResult&& result, size_t pathInd) {
switch (result.index()) {
case 0U:
actorSystem->Send(new IEventHandle(selfId, TActorId(), new TEvPrivate::TEvReadResult(std::get<IHTTPGateway::TContent>(std::move(result)))));
return;
case 1U:
- actorSystem->Send(new IEventHandle(selfId, TActorId(), new TEvPrivate::TEvReadError(std::get<TIssues>(std::move(result)), pathInd)));
+ actorSystem->Send(new IEventHandle(selfId, TActorId(), new TEvPrivate::TEvReadError(std::get<TIssues>(std::move(result)), pathInd)));
return;
default:
break;
@@ -174,21 +174,21 @@ private:
Callbacks->OnNewSourceDataArrived(InputIndex);
}
- void HandleRetry(TEvPrivate::TEvRetryEvent::TPtr& ev) {
- const auto pathInd = ev->Get()->PathIndex;
- RetriesPerPath[pathInd].IncRetries();
- Gateway->Download(Url + std::get<TString>(Paths[pathInd]),
- Headers, std::get<size_t>(Paths[pathInd]),
- std::bind(&TS3ReadActor::OnDownloadFinished, ActorSystem, SelfId(), std::placeholders::_1, pathInd));
- }
-
+ void HandleRetry(TEvPrivate::TEvRetryEvent::TPtr& ev) {
+ const auto pathInd = ev->Get()->PathIndex;
+ RetriesPerPath[pathInd].IncRetries();
+ Gateway->Download(Url + std::get<TString>(Paths[pathInd]),
+ Headers, std::get<size_t>(Paths[pathInd]),
+ std::bind(&TS3ReadActor::OnDownloadFinished, ActorSystem, SelfId(), std::placeholders::_1, pathInd));
+ }
+
void Handle(TEvPrivate::TEvReadError::TPtr& result) {
- const auto pathInd = result->Get()->PathIndex;
- Y_VERIFY(pathInd < RetriesPerPath.size());
- if (auto nextDelayMs = RetriesPerPath[pathInd].GetNextDelay(MaxRetriesPerPath)) {
- Schedule(nextDelayMs, new TEvPrivate::TEvRetryEvent(pathInd));
- return;
- }
+ const auto pathInd = result->Get()->PathIndex;
+ Y_VERIFY(pathInd < RetriesPerPath.size());
+ if (auto nextDelayMs = RetriesPerPath[pathInd].GetNextDelay(MaxRetriesPerPath)) {
+ Schedule(nextDelayMs, new TEvPrivate::TEvRetryEvent(pathInd));
+ return;
+ }
++IsDoneCounter;
Callbacks->OnSourceError(InputIndex, result->Get()->Error, true);
}
@@ -202,7 +202,7 @@ private:
return token.empty() ? IHTTPGateway::THeaders() : IHTTPGateway::THeaders{TString("X-YaCloud-SubjectToken:") += token};
}
-private:
+private:
size_t IsDoneCounter = 0U;
const IHTTPGateway::TPtr Gateway;
@@ -217,10 +217,10 @@ private:
const TPathList Paths;
std::queue<IHTTPGateway::TContent> Blocks;
-
- std::vector<TRetryParams> RetriesPerPath;
- const std::shared_ptr<NS3::TRetryConfig> RetryConfig;
- ui32 MaxRetriesPerPath = 3;
+
+ std::vector<TRetryParams> RetriesPerPath;
+ const std::shared_ptr<NS3::TRetryConfig> RetryConfig;
+ ui32 MaxRetriesPerPath = 3;
};
std::pair<NYql::NDq::IDqSourceActor*, IActor*> CreateS3ReadActor(
@@ -230,8 +230,8 @@ std::pair<NYql::NDq::IDqSourceActor*, IActor*> CreateS3ReadActor(
const THashMap<TString, TString>& secureParams,
const THashMap<TString, TString>& taskParams,
NYql::NDq::IDqSourceActor::ICallbacks* callback,
- ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory,
- const std::shared_ptr<NS3::TRetryConfig>& retryConfig)
+ ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory,
+ const std::shared_ptr<NS3::TRetryConfig>& retryConfig)
{
std::unordered_map<TString, size_t> map(params.GetPath().size());
for (auto i = 0; i < params.GetPath().size(); ++i)
@@ -254,7 +254,7 @@ std::pair<NYql::NDq::IDqSourceActor*, IActor*> CreateS3ReadActor(
const auto token = secureParams.Value(params.GetToken(), TString{});
const auto credentialsProviderFactory = CreateCredentialsProviderFactoryForStructuredToken(credentialsFactory, token);
const auto authToken = credentialsProviderFactory->CreateProvider()->GetAuthInfo();
- const auto actor = new TS3ReadActor(inputIndex, std::move(gateway), params.GetUrl(), authToken, std::move(paths), callback, retryConfig);
+ const auto actor = new TS3ReadActor(inputIndex, std::move(gateway), params.GetUrl(), authToken, std::move(paths), callback, retryConfig);
return {actor, actor};
}
diff --git a/ydb/library/yql/providers/s3/actors/yql_s3_read_actor.h b/ydb/library/yql/providers/s3/actors/yql_s3_read_actor.h
index e72f729486..9280e9ba2e 100644
--- a/ydb/library/yql/providers/s3/actors/yql_s3_read_actor.h
+++ b/ydb/library/yql/providers/s3/actors/yql_s3_read_actor.h
@@ -16,7 +16,7 @@ std::pair<NYql::NDq::IDqSourceActor*, NActors::IActor*> CreateS3ReadActor(
const THashMap<TString, TString>& secureParams,
const THashMap<TString, TString>& taskParams,
IDqSourceActor::ICallbacks* callback,
- ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory,
- const std::shared_ptr<NYql::NS3::TRetryConfig>& retryConfig = nullptr);
+ ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory,
+ const std::shared_ptr<NYql::NS3::TRetryConfig>& retryConfig = nullptr);
} // namespace NYql::NDq
diff --git a/ydb/library/yql/providers/s3/actors/yql_s3_source_factory.cpp b/ydb/library/yql/providers/s3/actors/yql_s3_source_factory.cpp
index 4034574a44..fb78240498 100644
--- a/ydb/library/yql/providers/s3/actors/yql_s3_source_factory.cpp
+++ b/ydb/library/yql/providers/s3/actors/yql_s3_source_factory.cpp
@@ -5,14 +5,14 @@
namespace NYql::NDq {
-void RegisterS3ReadActorFactory(
- TDqSourceFactory& factory,
- ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory,
- IHTTPGateway::TPtr gateway,
- const std::shared_ptr<NYql::NS3::TRetryConfig>& retryConfig) {
+void RegisterS3ReadActorFactory(
+ TDqSourceFactory& factory,
+ ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory,
+ IHTTPGateway::TPtr gateway,
+ const std::shared_ptr<NYql::NS3::TRetryConfig>& retryConfig) {
factory.Register<NS3::TSource>("S3Source",
- [credentialsFactory, gateway, retryConfig](NS3::TSource&& settings, IDqSourceActorFactory::TArguments&& args) {
- return CreateS3ReadActor(gateway, std::move(settings), args.InputIndex, args.SecureParams, args.TaskParams, args.Callback, credentialsFactory, retryConfig);
+ [credentialsFactory, gateway, retryConfig](NS3::TSource&& settings, IDqSourceActorFactory::TArguments&& args) {
+ return CreateS3ReadActor(gateway, std::move(settings), args.InputIndex, args.SecureParams, args.TaskParams, args.Callback, credentialsFactory, retryConfig);
});
}
diff --git a/ydb/library/yql/providers/s3/actors/yql_s3_source_factory.h b/ydb/library/yql/providers/s3/actors/yql_s3_source_factory.h
index 412377e2de..2df0a312b1 100644
--- a/ydb/library/yql/providers/s3/actors/yql_s3_source_factory.h
+++ b/ydb/library/yql/providers/s3/actors/yql_s3_source_factory.h
@@ -1,7 +1,7 @@
#pragma once
-#include <ydb/library/yql/providers/common/token_accessor/client/factory.h>
-
+#include <ydb/library/yql/providers/common/token_accessor/client/factory.h>
+
#include <ydb/library/yql/dq/actors/compute/dq_compute_actor_io_actors_factory.h>
#include <ydb/library/yql/dq/actors/compute/dq_compute_actor_sources.h>
#include <ydb/library/yql/providers/common/http_gateway/yql_http_gateway.h>
@@ -10,10 +10,10 @@
namespace NYql::NDq {
-void RegisterS3ReadActorFactory(
- TDqSourceFactory& factory,
- ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory,
- IHTTPGateway::TPtr gateway = IHTTPGateway::Make(),
- const std::shared_ptr<NYql::NS3::TRetryConfig>& retryConfig = nullptr);
+void RegisterS3ReadActorFactory(
+ TDqSourceFactory& factory,
+ ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory,
+ IHTTPGateway::TPtr gateway = IHTTPGateway::Make(),
+ const std::shared_ptr<NYql::NS3::TRetryConfig>& retryConfig = nullptr);
}
diff --git a/ydb/library/yql/providers/s3/proto/retry_config.proto b/ydb/library/yql/providers/s3/proto/retry_config.proto
index afec3b0536..3477562bb6 100644
--- a/ydb/library/yql/providers/s3/proto/retry_config.proto
+++ b/ydb/library/yql/providers/s3/proto/retry_config.proto
@@ -1,10 +1,10 @@
-syntax = "proto3";
-option cc_enable_arenas = true;
-
-package NYql.NS3;
-
-message TRetryConfig {
- uint64 InitialDelayMs = 1;
- uint32 MaxRetriesPerPath = 2;
- double Epsilon = 3;
-}
+syntax = "proto3";
+option cc_enable_arenas = true;
+
+package NYql.NS3;
+
+message TRetryConfig {
+ uint64 InitialDelayMs = 1;
+ uint32 MaxRetriesPerPath = 2;
+ double Epsilon = 3;
+}
diff --git a/ydb/library/yql/providers/s3/proto/ya.make b/ydb/library/yql/providers/s3/proto/ya.make
index 993f34c7e9..48215f2ab5 100644
--- a/ydb/library/yql/providers/s3/proto/ya.make
+++ b/ydb/library/yql/providers/s3/proto/ya.make
@@ -7,7 +7,7 @@ PROTO_LIBRARY()
SRCS(
range.proto
- retry_config.proto
+ retry_config.proto
source.proto
)
diff --git a/ydb/library/yql/providers/s3/provider/yql_s3_io_discovery.cpp b/ydb/library/yql/providers/s3/provider/yql_s3_io_discovery.cpp
index 8eafea99b8..3e0384c9d7 100644
--- a/ydb/library/yql/providers/s3/provider/yql_s3_io_discovery.cpp
+++ b/ydb/library/yql/providers/s3/provider/yql_s3_io_discovery.cpp
@@ -31,23 +31,23 @@ std::array<TExprNode::TPtr, 2U> GetSchema(const TExprNode& settings) {
using TItemsMap = std::map<TString, std::size_t>;
using TPendingBuckets = std::unordered_map<std::tuple<TString, TString, TString>, std::tuple<TNodeSet, TItemsMap, TIssues>, THash<std::tuple<TString, TString, TString>>>;
-void OnDiscovery(
- IHTTPGateway::TWeakPtr gateway,
- TPosition pos,
- IHTTPGateway::TResult&& result,
- const TPendingBuckets::key_type& keys,
- TPendingBuckets::mapped_type& output,
- NThreading::TPromise<void> promise,
- std::weak_ptr<TPendingBuckets> pendingBucketsWPtr,
- int promiseInd) {
- auto pendingBuckets = pendingBucketsWPtr.lock(); // keys and output could be used only when TPendingBuckets is alive
- if (!pendingBuckets) {
- return;
- }
- TString logMsg = TStringBuilder() << "promise #" << promiseInd << ": ";
+void OnDiscovery(
+ IHTTPGateway::TWeakPtr gateway,
+ TPosition pos,
+ IHTTPGateway::TResult&& result,
+ const TPendingBuckets::key_type& keys,
+ TPendingBuckets::mapped_type& output,
+ NThreading::TPromise<void> promise,
+ std::weak_ptr<TPendingBuckets> pendingBucketsWPtr,
+ int promiseInd) {
+ auto pendingBuckets = pendingBucketsWPtr.lock(); // keys and output could be used only when TPendingBuckets is alive
+ if (!pendingBuckets) {
+ return;
+ }
+ TString logMsg = TStringBuilder() << "promise #" << promiseInd << ": ";
switch (result.index()) {
case 0U: try {
- logMsg += "Result received";
+ logMsg += "Result received";
const NXml::TDocument xml(std::get<IHTTPGateway::TContent>(std::move(result)).Extract(), NXml::TDocument::String);
if (const auto& root = xml.Root(); root.Name() == "Error") {
const auto& code = root.Node("Code", true).Value<TString>();
@@ -80,30 +80,30 @@ void OnDiscovery(
if (const auto& token = std::get<2U>(keys); !token.empty())
headers.emplace_back(token);
return g->Download(std::get<0U>(keys) + "?list-type=2&prefix=" + prefix + "&continuation-token=" + next + "&max-keys=" + maxKeys, std::move(headers), 0U,
- std::bind(&OnDiscovery, gateway, pos, std::placeholders::_1, std::cref(keys), std::ref(output), std::move(promise), pendingBucketsWPtr, promiseInd));
+ std::bind(&OnDiscovery, gateway, pos, std::placeholders::_1, std::cref(keys), std::ref(output), std::move(promise), pendingBucketsWPtr, promiseInd));
}
- YQL_CLOG(INFO, ProviderS3) << "Gateway disappeared.";
+ YQL_CLOG(INFO, ProviderS3) << "Gateway disappeared.";
}
}
break;
} catch (const std::exception& ex) {
- logMsg += TStringBuilder() << "Exception occurred: " << ex.what();
+ logMsg += TStringBuilder() << "Exception occurred: " << ex.what();
std::get<TIssues>(output) = {TIssue(pos, TStringBuilder() << "Error '" << ex.what() << "' on parse discovery response.")};
break;
}
case 1U:
- logMsg += TStringBuilder() << "Issues occurred: " << std::get<TIssues>(result).ToString();
+ logMsg += TStringBuilder() << "Issues occurred: " << std::get<TIssues>(result).ToString();
std::get<TIssues>(output) = std::get<TIssues>(std::move(result));
break;
default:
- logMsg += TStringBuilder() << "Undefined variant index: " << result.index();
+ logMsg += TStringBuilder() << "Undefined variant index: " << result.index();
std::get<TIssues>(output) = {TIssue(pos, TStringBuilder() << "Unexpected variant index " << result.index() << " on discovery.")};
break;
}
- YQL_CLOG(DEBUG, ProviderS3) << "Set promise with log message: " << logMsg;
- promise.SetValue();
+ YQL_CLOG(DEBUG, ProviderS3) << "Set promise with log message: " << logMsg;
+ promise.SetValue();
}
@@ -187,15 +187,15 @@ public:
const auto credentialsProviderFactory = CreateCredentialsProviderFactoryForStructuredToken(State_->CredentialsFactory, token);
const auto authToken = credentialsProviderFactory->CreateProvider()->GetAuthInfo();
- std::get<TNodeSet>((*PendingBuckets_)[std::make_tuple(connect.Url, TString(prefix), authToken.empty() ? TString() : TString("X-YaCloud-SubjectToken:") += authToken)]).emplace(read.Raw());
+ std::get<TNodeSet>((*PendingBuckets_)[std::make_tuple(connect.Url, TString(prefix), authToken.empty() ? TString() : TString("X-YaCloud-SubjectToken:") += authToken)]).emplace(read.Raw());
}
}
std::vector<NThreading::TFuture<void>> handles;
- handles.reserve(PendingBuckets_->size());
+ handles.reserve(PendingBuckets_->size());
- int i = 0;
- for (auto& bucket : *PendingBuckets_) {
+ int i = 0;
+ for (auto& bucket : *PendingBuckets_) {
auto promise = NThreading::NewPromise();
handles.emplace_back(promise.GetFuture());
TString prefix(std::get<1U>(bucket.first));
@@ -203,10 +203,10 @@ public:
IHTTPGateway::THeaders headers;
if (const auto& token = std::get<2U>(bucket.first); !token.empty())
headers.emplace_back(token);
- std::weak_ptr<TPendingBuckets> pendingBucketsWPtr = PendingBuckets_;
+ std::weak_ptr<TPendingBuckets> pendingBucketsWPtr = PendingBuckets_;
Gateway_->Download(std::get<0U>(bucket.first) + "?list-type=2&prefix=" + prefix, headers, 0U, std::bind(&OnDiscovery,
- IHTTPGateway::TWeakPtr(Gateway_), ctx.GetPosition((*std::get<TNodeSet>(bucket.second).cbegin())->Pos()), std::placeholders::_1,
- std::cref(bucket.first), std::ref(bucket.second), std::move(promise), pendingBucketsWPtr, i++));
+ IHTTPGateway::TWeakPtr(Gateway_), ctx.GetPosition((*std::get<TNodeSet>(bucket.second).cbegin())->Pos()), std::placeholders::_1,
+ std::cref(bucket.first), std::ref(bucket.second), std::move(promise), pendingBucketsWPtr, i++));
YQL_CLOG(INFO, ProviderS3) << "Enumerate items in " << std::get<0U>(bucket.first) << std::get<1U>(bucket.first);
}
@@ -226,8 +226,8 @@ public:
// Raise errors if any
AllFuture_.GetValue();
- TNodeOnNodeOwnedMap replaces(PendingBuckets_->size());
- auto buckets = std::move(*PendingBuckets_);
+ TNodeOnNodeOwnedMap replaces(PendingBuckets_->size());
+ auto buckets = std::move(*PendingBuckets_);
auto count = 0ULL;
auto readSize = 0ULL;
for (auto& bucket : buckets) {
@@ -337,7 +337,7 @@ private:
const TS3State::TPtr State_;
const IHTTPGateway::TPtr Gateway_;
- const std::shared_ptr<TPendingBuckets> PendingBuckets_ = std::make_shared<TPendingBuckets>();
+ const std::shared_ptr<TPendingBuckets> PendingBuckets_ = std::make_shared<TPendingBuckets>();
NThreading::TFuture<void> AllFuture_;
};
diff --git a/ydb/library/yql/providers/solomon/async_io/dq_solomon_write_actor.cpp b/ydb/library/yql/providers/solomon/async_io/dq_solomon_write_actor.cpp
index 7910b8d740..fc07635176 100644
--- a/ydb/library/yql/providers/solomon/async_io/dq_solomon_write_actor.cpp
+++ b/ydb/library/yql/providers/solomon/async_io/dq_solomon_write_actor.cpp
@@ -76,8 +76,8 @@ struct TMetricsInflight {
class TDqSolomonWriteActor : public NActors::TActor<TDqSolomonWriteActor>, public IDqSinkActor {
public:
- static constexpr char ActorName[] = "DQ_SOLOMON_WRITE_ACTOR";
-
+ static constexpr char ActorName[] = "DQ_SOLOMON_WRITE_ACTOR";
+
TDqSolomonWriteActor(
ui64 outputIndex,
TDqSolomonWriteParams&& writeParams,
diff --git a/ydb/library/yql/providers/ydb/actors/yql_ydb_read_actor.cpp b/ydb/library/yql/providers/ydb/actors/yql_ydb_read_actor.cpp
index 5df2e3e535..7ab55cfcfc 100644
--- a/ydb/library/yql/providers/ydb/actors/yql_ydb_read_actor.cpp
+++ b/ydb/library/yql/providers/ydb/actors/yql_ydb_read_actor.cpp
@@ -102,9 +102,9 @@ public:
Become(&TYdbReadActor::StateFunc);
SendRequest();
}
-
- static constexpr char ActorName[] = "YQL_YDB_READ_ACTOR";
-
+
+ static constexpr char ActorName[] = "YQL_YDB_READ_ACTOR";
+
private:
void SaveState(const NDqProto::TCheckpoint&, NDqProto::TSourceState&) final {}
void LoadState(const NDqProto::TSourceState&) final {}
diff --git a/ydb/library/yql/providers/ydb/provider/ya.make b/ydb/library/yql/providers/ydb/provider/ya.make
index 128d752790..b220766ee5 100644
--- a/ydb/library/yql/providers/ydb/provider/ya.make
+++ b/ydb/library/yql/providers/ydb/provider/ya.make
@@ -10,7 +10,7 @@ SRCS(
yql_ydb_datasource_type_ann.cpp
yql_ydb_dq_integration.cpp
yql_ydb_exec.cpp
- yql_ydb_io_discovery.cpp
+ yql_ydb_io_discovery.cpp
yql_ydb_load_meta.cpp
yql_ydb_logical_opt.cpp
yql_ydb_physical_opt.cpp
@@ -21,8 +21,8 @@ SRCS(
)
PEERDIR(
- library/cpp/random_provider
- library/cpp/time_provider
+ library/cpp/random_provider
+ library/cpp/time_provider
library/cpp/yson/node
ydb/core/yq/libs/common
ydb/core/yq/libs/db_resolver
diff --git a/ydb/library/yql/providers/ydb/provider/yql_ydb_datasource.cpp b/ydb/library/yql/providers/ydb/provider/yql_ydb_datasource.cpp
index b1458294aa..3dfe4e856b 100644
--- a/ydb/library/yql/providers/ydb/provider/yql_ydb_datasource.cpp
+++ b/ydb/library/yql/providers/ydb/provider/yql_ydb_datasource.cpp
@@ -17,11 +17,11 @@ namespace {
class TYdbDataSourceProvider : public TDataProviderBase {
public:
- TYdbDataSourceProvider(
- TYdbState::TPtr state,
- NYdb::TDriver driver)
+ TYdbDataSourceProvider(
+ TYdbState::TPtr state,
+ NYdb::TDriver driver)
: State_(state)
- , IODiscoveryTransformer_(CreateYdbIODiscoveryTransformer(State_))
+ , IODiscoveryTransformer_(CreateYdbIODiscoveryTransformer(State_))
, LoadMetaDataTransformer_(CreateYdbLoadTableMetadataTransformer(State_, driver))
, CallableExecutionTransformer_(CreateYdbSourceCallableExecutionTransformer(State_))
, TypeAnnotationTransformer_(CreateYdbDataSourceTypeAnnotationTransformer(State_))
@@ -56,10 +56,10 @@ public:
return TypeAnnotationTransformer_->CanParse(node);
}
- IGraphTransformer& GetIODiscoveryTransformer() override {
- return *IODiscoveryTransformer_;
- }
-
+ IGraphTransformer& GetIODiscoveryTransformer() override {
+ return *IODiscoveryTransformer_;
+ }
+
IGraphTransformer& GetLoadTableMetadataTransformer() override {
return *LoadMetaDataTransformer_;
}
@@ -138,7 +138,7 @@ public:
private:
const TYdbState::TPtr State_;
- const THolder<IGraphTransformer> IODiscoveryTransformer_;
+ const THolder<IGraphTransformer> IODiscoveryTransformer_;
const THolder<IGraphTransformer> LoadMetaDataTransformer_;
const THolder<IGraphTransformer> CallableExecutionTransformer_;
const THolder<TVisitorTransformerBase> TypeAnnotationTransformer_;
@@ -147,9 +147,9 @@ private:
}
-TIntrusivePtr<IDataProvider> CreateYdbDataSource(
- TYdbState::TPtr state,
- NYdb::TDriver driver) {
+TIntrusivePtr<IDataProvider> CreateYdbDataSource(
+ TYdbState::TPtr state,
+ NYdb::TDriver driver) {
return new TYdbDataSourceProvider(state, driver);
}
diff --git a/ydb/library/yql/providers/ydb/provider/yql_ydb_io_discovery.cpp b/ydb/library/yql/providers/ydb/provider/yql_ydb_io_discovery.cpp
index 0d51b6638e..794c725164 100644
--- a/ydb/library/yql/providers/ydb/provider/yql_ydb_io_discovery.cpp
+++ b/ydb/library/yql/providers/ydb/provider/yql_ydb_io_discovery.cpp
@@ -1,107 +1,107 @@
-#include "yql_ydb_provider_impl.h"
+#include "yql_ydb_provider_impl.h"
#include <ydb/core/yq/libs/events/events.h>
-
+
#include <ydb/library/yql/providers/ydb/expr_nodes/yql_ydb_expr_nodes.h>
#include <ydb/library/yql/providers/common/provider/yql_provider_names.h>
#include <ydb/library/yql/core/expr_nodes/yql_expr_nodes.h>
-
+
#include <ydb/library/yql/core/yql_expr_optimize.h>
#include <ydb/library/yql/core/yql_graph_transformer.h>
#include <ydb/library/yql/utils/log/log.h>
#include <ydb/library/yql/public/udf/udf_types.h>
#include <ydb/library/yql/ast/yql_expr.h>
#include <ydb/core/yq/libs/events/events.h>
-
-namespace NYql {
-
-namespace {
-
-using namespace NNodes;
-
-class TYdbIODiscoveryTransformer : public TGraphTransformerBase {
- using TDbId2Endpoint = THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TEvEndpointResponse::TEndpoint>;
-public:
- TYdbIODiscoveryTransformer(TYdbState::TPtr state)
- : State_(std::move(state))
- {}
-
- TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final {
- output = input;
-
- if (ctx.Step.IsDone(TExprStep::DiscoveryIO))
- return TStatus::Ok;
-
- if (!State_->DbResolver)
- return TStatus::Ok;
-
- THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TDatabaseAuth> ids;
- if (auto reads = FindNodes(input, [&](const TExprNode::TPtr& node) {
- const TExprBase nodeExpr(node);
- if (!nodeExpr.Maybe<TYdbRead>())
- return false;
- const TYdbRead read(node);
- if (read.DataSource().Category().Value() != YdbProviderName) {
- return false;
- }
- return true;
- }); !reads.empty()) {
- for (auto& node : reads) {
- const TYdbRead read(node);
- const auto& cluster = read.DataSource().Cluster().StringValue();
- const auto& dbId = State_->Configuration->Clusters[cluster].DatabaseId;
- const auto idKey = std::make_pair(dbId, NYq::DatabaseType::Ydb);
- const auto iter = State_->DatabaseIds.find(idKey);
- if (iter != State_->DatabaseIds.end()) {
- ids[idKey] = iter->second;
- }
- }
- }
- if (ids.empty()) {
- return TStatus::Ok;
- }
- AsyncFuture_ = State_->DbResolver->ResolveIds({ids, State_->DbResolver->GetTraceId()}).Apply([resolvedIds_ = ResolvedIds_](const auto& future) {
- *resolvedIds_ = future.GetValue();
- });
- return TStatus::Async;
- }
-
- NThreading::TFuture<void> DoGetAsyncFuture(const TExprNode&) final {
- return AsyncFuture_;
- }
-
- TStatus DoApplyAsyncChanges(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext&) final {
- output = input;
- AsyncFuture_.GetValue();
- FullResolvedIds_.insert(ResolvedIds_->begin(), ResolvedIds_->end());
- ResolvedIds_->clear();
- auto& clusters = State_->Configuration->Clusters;
- const auto& id2Clusters = State_->Configuration->DbId2Clusters;
- for (const auto& [dbIdWithType, info] : FullResolvedIds_) {
- const auto& dbId = dbIdWithType.first;
- const auto iter = id2Clusters.find(dbId);
- if (iter == id2Clusters.end()) {
- continue;
- }
- for (const auto& clusterName : iter->second) {
- auto& c = clusters[clusterName];
- c.Endpoint = info.Endpoint;
- c.Database = info.Database;
- c.Secure = info.Secure;
- }
- }
- return TStatus::Ok;
- }
-private:
- const TYdbState::TPtr State_;
-
- NThreading::TFuture<void> AsyncFuture_;
- TDbId2Endpoint FullResolvedIds_;
- std::shared_ptr<TDbId2Endpoint> ResolvedIds_ = std::make_shared<TDbId2Endpoint>();
-};
-}
-
-THolder<IGraphTransformer> CreateYdbIODiscoveryTransformer(TYdbState::TPtr state) {
- return THolder(new TYdbIODiscoveryTransformer(std::move(state)));
-}
-
-}
+
+namespace NYql {
+
+namespace {
+
+using namespace NNodes;
+
+class TYdbIODiscoveryTransformer : public TGraphTransformerBase {
+ using TDbId2Endpoint = THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TEvEndpointResponse::TEndpoint>;
+public:
+ TYdbIODiscoveryTransformer(TYdbState::TPtr state)
+ : State_(std::move(state))
+ {}
+
+ TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final {
+ output = input;
+
+ if (ctx.Step.IsDone(TExprStep::DiscoveryIO))
+ return TStatus::Ok;
+
+ if (!State_->DbResolver)
+ return TStatus::Ok;
+
+ THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TDatabaseAuth> ids;
+ if (auto reads = FindNodes(input, [&](const TExprNode::TPtr& node) {
+ const TExprBase nodeExpr(node);
+ if (!nodeExpr.Maybe<TYdbRead>())
+ return false;
+ const TYdbRead read(node);
+ if (read.DataSource().Category().Value() != YdbProviderName) {
+ return false;
+ }
+ return true;
+ }); !reads.empty()) {
+ for (auto& node : reads) {
+ const TYdbRead read(node);
+ const auto& cluster = read.DataSource().Cluster().StringValue();
+ const auto& dbId = State_->Configuration->Clusters[cluster].DatabaseId;
+ const auto idKey = std::make_pair(dbId, NYq::DatabaseType::Ydb);
+ const auto iter = State_->DatabaseIds.find(idKey);
+ if (iter != State_->DatabaseIds.end()) {
+ ids[idKey] = iter->second;
+ }
+ }
+ }
+ if (ids.empty()) {
+ return TStatus::Ok;
+ }
+ AsyncFuture_ = State_->DbResolver->ResolveIds({ids, State_->DbResolver->GetTraceId()}).Apply([resolvedIds_ = ResolvedIds_](const auto& future) {
+ *resolvedIds_ = future.GetValue();
+ });
+ return TStatus::Async;
+ }
+
+ NThreading::TFuture<void> DoGetAsyncFuture(const TExprNode&) final {
+ return AsyncFuture_;
+ }
+
+ TStatus DoApplyAsyncChanges(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext&) final {
+ output = input;
+ AsyncFuture_.GetValue();
+ FullResolvedIds_.insert(ResolvedIds_->begin(), ResolvedIds_->end());
+ ResolvedIds_->clear();
+ auto& clusters = State_->Configuration->Clusters;
+ const auto& id2Clusters = State_->Configuration->DbId2Clusters;
+ for (const auto& [dbIdWithType, info] : FullResolvedIds_) {
+ const auto& dbId = dbIdWithType.first;
+ const auto iter = id2Clusters.find(dbId);
+ if (iter == id2Clusters.end()) {
+ continue;
+ }
+ for (const auto& clusterName : iter->second) {
+ auto& c = clusters[clusterName];
+ c.Endpoint = info.Endpoint;
+ c.Database = info.Database;
+ c.Secure = info.Secure;
+ }
+ }
+ return TStatus::Ok;
+ }
+private:
+ const TYdbState::TPtr State_;
+
+ NThreading::TFuture<void> AsyncFuture_;
+ TDbId2Endpoint FullResolvedIds_;
+ std::shared_ptr<TDbId2Endpoint> ResolvedIds_ = std::make_shared<TDbId2Endpoint>();
+};
+}
+
+THolder<IGraphTransformer> CreateYdbIODiscoveryTransformer(TYdbState::TPtr state) {
+ return THolder(new TYdbIODiscoveryTransformer(std::move(state)));
+}
+
+}
diff --git a/ydb/library/yql/providers/ydb/provider/yql_ydb_load_meta.cpp b/ydb/library/yql/providers/ydb/provider/yql_ydb_load_meta.cpp
index 7b348cb8d8..607b3ba7a8 100644
--- a/ydb/library/yql/providers/ydb/provider/yql_ydb_load_meta.cpp
+++ b/ydb/library/yql/providers/ydb/provider/yql_ydb_load_meta.cpp
@@ -56,10 +56,10 @@ NUdf::TDataTypeId GetYqlTypeId(NYdb::TTypeParser&& type, TExprContext& ctx) {
}
class TYdbLoadTableMetadataTransformer : public TGraphTransformerBase {
-
-using TCluster2ClientPerSnapshotHandle = std::unordered_map<std::string_view, std::pair<TMetaClient, std::optional<TCreateSnapshotHandleResult>>>;
-using TTableKey2DescribeTableResult = std::unordered_map<std::pair<TString, TString>, std::optional<TDescribeTableResult>, THash<std::pair<TString, TString>>>;
-
+
+using TCluster2ClientPerSnapshotHandle = std::unordered_map<std::string_view, std::pair<TMetaClient, std::optional<TCreateSnapshotHandleResult>>>;
+using TTableKey2DescribeTableResult = std::unordered_map<std::pair<TString, TString>, std::optional<TDescribeTableResult>, THash<std::pair<TString, TString>>>;
+
public:
TYdbLoadTableMetadataTransformer(TYdbState::TPtr state, NYdb::TDriver driver)
: Driver_(std::move(driver)), State_(std::move(state))
@@ -72,9 +72,9 @@ public:
return TStatus::Ok;
}
- Y_VERIFY(Clients_, "Clients_ was destroyed");
- Y_VERIFY(PendingTables_, "PendingTables_ was destroyed");
-
+ Y_VERIFY(Clients_, "Clients_ was destroyed");
+ Y_VERIFY(PendingTables_, "PendingTables_ was destroyed");
+
std::unordered_map<TString, std::unordered_set<TString>> tablesFromCluster;
TOptimizeExprSettings settings{ nullptr };
@@ -113,8 +113,8 @@ public:
table = database + '/' + table;
const auto& tableKey = std::make_pair(cluster, table);
if (State_->Tables.cend() == State_->Tables.find(tableKey)) {
- PendingTables_->emplace(tableKey, std::nullopt);
- tablesFromCluster[tableKey.first].emplace(tableKey.second);
+ PendingTables_->emplace(tableKey, std::nullopt);
+ tablesFromCluster[tableKey.first].emplace(tableKey.second);
}
switch (key.GetKeyType()) {
@@ -137,21 +137,21 @@ public:
}
}, ctx, settings);
- if (status == TStatus::Error || PendingTables_->empty()) {
+ if (status == TStatus::Error || PendingTables_->empty()) {
return status;
}
std::vector<NThreading::TFuture<void>> handles;
- handles.reserve(PendingTables_->size() + tablesFromCluster.size());
+ handles.reserve(PendingTables_->size() + tablesFromCluster.size());
for (const auto& item : tablesFromCluster) {
const auto& cluster = item.first;
TString token = State_->Configuration->Tokens.at(cluster);
const auto& config = State_->Configuration->Clusters[cluster];
-
+
std::shared_ptr<NYdb::ICredentialsProviderFactory> credentialsProviderFactory = CreateCredentialsProviderFactoryForStructuredToken(State_->CredentialsFactory, token, config.AddBearerToToken);
- const auto ins = Clients_->emplace(cluster, std::pair<TMetaClient, std::optional<TCreateSnapshotHandleResult>>{{Driver_, NYdb::TCommonClientSettings()
+ const auto ins = Clients_->emplace(cluster, std::pair<TMetaClient, std::optional<TCreateSnapshotHandleResult>>{{Driver_, NYdb::TCommonClientSettings()
.Database(config.Database)
.DiscoveryEndpoint(config.Endpoint)
.EnableSsl(config.Secure)
@@ -160,32 +160,32 @@ public:
YQL_CLOG(INFO, ProviderYdb) << "Take snapshot for: `" << cluster << "` from " << config.Endpoint;
- std::optional<TCreateSnapshotHandleResult>& snapshot = ins.first->second.second;
- std::weak_ptr<TCluster2ClientPerSnapshotHandle> clients = Clients_;
- handles.emplace_back(ins.first->second.first.CreateSnapshotHandle(TVector<TString>(item.second.cbegin(), item.second.cend())).Apply([clients, &snapshot]
+ std::optional<TCreateSnapshotHandleResult>& snapshot = ins.first->second.second;
+ std::weak_ptr<TCluster2ClientPerSnapshotHandle> clients = Clients_;
+ handles.emplace_back(ins.first->second.first.CreateSnapshotHandle(TVector<TString>(item.second.cbegin(), item.second.cend())).Apply([clients, &snapshot]
(const NThreading::TFuture<TCreateSnapshotHandleResult>& future) {
- if (future.HasException()) {
- const_cast<NThreading::TFuture<TCreateSnapshotHandleResult>&>(future).ExtractValue();
- } else if (clients.lock()) {
- snapshot.emplace(const_cast<NThreading::TFuture<TCreateSnapshotHandleResult>&>(future).ExtractValue());
- }
+ if (future.HasException()) {
+ const_cast<NThreading::TFuture<TCreateSnapshotHandleResult>&>(future).ExtractValue();
+ } else if (clients.lock()) {
+ snapshot.emplace(const_cast<NThreading::TFuture<TCreateSnapshotHandleResult>&>(future).ExtractValue());
+ }
}));
}
- for (auto& pair : *PendingTables_) {
- const auto find = Clients_->find(pair.first.first);
+ for (auto& pair : *PendingTables_) {
+ const auto find = Clients_->find(pair.first.first);
const auto& table = pair.first.second;
YQL_CLOG(INFO, ProviderYdb) << "Load table meta for: `" << table << "`";
- std::optional<TDescribeTableResult>& meta = pair.second;
- std::weak_ptr<TTableKey2DescribeTableResult> pendingTables = PendingTables_;
- handles.emplace_back(find->second.first.GetTableDescription(table, true).Apply([pendingTables, &meta]
+ std::optional<TDescribeTableResult>& meta = pair.second;
+ std::weak_ptr<TTableKey2DescribeTableResult> pendingTables = PendingTables_;
+ handles.emplace_back(find->second.first.GetTableDescription(table, true).Apply([pendingTables, &meta]
(const NThreading::TFuture<TDescribeTableResult>& future) {
- if (future.HasException()) {
- const_cast<NThreading::TFuture<TDescribeTableResult>&>(future).ExtractValue();
- } else if (pendingTables.lock()) {
- meta.emplace(const_cast<NThreading::TFuture<TDescribeTableResult>&>(future).ExtractValue());
- }
+ if (future.HasException()) {
+ const_cast<NThreading::TFuture<TDescribeTableResult>&>(future).ExtractValue();
+ } else if (pendingTables.lock()) {
+ meta.emplace(const_cast<NThreading::TFuture<TDescribeTableResult>&>(future).ExtractValue());
+ }
}));
}
@@ -205,12 +205,12 @@ public:
YQL_ENSURE(AsyncFuture_.HasValue());
output = input;
- Y_VERIFY(Clients_, "Clients_ was destroyed");
- Y_VERIFY(PendingTables_, "PendingTables_ was destroyed");
-
+ Y_VERIFY(Clients_, "Clients_ was destroyed");
+ Y_VERIFY(PendingTables_, "PendingTables_ was destroyed");
+
bool failed = false;
- std::unordered_map<std::string_view, TSnapshotHandle> snapshots(Clients_->size());
- for (auto& client : *Clients_) {
+ std::unordered_map<std::string_view, TSnapshotHandle> snapshots(Clients_->size());
+ for (auto& client : *Clients_) {
auto& snapshot = *client.second.second;
if (snapshot.IsSuccess()) {
snapshots.emplace(client.first, snapshot.ExtractResult());
@@ -223,7 +223,7 @@ public:
}
}
- for (auto& pair : *PendingTables_) {
+ for (auto& pair : *PendingTables_) {
const auto& result = *pair.second;
auto& meta = State_->Tables[pair.first];
meta.Snapshot = snapshots[pair.first.first];
@@ -260,16 +260,16 @@ public:
}
}
- Clients_->clear();
- PendingTables_->clear();
+ Clients_->clear();
+ PendingTables_->clear();
return failed ? TStatus::Error : TStatus::Ok;
}
private:
const NYdb::TDriver Driver_;
const TYdbState::TPtr State_;
- std::shared_ptr<TCluster2ClientPerSnapshotHandle> Clients_ = std::make_shared<TCluster2ClientPerSnapshotHandle>();
- std::shared_ptr<TTableKey2DescribeTableResult > PendingTables_ = std::make_shared<TTableKey2DescribeTableResult>();
+ std::shared_ptr<TCluster2ClientPerSnapshotHandle> Clients_ = std::make_shared<TCluster2ClientPerSnapshotHandle>();
+ std::shared_ptr<TTableKey2DescribeTableResult > PendingTables_ = std::make_shared<TTableKey2DescribeTableResult>();
NThreading::TFuture<void> AsyncFuture_;
};
diff --git a/ydb/library/yql/providers/ydb/provider/yql_ydb_provider.cpp b/ydb/library/yql/providers/ydb/provider/yql_ydb_provider.cpp
index ca69764a86..d17e353d3e 100644
--- a/ydb/library/yql/providers/ydb/provider/yql_ydb_provider.cpp
+++ b/ydb/library/yql/providers/ydb/provider/yql_ydb_provider.cpp
@@ -4,11 +4,11 @@
namespace NYql {
-TDataProviderInitializer GetYdbDataProviderInitializer(
- NYdb::TDriver driver,
- ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory,
- std::shared_ptr<NYq::TDatabaseAsyncResolverWithMeta> dbResolverWithMeta) {
- return [driver, credentialsFactory, dbResolverWithMeta] (
+TDataProviderInitializer GetYdbDataProviderInitializer(
+ NYdb::TDriver driver,
+ ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory,
+ std::shared_ptr<NYq::TDatabaseAsyncResolverWithMeta> dbResolverWithMeta) {
+ return [driver, credentialsFactory, dbResolverWithMeta] (
const TString& userName,
const TString& sessionId,
const TGatewaysConfig* gatewaysConfig,
@@ -30,9 +30,9 @@ TDataProviderInitializer GetYdbDataProviderInitializer(
state->Types = typeCtx.Get();
state->FunctionRegistry = functionRegistry;
state->CredentialsFactory = credentialsFactory;
- state->DbResolver = dbResolverWithMeta;
+ state->DbResolver = dbResolverWithMeta;
if (gatewaysConfig) {
- state->Configuration->Init(gatewaysConfig->GetYdb(), typeCtx, state->DbResolver, state->DatabaseIds);
+ state->Configuration->Init(gatewaysConfig->GetYdb(), typeCtx, state->DbResolver, state->DatabaseIds);
}
TDataProviderInfo info;
diff --git a/ydb/library/yql/providers/ydb/provider/yql_ydb_provider.h b/ydb/library/yql/providers/ydb/provider/yql_ydb_provider.h
index d38323f265..ac9d00d145 100644
--- a/ydb/library/yql/providers/ydb/provider/yql_ydb_provider.h
+++ b/ydb/library/yql/providers/ydb/provider/yql_ydb_provider.h
@@ -33,21 +33,21 @@ struct TYdbState : public TThrRefBase
TYdbConfiguration::TPtr Configuration = MakeIntrusive<TYdbConfiguration>();
const NKikimr::NMiniKQL::IFunctionRegistry* FunctionRegistry = nullptr;
ISecuredServiceAccountCredentialsFactory::TPtr CredentialsFactory;
- THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TDatabaseAuth> DatabaseIds;
- std::shared_ptr<NYq::TDatabaseAsyncResolverWithMeta> DbResolver;
+ THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TDatabaseAuth> DatabaseIds;
+ std::shared_ptr<NYq::TDatabaseAsyncResolverWithMeta> DbResolver;
};
-TDataProviderInitializer GetYdbDataProviderInitializer(
- NYdb::TDriver driver,
- ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory = nullptr,
- const std::shared_ptr<NYq::TDatabaseAsyncResolverWithMeta> dbResolverWithMeta = nullptr
-);
-
-TIntrusivePtr<IDataProvider> CreateYdbDataSource(
- TYdbState::TPtr state,
- NYdb::TDriver driver
-);
-
+TDataProviderInitializer GetYdbDataProviderInitializer(
+ NYdb::TDriver driver,
+ ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory = nullptr,
+ const std::shared_ptr<NYq::TDatabaseAsyncResolverWithMeta> dbResolverWithMeta = nullptr
+);
+
+TIntrusivePtr<IDataProvider> CreateYdbDataSource(
+ TYdbState::TPtr state,
+ NYdb::TDriver driver
+);
+
TIntrusivePtr<IDataProvider> CreateYdbDataSink(TYdbState::TPtr state);
} // namespace NYql
diff --git a/ydb/library/yql/providers/ydb/provider/yql_ydb_provider_impl.h b/ydb/library/yql/providers/ydb/provider/yql_ydb_provider_impl.h
index 3cabc992d8..454c019d58 100644
--- a/ydb/library/yql/providers/ydb/provider/yql_ydb_provider_impl.h
+++ b/ydb/library/yql/providers/ydb/provider/yql_ydb_provider_impl.h
@@ -12,7 +12,7 @@
namespace NYql {
-THolder<IGraphTransformer> CreateYdbIODiscoveryTransformer(TYdbState::TPtr state);
+THolder<IGraphTransformer> CreateYdbIODiscoveryTransformer(TYdbState::TPtr state);
THolder<IGraphTransformer> CreateYdbLoadTableMetadataTransformer(TYdbState::TPtr state, NYdb::TDriver driver);
THolder<TVisitorTransformerBase> CreateYdbDataSourceTypeAnnotationTransformer(TYdbState::TPtr state);
diff --git a/ydb/library/yql/providers/ydb/provider/yql_ydb_settings.cpp b/ydb/library/yql/providers/ydb/provider/yql_ydb_settings.cpp
index da9ae709a2..06d1919027 100644
--- a/ydb/library/yql/providers/ydb/provider/yql_ydb_settings.cpp
+++ b/ydb/library/yql/providers/ydb/provider/yql_ydb_settings.cpp
@@ -17,11 +17,11 @@ bool TYdbConfiguration::HasCluster(TStringBuf cluster) const {
return ValidClusters.contains(cluster);
}
-void TYdbConfiguration::Init(
- const TYdbGatewayConfig& config,
- TIntrusivePtr<TTypeAnnotationContext> typeCtx,
- const std::shared_ptr<NYq::TDatabaseAsyncResolverWithMeta> dbResolver,
- THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TDatabaseAuth>& databaseIds)
+void TYdbConfiguration::Init(
+ const TYdbGatewayConfig& config,
+ TIntrusivePtr<TTypeAnnotationContext> typeCtx,
+ const std::shared_ptr<NYq::TDatabaseAsyncResolverWithMeta> dbResolver,
+ THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TDatabaseAuth>& databaseIds)
{
TVector<TString> clusters(Reserve(config.ClusterMappingSize()));
for (auto& cluster: config.GetClusterMapping()) {
@@ -36,19 +36,19 @@ void TYdbConfiguration::Init(
for (const auto& cluster : config.GetClusterMapping()) {
this->Dispatch(cluster.GetName(), cluster.GetSettings());
-
- if (dbResolver) {
- dbResolver->TryAddDbIdToResolve(cluster.HasEndpoint(), cluster.GetName(), cluster.GetId(), NYq::DatabaseType::Ydb, databaseIds);
- if (cluster.GetId()) {
- DbId2Clusters[cluster.GetId()].emplace_back(cluster.GetName());
- }
- }
-
+
+ if (dbResolver) {
+ dbResolver->TryAddDbIdToResolve(cluster.HasEndpoint(), cluster.GetName(), cluster.GetId(), NYq::DatabaseType::Ydb, databaseIds);
+ if (cluster.GetId()) {
+ DbId2Clusters[cluster.GetId()].emplace_back(cluster.GetName());
+ }
+ }
+
auto& settings = Clusters[cluster.GetName()];
settings.Endpoint = cluster.HasEndpoint() ? cluster.GetEndpoint() : endpoint;
settings.Database = cluster.GetDatabase().empty() ? cluster.GetName() : cluster.GetDatabase();
- if (cluster.GetId())
- settings.DatabaseId = cluster.GetId();
+ if (cluster.GetId())
+ settings.DatabaseId = cluster.GetId();
if (cluster.HasSecure())
settings.Secure = cluster.GetSecure();
if (cluster.HasAddBearerToToken())
@@ -57,7 +57,7 @@ void TYdbConfiguration::Init(
const TString authToken = typeCtx->FindCredentialContent("cluster:default_" + cluster.GetName(), "default_ydb", cluster.GetToken());
Tokens[cluster.GetName()] = ComposeStructuredTokenJsonForServiceAccount(cluster.GetServiceAccountId(), cluster.GetServiceAccountIdSignature(), authToken);
settings.Raw = cluster;
-
+
}
this->FreezeDefaults();
}
diff --git a/ydb/library/yql/providers/ydb/provider/yql_ydb_settings.h b/ydb/library/yql/providers/ydb/provider/yql_ydb_settings.h
index 3640971a8f..fc6fb03ced 100644
--- a/ydb/library/yql/providers/ydb/provider/yql_ydb_settings.h
+++ b/ydb/library/yql/providers/ydb/provider/yql_ydb_settings.h
@@ -8,8 +8,8 @@
#include <ydb/core/yq/libs/events/events.h>
#include <ydb/core/yq/libs/db_resolver/db_async_resolver_with_meta.h>
#include <ydb/core/yq/libs/common/database_token_builder.h>
-
-
+
+
namespace NYql {
struct TYdbSettings {
@@ -19,7 +19,7 @@ struct TYdbSettings {
struct TClusterMainSettings {
TString Endpoint;
TString Database;
- TString DatabaseId;
+ TString DatabaseId;
bool Secure = false;
bool AddBearerToToken = false;
@@ -32,19 +32,19 @@ struct TYdbConfiguration : public TYdbSettings, public NCommon::TSettingDispatch
TYdbConfiguration();
TYdbConfiguration(const TYdbConfiguration&) = delete;
- void Init(
- const TYdbGatewayConfig& config,
- TIntrusivePtr<TTypeAnnotationContext> typeCtx,
- const std::shared_ptr<NYq::TDatabaseAsyncResolverWithMeta> dbResolver,
- THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TDatabaseAuth>& databaseIds
- );
+ void Init(
+ const TYdbGatewayConfig& config,
+ TIntrusivePtr<TTypeAnnotationContext> typeCtx,
+ const std::shared_ptr<NYq::TDatabaseAsyncResolverWithMeta> dbResolver,
+ THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TDatabaseAuth>& databaseIds
+ );
bool HasCluster(TStringBuf cluster) const;
TYdbSettings::TConstPtr Snapshot() const;
THashMap<TString, TString> Tokens;
std::unordered_map<TString, TClusterMainSettings> Clusters;
- THashMap<TString, TVector<TString>> DbId2Clusters; // DatabaseId -> ClusterNames
+ THashMap<TString, TVector<TString>> DbId2Clusters; // DatabaseId -> ClusterNames
};
} // NYql
diff --git a/ydb/library/yql/utils/actor_log/log.cpp b/ydb/library/yql/utils/actor_log/log.cpp
index 5441aec0cc..3b3002208c 100644
--- a/ydb/library/yql/utils/actor_log/log.cpp
+++ b/ydb/library/yql/utils/actor_log/log.cpp
@@ -57,16 +57,16 @@ NYql::NLog::ELevel GetYqlLogLevel(NActors::NLog::EPriority priority) {
} // namespace
void TActorYqlLogBackend::WriteData(const TLogRecord& rec) {
- std::visit([&](const auto* actorCtxOrSystem){
- Y_VERIFY(actorCtxOrSystem);
- if (TraceId.empty()) {
- LOG_LOG(*actorCtxOrSystem, GetActorLogPriority(rec.Priority), Component,
- "SessionId: %s %.*s", SessionId.c_str(), (int)rec.Len, rec.Data);
- } else {
- LOG_LOG(*actorCtxOrSystem, GetActorLogPriority(rec.Priority), Component,
- "TraceId: %s, SessionId: %s %.*s", TraceId.c_str(), SessionId.c_str(), (int)rec.Len, rec.Data);
- }
- }, ActorCtxOrSystem);
+ std::visit([&](const auto* actorCtxOrSystem){
+ Y_VERIFY(actorCtxOrSystem);
+ if (TraceId.empty()) {
+ LOG_LOG(*actorCtxOrSystem, GetActorLogPriority(rec.Priority), Component,
+ "SessionId: %s %.*s", SessionId.c_str(), (int)rec.Len, rec.Data);
+ } else {
+ LOG_LOG(*actorCtxOrSystem, GetActorLogPriority(rec.Priority), Component,
+ "TraceId: %s, SessionId: %s %.*s", TraceId.c_str(), SessionId.c_str(), (int)rec.Len, rec.Data);
+ }
+ }, ActorCtxOrSystem);
}
void SetYqlLogLevels(const NActors::NLog::EPriority& priority) {
diff --git a/ydb/library/yql/utils/actor_log/log.h b/ydb/library/yql/utils/actor_log/log.h
index b0f3e66161..08d525d28b 100644
--- a/ydb/library/yql/utils/actor_log/log.h
+++ b/ydb/library/yql/utils/actor_log/log.h
@@ -11,31 +11,31 @@ namespace NDq {
class TActorYqlLogBackend : public TLogBackend {
public:
- TActorYqlLogBackend(
- const NActors::TActorContext& actorCtx,
- int component,
- const TString& sessionId,
- const TString& traceId)
- : ActorCtxOrSystem(&actorCtx)
+ TActorYqlLogBackend(
+ const NActors::TActorContext& actorCtx,
+ int component,
+ const TString& sessionId,
+ const TString& traceId)
+ : ActorCtxOrSystem(&actorCtx)
+ , Component(component)
+ , SessionId(sessionId)
+ , TraceId(traceId) {}
+
+ TActorYqlLogBackend(
+ const NActors::TActorSystem* actorSystem,
+ int component,
+ const TString& sessionId,
+ const TString& traceId)
+ : ActorCtxOrSystem(actorSystem)
, Component(component)
, SessionId(sessionId)
, TraceId(traceId) {}
- TActorYqlLogBackend(
- const NActors::TActorSystem* actorSystem,
- int component,
- const TString& sessionId,
- const TString& traceId)
- : ActorCtxOrSystem(actorSystem)
- , Component(component)
- , SessionId(sessionId)
- , TraceId(traceId) {}
-
void WriteData(const TLogRecord& rec) override;
void ReopenLog() override {}
private:
- std::variant<const NActors::TActorContext*, const NActors::TActorSystem*> ActorCtxOrSystem;
+ std::variant<const NActors::TActorContext*, const NActors::TActorSystem*> ActorCtxOrSystem;
int Component;
TString SessionId;
TString TraceId;
@@ -43,11 +43,11 @@ private:
class TYqlLogScope : public NYql::NLog::TScopedBackend<TActorYqlLogBackend> {
public:
- TYqlLogScope(const NActors::TActorContext& actorCtx, int component, const TString& sessionId, const TString& traceId = "")
+ TYqlLogScope(const NActors::TActorContext& actorCtx, int component, const TString& sessionId, const TString& traceId = "")
: NYql::NLog::TScopedBackend<TActorYqlLogBackend>(actorCtx, component, sessionId, traceId) {};
-
- TYqlLogScope(const NActors::TActorSystem* actorSystem, int component, const TString& sessionId, const TString& traceId = "")
- : NYql::NLog::TScopedBackend<TActorYqlLogBackend>(actorSystem, component, sessionId, traceId) {};
+
+ TYqlLogScope(const NActors::TActorSystem* actorSystem, int component, const TString& sessionId, const TString& traceId = "")
+ : NYql::NLog::TScopedBackend<TActorYqlLogBackend>(actorSystem, component, sessionId, traceId) {};
};
class TLogWrapReceive: public NActors::TDecorator {
diff --git a/ydb/public/api/grpc/draft/yql_db_v1.proto b/ydb/public/api/grpc/draft/yql_db_v1.proto
index 3686684137..12b061eca3 100644
--- a/ydb/public/api/grpc/draft/yql_db_v1.proto
+++ b/ydb/public/api/grpc/draft/yql_db_v1.proto
@@ -1,19 +1,19 @@
syntax = "proto3";
-package Yq.Private.V1;
+package Yq.Private.V1;
import "ydb/public/api/protos/draft/yq_private.proto";
-service YqPrivateTaskService {
+service YqPrivateTaskService {
// gets new task
- rpc GetTask(Yq.Private.GetTaskRequest) returns (Yq.Private.GetTaskResponse);
+ rpc GetTask(Yq.Private.GetTaskRequest) returns (Yq.Private.GetTaskResponse);
// pings new task (also can update metadata)
- rpc PingTask(Yq.Private.PingTaskRequest) returns (Yq.Private.PingTaskResponse);
+ rpc PingTask(Yq.Private.PingTaskRequest) returns (Yq.Private.PingTaskResponse);
// writes rows
- rpc WriteTaskResult(Yq.Private.WriteTaskResultRequest) returns (Yq.Private.WriteTaskResultResponse);
-
- //Nodes
- rpc NodesHealthCheck(Yq.Private.NodesHealthCheckRequest) returns (Yq.Private.NodesHealthCheckResponse);
+ rpc WriteTaskResult(Yq.Private.WriteTaskResultRequest) returns (Yq.Private.WriteTaskResultResponse);
+
+ //Nodes
+ rpc NodesHealthCheck(Yq.Private.NodesHealthCheckRequest) returns (Yq.Private.NodesHealthCheckResponse);
}
diff --git a/ydb/public/api/protos/draft/yq_private.proto b/ydb/public/api/protos/draft/yq_private.proto
index e08b3726a9..2af1c2f3ed 100644
--- a/ydb/public/api/protos/draft/yq_private.proto
+++ b/ydb/public/api/protos/draft/yq_private.proto
@@ -1,7 +1,7 @@
syntax = "proto3";
option cc_enable_arenas = true;
-package Yq.Private;
+package Yq.Private;
import "ydb/public/api/protos/ydb_operation.proto";
import "ydb/public/api/protos/ydb_value.proto";
@@ -14,17 +14,17 @@ import "google/protobuf/timestamp.proto";
////////////////////////////////////////////////////////////
message GetTaskRequest {
- string tenant = 1;
- string owner_id = 2; // guid, should be refreshed on node restart
- string host = 3;
- Ydb.Operations.OperationParams operation_params = 4;
+ string tenant = 1;
+ string owner_id = 2; // guid, should be refreshed on node restart
+ string host = 3;
+ Ydb.Operations.OperationParams operation_params = 4;
+}
+
+message SignedIdentity {
+ string value = 1;
+ string signature = 2;
}
-message SignedIdentity {
- string value = 1;
- string signature = 2;
-}
-
message TopicConsumer {
string database_id = 1;
string database = 2;
@@ -37,39 +37,39 @@ message TopicConsumer {
}
message GetTaskResult {
- message Task {
- // come back later in 10 sec ?
- SignedIdentity result_id = 1;
- SignedIdentity query_id = 2;
- SignedIdentity job_id = 3;
- uint64 generation = 4;
-
- bool streaming = 5;
- repeated bytes dq_graph = 6;
- // text, connection and binding are empty if dq_graph is not empty
- string text = 7;
- repeated YandexQuery.Connection connection = 8;
- repeated YandexQuery.Binding binding = 9;
-
- string user_token = 10; // IAM token for debug
- repeated SignedIdentity service_accounts = 11;
- string user_id = 12;
- YandexQuery.QueryContent.QueryType query_type = 13;
- string scope = 14;
- YandexQuery.ExecuteMode execute_mode = 15;
- YandexQuery.StateLoadMode state_load_mode = 16;
- YandexQuery.QueryMeta.ComputeStatus status = 17;
- repeated YandexQuery.ResultSetMeta result_set_meta = 18;
- repeated TopicConsumer created_topic_consumers = 19;
- int32 dq_graph_index = 20;
- map<string, string> sensor_labels = 21;
-
- bool automatic = 22;
- string query_name = 23;
+ message Task {
+ // come back later in 10 sec ?
+ SignedIdentity result_id = 1;
+ SignedIdentity query_id = 2;
+ SignedIdentity job_id = 3;
+ uint64 generation = 4;
+
+ bool streaming = 5;
+ repeated bytes dq_graph = 6;
+ // text, connection and binding are empty if dq_graph is not empty
+ string text = 7;
+ repeated YandexQuery.Connection connection = 8;
+ repeated YandexQuery.Binding binding = 9;
+
+ string user_token = 10; // IAM token for debug
+ repeated SignedIdentity service_accounts = 11;
+ string user_id = 12;
+ YandexQuery.QueryContent.QueryType query_type = 13;
+ string scope = 14;
+ YandexQuery.ExecuteMode execute_mode = 15;
+ YandexQuery.StateLoadMode state_load_mode = 16;
+ YandexQuery.QueryMeta.ComputeStatus status = 17;
+ repeated YandexQuery.ResultSetMeta result_set_meta = 18;
+ repeated TopicConsumer created_topic_consumers = 19;
+ int32 dq_graph_index = 20;
+ map<string, string> sensor_labels = 21;
+
+ bool automatic = 22;
+ string query_name = 23;
google.protobuf.Timestamp deadline = 24;
YandexQuery.StreamingDisposition disposition = 25;
- }
- repeated Task tasks = 1;
+ }
+ repeated Task tasks = 1;
}
message GetTaskResponse {
@@ -77,34 +77,34 @@ message GetTaskResponse {
}
message PingTaskRequest {
- string owner_id = 1;
- SignedIdentity query_id = 2;
- SignedIdentity job_id = 3;
- SignedIdentity result_id = 4;
- YandexQuery.QueryMeta.ComputeStatus status = 5;
+ string owner_id = 1;
+ SignedIdentity query_id = 2;
+ SignedIdentity job_id = 3;
+ SignedIdentity result_id = 4;
+ YandexQuery.QueryMeta.ComputeStatus status = 5;
repeated Ydb.Issue.IssueMessage issues = 6;
repeated Ydb.Issue.IssueMessage transient_issues = 16;
uint32 result_set_count = 7;
string statistics = 8;
- repeated YandexQuery.ResultSetMeta result_set_meta = 9;
+ repeated YandexQuery.ResultSetMeta result_set_meta = 9;
string executer_info = 10;
repeated bytes dq_graph = 11;
int32 dq_graph_index = 20;
- string ast = 12;
- string plan = 13;
- bool resign_query = 14;
+ string ast = 12;
+ string plan = 13;
+ bool resign_query = 14;
repeated TopicConsumer created_topic_consumers = 17;
YandexQuery.StateLoadMode state_load_mode = 18;
YandexQuery.StreamingDisposition disposition = 19;
- Ydb.Operations.OperationParams operation_params = 15;
- string scope = 100; //TODO remove
- google.protobuf.Timestamp started_at = 101;
- google.protobuf.Timestamp finished_at = 102;
+ Ydb.Operations.OperationParams operation_params = 15;
+ string scope = 100; //TODO remove
+ google.protobuf.Timestamp started_at = 101;
+ google.protobuf.Timestamp finished_at = 102;
google.protobuf.Timestamp deadline = 103;
}
message PingTaskResult {
- YandexQuery.QueryAction action = 1;
+ YandexQuery.QueryAction action = 1;
}
message PingTaskResponse {
@@ -112,24 +112,24 @@ message PingTaskResponse {
}
message WriteTaskResultRequest {
- string owner_id = 1;
- SignedIdentity result_id = 2;
+ string owner_id = 1;
+ SignedIdentity result_id = 2;
Ydb.ResultSet result_set = 3;
uint32 result_set_id = 4;
uint64 offset = 5;
- uint64 request_id = 6;
- Ydb.Operations.OperationParams operation_params = 7;
+ uint64 request_id = 6;
+ Ydb.Operations.OperationParams operation_params = 7;
google.protobuf.Timestamp deadline = 8;
}
message WriteTaskResultResult {
- uint64 request_id = 1;
+ uint64 request_id = 1;
}
message WriteTaskResultResponse {
Ydb.Operations.Operation operation = 1; // WriteRowsResultResult
}
-
+
message NodeInfo {
uint32 node_id = 1;
string instance_id = 2;
@@ -137,20 +137,20 @@ message NodeInfo {
uint64 active_workers = 4;
uint64 memory_limit = 5;
uint64 memory_allocated = 6;
- uint32 interconnect_port = 7;
- string node_address = 8;
+ uint32 interconnect_port = 7;
+ string node_address = 8;
}
-message NodesHealthCheckRequest {
- string tenant = 1;
+message NodesHealthCheckRequest {
+ string tenant = 1;
NodeInfo node = 2;
- Ydb.Operations.OperationParams operation_params = 6;
-}
-
-message NodesHealthCheckResult {
- repeated NodeInfo nodes = 1;
-}
-
-message NodesHealthCheckResponse {
- Ydb.Operations.Operation operation = 1; // NodesHealthCheckResult
-}
+ Ydb.Operations.OperationParams operation_params = 6;
+}
+
+message NodesHealthCheckResult {
+ repeated NodeInfo nodes = 1;
+}
+
+message NodesHealthCheckResponse {
+ Ydb.Operations.Operation operation = 1; // NodesHealthCheckResult
+}
diff --git a/ydb/public/api/protos/ya.make b/ydb/public/api/protos/ya.make
index 4d02adac4d..8a156403a3 100644
--- a/ydb/public/api/protos/ya.make
+++ b/ydb/public/api/protos/ya.make
@@ -19,7 +19,7 @@ SRCS(
draft/persqueue_error_codes.proto
draft/ydb_long_tx.proto
draft/ydb_logstore.proto
- draft/yq_private.proto
+ draft/yq_private.proto
persqueue_error_codes_v1.proto
ydb_auth.proto
ydb_persqueue_v1.proto
diff --git a/ydb/public/api/protos/yq.proto b/ydb/public/api/protos/yq.proto
index cb24bbc1da..6aa7169a89 100644
--- a/ydb/public/api/protos/yq.proto
+++ b/ydb/public/api/protos/yq.proto
@@ -104,7 +104,7 @@ message QueryContent {
ANALYTICS = 1; // Analytical query (used for analytical data processing for example to work with YDB, ClickHouse, ...)
STREAMING = 2; // Streaming query (used for streaming data processing, such as working with YDS)
}
- QueryType type = 1;
+ QueryType type = 1;
string name = 2 [(Ydb.length).le = 1024];
Acl acl = 3;
Limits limits = 4;
@@ -160,7 +160,7 @@ message QueryMeta {
}
message BriefQuery {
- QueryContent.QueryType type = 1;
+ QueryContent.QueryType type = 1;
string name = 2 [(Ydb.length).le = 1024];
QueryMeta meta = 3;
Acl.Visibility visibility = 4;
diff --git a/ydb/services/yq/private_grpc.cpp b/ydb/services/yq/private_grpc.cpp
index 79a9f5e9ea..c4c076b482 100644
--- a/ydb/services/yq/private_grpc.cpp
+++ b/ydb/services/yq/private_grpc.cpp
@@ -1,66 +1,66 @@
-#include "private_grpc.h"
-
+#include "private_grpc.h"
+
#include <ydb/core/grpc_services/base/base.h>
#include <ydb/core/grpc_services/grpc_helper.h>
#include <ydb/core/grpc_services/service_analytics_internal.h>
#include <ydb/library/protobuf_printer/security_printer.h>
-
-namespace NKikimr {
-namespace NGRpcService {
-
-TGRpcYqPrivateTaskService::TGRpcYqPrivateTaskService(NActors::TActorSystem *system,
- TIntrusivePtr<NMonitoring::TDynamicCounters> counters, NActors::TActorId id)
- : ActorSystem_(system)
- , Counters_(counters)
- , GRpcRequestProxyId_(id) {}
-
-void TGRpcYqPrivateTaskService::InitService(grpc::ServerCompletionQueue* cq, NGrpc::TLoggerPtr logger) {
- CQ_ = cq;
- SetupIncomingRequests(std::move(logger));
-}
-
-void TGRpcYqPrivateTaskService::SetGlobalLimiterHandle(NGrpc::TGlobalLimiter* limiter) {
- Limiter_ = limiter;
-}
-
-bool TGRpcYqPrivateTaskService::IncRequest() {
- return Limiter_->Inc();
-}
-
-void TGRpcYqPrivateTaskService::DecRequest() {
- Limiter_->Dec();
- Y_ASSERT(Limiter_->GetCurrentInFlight() >= 0);
-}
-
-void TGRpcYqPrivateTaskService::SetupIncomingRequests(NGrpc::TLoggerPtr logger) {
- auto getCounterBlock = CreateCounterCb(Counters_, ActorSystem_);
-
-#ifdef ADD_REQUEST
-#error ADD_REQUEST macro already defined
-#endif
+
+namespace NKikimr {
+namespace NGRpcService {
+
+TGRpcYqPrivateTaskService::TGRpcYqPrivateTaskService(NActors::TActorSystem *system,
+ TIntrusivePtr<NMonitoring::TDynamicCounters> counters, NActors::TActorId id)
+ : ActorSystem_(system)
+ , Counters_(counters)
+ , GRpcRequestProxyId_(id) {}
+
+void TGRpcYqPrivateTaskService::InitService(grpc::ServerCompletionQueue* cq, NGrpc::TLoggerPtr logger) {
+ CQ_ = cq;
+ SetupIncomingRequests(std::move(logger));
+}
+
+void TGRpcYqPrivateTaskService::SetGlobalLimiterHandle(NGrpc::TGlobalLimiter* limiter) {
+ Limiter_ = limiter;
+}
+
+bool TGRpcYqPrivateTaskService::IncRequest() {
+ return Limiter_->Inc();
+}
+
+void TGRpcYqPrivateTaskService::DecRequest() {
+ Limiter_->Dec();
+ Y_ASSERT(Limiter_->GetCurrentInFlight() >= 0);
+}
+
+void TGRpcYqPrivateTaskService::SetupIncomingRequests(NGrpc::TLoggerPtr logger) {
+ auto getCounterBlock = CreateCounterCb(Counters_, ActorSystem_);
+
+#ifdef ADD_REQUEST
+#error ADD_REQUEST macro already defined
+#endif
#define ADD_REQUEST(NAME, CB) \
MakeIntrusive<TGRpcRequest<Yq::Private::NAME##Request, Yq::Private::NAME##Response, TGRpcYqPrivateTaskService, TSecurityTextFormatPrinter<Yq::Private::NAME##Request>, TSecurityTextFormatPrinter<Yq::Private::NAME##Response>>>( \
this, &Service_, CQ_, \
[this](NGrpc::IRequestContextBase *ctx) { \
NGRpcService::ReportGrpcReqToMon(*ActorSystem_, ctx->GetPeer()); \
ActorSystem_->Send(GRpcRequestProxyId_, \
- new TGrpcRequestOperationCall<Yq::Private::NAME##Request, Yq::Private::NAME##Response> \
+ new TGrpcRequestOperationCall<Yq::Private::NAME##Request, Yq::Private::NAME##Response> \
(ctx, &CB)); \
}, \
- &Yq::Private::V1::YqPrivateTaskService::AsyncService::Request##NAME, \
+ &Yq::Private::V1::YqPrivateTaskService::AsyncService::Request##NAME, \
#NAME, logger, getCounterBlock("yql_internal", #NAME)) \
->Run(); \
-
- ADD_REQUEST(PingTask, DoYqPrivatePingTaskRequest)
-
- ADD_REQUEST(GetTask, DoYqPrivateGetTaskRequest)
-
- ADD_REQUEST(WriteTaskResult, DoYqPrivateWriteTaskResultRequest)
-
- ADD_REQUEST(NodesHealthCheck, DoYqPrivateNodesHealthCheckRequest)
-
-#undef ADD_REQUEST
-}
-
-} // namespace NGRpcService
-} // namespace NKikimr
+
+ ADD_REQUEST(PingTask, DoYqPrivatePingTaskRequest)
+
+ ADD_REQUEST(GetTask, DoYqPrivateGetTaskRequest)
+
+ ADD_REQUEST(WriteTaskResult, DoYqPrivateWriteTaskResultRequest)
+
+ ADD_REQUEST(NodesHealthCheck, DoYqPrivateNodesHealthCheckRequest)
+
+#undef ADD_REQUEST
+}
+
+} // namespace NGRpcService
+} // namespace NKikimr
diff --git a/ydb/services/yq/private_grpc.h b/ydb/services/yq/private_grpc.h
index 5406b58446..f5107a7c10 100644
--- a/ydb/services/yq/private_grpc.h
+++ b/ydb/services/yq/private_grpc.h
@@ -1,34 +1,34 @@
-#pragma once
-
-#include <library/cpp/actors/core/actorsystem.h>
-#include <library/cpp/grpc/server/grpc_server.h>
+#pragma once
+
+#include <library/cpp/actors/core/actorsystem.h>
+#include <library/cpp/grpc/server/grpc_server.h>
#include <ydb/public/api/grpc/draft/yql_db_v1.grpc.pb.h>
-
-namespace NKikimr {
-namespace NGRpcService {
-
-class TGRpcYqPrivateTaskService
- : public NGrpc::TGrpcServiceBase<Yq::Private::V1::YqPrivateTaskService>
-{
-public:
- TGRpcYqPrivateTaskService(NActors::TActorSystem* system, TIntrusivePtr<NMonitoring::TDynamicCounters> counters,
- NActors::TActorId id);
-
- void InitService(grpc::ServerCompletionQueue* cq, NGrpc::TLoggerPtr logger) override;
- void SetGlobalLimiterHandle(NGrpc::TGlobalLimiter* limiter) override;
-
- bool IncRequest();
- void DecRequest();
-private:
- void SetupIncomingRequests(NGrpc::TLoggerPtr logger);
-
- NActors::TActorSystem* ActorSystem_;
+
+namespace NKikimr {
+namespace NGRpcService {
+
+class TGRpcYqPrivateTaskService
+ : public NGrpc::TGrpcServiceBase<Yq::Private::V1::YqPrivateTaskService>
+{
+public:
+ TGRpcYqPrivateTaskService(NActors::TActorSystem* system, TIntrusivePtr<NMonitoring::TDynamicCounters> counters,
+ NActors::TActorId id);
+
+ void InitService(grpc::ServerCompletionQueue* cq, NGrpc::TLoggerPtr logger) override;
+ void SetGlobalLimiterHandle(NGrpc::TGlobalLimiter* limiter) override;
+
+ bool IncRequest();
+ void DecRequest();
+private:
+ void SetupIncomingRequests(NGrpc::TLoggerPtr logger);
+
+ NActors::TActorSystem* ActorSystem_;
grpc::ServerCompletionQueue* CQ_ = nullptr;
-
- TIntrusivePtr<NMonitoring::TDynamicCounters> Counters_;
- NActors::TActorId GRpcRequestProxyId_;
+
+ TIntrusivePtr<NMonitoring::TDynamicCounters> Counters_;
+ NActors::TActorId GRpcRequestProxyId_;
NGrpc::TGlobalLimiter* Limiter_ = nullptr;
-};
-
-} // namespace NGRpcService
-} // namespace NKikimr
+};
+
+} // namespace NGRpcService
+} // namespace NKikimr
diff --git a/ydb/services/yq/ut_integration/ut_utils.cpp b/ydb/services/yq/ut_integration/ut_utils.cpp
index 0ab1382fd7..925f74f7b3 100644
--- a/ydb/services/yq/ut_integration/ut_utils.cpp
+++ b/ydb/services/yq/ut_integration/ut_utils.cpp
@@ -17,15 +17,15 @@
#include <util/string/printf.h>
#include <util/string/builder.h>
-#include <library/cpp/protobuf/json/proto2json.h>
-
+#include <library/cpp/protobuf/json/proto2json.h>
+
using namespace NYdb;
using namespace NYdb::NYq;
using namespace NActors;
-void UpsertToExistingTable(TDriver& driver, const TString& location){
- Y_UNUSED(location);
- const TString tablePrefix = "Root/yq";
+void UpsertToExistingTable(TDriver& driver, const TString& location){
+ Y_UNUSED(location);
+ const TString tablePrefix = "Root/yq";
NYdb::NTable::TClientSettings settings;
NYdb::NTable::TTableClient client(driver, settings);
auto sessionOp = client.CreateSession().ExtractValueSync();
@@ -36,7 +36,7 @@ void UpsertToExistingTable(TDriver& driver, const TString& location){
auto timeProvider = CreateDefaultTimeProvider();
auto now = timeProvider->Now();
NYdb::TParamsBuilder paramsBuilder;
-
+
paramsBuilder.AddParam("$now").Timestamp(now).Build();
auto params = paramsBuilder.Build();
@@ -90,4 +90,4 @@ void UpsertToExistingTable(TDriver& driver, const TString& location){
}
}
}
-
+
diff --git a/ydb/services/yq/ut_integration/ut_utils.h b/ydb/services/yq/ut_integration/ut_utils.h
index 08229a279c..a627c45929 100644
--- a/ydb/services/yq/ut_integration/ut_utils.h
+++ b/ydb/services/yq/ut_integration/ut_utils.h
@@ -3,7 +3,7 @@
#include <ydb/public/sdk/cpp/client/ydb_driver/driver.h>
#include <util/system/shellcommand.h>
#include <contrib/libs/curl/include/curl/curl.h>
-#include <library/cpp/json/json_reader.h>
+#include <library/cpp/json/json_reader.h>
-void UpsertToExistingTable(NYdb::TDriver& driver, const TString& location);
+void UpsertToExistingTable(NYdb::TDriver& driver, const TString& location);
diff --git a/ydb/services/yq/ut_integration/ya.make b/ydb/services/yq/ut_integration/ya.make
index fbbe452cf4..bc62df4563 100644
--- a/ydb/services/yq/ut_integration/ya.make
+++ b/ydb/services/yq/ut_integration/ya.make
@@ -17,8 +17,8 @@ SRCS(
PEERDIR(
library/cpp/getopt
- library/cpp/grpc/client
- library/cpp/regex/pcre
+ library/cpp/grpc/client
+ library/cpp/regex/pcre
library/cpp/svnversion
ydb/core/testlib
ydb/core/yq/libs/control_plane_storage
diff --git a/ydb/services/yq/ut_integration/yq_ut.cpp b/ydb/services/yq/ut_integration/yq_ut.cpp
index 3e43d91c65..d4a35f195e 100644
--- a/ydb/services/yq/ut_integration/yq_ut.cpp
+++ b/ydb/services/yq/ut_integration/yq_ut.cpp
@@ -14,221 +14,221 @@
#include <library/cpp/protobuf/util/pb_io.h>
#include <library/cpp/retry/retry.h>
-#include <util/system/mutex.h>
+#include <util/system/mutex.h>
#include "ut_utils.h"
-#include <google/protobuf/util/time_util.h>
+#include <google/protobuf/util/time_util.h>
#include <ydb/public/lib/yq/scope.h>
-#include <util/system/hostname.h>
-
-#include <util/string/split.h>
-
+#include <util/system/hostname.h>
+
+#include <util/string/split.h>
+
using namespace NYdb;
using namespace YandexQuery;;
using namespace NYdb::NYq;
-
-namespace {
- const ui32 Retries = 10;
-
- void PrintProtoIssues(const NProtoBuf::RepeatedPtrField<::Ydb::Issue::IssueMessage>& protoIssues) {
- if (protoIssues.empty()) {
- Cerr << "No Issues" << Endl;
- return;
- }
- NYql::TIssues issues;
- NYql::IssuesFromMessage(protoIssues, issues);
- Cerr << ">>> Issues: " << issues.ToString() << Endl;
- }
-
+
+namespace {
+ const ui32 Retries = 10;
+
+ void PrintProtoIssues(const NProtoBuf::RepeatedPtrField<::Ydb::Issue::IssueMessage>& protoIssues) {
+ if (protoIssues.empty()) {
+ Cerr << "No Issues" << Endl;
+ return;
+ }
+ NYql::TIssues issues;
+ NYql::IssuesFromMessage(protoIssues, issues);
+ Cerr << ">>> Issues: " << issues.ToString() << Endl;
+ }
+
TString CreateNewHistoryAndWaitFinish(const TString& folderId,
- NYdb::NYq::TClient& client, const TString& yqlText,
- const YandexQuery::QueryMeta::ComputeStatus& expectedStatusResult)
+ NYdb::NYq::TClient& client, const TString& yqlText,
+ const YandexQuery::QueryMeta::ComputeStatus& expectedStatusResult)
{
- //CreateQuery
- TString queryId;
- {
- auto request = ::NYq::TCreateQueryBuilder{}
- .SetText(yqlText)
- .Build();
- auto result = client.CreateQuery(
+ //CreateQuery
+ TString queryId;
+ {
+ auto request = ::NYq::TCreateQueryBuilder{}
+ .SetText(yqlText)
+ .Build();
+ auto result = client.CreateQuery(
request, CreateYqSettings<TCreateQuerySettings>(folderId))
- .ExtractValueSync();
+ .ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
- queryId = result.GetResult().query_id();
- }
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
+ queryId = result.GetResult().query_id();
+ }
// GetQueryStatus
const auto request = ::NYq::TGetQueryStatusBuilder{}
- .SetQueryId(queryId)
- .Build();
- const auto result = DoWithRetryOnRetCode([&]() {
+ .SetQueryId(queryId)
+ .Build();
+ const auto result = DoWithRetryOnRetCode([&]() {
auto result = client.GetQueryStatus(
request, CreateYqSettings<TGetQueryStatusSettings>(folderId))
- .ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
+ .ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
return result.GetResult().status() == expectedStatusResult;
- }, TRetryOptions(Retries));
- UNIT_ASSERT_C(result, "the execution of the query did not end within the time limit");
-
- return queryId;
- }
-
- void CheckGetResultData(
- NYdb::NYq::TClient& client,
- const TString& queryId,
- const TString& folderId,
- const ui64 rowsCount,
- const int colsSize,
- const int answer)
- {
- const auto request = ::NYq::TGetResultDataBuilder{}
- .SetQueryId(queryId)
- .Build();
- const auto result = client.GetResultData(
+ }, TRetryOptions(Retries));
+ UNIT_ASSERT_C(result, "the execution of the query did not end within the time limit");
+
+ return queryId;
+ }
+
+ void CheckGetResultData(
+ NYdb::NYq::TClient& client,
+ const TString& queryId,
+ const TString& folderId,
+ const ui64 rowsCount,
+ const int colsSize,
+ const int answer)
+ {
+ const auto request = ::NYq::TGetResultDataBuilder{}
+ .SetQueryId(queryId)
+ .Build();
+ const auto result = client.GetResultData(
request, CreateYqSettings<TGetResultDataSettings>(folderId))
- .ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
-
- const auto& resultSet = result.GetResult().result_set();
- UNIT_ASSERT_VALUES_EQUAL(resultSet.rows().size(), rowsCount);
- UNIT_ASSERT_VALUES_EQUAL(resultSet.columns().size(), colsSize);
- if (!resultSet.rows().empty()) {
- const auto& item = resultSet.rows(0).items(0);
- TString str = item.DebugString();
- TVector<TString> arr;
- StringSplitter(str).Split(' ').SkipEmpty().AddTo(&arr);
- Y_VERIFY(arr.size() == 2, "Incorrect numeric result");
- UNIT_ASSERT_VALUES_EQUAL(std::stoi(arr.back()), answer);
- }
- }
-}
-
+ .ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
+
+ const auto& resultSet = result.GetResult().result_set();
+ UNIT_ASSERT_VALUES_EQUAL(resultSet.rows().size(), rowsCount);
+ UNIT_ASSERT_VALUES_EQUAL(resultSet.columns().size(), colsSize);
+ if (!resultSet.rows().empty()) {
+ const auto& item = resultSet.rows(0).items(0);
+ TString str = item.DebugString();
+ TVector<TString> arr;
+ StringSplitter(str).Split(' ').SkipEmpty().AddTo(&arr);
+ Y_VERIFY(arr.size() == 2, "Incorrect numeric result");
+ UNIT_ASSERT_VALUES_EQUAL(std::stoi(arr.back()), answer);
+ }
+ }
+}
+
Y_UNIT_TEST_SUITE(Yq_1) {
Y_UNIT_TEST(Basic) {
TKikimrWithGrpcAndRootSchema server({}, {}, {}, true);
- ui16 grpc = server.GetPort();
+ ui16 grpc = server.GetPort();
TString location = TStringBuilder() << "localhost:" << grpc;
- auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken("root@builtin"));
- NYdb::NYq::TClient client(driver);
- const auto folderId = "some_folder_id";
+ auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken("root@builtin"));
+ NYdb::NYq::TClient client(driver);
+ const auto folderId = "some_folder_id";
const auto queryId = CreateNewHistoryAndWaitFinish(folderId, client, "select 1", YandexQuery::QueryMeta::COMPLETED);
CheckGetResultData(client, queryId, folderId, 1, 1, 1);
{
- const auto request = ::NYq::TDescribeQueryBuilder()
- .SetQueryId("foo")
- .Build();
- const auto result = DoWithRetryOnRetCode([&]() {
- auto result = client.DescribeQuery(
+ const auto request = ::NYq::TDescribeQueryBuilder()
+ .SetQueryId("foo")
+ .Build();
+ const auto result = DoWithRetryOnRetCode([&]() {
+ auto result = client.DescribeQuery(
request, CreateYqSettings<TDescribeQuerySettings>("WTF"))
- .ExtractValueSync();
- return result.GetStatus() == EStatus::BAD_REQUEST;
- }, TRetryOptions(Retries));
- UNIT_ASSERT_C(result, "the execution of the query did not end within the time limit");
+ .ExtractValueSync();
+ return result.GetStatus() == EStatus::BAD_REQUEST;
+ }, TRetryOptions(Retries));
+ UNIT_ASSERT_C(result, "the execution of the query did not end within the time limit");
}
{
- auto request = ::NYq::TListQueriesBuilder{}.Build();
- auto result = DoWithRetryOnRetCode([&]() {
- auto result = client.ListQueries(
+ auto request = ::NYq::TListQueriesBuilder{}.Build();
+ auto result = DoWithRetryOnRetCode([&]() {
+ auto result = client.ListQueries(
request, CreateYqSettings<TListQueriesSettings>("WTF"))
- .ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
- UNIT_ASSERT_VALUES_EQUAL(result.GetResult().query().size(), 0);
- return result.GetStatus() == EStatus::SUCCESS;
- }, TRetryOptions(Retries));
- UNIT_ASSERT_C(result, "the execution of the query did not end within the time limit");
- }
-
- {
- const auto request = ::NYq::TDescribeQueryBuilder()
- .SetQueryId(queryId)
- .Build();
- auto result = client.DescribeQuery(
+ .ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
+ UNIT_ASSERT_VALUES_EQUAL(result.GetResult().query().size(), 0);
+ return result.GetStatus() == EStatus::SUCCESS;
+ }, TRetryOptions(Retries));
+ UNIT_ASSERT_C(result, "the execution of the query did not end within the time limit");
+ }
+
+ {
+ const auto request = ::NYq::TDescribeQueryBuilder()
+ .SetQueryId(queryId)
+ .Build();
+ auto result = client.DescribeQuery(
request, CreateYqSettings<TDescribeQuerySettings>(folderId))
- .ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
- const auto status = result.GetResult().query().meta().status();
- UNIT_ASSERT(status == YandexQuery::QueryMeta::COMPLETED);
+ .ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
+ const auto status = result.GetResult().query().meta().status();
+ UNIT_ASSERT(status == YandexQuery::QueryMeta::COMPLETED);
}
{
- const auto request = ::NYq::TModifyQueryBuilder()
- .SetQueryId(queryId)
- .SetName("MODIFIED_NAME")
- .Build();
- const auto result = client.ModifyQuery(
+ const auto request = ::NYq::TModifyQueryBuilder()
+ .SetQueryId(queryId)
+ .SetName("MODIFIED_NAME")
+ .Build();
+ const auto result = client.ModifyQuery(
request, CreateYqSettings<TModifyQuerySettings>(folderId))
- .ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
+ .ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
}
{
- const auto request = ::NYq::TDescribeQueryBuilder()
- .SetQueryId(queryId)
- .Build();
- const auto result = client.DescribeQuery(
+ const auto request = ::NYq::TDescribeQueryBuilder()
+ .SetQueryId(queryId)
+ .Build();
+ const auto result = client.DescribeQuery(
request, CreateYqSettings<TDescribeQuerySettings>(folderId))
- .ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
- const auto& res = result.GetResult();
- UNIT_ASSERT_VALUES_EQUAL(res.query().content().name(), "MODIFIED_NAME");
- UNIT_ASSERT(res.query().content().acl().visibility() == static_cast<int>(Acl_Visibility_SCOPE));
+ .ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
+ const auto& res = result.GetResult();
+ UNIT_ASSERT_VALUES_EQUAL(res.query().content().name(), "MODIFIED_NAME");
+ UNIT_ASSERT(res.query().content().acl().visibility() == static_cast<int>(Acl_Visibility_SCOPE));
}
{
- const auto request = ::NYq::TDescribeQueryBuilder()
- .SetQueryId("")
- .Build();
- const auto result = client.DescribeQuery(
+ const auto request = ::NYq::TDescribeQueryBuilder()
+ .SetQueryId("")
+ .Build();
+ const auto result = client.DescribeQuery(
request, CreateYqSettings<TDescribeQuerySettings>(""))
- .ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::BAD_REQUEST, result.GetIssues().ToString());
+ .ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::BAD_REQUEST, result.GetIssues().ToString());
}
{
- const auto request = ::NYq::TGetResultDataBuilder()
- .SetQueryId("")
- .Build();
- const auto result = client.GetResultData(
+ const auto request = ::NYq::TGetResultDataBuilder()
+ .SetQueryId("")
+ .Build();
+ const auto result = client.GetResultData(
request, CreateYqSettings<TGetResultDataSettings>(""))
- .ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::BAD_REQUEST, result.GetIssues().ToString());
+ .ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::BAD_REQUEST, result.GetIssues().ToString());
}
}
Y_UNIT_TEST(Basic_EmptyTable) {
TKikimrWithGrpcAndRootSchema server({}, {}, {}, true);
- ui16 grpc = server.GetPort();
+ ui16 grpc = server.GetPort();
TString location = TStringBuilder() << "localhost:" << grpc;
- auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken("root@builtin"));
+ auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken("root@builtin"));
UpsertToExistingTable(driver, location);
- NYdb::NYq::TClient client(driver);
+ NYdb::NYq::TClient client(driver);
const TString folderId = "some_folder_id";
{
- const auto request = ::NYq::TCreateConnectionBuilder()
- .SetName("testdbempty")
- .CreateYdb("Root", location, "")
- .Build();
- const auto result = client
+ const auto request = ::NYq::TCreateConnectionBuilder()
+ .SetName("testdbempty")
+ .CreateYdb("Root", location, "")
+ .Build();
+ const auto result = client
.CreateConnection(request, CreateYqSettings<TCreateConnectionSettings>(folderId))
- .ExtractValueSync();
- UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString());
+ .ExtractValueSync();
+ UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString());
}
- const TString queryId = CreateNewHistoryAndWaitFinish(
+ const TString queryId = CreateNewHistoryAndWaitFinish(
folderId, client,
- "select count(*) from testdbempty.`yq/empty_table`",
- YandexQuery::QueryMeta::COMPLETED);
+ "select count(*) from testdbempty.`yq/empty_table`",
+ YandexQuery::QueryMeta::COMPLETED);
CheckGetResultData(client, queryId, folderId, 1, 1, 0);
}
Y_UNIT_TEST(Basic_EmptyList) {
TKikimrWithGrpcAndRootSchema server({}, {}, {}, true);
- ui16 grpc = server.GetPort();
+ ui16 grpc = server.GetPort();
TString location = TStringBuilder() << "localhost:" << grpc;
- auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken("root@builtin"));
- NYdb::NYq::TClient client(driver);
+ auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken("root@builtin"));
+ NYdb::NYq::TClient client(driver);
const TString folderId = "some_folder_id";
auto expectedStatus = YandexQuery::QueryMeta::COMPLETED;
CreateNewHistoryAndWaitFinish(folderId, client, "select []", expectedStatus);
@@ -256,7 +256,7 @@ Y_UNIT_TEST_SUITE(Yq_1) {
CreateNewHistoryAndWaitFinish(folderId, client, "select null", expectedStatus);
}
- SIMPLE_UNIT_FORKED_TEST(Basic_Tagged) {
+ SIMPLE_UNIT_FORKED_TEST(Basic_Tagged) {
TKikimrWithGrpcAndRootSchema server({}, {}, {}, true);
ui16 grpc = server.GetPort();
TString location = TStringBuilder() << "localhost:" << grpc;
@@ -295,104 +295,104 @@ Y_UNIT_TEST_SUITE(Yq_1) {
}
// use fork for data test due to ch initialization problem
- SIMPLE_UNIT_FORKED_TEST(ExtendedDatabaseId) {
+ SIMPLE_UNIT_FORKED_TEST(ExtendedDatabaseId) {
TKikimrWithGrpcAndRootSchema server({}, {}, {}, true);
- ui16 grpc = server.GetPort();
+ ui16 grpc = server.GetPort();
TString location = TStringBuilder() << "localhost:" << grpc;
- auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken("root@builtin"));
+ auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken("root@builtin"));
- NYdb::NYq::TClient client(driver);
+ NYdb::NYq::TClient client(driver);
const TString folderId = "folder_id_" + CreateGuidAsString();
{
- const auto request = ::NYq::TCreateConnectionBuilder()
- .SetName("testdb01")
- .CreateYdb("FakeDatabaseId", "")
- .Build();
- const auto result = client
+ const auto request = ::NYq::TCreateConnectionBuilder()
+ .SetName("testdb01")
+ .CreateYdb("FakeDatabaseId", "")
+ .Build();
+ const auto result = client
.CreateConnection(request, CreateYqSettings<TCreateConnectionSettings>(folderId))
- .ExtractValueSync();
- UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString());
+ .ExtractValueSync();
+ UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString());
}
{
- const auto request = ::NYq::TCreateConnectionBuilder()
- .SetName("testdb02")
- .CreateYdb("FakeDatabaseId", "")
- .Build();
- const auto result = client
+ const auto request = ::NYq::TCreateConnectionBuilder()
+ .SetName("testdb02")
+ .CreateYdb("FakeDatabaseId", "")
+ .Build();
+ const auto result = client
.CreateConnection(request, CreateYqSettings<TCreateConnectionSettings>(folderId))
- .ExtractValueSync();
- UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString());
+ .ExtractValueSync();
+ UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString());
}
- {
+ {
const auto queryId = CreateNewHistoryAndWaitFinish(folderId, client,
- "select count(*) from testdb01.`yq/connections`", YandexQuery::QueryMeta::COMPLETED);
+ "select count(*) from testdb01.`yq/connections`", YandexQuery::QueryMeta::COMPLETED);
CheckGetResultData(client, queryId, folderId, 1, 1, 2);
- }
-
- {
- // test connections db with 2 databaseId
+ }
+
+ {
+ // test connections db with 2 databaseId
const auto queryId = CreateNewHistoryAndWaitFinish(folderId, client,
- "select count(*) from testdb02.`yq/connections`", YandexQuery::QueryMeta::COMPLETED);
+ "select count(*) from testdb02.`yq/connections`", YandexQuery::QueryMeta::COMPLETED);
CheckGetResultData(client, queryId, folderId, 1, 1, 2);
- }
+ }
}
- Y_UNIT_TEST(DescribeConnection) {
+ Y_UNIT_TEST(DescribeConnection) {
TKikimrWithGrpcAndRootSchema server({}, {}, {}, true);
- ui16 grpc = server.GetPort();
+ ui16 grpc = server.GetPort();
TString location = TStringBuilder() << "localhost:" << grpc;
- auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken("root@builtin"));
- NYdb::NYq::TClient client(driver);
- const TString folderId = "some_folder_id";
- TString conId;
- {
- const auto request = ::NYq::TCreateConnectionBuilder()
- .SetName("created_conn")
- .CreateYdb("created_db", "")
- .Build();
- const auto result = client
+ auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken("root@builtin"));
+ NYdb::NYq::TClient client(driver);
+ const TString folderId = "some_folder_id";
+ TString conId;
+ {
+ const auto request = ::NYq::TCreateConnectionBuilder()
+ .SetName("created_conn")
+ .CreateYdb("created_db", "")
+ .Build();
+ const auto result = client
.CreateConnection(request, CreateYqSettings<TCreateConnectionSettings>(folderId))
- .ExtractValueSync();
- UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString());
- conId = result.GetResult().connection_id();
- }
- {
- auto request = ::NYq::TDescribeConnectionBuilder()
- .SetConnectionId(conId)
- .Build();
- auto result = client
+ .ExtractValueSync();
+ UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString());
+ conId = result.GetResult().connection_id();
+ }
+ {
+ auto request = ::NYq::TDescribeConnectionBuilder()
+ .SetConnectionId(conId)
+ .Build();
+ auto result = client
.DescribeConnection(request, CreateYqSettings<TDescribeConnectionSettings>(folderId))
- .ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- const auto& res = result.GetResult().connection();
- UNIT_ASSERT_VALUES_EQUAL(res.meta().id(), conId);
- UNIT_ASSERT_VALUES_EQUAL(res.meta().created_by(), "root@builtin");
- UNIT_ASSERT_VALUES_EQUAL(res.meta().modified_by(), "root@builtin");
- UNIT_ASSERT_VALUES_EQUAL(res.content().name(), "created_conn");
- }
- }
-
- Y_UNIT_TEST(ListConnections) {
+ .ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ const auto& res = result.GetResult().connection();
+ UNIT_ASSERT_VALUES_EQUAL(res.meta().id(), conId);
+ UNIT_ASSERT_VALUES_EQUAL(res.meta().created_by(), "root@builtin");
+ UNIT_ASSERT_VALUES_EQUAL(res.meta().modified_by(), "root@builtin");
+ UNIT_ASSERT_VALUES_EQUAL(res.content().name(), "created_conn");
+ }
+ }
+
+ Y_UNIT_TEST(ListConnections) {
TKikimrWithGrpcAndRootSchema server({}, {}, {}, true);
- ui16 grpc = server.GetPort();
- TString location = TStringBuilder() << "localhost:" << grpc;
- auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken("root@builtin"));
- NYdb::NYq::TClient client(driver);
+ ui16 grpc = server.GetPort();
+ TString location = TStringBuilder() << "localhost:" << grpc;
+ auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken("root@builtin"));
+ NYdb::NYq::TClient client(driver);
const size_t conns = 3;
- const auto folderId = TString(__func__) + "folder_id";
- {//CreateConnections
+ const auto folderId = TString(__func__) + "folder_id";
+ {//CreateConnections
for (size_t i = 0; i < conns - 1; ++i) {
- const auto request = ::NYq::TCreateConnectionBuilder()
- .SetName("testdb" + ToString(i))
- .CreateYdb("FakeDatabaseId", "")
- .Build();
- const auto result = client
+ const auto request = ::NYq::TCreateConnectionBuilder()
+ .SetName("testdb" + ToString(i))
+ .CreateYdb("FakeDatabaseId", "")
+ .Build();
+ const auto result = client
.CreateConnection(request, CreateYqSettings<TCreateConnectionSettings>(folderId))
- .ExtractValueSync();
- UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString());
- }
+ .ExtractValueSync();
+ UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString());
+ }
// yds
const auto request = ::NYq::TCreateConnectionBuilder()
@@ -403,391 +403,391 @@ Y_UNIT_TEST_SUITE(Yq_1) {
.CreateConnection(request, CreateYqSettings<TCreateConnectionSettings>(folderId))
.ExtractValueSync();
UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString());
- }
-
+ }
+
{
- const auto request = ::NYq::TListConnectionsBuilder().Build();
- auto result = client
+ const auto request = ::NYq::TListConnectionsBuilder().Build();
+ auto result = client
.ListConnections(request, CreateYqSettings<TListConnectionsSettings>(folderId))
- .ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
- UNIT_ASSERT_VALUES_EQUAL(result.GetResult().connection().size(), conns);
- size_t i = 0;
- auto res = result.GetResult();
- auto* conns = res.mutable_connection();
- std::sort(conns->begin(), conns->end(), [&](const auto& lhs, const auto& rhs) {
- return lhs.content().name() < rhs.content().name();
- });
- for (const auto& conn : *conns) {
- const auto& content = conn.content();
- const auto& meta = conn.meta();
- UNIT_ASSERT_VALUES_EQUAL(content.name(), "testdb" + ToString(i));
- UNIT_ASSERT_VALUES_EQUAL(meta.created_by(), "root@builtin");
- UNIT_ASSERT_VALUES_EQUAL(meta.modified_by(), "root@builtin");
+ .ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
+ UNIT_ASSERT_VALUES_EQUAL(result.GetResult().connection().size(), conns);
+ size_t i = 0;
+ auto res = result.GetResult();
+ auto* conns = res.mutable_connection();
+ std::sort(conns->begin(), conns->end(), [&](const auto& lhs, const auto& rhs) {
+ return lhs.content().name() < rhs.content().name();
+ });
+ for (const auto& conn : *conns) {
+ const auto& content = conn.content();
+ const auto& meta = conn.meta();
+ UNIT_ASSERT_VALUES_EQUAL(content.name(), "testdb" + ToString(i));
+ UNIT_ASSERT_VALUES_EQUAL(meta.created_by(), "root@builtin");
+ UNIT_ASSERT_VALUES_EQUAL(meta.modified_by(), "root@builtin");
if (i < 2) {
UNIT_ASSERT_C(content.setting().has_ydb_database(), content);
} else {
UNIT_ASSERT_C(content.setting().has_data_streams(), content);
}
- i++;
- }
+ i++;
+ }
}
}
-
- Y_UNIT_TEST(ListConnectionsOnEmptyConnectionsTable) {
+
+ Y_UNIT_TEST(ListConnectionsOnEmptyConnectionsTable) {
TKikimrWithGrpcAndRootSchema server({}, {}, {}, true);
- ui16 grpc = server.GetPort();
- TString location = TStringBuilder() << "localhost:" << grpc;
- auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken("root@builtin"));
- NYdb::NYq::TClient client(driver);
-
- {
- const auto request = ::NYq::TListConnectionsBuilder().Build();
- auto result = client
+ ui16 grpc = server.GetPort();
+ TString location = TStringBuilder() << "localhost:" << grpc;
+ auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken("root@builtin"));
+ NYdb::NYq::TClient client(driver);
+
+ {
+ const auto request = ::NYq::TListConnectionsBuilder().Build();
+ auto result = client
.ListConnections(request, CreateYqSettings<TListConnectionsSettings>("WTF"))
- .ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
- UNIT_ASSERT(result.GetResult().connection().empty());
- }
- }
-
- Y_UNIT_TEST(ModifyConnections) {
+ .ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
+ UNIT_ASSERT(result.GetResult().connection().empty());
+ }
+ }
+
+ Y_UNIT_TEST(ModifyConnections) {
TKikimrWithGrpcAndRootSchema server({}, {}, {}, true);
- ui16 grpc = server.GetPort();
- TString location = TStringBuilder() << "localhost:" << grpc;
- TString userToken = "root@builtin";
- TString userId = "root";
- auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken(userToken));
- NYdb::NYq::TClient client(driver);
- const auto folderId = TString(__func__) + "folder_id";
- TString conId;
-
- {
- const auto request = ::NYq::TCreateConnectionBuilder()
- .SetName("created_conn")
- .CreateYdb("created_db", "")
- .Build();
- const auto result = client
+ ui16 grpc = server.GetPort();
+ TString location = TStringBuilder() << "localhost:" << grpc;
+ TString userToken = "root@builtin";
+ TString userId = "root";
+ auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken(userToken));
+ NYdb::NYq::TClient client(driver);
+ const auto folderId = TString(__func__) + "folder_id";
+ TString conId;
+
+ {
+ const auto request = ::NYq::TCreateConnectionBuilder()
+ .SetName("created_conn")
+ .CreateYdb("created_db", "")
+ .Build();
+ const auto result = client
.CreateConnection(request, CreateYqSettings<TCreateConnectionSettings>(folderId))
- .ExtractValueSync();
- UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString());
- conId = result.GetResult().connection_id();
- }
-
- {//Modify
- const auto request = ::NYq::TModifyConnectionBuilder()
- .SetName("modified_name")
- .SetConnectionId(conId)
- .CreateYdb("new ydb", "")
- .SetDescription("Modified")
- .Build();
- const auto result = client
+ .ExtractValueSync();
+ UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString());
+ conId = result.GetResult().connection_id();
+ }
+
+ {//Modify
+ const auto request = ::NYq::TModifyConnectionBuilder()
+ .SetName("modified_name")
+ .SetConnectionId(conId)
+ .CreateYdb("new ydb", "")
+ .SetDescription("Modified")
+ .Build();
+ const auto result = client
.ModifyConnection(request, CreateYqSettings<TModifyConnectionSettings>(folderId))
- .ExtractValueSync();
- UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString());
- }
-
- {
- auto request = ::NYq::TDescribeConnectionBuilder()
- .SetConnectionId(conId)
- .Build();
- auto result = client
+ .ExtractValueSync();
+ UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString());
+ }
+
+ {
+ auto request = ::NYq::TDescribeConnectionBuilder()
+ .SetConnectionId(conId)
+ .Build();
+ auto result = client
.DescribeConnection(request, CreateYqSettings<TDescribeConnectionSettings>(folderId))
- .ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
- const auto& res = result.GetResult().connection();
- UNIT_ASSERT_VALUES_EQUAL(res.meta().id(), conId);
- UNIT_ASSERT_VALUES_EQUAL(res.meta().created_by(), "root@builtin");
- UNIT_ASSERT_VALUES_EQUAL(res.meta().modified_by(), "root@builtin");
- UNIT_ASSERT_VALUES_EQUAL(res.content().name(), "modified_name");
- UNIT_ASSERT_VALUES_EQUAL(res.content().description(), "Modified");
- }
- }
-
- Y_UNIT_TEST(DeleteConnections) {
+ .ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
+ const auto& res = result.GetResult().connection();
+ UNIT_ASSERT_VALUES_EQUAL(res.meta().id(), conId);
+ UNIT_ASSERT_VALUES_EQUAL(res.meta().created_by(), "root@builtin");
+ UNIT_ASSERT_VALUES_EQUAL(res.meta().modified_by(), "root@builtin");
+ UNIT_ASSERT_VALUES_EQUAL(res.content().name(), "modified_name");
+ UNIT_ASSERT_VALUES_EQUAL(res.content().description(), "Modified");
+ }
+ }
+
+ Y_UNIT_TEST(DeleteConnections) {
TKikimrWithGrpcAndRootSchema server({}, {}, {}, true);
- ui16 grpc = server.GetPort();
- TString location = TStringBuilder() << "localhost:" << grpc;
- auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken("root@builtin"));
- NYdb::NYq::TClient client(driver);
- const auto folderId = TString(__func__) + "folder_id";
- TString conId;
-
- {
- const auto request = ::NYq::TCreateConnectionBuilder()
- .SetName("created_conn")
- .CreateYdb("created_db", "")
- .Build();
- const auto result = client
+ ui16 grpc = server.GetPort();
+ TString location = TStringBuilder() << "localhost:" << grpc;
+ auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken("root@builtin"));
+ NYdb::NYq::TClient client(driver);
+ const auto folderId = TString(__func__) + "folder_id";
+ TString conId;
+
+ {
+ const auto request = ::NYq::TCreateConnectionBuilder()
+ .SetName("created_conn")
+ .CreateYdb("created_db", "")
+ .Build();
+ const auto result = client
.CreateConnection(request, CreateYqSettings<TCreateConnectionSettings>(folderId))
- .ExtractValueSync();
- UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString());
- conId = result.GetResult().connection_id();
- }
-
- {
- const auto request = ::NYq::TDeleteConnectionBuilder()
- .SetConnectionId(conId)
- .Build();
-
- const auto result = client
+ .ExtractValueSync();
+ UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString());
+ conId = result.GetResult().connection_id();
+ }
+
+ {
+ const auto request = ::NYq::TDeleteConnectionBuilder()
+ .SetConnectionId(conId)
+ .Build();
+
+ const auto result = client
.DeleteConnection(request, CreateYqSettings<TDeleteConnectionSettings>(folderId))
- .ExtractValueSync();
- UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString());
- }
- }
-
- Y_UNIT_TEST(Create_And_Modify_The_Same_Connection) {
+ .ExtractValueSync();
+ UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString());
+ }
+ }
+
+ Y_UNIT_TEST(Create_And_Modify_The_Same_Connection) {
TKikimrWithGrpcAndRootSchema server({}, {}, {}, true);
- ui16 grpc = server.GetPort();
- TString location = TStringBuilder() << "localhost:" << grpc;
- TString userToken = "root@builtin";
+ ui16 grpc = server.GetPort();
+ TString location = TStringBuilder() << "localhost:" << grpc;
+ TString userToken = "root@builtin";
auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken(userToken));
- NYdb::NYq::TClient client(driver);
+ NYdb::NYq::TClient client(driver);
- const auto folderId = TString(__func__) + "folder_id";
- TString conId;
+ const auto folderId = TString(__func__) + "folder_id";
+ TString conId;
- {
- const auto request = ::NYq::TCreateConnectionBuilder()
- .SetName("created_conn")
- .CreateYdb("created_db", "")
- .Build();
- const auto result = client
+ {
+ const auto request = ::NYq::TCreateConnectionBuilder()
+ .SetName("created_conn")
+ .CreateYdb("created_db", "")
+ .Build();
+ const auto result = client
.CreateConnection(request, CreateYqSettings<TCreateConnectionSettings>(folderId))
- .ExtractValueSync();
- UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString());
- conId = result.GetResult().connection_id();
- }
-
- {
- const auto request = ::NYq::TModifyConnectionBuilder()
- .SetConnectionId(conId)
- .CreateYdb("modified_db", "")//TODO remove
- .Build();
- const auto result = client
+ .ExtractValueSync();
+ UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString());
+ conId = result.GetResult().connection_id();
+ }
+
+ {
+ const auto request = ::NYq::TModifyConnectionBuilder()
+ .SetConnectionId(conId)
+ .CreateYdb("modified_db", "")//TODO remove
+ .Build();
+ const auto result = client
.ModifyConnection(request, CreateYqSettings<TModifyConnectionSettings>(folderId))
- .ExtractValueSync();
- UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString());
- }
- }
-
- Y_UNIT_TEST(CreateConnection_With_Existing_Name) {
+ .ExtractValueSync();
+ UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString());
+ }
+ }
+
+ Y_UNIT_TEST(CreateConnection_With_Existing_Name) {
TKikimrWithGrpcAndRootSchema server({}, {}, {}, true);
- ui16 grpc = server.GetPort();
- TString location = TStringBuilder() << "localhost:" << grpc;
- TString userToken = "root@builtin";
- auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken(userToken));
- NYdb::NYq::TClient client(driver);
-
- const auto folderId = TString(__func__) + "folder_id";
+ ui16 grpc = server.GetPort();
+ TString location = TStringBuilder() << "localhost:" << grpc;
+ TString userToken = "root@builtin";
+ auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken(userToken));
+ NYdb::NYq::TClient client(driver);
+
+ const auto folderId = TString(__func__) + "folder_id";
auto name = TString(__func__) + "_name";
name.to_lower();
-
- {
- const auto request = ::NYq::TCreateConnectionBuilder()
- .SetName(name)
- .CreateYdb("created_db", "")
- .Build();
- const auto result = client
+
+ {
+ const auto request = ::NYq::TCreateConnectionBuilder()
+ .SetName(name)
+ .CreateYdb("created_db", "")
+ .Build();
+ const auto result = client
.CreateConnection(request, CreateYqSettings<TCreateConnectionSettings>(folderId))
- .ExtractValueSync();
- UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString());
- }
-
- {
- const auto request = ::NYq::TCreateConnectionBuilder()
- .SetName(name)
- .CreateYdb("created_db", "")
- .Build();
- const auto result = client
+ .ExtractValueSync();
+ UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString());
+ }
+
+ {
+ const auto request = ::NYq::TCreateConnectionBuilder()
+ .SetName(name)
+ .CreateYdb("created_db", "")
+ .Build();
+ const auto result = client
.CreateConnection(request, CreateYqSettings<TCreateConnectionSettings>(folderId))
- .ExtractValueSync();
- UNIT_ASSERT_C(result.GetStatus() == EStatus::BAD_REQUEST, result.GetIssues().ToString()); //TODO status should be ALREADY_EXISTS
- }
- }
-
- Y_UNIT_TEST(CreateConnections_With_Idempotency) {
+ .ExtractValueSync();
+ UNIT_ASSERT_C(result.GetStatus() == EStatus::BAD_REQUEST, result.GetIssues().ToString()); //TODO status should be ALREADY_EXISTS
+ }
+ }
+
+ Y_UNIT_TEST(CreateConnections_With_Idempotency) {
TKikimrWithGrpcAndRootSchema server({}, {}, {}, true);
- ui16 grpc = server.GetPort();
- TString location = TStringBuilder() << "localhost:" << grpc;
- TString userToken = "root@builtin";
- auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken(userToken));
- NYdb::NYq::TClient client(driver);
-
- const auto folderId = TString(__func__) + "folder_id";
- const auto name = "connection_name";
- const TString idempotencyKey = "idempotency_key";
- TString conId;
-
- {
- const auto request = ::NYq::TCreateConnectionBuilder()
- .SetName(name)
- .SetIdempotencyKey(idempotencyKey)
- .CreateYdb("created_db", "")
- .Build();
- const auto result = client
+ ui16 grpc = server.GetPort();
+ TString location = TStringBuilder() << "localhost:" << grpc;
+ TString userToken = "root@builtin";
+ auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken(userToken));
+ NYdb::NYq::TClient client(driver);
+
+ const auto folderId = TString(__func__) + "folder_id";
+ const auto name = "connection_name";
+ const TString idempotencyKey = "idempotency_key";
+ TString conId;
+
+ {
+ const auto request = ::NYq::TCreateConnectionBuilder()
+ .SetName(name)
+ .SetIdempotencyKey(idempotencyKey)
+ .CreateYdb("created_db", "")
+ .Build();
+ const auto result = client
.CreateConnection(request, CreateYqSettings<TCreateConnectionSettings>(folderId))
- .ExtractValueSync();
- UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString());
- conId = result.GetResult().connection_id();
- }
-
- {
- const auto request = ::NYq::TCreateConnectionBuilder()
- .SetName(name)
- .SetIdempotencyKey(idempotencyKey)
- .CreateYdb("created_db", "")
- .Build();
- const auto result = client
+ .ExtractValueSync();
+ UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString());
+ conId = result.GetResult().connection_id();
+ }
+
+ {
+ const auto request = ::NYq::TCreateConnectionBuilder()
+ .SetName(name)
+ .SetIdempotencyKey(idempotencyKey)
+ .CreateYdb("created_db", "")
+ .Build();
+ const auto result = client
.CreateConnection(request, CreateYqSettings<TCreateConnectionSettings>(folderId))
- .ExtractValueSync();
- UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString());
- UNIT_ASSERT_VALUES_EQUAL(conId, result.GetResult().connection_id());
- }
- }
-
- Y_UNIT_TEST(CreateQuery_With_Idempotency) {//TODO Fix
+ .ExtractValueSync();
+ UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString());
+ UNIT_ASSERT_VALUES_EQUAL(conId, result.GetResult().connection_id());
+ }
+ }
+
+ Y_UNIT_TEST(CreateQuery_With_Idempotency) {//TODO Fix
TKikimrWithGrpcAndRootSchema server({}, {}, {}, true);
- ui16 grpc = server.GetPort();
+ ui16 grpc = server.GetPort();
TString location = TStringBuilder() << "localhost:" << grpc;
- TString userToken = "root@builtin";
+ TString userToken = "root@builtin";
auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken(userToken));
- NYdb::NYq::TClient client(driver);
+ NYdb::NYq::TClient client(driver);
- const auto folderId = TString(__func__) + "folder_id";
+ const auto folderId = TString(__func__) + "folder_id";
const TString idempotencyKey = "idempotency_key";
- const TString yqlText = "select 1";
- TString queryId;
- const auto request = ::NYq::TCreateQueryBuilder{}
- .SetText(yqlText)
- .SetIdempotencyKey(idempotencyKey)
- .Build();
+ const TString yqlText = "select 1";
+ TString queryId;
+ const auto request = ::NYq::TCreateQueryBuilder{}
+ .SetText(yqlText)
+ .SetIdempotencyKey(idempotencyKey)
+ .Build();
{
- auto result = client.CreateQuery(
+ auto result = client.CreateQuery(
request, CreateYqSettings<TCreateQuerySettings>(folderId))
- .ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
- queryId = result.GetResult().query_id();
+ .ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
+ queryId = result.GetResult().query_id();
}
{
- const auto req = ::NYq::TDescribeQueryBuilder{}
- .SetQueryId(queryId)
- .Build();
- const auto result = DoWithRetryOnRetCode([&]() {
- auto result = client.DescribeQuery(
+ const auto req = ::NYq::TDescribeQueryBuilder{}
+ .SetQueryId(queryId)
+ .Build();
+ const auto result = DoWithRetryOnRetCode([&]() {
+ auto result = client.DescribeQuery(
req, CreateYqSettings<TDescribeQuerySettings>(folderId))
- .ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
- const auto status = result.GetResult().query().meta().status();
- PrintProtoIssues(result.GetResult().query().issue());
- return status == YandexQuery::QueryMeta::COMPLETED;
- }, TRetryOptions(Retries));
- UNIT_ASSERT_C(result, "the execution of the query did not end within the time limit");
+ .ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
+ const auto status = result.GetResult().query().meta().status();
+ PrintProtoIssues(result.GetResult().query().issue());
+ return status == YandexQuery::QueryMeta::COMPLETED;
+ }, TRetryOptions(Retries));
+ UNIT_ASSERT_C(result, "the execution of the query did not end within the time limit");
}
{
- auto result = client.CreateQuery(
+ auto result = client.CreateQuery(
request, CreateYqSettings<TCreateQuerySettings>(folderId))
- .ExtractValueSync();
+ .ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
- UNIT_ASSERT_VALUES_EQUAL(queryId, result.GetResult().query_id());
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
+ UNIT_ASSERT_VALUES_EQUAL(queryId, result.GetResult().query_id());
}
CheckGetResultData(client, queryId, folderId, 1, 1, 1);
}
// use fork for data test due to ch initialization problem
- SIMPLE_UNIT_FORKED_TEST(CreateQuery_Without_Connection) {
+ SIMPLE_UNIT_FORKED_TEST(CreateQuery_Without_Connection) {
TKikimrWithGrpcAndRootSchema server({}, {}, {}, true);
- ui16 grpc = server.GetPort();
- TString location = TStringBuilder() << "localhost:" << grpc;
- TString userToken = "root@builtin";
- auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken(userToken));
- NYdb::NYq::TClient client(driver);
-
- const TString yqlText = "select count(*) from testdbWTF.`connections`";
+ ui16 grpc = server.GetPort();
+ TString location = TStringBuilder() << "localhost:" << grpc;
+ TString userToken = "root@builtin";
+ auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken(userToken));
+ NYdb::NYq::TClient client(driver);
+
+ const TString yqlText = "select count(*) from testdbWTF.`connections`";
CreateNewHistoryAndWaitFinish("folder_id_WTF", client,
- yqlText, YandexQuery::QueryMeta::FAILED);
- }
-
- Y_UNIT_TEST(DeleteQuery) {
+ yqlText, YandexQuery::QueryMeta::FAILED);
+ }
+
+ Y_UNIT_TEST(DeleteQuery) {
TKikimrWithGrpcAndRootSchema server({}, {}, {}, true);
- ui16 grpc = server.GetPort();
- TString location = TStringBuilder() << "localhost:" << grpc;
- TString userToken = "root@builtin";
- auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken(userToken));
- NYdb::NYq::TClient client(driver);
-
- const auto folderId = TString(__func__) + "folder_id";
- const TString yqlText = "select 1";
+ ui16 grpc = server.GetPort();
+ TString location = TStringBuilder() << "localhost:" << grpc;
+ TString userToken = "root@builtin";
+ auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken(userToken));
+ NYdb::NYq::TClient client(driver);
+
+ const auto folderId = TString(__func__) + "folder_id";
+ const TString yqlText = "select 1";
const TString queryId = CreateNewHistoryAndWaitFinish(folderId, client,
- yqlText, YandexQuery::QueryMeta::COMPLETED);
+ yqlText, YandexQuery::QueryMeta::COMPLETED);
CheckGetResultData(client, queryId, folderId, 1, 1, 1);
-
- {
- const auto request = ::NYq::TDeleteQueryBuilder()
- .SetQueryId(queryId)
- .Build();
- auto result = client
+
+ {
+ const auto request = ::NYq::TDeleteQueryBuilder()
+ .SetQueryId(queryId)
+ .Build();
+ auto result = client
.DeleteQuery(request, CreateYqSettings<TDeleteQuerySettings>(folderId))
- .ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
- }
-
- {
- auto request = ::NYq::TDescribeQueryBuilder()
- .SetQueryId(queryId)
- .Build();
- auto result = client
+ .ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
+ }
+
+ {
+ auto request = ::NYq::TDescribeQueryBuilder()
+ .SetQueryId(queryId)
+ .Build();
+ auto result = client
.DescribeQuery(request, CreateYqSettings<TDescribeQuerySettings>(folderId))
- .ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::BAD_REQUEST, result.GetIssues().ToString());
- }
- }
-
- Y_UNIT_TEST(ModifyQuery) {
+ .ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::BAD_REQUEST, result.GetIssues().ToString());
+ }
+ }
+
+ Y_UNIT_TEST(ModifyQuery) {
TKikimrWithGrpcAndRootSchema server({}, {}, {}, true);
- ui16 grpc = server.GetPort();
- TString location = TStringBuilder() << "localhost:" << grpc;
- TString userToken = "root@builtin";
- auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken(userToken));
- NYdb::NYq::TClient client(driver);
-
- const auto folderId = TString(__func__) + "folder_id";
- const TString yqlText = "select 1";
+ ui16 grpc = server.GetPort();
+ TString location = TStringBuilder() << "localhost:" << grpc;
+ TString userToken = "root@builtin";
+ auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken(userToken));
+ NYdb::NYq::TClient client(driver);
+
+ const auto folderId = TString(__func__) + "folder_id";
+ const TString yqlText = "select 1";
const TString queryId = CreateNewHistoryAndWaitFinish(folderId, client,
- yqlText, YandexQuery::QueryMeta::COMPLETED);
+ yqlText, YandexQuery::QueryMeta::COMPLETED);
CheckGetResultData(client, queryId, folderId, 1, 1, 1);
-
- {
- const auto request = ::NYq::TModifyQueryBuilder()
- .SetQueryId(queryId)
- .SetName("ModifiedName")
- .SetDescription("OK")
- .Build();
- auto result = client
+
+ {
+ const auto request = ::NYq::TModifyQueryBuilder()
+ .SetQueryId(queryId)
+ .SetName("ModifiedName")
+ .SetDescription("OK")
+ .Build();
+ auto result = client
.ModifyQuery(request, CreateYqSettings<TModifyQuerySettings>(folderId))
- .ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
- }
-
- {
- auto request = ::NYq::TDescribeQueryBuilder()
- .SetQueryId(queryId)
- .Build();
- auto result = client
+ .ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
+ }
+
+ {
+ auto request = ::NYq::TDescribeQueryBuilder()
+ .SetQueryId(queryId)
+ .Build();
+ auto result = client
.DescribeQuery(request, CreateYqSettings<TDescribeQuerySettings>(folderId))
- .ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
- const auto& query = result.GetResult().query();
- UNIT_ASSERT_VALUES_EQUAL(query.content().name(), "ModifiedName");
- UNIT_ASSERT_VALUES_EQUAL(query.content().description(), "OK");
- }
- }
+ .ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
+ const auto& query = result.GetResult().query();
+ UNIT_ASSERT_VALUES_EQUAL(query.content().name(), "ModifiedName");
+ UNIT_ASSERT_VALUES_EQUAL(query.content().description(), "OK");
+ }
+ }
Y_UNIT_TEST(DescribeJob) {
TKikimrWithGrpcAndRootSchema server({}, {}, {}, true);
@@ -822,7 +822,7 @@ Y_UNIT_TEST_SUITE(Yq_1) {
request, CreateYqSettings<TDescribeJobSettings>(folderId))
.ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
- UNIT_ASSERT_VALUES_EQUAL(result.GetResult().job().query_meta().common().id(), queryId);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetResult().job().query_meta().common().id(), queryId);
UNIT_ASSERT_VALUES_EQUAL(result.GetResult().job().meta().id(), jobId);
UNIT_ASSERT_VALUES_EQUAL(result.GetResult().job().query_name(), "test_query_name_1");
}
@@ -854,15 +854,15 @@ Y_UNIT_TEST_SUITE(Yq_1) {
}
}
}
-
+
Y_UNIT_TEST_SUITE(Yq_2) {
// use fork for data test due to ch initialization problem
- Y_UNIT_TEST(Test_HostNameTrasformation) {
- UNIT_ASSERT_VALUES_EQUAL(::NYq::TransformMdbHostToCorrectFormat("rc1c-p5waby2y5y1kb5ue.mdb.yandexcloud.net"), "rc1c-p5waby2y5y1kb5ue.db.yandex.net:8443");
- UNIT_ASSERT_VALUES_EQUAL(::NYq::TransformMdbHostToCorrectFormat("xxx.xxx"), "xxx.db.yandex.net:8443");
- UNIT_ASSERT_VALUES_EQUAL(::NYq::TransformMdbHostToCorrectFormat("host."), "host.db.yandex.net:8443");
- }
-
+ Y_UNIT_TEST(Test_HostNameTrasformation) {
+ UNIT_ASSERT_VALUES_EQUAL(::NYq::TransformMdbHostToCorrectFormat("rc1c-p5waby2y5y1kb5ue.mdb.yandexcloud.net"), "rc1c-p5waby2y5y1kb5ue.db.yandex.net:8443");
+ UNIT_ASSERT_VALUES_EQUAL(::NYq::TransformMdbHostToCorrectFormat("xxx.xxx"), "xxx.db.yandex.net:8443");
+ UNIT_ASSERT_VALUES_EQUAL(::NYq::TransformMdbHostToCorrectFormat("host."), "host.db.yandex.net:8443");
+ }
+
SIMPLE_UNIT_FORKED_TEST(ReadFromYdbOverYq) {
TKikimrWithGrpcAndRootSchema server({}, {}, {}, true);
ui16 grpc = server.GetPort();
@@ -883,11 +883,11 @@ Y_UNIT_TEST_SUITE(Yq_2) {
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
}
-
+
TString queryId;
{
auto request = ::NYq::TCreateQueryBuilder{}
- .SetText("select count(*) from testdb00.`yq/connections`")
+ .SetText("select count(*) from testdb00.`yq/connections`")
.Build();
auto result = client.CreateQuery(
request, CreateYqSettings<TCreateQuerySettings>(folderId))
@@ -905,7 +905,7 @@ Y_UNIT_TEST_SUITE(Yq_2) {
.ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
const auto status = result.GetResult().query().meta().status();
- PrintProtoIssues(result.GetResult().query().issue());
+ PrintProtoIssues(result.GetResult().query().issue());
return status == YandexQuery::QueryMeta::COMPLETED;
}, TRetryOptions(10));
UNIT_ASSERT_C(result, "the execution of the query did not end within the time limit");
@@ -921,78 +921,78 @@ Y_UNIT_TEST_SUITE(Yq_2) {
const auto& resultSet = result.GetResult().result_set();
UNIT_ASSERT_VALUES_EQUAL(resultSet.rows().size(), 1);
UNIT_ASSERT_VALUES_EQUAL(resultSet.columns().size(), 1);
- UNIT_ASSERT_VALUES_EQUAL(resultSet.rows(0).items(0).uint64_value(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(resultSet.rows(0).items(0).uint64_value(), 1);
}
}
}
-
-Y_UNIT_TEST_SUITE(PrivateApi) {
- Y_UNIT_TEST(PingTask) {
+
+Y_UNIT_TEST_SUITE(PrivateApi) {
+ Y_UNIT_TEST(PingTask) {
TKikimrWithGrpcAndRootSchema server({}, {}, {}, true);
- ui16 grpc = server.GetPort();
- TString location = TStringBuilder() << "localhost:" << grpc;
- auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken("root@builtin"));
- ::NYq::TPrivateClient client(driver);
- const TString historyId = "id";
- const TString folderId = "folder_id";
+ ui16 grpc = server.GetPort();
+ TString location = TStringBuilder() << "localhost:" << grpc;
+ auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken("root@builtin"));
+ ::NYq::TPrivateClient client(driver);
+ const TString historyId = "id";
+ const TString folderId = "folder_id";
const TScope scope(folderId);
- {
- Yq::Private::PingTaskRequest req;
- req.mutable_query_id()->set_value("id");
- req.set_scope(scope.ToString());
- req.set_owner_id("some_owner");
- req.set_status(YandexQuery::QueryMeta::COMPLETED);
- auto result = client.PingTask(std::move(req)).ExtractValueSync();
- result.GetIssues().PrintTo(Cerr);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::GENERIC_ERROR);
- }
- }
-
- Y_UNIT_TEST(GetTask) {//PendingFetcher can take task first
+ {
+ Yq::Private::PingTaskRequest req;
+ req.mutable_query_id()->set_value("id");
+ req.set_scope(scope.ToString());
+ req.set_owner_id("some_owner");
+ req.set_status(YandexQuery::QueryMeta::COMPLETED);
+ auto result = client.PingTask(std::move(req)).ExtractValueSync();
+ result.GetIssues().PrintTo(Cerr);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::GENERIC_ERROR);
+ }
+ }
+
+ Y_UNIT_TEST(GetTask) {//PendingFetcher can take task first
TKikimrWithGrpcAndRootSchema server({}, {}, {}, true);
- ui16 grpc = server.GetPort();
- TString location = TStringBuilder() << "localhost:" << grpc;
- auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken("root@builtin"));
- ::NYq::TPrivateClient client(driver);
- {
- Yq::Private::GetTaskRequest req;
- req.set_owner_id("owner_id");
- req.set_host("host");
- auto result = client.GetTask(std::move(req)).ExtractValueSync();
- result.GetIssues().PrintTo(Cerr);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- result.GetIssues().PrintTo(Cerr);
- }
- }
-
- Y_UNIT_TEST(Nodes) {
+ ui16 grpc = server.GetPort();
+ TString location = TStringBuilder() << "localhost:" << grpc;
+ auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken("root@builtin"));
+ ::NYq::TPrivateClient client(driver);
+ {
+ Yq::Private::GetTaskRequest req;
+ req.set_owner_id("owner_id");
+ req.set_host("host");
+ auto result = client.GetTask(std::move(req)).ExtractValueSync();
+ result.GetIssues().PrintTo(Cerr);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ result.GetIssues().PrintTo(Cerr);
+ }
+ }
+
+ Y_UNIT_TEST(Nodes) {
TKikimrWithGrpcAndRootSchema server({}, {}, {}, true);
- ui16 grpc = server.GetPort();
- TString location = TStringBuilder() << "localhost:" << grpc;
- auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken("root@builtin"));
- ::NYq::TPrivateClient client(driver);
- const auto instanceId = CreateGuidAsString();
- {
- Yq::Private::NodesHealthCheckRequest req;
- req.set_tenant("Tenant");
+ ui16 grpc = server.GetPort();
+ TString location = TStringBuilder() << "localhost:" << grpc;
+ auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken("root@builtin"));
+ ::NYq::TPrivateClient client(driver);
+ const auto instanceId = CreateGuidAsString();
+ {
+ Yq::Private::NodesHealthCheckRequest req;
+ req.set_tenant("Tenant");
auto& node = *req.mutable_node();
node.set_hostname("hostname");
node.set_node_id(100500);
node.set_instance_id(instanceId);
- const auto result = DoWithRetryOnRetCode([&]() {
- auto r = req;
- auto result = client.NodesHealthCheck(std::move(r)).ExtractValueSync();
- if (result.GetStatus() == EStatus::SUCCESS) {
- const auto& res = result.GetResult();
- UNIT_ASSERT(!res.nodes().empty());
- UNIT_ASSERT_VALUES_EQUAL(res.nodes(0).hostname(), "hostname");
- UNIT_ASSERT_VALUES_EQUAL(res.nodes(0).node_id(), 100500);
- UNIT_ASSERT_VALUES_EQUAL(res.nodes(0).instance_id(), instanceId);
- }
- // result.GetIssues().PrintTo(Cerr);
- return result.GetStatus() == EStatus::SUCCESS;
- }, TRetryOptions(Retries));
- UNIT_ASSERT_C(result, "the execution of the query did not end within the time limit");
- }
- }
+ const auto result = DoWithRetryOnRetCode([&]() {
+ auto r = req;
+ auto result = client.NodesHealthCheck(std::move(r)).ExtractValueSync();
+ if (result.GetStatus() == EStatus::SUCCESS) {
+ const auto& res = result.GetResult();
+ UNIT_ASSERT(!res.nodes().empty());
+ UNIT_ASSERT_VALUES_EQUAL(res.nodes(0).hostname(), "hostname");
+ UNIT_ASSERT_VALUES_EQUAL(res.nodes(0).node_id(), 100500);
+ UNIT_ASSERT_VALUES_EQUAL(res.nodes(0).instance_id(), instanceId);
+ }
+ // result.GetIssues().PrintTo(Cerr);
+ return result.GetStatus() == EStatus::SUCCESS;
+ }, TRetryOptions(Retries));
+ UNIT_ASSERT_C(result, "the execution of the query did not end within the time limit");
+ }
+ }
}
diff --git a/ydb/services/yq/ya.make b/ydb/services/yq/ya.make
index 92b1eb8f8b..905c48a496 100644
--- a/ydb/services/yq/ya.make
+++ b/ydb/services/yq/ya.make
@@ -7,12 +7,12 @@ OWNER(
SRCS(
grpc_service.cpp
- private_grpc.cpp
+ private_grpc.cpp
)
PEERDIR(
library/cpp/grpc/server
- library/cpp/retry
+ library/cpp/retry
ydb/core/grpc_services
ydb/core/grpc_services/base
ydb/library/protobuf_printer