diff options
author | dinmukhammed <dinmukhammed@yandex-team.ru> | 2022-02-10 16:51:14 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:51:14 +0300 |
commit | f4f0dcffead22e9e9feebedc03057853a44ccc75 (patch) | |
tree | 5d5cb817648f650d76cf1076100726fd9b8448e8 | |
parent | 8435bcab44202d0f1e0eacf7a0535e238b995b09 (diff) | |
download | ydb-f4f0dcffead22e9e9feebedc03057853a44ccc75.tar.gz |
Restoring authorship annotation for <dinmukhammed@yandex-team.ru>. Commit 2 of 2.
184 files changed, 6410 insertions, 6410 deletions
diff --git a/build/rules/kikimr.policy b/build/rules/kikimr.policy index 6cb346dc9b..5bb6426e30 100644 --- a/build/rules/kikimr.policy +++ b/build/rules/kikimr.policy @@ -146,10 +146,10 @@ ALLOW ydb/library/yql/providers/dq/worker_manager -> kikimr/core/kqp ALLOW ydb/library/yql/providers/clickhouse/provider -> kikimr/yq/libs/db_resolver ALLOW ydb/library/yql/providers/clickhouse/provider -> kikimr/yq/libs/common - + ALLOW ydb/library/yql/providers/pq/provider -> kikimr/yq/libs/db_resolver ALLOW ydb/library/yql/providers/pq/provider -> kikimr/yq/libs/common - + ALLOW yql/tools/yqlworker/dq/worker_manager -> kikimr/core/kqp ALLOW ydb/library/yql/dq/actors/compute -> kikimr/core/kqp/runtime diff --git a/ydb/core/base/events.h b/ydb/core/base/events.h index 10e6cfd323..f5fedfe19b 100644 --- a/ydb/core/base/events.h +++ b/ydb/core/base/events.h @@ -6,7 +6,7 @@ #include <ydb/library/yql/dq/actors/dq_events_ids.h> #include <ydb/core/yq/libs/events/event_ids.h> - + namespace NKikimr { struct TKikimrEvents : TEvents { @@ -126,7 +126,7 @@ struct TKikimrEvents : TEvents { ES_TX_COLUMNSHARD, ES_CROSSREF, ES_SCHEME_BOARD_MON, - ES_YQL_ANALYTICS_PROXY = NYq::TEventIds::ES_YQL_ANALYTICS_PROXY, + ES_YQL_ANALYTICS_PROXY = NYq::TEventIds::ES_YQL_ANALYTICS_PROXY, ES_BLOB_CACHE, ES_LONG_TX_SERVICE, ES_TEST_SHARD, diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp index 5fffe37aa4..819c1478d1 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp @@ -190,8 +190,8 @@ namespace NKikimr { namespace NKikimrServicesInitializers { -ui32 TYandexQueryInitializer::IcPort = 0; - +ui32 TYandexQueryInitializer::IcPort = 0; + IKikimrServicesInitializer::IKikimrServicesInitializer(const TKikimrRunConfig& runConfig) : Config(runConfig.AppConfig) , NodeId(runConfig.NodeId) @@ -594,7 +594,7 @@ void TBasicServicesInitializer::InitializeServices(NActors::TActorSystemSetup* s case NKikimrConfig::TStaticNameserviceConfig::NS_EXTERNAL: nameservice = NActors::CreateDynamicNameserver(table, TDuration::Seconds(3), appData->IOPoolId); break; - } + } setup->LocalServices.emplace_back( nameserviceId, @@ -707,12 +707,12 @@ void TBasicServicesInitializer::InitializeServices(NActors::TActorSystemSetup* s setup->Interconnect.ProxyActors.resize(maxNode + 1); setup->Interconnect.ProxyWrapperFactory = CreateProxyWrapperFactory(icCommon, interconnectPoolId); - std::unordered_set<ui32> staticIds; - - for (const auto& node : table->StaticNodeTable) { + std::unordered_set<ui32> staticIds; + + for (const auto& node : table->StaticNodeTable) { const ui32 destId = node.first; if (destId != NodeId) { - staticIds.insert(destId); + staticIds.insert(destId); setup->Interconnect.ProxyActors[destId] = TActorSetupCmd(new TInterconnectProxyTCP(destId, icCommon), TMailboxType::ReadAsFilled, interconnectPoolId); } else { @@ -2257,10 +2257,10 @@ TYandexQueryInitializer::TYandexQueryInitializer(const TKikimrRunConfig& runConf { } -void TYandexQueryInitializer::SetIcPort(ui32 icPort) { - IcPort = icPort; -} - +void TYandexQueryInitializer::SetIcPort(ui32 icPort) { + IcPort = icPort; +} + void TYandexQueryInitializer::InitializeServices(TActorSystemSetup* setup, const TAppData* appData) { const auto& protoConfig = Config.GetYandexQueryConfig(); if (!protoConfig.GetEnabled()) { @@ -2292,8 +2292,8 @@ void TYandexQueryInitializer::InitializeServices(TActorSystemSetup* setup, const YqSharedResources, Factories->FolderServiceFactory, Factories->YqAuditServiceFactory, - Factories->YdbCredentialProviderFactory, - IcPort + Factories->YdbCredentialProviderFactory, + IcPort ); } diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.h b/ydb/core/driver_lib/run/kikimr_services_initializers.h index d2362dc0f0..407ce1bb7b 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.h +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.h @@ -491,12 +491,12 @@ public: TYandexQueryInitializer(const TKikimrRunConfig& runConfig, std::shared_ptr<TModuleFactories> factories, NYq::IYqSharedResources::TPtr yqSharedResources); void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; - - static void SetIcPort(ui32 icPort); + + static void SetIcPort(ui32 icPort); private: std::shared_ptr<TModuleFactories> Factories; NYq::IYqSharedResources::TPtr YqSharedResources; - static ui32 IcPort; + static ui32 IcPort; }; } // namespace NKikimrServicesInitializers diff --git a/ydb/core/driver_lib/run/run.cpp b/ydb/core/driver_lib/run/run.cpp index c2405fc86d..a4f74aa4e0 100644 --- a/ydb/core/driver_lib/run/run.cpp +++ b/ydb/core/driver_lib/run/run.cpp @@ -715,7 +715,7 @@ void TKikimrRunner::InitializeGRpc(const TKikimrRunConfig& runConfig) { if (hasYandexQuery) { server.AddService(new NGRpcService::TGRpcYandexQueryService(ActorSystem.Get(), Counters, grpcRequestProxyId)); - server.AddService(new NGRpcService::TGRpcYqPrivateTaskService(ActorSystem.Get(), Counters, grpcRequestProxyId)); + server.AddService(new NGRpcService::TGRpcYqPrivateTaskService(ActorSystem.Get(), Counters, grpcRequestProxyId)); } if (hasLogStore) { @@ -1316,8 +1316,8 @@ TIntrusivePtr<TServiceInitializersList> TKikimrRunner::CreateServiceInitializers } if (serviceMask.EnableYandexQuery && runConfig.AppConfig.GetYandexQueryConfig().GetEnabled()) { - YqSharedResources = NYq::CreateYqSharedResources(runConfig.AppConfig.GetYandexQueryConfig(), - ModuleFactories->YdbCredentialProviderFactory, Counters->GetSubgroup("counters", "yq")); + YqSharedResources = NYq::CreateYqSharedResources(runConfig.AppConfig.GetYandexQueryConfig(), + ModuleFactories->YdbCredentialProviderFactory, Counters->GetSubgroup("counters", "yq")); sil->AddServiceInitializer(new TYandexQueryInitializer(runConfig, ModuleFactories, YqSharedResources)); } diff --git a/ydb/core/grpc_services/base/base.h b/ydb/core/grpc_services/base/base.h index dc7ca0bfd9..44b25c4a5f 100644 --- a/ydb/core/grpc_services/base/base.h +++ b/ydb/core/grpc_services/base/base.h @@ -198,10 +198,10 @@ struct TRpcServices { EvDropLogTable, EvAlterLogTable, EvLogin, - EvAnalyticsInternalPingTask, - EvAnalyticsInternalGetTask, - EvAnalyticsInternalWriteTaskResult, - EvAnalyticsInternalNodesHealthCheck, + EvAnalyticsInternalPingTask, + EvAnalyticsInternalGetTask, + EvAnalyticsInternalWriteTaskResult, + EvAnalyticsInternalNodesHealthCheck, EvCreateYndxRateLimiterResource, EvAlterYndxRateLimiterResource, EvDropYndxRateLimiterResource, diff --git a/ydb/core/grpc_services/grpc_request_proxy.cpp b/ydb/core/grpc_services/grpc_request_proxy.cpp index 2b5e5ba458..a1e7accb87 100644 --- a/ydb/core/grpc_services/grpc_request_proxy.cpp +++ b/ydb/core/grpc_services/grpc_request_proxy.cpp @@ -172,14 +172,14 @@ private: Handle(event, ctx); return; } - + auto state = requestBaseCtx->GetAuthState(); if (state.State == NGrpc::TAuthState::AS_FAIL) { requestBaseCtx->ReplyUnauthenticated(); return; } - + if (state.State == NGrpc::TAuthState::AS_UNAVAILABLE) { Counters->IncDatabaseUnavailableCounter(); const TString error = "Unable to resolve token"; @@ -317,7 +317,7 @@ void TGRpcRequestProxyImpl::Bootstrap(const TActorContext& ctx) { } Counters = MakeIntrusive<TGrpcProxyCounters>(AppData()->Counters); - + RootDatabase = DatabaseFromDomain(); TDatabaseInfo& database = Databases[RootDatabase]; database.DatabaseType = TDatabaseInfo::TDatabaseType::Root; @@ -334,7 +334,7 @@ void TGRpcRequestProxyImpl::ReplayEvents(const TString& databaseName, const TAct std::deque<TEventReqHolder> deferredEvents; std::swap(deferredEvents, queue); // we can put back event to DeferredEvents queue in StateFunc KIKIMR-12851 while (!deferredEvents.empty()) { - StateFunc(deferredEvents.front().Ev, ctx); + StateFunc(deferredEvents.front().Ev, ctx); deferredEvents.pop_front(); } if (queue.empty()) { @@ -552,7 +552,7 @@ void TGRpcRequestProxyImpl::StateFunc(TAutoPtr<IEventHandle>& ev, const TActorCo hFunc(TEvents::TEvUndelivered, HandleUndelivery); HFunc(TSchemeBoardEvents::TEvNotifyUpdate, HandleSchemeBoard); hFunc(TSchemeBoardEvents::TEvNotifyDelete, HandleSchemeBoard); - + default: handled = false; break; diff --git a/ydb/core/grpc_services/rpc_analytics_internal.cpp b/ydb/core/grpc_services/rpc_analytics_internal.cpp index 45cec13a64..48227a57fd 100644 --- a/ydb/core/grpc_services/rpc_analytics_internal.cpp +++ b/ydb/core/grpc_services/rpc_analytics_internal.cpp @@ -1,108 +1,108 @@ #include "service_analytics_internal.h" -#include "rpc_common.h" -#include "rpc_deferrable.h" - +#include "rpc_common.h" +#include "rpc_deferrable.h" + #include <ydb/core/yq/libs/events/events.h> #include <ydb/core/yq/libs/actors/proxy_private.h> - -#include <library/cpp/actors/core/hfunc.h> - + +#include <library/cpp/actors/core/hfunc.h> + #include <ydb/core/grpc_services/base/base.h> #include <ydb/public/api/protos/draft/yq_private.pb.h> -namespace NKikimr { -namespace NGRpcService { - -using TEvYqPrivatePingTaskRequest = - TGrpcRequestOperationCall<Yq::Private::PingTaskRequest, Yq::Private::PingTaskResponse>; -using TEvYqPrivateGetTaskRequest = - TGrpcRequestOperationCall<Yq::Private::GetTaskRequest, Yq::Private::GetTaskResponse>; -using TEvYqPrivateWriteTaskResultRequest = - TGrpcRequestOperationCall<Yq::Private::WriteTaskResultRequest, Yq::Private::WriteTaskResultResponse>; -using TEvYqPrivateNodesHealthCheckRequest = - TGrpcRequestOperationCall<Yq::Private::NodesHealthCheckRequest, Yq::Private::NodesHealthCheckResponse>; - -namespace { - template <typename TEv, typename TReq> - void SendResponse(const TEv& ev, TReq& req) { - if (!ev->Get()->Record) { +namespace NKikimr { +namespace NGRpcService { + +using TEvYqPrivatePingTaskRequest = + TGrpcRequestOperationCall<Yq::Private::PingTaskRequest, Yq::Private::PingTaskResponse>; +using TEvYqPrivateGetTaskRequest = + TGrpcRequestOperationCall<Yq::Private::GetTaskRequest, Yq::Private::GetTaskResponse>; +using TEvYqPrivateWriteTaskResultRequest = + TGrpcRequestOperationCall<Yq::Private::WriteTaskResultRequest, Yq::Private::WriteTaskResultResponse>; +using TEvYqPrivateNodesHealthCheckRequest = + TGrpcRequestOperationCall<Yq::Private::NodesHealthCheckRequest, Yq::Private::NodesHealthCheckResponse>; + +namespace { + template <typename TEv, typename TReq> + void SendResponse(const TEv& ev, TReq& req) { + if (!ev->Get()->Record) { req.RaiseIssues(ev->Get()->Issues); - req.ReplyWithYdbStatus(ev->Get()->Status); - } else { - req.SendResult(*ev->Get()->Record, ev->Get()->Status); - } - } - -} - -template <typename RpcRequestType, typename EvRequestType, typename EvResponseType> -class TYqPrivateRequestRPC : public TRpcOperationRequestActor< - TYqPrivateRequestRPC<RpcRequestType,EvRequestType,EvResponseType>, RpcRequestType> { - - using TBase = TRpcOperationRequestActor< - TYqPrivateRequestRPC<RpcRequestType,EvRequestType,EvResponseType>, - RpcRequestType>; - -public: - TYqPrivateRequestRPC(IRequestOpCtx* request) : TBase(request) {} - - void Bootstrap(const TActorContext& ctx) { - Y_UNUSED(ctx); - const auto req = this->GetProtoRequest(); - auto ev = MakeHolder<EvRequestType>(); - auto request = dynamic_cast<RpcRequestType*>(this->Request_.get()); - Y_VERIFY(request); - ev->Record = *req; - this->Send(NYq::MakeYqPrivateProxyId(), ev.Release()); - this->Become(&TYqPrivateRequestRPC<RpcRequestType, EvRequestType, EvResponseType>::StateFunc); - } - -private: - STRICT_STFUNC(StateFunc, - HFunc(EvResponseType, Handle); - ) - - void Handle(typename EvResponseType::TPtr& ev, const TActorContext& ctx) { - SendResponse(ev, *this->Request_); - this->Die(ctx); - } -}; - -using TYqPrivatePingTaskRPC = TYqPrivateRequestRPC< - TEvYqPrivatePingTaskRequest, - NYq::TEvents::TEvPingTaskRequest, - NYq::TEvents::TEvPingTaskResponse>; - -using TYqPrivateGetTaskRPC = TYqPrivateRequestRPC< - TEvYqPrivateGetTaskRequest, - NYq::TEvents::TEvGetTaskRequest, - NYq::TEvents::TEvGetTaskResponse>; - -using TYqPrivateWriteTaskResultRPC = TYqPrivateRequestRPC< - TEvYqPrivateWriteTaskResultRequest, - NYq::TEvents::TEvWriteTaskResultRequest, - NYq::TEvents::TEvWriteTaskResultResponse>; - -using TYqPrivateNodesHealthCheckRPC = TYqPrivateRequestRPC< - TEvYqPrivateNodesHealthCheckRequest, - NYq::TEvents::TEvNodesHealthCheckRequest, - NYq::TEvents::TEvNodesHealthCheckResponse>; - -void DoYqPrivatePingTaskRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&) { - TActivationContext::AsActorContext().Register(new TYqPrivatePingTaskRPC(p.release())); -} - -void DoYqPrivateGetTaskRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&) { - TActivationContext::AsActorContext().Register(new TYqPrivateGetTaskRPC(p.release())); -} - -void DoYqPrivateWriteTaskResultRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&) { - TActivationContext::AsActorContext().Register(new TYqPrivateWriteTaskResultRPC(p.release())); -} - -void DoYqPrivateNodesHealthCheckRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&) { - TActivationContext::AsActorContext().Register(new TYqPrivateNodesHealthCheckRPC(p.release())); -} - -} // namespace NGRpcService -} // namespace NKikimr + req.ReplyWithYdbStatus(ev->Get()->Status); + } else { + req.SendResult(*ev->Get()->Record, ev->Get()->Status); + } + } + +} + +template <typename RpcRequestType, typename EvRequestType, typename EvResponseType> +class TYqPrivateRequestRPC : public TRpcOperationRequestActor< + TYqPrivateRequestRPC<RpcRequestType,EvRequestType,EvResponseType>, RpcRequestType> { + + using TBase = TRpcOperationRequestActor< + TYqPrivateRequestRPC<RpcRequestType,EvRequestType,EvResponseType>, + RpcRequestType>; + +public: + TYqPrivateRequestRPC(IRequestOpCtx* request) : TBase(request) {} + + void Bootstrap(const TActorContext& ctx) { + Y_UNUSED(ctx); + const auto req = this->GetProtoRequest(); + auto ev = MakeHolder<EvRequestType>(); + auto request = dynamic_cast<RpcRequestType*>(this->Request_.get()); + Y_VERIFY(request); + ev->Record = *req; + this->Send(NYq::MakeYqPrivateProxyId(), ev.Release()); + this->Become(&TYqPrivateRequestRPC<RpcRequestType, EvRequestType, EvResponseType>::StateFunc); + } + +private: + STRICT_STFUNC(StateFunc, + HFunc(EvResponseType, Handle); + ) + + void Handle(typename EvResponseType::TPtr& ev, const TActorContext& ctx) { + SendResponse(ev, *this->Request_); + this->Die(ctx); + } +}; + +using TYqPrivatePingTaskRPC = TYqPrivateRequestRPC< + TEvYqPrivatePingTaskRequest, + NYq::TEvents::TEvPingTaskRequest, + NYq::TEvents::TEvPingTaskResponse>; + +using TYqPrivateGetTaskRPC = TYqPrivateRequestRPC< + TEvYqPrivateGetTaskRequest, + NYq::TEvents::TEvGetTaskRequest, + NYq::TEvents::TEvGetTaskResponse>; + +using TYqPrivateWriteTaskResultRPC = TYqPrivateRequestRPC< + TEvYqPrivateWriteTaskResultRequest, + NYq::TEvents::TEvWriteTaskResultRequest, + NYq::TEvents::TEvWriteTaskResultResponse>; + +using TYqPrivateNodesHealthCheckRPC = TYqPrivateRequestRPC< + TEvYqPrivateNodesHealthCheckRequest, + NYq::TEvents::TEvNodesHealthCheckRequest, + NYq::TEvents::TEvNodesHealthCheckResponse>; + +void DoYqPrivatePingTaskRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&) { + TActivationContext::AsActorContext().Register(new TYqPrivatePingTaskRPC(p.release())); +} + +void DoYqPrivateGetTaskRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&) { + TActivationContext::AsActorContext().Register(new TYqPrivateGetTaskRPC(p.release())); +} + +void DoYqPrivateWriteTaskResultRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&) { + TActivationContext::AsActorContext().Register(new TYqPrivateWriteTaskResultRPC(p.release())); +} + +void DoYqPrivateNodesHealthCheckRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&) { + TActivationContext::AsActorContext().Register(new TYqPrivateNodesHealthCheckRPC(p.release())); +} + +} // namespace NGRpcService +} // namespace NKikimr diff --git a/ydb/core/grpc_services/service_analytics_internal.h b/ydb/core/grpc_services/service_analytics_internal.h index 5fee70f346..f3d1ed966d 100644 --- a/ydb/core/grpc_services/service_analytics_internal.h +++ b/ydb/core/grpc_services/service_analytics_internal.h @@ -8,10 +8,10 @@ namespace NGRpcService { class IRequestOpCtx; class IFacilityProvider; -void DoYqPrivatePingTaskRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider& facility); -void DoYqPrivateGetTaskRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider& facility); -void DoYqPrivateWriteTaskResultRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider& facility); -void DoYqPrivateNodesHealthCheckRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider& facility); +void DoYqPrivatePingTaskRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider& facility); +void DoYqPrivateGetTaskRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider& facility); +void DoYqPrivateWriteTaskResultRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider& facility); +void DoYqPrivateNodesHealthCheckRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider& facility); } // namespace NGRpcService } // namespace NKikimr diff --git a/ydb/core/grpc_services/ya.make b/ydb/core/grpc_services/ya.make index c364c80745..05b156bf04 100644 --- a/ydb/core/grpc_services/ya.make +++ b/ydb/core/grpc_services/ya.make @@ -72,7 +72,7 @@ SRCS( rpc_yq.cpp table_profiles.cpp table_settings.cpp - rpc_analytics_internal.cpp + rpc_analytics_internal.cpp ) PEERDIR( diff --git a/ydb/core/protos/services.proto b/ydb/core/protos/services.proto index eb568d5e1f..c17c8a7dc3 100644 --- a/ydb/core/protos/services.proto +++ b/ydb/core/protos/services.proto @@ -262,10 +262,10 @@ enum EServiceKikimr { YQL_PROXY = 713; - YQL_PRIVATE_PROXY = 714; - - YQL_NODES_MANAGER = 715; - + YQL_PRIVATE_PROXY = 714; + + YQL_NODES_MANAGER = 715; + // Exports (& imports) EXPORT = 800; DATASHARD_RESTORE = 801; @@ -829,9 +829,9 @@ message TActivity { CHANGE_SENDER_ACTOR = 524; CHANGE_SENDER_ASYNC_INDEX_ACTOR_MAIN = 525; CHANGE_SENDER_ASYNC_INDEX_ACTOR_PARTITION = 526; - YQL_GET_CONNECTIONS_REQUEST_ACTOR = 527; - YQL_MODIFY_CONNECTIONS_REQUEST_ACTOR = 528; - YQL_MODIFY_HISTORY_REQUEST_ACTOR = 529; + YQL_GET_CONNECTIONS_REQUEST_ACTOR = 527; + YQL_MODIFY_CONNECTIONS_REQUEST_ACTOR = 528; + YQL_MODIFY_HISTORY_REQUEST_ACTOR = 529; YQL_WRITE_RESULT_DATA_ACTOR = 530; BS_SCRUB_ACTOR = 531; BS_BLOB_RECOVERY_ACTOR = 532; @@ -839,19 +839,19 @@ message TActivity { YQ_CONTROL_PLANE_STORAGE_ACTOR = 534; YQ_CONTROL_PLANE_PROXY_ACTOR = 535; DB_WATCHER_ACTOR = 495; - YQL_PENDING_FETCHER_ACTOR = 536; - YQL_PINGER_ACTOR = 537; - YQL_PRIVATE_PROXY_ACTOR = 538; - YQL_PRIVATE_PING_TASK_ACTOR = 539; + YQL_PENDING_FETCHER_ACTOR = 536; + YQL_PINGER_ACTOR = 537; + YQL_PRIVATE_PROXY_ACTOR = 538; + YQL_PRIVATE_PING_TASK_ACTOR = 539; EXT_COUNTERS_UPDATER_ACTOR = 540; - YQL_PRIVATE_GET_TASK_ACTOR = 541; - YQL_PRIVATE_WRITE_TASK_ACTOR = 542; - YQL_NODES_MANAGER_ACTOR = 543; + YQL_PRIVATE_GET_TASK_ACTOR = 541; + YQL_PRIVATE_WRITE_TASK_ACTOR = 542; + YQL_NODES_MANAGER_ACTOR = 543; KESUS_ACCOUNTING_ACTOR = 544; BLOCKSTORE_PARTITION_COMMON = 545; INTERCONNECT_PROXY_WRAPPER = 546; CLOUD_STORAGE_HIVE_PROXY = 547; - YQL_PRIVATE_NODES_HEALTH_CHECK_ACTOR = 548; + YQL_PRIVATE_NODES_HEALTH_CHECK_ACTOR = 548; CHANGE_SENDER_CDC_ACTOR_MAIN = 549; KQP_TEST_WORKLOAD = 550; PQ_PARTITION_WRITER_ACTOR = 551; diff --git a/ydb/core/testlib/test_client.cpp b/ydb/core/testlib/test_client.cpp index bdb3f4f7cc..d4907c26f1 100644 --- a/ydb/core/testlib/test_client.cpp +++ b/ydb/core/testlib/test_client.cpp @@ -319,7 +319,7 @@ namespace Tests { GRpcServer->AddService(new NGRpcService::TGRpcDataStreamsService(system, counters, grpcRequestProxyId)); if (Settings->EnableYq) { GRpcServer->AddService(new NGRpcService::TGRpcYandexQueryService(system, counters, grpcRequestProxyId)); - GRpcServer->AddService(new NGRpcService::TGRpcYqPrivateTaskService(system, counters, grpcRequestProxyId)); + GRpcServer->AddService(new NGRpcService::TGRpcYqPrivateTaskService(system, counters, grpcRequestProxyId)); } if (const auto& factory = Settings->GrpcServiceFactory) { // All services enabled by default for ut @@ -746,7 +746,7 @@ namespace Tests { controlPlaneStorageConfig.AddAvailableConnection("OBJECT_STORAGE"); controlPlaneStorageConfig.AddAvailableConnection("MONITORING"); } - + { auto& commonConfig = *protoConfig.MutableCommon(); commonConfig.SetYdbMvpCloudEndpoint(ydbMvpEndpoint); @@ -801,20 +801,20 @@ namespace Tests { }; const auto ydbCredFactory = NKikimr::CreateYdbCredentialsProviderFactory; - auto counters = MakeIntrusive<NMonitoring::TDynamicCounters>(); - auto yqSharedResources = NYq::CreateYqSharedResources(protoConfig, ydbCredFactory, counters); + auto counters = MakeIntrusive<NMonitoring::TDynamicCounters>(); + auto yqSharedResources = NYq::CreateYqSharedResources(protoConfig, ydbCredFactory, counters); NYq::Init( protoConfig, Runtime->GetNodeId(nodeIdx), actorRegistrator, - &appData, + &appData, "TestTenant", nullptr, // MakeIntrusive<NPq::NConfigurationManager::TConnections>(), yqSharedResources, NKikimr::NFolderService::CreateMockFolderServiceActor, NYq::CreateMockYqAuditServiceActor, - ydbCredFactory, - /*IcPort = */0 + ydbCredFactory, + /*IcPort = */0 ); NYq::InitTest(Runtime.Get(), port, Settings->GrpcPort, yqSharedResources); } @@ -843,7 +843,7 @@ namespace Tests { //Runtime->SetLogPriority(NKikimrServices::SCHEME_BOARD_REPLICA, NActors::NLog::PRI_DEBUG); //Runtime->SetLogPriority(NKikimrServices::SCHEME_BOARD_POPULATOR, NActors::NLog::PRI_DEBUG); //Runtime->SetLogPriority(NKikimrServices::SCHEME_BOARD_SUBSCRIBER, NActors::NLog::PRI_TRACE); - //Runtime->SetLogPriority(NKikimrServices::YQL_PROXY, NActors::NLog::PRI_DEBUG); + //Runtime->SetLogPriority(NKikimrServices::YQL_PROXY, NActors::NLog::PRI_DEBUG); if (Settings->LoggerInitializer) { Settings->LoggerInitializer(*Runtime); diff --git a/ydb/core/yq/libs/actors/clusters_from_connections.cpp b/ydb/core/yq/libs/actors/clusters_from_connections.cpp index c2927919b9..93e0471892 100644 --- a/ydb/core/yq/libs/actors/clusters_from_connections.cpp +++ b/ydb/core/yq/libs/actors/clusters_from_connections.cpp @@ -6,10 +6,10 @@ #include <util/string/builder.h> #include <util/system/env.h> -namespace NYq { +namespace NYq { + +using namespace NYql; -using namespace NYql; - namespace { template <typename TClusterConfig> @@ -30,8 +30,8 @@ void FillClusterAuth(TClusterConfig& clusterCfg, const YandexQuery::IamAuth& aut } } -} //namespace - +} //namespace + void AddClustersFromConnections(const THashMap<TString, YandexQuery::Connection>& connections, bool useBearerForYdb, const TString& objectStorageEndpoint, @@ -47,10 +47,10 @@ void AddClustersFromConnections(const THashMap<TString, YandexQuery::Connection> auto* clusterCfg = gatewaysConfig.MutableYdb()->AddClusterMapping(); clusterCfg->SetName(connectionName); clusterCfg->SetId(db.database_id()); - if (db.database()) - clusterCfg->SetDatabase(db.database()); - if (db.endpoint()) - clusterCfg->SetEndpoint(db.endpoint()); + if (db.database()) + clusterCfg->SetDatabase(db.database()); + if (db.endpoint()) + clusterCfg->SetEndpoint(db.endpoint()); clusterCfg->SetSecure(db.secure()); clusterCfg->SetAddBearerToToken(useBearerForYdb); FillClusterAuth(*clusterCfg, db.auth(), authToken, accountIdSignatures); @@ -61,11 +61,11 @@ void AddClustersFromConnections(const THashMap<TString, YandexQuery::Connection> const auto& ch = conn.content().setting().clickhouse_cluster(); auto* clusterCfg = gatewaysConfig.MutableClickHouse()->AddClusterMapping(); clusterCfg->SetName(connectionName); - clusterCfg->SetId(ch.database_id()); - if (ch.host()) - clusterCfg->SetCluster(ch.host()); - clusterCfg->SetNativeHostPort(9440); - clusterCfg->SetNativeSecure(true); + clusterCfg->SetId(ch.database_id()); + if (ch.host()) + clusterCfg->SetCluster(ch.host()); + clusterCfg->SetNativeHostPort(9440); + clusterCfg->SetNativeSecure(true); clusterCfg->SetCHToken(TStringBuilder() << "basic#" << ch.login() << "#" << ch.password()); clusters.emplace(connectionName, ClickHouseProviderName); break; @@ -89,8 +89,8 @@ void AddClustersFromConnections(const THashMap<TString, YandexQuery::Connection> const auto& ds = conn.content().setting().data_streams(); auto* clusterCfg = gatewaysConfig.MutablePq()->AddClusterMapping(); clusterCfg->SetName(connectionName); - if (ds.endpoint()) - clusterCfg->SetEndpoint(ds.endpoint()); + if (ds.endpoint()) + clusterCfg->SetEndpoint(ds.endpoint()); clusterCfg->SetDatabase(ds.database()); clusterCfg->SetDatabaseId(ds.database_id()); clusterCfg->SetUseSsl(ds.secure()); @@ -135,4 +135,4 @@ void AddClustersFromConnections(const THashMap<TString, YandexQuery::Connection> } } } -} //NYq +} //NYq diff --git a/ydb/core/yq/libs/actors/clusters_from_connections.h b/ydb/core/yq/libs/actors/clusters_from_connections.h index 0a3cc6783d..08db9704ea 100644 --- a/ydb/core/yq/libs/actors/clusters_from_connections.h +++ b/ydb/core/yq/libs/actors/clusters_from_connections.h @@ -3,14 +3,14 @@ #include <ydb/library/yql/providers/common/proto/gateways_config.pb.h> #include <ydb/public/api/protos/yq.pb.h> -namespace NYq { - +namespace NYq { + void AddClustersFromConnections(const THashMap<TString, YandexQuery::Connection>& connections, bool useBearerForYdb, const TString& objectStorageEndpoint, const TString& authToken, const THashMap<TString, TString>& accountIdSignatures, - NYql::TGatewaysConfig& gatewaysConfig, + NYql::TGatewaysConfig& gatewaysConfig, THashMap<TString, TString>& clusters); - -} //NYq + +} //NYq diff --git a/ydb/core/yq/libs/actors/database_resolver.cpp b/ydb/core/yq/libs/actors/database_resolver.cpp index 8aa27187c5..c784d80c00 100644 --- a/ydb/core/yq/libs/actors/database_resolver.cpp +++ b/ydb/core/yq/libs/actors/database_resolver.cpp @@ -15,93 +15,93 @@ #define LOG_D(stream) \ LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::YQL_PROXY, "DatabaseResolver - TraceId: " << TraceId << ": " << stream) -namespace NYq { +namespace NYq { using namespace NActors; -using namespace NYql; - +using namespace NYql; + using TEndpoint = TEvents::TEvEndpointResponse::TEndpoint; using TParsers = THashMap<DatabaseType, std::function<TEndpoint(NJson::TJsonValue& body, bool)>>; -using TCache = TTtlCache<std::tuple<TString, DatabaseType, TEvents::TDatabaseAuth>, std::variant<TEndpoint, TString>>; +using TCache = TTtlCache<std::tuple<TString, DatabaseType, TEvents::TDatabaseAuth>, std::variant<TEndpoint, TString>>; + +TString TransformMdbHostToCorrectFormat(const TString& mdbHost) { + return mdbHost.substr(0, mdbHost.find('.')) + ".db.yandex.net:8443"; +} -TString TransformMdbHostToCorrectFormat(const TString& mdbHost) { - return mdbHost.substr(0, mdbHost.find('.')) + ".db.yandex.net:8443"; -} - -class TResponseProcessor : public TActorBootstrapped<TResponseProcessor> +class TResponseProcessor : public TActorBootstrapped<TResponseProcessor> { public: - enum EWakeUp { - WU_Die_On_Ttl - }; - + enum EWakeUp { + WU_Die_On_Ttl + }; + TResponseProcessor( - const TActorId sender, + const TActorId sender, TCache& cache, const THashMap<std::pair<TString, DatabaseType>, TEndpoint>& ready, const THashMap<NHttp::THttpOutgoingRequestPtr, std::tuple<TString, DatabaseType, TEvents::TDatabaseAuth>>& requests, - const TString& traceId, + const TString& traceId, bool mdbTransformHost, const TParsers& parsers) - : Sender(sender) + : Sender(sender) , Cache(cache) , Requests(requests) , TraceId(traceId) - , MdbTransformHost(mdbTransformHost) + , MdbTransformHost(mdbTransformHost) , DatabaseId2Endpoint(ready) , Parsers(parsers) { } - static constexpr char ActorName[] = "YQ_RESPONSE_PROCESSOR"; - - void Bootstrap() { - Schedule(ResolvingTtl, new NActors::TEvents::TEvWakeup(WU_Die_On_Ttl)); - Become(&TResponseProcessor::StateFunc); - } - - STRICT_STFUNC(StateFunc, { - hFunc(NHttp::TEvHttpProxy::TEvHttpIncomingResponse, Handle); - hFunc(NActors::TEvents::TEvWakeup, HandleWakeup) - cFunc(NActors::TEvents::TEvPoison::EventType, PassAway); + static constexpr char ActorName[] = "YQ_RESPONSE_PROCESSOR"; + + void Bootstrap() { + Schedule(ResolvingTtl, new NActors::TEvents::TEvWakeup(WU_Die_On_Ttl)); + Become(&TResponseProcessor::StateFunc); + } + + STRICT_STFUNC(StateFunc, { + hFunc(NHttp::TEvHttpProxy::TEvHttpIncomingResponse, Handle); + hFunc(NActors::TEvents::TEvWakeup, HandleWakeup) + cFunc(NActors::TEvents::TEvPoison::EventType, PassAway); }); -private: - void HandleWakeup(NActors::TEvents::TEvWakeup::TPtr& ev) { - auto tag = ev->Get()->Tag; - switch (tag) { - case WU_Die_On_Ttl: - DieOnTtl(); - break; - } - } - - void DieOnTtl() { - Success = false; - - TString logMsg = "Could not resolve database ids: "; - bool firstUnresolvedDbId = true; - for (const auto& [_, info]: Requests) { - const auto& dbId = std::get<0>(info); - const auto& dbType = std::get<1>(info); - if (const auto it = DatabaseId2Endpoint.find(std::make_pair(dbId, dbType)); it == DatabaseId2Endpoint.end()) { - logMsg += (firstUnresolvedDbId ? TString{""} : TString{", "}) + dbId; - if (firstUnresolvedDbId) - firstUnresolvedDbId = false; - } - } - logMsg += TStringBuilder() << " in " << ResolvingTtl << " seconds."; - LOG_E(logMsg); - - SendResolvedEndpointsAndDie(); - } - - void SendResolvedEndpointsAndDie() { - Send(Sender, new TEvents::TEvEndpointResponse(std::move(DatabaseId2Endpoint), Success)); - PassAway(); - } - - void Handle(NHttp::TEvHttpProxy::TEvHttpIncomingResponse::TPtr& ev) +private: + void HandleWakeup(NActors::TEvents::TEvWakeup::TPtr& ev) { + auto tag = ev->Get()->Tag; + switch (tag) { + case WU_Die_On_Ttl: + DieOnTtl(); + break; + } + } + + void DieOnTtl() { + Success = false; + + TString logMsg = "Could not resolve database ids: "; + bool firstUnresolvedDbId = true; + for (const auto& [_, info]: Requests) { + const auto& dbId = std::get<0>(info); + const auto& dbType = std::get<1>(info); + if (const auto it = DatabaseId2Endpoint.find(std::make_pair(dbId, dbType)); it == DatabaseId2Endpoint.end()) { + logMsg += (firstUnresolvedDbId ? TString{""} : TString{", "}) + dbId; + if (firstUnresolvedDbId) + firstUnresolvedDbId = false; + } + } + logMsg += TStringBuilder() << " in " << ResolvingTtl << " seconds."; + LOG_E(logMsg); + + SendResolvedEndpointsAndDie(); + } + + void SendResolvedEndpointsAndDie() { + Send(Sender, new TEvents::TEvEndpointResponse(std::move(DatabaseId2Endpoint), Success)); + PassAway(); + } + + void Handle(NHttp::TEvHttpProxy::TEvHttpIncomingResponse::TPtr& ev) { TString status; TString errorMessage; @@ -109,7 +109,7 @@ private: DatabaseType databaseType = DatabaseType::Ydb; TEvents::TDatabaseAuth info; TMaybe<TEndpoint> result; - HandledIds++; + HandledIds++; if (ev->Get()->Error.empty() && (ev->Get()->Response && ((status = ev->Get()->Response->Status) == "200"))) { NJson::TJsonReaderConfig jsonConfig; NJson::TJsonValue databaseInfo; @@ -134,20 +134,20 @@ private: << databaseType << " Id: " << databaseId << "\n" << CurrentExceptionMessage(); - } + } } else { - errorMessage = TStringBuilder() << "Unable to parse database information." - << "Database Id: " << databaseId + errorMessage = TStringBuilder() << "Unable to parse database information." + << "Database Id: " << databaseId << ", Database Type: " << databaseType; } } } else { errorMessage = ev->Get()->Error; - const TString error = "Cannot resolve databaseId (status = " + ToString(status) + ")"; - if (!errorMessage.empty()) { - errorMessage += '\n'; + const TString error = "Cannot resolve databaseId (status = " + ToString(status) + ")"; + if (!errorMessage.empty()) { + errorMessage += '\n'; } - errorMessage += error; + errorMessage += error; } if (errorMessage) { @@ -157,18 +157,18 @@ private: if (databaseId) { auto key = std::make_tuple(databaseId, databaseType, info); - if (errorMessage) { - Cache.Put(key, errorMessage); - } else { - Cache.Put(key, result); - } + if (errorMessage) { + Cache.Put(key, errorMessage); + } else { + Cache.Put(key, result); + } } - if (HandledIds == Requests.size()) { - SendResolvedEndpointsAndDie(); + if (HandledIds == Requests.size()) { + SendResolvedEndpointsAndDie(); } - LOG_D(DatabaseId2Endpoint.size() << " of " << Requests.size() << " done"); + LOG_D(DatabaseId2Endpoint.size() << " of " << Requests.size() << " done"); } private: @@ -176,12 +176,12 @@ private: TCache& Cache; const THashMap<NHttp::THttpOutgoingRequestPtr, std::tuple<TString, DatabaseType, TEvents::TDatabaseAuth>> Requests; const TString TraceId; - const bool MdbTransformHost; + const bool MdbTransformHost; THashMap<std::pair<TString, DatabaseType>, TEvents::TEvEndpointResponse::TEndpoint> DatabaseId2Endpoint; - size_t HandledIds = 0; + size_t HandledIds = 0; bool Success = true; const TParsers& Parsers; - TDuration ResolvingTtl = TDuration::Seconds(30); //TODO: Use cfg + TDuration ResolvingTtl = TDuration::Seconds(30); //TODO: Use cfg }; class TDatabaseResolver: public TActor<TDatabaseResolver> @@ -248,8 +248,8 @@ public: }; } - static constexpr char ActorName[] = "YQ_DATABASE_RESOLVER"; - + static constexpr char ActorName[] = "YQ_DATABASE_RESOLVER"; + private: STRICT_STFUNC(State, { HFunc(TEvents::TEvEndpointRequest, Handle); @@ -264,13 +264,13 @@ private: THashMap<std::pair<TString, DatabaseType>, TEndpoint> ready; for (const auto& [p, info] : ev->Get()->DatabaseIds) { const auto& [databaseId, type] = p; - TMaybe<std::variant<TEndpoint, TString>> endpoint; + TMaybe<std::variant<TEndpoint, TString>> endpoint; auto key = std::make_tuple(databaseId, type, info); if (Cache.Get(key, &endpoint)) { if (endpoint) { - ready.insert(std::make_pair(p, - (endpoint->index() == 0 ? std::get<0>(*endpoint) : TEndpoint{}) - )); + ready.insert(std::make_pair(p, + (endpoint->index() == 0 ? std::get<0>(*endpoint) : TEndpoint{}) + )); } continue; } @@ -291,15 +291,15 @@ private: requests[httpRequest] = key; } catch (const std::exception& e) { - const TString msg = TStringBuilder() << " Error while preparing to resolve databaseId " << databaseId << ", details: " << e.what(); + const TString msg = TStringBuilder() << " Error while preparing to resolve databaseId " << databaseId << ", details: " << e.what(); LOG_E(msg); Cache.Put(key, endpoint); } } if (!requests.empty()) { - auto helper = Register( - new TResponseProcessor(ev->Sender, Cache, ready, requests, TraceId, ev->Get()->MdbTransformHost, Parsers)); + auto helper = Register( + new TResponseProcessor(ev->Sender, Cache, ready, requests, TraceId, ev->Get()->MdbTransformHost, Parsers)); for (const auto& [request, _] : requests) { ctx.Send(new IEventHandle(HttpProxy, helper, new NHttp::TEvHttpProxy::TEvHttpOutgoingRequest(request))); @@ -321,4 +321,4 @@ NActors::IActor* CreateDatabaseResolver(NActors::TActorId httpProxy, ISecuredSer return new TDatabaseResolver(httpProxy, credentialsFactory); } -} /* namespace NYq */ +} /* namespace NYq */ diff --git a/ydb/core/yq/libs/actors/database_resolver.h b/ydb/core/yq/libs/actors/database_resolver.h index f7b98beec0..3e50f46bc4 100644 --- a/ydb/core/yq/libs/actors/database_resolver.h +++ b/ydb/core/yq/libs/actors/database_resolver.h @@ -3,10 +3,10 @@ #include <library/cpp/actors/core/actor.h> #include <ydb/library/yql/providers/common/token_accessor/client/factory.h> -namespace NYq { +namespace NYq { -TString TransformMdbHostToCorrectFormat(const TString& mdbHost); - -NActors::IActor* CreateDatabaseResolver(NActors::TActorId httpProxy, NYql::ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory); +TString TransformMdbHostToCorrectFormat(const TString& mdbHost); -} /* namespace NYq */ +NActors::IActor* CreateDatabaseResolver(NActors::TActorId httpProxy, NYql::ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory); + +} /* namespace NYq */ diff --git a/ydb/core/yq/libs/actors/error.cpp b/ydb/core/yq/libs/actors/error.cpp index 22212eb4b3..7a847961c1 100644 --- a/ydb/core/yq/libs/actors/error.cpp +++ b/ydb/core/yq/libs/actors/error.cpp @@ -2,10 +2,10 @@ #include <util/system/hostname.h> -namespace NYq { +namespace NYq { TString MakeInternalError(const TString& text) { return TStringBuilder() << "Internal error (" << text << ", host: " << HostName() << ")"; } -} // namespace NYq +} // namespace NYq diff --git a/ydb/core/yq/libs/actors/nodes_health_check.cpp b/ydb/core/yq/libs/actors/nodes_health_check.cpp index 395b09cb89..96e7c1c1cd 100644 --- a/ydb/core/yq/libs/actors/nodes_health_check.cpp +++ b/ydb/core/yq/libs/actors/nodes_health_check.cpp @@ -1,132 +1,132 @@ -#include "proxy_private.h" - +#include "proxy_private.h" + #include <ydb/core/protos/services.pb.h> #include <ydb/library/yql/public/issue/yql_issue_message.h> - -#include <library/cpp/yson/node/node_io.h> -#include <library/cpp/actors/core/events.h> -#include <library/cpp/actors/core/hfunc.h> -#include <library/cpp/actors/core/actor_bootstrapped.h> -#include <library/cpp/actors/core/log.h> - + +#include <library/cpp/yson/node/node_io.h> +#include <library/cpp/actors/core/events.h> +#include <library/cpp/actors/core/hfunc.h> +#include <library/cpp/actors/core/actor_bootstrapped.h> +#include <library/cpp/actors/core/log.h> + #include <ydb/core/yq/libs/control_plane_storage/control_plane_storage.h> #include <ydb/core/yq/libs/control_plane_storage/events/events.h> -#include <google/protobuf/util/time_util.h> - +#include <google/protobuf/util/time_util.h> + #define LOG_E(stream) \ LOG_ERROR_S(*TlsActivationContext, NKikimrServices::YQL_NODES_MANAGER, stream) #define LOG_D(stream) \ LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::YQL_NODES_MANAGER, stream) - -namespace NYq { - -using namespace NActors; -using namespace NYql; -using namespace NMonitoring; - -class TNodesHealthCheckActor - : public NActors::TActorBootstrapped<TNodesHealthCheckActor> -{ -public: - TNodesHealthCheckActor( - const NActors::TActorId& sender, - TIntrusivePtr<ITimeProvider> timeProvider, - TAutoPtr<TEvents::TEvNodesHealthCheckRequest> ev, - TDynamicCounterPtr counters) + +namespace NYq { + +using namespace NActors; +using namespace NYql; +using namespace NMonitoring; + +class TNodesHealthCheckActor + : public NActors::TActorBootstrapped<TNodesHealthCheckActor> +{ +public: + TNodesHealthCheckActor( + const NActors::TActorId& sender, + TIntrusivePtr<ITimeProvider> timeProvider, + TAutoPtr<TEvents::TEvNodesHealthCheckRequest> ev, + TDynamicCounterPtr counters) : Sender(sender) - , TimeProvider(timeProvider) - , Ev(std::move(ev)) - , Counters(std::move(counters)) - , LifetimeDuration(Counters->GetSubgroup("subsystem", "private_api")->GetHistogram("NodesHealthCheckTask", ExponentialHistogram(10, 2, 50))) - , StartTime(TInstant::Now()) - {} - - static constexpr char ActorName[] = "YQ_PRIVATE_NODES_HEALTH_CHECK"; - - void OnUndelivered(NActors::TEvents::TEvUndelivered::TPtr& ev, const NActors::TActorContext& ctx) { - LOG_E("TNodesHealthCheckActor::OnUndelivered"); - auto res = MakeHolder<TEvents::TEvNodesHealthCheckResponse>(); - res->Status = Ydb::StatusIds::GENERIC_ERROR; - res->Issues.AddIssue("UNDELIVERED"); - ctx.Send(ev->Sender, res.Release()); - Die(ctx); - } - - void PassAway() final { - LifetimeDuration->Collect((TInstant::Now() - StartTime).MilliSeconds()); - NActors::IActor::PassAway(); - } - - void Fail(const TString& message, Ydb::StatusIds::StatusCode reqStatus = Ydb::StatusIds::INTERNAL_ERROR) { - Issues.AddIssue(message); - const auto codeStr = Ydb::StatusIds_StatusCode_Name(reqStatus); - LOG_E(TStringBuilder() - << "Failed with code: " << codeStr - << " Details: " << Issues.ToString()); - auto res = MakeHolder<TEvents::TEvNodesHealthCheckResponse>(); - res->Status = reqStatus; - res->Issues.AddIssues(Issues); - Send(Sender, res.Release()); - PassAway(); - } - - void Bootstrap(const TActorContext&) { - Become(&TNodesHealthCheckActor::StateFunc); - auto& req = Ev->Record; - Tenant = req.tenant(); - - Send(NYq::ControlPlaneStorageServiceActorId(), - new NYq::TEvControlPlaneStorage::TEvNodesHealthCheckRequest(std::move(req))); - } - -private: - STRICT_STFUNC( - StateFunc, - CFunc(NActors::TEvents::TEvPoison::EventType, Die) - HFunc(NYq::TEvControlPlaneStorage::TEvNodesHealthCheckResponse, HandleResponse) - HFunc(NActors::TEvents::TEvUndelivered, OnUndelivered) - ) - - void HandleResponse(NYq::TEvControlPlaneStorage::TEvNodesHealthCheckResponse::TPtr& ev, const TActorContext& ctx) { - auto res = MakeHolder<TEvents::TEvNodesHealthCheckResponse>(); - try { - const auto& issues = ev->Get()->Issues; - if (issues) { - ythrow yexception() << issues.ToString(); - } - res->Record.ConstructInPlace(); - res->Status = Ydb::StatusIds::SUCCESS; - res->Record = ev->Get()->Record; - ctx.Send(Sender, res.Release()); - Die(ctx); - } catch (...) { - const auto msg = TStringBuilder() << "Can't do NodesHealthCheck: " << CurrentExceptionMessage(); - Fail(msg); - } - } - -private: - const TActorId Sender; - TIntrusivePtr<ITimeProvider> TimeProvider; - TAutoPtr<TEvents::TEvNodesHealthCheckRequest> Ev; - TDynamicCounterPtr Counters; - const THistogramPtr LifetimeDuration; - const TInstant StartTime; - - NYql::TIssues Issues; - TString Tenant; -}; - -IActor* CreateNodesHealthCheckActor( - const NActors::TActorId& sender, - TIntrusivePtr<ITimeProvider> timeProvider, - TAutoPtr<TEvents::TEvNodesHealthCheckRequest> ev, - TDynamicCounterPtr counters) { - return new TNodesHealthCheckActor( - sender, - timeProvider, - std::move(ev), - std::move(counters)); -} - -} /* NYq */ + , TimeProvider(timeProvider) + , Ev(std::move(ev)) + , Counters(std::move(counters)) + , LifetimeDuration(Counters->GetSubgroup("subsystem", "private_api")->GetHistogram("NodesHealthCheckTask", ExponentialHistogram(10, 2, 50))) + , StartTime(TInstant::Now()) + {} + + static constexpr char ActorName[] = "YQ_PRIVATE_NODES_HEALTH_CHECK"; + + void OnUndelivered(NActors::TEvents::TEvUndelivered::TPtr& ev, const NActors::TActorContext& ctx) { + LOG_E("TNodesHealthCheckActor::OnUndelivered"); + auto res = MakeHolder<TEvents::TEvNodesHealthCheckResponse>(); + res->Status = Ydb::StatusIds::GENERIC_ERROR; + res->Issues.AddIssue("UNDELIVERED"); + ctx.Send(ev->Sender, res.Release()); + Die(ctx); + } + + void PassAway() final { + LifetimeDuration->Collect((TInstant::Now() - StartTime).MilliSeconds()); + NActors::IActor::PassAway(); + } + + void Fail(const TString& message, Ydb::StatusIds::StatusCode reqStatus = Ydb::StatusIds::INTERNAL_ERROR) { + Issues.AddIssue(message); + const auto codeStr = Ydb::StatusIds_StatusCode_Name(reqStatus); + LOG_E(TStringBuilder() + << "Failed with code: " << codeStr + << " Details: " << Issues.ToString()); + auto res = MakeHolder<TEvents::TEvNodesHealthCheckResponse>(); + res->Status = reqStatus; + res->Issues.AddIssues(Issues); + Send(Sender, res.Release()); + PassAway(); + } + + void Bootstrap(const TActorContext&) { + Become(&TNodesHealthCheckActor::StateFunc); + auto& req = Ev->Record; + Tenant = req.tenant(); + + Send(NYq::ControlPlaneStorageServiceActorId(), + new NYq::TEvControlPlaneStorage::TEvNodesHealthCheckRequest(std::move(req))); + } + +private: + STRICT_STFUNC( + StateFunc, + CFunc(NActors::TEvents::TEvPoison::EventType, Die) + HFunc(NYq::TEvControlPlaneStorage::TEvNodesHealthCheckResponse, HandleResponse) + HFunc(NActors::TEvents::TEvUndelivered, OnUndelivered) + ) + + void HandleResponse(NYq::TEvControlPlaneStorage::TEvNodesHealthCheckResponse::TPtr& ev, const TActorContext& ctx) { + auto res = MakeHolder<TEvents::TEvNodesHealthCheckResponse>(); + try { + const auto& issues = ev->Get()->Issues; + if (issues) { + ythrow yexception() << issues.ToString(); + } + res->Record.ConstructInPlace(); + res->Status = Ydb::StatusIds::SUCCESS; + res->Record = ev->Get()->Record; + ctx.Send(Sender, res.Release()); + Die(ctx); + } catch (...) { + const auto msg = TStringBuilder() << "Can't do NodesHealthCheck: " << CurrentExceptionMessage(); + Fail(msg); + } + } + +private: + const TActorId Sender; + TIntrusivePtr<ITimeProvider> TimeProvider; + TAutoPtr<TEvents::TEvNodesHealthCheckRequest> Ev; + TDynamicCounterPtr Counters; + const THistogramPtr LifetimeDuration; + const TInstant StartTime; + + NYql::TIssues Issues; + TString Tenant; +}; + +IActor* CreateNodesHealthCheckActor( + const NActors::TActorId& sender, + TIntrusivePtr<ITimeProvider> timeProvider, + TAutoPtr<TEvents::TEvNodesHealthCheckRequest> ev, + TDynamicCounterPtr counters) { + return new TNodesHealthCheckActor( + sender, + timeProvider, + std::move(ev), + std::move(counters)); +} + +} /* NYq */ diff --git a/ydb/core/yq/libs/actors/nodes_manager.cpp b/ydb/core/yq/libs/actors/nodes_manager.cpp index 62cf85e5c2..62d92aec0f 100644 --- a/ydb/core/yq/libs/actors/nodes_manager.cpp +++ b/ydb/core/yq/libs/actors/nodes_manager.cpp @@ -1,119 +1,119 @@ -#include "nodes_manager.h" +#include "nodes_manager.h" #include <ydb/core/yq/libs/config/protos/yq_config.pb.h> - -#include <library/cpp/actors/core/events.h> -#include <library/cpp/actors/core/hfunc.h> -#include <library/cpp/actors/core/actor_bootstrapped.h> + +#include <library/cpp/actors/core/events.h> +#include <library/cpp/actors/core/hfunc.h> +#include <library/cpp/actors/core/actor_bootstrapped.h> #include <ydb/library/yql/providers/dq/worker_manager/interface/events.h> #include <ydb/library/yql/public/issue/yql_issue_message.h> #include <ydb/public/sdk/cpp/client/ydb_driver/driver.h> #include <ydb/public/sdk/cpp/client/ydb_value/value.h> #include <ydb/core/yq/libs/common/entity_id.h> #include <ydb/core/yq/libs/private_client/private_client.h> -#include <library/cpp/actors/core/log.h> -#include <util/system/hostname.h> +#include <library/cpp/actors/core/log.h> +#include <util/system/hostname.h> #include <ydb/core/protos/services.pb.h> - - + + #define LOG_E(stream) \ LOG_ERROR_S(*TlsActivationContext, NKikimrServices::YQL_NODES_MANAGER, stream) #define LOG_I(stream) \ LOG_INFO_S(*TlsActivationContext, NKikimrServices::YQL_NODES_MANAGER, stream) #define LOG_D(stream) \ LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::YQL_NODES_MANAGER, stream) - -namespace NYq { - -using namespace NActors; -using namespace NYql; -using namespace NDqs; - -struct TEvHealthNodesResponse : public NActors::TEventLocal<TEvHealthNodesResponse, NActors::TEvents::TSystem::Completed>{ - bool Success; - NYdb::EStatus Status; - const NYql::TIssues Issues; - Yq::Private::NodesHealthCheckResult Record; - - explicit TEvHealthNodesResponse( - const bool success, - const NYdb::EStatus& status, - const TIssues& issues, - const Yq::Private::NodesHealthCheckResult& record) - : Success(success) - , Status(status) - , Issues(issues) - , Record(record) - { } -}; - -class TYqlNodesManagerActor : public NActors::TActorBootstrapped<TYqlNodesManagerActor> { -public: - enum EWakeUp { - WU_NodesHealthCheck - }; - - TYqlNodesManagerActor( + +namespace NYq { + +using namespace NActors; +using namespace NYql; +using namespace NDqs; + +struct TEvHealthNodesResponse : public NActors::TEventLocal<TEvHealthNodesResponse, NActors::TEvents::TSystem::Completed>{ + bool Success; + NYdb::EStatus Status; + const NYql::TIssues Issues; + Yq::Private::NodesHealthCheckResult Record; + + explicit TEvHealthNodesResponse( + const bool success, + const NYdb::EStatus& status, + const TIssues& issues, + const Yq::Private::NodesHealthCheckResult& record) + : Success(success) + , Status(status) + , Issues(issues) + , Record(record) + { } +}; + +class TYqlNodesManagerActor : public NActors::TActorBootstrapped<TYqlNodesManagerActor> { +public: + enum EWakeUp { + WU_NodesHealthCheck + }; + + TYqlNodesManagerActor( const NYq::TYqSharedResources::TPtr& yqSharedResources, - const NDqs::TWorkerManagerCounters& workerManagerCounters, - TIntrusivePtr<ITimeProvider> timeProvider, - TIntrusivePtr<IRandomProvider> randomProvider, + const NDqs::TWorkerManagerCounters& workerManagerCounters, + TIntrusivePtr<ITimeProvider> timeProvider, + TIntrusivePtr<IRandomProvider> randomProvider, const ::NYq::NCommon::TServiceCounters& serviceCounters, const NConfig::TPrivateApiConfig& privateApiConfig, - const ui32& icPort, - const TString& address, + const ui32& icPort, + const TString& address, const TString& tenant, - ui64 mkqlInitialMemoryLimit, - const NMonitoring::TDynamicCounterPtr& clientCounters) - : WorkerManagerCounters(workerManagerCounters) - , TimeProvider(timeProvider) - , RandomProvider(randomProvider) + ui64 mkqlInitialMemoryLimit, + const NMonitoring::TDynamicCounterPtr& clientCounters) + : WorkerManagerCounters(workerManagerCounters) + , TimeProvider(timeProvider) + , RandomProvider(randomProvider) , ServiceCounters(serviceCounters, "node_manager") , PrivateApiConfig(privateApiConfig) - , Tenant(tenant) + , Tenant(tenant) , MkqlInitialMemoryLimit(mkqlInitialMemoryLimit) , YqSharedResources(yqSharedResources) - , IcPort(icPort) - , Address(address) - , Client( + , IcPort(icPort) + , Address(address) + , Client( YqSharedResources->YdbDriver, - NYdb::TCommonClientSettings() + NYdb::TCommonClientSettings() .DiscoveryEndpoint(PrivateApiConfig.GetTaskServiceEndpoint()) - .Database(PrivateApiConfig.GetTaskServiceDatabase() ? PrivateApiConfig.GetTaskServiceDatabase() : TMaybe<TString>()), - clientCounters) - - { - InstanceId = GetGuidAsString(RandomProvider->GenUuid4()); - } - - static constexpr char ActorName[] = "YQ_NODES_MANAGER"; - - void PassAway() final { + .Database(PrivateApiConfig.GetTaskServiceDatabase() ? PrivateApiConfig.GetTaskServiceDatabase() : TMaybe<TString>()), + clientCounters) + + { + InstanceId = GetGuidAsString(RandomProvider->GenUuid4()); + } + + static constexpr char ActorName[] = "YQ_NODES_MANAGER"; + + void PassAway() final { LOG_I("PassAway STOPPED"); - NActors::IActor::PassAway(); - } - + NActors::IActor::PassAway(); + } + void Bootstrap(const TActorContext&) { - Become(&TYqlNodesManagerActor::StateFunc); + Become(&TYqlNodesManagerActor::StateFunc); ServiceCounters.Counters->GetCounter("EvBootstrap", true)->Inc(); LOG_I("Bootstrap STARTED"); NodesHealthCheck(); - } - -private: + } + +private: void Handle(NDqs::TEvAllocateWorkersRequest::TPtr& ev) { ServiceCounters.Counters->GetCounter("EvAllocateWorkersRequest", true)->Inc(); - const auto &rec = ev->Get()->Record; - const auto count = rec.GetCount(); - Y_ASSERT(count != 0); - auto resourceId = rec.GetResourceId(); - if (!resourceId) { + const auto &rec = ev->Get()->Record; + const auto count = rec.GetCount(); + Y_ASSERT(count != 0); + auto resourceId = rec.GetResourceId(); + if (!resourceId) { resourceId = (ui64(++ResourceIdPart) << 32) | SelfId().NodeId(); - } - - TVector<TPeer> nodes; - for (ui32 i = 0; i < count; ++i) { + } + + TVector<TPeer> nodes; + for (ui32 i = 0; i < count; ++i) { TPeer node = {SelfId().NodeId(), InstanceId + "," + HostName(), 0, 0, 0}; - if (!Peers.empty()) { + if (!Peers.empty()) { auto FirstPeer = NextPeer; while (true) { if (NextPeer >= Peers.size()) { @@ -132,57 +132,57 @@ private: node = nextNode; break; } - } - } - nodes.push_back(node); - } - - auto req = MakeHolder<NDqs::TEvAllocateWorkersResponse>(); - req->Record.ClearError(); - auto& group = *req->Record.MutableNodes(); - group.SetResourceId(resourceId); - for (const auto& node : nodes) { - auto* worker = group.AddWorker(); - *worker->MutableGuid() = node.InstanceId; - worker->SetNodeId(node.NodeId); - } + } + } + nodes.push_back(node); + } + + auto req = MakeHolder<NDqs::TEvAllocateWorkersResponse>(); + req->Record.ClearError(); + auto& group = *req->Record.MutableNodes(); + group.SetResourceId(resourceId); + for (const auto& node : nodes) { + auto* worker = group.AddWorker(); + *worker->MutableGuid() = node.InstanceId; + worker->SetNodeId(node.NodeId); + } LOG_D("TEvAllocateWorkersResponse " << req->Record.DebugString()); - - Send(ev->Sender, req.Release()); - } - + + Send(ev->Sender, req.Release()); + } + void Handle(NDqs::TEvFreeWorkersNotify::TPtr&) { ServiceCounters.Counters->GetCounter("EvFreeWorkersNotify", true)->Inc(); - } - - STRICT_STFUNC( - StateFunc, - + } + + STRICT_STFUNC( + StateFunc, + hFunc(NActors::TEvents::TEvWakeup, HandleWakeup) hFunc(NDqs::TEvAllocateWorkersRequest, Handle) hFunc(NDqs::TEvFreeWorkersNotify, Handle) hFunc(NActors::TEvents::TEvUndelivered, OnUndelivered) - hFunc(TEvHealthNodesResponse, HandleResponse) - ) - + hFunc(TEvHealthNodesResponse, HandleResponse) + ) + void HandleWakeup(NActors::TEvents::TEvWakeup::TPtr& ev) { ServiceCounters.Counters->GetCounter("EvWakeup", true)->Inc(); - auto tag = ev->Get()->Tag; - switch (tag) { - case WU_NodesHealthCheck: + auto tag = ev->Get()->Tag; + switch (tag) { + case WU_NodesHealthCheck: NodesHealthCheck(); - break; - } - } - + break; + } + } + void NodesHealthCheck() { - const TDuration ttl = TDuration::Seconds(5); + const TDuration ttl = TDuration::Seconds(5); Schedule(ttl, new NActors::TEvents::TEvWakeup(WU_NodesHealthCheck)); - + ServiceCounters.Counters->GetCounter("NodesHealthCheck", true)->Inc(); - - Yq::Private::NodesHealthCheckRequest request; - request.set_tenant(Tenant); + + Yq::Private::NodesHealthCheckRequest request; + request.set_tenant(Tenant); auto& node = *request.mutable_node(); node.set_node_id(SelfId().NodeId()); node.set_instance_id(InstanceId); @@ -190,122 +190,122 @@ private: node.set_active_workers(AtomicGet(WorkerManagerCounters.ActiveWorkers->GetAtomic())); node.set_memory_limit(AtomicGet(WorkerManagerCounters.MkqlMemoryLimit->GetAtomic())); node.set_memory_allocated(AtomicGet(WorkerManagerCounters.MkqlMemoryAllocated->GetAtomic())); - node.set_interconnect_port(IcPort); - node.set_node_address(Address); - const auto actorSystem = NActors::TActivationContext::ActorSystem(); - const auto selfId = SelfId(); - Client - .NodesHealthCheck(std::move(request)) - .Subscribe([actorSystem, selfId](const auto& future) { - const auto& wrappedResult = future.GetValue(); - if (wrappedResult.IsResultSet()) { - actorSystem->Send(selfId, - new TEvHealthNodesResponse{wrappedResult.IsSuccess(), wrappedResult.GetStatus(), - wrappedResult.GetIssues(), wrappedResult.GetResult()}); - } - }); - } - + node.set_interconnect_port(IcPort); + node.set_node_address(Address); + const auto actorSystem = NActors::TActivationContext::ActorSystem(); + const auto selfId = SelfId(); + Client + .NodesHealthCheck(std::move(request)) + .Subscribe([actorSystem, selfId](const auto& future) { + const auto& wrappedResult = future.GetValue(); + if (wrappedResult.IsResultSet()) { + actorSystem->Send(selfId, + new TEvHealthNodesResponse{wrappedResult.IsSuccess(), wrappedResult.GetStatus(), + wrappedResult.GetIssues(), wrappedResult.GetResult()}); + } + }); + } + void OnUndelivered(NActors::TEvents::TEvUndelivered::TPtr&) { - LOG_E("TYqlNodesManagerActor::OnUndelivered"); + LOG_E("TYqlNodesManagerActor::OnUndelivered"); ServiceCounters.Counters->GetCounter("OnUndelivered", true)->Inc(); - } - - void HandleResponse(TEvHealthNodesResponse::TPtr& ev) { - try { - const auto& status = ev->Get()->Status; - THolder<TEvInterconnect::TEvNodesInfo> nameServiceUpdateReq(new TEvInterconnect::TEvNodesInfo()); - if (!ev->Get()->Success) { - ythrow yexception() << status << '\n' << ev->Get()->Issues.ToString(); - } - const auto& res = ev->Get()->Record; - - auto& nodesInfo = nameServiceUpdateReq->Nodes; - nodesInfo.reserve(res.nodes().size()); - - Peers.clear(); - for (const auto& node : res.nodes()) { + } + + void HandleResponse(TEvHealthNodesResponse::TPtr& ev) { + try { + const auto& status = ev->Get()->Status; + THolder<TEvInterconnect::TEvNodesInfo> nameServiceUpdateReq(new TEvInterconnect::TEvNodesInfo()); + if (!ev->Get()->Success) { + ythrow yexception() << status << '\n' << ev->Get()->Issues.ToString(); + } + const auto& res = ev->Get()->Record; + + auto& nodesInfo = nameServiceUpdateReq->Nodes; + nodesInfo.reserve(res.nodes().size()); + + Peers.clear(); + for (const auto& node : res.nodes()) { Peers.push_back({node.node_id(), node.instance_id() + "," + node.hostname(), node.active_workers(), node.memory_limit(), node.memory_allocated()}); - - if (node.interconnect_port()) { - nodesInfo.emplace_back(TEvInterconnect::TNodeInfo{ - node.node_id(), - node.node_address(), - node.hostname(), // host - node.hostname(), // resolveHost - static_cast<ui16>(node.interconnect_port()), - /* NodeLocation = */{}}); - } - } - + + if (node.interconnect_port()) { + nodesInfo.emplace_back(TEvInterconnect::TNodeInfo{ + node.node_id(), + node.node_address(), + node.hostname(), // host + node.hostname(), // resolveHost + static_cast<ui16>(node.interconnect_port()), + /* NodeLocation = */{}}); + } + } + ServiceCounters.Counters->GetCounter("PeerCount", false)->Set(Peers.size()); ServiceCounters.Counters->GetCounter("NodesHealthCheckOk", true)->Inc(); - - LOG_D("Send NodeInfo with size: " << nodesInfo.size() << " to DynamicNameserver"); - if (!nodesInfo.empty()) { - Send(GetNameserviceActorId(), nameServiceUpdateReq.Release()); - } - } catch (yexception &e) { - LOG_E(e.what()); + + LOG_D("Send NodeInfo with size: " << nodesInfo.size() << " to DynamicNameserver"); + if (!nodesInfo.empty()) { + Send(GetNameserviceActorId(), nameServiceUpdateReq.Release()); + } + } catch (yexception &e) { + LOG_E(e.what()); ServiceCounters.Counters->GetCounter("NodesHealthCheckFail", true)->Inc(); - } - } - -private: - NDqs::TWorkerManagerCounters WorkerManagerCounters; - TIntrusivePtr<ITimeProvider> TimeProvider; - TIntrusivePtr<IRandomProvider> RandomProvider; + } + } + +private: + NDqs::TWorkerManagerCounters WorkerManagerCounters; + TIntrusivePtr<ITimeProvider> TimeProvider; + TIntrusivePtr<IRandomProvider> RandomProvider; ::NYq::NCommon::TServiceCounters ServiceCounters; NConfig::TPrivateApiConfig PrivateApiConfig; - TString Tenant; + TString Tenant; ui64 MkqlInitialMemoryLimit; - + NYq::TYqSharedResources::TPtr YqSharedResources; - const ui32 IcPort; // Interconnect Port - const TString Address; - - TPrivateClient Client; - - struct TPeer { - ui32 NodeId; - TString InstanceId; + const ui32 IcPort; // Interconnect Port + const TString Address; + + TPrivateClient Client; + + struct TPeer { + ui32 NodeId; + TString InstanceId; ui64 ActiveWorkers; ui64 MemoryLimit; ui64 MemoryAllocated; - }; - TVector<TPeer> Peers; - ui32 ResourceIdPart = 0; - ui32 NextPeer = 0; - TString InstanceId; -}; - -TActorId MakeYqlNodesManagerId() { - constexpr TStringBuf name = "YQLNODESCTRL"; - return NActors::TActorId(0, name); -} - -TActorId MakeYqlNodesManagerHttpId() { - constexpr TStringBuf name = "YQLHTTPNODES"; - return NActors::TActorId(0, name); -} - -IActor* CreateYqlNodesManager( - const NDqs::TWorkerManagerCounters& workerManagerCounters, - TIntrusivePtr<ITimeProvider> timeProvider, - TIntrusivePtr<IRandomProvider> randomProvider, + }; + TVector<TPeer> Peers; + ui32 ResourceIdPart = 0; + ui32 NextPeer = 0; + TString InstanceId; +}; + +TActorId MakeYqlNodesManagerId() { + constexpr TStringBuf name = "YQLNODESCTRL"; + return NActors::TActorId(0, name); +} + +TActorId MakeYqlNodesManagerHttpId() { + constexpr TStringBuf name = "YQLHTTPNODES"; + return NActors::TActorId(0, name); +} + +IActor* CreateYqlNodesManager( + const NDqs::TWorkerManagerCounters& workerManagerCounters, + TIntrusivePtr<ITimeProvider> timeProvider, + TIntrusivePtr<IRandomProvider> randomProvider, const ::NYq::NCommon::TServiceCounters& serviceCounters, const NConfig::TPrivateApiConfig& privateApiConfig, const NYq::TYqSharedResources::TPtr& yqSharedResources, - const ui32& icPort, - const TString& address, + const ui32& icPort, + const TString& address, const TString& tenant, - ui64 mkqlInitialMemoryLimit, - const NMonitoring::TDynamicCounterPtr& clientCounters) { + ui64 mkqlInitialMemoryLimit, + const NMonitoring::TDynamicCounterPtr& clientCounters) { return new TYqlNodesManagerActor(yqSharedResources, workerManagerCounters, - timeProvider, randomProvider, - serviceCounters, privateApiConfig, icPort, address, tenant, mkqlInitialMemoryLimit, clientCounters); -} - -} // namespace NYq + timeProvider, randomProvider, + serviceCounters, privateApiConfig, icPort, address, tenant, mkqlInitialMemoryLimit, clientCounters); +} + +} // namespace NYq diff --git a/ydb/core/yq/libs/actors/nodes_manager.h b/ydb/core/yq/libs/actors/nodes_manager.h index a6a365a682..7301219b58 100644 --- a/ydb/core/yq/libs/actors/nodes_manager.h +++ b/ydb/core/yq/libs/actors/nodes_manager.h @@ -1,5 +1,5 @@ -#pragma once - +#pragma once + #include <ydb/core/yq/libs/common/service_counters.h> #include <ydb/core/yq/libs/events/events.h> #include <ydb/core/yq/libs/shared_resources/shared_resources.h> @@ -9,29 +9,29 @@ #include <ydb/library/yql/providers/dq/worker_manager/interface/counters.h> #include <ydb/library/yql/providers/dq/actors/proto_builder.h> #include <ydb/library/yql/providers/common/http_gateway/yql_http_gateway.h> - -#include <library/cpp/actors/core/actorsystem.h> -#include <library/cpp/time_provider/time_provider.h> -#include <library/cpp/random_provider/random_provider.h> - -namespace NYq { - -using namespace NActors; - -TActorId MakeYqlNodesManagerId(); -TActorId MakeYqlNodesManagerHttpId(); - -IActor* CreateYqlNodesManager( - const NYql::NDqs::TWorkerManagerCounters& workerManagerCounters, - TIntrusivePtr<ITimeProvider> timeProvider, - TIntrusivePtr<IRandomProvider> randomProvider, + +#include <library/cpp/actors/core/actorsystem.h> +#include <library/cpp/time_provider/time_provider.h> +#include <library/cpp/random_provider/random_provider.h> + +namespace NYq { + +using namespace NActors; + +TActorId MakeYqlNodesManagerId(); +TActorId MakeYqlNodesManagerHttpId(); + +IActor* CreateYqlNodesManager( + const NYql::NDqs::TWorkerManagerCounters& workerManagerCounters, + TIntrusivePtr<ITimeProvider> timeProvider, + TIntrusivePtr<IRandomProvider> randomProvider, const ::NYq::NCommon::TServiceCounters& serviceCounters, const NConfig::TPrivateApiConfig& privateApiConfig, const NYq::TYqSharedResources::TPtr& yqSharedResources, - const ui32& icPort, - const TString& address, + const ui32& icPort, + const TString& address, const TString& tenant = "", - ui64 mkqlInitialMemoryLimit = 0, - const NMonitoring::TDynamicCounterPtr& clientCounters = MakeIntrusive<NMonitoring::TDynamicCounters>()); - -} + ui64 mkqlInitialMemoryLimit = 0, + const NMonitoring::TDynamicCounterPtr& clientCounters = MakeIntrusive<NMonitoring::TDynamicCounters>()); + +} diff --git a/ydb/core/yq/libs/actors/pending_fetcher.cpp b/ydb/core/yq/libs/actors/pending_fetcher.cpp index f7cd877d37..0095963c8f 100644 --- a/ydb/core/yq/libs/actors/pending_fetcher.cpp +++ b/ydb/core/yq/libs/actors/pending_fetcher.cpp @@ -1,7 +1,7 @@ #include <ydb/core/yq/libs/config/protos/pinger.pb.h> #include <ydb/core/yq/libs/config/protos/yq_config.pb.h> #include "proxy.h" -#include "nodes_manager.h" +#include "nodes_manager.h" #include "database_resolver.h" @@ -64,27 +64,27 @@ #define LOG_D(stream) \ LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::YQL_PROXY, "Fetcher: " << stream) -namespace NYq { +namespace NYq { using namespace NActors; namespace { -struct TEvGetTaskInternalResponse : public NActors::TEventLocal<TEvGetTaskInternalResponse, NActors::TEvents::TSystem::Completed> { - bool Success = false; - const TIssues Issues; - const Yq::Private::GetTaskResult Result; - - TEvGetTaskInternalResponse( - bool success, - const TIssues& issues, - const Yq::Private::GetTaskResult& result) - : Success(success) - , Issues(issues) - , Result(result) - { } -}; - +struct TEvGetTaskInternalResponse : public NActors::TEventLocal<TEvGetTaskInternalResponse, NActors::TEvents::TSystem::Completed> { + bool Success = false; + const TIssues Issues; + const Yq::Private::GetTaskResult Result; + + TEvGetTaskInternalResponse( + bool success, + const TIssues& issues, + const Yq::Private::GetTaskResult& result) + : Success(success) + , Issues(issues) + , Result(result) + { } +}; + template <class TElement> TVector<TElement> VectorFromProto(const ::google::protobuf::RepeatedPtrField<TElement>& field) { return { field.begin(), field.end() }; @@ -108,8 +108,8 @@ public: const ::NYq::NCommon::TServiceCounters& serviceCounters, ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory, IHTTPGateway::TPtr s3Gateway, - ::NPq::NConfigurationManager::IConnections::TPtr pqCmConnections, - const NMonitoring::TDynamicCounterPtr& clientCounters + ::NPq::NConfigurationManager::IConnections::TPtr pqCmConnections, + const NMonitoring::TDynamicCounterPtr& clientCounters ) : YqSharedResources(yqSharedResources) , CommonConfig(commonConfig) @@ -125,22 +125,22 @@ public: , CredentialsFactory(credentialsFactory) , S3Gateway(s3Gateway) , PqCmConnections(std::move(pqCmConnections)) - , Guid(CreateGuidAsString()) - , ClientCounters(clientCounters) - , Client( + , Guid(CreateGuidAsString()) + , ClientCounters(clientCounters) + , Client( YqSharedResources->YdbDriver, - NYdb::TCommonClientSettings() + NYdb::TCommonClientSettings() .DiscoveryEndpoint(PrivateApiConfig.GetTaskServiceEndpoint()) - .Database(PrivateApiConfig.GetTaskServiceDatabase() ? PrivateApiConfig.GetTaskServiceDatabase() : TMaybe<TString>()), - ClientCounters) + .Database(PrivateApiConfig.GetTaskServiceDatabase() ? PrivateApiConfig.GetTaskServiceDatabase() : TMaybe<TString>()), + ClientCounters) { Y_ENSURE(GetYqlDefaultModuleResolverWithContext(ModuleResolver)); } - static constexpr char ActorName[] = "YQ_PENDING_FETCHER"; + static constexpr char ActorName[] = "YQ_PENDING_FETCHER"; void PassAway() final { - LOG_D("Stop Fetcher"); + LOG_D("Stop Fetcher"); Send(DatabaseResolver, new NActors::TEvents::TEvPoison()); NActors::IActor::PassAway(); } @@ -151,10 +151,10 @@ public: Y_UNUSED(ctx); DatabaseResolver = Register(CreateDatabaseResolver(MakeYqlAnalyticsHttpProxyId(), CredentialsFactory)); - Send(SelfId(), new NActors::TEvents::TEvWakeup()); + Send(SelfId(), new NActors::TEvents::TEvWakeup()); - LOG_I("STARTED"); - LogScope.ConstructInPlace(NActors::TActivationContext::ActorSystem(), NKikimrServices::YQL_PROXY, Guid); + LOG_I("STARTED"); + LogScope.ConstructInPlace(NActors::TActivationContext::ActorSystem(), NKikimrServices::YQL_PROXY, Guid); } private: @@ -173,66 +173,66 @@ private: } } - void HandleResponse(TEvGetTaskInternalResponse::TPtr& ev) { + void HandleResponse(TEvGetTaskInternalResponse::TPtr& ev) { HasRunningRequest = false; - LOG_D("Got GetTask response from PrivateApi"); - if (!ev->Get()->Success) { + LOG_D("Got GetTask response from PrivateApi"); + if (!ev->Get()->Success) { LOG_E("Error with GetTask: "<< ev->Get()->Issues.ToString()); - return; - } - - const auto& res = ev->Get()->Result; + return; + } - LOG_D("Tasks count: " << res.tasks().size()); - if (!res.tasks().empty()) { + const auto& res = ev->Get()->Result; + + LOG_D("Tasks count: " << res.tasks().size()); + if (!res.tasks().empty()) { ProcessTask(res); - HasRunningRequest = true; - GetPendingTask(); - } - } + HasRunningRequest = true; + GetPendingTask(); + } + } - void GetPendingTask() { + void GetPendingTask() { LOG_D("Request Private::GetTask" << ", Owner: " << Guid << ", Host: " << HostName()); - Yq::Private::GetTaskRequest request; - request.set_owner_id(Guid); - request.set_host(HostName()); - const auto actorSystem = NActors::TActivationContext::ActorSystem(); - const auto selfId = SelfId(); - Client - .GetTask(std::move(request)) - .Subscribe([actorSystem, selfId](const NThreading::TFuture<TGetTaskResult>& future) { - const auto& wrappedResult = future.GetValue(); - if (wrappedResult.IsResultSet()) { - actorSystem->Send(selfId, new TEvGetTaskInternalResponse( - wrappedResult.IsSuccess(), wrappedResult.GetIssues(), wrappedResult.GetResult()) - ); + Yq::Private::GetTaskRequest request; + request.set_owner_id(Guid); + request.set_host(HostName()); + const auto actorSystem = NActors::TActivationContext::ActorSystem(); + const auto selfId = SelfId(); + Client + .GetTask(std::move(request)) + .Subscribe([actorSystem, selfId](const NThreading::TFuture<TGetTaskResult>& future) { + const auto& wrappedResult = future.GetValue(); + if (wrappedResult.IsResultSet()) { + actorSystem->Send(selfId, new TEvGetTaskInternalResponse( + wrappedResult.IsSuccess(), wrappedResult.GetIssues(), wrappedResult.GetResult()) + ); } else { actorSystem->Send(selfId, new TEvGetTaskInternalResponse( - false, TIssues{{TIssue{"grpc private api result is not set for get task call"}}}, Yq::Private::GetTaskResult{}) + false, TIssues{{TIssue{"grpc private api result is not set for get task call"}}}, Yq::Private::GetTaskResult{}) ); } - }); + }); } - void ProcessTask(const Yq::Private::GetTaskResult& result) { - for (const auto& task : result.tasks()) { - RunTask(task); - } - - } - - void RunTask(const Yq::Private::GetTaskResult::Task& task) { + void ProcessTask(const Yq::Private::GetTaskResult& result) { + for (const auto& task : result.tasks()) { + RunTask(task); + } + + } + + void RunTask(const Yq::Private::GetTaskResult::Task& task) { LOG_D("NewTask:" - << " Scope: " << task.scope() - << " Id: " << task.query_id().value() - << " UserId: " << task.user_id() - << " AuthToken: " << NKikimr::MaskTicket(task.user_token())); - - THashMap<TString, TString> serviceAccounts; - for (const auto& identity : task.service_accounts()) { - serviceAccounts[identity.value()] = identity.signature(); - } - + << " Scope: " << task.scope() + << " Id: " << task.query_id().value() + << " UserId: " << task.user_id() + << " AuthToken: " << NKikimr::MaskTicket(task.user_token())); + + THashMap<TString, TString> serviceAccounts; + for (const auto& identity : task.service_accounts()) { + serviceAccounts[identity.value()] = identity.signature(); + } + TRunActorParams params( YqSharedResources->YdbDriver, S3Gateway, FunctionRegistry, RandomProvider, @@ -240,31 +240,31 @@ private: DqCompFactory, PqCmConnections, CommonConfig, CheckpointCoordinatorConfig, PrivateApiConfig, GatewaysConfig, PingerConfig, - task.text(), task.scope(), task.user_token(), - DatabaseResolver, task.query_id().value(), - task.user_id(), Guid, task.generation(), - VectorFromProto(task.connection()), - VectorFromProto(task.binding()), + task.text(), task.scope(), task.user_token(), + DatabaseResolver, task.query_id().value(), + task.user_id(), Guid, task.generation(), + VectorFromProto(task.connection()), + VectorFromProto(task.binding()), CredentialsFactory, serviceAccounts, - task.query_type(), - task.execute_mode(), + task.query_type(), + task.execute_mode(), GetEntityIdAsString(CommonConfig.GetIdsPrefix(), EEntityType::RESULT), - task.state_load_mode(), + task.state_load_mode(), task.disposition(), - task.status(), - task.sensor_labels().at("cloud_id"), - VectorFromProto(task.result_set_meta()), - VectorFromProto(task.dq_graph()), - task.dq_graph_index(), - VectorFromProto(task.created_topic_consumers()), - task.automatic(), + task.status(), + task.sensor_labels().at("cloud_id"), + VectorFromProto(task.result_set_meta()), + VectorFromProto(task.dq_graph()), + task.dq_graph_index(), + VectorFromProto(task.created_topic_consumers()), + task.automatic(), task.query_name(), - NProtoInterop::CastFromProto(task.deadline()), - ClientCounters); - + NProtoInterop::CastFromProto(task.deadline()), + ClientCounters); + NDq::SetYqlLogLevels(NActors::NLog::PRI_TRACE); - Register(CreateRunActor(ServiceCounters, std::move(params))); + Register(CreateRunActor(ServiceCounters, std::move(params))); } STRICT_STFUNC( @@ -272,7 +272,7 @@ private: HFunc(NActors::TEvents::TEvWakeup, HandleWakeup) HFunc(NActors::TEvents::TEvUndelivered, OnUndelivered) - hFunc(TEvGetTaskInternalResponse, HandleResponse) + hFunc(TEvGetTaskInternalResponse, HandleResponse) ); NYq::TYqSharedResources::TPtr YqSharedResources; @@ -299,12 +299,12 @@ private: ISecuredServiceAccountCredentialsFactory::TPtr CredentialsFactory; const IHTTPGateway::TPtr S3Gateway; const ::NPq::NConfigurationManager::IConnections::TPtr PqCmConnections; - - const TString Guid; //OwnerId - const NMonitoring::TDynamicCounterPtr ClientCounters; - TPrivateClient Client; - - TMaybe<NYql::NLog::TScopedBackend<NYql::NDq::TYqlLogScope>> LogScope; + + const TString Guid; //OwnerId + const NMonitoring::TDynamicCounterPtr ClientCounters; + TPrivateClient Client; + + TMaybe<NYql::NLog::TScopedBackend<NYql::NDq::TYqlLogScope>> LogScope; }; @@ -322,8 +322,8 @@ NActors::IActor* CreatePendingFetcher( const ::NYq::NCommon::TServiceCounters& serviceCounters, ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory, IHTTPGateway::TPtr s3Gateway, - ::NPq::NConfigurationManager::IConnections::TPtr pqCmConnections, - const NMonitoring::TDynamicCounterPtr& clientCounters) + ::NPq::NConfigurationManager::IConnections::TPtr pqCmConnections, + const NMonitoring::TDynamicCounterPtr& clientCounters) { return new TYqlPendingFetcher( yqSharedResources, @@ -339,8 +339,8 @@ NActors::IActor* CreatePendingFetcher( serviceCounters, credentialsFactory, s3Gateway, - std::move(pqCmConnections), - clientCounters); + std::move(pqCmConnections), + clientCounters); } TActorId MakeYqlAnalyticsFetcherId(ui32 nodeId) { @@ -348,4 +348,4 @@ TActorId MakeYqlAnalyticsFetcherId(ui32 nodeId) { return NActors::TActorId(nodeId, name); } -} /* NYq */ +} /* NYq */ diff --git a/ydb/core/yq/libs/actors/pinger.cpp b/ydb/core/yq/libs/actors/pinger.cpp index 0978e42de6..55fc80d6a0 100644 --- a/ydb/core/yq/libs/actors/pinger.cpp +++ b/ydb/core/yq/libs/actors/pinger.cpp @@ -28,10 +28,10 @@ #define LOG_T(stream) \ LOG_TRACE_S(*TlsActivationContext, NKikimrServices::YQL_PROXY, "Pinger - " << "QueryId: " << Id << ", Owner: " << OwnerId << " " << stream) -namespace NYq { +namespace NYq { using namespace NActors; -using namespace NYql; +using namespace NYql; struct TEvPingResponse : public NActors::TEventLocal<TEvPingResponse, NActors::TEvents::TSystem::CallbackCompletion> { TPingTaskResult Result; @@ -59,12 +59,12 @@ private: class TPingerActor : public NActors::TActorBootstrapped<TPingerActor> { class TRetryState { public: - void Init(const TInstant& now, const TInstant& startLeaseTime, const TDuration& maxRetryTime) { + void Init(const TInstant& now, const TInstant& startLeaseTime, const TDuration& maxRetryTime) { StartRequestTime = now; StartLeaseTime = startLeaseTime; Delay = TDuration::Zero(); RetriesCount = 0; - MaxRetryTime = maxRetryTime; + MaxRetryTime = maxRetryTime; } void UpdateStartLeaseTime(TInstant startLeaseTime) { @@ -111,7 +111,7 @@ class TPingerActor : public NActors::TActorBootstrapped<TPingerActor> { TInstant StartRequestTime; size_t RetriesCount = 0; - TDuration MaxRetryTime; + TDuration MaxRetryTime; static constexpr TDuration MaxDelay = TDuration::Seconds(5); static constexpr TDuration MinDelay = TDuration::MilliSeconds(100); // from second retry }; @@ -134,9 +134,9 @@ class TPingerActor : public NActors::TActorBootstrapped<TPingerActor> { TConfig(const NConfig::TPingerConfig& config) : Proto(config) { - if (Proto.GetPingPeriod()) { - Y_VERIFY(TDuration::TryParse(Proto.GetPingPeriod(), PingPeriod)); - } + if (Proto.GetPingPeriod()) { + Y_VERIFY(TDuration::TryParse(Proto.GetPingPeriod(), PingPeriod)); + } } }; @@ -161,7 +161,7 @@ public: { } - static constexpr char ActorName[] = "YQ_PINGER"; + static constexpr char ActorName[] = "YQ_PINGER"; void Bootstrap() { LOG_D("Start Pinger"); @@ -175,7 +175,7 @@ private: StateFunc, cFunc(NActors::TEvents::TEvPoison::EventType, PassAway) hFunc(NActors::TEvents::TEvWakeup, Wakeup) - hFunc(TEvPingResponse, Handle) + hFunc(TEvPingResponse, Handle) hFunc(TEvents::TEvForwardPingRequest, Handle) ) @@ -187,7 +187,7 @@ private: void ScheduleNextPing() { if (!Finishing) { SchedulerCookieHolder.Reset(ISchedulerCookie::Make2Way()); - Schedule(Config.PingPeriod, new NActors::TEvents::TEvWakeup(ContinueLeaseWakeupTag), SchedulerCookieHolder.Get()); + Schedule(Config.PingPeriod, new NActors::TEvents::TEvWakeup(ContinueLeaseWakeupTag), SchedulerCookieHolder.Get()); } } @@ -275,7 +275,7 @@ private: return false; } - void Handle(TEvPingResponse::TPtr& ev) { + void Handle(TEvPingResponse::TPtr& ev) { if (FatalError) { LOG_D("Got ping response after fatal error. Ignore"); return; @@ -374,14 +374,14 @@ private: if (!retry) { RetryState.Init(TActivationContext::Now(), StartLeaseTime, Config.PingPeriod); } - Ping(Yq::Private::PingTaskRequest(), ContinueLeaseRequestCookie); + Ping(Yq::Private::PingTaskRequest(), ContinueLeaseRequestCookie); } - void Ping(Yq::Private::PingTaskRequest request, ui64 cookie) { + void Ping(Yq::Private::PingTaskRequest request, ui64 cookie) { // Fill ids - request.set_scope(Scope.ToString()); - request.set_owner_id(OwnerId); - request.mutable_query_id()->set_value(Id); + request.set_scope(Scope.ToString()); + request.set_owner_id(OwnerId); + request.mutable_query_id()->set_value(Id); *request.mutable_deadline() = NProtoInterop::CastToProto(Deadline); const auto* actorSystem = NActors::TActivationContext::ActorSystem(); @@ -400,7 +400,7 @@ private: << CurrentExceptionMessage()); } actorSystem->Send(new IEventHandle(selfId, selfId, ev.release(), 0, cookie)); - } + } ); } @@ -454,4 +454,4 @@ IActor* CreatePingerActor( deadline); } -} /* NYq */ +} /* NYq */ diff --git a/ydb/core/yq/libs/actors/proxy.cpp b/ydb/core/yq/libs/actors/proxy.cpp index 76da02bcc2..f06f2f8475 100644 --- a/ydb/core/yq/libs/actors/proxy.cpp +++ b/ydb/core/yq/libs/actors/proxy.cpp @@ -1,6 +1,6 @@ #include "proxy.h" -namespace NYq { +namespace NYq { using namespace NActors; @@ -9,4 +9,4 @@ NActors::TActorId MakeYqlAnalyticsHttpProxyId() { return NActors::TActorId(0, name); } -} // namespace NYq +} // namespace NYq diff --git a/ydb/core/yq/libs/actors/proxy.h b/ydb/core/yq/libs/actors/proxy.h index 2e274ffed4..442d29ac44 100644 --- a/ydb/core/yq/libs/actors/proxy.h +++ b/ydb/core/yq/libs/actors/proxy.h @@ -1,6 +1,6 @@ #pragma once #include <ydb/core/yq/libs/config/protos/pinger.pb.h> -#include "run_actor_params.h" +#include "run_actor_params.h" #include <util/datetime/base.h> #include <ydb/core/yq/libs/events/events.h> @@ -28,7 +28,7 @@ namespace NKikimr { } } -namespace NYq { +namespace NYq { NActors::TActorId MakeYqlAnalyticsHttpProxyId(); NActors::TActorId MakeYqlAnalyticsFetcherId(ui32 nodeId); @@ -45,10 +45,10 @@ NActors::IActor* CreatePendingFetcher( TIntrusivePtr<IRandomProvider> randomProvider, NKikimr::NMiniKQL::TComputationNodeFactory dqCompFactory, const ::NYq::NCommon::TServiceCounters& serviceCounters, - NYql::ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory, + NYql::ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory, NYql::IHTTPGateway::TPtr s3Gateway, - ::NPq::NConfigurationManager::IConnections::TPtr pqCmConnections, - const NMonitoring::TDynamicCounterPtr& clientCounters + ::NPq::NConfigurationManager::IConnections::TPtr pqCmConnections, + const NMonitoring::TDynamicCounterPtr& clientCounters ); NActors::IActor* CreateRunActor( @@ -59,8 +59,8 @@ NActors::IActor* CreateRunActor( struct TResultId { TString Id; int SetId; - TString HistoryId; - TString Owner; + TString HistoryId; + TString Owner; TString CloudId; }; @@ -72,20 +72,20 @@ NActors::IActor* CreateResultWriter( const TResultId& resultId, const TVector<TString>& columns, const TString& traceId, - const TInstant& deadline, - const NMonitoring::TDynamicCounterPtr& clientCounters - ); + const TInstant& deadline, + const NMonitoring::TDynamicCounterPtr& clientCounters + ); -NActors::IActor* CreatePingerActor( +NActors::IActor* CreatePingerActor( const TScope& scope, const TString& userId, const TString& id, const TString& owner, - const NYq::TPrivateClient& client, + const NYq::TPrivateClient& client, const NActors::TActorId parent, const NConfig::TPingerConfig& config, const TInstant& deadline); TString MakeInternalError(const TString& text); -} /* NYq */ +} /* NYq */ diff --git a/ydb/core/yq/libs/actors/proxy_private.cpp b/ydb/core/yq/libs/actors/proxy_private.cpp index 2e3d60a42a..f852894578 100644 --- a/ydb/core/yq/libs/actors/proxy_private.cpp +++ b/ydb/core/yq/libs/actors/proxy_private.cpp @@ -1,120 +1,120 @@ -#include "proxy_private.h" +#include "proxy_private.h" #include <ydb/core/yq/libs/config/protos/yq_config.pb.h> - + #include <ydb/core/yq/libs/events/events.h> #include <ydb/core/yq/libs/shared_resources/db_pool.h> -#include <library/cpp/actors/core/events.h> -#include <library/cpp/actors/core/hfunc.h> -#include <library/cpp/actors/core/actor_bootstrapped.h> +#include <library/cpp/actors/core/events.h> +#include <library/cpp/actors/core/hfunc.h> +#include <library/cpp/actors/core/actor_bootstrapped.h> #include <ydb/core/protos/services.pb.h> - -#include <library/cpp/actors/core/log.h> - -#include <util/generic/deque.h> -#include <util/generic/guid.h> -#include <util/system/hostname.h> - + +#include <library/cpp/actors/core/log.h> + +#include <util/generic/deque.h> +#include <util/generic/guid.h> +#include <util/system/hostname.h> + #define LOG_E(stream) \ LOG_ERROR_S(*TlsActivationContext, NKikimrServices::YQL_PRIVATE_PROXY, stream) #define LOG_I(stream) \ LOG_INFO_S(*TlsActivationContext, NKikimrServices::YQL_PRIVATE_PROXY, stream) #define LOG_D(stream) \ LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::YQL_PRIVATE_PROXY, stream) - -namespace NYq { - -using namespace NActors; -using namespace NMonitoring; - -class TYqlAnalyticsPrivateProxy : public NActors::TActorBootstrapped<TYqlAnalyticsPrivateProxy> { -public: - TYqlAnalyticsPrivateProxy( - TIntrusivePtr<ITimeProvider> timeProvider, - TIntrusivePtr<IRandomProvider> randomProvider, - NMonitoring::TDynamicCounterPtr counters, + +namespace NYq { + +using namespace NActors; +using namespace NMonitoring; + +class TYqlAnalyticsPrivateProxy : public NActors::TActorBootstrapped<TYqlAnalyticsPrivateProxy> { +public: + TYqlAnalyticsPrivateProxy( + TIntrusivePtr<ITimeProvider> timeProvider, + TIntrusivePtr<IRandomProvider> randomProvider, + NMonitoring::TDynamicCounterPtr counters, const NConfig::TTokenAccessorConfig& tokenAccessorConfig) : TokenAccessorConfig(tokenAccessorConfig) , TimeProvider(timeProvider) - , RandomProvider(randomProvider) - , Counters(counters->GetSubgroup("subsystem", "private_api")) - { } - - static constexpr char ActorName[] = "YQ_PRIVATE_PROXY"; - - void PassAway() final { - NActors::IActor::PassAway(); - } - - void Bootstrap(const TActorContext&) { - Become(&TYqlAnalyticsPrivateProxy::StateFunc); - Counters->GetCounter("EvBootstrap", true)->Inc(); - } - -private: - void Handle(TEvents::TEvPingTaskRequest::TPtr& ev, const TActorContext& ctx) { - Counters->GetCounter("EvPingTaskRequest", true)->Inc(); - - Register( + , RandomProvider(randomProvider) + , Counters(counters->GetSubgroup("subsystem", "private_api")) + { } + + static constexpr char ActorName[] = "YQ_PRIVATE_PROXY"; + + void PassAway() final { + NActors::IActor::PassAway(); + } + + void Bootstrap(const TActorContext&) { + Become(&TYqlAnalyticsPrivateProxy::StateFunc); + Counters->GetCounter("EvBootstrap", true)->Inc(); + } + +private: + void Handle(TEvents::TEvPingTaskRequest::TPtr& ev, const TActorContext& ctx) { + Counters->GetCounter("EvPingTaskRequest", true)->Inc(); + + Register( CreatePingTaskRequestActor(ev->Sender, TimeProvider, ev->Release(), Counters), - NActors::TMailboxType::HTSwap, ctx.SelfID.PoolID()); - } - - void Handle(TEvents::TEvGetTaskRequest::TPtr& ev, const TActorContext& ctx) { - Counters->GetCounter("EvGetTaskRequest", true)->Inc(); - Register( + NActors::TMailboxType::HTSwap, ctx.SelfID.PoolID()); + } + + void Handle(TEvents::TEvGetTaskRequest::TPtr& ev, const TActorContext& ctx) { + Counters->GetCounter("EvGetTaskRequest", true)->Inc(); + Register( CreateGetTaskRequestActor(ev->Sender, TokenAccessorConfig, TimeProvider, ev->Release(), Counters), - NActors::TMailboxType::HTSwap, ctx.SelfID.PoolID()); - } - - void Handle(TEvents::TEvWriteTaskResultRequest::TPtr& ev, const TActorContext& ctx) { - Counters->GetCounter("EvWriteTaskResultRequest", true)->Inc(); - - Register( + NActors::TMailboxType::HTSwap, ctx.SelfID.PoolID()); + } + + void Handle(TEvents::TEvWriteTaskResultRequest::TPtr& ev, const TActorContext& ctx) { + Counters->GetCounter("EvWriteTaskResultRequest", true)->Inc(); + + Register( CreateWriteTaskResultRequestActor(ev->Sender, TimeProvider, ev->Release(), Counters), - NActors::TMailboxType::HTSwap, ctx.SelfID.PoolID()); - } - - void Handle(TEvents::TEvNodesHealthCheckRequest::TPtr& ev, const TActorContext& ctx) { - Counters->GetCounter("EvNodesHealthCheckRequest", true)->Inc(); - - Register( + NActors::TMailboxType::HTSwap, ctx.SelfID.PoolID()); + } + + void Handle(TEvents::TEvNodesHealthCheckRequest::TPtr& ev, const TActorContext& ctx) { + Counters->GetCounter("EvNodesHealthCheckRequest", true)->Inc(); + + Register( CreateNodesHealthCheckActor(ev->Sender, TimeProvider, ev->Release(), Counters), - NActors::TMailboxType::HTSwap, ctx.SelfID.PoolID()); - } - - STRICT_STFUNC( - StateFunc, - HFunc(NActors::TEvents::TEvUndelivered, OnUndelivered) - HFunc(TEvents::TEvPingTaskRequest, Handle) - HFunc(TEvents::TEvGetTaskRequest, Handle) - HFunc(TEvents::TEvWriteTaskResultRequest, Handle) - HFunc(TEvents::TEvNodesHealthCheckRequest, Handle) - ) - - void OnUndelivered(NActors::TEvents::TEvUndelivered::TPtr&, const NActors::TActorContext&) { - LOG_E("TYqlAnalyticsPrivateProxy::OnUndelivered"); - Counters->GetCounter("OnUndelivered", true)->Inc(); - } - -private: + NActors::TMailboxType::HTSwap, ctx.SelfID.PoolID()); + } + + STRICT_STFUNC( + StateFunc, + HFunc(NActors::TEvents::TEvUndelivered, OnUndelivered) + HFunc(TEvents::TEvPingTaskRequest, Handle) + HFunc(TEvents::TEvGetTaskRequest, Handle) + HFunc(TEvents::TEvWriteTaskResultRequest, Handle) + HFunc(TEvents::TEvNodesHealthCheckRequest, Handle) + ) + + void OnUndelivered(NActors::TEvents::TEvUndelivered::TPtr&, const NActors::TActorContext&) { + LOG_E("TYqlAnalyticsPrivateProxy::OnUndelivered"); + Counters->GetCounter("OnUndelivered", true)->Inc(); + } + +private: const NConfig::TTokenAccessorConfig TokenAccessorConfig; - TIntrusivePtr<ITimeProvider> TimeProvider; - TIntrusivePtr<IRandomProvider> RandomProvider; - NMonitoring::TDynamicCounterPtr Counters; -}; - -TActorId MakeYqPrivateProxyId() { - constexpr TStringBuf name = "YQPRIVPROXY"; - return NActors::TActorId(0, name); -} - -IActor* CreateYqlAnalyticsPrivateProxy( - TIntrusivePtr<ITimeProvider> timeProvider, - TIntrusivePtr<IRandomProvider> randomProvider, - NMonitoring::TDynamicCounterPtr counters, + TIntrusivePtr<ITimeProvider> TimeProvider; + TIntrusivePtr<IRandomProvider> RandomProvider; + NMonitoring::TDynamicCounterPtr Counters; +}; + +TActorId MakeYqPrivateProxyId() { + constexpr TStringBuf name = "YQPRIVPROXY"; + return NActors::TActorId(0, name); +} + +IActor* CreateYqlAnalyticsPrivateProxy( + TIntrusivePtr<ITimeProvider> timeProvider, + TIntrusivePtr<IRandomProvider> randomProvider, + NMonitoring::TDynamicCounterPtr counters, const NConfig::TTokenAccessorConfig& tokenAccessorConfig) { return new TYqlAnalyticsPrivateProxy(timeProvider, randomProvider, counters, tokenAccessorConfig); -} - -} // namespace NYq +} + +} // namespace NYq diff --git a/ydb/core/yq/libs/actors/proxy_private.h b/ydb/core/yq/libs/actors/proxy_private.h index fbd17a7aee..f5378ebab8 100644 --- a/ydb/core/yq/libs/actors/proxy_private.h +++ b/ydb/core/yq/libs/actors/proxy_private.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include <ydb/core/yq/libs/shared_resources/db_pool.h> #include <ydb/core/yq/libs/events/events.h> @@ -6,56 +6,56 @@ #include <ydb/library/yql/providers/dq/provider/yql_dq_gateway.h> #include <ydb/library/yql/providers/dq/worker_manager/interface/counters.h> #include <ydb/library/yql/providers/dq/actors/proto_builder.h> - + #include <library/cpp/actors/core/actorsystem.h> #include <library/cpp/time_provider/time_provider.h> #include <library/cpp/random_provider/random_provider.h> -#include <library/cpp/monlib/metrics/histogram_collector.h> - -namespace NKikimr { - namespace NMiniKQL { - class IFunctionRegistry; - } -} - -namespace NYq { - -NActors::TActorId MakeYqPrivateProxyId(); - -NActors::IActor* CreateYqlAnalyticsPrivateProxy( - TIntrusivePtr<ITimeProvider> timeProvider, - TIntrusivePtr<IRandomProvider> randomProvider, - NMonitoring::TDynamicCounterPtr counters, +#include <library/cpp/monlib/metrics/histogram_collector.h> + +namespace NKikimr { + namespace NMiniKQL { + class IFunctionRegistry; + } +} + +namespace NYq { + +NActors::TActorId MakeYqPrivateProxyId(); + +NActors::IActor* CreateYqlAnalyticsPrivateProxy( + TIntrusivePtr<ITimeProvider> timeProvider, + TIntrusivePtr<IRandomProvider> randomProvider, + NMonitoring::TDynamicCounterPtr counters, const NConfig::TTokenAccessorConfig& tockenAccessorConfig -); - -NActors::IActor* CreatePingTaskRequestActor( - const NActors::TActorId& sender, - TIntrusivePtr<ITimeProvider> timeProvider, - TAutoPtr<TEvents::TEvPingTaskRequest> ev, - NMonitoring::TDynamicCounterPtr counters -); - -NActors::IActor* CreateGetTaskRequestActor( - const NActors::TActorId& sender, +); + +NActors::IActor* CreatePingTaskRequestActor( + const NActors::TActorId& sender, + TIntrusivePtr<ITimeProvider> timeProvider, + TAutoPtr<TEvents::TEvPingTaskRequest> ev, + NMonitoring::TDynamicCounterPtr counters +); + +NActors::IActor* CreateGetTaskRequestActor( + const NActors::TActorId& sender, const NConfig::TTokenAccessorConfig& tockenAccessorConfig, - TIntrusivePtr<ITimeProvider> timeProvider, - TAutoPtr<TEvents::TEvGetTaskRequest> ev, - NMonitoring::TDynamicCounterPtr counters -); - -NActors::IActor* CreateWriteTaskResultRequestActor( - const NActors::TActorId& sender, - TIntrusivePtr<ITimeProvider> timeProvider, - TAutoPtr<TEvents::TEvWriteTaskResultRequest> ev, - NMonitoring::TDynamicCounterPtr counters -); - -NActors::IActor* CreateNodesHealthCheckActor( - const NActors::TActorId& sender, - TIntrusivePtr<ITimeProvider> timeProvider, - TAutoPtr<TEvents::TEvNodesHealthCheckRequest> ev, - NMonitoring::TDynamicCounterPtr counters -); - -} /* NYq */ + TIntrusivePtr<ITimeProvider> timeProvider, + TAutoPtr<TEvents::TEvGetTaskRequest> ev, + NMonitoring::TDynamicCounterPtr counters +); + +NActors::IActor* CreateWriteTaskResultRequestActor( + const NActors::TActorId& sender, + TIntrusivePtr<ITimeProvider> timeProvider, + TAutoPtr<TEvents::TEvWriteTaskResultRequest> ev, + NMonitoring::TDynamicCounterPtr counters +); + +NActors::IActor* CreateNodesHealthCheckActor( + const NActors::TActorId& sender, + TIntrusivePtr<ITimeProvider> timeProvider, + TAutoPtr<TEvents::TEvNodesHealthCheckRequest> ev, + NMonitoring::TDynamicCounterPtr counters +); + +} /* NYq */ diff --git a/ydb/core/yq/libs/actors/result_writer.cpp b/ydb/core/yq/libs/actors/result_writer.cpp index d1ef15c428..d51b8aae9f 100644 --- a/ydb/core/yq/libs/actors/result_writer.cpp +++ b/ydb/core/yq/libs/actors/result_writer.cpp @@ -27,7 +27,7 @@ #define LOG_D(stream) \ LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::YQL_PROXY, "Writer: " << TraceId << ": " << stream) -namespace NYq { +namespace NYq { using namespace NActors; using namespace NYql; @@ -43,22 +43,22 @@ public: const TResultId& resultId, const TVector<TString>& columns, const TString& traceId, - const TInstant& deadline, - const NMonitoring::TDynamicCounterPtr& clientCounters) + const TInstant& deadline, + const NMonitoring::TDynamicCounterPtr& clientCounters) : ExecuterId(executerId) , ResultBuilder(MakeHolder<TProtoBuilder>(resultType, columns)) , ResultId({resultId}) , TraceId(traceId) , Deadline(deadline) - , Client( + , Client( driver, - NYdb::TCommonClientSettings() + NYdb::TCommonClientSettings() .DiscoveryEndpoint(privateApiConfig.GetTaskServiceEndpoint()) - .Database(privateApiConfig.GetTaskServiceDatabase() ? privateApiConfig.GetTaskServiceDatabase(): TMaybe<TString>()), - clientCounters) + .Database(privateApiConfig.GetTaskServiceDatabase() ? privateApiConfig.GetTaskServiceDatabase(): TMaybe<TString>()), + clientCounters) { } - static constexpr char ActorName[] = "YQ_RESULT_WRITER"; + static constexpr char ActorName[] = "YQ_RESULT_WRITER"; void Bootstrap(const TActorContext&) { LOG_I("Bootstrap"); @@ -84,20 +84,20 @@ private: } void OnUndelivered(NActors::TEvents::TEvUndelivered::TPtr&, const NActors::TActorContext& ) { - auto req = MakeHolder<TEvDqFailure>(TIssue("Undelivered").SetCode(NYql::DEFAULT_ERROR, TSeverityIds::S_ERROR), /*retriable=*/ false, /*needFallback=*/false); + auto req = MakeHolder<TEvDqFailure>(TIssue("Undelivered").SetCode(NYql::DEFAULT_ERROR, TSeverityIds::S_ERROR), /*retriable=*/ false, /*needFallback=*/false); Send(ExecuterId, req.Release()); HasError = true; } - void SendIssuesAndSetErrorFlag(const TIssues& issues) { - LOG_E("ControlPlane WriteResult Issues: " << issues.ToString()); - Issues.AddIssues(issues); - HasError = true; - auto req = MakeHolder<TEvDqFailure>(); - IssuesToMessage(issues, req->Record.MutableIssues()); - Send(ExecuterId, req.Release()); - } - + void SendIssuesAndSetErrorFlag(const TIssues& issues) { + LOG_E("ControlPlane WriteResult Issues: " << issues.ToString()); + Issues.AddIssues(issues); + HasError = true; + auto req = MakeHolder<TEvDqFailure>(); + IssuesToMessage(issues, req->Record.MutableIssues()); + Send(ExecuterId, req.Release()); + } + void MaybeFinish() { if (Finished && Requests.empty()) { Send(ExecuterId, new TEvGraphFinished()); @@ -106,7 +106,7 @@ private: void OnQueryResult(TEvQueryResponse::TPtr& ev, const TActorContext&) { Finished = true; - NYql::NDqProto::TQueryResponse queryResult(ev->Get()->Record); + NYql::NDqProto::TQueryResponse queryResult(ev->Get()->Record); *queryResult.MutableYson() = ResultBuilder->BuildYson(Head); if (!Issues.Empty()) { @@ -122,50 +122,50 @@ private: void HandleResponse(NYq::TEvControlPlaneStorage::TEvWriteResultDataResponse::TPtr& ev) { const auto& issues = ev->Get()->Issues; - auto it = Requests.find(ev->Get()->RequestId); - if (issues) { - SendIssuesAndSetErrorFlag(issues); - return; - } - + auto it = Requests.find(ev->Get()->RequestId); + if (issues) { + SendIssuesAndSetErrorFlag(issues); + return; + } + if (it == Requests.end()) { HasError = true; - auto req = MakeHolder<TEvDqFailure>(TIssue("Unknown RequestId").SetCode(NYql::DEFAULT_ERROR, TSeverityIds::S_ERROR), /*retriable=*/ false, /*needFallback=*/false); + auto req = MakeHolder<TEvDqFailure>(TIssue("Unknown RequestId").SetCode(NYql::DEFAULT_ERROR, TSeverityIds::S_ERROR), /*retriable=*/ false, /*needFallback=*/false); Send(ExecuterId, req.Release()); return; } auto& request = it->second; - if (request.MessagesNum > 0) - --request.MessagesNum; - - --InflightCounter; - - if (request.MessagesNum == 0) { - FreeSpace += request.Size; - - if (FreeSpace > 0) { - auto res = MakeHolder<NDq::TEvDqCompute::TEvChannelDataAck>(); - res->Record.SetChannelId(request.ChannelId); - res->Record.SetFreeSpace(FreeSpace); - res->Record.SetSeqNo(request.SeqNo); - res->Record.SetFinish(HasError); - Send(request.Sender, res.Release()); - - auto duration = (TInstant::Now()-StartTime); - - LOG_D("ChannelData, Records: " << RowIndex - << " HasError: " << HasError - << " Size: " << Size - << " Rows: " << Rows - << " FreeSpace: " << FreeSpace - << " Duration: " << duration - << " AvgSpeed: " << Size/(duration.Seconds()+1)/1024/1024); - } - - Requests.erase(it); - MaybeFinish(); + if (request.MessagesNum > 0) + --request.MessagesNum; + + --InflightCounter; + + if (request.MessagesNum == 0) { + FreeSpace += request.Size; + + if (FreeSpace > 0) { + auto res = MakeHolder<NDq::TEvDqCompute::TEvChannelDataAck>(); + res->Record.SetChannelId(request.ChannelId); + res->Record.SetFreeSpace(FreeSpace); + res->Record.SetSeqNo(request.SeqNo); + res->Record.SetFinish(HasError); + Send(request.Sender, res.Release()); + + auto duration = (TInstant::Now()-StartTime); + + LOG_D("ChannelData, Records: " << RowIndex + << " HasError: " << HasError + << " Size: " << Size + << " Rows: " << Rows + << " FreeSpace: " << FreeSpace + << " Duration: " << duration + << " AvgSpeed: " << Size/(duration.Seconds()+1)/1024/1024); + } + + Requests.erase(it); + MaybeFinish(); } - SendResult(); // Send remaining rows + SendResult(); // Send remaining rows } void StopChannel(NDq::TEvDqCompute::TEvChannelData::TPtr& ev) { @@ -177,82 +177,82 @@ private: Send(ev->Sender, res.Release()); } - Yq::Private::WriteTaskResultRequest CreateProtoRequestWithoutResultSet(ui64 startRowIndex) { - Yq::Private::WriteTaskResultRequest protoReq; - protoReq.set_owner_id(ResultId.Owner); - protoReq.mutable_result_id()->set_value(ResultId.Id); - protoReq.set_offset(startRowIndex); - protoReq.set_result_set_id(ResultId.SetId); - protoReq.set_request_id(Cookie); + Yq::Private::WriteTaskResultRequest CreateProtoRequestWithoutResultSet(ui64 startRowIndex) { + Yq::Private::WriteTaskResultRequest protoReq; + protoReq.set_owner_id(ResultId.Owner); + protoReq.mutable_result_id()->set_value(ResultId.Id); + protoReq.set_offset(startRowIndex); + protoReq.set_result_set_id(ResultId.SetId); + protoReq.set_request_id(Cookie); *protoReq.mutable_deadline() = NProtoInterop::CastToProto(Deadline); - return protoReq; - } - - void SendResult() { - if (InflightCounter || CurChunkInd >= ResultChunks.size()) { - return; - } - ++InflightCounter; - const auto actorSystem = NActors::TActivationContext::ActorSystem(); - const auto selfId = SelfId(); - Client - .WriteTaskResult(std::move(ResultChunks[CurChunkInd++])) - .Subscribe([actorSystem, selfId](const auto& future) { - try { - const auto& wrappedResult = future.GetValue(); - if (wrappedResult.IsResultSet()) { - actorSystem->Send(selfId, - new NYq::TEvControlPlaneStorage::TEvWriteResultDataResponse(wrappedResult.GetIssues(), wrappedResult.GetResult().request_id())); - } else { - auto issues = wrappedResult.GetIssues(); - issues.AddIssue("Error on writing result"); - actorSystem->Send(selfId, - new NYq::TEvControlPlaneStorage::TEvWriteResultDataResponse(issues, UINT64_MAX)); - } - } catch(...) { - TIssues issues; - issues.AddIssue(CurrentExceptionMessage()); - actorSystem->Send(selfId, - new NYq::TEvControlPlaneStorage::TEvWriteResultDataResponse(issues, UINT64_MAX)); - } - }); - } - - void ConstructResults(const Ydb::ResultSet& resultSet, ui64 startRowIndex) { - const size_t baseProtoByteSize = CreateProtoRequestWithoutResultSet(startRowIndex).ByteSizeLong(); - - NYq::TRowsProtoSplitter rowsSplitter(resultSet, ProtoMessageLimit, baseProtoByteSize, MaxRowsCountPerChunk); - const auto splittedResultSets = rowsSplitter.Split(); - - if (!splittedResultSets.Success) { - SendIssuesAndSetErrorFlag(splittedResultSets.Issues); - return; - } - - for (const auto& resultSet : splittedResultSets.ResultSets) { - auto protoReq = CreateProtoRequestWithoutResultSet(startRowIndex); - startRowIndex += resultSet.rows().size(); - *protoReq.mutable_result_set() = resultSet; - ResultChunks.emplace_back(std::move(protoReq)); - } - Requests[Cookie].MessagesNum = splittedResultSets.ResultSets.size(); - } - + return protoReq; + } + + void SendResult() { + if (InflightCounter || CurChunkInd >= ResultChunks.size()) { + return; + } + ++InflightCounter; + const auto actorSystem = NActors::TActivationContext::ActorSystem(); + const auto selfId = SelfId(); + Client + .WriteTaskResult(std::move(ResultChunks[CurChunkInd++])) + .Subscribe([actorSystem, selfId](const auto& future) { + try { + const auto& wrappedResult = future.GetValue(); + if (wrappedResult.IsResultSet()) { + actorSystem->Send(selfId, + new NYq::TEvControlPlaneStorage::TEvWriteResultDataResponse(wrappedResult.GetIssues(), wrappedResult.GetResult().request_id())); + } else { + auto issues = wrappedResult.GetIssues(); + issues.AddIssue("Error on writing result"); + actorSystem->Send(selfId, + new NYq::TEvControlPlaneStorage::TEvWriteResultDataResponse(issues, UINT64_MAX)); + } + } catch(...) { + TIssues issues; + issues.AddIssue(CurrentExceptionMessage()); + actorSystem->Send(selfId, + new NYq::TEvControlPlaneStorage::TEvWriteResultDataResponse(issues, UINT64_MAX)); + } + }); + } + + void ConstructResults(const Ydb::ResultSet& resultSet, ui64 startRowIndex) { + const size_t baseProtoByteSize = CreateProtoRequestWithoutResultSet(startRowIndex).ByteSizeLong(); + + NYq::TRowsProtoSplitter rowsSplitter(resultSet, ProtoMessageLimit, baseProtoByteSize, MaxRowsCountPerChunk); + const auto splittedResultSets = rowsSplitter.Split(); + + if (!splittedResultSets.Success) { + SendIssuesAndSetErrorFlag(splittedResultSets.Issues); + return; + } + + for (const auto& resultSet : splittedResultSets.ResultSets) { + auto protoReq = CreateProtoRequestWithoutResultSet(startRowIndex); + startRowIndex += resultSet.rows().size(); + *protoReq.mutable_result_set() = resultSet; + ResultChunks.emplace_back(std::move(protoReq)); + } + Requests[Cookie].MessagesNum = splittedResultSets.ResultSets.size(); + } + void ProcessData(NDq::TEvDqCompute::TEvChannelData::TPtr& ev) { auto& data = ev->Get()->Record.GetChannelData().GetData(); auto resultSet = ResultBuilder->BuildResultSet({data}); FreeSpace -= data.GetRaw().size(); - OccupiedSpace += data.GetRaw().size(); - - if (OccupiedSpace > SpaceLimitPerQuery) { - TIssues issues; - issues.AddIssue(TStringBuilder() << "Can not write results with size > " << SpaceLimitPerQuery / (1024 * 1024) << "_MB"); - SendIssuesAndSetErrorFlag(issues); - return; - } - + OccupiedSpace += data.GetRaw().size(); + + if (OccupiedSpace > SpaceLimitPerQuery) { + TIssues issues; + issues.AddIssue(TStringBuilder() << "Can not write results with size > " << SpaceLimitPerQuery / (1024 * 1024) << "_MB"); + SendIssuesAndSetErrorFlag(issues); + return; + } + ui64 startRowIndex = RowIndex; - RowIndex += resultSet.rows().size(); + RowIndex += resultSet.rows().size(); auto& request = Requests[Cookie]; request.Sender = ev->Sender; @@ -260,9 +260,9 @@ private: request.SeqNo = ev->Get()->Record.GetSeqNo(); request.Size = data.GetRaw().size(); - ConstructResults(resultSet, startRowIndex); - SendResult(); - + ConstructResults(resultSet, startRowIndex); + SendResult(); + if (!Truncated && (!AllResultsBytesLimit || Size + data.GetRaw().size() < *AllResultsBytesLimit) && (!RowsLimitPerWrite || Rows + data.GetRows() < *RowsLimitPerWrite)) { @@ -287,27 +287,27 @@ private: try { if (ev->Get()->Record.GetChannelData().HasData()) { ProcessData(ev); - } else { - auto res = MakeHolder<NDq::TEvDqCompute::TEvChannelDataAck>(); - res->Record.SetChannelId(ev->Get()->Record.GetChannelData().GetChannelId()); - res->Record.SetSeqNo(ev->Get()->Record.GetSeqNo()); - res->Record.SetFreeSpace(FreeSpace); - res->Record.SetFinish(HasError); - Send(ev->Sender, res.Release()); - - auto duration = (TInstant::Now()-StartTime); - - LOG_D("ChannelData, Records: " << RowIndex - << " HasError: " << HasError - << " Size: " << Size - << " Rows: " << Rows - << " FreeSpace: " << FreeSpace - << " Duration: " << duration - << " AvgSpeed: " << Size/(duration.Seconds()+1)/1024/1024); - } + } else { + auto res = MakeHolder<NDq::TEvDqCompute::TEvChannelDataAck>(); + res->Record.SetChannelId(ev->Get()->Record.GetChannelData().GetChannelId()); + res->Record.SetSeqNo(ev->Get()->Record.GetSeqNo()); + res->Record.SetFreeSpace(FreeSpace); + res->Record.SetFinish(HasError); + Send(ev->Sender, res.Release()); + + auto duration = (TInstant::Now()-StartTime); + + LOG_D("ChannelData, Records: " << RowIndex + << " HasError: " << HasError + << " Size: " << Size + << " Rows: " << Rows + << " FreeSpace: " << FreeSpace + << " Duration: " << duration + << " AvgSpeed: " << Size/(duration.Seconds()+1)/1024/1024); + } } catch (...) { LOG_E(CurrentExceptionMessage()); - auto req = MakeHolder<TEvDqFailure>(TIssue("Internal error on data write").SetCode(NYql::DEFAULT_ERROR, TSeverityIds::S_ERROR), /*retriable=*/ false, + auto req = MakeHolder<TEvDqFailure>(TIssue("Internal error on data write").SetCode(NYql::DEFAULT_ERROR, TSeverityIds::S_ERROR), /*retriable=*/ false, /*needFallback=*/false); Send(ExecuterId, req.Release()); HasError = true; @@ -319,11 +319,11 @@ private: ui64 ChannelId; ui64 SeqNo; i64 Size; - size_t MessagesNum; + size_t MessagesNum; }; THashMap<ui64, TRequest> Requests; - TVector<NYql::NDqProto::TData> Head; + TVector<NYql::NDqProto::TData> Head; bool Truncated = false; TMaybe<ui64> AllResultsBytesLimit = 10000; TMaybe<ui64> RowsLimitPerWrite = 1000; @@ -335,23 +335,23 @@ private: const TResultId ResultId; const TString TraceId; TInstant Deadline; - TPrivateClient Client; + TPrivateClient Client; const TInstant StartTime = TInstant::Now(); ui64 RowIndex = 0; ui64 Cookie = 0; i64 FreeSpace = 64_MB; // TODO: make global free space - const size_t ProtoMessageLimit = 10_MB; - const size_t MaxRowsCountPerChunk = 100'000; + const size_t ProtoMessageLimit = 10_MB; + const size_t MaxRowsCountPerChunk = 100'000; bool HasError = false; bool Finished = false; - NYql::TIssues Issues; - ui64 SpaceLimitPerQuery = 20_MB; - ui64 OccupiedSpace = 0; - - TVector<Yq::Private::WriteTaskResultRequest> ResultChunks; - size_t CurChunkInd = 0; - ui32 InflightCounter = 0; + NYql::TIssues Issues; + ui64 SpaceLimitPerQuery = 20_MB; + ui64 OccupiedSpace = 0; + + TVector<Yq::Private::WriteTaskResultRequest> ResultChunks; + size_t CurChunkInd = 0; + ui32 InflightCounter = 0; }; NActors::IActor* CreateResultWriter( @@ -362,10 +362,10 @@ NActors::IActor* CreateResultWriter( const TResultId& resultId, const TVector<TString>& columns, const TString& traceId, - const TInstant& deadline, - const NMonitoring::TDynamicCounterPtr& clientCounters) + const TInstant& deadline, + const NMonitoring::TDynamicCounterPtr& clientCounters) { - return new TResultWriter(driver, executerId, resultType, privateApiConfig, resultId, columns, traceId, deadline, clientCounters); + return new TResultWriter(driver, executerId, resultType, privateApiConfig, resultId, columns, traceId, deadline, clientCounters); } -} // namespace NYq +} // namespace NYq diff --git a/ydb/core/yq/libs/actors/run_actor.cpp b/ydb/core/yq/libs/actors/run_actor.cpp index 25cdacc16d..5549f8f254 100644 --- a/ydb/core/yq/libs/actors/run_actor.cpp +++ b/ydb/core/yq/libs/actors/run_actor.cpp @@ -58,11 +58,11 @@ #include <ydb/core/yq/libs/tasks_packer/tasks_packer.h> #include <util/system/hostname.h> -#include <library/cpp/json/yson/json2yson.h> - +#include <library/cpp/json/yson/json2yson.h> + #include <ydb/core/yq/libs/control_plane_storage/control_plane_storage.h> #include <ydb/core/yq/libs/control_plane_storage/events/events.h> -#include <google/protobuf/util/time_util.h> +#include <google/protobuf/util/time_util.h> #include <util/string/split.h> #include <ydb/core/yq/libs/checkpointing/checkpoint_coordinator.h> @@ -78,7 +78,7 @@ #define LOG_D(stream) \ LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::YQL_PROXY, Params.QueryId << " RunActor : " << stream) -namespace NYq { +namespace NYq { using namespace NActors; using namespace NYql; @@ -96,8 +96,8 @@ public: { } - static constexpr char ActorName[] = "YQ_DEFERRED_COUNTERS_CLEANUP"; - + static constexpr char ActorName[] = "YQ_DEFERRED_COUNTERS_CLEANUP"; + void Bootstrap() { Become(&TDeferredCountersCleanupActor::StateFunc, TDuration::Seconds(60), new NActors::TEvents::TEvWakeup()); } @@ -122,8 +122,8 @@ private: const TString QueryId; }; -class TRunActor : public NActors::TActorBootstrapped<TRunActor> { -public: +class TRunActor : public NActors::TActorBootstrapped<TRunActor> { +public: explicit TRunActor( const ::NYq::NCommon::TServiceCounters& serviceCounters , TRunActorParams&& params) @@ -132,11 +132,11 @@ public: , ServiceCounters(serviceCounters) , QueryCounters(serviceCounters) , EnableCheckpointCoordinator(Params.QueryType == YandexQuery::QueryContent::STREAMING && Params.CheckpointCoordinatorConfig.GetEnabled()) - , MaxTasksPerOperation(Params.CommonConfig.GetMaxTasksPerOperation() ? Params.CommonConfig.GetMaxTasksPerOperation() : 40) + , MaxTasksPerOperation(Params.CommonConfig.GetMaxTasksPerOperation() ? Params.CommonConfig.GetMaxTasksPerOperation() : 40) { } - static constexpr char ActorName[] = "YQ_RUN_ACTOR"; + static constexpr char ActorName[] = "YQ_RUN_ACTOR"; void Bootstrap() { LOG_D("Start run actor. Compute state: " << YandexQuery::QueryMeta::ComputeStatus_Name(Params.Status)); @@ -153,8 +153,8 @@ public: .DiscoveryEndpoint(Params.PrivateApiConfig.GetTaskServiceEndpoint()) .EnableSsl(Params.PrivateApiConfig.GetSecureTaskService()) .AuthToken(Params.AuthToken) - .Database(Params.PrivateApiConfig.GetTaskServiceDatabase() ? Params.PrivateApiConfig.GetTaskServiceDatabase() : TMaybe<TString>()), - Params.ClientCounters), + .Database(Params.PrivateApiConfig.GetTaskServiceDatabase() ? Params.PrivateApiConfig.GetTaskServiceDatabase() : TMaybe<TString>()), + Params.ClientCounters), SelfId(), Params.PingerConfig, Params.Deadline @@ -165,9 +165,9 @@ public: } catch (const std::exception&) { FailOnException(); } - } + } -private: +private: template <void (TRunActor::* DelegatedStateFunc)(STFUNC_SIG)> STFUNC(StateFuncWrapper) { try { @@ -181,7 +181,7 @@ private: HFunc(TEvents::TEvAsyncContinue, Handle); hFunc(NActors::TEvents::TEvUndelivered, Handle); hFunc(NYq::TEvents::TEvGraphParams, Handle); - hFunc(NYq::TEvents::TEvDataStreamsReadRulesCreationResult, Handle); + hFunc(NYq::TEvents::TEvDataStreamsReadRulesCreationResult, Handle); hFunc(NYql::NDqs::TEvQueryResponse, Handle); hFunc(TEvents::TEvQueryActionResult, Handle); hFunc(TEvents::TEvForwardPingResponse, Handle); @@ -189,7 +189,7 @@ private: ) STRICT_STFUNC(FinishStateFunc, - hFunc(NYq::TEvents::TEvDataStreamsReadRulesCreationResult, HandleFinish); + hFunc(NYq::TEvents::TEvDataStreamsReadRulesCreationResult, HandleFinish); hFunc(TEvents::TEvDataStreamsReadRulesDeletionResult, HandleFinish); hFunc(NYql::NDqs::TEvQueryResponse, HandleFinish); hFunc(TEvents::TEvForwardPingResponse, HandleFinish); @@ -260,8 +260,8 @@ private: Finish(GetFinalStatusFromFinalizingStatus(Params.Status)); break; case YandexQuery::QueryMeta::STARTING: - HandleConnections(); - RunProgram(); + HandleConnections(); + RunProgram(); break; case YandexQuery::QueryMeta::RESUMING: case YandexQuery::QueryMeta::RUNNING: @@ -273,22 +273,22 @@ private: } } - void HandleConnections() { + void HandleConnections() { LOG_D("HandleConnections"); - THashMap<std::pair<TString, DatabaseType>, TEvents::TDatabaseAuth> databaseIds; + THashMap<std::pair<TString, DatabaseType>, TEvents::TDatabaseAuth> databaseIds; for (const auto& connection : Params.Connections) { if (!connection.content().name()) { LOG_D("Connection with empty name " << connection.meta().id()); continue; } - Connections[connection.content().name()] = connection; // Necessary for TDatabaseAsyncResolverWithMeta + Connections[connection.content().name()] = connection; // Necessary for TDatabaseAsyncResolverWithMeta YqConnections.emplace(connection.meta().id(), connection); } } void RunProgram() { - LOG_D("RunProgram"); + LOG_D("RunProgram"); if (!CompileQuery()) { Abort("Failed to compile query", YandexQuery::QueryMeta::FAILED); } @@ -301,9 +301,9 @@ private: void Fail(const TString& errorMessage) { LOG_E("Fail for query " << Params.QueryId << ", finishing: " << Finishing << ", details: " << errorMessage); - if (YqConnections.empty()) { - Issues.AddIssue("YqConnections array is empty"); - } + if (YqConnections.empty()) { + Issues.AddIssue("YqConnections array is empty"); + } if (!Finishing) { Abort("Internal Error", YandexQuery::QueryMeta::FAILED); @@ -314,7 +314,7 @@ private: Issues.AddIssue("Internal Error"); if (!ConsumersAreDeleted) { - for (const Yq::Private::TopicConsumer& c : Params.CreatedTopicConsumers) { + for (const Yq::Private::TopicConsumer& c : Params.CreatedTopicConsumers) { TransientIssues.AddIssue(TStringBuilder() << "Created read rule `" << c.consumer_name() << "` for topic `" << c.topic_path() << "` (database id " << c.database_id() << ") maybe was left undeleted: internal error occurred"); TransientIssues.back().Severity = NYql::TSeverityIds::S_WARNING; } @@ -405,8 +405,8 @@ private: for (NYql::NDqProto::TTaskInput& taskInput : *task.MutableInputs()) { if (taskInput.GetTypeCase() == NYql::NDqProto::TTaskInput::kSource && taskInput.GetSource().GetType() == "PqSource") { google::protobuf::Any& settingsAny = *taskInput.MutableSource()->MutableSettings(); - YQL_ENSURE(settingsAny.Is<NYql::NPq::NProto::TDqPqTopicSource>()); - NYql::NPq::NProto::TDqPqTopicSource srcDesc; + YQL_ENSURE(settingsAny.Is<NYql::NPq::NProto::TDqPqTopicSource>()); + NYql::NPq::NProto::TDqPqTopicSource srcDesc; YQL_ENSURE(settingsAny.UnpackTo(&srcDesc)); if (!srcDesc.GetConsumerName()) { @@ -475,29 +475,29 @@ private: } } - TString CheckLimitsOfDqGraphs() { - size_t dqTasks = 0; - for (const auto& dqGraph : DqGraphParams) { - dqTasks += dqGraph.TasksSize(); - } - LOG_D("Overall dq tasks: " << dqTasks); - if (dqTasks > MaxTasksPerOperation) { - return TStringBuilder() << "Too many tasks per operation: " << dqTasks << ". Allowed: less than " << MaxTasksPerOperation; - } - return ""; - } - - bool AbortOnExceedingDqGraphsLimits() { - TString errorMsg = CheckLimitsOfDqGraphs(); - if (errorMsg) { - Abort(errorMsg, YandexQuery::QueryMeta::FAILED, Program->Issues()); - return true; - } - return false; - } - + TString CheckLimitsOfDqGraphs() { + size_t dqTasks = 0; + for (const auto& dqGraph : DqGraphParams) { + dqTasks += dqGraph.TasksSize(); + } + LOG_D("Overall dq tasks: " << dqTasks); + if (dqTasks > MaxTasksPerOperation) { + return TStringBuilder() << "Too many tasks per operation: " << dqTasks << ". Allowed: less than " << MaxTasksPerOperation; + } + return ""; + } + + bool AbortOnExceedingDqGraphsLimits() { + TString errorMsg = CheckLimitsOfDqGraphs(); + if (errorMsg) { + Abort(errorMsg, YandexQuery::QueryMeta::FAILED, Program->Issues()); + return true; + } + return false; + } + void Handle(NYq::TEvents::TEvGraphParams::TPtr& ev) { - LOG_D("Graph params with tasks: " << ev->Get()->GraphParams.TasksSize()); + LOG_D("Graph params with tasks: " << ev->Get()->GraphParams.TasksSize()); DqGraphParams.push_back(ev->Get()->GraphParams); } @@ -516,10 +516,10 @@ private: } void PrepareGraphs() { - if (AbortOnExceedingDqGraphsLimits()) { - return; - } - Yq::Private::PingTaskRequest request; + if (AbortOnExceedingDqGraphsLimits()) { + return; + } + Yq::Private::PingTaskRequest request; TStringStream exprOut; TStringStream planOut; @@ -544,7 +544,7 @@ private: Params.CreatedTopicConsumers.clear(); Params.CreatedTopicConsumers.reserve(TopicsForConsumersCreation.size()); - for (const NYql::NPq::NProto::TDqPqTopicSource& src : TopicsForConsumersCreation) { + for (const NYql::NPq::NProto::TDqPqTopicSource& src : TopicsForConsumersCreation) { auto& consumer = *request.add_created_topic_consumers(); consumer.set_database_id(src.GetDatabaseId()); consumer.set_database(src.GetDatabase()); @@ -733,7 +733,7 @@ private: ContinueFinish(); } - void Handle(NYq::TEvents::TEvDataStreamsReadRulesCreationResult::TPtr& ev) { + void Handle(NYq::TEvents::TEvDataStreamsReadRulesCreationResult::TPtr& ev) { LOG_D("Read rules creation finished. Issues: " << ev->Get()->Issues.Size()); ReadRulesCreatorId = {}; if (ev->Get()->Issues) { @@ -745,7 +745,7 @@ private: } } - void HandleFinish(NYq::TEvents::TEvDataStreamsReadRulesCreationResult::TPtr& ev) { + void HandleFinish(NYq::TEvents::TEvDataStreamsReadRulesCreationResult::TPtr& ev) { ReadRulesCreatorId = {}; if (ev->Get()->Issues) { TransientIssues.AddIssues(ev->Get()->Issues); @@ -779,7 +779,7 @@ private: void RunReadRulesDeletionActor() { TVector<std::shared_ptr<NYdb::ICredentialsProviderFactory>> credentials; credentials.reserve(Params.CreatedTopicConsumers.size()); - for (const Yq::Private::TopicConsumer& c : Params.CreatedTopicConsumers) { + for (const Yq::Private::TopicConsumer& c : Params.CreatedTopicConsumers) { if (const TString& tokenName = c.token_name()) { credentials.emplace_back( CreateCredentialsProviderFactoryForStructuredToken(Params.CredentialsFactory, FindTokenByName(tokenName), c.add_bearer_to_token())); @@ -810,7 +810,7 @@ private: { Params.Status = YandexQuery::QueryMeta::RUNNING; - Yq::Private::PingTaskRequest request; + Yq::Private::PingTaskRequest request; request.set_status(YandexQuery::QueryMeta::RUNNING); *request.mutable_started_at() = google::protobuf::util::TimeUtil::MillisecondsToTimestamp(Now().MilliSeconds()); Send(Pinger, new TEvents::TEvForwardPingRequest(request), 0, UpdateQueryInfoCookie); @@ -818,7 +818,7 @@ private: /* { // Failure test -- keep it until integrational tests work - Yq::Private::PingTaskRequest request; + Yq::Private::PingTaskRequest request; request.set_status(YandexQuery::QueryMeta::RUNNING); request.set_resign_query(true); Send(Pinger, new TEvents::TEvForwardPingRequest(request, true), 0, UpdateQueryInfoCookie); @@ -862,7 +862,7 @@ private: NActors::TActorId resultId; if (dqGraphParams.GetResultType()) { - TResultId writerResultId; + TResultId writerResultId; { writerResultId.HistoryId = Params.QueryId; writerResultId.Id = Params.ResultId; @@ -874,9 +874,9 @@ private: columns.emplace_back(column); } resultId = NActors::TActivationContext::Register( - CreateResultWriter( - Params.Driver, ExecuterId, dqGraphParams.GetResultType(), Params.PrivateApiConfig, - writerResultId, columns, dqGraphParams.GetSession(), Params.Deadline, Params.ClientCounters)); + CreateResultWriter( + Params.Driver, ExecuterId, dqGraphParams.GetResultType(), Params.PrivateApiConfig, + writerResultId, columns, dqGraphParams.GetSession(), Params.Deadline, Params.ClientCounters)); } else { LOG_D("ResultWriter was NOT CREATED since ResultType is empty"); resultId = ExecuterId; @@ -915,7 +915,7 @@ private: attr = dqSettings.Add(); attr->SetName("MaxTasksPerOperation"); - attr->SetValue(ToString(MaxTasksPerOperation)); + attr->SetValue(ToString(MaxTasksPerOperation)); attr = dqSettings.Add(); attr->SetName("EnableComputeActor"); @@ -945,7 +945,7 @@ private: } } - void AddClustersFromConfig(NYql::TGatewaysConfig& gatewaysConfig, THashMap<TString, TString>& clusters) const { + void AddClustersFromConfig(NYql::TGatewaysConfig& gatewaysConfig, THashMap<TString, TString>& clusters) const { for (const auto& pq : Params.GatewaysConfig.GetPq().GetClusterMapping()) { auto& clusterCfg = *gatewaysConfig.MutablePq()->AddClusterMapping(); clusterCfg = pq; @@ -1036,12 +1036,12 @@ private: } void Finish(YandexQuery::QueryMeta::ComputeStatus status) { - LOG_D("Is about to finish query with status " << YandexQuery::QueryMeta::ComputeStatus_Name(status));; + LOG_D("Is about to finish query with status " << YandexQuery::QueryMeta::ComputeStatus_Name(status));; Finishing = true; RetryNeeded = false; FinalQueryStatus = status; QueryStateUpdateRequest.set_status(FinalQueryStatus); // Can be changed later. - *QueryStateUpdateRequest.mutable_finished_at() = google::protobuf::util::TimeUtil::MillisecondsToTimestamp(TInstant::Now().MilliSeconds()); + *QueryStateUpdateRequest.mutable_finished_at() = google::protobuf::util::TimeUtil::MillisecondsToTimestamp(TInstant::Now().MilliSeconds()); Become(&TRunActor::StateFuncWrapper<&TRunActor::FinishStateFunc>); if (!FinalizingStatusIsWritten) { @@ -1088,18 +1088,18 @@ private: Finish(status); } - void FillDqGraphParams() { + void FillDqGraphParams() { for (const auto& s : Params.DqGraphs) { NYq::NProto::TGraphParams dqGraphParams; Y_VERIFY(dqGraphParams.ParseFromString(s)); DqGraphParams.emplace_back(std::move(dqGraphParams)); } - } - - void ReRunQuery() { - if (AbortOnExceedingDqGraphsLimits()) { - return; - } + } + + void ReRunQuery() { + if (AbortOnExceedingDqGraphsLimits()) { + return; + } for (const auto& m : Params.ResultSetMetas) { *QueryStateUpdateRequest.add_result_set_meta() = m; } @@ -1110,8 +1110,8 @@ private: } bool CompileQuery() { - LOG_D("Compiling query ..."); - NYql::TGatewaysConfig gatewaysConfig; + LOG_D("Compiling query ..."); + NYql::TGatewaysConfig gatewaysConfig; SetupDqSettings(*gatewaysConfig.MutableDq()->MutableDefaultSettings()); THashMap<TString, TString> clusters; @@ -1128,24 +1128,24 @@ private: gatewaysConfig, clusters); - TVector<TDataProviderInitializer> dataProvidersInit; - const auto dbResolver = std::make_shared<TDatabaseAsyncResolverWithMeta>(TDatabaseAsyncResolverWithMeta(NActors::TActivationContext::ActorSystem(), Params.DatabaseResolver, + TVector<TDataProviderInitializer> dataProvidersInit; + const auto dbResolver = std::make_shared<TDatabaseAsyncResolverWithMeta>(TDatabaseAsyncResolverWithMeta(NActors::TActivationContext::ActorSystem(), Params.DatabaseResolver, Params.CommonConfig.GetYdbMvpCloudEndpoint(), Params.CommonConfig.GetMdbGateway(), Params.CommonConfig.GetMdbTransformHost(), Params.QueryId, Params.AuthToken, Params.AccountIdSignatures, Connections)); - { + { // TBD: move init to better place QueryStateUpdateRequest.set_scope(Params.Scope.ToString()); QueryStateUpdateRequest.mutable_query_id()->set_value(Params.QueryId); QueryStateUpdateRequest.set_owner_id(Params.Owner); dataProvidersInit.push_back(GetDqDataProviderInitializer(&CreateInMemoryExecTransformer, NYq::CreateEmptyGateway(SelfId()), Params.DqCompFactory, {}, nullptr)); - } + } - { - dataProvidersInit.push_back(GetYdbDataProviderInitializer(Params.Driver, Params.CredentialsFactory, dbResolver)); - } + { + dataProvidersInit.push_back(GetYdbDataProviderInitializer(Params.Driver, Params.CredentialsFactory, dbResolver)); + } - { + { dataProvidersInit.push_back(GetClickHouseDataProviderInitializer(Params.S3Gateway, dbResolver)); - } + } { dataProvidersInit.push_back(GetS3DataProviderInitializer(Params.S3Gateway, Params.CredentialsFactory)); @@ -1160,7 +1160,7 @@ private: Params.FunctionRegistry ); const auto pqGateway = NYql::CreatePqNativeGateway(pqServices); - dataProvidersInit.push_back(GetPqDataProviderInitializer(pqGateway, false, dbResolver)); + dataProvidersInit.push_back(GetPqDataProviderInitializer(pqGateway, false, dbResolver)); } { @@ -1170,24 +1170,24 @@ private: } TProgramFactory progFactory(false, Params.FunctionRegistry, Params.NextUniqueId, dataProvidersInit, "yq"); - progFactory.SetModules(Params.ModuleResolver); - progFactory.SetUdfResolver(NYql::NCommon::CreateSimpleUdfResolver(Params.FunctionRegistry, nullptr)); - progFactory.SetGatewaysConfig(&gatewaysConfig); + progFactory.SetModules(Params.ModuleResolver); + progFactory.SetUdfResolver(NYql::NCommon::CreateSimpleUdfResolver(Params.FunctionRegistry, nullptr)); + progFactory.SetGatewaysConfig(&gatewaysConfig); SessionId = TStringBuilder() << Params.QueryId << '#' << Params.ResultId << '#' - << Params.Scope.ToString() << '#' + << Params.Scope.ToString() << '#' << Params.Owner << '#' << Params.CloudId; Program = progFactory.Create("-stdin-", Params.Sql, SessionId); - Program->EnableResultPosition(); - - NSQLTranslation::TTranslationSettings sqlSettings; - sqlSettings.ClusterMapping = clusters; - sqlSettings.SyntaxVersion = 1; - sqlSettings.V0Behavior = NSQLTranslation::EV0Behavior::Disable; + Program->EnableResultPosition(); + + NSQLTranslation::TTranslationSettings sqlSettings; + sqlSettings.ClusterMapping = clusters; + sqlSettings.SyntaxVersion = 1; + sqlSettings.V0Behavior = NSQLTranslation::EV0Behavior::Disable; sqlSettings.Flags.insert({ "DqEngineEnable", "DqEngineForce" }); try { AddTableBindingsFromBindings(Params.Bindings, YqConnections, sqlSettings); @@ -1235,45 +1235,45 @@ private: break; default: Issues.AddIssue(TStringBuilder() << "Unexpected execute mode " << static_cast<int>(Params.ExecuteMode)); - return false; - } + return false; + } futureStatus.Subscribe([actorSystem = NActors::TActivationContext::ActorSystem(), selfId = SelfId()](const TProgram::TFutureStatus& f) { - actorSystem->Send(selfId, new TEvents::TEvAsyncContinue(f)); - }); - return true; - } - - void Handle(TEvents::TEvAsyncContinue::TPtr& ev, const TActorContext& ctx) { - LOG_D("Compiling finished"); + actorSystem->Send(selfId, new TEvents::TEvAsyncContinue(f)); + }); + return true; + } + + void Handle(TEvents::TEvAsyncContinue::TPtr& ev, const TActorContext& ctx) { + LOG_D("Compiling finished"); NYql::TProgram::TStatus status = TProgram::TStatus::Error; - const auto& f = ev->Get()->Future; - try { + const auto& f = ev->Get()->Future; + try { status = f.GetValue(); - if (status == TProgram::TStatus::Async) { - auto futureStatus = Program->ContinueAsync(); - auto actorSystem = ctx.ActorSystem(); - auto selfId = ctx.SelfID; - futureStatus.Subscribe([actorSystem, selfId](const TProgram::TFutureStatus& f) { - actorSystem->Send(selfId, new TEvents::TEvAsyncContinue(f)); - }); - return; - } + if (status == TProgram::TStatus::Async) { + auto futureStatus = Program->ContinueAsync(); + auto actorSystem = ctx.ActorSystem(); + auto selfId = ctx.SelfID; + futureStatus.Subscribe([actorSystem, selfId](const TProgram::TFutureStatus& f) { + actorSystem->Send(selfId, new TEvents::TEvAsyncContinue(f)); + }); + return; + } } catch (const std::exception& err) { Issues.AddIssue(ExceptionToIssue(err)); } - + if (status == TProgram::TStatus::Ok || (DqGraphParams.size() > 0 && !DqGraphParams[0].GetResultType())) { PrepareGraphs(); } else { Abort(TStringBuilder() << "Run query failed: " << ToString(status), YandexQuery::QueryMeta::FAILED, Program->Issues()); - } - } + } + } void Handle(NActors::TEvents::TEvUndelivered::TPtr&) { Fail("TRunActor::OnUndelivered"); - } + } TString FindTokenByName(const TString& tokenName) const { for (auto& graphParams : DqGraphParams) { @@ -1319,11 +1319,11 @@ private: TRunActorParams Params; THashMap<TString, YandexQuery::Connection> YqConnections; - TProgramPtr Program; - TIssues Issues; + TProgramPtr Program; + TIssues Issues; TIssues TransientIssues; - TQueryResult QueryResult; - TInstant Deadline; + TQueryResult QueryResult; + TInstant Deadline; TActorId Pinger; TInstant CreatedAt; YandexQuery::QueryAction Action = YandexQuery::QueryAction::QUERY_ACTION_UNSPECIFIED; @@ -1340,11 +1340,11 @@ private: ::NYq::NCommon::TServiceCounters QueryCounters; bool EnableCheckpointCoordinator = false; bool RetryNeeded = false; - Yq::Private::PingTaskRequest QueryStateUpdateRequest; + Yq::Private::PingTaskRequest QueryStateUpdateRequest; THashMap<TString, YandexQuery::Connection> Connections; // Necessary for DbAsyncResolver - const ui64 MaxTasksPerOperation = 100; - + const ui64 MaxTasksPerOperation = 100; + // Consumers creation TVector<NYql::NPq::NProto::TDqPqTopicSource> TopicsForConsumersCreation; TVector<std::shared_ptr<NYdb::ICredentialsProviderFactory>> CredentialsForConsumersCreation; @@ -1365,9 +1365,9 @@ private: SaveFinalizingStatusCookie, SetLoadFromCheckpointModeCookie, }; -}; - - +}; + + IActor* CreateRunActor( const ::NYq::NCommon::TServiceCounters& serviceCounters, TRunActorParams&& params @@ -1375,4 +1375,4 @@ IActor* CreateRunActor( return new TRunActor(serviceCounters, std::move(params)); } -} /* NYq */ +} /* NYq */ diff --git a/ydb/core/yq/libs/actors/run_actor_params.cpp b/ydb/core/yq/libs/actors/run_actor_params.cpp index 933c9ef631..1446cae1bb 100644 --- a/ydb/core/yq/libs/actors/run_actor_params.cpp +++ b/ydb/core/yq/libs/actors/run_actor_params.cpp @@ -1,26 +1,26 @@ -#include "run_actor_params.h" - -namespace NYq { - -using namespace NActors; - -TRunActorParams::TRunActorParams( - NYdb::TDriver driver, +#include "run_actor_params.h" + +namespace NYq { + +using namespace NActors; + +TRunActorParams::TRunActorParams( + NYdb::TDriver driver, NYql::IHTTPGateway::TPtr s3Gateway, - const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry, - TIntrusivePtr<IRandomProvider> randomProvider, - NYql::IModuleResolver::TPtr& moduleResolver, - ui64 nextUniqueId, - NKikimr::NMiniKQL::TComputationNodeFactory dqCompFactory, + const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry, + TIntrusivePtr<IRandomProvider> randomProvider, + NYql::IModuleResolver::TPtr& moduleResolver, + ui64 nextUniqueId, + NKikimr::NMiniKQL::TComputationNodeFactory dqCompFactory, ::NPq::NConfigurationManager::IConnections::TPtr pqCmConnections, const ::NYq::NConfig::TCommonConfig& commonConfig, const ::NYq::NConfig::TCheckpointCoordinatorConfig& checkpointCoordinatorConfig, const ::NYq::NConfig::TPrivateApiConfig& privateApiConfig, const ::NYq::NConfig::TGatewaysConfig& gatewaysConfig, const ::NYq::NConfig::TPingerConfig& pingerConfig, - const TString& sql, + const TString& sql, const TScope& scope, - const TString& authToken, + const TString& authToken, const TActorId& databaseResolver, const TString& queryId, const TString& userId, @@ -28,9 +28,9 @@ TRunActorParams::TRunActorParams( const int64_t previousQueryRevision, TVector<YandexQuery::Connection> connections, TVector<YandexQuery::Binding> bindings, - NYql::ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory, + NYql::ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory, THashMap<TString, TString> accountIdSignatures, - YandexQuery::QueryContent::QueryType queryType, + YandexQuery::QueryContent::QueryType queryType, YandexQuery::ExecuteMode executeMode, const TString& resultId, const YandexQuery::StateLoadMode stateLoadMode, @@ -40,28 +40,28 @@ TRunActorParams::TRunActorParams( TVector<YandexQuery::ResultSetMeta> resultSetMetas, TVector<TString> dqGraphs, int32_t dqGraphIndex, - TVector<Yq::Private::TopicConsumer> createdTopicConsumers, - bool automatic, + TVector<Yq::Private::TopicConsumer> createdTopicConsumers, + bool automatic, const TString& queryName, - const TInstant& deadline, - const NMonitoring::TDynamicCounterPtr& clientCounters - ) - : Driver(driver) + const TInstant& deadline, + const NMonitoring::TDynamicCounterPtr& clientCounters + ) + : Driver(driver) , S3Gateway(s3Gateway) - , FunctionRegistry(functionRegistry) - , RandomProvider(randomProvider) - , ModuleResolver(moduleResolver) - , NextUniqueId(nextUniqueId) - , DqCompFactory(dqCompFactory) + , FunctionRegistry(functionRegistry) + , RandomProvider(randomProvider) + , ModuleResolver(moduleResolver) + , NextUniqueId(nextUniqueId) + , DqCompFactory(dqCompFactory) , PqCmConnections(std::move(pqCmConnections)) , CommonConfig(commonConfig) , CheckpointCoordinatorConfig(checkpointCoordinatorConfig) , PrivateApiConfig(privateApiConfig) , GatewaysConfig(gatewaysConfig) , PingerConfig(pingerConfig) - , Sql(sql) + , Sql(sql) , Scope(scope) - , AuthToken(authToken) + , AuthToken(authToken) , DatabaseResolver(databaseResolver) , QueryId(queryId) , UserId(userId) @@ -73,7 +73,7 @@ TRunActorParams::TRunActorParams( , AccountIdSignatures(std::move(accountIdSignatures)) , QueryType(queryType) , ExecuteMode(executeMode) - , ResultId(resultId) + , ResultId(resultId) , StateLoadMode(stateLoadMode) , StreamingDisposition(streamingDisposition) , Status(status) @@ -82,11 +82,11 @@ TRunActorParams::TRunActorParams( , DqGraphs(std::move(dqGraphs)) , DqGraphIndex(dqGraphIndex) , CreatedTopicConsumers(std::move(createdTopicConsumers)) - , Automatic(automatic) - , QueryName(queryName) + , Automatic(automatic) + , QueryName(queryName) , Deadline(deadline) - , ClientCounters(clientCounters) - { - } - -} /* NYq */ + , ClientCounters(clientCounters) + { + } + +} /* NYq */ diff --git a/ydb/core/yq/libs/actors/run_actor_params.h b/ydb/core/yq/libs/actors/run_actor_params.h index c1d63bc1f8..bad5c3c32b 100644 --- a/ydb/core/yq/libs/actors/run_actor_params.h +++ b/ydb/core/yq/libs/actors/run_actor_params.h @@ -10,41 +10,41 @@ #include <ydb/library/yql/providers/dq/worker_manager/interface/counters.h> #include <ydb/library/yql/providers/solomon/provider/yql_solomon_gateway.h> #include <ydb/library/yql/providers/pq/cm_client/interface/client.h> - + #include <library/cpp/actors/core/actorsystem.h> #include <library/cpp/time_provider/time_provider.h> #include <library/cpp/random_provider/random_provider.h> -namespace NYq { - -struct TRunActorParams { // TODO2 : Change name - TRunActorParams( - NYdb::TDriver driver, +namespace NYq { + +struct TRunActorParams { // TODO2 : Change name + TRunActorParams( + NYdb::TDriver driver, NYql::IHTTPGateway::TPtr s3Gateway, - const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry, - TIntrusivePtr<IRandomProvider> randomProvider, - NYql::IModuleResolver::TPtr& moduleResolver, - ui64 nextUniqueId, - NKikimr::NMiniKQL::TComputationNodeFactory dqCompFactory, + const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry, + TIntrusivePtr<IRandomProvider> randomProvider, + NYql::IModuleResolver::TPtr& moduleResolver, + ui64 nextUniqueId, + NKikimr::NMiniKQL::TComputationNodeFactory dqCompFactory, ::NPq::NConfigurationManager::IConnections::TPtr pqCmConnections, const ::NYq::NConfig::TCommonConfig& commonConfig, const ::NYq::NConfig::TCheckpointCoordinatorConfig& checkpointCoordinatorConfig, const ::NYq::NConfig::TPrivateApiConfig& privateApiConfig, const ::NYq::NConfig::TGatewaysConfig& gatewaysConfig, const ::NYq::NConfig::TPingerConfig& pingerConfig, - const TString& sql, + const TString& sql, const TScope& scope, - const TString& authToken, - const NActors::TActorId& databaseResolver, + const TString& authToken, + const NActors::TActorId& databaseResolver, const TString& queryId, const TString& userId, const TString& owner, const int64_t previousQueryRevision, TVector<YandexQuery::Connection> connections, TVector<YandexQuery::Binding> bindings, - NYql::ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory, + NYql::ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory, THashMap<TString, TString> accountIdSignatures, - YandexQuery::QueryContent::QueryType queryType, + YandexQuery::QueryContent::QueryType queryType, YandexQuery::ExecuteMode executeMode, const TString& resultId, const YandexQuery::StateLoadMode stateLoadMode, @@ -54,23 +54,23 @@ struct TRunActorParams { // TODO2 : Change name TVector<YandexQuery::ResultSetMeta> resultSetMetas, TVector<TString> dqGraphs, int32_t dqGraphIndex, - TVector<Yq::Private::TopicConsumer> createdTopicConsumers, - bool automatic, + TVector<Yq::Private::TopicConsumer> createdTopicConsumers, + bool automatic, const TString& queryName, - const TInstant& deadline, - const NMonitoring::TDynamicCounterPtr& clientCounters - ); - + const TInstant& deadline, + const NMonitoring::TDynamicCounterPtr& clientCounters + ); + TRunActorParams(const TRunActorParams& params) = default; TRunActorParams(TRunActorParams&& params) = default; - - NYdb::TDriver Driver; + + NYdb::TDriver Driver; NYql::IHTTPGateway::TPtr S3Gateway; - const NKikimr::NMiniKQL::IFunctionRegistry* FunctionRegistry; - TIntrusivePtr<IRandomProvider> RandomProvider; - NYql::IModuleResolver::TPtr ModuleResolver; - ui64 NextUniqueId; - NKikimr::NMiniKQL::TComputationNodeFactory DqCompFactory; + const NKikimr::NMiniKQL::IFunctionRegistry* FunctionRegistry; + TIntrusivePtr<IRandomProvider> RandomProvider; + NYql::IModuleResolver::TPtr ModuleResolver; + ui64 NextUniqueId; + NKikimr::NMiniKQL::TComputationNodeFactory DqCompFactory; ::NPq::NConfigurationManager::IConnections::TPtr PqCmConnections; const ::NYq::NConfig::TCommonConfig CommonConfig; @@ -78,21 +78,21 @@ struct TRunActorParams { // TODO2 : Change name const ::NYq::NConfig::TPrivateApiConfig PrivateApiConfig; const ::NYq::NConfig::TGatewaysConfig GatewaysConfig; const ::NYq::NConfig::TPingerConfig PingerConfig; - const TString Sql; + const TString Sql; const TScope Scope; - const TString AuthToken; - const NActors::TActorId DatabaseResolver; + const TString AuthToken; + const NActors::TActorId DatabaseResolver; const TString QueryId; const TString UserId; const TString Owner; const int64_t PreviousQueryRevision; const TVector<YandexQuery::Connection> Connections; const TVector<YandexQuery::Binding> Bindings; - const NYql::ISecuredServiceAccountCredentialsFactory::TPtr CredentialsFactory; + const NYql::ISecuredServiceAccountCredentialsFactory::TPtr CredentialsFactory; const THashMap<TString, TString> AccountIdSignatures; const YandexQuery::QueryContent::QueryType QueryType; const YandexQuery::ExecuteMode ExecuteMode; - const TString ResultId; + const TString ResultId; const YandexQuery::StateLoadMode StateLoadMode; const YandexQuery::StreamingDisposition StreamingDisposition; YandexQuery::QueryMeta::ComputeStatus Status; @@ -100,13 +100,13 @@ struct TRunActorParams { // TODO2 : Change name const TVector<YandexQuery::ResultSetMeta> ResultSetMetas; const TVector<TString> DqGraphs; const int32_t DqGraphIndex; - TVector<Yq::Private::TopicConsumer> CreatedTopicConsumers; - - bool Automatic = false; - TString QueryName; + TVector<Yq::Private::TopicConsumer> CreatedTopicConsumers; + + bool Automatic = false; + TString QueryName; TInstant Deadline; - - const NMonitoring::TDynamicCounterPtr ClientCounters; -}; - -} /* NYq */ + + const NMonitoring::TDynamicCounterPtr ClientCounters; +}; + +} /* NYq */ diff --git a/ydb/core/yq/libs/actors/system_clusters.cpp b/ydb/core/yq/libs/actors/system_clusters.cpp index 2cfb0ad41f..2968ae6487 100644 --- a/ydb/core/yq/libs/actors/system_clusters.cpp +++ b/ydb/core/yq/libs/actors/system_clusters.cpp @@ -3,10 +3,10 @@ #include <ydb/library/yql/providers/common/provider/yql_provider_names.h> #include <util/generic/hash.h> -namespace NYq { +namespace NYq { + +using namespace NYql; -using namespace NYql; - void AddSystemClusters(TGatewaysConfig& gatewaysConfig, THashMap<TString, TString>& clusters, const TString& authToken) { { const auto clusterCfg = gatewaysConfig.MutablePq()->AddClusterMapping(); @@ -117,5 +117,5 @@ void AddSystemClusters(TGatewaysConfig& gatewaysConfig, THashMap<TString, TStrin clusters.emplace(clusterCfg->GetName(), SolomonProviderName); } } - -} //NYq + +} //NYq diff --git a/ydb/core/yq/libs/actors/system_clusters.h b/ydb/core/yq/libs/actors/system_clusters.h index 2c496d7b99..c842948fdc 100644 --- a/ydb/core/yq/libs/actors/system_clusters.h +++ b/ydb/core/yq/libs/actors/system_clusters.h @@ -2,8 +2,8 @@ #include <ydb/library/yql/providers/common/proto/gateways_config.pb.h> -namespace NYq { - -void AddSystemClusters(NYql::TGatewaysConfig& gatewaysConfig, THashMap<TString, TString>& clusters, const TString& authToken); - -} //NYq +namespace NYq { + +void AddSystemClusters(NYql::TGatewaysConfig& gatewaysConfig, THashMap<TString, TString>& clusters, const TString& authToken); + +} //NYq diff --git a/ydb/core/yq/libs/actors/table_bindings_from_bindings.cpp b/ydb/core/yq/libs/actors/table_bindings_from_bindings.cpp index 7be0c1b98c..e353a194a4 100644 --- a/ydb/core/yq/libs/actors/table_bindings_from_bindings.cpp +++ b/ydb/core/yq/libs/actors/table_bindings_from_bindings.cpp @@ -4,12 +4,12 @@ #include <ydb/core/yq/libs/result_formatter/result_formatter.h> #include <util/generic/vector.h> -namespace NYq { +namespace NYq { + +using namespace NYql; -using namespace NYql; - namespace { - + void FillBinding(NSQLTranslation::TTranslationSettings& sqlSettings, const YandexQuery::Binding& binding, const THashMap<TString, YandexQuery::Connection>& connections) { TString clusterType; TString path; @@ -67,12 +67,12 @@ void FillBinding(NSQLTranslation::TTranslationSettings& sqlSettings, const Yande sqlSettings.PrivateBindings[binding.content().name()] = std::move(bindSettings); } -} //namespace +} //namespace void AddTableBindingsFromBindings(const TVector<YandexQuery::Binding>& bindings, const THashMap<TString, YandexQuery::Connection>& connections, NSQLTranslation::TTranslationSettings& sqlSettings) { for (const auto& binding : bindings) { FillBinding(sqlSettings, binding, connections); } } - -} //NYq + +} //NYq diff --git a/ydb/core/yq/libs/actors/table_bindings_from_bindings.h b/ydb/core/yq/libs/actors/table_bindings_from_bindings.h index afbbcf68f1..76343b06dc 100644 --- a/ydb/core/yq/libs/actors/table_bindings_from_bindings.h +++ b/ydb/core/yq/libs/actors/table_bindings_from_bindings.h @@ -3,8 +3,8 @@ #include <ydb/library/yql/sql/settings/translation_settings.h> #include <ydb/public/api/protos/yq.pb.h> -namespace NYq { - +namespace NYq { + void AddTableBindingsFromBindings(const TVector<YandexQuery::Binding>& bindings, const THashMap<TString, YandexQuery::Connection>& connections, NSQLTranslation::TTranslationSettings& sqlSettings); - -} //NYq + +} //NYq diff --git a/ydb/core/yq/libs/actors/task_get.cpp b/ydb/core/yq/libs/actors/task_get.cpp index c9081ad884..d31517aa9f 100644 --- a/ydb/core/yq/libs/actors/task_get.cpp +++ b/ydb/core/yq/libs/actors/task_get.cpp @@ -1,238 +1,238 @@ #include <ydb/core/yq/libs/config/protos/yq_config.pb.h> -#include "proxy_private.h" -#include "proxy.h" - +#include "proxy_private.h" +#include "proxy.h" + #include <ydb/core/protos/services.pb.h> #include <ydb/library/yql/public/issue/yql_issue_message.h> - -#include <library/cpp/yson/node/node_io.h> -#include <library/cpp/actors/core/events.h> -#include <library/cpp/actors/core/hfunc.h> -#include <library/cpp/actors/core/actor_bootstrapped.h> -#include <library/cpp/actors/core/log.h> + +#include <library/cpp/yson/node/node_io.h> +#include <library/cpp/actors/core/events.h> +#include <library/cpp/actors/core/hfunc.h> +#include <library/cpp/actors/core/actor_bootstrapped.h> +#include <library/cpp/actors/core/log.h> #include <library/cpp/protobuf/interop/cast.h> - + #include <ydb/core/yq/libs/common/entity_id.h> - + #include <ydb/core/yq/libs/control_plane_storage/control_plane_storage.h> #include <ydb/core/yq/libs/control_plane_storage/events/events.h> #include <ydb/library/security/util.h> - + #define LOG_E(stream) \ LOG_ERROR_S(*TlsActivationContext, NKikimrServices::YQL_PRIVATE_PROXY, "PrivateGetTask - Owner: " << OwnerId << ", " << "Host: " << Host << ", "<< stream) #define LOG_D(stream) \ LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::YQL_PRIVATE_PROXY, "PrivateGetTask - Owner: " << OwnerId << ", " << "Host: " << Host << ", " << stream) - -namespace NYq { - -using namespace NActors; -using namespace NMonitoring; - -class TGetTaskRequestActor - : public NActors::TActorBootstrapped<TGetTaskRequestActor> -{ -public: - TGetTaskRequestActor( - const NActors::TActorId& sender, + +namespace NYq { + +using namespace NActors; +using namespace NMonitoring; + +class TGetTaskRequestActor + : public NActors::TActorBootstrapped<TGetTaskRequestActor> +{ +public: + TGetTaskRequestActor( + const NActors::TActorId& sender, const NConfig::TTokenAccessorConfig& tokenAccessorConfig, - TIntrusivePtr<ITimeProvider> timeProvider, - TAutoPtr<TEvents::TEvGetTaskRequest> ev, - TDynamicCounterPtr counters) + TIntrusivePtr<ITimeProvider> timeProvider, + TAutoPtr<TEvents::TEvGetTaskRequest> ev, + TDynamicCounterPtr counters) : TokenAccessorConfig(tokenAccessorConfig) - , Sender(sender) - , TimeProvider(timeProvider) - , Ev(std::move(ev)) - , Counters(std::move(counters->GetSubgroup("subsystem", "private_api")->GetSubgroup("subcomponent", "GetTask"))) - , LifetimeDuration(Counters->GetHistogram("LifetimeDurationMs", ExponentialHistogram(10, 2, 50))) - , RequestedMBytes(Counters->GetHistogram("RequestedMB", ExponentialHistogram(6, 2, 3))) - , StartTime(TInstant::Now()) - { + , Sender(sender) + , TimeProvider(timeProvider) + , Ev(std::move(ev)) + , Counters(std::move(counters->GetSubgroup("subsystem", "private_api")->GetSubgroup("subcomponent", "GetTask"))) + , LifetimeDuration(Counters->GetHistogram("LifetimeDurationMs", ExponentialHistogram(10, 2, 50))) + , RequestedMBytes(Counters->GetHistogram("RequestedMB", ExponentialHistogram(6, 2, 3))) + , StartTime(TInstant::Now()) + { if (TokenAccessorConfig.GetHmacSecretFile()) { Signer = ::NYq::CreateSignerFromFile(TokenAccessorConfig.GetHmacSecretFile()); - } - } - - static constexpr char ActorName[] = "YQ_PRIVATE_GET_TASK"; - - void OnUndelivered(NActors::TEvents::TEvUndelivered::TPtr& ev, const NActors::TActorContext& ctx) { - LOG_E("TGetTaskRequestActor::OnUndelivered"); - auto Res = MakeHolder<TEvents::TEvGetTaskResponse>(); - Res->Status = Ydb::StatusIds::GENERIC_ERROR; - Res->Issues.AddIssue("UNDELIVERED"); - ctx.Send(ev->Sender, Res.Release()); - Die(ctx); - } - - void PassAway() final { - LifetimeDuration->Collect((TInstant::Now() - StartTime).MilliSeconds()); - NActors::IActor::PassAway(); - } - - void Fail(const TString& message, Ydb::StatusIds::StatusCode reqStatus = Ydb::StatusIds::INTERNAL_ERROR) { - Issues.AddIssue(message); - const auto codeStr = Ydb::StatusIds_StatusCode_Name(reqStatus); - LOG_E(TStringBuilder() - << "Failed with code: " << codeStr - << " Details: " << Issues.ToString()); - auto Res = MakeHolder<TEvents::TEvGetTaskResponse>(); - Res->Status = reqStatus; - Res->Issues.AddIssues(Issues); - Send(Sender, Res.Release()); - PassAway(); - } - - void Bootstrap(const TActorContext& ctx) { - Become(&TGetTaskRequestActor::StateFunc); - const auto& req = Ev->Record; - OwnerId = req.owner_id(); - Host = req.host(); - LOG_D("Request CP::GetTask with size: " << req.ByteSize() << " bytes"); - RequestedMBytes->Collect(req.ByteSize() / 1024 / 1024); - ctx.Send(NYq::ControlPlaneStorageServiceActorId(), - new NYq::TEvControlPlaneStorage::TEvGetTaskRequest(OwnerId, Host)); - } - + } + } + + static constexpr char ActorName[] = "YQ_PRIVATE_GET_TASK"; + + void OnUndelivered(NActors::TEvents::TEvUndelivered::TPtr& ev, const NActors::TActorContext& ctx) { + LOG_E("TGetTaskRequestActor::OnUndelivered"); + auto Res = MakeHolder<TEvents::TEvGetTaskResponse>(); + Res->Status = Ydb::StatusIds::GENERIC_ERROR; + Res->Issues.AddIssue("UNDELIVERED"); + ctx.Send(ev->Sender, Res.Release()); + Die(ctx); + } + + void PassAway() final { + LifetimeDuration->Collect((TInstant::Now() - StartTime).MilliSeconds()); + NActors::IActor::PassAway(); + } + + void Fail(const TString& message, Ydb::StatusIds::StatusCode reqStatus = Ydb::StatusIds::INTERNAL_ERROR) { + Issues.AddIssue(message); + const auto codeStr = Ydb::StatusIds_StatusCode_Name(reqStatus); + LOG_E(TStringBuilder() + << "Failed with code: " << codeStr + << " Details: " << Issues.ToString()); + auto Res = MakeHolder<TEvents::TEvGetTaskResponse>(); + Res->Status = reqStatus; + Res->Issues.AddIssues(Issues); + Send(Sender, Res.Release()); + PassAway(); + } + + void Bootstrap(const TActorContext& ctx) { + Become(&TGetTaskRequestActor::StateFunc); + const auto& req = Ev->Record; + OwnerId = req.owner_id(); + Host = req.host(); + LOG_D("Request CP::GetTask with size: " << req.ByteSize() << " bytes"); + RequestedMBytes->Collect(req.ByteSize() / 1024 / 1024); + ctx.Send(NYq::ControlPlaneStorageServiceActorId(), + new NYq::TEvControlPlaneStorage::TEvGetTaskRequest(OwnerId, Host)); + } + static TString GetServiceAccountId(const YandexQuery::IamAuth& auth) { return auth.has_service_account() ? auth.service_account().id() : TString{}; } - static TString ExtractServiceAccountId(const YandexQuery::Connection& c) { - switch (c.content().setting().connection_case()) { - case YandexQuery::ConnectionSetting::kYdbDatabase: { + static TString ExtractServiceAccountId(const YandexQuery::Connection& c) { + switch (c.content().setting().connection_case()) { + case YandexQuery::ConnectionSetting::kYdbDatabase: { return GetServiceAccountId(c.content().setting().ydb_database().auth()); - } - case YandexQuery::ConnectionSetting::kDataStreams: { + } + case YandexQuery::ConnectionSetting::kDataStreams: { return GetServiceAccountId(c.content().setting().data_streams().auth()); - } - case YandexQuery::ConnectionSetting::kObjectStorage: { + } + case YandexQuery::ConnectionSetting::kObjectStorage: { return GetServiceAccountId(c.content().setting().object_storage().auth()); - } - case YandexQuery::ConnectionSetting::kMonitoring: { + } + case YandexQuery::ConnectionSetting::kMonitoring: { return GetServiceAccountId(c.content().setting().monitoring().auth()); - } + } case YandexQuery::ConnectionSetting::kClickhouseCluster: { return GetServiceAccountId(c.content().setting().clickhouse_cluster().auth()); } // Do not replace with default. Adding a new connection should cause a compilation error case YandexQuery::ConnectionSetting::CONNECTION_NOT_SET: break; - } - return {}; - } - -private: - void HandleResponse(NYq::TEvControlPlaneStorage::TEvGetTaskResponse::TPtr& ev, const TActorContext& ctx) { // YQ - LOG_D("Got CP::GetTask Response"); - const auto& tasks = ev->Get()->Tasks; - Res->Record.ConstructInPlace(); - Res->Status = Ydb::StatusIds::SUCCESS; - const auto& issues = ev->Get()->Issues; - if (issues) { - Issues.AddIssues(issues); - Fail("ControlPlane::GetTaskError", Ydb::StatusIds::GENERIC_ERROR); - return; - } - - try { - for (const auto& task : tasks) { - const auto& queryType = task.Query.content().type(); - if (queryType != YandexQuery::QueryContent::ANALYTICS && queryType != YandexQuery::QueryContent::STREAMING) { //TODO: fix - ythrow yexception() - << "query type " - << YandexQuery::QueryContent::QueryType_Name(queryType) - << " unsupported"; - } - auto* newTask = Res->Record->add_tasks(); - newTask->set_query_type(queryType); - newTask->set_execute_mode(task.Query.meta().execute_mode()); - newTask->set_state_load_mode(task.Internal.state_load_mode()); - auto* queryId = newTask->mutable_query_id(); - queryId->set_value(task.Query.meta().common().id()); - newTask->set_streaming(queryType == YandexQuery::QueryContent::STREAMING); - newTask->set_text(task.Query.content().text()); - *newTask->mutable_connection() = task.Internal.connection(); - *newTask->mutable_binding() = task.Internal.binding(); - newTask->set_user_token(task.Internal.token()); - newTask->set_user_id(task.Query.meta().common().created_by()); - newTask->set_generation(task.Generation); - newTask->set_status(task.Query.meta().status()); - *newTask->mutable_created_topic_consumers() = task.Internal.created_topic_consumers(); - newTask->mutable_sensor_labels()->insert({"cloud_id", task.Internal.cloud_id()}); - newTask->set_automatic(task.Query.content().automatic()); - newTask->set_query_name(task.Query.content().name()); + } + return {}; + } + +private: + void HandleResponse(NYq::TEvControlPlaneStorage::TEvGetTaskResponse::TPtr& ev, const TActorContext& ctx) { // YQ + LOG_D("Got CP::GetTask Response"); + const auto& tasks = ev->Get()->Tasks; + Res->Record.ConstructInPlace(); + Res->Status = Ydb::StatusIds::SUCCESS; + const auto& issues = ev->Get()->Issues; + if (issues) { + Issues.AddIssues(issues); + Fail("ControlPlane::GetTaskError", Ydb::StatusIds::GENERIC_ERROR); + return; + } + + try { + for (const auto& task : tasks) { + const auto& queryType = task.Query.content().type(); + if (queryType != YandexQuery::QueryContent::ANALYTICS && queryType != YandexQuery::QueryContent::STREAMING) { //TODO: fix + ythrow yexception() + << "query type " + << YandexQuery::QueryContent::QueryType_Name(queryType) + << " unsupported"; + } + auto* newTask = Res->Record->add_tasks(); + newTask->set_query_type(queryType); + newTask->set_execute_mode(task.Query.meta().execute_mode()); + newTask->set_state_load_mode(task.Internal.state_load_mode()); + auto* queryId = newTask->mutable_query_id(); + queryId->set_value(task.Query.meta().common().id()); + newTask->set_streaming(queryType == YandexQuery::QueryContent::STREAMING); + newTask->set_text(task.Query.content().text()); + *newTask->mutable_connection() = task.Internal.connection(); + *newTask->mutable_binding() = task.Internal.binding(); + newTask->set_user_token(task.Internal.token()); + newTask->set_user_id(task.Query.meta().common().created_by()); + newTask->set_generation(task.Generation); + newTask->set_status(task.Query.meta().status()); + *newTask->mutable_created_topic_consumers() = task.Internal.created_topic_consumers(); + newTask->mutable_sensor_labels()->insert({"cloud_id", task.Internal.cloud_id()}); + newTask->set_automatic(task.Query.content().automatic()); + newTask->set_query_name(task.Query.content().name()); *newTask->mutable_deadline() = NProtoInterop::CastToProto(task.Deadline); newTask->mutable_disposition()->CopyFrom(task.Internal.disposition()); - - THashMap<TString, TString> accountIdSignatures; - for (const auto& connection: task.Internal.connection()) { - const auto serviceAccountId = ExtractServiceAccountId(connection); - if (!serviceAccountId) { - continue; - } - - auto& signature = accountIdSignatures[serviceAccountId]; - if (!signature && Signer) { - signature = Signer->SignAccountId(serviceAccountId); - } - auto* account = newTask->add_service_accounts(); - account->set_value(serviceAccountId); - account->set_signature(signature); - } - - *newTask->mutable_dq_graph() = task.Internal.dq_graph(); - newTask->set_dq_graph_index(task.Internal.dq_graph_index()); - - *newTask->mutable_result_set_meta() = task.Query.result_set_meta(); - newTask->set_scope(task.Scope); - } - ctx.Send(Sender, Res.Release()); - Die(ctx); - } catch (...) { - const auto msg = TStringBuilder() << "Can't do GetTask: " << CurrentExceptionMessage(); - Fail(msg); - } - } - -private: - STRICT_STFUNC( - StateFunc, - CFunc(NActors::TEvents::TEvPoison::EventType, Die) - HFunc(NActors::TEvents::TEvUndelivered, OnUndelivered) - HFunc(NYq::TEvControlPlaneStorage::TEvGetTaskResponse, HandleResponse) - ) - + + THashMap<TString, TString> accountIdSignatures; + for (const auto& connection: task.Internal.connection()) { + const auto serviceAccountId = ExtractServiceAccountId(connection); + if (!serviceAccountId) { + continue; + } + + auto& signature = accountIdSignatures[serviceAccountId]; + if (!signature && Signer) { + signature = Signer->SignAccountId(serviceAccountId); + } + auto* account = newTask->add_service_accounts(); + account->set_value(serviceAccountId); + account->set_signature(signature); + } + + *newTask->mutable_dq_graph() = task.Internal.dq_graph(); + newTask->set_dq_graph_index(task.Internal.dq_graph_index()); + + *newTask->mutable_result_set_meta() = task.Query.result_set_meta(); + newTask->set_scope(task.Scope); + } + ctx.Send(Sender, Res.Release()); + Die(ctx); + } catch (...) { + const auto msg = TStringBuilder() << "Can't do GetTask: " << CurrentExceptionMessage(); + Fail(msg); + } + } + +private: + STRICT_STFUNC( + StateFunc, + CFunc(NActors::TEvents::TEvPoison::EventType, Die) + HFunc(NActors::TEvents::TEvUndelivered, OnUndelivered) + HFunc(NYq::TEvControlPlaneStorage::TEvGetTaskResponse, HandleResponse) + ) + const NConfig::TTokenAccessorConfig TokenAccessorConfig; - const TActorId Sender; - TIntrusivePtr<ITimeProvider> TimeProvider; - TAutoPtr<TEvents::TEvGetTaskRequest> Ev; - TDynamicCounterPtr Counters; - const THistogramPtr LifetimeDuration; - const THistogramPtr RequestedMBytes; - const TInstant StartTime; - + const TActorId Sender; + TIntrusivePtr<ITimeProvider> TimeProvider; + TAutoPtr<TEvents::TEvGetTaskRequest> Ev; + TDynamicCounterPtr Counters; + const THistogramPtr LifetimeDuration; + const THistogramPtr RequestedMBytes; + const TInstant StartTime; + ::NYq::TSigner::TPtr Signer; - - NYql::TIssues Issues; - TString OwnerId; - TString Host; - - THolder<TEvents::TEvGetTaskResponse> Res = MakeHolder<TEvents::TEvGetTaskResponse>(); -}; - -IActor* CreateGetTaskRequestActor( - const NActors::TActorId& sender, + + NYql::TIssues Issues; + TString OwnerId; + TString Host; + + THolder<TEvents::TEvGetTaskResponse> Res = MakeHolder<TEvents::TEvGetTaskResponse>(); +}; + +IActor* CreateGetTaskRequestActor( + const NActors::TActorId& sender, const NConfig::TTokenAccessorConfig& tokenAccessorConfig, - TIntrusivePtr<ITimeProvider> timeProvider, - TAutoPtr<TEvents::TEvGetTaskRequest> ev, - TDynamicCounterPtr counters) { - return new TGetTaskRequestActor( - sender, + TIntrusivePtr<ITimeProvider> timeProvider, + TAutoPtr<TEvents::TEvGetTaskRequest> ev, + TDynamicCounterPtr counters) { + return new TGetTaskRequestActor( + sender, tokenAccessorConfig, - timeProvider, - std::move(ev), - counters); -} - -} /* NYq */ + timeProvider, + std::move(ev), + counters); +} + +} /* NYq */ diff --git a/ydb/core/yq/libs/actors/task_ping.cpp b/ydb/core/yq/libs/actors/task_ping.cpp index 95623f0d9d..29410dccd1 100644 --- a/ydb/core/yq/libs/actors/task_ping.cpp +++ b/ydb/core/yq/libs/actors/task_ping.cpp @@ -1,157 +1,157 @@ -#include "proxy_private.h" +#include "proxy_private.h" #include <util/datetime/base.h> - + #include <ydb/core/protos/services.pb.h> #include <ydb/library/yql/public/issue/yql_issue_message.h> - -#include <library/cpp/yson/node/node_io.h> -#include <library/cpp/actors/core/events.h> -#include <library/cpp/actors/core/hfunc.h> -#include <library/cpp/actors/core/actor_bootstrapped.h> -#include <library/cpp/actors/core/log.h> + +#include <library/cpp/yson/node/node_io.h> +#include <library/cpp/actors/core/events.h> +#include <library/cpp/actors/core/hfunc.h> +#include <library/cpp/actors/core/actor_bootstrapped.h> +#include <library/cpp/actors/core/log.h> #include <library/cpp/protobuf/interop/cast.h> - + #include <ydb/core/yq/libs/control_plane_storage/control_plane_storage.h> #include <ydb/core/yq/libs/control_plane_storage/events/events.h> -#include <google/protobuf/util/time_util.h> - +#include <google/protobuf/util/time_util.h> + #define LOG_E(stream) \ LOG_ERROR_S(*TlsActivationContext, NKikimrServices::YQL_PRIVATE_PROXY, "PrivatePingTask - QueryId: " << OperationId << ", Owner: " << OwnerId << ", " << stream) #define LOG_D(stream) \ LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::YQL_PRIVATE_PROXY, "PrivatePingTask - QueryId: " << OperationId << ", Owner: " << OwnerId << ", "<< stream) - -namespace NYq { - -using namespace NActors; -using namespace NMonitoring; - -class TTaskPingRequestActor - : public NActors::TActorBootstrapped<TTaskPingRequestActor> -{ -public: - TTaskPingRequestActor( - const NActors::TActorId& sender, - TIntrusivePtr<ITimeProvider> timeProvider, - TAutoPtr<TEvents::TEvPingTaskRequest> ev, - TDynamicCounterPtr counters) + +namespace NYq { + +using namespace NActors; +using namespace NMonitoring; + +class TTaskPingRequestActor + : public NActors::TActorBootstrapped<TTaskPingRequestActor> +{ +public: + TTaskPingRequestActor( + const NActors::TActorId& sender, + TIntrusivePtr<ITimeProvider> timeProvider, + TAutoPtr<TEvents::TEvPingTaskRequest> ev, + TDynamicCounterPtr counters) : Sender(sender) - , TimeProvider(timeProvider) - , Ev(std::move(ev)) - , Counters(std::move(counters->GetSubgroup("subsystem", "private_api")->GetSubgroup("subcomponent", "PingTask"))) - , LifetimeDuration(Counters->GetHistogram("LifetimeDurationMs", ExponentialHistogram(10, 2, 50))) - , RequestedMBytes(Counters->GetHistogram("RequestedMB", ExponentialHistogram(6, 2, 3))) - , StartTime(TInstant::Now()) - {} - - static constexpr char ActorName[] = "YQ_PRIVATE_PING_TASK"; - - void OnUndelivered(NActors::TEvents::TEvUndelivered::TPtr& ev, const NActors::TActorContext& ctx) { - LOG_E("TTaskPingRequestActor::OnUndelivered"); - auto res = MakeHolder<TEvents::TEvPingTaskResponse>(); - res->Status = Ydb::StatusIds::GENERIC_ERROR; - res->Issues.AddIssue("UNDELIVERED"); - ctx.Send(ev->Sender, res.Release()); - Die(ctx); - } - - void PassAway() final { - LifetimeDuration->Collect((TInstant::Now() - StartTime).MilliSeconds()); - NActors::IActor::PassAway(); - } - - void Fail(const TString& message, Ydb::StatusIds::StatusCode reqStatus = Ydb::StatusIds::INTERNAL_ERROR) { - Issues.AddIssue(message); - const auto codeStr = Ydb::StatusIds_StatusCode_Name(reqStatus); - LOG_E(TStringBuilder() - << "Failed with code: " << codeStr - << " Details: " << Issues.ToString()); - auto res = MakeHolder<TEvents::TEvPingTaskResponse>(); - res->Status = reqStatus; - res->Issues.AddIssues(Issues); - Send(Sender, res.Release()); - PassAway(); - } - - void Bootstrap(const TActorContext& ctx) { - Y_UNUSED(ctx); - Become(&TTaskPingRequestActor::StateFunc); - const auto& req = Ev->Record; - OperationId = req.query_id().value(); + , TimeProvider(timeProvider) + , Ev(std::move(ev)) + , Counters(std::move(counters->GetSubgroup("subsystem", "private_api")->GetSubgroup("subcomponent", "PingTask"))) + , LifetimeDuration(Counters->GetHistogram("LifetimeDurationMs", ExponentialHistogram(10, 2, 50))) + , RequestedMBytes(Counters->GetHistogram("RequestedMB", ExponentialHistogram(6, 2, 3))) + , StartTime(TInstant::Now()) + {} + + static constexpr char ActorName[] = "YQ_PRIVATE_PING_TASK"; + + void OnUndelivered(NActors::TEvents::TEvUndelivered::TPtr& ev, const NActors::TActorContext& ctx) { + LOG_E("TTaskPingRequestActor::OnUndelivered"); + auto res = MakeHolder<TEvents::TEvPingTaskResponse>(); + res->Status = Ydb::StatusIds::GENERIC_ERROR; + res->Issues.AddIssue("UNDELIVERED"); + ctx.Send(ev->Sender, res.Release()); + Die(ctx); + } + + void PassAway() final { + LifetimeDuration->Collect((TInstant::Now() - StartTime).MilliSeconds()); + NActors::IActor::PassAway(); + } + + void Fail(const TString& message, Ydb::StatusIds::StatusCode reqStatus = Ydb::StatusIds::INTERNAL_ERROR) { + Issues.AddIssue(message); + const auto codeStr = Ydb::StatusIds_StatusCode_Name(reqStatus); + LOG_E(TStringBuilder() + << "Failed with code: " << codeStr + << " Details: " << Issues.ToString()); + auto res = MakeHolder<TEvents::TEvPingTaskResponse>(); + res->Status = reqStatus; + res->Issues.AddIssues(Issues); + Send(Sender, res.Release()); + PassAway(); + } + + void Bootstrap(const TActorContext& ctx) { + Y_UNUSED(ctx); + Become(&TTaskPingRequestActor::StateFunc); + const auto& req = Ev->Record; + OperationId = req.query_id().value(); OwnerId = req.owner_id(); - Scope = req.scope(); + Scope = req.scope(); Deadline = NProtoInterop::CastFromProto(req.deadline()); - LOG_D("Request CP::PingTask with size: " << req.ByteSize() << " bytes"); - RequestedMBytes->Collect(req.ByteSize() / 1024 / 1024); - try { - auto event = CreateControlPlaneEvent(); - Send(NYq::ControlPlaneStorageServiceActorId(), event.release()); - } catch (const std::exception& err) { - const auto msg = TStringBuilder() << "PingTask Boostrap Error: " << CurrentExceptionMessage(); - Fail(msg); - } - } - -private: - STRICT_STFUNC( - StateFunc, - CFunc(NActors::TEvents::TEvPoison::EventType, Die) - HFunc(NYq::TEvControlPlaneStorage::TEvPingTaskResponse, HandleResponse) - HFunc(NActors::TEvents::TEvUndelivered, OnUndelivered) - ) - - std::unique_ptr<NYq::TEvControlPlaneStorage::TEvPingTaskRequest> CreateControlPlaneEvent() { + LOG_D("Request CP::PingTask with size: " << req.ByteSize() << " bytes"); + RequestedMBytes->Collect(req.ByteSize() / 1024 / 1024); + try { + auto event = CreateControlPlaneEvent(); + Send(NYq::ControlPlaneStorageServiceActorId(), event.release()); + } catch (const std::exception& err) { + const auto msg = TStringBuilder() << "PingTask Boostrap Error: " << CurrentExceptionMessage(); + Fail(msg); + } + } + +private: + STRICT_STFUNC( + StateFunc, + CFunc(NActors::TEvents::TEvPoison::EventType, Die) + HFunc(NYq::TEvControlPlaneStorage::TEvPingTaskResponse, HandleResponse) + HFunc(NActors::TEvents::TEvUndelivered, OnUndelivered) + ) + + std::unique_ptr<NYq::TEvControlPlaneStorage::TEvPingTaskRequest> CreateControlPlaneEvent() { auto event = std::make_unique<NYq::TEvControlPlaneStorage::TEvPingTaskRequest>(Scope, OperationId, OwnerId, Deadline); - const auto& req = Ev->Record; - ui64 issuesByteSize = 0; - ui64 transientIssuesByteSize = 0; - ui64 resultSetMetaByteSize = 0; - ui64 dqGraphBytesSize = 0; - - //TODO use all fields - if (req.status() != YandexQuery::QueryMeta::COMPUTE_STATUS_UNSPECIFIED) { - event->Status = req.status(); - } - if (!req.issues().empty()) { - NYql::TIssues reqIssues; - for (const auto& issue : req.issues()) { - issuesByteSize += issue.ByteSize(); - } - NYql::IssuesFromMessage(req.issues(), reqIssues); - Issues.AddIssues(reqIssues); - event->Issues = Issues; - } + const auto& req = Ev->Record; + ui64 issuesByteSize = 0; + ui64 transientIssuesByteSize = 0; + ui64 resultSetMetaByteSize = 0; + ui64 dqGraphBytesSize = 0; + + //TODO use all fields + if (req.status() != YandexQuery::QueryMeta::COMPUTE_STATUS_UNSPECIFIED) { + event->Status = req.status(); + } + if (!req.issues().empty()) { + NYql::TIssues reqIssues; + for (const auto& issue : req.issues()) { + issuesByteSize += issue.ByteSize(); + } + NYql::IssuesFromMessage(req.issues(), reqIssues); + Issues.AddIssues(reqIssues); + event->Issues = Issues; + } if (!req.transient_issues().empty()) { NYql::TIssues transientIssues; - for (const auto& issue : req.transient_issues()) { - transientIssuesByteSize += issue.ByteSize(); - } + for (const auto& issue : req.transient_issues()) { + transientIssuesByteSize += issue.ByteSize(); + } NYql::IssuesFromMessage(req.transient_issues(), transientIssues); event->TransientIssues = transientIssues; } - if (req.statistics()) { - event->Statistics = req.statistics(); - } - if (req.ast()) { - event->Ast = req.ast(); - } - if (req.result_id().value()) { - event->ResultId = req.result_id().value(); - } - if (req.plan()) { - event->Plan = req.plan(); - } - if (!req.result_set_meta().empty()) { - for (const auto& rsMeta : req.result_set_meta()) { - resultSetMetaByteSize += rsMeta.ByteSize(); - } + if (req.statistics()) { + event->Statistics = req.statistics(); + } + if (req.ast()) { + event->Ast = req.ast(); + } + if (req.result_id().value()) { + event->ResultId = req.result_id().value(); + } + if (req.plan()) { + event->Plan = req.plan(); + } + if (!req.result_set_meta().empty()) { + for (const auto& rsMeta : req.result_set_meta()) { + resultSetMetaByteSize += rsMeta.ByteSize(); + } event->ResultSetMetas = {req.result_set_meta().begin(), req.result_set_meta().end()}; - } - if (req.has_started_at()) { + } + if (req.has_started_at()) { event->StartedAt = TInstant::FromValue(google::protobuf::util::TimeUtil::TimestampToMicroseconds(req.started_at())); - } - if (req.has_finished_at()) { + } + if (req.has_finished_at()) { event->FinishedAt = TInstant::FromValue(google::protobuf::util::TimeUtil::TimestampToMicroseconds(req.finished_at())); - } + } event->ResignQuery = req.resign_query(); event->CreatedTopicConsumers.reserve(req.created_topic_consumers_size()); @@ -169,76 +169,76 @@ private: event->DqGraphs.reserve(req.dq_graph_size()); for (const auto& g : req.dq_graph()) { - dqGraphBytesSize += g.size(); + dqGraphBytesSize += g.size(); event->DqGraphs.emplace_back(g); } if (req.state_load_mode()) { event->StateLoadMode = req.state_load_mode(); } - + if (req.has_disposition()) { event->StreamingDisposition = req.disposition(); } - LOG_D("Statistics length: " << req.statistics().size() << ", " - << "Ast length: " << req.ast().size() << " bytes, " - << "Plan length: " << req.plan().size() << " bytes, " - << "Result set meta size: " << resultSetMetaByteSize << " bytes, " - << "Topic consumers size: " << event->CreatedTopicConsumers.size() * sizeof(TEvControlPlaneStorage::TTopicConsumer) << " bytes, " - << "Dq graphs size: " << dqGraphBytesSize << " bytes, " - << "Issues size: " << issuesByteSize << " bytes, " - << "Transient issues size: " << transientIssuesByteSize << " bytes"); - + LOG_D("Statistics length: " << req.statistics().size() << ", " + << "Ast length: " << req.ast().size() << " bytes, " + << "Plan length: " << req.plan().size() << " bytes, " + << "Result set meta size: " << resultSetMetaByteSize << " bytes, " + << "Topic consumers size: " << event->CreatedTopicConsumers.size() * sizeof(TEvControlPlaneStorage::TTopicConsumer) << " bytes, " + << "Dq graphs size: " << dqGraphBytesSize << " bytes, " + << "Issues size: " << issuesByteSize << " bytes, " + << "Transient issues size: " << transientIssuesByteSize << " bytes"); + event->DqGraphIndex = req.dq_graph_index(); - return std::move(event); - } - - void HandleResponse(NYq::TEvControlPlaneStorage::TEvPingTaskResponse::TPtr& ev, const TActorContext& ctx) { - LOG_D("Got CP::PingTaskResponse"); - const auto& issues = ev->Get()->Issues; - if (issues) { - Issues.AddIssues(issues); - Fail("ControlPlane PingTaskError", Ydb::StatusIds::GENERIC_ERROR); - return; - } - auto res = MakeHolder<TEvents::TEvPingTaskResponse>(); - res->Status = Ydb::StatusIds::SUCCESS; - Yq::Private::PingTaskResult result; + return std::move(event); + } + + void HandleResponse(NYq::TEvControlPlaneStorage::TEvPingTaskResponse::TPtr& ev, const TActorContext& ctx) { + LOG_D("Got CP::PingTaskResponse"); + const auto& issues = ev->Get()->Issues; + if (issues) { + Issues.AddIssues(issues); + Fail("ControlPlane PingTaskError", Ydb::StatusIds::GENERIC_ERROR); + return; + } + auto res = MakeHolder<TEvents::TEvPingTaskResponse>(); + res->Status = Ydb::StatusIds::SUCCESS; + Yq::Private::PingTaskResult result; result.set_action(ev->Get()->Action); res->Record.ConstructInPlace(result); - ctx.Send(Sender, res.Release()); - Die(ctx); - } - -private: - const TActorId Sender; - TIntrusivePtr<ITimeProvider> TimeProvider; - TAutoPtr<TEvents::TEvPingTaskRequest> Ev; - TDynamicCounterPtr Counters; - const THistogramPtr LifetimeDuration; - const THistogramPtr RequestedMBytes; - const TInstant StartTime; - - TString OperationId; + ctx.Send(Sender, res.Release()); + Die(ctx); + } + +private: + const TActorId Sender; + TIntrusivePtr<ITimeProvider> TimeProvider; + TAutoPtr<TEvents::TEvPingTaskRequest> Ev; + TDynamicCounterPtr Counters; + const THistogramPtr LifetimeDuration; + const THistogramPtr RequestedMBytes; + const TInstant StartTime; + + TString OperationId; TString OwnerId; - TString Scope; + TString Scope; TInstant Deadline; - NYql::TIssues Issues; -}; - -IActor* CreatePingTaskRequestActor( - const NActors::TActorId& sender, - TIntrusivePtr<ITimeProvider> timeProvider, - TAutoPtr<TEvents::TEvPingTaskRequest> ev, - TDynamicCounterPtr counters) { - return new TTaskPingRequestActor( - sender, - timeProvider, - std::move(ev), - std::move(counters)); -} - -} /* NYq */ + NYql::TIssues Issues; +}; + +IActor* CreatePingTaskRequestActor( + const NActors::TActorId& sender, + TIntrusivePtr<ITimeProvider> timeProvider, + TAutoPtr<TEvents::TEvPingTaskRequest> ev, + TDynamicCounterPtr counters) { + return new TTaskPingRequestActor( + sender, + timeProvider, + std::move(ev), + std::move(counters)); +} + +} /* NYq */ diff --git a/ydb/core/yq/libs/actors/task_result_write.cpp b/ydb/core/yq/libs/actors/task_result_write.cpp index 760b1b56c4..5919ddbaab 100644 --- a/ydb/core/yq/libs/actors/task_result_write.cpp +++ b/ydb/core/yq/libs/actors/task_result_write.cpp @@ -1,144 +1,144 @@ -#include "proxy_private.h" - +#include "proxy_private.h" + #include <ydb/core/protos/services.pb.h> - + #include <ydb/library/yql/public/issue/yql_issue_message.h> - -#include <library/cpp/yson/node/node_io.h> -#include <library/cpp/actors/core/events.h> -#include <library/cpp/actors/core/hfunc.h> -#include <library/cpp/actors/core/actor_bootstrapped.h> -#include <library/cpp/actors/core/log.h> + +#include <library/cpp/yson/node/node_io.h> +#include <library/cpp/actors/core/events.h> +#include <library/cpp/actors/core/hfunc.h> +#include <library/cpp/actors/core/actor_bootstrapped.h> +#include <library/cpp/actors/core/log.h> #include <library/cpp/protobuf/interop/cast.h> - + #include <ydb/core/yq/libs/control_plane_storage/events/events.h> #include <ydb/core/yq/libs/control_plane_storage/control_plane_storage.h> - + #define LOG_E(stream) \ LOG_ERROR_S(*TlsActivationContext, NKikimrServices::YQL_PRIVATE_PROXY, "PrivateWriteTask - ResultId: " << ResultId << ", RequestId: " << RequestId << ", " << stream) #define LOG_D(stream) \ LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::YQL_PRIVATE_PROXY, "PrivateWriteTask - ResultId: " << ResultId << ", RequestId: " << RequestId << ", " << stream) - -namespace NYq { - -using namespace NActors; -using namespace NMonitoring; - - -class TWriteTaskRequestActor - : public NActors::TActorBootstrapped<TWriteTaskRequestActor> -{ -public: - TWriteTaskRequestActor( - const NActors::TActorId& sender, - TIntrusivePtr<ITimeProvider> timeProvider, - TAutoPtr<TEvents::TEvWriteTaskResultRequest> ev, - TDynamicCounterPtr counters) + +namespace NYq { + +using namespace NActors; +using namespace NMonitoring; + + +class TWriteTaskRequestActor + : public NActors::TActorBootstrapped<TWriteTaskRequestActor> +{ +public: + TWriteTaskRequestActor( + const NActors::TActorId& sender, + TIntrusivePtr<ITimeProvider> timeProvider, + TAutoPtr<TEvents::TEvWriteTaskResultRequest> ev, + TDynamicCounterPtr counters) : Sender(sender) - , TimeProvider(timeProvider) - , Ev(std::move(ev)) - , Counters(std::move(counters->GetSubgroup("subsystem", "private_api")->GetSubgroup("subcomponent", "WriteTaskResult"))) - , LifetimeDuration(Counters->GetHistogram("LifetimeDurationMs", ExponentialHistogram(10, 2, 50))) - , RequestedMBytes(Counters->GetHistogram("RequestedMB", ExponentialHistogram(6, 2, 3))) - , StartTime(TInstant::Now()) - - {} - - static constexpr char ActorName[] = "YQ_PRIVATE_WRITE_RESULT_TASK"; - - void OnUndelivered(NActors::TEvents::TEvUndelivered::TPtr& ev, const NActors::TActorContext& ctx) { - LOG_E("TWriteTaskRequestActor::OnUndelivered"); - Res->Status = Ydb::StatusIds::GENERIC_ERROR; - Res->Issues.AddIssue("UNDELIVERED"); - ctx.Send(ev->Sender, Res.Release()); - Die(ctx); - } - - void PassAway() final { - LifetimeDuration->Collect((TInstant::Now() - StartTime).MilliSeconds()); - NActors::IActor::PassAway(); - } - - void Fail(const TString& message, Ydb::StatusIds::StatusCode reqStatus = Ydb::StatusIds::INTERNAL_ERROR) { - Issues.AddIssue(message); - const auto codeStr = Ydb::StatusIds_StatusCode_Name(reqStatus); - LOG_E(TStringBuilder() - << "Failed with code: " << codeStr - << " Details: " << Issues.ToString()); - Res->Status = reqStatus; - Res->Issues.AddIssues(Issues); - Send(Sender, Res.Release()); - PassAway(); - } - + , TimeProvider(timeProvider) + , Ev(std::move(ev)) + , Counters(std::move(counters->GetSubgroup("subsystem", "private_api")->GetSubgroup("subcomponent", "WriteTaskResult"))) + , LifetimeDuration(Counters->GetHistogram("LifetimeDurationMs", ExponentialHistogram(10, 2, 50))) + , RequestedMBytes(Counters->GetHistogram("RequestedMB", ExponentialHistogram(6, 2, 3))) + , StartTime(TInstant::Now()) + + {} + + static constexpr char ActorName[] = "YQ_PRIVATE_WRITE_RESULT_TASK"; + + void OnUndelivered(NActors::TEvents::TEvUndelivered::TPtr& ev, const NActors::TActorContext& ctx) { + LOG_E("TWriteTaskRequestActor::OnUndelivered"); + Res->Status = Ydb::StatusIds::GENERIC_ERROR; + Res->Issues.AddIssue("UNDELIVERED"); + ctx.Send(ev->Sender, Res.Release()); + Die(ctx); + } + + void PassAway() final { + LifetimeDuration->Collect((TInstant::Now() - StartTime).MilliSeconds()); + NActors::IActor::PassAway(); + } + + void Fail(const TString& message, Ydb::StatusIds::StatusCode reqStatus = Ydb::StatusIds::INTERNAL_ERROR) { + Issues.AddIssue(message); + const auto codeStr = Ydb::StatusIds_StatusCode_Name(reqStatus); + LOG_E(TStringBuilder() + << "Failed with code: " << codeStr + << " Details: " << Issues.ToString()); + Res->Status = reqStatus; + Res->Issues.AddIssues(Issues); + Send(Sender, Res.Release()); + PassAway(); + } + void Bootstrap(const TActorContext&) { - Become(&TWriteTaskRequestActor::StateFunc); - const auto& req = Ev->Record; - + Become(&TWriteTaskRequestActor::StateFunc); + const auto& req = Ev->Record; + Deadline = NProtoInterop::CastFromProto(req.deadline()); - - const auto& resultSet = req.result_set(); - ResultId = req.result_id().value(); - const auto& resultSetId = req.result_set_id(); - const auto& startRowIndex = req.offset(); - RequestId = req.request_id(); - LOG_D("Request CP::WriteTaskResult with size: " << req.ByteSize() << " bytes"); - RequestedMBytes->Collect(req.ByteSize() / 1024 / 1024); - Send(NYq::ControlPlaneStorageServiceActorId(), - new NYq::TEvControlPlaneStorage::TEvWriteResultDataRequest(ResultId, resultSetId, startRowIndex, Deadline, resultSet), 0, RequestId); - } - -private: - STRICT_STFUNC( - StateFunc, - CFunc(NActors::TEvents::TEvPoison::EventType, Die) - HFunc(NActors::TEvents::TEvUndelivered, OnUndelivered) - HFunc(NYq::TEvControlPlaneStorage::TEvWriteResultDataResponse, HandleResponse); - ) - - void HandleResponse(NYq::TEvControlPlaneStorage::TEvWriteResultDataResponse::TPtr& ev, const TActorContext& ctx) { - LOG_D("Got CP::WriteTaskResult Response"); - const auto& issues = ev->Get()->Issues; - if (issues) { - Issues.AddIssues(issues); - Fail("ControlPlane WriteTaskResult Error", Ydb::StatusIds::GENERIC_ERROR); - return; - } - Res->Record.ConstructInPlace(); - Res->Record->set_request_id(RequestId); - Res->Issues.AddIssues(Issues); - Res->Status = Ydb::StatusIds::SUCCESS; - ctx.Send(Sender, Res.Release()); - Die(ctx); - } - - const TActorId Sender; - TIntrusivePtr<ITimeProvider> TimeProvider; - TAutoPtr<TEvents::TEvWriteTaskResultRequest> Ev; - TDynamicCounterPtr Counters; - const THistogramPtr LifetimeDuration; - const THistogramPtr RequestedMBytes; - const TInstant StartTime; - - TString ResultId; - ui64 RequestId = 0; - TInstant Deadline; - - NYql::TIssues Issues; - - THolder<TEvents::TEvWriteTaskResultResponse> Res = MakeHolder<TEvents::TEvWriteTaskResultResponse>(); -}; - -IActor* CreateWriteTaskResultRequestActor( - const NActors::TActorId& sender, - TIntrusivePtr<ITimeProvider> timeProvider, - TAutoPtr<TEvents::TEvWriteTaskResultRequest> ev, - TDynamicCounterPtr counters) { - return new TWriteTaskRequestActor( - sender, - timeProvider, - std::move(ev), - std::move(counters)); -} - -} /* NYq */ + + const auto& resultSet = req.result_set(); + ResultId = req.result_id().value(); + const auto& resultSetId = req.result_set_id(); + const auto& startRowIndex = req.offset(); + RequestId = req.request_id(); + LOG_D("Request CP::WriteTaskResult with size: " << req.ByteSize() << " bytes"); + RequestedMBytes->Collect(req.ByteSize() / 1024 / 1024); + Send(NYq::ControlPlaneStorageServiceActorId(), + new NYq::TEvControlPlaneStorage::TEvWriteResultDataRequest(ResultId, resultSetId, startRowIndex, Deadline, resultSet), 0, RequestId); + } + +private: + STRICT_STFUNC( + StateFunc, + CFunc(NActors::TEvents::TEvPoison::EventType, Die) + HFunc(NActors::TEvents::TEvUndelivered, OnUndelivered) + HFunc(NYq::TEvControlPlaneStorage::TEvWriteResultDataResponse, HandleResponse); + ) + + void HandleResponse(NYq::TEvControlPlaneStorage::TEvWriteResultDataResponse::TPtr& ev, const TActorContext& ctx) { + LOG_D("Got CP::WriteTaskResult Response"); + const auto& issues = ev->Get()->Issues; + if (issues) { + Issues.AddIssues(issues); + Fail("ControlPlane WriteTaskResult Error", Ydb::StatusIds::GENERIC_ERROR); + return; + } + Res->Record.ConstructInPlace(); + Res->Record->set_request_id(RequestId); + Res->Issues.AddIssues(Issues); + Res->Status = Ydb::StatusIds::SUCCESS; + ctx.Send(Sender, Res.Release()); + Die(ctx); + } + + const TActorId Sender; + TIntrusivePtr<ITimeProvider> TimeProvider; + TAutoPtr<TEvents::TEvWriteTaskResultRequest> Ev; + TDynamicCounterPtr Counters; + const THistogramPtr LifetimeDuration; + const THistogramPtr RequestedMBytes; + const TInstant StartTime; + + TString ResultId; + ui64 RequestId = 0; + TInstant Deadline; + + NYql::TIssues Issues; + + THolder<TEvents::TEvWriteTaskResultResponse> Res = MakeHolder<TEvents::TEvWriteTaskResultResponse>(); +}; + +IActor* CreateWriteTaskResultRequestActor( + const NActors::TActorId& sender, + TIntrusivePtr<ITimeProvider> timeProvider, + TAutoPtr<TEvents::TEvWriteTaskResultRequest> ev, + TDynamicCounterPtr counters) { + return new TWriteTaskRequestActor( + sender, + timeProvider, + std::move(ev), + std::move(counters)); +} + +} /* NYq */ diff --git a/ydb/core/yq/libs/actors/ya.make b/ydb/core/yq/libs/actors/ya.make index 347e1dbea7..3638593814 100644 --- a/ydb/core/yq/libs/actors/ya.make +++ b/ydb/core/yq/libs/actors/ya.make @@ -7,24 +7,24 @@ SRCS( database_resolver.cpp error.cpp nodes_health_check.cpp - nodes_manager.cpp + nodes_manager.cpp pending_fetcher.cpp pinger.cpp proxy.cpp proxy_private.cpp result_writer.cpp run_actor.cpp - run_actor_params.cpp + run_actor_params.cpp system_clusters.cpp table_bindings_from_bindings.cpp task_get.cpp - task_ping.cpp - task_result_write.cpp + task_ping.cpp + task_result_write.cpp ) PEERDIR( library/cpp/actors/core - library/cpp/actors/interconnect + library/cpp/actors/interconnect library/cpp/json/yson library/cpp/monlib/dynamic_counters library/cpp/random_provider diff --git a/ydb/core/yq/libs/checkpoint_storage/gc.cpp b/ydb/core/yq/libs/checkpoint_storage/gc.cpp index 88a141ee65..2a0e73ecb4 100644 --- a/ydb/core/yq/libs/checkpoint_storage/gc.cpp +++ b/ydb/core/yq/libs/checkpoint_storage/gc.cpp @@ -61,7 +61,7 @@ public: void Bootstrap(const TActorContext& ctx); static constexpr char ActorName[] = "YQ_GC_ACTOR"; - + private: STRICT_STFUNC(StateFunc, HFunc(TEvCheckpointStorage::TEvNewCheckpointSucceeded, Handle); diff --git a/ydb/core/yq/libs/checkpoint_storage/storage_proxy.cpp b/ydb/core/yq/libs/checkpoint_storage/storage_proxy.cpp index e92fc1b208..07c8d5101a 100644 --- a/ydb/core/yq/libs/checkpoint_storage/storage_proxy.cpp +++ b/ydb/core/yq/libs/checkpoint_storage/storage_proxy.cpp @@ -51,7 +51,7 @@ public: void Bootstrap(); static constexpr char ActorName[] = "YQ_STORAGE_PROXY"; - + private: STRICT_STFUNC(StateFunc, hFunc(TEvCheckpointStorage::TEvRegisterCoordinatorRequest, Handle); diff --git a/ydb/core/yq/libs/checkpointing/checkpoint_coordinator.h b/ydb/core/yq/libs/checkpointing/checkpoint_coordinator.h index cd2b68a35e..0bb76685e1 100644 --- a/ydb/core/yq/libs/checkpointing/checkpoint_coordinator.h +++ b/ydb/core/yq/libs/checkpointing/checkpoint_coordinator.h @@ -76,8 +76,8 @@ public: void Bootstrap(); - static constexpr char ActorName[] = "YQ_CHECKPOINT_COORDINATOR"; - + static constexpr char ActorName[] = "YQ_CHECKPOINT_COORDINATOR"; + private: void InitCheckpoint(); void InjectCheckpoint(const TCheckpointId& checkpointId); diff --git a/ydb/core/yq/libs/common/cache.h b/ydb/core/yq/libs/common/cache.h index 0a27831310..e79eb32839 100644 --- a/ydb/core/yq/libs/common/cache.h +++ b/ydb/core/yq/libs/common/cache.h @@ -4,7 +4,7 @@ #include <util/datetime/base.h> #include <util/system/mutex.h> -namespace NYq { +namespace NYq { struct TTtlCacheSettings { TDuration Ttl = TDuration::Minutes(10); @@ -113,4 +113,4 @@ private: TTtlCacheSettings Config; }; -} // namespace NYq +} // namespace NYq diff --git a/ydb/core/yq/libs/common/cache_ut.cpp b/ydb/core/yq/libs/common/cache_ut.cpp index 005689a709..4e5ace033d 100644 --- a/ydb/core/yq/libs/common/cache_ut.cpp +++ b/ydb/core/yq/libs/common/cache_ut.cpp @@ -2,7 +2,7 @@ #include <ydb/services/ydb/ydb_common_ut.h> -using namespace NYq; +using namespace NYq; using TCache = TTtlCache<int,int>; diff --git a/ydb/core/yq/libs/common/database_token_builder.cpp b/ydb/core/yq/libs/common/database_token_builder.cpp index 14991504bb..89d36cfba6 100644 --- a/ydb/core/yq/libs/common/database_token_builder.cpp +++ b/ydb/core/yq/libs/common/database_token_builder.cpp @@ -1,45 +1,45 @@ -#include "database_token_builder.h" +#include "database_token_builder.h" #include <ydb/library/yql/providers/common/structured_token/yql_token_builder.h> - -namespace NYq { - -using namespace NYql; - - TString BuildStructuredToken(const YandexQuery::IamAuth& auth, const TString& authToken, const THashMap<TString, TString>& accountIdSignatures) { - TStructuredTokenBuilder result; - switch (auth.identity_case()) { - case YandexQuery::IamAuth::kCurrentIam: - result.SetIAMToken(authToken); - break; - case YandexQuery::IamAuth::kServiceAccount: { - const auto& signature = accountIdSignatures.at(auth.service_account().id()); - result.SetServiceAccountIdAuth(auth.service_account().id(), signature); - break; - } - default: - result.SetNoAuth(); - break; - } - - return result.ToJson(); - } - - void TryAddDatabaseToResolve( - const YandexQuery::IamAuth& auth, - const TString& databaseId, - DatabaseType type, - const TString& authToken, - const THashMap<TString, TString>& accountIdSignatures, - THashMap<std::pair<TString, DatabaseType>, TEvents::TDatabaseAuth>& databaseIds) { - if (!databaseId) { - return; - } - - TEvents::TDatabaseAuth info; - info.StructuredToken = BuildStructuredToken(auth, authToken, accountIdSignatures); - info.AddBearerToToken = true; // XXX - databaseIds[std::make_pair(databaseId, type)] = info; - } - + +namespace NYq { + +using namespace NYql; + + TString BuildStructuredToken(const YandexQuery::IamAuth& auth, const TString& authToken, const THashMap<TString, TString>& accountIdSignatures) { + TStructuredTokenBuilder result; + switch (auth.identity_case()) { + case YandexQuery::IamAuth::kCurrentIam: + result.SetIAMToken(authToken); + break; + case YandexQuery::IamAuth::kServiceAccount: { + const auto& signature = accountIdSignatures.at(auth.service_account().id()); + result.SetServiceAccountIdAuth(auth.service_account().id(), signature); + break; + } + default: + result.SetNoAuth(); + break; + } + + return result.ToJson(); + } + + void TryAddDatabaseToResolve( + const YandexQuery::IamAuth& auth, + const TString& databaseId, + DatabaseType type, + const TString& authToken, + const THashMap<TString, TString>& accountIdSignatures, + THashMap<std::pair<TString, DatabaseType>, TEvents::TDatabaseAuth>& databaseIds) { + if (!databaseId) { + return; + } + + TEvents::TDatabaseAuth info; + info.StructuredToken = BuildStructuredToken(auth, authToken, accountIdSignatures); + info.AddBearerToToken = true; // XXX + databaseIds[std::make_pair(databaseId, type)] = info; + } + } // NYq diff --git a/ydb/core/yq/libs/common/database_token_builder.h b/ydb/core/yq/libs/common/database_token_builder.h index c903b756ea..a3e00efab5 100644 --- a/ydb/core/yq/libs/common/database_token_builder.h +++ b/ydb/core/yq/libs/common/database_token_builder.h @@ -2,23 +2,23 @@ #include <ydb/core/yq/libs/control_plane_storage/events/events.h> #include <ydb/core/yq/libs/events/events.h> - -namespace NYq { - -using namespace NActors; -using namespace NYql; - - TString BuildStructuredToken( - const YandexQuery::IamAuth& auth, - const TString& authToken, - const THashMap<TString, TString>& accountIdSignatures); - - void TryAddDatabaseToResolve( - const YandexQuery::IamAuth& auth, - const TString& databaseId, - DatabaseType type, - const TString& authToken, - const THashMap<TString, TString>& accountIdSignatures, - THashMap<std::pair<TString, DatabaseType>, TEvents::TDatabaseAuth>& databaseIds); - + +namespace NYq { + +using namespace NActors; +using namespace NYql; + + TString BuildStructuredToken( + const YandexQuery::IamAuth& auth, + const TString& authToken, + const THashMap<TString, TString>& accountIdSignatures); + + void TryAddDatabaseToResolve( + const YandexQuery::IamAuth& auth, + const TString& databaseId, + DatabaseType type, + const TString& authToken, + const THashMap<TString, TString>& accountIdSignatures, + THashMap<std::pair<TString, DatabaseType>, TEvents::TDatabaseAuth>& databaseIds); + } // NYq diff --git a/ydb/core/yq/libs/common/entity_id.cpp b/ydb/core/yq/libs/common/entity_id.cpp index ea3ea069cb..dab1d1abca 100644 --- a/ydb/core/yq/libs/common/entity_id.cpp +++ b/ydb/core/yq/libs/common/entity_id.cpp @@ -7,7 +7,7 @@ #include <util/stream/format.h> #include <util/string/ascii.h> -namespace NYq { +namespace NYq { // used ascii order: IntToChar[i] < IntToChar[i+1] constexpr char IntToChar[] = { @@ -84,4 +84,4 @@ IEntityIdGenerator::TPtr CreateEntityIdGenerator(const TString& prefix) { return MakeIntrusive<TEntityIdGenerator>(prefix); } -} // namespace NYq +} // namespace NYq diff --git a/ydb/core/yq/libs/common/entity_id.h b/ydb/core/yq/libs/common/entity_id.h index 5175a04c7b..d34dee0f55 100644 --- a/ydb/core/yq/libs/common/entity_id.h +++ b/ydb/core/yq/libs/common/entity_id.h @@ -3,7 +3,7 @@ #include <util/generic/string.h> #include <util/datetime/base.h> -namespace NYq { +namespace NYq { enum class EEntityType : char { UNDEFINED = 'u', @@ -26,4 +26,4 @@ struct IEntityIdGenerator : public TThrRefBase { IEntityIdGenerator::TPtr CreateEntityIdGenerator(const TString& prefix); -} // namespace NYq +} // namespace NYq diff --git a/ydb/core/yq/libs/common/entity_id_ut.cpp b/ydb/core/yq/libs/common/entity_id_ut.cpp index 1d785c1865..8533bdcca0 100644 --- a/ydb/core/yq/libs/common/entity_id_ut.cpp +++ b/ydb/core/yq/libs/common/entity_id_ut.cpp @@ -3,7 +3,7 @@ #include <ydb/services/ydb/ydb_common_ut.h> #include <limits> -using namespace NYq; +using namespace NYq; Y_UNIT_TEST_SUITE(EntityId) { Y_UNIT_TEST(Distinct) { diff --git a/ydb/core/yq/libs/common/rows_proto_splitter.cpp b/ydb/core/yq/libs/common/rows_proto_splitter.cpp index 38fbd99812..00132486dd 100644 --- a/ydb/core/yq/libs/common/rows_proto_splitter.cpp +++ b/ydb/core/yq/libs/common/rows_proto_splitter.cpp @@ -1,77 +1,77 @@ -#include "rows_proto_splitter.h" - -#include <util/string/builder.h> - -namespace NYq { - -TRowsProtoSplitter::TRowsProtoSplitter( - const Ydb::ResultSet& resultSet, - const ui64 chunkLimit, - const ui64 headerProtoByteSize, - const ui64 maxRowsCountPerChunk) - : ResultSet(resultSet) - , ChunkLimit(chunkLimit) - , MaxRowsCountPerChunk(maxRowsCountPerChunk) - { - ui64 colsBytes = 0; - for (const auto& column : resultSet.columns()) { - colsBytes += column.ByteSizeLong(); - } - BaseProtoBytesSize = colsBytes + headerProtoByteSize; - } - -TSplittedResultSets TRowsProtoSplitter::MakeResultWithIssues(const TString& msg) { - Issues.AddIssue(msg); - Success = false; - return TSplittedResultSets{{}, std::move(Issues), Success}; -} - -TString TRowsProtoSplitter::CheckLimits(ui64 curRowBytes, size_t rowInd) { - TString issueMsg; - - if (curRowBytes + BaseProtoBytesSize > ChunkLimit) { - issueMsg += TStringBuilder() << "Can not write Row["<< rowInd << "] with size: " - << curRowBytes + BaseProtoBytesSize << " bytes (> " << ChunkLimit / (1024 * 1024) << "_MB)\n"; - } - - return issueMsg; -} - -void TRowsProtoSplitter::MakeNewChunk(Ydb::ResultSet& resultSet, ui64& chunkRowsCounter, ui64& chunkSize) { - resultSet.Clear(); - resultSet.mutable_columns()->CopyFrom(ResultSet.columns()); - - chunkRowsCounter = 0; - - chunkSize = BaseProtoBytesSize; -} - -TSplittedResultSets TRowsProtoSplitter::Split() { - size_t rowInd = 0; - size_t curChunkRowsCounter = 0; - - Ydb::ResultSet curResultSet; - ui64 curChunkSize = 0; - MakeNewChunk(curResultSet, curChunkRowsCounter, curChunkSize); - - for (const auto& row : ResultSet.rows()) { - if (const auto issueMsg = CheckLimits(row.ByteSizeLong(), rowInd++)) { - return MakeResultWithIssues(issueMsg); - } - - if (row.ByteSizeLong() + curChunkSize <= ChunkLimit && ++curChunkRowsCounter <= MaxRowsCountPerChunk) { - curChunkSize += row.ByteSizeLong(); - *curResultSet.add_rows() = row; - continue; - } - - SplittedResultSets.emplace_back(curResultSet); - MakeNewChunk(curResultSet, curChunkRowsCounter, curChunkSize); - *curResultSet.add_rows() = row; - - } - SplittedResultSets.emplace_back(curResultSet); //last rs - return TSplittedResultSets{std::move(SplittedResultSets), std::move(Issues), Success}; -} - -} //NYq +#include "rows_proto_splitter.h" + +#include <util/string/builder.h> + +namespace NYq { + +TRowsProtoSplitter::TRowsProtoSplitter( + const Ydb::ResultSet& resultSet, + const ui64 chunkLimit, + const ui64 headerProtoByteSize, + const ui64 maxRowsCountPerChunk) + : ResultSet(resultSet) + , ChunkLimit(chunkLimit) + , MaxRowsCountPerChunk(maxRowsCountPerChunk) + { + ui64 colsBytes = 0; + for (const auto& column : resultSet.columns()) { + colsBytes += column.ByteSizeLong(); + } + BaseProtoBytesSize = colsBytes + headerProtoByteSize; + } + +TSplittedResultSets TRowsProtoSplitter::MakeResultWithIssues(const TString& msg) { + Issues.AddIssue(msg); + Success = false; + return TSplittedResultSets{{}, std::move(Issues), Success}; +} + +TString TRowsProtoSplitter::CheckLimits(ui64 curRowBytes, size_t rowInd) { + TString issueMsg; + + if (curRowBytes + BaseProtoBytesSize > ChunkLimit) { + issueMsg += TStringBuilder() << "Can not write Row["<< rowInd << "] with size: " + << curRowBytes + BaseProtoBytesSize << " bytes (> " << ChunkLimit / (1024 * 1024) << "_MB)\n"; + } + + return issueMsg; +} + +void TRowsProtoSplitter::MakeNewChunk(Ydb::ResultSet& resultSet, ui64& chunkRowsCounter, ui64& chunkSize) { + resultSet.Clear(); + resultSet.mutable_columns()->CopyFrom(ResultSet.columns()); + + chunkRowsCounter = 0; + + chunkSize = BaseProtoBytesSize; +} + +TSplittedResultSets TRowsProtoSplitter::Split() { + size_t rowInd = 0; + size_t curChunkRowsCounter = 0; + + Ydb::ResultSet curResultSet; + ui64 curChunkSize = 0; + MakeNewChunk(curResultSet, curChunkRowsCounter, curChunkSize); + + for (const auto& row : ResultSet.rows()) { + if (const auto issueMsg = CheckLimits(row.ByteSizeLong(), rowInd++)) { + return MakeResultWithIssues(issueMsg); + } + + if (row.ByteSizeLong() + curChunkSize <= ChunkLimit && ++curChunkRowsCounter <= MaxRowsCountPerChunk) { + curChunkSize += row.ByteSizeLong(); + *curResultSet.add_rows() = row; + continue; + } + + SplittedResultSets.emplace_back(curResultSet); + MakeNewChunk(curResultSet, curChunkRowsCounter, curChunkSize); + *curResultSet.add_rows() = row; + + } + SplittedResultSets.emplace_back(curResultSet); //last rs + return TSplittedResultSets{std::move(SplittedResultSets), std::move(Issues), Success}; +} + +} //NYq diff --git a/ydb/core/yq/libs/common/rows_proto_splitter.h b/ydb/core/yq/libs/common/rows_proto_splitter.h index a666e8e894..b9ef0a57a7 100644 --- a/ydb/core/yq/libs/common/rows_proto_splitter.h +++ b/ydb/core/yq/libs/common/rows_proto_splitter.h @@ -1,42 +1,42 @@ -#pragma once +#pragma once #include <ydb/public/api/protos/ydb_value.pb.h> #include <ydb/library/yql/public/issue/yql_issue_message.h> - -namespace NYq { - -struct TSplittedResultSets { - TVector<Ydb::ResultSet> ResultSets; - NYql::TIssues Issues; - bool Success = true; -}; - -class TRowsProtoSplitter { -public: - TRowsProtoSplitter( - const Ydb::ResultSet& resultSet, - const ui64 chunkLimit, - const ui64 headerProtoByteSize = 0, + +namespace NYq { + +struct TSplittedResultSets { + TVector<Ydb::ResultSet> ResultSets; + NYql::TIssues Issues; + bool Success = true; +}; + +class TRowsProtoSplitter { +public: + TRowsProtoSplitter( + const Ydb::ResultSet& resultSet, + const ui64 chunkLimit, + const ui64 headerProtoByteSize = 0, const ui64 maxRowsCountPerChunk = 100'000 ); - void MakeNewChunk(Ydb::ResultSet& resultSet, ui64& chunkRowsCounter, ui64& chunkSize); - - TString CheckLimits(ui64 curRowBytes, size_t rowInd); - - TSplittedResultSets MakeResultWithIssues(const TString& msg); - - TSplittedResultSets Split(); - - -private: - Ydb::ResultSet ResultSet; - const ui64 ChunkLimit = 0; - ui64 BaseProtoBytesSize = 0; - const ui64 MaxRowsCountPerChunk = 0; - - TVector<Ydb::ResultSet> SplittedResultSets; - NYql::TIssues Issues; - bool Success = true; -}; - -} //NYq + void MakeNewChunk(Ydb::ResultSet& resultSet, ui64& chunkRowsCounter, ui64& chunkSize); + + TString CheckLimits(ui64 curRowBytes, size_t rowInd); + + TSplittedResultSets MakeResultWithIssues(const TString& msg); + + TSplittedResultSets Split(); + + +private: + Ydb::ResultSet ResultSet; + const ui64 ChunkLimit = 0; + ui64 BaseProtoBytesSize = 0; + const ui64 MaxRowsCountPerChunk = 0; + + TVector<Ydb::ResultSet> SplittedResultSets; + NYql::TIssues Issues; + bool Success = true; +}; + +} //NYq diff --git a/ydb/core/yq/libs/common/rows_proto_splitter_ut.cpp b/ydb/core/yq/libs/common/rows_proto_splitter_ut.cpp index bd8befadc2..ee7dd0dd4d 100644 --- a/ydb/core/yq/libs/common/rows_proto_splitter_ut.cpp +++ b/ydb/core/yq/libs/common/rows_proto_splitter_ut.cpp @@ -1,115 +1,115 @@ -#include "rows_proto_splitter.h" - +#include "rows_proto_splitter.h" + #include <ydb/public/sdk/cpp/client/ydb_result/result.h> #include <ydb/services/ydb/ydb_common_ut.h> - -namespace NYq { - -namespace { - -Ydb::ResultSet MakeProtoResultSet() { - const TString resultSetString = - "columns {\n" - " name: \"colName\"\n" - " type {\n" - " list_type {\n" - " item {\n" - " type_id: INT32\n" - " }\n" - " }\n" - " }\n" - "}\n" - "rows {\n" - " items {\n" - " items {\n" - " int32_value: 42\n" - " }\n" - " items {\n" - " int32_value: 43\n" - " }\n" - " items {\n" - " int32_value: 44\n" - " }\n" - " }\n" - "}\n" - "rows {\n" - " items {\n" - " items {\n" - " int32_value: 45\n" - " }\n" - " items {\n" - " int32_value: 46\n" - " }\n" - " items {\n" - " int32_value: 47\n" - " }\n" - " }\n" - "}\n"; - Ydb::ResultSet rsProto; - google::protobuf::TextFormat::ParseFromString(resultSetString, &rsProto); - return rsProto; -} - -} //namespace - -Y_UNIT_TEST_SUITE(SplitterBasic) { - Y_UNIT_TEST(EqualSplitByMaxBytesLimitPerChunk) { - Ydb::ResultSet resultSet = MakeProtoResultSet(); - TRowsProtoSplitter splitter(resultSet, 41, 0); - const auto& result = splitter.Split(); - UNIT_ASSERT_C(result.Success, result.Issues.ToString()); - UNIT_ASSERT_VALUES_EQUAL(result.ResultSets.size(), 2); - - int expectedVal = 42; - for (const auto& rs : result.ResultSets) { - NYdb::TResultSetParser rsParser(rs); - UNIT_ASSERT_EQUAL(rsParser.ColumnsCount(), 1); - UNIT_ASSERT_EQUAL(rsParser.RowsCount(), 1); - UNIT_ASSERT_EQUAL(rsParser.ColumnIndex("colName"), 0); - UNIT_ASSERT_EQUAL(rsParser.ColumnIndex("otherName"), -1); - auto& column0 = rsParser.ColumnParser(0); - while (rsParser.TryNextRow()) { - column0.OpenList(); - while (column0.TryNextListItem()) { - UNIT_ASSERT_EQUAL(column0.GetInt32(), expectedVal++); - } - } - } - } - - Y_UNIT_TEST(EqualSplitByMaxRowsLimitPerChunk) { - Ydb::ResultSet resultSet = MakeProtoResultSet(); - TRowsProtoSplitter splitter(resultSet, 1024 * 1024, 0, 1); - const auto& result = splitter.Split(); - UNIT_ASSERT_C(result.Success, result.Issues.ToString()); - UNIT_ASSERT_VALUES_EQUAL(result.ResultSets.size(), 2); - - int expectedVal = 42; - for (const auto& rs : result.ResultSets) { - NYdb::TResultSetParser rsParser(rs); - UNIT_ASSERT_EQUAL(rsParser.ColumnsCount(), 1); - UNIT_ASSERT_EQUAL_C(rsParser.RowsCount(), 1, rsParser.RowsCount()); - UNIT_ASSERT_EQUAL(rsParser.ColumnIndex("colName"), 0); - UNIT_ASSERT_EQUAL(rsParser.ColumnIndex("otherName"), -1); - auto& column0 = rsParser.ColumnParser(0); - while (rsParser.TryNextRow()) { - column0.OpenList(); - while (column0.TryNextListItem()) { - UNIT_ASSERT_EQUAL(column0.GetInt32(), expectedVal++); - } - } - } - } - - Y_UNIT_TEST(LimitExceed) { - Ydb::ResultSet resultSet = MakeProtoResultSet(); - TRowsProtoSplitter splitter(resultSet, 1, 0); - const auto& result = splitter.Split(); - const auto& issuesStr = result.Issues.ToString(); - UNIT_ASSERT_C(!result.Success, issuesStr); - UNIT_ASSERT_VALUES_EQUAL(result.ResultSets.size(), 0); - UNIT_ASSERT_C(issuesStr.Contains("Can not write Row[0] with size: 41 bytes"), issuesStr); - } -} - -} // NYq + +namespace NYq { + +namespace { + +Ydb::ResultSet MakeProtoResultSet() { + const TString resultSetString = + "columns {\n" + " name: \"colName\"\n" + " type {\n" + " list_type {\n" + " item {\n" + " type_id: INT32\n" + " }\n" + " }\n" + " }\n" + "}\n" + "rows {\n" + " items {\n" + " items {\n" + " int32_value: 42\n" + " }\n" + " items {\n" + " int32_value: 43\n" + " }\n" + " items {\n" + " int32_value: 44\n" + " }\n" + " }\n" + "}\n" + "rows {\n" + " items {\n" + " items {\n" + " int32_value: 45\n" + " }\n" + " items {\n" + " int32_value: 46\n" + " }\n" + " items {\n" + " int32_value: 47\n" + " }\n" + " }\n" + "}\n"; + Ydb::ResultSet rsProto; + google::protobuf::TextFormat::ParseFromString(resultSetString, &rsProto); + return rsProto; +} + +} //namespace + +Y_UNIT_TEST_SUITE(SplitterBasic) { + Y_UNIT_TEST(EqualSplitByMaxBytesLimitPerChunk) { + Ydb::ResultSet resultSet = MakeProtoResultSet(); + TRowsProtoSplitter splitter(resultSet, 41, 0); + const auto& result = splitter.Split(); + UNIT_ASSERT_C(result.Success, result.Issues.ToString()); + UNIT_ASSERT_VALUES_EQUAL(result.ResultSets.size(), 2); + + int expectedVal = 42; + for (const auto& rs : result.ResultSets) { + NYdb::TResultSetParser rsParser(rs); + UNIT_ASSERT_EQUAL(rsParser.ColumnsCount(), 1); + UNIT_ASSERT_EQUAL(rsParser.RowsCount(), 1); + UNIT_ASSERT_EQUAL(rsParser.ColumnIndex("colName"), 0); + UNIT_ASSERT_EQUAL(rsParser.ColumnIndex("otherName"), -1); + auto& column0 = rsParser.ColumnParser(0); + while (rsParser.TryNextRow()) { + column0.OpenList(); + while (column0.TryNextListItem()) { + UNIT_ASSERT_EQUAL(column0.GetInt32(), expectedVal++); + } + } + } + } + + Y_UNIT_TEST(EqualSplitByMaxRowsLimitPerChunk) { + Ydb::ResultSet resultSet = MakeProtoResultSet(); + TRowsProtoSplitter splitter(resultSet, 1024 * 1024, 0, 1); + const auto& result = splitter.Split(); + UNIT_ASSERT_C(result.Success, result.Issues.ToString()); + UNIT_ASSERT_VALUES_EQUAL(result.ResultSets.size(), 2); + + int expectedVal = 42; + for (const auto& rs : result.ResultSets) { + NYdb::TResultSetParser rsParser(rs); + UNIT_ASSERT_EQUAL(rsParser.ColumnsCount(), 1); + UNIT_ASSERT_EQUAL_C(rsParser.RowsCount(), 1, rsParser.RowsCount()); + UNIT_ASSERT_EQUAL(rsParser.ColumnIndex("colName"), 0); + UNIT_ASSERT_EQUAL(rsParser.ColumnIndex("otherName"), -1); + auto& column0 = rsParser.ColumnParser(0); + while (rsParser.TryNextRow()) { + column0.OpenList(); + while (column0.TryNextListItem()) { + UNIT_ASSERT_EQUAL(column0.GetInt32(), expectedVal++); + } + } + } + } + + Y_UNIT_TEST(LimitExceed) { + Ydb::ResultSet resultSet = MakeProtoResultSet(); + TRowsProtoSplitter splitter(resultSet, 1, 0); + const auto& result = splitter.Split(); + const auto& issuesStr = result.Issues.ToString(); + UNIT_ASSERT_C(!result.Success, issuesStr); + UNIT_ASSERT_VALUES_EQUAL(result.ResultSets.size(), 0); + UNIT_ASSERT_C(issuesStr.Contains("Can not write Row[0] with size: 41 bytes"), issuesStr); + } +} + +} // NYq diff --git a/ydb/core/yq/libs/common/ut/ya.make b/ydb/core/yq/libs/common/ut/ya.make index 21401fe7d1..320ef17e66 100644 --- a/ydb/core/yq/libs/common/ut/ya.make +++ b/ydb/core/yq/libs/common/ut/ya.make @@ -14,7 +14,7 @@ ENDIF() SRCS( cache_ut.cpp entity_id_ut.cpp - rows_proto_splitter_ut.cpp + rows_proto_splitter_ut.cpp ) PEERDIR( diff --git a/ydb/core/yq/libs/common/ya.make b/ydb/core/yq/libs/common/ya.make index ab214224a1..4622695700 100644 --- a/ydb/core/yq/libs/common/ya.make +++ b/ydb/core/yq/libs/common/ya.make @@ -1,24 +1,24 @@ OWNER(g:yq) - -LIBRARY() - -SRCS( - database_token_builder.cpp + +LIBRARY() + +SRCS( + database_token_builder.cpp entity_id.cpp - rows_proto_splitter.cpp -) - -PEERDIR( + rows_proto_splitter.cpp +) + +PEERDIR( library/cpp/monlib/dynamic_counters ydb/core/yq/libs/control_plane_storage/events ydb/core/yq/libs/events ydb/library/yql/providers/common/structured_token ydb/library/yql/public/issue ydb/public/api/protos -) - -YQL_LAST_ABI_VERSION() - +) + +YQL_LAST_ABI_VERSION() + END() RECURSE_FOR_TESTS( diff --git a/ydb/core/yq/libs/config/protos/common.proto b/ydb/core/yq/libs/config/protos/common.proto index 2b0b850e3e..e86548cf3c 100644 --- a/ydb/core/yq/libs/config/protos/common.proto +++ b/ydb/core/yq/libs/config/protos/common.proto @@ -20,5 +20,5 @@ message TCommonConfig { bool MdbTransformHost = 5; string ObjectStorageEndpoint = 6; string IdsPrefix = 7; - uint64 MaxTasksPerOperation = 8; + uint64 MaxTasksPerOperation = 8; } diff --git a/ydb/core/yq/libs/config/protos/gateways.proto b/ydb/core/yq/libs/config/protos/gateways.proto index d416454619..b953143764 100644 --- a/ydb/core/yq/libs/config/protos/gateways.proto +++ b/ydb/core/yq/libs/config/protos/gateways.proto @@ -15,5 +15,5 @@ message TGatewaysConfig { NYql.TYdbGatewayConfig Ydb = 4; NYql.TS3GatewayConfig S3 = 5; NYql.TSolomonGatewayConfig Solomon = 6; - NYql.THttpGatewayConfig HttpGateway = 7; + NYql.THttpGatewayConfig HttpGateway = 7; } diff --git a/ydb/core/yq/libs/config/protos/read_actors_factory.proto b/ydb/core/yq/libs/config/protos/read_actors_factory.proto index c30111c13c..6addaab49f 100644 --- a/ydb/core/yq/libs/config/protos/read_actors_factory.proto +++ b/ydb/core/yq/libs/config/protos/read_actors_factory.proto @@ -1,17 +1,17 @@ -syntax = "proto3"; -option cc_enable_arenas = true; - -package NYq.NConfig; -option java_package = "ru.yandex.kikimr.proto"; - -import "ydb/library/yql/providers/s3/proto/retry_config.proto"; - -//////////////////////////////////////////////////////////// - -message TS3ReadActorFactoryConfig { - NYql.NS3.TRetryConfig RetryConfig = 1; -} - -message TReadActorsFactoryConfig { - TS3ReadActorFactoryConfig S3ReadActorFactoryConfig = 1; -} +syntax = "proto3"; +option cc_enable_arenas = true; + +package NYq.NConfig; +option java_package = "ru.yandex.kikimr.proto"; + +import "ydb/library/yql/providers/s3/proto/retry_config.proto"; + +//////////////////////////////////////////////////////////// + +message TS3ReadActorFactoryConfig { + NYql.NS3.TRetryConfig RetryConfig = 1; +} + +message TReadActorsFactoryConfig { + TS3ReadActorFactoryConfig S3ReadActorFactoryConfig = 1; +} diff --git a/ydb/core/yq/libs/config/protos/ya.make b/ydb/core/yq/libs/config/protos/ya.make index 4ee4b0cd1a..72000d86fa 100644 --- a/ydb/core/yq/libs/config/protos/ya.make +++ b/ydb/core/yq/libs/config/protos/ya.make @@ -16,7 +16,7 @@ SRCS( pinger.proto private_api.proto private_proxy.proto - read_actors_factory.proto + read_actors_factory.proto resource_manager.proto storage.proto test_connection.proto @@ -27,7 +27,7 @@ SRCS( PEERDIR( ydb/library/folder_service/proto ydb/library/yql/providers/common/proto - ydb/library/yql/providers/s3/proto + ydb/library/yql/providers/s3/proto ) EXCLUDE_TAGS(GO_PROTO) diff --git a/ydb/core/yq/libs/config/protos/yq_config.proto b/ydb/core/yq/libs/config/protos/yq_config.proto index 8bee641c10..841b816821 100644 --- a/ydb/core/yq/libs/config/protos/yq_config.proto +++ b/ydb/core/yq/libs/config/protos/yq_config.proto @@ -16,7 +16,7 @@ import "ydb/core/yq/libs/config/protos/pending_fetcher.proto"; import "ydb/core/yq/libs/config/protos/pinger.proto"; import "ydb/core/yq/libs/config/protos/private_api.proto"; import "ydb/core/yq/libs/config/protos/private_proxy.proto"; -import "ydb/core/yq/libs/config/protos/read_actors_factory.proto"; +import "ydb/core/yq/libs/config/protos/read_actors_factory.proto"; import "ydb/core/yq/libs/config/protos/resource_manager.proto"; import "ydb/core/yq/libs/config/protos/test_connection.proto"; import "ydb/core/yq/libs/config/protos/token_accessor.proto"; @@ -41,7 +41,7 @@ message TConfig { TNodesManagerConfig NodesManager = 14; TPendingFetcherConfig PendingFetcher = 15; TPingerConfig Pinger = 16; - bool EnableDynamicNameservice = 17; + bool EnableDynamicNameservice = 17; TTestConnectionConfig TestConnection = 18; - TReadActorsFactoryConfig ReadActorsFactoryConfig = 19; + TReadActorsFactoryConfig ReadActorsFactoryConfig = 19; } diff --git a/ydb/core/yq/libs/control_plane_proxy/control_plane_proxy.cpp b/ydb/core/yq/libs/control_plane_proxy/control_plane_proxy.cpp index 33d031643b..53838351b4 100644 --- a/ydb/core/yq/libs/control_plane_proxy/control_plane_proxy.cpp +++ b/ydb/core/yq/libs/control_plane_proxy/control_plane_proxy.cpp @@ -104,7 +104,7 @@ class TRequestActor : public NActors::TActorBootstrapped<TRequestActor<TRequestP public: static constexpr char ActorName[] = "YQ_CONTROL_PLANE_PROXY_REQUEST_ACTOR"; - + explicit TRequestActor(const NConfig::TControlPlaneProxyConfig& config, TActorId sender, ui32 cookie, const TString& scope, const TString& folderId, TRequestProto&& requestProto, @@ -331,7 +331,7 @@ public: { } - static constexpr char ActorName[] = "YQ_CONTROL_PLANE_PROXY"; + static constexpr char ActorName[] = "YQ_CONTROL_PLANE_PROXY"; void Bootstrap() { CPP_LOG_D("Starting yandex query control plane proxy. Actor id: " << SelfId()); diff --git a/ydb/core/yq/libs/control_plane_storage/control_plane_storage_counters.cpp b/ydb/core/yq/libs/control_plane_storage/control_plane_storage_counters.cpp index 17ed9a8aa7..8274a88589 100644 --- a/ydb/core/yq/libs/control_plane_storage/control_plane_storage_counters.cpp +++ b/ydb/core/yq/libs/control_plane_storage/control_plane_storage_counters.cpp @@ -1,11 +1,11 @@ -#include "control_plane_storage_counters.h" - -namespace NYq { - -TRequestCounters::TRequestCounters(const TString& name) - : Name(name) {} - -void TRequestCounters::Register(const NMonitoring::TDynamicCounterPtr& counters) { +#include "control_plane_storage_counters.h" + +namespace NYq { + +TRequestCounters::TRequestCounters(const TString& name) + : Name(name) {} + +void TRequestCounters::Register(const NMonitoring::TDynamicCounterPtr& counters) { auto requestCounters = counters->GetSubgroup("request", Name); InFly = requestCounters->GetCounter("InFly", false); Ok = requestCounters->GetCounter("Ok", true); @@ -13,52 +13,52 @@ void TRequestCounters::Register(const NMonitoring::TDynamicCounterPtr& counters) Retry = requestCounters->GetCounter("Retry", true); LatencyMs = requestCounters->GetHistogram("LatencyMs", GetLatencyHistogramBuckets()); Issues = requestCounters->GetSubgroup("subcomponent", "Issues"); -} - -NMonitoring::IHistogramCollectorPtr TRequestCounters::GetLatencyHistogramBuckets() { - return NMonitoring::ExplicitHistogram({0, 1, 2, 5, 10, 20, 50, 100, 500, 1000, 2000, 5000, 10000, 30000, 50000, 500000}); -} - - -TFinalStatusCounters::TFinalStatusCounters(const NMonitoring::TDynamicCounterPtr& counters) { - auto subgroup = counters->GetSubgroup("subcomponent", "FinalStatus"); - Completed = subgroup->GetCounter("COMPLETED", true); - AbortedBySystem = subgroup->GetCounter("ABORTED_BY_SYSTEM", true); - AbortedByUser = subgroup->GetCounter("ABORTED_BY_USER", true); - Failed = subgroup->GetCounter("FAILED", true); - Paused = subgroup->GetCounter("PAUSED", true); -} - -void TFinalStatusCounters::IncByStatus(YandexQuery::QueryMeta::ComputeStatus finalStatus) { - switch (finalStatus) { - case YandexQuery::QueryMeta::COMPLETED: - Completed->Inc(); - break; - case YandexQuery::QueryMeta::FAILED: - Failed->Inc(); - break; - case YandexQuery::QueryMeta::ABORTED_BY_SYSTEM: - AbortedBySystem->Inc(); - break; - case YandexQuery::QueryMeta::ABORTED_BY_USER: - AbortedByUser->Inc(); - break; - case YandexQuery::QueryMeta::PAUSED: - Paused->Inc(); - break; - case YandexQuery::QueryMeta::COMPUTE_STATUS_UNSPECIFIED: - case YandexQuery::QueryMeta::STARTING: - case YandexQuery::QueryMeta::ABORTING_BY_USER: - case YandexQuery::QueryMeta::ABORTING_BY_SYSTEM: - case YandexQuery::QueryMeta::RESUMING: - case YandexQuery::QueryMeta::RUNNING: - case YandexQuery::QueryMeta::COMPLETING: - case YandexQuery::QueryMeta::FAILING: - case YandexQuery::QueryMeta::PAUSING: - break; - default: - Y_ENSURE(true, "Unexpected status: " << YandexQuery::QueryMeta_ComputeStatus_Name(finalStatus)); - } -} - -} // NYq +} + +NMonitoring::IHistogramCollectorPtr TRequestCounters::GetLatencyHistogramBuckets() { + return NMonitoring::ExplicitHistogram({0, 1, 2, 5, 10, 20, 50, 100, 500, 1000, 2000, 5000, 10000, 30000, 50000, 500000}); +} + + +TFinalStatusCounters::TFinalStatusCounters(const NMonitoring::TDynamicCounterPtr& counters) { + auto subgroup = counters->GetSubgroup("subcomponent", "FinalStatus"); + Completed = subgroup->GetCounter("COMPLETED", true); + AbortedBySystem = subgroup->GetCounter("ABORTED_BY_SYSTEM", true); + AbortedByUser = subgroup->GetCounter("ABORTED_BY_USER", true); + Failed = subgroup->GetCounter("FAILED", true); + Paused = subgroup->GetCounter("PAUSED", true); +} + +void TFinalStatusCounters::IncByStatus(YandexQuery::QueryMeta::ComputeStatus finalStatus) { + switch (finalStatus) { + case YandexQuery::QueryMeta::COMPLETED: + Completed->Inc(); + break; + case YandexQuery::QueryMeta::FAILED: + Failed->Inc(); + break; + case YandexQuery::QueryMeta::ABORTED_BY_SYSTEM: + AbortedBySystem->Inc(); + break; + case YandexQuery::QueryMeta::ABORTED_BY_USER: + AbortedByUser->Inc(); + break; + case YandexQuery::QueryMeta::PAUSED: + Paused->Inc(); + break; + case YandexQuery::QueryMeta::COMPUTE_STATUS_UNSPECIFIED: + case YandexQuery::QueryMeta::STARTING: + case YandexQuery::QueryMeta::ABORTING_BY_USER: + case YandexQuery::QueryMeta::ABORTING_BY_SYSTEM: + case YandexQuery::QueryMeta::RESUMING: + case YandexQuery::QueryMeta::RUNNING: + case YandexQuery::QueryMeta::COMPLETING: + case YandexQuery::QueryMeta::FAILING: + case YandexQuery::QueryMeta::PAUSING: + break; + default: + Y_ENSURE(true, "Unexpected status: " << YandexQuery::QueryMeta_ComputeStatus_Name(finalStatus)); + } +} + +} // NYq diff --git a/ydb/core/yq/libs/control_plane_storage/control_plane_storage_counters.h b/ydb/core/yq/libs/control_plane_storage/control_plane_storage_counters.h index bec7d629bf..58d3c56b27 100644 --- a/ydb/core/yq/libs/control_plane_storage/control_plane_storage_counters.h +++ b/ydb/core/yq/libs/control_plane_storage/control_plane_storage_counters.h @@ -1,40 +1,40 @@ -#pragma once - -#include <library/cpp/monlib/dynamic_counters/counters.h> -#include <ydb/public/api/protos/yq.pb.h> - -namespace NYq { - -class TRequestCounters: public virtual TThrRefBase { -public: - const TString Name; - - NMonitoring::TDynamicCounters::TCounterPtr InFly; - NMonitoring::TDynamicCounters::TCounterPtr Ok; - NMonitoring::TDynamicCounters::TCounterPtr Error; - NMonitoring::TDynamicCounters::TCounterPtr Retry; - NMonitoring::THistogramPtr LatencyMs; +#pragma once + +#include <library/cpp/monlib/dynamic_counters/counters.h> +#include <ydb/public/api/protos/yq.pb.h> + +namespace NYq { + +class TRequestCounters: public virtual TThrRefBase { +public: + const TString Name; + + NMonitoring::TDynamicCounters::TCounterPtr InFly; + NMonitoring::TDynamicCounters::TCounterPtr Ok; + NMonitoring::TDynamicCounters::TCounterPtr Error; + NMonitoring::TDynamicCounters::TCounterPtr Retry; + NMonitoring::THistogramPtr LatencyMs; NMonitoring::TDynamicCounterPtr Issues; - - explicit TRequestCounters(const TString& name); - - void Register(const NMonitoring::TDynamicCounterPtr& counters); - -private: - static NMonitoring::IHistogramCollectorPtr GetLatencyHistogramBuckets(); -}; - -class TFinalStatusCounters: public virtual TThrRefBase { - NMonitoring::TDynamicCounters::TCounterPtr Completed; - NMonitoring::TDynamicCounters::TCounterPtr AbortedBySystem; - NMonitoring::TDynamicCounters::TCounterPtr AbortedByUser; - NMonitoring::TDynamicCounters::TCounterPtr Failed; - NMonitoring::TDynamicCounters::TCounterPtr Paused; - -public: - TFinalStatusCounters(const NMonitoring::TDynamicCounterPtr& counters); - - void IncByStatus(YandexQuery::QueryMeta::ComputeStatus finalStatus); -}; - -} // NYq + + explicit TRequestCounters(const TString& name); + + void Register(const NMonitoring::TDynamicCounterPtr& counters); + +private: + static NMonitoring::IHistogramCollectorPtr GetLatencyHistogramBuckets(); +}; + +class TFinalStatusCounters: public virtual TThrRefBase { + NMonitoring::TDynamicCounters::TCounterPtr Completed; + NMonitoring::TDynamicCounters::TCounterPtr AbortedBySystem; + NMonitoring::TDynamicCounters::TCounterPtr AbortedByUser; + NMonitoring::TDynamicCounters::TCounterPtr Failed; + NMonitoring::TDynamicCounters::TCounterPtr Paused; + +public: + TFinalStatusCounters(const NMonitoring::TDynamicCounterPtr& counters); + + void IncByStatus(YandexQuery::QueryMeta::ComputeStatus finalStatus); +}; + +} // NYq diff --git a/ydb/core/yq/libs/control_plane_storage/events/events.h b/ydb/core/yq/libs/control_plane_storage/events/events.h index 1ef2323a62..8c98e790d7 100644 --- a/ydb/core/yq/libs/control_plane_storage/events/events.h +++ b/ydb/core/yq/libs/control_plane_storage/events/events.h @@ -27,12 +27,12 @@ struct TAuditDetails { TString CloudId; }; -struct TNodeInfo { - ui32 NodeId; - TString InstanceId; - TString HostName; -}; - +struct TNodeInfo { + ui32 NodeId; + TString InstanceId; + TString HostName; +}; + struct TDebugItem { TString Query; NYdb::TParams Params; @@ -108,7 +108,7 @@ struct TEvControlPlaneStorage { EvDeleteQueryRequest, EvDeleteQueryResponse, EvControlQueryRequest, - EvControlQueryResponse, + EvControlQueryResponse, EvGetResultDataRequest, EvGetResultDataResponse, EvListJobsRequest, @@ -141,10 +141,10 @@ struct TEvControlPlaneStorage { EvGetTaskResponse, EvPingTaskRequest, EvPingTaskResponse, - EvNodesHealthCheckRequest, - EvNodesHealthCheckResponse, - EvGetHealthNodesRequest, - EvGetHealthNodesResponse, + EvNodesHealthCheckRequest, + EvNodesHealthCheckResponse, + EvGetHealthNodesRequest, + EvGetHealthNodesResponse, EvEnd, }; @@ -931,12 +931,12 @@ struct TEvControlPlaneStorage { // internal messages struct TEvWriteResultDataRequest : NActors::TEventLocal<TEvWriteResultDataRequest, EvWriteResultDataRequest> { - explicit TEvWriteResultDataRequest(const TString& resultId, + explicit TEvWriteResultDataRequest(const TString& resultId, const int32_t resultSetId, const int64_t startRowId, const TInstant& deadline, const Ydb::ResultSet& resultSet) - : ResultId(resultId) + : ResultId(resultId) , ResultSetId(resultSetId) , StartRowId(startRowId) , Deadline(deadline) @@ -944,7 +944,7 @@ struct TEvControlPlaneStorage { { } - TString ResultId; + TString ResultId; int32_t ResultSetId = 0; int64_t StartRowId = 0; TInstant Deadline; @@ -957,23 +957,23 @@ struct TEvControlPlaneStorage { { } - explicit TEvWriteResultDataResponse( - const NYql::TIssues& issues, - const ui64 requestId) - : Issues(issues) - , RequestId(requestId) - { - } - + explicit TEvWriteResultDataResponse( + const NYql::TIssues& issues, + const ui64 requestId) + : Issues(issues) + , RequestId(requestId) + { + } + NYql::TIssues Issues; - const ui64 RequestId = 0; + const ui64 RequestId = 0; TDebugInfoPtr DebugInfo; }; struct TEvGetTaskRequest : NActors::TEventLocal<TEvGetTaskRequest, EvGetTaskRequest> { - explicit TEvGetTaskRequest( - const TString& owner, - const TString& hostName) + explicit TEvGetTaskRequest( + const TString& owner, + const TString& hostName) : Owner(owner) , HostName(hostName) { @@ -983,19 +983,19 @@ struct TEvControlPlaneStorage { TString HostName; }; - struct TTask { - TString Scope; - TString QueryId; - YandexQuery::Query Query; - YandexQuery::Internal::QueryInternal Internal; - ui64 Generation = 0; + struct TTask { + TString Scope; + TString QueryId; + YandexQuery::Query Query; + YandexQuery::Internal::QueryInternal Internal; + ui64 Generation = 0; TInstant Deadline; - }; - + }; + struct TEvGetTaskResponse : NActors::TEventLocal<TEvGetTaskResponse, EvGetTaskResponse> { - explicit TEvGetTaskResponse(const TVector<TTask>& tasks, const TString& owner) - : Tasks(tasks) - , Owner(owner) + explicit TEvGetTaskResponse(const TVector<TTask>& tasks, const TString& owner) + : Tasks(tasks) + , Owner(owner) { } @@ -1005,8 +1005,8 @@ struct TEvControlPlaneStorage { } NYql::TIssues Issues; - TVector<TTask> Tasks; - TString Owner; + TVector<TTask> Tasks; + TString Owner; TDebugInfoPtr DebugInfo; }; @@ -1026,17 +1026,17 @@ struct TEvControlPlaneStorage { explicit TEvPingTaskRequest(const TString& scope, const TString& queryId, const TString& owner, const TInstant& deadline, const TString& resultId = "") : Scope(scope) , QueryId(queryId) - , Owner(owner) + , Owner(owner) , Deadline(deadline) - , ResultId(resultId) + , ResultId(resultId) { } - const TString Scope; - const TString QueryId; - const TString Owner; + const TString Scope; + const TString QueryId; + const TString Owner; const TInstant Deadline; - TString ResultId; + TString ResultId; TMaybe<YandexQuery::QueryMeta::ComputeStatus> Status; TMaybe<NYql::TIssues> Issues; TMaybe<NYql::TIssues> TransientIssues; @@ -1069,32 +1069,32 @@ struct TEvControlPlaneStorage { NYql::TIssues Issues; TDebugInfoPtr DebugInfo; }; - - struct TEvNodesHealthCheckRequest : NActors::TEventLocal<TEvNodesHealthCheckRequest, EvNodesHealthCheckRequest> { - explicit TEvNodesHealthCheckRequest( - Yq::Private::NodesHealthCheckRequest&& request) - : Request(std::move(request)) - {} - Yq::Private::NodesHealthCheckRequest Request; - }; - - struct TEvNodesHealthCheckResponse : NActors::TEventLocal<TEvNodesHealthCheckResponse, EvNodesHealthCheckResponse> { - - explicit TEvNodesHealthCheckResponse( - const Yq::Private::NodesHealthCheckResult& record) - : Record(record) - {} - - explicit TEvNodesHealthCheckResponse( - const NYql::TIssues& issues) - : Issues(issues) - {} - - Yq::Private::NodesHealthCheckResult Record; - NYql::TIssues Issues; + + struct TEvNodesHealthCheckRequest : NActors::TEventLocal<TEvNodesHealthCheckRequest, EvNodesHealthCheckRequest> { + explicit TEvNodesHealthCheckRequest( + Yq::Private::NodesHealthCheckRequest&& request) + : Request(std::move(request)) + {} + Yq::Private::NodesHealthCheckRequest Request; + }; + + struct TEvNodesHealthCheckResponse : NActors::TEventLocal<TEvNodesHealthCheckResponse, EvNodesHealthCheckResponse> { + + explicit TEvNodesHealthCheckResponse( + const Yq::Private::NodesHealthCheckResult& record) + : Record(record) + {} + + explicit TEvNodesHealthCheckResponse( + const NYql::TIssues& issues) + : Issues(issues) + {} + + Yq::Private::NodesHealthCheckResult Record; + NYql::TIssues Issues; TDebugInfoPtr DebugInfo; - }; - + }; + }; } diff --git a/ydb/core/yq/libs/control_plane_storage/in_memory_control_plane_storage.cpp b/ydb/core/yq/libs/control_plane_storage/in_memory_control_plane_storage.cpp index 8f4d4931be..783b924238 100644 --- a/ydb/core/yq/libs/control_plane_storage/in_memory_control_plane_storage.cpp +++ b/ydb/core/yq/libs/control_plane_storage/in_memory_control_plane_storage.cpp @@ -318,7 +318,7 @@ private: { SendEmptyResponse< TEvControlPlaneStorage::TEvNodesHealthCheckRequest::TPtr, - Yq::Private::NodesHealthCheckResult, + Yq::Private::NodesHealthCheckResult, TEvControlPlaneStorage::TEvNodesHealthCheckResponse>(ev, "NodesHealthCheckRequest"); } diff --git a/ydb/core/yq/libs/control_plane_storage/internal/nodes_health_check.cpp b/ydb/core/yq/libs/control_plane_storage/internal/nodes_health_check.cpp index 701fa89447..eb25dac4b1 100644 --- a/ydb/core/yq/libs/control_plane_storage/internal/nodes_health_check.cpp +++ b/ydb/core/yq/libs/control_plane_storage/internal/nodes_health_check.cpp @@ -1,17 +1,17 @@ -#include "utils.h" - +#include "utils.h" + #include <ydb/core/yq/libs/db_schema/db_schema.h> -namespace NYq { - -void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvNodesHealthCheckRequest::TPtr& ev) -{ - TInstant startTime = TInstant::Now(); - TRequestCountersPtr requestCounters = Counters.Requests[RT_NODES_HEALTH_CHECK]; - requestCounters->InFly->Inc(); - - const auto& request = ev->Get()->Request; - const TString tenant = request.tenant(); +namespace NYq { + +void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvNodesHealthCheckRequest::TPtr& ev) +{ + TInstant startTime = TInstant::Now(); + TRequestCountersPtr requestCounters = Counters.Requests[RT_NODES_HEALTH_CHECK]; + requestCounters->InFly->Inc(); + + const auto& request = ev->Get()->Request; + const TString tenant = request.tenant(); const auto& node = request.node(); const ui32 nodeId = node.node_id(); const TString instanceId = node.instance_id(); @@ -19,23 +19,23 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvNodesHealth const ui64 activeWorkers = node.active_workers(); const ui64 memoryLimit = node.memory_limit(); const ui64 memoryAllocated = node.memory_allocated(); - const ui32 icPort = node.interconnect_port(); - const TString nodeAddress = node.node_address(); - const auto ttl = TDuration::Seconds(5); - const auto deadline = startTime + ttl * 3; - - CPS_LOG_T("NodesHealthCheck: " << tenant << " " << nodeId << " " << instanceId << " " << hostName << " " << node.node_address() << ":" << node.interconnect_port()); - - NYql::TIssues issues = ValidateNodesHealthCheck(tenant, instanceId, hostName); - if (issues) { - CPS_LOG_D("NodesHealthCheckRequest, validation failed: " << issues.ToString()); - const TDuration delta = TInstant::Now() - startTime; - SendResponseIssues<TEvControlPlaneStorage::TEvNodesHealthCheckResponse>(ev->Sender, issues, ev->Cookie, delta, requestCounters); - LWPROBE(NodesHealthCheckRequest, "", 0, "", "", delta, false); - return; - } - - std::shared_ptr<Yq::Private::NodesHealthCheckResult> response = std::make_shared<Yq::Private::NodesHealthCheckResult>(); + const ui32 icPort = node.interconnect_port(); + const TString nodeAddress = node.node_address(); + const auto ttl = TDuration::Seconds(5); + const auto deadline = startTime + ttl * 3; + + CPS_LOG_T("NodesHealthCheck: " << tenant << " " << nodeId << " " << instanceId << " " << hostName << " " << node.node_address() << ":" << node.interconnect_port()); + + NYql::TIssues issues = ValidateNodesHealthCheck(tenant, instanceId, hostName); + if (issues) { + CPS_LOG_D("NodesHealthCheckRequest, validation failed: " << issues.ToString()); + const TDuration delta = TInstant::Now() - startTime; + SendResponseIssues<TEvControlPlaneStorage::TEvNodesHealthCheckResponse>(ev->Sender, issues, ev->Cookie, delta, requestCounters); + LWPROBE(NodesHealthCheckRequest, "", 0, "", "", delta, false); + return; + } + + std::shared_ptr<Yq::Private::NodesHealthCheckResult> response = std::make_shared<Yq::Private::NodesHealthCheckResult>(); { auto* node = response->add_nodes(); node->set_node_id(nodeId); @@ -44,22 +44,22 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvNodesHealth node->set_active_workers(activeWorkers); node->set_memory_limit(memoryLimit); node->set_memory_allocated(memoryAllocated); - node->set_node_address(nodeAddress); + node->set_node_address(nodeAddress); } - - TSqlQueryBuilder readQueryBuilder(YdbConnection->TablePathPrefix, "NodesHealthCheck(read)"); + + TSqlQueryBuilder readQueryBuilder(YdbConnection->TablePathPrefix, "NodesHealthCheck(read)"); readQueryBuilder.AddTimestamp("now", TInstant::Now()); readQueryBuilder.AddString("tenant", tenant); readQueryBuilder.AddText( - "SELECT `" NODE_ID_COLUMN_NAME "`, `" INSTANCE_ID_COLUMN_NAME "`, `" HOST_NAME_COLUMN_NAME "`, `" ACTIVE_WORKERS_COLUMN_NAME"`, `" MEMORY_LIMIT_COLUMN_NAME"`, " - "`" MEMORY_ALLOCATED_COLUMN_NAME"`, `" INTERCONNECT_PORT_COLUMN_NAME "`, `" NODE_ADDRESS_COLUMN_NAME "` FROM `" NODES_TABLE_NAME "`\n" - "WHERE `" TENANT_COLUMN_NAME"` = $tenant AND `" EXPIRE_AT_COLUMN_NAME "` >= $now;\n" + "SELECT `" NODE_ID_COLUMN_NAME "`, `" INSTANCE_ID_COLUMN_NAME "`, `" HOST_NAME_COLUMN_NAME "`, `" ACTIVE_WORKERS_COLUMN_NAME"`, `" MEMORY_LIMIT_COLUMN_NAME"`, " + "`" MEMORY_ALLOCATED_COLUMN_NAME"`, `" INTERCONNECT_PORT_COLUMN_NAME "`, `" NODE_ADDRESS_COLUMN_NAME "` FROM `" NODES_TABLE_NAME "`\n" + "WHERE `" TENANT_COLUMN_NAME"` = $tenant AND `" EXPIRE_AT_COLUMN_NAME "` >= $now;\n" ); - - auto prepareParams = [=](const TVector<TResultSet>& resultSets) { - for (const auto& resultSet : resultSets) { - TResultSetParser parser(resultSet); - while (parser.TryNextRow()) { + + auto prepareParams = [=](const TVector<TResultSet>& resultSets) { + for (const auto& resultSet : resultSets) { + TResultSetParser parser(resultSet); + while (parser.TryNextRow()) { auto nid = *parser.ColumnParser(NODE_ID_COLUMN_NAME).GetOptionalUint32(); if (nid != nodeId) { auto* node = response->add_nodes(); @@ -69,13 +69,13 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvNodesHealth node->set_active_workers(*parser.ColumnParser(ACTIVE_WORKERS_COLUMN_NAME).GetOptionalUint64()); node->set_memory_limit(*parser.ColumnParser(MEMORY_LIMIT_COLUMN_NAME).GetOptionalUint64()); node->set_memory_allocated(*parser.ColumnParser(MEMORY_ALLOCATED_COLUMN_NAME).GetOptionalUint64()); - node->set_interconnect_port(parser.ColumnParser(INTERCONNECT_PORT_COLUMN_NAME).GetOptionalUint32().GetOrElse(0)); - node->set_node_address(*parser.ColumnParser(NODE_ADDRESS_COLUMN_NAME).GetOptionalString()); + node->set_interconnect_port(parser.ColumnParser(INTERCONNECT_PORT_COLUMN_NAME).GetOptionalUint32().GetOrElse(0)); + node->set_node_address(*parser.ColumnParser(NODE_ADDRESS_COLUMN_NAME).GetOptionalString()); } - } - } + } + } - TSqlQueryBuilder writeQueryBuilder(YdbConnection->TablePathPrefix, "NodesHealthCheck(write)"); + TSqlQueryBuilder writeQueryBuilder(YdbConnection->TablePathPrefix, "NodesHealthCheck(write)"); writeQueryBuilder.AddString("tenant", tenant); writeQueryBuilder.AddUint32("node_id", nodeId); writeQueryBuilder.AddString("instance_id", instanceId); @@ -94,27 +94,27 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvNodesHealth ); const auto writeQuery = writeQueryBuilder.Build(); return std::make_pair(writeQuery.Sql, writeQuery.Params); - }; - + }; + const auto readQuery = readQueryBuilder.Build(); auto debugInfo = Config.Proto.GetEnableDebugMode() ? std::make_shared<TDebugInfo>() : TDebugInfoPtr{}; TAsyncStatus status = ReadModifyWrite(NActors::TActivationContext::ActorSystem(), readQuery.Sql, readQuery.Params, prepareParams, requestCounters, debugInfo); - auto prepare = [response] { return *response; }; - auto success = SendResponse<TEvControlPlaneStorage::TEvNodesHealthCheckResponse, Yq::Private::NodesHealthCheckResult>( - "NodesHealthCheckRequest", - NActors::TActivationContext::ActorSystem(), - status, - SelfId(), - ev, - startTime, - requestCounters, - prepare, - debugInfo); - - success.Apply([=](const auto& future) { - TDuration delta = TInstant::Now() - startTime; - LWPROBE(NodesHealthCheckRequest, tenant, nodeId, instanceId, hostName, delta, future.GetValue()); - }); -} - -} // NYq + auto prepare = [response] { return *response; }; + auto success = SendResponse<TEvControlPlaneStorage::TEvNodesHealthCheckResponse, Yq::Private::NodesHealthCheckResult>( + "NodesHealthCheckRequest", + NActors::TActivationContext::ActorSystem(), + status, + SelfId(), + ev, + startTime, + requestCounters, + prepare, + debugInfo); + + success.Apply([=](const auto& future) { + TDuration delta = TInstant::Now() - startTime; + LWPROBE(NodesHealthCheckRequest, tenant, nodeId, instanceId, hostName, delta, future.GetValue()); + }); +} + +} // NYq diff --git a/ydb/core/yq/libs/control_plane_storage/internal/response_tasks.cpp b/ydb/core/yq/libs/control_plane_storage/internal/response_tasks.cpp index a3feec3252..1d64d9ed58 100644 --- a/ydb/core/yq/libs/control_plane_storage/internal/response_tasks.cpp +++ b/ydb/core/yq/libs/control_plane_storage/internal/response_tasks.cpp @@ -1,47 +1,47 @@ -#include "response_tasks.h" - -namespace NYq { - -void TResponseTasks::AddTaskNonBlocking(const TString& key, const TEvControlPlaneStorage::TTask& task) { - Tasks[key] = task; -} - -void TResponseTasks::AddTaskBlocking(const TString& key, const TEvControlPlaneStorage::TTask& task) { - with_lock (Mutex) { - Tasks[key] = task; - } -} - -void TResponseTasks::SafeEraseTaskNonBlocking(const TString& key) { - if (auto it = Tasks.find(key); it != Tasks.end()) - Tasks.erase(it); -} - -void TResponseTasks::SafeEraseTaskBlocking(const TString& key) { - with_lock (Mutex) { - if (auto it = Tasks.find(key); it != Tasks.end()) - Tasks.erase(it); - } -} - -bool TResponseTasks::EmptyNonBlocking() { - return Tasks.empty(); -} - -bool TResponseTasks::EmptyBlocking() { - with_lock (Mutex) { - return Tasks.empty(); - } -} - -const THashMap<TString, TEvControlPlaneStorage::TTask>& TResponseTasks::GetTasksNonBlocking() { - return Tasks; -} - -const THashMap<TString, TEvControlPlaneStorage::TTask>& TResponseTasks::GetTasksBlocking() { - with_lock (Mutex) { - return Tasks; - } -} - -} //NYq
\ No newline at end of file +#include "response_tasks.h" + +namespace NYq { + +void TResponseTasks::AddTaskNonBlocking(const TString& key, const TEvControlPlaneStorage::TTask& task) { + Tasks[key] = task; +} + +void TResponseTasks::AddTaskBlocking(const TString& key, const TEvControlPlaneStorage::TTask& task) { + with_lock (Mutex) { + Tasks[key] = task; + } +} + +void TResponseTasks::SafeEraseTaskNonBlocking(const TString& key) { + if (auto it = Tasks.find(key); it != Tasks.end()) + Tasks.erase(it); +} + +void TResponseTasks::SafeEraseTaskBlocking(const TString& key) { + with_lock (Mutex) { + if (auto it = Tasks.find(key); it != Tasks.end()) + Tasks.erase(it); + } +} + +bool TResponseTasks::EmptyNonBlocking() { + return Tasks.empty(); +} + +bool TResponseTasks::EmptyBlocking() { + with_lock (Mutex) { + return Tasks.empty(); + } +} + +const THashMap<TString, TEvControlPlaneStorage::TTask>& TResponseTasks::GetTasksNonBlocking() { + return Tasks; +} + +const THashMap<TString, TEvControlPlaneStorage::TTask>& TResponseTasks::GetTasksBlocking() { + with_lock (Mutex) { + return Tasks; + } +} + +} //NYq
\ No newline at end of file diff --git a/ydb/core/yq/libs/control_plane_storage/internal/response_tasks.h b/ydb/core/yq/libs/control_plane_storage/internal/response_tasks.h index 538c333967..1ab660d43d 100644 --- a/ydb/core/yq/libs/control_plane_storage/internal/response_tasks.h +++ b/ydb/core/yq/libs/control_plane_storage/internal/response_tasks.h @@ -1,31 +1,31 @@ #pragma once -#include <util/system/mutex.h> - +#include <util/system/mutex.h> + #include <ydb/core/yq/libs/control_plane_storage/events/events.h> -namespace NYq { - -class TResponseTasks { -public: - void AddTaskNonBlocking(const TString& key, const TEvControlPlaneStorage::TTask& task); - - void AddTaskBlocking(const TString& key, const TEvControlPlaneStorage::TTask& task); - - void SafeEraseTaskNonBlocking(const TString& key); - - void SafeEraseTaskBlocking(const TString& key); - - bool EmptyNonBlocking(); - - bool EmptyBlocking(); - - const THashMap<TString, TEvControlPlaneStorage::TTask>& GetTasksNonBlocking(); - - const THashMap<TString, TEvControlPlaneStorage::TTask>& GetTasksBlocking(); - -private: - TMutex Mutex; - THashMap<TString, TEvControlPlaneStorage::TTask> Tasks; -}; - +namespace NYq { + +class TResponseTasks { +public: + void AddTaskNonBlocking(const TString& key, const TEvControlPlaneStorage::TTask& task); + + void AddTaskBlocking(const TString& key, const TEvControlPlaneStorage::TTask& task); + + void SafeEraseTaskNonBlocking(const TString& key); + + void SafeEraseTaskBlocking(const TString& key); + + bool EmptyNonBlocking(); + + bool EmptyBlocking(); + + const THashMap<TString, TEvControlPlaneStorage::TTask>& GetTasksNonBlocking(); + + const THashMap<TString, TEvControlPlaneStorage::TTask>& GetTasksBlocking(); + +private: + TMutex Mutex; + THashMap<TString, TEvControlPlaneStorage::TTask> Tasks; +}; + } //NYq diff --git a/ydb/core/yq/libs/control_plane_storage/internal/task_get.cpp b/ydb/core/yq/libs/control_plane_storage/internal/task_get.cpp index bad099564d..42b3c1a1ac 100644 --- a/ydb/core/yq/libs/control_plane_storage/internal/task_get.cpp +++ b/ydb/core/yq/libs/control_plane_storage/internal/task_get.cpp @@ -1,36 +1,36 @@ -#include "utils.h" - -#include <random> - +#include "utils.h" + +#include <random> + #include <util/datetime/base.h> #include <ydb/core/yq/libs/control_plane_storage/schema.h> #include <ydb/core/yq/libs/db_schema/db_schema.h> -namespace NYq { - -namespace { - -struct TTaskInternal { - TEvControlPlaneStorage::TTask Task; - ui64 RetryCounter = 0; - TInstant RetryCounterUpdatedAt = TInstant::Zero(); - bool ShouldAbortTask = false; - TString TablePathPrefix; - TString Owner; - TString HostName; - TMaybe<YandexQuery::Job> Job; +namespace NYq { + +namespace { + +struct TTaskInternal { + TEvControlPlaneStorage::TTask Task; + ui64 RetryCounter = 0; + TInstant RetryCounterUpdatedAt = TInstant::Zero(); + bool ShouldAbortTask = false; + TString TablePathPrefix; + TString Owner; + TString HostName; + TMaybe<YandexQuery::Job> Job; TInstant Deadline; -}; - -std::pair<TString, NYdb::TParams> MakeSql(const TTaskInternal& taskInternal) { - const auto& task = taskInternal.Task; - const auto& retryCounter = taskInternal.RetryCounter; - const auto& retryCounterUpdatedAt = taskInternal.RetryCounterUpdatedAt; - const auto& hostName = taskInternal.HostName; - const auto& owner = taskInternal.Owner; - - TSqlQueryBuilder queryBuilder(taskInternal.TablePathPrefix, "GetTask(write)"); +}; + +std::pair<TString, NYdb::TParams> MakeSql(const TTaskInternal& taskInternal) { + const auto& task = taskInternal.Task; + const auto& retryCounter = taskInternal.RetryCounter; + const auto& retryCounterUpdatedAt = taskInternal.RetryCounterUpdatedAt; + const auto& hostName = taskInternal.HostName; + const auto& owner = taskInternal.Owner; + + TSqlQueryBuilder queryBuilder(taskInternal.TablePathPrefix, "GetTask(write)"); queryBuilder.AddString("scope", task.Scope); queryBuilder.AddString("query_id", task.QueryId); queryBuilder.AddString("query", task.Query.SerializeAsString()); @@ -44,213 +44,213 @@ std::pair<TString, NYdb::TParams> MakeSql(const TTaskInternal& taskInternal) { // update queries queryBuilder.AddText( - "UPDATE `" QUERIES_TABLE_NAME "` SET `" GENERATION_COLUMN_NAME "` = $generation, `" QUERY_COLUMN_NAME "` = $query, `" INTERNAL_COLUMN_NAME "` = $internal\n" + "UPDATE `" QUERIES_TABLE_NAME "` SET `" GENERATION_COLUMN_NAME "` = $generation, `" QUERY_COLUMN_NAME "` = $query, `" INTERNAL_COLUMN_NAME "` = $internal\n" "WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id;\n" ); // update pending small queryBuilder.AddText( - "UPDATE `" PENDING_SMALL_TABLE_NAME "` SET `" LAST_SEEN_AT_COLUMN_NAME "` = $now,\n" - "`" RETRY_COUNTER_COLUMN_NAME "` = $retry_counter, `" RETRY_COUNTER_UPDATE_COLUMN_NAME "` = $retry_counter_update_time, `" IS_RESIGN_QUERY_COLUMN_NAME "` = false,\n" - "`" HOST_NAME_COLUMN_NAME "` = $host, `" OWNER_COLUMN_NAME "` = $owner\n" + "UPDATE `" PENDING_SMALL_TABLE_NAME "` SET `" LAST_SEEN_AT_COLUMN_NAME "` = $now,\n" + "`" RETRY_COUNTER_COLUMN_NAME "` = $retry_counter, `" RETRY_COUNTER_UPDATE_COLUMN_NAME "` = $retry_counter_update_time, `" IS_RESIGN_QUERY_COLUMN_NAME "` = false,\n" + "`" HOST_NAME_COLUMN_NAME "` = $host, `" OWNER_COLUMN_NAME "` = $owner\n" "WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id;\n" ); - + const auto query = queryBuilder.Build(); return std::make_pair(query.Sql, query.Params); -} - -} // namespace - -std::tuple<TString, NYdb::TParams, std::function<std::pair<TString, NYdb::TParams>(const TVector<NYdb::TResultSet>&)>> MakeGetTaskUpdateQuery( - const TTaskInternal& taskInternal, - const std::shared_ptr<TResponseTasks>& responseTasks, +} + +} // namespace + +std::tuple<TString, NYdb::TParams, std::function<std::pair<TString, NYdb::TParams>(const TVector<NYdb::TResultSet>&)>> MakeGetTaskUpdateQuery( + const TTaskInternal& taskInternal, + const std::shared_ptr<TResponseTasks>& responseTasks, const TInstant& taskLeaseTimestamp, bool disableCurrentIam, const TDuration& automaticQueriesTtl, const TDuration& resultSetsTtl) -{ - const auto& task = taskInternal.Task; - - TSqlQueryBuilder queryBuilder(taskInternal.TablePathPrefix, "GetTask(read)"); +{ + const auto& task = taskInternal.Task; + + TSqlQueryBuilder queryBuilder(taskInternal.TablePathPrefix, "GetTask(read)"); queryBuilder.AddString("scope", task.Scope); queryBuilder.AddString("query_id", task.QueryId); queryBuilder.AddTimestamp("from", taskLeaseTimestamp); - + queryBuilder.AddText( - "SELECT `" GENERATION_COLUMN_NAME "`, `" INTERNAL_COLUMN_NAME "`, `" QUERY_COLUMN_NAME "`\n" - "FROM `" QUERIES_TABLE_NAME "`\n" - "WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id;\n" - "SELECT `" LAST_SEEN_AT_COLUMN_NAME "`\n" - "FROM `" PENDING_SMALL_TABLE_NAME "`\n" - "WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id AND `" LAST_SEEN_AT_COLUMN_NAME "` < $from;\n" + "SELECT `" GENERATION_COLUMN_NAME "`, `" INTERNAL_COLUMN_NAME "`, `" QUERY_COLUMN_NAME "`\n" + "FROM `" QUERIES_TABLE_NAME "`\n" + "WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id;\n" + "SELECT `" LAST_SEEN_AT_COLUMN_NAME "`\n" + "FROM `" PENDING_SMALL_TABLE_NAME "`\n" + "WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id AND `" LAST_SEEN_AT_COLUMN_NAME "` < $from;\n" ); - - auto prepareParams = [=, taskInternal=taskInternal, responseTasks=responseTasks](const TVector<TResultSet>& resultSets) mutable { - auto& task = taskInternal.Task; - const auto shouldAbortTask = taskInternal.ShouldAbortTask; - constexpr size_t expectedResultSetsSize = 2; - - if (resultSets.size() != expectedResultSetsSize || !resultSets[1].RowsCount()) { - return std::make_pair(TString{}, TParamsBuilder{}.Build()); - } - - { - TResultSetParser parser(resultSets[0]); - while (parser.TryNextRow()) { - task.Generation = parser.ColumnParser(GENERATION_COLUMN_NAME).GetOptionalUint64().GetOrElse(0) + 1; - - if (!task.Query.ParseFromString(*parser.ColumnParser(QUERY_COLUMN_NAME).GetOptionalString())) { - throw TControlPlaneStorageException(TIssuesIds::INTERNAL_ERROR) << "Error parsing proto message for query. Please contact internal support"; - } + + auto prepareParams = [=, taskInternal=taskInternal, responseTasks=responseTasks](const TVector<TResultSet>& resultSets) mutable { + auto& task = taskInternal.Task; + const auto shouldAbortTask = taskInternal.ShouldAbortTask; + constexpr size_t expectedResultSetsSize = 2; + + if (resultSets.size() != expectedResultSetsSize || !resultSets[1].RowsCount()) { + return std::make_pair(TString{}, TParamsBuilder{}.Build()); + } + + { + TResultSetParser parser(resultSets[0]); + while (parser.TryNextRow()) { + task.Generation = parser.ColumnParser(GENERATION_COLUMN_NAME).GetOptionalUint64().GetOrElse(0) + 1; + + if (!task.Query.ParseFromString(*parser.ColumnParser(QUERY_COLUMN_NAME).GetOptionalString())) { + throw TControlPlaneStorageException(TIssuesIds::INTERNAL_ERROR) << "Error parsing proto message for query. Please contact internal support"; + } const TInstant deadline = TInstant::Now() + (task.Query.content().automatic() ? std::min(automaticQueriesTtl, resultSetsTtl) : resultSetsTtl); task.Deadline = deadline; - if (!task.Internal.ParseFromString(*parser.ColumnParser(INTERNAL_COLUMN_NAME).GetOptionalString())) { - throw TControlPlaneStorageException(TIssuesIds::INTERNAL_ERROR) << "Error parsing proto message for query internal. Please contact internal support"; - } - + if (!task.Internal.ParseFromString(*parser.ColumnParser(INTERNAL_COLUMN_NAME).GetOptionalString())) { + throw TControlPlaneStorageException(TIssuesIds::INTERNAL_ERROR) << "Error parsing proto message for query internal. Please contact internal support"; + } + if (disableCurrentIam) { task.Internal.clear_token(); } - } - } - - if (shouldAbortTask) { - Ydb::Issue::IssueMessage abortIssueMsg; - abortIssueMsg.set_message("Query was aborted by system due to high failure rate"); - abortIssueMsg.set_severity(NYql::TSeverityIds::S_ERROR); - *task.Query.add_issue() = abortIssueMsg; - task.Query.mutable_meta()->set_status(YandexQuery::QueryMeta::ABORTING_BY_SYSTEM); - } - - responseTasks->AddTaskBlocking(task.QueryId, task); - - return MakeSql(taskInternal); - }; - + } + } + + if (shouldAbortTask) { + Ydb::Issue::IssueMessage abortIssueMsg; + abortIssueMsg.set_message("Query was aborted by system due to high failure rate"); + abortIssueMsg.set_severity(NYql::TSeverityIds::S_ERROR); + *task.Query.add_issue() = abortIssueMsg; + task.Query.mutable_meta()->set_status(YandexQuery::QueryMeta::ABORTING_BY_SYSTEM); + } + + responseTasks->AddTaskBlocking(task.QueryId, task); + + return MakeSql(taskInternal); + }; + const auto query = queryBuilder.Build(); return std::make_tuple(query.Sql, query.Params, prepareParams); -} - -void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvGetTaskRequest::TPtr& ev) -{ - TInstant startTime = TInstant::Now(); - TRequestCountersPtr requestCounters = Counters.Requests[RT_GET_TASK]; - requestCounters->InFly->Inc(); - - TEvControlPlaneStorage::TEvGetTaskRequest& request = *ev->Get(); - const TString owner = request.Owner; - const TString hostName = request.HostName; +} + +void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvGetTaskRequest::TPtr& ev) +{ + TInstant startTime = TInstant::Now(); + TRequestCountersPtr requestCounters = Counters.Requests[RT_GET_TASK]; + requestCounters->InFly->Inc(); + + TEvControlPlaneStorage::TEvGetTaskRequest& request = *ev->Get(); + const TString owner = request.Owner; + const TString hostName = request.HostName; const ui64 tasksBatchSize = Config.Proto.GetTasksBatchSize(); const ui64 numTasksProportion = Config.Proto.GetNumTasksProportion(); - - CPS_LOG_T("GetTaskRequest: " << owner << " " << hostName); - - NYql::TIssues issues = ValidateGetTask(owner, hostName); - if (issues) { - CPS_LOG_D("GetTaskRequest, validation failed: " << owner << " " << hostName << " " << issues.ToString()); - const TDuration delta = TInstant::Now() - startTime; - SendResponseIssues<TEvControlPlaneStorage::TEvGetTaskResponse>(ev->Sender, issues, ev->Cookie, delta, requestCounters); - LWPROBE(GetTaskRequest, owner, hostName, delta, false); - return; - } - + + CPS_LOG_T("GetTaskRequest: " << owner << " " << hostName); + + NYql::TIssues issues = ValidateGetTask(owner, hostName); + if (issues) { + CPS_LOG_D("GetTaskRequest, validation failed: " << owner << " " << hostName << " " << issues.ToString()); + const TDuration delta = TInstant::Now() - startTime; + SendResponseIssues<TEvControlPlaneStorage::TEvGetTaskResponse>(ev->Sender, issues, ev->Cookie, delta, requestCounters); + LWPROBE(GetTaskRequest, owner, hostName, delta, false); + return; + } + auto debugInfo = Config.Proto.GetEnableDebugMode() ? std::make_shared<TDebugInfo>() : TDebugInfoPtr{}; - - auto response = std::make_shared<std::tuple<TVector<TEvControlPlaneStorage::TTask>, TString>>(); //tasks, owner - - TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "GetTask(read stale ro)"); + + auto response = std::make_shared<std::tuple<TVector<TEvControlPlaneStorage::TTask>, TString>>(); //tasks, owner + + TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "GetTask(read stale ro)"); const auto taskLeaseTimestamp = TInstant::Now() - Config.TaskLeaseTtl; queryBuilder.AddTimestamp("from", taskLeaseTimestamp); queryBuilder.AddUint64("tasks_limit", tasksBatchSize); queryBuilder.AddText( - "SELECT `" SCOPE_COLUMN_NAME "`, `" QUERY_ID_COLUMN_NAME "`,\n" - "`" RETRY_COUNTER_COLUMN_NAME "`, `" RETRY_COUNTER_UPDATE_COLUMN_NAME "`, `" QUERY_TYPE_COLUMN_NAME "`, `" IS_RESIGN_QUERY_COLUMN_NAME "`\n" - "FROM `" PENDING_SMALL_TABLE_NAME "`\n" - "WHERE `" LAST_SEEN_AT_COLUMN_NAME "` < $from ORDER BY `" QUERY_ID_COLUMN_NAME "` DESC LIMIT $tasks_limit;\n" + "SELECT `" SCOPE_COLUMN_NAME "`, `" QUERY_ID_COLUMN_NAME "`,\n" + "`" RETRY_COUNTER_COLUMN_NAME "`, `" RETRY_COUNTER_UPDATE_COLUMN_NAME "`, `" QUERY_TYPE_COLUMN_NAME "`, `" IS_RESIGN_QUERY_COLUMN_NAME "`\n" + "FROM `" PENDING_SMALL_TABLE_NAME "`\n" + "WHERE `" LAST_SEEN_AT_COLUMN_NAME "` < $from ORDER BY `" QUERY_ID_COLUMN_NAME "` DESC LIMIT $tasks_limit;\n" ); - - auto responseTasks = std::make_shared<TResponseTasks>(); - auto prepareParams = [=, responseTasks=responseTasks](const TVector<TResultSet>& resultSets) mutable { - TVector<TTaskInternal> tasks; - TVector<TPickTaskParams> pickTaskParams; - const auto now = TInstant::Now(); - if (resultSets.empty() || !resultSets.back().RowsCount()) { + + auto responseTasks = std::make_shared<TResponseTasks>(); + auto prepareParams = [=, responseTasks=responseTasks](const TVector<TResultSet>& resultSets) mutable { + TVector<TTaskInternal> tasks; + TVector<TPickTaskParams> pickTaskParams; + const auto now = TInstant::Now(); + if (resultSets.empty() || !resultSets.back().RowsCount()) { return pickTaskParams; - } - - TResultSetParser parser(resultSets.back()); - while (parser.TryNextRow()) { - TTaskInternal& taskInternal = tasks.emplace_back(); - taskInternal.Owner = owner; - taskInternal.HostName = hostName; - taskInternal.TablePathPrefix = YdbConnection->TablePathPrefix; - - auto& task = taskInternal.Task; - - task.Scope = *parser.ColumnParser(SCOPE_COLUMN_NAME).GetOptionalString(); - task.QueryId = *parser.ColumnParser(QUERY_ID_COLUMN_NAME).GetOptionalString(); - - bool isResignQuery = parser.ColumnParser(IS_RESIGN_QUERY_COLUMN_NAME).GetOptionalBool().GetOrElse(false); - taskInternal.RetryCounter = parser.ColumnParser(RETRY_COUNTER_COLUMN_NAME).GetOptionalUint64().GetOrElse(0); - taskInternal.RetryCounterUpdatedAt = parser.ColumnParser(RETRY_COUNTER_UPDATE_COLUMN_NAME).GetOptionalTimestamp().GetOrElse(TInstant::Zero()); - - YandexQuery::QueryContent::QueryType queryType = static_cast<YandexQuery::QueryContent::QueryType>(parser.ColumnParser(QUERY_TYPE_COLUMN_NAME).GetOptionalInt64().GetOrElse(0)); - + } + + TResultSetParser parser(resultSets.back()); + while (parser.TryNextRow()) { + TTaskInternal& taskInternal = tasks.emplace_back(); + taskInternal.Owner = owner; + taskInternal.HostName = hostName; + taskInternal.TablePathPrefix = YdbConnection->TablePathPrefix; + + auto& task = taskInternal.Task; + + task.Scope = *parser.ColumnParser(SCOPE_COLUMN_NAME).GetOptionalString(); + task.QueryId = *parser.ColumnParser(QUERY_ID_COLUMN_NAME).GetOptionalString(); + + bool isResignQuery = parser.ColumnParser(IS_RESIGN_QUERY_COLUMN_NAME).GetOptionalBool().GetOrElse(false); + taskInternal.RetryCounter = parser.ColumnParser(RETRY_COUNTER_COLUMN_NAME).GetOptionalUint64().GetOrElse(0); + taskInternal.RetryCounterUpdatedAt = parser.ColumnParser(RETRY_COUNTER_UPDATE_COLUMN_NAME).GetOptionalTimestamp().GetOrElse(TInstant::Zero()); + + YandexQuery::QueryContent::QueryType queryType = static_cast<YandexQuery::QueryContent::QueryType>(parser.ColumnParser(QUERY_TYPE_COLUMN_NAME).GetOptionalInt64().GetOrElse(0)); + const auto retryCounterLimit = queryType == YandexQuery::QueryContent::ANALYTICS ? Config.Proto.GetAnalyticsRetryCounterLimit() : Config.Proto.GetStreamingRetryCounterLimit(); const auto retryCounterUpdateTime = queryType == YandexQuery::QueryContent::ANALYTICS ? Config.AnalyticsRetryCounterUpdateTime : Config.StreamingRetryCounterUpdateTime; - - if (isResignQuery) { - continue; - } - - ++taskInternal.RetryCounter; - - if (now > taskInternal.RetryCounterUpdatedAt + retryCounterUpdateTime) { - taskInternal.RetryCounter = 0; - taskInternal.RetryCounterUpdatedAt = now; - } - - if (taskInternal.RetryCounter >= retryCounterLimit) { - taskInternal.ShouldAbortTask = true; - } - } - - std::shuffle(tasks.begin(), tasks.end(), std::default_random_engine()); - const size_t numTasks = (std::min(tasks.size(), tasksBatchSize) + numTasksProportion - 1) / numTasksProportion; - - for (size_t i = 0; i < numTasks; ++i) { - auto tupleParams = MakeGetTaskUpdateQuery(tasks[i], - responseTasks, taskLeaseTimestamp, Config.Proto.GetDisableCurrentIam(), - Config.AutomaticQueriesTtl, Config.ResultSetsTtl); // using for win32 build - auto readQuery = std::get<0>(tupleParams); - auto readParams = std::get<1>(tupleParams); - auto prepareParams = std::get<2>(tupleParams); - pickTaskParams.emplace_back(TPickTaskParams{readQuery, readParams, prepareParams, tasks[i].Task.QueryId, tasks[i].ShouldAbortTask}); - } - return pickTaskParams; - }; - + + if (isResignQuery) { + continue; + } + + ++taskInternal.RetryCounter; + + if (now > taskInternal.RetryCounterUpdatedAt + retryCounterUpdateTime) { + taskInternal.RetryCounter = 0; + taskInternal.RetryCounterUpdatedAt = now; + } + + if (taskInternal.RetryCounter >= retryCounterLimit) { + taskInternal.ShouldAbortTask = true; + } + } + + std::shuffle(tasks.begin(), tasks.end(), std::default_random_engine()); + const size_t numTasks = (std::min(tasks.size(), tasksBatchSize) + numTasksProportion - 1) / numTasksProportion; + + for (size_t i = 0; i < numTasks; ++i) { + auto tupleParams = MakeGetTaskUpdateQuery(tasks[i], + responseTasks, taskLeaseTimestamp, Config.Proto.GetDisableCurrentIam(), + Config.AutomaticQueriesTtl, Config.ResultSetsTtl); // using for win32 build + auto readQuery = std::get<0>(tupleParams); + auto readParams = std::get<1>(tupleParams); + auto prepareParams = std::get<2>(tupleParams); + pickTaskParams.emplace_back(TPickTaskParams{readQuery, readParams, prepareParams, tasks[i].Task.QueryId, tasks[i].ShouldAbortTask}); + } + return pickTaskParams; + }; + const auto query = queryBuilder.Build(); auto [readStatus, resultSets] = Read(query.Sql, query.Params, requestCounters, debugInfo, TTxSettings::StaleRO()); - auto result = readStatus.Apply( - [=, - resultSets=resultSets, - requestCounters=requestCounters, - debugInfo=debugInfo, - responseTasks=responseTasks] (const auto& readFuture) mutable - { - try { - if (!readFuture.GetValue().IsSuccess()) - return readFuture; - } catch (...) { - return readFuture; - } - - auto pickTaskParams = prepareParams(*resultSets); - if (pickTaskParams.empty()) - return readFuture; - + auto result = readStatus.Apply( + [=, + resultSets=resultSets, + requestCounters=requestCounters, + debugInfo=debugInfo, + responseTasks=responseTasks] (const auto& readFuture) mutable + { + try { + if (!readFuture.GetValue().IsSuccess()) + return readFuture; + } catch (...) { + return readFuture; + } + + auto pickTaskParams = prepareParams(*resultSets); + if (pickTaskParams.empty()) + return readFuture; + auto debugInfos = std::make_shared<TVector<TDebugInfoPtr>>(pickTaskParams.size()); if (Config.Proto.GetEnableDebugMode()) { for (size_t i = 0; i < pickTaskParams.size(); i++) { @@ -258,53 +258,53 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvGetTaskRequ } } - TVector<TFuture<void>> futures; - for (size_t i = 0; i < pickTaskParams.size(); ++i) { + TVector<TFuture<void>> futures; + for (size_t i = 0; i < pickTaskParams.size(); ++i) { futures.emplace_back(PickTask(pickTaskParams[i], requestCounters, (*debugInfos)[i], responseTasks)); - } - - auto allFuture = NThreading::WaitExceptionOrAll(futures); - return allFuture.Apply([=, responseTasks=responseTasks](const auto& future) mutable { + } + + auto allFuture = NThreading::WaitExceptionOrAll(futures); + return allFuture.Apply([=, responseTasks=responseTasks](const auto& future) mutable { if (debugInfo) { for (const auto& info: *debugInfos) { debugInfo->insert(debugInfo->end(), info->begin(), info->end()); } } - NYql::TIssues issues; - auto status = MakeFuture(TStatus{EStatus::SUCCESS, std::move(issues)}); - try { - future.GetValue(); - TVector<TEvControlPlaneStorage::TTask> tasks; - for (const auto& [_, task] : responseTasks->GetTasksNonBlocking()) { - tasks.emplace_back(task); - } - *response = std::make_tuple(tasks, owner); - } catch (...) { - issues.AddIssue(CurrentExceptionMessage()); - status = MakeFuture(TStatus{EStatus::GENERIC_ERROR, std::move(issues)}); - } - return status; - }); - }); - - auto prepare = [response] { return *response; }; - auto success = SendResponseTuple - <TEvControlPlaneStorage::TEvGetTaskResponse, - std::tuple<TVector<TEvControlPlaneStorage::TTask>, TString>> //tasks, owner - ("GetTaskRequest", - NActors::TActivationContext::ActorSystem(), - result, - SelfId(), - ev, - startTime, - requestCounters, - prepare, - debugInfo); - - success.Apply([=](const auto& future) { - TDuration delta = TInstant::Now() - startTime; - LWPROBE(GetTaskRequest, owner, hostName, delta, future.GetValue()); - }); -} - -} // NYq + NYql::TIssues issues; + auto status = MakeFuture(TStatus{EStatus::SUCCESS, std::move(issues)}); + try { + future.GetValue(); + TVector<TEvControlPlaneStorage::TTask> tasks; + for (const auto& [_, task] : responseTasks->GetTasksNonBlocking()) { + tasks.emplace_back(task); + } + *response = std::make_tuple(tasks, owner); + } catch (...) { + issues.AddIssue(CurrentExceptionMessage()); + status = MakeFuture(TStatus{EStatus::GENERIC_ERROR, std::move(issues)}); + } + return status; + }); + }); + + auto prepare = [response] { return *response; }; + auto success = SendResponseTuple + <TEvControlPlaneStorage::TEvGetTaskResponse, + std::tuple<TVector<TEvControlPlaneStorage::TTask>, TString>> //tasks, owner + ("GetTaskRequest", + NActors::TActivationContext::ActorSystem(), + result, + SelfId(), + ev, + startTime, + requestCounters, + prepare, + debugInfo); + + success.Apply([=](const auto& future) { + TDuration delta = TInstant::Now() - startTime; + LWPROBE(GetTaskRequest, owner, hostName, delta, future.GetValue()); + }); +} + +} // NYq diff --git a/ydb/core/yq/libs/control_plane_storage/internal/task_ping.cpp b/ydb/core/yq/libs/control_plane_storage/internal/task_ping.cpp index b3a81355af..834d40c0f7 100644 --- a/ydb/core/yq/libs/control_plane_storage/internal/task_ping.cpp +++ b/ydb/core/yq/libs/control_plane_storage/internal/task_ping.cpp @@ -1,11 +1,11 @@ -#include "utils.h" - +#include "utils.h" + #include <util/datetime/base.h> #include <ydb/core/yq/libs/db_schema/db_schema.h> -namespace NYq { - +namespace NYq { + namespace { bool IsFinishedStatus(YandexQuery::QueryMeta::ComputeStatus status) { @@ -17,129 +17,129 @@ bool IsFinishedStatus(YandexQuery::QueryMeta::ComputeStatus status) { } // namespace -std::tuple<TString, TParams, const std::function<std::pair<TString, NYdb::TParams>(const TVector<NYdb::TResultSet>&)>> ConstructHardPingTask( - const TEvControlPlaneStorage::TEvPingTaskRequest* request, std::shared_ptr<YandexQuery::QueryAction> response, +std::tuple<TString, TParams, const std::function<std::pair<TString, NYdb::TParams>(const TVector<NYdb::TResultSet>&)>> ConstructHardPingTask( + const TEvControlPlaneStorage::TEvPingTaskRequest* request, std::shared_ptr<YandexQuery::QueryAction> response, const TString& tablePathPrefix, const TDuration& automaticQueriesTtl) { - - TSqlQueryBuilder readQueryBuilder(tablePathPrefix, "HardPingTask(read)"); + + TSqlQueryBuilder readQueryBuilder(tablePathPrefix, "HardPingTask(read)"); readQueryBuilder.AddString("scope", request->Scope); readQueryBuilder.AddString("query_id", request->QueryId); readQueryBuilder.AddText( - "$last_job_id = SELECT `" LAST_JOB_ID_COLUMN_NAME "` FROM `" QUERIES_TABLE_NAME "`\n" - " WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id;\n" - "SELECT `" QUERY_COLUMN_NAME "`, `" INTERNAL_COLUMN_NAME "` FROM `" QUERIES_TABLE_NAME "`\n" - " WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id;\n" - "SELECT `" JOB_ID_COLUMN_NAME "`, `" JOB_COLUMN_NAME "` FROM `" JOBS_TABLE_NAME "`\n" - " WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id AND `" JOB_ID_COLUMN_NAME "` = $last_job_id;\n" - "SELECT `" OWNER_COLUMN_NAME "` FROM `" PENDING_SMALL_TABLE_NAME "`\n" - " WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id;\n" + "$last_job_id = SELECT `" LAST_JOB_ID_COLUMN_NAME "` FROM `" QUERIES_TABLE_NAME "`\n" + " WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id;\n" + "SELECT `" QUERY_COLUMN_NAME "`, `" INTERNAL_COLUMN_NAME "` FROM `" QUERIES_TABLE_NAME "`\n" + " WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id;\n" + "SELECT `" JOB_ID_COLUMN_NAME "`, `" JOB_COLUMN_NAME "` FROM `" JOBS_TABLE_NAME "`\n" + " WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id AND `" JOB_ID_COLUMN_NAME "` = $last_job_id;\n" + "SELECT `" OWNER_COLUMN_NAME "` FROM `" PENDING_SMALL_TABLE_NAME "`\n" + " WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id;\n" ); - - auto prepareParams = [=](const TVector<TResultSet>& resultSets) { - TString jobId; - YandexQuery::Query query; - YandexQuery::Internal::QueryInternal internal; - YandexQuery::Job job; - TString selectedOwner; - - if (resultSets.size() != 3) { - ythrow TControlPlaneStorageException(TIssuesIds::INTERNAL_ERROR) << "Result set size is not equal to 3 but equal " << resultSets.size() << ". Please contact internal support"; - } - - { - TResultSetParser parser(resultSets[0]); - if (parser.TryNextRow()) { - if (!query.ParseFromString(*parser.ColumnParser(QUERY_COLUMN_NAME).GetOptionalString())) { - ythrow TControlPlaneStorageException(TIssuesIds::INTERNAL_ERROR) << "Error parsing proto message for query. Please contact internal support"; - } - if (!internal.ParseFromString(*parser.ColumnParser(INTERNAL_COLUMN_NAME).GetOptionalString())) { - ythrow TControlPlaneStorageException(TIssuesIds::INTERNAL_ERROR) << "Error parsing proto message for query internal. Please contact internal support"; - } - } - } - - { - TResultSetParser parser(resultSets[1]); - if (parser.TryNextRow()) { - if (!job.ParseFromString(*parser.ColumnParser(JOB_COLUMN_NAME).GetOptionalString())) { - ythrow TControlPlaneStorageException(TIssuesIds::INTERNAL_ERROR) << "Error parsing proto message for job. Please contact internal support"; - } - jobId = *parser.ColumnParser(JOB_ID_COLUMN_NAME).GetOptionalString(); - } - } - - { - TResultSetParser parser(resultSets[2]); - if (parser.TryNextRow()) { - selectedOwner = *parser.ColumnParser(OWNER_COLUMN_NAME).GetOptionalString(); - } - } - - if (selectedOwner != request->Owner) { - ythrow TControlPlaneStorageException(TIssuesIds::BAD_REQUEST) << "Query with the specified Owner: \"" << request->Owner << "\" does not exist. Selected owner: \"" << selectedOwner << "\""; - } - - if (request->Status) { - query.mutable_meta()->set_status(*request->Status); - job.mutable_query_meta()->set_status(*request->Status); - } - - if (request->Issues) { - NYql::IssuesToMessage(*request->Issues, query.mutable_issue()); - NYql::IssuesToMessage(*request->Issues, job.mutable_issue()); - } - - if (request->TransientIssues) { - NYql::TIssues issues = *request->TransientIssues; - for (const auto& issue: *query.mutable_transient_issue()) { - issues.AddIssue(NYql::IssueFromMessage(issue)); - } - - NYql::TIssues newIssues; - std::for_each_n(issues.begin(), std::min(static_cast<unsigned long long>(issues.Size()), 20ULL), [&](auto& issue){ newIssues.AddIssue(issue); }); - - NYql::IssuesToMessage(newIssues, query.mutable_transient_issue()); - } - - if (request->Statistics) { - *query.mutable_statistics()->mutable_json() = *request->Statistics; - *job.mutable_statistics()->mutable_json() = *request->Statistics; - } - - if (request->ResultSetMetas) { - // we will overwrite result_set_meta's COMPLETELY - query.clear_result_set_meta(); - job.clear_result_set_meta(); - for (const auto& resultSetMeta : *request->ResultSetMetas) { - *query.add_result_set_meta() = resultSetMeta; - *job.add_result_set_meta() = resultSetMeta; - } - } - - if (request->Ast) { - query.mutable_ast()->set_data(*request->Ast); - job.mutable_ast()->set_data(*request->Ast); - } - - if (request->Plan) { - query.mutable_plan()->set_json(*request->Plan); - job.mutable_plan()->set_json(*request->Plan); - } - - if (request->StartedAt) { + + auto prepareParams = [=](const TVector<TResultSet>& resultSets) { + TString jobId; + YandexQuery::Query query; + YandexQuery::Internal::QueryInternal internal; + YandexQuery::Job job; + TString selectedOwner; + + if (resultSets.size() != 3) { + ythrow TControlPlaneStorageException(TIssuesIds::INTERNAL_ERROR) << "Result set size is not equal to 3 but equal " << resultSets.size() << ". Please contact internal support"; + } + + { + TResultSetParser parser(resultSets[0]); + if (parser.TryNextRow()) { + if (!query.ParseFromString(*parser.ColumnParser(QUERY_COLUMN_NAME).GetOptionalString())) { + ythrow TControlPlaneStorageException(TIssuesIds::INTERNAL_ERROR) << "Error parsing proto message for query. Please contact internal support"; + } + if (!internal.ParseFromString(*parser.ColumnParser(INTERNAL_COLUMN_NAME).GetOptionalString())) { + ythrow TControlPlaneStorageException(TIssuesIds::INTERNAL_ERROR) << "Error parsing proto message for query internal. Please contact internal support"; + } + } + } + + { + TResultSetParser parser(resultSets[1]); + if (parser.TryNextRow()) { + if (!job.ParseFromString(*parser.ColumnParser(JOB_COLUMN_NAME).GetOptionalString())) { + ythrow TControlPlaneStorageException(TIssuesIds::INTERNAL_ERROR) << "Error parsing proto message for job. Please contact internal support"; + } + jobId = *parser.ColumnParser(JOB_ID_COLUMN_NAME).GetOptionalString(); + } + } + + { + TResultSetParser parser(resultSets[2]); + if (parser.TryNextRow()) { + selectedOwner = *parser.ColumnParser(OWNER_COLUMN_NAME).GetOptionalString(); + } + } + + if (selectedOwner != request->Owner) { + ythrow TControlPlaneStorageException(TIssuesIds::BAD_REQUEST) << "Query with the specified Owner: \"" << request->Owner << "\" does not exist. Selected owner: \"" << selectedOwner << "\""; + } + + if (request->Status) { + query.mutable_meta()->set_status(*request->Status); + job.mutable_query_meta()->set_status(*request->Status); + } + + if (request->Issues) { + NYql::IssuesToMessage(*request->Issues, query.mutable_issue()); + NYql::IssuesToMessage(*request->Issues, job.mutable_issue()); + } + + if (request->TransientIssues) { + NYql::TIssues issues = *request->TransientIssues; + for (const auto& issue: *query.mutable_transient_issue()) { + issues.AddIssue(NYql::IssueFromMessage(issue)); + } + + NYql::TIssues newIssues; + std::for_each_n(issues.begin(), std::min(static_cast<unsigned long long>(issues.Size()), 20ULL), [&](auto& issue){ newIssues.AddIssue(issue); }); + + NYql::IssuesToMessage(newIssues, query.mutable_transient_issue()); + } + + if (request->Statistics) { + *query.mutable_statistics()->mutable_json() = *request->Statistics; + *job.mutable_statistics()->mutable_json() = *request->Statistics; + } + + if (request->ResultSetMetas) { + // we will overwrite result_set_meta's COMPLETELY + query.clear_result_set_meta(); + job.clear_result_set_meta(); + for (const auto& resultSetMeta : *request->ResultSetMetas) { + *query.add_result_set_meta() = resultSetMeta; + *job.add_result_set_meta() = resultSetMeta; + } + } + + if (request->Ast) { + query.mutable_ast()->set_data(*request->Ast); + job.mutable_ast()->set_data(*request->Ast); + } + + if (request->Plan) { + query.mutable_plan()->set_json(*request->Plan); + job.mutable_plan()->set_json(*request->Plan); + } + + if (request->StartedAt) { *query.mutable_meta()->mutable_started_at() = NProtoInterop::CastToProto(*request->StartedAt); *job.mutable_query_meta()->mutable_started_at() = NProtoInterop::CastToProto(*request->StartedAt); - } - - if (request->FinishedAt) { + } + + if (request->FinishedAt) { *query.mutable_meta()->mutable_finished_at() = NProtoInterop::CastToProto(*request->FinishedAt); *job.mutable_query_meta()->mutable_finished_at() = NProtoInterop::CastToProto(*request->FinishedAt); - if (!query.meta().has_started_at()) { + if (!query.meta().has_started_at()) { *query.mutable_meta()->mutable_started_at() = NProtoInterop::CastToProto(*request->FinishedAt); *job.mutable_query_meta()->mutable_started_at() = NProtoInterop::CastToProto(*request->FinishedAt); - } - } - + } + } + TInstant expireAt = TInstant::Now() + automaticQueriesTtl; if (IsTerminalStatus(query.meta().status()) && query.content().automatic()) { *query.mutable_meta()->mutable_expire_at() = NProtoInterop::CastToProto(expireAt); @@ -168,42 +168,42 @@ std::tuple<TString, TParams, const std::function<std::pair<TString, NYdb::TParam internal.clear_dq_graph_index(); } - if (!request->CreatedTopicConsumers.empty()) { - std::set<Yq::Private::TopicConsumer, TTopicConsumerLess> mergedConsumers; - for (auto&& c : *internal.mutable_created_topic_consumers()) { - mergedConsumers.emplace(std::move(c)); - } - - for (const auto& c : request->CreatedTopicConsumers) { - Yq::Private::TopicConsumer proto; - proto.set_database_id(c.DatabaseId); - proto.set_database(c.Database); - proto.set_topic_path(c.TopicPath); - proto.set_consumer_name(c.ConsumerName); - proto.set_cluster_endpoint(c.ClusterEndpoint); - proto.set_use_ssl(c.UseSsl); - proto.set_token_name(c.TokenName); - proto.set_add_bearer_to_token(c.AddBearerToToken); - mergedConsumers.emplace(std::move(proto)); - } - internal.clear_created_topic_consumers(); - for (auto&& c : mergedConsumers) { - *internal.add_created_topic_consumers() = std::move(c); - } - } - - if (!request->DqGraphs.empty()) { - internal.clear_dq_graph(); - for (const auto& g : request->DqGraphs) { - internal.add_dq_graph(g); - } - } - - if (request->DqGraphIndex) { - internal.set_dq_graph_index(request->DqGraphIndex); - } - - TSqlQueryBuilder writeQueryBuilder(tablePathPrefix, "HardPingTask(write)"); + if (!request->CreatedTopicConsumers.empty()) { + std::set<Yq::Private::TopicConsumer, TTopicConsumerLess> mergedConsumers; + for (auto&& c : *internal.mutable_created_topic_consumers()) { + mergedConsumers.emplace(std::move(c)); + } + + for (const auto& c : request->CreatedTopicConsumers) { + Yq::Private::TopicConsumer proto; + proto.set_database_id(c.DatabaseId); + proto.set_database(c.Database); + proto.set_topic_path(c.TopicPath); + proto.set_consumer_name(c.ConsumerName); + proto.set_cluster_endpoint(c.ClusterEndpoint); + proto.set_use_ssl(c.UseSsl); + proto.set_token_name(c.TokenName); + proto.set_add_bearer_to_token(c.AddBearerToToken); + mergedConsumers.emplace(std::move(proto)); + } + internal.clear_created_topic_consumers(); + for (auto&& c : mergedConsumers) { + *internal.add_created_topic_consumers() = std::move(c); + } + } + + if (!request->DqGraphs.empty()) { + internal.clear_dq_graph(); + for (const auto& g : request->DqGraphs) { + internal.add_dq_graph(g); + } + } + + if (request->DqGraphIndex) { + internal.set_dq_graph_index(request->DqGraphIndex); + } + + TSqlQueryBuilder writeQueryBuilder(tablePathPrefix, "HardPingTask(write)"); writeQueryBuilder.AddString("scope", request->Scope); writeQueryBuilder.AddString("job_id", jobId); writeQueryBuilder.AddString("job", job.SerializeAsString()); @@ -212,7 +212,7 @@ std::tuple<TString, TParams, const std::function<std::pair<TString, NYdb::TParam writeQueryBuilder.AddString("internal", internal.SerializeAsString()); writeQueryBuilder.AddString("result_id", request->ResultId); writeQueryBuilder.AddString("query_id", request->QueryId); - + if (IsTerminalStatus(query.meta().status())) { // delete pending writeQueryBuilder.AddText( @@ -220,7 +220,7 @@ std::tuple<TString, TParams, const std::function<std::pair<TString, NYdb::TParam "WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id;\n" ); } else { - // update pending small + // update pending small writeQueryBuilder.AddTimestamp("now", request->ResignQuery ? TInstant::Zero() : TInstant::Now()); const TString updateResignQueryFlag = request->ResignQuery ? ", `" IS_RESIGN_QUERY_COLUMN_NAME "` = true" : ""; writeQueryBuilder.AddText( @@ -228,7 +228,7 @@ std::tuple<TString, TParams, const std::function<std::pair<TString, NYdb::TParam "WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id;\n" ); } - + if (IsTerminalStatus(query.meta().status()) && query.content().automatic()) { // set jobs ttl writeQueryBuilder.AddText( @@ -242,7 +242,7 @@ std::tuple<TString, TParams, const std::function<std::pair<TString, NYdb::TParam "WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id AND `" EXPIRE_AT_COLUMN_NAME "` IS NOT NULL;\n" ); } - + TString updateResultSetsExpire; if (query.meta().status() == YandexQuery::QueryMeta::COMPLETED) { writeQueryBuilder.AddTimestamp("result_sets_expire_at", request->Deadline); @@ -250,7 +250,7 @@ std::tuple<TString, TParams, const std::function<std::pair<TString, NYdb::TParam } else { updateResultSetsExpire = "`" RESULT_SETS_EXPIRE_AT_COLUMN_NAME "` = NULL"; } - + TString updateQueryTtl; if (IsTerminalStatus(query.meta().status()) && query.content().automatic()) { writeQueryBuilder.AddTimestamp("expire_at", expireAt); @@ -260,134 +260,134 @@ std::tuple<TString, TParams, const std::function<std::pair<TString, NYdb::TParam } writeQueryBuilder.AddText( - "UPSERT INTO `" JOBS_TABLE_NAME "` (`" SCOPE_COLUMN_NAME "`, `" QUERY_ID_COLUMN_NAME "`, `" JOB_ID_COLUMN_NAME "`, `" JOB_COLUMN_NAME "`) VALUES($scope, $query_id, $job_id, $job);\n" + "UPSERT INTO `" JOBS_TABLE_NAME "` (`" SCOPE_COLUMN_NAME "`, `" QUERY_ID_COLUMN_NAME "`, `" JOB_ID_COLUMN_NAME "`, `" JOB_COLUMN_NAME "`) VALUES($scope, $query_id, $job_id, $job);\n" "UPDATE `" QUERIES_TABLE_NAME "` SET `" QUERY_COLUMN_NAME "` = $query, `" STATUS_COLUMN_NAME "` = $status, `" INTERNAL_COLUMN_NAME "` = $internal, `" RESULT_ID_COLUMN_NAME "` = $result_id, " + updateResultSetsExpire + ", " + updateQueryTtl + ", `" META_REVISION_COLUMN_NAME "` = `" META_REVISION_COLUMN_NAME "` + 1\n" - "WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id;\n" + "WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id;\n" ); - - *response = internal.action(); + + *response = internal.action(); const auto writeQuery = writeQueryBuilder.Build(); return std::make_pair(writeQuery.Sql, writeQuery.Params); - }; + }; const auto readQuery = readQueryBuilder.Build(); return std::make_tuple(readQuery.Sql, readQuery.Params, prepareParams); -} - -std::tuple<TString, TParams, const std::function<std::pair<TString, NYdb::TParams>(const TVector<NYdb::TResultSet>&)>> ConstructSoftPingTask( - const TEvControlPlaneStorage::TEvPingTaskRequest* request, std::shared_ptr<YandexQuery::QueryAction> response, - const TString& tablePathPrefix) { - TSqlQueryBuilder readQueryBuilder(tablePathPrefix, "SoftPingTask(read)"); +} + +std::tuple<TString, TParams, const std::function<std::pair<TString, NYdb::TParams>(const TVector<NYdb::TResultSet>&)>> ConstructSoftPingTask( + const TEvControlPlaneStorage::TEvPingTaskRequest* request, std::shared_ptr<YandexQuery::QueryAction> response, + const TString& tablePathPrefix) { + TSqlQueryBuilder readQueryBuilder(tablePathPrefix, "SoftPingTask(read)"); readQueryBuilder.AddString("scope", request->Scope); readQueryBuilder.AddString("query_id", request->QueryId); readQueryBuilder.AddText( - "SELECT `" INTERNAL_COLUMN_NAME "`\n" - "FROM `" QUERIES_TABLE_NAME "` WHERE `" QUERY_ID_COLUMN_NAME "` = $query_id AND `" SCOPE_COLUMN_NAME "` = $scope;\n" - "SELECT `" OWNER_COLUMN_NAME "`\n" - "FROM `" PENDING_SMALL_TABLE_NAME "` WHERE `" QUERY_ID_COLUMN_NAME "` = $query_id AND `" SCOPE_COLUMN_NAME "` = $scope;\n" + "SELECT `" INTERNAL_COLUMN_NAME "`\n" + "FROM `" QUERIES_TABLE_NAME "` WHERE `" QUERY_ID_COLUMN_NAME "` = $query_id AND `" SCOPE_COLUMN_NAME "` = $scope;\n" + "SELECT `" OWNER_COLUMN_NAME "`\n" + "FROM `" PENDING_SMALL_TABLE_NAME "` WHERE `" QUERY_ID_COLUMN_NAME "` = $query_id AND `" SCOPE_COLUMN_NAME "` = $scope;\n" ); - - auto prepareParams = [=](const TVector<TResultSet>& resultSets) { - TString selectedOwner; - YandexQuery::Internal::QueryInternal internal; - - if (resultSets.size() != 2) { - ythrow TControlPlaneStorageException(TIssuesIds::INTERNAL_ERROR) << "Result set size is not equal to 2 but equal " << resultSets.size() << ". Please contact internal support"; - } - - { - TResultSetParser parser(resultSets[0]); - if (parser.TryNextRow()) { - if (!internal.ParseFromString(*parser.ColumnParser(INTERNAL_COLUMN_NAME).GetOptionalString())) { - ythrow TControlPlaneStorageException(TIssuesIds::INTERNAL_ERROR) << "Error parsing proto message for pending internal query. Please contact internal support"; - } - } - } - - { - TResultSetParser parser(resultSets[1]); - if (parser.TryNextRow()) { - selectedOwner = *parser.ColumnParser(OWNER_COLUMN_NAME).GetOptionalString(); - } - } - *response = internal.action(); - - if (selectedOwner != request->Owner) { - ythrow TControlPlaneStorageException(TIssuesIds::BAD_REQUEST) << "query with the specified Owner: \"" << request->Owner << "\" does not exist. Selected owner: \"" << selectedOwner << "\""; - } - - TSqlQueryBuilder writeQueryBuilder(tablePathPrefix, "SoftPingTask(write)"); + + auto prepareParams = [=](const TVector<TResultSet>& resultSets) { + TString selectedOwner; + YandexQuery::Internal::QueryInternal internal; + + if (resultSets.size() != 2) { + ythrow TControlPlaneStorageException(TIssuesIds::INTERNAL_ERROR) << "Result set size is not equal to 2 but equal " << resultSets.size() << ". Please contact internal support"; + } + + { + TResultSetParser parser(resultSets[0]); + if (parser.TryNextRow()) { + if (!internal.ParseFromString(*parser.ColumnParser(INTERNAL_COLUMN_NAME).GetOptionalString())) { + ythrow TControlPlaneStorageException(TIssuesIds::INTERNAL_ERROR) << "Error parsing proto message for pending internal query. Please contact internal support"; + } + } + } + + { + TResultSetParser parser(resultSets[1]); + if (parser.TryNextRow()) { + selectedOwner = *parser.ColumnParser(OWNER_COLUMN_NAME).GetOptionalString(); + } + } + *response = internal.action(); + + if (selectedOwner != request->Owner) { + ythrow TControlPlaneStorageException(TIssuesIds::BAD_REQUEST) << "query with the specified Owner: \"" << request->Owner << "\" does not exist. Selected owner: \"" << selectedOwner << "\""; + } + + TSqlQueryBuilder writeQueryBuilder(tablePathPrefix, "SoftPingTask(write)"); writeQueryBuilder.AddTimestamp("now", request->ResignQuery ? TInstant::Zero() : TInstant::Now()); writeQueryBuilder.AddString("scope", request->Scope); writeQueryBuilder.AddString("query_id", request->QueryId); writeQueryBuilder.AddString("owner", request->Owner); - - const TString updateResignQueryFlag = request->ResignQuery ? ", `" IS_RESIGN_QUERY_COLUMN_NAME "` = true" : ""; + + const TString updateResignQueryFlag = request->ResignQuery ? ", `" IS_RESIGN_QUERY_COLUMN_NAME "` = true" : ""; writeQueryBuilder.AddText( "UPDATE `" PENDING_SMALL_TABLE_NAME "` SET `" LAST_SEEN_AT_COLUMN_NAME "` = $now " + updateResignQueryFlag + "\n" - "WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id;\n" - ); - + "WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id;\n" + ); + const auto writeQuery = writeQueryBuilder.Build(); return std::make_pair(writeQuery.Sql, writeQuery.Params); - }; + }; const auto readQuery = readQueryBuilder.Build(); return std::make_tuple(readQuery.Sql, readQuery.Params, prepareParams); -} - -void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvPingTaskRequest::TPtr& ev) -{ - TInstant startTime = TInstant::Now(); - TRequestCountersPtr requestCounters = Counters.Requests[RT_PING_TASK]; - requestCounters->InFly->Inc(); - - TEvControlPlaneStorage::TEvPingTaskRequest* request = ev->Get(); - const TString scope = request->Scope; - const TString queryId = request->QueryId; - const TString owner = request->Owner; +} + +void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvPingTaskRequest::TPtr& ev) +{ + TInstant startTime = TInstant::Now(); + TRequestCountersPtr requestCounters = Counters.Requests[RT_PING_TASK]; + requestCounters->InFly->Inc(); + + TEvControlPlaneStorage::TEvPingTaskRequest* request = ev->Get(); + const TString scope = request->Scope; + const TString queryId = request->QueryId; + const TString owner = request->Owner; const TInstant deadline = request->Deadline; - - CPS_LOG_T("PingTaskRequest: " << scope << " " << queryId - << " " << owner << " " << deadline << " " - << (request->Status ? YandexQuery::QueryMeta_ComputeStatus_Name(*request->Status) : "no status")); - + + CPS_LOG_T("PingTaskRequest: " << scope << " " << queryId + << " " << owner << " " << deadline << " " + << (request->Status ? YandexQuery::QueryMeta_ComputeStatus_Name(*request->Status) : "no status")); + NYql::TIssues issues = ValidatePingTask(scope, queryId, owner, deadline, Config.ResultSetsTtl); - if (issues) { + if (issues) { CPS_LOG_D("PingTaskRequest, validation failed: " << scope << " " << queryId << " " << owner << " " << deadline << issues.ToString()); - const TDuration delta = TInstant::Now() - startTime; - SendResponseIssues<TEvControlPlaneStorage::TEvPingTaskResponse>(ev->Sender, issues, ev->Cookie, delta, requestCounters); - LWPROBE(PingTaskRequest, queryId, delta, false); - return; - } - - std::shared_ptr<YandexQuery::QueryAction> response = std::make_shared<YandexQuery::QueryAction>(); - - if (request->Status) - FinalStatusCounters.IncByStatus(*request->Status); - auto pingTaskParams = DoesPingTaskUpdateQueriesTable(request) ? + const TDuration delta = TInstant::Now() - startTime; + SendResponseIssues<TEvControlPlaneStorage::TEvPingTaskResponse>(ev->Sender, issues, ev->Cookie, delta, requestCounters); + LWPROBE(PingTaskRequest, queryId, delta, false); + return; + } + + std::shared_ptr<YandexQuery::QueryAction> response = std::make_shared<YandexQuery::QueryAction>(); + + if (request->Status) + FinalStatusCounters.IncByStatus(*request->Status); + auto pingTaskParams = DoesPingTaskUpdateQueriesTable(request) ? ConstructHardPingTask(request, response, YdbConnection->TablePathPrefix, Config.AutomaticQueriesTtl) : - ConstructSoftPingTask(request, response, YdbConnection->TablePathPrefix); - auto readQuery = std::get<0>(pingTaskParams); // Use std::get for win compiler - auto readParams = std::get<1>(pingTaskParams); - auto prepareParams = std::get<2>(pingTaskParams); - + ConstructSoftPingTask(request, response, YdbConnection->TablePathPrefix); + auto readQuery = std::get<0>(pingTaskParams); // Use std::get for win compiler + auto readParams = std::get<1>(pingTaskParams); + auto prepareParams = std::get<2>(pingTaskParams); + auto debugInfo = Config.Proto.GetEnableDebugMode() ? std::make_shared<TDebugInfo>() : TDebugInfoPtr{}; - auto result = ReadModifyWrite(NActors::TActivationContext::ActorSystem(), readQuery, readParams, prepareParams, requestCounters, debugInfo); - auto prepare = [response] { return std::make_tuple(*response); }; - auto success = SendResponseTuple<TEvControlPlaneStorage::TEvPingTaskResponse, std::tuple<YandexQuery::QueryAction>>( - "PingTaskRequest", - NActors::TActivationContext::ActorSystem(), - result, - SelfId(), - ev, - startTime, - requestCounters, - prepare, - debugInfo); - - success.Apply([=](const auto& future) { - TDuration delta = TInstant::Now() - startTime; - LWPROBE(PingTaskRequest, queryId, delta, future.GetValue()); - }); -} - -} // NYq + auto result = ReadModifyWrite(NActors::TActivationContext::ActorSystem(), readQuery, readParams, prepareParams, requestCounters, debugInfo); + auto prepare = [response] { return std::make_tuple(*response); }; + auto success = SendResponseTuple<TEvControlPlaneStorage::TEvPingTaskResponse, std::tuple<YandexQuery::QueryAction>>( + "PingTaskRequest", + NActors::TActivationContext::ActorSystem(), + result, + SelfId(), + ev, + startTime, + requestCounters, + prepare, + debugInfo); + + success.Apply([=](const auto& future) { + TDuration delta = TInstant::Now() - startTime; + LWPROBE(PingTaskRequest, queryId, delta, future.GetValue()); + }); +} + +} // NYq diff --git a/ydb/core/yq/libs/control_plane_storage/internal/task_result_write.cpp b/ydb/core/yq/libs/control_plane_storage/internal/task_result_write.cpp index f6c16f084f..db945dcca9 100644 --- a/ydb/core/yq/libs/control_plane_storage/internal/task_result_write.cpp +++ b/ydb/core/yq/libs/control_plane_storage/internal/task_result_write.cpp @@ -1,83 +1,83 @@ -#include "utils.h" - -namespace NYq { - -void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvWriteResultDataRequest::TPtr& ev) -{ - TInstant startTime = TInstant::Now(); - TRequestCountersPtr requestCounters = Counters.Requests[RT_WRITE_RESULT_DATA]; - requestCounters->InFly->Inc(); - - TEvControlPlaneStorage::TEvWriteResultDataRequest& request = *ev->Get(); - const TString resultId = request.ResultId; - const int32_t resultSetId = request.ResultSetId; - const int64_t startRowId = request.StartRowId; +#include "utils.h" + +namespace NYq { + +void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvWriteResultDataRequest::TPtr& ev) +{ + TInstant startTime = TInstant::Now(); + TRequestCountersPtr requestCounters = Counters.Requests[RT_WRITE_RESULT_DATA]; + requestCounters->InFly->Inc(); + + TEvControlPlaneStorage::TEvWriteResultDataRequest& request = *ev->Get(); + const TString resultId = request.ResultId; + const int32_t resultSetId = request.ResultSetId; + const int64_t startRowId = request.StartRowId; const TInstant deadline = request.Deadline; - const Ydb::ResultSet& resultSet = request.ResultSet; - const int byteSize = resultSet.ByteSize(); - + const Ydb::ResultSet& resultSet = request.ResultSet; + const int byteSize = resultSet.ByteSize(); + CPS_LOG_T("WriteResultDataRequest: " << resultId << " " << resultSetId << " " << startRowId << " " << resultSet.ByteSize() << " " << deadline); - + NYql::TIssues issues = ValidateWriteResultData(resultId, resultSet, deadline, Config.ResultSetsTtl); - if (issues) { + if (issues) { CPS_LOG_D("WriteResultDataRequest, validation failed: " << resultId << " " << resultSetId << " " << startRowId << " " << resultSet.DebugString() << " " << deadline << " error: " << issues.ToString()); - const TDuration delta = TInstant::Now() - startTime; - SendResponseIssues<TEvControlPlaneStorage::TEvWriteResultDataResponse>(ev->Sender, issues, ev->Cookie, delta, requestCounters); - LWPROBE(WriteResultDataRequest, resultId, resultSetId, startRowId, resultSet.rows().size(), delta, deadline, byteSize, false); - return; - } - - NYdb::TValueBuilder itemsAsList; - itemsAsList.BeginList(); - - int64_t rowId = startRowId; - for (const auto& row : resultSet.rows()) { - TString serializedRow; - if (!row.SerializeToString(&serializedRow)) { - ythrow TControlPlaneStorageException(TIssuesIds::INTERNAL_ERROR) << "Error serialize proto message for row. Please contact internal support"; - } - - itemsAsList.AddListItem() - .BeginStruct() - .AddMember("row_id").Int64(rowId) + const TDuration delta = TInstant::Now() - startTime; + SendResponseIssues<TEvControlPlaneStorage::TEvWriteResultDataResponse>(ev->Sender, issues, ev->Cookie, delta, requestCounters); + LWPROBE(WriteResultDataRequest, resultId, resultSetId, startRowId, resultSet.rows().size(), delta, deadline, byteSize, false); + return; + } + + NYdb::TValueBuilder itemsAsList; + itemsAsList.BeginList(); + + int64_t rowId = startRowId; + for (const auto& row : resultSet.rows()) { + TString serializedRow; + if (!row.SerializeToString(&serializedRow)) { + ythrow TControlPlaneStorageException(TIssuesIds::INTERNAL_ERROR) << "Error serialize proto message for row. Please contact internal support"; + } + + itemsAsList.AddListItem() + .BeginStruct() + .AddMember("row_id").Int64(rowId) .AddMember("result_set").String(row.SerializeAsString()) - .EndStruct(); - rowId++; - } - - itemsAsList.EndList(); - - TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "TaskResultWrite"); + .EndStruct(); + rowId++; + } + + itemsAsList.EndList(); + + TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "TaskResultWrite"); queryBuilder.AddString("result_id", resultId); queryBuilder.AddInt32("result_set_id", resultSetId); queryBuilder.AddTimestamp("expire_at", deadline); queryBuilder.AddValue("items", itemsAsList.Build()); - + queryBuilder.AddText( - "UPSERT INTO `" RESULT_SETS_TABLE_NAME "`\n" + "UPSERT INTO `" RESULT_SETS_TABLE_NAME "`\n" "SELECT $result_id as result_id, $result_set_id as result_set_id,\n" " T.*, $expire_at as expire_at FROM as_table($items) AS T;\n" ); - + const auto query = queryBuilder.Build(); auto debugInfo = Config.Proto.GetEnableDebugMode() ? std::make_shared<TDebugInfo>() : TDebugInfoPtr{}; TAsyncStatus result = Write(NActors::TActivationContext::ActorSystem(), query.Sql, query.Params, requestCounters, debugInfo); - auto prepare = []() { return std::make_tuple<NYql::TIssues>(NYql::TIssues{}); }; - auto success = SendResponseTuple<TEvControlPlaneStorage::TEvWriteResultDataResponse, std::tuple<NYql::TIssues>>( - "WriteResultDataRequest", - NActors::TActivationContext::ActorSystem(), - result, - SelfId(), - ev, - startTime, - requestCounters, - prepare, - debugInfo); - - success.Apply([=](const auto& future) { - TDuration delta = TInstant::Now() - startTime; - LWPROBE(WriteResultDataRequest, resultId, resultSetId, startRowId, resultSet.rows().size(), delta, deadline, byteSize, future.GetValue()); - }); -} - -} // NYq + auto prepare = []() { return std::make_tuple<NYql::TIssues>(NYql::TIssues{}); }; + auto success = SendResponseTuple<TEvControlPlaneStorage::TEvWriteResultDataResponse, std::tuple<NYql::TIssues>>( + "WriteResultDataRequest", + NActors::TActivationContext::ActorSystem(), + result, + SelfId(), + ev, + startTime, + requestCounters, + prepare, + debugInfo); + + success.Apply([=](const auto& future) { + TDuration delta = TInstant::Now() - startTime; + LWPROBE(WriteResultDataRequest, resultId, resultSetId, startRowId, resultSet.rows().size(), delta, deadline, byteSize, future.GetValue()); + }); +} + +} // NYq diff --git a/ydb/core/yq/libs/control_plane_storage/internal/utils.cpp b/ydb/core/yq/libs/control_plane_storage/internal/utils.cpp index 8e3a78deb0..bca91f1cfb 100644 --- a/ydb/core/yq/libs/control_plane_storage/internal/utils.cpp +++ b/ydb/core/yq/libs/control_plane_storage/internal/utils.cpp @@ -1,83 +1,83 @@ -#include "utils.h" - -namespace NYq { - +#include "utils.h" + +namespace NYq { + NYql::TIssues ValidateWriteResultData(const TString& resultId, const Ydb::ResultSet& resultSet, const TInstant& deadline, const TDuration& ttl) -{ - NYql::TIssues issues; - if (!resultId) { - issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "result id is not specified")); - } - - if (resultSet.rows().size() == 0) { - issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "result set rows count is empty")); - } - +{ + NYql::TIssues issues; + if (!resultId) { + issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "result id is not specified")); + } + + if (resultSet.rows().size() == 0) { + issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "result set rows count is empty")); + } + const auto hardLimit = TInstant::Now() + ttl; if (deadline > hardLimit) { issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "deadline " + deadline.ToString() + " must be less than " + hardLimit.ToString())); } - return issues; -} - -NYql::TIssues ValidateGetTask(const TString& owner, const TString& hostName) -{ - NYql::TIssues issues; - if (!owner) { - issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "owner is not specified")); - } - - if (!hostName) { - issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "host name is not specified")); - } - - return issues; -} - + return issues; +} + +NYql::TIssues ValidateGetTask(const TString& owner, const TString& hostName) +{ + NYql::TIssues issues; + if (!owner) { + issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "owner is not specified")); + } + + if (!hostName) { + issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "host name is not specified")); + } + + return issues; +} + NYql::TIssues ValidatePingTask(const TString& scope, const TString& queryId, const TString& owner, const TInstant& deadline, const TDuration& ttl) -{ - NYql::TIssues issues; - if (!scope) { - issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "scope is not specified")); - } - - if (!queryId) { - issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "query id is not specified")); - } - - if (!owner) { - issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "owner is not specified")); - } - +{ + NYql::TIssues issues; + if (!scope) { + issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "scope is not specified")); + } + + if (!queryId) { + issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "query id is not specified")); + } + + if (!owner) { + issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "owner is not specified")); + } + const auto hardLimit = TInstant::Now() + ttl; if (deadline > hardLimit) { issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "deadline " + deadline.ToString() + " must be less than " + hardLimit.ToString())); } - return issues; -} - -NYql::TIssues ValidateNodesHealthCheck( - const TString& tenant, - const TString& instanceId, - const TString& hostName - ) -{ - NYql::TIssues issues; - if (!tenant) { - issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "tenant is not specified")); - } - - if (!instanceId) { - issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "instance id is not specified")); - } - - if (!hostName) { - issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "hostName is not specified")); - } - - return issues; -} - -};
\ No newline at end of file + return issues; +} + +NYql::TIssues ValidateNodesHealthCheck( + const TString& tenant, + const TString& instanceId, + const TString& hostName + ) +{ + NYql::TIssues issues; + if (!tenant) { + issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "tenant is not specified")); + } + + if (!instanceId) { + issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "instance id is not specified")); + } + + if (!hostName) { + issues.AddIssue(MakeErrorIssue(TIssuesIds::BAD_REQUEST, "hostName is not specified")); + } + + return issues; +} + +};
\ No newline at end of file diff --git a/ydb/core/yq/libs/control_plane_storage/internal/utils.h b/ydb/core/yq/libs/control_plane_storage/internal/utils.h index 58ffe6b340..2fd0f708d1 100644 --- a/ydb/core/yq/libs/control_plane_storage/internal/utils.h +++ b/ydb/core/yq/libs/control_plane_storage/internal/utils.h @@ -1,5 +1,5 @@ -#pragma once - +#pragma once + #include <tuple> #include <ydb/public/sdk/cpp/client/ydb_value/value.h> @@ -8,25 +8,25 @@ #include <ydb/core/yq/libs/config/protos/issue_id.pb.h> #include <ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage_impl.h> - -namespace NYq { - -struct TTopicConsumerLess { - bool operator()(const Yq::Private::TopicConsumer& c1, const Yq::Private::TopicConsumer& c2) const { - // Cluster endpoint/use ssl are not in key - return std::tie(c1.database_id(), c1.database(), c1.topic_path(), c1.consumer_name()) < std::tie(c2.database_id(), c2.database(), c2.topic_path(), c2.consumer_name()); - } -}; - + +namespace NYq { + +struct TTopicConsumerLess { + bool operator()(const Yq::Private::TopicConsumer& c1, const Yq::Private::TopicConsumer& c2) const { + // Cluster endpoint/use ssl are not in key + return std::tie(c1.database_id(), c1.database(), c1.topic_path(), c1.consumer_name()) < std::tie(c2.database_id(), c2.database(), c2.topic_path(), c2.consumer_name()); + } +}; + NYql::TIssues ValidateWriteResultData(const TString& resultId, const Ydb::ResultSet& resultSet, const TInstant& deadline, const TDuration& ttl); - -NYql::TIssues ValidateGetTask(const TString& owner, const TString& hostName); - + +NYql::TIssues ValidateGetTask(const TString& owner, const TString& hostName); + NYql::TIssues ValidatePingTask(const TString& scope, const TString& queryId, const TString& owner, const TInstant& deadline, const TDuration& ttl); - -NYql::TIssues ValidateNodesHealthCheck( - const TString& tenant, - const TString& instanceId, - const TString& hostName); - + +NYql::TIssues ValidateNodesHealthCheck( + const TString& tenant, + const TString& instanceId, + const TString& hostName); + }; diff --git a/ydb/core/yq/libs/control_plane_storage/internal/ya.make b/ydb/core/yq/libs/control_plane_storage/internal/ya.make index 2e268259eb..f38b3b070c 100644 --- a/ydb/core/yq/libs/control_plane_storage/internal/ya.make +++ b/ydb/core/yq/libs/control_plane_storage/internal/ya.make @@ -1,20 +1,20 @@ OWNER(g:yq) - -LIBRARY() - -SRCS( - nodes_health_check.cpp - response_tasks.cpp - task_get.cpp - task_ping.cpp - task_result_write.cpp - utils.cpp -) - -PEERDIR( - library/cpp/actors/core - library/cpp/lwtrace/mon - library/cpp/monlib/service/pages + +LIBRARY() + +SRCS( + nodes_health_check.cpp + response_tasks.cpp + task_get.cpp + task_ping.cpp + task_result_write.cpp + utils.cpp +) + +PEERDIR( + library/cpp/actors/core + library/cpp/lwtrace/mon + library/cpp/monlib/service/pages ydb/core/base ydb/core/mon ydb/core/yq/libs/common @@ -25,8 +25,8 @@ PEERDIR( ydb/public/sdk/cpp/client/ydb_scheme ydb/public/sdk/cpp/client/ydb_value ydb/library/yql/public/issue -) - -YQL_LAST_ABI_VERSION() - -END() +) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/ydb/core/yq/libs/control_plane_storage/message_builders.h b/ydb/core/yq/libs/control_plane_storage/message_builders.h index d7ebdb7098..75ee6f1f9f 100644 --- a/ydb/core/yq/libs/control_plane_storage/message_builders.h +++ b/ydb/core/yq/libs/control_plane_storage/message_builders.h @@ -151,7 +151,7 @@ public: Request.set_query_id(queryId); return *this; } - + TDeleteQueryBuilder& SetIdempotencyKey(const TString& idempotencyKey) { Request.set_idempotency_key(idempotencyKey); @@ -249,12 +249,12 @@ public: return *this; } - TModifyQueryBuilder& SetDescription(const TString& description) - { - Request.mutable_content()->set_description(description); - return *this; - } - + TModifyQueryBuilder& SetDescription(const TString& description) + { + Request.mutable_content()->set_description(description); + return *this; + } + const YandexQuery::ModifyQueryRequest& Build() { return Request; @@ -281,7 +281,7 @@ public: Request.set_query_id(queryId); return *this; } - + TControlQueryBuilder& SetIdempotencyKey(const TString& idempotencyKey) { Request.set_idempotency_key(idempotencyKey); @@ -625,7 +625,7 @@ public: TModifyConnectionBuilder& SetDescription(const TString& description) { - Request.mutable_content()->set_description(description); + Request.mutable_content()->set_description(description); return *this; } @@ -662,7 +662,7 @@ public: Request.set_connection_id(connectionId); return *this; } - + TDeleteConnectionBuilder& SetIdempotencyKey(const TString& idempotencyKey) { Request.set_idempotency_key(idempotencyKey); @@ -885,7 +885,7 @@ public: Request.set_binding_id(bindingId); return *this; } - + TDeleteBindingBuilder& SetIdempotencyKey(const TString& idempotencyKey) { Request.set_idempotency_key(idempotencyKey); @@ -907,7 +907,7 @@ public: // internal class TWriteResultDataBuilder { - TString ResultId; + TString ResultId; int32_t ResultSetId = 0; int64_t StartRowId = 0; TInstant Deadline; @@ -923,9 +923,9 @@ public: SetResultSet(resultSet); } - TWriteResultDataBuilder& SetResultId(const TString& resultId) + TWriteResultDataBuilder& SetResultId(const TString& resultId) { - ResultId = resultId; + ResultId = resultId; return *this; } @@ -955,7 +955,7 @@ public: std::unique_ptr<TEvControlPlaneStorage::TEvWriteResultDataRequest> Build() { - return std::make_unique<TEvControlPlaneStorage::TEvWriteResultDataRequest>(ResultId, ResultSetId, StartRowId, Deadline, ResultSet); + return std::make_unique<TEvControlPlaneStorage::TEvWriteResultDataRequest>(ResultId, ResultSetId, StartRowId, Deadline, ResultSet); } }; @@ -984,15 +984,15 @@ public: std::unique_ptr<TEvControlPlaneStorage::TEvGetTaskRequest> Build() { - return std::make_unique<TEvControlPlaneStorage::TEvGetTaskRequest>(Owner, HostName); + return std::make_unique<TEvControlPlaneStorage::TEvGetTaskRequest>(Owner, HostName); } }; class TPingTaskBuilder { TString Scope; TString QueryId; - TString ResultId; - TString Owner; + TString ResultId; + TString Owner; TInstant Deadline; TMaybe<YandexQuery::QueryMeta::ComputeStatus> Status; TMaybe<NYql::TIssues> Issues; @@ -1026,15 +1026,15 @@ public: return *this; } - TPingTaskBuilder& SetResultId(const TString& resultId) - { - ResultId = resultId; - return *this; - } - - TPingTaskBuilder& SetOwner(const TString& owner) + TPingTaskBuilder& SetResultId(const TString& resultId) { - Owner = owner; + ResultId = resultId; + return *this; + } + + TPingTaskBuilder& SetOwner(const TString& owner) + { + Owner = owner; return *this; } @@ -1142,31 +1142,31 @@ public: } }; -class TNodesHealthCheckBuilder { - TString Tenant; +class TNodesHealthCheckBuilder { + TString Tenant; ui32 NodeId = 0; - TString HostName; - TString InstanceId; + TString HostName; + TString InstanceId; ui64 ActiveWorkers = 0; ui64 MemoryLimit = 0; ui64 MemoryAllocated = 0; -public: - TNodesHealthCheckBuilder() - {} - - TNodesHealthCheckBuilder& SetTenant(const TString& tenant) - { - Tenant = tenant; - return *this; - } - - TNodesHealthCheckBuilder& SetNodeId(const ui32& nodeId) - { - NodeId = nodeId; - return *this; - } - +public: + TNodesHealthCheckBuilder() + {} + + TNodesHealthCheckBuilder& SetTenant(const TString& tenant) + { + Tenant = tenant; + return *this; + } + + TNodesHealthCheckBuilder& SetNodeId(const ui32& nodeId) + { + NodeId = nodeId; + return *this; + } + TNodesHealthCheckBuilder& SetHostName(const TString& hostName) { HostName = hostName; @@ -1179,28 +1179,28 @@ public: return *this; } - TNodesHealthCheckBuilder& SetActiveWorkers(const ui64& activeWorkers) - { - ActiveWorkers = activeWorkers; - return *this; - } - + TNodesHealthCheckBuilder& SetActiveWorkers(const ui64& activeWorkers) + { + ActiveWorkers = activeWorkers; + return *this; + } + TNodesHealthCheckBuilder& SetMemoryLimit(const ui64& memoryLimit) - { + { MemoryLimit = memoryLimit; - return *this; - } - + return *this; + } + TNodesHealthCheckBuilder& SetMemoryAllocated(const ui64& memoryAllocated) - { + { MemoryAllocated = memoryAllocated; - return *this; - } - - std::unique_ptr<TEvControlPlaneStorage::TEvNodesHealthCheckRequest> Build() - { - Yq::Private::NodesHealthCheckRequest request; - request.set_tenant(Tenant); + return *this; + } + + std::unique_ptr<TEvControlPlaneStorage::TEvNodesHealthCheckRequest> Build() + { + Yq::Private::NodesHealthCheckRequest request; + request.set_tenant(Tenant); auto& node = *request.mutable_node(); node.set_node_id(NodeId); node.set_instance_id(InstanceId); @@ -1208,8 +1208,8 @@ public: node.set_active_workers(ActiveWorkers); node.set_memory_limit(MemoryLimit); node.set_memory_allocated(MemoryAllocated); - return std::make_unique<TEvControlPlaneStorage::TEvNodesHealthCheckRequest>(std::move(request)); - } -}; - + return std::make_unique<TEvControlPlaneStorage::TEvNodesHealthCheckRequest>(std::move(request)); + } +}; + } diff --git a/ydb/core/yq/libs/control_plane_storage/probes.h b/ydb/core/yq/libs/control_plane_storage/probes.h index e4322110cc..c82c68e5ef 100644 --- a/ydb/core/yq/libs/control_plane_storage/probes.h +++ b/ydb/core/yq/libs/control_plane_storage/probes.h @@ -86,7 +86,7 @@ PROBE(WriteResultDataRequest, \ GROUPS(), \ TYPES(TString, i32, i64, i64, TDuration, TInstant, i64, bool), \ - NAMES("resultId", "resulSetId", "startRowId", "countRows", "latencyMs", "deadlineSec", "size", "success")) \ + NAMES("resultId", "resulSetId", "startRowId", "countRows", "latencyMs", "deadlineSec", "size", "success")) \ PROBE(GetTaskRequest, \ GROUPS(), \ TYPES(TString, TString, TDuration, bool), \ @@ -95,10 +95,10 @@ GROUPS(), \ TYPES(TString, TDuration, bool), \ NAMES("queryId", "latencyMs", "success")) \ - PROBE(NodesHealthCheckRequest, \ - GROUPS(), \ - TYPES(TString, ui32, TString, TString, TDuration, bool), \ - NAMES("tenant", "nodeId", "instanceId", "hostName", "latencyMs", "success")) \ + PROBE(NodesHealthCheckRequest, \ + GROUPS(), \ + TYPES(TString, ui32, TString, TString, TDuration, bool), \ + NAMES("tenant", "nodeId", "instanceId", "hostName", "latencyMs", "success")) \ // YQ_CONTROL_PLANE_STORAGE_PROVIDER diff --git a/ydb/core/yq/libs/control_plane_storage/proto/yq_internal.proto b/ydb/core/yq/libs/control_plane_storage/proto/yq_internal.proto index c941f1e965..e6ebff0c84 100644 --- a/ydb/core/yq/libs/control_plane_storage/proto/yq_internal.proto +++ b/ydb/core/yq/libs/control_plane_storage/proto/yq_internal.proto @@ -26,7 +26,7 @@ message QueryInternal { ExecuteMode execute_mode = 9; StateLoadMode state_load_mode = 10; string cloud_id = 11; - repeated Yq.Private.TopicConsumer created_topic_consumers = 12; + repeated Yq.Private.TopicConsumer created_topic_consumers = 12; repeated bytes dq_graph = 13; int32 dq_graph_index = 14; StreamingDisposition disposition = 15; diff --git a/ydb/core/yq/libs/control_plane_storage/schema.h b/ydb/core/yq/libs/control_plane_storage/schema.h index a1c6700df5..613eb7bcc1 100644 --- a/ydb/core/yq/libs/control_plane_storage/schema.h +++ b/ydb/core/yq/libs/control_plane_storage/schema.h @@ -4,7 +4,7 @@ namespace NYq { // tables #define QUERIES_TABLE_NAME "queries" -#define PENDING_SMALL_TABLE_NAME "pending_small" +#define PENDING_SMALL_TABLE_NAME "pending_small" #define CONNECTIONS_TABLE_NAME "connections" #define BINDINGS_TABLE_NAME "bindings" #define RESULT_SETS_TABLE_NAME "result_sets" @@ -21,10 +21,10 @@ namespace NYq { #define META_REVISION_COLUMN_NAME "meta_revision" #define INTERNAL_COLUMN_NAME "internal" #define GENERATION_COLUMN_NAME "generation" -#define RETRY_COUNTER_COLUMN_NAME "retry_counter" -#define RETRY_COUNTER_UPDATE_COLUMN_NAME "retry_counter_updated_at" -#define IS_RESIGN_QUERY_COLUMN_NAME "is_resign_query" - +#define RETRY_COUNTER_COLUMN_NAME "retry_counter" +#define RETRY_COUNTER_UPDATE_COLUMN_NAME "retry_counter_updated_at" +#define IS_RESIGN_QUERY_COLUMN_NAME "is_resign_query" + #define STATUS_COLUMN_NAME "status" #define QUERY_TYPE_COLUMN_NAME "query_type" #define EXECUTE_MODE_COLUMN_NAME "execute_mode" @@ -60,8 +60,8 @@ namespace NYq { #define ACTIVE_WORKERS_COLUMN_NAME "active_workers" #define MEMORY_LIMIT_COLUMN_NAME "memory_limit" #define MEMORY_ALLOCATED_COLUMN_NAME "memory_allocated" -#define INTERCONNECT_PORT_COLUMN_NAME "interconnect_port" -#define NODE_ADDRESS_COLUMN_NAME "node_address" +#define INTERCONNECT_PORT_COLUMN_NAME "interconnect_port" +#define NODE_ADDRESS_COLUMN_NAME "node_address" #define HOST_NAME_COLUMN_NAME "hostname" #define OWNER_COLUMN_NAME "owner" diff --git a/ydb/core/yq/libs/control_plane_storage/util.cpp b/ydb/core/yq/libs/control_plane_storage/util.cpp index ca9281b495..288f25a1cb 100644 --- a/ydb/core/yq/libs/control_plane_storage/util.cpp +++ b/ydb/core/yq/libs/control_plane_storage/util.cpp @@ -43,47 +43,47 @@ NConfig::TControlPlaneStorageConfig FillDefaultParameters(NConfig::TControlPlane if (!config.GetMaxCountJobs()) { config.SetMaxCountJobs(20); } - - if (!config.GetTasksBatchSize()) { - config.SetTasksBatchSize(100); - } - - if (!config.GetNumTasksProportion()) { - config.SetNumTasksProportion(4); - } - - if (!config.GetNumTasksProportion()) { - config.SetNumTasksProportion(4); - } - - if (!config.GetAnalyticsRetryCounterLimit()) { - config.SetAnalyticsRetryCounterLimit(20); - } - - if (!config.GetStreamingRetryCounterLimit()) { - config.SetStreamingRetryCounterLimit(20); - } - + + if (!config.GetTasksBatchSize()) { + config.SetTasksBatchSize(100); + } + + if (!config.GetNumTasksProportion()) { + config.SetNumTasksProportion(4); + } + + if (!config.GetNumTasksProportion()) { + config.SetNumTasksProportion(4); + } + + if (!config.GetAnalyticsRetryCounterLimit()) { + config.SetAnalyticsRetryCounterLimit(20); + } + + if (!config.GetStreamingRetryCounterLimit()) { + config.SetStreamingRetryCounterLimit(20); + } + if (!config.GetAnalyticsRetryCounterUpdateTime()) { config.SetAnalyticsRetryCounterUpdateTime("1d"); - } - + } + if (!config.GetStreamingRetryCounterUpdateTime()) { config.SetStreamingRetryCounterUpdateTime("1d"); - } - + } + if (!config.GetAutomaticQueriesTtl()) { config.SetAutomaticQueriesTtl("1d"); } - + if (!config.GetTaskLeaseTtl()) { config.SetTaskLeaseTtl("30s"); - } + } if (!config.GetStorage().GetToken() && config.GetStorage().GetOAuthFile()) { config.MutableStorage()->SetToken(StripString(TFileInput(config.GetStorage().GetOAuthFile()).ReadAll())); } - + if (!config.GetResultSetsTtl()) { config.SetResultSetsTtl("1d"); } @@ -92,39 +92,39 @@ NConfig::TControlPlaneStorageConfig FillDefaultParameters(NConfig::TControlPlane } bool DoesPingTaskUpdateQueriesTable(const TEvControlPlaneStorage::TEvPingTaskRequest* request) { - if (!request) { - return false; - } - return request->Status || - request->Issues || - request->TransientIssues || - request->Statistics || + if (!request) { + return false; + } + return request->Status || + request->Issues || + request->TransientIssues || + request->Statistics || request->ResultSetMetas || - request->Ast || - request->Plan || - request->StartedAt || + request->Ast || + request->Plan || + request->StartedAt || request->FinishedAt || !request->CreatedTopicConsumers.empty() || !request->DqGraphs.empty() || request->DqGraphIndex || request->StateLoadMode || request->StreamingDisposition; -} - -NYdb::TValue PackItemsToList(const TVector<NYdb::TValue>& items) { - NYdb::TValueBuilder itemsAsList; - itemsAsList.BeginList(); - for (const NYdb::TValue& item: items) { - itemsAsList.AddListItem(item); - } - itemsAsList.EndList(); - return itemsAsList.Build(); -} - -std::pair<TString, TString> SplitId(const TString& id, char delim) { - auto it = std::find(id.begin(), id.end(), delim); - return std::make_pair(id.substr(0, it - id.begin()), - (it != id.end() ? id.substr(it - id.begin() + 1) : TString{""})); -} - +} + +NYdb::TValue PackItemsToList(const TVector<NYdb::TValue>& items) { + NYdb::TValueBuilder itemsAsList; + itemsAsList.BeginList(); + for (const NYdb::TValue& item: items) { + itemsAsList.AddListItem(item); + } + itemsAsList.EndList(); + return itemsAsList.Build(); +} + +std::pair<TString, TString> SplitId(const TString& id, char delim) { + auto it = std::find(id.begin(), id.end(), delim); + return std::make_pair(id.substr(0, it - id.begin()), + (it != id.end() ? id.substr(it - id.begin() + 1) : TString{""})); +} + } //namespace NYq diff --git a/ydb/core/yq/libs/control_plane_storage/util.h b/ydb/core/yq/libs/control_plane_storage/util.h index 9edc20d84e..8deddc25ca 100644 --- a/ydb/core/yq/libs/control_plane_storage/util.h +++ b/ydb/core/yq/libs/control_plane_storage/util.h @@ -25,9 +25,9 @@ auto CreateArray(const T(&list)[N]) -> std::array<T, K> { } bool DoesPingTaskUpdateQueriesTable(const TEvControlPlaneStorage::TEvPingTaskRequest* request); - -NYdb::TValue PackItemsToList(const TVector<NYdb::TValue>& items); - -std::pair<TString, TString> SplitId(const TString& id, char delim = '-'); - -} // namespace NYq + +NYdb::TValue PackItemsToList(const TVector<NYdb::TValue>& items); + +std::pair<TString, TString> SplitId(const TString& id, char delim = '-'); + +} // namespace NYq diff --git a/ydb/core/yq/libs/control_plane_storage/ya.make b/ydb/core/yq/libs/control_plane_storage/ya.make index b8f9939079..8304e50db5 100644 --- a/ydb/core/yq/libs/control_plane_storage/ya.make +++ b/ydb/core/yq/libs/control_plane_storage/ya.make @@ -3,7 +3,7 @@ OWNER(g:yq) LIBRARY() SRCS( - control_plane_storage_counters.cpp + control_plane_storage_counters.cpp exceptions.cpp in_memory_control_plane_storage.cpp probes.cpp diff --git a/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage.cpp b/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage.cpp index 5cdddd2619..222ebb6e8e 100644 --- a/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage.cpp +++ b/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage.cpp @@ -37,18 +37,18 @@ void TYdbControlPlaneStorageActor::Bootstrap() { CPS_LOG_I("Starting ydb control plane storage service. Actor id: " << SelfId()); NLwTraceMonPage::ProbeRegistry().AddProbesList(LWTRACE_GET_PROBES(YQ_CONTROL_PLANE_STORAGE_PROVIDER)); - DbPool = YqSharedResources->DbPoolHolder->GetOrCreate(EDbPoolId::MAIN, 10); + DbPool = YqSharedResources->DbPoolHolder->GetOrCreate(EDbPoolId::MAIN, 10); YdbConnection = NewYdbConnection(Config.Proto.GetStorage(), CredProviderFactory); auto as = NActors::TActivationContext::ActorSystem(); CreateDirectory(as); CreateQueriesTable(as); - CreatePendingSmallTable(as); + CreatePendingSmallTable(as); CreateConnectionsTable(as); CreateBindingsTable(as); CreateIdempotencyKeysTable(as); CreateResultSetsTable(as); CreateJobsTable(as); - CreateNodesTable(as); + CreateNodesTable(as); Become(&TThis::StateFunc); } @@ -81,8 +81,8 @@ TAsyncStatus TYdbControlPlaneStorageActor::CreateQueriesTable(TActorSystem* as) auto description = TTableBuilder() .AddNullableColumn(SCOPE_COLUMN_NAME, EPrimitiveType::String) .AddNullableColumn(QUERY_ID_COLUMN_NAME, EPrimitiveType::String) - .AddNullableColumn(RESULT_ID_COLUMN_NAME, EPrimitiveType::String) - .AddNullableColumn(GENERATION_COLUMN_NAME, EPrimitiveType::Uint64) + .AddNullableColumn(RESULT_ID_COLUMN_NAME, EPrimitiveType::String) + .AddNullableColumn(GENERATION_COLUMN_NAME, EPrimitiveType::Uint64) .AddNullableColumn(NAME_COLUMN_NAME, EPrimitiveType::String) .AddNullableColumn(USER_COLUMN_NAME, EPrimitiveType::String) .AddNullableColumn(VISIBILITY_COLUMN_NAME, EPrimitiveType::Int64) @@ -115,20 +115,20 @@ TAsyncStatus TYdbControlPlaneStorageActor::CreateQueriesTable(TActorSystem* as) }); } -TAsyncStatus TYdbControlPlaneStorageActor::CreatePendingSmallTable(TActorSystem* as) -{ - auto tablePath = JoinPath(YdbConnection->TablePathPrefix, PENDING_SMALL_TABLE_NAME); - - auto description = TTableBuilder() - .AddNullableColumn(SCOPE_COLUMN_NAME, EPrimitiveType::String) - .AddNullableColumn(QUERY_ID_COLUMN_NAME, EPrimitiveType::String) +TAsyncStatus TYdbControlPlaneStorageActor::CreatePendingSmallTable(TActorSystem* as) +{ + auto tablePath = JoinPath(YdbConnection->TablePathPrefix, PENDING_SMALL_TABLE_NAME); + + auto description = TTableBuilder() + .AddNullableColumn(SCOPE_COLUMN_NAME, EPrimitiveType::String) + .AddNullableColumn(QUERY_ID_COLUMN_NAME, EPrimitiveType::String) .AddNullableColumn(LAST_SEEN_AT_COLUMN_NAME, EPrimitiveType::Timestamp) - .AddNullableColumn(RETRY_COUNTER_COLUMN_NAME, EPrimitiveType::Uint64) - .AddNullableColumn(RETRY_COUNTER_UPDATE_COLUMN_NAME, EPrimitiveType::Timestamp) - .AddNullableColumn(QUERY_TYPE_COLUMN_NAME, EPrimitiveType::Int64) - .AddNullableColumn(IS_RESIGN_QUERY_COLUMN_NAME, EPrimitiveType::Bool) - .AddNullableColumn(HOST_NAME_COLUMN_NAME, EPrimitiveType::String) - .AddNullableColumn(OWNER_COLUMN_NAME, EPrimitiveType::String) + .AddNullableColumn(RETRY_COUNTER_COLUMN_NAME, EPrimitiveType::Uint64) + .AddNullableColumn(RETRY_COUNTER_UPDATE_COLUMN_NAME, EPrimitiveType::Timestamp) + .AddNullableColumn(QUERY_TYPE_COLUMN_NAME, EPrimitiveType::Int64) + .AddNullableColumn(IS_RESIGN_QUERY_COLUMN_NAME, EPrimitiveType::Bool) + .AddNullableColumn(HOST_NAME_COLUMN_NAME, EPrimitiveType::String) + .AddNullableColumn(OWNER_COLUMN_NAME, EPrimitiveType::String) .SetPrimaryKeyColumns({SCOPE_COLUMN_NAME, QUERY_ID_COLUMN_NAME}) .Build(); @@ -140,7 +140,7 @@ TAsyncStatus TYdbControlPlaneStorageActor::CreatePendingSmallTable(TActorSystem* auto status = future.GetValue(); if (!IsTableCreated(status)) { CPS_LOG_AS_E(*as, "create pending table error: " << status.GetIssues().ToString()); - return CreatePendingSmallTable(as); + return CreatePendingSmallTable(as); } return future; }); @@ -202,7 +202,7 @@ TAsyncStatus TYdbControlPlaneStorageActor::CreateJobsTable(TActorSystem* as) .AddNullableColumn(USER_COLUMN_NAME, EPrimitiveType::String) .AddNullableColumn(VISIBILITY_COLUMN_NAME, EPrimitiveType::Int64) .AddNullableColumn(EXPIRE_AT_COLUMN_NAME, EPrimitiveType::Timestamp) - .SetPrimaryKeyColumns({SCOPE_COLUMN_NAME, QUERY_ID_COLUMN_NAME, JOB_ID_COLUMN_NAME}) + .SetPrimaryKeyColumns({SCOPE_COLUMN_NAME, QUERY_ID_COLUMN_NAME, JOB_ID_COLUMN_NAME}) .SetTtlSettings(EXPIRE_AT_COLUMN_NAME) .Build(); @@ -220,39 +220,39 @@ TAsyncStatus TYdbControlPlaneStorageActor::CreateJobsTable(TActorSystem* as) }); } -TAsyncStatus TYdbControlPlaneStorageActor::CreateNodesTable(TActorSystem* as) -{ +TAsyncStatus TYdbControlPlaneStorageActor::CreateNodesTable(TActorSystem* as) +{ auto tablePath = JoinPath(YdbConnection->TablePathPrefix, NODES_TABLE_NAME); - - auto description = TTableBuilder() - .AddNullableColumn(TENANT_COLUMN_NAME, EPrimitiveType::String) - .AddNullableColumn(INSTANCE_ID_COLUMN_NAME, EPrimitiveType::String) - .AddNullableColumn(NODE_ID_COLUMN_NAME, EPrimitiveType::Uint32) - .AddNullableColumn(HOST_NAME_COLUMN_NAME, EPrimitiveType::String) - .AddNullableColumn(ACTIVE_WORKERS_COLUMN_NAME, EPrimitiveType::Uint64) + + auto description = TTableBuilder() + .AddNullableColumn(TENANT_COLUMN_NAME, EPrimitiveType::String) + .AddNullableColumn(INSTANCE_ID_COLUMN_NAME, EPrimitiveType::String) + .AddNullableColumn(NODE_ID_COLUMN_NAME, EPrimitiveType::Uint32) + .AddNullableColumn(HOST_NAME_COLUMN_NAME, EPrimitiveType::String) + .AddNullableColumn(ACTIVE_WORKERS_COLUMN_NAME, EPrimitiveType::Uint64) .AddNullableColumn(MEMORY_LIMIT_COLUMN_NAME, EPrimitiveType::Uint64) .AddNullableColumn(MEMORY_ALLOCATED_COLUMN_NAME, EPrimitiveType::Uint64) - .AddNullableColumn(EXPIRE_AT_COLUMN_NAME, EPrimitiveType::Timestamp) - .AddNullableColumn(INTERCONNECT_PORT_COLUMN_NAME, EPrimitiveType::Uint32) - .AddNullableColumn(NODE_ADDRESS_COLUMN_NAME, EPrimitiveType::String) - .SetTtlSettings(EXPIRE_AT_COLUMN_NAME) + .AddNullableColumn(EXPIRE_AT_COLUMN_NAME, EPrimitiveType::Timestamp) + .AddNullableColumn(INTERCONNECT_PORT_COLUMN_NAME, EPrimitiveType::Uint32) + .AddNullableColumn(NODE_ADDRESS_COLUMN_NAME, EPrimitiveType::String) + .SetTtlSettings(EXPIRE_AT_COLUMN_NAME) .SetPrimaryKeyColumns({TENANT_COLUMN_NAME, NODE_ID_COLUMN_NAME}) - .Build(); - - return YdbConnection->Client.RetryOperation( - [tablePath = std::move(tablePath), description = std::move(description)] (TSession session) mutable { - return session.CreateTable(tablePath, TTableDescription(description)); - }) - .Apply([=](const auto& future) { - auto status = future.GetValue(); + .Build(); + + return YdbConnection->Client.RetryOperation( + [tablePath = std::move(tablePath), description = std::move(description)] (TSession session) mutable { + return session.CreateTable(tablePath, TTableDescription(description)); + }) + .Apply([=](const auto& future) { + auto status = future.GetValue(); if (!IsTableCreated(status)) { - CPS_LOG_AS_E(*as, "create nodes table error: " << status.GetIssues().ToString()); - return CreateNodesTable(as); - } - return future; - }); -} - + CPS_LOG_AS_E(*as, "create nodes table error: " << status.GetIssues().ToString()); + return CreateNodesTable(as); + } + return future; + }); +} + TAsyncStatus TYdbControlPlaneStorageActor::CreateBindingsTable(TActorSystem* as) { auto tablePath = JoinPath(YdbConnection->TablePathPrefix, BINDINGS_TABLE_NAME); @@ -286,7 +286,7 @@ TAsyncStatus TYdbControlPlaneStorageActor::CreateBindingsTable(TActorSystem* as) TAsyncStatus TYdbControlPlaneStorageActor::CreateIdempotencyKeysTable(TActorSystem* as) { - + auto tablePath = JoinPath(YdbConnection->TablePathPrefix, IDEMPOTENCY_KEYS_TABLE_NAME); auto description = TTableBuilder() @@ -317,13 +317,13 @@ TAsyncStatus TYdbControlPlaneStorageActor::CreateResultSetsTable(TActorSystem* a auto tablePath = JoinPath(YdbConnection->TablePathPrefix, RESULT_SETS_TABLE_NAME); auto description = TTableBuilder() - .AddNullableColumn(RESULT_ID_COLUMN_NAME, EPrimitiveType::String) + .AddNullableColumn(RESULT_ID_COLUMN_NAME, EPrimitiveType::String) .AddNullableColumn(RESULT_SET_ID_COLUMN_NAME, EPrimitiveType::Int32) .AddNullableColumn(ROW_ID_COLUMN_NAME, EPrimitiveType::Int64) .AddNullableColumn(RESULT_SET_COLUMN_NAME, EPrimitiveType::String) .AddNullableColumn(EXPIRE_AT_COLUMN_NAME, EPrimitiveType::Timestamp) .SetTtlSettings(EXPIRE_AT_COLUMN_NAME) - .SetPrimaryKeyColumns({RESULT_ID_COLUMN_NAME, RESULT_SET_ID_COLUMN_NAME, ROW_ID_COLUMN_NAME}) + .SetPrimaryKeyColumns({RESULT_ID_COLUMN_NAME, RESULT_SET_ID_COLUMN_NAME, ROW_ID_COLUMN_NAME}) .Build(); return YdbConnection->Client.RetryOperation( @@ -373,31 +373,31 @@ void TYdbControlPlaneStorageActor::ReadIdempotencyKeyQuery(TSqlQueryBuilder& bui class TDbRequest: public NActors::TActorBootstrapped<TDbRequest> { using TFunction = std::function<NYdb::TAsyncStatus(NYdb::NTable::TSession&)>; - TDbPool::TPtr DbPool; + TDbPool::TPtr DbPool; TPromise<NYdb::TStatus> Promise; TFunction Handler; public: - TDbRequest(const TDbPool::TPtr& dbPool, const TPromise<NYdb::TStatus>& promise, const TFunction& handler) + TDbRequest(const TDbPool::TPtr& dbPool, const TPromise<NYdb::TStatus>& promise, const TFunction& handler) : DbPool(dbPool) , Promise(promise) , Handler(handler) {} static constexpr char ActorName[] = "YQ_CONTROL_PLANE_STORAGE_DB_REQUEST"; - + void Bootstrap() { CPS_LOG_T("DbRequest actor request. Actor id: " << SelfId()); Become(&TDbRequest::StateFunc); - Send(DbPool->GetNextActor(), new TEvents::TEvDbFunctionRequest(Handler), IEventHandle::FlagTrackDelivery); + Send(DbPool->GetNextActor(), new TEvents::TEvDbFunctionRequest(Handler), IEventHandle::FlagTrackDelivery); } STRICT_STFUNC(StateFunc, - hFunc(TEvents::TEvDbFunctionResponse, Handle); + hFunc(TEvents::TEvDbFunctionResponse, Handle); hFunc(NActors::TEvents::TEvUndelivered, OnUndelivered) ) - void Handle(TEvents::TEvDbFunctionResponse::TPtr& ev) { + void Handle(TEvents::TEvDbFunctionResponse::TPtr& ev) { CPS_LOG_T("DbRequest actor response. Actor id: " << SelfId()); Promise.SetValue(ev->Get()->Status); PassAway(); @@ -405,21 +405,21 @@ public: void OnUndelivered(NActors::TEvents::TEvUndelivered::TPtr&) { CPS_LOG_E("On delivered. Actor id: " << SelfId()); - Send(DbPool->GetNextActor(), new TEvents::TEvDbFunctionRequest(Handler), IEventHandle::FlagTrackDelivery); + Send(DbPool->GetNextActor(), new TEvents::TEvDbFunctionRequest(Handler), IEventHandle::FlagTrackDelivery); } }; -std::pair<TAsyncStatus, std::shared_ptr<TVector<NYdb::TResultSet>>> TYdbControlPlaneStorageActor::Read( - const TString& query, - const NYdb::TParams& params, - const TRequestCountersPtr& requestCounters, - TDebugInfoPtr debugInfo, - TTxSettings transactionMode, - bool retryOnTli) +std::pair<TAsyncStatus, std::shared_ptr<TVector<NYdb::TResultSet>>> TYdbControlPlaneStorageActor::Read( + const TString& query, + const NYdb::TParams& params, + const TRequestCountersPtr& requestCounters, + TDebugInfoPtr debugInfo, + TTxSettings transactionMode, + bool retryOnTli) { auto resultSet = std::make_shared<TVector<NYdb::TResultSet>>(); - + std::shared_ptr<int> retryCount = std::make_shared<int>(); auto handler = [=](TSession& session) { if (*retryCount != 0) { @@ -427,17 +427,17 @@ std::pair<TAsyncStatus, std::shared_ptr<TVector<NYdb::TResultSet>>> TYdbControlP } ++(*retryCount); CollectDebugInfo(query, params, session, debugInfo); - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx(transactionMode).CommitTx(), params, NYdb::NTable::TExecDataQuerySettings().KeepInQueryCache(true)); - return result.Apply([retryOnTli, resultSet] (const TFuture<TDataQueryResult>& future) { + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx(transactionMode).CommitTx(), params, NYdb::NTable::TExecDataQuerySettings().KeepInQueryCache(true)); + return result.Apply([retryOnTli, resultSet] (const TFuture<TDataQueryResult>& future) { NYdb::NTable::TDataQueryResult result = future.GetValue(); *resultSet = result.GetResultSets(); auto status = static_cast<TStatus>(result); if (status.GetStatus() == EStatus::SCHEME_ERROR) { // retry if table does not exist return TStatus{EStatus::UNAVAILABLE, NYql::TIssues{status.GetIssues()}}; } - if (!retryOnTli && status.GetStatus() == EStatus::ABORTED) { - return TStatus{EStatus::GENERIC_ERROR, NYql::TIssues{status.GetIssues()}}; - } + if (!retryOnTli && status.GetStatus() == EStatus::ABORTED) { + return TStatus{EStatus::GENERIC_ERROR, NYql::TIssues{status.GetIssues()}}; + } return status; }); }; @@ -447,15 +447,15 @@ std::pair<TAsyncStatus, std::shared_ptr<TVector<NYdb::TResultSet>>> TYdbControlP return {promise.GetFuture(), resultSet}; } -TAsyncStatus TYdbControlPlaneStorageActor::Validate( - std::shared_ptr<TMaybe<TTransaction>> transaction, - size_t item, - const TVector<TValidationQuery>& validators, - TSession session, - std::shared_ptr<bool> successFinish, - TDebugInfoPtr debugInfo, - TTxSettings transactionMode) -{ +TAsyncStatus TYdbControlPlaneStorageActor::Validate( + std::shared_ptr<TMaybe<TTransaction>> transaction, + size_t item, + const TVector<TValidationQuery>& validators, + TSession session, + std::shared_ptr<bool> successFinish, + TDebugInfoPtr debugInfo, + TTxSettings transactionMode) +{ if (item >= validators.size()) { return MakeFuture(TStatus{EStatus::SUCCESS, NYql::TIssues{}}); } @@ -481,30 +481,30 @@ TAsyncStatus TYdbControlPlaneStorageActor::Validate( }); } -TAsyncStatus TYdbControlPlaneStorageActor::Write( - NActors::TActorSystem* actorSystem, - const TString& query, - const NYdb::TParams& params, - const TRequestCountersPtr& requestCounters, - TDebugInfoPtr debugInfo, - const TVector<TValidationQuery>& validators, - TTxSettings transactionMode, - bool retryOnTli) +TAsyncStatus TYdbControlPlaneStorageActor::Write( + NActors::TActorSystem* actorSystem, + const TString& query, + const NYdb::TParams& params, + const TRequestCountersPtr& requestCounters, + TDebugInfoPtr debugInfo, + const TVector<TValidationQuery>& validators, + TTxSettings transactionMode, + bool retryOnTli) { std::shared_ptr<int> retryCount = std::make_shared<int>(); auto transaction = std::make_shared<TMaybe<TTransaction>>(); - auto writeHandler = [=, retryOnTli=retryOnTli] (TSession session) { + auto writeHandler = [=, retryOnTli=retryOnTli] (TSession session) { CollectDebugInfo(query, params, session, debugInfo); - auto result = session.ExecuteDataQuery(query, validators ? TTxControl::Tx(**transaction).CommitTx() : TTxControl::BeginTx(transactionMode).CommitTx(), params, NYdb::NTable::TExecDataQuerySettings().KeepInQueryCache(true)); - return result.Apply([=] (const TFuture<TDataQueryResult>& future) { + auto result = session.ExecuteDataQuery(query, validators ? TTxControl::Tx(**transaction).CommitTx() : TTxControl::BeginTx(transactionMode).CommitTx(), params, NYdb::NTable::TExecDataQuerySettings().KeepInQueryCache(true)); + return result.Apply([=] (const TFuture<TDataQueryResult>& future) { NYdb::NTable::TDataQueryResult result = future.GetValue(); auto status = static_cast<TStatus>(result); if (status.GetStatus() == EStatus::SCHEME_ERROR) { // retry if table does not exist return TStatus{EStatus::UNAVAILABLE, NYql::TIssues{status.GetIssues()}}; } - if (!retryOnTli && status.GetStatus() == EStatus::ABORTED) { - return TStatus{EStatus::GENERIC_ERROR, NYql::TIssues{status.GetIssues()}}; - } + if (!retryOnTli && status.GetStatus() == EStatus::ABORTED) { + return TStatus{EStatus::GENERIC_ERROR, NYql::TIssues{status.GetIssues()}}; + } return status; }); }; @@ -542,42 +542,42 @@ TAsyncStatus TYdbControlPlaneStorageActor::Write( return promise.GetFuture(); } -NThreading::TFuture<void> TYdbControlPlaneStorageActor::PickTask( - const TPickTaskParams& taskParams, - const TRequestCountersPtr& requestCounters, - TDebugInfoPtr debugInfo, - std::shared_ptr<TResponseTasks> responseTasks, - const TVector<TValidationQuery>& validators, - TTxSettings transactionMode) +NThreading::TFuture<void> TYdbControlPlaneStorageActor::PickTask( + const TPickTaskParams& taskParams, + const TRequestCountersPtr& requestCounters, + TDebugInfoPtr debugInfo, + std::shared_ptr<TResponseTasks> responseTasks, + const TVector<TValidationQuery>& validators, + TTxSettings transactionMode) +{ + return ReadModifyWrite(NActors::TActivationContext::ActorSystem(), taskParams.ReadQuery, taskParams.ReadParams, + taskParams.PrepareParams, requestCounters, debugInfo, validators, transactionMode, taskParams.RetryOnTli) + .Apply([=, responseTasks=responseTasks, queryId = taskParams.QueryId](const auto& future) { + const auto status = future.GetValue(); + if (responseTasks && status.GetStatus() == EStatus::GENERIC_ERROR) { + responseTasks->SafeEraseTaskBlocking(queryId); + } + }); +} + +TAsyncStatus TYdbControlPlaneStorageActor::ReadModifyWrite( + NActors::TActorSystem* actorSystem, + const TString& readQuery, + const NYdb::TParams& readParams, + const std::function<std::pair<TString, NYdb::TParams>(const TVector<NYdb::TResultSet>&)>& prepare, + const TRequestCountersPtr& requestCounters, + TDebugInfoPtr debugInfo, + const TVector<TValidationQuery>& validators, + TTxSettings transactionMode, + bool retryOnTli) { - return ReadModifyWrite(NActors::TActivationContext::ActorSystem(), taskParams.ReadQuery, taskParams.ReadParams, - taskParams.PrepareParams, requestCounters, debugInfo, validators, transactionMode, taskParams.RetryOnTli) - .Apply([=, responseTasks=responseTasks, queryId = taskParams.QueryId](const auto& future) { - const auto status = future.GetValue(); - if (responseTasks && status.GetStatus() == EStatus::GENERIC_ERROR) { - responseTasks->SafeEraseTaskBlocking(queryId); - } - }); -} - -TAsyncStatus TYdbControlPlaneStorageActor::ReadModifyWrite( - NActors::TActorSystem* actorSystem, - const TString& readQuery, - const NYdb::TParams& readParams, - const std::function<std::pair<TString, NYdb::TParams>(const TVector<NYdb::TResultSet>&)>& prepare, - const TRequestCountersPtr& requestCounters, - TDebugInfoPtr debugInfo, - const TVector<TValidationQuery>& validators, - TTxSettings transactionMode, - bool retryOnTli) -{ std::shared_ptr<int> retryCount = std::make_shared<int>(); auto resultSets = std::make_shared<TVector<NYdb::TResultSet>>(); auto transaction = std::make_shared<TMaybe<TTransaction>>(); auto readModifyWriteHandler = [=](TSession session) { CollectDebugInfo(readQuery, readParams, session, debugInfo); - auto readResult = session.ExecuteDataQuery(readQuery, validators ? TTxControl::Tx(**transaction) : TTxControl::BeginTx(transactionMode), readParams, NYdb::NTable::TExecDataQuerySettings().KeepInQueryCache(true)); + auto readResult = session.ExecuteDataQuery(readQuery, validators ? TTxControl::Tx(**transaction) : TTxControl::BeginTx(transactionMode), readParams, NYdb::NTable::TExecDataQuerySettings().KeepInQueryCache(true)); auto readResultStatus = readResult.Apply([resultSets, transaction] (const TFuture<TDataQueryResult>& future) { NYdb::NTable::TDataQueryResult result = future.GetValue(); *resultSets = result.GetResultSets(); @@ -597,7 +597,7 @@ TAsyncStatus TYdbControlPlaneStorageActor::ReadModifyWrite( if (!readResultStatus.GetValue().IsSuccess()) { return readResultStatus; } - + try { auto [writeQuery, params] = future.GetValue(); if (!writeQuery) { @@ -612,15 +612,15 @@ TAsyncStatus TYdbControlPlaneStorageActor::ReadModifyWrite( } CollectDebugInfo(writeQuery, params, session, debugInfo); auto writeResult = session.ExecuteDataQuery(writeQuery, TTxControl::Tx(**transaction).CommitTx(), params, NYdb::NTable::TExecDataQuerySettings().KeepInQueryCache(true)); - return writeResult.Apply([retryOnTli] (const TFuture<TDataQueryResult>& future) { + return writeResult.Apply([retryOnTli] (const TFuture<TDataQueryResult>& future) { NYdb::NTable::TDataQueryResult result = future.GetValue(); auto status = static_cast<TStatus>(result); if (status.GetStatus() == EStatus::SCHEME_ERROR) { // retry if table does not exist return TStatus{EStatus::UNAVAILABLE, NYql::TIssues{status.GetIssues()}}; } - if (!retryOnTli && status.GetStatus() == EStatus::ABORTED) { - return TStatus{EStatus::GENERIC_ERROR, NYql::TIssues{status.GetIssues()}}; - } + if (!retryOnTli && status.GetStatus() == EStatus::ABORTED) { + return TStatus{EStatus::GENERIC_ERROR, NYql::TIssues{status.GetIssues()}}; + } return status; }); } catch (const TControlPlaneStorageException& exception) { @@ -634,7 +634,7 @@ TAsyncStatus TYdbControlPlaneStorageActor::ReadModifyWrite( return MakeFuture(TStatus{EStatus::GENERIC_ERROR, NYql::TIssues{NYql::TIssue{CurrentExceptionMessage()}}}); } }); - }; + }; auto handler = [=] (TSession session) { if (*retryCount != 0) { diff --git a/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage_bindings.cpp b/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage_bindings.cpp index 4612cce5bf..fc617fe5f0 100644 --- a/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage_bindings.cpp +++ b/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage_bindings.cpp @@ -64,7 +64,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvCreateBindi response->second.After.ConstructInPlace().CopyFrom(binding); response->second.CloudId = cloudId; - TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "CreateBinding"); + TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "CreateBinding"); queryBuilder.AddString("scope", scope); queryBuilder.AddString("binding_id", bindingId); queryBuilder.AddString("connection_id", connectionId); @@ -172,7 +172,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvListBinding return; } - TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "ListBindings"); + TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "ListBindings"); queryBuilder.AddString("scope", scope); queryBuilder.AddString("last_binding", pageToken); queryBuilder.AddUint64("limit", limit + 1); @@ -307,7 +307,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvDescribeBin return; } - TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "DescribeBinding"); + TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "DescribeBinding"); queryBuilder.AddString("scope", scope); queryBuilder.AddString("binding_id", bindingId); queryBuilder.AddText( @@ -395,7 +395,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvModifyBindi return; } - TSqlQueryBuilder readQueryBuilder(YdbConnection->TablePathPrefix, "ModifyBinding(read)"); + TSqlQueryBuilder readQueryBuilder(YdbConnection->TablePathPrefix, "ModifyBinding(read)"); readQueryBuilder.AddString("scope", scope); readQueryBuilder.AddString("binding_id", bindingId); readQueryBuilder.AddText( @@ -451,7 +451,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvModifyBindi response->second.After.ConstructInPlace().CopyFrom(binding); response->second.CloudId = bindingInternal.cloud_id(); - TSqlQueryBuilder writeQueryBuilder(YdbConnection->TablePathPrefix, "ModifyBinding(write)"); + TSqlQueryBuilder writeQueryBuilder(YdbConnection->TablePathPrefix, "ModifyBinding(write)"); writeQueryBuilder.AddString("scope", scope); writeQueryBuilder.AddString("binding_id", bindingId); writeQueryBuilder.AddInt64("visibility", YandexQuery::Acl::SCOPE); // TODO: fix me @@ -572,7 +572,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvDeleteBindi std::shared_ptr<std::pair<YandexQuery::DeleteBindingResult, TAuditDetails<YandexQuery::Binding>>> response = std::make_shared<std::pair<YandexQuery::DeleteBindingResult, TAuditDetails<YandexQuery::Binding>>>(); - TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "DeleteBinding"); + TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "DeleteBinding"); queryBuilder.AddString("scope", scope); queryBuilder.AddString("binding_id", bindingId); diff --git a/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage_connections.cpp b/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage_connections.cpp index a9b920ad9e..ddb15b16c4 100644 --- a/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage_connections.cpp +++ b/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage_connections.cpp @@ -49,7 +49,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvCreateConne const TDuration delta = TInstant::Now() - startTime; SendResponseIssues<TEvControlPlaneStorage::TEvCreateConnectionResponse>(ev->Sender, issues, ev->Cookie, delta, requestCounters); LWPROBE(CreateConnectionRequest, scope, user, delta, byteSize, false); - return; + return; } YandexQuery::Connection connection; @@ -65,7 +65,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvCreateConne response->second.After.ConstructInPlace().CopyFrom(connection); response->second.CloudId = cloudId; - TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "CreateConnection"); + TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "CreateConnection"); queryBuilder.AddString("scope", scope); queryBuilder.AddString("connection_id", connectionId); queryBuilder.AddString("user", user); @@ -161,10 +161,10 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvListConnect const TDuration delta = TInstant::Now() - startTime; SendResponseIssues<TEvControlPlaneStorage::TEvListConnectionsResponse>(ev->Sender, issues, ev->Cookie, delta, requestCounters); LWPROBE(ListConnectionsRequest, scope, user, delta, byteSize, false); - return; + return; } - TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "ListConnections"); + TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "ListConnections"); queryBuilder.AddString("scope", scope); queryBuilder.AddString("last_connection", pageToken); queryBuilder.AddUint64("limit", limit + 1); @@ -286,10 +286,10 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvDescribeCon const TDuration delta = TInstant::Now() - startTime; SendResponseIssues<TEvControlPlaneStorage::TEvDescribeConnectionResponse>(ev->Sender, issues, ev->Cookie, delta, requestCounters); LWPROBE(DescribeConnectionRequest, scope, connectionId, user, delta, byteSize, false); - return; + return; } - TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "DescribeConnection"); + TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "DescribeConnection"); queryBuilder.AddString("scope", scope); queryBuilder.AddString("connection_id", connectionId); @@ -381,10 +381,10 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvModifyConne const TDuration delta = TInstant::Now() - startTime; SendResponseIssues<TEvControlPlaneStorage::TEvModifyConnectionResponse>(ev->Sender, issues, ev->Cookie, delta, requestCounters); LWPROBE(ModifyConnectionRequest, scope, connectionId, user, delta, byteSize, false); - return; + return; } - - TSqlQueryBuilder readQueryBuilder(YdbConnection->TablePathPrefix, "ModifyConnection(read)"); + + TSqlQueryBuilder readQueryBuilder(YdbConnection->TablePathPrefix, "ModifyConnection(read)"); readQueryBuilder.AddString("scope", scope); readQueryBuilder.AddString("connection_id", connectionId); readQueryBuilder.AddText( @@ -441,7 +441,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvModifyConne response->second.After.ConstructInPlace().CopyFrom(connection); response->second.CloudId = connectionInternal.cloud_id(); - TSqlQueryBuilder writeQueryBuilder(YdbConnection->TablePathPrefix, "ModifyConnection(write)"); + TSqlQueryBuilder writeQueryBuilder(YdbConnection->TablePathPrefix, "ModifyConnection(write)"); writeQueryBuilder.AddString("scope", scope); writeQueryBuilder.AddString("connection_id", connectionId); writeQueryBuilder.AddInt64("visibility", connection.content().acl().visibility()); @@ -545,7 +545,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvDeleteConne << "DeleteConnectionRequest: " << NKikimr::MaskTicket(token) << " " << request.DebugString()); - + NYql::TIssues issues = ValidateEvent(ev); if (issues) { CPS_LOG_D(MakeLogPrefix(scope, user, connectionId) @@ -556,12 +556,12 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvDeleteConne const TDuration delta = TInstant::Now() - startTime; SendResponseIssues<TEvControlPlaneStorage::TEvDeleteConnectionResponse>(ev->Sender, issues, ev->Cookie, delta, requestCounters); LWPROBE(DeleteConnectionRequest, scope, connectionId, user, delta, byteSize, false); - return; + return; } - + std::shared_ptr<std::pair<YandexQuery::DeleteConnectionResult, TAuditDetails<YandexQuery::Connection>>> response = std::make_shared<std::pair<YandexQuery::DeleteConnectionResult, TAuditDetails<YandexQuery::Connection>>>(); - TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "DeleteConnection"); + TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "DeleteConnection"); queryBuilder.AddString("scope", scope); queryBuilder.AddString("connection_id", connectionId); diff --git a/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage_impl.h b/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage_impl.h index f338968f79..1c8cda7ee7 100644 --- a/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage_impl.h +++ b/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage_impl.h @@ -1,7 +1,7 @@ #pragma once #include "control_plane_storage.h" -#include "control_plane_storage_counters.h" +#include "control_plane_storage_counters.h" #include "exceptions.h" #include "extractors.h" #include <ydb/core/yq/libs/control_plane_storage/internal/response_tasks.h> @@ -113,7 +113,7 @@ class TYdbControlPlaneStorageActor : public NActors::TActorBootstrapped<TYdbCont RT_WRITE_RESULT_DATA, RT_GET_TASK, RT_PING_TASK, - RT_NODES_HEALTH_CHECK, + RT_NODES_HEALTH_CHECK, RT_MAX, }; @@ -142,7 +142,7 @@ class TYdbControlPlaneStorageActor : public NActors::TActorBootstrapped<TYdbCont { MakeIntrusive<TRequestCounters>("WriteResultData") }, { MakeIntrusive<TRequestCounters>("GetTask") }, { MakeIntrusive<TRequestCounters>("PingTask") }, - { MakeIntrusive<TRequestCounters>("NodesHealthCheck") }, + { MakeIntrusive<TRequestCounters>("NodesHealthCheck") }, }); NMonitoring::TDynamicCounterPtr Counters; @@ -172,17 +172,17 @@ class TYdbControlPlaneStorageActor : public NActors::TActorBootstrapped<TYdbCont }; TCounters Counters; - TFinalStatusCounters FinalStatusCounters; + TFinalStatusCounters FinalStatusCounters; TConfig Config; TYdbConnectionPtr YdbConnection; ::NYq::TYqSharedResources::TPtr YqSharedResources; - TDbPool::TPtr DbPool; + TDbPool::TPtr DbPool; static constexpr int64_t InitialRevision = 1; - + NKikimr::TYdbCredentialsProviderFactory CredProviderFactory; public: @@ -193,15 +193,15 @@ public: const ::NYq::TYqSharedResources::TPtr& yqSharedResources, const NKikimr::TYdbCredentialsProviderFactory& credProviderFactory) : Counters(counters) - , FinalStatusCounters(counters) + , FinalStatusCounters(counters) , Config(config, common) , YqSharedResources(yqSharedResources) , CredProviderFactory(credProviderFactory) { } - static constexpr char ActorName[] = "YQ_CONTROL_PLANE_STORAGE"; - + static constexpr char ActorName[] = "YQ_CONTROL_PLANE_STORAGE"; + void Bootstrap(); STRICT_STFUNC(StateFunc, @@ -228,7 +228,7 @@ public: hFunc(TEvControlPlaneStorage::TEvWriteResultDataRequest, Handle); hFunc(TEvControlPlaneStorage::TEvGetTaskRequest, Handle); hFunc(TEvControlPlaneStorage::TEvPingTaskRequest, Handle); - hFunc(TEvControlPlaneStorage::TEvNodesHealthCheckRequest, Handle); + hFunc(TEvControlPlaneStorage::TEvNodesHealthCheckRequest, Handle); hFunc(NMon::TEvHttpInfo, Handle); ) @@ -259,8 +259,8 @@ public: void Handle(TEvControlPlaneStorage::TEvGetTaskRequest::TPtr& ev); void Handle(TEvControlPlaneStorage::TEvPingTaskRequest::TPtr& ev); - void Handle(TEvControlPlaneStorage::TEvNodesHealthCheckRequest::TPtr& ev); - + void Handle(TEvControlPlaneStorage::TEvNodesHealthCheckRequest::TPtr& ev); + template<typename T> NYql::TIssues ValidateConnection(T& ev, bool clickHousePasswordRequire = true) { @@ -512,10 +512,10 @@ public: TAsyncStatus CreateDirectory(TActorSystem* as); TAsyncStatus CreateQueriesTable(TActorSystem* as); TAsyncStatus CreatePendingTable(TActorSystem* as); - TAsyncStatus CreatePendingSmallTable(TActorSystem* as); + TAsyncStatus CreatePendingSmallTable(TActorSystem* as); TAsyncStatus CreateConnectionsTable(TActorSystem* as); TAsyncStatus CreateJobsTable(TActorSystem* as); - TAsyncStatus CreateNodesTable(TActorSystem* as); + TAsyncStatus CreateNodesTable(TActorSystem* as); TAsyncStatus CreateBindingsTable(TActorSystem* as); TAsyncStatus CreateIdempotencyKeysTable(TActorSystem* as); TAsyncStatus CreateResultSetsTable(TActorSystem* as); @@ -530,42 +530,42 @@ private: void ReadIdempotencyKeyQuery(TSqlQueryBuilder& builder, const TString& scope, const TString& idempotencyKey); - std::pair<TAsyncStatus, std::shared_ptr<TVector<NYdb::TResultSet>>> Read( - const TString& query, - const NYdb::TParams& params, - const TRequestCountersPtr& requestCounters, - TDebugInfoPtr debugInfo, - TTxSettings transactionMode = TTxSettings::SerializableRW(), - bool retryOnTli = true); - - TAsyncStatus Validate( - std::shared_ptr<TMaybe<TTransaction>> transaction, - size_t item, const TVector<TValidationQuery>& validators, - TSession session, - std::shared_ptr<bool> successFinish, - TDebugInfoPtr debugInfo, - TTxSettings transactionMode = TTxSettings::SerializableRW()); - - TAsyncStatus Write( - NActors::TActorSystem* actorSystem, - const TString& query, - const NYdb::TParams& params, - const TRequestCountersPtr& requestCounters, - TDebugInfoPtr debugInfo, - const TVector<TValidationQuery>& validators = {}, - TTxSettings transactionMode = TTxSettings::SerializableRW(), - bool retryTli = true); - - TAsyncStatus ReadModifyWrite( - NActors::TActorSystem* actorSystem, - const TString& readQuery, - const NYdb::TParams& readParams, - const std::function<std::pair<TString, NYdb::TParams>(const TVector<NYdb::TResultSet>&)>& prepare, - const TRequestCountersPtr& requestCounters, - TDebugInfoPtr debugInfo = {}, - const TVector<TValidationQuery>& validators = {}, - TTxSettings transactionMode = TTxSettings::SerializableRW(), - bool retryOnTli = true); + std::pair<TAsyncStatus, std::shared_ptr<TVector<NYdb::TResultSet>>> Read( + const TString& query, + const NYdb::TParams& params, + const TRequestCountersPtr& requestCounters, + TDebugInfoPtr debugInfo, + TTxSettings transactionMode = TTxSettings::SerializableRW(), + bool retryOnTli = true); + + TAsyncStatus Validate( + std::shared_ptr<TMaybe<TTransaction>> transaction, + size_t item, const TVector<TValidationQuery>& validators, + TSession session, + std::shared_ptr<bool> successFinish, + TDebugInfoPtr debugInfo, + TTxSettings transactionMode = TTxSettings::SerializableRW()); + + TAsyncStatus Write( + NActors::TActorSystem* actorSystem, + const TString& query, + const NYdb::TParams& params, + const TRequestCountersPtr& requestCounters, + TDebugInfoPtr debugInfo, + const TVector<TValidationQuery>& validators = {}, + TTxSettings transactionMode = TTxSettings::SerializableRW(), + bool retryTli = true); + + TAsyncStatus ReadModifyWrite( + NActors::TActorSystem* actorSystem, + const TString& readQuery, + const NYdb::TParams& readParams, + const std::function<std::pair<TString, NYdb::TParams>(const TVector<NYdb::TResultSet>&)>& prepare, + const TRequestCountersPtr& requestCounters, + TDebugInfoPtr debugInfo = {}, + const TVector<TValidationQuery>& validators = {}, + TTxSettings transactionMode = TTxSettings::SerializableRW(), + bool retryOnTli = true); template<typename T> THashMap<TString, T> GetEntitiesWithVisibilityPriority(const TResultSet& resultSet, const TString& columnName) @@ -821,22 +821,22 @@ private: static TString MakeLogPrefix(const TString& scope, const TString& user, const TString& id = "") { return "[" + scope + ", " + user + (id ? ", " + id : "") + "] "; } - - struct TPickTaskParams { - TString ReadQuery; - TParams ReadParams; - std::function<std::pair<TString, NYdb::TParams>(const TVector<NYdb::TResultSet>&)> PrepareParams; - TString QueryId; - bool RetryOnTli = false; - }; - - NThreading::TFuture<void> PickTask( - const TPickTaskParams& taskParams, - const TRequestCountersPtr& requestCounters, - TDebugInfoPtr debugInfo, - std::shared_ptr<TResponseTasks> responseTasks, - const TVector<TValidationQuery>& validators = {}, - TTxSettings transactionMode = TTxSettings::SerializableRW()); + + struct TPickTaskParams { + TString ReadQuery; + TParams ReadParams; + std::function<std::pair<TString, NYdb::TParams>(const TVector<NYdb::TResultSet>&)> PrepareParams; + TString QueryId; + bool RetryOnTli = false; + }; + + NThreading::TFuture<void> PickTask( + const TPickTaskParams& taskParams, + const TRequestCountersPtr& requestCounters, + TDebugInfoPtr debugInfo, + std::shared_ptr<TResponseTasks> responseTasks, + const TVector<TValidationQuery>& validators = {}, + TTxSettings transactionMode = TTxSettings::SerializableRW()); }; } diff --git a/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage_queries.cpp b/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage_queries.cpp index df7b3c9dde..398725f7f4 100644 --- a/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage_queries.cpp +++ b/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage_queries.cpp @@ -84,7 +84,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvCreateQuery const TDuration delta = TInstant::Now() - startTime; SendResponseIssues<TEvControlPlaneStorage::TEvCreateQueryResponse>(ev->Sender, issues, ev->Cookie, delta, requestCounters); LWPROBE(CreateQueryRequest, scope, user, delta, byteSize, false); - return; + return; } const TString idempotencyKey = request.idempotency_key(); @@ -116,7 +116,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvCreateQuery response->first.set_query_id(queryId); response->second.CloudId = cloudId; - TSqlQueryBuilder readQueryBuilder(YdbConnection->TablePathPrefix, "CreateQuery(read)"); + TSqlQueryBuilder readQueryBuilder(YdbConnection->TablePathPrefix, "CreateQuery(read)"); ReadIdempotencyKeyQuery(readQueryBuilder, scope, idempotencyKey); if (request.execute_mode() != YandexQuery::SAVE) { @@ -200,7 +200,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvCreateQuery response->second.After.ConstructInPlace().CopyFrom(query); - TSqlQueryBuilder writeQueryBuilder(YdbConnection->TablePathPrefix, "CreateQuery(write)"); + TSqlQueryBuilder writeQueryBuilder(YdbConnection->TablePathPrefix, "CreateQuery(write)"); writeQueryBuilder.AddString("scope", scope); writeQueryBuilder.AddString("query_id", queryId); writeQueryBuilder.AddString("name", query.content().name()); @@ -224,20 +224,20 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvCreateQuery // insert job writeQueryBuilder.AddText( - "INSERT INTO `" JOBS_TABLE_NAME "`\n" - "(`" SCOPE_COLUMN_NAME "`, `" QUERY_ID_COLUMN_NAME "`, `" JOB_ID_COLUMN_NAME "`, \n" - "`" JOB_COLUMN_NAME "`, `" USER_COLUMN_NAME "`, `" VISIBILITY_COLUMN_NAME "`)\n" - "VALUES\n" - " ($scope, $query_id, $job_id, $job, $user, $visibility);" + "INSERT INTO `" JOBS_TABLE_NAME "`\n" + "(`" SCOPE_COLUMN_NAME "`, `" QUERY_ID_COLUMN_NAME "`, `" JOB_ID_COLUMN_NAME "`, \n" + "`" JOB_COLUMN_NAME "`, `" USER_COLUMN_NAME "`, `" VISIBILITY_COLUMN_NAME "`)\n" + "VALUES\n" + " ($scope, $query_id, $job_id, $job, $user, $visibility);" ); // insert pending small writeQueryBuilder.AddText( - "INSERT INTO `" PENDING_SMALL_TABLE_NAME "`\n" - "(`" SCOPE_COLUMN_NAME "`, `" QUERY_ID_COLUMN_NAME "`, `" QUERY_TYPE_COLUMN_NAME "`, `" LAST_SEEN_AT_COLUMN_NAME "`,\n" - "`" RETRY_COUNTER_COLUMN_NAME "`, `" RETRY_COUNTER_UPDATE_COLUMN_NAME "`, `" HOST_NAME_COLUMN_NAME "`, `" OWNER_COLUMN_NAME "`)\n" - "VALUES\n" - " ($scope, $query_id, $query_type, $zero_timestamp, 0, $now, \"\", \"\");" + "INSERT INTO `" PENDING_SMALL_TABLE_NAME "`\n" + "(`" SCOPE_COLUMN_NAME "`, `" QUERY_ID_COLUMN_NAME "`, `" QUERY_TYPE_COLUMN_NAME "`, `" LAST_SEEN_AT_COLUMN_NAME "`,\n" + "`" RETRY_COUNTER_COLUMN_NAME "`, `" RETRY_COUNTER_UPDATE_COLUMN_NAME "`, `" HOST_NAME_COLUMN_NAME "`, `" OWNER_COLUMN_NAME "`)\n" + "VALUES\n" + " ($scope, $query_id, $query_type, $zero_timestamp, 0, $now, \"\", \"\");" ); } @@ -292,8 +292,8 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvListQueries if (IsSuperUser(user)) { permissions.SetAll(); } - - const TString pageToken = request.page_token(); + + const TString pageToken = request.page_token(); const int byteSize = request.ByteSize(); const int64_t limit = request.limit(); @@ -312,10 +312,10 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvListQueries const TDuration delta = TInstant::Now() - startTime; SendResponseIssues<TEvControlPlaneStorage::TEvListQueriesResponse>(ev->Sender, issues, ev->Cookie, delta, requestCounters); LWPROBE(ListQueriesRequest, scope, user, delta, byteSize, false); - return; + return; } - TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "ListQueries"); + TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "ListQueries"); queryBuilder.AddString("scope", scope); queryBuilder.AddString("last_query", pageToken); queryBuilder.AddTimestamp("now", TInstant::Now()); @@ -412,8 +412,8 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvListQueries ythrow TControlPlaneStorageException(TIssuesIds::INTERNAL_ERROR) << "Error parsing proto message for query. Please contact internal support"; } YandexQuery::BriefQuery briefQuery; - const auto lastJobId = query.meta().last_job_id(); - query.mutable_meta()->set_last_job_id(lastJobId + "-" + query.meta().common().id()); + const auto lastJobId = query.meta().last_job_id(); + query.mutable_meta()->set_last_job_id(lastJobId + "-" + query.meta().common().id()); *briefQuery.mutable_meta() = query.meta(); briefQuery.set_name(query.content().name()); briefQuery.set_type(query.content().type()); @@ -478,10 +478,10 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvDescribeQue const TDuration delta = TInstant::Now() - startTime; SendResponseIssues<TEvControlPlaneStorage::TEvDescribeQueryResponse>(ev->Sender, issues, ev->Cookie, delta, requestCounters); LWPROBE(DescribeQueryRequest, scope, user, queryId, delta, byteSize, false); - return; + return; } - TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "DescribeQuery"); + TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "DescribeQuery"); queryBuilder.AddString("scope", scope); queryBuilder.AddString("query_id", queryId); queryBuilder.AddTimestamp("now", TInstant::Now()); @@ -507,9 +507,9 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvDescribeQue ythrow TControlPlaneStorageException(TIssuesIds::INTERNAL_ERROR) << "Error parsing proto message for query. Please contact internal support"; } - const auto lastJobId = result.query().meta().last_job_id(); - result.mutable_query()->mutable_meta()->set_last_job_id(lastJobId + "-" + result.query().meta().common().id()); - + const auto lastJobId = result.query().meta().last_job_id(); + result.mutable_query()->mutable_meta()->set_last_job_id(lastJobId + "-" + result.query().meta().common().id()); + const auto queryVisibility = result.query().content().acl().visibility(); const auto queryUser = result.query().meta().common().created_by(); const bool hasViewAccess = HasViewAccess(permissions, queryVisibility, queryUser, user); @@ -576,7 +576,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvGetQuerySta return; } - TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "GetQueryStatus"); + TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "GetQueryStatus"); queryBuilder.AddString("scope", scope); queryBuilder.AddString("query_id", queryId); queryBuilder.AddTimestamp("now", TInstant::Now()); @@ -679,15 +679,15 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvModifyQuery const TDuration delta = TInstant::Now() - startTime; SendResponseIssues<TEvControlPlaneStorage::TEvModifyQueryResponse>(ev->Sender, issues, ev->Cookie, delta, requestCounters); LWPROBE(ModifyQueryRequest, scope, user, queryId, delta, byteSize, false); - return; + return; } const TString idempotencyKey = request.idempotency_key(); - std::shared_ptr<std::pair<YandexQuery::ModifyQueryResult, TAuditDetails<YandexQuery::Query>>> response = - std::make_shared<std::pair<YandexQuery::ModifyQueryResult, TAuditDetails<YandexQuery::Query>>>(); + std::shared_ptr<std::pair<YandexQuery::ModifyQueryResult, TAuditDetails<YandexQuery::Query>>> response = + std::make_shared<std::pair<YandexQuery::ModifyQueryResult, TAuditDetails<YandexQuery::Query>>>(); - TSqlQueryBuilder readQueryBuilder(YdbConnection->TablePathPrefix, "ModifyQuery(read)"); + TSqlQueryBuilder readQueryBuilder(YdbConnection->TablePathPrefix, "ModifyQuery(read)"); readQueryBuilder.AddString("scope", scope); readQueryBuilder.AddString("query_id", queryId); readQueryBuilder.AddTimestamp("now", TInstant::Now()); @@ -700,7 +700,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvModifyQuery "SELECT `" CONNECTION_ID_COLUMN_NAME "`, `" CONNECTION_COLUMN_NAME "` FROM `" CONNECTIONS_TABLE_NAME "`\n" "WHERE `" SCOPE_COLUMN_NAME "` = $scope AND (`" VISIBILITY_COLUMN_NAME "` = $scope_visibility OR `" USER_COLUMN_NAME "` = $user);\n" ); - + // user bindings readQueryBuilder.AddText( "SELECT `" BINDING_ID_COLUMN_NAME "`, `" BINDING_COLUMN_NAME "` FROM `" BINDINGS_TABLE_NAME "`\n" @@ -709,7 +709,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvModifyQuery } readQueryBuilder.AddText( - "SELECT `" QUERY_COLUMN_NAME "`, `" INTERNAL_COLUMN_NAME "`, `" RESULT_ID_COLUMN_NAME "` FROM `" QUERIES_TABLE_NAME "`\n" + "SELECT `" QUERY_COLUMN_NAME "`, `" INTERNAL_COLUMN_NAME "`, `" RESULT_ID_COLUMN_NAME "` FROM `" QUERIES_TABLE_NAME "`\n" "WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id AND (`" EXPIRE_AT_COLUMN_NAME "` is NULL OR `" EXPIRE_AT_COLUMN_NAME "` > $now);" ); @@ -736,8 +736,8 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvModifyQuery ythrow TControlPlaneStorageException(TIssuesIds::INTERNAL_ERROR) << "Error parsing proto message for query internal. Please contact internal support"; } - const TString resultId = request.execute_mode() == YandexQuery::SAVE ? parser.ColumnParser(RESULT_ID_COLUMN_NAME).GetOptionalString().GetOrElse("") : ""; - + const TString resultId = request.execute_mode() == YandexQuery::SAVE ? parser.ColumnParser(RESULT_ID_COLUMN_NAME).GetOptionalString().GetOrElse("") : ""; + const auto queryVisibility = query.content().acl().visibility(); const auto queryUser = query.meta().common().created_by(); const bool hasManageAccess = HasManageAccess(permissions, queryVisibility, queryUser, user); @@ -781,11 +781,11 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvModifyQuery internal.set_token(token); } if (request.execute_mode() != YandexQuery::SAVE) { - if (request.state_load_mode() != YandexQuery::StateLoadMode::STATE_LOAD_MODE_UNSPECIFIED) { - internal.set_state_load_mode(request.state_load_mode()); - } - internal.mutable_disposition()->CopyFrom(request.disposition()); - + if (request.state_load_mode() != YandexQuery::StateLoadMode::STATE_LOAD_MODE_UNSPECIFIED) { + internal.set_state_load_mode(request.state_load_mode()); + } + internal.mutable_disposition()->CopyFrom(request.disposition()); + internal.clear_binding(); internal.clear_connection(); // TODO: move to run actor priority selection @@ -862,7 +862,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvModifyQuery response->second.After.ConstructInPlace().CopyFrom(query); response->second.CloudId = internal.cloud_id(); - TSqlQueryBuilder writeQueryBuilder(YdbConnection->TablePathPrefix, "ModifyQuery(write)"); + TSqlQueryBuilder writeQueryBuilder(YdbConnection->TablePathPrefix, "ModifyQuery(write)"); writeQueryBuilder.AddString("scope", scope); writeQueryBuilder.AddString("query_id", queryId); writeQueryBuilder.AddUint64("max_count_jobs", Config.Proto.GetMaxCountJobs()); @@ -905,34 +905,34 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvModifyQuery writeQueryBuilder.AddTimestamp("now", TInstant::Now()); // insert job writeQueryBuilder.AddText( - "UPSERT INTO `" JOBS_TABLE_NAME "` (`" SCOPE_COLUMN_NAME "`, `" QUERY_ID_COLUMN_NAME "`, `" JOB_ID_COLUMN_NAME "`, `" JOB_COLUMN_NAME "`, `" USER_COLUMN_NAME "`, `" VISIBILITY_COLUMN_NAME "`) VALUES\n" - " ($scope, $query_id, $job_id, $job, $user, $visibility);\n" + "UPSERT INTO `" JOBS_TABLE_NAME "` (`" SCOPE_COLUMN_NAME "`, `" QUERY_ID_COLUMN_NAME "`, `" JOB_ID_COLUMN_NAME "`, `" JOB_COLUMN_NAME "`, `" USER_COLUMN_NAME "`, `" VISIBILITY_COLUMN_NAME "`) VALUES\n" + " ($scope, $query_id, $job_id, $job, $user, $visibility);\n" ); // insert pending small writeQueryBuilder.AddText( "INSERT INTO `" PENDING_SMALL_TABLE_NAME "`\n" - " (`" SCOPE_COLUMN_NAME "`, `" QUERY_ID_COLUMN_NAME "`, `" LAST_SEEN_AT_COLUMN_NAME "`, `" RETRY_COUNTER_COLUMN_NAME "`, \n" - " `" RETRY_COUNTER_UPDATE_COLUMN_NAME "`, `" QUERY_TYPE_COLUMN_NAME "`, `" HOST_NAME_COLUMN_NAME "`, `" OWNER_COLUMN_NAME "`)\n" + " (`" SCOPE_COLUMN_NAME "`, `" QUERY_ID_COLUMN_NAME "`, `" LAST_SEEN_AT_COLUMN_NAME "`, `" RETRY_COUNTER_COLUMN_NAME "`, \n" + " `" RETRY_COUNTER_UPDATE_COLUMN_NAME "`, `" QUERY_TYPE_COLUMN_NAME "`, `" HOST_NAME_COLUMN_NAME "`, `" OWNER_COLUMN_NAME "`)\n" "VALUES\n" - " ($scope, $query_id, $zero_timestamp, 0, $now, $query_type, \"\", \"\");\n" + " ($scope, $query_id, $zero_timestamp, 0, $now, $query_type, \"\", \"\");\n" ); } writeQueryBuilder.AddText( "DELETE FROM `" JOBS_TABLE_NAME "` ON\n" "SELECT * FROM $to_delete;\n" - "UPDATE `" QUERIES_TABLE_NAME "` SET \n" - " `" LAST_JOB_ID_COLUMN_NAME "` = $job_id, `" VISIBILITY_COLUMN_NAME "` = $visibility, `" AUTOMATIC_COLUMN_NAME "` = $automatic,\n" - " `" NAME_COLUMN_NAME "` = $name, `" EXECUTE_MODE_COLUMN_NAME "` = $execute_mode,\n" - " `" REVISION_COLUMN_NAME "` = $revision, `" STATUS_COLUMN_NAME "` = $status, " - ); - writeQueryBuilder.AddText( - (request.execute_mode() != YandexQuery::SAVE ? "`" INTERNAL_COLUMN_NAME "` = $internal,\n" : TString{"\n"}) - ); - writeQueryBuilder.AddText( - " `" QUERY_TYPE_COLUMN_NAME "` = $query_type, `" QUERY_COLUMN_NAME "` = $query,\n" - " `" RESULT_ID_COLUMN_NAME "` = $result_id, `" META_REVISION_COLUMN_NAME "` = `" META_REVISION_COLUMN_NAME "` + 1\n" + "UPDATE `" QUERIES_TABLE_NAME "` SET \n" + " `" LAST_JOB_ID_COLUMN_NAME "` = $job_id, `" VISIBILITY_COLUMN_NAME "` = $visibility, `" AUTOMATIC_COLUMN_NAME "` = $automatic,\n" + " `" NAME_COLUMN_NAME "` = $name, `" EXECUTE_MODE_COLUMN_NAME "` = $execute_mode,\n" + " `" REVISION_COLUMN_NAME "` = $revision, `" STATUS_COLUMN_NAME "` = $status, " + ); + writeQueryBuilder.AddText( + (request.execute_mode() != YandexQuery::SAVE ? "`" INTERNAL_COLUMN_NAME "` = $internal,\n" : TString{"\n"}) + ); + writeQueryBuilder.AddText( + " `" QUERY_TYPE_COLUMN_NAME "` = $query_type, `" QUERY_COLUMN_NAME "` = $query,\n" + " `" RESULT_ID_COLUMN_NAME "` = $result_id, `" META_REVISION_COLUMN_NAME "` = `" META_REVISION_COLUMN_NAME "` + 1\n" "WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id;" ); @@ -1033,10 +1033,10 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvDeleteQuery const TDuration delta = TInstant::Now() - startTime; SendResponseIssues<TEvControlPlaneStorage::TEvDeleteQueryResponse>(ev->Sender, issues, ev->Cookie, delta, requestCounters); LWPROBE(DeleteQueryRequest, scope, queryId, user, delta, byteSize, false); - return; + return; } std::shared_ptr<std::pair<YandexQuery::DeleteQueryResult, TAuditDetails<YandexQuery::Query>>> response = std::make_shared<std::pair<YandexQuery::DeleteQueryResult, TAuditDetails<YandexQuery::Query>>>(); - TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "DeleteQuery"); + TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "DeleteQuery"); queryBuilder.AddString("scope", scope); queryBuilder.AddString("query_id", queryId); @@ -1044,8 +1044,8 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvDeleteQuery queryBuilder.AddText( "DELETE FROM `" JOBS_TABLE_NAME "`\n" "WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id;\n" - "DELETE FROM `" PENDING_SMALL_TABLE_NAME "`\n" - "WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id;\n" + "DELETE FROM `" PENDING_SMALL_TABLE_NAME "`\n" + "WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id;\n" "DELETE FROM `" QUERIES_TABLE_NAME "`\n" "WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id;" ); @@ -1134,7 +1134,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvControlQuer << "ControlQueryRequest: " << NKikimr::MaskTicket(token) << " " << request.DebugString()); - + NYql::TIssues issues = ValidateEvent(ev); if (issues) { CPS_LOG_D(MakeLogPrefix(scope, user, queryId) @@ -1145,12 +1145,12 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvControlQuer const TDuration delta = TInstant::Now() - startTime; SendResponseIssues<TEvControlPlaneStorage::TEvControlQueryResponse>(ev->Sender, issues, ev->Cookie, delta, requestCounters); LWPROBE(ControlQueryRequest, scope, user, queryId, delta, byteSize, false); - return; + return; } std::shared_ptr<std::pair<YandexQuery::ControlQueryResult, TAuditDetails<YandexQuery::Query>>> response = std::make_shared<std::pair<YandexQuery::ControlQueryResult, TAuditDetails<YandexQuery::Query>>>(); - TSqlQueryBuilder readQueryBuilder(YdbConnection->TablePathPrefix, "ControlQuery(read)"); + TSqlQueryBuilder readQueryBuilder(YdbConnection->TablePathPrefix, "ControlQuery(read)"); readQueryBuilder.AddString("scope", scope); readQueryBuilder.AddString("query_id", queryId); @@ -1160,11 +1160,11 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvControlQuer "$job_id = SELECT `" LAST_JOB_ID_COLUMN_NAME "` FROM $selected;\n" "SELECT * FROM $selected;" "SELECT `" JOB_COLUMN_NAME "` FROM `" JOBS_TABLE_NAME "`\n" - "WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id AND `" JOB_ID_COLUMN_NAME "` = $job_id;\n" + "WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id AND `" JOB_ID_COLUMN_NAME "` = $job_id;\n" ); auto prepareParams = [=, config=Config](const TVector<TResultSet>& resultSets) { - if (resultSets.size() != 2) { + if (resultSets.size() != 2) { ythrow TControlPlaneStorageException(TIssuesIds::INTERNAL_ERROR) << "Result set size is not equal to 2 but equal " << resultSets.size() << ". Please contact internal support"; } @@ -1172,7 +1172,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvControlQuer YandexQuery::Query query; YandexQuery::Internal::QueryInternal queryInternal; { - TResultSetParser parser(resultSets[0]); + TResultSetParser parser(resultSets[0]); if (!parser.TryNextRow()) { ythrow TControlPlaneStorageException(TIssuesIds::ACCESS_DENIED) << "Query does not exist or permission denied. Please check the id query or your access rights"; } @@ -1189,7 +1189,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvControlQuer YandexQuery::Job job; TString jobId; { - TResultSetParser parser(resultSets[1]); + TResultSetParser parser(resultSets[1]); if (!parser.TryNextRow()) { ythrow TControlPlaneStorageException(TIssuesIds::ACCESS_DENIED) << "Job does not exist or permission denied. Please check the id query or your access rights"; } @@ -1208,25 +1208,25 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvControlQuer queryInternal.set_action(action); - auto& metaQuery = *query.mutable_meta(); - auto& commonQuery = *metaQuery.mutable_common(); - commonQuery.set_revision(commonQuery.revision() + 1); - commonQuery.set_modified_by(user); + auto& metaQuery = *query.mutable_meta(); + auto& commonQuery = *metaQuery.mutable_common(); + commonQuery.set_revision(commonQuery.revision() + 1); + commonQuery.set_modified_by(user); *commonQuery.mutable_modified_at() = NProtoInterop::CastToProto(now); - + if (action == YandexQuery::ABORT || action == YandexQuery::ABORT_GRACEFULLY) { const bool isValidStatusForAbort = IsIn({ YandexQuery::QueryMeta::STARTING, YandexQuery::QueryMeta::RESUMING, YandexQuery::QueryMeta::RUNNING, YandexQuery::QueryMeta::PAUSING - }, metaQuery.status()); - + }, metaQuery.status()); + if (isValidStatusForAbort) { - metaQuery.set_status(YandexQuery::QueryMeta::ABORTING_BY_USER); + metaQuery.set_status(YandexQuery::QueryMeta::ABORTING_BY_USER); metaQuery.set_aborted_by(user); } else { - ythrow TControlPlaneStorageException(TIssuesIds::INTERNAL_ERROR) << "Conversion from status " << YandexQuery::QueryMeta::ComputeStatus_Name(metaQuery.status()) << " to " << YandexQuery::QueryMeta::ComputeStatus_Name(YandexQuery::QueryMeta::ABORTING_BY_USER) << " is not possible. Please wait for the previous operation to be completed"; + ythrow TControlPlaneStorageException(TIssuesIds::INTERNAL_ERROR) << "Conversion from status " << YandexQuery::QueryMeta::ComputeStatus_Name(metaQuery.status()) << " to " << YandexQuery::QueryMeta::ComputeStatus_Name(YandexQuery::QueryMeta::ABORTING_BY_USER) << " is not possible. Please wait for the previous operation to be completed"; } } @@ -1234,34 +1234,34 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvControlQuer const bool isValidStatusForPause = IsIn({ YandexQuery::QueryMeta::RESUMING, YandexQuery::QueryMeta::RUNNING - }, metaQuery.status()); - + }, metaQuery.status()); + if (isValidStatusForPause) { - metaQuery.set_status(YandexQuery::QueryMeta::PAUSING); + metaQuery.set_status(YandexQuery::QueryMeta::PAUSING); metaQuery.set_paused_by(user); } else { - ythrow TControlPlaneStorageException(TIssuesIds::BAD_REQUEST) << "Conversion from status " << YandexQuery::QueryMeta::ComputeStatus_Name(metaQuery.status()) << " to " << YandexQuery::QueryMeta::ComputeStatus_Name(YandexQuery::QueryMeta::PAUSING) << " is not possible. Please wait for the previous operation to be completed"; + ythrow TControlPlaneStorageException(TIssuesIds::BAD_REQUEST) << "Conversion from status " << YandexQuery::QueryMeta::ComputeStatus_Name(metaQuery.status()) << " to " << YandexQuery::QueryMeta::ComputeStatus_Name(YandexQuery::QueryMeta::PAUSING) << " is not possible. Please wait for the previous operation to be completed"; } } if (action == YandexQuery::RESUME) { - const bool isValidStatusForResume = metaQuery.status() == YandexQuery::QueryMeta::PAUSED; + const bool isValidStatusForResume = metaQuery.status() == YandexQuery::QueryMeta::PAUSED; if (isValidStatusForResume) { - metaQuery.set_status(YandexQuery::QueryMeta::RESUMING); + metaQuery.set_status(YandexQuery::QueryMeta::RESUMING); } else { - ythrow TControlPlaneStorageException(TIssuesIds::BAD_REQUEST) << "Conversion from status " << YandexQuery::QueryMeta::ComputeStatus_Name(metaQuery.status()) << " to " << YandexQuery::QueryMeta::ComputeStatus_Name(YandexQuery::QueryMeta::RESUMING) << " is not possible. Please wait for the previous operation to be completed"; + ythrow TControlPlaneStorageException(TIssuesIds::BAD_REQUEST) << "Conversion from status " << YandexQuery::QueryMeta::ComputeStatus_Name(metaQuery.status()) << " to " << YandexQuery::QueryMeta::ComputeStatus_Name(YandexQuery::QueryMeta::RESUMING) << " is not possible. Please wait for the previous operation to be completed"; } } - job.mutable_query_meta()->CopyFrom(metaQuery); + job.mutable_query_meta()->CopyFrom(metaQuery); job.mutable_meta()->set_revision(job.meta().revision() + 1); job.mutable_meta()->set_modified_by(user); *job.mutable_meta()->mutable_modified_at() = NProtoInterop::CastToProto(now); - + response->second.After.ConstructInPlace().CopyFrom(query); response->second.CloudId = queryInternal.cloud_id(); - TSqlQueryBuilder writeQueryBuilder(YdbConnection->TablePathPrefix, "ControlQuery(write)"); + TSqlQueryBuilder writeQueryBuilder(YdbConnection->TablePathPrefix, "ControlQuery(write)"); writeQueryBuilder.AddString("scope", scope); writeQueryBuilder.AddString("job", job.SerializeAsString()); writeQueryBuilder.AddString("job_id", jobId); @@ -1274,10 +1274,10 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvControlQuer InsertIdempotencyKey(writeQueryBuilder, scope, idempotencyKey, response->first.SerializeAsString(), now + Config.IdempotencyKeyTtl); writeQueryBuilder.AddText( "UPDATE `" JOBS_TABLE_NAME "` SET `" JOB_COLUMN_NAME "` = $job\n" - "WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id AND `" JOB_ID_COLUMN_NAME "` = $job_id;\n" + "WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id AND `" JOB_ID_COLUMN_NAME "` = $job_id;\n" "UPDATE `" QUERIES_TABLE_NAME "` SET `" QUERY_COLUMN_NAME "` = $query, `" REVISION_COLUMN_NAME "` = $revision, `" STATUS_COLUMN_NAME "` = $status, `" INTERNAL_COLUMN_NAME "` = $internal, `" META_REVISION_COLUMN_NAME "` = `" META_REVISION_COLUMN_NAME "` + 1\n" "WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id;\n" - "UPDATE `" PENDING_SMALL_TABLE_NAME "` SET `" HOST_NAME_COLUMN_NAME "` = \"\"\n" + "UPDATE `" PENDING_SMALL_TABLE_NAME "` SET `" HOST_NAME_COLUMN_NAME "` = \"\"\n" "WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id;" ); @@ -1373,7 +1373,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvGetResultDa return; } - TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "GetResultData"); + TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "GetResultData"); queryBuilder.AddString("scope", scope); queryBuilder.AddString("query_id", queryId); queryBuilder.AddTimestamp("now", TInstant::Now()); @@ -1388,7 +1388,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvGetResultDa "$result_id = SELECT `" RESULT_ID_COLUMN_NAME "` FROM $query_info\n" "WHERE `" RESULT_SETS_EXPIRE_AT_COLUMN_NAME "` >= $now;\n" "SELECT `" RESULT_SET_ID_COLUMN_NAME "`, `" RESULT_SET_COLUMN_NAME "`, `" ROW_ID_COLUMN_NAME "` FROM `" RESULT_SETS_TABLE_NAME "`\n" - "WHERE `" RESULT_ID_COLUMN_NAME "` = $result_id AND `" RESULT_SET_ID_COLUMN_NAME "` = $result_set_index AND `" ROW_ID_COLUMN_NAME "` >= $offset\n" + "WHERE `" RESULT_ID_COLUMN_NAME "` = $result_id AND `" RESULT_SET_ID_COLUMN_NAME "` = $result_set_index AND `" ROW_ID_COLUMN_NAME "` >= $offset\n" "ORDER BY `" ROW_ID_COLUMN_NAME "`\n" "LIMIT $limit;\n" ); @@ -1435,7 +1435,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvGetResultDa if (!resultSetsExpireAtParser) { ythrow TControlPlaneStorageException(TIssuesIds::BAD_REQUEST) << "Result doesn't exist"; } - + if (*resultSetsExpireAtParser < TInstant::Now()) { ythrow TControlPlaneStorageException(TIssuesIds::EXPIRED) << "Result removed by TTL"; } @@ -1485,11 +1485,11 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvListJobsReq if (request.has_filter() && request.filter().query_id()) { queryId = request.filter().query_id(); } - - auto splittedPageToken = SplitId(request.page_token()); - const auto jobIdToken = std::move(splittedPageToken.first); - const auto queryIdToken = std::move(splittedPageToken.second); - + + auto splittedPageToken = SplitId(request.page_token()); + const auto jobIdToken = std::move(splittedPageToken.first); + const auto queryIdToken = std::move(splittedPageToken.second); + const int byteSize = ev->Get()->Request.ByteSize(); const TString token = ev->Get()->Token; TPermissions permissions = Config.Proto.GetEnablePermissions() @@ -1514,19 +1514,19 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvListJobsReq const TDuration delta = TInstant::Now() - startTime; SendResponseIssues<TEvControlPlaneStorage::TEvListJobsResponse>(ev->Sender, issues, ev->Cookie, delta, requestCounters); LWPROBE(ListJobsRequest, scope, user, queryId, delta, byteSize, false); - return; + return; } - TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "ListJobs"); + TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "ListJobs"); queryBuilder.AddString("scope", scope); - queryBuilder.AddString("last_job", jobIdToken); - queryBuilder.AddString("last_query", queryIdToken); + queryBuilder.AddString("last_job", jobIdToken); + queryBuilder.AddString("last_query", queryIdToken); queryBuilder.AddTimestamp("now", TInstant::Now()); queryBuilder.AddUint64("limit", limit + 1); queryBuilder.AddText( "SELECT `" JOB_ID_COLUMN_NAME "`, `" JOB_COLUMN_NAME "` FROM `" JOBS_TABLE_NAME "`\n" - "WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` >= $last_query\n" - "AND `" JOB_ID_COLUMN_NAME "` >= $last_job AND (`" EXPIRE_AT_COLUMN_NAME "` is NULL OR `" EXPIRE_AT_COLUMN_NAME "` > $now) " + "WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` >= $last_query\n" + "AND `" JOB_ID_COLUMN_NAME "` >= $last_job AND (`" EXPIRE_AT_COLUMN_NAME "` is NULL OR `" EXPIRE_AT_COLUMN_NAME "` > $now) " ); TString filter; @@ -1571,9 +1571,9 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvListJobsReq if (!job.ParseFromString(*parser.ColumnParser(JOB_COLUMN_NAME).GetOptionalString())) { ythrow TControlPlaneStorageException(TIssuesIds::INTERNAL_ERROR) << "Error parsing proto message for job. Please contact internal support"; } - const TString mergedId = job.meta().id() + "-" + job.query_meta().common().id(); - job.mutable_meta()->set_id(mergedId); - job.mutable_query_meta()->set_last_job_id(mergedId); + const TString mergedId = job.meta().id() + "-" + job.query_meta().common().id(); + job.mutable_meta()->set_id(mergedId); + job.mutable_query_meta()->set_last_job_id(mergedId); YandexQuery::BriefJob briefJob; *briefJob.mutable_meta() = job.meta(); *briefJob.mutable_query_meta() = job.query_meta(); @@ -1585,8 +1585,8 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvListJobsReq } if (result.job_size() == limit + 1) { - result.set_next_page_token( - result.job(result.job_size() - 1).meta().id() + "-" + result.job(result.job_size() - 1).query_meta().common().id()); + result.set_next_page_token( + result.job(result.job_size() - 1).meta().id() + "-" + result.job(result.job_size() - 1).query_meta().common().id()); result.mutable_job()->RemoveLast(); } return result; @@ -1618,11 +1618,11 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvDescribeJob const YandexQuery::DescribeJobRequest& request = ev->Get()->Request; const TString scope = ev->Get()->Scope; const TString user = ev->Get()->User; - - auto splittedId = SplitId(request.job_id()); - const auto jobId = std::move(splittedId.first); - const auto queryId = std::move(splittedId.second); - + + auto splittedId = SplitId(request.job_id()); + const auto jobId = std::move(splittedId.first); + const auto queryId = std::move(splittedId.second); + const int byteSize = ev->Get()->Request.ByteSize(); const TString token = ev->Get()->Token; TPermissions permissions = Config.Proto.GetEnablePermissions() @@ -1648,23 +1648,23 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvDescribeJob return; } - TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "DescribeJob"); + TSqlQueryBuilder queryBuilder(YdbConnection->TablePathPrefix, "DescribeJob"); queryBuilder.AddString("scope", scope); queryBuilder.AddString("job_id", jobId); - queryBuilder.AddString("query_id", queryId); + queryBuilder.AddString("query_id", queryId); queryBuilder.AddTimestamp("now", TInstant::Now()); queryBuilder.AddText( "SELECT `" JOB_COLUMN_NAME "`, `" VISIBILITY_COLUMN_NAME "` FROM `" JOBS_TABLE_NAME "`\n" - "WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id\n" - " AND `" JOB_ID_COLUMN_NAME "` = $job_id AND (`" EXPIRE_AT_COLUMN_NAME "` is NULL OR `" EXPIRE_AT_COLUMN_NAME "` > $now);\n" + "WHERE `" SCOPE_COLUMN_NAME "` = $scope AND `" QUERY_ID_COLUMN_NAME "` = $query_id\n" + " AND `" JOB_ID_COLUMN_NAME "` = $job_id AND (`" EXPIRE_AT_COLUMN_NAME "` is NULL OR `" EXPIRE_AT_COLUMN_NAME "` > $now);\n" ); const auto query = queryBuilder.Build(); auto debugInfo = Config.Proto.GetEnableDebugMode() ? std::make_shared<TDebugInfo>() : TDebugInfoPtr{}; auto [result, resultSets] = Read(query.Sql, query.Params, requestCounters, debugInfo); - auto prepare = [=, id=request.job_id(), resultSets=resultSets] { + auto prepare = [=, id=request.job_id(), resultSets=resultSets] { if (resultSets->size() != 1) { ythrow TControlPlaneStorageException(TIssuesIds::INTERNAL_ERROR) << "Result set size is not equal to 1 but equal " << resultSets->size() << ". Please contact internal support"; } @@ -1678,7 +1678,7 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvDescribeJob ythrow TControlPlaneStorageException(TIssuesIds::INTERNAL_ERROR) << "Error parsing proto message for job. Please contact internal support"; } auto visibility = static_cast<YandexQuery::Acl::Visibility>(*parser.ColumnParser(VISIBILITY_COLUMN_NAME).GetOptionalInt64()); - result.mutable_job()->mutable_meta()->set_id(id); + result.mutable_job()->mutable_meta()->set_id(id); bool hasViewAccces = HasViewAccess(permissions, visibility, result.job().meta().created_by(), user); if (!hasViewAccces) { @@ -1704,4 +1704,4 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvDescribeJob }); } -} // NYq +} // NYq diff --git a/ydb/core/yq/libs/db_resolver/db_async_resolver.h b/ydb/core/yq/libs/db_resolver/db_async_resolver.h index 02f6f3681a..ae81326839 100644 --- a/ydb/core/yq/libs/db_resolver/db_async_resolver.h +++ b/ydb/core/yq/libs/db_resolver/db_async_resolver.h @@ -1,22 +1,22 @@ -#pragma once -#include <library/cpp/threading/future/future.h> +#pragma once +#include <library/cpp/threading/future/future.h> #include <ydb/core/yq/libs/events/events.h> - - -namespace NYq { - -struct TResolveParams { //TODO: remove - THashMap<std::pair<TString, DatabaseType>, TEvents::TDatabaseAuth> Ids; - TString TraceId; -}; - -class IDatabaseAsyncResolver { -public: - using TEndpoint = TEvents::TEvEndpointResponse::TEndpoint; - - virtual NThreading::TFuture<THashMap<std::pair<TString, DatabaseType>, TEndpoint>> ResolveIds(const TResolveParams& params) const = 0; - - virtual ~IDatabaseAsyncResolver() = default; -}; - -} // NYq + + +namespace NYq { + +struct TResolveParams { //TODO: remove + THashMap<std::pair<TString, DatabaseType>, TEvents::TDatabaseAuth> Ids; + TString TraceId; +}; + +class IDatabaseAsyncResolver { +public: + using TEndpoint = TEvents::TEvEndpointResponse::TEndpoint; + + virtual NThreading::TFuture<THashMap<std::pair<TString, DatabaseType>, TEndpoint>> ResolveIds(const TResolveParams& params) const = 0; + + virtual ~IDatabaseAsyncResolver() = default; +}; + +} // NYq diff --git a/ydb/core/yq/libs/db_resolver/db_async_resolver_impl.cpp b/ydb/core/yq/libs/db_resolver/db_async_resolver_impl.cpp index c798cc0d0e..253c205cee 100644 --- a/ydb/core/yq/libs/db_resolver/db_async_resolver_impl.cpp +++ b/ydb/core/yq/libs/db_resolver/db_async_resolver_impl.cpp @@ -1,41 +1,41 @@ -#include "db_async_resolver_impl.h" - -namespace NYq { - -using TEndpoint = TEvents::TEvEndpointResponse::TEndpoint; - -TDatabaseAsyncResolver::TDatabaseAsyncResolver( - NActors::TActorSystem* actorSystem, - const NActors::TActorId& recipient, - const TString& ydbMvpEndpoint, - const TString& mdbGateway, - const bool mdbTransformHost) - : ActorSystem(actorSystem) - , Recipient(recipient) - , YdbMvpEndpoint(ydbMvpEndpoint) - , MdbGateway(mdbGateway) - , MdbTransformHost(mdbTransformHost) -{} - -NThreading::TFuture<THashMap<std::pair<TString, DatabaseType>, TEndpoint>> TDatabaseAsyncResolver::ResolveIds(const TResolveParams& params) const { - auto promise = NThreading::NewPromise<THashMap<std::pair<TString, DatabaseType>, TEndpoint>>(); - auto callback = MakeHolder<NYql::TRichActorFutureCallback<TEvents::TEvEndpointResponse>>( - [promise] (TAutoPtr<NActors::TEventHandle<TEvents::TEvEndpointResponse>>& event) mutable { - promise.SetValue(event->Get()->DatabaseId2Endpoint); - }, - [promise] () mutable { - //TODO add logs - promise.SetException("Error occurred on resolving ids. Message was undelivered."); - }, - TDuration::Seconds(10) - ); - - NActors::TActorId callbackId = ActorSystem->Register(callback.Release()); - - ActorSystem->Send(new NActors::IEventHandle(Recipient, callbackId, - new TEvents::TEvEndpointRequest(params.Ids, YdbMvpEndpoint, MdbGateway, - params.TraceId, MdbTransformHost))); - return promise.GetFuture(); -} - -} // NYq +#include "db_async_resolver_impl.h" + +namespace NYq { + +using TEndpoint = TEvents::TEvEndpointResponse::TEndpoint; + +TDatabaseAsyncResolver::TDatabaseAsyncResolver( + NActors::TActorSystem* actorSystem, + const NActors::TActorId& recipient, + const TString& ydbMvpEndpoint, + const TString& mdbGateway, + const bool mdbTransformHost) + : ActorSystem(actorSystem) + , Recipient(recipient) + , YdbMvpEndpoint(ydbMvpEndpoint) + , MdbGateway(mdbGateway) + , MdbTransformHost(mdbTransformHost) +{} + +NThreading::TFuture<THashMap<std::pair<TString, DatabaseType>, TEndpoint>> TDatabaseAsyncResolver::ResolveIds(const TResolveParams& params) const { + auto promise = NThreading::NewPromise<THashMap<std::pair<TString, DatabaseType>, TEndpoint>>(); + auto callback = MakeHolder<NYql::TRichActorFutureCallback<TEvents::TEvEndpointResponse>>( + [promise] (TAutoPtr<NActors::TEventHandle<TEvents::TEvEndpointResponse>>& event) mutable { + promise.SetValue(event->Get()->DatabaseId2Endpoint); + }, + [promise] () mutable { + //TODO add logs + promise.SetException("Error occurred on resolving ids. Message was undelivered."); + }, + TDuration::Seconds(10) + ); + + NActors::TActorId callbackId = ActorSystem->Register(callback.Release()); + + ActorSystem->Send(new NActors::IEventHandle(Recipient, callbackId, + new TEvents::TEvEndpointRequest(params.Ids, YdbMvpEndpoint, MdbGateway, + params.TraceId, MdbTransformHost))); + return promise.GetFuture(); +} + +} // NYq diff --git a/ydb/core/yq/libs/db_resolver/db_async_resolver_impl.h b/ydb/core/yq/libs/db_resolver/db_async_resolver_impl.h index 1bfe0b5d76..cb13cde383 100644 --- a/ydb/core/yq/libs/db_resolver/db_async_resolver_impl.h +++ b/ydb/core/yq/libs/db_resolver/db_async_resolver_impl.h @@ -1,28 +1,28 @@ -#pragma once -#include "db_async_resolver.h" +#pragma once +#include "db_async_resolver.h" #include <ydb/library/yql/providers/dq/actors/actor_helpers.h> - -namespace NYq { - -using TEndpoint = TEvents::TEvEndpointResponse::TEndpoint; - -class TDatabaseAsyncResolver : public IDatabaseAsyncResolver { -public: - TDatabaseAsyncResolver( - NActors::TActorSystem* actorSystem, - const NActors::TActorId& recipient, - const TString& ydbMvpEndpoint, - const TString& mdbGateway, - const bool mdbTransformHost - ); - - NThreading::TFuture<THashMap<std::pair<TString, DatabaseType>, TEndpoint>> ResolveIds(const TResolveParams& params) const override; -private: - NActors::TActorSystem* ActorSystem; - const NActors::TActorId Recipient; - const TString YdbMvpEndpoint; - const TString MdbGateway; - const bool MdbTransformHost = false; -}; - -} // NYq + +namespace NYq { + +using TEndpoint = TEvents::TEvEndpointResponse::TEndpoint; + +class TDatabaseAsyncResolver : public IDatabaseAsyncResolver { +public: + TDatabaseAsyncResolver( + NActors::TActorSystem* actorSystem, + const NActors::TActorId& recipient, + const TString& ydbMvpEndpoint, + const TString& mdbGateway, + const bool mdbTransformHost + ); + + NThreading::TFuture<THashMap<std::pair<TString, DatabaseType>, TEndpoint>> ResolveIds(const TResolveParams& params) const override; +private: + NActors::TActorSystem* ActorSystem; + const NActors::TActorId Recipient; + const TString YdbMvpEndpoint; + const TString MdbGateway; + const bool MdbTransformHost = false; +}; + +} // NYq diff --git a/ydb/core/yq/libs/db_resolver/db_async_resolver_with_meta.cpp b/ydb/core/yq/libs/db_resolver/db_async_resolver_with_meta.cpp index 373c3825f5..4d38454f62 100644 --- a/ydb/core/yq/libs/db_resolver/db_async_resolver_with_meta.cpp +++ b/ydb/core/yq/libs/db_resolver/db_async_resolver_with_meta.cpp @@ -1,72 +1,72 @@ -#include "db_async_resolver_with_meta.h" +#include "db_async_resolver_with_meta.h" #include <ydb/core/yq/libs/common/database_token_builder.h> - -namespace NYq { - - TDatabaseAsyncResolverWithMeta::TDatabaseAsyncResolverWithMeta( - NActors::TActorSystem* actorSystem, - const NActors::TActorId& recipient, - const TString& ydbMvpEndpoint, - const TString& mdbGateway, - const bool mdbTransformHost, - const TString& traceId, - const TString& token, - const THashMap<TString, TString>& accountIdSignatures, - const THashMap<TString, YandexQuery::Connection>& connections) - : DbResolver(actorSystem, recipient, ydbMvpEndpoint, mdbGateway, mdbTransformHost) - , TraceId(traceId) - , Token(token) - , AccountIdSignatures(accountIdSignatures) - , Connections(connections) - {} - - NThreading::TFuture<THashMap<std::pair<TString, DatabaseType>, TEndpoint>> TDatabaseAsyncResolverWithMeta::ResolveIds(const TResolveParams& params) const { - return DbResolver.ResolveIds(params); - } - - TString TDatabaseAsyncResolverWithMeta::GetTraceId() const { - return TraceId; - } - - TString TDatabaseAsyncResolverWithMeta::GetToken() const { - return Token; - } - - const THashMap<TString, TString>& TDatabaseAsyncResolverWithMeta::GetAccountIdSignatures() const { - return AccountIdSignatures; - } - - void TDatabaseAsyncResolverWithMeta::TryAddDbIdToResolve( - const bool isEndpoint, - const TString& clusterName, - const TString& dbId, - const DatabaseType type, - THashMap<std::pair<TString, DatabaseType>, TEvents::TDatabaseAuth>& databaseIds) const { - if (isEndpoint) { - return; - } - const auto iter = Connections.find(clusterName); - if (iter == Connections.end()) { - return; - } - const auto& conn = iter->second; - const auto& setting = conn.content().setting(); - YandexQuery::IamAuth auth; - switch (type) { - case DatabaseType::Ydb: - auth = setting.ydb_database().auth(); - break; - case DatabaseType::ClickHouse: - auth = setting.clickhouse_cluster().auth(); - break; - case DatabaseType::DataStreams: - auth = setting.data_streams().auth(); - break; - default: - return; - } - TryAddDatabaseToResolve(auth, dbId, type, Token, AccountIdSignatures, databaseIds); - } - + +namespace NYq { + + TDatabaseAsyncResolverWithMeta::TDatabaseAsyncResolverWithMeta( + NActors::TActorSystem* actorSystem, + const NActors::TActorId& recipient, + const TString& ydbMvpEndpoint, + const TString& mdbGateway, + const bool mdbTransformHost, + const TString& traceId, + const TString& token, + const THashMap<TString, TString>& accountIdSignatures, + const THashMap<TString, YandexQuery::Connection>& connections) + : DbResolver(actorSystem, recipient, ydbMvpEndpoint, mdbGateway, mdbTransformHost) + , TraceId(traceId) + , Token(token) + , AccountIdSignatures(accountIdSignatures) + , Connections(connections) + {} + + NThreading::TFuture<THashMap<std::pair<TString, DatabaseType>, TEndpoint>> TDatabaseAsyncResolverWithMeta::ResolveIds(const TResolveParams& params) const { + return DbResolver.ResolveIds(params); + } + + TString TDatabaseAsyncResolverWithMeta::GetTraceId() const { + return TraceId; + } + + TString TDatabaseAsyncResolverWithMeta::GetToken() const { + return Token; + } + + const THashMap<TString, TString>& TDatabaseAsyncResolverWithMeta::GetAccountIdSignatures() const { + return AccountIdSignatures; + } + + void TDatabaseAsyncResolverWithMeta::TryAddDbIdToResolve( + const bool isEndpoint, + const TString& clusterName, + const TString& dbId, + const DatabaseType type, + THashMap<std::pair<TString, DatabaseType>, TEvents::TDatabaseAuth>& databaseIds) const { + if (isEndpoint) { + return; + } + const auto iter = Connections.find(clusterName); + if (iter == Connections.end()) { + return; + } + const auto& conn = iter->second; + const auto& setting = conn.content().setting(); + YandexQuery::IamAuth auth; + switch (type) { + case DatabaseType::Ydb: + auth = setting.ydb_database().auth(); + break; + case DatabaseType::ClickHouse: + auth = setting.clickhouse_cluster().auth(); + break; + case DatabaseType::DataStreams: + auth = setting.data_streams().auth(); + break; + default: + return; + } + TryAddDatabaseToResolve(auth, dbId, type, Token, AccountIdSignatures, databaseIds); + } + } // NYq diff --git a/ydb/core/yq/libs/db_resolver/db_async_resolver_with_meta.h b/ydb/core/yq/libs/db_resolver/db_async_resolver_with_meta.h index 99f422b3b8..4e0f5ae468 100644 --- a/ydb/core/yq/libs/db_resolver/db_async_resolver_with_meta.h +++ b/ydb/core/yq/libs/db_resolver/db_async_resolver_with_meta.h @@ -1,46 +1,46 @@ -#pragma once -#include "db_async_resolver_impl.h" - -namespace NYq { - -using TEndpoint = TEvents::TEvEndpointResponse::TEndpoint; - -class TDatabaseAsyncResolverWithMeta : public IDatabaseAsyncResolver { -public: - TDatabaseAsyncResolverWithMeta( - NActors::TActorSystem* actorSystem, - const NActors::TActorId& recipient, - const TString& ydbMvpEndpoint, - const TString& mdbGateway, - const bool mdbTransformHost, - const TString& traceId, - const TString& token, - const THashMap<TString, TString>& accountIdSignatures, - const THashMap<TString, YandexQuery::Connection>& connections - ); - - NThreading::TFuture<THashMap<std::pair<TString, DatabaseType>, TEndpoint>> ResolveIds(const TResolveParams& params) const override; - - TString GetTraceId() const; - - TString GetToken() const; - - const THashMap<TString, TString>& GetAccountIdSignatures() const; - - void TryAddDbIdToResolve( - const bool isEndpoint, - const TString& clusterName, - const TString& dbId, - const DatabaseType type, - THashMap<std::pair<TString, DatabaseType>, TEvents::TDatabaseAuth>& databaseIds - ) const; - -private: - TDatabaseAsyncResolver DbResolver; - const TString TraceId; - const TString Token; - const THashMap<TString, TString> AccountIdSignatures; - const THashMap<TString, YandexQuery::Connection> Connections; -}; - -} // NYq
\ No newline at end of file +#pragma once +#include "db_async_resolver_impl.h" + +namespace NYq { + +using TEndpoint = TEvents::TEvEndpointResponse::TEndpoint; + +class TDatabaseAsyncResolverWithMeta : public IDatabaseAsyncResolver { +public: + TDatabaseAsyncResolverWithMeta( + NActors::TActorSystem* actorSystem, + const NActors::TActorId& recipient, + const TString& ydbMvpEndpoint, + const TString& mdbGateway, + const bool mdbTransformHost, + const TString& traceId, + const TString& token, + const THashMap<TString, TString>& accountIdSignatures, + const THashMap<TString, YandexQuery::Connection>& connections + ); + + NThreading::TFuture<THashMap<std::pair<TString, DatabaseType>, TEndpoint>> ResolveIds(const TResolveParams& params) const override; + + TString GetTraceId() const; + + TString GetToken() const; + + const THashMap<TString, TString>& GetAccountIdSignatures() const; + + void TryAddDbIdToResolve( + const bool isEndpoint, + const TString& clusterName, + const TString& dbId, + const DatabaseType type, + THashMap<std::pair<TString, DatabaseType>, TEvents::TDatabaseAuth>& databaseIds + ) const; + +private: + TDatabaseAsyncResolver DbResolver; + const TString TraceId; + const TString Token; + const THashMap<TString, TString> AccountIdSignatures; + const THashMap<TString, YandexQuery::Connection> Connections; +}; + +} // NYq
\ No newline at end of file diff --git a/ydb/core/yq/libs/db_resolver/ya.make b/ydb/core/yq/libs/db_resolver/ya.make index b613350874..2262759e36 100644 --- a/ydb/core/yq/libs/db_resolver/ya.make +++ b/ydb/core/yq/libs/db_resolver/ya.make @@ -1,17 +1,17 @@ OWNER(g:yq) - -LIBRARY() - -SRCS( - db_async_resolver_impl.cpp - db_async_resolver_with_meta.cpp -) - -PEERDIR( + +LIBRARY() + +SRCS( + db_async_resolver_impl.cpp + db_async_resolver_with_meta.cpp +) + +PEERDIR( ydb/core/yq/libs/common ydb/core/yq/libs/events -) - -YQL_LAST_ABI_VERSION() - -END() +) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/ydb/core/yq/libs/db_schema/db_schema.cpp b/ydb/core/yq/libs/db_schema/db_schema.cpp index 44f57e171a..fdda16df40 100644 --- a/ydb/core/yq/libs/db_schema/db_schema.cpp +++ b/ydb/core/yq/libs/db_schema/db_schema.cpp @@ -3,13 +3,13 @@ #include <ydb/public/sdk/cpp/client/ydb_table/table.h> #include <util/string/printf.h> -namespace NYq { +namespace NYq { using namespace NYdb; -TSqlQueryBuilder::TSqlQueryBuilder(const TString& tablePrefix, const TString& queryName) +TSqlQueryBuilder::TSqlQueryBuilder(const TString& tablePrefix, const TString& queryName) : TablePrefix(tablePrefix) - , QueryName(queryName) + , QueryName(queryName) { } TSqlQueryBuilder& TSqlQueryBuilder::PushPk() { @@ -65,12 +65,12 @@ TSqlQueryBuilder& TSqlQueryBuilder::AddValue(const TString& name, const TValue& return *this; } -TSqlQueryBuilder& TSqlQueryBuilder::AddColNames(TVector<TString>&& colNames) -{ - ColNames = colNames; - return *this; -} - +TSqlQueryBuilder& TSqlQueryBuilder::AddColNames(TVector<TString>&& colNames) +{ + ColNames = colNames; + return *this; +} + TSqlQueryBuilder& TSqlQueryBuilder::MaybeAddValue(const TString& name, const TMaybe<TValue>& value) { if (value) { return AddValue(name, *value, "=="); @@ -89,28 +89,28 @@ TSqlQueryBuilder& TSqlQueryBuilder::AddText(const TString& sql, const TString& t return *this; } -TSqlQueryBuilder& TSqlQueryBuilder::AddBeforeQuery(const TString& sql, const TString& tableName ) { - if (tableName) { +TSqlQueryBuilder& TSqlQueryBuilder::AddBeforeQuery(const TString& sql, const TString& tableName ) { + if (tableName) { BeforeQuery << SubstGlobalCopy(sql, TString("{TABLENAME}"), tableName); - } else { - BeforeQuery << sql; - } - return *this; - -} - + } else { + BeforeQuery << sql; + } + return *this; + +} + TSqlQueryBuilder& TSqlQueryBuilder::Upsert(const TString& tableName) { Op = "UPSERT"; Table = tableName; return *this; } -TSqlQueryBuilder& TSqlQueryBuilder::Update(const TString& tableName) { - Op = "UPDATE"; - Table = tableName; - return *this; -} - +TSqlQueryBuilder& TSqlQueryBuilder::Update(const TString& tableName) { + Op = "UPDATE"; + Table = tableName; + return *this; +} + TSqlQueryBuilder& TSqlQueryBuilder::Insert(const TString& tableName) { Op = "INSERT"; Table = tableName; @@ -125,47 +125,47 @@ TSqlQueryBuilder& TSqlQueryBuilder::Delete(const TString& tableName, int limit) } TSqlQueryBuilder& TSqlQueryBuilder::Get(const TString& tableName, int limit) { - Op = "SELECT"; - Table = tableName; + Op = "SELECT"; + Table = tableName; Limit = limit; - return *this; -} - -TSqlQueryBuilder& TSqlQueryBuilder::GetCount(const TString& tableName) { - Op = "SELECT COUNT(*)"; - Table = tableName; - return *this; -} - + return *this; +} + +TSqlQueryBuilder& TSqlQueryBuilder::GetCount(const TString& tableName) { + Op = "SELECT COUNT(*)"; + Table = tableName; + return *this; +} + TSqlQueryBuilder& TSqlQueryBuilder::GetLastDeleted() { Op = "SELECT LAST DELETED"; return *this; } -void TSqlQueryBuilder::ConstructWhereFilter(TStringBuilder& text) { - text << "WHERE "; - size_t i = 0; - for (const auto& pk : Pk) { - text << "`" << pk << "` == " << "$" << pk; - ++i; - if (Fields || i != Pk.size()) { - text << " and "; - } - } - i = 0; +void TSqlQueryBuilder::ConstructWhereFilter(TStringBuilder& text) { + text << "WHERE "; + size_t i = 0; + for (const auto& pk : Pk) { + text << "`" << pk << "` == " << "$" << pk; + ++i; + if (Fields || i != Pk.size()) { + text << " and "; + } + } + i = 0; for (const auto& t : Fields) { auto k = std::get<0>(t); // structured capture don't work on win32 auto v = std::get<1>(t); // structured capture don't work on win32 auto p = std::get<2>(t); // structured capture don't work on win32 text << "`" << k << "` " << p << " " << "$" << v; - ++i; - if (i != Fields.size()) { - text << " and "; - } - } -} - + ++i; + if (i != Fields.size()) { + text << " and "; + } + } +} + TSqlQueryBuilder& TSqlQueryBuilder::End() { if (!Op) { @@ -191,22 +191,22 @@ TSqlQueryBuilder& TSqlQueryBuilder::End() text << "DELETE FROM `" << Table << "`\n"; text << "ON (SELECT * from $todelete" << Counter << ");\n"; - } else if (Op == "SELECT") { - text << Op << ' '; - size_t i = 0; - for (const auto& name : ColNames) { - text << '`' << name << '`'; - ++i; - if (i != ColNames.size()) { - text << ", "; - } - } + } else if (Op == "SELECT") { + text << Op << ' '; + size_t i = 0; + for (const auto& name : ColNames) { + text << '`' << name << '`'; + ++i; + if (i != ColNames.size()) { + text << ", "; + } + } text << "\nFROM `" << Table << "`\n"; - ConstructWhereFilter(text); + ConstructWhereFilter(text); if (Limit > 0) { text << " LIMIT " << Limit; } - text << ";\n"; + text << ";\n"; } else if (Op == "SELECT LAST DELETED") { text << "SELECT "; size_t i = 0; @@ -218,48 +218,48 @@ TSqlQueryBuilder& TSqlQueryBuilder::End() } } text << "\n FROM $todelete" << (Counter-1) << ";\n"; - } else if (Op == "SELECT COUNT(*)") { - text << Op ; + } else if (Op == "SELECT COUNT(*)") { + text << Op ; text << "\nFROM `" << Table << "`\n"; - ConstructWhereFilter(text); - text << ";\n"; - } else if (Op == "UPDATE") { + ConstructWhereFilter(text); + text << ";\n"; + } else if (Op == "UPDATE") { text << Op << " `" << Table << "`\n"; - if (Pk.empty() && Fields.empty()) { - ythrow yexception() << "dangerous sql query"; - } - text << "SET\n"; - size_t i = 0; - for (const auto& t : Fields) { - auto k = std::get<0>(t); // structured capture don't work on win32 - auto v = std::get<1>(t); // structured capture don't work on win32 - auto p = std::get<2>(t); // structured capture don't work on win32 - - text << "`" << k << "` " << " = " << "$" << v; - ++i; - if (i != Fields.size()) { - text << ", "; - } - } - text << "\nWHERE\n"; - for (const auto& pk : Pk) { - text << "$" << pk << " == " << "`" << pk << "` and "; - } - text << " $count != 0;\n"; - BeforeQuery << text; - text.clear(); + if (Pk.empty() && Fields.empty()) { + ythrow yexception() << "dangerous sql query"; + } + text << "SET\n"; + size_t i = 0; + for (const auto& t : Fields) { + auto k = std::get<0>(t); // structured capture don't work on win32 + auto v = std::get<1>(t); // structured capture don't work on win32 + auto p = std::get<2>(t); // structured capture don't work on win32 + + text << "`" << k << "` " << " = " << "$" << v; + ++i; + if (i != Fields.size()) { + text << ", "; + } + } + text << "\nWHERE\n"; + for (const auto& pk : Pk) { + text << "$" << pk << " == " << "`" << pk << "` and "; + } + text << " $count != 0;\n"; + BeforeQuery << text; + text.clear(); } else { text << Op << " INTO `" << Table << "`\n"; text << "(\n" << " "; for (const auto& pk : Pk) { text << "`" << pk << "`" << ","; } - size_t i = 0; + size_t i = 0; for (const auto& t : Fields) { auto field = std::get<0>(t); // structured capture don't work on win32 text << "`" << field << "`"; i++; - if (i != Fields.size()) { + if (i != Fields.size()) { text << ","; } } @@ -274,7 +274,7 @@ TSqlQueryBuilder& TSqlQueryBuilder::End() auto field = std::get<1>(t); // structured capture don't work on win32 text << "$" << field; i++; - if (i != Fields.size()) { + if (i != Fields.size()) { text << ", "; } } @@ -290,7 +290,7 @@ TSqlQueryBuilder::TResult TSqlQueryBuilder::Build() { TStringBuilder declr; declr << "--!syntax_v1\n"; - declr << "-- Query name: " << QueryName << "\n"; + declr << "-- Query name: " << QueryName << "\n"; declr << "PRAGMA TablePathPrefix(\"" << TablePrefix << "\");\n"; for (const auto& [k, v] : ParametersOrdered) { declr << "DECLARE $" << k << " as " << v << ";\n"; @@ -301,4 +301,4 @@ TSqlQueryBuilder::TResult TSqlQueryBuilder::Build() return {declr, ParamsBuilder.Build()}; } -} // namespace NYq +} // namespace NYq diff --git a/ydb/core/yq/libs/db_schema/db_schema.h b/ydb/core/yq/libs/db_schema/db_schema.h index 4359ad7f04..c14a7d21c0 100644 --- a/ydb/core/yq/libs/db_schema/db_schema.h +++ b/ydb/core/yq/libs/db_schema/db_schema.h @@ -6,11 +6,11 @@ #include <util/string/builder.h> #include <util/generic/hash_set.h> -namespace NYq { +namespace NYq { class TSqlQueryBuilder { public: - TSqlQueryBuilder(const TString& tablePrefix, const TString& queryName = "Unknown query name"); + TSqlQueryBuilder(const TString& tablePrefix, const TString& queryName = "Unknown query name"); TSqlQueryBuilder& PushPk(); TSqlQueryBuilder& PopPk(); @@ -54,15 +54,15 @@ public: TSqlQueryBuilder& AddValue(const TString& name, const NYdb::TValue& value, const TString& pred); TSqlQueryBuilder& AddValue(const TString& name, const TString& value, const TString& pred) { - return AddValue(name, NYdb::TValueBuilder().String(value).Build(), pred); + return AddValue(name, NYdb::TValueBuilder().String(value).Build(), pred); } TSqlQueryBuilder& AddValue(const TString& name, ui32 value, const TString& pred) { - return AddValue(name, NYdb::TValueBuilder().Uint32(value).Build(), pred); + return AddValue(name, NYdb::TValueBuilder().Uint32(value).Build(), pred); } TSqlQueryBuilder& AddValue(const TString& name, ui64 value, const TString& pred) { - return AddValue(name, NYdb::TValueBuilder().Uint64(value).Build(), pred); + return AddValue(name, NYdb::TValueBuilder().Uint64(value).Build(), pred); } TSqlQueryBuilder& AddValue(const TString& name, TInstant value, const TString& pred) { @@ -79,17 +79,17 @@ public: return *this; } - TSqlQueryBuilder& AddColNames(TVector<TString>&& colNames); - + TSqlQueryBuilder& AddColNames(TVector<TString>&& colNames); + TSqlQueryBuilder& AddText(const TString& sql, const TString& tableName = ""); - TSqlQueryBuilder& AddBeforeQuery(const TString& sql, const TString& tableName = ""); + TSqlQueryBuilder& AddBeforeQuery(const TString& sql, const TString& tableName = ""); TSqlQueryBuilder& Upsert(const TString& tableName); - TSqlQueryBuilder& Update(const TString& tableName); + TSqlQueryBuilder& Update(const TString& tableName); TSqlQueryBuilder& Insert(const TString& tableName); TSqlQueryBuilder& Delete(const TString& tableName, int limit = 0); TSqlQueryBuilder& Get(const TString& tableName, int limit = 0); - TSqlQueryBuilder& GetCount(const TString& tableName); + TSqlQueryBuilder& GetCount(const TString& tableName); TSqlQueryBuilder& GetLastDeleted(); TSqlQueryBuilder& End(); @@ -108,7 +108,7 @@ private: void ConstructWhereFilter(TStringBuilder& text); const TString TablePrefix; - const TString QueryName; + const TString QueryName; TString Table; int Limit = 0; TString Op; @@ -123,8 +123,8 @@ private: TStringBuilder BeforeQuery; TStringBuilder Text; int Counter = 0; - - TVector<TString> ColNames; + + TVector<TString> ColNames; }; -} // namespace NYq +} // namespace NYq diff --git a/ydb/core/yq/libs/events/event_ids.h b/ydb/core/yq/libs/events/event_ids.h index 83ed11b390..56032d50ea 100644 --- a/ydb/core/yq/libs/events/event_ids.h +++ b/ydb/core/yq/libs/events/event_ids.h @@ -1,54 +1,54 @@ -#pragma once -#include <library/cpp/actors/core/event_local.h> -#include <library/cpp/actors/core/events.h> - -namespace NYq { - -struct TEventIds { - enum EEventSpaceYqlProxy { - ES_YQL_ANALYTICS_PROXY = 4205 //TKikimrEvents::ES_YQL_ANALYTICS_PROXY - }; - - static constexpr ui32 EventSpace = ES_YQL_ANALYTICS_PROXY; - - // Event ids. - enum EEv : ui32 { - EvBegin = EventSpaceBegin(EventSpace), - - //YQL Internal - EvPingTaskRequest = EvBegin, - EvPingTaskResponse, - EvGetTaskRequest, - EvGetTaskResponse, - EvWriteTaskResultRequest, - EvWriteTaskResultResponse, - - EvNodesHealthCheckRequest, - EvNodesHealthCheckResponse, - - EvUpdateConfig, - - // Internal events - EvAsyncContinue, - EvDbRequest, - EvDbResponse, - EvDbFunctionRequest, - EvDbFunctionResponse, - EvEndpointRequest, - EvEndpointResponse, - EvDataStreamsReadRulesCreationResult, - EvDataStreamsReadRulesDeletionResult, - EvQueryActionResult, - EvForwardPingRequest, - EvForwardPingResponse, +#pragma once +#include <library/cpp/actors/core/event_local.h> +#include <library/cpp/actors/core/events.h> + +namespace NYq { + +struct TEventIds { + enum EEventSpaceYqlProxy { + ES_YQL_ANALYTICS_PROXY = 4205 //TKikimrEvents::ES_YQL_ANALYTICS_PROXY + }; + + static constexpr ui32 EventSpace = ES_YQL_ANALYTICS_PROXY; + + // Event ids. + enum EEv : ui32 { + EvBegin = EventSpaceBegin(EventSpace), + + //YQL Internal + EvPingTaskRequest = EvBegin, + EvPingTaskResponse, + EvGetTaskRequest, + EvGetTaskResponse, + EvWriteTaskResultRequest, + EvWriteTaskResultResponse, + + EvNodesHealthCheckRequest, + EvNodesHealthCheckResponse, + + EvUpdateConfig, + + // Internal events + EvAsyncContinue, + EvDbRequest, + EvDbResponse, + EvDbFunctionRequest, + EvDbFunctionResponse, + EvEndpointRequest, + EvEndpointResponse, + EvDataStreamsReadRulesCreationResult, + EvDataStreamsReadRulesDeletionResult, + EvQueryActionResult, + EvForwardPingRequest, + EvForwardPingResponse, EvGraphParams, - - // Special events - EvEnd - }; - - static_assert(EvEnd < EventSpaceEnd(EventSpace), "expect EvEnd < EventSpaceEnd(EventSpace)"); - -}; - -} // namespace NYq + + // Special events + EvEnd + }; + + static_assert(EvEnd < EventSpaceEnd(EventSpace), "expect EvEnd < EventSpaceEnd(EventSpace)"); + +}; + +} // namespace NYq diff --git a/ydb/core/yq/libs/events/events.h b/ydb/core/yq/libs/events/events.h index eca0b2a543..b3c70e5c5f 100644 --- a/ydb/core/yq/libs/events/events.h +++ b/ydb/core/yq/libs/events/events.h @@ -1,5 +1,5 @@ #pragma once -#include "event_ids.h" +#include "event_ids.h" #include <ydb/library/yql/core/facade/yql_facade.h> #include <ydb/library/yql/providers/dq/provider/yql_dq_gateway.h> @@ -14,16 +14,16 @@ #include <util/digest/multi.h> -namespace NYq { +namespace NYq { using NYdb::NYq::TScope; -enum class DatabaseType { - Ydb, +enum class DatabaseType { + Ydb, ClickHouse, DataStreams, ObjectStorage -}; +}; struct TQueryResult { TVector<Ydb::ResultSet> Sets; @@ -39,55 +39,55 @@ struct TEvents { bool HasPerm = false; }; - struct TEvPingTaskRequest : NActors::TEventLocal<TEvPingTaskRequest, TEventIds::EvPingTaskRequest>, TEvAnalyticsBase { - Yq::Private::PingTaskRequest Record; - }; - - struct TEvPingTaskResponse : NActors::TEventLocal<TEvPingTaskResponse, TEventIds::EvPingTaskResponse> { - Ydb::StatusIds::StatusCode Status; - NYql::TIssues Issues; - TMaybe<Yq::Private::PingTaskResult> Record; - }; - - struct TEvGetTaskRequest : NActors::TEventLocal<TEvGetTaskRequest, TEventIds::EvGetTaskRequest>, TEvAnalyticsBase { - Yq::Private::GetTaskRequest Record; - }; - - struct TEvGetTaskResponse : NActors::TEventLocal<TEvGetTaskResponse, TEventIds::EvGetTaskResponse> { - Ydb::StatusIds::StatusCode Status; - NYql::TIssues Issues; - TMaybe<Yq::Private::GetTaskResult> Record; - }; - - struct TEvWriteTaskResultRequest : NActors::TEventLocal<TEvWriteTaskResultRequest, TEventIds::EvWriteTaskResultRequest>, TEvAnalyticsBase { - Yq::Private::WriteTaskResultRequest Record; - }; - - struct TEvWriteTaskResultResponse : NActors::TEventLocal<TEvWriteTaskResultResponse, TEventIds::EvWriteTaskResultResponse> { - Ydb::StatusIds::StatusCode Status; - NYql::TIssues Issues; - TMaybe<Yq::Private::WriteTaskResultResult> Record; - }; - - struct TEvNodesHealthCheckRequest : NActors::TEventLocal<TEvNodesHealthCheckRequest, TEventIds::EvNodesHealthCheckRequest>{ - Yq::Private::NodesHealthCheckRequest Record; - }; - - struct TEvNodesHealthCheckResponse : NActors::TEventLocal<TEvNodesHealthCheckResponse, TEventIds::EvNodesHealthCheckResponse> { - Ydb::StatusIds::StatusCode Status; - NYql::TIssues Issues; - TMaybe<Yq::Private::NodesHealthCheckResult> Record; - }; - - struct TEvAsyncContinue : NActors::TEventLocal<TEvAsyncContinue, TEventIds::EvAsyncContinue> { - const NYql::TProgram::TFutureStatus Future; - - explicit TEvAsyncContinue(const NYql::TProgram::TFutureStatus& future) + struct TEvPingTaskRequest : NActors::TEventLocal<TEvPingTaskRequest, TEventIds::EvPingTaskRequest>, TEvAnalyticsBase { + Yq::Private::PingTaskRequest Record; + }; + + struct TEvPingTaskResponse : NActors::TEventLocal<TEvPingTaskResponse, TEventIds::EvPingTaskResponse> { + Ydb::StatusIds::StatusCode Status; + NYql::TIssues Issues; + TMaybe<Yq::Private::PingTaskResult> Record; + }; + + struct TEvGetTaskRequest : NActors::TEventLocal<TEvGetTaskRequest, TEventIds::EvGetTaskRequest>, TEvAnalyticsBase { + Yq::Private::GetTaskRequest Record; + }; + + struct TEvGetTaskResponse : NActors::TEventLocal<TEvGetTaskResponse, TEventIds::EvGetTaskResponse> { + Ydb::StatusIds::StatusCode Status; + NYql::TIssues Issues; + TMaybe<Yq::Private::GetTaskResult> Record; + }; + + struct TEvWriteTaskResultRequest : NActors::TEventLocal<TEvWriteTaskResultRequest, TEventIds::EvWriteTaskResultRequest>, TEvAnalyticsBase { + Yq::Private::WriteTaskResultRequest Record; + }; + + struct TEvWriteTaskResultResponse : NActors::TEventLocal<TEvWriteTaskResultResponse, TEventIds::EvWriteTaskResultResponse> { + Ydb::StatusIds::StatusCode Status; + NYql::TIssues Issues; + TMaybe<Yq::Private::WriteTaskResultResult> Record; + }; + + struct TEvNodesHealthCheckRequest : NActors::TEventLocal<TEvNodesHealthCheckRequest, TEventIds::EvNodesHealthCheckRequest>{ + Yq::Private::NodesHealthCheckRequest Record; + }; + + struct TEvNodesHealthCheckResponse : NActors::TEventLocal<TEvNodesHealthCheckResponse, TEventIds::EvNodesHealthCheckResponse> { + Ydb::StatusIds::StatusCode Status; + NYql::TIssues Issues; + TMaybe<Yq::Private::NodesHealthCheckResult> Record; + }; + + struct TEvAsyncContinue : NActors::TEventLocal<TEvAsyncContinue, TEventIds::EvAsyncContinue> { + const NYql::TProgram::TFutureStatus Future; + + explicit TEvAsyncContinue(const NYql::TProgram::TFutureStatus& future) : Future(future) {} }; - struct TEvDbRequest : NActors::TEventLocal<TEvDbRequest, TEventIds::EvDbRequest> { + struct TEvDbRequest : NActors::TEventLocal<TEvDbRequest, TEventIds::EvDbRequest> { TString Sql; NYdb::TParams Params; bool Idempotent; @@ -99,7 +99,7 @@ struct TEvents { {} }; - struct TEvDbResponse : NActors::TEventLocal<TEvDbResponse, TEventIds::EvDbResponse> { + struct TEvDbResponse : NActors::TEventLocal<TEvDbResponse, TEventIds::EvDbResponse> { NYdb::TStatus Status; TVector<NYdb::TResultSet> ResultSets; @@ -109,7 +109,7 @@ struct TEvents { {} }; - struct TEvDbFunctionRequest : NActors::TEventLocal<TEvDbFunctionRequest, TEventIds::EvDbFunctionRequest> { + struct TEvDbFunctionRequest : NActors::TEventLocal<TEvDbFunctionRequest, TEventIds::EvDbFunctionRequest> { using TFunction = std::function<NYdb::TAsyncStatus(NYdb::NTable::TSession&)>; TFunction Handler; @@ -118,7 +118,7 @@ struct TEvents { {} }; - struct TEvDbFunctionResponse : NActors::TEventLocal<TEvDbFunctionResponse, TEventIds::EvDbFunctionResponse> { + struct TEvDbFunctionResponse : NActors::TEventLocal<TEvDbFunctionResponse, TEventIds::EvDbFunctionResponse> { NYdb::TStatus Status; explicit TEvDbFunctionResponse(NYdb::TStatus status) @@ -126,21 +126,21 @@ struct TEvents { {} }; - struct TEvEndpointResponse : NActors::TEventLocal<TEvEndpointResponse, TEventIds::EvEndpointResponse> { - struct TEndpoint { - TString Endpoint; - TString Database; + struct TEvEndpointResponse : NActors::TEventLocal<TEvEndpointResponse, TEventIds::EvEndpointResponse> { + struct TEndpoint { + TString Endpoint; + TString Database; bool Secure = false; - }; + }; THashMap<std::pair<TString, DatabaseType>, TEndpoint> DatabaseId2Endpoint; - bool Success; - + bool Success; + TEvEndpointResponse(THashMap<std::pair<TString, DatabaseType>, TEndpoint>&& res, bool success) : DatabaseId2Endpoint(std::move(res)) - , Success(success) - { } - }; - + , Success(success) + { } + }; + struct TDatabaseAuth { TString StructuredToken; bool AddBearerToToken = false; @@ -153,12 +153,12 @@ struct TEvents { } }; - struct TEvEndpointRequest : NActors::TEventLocal<TEvEndpointRequest, TEventIds::EvEndpointRequest> { + struct TEvEndpointRequest : NActors::TEventLocal<TEvEndpointRequest, TEventIds::EvEndpointRequest> { THashMap<std::pair<TString, DatabaseType>, TDatabaseAuth> DatabaseIds; // DbId, DatabaseType => database auth - TString YdbMvpEndpoint; + TString YdbMvpEndpoint; TString MdbGateway; TString TraceId; - bool MdbTransformHost; + bool MdbTransformHost; TEvEndpointRequest( const THashMap<std::pair<TString, DatabaseType>, TDatabaseAuth>& databaseIds, @@ -167,14 +167,14 @@ struct TEvents { const TString& traceId, bool mdbTransformHost) : DatabaseIds(databaseIds) - , YdbMvpEndpoint(ydbMvpEndpoint) + , YdbMvpEndpoint(ydbMvpEndpoint) , MdbGateway(mdbGateway) , TraceId(traceId) - , MdbTransformHost(mdbTransformHost) + , MdbTransformHost(mdbTransformHost) { } }; - struct TEvDataStreamsReadRulesCreationResult : NActors::TEventLocal<TEvDataStreamsReadRulesCreationResult, TEventIds::EvDataStreamsReadRulesCreationResult> { + struct TEvDataStreamsReadRulesCreationResult : NActors::TEventLocal<TEvDataStreamsReadRulesCreationResult, TEventIds::EvDataStreamsReadRulesCreationResult> { explicit TEvDataStreamsReadRulesCreationResult(NYql::TIssues issues) : Issues(std::move(issues)) { @@ -183,7 +183,7 @@ struct TEvents { NYql::TIssues Issues; }; - struct TEvDataStreamsReadRulesDeletionResult : NActors::TEventLocal<TEvDataStreamsReadRulesDeletionResult, TEventIds::EvDataStreamsReadRulesDeletionResult> { + struct TEvDataStreamsReadRulesDeletionResult : NActors::TEventLocal<TEvDataStreamsReadRulesDeletionResult, TEventIds::EvDataStreamsReadRulesDeletionResult> { explicit TEvDataStreamsReadRulesDeletionResult(NYql::TIssues transientIssues) : TransientIssues(std::move(transientIssues)) { @@ -192,7 +192,7 @@ struct TEvents { NYql::TIssues TransientIssues; }; - struct TEvQueryActionResult : NActors::TEventLocal<TEvQueryActionResult, TEventIds::EvQueryActionResult> { + struct TEvQueryActionResult : NActors::TEventLocal<TEvQueryActionResult, TEventIds::EvQueryActionResult> { explicit TEvQueryActionResult(YandexQuery::QueryAction action) : Action(action) { @@ -201,17 +201,17 @@ struct TEvents { YandexQuery::QueryAction Action; }; - struct TEvForwardPingRequest : NActors::TEventLocal<TEvForwardPingRequest, TEventIds::EvForwardPingRequest> { - explicit TEvForwardPingRequest(const Yq::Private::PingTaskRequest& request, bool final = false) + struct TEvForwardPingRequest : NActors::TEventLocal<TEvForwardPingRequest, TEventIds::EvForwardPingRequest> { + explicit TEvForwardPingRequest(const Yq::Private::PingTaskRequest& request, bool final = false) : Request(request) , Final(final) { } - Yq::Private::PingTaskRequest Request; + Yq::Private::PingTaskRequest Request; bool Final; // Is this the last ping request. }; - struct TEvForwardPingResponse : NActors::TEventLocal<TEvForwardPingResponse, TEventIds::EvForwardPingResponse> { + struct TEvForwardPingResponse : NActors::TEventLocal<TEvForwardPingResponse, TEventIds::EvForwardPingResponse> { TEvForwardPingResponse(bool success, YandexQuery::QueryAction action) : Success(success) , Action(action) @@ -235,11 +235,11 @@ struct TEvents { }; }; -} // namespace NYq +} // namespace NYq template<> -struct THash<NYq::TEvents::TDatabaseAuth> { - inline ui64 operator()(const NYq::TEvents::TDatabaseAuth& x) const noexcept { +struct THash<NYq::TEvents::TDatabaseAuth> { + inline ui64 operator()(const NYq::TEvents::TDatabaseAuth& x) const noexcept { return MultiHash(x.StructuredToken, x.AddBearerToToken); } }; diff --git a/ydb/core/yq/libs/events/ya.make b/ydb/core/yq/libs/events/ya.make index 1f91ea511b..c12bdd681f 100644 --- a/ydb/core/yq/libs/events/ya.make +++ b/ydb/core/yq/libs/events/ya.make @@ -4,7 +4,7 @@ LIBRARY() GENERATE_ENUM_SERIALIZATION(events.h) -GENERATE_ENUM_SERIALIZATION(event_ids.h) +GENERATE_ENUM_SERIALIZATION(event_ids.h) PEERDIR( library/cpp/actors/core diff --git a/ydb/core/yq/libs/init/init.cpp b/ydb/core/yq/libs/init/init.cpp index 747d25f192..d66ac202bf 100644 --- a/ydb/core/yq/libs/init/init.cpp +++ b/ydb/core/yq/libs/init/init.cpp @@ -24,7 +24,7 @@ #include <ydb/library/yql/providers/dq/task_runner/tasks_runner_local.h> #include <ydb/library/yql/providers/dq/worker_manager/local_worker_manager.h> #include <ydb/library/yql/providers/s3/actors/yql_s3_source_factory.h> -#include <ydb/library/yql/providers/s3/proto/retry_config.pb.h> +#include <ydb/library/yql/providers/s3/proto/retry_config.pb.h> #include <ydb/library/yql/providers/clickhouse/actors/yql_ch_source_factory.h> #include <ydb/library/yql/providers/pq/async_io/dq_pq_read_actor.h> #include <ydb/library/yql/providers/pq/async_io/dq_pq_write_actor.h> @@ -35,62 +35,62 @@ #include <ydb/library/yql/providers/ydb/actors/yql_ydb_source_factory.h> #include <util/stream/file.h> -#include <util/system/hostname.h> - -namespace { - -std::tuple<TString, TString> GetLocalAddress(const TString* overrideHostname = nullptr) { - constexpr auto MaxLocalHostNameLength = 4096; - std::array<char, MaxLocalHostNameLength> buffer; - buffer.fill(0); - TString hostName; - TString localAddress; - - int result = gethostname(buffer.data(), buffer.size() - 1); - if (result != 0) { - Cerr << "gethostname failed " << strerror(errno) << Endl; - return std::make_tuple(hostName, localAddress); - } - - if (overrideHostname) { - memcpy(&buffer[0], overrideHostname->c_str(), Min<int>( - overrideHostname->size()+1, buffer.size()-1 - )); - } - - hostName = &buffer[0]; - - addrinfo request; - memset(&request, 0, sizeof(request)); - request.ai_family = AF_INET6; - request.ai_socktype = SOCK_STREAM; - - addrinfo* response = nullptr; - result = getaddrinfo(buffer.data(), nullptr, &request, &response); - if (result != 0) { - Cerr << "getaddrinfo failed " << gai_strerror(result) << Endl; - return std::make_tuple(hostName, localAddress); - } - - std::unique_ptr<addrinfo, void (*)(addrinfo*)> holder(response, &freeaddrinfo); - - if (!response->ai_addr) { - Cerr << "getaddrinfo failed: no ai_addr" << Endl; - return std::make_tuple(hostName, localAddress); - } - - auto* sa = response->ai_addr; - Y_VERIFY(sa->sa_family == AF_INET6); - inet_ntop(AF_INET6, &(((struct sockaddr_in6*)sa)->sin6_addr), - &buffer[0], buffer.size() - 1); - - localAddress = &buffer[0]; - - return std::make_tuple(hostName, localAddress); -} - -} - +#include <util/system/hostname.h> + +namespace { + +std::tuple<TString, TString> GetLocalAddress(const TString* overrideHostname = nullptr) { + constexpr auto MaxLocalHostNameLength = 4096; + std::array<char, MaxLocalHostNameLength> buffer; + buffer.fill(0); + TString hostName; + TString localAddress; + + int result = gethostname(buffer.data(), buffer.size() - 1); + if (result != 0) { + Cerr << "gethostname failed " << strerror(errno) << Endl; + return std::make_tuple(hostName, localAddress); + } + + if (overrideHostname) { + memcpy(&buffer[0], overrideHostname->c_str(), Min<int>( + overrideHostname->size()+1, buffer.size()-1 + )); + } + + hostName = &buffer[0]; + + addrinfo request; + memset(&request, 0, sizeof(request)); + request.ai_family = AF_INET6; + request.ai_socktype = SOCK_STREAM; + + addrinfo* response = nullptr; + result = getaddrinfo(buffer.data(), nullptr, &request, &response); + if (result != 0) { + Cerr << "getaddrinfo failed " << gai_strerror(result) << Endl; + return std::make_tuple(hostName, localAddress); + } + + std::unique_ptr<addrinfo, void (*)(addrinfo*)> holder(response, &freeaddrinfo); + + if (!response->ai_addr) { + Cerr << "getaddrinfo failed: no ai_addr" << Endl; + return std::make_tuple(hostName, localAddress); + } + + auto* sa = response->ai_addr; + Y_VERIFY(sa->sa_family == AF_INET6); + inet_ntop(AF_INET6, &(((struct sockaddr_in6*)sa)->sin6_addr), + &buffer[0], buffer.size() - 1); + + localAddress = &buffer[0]; + + return std::make_tuple(hostName, localAddress); +} + +} + namespace NYq { using namespace NKikimr; @@ -100,18 +100,18 @@ void Init( ui32 nodeId, const TActorRegistrator& actorRegistrator, const TAppData* appData, - const TString& tenant, + const TString& tenant, ::NPq::NConfigurationManager::IConnections::TPtr pqCmConnections, const IYqSharedResources::TPtr& iyqSharedResources, const std::function<IActor*(const NKikimrProto::NFolderService::TFolderServiceConfig& authConfig)>& folderServiceFactory, const std::function<IActor*(const NYq::NConfig::TAuditConfig& auditConfig)>& auditServiceFactory, - const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory, - const ui32& icPort + const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory, + const ui32& icPort ) { Y_VERIFY(iyqSharedResources, "No YQ shared resources created"); TYqSharedResources::TPtr yqSharedResources = TYqSharedResources::Cast(iyqSharedResources); - const auto clientCounters = appData->Counters->GetSubgroup("counters", "yq")->GetSubgroup("subsystem", "ClientMetrics"); + const auto clientCounters = appData->Counters->GetSubgroup("counters", "yq")->GetSubgroup("subsystem", "ClientMetrics"); if (protoConfig.GetControlPlaneStorage().GetEnabled()) { auto controlPlaneStorage = protoConfig.GetControlPlaneStorage().GetUseInMemory() @@ -154,8 +154,8 @@ void Init( actorRegistrator(NYql::NDq::MakeCheckpointStorageID(), checkpointStorage.release()); } - auto yqCounters = appData->Counters->GetSubgroup("counters", "yq"); - auto workerManagerCounters = NYql::NDqs::TWorkerManagerCounters(yqCounters->GetSubgroup("subsystem", "worker_manager")); + auto yqCounters = appData->Counters->GetSubgroup("counters", "yq"); + auto workerManagerCounters = NYql::NDqs::TWorkerManagerCounters(yqCounters->GetSubgroup("subsystem", "worker_manager")); NKikimr::NMiniKQL::TComputationNodeFactory dqCompFactory = NKikimr::NMiniKQL::GetCompositeWithBuiltinFactory({ NYql::GetCommonDqFactory(), @@ -172,17 +172,17 @@ void Init( auto sinkActorFactory = MakeIntrusive<NYql::NDq::TDqSinkFactory>(); NYql::ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory; - const auto httpGateway = NYql::IHTTPGateway::Make( - &protoConfig.GetGateways().GetHttpGateway(), - yqCounters->GetSubgroup("subcomponent", "http_gateway")); - + const auto httpGateway = NYql::IHTTPGateway::Make( + &protoConfig.GetGateways().GetHttpGateway(), + yqCounters->GetSubgroup("subcomponent", "http_gateway")); + if (protoConfig.GetTokenAccessor().GetEnabled()) { credentialsFactory = NYql::CreateSecuredServiceAccountCredentialsOverTokenAccessorFactory(protoConfig.GetTokenAccessor().GetEndpoint(), protoConfig.GetTokenAccessor().GetUseSsl()); RegisterDqPqReadActorFactory(*sourceActorFactory, yqSharedResources->YdbDriver, credentialsFactory); RegisterYdbReadActorFactory(*sourceActorFactory, yqSharedResources->YdbDriver, credentialsFactory); - RegisterS3ReadActorFactory(*sourceActorFactory, credentialsFactory, - httpGateway, std::make_shared<NYql::NS3::TRetryConfig>(protoConfig.GetReadActorsFactoryConfig().GetS3ReadActorFactoryConfig().GetRetryConfig())); - RegisterClickHouseReadActorFactory(*sourceActorFactory, credentialsFactory, httpGateway); + RegisterS3ReadActorFactory(*sourceActorFactory, credentialsFactory, + httpGateway, std::make_shared<NYql::NS3::TRetryConfig>(protoConfig.GetReadActorsFactoryConfig().GetS3ReadActorFactoryConfig().GetRetryConfig())); + RegisterClickHouseReadActorFactory(*sourceActorFactory, credentialsFactory, httpGateway); RegisterDqPqWriteActorFactory(*sinkActorFactory, yqSharedResources->YdbDriver, credentialsFactory); RegisterDQSolomonWriteActorFactory(*sinkActorFactory, credentialsFactory); @@ -209,7 +209,7 @@ void Init( lwmOptions.MkqlInitialMemoryLimit = mkqlInitialMemoryLimit; lwmOptions.MkqlTotalMemoryLimit = mkqlTotalMemoryLimit; lwmOptions.MkqlMinAllocSize = mkqlAllocSize; - auto resman = NYql::NDqs::CreateLocalWorkerManager(lwmOptions); + auto resman = NYql::NDqs::CreateLocalWorkerManager(lwmOptions); actorRegistrator(NYql::NDqs::MakeWorkerManagerActorID(nodeId), resman); } @@ -217,7 +217,7 @@ void Init( ::NYq::NCommon::TServiceCounters serviceCounters(appData->Counters); if (protoConfig.GetNodesManager().GetEnabled()) { - const auto localAddr = GetLocalAddress(&HostName()); + const auto localAddr = GetLocalAddress(&HostName()); auto nodesManager = CreateYqlNodesManager( workerManagerCounters, TAppData::TimeProvider, @@ -225,17 +225,17 @@ void Init( serviceCounters, protoConfig.GetPrivateApi(), yqSharedResources, - icPort, - std::get<1>(localAddr), + icPort, + std::get<1>(localAddr), tenant, - mkqlInitialMemoryLimit, - clientCounters); + mkqlInitialMemoryLimit, + clientCounters); actorRegistrator(MakeYqlNodesManagerId(), nodesManager); } auto httpProxy = NHttp::CreateHttpProxy(*NMonitoring::TMetricRegistry::Instance()); - actorRegistrator(MakeYqlAnalyticsHttpProxyId(), httpProxy); + actorRegistrator(MakeYqlAnalyticsHttpProxyId(), httpProxy); if (protoConfig.GetPendingFetcher().GetEnabled()) { auto fetcher = CreatePendingFetcher( @@ -251,31 +251,31 @@ void Init( dqCompFactory, serviceCounters, credentialsFactory, - httpGateway, - std::move(pqCmConnections), - clientCounters + httpGateway, + std::move(pqCmConnections), + clientCounters ); actorRegistrator(MakeYqlAnalyticsFetcherId(nodeId), fetcher); } - + if (protoConfig.GetPrivateProxy().GetEnabled()) { auto proxyPrivate = CreateYqlAnalyticsPrivateProxy( TAppData::TimeProvider, TAppData::RandomProvider, serviceCounters.Counters, protoConfig.GetTokenAccessor()); - - actorRegistrator(MakeYqPrivateProxyId(), proxyPrivate); + + actorRegistrator(MakeYqPrivateProxyId(), proxyPrivate); } } IYqSharedResources::TPtr CreateYqSharedResources( const NYq::NConfig::TConfig& config, - const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory, - const NMonitoring::TDynamicCounterPtr& counters) + const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory, + const NMonitoring::TDynamicCounterPtr& counters) { - return CreateYqSharedResourcesImpl(config, credentialsProviderFactory, counters); + return CreateYqSharedResourcesImpl(config, credentialsProviderFactory, counters); } } // NYq diff --git a/ydb/core/yq/libs/init/init.h b/ydb/core/yq/libs/init/init.h index 5aee6d8878..fa26651f18 100644 --- a/ydb/core/yq/libs/init/init.h +++ b/ydb/core/yq/libs/init/init.h @@ -23,8 +23,8 @@ using TActorRegistrator = std::function<void(NActors::TActorId, NActors::IActor* IYqSharedResources::TPtr CreateYqSharedResources( const NYq::NConfig::TConfig& config, - const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory, - const NMonitoring::TDynamicCounterPtr& counters); + const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory, + const NMonitoring::TDynamicCounterPtr& counters); void Init( const NYq::NConfig::TConfig& config, @@ -36,8 +36,8 @@ void Init( const IYqSharedResources::TPtr& yqSharedResources, const std::function<IActor*(const NKikimrProto::NFolderService::TFolderServiceConfig& authConfig)>& folderServiceFactory, const std::function<IActor*(const NYq::NConfig::TAuditConfig& auditConfig)>& auditServiceFactory, - const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory, - const ui32& icPort + const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory, + const ui32& icPort ); } // NYq diff --git a/ydb/core/yq/libs/init/ya.make b/ydb/core/yq/libs/init/ya.make index 330275a50b..129103b024 100644 --- a/ydb/core/yq/libs/init/ya.make +++ b/ydb/core/yq/libs/init/ya.make @@ -40,7 +40,7 @@ PEERDIR( ydb/library/yql/providers/pq/gateway/native ydb/library/yql/providers/pq/provider ydb/library/yql/providers/s3/actors - ydb/library/yql/providers/s3/proto + ydb/library/yql/providers/s3/proto ydb/library/yql/providers/s3/provider ydb/library/yql/providers/solomon/async_io ydb/library/yql/providers/solomon/gateway diff --git a/ydb/core/yq/libs/logs/log.cpp b/ydb/core/yq/libs/logs/log.cpp index f26c6a23c1..91c4af73ca 100644 --- a/ydb/core/yq/libs/logs/log.cpp +++ b/ydb/core/yq/libs/logs/log.cpp @@ -13,8 +13,8 @@ using namespace NActors; class TYqlLogsUpdater : public TActorBootstrapped<TYqlLogsUpdater> { public: - static constexpr char ActorName[] = "YQ_LOGS_UPDATER"; - + static constexpr char ActorName[] = "YQ_LOGS_UPDATER"; + TYqlLogsUpdater(const NKikimrConfig::TLogConfig& logConfig) : YqlLoggerScope(new NYql::NLog::TTlsLogBackend(new TNullLogBackend())) , LogConfig(logConfig) diff --git a/ydb/core/yq/libs/mock/yql_mock.cpp b/ydb/core/yq/libs/mock/yql_mock.cpp index 2e2d412453..8fe694f35b 100644 --- a/ydb/core/yq/libs/mock/yql_mock.cpp +++ b/ydb/core/yq/libs/mock/yql_mock.cpp @@ -36,8 +36,8 @@ public: Location.Endpoint = TStringBuilder() << "localhost:" << grpcPort; } - static constexpr char ActorName[] = "YQL_MOCK"; - + static constexpr char ActorName[] = "YQL_MOCK"; + private: STRICT_STFUNC(Handler, { HFunc(NHttp::TEvHttpProxy::TEvHttpIncomingRequest, Handle); @@ -76,7 +76,7 @@ void InitTest(NActors::TTestActorRuntime* runtime, int httpPort, int grpcPort, c { yqSharedResources->Init(runtime->GetAnyNodeActorSystem()); - auto httpProxyId = NYq::MakeYqlAnalyticsHttpProxyId(); + auto httpProxyId = NYq::MakeYqlAnalyticsHttpProxyId(); TActorId mockActorId = runtime->Register(CreateYqlMockActor(grpcPort)); diff --git a/ydb/core/yq/libs/private_client/private_client.cpp b/ydb/core/yq/libs/private_client/private_client.cpp index bce67cbd3b..bccb8ddd1d 100644 --- a/ydb/core/yq/libs/private_client/private_client.cpp +++ b/ydb/core/yq/libs/private_client/private_client.cpp @@ -1,183 +1,183 @@ -#include "private_client.h" +#include "private_client.h" #include <ydb/library/yql/public/issue/yql_issue.h> #include <ydb/library/yql/public/issue/yql_issue_message.h> #include <ydb/public/sdk/cpp/client/ydb_common_client/impl/client.h> - -namespace NYq { - -using namespace NYdb; - -class TPrivateClient::TImpl : public TClientImplCommon<TPrivateClient::TImpl> { -public: - TImpl( - std::shared_ptr<TGRpcConnectionsImpl>&& connections, - const TCommonClientSettings& settings, - const NMonitoring::TDynamicCounterPtr& counters) - : TClientImplCommon(std::move(connections), settings) - , Counters(counters->GetSubgroup("subsystem", "private_api")->GetSubgroup("subcomponent", "ClientMetrics")) - , PingTaskTime(Counters->GetHistogram("PingTaskMs", NMonitoring::ExponentialHistogram(10, 2, 50))) - , GetTaskTime(Counters->GetHistogram("GetTaskMs", NMonitoring::ExponentialHistogram(10, 2, 50))) - , WriteTaskResultTime(Counters->GetHistogram("WriteTaskResultMs", NMonitoring::ExponentialHistogram(10, 2, 50))) - , NodesHealthCheckTime(Counters->GetHistogram("NodesHealthCheckMs", NMonitoring::ExponentialHistogram(10, 2, 50))) - {} - - template<class TProtoResult, class TResultWrapper> - auto MakeResultExtractor(NThreading::TPromise<TResultWrapper> promise, const NMonitoring::THistogramPtr& hist, TInstant startedAt) { - return [=, promise = std::move(promise)] - (google::protobuf::Any* any, TPlainStatus status) mutable { - std::unique_ptr<TProtoResult> result; - if (any) { - result.reset(new TProtoResult); - any->UnpackTo(result.get()); - } - - hist->Collect((TInstant::Now() - startedAt).MilliSeconds()); - - promise.SetValue( - TResultWrapper( - TStatus(std::move(status)), - std::move(result))); - }; - } - - TAsyncPingTaskResult PingTask( - Yq::Private::PingTaskRequest&& request, - const TPingTaskSettings& settings) { - const auto startedAt = TInstant::Now(); - auto promise = NThreading::NewPromise<TPingTaskResult>(); - auto future = promise.GetFuture(); - - auto extractor = MakeResultExtractor< - Yq::Private::PingTaskResult, - TPingTaskResult>(std::move(promise), PingTaskTime, startedAt); - - Connections_->RunDeferred< - Yq::Private::V1::YqPrivateTaskService, - Yq::Private::PingTaskRequest, - Yq::Private::PingTaskResponse>( - std::move(request), - std::move(extractor), - &Yq::Private::V1::YqPrivateTaskService::Stub::AsyncPingTask, - DbDriverState_, - INITIAL_DEFERRED_CALL_DELAY, - TRpcRequestSettings::Make(settings), - settings.ClientTimeout_); - - return future; - } - - TAsyncGetTaskResult GetTask( - Yq::Private::GetTaskRequest&& request, - const TGetTaskSettings& settings) { - const auto startedAt = TInstant::Now(); - auto promise = NThreading::NewPromise<TGetTaskResult>(); - auto future = promise.GetFuture(); - auto extractor = MakeResultExtractor< - Yq::Private::GetTaskResult, - TGetTaskResult>(std::move(promise), GetTaskTime, startedAt); - - Connections_->RunDeferred< - Yq::Private::V1::YqPrivateTaskService, - Yq::Private::GetTaskRequest, - Yq::Private::GetTaskResponse>( - std::move(request), - std::move(extractor), - &Yq::Private::V1::YqPrivateTaskService::Stub::AsyncGetTask, - DbDriverState_, - INITIAL_DEFERRED_CALL_DELAY, - TRpcRequestSettings::Make(settings), - settings.ClientTimeout_); - - return future; - } - - TAsyncWriteTaskResult WriteTaskResult( - Yq::Private::WriteTaskResultRequest&& request, - const TWriteTaskResultSettings& settings) { - const auto startedAt = TInstant::Now(); - auto promise = NThreading::NewPromise<TWriteTaskResult>(); - auto future = promise.GetFuture(); - auto extractor = MakeResultExtractor< - Yq::Private::WriteTaskResultResult, - TWriteTaskResult>(std::move(promise), WriteTaskResultTime, startedAt); - - Connections_->RunDeferred< - Yq::Private::V1::YqPrivateTaskService, - Yq::Private::WriteTaskResultRequest, - Yq::Private::WriteTaskResultResponse>( - std::move(request), - std::move(extractor), - &Yq::Private::V1::YqPrivateTaskService::Stub::AsyncWriteTaskResult, - DbDriverState_, - INITIAL_DEFERRED_CALL_DELAY, - TRpcRequestSettings::Make(settings), - settings.ClientTimeout_); - - return future; - } - - TAsyncNodesHealthCheckResult NodesHealthCheck( - Yq::Private::NodesHealthCheckRequest&& request, - const TNodesHealthCheckSettings& settings) { - const auto startedAt = TInstant::Now(); - auto promise = NThreading::NewPromise<TNodesHealthCheckResult>(); - auto future = promise.GetFuture(); - auto extractor = MakeResultExtractor< - Yq::Private::NodesHealthCheckResult, - TNodesHealthCheckResult>(std::move(promise), NodesHealthCheckTime, startedAt); - - Connections_->RunDeferred< - Yq::Private::V1::YqPrivateTaskService, - Yq::Private::NodesHealthCheckRequest, - Yq::Private::NodesHealthCheckResponse>( - std::move(request), - std::move(extractor), - &Yq::Private::V1::YqPrivateTaskService::Stub::AsyncNodesHealthCheck, - DbDriverState_, - INITIAL_DEFERRED_CALL_DELAY, - TRpcRequestSettings::Make(settings), - settings.ClientTimeout_); - - return future; - } -private: - const NMonitoring::TDynamicCounterPtr Counters; - const NMonitoring::THistogramPtr PingTaskTime; - const NMonitoring::THistogramPtr GetTaskTime; - const NMonitoring::THistogramPtr WriteTaskResultTime; - const NMonitoring::THistogramPtr NodesHealthCheckTime; -}; - -TPrivateClient::TPrivateClient( - const TDriver& driver, - const TCommonClientSettings& settings, - const NMonitoring::TDynamicCounterPtr& counters) - : Impl(new TImpl(CreateInternalInterface(driver), settings, counters)) -{} - -TAsyncPingTaskResult TPrivateClient::PingTask( - Yq::Private::PingTaskRequest&& request, - const TPingTaskSettings& settings) { + +namespace NYq { + +using namespace NYdb; + +class TPrivateClient::TImpl : public TClientImplCommon<TPrivateClient::TImpl> { +public: + TImpl( + std::shared_ptr<TGRpcConnectionsImpl>&& connections, + const TCommonClientSettings& settings, + const NMonitoring::TDynamicCounterPtr& counters) + : TClientImplCommon(std::move(connections), settings) + , Counters(counters->GetSubgroup("subsystem", "private_api")->GetSubgroup("subcomponent", "ClientMetrics")) + , PingTaskTime(Counters->GetHistogram("PingTaskMs", NMonitoring::ExponentialHistogram(10, 2, 50))) + , GetTaskTime(Counters->GetHistogram("GetTaskMs", NMonitoring::ExponentialHistogram(10, 2, 50))) + , WriteTaskResultTime(Counters->GetHistogram("WriteTaskResultMs", NMonitoring::ExponentialHistogram(10, 2, 50))) + , NodesHealthCheckTime(Counters->GetHistogram("NodesHealthCheckMs", NMonitoring::ExponentialHistogram(10, 2, 50))) + {} + + template<class TProtoResult, class TResultWrapper> + auto MakeResultExtractor(NThreading::TPromise<TResultWrapper> promise, const NMonitoring::THistogramPtr& hist, TInstant startedAt) { + return [=, promise = std::move(promise)] + (google::protobuf::Any* any, TPlainStatus status) mutable { + std::unique_ptr<TProtoResult> result; + if (any) { + result.reset(new TProtoResult); + any->UnpackTo(result.get()); + } + + hist->Collect((TInstant::Now() - startedAt).MilliSeconds()); + + promise.SetValue( + TResultWrapper( + TStatus(std::move(status)), + std::move(result))); + }; + } + + TAsyncPingTaskResult PingTask( + Yq::Private::PingTaskRequest&& request, + const TPingTaskSettings& settings) { + const auto startedAt = TInstant::Now(); + auto promise = NThreading::NewPromise<TPingTaskResult>(); + auto future = promise.GetFuture(); + + auto extractor = MakeResultExtractor< + Yq::Private::PingTaskResult, + TPingTaskResult>(std::move(promise), PingTaskTime, startedAt); + + Connections_->RunDeferred< + Yq::Private::V1::YqPrivateTaskService, + Yq::Private::PingTaskRequest, + Yq::Private::PingTaskResponse>( + std::move(request), + std::move(extractor), + &Yq::Private::V1::YqPrivateTaskService::Stub::AsyncPingTask, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, + TRpcRequestSettings::Make(settings), + settings.ClientTimeout_); + + return future; + } + + TAsyncGetTaskResult GetTask( + Yq::Private::GetTaskRequest&& request, + const TGetTaskSettings& settings) { + const auto startedAt = TInstant::Now(); + auto promise = NThreading::NewPromise<TGetTaskResult>(); + auto future = promise.GetFuture(); + auto extractor = MakeResultExtractor< + Yq::Private::GetTaskResult, + TGetTaskResult>(std::move(promise), GetTaskTime, startedAt); + + Connections_->RunDeferred< + Yq::Private::V1::YqPrivateTaskService, + Yq::Private::GetTaskRequest, + Yq::Private::GetTaskResponse>( + std::move(request), + std::move(extractor), + &Yq::Private::V1::YqPrivateTaskService::Stub::AsyncGetTask, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, + TRpcRequestSettings::Make(settings), + settings.ClientTimeout_); + + return future; + } + + TAsyncWriteTaskResult WriteTaskResult( + Yq::Private::WriteTaskResultRequest&& request, + const TWriteTaskResultSettings& settings) { + const auto startedAt = TInstant::Now(); + auto promise = NThreading::NewPromise<TWriteTaskResult>(); + auto future = promise.GetFuture(); + auto extractor = MakeResultExtractor< + Yq::Private::WriteTaskResultResult, + TWriteTaskResult>(std::move(promise), WriteTaskResultTime, startedAt); + + Connections_->RunDeferred< + Yq::Private::V1::YqPrivateTaskService, + Yq::Private::WriteTaskResultRequest, + Yq::Private::WriteTaskResultResponse>( + std::move(request), + std::move(extractor), + &Yq::Private::V1::YqPrivateTaskService::Stub::AsyncWriteTaskResult, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, + TRpcRequestSettings::Make(settings), + settings.ClientTimeout_); + + return future; + } + + TAsyncNodesHealthCheckResult NodesHealthCheck( + Yq::Private::NodesHealthCheckRequest&& request, + const TNodesHealthCheckSettings& settings) { + const auto startedAt = TInstant::Now(); + auto promise = NThreading::NewPromise<TNodesHealthCheckResult>(); + auto future = promise.GetFuture(); + auto extractor = MakeResultExtractor< + Yq::Private::NodesHealthCheckResult, + TNodesHealthCheckResult>(std::move(promise), NodesHealthCheckTime, startedAt); + + Connections_->RunDeferred< + Yq::Private::V1::YqPrivateTaskService, + Yq::Private::NodesHealthCheckRequest, + Yq::Private::NodesHealthCheckResponse>( + std::move(request), + std::move(extractor), + &Yq::Private::V1::YqPrivateTaskService::Stub::AsyncNodesHealthCheck, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, + TRpcRequestSettings::Make(settings), + settings.ClientTimeout_); + + return future; + } +private: + const NMonitoring::TDynamicCounterPtr Counters; + const NMonitoring::THistogramPtr PingTaskTime; + const NMonitoring::THistogramPtr GetTaskTime; + const NMonitoring::THistogramPtr WriteTaskResultTime; + const NMonitoring::THistogramPtr NodesHealthCheckTime; +}; + +TPrivateClient::TPrivateClient( + const TDriver& driver, + const TCommonClientSettings& settings, + const NMonitoring::TDynamicCounterPtr& counters) + : Impl(new TImpl(CreateInternalInterface(driver), settings, counters)) +{} + +TAsyncPingTaskResult TPrivateClient::PingTask( + Yq::Private::PingTaskRequest&& request, + const TPingTaskSettings& settings) { return Impl->PingTask(std::move(request), settings); -} - -TAsyncGetTaskResult TPrivateClient::GetTask( - Yq::Private::GetTaskRequest&& request, - const TGetTaskSettings& settings) { +} + +TAsyncGetTaskResult TPrivateClient::GetTask( + Yq::Private::GetTaskRequest&& request, + const TGetTaskSettings& settings) { return Impl->GetTask(std::move(request), settings); -} - -TAsyncWriteTaskResult TPrivateClient::WriteTaskResult( - Yq::Private::WriteTaskResultRequest&& request, - const TWriteTaskResultSettings& settings) { +} + +TAsyncWriteTaskResult TPrivateClient::WriteTaskResult( + Yq::Private::WriteTaskResultRequest&& request, + const TWriteTaskResultSettings& settings) { return Impl->WriteTaskResult(std::move(request), settings); -} - -TAsyncNodesHealthCheckResult TPrivateClient::NodesHealthCheck( - Yq::Private::NodesHealthCheckRequest&& request, - const TNodesHealthCheckSettings& settings) { +} + +TAsyncNodesHealthCheckResult TPrivateClient::NodesHealthCheck( + Yq::Private::NodesHealthCheckRequest&& request, + const TNodesHealthCheckSettings& settings) { return Impl->NodesHealthCheck(std::move(request), settings); -} - -} //NYq - +} + +} //NYq + diff --git a/ydb/core/yq/libs/private_client/private_client.h b/ydb/core/yq/libs/private_client/private_client.h index 35c8e1c0a4..e63b67eddb 100644 --- a/ydb/core/yq/libs/private_client/private_client.h +++ b/ydb/core/yq/libs/private_client/private_client.h @@ -1,81 +1,81 @@ -#pragma once - -#include <library/cpp/monlib/dynamic_counters/counters.h> - +#pragma once + +#include <library/cpp/monlib/dynamic_counters/counters.h> + #include <ydb/public/sdk/cpp/client/ydb_table/table.h> #include <ydb/public/api/grpc/draft/yql_db_v1.grpc.pb.h> - -namespace NYq { - -template<class TProtoResult> -class TProtoResultInternalWrapper : public NYdb::TStatus { - friend class TPrivateClient; - + +namespace NYq { + +template<class TProtoResult> +class TProtoResultInternalWrapper : public NYdb::TStatus { + friend class TPrivateClient; + public: - TProtoResultInternalWrapper( - NYdb::TStatus&& status, - std::unique_ptr<TProtoResult> result) - : TStatus(std::move(status)) - , Result(std::move(result)) - { } - -public: - const TProtoResult& GetResult() const { - Y_VERIFY(Result, "Uninitialized result"); - return *Result; - } - - bool IsResultSet() const { - return Result ? true : false; - } - -private: - std::unique_ptr<TProtoResult> Result; -}; - - -using TGetTaskResult = TProtoResultInternalWrapper<Yq::Private::GetTaskResult>; -using TPingTaskResult = TProtoResultInternalWrapper<Yq::Private::PingTaskResult>; -using TWriteTaskResult = TProtoResultInternalWrapper<Yq::Private::WriteTaskResultResult>; -using TNodesHealthCheckResult = TProtoResultInternalWrapper<Yq::Private::NodesHealthCheckResult>; - -using TAsyncGetTaskResult = NThreading::TFuture<TGetTaskResult>; -using TAsyncPingTaskResult = NThreading::TFuture<TPingTaskResult>; -using TAsyncWriteTaskResult = NThreading::TFuture<TWriteTaskResult>; -using TAsyncNodesHealthCheckResult = NThreading::TFuture<TNodesHealthCheckResult>; - -struct TGetTaskSettings : public NYdb::TOperationRequestSettings<TGetTaskSettings> {}; -struct TPingTaskSettings : public NYdb::TOperationRequestSettings<TPingTaskSettings> {}; -struct TWriteTaskResultSettings : public NYdb::TOperationRequestSettings<TWriteTaskResultSettings> {}; -struct TNodesHealthCheckSettings : public NYdb::TOperationRequestSettings<TNodesHealthCheckSettings> {}; - -class TPrivateClient { - class TImpl; - -public: - TPrivateClient( - const NYdb::TDriver& driver, - const NYdb::TCommonClientSettings& settings = NYdb::TCommonClientSettings(), - const NMonitoring::TDynamicCounterPtr& counters = MakeIntrusive<NMonitoring::TDynamicCounters>()); - - TAsyncGetTaskResult GetTask( - Yq::Private::GetTaskRequest&& request, - const TGetTaskSettings& settings = TGetTaskSettings()); - - TAsyncPingTaskResult PingTask( - Yq::Private::PingTaskRequest&& request, - const TPingTaskSettings& settings = TPingTaskSettings()); - - TAsyncWriteTaskResult WriteTaskResult( - Yq::Private::WriteTaskResultRequest&& request, - const TWriteTaskResultSettings& settings = TWriteTaskResultSettings()); - - TAsyncNodesHealthCheckResult NodesHealthCheck( - Yq::Private::NodesHealthCheckRequest&& request, - const TNodesHealthCheckSettings& settings = TNodesHealthCheckSettings()); - -private: + TProtoResultInternalWrapper( + NYdb::TStatus&& status, + std::unique_ptr<TProtoResult> result) + : TStatus(std::move(status)) + , Result(std::move(result)) + { } + +public: + const TProtoResult& GetResult() const { + Y_VERIFY(Result, "Uninitialized result"); + return *Result; + } + + bool IsResultSet() const { + return Result ? true : false; + } + +private: + std::unique_ptr<TProtoResult> Result; +}; + + +using TGetTaskResult = TProtoResultInternalWrapper<Yq::Private::GetTaskResult>; +using TPingTaskResult = TProtoResultInternalWrapper<Yq::Private::PingTaskResult>; +using TWriteTaskResult = TProtoResultInternalWrapper<Yq::Private::WriteTaskResultResult>; +using TNodesHealthCheckResult = TProtoResultInternalWrapper<Yq::Private::NodesHealthCheckResult>; + +using TAsyncGetTaskResult = NThreading::TFuture<TGetTaskResult>; +using TAsyncPingTaskResult = NThreading::TFuture<TPingTaskResult>; +using TAsyncWriteTaskResult = NThreading::TFuture<TWriteTaskResult>; +using TAsyncNodesHealthCheckResult = NThreading::TFuture<TNodesHealthCheckResult>; + +struct TGetTaskSettings : public NYdb::TOperationRequestSettings<TGetTaskSettings> {}; +struct TPingTaskSettings : public NYdb::TOperationRequestSettings<TPingTaskSettings> {}; +struct TWriteTaskResultSettings : public NYdb::TOperationRequestSettings<TWriteTaskResultSettings> {}; +struct TNodesHealthCheckSettings : public NYdb::TOperationRequestSettings<TNodesHealthCheckSettings> {}; + +class TPrivateClient { + class TImpl; + +public: + TPrivateClient( + const NYdb::TDriver& driver, + const NYdb::TCommonClientSettings& settings = NYdb::TCommonClientSettings(), + const NMonitoring::TDynamicCounterPtr& counters = MakeIntrusive<NMonitoring::TDynamicCounters>()); + + TAsyncGetTaskResult GetTask( + Yq::Private::GetTaskRequest&& request, + const TGetTaskSettings& settings = TGetTaskSettings()); + + TAsyncPingTaskResult PingTask( + Yq::Private::PingTaskRequest&& request, + const TPingTaskSettings& settings = TPingTaskSettings()); + + TAsyncWriteTaskResult WriteTaskResult( + Yq::Private::WriteTaskResultRequest&& request, + const TWriteTaskResultSettings& settings = TWriteTaskResultSettings()); + + TAsyncNodesHealthCheckResult NodesHealthCheck( + Yq::Private::NodesHealthCheckRequest&& request, + const TNodesHealthCheckSettings& settings = TNodesHealthCheckSettings()); + +private: std::shared_ptr<TImpl> Impl; -}; - -} // namespace NYq +}; + +} // namespace NYq diff --git a/ydb/core/yq/libs/private_client/utils.cpp b/ydb/core/yq/libs/private_client/utils.cpp index e90b6f4513..f1ba5c7e22 100644 --- a/ydb/core/yq/libs/private_client/utils.cpp +++ b/ydb/core/yq/libs/private_client/utils.cpp @@ -1,57 +1,57 @@ -#include "utils.h" -#include <library/cpp/protobuf/json/json2proto.h> -#include <library/cpp/protobuf/json/proto2json.h> +#include "utils.h" +#include <library/cpp/protobuf/json/json2proto.h> +#include <library/cpp/protobuf/json/proto2json.h> #include <ydb/public/lib/ydb_cli/commands/ydb_common.h> - -namespace NYq { - + +namespace NYq { + using namespace NYdb::NYq; -void UpdateConnections( - TClient& client, +void UpdateConnections( + TClient& client, const TString& folderId, const TString& connectionsStr) { - NJson::TJsonValue value; - TStringStream in(connectionsStr); - NJson::ReadJsonTree(&in, &value); - YandexQuery::ListConnectionsRequest listConsRequest; - listConsRequest.set_limit(100); - auto result = client - .ListConnections(listConsRequest, CreateYqSettings<TListConnectionsSettings>(folderId)) - .ExtractValueSync(); - NYdb::NConsoleClient::ThrowOnError(result); - THashMap<TString, YandexQuery::Connection> connections; - for (const auto& connection : result.GetResult().connection()) { - if (!connection.content().name()) { - continue; - } - connections.emplace(connection.content().name(), connection); - } - for (auto conJson : value.GetArraySafe()) { - const auto& content = conJson["content"]; - const TString conName = content["name"].GetString(); - auto it = connections.find(conName); - if (it == connections.end()) { - Cerr << " Add " << conName << Endl; - YandexQuery::CreateConnectionRequest request; - NProtobufJson::Json2Proto(conJson, request); - const auto result = client - .CreateConnection(request, CreateYqSettings<TCreateConnectionSettings>(folderId)) - .ExtractValueSync(); - result.GetIssues().PrintTo(Cerr); - NYdb::NConsoleClient::ThrowOnError(result); - } else { - Cerr << " Update " << conName << Endl; - YandexQuery::ModifyConnectionRequest request; - conJson["connection_id"] = it->second.meta().id(); - NProtobufJson::Json2Proto(conJson, request); - const auto result = client - .ModifyConnection(request, CreateYqSettings<TModifyConnectionSettings>(folderId)) - .ExtractValueSync(); - result.GetIssues().PrintTo(Cerr); - NYdb::NConsoleClient::ThrowOnError(result); - } - } -} - -} //NYq + NJson::TJsonValue value; + TStringStream in(connectionsStr); + NJson::ReadJsonTree(&in, &value); + YandexQuery::ListConnectionsRequest listConsRequest; + listConsRequest.set_limit(100); + auto result = client + .ListConnections(listConsRequest, CreateYqSettings<TListConnectionsSettings>(folderId)) + .ExtractValueSync(); + NYdb::NConsoleClient::ThrowOnError(result); + THashMap<TString, YandexQuery::Connection> connections; + for (const auto& connection : result.GetResult().connection()) { + if (!connection.content().name()) { + continue; + } + connections.emplace(connection.content().name(), connection); + } + for (auto conJson : value.GetArraySafe()) { + const auto& content = conJson["content"]; + const TString conName = content["name"].GetString(); + auto it = connections.find(conName); + if (it == connections.end()) { + Cerr << " Add " << conName << Endl; + YandexQuery::CreateConnectionRequest request; + NProtobufJson::Json2Proto(conJson, request); + const auto result = client + .CreateConnection(request, CreateYqSettings<TCreateConnectionSettings>(folderId)) + .ExtractValueSync(); + result.GetIssues().PrintTo(Cerr); + NYdb::NConsoleClient::ThrowOnError(result); + } else { + Cerr << " Update " << conName << Endl; + YandexQuery::ModifyConnectionRequest request; + conJson["connection_id"] = it->second.meta().id(); + NProtobufJson::Json2Proto(conJson, request); + const auto result = client + .ModifyConnection(request, CreateYqSettings<TModifyConnectionSettings>(folderId)) + .ExtractValueSync(); + result.GetIssues().PrintTo(Cerr); + NYdb::NConsoleClient::ThrowOnError(result); + } + } +} + +} //NYq diff --git a/ydb/core/yq/libs/private_client/utils.h b/ydb/core/yq/libs/private_client/utils.h index b2131b2806..ff1b255c31 100644 --- a/ydb/core/yq/libs/private_client/utils.h +++ b/ydb/core/yq/libs/private_client/utils.h @@ -1,12 +1,12 @@ -#pragma once +#pragma once #include <ydb/public/lib/yq/yq.h> #include <ydb/public/lib/yq/helpers.h> - -namespace NYq { - -void UpdateConnections( + +namespace NYq { + +void UpdateConnections( NYdb::NYq::TClient& client, const TString& folderId, const TString& connectionsStr); - -} // NYq + +} // NYq diff --git a/ydb/core/yq/libs/private_client/ya.make b/ydb/core/yq/libs/private_client/ya.make index 50dd2d6cb6..247be520a7 100644 --- a/ydb/core/yq/libs/private_client/ya.make +++ b/ydb/core/yq/libs/private_client/ya.make @@ -3,13 +3,13 @@ LIBRARY() OWNER(g:kikimr) SRCS( - utils.cpp - private_client.cpp + utils.cpp + private_client.cpp ) PEERDIR( - library/cpp/monlib/dynamic_counters - library/cpp/protobuf/json + library/cpp/monlib/dynamic_counters + library/cpp/protobuf/json ydb/public/api/grpc/draft ydb/public/sdk/cpp/client/ydb_table ) diff --git a/ydb/core/yq/libs/read_rule/read_rule_creator.cpp b/ydb/core/yq/libs/read_rule/read_rule_creator.cpp index 406cd3df9f..4ca1a5ca47 100644 --- a/ydb/core/yq/libs/read_rule/read_rule_creator.cpp +++ b/ydb/core/yq/libs/read_rule/read_rule_creator.cpp @@ -89,7 +89,7 @@ public: } static constexpr char ActorName[] = "YQ_SINGLE_READ_RULE_CREATOR"; - + TString GetTopicPath() const { TStringBuilder ret; ret << Topic.GetDatabase(); @@ -146,13 +146,13 @@ public: PassAway(); } else { if (!CheckFinish()) { - Schedule(*nextRetryDelay, new NActors::TEvents::TEvWakeup()); + Schedule(*nextRetryDelay, new NActors::TEvents::TEvWakeup()); } } } } - void Handle(NActors::TEvents::TEvWakeup::TPtr&) { + void Handle(NActors::TEvents::TEvWakeup::TPtr&) { if (!CheckFinish()) { StartRequest(); } @@ -175,7 +175,7 @@ public: STRICT_STFUNC(StateFunc, hFunc(TEvPrivate::TEvAddReadRuleStatus, Handle); - hFunc(NActors::TEvents::TEvWakeup, Handle); + hFunc(NActors::TEvents::TEvWakeup, Handle); hFunc(NActors::TEvents::TEvPoison, Handle); ) @@ -223,7 +223,7 @@ public: } static constexpr char ActorName[] = "YQ_READ_RULE_CREATOR"; - + void Bootstrap() { Become(&TReadRuleCreator::StateFunc); @@ -266,7 +266,7 @@ public: } issues.AddIssue(std::move(mainIssue)); } - Send(Owner, MakeHolder<NYq::TEvents::TEvDataStreamsReadRulesCreationResult>(std::move(issues))); + Send(Owner, MakeHolder<NYq::TEvents::TEvDataStreamsReadRulesCreationResult>(std::move(issues))); PassAway(); } } diff --git a/ydb/core/yq/libs/read_rule/read_rule_deleter.cpp b/ydb/core/yq/libs/read_rule/read_rule_deleter.cpp index 71c5e8f8c1..b5c2e4d549 100644 --- a/ydb/core/yq/libs/read_rule/read_rule_deleter.cpp +++ b/ydb/core/yq/libs/read_rule/read_rule_deleter.cpp @@ -66,7 +66,7 @@ public: NActors::TActorId owner, TString queryId, NYdb::TDriver ydbDriver, - Yq::Private::TopicConsumer topic, + Yq::Private::TopicConsumer topic, std::shared_ptr<NYdb::ICredentialsProviderFactory> credentialsProvider, ui64 index, size_t maxRetries @@ -82,7 +82,7 @@ public: } static constexpr char ActorName[] = "YQ_SINGLE_READ_RULE_DELETER"; - + void Bootstrap() { Become(&TSingleReadRuleDeleter::StateFunc); StartRequest(); @@ -139,18 +139,18 @@ public: Send(Owner, MakeHolder<TEvPrivate::TEvSingleReadRuleDeleterResult>(status.GetIssues()), 0, Index); PassAway(); } else { - Schedule(*nextRetryDelay, new NActors::TEvents::TEvWakeup()); + Schedule(*nextRetryDelay, new NActors::TEvents::TEvWakeup()); } } } - void Handle(NActors::TEvents::TEvWakeup::TPtr&) { + void Handle(NActors::TEvents::TEvWakeup::TPtr&) { StartRequest(); } STRICT_STFUNC(StateFunc, hFunc(TEvPrivate::TEvRemoveReadRuleStatus, Handle); - hFunc(NActors::TEvents::TEvWakeup, Handle); + hFunc(NActors::TEvents::TEvWakeup, Handle); cFunc(NActors::TEvents::TEvPoison::EventType, PassAway); ) @@ -168,7 +168,7 @@ private: private: const NActors::TActorId Owner; const TString QueryId; - const Yq::Private::TopicConsumer Topic; + const Yq::Private::TopicConsumer Topic; NYdb::TDriver YdbDriver; NYdb::NPersQueue::TPersQueueClient PqClient; ui64 Index = 0; @@ -183,7 +183,7 @@ public: NActors::TActorId owner, TString queryId, NYdb::TDriver ydbDriver, - TVector<Yq::Private::TopicConsumer> topics, + TVector<Yq::Private::TopicConsumer> topics, TVector<std::shared_ptr<NYdb::ICredentialsProviderFactory>> credentials, size_t maxRetries ) @@ -210,7 +210,7 @@ public: } static constexpr char ActorName[] = "YQ_READ_RULE_DELETER"; - + void Handle(TEvPrivate::TEvSingleReadRuleDeleterResult::TPtr& ev) { const ui64 index = ev->Cookie; Y_VERIFY(!Results[index]); @@ -242,7 +242,7 @@ public: } issues.AddIssue(std::move(mainIssue)); } - Send(Owner, MakeHolder<TEvents::TEvDataStreamsReadRulesDeletionResult>(std::move(issues))); + Send(Owner, MakeHolder<TEvents::TEvDataStreamsReadRulesDeletionResult>(std::move(issues))); PassAway(); } } @@ -256,7 +256,7 @@ private: const NActors::TActorId Owner; const TString QueryId; NYdb::TDriver YdbDriver; - const TVector<Yq::Private::TopicConsumer> Topics; + const TVector<Yq::Private::TopicConsumer> Topics; const TVector<std::shared_ptr<NYdb::ICredentialsProviderFactory>> Credentials; const size_t MaxRetries; size_t ResultsGot = 0; @@ -271,7 +271,7 @@ NActors::IActor* MakeReadRuleDeleterActor( NActors::TActorId owner, TString queryId, NYdb::TDriver ydbDriver, - TVector<Yq::Private::TopicConsumer> topics, + TVector<Yq::Private::TopicConsumer> topics, TVector<std::shared_ptr<NYdb::ICredentialsProviderFactory>> credentials, // For each topic size_t maxRetries ) diff --git a/ydb/core/yq/libs/read_rule/read_rule_deleter.h b/ydb/core/yq/libs/read_rule/read_rule_deleter.h index e76e17340f..b3a92745a0 100644 --- a/ydb/core/yq/libs/read_rule/read_rule_deleter.h +++ b/ydb/core/yq/libs/read_rule/read_rule_deleter.h @@ -11,7 +11,7 @@ NActors::IActor* MakeReadRuleDeleterActor( NActors::TActorId owner, TString queryId, NYdb::TDriver ydbDriver, - TVector<Yq::Private::TopicConsumer> topics, + TVector<Yq::Private::TopicConsumer> topics, TVector<std::shared_ptr<NYdb::ICredentialsProviderFactory>> credentials, // For each topic size_t maxRetries = 15 ); diff --git a/ydb/core/yq/libs/result_formatter/result_formatter.cpp b/ydb/core/yq/libs/result_formatter/result_formatter.cpp index b4da61874d..394832b7e5 100644 --- a/ydb/core/yq/libs/result_formatter/result_formatter.cpp +++ b/ydb/core/yq/libs/result_formatter/result_formatter.cpp @@ -13,10 +13,10 @@ #include <library/cpp/json/yson/json2yson.h> -namespace NYq { +namespace NYq { using namespace NKikimr::NMiniKQL; -using NYql::NUdf::TUnboxedValuePod; +using NYql::NUdf::TUnboxedValuePod; namespace { @@ -154,7 +154,7 @@ NKikimr::NMiniKQL::TType* MakeEmptyDictType(NKikimr::NMiniKQL::TTypeEnvironment& return env.GetTypeOfEmptyDict(); } -const NYql::TTypeAnnotationNode* MakeVariantType(const NYql::TTypeAnnotationNode* underlyingType, NYql::TExprContext& ctx) +const NYql::TTypeAnnotationNode* MakeVariantType(const NYql::TTypeAnnotationNode* underlyingType, NYql::TExprContext& ctx) { return ctx.MakeType<NYql::TVariantExprType>(underlyingType); } @@ -164,7 +164,7 @@ NKikimr::NMiniKQL::TType* MakeVariantType(NKikimr::NMiniKQL::TType* underlyingTy return TVariantType::Create(underlyingType, env); } -const NYql::TTypeAnnotationNode* MakeTaggedType(const TString& tag, const NYql::TTypeAnnotationNode* underlyingType, NYql::TExprContext& ctx) +const NYql::TTypeAnnotationNode* MakeTaggedType(const TString& tag, const NYql::TTypeAnnotationNode* underlyingType, NYql::TExprContext& ctx) { return ctx.MakeType<NYql::TTaggedExprType>(underlyingType, tag); } @@ -323,7 +323,7 @@ void FormatColumnValue( { const Ydb::Value& rawProtoValue = NYdb::TProtoAccessor::GetProto(value); - NYql::NUdf::TUnboxedValue unboxed = ImportValueFromProto( + NYql::NUdf::TUnboxedValue unboxed = ImportValueFromProto( type, rawProtoValue, holderFactory); @@ -389,4 +389,4 @@ void FormatResultSet(NJson::TJsonValue& root, const NYdb::TResultSet& resultSet) } } -} // namespace NYq +} // namespace NYq diff --git a/ydb/core/yq/libs/result_formatter/result_formatter.h b/ydb/core/yq/libs/result_formatter/result_formatter.h index bbe4724c41..031196e050 100644 --- a/ydb/core/yq/libs/result_formatter/result_formatter.h +++ b/ydb/core/yq/libs/result_formatter/result_formatter.h @@ -8,9 +8,9 @@ #include <library/cpp/json/json_writer.h> -namespace NYq { +namespace NYq { void FormatResultSet(NJson::TJsonValue& root, const NYdb::TResultSet& resultSet); TString FormatSchema(const YandexQuery::Schema& schema); -} // namespace NYq +} // namespace NYq diff --git a/ydb/core/yq/libs/result_formatter/result_formatter_ut.cpp b/ydb/core/yq/libs/result_formatter/result_formatter_ut.cpp index a7a49ebe00..bebbf68ab3 100644 --- a/ydb/core/yq/libs/result_formatter/result_formatter_ut.cpp +++ b/ydb/core/yq/libs/result_formatter/result_formatter_ut.cpp @@ -2,7 +2,7 @@ #include <ydb/services/ydb/ydb_common_ut.h> -using namespace NYq; +using namespace NYq; Y_UNIT_TEST_SUITE(ResultFormatter) { Y_UNIT_TEST(Primitive) { diff --git a/ydb/core/yq/libs/result_formatter/ut/ya.make b/ydb/core/yq/libs/result_formatter/ut/ya.make index b6cf21e339..371714623b 100644 --- a/ydb/core/yq/libs/result_formatter/ut/ya.make +++ b/ydb/core/yq/libs/result_formatter/ut/ya.make @@ -1,25 +1,25 @@ UNITTEST_FOR(ydb/core/yq/libs/result_formatter) - + OWNER( g:kikimr g:yq ) -FORK_SUBTESTS() - -IF (SANITIZER_TYPE OR WITH_VALGRIND) - SIZE(MEDIUM) -ENDIF() - -SRCS( - result_formatter_ut.cpp -) - -PEERDIR( +FORK_SUBTESTS() + +IF (SANITIZER_TYPE OR WITH_VALGRIND) + SIZE(MEDIUM) +ENDIF() + +SRCS( + result_formatter_ut.cpp +) + +PEERDIR( ydb/library/yql/public/udf/service/stub ydb/services/ydb -) - -YQL_LAST_ABI_VERSION() - -END() +) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/ydb/core/yq/libs/result_formatter/ya.make b/ydb/core/yq/libs/result_formatter/ya.make index 9a166bf337..16a912791d 100644 --- a/ydb/core/yq/libs/result_formatter/ya.make +++ b/ydb/core/yq/libs/result_formatter/ya.make @@ -1,14 +1,14 @@ OWNER(g:yq) - -LIBRARY() - -SRCS( - result_formatter.cpp -) - -PEERDIR( - library/cpp/json - library/cpp/json/yson + +LIBRARY() + +SRCS( + result_formatter.cpp +) + +PEERDIR( + library/cpp/json + library/cpp/json/yson ydb/core/engine ydb/library/yql/ast ydb/library/yql/minikql @@ -21,12 +21,12 @@ PEERDIR( ydb/library/yql/providers/common/codec ydb/library/yql/providers/common/schema/expr ydb/library/yql/providers/common/schema/mkql -) - -YQL_LAST_ABI_VERSION() - -END() - +) + +YQL_LAST_ABI_VERSION() + +END() + RECURSE_FOR_TESTS( ut ) diff --git a/ydb/core/yq/libs/shared_resources/db_pool.cpp b/ydb/core/yq/libs/shared_resources/db_pool.cpp index aec0866c7e..372dfd72cd 100644 --- a/ydb/core/yq/libs/shared_resources/db_pool.cpp +++ b/ydb/core/yq/libs/shared_resources/db_pool.cpp @@ -12,24 +12,24 @@ #define LOG_E(stream) \ LOG_ERROR_S(*TlsActivationContext, NKikimrServices::YQL_PROXY, stream) -namespace NYq { +namespace NYq { using namespace NActors; using namespace NYql; class TDbPoolActor : public NActors::TActor<TDbPoolActor> { public: - TDbPoolActor( - const NYdb::NTable::TTableClient& tableClient, - const NMonitoring::TDynamicCounterPtr& counters) + TDbPoolActor( + const NYdb::NTable::TTableClient& tableClient, + const NMonitoring::TDynamicCounterPtr& counters) : TActor(&TThis::WorkingState) , TableClient(tableClient) - , QueueSize(counters->GetSubgroup("subcomponent", "DbPool")->GetHistogram("InFlight", NMonitoring::ExponentialHistogram(10, 2, 10))) - , TotalInFlight(counters->GetSubgroup("subcomponent", "DbPool")->GetCounter("TotalInflight")) - , RequestsTime(counters->GetSubgroup("subcomponent", "DbPool")->GetHistogram("RequestTimeMs", NMonitoring::ExponentialHistogram(6, 3, 100))) + , QueueSize(counters->GetSubgroup("subcomponent", "DbPool")->GetHistogram("InFlight", NMonitoring::ExponentialHistogram(10, 2, 10))) + , TotalInFlight(counters->GetSubgroup("subcomponent", "DbPool")->GetCounter("TotalInflight")) + , RequestsTime(counters->GetSubgroup("subcomponent", "DbPool")->GetHistogram("RequestTimeMs", NMonitoring::ExponentialHistogram(6, 3, 100))) {} - static constexpr char ActorName[] = "YQ_DB_POOL"; + static constexpr char ActorName[] = "YQ_DB_POOL"; STRICT_STFUNC(WorkingState, CFunc(NActors::TEvents::TEvPoison::EventType, Die) @@ -56,14 +56,14 @@ public: } void ProcessQueue(const TActorContext& ctx) { - QueueSize->Collect(Requests.size()); + QueueSize->Collect(Requests.size()); if (Requests.empty() || RequestInProgress) { return; } - TotalInFlight->Inc(); + TotalInFlight->Inc(); RequestInProgress = true; - RequestInProgressTimestamp = TInstant::Now(); + RequestInProgressTimestamp = TInstant::Now(); const auto& requestVariant = Requests.front(); if (auto pRequest = std::get_if<TRequest>(&requestVariant)) { @@ -110,20 +110,20 @@ public: ProcessQueue(ctx); } - void PopFromQueueAndProcess(const TActorContext& ctx) { + void PopFromQueueAndProcess(const TActorContext& ctx) { RequestInProgress = false; - RequestsTime->Collect(TInstant::Now().MilliSeconds() - RequestInProgressTimestamp.MilliSeconds()); + RequestsTime->Collect(TInstant::Now().MilliSeconds() - RequestInProgressTimestamp.MilliSeconds()); Requests.pop_front(); - TotalInFlight->Dec(); + TotalInFlight->Dec(); ProcessQueue(ctx); } - void HandleResponse(TEvents::TEvDbResponse::TPtr& ev, const TActorContext& ctx) { - const auto& request = Requests.front(); - ctx.Send(ev->Forward(std::visit([](const auto& arg) { return arg.Sender; }, request))); - PopFromQueueAndProcess(ctx); - } - + void HandleResponse(TEvents::TEvDbResponse::TPtr& ev, const TActorContext& ctx) { + const auto& request = Requests.front(); + ctx.Send(ev->Forward(std::visit([](const auto& arg) { return arg.Sender; }, request))); + PopFromQueueAndProcess(ctx); + } + void HandleRequest(TEvents::TEvDbFunctionRequest::TPtr& ev, const TActorContext& ctx) { auto request = ev->Get(); Requests.emplace_back(TFunctionRequest{ev->Sender, ev->Cookie, std::move(request->Handler)}); @@ -133,7 +133,7 @@ public: void HandleResponse(TEvents::TEvDbFunctionResponse::TPtr& ev, const TActorContext& ctx) { const auto& request = Requests.front(); ctx.Send(ev->Forward(std::visit([](const auto& arg) { return arg.Sender; }, request))); - PopFromQueueAndProcess(ctx); + PopFromQueueAndProcess(ctx); } private: @@ -171,17 +171,17 @@ private: NYdb::NTable::TTableClient TableClient; TDeque<std::variant<TRequest, TFunctionRequest>> Requests; bool RequestInProgress = false; - TInstant RequestInProgressTimestamp = TInstant::Now(); + TInstant RequestInProgressTimestamp = TInstant::Now(); std::shared_ptr<int> State = std::make_shared<int>(); - const NMonitoring::THistogramPtr QueueSize; - const NMonitoring::TDynamicCounters::TCounterPtr TotalInFlight; - const NMonitoring::THistogramPtr RequestsTime; + const NMonitoring::THistogramPtr QueueSize; + const NMonitoring::TDynamicCounters::TCounterPtr TotalInFlight; + const NMonitoring::THistogramPtr RequestsTime; }; -TDbPool::TDbPool( - ui32 sessionsCount, - const NYdb::NTable::TTableClient& tableClient, - const NMonitoring::TDynamicCounterPtr& counters) +TDbPool::TDbPool( + ui32 sessionsCount, + const NYdb::NTable::TTableClient& tableClient, + const NMonitoring::TDynamicCounterPtr& counters) { const auto& ctx = NActors::TActivationContext::AsActorContext(); auto parentId = ctx.SelfID; @@ -189,7 +189,7 @@ TDbPool::TDbPool( for (ui32 i = 0; i < sessionsCount; ++i) { Actors.emplace_back( NActors::TActivationContext::Register( - new TDbPoolActor(tableClient, counters), + new TDbPoolActor(tableClient, counters), parentId, NActors::TMailboxType::HTSwap, parentId.PoolID())); } } @@ -224,12 +224,12 @@ static void PrepareConfig(NYq::NConfig::TDbPoolConfig& config) { TDbPoolMap::TDbPoolMap( const NYq::NConfig::TDbPoolConfig& config, NYdb::TDriver driver, - const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory, - const NMonitoring::TDynamicCounterPtr& counters) + const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory, + const NMonitoring::TDynamicCounterPtr& counters) : Config(config) , Driver(driver) , CredentialsProviderFactory(credentialsProviderFactory) - , Counters(counters) + , Counters(counters) { PrepareConfig(Config); } @@ -237,10 +237,10 @@ TDbPoolMap::TDbPoolMap( TDbPoolHolder::TDbPoolHolder( const NYq::NConfig::TDbPoolConfig& config, const NYdb::TDriver& driver, - const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory, - const NMonitoring::TDynamicCounterPtr& counters) + const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory, + const NMonitoring::TDynamicCounterPtr& counters) : Driver(driver) - , Pools(new TDbPoolMap(config, Driver, credentialsProviderFactory, counters)) + , Pools(new TDbPoolMap(config, Driver, credentialsProviderFactory, counters)) { } TDbPoolHolder::~TDbPoolHolder() @@ -297,7 +297,7 @@ TDbPool::TPtr TDbPoolMap::GetOrCreate(EDbPoolId dbPoolId, ui32 sessionsCount) { TableClient = MakeHolder<NYdb::NTable::TTableClient>(Driver, clientSettings); } - TDbPool::TPtr dbPool = new TDbPool(sessionsCount, *TableClient, Counters); + TDbPool::TPtr dbPool = new TDbPool(sessionsCount, *TableClient, Counters); Pools.emplace(dbPoolId, dbPool); return dbPool; } @@ -306,4 +306,4 @@ NYdb::TDriver& TDbPoolHolder::GetDriver() { return Driver; } -} /* namespace NYq */ +} /* namespace NYq */ diff --git a/ydb/core/yq/libs/shared_resources/db_pool.h b/ydb/core/yq/libs/shared_resources/db_pool.h index 461726990f..68d9adc2fc 100644 --- a/ydb/core/yq/libs/shared_resources/db_pool.h +++ b/ydb/core/yq/libs/shared_resources/db_pool.h @@ -7,11 +7,11 @@ #include <ydb/library/security/ydb_credentials_provider_factory.h> #include <library/cpp/actors/core/actor.h> -#include <library/cpp/monlib/dynamic_counters/counters.h> +#include <library/cpp/monlib/dynamic_counters/counters.h> #include <util/system/mutex.h> -namespace NYq { +namespace NYq { class TDbPool: public TThrRefBase { public: @@ -24,12 +24,12 @@ public: private: friend class TDbPoolMap; - TDbPool(ui32 sessionsCount, const NYdb::NTable::TTableClient& tableClient, const NMonitoring::TDynamicCounterPtr& counters); + TDbPool(ui32 sessionsCount, const NYdb::NTable::TTableClient& tableClient, const NMonitoring::TDynamicCounterPtr& counters); TMutex Mutex; TVector<NActors::TActorId> Actors; ui32 Index = 0; - const NMonitoring::TDynamicCounterPtr Counters; + const NMonitoring::TDynamicCounterPtr Counters; }; enum class EDbPoolId { @@ -48,9 +48,9 @@ private: TDbPoolMap(const NYq::NConfig::TDbPoolConfig& config, NYdb::TDriver driver, - const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory, - const NMonitoring::TDynamicCounterPtr& counters); - + const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory, + const NMonitoring::TDynamicCounterPtr& counters); + void Reset(const NYq::NConfig::TDbPoolConfig& config); TMutex Mutex; NYq::NConfig::TDbPoolConfig Config; @@ -58,7 +58,7 @@ private: THashMap<EDbPoolId, TDbPool::TPtr> Pools; THolder<NYdb::NTable::TTableClient> TableClient; NKikimr::TYdbCredentialsProviderFactory CredentialsProviderFactory; - const NMonitoring::TDynamicCounterPtr Counters; + const NMonitoring::TDynamicCounterPtr Counters; }; class TDbPoolHolder: public TThrRefBase { @@ -67,9 +67,9 @@ public: TDbPoolHolder( const NYq::NConfig::TDbPoolConfig& config, const NYdb::TDriver& driver, - const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory, - const NMonitoring::TDynamicCounterPtr& counters); - + const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory, + const NMonitoring::TDynamicCounterPtr& counters); + ~TDbPoolHolder(); void Reset(const NYq::NConfig::TDbPoolConfig& config); @@ -82,4 +82,4 @@ public: TDbPoolMap::TPtr Pools; }; -} /* NYq */ +} /* NYq */ diff --git a/ydb/core/yq/libs/shared_resources/shared_resources.cpp b/ydb/core/yq/libs/shared_resources/shared_resources.cpp index 2504a6abdf..3dfd566232 100644 --- a/ydb/core/yq/libs/shared_resources/shared_resources.cpp +++ b/ydb/core/yq/libs/shared_resources/shared_resources.cpp @@ -86,11 +86,11 @@ struct TActorSystemPtrMixin { struct TYqSharedResourcesImpl : public TActorSystemPtrMixin, public TYqSharedResources { explicit TYqSharedResourcesImpl( const NYq::NConfig::TConfig& config, - const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory, - const NMonitoring::TDynamicCounterPtr& counters) + const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory, + const NMonitoring::TDynamicCounterPtr& counters) : TYqSharedResources(NYdb::TDriver(GetYdbDriverConfig(config.GetCommon().GetYdbDriverConfig()))) { - CreateDbPoolHolder(config.GetDbPool(), credentialsProviderFactory, counters); + CreateDbPoolHolder(config.GetDbPool(), credentialsProviderFactory, counters); } void Init(NActors::TActorSystem* actorSystem) override { @@ -120,9 +120,9 @@ struct TYqSharedResourcesImpl : public TActorSystemPtrMixin, public TYqSharedRes void CreateDbPoolHolder( const NYq::NConfig::TDbPoolConfig& config, - const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory, - const NMonitoring::TDynamicCounterPtr& counters) { - DbPoolHolder = MakeIntrusive<NYq::TDbPoolHolder>(config, YdbDriver, credentialsProviderFactory, counters); + const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory, + const NMonitoring::TDynamicCounterPtr& counters) { + DbPoolHolder = MakeIntrusive<NYq::TDbPoolHolder>(config, YdbDriver, credentialsProviderFactory, counters); } }; @@ -130,9 +130,9 @@ struct TYqSharedResourcesImpl : public TActorSystemPtrMixin, public TYqSharedRes TYqSharedResources::TPtr CreateYqSharedResourcesImpl( const NYq::NConfig::TConfig& config, - const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory, - const NMonitoring::TDynamicCounterPtr& counters) { - return MakeIntrusive<TYqSharedResourcesImpl>(config, credentialsProviderFactory, counters); + const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory, + const NMonitoring::TDynamicCounterPtr& counters) { + return MakeIntrusive<TYqSharedResourcesImpl>(config, credentialsProviderFactory, counters); } TYqSharedResources::TYqSharedResources(NYdb::TDriver driver) diff --git a/ydb/core/yq/libs/shared_resources/shared_resources.h b/ydb/core/yq/libs/shared_resources/shared_resources.h index 2f24ab9258..bfd1ec6695 100644 --- a/ydb/core/yq/libs/shared_resources/shared_resources.h +++ b/ydb/core/yq/libs/shared_resources/shared_resources.h @@ -17,7 +17,7 @@ struct TYqSharedResources : public IYqSharedResources { // Resources NYdb::TDriver YdbDriver; - TDbPoolHolder::TPtr DbPoolHolder; + TDbPoolHolder::TPtr DbPoolHolder; protected: explicit TYqSharedResources(NYdb::TDriver driver); @@ -25,7 +25,7 @@ protected: TYqSharedResources::TPtr CreateYqSharedResourcesImpl( const NYq::NConfig::TConfig& config, - const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory, - const NMonitoring::TDynamicCounterPtr& counters); + const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory, + const NMonitoring::TDynamicCounterPtr& counters); } // namespace NYq diff --git a/ydb/core/yq/libs/shared_resources/ya.make b/ydb/core/yq/libs/shared_resources/ya.make index fed68026c5..46c9a3bb2b 100644 --- a/ydb/core/yq/libs/shared_resources/ya.make +++ b/ydb/core/yq/libs/shared_resources/ya.make @@ -9,7 +9,7 @@ SRCS( PEERDIR( library/cpp/actors/core - library/cpp/monlib/dynamic_counters + library/cpp/monlib/dynamic_counters ydb/core/protos ydb/core/yq/libs/events ydb/core/yq/libs/shared_resources/interface diff --git a/ydb/core/yq/libs/ya.make b/ydb/core/yq/libs/ya.make index 613e2d963c..ccc88a55b6 100644 --- a/ydb/core/yq/libs/ya.make +++ b/ydb/core/yq/libs/ya.make @@ -6,11 +6,11 @@ RECURSE( checkpoint_storage checkpointing checkpointing_common - common + common config control_plane_proxy control_plane_storage - db_resolver + db_resolver db_schema events gateway @@ -20,9 +20,9 @@ RECURSE( logs mock pretty_printers - private_client + private_client read_rule - result_formatter + result_formatter shared_resources signer tasks_packer diff --git a/ydb/library/mkql_proto/mkql_proto.cpp b/ydb/library/mkql_proto/mkql_proto.cpp index 52bef4723a..53d7722dee 100644 --- a/ydb/library/mkql_proto/mkql_proto.cpp +++ b/ydb/library/mkql_proto/mkql_proto.cpp @@ -425,8 +425,8 @@ Y_FORCE_INLINE void HandleKindDataExport(const TType* type, const NUdf::TUnboxed void ExportValueToProtoImpl(TType* type, const NUdf::TUnboxedValuePod& value, NKikimrMiniKQL::TValue& res, const TVector<ui32>* columnOrder = nullptr) { switch (type->GetKind()) { case TType::EKind::Void: - case TType::EKind::EmptyList: - case TType::EKind::EmptyDict: + case TType::EKind::EmptyList: + case TType::EKind::EmptyDict: break; case TType::EKind::Null: { @@ -507,12 +507,12 @@ void ExportValueToProtoImpl(TType* type, const NUdf::TUnboxedValuePod& value, NK break; } - case TType::EKind::Tagged: { - auto taggedType = static_cast<TTaggedType*>(type); - ExportValueToProtoImpl(taggedType->GetBaseType(), value, res); - break; - } - + case TType::EKind::Tagged: { + auto taggedType = static_cast<TTaggedType*>(type); + ExportValueToProtoImpl(taggedType->GetBaseType(), value, res); + break; + } + default: MKQL_ENSURE(false, TStringBuilder() << "Unknown kind: " << type->GetKindAsStr()); } @@ -521,9 +521,9 @@ void ExportValueToProtoImpl(TType* type, const NUdf::TUnboxedValuePod& value, NK void ExportValueToProtoImpl(TType* type, const NUdf::TUnboxedValuePod& value, Ydb::Value& res, const TVector<ui32>* columnOrder = nullptr) { switch (type->GetKind()) { case TType::EKind::Void: - case TType::EKind::EmptyList: - case TType::EKind::EmptyDict: - break; + case TType::EKind::EmptyList: + case TType::EKind::EmptyDict: + break; case TType::EKind::Null: { res.set_null_flag_value(::google::protobuf::NULL_VALUE); diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_actor_checkpoints.h b/ydb/library/yql/dq/actors/compute/dq_compute_actor_checkpoints.h index 5efedb370f..27fd851338 100644 --- a/ydb/library/yql/dq/actors/compute/dq_compute_actor_checkpoints.h +++ b/ydb/library/yql/dq/actors/compute/dq_compute_actor_checkpoints.h @@ -64,8 +64,8 @@ class TDqComputeActorCheckpoints : public NActors::TActor<TDqComputeActorCheckpo }; public: - static constexpr char ActorName[] = "DQ_COMPUTE_ACTOR_CHECKPOINTS"; - + static constexpr char ActorName[] = "DQ_COMPUTE_ACTOR_CHECKPOINTS"; + struct ICallbacks { [[nodiscard]] virtual bool ReadyToCheckpoint() const = 0; diff --git a/ydb/library/yql/providers/clickhouse/provider/ya.make b/ydb/library/yql/providers/clickhouse/provider/ya.make index e2cfd1cb42..05effd38eb 100644 --- a/ydb/library/yql/providers/clickhouse/provider/ya.make +++ b/ydb/library/yql/providers/clickhouse/provider/ya.make @@ -9,7 +9,7 @@ SRCS( yql_clickhouse_datasource.cpp yql_clickhouse_datasource_type_ann.cpp yql_clickhouse_dq_integration.cpp - yql_clickhouse_io_discovery.cpp + yql_clickhouse_io_discovery.cpp yql_clickhouse_load_meta.cpp yql_clickhouse_logical_opt.cpp yql_clickhouse_mkql_compiler.cpp @@ -27,8 +27,8 @@ YQL_LAST_ABI_VERSION() PEERDIR( library/cpp/json - library/cpp/random_provider - library/cpp/time_provider + library/cpp/random_provider + library/cpp/time_provider ydb/core/yq/libs/common ydb/core/yq/libs/db_resolver ydb/library/yql/ast diff --git a/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_datasource.cpp b/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_datasource.cpp index c371d97991..e23600f80f 100644 --- a/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_datasource.cpp +++ b/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_datasource.cpp @@ -19,7 +19,7 @@ class TClickHouseDataSource : public TDataProviderBase { public: TClickHouseDataSource(TClickHouseState::TPtr state, IHTTPGateway::TPtr gateway) : State_(state) - , IODiscoveryTransformer_(CreateClickHouseIODiscoveryTransformer(State_)) + , IODiscoveryTransformer_(CreateClickHouseIODiscoveryTransformer(State_)) , LoadMetaDataTransformer_(CreateClickHouseLoadTableMetadataTransformer(State_, std::move(gateway))) , TypeAnnotationTransformer_(CreateClickHouseDataSourceTypeAnnotationTransformer(State_)) , DqIntegration_(CreateClickHouseDqIntegration(State_)) @@ -53,10 +53,10 @@ public: return TypeAnnotationTransformer_->CanParse(node); } - IGraphTransformer& GetIODiscoveryTransformer() override { - return *IODiscoveryTransformer_; - } - + IGraphTransformer& GetIODiscoveryTransformer() override { + return *IODiscoveryTransformer_; + } + IGraphTransformer& GetLoadTableMetadataTransformer() override { return *LoadMetaDataTransformer_; } diff --git a/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_io_discovery.cpp b/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_io_discovery.cpp index 9bc674949f..193c1e7bae 100644 --- a/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_io_discovery.cpp +++ b/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_io_discovery.cpp @@ -1,118 +1,118 @@ -#include "yql_clickhouse_provider_impl.h" +#include "yql_clickhouse_provider_impl.h" #include <ydb/library/yql/providers/clickhouse/expr_nodes/yql_clickhouse_expr_nodes.h> #include <ydb/library/yql/providers/common/provider/yql_provider_names.h> #include <ydb/library/yql/core/expr_nodes/yql_expr_nodes.h> #include <ydb/library/yql/core/yql_expr_optimize.h> #include <ydb/library/yql/core/yql_graph_transformer.h> #include <ydb/library/yql/ast/yql_expr.h> - + #include <ydb/core/yq/libs/events/events.h> - -namespace NYql { - -namespace { - -using namespace NNodes; - -class TClickHouseIODiscoveryTransformer : public TGraphTransformerBase { - -using TDbId2Endpoint = THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TEvEndpointResponse::TEndpoint>; - -public: - TClickHouseIODiscoveryTransformer(TClickHouseState::TPtr state) - : State_(std::move(state)) - {} - - TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final { - output = input; - - if (ctx.Step.IsDone(TExprStep::DiscoveryIO)) - return TStatus::Ok; - - if (!State_->DbResolver) - return TStatus::Ok; - - THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TDatabaseAuth> ids; - if (auto reads = FindNodes(input, [&](const TExprNode::TPtr& node) { - const TExprBase nodeExpr(node); - if (!nodeExpr.Maybe<TClRead>()) - return false; - - auto read = nodeExpr.Maybe<TClRead>().Cast(); - if (read.DataSource().Category().Value() != ClickHouseProviderName) { - return false; - } - return true; - }); !reads.empty()) { - for (auto& node : reads) { - const TClRead read(node); - const auto cluster = read.DataSource().Cluster().StringValue(); - YQL_CLOG(DEBUG, ProviderClickHouse) << "Found cluster: " << cluster; - auto dbId = State_->Configuration->Endpoints[cluster].first; - dbId = dbId.substr(0, dbId.find(':')); - YQL_CLOG(DEBUG, ProviderClickHouse) << "Found dbId: " << dbId; - const auto idKey = std::make_pair(dbId, NYq::DatabaseType::ClickHouse); - const auto iter = State_->DatabaseIds.find(idKey); - if (iter != State_->DatabaseIds.end()) { - YQL_CLOG(DEBUG, ProviderClickHouse) << "Resolve CH id: " << dbId; - ids[idKey] = iter->second; - } - } - } - YQL_CLOG(DEBUG, ProviderClickHouse) << "Ids to resolve: " << ids.size(); - if (ids.empty()) { - return TStatus::Ok; - } - AsyncFuture_ = State_->DbResolver->ResolveIds({ids, State_->DbResolver->GetTraceId()}).Apply([resolvedIds_ = ResolvedIds_](const auto& future) { - *resolvedIds_ = future.GetValue(); - }); - return TStatus::Async; - } - - NThreading::TFuture<void> DoGetAsyncFuture(const TExprNode&) final { - return AsyncFuture_; - } - - TStatus DoApplyAsyncChanges(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext&) final { - output = input; - AsyncFuture_.GetValue(); - FullResolvedIds_.insert(ResolvedIds_->begin(), ResolvedIds_->end()); - ResolvedIds_->clear(); - YQL_CLOG(DEBUG, ProviderClickHouse) << "ResolvedIds: " << FullResolvedIds_.size(); - auto& endpoints = State_->Configuration->Endpoints; - const auto& id2Clusters = State_->Configuration->DbId2Clusters; - for (const auto& [dbIdWithType, info] : FullResolvedIds_) { - const auto& dbId = dbIdWithType.first; - const auto iter = id2Clusters.find(dbId); - if (iter == id2Clusters.end()) { - continue; - } - for (const auto& clusterName : iter->second) { - YQL_CLOG(DEBUG, ProviderClickHouse) << "Resolved endpoint: " << info.Endpoint << " for id: "<< dbId; - auto& endpoint = endpoints[clusterName].first; - auto& secure = endpoints[clusterName].second; - if (const auto it = endpoint.find(':'); it != TString::npos) { - secure = info.Secure; - const auto port = endpoint.substr(it); - endpoint = info.Endpoint; - if (info.Endpoint.find(':') == TString::npos) - endpoint += port; - } - } - } - return TStatus::Ok; - } -private: - const TClickHouseState::TPtr State_; - - NThreading::TFuture<void> AsyncFuture_; - TDbId2Endpoint FullResolvedIds_; - std::shared_ptr<TDbId2Endpoint> ResolvedIds_ = std::make_shared<TDbId2Endpoint>(); -}; -} - -THolder<IGraphTransformer> CreateClickHouseIODiscoveryTransformer(TClickHouseState::TPtr state) { - return THolder(new TClickHouseIODiscoveryTransformer(std::move(state))); -} - -} + +namespace NYql { + +namespace { + +using namespace NNodes; + +class TClickHouseIODiscoveryTransformer : public TGraphTransformerBase { + +using TDbId2Endpoint = THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TEvEndpointResponse::TEndpoint>; + +public: + TClickHouseIODiscoveryTransformer(TClickHouseState::TPtr state) + : State_(std::move(state)) + {} + + TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final { + output = input; + + if (ctx.Step.IsDone(TExprStep::DiscoveryIO)) + return TStatus::Ok; + + if (!State_->DbResolver) + return TStatus::Ok; + + THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TDatabaseAuth> ids; + if (auto reads = FindNodes(input, [&](const TExprNode::TPtr& node) { + const TExprBase nodeExpr(node); + if (!nodeExpr.Maybe<TClRead>()) + return false; + + auto read = nodeExpr.Maybe<TClRead>().Cast(); + if (read.DataSource().Category().Value() != ClickHouseProviderName) { + return false; + } + return true; + }); !reads.empty()) { + for (auto& node : reads) { + const TClRead read(node); + const auto cluster = read.DataSource().Cluster().StringValue(); + YQL_CLOG(DEBUG, ProviderClickHouse) << "Found cluster: " << cluster; + auto dbId = State_->Configuration->Endpoints[cluster].first; + dbId = dbId.substr(0, dbId.find(':')); + YQL_CLOG(DEBUG, ProviderClickHouse) << "Found dbId: " << dbId; + const auto idKey = std::make_pair(dbId, NYq::DatabaseType::ClickHouse); + const auto iter = State_->DatabaseIds.find(idKey); + if (iter != State_->DatabaseIds.end()) { + YQL_CLOG(DEBUG, ProviderClickHouse) << "Resolve CH id: " << dbId; + ids[idKey] = iter->second; + } + } + } + YQL_CLOG(DEBUG, ProviderClickHouse) << "Ids to resolve: " << ids.size(); + if (ids.empty()) { + return TStatus::Ok; + } + AsyncFuture_ = State_->DbResolver->ResolveIds({ids, State_->DbResolver->GetTraceId()}).Apply([resolvedIds_ = ResolvedIds_](const auto& future) { + *resolvedIds_ = future.GetValue(); + }); + return TStatus::Async; + } + + NThreading::TFuture<void> DoGetAsyncFuture(const TExprNode&) final { + return AsyncFuture_; + } + + TStatus DoApplyAsyncChanges(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext&) final { + output = input; + AsyncFuture_.GetValue(); + FullResolvedIds_.insert(ResolvedIds_->begin(), ResolvedIds_->end()); + ResolvedIds_->clear(); + YQL_CLOG(DEBUG, ProviderClickHouse) << "ResolvedIds: " << FullResolvedIds_.size(); + auto& endpoints = State_->Configuration->Endpoints; + const auto& id2Clusters = State_->Configuration->DbId2Clusters; + for (const auto& [dbIdWithType, info] : FullResolvedIds_) { + const auto& dbId = dbIdWithType.first; + const auto iter = id2Clusters.find(dbId); + if (iter == id2Clusters.end()) { + continue; + } + for (const auto& clusterName : iter->second) { + YQL_CLOG(DEBUG, ProviderClickHouse) << "Resolved endpoint: " << info.Endpoint << " for id: "<< dbId; + auto& endpoint = endpoints[clusterName].first; + auto& secure = endpoints[clusterName].second; + if (const auto it = endpoint.find(':'); it != TString::npos) { + secure = info.Secure; + const auto port = endpoint.substr(it); + endpoint = info.Endpoint; + if (info.Endpoint.find(':') == TString::npos) + endpoint += port; + } + } + } + return TStatus::Ok; + } +private: + const TClickHouseState::TPtr State_; + + NThreading::TFuture<void> AsyncFuture_; + TDbId2Endpoint FullResolvedIds_; + std::shared_ptr<TDbId2Endpoint> ResolvedIds_ = std::make_shared<TDbId2Endpoint>(); +}; +} + +THolder<IGraphTransformer> CreateClickHouseIODiscoveryTransformer(TClickHouseState::TPtr state) { + return THolder(new TClickHouseIODiscoveryTransformer(std::move(state))); +} + +} diff --git a/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_provider.cpp b/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_provider.cpp index 7287848b5d..15c4d834e3 100644 --- a/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_provider.cpp +++ b/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_provider.cpp @@ -4,10 +4,10 @@ namespace NYql { -TDataProviderInitializer GetClickHouseDataProviderInitializer( +TDataProviderInitializer GetClickHouseDataProviderInitializer( IHTTPGateway::TPtr gateway, - const std::shared_ptr<NYq::TDatabaseAsyncResolverWithMeta> dbResolverWithMeta) -{ + const std::shared_ptr<NYq::TDatabaseAsyncResolverWithMeta> dbResolverWithMeta) +{ return [gateway, dbResolverWithMeta] ( const TString& userName, const TString& sessionId, @@ -29,9 +29,9 @@ TDataProviderInitializer GetClickHouseDataProviderInitializer( state->Types = typeCtx.Get(); state->FunctionRegistry = functionRegistry; - state->DbResolver = dbResolverWithMeta; + state->DbResolver = dbResolverWithMeta; if (gatewaysConfig) { - state->Configuration->Init(gatewaysConfig->GetClickHouse(), state->DbResolver, state->DatabaseIds); + state->Configuration->Init(gatewaysConfig->GetClickHouse(), state->DbResolver, state->DatabaseIds); } TDataProviderInfo info; diff --git a/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_provider.h b/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_provider.h index 3b918ca877..7184fe1ba3 100644 --- a/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_provider.h +++ b/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_provider.h @@ -26,14 +26,14 @@ struct TClickHouseState : public TThrRefBase TTypeAnnotationContext* Types = nullptr; TClickHouseConfiguration::TPtr Configuration = MakeIntrusive<TClickHouseConfiguration>(); const NKikimr::NMiniKQL::IFunctionRegistry* FunctionRegistry = nullptr; - THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TDatabaseAuth> DatabaseIds; - std::shared_ptr<NYq::TDatabaseAsyncResolverWithMeta> DbResolver; + THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TDatabaseAuth> DatabaseIds; + std::shared_ptr<NYq::TDatabaseAsyncResolverWithMeta> DbResolver; }; -TDataProviderInitializer GetClickHouseDataProviderInitializer( +TDataProviderInitializer GetClickHouseDataProviderInitializer( IHTTPGateway::TPtr gateway, - std::shared_ptr<NYq::TDatabaseAsyncResolverWithMeta> dbResolverWithMeta = nullptr -); + std::shared_ptr<NYq::TDatabaseAsyncResolverWithMeta> dbResolverWithMeta = nullptr +); TIntrusivePtr<IDataProvider> CreateClickHouseDataSource(TClickHouseState::TPtr state, IHTTPGateway::TPtr gateway); TIntrusivePtr<IDataProvider> CreateClickHouseDataSink(TClickHouseState::TPtr state); diff --git a/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_provider_impl.h b/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_provider_impl.h index 026b613b1c..cdf5c20405 100644 --- a/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_provider_impl.h +++ b/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_provider_impl.h @@ -10,7 +10,7 @@ namespace NYql { -THolder<IGraphTransformer> CreateClickHouseIODiscoveryTransformer(TClickHouseState::TPtr state); +THolder<IGraphTransformer> CreateClickHouseIODiscoveryTransformer(TClickHouseState::TPtr state); THolder<IGraphTransformer> CreateClickHouseLoadTableMetadataTransformer(TClickHouseState::TPtr state, IHTTPGateway::TPtr gateway); THolder<TVisitorTransformerBase> CreateClickHouseDataSourceTypeAnnotationTransformer(TClickHouseState::TPtr state); diff --git a/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_settings.h b/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_settings.h index 5a29448cf3..8d0b094af4 100644 --- a/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_settings.h +++ b/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_settings.h @@ -7,7 +7,7 @@ #include <ydb/core/yq/libs/events/events.h> #include <ydb/core/yq/libs/db_resolver/db_async_resolver_with_meta.h> #include <ydb/core/yq/libs/common/database_token_builder.h> - + namespace NYql { struct TClickHouseSettings { @@ -21,10 +21,10 @@ struct TClickHouseConfiguration : public TClickHouseSettings, public NCommon::TS TClickHouseConfiguration(const TClickHouseConfiguration&) = delete; template <typename TProtoConfig> - void Init( - const TProtoConfig& config, - const std::shared_ptr<NYq::TDatabaseAsyncResolverWithMeta> dbResolver, - THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TDatabaseAuth>& databaseIds) + void Init( + const TProtoConfig& config, + const std::shared_ptr<NYq::TDatabaseAsyncResolverWithMeta> dbResolver, + THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TDatabaseAuth>& databaseIds) { TVector<TString> clusters(Reserve(config.ClusterMappingSize())); for (auto& cluster: config.GetClusterMapping()) { @@ -36,28 +36,28 @@ struct TClickHouseConfiguration : public TClickHouseSettings, public NCommon::TS this->Dispatch(config.GetDefaultSettings()); for (auto& cluster: config.GetClusterMapping()) { this->Dispatch(cluster.GetName(), cluster.GetSettings()); - - if (dbResolver) { //TODO: change log level - YQL_CLOG(DEBUG, ProviderClickHouse) << "Settings: clusterName = " << cluster.GetName() - << ", clusterDbId = " << cluster.GetId() << ", cluster.GetCluster(): " << cluster.GetCluster() << ", HasCluster: " << (cluster.HasCluster() ? "TRUE" : "FALSE") ; - dbResolver->TryAddDbIdToResolve(cluster.HasCluster(), cluster.GetName(), cluster.GetId(), NYq::DatabaseType::ClickHouse, databaseIds); - if (cluster.GetId()) { - DbId2Clusters[cluster.GetId()].emplace_back(cluster.GetName()); - YQL_CLOG(DEBUG, ProviderClickHouse) << "Add dbId: " << cluster.GetId() << " to DbId2Clusters"; - } - } - + + if (dbResolver) { //TODO: change log level + YQL_CLOG(DEBUG, ProviderClickHouse) << "Settings: clusterName = " << cluster.GetName() + << ", clusterDbId = " << cluster.GetId() << ", cluster.GetCluster(): " << cluster.GetCluster() << ", HasCluster: " << (cluster.HasCluster() ? "TRUE" : "FALSE") ; + dbResolver->TryAddDbIdToResolve(cluster.HasCluster(), cluster.GetName(), cluster.GetId(), NYq::DatabaseType::ClickHouse, databaseIds); + if (cluster.GetId()) { + DbId2Clusters[cluster.GetId()].emplace_back(cluster.GetName()); + YQL_CLOG(DEBUG, ProviderClickHouse) << "Add dbId: " << cluster.GetId() << " to DbId2Clusters"; + } + } + Tokens[cluster.GetName()] = cluster.GetCHToken(); // TODO: Drop later - TString endpoint; - if (cluster.HasCluster()) { - endpoint = cluster.GetCluster(); - if (endpoint.StartsWith("https://")) { - endpoint = endpoint.substr(8); - } - endpoint = endpoint.substr(0, endpoint.find(':')); - } else { - endpoint = cluster.GetId(); + TString endpoint; + if (cluster.HasCluster()) { + endpoint = cluster.GetCluster(); + if (endpoint.StartsWith("https://")) { + endpoint = endpoint.substr(8); + } + endpoint = endpoint.substr(0, endpoint.find(':')); + } else { + endpoint = cluster.GetId(); } Endpoints[cluster.GetName()] = std::make_pair( endpoint + ":" + ToString(cluster.GetNativeHostPort()), cluster.GetNativeSecure()); @@ -100,7 +100,7 @@ struct TClickHouseConfiguration : public TClickHouseSettings, public NCommon::TS THashMap<TString, TString> Tokens; THashMap<TString, std::tuple<TString, EHostScheme, ui16>> Urls; THashMap<TString, std::pair<TString, bool>> Endpoints; - THashMap<TString, TVector<TString>> DbId2Clusters; // DatabaseId -> ClusterNames + THashMap<TString, TVector<TString>> DbId2Clusters; // DatabaseId -> ClusterNames }; } // NYql diff --git a/ydb/library/yql/providers/common/http_gateway/ya.make b/ydb/library/yql/providers/common/http_gateway/ya.make index 8fdcdfe9a3..f53489973e 100644 --- a/ydb/library/yql/providers/common/http_gateway/ya.make +++ b/ydb/library/yql/providers/common/http_gateway/ya.make @@ -10,9 +10,9 @@ SRCS( ) PEERDIR( - contrib/libs/curl - library/cpp/monlib/dynamic_counters - ydb/library/yql/providers/common/proto + contrib/libs/curl + library/cpp/monlib/dynamic_counters + ydb/library/yql/providers/common/proto ydb/library/yql/public/issue ) diff --git a/ydb/library/yql/providers/common/http_gateway/yql_http_gateway.cpp b/ydb/library/yql/providers/common/http_gateway/yql_http_gateway.cpp index 10897fc634..5c2a1451b4 100644 --- a/ydb/library/yql/providers/common/http_gateway/yql_http_gateway.cpp +++ b/ydb/library/yql/providers/common/http_gateway/yql_http_gateway.cpp @@ -131,24 +131,24 @@ public: using TPtr = std::shared_ptr<THTTPMultiGateway>; using TWeakPtr = std::weak_ptr<THTTPMultiGateway>; - explicit THTTPMultiGateway( - const THttpGatewayConfig* httpGatewaysCfg, - NMonitoring::TDynamicCounterPtr counters) - : Counters(std::move(counters)) - , Rps(Counters->GetCounter("RPS", true)) - , InFlight(Counters->GetCounter("InFlight")) - , AllocatedMemory(Counters->GetCounter("AllocatedMemory")) - { - if (!httpGatewaysCfg) { - return; - } - if (httpGatewaysCfg->HasMaxInFlightCount()) { - MaxHandlers = httpGatewaysCfg->GetMaxInFlightCount(); - } - if (httpGatewaysCfg->HasMaxSimulatenousDownloadsSize()) { - MaxSimulatenousDownloadsSize = httpGatewaysCfg->GetMaxSimulatenousDownloadsSize(); - } - } + explicit THTTPMultiGateway( + const THttpGatewayConfig* httpGatewaysCfg, + NMonitoring::TDynamicCounterPtr counters) + : Counters(std::move(counters)) + , Rps(Counters->GetCounter("RPS", true)) + , InFlight(Counters->GetCounter("InFlight")) + , AllocatedMemory(Counters->GetCounter("AllocatedMemory")) + { + if (!httpGatewaysCfg) { + return; + } + if (httpGatewaysCfg->HasMaxInFlightCount()) { + MaxHandlers = httpGatewaysCfg->GetMaxInFlightCount(); + } + if (httpGatewaysCfg->HasMaxSimulatenousDownloadsSize()) { + MaxSimulatenousDownloadsSize = httpGatewaysCfg->GetMaxSimulatenousDownloadsSize(); + } + } ~THTTPMultiGateway() { if (Handle) @@ -156,17 +156,17 @@ public: Thread.join(); } private: - std::size_t MaxHandlers = 1024U; - std::size_t MaxSimulatenousDownloadsSize = 8_GB; + std::size_t MaxHandlers = 1024U; + std::size_t MaxSimulatenousDownloadsSize = 8_GB; static void Perform(const TWeakPtr& weak) { OutputSize.store(0ULL); curl_global_init(CURL_GLOBAL_ALL); if (const auto handle = curl_multi_init()) { - if (const auto& initHandle = weak.lock()) { - initHandle->Handle = handle; - } + if (const auto& initHandle = weak.lock()) { + initHandle->Handle = handle; + } for (size_t handlers = 0U;;) { if (const auto& self = weak.lock()) @@ -218,7 +218,7 @@ private: curl_multi_add_handle(Handle, handle); } - AllocatedMemory->Set(AllocatedSize); + AllocatedMemory->Set(AllocatedSize); return Allocated.size(); } @@ -235,10 +235,10 @@ private: if (Await.empty() && Allocated.empty()) Requests.clear(); } - if (easy) { - InFlight->Dec(); + if (easy) { + InFlight->Dec(); easy->Done(result); - } + } } void Fail(CURLMcode result) { @@ -258,7 +258,7 @@ private: } const TIssue error(curl_multi_strerror(result)); - InFlight->Sub(works.size()); + InFlight->Sub(works.size()); while (!works.empty()) { works.top()->Fail(error); works.pop(); @@ -266,9 +266,9 @@ private: } void Download(TString url, THeaders headers, std::size_t expectedSize, TOnResult callback, TString data) final { - Rps->Inc(); - InFlight->Inc(); - + Rps->Inc(); + InFlight->Inc(); + const std::unique_lock lock(Sync); auto& entry = Requests[TKeyType(url, headers, data)]; if (const auto& easy = entry.lock()) @@ -298,11 +298,11 @@ private: static std::mutex CreateSync; static TWeakPtr Singleton; - - const NMonitoring::TDynamicCounterPtr Counters; - const NMonitoring::TDynamicCounters::TCounterPtr Rps; - const NMonitoring::TDynamicCounters::TCounterPtr InFlight; - const NMonitoring::TDynamicCounters::TCounterPtr AllocatedMemory; + + const NMonitoring::TDynamicCounterPtr Counters; + const NMonitoring::TDynamicCounters::TCounterPtr Rps; + const NMonitoring::TDynamicCounters::TCounterPtr InFlight; + const NMonitoring::TDynamicCounters::TCounterPtr AllocatedMemory; }; std::atomic_size_t THTTPMultiGateway::OutputSize = 0ULL; @@ -386,12 +386,12 @@ IHTTPGateway::TContent::~TContent() template<> IHTTPGateway::TPtr -IHTTPGateway::Make<true>(const THttpGatewayConfig* httpGatewaysCfg, NMonitoring::TDynamicCounterPtr counters) { +IHTTPGateway::Make<true>(const THttpGatewayConfig* httpGatewaysCfg, NMonitoring::TDynamicCounterPtr counters) { const std::unique_lock lock(THTTPMultiGateway::CreateSync); if (const auto g = THTTPMultiGateway::Singleton.lock()) return g; - const auto gateway = std::make_shared<THTTPMultiGateway>(httpGatewaysCfg, std::move(counters)); + const auto gateway = std::make_shared<THTTPMultiGateway>(httpGatewaysCfg, std::move(counters)); THTTPMultiGateway::Singleton = gateway; gateway->Thread = std::thread(std::bind(&THTTPMultiGateway::Perform, THTTPMultiGateway::Singleton)); return gateway; @@ -399,7 +399,7 @@ IHTTPGateway::Make<true>(const THttpGatewayConfig* httpGatewaysCfg, NMonitoring: template<> IHTTPGateway::TPtr -IHTTPGateway::Make<false>(const THttpGatewayConfig*, NMonitoring::TDynamicCounterPtr) { +IHTTPGateway::Make<false>(const THttpGatewayConfig*, NMonitoring::TDynamicCounterPtr) { return std::make_shared<THTTPEasyGateway>(); } diff --git a/ydb/library/yql/providers/common/http_gateway/yql_http_gateway.h b/ydb/library/yql/providers/common/http_gateway/yql_http_gateway.h index a6ddfdcf27..4e15aa9c78 100644 --- a/ydb/library/yql/providers/common/http_gateway/yql_http_gateway.h +++ b/ydb/library/yql/providers/common/http_gateway/yql_http_gateway.h @@ -1,10 +1,10 @@ #pragma once -#include <ydb/library/yql/providers/common/proto/gateways_config.pb.h> - +#include <ydb/library/yql/providers/common/proto/gateways_config.pb.h> + #include <ydb/library/yql/public/issue/yql_issue.h> #include <library/cpp/containers/stack_vector/stack_vec.h> -#include <library/cpp/monlib/dynamic_counters/counters.h> +#include <library/cpp/monlib/dynamic_counters/counters.h> #include <variant> #include <functional> @@ -19,9 +19,9 @@ public: virtual ~IHTTPGateway() = default; template<bool SingleThread = true> - static TPtr Make( - const THttpGatewayConfig* httpGatewaysCfg = nullptr, - NMonitoring::TDynamicCounterPtr counters = MakeIntrusive<NMonitoring::TDynamicCounters>()); + static TPtr Make( + const THttpGatewayConfig* httpGatewaysCfg = nullptr, + NMonitoring::TDynamicCounterPtr counters = MakeIntrusive<NMonitoring::TDynamicCounters>()); class TContent : private TString { friend class TEasyCurl; diff --git a/ydb/library/yql/providers/common/proto/gateways_config.proto b/ydb/library/yql/providers/common/proto/gateways_config.proto index 42c53e1e8e..8561fe2536 100644 --- a/ydb/library/yql/providers/common/proto/gateways_config.proto +++ b/ydb/library/yql/providers/common/proto/gateways_config.proto @@ -50,13 +50,13 @@ enum ETokenType { CREDENTIALS = 3; }; -/////////////////////////////// HTTP GATEWAY ////////////////////// - -message THttpGatewayConfig { - optional uint32 MaxInFlightCount = 1; - optional uint64 MaxSimulatenousDownloadsSize = 2; -} - +/////////////////////////////// HTTP GATEWAY ////////////////////// + +message THttpGatewayConfig { + optional uint32 MaxInFlightCount = 1; + optional uint64 MaxSimulatenousDownloadsSize = 2; +} + /////////////////////////////// YT /////////////////////////////// message TYtClusterConfig { @@ -179,7 +179,7 @@ message TClickHouseClusterConfig { optional NYql.NProto.TClickHouseRunnerConfig Runner = 10; optional uint32 NativeHostPort = 11 [default = 9000]; optional bool NativeSecure = 12 [default = false]; - optional string Id = 13; + optional string Id = 13; } message TClickHouseGatewayConfig { @@ -453,5 +453,5 @@ message TGatewaysConfig { optional TYdbGatewayConfig Ydb = 15; optional TPqGatewayConfig Pq = 16; optional TS3GatewayConfig S3 = 17; - optional THttpGatewayConfig HttpGateway = 18; + optional THttpGatewayConfig HttpGateway = 18; } diff --git a/ydb/library/yql/providers/dq/actors/worker_actor.cpp b/ydb/library/yql/providers/dq/actors/worker_actor.cpp index a7f27f1d8d..f4503a60b2 100644 --- a/ydb/library/yql/providers/dq/actors/worker_actor.cpp +++ b/ydb/library/yql/providers/dq/actors/worker_actor.cpp @@ -71,7 +71,7 @@ class TDqWorker: public TRichActor<TDqWorker> public: static constexpr char ActorName[] = "YQL_DQ_WORKER"; - + explicit TDqWorker( const ITaskRunnerActorFactory::TPtr& taskRunnerActorFactory, const IDqSourceActorFactory::TPtr& sourceActorFactory, diff --git a/ydb/library/yql/providers/dq/api/protos/dqs.proto b/ydb/library/yql/providers/dq/api/protos/dqs.proto index 6d8929be78..09aecfcf92 100644 --- a/ydb/library/yql/providers/dq/api/protos/dqs.proto +++ b/ydb/library/yql/providers/dq/api/protos/dqs.proto @@ -176,7 +176,7 @@ message TQueryResponse { bool NeedFallback = 6; repeated TMetric Metric = 5; bool Truncated = 7; - uint64 RowsCount = 8; + uint64 RowsCount = 8; } message TDqFailure { diff --git a/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp b/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp index f33d6264e3..8759f44221 100644 --- a/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp +++ b/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp @@ -764,10 +764,10 @@ private: } auto raw = NYT::NodeToYsonString(item); - const bool truncated = res.Truncated; - const ui64 rowsCount = res.RowsCount; + const bool truncated = res.Truncated; + const ui64 rowsCount = res.RowsCount; - if (truncated && item.IsList()) { + if (truncated && item.IsList()) { ui64 bytes = 0; ui64 rows = 0; writer.OnBeginList(); @@ -804,11 +804,11 @@ private: writer.OnRaw(raw); } - if (rowsCount) { - writer.OnKeyedItem("RowsCount"); - writer.OnUint64Scalar(rowsCount); - } - + if (rowsCount) { + writer.OnKeyedItem("RowsCount"); + writer.OnUint64Scalar(rowsCount); + } + writer.OnEndMap(); input->SetResult(ctx.NewAtom(input->Pos(), out.Str())); @@ -1100,8 +1100,8 @@ private: auto raw = NYT::NodeToYsonString(item); TString trStr = ""; - const bool truncated = res.Truncated; - const ui64 rowsCount = res.RowsCount; + const bool truncated = res.Truncated; + const ui64 rowsCount = res.RowsCount; if (truncated && !state->TypeCtx->ForceDq && !enableFullResultWrite) { auto issue = TIssue(ctx.GetPosition(input->Pos()), TStringBuilder() << "DQ cannot execute the query. Cause: " << "too big result " << trStr).SetCode(TIssuesIds::DQ_GATEWAY_NEED_FALLBACK_ERROR, TSeverityIds::S_INFO); @@ -1158,11 +1158,11 @@ private: writer.OnRaw(raw); } - if (rowsCount) { - writer.OnKeyedItem("RowsCount"); - writer.OnUint64Scalar(rowsCount); - } - + if (rowsCount) { + writer.OnKeyedItem("RowsCount"); + writer.OnUint64Scalar(rowsCount); + } + writer.OnEndMap(); ctx.IssueManager.RaiseIssues(res.Issues); diff --git a/ydb/library/yql/providers/dq/provider/yql_dq_gateway.h b/ydb/library/yql/providers/dq/provider/yql_dq_gateway.h index 64385ad351..d562776033 100644 --- a/ydb/library/yql/providers/dq/provider/yql_dq_gateway.h +++ b/ydb/library/yql/providers/dq/provider/yql_dq_gateway.h @@ -49,7 +49,7 @@ public: bool ForceFallback = false; bool Retriable = false; bool Truncated = false; - ui64 RowsCount = 0; + ui64 RowsCount = 0; TIssues Issues; TOperationStatistics Statistics; diff --git a/ydb/library/yql/providers/pq/async_io/dq_pq_read_actor.cpp b/ydb/library/yql/providers/pq/async_io/dq_pq_read_actor.cpp index a58129d428..e26956545e 100644 --- a/ydb/library/yql/providers/pq/async_io/dq_pq_read_actor.cpp +++ b/ydb/library/yql/providers/pq/async_io/dq_pq_read_actor.cpp @@ -126,8 +126,8 @@ public: return opts; } - static constexpr char ActorName[] = "DQ_PQ_READ_ACTOR"; - + static constexpr char ActorName[] = "DQ_PQ_READ_ACTOR"; + public: void SaveState(const NDqProto::TCheckpoint& checkpoint, NDqProto::TSourceState& state) override { NPq::NProto::TDqPqTopicSourceState stateProto; diff --git a/ydb/library/yql/providers/pq/async_io/dq_pq_write_actor.cpp b/ydb/library/yql/providers/pq/async_io/dq_pq_write_actor.cpp index 54fb25a147..85fbc91c01 100644 --- a/ydb/library/yql/providers/pq/async_io/dq_pq_write_actor.cpp +++ b/ydb/library/yql/providers/pq/async_io/dq_pq_write_actor.cpp @@ -109,8 +109,8 @@ public: , PersQueueClient(Driver, GetPersQueueClientSettings()) { } - static constexpr char ActorName[] = "DQ_PQ_WRITE_ACTOR"; - + static constexpr char ActorName[] = "DQ_PQ_WRITE_ACTOR"; + public: void SendData( NKikimr::NMiniKQL::TUnboxedValueVector&& batch, diff --git a/ydb/library/yql/providers/pq/gateway/dummy/yql_pq_dummy_gateway.cpp b/ydb/library/yql/providers/pq/gateway/dummy/yql_pq_dummy_gateway.cpp index 8a37bb133e..b1d64befb3 100644 --- a/ydb/library/yql/providers/pq/gateway/dummy/yql_pq_dummy_gateway.cpp +++ b/ydb/library/yql/providers/pq/gateway/dummy/yql_pq_dummy_gateway.cpp @@ -52,16 +52,16 @@ TDummyPqGateway& TDummyPqGateway::AddDummyTopic(const TDummyTopic& topic) { } } -void TDummyPqGateway::UpdateClusterConfigs( - const TString& clusterName, - const TString& endpoint, - const TString& database, - bool secure) -{ - Y_UNUSED(clusterName); - Y_UNUSED(endpoint); - Y_UNUSED(database); - Y_UNUSED(secure); -} - +void TDummyPqGateway::UpdateClusterConfigs( + const TString& clusterName, + const TString& endpoint, + const TString& database, + bool secure) +{ + Y_UNUSED(clusterName); + Y_UNUSED(endpoint); + Y_UNUSED(database); + Y_UNUSED(secure); +} + } // namespace NYql diff --git a/ydb/library/yql/providers/pq/gateway/dummy/yql_pq_dummy_gateway.h b/ydb/library/yql/providers/pq/gateway/dummy/yql_pq_dummy_gateway.h index 78262dc0a1..753c849c20 100644 --- a/ydb/library/yql/providers/pq/gateway/dummy/yql_pq_dummy_gateway.h +++ b/ydb/library/yql/providers/pq/gateway/dummy/yql_pq_dummy_gateway.h @@ -36,12 +36,12 @@ public: NPq::NConfigurationManager::TAsyncDescribePathResult DescribePath(const TString& sessionId, const TString& cluster, const TString& database, const TString& path, const TString& token) override; - void UpdateClusterConfigs( - const TString& clusterName, - const TString& endpoint, - const TString& database, - bool secure) override; - + void UpdateClusterConfigs( + const TString& clusterName, + const TString& endpoint, + const TString& database, + bool secure) override; + private: mutable TMutex Mutex; THashMap<std::pair<TString, TString>, TDummyTopic> Topics; diff --git a/ydb/library/yql/providers/pq/gateway/native/yql_pq_gateway.cpp b/ydb/library/yql/providers/pq/gateway/native/yql_pq_gateway.cpp index dafe21ea2f..bd89565050 100644 --- a/ydb/library/yql/providers/pq/gateway/native/yql_pq_gateway.cpp +++ b/ydb/library/yql/providers/pq/gateway/native/yql_pq_gateway.cpp @@ -26,12 +26,12 @@ public: const TString& path, const TString& token) override; - void UpdateClusterConfigs( - const TString& clusterName, - const TString& endpoint, - const TString& database, - bool secure) override; - + void UpdateClusterConfigs( + const TString& clusterName, + const TString& endpoint, + const TString& database, + bool secure) override; + private: void InitClusterConfigs(); TPqSession::TPtr GetExistingSession(const TString& sessionId) const; @@ -68,22 +68,22 @@ void TPqNativeGateway::InitClusterConfigs() { } } -void TPqNativeGateway::UpdateClusterConfigs( - const TString& clusterName, - const TString& endpoint, - const TString& database, - bool secure) -{ - with_lock (Mutex) { - const auto foundCluster = ClusterConfigs->find(clusterName); - Y_VERIFY(foundCluster != ClusterConfigs->end()); - auto& cluster = foundCluster->second; - cluster.SetEndpoint(endpoint); - cluster.SetDatabase(database); - cluster.SetUseSsl(secure); - } -} - +void TPqNativeGateway::UpdateClusterConfigs( + const TString& clusterName, + const TString& endpoint, + const TString& database, + bool secure) +{ + with_lock (Mutex) { + const auto foundCluster = ClusterConfigs->find(clusterName); + Y_VERIFY(foundCluster != ClusterConfigs->end()); + auto& cluster = foundCluster->second; + cluster.SetEndpoint(endpoint); + cluster.SetDatabase(database); + cluster.SetUseSsl(secure); + } +} + NThreading::TFuture<void> TPqNativeGateway::OpenSession(const TString& sessionId, const TString& username) { with_lock (Mutex) { auto [sessionIt, isNewSession] = Sessions.emplace(sessionId, diff --git a/ydb/library/yql/providers/pq/provider/ya.make b/ydb/library/yql/providers/pq/provider/ya.make index ad331ccf71..ead04a48fc 100644 --- a/ydb/library/yql/providers/pq/provider/ya.make +++ b/ydb/library/yql/providers/pq/provider/ya.make @@ -9,7 +9,7 @@ OWNER( SRCS( yql_pq_datasink.cpp yql_pq_datasink_execution.cpp - yql_pq_datasink_io_discovery.cpp + yql_pq_datasink_io_discovery.cpp yql_pq_datasink_type_ann.cpp yql_pq_datasource.cpp yql_pq_datasource_type_ann.cpp diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_datasink.cpp b/ydb/library/yql/providers/pq/provider/yql_pq_datasink.cpp index 24e01d3eae..380df33d52 100644 --- a/ydb/library/yql/providers/pq/provider/yql_pq_datasink.cpp +++ b/ydb/library/yql/providers/pq/provider/yql_pq_datasink.cpp @@ -33,7 +33,7 @@ public: TPqDataSinkProvider(TPqState::TPtr state, IPqGateway::TPtr gateway) : State_(state) , Gateway_(gateway) - , IODiscoveryTransformer_(CreatePqDataSinkIODiscoveryTransformer(State_)) + , IODiscoveryTransformer_(CreatePqDataSinkIODiscoveryTransformer(State_)) , TypeAnnotationTransformer_(CreatePqDataSinkTypeAnnotationTransformer(State_)) , ExecutionTransformer_(CreatePqDataSinkExecTransformer(State_)) , LogicalOptProposalTransformer_(CreatePqLogicalOptProposalTransformer(State_)) @@ -53,10 +53,10 @@ public: return TypeAnnotationTransformer_->CanParse(node); } - IGraphTransformer& GetIODiscoveryTransformer() override { - return *IODiscoveryTransformer_; - } - + IGraphTransformer& GetIODiscoveryTransformer() override { + return *IODiscoveryTransformer_; + } + IGraphTransformer& GetTypeAnnotationTransformer(bool instantOnly) override { Y_UNUSED(instantOnly); return *TypeAnnotationTransformer_; @@ -174,7 +174,7 @@ public: private: TPqState::TPtr State_; IPqGateway::TPtr Gateway_; - THolder<IGraphTransformer> IODiscoveryTransformer_; + THolder<IGraphTransformer> IODiscoveryTransformer_; THolder<TVisitorTransformerBase> TypeAnnotationTransformer_; THolder<TExecTransformerBase> ExecutionTransformer_; THolder<IGraphTransformer> LogicalOptProposalTransformer_; diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_datasink_io_discovery.cpp b/ydb/library/yql/providers/pq/provider/yql_pq_datasink_io_discovery.cpp index 21abaccb95..847afd3b3d 100644 --- a/ydb/library/yql/providers/pq/provider/yql_pq_datasink_io_discovery.cpp +++ b/ydb/library/yql/providers/pq/provider/yql_pq_datasink_io_discovery.cpp @@ -1,70 +1,70 @@ -#include "yql_pq_provider_impl.h" -#include "yql_pq_helpers.h" - +#include "yql_pq_provider_impl.h" +#include "yql_pq_helpers.h" + #include <ydb/library/yql/core/yql_expr_optimize.h> #include <ydb/library/yql/utils/log/log.h> #include <ydb/library/yql/providers/pq/expr_nodes/yql_pq_expr_nodes.h> - -namespace NYql { - -namespace { - -using namespace NNodes; - -class TPqDataSinkIODiscoveryTransformer : public TGraphTransformerBase { - -using TDbId2Endpoint = THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TEvEndpointResponse::TEndpoint>; - -public: - explicit TPqDataSinkIODiscoveryTransformer(TPqState::TPtr state) - : State_(state) - { - } - - TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final { - Y_UNUSED(ctx); - output = input; - if (ctx.Step.IsDone(TExprStep::DiscoveryIO)) - return TStatus::Ok; - - if (!State_->DbResolver) - return TStatus::Ok; - - THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TDatabaseAuth> ids; - FindYdsDbIdsForResolving(State_, input, ids); - - if (ids.empty()) - return TStatus::Ok; - - AsyncFuture_ = State_->DbResolver->ResolveIds({ids, State_->DbResolver->GetTraceId()}).Apply([resolvedIds_ = ResolvedIds_](const auto& future) { - *resolvedIds_ = future.GetValue(); - }); - return TStatus::Async; - } - - NThreading::TFuture<void> DoGetAsyncFuture(const TExprNode&) final { - return AsyncFuture_; - } - - TStatus DoApplyAsyncChanges(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext&) final { - output = input; - AsyncFuture_.GetValue(); - FullResolvedIds_.insert(ResolvedIds_->begin(), ResolvedIds_->end()); - ResolvedIds_->clear(); - FillSettingsWithResolvedYdsIds(State_, FullResolvedIds_); - return TStatus::Ok; - } - -private: - const TPqState::TPtr State_; - NThreading::TFuture<void> AsyncFuture_; - TDbId2Endpoint FullResolvedIds_; - std::shared_ptr<TDbId2Endpoint> ResolvedIds_ = std::make_shared<TDbId2Endpoint>(); -}; -} - -THolder<IGraphTransformer> CreatePqDataSinkIODiscoveryTransformer(TPqState::TPtr state) { - return THolder(new TPqDataSinkIODiscoveryTransformer(state)); -} - -} + +namespace NYql { + +namespace { + +using namespace NNodes; + +class TPqDataSinkIODiscoveryTransformer : public TGraphTransformerBase { + +using TDbId2Endpoint = THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TEvEndpointResponse::TEndpoint>; + +public: + explicit TPqDataSinkIODiscoveryTransformer(TPqState::TPtr state) + : State_(state) + { + } + + TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final { + Y_UNUSED(ctx); + output = input; + if (ctx.Step.IsDone(TExprStep::DiscoveryIO)) + return TStatus::Ok; + + if (!State_->DbResolver) + return TStatus::Ok; + + THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TDatabaseAuth> ids; + FindYdsDbIdsForResolving(State_, input, ids); + + if (ids.empty()) + return TStatus::Ok; + + AsyncFuture_ = State_->DbResolver->ResolveIds({ids, State_->DbResolver->GetTraceId()}).Apply([resolvedIds_ = ResolvedIds_](const auto& future) { + *resolvedIds_ = future.GetValue(); + }); + return TStatus::Async; + } + + NThreading::TFuture<void> DoGetAsyncFuture(const TExprNode&) final { + return AsyncFuture_; + } + + TStatus DoApplyAsyncChanges(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext&) final { + output = input; + AsyncFuture_.GetValue(); + FullResolvedIds_.insert(ResolvedIds_->begin(), ResolvedIds_->end()); + ResolvedIds_->clear(); + FillSettingsWithResolvedYdsIds(State_, FullResolvedIds_); + return TStatus::Ok; + } + +private: + const TPqState::TPtr State_; + NThreading::TFuture<void> AsyncFuture_; + TDbId2Endpoint FullResolvedIds_; + std::shared_ptr<TDbId2Endpoint> ResolvedIds_ = std::make_shared<TDbId2Endpoint>(); +}; +} + +THolder<IGraphTransformer> CreatePqDataSinkIODiscoveryTransformer(TPqState::TPtr state) { + return THolder(new TPqDataSinkIODiscoveryTransformer(state)); +} + +} diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_gateway.h b/ydb/library/yql/providers/pq/provider/yql_pq_gateway.h index 4b27e0aa6c..118b37c012 100644 --- a/ydb/library/yql/providers/pq/provider/yql_pq_gateway.h +++ b/ydb/library/yql/providers/pq/provider/yql_pq_gateway.h @@ -17,12 +17,12 @@ struct IPqGateway : public TThrRefBase { // CM API. virtual ::NPq::NConfigurationManager::TAsyncDescribePathResult DescribePath(const TString& sessionId, const TString& cluster, const TString& database, const TString& path, const TString& token) = 0; - - virtual void UpdateClusterConfigs( - const TString& clusterName, - const TString& endpoint, - const TString& database, - bool secure) = 0; + + virtual void UpdateClusterConfigs( + const TString& clusterName, + const TString& endpoint, + const TString& database, + bool secure) = 0; }; } // namespace NYql diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_helpers.cpp b/ydb/library/yql/providers/pq/provider/yql_pq_helpers.cpp index e3a780aa79..47a2134147 100644 --- a/ydb/library/yql/providers/pq/provider/yql_pq_helpers.cpp +++ b/ydb/library/yql/providers/pq/provider/yql_pq_helpers.cpp @@ -1,6 +1,6 @@ #include "yql_pq_helpers.h" -#include "yql_pq_provider_impl.h" +#include "yql_pq_provider_impl.h" #include <ydb/library/yql/core/yql_expr_optimize.h> #include <ydb/library/yql/utils/log/log.h> #include <ydb/library/yql/providers/pq/common/yql_names.h> @@ -26,80 +26,80 @@ TCoNameValueTupleList BuildTopicPropsList(const TPqState::TTopicMeta& meta, TPos .Done(); } -void FindYdsDbIdsForResolving( - const TPqState::TPtr& state, - TExprNode::TPtr input, - THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TDatabaseAuth>& ids) -{ - if (auto pqNodes = FindNodes(input, [&](const TExprNode::TPtr& node) { - if (auto maybePqRead = TMaybeNode<TPqRead>(node)) { - TPqRead read = maybePqRead.Cast(); - if (read.DataSource().Category().Value() == PqProviderName) { - return true; - } - } else if (auto maybePqWrite = TMaybeNode<TPqWrite>(node)) { - TPqWrite write = maybePqWrite.Cast(); - if (write.DataSink().Category().Value() == PqProviderName) { - return true; - } - } - return false; - }); !pqNodes.empty()) { - TString cluster; - for (auto& node : pqNodes) { - if (auto maybePqRead = TMaybeNode<TPqRead>(node)) { - TPqRead read = maybePqRead.Cast(); - cluster = read.DataSource().Cluster().StringValue(); - } else if (auto maybePqWrite = TMaybeNode<TPqWrite>(node)) { - TPqWrite write = maybePqWrite.Cast(); - cluster = write.DataSink().Cluster().StringValue(); - } else { - Y_FAIL("Unrecognized pq node"); - } - YQL_CLOG(INFO, ProviderPq) << "Found cluster: " << cluster; - const auto& clusterCfgSettings = state->Configuration->ClustersConfigurationSettings; - const auto foundSetting = clusterCfgSettings.find(cluster); - if (foundSetting == clusterCfgSettings.end() - || foundSetting->second.ClusterType != NYql::TPqClusterConfig::CT_DATA_STREAMS - || foundSetting->second.Endpoint) - continue; - YQL_CLOG(INFO, ProviderPq) << "Found dbId: " << foundSetting->second.DatabaseId; - if (!foundSetting->second.DatabaseId) - continue; - YQL_CLOG(INFO, ProviderPq) << "Resolve YDS id: " << foundSetting->second.DatabaseId; - const auto idKey = std::make_pair(foundSetting->second.DatabaseId, NYq::DatabaseType::DataStreams); - const auto foundDbId = state->DatabaseIds.find(idKey); - if (foundDbId != state->DatabaseIds.end()) { - ids[idKey] = foundDbId->second; - } - } - } - YQL_CLOG(INFO, ProviderPq) << "Ids to resolve: " << ids.size(); -} - -void FillSettingsWithResolvedYdsIds( - const TPqState::TPtr& state, - const THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TEvEndpointResponse::TEndpoint>& fullResolvedIds) -{ - YQL_CLOG(INFO, ProviderPq) << "FullResolvedIds size: " << fullResolvedIds.size(); - auto& clusters = state->Configuration->ClustersConfigurationSettings; - const auto& id2Clusters = state->Configuration->DbId2Clusters; - for (const auto& [dbIdWithType, info] : fullResolvedIds) { - const auto& dbId = dbIdWithType.first; - YQL_CLOG(INFO, ProviderPq) << "DbId = " << dbId; - const auto iter = id2Clusters.find(dbId); - if (iter == id2Clusters.end()) { - continue; - } - for (const auto& clusterName : iter->second) { - auto& setting = clusters[clusterName]; - setting.Endpoint = info.Endpoint; - setting.Database = info.Database; - setting.UseSsl = info.Secure; - state->Gateway->UpdateClusterConfigs(clusterName, info.Endpoint, info.Database, info.Secure); - } - } -} - - +void FindYdsDbIdsForResolving( + const TPqState::TPtr& state, + TExprNode::TPtr input, + THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TDatabaseAuth>& ids) +{ + if (auto pqNodes = FindNodes(input, [&](const TExprNode::TPtr& node) { + if (auto maybePqRead = TMaybeNode<TPqRead>(node)) { + TPqRead read = maybePqRead.Cast(); + if (read.DataSource().Category().Value() == PqProviderName) { + return true; + } + } else if (auto maybePqWrite = TMaybeNode<TPqWrite>(node)) { + TPqWrite write = maybePqWrite.Cast(); + if (write.DataSink().Category().Value() == PqProviderName) { + return true; + } + } + return false; + }); !pqNodes.empty()) { + TString cluster; + for (auto& node : pqNodes) { + if (auto maybePqRead = TMaybeNode<TPqRead>(node)) { + TPqRead read = maybePqRead.Cast(); + cluster = read.DataSource().Cluster().StringValue(); + } else if (auto maybePqWrite = TMaybeNode<TPqWrite>(node)) { + TPqWrite write = maybePqWrite.Cast(); + cluster = write.DataSink().Cluster().StringValue(); + } else { + Y_FAIL("Unrecognized pq node"); + } + YQL_CLOG(INFO, ProviderPq) << "Found cluster: " << cluster; + const auto& clusterCfgSettings = state->Configuration->ClustersConfigurationSettings; + const auto foundSetting = clusterCfgSettings.find(cluster); + if (foundSetting == clusterCfgSettings.end() + || foundSetting->second.ClusterType != NYql::TPqClusterConfig::CT_DATA_STREAMS + || foundSetting->second.Endpoint) + continue; + YQL_CLOG(INFO, ProviderPq) << "Found dbId: " << foundSetting->second.DatabaseId; + if (!foundSetting->second.DatabaseId) + continue; + YQL_CLOG(INFO, ProviderPq) << "Resolve YDS id: " << foundSetting->second.DatabaseId; + const auto idKey = std::make_pair(foundSetting->second.DatabaseId, NYq::DatabaseType::DataStreams); + const auto foundDbId = state->DatabaseIds.find(idKey); + if (foundDbId != state->DatabaseIds.end()) { + ids[idKey] = foundDbId->second; + } + } + } + YQL_CLOG(INFO, ProviderPq) << "Ids to resolve: " << ids.size(); +} + +void FillSettingsWithResolvedYdsIds( + const TPqState::TPtr& state, + const THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TEvEndpointResponse::TEndpoint>& fullResolvedIds) +{ + YQL_CLOG(INFO, ProviderPq) << "FullResolvedIds size: " << fullResolvedIds.size(); + auto& clusters = state->Configuration->ClustersConfigurationSettings; + const auto& id2Clusters = state->Configuration->DbId2Clusters; + for (const auto& [dbIdWithType, info] : fullResolvedIds) { + const auto& dbId = dbIdWithType.first; + YQL_CLOG(INFO, ProviderPq) << "DbId = " << dbId; + const auto iter = id2Clusters.find(dbId); + if (iter == id2Clusters.end()) { + continue; + } + for (const auto& clusterName : iter->second) { + auto& setting = clusters[clusterName]; + setting.Endpoint = info.Endpoint; + setting.Database = info.Database; + setting.UseSsl = info.Secure; + state->Gateway->UpdateClusterConfigs(clusterName, info.Endpoint, info.Database, info.Secure); + } + } +} + + } // namespace NYql diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_helpers.h b/ydb/library/yql/providers/pq/provider/yql_pq_helpers.h index 181a25cce8..628ab2f65d 100644 --- a/ydb/library/yql/providers/pq/provider/yql_pq_helpers.h +++ b/ydb/library/yql/providers/pq/provider/yql_pq_helpers.h @@ -1,6 +1,6 @@ #pragma once -#include "yql_pq_provider_impl.h" +#include "yql_pq_provider_impl.h" #include <ydb/library/yql/ast/yql_expr.h> #include <ydb/library/yql/ast/yql_pos_handle.h> #include <ydb/library/yql/core/expr_nodes/yql_expr_nodes.h> @@ -12,13 +12,13 @@ void Add(TVector<NNodes::TCoNameValueTuple>& settings, TStringBuf name, TStringB NNodes::TCoNameValueTupleList BuildTopicPropsList(const TPqState::TTopicMeta& meta, TPositionHandle pos, TExprContext& ctx); -void FindYdsDbIdsForResolving( - const TPqState::TPtr& state, - TExprNode::TPtr input, - THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TDatabaseAuth>& ids); - -void FillSettingsWithResolvedYdsIds( - const TPqState::TPtr& state, - const THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TEvEndpointResponse::TEndpoint>& fullResolvedIds); - +void FindYdsDbIdsForResolving( + const TPqState::TPtr& state, + TExprNode::TPtr input, + THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TDatabaseAuth>& ids); + +void FillSettingsWithResolvedYdsIds( + const TPqState::TPtr& state, + const THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TEvEndpointResponse::TEndpoint>& fullResolvedIds); + } // namespace NYql diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_io_discovery.cpp b/ydb/library/yql/providers/pq/provider/yql_pq_io_discovery.cpp index d8cbcc896b..f2e347cf73 100644 --- a/ydb/library/yql/providers/pq/provider/yql_pq_io_discovery.cpp +++ b/ydb/library/yql/providers/pq/provider/yql_pq_io_discovery.cpp @@ -1,5 +1,5 @@ #include "yql_pq_provider_impl.h" -#include "yql_pq_helpers.h" +#include "yql_pq_helpers.h" #include <ydb/library/yql/core/yql_expr_optimize.h> #include <ydb/library/yql/utils/log/log.h> @@ -12,9 +12,9 @@ namespace { using namespace NNodes; class TPqIODiscoveryTransformer : public TGraphTransformerBase { - -using TDbId2Endpoint = THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TEvEndpointResponse::TEndpoint>; - + +using TDbId2Endpoint = THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TEvEndpointResponse::TEndpoint>; + public: explicit TPqIODiscoveryTransformer(TPqState::TPtr state) : State_(state) @@ -24,42 +24,42 @@ public: TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final { Y_UNUSED(ctx); output = input; - if (ctx.Step.IsDone(TExprStep::DiscoveryIO)) - return TStatus::Ok; - - if (!State_->DbResolver) - return TStatus::Ok; - - THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TDatabaseAuth> ids; - FindYdsDbIdsForResolving(State_, input, ids); - - if (ids.empty()) - return TStatus::Ok; - - AsyncFuture_ = State_->DbResolver->ResolveIds({ids, State_->DbResolver->GetTraceId()}).Apply([resolvedIds_ = ResolvedIds_](const auto& future) { - *resolvedIds_ = future.GetValue(); - }); - return TStatus::Async; + if (ctx.Step.IsDone(TExprStep::DiscoveryIO)) + return TStatus::Ok; + + if (!State_->DbResolver) + return TStatus::Ok; + + THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TDatabaseAuth> ids; + FindYdsDbIdsForResolving(State_, input, ids); + + if (ids.empty()) + return TStatus::Ok; + + AsyncFuture_ = State_->DbResolver->ResolveIds({ids, State_->DbResolver->GetTraceId()}).Apply([resolvedIds_ = ResolvedIds_](const auto& future) { + *resolvedIds_ = future.GetValue(); + }); + return TStatus::Async; } NThreading::TFuture<void> DoGetAsyncFuture(const TExprNode&) final { - return AsyncFuture_; + return AsyncFuture_; } TStatus DoApplyAsyncChanges(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext&) final { output = input; - AsyncFuture_.GetValue(); - FullResolvedIds_.insert(ResolvedIds_->begin(), ResolvedIds_->end()); - ResolvedIds_->clear(); - FillSettingsWithResolvedYdsIds(State_, FullResolvedIds_); + AsyncFuture_.GetValue(); + FullResolvedIds_.insert(ResolvedIds_->begin(), ResolvedIds_->end()); + ResolvedIds_->clear(); + FillSettingsWithResolvedYdsIds(State_, FullResolvedIds_); return TStatus::Ok; } private: const TPqState::TPtr State_; - NThreading::TFuture<void> AsyncFuture_; - TDbId2Endpoint FullResolvedIds_; - std::shared_ptr<TDbId2Endpoint> ResolvedIds_ = std::make_shared<TDbId2Endpoint>(); + NThreading::TFuture<void> AsyncFuture_; + TDbId2Endpoint FullResolvedIds_; + std::shared_ptr<TDbId2Endpoint> ResolvedIds_ = std::make_shared<TDbId2Endpoint>(); }; } diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_provider.cpp b/ydb/library/yql/providers/pq/provider/yql_pq_provider.cpp index 9dc52ed7b8..b3f700c147 100644 --- a/ydb/library/yql/providers/pq/provider/yql_pq_provider.cpp +++ b/ydb/library/yql/providers/pq/provider/yql_pq_provider.cpp @@ -9,11 +9,11 @@ namespace NYql { -TDataProviderInitializer GetPqDataProviderInitializer( - IPqGateway::TPtr gateway, - bool supportRtmrMode, - std::shared_ptr<NYq::TDatabaseAsyncResolverWithMeta> dbResolverWithMeta) { - return [gateway, supportRtmrMode, dbResolverWithMeta] ( +TDataProviderInitializer GetPqDataProviderInitializer( + IPqGateway::TPtr gateway, + bool supportRtmrMode, + std::shared_ptr<NYq::TDatabaseAsyncResolverWithMeta> dbResolverWithMeta) { + return [gateway, supportRtmrMode, dbResolverWithMeta] ( const TString& userName, const TString& sessionId, const TGatewaysConfig* gatewaysConfig, @@ -33,9 +33,9 @@ TDataProviderInitializer GetPqDataProviderInitializer( state->SupportRtmrMode = supportRtmrMode; state->Types = typeCtx.Get(); state->FunctionRegistry = functionRegistry; - state->DbResolver = dbResolverWithMeta; + state->DbResolver = dbResolverWithMeta; if (gatewaysConfig) { - state->Configuration->Init(gatewaysConfig->GetPq(), typeCtx, dbResolverWithMeta, state->DatabaseIds); + state->Configuration->Init(gatewaysConfig->GetPq(), typeCtx, dbResolverWithMeta, state->DatabaseIds); } state->Gateway = gateway; state->DqIntegration = CreatePqDqIntegration(state); diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_provider.h b/ydb/library/yql/providers/pq/provider/yql_pq_provider.h index 7aadfd904f..a890e32b69 100644 --- a/ydb/library/yql/providers/pq/provider/yql_pq_provider.h +++ b/ydb/library/yql/providers/pq/provider/yql_pq_provider.h @@ -52,14 +52,14 @@ public: const NKikimr::NMiniKQL::IFunctionRegistry* FunctionRegistry = nullptr; IPqGateway::TPtr Gateway; THolder<IDqIntegration> DqIntegration; - THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TDatabaseAuth> DatabaseIds; - std::shared_ptr<NYq::TDatabaseAsyncResolverWithMeta> DbResolver; + THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TDatabaseAuth> DatabaseIds; + std::shared_ptr<NYq::TDatabaseAsyncResolverWithMeta> DbResolver; }; -TDataProviderInitializer GetPqDataProviderInitializer( - IPqGateway::TPtr gateway, - bool supportRtmrMode = false, - std::shared_ptr<NYq::TDatabaseAsyncResolverWithMeta> dbResolverWithMeta = nullptr -); +TDataProviderInitializer GetPqDataProviderInitializer( + IPqGateway::TPtr gateway, + bool supportRtmrMode = false, + std::shared_ptr<NYq::TDatabaseAsyncResolverWithMeta> dbResolverWithMeta = nullptr +); } // namespace NYql diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_provider_impl.h b/ydb/library/yql/providers/pq/provider/yql_pq_provider_impl.h index 50c620f10c..619eb468e2 100644 --- a/ydb/library/yql/providers/pq/provider/yql_pq_provider_impl.h +++ b/ydb/library/yql/providers/pq/provider/yql_pq_provider_impl.h @@ -15,7 +15,7 @@ TIntrusivePtr<IDataProvider> CreatePqDataSink(TPqState::TPtr state, IPqGateway:: THolder<IGraphTransformer> CreatePqLoadTopicMetadataTransformer(TPqState::TPtr state); -THolder<IGraphTransformer> CreatePqDataSinkIODiscoveryTransformer(TPqState::TPtr state); +THolder<IGraphTransformer> CreatePqDataSinkIODiscoveryTransformer(TPqState::TPtr state); THolder<TVisitorTransformerBase> CreatePqDataSourceTypeAnnotationTransformer(TPqState::TPtr state); THolder<TVisitorTransformerBase> CreatePqDataSinkTypeAnnotationTransformer(TPqState::TPtr state); diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_settings.cpp b/ydb/library/yql/providers/pq/provider/yql_pq_settings.cpp index e6fe651538..278db980f4 100644 --- a/ydb/library/yql/providers/pq/provider/yql_pq_settings.cpp +++ b/ydb/library/yql/providers/pq/provider/yql_pq_settings.cpp @@ -14,59 +14,59 @@ TPqSettings::TConstPtr TPqConfiguration::Snapshot() const { return std::make_shared<const TPqSettings>(*this); } -void TPqConfiguration::Init( - const TPqGatewayConfig& config, - TIntrusivePtr<TTypeAnnotationContext> typeCtx, - const std::shared_ptr<NYq::TDatabaseAsyncResolverWithMeta> dbResolver, - THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TDatabaseAuth>& databaseIds) -{ - TVector<TString> clusters(Reserve(config.ClusterMappingSize())); - for (auto& cluster: config.GetClusterMapping()) { - clusters.push_back(cluster.GetName()); - } - - SetValidClusters(clusters); - - Dispatch(config.GetDefaultSettings()); - - for (auto& cluster: config.GetClusterMapping()) { - Dispatch(cluster.GetName(), cluster.GetSettings()); - TPqClusterConfigurationSettings& clusterSettings = ClustersConfigurationSettings[cluster.GetName()]; - - clusterSettings.ClusterName = cluster.GetName(); - clusterSettings.ClusterType = cluster.GetClusterType(); - clusterSettings.Endpoint = cluster.GetEndpoint(); - clusterSettings.ConfigManagerEndpoint = cluster.GetConfigManagerEndpoint(); - clusterSettings.Database = cluster.GetDatabase(); - clusterSettings.DatabaseId = cluster.GetDatabaseId(); - clusterSettings.TvmId = cluster.GetTvmId(); - clusterSettings.UseSsl = cluster.GetUseSsl(); - clusterSettings.AddBearerToToken = cluster.GetAddBearerToToken(); - - if (dbResolver) { - YQL_CLOG(DEBUG, ProviderPq) << "Settings: clusterName = " << cluster.GetName() - << ", clusterDbId = " << cluster.GetDatabaseId() << ", cluster.GetEndpoint(): " << cluster.GetEndpoint() << ", HasEndpoint = " << (cluster.HasEndpoint() ? "TRUE" : "FALSE") ; - dbResolver->TryAddDbIdToResolve(cluster.HasEndpoint(), cluster.GetName(), cluster.GetDatabaseId(), NYq::DatabaseType::DataStreams, databaseIds); - if (cluster.GetDatabaseId()) { - DbId2Clusters[cluster.GetDatabaseId()].emplace_back(cluster.GetName()); - YQL_CLOG(DEBUG, ProviderPq) << "Add dbId: " << cluster.GetDatabaseId() << " to DbId2Clusters"; - } - } - - const TString authToken = typeCtx->FindCredentialContent("cluster:default_" + clusterSettings.ClusterName, "default_pq", cluster.GetToken()); - clusterSettings.AuthToken = authToken; - Tokens[clusterSettings.ClusterName] = ComposeStructuredTokenJsonForServiceAccount(cluster.GetServiceAccountId(), cluster.GetServiceAccountIdSignature(), authToken); - } - FreezeDefaults(); -} - -TString TPqConfiguration::GetDatabaseForTopic(const TString& cluster) const { - if (TMaybe<TString> explicitDb = Database.Get()) { - return *explicitDb; - } - const auto clusterSetting = ClustersConfigurationSettings.FindPtr(cluster); - YQL_ENSURE(clusterSetting, "Unknown cluster " << cluster); - return clusterSetting->Database; -} - +void TPqConfiguration::Init( + const TPqGatewayConfig& config, + TIntrusivePtr<TTypeAnnotationContext> typeCtx, + const std::shared_ptr<NYq::TDatabaseAsyncResolverWithMeta> dbResolver, + THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TDatabaseAuth>& databaseIds) +{ + TVector<TString> clusters(Reserve(config.ClusterMappingSize())); + for (auto& cluster: config.GetClusterMapping()) { + clusters.push_back(cluster.GetName()); + } + + SetValidClusters(clusters); + + Dispatch(config.GetDefaultSettings()); + + for (auto& cluster: config.GetClusterMapping()) { + Dispatch(cluster.GetName(), cluster.GetSettings()); + TPqClusterConfigurationSettings& clusterSettings = ClustersConfigurationSettings[cluster.GetName()]; + + clusterSettings.ClusterName = cluster.GetName(); + clusterSettings.ClusterType = cluster.GetClusterType(); + clusterSettings.Endpoint = cluster.GetEndpoint(); + clusterSettings.ConfigManagerEndpoint = cluster.GetConfigManagerEndpoint(); + clusterSettings.Database = cluster.GetDatabase(); + clusterSettings.DatabaseId = cluster.GetDatabaseId(); + clusterSettings.TvmId = cluster.GetTvmId(); + clusterSettings.UseSsl = cluster.GetUseSsl(); + clusterSettings.AddBearerToToken = cluster.GetAddBearerToToken(); + + if (dbResolver) { + YQL_CLOG(DEBUG, ProviderPq) << "Settings: clusterName = " << cluster.GetName() + << ", clusterDbId = " << cluster.GetDatabaseId() << ", cluster.GetEndpoint(): " << cluster.GetEndpoint() << ", HasEndpoint = " << (cluster.HasEndpoint() ? "TRUE" : "FALSE") ; + dbResolver->TryAddDbIdToResolve(cluster.HasEndpoint(), cluster.GetName(), cluster.GetDatabaseId(), NYq::DatabaseType::DataStreams, databaseIds); + if (cluster.GetDatabaseId()) { + DbId2Clusters[cluster.GetDatabaseId()].emplace_back(cluster.GetName()); + YQL_CLOG(DEBUG, ProviderPq) << "Add dbId: " << cluster.GetDatabaseId() << " to DbId2Clusters"; + } + } + + const TString authToken = typeCtx->FindCredentialContent("cluster:default_" + clusterSettings.ClusterName, "default_pq", cluster.GetToken()); + clusterSettings.AuthToken = authToken; + Tokens[clusterSettings.ClusterName] = ComposeStructuredTokenJsonForServiceAccount(cluster.GetServiceAccountId(), cluster.GetServiceAccountIdSignature(), authToken); + } + FreezeDefaults(); +} + +TString TPqConfiguration::GetDatabaseForTopic(const TString& cluster) const { + if (TMaybe<TString> explicitDb = Database.Get()) { + return *explicitDb; + } + const auto clusterSetting = ClustersConfigurationSettings.FindPtr(cluster); + YQL_ENSURE(clusterSetting, "Unknown cluster " << cluster); + return clusterSetting->Database; +} + } // NYql diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_settings.h b/ydb/library/yql/providers/pq/provider/yql_pq_settings.h index 4bfd0abfaa..5ed40d07d9 100644 --- a/ydb/library/yql/providers/pq/provider/yql_pq_settings.h +++ b/ydb/library/yql/providers/pq/provider/yql_pq_settings.h @@ -9,7 +9,7 @@ #include <ydb/core/yq/libs/events/events.h> #include <ydb/core/yq/libs/db_resolver/db_async_resolver_with_meta.h> #include <ydb/core/yq/libs/common/database_token_builder.h> - + namespace NYql { struct TPqSettings { @@ -39,18 +39,18 @@ struct TPqConfiguration : public TPqSettings, public NCommon::TSettingDispatcher TPqConfiguration(); TPqConfiguration(const TPqConfiguration&) = delete; - void Init( - const TPqGatewayConfig& config, - TIntrusivePtr<TTypeAnnotationContext> typeCtx, - const std::shared_ptr<NYq::TDatabaseAsyncResolverWithMeta> dbResolver, - THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TDatabaseAuth>& databaseIds); + void Init( + const TPqGatewayConfig& config, + TIntrusivePtr<TTypeAnnotationContext> typeCtx, + const std::shared_ptr<NYq::TDatabaseAsyncResolverWithMeta> dbResolver, + THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TDatabaseAuth>& databaseIds); - TString GetDatabaseForTopic(const TString& cluster) const; + TString GetDatabaseForTopic(const TString& cluster) const; TPqSettings::TConstPtr Snapshot() const; THashMap<TString, TPqClusterConfigurationSettings> ClustersConfigurationSettings; THashMap<TString, TString> Tokens; - THashMap<TString, TVector<TString>> DbId2Clusters; // DatabaseId -> ClusterNames + THashMap<TString, TVector<TString>> DbId2Clusters; // DatabaseId -> ClusterNames }; } // NYql diff --git a/ydb/library/yql/providers/s3/actors/yql_s3_read_actor.cpp b/ydb/library/yql/providers/s3/actors/yql_s3_read_actor.cpp index d1ec6141d6..eb5410a24e 100644 --- a/ydb/library/yql/providers/s3/actors/yql_s3_read_actor.cpp +++ b/ydb/library/yql/providers/s3/actors/yql_s3_read_actor.cpp @@ -25,7 +25,7 @@ struct TEvPrivate { EvReadResult = EvBegin, EvReadError, - EvRetry, + EvRetry, EvEnd }; @@ -39,51 +39,51 @@ struct TEvPrivate { }; struct TEvReadError : public TEventLocal<TEvReadError, EvReadError> { - TEvReadError(TIssues&& error, size_t pathInd): Error(std::move(error)), PathIndex(pathInd) {} + TEvReadError(TIssues&& error, size_t pathInd): Error(std::move(error)), PathIndex(pathInd) {} TIssues Error; - size_t PathIndex = 0; + size_t PathIndex = 0; + }; + + struct TEvRetryEvent : public NActors::TEventLocal<TEvRetryEvent, EvRetry> { + explicit TEvRetryEvent(size_t pathIndex) : PathIndex(pathIndex) {} + size_t PathIndex = 0; }; - - struct TEvRetryEvent : public NActors::TEventLocal<TEvRetryEvent, EvRetry> { - explicit TEvRetryEvent(size_t pathIndex) : PathIndex(pathIndex) {} - size_t PathIndex = 0; - }; }; } // namespace class TS3ReadActor : public TActorBootstrapped<TS3ReadActor>, public IDqSourceActor { -private: - class TRetryParams { - public: - TRetryParams(const std::shared_ptr<NS3::TRetryConfig>& retryConfig) { - if (retryConfig) { - DelayMs = retryConfig->GetInitialDelayMs() ? TDuration::MilliSeconds(retryConfig->GetInitialDelayMs()) : DelayMs; - Epsilon = retryConfig->GetEpsilon() ? retryConfig->GetEpsilon() : Epsilon; - Y_VERIFY(0.0 < Epsilon && Epsilon < 1.0); - } - } - - TDuration GetNextDelay(ui32 maxRetries) { - if (Retries > maxRetries) return TDuration::Zero(); - return DelayMs = GenerateNextDelay(); - }; - - void IncRetries() { - ++Retries; - } - - private: - TDuration GenerateNextDelay() { - double low = 1 - Epsilon; - auto jitter = low + std::rand() / (RAND_MAX / (2 * Epsilon)); - return DelayMs * jitter; - } - - ui32 Retries = 0; - TDuration DelayMs = TDuration::MilliSeconds(100); - double Epsilon = 0.1; - }; +private: + class TRetryParams { + public: + TRetryParams(const std::shared_ptr<NS3::TRetryConfig>& retryConfig) { + if (retryConfig) { + DelayMs = retryConfig->GetInitialDelayMs() ? TDuration::MilliSeconds(retryConfig->GetInitialDelayMs()) : DelayMs; + Epsilon = retryConfig->GetEpsilon() ? retryConfig->GetEpsilon() : Epsilon; + Y_VERIFY(0.0 < Epsilon && Epsilon < 1.0); + } + } + + TDuration GetNextDelay(ui32 maxRetries) { + if (Retries > maxRetries) return TDuration::Zero(); + return DelayMs = GenerateNextDelay(); + }; + + void IncRetries() { + ++Retries; + } + + private: + TDuration GenerateNextDelay() { + double low = 1 - Epsilon; + auto jitter = low + std::rand() / (RAND_MAX / (2 * Epsilon)); + return DelayMs * jitter; + } + + ui32 Retries = 0; + TDuration DelayMs = TDuration::MilliSeconds(100); + double Epsilon = 0.1; + }; public: using TPath = std::tuple<TString, size_t>; using TPathList = std::vector<TPath>; @@ -93,8 +93,8 @@ using TPathList = std::vector<TPath>; const TString& url, const TString& token, TPathList&& paths, - ICallbacks* callbacks, - const std::shared_ptr<NS3::TRetryConfig>& retryConfig + ICallbacks* callbacks, + const std::shared_ptr<NS3::TRetryConfig>& retryConfig ) : Gateway(std::move(gateway)) , InputIndex(inputIndex) , Callbacks(callbacks) @@ -102,26 +102,26 @@ using TPathList = std::vector<TPath>; , Url(url) , Headers(MakeHeader(token)) , Paths(std::move(paths)) - , RetryConfig(retryConfig) - { - if (RetryConfig) { - MaxRetriesPerPath = RetryConfig->GetMaxRetriesPerPath() ? RetryConfig->GetMaxRetriesPerPath() : MaxRetriesPerPath; - } - } + , RetryConfig(retryConfig) + { + if (RetryConfig) { + MaxRetriesPerPath = RetryConfig->GetMaxRetriesPerPath() ? RetryConfig->GetMaxRetriesPerPath() : MaxRetriesPerPath; + } + } void Bootstrap() { Become(&TS3ReadActor::StateFunc); - RetriesPerPath.resize(Paths.size(), TRetryParams(RetryConfig)); - for (size_t pathInd = 0; pathInd < Paths.size(); ++pathInd) { - const TPath& path = Paths[pathInd]; - Gateway->Download(Url + std::get<TString>(path), - Headers, std::get<size_t>(path), - std::bind(&TS3ReadActor::OnDownloadFinished, ActorSystem, SelfId(), std::placeholders::_1, pathInd)); - }; + RetriesPerPath.resize(Paths.size(), TRetryParams(RetryConfig)); + for (size_t pathInd = 0; pathInd < Paths.size(); ++pathInd) { + const TPath& path = Paths[pathInd]; + Gateway->Download(Url + std::get<TString>(path), + Headers, std::get<size_t>(path), + std::bind(&TS3ReadActor::OnDownloadFinished, ActorSystem, SelfId(), std::placeholders::_1, pathInd)); + }; } - - static constexpr char ActorName[] = "S3_READ_ACTOR"; - + + static constexpr char ActorName[] = "S3_READ_ACTOR"; + private: void SaveState(const NDqProto::TCheckpoint&, NDqProto::TSourceState&) final {} void LoadState(const NDqProto::TSourceState&) final {} @@ -131,16 +131,16 @@ private: STRICT_STFUNC(StateFunc, hFunc(TEvPrivate::TEvReadResult, Handle); hFunc(TEvPrivate::TEvReadError, Handle); - hFunc(TEvPrivate::TEvRetryEvent, HandleRetry); + hFunc(TEvPrivate::TEvRetryEvent, HandleRetry); ) - static void OnDownloadFinished(TActorSystem* actorSystem, TActorId selfId, IHTTPGateway::TResult&& result, size_t pathInd) { + static void OnDownloadFinished(TActorSystem* actorSystem, TActorId selfId, IHTTPGateway::TResult&& result, size_t pathInd) { switch (result.index()) { case 0U: actorSystem->Send(new IEventHandle(selfId, TActorId(), new TEvPrivate::TEvReadResult(std::get<IHTTPGateway::TContent>(std::move(result))))); return; case 1U: - actorSystem->Send(new IEventHandle(selfId, TActorId(), new TEvPrivate::TEvReadError(std::get<TIssues>(std::move(result)), pathInd))); + actorSystem->Send(new IEventHandle(selfId, TActorId(), new TEvPrivate::TEvReadError(std::get<TIssues>(std::move(result)), pathInd))); return; default: break; @@ -174,21 +174,21 @@ private: Callbacks->OnNewSourceDataArrived(InputIndex); } - void HandleRetry(TEvPrivate::TEvRetryEvent::TPtr& ev) { - const auto pathInd = ev->Get()->PathIndex; - RetriesPerPath[pathInd].IncRetries(); - Gateway->Download(Url + std::get<TString>(Paths[pathInd]), - Headers, std::get<size_t>(Paths[pathInd]), - std::bind(&TS3ReadActor::OnDownloadFinished, ActorSystem, SelfId(), std::placeholders::_1, pathInd)); - } - + void HandleRetry(TEvPrivate::TEvRetryEvent::TPtr& ev) { + const auto pathInd = ev->Get()->PathIndex; + RetriesPerPath[pathInd].IncRetries(); + Gateway->Download(Url + std::get<TString>(Paths[pathInd]), + Headers, std::get<size_t>(Paths[pathInd]), + std::bind(&TS3ReadActor::OnDownloadFinished, ActorSystem, SelfId(), std::placeholders::_1, pathInd)); + } + void Handle(TEvPrivate::TEvReadError::TPtr& result) { - const auto pathInd = result->Get()->PathIndex; - Y_VERIFY(pathInd < RetriesPerPath.size()); - if (auto nextDelayMs = RetriesPerPath[pathInd].GetNextDelay(MaxRetriesPerPath)) { - Schedule(nextDelayMs, new TEvPrivate::TEvRetryEvent(pathInd)); - return; - } + const auto pathInd = result->Get()->PathIndex; + Y_VERIFY(pathInd < RetriesPerPath.size()); + if (auto nextDelayMs = RetriesPerPath[pathInd].GetNextDelay(MaxRetriesPerPath)) { + Schedule(nextDelayMs, new TEvPrivate::TEvRetryEvent(pathInd)); + return; + } ++IsDoneCounter; Callbacks->OnSourceError(InputIndex, result->Get()->Error, true); } @@ -202,7 +202,7 @@ private: return token.empty() ? IHTTPGateway::THeaders() : IHTTPGateway::THeaders{TString("X-YaCloud-SubjectToken:") += token}; } -private: +private: size_t IsDoneCounter = 0U; const IHTTPGateway::TPtr Gateway; @@ -217,10 +217,10 @@ private: const TPathList Paths; std::queue<IHTTPGateway::TContent> Blocks; - - std::vector<TRetryParams> RetriesPerPath; - const std::shared_ptr<NS3::TRetryConfig> RetryConfig; - ui32 MaxRetriesPerPath = 3; + + std::vector<TRetryParams> RetriesPerPath; + const std::shared_ptr<NS3::TRetryConfig> RetryConfig; + ui32 MaxRetriesPerPath = 3; }; std::pair<NYql::NDq::IDqSourceActor*, IActor*> CreateS3ReadActor( @@ -230,8 +230,8 @@ std::pair<NYql::NDq::IDqSourceActor*, IActor*> CreateS3ReadActor( const THashMap<TString, TString>& secureParams, const THashMap<TString, TString>& taskParams, NYql::NDq::IDqSourceActor::ICallbacks* callback, - ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory, - const std::shared_ptr<NS3::TRetryConfig>& retryConfig) + ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory, + const std::shared_ptr<NS3::TRetryConfig>& retryConfig) { std::unordered_map<TString, size_t> map(params.GetPath().size()); for (auto i = 0; i < params.GetPath().size(); ++i) @@ -254,7 +254,7 @@ std::pair<NYql::NDq::IDqSourceActor*, IActor*> CreateS3ReadActor( const auto token = secureParams.Value(params.GetToken(), TString{}); const auto credentialsProviderFactory = CreateCredentialsProviderFactoryForStructuredToken(credentialsFactory, token); const auto authToken = credentialsProviderFactory->CreateProvider()->GetAuthInfo(); - const auto actor = new TS3ReadActor(inputIndex, std::move(gateway), params.GetUrl(), authToken, std::move(paths), callback, retryConfig); + const auto actor = new TS3ReadActor(inputIndex, std::move(gateway), params.GetUrl(), authToken, std::move(paths), callback, retryConfig); return {actor, actor}; } diff --git a/ydb/library/yql/providers/s3/actors/yql_s3_read_actor.h b/ydb/library/yql/providers/s3/actors/yql_s3_read_actor.h index e72f729486..9280e9ba2e 100644 --- a/ydb/library/yql/providers/s3/actors/yql_s3_read_actor.h +++ b/ydb/library/yql/providers/s3/actors/yql_s3_read_actor.h @@ -16,7 +16,7 @@ std::pair<NYql::NDq::IDqSourceActor*, NActors::IActor*> CreateS3ReadActor( const THashMap<TString, TString>& secureParams, const THashMap<TString, TString>& taskParams, IDqSourceActor::ICallbacks* callback, - ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory, - const std::shared_ptr<NYql::NS3::TRetryConfig>& retryConfig = nullptr); + ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory, + const std::shared_ptr<NYql::NS3::TRetryConfig>& retryConfig = nullptr); } // namespace NYql::NDq diff --git a/ydb/library/yql/providers/s3/actors/yql_s3_source_factory.cpp b/ydb/library/yql/providers/s3/actors/yql_s3_source_factory.cpp index 4034574a44..fb78240498 100644 --- a/ydb/library/yql/providers/s3/actors/yql_s3_source_factory.cpp +++ b/ydb/library/yql/providers/s3/actors/yql_s3_source_factory.cpp @@ -5,14 +5,14 @@ namespace NYql::NDq { -void RegisterS3ReadActorFactory( - TDqSourceFactory& factory, - ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory, - IHTTPGateway::TPtr gateway, - const std::shared_ptr<NYql::NS3::TRetryConfig>& retryConfig) { +void RegisterS3ReadActorFactory( + TDqSourceFactory& factory, + ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory, + IHTTPGateway::TPtr gateway, + const std::shared_ptr<NYql::NS3::TRetryConfig>& retryConfig) { factory.Register<NS3::TSource>("S3Source", - [credentialsFactory, gateway, retryConfig](NS3::TSource&& settings, IDqSourceActorFactory::TArguments&& args) { - return CreateS3ReadActor(gateway, std::move(settings), args.InputIndex, args.SecureParams, args.TaskParams, args.Callback, credentialsFactory, retryConfig); + [credentialsFactory, gateway, retryConfig](NS3::TSource&& settings, IDqSourceActorFactory::TArguments&& args) { + return CreateS3ReadActor(gateway, std::move(settings), args.InputIndex, args.SecureParams, args.TaskParams, args.Callback, credentialsFactory, retryConfig); }); } diff --git a/ydb/library/yql/providers/s3/actors/yql_s3_source_factory.h b/ydb/library/yql/providers/s3/actors/yql_s3_source_factory.h index 412377e2de..2df0a312b1 100644 --- a/ydb/library/yql/providers/s3/actors/yql_s3_source_factory.h +++ b/ydb/library/yql/providers/s3/actors/yql_s3_source_factory.h @@ -1,7 +1,7 @@ #pragma once -#include <ydb/library/yql/providers/common/token_accessor/client/factory.h> - +#include <ydb/library/yql/providers/common/token_accessor/client/factory.h> + #include <ydb/library/yql/dq/actors/compute/dq_compute_actor_io_actors_factory.h> #include <ydb/library/yql/dq/actors/compute/dq_compute_actor_sources.h> #include <ydb/library/yql/providers/common/http_gateway/yql_http_gateway.h> @@ -10,10 +10,10 @@ namespace NYql::NDq { -void RegisterS3ReadActorFactory( - TDqSourceFactory& factory, - ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory, - IHTTPGateway::TPtr gateway = IHTTPGateway::Make(), - const std::shared_ptr<NYql::NS3::TRetryConfig>& retryConfig = nullptr); +void RegisterS3ReadActorFactory( + TDqSourceFactory& factory, + ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory, + IHTTPGateway::TPtr gateway = IHTTPGateway::Make(), + const std::shared_ptr<NYql::NS3::TRetryConfig>& retryConfig = nullptr); } diff --git a/ydb/library/yql/providers/s3/proto/retry_config.proto b/ydb/library/yql/providers/s3/proto/retry_config.proto index afec3b0536..3477562bb6 100644 --- a/ydb/library/yql/providers/s3/proto/retry_config.proto +++ b/ydb/library/yql/providers/s3/proto/retry_config.proto @@ -1,10 +1,10 @@ -syntax = "proto3"; -option cc_enable_arenas = true; - -package NYql.NS3; - -message TRetryConfig { - uint64 InitialDelayMs = 1; - uint32 MaxRetriesPerPath = 2; - double Epsilon = 3; -} +syntax = "proto3"; +option cc_enable_arenas = true; + +package NYql.NS3; + +message TRetryConfig { + uint64 InitialDelayMs = 1; + uint32 MaxRetriesPerPath = 2; + double Epsilon = 3; +} diff --git a/ydb/library/yql/providers/s3/proto/ya.make b/ydb/library/yql/providers/s3/proto/ya.make index 993f34c7e9..48215f2ab5 100644 --- a/ydb/library/yql/providers/s3/proto/ya.make +++ b/ydb/library/yql/providers/s3/proto/ya.make @@ -7,7 +7,7 @@ PROTO_LIBRARY() SRCS( range.proto - retry_config.proto + retry_config.proto source.proto ) diff --git a/ydb/library/yql/providers/s3/provider/yql_s3_io_discovery.cpp b/ydb/library/yql/providers/s3/provider/yql_s3_io_discovery.cpp index 8eafea99b8..3e0384c9d7 100644 --- a/ydb/library/yql/providers/s3/provider/yql_s3_io_discovery.cpp +++ b/ydb/library/yql/providers/s3/provider/yql_s3_io_discovery.cpp @@ -31,23 +31,23 @@ std::array<TExprNode::TPtr, 2U> GetSchema(const TExprNode& settings) { using TItemsMap = std::map<TString, std::size_t>; using TPendingBuckets = std::unordered_map<std::tuple<TString, TString, TString>, std::tuple<TNodeSet, TItemsMap, TIssues>, THash<std::tuple<TString, TString, TString>>>; -void OnDiscovery( - IHTTPGateway::TWeakPtr gateway, - TPosition pos, - IHTTPGateway::TResult&& result, - const TPendingBuckets::key_type& keys, - TPendingBuckets::mapped_type& output, - NThreading::TPromise<void> promise, - std::weak_ptr<TPendingBuckets> pendingBucketsWPtr, - int promiseInd) { - auto pendingBuckets = pendingBucketsWPtr.lock(); // keys and output could be used only when TPendingBuckets is alive - if (!pendingBuckets) { - return; - } - TString logMsg = TStringBuilder() << "promise #" << promiseInd << ": "; +void OnDiscovery( + IHTTPGateway::TWeakPtr gateway, + TPosition pos, + IHTTPGateway::TResult&& result, + const TPendingBuckets::key_type& keys, + TPendingBuckets::mapped_type& output, + NThreading::TPromise<void> promise, + std::weak_ptr<TPendingBuckets> pendingBucketsWPtr, + int promiseInd) { + auto pendingBuckets = pendingBucketsWPtr.lock(); // keys and output could be used only when TPendingBuckets is alive + if (!pendingBuckets) { + return; + } + TString logMsg = TStringBuilder() << "promise #" << promiseInd << ": "; switch (result.index()) { case 0U: try { - logMsg += "Result received"; + logMsg += "Result received"; const NXml::TDocument xml(std::get<IHTTPGateway::TContent>(std::move(result)).Extract(), NXml::TDocument::String); if (const auto& root = xml.Root(); root.Name() == "Error") { const auto& code = root.Node("Code", true).Value<TString>(); @@ -80,30 +80,30 @@ void OnDiscovery( if (const auto& token = std::get<2U>(keys); !token.empty()) headers.emplace_back(token); return g->Download(std::get<0U>(keys) + "?list-type=2&prefix=" + prefix + "&continuation-token=" + next + "&max-keys=" + maxKeys, std::move(headers), 0U, - std::bind(&OnDiscovery, gateway, pos, std::placeholders::_1, std::cref(keys), std::ref(output), std::move(promise), pendingBucketsWPtr, promiseInd)); + std::bind(&OnDiscovery, gateway, pos, std::placeholders::_1, std::cref(keys), std::ref(output), std::move(promise), pendingBucketsWPtr, promiseInd)); } - YQL_CLOG(INFO, ProviderS3) << "Gateway disappeared."; + YQL_CLOG(INFO, ProviderS3) << "Gateway disappeared."; } } break; } catch (const std::exception& ex) { - logMsg += TStringBuilder() << "Exception occurred: " << ex.what(); + logMsg += TStringBuilder() << "Exception occurred: " << ex.what(); std::get<TIssues>(output) = {TIssue(pos, TStringBuilder() << "Error '" << ex.what() << "' on parse discovery response.")}; break; } case 1U: - logMsg += TStringBuilder() << "Issues occurred: " << std::get<TIssues>(result).ToString(); + logMsg += TStringBuilder() << "Issues occurred: " << std::get<TIssues>(result).ToString(); std::get<TIssues>(output) = std::get<TIssues>(std::move(result)); break; default: - logMsg += TStringBuilder() << "Undefined variant index: " << result.index(); + logMsg += TStringBuilder() << "Undefined variant index: " << result.index(); std::get<TIssues>(output) = {TIssue(pos, TStringBuilder() << "Unexpected variant index " << result.index() << " on discovery.")}; break; } - YQL_CLOG(DEBUG, ProviderS3) << "Set promise with log message: " << logMsg; - promise.SetValue(); + YQL_CLOG(DEBUG, ProviderS3) << "Set promise with log message: " << logMsg; + promise.SetValue(); } @@ -187,15 +187,15 @@ public: const auto credentialsProviderFactory = CreateCredentialsProviderFactoryForStructuredToken(State_->CredentialsFactory, token); const auto authToken = credentialsProviderFactory->CreateProvider()->GetAuthInfo(); - std::get<TNodeSet>((*PendingBuckets_)[std::make_tuple(connect.Url, TString(prefix), authToken.empty() ? TString() : TString("X-YaCloud-SubjectToken:") += authToken)]).emplace(read.Raw()); + std::get<TNodeSet>((*PendingBuckets_)[std::make_tuple(connect.Url, TString(prefix), authToken.empty() ? TString() : TString("X-YaCloud-SubjectToken:") += authToken)]).emplace(read.Raw()); } } std::vector<NThreading::TFuture<void>> handles; - handles.reserve(PendingBuckets_->size()); + handles.reserve(PendingBuckets_->size()); - int i = 0; - for (auto& bucket : *PendingBuckets_) { + int i = 0; + for (auto& bucket : *PendingBuckets_) { auto promise = NThreading::NewPromise(); handles.emplace_back(promise.GetFuture()); TString prefix(std::get<1U>(bucket.first)); @@ -203,10 +203,10 @@ public: IHTTPGateway::THeaders headers; if (const auto& token = std::get<2U>(bucket.first); !token.empty()) headers.emplace_back(token); - std::weak_ptr<TPendingBuckets> pendingBucketsWPtr = PendingBuckets_; + std::weak_ptr<TPendingBuckets> pendingBucketsWPtr = PendingBuckets_; Gateway_->Download(std::get<0U>(bucket.first) + "?list-type=2&prefix=" + prefix, headers, 0U, std::bind(&OnDiscovery, - IHTTPGateway::TWeakPtr(Gateway_), ctx.GetPosition((*std::get<TNodeSet>(bucket.second).cbegin())->Pos()), std::placeholders::_1, - std::cref(bucket.first), std::ref(bucket.second), std::move(promise), pendingBucketsWPtr, i++)); + IHTTPGateway::TWeakPtr(Gateway_), ctx.GetPosition((*std::get<TNodeSet>(bucket.second).cbegin())->Pos()), std::placeholders::_1, + std::cref(bucket.first), std::ref(bucket.second), std::move(promise), pendingBucketsWPtr, i++)); YQL_CLOG(INFO, ProviderS3) << "Enumerate items in " << std::get<0U>(bucket.first) << std::get<1U>(bucket.first); } @@ -226,8 +226,8 @@ public: // Raise errors if any AllFuture_.GetValue(); - TNodeOnNodeOwnedMap replaces(PendingBuckets_->size()); - auto buckets = std::move(*PendingBuckets_); + TNodeOnNodeOwnedMap replaces(PendingBuckets_->size()); + auto buckets = std::move(*PendingBuckets_); auto count = 0ULL; auto readSize = 0ULL; for (auto& bucket : buckets) { @@ -337,7 +337,7 @@ private: const TS3State::TPtr State_; const IHTTPGateway::TPtr Gateway_; - const std::shared_ptr<TPendingBuckets> PendingBuckets_ = std::make_shared<TPendingBuckets>(); + const std::shared_ptr<TPendingBuckets> PendingBuckets_ = std::make_shared<TPendingBuckets>(); NThreading::TFuture<void> AllFuture_; }; diff --git a/ydb/library/yql/providers/solomon/async_io/dq_solomon_write_actor.cpp b/ydb/library/yql/providers/solomon/async_io/dq_solomon_write_actor.cpp index 7910b8d740..fc07635176 100644 --- a/ydb/library/yql/providers/solomon/async_io/dq_solomon_write_actor.cpp +++ b/ydb/library/yql/providers/solomon/async_io/dq_solomon_write_actor.cpp @@ -76,8 +76,8 @@ struct TMetricsInflight { class TDqSolomonWriteActor : public NActors::TActor<TDqSolomonWriteActor>, public IDqSinkActor { public: - static constexpr char ActorName[] = "DQ_SOLOMON_WRITE_ACTOR"; - + static constexpr char ActorName[] = "DQ_SOLOMON_WRITE_ACTOR"; + TDqSolomonWriteActor( ui64 outputIndex, TDqSolomonWriteParams&& writeParams, diff --git a/ydb/library/yql/providers/ydb/actors/yql_ydb_read_actor.cpp b/ydb/library/yql/providers/ydb/actors/yql_ydb_read_actor.cpp index 5df2e3e535..7ab55cfcfc 100644 --- a/ydb/library/yql/providers/ydb/actors/yql_ydb_read_actor.cpp +++ b/ydb/library/yql/providers/ydb/actors/yql_ydb_read_actor.cpp @@ -102,9 +102,9 @@ public: Become(&TYdbReadActor::StateFunc); SendRequest(); } - - static constexpr char ActorName[] = "YQL_YDB_READ_ACTOR"; - + + static constexpr char ActorName[] = "YQL_YDB_READ_ACTOR"; + private: void SaveState(const NDqProto::TCheckpoint&, NDqProto::TSourceState&) final {} void LoadState(const NDqProto::TSourceState&) final {} diff --git a/ydb/library/yql/providers/ydb/provider/ya.make b/ydb/library/yql/providers/ydb/provider/ya.make index 128d752790..b220766ee5 100644 --- a/ydb/library/yql/providers/ydb/provider/ya.make +++ b/ydb/library/yql/providers/ydb/provider/ya.make @@ -10,7 +10,7 @@ SRCS( yql_ydb_datasource_type_ann.cpp yql_ydb_dq_integration.cpp yql_ydb_exec.cpp - yql_ydb_io_discovery.cpp + yql_ydb_io_discovery.cpp yql_ydb_load_meta.cpp yql_ydb_logical_opt.cpp yql_ydb_physical_opt.cpp @@ -21,8 +21,8 @@ SRCS( ) PEERDIR( - library/cpp/random_provider - library/cpp/time_provider + library/cpp/random_provider + library/cpp/time_provider library/cpp/yson/node ydb/core/yq/libs/common ydb/core/yq/libs/db_resolver diff --git a/ydb/library/yql/providers/ydb/provider/yql_ydb_datasource.cpp b/ydb/library/yql/providers/ydb/provider/yql_ydb_datasource.cpp index b1458294aa..3dfe4e856b 100644 --- a/ydb/library/yql/providers/ydb/provider/yql_ydb_datasource.cpp +++ b/ydb/library/yql/providers/ydb/provider/yql_ydb_datasource.cpp @@ -17,11 +17,11 @@ namespace { class TYdbDataSourceProvider : public TDataProviderBase { public: - TYdbDataSourceProvider( - TYdbState::TPtr state, - NYdb::TDriver driver) + TYdbDataSourceProvider( + TYdbState::TPtr state, + NYdb::TDriver driver) : State_(state) - , IODiscoveryTransformer_(CreateYdbIODiscoveryTransformer(State_)) + , IODiscoveryTransformer_(CreateYdbIODiscoveryTransformer(State_)) , LoadMetaDataTransformer_(CreateYdbLoadTableMetadataTransformer(State_, driver)) , CallableExecutionTransformer_(CreateYdbSourceCallableExecutionTransformer(State_)) , TypeAnnotationTransformer_(CreateYdbDataSourceTypeAnnotationTransformer(State_)) @@ -56,10 +56,10 @@ public: return TypeAnnotationTransformer_->CanParse(node); } - IGraphTransformer& GetIODiscoveryTransformer() override { - return *IODiscoveryTransformer_; - } - + IGraphTransformer& GetIODiscoveryTransformer() override { + return *IODiscoveryTransformer_; + } + IGraphTransformer& GetLoadTableMetadataTransformer() override { return *LoadMetaDataTransformer_; } @@ -138,7 +138,7 @@ public: private: const TYdbState::TPtr State_; - const THolder<IGraphTransformer> IODiscoveryTransformer_; + const THolder<IGraphTransformer> IODiscoveryTransformer_; const THolder<IGraphTransformer> LoadMetaDataTransformer_; const THolder<IGraphTransformer> CallableExecutionTransformer_; const THolder<TVisitorTransformerBase> TypeAnnotationTransformer_; @@ -147,9 +147,9 @@ private: } -TIntrusivePtr<IDataProvider> CreateYdbDataSource( - TYdbState::TPtr state, - NYdb::TDriver driver) { +TIntrusivePtr<IDataProvider> CreateYdbDataSource( + TYdbState::TPtr state, + NYdb::TDriver driver) { return new TYdbDataSourceProvider(state, driver); } diff --git a/ydb/library/yql/providers/ydb/provider/yql_ydb_io_discovery.cpp b/ydb/library/yql/providers/ydb/provider/yql_ydb_io_discovery.cpp index 0d51b6638e..794c725164 100644 --- a/ydb/library/yql/providers/ydb/provider/yql_ydb_io_discovery.cpp +++ b/ydb/library/yql/providers/ydb/provider/yql_ydb_io_discovery.cpp @@ -1,107 +1,107 @@ -#include "yql_ydb_provider_impl.h" +#include "yql_ydb_provider_impl.h" #include <ydb/core/yq/libs/events/events.h> - + #include <ydb/library/yql/providers/ydb/expr_nodes/yql_ydb_expr_nodes.h> #include <ydb/library/yql/providers/common/provider/yql_provider_names.h> #include <ydb/library/yql/core/expr_nodes/yql_expr_nodes.h> - + #include <ydb/library/yql/core/yql_expr_optimize.h> #include <ydb/library/yql/core/yql_graph_transformer.h> #include <ydb/library/yql/utils/log/log.h> #include <ydb/library/yql/public/udf/udf_types.h> #include <ydb/library/yql/ast/yql_expr.h> #include <ydb/core/yq/libs/events/events.h> - -namespace NYql { - -namespace { - -using namespace NNodes; - -class TYdbIODiscoveryTransformer : public TGraphTransformerBase { - using TDbId2Endpoint = THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TEvEndpointResponse::TEndpoint>; -public: - TYdbIODiscoveryTransformer(TYdbState::TPtr state) - : State_(std::move(state)) - {} - - TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final { - output = input; - - if (ctx.Step.IsDone(TExprStep::DiscoveryIO)) - return TStatus::Ok; - - if (!State_->DbResolver) - return TStatus::Ok; - - THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TDatabaseAuth> ids; - if (auto reads = FindNodes(input, [&](const TExprNode::TPtr& node) { - const TExprBase nodeExpr(node); - if (!nodeExpr.Maybe<TYdbRead>()) - return false; - const TYdbRead read(node); - if (read.DataSource().Category().Value() != YdbProviderName) { - return false; - } - return true; - }); !reads.empty()) { - for (auto& node : reads) { - const TYdbRead read(node); - const auto& cluster = read.DataSource().Cluster().StringValue(); - const auto& dbId = State_->Configuration->Clusters[cluster].DatabaseId; - const auto idKey = std::make_pair(dbId, NYq::DatabaseType::Ydb); - const auto iter = State_->DatabaseIds.find(idKey); - if (iter != State_->DatabaseIds.end()) { - ids[idKey] = iter->second; - } - } - } - if (ids.empty()) { - return TStatus::Ok; - } - AsyncFuture_ = State_->DbResolver->ResolveIds({ids, State_->DbResolver->GetTraceId()}).Apply([resolvedIds_ = ResolvedIds_](const auto& future) { - *resolvedIds_ = future.GetValue(); - }); - return TStatus::Async; - } - - NThreading::TFuture<void> DoGetAsyncFuture(const TExprNode&) final { - return AsyncFuture_; - } - - TStatus DoApplyAsyncChanges(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext&) final { - output = input; - AsyncFuture_.GetValue(); - FullResolvedIds_.insert(ResolvedIds_->begin(), ResolvedIds_->end()); - ResolvedIds_->clear(); - auto& clusters = State_->Configuration->Clusters; - const auto& id2Clusters = State_->Configuration->DbId2Clusters; - for (const auto& [dbIdWithType, info] : FullResolvedIds_) { - const auto& dbId = dbIdWithType.first; - const auto iter = id2Clusters.find(dbId); - if (iter == id2Clusters.end()) { - continue; - } - for (const auto& clusterName : iter->second) { - auto& c = clusters[clusterName]; - c.Endpoint = info.Endpoint; - c.Database = info.Database; - c.Secure = info.Secure; - } - } - return TStatus::Ok; - } -private: - const TYdbState::TPtr State_; - - NThreading::TFuture<void> AsyncFuture_; - TDbId2Endpoint FullResolvedIds_; - std::shared_ptr<TDbId2Endpoint> ResolvedIds_ = std::make_shared<TDbId2Endpoint>(); -}; -} - -THolder<IGraphTransformer> CreateYdbIODiscoveryTransformer(TYdbState::TPtr state) { - return THolder(new TYdbIODiscoveryTransformer(std::move(state))); -} - -} + +namespace NYql { + +namespace { + +using namespace NNodes; + +class TYdbIODiscoveryTransformer : public TGraphTransformerBase { + using TDbId2Endpoint = THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TEvEndpointResponse::TEndpoint>; +public: + TYdbIODiscoveryTransformer(TYdbState::TPtr state) + : State_(std::move(state)) + {} + + TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final { + output = input; + + if (ctx.Step.IsDone(TExprStep::DiscoveryIO)) + return TStatus::Ok; + + if (!State_->DbResolver) + return TStatus::Ok; + + THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TDatabaseAuth> ids; + if (auto reads = FindNodes(input, [&](const TExprNode::TPtr& node) { + const TExprBase nodeExpr(node); + if (!nodeExpr.Maybe<TYdbRead>()) + return false; + const TYdbRead read(node); + if (read.DataSource().Category().Value() != YdbProviderName) { + return false; + } + return true; + }); !reads.empty()) { + for (auto& node : reads) { + const TYdbRead read(node); + const auto& cluster = read.DataSource().Cluster().StringValue(); + const auto& dbId = State_->Configuration->Clusters[cluster].DatabaseId; + const auto idKey = std::make_pair(dbId, NYq::DatabaseType::Ydb); + const auto iter = State_->DatabaseIds.find(idKey); + if (iter != State_->DatabaseIds.end()) { + ids[idKey] = iter->second; + } + } + } + if (ids.empty()) { + return TStatus::Ok; + } + AsyncFuture_ = State_->DbResolver->ResolveIds({ids, State_->DbResolver->GetTraceId()}).Apply([resolvedIds_ = ResolvedIds_](const auto& future) { + *resolvedIds_ = future.GetValue(); + }); + return TStatus::Async; + } + + NThreading::TFuture<void> DoGetAsyncFuture(const TExprNode&) final { + return AsyncFuture_; + } + + TStatus DoApplyAsyncChanges(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext&) final { + output = input; + AsyncFuture_.GetValue(); + FullResolvedIds_.insert(ResolvedIds_->begin(), ResolvedIds_->end()); + ResolvedIds_->clear(); + auto& clusters = State_->Configuration->Clusters; + const auto& id2Clusters = State_->Configuration->DbId2Clusters; + for (const auto& [dbIdWithType, info] : FullResolvedIds_) { + const auto& dbId = dbIdWithType.first; + const auto iter = id2Clusters.find(dbId); + if (iter == id2Clusters.end()) { + continue; + } + for (const auto& clusterName : iter->second) { + auto& c = clusters[clusterName]; + c.Endpoint = info.Endpoint; + c.Database = info.Database; + c.Secure = info.Secure; + } + } + return TStatus::Ok; + } +private: + const TYdbState::TPtr State_; + + NThreading::TFuture<void> AsyncFuture_; + TDbId2Endpoint FullResolvedIds_; + std::shared_ptr<TDbId2Endpoint> ResolvedIds_ = std::make_shared<TDbId2Endpoint>(); +}; +} + +THolder<IGraphTransformer> CreateYdbIODiscoveryTransformer(TYdbState::TPtr state) { + return THolder(new TYdbIODiscoveryTransformer(std::move(state))); +} + +} diff --git a/ydb/library/yql/providers/ydb/provider/yql_ydb_load_meta.cpp b/ydb/library/yql/providers/ydb/provider/yql_ydb_load_meta.cpp index 7b348cb8d8..607b3ba7a8 100644 --- a/ydb/library/yql/providers/ydb/provider/yql_ydb_load_meta.cpp +++ b/ydb/library/yql/providers/ydb/provider/yql_ydb_load_meta.cpp @@ -56,10 +56,10 @@ NUdf::TDataTypeId GetYqlTypeId(NYdb::TTypeParser&& type, TExprContext& ctx) { } class TYdbLoadTableMetadataTransformer : public TGraphTransformerBase { - -using TCluster2ClientPerSnapshotHandle = std::unordered_map<std::string_view, std::pair<TMetaClient, std::optional<TCreateSnapshotHandleResult>>>; -using TTableKey2DescribeTableResult = std::unordered_map<std::pair<TString, TString>, std::optional<TDescribeTableResult>, THash<std::pair<TString, TString>>>; - + +using TCluster2ClientPerSnapshotHandle = std::unordered_map<std::string_view, std::pair<TMetaClient, std::optional<TCreateSnapshotHandleResult>>>; +using TTableKey2DescribeTableResult = std::unordered_map<std::pair<TString, TString>, std::optional<TDescribeTableResult>, THash<std::pair<TString, TString>>>; + public: TYdbLoadTableMetadataTransformer(TYdbState::TPtr state, NYdb::TDriver driver) : Driver_(std::move(driver)), State_(std::move(state)) @@ -72,9 +72,9 @@ public: return TStatus::Ok; } - Y_VERIFY(Clients_, "Clients_ was destroyed"); - Y_VERIFY(PendingTables_, "PendingTables_ was destroyed"); - + Y_VERIFY(Clients_, "Clients_ was destroyed"); + Y_VERIFY(PendingTables_, "PendingTables_ was destroyed"); + std::unordered_map<TString, std::unordered_set<TString>> tablesFromCluster; TOptimizeExprSettings settings{ nullptr }; @@ -113,8 +113,8 @@ public: table = database + '/' + table; const auto& tableKey = std::make_pair(cluster, table); if (State_->Tables.cend() == State_->Tables.find(tableKey)) { - PendingTables_->emplace(tableKey, std::nullopt); - tablesFromCluster[tableKey.first].emplace(tableKey.second); + PendingTables_->emplace(tableKey, std::nullopt); + tablesFromCluster[tableKey.first].emplace(tableKey.second); } switch (key.GetKeyType()) { @@ -137,21 +137,21 @@ public: } }, ctx, settings); - if (status == TStatus::Error || PendingTables_->empty()) { + if (status == TStatus::Error || PendingTables_->empty()) { return status; } std::vector<NThreading::TFuture<void>> handles; - handles.reserve(PendingTables_->size() + tablesFromCluster.size()); + handles.reserve(PendingTables_->size() + tablesFromCluster.size()); for (const auto& item : tablesFromCluster) { const auto& cluster = item.first; TString token = State_->Configuration->Tokens.at(cluster); const auto& config = State_->Configuration->Clusters[cluster]; - + std::shared_ptr<NYdb::ICredentialsProviderFactory> credentialsProviderFactory = CreateCredentialsProviderFactoryForStructuredToken(State_->CredentialsFactory, token, config.AddBearerToToken); - const auto ins = Clients_->emplace(cluster, std::pair<TMetaClient, std::optional<TCreateSnapshotHandleResult>>{{Driver_, NYdb::TCommonClientSettings() + const auto ins = Clients_->emplace(cluster, std::pair<TMetaClient, std::optional<TCreateSnapshotHandleResult>>{{Driver_, NYdb::TCommonClientSettings() .Database(config.Database) .DiscoveryEndpoint(config.Endpoint) .EnableSsl(config.Secure) @@ -160,32 +160,32 @@ public: YQL_CLOG(INFO, ProviderYdb) << "Take snapshot for: `" << cluster << "` from " << config.Endpoint; - std::optional<TCreateSnapshotHandleResult>& snapshot = ins.first->second.second; - std::weak_ptr<TCluster2ClientPerSnapshotHandle> clients = Clients_; - handles.emplace_back(ins.first->second.first.CreateSnapshotHandle(TVector<TString>(item.second.cbegin(), item.second.cend())).Apply([clients, &snapshot] + std::optional<TCreateSnapshotHandleResult>& snapshot = ins.first->second.second; + std::weak_ptr<TCluster2ClientPerSnapshotHandle> clients = Clients_; + handles.emplace_back(ins.first->second.first.CreateSnapshotHandle(TVector<TString>(item.second.cbegin(), item.second.cend())).Apply([clients, &snapshot] (const NThreading::TFuture<TCreateSnapshotHandleResult>& future) { - if (future.HasException()) { - const_cast<NThreading::TFuture<TCreateSnapshotHandleResult>&>(future).ExtractValue(); - } else if (clients.lock()) { - snapshot.emplace(const_cast<NThreading::TFuture<TCreateSnapshotHandleResult>&>(future).ExtractValue()); - } + if (future.HasException()) { + const_cast<NThreading::TFuture<TCreateSnapshotHandleResult>&>(future).ExtractValue(); + } else if (clients.lock()) { + snapshot.emplace(const_cast<NThreading::TFuture<TCreateSnapshotHandleResult>&>(future).ExtractValue()); + } })); } - for (auto& pair : *PendingTables_) { - const auto find = Clients_->find(pair.first.first); + for (auto& pair : *PendingTables_) { + const auto find = Clients_->find(pair.first.first); const auto& table = pair.first.second; YQL_CLOG(INFO, ProviderYdb) << "Load table meta for: `" << table << "`"; - std::optional<TDescribeTableResult>& meta = pair.second; - std::weak_ptr<TTableKey2DescribeTableResult> pendingTables = PendingTables_; - handles.emplace_back(find->second.first.GetTableDescription(table, true).Apply([pendingTables, &meta] + std::optional<TDescribeTableResult>& meta = pair.second; + std::weak_ptr<TTableKey2DescribeTableResult> pendingTables = PendingTables_; + handles.emplace_back(find->second.first.GetTableDescription(table, true).Apply([pendingTables, &meta] (const NThreading::TFuture<TDescribeTableResult>& future) { - if (future.HasException()) { - const_cast<NThreading::TFuture<TDescribeTableResult>&>(future).ExtractValue(); - } else if (pendingTables.lock()) { - meta.emplace(const_cast<NThreading::TFuture<TDescribeTableResult>&>(future).ExtractValue()); - } + if (future.HasException()) { + const_cast<NThreading::TFuture<TDescribeTableResult>&>(future).ExtractValue(); + } else if (pendingTables.lock()) { + meta.emplace(const_cast<NThreading::TFuture<TDescribeTableResult>&>(future).ExtractValue()); + } })); } @@ -205,12 +205,12 @@ public: YQL_ENSURE(AsyncFuture_.HasValue()); output = input; - Y_VERIFY(Clients_, "Clients_ was destroyed"); - Y_VERIFY(PendingTables_, "PendingTables_ was destroyed"); - + Y_VERIFY(Clients_, "Clients_ was destroyed"); + Y_VERIFY(PendingTables_, "PendingTables_ was destroyed"); + bool failed = false; - std::unordered_map<std::string_view, TSnapshotHandle> snapshots(Clients_->size()); - for (auto& client : *Clients_) { + std::unordered_map<std::string_view, TSnapshotHandle> snapshots(Clients_->size()); + for (auto& client : *Clients_) { auto& snapshot = *client.second.second; if (snapshot.IsSuccess()) { snapshots.emplace(client.first, snapshot.ExtractResult()); @@ -223,7 +223,7 @@ public: } } - for (auto& pair : *PendingTables_) { + for (auto& pair : *PendingTables_) { const auto& result = *pair.second; auto& meta = State_->Tables[pair.first]; meta.Snapshot = snapshots[pair.first.first]; @@ -260,16 +260,16 @@ public: } } - Clients_->clear(); - PendingTables_->clear(); + Clients_->clear(); + PendingTables_->clear(); return failed ? TStatus::Error : TStatus::Ok; } private: const NYdb::TDriver Driver_; const TYdbState::TPtr State_; - std::shared_ptr<TCluster2ClientPerSnapshotHandle> Clients_ = std::make_shared<TCluster2ClientPerSnapshotHandle>(); - std::shared_ptr<TTableKey2DescribeTableResult > PendingTables_ = std::make_shared<TTableKey2DescribeTableResult>(); + std::shared_ptr<TCluster2ClientPerSnapshotHandle> Clients_ = std::make_shared<TCluster2ClientPerSnapshotHandle>(); + std::shared_ptr<TTableKey2DescribeTableResult > PendingTables_ = std::make_shared<TTableKey2DescribeTableResult>(); NThreading::TFuture<void> AsyncFuture_; }; diff --git a/ydb/library/yql/providers/ydb/provider/yql_ydb_provider.cpp b/ydb/library/yql/providers/ydb/provider/yql_ydb_provider.cpp index ca69764a86..d17e353d3e 100644 --- a/ydb/library/yql/providers/ydb/provider/yql_ydb_provider.cpp +++ b/ydb/library/yql/providers/ydb/provider/yql_ydb_provider.cpp @@ -4,11 +4,11 @@ namespace NYql { -TDataProviderInitializer GetYdbDataProviderInitializer( - NYdb::TDriver driver, - ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory, - std::shared_ptr<NYq::TDatabaseAsyncResolverWithMeta> dbResolverWithMeta) { - return [driver, credentialsFactory, dbResolverWithMeta] ( +TDataProviderInitializer GetYdbDataProviderInitializer( + NYdb::TDriver driver, + ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory, + std::shared_ptr<NYq::TDatabaseAsyncResolverWithMeta> dbResolverWithMeta) { + return [driver, credentialsFactory, dbResolverWithMeta] ( const TString& userName, const TString& sessionId, const TGatewaysConfig* gatewaysConfig, @@ -30,9 +30,9 @@ TDataProviderInitializer GetYdbDataProviderInitializer( state->Types = typeCtx.Get(); state->FunctionRegistry = functionRegistry; state->CredentialsFactory = credentialsFactory; - state->DbResolver = dbResolverWithMeta; + state->DbResolver = dbResolverWithMeta; if (gatewaysConfig) { - state->Configuration->Init(gatewaysConfig->GetYdb(), typeCtx, state->DbResolver, state->DatabaseIds); + state->Configuration->Init(gatewaysConfig->GetYdb(), typeCtx, state->DbResolver, state->DatabaseIds); } TDataProviderInfo info; diff --git a/ydb/library/yql/providers/ydb/provider/yql_ydb_provider.h b/ydb/library/yql/providers/ydb/provider/yql_ydb_provider.h index d38323f265..ac9d00d145 100644 --- a/ydb/library/yql/providers/ydb/provider/yql_ydb_provider.h +++ b/ydb/library/yql/providers/ydb/provider/yql_ydb_provider.h @@ -33,21 +33,21 @@ struct TYdbState : public TThrRefBase TYdbConfiguration::TPtr Configuration = MakeIntrusive<TYdbConfiguration>(); const NKikimr::NMiniKQL::IFunctionRegistry* FunctionRegistry = nullptr; ISecuredServiceAccountCredentialsFactory::TPtr CredentialsFactory; - THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TDatabaseAuth> DatabaseIds; - std::shared_ptr<NYq::TDatabaseAsyncResolverWithMeta> DbResolver; + THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TDatabaseAuth> DatabaseIds; + std::shared_ptr<NYq::TDatabaseAsyncResolverWithMeta> DbResolver; }; -TDataProviderInitializer GetYdbDataProviderInitializer( - NYdb::TDriver driver, - ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory = nullptr, - const std::shared_ptr<NYq::TDatabaseAsyncResolverWithMeta> dbResolverWithMeta = nullptr -); - -TIntrusivePtr<IDataProvider> CreateYdbDataSource( - TYdbState::TPtr state, - NYdb::TDriver driver -); - +TDataProviderInitializer GetYdbDataProviderInitializer( + NYdb::TDriver driver, + ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory = nullptr, + const std::shared_ptr<NYq::TDatabaseAsyncResolverWithMeta> dbResolverWithMeta = nullptr +); + +TIntrusivePtr<IDataProvider> CreateYdbDataSource( + TYdbState::TPtr state, + NYdb::TDriver driver +); + TIntrusivePtr<IDataProvider> CreateYdbDataSink(TYdbState::TPtr state); } // namespace NYql diff --git a/ydb/library/yql/providers/ydb/provider/yql_ydb_provider_impl.h b/ydb/library/yql/providers/ydb/provider/yql_ydb_provider_impl.h index 3cabc992d8..454c019d58 100644 --- a/ydb/library/yql/providers/ydb/provider/yql_ydb_provider_impl.h +++ b/ydb/library/yql/providers/ydb/provider/yql_ydb_provider_impl.h @@ -12,7 +12,7 @@ namespace NYql { -THolder<IGraphTransformer> CreateYdbIODiscoveryTransformer(TYdbState::TPtr state); +THolder<IGraphTransformer> CreateYdbIODiscoveryTransformer(TYdbState::TPtr state); THolder<IGraphTransformer> CreateYdbLoadTableMetadataTransformer(TYdbState::TPtr state, NYdb::TDriver driver); THolder<TVisitorTransformerBase> CreateYdbDataSourceTypeAnnotationTransformer(TYdbState::TPtr state); diff --git a/ydb/library/yql/providers/ydb/provider/yql_ydb_settings.cpp b/ydb/library/yql/providers/ydb/provider/yql_ydb_settings.cpp index da9ae709a2..06d1919027 100644 --- a/ydb/library/yql/providers/ydb/provider/yql_ydb_settings.cpp +++ b/ydb/library/yql/providers/ydb/provider/yql_ydb_settings.cpp @@ -17,11 +17,11 @@ bool TYdbConfiguration::HasCluster(TStringBuf cluster) const { return ValidClusters.contains(cluster); } -void TYdbConfiguration::Init( - const TYdbGatewayConfig& config, - TIntrusivePtr<TTypeAnnotationContext> typeCtx, - const std::shared_ptr<NYq::TDatabaseAsyncResolverWithMeta> dbResolver, - THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TDatabaseAuth>& databaseIds) +void TYdbConfiguration::Init( + const TYdbGatewayConfig& config, + TIntrusivePtr<TTypeAnnotationContext> typeCtx, + const std::shared_ptr<NYq::TDatabaseAsyncResolverWithMeta> dbResolver, + THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TDatabaseAuth>& databaseIds) { TVector<TString> clusters(Reserve(config.ClusterMappingSize())); for (auto& cluster: config.GetClusterMapping()) { @@ -36,19 +36,19 @@ void TYdbConfiguration::Init( for (const auto& cluster : config.GetClusterMapping()) { this->Dispatch(cluster.GetName(), cluster.GetSettings()); - - if (dbResolver) { - dbResolver->TryAddDbIdToResolve(cluster.HasEndpoint(), cluster.GetName(), cluster.GetId(), NYq::DatabaseType::Ydb, databaseIds); - if (cluster.GetId()) { - DbId2Clusters[cluster.GetId()].emplace_back(cluster.GetName()); - } - } - + + if (dbResolver) { + dbResolver->TryAddDbIdToResolve(cluster.HasEndpoint(), cluster.GetName(), cluster.GetId(), NYq::DatabaseType::Ydb, databaseIds); + if (cluster.GetId()) { + DbId2Clusters[cluster.GetId()].emplace_back(cluster.GetName()); + } + } + auto& settings = Clusters[cluster.GetName()]; settings.Endpoint = cluster.HasEndpoint() ? cluster.GetEndpoint() : endpoint; settings.Database = cluster.GetDatabase().empty() ? cluster.GetName() : cluster.GetDatabase(); - if (cluster.GetId()) - settings.DatabaseId = cluster.GetId(); + if (cluster.GetId()) + settings.DatabaseId = cluster.GetId(); if (cluster.HasSecure()) settings.Secure = cluster.GetSecure(); if (cluster.HasAddBearerToToken()) @@ -57,7 +57,7 @@ void TYdbConfiguration::Init( const TString authToken = typeCtx->FindCredentialContent("cluster:default_" + cluster.GetName(), "default_ydb", cluster.GetToken()); Tokens[cluster.GetName()] = ComposeStructuredTokenJsonForServiceAccount(cluster.GetServiceAccountId(), cluster.GetServiceAccountIdSignature(), authToken); settings.Raw = cluster; - + } this->FreezeDefaults(); } diff --git a/ydb/library/yql/providers/ydb/provider/yql_ydb_settings.h b/ydb/library/yql/providers/ydb/provider/yql_ydb_settings.h index 3640971a8f..fc6fb03ced 100644 --- a/ydb/library/yql/providers/ydb/provider/yql_ydb_settings.h +++ b/ydb/library/yql/providers/ydb/provider/yql_ydb_settings.h @@ -8,8 +8,8 @@ #include <ydb/core/yq/libs/events/events.h> #include <ydb/core/yq/libs/db_resolver/db_async_resolver_with_meta.h> #include <ydb/core/yq/libs/common/database_token_builder.h> - - + + namespace NYql { struct TYdbSettings { @@ -19,7 +19,7 @@ struct TYdbSettings { struct TClusterMainSettings { TString Endpoint; TString Database; - TString DatabaseId; + TString DatabaseId; bool Secure = false; bool AddBearerToToken = false; @@ -32,19 +32,19 @@ struct TYdbConfiguration : public TYdbSettings, public NCommon::TSettingDispatch TYdbConfiguration(); TYdbConfiguration(const TYdbConfiguration&) = delete; - void Init( - const TYdbGatewayConfig& config, - TIntrusivePtr<TTypeAnnotationContext> typeCtx, - const std::shared_ptr<NYq::TDatabaseAsyncResolverWithMeta> dbResolver, - THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TDatabaseAuth>& databaseIds - ); + void Init( + const TYdbGatewayConfig& config, + TIntrusivePtr<TTypeAnnotationContext> typeCtx, + const std::shared_ptr<NYq::TDatabaseAsyncResolverWithMeta> dbResolver, + THashMap<std::pair<TString, NYq::DatabaseType>, NYq::TEvents::TDatabaseAuth>& databaseIds + ); bool HasCluster(TStringBuf cluster) const; TYdbSettings::TConstPtr Snapshot() const; THashMap<TString, TString> Tokens; std::unordered_map<TString, TClusterMainSettings> Clusters; - THashMap<TString, TVector<TString>> DbId2Clusters; // DatabaseId -> ClusterNames + THashMap<TString, TVector<TString>> DbId2Clusters; // DatabaseId -> ClusterNames }; } // NYql diff --git a/ydb/library/yql/utils/actor_log/log.cpp b/ydb/library/yql/utils/actor_log/log.cpp index 5441aec0cc..3b3002208c 100644 --- a/ydb/library/yql/utils/actor_log/log.cpp +++ b/ydb/library/yql/utils/actor_log/log.cpp @@ -57,16 +57,16 @@ NYql::NLog::ELevel GetYqlLogLevel(NActors::NLog::EPriority priority) { } // namespace void TActorYqlLogBackend::WriteData(const TLogRecord& rec) { - std::visit([&](const auto* actorCtxOrSystem){ - Y_VERIFY(actorCtxOrSystem); - if (TraceId.empty()) { - LOG_LOG(*actorCtxOrSystem, GetActorLogPriority(rec.Priority), Component, - "SessionId: %s %.*s", SessionId.c_str(), (int)rec.Len, rec.Data); - } else { - LOG_LOG(*actorCtxOrSystem, GetActorLogPriority(rec.Priority), Component, - "TraceId: %s, SessionId: %s %.*s", TraceId.c_str(), SessionId.c_str(), (int)rec.Len, rec.Data); - } - }, ActorCtxOrSystem); + std::visit([&](const auto* actorCtxOrSystem){ + Y_VERIFY(actorCtxOrSystem); + if (TraceId.empty()) { + LOG_LOG(*actorCtxOrSystem, GetActorLogPriority(rec.Priority), Component, + "SessionId: %s %.*s", SessionId.c_str(), (int)rec.Len, rec.Data); + } else { + LOG_LOG(*actorCtxOrSystem, GetActorLogPriority(rec.Priority), Component, + "TraceId: %s, SessionId: %s %.*s", TraceId.c_str(), SessionId.c_str(), (int)rec.Len, rec.Data); + } + }, ActorCtxOrSystem); } void SetYqlLogLevels(const NActors::NLog::EPriority& priority) { diff --git a/ydb/library/yql/utils/actor_log/log.h b/ydb/library/yql/utils/actor_log/log.h index b0f3e66161..08d525d28b 100644 --- a/ydb/library/yql/utils/actor_log/log.h +++ b/ydb/library/yql/utils/actor_log/log.h @@ -11,31 +11,31 @@ namespace NDq { class TActorYqlLogBackend : public TLogBackend { public: - TActorYqlLogBackend( - const NActors::TActorContext& actorCtx, - int component, - const TString& sessionId, - const TString& traceId) - : ActorCtxOrSystem(&actorCtx) + TActorYqlLogBackend( + const NActors::TActorContext& actorCtx, + int component, + const TString& sessionId, + const TString& traceId) + : ActorCtxOrSystem(&actorCtx) + , Component(component) + , SessionId(sessionId) + , TraceId(traceId) {} + + TActorYqlLogBackend( + const NActors::TActorSystem* actorSystem, + int component, + const TString& sessionId, + const TString& traceId) + : ActorCtxOrSystem(actorSystem) , Component(component) , SessionId(sessionId) , TraceId(traceId) {} - TActorYqlLogBackend( - const NActors::TActorSystem* actorSystem, - int component, - const TString& sessionId, - const TString& traceId) - : ActorCtxOrSystem(actorSystem) - , Component(component) - , SessionId(sessionId) - , TraceId(traceId) {} - void WriteData(const TLogRecord& rec) override; void ReopenLog() override {} private: - std::variant<const NActors::TActorContext*, const NActors::TActorSystem*> ActorCtxOrSystem; + std::variant<const NActors::TActorContext*, const NActors::TActorSystem*> ActorCtxOrSystem; int Component; TString SessionId; TString TraceId; @@ -43,11 +43,11 @@ private: class TYqlLogScope : public NYql::NLog::TScopedBackend<TActorYqlLogBackend> { public: - TYqlLogScope(const NActors::TActorContext& actorCtx, int component, const TString& sessionId, const TString& traceId = "") + TYqlLogScope(const NActors::TActorContext& actorCtx, int component, const TString& sessionId, const TString& traceId = "") : NYql::NLog::TScopedBackend<TActorYqlLogBackend>(actorCtx, component, sessionId, traceId) {}; - - TYqlLogScope(const NActors::TActorSystem* actorSystem, int component, const TString& sessionId, const TString& traceId = "") - : NYql::NLog::TScopedBackend<TActorYqlLogBackend>(actorSystem, component, sessionId, traceId) {}; + + TYqlLogScope(const NActors::TActorSystem* actorSystem, int component, const TString& sessionId, const TString& traceId = "") + : NYql::NLog::TScopedBackend<TActorYqlLogBackend>(actorSystem, component, sessionId, traceId) {}; }; class TLogWrapReceive: public NActors::TDecorator { diff --git a/ydb/public/api/grpc/draft/yql_db_v1.proto b/ydb/public/api/grpc/draft/yql_db_v1.proto index 3686684137..12b061eca3 100644 --- a/ydb/public/api/grpc/draft/yql_db_v1.proto +++ b/ydb/public/api/grpc/draft/yql_db_v1.proto @@ -1,19 +1,19 @@ syntax = "proto3"; -package Yq.Private.V1; +package Yq.Private.V1; import "ydb/public/api/protos/draft/yq_private.proto"; -service YqPrivateTaskService { +service YqPrivateTaskService { // gets new task - rpc GetTask(Yq.Private.GetTaskRequest) returns (Yq.Private.GetTaskResponse); + rpc GetTask(Yq.Private.GetTaskRequest) returns (Yq.Private.GetTaskResponse); // pings new task (also can update metadata) - rpc PingTask(Yq.Private.PingTaskRequest) returns (Yq.Private.PingTaskResponse); + rpc PingTask(Yq.Private.PingTaskRequest) returns (Yq.Private.PingTaskResponse); // writes rows - rpc WriteTaskResult(Yq.Private.WriteTaskResultRequest) returns (Yq.Private.WriteTaskResultResponse); - - //Nodes - rpc NodesHealthCheck(Yq.Private.NodesHealthCheckRequest) returns (Yq.Private.NodesHealthCheckResponse); + rpc WriteTaskResult(Yq.Private.WriteTaskResultRequest) returns (Yq.Private.WriteTaskResultResponse); + + //Nodes + rpc NodesHealthCheck(Yq.Private.NodesHealthCheckRequest) returns (Yq.Private.NodesHealthCheckResponse); } diff --git a/ydb/public/api/protos/draft/yq_private.proto b/ydb/public/api/protos/draft/yq_private.proto index e08b3726a9..2af1c2f3ed 100644 --- a/ydb/public/api/protos/draft/yq_private.proto +++ b/ydb/public/api/protos/draft/yq_private.proto @@ -1,7 +1,7 @@ syntax = "proto3"; option cc_enable_arenas = true; -package Yq.Private; +package Yq.Private; import "ydb/public/api/protos/ydb_operation.proto"; import "ydb/public/api/protos/ydb_value.proto"; @@ -14,17 +14,17 @@ import "google/protobuf/timestamp.proto"; //////////////////////////////////////////////////////////// message GetTaskRequest { - string tenant = 1; - string owner_id = 2; // guid, should be refreshed on node restart - string host = 3; - Ydb.Operations.OperationParams operation_params = 4; + string tenant = 1; + string owner_id = 2; // guid, should be refreshed on node restart + string host = 3; + Ydb.Operations.OperationParams operation_params = 4; +} + +message SignedIdentity { + string value = 1; + string signature = 2; } -message SignedIdentity { - string value = 1; - string signature = 2; -} - message TopicConsumer { string database_id = 1; string database = 2; @@ -37,39 +37,39 @@ message TopicConsumer { } message GetTaskResult { - message Task { - // come back later in 10 sec ? - SignedIdentity result_id = 1; - SignedIdentity query_id = 2; - SignedIdentity job_id = 3; - uint64 generation = 4; - - bool streaming = 5; - repeated bytes dq_graph = 6; - // text, connection and binding are empty if dq_graph is not empty - string text = 7; - repeated YandexQuery.Connection connection = 8; - repeated YandexQuery.Binding binding = 9; - - string user_token = 10; // IAM token for debug - repeated SignedIdentity service_accounts = 11; - string user_id = 12; - YandexQuery.QueryContent.QueryType query_type = 13; - string scope = 14; - YandexQuery.ExecuteMode execute_mode = 15; - YandexQuery.StateLoadMode state_load_mode = 16; - YandexQuery.QueryMeta.ComputeStatus status = 17; - repeated YandexQuery.ResultSetMeta result_set_meta = 18; - repeated TopicConsumer created_topic_consumers = 19; - int32 dq_graph_index = 20; - map<string, string> sensor_labels = 21; - - bool automatic = 22; - string query_name = 23; + message Task { + // come back later in 10 sec ? + SignedIdentity result_id = 1; + SignedIdentity query_id = 2; + SignedIdentity job_id = 3; + uint64 generation = 4; + + bool streaming = 5; + repeated bytes dq_graph = 6; + // text, connection and binding are empty if dq_graph is not empty + string text = 7; + repeated YandexQuery.Connection connection = 8; + repeated YandexQuery.Binding binding = 9; + + string user_token = 10; // IAM token for debug + repeated SignedIdentity service_accounts = 11; + string user_id = 12; + YandexQuery.QueryContent.QueryType query_type = 13; + string scope = 14; + YandexQuery.ExecuteMode execute_mode = 15; + YandexQuery.StateLoadMode state_load_mode = 16; + YandexQuery.QueryMeta.ComputeStatus status = 17; + repeated YandexQuery.ResultSetMeta result_set_meta = 18; + repeated TopicConsumer created_topic_consumers = 19; + int32 dq_graph_index = 20; + map<string, string> sensor_labels = 21; + + bool automatic = 22; + string query_name = 23; google.protobuf.Timestamp deadline = 24; YandexQuery.StreamingDisposition disposition = 25; - } - repeated Task tasks = 1; + } + repeated Task tasks = 1; } message GetTaskResponse { @@ -77,34 +77,34 @@ message GetTaskResponse { } message PingTaskRequest { - string owner_id = 1; - SignedIdentity query_id = 2; - SignedIdentity job_id = 3; - SignedIdentity result_id = 4; - YandexQuery.QueryMeta.ComputeStatus status = 5; + string owner_id = 1; + SignedIdentity query_id = 2; + SignedIdentity job_id = 3; + SignedIdentity result_id = 4; + YandexQuery.QueryMeta.ComputeStatus status = 5; repeated Ydb.Issue.IssueMessage issues = 6; repeated Ydb.Issue.IssueMessage transient_issues = 16; uint32 result_set_count = 7; string statistics = 8; - repeated YandexQuery.ResultSetMeta result_set_meta = 9; + repeated YandexQuery.ResultSetMeta result_set_meta = 9; string executer_info = 10; repeated bytes dq_graph = 11; int32 dq_graph_index = 20; - string ast = 12; - string plan = 13; - bool resign_query = 14; + string ast = 12; + string plan = 13; + bool resign_query = 14; repeated TopicConsumer created_topic_consumers = 17; YandexQuery.StateLoadMode state_load_mode = 18; YandexQuery.StreamingDisposition disposition = 19; - Ydb.Operations.OperationParams operation_params = 15; - string scope = 100; //TODO remove - google.protobuf.Timestamp started_at = 101; - google.protobuf.Timestamp finished_at = 102; + Ydb.Operations.OperationParams operation_params = 15; + string scope = 100; //TODO remove + google.protobuf.Timestamp started_at = 101; + google.protobuf.Timestamp finished_at = 102; google.protobuf.Timestamp deadline = 103; } message PingTaskResult { - YandexQuery.QueryAction action = 1; + YandexQuery.QueryAction action = 1; } message PingTaskResponse { @@ -112,24 +112,24 @@ message PingTaskResponse { } message WriteTaskResultRequest { - string owner_id = 1; - SignedIdentity result_id = 2; + string owner_id = 1; + SignedIdentity result_id = 2; Ydb.ResultSet result_set = 3; uint32 result_set_id = 4; uint64 offset = 5; - uint64 request_id = 6; - Ydb.Operations.OperationParams operation_params = 7; + uint64 request_id = 6; + Ydb.Operations.OperationParams operation_params = 7; google.protobuf.Timestamp deadline = 8; } message WriteTaskResultResult { - uint64 request_id = 1; + uint64 request_id = 1; } message WriteTaskResultResponse { Ydb.Operations.Operation operation = 1; // WriteRowsResultResult } - + message NodeInfo { uint32 node_id = 1; string instance_id = 2; @@ -137,20 +137,20 @@ message NodeInfo { uint64 active_workers = 4; uint64 memory_limit = 5; uint64 memory_allocated = 6; - uint32 interconnect_port = 7; - string node_address = 8; + uint32 interconnect_port = 7; + string node_address = 8; } -message NodesHealthCheckRequest { - string tenant = 1; +message NodesHealthCheckRequest { + string tenant = 1; NodeInfo node = 2; - Ydb.Operations.OperationParams operation_params = 6; -} - -message NodesHealthCheckResult { - repeated NodeInfo nodes = 1; -} - -message NodesHealthCheckResponse { - Ydb.Operations.Operation operation = 1; // NodesHealthCheckResult -} + Ydb.Operations.OperationParams operation_params = 6; +} + +message NodesHealthCheckResult { + repeated NodeInfo nodes = 1; +} + +message NodesHealthCheckResponse { + Ydb.Operations.Operation operation = 1; // NodesHealthCheckResult +} diff --git a/ydb/public/api/protos/ya.make b/ydb/public/api/protos/ya.make index 4d02adac4d..8a156403a3 100644 --- a/ydb/public/api/protos/ya.make +++ b/ydb/public/api/protos/ya.make @@ -19,7 +19,7 @@ SRCS( draft/persqueue_error_codes.proto draft/ydb_long_tx.proto draft/ydb_logstore.proto - draft/yq_private.proto + draft/yq_private.proto persqueue_error_codes_v1.proto ydb_auth.proto ydb_persqueue_v1.proto diff --git a/ydb/public/api/protos/yq.proto b/ydb/public/api/protos/yq.proto index cb24bbc1da..6aa7169a89 100644 --- a/ydb/public/api/protos/yq.proto +++ b/ydb/public/api/protos/yq.proto @@ -104,7 +104,7 @@ message QueryContent { ANALYTICS = 1; // Analytical query (used for analytical data processing for example to work with YDB, ClickHouse, ...) STREAMING = 2; // Streaming query (used for streaming data processing, such as working with YDS) } - QueryType type = 1; + QueryType type = 1; string name = 2 [(Ydb.length).le = 1024]; Acl acl = 3; Limits limits = 4; @@ -160,7 +160,7 @@ message QueryMeta { } message BriefQuery { - QueryContent.QueryType type = 1; + QueryContent.QueryType type = 1; string name = 2 [(Ydb.length).le = 1024]; QueryMeta meta = 3; Acl.Visibility visibility = 4; diff --git a/ydb/services/yq/private_grpc.cpp b/ydb/services/yq/private_grpc.cpp index 79a9f5e9ea..c4c076b482 100644 --- a/ydb/services/yq/private_grpc.cpp +++ b/ydb/services/yq/private_grpc.cpp @@ -1,66 +1,66 @@ -#include "private_grpc.h" - +#include "private_grpc.h" + #include <ydb/core/grpc_services/base/base.h> #include <ydb/core/grpc_services/grpc_helper.h> #include <ydb/core/grpc_services/service_analytics_internal.h> #include <ydb/library/protobuf_printer/security_printer.h> - -namespace NKikimr { -namespace NGRpcService { - -TGRpcYqPrivateTaskService::TGRpcYqPrivateTaskService(NActors::TActorSystem *system, - TIntrusivePtr<NMonitoring::TDynamicCounters> counters, NActors::TActorId id) - : ActorSystem_(system) - , Counters_(counters) - , GRpcRequestProxyId_(id) {} - -void TGRpcYqPrivateTaskService::InitService(grpc::ServerCompletionQueue* cq, NGrpc::TLoggerPtr logger) { - CQ_ = cq; - SetupIncomingRequests(std::move(logger)); -} - -void TGRpcYqPrivateTaskService::SetGlobalLimiterHandle(NGrpc::TGlobalLimiter* limiter) { - Limiter_ = limiter; -} - -bool TGRpcYqPrivateTaskService::IncRequest() { - return Limiter_->Inc(); -} - -void TGRpcYqPrivateTaskService::DecRequest() { - Limiter_->Dec(); - Y_ASSERT(Limiter_->GetCurrentInFlight() >= 0); -} - -void TGRpcYqPrivateTaskService::SetupIncomingRequests(NGrpc::TLoggerPtr logger) { - auto getCounterBlock = CreateCounterCb(Counters_, ActorSystem_); - -#ifdef ADD_REQUEST -#error ADD_REQUEST macro already defined -#endif + +namespace NKikimr { +namespace NGRpcService { + +TGRpcYqPrivateTaskService::TGRpcYqPrivateTaskService(NActors::TActorSystem *system, + TIntrusivePtr<NMonitoring::TDynamicCounters> counters, NActors::TActorId id) + : ActorSystem_(system) + , Counters_(counters) + , GRpcRequestProxyId_(id) {} + +void TGRpcYqPrivateTaskService::InitService(grpc::ServerCompletionQueue* cq, NGrpc::TLoggerPtr logger) { + CQ_ = cq; + SetupIncomingRequests(std::move(logger)); +} + +void TGRpcYqPrivateTaskService::SetGlobalLimiterHandle(NGrpc::TGlobalLimiter* limiter) { + Limiter_ = limiter; +} + +bool TGRpcYqPrivateTaskService::IncRequest() { + return Limiter_->Inc(); +} + +void TGRpcYqPrivateTaskService::DecRequest() { + Limiter_->Dec(); + Y_ASSERT(Limiter_->GetCurrentInFlight() >= 0); +} + +void TGRpcYqPrivateTaskService::SetupIncomingRequests(NGrpc::TLoggerPtr logger) { + auto getCounterBlock = CreateCounterCb(Counters_, ActorSystem_); + +#ifdef ADD_REQUEST +#error ADD_REQUEST macro already defined +#endif #define ADD_REQUEST(NAME, CB) \ MakeIntrusive<TGRpcRequest<Yq::Private::NAME##Request, Yq::Private::NAME##Response, TGRpcYqPrivateTaskService, TSecurityTextFormatPrinter<Yq::Private::NAME##Request>, TSecurityTextFormatPrinter<Yq::Private::NAME##Response>>>( \ this, &Service_, CQ_, \ [this](NGrpc::IRequestContextBase *ctx) { \ NGRpcService::ReportGrpcReqToMon(*ActorSystem_, ctx->GetPeer()); \ ActorSystem_->Send(GRpcRequestProxyId_, \ - new TGrpcRequestOperationCall<Yq::Private::NAME##Request, Yq::Private::NAME##Response> \ + new TGrpcRequestOperationCall<Yq::Private::NAME##Request, Yq::Private::NAME##Response> \ (ctx, &CB)); \ }, \ - &Yq::Private::V1::YqPrivateTaskService::AsyncService::Request##NAME, \ + &Yq::Private::V1::YqPrivateTaskService::AsyncService::Request##NAME, \ #NAME, logger, getCounterBlock("yql_internal", #NAME)) \ ->Run(); \ - - ADD_REQUEST(PingTask, DoYqPrivatePingTaskRequest) - - ADD_REQUEST(GetTask, DoYqPrivateGetTaskRequest) - - ADD_REQUEST(WriteTaskResult, DoYqPrivateWriteTaskResultRequest) - - ADD_REQUEST(NodesHealthCheck, DoYqPrivateNodesHealthCheckRequest) - -#undef ADD_REQUEST -} - -} // namespace NGRpcService -} // namespace NKikimr + + ADD_REQUEST(PingTask, DoYqPrivatePingTaskRequest) + + ADD_REQUEST(GetTask, DoYqPrivateGetTaskRequest) + + ADD_REQUEST(WriteTaskResult, DoYqPrivateWriteTaskResultRequest) + + ADD_REQUEST(NodesHealthCheck, DoYqPrivateNodesHealthCheckRequest) + +#undef ADD_REQUEST +} + +} // namespace NGRpcService +} // namespace NKikimr diff --git a/ydb/services/yq/private_grpc.h b/ydb/services/yq/private_grpc.h index 5406b58446..f5107a7c10 100644 --- a/ydb/services/yq/private_grpc.h +++ b/ydb/services/yq/private_grpc.h @@ -1,34 +1,34 @@ -#pragma once - -#include <library/cpp/actors/core/actorsystem.h> -#include <library/cpp/grpc/server/grpc_server.h> +#pragma once + +#include <library/cpp/actors/core/actorsystem.h> +#include <library/cpp/grpc/server/grpc_server.h> #include <ydb/public/api/grpc/draft/yql_db_v1.grpc.pb.h> - -namespace NKikimr { -namespace NGRpcService { - -class TGRpcYqPrivateTaskService - : public NGrpc::TGrpcServiceBase<Yq::Private::V1::YqPrivateTaskService> -{ -public: - TGRpcYqPrivateTaskService(NActors::TActorSystem* system, TIntrusivePtr<NMonitoring::TDynamicCounters> counters, - NActors::TActorId id); - - void InitService(grpc::ServerCompletionQueue* cq, NGrpc::TLoggerPtr logger) override; - void SetGlobalLimiterHandle(NGrpc::TGlobalLimiter* limiter) override; - - bool IncRequest(); - void DecRequest(); -private: - void SetupIncomingRequests(NGrpc::TLoggerPtr logger); - - NActors::TActorSystem* ActorSystem_; + +namespace NKikimr { +namespace NGRpcService { + +class TGRpcYqPrivateTaskService + : public NGrpc::TGrpcServiceBase<Yq::Private::V1::YqPrivateTaskService> +{ +public: + TGRpcYqPrivateTaskService(NActors::TActorSystem* system, TIntrusivePtr<NMonitoring::TDynamicCounters> counters, + NActors::TActorId id); + + void InitService(grpc::ServerCompletionQueue* cq, NGrpc::TLoggerPtr logger) override; + void SetGlobalLimiterHandle(NGrpc::TGlobalLimiter* limiter) override; + + bool IncRequest(); + void DecRequest(); +private: + void SetupIncomingRequests(NGrpc::TLoggerPtr logger); + + NActors::TActorSystem* ActorSystem_; grpc::ServerCompletionQueue* CQ_ = nullptr; - - TIntrusivePtr<NMonitoring::TDynamicCounters> Counters_; - NActors::TActorId GRpcRequestProxyId_; + + TIntrusivePtr<NMonitoring::TDynamicCounters> Counters_; + NActors::TActorId GRpcRequestProxyId_; NGrpc::TGlobalLimiter* Limiter_ = nullptr; -}; - -} // namespace NGRpcService -} // namespace NKikimr +}; + +} // namespace NGRpcService +} // namespace NKikimr diff --git a/ydb/services/yq/ut_integration/ut_utils.cpp b/ydb/services/yq/ut_integration/ut_utils.cpp index 0ab1382fd7..925f74f7b3 100644 --- a/ydb/services/yq/ut_integration/ut_utils.cpp +++ b/ydb/services/yq/ut_integration/ut_utils.cpp @@ -17,15 +17,15 @@ #include <util/string/printf.h> #include <util/string/builder.h> -#include <library/cpp/protobuf/json/proto2json.h> - +#include <library/cpp/protobuf/json/proto2json.h> + using namespace NYdb; using namespace NYdb::NYq; using namespace NActors; -void UpsertToExistingTable(TDriver& driver, const TString& location){ - Y_UNUSED(location); - const TString tablePrefix = "Root/yq"; +void UpsertToExistingTable(TDriver& driver, const TString& location){ + Y_UNUSED(location); + const TString tablePrefix = "Root/yq"; NYdb::NTable::TClientSettings settings; NYdb::NTable::TTableClient client(driver, settings); auto sessionOp = client.CreateSession().ExtractValueSync(); @@ -36,7 +36,7 @@ void UpsertToExistingTable(TDriver& driver, const TString& location){ auto timeProvider = CreateDefaultTimeProvider(); auto now = timeProvider->Now(); NYdb::TParamsBuilder paramsBuilder; - + paramsBuilder.AddParam("$now").Timestamp(now).Build(); auto params = paramsBuilder.Build(); @@ -90,4 +90,4 @@ void UpsertToExistingTable(TDriver& driver, const TString& location){ } } } - + diff --git a/ydb/services/yq/ut_integration/ut_utils.h b/ydb/services/yq/ut_integration/ut_utils.h index 08229a279c..a627c45929 100644 --- a/ydb/services/yq/ut_integration/ut_utils.h +++ b/ydb/services/yq/ut_integration/ut_utils.h @@ -3,7 +3,7 @@ #include <ydb/public/sdk/cpp/client/ydb_driver/driver.h> #include <util/system/shellcommand.h> #include <contrib/libs/curl/include/curl/curl.h> -#include <library/cpp/json/json_reader.h> +#include <library/cpp/json/json_reader.h> -void UpsertToExistingTable(NYdb::TDriver& driver, const TString& location); +void UpsertToExistingTable(NYdb::TDriver& driver, const TString& location); diff --git a/ydb/services/yq/ut_integration/ya.make b/ydb/services/yq/ut_integration/ya.make index fbbe452cf4..bc62df4563 100644 --- a/ydb/services/yq/ut_integration/ya.make +++ b/ydb/services/yq/ut_integration/ya.make @@ -17,8 +17,8 @@ SRCS( PEERDIR( library/cpp/getopt - library/cpp/grpc/client - library/cpp/regex/pcre + library/cpp/grpc/client + library/cpp/regex/pcre library/cpp/svnversion ydb/core/testlib ydb/core/yq/libs/control_plane_storage diff --git a/ydb/services/yq/ut_integration/yq_ut.cpp b/ydb/services/yq/ut_integration/yq_ut.cpp index 3e43d91c65..d4a35f195e 100644 --- a/ydb/services/yq/ut_integration/yq_ut.cpp +++ b/ydb/services/yq/ut_integration/yq_ut.cpp @@ -14,221 +14,221 @@ #include <library/cpp/protobuf/util/pb_io.h> #include <library/cpp/retry/retry.h> -#include <util/system/mutex.h> +#include <util/system/mutex.h> #include "ut_utils.h" -#include <google/protobuf/util/time_util.h> +#include <google/protobuf/util/time_util.h> #include <ydb/public/lib/yq/scope.h> -#include <util/system/hostname.h> - -#include <util/string/split.h> - +#include <util/system/hostname.h> + +#include <util/string/split.h> + using namespace NYdb; using namespace YandexQuery;; using namespace NYdb::NYq; - -namespace { - const ui32 Retries = 10; - - void PrintProtoIssues(const NProtoBuf::RepeatedPtrField<::Ydb::Issue::IssueMessage>& protoIssues) { - if (protoIssues.empty()) { - Cerr << "No Issues" << Endl; - return; - } - NYql::TIssues issues; - NYql::IssuesFromMessage(protoIssues, issues); - Cerr << ">>> Issues: " << issues.ToString() << Endl; - } - + +namespace { + const ui32 Retries = 10; + + void PrintProtoIssues(const NProtoBuf::RepeatedPtrField<::Ydb::Issue::IssueMessage>& protoIssues) { + if (protoIssues.empty()) { + Cerr << "No Issues" << Endl; + return; + } + NYql::TIssues issues; + NYql::IssuesFromMessage(protoIssues, issues); + Cerr << ">>> Issues: " << issues.ToString() << Endl; + } + TString CreateNewHistoryAndWaitFinish(const TString& folderId, - NYdb::NYq::TClient& client, const TString& yqlText, - const YandexQuery::QueryMeta::ComputeStatus& expectedStatusResult) + NYdb::NYq::TClient& client, const TString& yqlText, + const YandexQuery::QueryMeta::ComputeStatus& expectedStatusResult) { - //CreateQuery - TString queryId; - { - auto request = ::NYq::TCreateQueryBuilder{} - .SetText(yqlText) - .Build(); - auto result = client.CreateQuery( + //CreateQuery + TString queryId; + { + auto request = ::NYq::TCreateQueryBuilder{} + .SetText(yqlText) + .Build(); + auto result = client.CreateQuery( request, CreateYqSettings<TCreateQuerySettings>(folderId)) - .ExtractValueSync(); + .ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - queryId = result.GetResult().query_id(); - } + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + queryId = result.GetResult().query_id(); + } // GetQueryStatus const auto request = ::NYq::TGetQueryStatusBuilder{} - .SetQueryId(queryId) - .Build(); - const auto result = DoWithRetryOnRetCode([&]() { + .SetQueryId(queryId) + .Build(); + const auto result = DoWithRetryOnRetCode([&]() { auto result = client.GetQueryStatus( request, CreateYqSettings<TGetQueryStatusSettings>(folderId)) - .ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + .ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); return result.GetResult().status() == expectedStatusResult; - }, TRetryOptions(Retries)); - UNIT_ASSERT_C(result, "the execution of the query did not end within the time limit"); - - return queryId; - } - - void CheckGetResultData( - NYdb::NYq::TClient& client, - const TString& queryId, - const TString& folderId, - const ui64 rowsCount, - const int colsSize, - const int answer) - { - const auto request = ::NYq::TGetResultDataBuilder{} - .SetQueryId(queryId) - .Build(); - const auto result = client.GetResultData( + }, TRetryOptions(Retries)); + UNIT_ASSERT_C(result, "the execution of the query did not end within the time limit"); + + return queryId; + } + + void CheckGetResultData( + NYdb::NYq::TClient& client, + const TString& queryId, + const TString& folderId, + const ui64 rowsCount, + const int colsSize, + const int answer) + { + const auto request = ::NYq::TGetResultDataBuilder{} + .SetQueryId(queryId) + .Build(); + const auto result = client.GetResultData( request, CreateYqSettings<TGetResultDataSettings>(folderId)) - .ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - - const auto& resultSet = result.GetResult().result_set(); - UNIT_ASSERT_VALUES_EQUAL(resultSet.rows().size(), rowsCount); - UNIT_ASSERT_VALUES_EQUAL(resultSet.columns().size(), colsSize); - if (!resultSet.rows().empty()) { - const auto& item = resultSet.rows(0).items(0); - TString str = item.DebugString(); - TVector<TString> arr; - StringSplitter(str).Split(' ').SkipEmpty().AddTo(&arr); - Y_VERIFY(arr.size() == 2, "Incorrect numeric result"); - UNIT_ASSERT_VALUES_EQUAL(std::stoi(arr.back()), answer); - } - } -} - + .ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + + const auto& resultSet = result.GetResult().result_set(); + UNIT_ASSERT_VALUES_EQUAL(resultSet.rows().size(), rowsCount); + UNIT_ASSERT_VALUES_EQUAL(resultSet.columns().size(), colsSize); + if (!resultSet.rows().empty()) { + const auto& item = resultSet.rows(0).items(0); + TString str = item.DebugString(); + TVector<TString> arr; + StringSplitter(str).Split(' ').SkipEmpty().AddTo(&arr); + Y_VERIFY(arr.size() == 2, "Incorrect numeric result"); + UNIT_ASSERT_VALUES_EQUAL(std::stoi(arr.back()), answer); + } + } +} + Y_UNIT_TEST_SUITE(Yq_1) { Y_UNIT_TEST(Basic) { TKikimrWithGrpcAndRootSchema server({}, {}, {}, true); - ui16 grpc = server.GetPort(); + ui16 grpc = server.GetPort(); TString location = TStringBuilder() << "localhost:" << grpc; - auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken("root@builtin")); - NYdb::NYq::TClient client(driver); - const auto folderId = "some_folder_id"; + auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken("root@builtin")); + NYdb::NYq::TClient client(driver); + const auto folderId = "some_folder_id"; const auto queryId = CreateNewHistoryAndWaitFinish(folderId, client, "select 1", YandexQuery::QueryMeta::COMPLETED); CheckGetResultData(client, queryId, folderId, 1, 1, 1); { - const auto request = ::NYq::TDescribeQueryBuilder() - .SetQueryId("foo") - .Build(); - const auto result = DoWithRetryOnRetCode([&]() { - auto result = client.DescribeQuery( + const auto request = ::NYq::TDescribeQueryBuilder() + .SetQueryId("foo") + .Build(); + const auto result = DoWithRetryOnRetCode([&]() { + auto result = client.DescribeQuery( request, CreateYqSettings<TDescribeQuerySettings>("WTF")) - .ExtractValueSync(); - return result.GetStatus() == EStatus::BAD_REQUEST; - }, TRetryOptions(Retries)); - UNIT_ASSERT_C(result, "the execution of the query did not end within the time limit"); + .ExtractValueSync(); + return result.GetStatus() == EStatus::BAD_REQUEST; + }, TRetryOptions(Retries)); + UNIT_ASSERT_C(result, "the execution of the query did not end within the time limit"); } { - auto request = ::NYq::TListQueriesBuilder{}.Build(); - auto result = DoWithRetryOnRetCode([&]() { - auto result = client.ListQueries( + auto request = ::NYq::TListQueriesBuilder{}.Build(); + auto result = DoWithRetryOnRetCode([&]() { + auto result = client.ListQueries( request, CreateYqSettings<TListQueriesSettings>("WTF")) - .ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - UNIT_ASSERT_VALUES_EQUAL(result.GetResult().query().size(), 0); - return result.GetStatus() == EStatus::SUCCESS; - }, TRetryOptions(Retries)); - UNIT_ASSERT_C(result, "the execution of the query did not end within the time limit"); - } - - { - const auto request = ::NYq::TDescribeQueryBuilder() - .SetQueryId(queryId) - .Build(); - auto result = client.DescribeQuery( + .ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + UNIT_ASSERT_VALUES_EQUAL(result.GetResult().query().size(), 0); + return result.GetStatus() == EStatus::SUCCESS; + }, TRetryOptions(Retries)); + UNIT_ASSERT_C(result, "the execution of the query did not end within the time limit"); + } + + { + const auto request = ::NYq::TDescribeQueryBuilder() + .SetQueryId(queryId) + .Build(); + auto result = client.DescribeQuery( request, CreateYqSettings<TDescribeQuerySettings>(folderId)) - .ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - const auto status = result.GetResult().query().meta().status(); - UNIT_ASSERT(status == YandexQuery::QueryMeta::COMPLETED); + .ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + const auto status = result.GetResult().query().meta().status(); + UNIT_ASSERT(status == YandexQuery::QueryMeta::COMPLETED); } { - const auto request = ::NYq::TModifyQueryBuilder() - .SetQueryId(queryId) - .SetName("MODIFIED_NAME") - .Build(); - const auto result = client.ModifyQuery( + const auto request = ::NYq::TModifyQueryBuilder() + .SetQueryId(queryId) + .SetName("MODIFIED_NAME") + .Build(); + const auto result = client.ModifyQuery( request, CreateYqSettings<TModifyQuerySettings>(folderId)) - .ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + .ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); } { - const auto request = ::NYq::TDescribeQueryBuilder() - .SetQueryId(queryId) - .Build(); - const auto result = client.DescribeQuery( + const auto request = ::NYq::TDescribeQueryBuilder() + .SetQueryId(queryId) + .Build(); + const auto result = client.DescribeQuery( request, CreateYqSettings<TDescribeQuerySettings>(folderId)) - .ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - const auto& res = result.GetResult(); - UNIT_ASSERT_VALUES_EQUAL(res.query().content().name(), "MODIFIED_NAME"); - UNIT_ASSERT(res.query().content().acl().visibility() == static_cast<int>(Acl_Visibility_SCOPE)); + .ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + const auto& res = result.GetResult(); + UNIT_ASSERT_VALUES_EQUAL(res.query().content().name(), "MODIFIED_NAME"); + UNIT_ASSERT(res.query().content().acl().visibility() == static_cast<int>(Acl_Visibility_SCOPE)); } { - const auto request = ::NYq::TDescribeQueryBuilder() - .SetQueryId("") - .Build(); - const auto result = client.DescribeQuery( + const auto request = ::NYq::TDescribeQueryBuilder() + .SetQueryId("") + .Build(); + const auto result = client.DescribeQuery( request, CreateYqSettings<TDescribeQuerySettings>("")) - .ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::BAD_REQUEST, result.GetIssues().ToString()); + .ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::BAD_REQUEST, result.GetIssues().ToString()); } { - const auto request = ::NYq::TGetResultDataBuilder() - .SetQueryId("") - .Build(); - const auto result = client.GetResultData( + const auto request = ::NYq::TGetResultDataBuilder() + .SetQueryId("") + .Build(); + const auto result = client.GetResultData( request, CreateYqSettings<TGetResultDataSettings>("")) - .ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::BAD_REQUEST, result.GetIssues().ToString()); + .ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::BAD_REQUEST, result.GetIssues().ToString()); } } Y_UNIT_TEST(Basic_EmptyTable) { TKikimrWithGrpcAndRootSchema server({}, {}, {}, true); - ui16 grpc = server.GetPort(); + ui16 grpc = server.GetPort(); TString location = TStringBuilder() << "localhost:" << grpc; - auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken("root@builtin")); + auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken("root@builtin")); UpsertToExistingTable(driver, location); - NYdb::NYq::TClient client(driver); + NYdb::NYq::TClient client(driver); const TString folderId = "some_folder_id"; { - const auto request = ::NYq::TCreateConnectionBuilder() - .SetName("testdbempty") - .CreateYdb("Root", location, "") - .Build(); - const auto result = client + const auto request = ::NYq::TCreateConnectionBuilder() + .SetName("testdbempty") + .CreateYdb("Root", location, "") + .Build(); + const auto result = client .CreateConnection(request, CreateYqSettings<TCreateConnectionSettings>(folderId)) - .ExtractValueSync(); - UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString()); + .ExtractValueSync(); + UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString()); } - const TString queryId = CreateNewHistoryAndWaitFinish( + const TString queryId = CreateNewHistoryAndWaitFinish( folderId, client, - "select count(*) from testdbempty.`yq/empty_table`", - YandexQuery::QueryMeta::COMPLETED); + "select count(*) from testdbempty.`yq/empty_table`", + YandexQuery::QueryMeta::COMPLETED); CheckGetResultData(client, queryId, folderId, 1, 1, 0); } Y_UNIT_TEST(Basic_EmptyList) { TKikimrWithGrpcAndRootSchema server({}, {}, {}, true); - ui16 grpc = server.GetPort(); + ui16 grpc = server.GetPort(); TString location = TStringBuilder() << "localhost:" << grpc; - auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken("root@builtin")); - NYdb::NYq::TClient client(driver); + auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken("root@builtin")); + NYdb::NYq::TClient client(driver); const TString folderId = "some_folder_id"; auto expectedStatus = YandexQuery::QueryMeta::COMPLETED; CreateNewHistoryAndWaitFinish(folderId, client, "select []", expectedStatus); @@ -256,7 +256,7 @@ Y_UNIT_TEST_SUITE(Yq_1) { CreateNewHistoryAndWaitFinish(folderId, client, "select null", expectedStatus); } - SIMPLE_UNIT_FORKED_TEST(Basic_Tagged) { + SIMPLE_UNIT_FORKED_TEST(Basic_Tagged) { TKikimrWithGrpcAndRootSchema server({}, {}, {}, true); ui16 grpc = server.GetPort(); TString location = TStringBuilder() << "localhost:" << grpc; @@ -295,104 +295,104 @@ Y_UNIT_TEST_SUITE(Yq_1) { } // use fork for data test due to ch initialization problem - SIMPLE_UNIT_FORKED_TEST(ExtendedDatabaseId) { + SIMPLE_UNIT_FORKED_TEST(ExtendedDatabaseId) { TKikimrWithGrpcAndRootSchema server({}, {}, {}, true); - ui16 grpc = server.GetPort(); + ui16 grpc = server.GetPort(); TString location = TStringBuilder() << "localhost:" << grpc; - auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken("root@builtin")); + auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken("root@builtin")); - NYdb::NYq::TClient client(driver); + NYdb::NYq::TClient client(driver); const TString folderId = "folder_id_" + CreateGuidAsString(); { - const auto request = ::NYq::TCreateConnectionBuilder() - .SetName("testdb01") - .CreateYdb("FakeDatabaseId", "") - .Build(); - const auto result = client + const auto request = ::NYq::TCreateConnectionBuilder() + .SetName("testdb01") + .CreateYdb("FakeDatabaseId", "") + .Build(); + const auto result = client .CreateConnection(request, CreateYqSettings<TCreateConnectionSettings>(folderId)) - .ExtractValueSync(); - UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString()); + .ExtractValueSync(); + UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString()); } { - const auto request = ::NYq::TCreateConnectionBuilder() - .SetName("testdb02") - .CreateYdb("FakeDatabaseId", "") - .Build(); - const auto result = client + const auto request = ::NYq::TCreateConnectionBuilder() + .SetName("testdb02") + .CreateYdb("FakeDatabaseId", "") + .Build(); + const auto result = client .CreateConnection(request, CreateYqSettings<TCreateConnectionSettings>(folderId)) - .ExtractValueSync(); - UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString()); + .ExtractValueSync(); + UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString()); } - { + { const auto queryId = CreateNewHistoryAndWaitFinish(folderId, client, - "select count(*) from testdb01.`yq/connections`", YandexQuery::QueryMeta::COMPLETED); + "select count(*) from testdb01.`yq/connections`", YandexQuery::QueryMeta::COMPLETED); CheckGetResultData(client, queryId, folderId, 1, 1, 2); - } - - { - // test connections db with 2 databaseId + } + + { + // test connections db with 2 databaseId const auto queryId = CreateNewHistoryAndWaitFinish(folderId, client, - "select count(*) from testdb02.`yq/connections`", YandexQuery::QueryMeta::COMPLETED); + "select count(*) from testdb02.`yq/connections`", YandexQuery::QueryMeta::COMPLETED); CheckGetResultData(client, queryId, folderId, 1, 1, 2); - } + } } - Y_UNIT_TEST(DescribeConnection) { + Y_UNIT_TEST(DescribeConnection) { TKikimrWithGrpcAndRootSchema server({}, {}, {}, true); - ui16 grpc = server.GetPort(); + ui16 grpc = server.GetPort(); TString location = TStringBuilder() << "localhost:" << grpc; - auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken("root@builtin")); - NYdb::NYq::TClient client(driver); - const TString folderId = "some_folder_id"; - TString conId; - { - const auto request = ::NYq::TCreateConnectionBuilder() - .SetName("created_conn") - .CreateYdb("created_db", "") - .Build(); - const auto result = client + auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken("root@builtin")); + NYdb::NYq::TClient client(driver); + const TString folderId = "some_folder_id"; + TString conId; + { + const auto request = ::NYq::TCreateConnectionBuilder() + .SetName("created_conn") + .CreateYdb("created_db", "") + .Build(); + const auto result = client .CreateConnection(request, CreateYqSettings<TCreateConnectionSettings>(folderId)) - .ExtractValueSync(); - UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString()); - conId = result.GetResult().connection_id(); - } - { - auto request = ::NYq::TDescribeConnectionBuilder() - .SetConnectionId(conId) - .Build(); - auto result = client + .ExtractValueSync(); + UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString()); + conId = result.GetResult().connection_id(); + } + { + auto request = ::NYq::TDescribeConnectionBuilder() + .SetConnectionId(conId) + .Build(); + auto result = client .DescribeConnection(request, CreateYqSettings<TDescribeConnectionSettings>(folderId)) - .ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - const auto& res = result.GetResult().connection(); - UNIT_ASSERT_VALUES_EQUAL(res.meta().id(), conId); - UNIT_ASSERT_VALUES_EQUAL(res.meta().created_by(), "root@builtin"); - UNIT_ASSERT_VALUES_EQUAL(res.meta().modified_by(), "root@builtin"); - UNIT_ASSERT_VALUES_EQUAL(res.content().name(), "created_conn"); - } - } - - Y_UNIT_TEST(ListConnections) { + .ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + const auto& res = result.GetResult().connection(); + UNIT_ASSERT_VALUES_EQUAL(res.meta().id(), conId); + UNIT_ASSERT_VALUES_EQUAL(res.meta().created_by(), "root@builtin"); + UNIT_ASSERT_VALUES_EQUAL(res.meta().modified_by(), "root@builtin"); + UNIT_ASSERT_VALUES_EQUAL(res.content().name(), "created_conn"); + } + } + + Y_UNIT_TEST(ListConnections) { TKikimrWithGrpcAndRootSchema server({}, {}, {}, true); - ui16 grpc = server.GetPort(); - TString location = TStringBuilder() << "localhost:" << grpc; - auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken("root@builtin")); - NYdb::NYq::TClient client(driver); + ui16 grpc = server.GetPort(); + TString location = TStringBuilder() << "localhost:" << grpc; + auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken("root@builtin")); + NYdb::NYq::TClient client(driver); const size_t conns = 3; - const auto folderId = TString(__func__) + "folder_id"; - {//CreateConnections + const auto folderId = TString(__func__) + "folder_id"; + {//CreateConnections for (size_t i = 0; i < conns - 1; ++i) { - const auto request = ::NYq::TCreateConnectionBuilder() - .SetName("testdb" + ToString(i)) - .CreateYdb("FakeDatabaseId", "") - .Build(); - const auto result = client + const auto request = ::NYq::TCreateConnectionBuilder() + .SetName("testdb" + ToString(i)) + .CreateYdb("FakeDatabaseId", "") + .Build(); + const auto result = client .CreateConnection(request, CreateYqSettings<TCreateConnectionSettings>(folderId)) - .ExtractValueSync(); - UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString()); - } + .ExtractValueSync(); + UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString()); + } // yds const auto request = ::NYq::TCreateConnectionBuilder() @@ -403,391 +403,391 @@ Y_UNIT_TEST_SUITE(Yq_1) { .CreateConnection(request, CreateYqSettings<TCreateConnectionSettings>(folderId)) .ExtractValueSync(); UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString()); - } - + } + { - const auto request = ::NYq::TListConnectionsBuilder().Build(); - auto result = client + const auto request = ::NYq::TListConnectionsBuilder().Build(); + auto result = client .ListConnections(request, CreateYqSettings<TListConnectionsSettings>(folderId)) - .ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - UNIT_ASSERT_VALUES_EQUAL(result.GetResult().connection().size(), conns); - size_t i = 0; - auto res = result.GetResult(); - auto* conns = res.mutable_connection(); - std::sort(conns->begin(), conns->end(), [&](const auto& lhs, const auto& rhs) { - return lhs.content().name() < rhs.content().name(); - }); - for (const auto& conn : *conns) { - const auto& content = conn.content(); - const auto& meta = conn.meta(); - UNIT_ASSERT_VALUES_EQUAL(content.name(), "testdb" + ToString(i)); - UNIT_ASSERT_VALUES_EQUAL(meta.created_by(), "root@builtin"); - UNIT_ASSERT_VALUES_EQUAL(meta.modified_by(), "root@builtin"); + .ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + UNIT_ASSERT_VALUES_EQUAL(result.GetResult().connection().size(), conns); + size_t i = 0; + auto res = result.GetResult(); + auto* conns = res.mutable_connection(); + std::sort(conns->begin(), conns->end(), [&](const auto& lhs, const auto& rhs) { + return lhs.content().name() < rhs.content().name(); + }); + for (const auto& conn : *conns) { + const auto& content = conn.content(); + const auto& meta = conn.meta(); + UNIT_ASSERT_VALUES_EQUAL(content.name(), "testdb" + ToString(i)); + UNIT_ASSERT_VALUES_EQUAL(meta.created_by(), "root@builtin"); + UNIT_ASSERT_VALUES_EQUAL(meta.modified_by(), "root@builtin"); if (i < 2) { UNIT_ASSERT_C(content.setting().has_ydb_database(), content); } else { UNIT_ASSERT_C(content.setting().has_data_streams(), content); } - i++; - } + i++; + } } } - - Y_UNIT_TEST(ListConnectionsOnEmptyConnectionsTable) { + + Y_UNIT_TEST(ListConnectionsOnEmptyConnectionsTable) { TKikimrWithGrpcAndRootSchema server({}, {}, {}, true); - ui16 grpc = server.GetPort(); - TString location = TStringBuilder() << "localhost:" << grpc; - auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken("root@builtin")); - NYdb::NYq::TClient client(driver); - - { - const auto request = ::NYq::TListConnectionsBuilder().Build(); - auto result = client + ui16 grpc = server.GetPort(); + TString location = TStringBuilder() << "localhost:" << grpc; + auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken("root@builtin")); + NYdb::NYq::TClient client(driver); + + { + const auto request = ::NYq::TListConnectionsBuilder().Build(); + auto result = client .ListConnections(request, CreateYqSettings<TListConnectionsSettings>("WTF")) - .ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - UNIT_ASSERT(result.GetResult().connection().empty()); - } - } - - Y_UNIT_TEST(ModifyConnections) { + .ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + UNIT_ASSERT(result.GetResult().connection().empty()); + } + } + + Y_UNIT_TEST(ModifyConnections) { TKikimrWithGrpcAndRootSchema server({}, {}, {}, true); - ui16 grpc = server.GetPort(); - TString location = TStringBuilder() << "localhost:" << grpc; - TString userToken = "root@builtin"; - TString userId = "root"; - auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken(userToken)); - NYdb::NYq::TClient client(driver); - const auto folderId = TString(__func__) + "folder_id"; - TString conId; - - { - const auto request = ::NYq::TCreateConnectionBuilder() - .SetName("created_conn") - .CreateYdb("created_db", "") - .Build(); - const auto result = client + ui16 grpc = server.GetPort(); + TString location = TStringBuilder() << "localhost:" << grpc; + TString userToken = "root@builtin"; + TString userId = "root"; + auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken(userToken)); + NYdb::NYq::TClient client(driver); + const auto folderId = TString(__func__) + "folder_id"; + TString conId; + + { + const auto request = ::NYq::TCreateConnectionBuilder() + .SetName("created_conn") + .CreateYdb("created_db", "") + .Build(); + const auto result = client .CreateConnection(request, CreateYqSettings<TCreateConnectionSettings>(folderId)) - .ExtractValueSync(); - UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString()); - conId = result.GetResult().connection_id(); - } - - {//Modify - const auto request = ::NYq::TModifyConnectionBuilder() - .SetName("modified_name") - .SetConnectionId(conId) - .CreateYdb("new ydb", "") - .SetDescription("Modified") - .Build(); - const auto result = client + .ExtractValueSync(); + UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString()); + conId = result.GetResult().connection_id(); + } + + {//Modify + const auto request = ::NYq::TModifyConnectionBuilder() + .SetName("modified_name") + .SetConnectionId(conId) + .CreateYdb("new ydb", "") + .SetDescription("Modified") + .Build(); + const auto result = client .ModifyConnection(request, CreateYqSettings<TModifyConnectionSettings>(folderId)) - .ExtractValueSync(); - UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString()); - } - - { - auto request = ::NYq::TDescribeConnectionBuilder() - .SetConnectionId(conId) - .Build(); - auto result = client + .ExtractValueSync(); + UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { + auto request = ::NYq::TDescribeConnectionBuilder() + .SetConnectionId(conId) + .Build(); + auto result = client .DescribeConnection(request, CreateYqSettings<TDescribeConnectionSettings>(folderId)) - .ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - const auto& res = result.GetResult().connection(); - UNIT_ASSERT_VALUES_EQUAL(res.meta().id(), conId); - UNIT_ASSERT_VALUES_EQUAL(res.meta().created_by(), "root@builtin"); - UNIT_ASSERT_VALUES_EQUAL(res.meta().modified_by(), "root@builtin"); - UNIT_ASSERT_VALUES_EQUAL(res.content().name(), "modified_name"); - UNIT_ASSERT_VALUES_EQUAL(res.content().description(), "Modified"); - } - } - - Y_UNIT_TEST(DeleteConnections) { + .ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + const auto& res = result.GetResult().connection(); + UNIT_ASSERT_VALUES_EQUAL(res.meta().id(), conId); + UNIT_ASSERT_VALUES_EQUAL(res.meta().created_by(), "root@builtin"); + UNIT_ASSERT_VALUES_EQUAL(res.meta().modified_by(), "root@builtin"); + UNIT_ASSERT_VALUES_EQUAL(res.content().name(), "modified_name"); + UNIT_ASSERT_VALUES_EQUAL(res.content().description(), "Modified"); + } + } + + Y_UNIT_TEST(DeleteConnections) { TKikimrWithGrpcAndRootSchema server({}, {}, {}, true); - ui16 grpc = server.GetPort(); - TString location = TStringBuilder() << "localhost:" << grpc; - auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken("root@builtin")); - NYdb::NYq::TClient client(driver); - const auto folderId = TString(__func__) + "folder_id"; - TString conId; - - { - const auto request = ::NYq::TCreateConnectionBuilder() - .SetName("created_conn") - .CreateYdb("created_db", "") - .Build(); - const auto result = client + ui16 grpc = server.GetPort(); + TString location = TStringBuilder() << "localhost:" << grpc; + auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken("root@builtin")); + NYdb::NYq::TClient client(driver); + const auto folderId = TString(__func__) + "folder_id"; + TString conId; + + { + const auto request = ::NYq::TCreateConnectionBuilder() + .SetName("created_conn") + .CreateYdb("created_db", "") + .Build(); + const auto result = client .CreateConnection(request, CreateYqSettings<TCreateConnectionSettings>(folderId)) - .ExtractValueSync(); - UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString()); - conId = result.GetResult().connection_id(); - } - - { - const auto request = ::NYq::TDeleteConnectionBuilder() - .SetConnectionId(conId) - .Build(); - - const auto result = client + .ExtractValueSync(); + UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString()); + conId = result.GetResult().connection_id(); + } + + { + const auto request = ::NYq::TDeleteConnectionBuilder() + .SetConnectionId(conId) + .Build(); + + const auto result = client .DeleteConnection(request, CreateYqSettings<TDeleteConnectionSettings>(folderId)) - .ExtractValueSync(); - UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString()); - } - } - - Y_UNIT_TEST(Create_And_Modify_The_Same_Connection) { + .ExtractValueSync(); + UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString()); + } + } + + Y_UNIT_TEST(Create_And_Modify_The_Same_Connection) { TKikimrWithGrpcAndRootSchema server({}, {}, {}, true); - ui16 grpc = server.GetPort(); - TString location = TStringBuilder() << "localhost:" << grpc; - TString userToken = "root@builtin"; + ui16 grpc = server.GetPort(); + TString location = TStringBuilder() << "localhost:" << grpc; + TString userToken = "root@builtin"; auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken(userToken)); - NYdb::NYq::TClient client(driver); + NYdb::NYq::TClient client(driver); - const auto folderId = TString(__func__) + "folder_id"; - TString conId; + const auto folderId = TString(__func__) + "folder_id"; + TString conId; - { - const auto request = ::NYq::TCreateConnectionBuilder() - .SetName("created_conn") - .CreateYdb("created_db", "") - .Build(); - const auto result = client + { + const auto request = ::NYq::TCreateConnectionBuilder() + .SetName("created_conn") + .CreateYdb("created_db", "") + .Build(); + const auto result = client .CreateConnection(request, CreateYqSettings<TCreateConnectionSettings>(folderId)) - .ExtractValueSync(); - UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString()); - conId = result.GetResult().connection_id(); - } - - { - const auto request = ::NYq::TModifyConnectionBuilder() - .SetConnectionId(conId) - .CreateYdb("modified_db", "")//TODO remove - .Build(); - const auto result = client + .ExtractValueSync(); + UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString()); + conId = result.GetResult().connection_id(); + } + + { + const auto request = ::NYq::TModifyConnectionBuilder() + .SetConnectionId(conId) + .CreateYdb("modified_db", "")//TODO remove + .Build(); + const auto result = client .ModifyConnection(request, CreateYqSettings<TModifyConnectionSettings>(folderId)) - .ExtractValueSync(); - UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString()); - } - } - - Y_UNIT_TEST(CreateConnection_With_Existing_Name) { + .ExtractValueSync(); + UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString()); + } + } + + Y_UNIT_TEST(CreateConnection_With_Existing_Name) { TKikimrWithGrpcAndRootSchema server({}, {}, {}, true); - ui16 grpc = server.GetPort(); - TString location = TStringBuilder() << "localhost:" << grpc; - TString userToken = "root@builtin"; - auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken(userToken)); - NYdb::NYq::TClient client(driver); - - const auto folderId = TString(__func__) + "folder_id"; + ui16 grpc = server.GetPort(); + TString location = TStringBuilder() << "localhost:" << grpc; + TString userToken = "root@builtin"; + auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken(userToken)); + NYdb::NYq::TClient client(driver); + + const auto folderId = TString(__func__) + "folder_id"; auto name = TString(__func__) + "_name"; name.to_lower(); - - { - const auto request = ::NYq::TCreateConnectionBuilder() - .SetName(name) - .CreateYdb("created_db", "") - .Build(); - const auto result = client + + { + const auto request = ::NYq::TCreateConnectionBuilder() + .SetName(name) + .CreateYdb("created_db", "") + .Build(); + const auto result = client .CreateConnection(request, CreateYqSettings<TCreateConnectionSettings>(folderId)) - .ExtractValueSync(); - UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString()); - } - - { - const auto request = ::NYq::TCreateConnectionBuilder() - .SetName(name) - .CreateYdb("created_db", "") - .Build(); - const auto result = client + .ExtractValueSync(); + UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { + const auto request = ::NYq::TCreateConnectionBuilder() + .SetName(name) + .CreateYdb("created_db", "") + .Build(); + const auto result = client .CreateConnection(request, CreateYqSettings<TCreateConnectionSettings>(folderId)) - .ExtractValueSync(); - UNIT_ASSERT_C(result.GetStatus() == EStatus::BAD_REQUEST, result.GetIssues().ToString()); //TODO status should be ALREADY_EXISTS - } - } - - Y_UNIT_TEST(CreateConnections_With_Idempotency) { + .ExtractValueSync(); + UNIT_ASSERT_C(result.GetStatus() == EStatus::BAD_REQUEST, result.GetIssues().ToString()); //TODO status should be ALREADY_EXISTS + } + } + + Y_UNIT_TEST(CreateConnections_With_Idempotency) { TKikimrWithGrpcAndRootSchema server({}, {}, {}, true); - ui16 grpc = server.GetPort(); - TString location = TStringBuilder() << "localhost:" << grpc; - TString userToken = "root@builtin"; - auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken(userToken)); - NYdb::NYq::TClient client(driver); - - const auto folderId = TString(__func__) + "folder_id"; - const auto name = "connection_name"; - const TString idempotencyKey = "idempotency_key"; - TString conId; - - { - const auto request = ::NYq::TCreateConnectionBuilder() - .SetName(name) - .SetIdempotencyKey(idempotencyKey) - .CreateYdb("created_db", "") - .Build(); - const auto result = client + ui16 grpc = server.GetPort(); + TString location = TStringBuilder() << "localhost:" << grpc; + TString userToken = "root@builtin"; + auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken(userToken)); + NYdb::NYq::TClient client(driver); + + const auto folderId = TString(__func__) + "folder_id"; + const auto name = "connection_name"; + const TString idempotencyKey = "idempotency_key"; + TString conId; + + { + const auto request = ::NYq::TCreateConnectionBuilder() + .SetName(name) + .SetIdempotencyKey(idempotencyKey) + .CreateYdb("created_db", "") + .Build(); + const auto result = client .CreateConnection(request, CreateYqSettings<TCreateConnectionSettings>(folderId)) - .ExtractValueSync(); - UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString()); - conId = result.GetResult().connection_id(); - } - - { - const auto request = ::NYq::TCreateConnectionBuilder() - .SetName(name) - .SetIdempotencyKey(idempotencyKey) - .CreateYdb("created_db", "") - .Build(); - const auto result = client + .ExtractValueSync(); + UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString()); + conId = result.GetResult().connection_id(); + } + + { + const auto request = ::NYq::TCreateConnectionBuilder() + .SetName(name) + .SetIdempotencyKey(idempotencyKey) + .CreateYdb("created_db", "") + .Build(); + const auto result = client .CreateConnection(request, CreateYqSettings<TCreateConnectionSettings>(folderId)) - .ExtractValueSync(); - UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString()); - UNIT_ASSERT_VALUES_EQUAL(conId, result.GetResult().connection_id()); - } - } - - Y_UNIT_TEST(CreateQuery_With_Idempotency) {//TODO Fix + .ExtractValueSync(); + UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString()); + UNIT_ASSERT_VALUES_EQUAL(conId, result.GetResult().connection_id()); + } + } + + Y_UNIT_TEST(CreateQuery_With_Idempotency) {//TODO Fix TKikimrWithGrpcAndRootSchema server({}, {}, {}, true); - ui16 grpc = server.GetPort(); + ui16 grpc = server.GetPort(); TString location = TStringBuilder() << "localhost:" << grpc; - TString userToken = "root@builtin"; + TString userToken = "root@builtin"; auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken(userToken)); - NYdb::NYq::TClient client(driver); + NYdb::NYq::TClient client(driver); - const auto folderId = TString(__func__) + "folder_id"; + const auto folderId = TString(__func__) + "folder_id"; const TString idempotencyKey = "idempotency_key"; - const TString yqlText = "select 1"; - TString queryId; - const auto request = ::NYq::TCreateQueryBuilder{} - .SetText(yqlText) - .SetIdempotencyKey(idempotencyKey) - .Build(); + const TString yqlText = "select 1"; + TString queryId; + const auto request = ::NYq::TCreateQueryBuilder{} + .SetText(yqlText) + .SetIdempotencyKey(idempotencyKey) + .Build(); { - auto result = client.CreateQuery( + auto result = client.CreateQuery( request, CreateYqSettings<TCreateQuerySettings>(folderId)) - .ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - queryId = result.GetResult().query_id(); + .ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + queryId = result.GetResult().query_id(); } { - const auto req = ::NYq::TDescribeQueryBuilder{} - .SetQueryId(queryId) - .Build(); - const auto result = DoWithRetryOnRetCode([&]() { - auto result = client.DescribeQuery( + const auto req = ::NYq::TDescribeQueryBuilder{} + .SetQueryId(queryId) + .Build(); + const auto result = DoWithRetryOnRetCode([&]() { + auto result = client.DescribeQuery( req, CreateYqSettings<TDescribeQuerySettings>(folderId)) - .ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - const auto status = result.GetResult().query().meta().status(); - PrintProtoIssues(result.GetResult().query().issue()); - return status == YandexQuery::QueryMeta::COMPLETED; - }, TRetryOptions(Retries)); - UNIT_ASSERT_C(result, "the execution of the query did not end within the time limit"); + .ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + const auto status = result.GetResult().query().meta().status(); + PrintProtoIssues(result.GetResult().query().issue()); + return status == YandexQuery::QueryMeta::COMPLETED; + }, TRetryOptions(Retries)); + UNIT_ASSERT_C(result, "the execution of the query did not end within the time limit"); } { - auto result = client.CreateQuery( + auto result = client.CreateQuery( request, CreateYqSettings<TCreateQuerySettings>(folderId)) - .ExtractValueSync(); + .ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - UNIT_ASSERT_VALUES_EQUAL(queryId, result.GetResult().query_id()); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + UNIT_ASSERT_VALUES_EQUAL(queryId, result.GetResult().query_id()); } CheckGetResultData(client, queryId, folderId, 1, 1, 1); } // use fork for data test due to ch initialization problem - SIMPLE_UNIT_FORKED_TEST(CreateQuery_Without_Connection) { + SIMPLE_UNIT_FORKED_TEST(CreateQuery_Without_Connection) { TKikimrWithGrpcAndRootSchema server({}, {}, {}, true); - ui16 grpc = server.GetPort(); - TString location = TStringBuilder() << "localhost:" << grpc; - TString userToken = "root@builtin"; - auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken(userToken)); - NYdb::NYq::TClient client(driver); - - const TString yqlText = "select count(*) from testdbWTF.`connections`"; + ui16 grpc = server.GetPort(); + TString location = TStringBuilder() << "localhost:" << grpc; + TString userToken = "root@builtin"; + auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken(userToken)); + NYdb::NYq::TClient client(driver); + + const TString yqlText = "select count(*) from testdbWTF.`connections`"; CreateNewHistoryAndWaitFinish("folder_id_WTF", client, - yqlText, YandexQuery::QueryMeta::FAILED); - } - - Y_UNIT_TEST(DeleteQuery) { + yqlText, YandexQuery::QueryMeta::FAILED); + } + + Y_UNIT_TEST(DeleteQuery) { TKikimrWithGrpcAndRootSchema server({}, {}, {}, true); - ui16 grpc = server.GetPort(); - TString location = TStringBuilder() << "localhost:" << grpc; - TString userToken = "root@builtin"; - auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken(userToken)); - NYdb::NYq::TClient client(driver); - - const auto folderId = TString(__func__) + "folder_id"; - const TString yqlText = "select 1"; + ui16 grpc = server.GetPort(); + TString location = TStringBuilder() << "localhost:" << grpc; + TString userToken = "root@builtin"; + auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken(userToken)); + NYdb::NYq::TClient client(driver); + + const auto folderId = TString(__func__) + "folder_id"; + const TString yqlText = "select 1"; const TString queryId = CreateNewHistoryAndWaitFinish(folderId, client, - yqlText, YandexQuery::QueryMeta::COMPLETED); + yqlText, YandexQuery::QueryMeta::COMPLETED); CheckGetResultData(client, queryId, folderId, 1, 1, 1); - - { - const auto request = ::NYq::TDeleteQueryBuilder() - .SetQueryId(queryId) - .Build(); - auto result = client + + { + const auto request = ::NYq::TDeleteQueryBuilder() + .SetQueryId(queryId) + .Build(); + auto result = client .DeleteQuery(request, CreateYqSettings<TDeleteQuerySettings>(folderId)) - .ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - - { - auto request = ::NYq::TDescribeQueryBuilder() - .SetQueryId(queryId) - .Build(); - auto result = client + .ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { + auto request = ::NYq::TDescribeQueryBuilder() + .SetQueryId(queryId) + .Build(); + auto result = client .DescribeQuery(request, CreateYqSettings<TDescribeQuerySettings>(folderId)) - .ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::BAD_REQUEST, result.GetIssues().ToString()); - } - } - - Y_UNIT_TEST(ModifyQuery) { + .ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::BAD_REQUEST, result.GetIssues().ToString()); + } + } + + Y_UNIT_TEST(ModifyQuery) { TKikimrWithGrpcAndRootSchema server({}, {}, {}, true); - ui16 grpc = server.GetPort(); - TString location = TStringBuilder() << "localhost:" << grpc; - TString userToken = "root@builtin"; - auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken(userToken)); - NYdb::NYq::TClient client(driver); - - const auto folderId = TString(__func__) + "folder_id"; - const TString yqlText = "select 1"; + ui16 grpc = server.GetPort(); + TString location = TStringBuilder() << "localhost:" << grpc; + TString userToken = "root@builtin"; + auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken(userToken)); + NYdb::NYq::TClient client(driver); + + const auto folderId = TString(__func__) + "folder_id"; + const TString yqlText = "select 1"; const TString queryId = CreateNewHistoryAndWaitFinish(folderId, client, - yqlText, YandexQuery::QueryMeta::COMPLETED); + yqlText, YandexQuery::QueryMeta::COMPLETED); CheckGetResultData(client, queryId, folderId, 1, 1, 1); - - { - const auto request = ::NYq::TModifyQueryBuilder() - .SetQueryId(queryId) - .SetName("ModifiedName") - .SetDescription("OK") - .Build(); - auto result = client + + { + const auto request = ::NYq::TModifyQueryBuilder() + .SetQueryId(queryId) + .SetName("ModifiedName") + .SetDescription("OK") + .Build(); + auto result = client .ModifyQuery(request, CreateYqSettings<TModifyQuerySettings>(folderId)) - .ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - - { - auto request = ::NYq::TDescribeQueryBuilder() - .SetQueryId(queryId) - .Build(); - auto result = client + .ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { + auto request = ::NYq::TDescribeQueryBuilder() + .SetQueryId(queryId) + .Build(); + auto result = client .DescribeQuery(request, CreateYqSettings<TDescribeQuerySettings>(folderId)) - .ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - const auto& query = result.GetResult().query(); - UNIT_ASSERT_VALUES_EQUAL(query.content().name(), "ModifiedName"); - UNIT_ASSERT_VALUES_EQUAL(query.content().description(), "OK"); - } - } + .ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + const auto& query = result.GetResult().query(); + UNIT_ASSERT_VALUES_EQUAL(query.content().name(), "ModifiedName"); + UNIT_ASSERT_VALUES_EQUAL(query.content().description(), "OK"); + } + } Y_UNIT_TEST(DescribeJob) { TKikimrWithGrpcAndRootSchema server({}, {}, {}, true); @@ -822,7 +822,7 @@ Y_UNIT_TEST_SUITE(Yq_1) { request, CreateYqSettings<TDescribeJobSettings>(folderId)) .ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - UNIT_ASSERT_VALUES_EQUAL(result.GetResult().job().query_meta().common().id(), queryId); + UNIT_ASSERT_VALUES_EQUAL(result.GetResult().job().query_meta().common().id(), queryId); UNIT_ASSERT_VALUES_EQUAL(result.GetResult().job().meta().id(), jobId); UNIT_ASSERT_VALUES_EQUAL(result.GetResult().job().query_name(), "test_query_name_1"); } @@ -854,15 +854,15 @@ Y_UNIT_TEST_SUITE(Yq_1) { } } } - + Y_UNIT_TEST_SUITE(Yq_2) { // use fork for data test due to ch initialization problem - Y_UNIT_TEST(Test_HostNameTrasformation) { - UNIT_ASSERT_VALUES_EQUAL(::NYq::TransformMdbHostToCorrectFormat("rc1c-p5waby2y5y1kb5ue.mdb.yandexcloud.net"), "rc1c-p5waby2y5y1kb5ue.db.yandex.net:8443"); - UNIT_ASSERT_VALUES_EQUAL(::NYq::TransformMdbHostToCorrectFormat("xxx.xxx"), "xxx.db.yandex.net:8443"); - UNIT_ASSERT_VALUES_EQUAL(::NYq::TransformMdbHostToCorrectFormat("host."), "host.db.yandex.net:8443"); - } - + Y_UNIT_TEST(Test_HostNameTrasformation) { + UNIT_ASSERT_VALUES_EQUAL(::NYq::TransformMdbHostToCorrectFormat("rc1c-p5waby2y5y1kb5ue.mdb.yandexcloud.net"), "rc1c-p5waby2y5y1kb5ue.db.yandex.net:8443"); + UNIT_ASSERT_VALUES_EQUAL(::NYq::TransformMdbHostToCorrectFormat("xxx.xxx"), "xxx.db.yandex.net:8443"); + UNIT_ASSERT_VALUES_EQUAL(::NYq::TransformMdbHostToCorrectFormat("host."), "host.db.yandex.net:8443"); + } + SIMPLE_UNIT_FORKED_TEST(ReadFromYdbOverYq) { TKikimrWithGrpcAndRootSchema server({}, {}, {}, true); ui16 grpc = server.GetPort(); @@ -883,11 +883,11 @@ Y_UNIT_TEST_SUITE(Yq_2) { UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); } - + TString queryId; { auto request = ::NYq::TCreateQueryBuilder{} - .SetText("select count(*) from testdb00.`yq/connections`") + .SetText("select count(*) from testdb00.`yq/connections`") .Build(); auto result = client.CreateQuery( request, CreateYqSettings<TCreateQuerySettings>(folderId)) @@ -905,7 +905,7 @@ Y_UNIT_TEST_SUITE(Yq_2) { .ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); const auto status = result.GetResult().query().meta().status(); - PrintProtoIssues(result.GetResult().query().issue()); + PrintProtoIssues(result.GetResult().query().issue()); return status == YandexQuery::QueryMeta::COMPLETED; }, TRetryOptions(10)); UNIT_ASSERT_C(result, "the execution of the query did not end within the time limit"); @@ -921,78 +921,78 @@ Y_UNIT_TEST_SUITE(Yq_2) { const auto& resultSet = result.GetResult().result_set(); UNIT_ASSERT_VALUES_EQUAL(resultSet.rows().size(), 1); UNIT_ASSERT_VALUES_EQUAL(resultSet.columns().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(resultSet.rows(0).items(0).uint64_value(), 1); + UNIT_ASSERT_VALUES_EQUAL(resultSet.rows(0).items(0).uint64_value(), 1); } } } - -Y_UNIT_TEST_SUITE(PrivateApi) { - Y_UNIT_TEST(PingTask) { + +Y_UNIT_TEST_SUITE(PrivateApi) { + Y_UNIT_TEST(PingTask) { TKikimrWithGrpcAndRootSchema server({}, {}, {}, true); - ui16 grpc = server.GetPort(); - TString location = TStringBuilder() << "localhost:" << grpc; - auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken("root@builtin")); - ::NYq::TPrivateClient client(driver); - const TString historyId = "id"; - const TString folderId = "folder_id"; + ui16 grpc = server.GetPort(); + TString location = TStringBuilder() << "localhost:" << grpc; + auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken("root@builtin")); + ::NYq::TPrivateClient client(driver); + const TString historyId = "id"; + const TString folderId = "folder_id"; const TScope scope(folderId); - { - Yq::Private::PingTaskRequest req; - req.mutable_query_id()->set_value("id"); - req.set_scope(scope.ToString()); - req.set_owner_id("some_owner"); - req.set_status(YandexQuery::QueryMeta::COMPLETED); - auto result = client.PingTask(std::move(req)).ExtractValueSync(); - result.GetIssues().PrintTo(Cerr); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::GENERIC_ERROR); - } - } - - Y_UNIT_TEST(GetTask) {//PendingFetcher can take task first + { + Yq::Private::PingTaskRequest req; + req.mutable_query_id()->set_value("id"); + req.set_scope(scope.ToString()); + req.set_owner_id("some_owner"); + req.set_status(YandexQuery::QueryMeta::COMPLETED); + auto result = client.PingTask(std::move(req)).ExtractValueSync(); + result.GetIssues().PrintTo(Cerr); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::GENERIC_ERROR); + } + } + + Y_UNIT_TEST(GetTask) {//PendingFetcher can take task first TKikimrWithGrpcAndRootSchema server({}, {}, {}, true); - ui16 grpc = server.GetPort(); - TString location = TStringBuilder() << "localhost:" << grpc; - auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken("root@builtin")); - ::NYq::TPrivateClient client(driver); - { - Yq::Private::GetTaskRequest req; - req.set_owner_id("owner_id"); - req.set_host("host"); - auto result = client.GetTask(std::move(req)).ExtractValueSync(); - result.GetIssues().PrintTo(Cerr); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - result.GetIssues().PrintTo(Cerr); - } - } - - Y_UNIT_TEST(Nodes) { + ui16 grpc = server.GetPort(); + TString location = TStringBuilder() << "localhost:" << grpc; + auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken("root@builtin")); + ::NYq::TPrivateClient client(driver); + { + Yq::Private::GetTaskRequest req; + req.set_owner_id("owner_id"); + req.set_host("host"); + auto result = client.GetTask(std::move(req)).ExtractValueSync(); + result.GetIssues().PrintTo(Cerr); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + result.GetIssues().PrintTo(Cerr); + } + } + + Y_UNIT_TEST(Nodes) { TKikimrWithGrpcAndRootSchema server({}, {}, {}, true); - ui16 grpc = server.GetPort(); - TString location = TStringBuilder() << "localhost:" << grpc; - auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken("root@builtin")); - ::NYq::TPrivateClient client(driver); - const auto instanceId = CreateGuidAsString(); - { - Yq::Private::NodesHealthCheckRequest req; - req.set_tenant("Tenant"); + ui16 grpc = server.GetPort(); + TString location = TStringBuilder() << "localhost:" << grpc; + auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken("root@builtin")); + ::NYq::TPrivateClient client(driver); + const auto instanceId = CreateGuidAsString(); + { + Yq::Private::NodesHealthCheckRequest req; + req.set_tenant("Tenant"); auto& node = *req.mutable_node(); node.set_hostname("hostname"); node.set_node_id(100500); node.set_instance_id(instanceId); - const auto result = DoWithRetryOnRetCode([&]() { - auto r = req; - auto result = client.NodesHealthCheck(std::move(r)).ExtractValueSync(); - if (result.GetStatus() == EStatus::SUCCESS) { - const auto& res = result.GetResult(); - UNIT_ASSERT(!res.nodes().empty()); - UNIT_ASSERT_VALUES_EQUAL(res.nodes(0).hostname(), "hostname"); - UNIT_ASSERT_VALUES_EQUAL(res.nodes(0).node_id(), 100500); - UNIT_ASSERT_VALUES_EQUAL(res.nodes(0).instance_id(), instanceId); - } - // result.GetIssues().PrintTo(Cerr); - return result.GetStatus() == EStatus::SUCCESS; - }, TRetryOptions(Retries)); - UNIT_ASSERT_C(result, "the execution of the query did not end within the time limit"); - } - } + const auto result = DoWithRetryOnRetCode([&]() { + auto r = req; + auto result = client.NodesHealthCheck(std::move(r)).ExtractValueSync(); + if (result.GetStatus() == EStatus::SUCCESS) { + const auto& res = result.GetResult(); + UNIT_ASSERT(!res.nodes().empty()); + UNIT_ASSERT_VALUES_EQUAL(res.nodes(0).hostname(), "hostname"); + UNIT_ASSERT_VALUES_EQUAL(res.nodes(0).node_id(), 100500); + UNIT_ASSERT_VALUES_EQUAL(res.nodes(0).instance_id(), instanceId); + } + // result.GetIssues().PrintTo(Cerr); + return result.GetStatus() == EStatus::SUCCESS; + }, TRetryOptions(Retries)); + UNIT_ASSERT_C(result, "the execution of the query did not end within the time limit"); + } + } } diff --git a/ydb/services/yq/ya.make b/ydb/services/yq/ya.make index 92b1eb8f8b..905c48a496 100644 --- a/ydb/services/yq/ya.make +++ b/ydb/services/yq/ya.make @@ -7,12 +7,12 @@ OWNER( SRCS( grpc_service.cpp - private_grpc.cpp + private_grpc.cpp ) PEERDIR( library/cpp/grpc/server - library/cpp/retry + library/cpp/retry ydb/core/grpc_services ydb/core/grpc_services/base ydb/library/protobuf_printer |