diff options
author | Alexey Bogolyubskiy <i@bogolyubskiyalexey.ru> | 2022-02-10 16:51:14 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:51:14 +0300 |
commit | 49023e629b5849b5f604b412acaf64558162d1d3 (patch) | |
tree | 5d5cb817648f650d76cf1076100726fd9b8448e8 | |
parent | 4d3f24f89a157c192be964fae784c6597dbbeb36 (diff) | |
download | ydb-49023e629b5849b5f604b412acaf64558162d1d3.tar.gz |
Restoring authorship annotation for Alexey Bogolyubskiy <i@bogolyubskiyalexey.ru>. Commit 2 of 2.
306 files changed, 10399 insertions, 10399 deletions
diff --git a/build/rules/kikimr.policy b/build/rules/kikimr.policy index 114dcf93c6..5bb6426e30 100644 --- a/build/rules/kikimr.policy +++ b/build/rules/kikimr.policy @@ -69,8 +69,8 @@ ALLOW .* -> kikimr/public/lib/json_value ALLOW .* -> kikimr/public/lib/yson_value ALLOW .* -> kikimr/public/lib/ydb_cli -ALLOW .* -> kikimr/yndx/api - +ALLOW .* -> kikimr/yndx/api + ALLOW cloud/blockstore -> kikimr/blockstore ALLOW cloud/blockstore -> kikimr/core ALLOW cloud/blockstore -> kikimr/driver_lib/run @@ -119,7 +119,7 @@ ALLOW logbroker/config-manager -> kikimr/core/util ALLOW logbroker/config-manager -> kikimr/library/persqueue/topic_parser ALLOW logbroker/config-manager -> kikimr/public/lib/deprecated/kicli ALLOW logbroker/config-manager -> kikimr/library/security -ALLOW logbroker/config-manager -> kikimr/yndx/security +ALLOW logbroker/config-manager -> kikimr/yndx/security ALLOW logbroker/config-manager -> kikimr/core/testlib ALLOW logbroker/push-agent/ut -> kikimr/core/testlib diff --git a/ydb/core/base/appdata.h b/ydb/core/base/appdata.h index dc3e2ccd8e..c666f7468c 100644 --- a/ydb/core/base/appdata.h +++ b/ydb/core/base/appdata.h @@ -58,7 +58,7 @@ namespace NSQS { } namespace NPQ { - class IPersQueueMirrorReaderFactory; + class IPersQueueMirrorReaderFactory; } class TFormatFactory; @@ -83,7 +83,7 @@ struct TAppData { IActor*(*FolderServiceFactory)(const NKikimrProto::NFolderService::TFolderServiceConfig&); - const NPQ::IPersQueueMirrorReaderFactory* PersQueueMirrorReaderFactory = nullptr; + const NPQ::IPersQueueMirrorReaderFactory* PersQueueMirrorReaderFactory = nullptr; NYdb::TDriver* YdbDriver = nullptr; const NPDisk::IIoContextFactory* IoContextFactory = nullptr; diff --git a/ydb/core/base/grpc_service_factory.h b/ydb/core/base/grpc_service_factory.h index bdba3349f9..0ae4fd9dd8 100644 --- a/ydb/core/base/grpc_service_factory.h +++ b/ydb/core/base/grpc_service_factory.h @@ -8,66 +8,66 @@ namespace NKikimr { class TGrpcServiceFactory { private: using TServicePtr = TIntrusivePtr<NGrpc::IGRpcService>; - using TFactoryMethod = std::function< - TServicePtr( - NActors::TActorSystem*, - TIntrusivePtr<NMonitoring::TDynamicCounters>, - NActors::TActorId - ) - >; - - struct TServiceParams { - TServiceParams(TFactoryMethod method, bool enableByDefault, std::optional<NActors::TActorId> proxyId) - : Method(method) - , EnableByDefault(enableByDefault) - , GrpcRequestProxyId(proxyId) - {} - - TFactoryMethod Method; - bool EnableByDefault; - std::optional<NActors::TActorId> GrpcRequestProxyId; - }; - -private: - std::unordered_map<TString, std::vector<TServiceParams>> Registry; - + using TFactoryMethod = std::function< + TServicePtr( + NActors::TActorSystem*, + TIntrusivePtr<NMonitoring::TDynamicCounters>, + NActors::TActorId + ) + >; + + struct TServiceParams { + TServiceParams(TFactoryMethod method, bool enableByDefault, std::optional<NActors::TActorId> proxyId) + : Method(method) + , EnableByDefault(enableByDefault) + , GrpcRequestProxyId(proxyId) + {} + + TFactoryMethod Method; + bool EnableByDefault; + std::optional<NActors::TActorId> GrpcRequestProxyId; + }; + +private: + std::unordered_map<TString, std::vector<TServiceParams>> Registry; + public: template <class TService> - void Register( - const TString& name, - bool enableByDefault = false, - std::optional<NActors::TActorId> grpcRequestProxyIdForService = std::nullopt - ) { - auto method = []( - NActors::TActorSystem* actorSystem, - TIntrusivePtr<NMonitoring::TDynamicCounters> counters, - NActors::TActorId grpcRequestProxyId - ) { + void Register( + const TString& name, + bool enableByDefault = false, + std::optional<NActors::TActorId> grpcRequestProxyIdForService = std::nullopt + ) { + auto method = []( + NActors::TActorSystem* actorSystem, + TIntrusivePtr<NMonitoring::TDynamicCounters> counters, + NActors::TActorId grpcRequestProxyId + ) { return TServicePtr(new TService(actorSystem, counters, grpcRequestProxyId)); - }; - Registry[name].emplace_back( - method, - enableByDefault, - grpcRequestProxyIdForService - ); + }; + Registry[name].emplace_back( + method, + enableByDefault, + grpcRequestProxyIdForService + ); } - std::vector<TServicePtr> Create( - const std::unordered_set<TString>& enabled, - const std::unordered_set<TString>& disabled, - NActors::TActorSystem* actorSystem, - TIntrusivePtr<NMonitoring::TDynamicCounters> counters, - NActors::TActorId grpcRequestProxyId - ) { + std::vector<TServicePtr> Create( + const std::unordered_set<TString>& enabled, + const std::unordered_set<TString>& disabled, + NActors::TActorSystem* actorSystem, + TIntrusivePtr<NMonitoring::TDynamicCounters> counters, + NActors::TActorId grpcRequestProxyId + ) { std::vector<TServicePtr> services; for (const auto& [name, methods] : Registry) { - for (const auto& [method, enableByDefault, grpcRequestProxyIdForService] : methods) { + for (const auto& [method, enableByDefault, grpcRequestProxyIdForService] : methods) { if (!disabled.count(name) && (enabled.count(name) || enableByDefault)) { - services.emplace_back(method( - actorSystem, - counters, - grpcRequestProxyIdForService.value_or(grpcRequestProxyId) - )); + services.emplace_back(method( + actorSystem, + counters, + grpcRequestProxyIdForService.value_or(grpcRequestProxyId) + )); } } } diff --git a/ydb/core/client/server/msgbus_server.cpp b/ydb/core/client/server/msgbus_server.cpp index 58afb3a0e6..8a4d8a3bbd 100644 --- a/ydb/core/client/server/msgbus_server.cpp +++ b/ydb/core/client/server/msgbus_server.cpp @@ -435,15 +435,15 @@ protected: const NBus::TBusServerSessionConfig &SessionConfig; }; -TMessageBusServer::TMessageBusServer( - const NBus::TBusServerSessionConfig &sessionConfig, - NBus::TBusMessageQueue *busQueue, - ui32 bindPort, - std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory -) +TMessageBusServer::TMessageBusServer( + const NBus::TBusServerSessionConfig &sessionConfig, + NBus::TBusMessageQueue *busQueue, + ui32 bindPort, + std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory +) : SessionConfig(sessionConfig) , BusQueue(busQueue) - , PQReadSessionsInfoWorkerFactory(pqReadSessionsInfoWorkerFactory) + , PQReadSessionsInfoWorkerFactory(pqReadSessionsInfoWorkerFactory) , Protocol(bindPort) {} @@ -599,20 +599,20 @@ void TMessageBusServer::UnknownMessage(TBusMessageContext &msg) { } IActor* TMessageBusServer::CreateProxy() { - return CreateMessageBusServerProxy(this, PQReadSessionsInfoWorkerFactory); + return CreateMessageBusServerProxy(this, PQReadSessionsInfoWorkerFactory); } IActor* TMessageBusServer::CreateMessageBusTraceService() { return nullptr; } -IMessageBusServer* CreateMsgBusServer( - NBus::TBusMessageQueue *queue, - const NBus::TBusServerSessionConfig &config, - std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory, - ui32 bindPort -) { - return new TMessageBusServer(config, queue, bindPort, pqReadSessionsInfoWorkerFactory); +IMessageBusServer* CreateMsgBusServer( + NBus::TBusMessageQueue *queue, + const NBus::TBusServerSessionConfig &config, + std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory, + ui32 bindPort +) { + return new TMessageBusServer(config, queue, bindPort, pqReadSessionsInfoWorkerFactory); } } diff --git a/ydb/core/client/server/msgbus_server.h b/ydb/core/client/server/msgbus_server.h index 2416df0981..3d40fc9001 100644 --- a/ydb/core/client/server/msgbus_server.h +++ b/ydb/core/client/server/msgbus_server.h @@ -136,18 +136,18 @@ class TMessageBusServer : public IMessageBusServer, public NBus::IBusServerHandl NBus::TBusServerSessionPtr Session; TActorId Proxy; TActorId Monitor; - std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> PQReadSessionsInfoWorkerFactory; + std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> PQReadSessionsInfoWorkerFactory; protected: TProtocol Protocol; TActorSystem *ActorSystem = nullptr; TIntrusivePtr<IMessageBusHttpServer> HttpServer; public: - TMessageBusServer( - const NBus::TBusServerSessionConfig &sessionConfig, - NBus::TBusMessageQueue *busQueue, - ui32 bindPort, - std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory - ); + TMessageBusServer( + const NBus::TBusServerSessionConfig &sessionConfig, + NBus::TBusMessageQueue *busQueue, + ui32 bindPort, + std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory + ); ~TMessageBusServer(); void InitSession(TActorSystem *actorSystem, const TActorId &proxy); @@ -272,13 +272,13 @@ void CopyProtobufsByFieldName(TProtoTo& protoTo, const TProtoFrom& protoFrom) { } } -class IPersQueueGetReadSessionsInfoWorkerFactory; +class IPersQueueGetReadSessionsInfoWorkerFactory; + +IActor* CreateMessageBusServerProxy( + TMessageBusServer *server, + std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory +); -IActor* CreateMessageBusServerProxy( - TMessageBusServer *server, - std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory -); - //IActor* CreateMessageBusDatashardSetConfig(TBusMessageContext &msg); IActor* CreateMessageBusTabletCountersRequest(TBusMessageContext &msg); IActor* CreateMessageBusLocalMKQL(TBusMessageContext &msg); diff --git a/ydb/core/client/server/msgbus_server_persqueue.cpp b/ydb/core/client/server/msgbus_server_persqueue.cpp index d796fa65e6..78b83ef3a3 100644 --- a/ydb/core/client/server/msgbus_server_persqueue.cpp +++ b/ydb/core/client/server/msgbus_server_persqueue.cpp @@ -1388,12 +1388,12 @@ public: TString RequestId; }; -template <template <class TImpl, class... TArgs> class TSenderImpl, class... T> -IActor* CreatePersQueueRequestProcessor( - const NKikimrClient::TPersQueueRequest& request, - std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory, - T&&... constructorParams -) { +template <template <class TImpl, class... TArgs> class TSenderImpl, class... T> +IActor* CreatePersQueueRequestProcessor( + const NKikimrClient::TPersQueueRequest& request, + std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory, + T&&... constructorParams +) { try { if (request.HasMetaRequest() + request.HasPartitionRequest() + request.HasFetchRequest() > 1) { throw std::runtime_error("only one from meta partition or fetch requests must be filled"); @@ -1414,10 +1414,10 @@ IActor* CreatePersQueueRequestProcessor( } else if (meta.HasCmdGetPartitionStatus()) { return new TSenderImpl<TPersQueueGetPartitionStatusProcessor>(std::forward<T>(constructorParams)...); } else if (meta.HasCmdGetReadSessionsInfo()) { - return new TSenderImpl<TPersQueueGetReadSessionsInfoProcessor>( - std::forward<T>(constructorParams)..., - pqReadSessionsInfoWorkerFactory - ); + return new TSenderImpl<TPersQueueGetReadSessionsInfoProcessor>( + std::forward<T>(constructorParams)..., + pqReadSessionsInfoWorkerFactory + ); } else { throw std::runtime_error("Not implemented yet"); } @@ -1439,9 +1439,9 @@ IActor* CreatePersQueueRequestProcessor( template <class TImplActor> class TMessageBusServerPersQueue : public TImplActor, TMessageBusSessionIdentHolder { public: - template <class... T> - TMessageBusServerPersQueue(TBusMessageContext& msg, T&&... constructorParams) - : TImplActor(static_cast<TBusPersQueue*>(msg.GetMessage())->Record, std::forward<T>(constructorParams)...) + template <class... T> + TMessageBusServerPersQueue(TBusMessageContext& msg, T&&... constructorParams) + : TImplActor(static_cast<TBusPersQueue*>(msg.GetMessage())->Record, std::forward<T>(constructorParams)...) , TMessageBusSessionIdentHolder(msg) {} @@ -1459,33 +1459,33 @@ public: }; -IActor* CreateMessageBusServerPersQueue( - TBusMessageContext& msg, - const TActorId& schemeCache, - std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory -) { +IActor* CreateMessageBusServerPersQueue( + TBusMessageContext& msg, + const TActorId& schemeCache, + std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory +) { const NKikimrClient::TPersQueueRequest& request = static_cast<TBusPersQueue*>(msg.GetMessage())->Record; - return CreatePersQueueRequestProcessor<TMessageBusServerPersQueue>( - request, - pqReadSessionsInfoWorkerFactory, - msg, - schemeCache - ); + return CreatePersQueueRequestProcessor<TMessageBusServerPersQueue>( + request, + pqReadSessionsInfoWorkerFactory, + msg, + schemeCache + ); } -IActor* CreateActorServerPersQueue( - const TActorId& parentId, - const NKikimrClient::TPersQueueRequest& request, - const TActorId& schemeCache, - std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory -) { - return CreatePersQueueRequestProcessor<TReplierToParent>( - request, - pqReadSessionsInfoWorkerFactory, - parentId, - request, - schemeCache - ); +IActor* CreateActorServerPersQueue( + const TActorId& parentId, + const NKikimrClient::TPersQueueRequest& request, + const TActorId& schemeCache, + std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory +) { + return CreatePersQueueRequestProcessor<TReplierToParent>( + request, + pqReadSessionsInfoWorkerFactory, + parentId, + request, + schemeCache + ); } } diff --git a/ydb/core/client/server/msgbus_server_persqueue.h b/ydb/core/client/server/msgbus_server_persqueue.h index 3cecd75360..52c67832e4 100644 --- a/ydb/core/client/server/msgbus_server_persqueue.h +++ b/ydb/core/client/server/msgbus_server_persqueue.h @@ -28,17 +28,17 @@ TProcessingResult ProcessMetaCacheAllTopicsResponse(NPqMetaCacheV2::TEvPqNewMeta TProcessingResult ProcessMetaCacheSingleTopicsResponse(const NSchemeCache::TSchemeCacheNavigate::TEntry& entry); // Worker actor creation -IActor* CreateMessageBusServerPersQueue( - TBusMessageContext& msg, - const TActorId& schemeCache, - std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory = nullptr -); -IActor* CreateActorServerPersQueue( - const TActorId& parentId, - const NKikimrClient::TPersQueueRequest& request, - const TActorId& schemeCache, - std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory = nullptr -); +IActor* CreateMessageBusServerPersQueue( + TBusMessageContext& msg, + const TActorId& schemeCache, + std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory = nullptr +); +IActor* CreateActorServerPersQueue( + const TActorId& parentId, + const NKikimrClient::TPersQueueRequest& request, + const TActorId& schemeCache, + std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory = nullptr +); NKikimrClient::TResponse CreateErrorReply(EResponseStatus status, NPersQueue::NErrorCode::EErrorCode code, const TString& errorReason); diff --git a/ydb/core/client/server/msgbus_server_pq_metacache.cpp b/ydb/core/client/server/msgbus_server_pq_metacache.cpp index c8f6247718..f44ab6f480 100644 --- a/ydb/core/client/server/msgbus_server_pq_metacache.cpp +++ b/ydb/core/client/server/msgbus_server_pq_metacache.cpp @@ -35,7 +35,7 @@ public: : Counters(counters) , ClientWrapper(std::move(std::make_unique<TClientWrapper>(grpcPort))) , VersionCheckInterval(versionCheckInterval) - , Generation(std::make_shared<TAtomicCounter>()) + , Generation(std::make_shared<TAtomicCounter>()) { } @@ -43,7 +43,7 @@ public: const TDuration& versionCheckInterval) : Counters(counters) , VersionCheckInterval(versionCheckInterval) - , Generation(std::make_shared<TAtomicCounter>()) + , Generation(std::make_shared<TAtomicCounter>()) { } @@ -86,22 +86,22 @@ public: } private: - template<class TEventType, class TFutureType, class... TArgs> - void SubscribeEvent(const TFutureType& future, TArgs... args) { - std::weak_ptr<TAtomicCounter> weakGeneration(Generation); + template<class TEventType, class TFutureType, class... TArgs> + void SubscribeEvent(const TFutureType& future, TArgs... args) { + std::weak_ptr<TAtomicCounter> weakGeneration(Generation); future.Subscribe( - [ - id = SelfId(), - originalGen = Generation->Val(), - weakGen = weakGeneration, - as = ActorSystem, - ... args = std::forward<TArgs>(args) - ](const auto&) mutable { - auto currentGen = weakGen.lock(); - if (currentGen && originalGen == currentGen->Val()) { - as->Send(id, new TEventType(args...)); + [ + id = SelfId(), + originalGen = Generation->Val(), + weakGen = weakGeneration, + as = ActorSystem, + ... args = std::forward<TArgs>(args) + ](const auto&) mutable { + auto currentGen = weakGen.lock(); + if (currentGen && originalGen == currentGen->Val()) { + as->Send(id, new TEventType(args...)); } - } + } ); } @@ -119,7 +119,7 @@ private: void StartSession(const TActorContext& ctx) { LOG_DEBUG_S(ctx, NKikimrServices::PQ_METACACHE, "Start new session"); SessionFuture = ClientWrapper->GetClient()->GetSession(); - SubscribeEvent<TEvPqNewMetaCache::TEvSessionStarted>(SessionFuture); + SubscribeEvent<TEvPqNewMetaCache::TEvSessionStarted>(SessionFuture); } void HandleSessionStarted(const TActorContext& ctx) { @@ -136,7 +136,7 @@ private: void PrepareTopicsQuery(const TActorContext& ctx) { Y_UNUSED(ctx); TopicsQueryFuture = YdbSession->PrepareDataQuery(TopicsQuery); - SubscribeEvent<TEvPqNewMetaCache::TEvQueryPrepared>(TopicsQueryFuture); + SubscribeEvent<TEvPqNewMetaCache::TEvQueryPrepared>(TopicsQueryFuture); } void HandleTopicsQueryPrepared(const TActorContext& ctx) { @@ -176,7 +176,7 @@ private: } AsyncQueryResult = PreparedTopicsQuery->Execute(TTxControl::BeginTx().CommitTx(), builder.Build()); } - SubscribeEvent<TEvPqNewMetaCache::TEvQueryComplete>(AsyncQueryResult, type); + SubscribeEvent<TEvPqNewMetaCache::TEvQueryComplete>(AsyncQueryResult, type); } void HandleCheckVersionResult(const TActorContext& ctx) { @@ -522,7 +522,7 @@ private: NActors::TActorId SchemeCacheId; TString PathPrefix; TVector<TString> PathPrefixParts; - std::shared_ptr<TAtomicCounter> Generation; + std::shared_ptr<TAtomicCounter> Generation; bool SkipVersionCheck = false; }; diff --git a/ydb/core/client/server/msgbus_server_pq_metarequest.cpp b/ydb/core/client/server/msgbus_server_pq_metarequest.cpp index a32f14495b..32f37251cc 100644 --- a/ydb/core/client/server/msgbus_server_pq_metarequest.cpp +++ b/ydb/core/client/server/msgbus_server_pq_metarequest.cpp @@ -1,5 +1,5 @@ #include "msgbus_server_pq_metarequest.h" -#include "msgbus_server_pq_read_session_info.h" +#include "msgbus_server_pq_read_session_info.h" namespace NKikimr { namespace NMsgBusProxy { @@ -439,12 +439,12 @@ void TPersQueueGetPartitionLocationsTopicWorker::Answer( // TPersQueueGetReadSessionsInfoProcessor::TPersQueueGetReadSessionsInfoProcessor( - const NKikimrClient::TPersQueueRequest& request, - const TActorId& schemeCache, - std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory + const NKikimrClient::TPersQueueRequest& request, + const TActorId& schemeCache, + std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory ) : TPersQueueBaseRequestProcessor(request, schemeCache, true) - , PQReadSessionsInfoWorkerFactory(pqReadSessionsInfoWorkerFactory) + , PQReadSessionsInfoWorkerFactory(pqReadSessionsInfoWorkerFactory) { const auto& cmd = RequestProto->GetMetaRequest().GetCmdGetReadSessionsInfo(); const auto& topics = cmd.GetTopic(); @@ -531,13 +531,13 @@ bool TPersQueueGetReadSessionsInfoTopicWorker::WaitAllPipeEvents(const TActorCon return false; } -THolder<IActor> TPersQueueGetReadSessionsInfoProcessor::CreateSessionsSubactor( - const THashMap<TString, TActorId>&& readSessions -) { - if (PQReadSessionsInfoWorkerFactory) { - return PQReadSessionsInfoWorkerFactory->Create(SelfId(), std::move(readSessions), NodesInfo); +THolder<IActor> TPersQueueGetReadSessionsInfoProcessor::CreateSessionsSubactor( + const THashMap<TString, TActorId>&& readSessions +) { + if (PQReadSessionsInfoWorkerFactory) { + return PQReadSessionsInfoWorkerFactory->Create(SelfId(), std::move(readSessions), NodesInfo); } - return MakeHolder<TPersQueueGetReadSessionsInfoWorker>(SelfId(), std::move(readSessions), NodesInfo); + return MakeHolder<TPersQueueGetReadSessionsInfoWorker>(SelfId(), std::move(readSessions), NodesInfo); } STFUNC(TPersQueueGetReadSessionsInfoTopicWorker::WaitAllPipeEventsStateFunc) { diff --git a/ydb/core/client/server/msgbus_server_pq_metarequest.h b/ydb/core/client/server/msgbus_server_pq_metarequest.h index 32cf7fddd1..1d56c28dc0 100644 --- a/ydb/core/client/server/msgbus_server_pq_metarequest.h +++ b/ydb/core/client/server/msgbus_server_pq_metarequest.h @@ -131,11 +131,11 @@ private: class TPersQueueGetReadSessionsInfoProcessor : public TPersQueueBaseRequestProcessor { public: - TPersQueueGetReadSessionsInfoProcessor( - const NKikimrClient::TPersQueueRequest& request, - const TActorId& schemeCache, - std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory - ); + TPersQueueGetReadSessionsInfoProcessor( + const NKikimrClient::TPersQueueRequest& request, + const TActorId& schemeCache, + std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory + ); bool ReadyForAnswer(const TActorContext& ctx) override { if (TPersQueueBaseRequestProcessor::ReadyForAnswer(ctx)) { @@ -172,7 +172,7 @@ private: THolder<IActor> CreateTopicSubactor(const TSchemeEntry& topicEntry, const TString& name) override; THolder<IActor> CreateSessionsSubactor(const THashMap<TString, TActorId>&& readSessions); - std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> PQReadSessionsInfoWorkerFactory; + std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> PQReadSessionsInfoWorkerFactory; mutable bool HasSessionsRequest = false; THashMap<TString, TActorId> ReadSessions; }; diff --git a/ydb/core/client/server/msgbus_server_pq_metarequest_ut.cpp b/ydb/core/client/server/msgbus_server_pq_metarequest_ut.cpp index 67eb9645c9..237bba147b 100644 --- a/ydb/core/client/server/msgbus_server_pq_metarequest_ut.cpp +++ b/ydb/core/client/server/msgbus_server_pq_metarequest_ut.cpp @@ -1,5 +1,5 @@ #include "msgbus_server_persqueue.h" -#include "msgbus_server_pq_read_session_info.h" +#include "msgbus_server_pq_read_session_info.h" #include <ydb/core/base/tabletid.h> #include <ydb/core/engine/minikql/flat_local_tx_factory.h> @@ -225,12 +225,12 @@ protected: // Register tested actor void RegisterActor(const NKikimrClient::TPersQueueRequest& request) { - Actor = CreateActorServerPersQueue( - EdgeActorId, - request, - GetMockPQMetaCache().SelfId(), - nullptr - ); + Actor = CreateActorServerPersQueue( + EdgeActorId, + request, + GetMockPQMetaCache().SelfId(), + nullptr + ); TestMainActorId = Runtime->Register(Actor); TestActors.insert(TestMainActorId); } diff --git a/ydb/core/client/server/msgbus_server_proxy.cpp b/ydb/core/client/server/msgbus_server_proxy.cpp index 4701cea25f..7f3146d21a 100644 --- a/ydb/core/client/server/msgbus_server_proxy.cpp +++ b/ydb/core/client/server/msgbus_server_proxy.cpp @@ -100,11 +100,11 @@ public: } }; -IActor* CreateMessageBusServerProxy( - TMessageBusServer* server, - std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory -) { - return new TMessageBusServerProxy(server, pqReadSessionsInfoWorkerFactory); +IActor* CreateMessageBusServerProxy( + TMessageBusServer* server, + std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory +) { + return new TMessageBusServerProxy(server, pqReadSessionsInfoWorkerFactory); } TBusResponse* ProposeTransactionStatusToResponse(EResponseStatus status, diff --git a/ydb/core/client/server/msgbus_server_proxy.h b/ydb/core/client/server/msgbus_server_proxy.h index 13b62aad64..cb9f9c19d3 100644 --- a/ydb/core/client/server/msgbus_server_proxy.h +++ b/ydb/core/client/server/msgbus_server_proxy.h @@ -31,8 +31,8 @@ struct TMessageBusDbOpsCounters : TAtomicRefCount<TMessageBusDbOpsCounters> { }; class TMessageBusServerProxy : public TActorBootstrapped<TMessageBusServerProxy> { - TMessageBusServer* const Server; - std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> PQReadSessionsInfoWorkerFactory; + TMessageBusServer* const Server; + std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> PQReadSessionsInfoWorkerFactory; TIntrusivePtr<NMonitoring::TDynamicCounters> SchemeCacheCounters; @@ -61,12 +61,12 @@ public: return NKikimrServices::TActivity::MSGBUS_PROXY_ACTOR; } - TMessageBusServerProxy( - TMessageBusServer* server, - std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory - ) + TMessageBusServerProxy( + TMessageBusServer* server, + std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory + ) : Server(server) - , PQReadSessionsInfoWorkerFactory(pqReadSessionsInfoWorkerFactory) + , PQReadSessionsInfoWorkerFactory(pqReadSessionsInfoWorkerFactory) { } diff --git a/ydb/core/client/server/msgbus_server_scheme_request.cpp b/ydb/core/client/server/msgbus_server_scheme_request.cpp index 4b19b9e349..7df5e262d0 100644 --- a/ydb/core/client/server/msgbus_server_scheme_request.cpp +++ b/ydb/core/client/server/msgbus_server_scheme_request.cpp @@ -218,7 +218,7 @@ void TMessageBusServerProxy::Handle(TEvBusProxy::TEvPersQueue::TPtr& ev, const T ctx.Register(new TMessageBusServerSchemeRequest<TBusPersQueue>(ev->Get()), TMailboxType::HTSwap, AppData()->UserPoolId); return; } - ctx.Register(CreateMessageBusServerPersQueue(msg->MsgContext, PqMetaCache, PQReadSessionsInfoWorkerFactory)); + ctx.Register(CreateMessageBusServerPersQueue(msg->MsgContext, PqMetaCache, PQReadSessionsInfoWorkerFactory)); } void TMessageBusServerProxy::Handle(TEvBusProxy::TEvFlatTxRequest::TPtr& ev, const TActorContext& ctx) { diff --git a/ydb/core/client/server/msgbus_server_tracer.cpp b/ydb/core/client/server/msgbus_server_tracer.cpp index 68098aa184..ff93f8d1cb 100644 --- a/ydb/core/client/server/msgbus_server_tracer.cpp +++ b/ydb/core/client/server/msgbus_server_tracer.cpp @@ -6,14 +6,14 @@ namespace NKikimr { namespace NMessageBusTracer { -TMessageBusTracingServer::TMessageBusTracingServer( - const NBus::TBusServerSessionConfig &sessionConfig, - NBus::TBusMessageQueue *busQueue, - const TString& tracePath, - ui32 bindPort, - std::shared_ptr<NMsgBusProxy::IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory -) - : NKikimr::NMsgBusProxy::TMessageBusServer(sessionConfig, busQueue, bindPort, pqReadSessionsInfoWorkerFactory) +TMessageBusTracingServer::TMessageBusTracingServer( + const NBus::TBusServerSessionConfig &sessionConfig, + NBus::TBusMessageQueue *busQueue, + const TString& tracePath, + ui32 bindPort, + std::shared_ptr<NMsgBusProxy::IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory +) + : NKikimr::NMsgBusProxy::TMessageBusServer(sessionConfig, busQueue, bindPort, pqReadSessionsInfoWorkerFactory) , MessageBusTracerActorID(MakeMessageBusTraceServiceID()) , TraceActive(false) , TracePath(tracePath) @@ -239,20 +239,20 @@ IActor* CreateMessageBusTracerStopTrace(NMsgBusProxy::TBusMessageContext &msg) { namespace NMsgBusProxy { -IMessageBusServer* CreateMsgBusTracingServer( - NBus::TBusMessageQueue *queue, - const NBus::TBusServerSessionConfig &config, - const TString &tracePath, - std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory, - ui32 bindPort -) { - return new NMessageBusTracer::TMessageBusTracingServer( - config, - queue, - tracePath, - bindPort, - pqReadSessionsInfoWorkerFactory - ); +IMessageBusServer* CreateMsgBusTracingServer( + NBus::TBusMessageQueue *queue, + const NBus::TBusServerSessionConfig &config, + const TString &tracePath, + std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory, + ui32 bindPort +) { + return new NMessageBusTracer::TMessageBusTracingServer( + config, + queue, + tracePath, + bindPort, + pqReadSessionsInfoWorkerFactory + ); } } diff --git a/ydb/core/client/server/msgbus_server_tracer.h b/ydb/core/client/server/msgbus_server_tracer.h index 487af92085..57a3812f5d 100644 --- a/ydb/core/client/server/msgbus_server_tracer.h +++ b/ydb/core/client/server/msgbus_server_tracer.h @@ -17,13 +17,13 @@ namespace NMessageBusTracer { class TMessageBusTracingServer : public NKikimr::NMsgBusProxy::TMessageBusServer, public NMsgBusProxy::IMessageWatcher { public: using TBusKey = NBus::TBusKey; - TMessageBusTracingServer( - const NBus::TBusServerSessionConfig &sessionConfig, - NBus::TBusMessageQueue *busQueue, - const TString &tracePath, - ui32 bindPort, - std::shared_ptr<NMsgBusProxy::IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory - ); + TMessageBusTracingServer( + const NBus::TBusServerSessionConfig &sessionConfig, + NBus::TBusMessageQueue *busQueue, + const TString &tracePath, + ui32 bindPort, + std::shared_ptr<NMsgBusProxy::IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory + ); IActor* CreateMessageBusTraceService() override; protected: TActorId MessageBusTracerActorID; diff --git a/ydb/core/client/server/ya.make b/ydb/core/client/server/ya.make index 309164a89e..8cbeec21b7 100644 --- a/ydb/core/client/server/ya.make +++ b/ydb/core/client/server/ya.make @@ -3,7 +3,7 @@ LIBRARY() OWNER( ddoarn g:kikimr - g:logbroker + g:logbroker ) SRCS( @@ -31,7 +31,7 @@ SRCS( msgbus_server_pq_metacache.cpp msgbus_server_pq_metarequest.h msgbus_server_pq_metarequest.cpp - msgbus_server_pq_read_session_info.cpp + msgbus_server_pq_read_session_info.cpp msgbus_server_resolve_node.cpp msgbus_server_ic_debug.cpp msgbus_server_load.cpp diff --git a/ydb/core/cms/console/net_classifier_updater.cpp b/ydb/core/cms/console/net_classifier_updater.cpp index 5667e9c376..121a65efc3 100644 --- a/ydb/core/cms/console/net_classifier_updater.cpp +++ b/ydb/core/cms/console/net_classifier_updater.cpp @@ -175,7 +175,7 @@ private: auto FormNetDataFromJson(TStringBuf jsonData) const { NKikimrNetClassifier::TNetData netData; - TVector<TString> tagsToFilter(UpdaterConfig().GetNetBoxTags().begin(), UpdaterConfig().GetNetBoxTags().end()); + TVector<TString> tagsToFilter(UpdaterConfig().GetNetBoxTags().begin(), UpdaterConfig().GetNetBoxTags().end()); NJson::TJsonValue value; bool res = NJson::ReadJsonTree(jsonData, &value); if (!res) @@ -186,25 +186,25 @@ private: if (!v["prefix"].IsString()) return NKikimrNetClassifier::TNetData{}; TString mask = v["prefix"].GetString(); - + if (!v["tags"].IsArray() || v["tags"].GetArray().size() == 0) return NKikimrNetClassifier::TNetData{}; - const auto& tags = v["tags"].GetArray(); - TString label; - for (auto& tag : tags) { - if (!tag.IsString()) - return NKikimrNetClassifier::TNetData{}; - if (std::count(tagsToFilter.begin(), tagsToFilter.end(), tag.GetString())) { - label = tag.GetString(); - break; - } - } - if (tagsToFilter.empty()) { - label = tags.front().GetString(); - } - if (!label) { - continue; - } + const auto& tags = v["tags"].GetArray(); + TString label; + for (auto& tag : tags) { + if (!tag.IsString()) + return NKikimrNetClassifier::TNetData{}; + if (std::count(tagsToFilter.begin(), tagsToFilter.end(), tag.GetString())) { + label = tag.GetString(); + break; + } + } + if (tagsToFilter.empty()) { + label = tags.front().GetString(); + } + if (!label) { + continue; + } auto& subnet = *netData.AddSubnets(); subnet.SetMask(mask); subnet.SetLabel(label); @@ -241,28 +241,28 @@ private: } void HandleWhileWorking(NHttp::TEvHttpProxy::TEvHttpIncomingResponse::TPtr& ev) { - if (ev->Get()->Error.empty()) { - if (ev->Get()->Response->Status == "200") { - const auto netData = UpdaterConfig().GetFormat() == NKikimrNetClassifier::TNetClassifierUpdaterConfig::TSV - ? FormNetData(ev->Get()->Response->Body) - : FormNetDataFromJson(ev->Get()->Response->Body); - if (netData.SubnetsSize() != 0) { - PackedNetData = PackAcquiredSubnets(netData); - - LastUpdateTimestamp = TActivationContext::Now(); - LastUpdateDatetimeUTC = LastUpdateTimestamp.ToRfc822String(); // for viewer - - // To modify the config it's essential to find the current id and generation - RequestCurrentConfigViaCookie(); - return; - } else { - BLOG_ERROR("NetClassifierUpdater failed to get subnets: got empty subnets list"); - } - } else { - BLOG_ERROR("NetClassifierUpdater failed to get subnets: http_status=" <<ev->Get()->Response->Status); + if (ev->Get()->Error.empty()) { + if (ev->Get()->Response->Status == "200") { + const auto netData = UpdaterConfig().GetFormat() == NKikimrNetClassifier::TNetClassifierUpdaterConfig::TSV + ? FormNetData(ev->Get()->Response->Body) + : FormNetDataFromJson(ev->Get()->Response->Body); + if (netData.SubnetsSize() != 0) { + PackedNetData = PackAcquiredSubnets(netData); + + LastUpdateTimestamp = TActivationContext::Now(); + LastUpdateDatetimeUTC = LastUpdateTimestamp.ToRfc822String(); // for viewer + + // To modify the config it's essential to find the current id and generation + RequestCurrentConfigViaCookie(); + return; + } else { + BLOG_ERROR("NetClassifierUpdater failed to get subnets: got empty subnets list"); + } + } else { + BLOG_ERROR("NetClassifierUpdater failed to get subnets: http_status=" <<ev->Get()->Response->Status); } - } else { - BLOG_ERROR("NetClassifierUpdater failed to get subnets: " << ev->Get()->Error); + } else { + BLOG_ERROR("NetClassifierUpdater failed to get subnets: " << ev->Get()->Error); } InitializeAgain(); } diff --git a/ydb/core/cms/console/net_classifier_updater_ut.cpp b/ydb/core/cms/console/net_classifier_updater_ut.cpp index 4b50f496b4..da41ee51f2 100644 --- a/ydb/core/cms/console/net_classifier_updater_ut.cpp +++ b/ydb/core/cms/console/net_classifier_updater_ut.cpp @@ -19,9 +19,9 @@ namespace NKikimr::NNetClassifierUpdaterTests { using namespace NConsole; using namespace Tests; -using TNetClassifierUpdaterConfig = NKikimrNetClassifier::TNetClassifierUpdaterConfig; -const TString NETWORKS_URI = "/fancy_path/networks.tsv"; - +using TNetClassifierUpdaterConfig = NKikimrNetClassifier::TNetClassifierUpdaterConfig; +const TString NETWORKS_URI = "/fancy_path/networks.tsv"; + static NHttp::TEvHttpProxy::TEvHttpOutgoingResponse* MakeHttpResponse(NHttp::TEvHttpProxy::TEvHttpIncomingRequest* request, const TString& netData) { const TString content = TStringBuilder() << "HTTP/1.1 200 OK\r\nConnection: Close\r\nContent-Type: application/octet-stream\r\nContent-Length: " << netData.size() << "\r\n\r\n" << netData; @@ -113,26 +113,26 @@ static TString ConvertToJson(const NKikimrNetClassifier::TNetData& netData) { return res; } -NKikimrNetClassifier::TNetClassifierUpdaterConfig CreateUpdaterConfig( - ui16 netDataSourcePort, - TNetClassifierUpdaterConfig::EFormat format, - const TVector<TString>& netBoxTags = {} -) { - const TString url = TStringBuilder() << "http://[::1]:" << netDataSourcePort << NETWORKS_URI; - NKikimrNetClassifier::TNetClassifierUpdaterConfig updaterConfig; - updaterConfig.SetNetDataSourceUrl(url); - updaterConfig.SetFormat(format); - *updaterConfig.MutableNetBoxTags() = {netBoxTags.begin(), netBoxTags.end()}; - return updaterConfig; -} +NKikimrNetClassifier::TNetClassifierUpdaterConfig CreateUpdaterConfig( + ui16 netDataSourcePort, + TNetClassifierUpdaterConfig::EFormat format, + const TVector<TString>& netBoxTags = {} +) { + const TString url = TStringBuilder() << "http://[::1]:" << netDataSourcePort << NETWORKS_URI; + NKikimrNetClassifier::TNetClassifierUpdaterConfig updaterConfig; + updaterConfig.SetNetDataSourceUrl(url); + updaterConfig.SetFormat(format); + *updaterConfig.MutableNetBoxTags() = {netBoxTags.begin(), netBoxTags.end()}; + return updaterConfig; +} Y_UNIT_TEST_SUITE(TNetClassifierUpdaterTest) { - void TestGetUpdatesFromHttpServer( - const TString& sourceResponce, - const NKikimrNetClassifier::TNetData& expectedNetData, - TNetClassifierUpdaterConfig::EFormat format = TNetClassifierUpdaterConfig::TSV, - const TVector<TString>& netBoxTags = {} - ) { + void TestGetUpdatesFromHttpServer( + const TString& sourceResponce, + const NKikimrNetClassifier::TNetData& expectedNetData, + TNetClassifierUpdaterConfig::EFormat format = TNetClassifierUpdaterConfig::TSV, + const TVector<TString>& netBoxTags = {} + ) { NMonitoring::TMetricRegistry sensors; TPortManager pm; @@ -140,7 +140,7 @@ Y_UNIT_TEST_SUITE(TNetClassifierUpdaterTest) { const ui64 netDataSourcePort = pm.GetPort(13334); TServerSettings settings(port); auto& updaterConfig = *settings.NetClassifierConfig.MutableUpdaterConfig(); - updaterConfig = CreateUpdaterConfig(netDataSourcePort, format, netBoxTags); + updaterConfig = CreateUpdaterConfig(netDataSourcePort, format, netBoxTags); TServer cleverServer = TServer(settings); auto& actorSystem = *cleverServer.GetRuntime(); @@ -153,13 +153,13 @@ Y_UNIT_TEST_SUITE(TNetClassifierUpdaterTest) { NActors::TActorId serverId = actorSystem.AllocateEdgeActor(); - actorSystem.Send(new NActors::IEventHandle(proxyId, serverId, new NHttp::TEvHttpProxy::TEvRegisterHandler(NETWORKS_URI, serverId)), 0, true); - + actorSystem.Send(new NActors::IEventHandle(proxyId, serverId, new NHttp::TEvHttpProxy::TEvRegisterHandler(NETWORKS_URI, serverId)), 0, true); + TAutoPtr<NActors::IEventHandle> handle; NHttp::TEvHttpProxy::TEvHttpIncomingRequest* request = actorSystem.GrabEdgeEvent<NHttp::TEvHttpProxy::TEvHttpIncomingRequest>(handle); - UNIT_ASSERT_EQUAL(request->Request->URL, NETWORKS_URI); + UNIT_ASSERT_EQUAL(request->Request->URL, NETWORKS_URI); - actorSystem.Send(new NActors::IEventHandle(handle->Sender, serverId, MakeHttpResponse(request, sourceResponce)), 0, true); + actorSystem.Send(new NActors::IEventHandle(handle->Sender, serverId, MakeHttpResponse(request, sourceResponce)), 0, true); const TActorId sender = actorSystem.AllocateEdgeActor(); size_t iterations = 0; @@ -174,7 +174,7 @@ Y_UNIT_TEST_SUITE(TNetClassifierUpdaterTest) { const auto event = cleverServer.GetRuntime()->GrabEdgeEvent<TEvConfigsDispatcher::TEvGetConfigResponse>(handle); - if (CheckDistributableConfig(event->Config->GetNetClassifierDistributableConfig(), expectedNetData)) { + if (CheckDistributableConfig(event->Config->GetNetClassifierDistributableConfig(), expectedNetData)) { break; } @@ -184,74 +184,74 @@ Y_UNIT_TEST_SUITE(TNetClassifierUpdaterTest) { } Y_UNIT_TEST(TestGetUpdatesFromHttpServer) { - auto netData = FormNetData(); - TestGetUpdatesFromHttpServer(ConvertToTsv(netData), netData); - TestGetUpdatesFromHttpServer(ConvertToJson(netData), netData, TNetClassifierUpdaterConfig::NETBOX); + auto netData = FormNetData(); + TestGetUpdatesFromHttpServer(ConvertToTsv(netData), netData); + TestGetUpdatesFromHttpServer(ConvertToJson(netData), netData, TNetClassifierUpdaterConfig::NETBOX); } - Y_UNIT_TEST(TestFiltrationByNetboxTags) { - const TString netboxResponce = "{ \ - \"count\": 5, \ - \"results\": [ \ - {\"prefix\": \"5.45.192.0/18\", \"tags\": [\"asd\", \"zxcv\"]}, \ - {\"prefix\": \"5.255.192.0/18\", \"tags\": [\"zxcv\", \"asd\"]}, \ - {\"prefix\": \"37.9.64.0/18\", \"tags\": [\"zxcv\"]}, \ - {\"prefix\": \"95.108.128.0/17\", \"tags\": [\"asd\"]}, \ - {\"prefix\": \"172.24.0.0/13\", \"tags\": [\"qwerty\"]} \ - ]}"; - - auto addMask = [](NKikimrNetClassifier::TNetData& data, const TString& mask, const TString& label) { - auto& subnet = *data.AddSubnets(); - subnet.SetMask(mask); - subnet.SetLabel(label); - }; - - { - NKikimrNetClassifier::TNetData data; - addMask(data, "5.45.192.0/18", "asd"); - addMask(data, "5.255.192.0/18", "zxcv"); - addMask(data, "37.9.64.0/18", "zxcv"); - addMask(data, "95.108.128.0/17", "asd"); - addMask(data, "172.24.0.0/13", "qwerty"); - TestGetUpdatesFromHttpServer(netboxResponce, data, TNetClassifierUpdaterConfig::NETBOX); - } - { - NKikimrNetClassifier::TNetData data; - addMask(data, "5.45.192.0/18", "asd"); - addMask(data, "5.255.192.0/18", "zxcv"); - addMask(data, "37.9.64.0/18", "zxcv"); - addMask(data, "95.108.128.0/17", "asd"); - addMask(data, "172.24.0.0/13", "qwerty"); - TestGetUpdatesFromHttpServer(netboxResponce, data, TNetClassifierUpdaterConfig::NETBOX, {"asd", "zxcv", "qwerty", "faketag"}); - } - { - NKikimrNetClassifier::TNetData data; - addMask(data, "5.45.192.0/18", "asd"); - addMask(data, "5.255.192.0/18", "zxcv"); - addMask(data, "37.9.64.0/18", "zxcv"); - addMask(data, "95.108.128.0/17", "asd"); - TestGetUpdatesFromHttpServer(netboxResponce, data, TNetClassifierUpdaterConfig::NETBOX, {"zxcv", "asd"}); - } - { - NKikimrNetClassifier::TNetData data; - addMask(data, "5.45.192.0/18", "zxcv"); - addMask(data, "5.255.192.0/18", "zxcv"); - addMask(data, "37.9.64.0/18", "zxcv"); - TestGetUpdatesFromHttpServer(netboxResponce, data, TNetClassifierUpdaterConfig::NETBOX, {"zxcv"}); - } - { - NKikimrNetClassifier::TNetData data; - addMask(data, "5.45.192.0/18", "asd"); - addMask(data, "5.255.192.0/18", "asd"); - addMask(data, "95.108.128.0/17", "asd"); - TestGetUpdatesFromHttpServer(netboxResponce, data, TNetClassifierUpdaterConfig::NETBOX, {"asd"}); - } - { - NKikimrNetClassifier::TNetData data; - addMask(data, "172.24.0.0/13", "qwerty"); - TestGetUpdatesFromHttpServer(netboxResponce, data, TNetClassifierUpdaterConfig::NETBOX, {"qwerty"}); - } - } + Y_UNIT_TEST(TestFiltrationByNetboxTags) { + const TString netboxResponce = "{ \ + \"count\": 5, \ + \"results\": [ \ + {\"prefix\": \"5.45.192.0/18\", \"tags\": [\"asd\", \"zxcv\"]}, \ + {\"prefix\": \"5.255.192.0/18\", \"tags\": [\"zxcv\", \"asd\"]}, \ + {\"prefix\": \"37.9.64.0/18\", \"tags\": [\"zxcv\"]}, \ + {\"prefix\": \"95.108.128.0/17\", \"tags\": [\"asd\"]}, \ + {\"prefix\": \"172.24.0.0/13\", \"tags\": [\"qwerty\"]} \ + ]}"; + + auto addMask = [](NKikimrNetClassifier::TNetData& data, const TString& mask, const TString& label) { + auto& subnet = *data.AddSubnets(); + subnet.SetMask(mask); + subnet.SetLabel(label); + }; + + { + NKikimrNetClassifier::TNetData data; + addMask(data, "5.45.192.0/18", "asd"); + addMask(data, "5.255.192.0/18", "zxcv"); + addMask(data, "37.9.64.0/18", "zxcv"); + addMask(data, "95.108.128.0/17", "asd"); + addMask(data, "172.24.0.0/13", "qwerty"); + TestGetUpdatesFromHttpServer(netboxResponce, data, TNetClassifierUpdaterConfig::NETBOX); + } + { + NKikimrNetClassifier::TNetData data; + addMask(data, "5.45.192.0/18", "asd"); + addMask(data, "5.255.192.0/18", "zxcv"); + addMask(data, "37.9.64.0/18", "zxcv"); + addMask(data, "95.108.128.0/17", "asd"); + addMask(data, "172.24.0.0/13", "qwerty"); + TestGetUpdatesFromHttpServer(netboxResponce, data, TNetClassifierUpdaterConfig::NETBOX, {"asd", "zxcv", "qwerty", "faketag"}); + } + { + NKikimrNetClassifier::TNetData data; + addMask(data, "5.45.192.0/18", "asd"); + addMask(data, "5.255.192.0/18", "zxcv"); + addMask(data, "37.9.64.0/18", "zxcv"); + addMask(data, "95.108.128.0/17", "asd"); + TestGetUpdatesFromHttpServer(netboxResponce, data, TNetClassifierUpdaterConfig::NETBOX, {"zxcv", "asd"}); + } + { + NKikimrNetClassifier::TNetData data; + addMask(data, "5.45.192.0/18", "zxcv"); + addMask(data, "5.255.192.0/18", "zxcv"); + addMask(data, "37.9.64.0/18", "zxcv"); + TestGetUpdatesFromHttpServer(netboxResponce, data, TNetClassifierUpdaterConfig::NETBOX, {"zxcv"}); + } + { + NKikimrNetClassifier::TNetData data; + addMask(data, "5.45.192.0/18", "asd"); + addMask(data, "5.255.192.0/18", "asd"); + addMask(data, "95.108.128.0/17", "asd"); + TestGetUpdatesFromHttpServer(netboxResponce, data, TNetClassifierUpdaterConfig::NETBOX, {"asd"}); + } + { + NKikimrNetClassifier::TNetData data; + addMask(data, "172.24.0.0/13", "qwerty"); + TestGetUpdatesFromHttpServer(netboxResponce, data, TNetClassifierUpdaterConfig::NETBOX, {"qwerty"}); + } + } } } // namespace NKikimr::NNetClassifierUpdaterTests diff --git a/ydb/core/driver_lib/run/factories.h b/ydb/core/driver_lib/run/factories.h index eb35a94f2b..41a45b44e0 100644 --- a/ydb/core/driver_lib/run/factories.h +++ b/ydb/core/driver_lib/run/factories.h @@ -33,8 +33,8 @@ struct TModuleFactories { std::shared_ptr<ILogBackendFactory> LogBackendFactory; // A backend factory for Query Replay std::shared_ptr<NKqp::IQueryReplayBackendFactory> QueryReplayBackendFactory; - // - std::shared_ptr<NMsgBusProxy::IPersQueueGetReadSessionsInfoWorkerFactory> PQReadSessionsInfoWorkerFactory; + // + std::shared_ptr<NMsgBusProxy::IPersQueueGetReadSessionsInfoWorkerFactory> PQReadSessionsInfoWorkerFactory; // Can be nullptr. In that case there would be no ability to work with Yandex Logbroker in Yandex Query. NPq::NConfigurationManager::IConnections::TPtr PqCmConnections; // Export implementation for Data Shards @@ -50,7 +50,7 @@ struct TModuleFactories { // Factory for grpc services TGrpcServiceFactory GrpcServiceFactory; - std::shared_ptr<NPQ::IPersQueueMirrorReaderFactory> PersQueueMirrorReaderFactory; + std::shared_ptr<NPQ::IPersQueueMirrorReaderFactory> PersQueueMirrorReaderFactory; /// Factory for pdisk's aio engines std::shared_ptr<NPDisk::IIoContextFactory> IoContextFactory; diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp index d8086f97d1..819c1478d1 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp @@ -1471,12 +1471,12 @@ void TSecurityServicesInitializer::InitializeServices(NActors::TActorSystemSetup // TGRpcServicesInitializer -TGRpcServicesInitializer::TGRpcServicesInitializer( - const TKikimrRunConfig& runConfig, - std::shared_ptr<TModuleFactories> factories -) +TGRpcServicesInitializer::TGRpcServicesInitializer( + const TKikimrRunConfig& runConfig, + std::shared_ptr<TModuleFactories> factories +) : IKikimrServicesInitializer(runConfig) - , Factories(factories) + , Factories(factories) {} void TGRpcServicesInitializer::InitializeServices(NActors::TActorSystemSetup* setup, @@ -1484,10 +1484,10 @@ void TGRpcServicesInitializer::InitializeServices(NActors::TActorSystemSetup* se { if (!IsServiceInitialized(setup, NMsgBusProxy::CreateMsgBusProxyId()) && Config.HasGRpcConfig() && Config.GetGRpcConfig().GetStartGRpcProxy()) { - IActor * proxy = NMsgBusProxy::CreateMessageBusServerProxy( - nullptr, - Factories ? Factories->PQReadSessionsInfoWorkerFactory : nullptr - ); + IActor * proxy = NMsgBusProxy::CreateMessageBusServerProxy( + nullptr, + Factories ? Factories->PQReadSessionsInfoWorkerFactory : nullptr + ); Y_VERIFY(proxy); setup->LocalServices.emplace_back( NMsgBusProxy::CreateMsgBusProxyId(), @@ -1799,11 +1799,11 @@ TPersQueueL2CacheInitializer::TPersQueueL2CacheInitializer(const TKikimrRunConfi void TPersQueueL2CacheInitializer::InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) { static const ui64 DEFAULT_PQ_L2_MAX_SIZE_MB = 8 * 1024; - static const TDuration DEFAULT_PQ_L2_KEEP_TIMEOUT = TDuration::Seconds(10); + static const TDuration DEFAULT_PQ_L2_KEEP_TIMEOUT = TDuration::Seconds(10); NPQ::TCacheL2Parameters params; params.MaxSizeMB = DEFAULT_PQ_L2_MAX_SIZE_MB; - params.KeepTime = DEFAULT_PQ_L2_KEEP_TIMEOUT; + params.KeepTime = DEFAULT_PQ_L2_KEEP_TIMEOUT; if (Config.HasBootstrapConfig() && Config.GetBootstrapConfig().HasNodeLimits()) { auto nodeLimits = Config.GetBootstrapConfig().GetNodeLimits(); @@ -1812,7 +1812,7 @@ void TPersQueueL2CacheInitializer::InitializeServices(NActors::TActorSystemSetup if (cfg.HasSharedCacheSizeMb()) params.MaxSizeMB = cfg.GetSharedCacheSizeMb(); if (cfg.HasCacheKeepTimeSec()) - params.KeepTime = TDuration::Seconds(cfg.GetCacheKeepTimeSec()); + params.KeepTime = TDuration::Seconds(cfg.GetCacheKeepTimeSec()); } } @@ -1860,24 +1860,24 @@ void TPersQueueClusterTrackerInitializer::InitializeServices(NActors::TActorSyst TActorSetupCmd(actor, TMailboxType::HTSwap, appData->UserPoolId))); } -// TPersQueueLibSharedInstanceInitializer +// TPersQueueLibSharedInstanceInitializer + +TPersQueueLibSharedInstanceInitializer::TPersQueueLibSharedInstanceInitializer(const TKikimrRunConfig& runConfig) + : IKikimrServicesInitializer(runConfig) +{} -TPersQueueLibSharedInstanceInitializer::TPersQueueLibSharedInstanceInitializer(const TKikimrRunConfig& runConfig) - : IKikimrServicesInitializer(runConfig) -{} - void TPersQueueLibSharedInstanceInitializer::InitializeServices(NActors::TActorSystemSetup*, const NKikimr::TAppData* appData) { - if (Config.HasPQConfig() && Config.GetPQConfig().GetEnabled()) { - if (Config.GetPQConfig().GetMirrorConfig().GetEnabled()) { - if (appData->PersQueueMirrorReaderFactory) { - appData->PersQueueMirrorReaderFactory->Initialize(Config.GetPQConfig().GetMirrorConfig().GetPQLibSettings()); + if (Config.HasPQConfig() && Config.GetPQConfig().GetEnabled()) { + if (Config.GetPQConfig().GetMirrorConfig().GetEnabled()) { + if (appData->PersQueueMirrorReaderFactory) { + appData->PersQueueMirrorReaderFactory->Initialize(Config.GetPQConfig().GetMirrorConfig().GetPQLibSettings()); } - } - } -} - -// TMemProfMonitorInitializer - + } + } +} + +// TMemProfMonitorInitializer + TMemProfMonitorInitializer::TMemProfMonitorInitializer(const TKikimrRunConfig& runConfig) : IKikimrServicesInitializer(runConfig) {} diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.h b/ydb/core/driver_lib/run/kikimr_services_initializers.h index 331859b8eb..407ce1bb7b 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.h +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.h @@ -254,11 +254,11 @@ public: // grpc_proxy class TGRpcServicesInitializer : public IKikimrServicesInitializer { -private: - std::shared_ptr<TModuleFactories> Factories; - +private: + std::shared_ptr<TModuleFactories> Factories; + public: - TGRpcServicesInitializer(const TKikimrRunConfig& runConfig, std::shared_ptr<TModuleFactories> factories); + TGRpcServicesInitializer(const TKikimrRunConfig& runConfig, std::shared_ptr<TModuleFactories> factories); void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; }; @@ -350,13 +350,13 @@ public: void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; }; -class TPersQueueLibSharedInstanceInitializer : public IKikimrServicesInitializer { -public: - TPersQueueLibSharedInstanceInitializer(const TKikimrRunConfig& runConfig); +class TPersQueueLibSharedInstanceInitializer : public IKikimrServicesInitializer { +public: + TPersQueueLibSharedInstanceInitializer(const TKikimrRunConfig& runConfig); void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; -}; - +}; + class TMemProfMonitorInitializer : public IKikimrServicesInitializer { public: TMemProfMonitorInitializer(const TKikimrRunConfig& runConfig); diff --git a/ydb/core/driver_lib/run/run.cpp b/ydb/core/driver_lib/run/run.cpp index 86a905d5b3..a4f74aa4e0 100644 --- a/ydb/core/driver_lib/run/run.cpp +++ b/ydb/core/driver_lib/run/run.cpp @@ -398,10 +398,10 @@ void TKikimrRunner::InitializeControlBoard(const TKikimrRunConfig& runConfig) } } -void TKikimrRunner::InitializeMessageBus( - const TKikimrRunConfig& runConfig, - std::shared_ptr<TModuleFactories> factories -) { +void TKikimrRunner::InitializeMessageBus( + const TKikimrRunConfig& runConfig, + std::shared_ptr<TModuleFactories> factories +) { if (runConfig.AppConfig.HasMessageBusConfig() && runConfig.AppConfig.GetMessageBusConfig().GetStartBusProxy()) { const auto& msgbusConfig = runConfig.AppConfig.GetMessageBusConfig(); @@ -440,21 +440,21 @@ void TKikimrRunner::InitializeMessageBus( Bus.Reset(NBus::CreateMessageQueue(ProxyBusQueueConfig, "server proxy")); if (msgbusConfig.GetStartTracingBusProxy()) { - BusServer.Reset(NMsgBusProxy::CreateMsgBusTracingServer( - Bus.Get(), - ProxyBusSessionConfig, - msgbusConfig.GetTracePath(), - factories ? factories->PQReadSessionsInfoWorkerFactory : nullptr, - msgbusConfig.GetBusProxyPort()) - ); + BusServer.Reset(NMsgBusProxy::CreateMsgBusTracingServer( + Bus.Get(), + ProxyBusSessionConfig, + msgbusConfig.GetTracePath(), + factories ? factories->PQReadSessionsInfoWorkerFactory : nullptr, + msgbusConfig.GetBusProxyPort()) + ); } else { - BusServer.Reset(NMsgBusProxy::CreateMsgBusServer( - Bus.Get(), - ProxyBusSessionConfig, - factories ? factories->PQReadSessionsInfoWorkerFactory : nullptr, - msgbusConfig.GetBusProxyPort() - )); + BusServer.Reset(NMsgBusProxy::CreateMsgBusServer( + Bus.Get(), + ProxyBusSessionConfig, + factories ? factories->PQReadSessionsInfoWorkerFactory : nullptr, + msgbusConfig.GetBusProxyPort() + )); } if (Monitoring) { @@ -852,7 +852,7 @@ void TKikimrRunner::InitializeAppData(const TKikimrRunConfig& runConfig) &KikimrShouldContinue)); AppData->DataShardExportFactory = ModuleFactories ? ModuleFactories->DataShardExportFactory.get() : nullptr; AppData->SqsEventsWriterFactory = ModuleFactories ? ModuleFactories->SqsEventsWriterFactory.get() : nullptr; - AppData->PersQueueMirrorReaderFactory = ModuleFactories ? ModuleFactories->PersQueueMirrorReaderFactory.get() : nullptr; + AppData->PersQueueMirrorReaderFactory = ModuleFactories ? ModuleFactories->PersQueueMirrorReaderFactory.get() : nullptr; AppData->IoContextFactory = ModuleFactories ? ModuleFactories->IoContextFactory.get() : nullptr; AppData->SqsAuthFactory = ModuleFactories @@ -1223,7 +1223,7 @@ TIntrusivePtr<TServiceInitializersList> TKikimrRunner::CreateServiceInitializers } if (serviceMask.EnableGRpcService) { - sil->AddServiceInitializer(new TGRpcServicesInitializer(runConfig, ModuleFactories)); + sil->AddServiceInitializer(new TGRpcServicesInitializer(runConfig, ModuleFactories)); } #ifdef ACTORSLIB_COLLECT_EXEC_STATS @@ -1260,8 +1260,8 @@ TIntrusivePtr<TServiceInitializersList> TKikimrRunner::CreateServiceInitializers sil->AddServiceInitializer(new TPersQueueClusterTrackerInitializer(runConfig)); } - sil->AddServiceInitializer(new TPersQueueLibSharedInstanceInitializer(runConfig)); - + sil->AddServiceInitializer(new TPersQueueLibSharedInstanceInitializer(runConfig)); + sil->AddServiceInitializer(new TMemProfMonitorInitializer(runConfig)); #if defined(ENABLE_MEMORY_TRACKING) @@ -1559,12 +1559,12 @@ TIntrusivePtr<TKikimrRunner> TKikimrRunner::CreateKikimrRunner( std::shared_ptr<TModuleFactories> factories) { TBasicKikimrServicesMask servicesMask; // all services enabled by default - TIntrusivePtr<TKikimrRunner> runner(new TKikimrRunner(factories)); + TIntrusivePtr<TKikimrRunner> runner(new TKikimrRunner(factories)); runner->InitializeAllocator(runConfig); runner->InitializeRegistries(runConfig); runner->InitializeMonitoring(runConfig); runner->InitializeControlBoard(runConfig); - runner->InitializeMessageBus(runConfig, factories); + runner->InitializeMessageBus(runConfig, factories); runner->InitializeAppData(runConfig); runner->InitializeLogSettings(runConfig); TIntrusivePtr<TServiceInitializersList> sil(runner->CreateServiceInitializersList(runConfig, servicesMask)); diff --git a/ydb/core/driver_lib/run/run.h b/ydb/core/driver_lib/run/run.h index 1e1c9c2c29..dfd93a24e7 100644 --- a/ydb/core/driver_lib/run/run.h +++ b/ydb/core/driver_lib/run/run.h @@ -81,10 +81,10 @@ protected: void InitializeMonitoringLogin(const TKikimrRunConfig& runConfig); - void InitializeMessageBus( - const TKikimrRunConfig& runConfig, - std::shared_ptr<TModuleFactories> factories = nullptr - ); + void InitializeMessageBus( + const TKikimrRunConfig& runConfig, + std::shared_ptr<TModuleFactories> factories = nullptr + ); void InitializeGRpc(const TKikimrRunConfig& runConfig); diff --git a/ydb/core/persqueue/actor_persqueue_client_iface.h b/ydb/core/persqueue/actor_persqueue_client_iface.h index 4312d21e52..2f8a8d4ab8 100644 --- a/ydb/core/persqueue/actor_persqueue_client_iface.h +++ b/ydb/core/persqueue/actor_persqueue_client_iface.h @@ -3,88 +3,88 @@ #include <ydb/core/protos/pqconfig.pb.h> #include <ydb/public/sdk/cpp/client/ydb_driver/driver.h> #include <ydb/public/sdk/cpp/client/ydb_persqueue_core/persqueue.h> - + #include <library/cpp/actors/core/actor.h> -#include <library/cpp/logger/log.h> +#include <library/cpp/logger/log.h> + +#include <util/datetime/base.h> -#include <util/datetime/base.h> - namespace NKikimr::NPQ { -class IPersQueueMirrorReaderFactory { +class IPersQueueMirrorReaderFactory { public: - virtual void Initialize(const NKikimrPQ::TPQConfig::TPQLibSettings& settings) const = 0; + virtual void Initialize(const NKikimrPQ::TPQConfig::TPQLibSettings& settings) const = 0; - virtual std::shared_ptr<NYdb::ICredentialsProviderFactory> GetCredentialsProvider( - const NKikimrPQ::TMirrorPartitionConfig::TCredentials& cred - ) const = 0; + virtual std::shared_ptr<NYdb::ICredentialsProviderFactory> GetCredentialsProvider( + const NKikimrPQ::TMirrorPartitionConfig::TCredentials& cred + ) const = 0; - virtual std::shared_ptr<NYdb::NPersQueue::IReadSession> GetReadSession( - const NKikimrPQ::TMirrorPartitionConfig& config, - ui32 partition, - std::shared_ptr<NYdb::ICredentialsProviderFactory> credentialsProviderFactory, - ui64 maxMemoryUsageBytes - ) const = 0; + virtual std::shared_ptr<NYdb::NPersQueue::IReadSession> GetReadSession( + const NKikimrPQ::TMirrorPartitionConfig& config, + ui32 partition, + std::shared_ptr<NYdb::ICredentialsProviderFactory> credentialsProviderFactory, + ui64 maxMemoryUsageBytes + ) const = 0; - virtual ~IPersQueueMirrorReaderFactory() = default; + virtual ~IPersQueueMirrorReaderFactory() = default; }; -class TPersQueueMirrorReaderFactory : public IPersQueueMirrorReaderFactory { +class TPersQueueMirrorReaderFactory : public IPersQueueMirrorReaderFactory { public: - void Initialize(const NKikimrPQ::TPQConfig::TPQLibSettings& settings) const override { - auto driverConfig = NYdb::TDriverConfig() - .SetNetworkThreadsNum(settings.GetThreadsCount()); - Driver = std::make_shared<NYdb::TDriver>(driverConfig); - } - - std::shared_ptr<NYdb::ICredentialsProviderFactory> GetCredentialsProvider( - const NKikimrPQ::TMirrorPartitionConfig::TCredentials& cred - ) const override { - switch (cred.GetCredentialsCase()) { - case NKikimrPQ::TMirrorPartitionConfig::TCredentials::CREDENTIALS_NOT_SET: { - return NYdb::CreateInsecureCredentialsProviderFactory(); - } - default: { - ythrow yexception() << "unsupported credentials type " << ui64(cred.GetCredentialsCase()); - } - } - } - - std::shared_ptr<NYdb::NPersQueue::IReadSession> GetReadSession( - const NKikimrPQ::TMirrorPartitionConfig& config, - ui32 partition, - std::shared_ptr<NYdb::ICredentialsProviderFactory> credentialsProviderFactory, - ui64 maxMemoryUsageBytes - ) const override { - NYdb::NPersQueue::TPersQueueClientSettings clientSettings = NYdb::NPersQueue::TPersQueueClientSettings() - .DiscoveryEndpoint(TStringBuilder() << config.GetEndpoint() << ":" << config.GetEndpointPort()) - .CredentialsProviderFactory(credentialsProviderFactory) - .EnableSsl(config.GetUseSecureConnection()); - if (config.HasDatabase()) { - clientSettings.Database(config.GetDatabase()); - } - - NYdb::NPersQueue::TReadSessionSettings settings = NYdb::NPersQueue::TReadSessionSettings() - .ConsumerName(config.GetConsumer()) - .MaxMemoryUsageBytes(maxMemoryUsageBytes) - .Decompress(false) - .DisableClusterDiscovery(true) - .ReadOnlyOriginal(true) - .RetryPolicy(NYdb::NPersQueue::IRetryPolicy::GetNoRetryPolicy()); - if (config.HasReadFromTimestampsMs()) { - settings.StartingMessageTimestamp(TInstant::MilliSeconds(config.GetReadFromTimestampsMs())); - } - NYdb::NPersQueue::TTopicReadSettings topicSettings(config.GetTopic()); - topicSettings.AppendPartitionGroupIds({partition + 1}); - settings.AppendTopics(topicSettings); - - NYdb::NPersQueue::TPersQueueClient persQueueClient(*Driver, clientSettings); - return persQueueClient.CreateReadSession(settings); - } - -private: - mutable std::shared_ptr<NYdb::TDriver> Driver; + void Initialize(const NKikimrPQ::TPQConfig::TPQLibSettings& settings) const override { + auto driverConfig = NYdb::TDriverConfig() + .SetNetworkThreadsNum(settings.GetThreadsCount()); + Driver = std::make_shared<NYdb::TDriver>(driverConfig); + } + + std::shared_ptr<NYdb::ICredentialsProviderFactory> GetCredentialsProvider( + const NKikimrPQ::TMirrorPartitionConfig::TCredentials& cred + ) const override { + switch (cred.GetCredentialsCase()) { + case NKikimrPQ::TMirrorPartitionConfig::TCredentials::CREDENTIALS_NOT_SET: { + return NYdb::CreateInsecureCredentialsProviderFactory(); + } + default: { + ythrow yexception() << "unsupported credentials type " << ui64(cred.GetCredentialsCase()); + } + } + } + + std::shared_ptr<NYdb::NPersQueue::IReadSession> GetReadSession( + const NKikimrPQ::TMirrorPartitionConfig& config, + ui32 partition, + std::shared_ptr<NYdb::ICredentialsProviderFactory> credentialsProviderFactory, + ui64 maxMemoryUsageBytes + ) const override { + NYdb::NPersQueue::TPersQueueClientSettings clientSettings = NYdb::NPersQueue::TPersQueueClientSettings() + .DiscoveryEndpoint(TStringBuilder() << config.GetEndpoint() << ":" << config.GetEndpointPort()) + .CredentialsProviderFactory(credentialsProviderFactory) + .EnableSsl(config.GetUseSecureConnection()); + if (config.HasDatabase()) { + clientSettings.Database(config.GetDatabase()); + } + + NYdb::NPersQueue::TReadSessionSettings settings = NYdb::NPersQueue::TReadSessionSettings() + .ConsumerName(config.GetConsumer()) + .MaxMemoryUsageBytes(maxMemoryUsageBytes) + .Decompress(false) + .DisableClusterDiscovery(true) + .ReadOnlyOriginal(true) + .RetryPolicy(NYdb::NPersQueue::IRetryPolicy::GetNoRetryPolicy()); + if (config.HasReadFromTimestampsMs()) { + settings.StartingMessageTimestamp(TInstant::MilliSeconds(config.GetReadFromTimestampsMs())); + } + NYdb::NPersQueue::TTopicReadSettings topicSettings(config.GetTopic()); + topicSettings.AppendPartitionGroupIds({partition + 1}); + settings.AppendTopics(topicSettings); + + NYdb::NPersQueue::TPersQueueClient persQueueClient(*Driver, clientSettings); + return persQueueClient.CreateReadSession(settings); + } + +private: + mutable std::shared_ptr<NYdb::TDriver> Driver; }; } // namespace NKikimr::NSQS diff --git a/ydb/core/persqueue/blob.cpp b/ydb/core/persqueue/blob.cpp index 044f02a32a..7727f12a13 100644 --- a/ydb/core/persqueue/blob.cpp +++ b/ydb/core/persqueue/blob.cpp @@ -1,7 +1,7 @@ #include "blob.h" #include "type_codecs.h" -#include <util/string/builder.h> +#include <util/string/builder.h> #include <util/string/escape.h> #include <util/system/unaligned_mem.h> @@ -97,10 +97,10 @@ void TClientBlob::Serialize(TBuffer& res) const res.Append(ExplicitHashKey.data(), ExplicitHashKey.size()); } - ui64 writeTimestampMs = WriteTimestamp.MilliSeconds(); - ui64 createTimestampMs = CreateTimestamp.MilliSeconds(); - res.Append((const char*)&writeTimestampMs, sizeof(ui64)); - res.Append((const char*)&createTimestampMs, sizeof(ui64)); + ui64 writeTimestampMs = WriteTimestamp.MilliSeconds(); + ui64 createTimestampMs = CreateTimestamp.MilliSeconds(); + res.Append((const char*)&writeTimestampMs, sizeof(ui64)); + res.Append((const char*)&createTimestampMs, sizeof(ui64)); if (outputUncompressedSize) res.Append((const char*)&(UncompressedSize), sizeof(ui32)); @@ -153,15 +153,15 @@ TClientBlob TClientBlob::Deserialize(const char* data, ui32 size) data += explicitHashKey.Size(); } - TInstant writeTimestamp; - TInstant createTimestamp; + TInstant writeTimestamp; + TInstant createTimestamp; ui32 us = 0; if (hasTS) { - writeTimestamp = TInstant::MilliSeconds(ReadUnaligned<ui64>(data)); + writeTimestamp = TInstant::MilliSeconds(ReadUnaligned<ui64>(data)); data += sizeof(ui64); } if (hasTS2) { - createTimestamp = TInstant::MilliSeconds(ReadUnaligned<ui64>(data)); + createTimestamp = TInstant::MilliSeconds(ReadUnaligned<ui64>(data)); data += sizeof(ui64); } if (hasUS) { @@ -317,8 +317,8 @@ void TBatch::Pack() { { auto chunk = MakeChunk<NScheme::TDeltaVarIntCodec<ui64, false>>(output); for (ui32 i = 0; i < Blobs.size(); ++i) { - ui64 writeTimestampMs = Blobs[i].WriteTimestamp.MilliSeconds(); - chunk->AddData((const char*)&writeTimestampMs, sizeof(ui64)); + ui64 writeTimestampMs = Blobs[i].WriteTimestamp.MilliSeconds(); + chunk->AddData((const char*)&writeTimestampMs, sizeof(ui64)); } OutputChunk(chunk, output, res); } @@ -345,8 +345,8 @@ void TBatch::Pack() { { auto chunk = MakeChunk<NScheme::TDeltaVarIntCodec<ui64, false>>(output); for (ui32 i = 0; i < Blobs.size(); ++i) { - ui64 createTimestampMs = Blobs[i].CreateTimestamp.MilliSeconds(); - chunk->AddData((const char*)&createTimestampMs, sizeof(ui64)); + ui64 createTimestampMs = Blobs[i].CreateTimestamp.MilliSeconds(); + chunk->AddData((const char*)&createTimestampMs, sizeof(ui64)); } OutputChunk(chunk, output, res); } @@ -495,17 +495,17 @@ void TBatch::UnpackToType1(TVector<TClientBlob> *blobs) { partData.insert(std::make_pair(ps, TPartData(partNo, totalParts, totalSize))); } } - TVector<TInstant> wtime; + TVector<TInstant> wtime; wtime.reserve(totalBlobs); { auto chunk = NScheme::IChunkDecoder::ReadChunk(GetChunk(data, dataEnd), &ui64Codecs); auto iter = chunk->MakeIterator(); for (ui32 i = 0; i < totalBlobs; ++i) { - ui64 timestampMs = *(ui64*)iter->Next().Data(); - wtime.push_back(TInstant::MilliSeconds(timestampMs)); + ui64 timestampMs = *(ui64*)iter->Next().Data(); + wtime.push_back(TInstant::MilliSeconds(timestampMs)); } } - TVector<TInstant> ctime; + TVector<TInstant> ctime; ctime.reserve(totalBlobs); TVector<TString> partitionKey; @@ -540,8 +540,8 @@ void TBatch::UnpackToType1(TVector<TClientBlob> *blobs) { auto chunk = NScheme::IChunkDecoder::ReadChunk(GetChunk(data, dataEnd), &ui64Codecs); auto iter = chunk->MakeIterator(); for (ui32 i = 0; i < totalBlobs; ++i) { - ui64 timestampMs = *(ui64*)iter->Next().Data(); - ctime.push_back(TInstant::MilliSeconds(timestampMs)); + ui64 timestampMs = *(ui64*)iter->Next().Data(); + ctime.push_back(TInstant::MilliSeconds(timestampMs)); } } } else { diff --git a/ydb/core/persqueue/blob.h b/ydb/core/persqueue/blob.h index c483b4aebf..633ec2499a 100644 --- a/ydb/core/persqueue/blob.h +++ b/ydb/core/persqueue/blob.h @@ -2,11 +2,11 @@ #include "header.h" #include "key.h" -#include <util/datetime/base.h> -#include <util/generic/maybe.h> -#include <util/generic/vector.h> - -#include <deque> +#include <util/datetime/base.h> +#include <util/generic/maybe.h> +#include <util/generic/vector.h> + +#include <deque> namespace NKikimr { namespace NPQ { @@ -38,8 +38,8 @@ struct TClientBlob { ui64 SeqNo; TString Data; TMaybe<TPartData> PartData; - TInstant WriteTimestamp; - TInstant CreateTimestamp; + TInstant WriteTimestamp; + TInstant CreateTimestamp; ui32 UncompressedSize; TString PartitionKey; TString ExplicitHashKey; @@ -49,7 +49,7 @@ struct TClientBlob { , UncompressedSize(0) {} - TClientBlob(const TString& sourceId, const ui64 seqNo, const TString& data, TMaybe<TPartData> &&partData, TInstant writeTimestamp, TInstant createTimestamp, + TClientBlob(const TString& sourceId, const ui64 seqNo, const TString& data, TMaybe<TPartData> &&partData, TInstant writeTimestamp, TInstant createTimestamp, const ui64 uncompressedSize, const TString& partitionKey, const TString& explicitHashKey) : SourceId(sourceId) , SeqNo(seqNo) diff --git a/ydb/core/persqueue/cache_eviction.h b/ydb/core/persqueue/cache_eviction.h index afe070a2d4..07ae210f85 100644 --- a/ydb/core/persqueue/cache_eviction.h +++ b/ydb/core/persqueue/cache_eviction.h @@ -1,5 +1,5 @@ #pragma once - + #include "blob.h" #include "pq_l2_service.h" diff --git a/ydb/core/persqueue/cluster_tracker.cpp b/ydb/core/persqueue/cluster_tracker.cpp index 60f4750c21..59fdea5c2b 100644 --- a/ydb/core/persqueue/cluster_tracker.cpp +++ b/ydb/core/persqueue/cluster_tracker.cpp @@ -174,10 +174,10 @@ private: cluster.IsLocal = t.GetList(i).GetStruct(2).GetOptional().GetBool(); cluster.IsEnabled = t.GetList(i).GetStruct(3).GetOptional().GetBool(); - cluster.Weight = t.GetList(i).GetStruct(4).GetOptional().GetUint64(); + cluster.Weight = t.GetList(i).GetStruct(4).GetOptional().GetUint64(); } - clustersList->Version = t.GetList(0).GetStruct(5).GetOptional().GetInt64(); + clustersList->Version = t.GetList(0).GetStruct(5).GetOptional().GetInt64(); ClustersList = std::move(clustersList); ClustersListUpdateTimestamp = Ctx().Now(); diff --git a/ydb/core/persqueue/config/ya.make b/ydb/core/persqueue/config/ya.make index 8ace455da0..08ff7c6de1 100644 --- a/ydb/core/persqueue/config/ya.make +++ b/ydb/core/persqueue/config/ya.make @@ -3,7 +3,7 @@ LIBRARY() OWNER( alexnick g:kikimr - g:logbroker + g:logbroker ) SRCS( diff --git a/ydb/core/persqueue/event_helpers.cpp b/ydb/core/persqueue/event_helpers.cpp index 392dc5e601..320002a936 100644 --- a/ydb/core/persqueue/event_helpers.cpp +++ b/ydb/core/persqueue/event_helpers.cpp @@ -1,48 +1,48 @@ -#include "event_helpers.h" - +#include "event_helpers.h" + #include <ydb/core/persqueue/events/internal.h> #include <ydb/core/protos/counters_pq.pb.h> - -#include <library/cpp/actors/core/log.h> - -#include <util/string/builder.h> - -namespace NKikimr { -namespace NPQ { - -void ReplyPersQueueError( - TActorId dstActor, - const TActorContext& ctx, - ui64 tabletId, - const TString& topicName, - TMaybe<ui32> partition, - NKikimr::TTabletCountersBase& counters, - NKikimrServices::EServiceKikimr service, - const ui64 responseCookie, - NPersQueue::NErrorCode::EErrorCode errorCode, + +#include <library/cpp/actors/core/log.h> + +#include <util/string/builder.h> + +namespace NKikimr { +namespace NPQ { + +void ReplyPersQueueError( + TActorId dstActor, + const TActorContext& ctx, + ui64 tabletId, + const TString& topicName, + TMaybe<ui32> partition, + NKikimr::TTabletCountersBase& counters, + NKikimrServices::EServiceKikimr service, + const ui64 responseCookie, + NPersQueue::NErrorCode::EErrorCode errorCode, const TString& error, bool logDebug -) { - if (errorCode == NPersQueue::NErrorCode::BAD_REQUEST) { - counters.Cumulative()[COUNTER_PQ_BAD_REQUEST].Increment(1); - } else if (errorCode == NPersQueue::NErrorCode::INITIALIZING) { - counters.Cumulative()[COUNTER_PQ_INITIALIZING].Increment(1); - } - - TStringBuilder logStr; - logStr << "tablet " << tabletId << " topic '" << topicName; - if (partition) { - logStr << "' partition " << partition; - } - logStr << " error: " << error; - +) { + if (errorCode == NPersQueue::NErrorCode::BAD_REQUEST) { + counters.Cumulative()[COUNTER_PQ_BAD_REQUEST].Increment(1); + } else if (errorCode == NPersQueue::NErrorCode::INITIALIZING) { + counters.Cumulative()[COUNTER_PQ_INITIALIZING].Increment(1); + } + + TStringBuilder logStr; + logStr << "tablet " << tabletId << " topic '" << topicName; + if (partition) { + logStr << "' partition " << partition; + } + logStr << " error: " << error; + if (logDebug) { LOG_DEBUG_S(ctx, service, logStr); } else { LOG_WARN_S(ctx, service, logStr); } - ctx.Send(dstActor, new TEvPQ::TEvError(errorCode, error, responseCookie)); -} - -}// NPQ -}// NKikimr + ctx.Send(dstActor, new TEvPQ::TEvError(errorCode, error, responseCookie)); +} + +}// NPQ +}// NKikimr diff --git a/ydb/core/persqueue/event_helpers.h b/ydb/core/persqueue/event_helpers.h index 9b855b9637..51bb814c92 100644 --- a/ydb/core/persqueue/event_helpers.h +++ b/ydb/core/persqueue/event_helpers.h @@ -1,27 +1,27 @@ -#pragma once - +#pragma once + #include <ydb/core/tablet/tablet_counters.h> #include <ydb/core/protos/services.pb.h> #include <ydb/public/api/protos/draft/persqueue_error_codes.pb.h> - -#include <library/cpp/actors/core/actor.h> - -namespace NKikimr { -namespace NPQ { - -void ReplyPersQueueError( - TActorId dstActor, - const TActorContext& ctx, - ui64 tabletId, - const TString& topicName, - TMaybe<ui32> partition, - NKikimr::TTabletCountersBase& counters, - NKikimrServices::EServiceKikimr service, - const ui64 responseCookie, - NPersQueue::NErrorCode::EErrorCode errorCode, + +#include <library/cpp/actors/core/actor.h> + +namespace NKikimr { +namespace NPQ { + +void ReplyPersQueueError( + TActorId dstActor, + const TActorContext& ctx, + ui64 tabletId, + const TString& topicName, + TMaybe<ui32> partition, + NKikimr::TTabletCountersBase& counters, + NKikimrServices::EServiceKikimr service, + const ui64 responseCookie, + NPersQueue::NErrorCode::EErrorCode errorCode, const TString& error, bool logDebug = false -); - -}// NPQ -}// NKikimr +); + +}// NPQ +}// NKikimr diff --git a/ydb/core/persqueue/events/events.cpp b/ydb/core/persqueue/events/events.cpp index 97707bebad..2b31588b56 100644 --- a/ydb/core/persqueue/events/events.cpp +++ b/ydb/core/persqueue/events/events.cpp @@ -1,2 +1,2 @@ -#include "global.h" -#include "internal.h" +#include "global.h" +#include "internal.h" diff --git a/ydb/core/persqueue/events/global.h b/ydb/core/persqueue/events/global.h index c1f623dce5..e1957e7a16 100644 --- a/ydb/core/persqueue/events/global.h +++ b/ydb/core/persqueue/events/global.h @@ -42,7 +42,7 @@ struct TEvPersQueue { EvError, EvGetPartitionIdForWrite, EvGetPartitionIdForWriteResponse, - EvReportPartitionError, + EvReportPartitionError, EvResponse = EvRequest + 256, EvInternalEvents = EvResponse + 256, EvEnd @@ -199,7 +199,7 @@ struct TEvPersQueue { }; struct TEvError : public TEventPB<TEvError, - NPersQueueCommon::TError, EvError> { + NPersQueueCommon::TError, EvError> { TEvError() {} }; @@ -213,10 +213,10 @@ struct TEvPersQueue { {}; }; - struct TEvReportPartitionError : public TEventPB<TEvReportPartitionError, NKikimrPQ::TStatusResponse::TErrorMessage, EvReportPartitionError> { - TEvReportPartitionError() - {} - }; - + struct TEvReportPartitionError : public TEventPB<TEvReportPartitionError, NKikimrPQ::TStatusResponse::TErrorMessage, EvReportPartitionError> { + TEvReportPartitionError() + {} + }; + }; } //NKikimr diff --git a/ydb/core/persqueue/events/internal.h b/ydb/core/persqueue/events/internal.h index f47b60d47a..bc892988b4 100644 --- a/ydb/core/persqueue/events/internal.h +++ b/ydb/core/persqueue/events/internal.h @@ -1,10 +1,10 @@ #pragma once -#include "global.h" - +#include "global.h" + #include <ydb/core/protos/pqconfig.pb.h> #include <ydb/core/tablet/tablet_counters.h> - + #include <library/cpp/actors/core/event_local.h> #include <library/cpp/actors/core/actorid.h> @@ -97,17 +97,17 @@ struct TEvPQ { EvRegisterMessageGroup, EvDeregisterMessageGroup, EvSplitMessageGroup, - EvUpdateCounters, - EvMirrorerCounters, - EvReadLimiterRequest, - EvReadLimiterResponse, - EvReadLimiterConsumed, - EvReadLimiterCounters, - EvRetryWrite, - EvInitCredentials, - EvCreateConsumer, - EvRequestPartitionStatus, - EvReaderEventArrived, + EvUpdateCounters, + EvMirrorerCounters, + EvReadLimiterRequest, + EvReadLimiterResponse, + EvReadLimiterConsumed, + EvReadLimiterCounters, + EvRetryWrite, + EvInitCredentials, + EvCreateConsumer, + EvRequestPartitionStatus, + EvReaderEventArrived, EvEnd }; @@ -580,44 +580,44 @@ struct TEvPQ { TVector<TDeregisterBody> Deregistrations; TVector<TRegisterBody> Registrations; }; - - struct TEvUpdateCounters : public TEventLocal<TEvUpdateCounters, EvUpdateCounters> { - TEvUpdateCounters() - {} - }; - - struct TEvMirrorerCounters : public TEventLocal<TEvMirrorerCounters, EvMirrorerCounters> { - TEvMirrorerCounters(const NKikimr::TTabletCountersBase& counters) { - Counters.Populate(counters); - } - - NKikimr::TTabletCountersBase Counters; - }; - - struct TEvRetryWrite : public TEventLocal<TEvRetryWrite, EvRetryWrite> { - TEvRetryWrite() - {} - }; - - struct TEvInitCredentials : public TEventLocal<TEvInitCredentials, EvInitCredentials> { - TEvInitCredentials() - {} - }; - - struct TEvCreateConsumer : public TEventLocal<TEvCreateConsumer, EvCreateConsumer> { - TEvCreateConsumer() - {} - }; - - struct TEvRequestPartitionStatus : public TEventLocal<TEvRequestPartitionStatus, EvRequestPartitionStatus> { - TEvRequestPartitionStatus() - {} - }; - - struct TEvReaderEventArrived : public TEventLocal<TEvReaderEventArrived, EvReaderEventArrived> { - TEvReaderEventArrived() - {} - }; + + struct TEvUpdateCounters : public TEventLocal<TEvUpdateCounters, EvUpdateCounters> { + TEvUpdateCounters() + {} + }; + + struct TEvMirrorerCounters : public TEventLocal<TEvMirrorerCounters, EvMirrorerCounters> { + TEvMirrorerCounters(const NKikimr::TTabletCountersBase& counters) { + Counters.Populate(counters); + } + + NKikimr::TTabletCountersBase Counters; + }; + + struct TEvRetryWrite : public TEventLocal<TEvRetryWrite, EvRetryWrite> { + TEvRetryWrite() + {} + }; + + struct TEvInitCredentials : public TEventLocal<TEvInitCredentials, EvInitCredentials> { + TEvInitCredentials() + {} + }; + + struct TEvCreateConsumer : public TEventLocal<TEvCreateConsumer, EvCreateConsumer> { + TEvCreateConsumer() + {} + }; + + struct TEvRequestPartitionStatus : public TEventLocal<TEvRequestPartitionStatus, EvRequestPartitionStatus> { + TEvRequestPartitionStatus() + {} + }; + + struct TEvReaderEventArrived : public TEventLocal<TEvReaderEventArrived, EvReaderEventArrived> { + TEvReaderEventArrived() + {} + }; }; } //NKikimr diff --git a/ydb/core/persqueue/events/ya.make b/ydb/core/persqueue/events/ya.make index acc4439f81..2fd0e80ca4 100644 --- a/ydb/core/persqueue/events/ya.make +++ b/ydb/core/persqueue/events/ya.make @@ -1,21 +1,21 @@ -LIBRARY() - -OWNER( - alexnick - g:kikimr -) - -SRCS( - events.cpp -) - -PEERDIR( - library/cpp/actors/core +LIBRARY() + +OWNER( + alexnick + g:kikimr +) + +SRCS( + events.cpp +) + +PEERDIR( + library/cpp/actors/core ydb/core/base ydb/core/keyvalue ydb/core/protos ydb/core/tablet ydb/public/api/protos -) - -END() +) + +END() diff --git a/ydb/core/persqueue/internals_ut.cpp b/ydb/core/persqueue/internals_ut.cpp index 24f6a1533f..d72ffe5e11 100644 --- a/ydb/core/persqueue/internals_ut.cpp +++ b/ydb/core/persqueue/internals_ut.cpp @@ -37,10 +37,10 @@ void Test(bool headCompacted, ui32 parts, ui32 partSize, ui32 leftInHead) TString value(102400, 'a'); head.Batches.push_back(TBatch(head.Offset, 0, TVector<TClientBlob>())); for (ui32 i = 0; i < 50; ++i) { - head.Batches.back().AddBlob(TClientBlob( - "sourceId" + TString(1,'a' + rand() % 26), i + 1, value, TMaybe<TPartData>(), + head.Batches.back().AddBlob(TClientBlob( + "sourceId" + TString(1,'a' + rand() % 26), i + 1, value, TMaybe<TPartData>(), TInstant::MilliSeconds(i + 1), TInstant::MilliSeconds(i + 1), 1, "", "" - )); + )); if (!headCompacted) all.push_back(head.Batches.back().Blobs.back()); } @@ -59,10 +59,10 @@ void Test(bool headCompacted, ui32 parts, ui32 partSize, ui32 leftInHead) newHead.Offset = head.GetNextOffset(); newHead.Batches.push_back(TBatch(newHead.Offset, 0, TVector<TClientBlob>())); for (ui32 i = 0; i < 10; ++i) { - newHead.Batches.back().AddBlob(TClientBlob( - "sourceId2", i + 1, value, TMaybe<TPartData>(), + newHead.Batches.back().AddBlob(TClientBlob( + "sourceId2", i + 1, value, TMaybe<TPartData>(), TInstant::MilliSeconds(i + 1000), TInstant::MilliSeconds(i + 1000), 1, "", "" - )); + )); all.push_back(newHead.Batches.back().Blobs.back()); //newHead always glued } newHead.PackedSize = newHead.Batches.back().GetUnpackedSize(); @@ -77,10 +77,10 @@ void Test(bool headCompacted, ui32 parts, ui32 partSize, ui32 leftInHead) UNIT_ASSERT(!blob.IsComplete()); UNIT_ASSERT(blob.IsNextPart("sourceId3", 1, i, &error)); TMaybe<TPartData> partData = TPartData(i, parts, value2.size()); - TClientBlob clientBlob( - "soruceId3", 1, value2, std::move(partData), + TClientBlob clientBlob( + "soruceId3", 1, value2, std::move(partData), TInstant::MilliSeconds(1), TInstant::MilliSeconds(1), 1, "", "" - ); + ); all.push_back(clientBlob); auto res = blob.Add(std::move(clientBlob)); if (!res.second.empty()) @@ -171,10 +171,10 @@ Y_UNIT_TEST(TestBatchPacking) { TString value(10, 'a'); TBatch batch; for (ui32 i = 0; i < 100; ++i) { - batch.AddBlob(TClientBlob( - "sourceId1", i + 1, value, TMaybe<TPartData>(), + batch.AddBlob(TClientBlob( + "sourceId1", i + 1, value, TMaybe<TPartData>(), TInstant::MilliSeconds(1), TInstant::MilliSeconds(1), 0, "", "" - )); + )); } batch.Pack(); TString s = batch.PackedData; @@ -189,10 +189,10 @@ Y_UNIT_TEST(TestBatchPacking) { Y_VERIFY(batch2.Blobs.size() == 100); TBatch batch3; - batch3.AddBlob(TClientBlob( - "sourceId", 999999999999999ll, "abacaba", TPartData{33,66,4000000000u}, + batch3.AddBlob(TClientBlob( + "sourceId", 999999999999999ll, "abacaba", TPartData{33,66,4000000000u}, TInstant::MilliSeconds(999999999999ll), TInstant::MilliSeconds(1000), 0, "", "" - )); + )); batch3.Pack(); UNIT_ASSERT(batch3.Header.GetFormat() == NKikimrPQ::TBatchHeader::EUncompressed); batch3.Unpack(); diff --git a/ydb/core/persqueue/key.h b/ydb/core/persqueue/key.h index ec3fe5ad85..098c67cf7d 100644 --- a/ydb/core/persqueue/key.h +++ b/ydb/core/persqueue/key.h @@ -1,8 +1,8 @@ #pragma once #include <util/generic/buffer.h> -#include <util/string/cast.h> -#include <util/string/printf.h> +#include <util/string/cast.h> +#include <util/string/printf.h> namespace NKikimr { namespace NPQ { diff --git a/ydb/core/persqueue/mirrorer.cpp b/ydb/core/persqueue/mirrorer.cpp index c5b720146a..37aea95ad1 100644 --- a/ydb/core/persqueue/mirrorer.cpp +++ b/ydb/core/persqueue/mirrorer.cpp @@ -1,557 +1,557 @@ -#include "mirrorer.h" -#include "write_meta.h" - +#include "mirrorer.h" +#include "write_meta.h" + #include <ydb/core/persqueue/writer/source_id_encoding.h> #include <ydb/core/protos/grpc_pq_old.pb.h> #include <ydb/library/persqueue/topic_parser/topic_parser.h> #include <ydb/core/base/counters.h> #include <ydb/core/util/yverify_stream.h> - -#include <library/cpp/time_provider/time_provider.h> - + +#include <library/cpp/time_provider/time_provider.h> + #include <google/protobuf/util/message_differencer.h> - -#include <util/string/join.h> - - -namespace NKikimr { -namespace NPQ { - -using TPersQueueReadEvent = NYdb::NPersQueue::TReadSessionEvent; - -constexpr NKikimrServices::TActivity::EType TMirrorer::ActorActivityType() { - return NKikimrServices::TActivity::PERSQUEUE_MIRRORER; -} - -TMirrorer::TMirrorer( - TActorId tabletActor, + +#include <util/string/join.h> + + +namespace NKikimr { +namespace NPQ { + +using TPersQueueReadEvent = NYdb::NPersQueue::TReadSessionEvent; + +constexpr NKikimrServices::TActivity::EType TMirrorer::ActorActivityType() { + return NKikimrServices::TActivity::PERSQUEUE_MIRRORER; +} + +TMirrorer::TMirrorer( + TActorId tabletActor, TActorId partitionActor, - const TString& topicName, - ui32 partition, - bool localDC, - ui64 endOffset, - const NKikimrPQ::TMirrorPartitionConfig& config, - const TTabletCountersBase& counters -) - : TabletActor(tabletActor) - , PartitionActor(partitionActor) - , TopicName(topicName) - , Partition(partition) - , LocalDC(localDC) - , EndOffset(endOffset) - , OffsetToRead(endOffset) - , Config(config) -{ - Counters.Populate(counters); -} - -void TMirrorer::Bootstrap(const TActorContext& ctx) { - Become(&TThis::StateInitConsumer); - - StartInit(ctx); - ctx.Schedule(UPDATE_COUNTERS_INTERVAL, new TEvPQ::TEvUpdateCounters); - - if (AppData(ctx)->Counters) { - if (TopicName.find("--") != TString::npos) { - TVector<std::pair<ui64, TString>> lagsIntervals{{100, "100ms"}, {200, "200ms"}, {500, "500ms"}, - {1000, "1000ms"}, {2000, "2000ms"}, {5000, "5000ms"}, {10000, "10000ms"}, - {30000, "30000ms"}, {60000, "60000ms"}, {180000,"180000ms"}, {9999999, "999999ms"}}; - - auto counters = AppData(ctx)->Counters; - TString suffix = LocalDC ? "Remote" : "Internal"; - MirrorerErrors = NKikimr::NPQ::TMultiCounter( + const TString& topicName, + ui32 partition, + bool localDC, + ui64 endOffset, + const NKikimrPQ::TMirrorPartitionConfig& config, + const TTabletCountersBase& counters +) + : TabletActor(tabletActor) + , PartitionActor(partitionActor) + , TopicName(topicName) + , Partition(partition) + , LocalDC(localDC) + , EndOffset(endOffset) + , OffsetToRead(endOffset) + , Config(config) +{ + Counters.Populate(counters); +} + +void TMirrorer::Bootstrap(const TActorContext& ctx) { + Become(&TThis::StateInitConsumer); + + StartInit(ctx); + ctx.Schedule(UPDATE_COUNTERS_INTERVAL, new TEvPQ::TEvUpdateCounters); + + if (AppData(ctx)->Counters) { + if (TopicName.find("--") != TString::npos) { + TVector<std::pair<ui64, TString>> lagsIntervals{{100, "100ms"}, {200, "200ms"}, {500, "500ms"}, + {1000, "1000ms"}, {2000, "2000ms"}, {5000, "5000ms"}, {10000, "10000ms"}, + {30000, "30000ms"}, {60000, "60000ms"}, {180000,"180000ms"}, {9999999, "999999ms"}}; + + auto counters = AppData(ctx)->Counters; + TString suffix = LocalDC ? "Remote" : "Internal"; + MirrorerErrors = NKikimr::NPQ::TMultiCounter( GetServiceCounters(counters, "pqproxy|writeSession"), GetLabels(TopicName), {}, {"MirrorerErrors" + suffix}, true - ); - MirrorerTimeLags = THolder<TPercentileCounter>(new TPercentileCounter( + ); + MirrorerTimeLags = THolder<TPercentileCounter>(new TPercentileCounter( GetServiceCounters(counters, "pqproxy|mirrorWriteTimeLag"), - GetLabels(TopicName), - {{"sensor", "TimeLags" + suffix}}, - "Interval", lagsIntervals, true - )); - InitTimeoutCounter = NKikimr::NPQ::TMultiCounter( - GetServiceCounters(counters, "pqproxy|writeSession"), + GetLabels(TopicName), + {{"sensor", "TimeLags" + suffix}}, + "Interval", lagsIntervals, true + )); + InitTimeoutCounter = NKikimr::NPQ::TMultiCounter( + GetServiceCounters(counters, "pqproxy|writeSession"), GetLabels(TopicName), {}, {"MirrorerInitTimeout" + suffix}, true - ); - WriteTimeoutCounter = NKikimr::NPQ::TMultiCounter( - GetServiceCounters(counters, "pqproxy|writeSession"), + ); + WriteTimeoutCounter = NKikimr::NPQ::TMultiCounter( + GetServiceCounters(counters, "pqproxy|writeSession"), {}, {}, {"MirrorerWriteTimeout"}, true, "sensor", false - ); - } - } -} - -void TMirrorer::StartInit(const TActorContext& ctx) { - Become(&TThis::StateInitConsumer); - LastInitStageTimestamp = ctx.Now(); - ctx.Send(SelfId(), new TEvPQ::TEvInitCredentials); -} - -void TMirrorer::Handle(TEvents::TEvPoisonPill::TPtr&, const TActorContext& ctx) { - LOG_NOTICE_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription() << " killed"); - ReadSession = nullptr; - PartitionStream = nullptr; + ); + } + } +} + +void TMirrorer::StartInit(const TActorContext& ctx) { + Become(&TThis::StateInitConsumer); + LastInitStageTimestamp = ctx.Now(); + ctx.Send(SelfId(), new TEvPQ::TEvInitCredentials); +} + +void TMirrorer::Handle(TEvents::TEvPoisonPill::TPtr&, const TActorContext& ctx) { + LOG_NOTICE_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription() << " killed"); + ReadSession = nullptr; + PartitionStream = nullptr; CredentialsProvider = nullptr; - Die(ctx); -} - -bool TMirrorer::AddToWriteRequest( - NKikimrClient::TPersQueuePartitionRequest& request, - TPersQueueReadEvent::TDataReceivedEvent::TCompressedMessage& message -) { - if (!request.HasCmdWriteOffset()) { - Y_VERIFY(request.CmdWriteSize() == 0); - request.SetCmdWriteOffset(message.GetOffset(0)); - } - if (request.GetCmdWriteOffset() + request.CmdWriteSize() != message.GetOffset(0)) { - return false; - } - - auto write = request.AddCmdWrite(); - write->SetData(GetSerializedData(message)); - write->SetSourceId(NSourceIdEncoding::EncodeSimple(message.GetMessageGroupId(0))); - write->SetSeqNo(message.GetSeqNo(0)); - write->SetCreateTimeMS(message.GetCreateTime(0).MilliSeconds()); - if (Config.GetSyncWriteTime()) { - write->SetWriteTimeMS(message.GetWriteTime(0).MilliSeconds()); - } - write->SetDisableDeduplication(true); - write->SetUncompressedSize(message.GetUncompressedSize(0)); - return true; -} - -void TMirrorer::ProcessError(const TActorContext& ctx, const TString& msg) { - if (MirrorerErrors) { - MirrorerErrors.Inc(1); - } - + Die(ctx); +} + +bool TMirrorer::AddToWriteRequest( + NKikimrClient::TPersQueuePartitionRequest& request, + TPersQueueReadEvent::TDataReceivedEvent::TCompressedMessage& message +) { + if (!request.HasCmdWriteOffset()) { + Y_VERIFY(request.CmdWriteSize() == 0); + request.SetCmdWriteOffset(message.GetOffset(0)); + } + if (request.GetCmdWriteOffset() + request.CmdWriteSize() != message.GetOffset(0)) { + return false; + } + + auto write = request.AddCmdWrite(); + write->SetData(GetSerializedData(message)); + write->SetSourceId(NSourceIdEncoding::EncodeSimple(message.GetMessageGroupId(0))); + write->SetSeqNo(message.GetSeqNo(0)); + write->SetCreateTimeMS(message.GetCreateTime(0).MilliSeconds()); + if (Config.GetSyncWriteTime()) { + write->SetWriteTimeMS(message.GetWriteTime(0).MilliSeconds()); + } + write->SetDisableDeduplication(true); + write->SetUncompressedSize(message.GetUncompressedSize(0)); + return true; +} + +void TMirrorer::ProcessError(const TActorContext& ctx, const TString& msg) { + if (MirrorerErrors) { + MirrorerErrors.Inc(1); + } + THolder<TEvPersQueue::TEvReportPartitionError> request = MakeHolder<TEvPersQueue::TEvReportPartitionError>(); - auto& record = request->Record; - record.SetTimestamp(ctx.Now().Seconds()); - record.SetService(NKikimrServices::PQ_MIRRORER); - record.SetMessage(TStringBuilder() << MirrorerDescription() << ": " << msg); - - Send(PartitionActor, request.Release()); -} - -void TMirrorer::ProcessError(const TActorContext& ctx, const TString& msg, const NKikimrClient::TResponse& response) { - ProcessError(ctx, msg); - - if (response.HasPartitionResponse()) { - bool isWriteRequest = response.GetPartitionResponse().HasCookie() && response.GetPartitionResponse().GetCookie() == EEventCookie::WRITE_REQUEST_COOKIE; - if (isWriteRequest) { - ScheduleWithIncreasingTimeout<TEvPQ::TEvRetryWrite>(SelfId(), WriteRetryTimeout, WRITE_RETRY_TIMEOUT_MAX, ctx); - } - } -} - -void TMirrorer::AfterSuccesWrite(const TActorContext& ctx) { - Y_VERIFY(WriteInFlight.empty()); - Y_VERIFY(WriteRequestInFlight); - LOG_INFO_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription() - << " written " << WriteRequestInFlight.value().CmdWriteSize() - << " messages with first offset=" << WriteRequestInFlight.value().GetCmdWriteOffset() - << ", current queue size: " << Queue.size() << "(" << BytesInFlight << "bytes)"); - - WriteRequestInFlight.reset(); - Send(SelfId(), new TEvents::TEvWakeup()); - - WriteRetryTimeout = WRITE_RETRY_TIMEOUT_START; - - TryUpdateWriteTimetsamp(ctx); -} - -void TMirrorer::ProcessWriteResponse( - const TActorContext& ctx, - const NKikimrClient::TPersQueuePartitionResponse& response -) { - Y_VERIFY_S(response.CmdWriteResultSize() == WriteInFlight.size(), MirrorerDescription() - << "CmdWriteResultSize=" << response.CmdWriteResultSize() << ", but expected=" << WriteInFlight.size() - << ". First expected offset= " << (WriteInFlight.empty() ? -1 : WriteInFlight.front().Offset) - << " response: " << response); - - - NYdb::NPersQueue::TDeferredCommit deferredCommit; - for (auto& result : response.GetCmdWriteResult()) { - if (result.GetAlreadyWritten()) { - Y_VERIFY_S( - !WasSuccessfulRecording, - MirrorerDescription() << "got write result 'already written'," - <<" but we have successful recording before: " << result - ); - ProcessError(ctx, TStringBuilder() << "got write result 'already written': " << result); - } else { - WasSuccessfulRecording = true; - } - auto& writtenMessageInfo = WriteInFlight.front(); - if (MirrorerTimeLags) { - TDuration lag = TInstant::MilliSeconds(result.GetWriteTimestampMS()) - writtenMessageInfo.WriteTime; - MirrorerTimeLags->IncFor(lag.MilliSeconds(), 1); - } - ui64 offset = writtenMessageInfo.Offset; - Y_VERIFY((ui64)result.GetOffset() == offset); - Y_VERIFY_S(EndOffset <= offset, MirrorerDescription() - << "end offset more the written " << EndOffset << ">" << offset); - EndOffset = offset + 1; - BytesInFlight -= writtenMessageInfo.Size; - - if (PartitionStream) { - deferredCommit.Add(PartitionStream, offset); - } - WriteInFlight.pop_front(); - } - deferredCommit.Commit(); - AfterSuccesWrite(ctx); -} - -void TMirrorer::Handle(TEvPersQueue::TEvResponse::TPtr& ev, const TActorContext& ctx) { - auto& response = ev->Get()->Record; - if (response.GetStatus() != NMsgBusProxy::MSTATUS_OK) { - ProcessError(ctx, "status is not ok: " + response.GetErrorReason(), response); - return; - } - if (response.GetErrorCode() != NPersQueue::NErrorCode::OK) { - ProcessError(ctx, response.GetErrorReason(), response); - return; - } - if (response.HasPartitionResponse()) { - const auto& partitionResponse = response.GetPartitionResponse(); - switch(partitionResponse.GetCookie()) { - case EEventCookie::WRITE_REQUEST_COOKIE: { - ProcessWriteResponse(ctx, partitionResponse); - return; - } - case EEventCookie::UPDATE_WRITE_TIMESTAMP: { - LOG_DEBUG_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription() - << " got response to update write timestamp request: " << partitionResponse); - return; - } - default: { - ProcessError(ctx, TStringBuilder() << "unexpected partition response: " << response); - return; - } - } - } - ProcessError(ctx, TStringBuilder() << "unexpected response: " << response); -} - -void TMirrorer::Handle(TEvPQ::TEvUpdateCounters::TPtr& /*ev*/, const TActorContext& ctx) { - ctx.Schedule(UPDATE_COUNTERS_INTERVAL, new TEvPQ::TEvUpdateCounters); - ctx.Send(PartitionActor, new TEvPQ::TEvMirrorerCounters(Counters)); - - if (ctx.Now() - LastStateLogTimestamp > LOG_STATE_INTERVAL) { - LastStateLogTimestamp = ctx.Now(); - LOG_NOTICE_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription() - << "[STATE] current state=" << GetCurrentState() - << ", read session=" << bool(ReadSession) << ", credentials provider=" << bool(CredentialsProvider)); - if (ReadSession) { - LOG_NOTICE_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription() - << "[STATE] read session id " << ReadSession->GetSessionId()); - } - if (PartitionStream) { - LOG_NOTICE_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription() + auto& record = request->Record; + record.SetTimestamp(ctx.Now().Seconds()); + record.SetService(NKikimrServices::PQ_MIRRORER); + record.SetMessage(TStringBuilder() << MirrorerDescription() << ": " << msg); + + Send(PartitionActor, request.Release()); +} + +void TMirrorer::ProcessError(const TActorContext& ctx, const TString& msg, const NKikimrClient::TResponse& response) { + ProcessError(ctx, msg); + + if (response.HasPartitionResponse()) { + bool isWriteRequest = response.GetPartitionResponse().HasCookie() && response.GetPartitionResponse().GetCookie() == EEventCookie::WRITE_REQUEST_COOKIE; + if (isWriteRequest) { + ScheduleWithIncreasingTimeout<TEvPQ::TEvRetryWrite>(SelfId(), WriteRetryTimeout, WRITE_RETRY_TIMEOUT_MAX, ctx); + } + } +} + +void TMirrorer::AfterSuccesWrite(const TActorContext& ctx) { + Y_VERIFY(WriteInFlight.empty()); + Y_VERIFY(WriteRequestInFlight); + LOG_INFO_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription() + << " written " << WriteRequestInFlight.value().CmdWriteSize() + << " messages with first offset=" << WriteRequestInFlight.value().GetCmdWriteOffset() + << ", current queue size: " << Queue.size() << "(" << BytesInFlight << "bytes)"); + + WriteRequestInFlight.reset(); + Send(SelfId(), new TEvents::TEvWakeup()); + + WriteRetryTimeout = WRITE_RETRY_TIMEOUT_START; + + TryUpdateWriteTimetsamp(ctx); +} + +void TMirrorer::ProcessWriteResponse( + const TActorContext& ctx, + const NKikimrClient::TPersQueuePartitionResponse& response +) { + Y_VERIFY_S(response.CmdWriteResultSize() == WriteInFlight.size(), MirrorerDescription() + << "CmdWriteResultSize=" << response.CmdWriteResultSize() << ", but expected=" << WriteInFlight.size() + << ". First expected offset= " << (WriteInFlight.empty() ? -1 : WriteInFlight.front().Offset) + << " response: " << response); + + + NYdb::NPersQueue::TDeferredCommit deferredCommit; + for (auto& result : response.GetCmdWriteResult()) { + if (result.GetAlreadyWritten()) { + Y_VERIFY_S( + !WasSuccessfulRecording, + MirrorerDescription() << "got write result 'already written'," + <<" but we have successful recording before: " << result + ); + ProcessError(ctx, TStringBuilder() << "got write result 'already written': " << result); + } else { + WasSuccessfulRecording = true; + } + auto& writtenMessageInfo = WriteInFlight.front(); + if (MirrorerTimeLags) { + TDuration lag = TInstant::MilliSeconds(result.GetWriteTimestampMS()) - writtenMessageInfo.WriteTime; + MirrorerTimeLags->IncFor(lag.MilliSeconds(), 1); + } + ui64 offset = writtenMessageInfo.Offset; + Y_VERIFY((ui64)result.GetOffset() == offset); + Y_VERIFY_S(EndOffset <= offset, MirrorerDescription() + << "end offset more the written " << EndOffset << ">" << offset); + EndOffset = offset + 1; + BytesInFlight -= writtenMessageInfo.Size; + + if (PartitionStream) { + deferredCommit.Add(PartitionStream, offset); + } + WriteInFlight.pop_front(); + } + deferredCommit.Commit(); + AfterSuccesWrite(ctx); +} + +void TMirrorer::Handle(TEvPersQueue::TEvResponse::TPtr& ev, const TActorContext& ctx) { + auto& response = ev->Get()->Record; + if (response.GetStatus() != NMsgBusProxy::MSTATUS_OK) { + ProcessError(ctx, "status is not ok: " + response.GetErrorReason(), response); + return; + } + if (response.GetErrorCode() != NPersQueue::NErrorCode::OK) { + ProcessError(ctx, response.GetErrorReason(), response); + return; + } + if (response.HasPartitionResponse()) { + const auto& partitionResponse = response.GetPartitionResponse(); + switch(partitionResponse.GetCookie()) { + case EEventCookie::WRITE_REQUEST_COOKIE: { + ProcessWriteResponse(ctx, partitionResponse); + return; + } + case EEventCookie::UPDATE_WRITE_TIMESTAMP: { + LOG_DEBUG_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription() + << " got response to update write timestamp request: " << partitionResponse); + return; + } + default: { + ProcessError(ctx, TStringBuilder() << "unexpected partition response: " << response); + return; + } + } + } + ProcessError(ctx, TStringBuilder() << "unexpected response: " << response); +} + +void TMirrorer::Handle(TEvPQ::TEvUpdateCounters::TPtr& /*ev*/, const TActorContext& ctx) { + ctx.Schedule(UPDATE_COUNTERS_INTERVAL, new TEvPQ::TEvUpdateCounters); + ctx.Send(PartitionActor, new TEvPQ::TEvMirrorerCounters(Counters)); + + if (ctx.Now() - LastStateLogTimestamp > LOG_STATE_INTERVAL) { + LastStateLogTimestamp = ctx.Now(); + LOG_NOTICE_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription() + << "[STATE] current state=" << GetCurrentState() + << ", read session=" << bool(ReadSession) << ", credentials provider=" << bool(CredentialsProvider)); + if (ReadSession) { + LOG_NOTICE_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription() + << "[STATE] read session id " << ReadSession->GetSessionId()); + } + if (PartitionStream) { + LOG_NOTICE_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription() << "[STATE] has partition stream " << PartitionStream->GetTopicPath() - << ":" << PartitionStream->GetPartitionId() - << " with id " << PartitionStream->GetPartitionStreamId()); - } else { - LOG_NOTICE_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription() << "[STATE] hasn't partition stream"); - } - if (StreamStatus) { - LOG_NOTICE_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription() - << "[STATE] last source partition status: " << StreamStatus->DebugString()); - } - LOG_NOTICE_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription() - << "[STATE] next offset to read " << OffsetToRead << ", current end offset " << EndOffset); - LOG_NOTICE_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription() - << "[STATE] bytes in flight " << BytesInFlight - << ", messages in write request " << WriteInFlight.size() - << ", queue to write: " << Queue.size()); - } - if (!ReadSession && LastInitStageTimestamp + INIT_TIMEOUT < ctx.Now()) { - ProcessError(ctx, TStringBuilder() << "read session was not created, the last stage of initialization was at " - << LastInitStageTimestamp.Seconds()); - if (InitTimeoutCounter) { - InitTimeoutCounter.Inc(1); - } - StartInit(ctx); - } - if (WriteRequestInFlight && WriteRequestTimestamp + WRITE_TIMEOUT < ctx.Now()) { - LOG_ERROR_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription() << " write request was sent at " - << WriteRequestTimestamp.Seconds() << ", but no response has been received yet. Tablet will be killed."); - if (WriteTimeoutCounter) { - WriteTimeoutCounter.Inc(1); - } - ctx.Send(TabletActor, new TEvents::TEvPoisonPill()); - } -} - -void TMirrorer::HandleChangeConfig(TEvPQ::TEvChangeConfig::TPtr& ev, const TActorContext& ctx) { - bool equalConfigs = google::protobuf::util::MessageDifferencer::Equals( - Config, - ev->Get()->Config.GetPartitionConfig().GetMirrorFrom() - ); - LOG_NOTICE_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription() << " got new config, equal with previous: " << equalConfigs); - if (!equalConfigs) { - Config = ev->Get()->Config.GetPartitionConfig().GetMirrorFrom(); - LOG_NOTICE_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription() << " changing config"); - - Become(&TThis::StateInitConsumer); - ctx.Send(SelfId(), new TEvPQ::TEvInitCredentials); - } -} - -void TMirrorer::TryToRead(const TActorContext& ctx) { - if (BytesInFlight < MAX_BYTES_IN_FLIGHT && ReadSession) { - StartWaitNextReaderEvent(ctx); - } -} - -void TMirrorer::TryToWrite(const TActorContext& ctx) { - if (WriteRequestInFlight || Queue.empty()) { - return; - } - + << ":" << PartitionStream->GetPartitionId() + << " with id " << PartitionStream->GetPartitionStreamId()); + } else { + LOG_NOTICE_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription() << "[STATE] hasn't partition stream"); + } + if (StreamStatus) { + LOG_NOTICE_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription() + << "[STATE] last source partition status: " << StreamStatus->DebugString()); + } + LOG_NOTICE_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription() + << "[STATE] next offset to read " << OffsetToRead << ", current end offset " << EndOffset); + LOG_NOTICE_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription() + << "[STATE] bytes in flight " << BytesInFlight + << ", messages in write request " << WriteInFlight.size() + << ", queue to write: " << Queue.size()); + } + if (!ReadSession && LastInitStageTimestamp + INIT_TIMEOUT < ctx.Now()) { + ProcessError(ctx, TStringBuilder() << "read session was not created, the last stage of initialization was at " + << LastInitStageTimestamp.Seconds()); + if (InitTimeoutCounter) { + InitTimeoutCounter.Inc(1); + } + StartInit(ctx); + } + if (WriteRequestInFlight && WriteRequestTimestamp + WRITE_TIMEOUT < ctx.Now()) { + LOG_ERROR_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription() << " write request was sent at " + << WriteRequestTimestamp.Seconds() << ", but no response has been received yet. Tablet will be killed."); + if (WriteTimeoutCounter) { + WriteTimeoutCounter.Inc(1); + } + ctx.Send(TabletActor, new TEvents::TEvPoisonPill()); + } +} + +void TMirrorer::HandleChangeConfig(TEvPQ::TEvChangeConfig::TPtr& ev, const TActorContext& ctx) { + bool equalConfigs = google::protobuf::util::MessageDifferencer::Equals( + Config, + ev->Get()->Config.GetPartitionConfig().GetMirrorFrom() + ); + LOG_NOTICE_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription() << " got new config, equal with previous: " << equalConfigs); + if (!equalConfigs) { + Config = ev->Get()->Config.GetPartitionConfig().GetMirrorFrom(); + LOG_NOTICE_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription() << " changing config"); + + Become(&TThis::StateInitConsumer); + ctx.Send(SelfId(), new TEvPQ::TEvInitCredentials); + } +} + +void TMirrorer::TryToRead(const TActorContext& ctx) { + if (BytesInFlight < MAX_BYTES_IN_FLIGHT && ReadSession) { + StartWaitNextReaderEvent(ctx); + } +} + +void TMirrorer::TryToWrite(const TActorContext& ctx) { + if (WriteRequestInFlight || Queue.empty()) { + return; + } + THolder<TEvPersQueue::TEvRequest> request = MakeHolder<TEvPersQueue::TEvRequest>(); - auto req = request->Record.MutablePartitionRequest(); - req->SetTopic(TopicName); - req->SetPartition(Partition); - req->SetMessageNo(0); - req->SetCookie(WRITE_REQUEST_COOKIE); - - while (!Queue.empty() && AddToWriteRequest(*req, Queue.front())) { - auto& message = Queue.front(); - ui64 dataSize = message.GetData().size(); - WriteInFlight.emplace_back(message.GetOffset(0), dataSize, message.GetWriteTime(0)); - Queue.pop_front(); - } - - WriteRequestInFlight = request->Record.GetPartitionRequest(); - - Send(TabletActor, request.Release()); - WriteRequestTimestamp = ctx.Now(); -} - - -void TMirrorer::HandleInitCredentials(TEvPQ::TEvInitCredentials::TPtr& /*ev*/, const TActorContext& ctx) { - LastInitStageTimestamp = ctx.Now(); - try { + auto req = request->Record.MutablePartitionRequest(); + req->SetTopic(TopicName); + req->SetPartition(Partition); + req->SetMessageNo(0); + req->SetCookie(WRITE_REQUEST_COOKIE); + + while (!Queue.empty() && AddToWriteRequest(*req, Queue.front())) { + auto& message = Queue.front(); + ui64 dataSize = message.GetData().size(); + WriteInFlight.emplace_back(message.GetOffset(0), dataSize, message.GetWriteTime(0)); + Queue.pop_front(); + } + + WriteRequestInFlight = request->Record.GetPartitionRequest(); + + Send(TabletActor, request.Release()); + WriteRequestTimestamp = ctx.Now(); +} + + +void TMirrorer::HandleInitCredentials(TEvPQ::TEvInitCredentials::TPtr& /*ev*/, const TActorContext& ctx) { + LastInitStageTimestamp = ctx.Now(); + try { RecreateCredentialsProvider(ctx); - } catch(...) { - ProcessError(ctx, "cannot initialize credentials provider: " + CurrentExceptionMessage()); - ScheduleWithIncreasingTimeout<TEvPQ::TEvInitCredentials>(SelfId(), ConsumerInitInterval, CONSUMER_INIT_INTERVAL_MAX, ctx); - return; - } - LOG_NOTICE_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription() << " credentials provider created"); - ConsumerInitInterval = CONSUMER_INIT_INTERVAL_START; - ScheduleConsumerCreation(ctx); -} - -void TMirrorer::RetryWrite(const TActorContext& ctx) { - Y_VERIFY(WriteRequestInFlight); - + } catch(...) { + ProcessError(ctx, "cannot initialize credentials provider: " + CurrentExceptionMessage()); + ScheduleWithIncreasingTimeout<TEvPQ::TEvInitCredentials>(SelfId(), ConsumerInitInterval, CONSUMER_INIT_INTERVAL_MAX, ctx); + return; + } + LOG_NOTICE_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription() << " credentials provider created"); + ConsumerInitInterval = CONSUMER_INIT_INTERVAL_START; + ScheduleConsumerCreation(ctx); +} + +void TMirrorer::RetryWrite(const TActorContext& ctx) { + Y_VERIFY(WriteRequestInFlight); + THolder<TEvPersQueue::TEvRequest> request = MakeHolder<TEvPersQueue::TEvRequest>(); - auto req = request->Record.MutablePartitionRequest(); - req->CopyFrom(WriteRequestInFlight.value()); - - Send(TabletActor, request.Release()); - WriteRequestTimestamp = ctx.Now(); -} - -void TMirrorer::HandleRetryWrite(TEvPQ::TEvRetryWrite::TPtr& /*ev*/, const TActorContext& ctx) { - RetryWrite(ctx); -} - -void TMirrorer::HandleWakeup(const TActorContext& ctx) { - TryToRead(ctx); - TryToWrite(ctx); -} - -void TMirrorer::CreateConsumer(TEvPQ::TEvCreateConsumer::TPtr&, const TActorContext& ctx) { - LastInitStageTimestamp = ctx.Now(); - LOG_NOTICE_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription() << " creating new read session"); - - if (!Queue.empty()) { - OffsetToRead = Queue.front().GetOffset(0); - while (!Queue.empty()) { - ui64 dataSize = Queue.back().GetData().size(); - Y_VERIFY(BytesInFlight >= dataSize); - BytesInFlight -= dataSize; - Queue.pop_back(); - } - } - - ReadSession.reset(); - PartitionStream.Reset(); - - auto factory = AppData(ctx)->PersQueueMirrorReaderFactory; + auto req = request->Record.MutablePartitionRequest(); + req->CopyFrom(WriteRequestInFlight.value()); + + Send(TabletActor, request.Release()); + WriteRequestTimestamp = ctx.Now(); +} + +void TMirrorer::HandleRetryWrite(TEvPQ::TEvRetryWrite::TPtr& /*ev*/, const TActorContext& ctx) { + RetryWrite(ctx); +} + +void TMirrorer::HandleWakeup(const TActorContext& ctx) { + TryToRead(ctx); + TryToWrite(ctx); +} + +void TMirrorer::CreateConsumer(TEvPQ::TEvCreateConsumer::TPtr&, const TActorContext& ctx) { + LastInitStageTimestamp = ctx.Now(); + LOG_NOTICE_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription() << " creating new read session"); + + if (!Queue.empty()) { + OffsetToRead = Queue.front().GetOffset(0); + while (!Queue.empty()) { + ui64 dataSize = Queue.back().GetData().size(); + Y_VERIFY(BytesInFlight >= dataSize); + BytesInFlight -= dataSize; + Queue.pop_back(); + } + } + + ReadSession.reset(); + PartitionStream.Reset(); + + auto factory = AppData(ctx)->PersQueueMirrorReaderFactory; Y_VERIFY(factory); - - ReadSession = factory->GetReadSession(Config, Partition, CredentialsProvider, MAX_BYTES_IN_FLIGHT); - - LOG_NOTICE_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription() - << " read session created: " << ReadSession->GetSessionId()); - - Send(SelfId(), new TEvents::TEvWakeup()); - Become(&TThis::StateWork); -} - -void TMirrorer::RequestSourcePartitionStatus(TEvPQ::TEvRequestPartitionStatus::TPtr&, const TActorContext&) { - RequestSourcePartitionStatus(); -} - -void TMirrorer::RequestSourcePartitionStatus() { - if (Config.GetSyncWriteTime() && PartitionStream && ReadSession) { - PartitionStream->RequestStatus(); - } -} - -void TMirrorer::TryUpdateWriteTimetsamp(const TActorContext &ctx) { - if (!WriteRequestInFlight && StreamStatus && EndOffset == StreamStatus->GetEndOffset()) { - LOG_INFO_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription() - << " update write timestamp from original topic: " << StreamStatus->DebugString()); - THolder<TEvPersQueue::TEvRequest> request = MakeHolder<TEvPersQueue::TEvRequest>(); - auto req = request->Record.MutablePartitionRequest(); - req->SetTopic(TopicName); - req->SetPartition(Partition); - req->SetCookie(UPDATE_WRITE_TIMESTAMP); - req->MutableCmdUpdateWriteTimestamp()->SetWriteTimeMS(StreamStatus->GetWriteWatermark().MilliSeconds()); - ctx.Send(TabletActor, request.Release()); - } -} - -void TMirrorer::AddMessagesToQueue(TVector<TPersQueueReadEvent::TDataReceivedEvent::TCompressedMessage>&& messages) { - for (auto& msg : messages) { - Y_VERIFY(msg.GetBlocksCount() == 1); // TODO support several compressed messages - ui64 offset = msg.GetOffset(0); - Y_VERIFY(OffsetToRead <= offset); - ui64 messageSize = msg.GetData().size(); - - Counters.Cumulative()[COUNTER_PQ_TABLET_NETWORK_BYTES_USAGE].Increment(messageSize); - BytesInFlight += messageSize; - - OffsetToRead = offset + 1; - Queue.emplace_back(std::move(msg)); - } -} - + + ReadSession = factory->GetReadSession(Config, Partition, CredentialsProvider, MAX_BYTES_IN_FLIGHT); + + LOG_NOTICE_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription() + << " read session created: " << ReadSession->GetSessionId()); + + Send(SelfId(), new TEvents::TEvWakeup()); + Become(&TThis::StateWork); +} + +void TMirrorer::RequestSourcePartitionStatus(TEvPQ::TEvRequestPartitionStatus::TPtr&, const TActorContext&) { + RequestSourcePartitionStatus(); +} + +void TMirrorer::RequestSourcePartitionStatus() { + if (Config.GetSyncWriteTime() && PartitionStream && ReadSession) { + PartitionStream->RequestStatus(); + } +} + +void TMirrorer::TryUpdateWriteTimetsamp(const TActorContext &ctx) { + if (!WriteRequestInFlight && StreamStatus && EndOffset == StreamStatus->GetEndOffset()) { + LOG_INFO_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription() + << " update write timestamp from original topic: " << StreamStatus->DebugString()); + THolder<TEvPersQueue::TEvRequest> request = MakeHolder<TEvPersQueue::TEvRequest>(); + auto req = request->Record.MutablePartitionRequest(); + req->SetTopic(TopicName); + req->SetPartition(Partition); + req->SetCookie(UPDATE_WRITE_TIMESTAMP); + req->MutableCmdUpdateWriteTimestamp()->SetWriteTimeMS(StreamStatus->GetWriteWatermark().MilliSeconds()); + ctx.Send(TabletActor, request.Release()); + } +} + +void TMirrorer::AddMessagesToQueue(TVector<TPersQueueReadEvent::TDataReceivedEvent::TCompressedMessage>&& messages) { + for (auto& msg : messages) { + Y_VERIFY(msg.GetBlocksCount() == 1); // TODO support several compressed messages + ui64 offset = msg.GetOffset(0); + Y_VERIFY(OffsetToRead <= offset); + ui64 messageSize = msg.GetData().size(); + + Counters.Cumulative()[COUNTER_PQ_TABLET_NETWORK_BYTES_USAGE].Increment(messageSize); + BytesInFlight += messageSize; + + OffsetToRead = offset + 1; + Queue.emplace_back(std::move(msg)); + } +} + void TMirrorer::ScheduleConsumerCreation(const TActorContext& ctx) { - LastInitStageTimestamp = ctx.Now(); - ReadSession = nullptr; - PartitionStream = nullptr; - Become(&TThis::StateInitConsumer); - - LOG_NOTICE_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription() << " schedule consumer creation"); - ScheduleWithIncreasingTimeout<TEvPQ::TEvCreateConsumer>(SelfId(), ConsumerInitInterval, CONSUMER_INIT_INTERVAL_MAX, ctx); -} - + LastInitStageTimestamp = ctx.Now(); + ReadSession = nullptr; + PartitionStream = nullptr; + Become(&TThis::StateInitConsumer); + + LOG_NOTICE_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription() << " schedule consumer creation"); + ScheduleWithIncreasingTimeout<TEvPQ::TEvCreateConsumer>(SelfId(), ConsumerInitInterval, CONSUMER_INIT_INTERVAL_MAX, ctx); +} + void TMirrorer::RecreateCredentialsProvider(const TActorContext& ctx) { CredentialsProvider = nullptr; - auto factory = AppData(ctx)->PersQueueMirrorReaderFactory; - Y_VERIFY(factory); - CredentialsProvider = factory->GetCredentialsProvider(Config.GetCredentials()); -} - -TString TMirrorer::MirrorerDescription() const { - return TStringBuilder() << "[mirrorer for " << TopicName << ':' << Partition << ']'; -} - -TString TMirrorer::GetCurrentState() const { - if (CurrentStateFunc() == &TThis::StateInitConsumer) { - return "StateInitConsumer"; - } else if (CurrentStateFunc() == &TThis::StateWork) { - return "StateWork"; - } - return "UNKNOWN"; -} - -void TMirrorer::StartWaitNextReaderEvent(const TActorContext& ctx) { - if (WaitNextReaderEventInFlight) { - return; - } - auto future = ReadSession->WaitEvent(); - future.Subscribe( - [ - actorSystem = ctx.ExecutorThread.ActorSystem, - selfId = SelfId() - ](const NThreading::TFuture<void>&) { - actorSystem->Send(new NActors::IEventHandle(selfId, selfId, new TEvPQ::TEvReaderEventArrived())); - } - ); -} - -void TMirrorer::ProcessNextReaderEvent(TEvPQ::TEvReaderEventArrived::TPtr&, const TActorContext& ctx) { - TMaybe<NYdb::NPersQueue::TReadSessionEvent::TEvent> event = ReadSession->GetEvent(false); - - WaitNextReaderEventInFlight = false; - if (!event) { - StartWaitNextReaderEvent(ctx); - return; - } - - if (auto* dataEvent = std::get_if<TPersQueueReadEvent::TDataReceivedEvent>(&event.GetRef())) { - AddMessagesToQueue(std::move(dataEvent->GetCompressedMessages())); - } else if (auto* createStream = std::get_if<TPersQueueReadEvent::TCreatePartitionStreamEvent>(&event.GetRef())) { - Y_VERIFY_S( + auto factory = AppData(ctx)->PersQueueMirrorReaderFactory; + Y_VERIFY(factory); + CredentialsProvider = factory->GetCredentialsProvider(Config.GetCredentials()); +} + +TString TMirrorer::MirrorerDescription() const { + return TStringBuilder() << "[mirrorer for " << TopicName << ':' << Partition << ']'; +} + +TString TMirrorer::GetCurrentState() const { + if (CurrentStateFunc() == &TThis::StateInitConsumer) { + return "StateInitConsumer"; + } else if (CurrentStateFunc() == &TThis::StateWork) { + return "StateWork"; + } + return "UNKNOWN"; +} + +void TMirrorer::StartWaitNextReaderEvent(const TActorContext& ctx) { + if (WaitNextReaderEventInFlight) { + return; + } + auto future = ReadSession->WaitEvent(); + future.Subscribe( + [ + actorSystem = ctx.ExecutorThread.ActorSystem, + selfId = SelfId() + ](const NThreading::TFuture<void>&) { + actorSystem->Send(new NActors::IEventHandle(selfId, selfId, new TEvPQ::TEvReaderEventArrived())); + } + ); +} + +void TMirrorer::ProcessNextReaderEvent(TEvPQ::TEvReaderEventArrived::TPtr&, const TActorContext& ctx) { + TMaybe<NYdb::NPersQueue::TReadSessionEvent::TEvent> event = ReadSession->GetEvent(false); + + WaitNextReaderEventInFlight = false; + if (!event) { + StartWaitNextReaderEvent(ctx); + return; + } + + if (auto* dataEvent = std::get_if<TPersQueueReadEvent::TDataReceivedEvent>(&event.GetRef())) { + AddMessagesToQueue(std::move(dataEvent->GetCompressedMessages())); + } else if (auto* createStream = std::get_if<TPersQueueReadEvent::TCreatePartitionStreamEvent>(&event.GetRef())) { + Y_VERIFY_S( !PartitionStream, MirrorerDescription() - << " already has stream " << PartitionStream->GetPartitionStreamId() - << ", new stream " << createStream->GetPartitionStream()->GetPartitionStreamId() - ); - PartitionStream = createStream->GetPartitionStream(); - Y_VERIFY_S( - Partition == PartitionStream->GetPartitionId(), - MirrorerDescription() + << " already has stream " << PartitionStream->GetPartitionStreamId() + << ", new stream " << createStream->GetPartitionStream()->GetPartitionStreamId() + ); + PartitionStream = createStream->GetPartitionStream(); + Y_VERIFY_S( + Partition == PartitionStream->GetPartitionId(), + MirrorerDescription() << " got stream for incorrect partition, stream: topic=" << PartitionStream->GetTopicPath() - << " partition=" << PartitionStream->GetPartitionId() - ); - if (OffsetToRead < createStream->GetCommittedOffset()) { + << " partition=" << PartitionStream->GetPartitionId() + ); + if (OffsetToRead < createStream->GetCommittedOffset()) { ProcessError(ctx, TStringBuilder() << "stream has commit offset more then partition end offset," - << "gap will be created [" << OffsetToRead << ";" << createStream->GetCommittedOffset() << ")" - ); - } - LOG_INFO_S(ctx, NKikimrServices::PQ_MIRRORER, + << "gap will be created [" << OffsetToRead << ";" << createStream->GetCommittedOffset() << ")" + ); + } + LOG_INFO_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription() << " got create stream event for '" << createStream->DebugString() - << " and will set offset=" << OffsetToRead); - - createStream->Confirm(OffsetToRead, OffsetToRead); - RequestSourcePartitionStatus(); - } else if (auto* destroyStream = std::get_if<TPersQueueReadEvent::TDestroyPartitionStreamEvent>(&event.GetRef())) { - destroyStream->Confirm(); - - PartitionStream.Reset(); - LOG_INFO_S(ctx, NKikimrServices::PQ_MIRRORER, - MirrorerDescription() - << " got destroy stream event for partition stream id: " - << destroyStream->GetPartitionStream()->GetPartitionStreamId()); - } else if (auto* streamClosed = std::get_if<TPersQueueReadEvent::TPartitionStreamClosedEvent>(&event.GetRef())) { - PartitionStream.Reset(); - LOG_INFO_S(ctx, NKikimrServices::PQ_MIRRORER, - MirrorerDescription() - << " got stream closed event for partition stream id: " - << streamClosed->GetPartitionStream()->GetPartitionStreamId() - << " reason: " << streamClosed->GetReason()); - } else if (auto* streamStatus = std::get_if<TPersQueueReadEvent::TPartitionStreamStatusEvent >(&event.GetRef())) { - if (PartitionStream - && PartitionStream->GetPartitionStreamId() == streamStatus->GetPartitionStream()->GetPartitionStreamId() - ) { - StreamStatus = MakeHolder<TPersQueueReadEvent::TPartitionStreamStatusEvent>(*streamStatus); - - ctx.Schedule(TDuration::Seconds(1), new TEvPQ::TEvRequestPartitionStatus); - TryUpdateWriteTimetsamp(ctx); - } - } else if (auto* commitAck = std::get_if<TPersQueueReadEvent::TCommitAcknowledgementEvent>(&event.GetRef())) { - LOG_INFO_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription() - << " got commit responce, commited offset: " << commitAck->GetCommittedOffset()); - } else if (auto* closeSessionEvent = std::get_if<NYdb::NPersQueue::TSessionClosedEvent>(&event.GetRef())) { - ProcessError(ctx, TStringBuilder() << " read session closed: " << closeSessionEvent->DebugString()); - ScheduleConsumerCreation(ctx); - return; - } - - Send(SelfId(), new TEvents::TEvWakeup()); - ConsumerInitInterval = CONSUMER_INIT_INTERVAL_START; -} - -}// NPQ -}// NKikimr + << " and will set offset=" << OffsetToRead); + + createStream->Confirm(OffsetToRead, OffsetToRead); + RequestSourcePartitionStatus(); + } else if (auto* destroyStream = std::get_if<TPersQueueReadEvent::TDestroyPartitionStreamEvent>(&event.GetRef())) { + destroyStream->Confirm(); + + PartitionStream.Reset(); + LOG_INFO_S(ctx, NKikimrServices::PQ_MIRRORER, + MirrorerDescription() + << " got destroy stream event for partition stream id: " + << destroyStream->GetPartitionStream()->GetPartitionStreamId()); + } else if (auto* streamClosed = std::get_if<TPersQueueReadEvent::TPartitionStreamClosedEvent>(&event.GetRef())) { + PartitionStream.Reset(); + LOG_INFO_S(ctx, NKikimrServices::PQ_MIRRORER, + MirrorerDescription() + << " got stream closed event for partition stream id: " + << streamClosed->GetPartitionStream()->GetPartitionStreamId() + << " reason: " << streamClosed->GetReason()); + } else if (auto* streamStatus = std::get_if<TPersQueueReadEvent::TPartitionStreamStatusEvent >(&event.GetRef())) { + if (PartitionStream + && PartitionStream->GetPartitionStreamId() == streamStatus->GetPartitionStream()->GetPartitionStreamId() + ) { + StreamStatus = MakeHolder<TPersQueueReadEvent::TPartitionStreamStatusEvent>(*streamStatus); + + ctx.Schedule(TDuration::Seconds(1), new TEvPQ::TEvRequestPartitionStatus); + TryUpdateWriteTimetsamp(ctx); + } + } else if (auto* commitAck = std::get_if<TPersQueueReadEvent::TCommitAcknowledgementEvent>(&event.GetRef())) { + LOG_INFO_S(ctx, NKikimrServices::PQ_MIRRORER, MirrorerDescription() + << " got commit responce, commited offset: " << commitAck->GetCommittedOffset()); + } else if (auto* closeSessionEvent = std::get_if<NYdb::NPersQueue::TSessionClosedEvent>(&event.GetRef())) { + ProcessError(ctx, TStringBuilder() << " read session closed: " << closeSessionEvent->DebugString()); + ScheduleConsumerCreation(ctx); + return; + } + + Send(SelfId(), new TEvents::TEvWakeup()); + ConsumerInitInterval = CONSUMER_INIT_INTERVAL_START; +} + +}// NPQ +}// NKikimr diff --git a/ydb/core/persqueue/mirrorer.h b/ydb/core/persqueue/mirrorer.h index 0e7586fe04..6819891f6c 100644 --- a/ydb/core/persqueue/mirrorer.h +++ b/ydb/core/persqueue/mirrorer.h @@ -1,9 +1,9 @@ -#pragma once - +#pragma once + #include "actor_persqueue_client_iface.h" - -#include <library/cpp/actors/core/hfunc.h> -#include <library/cpp/actors/core/log.h> + +#include <library/cpp/actors/core/hfunc.h> +#include <library/cpp/actors/core/log.h> #include <ydb/core/base/appdata.h> #include <ydb/core/persqueue/percentile_counter.h> #include <ydb/core/protos/counters_pq.pb.h> @@ -12,188 +12,188 @@ #include <ydb/core/persqueue/events/internal.h> #include <ydb/library/persqueue/counter_time_keeper/counter_time_keeper.h> #include <ydb/public/sdk/cpp/client/ydb_persqueue_core/persqueue.h> - - -namespace NKikimr { -namespace NPQ { - -class TMirrorer : public TActorBootstrapped<TMirrorer> { -private: - const ui64 MAX_BYTES_IN_FLIGHT = 8 * 1024 * 1024; - const TDuration WRITE_RETRY_TIMEOUT_MAX = TDuration::Seconds(1); - const TDuration WRITE_RETRY_TIMEOUT_START = TDuration::MilliSeconds(1); - - const TDuration CONSUMER_INIT_TIMEOUT_MAX = TDuration::Seconds(60); - const TDuration CONSUMER_INIT_TIMEOUT_START = TDuration::Seconds(5); - - const TDuration CONSUMER_INIT_INTERVAL_MAX = TDuration::Seconds(60); - const TDuration CONSUMER_INIT_INTERVAL_START = TDuration::Seconds(1); - - const TDuration READ_RETRY_TIMEOUT_MAX = TDuration::Seconds(1); - const TDuration READ_RETRY_TIMEOUT_START = TDuration::MilliSeconds(1); - - const TDuration UPDATE_COUNTERS_INTERVAL = TDuration::Seconds(5); - - const TDuration LOG_STATE_INTERVAL = TDuration::Minutes(1); - const TDuration INIT_TIMEOUT = TDuration::Minutes(1); - const TDuration WRITE_TIMEOUT = TDuration::Minutes(10); - - -private: - enum EEventCookie : ui64 { - WRITE_REQUEST_COOKIE = 1, - UPDATE_WRITE_TIMESTAMP = 2 - }; - - struct TMessageInfo { - ui64 Offset; - ui64 Size; - TInstant WriteTime; - - TMessageInfo(ui64 offset, ui64 size, TInstant writeTime) - : Offset(offset) - , Size(size) - , WriteTime(writeTime) - {} - }; - -private: - - STFUNC(StateInitConsumer) - { - NPersQueue::TCounterTimeKeeper keeper(Counters.Cumulative()[COUNTER_PQ_TABLET_CPU_USAGE]); - - TRACE_EVENT(NKikimrServices::PQ_MIRRORER); - switch (ev->GetTypeRewrite()) { - HFuncTraced(TEvPQ::TEvInitCredentials, HandleInitCredentials); - HFuncTraced(TEvPQ::TEvChangeConfig, HandleChangeConfig); - HFuncTraced(TEvPQ::TEvCreateConsumer, CreateConsumer); - HFuncTraced(TEvPQ::TEvRetryWrite, HandleRetryWrite); - HFuncTraced(TEvPersQueue::TEvResponse, Handle); - HFuncTraced(TEvPQ::TEvUpdateCounters, Handle); - HFuncTraced(TEvents::TEvPoisonPill, Handle); - default: - break; - }; - } - - STFUNC(StateWork) - { - NPersQueue::TCounterTimeKeeper keeper(Counters.Cumulative()[COUNTER_PQ_TABLET_CPU_USAGE]); - - TRACE_EVENT(NKikimrServices::PQ_MIRRORER); - switch (ev->GetTypeRewrite()) { - HFuncTraced(TEvPQ::TEvChangeConfig, HandleChangeConfig); - CFunc(TEvents::TSystem::Wakeup, HandleWakeup); - HFuncTraced(TEvPQ::TEvRequestPartitionStatus, RequestSourcePartitionStatus); - HFuncTraced(TEvPQ::TEvRetryWrite, HandleRetryWrite); - HFuncTraced(TEvPersQueue::TEvResponse, Handle); - HFuncTraced(TEvPQ::TEvUpdateCounters, Handle); - HFuncTraced(TEvPQ::TEvReaderEventArrived, ProcessNextReaderEvent); - HFuncTraced(TEvents::TEvPoisonPill, Handle); - default: - break; - }; - } - -private: - template<class TEvent> + + +namespace NKikimr { +namespace NPQ { + +class TMirrorer : public TActorBootstrapped<TMirrorer> { +private: + const ui64 MAX_BYTES_IN_FLIGHT = 8 * 1024 * 1024; + const TDuration WRITE_RETRY_TIMEOUT_MAX = TDuration::Seconds(1); + const TDuration WRITE_RETRY_TIMEOUT_START = TDuration::MilliSeconds(1); + + const TDuration CONSUMER_INIT_TIMEOUT_MAX = TDuration::Seconds(60); + const TDuration CONSUMER_INIT_TIMEOUT_START = TDuration::Seconds(5); + + const TDuration CONSUMER_INIT_INTERVAL_MAX = TDuration::Seconds(60); + const TDuration CONSUMER_INIT_INTERVAL_START = TDuration::Seconds(1); + + const TDuration READ_RETRY_TIMEOUT_MAX = TDuration::Seconds(1); + const TDuration READ_RETRY_TIMEOUT_START = TDuration::MilliSeconds(1); + + const TDuration UPDATE_COUNTERS_INTERVAL = TDuration::Seconds(5); + + const TDuration LOG_STATE_INTERVAL = TDuration::Minutes(1); + const TDuration INIT_TIMEOUT = TDuration::Minutes(1); + const TDuration WRITE_TIMEOUT = TDuration::Minutes(10); + + +private: + enum EEventCookie : ui64 { + WRITE_REQUEST_COOKIE = 1, + UPDATE_WRITE_TIMESTAMP = 2 + }; + + struct TMessageInfo { + ui64 Offset; + ui64 Size; + TInstant WriteTime; + + TMessageInfo(ui64 offset, ui64 size, TInstant writeTime) + : Offset(offset) + , Size(size) + , WriteTime(writeTime) + {} + }; + +private: + + STFUNC(StateInitConsumer) + { + NPersQueue::TCounterTimeKeeper keeper(Counters.Cumulative()[COUNTER_PQ_TABLET_CPU_USAGE]); + + TRACE_EVENT(NKikimrServices::PQ_MIRRORER); + switch (ev->GetTypeRewrite()) { + HFuncTraced(TEvPQ::TEvInitCredentials, HandleInitCredentials); + HFuncTraced(TEvPQ::TEvChangeConfig, HandleChangeConfig); + HFuncTraced(TEvPQ::TEvCreateConsumer, CreateConsumer); + HFuncTraced(TEvPQ::TEvRetryWrite, HandleRetryWrite); + HFuncTraced(TEvPersQueue::TEvResponse, Handle); + HFuncTraced(TEvPQ::TEvUpdateCounters, Handle); + HFuncTraced(TEvents::TEvPoisonPill, Handle); + default: + break; + }; + } + + STFUNC(StateWork) + { + NPersQueue::TCounterTimeKeeper keeper(Counters.Cumulative()[COUNTER_PQ_TABLET_CPU_USAGE]); + + TRACE_EVENT(NKikimrServices::PQ_MIRRORER); + switch (ev->GetTypeRewrite()) { + HFuncTraced(TEvPQ::TEvChangeConfig, HandleChangeConfig); + CFunc(TEvents::TSystem::Wakeup, HandleWakeup); + HFuncTraced(TEvPQ::TEvRequestPartitionStatus, RequestSourcePartitionStatus); + HFuncTraced(TEvPQ::TEvRetryWrite, HandleRetryWrite); + HFuncTraced(TEvPersQueue::TEvResponse, Handle); + HFuncTraced(TEvPQ::TEvUpdateCounters, Handle); + HFuncTraced(TEvPQ::TEvReaderEventArrived, ProcessNextReaderEvent); + HFuncTraced(TEvents::TEvPoisonPill, Handle); + default: + break; + }; + } + +private: + template<class TEvent> void ScheduleWithIncreasingTimeout(const TActorId& recipient, TDuration& timeout, const TDuration& maxTimeout, const TActorContext &ctx) { - ctx.ExecutorThread.ActorSystem->Schedule(timeout, new IEventHandle(recipient, SelfId(), new TEvent())); - timeout = Min(timeout * 2, maxTimeout); - } - - bool AddToWriteRequest( - NKikimrClient::TPersQueuePartitionRequest& request, - NYdb::NPersQueue::TReadSessionEvent::TDataReceivedEvent::TCompressedMessage& message - ); - void ProcessError(const TActorContext& ctx, const TString& msg); - void ProcessError(const TActorContext& ctx, const TString& msg, const NKikimrClient::TResponse& response); - void AfterSuccesWrite(const TActorContext& ctx); - void ProcessWriteResponse( - const TActorContext& ctx, - const NKikimrClient::TPersQueuePartitionResponse& response - ); + ctx.ExecutorThread.ActorSystem->Schedule(timeout, new IEventHandle(recipient, SelfId(), new TEvent())); + timeout = Min(timeout * 2, maxTimeout); + } + + bool AddToWriteRequest( + NKikimrClient::TPersQueuePartitionRequest& request, + NYdb::NPersQueue::TReadSessionEvent::TDataReceivedEvent::TCompressedMessage& message + ); + void ProcessError(const TActorContext& ctx, const TString& msg); + void ProcessError(const TActorContext& ctx, const TString& msg, const NKikimrClient::TResponse& response); + void AfterSuccesWrite(const TActorContext& ctx); + void ProcessWriteResponse( + const TActorContext& ctx, + const NKikimrClient::TPersQueuePartitionResponse& response + ); void ScheduleConsumerCreation(const TActorContext& ctx); void RecreateCredentialsProvider(const TActorContext& ctx); - void StartInit(const TActorContext& ctx); - void RetryWrite(const TActorContext& ctx); - - void ProcessNextReaderEvent(TEvPQ::TEvReaderEventArrived::TPtr& ev, const TActorContext& ctx); - - TString MirrorerDescription() const; - - TString GetCurrentState() const; - -public: - static constexpr NKikimrServices::TActivity::EType ActorActivityType(); - TMirrorer( - TActorId tabletActor, - TActorId partitionActor, - const TString& topicName, - ui32 partition, - bool localDC, - ui64 endOffset, - const NKikimrPQ::TMirrorPartitionConfig& config, - const TTabletCountersBase& counters - ); - void Bootstrap(const TActorContext& ctx); - void Handle(TEvents::TEvPoisonPill::TPtr& ev, const TActorContext& ctx); - void Handle(TEvPersQueue::TEvResponse::TPtr& ev, const TActorContext& ctx); - void Handle(TEvPQ::TEvUpdateCounters::TPtr& ev, const TActorContext& ctx); - void HandleChangeConfig(TEvPQ::TEvChangeConfig::TPtr& ev, const TActorContext& ctx); - void TryToRead(const TActorContext& ctx); - void TryToWrite(const TActorContext& ctx); - void HandleInitCredentials(TEvPQ::TEvInitCredentials::TPtr& ev, const TActorContext& ctx); - void HandleRetryWrite(TEvPQ::TEvRetryWrite::TPtr& ev, const TActorContext& ctx); - void HandleWakeup(const TActorContext& ctx); - void CreateConsumer(TEvPQ::TEvCreateConsumer::TPtr& ev, const TActorContext& ctx); - void RequestSourcePartitionStatus(TEvPQ::TEvRequestPartitionStatus::TPtr& ev, const TActorContext& ctx); - void RequestSourcePartitionStatus(); - void TryUpdateWriteTimetsamp(const TActorContext &ctx); - void AddMessagesToQueue( - TVector<NYdb::NPersQueue::TReadSessionEvent::TDataReceivedEvent::TCompressedMessage>&& messages - ); - void StartWaitNextReaderEvent(const TActorContext& ctx); - -private: - TActorId TabletActor; + void StartInit(const TActorContext& ctx); + void RetryWrite(const TActorContext& ctx); + + void ProcessNextReaderEvent(TEvPQ::TEvReaderEventArrived::TPtr& ev, const TActorContext& ctx); + + TString MirrorerDescription() const; + + TString GetCurrentState() const; + +public: + static constexpr NKikimrServices::TActivity::EType ActorActivityType(); + TMirrorer( + TActorId tabletActor, + TActorId partitionActor, + const TString& topicName, + ui32 partition, + bool localDC, + ui64 endOffset, + const NKikimrPQ::TMirrorPartitionConfig& config, + const TTabletCountersBase& counters + ); + void Bootstrap(const TActorContext& ctx); + void Handle(TEvents::TEvPoisonPill::TPtr& ev, const TActorContext& ctx); + void Handle(TEvPersQueue::TEvResponse::TPtr& ev, const TActorContext& ctx); + void Handle(TEvPQ::TEvUpdateCounters::TPtr& ev, const TActorContext& ctx); + void HandleChangeConfig(TEvPQ::TEvChangeConfig::TPtr& ev, const TActorContext& ctx); + void TryToRead(const TActorContext& ctx); + void TryToWrite(const TActorContext& ctx); + void HandleInitCredentials(TEvPQ::TEvInitCredentials::TPtr& ev, const TActorContext& ctx); + void HandleRetryWrite(TEvPQ::TEvRetryWrite::TPtr& ev, const TActorContext& ctx); + void HandleWakeup(const TActorContext& ctx); + void CreateConsumer(TEvPQ::TEvCreateConsumer::TPtr& ev, const TActorContext& ctx); + void RequestSourcePartitionStatus(TEvPQ::TEvRequestPartitionStatus::TPtr& ev, const TActorContext& ctx); + void RequestSourcePartitionStatus(); + void TryUpdateWriteTimetsamp(const TActorContext &ctx); + void AddMessagesToQueue( + TVector<NYdb::NPersQueue::TReadSessionEvent::TDataReceivedEvent::TCompressedMessage>&& messages + ); + void StartWaitNextReaderEvent(const TActorContext& ctx); + +private: + TActorId TabletActor; TActorId PartitionActor; - TString TopicName; - ui32 Partition; - bool LocalDC; - ui64 EndOffset; - ui64 OffsetToRead; - NKikimrPQ::TMirrorPartitionConfig Config; - - TDeque<NYdb::NPersQueue::TReadSessionEvent::TDataReceivedEvent::TCompressedMessage> Queue; - TDeque<TMessageInfo> WriteInFlight; - ui64 BytesInFlight = 0; - std::optional<NKikimrClient::TPersQueuePartitionRequest> WriteRequestInFlight; - TDuration WriteRetryTimeout = WRITE_RETRY_TIMEOUT_START; - TInstant WriteRequestTimestamp; - std::shared_ptr<NYdb::ICredentialsProviderFactory> CredentialsProvider; - std::shared_ptr<NYdb::NPersQueue::IReadSession> ReadSession; - NYdb::NPersQueue::TPartitionStream::TPtr PartitionStream; - THolder<NYdb::NPersQueue::TReadSessionEvent::TPartitionStreamStatusEvent> StreamStatus; - TInstant LastInitStageTimestamp; - - TDuration ConsumerInitTimeout = CONSUMER_INIT_TIMEOUT_START; - TDuration ConsumerInitInterval = CONSUMER_INIT_INTERVAL_START; - TDuration ReadRetryTimeout = READ_RETRY_TIMEOUT_START; - - TTabletCountersBase Counters; - - bool WaitNextReaderEventInFlight = false; - - bool WasSuccessfulRecording = false; - TInstant LastStateLogTimestamp; - - TMultiCounter MirrorerErrors; - TMultiCounter InitTimeoutCounter; - TMultiCounter WriteTimeoutCounter; - THolder<TPercentileCounter> MirrorerTimeLags; -}; - -}// NPQ -}// NKikimr + TString TopicName; + ui32 Partition; + bool LocalDC; + ui64 EndOffset; + ui64 OffsetToRead; + NKikimrPQ::TMirrorPartitionConfig Config; + + TDeque<NYdb::NPersQueue::TReadSessionEvent::TDataReceivedEvent::TCompressedMessage> Queue; + TDeque<TMessageInfo> WriteInFlight; + ui64 BytesInFlight = 0; + std::optional<NKikimrClient::TPersQueuePartitionRequest> WriteRequestInFlight; + TDuration WriteRetryTimeout = WRITE_RETRY_TIMEOUT_START; + TInstant WriteRequestTimestamp; + std::shared_ptr<NYdb::ICredentialsProviderFactory> CredentialsProvider; + std::shared_ptr<NYdb::NPersQueue::IReadSession> ReadSession; + NYdb::NPersQueue::TPartitionStream::TPtr PartitionStream; + THolder<NYdb::NPersQueue::TReadSessionEvent::TPartitionStreamStatusEvent> StreamStatus; + TInstant LastInitStageTimestamp; + + TDuration ConsumerInitTimeout = CONSUMER_INIT_TIMEOUT_START; + TDuration ConsumerInitInterval = CONSUMER_INIT_INTERVAL_START; + TDuration ReadRetryTimeout = READ_RETRY_TIMEOUT_START; + + TTabletCountersBase Counters; + + bool WaitNextReaderEventInFlight = false; + + bool WasSuccessfulRecording = false; + TInstant LastStateLogTimestamp; + + TMultiCounter MirrorerErrors; + TMultiCounter InitTimeoutCounter; + TMultiCounter WriteTimeoutCounter; + THolder<TPercentileCounter> MirrorerTimeLags; +}; + +}// NPQ +}// NKikimr diff --git a/ydb/core/persqueue/ownerinfo.h b/ydb/core/persqueue/ownerinfo.h index ed7c31be04..a6f5aef716 100644 --- a/ydb/core/persqueue/ownerinfo.h +++ b/ydb/core/persqueue/ownerinfo.h @@ -1,10 +1,10 @@ #pragma once #include <ydb/core/keyvalue/keyvalue_events.h> #include <ydb/core/persqueue/events/internal.h> - + #include <library/cpp/actors/core/actor.h> - + namespace NKikimr { namespace NPQ { diff --git a/ydb/core/persqueue/partition.cpp b/ydb/core/persqueue/partition.cpp index 739b3a3465..cc8e2419cf 100644 --- a/ydb/core/persqueue/partition.cpp +++ b/ydb/core/persqueue/partition.cpp @@ -1,10 +1,10 @@ #include "partition.h" -#include "event_helpers.h" +#include "event_helpers.h" #include "read.h" #include "sourceid.h" #include "ownerinfo.h" -#include "mirrorer.h" - +#include "mirrorer.h" + #include <ydb/core/base/appdata.h> #include <ydb/core/base/blobstorage.h> #include <ydb/core/base/counters.h> @@ -28,19 +28,19 @@ namespace NKikimr { namespace NPQ { -static const ui32 BATCH_UNPACK_SIZE_BORDER = 500 * 1024; //500kb - +static const ui32 BATCH_UNPACK_SIZE_BORDER = 500 * 1024; //500kb + static const ui32 MAX_WRITE_CYCLE_SIZE = 16 << 20; //16MB static const ui32 MAX_USER_ACTS = 1000; -static const TDuration WAKE_TIMEOUT = TDuration::Seconds(5); +static const TDuration WAKE_TIMEOUT = TDuration::Seconds(5); static const ui32 MAX_INLINE_SIZE = 1000; static const ui32 LEVEL0 = 32; -static const TDuration UPDATE_AVAIL_SIZE_INTERVAL = TDuration::MilliSeconds(100); +static const TDuration UPDATE_AVAIL_SIZE_INTERVAL = TDuration::MilliSeconds(100); static const TString WRITE_QUOTA_ROOT_PATH = "write-quota"; @@ -58,7 +58,7 @@ struct TPartition::THasDataReq { }; struct TPartition::THasDataDeadline { - TInstant Deadline; + TInstant Deadline; TPartition::THasDataReq Request; bool operator < (const THasDataDeadline& dl) const @@ -215,12 +215,12 @@ IOutputStream& operator <<(IOutputStream& out, const TKeyLevel& value) { } -ui64 GetOffsetEstimate(const std::deque<TDataKey>& container, TInstant timestamp, ui64 offset) { +ui64 GetOffsetEstimate(const std::deque<TDataKey>& container, TInstant timestamp, ui64 offset) { if (container.empty()) { return offset; } auto it = std::lower_bound(container.begin(), container.end(), timestamp, - [](const TDataKey& p, const TInstant timestamp) { return timestamp > p.Timestamp; }); + [](const TDataKey& p, const TInstant timestamp) { return timestamp > p.Timestamp; }); if (it == container.end()) { return offset; } else { @@ -228,23 +228,23 @@ ui64 GetOffsetEstimate(const std::deque<TDataKey>& container, TInstant timestamp } } -struct TMirrorerInfo { - TMirrorerInfo(const TActorId& actor, const TTabletCountersBase& baseline) - : Actor(actor) - { - Baseline.Populate(baseline); - } - - TActorId Actor; - TTabletCountersBase Baseline; -}; - +struct TMirrorerInfo { + TMirrorerInfo(const TActorId& actor, const TTabletCountersBase& baseline) + : Actor(actor) + { + Baseline.Populate(baseline); + } + + TActorId Actor; + TTabletCountersBase Baseline; +}; + void TPartition::ReplyError(const TActorContext& ctx, const ui64 dst, NPersQueue::NErrorCode::EErrorCode errorCode, const TString& error) { - ReplyPersQueueError( - dst == 0 ? ctx.SelfID : Tablet, ctx, TabletID, TopicName, Partition, Counters, NKikimrServices::PERSQUEUE, + ReplyPersQueueError( + dst == 0 ? ctx.SelfID : Tablet, ctx, TabletID, TopicName, Partition, Counters, NKikimrServices::PERSQUEUE, dst, errorCode, error, true - ); + ); } void TPartition::ReplyOk(const TActorContext& ctx, const ui64 dst) @@ -266,10 +266,10 @@ void TPartition::ReplyOwnerOk(const TActorContext& ctx, const ui64 dst, const TS ctx.Send(Tablet, response.Release()); } -void TPartition::ReplyWrite( - const TActorContext& ctx, const ui64 dst, const TString& sourceId, const ui64 seqNo, const ui16 partNo, const ui16 totalParts, - const ui64 offset, const TInstant writeTimestamp, bool already, const ui64 maxSeqNo, - const ui64 partitionQuotedTime, const TDuration topicQuotedTime, const ui64 queueTime, const ui64 writeTime) +void TPartition::ReplyWrite( + const TActorContext& ctx, const ui64 dst, const TString& sourceId, const ui64 seqNo, const ui16 partNo, const ui16 totalParts, + const ui64 offset, const TInstant writeTimestamp, bool already, const ui64 maxSeqNo, + const ui64 partitionQuotedTime, const TDuration topicQuotedTime, const ui64 queueTime, const ui64 writeTime) { Y_VERIFY(offset <= (ui64)Max<i64>(), "Offset is too big: %" PRIu64, offset); Y_VERIFY(seqNo <= (ui64)Max<i64>(), "SeqNo is too big: %" PRIu64, seqNo); @@ -281,7 +281,7 @@ void TPartition::ReplyWrite( auto write = resp.MutablePartitionResponse()->AddCmdWriteResult(); write->SetSourceId(sourceId); write->SetSeqNo(seqNo); - write->SetWriteTimestampMS(writeTimestamp.MilliSeconds()); + write->SetWriteTimestampMS(writeTimestamp.MilliSeconds()); if (totalParts > 1) write->SetPartNo(partNo); write->SetAlreadyWritten(already); @@ -290,7 +290,7 @@ void TPartition::ReplyWrite( write->SetOffset(offset); write->SetPartitionQuotedTimeMs(partitionQuotedTime); - write->SetTopicQuotedTimeMs(topicQuotedTime.MilliSeconds()); + write->SetTopicQuotedTimeMs(topicQuotedTime.MilliSeconds()); write->SetTotalTimeInPartitionQueueMs(queueTime); write->SetWriteTimeMs(writeTime); @@ -299,7 +299,7 @@ void TPartition::ReplyWrite( void TPartition::ReplyGetClientOffsetOk(const TActorContext& ctx, const ui64 dst, const i64 offset, - const TInstant writeTimestamp, const TInstant createTimestamp) + const TInstant writeTimestamp, const TInstant createTimestamp) { THolder<TEvPQ::TEvProxyResponse> response = MakeHolder<TEvPQ::TEvProxyResponse>(dst); NKikimrClient::TResponse& resp = response->Response; @@ -309,11 +309,11 @@ void TPartition::ReplyGetClientOffsetOk(const TActorContext& ctx, const ui64 dst auto user = resp.MutablePartitionResponse()->MutableCmdGetClientOffsetResult(); if (offset > -1) user->SetOffset(offset); - if (writeTimestamp) - user->SetWriteTimestampMS(writeTimestamp.MilliSeconds()); - if (createTimestamp) { - Y_VERIFY(writeTimestamp); - user->SetCreateTimestampMS(createTimestamp.MilliSeconds()); + if (writeTimestamp) + user->SetWriteTimestampMS(writeTimestamp.MilliSeconds()); + if (createTimestamp) { + Y_VERIFY(writeTimestamp); + user->SetCreateTimestampMS(createTimestamp.MilliSeconds()); } user->SetEndOffset(EndOffset); user->SetSizeLag(GetSizeLag(offset)); @@ -412,17 +412,17 @@ void RequestDataRange(const TActorContext& ctx, const TActorId& dst, ui32 partit void TPartition::FillReadFromTimestamps(const NKikimrPQ::TPQTabletConfig& config, const TActorContext& ctx) { TSet<TString> hasReadRule; - for (auto& userInfo : UsersInfoStorage.GetAll()) { - userInfo.second.ReadFromTimestamp = TInstant::Zero(); - if (userInfo.second.HasReadRule) { - userInfo.second.HasReadRule = false; - hasReadRule.insert(userInfo.first); + for (auto& userInfo : UsersInfoStorage.GetAll()) { + userInfo.second.ReadFromTimestamp = TInstant::Zero(); + if (userInfo.second.HasReadRule) { + userInfo.second.HasReadRule = false; + hasReadRule.insert(userInfo.first); } } for (ui32 i = 0; i < config.ReadRulesSize(); ++i) { const auto& consumer = config.GetReadRules(i); - auto& userInfo = UsersInfoStorage.GetOrCreate(consumer, ctx); - userInfo.HasReadRule = true; + auto& userInfo = UsersInfoStorage.GetOrCreate(consumer, ctx); + userInfo.HasReadRule = true; ui64 rrGen = i < config.ReadRuleGenerationsSize() ? config.GetReadRuleGenerations(i) : 0; if (userInfo.ReadRuleGeneration != rrGen) { THolder<TEvPQ::TEvSetClientInfo> event = MakeHolder<TEvPQ::TEvSetClientInfo>(0, consumer, 0, "", 0, 0, @@ -433,15 +433,15 @@ void TPartition::FillReadFromTimestamps(const NKikimrPQ::TPQTabletConfig& config userInfo.Step = userInfo.Generation = 0; } hasReadRule.erase(consumer); - TInstant ts = i < config.ReadFromTimestampsMsSize() ? TInstant::MilliSeconds(config.GetReadFromTimestampsMs(i)) : TInstant::Zero(); - if (!ts) ts += TDuration::MilliSeconds(1); - if (!userInfo.ReadFromTimestamp|| userInfo.ReadFromTimestamp > ts) - userInfo.ReadFromTimestamp = ts; + TInstant ts = i < config.ReadFromTimestampsMsSize() ? TInstant::MilliSeconds(config.GetReadFromTimestampsMs(i)) : TInstant::Zero(); + if (!ts) ts += TDuration::MilliSeconds(1); + if (!userInfo.ReadFromTimestamp|| userInfo.ReadFromTimestamp > ts) + userInfo.ReadFromTimestamp = ts; } for (auto& consumer : hasReadRule) { - auto& userInfo = UsersInfoStorage.GetOrCreate(consumer, ctx); - if (!userInfo.Important) { - ctx.Send(Tablet, new TEvPQ::TEvPartitionLabeledCountersDrop(Partition, userInfo.LabeledCounters.GetGroup())); + auto& userInfo = UsersInfoStorage.GetOrCreate(consumer, ctx); + if (!userInfo.Important) { + ctx.Send(Tablet, new TEvPQ::TEvPartitionLabeledCountersDrop(Partition, userInfo.LabeledCounters.GetGroup())); } THolder<TEvPQ::TEvSetClientInfo> event = MakeHolder<TEvPQ::TEvSetClientInfo>(0, consumer, 0, "", 0, 0, TEvPQ::TEvSetClientInfo::ESCI_DROP_READ_RULE, 0); @@ -470,7 +470,7 @@ TPartition::TPartition(ui64 tabletId, ui32 partition, const TActorId& tablet, co , BlobCache(blobCache) , InitState(WaitDiskStatus) , PartitionedBlob(partition, 0, 0, 0, 0, 0, Head, NewHead, true, false, 8 << 20) - , NewHeadKey{TKey{}, 0, TInstant::Zero(), 0} + , NewHeadKey{TKey{}, 0, TInstant::Zero(), 0} , BodySize(0) , MaxWriteResponsesSize(0) , GapSize(0) @@ -508,13 +508,13 @@ TPartition::TPartition(ui64 tabletId, ui32 partition, const TActorId& tablet, co { if (Config.GetPartitionConfig().HasMirrorFrom()) { ManageWriteTimestampEstimate = !Config.GetPartitionConfig().GetMirrorFrom().GetSyncWriteTime(); - } else { - ManageWriteTimestampEstimate = LocalDC; - } + } else { + ManageWriteTimestampEstimate = LocalDC; + } - WriteTimestampEstimate = ManageWriteTimestampEstimate ? ctx.Now() : TInstant::Zero(); - + WriteTimestampEstimate = ManageWriteTimestampEstimate ? ctx.Now() : TInstant::Zero(); + CalcTopicWriteQuotaParams(); @@ -588,7 +588,7 @@ void TPartition::HandleMonitoring(TEvPQ::TEvMonRequest::TPtr& ev, const TActorCo TABLER() { TABLED() {out << "DataBody";} TABLED() {out << i++;} - TABLED() {out << ToStringLocalTimeUpToSeconds(d.Timestamp);} + TABLED() {out << ToStringLocalTimeUpToSeconds(d.Timestamp);} TABLED() {out << d.Key.GetOffset();} TABLED() {out << d.Key.GetPartNo();} TABLED() {out << d.Key.GetCount();} @@ -605,7 +605,7 @@ void TPartition::HandleMonitoring(TEvPQ::TEvMonRequest::TPtr& ev, const TActorCo TABLER() { TABLED() {out << "DataHead[" << currentLevel << "]";} TABLED() {out << i++;} - TABLED() {out << ToStringLocalTimeUpToSeconds(HeadKeys[p].Timestamp);} + TABLED() {out << ToStringLocalTimeUpToSeconds(HeadKeys[p].Timestamp);} TABLED() {out << HeadKeys[p].Key.GetOffset();} TABLED() {out << HeadKeys[p].Key.GetPartNo();} TABLED() {out << HeadKeys[p].Key.GetCount();} @@ -692,17 +692,17 @@ void TPartition::HandleMonitoring(TEvPQ::TEvMonRequest::TPtr& ev, const TActorCo } } TABLEBODY() { - for (auto& d: UsersInfoStorage.GetAll()) { + for (auto& d: UsersInfoStorage.GetAll()) { TABLER() { TABLED() {out << EncodeHtmlPcdata(d.first);} TABLED() {out << d.second.Offset;} TABLED() {out << (EndOffset - d.second.Offset);} - TABLED() {out << ToStringLocalTimeUpToSeconds(d.second.ReadFromTimestamp);} - TABLED() {out << ToStringLocalTimeUpToSeconds(d.second.WriteTimestamp);} - TABLED() {out << ToStringLocalTimeUpToSeconds(d.second.CreateTimestamp);} + TABLED() {out << ToStringLocalTimeUpToSeconds(d.second.ReadFromTimestamp);} + TABLED() {out << ToStringLocalTimeUpToSeconds(d.second.WriteTimestamp);} + TABLED() {out << ToStringLocalTimeUpToSeconds(d.second.CreateTimestamp);} TABLED() {out << (d.second.GetReadOffset());} - TABLED() {out << ToStringLocalTimeUpToSeconds(d.second.GetReadWriteTimestamp());} - TABLED() {out << ToStringLocalTimeUpToSeconds(d.second.GetReadCreateTimestamp());} + TABLED() {out << ToStringLocalTimeUpToSeconds(d.second.GetReadWriteTimestamp());} + TABLED() {out << ToStringLocalTimeUpToSeconds(d.second.GetReadCreateTimestamp());} TABLED() {out << (d.second.ReadOffsetRewindSum);} TABLED() {out << d.second.ActiveReads;} TABLED() {out << d.second.Subscriptions;} @@ -720,8 +720,8 @@ void TPartition::HandleMonitoring(TEvPQ::TEvMonRequest::TPtr& ev, const TActorCo void TPartition::Bootstrap(const TActorContext& ctx) { - UsersInfoStorage.Init(Tablet, SelfId()); - + UsersInfoStorage.Init(Tablet, SelfId()); + Y_VERIFY(AppData(ctx)->PQConfig.GetMaxBlobsPerLevel() > 0); ui32 border = LEVEL0; MaxSizeCheck = 0; @@ -744,8 +744,8 @@ void TPartition::Bootstrap(const TActorContext& ctx) } for (const auto& readQuota : Config.GetPartitionConfig().GetReadQuota()) { - auto &userInfo = UsersInfoStorage.GetOrCreate(readQuota.GetClientId(), ctx); - userInfo.ReadQuota.UpdateConfig(readQuota.GetBurstSize(), readQuota.GetSpeedInBytesPerSecond()); + auto &userInfo = UsersInfoStorage.GetOrCreate(readQuota.GetClientId(), ctx); + userInfo.ReadQuota.UpdateConfig(readQuota.GetBurstSize(), readQuota.GetSpeedInBytesPerSecond()); } LOG_INFO_S(ctx, NKikimrServices::PERSQUEUE, "boostrapping " << Partition << " " << ctx.SelfID); @@ -930,8 +930,8 @@ void TPartition::ProcessHasDataRequests(const TActorContext& ctx) { res->Record.SetCookie(*(it->Cookie)); ctx.Send(it->Sender, res.Release()); if (!it->ClientId.empty()) { - auto& userInfo = UsersInfoStorage.GetOrCreate(it->ClientId, ctx); - userInfo.ForgetSubscription(ctx.Now()); + auto& userInfo = UsersInfoStorage.GetOrCreate(it->ClientId, ctx); + userInfo.ForgetSubscription(ctx.Now()); } it = HasDataRequests.erase(it); } else { @@ -939,7 +939,7 @@ void TPartition::ProcessHasDataRequests(const TActorContext& ctx) { } } for (auto it = HasDataDeadlines.begin(); it != HasDataDeadlines.end();) { - if (it->Deadline <= ctx.Now()) { + if (it->Deadline <= ctx.Now()) { auto jt = HasDataRequests.find(it->Request); if (jt != HasDataRequests.end()) { TAutoPtr<TEvPersQueue::TEvHasDataInfoResponse> res(new TEvPersQueue::TEvHasDataInfoResponse()); @@ -950,8 +950,8 @@ void TPartition::ProcessHasDataRequests(const TActorContext& ctx) { res->Record.SetCookie(*(it->Request.Cookie)); ctx.Send(it->Request.Sender, res.Release()); if (!it->Request.ClientId.empty()) { - auto& userInfo = UsersInfoStorage.GetOrCreate(it->Request.ClientId, ctx); - userInfo.ForgetSubscription(ctx.Now()); + auto& userInfo = UsersInfoStorage.GetOrCreate(it->Request.ClientId, ctx); + userInfo.ForgetSubscription(ctx.Now()); } HasDataRequests.erase(jt); } @@ -969,7 +969,7 @@ void TPartition::UpdateAvailableSize(const TActorContext& ctx) { auto now = ctx.Now(); WriteQuota.Update(now); - for (auto &c : UsersInfoStorage.GetAll()) { + for (auto &c : UsersInfoStorage.GetAll()) { while (true) { c.second.ReadQuota.Update(now); if (!c.second.ReadQuota.CanExaust() && !c.second.ReadRequests.empty()) { @@ -1001,7 +1001,7 @@ void TPartition::HandleOnWrite(TEvPQ::TEvUpdateAvailableSize::TPtr&, const TActo void TPartition::HandleWakeup(const TActorContext& ctx) { FilterDeadlinedWrites(ctx); - ctx.Schedule(WAKE_TIMEOUT, new TEvents::TEvWakeup()); + ctx.Schedule(WAKE_TIMEOUT, new TEvents::TEvWakeup()); ctx.Send(Tablet, new TEvPQ::TEvPartitionCounters(Partition, Counters)); ReportLabeledCounters(ctx); @@ -1009,9 +1009,9 @@ void TPartition::HandleWakeup(const TActorContext& ctx) { ProcessHasDataRequests(ctx); auto now = ctx.Now(); - for (auto& userInfo : UsersInfoStorage.GetAll()) { - userInfo.second.UpdateReadingTimeAndState(now); - for (auto& avg : userInfo.second.AvgReadBytes) { + for (auto& userInfo : UsersInfoStorage.GetAll()) { + userInfo.second.UpdateReadingTimeAndState(now); + for (auto& avg : userInfo.second.AvgReadBytes) { avg.Update(now); } } @@ -1032,7 +1032,7 @@ void TPartition::HandleWakeup(const TActorContext& ctx) { } Y_VERIFY(CurrentStateFunc() == &TThis::StateIdle); - if (ManageWriteTimestampEstimate) + if (ManageWriteTimestampEstimate) WriteTimestampEstimate = now; THolder <TEvKeyValue::TEvRequest> request = MakeHolder<TEvKeyValue::TEvRequest>(); @@ -1094,10 +1094,10 @@ bool TPartition::DropOldData(TEvKeyValue::TEvRequest *request, bool hasWrites, c ui64 minOffset = EndOffset; for (const auto& importantClientId : Config.GetPartitionConfig().GetImportantClientId()) { - TUserInfo* userInfo = UsersInfoStorage.GetIfExists(importantClientId); + TUserInfo* userInfo = UsersInfoStorage.GetIfExists(importantClientId); ui64 curOffset = StartOffset; - if (userInfo && userInfo->Offset >= 0) //-1 means no offset - curOffset = userInfo->Offset; + if (userInfo && userInfo->Offset >= 0) //-1 means no offset + curOffset = userInfo->Offset; minOffset = Min<ui64>(minOffset, curOffset); } @@ -1192,39 +1192,39 @@ void TPartition::Handle(TEvPersQueue::TEvHasDataInfo::TPtr& ev, const TActorCont } else { THasDataReq req{++HasDataReqNum, (ui64)record.GetOffset(), sender, record.HasCookie() ? TMaybe<ui64>(record.GetCookie()) : TMaybe<ui64>(), record.HasClientId() && InitDone ? record.GetClientId() : ""}; - THasDataDeadline dl{TInstant::MilliSeconds(record.GetDeadline()), req}; + THasDataDeadline dl{TInstant::MilliSeconds(record.GetDeadline()), req}; auto res = HasDataRequests.insert(req); HasDataDeadlines.insert(dl); Y_VERIFY(res.second); if (InitDone && record.HasClientId() && !record.GetClientId().empty()) { - auto& userInfo = UsersInfoStorage.GetOrCreate(record.GetClientId(), ctx); - ++userInfo.Subscriptions; - userInfo.UpdateReadOffset((i64)EndOffset - 1, ctx.Now(), ctx.Now(), ctx.Now()); - userInfo.UpdateReadingTimeAndState(ctx.Now()); - } - } -} - -void TPartition::Handle(TEvPQ::TEvMirrorerCounters::TPtr& ev, const TActorContext& /*ctx*/) { - if (Mirrorer) { - auto diff = ev->Get()->Counters.MakeDiffForAggr(Mirrorer->Baseline); - Counters.Populate(*diff.Get()); - ev->Get()->Counters.RememberCurrentStateAsBaseline(Mirrorer->Baseline); - } -} - -void TPartition::Handle(NReadSpeedLimiterEvents::TEvCounters::TPtr& ev, const TActorContext& /*ctx*/) { - auto userInfo = UsersInfoStorage.GetIfExists(ev->Get()->User); - Y_VERIFY(userInfo); - if (userInfo) { - Y_VERIFY(userInfo->ReadSpeedLimiter); - auto diff = ev->Get()->Counters.MakeDiffForAggr(userInfo->ReadSpeedLimiter->Baseline); - Counters.Populate(*diff.Get()); - ev->Get()->Counters.RememberCurrentStateAsBaseline(userInfo->ReadSpeedLimiter->Baseline); - } -} - + auto& userInfo = UsersInfoStorage.GetOrCreate(record.GetClientId(), ctx); + ++userInfo.Subscriptions; + userInfo.UpdateReadOffset((i64)EndOffset - 1, ctx.Now(), ctx.Now(), ctx.Now()); + userInfo.UpdateReadingTimeAndState(ctx.Now()); + } + } +} + +void TPartition::Handle(TEvPQ::TEvMirrorerCounters::TPtr& ev, const TActorContext& /*ctx*/) { + if (Mirrorer) { + auto diff = ev->Get()->Counters.MakeDiffForAggr(Mirrorer->Baseline); + Counters.Populate(*diff.Get()); + ev->Get()->Counters.RememberCurrentStateAsBaseline(Mirrorer->Baseline); + } +} + +void TPartition::Handle(NReadSpeedLimiterEvents::TEvCounters::TPtr& ev, const TActorContext& /*ctx*/) { + auto userInfo = UsersInfoStorage.GetIfExists(ev->Get()->User); + Y_VERIFY(userInfo); + if (userInfo) { + Y_VERIFY(userInfo->ReadSpeedLimiter); + auto diff = ev->Get()->Counters.MakeDiffForAggr(userInfo->ReadSpeedLimiter->Baseline); + Counters.Populate(*diff.Get()); + ev->Get()->Counters.RememberCurrentStateAsBaseline(userInfo->ReadSpeedLimiter->Baseline); + } +} + void TPartition::Handle(TEvents::TEvPoisonPill::TPtr&, const TActorContext& ctx) { // Reply to all outstanding requests in order to destroy corresponding actors @@ -1248,11 +1248,11 @@ void TPartition::Handle(TEvents::TEvPoisonPill::TPtr&, const TActorContext& ctx) TStringBuilder() << ss << " (ReadInfo) cookie " << ri.first); } - if (Mirrorer) { - Send(Mirrorer->Actor, new TEvents::TEvPoisonPill()); - } - UsersInfoStorage.Clear(ctx); - + if (Mirrorer) { + Send(Mirrorer->Actor, new TEvents::TEvPoisonPill()); + } + UsersInfoStorage.Clear(ctx); + Die(ctx); } @@ -1331,9 +1331,9 @@ void TPartition::HandleGetDiskStatus(const NKikimrClient::TResponse& response, c diskIsOk = diskIsOk && CheckDiskStatus(res.GetStatusFlags()); } DiskIsFull = !diskIsOk; - if (DiskIsFull) { - LogAndCollectError(NKikimrServices::PERSQUEUE, "disk is full", ctx); - } + if (DiskIsFull) { + LogAndCollectError(NKikimrServices::PERSQUEUE, "disk is full", ctx); + } InitState = WaitMetaRead; RequestMetaRead(ctx, Tablet, Partition); @@ -1399,9 +1399,9 @@ void TPartition::HandleInfoRangeRead(const NKikimrClient::TKeyValueResponse::TRe } else if ((*key)[TKeyPrefix::MarkPosition()] == TKeyPrefix::MarkProtoSourceId) { SourceIdStorage.LoadSourceIdInfo(*key, pair.GetValue(), ctx.Now()); } else if ((*key)[TKeyPrefix::MarkPosition()] == TKeyPrefix::MarkUser) { - UsersInfoStorage.Parse(*key, pair.GetValue(), ctx); + UsersInfoStorage.Parse(*key, pair.GetValue(), ctx); } else if ((*key)[TKeyPrefix::MarkPosition()] == TKeyPrefix::MarkUserDeprecated) { - UsersInfoStorage.ParseDeprecated(*key, pair.GetValue(), ctx); + UsersInfoStorage.ParseDeprecated(*key, pair.GetValue(), ctx); } } //make next step @@ -1453,7 +1453,7 @@ void TPartition::FillBlobsMetaData(const NKikimrClient::TKeyValueResponse::TRead LOG_DEBUG_S(ctx, NKikimrServices::PERSQUEUE, "Got data topic " << TopicName << " partition " << k.GetPartition() << " offset " << k.GetOffset() << " count " << k.GetCount() << " size " << pair.GetValueSize() << " so " << StartOffset << " eo " << EndOffset << " " << pair.GetKey()); - DataKeysBody.push_back({k, pair.GetValueSize(), TInstant::Seconds(pair.GetCreationUnixTime()), DataKeysBody.empty() ? 0 : DataKeysBody.back().CumulativeSize + DataKeysBody.back().Size}); + DataKeysBody.push_back({k, pair.GetValueSize(), TInstant::Seconds(pair.GetCreationUnixTime()), DataKeysBody.empty() ? 0 : DataKeysBody.back().CumulativeSize + DataKeysBody.back().Size}); } Y_VERIFY(EndOffset >= StartOffset); @@ -1682,9 +1682,9 @@ void TPartition::InitComplete(const TActorContext& ctx) { InitUserInfoForImportantClients(ctx); - for (auto& userInfoPair : UsersInfoStorage.GetAll()) { - Y_VERIFY(userInfoPair.second.Offset >= 0); - ReadTimestampForOffset(userInfoPair.first, userInfoPair.second, ctx); + for (auto& userInfoPair : UsersInfoStorage.GetAll()) { + Y_VERIFY(userInfoPair.second.Offset >= 0); + ReadTimestampForOffset(userInfoPair.first, userInfoPair.second, ctx); } PartitionLabeledCounters.GetCounters()[METRIC_INIT_TIME] = InitDuration.MilliSeconds(); @@ -1695,17 +1695,17 @@ void TPartition::InitComplete(const TActorContext& ctx) { UpdateUserInfoEndOffset(ctx.Now()); ScheduleUpdateAvailableSize(ctx); - + if (Config.GetPartitionConfig().HasMirrorFrom()) { - CreateMirrorerActor(); - } + CreateMirrorerActor(); + } } void TPartition::UpdateUserInfoEndOffset(const TInstant& now) { - for (auto& userInfo : UsersInfoStorage.GetAll()) { - userInfo.second.EndOffset = (i64)EndOffset; - userInfo.second.UpdateReadingTimeAndState(now); + for (auto& userInfo : UsersInfoStorage.GetAll()) { + userInfo.second.EndOffset = (i64)EndOffset; + userInfo.second.UpdateReadingTimeAndState(now); } } @@ -1755,27 +1755,27 @@ THashMap<TString, NKikimr::NPQ::TOwnerInfo>::iterator TPartition::DropOwner(THas void TPartition::InitUserInfoForImportantClients(const TActorContext& ctx) { TSet<TString> important; for (const auto& importantUser : Config.GetPartitionConfig().GetImportantClientId()) { - important.insert(importantUser); - TUserInfo* userInfo = UsersInfoStorage.GetIfExists(importantUser); - if (userInfo && !userInfo->Important) { - ctx.Send(Tablet, new TEvPQ::TEvPartitionLabeledCountersDrop(Partition, userInfo->LabeledCounters.GetGroup())); - userInfo->SetImportant(true); + important.insert(importantUser); + TUserInfo* userInfo = UsersInfoStorage.GetIfExists(importantUser); + if (userInfo && !userInfo->Important) { + ctx.Send(Tablet, new TEvPQ::TEvPartitionLabeledCountersDrop(Partition, userInfo->LabeledCounters.GetGroup())); + userInfo->SetImportant(true); continue; } - if (!userInfo) { + if (!userInfo) { userInfo = &UsersInfoStorage.Create(ctx, importantUser, 0, true, "", 0, 0, 0, 0, TInstant::Zero()); - } - if (userInfo->Offset < (i64)StartOffset) - userInfo->Offset = StartOffset; - ReadTimestampForOffset(importantUser, *userInfo, ctx); - } - for (auto& userInfoPair : UsersInfoStorage.GetAll()) { - if (!important.contains(userInfoPair.first) && userInfoPair.second.Important) { - ctx.Send( - Tablet, - new TEvPQ::TEvPartitionLabeledCountersDrop(Partition, userInfoPair.second.LabeledCounters.GetGroup()) - ); - userInfoPair.second.SetImportant(false); + } + if (userInfo->Offset < (i64)StartOffset) + userInfo->Offset = StartOffset; + ReadTimestampForOffset(importantUser, *userInfo, ctx); + } + for (auto& userInfoPair : UsersInfoStorage.GetAll()) { + if (!important.contains(userInfoPair.first) && userInfoPair.second.Important) { + ctx.Send( + Tablet, + new TEvPQ::TEvPartitionLabeledCountersDrop(Partition, userInfoPair.second.LabeledCounters.GetGroup()) + ); + userInfoPair.second.SetImportant(false); } } } @@ -1798,27 +1798,27 @@ void TPartition::Handle(TEvPQ::TEvChangeConfig::TPtr& ev, const TActorContext& c } for (const auto& readQuota : Config.GetPartitionConfig().GetReadQuota()) { - auto& userInfo = UsersInfoStorage.GetOrCreate(readQuota.GetClientId(), ctx); - userInfo.ReadQuota.UpdateConfig(readQuota.GetBurstSize(), readQuota.GetSpeedInBytesPerSecond()); + auto& userInfo = UsersInfoStorage.GetOrCreate(readQuota.GetClientId(), ctx); + userInfo.ReadQuota.UpdateConfig(readQuota.GetBurstSize(), readQuota.GetSpeedInBytesPerSecond()); } if (CurrentStateFunc() != &TThis::StateInit) { InitUserInfoForImportantClients(ctx); FillReadFromTimestamps(Config, ctx); } - + if (Config.GetPartitionConfig().HasMirrorFrom()) { - if (Mirrorer) { - ctx.Send(ev->Forward(Mirrorer->Actor)); - } else { - CreateMirrorerActor(); - } - } else { - if (Mirrorer) { - ctx.Send(Mirrorer->Actor, new TEvents::TEvPoisonPill()); - Mirrorer.Reset(); - } - } + if (Mirrorer) { + ctx.Send(ev->Forward(Mirrorer->Actor)); + } else { + CreateMirrorerActor(); + } + } else { + if (Mirrorer) { + ctx.Send(Mirrorer->Actor, new TEvents::TEvPoisonPill()); + Mirrorer.Reset(); + } + } } @@ -1924,17 +1924,17 @@ void TPartition::Handle(TEvPQ::TEvPartitionOffsets::TPtr& ev, const TActorContex result.SetWriteTimestampEstimateMS(WriteTimestampEstimate.MilliSeconds()); if (!ev->Get()->ClientId.empty()) { - TUserInfo* userInfo = UsersInfoStorage.GetIfExists(ev->Get()->ClientId); - if (userInfo) { - i64 offset = Max<i64>(userInfo->Offset, 0); - result.SetClientOffset(userInfo->Offset); - TInstant tmp = userInfo->GetWriteTimestamp() ? userInfo->GetWriteTimestamp() : GetWriteTimeEstimate(offset); - result.SetWriteTimestampMS(tmp.MilliSeconds()); - result.SetCreateTimestampMS(userInfo->GetCreateTimestamp().MilliSeconds()); - result.SetClientReadOffset(userInfo->GetReadOffset()); - tmp = userInfo->GetReadWriteTimestamp() ? userInfo->GetReadWriteTimestamp() : GetWriteTimeEstimate(userInfo->GetReadOffset()); - result.SetReadWriteTimestampMS(tmp.MilliSeconds()); - result.SetReadCreateTimestampMS(userInfo->GetReadCreateTimestamp().MilliSeconds()); + TUserInfo* userInfo = UsersInfoStorage.GetIfExists(ev->Get()->ClientId); + if (userInfo) { + i64 offset = Max<i64>(userInfo->Offset, 0); + result.SetClientOffset(userInfo->Offset); + TInstant tmp = userInfo->GetWriteTimestamp() ? userInfo->GetWriteTimestamp() : GetWriteTimeEstimate(offset); + result.SetWriteTimestampMS(tmp.MilliSeconds()); + result.SetCreateTimestampMS(userInfo->GetCreateTimestamp().MilliSeconds()); + result.SetClientReadOffset(userInfo->GetReadOffset()); + tmp = userInfo->GetReadWriteTimestamp() ? userInfo->GetReadWriteTimestamp() : GetWriteTimeEstimate(userInfo->GetReadOffset()); + result.SetReadWriteTimestampMS(tmp.MilliSeconds()); + result.SetReadCreateTimestampMS(userInfo->GetReadCreateTimestamp().MilliSeconds()); } } ctx.Send(ev->Get()->Sender, new TEvPQ::TEvPartitionOffsetsResponse(result)); @@ -1979,43 +1979,43 @@ void TPartition::Handle(TEvPQ::TEvPartitionStatus::TPtr& ev, const TActorContext result.SetAvgQuotaSpeedPerDay(AvgQuotaBytes[3].GetValue()); result.SetSourceIdCount(SourceIdStorage.GetInMemorySourceIds().size()); - result.SetSourceIdRetentionPeriodSec((ctx.Now() - SourceIdStorage.MinAvailableTimestamp(ctx.Now())).Seconds()); + result.SetSourceIdRetentionPeriodSec((ctx.Now() - SourceIdStorage.MinAvailableTimestamp(ctx.Now())).Seconds()); result.SetWriteBytesQuota(WriteQuota.GetTotalSpeed()); TVector<ui64> resSpeed; resSpeed.resize(4); ui64 maxQuota = 0; - for (auto& userInfoPair : UsersInfoStorage.GetAll()) { - auto& userInfo = userInfoPair.second; - if (ev->Get()->ClientId.empty() || ev->Get()->ClientId == userInfo.User) { - Y_VERIFY(userInfo.AvgReadBytes.size() == 4); + for (auto& userInfoPair : UsersInfoStorage.GetAll()) { + auto& userInfo = userInfoPair.second; + if (ev->Get()->ClientId.empty() || ev->Get()->ClientId == userInfo.User) { + Y_VERIFY(userInfo.AvgReadBytes.size() == 4); for (ui32 i = 0; i < 4; ++i) { - resSpeed[i] += userInfo.AvgReadBytes[i].GetValue(); + resSpeed[i] += userInfo.AvgReadBytes[i].GetValue(); } - maxQuota += userInfo.ReadQuota.GetTotalSpeed(); + maxQuota += userInfo.ReadQuota.GetTotalSpeed(); } - if (ev->Get()->ClientId == userInfo.User) { //fill lags + if (ev->Get()->ClientId == userInfo.User) { //fill lags NKikimrPQ::TClientInfo* clientInfo = result.MutableLagsInfo(); - clientInfo->SetClientId(userInfo.User); + clientInfo->SetClientId(userInfo.User); auto write = clientInfo->MutableWritePosition(); write->SetOffset(userInfo.Offset); userInfo.EndOffset = EndOffset; - write->SetWriteTimestamp((userInfo.GetWriteTimestamp() ? userInfo.GetWriteTimestamp() : GetWriteTimeEstimate(userInfo.Offset)).MilliSeconds()); - write->SetCreateTimestamp(userInfo.GetCreateTimestamp().MilliSeconds()); + write->SetWriteTimestamp((userInfo.GetWriteTimestamp() ? userInfo.GetWriteTimestamp() : GetWriteTimeEstimate(userInfo.Offset)).MilliSeconds()); + write->SetCreateTimestamp(userInfo.GetCreateTimestamp().MilliSeconds()); auto read = clientInfo->MutableReadPosition(); read->SetOffset(userInfo.GetReadOffset()); - read->SetWriteTimestamp((userInfo.GetReadWriteTimestamp() ? userInfo.GetReadWriteTimestamp() : GetWriteTimeEstimate(userInfo.GetReadOffset())).MilliSeconds()); - read->SetCreateTimestamp(userInfo.GetReadCreateTimestamp().MilliSeconds()); + read->SetWriteTimestamp((userInfo.GetReadWriteTimestamp() ? userInfo.GetReadWriteTimestamp() : GetWriteTimeEstimate(userInfo.GetReadOffset())).MilliSeconds()); + read->SetCreateTimestamp(userInfo.GetReadCreateTimestamp().MilliSeconds()); write->SetSize(GetSizeLag(userInfo.Offset)); read->SetSize(GetSizeLag(userInfo.GetReadOffset())); clientInfo->SetReadLagMs(userInfo.GetReadOffset() < (i64)EndOffset - ? (userInfo.GetReadTimestamp() - TInstant::MilliSeconds(read->GetWriteTimestamp())).MilliSeconds() + ? (userInfo.GetReadTimestamp() - TInstant::MilliSeconds(read->GetWriteTimestamp())).MilliSeconds() : 0); - clientInfo->SetLastReadTimestampMs(userInfo.GetReadTimestamp().MilliSeconds()); + clientInfo->SetLastReadTimestampMs(userInfo.GetReadTimestamp().MilliSeconds()); clientInfo->SetWriteLagMs(userInfo.GetWriteLagMs()); - ui64 totalLag = clientInfo->GetReadLagMs() + userInfo.GetWriteLagMs() + (ctx.Now() - userInfo.GetReadTimestamp()).MilliSeconds(); + ui64 totalLag = clientInfo->GetReadLagMs() + userInfo.GetWriteLagMs() + (ctx.Now() - userInfo.GetReadTimestamp()).MilliSeconds(); clientInfo->SetTotalLagMs(totalLag); } } @@ -2033,8 +2033,8 @@ void TPartition::Handle(TEvPQ::TEvPartitionStatus::TPtr& ev, const TActorContext result.SetLastWriteTimestampMs(WriteTimestamp.MilliSeconds()); result.SetWriteLagMs(WriteLagMs.GetValue()); - *result.MutableErrors() = {Errors.begin(), Errors.end()}; - + *result.MutableErrors() = {Errors.begin(), Errors.end()}; + ctx.Send(ev->Get()->Sender, new TEvPQ::TEvPartitionStatusResponse(result)); } @@ -2055,57 +2055,57 @@ void TPartition::Handle(TEvPQ::TEvGetPartitionClientInfo::TPtr& ev, const TActor result.SetStartOffset(StartOffset); result.SetEndOffset(EndOffset); result.SetResponseTimestamp(ctx.Now().MilliSeconds()); - for (auto& pr : UsersInfoStorage.GetAll()) { + for (auto& pr : UsersInfoStorage.GetAll()) { TUserInfo& userInfo(pr.second); NKikimrPQ::TClientInfo& clientInfo = *result.AddClientInfo(); clientInfo.SetClientId(pr.first); auto& write = *clientInfo.MutableWritePosition(); write.SetOffset(userInfo.Offset); userInfo.EndOffset = EndOffset; - write.SetWriteTimestamp((userInfo.GetWriteTimestamp() ? userInfo.GetWriteTimestamp() : GetWriteTimeEstimate(userInfo.Offset)).MilliSeconds()); - write.SetCreateTimestamp(userInfo.GetCreateTimestamp().MilliSeconds()); + write.SetWriteTimestamp((userInfo.GetWriteTimestamp() ? userInfo.GetWriteTimestamp() : GetWriteTimeEstimate(userInfo.Offset)).MilliSeconds()); + write.SetCreateTimestamp(userInfo.GetCreateTimestamp().MilliSeconds()); auto& read = *clientInfo.MutableReadPosition(); read.SetOffset(userInfo.GetReadOffset()); - read.SetWriteTimestamp((userInfo.GetReadWriteTimestamp() ? userInfo.GetReadWriteTimestamp() : GetWriteTimeEstimate(userInfo.GetReadOffset())).MilliSeconds()); - read.SetCreateTimestamp(userInfo.GetReadCreateTimestamp().MilliSeconds()); + read.SetWriteTimestamp((userInfo.GetReadWriteTimestamp() ? userInfo.GetReadWriteTimestamp() : GetWriteTimeEstimate(userInfo.GetReadOffset())).MilliSeconds()); + read.SetCreateTimestamp(userInfo.GetReadCreateTimestamp().MilliSeconds()); write.SetSize(GetSizeLag(userInfo.Offset)); read.SetSize(GetSizeLag(userInfo.GetReadOffset())); } ctx.Send(ev->Get()->Sender, response.Release(), 0, ev->Cookie); } -void TPartition::Handle(TEvPersQueue::TEvReportPartitionError::TPtr& ev, const TActorContext& ctx) { - LogAndCollectError(ev->Get()->Record, ctx); -} - -void TPartition::LogAndCollectError(const NKikimrPQ::TStatusResponse::TErrorMessage& error, const TActorContext& ctx) { - if (Errors.size() == MAX_ERRORS_COUNT_TO_STORE) { - Errors.pop_front(); - } - Errors.push_back(error); - LOG_ERROR_S(ctx, error.GetService(), error.GetMessage()); -} - -void TPartition::LogAndCollectError(NKikimrServices::EServiceKikimr service, const TString& msg, const TActorContext& ctx) { - NKikimrPQ::TStatusResponse::TErrorMessage error; - error.SetTimestamp(ctx.Now().Seconds()); - error.SetService(service); - error.SetMessage(TStringBuilder() << "topic '" << TopicName << "' partition " << Partition << " got error: " << msg); - LogAndCollectError(error, ctx); -} - -std::pair<TInstant, TInstant> TPartition::GetTime(const TUserInfo& userInfo, ui64 offset) const +void TPartition::Handle(TEvPersQueue::TEvReportPartitionError::TPtr& ev, const TActorContext& ctx) { + LogAndCollectError(ev->Get()->Record, ctx); +} + +void TPartition::LogAndCollectError(const NKikimrPQ::TStatusResponse::TErrorMessage& error, const TActorContext& ctx) { + if (Errors.size() == MAX_ERRORS_COUNT_TO_STORE) { + Errors.pop_front(); + } + Errors.push_back(error); + LOG_ERROR_S(ctx, error.GetService(), error.GetMessage()); +} + +void TPartition::LogAndCollectError(NKikimrServices::EServiceKikimr service, const TString& msg, const TActorContext& ctx) { + NKikimrPQ::TStatusResponse::TErrorMessage error; + error.SetTimestamp(ctx.Now().Seconds()); + error.SetService(service); + error.SetMessage(TStringBuilder() << "topic '" << TopicName << "' partition " << Partition << " got error: " << msg); + LogAndCollectError(error, ctx); +} + +std::pair<TInstant, TInstant> TPartition::GetTime(const TUserInfo& userInfo, ui64 offset) const { - TInstant wtime = userInfo.WriteTimestamp > TInstant::Zero() ? userInfo.WriteTimestamp : GetWriteTimeEstimate(offset); + TInstant wtime = userInfo.WriteTimestamp > TInstant::Zero() ? userInfo.WriteTimestamp : GetWriteTimeEstimate(offset); return std::make_pair(wtime, userInfo.CreateTimestamp); } //zero means no such record -TInstant TPartition::GetWriteTimeEstimate(ui64 offset) const +TInstant TPartition::GetWriteTimeEstimate(ui64 offset) const { if (offset < StartOffset) offset = StartOffset; if (offset >= EndOffset) - return TInstant::Zero(); + return TInstant::Zero(); const std::deque<TDataKey>& container = (offset < Head.Offset || offset == Head.Offset && Head.PartNo > 0) ? DataKeysBody : HeadKeys; Y_VERIFY(!container.empty()); auto it = std::upper_bound(container.begin(), container.end(), offset, @@ -2116,17 +2116,17 @@ TInstant TPartition::GetWriteTimeEstimate(ui64 offset) const --it; if (it != container.begin()) --it; - return it->Timestamp; + return it->Timestamp; } void TPartition::Handle(TEvPQ::TEvGetClientOffset::TPtr& ev, const TActorContext& ctx) { - auto& userInfo = UsersInfoStorage.GetOrCreate(ev->Get()->ClientId, ctx); - Y_VERIFY(userInfo.Offset >= -1, "Unexpected Offset: %" PRIi64, userInfo.Offset); - ui64 offset = Max<i64>(userInfo.Offset, 0); - auto ts = GetTime(userInfo, offset); + auto& userInfo = UsersInfoStorage.GetOrCreate(ev->Get()->ClientId, ctx); + Y_VERIFY(userInfo.Offset >= -1, "Unexpected Offset: %" PRIi64, userInfo.Offset); + ui64 offset = Max<i64>(userInfo.Offset, 0); + auto ts = GetTime(userInfo, offset); Counters.Cumulative()[COUNTER_PQ_GET_CLIENT_OFFSET_OK].Increment(1); - ReplyGetClientOffsetOk(ctx, ev->Get()->Cookie, userInfo.Offset, ts.first, ts.second); + ReplyGetClientOffsetOk(ctx, ev->Get()->Cookie, userInfo.Offset, ts.first, ts.second); } void TPartition::Handle(TEvPQ::TEvUpdateWriteTimestamp::TPtr& ev, const TActorContext& ctx) { @@ -2143,12 +2143,12 @@ void TPartition::Handle(TEvPQ::TEvUpdateWriteTimestamp::TPtr& ev, const TActorCo void TPartition::Handle(TEvPQ::TEvSetClientInfo::TPtr& ev, const TActorContext& ctx) { const TString& user = ev->Get()->ClientId; - auto& userInfo = UsersInfoStorage.GetOrCreate(user, ctx); + auto& userInfo = UsersInfoStorage.GetOrCreate(user, ctx); - if (userInfo.UserActs.size() > MAX_USER_ACTS) { + if (userInfo.UserActs.size() > MAX_USER_ACTS) { Counters.Cumulative()[COUNTER_PQ_SET_CLIENT_OFFSET_ERROR].Increment(1); ReplyError(ctx, ev->Get()->Cookie, NPersQueue::NErrorCode::OVERLOAD, - TStringBuilder() << "too big inflight: " << userInfo.UserActs.size()); + TStringBuilder() << "too big inflight: " << userInfo.UserActs.size()); return; } @@ -2156,7 +2156,7 @@ void TPartition::Handle(TEvPQ::TEvSetClientInfo::TPtr& ev, const TActorContext& const ui64& offset = ev->Get()->Offset; Y_VERIFY(offset <= (ui64)Max<i64>(), "Unexpected Offset: %" PRIu64, offset); - userInfo.UserActs.push_back(ev->Release()); + userInfo.UserActs.push_back(ev->Release()); ProcessUserActs(userInfo, ctx); } @@ -2208,33 +2208,33 @@ void TPartition::Handle(TEvPQ::TEvBlobResponse::TPtr& ev, const TActorContext& c auto it = ReadInfo.find(cookie); Y_VERIFY(it != ReadInfo.end()); - TReadInfo info = std::move(it->second); - ReadInfo.erase(it); + TReadInfo info = std::move(it->second); + ReadInfo.erase(it); //make readinfo class - TReadAnswer answer(info.FormAnswer( - ctx, *ev->Get(), EndOffset, Partition, &UsersInfoStorage.GetOrCreate(info.User, ctx), - info.Destination, GetSizeLag(info.Offset) - )); + TReadAnswer answer(info.FormAnswer( + ctx, *ev->Get(), EndOffset, Partition, &UsersInfoStorage.GetOrCreate(info.User, ctx), + info.Destination, GetSizeLag(info.Offset) + )); if (HasError(*ev->Get())) { - if (info.IsSubscription) { + if (info.IsSubscription) { Counters.Cumulative()[COUNTER_PQ_READ_SUBSCRIPTION_ERROR].Increment(1); } Counters.Cumulative()[COUNTER_PQ_READ_ERROR].Increment(1); - Counters.Percentile()[COUNTER_LATENCY_PQ_READ_ERROR].IncrementFor((ctx.Now() - info.Timestamp).MilliSeconds()); + Counters.Percentile()[COUNTER_LATENCY_PQ_READ_ERROR].IncrementFor((ctx.Now() - info.Timestamp).MilliSeconds()); } else { - if (info.IsSubscription) { + if (info.IsSubscription) { Counters.Cumulative()[COUNTER_PQ_READ_SUBSCRIPTION_OK].Increment(1); } - const auto& resp = dynamic_cast<TEvPQ::TEvProxyResponse*>(answer.Event.Get())->Response; + const auto& resp = dynamic_cast<TEvPQ::TEvProxyResponse*>(answer.Event.Get())->Response; Counters.Cumulative()[COUNTER_PQ_READ_OK].Increment(1); - Counters.Percentile()[COUNTER_LATENCY_PQ_READ_OK].IncrementFor((ctx.Now() - info.Timestamp).MilliSeconds()); + Counters.Percentile()[COUNTER_LATENCY_PQ_READ_OK].IncrementFor((ctx.Now() - info.Timestamp).MilliSeconds()); Counters.Cumulative()[COUNTER_PQ_READ_BYTES].Increment(resp.ByteSize()); } - ctx.Send(info.Destination != 0 ? Tablet : ctx.SelfID, answer.Event.Release()); + ctx.Send(info.Destination != 0 ? Tablet : ctx.SelfID, answer.Event.Release()); ReportLabeledCounters(ctx); - OnReadRequestFinished(std::move(info), answer.Size); + OnReadRequestFinished(std::move(info), answer.Size); } @@ -2246,8 +2246,8 @@ static void AddResultBlob(T* read, const TClientBlob& blob, ui64 offset) cc->SetData(blob.Data); cc->SetSourceId(blob.SourceId); cc->SetSeqNo(blob.SeqNo); - cc->SetWriteTimestampMS(blob.WriteTimestamp.MilliSeconds()); - cc->SetCreateTimestampMS(blob.CreateTimestamp.MilliSeconds()); + cc->SetWriteTimestampMS(blob.WriteTimestamp.MilliSeconds()); + cc->SetCreateTimestampMS(blob.CreateTimestamp.MilliSeconds()); cc->SetUncompressedSize(blob.UncompressedSize); cc->SetPartitionKey(blob.PartitionKey); cc->SetExplicitHash(blob.ExplicitHashKey); @@ -2280,31 +2280,31 @@ static void AddResultDebugInfo(const TEvPQ::TEvBlobResponse* response, T* readRe readResult->SetBlobsFromDisk(diskBlobs); } -TReadAnswer TReadInfo::FormAnswer( - const TActorContext& ctx, - const TEvPQ::TEvBlobResponse& blobResponse, - const ui64 endOffset, - const ui32 partition, - TUserInfo* userInfo, - const ui64 cookie, - const ui64 sizeLag -) { +TReadAnswer TReadInfo::FormAnswer( + const TActorContext& ctx, + const TEvPQ::TEvBlobResponse& blobResponse, + const ui64 endOffset, + const ui32 partition, + TUserInfo* userInfo, + const ui64 cookie, + const ui64 sizeLag +) { Y_UNUSED(partition); THolder<TEvPQ::TEvProxyResponse> answer = MakeHolder<TEvPQ::TEvProxyResponse>(cookie); NKikimrClient::TResponse& res = answer->Response; const TEvPQ::TEvBlobResponse* response = &blobResponse; if (HasError(blobResponse)) { - return TReadAnswer{ - blobResponse.Error.ErrorStr.size(), + return TReadAnswer{ + blobResponse.Error.ErrorStr.size(), MakeHolder<TEvPQ::TEvError>(blobResponse.Error.ErrorCode, blobResponse.Error.ErrorStr, cookie) - }; + }; } res.SetStatus(NMsgBusProxy::MSTATUS_OK); res.SetErrorCode(NPersQueue::NErrorCode::OK); auto readResult = res.MutablePartitionResponse()->MutableCmdReadResult(); - readResult->SetWaitQuotaTimeMs(WaitQuotaTime.MilliSeconds()); + readResult->SetWaitQuotaTimeMs(WaitQuotaTime.MilliSeconds()); readResult->SetMaxOffset(endOffset); readResult->SetRealReadOffset(Offset); Y_VERIFY(endOffset <= (ui64)Max<i64>(), "Max offset is too big: %" PRIu64, endOffset); @@ -2332,8 +2332,8 @@ TReadAnswer TReadInfo::FormAnswer( if (blobValue.empty()) { // this is ok. Means that someone requested too much data LOG_DEBUG(ctx, NKikimrServices::PERSQUEUE, "Not full answer here!"); - ui64 answerSize = answer->Response.ByteSize(); - if (userInfo && Destination != 0) { + ui64 answerSize = answer->Response.ByteSize(); + if (userInfo && Destination != 0) { userInfo->ReadDone(ctx, ctx.Now(), answerSize, cnt, ClientDC); } readResult->SetSizeLag(sizeLag - size); @@ -2393,7 +2393,7 @@ TReadAnswer TReadInfo::FormAnswer( if (res.IsLastPart()) { ++cnt; } - } + } if (res.IsLastPart()) { PartNo = 0; @@ -2426,12 +2426,12 @@ TReadAnswer TReadInfo::FormAnswer( readResult->SetBlobsCachedSize(readResult->GetBlobsCachedSize() + writeBlob.GetBlobSize()); size += writeBlob.GetBlobSize(); - if (userInfo) { - userInfo->AddTimestampToCache( - Offset, writeBlob.WriteTimestamp, writeBlob.CreateTimestamp, - Destination != 0, ctx.Now() - ); - } + if (userInfo) { + userInfo->AddTimestampToCache( + Offset, writeBlob.WriteTimestamp, writeBlob.CreateTimestamp, + Destination != 0, ctx.Now() + ); + } AddResultBlob(readResult, writeBlob, Offset); if (writeBlob.IsLastPart()) { ++Offset; @@ -2442,8 +2442,8 @@ TReadAnswer TReadInfo::FormAnswer( Y_VERIFY(pcnt <= Count && psize <= Size); Y_VERIFY(pcnt <= cnt && psize <= size); Y_VERIFY(Offset <= (ui64)Max<i64>(), "Offset is too big: %" PRIu64, Offset); - ui64 answerSize = answer->Response.ByteSize(); - if (userInfo && Destination != 0) { + ui64 answerSize = answer->Response.ByteSize(); + if (userInfo && Destination != 0) { userInfo->ReadDone(ctx, ctx.Now(), answerSize, cnt, ClientDC); } readResult->SetSizeLag(sizeLag - size); @@ -2458,17 +2458,17 @@ void TPartition::HandleOnIdle(TEvPQ::TEvWrite::TPtr& ev, const TActorContext& ct void TPartition::Handle(TEvPQ::TEvReadTimeout::TPtr& ev, const TActorContext& ctx) { - auto res = Subscriber.OnTimeout(ev); + auto res = Subscriber.OnTimeout(ev); if (!res) return; - TReadAnswer answer(res->FormAnswer(ctx, res->Offset, Partition, nullptr, res->Destination, 0)); - ctx.Send(Tablet, answer.Event.Release()); - LOG_DEBUG_S(ctx, NKikimrServices::PERSQUEUE, " waiting read cookie " << ev->Get()->Cookie - << " partition " << Partition << " read timeout for " << res->User << " offset " << res->Offset); - auto& userInfo = UsersInfoStorage.GetOrCreate(res->User, ctx); - - userInfo.ForgetSubscription(ctx.Now()); - OnReadRequestFinished(std::move(res.GetRef()), answer.Size); + TReadAnswer answer(res->FormAnswer(ctx, res->Offset, Partition, nullptr, res->Destination, 0)); + ctx.Send(Tablet, answer.Event.Release()); + LOG_DEBUG_S(ctx, NKikimrServices::PERSQUEUE, " waiting read cookie " << ev->Get()->Cookie + << " partition " << Partition << " read timeout for " << res->User << " offset " << res->Offset); + auto& userInfo = UsersInfoStorage.GetOrCreate(res->User, ctx); + + userInfo.ForgetSubscription(ctx.Now()); + OnReadRequestFinished(std::move(res.GetRef()), answer.Size); } @@ -2605,10 +2605,10 @@ void TPartition::Handle(TEvPQ::TEvRead::TPtr& ev, const TActorContext& ctx) { Y_VERIFY(read->Offset <= EndOffset); - auto& userInfo = UsersInfoStorage.GetOrCreate(user, ctx); + auto& userInfo = UsersInfoStorage.GetOrCreate(user, ctx); if (!read->SessionId.empty()) { - if (userInfo.Session != read->SessionId) { + if (userInfo.Session != read->SessionId) { Counters.Cumulative()[COUNTER_PQ_READ_ERROR_NO_SESSION].Increment(1); Counters.Percentile()[COUNTER_LATENCY_PQ_READ_ERROR].IncrementFor(0); ReplyError(ctx, read->Cookie, NPersQueue::NErrorCode::READ_ERROR_NO_SESSION, @@ -2617,32 +2617,32 @@ void TPartition::Handle(TEvPQ::TEvRead::TPtr& ev, const TActorContext& ctx) { } } - if (userInfo.ReadSpeedLimiter) { - Send(userInfo.ReadSpeedLimiter->Actor, new NReadSpeedLimiterEvents::TEvRequest(ev.Release())); - } else { - DoRead(ev.Release(), TDuration::Zero(), ctx); - } -} - -void TPartition::Handle(NReadSpeedLimiterEvents::TEvResponse::TPtr& ev, const TActorContext& ctx) { - DoRead(ev->Get()->ReadRequest.Release(), ev->Get()->WaitTime, ctx); -} - -void TPartition::DoRead(TEvPQ::TEvRead::TPtr ev, TDuration waitQuotaTime, const TActorContext& ctx) { - auto read = ev->Get(); - const TString& user = read->ClientId; - auto& userInfo = UsersInfoStorage.GetOrCreate(user, ctx); - + if (userInfo.ReadSpeedLimiter) { + Send(userInfo.ReadSpeedLimiter->Actor, new NReadSpeedLimiterEvents::TEvRequest(ev.Release())); + } else { + DoRead(ev.Release(), TDuration::Zero(), ctx); + } +} + +void TPartition::Handle(NReadSpeedLimiterEvents::TEvResponse::TPtr& ev, const TActorContext& ctx) { + DoRead(ev->Get()->ReadRequest.Release(), ev->Get()->WaitTime, ctx); +} + +void TPartition::DoRead(TEvPQ::TEvRead::TPtr ev, TDuration waitQuotaTime, const TActorContext& ctx) { + auto read = ev->Get(); + const TString& user = read->ClientId; + auto& userInfo = UsersInfoStorage.GetOrCreate(user, ctx); + ui64 offset = read->Offset; - if (read->MaxTimeLagMs > 0 || read->ReadTimestampMs > 0 || userInfo.ReadFromTimestamp > TInstant::MilliSeconds(1)) { - TInstant timestamp = read->MaxTimeLagMs > 0 ? ctx.Now() - TDuration::MilliSeconds(read->MaxTimeLagMs) : TInstant::Zero(); - timestamp = Max(timestamp, TInstant::MilliSeconds(read->ReadTimestampMs)); - timestamp = Max(timestamp, userInfo.ReadFromTimestamp); + if (read->MaxTimeLagMs > 0 || read->ReadTimestampMs > 0 || userInfo.ReadFromTimestamp > TInstant::MilliSeconds(1)) { + TInstant timestamp = read->MaxTimeLagMs > 0 ? ctx.Now() - TDuration::MilliSeconds(read->MaxTimeLagMs) : TInstant::Zero(); + timestamp = Max(timestamp, TInstant::MilliSeconds(read->ReadTimestampMs)); + timestamp = Max(timestamp, userInfo.ReadFromTimestamp); offset = Max(GetOffsetEstimate(DataKeysBody, timestamp, Min(Head.Offset, EndOffset - 1)), offset); - userInfo.ReadOffsetRewindSum += offset - read->Offset; + userInfo.ReadOffsetRewindSum += offset - read->Offset; } - TReadInfo info(user, read->ClientDC, offset, read->PartNo, read->Count, read->Size, read->Cookie, read->ReadTimestampMs, waitQuotaTime); + TReadInfo info(user, read->ClientDC, offset, read->PartNo, read->Count, read->Size, read->Cookie, read->ReadTimestampMs, waitQuotaTime); ui64 cookie = Cookie++; @@ -2659,8 +2659,8 @@ void TPartition::DoRead(TEvPQ::TEvRead::TPtr ev, TDuration waitQuotaTime, const read->Timeout = 30000; } Subscriber.AddSubscription(std::move(info), read->Timeout, cookie, ctx); - ++userInfo.Subscriptions; - userInfo.UpdateReadOffset((i64)offset - 1, userInfo.WriteTimestamp, userInfo.CreateTimestamp, ctx.Now()); + ++userInfo.Subscriptions; + userInfo.UpdateReadOffset((i64)offset - 1, userInfo.WriteTimestamp, userInfo.CreateTimestamp, ctx.Now()); return; } @@ -2670,18 +2670,18 @@ void TPartition::DoRead(TEvPQ::TEvRead::TPtr ev, TDuration waitQuotaTime, const ProcessRead(ctx, std::move(info), cookie, false); } -void TPartition::OnReadRequestFinished(TReadInfo&& info, ui64 answerSize) { - auto userInfo = UsersInfoStorage.GetIfExists(info.User); - Y_VERIFY(userInfo); +void TPartition::OnReadRequestFinished(TReadInfo&& info, ui64 answerSize) { + auto userInfo = UsersInfoStorage.GetIfExists(info.User); + Y_VERIFY(userInfo); + + if (userInfo->ReadSpeedLimiter) { + Send( + userInfo->ReadSpeedLimiter->Actor, + new NReadSpeedLimiterEvents::TEvConsumed(answerSize, info.Destination) + ); + } +} - if (userInfo->ReadSpeedLimiter) { - Send( - userInfo->ReadSpeedLimiter->Actor, - new NReadSpeedLimiterEvents::TEvConsumed(answerSize, info.Destination) - ); - } -} - void TPartition::AnswerCurrentWrites(const TActorContext& ctx) { ui64 offset = EndOffset; @@ -2729,11 +2729,11 @@ void TPartition::AnswerCurrentWrites(const TActorContext& ctx) Counters.Cumulative()[COUNTER_PQ_WRITE_OK].Increment(1); } - ReplyWrite( - ctx, writeResponse.Cookie, s, seqNo, partNo, totalParts, - already ? maxOffset : offset, CurrentTimestamp, already, maxSeqNo, - quotedTime, TopicQuotaWaitTimeForCurrentBlob, queueTime, writeTime - ); + ReplyWrite( + ctx, writeResponse.Cookie, s, seqNo, partNo, totalParts, + already ? maxOffset : offset, CurrentTimestamp, already, maxSeqNo, + quotedTime, TopicQuotaWaitTimeForCurrentBlob, queueTime, writeTime + ); LOG_DEBUG_S( ctx, NKikimrServices::PERSQUEUE, @@ -2797,28 +2797,28 @@ void TPartition::AnswerCurrentWrites(const TActorContext& ctx) } -void TPartition::ReadTimestampForOffset(const TString& user, TUserInfo& userInfo, const TActorContext& ctx) +void TPartition::ReadTimestampForOffset(const TString& user, TUserInfo& userInfo, const TActorContext& ctx) { - if (userInfo.ReadScheduled) + if (userInfo.ReadScheduled) return; - userInfo.ReadScheduled = true; + userInfo.ReadScheduled = true; LOG_DEBUG_S(ctx, NKikimrServices::PERSQUEUE, "Topic '" << TopicName << "' partition " << Partition - << " user " << user << " readTimeStamp for offset " << userInfo.Offset << " initiated " << " queuesize " << UpdateUserInfoTimestamp.size() << " startOffset " << StartOffset << " ReadingTimestamp " << ReadingTimestamp); + << " user " << user << " readTimeStamp for offset " << userInfo.Offset << " initiated " << " queuesize " << UpdateUserInfoTimestamp.size() << " startOffset " << StartOffset << " ReadingTimestamp " << ReadingTimestamp); if (ReadingTimestamp) { UpdateUserInfoTimestamp.push_back(user); return; } - if (userInfo.Offset < (i64)StartOffset) { - userInfo.ReadScheduled = false; - auto now = ctx.Now(); + if (userInfo.Offset < (i64)StartOffset) { + userInfo.ReadScheduled = false; + auto now = ctx.Now(); userInfo.CreateTimestamp = now - TDuration::Seconds(Max(86400, Config.GetPartitionConfig().GetLifetimeSeconds())); userInfo.WriteTimestamp = now - TDuration::Seconds(Max(86400, Config.GetPartitionConfig().GetLifetimeSeconds())); - userInfo.ActualTimestamps = true; - if (userInfo.ReadOffset + 1 < userInfo.Offset) { - userInfo.ReadOffset = userInfo.Offset - 1; - userInfo.ReadCreateTimestamp = userInfo.CreateTimestamp; - userInfo.ReadWriteTimestamp = userInfo.WriteTimestamp; + userInfo.ActualTimestamps = true; + if (userInfo.ReadOffset + 1 < userInfo.Offset) { + userInfo.ReadOffset = userInfo.Offset - 1; + userInfo.ReadCreateTimestamp = userInfo.CreateTimestamp; + userInfo.ReadWriteTimestamp = userInfo.WriteTimestamp; } Counters.Cumulative()[COUNTER_PQ_WRITE_TIMESTAMP_OFFSET_IS_LOST].Increment(1); @@ -2827,7 +2827,7 @@ void TPartition::ReadTimestampForOffset(const TString& user, TUserInfo& userInfo } if (userInfo.Offset >= (i64)EndOffset || StartOffset == EndOffset) { - userInfo.ReadScheduled = false; + userInfo.ReadScheduled = false; ReportLabeledCounters(ctx); return; } @@ -2836,7 +2836,7 @@ void TPartition::ReadTimestampForOffset(const TString& user, TUserInfo& userInfo ReadingTimestamp = true; ReadingForUser = user; - ReadingForOffset = userInfo.Offset; + ReadingForOffset = userInfo.Offset; ReadingForUserReadRuleGeneration = userInfo.ReadRuleGeneration; for (const auto& user : UpdateUserInfoTimestamp) { @@ -2844,7 +2844,7 @@ void TPartition::ReadTimestampForOffset(const TString& user, TUserInfo& userInfo } LOG_DEBUG_S(ctx, NKikimrServices::PERSQUEUE, "Topic '" << TopicName << "' partition " << Partition - << " user " << user << " send read request for offset " << userInfo.Offset << " initiated " << " queuesize " << UpdateUserInfoTimestamp.size() << " startOffset " << StartOffset << " ReadingTimestamp " << ReadingTimestamp); + << " user " << user << " send read request for offset " << userInfo.Offset << " initiated " << " queuesize " << UpdateUserInfoTimestamp.size() << " startOffset " << StartOffset << " ReadingTimestamp " << ReadingTimestamp); THolder<TEvPQ::TEvRead> event = MakeHolder<TEvPQ::TEvRead>(0, userInfo.Offset, 0, 1, "", @@ -2856,9 +2856,9 @@ void TPartition::ReadTimestampForOffset(const TString& user, TUserInfo& userInfo void TPartition::ProcessTimestampsForNewData(const ui64 prevEndOffset, const TActorContext& ctx) { - for (auto& userInfoPair : UsersInfoStorage.GetAll()) { - if (userInfoPair.second.Offset >= (i64)prevEndOffset && userInfoPair.second.Offset < (i64)EndOffset) { - ReadTimestampForOffset(userInfoPair.first, userInfoPair.second, ctx); + for (auto& userInfoPair : UsersInfoStorage.GetAll()) { + if (userInfoPair.second.Offset >= (i64)prevEndOffset && userInfoPair.second.Offset < (i64)EndOffset) { + ReadTimestampForOffset(userInfoPair.first, userInfoPair.second, ctx); } } } @@ -2880,7 +2880,7 @@ void TPartition::Handle(TEvPQ::TEvProxyResponse::TPtr& ev, const TActorContext& << " queuesize " << UpdateUserInfoTimestamp.size() << " startOffset " << StartOffset); if (!userInfo->ActualTimestamps) { - LOG_INFO_S( + LOG_INFO_S( ctx, NKikimrServices::PERSQUEUE, "Reading Timestamp failed for offset " << ReadingForOffset << " ( "<< userInfo->Offset << " ) " << ev->Get()->Response.DebugString() @@ -2992,7 +2992,7 @@ void TPartition::SyncMemoryStateWithKVState(const TActorContext& ctx) HeadKeys.pop_back(); } HeadKeys.push_back(NewHeadKey); - NewHeadKey = TDataKey{TKey{}, 0, TInstant::Zero(), 0}; + NewHeadKey = TDataKey{TKey{}, 0, TInstant::Zero(), 0}; } if (CompactedKeys.empty() && NewHead.PackedSize == 0) { //Nothing writed at all @@ -3022,7 +3022,7 @@ void TPartition::SyncMemoryStateWithKVState(const TActorContext& ctx) GapSize += ck.first.GetOffset() - lastOffset; } } - DataKeysBody.push_back({ck.first, ck.second, ctx.Now(), DataKeysBody.empty() ? 0 : DataKeysBody.back().CumulativeSize + DataKeysBody.back().Size}); + DataKeysBody.push_back({ck.first, ck.second, ctx.Now(), DataKeysBody.empty() ? 0 : DataKeysBody.back().CumulativeSize + DataKeysBody.back().Size}); CompactedKeys.pop_front(); } // head cleared, all data moved to body @@ -3072,108 +3072,108 @@ void TPartition::ReportLabeledCounters(const TActorContext& ctx) { //per client counters const auto now = ctx.Now(); - for (auto& userInfoPair : UsersInfoStorage.GetAll()) { - auto& userInfo = userInfoPair.second; - if (!userInfo.HasReadRule && !userInfo.Important) + for (auto& userInfoPair : UsersInfoStorage.GetAll()) { + auto& userInfo = userInfoPair.second; + if (!userInfo.HasReadRule && !userInfo.Important) continue; bool haveChanges = false; - userInfo.EndOffset = EndOffset; - userInfo.UpdateReadingTimeAndState(now); - ui64 ts = userInfo.GetWriteTimestamp().MilliSeconds(); + userInfo.EndOffset = EndOffset; + userInfo.UpdateReadingTimeAndState(now); + ui64 ts = userInfo.GetWriteTimestamp().MilliSeconds(); if (ts < MIN_TIMESTAMP_MS) ts = Max<i64>(); - if (userInfo.LabeledCounters.GetCounters()[METRIC_COMMIT_WRITE_TIME].Get() != ts) { + if (userInfo.LabeledCounters.GetCounters()[METRIC_COMMIT_WRITE_TIME].Get() != ts) { haveChanges = true; - userInfo.LabeledCounters.GetCounters()[METRIC_COMMIT_WRITE_TIME].Set(ts); + userInfo.LabeledCounters.GetCounters()[METRIC_COMMIT_WRITE_TIME].Set(ts); } - ts = userInfo.GetCreateTimestamp().MilliSeconds(); + ts = userInfo.GetCreateTimestamp().MilliSeconds(); if (ts < MIN_TIMESTAMP_MS) ts = Max<i64>(); - if (userInfo.LabeledCounters.GetCounters()[METRIC_COMMIT_CREATE_TIME].Get() != ts) { + if (userInfo.LabeledCounters.GetCounters()[METRIC_COMMIT_CREATE_TIME].Get() != ts) { haveChanges = true; - userInfo.LabeledCounters.GetCounters()[METRIC_COMMIT_CREATE_TIME].Set(ts); + userInfo.LabeledCounters.GetCounters()[METRIC_COMMIT_CREATE_TIME].Set(ts); } - ts = userInfo.GetReadWriteTimestamp().MilliSeconds(); - if (userInfo.LabeledCounters.GetCounters()[METRIC_READ_WRITE_TIME].Get() != ts) { + ts = userInfo.GetReadWriteTimestamp().MilliSeconds(); + if (userInfo.LabeledCounters.GetCounters()[METRIC_READ_WRITE_TIME].Get() != ts) { haveChanges = true; - userInfo.LabeledCounters.GetCounters()[METRIC_READ_WRITE_TIME].Set(ts); + userInfo.LabeledCounters.GetCounters()[METRIC_READ_WRITE_TIME].Set(ts); } - i64 off = userInfo.GetReadOffset(); //we want to track first not-readed offset - TInstant wts = userInfo.GetReadWriteTimestamp() ? userInfo.GetReadWriteTimestamp() : GetWriteTimeEstimate(userInfo.GetReadOffset()); - TInstant readTimestamp = userInfo.GetReadTimestamp(); - ui64 readTimeLag = off >= (i64)EndOffset ? 0 : (readTimestamp - wts).MilliSeconds(); - ui64 totalLag = userInfo.GetWriteLagMs() + readTimeLag + (now - readTimestamp).MilliSeconds(); + i64 off = userInfo.GetReadOffset(); //we want to track first not-readed offset + TInstant wts = userInfo.GetReadWriteTimestamp() ? userInfo.GetReadWriteTimestamp() : GetWriteTimeEstimate(userInfo.GetReadOffset()); + TInstant readTimestamp = userInfo.GetReadTimestamp(); + ui64 readTimeLag = off >= (i64)EndOffset ? 0 : (readTimestamp - wts).MilliSeconds(); + ui64 totalLag = userInfo.GetWriteLagMs() + readTimeLag + (now - readTimestamp).MilliSeconds(); - if (userInfo.LabeledCounters.GetCounters()[METRIC_READ_TOTAL_TIME].Get() != totalLag) { + if (userInfo.LabeledCounters.GetCounters()[METRIC_READ_TOTAL_TIME].Get() != totalLag) { haveChanges = true; - userInfo.LabeledCounters.GetCounters()[METRIC_READ_TOTAL_TIME].Set(totalLag); + userInfo.LabeledCounters.GetCounters()[METRIC_READ_TOTAL_TIME].Set(totalLag); } - ts = readTimestamp.MilliSeconds(); - if (userInfo.LabeledCounters.GetCounters()[METRIC_LAST_READ_TIME].Get() != ts) { + ts = readTimestamp.MilliSeconds(); + if (userInfo.LabeledCounters.GetCounters()[METRIC_LAST_READ_TIME].Get() != ts) { haveChanges = true; - userInfo.LabeledCounters.GetCounters()[METRIC_LAST_READ_TIME].Set(ts); + userInfo.LabeledCounters.GetCounters()[METRIC_LAST_READ_TIME].Set(ts); } - ui64 timeLag = userInfo.GetWriteLagMs(); - if (userInfo.LabeledCounters.GetCounters()[METRIC_WRITE_TIME_LAG].Get() != timeLag) { + ui64 timeLag = userInfo.GetWriteLagMs(); + if (userInfo.LabeledCounters.GetCounters()[METRIC_WRITE_TIME_LAG].Get() != timeLag) { haveChanges = true; - userInfo.LabeledCounters.GetCounters()[METRIC_WRITE_TIME_LAG].Set(timeLag); + userInfo.LabeledCounters.GetCounters()[METRIC_WRITE_TIME_LAG].Set(timeLag); } - if (userInfo.LabeledCounters.GetCounters()[METRIC_READ_TIME_LAG].Get() != readTimeLag) { + if (userInfo.LabeledCounters.GetCounters()[METRIC_READ_TIME_LAG].Get() != readTimeLag) { haveChanges = true; - userInfo.LabeledCounters.GetCounters()[METRIC_READ_TIME_LAG].Set(readTimeLag); + userInfo.LabeledCounters.GetCounters()[METRIC_READ_TIME_LAG].Set(readTimeLag); } - if (userInfo.LabeledCounters.GetCounters()[METRIC_COMMIT_MESSAGE_LAG].Get() != EndOffset - userInfo.Offset) { + if (userInfo.LabeledCounters.GetCounters()[METRIC_COMMIT_MESSAGE_LAG].Get() != EndOffset - userInfo.Offset) { haveChanges = true; - userInfo.LabeledCounters.GetCounters()[METRIC_COMMIT_MESSAGE_LAG].Set(EndOffset - userInfo.Offset); + userInfo.LabeledCounters.GetCounters()[METRIC_COMMIT_MESSAGE_LAG].Set(EndOffset - userInfo.Offset); } - if (userInfo.LabeledCounters.GetCounters()[METRIC_READ_MESSAGE_LAG].Get() != EndOffset - off) { + if (userInfo.LabeledCounters.GetCounters()[METRIC_READ_MESSAGE_LAG].Get() != EndOffset - off) { haveChanges = true; - userInfo.LabeledCounters.GetCounters()[METRIC_READ_MESSAGE_LAG].Set(EndOffset - off); - userInfo.LabeledCounters.GetCounters()[METRIC_READ_TOTAL_MESSAGE_LAG].Set(EndOffset - off); + userInfo.LabeledCounters.GetCounters()[METRIC_READ_MESSAGE_LAG].Set(EndOffset - off); + userInfo.LabeledCounters.GetCounters()[METRIC_READ_TOTAL_MESSAGE_LAG].Set(EndOffset - off); } - ui64 sizeLag = GetSizeLag(userInfo.Offset); - ui64 sizeLagRead = GetSizeLag(userInfo.ReadOffset); - if (userInfo.LabeledCounters.GetCounters()[METRIC_COMMIT_SIZE_LAG].Get() != sizeLag) { + ui64 sizeLag = GetSizeLag(userInfo.Offset); + ui64 sizeLagRead = GetSizeLag(userInfo.ReadOffset); + if (userInfo.LabeledCounters.GetCounters()[METRIC_COMMIT_SIZE_LAG].Get() != sizeLag) { haveChanges = true; - userInfo.LabeledCounters.GetCounters()[METRIC_COMMIT_SIZE_LAG].Set(sizeLag); + userInfo.LabeledCounters.GetCounters()[METRIC_COMMIT_SIZE_LAG].Set(sizeLag); } - if (userInfo.LabeledCounters.GetCounters()[METRIC_READ_SIZE_LAG].Get() != sizeLagRead) { + if (userInfo.LabeledCounters.GetCounters()[METRIC_READ_SIZE_LAG].Get() != sizeLagRead) { haveChanges = true; - userInfo.LabeledCounters.GetCounters()[METRIC_READ_SIZE_LAG].Set(sizeLagRead); - userInfo.LabeledCounters.GetCounters()[METRIC_READ_TOTAL_SIZE_LAG].Set(sizeLag); + userInfo.LabeledCounters.GetCounters()[METRIC_READ_SIZE_LAG].Set(sizeLagRead); + userInfo.LabeledCounters.GetCounters()[METRIC_READ_TOTAL_SIZE_LAG].Set(sizeLag); } - if (userInfo.LabeledCounters.GetCounters()[METRIC_USER_PARTITIONS].Get() == 0) { + if (userInfo.LabeledCounters.GetCounters()[METRIC_USER_PARTITIONS].Get() == 0) { haveChanges = true; - userInfo.LabeledCounters.GetCounters()[METRIC_USER_PARTITIONS].Set(1); + userInfo.LabeledCounters.GetCounters()[METRIC_USER_PARTITIONS].Set(1); } - ui64 speed = userInfo.ReadQuota.GetTotalSpeed(); - if (speed != userInfo.LabeledCounters.GetCounters()[METRIC_READ_QUOTA_BYTES].Get()) { + ui64 speed = userInfo.ReadQuota.GetTotalSpeed(); + if (speed != userInfo.LabeledCounters.GetCounters()[METRIC_READ_QUOTA_BYTES].Get()) { haveChanges = true; - userInfo.LabeledCounters.GetCounters()[METRIC_READ_QUOTA_BYTES].Set(speed); + userInfo.LabeledCounters.GetCounters()[METRIC_READ_QUOTA_BYTES].Set(speed); } - ui64 availSec = userInfo.ReadQuota.GetAvailableAvgSec(ctx.Now()); - if (availSec != userInfo.LabeledCounters.GetCounters()[METRIC_MIN_READ_QUOTA_BYTES_AVAIL_SEC].Get()) { + ui64 availSec = userInfo.ReadQuota.GetAvailableAvgSec(ctx.Now()); + if (availSec != userInfo.LabeledCounters.GetCounters()[METRIC_MIN_READ_QUOTA_BYTES_AVAIL_SEC].Get()) { haveChanges = true; - userInfo.LabeledCounters.GetCounters()[METRIC_MIN_READ_QUOTA_BYTES_AVAIL_SEC].Set(availSec); + userInfo.LabeledCounters.GetCounters()[METRIC_MIN_READ_QUOTA_BYTES_AVAIL_SEC].Set(availSec); } - ui64 availMin = userInfo.ReadQuota.GetAvailableAvgMin(ctx.Now()); - if (availMin != userInfo.LabeledCounters.GetCounters()[METRIC_MIN_READ_QUOTA_BYTES_AVAIL_MIN].Get()) { + ui64 availMin = userInfo.ReadQuota.GetAvailableAvgMin(ctx.Now()); + if (availMin != userInfo.LabeledCounters.GetCounters()[METRIC_MIN_READ_QUOTA_BYTES_AVAIL_MIN].Get()) { haveChanges = true; - userInfo.LabeledCounters.GetCounters()[METRIC_MIN_READ_QUOTA_BYTES_AVAIL_MIN].Set(availMin); + userInfo.LabeledCounters.GetCounters()[METRIC_MIN_READ_QUOTA_BYTES_AVAIL_MIN].Set(availMin); } - ui64 readOffsetRewindSum = userInfo.ReadOffsetRewindSum; - if (readOffsetRewindSum != userInfo.LabeledCounters.GetCounters()[METRIC_READ_OFFSET_REWIND_SUM].Get()) { + ui64 readOffsetRewindSum = userInfo.ReadOffsetRewindSum; + if (readOffsetRewindSum != userInfo.LabeledCounters.GetCounters()[METRIC_READ_OFFSET_REWIND_SUM].Get()) { haveChanges = true; - userInfo.LabeledCounters.GetCounters()[METRIC_READ_OFFSET_REWIND_SUM].Set(readOffsetRewindSum); + userInfo.LabeledCounters.GetCounters()[METRIC_READ_OFFSET_REWIND_SUM].Set(readOffsetRewindSum); } if (readOffsetRewindSum != userInfo.LabeledCounters.GetCounters()[METRIC_READ_OFFSET_REWIND_TOTAL].Get()) { haveChanges = true; @@ -3181,25 +3181,25 @@ void TPartition::ReportLabeledCounters(const TActorContext& ctx) } ui32 id = METRIC_TOTAL_READ_SPEED_1; - for (ui32 i = 0; i < userInfo.AvgReadBytes.size(); ++i) { - ui64 avg = userInfo.AvgReadBytes[i].GetValue(); - if (avg != userInfo.LabeledCounters.GetCounters()[id].Get()) { + for (ui32 i = 0; i < userInfo.AvgReadBytes.size(); ++i) { + ui64 avg = userInfo.AvgReadBytes[i].GetValue(); + if (avg != userInfo.LabeledCounters.GetCounters()[id].Get()) { haveChanges = true; - userInfo.LabeledCounters.GetCounters()[id].Set(avg); //total - userInfo.LabeledCounters.GetCounters()[id + 1].Set(avg); //max + userInfo.LabeledCounters.GetCounters()[id].Set(avg); //total + userInfo.LabeledCounters.GetCounters()[id + 1].Set(avg); //max } id += 2; } Y_VERIFY(id == METRIC_MAX_READ_SPEED_4 + 1); - if (userInfo.ReadQuota.GetTotalSpeed()) { - ui64 quotaUsage = ui64(userInfo.AvgReadBytes[1].GetValue()) * 1000000 / userInfo.ReadQuota.GetTotalSpeed() / 60; - if (quotaUsage != userInfo.LabeledCounters.GetCounters()[METRIC_READ_QUOTA_USAGE].Get()) { + if (userInfo.ReadQuota.GetTotalSpeed()) { + ui64 quotaUsage = ui64(userInfo.AvgReadBytes[1].GetValue()) * 1000000 / userInfo.ReadQuota.GetTotalSpeed() / 60; + if (quotaUsage != userInfo.LabeledCounters.GetCounters()[METRIC_READ_QUOTA_USAGE].Get()) { haveChanges = true; - userInfo.LabeledCounters.GetCounters()[METRIC_READ_QUOTA_USAGE].Set(quotaUsage); + userInfo.LabeledCounters.GetCounters()[METRIC_READ_QUOTA_USAGE].Set(quotaUsage); } } if (haveChanges) { - ctx.Send(Tablet, new TEvPQ::TEvPartitionLabeledCounters(Partition, userInfo.LabeledCounters)); + ctx.Send(Tablet, new TEvPQ::TEvPartitionLabeledCounters(Partition, userInfo.LabeledCounters)); } } //Partition counters @@ -3210,10 +3210,10 @@ void TPartition::ReportLabeledCounters(const TActorContext& ctx) PartitionLabeledCounters.GetCounters()[METRIC_NUM_SIDS].Set(SourceIdStorage.GetInMemorySourceIds().size()); } - TDuration lifetimeNow = ctx.Now() - SourceIdStorage.MinAvailableTimestamp(ctx.Now()); - if (lifetimeNow.MilliSeconds() != PartitionLabeledCounters.GetCounters()[METRIC_MIN_SID_LIFETIME].Get()) { + TDuration lifetimeNow = ctx.Now() - SourceIdStorage.MinAvailableTimestamp(ctx.Now()); + if (lifetimeNow.MilliSeconds() != PartitionLabeledCounters.GetCounters()[METRIC_MIN_SID_LIFETIME].Get()) { haveChanges = true; - PartitionLabeledCounters.GetCounters()[METRIC_MIN_SID_LIFETIME].Set(lifetimeNow.MilliSeconds()); + PartitionLabeledCounters.GetCounters()[METRIC_MIN_SID_LIFETIME].Set(lifetimeNow.MilliSeconds()); } ui64 headGapSize = DataKeysBody.empty() ? 0 : (Head.Offset - (DataKeysBody.back().Key.GetOffset() + DataKeysBody.back().Key.GetCount())); @@ -3381,8 +3381,8 @@ void TPartition::HandleSetOffsetResponse(NKikimrClient::TResponse& response, con TString user = it->second; CookieToUser.erase(it); - TUserInfo* userInfo = UsersInfoStorage.GetIfExists(user); - Y_VERIFY(userInfo); + TUserInfo* userInfo = UsersInfoStorage.GetIfExists(user); + Y_VERIFY(userInfo); Y_VERIFY(!userInfo->UserActs.empty()); @@ -3419,7 +3419,7 @@ void TPartition::HandleSetOffsetResponse(NKikimrClient::TResponse& response, con LOG_DEBUG_S(ctx, NKikimrServices::PERSQUEUE, "Topic '" << TopicName << "' partition " << Partition << " user " << user << " reinit with generation " << readRuleGeneration << " done"); userInfo->ReadRuleGeneration = readRuleGeneration; - userInfo->Session = ""; + userInfo->Session = ""; userInfo->Generation = userInfo->Step = 0; userInfo->Offset = 0; } else { @@ -3463,7 +3463,7 @@ void TPartition::HandleSetOffsetResponse(NKikimrClient::TResponse& response, con void TPartition::ScheduleUpdateAvailableSize(const TActorContext& ctx) { - ctx.Schedule(UPDATE_AVAIL_SIZE_INTERVAL, new TEvPQ::TEvUpdateAvailableSize()); + ctx.Schedule(UPDATE_AVAIL_SIZE_INTERVAL, new TEvPQ::TEvUpdateAvailableSize()); } @@ -3556,25 +3556,25 @@ void TPartition::HandleOnWrite(TEvPQ::TEvWrite::TPtr& ev, const TActorContext& c }); bool mirroredPartition = Config.GetPartitionConfig().HasMirrorFrom(); - if (mirroredPartition && !ev->Get()->OwnerCookie.empty()) { + if (mirroredPartition && !ev->Get()->OwnerCookie.empty()) { ReplyError(ctx, ev->Get()->Cookie, NPersQueue::NErrorCode::BAD_REQUEST, - TStringBuilder() << "Write to mirrored topic is forbiden "); + TStringBuilder() << "Write to mirrored topic is forbiden "); return; } - ui64 decReservedSize = 0; + ui64 decReservedSize = 0; TStringBuf owner; if (!mirroredPartition && !ev->Get()->IsDirectWrite) { owner = TOwnerInfo::GetOwnerFromOwnerCookie(ev->Get()->OwnerCookie); - auto it = Owners.find(owner); - - if (it == Owners.end() || it->second.NeedResetOwner) { - ReplyError(ctx, ev->Get()->Cookie, NPersQueue::NErrorCode::WRONG_COOKIE, - TStringBuilder() << "new GetOwnership request needed for owner " << owner); - return; - } - + auto it = Owners.find(owner); + + if (it == Owners.end() || it->second.NeedResetOwner) { + ReplyError(ctx, ev->Get()->Cookie, NPersQueue::NErrorCode::WRONG_COOKIE, + TStringBuilder() << "new GetOwnership request needed for owner " << owner); + return; + } + if (it->second.SourceIdDeleted) { ReplyError(ctx, ev->Get()->Cookie, NPersQueue::NErrorCode::SOURCEID_DELETED, TStringBuilder() << "Yours maximum written sequence number for session was deleted, need to recreate session. " @@ -3584,21 +3584,21 @@ void TPartition::HandleOnWrite(TEvPQ::TEvWrite::TPtr& ev, const TActorContext& c return; } - if (it->second.OwnerCookie != ev->Get()->OwnerCookie) { - ReplyError(ctx, ev->Get()->Cookie, NPersQueue::NErrorCode::WRONG_COOKIE, - TStringBuilder() << "incorrect ownerCookie " << ev->Get()->OwnerCookie << ", must be " << it->second.OwnerCookie); - return; - } - - if (ev->Get()->MessageNo != it->second.NextMessageNo) { - ReplyError(ctx, ev->Get()->Cookie, NPersQueue::NErrorCode::BAD_REQUEST, - TStringBuilder() << "reorder in requests, waiting " << it->second.NextMessageNo << ", but got " << ev->Get()->MessageNo); - DropOwner(it, ctx); - return; - } - - ++it->second.NextMessageNo; - decReservedSize = it->second.DecReservedSize(); + if (it->second.OwnerCookie != ev->Get()->OwnerCookie) { + ReplyError(ctx, ev->Get()->Cookie, NPersQueue::NErrorCode::WRONG_COOKIE, + TStringBuilder() << "incorrect ownerCookie " << ev->Get()->OwnerCookie << ", must be " << it->second.OwnerCookie); + return; + } + + if (ev->Get()->MessageNo != it->second.NextMessageNo) { + ReplyError(ctx, ev->Get()->Cookie, NPersQueue::NErrorCode::BAD_REQUEST, + TStringBuilder() << "reorder in requests, waiting " << it->second.NextMessageNo << ", but got " << ev->Get()->MessageNo); + DropOwner(it, ctx); + return; + } + + ++it->second.NextMessageNo; + decReservedSize = it->second.DecReservedSize(); } TMaybe<ui64> offset = ev->Get()->Offset; @@ -3661,9 +3661,9 @@ void TPartition::HandleOnWrite(TEvPQ::TEvWrite::TPtr& ev, const TActorContext& c ++*offset; } WriteInflightSize += size; - - ReservedSize -= decReservedSize; - Y_VERIFY(size <= decReservedSize || decReservedSize == 0); //TODO: remove decReservedSize == 0 + + ReservedSize -= decReservedSize; + Y_VERIFY(size <= decReservedSize || decReservedSize == 0); //TODO: remove decReservedSize == 0 Counters.Simple()[COUNTER_PQ_TABLET_RESERVED_BYTES_SIZE].Set(ReservedSize); UpdateWriteBufferIsFullState(ctx.Now()); } @@ -3808,13 +3808,13 @@ void TPartition::BecomeIdle(const TActorContext&) } -void TPartition::WriteClientInfo(const ui64 cookie, TUserInfo& userInfo, const TActorContext& ctx) { +void TPartition::WriteClientInfo(const ui64 cookie, TUserInfo& userInfo, const TActorContext& ctx) { THolder<TEvKeyValue::TEvRequest> request(new TEvKeyValue::TEvRequest); Y_VERIFY(!userInfo.WriteInProgress); - Y_VERIFY(!userInfo.UserActs.empty()); - while (!userInfo.UserActs.empty()) { - const auto ev = userInfo.UserActs.front().Get(); + Y_VERIFY(!userInfo.UserActs.empty()); + while (!userInfo.UserActs.empty()) { + const auto ev = userInfo.UserActs.front().Get(); TKeyPrefix ikey(TKeyPrefix::TypeInfo, Partition, TKeyPrefix::MarkUser); ikey.Append(ev->ClientId.c_str(), ev->ClientId.size()); @@ -3839,9 +3839,9 @@ void TPartition::WriteClientInfo(const ui64 cookie, TUserInfo& userInfo, const T } if (ev->Type == TEvPQ::TEvSetClientInfo::ESCI_CREATE_SESSION && ev->SessionId == userInfo.Session) { //this is retry of current request, answer ok - auto ts = GetTime(userInfo, userInfo.Offset); - ReplyGetClientOffsetOk(ctx, ev->Cookie, userInfo.Offset, ts.first, ts.second); - userInfo.UserActs.pop_front(); + auto ts = GetTime(userInfo, userInfo.Offset); + ReplyGetClientOffsetOk(ctx, ev->Cookie, userInfo.Offset, ts.first, ts.second); + userInfo.UserActs.pop_front(); continue; } @@ -3849,17 +3849,17 @@ void TPartition::WriteClientInfo(const ui64 cookie, TUserInfo& userInfo, const T && !ev->SessionId.empty() && userInfo.Session != ev->SessionId //request to wrong session && (ev->Type != TEvPQ::TEvSetClientInfo::ESCI_DROP_SESSION || !userInfo.Session.empty()) //but allow DropSession request when session is already dropped - for idempotence || (ev->Type == TEvPQ::TEvSetClientInfo::ESCI_CREATE_SESSION && !userInfo.Session.empty() - && (ev->Generation < userInfo.Generation || ev->Generation == userInfo.Generation && ev->Step <= userInfo.Step))) { //old generation request + && (ev->Generation < userInfo.Generation || ev->Generation == userInfo.Generation && ev->Step <= userInfo.Step))) { //old generation request Counters.Cumulative()[COUNTER_PQ_SET_CLIENT_OFFSET_ERROR].Increment(1); ReplyError(ctx, ev->Cookie, NPersQueue::NErrorCode::WRONG_COOKIE, - TStringBuilder() << "set offset in already dead session " << ev->SessionId << " actual is " << userInfo.Session); - userInfo.UserActs.pop_front(); + TStringBuilder() << "set offset in already dead session " << ev->SessionId << " actual is " << userInfo.Session); + userInfo.UserActs.pop_front(); continue; } - if (!ev->SessionId.empty() && ev->Type == TEvPQ::TEvSetClientInfo::ESCI_OFFSET && (i64)ev->Offset <= userInfo.Offset) { //this is stale request, answer ok for it + if (!ev->SessionId.empty() && ev->Type == TEvPQ::TEvSetClientInfo::ESCI_OFFSET && (i64)ev->Offset <= userInfo.Offset) { //this is stale request, answer ok for it ReplyOk(ctx, ev->Cookie); - userInfo.UserActs.pop_front(); + userInfo.UserActs.pop_front(); continue; } @@ -3868,7 +3868,7 @@ void TPartition::WriteClientInfo(const ui64 cookie, TUserInfo& userInfo, const T TString session = (ev->Type == TEvPQ::TEvSetClientInfo::ESCI_CREATE_SESSION ? ev->SessionId : (ev->Type == TEvPQ::TEvSetClientInfo::ESCI_DROP_SESSION ? "" : userInfo.Session)); ui32 gen = (ev->Type == TEvPQ::TEvSetClientInfo::ESCI_CREATE_SESSION ? ev->Generation : (ev->Type == TEvPQ::TEvSetClientInfo::ESCI_DROP_SESSION ? 0 : userInfo.Generation)); ui32 step = (ev->Type == TEvPQ::TEvSetClientInfo::ESCI_CREATE_SESSION ? ev->Step : (ev->Type == TEvPQ::TEvSetClientInfo::ESCI_DROP_SESSION ? 0 : userInfo.Step)); - ui64 offset = (ev->Type == TEvPQ::TEvSetClientInfo::ESCI_OFFSET ? ev->Offset : userInfo.Offset); + ui64 offset = (ev->Type == TEvPQ::TEvSetClientInfo::ESCI_OFFSET ? ev->Offset : userInfo.Offset); ui64 readRuleGeneration = userInfo.ReadRuleGeneration; if (ev->Type == TEvPQ::TEvSetClientInfo::ESCI_INIT_READ_RULE) { readRuleGeneration = ev->ReadRuleGeneration; @@ -3889,24 +3889,24 @@ void TPartition::WriteClientInfo(const ui64 cookie, TUserInfo& userInfo, const T /* Counters.Cumulative()[COUNTER_PQ_SET_CLIENT_OFFSET_ERROR].Increment(1); ReplyError(ctx, ev->Cookie, NPersQueue::NErrorCode::SET_OFFSET_ERROR_COMMIT_TO_FUTURE, TStringBuilder() << "can't commit to future. Offset " << offset << " EndOffset " << EndOffset); - userInfo.UserActrs.pop_front(); + userInfo.UserActrs.pop_front(); continue;*/ } TBuffer idata; - { - NKikimrPQ::TUserInfo userData; - userData.SetOffset(offset); - userData.SetGeneration(gen); - userData.SetStep(step); - userData.SetSession(session); - userData.SetOffsetRewindSum(userInfo.ReadOffsetRewindSum); + { + NKikimrPQ::TUserInfo userData; + userData.SetOffset(offset); + userData.SetGeneration(gen); + userData.SetStep(step); + userData.SetSession(session); + userData.SetOffsetRewindSum(userInfo.ReadOffsetRewindSum); userData.SetReadRuleGeneration(readRuleGeneration); - TString out; + TString out; Y_PROTOBUF_SUPPRESS_NODISCARD userData.SerializeToString(&out); - idata.Append(out.c_str(), out.size()); - } + idata.Append(out.c_str(), out.size()); + } TBuffer idataDeprecated = NDeprecatedUserData::Serialize(offset, gen, step, session); auto write = request->Record.AddCmdWrite(); @@ -3968,7 +3968,7 @@ bool TPartition::AppendHeadWithNewWrites(TEvKeyValue::TEvRequest* request, const WriteNewSizeUncompressed = 0; WriteNewMessages = 0; UpdateWriteBufferIsFullState(ctx.Now()); - CurrentTimestamp = ctx.Now(); + CurrentTimestamp = ctx.Now(); NewHead.Offset = EndOffset; NewHead.PartNo = 0; @@ -4143,14 +4143,14 @@ bool TPartition::AppendHeadWithNewWrites(TEvKeyValue::TEvRequest* request, const } WriteTimestamp = ctx.Now(); WriteTimestampEstimate = p.Msg.WriteTimestamp > 0 ? TInstant::MilliSeconds(p.Msg.WriteTimestamp) : WriteTimestamp; - TClientBlob blob(p.Msg.SourceId, p.Msg.SeqNo, p.Msg.Data, std::move(partData), WriteTimestampEstimate, + TClientBlob blob(p.Msg.SourceId, p.Msg.SeqNo, p.Msg.Data, std::move(partData), WriteTimestampEstimate, TInstant::MilliSeconds(p.Msg.CreateTimestamp == 0 ? curOffset : p.Msg.CreateTimestamp), p.Msg.UncompressedSize, p.Msg.PartitionKey, p.Msg.ExplicitHashKey); //remove curOffset when LB will report CTime - ui64 writeLagMs = (WriteTimestamp - TInstant::MilliSeconds(p.Msg.CreateTimestamp)).MilliSeconds(); - WriteLagMs.Update(writeLagMs, WriteTimestamp); + ui64 writeLagMs = (WriteTimestamp - TInstant::MilliSeconds(p.Msg.CreateTimestamp)).MilliSeconds(); + WriteLagMs.Update(writeLagMs, WriteTimestamp); if (InputTimeLag) { - InputTimeLag->IncFor(writeLagMs, 1); + InputTimeLag->IncFor(writeLagMs, 1); if (p.Msg.PartNo == 0) { MessageSize->IncFor(p.Msg.TotalSize + p.Msg.SourceId.size(), 1); } @@ -4403,7 +4403,7 @@ void TPartition::AddNewWriteBlob(std::pair<TKey, ui32>& res, TEvKeyValue::TEvReq NewHead.PartNo = 0; } else { Y_VERIFY(NewHeadKey.Size == 0); - NewHeadKey = {key, res.second, CurrentTimestamp, 0}; + NewHeadKey = {key, res.second, CurrentTimestamp, 0}; } WriteCycleSize += write->GetValue().size(); UpdateWriteBufferIsFullState(ctx.Now()); @@ -4567,15 +4567,15 @@ void TPartition::ProcessRead(const TActorContext& ctx, TReadInfo&& info, const u ui32 size = 0; Y_VERIFY(!info.User.empty()); - auto& userInfo = UsersInfoStorage.GetOrCreate(info.User, ctx); + auto& userInfo = UsersInfoStorage.GetOrCreate(info.User, ctx); if (subscription) { - userInfo.ForgetSubscription(ctx.Now()); + userInfo.ForgetSubscription(ctx.Now()); } - if (!userInfo.ReadQuota.CanExaust()) { - userInfo.ReadRequests.push_back({std::move(info), cookie}); - userInfo.UpdateReadingTimeAndState(ctx.Now()); + if (!userInfo.ReadQuota.CanExaust()) { + userInfo.ReadRequests.push_back({std::move(info), cookie}); + userInfo.UpdateReadingTimeAndState(ctx.Now()); return; } TVector<TRequestedBlob> blobs = GetReadRequestFromBody(info.Offset, info.PartNo, info.Count, info.Size, &count, &size); @@ -4589,27 +4589,27 @@ void TPartition::ProcessRead(const TActorContext& ctx, TReadInfo&& info, const u info.CachedOffset = Head.Offset > 0 ? Head.Offset : insideHeadOffset; if (info.Destination != 0) { - ++userInfo.ActiveReads; - userInfo.UpdateReadingTimeAndState(ctx.Now()); + ++userInfo.ActiveReads; + userInfo.UpdateReadingTimeAndState(ctx.Now()); } if (info.Blobs.empty()) { //all from head, answer right now LOG_DEBUG_S(ctx, NKikimrServices::PERSQUEUE, "Reading cookie " << cookie << ". All data is from uncompacted head."); - TReadAnswer answer(info.FormAnswer( - ctx, EndOffset, Partition, &UsersInfoStorage.GetOrCreate(info.User, ctx), - info.Destination, GetSizeLag(info.Offset) - )); - const auto& resp = dynamic_cast<TEvPQ::TEvProxyResponse*>(answer.Event.Get())->Response; + TReadAnswer answer(info.FormAnswer( + ctx, EndOffset, Partition, &UsersInfoStorage.GetOrCreate(info.User, ctx), + info.Destination, GetSizeLag(info.Offset) + )); + const auto& resp = dynamic_cast<TEvPQ::TEvProxyResponse*>(answer.Event.Get())->Response; if (info.IsSubscription) { Counters.Cumulative()[COUNTER_PQ_READ_SUBSCRIPTION_OK].Increment(1); } Counters.Cumulative()[COUNTER_PQ_READ_HEAD_ONLY_OK].Increment(1); Counters.Percentile()[COUNTER_LATENCY_PQ_READ_HEAD_ONLY].IncrementFor((ctx.Now() - info.Timestamp).MilliSeconds()); Counters.Cumulative()[COUNTER_PQ_READ_BYTES].Increment(resp.ByteSize()); - ctx.Send(info.Destination != 0 ? Tablet : ctx.SelfID, answer.Event.Release()); + ctx.Send(info.Destination != 0 ? Tablet : ctx.SelfID, answer.Event.Release()); ReportLabeledCounters(ctx); - OnReadRequestFinished(std::move(info), answer.Size); + OnReadRequestFinished(std::move(info), answer.Size); return; } @@ -4740,13 +4740,13 @@ void TPartition::CalcTopicWriteQuotaParams() } } -void TPartition::CreateMirrorerActor() { - Mirrorer = MakeHolder<TMirrorerInfo>( +void TPartition::CreateMirrorerActor() { + Mirrorer = MakeHolder<TMirrorerInfo>( Register(new TMirrorer(Tablet, SelfId(), TopicName, Partition, LocalDC, EndOffset, Config.GetPartitionConfig().GetMirrorFrom(), Counters)), - Counters - ); -} - + Counters + ); +} + bool TPartition::IsQuotingEnabled() const { const auto& pqConfig = AppData()->PQConfig; const auto& quotingConfig = pqConfig.GetQuotingConfig(); diff --git a/ydb/core/persqueue/partition.h b/ydb/core/persqueue/partition.h index d6fe2ea41b..2eec26f251 100644 --- a/ydb/core/persqueue/partition.h +++ b/ydb/core/persqueue/partition.h @@ -11,7 +11,7 @@ #include <ydb/core/protos/pqconfig.pb.h> #include <ydb/core/persqueue/events/internal.h> #include <ydb/library/persqueue/counter_time_keeper/counter_time_keeper.h> - + #include "key.h" #include "blob.h" #include "subscriber.h" @@ -35,19 +35,19 @@ typedef TProtobufTabletLabeledCounters<EPartitionLabeledCounters_descriptor> TPa struct TDataKey { TKey Key; ui32 Size; - TInstant Timestamp; + TInstant Timestamp; ui64 CumulativeSize; }; -ui64 GetOffsetEstimate(const std::deque<TDataKey>& container, TInstant timestamp, ui64 headOffset); +ui64 GetOffsetEstimate(const std::deque<TDataKey>& container, TInstant timestamp, ui64 headOffset); + +struct TMirrorerInfo; -struct TMirrorerInfo; - class TPartition : public TActorBootstrapped<TPartition> { -private: - static constexpr ui32 MAX_ERRORS_COUNT_TO_STORE = 10; +private: + static constexpr ui32 MAX_ERRORS_COUNT_TO_STORE = 10; -private: +private: struct THasDataReq; struct THasDataDeadline; @@ -60,20 +60,20 @@ private: void ReplyError(const TActorContext& ctx, const ui64 dst, NPersQueue::NErrorCode::EErrorCode errorCode, const TString& error); void ReplyErrorForStoredWrites(const TActorContext& ctx); void ReplyOk(const TActorContext& ctx, const ui64 dst); - void ReplyWrite( - const TActorContext& ctx, ui64 dst, const TString& sourceId, ui64 seqNo, ui16 partNo, ui16 totalParts, - ui64 offset, TInstant writeTimestamp, bool already, ui64 maxSeqNo, - ui64 partitionQuotedTime, TDuration topicQuotedTime, ui64 queueTime, ui64 writeTime); + void ReplyWrite( + const TActorContext& ctx, ui64 dst, const TString& sourceId, ui64 seqNo, ui16 partNo, ui16 totalParts, + ui64 offset, TInstant writeTimestamp, bool already, ui64 maxSeqNo, + ui64 partitionQuotedTime, TDuration topicQuotedTime, ui64 queueTime, ui64 writeTime); void ReplyGetClientOffsetOk(const TActorContext& ctx, const ui64 dst, const i64 offset, - const TInstant writeTimestamp, const TInstant createTimestamp); + const TInstant writeTimestamp, const TInstant createTimestamp); void ReplyOwnerOk(const TActorContext& ctx, const ui64 dst, const TString& ownerCookie); void Handle(TEvPersQueue::TEvHasDataInfo::TPtr& ev, const TActorContext& ctx); - void Handle(TEvPQ::TEvMirrorerCounters::TPtr& ev, const TActorContext& ctx); - void Handle(NReadSpeedLimiterEvents::TEvCounters::TPtr& ev, const TActorContext& ctx); - + void Handle(TEvPQ::TEvMirrorerCounters::TPtr& ev, const TActorContext& ctx); + void Handle(NReadSpeedLimiterEvents::TEvCounters::TPtr& ev, const TActorContext& ctx); + //answer for reads for Timestamps void Handle(TEvPQ::TEvProxyResponse::TPtr& ev, const TActorContext& ctx); void Handle(TEvPQ::TEvError::TPtr& ev, const TActorContext& ctx); @@ -117,10 +117,10 @@ private: void Handle(TEvPQ::TEvPartitionStatus::TPtr& ev, const TActorContext& ctx); void Handle(TEvPQ::TEvGetPartitionClientInfo::TPtr& ev, const TActorContext& ctx); - void Handle(TEvPersQueue::TEvReportPartitionError::TPtr& ev, const TActorContext& ctx); - void LogAndCollectError(const NKikimrPQ::TStatusResponse::TErrorMessage& error, const TActorContext& ctx); - void LogAndCollectError(NKikimrServices::EServiceKikimr service, const TString& msg, const TActorContext& ctx); - + void Handle(TEvPersQueue::TEvReportPartitionError::TPtr& ev, const TActorContext& ctx); + void LogAndCollectError(const NKikimrPQ::TStatusResponse::TErrorMessage& error, const TActorContext& ctx); + void LogAndCollectError(NKikimrServices::EServiceKikimr service, const TString& msg, const TActorContext& ctx); + void HandleOnIdle(TEvPQ::TEvUpdateAvailableSize::TPtr& ev, const TActorContext& ctx); void HandleOnWrite(TEvPQ::TEvUpdateAvailableSize::TPtr& ev, const TActorContext& ctx); @@ -136,9 +136,9 @@ private: void Handle(TEvents::TEvPoisonPill::TPtr& ev, const TActorContext& ctx); void Handle(TEvPQ::TEvRead::TPtr& ev, const TActorContext& ctx); - void Handle(NReadSpeedLimiterEvents::TEvResponse::TPtr& ev, const TActorContext& ctx); - void DoRead(TEvPQ::TEvRead::TPtr ev, TDuration waitQuotaTime, const TActorContext& ctx); - void OnReadRequestFinished(TReadInfo&& info, ui64 answerSize); + void Handle(NReadSpeedLimiterEvents::TEvResponse::TPtr& ev, const TActorContext& ctx); + void DoRead(TEvPQ::TEvRead::TPtr ev, TDuration waitQuotaTime, const TActorContext& ctx); + void OnReadRequestFinished(TReadInfo&& info, ui64 answerSize); // will return rcount and rsize also TVector<TRequestedBlob> GetReadRequestFromBody(const ui64 startOffset, const ui16 partNo, const ui32 maxCount, const ui32 maxSize, ui32* rcount, ui32* rsize); @@ -207,7 +207,7 @@ private: void Handle(TEvPQ::TEvReserveBytes::TPtr& ev, const TActorContext& ctx); void ProcessReserveRequests(const TActorContext& ctx); - void CreateMirrorerActor(); + void CreateMirrorerActor(); bool IsQuotingEnabled() const; void SetupTopicCounters(const TActorContext& ctx); @@ -252,7 +252,7 @@ private: STFUNC(StateInit) { - NPersQueue::TCounterTimeKeeper keeper(Counters.Cumulative()[COUNTER_PQ_TABLET_CPU_USAGE]); + NPersQueue::TCounterTimeKeeper keeper(Counters.Cumulative()[COUNTER_PQ_TABLET_CPU_USAGE]); LOG_TRACE_S(ctx, NKikimrServices::PERSQUEUE, EventStr("StateInit", ev)); @@ -265,10 +265,10 @@ private: HFuncTraced(TEvPQ::TEvChangeConfig, Handle); HFuncTraced(TEvPQ::TEvPartitionOffsets, HandleOnInit); HFuncTraced(TEvPQ::TEvPartitionStatus, HandleOnInit); - HFuncTraced(TEvPersQueue::TEvReportPartitionError, Handle); + HFuncTraced(TEvPersQueue::TEvReportPartitionError, Handle); HFuncTraced(TEvPersQueue::TEvHasDataInfo, Handle); - HFuncTraced(TEvPQ::TEvMirrorerCounters, Handle); - HFuncTraced(NReadSpeedLimiterEvents::TEvCounters, Handle); + HFuncTraced(TEvPQ::TEvMirrorerCounters, Handle); + HFuncTraced(NReadSpeedLimiterEvents::TEvCounters, Handle); HFuncTraced(TEvPQ::TEvGetPartitionClientInfo, Handle); default: LOG_ERROR_S(ctx, NKikimrServices::PERSQUEUE, "Unexpected " << EventStr("StateInit", ev)); @@ -278,7 +278,7 @@ private: STFUNC(StateIdle) { - NPersQueue::TCounterTimeKeeper keeper(Counters.Cumulative()[COUNTER_PQ_TABLET_CPU_USAGE]); + NPersQueue::TCounterTimeKeeper keeper(Counters.Cumulative()[COUNTER_PQ_TABLET_CPU_USAGE]); LOG_TRACE_S(ctx, NKikimrServices::PERSQUEUE, EventStr("StateIdle", ev)); @@ -289,7 +289,7 @@ private: HFuncTraced(TEvPQ::TEvBlobResponse, Handle); HFuncTraced(TEvPQ::TEvWrite, HandleOnIdle); HFuncTraced(TEvPQ::TEvRead, Handle); - HFuncTraced(NReadSpeedLimiterEvents::TEvResponse, Handle); + HFuncTraced(NReadSpeedLimiterEvents::TEvResponse, Handle); HFuncTraced(TEvPQ::TEvReadTimeout, Handle); HFuncTraced(TEvents::TEvPoisonPill, Handle); HFuncTraced(TEvPQ::TEvMonRequest, HandleMonitoring); @@ -300,11 +300,11 @@ private: HFuncTraced(TEvPQ::TEvSetClientInfo, Handle); HFuncTraced(TEvPQ::TEvPartitionOffsets, Handle); HFuncTraced(TEvPQ::TEvPartitionStatus, Handle); - HFuncTraced(TEvPersQueue::TEvReportPartitionError, Handle); + HFuncTraced(TEvPersQueue::TEvReportPartitionError, Handle); HFuncTraced(TEvPQ::TEvChangeOwner, Handle); HFuncTraced(TEvPersQueue::TEvHasDataInfo, Handle); - HFuncTraced(TEvPQ::TEvMirrorerCounters, Handle); - HFuncTraced(NReadSpeedLimiterEvents::TEvCounters, Handle); + HFuncTraced(TEvPQ::TEvMirrorerCounters, Handle); + HFuncTraced(NReadSpeedLimiterEvents::TEvCounters, Handle); HFuncTraced(TEvPQ::TEvProxyResponse, Handle); HFuncTraced(TEvPQ::TEvError, Handle); HFuncTraced(TEvPQ::TEvGetPartitionClientInfo, Handle); @@ -325,7 +325,7 @@ private: STFUNC(StateWrite) { - NPersQueue::TCounterTimeKeeper keeper(Counters.Cumulative()[COUNTER_PQ_TABLET_CPU_USAGE]); + NPersQueue::TCounterTimeKeeper keeper(Counters.Cumulative()[COUNTER_PQ_TABLET_CPU_USAGE]); LOG_TRACE_S(ctx, NKikimrServices::PERSQUEUE, EventStr("StateWrite", ev)); @@ -337,7 +337,7 @@ private: HFuncTraced(TEvPQ::TEvBlobResponse, Handle); HFuncTraced(TEvPQ::TEvWrite, HandleOnWrite); HFuncTraced(TEvPQ::TEvRead, Handle); - HFuncTraced(NReadSpeedLimiterEvents::TEvResponse, Handle); + HFuncTraced(NReadSpeedLimiterEvents::TEvResponse, Handle); HFuncTraced(TEvPQ::TEvReadTimeout, Handle); HFuncTraced(TEvents::TEvPoisonPill, Handle); HFuncTraced(TEvPQ::TEvMonRequest, HandleMonitoring); @@ -347,12 +347,12 @@ private: HFuncTraced(TEvPQ::TEvSetClientInfo, Handle); HFuncTraced(TEvPQ::TEvPartitionOffsets, Handle); HFuncTraced(TEvPQ::TEvPartitionStatus, Handle); - HFuncTraced(TEvPersQueue::TEvReportPartitionError, Handle); + HFuncTraced(TEvPersQueue::TEvReportPartitionError, Handle); HFuncTraced(TEvPQ::TEvChangeOwner, Handle); HFuncTraced(TEvPQ::TEvChangeConfig, Handle); HFuncTraced(TEvPersQueue::TEvHasDataInfo, Handle); - HFuncTraced(TEvPQ::TEvMirrorerCounters, Handle); - HFuncTraced(NReadSpeedLimiterEvents::TEvCounters, Handle); + HFuncTraced(TEvPQ::TEvMirrorerCounters, Handle); + HFuncTraced(NReadSpeedLimiterEvents::TEvCounters, Handle); HFuncTraced(TEvPQ::TEvProxyResponse, Handle); HFuncTraced(TEvPQ::TEvError, Handle); HFuncTraced(TEvPQ::TEvReserveBytes, Handle); @@ -379,8 +379,8 @@ private: void CheckHeadConsistency() const; - std::pair<TInstant, TInstant> GetTime(const TUserInfo& userInfo, ui64 offset) const; - TInstant GetWriteTimeEstimate(ui64 offset) const; + std::pair<TInstant, TInstant> GetTime(const TUserInfo& userInfo, ui64 offset) const; + TInstant GetWriteTimeEstimate(ui64 offset) const; ui32 NextChannel(bool isHead, ui32 blobSize); @@ -568,7 +568,7 @@ private: TString DbId; TString FolderId; - TUsersInfoStorage UsersInfoStorage; + TUsersInfoStorage UsersInfoStorage; std::deque<TString> UpdateUserInfoTimestamp; bool ReadingTimestamp; @@ -606,7 +606,7 @@ private: ui32 WriteNewMessages; ui32 WriteNewMessagesInternal; - TInstant CurrentTimestamp; + TInstant CurrentTimestamp; bool DiskIsFull; @@ -632,7 +632,7 @@ private: TInstant WriteTimestamp; TInstant WriteTimestampEstimate; - bool ManageWriteTimestampEstimate = true; + bool ManageWriteTimestampEstimate = true; NSlidingWindow::TSlidingWindow<NSlidingWindow::TMaxOperation<ui64>> WriteLagMs; THolder<TPercentileCounter> InputTimeLag; THolder<TPercentileCounter> MessageSize; @@ -656,10 +656,10 @@ private: TString TopicWriteQuoterPath; TString TopicWriteQuotaResourcePath; ui64 NextTopicWriteQuotaRequestCookie = 1; - - TDeque<NKikimrPQ::TStatusResponse::TErrorMessage> Errors; - - THolder<TMirrorerInfo> Mirrorer; + + TDeque<NKikimrPQ::TStatusResponse::TErrorMessage> Errors; + + THolder<TMirrorerInfo> Mirrorer; }; }// NPQ diff --git a/ydb/core/persqueue/pq.h b/ydb/core/persqueue/pq.h index 8f4bdadd5e..e5f3309ba9 100644 --- a/ydb/core/persqueue/pq.h +++ b/ydb/core/persqueue/pq.h @@ -2,7 +2,7 @@ #include <ydb/core/base/blobstorage.h> -#include <library/cpp/actors/core/actorid.h> +#include <library/cpp/actors/core/actorid.h> namespace NKikimr { diff --git a/ydb/core/persqueue/pq_impl.cpp b/ydb/core/persqueue/pq_impl.cpp index 7fffb27cb7..9937dbe561 100644 --- a/ydb/core/persqueue/pq_impl.cpp +++ b/ydb/core/persqueue/pq_impl.cpp @@ -1,6 +1,6 @@ - + #include "pq_impl.h" -#include "event_helpers.h" +#include "event_helpers.h" #include "partition.h" #include "read.h" #include <ydb/core/persqueue/config/config.h> @@ -25,7 +25,7 @@ namespace NPQ { const TString TMP_REQUEST_MARKER = "__TMP__REQUEST__MARKER__"; const ui32 CACHE_SIZE = 100 << 20; //100mb per tablet by default const ui32 MAX_BYTES = 25 * 1024 * 1024; -const TDuration TOTAL_TIMEOUT = TDuration::Seconds(120); +const TDuration TOTAL_TIMEOUT = TDuration::Seconds(120); static constexpr ui32 MAX_SOURCE_ID_LENGTH = 10240; struct TPartitionInfo { @@ -34,17 +34,17 @@ struct TPartitionInfo { : Actor(actor) , KeyRange(std::move(keyRange)) , InitDone(initDone) - { - Baseline.Populate(baseline); - } + { + Baseline.Populate(baseline); + } TPartitionInfo(const TPartitionInfo& info) : Actor(info.Actor) , KeyRange(info.KeyRange) , InitDone(info.InitDone) - { - Baseline.Populate(info.Baseline); - } + { + Baseline.Populate(info.Baseline); + } TActorId Actor; TMaybe<TPartitionKeyRange> KeyRange; @@ -143,9 +143,9 @@ private: auto partResp = Response->Record.MutablePartitionResponse()->MutableCmdReadResult(); - partResp->SetMaxOffset(res.GetMaxOffset()); - partResp->SetSizeLag(res.GetSizeLag()); - partResp->SetWaitQuotaTimeMs(partResp->GetWaitQuotaTimeMs() + res.GetWaitQuotaTimeMs()); + partResp->SetMaxOffset(res.GetMaxOffset()); + partResp->SetSizeLag(res.GetSizeLag()); + partResp->SetWaitQuotaTimeMs(partResp->GetWaitQuotaTimeMs() + res.GetWaitQuotaTimeMs()); for (ui32 i = 0; i < res.ResultSize(); ++i) { if (!res.GetResult(i).HasPartNo() || res.GetResult(i).GetPartNo() == 0) { @@ -343,7 +343,7 @@ public: AnswerAndDie(ctx); return; } - ctx.Schedule(TOTAL_TIMEOUT, new TEvents::TEvWakeup()); + ctx.Schedule(TOTAL_TIMEOUT, new TEvents::TEvWakeup()); } private: @@ -538,10 +538,10 @@ private: void TPersQueue::ReplyError(const TActorContext& ctx, const ui64 responseCookie, NPersQueue::NErrorCode::EErrorCode errorCode, const TString& error) { - ReplyPersQueueError( - ctx.SelfID, ctx, TabletID(), TopicName, Nothing(), *Counters, NKikimrServices::PERSQUEUE, - responseCookie, errorCode, error - ); + ReplyPersQueueError( + ctx.SelfID, ctx, TabletID(), TopicName, Nothing(), *Counters, NKikimrServices::PERSQUEUE, + responseCookie, errorCode, error + ); } void TPersQueue::FillMeteringParams(const TActorContext& ctx) @@ -621,8 +621,8 @@ void TPersQueue::ApplyNewConfigAndReply(const TActorContext& ctx) Partitions.emplace(partitionId, TPartitionInfo( ctx.Register(new TPartition(TabletID(), partitionId, ctx.SelfID, CacheActor, TopicName, TopicPath, LocalDC, DCId, Config, *Counters, ctx, true)), GetPartitionKeyRange(partition), - true, - *Counters + true, + *Counters )); // InitCompleted is true because this partition is empty @@ -729,8 +729,8 @@ void TPersQueue::ReadConfig(const NKikimrClient::TKeyValueResponse::TReadResult& Partitions.emplace(partitionId, TPartitionInfo( ctx.Register(new TPartition(TabletID(), partitionId, ctx.SelfID, CacheActor, TopicName, TopicPath, LocalDC, DCId, Config, *Counters, ctx, false)), GetPartitionKeyRange(partition), - false, - *Counters + false, + *Counters )); } ConfigInited = true; @@ -849,19 +849,19 @@ void TPersQueue::Handle(TEvPQ::TEvPartitionCounters::TPtr& ev, const TActorConte Y_VERIFY(it != Partitions.end()); auto diff = ev->Get()->Counters.MakeDiffForAggr(it->second.Baseline); ui64 cpuUsage = diff->Cumulative()[COUNTER_PQ_TABLET_CPU_USAGE].Get(); - ui64 networkBytesUsage = diff->Cumulative()[COUNTER_PQ_TABLET_NETWORK_BYTES_USAGE].Get(); - if (ResourceMetrics) { - if (cpuUsage > 0) { + ui64 networkBytesUsage = diff->Cumulative()[COUNTER_PQ_TABLET_NETWORK_BYTES_USAGE].Get(); + if (ResourceMetrics) { + if (cpuUsage > 0) { ResourceMetrics->CPU.Increment(cpuUsage); - } - if (networkBytesUsage > 0) { - ResourceMetrics->Network.Increment(networkBytesUsage); - } - if (cpuUsage > 0 || networkBytesUsage > 0) { + } + if (networkBytesUsage > 0) { + ResourceMetrics->Network.Increment(networkBytesUsage); + } + if (cpuUsage > 0 || networkBytesUsage > 0) { ResourceMetrics->TryUpdate(ctx); - } + } } - + Counters->Populate(*diff.Get()); ev->Get()->Counters.RememberCurrentStateAsBaseline(it->second.Baseline); @@ -1413,8 +1413,8 @@ void TPersQueue::HandleWriteRequest(const ui64 responseCookie, const TActorId& p TVector <TEvPQ::TEvWrite::TMsg> msgs; - bool mirroredPartition = Config.GetPartitionConfig().HasMirrorFrom(); - + bool mirroredPartition = Config.GetPartitionConfig().HasMirrorFrom(); + if (!req.GetIsDirectWrite()) { if (!req.HasMessageNo()) { ReplyError(ctx, responseCookie, NPersQueue::NErrorCode::BAD_REQUEST, "MessageNo must be set for writes"); @@ -1427,7 +1427,7 @@ void TPersQueue::HandleWriteRequest(const ui64 responseCookie, const TActorId& p return; } } - + if (req.HasCmdWriteOffset() && req.GetCmdWriteOffset() < 0) { ReplyError(ctx, responseCookie, NPersQueue::NErrorCode::BAD_REQUEST, "CmdWriteOffset can't be negative"); return; @@ -1479,14 +1479,14 @@ void TPersQueue::HandleWriteRequest(const ui64 responseCookie, const TActorId& p errorStr = "TotalSize is incorrect"; } else if (cmd.GetSourceId().size() > MAX_SOURCE_ID_LENGTH) { errorStr = "Too big SourceId"; - } else if (mirroredPartition && !cmd.GetDisableDeduplication()) { - errorStr = "Write to mirrored topic is forbiden"; + } else if (mirroredPartition && !cmd.GetDisableDeduplication()) { + errorStr = "Write to mirrored topic is forbiden"; } - ui64 createTimestampMs = 0, writeTimestampMs = 0; + ui64 createTimestampMs = 0, writeTimestampMs = 0; if (cmd.HasCreateTimeMS() && cmd.GetCreateTimeMS() >= 0) - createTimestampMs = cmd.GetCreateTimeMS(); + createTimestampMs = cmd.GetCreateTimeMS(); if (cmd.HasWriteTimeMS() && cmd.GetWriteTimeMS() > 0) { - writeTimestampMs = cmd.GetWriteTimeMS(); + writeTimestampMs = cmd.GetWriteTimeMS(); if (!cmd.GetDisableDeduplication()) { errorStr = "WriteTimestamp avail only without deduplication"; } @@ -1498,7 +1498,7 @@ void TPersQueue::HandleWriteRequest(const ui64 responseCookie, const TActorId& p } ui32 mSize = MAX_BLOB_PART_SIZE - cmd.GetSourceId().size() - sizeof(ui32) - TClientBlob::OVERHEAD; //megaqc - remove this Y_VERIFY(mSize > 204800); - ui64 receiveTimestampMs = TAppData::TimeProvider->Now().MilliSeconds(); + ui64 receiveTimestampMs = TAppData::TimeProvider->Now().MilliSeconds(); bool disableDeduplication = cmd.GetDisableDeduplication(); if (cmd.GetData().size() > mSize) { if (cmd.HasPartNo()) { @@ -1525,10 +1525,10 @@ void TPersQueue::HandleWriteRequest(const ui64 responseCookie, const TActorId& p pos += mSize - diff; diff = 0; msgs.push_back({cmd.GetSourceId(), static_cast<ui64>(cmd.GetSeqNo()), partNo, - totalParts, totalSize, createTimestampMs, receiveTimestampMs, + totalParts, totalSize, createTimestampMs, receiveTimestampMs, disableDeduplication, writeTimestampMs, data, uncompressedSize, cmd.GetPartitionKey(), cmd.GetExplicitHash(), cmd.GetExternalOperation() - }); + }); partNo++; uncompressedSize = 0; LOG_DEBUG_S(ctx, NKikimrServices::PERSQUEUE, "got client PART message topic: " << TopicName << " partition: " << req.GetPartition() @@ -1539,12 +1539,12 @@ void TPersQueue::HandleWriteRequest(const ui64 responseCookie, const TActorId& p Y_VERIFY(partNo == totalParts); } else { msgs.push_back({cmd.GetSourceId(), static_cast<ui64>(cmd.GetSeqNo()), static_cast<ui16>(cmd.HasPartNo() ? cmd.GetPartNo() : 0), - static_cast<ui16>(cmd.HasPartNo() ? cmd.GetTotalParts() : 1), - static_cast<ui32>(cmd.HasTotalSize() ? cmd.GetTotalSize() : cmd.GetData().Size()), - createTimestampMs, receiveTimestampMs, disableDeduplication, writeTimestampMs, cmd.GetData(), + static_cast<ui16>(cmd.HasPartNo() ? cmd.GetTotalParts() : 1), + static_cast<ui32>(cmd.HasTotalSize() ? cmd.GetTotalSize() : cmd.GetData().Size()), + createTimestampMs, receiveTimestampMs, disableDeduplication, writeTimestampMs, cmd.GetData(), cmd.HasUncompressedSize() ? cmd.GetUncompressedSize() : 0u, cmd.GetPartitionKey(), cmd.GetExplicitHash(), cmd.GetExternalOperation() - }); + }); } LOG_DEBUG_S(ctx, NKikimrServices::PERSQUEUE, "got client message topic: " << TopicName << " partition: " << req.GetPartition() << diff --git a/ydb/core/persqueue/pq_impl.h b/ydb/core/persqueue/pq_impl.h index 60e04cf9bb..bc3bfb10ba 100644 --- a/ydb/core/persqueue/pq_impl.h +++ b/ydb/core/persqueue/pq_impl.h @@ -1,5 +1,5 @@ #pragma once - + #include "percentile_counter.h" #include <ydb/core/keyvalue/keyvalue_flat_impl.h> #include <ydb/core/tablet/tablet_counters.h> diff --git a/ydb/core/persqueue/pq_l2_cache.h b/ydb/core/persqueue/pq_l2_cache.h index c672b190ab..f9fcccbc8e 100644 --- a/ydb/core/persqueue/pq_l2_cache.h +++ b/ydb/core/persqueue/pq_l2_cache.h @@ -1,5 +1,5 @@ #pragma once - + #include "read.h" #include "pq_l2_service.h" @@ -81,8 +81,8 @@ public: : Cache(SizeInBytes(1024*1024)/MAX_BLOB_SIZE) // It's some "much bigger then we need" size here. , MaxSize(SizeInBytes(params.MaxSizeMB)) , CurrentSize(0) - , KeepTime(params.KeepTime) - , RetentionTime(TDuration::Zero()) + , KeepTime(params.KeepTime) + , RetentionTime(TDuration::Zero()) , Counters(countersGroup) {} diff --git a/ydb/core/persqueue/pq_l2_service.h b/ydb/core/persqueue/pq_l2_service.h index c71418c34d..a9c5eefd16 100644 --- a/ydb/core/persqueue/pq_l2_service.h +++ b/ydb/core/persqueue/pq_l2_service.h @@ -16,7 +16,7 @@ inline TActorId MakePersQueueL2CacheID() { struct TCacheL2Parameters { ui32 MaxSizeMB; - TDuration KeepTime; + TDuration KeepTime; }; IActor* CreateNodePersQueueL2Cache(const TCacheL2Parameters&, TIntrusivePtr<NMonitoring::TDynamicCounters>); diff --git a/ydb/core/persqueue/pq_ut.cpp b/ydb/core/persqueue/pq_ut.cpp index 5d7a2d05c5..13a6122910 100644 --- a/ydb/core/persqueue/pq_ut.cpp +++ b/ydb/core/persqueue/pq_ut.cpp @@ -1905,22 +1905,22 @@ Y_UNIT_TEST(TestPQCacheSizeManagement) { Y_UNIT_TEST(TestOffsetEstimation) { std::deque<NPQ::TDataKey> container = { - {NPQ::TKey(NPQ::TKeyPrefix::EType::TypeNone, 0, 1, 0, 0, 0), 0, TInstant::Seconds(1), 10}, - {NPQ::TKey(NPQ::TKeyPrefix::EType::TypeNone, 0, 2, 0, 0, 0), 0, TInstant::Seconds(1), 10}, - {NPQ::TKey(NPQ::TKeyPrefix::EType::TypeNone, 0, 3, 0, 0, 0), 0, TInstant::Seconds(2), 10}, - {NPQ::TKey(NPQ::TKeyPrefix::EType::TypeNone, 0, 4, 0, 0, 0), 0, TInstant::Seconds(2), 10}, - {NPQ::TKey(NPQ::TKeyPrefix::EType::TypeNone, 0, 5, 0, 0, 0), 0, TInstant::Seconds(3), 10}, - {NPQ::TKey(NPQ::TKeyPrefix::EType::TypeNone, 0, 6, 0, 0, 0), 0, TInstant::Seconds(3), 10}, + {NPQ::TKey(NPQ::TKeyPrefix::EType::TypeNone, 0, 1, 0, 0, 0), 0, TInstant::Seconds(1), 10}, + {NPQ::TKey(NPQ::TKeyPrefix::EType::TypeNone, 0, 2, 0, 0, 0), 0, TInstant::Seconds(1), 10}, + {NPQ::TKey(NPQ::TKeyPrefix::EType::TypeNone, 0, 3, 0, 0, 0), 0, TInstant::Seconds(2), 10}, + {NPQ::TKey(NPQ::TKeyPrefix::EType::TypeNone, 0, 4, 0, 0, 0), 0, TInstant::Seconds(2), 10}, + {NPQ::TKey(NPQ::TKeyPrefix::EType::TypeNone, 0, 5, 0, 0, 0), 0, TInstant::Seconds(3), 10}, + {NPQ::TKey(NPQ::TKeyPrefix::EType::TypeNone, 0, 6, 0, 0, 0), 0, TInstant::Seconds(3), 10}, }; - UNIT_ASSERT_EQUAL(NPQ::GetOffsetEstimate({}, TInstant::MilliSeconds(0), 9999), 9999); - UNIT_ASSERT_EQUAL(NPQ::GetOffsetEstimate(container, TInstant::MilliSeconds(0), 9999), 1); - UNIT_ASSERT_EQUAL(NPQ::GetOffsetEstimate(container, TInstant::MilliSeconds(500), 9999), 1); - UNIT_ASSERT_EQUAL(NPQ::GetOffsetEstimate(container, TInstant::MilliSeconds(1000), 9999), 1); - UNIT_ASSERT_EQUAL(NPQ::GetOffsetEstimate(container, TInstant::MilliSeconds(1500), 9999), 3); - UNIT_ASSERT_EQUAL(NPQ::GetOffsetEstimate(container, TInstant::MilliSeconds(2000), 9999), 3); - UNIT_ASSERT_EQUAL(NPQ::GetOffsetEstimate(container, TInstant::MilliSeconds(2500), 9999), 5); - UNIT_ASSERT_EQUAL(NPQ::GetOffsetEstimate(container, TInstant::MilliSeconds(3000), 9999), 5); - UNIT_ASSERT_EQUAL(NPQ::GetOffsetEstimate(container, TInstant::MilliSeconds(3500), 9999), 9999); + UNIT_ASSERT_EQUAL(NPQ::GetOffsetEstimate({}, TInstant::MilliSeconds(0), 9999), 9999); + UNIT_ASSERT_EQUAL(NPQ::GetOffsetEstimate(container, TInstant::MilliSeconds(0), 9999), 1); + UNIT_ASSERT_EQUAL(NPQ::GetOffsetEstimate(container, TInstant::MilliSeconds(500), 9999), 1); + UNIT_ASSERT_EQUAL(NPQ::GetOffsetEstimate(container, TInstant::MilliSeconds(1000), 9999), 1); + UNIT_ASSERT_EQUAL(NPQ::GetOffsetEstimate(container, TInstant::MilliSeconds(1500), 9999), 3); + UNIT_ASSERT_EQUAL(NPQ::GetOffsetEstimate(container, TInstant::MilliSeconds(2000), 9999), 3); + UNIT_ASSERT_EQUAL(NPQ::GetOffsetEstimate(container, TInstant::MilliSeconds(2500), 9999), 5); + UNIT_ASSERT_EQUAL(NPQ::GetOffsetEstimate(container, TInstant::MilliSeconds(3000), 9999), 5); + UNIT_ASSERT_EQUAL(NPQ::GetOffsetEstimate(container, TInstant::MilliSeconds(3500), 9999), 9999); } Y_UNIT_TEST(TestMaxTimeLagRewind) { diff --git a/ydb/core/persqueue/pq_ut.h b/ydb/core/persqueue/pq_ut.h index e21ba73316..3db93bc104 100644 --- a/ydb/core/persqueue/pq_ut.h +++ b/ydb/core/persqueue/pq_ut.h @@ -1,7 +1,7 @@ #pragma once - -#include "pq.h" -#include "user_info.h" + +#include "pq.h" +#include "user_info.h" #include <ydb/core/testlib/basics/runtime.h> #include <ydb/core/tablet_flat/tablet_flat_executed.h> @@ -1093,7 +1093,7 @@ void FillDeprecatedUserInfo(NKikimrClient::TKeyValueRequest_TCmdWrite* write, TS NPQ::TKeyPrefix ikeyDeprecated(NPQ::TKeyPrefix::TypeInfo, partition, NPQ::TKeyPrefix::MarkUserDeprecated); ikeyDeprecated.Append(client.c_str(), client.size()); - TBuffer idataDeprecated = NPQ::NDeprecatedUserData::Serialize(offset, gen, step, session); + TBuffer idataDeprecated = NPQ::NDeprecatedUserData::Serialize(offset, gen, step, session); write->SetKey(ikeyDeprecated.Data(), ikeyDeprecated.Size()); write->SetValue(idataDeprecated.Data(), idataDeprecated.Size()); } diff --git a/ydb/core/persqueue/read.h b/ydb/core/persqueue/read.h index 483add46e8..d0bc79c692 100644 --- a/ydb/core/persqueue/read.h +++ b/ydb/core/persqueue/read.h @@ -1,5 +1,5 @@ #pragma once - + #include "partition.h" #include "pq_l2_service.h" #include "cache_eviction.h" diff --git a/ydb/core/persqueue/read_balancer.cpp b/ydb/core/persqueue/read_balancer.cpp index 665c854cb2..4eb9c7206d 100644 --- a/ydb/core/persqueue/read_balancer.cpp +++ b/ydb/core/persqueue/read_balancer.cpp @@ -9,9 +9,9 @@ namespace NPQ { using namespace NTabletFlatExecutor; -static constexpr TDuration ACL_SUCCESS_RETRY_TIMEOUT = TDuration::Seconds(30); -static constexpr TDuration ACL_ERROR_RETRY_TIMEOUT = TDuration::Seconds(5); -static constexpr TDuration ACL_EXPIRATION_TIMEOUT = TDuration::Minutes(5); +static constexpr TDuration ACL_SUCCESS_RETRY_TIMEOUT = TDuration::Seconds(30); +static constexpr TDuration ACL_ERROR_RETRY_TIMEOUT = TDuration::Seconds(5); +static constexpr TDuration ACL_EXPIRATION_TIMEOUT = TDuration::Minutes(5); bool TPersQueueReadBalancer::TTxPreInit::Execute(TTransactionContext& txc, const TActorContext& ctx) { Y_UNUSED(ctx); @@ -319,7 +319,7 @@ void TPersQueueReadBalancer::Handle(TEvPersQueue::TEvCheckACL::TPtr &ev, const T return; } - if (ctx.Now() > LastACLUpdate + ACL_EXPIRATION_TIMEOUT || Topic.empty()) { //Topic.empty is only for tests + if (ctx.Now() > LastACLUpdate + ACL_EXPIRATION_TIMEOUT || Topic.empty()) { //Topic.empty is only for tests WaitingACLRequests.push_back(ev); return; } @@ -394,7 +394,7 @@ void TPersQueueReadBalancer::Handle(TEvPersQueue::TEvWakeupClient::TPtr &ev, con } void TPersQueueReadBalancer::Handle(TEvPersQueue::TEvDescribe::TPtr &ev, const TActorContext& ctx) { - if (ctx.Now() > LastACLUpdate + ACL_EXPIRATION_TIMEOUT || Topic.empty()) { //Topic.empty is only for tests + if (ctx.Now() > LastACLUpdate + ACL_EXPIRATION_TIMEOUT || Topic.empty()) { //Topic.empty is only for tests WaitingDescribeRequests.push_back(ev); return; } else { @@ -685,7 +685,7 @@ void TPersQueueReadBalancer::Handle(NSchemeShard::TEvSchemeShard::TEvDescribeSch AnswerWaitingRequests(ctx); } else { LOG_DEBUG_S(ctx, NKikimrServices::PERSQUEUE_READ_BALANCER, GetPrefix() << "couldn't receive ACL due to " << record.GetStatus()); - ctx.Schedule(ACL_ERROR_RETRY_TIMEOUT, new TEvPersQueue::TEvUpdateACL()); + ctx.Schedule(ACL_ERROR_RETRY_TIMEOUT, new TEvPersQueue::TEvUpdateACL()); } } @@ -715,7 +715,7 @@ void TPersQueueReadBalancer::GetACL(const TActorContext& ctx) { if (WaitingForACL) // if there is request infly return; if (SchemeShardId == 0) { - ctx.Schedule(ACL_SUCCESS_RETRY_TIMEOUT, new TEvPersQueue::TEvUpdateACL()); + ctx.Schedule(ACL_SUCCESS_RETRY_TIMEOUT, new TEvPersQueue::TEvUpdateACL()); } else { WaitingForACL = true; RequestTabletIfNeeded(SchemeShardId, ctx); @@ -771,14 +771,14 @@ void TPersQueueReadBalancer::TClientInfo::AddSession(const ui32 group, const THa } auto it = ClientGroupsInfo.find(group); - it->second.SessionsInfo.insert({ - std::make_pair(pipe, it->second.RandomNumber), - TClientGroupInfo::TSessionInfo( - record.GetSession(), sender, - record.HasClientNode() ? record.GetClientNode() : "none", - sender.NodeId(), TAppData::TimeProvider->Now() - ) - }); + it->second.SessionsInfo.insert({ + std::make_pair(pipe, it->second.RandomNumber), + TClientGroupInfo::TSessionInfo( + record.GetSession(), sender, + record.HasClientNode() ? record.GetClientNode() : "none", + sender.NodeId(), TAppData::TimeProvider->Now() + ) + }); } @@ -888,7 +888,7 @@ void TPersQueueReadBalancer::Handle(TEvPersQueue::TEvGetReadSessionsInfo::TPtr& pi->SetClientNode(jt->second.ClientNode); pi->SetProxyNodeId(jt->second.ProxyNodeId); pi->SetSession(jt->second.Session); - pi->SetTimestamp(jt->second.Timestamp.Seconds()); + pi->SetTimestamp(jt->second.Timestamp.Seconds()); } else { pi->SetClientNode(""); pi->SetProxyNodeId(0); diff --git a/ydb/core/persqueue/read_balancer.h b/ydb/core/persqueue/read_balancer.h index 1fefe3f49f..6a07edcb78 100644 --- a/ydb/core/persqueue/read_balancer.h +++ b/ydb/core/persqueue/read_balancer.h @@ -334,7 +334,7 @@ class TPersQueueReadBalancer : public TActor<TPersQueueReadBalancer>, public TTa struct TClientGroupInfo { struct TSessionInfo { - TSessionInfo(const TString& session, const TActorId sender, const TString& clientNode, ui32 proxyNodeId, TInstant ts) + TSessionInfo(const TString& session, const TActorId sender, const TString& clientNode, ui32 proxyNodeId, TInstant ts) : Session(session) , Sender(sender) , NumSuspended(0) @@ -351,7 +351,7 @@ class TPersQueueReadBalancer : public TActor<TPersQueueReadBalancer>, public TTa TString ClientNode; ui32 ProxyNodeId; - TInstant Timestamp; + TInstant Timestamp; }; TString ClientId; diff --git a/ydb/core/persqueue/read_speed_limiter.cpp b/ydb/core/persqueue/read_speed_limiter.cpp index 47be44036a..793338420d 100644 --- a/ydb/core/persqueue/read_speed_limiter.cpp +++ b/ydb/core/persqueue/read_speed_limiter.cpp @@ -1,184 +1,184 @@ -#include "read_speed_limiter.h" -#include "event_helpers.h" - +#include "read_speed_limiter.h" +#include "event_helpers.h" + #include <ydb/core/base/appdata.h> #include <ydb/core/base/counters.h> #include <ydb/core/persqueue/percentile_counter.h> #include <ydb/library/persqueue/topic_parser/topic_parser.h> - -#include <library/cpp/actors/core/log.h> - -#include <util/string/join.h> -#include <util/string/vector.h> - -namespace NKikimr { -namespace NPQ { - -const TDuration UPDATE_COUNTERS_INTERVAL = TDuration::Seconds(5); -const TDuration DO_NOT_QUOTE_AFTER_ERROR_PERIOD = TDuration::Seconds(5); - -const TString TReadSpeedLimiter::READ_QUOTA_ROOT_PATH = "read-quota"; - -constexpr NKikimrServices::TActivity::EType TReadSpeedLimiter::ActorActivityType() { - return NKikimrServices::TActivity::PERSQUEUE_READ_SPEED_LIMITER; -} - -TReadSpeedLimiter::TReadSpeedLimiter( - TActorId tabletActor, - TActorId partitionActor, - ui64 tabletId, - const TString& topicName, - ui32 partition, - const TString& user, - const TTabletCountersBase& counters -) - : TabletActor(tabletActor) - , PartitionActor(partitionActor) - , TabletId(tabletId) - , TopicName(topicName) - , Partition(partition) - , User(user) - , ConsumerPath(NPersQueue::ConvertOldConsumerName(user)) - , ReadCreditBytes(AppData()->PQConfig.GetQuotingConfig().GetReadCreditBytes()) -{ - Counters.Populate(counters); - - auto userParts = SplitString(ConsumerPath, "/"); // account/folder/topic // account is first element - - const TString account = userParts[0]; - userParts[0] = READ_QUOTA_ROOT_PATH; // read-quota/folder/topic - - const auto& quotingConfig = AppData()->PQConfig.GetQuotingConfig(); - KesusPath = Join("/", quotingConfig.GetQuotersDirectoryPath(), account); - QuotaResourcePath = JoinSeq("/", userParts); - LOG_INFO_S(TActivationContext::AsActorContext(), NKikimrServices::PQ_READ_SPEED_LIMITER, - LimiterDescription() <<" kesus=" << KesusPath << " resource_path=" << QuotaResourcePath); -} - - -void TReadSpeedLimiter::Bootstrap(const TActorContext& ctx) { - Become(&TThis::StateWork); - ctx.Schedule(UPDATE_COUNTERS_INTERVAL, new TEvPQ::TEvUpdateCounters); - - auto counters = AppData()->Counters; - if (counters && TopicName.Contains("--")) { + +#include <library/cpp/actors/core/log.h> + +#include <util/string/join.h> +#include <util/string/vector.h> + +namespace NKikimr { +namespace NPQ { + +const TDuration UPDATE_COUNTERS_INTERVAL = TDuration::Seconds(5); +const TDuration DO_NOT_QUOTE_AFTER_ERROR_PERIOD = TDuration::Seconds(5); + +const TString TReadSpeedLimiter::READ_QUOTA_ROOT_PATH = "read-quota"; + +constexpr NKikimrServices::TActivity::EType TReadSpeedLimiter::ActorActivityType() { + return NKikimrServices::TActivity::PERSQUEUE_READ_SPEED_LIMITER; +} + +TReadSpeedLimiter::TReadSpeedLimiter( + TActorId tabletActor, + TActorId partitionActor, + ui64 tabletId, + const TString& topicName, + ui32 partition, + const TString& user, + const TTabletCountersBase& counters +) + : TabletActor(tabletActor) + , PartitionActor(partitionActor) + , TabletId(tabletId) + , TopicName(topicName) + , Partition(partition) + , User(user) + , ConsumerPath(NPersQueue::ConvertOldConsumerName(user)) + , ReadCreditBytes(AppData()->PQConfig.GetQuotingConfig().GetReadCreditBytes()) +{ + Counters.Populate(counters); + + auto userParts = SplitString(ConsumerPath, "/"); // account/folder/topic // account is first element + + const TString account = userParts[0]; + userParts[0] = READ_QUOTA_ROOT_PATH; // read-quota/folder/topic + + const auto& quotingConfig = AppData()->PQConfig.GetQuotingConfig(); + KesusPath = Join("/", quotingConfig.GetQuotersDirectoryPath(), account); + QuotaResourcePath = JoinSeq("/", userParts); + LOG_INFO_S(TActivationContext::AsActorContext(), NKikimrServices::PQ_READ_SPEED_LIMITER, + LimiterDescription() <<" kesus=" << KesusPath << " resource_path=" << QuotaResourcePath); +} + + +void TReadSpeedLimiter::Bootstrap(const TActorContext& ctx) { + Become(&TThis::StateWork); + ctx.Schedule(UPDATE_COUNTERS_INTERVAL, new TEvPQ::TEvUpdateCounters); + + auto counters = AppData()->Counters; + if (counters && TopicName.Contains("--")) { QuotaWaitCounter.Reset(new TPercentileCounter( - GetServiceCounters(counters, "pqproxy|consumerReadQuotaWait"), - GetLabels(TopicName), - { - {"Client", User}, - {"ConsumerPath", ConsumerPath}, - {"sensor", "ConsumerReadQuotaWait"} - }, - "Interval", - TVector<std::pair<ui64, TString>>{{0, "0ms"}, {1, "1ms"}, {5, "5ms"}, {10, "10ms"}, - {20, "20ms"}, {50, "50ms"}, {100, "100ms"}, {500, "500ms"}, - {1000, "1000ms"}, {2500, "2500ms"}, {5000, "5000ms"}, {10000, "10000ms"}, {9999999, "999999ms"}}, - true + GetServiceCounters(counters, "pqproxy|consumerReadQuotaWait"), + GetLabels(TopicName), + { + {"Client", User}, + {"ConsumerPath", ConsumerPath}, + {"sensor", "ConsumerReadQuotaWait"} + }, + "Interval", + TVector<std::pair<ui64, TString>>{{0, "0ms"}, {1, "1ms"}, {5, "5ms"}, {10, "10ms"}, + {20, "20ms"}, {50, "50ms"}, {100, "100ms"}, {500, "500ms"}, + {1000, "1000ms"}, {2500, "2500ms"}, {5000, "5000ms"}, {10000, "10000ms"}, {9999999, "999999ms"}}, + true )); - } -} - -void TReadSpeedLimiter::Handle(TEvents::TEvPoisonPill::TPtr&, const TActorContext& ctx) { - LOG_INFO_S(ctx, NKikimrServices::PQ_READ_SPEED_LIMITER, LimiterDescription() << " killed"); - for (const auto& event : Queue) { - auto cookie = event.Event->Get()->Cookie; - ReplyPersQueueError( - TabletActor, ctx, TabletId, TopicName, Partition, Counters, NKikimrServices::PQ_READ_SPEED_LIMITER, - cookie, NPersQueue::NErrorCode::INITIALIZING, - TStringBuilder() << "Tablet is restarting, topic " << TopicName << " (ReadInfo) cookie " << cookie - ); - } - Die(ctx); -} - -void TReadSpeedLimiter::HandleUpdateCounters(TEvPQ::TEvUpdateCounters::TPtr&, const TActorContext& ctx) { - ctx.Schedule(UPDATE_COUNTERS_INTERVAL, new TEvPQ::TEvUpdateCounters); - ctx.Send(PartitionActor, new NReadSpeedLimiterEvents::TEvCounters(Counters, User)); -} - -void TReadSpeedLimiter::HandleReadQuotaRequest(NReadSpeedLimiterEvents::TEvRequest::TPtr& ev, const TActorContext& ctx) { - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_SPEED_LIMITER, - LimiterDescription() << " quota required for cookie=" << ev->Get()->ReadRequest->Get()->Cookie - ); - bool hasActualErrors = ctx.Now() - LastReportedErrorTime <= DO_NOT_QUOTE_AFTER_ERROR_PERIOD; - if ((QuotaRequestInFlight || !InProcessReadRequestCookies.empty()) && !hasActualErrors) { - Queue.emplace_back(std::move(ev->Get()->ReadRequest), ctx.Now()); - } else { - ApproveRead(ev->Get()->ReadRequest, ctx.Now(), ctx); - } -} - -void TReadSpeedLimiter::HandleReadQuotaConsumed(NReadSpeedLimiterEvents::TEvConsumed::TPtr& ev, const TActorContext& ctx) { - ConsumedBytesInCredit += ev->Get()->ReadBytes; - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_SPEED_LIMITER, LimiterDescription() - << "consumed read quota " << ev->Get()->ReadBytes - << " bytes by cookie=" << ev->Get()->ReadRequestCookie - << ", consumed in credit " << ConsumedBytesInCredit << "/" << ReadCreditBytes - ); - auto it = InProcessReadRequestCookies.find(ev->Get()->ReadRequestCookie); - Y_VERIFY(it != InProcessReadRequestCookies.end()); - InProcessReadRequestCookies.erase(it); - - if (!QuotaRequestInFlight) { - if (ConsumedBytesInCredit >= ReadCreditBytes) { - Send(MakeQuoterServiceID(), - new TEvQuota::TEvRequest( - TEvQuota::EResourceOperator::And, - { TEvQuota::TResourceLeaf(KesusPath, QuotaResourcePath, ConsumedBytesInCredit) }, - TDuration::Max()), - 0, - ++CurrentQuotaRequestCookie - ); - QuotaRequestInFlight = true; - ConsumedBytesInCredit = 0; - } else if (!Queue.empty()){ - ApproveRead(std::move(Queue.front().Event), Queue.front().StartWait, ctx); - Queue.pop_front(); - } - } -} - -void TReadSpeedLimiter::HandleClearance(TEvQuota::TEvClearance::TPtr& ev, const TActorContext& ctx) { - QuotaRequestInFlight = false; - const ui64 cookie = ev->Cookie; - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_SPEED_LIMITER, - LimiterDescription() << "Got read quota:" << ev->Get()->Result << ". Cookie: " << cookie - ); - - Y_VERIFY(CurrentQuotaRequestCookie == cookie); - if (!Queue.empty()) { - ApproveRead(std::move(Queue.front().Event), Queue.front().StartWait, ctx); - Queue.pop_front(); - } - - if (Y_UNLIKELY(ev->Get()->Result != TEvQuota::TEvClearance::EResult::Success)) { - Y_VERIFY(ev->Get()->Result != TEvQuota::TEvClearance::EResult::Deadline); // We set deadline == inf in quota request. - if (ctx.Now() - LastReportedErrorTime > TDuration::Minutes(1)) { - LOG_ERROR_S(ctx, NKikimrServices::PQ_READ_SPEED_LIMITER, LimiterDescription() << "Got read quota error: " << ev->Get()->Result); - LastReportedErrorTime = ctx.Now(); - } - return; - } -} - -void TReadSpeedLimiter::TReadSpeedLimiter::ApproveRead(TEvPQ::TEvRead::TPtr ev, TInstant startWait, const TActorContext& ctx) { - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_SPEED_LIMITER, - LimiterDescription() << " approve read for cookie=" << ev->Get()->Cookie - ); - InProcessReadRequestCookies.insert(ev->Get()->Cookie); - - auto waitTime = ctx.Now() - startWait; - Send(PartitionActor, new NReadSpeedLimiterEvents::TEvResponse(ev.Release(), waitTime)); - - if (QuotaWaitCounter) { - QuotaWaitCounter->IncFor(waitTime.MilliSeconds()); - } -} - -TString TReadSpeedLimiter::LimiterDescription() const { - return TStringBuilder() << "topic=" << TopicName << ":" << Partition << " user=" << User << ": "; -} - -}// NPQ -}// NKikimr + } +} + +void TReadSpeedLimiter::Handle(TEvents::TEvPoisonPill::TPtr&, const TActorContext& ctx) { + LOG_INFO_S(ctx, NKikimrServices::PQ_READ_SPEED_LIMITER, LimiterDescription() << " killed"); + for (const auto& event : Queue) { + auto cookie = event.Event->Get()->Cookie; + ReplyPersQueueError( + TabletActor, ctx, TabletId, TopicName, Partition, Counters, NKikimrServices::PQ_READ_SPEED_LIMITER, + cookie, NPersQueue::NErrorCode::INITIALIZING, + TStringBuilder() << "Tablet is restarting, topic " << TopicName << " (ReadInfo) cookie " << cookie + ); + } + Die(ctx); +} + +void TReadSpeedLimiter::HandleUpdateCounters(TEvPQ::TEvUpdateCounters::TPtr&, const TActorContext& ctx) { + ctx.Schedule(UPDATE_COUNTERS_INTERVAL, new TEvPQ::TEvUpdateCounters); + ctx.Send(PartitionActor, new NReadSpeedLimiterEvents::TEvCounters(Counters, User)); +} + +void TReadSpeedLimiter::HandleReadQuotaRequest(NReadSpeedLimiterEvents::TEvRequest::TPtr& ev, const TActorContext& ctx) { + LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_SPEED_LIMITER, + LimiterDescription() << " quota required for cookie=" << ev->Get()->ReadRequest->Get()->Cookie + ); + bool hasActualErrors = ctx.Now() - LastReportedErrorTime <= DO_NOT_QUOTE_AFTER_ERROR_PERIOD; + if ((QuotaRequestInFlight || !InProcessReadRequestCookies.empty()) && !hasActualErrors) { + Queue.emplace_back(std::move(ev->Get()->ReadRequest), ctx.Now()); + } else { + ApproveRead(ev->Get()->ReadRequest, ctx.Now(), ctx); + } +} + +void TReadSpeedLimiter::HandleReadQuotaConsumed(NReadSpeedLimiterEvents::TEvConsumed::TPtr& ev, const TActorContext& ctx) { + ConsumedBytesInCredit += ev->Get()->ReadBytes; + LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_SPEED_LIMITER, LimiterDescription() + << "consumed read quota " << ev->Get()->ReadBytes + << " bytes by cookie=" << ev->Get()->ReadRequestCookie + << ", consumed in credit " << ConsumedBytesInCredit << "/" << ReadCreditBytes + ); + auto it = InProcessReadRequestCookies.find(ev->Get()->ReadRequestCookie); + Y_VERIFY(it != InProcessReadRequestCookies.end()); + InProcessReadRequestCookies.erase(it); + + if (!QuotaRequestInFlight) { + if (ConsumedBytesInCredit >= ReadCreditBytes) { + Send(MakeQuoterServiceID(), + new TEvQuota::TEvRequest( + TEvQuota::EResourceOperator::And, + { TEvQuota::TResourceLeaf(KesusPath, QuotaResourcePath, ConsumedBytesInCredit) }, + TDuration::Max()), + 0, + ++CurrentQuotaRequestCookie + ); + QuotaRequestInFlight = true; + ConsumedBytesInCredit = 0; + } else if (!Queue.empty()){ + ApproveRead(std::move(Queue.front().Event), Queue.front().StartWait, ctx); + Queue.pop_front(); + } + } +} + +void TReadSpeedLimiter::HandleClearance(TEvQuota::TEvClearance::TPtr& ev, const TActorContext& ctx) { + QuotaRequestInFlight = false; + const ui64 cookie = ev->Cookie; + LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_SPEED_LIMITER, + LimiterDescription() << "Got read quota:" << ev->Get()->Result << ". Cookie: " << cookie + ); + + Y_VERIFY(CurrentQuotaRequestCookie == cookie); + if (!Queue.empty()) { + ApproveRead(std::move(Queue.front().Event), Queue.front().StartWait, ctx); + Queue.pop_front(); + } + + if (Y_UNLIKELY(ev->Get()->Result != TEvQuota::TEvClearance::EResult::Success)) { + Y_VERIFY(ev->Get()->Result != TEvQuota::TEvClearance::EResult::Deadline); // We set deadline == inf in quota request. + if (ctx.Now() - LastReportedErrorTime > TDuration::Minutes(1)) { + LOG_ERROR_S(ctx, NKikimrServices::PQ_READ_SPEED_LIMITER, LimiterDescription() << "Got read quota error: " << ev->Get()->Result); + LastReportedErrorTime = ctx.Now(); + } + return; + } +} + +void TReadSpeedLimiter::TReadSpeedLimiter::ApproveRead(TEvPQ::TEvRead::TPtr ev, TInstant startWait, const TActorContext& ctx) { + LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_SPEED_LIMITER, + LimiterDescription() << " approve read for cookie=" << ev->Get()->Cookie + ); + InProcessReadRequestCookies.insert(ev->Get()->Cookie); + + auto waitTime = ctx.Now() - startWait; + Send(PartitionActor, new NReadSpeedLimiterEvents::TEvResponse(ev.Release(), waitTime)); + + if (QuotaWaitCounter) { + QuotaWaitCounter->IncFor(waitTime.MilliSeconds()); + } +} + +TString TReadSpeedLimiter::LimiterDescription() const { + return TStringBuilder() << "topic=" << TopicName << ":" << Partition << " user=" << User << ": "; +} + +}// NPQ +}// NKikimr diff --git a/ydb/core/persqueue/read_speed_limiter.h b/ydb/core/persqueue/read_speed_limiter.h index 9504946722..23ca8dad95 100644 --- a/ydb/core/persqueue/read_speed_limiter.h +++ b/ydb/core/persqueue/read_speed_limiter.h @@ -1,137 +1,137 @@ -#pragma once - +#pragma once + #include <ydb/core/base/quoter.h> #include <ydb/core/persqueue/events/internal.h> - -#include <library/cpp/actors/core/hfunc.h> - - -namespace NKikimr { -namespace NPQ { - -class TPercentileCounter; - -namespace NReadSpeedLimiterEvents { - struct TEvRequest : public TEventLocal<TEvRequest, TEvPQ::EvReadLimiterRequest> { - TEvRequest(TEvPQ::TEvRead::TPtr readRequest) - : ReadRequest(std::move(readRequest)) - {} - - TEvPQ::TEvRead::TPtr ReadRequest; - }; - - struct TEvResponse : public TEventLocal<TEvResponse, TEvPQ::EvReadLimiterResponse> { - TEvResponse(TEvPQ::TEvRead::TPtr readRequest, TDuration waitTime) - : ReadRequest(std::move(readRequest)) - , WaitTime(waitTime) - {} - - TEvPQ::TEvRead::TPtr ReadRequest; - TDuration WaitTime; - }; - - struct TEvConsumed : public TEventLocal<TEvConsumed, TEvPQ::EvReadLimiterConsumed> { - TEvConsumed(ui64 readBytes, ui64 readRequestCookie) - : ReadBytes(readBytes) - , ReadRequestCookie(readRequestCookie) - {} - - ui64 ReadBytes; - ui64 ReadRequestCookie; - }; - - struct TEvCounters : public TEventLocal<TEvCounters, TEvPQ::EvReadLimiterCounters> { - TEvCounters(const NKikimr::TTabletCountersBase& counters, const TString& user) - : User(user) - { - Counters.Populate(counters); - } - - NKikimr::TTabletCountersBase Counters; - const TString User; - }; -} - -class TReadSpeedLimiter : public TActorBootstrapped<TReadSpeedLimiter> { -private: - static const TString READ_QUOTA_ROOT_PATH; - - struct TQueueEvent { - TQueueEvent(TEvPQ::TEvRead::TPtr&& event, TInstant startWait) - : Event(event) - , StartWait(startWait) - {} - - TEvPQ::TEvRead::TPtr Event; - TInstant StartWait; - }; - -private: - STFUNC(StateWork) - { - TRACE_EVENT(NKikimrServices::PQ_READ_SPEED_LIMITER); - switch (ev->GetTypeRewrite()) { - HFuncTraced(TEvPQ::TEvUpdateCounters, HandleUpdateCounters); - HFuncTraced(NReadSpeedLimiterEvents::TEvRequest, HandleReadQuotaRequest); - HFuncTraced(NReadSpeedLimiterEvents::TEvConsumed, HandleReadQuotaConsumed); - HFuncTraced(TEvQuota::TEvClearance, HandleClearance); - HFuncTraced(TEvents::TEvPoisonPill, Handle); - default: - break; - }; - } - -public: - static constexpr NKikimrServices::TActivity::EType ActorActivityType(); - - TReadSpeedLimiter( - TActorId tabletActor, - TActorId partitionActor, - ui64 tabletId, - const TString& topicName, - ui32 partition, - const TString& user, - const TTabletCountersBase& counters - ); - - void Bootstrap(const TActorContext& ctx); - void Handle(TEvents::TEvPoisonPill::TPtr& ev, const TActorContext& ctx); - void HandleUpdateCounters(TEvPQ::TEvUpdateCounters::TPtr& ev, const TActorContext& ctx); - void HandleReadQuotaRequest(NReadSpeedLimiterEvents::TEvRequest::TPtr& ev, const TActorContext& ctx); - void HandleReadQuotaConsumed(NReadSpeedLimiterEvents::TEvConsumed::TPtr& ev, const TActorContext& ctx); - void HandleClearance(TEvQuota::TEvClearance::TPtr& ev, const TActorContext& ctx); - - void ApproveRead(TEvPQ::TEvRead::TPtr ev, TInstant startWait, const TActorContext& ctx); - -private: - TString LimiterDescription() const; - -private: - const TActorId TabletActor; - const TActorId PartitionActor; - const ui64 TabletId; - const TString TopicName; - const ui32 Partition; - const TString User; - const TString ConsumerPath; - const ui64 ReadCreditBytes; - - ui64 ConsumedBytesInCredit = 0; - - TString KesusPath; - TString QuotaResourcePath; - - TDeque<TQueueEvent> Queue; - - bool QuotaRequestInFlight = false; - ui64 CurrentQuotaRequestCookie = 0; - THashSet<ui64> InProcessReadRequestCookies; - - - TTabletCountersBase Counters; - THolder<TPercentileCounter> QuotaWaitCounter; - TInstant LastReportedErrorTime; -}; - -}// NPQ -}// NKikimr + +#include <library/cpp/actors/core/hfunc.h> + + +namespace NKikimr { +namespace NPQ { + +class TPercentileCounter; + +namespace NReadSpeedLimiterEvents { + struct TEvRequest : public TEventLocal<TEvRequest, TEvPQ::EvReadLimiterRequest> { + TEvRequest(TEvPQ::TEvRead::TPtr readRequest) + : ReadRequest(std::move(readRequest)) + {} + + TEvPQ::TEvRead::TPtr ReadRequest; + }; + + struct TEvResponse : public TEventLocal<TEvResponse, TEvPQ::EvReadLimiterResponse> { + TEvResponse(TEvPQ::TEvRead::TPtr readRequest, TDuration waitTime) + : ReadRequest(std::move(readRequest)) + , WaitTime(waitTime) + {} + + TEvPQ::TEvRead::TPtr ReadRequest; + TDuration WaitTime; + }; + + struct TEvConsumed : public TEventLocal<TEvConsumed, TEvPQ::EvReadLimiterConsumed> { + TEvConsumed(ui64 readBytes, ui64 readRequestCookie) + : ReadBytes(readBytes) + , ReadRequestCookie(readRequestCookie) + {} + + ui64 ReadBytes; + ui64 ReadRequestCookie; + }; + + struct TEvCounters : public TEventLocal<TEvCounters, TEvPQ::EvReadLimiterCounters> { + TEvCounters(const NKikimr::TTabletCountersBase& counters, const TString& user) + : User(user) + { + Counters.Populate(counters); + } + + NKikimr::TTabletCountersBase Counters; + const TString User; + }; +} + +class TReadSpeedLimiter : public TActorBootstrapped<TReadSpeedLimiter> { +private: + static const TString READ_QUOTA_ROOT_PATH; + + struct TQueueEvent { + TQueueEvent(TEvPQ::TEvRead::TPtr&& event, TInstant startWait) + : Event(event) + , StartWait(startWait) + {} + + TEvPQ::TEvRead::TPtr Event; + TInstant StartWait; + }; + +private: + STFUNC(StateWork) + { + TRACE_EVENT(NKikimrServices::PQ_READ_SPEED_LIMITER); + switch (ev->GetTypeRewrite()) { + HFuncTraced(TEvPQ::TEvUpdateCounters, HandleUpdateCounters); + HFuncTraced(NReadSpeedLimiterEvents::TEvRequest, HandleReadQuotaRequest); + HFuncTraced(NReadSpeedLimiterEvents::TEvConsumed, HandleReadQuotaConsumed); + HFuncTraced(TEvQuota::TEvClearance, HandleClearance); + HFuncTraced(TEvents::TEvPoisonPill, Handle); + default: + break; + }; + } + +public: + static constexpr NKikimrServices::TActivity::EType ActorActivityType(); + + TReadSpeedLimiter( + TActorId tabletActor, + TActorId partitionActor, + ui64 tabletId, + const TString& topicName, + ui32 partition, + const TString& user, + const TTabletCountersBase& counters + ); + + void Bootstrap(const TActorContext& ctx); + void Handle(TEvents::TEvPoisonPill::TPtr& ev, const TActorContext& ctx); + void HandleUpdateCounters(TEvPQ::TEvUpdateCounters::TPtr& ev, const TActorContext& ctx); + void HandleReadQuotaRequest(NReadSpeedLimiterEvents::TEvRequest::TPtr& ev, const TActorContext& ctx); + void HandleReadQuotaConsumed(NReadSpeedLimiterEvents::TEvConsumed::TPtr& ev, const TActorContext& ctx); + void HandleClearance(TEvQuota::TEvClearance::TPtr& ev, const TActorContext& ctx); + + void ApproveRead(TEvPQ::TEvRead::TPtr ev, TInstant startWait, const TActorContext& ctx); + +private: + TString LimiterDescription() const; + +private: + const TActorId TabletActor; + const TActorId PartitionActor; + const ui64 TabletId; + const TString TopicName; + const ui32 Partition; + const TString User; + const TString ConsumerPath; + const ui64 ReadCreditBytes; + + ui64 ConsumedBytesInCredit = 0; + + TString KesusPath; + TString QuotaResourcePath; + + TDeque<TQueueEvent> Queue; + + bool QuotaRequestInFlight = false; + ui64 CurrentQuotaRequestCookie = 0; + THashSet<ui64> InProcessReadRequestCookies; + + + TTabletCountersBase Counters; + THolder<TPercentileCounter> QuotaWaitCounter; + TInstant LastReportedErrorTime; +}; + +}// NPQ +}// NKikimr diff --git a/ydb/core/persqueue/subscriber.cpp b/ydb/core/persqueue/subscriber.cpp index 0d5fa02f79..641c566a1c 100644 --- a/ydb/core/persqueue/subscriber.cpp +++ b/ydb/core/persqueue/subscriber.cpp @@ -55,11 +55,11 @@ TSubscriber::TSubscriber(const ui32 partition, TTabletCountersBase& counters, co , Tablet(tablet) {} -TMaybe<TReadInfo> TSubscriber::OnTimeout(TEvPQ::TEvReadTimeout::TPtr& ev) { +TMaybe<TReadInfo> TSubscriber::OnTimeout(TEvPQ::TEvReadTimeout::TPtr& ev) { TMaybe<TReadInfo> res = Subscriber.ForgetSubscription(ev->Get()->Cookie); - if (res) { - Counters.Cumulative()[COUNTER_PQ_READ_SUBSCRIPTION_TIMEOUT].Increment(1); - } + if (res) { + Counters.Cumulative()[COUNTER_PQ_READ_SUBSCRIPTION_TIMEOUT].Increment(1); + } return res; } diff --git a/ydb/core/persqueue/subscriber.h b/ydb/core/persqueue/subscriber.h index 13c9fd08c6..7bbfe41e1f 100644 --- a/ydb/core/persqueue/subscriber.h +++ b/ydb/core/persqueue/subscriber.h @@ -1,8 +1,8 @@ #pragma once - + #include "header.h" #include "blob.h" - + #include <ydb/core/tablet/tablet_counters.h> #include <ydb/core/base/appdata.h> #include <ydb/core/persqueue/events/internal.h> @@ -12,11 +12,11 @@ namespace NPQ { struct TUserInfo; -struct TReadAnswer { - ui64 Size; - THolder<IEventBase> Event; -}; - +struct TReadAnswer { + ui64 Size; + THolder<IEventBase> Event; +}; + struct TReadInfo { TString User; TString ClientDC; @@ -27,7 +27,7 @@ struct TReadInfo { ui64 Destination; TInstant Timestamp; ui64 ReadTimestampMs; - TDuration WaitQuotaTime; + TDuration WaitQuotaTime; bool IsSubscription; @@ -36,17 +36,17 @@ struct TReadInfo { TVector<TClientBlob> Cached; //records from head TReadInfo() = delete; - TReadInfo( - const TString& user, - const TString& clientDC, - const ui64 offset, - const ui16 partNo, - const ui64 count, - const ui32 size, - const ui64 dst, - ui64 readTimestampMs, - TDuration waitQuotaTime - ) + TReadInfo( + const TString& user, + const TString& clientDC, + const ui64 offset, + const ui16 partNo, + const ui64 count, + const ui32 size, + const ui64 dst, + ui64 readTimestampMs, + TDuration waitQuotaTime + ) : User(user) , ClientDC(clientDC) , Offset(offset) @@ -55,30 +55,30 @@ struct TReadInfo { , Size(size) , Destination(dst) , Timestamp(TAppData::TimeProvider->Now()) - , ReadTimestampMs(readTimestampMs) - , WaitQuotaTime(waitQuotaTime) + , ReadTimestampMs(readTimestampMs) + , WaitQuotaTime(waitQuotaTime) , IsSubscription(false) , CachedOffset(0) {} - TReadAnswer FormAnswer( - const TActorContext& ctx, - const TEvPQ::TEvBlobResponse& response, - const ui64 endOffset, - const ui32 partition, - TUserInfo* ui, - const ui64 dst, - const ui64 sizeLag - ); - - TReadAnswer FormAnswer( - const TActorContext& ctx, - const ui64 endOffset, - const ui32 partition, - TUserInfo* ui, - const ui64 dst, - const ui64 sizeLag - ) { + TReadAnswer FormAnswer( + const TActorContext& ctx, + const TEvPQ::TEvBlobResponse& response, + const ui64 endOffset, + const ui32 partition, + TUserInfo* ui, + const ui64 dst, + const ui64 sizeLag + ); + + TReadAnswer FormAnswer( + const TActorContext& ctx, + const ui64 endOffset, + const ui32 partition, + TUserInfo* ui, + const ui64 dst, + const ui64 sizeLag + ) { TEvPQ::TEvBlobResponse response(0, TVector<TRequestedBlob>()); return FormAnswer(ctx, response, endOffset, partition, ui, dst, sizeLag); } @@ -116,7 +116,7 @@ public: void AddSubscription(TReadInfo&& info, const ui32 timeout, const ui64 cookie, const TActorContext& ctx); //handle of timeout for some read - TMaybe<TReadInfo> OnTimeout(TEvPQ::TEvReadTimeout::TPtr& ev); + TMaybe<TReadInfo> OnTimeout(TEvPQ::TEvReadTimeout::TPtr& ev); //get completed subscriptions TVector<std::pair<TReadInfo, ui64>> GetReads(const ui64 endOffsets); diff --git a/ydb/core/persqueue/user_info.cpp b/ydb/core/persqueue/user_info.cpp index 5b987c6b3a..26811765e2 100644 --- a/ydb/core/persqueue/user_info.cpp +++ b/ydb/core/persqueue/user_info.cpp @@ -1,129 +1,129 @@ -#include "user_info.h" - -namespace NKikimr { -namespace NPQ { - -namespace NDeprecatedUserData { - TBuffer Serialize(ui64 offset, ui32 gen, ui32 step, const TString& session) { - TBuffer data; - data.Resize(sizeof(ui64) + sizeof(ui32) * 2 + session.size()); - memcpy(data.Data(), &offset, sizeof(ui64)); - memcpy(data.Data() + sizeof(ui64), &gen, sizeof(ui32)); - memcpy(data.Data() + sizeof(ui64) + sizeof(ui32), &step, sizeof(ui32)); - memcpy(data.Data() + sizeof(ui64) + 2 * sizeof(ui32), session.data(), session.size()); - return data; - } - - void Parse(const TString& data, ui64& offset, ui32& gen, ui32& step, TString& session) { - Y_VERIFY(sizeof(ui64) <= data.size()); - - offset = *reinterpret_cast<const ui64*>(data.c_str()); - gen = 0; - step = 0; - if (data.size() > sizeof(ui64)) { - gen = reinterpret_cast<const ui32*>(data.c_str() + sizeof(ui64))[0]; - step = reinterpret_cast<const ui32*>(data.c_str() + sizeof(ui64))[1]; - session = data.substr(sizeof(ui64) + 2 * sizeof(ui32)); - } - } -} // NDeprecatedUserData - -TUsersInfoStorage::TUsersInfoStorage( +#include "user_info.h" + +namespace NKikimr { +namespace NPQ { + +namespace NDeprecatedUserData { + TBuffer Serialize(ui64 offset, ui32 gen, ui32 step, const TString& session) { + TBuffer data; + data.Resize(sizeof(ui64) + sizeof(ui32) * 2 + session.size()); + memcpy(data.Data(), &offset, sizeof(ui64)); + memcpy(data.Data() + sizeof(ui64), &gen, sizeof(ui32)); + memcpy(data.Data() + sizeof(ui64) + sizeof(ui32), &step, sizeof(ui32)); + memcpy(data.Data() + sizeof(ui64) + 2 * sizeof(ui32), session.data(), session.size()); + return data; + } + + void Parse(const TString& data, ui64& offset, ui32& gen, ui32& step, TString& session) { + Y_VERIFY(sizeof(ui64) <= data.size()); + + offset = *reinterpret_cast<const ui64*>(data.c_str()); + gen = 0; + step = 0; + if (data.size() > sizeof(ui64)) { + gen = reinterpret_cast<const ui32*>(data.c_str() + sizeof(ui64))[0]; + step = reinterpret_cast<const ui32*>(data.c_str() + sizeof(ui64))[1]; + session = data.substr(sizeof(ui64) + 2 * sizeof(ui32)); + } + } +} // NDeprecatedUserData + +TUsersInfoStorage::TUsersInfoStorage( TString dcId, - ui64 tabletId, - const TString& topicName, - ui32 partition, + ui64 tabletId, + const TString& topicName, + ui32 partition, const TTabletCountersBase& counters, const NKikimrPQ::TPQTabletConfig& config, const TString& cloudId, const TString& dbId, const TString& folderId -) +) : DCId(std::move(dcId)) - , TabletId(tabletId) - , TopicName(topicName) - , Partition(partition) + , TabletId(tabletId) + , TopicName(topicName) + , Partition(partition) , Config(config) , CloudId(cloudId) , DbId(dbId) , FolderId(folderId) -{ - Counters.Populate(counters); -} - -void TUsersInfoStorage::Init(TActorId tabletActor, TActorId partitionActor) { - Y_VERIFY(!TabletActor); - Y_VERIFY(!PartitionActor); - TabletActor = tabletActor; - PartitionActor = partitionActor; - - for (auto& userInfoPair : UsersInfo) { - auto& userInfo = userInfoPair.second; - Y_VERIFY(!userInfo.ReadSpeedLimiter); - userInfo.ReadSpeedLimiter = CreateReadSpeedLimiter(userInfo.User); - } -} - -void TUsersInfoStorage::ParseDeprecated(const TString& key, const TString& data, const TActorContext& ctx) { - Y_VERIFY(key.size() >= TKeyPrefix::MarkedSize()); - Y_VERIFY(key[TKeyPrefix::MarkPosition()] == TKeyPrefix::MarkUserDeprecated); - TString user = key.substr(TKeyPrefix::MarkedSize()); - +{ + Counters.Populate(counters); +} + +void TUsersInfoStorage::Init(TActorId tabletActor, TActorId partitionActor) { + Y_VERIFY(!TabletActor); + Y_VERIFY(!PartitionActor); + TabletActor = tabletActor; + PartitionActor = partitionActor; + + for (auto& userInfoPair : UsersInfo) { + auto& userInfo = userInfoPair.second; + Y_VERIFY(!userInfo.ReadSpeedLimiter); + userInfo.ReadSpeedLimiter = CreateReadSpeedLimiter(userInfo.User); + } +} + +void TUsersInfoStorage::ParseDeprecated(const TString& key, const TString& data, const TActorContext& ctx) { + Y_VERIFY(key.size() >= TKeyPrefix::MarkedSize()); + Y_VERIFY(key[TKeyPrefix::MarkPosition()] == TKeyPrefix::MarkUserDeprecated); + TString user = key.substr(TKeyPrefix::MarkedSize()); + TUserInfo* userInfo = GetIfExists(user); if (userInfo && userInfo->Parsed) { return; } - ui64 offset = 0; - ui32 gen = 0; - ui32 step = 0; - TString session; - NDeprecatedUserData::Parse(data, offset, gen, step, session); - Y_VERIFY(offset <= (ui64)Max<i64>(), "Offset is too big: %" PRIu64, offset); - - if (!userInfo) { + ui64 offset = 0; + ui32 gen = 0; + ui32 step = 0; + TString session; + NDeprecatedUserData::Parse(data, offset, gen, step, session); + Y_VERIFY(offset <= (ui64)Max<i64>(), "Offset is too big: %" PRIu64, offset); + + if (!userInfo) { Create(ctx, user, 0, false, session, gen, step, static_cast<i64>(offset), 0, TInstant::Zero()); - } else { - userInfo->Session = session; - userInfo->Generation = gen; - userInfo->Step = step; - userInfo->Offset = static_cast<i64>(offset); - } -} - -void TUsersInfoStorage::Parse(const TString& key, const TString& data, const TActorContext& ctx) { - Y_VERIFY(key.size() >= TKeyPrefix::MarkedSize()); - Y_VERIFY(key[TKeyPrefix::MarkPosition()] == TKeyPrefix::MarkUser); - TString user = key.substr(TKeyPrefix::MarkedSize()); - - Y_VERIFY(sizeof(ui64) <= data.size()); - - NKikimrPQ::TUserInfo userData; - bool res = userData.ParseFromString(data); - Y_VERIFY(res); - - Y_VERIFY(userData.GetOffset() <= (ui64)Max<i64>(), "Offset is too big: %" PRIu64, userData.GetOffset()); - i64 offset = static_cast<i64>(userData.GetOffset()); - - TUserInfo* userInfo = GetIfExists(user); - if (!userInfo) { - Create( + } else { + userInfo->Session = session; + userInfo->Generation = gen; + userInfo->Step = step; + userInfo->Offset = static_cast<i64>(offset); + } +} + +void TUsersInfoStorage::Parse(const TString& key, const TString& data, const TActorContext& ctx) { + Y_VERIFY(key.size() >= TKeyPrefix::MarkedSize()); + Y_VERIFY(key[TKeyPrefix::MarkPosition()] == TKeyPrefix::MarkUser); + TString user = key.substr(TKeyPrefix::MarkedSize()); + + Y_VERIFY(sizeof(ui64) <= data.size()); + + NKikimrPQ::TUserInfo userData; + bool res = userData.ParseFromString(data); + Y_VERIFY(res); + + Y_VERIFY(userData.GetOffset() <= (ui64)Max<i64>(), "Offset is too big: %" PRIu64, userData.GetOffset()); + i64 offset = static_cast<i64>(userData.GetOffset()); + + TUserInfo* userInfo = GetIfExists(user); + if (!userInfo) { + Create( ctx, user, userData.GetReadRuleGeneration(), false, userData.GetSession(), - userData.GetGeneration(), userData.GetStep(), offset, userData.GetOffsetRewindSum(), TInstant::Zero() - ); - } else { - userInfo->Session = userData.GetSession(); - userInfo->Generation = userData.GetGeneration(); - userInfo->Step = userData.GetStep(); - userInfo->Offset = offset; - userInfo->ReadOffsetRewindSum = userData.GetOffsetRewindSum(); + userData.GetGeneration(), userData.GetStep(), offset, userData.GetOffsetRewindSum(), TInstant::Zero() + ); + } else { + userInfo->Session = userData.GetSession(); + userInfo->Generation = userData.GetGeneration(); + userInfo->Step = userData.GetStep(); + userInfo->Offset = offset; + userInfo->ReadOffsetRewindSum = userData.GetOffsetRewindSum(); userInfo->ReadRuleGeneration = userData.GetReadRuleGeneration(); - } + } userInfo = GetIfExists(user); Y_VERIFY(userInfo); userInfo->Parsed = true; -} - +} + void TUsersInfoStorage::Remove(const TString& user, const TActorContext& ctx) { auto it = UsersInfo.find(user); Y_VERIFY(it != UsersInfo.end()); @@ -131,76 +131,76 @@ void TUsersInfoStorage::Remove(const TString& user, const TActorContext& ctx) { UsersInfo.erase(it); } -TUserInfo& TUsersInfoStorage::GetOrCreate(const TString& user, const TActorContext& ctx) { - Y_VERIFY(!user.empty()); - auto it = UsersInfo.find(user); - if (it == UsersInfo.end()) { +TUserInfo& TUsersInfoStorage::GetOrCreate(const TString& user, const TActorContext& ctx) { + Y_VERIFY(!user.empty()); + auto it = UsersInfo.find(user); + if (it == UsersInfo.end()) { return Create(ctx, user, 0, false, "", 0, 0, 0, 0, TInstant::Zero()); - } - return it->second; -} - -TUserInfo* TUsersInfoStorage::GetIfExists(const TString& user) { - auto it = UsersInfo.find(user); - return it != UsersInfo.end() ? &it->second : nullptr; -} - -THashMap<TString, TUserInfo>& TUsersInfoStorage::GetAll() { - return UsersInfo; -} - -TUserInfo& TUsersInfoStorage::Create( + } + return it->second; +} + +TUserInfo* TUsersInfoStorage::GetIfExists(const TString& user) { + auto it = UsersInfo.find(user); + return it != UsersInfo.end() ? &it->second : nullptr; +} + +THashMap<TString, TUserInfo>& TUsersInfoStorage::GetAll() { + return UsersInfo; +} + +TUserInfo& TUsersInfoStorage::Create( const TActorContext& ctx, const TString& user, const ui64 readRuleGeneration, bool important, const TString& session, - ui32 gen, ui32 step, i64 offset, ui64 readOffsetRewindSum, TInstant readFromTimestamp -) { + ui32 gen, ui32 step, i64 offset, ui64 readOffsetRewindSum, TInstant readFromTimestamp +) { ui64 burst = 1'000'000'000, speed = 1'000'000'000; if (AppData(ctx)->PQConfig.GetQuotingConfig().GetPartitionReadQuotaIsTwiceWriteQuota()) { burst = Config.GetPartitionConfig().GetBurstSize() * 2; speed = Config.GetPartitionConfig().GetWriteSpeedInBytesPerSecond() * 2; } - auto result = UsersInfo.emplace( - std::piecewise_construct, - std::forward_as_tuple(user), + auto result = UsersInfo.emplace( + std::piecewise_construct, + std::forward_as_tuple(user), std::forward_as_tuple(ctx, CreateReadSpeedLimiter(user), user, readRuleGeneration, important, TopicName, Partition, session, gen, step, offset, readOffsetRewindSum, DCId, readFromTimestamp, CloudId, DbId, FolderId, burst, speed) - ); - Y_VERIFY(result.second); - return result.first->second; -} - -void TUsersInfoStorage::Clear(const TActorContext& ctx) { - for (auto& userInfoPair : UsersInfo) { + ); + Y_VERIFY(result.second); + return result.first->second; +} + +void TUsersInfoStorage::Clear(const TActorContext& ctx) { + for (auto& userInfoPair : UsersInfo) { userInfoPair.second.Clear(ctx); - } + } UsersInfo.clear(); -} - +} + void TUserInfo::Clear(const TActorContext& ctx) { if (ReadSpeedLimiter) { ctx.Send(ReadSpeedLimiter->Actor, new TEvents::TEvPoisonPill()); } } -THolder<TReadSpeedLimiterHolder> TUsersInfoStorage::CreateReadSpeedLimiter(const TString& user) const { - const auto& quotingConfig = AppData()->PQConfig.GetQuotingConfig(); - if (TabletActor && quotingConfig.GetEnableQuoting() && quotingConfig.GetEnableReadQuoting()) { - TActorId actorId = TActivationContext::Register( - new TReadSpeedLimiter( - TabletActor.GetRef(), - PartitionActor.GetRef(), - TabletId, - TopicName, - Partition, - user, - Counters - ), - PartitionActor.GetRef() - ); - return MakeHolder<TReadSpeedLimiterHolder>(actorId, Counters); - } - return nullptr; -} - -} //NPQ -} //NKikimr +THolder<TReadSpeedLimiterHolder> TUsersInfoStorage::CreateReadSpeedLimiter(const TString& user) const { + const auto& quotingConfig = AppData()->PQConfig.GetQuotingConfig(); + if (TabletActor && quotingConfig.GetEnableQuoting() && quotingConfig.GetEnableReadQuoting()) { + TActorId actorId = TActivationContext::Register( + new TReadSpeedLimiter( + TabletActor.GetRef(), + PartitionActor.GetRef(), + TabletId, + TopicName, + Partition, + user, + Counters + ), + PartitionActor.GetRef() + ); + return MakeHolder<TReadSpeedLimiterHolder>(actorId, Counters); + } + return nullptr; +} + +} //NPQ +} //NKikimr diff --git a/ydb/core/persqueue/user_info.h b/ydb/core/persqueue/user_info.h index 04b6d4d443..258e947c46 100644 --- a/ydb/core/persqueue/user_info.h +++ b/ydb/core/persqueue/user_info.h @@ -3,8 +3,8 @@ #include "working_time_counter.h" #include "subscriber.h" #include "percentile_counter.h" -#include "read_speed_limiter.h" - +#include "read_speed_limiter.h" + #include <ydb/core/base/counters.h> #include <ydb/core/protos/counters_pq.pb.h> #include <ydb/core/protos/pqconfig.pb.h> @@ -13,17 +13,17 @@ #include <library/cpp/sliding_window/sliding_window.h> -#include <util/generic/set.h> - +#include <util/generic/set.h> + namespace NKikimr { namespace NPQ { -namespace NDeprecatedUserData { - // [offset:64bit][generation:32bit][step:32bit][session:other bytes] - TBuffer Serialize(ui64 offset, ui32 gen, ui32 step, const TString& session); - void Parse(const TString& data, ui64& offset, ui32& gen, ui32& step, TString& session); -} // NDeprecatedUserInfo - +namespace NDeprecatedUserData { + // [offset:64bit][generation:32bit][step:32bit][session:other bytes] + TBuffer Serialize(ui64 offset, ui32 gen, ui32 step, const TString& session); + void Parse(const TString& data, ui64& offset, ui32& gen, ui32& step, TString& session); +} // NDeprecatedUserInfo + static const ui32 MAX_USER_TS_CACHE_SIZE = 10'000; static const ui64 MIN_TIMESTAMP_MS = 1'000'000'000'000ll; // around 2002 year static const TString CLIENTID_TO_READ_INTERNALLY = "$without_consumer"; @@ -137,42 +137,42 @@ private: ui64 QuotedTime; }; -struct TReadSpeedLimiterHolder { - TReadSpeedLimiterHolder(const TActorId& actor, const TTabletCountersBase& baseline) - : Actor(actor) - { - Baseline.Populate(baseline); - } - - TActorId Actor; - TTabletCountersBase Baseline; -}; - +struct TReadSpeedLimiterHolder { + TReadSpeedLimiterHolder(const TActorId& actor, const TTabletCountersBase& baseline) + : Actor(actor) + { + Baseline.Populate(baseline); + } + + TActorId Actor; + TTabletCountersBase Baseline; +}; + struct TUserInfo { - THolder<TReadSpeedLimiterHolder> ReadSpeedLimiter; - + THolder<TReadSpeedLimiterHolder> ReadSpeedLimiter; + TString Session = ""; ui32 Generation = 0; ui32 Step = 0; i64 Offset = 0; - TInstant WriteTimestamp; - TInstant CreateTimestamp; - TInstant ReadTimestamp; + TInstant WriteTimestamp; + TInstant CreateTimestamp; + TInstant ReadTimestamp; bool ActualTimestamps = false; i64 ReadOffset = -1; - TInstant ReadWriteTimestamp; - TInstant ReadCreateTimestamp; + TInstant ReadWriteTimestamp; + TInstant ReadCreateTimestamp; ui64 ReadOffsetRewindSum = 0; bool ReadScheduled = false; //cache is used for storing WriteTime;CreateTime for offsets. //When client will commit to new position, timestamps for this offset could be in cache - not insane client should read data before commit - std::deque<std::pair<ui64, std::pair<TInstant, TInstant>>> Cache; + std::deque<std::pair<ui64, std::pair<TInstant, TInstant>>> Cache; bool Important = false; - TInstant ReadFromTimestamp; + TInstant ReadFromTimestamp; bool HasReadRule = false; TUserLabeledCounters LabeledCounters; TString User; @@ -209,23 +209,23 @@ struct TUserInfo { const ui64 readRuleGeneration, bool important, const TString& topic, const ui32 partition, bool doExternalRead, ui64 burst = 1'000'000'000, ui64 speed = 1'000'000'000) - : ReadSpeedLimiter(std::move(readSpeedLimiter)) - , Important(important) - , LabeledCounters(user + "/" + (important ? "1" : "0") + "/" + topic, partition) - , User(user) + : ReadSpeedLimiter(std::move(readSpeedLimiter)) + , Important(important) + , LabeledCounters(user + "/" + (important ? "1" : "0") + "/" + topic, partition) + , User(user) , ReadRuleGeneration(readRuleGeneration) - , Topic(topic) + , Topic(topic) , ReadQuota(burst, speed, TAppData::TimeProvider->Now()) - , Counter(nullptr) - , BytesRead() - , MsgsRead() - , ActiveReads(0) - , Subscriptions(0) - , EndOffset(0) - , WriteLagMs(TDuration::Minutes(1), 100) + , Counter(nullptr) + , BytesRead() + , MsgsRead() + , ActiveReads(0) + , Subscriptions(0) + , EndOffset(0) + , WriteLagMs(TDuration::Minutes(1), 100) , DoExternalRead(doExternalRead) - { - } + { + } void ForgetSubscription(const TInstant& now) { if(Subscriptions > 0) @@ -237,7 +237,7 @@ struct TUserInfo { Counter.UpdateState(Subscriptions > 0 || ActiveReads > 0 || ReadRequests.size() > 0); //no data for read or got read requests from client } - void UpdateReadingTimeAndState(TInstant now) { + void UpdateReadingTimeAndState(TInstant now) { Counter.UpdateWorkingTime(now); UpdateReadingState(); @@ -245,7 +245,7 @@ struct TUserInfo { WriteLagMs.Update(0, now); } if (Subscriptions > 0) { - ReadTimestamp = now; + ReadTimestamp = now; } } @@ -282,32 +282,32 @@ struct TUserInfo { Y_VERIFY(ActiveReads > 0); --ActiveReads; UpdateReadingTimeAndState(now); - ReadTimestamp = now; + ReadTimestamp = now; } - TUserInfo( + TUserInfo( const TActorContext& ctx, THolder<TReadSpeedLimiterHolder> readSpeedLimiter, const TString& user, const ui64 readRuleGeneration, const bool important, const TString& topic, const ui32 partition, const TString &session, ui32 gen, ui32 step, i64 offset, const ui64 readOffsetRewindSum, const TString& dcId, TInstant readFromTimestamp, const TString& cloudId, const TString& dbId, const TString& folderId, ui64 burst = 1'000'000'000, ui64 speed = 1'000'000'000 - ) - : ReadSpeedLimiter(std::move(readSpeedLimiter)) - , Session(session) + ) + : ReadSpeedLimiter(std::move(readSpeedLimiter)) + , Session(session) , Generation(gen) , Step(step) , Offset(offset) - , WriteTimestamp(TAppData::TimeProvider->Now()) - , CreateTimestamp(TAppData::TimeProvider->Now()) - , ReadTimestamp(TAppData::TimeProvider->Now()) + , WriteTimestamp(TAppData::TimeProvider->Now()) + , CreateTimestamp(TAppData::TimeProvider->Now()) + , ReadTimestamp(TAppData::TimeProvider->Now()) , ActualTimestamps(false) , ReadOffset(-1) - , ReadWriteTimestamp(TAppData::TimeProvider->Now()) - , ReadCreateTimestamp(TAppData::TimeProvider->Now()) + , ReadWriteTimestamp(TAppData::TimeProvider->Now()) + , ReadCreateTimestamp(TAppData::TimeProvider->Now()) , ReadOffsetRewindSum(readOffsetRewindSum) , ReadScheduled(false) , Important(important) - , ReadFromTimestamp(readFromTimestamp) + , ReadFromTimestamp(readFromTimestamp) , HasReadRule(false) , LabeledCounters(user + "/" +(important ? "1" : "0") + "/" + topic, partition) , User(user) @@ -415,35 +415,35 @@ struct TUserInfo { void Clear(const TActorContext& ctx); - void UpdateReadOffset(const i64 offset, TInstant writeTimestamp, TInstant createTimestamp, TInstant now) { + void UpdateReadOffset(const i64 offset, TInstant writeTimestamp, TInstant createTimestamp, TInstant now) { ReadOffset = offset; - ReadWriteTimestamp = writeTimestamp; - ReadCreateTimestamp = createTimestamp; - WriteLagMs.Update((ReadWriteTimestamp - ReadCreateTimestamp).MilliSeconds(), ReadWriteTimestamp); + ReadWriteTimestamp = writeTimestamp; + ReadCreateTimestamp = createTimestamp; + WriteLagMs.Update((ReadWriteTimestamp - ReadCreateTimestamp).MilliSeconds(), ReadWriteTimestamp); if (Subscriptions > 0) { - ReadTimestamp = now; + ReadTimestamp = now; } } - void AddTimestampToCache(const ui64 offset, TInstant writeTimestamp, TInstant createTimestamp, bool isUserRead, TInstant now) + void AddTimestampToCache(const ui64 offset, TInstant writeTimestamp, TInstant createTimestamp, bool isUserRead, TInstant now) { if ((ui64)Max<i64>(Offset, 0) == offset) { - WriteTimestamp = writeTimestamp; - CreateTimestamp = createTimestamp; + WriteTimestamp = writeTimestamp; + CreateTimestamp = createTimestamp; ActualTimestamps = true; if (ReadOffset == -1) { - UpdateReadOffset(offset, writeTimestamp, createTimestamp, now); + UpdateReadOffset(offset, writeTimestamp, createTimestamp, now); } } if (isUserRead) { - UpdateReadOffset(offset, writeTimestamp, createTimestamp, now); + UpdateReadOffset(offset, writeTimestamp, createTimestamp, now); if (ReadTimeLag) { - ReadTimeLag->IncFor((now - createTimestamp).MilliSeconds(), 1); + ReadTimeLag->IncFor((now - createTimestamp).MilliSeconds(), 1); } } if (!Cache.empty() && Cache.back().first >= offset) //already got data in cache return; - Cache.push_back(std::make_pair(offset, std::make_pair(writeTimestamp, createTimestamp))); + Cache.push_back(std::make_pair(offset, std::make_pair(writeTimestamp, createTimestamp))); if (Cache.size() > MAX_USER_TS_CACHE_SIZE) Cache.pop_front(); } @@ -454,11 +454,11 @@ struct TUserInfo { Cache.pop_front(); } if (!Cache.empty() && Cache.front().first == (ui64)Max<i64>(Offset, 0)) { - WriteTimestamp = Cache.front().second.first; - CreateTimestamp = Cache.front().second.second; + WriteTimestamp = Cache.front().second.first; + CreateTimestamp = Cache.front().second.second; ActualTimestamps = true; if (ReadOffset == -1) { - UpdateReadOffset(Offset - 1, Cache.front().second.first, Cache.front().second.second, TAppData::TimeProvider->Now()); + UpdateReadOffset(Offset - 1, Cache.front().second.first, Cache.front().second.second, TAppData::TimeProvider->Now()); } return true; } @@ -475,21 +475,21 @@ struct TUserInfo { return ReadOffset == -1 ? Offset : (ReadOffset + 1); //+1 because we want to track first not readed offset } - TInstant GetReadTimestamp() const { + TInstant GetReadTimestamp() const { return ReadTimestamp; } - TInstant GetWriteTimestamp() const { - return Offset == EndOffset ? TAppData::TimeProvider->Now() : WriteTimestamp; + TInstant GetWriteTimestamp() const { + return Offset == EndOffset ? TAppData::TimeProvider->Now() : WriteTimestamp; } - TInstant GetCreateTimestamp() const { - return Offset == EndOffset ? TAppData::TimeProvider->Now() : CreateTimestamp; + TInstant GetCreateTimestamp() const { + return Offset == EndOffset ? TAppData::TimeProvider->Now() : CreateTimestamp; } - TInstant GetReadWriteTimestamp() const { - TInstant ts = ReadOffset == -1 ? WriteTimestamp : ReadWriteTimestamp; - ts = GetReadOffset() >= EndOffset ? TAppData::TimeProvider->Now() : ts; + TInstant GetReadWriteTimestamp() const { + TInstant ts = ReadOffset == -1 ? WriteTimestamp : ReadWriteTimestamp; + ts = GetReadOffset() >= EndOffset ? TAppData::TimeProvider->Now() : ts; return ts; } @@ -497,63 +497,63 @@ struct TUserInfo { return WriteLagMs.GetValue(); } - TInstant GetReadCreateTimestamp() const { - TInstant ts = ReadOffset == -1 ? CreateTimestamp : ReadCreateTimestamp; - ts = GetReadOffset() >= EndOffset ? TAppData::TimeProvider->Now() : ts; + TInstant GetReadCreateTimestamp() const { + TInstant ts = ReadOffset == -1 ? CreateTimestamp : ReadCreateTimestamp; + ts = GetReadOffset() >= EndOffset ? TAppData::TimeProvider->Now() : ts; return ts; } }; -class TUsersInfoStorage { -public: +class TUsersInfoStorage { +public: TUsersInfoStorage(TString dcId, ui64 tabletId, const TString& topicName, ui32 partition, const TTabletCountersBase& counters, const NKikimrPQ::TPQTabletConfig& config, const TString& CloudId, const TString& DbId, const TString& FolderId); - - void Init(TActorId tabletActor, TActorId partitionActor); - - void ParseDeprecated(const TString& key, const TString& data, const TActorContext& ctx); - void Parse(const TString& key, const TString& data, const TActorContext& ctx); - - TUserInfo& GetOrCreate(const TString& user, const TActorContext& ctx); - TUserInfo* GetIfExists(const TString& user); - + + void Init(TActorId tabletActor, TActorId partitionActor); + + void ParseDeprecated(const TString& key, const TString& data, const TActorContext& ctx); + void Parse(const TString& key, const TString& data, const TActorContext& ctx); + + TUserInfo& GetOrCreate(const TString& user, const TActorContext& ctx); + TUserInfo* GetIfExists(const TString& user); + void UpdateConfig(const NKikimrPQ::TPQTabletConfig& config) { Config = config; } - THashMap<TString, TUserInfo>& GetAll(); - - TUserInfo& Create( + THashMap<TString, TUserInfo>& GetAll(); + + TUserInfo& Create( const TActorContext& ctx, const TString& user, const ui64 readRuleGeneration, bool important, const TString &session, - ui32 gen, ui32 step, i64 offset, ui64 readOffsetRewindSum, TInstant readFromTimestamp - ); + ui32 gen, ui32 step, i64 offset, ui64 readOffsetRewindSum, TInstant readFromTimestamp + ); + + void Clear(const TActorContext& ctx); - void Clear(const TActorContext& ctx); - void Remove(const TString& user, const TActorContext& ctx); -private: - THolder<TReadSpeedLimiterHolder> CreateReadSpeedLimiter(const TString& user) const; - -private: - THashMap<TString, TUserInfo> UsersInfo; - +private: + THolder<TReadSpeedLimiterHolder> CreateReadSpeedLimiter(const TString& user) const; + +private: + THashMap<TString, TUserInfo> UsersInfo; + const TString DCId; - ui64 TabletId; - const TString TopicName; - const ui32 Partition; - TTabletCountersBase Counters; - - TMaybe<TActorId> TabletActor; - TMaybe<TActorId> PartitionActor; + ui64 TabletId; + const TString TopicName; + const ui32 Partition; + TTabletCountersBase Counters; + + TMaybe<TActorId> TabletActor; + TMaybe<TActorId> PartitionActor; NKikimrPQ::TPQTabletConfig Config; TString CloudId; TString DbId; TString FolderId; -}; - +}; + } //NPQ } //NKikimr diff --git a/ydb/core/persqueue/user_info_ut.cpp b/ydb/core/persqueue/user_info_ut.cpp index c48148a21a..8f8ec0aa58 100644 --- a/ydb/core/persqueue/user_info_ut.cpp +++ b/ydb/core/persqueue/user_info_ut.cpp @@ -1,35 +1,35 @@ -#include "user_info.h" - -#include <library/cpp/testing/unittest/registar.h> - -namespace NKikimr::NPQ { - - Y_UNIT_TEST_SUITE(TPQUserInfoTest) { - Y_UNIT_TEST(UserDataDeprecatedSerializaion) { - ui64 offset = 1234556789012345678ULL; - ui32 gen = 987654321; - ui32 step = 1234567890; - const TString session = "some_source_id"; - TBuffer buffer = NDeprecatedUserData::Serialize(offset, gen, step, session); - TString data(buffer.data(), buffer.size()); - { - UNIT_ASSERT_EQUAL(data.size(), sizeof(offset) + sizeof(gen) + sizeof(step) + session.size()); - UNIT_ASSERT_EQUAL(offset, *reinterpret_cast<const ui64*>(data.c_str())); - UNIT_ASSERT_EQUAL(gen, reinterpret_cast<const ui32*>(data.c_str() + sizeof(ui64))[0]); - UNIT_ASSERT_EQUAL(step, reinterpret_cast<const ui32*>(data.c_str() + sizeof(ui64))[1]); - UNIT_ASSERT_EQUAL(session, data.substr(sizeof(ui64) + 2 * sizeof(ui32))); - } - { - ui64 parsedOffset = 0; - ui32 parsedGen = 0; - ui32 parsedStep = 0; - TString parsedSession; - NDeprecatedUserData::Parse(data, parsedOffset, parsedGen, parsedStep, parsedSession); - UNIT_ASSERT_EQUAL(offset, parsedOffset); - UNIT_ASSERT_EQUAL(gen, parsedGen); - UNIT_ASSERT_EQUAL(step, parsedStep); - UNIT_ASSERT_EQUAL(session, parsedSession); - } - } - } -} +#include "user_info.h" + +#include <library/cpp/testing/unittest/registar.h> + +namespace NKikimr::NPQ { + + Y_UNIT_TEST_SUITE(TPQUserInfoTest) { + Y_UNIT_TEST(UserDataDeprecatedSerializaion) { + ui64 offset = 1234556789012345678ULL; + ui32 gen = 987654321; + ui32 step = 1234567890; + const TString session = "some_source_id"; + TBuffer buffer = NDeprecatedUserData::Serialize(offset, gen, step, session); + TString data(buffer.data(), buffer.size()); + { + UNIT_ASSERT_EQUAL(data.size(), sizeof(offset) + sizeof(gen) + sizeof(step) + session.size()); + UNIT_ASSERT_EQUAL(offset, *reinterpret_cast<const ui64*>(data.c_str())); + UNIT_ASSERT_EQUAL(gen, reinterpret_cast<const ui32*>(data.c_str() + sizeof(ui64))[0]); + UNIT_ASSERT_EQUAL(step, reinterpret_cast<const ui32*>(data.c_str() + sizeof(ui64))[1]); + UNIT_ASSERT_EQUAL(session, data.substr(sizeof(ui64) + 2 * sizeof(ui32))); + } + { + ui64 parsedOffset = 0; + ui32 parsedGen = 0; + ui32 parsedStep = 0; + TString parsedSession; + NDeprecatedUserData::Parse(data, parsedOffset, parsedGen, parsedStep, parsedSession); + UNIT_ASSERT_EQUAL(offset, parsedOffset); + UNIT_ASSERT_EQUAL(gen, parsedGen); + UNIT_ASSERT_EQUAL(step, parsedStep); + UNIT_ASSERT_EQUAL(session, parsedSession); + } + } + } +} diff --git a/ydb/core/persqueue/ut/ya.make b/ydb/core/persqueue/ut/ya.make index 39578c05aa..7cbb0b9496 100644 --- a/ydb/core/persqueue/ut/ya.make +++ b/ydb/core/persqueue/ut/ya.make @@ -1,10 +1,10 @@ UNITTEST_FOR(ydb/core/persqueue) -OWNER( - alexnick +OWNER( + alexnick g:kikimr - g:logbroker -) + g:logbroker +) FORK_SUBTESTS() @@ -31,12 +31,12 @@ YQL_LAST_ABI_VERSION() SRCS( internals_ut.cpp - mirrorer_ut.cpp + mirrorer_ut.cpp pq_ut.cpp type_codecs_ut.cpp pq_ut.h sourceid_ut.cpp - user_info_ut.cpp + user_info_ut.cpp ) END() diff --git a/ydb/core/persqueue/ut_slow/ya.make b/ydb/core/persqueue/ut_slow/ya.make index c7fd14c5eb..9e018d205b 100644 --- a/ydb/core/persqueue/ut_slow/ya.make +++ b/ydb/core/persqueue/ut_slow/ya.make @@ -1,10 +1,10 @@ UNITTEST_FOR(ydb/core/persqueue) -OWNER( - alexnick +OWNER( + alexnick g:kikimr - g:logbroker -) + g:logbroker +) FORK_SUBTESTS() diff --git a/ydb/core/persqueue/working_time_counter.h b/ydb/core/persqueue/working_time_counter.h index 5e0cfb1481..4f4d81d263 100644 --- a/ydb/core/persqueue/working_time_counter.h +++ b/ydb/core/persqueue/working_time_counter.h @@ -2,8 +2,8 @@ #include <ydb/core/protos/counters_pq.pb.h> -#include <library/cpp/monlib/dynamic_counters/counters.h> - +#include <library/cpp/monlib/dynamic_counters/counters.h> + namespace NKikimr { namespace NPQ { @@ -11,26 +11,26 @@ namespace NPQ { class TWorkingTimeCounter { private: bool IsInWorkingState; - NMonitoring::TDynamicCounters::TCounterPtr WorkingTimeMicroSec; + NMonitoring::TDynamicCounters::TCounterPtr WorkingTimeMicroSec; TInstant LastUpdateTimestamp; public: TWorkingTimeCounter(NMonitoring::TDynamicCounters::TCounterPtr counter) : IsInWorkingState(false) - , WorkingTimeMicroSec(counter) + , WorkingTimeMicroSec(counter) {} void UpdateState(bool state) { IsInWorkingState = state; } - void UpdateWorkingTime(const TInstant now) { - if (!WorkingTimeMicroSec) //no counter + void UpdateWorkingTime(const TInstant now) { + if (!WorkingTimeMicroSec) //no counter return; if (IsInWorkingState && LastUpdateTimestamp > TInstant::Zero()) { - TDuration res = now - LastUpdateTimestamp; - (*WorkingTimeMicroSec) += res.MicroSeconds(); - LastUpdateTimestamp += res; + TDuration res = now - LastUpdateTimestamp; + (*WorkingTimeMicroSec) += res.MicroSeconds(); + LastUpdateTimestamp += res; } else { LastUpdateTimestamp = now; } diff --git a/ydb/core/persqueue/write_meta.cpp b/ydb/core/persqueue/write_meta.cpp index 26d090260c..58bed2badf 100644 --- a/ydb/core/persqueue/write_meta.cpp +++ b/ydb/core/persqueue/write_meta.cpp @@ -1,53 +1,53 @@ -#include "write_meta.h" - +#include "write_meta.h" + #include <ydb/core/persqueue/codecs/pqv1.h> - - -namespace NKikimr { - - -void SetMetaField(NKikimrPQClient::TDataChunk& proto, const TString& key, const TString& value) { - if (key == "server") { - proto.MutableMeta()->SetServer(value); - } else if (key == "ident") { - proto.MutableMeta()->SetIdent(value); - } else if (key == "logtype") { - proto.MutableMeta()->SetLogType(value); - } else if (key == "file") { - proto.MutableMeta()->SetFile(value); - } else { - auto res = proto.MutableExtraFields()->AddItems(); - res->SetKey(key); - res->SetValue(value); - } -} - -TString GetSerializedData(const NYdb::NPersQueue::TReadSessionEvent::TDataReceivedEvent::TCompressedMessage& message) { - NKikimrPQClient::TDataChunk proto; - for (const auto& item : message.GetMeta(0)->Fields) { - SetMetaField(proto, item.first, item.second); - } - proto.SetIp(message.GetIp(0)); - proto.SetSeqNo(message.GetSeqNo(0)); - proto.SetCreateTime(message.GetCreateTime(0).MilliSeconds()); - auto codec = NPQ::FromV1Codec(message.GetCodec()); - Y_VERIFY(codec); - proto.SetCodec(codec.value()); - proto.SetData(message.GetData()); - - TString str; - bool res = proto.SerializeToString(&str); - Y_VERIFY(res); - return str; -} - -NKikimrPQClient::TDataChunk GetDeserializedData(const TString& string) { - NKikimrPQClient::TDataChunk proto; - bool res = proto.ParseFromString(string); - Y_UNUSED(res); - //TODO: check errors of parsing - return proto; -} - -} // NKikimr - + + +namespace NKikimr { + + +void SetMetaField(NKikimrPQClient::TDataChunk& proto, const TString& key, const TString& value) { + if (key == "server") { + proto.MutableMeta()->SetServer(value); + } else if (key == "ident") { + proto.MutableMeta()->SetIdent(value); + } else if (key == "logtype") { + proto.MutableMeta()->SetLogType(value); + } else if (key == "file") { + proto.MutableMeta()->SetFile(value); + } else { + auto res = proto.MutableExtraFields()->AddItems(); + res->SetKey(key); + res->SetValue(value); + } +} + +TString GetSerializedData(const NYdb::NPersQueue::TReadSessionEvent::TDataReceivedEvent::TCompressedMessage& message) { + NKikimrPQClient::TDataChunk proto; + for (const auto& item : message.GetMeta(0)->Fields) { + SetMetaField(proto, item.first, item.second); + } + proto.SetIp(message.GetIp(0)); + proto.SetSeqNo(message.GetSeqNo(0)); + proto.SetCreateTime(message.GetCreateTime(0).MilliSeconds()); + auto codec = NPQ::FromV1Codec(message.GetCodec()); + Y_VERIFY(codec); + proto.SetCodec(codec.value()); + proto.SetData(message.GetData()); + + TString str; + bool res = proto.SerializeToString(&str); + Y_VERIFY(res); + return str; +} + +NKikimrPQClient::TDataChunk GetDeserializedData(const TString& string) { + NKikimrPQClient::TDataChunk proto; + bool res = proto.ParseFromString(string); + Y_UNUSED(res); + //TODO: check errors of parsing + return proto; +} + +} // NKikimr + diff --git a/ydb/core/persqueue/write_meta.h b/ydb/core/persqueue/write_meta.h index fe7c4f349e..0fba086633 100644 --- a/ydb/core/persqueue/write_meta.h +++ b/ydb/core/persqueue/write_meta.h @@ -1,73 +1,73 @@ -#pragma once - +#pragma once + #include <ydb/core/protos/grpc_pq_old.pb.h> #include <ydb/public/sdk/cpp/client/ydb_persqueue_core/persqueue.h> - -#include <util/string/vector.h> - -namespace NKikimr { - -template <class TWriteRequestInit> -void FillExtraFieldsForDataChunk( - const TWriteRequestInit& init, NKikimrPQClient::TDataChunk& data, TString& server, TString& ident, TString& logType, TString& file); - - -template <class TWriteRequestInit> -NKikimrPQClient::TDataChunk GetInitialDataChunk(const TWriteRequestInit& init, const TString& topic, const TString& peerName) { - NKikimrPQClient::TDataChunk data; - TString server, ident, logType, file; - - FillExtraFieldsForDataChunk(init, data, server, ident, logType, file); - - if (server.empty()) { - server = peerName; - } - if (ident.empty()) { - auto p = SplitString(topic, "--"); - if (p.size() == 3) - ident = p[1]; - else - ident = "unknown"; - } - if (logType.empty()) { - auto p = SplitString(topic, "--"); - if (p.size() == 3) - logType = p.back(); - else - logType = "unknown"; - } - - data.MutableMeta()->SetServer(server); - data.MutableMeta()->SetIdent(ident); - data.MutableMeta()->SetLogType(logType); - if (!file.empty()) - data.MutableMeta()->SetFile(file); - - data.SetIp(peerName); - return data; -} - -template <class TData> -void FillChunkDataFromReq(NKikimrPQClient::TDataChunk& proto, const TData& data, int); - -template <class TData> -void FillChunkDataFromReq(NKikimrPQClient::TDataChunk& proto, const TData& data); - -template <typename... TArgs> -TString GetSerializedData(const NKikimrPQClient::TDataChunk& init, TArgs&...args) { - NKikimrPQClient::TDataChunk proto; - proto.CopyFrom(init); - FillChunkDataFromReq(proto, std::forward<TArgs>(args)...); - - TString str; - bool res = proto.SerializeToString(&str); - Y_VERIFY(res); - return str; -} - -TString GetSerializedData(const NYdb::NPersQueue::TReadSessionEvent::TDataReceivedEvent::TCompressedMessage& message); - -NKikimrPQClient::TDataChunk GetDeserializedData(const TString& string); - -} // NKikimr - + +#include <util/string/vector.h> + +namespace NKikimr { + +template <class TWriteRequestInit> +void FillExtraFieldsForDataChunk( + const TWriteRequestInit& init, NKikimrPQClient::TDataChunk& data, TString& server, TString& ident, TString& logType, TString& file); + + +template <class TWriteRequestInit> +NKikimrPQClient::TDataChunk GetInitialDataChunk(const TWriteRequestInit& init, const TString& topic, const TString& peerName) { + NKikimrPQClient::TDataChunk data; + TString server, ident, logType, file; + + FillExtraFieldsForDataChunk(init, data, server, ident, logType, file); + + if (server.empty()) { + server = peerName; + } + if (ident.empty()) { + auto p = SplitString(topic, "--"); + if (p.size() == 3) + ident = p[1]; + else + ident = "unknown"; + } + if (logType.empty()) { + auto p = SplitString(topic, "--"); + if (p.size() == 3) + logType = p.back(); + else + logType = "unknown"; + } + + data.MutableMeta()->SetServer(server); + data.MutableMeta()->SetIdent(ident); + data.MutableMeta()->SetLogType(logType); + if (!file.empty()) + data.MutableMeta()->SetFile(file); + + data.SetIp(peerName); + return data; +} + +template <class TData> +void FillChunkDataFromReq(NKikimrPQClient::TDataChunk& proto, const TData& data, int); + +template <class TData> +void FillChunkDataFromReq(NKikimrPQClient::TDataChunk& proto, const TData& data); + +template <typename... TArgs> +TString GetSerializedData(const NKikimrPQClient::TDataChunk& init, TArgs&...args) { + NKikimrPQClient::TDataChunk proto; + proto.CopyFrom(init); + FillChunkDataFromReq(proto, std::forward<TArgs>(args)...); + + TString str; + bool res = proto.SerializeToString(&str); + Y_VERIFY(res); + return str; +} + +TString GetSerializedData(const NYdb::NPersQueue::TReadSessionEvent::TDataReceivedEvent::TCompressedMessage& message); + +NKikimrPQClient::TDataChunk GetDeserializedData(const TString& string); + +} // NKikimr + diff --git a/ydb/core/persqueue/writer/source_id_encoding.cpp b/ydb/core/persqueue/writer/source_id_encoding.cpp index 81ede360c9..13a37c2c61 100644 --- a/ydb/core/persqueue/writer/source_id_encoding.cpp +++ b/ydb/core/persqueue/writer/source_id_encoding.cpp @@ -1,14 +1,14 @@ -#include "source_id_encoding.h" - -#include <library/cpp/string_utils/base64/base64.h> - -#include <util/generic/yexception.h> -#include <util/string/strip.h> - -namespace NKikimr { +#include "source_id_encoding.h" + +#include <library/cpp/string_utils/base64/base64.h> + +#include <util/generic/yexception.h> +#include <util/string/strip.h> + +namespace NKikimr { namespace NPQ { namespace NSourceIdEncoding { - + struct TTags { static constexpr char Simple = 0; static constexpr char Base64 = 1; @@ -16,10 +16,10 @@ struct TTags { static constexpr TStringBuf Base64Prefix = "base64:"; -TString EncodeSimple(const TString& sourceId) { +TString EncodeSimple(const TString& sourceId) { return TString(1, TTags::Simple) + sourceId; -} - +} + TString DecodeSimple(const TString& sourceId) { Y_VERIFY(!sourceId.empty() && sourceId[0] == TTags::Simple); return sourceId.substr(1); @@ -49,14 +49,14 @@ TString DecodeBase64(const TString& sourceId) { return Base64Prefix + StripStringRight(Base64EncodeUrl(sourceId.substr(1)), EqualsStripAdapter(',')); } -TString Encode(const TString& sourceId) { +TString Encode(const TString& sourceId) { if (sourceId.StartsWith(Base64Prefix)) { return EncodeBase64(sourceId); - } else { + } else { return EncodeSimple(sourceId); - } -} - + } +} + TString Decode(const TString& sourceId) { Y_VERIFY(!sourceId.empty()); @@ -67,9 +67,9 @@ TString Decode(const TString& sourceId) { return DecodeBase64(sourceId); default: return sourceId; - } -} - + } +} + bool IsValidEncoded(const TString& sourceId) { if (sourceId.empty()) { return false; @@ -82,8 +82,8 @@ bool IsValidEncoded(const TString& sourceId) { default: return false; } -} - +} + } // NSourceIdEncoding } // NPQ } // NKikimr diff --git a/ydb/core/persqueue/writer/source_id_encoding.h b/ydb/core/persqueue/writer/source_id_encoding.h index bfd6727cfc..6472d8b4ac 100644 --- a/ydb/core/persqueue/writer/source_id_encoding.h +++ b/ydb/core/persqueue/writer/source_id_encoding.h @@ -1,18 +1,18 @@ -#pragma once - -#include <util/generic/fwd.h> - -namespace NKikimr { +#pragma once + +#include <util/generic/fwd.h> + +namespace NKikimr { namespace NPQ { namespace NSourceIdEncoding { - -TString EncodeSimple(const TString& sourceId); -TString Encode(const TString& sourceId); - -TString Decode(const TString& encodedSourceId); - -bool IsValidEncoded(const TString& encodedSourceId); - + +TString EncodeSimple(const TString& sourceId); +TString Encode(const TString& sourceId); + +TString Decode(const TString& encodedSourceId); + +bool IsValidEncoded(const TString& encodedSourceId); + } // NSourceIdEncoding } // NPQ } // NKikimr diff --git a/ydb/core/persqueue/ya.make b/ydb/core/persqueue/ya.make index 0adff6544e..50e89676ef 100644 --- a/ydb/core/persqueue/ya.make +++ b/ydb/core/persqueue/ya.make @@ -3,30 +3,30 @@ LIBRARY() OWNER( alexnick g:kikimr - g:logbroker + g:logbroker ) SRCS( cluster_tracker.cpp blob.cpp - event_helpers.cpp + event_helpers.cpp header.cpp percentile_counter.cpp pq.cpp pq_database.cpp pq_impl.cpp sourceid.cpp - mirrorer.cpp - mirrorer.h + mirrorer.cpp + mirrorer.h ownerinfo.cpp partition.cpp pq_l2_cache.cpp read_balancer.cpp - read_speed_limiter.cpp + read_speed_limiter.cpp subscriber.cpp type_codecs_defs.cpp - user_info.cpp - write_meta.cpp + user_info.cpp + write_meta.cpp actor_persqueue_client_iface.h ) diff --git a/ydb/core/protos/config.proto b/ydb/core/protos/config.proto index 1c5711a15d..d64169d4fc 100644 --- a/ydb/core/protos/config.proto +++ b/ydb/core/protos/config.proto @@ -839,8 +839,8 @@ message TSqsConfig { optional uint64 AddMessagesToInflyMinCheckAttempts = 63 [default = 10]; optional uint64 MinimumGarbageAgeSeconds = 64 [default = 3600]; - - optional bool MeteringByNetClassifierOnly = 65 [default = false]; + + optional bool MeteringByNetClassifierOnly = 65 [default = false]; message TYcSearchEventsConfig { optional bool EnableYcSearch = 1 [default = false]; diff --git a/ydb/core/protos/counters_pq.proto b/ydb/core/protos/counters_pq.proto index e2aa4c8d8c..7233758135 100644 --- a/ydb/core/protos/counters_pq.proto +++ b/ydb/core/protos/counters_pq.proto @@ -55,7 +55,7 @@ enum ECumulativeCounters { COUNTER_PQ_WRITE_SMALL_OFFSET = 34 [(CounterOpts) = {Name: "WriteSmallOffset"}]; COUNTER_PQ_TABLET_CPU_USAGE = 35 [(CounterOpts) = {Name: "CPUUsage"}]; - COUNTER_PQ_TABLET_NETWORK_BYTES_USAGE = 36 [(CounterOpts) = {Name: "NetworkUsage"}]; + COUNTER_PQ_TABLET_NETWORK_BYTES_USAGE = 36 [(CounterOpts) = {Name: "NetworkUsage"}]; } enum ESimpleCounters { diff --git a/ydb/core/protos/msgbus_pq.proto b/ydb/core/protos/msgbus_pq.proto index 2fc298d01e..849b292383 100644 --- a/ydb/core/protos/msgbus_pq.proto +++ b/ydb/core/protos/msgbus_pq.proto @@ -378,7 +378,7 @@ message TCmdReadResult { optional uint64 BlobsCachedSize = 8; optional uint64 SizeLag = 9; optional uint64 RealReadOffset = 10; - optional uint64 WaitQuotaTimeMs = 11; + optional uint64 WaitQuotaTimeMs = 11; } diff --git a/ydb/core/protos/netclassifier.proto b/ydb/core/protos/netclassifier.proto index 7f76031514..0c71eae7bf 100644 --- a/ydb/core/protos/netclassifier.proto +++ b/ydb/core/protos/netclassifier.proto @@ -12,7 +12,7 @@ message TNetClassifierUpdaterConfig { optional uint32 RetryIntervalSeconds = 2; optional uint32 NetDataUpdateIntervalSeconds = 3; optional EFormat Format = 4 [default = TSV]; - repeated string NetBoxTags = 5; + repeated string NetBoxTags = 5; } message TSubnet { diff --git a/ydb/core/protos/out/out.cpp b/ydb/core/protos/out/out.cpp index 2609cf635c..28c49a5c36 100644 --- a/ydb/core/protos/out/out.cpp +++ b/ydb/core/protos/out/out.cpp @@ -10,65 +10,65 @@ #include <ydb/core/protos/resource_broker.pb.h> #include <ydb/core/protos/tenant_pool.pb.h> #include <ydb/core/protos/tx_datashard.pb.h> - -#include <util/stream/output.h> - -Y_DECLARE_OUT_SPEC(, NKikimrBlobStorage::EPutHandleClass, stream, value) { - stream << NKikimrBlobStorage::EPutHandleClass_Name(value); -} - + +#include <util/stream/output.h> + +Y_DECLARE_OUT_SPEC(, NKikimrBlobStorage::EPutHandleClass, stream, value) { + stream << NKikimrBlobStorage::EPutHandleClass_Name(value); +} + Y_DECLARE_OUT_SPEC(, NKikimrBlobStorage::EGetHandleClass, stream, value) { stream << NKikimrBlobStorage::EGetHandleClass_Name(value); } -Y_DECLARE_OUT_SPEC(, NKikimrBlobStorage::TVDiskKind::EVDiskKind, stream, value) { - stream << NKikimrBlobStorage::TVDiskKind::EVDiskKind_Name(value); -} - -Y_DECLARE_OUT_SPEC(, NKikimrVDiskData::TSyncerVDiskEntry::ESyncStatus, stream, value) { - stream << NKikimrVDiskData::TSyncerVDiskEntry::ESyncStatus_Name(value); -} - -Y_DECLARE_OUT_SPEC(, NKikimrBlobStorage::TSyncGuidInfo::EState, stream, value) { - stream << NKikimrBlobStorage::TSyncGuidInfo::EState_Name(value); -} - -Y_DECLARE_OUT_SPEC(, NKikimrBlobStorage::TLocalGuidInfo::EState, stream, value) { - stream << NKikimrBlobStorage::TLocalGuidInfo::EState_Name(value); -} - -Y_DECLARE_OUT_SPEC(, NKikimrBlobStorage::TSyncerStatus::EPhase, stream, value) { - stream << NKikimrBlobStorage::TSyncerStatus::EPhase_Name(value); -} - -Y_DECLARE_OUT_SPEC(, NKikimrBlobStorage::EVDiskQueueId, stream, value) { - stream << NKikimrBlobStorage::EVDiskQueueId_Name(value); -} - -Y_DECLARE_OUT_SPEC(, NKikimrBlobStorage::EVDiskInternalQueueId, stream, value) { - stream << NKikimrBlobStorage::EVDiskInternalQueueId_Name(value); -} - -Y_DECLARE_OUT_SPEC(, NKikimrBlobStorage::ESyncFullStage, stream, value) { - stream << NKikimrBlobStorage::ESyncFullStage_Name(value); -} - -Y_DECLARE_OUT_SPEC(, NKikimrProto::EReplyStatus, stream, value) { - stream << NKikimrProto::EReplyStatus_Name(value); -} - -Y_DECLARE_OUT_SPEC(, NKikimrCms::TStatus::ECode, stream, value) { - stream << NKikimrCms::TStatus::ECode_Name(value); -} - -Y_DECLARE_OUT_SPEC(, NKikimrCms::EState, stream, value) { - stream << NKikimrCms::EState_Name(value); -} - -Y_DECLARE_OUT_SPEC(, NKikimrBlobStorage::EDriveStatus, stream, value) { - stream << NKikimrBlobStorage::EDriveStatus_Name(value); -} - +Y_DECLARE_OUT_SPEC(, NKikimrBlobStorage::TVDiskKind::EVDiskKind, stream, value) { + stream << NKikimrBlobStorage::TVDiskKind::EVDiskKind_Name(value); +} + +Y_DECLARE_OUT_SPEC(, NKikimrVDiskData::TSyncerVDiskEntry::ESyncStatus, stream, value) { + stream << NKikimrVDiskData::TSyncerVDiskEntry::ESyncStatus_Name(value); +} + +Y_DECLARE_OUT_SPEC(, NKikimrBlobStorage::TSyncGuidInfo::EState, stream, value) { + stream << NKikimrBlobStorage::TSyncGuidInfo::EState_Name(value); +} + +Y_DECLARE_OUT_SPEC(, NKikimrBlobStorage::TLocalGuidInfo::EState, stream, value) { + stream << NKikimrBlobStorage::TLocalGuidInfo::EState_Name(value); +} + +Y_DECLARE_OUT_SPEC(, NKikimrBlobStorage::TSyncerStatus::EPhase, stream, value) { + stream << NKikimrBlobStorage::TSyncerStatus::EPhase_Name(value); +} + +Y_DECLARE_OUT_SPEC(, NKikimrBlobStorage::EVDiskQueueId, stream, value) { + stream << NKikimrBlobStorage::EVDiskQueueId_Name(value); +} + +Y_DECLARE_OUT_SPEC(, NKikimrBlobStorage::EVDiskInternalQueueId, stream, value) { + stream << NKikimrBlobStorage::EVDiskInternalQueueId_Name(value); +} + +Y_DECLARE_OUT_SPEC(, NKikimrBlobStorage::ESyncFullStage, stream, value) { + stream << NKikimrBlobStorage::ESyncFullStage_Name(value); +} + +Y_DECLARE_OUT_SPEC(, NKikimrProto::EReplyStatus, stream, value) { + stream << NKikimrProto::EReplyStatus_Name(value); +} + +Y_DECLARE_OUT_SPEC(, NKikimrCms::TStatus::ECode, stream, value) { + stream << NKikimrCms::TStatus::ECode_Name(value); +} + +Y_DECLARE_OUT_SPEC(, NKikimrCms::EState, stream, value) { + stream << NKikimrCms::EState_Name(value); +} + +Y_DECLARE_OUT_SPEC(, NKikimrBlobStorage::EDriveStatus, stream, value) { + stream << NKikimrBlobStorage::EDriveStatus_Name(value); +} + Y_DECLARE_OUT_SPEC(, NKikimrBlobStorage::TGroupStatus::E, stream, value) { stream << NKikimrBlobStorage::TGroupStatus::E_Name(value); } @@ -81,65 +81,65 @@ Y_DECLARE_OUT_SPEC(, NKikimrBlobStorage::TSerialManagementStage::E, stream, valu stream << NKikimrBlobStorage::TSerialManagementStage::E_Name(value); } -Y_DECLARE_OUT_SPEC(, NKikimrResourceBroker::EResourceType, stream, value) { - stream << NKikimrResourceBroker::EResourceType_Name(value); -} +Y_DECLARE_OUT_SPEC(, NKikimrResourceBroker::EResourceType, stream, value) { + stream << NKikimrResourceBroker::EResourceType_Name(value); +} /* FIXME -Y_DECLARE_OUT_SPEC(, Ydb::Cms::GetDatabaseStatusResult::State, stream, value) { - stream << Ydb::Cms::GetDatabaseStatusResult::State_Name(value); -} - -Y_DECLARE_OUT_SPEC(, Ydb::StatusIds::StatusCode, stream, value) { - stream << Ydb::StatusIds::StatusCode_Name(value); -} +Y_DECLARE_OUT_SPEC(, Ydb::Cms::GetDatabaseStatusResult::State, stream, value) { + stream << Ydb::Cms::GetDatabaseStatusResult::State_Name(value); +} + +Y_DECLARE_OUT_SPEC(, Ydb::StatusIds::StatusCode, stream, value) { + stream << Ydb::StatusIds::StatusCode_Name(value); +} */ -Y_DECLARE_OUT_SPEC(, NKikimrConsole::TConfigItem::EKind, stream, value) { - stream << NKikimrConsole::TConfigItem::EKind_Name(value); -} - -Y_DECLARE_OUT_SPEC(, NKikimrTenantPool::EStatus, stream, value) { - stream << NKikimrTenantPool::EStatus_Name(value); -} - -Y_DECLARE_OUT_SPEC(, NKikimrBlobStorage::EPDiskType, stream, value) { - stream << NKikimrBlobStorage::EPDiskType_Name(value); -} - +Y_DECLARE_OUT_SPEC(, NKikimrConsole::TConfigItem::EKind, stream, value) { + stream << NKikimrConsole::TConfigItem::EKind_Name(value); +} + +Y_DECLARE_OUT_SPEC(, NKikimrTenantPool::EStatus, stream, value) { + stream << NKikimrTenantPool::EStatus_Name(value); +} + +Y_DECLARE_OUT_SPEC(, NKikimrBlobStorage::EPDiskType, stream, value) { + stream << NKikimrBlobStorage::EPDiskType_Name(value); +} + Y_DECLARE_OUT_SPEC(, NKikimrBlobStorage::EVDiskStatus, stream, value) { stream << NKikimrBlobStorage::EVDiskStatus_Name(value); } -Y_DECLARE_OUT_SPEC(, NKikimrNodeBroker::TStatus::ECode, stream, value) { - stream << NKikimrNodeBroker::TStatus::ECode_Name(value); -} - -Y_DECLARE_OUT_SPEC(, NKikimrCms::EMarker, stream, value) { - stream << NKikimrCms::EMarker_Name(value); -} - -Y_DECLARE_OUT_SPEC(, NKikimrTxDataShard::ETransactionKind, stream, value) { - stream << NKikimrTxDataShard::ETransactionKind_Name(value); -} - -Y_DECLARE_OUT_SPEC(, NKikimrCms::ETextFormat, stream, value) { - stream << NKikimrCms::ETextFormat_Name(value); -} - -Y_DECLARE_OUT_SPEC(, NKikimrCms::TLogRecordData::EType, stream, value) { - stream << NKikimrCms::TLogRecordData::EType_Name(value); -} - -Y_DECLARE_OUT_SPEC(, NKikimrConfig::TBootstrap::ETabletType, stream, value) { - stream << NKikimrConfig::TBootstrap::ETabletType_Name(value); -} - -Y_DECLARE_OUT_SPEC(, NKikimrTxDataShard::TEvProposeTransactionResult::EStatus, stream, value) { - stream << NKikimrTxDataShard::TEvProposeTransactionResult::EStatus_Name(value); -} - -Y_DECLARE_OUT_SPEC(, NKikimrTxDataShard::EDatashardState, stream, value) { - stream << NKikimrTxDataShard::EDatashardState_Name(value); -} +Y_DECLARE_OUT_SPEC(, NKikimrNodeBroker::TStatus::ECode, stream, value) { + stream << NKikimrNodeBroker::TStatus::ECode_Name(value); +} + +Y_DECLARE_OUT_SPEC(, NKikimrCms::EMarker, stream, value) { + stream << NKikimrCms::EMarker_Name(value); +} + +Y_DECLARE_OUT_SPEC(, NKikimrTxDataShard::ETransactionKind, stream, value) { + stream << NKikimrTxDataShard::ETransactionKind_Name(value); +} + +Y_DECLARE_OUT_SPEC(, NKikimrCms::ETextFormat, stream, value) { + stream << NKikimrCms::ETextFormat_Name(value); +} + +Y_DECLARE_OUT_SPEC(, NKikimrCms::TLogRecordData::EType, stream, value) { + stream << NKikimrCms::TLogRecordData::EType_Name(value); +} + +Y_DECLARE_OUT_SPEC(, NKikimrConfig::TBootstrap::ETabletType, stream, value) { + stream << NKikimrConfig::TBootstrap::ETabletType_Name(value); +} + +Y_DECLARE_OUT_SPEC(, NKikimrTxDataShard::TEvProposeTransactionResult::EStatus, stream, value) { + stream << NKikimrTxDataShard::TEvProposeTransactionResult::EStatus_Name(value); +} + +Y_DECLARE_OUT_SPEC(, NKikimrTxDataShard::EDatashardState, stream, value) { + stream << NKikimrTxDataShard::EDatashardState_Name(value); +} Y_DECLARE_OUT_SPEC(, NKikimrBlobStorage::TPDiskState::E, stream, value) { stream << NKikimrBlobStorage::TPDiskState::E_Name(value); diff --git a/ydb/core/protos/out/ya.make b/ydb/core/protos/out/ya.make index d56ff87819..bb7ca44717 100644 --- a/ydb/core/protos/out/ya.make +++ b/ydb/core/protos/out/ya.make @@ -1,14 +1,14 @@ -LIBRARY() +LIBRARY() -OWNER(g:kikimr) - -SRCS( - out.cpp +OWNER(g:kikimr) + +SRCS( + out.cpp out_sequenceshard.cpp -) - -PEERDIR( +) + +PEERDIR( ydb/core/protos -) - -END() +) + +END() diff --git a/ydb/core/protos/pqconfig.proto b/ydb/core/protos/pqconfig.proto index 14b11d3678..7c85927449 100644 --- a/ydb/core/protos/pqconfig.proto +++ b/ydb/core/protos/pqconfig.proto @@ -67,8 +67,8 @@ message TPQConfig { WRITTEN_BLOB_SIZE = 1; // Written blob size, including additional data for compaction. USER_PAYLOAD_SIZE = 2; // Message + source id size. } - optional bool EnableReadQuoting = 4 [default = false]; - optional uint64 ReadCreditBytes = 5 [default = 100000]; + optional bool EnableReadQuoting = 4 [default = false]; + optional uint64 ReadCreditBytes = 5 [default = 100000]; optional uint64 QuotaWaitDurationMs = 6 [default = 0]; // 0 means infinity @@ -85,18 +85,18 @@ message TPQConfig { optional string Root = 27 [default = "/Root/PQ"]; - message TPQLibSettings { - optional uint32 ThreadsCount = 1 [default = 1]; - optional uint32 CompressionPoolThreads = 2 [default = 1]; - optional uint32 GRpcThreads = 3 [default = 1]; - } - - message TMirrorConfig { - optional bool Enabled = 1 [default = true]; - optional TPQLibSettings PQLibSettings = 2; - } - - optional TMirrorConfig MirrorConfig = 29; + message TPQLibSettings { + optional uint32 ThreadsCount = 1 [default = 1]; + optional uint32 CompressionPoolThreads = 2 [default = 1]; + optional uint32 GRpcThreads = 3 [default = 1]; + } + + message TMirrorConfig { + optional bool Enabled = 1 [default = true]; + optional TPQLibSettings PQLibSettings = 2; + } + + optional TMirrorConfig MirrorConfig = 29; optional uint64 MinWriteLatencyMs = 30 [default = 0]; @@ -108,20 +108,20 @@ message TPQConfig { repeated uint32 ValidWriteSpeedLimitsKbPerSec = 34; - message TReadMeteringConfig { - message TPricing { - optional string Name = 1 [default = "unknown"]; - } - repeated TPricing Pricing = 2; - optional TPricing UnknownPricing = 3; - } - + message TReadMeteringConfig { + message TPricing { + optional string Name = 1 [default = "unknown"]; + } + repeated TPricing Pricing = 2; + optional TPricing UnknownPricing = 3; + } + message TBillingMeteringConfig { optional bool Enabled = 1 [default = false]; optional uint64 FlushIntervalSec = 2 [default = 30]; - optional TReadMeteringConfig Read = 3; + optional TReadMeteringConfig Read = 3; } - + optional TBillingMeteringConfig BillingMeteringConfig = 35; optional NKikimrNodeLimits.TNodeLimitsConfig.TPersQueueNodeConfig PersQueueNodeConfig = 36; @@ -129,16 +129,16 @@ message TPQConfig { optional bool EnableProtoSourceIdInfo = 37 [default = false]; optional string Database = 38; - - message TClientServiceType { - optional string Name = 1 [default = "data-transfer"]; - optional string ReadPricingName = 2; + + message TClientServiceType { + optional string Name = 1 [default = "data-transfer"]; + optional string ReadPricingName = 2; optional uint32 MaxReadRulesCountPerTopic = 3 [default = 0]; // 0 means no limit. If you want to allow only zero count, then do not specify ClientServiceType. - } - - repeated TClientServiceType ClientServiceType = 39; - optional TClientServiceType DefaultClientServiceType = 40; - optional bool DisallowDefaultClientServiceType = 43 [default = false]; + } + + repeated TClientServiceType ClientServiceType = 39; + optional TClientServiceType DefaultClientServiceType = 40; + optional bool DisallowDefaultClientServiceType = 43 [default = false]; optional uint32 MetaCacheRefreshIntervalMilliSeconds = 41 [default = 10000]; optional bool MetaCacheSkipVersionCheck = 42 [default = false]; @@ -153,30 +153,30 @@ message TChannelProfile { optional double WriteBandwidth = 6; // required bandwidth (in bytes/sec) for write requests } -message TMirrorPartitionConfig { - message TCredentials { - message IamCredentials { - optional string Endpoint = 1; - optional string ServiceAccountKey = 2; - } - oneof Credentials { - string OauthToken = 1; - string JwtParams = 2; - IamCredentials Iam = 3; - } - } - - optional string Endpoint = 1; - optional uint32 EndpointPort = 6; - optional string Topic = 2; - optional string Consumer = 3; - optional uint64 ReadFromTimestampsMs = 5; - optional TCredentials Credentials = 7; - optional string Database = 8; - optional bool UseSecureConnection = 9 [default = false]; - optional bool SyncWriteTime = 10 [default = false]; -} - +message TMirrorPartitionConfig { + message TCredentials { + message IamCredentials { + optional string Endpoint = 1; + optional string ServiceAccountKey = 2; + } + oneof Credentials { + string OauthToken = 1; + string JwtParams = 2; + IamCredentials Iam = 3; + } + } + + optional string Endpoint = 1; + optional uint32 EndpointPort = 6; + optional string Topic = 2; + optional string Consumer = 3; + optional uint64 ReadFromTimestampsMs = 5; + optional TCredentials Credentials = 7; + optional string Database = 8; + optional bool UseSecureConnection = 9 [default = false]; + optional bool SyncWriteTime = 10 [default = false]; +} + message TPartitionConfig { optional int32 MaxCountInPartition = 1 [default = 10000000]; optional int64 MaxSizeInPartition = 2 [default = 1099511627776]; @@ -206,8 +206,8 @@ message TPartitionConfig { optional uint32 TotalPartitions = 13 [default = 1]; repeated TChannelProfile ExplicitChannelProfiles = 14; - - optional TMirrorPartitionConfig MirrorFrom = 15; + + optional TMirrorPartitionConfig MirrorFrom = 15; }; message TPartitionKeyRange { @@ -247,7 +247,7 @@ message TPQTabletConfig { repeated string Codecs = 2; } repeated TCodecs ConsumerCodecs = 16; - repeated string ReadRuleServiceTypes = 17; + repeated string ReadRuleServiceTypes = 17; optional uint64 FormatVersion = 20; optional TCodecs Codecs = 21; @@ -357,7 +357,7 @@ message TDescribeResponse { message TCheckACL { optional EOperation Operation = 2; - optional NPersQueueCommon.Credentials Auth = 4; //leaved for compatibility + optional NPersQueueCommon.Credentials Auth = 4; //leaved for compatibility optional string User = 5; optional bytes Token = 6; } @@ -571,12 +571,12 @@ message TStatusResponse { STATUS_DISK_IS_FULL = 4; } - message TErrorMessage { - optional uint64 Timestamp = 1; - optional NKikimrServices.EServiceKikimr Service = 2; - optional string Message = 3; - } - + message TErrorMessage { + optional uint64 Timestamp = 1; + optional NKikimrServices.EServiceKikimr Service = 2; + optional string Message = 3; + } + message TPartResult { optional int32 Partition = 1; // mandatory optional EStatus Status = 2; @@ -615,8 +615,8 @@ message TStatusResponse { optional int64 SourceIdCount = 27; optional int64 SourceIdRetentionPeriodSec = 28; - - repeated TErrorMessage Errors = 29; + + repeated TErrorMessage Errors = 29; } optional uint64 TabletId = 1; diff --git a/ydb/core/protos/services.proto b/ydb/core/protos/services.proto index f2d7b219ea..c17c8a7dc3 100644 --- a/ydb/core/protos/services.proto +++ b/ydb/core/protos/services.proto @@ -164,8 +164,8 @@ enum EServiceKikimr { PQ_METACACHE = 441; PQ_READ_PROXY = 442; PQ_WRITE_PROXY = 443; - PQ_MIRRORER = 446; - PQ_READ_SPEED_LIMITER = 447; + PQ_MIRRORER = 446; + PQ_READ_SPEED_LIMITER = 447; PERSQUEUE_READ_BALANCER = 448; CHOOSE_PROXY = 444; diff --git a/ydb/core/tablet/tablet_counters_aggregator.cpp b/ydb/core/tablet/tablet_counters_aggregator.cpp index 18a589e6f0..1885b26e09 100644 --- a/ydb/core/tablet/tablet_counters_aggregator.cpp +++ b/ydb/core/tablet/tablet_counters_aggregator.cpp @@ -15,10 +15,10 @@ #include <ydb/core/tx/scheme_cache/scheme_cache.h> #include <ydb/core/util/wildcard.h> #include <ydb/library/persqueue/topic_parser/topic_parser.h> - + #include <library/cpp/monlib/service/pages/templates.h> #include <library/cpp/monlib/dynamic_counters/encode.h> - + #include <util/generic/xrange.h> #include <util/string/vector.h> #include <util/string/split.h> diff --git a/ydb/core/testlib/tablet_helpers.cpp b/ydb/core/testlib/tablet_helpers.cpp index eed935e589..a853040b93 100644 --- a/ydb/core/testlib/tablet_helpers.cpp +++ b/ydb/core/testlib/tablet_helpers.cpp @@ -53,7 +53,7 @@ const bool TRACE_DELAY_TIMING = true; const bool SUPPRESS_DELAYS = false; const bool VARIATE_RANDOM_SEED = false; const ui64 PQ_CACHE_MAX_SIZE_MB = 32; -const TDuration PQ_CACHE_KEEP_TIMEOUT = TDuration::Seconds(10); +const TDuration PQ_CACHE_KEEP_TIMEOUT = TDuration::Seconds(10); static NActors::TTestActorRuntime& AsKikimrRuntime(NActors::TTestActorRuntimeBase& r) { try { @@ -113,7 +113,7 @@ namespace NKikimr { void SetupPQNodeCache(TTestActorRuntime& runtime, ui32 nodeIndex) { - struct NPQ::TCacheL2Parameters l2Params = {PQ_CACHE_MAX_SIZE_MB, PQ_CACHE_KEEP_TIMEOUT}; + struct NPQ::TCacheL2Parameters l2Params = {PQ_CACHE_MAX_SIZE_MB, PQ_CACHE_KEEP_TIMEOUT}; runtime.AddLocalService(NPQ::MakePersQueueL2CacheID(), TActorSetupCmd( NPQ::CreateNodePersQueueL2Cache(l2Params, runtime.GetDynamicCounters(0)), diff --git a/ydb/core/testlib/test_client.cpp b/ydb/core/testlib/test_client.cpp index 13171086ec..d4907c26f1 100644 --- a/ydb/core/testlib/test_client.cpp +++ b/ydb/core/testlib/test_client.cpp @@ -226,20 +226,20 @@ namespace Tests { if (port) { Bus = NBus::CreateMessageQueue(NBus::TBusQueueConfig()); if (tracePath) { - BusServer.Reset(NMsgBusProxy::CreateMsgBusTracingServer( - Bus.Get(), - BusServerSessionConfig, - tracePath, - Settings->PersQueueGetReadSessionsInfoWorkerFactory, - port - )); + BusServer.Reset(NMsgBusProxy::CreateMsgBusTracingServer( + Bus.Get(), + BusServerSessionConfig, + tracePath, + Settings->PersQueueGetReadSessionsInfoWorkerFactory, + port + )); } else { - BusServer.Reset(NMsgBusProxy::CreateMsgBusServer( - Bus.Get(), - BusServerSessionConfig, - Settings->PersQueueGetReadSessionsInfoWorkerFactory, - port - )); + BusServer.Reset(NMsgBusProxy::CreateMsgBusServer( + Bus.Get(), + BusServerSessionConfig, + Settings->PersQueueGetReadSessionsInfoWorkerFactory, + port + )); } } } @@ -704,7 +704,7 @@ namespace Tests { TActorId netClassifierId = Runtime->Register(netClassifier, nodeIdx); Runtime->RegisterService(NNetClassifier::MakeNetClassifierID(), netClassifierId, nodeIdx); } - + if (Settings->EnableYq) { NYq::NConfig::TConfig protoConfig; protoConfig.SetEnabled(true); diff --git a/ydb/core/testlib/test_client.h b/ydb/core/testlib/test_client.h index 39f644357e..2064752ab4 100644 --- a/ydb/core/testlib/test_client.h +++ b/ydb/core/testlib/test_client.h @@ -120,7 +120,7 @@ namespace Tests { NKikimrConfig::TCompactionConfig CompactionConfig; TMap<ui32, TString> NodeKeys; ui64 DomainPlanResolution = 0; - std::shared_ptr<NKikimr::NMsgBusProxy::IPersQueueGetReadSessionsInfoWorkerFactory> PersQueueGetReadSessionsInfoWorkerFactory; + std::shared_ptr<NKikimr::NMsgBusProxy::IPersQueueGetReadSessionsInfoWorkerFactory> PersQueueGetReadSessionsInfoWorkerFactory; bool EnableMetering = false; TString MeteringFilePath; @@ -159,23 +159,23 @@ namespace Tests { TServerSettings& SetChangesQueueItemsLimit(ui64 value) { ChangesQueueItemsLimit = value; return *this; } TServerSettings& SetChangesQueueBytesLimit(ui64 value) { ChangesQueueBytesLimit = value; return *this; } TServerSettings& SetMeteringFilePath(const TString& path) { EnableMetering = true; MeteringFilePath = path; return *this; } - TServerSettings& SetPersQueueGetReadSessionsInfoWorkerFactory( - std::shared_ptr<NKikimr::NMsgBusProxy::IPersQueueGetReadSessionsInfoWorkerFactory> factory - ) { - PersQueueGetReadSessionsInfoWorkerFactory = factory; - return *this; - } + TServerSettings& SetPersQueueGetReadSessionsInfoWorkerFactory( + std::shared_ptr<NKikimr::NMsgBusProxy::IPersQueueGetReadSessionsInfoWorkerFactory> factory + ) { + PersQueueGetReadSessionsInfoWorkerFactory = factory; + return *this; + } // Add additional grpc services template <typename TService> - TServerSettings& RegisterGrpcService( - const TString& name, - std::optional<NActors::TActorId> proxyId = std::nullopt - ) { + TServerSettings& RegisterGrpcService( + const TString& name, + std::optional<NActors::TActorId> proxyId = std::nullopt + ) { if (!GrpcServiceFactory) { GrpcServiceFactory = std::make_shared<TGrpcServiceFactory>(); } - GrpcServiceFactory->Register<TService>(name, true, proxyId); + GrpcServiceFactory->Register<TService>(name, true, proxyId); return *this; } diff --git a/ydb/core/testlib/test_pq_client.h b/ydb/core/testlib/test_pq_client.h index 67e08f9809..f059923250 100644 --- a/ydb/core/testlib/test_pq_client.h +++ b/ydb/core/testlib/test_pq_client.h @@ -74,54 +74,54 @@ inline Tests::TServerSettings PQSettings(ui16 port, ui32 nodesCount = 2, bool ro const TString TopicPrefix = "/Root/PQ/"; const static TString DEFAULT_SRC_IDS_PATH = "/Root/PQ/SourceIdMeta2"; - + struct TRequestCreatePQ { - TRequestCreatePQ( - const TString& topic, - ui32 numParts, - ui32 cacheSize = 0, - ui64 lifetimeS = 86400, - ui32 lowWatermark = 8 * 1024 * 1024, - ui64 writeSpeed = 20000000, - const TString& user = "", - ui64 readSpeed = 20000000, - const TVector<TString>& readRules = {}, - const TVector<TString>& important = {}, - std::optional<NKikimrPQ::TMirrorPartitionConfig> mirrorFrom = {}, - ui64 sourceIdMaxCount = 6000000, - ui64 sourceIdLifetime = 86400 - ) - : Topic(topic) - , NumParts(numParts) - , CacheSize(cacheSize) - , LifetimeS(lifetimeS) - , LowWatermark(lowWatermark) - , WriteSpeed(writeSpeed) - , User(user) - , ReadSpeed(readSpeed) - , ReadRules(readRules) - , Important(important) - , MirrorFrom(mirrorFrom) - , SourceIdMaxCount(sourceIdMaxCount) - , SourceIdLifetime(sourceIdLifetime) - {} - + TRequestCreatePQ( + const TString& topic, + ui32 numParts, + ui32 cacheSize = 0, + ui64 lifetimeS = 86400, + ui32 lowWatermark = 8 * 1024 * 1024, + ui64 writeSpeed = 20000000, + const TString& user = "", + ui64 readSpeed = 20000000, + const TVector<TString>& readRules = {}, + const TVector<TString>& important = {}, + std::optional<NKikimrPQ::TMirrorPartitionConfig> mirrorFrom = {}, + ui64 sourceIdMaxCount = 6000000, + ui64 sourceIdLifetime = 86400 + ) + : Topic(topic) + , NumParts(numParts) + , CacheSize(cacheSize) + , LifetimeS(lifetimeS) + , LowWatermark(lowWatermark) + , WriteSpeed(writeSpeed) + , User(user) + , ReadSpeed(readSpeed) + , ReadRules(readRules) + , Important(important) + , MirrorFrom(mirrorFrom) + , SourceIdMaxCount(sourceIdMaxCount) + , SourceIdLifetime(sourceIdLifetime) + {} + TString Topic; ui32 NumParts; ui32 CacheSize; - ui64 LifetimeS; - ui32 LowWatermark; + ui64 LifetimeS; + ui32 LowWatermark; - ui64 WriteSpeed; + ui64 WriteSpeed; TString User; - ui64 ReadSpeed; + ui64 ReadSpeed; TVector<TString> ReadRules; TVector<TString> Important; - std::optional<NKikimrPQ::TMirrorPartitionConfig> MirrorFrom; - + std::optional<NKikimrPQ::TMirrorPartitionConfig> MirrorFrom; + ui64 SourceIdMaxCount; ui64 SourceIdLifetime; @@ -156,10 +156,10 @@ struct TRequestCreatePQ { config->MutablePartitionConfig()->SetBurstSize(WriteSpeed); for (auto& rr : ReadRules) { config->AddReadRules(rr); - config->AddReadFromTimestampsMs(0); - config->AddConsumerFormatVersions(0); - config->AddReadRuleVersions(0); - config->AddConsumerCodecs()->AddIds(0); + config->AddReadFromTimestampsMs(0); + config->AddConsumerFormatVersions(0); + config->AddReadRuleVersions(0); + config->AddConsumerCodecs()->AddIds(0); } if (!ReadRules.empty()) { config->SetRequireAuthRead(true); @@ -171,38 +171,38 @@ struct TRequestCreatePQ { rq->SetClientId(User); } - if (MirrorFrom) { - auto mirrorFromConfig = config->MutablePartitionConfig()->MutableMirrorFrom(); - mirrorFromConfig->CopyFrom(MirrorFrom.value()); - } + if (MirrorFrom) { + auto mirrorFromConfig = config->MutablePartitionConfig()->MutableMirrorFrom(); + mirrorFromConfig->CopyFrom(MirrorFrom.value()); + } return request; } }; struct TRequestAlterPQ { - TRequestAlterPQ( - const TString& topic, - ui32 numParts, - ui64 cacheSize = 0, - ui64 lifetimeS = 86400, - bool fillPartitionConfig = false, - std::optional<NKikimrPQ::TMirrorPartitionConfig> mirrorFrom = {} - ) - : Topic(topic) - , NumParts(numParts) - , CacheSize(cacheSize) - , LifetimeS(lifetimeS) - , FillPartitionConfig(fillPartitionConfig) - , MirrorFrom(mirrorFrom) - {} - + TRequestAlterPQ( + const TString& topic, + ui32 numParts, + ui64 cacheSize = 0, + ui64 lifetimeS = 86400, + bool fillPartitionConfig = false, + std::optional<NKikimrPQ::TMirrorPartitionConfig> mirrorFrom = {} + ) + : Topic(topic) + , NumParts(numParts) + , CacheSize(cacheSize) + , LifetimeS(lifetimeS) + , FillPartitionConfig(fillPartitionConfig) + , MirrorFrom(mirrorFrom) + {} + TString Topic; ui32 NumParts; ui64 CacheSize; ui64 LifetimeS; - bool FillPartitionConfig; - std::optional<NKikimrPQ::TMirrorPartitionConfig> MirrorFrom; + bool FillPartitionConfig; + std::optional<NKikimrPQ::TMirrorPartitionConfig> MirrorFrom; THolder<NMsgBusProxy::TBusPersQueue> GetRequest() { THolder<NMsgBusProxy::TBusPersQueue> request(new NMsgBusProxy::TBusPersQueue); @@ -213,21 +213,21 @@ struct TRequestAlterPQ { auto config = req->MutableConfig(); config->SetCacheSize(CacheSize); } - if (FillPartitionConfig) { - req->MutableConfig()->MutablePartitionConfig()->SetLifetimeSeconds(LifetimeS); - if (MirrorFrom) { - req->MutableConfig()->MutablePartitionConfig()->MutableMirrorFrom()->CopyFrom(MirrorFrom.value()); - } - } + if (FillPartitionConfig) { + req->MutableConfig()->MutablePartitionConfig()->SetLifetimeSeconds(LifetimeS); + if (MirrorFrom) { + req->MutableConfig()->MutablePartitionConfig()->MutableMirrorFrom()->CopyFrom(MirrorFrom.value()); + } + } return request; } }; struct TRequestDeletePQ { - TRequestDeletePQ(const TString& topic) - : Topic(topic) - {} - + TRequestDeletePQ(const TString& topic) + : Topic(topic) + {} + TString Topic; THolder<NMsgBusProxy::TBusPersQueue> GetRequest() { @@ -239,11 +239,11 @@ struct TRequestDeletePQ { }; struct TRequestGetOwnership { - TRequestGetOwnership(const TString& topic, ui32 partition) - : Topic(topic) - , Partition(partition) - {} - + TRequestGetOwnership(const TString& topic, ui32 partition) + : Topic(topic) + , Partition(partition) + {} + TString Topic; ui32 Partition; @@ -259,13 +259,13 @@ struct TRequestGetOwnership { struct TRequestWritePQ { - TRequestWritePQ(const TString& topic, ui32 partition, const TString& sourceId, ui64 seqNo) - : Topic(topic) - , Partition(partition) - , SourceId(sourceId) - , SeqNo(seqNo) - {} - + TRequestWritePQ(const TString& topic, ui32 partition, const TString& sourceId, ui64 seqNo) + : Topic(topic) + , Partition(partition) + , SourceId(sourceId) + , SeqNo(seqNo) + {} + TString Topic; ui32 Partition; TString SourceId; @@ -287,20 +287,20 @@ struct TRequestWritePQ { }; struct TRequestReadPQ { - TRequestReadPQ( - const TString& topic, - ui32 partition, - ui64 startOffset, - ui32 count, - const TString& user - ) - : Topic(topic) - , Partition(partition) - , StartOffset(startOffset) - , Count(count) - , User(user) - {} - + TRequestReadPQ( + const TString& topic, + ui32 partition, + ui64 startOffset, + ui32 count, + const TString& user + ) + : Topic(topic) + , Partition(partition) + , StartOffset(startOffset) + , Count(count) + , User(user) + {} + TString Topic; ui32 Partition; ui64 StartOffset; @@ -321,18 +321,18 @@ struct TRequestReadPQ { }; struct TRequestSetClientOffsetPQ { - TRequestSetClientOffsetPQ( - const TString& topic, - ui32 partition, - ui64 offset, - const TString& user - ) - : Topic(topic) - , Partition(partition) - , Offset(offset) - , User(user) - {} - + TRequestSetClientOffsetPQ( + const TString& topic, + ui32 partition, + ui64 offset, + const TString& user + ) + : Topic(topic) + , Partition(partition) + , Offset(offset) + , User(user) + {} + TString Topic; ui32 Partition; ui64 Offset; @@ -459,7 +459,7 @@ enum class ETransport { struct TPQTestClusterInfo { TString Balancer; bool Enabled; - ui64 Weight = 1000; + ui64 Weight = 1000; }; static THashMap<TString, TPQTestClusterInfo> DEFAULT_CLUSTERS_LIST = { @@ -615,7 +615,7 @@ public: balancer Utf8, local Bool, enabled Bool, - weight Uint64, + weight Uint64, PRIMARY KEY (name) ); CREATE TABLE [/Root/PQ/Config/V2/Topics] ( @@ -634,14 +634,14 @@ public: )___"); TStringBuilder upsertClusters; - upsertClusters << "UPSERT INTO [/Root/PQ/Config/V2/Cluster] (name, balancer, local, enabled, weight) VALUES "; + upsertClusters << "UPSERT INTO [/Root/PQ/Config/V2/Cluster] (name, balancer, local, enabled, weight) VALUES "; bool first = true; for (auto& [cluster, info] : clusters) { bool isLocal = localCluster.empty() ? first : localCluster == cluster; if (!first) upsertClusters << ", "; upsertClusters << "(\"" << cluster << "\", \"" << info.Balancer << "\", " << (isLocal ? "true" : "false") - << ", " << (info.Enabled ? "true" : "false") << ", " << info.Weight << ")"; + << ", " << (info.Enabled ? "true" : "false") << ", " << info.Weight << ")"; first = false; } upsertClusters << ";\n"; @@ -661,21 +661,21 @@ public: RunYqlDataQuery(query); } - TPQTestClusterInfo GetDcInfo(const TString& name) { - TStringBuilder query; - query << "SELECT balancer, enabled, weight FROM [/Root/PQ/Config/V2/Cluster] where name = \"" << name << "\";"; - auto result = RunYqlDataQuery(query); - NYdb::TResultSetParser parser(*result); - UNIT_ASSERT_VALUES_EQUAL(parser.RowsCount(), 1); - parser.TryNextRow(); - - TPQTestClusterInfo info; - info.Balancer = *parser.ColumnParser("balancer").GetOptionalUtf8(); - info.Enabled = *parser.ColumnParser("enabled").GetOptionalBool(); - info.Weight = *parser.ColumnParser("weight").GetOptionalUint64(); - return info; - } - + TPQTestClusterInfo GetDcInfo(const TString& name) { + TStringBuilder query; + query << "SELECT balancer, enabled, weight FROM [/Root/PQ/Config/V2/Cluster] where name = \"" << name << "\";"; + auto result = RunYqlDataQuery(query); + NYdb::TResultSetParser parser(*result); + UNIT_ASSERT_VALUES_EQUAL(parser.RowsCount(), 1); + parser.TryNextRow(); + + TPQTestClusterInfo info; + info.Balancer = *parser.ColumnParser("balancer").GetOptionalUtf8(); + info.Enabled = *parser.ColumnParser("enabled").GetOptionalBool(); + info.Weight = *parser.ColumnParser("weight").GetOptionalUint64(); + return info; + } + void InitUserRegistry() { MkDir("/Root/PQ", "Config"); MkDir("/Root/PQ/Config", "V2"); @@ -915,27 +915,27 @@ public: } } - void CreateTopic( - const TString& name, - ui32 nParts, - ui32 lowWatermark = 8*1024*1024, - ui64 lifetimeS = 86400, - ui64 writeSpeed = 20000000, - TString user = "", - ui64 readSpeed = 200000000, - TVector<TString> rr = {}, - TVector<TString> important = {}, + void CreateTopic( + const TString& name, + ui32 nParts, + ui32 lowWatermark = 8*1024*1024, + ui64 lifetimeS = 86400, + ui64 writeSpeed = 20000000, + TString user = "", + ui64 readSpeed = 200000000, + TVector<TString> rr = {}, + TVector<TString> important = {}, std::optional<NKikimrPQ::TMirrorPartitionConfig> mirrorFrom = {}, ui64 sourceIdMaxCount = 6000000, ui64 sourceIdLifetime = 86400 - ) { + ) { Y_VERIFY(name.StartsWith("rt3.")); Cerr << "PQ Client: create topic: " << name << " with " << nParts << " partitions" << Endl; - auto request = TRequestCreatePQ( + auto request = TRequestCreatePQ( name, nParts, 0, lifetimeS, lowWatermark, writeSpeed, user, readSpeed, rr, important, mirrorFrom, sourceIdMaxCount, sourceIdLifetime - ); + ); return CreateTopic(request); } @@ -955,17 +955,17 @@ public: Cerr << "Alter topic (" << path << ") response: " << res.GetValue().IsSuccess() << " " << res.GetValue().GetIssues().ToString() << Endl; } - void AlterTopic( - const TString& name, - ui32 nParts, - ui32 cacheSize = 0, - ui64 lifetimeS = 86400, - bool fillPartitionConfig = false, - std::optional<NKikimrPQ::TMirrorPartitionConfig> mirrorFrom = {} - ) { + void AlterTopic( + const TString& name, + ui32 nParts, + ui32 cacheSize = 0, + ui64 lifetimeS = 86400, + bool fillPartitionConfig = false, + std::optional<NKikimrPQ::TMirrorPartitionConfig> mirrorFrom = {} + ) { Y_VERIFY(name.StartsWith("rt3.")); - TRequestAlterPQ requestDescr(name, nParts, cacheSize, lifetimeS, fillPartitionConfig, mirrorFrom); - THolder<NMsgBusProxy::TBusPersQueue> request = requestDescr.GetRequest(); + TRequestAlterPQ requestDescr(name, nParts, cacheSize, lifetimeS, fillPartitionConfig, mirrorFrom); + THolder<NMsgBusProxy::TBusPersQueue> request = requestDescr.GetRequest(); ui32 prevVersion = TopicCreated(name); diff --git a/ydb/core/tx/scheme_board/cache.cpp b/ydb/core/tx/scheme_board/cache.cpp index 75bc4425e4..8e778a4a1d 100644 --- a/ydb/core/tx/scheme_board/cache.cpp +++ b/ydb/core/tx/scheme_board/cache.cpp @@ -190,7 +190,7 @@ namespace { break; } - entry.Self.Drop(); + entry.Self.Drop(); entry.SecurityObject.Drop(); entry.DomainInfo.Drop(); entry.Kind = TNavigate::KindUnknown; @@ -697,8 +697,8 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { CdcStreams.clear(); Partitioning.clear(); - Self.Drop(); - + Self.Drop(); + DomainDescription.Drop(); RtmrVolumeInfo.Drop(); KesusInfo.Drop(); @@ -1317,8 +1317,8 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { Y_VERIFY(pathDesc.HasSelf()); const auto& entryDesc = pathDesc.GetSelf(); - Self = new TNavigate::TDirEntryInfo(); - Self->Info.CopyFrom(entryDesc); + Self = new TNavigate::TDirEntryInfo(); + Self->Info.CopyFrom(entryDesc); Created = entryDesc.HasCreateFinished() && entryDesc.GetCreateFinished(); CreateStep = entryDesc.GetCreateStep(); @@ -1665,7 +1665,7 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { } // specific (it's safe to fill them all) - entry.Self = Self; + entry.Self = Self; entry.Columns = Columns; entry.NotNullColumns = NotNullColumns; entry.Indexes = Indexes; @@ -1916,8 +1916,8 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { TVector<NKikimrSchemeOp::TCdcStreamDescription> CdcStreams; TVector<TKeyDesc::TPartitionInfo> Partitioning; - TIntrusivePtr<TNavigate::TDirEntryInfo> Self; - + TIntrusivePtr<TNavigate::TDirEntryInfo> Self; + // RTMR specific TIntrusivePtr<TNavigate::TRtmrVolumeInfo> RtmrVolumeInfo; diff --git a/ydb/core/tx/scheme_cache/scheme_cache.h b/ydb/core/tx/scheme_cache/scheme_cache.h index ff1759b08a..26a07897a1 100644 --- a/ydb/core/tx/scheme_cache/scheme_cache.h +++ b/ydb/core/tx/scheme_cache/scheme_cache.h @@ -145,10 +145,10 @@ struct TSchemeCacheNavigate { TVector<TChild> Children; }; - struct TDirEntryInfo : public TAtomicRefCount<TDirEntryInfo>{ + struct TDirEntryInfo : public TAtomicRefCount<TDirEntryInfo>{ NKikimrSchemeOp::TDirEntry Info; - }; - + }; + struct TDomainDescription : public TAtomicRefCount<TDomainDescription> { EKind Kind = KindUnknown; NKikimrSubDomains::TDomainDescription Description; diff --git a/ydb/core/viewer/json_pqconsumerinfo.h b/ydb/core/viewer/json_pqconsumerinfo.h index c4d55aad7b..b98dfc2632 100644 --- a/ydb/core/viewer/json_pqconsumerinfo.h +++ b/ydb/core/viewer/json_pqconsumerinfo.h @@ -61,14 +61,14 @@ public: NKikimrClient::TPersQueueRequest request; request.MutableMetaRequest()->MutableCmdGetPartitionStatus()->SetClientId(Client); request.MutableMetaRequest()->MutableCmdGetPartitionStatus()->AddTopicRequest()->SetTopic(Topic); - ctx.Register(NMsgBusProxy::CreateActorServerPersQueue(ctx.SelfID, request, NMsgBusProxy::CreatePersQueueMetaCacheV2Id(), nullptr)); + ctx.Register(NMsgBusProxy::CreateActorServerPersQueue(ctx.SelfID, request, NMsgBusProxy::CreatePersQueueMetaCacheV2Id(), nullptr)); ++Requests; } { NKikimrClient::TPersQueueRequest request; request.MutableMetaRequest()->MutableCmdGetReadSessionsInfo()->SetClientId(Client); request.MutableMetaRequest()->MutableCmdGetReadSessionsInfo()->AddTopic(Topic); - ctx.Register(NMsgBusProxy::CreateActorServerPersQueue(ctx.SelfID, request, NMsgBusProxy::CreatePersQueueMetaCacheV2Id(), nullptr)); + ctx.Register(NMsgBusProxy::CreateActorServerPersQueue(ctx.SelfID, request, NMsgBusProxy::CreatePersQueueMetaCacheV2Id(), nullptr)); ++Requests; } Become(&TThis::StateRequestedTopicInfo, ctx, TDuration::MilliSeconds(Timeout), new TEvents::TEvWakeup()); diff --git a/ydb/core/ymq/actor/metering.cpp b/ydb/core/ymq/actor/metering.cpp index 144325410e..94ccef7eb0 100644 --- a/ydb/core/ymq/actor/metering.cpp +++ b/ydb/core/ymq/actor/metering.cpp @@ -5,7 +5,7 @@ #include "proxy_actor.h" #include <ydb/core/mind/address_classification/net_classifier.h> - + #include <ydb/core/ymq/base/action.h> #include <ydb/core/ymq/base/counters.h> #include <ydb/core/ymq/base/processed_request_attributes.h> @@ -14,7 +14,7 @@ #include <library/cpp/logger/record.h> #include <util/generic/guid.h> -#include <util/generic/serialized_enum.h> +#include <util/generic/serialized_enum.h> #include <util/string/builder.h> #include <util/system/hostname.h> @@ -24,28 +24,28 @@ namespace NKikimr::NSQS { using namespace NAddressClassifier; -const THashMap<TString, TString> TProcessedRequestsAggregator::LabelTransformation = { - {"yacloud", ToString(ENetworkClass::cloud)} -}; - -TProcessedRequestsAggregator::TProcessedRequestsAggregator(const NKikimrConfig::TSqsConfig& config) - : NetClassifierOnly(config.GetMeteringByNetClassifierOnly()) -{ - auto enumValues = GetEnumNames<ENetworkClass>(); - TVector<TString> labels(enumValues.size()); - for (auto enumItem : enumValues) { - Y_VERIFY(enumItem.first < labels.size()); - labels[enumItem.first] = enumItem.second; - } - Counters = new TMeteringCounters(config, GetSqsServiceCounters(AppData()->Counters, "metering"), labels); - if (!NetClassifierOnly) { - InitAddressClassifier(config, std::move(labels)); - } -} +const THashMap<TString, TString> TProcessedRequestsAggregator::LabelTransformation = { + {"yacloud", ToString(ENetworkClass::cloud)} +}; + +TProcessedRequestsAggregator::TProcessedRequestsAggregator(const NKikimrConfig::TSqsConfig& config) + : NetClassifierOnly(config.GetMeteringByNetClassifierOnly()) +{ + auto enumValues = GetEnumNames<ENetworkClass>(); + TVector<TString> labels(enumValues.size()); + for (auto enumItem : enumValues) { + Y_VERIFY(enumItem.first < labels.size()); + labels[enumItem.first] = enumItem.second; + } + Counters = new TMeteringCounters(config, GetSqsServiceCounters(AppData()->Counters, "metering"), labels); + if (!NetClassifierOnly) { + InitAddressClassifier(config, std::move(labels)); + } +} bool TProcessedRequestsAggregator::TReportedTrafficKey::operator<(const TReportedTrafficKey& rhs) const { - return std::tie(ResourceId, TrafficType, NetworkClassLabel) < - std::tie(rhs.ResourceId, rhs.TrafficType, rhs.NetworkClassLabel); + return std::tie(ResourceId, TrafficType, NetworkClassLabel) < + std::tie(rhs.ResourceId, rhs.TrafficType, rhs.NetworkClassLabel); } bool TProcessedRequestsAggregator::TReportedRequestsKey::operator<(const TReportedRequestsKey& rhs) const { @@ -53,33 +53,33 @@ bool TProcessedRequestsAggregator::TReportedRequestsKey::operator<(const TReport std::tie(rhs.ResourceId, rhs.QueueType); } -TString TProcessedRequestsAggregator::ClassifyNetwork(NAddressClassifier::TLabeledAddressClassifier::TConstPtr classifier, const TString& address) { - if (classifier) { - auto result = classifier->ClassifyAddress(address); - if (!result) { - return ToString(ENetworkClass::inet); - } - auto it = LabelTransformation.find(result.GetRef()); - if (it != LabelTransformation.end()) { - return it->second; - } - return result.GetRef(); - } - return ToString(ENetworkClass::unknown); -} - -TString TProcessedRequestsAggregator::ClassifyNetwork(const TString& address) const { - TString netClassifierResult = ClassifyNetwork(NetClassifier, address); - - TString result; - if (NetClassifierOnly) { - result = std::move(netClassifierResult); - } else { - result = ClassifyNetwork(AddressClassifier, address); - INC_COUNTER(Counters, IdleClassifierRequestsResults[netClassifierResult]); +TString TProcessedRequestsAggregator::ClassifyNetwork(NAddressClassifier::TLabeledAddressClassifier::TConstPtr classifier, const TString& address) { + if (classifier) { + auto result = classifier->ClassifyAddress(address); + if (!result) { + return ToString(ENetworkClass::inet); + } + auto it = LabelTransformation.find(result.GetRef()); + if (it != LabelTransformation.end()) { + return it->second; + } + return result.GetRef(); } - INC_COUNTER(Counters, ClassifierRequestsResults[result]); - return result; + return ToString(ENetworkClass::unknown); +} + +TString TProcessedRequestsAggregator::ClassifyNetwork(const TString& address) const { + TString netClassifierResult = ClassifyNetwork(NetClassifier, address); + + TString result; + if (NetClassifierOnly) { + result = std::move(netClassifierResult); + } else { + result = ClassifyNetwork(AddressClassifier, address); + INC_COUNTER(Counters, IdleClassifierRequestsResults[netClassifierResult]); + } + INC_COUNTER(Counters, ClassifierRequestsResults[result]); + return result; } ui64 TProcessedRequestsAggregator::CountBlocks(const ui64 bytes, const ui64 blockSize) const { @@ -97,13 +97,13 @@ bool TProcessedRequestsAggregator::Add(const TProcessedRequestAttributes& attrs) queueType = EQueueType::other; resourceId = ""; } - const auto networkClassLabel = ClassifyNetwork(attrs.SourceAddress); + const auto networkClassLabel = ClassifyNetwork(attrs.SourceAddress); Y_VERIFY(attrs.RequestSizeInBytes); Y_VERIFY(attrs.ResponseSizeInBytes); - ReportedTraffic[attrs.FolderId][{resourceId, ETrafficType::ingress, networkClassLabel}] += attrs.RequestSizeInBytes; - ReportedTraffic[attrs.FolderId][{resourceId, ETrafficType::egress, networkClassLabel}] += attrs.ResponseSizeInBytes; + ReportedTraffic[attrs.FolderId][{resourceId, ETrafficType::ingress, networkClassLabel}] += attrs.RequestSizeInBytes; + ReportedTraffic[attrs.FolderId][{resourceId, ETrafficType::egress, networkClassLabel}] += attrs.ResponseSizeInBytes; static const ui64 defaultRequestBlockSize = 64 * 1024; // SQS-22 ReportedRequests[attrs.FolderId][{resourceId, queueType}] += CountBlocks(attrs.RequestSizeInBytes + attrs.ResponseSizeInBytes, @@ -112,16 +112,16 @@ bool TProcessedRequestsAggregator::Add(const TProcessedRequestAttributes& attrs) return true; } -NSc::TValue CreateMeteringBillingRecord( - const TString& folderId, - const TString& resourceId, - const TString& schema, - const TString& fqdn, - const TInstant& now, - const ui64 quantity, - const TString& unit, - const NSc::TValue& tags -) { +NSc::TValue CreateMeteringBillingRecord( + const TString& folderId, + const TString& resourceId, + const TString& schema, + const TString& fqdn, + const TInstant& now, + const ui64 quantity, + const TString& unit, + const NSc::TValue& tags +) { const TString& billingRecordVersion = "v1"; const ui64 utcSeconds = now.Seconds(); @@ -157,10 +157,10 @@ TVector<NSc::TValue> TProcessedRequestsAggregator::DumpReportedTrafficAsJsonArra for (const auto& [trafficKey, trafficValue] : perFolderData.second) { NSc::TValue tags; tags.SetDict(); - tags["type"] = trafficKey.NetworkClassLabel; + tags["type"] = trafficKey.NetworkClassLabel; tags["direction"] = ToString(trafficKey.TrafficType); - result.push_back(CreateMeteringBillingRecord(folderId, + result.push_back(CreateMeteringBillingRecord(folderId, trafficKey.ResourceId, "ymq.traffic.v1", fqdn, @@ -184,7 +184,7 @@ TVector<NSc::TValue> TProcessedRequestsAggregator::DumpReportedRequestsAsJsonArr tags.SetDict(); tags["queue_type"] = ToString(requestsKey.QueueType); - result.push_back(CreateMeteringBillingRecord(folderId, + result.push_back(CreateMeteringBillingRecord(folderId, requestsKey.ResourceId, "ymq.requests.v1", fqdn, @@ -203,23 +203,23 @@ void TProcessedRequestsAggregator::ResetReportsStorage() { ReportedRequests.clear(); } -void TProcessedRequestsAggregator::UpdateNetClassifier(NAddressClassifier::TLabeledAddressClassifier::TConstPtr classifier, TMaybe<TInstant> /*netDataUpdateTimestamp*/) { - NetClassifier = classifier; -} - -void TProcessedRequestsAggregator::InitAddressClassifier(const NKikimrConfig::TSqsConfig& config, TVector<TString>&& labels) { - TAddressClassifier addressClassifier; - for (size_t i = 0; i < config.MeteringCloudNetCidrSize(); ++i) { - addressClassifier.AddNetByCidrAndLabel(config.GetMeteringCloudNetCidr(i) , ENetworkClass::cloud); - } - - for (size_t i = 0; i < config.MeteringYandexNetCidrSize(); ++i) { - addressClassifier.AddNetByCidrAndLabel(config.GetMeteringYandexNetCidr(i) , ENetworkClass::yandex); - } - - AddressClassifier = TLabeledAddressClassifier::MakeLabeledAddressClassifier(std::move(addressClassifier), std::move(labels)); -} - +void TProcessedRequestsAggregator::UpdateNetClassifier(NAddressClassifier::TLabeledAddressClassifier::TConstPtr classifier, TMaybe<TInstant> /*netDataUpdateTimestamp*/) { + NetClassifier = classifier; +} + +void TProcessedRequestsAggregator::InitAddressClassifier(const NKikimrConfig::TSqsConfig& config, TVector<TString>&& labels) { + TAddressClassifier addressClassifier; + for (size_t i = 0; i < config.MeteringCloudNetCidrSize(); ++i) { + addressClassifier.AddNetByCidrAndLabel(config.GetMeteringCloudNetCidr(i) , ENetworkClass::cloud); + } + + for (size_t i = 0; i < config.MeteringYandexNetCidrSize(); ++i) { + addressClassifier.AddNetByCidrAndLabel(config.GetMeteringYandexNetCidr(i) , ENetworkClass::yandex); + } + + AddressClassifier = TLabeledAddressClassifier::MakeLabeledAddressClassifier(std::move(addressClassifier), std::move(labels)); +} + class TMeteringActor : public TActorBootstrapped<TMeteringActor> { @@ -229,11 +229,11 @@ public: { } - void Bootstrap(const TActorContext& ctx) { + void Bootstrap(const TActorContext& ctx) { Become(&TThis::Work); - ctx.Send(NNetClassifier::MakeNetClassifierID(), new NNetClassifier::TEvNetClassifier::TEvSubscribe); - + ctx.Send(NNetClassifier::MakeNetClassifierID(), new NNetClassifier::TEvNetClassifier::TEvSubscribe); + Aggregator = MakeHolder<TProcessedRequestsAggregator>(Cfg()); FlushProcessedRequestsAttributes(); @@ -286,17 +286,17 @@ public: STATEFN(Work) { switch (ev->GetTypeRewrite()) { hFunc(TEvWakeup, HandleWakeup); - hFunc(NNetClassifier::TEvNetClassifier::TEvClassifierUpdate, HandleNetClassifierUpdate); + hFunc(NNetClassifier::TEvNetClassifier::TEvClassifierUpdate, HandleNetClassifierUpdate); hFunc(TSqsEvents::TEvReportProcessedRequestAttributes, HandleReportProcessedRequestAttributes); } } - + +private: + void HandleNetClassifierUpdate(NNetClassifier::TEvNetClassifier::TEvClassifierUpdate::TPtr& ev) { + Aggregator->UpdateNetClassifier(ev->Get()->Classifier, ev->Get()->NetDataUpdateTimestamp); + } + private: - void HandleNetClassifierUpdate(NNetClassifier::TEvNetClassifier::TEvClassifierUpdate::TPtr& ev) { - Aggregator->UpdateNetClassifier(ev->Get()->Classifier, ev->Get()->NetDataUpdateTimestamp); - } - -private: const TString HostFQDN; THolder<TProcessedRequestsAggregator> Aggregator; }; diff --git a/ydb/core/ymq/actor/metering.h b/ydb/core/ymq/actor/metering.h index 3e4fb93c03..c9d5a58991 100644 --- a/ydb/core/ymq/actor/metering.h +++ b/ydb/core/ymq/actor/metering.h @@ -15,23 +15,23 @@ namespace NKikimr::NSQS { -struct TMeteringCounters; - -NSc::TValue CreateMeteringBillingRecord( - const TString& folderId, - const TString& resourceId, - const TString& schema, - const TString& fqdn, - const TInstant& now, - const ui64 quantity, - const TString& unit, - const NSc::TValue& tags -); - +struct TMeteringCounters; + +NSc::TValue CreateMeteringBillingRecord( + const TString& folderId, + const TString& resourceId, + const TString& schema, + const TString& fqdn, + const TInstant& now, + const ui64 quantity, + const TString& unit, + const NSc::TValue& tags +); + class TProcessedRequestsAggregator { public: - static const THashMap<TString, TString> LabelTransformation; -public: + static const THashMap<TString, TString> LabelTransformation; +public: TProcessedRequestsAggregator(const NKikimrConfig::TSqsConfig& config); // Do not rename enum members in the enums below (SQS-22) @@ -43,8 +43,8 @@ public: enum ENetworkClass { inet = 0, cloud, - yandex, - unknown + yandex, + unknown }; enum EQueueType { @@ -56,7 +56,7 @@ public: struct TReportedTrafficKey { TString ResourceId; ETrafficType TrafficType; - TString NetworkClassLabel; + TString NetworkClassLabel; bool operator<(const TReportedTrafficKey& rhs) const; }; @@ -68,7 +68,7 @@ public: bool operator<(const TReportedRequestsKey& rhs) const; }; - TString ClassifyNetwork(const TString& address) const; + TString ClassifyNetwork(const TString& address) const; ui64 CountBlocks(const ui64 bytes, const ui64 blockSize) const; @@ -79,22 +79,22 @@ public: void ResetReportsStorage(); - void UpdateNetClassifier(NAddressClassifier::TLabeledAddressClassifier::TConstPtr classifier, TMaybe<TInstant> netDataUpdateTimestamp); - + void UpdateNetClassifier(NAddressClassifier::TLabeledAddressClassifier::TConstPtr classifier, TMaybe<TInstant> netDataUpdateTimestamp); + +private: + static TString ClassifyNetwork(NAddressClassifier::TLabeledAddressClassifier::TConstPtr classifier, const TString& address); + + void InitAddressClassifier(const NKikimrConfig::TSqsConfig& config, TVector<TString>&& labels); + private: - static TString ClassifyNetwork(NAddressClassifier::TLabeledAddressClassifier::TConstPtr classifier, const TString& address); - - void InitAddressClassifier(const NKikimrConfig::TSqsConfig& config, TVector<TString>&& labels); - -private: - bool NetClassifierOnly; - + bool NetClassifierOnly; + THashMap<TString, TMap<TReportedTrafficKey, ui64>> ReportedTraffic; THashMap<TString, TMap<TReportedRequestsKey, ui64>> ReportedRequests; - NAddressClassifier::TLabeledAddressClassifier::TConstPtr AddressClassifier; - NAddressClassifier::TLabeledAddressClassifier::TConstPtr NetClassifier; - - TIntrusivePtr<TMeteringCounters> Counters; + NAddressClassifier::TLabeledAddressClassifier::TConstPtr AddressClassifier; + NAddressClassifier::TLabeledAddressClassifier::TConstPtr NetClassifier; + + TIntrusivePtr<TMeteringCounters> Counters; }; } // namespace NKikimr::NSQS diff --git a/ydb/core/ymq/actor/queue_schema.cpp b/ydb/core/ymq/actor/queue_schema.cpp index 629af7ddea..91d3c5d3df 100644 --- a/ydb/core/ymq/actor/queue_schema.cpp +++ b/ydb/core/ymq/actor/queue_schema.cpp @@ -9,22 +9,22 @@ #include <ydb/core/ymq/queues/fifo/schema.h> #include <ydb/core/ymq/queues/std/schema.h> -#include <util/digest/city.h> +#include <util/digest/city.h> #include <util/generic/utility.h> -#include <util/string/join.h> +#include <util/string/join.h> using NKikimr::NClient::TValue; namespace NKikimr::NSQS { -constexpr TStringBuf FIFO_TABLES_DIR = ".FIFO"; -constexpr TStringBuf STD_TABLES_DIR = ".STD"; - -template <class... TArgs> -ui64 GetHash(TArgs&&... args) { - return CityHash64(Join("#", args...)); -} - +constexpr TStringBuf FIFO_TABLES_DIR = ".FIFO"; +constexpr TStringBuf STD_TABLES_DIR = ".STD"; + +template <class... TArgs> +ui64 GetHash(TArgs&&... args) { + return CityHash64(Join("#", args...)); +} + static bool IsGoodStatusCode(ui32 code) { switch (NTxProxy::TResultStatus::EStatus(code)) { case NTxProxy::TResultStatus::EStatus::ExecComplete: @@ -411,40 +411,40 @@ void TCreateQueueSchemaActorV2::OnAtomicCounterIncrement(TSqsEvents::TEvAtomicCo PassAway(); } -static const char* const GetTablesFormatQuery = R"__( - ( - (let defaultTablesFormat (Parameter 'DEFAULT_TABLES_FORMAT (DataType 'Uint32))) - (let userName (Parameter 'USER_NAME (DataType 'Utf8String))) - (let settingsTable '%1$s/.Settings) - (let tablesFormatSettingRow '( - '('Account userName) - '('Name (Utf8String '"CreateQueuesWithTabletFormat")))) - (let tablesFormatSettingSelect '('Value)) - (let tablesFormatSettingRead (SelectRow settingsTable tablesFormatSettingRow tablesFormatSettingSelect)) - (let tablesFormatSetting - (If (Exists tablesFormatSettingRead) - (Cast (Member tablesFormatSettingRead 'Value) 'Uint32) - defaultTablesFormat - ) - ) - - (return (AsList - (SetResult 'tablesFormat tablesFormatSetting) - ) - ) - ) -)__"; - -void TCreateQueueSchemaActorV2::RequestTablesFormatSettings(const TString& accountName) { - auto ev = MakeExecuteEvent(Sprintf(GetTablesFormatQuery, Cfg().GetRoot().c_str())); - auto* trans = ev->Record.MutableTransaction()->MutableMiniKQLTransaction(); - TParameters(trans->MutableParams()->MutableProto()) - .Utf8("USER_NAME", accountName) - .Uint32("DEFAULT_TABLES_FORMAT", 0); - - Register(new TMiniKqlExecutionActor(SelfId(), RequestId_, std::move(ev), false, QueuePath_, GetTransactionCounters(UserCounters_))); -} - +static const char* const GetTablesFormatQuery = R"__( + ( + (let defaultTablesFormat (Parameter 'DEFAULT_TABLES_FORMAT (DataType 'Uint32))) + (let userName (Parameter 'USER_NAME (DataType 'Utf8String))) + (let settingsTable '%1$s/.Settings) + (let tablesFormatSettingRow '( + '('Account userName) + '('Name (Utf8String '"CreateQueuesWithTabletFormat")))) + (let tablesFormatSettingSelect '('Value)) + (let tablesFormatSettingRead (SelectRow settingsTable tablesFormatSettingRow tablesFormatSettingSelect)) + (let tablesFormatSetting + (If (Exists tablesFormatSettingRead) + (Cast (Member tablesFormatSettingRead 'Value) 'Uint32) + defaultTablesFormat + ) + ) + + (return (AsList + (SetResult 'tablesFormat tablesFormatSetting) + ) + ) + ) +)__"; + +void TCreateQueueSchemaActorV2::RequestTablesFormatSettings(const TString& accountName) { + auto ev = MakeExecuteEvent(Sprintf(GetTablesFormatQuery, Cfg().GetRoot().c_str())); + auto* trans = ev->Record.MutableTransaction()->MutableMiniKQLTransaction(); + TParameters(trans->MutableParams()->MutableProto()) + .Utf8("USER_NAME", accountName) + .Uint32("DEFAULT_TABLES_FORMAT", 0); + + Register(new TMiniKqlExecutionActor(SelfId(), RequestId_, std::move(ev), false, QueuePath_, GetTransactionCounters(UserCounters_))); +} + void TCreateQueueSchemaActorV2::RegisterMakeDirActor(const TString& workingDir, const TString& dirName) { auto ev = MakeHolder<TEvTxUserProxy::TEvProposeTransaction>(); auto* trans = ev->Record.MutableTransaction()->MutableModifyScheme(); @@ -466,10 +466,10 @@ void TCreateQueueSchemaActorV2::RequestLeaderTabletId() { void TCreateQueueSchemaActorV2::CreateComponents() { switch (CurrentCreationStep_) { - case ECreateComponentsStep::GetTablesFormatSetting: { - RequestTablesFormatSettings(QueuePath_.UserName); - break; - } + case ECreateComponentsStep::GetTablesFormatSetting: { + RequestTablesFormatSettings(QueuePath_.UserName); + break; + } case ECreateComponentsStep::MakeQueueDir: { RegisterMakeDirActor(QueuePath_.GetUserPath(), QueuePath_.QueueName); break; @@ -500,10 +500,10 @@ void TCreateQueueSchemaActorV2::CreateComponents() { break; } - case ECreateComponentsStep::DescribeTableForSetSchemeShardId: { - SendDescribeTable(); - break; - } + case ECreateComponentsStep::DescribeTableForSetSchemeShardId: { + SendDescribeTable(); + break; + } case ECreateComponentsStep::DiscoverLeaderTabletId: { RequestLeaderTabletId(); break; @@ -519,7 +519,7 @@ STATEFN(TCreateQueueSchemaActorV2::CreateComponentsState) { switch (ev->GetTypeRewrite()) { hFunc(TSqsEvents::TEvExecuted, OnExecuted); hFunc(NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult, OnDescribeSchemeResult); - hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, HandleTableDescription); + hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, HandleTableDescription); hFunc(NKesus::TEvKesus::TEvAddQuoterResourceResult, HandleAddQuoterResource); cFunc(TEvPoisonPill::EventType, PassAway); } @@ -528,16 +528,16 @@ STATEFN(TCreateQueueSchemaActorV2::CreateComponentsState) { void TCreateQueueSchemaActorV2::Step() { RLOG_SQS_TRACE("Next step. Step: " << (int)CurrentCreationStep_); switch (CurrentCreationStep_) { - case ECreateComponentsStep::GetTablesFormatSetting: { - CurrentCreationStep_ = ECreateComponentsStep::MakeQueueDir; - break; - } + case ECreateComponentsStep::GetTablesFormatSetting: { + CurrentCreationStep_ = ECreateComponentsStep::MakeQueueDir; + break; + } case ECreateComponentsStep::MakeQueueDir: { - if (TablesFormat_ == 0) { - CurrentCreationStep_ = ECreateComponentsStep::MakeQueueVersionDir; - } else { - CurrentCreationStep_ = ECreateComponentsStep::DescribeTableForSetSchemeShardId; - } + if (TablesFormat_ == 0) { + CurrentCreationStep_ = ECreateComponentsStep::MakeQueueVersionDir; + } else { + CurrentCreationStep_ = ECreateComponentsStep::DescribeTableForSetSchemeShardId; + } break; } case ECreateComponentsStep::MakeQueueVersionDir: { @@ -566,12 +566,12 @@ void TCreateQueueSchemaActorV2::Step() { CurrentCreationStep_ = ECreateComponentsStep::DiscoverLeaderTabletId; break; } - case ECreateComponentsStep::DescribeTableForSetSchemeShardId: { - Y_VERIFY(TablesFormat_ == 1); - Y_VERIFY(TableWithLeaderPathId_.first && TableWithLeaderPathId_.second); - CurrentCreationStep_ = ECreateComponentsStep::DiscoverLeaderTabletId; - break; - } + case ECreateComponentsStep::DescribeTableForSetSchemeShardId: { + Y_VERIFY(TablesFormat_ == 1); + Y_VERIFY(TableWithLeaderPathId_.first && TableWithLeaderPathId_.second); + CurrentCreationStep_ = ECreateComponentsStep::DiscoverLeaderTabletId; + break; + } case ECreateComponentsStep::DiscoverLeaderTabletId: { Y_VERIFY(Cfg().GetQuotingConfig().GetEnableQuoting() && Cfg().GetQuotingConfig().HasKesusQuoterConfig()); CurrentCreationStep_ = ECreateComponentsStep::AddQuoterResource; @@ -601,28 +601,28 @@ void TCreateQueueSchemaActorV2::OnExecuted(TSqsEvents::TEvExecuted::TPtr& ev) { // SS finishes transaction immediately if the specified path already exists // DO NOT add any special logic based on the result type (except for an error) if (IsGoodStatusCode(status)) { - if (CurrentCreationStep_ == ECreateComponentsStep::GetTablesFormatSetting) { - const TValue value(TValue::Create(record.GetExecutionEngineEvaluatedResponse())); - const TValue formatValue = value["tablesFormat"]; - if (formatValue.HaveValue()) { - TablesFormat_ = static_cast<ui32>(formatValue); - } - if (!formatValue.HaveValue() || TablesFormat_ > 1) { - RLOG_SQS_WARN("Incorrect TablesFormat settings for account " - << QueuePath_.UserName << ", responce:" << record); - - auto resp = MakeErrorResponse(NErrors::INTERNAL_FAILURE); - resp->State = EQueueState::Creating; - resp->Error = "Incorrect TablesFormat settings for account"; - - Send(Sender_, std::move(resp)); - PassAway(); - return; - } - RLOG_SQS_DEBUG("Got table format '" << TablesFormat_ << "' for " - << QueuePath_.UserName << record); - } - + if (CurrentCreationStep_ == ECreateComponentsStep::GetTablesFormatSetting) { + const TValue value(TValue::Create(record.GetExecutionEngineEvaluatedResponse())); + const TValue formatValue = value["tablesFormat"]; + if (formatValue.HaveValue()) { + TablesFormat_ = static_cast<ui32>(formatValue); + } + if (!formatValue.HaveValue() || TablesFormat_ > 1) { + RLOG_SQS_WARN("Incorrect TablesFormat settings for account " + << QueuePath_.UserName << ", responce:" << record); + + auto resp = MakeErrorResponse(NErrors::INTERNAL_FAILURE); + resp->State = EQueueState::Creating; + resp->Error = "Incorrect TablesFormat settings for account"; + + Send(Sender_, std::move(resp)); + PassAway(); + return; + } + RLOG_SQS_DEBUG("Got table format '" << TablesFormat_ << "' for " + << QueuePath_.UserName << record); + } + Step(); } else { RLOG_SQS_WARN("Component creation request execution error: " << record); @@ -667,30 +667,30 @@ void TCreateQueueSchemaActorV2::OnDescribeSchemeResult(NSchemeShard::TEvSchemeSh } } -void TCreateQueueSchemaActorV2::SendDescribeTable() { - auto navigateRequest = std::make_unique<NSchemeCache::TSchemeCacheNavigate>(); - - NSchemeCache::TSchemeCacheNavigate::TEntry entry; - entry.Path = NKikimr::SplitPath(Cfg().GetRoot() + "/.Queues"); - entry.Operation = NSchemeCache::TSchemeCacheNavigate::OpTable; - navigateRequest->ResultSet.emplace_back(entry); - - Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvNavigateKeySet(navigateRequest.release())); -} - -void TCreateQueueSchemaActorV2::HandleTableDescription(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) { - const NSchemeCache::TSchemeCacheNavigate* result = ev->Get()->Request.Get(); - Y_VERIFY(result->ResultSet.size() == 1); - const auto& response = result->ResultSet.front(); - - TableWithLeaderPathId_ = std::make_pair( - response.TableId.PathId.OwnerId, - response.TableId.PathId.LocalPathId - ); - - Step(); -} - +void TCreateQueueSchemaActorV2::SendDescribeTable() { + auto navigateRequest = std::make_unique<NSchemeCache::TSchemeCacheNavigate>(); + + NSchemeCache::TSchemeCacheNavigate::TEntry entry; + entry.Path = NKikimr::SplitPath(Cfg().GetRoot() + "/.Queues"); + entry.Operation = NSchemeCache::TSchemeCacheNavigate::OpTable; + navigateRequest->ResultSet.emplace_back(entry); + + Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvNavigateKeySet(navigateRequest.release())); +} + +void TCreateQueueSchemaActorV2::HandleTableDescription(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) { + const NSchemeCache::TSchemeCacheNavigate* result = ev->Get()->Request.Get(); + Y_VERIFY(result->ResultSet.size() == 1); + const auto& response = result->ResultSet.front(); + + TableWithLeaderPathId_ = std::make_pair( + response.TableId.PathId.OwnerId, + response.TableId.PathId.LocalPathId + ); + + Step(); +} + void TCreateQueueSchemaActorV2::AddRPSQuota() { NKikimrKesus::TEvAddQuoterResource cmd; auto& res = *cmd.MutableResource(); @@ -730,9 +730,9 @@ static const char* const CommitQueueParamsQuery = R"__( (let shards (Parameter 'SHARDS (DataType 'Uint64))) (let partitions (Parameter 'PARTITIONS (DataType 'Uint64))) (let masterTabletId (Parameter 'MASTER_TABLET_ID (DataType 'Uint64))) - (let tablesFormat (Parameter 'TABLES_FORMAT (DataType 'Uint32))) + (let tablesFormat (Parameter 'TABLES_FORMAT (DataType 'Uint32))) (let version (Parameter 'VERSION (DataType 'Uint64))) - (let queueIdNumberHash (Parameter 'QUEUE_ID_NUMBER_HASH (DataType 'Uint64))) + (let queueIdNumberHash (Parameter 'QUEUE_ID_NUMBER_HASH (DataType 'Uint64))) (let maxSize (Parameter 'MAX_SIZE (DataType 'Uint64))) (let delay (Parameter 'DELAY (DataType 'Uint64))) (let visibility (Parameter 'VISIBILITY (DataType 'Uint64))) @@ -744,11 +744,11 @@ static const char* const CommitQueueParamsQuery = R"__( (let defaultMaxQueuesCount (Parameter 'DEFAULT_MAX_QUEUES_COUNT (DataType 'Uint64))) (let userName (Parameter 'USER_NAME (DataType 'Utf8String))) - (let attrsTable '%1$s/Attributes) - (let stateTable '%1$s/State) - (let settingsTable '%2$s/.Settings) - (let queuesTable '%2$s/.Queues) - (let eventsTable '%2$s/.Events) + (let attrsTable '%1$s/Attributes) + (let stateTable '%1$s/State) + (let settingsTable '%2$s/.Settings) + (let queuesTable '%2$s/.Queues) + (let eventsTable '%2$s/.Events) (let maxQueuesCountSettingRow '( '('Account userName) @@ -833,16 +833,16 @@ static const char* const CommitQueueParamsQuery = R"__( '('Partitions partitions) '('Version version) '('DlqName dlqName) - '('MasterTabletId masterTabletId) - '('TablesFormat tablesFormat))) + '('MasterTabletId masterTabletId) + '('TablesFormat tablesFormat))) (let eventsUpdate '( '('CustomQueueName customName) '('EventTimestamp now) '('FolderId folderId))) - (let attrRow '(%3$s)) - + (let attrRow '(%3$s)) + (let attrUpdate '( '('ContentBasedDeduplication contentBasedDeduplication) '('DelaySeconds delay) @@ -860,19 +860,19 @@ static const char* const CommitQueueParamsQuery = R"__( (Not queueExists) (Not overLimit))) - (let stateUpdate '( - '('CleanupTimestamp now) - '('CreatedTimestamp now) - '('LastModifiedTimestamp now) - '('InflyCount (Int64 '0)) - '('MessageCount (Int64 '0)) - '('RetentionBoundary (Uint64 '0)) - '('ReadOffset (Uint64 '0)) - '('WriteOffset (Uint64 '0)) - '('CleanupVersion (Uint64 '0)))) - - (let queueIdNumberAndShardHashes (AsList %4$s)) - + (let stateUpdate '( + '('CleanupTimestamp now) + '('CreatedTimestamp now) + '('LastModifiedTimestamp now) + '('InflyCount (Int64 '0)) + '('MessageCount (Int64 '0)) + '('RetentionBoundary (Uint64 '0)) + '('ReadOffset (Uint64 '0)) + '('WriteOffset (Uint64 '0)) + '('CleanupVersion (Uint64 '0)))) + + (let queueIdNumberAndShardHashes (AsList %4$s)) + (return (Extend (AsList (SetResult 'exists queueExists) @@ -886,14 +886,14 @@ static const char* const CommitQueueParamsQuery = R"__( (ListIf willCommit (UpdateRow queuesTable queuesRow queuesUpdate)) (ListIf willCommit (UpdateRow eventsTable eventsRow eventsUpdate)) (ListIf willCommit (UpdateRow attrsTable attrRow attrUpdate)) - - (If (Not willCommit) (AsList (Void)) - (Map (Enumerate queueIdNumberAndShardHashes) (lambda '(item) (block '( - (let shardOriginal (Nth item '0)) - (let shard (Cast shardOriginal 'Uint32)) - (let queueIdNumberAndShardHash (Nth item '1)) - - (let row '(%5$s)) + + (If (Not willCommit) (AsList (Void)) + (Map (Enumerate queueIdNumberAndShardHashes) (lambda '(item) (block '( + (let shardOriginal (Nth item '0)) + (let shard (Cast shardOriginal 'Uint32)) + (let queueIdNumberAndShardHash (Nth item '1)) + + (let row '(%5$s)) (let update '( '('CleanupTimestamp now) '('CreatedTimestamp now) @@ -909,62 +909,62 @@ static const char* const CommitQueueParamsQuery = R"__( ) )__"; -TString GetStateTableKeys(ui32 tablesFormat, bool isFifo) { - if (tablesFormat == 1) { - if (isFifo) { - return R"__( - '('QueueIdNumberHash queueIdNumberHash) - '('QueueIdNumber version) - )__"; - } - return R"__( - '('QueueIdNumberAndShardHash queueIdNumberAndShardHash) - '('QueueIdNumber version) - '('Shard shard) - )__"; - - } - return "'('State shardOriginal)"; -} - -TString GetAttrTableKeys(ui32 tablesFormat) { - if (tablesFormat == 1) { - return R"__( - '('QueueIdNumberHash queueIdNumberHash) - '('QueueIdNumber version) - )__"; - } - return "'('State (Uint64 '0))"; -} - -TString GetQueueIdAndShardHashesList(ui64 version, ui32 shards) { - TStringBuilder hashes; - for (ui32 i = 0; i < shards; ++i) { - hashes << "(Uint64 '" << GetHash(version, i) << ") "; - } - return hashes; -} - +TString GetStateTableKeys(ui32 tablesFormat, bool isFifo) { + if (tablesFormat == 1) { + if (isFifo) { + return R"__( + '('QueueIdNumberHash queueIdNumberHash) + '('QueueIdNumber version) + )__"; + } + return R"__( + '('QueueIdNumberAndShardHash queueIdNumberAndShardHash) + '('QueueIdNumber version) + '('Shard shard) + )__"; + + } + return "'('State shardOriginal)"; +} + +TString GetAttrTableKeys(ui32 tablesFormat) { + if (tablesFormat == 1) { + return R"__( + '('QueueIdNumberHash queueIdNumberHash) + '('QueueIdNumber version) + )__"; + } + return "'('State (Uint64 '0))"; +} + +TString GetQueueIdAndShardHashesList(ui64 version, ui32 shards) { + TStringBuilder hashes; + for (ui32 i = 0; i < shards; ++i) { + hashes << "(Uint64 '" << GetHash(version, i) << ") "; + } + return hashes; +} + void TCreateQueueSchemaActorV2::CommitNewVersion() { Become(&TCreateQueueSchemaActorV2::FinalizeAndCommit); - TString queuePath; - if (TablesFormat_ == 0) { - queuePath = Join("/", QueuePath_.GetUserPath(), QueuePath_.QueueName, VersionName_); - } else { - queuePath = Join("/", Cfg().GetRoot(), IsFifo_ ? FIFO_TABLES_DIR : STD_TABLES_DIR); - } - - TString query = Sprintf( - CommitQueueParamsQuery, - queuePath.c_str(), - Cfg().GetRoot().c_str(), - GetAttrTableKeys(TablesFormat_).c_str(), - GetQueueIdAndShardHashesList(Version_, RequiredShardsCount_).c_str(), - GetStateTableKeys(TablesFormat_, IsFifo_).c_str() - ); - - auto ev = MakeExecuteEvent(query); + TString queuePath; + if (TablesFormat_ == 0) { + queuePath = Join("/", QueuePath_.GetUserPath(), QueuePath_.QueueName, VersionName_); + } else { + queuePath = Join("/", Cfg().GetRoot(), IsFifo_ ? FIFO_TABLES_DIR : STD_TABLES_DIR); + } + + TString query = Sprintf( + CommitQueueParamsQuery, + queuePath.c_str(), + Cfg().GetRoot().c_str(), + GetAttrTableKeys(TablesFormat_).c_str(), + GetQueueIdAndShardHashesList(Version_, RequiredShardsCount_).c_str(), + GetStateTableKeys(TablesFormat_, IsFifo_).c_str() + ); + + auto ev = MakeExecuteEvent(query); auto* trans = ev->Record.MutableTransaction()->MutableMiniKQLTransaction(); Y_VERIFY(LeaderTabletId_ != 0); TParameters(trans->MutableParams()->MutableProto()) @@ -978,9 +978,9 @@ void TCreateQueueSchemaActorV2::CommitNewVersion() { .Uint64("SHARDS", RequiredShardsCount_) .Uint64("PARTITIONS", Request_.GetPartitions()) .Uint64("MASTER_TABLET_ID", LeaderTabletId_) - .Uint32("TABLES_FORMAT", TablesFormat_) + .Uint32("TABLES_FORMAT", TablesFormat_) .Uint64("VERSION", Version_) - .Uint64("QUEUE_ID_NUMBER_HASH", GetHash(Version_)) + .Uint64("QUEUE_ID_NUMBER_HASH", GetHash(Version_)) .Uint64("MAX_SIZE", *ValidatedAttributes_.MaximumMessageSize) .Uint64("DELAY", SecondsToMs(*ValidatedAttributes_.DelaySeconds)) .Uint64("VISIBILITY", SecondsToMs(*ValidatedAttributes_.VisibilityTimeout)) diff --git a/ydb/core/ymq/actor/queue_schema.h b/ydb/core/ymq/actor/queue_schema.h index 8cbdf86420..793ba1c06a 100644 --- a/ydb/core/ymq/actor/queue_schema.h +++ b/ydb/core/ymq/actor/queue_schema.h @@ -47,7 +47,7 @@ public: void RequestCreateQueueQuota(); void OnCreateQueueQuota(TEvQuota::TEvClearance::TPtr& ev); - void RequestTablesFormatSettings(const TString& accountName); + void RequestTablesFormatSettings(const TString& accountName); void RegisterMakeDirActor(const TString& workingDir, const TString& dirName); void RequestLeaderTabletId(); @@ -62,9 +62,9 @@ public: void OnDescribeSchemeResult(NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult::TPtr& ev); - void SendDescribeTable(); - void HandleTableDescription(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev); - + void SendDescribeTable(); + void HandleTableDescription(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev); + template <typename T> T GetAttribute(const TStringBuf name, const T& defaultValue) const; @@ -92,12 +92,12 @@ public: private: enum class ECreateComponentsStep : ui32 { - GetTablesFormatSetting, - MakeQueueDir, + GetTablesFormatSetting, + MakeQueueDir, MakeQueueVersionDir, MakeShards, MakeTables, - DescribeTableForSetSchemeShardId, + DescribeTableForSetSchemeShardId, DiscoverLeaderTabletId, AddQuoterResource, }; @@ -115,7 +115,7 @@ private: bool IsCloudMode_ = false; bool EnableQueueAttributesValidation_ = true; - ui32 TablesFormat_ = 0; + ui32 TablesFormat_ = 0; ui64 Version_ = 0; TString VersionName_; @@ -134,7 +134,7 @@ private: ui64 CreateTableWithLeaderTabletTxId_ = 0; std::pair<ui64, ui64> TableWithLeaderPathId_ = std::make_pair(0, 0); // (scheme shard, path id) are required for describing table - ECreateComponentsStep CurrentCreationStep_ = ECreateComponentsStep::GetTablesFormatSetting; + ECreateComponentsStep CurrentCreationStep_ = ECreateComponentsStep::GetTablesFormatSetting; TActorId AddQuoterResourceActor_; }; diff --git a/ydb/core/ymq/actor/ut/metering_ut.cpp b/ydb/core/ymq/actor/ut/metering_ut.cpp index 4fc150676b..bb6cbc6458 100644 --- a/ydb/core/ymq/actor/ut/metering_ut.cpp +++ b/ydb/core/ymq/actor/ut/metering_ut.cpp @@ -1,352 +1,352 @@ #include <ydb/core/ymq/actor/serviceid.h> - + #include <ydb/core/ymq/http/types.h> #include <ydb/core/ymq/actor/events.h> #include <ydb/core/ymq/actor/metering.h> - + #include <ydb/core/mind/address_classification/net_classifier.h> #include <ydb/core/testlib/test_client.h> - -#include <library/cpp/testing/unittest/registar.h> -#include <library/cpp/logger/global/global.h> - -#include <util/generic/serialized_enum.h> -#include <util/stream/file.h> - -namespace NKikimr::NSQS { - -using namespace Tests; - -bool IsSameRecords(const NSc::TValue& a, const NSc::TValue& b) { - auto cmp = [&](const TString& key, const TString& subkey = "") { - auto& va = a.Get(key); - auto& vb = b.Get(key); - if (subkey) { - return NSc::TValue::Equal(va.Get(subkey), vb.Get(subkey)); - } - return NSc::TValue::Equal(va, vb); - }; - return cmp("folder_id") && cmp("resource_id") && cmp("schema") && cmp("version") - && cmp("usage", "type") && cmp("usage", "unit") && cmp("usage", "quantity") - && cmp("tags", "direction") && cmp("tags", "type"); -} - -TVector<NSc::TValue>::const_iterator MatchBillingRecord(const TVector<NSc::TValue>& records, const NSc::TValue& expectedRecord) { - for (auto it = records.begin(); it != records.end(); ++it) { - if (IsSameRecords(*it, expectedRecord)) { - return it; - } - } - return records.end(); -} - -void CheckBillingRecord(const TVector<NSc::TValue>& records, const TVector<NSc::TValue>& expectedRecords) { - UNIT_ASSERT_VALUES_EQUAL(records.size(), expectedRecords.size()); - TVector<NSc::TValue> unmatchedRecords = records; - for (auto& record : expectedRecords) { - auto it = MatchBillingRecord(unmatchedRecords, record); - UNIT_ASSERT(unmatchedRecords.end() != it); - unmatchedRecords.erase(it); - } -} - -TVector<NSc::TValue> LoadBillingRecords(const TString& filepath) { - TString data = TFileInput(filepath).ReadAll(); - auto rawRecords = SplitString(data, "\n"); - TVector<NSc::TValue> records; - for (auto& record : rawRecords) { - records.push_back(NSc::TValue::FromJson(record)); - } - return records; -} - -void AddRequestToQueue( - TList<THolder<TSqsEvents::TEvReportProcessedRequestAttributes>>& requests, - const TString sourceAddress, - ui32 requestSizeInBytes, - ui32 responseSizeInBytes, - int statusCode, - const TString& folderId = "", - const TString& resourceId = "", - bool isFifo = false -) { - auto request = MakeHolder<TSqsEvents::TEvReportProcessedRequestAttributes>(); - TProcessedRequestAttributes& requestAttributes = request->Data; - requestAttributes.HttpStatusCode = statusCode; - requestAttributes.IsFifo = isFifo; - requestAttributes.FolderId = folderId; - requestAttributes.RequestSizeInBytes = requestSizeInBytes; - requestAttributes.ResponseSizeInBytes = responseSizeInBytes; - requestAttributes.SourceAddress = sourceAddress; - requestAttributes.ResourceId = resourceId; - requestAttributes.Action = EAction::ModifyPermissions; - - requests.push_back(std::move(request)); -} - -auto AddToExpectedRecords(TVector<NSc::TValue>& expectedRecords, const TProcessedRequestAttributes& attrs, TProcessedRequestsAggregator::ENetworkClass networkClass) { - { - NSc::TValue tags; - tags.SetDict(); - tags["type"] = ToString(networkClass); - tags["direction"] = ToString(TProcessedRequestsAggregator::ETrafficType::ingress); - expectedRecords.push_back(CreateMeteringBillingRecord( - attrs.FolderId, attrs.ResourceId, - "ymq.traffic.v1", "" /*fqdn*/, TInstant::Now(), attrs.RequestSizeInBytes, "byte", tags - )); - } - { - NSc::TValue tags; - tags.SetDict(); - tags["type"] = ToString(networkClass); - tags["direction"] = ToString(TProcessedRequestsAggregator::ETrafficType::egress); - expectedRecords.push_back(CreateMeteringBillingRecord( - attrs.FolderId, attrs.ResourceId, - "ymq.traffic.v1", "" /*fqdn*/, TInstant::Now(), attrs.ResponseSizeInBytes, "byte", tags - )); - } - { - NSc::TValue tags; - tags.SetDict(); - tags["queue_type"] = ToString(TProcessedRequestsAggregator::EQueueType::other); - expectedRecords.push_back(CreateMeteringBillingRecord( - attrs.FolderId, attrs.ResourceId, - "ymq.requests.v1", "" /*fqdn*/, TInstant::Now(), attrs.RequestSizeInBytes, "request", tags - )); - } -} - -Y_UNIT_TEST_SUITE(Metering) { - Y_UNIT_TEST(BillingRecords) { - TServerSettings serverSettings(0); - TServer server = TServer(serverSettings, true); - TTestActorRuntime* runtime = server.GetRuntime(); - - auto& sqsConfig = runtime->GetAppData().SqsConfig; - - sqsConfig.SetMeteringFlushingIntervalMs(100); - sqsConfig.SetMeteringLogFilePath("sqs_meterig.log"); - TFsPath(sqsConfig.GetMeteringLogFilePath()).DeleteIfExists(); - - sqsConfig.AddMeteringCloudNetCidr("5.45.196.0/24"); - sqsConfig.AddMeteringCloudNetCidr("2a0d:d6c0::/29"); - sqsConfig.AddMeteringYandexNetCidr("127.0.0.0/8"); - sqsConfig.AddMeteringYandexNetCidr("5.45.217.0/24"); - - DoInitGlobalLog(CreateOwningThreadedLogBackend(sqsConfig.GetMeteringLogFilePath(), 0)); - - runtime->RegisterService( - MakeSqsMeteringServiceID(), - runtime->Register(NSQS::CreateSqsMeteringService()) - ); - - TList<THolder<TSqsEvents::TEvReportProcessedRequestAttributes>> requests; - TVector<NSc::TValue> expectedRecords; - - - - AddRequestToQueue(requests, "5.45.196.0", 1, 2, 200, "folder1"); - AddToExpectedRecords(expectedRecords, requests.back()->Data, TProcessedRequestsAggregator::ENetworkClass::cloud); - AddRequestToQueue(requests, "5.45.196.0", 10, 20, 500, "folder1"); - AddRequestToQueue(requests, "5.45.196.0", 100, 200, 200); - - AddRequestToQueue(requests, "5.45.217.0", 1, 2, 200, "folder2"); - AddToExpectedRecords(expectedRecords, requests.back()->Data, TProcessedRequestsAggregator::ENetworkClass::yandex); - AddRequestToQueue(requests, "5.45.217.0", 10, 20, 500, "folder2"); - AddRequestToQueue(requests, "5.45.217.0", 100, 200, 200); - - AddRequestToQueue(requests, "5.45.215.192", 1, 2, 200, "folder3"); - AddToExpectedRecords(expectedRecords, requests.back()->Data, TProcessedRequestsAggregator::ENetworkClass::inet); - AddRequestToQueue(requests, "5.45.215.192", 10, 20, 500, "folder3"); - AddRequestToQueue(requests, "5.45.215.192", 100, 200, 200); - - for (auto& req : requests) { - runtime->Send(new IEventHandle( - MakeSqsMeteringServiceID(), runtime->AllocateEdgeActor(), req.Release() - )); - } - - Sleep(TDuration::Seconds(5)); - TVector<NSc::TValue> records = LoadBillingRecords(sqsConfig.GetMeteringLogFilePath()); - CheckBillingRecord(records, expectedRecords); - } - - Y_UNIT_TEST(MockedNetClassifierOnly) { - TServerSettings serverSettings(0); - TServer server = TServer(serverSettings, true); - TTestActorRuntime* runtime = server.GetRuntime(); - - auto& sqsConfig = runtime->GetAppData().SqsConfig; - - sqsConfig.SetMeteringByNetClassifierOnly(true); - sqsConfig.SetMeteringFlushingIntervalMs(100); - sqsConfig.SetMeteringLogFilePath("sqs_meterig.log"); - TFsPath(sqsConfig.GetMeteringLogFilePath()).DeleteIfExists(); - - sqsConfig.AddMeteringCloudNetCidr("5.45.196.0/24"); - sqsConfig.AddMeteringYandexNetCidr("5.45.217.0/24"); - - DoInitGlobalLog(CreateOwningThreadedLogBackend(sqsConfig.GetMeteringLogFilePath(), 0)); - - runtime->RegisterService( - MakeSqsMeteringServiceID(), - runtime->Register(NSQS::CreateSqsMeteringService()) - ); - - TList<THolder<TSqsEvents::TEvReportProcessedRequestAttributes>> requests; - TVector<NSc::TValue> expectedRecords; - - { - AddRequestToQueue(requests, "5.45.196.0", 1, 2, 200, "folder1"); - AddToExpectedRecords(expectedRecords, requests.back()->Data, TProcessedRequestsAggregator::ENetworkClass::unknown); - AddRequestToQueue(requests, "5.45.196.0", 10, 20, 500, "folder1"); - AddRequestToQueue(requests, "5.45.196.0", 100, 200, 200); - - AddRequestToQueue(requests, "5.45.217.0", 1, 2, 200, "folder2"); - AddToExpectedRecords(expectedRecords, requests.back()->Data, TProcessedRequestsAggregator::ENetworkClass::unknown); - AddRequestToQueue(requests, "5.45.217.0", 10, 20, 500, "folder2"); - AddRequestToQueue(requests, "5.45.217.0", 100, 200, 200); - - AddRequestToQueue(requests, "5.45.215.192", 1, 2, 200, "folder3"); - AddToExpectedRecords(expectedRecords, requests.back()->Data, TProcessedRequestsAggregator::ENetworkClass::unknown); - AddRequestToQueue(requests, "5.45.215.192", 10, 20, 500, "folder3"); - AddRequestToQueue(requests, "5.45.215.192", 100, 200, 200); - } - - for (auto& req : requests) { - runtime->Send(new IEventHandle( - MakeSqsMeteringServiceID(), runtime->AllocateEdgeActor(), req.Release() - )); - } - - Sleep(TDuration::Seconds(5)); - TVector<NSc::TValue> records = LoadBillingRecords(sqsConfig.GetMeteringLogFilePath()); - CheckBillingRecord(records, expectedRecords); - - { - NAddressClassifier::TAddressClassifier rawClassifier; - rawClassifier.AddNetByCidrAndLabel("127.0.0.0/8" , TProcessedRequestsAggregator::ENetworkClass::cloud); - rawClassifier.AddNetByCidrAndLabel("2a0d:d6c0::/29" , TProcessedRequestsAggregator::ENetworkClass::yandex); - - auto enumValues = GetEnumNames<TProcessedRequestsAggregator::ENetworkClass>(); - TVector<TString> labels(enumValues.size()); - for (auto enumItem : enumValues) { - Y_VERIFY(enumItem.first < labels.size()); - labels[enumItem.first] = enumItem.second; - } - auto classifier = NAddressClassifier::TLabeledAddressClassifier::MakeLabeledAddressClassifier(std::move(rawClassifier), std::move(labels)); - - - auto classifierUpdateRequest = MakeHolder<NNetClassifier::TEvNetClassifier::TEvClassifierUpdate>(); - classifierUpdateRequest->Classifier = classifier; - classifierUpdateRequest->NetDataUpdateTimestamp = TInstant::Now(); - runtime->Send(new IEventHandle( - MakeSqsMeteringServiceID(), runtime->AllocateEdgeActor(), classifierUpdateRequest.Release() - )); - } - Sleep(TDuration::MilliSeconds(100)); - { - requests.clear(); - AddRequestToQueue(requests, "5.45.196.0", 1, 2, 200, "folder4"); - AddToExpectedRecords(expectedRecords, requests.back()->Data, TProcessedRequestsAggregator::ENetworkClass::inet); - AddRequestToQueue(requests, "5.45.196.0", 10, 20, 500, "folder4"); - AddRequestToQueue(requests, "5.45.196.0", 100, 200, 200); - - AddRequestToQueue(requests, "5.45.217.0", 1, 2, 200, "folder5"); - AddToExpectedRecords(expectedRecords, requests.back()->Data, TProcessedRequestsAggregator::ENetworkClass::inet); - AddRequestToQueue(requests, "5.45.217.0", 10, 20, 500, "folder5"); - AddRequestToQueue(requests, "5.45.217.0", 100, 200, 200); - - AddRequestToQueue(requests, "5.45.215.192", 1, 2, 200, "folder6"); - AddToExpectedRecords(expectedRecords, requests.back()->Data, TProcessedRequestsAggregator::ENetworkClass::inet); - AddRequestToQueue(requests, "5.45.215.192", 10, 20, 500, "folder6"); - AddRequestToQueue(requests, "5.45.215.192", 100, 200, 200); - - AddRequestToQueue(requests, "127.255.255.255", 1, 2, 200, "folder7"); - AddToExpectedRecords(expectedRecords, requests.back()->Data, TProcessedRequestsAggregator::ENetworkClass::cloud); - AddRequestToQueue(requests, "127.255.255.255", 10, 20, 500, "folder7"); - AddRequestToQueue(requests, "127.255.255.255", 100, 200, 200); - - AddRequestToQueue(requests, "2a0d:d6c0:bbbb:bbbb:bbbb:bbbb:bbbb:bbbb", 1, 2, 200, "folder8"); - AddToExpectedRecords(expectedRecords, requests.back()->Data, TProcessedRequestsAggregator::ENetworkClass::yandex); - AddRequestToQueue(requests, "2a0d:d6c0:bbbb:bbbb:bbbb:bbbb:bbbb:bbbb", 10, 20, 500, "folder8"); - AddRequestToQueue(requests, "2a0d:d6c0:bbbb:bbbb:bbbb:bbbb:bbbb:bbbb", 100, 200, 200); - } - for (auto& req : requests) { - runtime->Send(new IEventHandle( - MakeSqsMeteringServiceID(), runtime->AllocateEdgeActor(), req.Release() - )); - } - - Sleep(TDuration::Seconds(5)); - records = LoadBillingRecords(sqsConfig.GetMeteringLogFilePath()); - CheckBillingRecord(records, expectedRecords); - } - - Y_UNIT_TEST(MockedNetClassifierLabelTransformation) { - TServerSettings serverSettings(0); - TServer server = TServer(serverSettings, true); - TTestActorRuntime* runtime = server.GetRuntime(); - - auto& sqsConfig = runtime->GetAppData().SqsConfig; - - sqsConfig.SetMeteringByNetClassifierOnly(true); - sqsConfig.SetMeteringFlushingIntervalMs(100); - sqsConfig.SetMeteringLogFilePath("sqs_meterig.log"); - TFsPath(sqsConfig.GetMeteringLogFilePath()).DeleteIfExists(); - - DoInitGlobalLog(CreateOwningThreadedLogBackend(sqsConfig.GetMeteringLogFilePath(), 0)); - - runtime->RegisterService( - MakeSqsMeteringServiceID(), - runtime->Register(NSQS::CreateSqsMeteringService()) - ); - Sleep(TDuration::MilliSeconds(500)); // waiting to override classifier from real NetClassifier service - { - NAddressClassifier::TAddressClassifier rawClassifier; - TVector<std::pair<TString, TString>> cidrWithLabel = { - {"127.0.0.0/8", ToString(TProcessedRequestsAggregator::ENetworkClass::yandex)}, - {"2a0d:d6c0::/29", ToString(TProcessedRequestsAggregator::ENetworkClass::cloud)}, - {"5.45.196.0/24", "yacloud"} - }; - TVector<TString> labels(cidrWithLabel.size()); - for (ui32 index = 0; index < cidrWithLabel.size(); ++index) { - auto& [cidr, label] = cidrWithLabel[index]; - rawClassifier.AddNetByCidrAndLabel(cidr , index); - labels[index] = label; - } - auto classifier = NAddressClassifier::TLabeledAddressClassifier::MakeLabeledAddressClassifier(std::move(rawClassifier), std::move(labels)); - - auto classifierUpdateRequest = MakeHolder<NNetClassifier::TEvNetClassifier::TEvClassifierUpdate>(); - classifierUpdateRequest->Classifier = classifier; - classifierUpdateRequest->NetDataUpdateTimestamp = TInstant::Now(); - runtime->Send(new IEventHandle( - MakeSqsMeteringServiceID(), runtime->AllocateEdgeActor(), classifierUpdateRequest.Release() - )); - } - Sleep(TDuration::MilliSeconds(100)); - TList<THolder<TSqsEvents::TEvReportProcessedRequestAttributes>> requests; - TVector<NSc::TValue> expectedRecords; - { - AddRequestToQueue(requests, "127.255.255.255", 1, 2, 200, "folder1"); - AddToExpectedRecords(expectedRecords, requests.back()->Data, TProcessedRequestsAggregator::ENetworkClass::yandex); - - AddRequestToQueue(requests, "2a0d:d6c0:bbbb:bbbb:bbbb:bbbb:bbbb:bbbb", 1, 2, 200, "folder2"); - AddToExpectedRecords(expectedRecords, requests.back()->Data, TProcessedRequestsAggregator::ENetworkClass::cloud); - - AddRequestToQueue(requests, "5.45.196.0", 1, 2, 200, "folder3"); - AddToExpectedRecords(expectedRecords, requests.back()->Data, TProcessedRequestsAggregator::ENetworkClass::cloud); - } - for (auto& req : requests) { - runtime->Send(new IEventHandle( - MakeSqsMeteringServiceID(), runtime->AllocateEdgeActor(), req.Release() - )); - } - - Sleep(TDuration::Seconds(5)); - auto records = LoadBillingRecords(sqsConfig.GetMeteringLogFilePath()); - CheckBillingRecord(records, expectedRecords); - } -} - -} // NKikimr::NSQS + +#include <library/cpp/testing/unittest/registar.h> +#include <library/cpp/logger/global/global.h> + +#include <util/generic/serialized_enum.h> +#include <util/stream/file.h> + +namespace NKikimr::NSQS { + +using namespace Tests; + +bool IsSameRecords(const NSc::TValue& a, const NSc::TValue& b) { + auto cmp = [&](const TString& key, const TString& subkey = "") { + auto& va = a.Get(key); + auto& vb = b.Get(key); + if (subkey) { + return NSc::TValue::Equal(va.Get(subkey), vb.Get(subkey)); + } + return NSc::TValue::Equal(va, vb); + }; + return cmp("folder_id") && cmp("resource_id") && cmp("schema") && cmp("version") + && cmp("usage", "type") && cmp("usage", "unit") && cmp("usage", "quantity") + && cmp("tags", "direction") && cmp("tags", "type"); +} + +TVector<NSc::TValue>::const_iterator MatchBillingRecord(const TVector<NSc::TValue>& records, const NSc::TValue& expectedRecord) { + for (auto it = records.begin(); it != records.end(); ++it) { + if (IsSameRecords(*it, expectedRecord)) { + return it; + } + } + return records.end(); +} + +void CheckBillingRecord(const TVector<NSc::TValue>& records, const TVector<NSc::TValue>& expectedRecords) { + UNIT_ASSERT_VALUES_EQUAL(records.size(), expectedRecords.size()); + TVector<NSc::TValue> unmatchedRecords = records; + for (auto& record : expectedRecords) { + auto it = MatchBillingRecord(unmatchedRecords, record); + UNIT_ASSERT(unmatchedRecords.end() != it); + unmatchedRecords.erase(it); + } +} + +TVector<NSc::TValue> LoadBillingRecords(const TString& filepath) { + TString data = TFileInput(filepath).ReadAll(); + auto rawRecords = SplitString(data, "\n"); + TVector<NSc::TValue> records; + for (auto& record : rawRecords) { + records.push_back(NSc::TValue::FromJson(record)); + } + return records; +} + +void AddRequestToQueue( + TList<THolder<TSqsEvents::TEvReportProcessedRequestAttributes>>& requests, + const TString sourceAddress, + ui32 requestSizeInBytes, + ui32 responseSizeInBytes, + int statusCode, + const TString& folderId = "", + const TString& resourceId = "", + bool isFifo = false +) { + auto request = MakeHolder<TSqsEvents::TEvReportProcessedRequestAttributes>(); + TProcessedRequestAttributes& requestAttributes = request->Data; + requestAttributes.HttpStatusCode = statusCode; + requestAttributes.IsFifo = isFifo; + requestAttributes.FolderId = folderId; + requestAttributes.RequestSizeInBytes = requestSizeInBytes; + requestAttributes.ResponseSizeInBytes = responseSizeInBytes; + requestAttributes.SourceAddress = sourceAddress; + requestAttributes.ResourceId = resourceId; + requestAttributes.Action = EAction::ModifyPermissions; + + requests.push_back(std::move(request)); +} + +auto AddToExpectedRecords(TVector<NSc::TValue>& expectedRecords, const TProcessedRequestAttributes& attrs, TProcessedRequestsAggregator::ENetworkClass networkClass) { + { + NSc::TValue tags; + tags.SetDict(); + tags["type"] = ToString(networkClass); + tags["direction"] = ToString(TProcessedRequestsAggregator::ETrafficType::ingress); + expectedRecords.push_back(CreateMeteringBillingRecord( + attrs.FolderId, attrs.ResourceId, + "ymq.traffic.v1", "" /*fqdn*/, TInstant::Now(), attrs.RequestSizeInBytes, "byte", tags + )); + } + { + NSc::TValue tags; + tags.SetDict(); + tags["type"] = ToString(networkClass); + tags["direction"] = ToString(TProcessedRequestsAggregator::ETrafficType::egress); + expectedRecords.push_back(CreateMeteringBillingRecord( + attrs.FolderId, attrs.ResourceId, + "ymq.traffic.v1", "" /*fqdn*/, TInstant::Now(), attrs.ResponseSizeInBytes, "byte", tags + )); + } + { + NSc::TValue tags; + tags.SetDict(); + tags["queue_type"] = ToString(TProcessedRequestsAggregator::EQueueType::other); + expectedRecords.push_back(CreateMeteringBillingRecord( + attrs.FolderId, attrs.ResourceId, + "ymq.requests.v1", "" /*fqdn*/, TInstant::Now(), attrs.RequestSizeInBytes, "request", tags + )); + } +} + +Y_UNIT_TEST_SUITE(Metering) { + Y_UNIT_TEST(BillingRecords) { + TServerSettings serverSettings(0); + TServer server = TServer(serverSettings, true); + TTestActorRuntime* runtime = server.GetRuntime(); + + auto& sqsConfig = runtime->GetAppData().SqsConfig; + + sqsConfig.SetMeteringFlushingIntervalMs(100); + sqsConfig.SetMeteringLogFilePath("sqs_meterig.log"); + TFsPath(sqsConfig.GetMeteringLogFilePath()).DeleteIfExists(); + + sqsConfig.AddMeteringCloudNetCidr("5.45.196.0/24"); + sqsConfig.AddMeteringCloudNetCidr("2a0d:d6c0::/29"); + sqsConfig.AddMeteringYandexNetCidr("127.0.0.0/8"); + sqsConfig.AddMeteringYandexNetCidr("5.45.217.0/24"); + + DoInitGlobalLog(CreateOwningThreadedLogBackend(sqsConfig.GetMeteringLogFilePath(), 0)); + + runtime->RegisterService( + MakeSqsMeteringServiceID(), + runtime->Register(NSQS::CreateSqsMeteringService()) + ); + + TList<THolder<TSqsEvents::TEvReportProcessedRequestAttributes>> requests; + TVector<NSc::TValue> expectedRecords; + + + + AddRequestToQueue(requests, "5.45.196.0", 1, 2, 200, "folder1"); + AddToExpectedRecords(expectedRecords, requests.back()->Data, TProcessedRequestsAggregator::ENetworkClass::cloud); + AddRequestToQueue(requests, "5.45.196.0", 10, 20, 500, "folder1"); + AddRequestToQueue(requests, "5.45.196.0", 100, 200, 200); + + AddRequestToQueue(requests, "5.45.217.0", 1, 2, 200, "folder2"); + AddToExpectedRecords(expectedRecords, requests.back()->Data, TProcessedRequestsAggregator::ENetworkClass::yandex); + AddRequestToQueue(requests, "5.45.217.0", 10, 20, 500, "folder2"); + AddRequestToQueue(requests, "5.45.217.0", 100, 200, 200); + + AddRequestToQueue(requests, "5.45.215.192", 1, 2, 200, "folder3"); + AddToExpectedRecords(expectedRecords, requests.back()->Data, TProcessedRequestsAggregator::ENetworkClass::inet); + AddRequestToQueue(requests, "5.45.215.192", 10, 20, 500, "folder3"); + AddRequestToQueue(requests, "5.45.215.192", 100, 200, 200); + + for (auto& req : requests) { + runtime->Send(new IEventHandle( + MakeSqsMeteringServiceID(), runtime->AllocateEdgeActor(), req.Release() + )); + } + + Sleep(TDuration::Seconds(5)); + TVector<NSc::TValue> records = LoadBillingRecords(sqsConfig.GetMeteringLogFilePath()); + CheckBillingRecord(records, expectedRecords); + } + + Y_UNIT_TEST(MockedNetClassifierOnly) { + TServerSettings serverSettings(0); + TServer server = TServer(serverSettings, true); + TTestActorRuntime* runtime = server.GetRuntime(); + + auto& sqsConfig = runtime->GetAppData().SqsConfig; + + sqsConfig.SetMeteringByNetClassifierOnly(true); + sqsConfig.SetMeteringFlushingIntervalMs(100); + sqsConfig.SetMeteringLogFilePath("sqs_meterig.log"); + TFsPath(sqsConfig.GetMeteringLogFilePath()).DeleteIfExists(); + + sqsConfig.AddMeteringCloudNetCidr("5.45.196.0/24"); + sqsConfig.AddMeteringYandexNetCidr("5.45.217.0/24"); + + DoInitGlobalLog(CreateOwningThreadedLogBackend(sqsConfig.GetMeteringLogFilePath(), 0)); + + runtime->RegisterService( + MakeSqsMeteringServiceID(), + runtime->Register(NSQS::CreateSqsMeteringService()) + ); + + TList<THolder<TSqsEvents::TEvReportProcessedRequestAttributes>> requests; + TVector<NSc::TValue> expectedRecords; + + { + AddRequestToQueue(requests, "5.45.196.0", 1, 2, 200, "folder1"); + AddToExpectedRecords(expectedRecords, requests.back()->Data, TProcessedRequestsAggregator::ENetworkClass::unknown); + AddRequestToQueue(requests, "5.45.196.0", 10, 20, 500, "folder1"); + AddRequestToQueue(requests, "5.45.196.0", 100, 200, 200); + + AddRequestToQueue(requests, "5.45.217.0", 1, 2, 200, "folder2"); + AddToExpectedRecords(expectedRecords, requests.back()->Data, TProcessedRequestsAggregator::ENetworkClass::unknown); + AddRequestToQueue(requests, "5.45.217.0", 10, 20, 500, "folder2"); + AddRequestToQueue(requests, "5.45.217.0", 100, 200, 200); + + AddRequestToQueue(requests, "5.45.215.192", 1, 2, 200, "folder3"); + AddToExpectedRecords(expectedRecords, requests.back()->Data, TProcessedRequestsAggregator::ENetworkClass::unknown); + AddRequestToQueue(requests, "5.45.215.192", 10, 20, 500, "folder3"); + AddRequestToQueue(requests, "5.45.215.192", 100, 200, 200); + } + + for (auto& req : requests) { + runtime->Send(new IEventHandle( + MakeSqsMeteringServiceID(), runtime->AllocateEdgeActor(), req.Release() + )); + } + + Sleep(TDuration::Seconds(5)); + TVector<NSc::TValue> records = LoadBillingRecords(sqsConfig.GetMeteringLogFilePath()); + CheckBillingRecord(records, expectedRecords); + + { + NAddressClassifier::TAddressClassifier rawClassifier; + rawClassifier.AddNetByCidrAndLabel("127.0.0.0/8" , TProcessedRequestsAggregator::ENetworkClass::cloud); + rawClassifier.AddNetByCidrAndLabel("2a0d:d6c0::/29" , TProcessedRequestsAggregator::ENetworkClass::yandex); + + auto enumValues = GetEnumNames<TProcessedRequestsAggregator::ENetworkClass>(); + TVector<TString> labels(enumValues.size()); + for (auto enumItem : enumValues) { + Y_VERIFY(enumItem.first < labels.size()); + labels[enumItem.first] = enumItem.second; + } + auto classifier = NAddressClassifier::TLabeledAddressClassifier::MakeLabeledAddressClassifier(std::move(rawClassifier), std::move(labels)); + + + auto classifierUpdateRequest = MakeHolder<NNetClassifier::TEvNetClassifier::TEvClassifierUpdate>(); + classifierUpdateRequest->Classifier = classifier; + classifierUpdateRequest->NetDataUpdateTimestamp = TInstant::Now(); + runtime->Send(new IEventHandle( + MakeSqsMeteringServiceID(), runtime->AllocateEdgeActor(), classifierUpdateRequest.Release() + )); + } + Sleep(TDuration::MilliSeconds(100)); + { + requests.clear(); + AddRequestToQueue(requests, "5.45.196.0", 1, 2, 200, "folder4"); + AddToExpectedRecords(expectedRecords, requests.back()->Data, TProcessedRequestsAggregator::ENetworkClass::inet); + AddRequestToQueue(requests, "5.45.196.0", 10, 20, 500, "folder4"); + AddRequestToQueue(requests, "5.45.196.0", 100, 200, 200); + + AddRequestToQueue(requests, "5.45.217.0", 1, 2, 200, "folder5"); + AddToExpectedRecords(expectedRecords, requests.back()->Data, TProcessedRequestsAggregator::ENetworkClass::inet); + AddRequestToQueue(requests, "5.45.217.0", 10, 20, 500, "folder5"); + AddRequestToQueue(requests, "5.45.217.0", 100, 200, 200); + + AddRequestToQueue(requests, "5.45.215.192", 1, 2, 200, "folder6"); + AddToExpectedRecords(expectedRecords, requests.back()->Data, TProcessedRequestsAggregator::ENetworkClass::inet); + AddRequestToQueue(requests, "5.45.215.192", 10, 20, 500, "folder6"); + AddRequestToQueue(requests, "5.45.215.192", 100, 200, 200); + + AddRequestToQueue(requests, "127.255.255.255", 1, 2, 200, "folder7"); + AddToExpectedRecords(expectedRecords, requests.back()->Data, TProcessedRequestsAggregator::ENetworkClass::cloud); + AddRequestToQueue(requests, "127.255.255.255", 10, 20, 500, "folder7"); + AddRequestToQueue(requests, "127.255.255.255", 100, 200, 200); + + AddRequestToQueue(requests, "2a0d:d6c0:bbbb:bbbb:bbbb:bbbb:bbbb:bbbb", 1, 2, 200, "folder8"); + AddToExpectedRecords(expectedRecords, requests.back()->Data, TProcessedRequestsAggregator::ENetworkClass::yandex); + AddRequestToQueue(requests, "2a0d:d6c0:bbbb:bbbb:bbbb:bbbb:bbbb:bbbb", 10, 20, 500, "folder8"); + AddRequestToQueue(requests, "2a0d:d6c0:bbbb:bbbb:bbbb:bbbb:bbbb:bbbb", 100, 200, 200); + } + for (auto& req : requests) { + runtime->Send(new IEventHandle( + MakeSqsMeteringServiceID(), runtime->AllocateEdgeActor(), req.Release() + )); + } + + Sleep(TDuration::Seconds(5)); + records = LoadBillingRecords(sqsConfig.GetMeteringLogFilePath()); + CheckBillingRecord(records, expectedRecords); + } + + Y_UNIT_TEST(MockedNetClassifierLabelTransformation) { + TServerSettings serverSettings(0); + TServer server = TServer(serverSettings, true); + TTestActorRuntime* runtime = server.GetRuntime(); + + auto& sqsConfig = runtime->GetAppData().SqsConfig; + + sqsConfig.SetMeteringByNetClassifierOnly(true); + sqsConfig.SetMeteringFlushingIntervalMs(100); + sqsConfig.SetMeteringLogFilePath("sqs_meterig.log"); + TFsPath(sqsConfig.GetMeteringLogFilePath()).DeleteIfExists(); + + DoInitGlobalLog(CreateOwningThreadedLogBackend(sqsConfig.GetMeteringLogFilePath(), 0)); + + runtime->RegisterService( + MakeSqsMeteringServiceID(), + runtime->Register(NSQS::CreateSqsMeteringService()) + ); + Sleep(TDuration::MilliSeconds(500)); // waiting to override classifier from real NetClassifier service + { + NAddressClassifier::TAddressClassifier rawClassifier; + TVector<std::pair<TString, TString>> cidrWithLabel = { + {"127.0.0.0/8", ToString(TProcessedRequestsAggregator::ENetworkClass::yandex)}, + {"2a0d:d6c0::/29", ToString(TProcessedRequestsAggregator::ENetworkClass::cloud)}, + {"5.45.196.0/24", "yacloud"} + }; + TVector<TString> labels(cidrWithLabel.size()); + for (ui32 index = 0; index < cidrWithLabel.size(); ++index) { + auto& [cidr, label] = cidrWithLabel[index]; + rawClassifier.AddNetByCidrAndLabel(cidr , index); + labels[index] = label; + } + auto classifier = NAddressClassifier::TLabeledAddressClassifier::MakeLabeledAddressClassifier(std::move(rawClassifier), std::move(labels)); + + auto classifierUpdateRequest = MakeHolder<NNetClassifier::TEvNetClassifier::TEvClassifierUpdate>(); + classifierUpdateRequest->Classifier = classifier; + classifierUpdateRequest->NetDataUpdateTimestamp = TInstant::Now(); + runtime->Send(new IEventHandle( + MakeSqsMeteringServiceID(), runtime->AllocateEdgeActor(), classifierUpdateRequest.Release() + )); + } + Sleep(TDuration::MilliSeconds(100)); + TList<THolder<TSqsEvents::TEvReportProcessedRequestAttributes>> requests; + TVector<NSc::TValue> expectedRecords; + { + AddRequestToQueue(requests, "127.255.255.255", 1, 2, 200, "folder1"); + AddToExpectedRecords(expectedRecords, requests.back()->Data, TProcessedRequestsAggregator::ENetworkClass::yandex); + + AddRequestToQueue(requests, "2a0d:d6c0:bbbb:bbbb:bbbb:bbbb:bbbb:bbbb", 1, 2, 200, "folder2"); + AddToExpectedRecords(expectedRecords, requests.back()->Data, TProcessedRequestsAggregator::ENetworkClass::cloud); + + AddRequestToQueue(requests, "5.45.196.0", 1, 2, 200, "folder3"); + AddToExpectedRecords(expectedRecords, requests.back()->Data, TProcessedRequestsAggregator::ENetworkClass::cloud); + } + for (auto& req : requests) { + runtime->Send(new IEventHandle( + MakeSqsMeteringServiceID(), runtime->AllocateEdgeActor(), req.Release() + )); + } + + Sleep(TDuration::Seconds(5)); + auto records = LoadBillingRecords(sqsConfig.GetMeteringLogFilePath()); + CheckBillingRecord(records, expectedRecords); + } +} + +} // NKikimr::NSQS diff --git a/ydb/core/ymq/actor/ut/ya.make b/ydb/core/ymq/actor/ut/ya.make index 1f41f730aa..a3b4f59f37 100644 --- a/ydb/core/ymq/actor/ut/ya.make +++ b/ydb/core/ymq/actor/ut/ya.make @@ -20,7 +20,7 @@ SRCS( infly_ut.cpp message_delay_stats_ut.cpp sha256_ut.cpp - metering_ut.cpp + metering_ut.cpp ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/ymq/base/counters.cpp b/ydb/core/ymq/base/counters.cpp index 77f203d97c..90a9f5000f 100644 --- a/ydb/core/ymq/base/counters.cpp +++ b/ydb/core/ymq/base/counters.cpp @@ -938,13 +938,13 @@ void TCloudAuthCounters::InitCounters(TIntrusivePtr<NMonitoring::TDynamicCounter INIT_HISTOGRAM_COUNTER(cloudAuthCounters, GetFolderIdDuration, ELifetime::Persistent, DurationBucketsMs, Lazy(*Cfg)); } -void TMeteringCounters::InitCounters(const TVector<TString>& classifierLabels) { - const TString classifierRequestsLabel("ClassifierRequests_"); - const TString idleClassifierRequestsLabel("IdleClassifierRequests_"); - for (auto label : classifierLabels) { - INIT_COUNTER_WITH_NAME(SqsMeteringCounters, ClassifierRequestsResults[label], TStringBuilder() << classifierRequestsLabel << label, ELifetime::Persistent, EValueType::Derivative, Lazy(Config)); - INIT_COUNTER_WITH_NAME(SqsMeteringCounters, IdleClassifierRequestsResults[label], TStringBuilder() << idleClassifierRequestsLabel << label, ELifetime::Persistent, EValueType::Derivative, Lazy(Config)); - } -} - +void TMeteringCounters::InitCounters(const TVector<TString>& classifierLabels) { + const TString classifierRequestsLabel("ClassifierRequests_"); + const TString idleClassifierRequestsLabel("IdleClassifierRequests_"); + for (auto label : classifierLabels) { + INIT_COUNTER_WITH_NAME(SqsMeteringCounters, ClassifierRequestsResults[label], TStringBuilder() << classifierRequestsLabel << label, ELifetime::Persistent, EValueType::Derivative, Lazy(Config)); + INIT_COUNTER_WITH_NAME(SqsMeteringCounters, IdleClassifierRequestsResults[label], TStringBuilder() << idleClassifierRequestsLabel << label, ELifetime::Persistent, EValueType::Derivative, Lazy(Config)); + } +} + } // namespace NKikimr::NSQS diff --git a/ydb/core/ymq/base/counters.h b/ydb/core/ymq/base/counters.h index b246fe4b90..3c2667b241 100644 --- a/ydb/core/ymq/base/counters.h +++ b/ydb/core/ymq/base/counters.h @@ -788,24 +788,24 @@ private: [GRPC_STATUSES_COUNT]; // 18 types. }; -// Metering counters in SQS core subsystem. -struct TMeteringCounters : public TAtomicRefCount<TMeteringCounters> { - THashMap<TString, TLazyCachedCounter> ClassifierRequestsResults; - THashMap<TString, TLazyCachedCounter> IdleClassifierRequestsResults; - - TMeteringCounters(const NKikimrConfig::TSqsConfig& config, const TIntrusivePtr<NMonitoring::TDynamicCounters>& sqsMeteringCounters, const TVector<TString>& classifierLabels) - : SqsMeteringCounters(sqsMeteringCounters) - , Config(config) - { - InitCounters(classifierLabels); - } - -private: - void InitCounters(const TVector<TString>& classifierLabels); - -private: - TIntrusivePtr<NMonitoring::TDynamicCounters> SqsMeteringCounters; - const NKikimrConfig::TSqsConfig& Config; -}; - +// Metering counters in SQS core subsystem. +struct TMeteringCounters : public TAtomicRefCount<TMeteringCounters> { + THashMap<TString, TLazyCachedCounter> ClassifierRequestsResults; + THashMap<TString, TLazyCachedCounter> IdleClassifierRequestsResults; + + TMeteringCounters(const NKikimrConfig::TSqsConfig& config, const TIntrusivePtr<NMonitoring::TDynamicCounters>& sqsMeteringCounters, const TVector<TString>& classifierLabels) + : SqsMeteringCounters(sqsMeteringCounters) + , Config(config) + { + InitCounters(classifierLabels); + } + +private: + void InitCounters(const TVector<TString>& classifierLabels); + +private: + TIntrusivePtr<NMonitoring::TDynamicCounters> SqsMeteringCounters; + const NKikimrConfig::TSqsConfig& Config; +}; + } // namespace NKikimr::NSQS diff --git a/ydb/core/ymq/base/ut/counters_ut.cpp b/ydb/core/ymq/base/ut/counters_ut.cpp index b82cd7cff2..5ff3d26e36 100644 --- a/ydb/core/ymq/base/ut/counters_ut.cpp +++ b/ydb/core/ymq/base/ut/counters_ut.cpp @@ -500,31 +500,31 @@ Y_UNIT_TEST_SUITE(HttpCountersTest) { } } -Y_UNIT_TEST_SUITE(MeteringCountersTest) { - Y_UNIT_TEST(CountersAggregationTest) { - NKikimrConfig::TSqsConfig config; - config.SetCreateLazyCounters(false); - TIntrusivePtr<NMonitoring::TDynamicCounters> metering = new NMonitoring::TDynamicCounters(); - TIntrusivePtr<TMeteringCounters> counters = new TMeteringCounters(config, metering, {"inet", "yandex", "unknown", "cloud"}); - *counters->ClassifierRequestsResults["inet"] = 33; - *counters->ClassifierRequestsResults["yandex"] = 42; - *counters->ClassifierRequestsResults["unknown"] = 113; - *counters->IdleClassifierRequestsResults["cloud"] = 128; - - AssertCounterValues(metering, - { - { "ConnectionsCount", 0 }, - { "ClassifierRequests_cloud", 0}, - { "ClassifierRequests_inet", 33}, - { "ClassifierRequests_unknown", 113}, - { "ClassifierRequests_yandex", 42}, - { "IdleClassifierRequests_cloud", 128}, - { "IdleClassifierRequests_inet", 0}, - { "IdleClassifierRequests_unknown", 0}, - { "IdleClassifierRequests_yandex", 0}, - { "RequestExceptions", 0} - }); - } -} - +Y_UNIT_TEST_SUITE(MeteringCountersTest) { + Y_UNIT_TEST(CountersAggregationTest) { + NKikimrConfig::TSqsConfig config; + config.SetCreateLazyCounters(false); + TIntrusivePtr<NMonitoring::TDynamicCounters> metering = new NMonitoring::TDynamicCounters(); + TIntrusivePtr<TMeteringCounters> counters = new TMeteringCounters(config, metering, {"inet", "yandex", "unknown", "cloud"}); + *counters->ClassifierRequestsResults["inet"] = 33; + *counters->ClassifierRequestsResults["yandex"] = 42; + *counters->ClassifierRequestsResults["unknown"] = 113; + *counters->IdleClassifierRequestsResults["cloud"] = 128; + + AssertCounterValues(metering, + { + { "ConnectionsCount", 0 }, + { "ClassifierRequests_cloud", 0}, + { "ClassifierRequests_inet", 33}, + { "ClassifierRequests_unknown", 113}, + { "ClassifierRequests_yandex", 42}, + { "IdleClassifierRequests_cloud", 128}, + { "IdleClassifierRequests_inet", 0}, + { "IdleClassifierRequests_unknown", 0}, + { "IdleClassifierRequests_yandex", 0}, + { "RequestExceptions", 0} + }); + } +} + } // namespace NKikimr::NSQS diff --git a/ydb/library/persqueue/counter_time_keeper/counter_time_keeper.cpp b/ydb/library/persqueue/counter_time_keeper/counter_time_keeper.cpp index 72867be7d6..64a6d6c5e2 100644 --- a/ydb/library/persqueue/counter_time_keeper/counter_time_keeper.cpp +++ b/ydb/library/persqueue/counter_time_keeper/counter_time_keeper.cpp @@ -1 +1 @@ -#include "counter_time_keeper.h" +#include "counter_time_keeper.h" diff --git a/ydb/library/persqueue/counter_time_keeper/counter_time_keeper.h b/ydb/library/persqueue/counter_time_keeper/counter_time_keeper.h index 0d4cf81479..313de598d3 100644 --- a/ydb/library/persqueue/counter_time_keeper/counter_time_keeper.h +++ b/ydb/library/persqueue/counter_time_keeper/counter_time_keeper.h @@ -1,22 +1,22 @@ -#pragma once - +#pragma once + #include <ydb/core/tablet/tablet_counters.h> - -#include <util/system/hp_timer.h> - -namespace NPersQueue { - -class TCounterTimeKeeper { -public: - TCounterTimeKeeper (NKikimr::TTabletCumulativeCounter& counter) - : Counter(counter) - {} - ~TCounterTimeKeeper() { - Counter += ui64(CpuTimer.PassedReset() * 1000000.); - } -private: - NKikimr::TTabletCumulativeCounter& Counter; - THPTimer CpuTimer; -}; - -} // namespace NPersQueue + +#include <util/system/hp_timer.h> + +namespace NPersQueue { + +class TCounterTimeKeeper { +public: + TCounterTimeKeeper (NKikimr::TTabletCumulativeCounter& counter) + : Counter(counter) + {} + ~TCounterTimeKeeper() { + Counter += ui64(CpuTimer.PassedReset() * 1000000.); + } +private: + NKikimr::TTabletCumulativeCounter& Counter; + THPTimer CpuTimer; +}; + +} // namespace NPersQueue diff --git a/ydb/library/persqueue/counter_time_keeper/ya.make b/ydb/library/persqueue/counter_time_keeper/ya.make index 2980fbcada..106ad89431 100644 --- a/ydb/library/persqueue/counter_time_keeper/ya.make +++ b/ydb/library/persqueue/counter_time_keeper/ya.make @@ -1,15 +1,15 @@ OWNER(g:logbroker) - -LIBRARY() - -PEERDIR( + +LIBRARY() + +PEERDIR( library/cpp/actors/protos ydb/core/protos -) - -SRCS( - counter_time_keeper.h - counter_time_keeper.cpp -) - -END() +) + +SRCS( + counter_time_keeper.h + counter_time_keeper.cpp +) + +END() diff --git a/ydb/library/persqueue/obfuscate/obfuscate.cpp b/ydb/library/persqueue/obfuscate/obfuscate.cpp index 30f9bcc068..c1a9b0b1a7 100644 --- a/ydb/library/persqueue/obfuscate/obfuscate.cpp +++ b/ydb/library/persqueue/obfuscate/obfuscate.cpp @@ -1,15 +1,15 @@ -#include "obfuscate.h" - -#include <util/generic/string.h> - -namespace NPersQueue { - -TString ObfuscateString(TString str) { - ui32 publicPartSize = Min<ui32>(4, str.size() / 4); - for (ui32 i = publicPartSize; i < str.size() - publicPartSize; ++i) { - str[i] = '*'; - } - return str; -} - -} // namespace NPersQueue +#include "obfuscate.h" + +#include <util/generic/string.h> + +namespace NPersQueue { + +TString ObfuscateString(TString str) { + ui32 publicPartSize = Min<ui32>(4, str.size() / 4); + for (ui32 i = publicPartSize; i < str.size() - publicPartSize; ++i) { + str[i] = '*'; + } + return str; +} + +} // namespace NPersQueue diff --git a/ydb/library/persqueue/obfuscate/obfuscate.h b/ydb/library/persqueue/obfuscate/obfuscate.h index 8fd8bf2ff2..5eb7302fe1 100644 --- a/ydb/library/persqueue/obfuscate/obfuscate.h +++ b/ydb/library/persqueue/obfuscate/obfuscate.h @@ -1,9 +1,9 @@ -#pragma once - -#include <util/generic/fwd.h> - -namespace NPersQueue { - -TString ObfuscateString(TString str); - -} // namespace NPersQueue +#pragma once + +#include <util/generic/fwd.h> + +namespace NPersQueue { + +TString ObfuscateString(TString str); + +} // namespace NPersQueue diff --git a/ydb/library/persqueue/obfuscate/ya.make b/ydb/library/persqueue/obfuscate/ya.make index f361e8ddf9..88df0eac37 100644 --- a/ydb/library/persqueue/obfuscate/ya.make +++ b/ydb/library/persqueue/obfuscate/ya.make @@ -1,10 +1,10 @@ OWNER(g:logbroker) - -LIBRARY() - -SRCS( - obfuscate.h - obfuscate.cpp -) - -END() + +LIBRARY() + +SRCS( + obfuscate.h + obfuscate.cpp +) + +END() diff --git a/ydb/library/persqueue/topic_parser/ya.make b/ydb/library/persqueue/topic_parser/ya.make index 200d6cde9a..2b4f126a59 100644 --- a/ydb/library/persqueue/topic_parser/ya.make +++ b/ydb/library/persqueue/topic_parser/ya.make @@ -1,23 +1,23 @@ -OWNER( +OWNER( komels - g:kikimr - g:logbroker -) - -LIBRARY() - -PEERDIR( + g:kikimr + g:logbroker +) + +LIBRARY() + +PEERDIR( ydb/core/base ydb/library/persqueue/topic_parser_public ydb/public/api/protos -) - -SRCS( - topic_parser.h - topic_parser.cpp -) +) + +SRCS( + topic_parser.h + topic_parser.cpp +) -END() +END() RECURSE_FOR_TESTS( ut diff --git a/ydb/library/persqueue/ya.make b/ydb/library/persqueue/ya.make index 6dc12584fe..d28331e52e 100644 --- a/ydb/library/persqueue/ya.make +++ b/ydb/library/persqueue/ya.make @@ -1,9 +1,9 @@ -OWNER(g:logbroker) - -RECURSE( - counter_time_keeper - obfuscate +OWNER(g:logbroker) + +RECURSE( + counter_time_keeper + obfuscate tests - topic_parser + topic_parser topic_parser_public ) diff --git a/ydb/library/pretty_types_print/protobuf/out.cpp b/ydb/library/pretty_types_print/protobuf/out.cpp index 624ae35086..1013108dd2 100644 --- a/ydb/library/pretty_types_print/protobuf/out.cpp +++ b/ydb/library/pretty_types_print/protobuf/out.cpp @@ -1,9 +1,9 @@ -#include <util/stream/output.h> -#include <util/string/join.h> - -#include <google/protobuf/repeated_field.h> - -Y_DECLARE_OUT_SPEC(, google::protobuf::RepeatedPtrField<TString>, stream, value) -{ - stream << JoinSeq(", ", value); -} +#include <util/stream/output.h> +#include <util/string/join.h> + +#include <google/protobuf/repeated_field.h> + +Y_DECLARE_OUT_SPEC(, google::protobuf::RepeatedPtrField<TString>, stream, value) +{ + stream << JoinSeq(", ", value); +} diff --git a/ydb/library/pretty_types_print/protobuf/ya.make b/ydb/library/pretty_types_print/protobuf/ya.make index 1c95e9e9e1..e41c037f03 100644 --- a/ydb/library/pretty_types_print/protobuf/ya.make +++ b/ydb/library/pretty_types_print/protobuf/ya.make @@ -1,13 +1,13 @@ -LIBRARY() - -OWNER(g:kikimr) - -SRCS( - out.cpp -) - -PEERDIR( - contrib/libs/protobuf -) - -END() +LIBRARY() + +OWNER(g:kikimr) + +SRCS( + out.cpp +) + +PEERDIR( + contrib/libs/protobuf +) + +END() diff --git a/ydb/library/pretty_types_print/wilson/out.cpp b/ydb/library/pretty_types_print/wilson/out.cpp index a1e88f0591..9b8c60f981 100644 --- a/ydb/library/pretty_types_print/wilson/out.cpp +++ b/ydb/library/pretty_types_print/wilson/out.cpp @@ -1,7 +1,7 @@ #include <ydb/library/wilson/wilson_trace.h> - -#include <util/stream/output.h> - -Y_DECLARE_OUT_SPEC(, NWilson::TTraceId, stream, value) { - value.OutputSpanId(stream); -} + +#include <util/stream/output.h> + +Y_DECLARE_OUT_SPEC(, NWilson::TTraceId, stream, value) { + value.OutputSpanId(stream); +} diff --git a/ydb/library/pretty_types_print/wilson/ya.make b/ydb/library/pretty_types_print/wilson/ya.make index 99e27e74c4..6dba9499b1 100644 --- a/ydb/library/pretty_types_print/wilson/ya.make +++ b/ydb/library/pretty_types_print/wilson/ya.make @@ -1,13 +1,13 @@ -LIBRARY() - -OWNER(g:kikimr) - -SRCS( - out.cpp -) - -PEERDIR( +LIBRARY() + +OWNER(g:kikimr) + +SRCS( + out.cpp +) + +PEERDIR( ydb/library/wilson -) - -END() +) + +END() diff --git a/ydb/library/pretty_types_print/ya.make b/ydb/library/pretty_types_print/ya.make index 191cca6931..957b484f05 100644 --- a/ydb/library/pretty_types_print/ya.make +++ b/ydb/library/pretty_types_print/ya.make @@ -1,4 +1,4 @@ -RECURSE( - protobuf - wilson -) +RECURSE( + protobuf + wilson +) diff --git a/ydb/public/api/protos/draft/persqueue_common.proto b/ydb/public/api/protos/draft/persqueue_common.proto index 3a0afbf2ca..87bb3f236d 100644 --- a/ydb/public/api/protos/draft/persqueue_common.proto +++ b/ydb/public/api/protos/draft/persqueue_common.proto @@ -1,42 +1,42 @@ -syntax = "proto3"; -import "google/protobuf/descriptor.proto"; +syntax = "proto3"; +import "google/protobuf/descriptor.proto"; import "ydb/public/api/protos/draft/persqueue_error_codes.proto"; - -package NPersQueueCommon; - -option java_package = "com.yandex.ydb.persqueue"; -option cc_enable_arenas = true; - -extend google.protobuf.FileOptions { - bool GenerateYaStyle = 66678; -} - -// TODO: Change to IssueMessage -message Error { - NPersQueue.NErrorCode.EErrorCode code = 1; - string description = 2; -} - -enum ECodec { - RAW = 0; - GZIP = 1; - LZOP = 2; - ZSTD = 3; - DEFAULT = 100; // TODO: move to pqlib -} - -/** - * Structure for storing client credentials. - * Client must provide this structre in first request (init of write\read) and in case of changing, - * e.g. client gets new TVM Service ticket once per hour. Not all requests (read/write) could contain TCredentials - server will take already known credentials. - * Provided auth credentials will be checked once per some time at serverside. This means that if TVM ticket is too old, then read\write session will die. - * If no auth provided at all - client will be using guest rights. So, reading/writing is still possible without credentials. - */ - -message Credentials { - oneof credentials { - bytes tvm_service_ticket = 1; - bytes oauth_token = 2; - } -} - + +package NPersQueueCommon; + +option java_package = "com.yandex.ydb.persqueue"; +option cc_enable_arenas = true; + +extend google.protobuf.FileOptions { + bool GenerateYaStyle = 66678; +} + +// TODO: Change to IssueMessage +message Error { + NPersQueue.NErrorCode.EErrorCode code = 1; + string description = 2; +} + +enum ECodec { + RAW = 0; + GZIP = 1; + LZOP = 2; + ZSTD = 3; + DEFAULT = 100; // TODO: move to pqlib +} + +/** + * Structure for storing client credentials. + * Client must provide this structre in first request (init of write\read) and in case of changing, + * e.g. client gets new TVM Service ticket once per hour. Not all requests (read/write) could contain TCredentials - server will take already known credentials. + * Provided auth credentials will be checked once per some time at serverside. This means that if TVM ticket is too old, then read\write session will die. + * If no auth provided at all - client will be using guest rights. So, reading/writing is still possible without credentials. + */ + +message Credentials { + oneof credentials { + bytes tvm_service_ticket = 1; + bytes oauth_token = 2; + } +} + diff --git a/ydb/public/api/protos/out/out.cpp b/ydb/public/api/protos/out/out.cpp index 850f902e77..9e8b7263ee 100644 --- a/ydb/public/api/protos/out/out.cpp +++ b/ydb/public/api/protos/out/out.cpp @@ -1,12 +1,12 @@ #include <ydb/public/api/protos/ydb_cms.pb.h> #include <ydb/public/api/protos/ydb_status_codes.pb.h> - -#include <util/stream/output.h> - -Y_DECLARE_OUT_SPEC(, Ydb::Cms::GetDatabaseStatusResult::State, stream, value) { - stream << Ydb::Cms::GetDatabaseStatusResult::State_Name(value); -} - -Y_DECLARE_OUT_SPEC(, Ydb::StatusIds::StatusCode, stream, value) { - stream << Ydb::StatusIds::StatusCode_Name(value); -} + +#include <util/stream/output.h> + +Y_DECLARE_OUT_SPEC(, Ydb::Cms::GetDatabaseStatusResult::State, stream, value) { + stream << Ydb::Cms::GetDatabaseStatusResult::State_Name(value); +} + +Y_DECLARE_OUT_SPEC(, Ydb::StatusIds::StatusCode, stream, value) { + stream << Ydb::StatusIds::StatusCode_Name(value); +} diff --git a/ydb/public/api/protos/out/ya.make b/ydb/public/api/protos/out/ya.make index 82ce9b754e..c984157c89 100644 --- a/ydb/public/api/protos/out/ya.make +++ b/ydb/public/api/protos/out/ya.make @@ -1,13 +1,13 @@ -LIBRARY() - -OWNER(g:kikimr) - -SRCS( - out.cpp -) - -PEERDIR( +LIBRARY() + +OWNER(g:kikimr) + +SRCS( + out.cpp +) + +PEERDIR( ydb/public/api/protos -) - -END() +) + +END() diff --git a/ydb/public/api/protos/ya.make b/ydb/public/api/protos/ya.make index b6e03aa5cb..8a156403a3 100644 --- a/ydb/public/api/protos/ya.make +++ b/ydb/public/api/protos/ya.make @@ -15,7 +15,7 @@ PEERDIR( SRCS( draft/datastreams.proto - draft/persqueue_common.proto + draft/persqueue_common.proto draft/persqueue_error_codes.proto draft/ydb_long_tx.proto draft/ydb_logstore.proto @@ -51,7 +51,7 @@ CPP_PROTO_PLUGIN0(validation ydb/core/grpc_services/validation) # .pb.h are only available in C++ variant of PROTO_LIBRARY IF (MODULE_TAG == "CPP_PROTO") - GENERATE_ENUM_SERIALIZATION(draft/persqueue_common.pb.h) + GENERATE_ENUM_SERIALIZATION(draft/persqueue_common.pb.h) GENERATE_ENUM_SERIALIZATION(ydb_persqueue_cluster_discovery.pb.h) GENERATE_ENUM_SERIALIZATION(draft/datastreams.pb.h) ENDIF() diff --git a/ydb/public/api/protos/ydb_persqueue_v1.proto b/ydb/public/api/protos/ydb_persqueue_v1.proto index 6c33f2afa0..93a7fb6c79 100644 --- a/ydb/public/api/protos/ydb_persqueue_v1.proto +++ b/ydb/public/api/protos/ydb_persqueue_v1.proto @@ -1108,22 +1108,22 @@ message DropTopicResult { } /** - * Credentials settings - */ - -message Credentials { - message Iam { - string endpoint = 1; - string service_account_key = 2; - } - oneof credentials { - string oauth_token = 1; - string jwt_params = 2; - Iam iam = 3; - } -} - -/** + * Credentials settings + */ + +message Credentials { + message Iam { + string endpoint = 1; + string service_account_key = 2; + } + oneof credentials { + string oauth_token = 1; + string jwt_params = 2; + Iam iam = 3; + } +} + +/** * Message for describing topic internals. */ @@ -1134,9 +1134,9 @@ message TopicSettings { } // How many partitions in topic. Must less than database limit. Default limit - 10. - int32 partitions_count = 1 [(value) = "> 0"]; + int32 partitions_count = 1 [(value) = "> 0"]; // How long data in partition should be stored. Must be greater than 0 and less than limit for this database. Default limit - 36 hours. - int64 retention_period_ms = 2 [(value) = "> 0"]; + int64 retention_period_ms = 2 [(value) = "> 0"]; // How long last written seqno for message group should be stored. Must be greater then retention_period_ms and less then limit for this database. Default limit - 16 days. int64 message_group_seqno_retention_period_ms = 12 [(value) = ">= 0"]; // How many last written seqno for various message groups should be stored per partition. Must be less than limit for this database. Default limit - 6*10^6 values. @@ -1148,11 +1148,11 @@ message TopicSettings { // Writes with codec not from this list are forbiden. repeated Codec supported_codecs = 4 [(size).le = 100]; // Max storage usage for each topic's partition. Must be less than database limit. Default limit - 130 GB. - int64 max_partition_storage_size = 5 [(value) = ">= 0"]; + int64 max_partition_storage_size = 5 [(value) = ">= 0"]; // Partition write speed in bytes per second. Must be less than database limit. Default limit - 1 MB/s. - int64 max_partition_write_speed = 6 [(value) = ">= 0"]; + int64 max_partition_write_speed = 6 [(value) = ">= 0"]; // Burst size for write in partition, in bytes. Must be less than database limit. Default limit - 1 MB. - int64 max_partition_write_burst = 7 [(value) = ">= 0"]; + int64 max_partition_write_burst = 7 [(value) = ">= 0"]; // Disallows client writes. Used for mirrored topics in federation. bool client_write_disabled = 8; @@ -1164,7 +1164,7 @@ message TopicSettings { // Flag that this consumer is important. bool important = 2; // All messages with smaller timestamp of write will be skipped. - int64 starting_message_timestamp_ms = 3 [(value) = ">= 0"]; + int64 starting_message_timestamp_ms = 3 [(value) = ">= 0"]; // Max format version that is supported by this consumer. // supported_format on topic must not be greater. Format supported_format = 4; @@ -1173,10 +1173,10 @@ message TopicSettings { repeated Codec supported_codecs = 5 [(size).le = 100]; // Read rule version. Any non-negative integer. - int64 version = 6 [(value) = ">= 0"]; - - // Client service type. - string service_type = 7; + int64 version = 6 [(value) = ">= 0"]; + + // Client service type. + string service_type = 7; } // List of consumer read rules for this topic. @@ -1184,24 +1184,24 @@ message TopicSettings { // User and server attributes of topic. Server attributes starts from "_" and will be validated by server. map<string, string> attributes = 10; - - // Message for remote mirror rule description. - message RemoteMirrorRule { - // Source cluster endpoint in format server:port. - string endpoint = 1; - // Source topic that we want to mirror. - string topic_path = 2; - // Source consumer for reading source topic. - string consumer_name = 3; - // Credentials for reading source topic by source consumer. - Credentials credentials = 4; - // All messages with smaller timestamp of write will be skipped. - int64 starting_message_timestamp_ms = 5 [(value) = ">= 0"]; - // Database - string database = 6; - } - // remote mirror rule for this topic. - RemoteMirrorRule remote_mirror_rule = 11; + + // Message for remote mirror rule description. + message RemoteMirrorRule { + // Source cluster endpoint in format server:port. + string endpoint = 1; + // Source topic that we want to mirror. + string topic_path = 2; + // Source consumer for reading source topic. + string consumer_name = 3; + // Credentials for reading source topic by source consumer. + Credentials credentials = 4; + // All messages with smaller timestamp of write will be skipped. + int64 starting_message_timestamp_ms = 5 [(value) = ">= 0"]; + // Database + string database = 6; + } + // remote mirror rule for this topic. + RemoteMirrorRule remote_mirror_rule = 11; } /** diff --git a/ydb/public/lib/base/msgbus.h b/ydb/public/lib/base/msgbus.h index d97e36e880..f11bd9771f 100644 --- a/ydb/public/lib/base/msgbus.h +++ b/ydb/public/lib/base/msgbus.h @@ -258,22 +258,22 @@ public: virtual ~IMessageBusServer() {} }; -class IPersQueueGetReadSessionsInfoWorkerFactory; +class IPersQueueGetReadSessionsInfoWorkerFactory; + +IMessageBusServer* CreateMsgBusServer( + NBus::TBusMessageQueue *queue, + const NBus::TBusServerSessionConfig &config, + std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory, + ui32 bindPort = TProtocol::DefaultPort +); +IMessageBusServer* CreateMsgBusTracingServer( + NBus::TBusMessageQueue *queue, + const NBus::TBusServerSessionConfig &config, + const TString &tracePath, + std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory, + ui32 bindPort = TProtocol::DefaultPort +); -IMessageBusServer* CreateMsgBusServer( - NBus::TBusMessageQueue *queue, - const NBus::TBusServerSessionConfig &config, - std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory, - ui32 bindPort = TProtocol::DefaultPort -); -IMessageBusServer* CreateMsgBusTracingServer( - NBus::TBusMessageQueue *queue, - const NBus::TBusServerSessionConfig &config, - const TString &tracePath, - std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory, - ui32 bindPort = TProtocol::DefaultPort -); - inline NActors::TActorId CreateMsgBusProxyId() { return NActors::TActorId(0, "MsgBusProxy"); } diff --git a/ydb/public/lib/experimental/ydb_clickhouse_internal.cpp b/ydb/public/lib/experimental/ydb_clickhouse_internal.cpp index 179d8b9791..2984f680b7 100644 --- a/ydb/public/lib/experimental/ydb_clickhouse_internal.cpp +++ b/ydb/public/lib/experimental/ydb_clickhouse_internal.cpp @@ -1,9 +1,9 @@ #include "ydb_clickhouse_internal.h" -#define INCLUDE_YDB_INTERNAL_H +#define INCLUDE_YDB_INTERNAL_H #include <ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.h> -#undef INCLUDE_YDB_INTERNAL_H - +#undef INCLUDE_YDB_INTERNAL_H + #include <ydb/public/api/grpc/draft/ydb_clickhouse_internal_v1.grpc.pb.h> #include <library/cpp/grpc/client/grpc_client_low.h> #include <ydb/public/sdk/cpp/client/ydb_common_client/impl/client.h> diff --git a/ydb/public/lib/experimental/ydb_experimental.cpp b/ydb/public/lib/experimental/ydb_experimental.cpp index 939468e0be..8765c0c750 100644 --- a/ydb/public/lib/experimental/ydb_experimental.cpp +++ b/ydb/public/lib/experimental/ydb_experimental.cpp @@ -1,9 +1,9 @@ #include "ydb_experimental.h" -#define INCLUDE_YDB_INTERNAL_H +#define INCLUDE_YDB_INTERNAL_H #include <ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.h> -#undef INCLUDE_YDB_INTERNAL_H - +#undef INCLUDE_YDB_INTERNAL_H + #include <ydb/public/api/grpc/draft/ydb_experimental_v1.grpc.pb.h> #include <library/cpp/grpc/client/grpc_client_low.h> #include <ydb/public/sdk/cpp/client/ydb_proto/accessor.h> @@ -112,7 +112,7 @@ public: auto request = MakeRequest<Ydb::Experimental::ExecuteStreamQueryRequest>(); request.set_yql_text(query); if (params) { - *request.mutable_parameters() = params->GetProtoMap(); + *request.mutable_parameters() = params->GetProtoMap(); } switch (settings.ProfileMode_) { diff --git a/ydb/public/lib/experimental/ydb_s3_internal.cpp b/ydb/public/lib/experimental/ydb_s3_internal.cpp index c9486a6e1a..ac1c5025ed 100644 --- a/ydb/public/lib/experimental/ydb_s3_internal.cpp +++ b/ydb/public/lib/experimental/ydb_s3_internal.cpp @@ -1,9 +1,9 @@ #include "ydb_s3_internal.h" -#define INCLUDE_YDB_INTERNAL_H +#define INCLUDE_YDB_INTERNAL_H #include <ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.h> -#undef INCLUDE_YDB_INTERNAL_H - +#undef INCLUDE_YDB_INTERNAL_H + #include <ydb/public/api/grpc/draft/ydb_s3_internal_v1.grpc.pb.h> #include <ydb/library/yql/public/issue/yql_issue.h> diff --git a/ydb/public/lib/idx_test/idx_test_data_provider.cpp b/ydb/public/lib/idx_test/idx_test_data_provider.cpp index 2d173baf13..0006551f29 100644 --- a/ydb/public/lib/idx_test/idx_test_data_provider.cpp +++ b/ydb/public/lib/idx_test/idx_test_data_provider.cpp @@ -4,8 +4,8 @@ #include <library/cpp/string_utils/base64/base64.h> -#include <util/string/builder.h> - +#include <util/string/builder.h> + using namespace NYdb; using namespace NYdb::NTable; diff --git a/ydb/public/sdk/cpp/client/draft/ut/ya.make b/ydb/public/sdk/cpp/client/draft/ut/ya.make index 9e40de167b..6e210f3fe3 100644 --- a/ydb/public/sdk/cpp/client/draft/ut/ya.make +++ b/ydb/public/sdk/cpp/client/draft/ut/ya.make @@ -1,23 +1,23 @@ UNITTEST_FOR(ydb/public/sdk/cpp/client/draft) - -OWNER( - dcherednik - g:kikimr -) - -IF (SANITIZER_TYPE) - TIMEOUT(1200) - SIZE(LARGE) - TAG(ya:fat) -ELSE() - TIMEOUT(600) - SIZE(MEDIUM) -ENDIF() - -FORK_SUBTESTS() - -SRCS( - ydb_scripting_response_headers_ut.cpp -) - -END() + +OWNER( + dcherednik + g:kikimr +) + +IF (SANITIZER_TYPE) + TIMEOUT(1200) + SIZE(LARGE) + TAG(ya:fat) +ELSE() + TIMEOUT(600) + SIZE(MEDIUM) +ENDIF() + +FORK_SUBTESTS() + +SRCS( + ydb_scripting_response_headers_ut.cpp +) + +END() diff --git a/ydb/public/sdk/cpp/client/draft/ya.make b/ydb/public/sdk/cpp/client/draft/ya.make index b2e982a502..a5f5fbf597 100644 --- a/ydb/public/sdk/cpp/client/draft/ya.make +++ b/ydb/public/sdk/cpp/client/draft/ya.make @@ -18,7 +18,7 @@ PEERDIR( ) END() - + RECURSE_FOR_TESTS( ut ) diff --git a/ydb/public/sdk/cpp/client/draft/ydb_scripting.cpp b/ydb/public/sdk/cpp/client/draft/ydb_scripting.cpp index 6e39163d4d..38b8c8123d 100644 --- a/ydb/public/sdk/cpp/client/draft/ydb_scripting.cpp +++ b/ydb/public/sdk/cpp/client/draft/ydb_scripting.cpp @@ -1,10 +1,10 @@ #include "ydb_scripting.h" -#define INCLUDE_YDB_INTERNAL_H +#define INCLUDE_YDB_INTERNAL_H #include <ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.h> #include <ydb/public/sdk/cpp/client/impl/ydb_internal/table_helpers/helpers.h> -#undef INCLUDE_YDB_INTERNAL_H - +#undef INCLUDE_YDB_INTERNAL_H + #include <ydb/public/api/grpc/ydb_scripting_v1.grpc.pb.h> #include <ydb/public/api/protos/ydb_scripting.pb.h> #include <ydb/public/sdk/cpp/client/ydb_common_client/impl/client.h> @@ -322,7 +322,7 @@ TParamsBuilder TScriptingClient::GetParamsBuilder() { TAsyncExecuteYqlResult TScriptingClient::ExecuteYqlScript(const TString &query, NYdb::TParams&& params, const TExecuteYqlRequestSettings &settings) { - auto paramsPtr = params.Empty() ? nullptr : params.GetProtoMapPtr(); + auto paramsPtr = params.Empty() ? nullptr : params.GetProtoMapPtr(); return Impl_->ExecuteYqlScript(query, paramsPtr, settings); } @@ -338,7 +338,7 @@ TAsyncExecuteYqlResult TScriptingClient::ExecuteYqlScript(const TString &query, using TProtoParamsType = const ::google::protobuf::Map<TString, Ydb::TypedValue>; return Impl_->ExecuteYqlScript<TProtoParamsType&>( query, - params.GetProtoMap(), + params.GetProtoMap(), settings); } } diff --git a/ydb/public/sdk/cpp/client/extensions/discovery_mutator/discovery_mutator_ut.cpp b/ydb/public/sdk/cpp/client/extensions/discovery_mutator/discovery_mutator_ut.cpp index a9db12c0df..7a5fa43cc3 100644 --- a/ydb/public/sdk/cpp/client/extensions/discovery_mutator/discovery_mutator_ut.cpp +++ b/ydb/public/sdk/cpp/client/extensions/discovery_mutator/discovery_mutator_ut.cpp @@ -2,7 +2,7 @@ #include <ydb/public/sdk/cpp/client/extensions/discovery_mutator/discovery_mutator.h> #include <ydb/public/sdk/cpp/client/ydb_extension/extension.h> #include <ydb/public/sdk/cpp/client/ydb_table/table.h> - + #include <library/cpp/testing/unittest/registar.h> #include <library/cpp/testing/unittest/tests_data.h> diff --git a/ydb/public/sdk/cpp/client/extensions/discovery_mutator/ut/ya.make b/ydb/public/sdk/cpp/client/extensions/discovery_mutator/ut/ya.make index 249e4409f8..d6688b8e0a 100644 --- a/ydb/public/sdk/cpp/client/extensions/discovery_mutator/ut/ya.make +++ b/ydb/public/sdk/cpp/client/extensions/discovery_mutator/ut/ya.make @@ -1,27 +1,27 @@ UNITTEST_FOR(ydb/public/sdk/cpp/client/extensions/discovery_mutator) - -OWNER( - dcherednik - g:kikimr -) - -IF (SANITIZER_TYPE) - TIMEOUT(1200) - SIZE(LARGE) - TAG(ya:fat) -ELSE() - TIMEOUT(600) - SIZE(MEDIUM) -ENDIF() - -FORK_SUBTESTS() - -PEERDIR( + +OWNER( + dcherednik + g:kikimr +) + +IF (SANITIZER_TYPE) + TIMEOUT(1200) + SIZE(LARGE) + TAG(ya:fat) +ELSE() + TIMEOUT(600) + SIZE(MEDIUM) +ENDIF() + +FORK_SUBTESTS() + +PEERDIR( ydb/public/sdk/cpp/client/ydb_table -) - -SRCS( - discovery_mutator_ut.cpp -) - -END() +) + +SRCS( + discovery_mutator_ut.cpp +) + +END() diff --git a/ydb/public/sdk/cpp/client/extensions/discovery_mutator/ya.make b/ydb/public/sdk/cpp/client/extensions/discovery_mutator/ya.make index a1ec29b162..7d4997e94c 100644 --- a/ydb/public/sdk/cpp/client/extensions/discovery_mutator/ya.make +++ b/ydb/public/sdk/cpp/client/extensions/discovery_mutator/ya.make @@ -14,7 +14,7 @@ PEERDIR( ) END() - + RECURSE_FOR_TESTS( ut ) diff --git a/ydb/public/sdk/cpp/client/extensions/solomon_stats/README.md b/ydb/public/sdk/cpp/client/extensions/solomon_stats/README.md index e934f99a89..869678d8dd 100644 --- a/ydb/public/sdk/cpp/client/extensions/solomon_stats/README.md +++ b/ydb/public/sdk/cpp/client/extensions/solomon_stats/README.md @@ -38,7 +38,7 @@ After creating NYdb::TDriver you need to add Solomon Monitoring extension. If yo ... { - auto config = NYdb::TDriverConfig(); + auto config = NYdb::TDriverConfig(); NYdb::TDriver driver(config); try { diff --git a/ydb/public/sdk/cpp/client/extensions/solomon_stats/pull_client.h b/ydb/public/sdk/cpp/client/extensions/solomon_stats/pull_client.h index 5ef7363256..8d410c2fa2 100644 --- a/ydb/public/sdk/cpp/client/extensions/solomon_stats/pull_client.h +++ b/ydb/public/sdk/cpp/client/extensions/solomon_stats/pull_client.h @@ -1,7 +1,7 @@ #pragma once #include <ydb/public/sdk/cpp/client/ydb_extension/extension.h> - + #include <library/cpp/http/server/response.h> #include <library/cpp/monlib/metrics/metric_consumer.h> #include <library/cpp/monlib/encode/json/json.h> @@ -9,8 +9,8 @@ #include <library/cpp/monlib/service/pages/mon_page.h> #include <library/cpp/monlib/service/monservice.h> -#include <util/generic/vector.h> - +#include <util/generic/vector.h> + namespace NSolomonStatExtension { class TSolomonStatPullExtension: public NYdb::IExtension { diff --git a/ydb/public/sdk/cpp/client/extensions/ya.make b/ydb/public/sdk/cpp/client/extensions/ya.make index 6b76506504..c6b7c59c93 100644 --- a/ydb/public/sdk/cpp/client/extensions/ya.make +++ b/ydb/public/sdk/cpp/client/extensions/ya.make @@ -1,6 +1,6 @@ -OWNER(g:kikimr) - -RECURSE( - solomon_stats - discovery_mutator -) +OWNER(g:kikimr) + +RECURSE( + solomon_stats + discovery_mutator +) diff --git a/ydb/public/sdk/cpp/client/impl/ydb_endpoints/endpoints.cpp b/ydb/public/sdk/cpp/client/impl/ydb_endpoints/endpoints.cpp index 76284cd53a..b6d03cc50a 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_endpoints/endpoints.cpp +++ b/ydb/public/sdk/cpp/client/impl/ydb_endpoints/endpoints.cpp @@ -1,13 +1,13 @@ -#include "endpoints.h" +#include "endpoints.h" #include <library/cpp/monlib/metrics/metric_registry.h> #include <library/cpp/string_utils/quote/quote.h> - -#include <util/random/random.h> -#include <set> -#include <unordered_set> - +#include <util/random/random.h> + +#include <set> +#include <unordered_set> + namespace NYdb { class TEndpointElectorSafe::TObjRegistry : public IObjRegistryHandle { diff --git a/ydb/public/sdk/cpp/client/impl/ydb_endpoints/endpoints_ut.cpp b/ydb/public/sdk/cpp/client/impl/ydb_endpoints/endpoints_ut.cpp index eef0a8671a..7823fb6460 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_endpoints/endpoints_ut.cpp +++ b/ydb/public/sdk/cpp/client/impl/ydb_endpoints/endpoints_ut.cpp @@ -1,13 +1,13 @@ #include <ydb/public/sdk/cpp/client/impl/ydb_endpoints/endpoints.h> - + #include <library/cpp/testing/unittest/registar.h> #include <library/cpp/testing/unittest/tests_data.h> -#include <library/cpp/threading/future/core/future.h> +#include <library/cpp/threading/future/core/future.h> #include <util/system/thread.h> #include <util/random/random.h> -#include <unordered_set> +#include <unordered_set> using namespace NYdb; diff --git a/ydb/public/sdk/cpp/client/impl/ydb_endpoints/ut/ya.make b/ydb/public/sdk/cpp/client/impl/ydb_endpoints/ut/ya.make index 4300971cb9..1f9dfbbe1d 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_endpoints/ut/ya.make +++ b/ydb/public/sdk/cpp/client/impl/ydb_endpoints/ut/ya.make @@ -1,23 +1,23 @@ UNITTEST_FOR(ydb/public/sdk/cpp/client/impl/ydb_endpoints) - -OWNER( - dcherednik - g:kikimr -) - -IF (SANITIZER_TYPE) - TIMEOUT(1200) - SIZE(LARGE) - TAG(ya:fat) -ELSE() - TIMEOUT(600) - SIZE(MEDIUM) -ENDIF() - -FORK_SUBTESTS() - -SRCS( - endpoints_ut.cpp -) - -END() + +OWNER( + dcherednik + g:kikimr +) + +IF (SANITIZER_TYPE) + TIMEOUT(1200) + SIZE(LARGE) + TAG(ya:fat) +ELSE() + TIMEOUT(600) + SIZE(MEDIUM) +ENDIF() + +FORK_SUBTESTS() + +SRCS( + endpoints_ut.cpp +) + +END() diff --git a/ydb/public/sdk/cpp/client/impl/ydb_endpoints/ya.make b/ydb/public/sdk/cpp/client/impl/ydb_endpoints/ya.make index 85e1fd466b..da6a27f707 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_endpoints/ya.make +++ b/ydb/public/sdk/cpp/client/impl/ydb_endpoints/ya.make @@ -6,7 +6,7 @@ OWNER( ) SRCS( - endpoints.cpp + endpoints.cpp ) PEERDIR( @@ -15,7 +15,7 @@ PEERDIR( ) END() - + RECURSE_FOR_TESTS( ut ) diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/common/types.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/common/types.h index 8048dd830f..c789a4b5e2 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/common/types.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/common/types.h @@ -1,26 +1,26 @@ -#pragma once - +#pragma once + #include <ydb/public/sdk/cpp/client/impl/ydb_internal/internal_header.h> - + #include <ydb/public/sdk/cpp/client/impl/ydb_internal/common/type_switcher.h> #include <ydb/public/sdk/cpp/client/ydb_types/status_codes.h> #include <ydb/public/sdk/cpp/client/ydb_types/ydb.h> - + #include <ydb/library/yql/public/issue/yql_issue.h> - -#include <library/cpp/grpc/client/grpc_client_low.h> - - -namespace NYdb { - -// Other callbacks -using TSimpleCb = std::function<void()>; -using TErrorCb = std::function<void(NGrpc::TGrpcStatus&)>; - -struct TBalancingSettings { - EBalancingPolicy Policy; + +#include <library/cpp/grpc/client/grpc_client_low.h> + + +namespace NYdb { + +// Other callbacks +using TSimpleCb = std::function<void()>; +using TErrorCb = std::function<void(NGrpc::TGrpcStatus&)>; + +struct TBalancingSettings { + EBalancingPolicy Policy; TStringType PolicyParams; -}; - -} // namespace NYdb +}; + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/common/ya.make b/ydb/public/sdk/cpp/client/impl/ydb_internal/common/ya.make index 5072fa2259..db5fd8bf4e 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/common/ya.make +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/common/ya.make @@ -1,16 +1,16 @@ -LIBRARY() - +LIBRARY() + OWNER(g:kikimr) - + SRCS( parser.cpp getenv.cpp string_helpers.cpp ) -PEERDIR( +PEERDIR( library/cpp/grpc/client ydb/library/yql/public/issue -) - -END() +) + +END() diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/authenticator.cpp b/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/authenticator.cpp index 3ff50c3fbd..f8862d4c71 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/authenticator.cpp +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/authenticator.cpp @@ -1,34 +1,34 @@ -#define INCLUDE_YDB_INTERNAL_H -#include "authenticator.h" - +#define INCLUDE_YDB_INTERNAL_H +#include "authenticator.h" + #include <ydb/public/sdk/cpp/client/resources/ydb_resources.h> - -namespace NYdb { - -TYdbAuthenticator::TYdbAuthenticator(std::shared_ptr<ICredentialsProvider> credentialsProvider) - : CredentialsProvider_(std::move(credentialsProvider)) -{} - -grpc::Status TYdbAuthenticator::GetMetadata( - grpc::string_ref, - grpc::string_ref, - const grpc::AuthContext&, - std::multimap<grpc::string, grpc::string>* metadata -) { - try { - metadata->insert(std::make_pair(YDB_AUTH_TICKET_HEADER, CredentialsProvider_->GetAuthInfo())); - } catch (const yexception& e) { - return grpc::Status( - grpc::StatusCode::UNAUTHENTICATED, - "Can't get Authentication info from CredentialsProvider", - e.what() - ); - } - return grpc::Status::OK; -} - -bool TYdbAuthenticator::IsBlocking() const { - return false; -} - -} // namespace NYdb + +namespace NYdb { + +TYdbAuthenticator::TYdbAuthenticator(std::shared_ptr<ICredentialsProvider> credentialsProvider) + : CredentialsProvider_(std::move(credentialsProvider)) +{} + +grpc::Status TYdbAuthenticator::GetMetadata( + grpc::string_ref, + grpc::string_ref, + const grpc::AuthContext&, + std::multimap<grpc::string, grpc::string>* metadata +) { + try { + metadata->insert(std::make_pair(YDB_AUTH_TICKET_HEADER, CredentialsProvider_->GetAuthInfo())); + } catch (const yexception& e) { + return grpc::Status( + grpc::StatusCode::UNAUTHENTICATED, + "Can't get Authentication info from CredentialsProvider", + e.what() + ); + } + return grpc::Status::OK; +} + +bool TYdbAuthenticator::IsBlocking() const { + return false; +} + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/authenticator.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/authenticator.h index 7744d6f9cc..e5e21bbb04 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/authenticator.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/authenticator.h @@ -1,28 +1,28 @@ -#pragma once - +#pragma once + #include <ydb/public/sdk/cpp/client/impl/ydb_internal/internal_header.h> - + #include <ydb/public/sdk/cpp/client/ydb_types/credentials/credentials.h> - -#include <library/cpp/grpc/client/grpc_client_low.h> - -namespace NYdb { - -class TYdbAuthenticator : public grpc::MetadataCredentialsPlugin { -public: - TYdbAuthenticator(std::shared_ptr<ICredentialsProvider> credentialsProvider); - - grpc::Status GetMetadata( - grpc::string_ref, - grpc::string_ref, - const grpc::AuthContext&, - std::multimap<grpc::string, grpc::string>* metadata - ) override; - - bool IsBlocking() const override; - -private: - std::shared_ptr<ICredentialsProvider> CredentialsProvider_; -}; - -} // namespace NYdb + +#include <library/cpp/grpc/client/grpc_client_low.h> + +namespace NYdb { + +class TYdbAuthenticator : public grpc::MetadataCredentialsPlugin { +public: + TYdbAuthenticator(std::shared_ptr<ICredentialsProvider> credentialsProvider); + + grpc::Status GetMetadata( + grpc::string_ref, + grpc::string_ref, + const grpc::AuthContext&, + std::multimap<grpc::string, grpc::string>* metadata + ) override; + + bool IsBlocking() const override; + +private: + std::shared_ptr<ICredentialsProvider> CredentialsProvider_; +}; + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/endpoint_pool.cpp b/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/endpoint_pool.cpp index 62253ced79..7b7947578d 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/endpoint_pool.cpp +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/endpoint_pool.cpp @@ -1,14 +1,14 @@ -#define INCLUDE_YDB_INTERNAL_H -#include "endpoint_pool.h" - -namespace NYdb { -TEndpointPool::TEndpointPool(TListEndpointsResultProvider&& provider, const IInternalClient* client) - : Provider_(provider) - , LastUpdateTime_(TInstant::Zero().MicroSeconds()) - , BalancingSettings_(client->GetBalancingSettings()) -{} - -TEndpointPool::~TEndpointPool() { +#define INCLUDE_YDB_INTERNAL_H +#include "endpoint_pool.h" + +namespace NYdb { +TEndpointPool::TEndpointPool(TListEndpointsResultProvider&& provider, const IInternalClient* client) + : Provider_(provider) + , LastUpdateTime_(TInstant::Zero().MicroSeconds()) + , BalancingSettings_(client->GetBalancingSettings()) +{} + +TEndpointPool::~TEndpointPool() { try { NThreading::TFuture<TEndpointUpdateResult> future; { @@ -16,43 +16,43 @@ TEndpointPool::~TEndpointPool() { if (DiscoveryPromise_.Initialized()) { future = DiscoveryPromise_.GetFuture(); } - } + } if (future.Initialized()) { future.Wait(); } } catch (...) { Y_FAIL("Unexpected exception from endpoint pool dtor"); - } -} - -std::pair<NThreading::TFuture<TEndpointUpdateResult>, bool> TEndpointPool::UpdateAsync() { - NThreading::TFuture<TEndpointUpdateResult> future; + } +} + +std::pair<NThreading::TFuture<TEndpointUpdateResult>, bool> TEndpointPool::UpdateAsync() { + NThreading::TFuture<TEndpointUpdateResult> future; { std::lock_guard guard(Mutex_); - if (DiscoveryPromise_.Initialized()) { - return {DiscoveryPromise_.GetFuture(), false}; - } else { - DiscoveryPromise_ = NThreading::NewPromise<TEndpointUpdateResult>(); - future = DiscoveryPromise_.GetFuture(); - } - } - auto handler = [this](const TAsyncListEndpointsResult& future) { - TListEndpointsResult result = future.GetValue(); + if (DiscoveryPromise_.Initialized()) { + return {DiscoveryPromise_.GetFuture(), false}; + } else { + DiscoveryPromise_ = NThreading::NewPromise<TEndpointUpdateResult>(); + future = DiscoveryPromise_.GetFuture(); + } + } + auto handler = [this](const TAsyncListEndpointsResult& future) { + TListEndpointsResult result = future.GetValue(); std::vector<TStringType> removed; - if (result.DiscoveryStatus.Status == EStatus::SUCCESS) { + if (result.DiscoveryStatus.Status == EStatus::SUCCESS) { std::vector<TEndpointRecord> records; - // Is used to convert float to integer load factor - // same integer values will be selected randomly. - const float multiplicator = 10.0; - const auto& preferedLocation = GetPreferedLocation(result.Result.self_location()); - for (const auto& endpoint : result.Result.endpoints()) { - i32 loadFactor = (i32)(multiplicator * Min(LoadMax, Max(LoadMin, endpoint.load_factor()))); - if (BalancingSettings_.Policy != EBalancingPolicy::UseAllNodes) { - if (endpoint.location() != preferedLocation) { - // Location missmatch, shift this endpoint - loadFactor += GetLocalityShift(); - } - } + // Is used to convert float to integer load factor + // same integer values will be selected randomly. + const float multiplicator = 10.0; + const auto& preferedLocation = GetPreferedLocation(result.Result.self_location()); + for (const auto& endpoint : result.Result.endpoints()) { + i32 loadFactor = (i32)(multiplicator * Min(LoadMax, Max(LoadMin, endpoint.load_factor()))); + if (BalancingSettings_.Policy != EBalancingPolicy::UseAllNodes) { + if (endpoint.location() != preferedLocation) { + // Location missmatch, shift this endpoint + loadFactor += GetLocalityShift(); + } + } TStringType sslTargetNameOverride = endpoint.ssl_target_name_override(); auto getIpSslTargetNameOverride = [&]() -> TStringType { @@ -105,82 +105,82 @@ std::pair<NThreading::TFuture<TEndpointUpdateResult>, bool> TEndpointPool::Updat << endpoint.port(); records.emplace_back(std::move(endpointString), loadFactor, std::move(sslTargetNameOverride)); } - } - LastUpdateTime_ = TInstant::Now().MicroSeconds(); - removed = Elector_.SetNewState(std::move(records)); - } - NThreading::TPromise<TEndpointUpdateResult> promise; + } + LastUpdateTime_ = TInstant::Now().MicroSeconds(); + removed = Elector_.SetNewState(std::move(records)); + } + NThreading::TPromise<TEndpointUpdateResult> promise; { std::lock_guard guard(Mutex_); - DiscoveryPromise_.Swap(promise); - } - promise.SetValue({std::move(removed), result.DiscoveryStatus}); - }; - - Provider_().Subscribe(handler); - return {future, true}; -} - + DiscoveryPromise_.Swap(promise); + } + promise.SetValue({std::move(removed), result.DiscoveryStatus}); + }; + + Provider_().Subscribe(handler); + return {future, true}; +} + TEndpointRecord TEndpointPool::GetEndpoint(const TStringType& preferredEndpoint) const { - return Elector_.GetEndpoint(preferredEndpoint); -} - -TDuration TEndpointPool::TimeSinceLastUpdate() const { - auto now = TInstant::Now().MicroSeconds(); + return Elector_.GetEndpoint(preferredEndpoint); +} + +TDuration TEndpointPool::TimeSinceLastUpdate() const { + auto now = TInstant::Now().MicroSeconds(); return TDuration::MicroSeconds(now - LastUpdateTime_.load()); -} - +} + void TEndpointPool::BanEndpoint(const TStringType& endpoint) { - Elector_.PessimizeEndpoint(endpoint); -} - -int TEndpointPool::GetPessimizationRatio() { - return Elector_.GetPessimizationRatio(); -} - + Elector_.PessimizeEndpoint(endpoint); +} + +int TEndpointPool::GetPessimizationRatio() { + return Elector_.GetPessimizationRatio(); +} + bool TEndpointPool::LinkObjToEndpoint(const TStringType& endpoint, TEndpointObj* obj, const void* tag) { - return Elector_.LinkObjToEndpoint(endpoint, obj, tag); -} - -void TEndpointPool::ForEachEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const { - return Elector_.ForEachEndpoint(cb, 0, Max<i32>(), tag); -} - -void TEndpointPool::ForEachLocalEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const { - return Elector_.ForEachEndpoint(cb, 0, GetLocalityShift() - 1, tag); -} - -void TEndpointPool::ForEachForeignEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const { - return Elector_.ForEachEndpoint(cb, GetLocalityShift(), Max<i32>() - 1, tag); -} - -EBalancingPolicy TEndpointPool::GetBalancingPolicy() const { - return BalancingSettings_.Policy; -} - -void TEndpointPool::SetStatCollector(NSdkStats::TStatCollector& statCollector) { - if (!statCollector.IsCollecting()) - return; - Elector_.SetStatCollector(statCollector.GetEndpointElectorStatCollector()); - StatCollector_ = &statCollector; -} - -constexpr i32 TEndpointPool::GetLocalityShift() { - return LoadMax * Multiplicator; -} - + return Elector_.LinkObjToEndpoint(endpoint, obj, tag); +} + +void TEndpointPool::ForEachEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const { + return Elector_.ForEachEndpoint(cb, 0, Max<i32>(), tag); +} + +void TEndpointPool::ForEachLocalEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const { + return Elector_.ForEachEndpoint(cb, 0, GetLocalityShift() - 1, tag); +} + +void TEndpointPool::ForEachForeignEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const { + return Elector_.ForEachEndpoint(cb, GetLocalityShift(), Max<i32>() - 1, tag); +} + +EBalancingPolicy TEndpointPool::GetBalancingPolicy() const { + return BalancingSettings_.Policy; +} + +void TEndpointPool::SetStatCollector(NSdkStats::TStatCollector& statCollector) { + if (!statCollector.IsCollecting()) + return; + Elector_.SetStatCollector(statCollector.GetEndpointElectorStatCollector()); + StatCollector_ = &statCollector; +} + +constexpr i32 TEndpointPool::GetLocalityShift() { + return LoadMax * Multiplicator; +} + TStringType TEndpointPool::GetPreferedLocation(const TStringType& selfLocation) { - switch (BalancingSettings_.Policy) { - case EBalancingPolicy::UseAllNodes: - return {}; - case EBalancingPolicy::UsePreferableLocation: - if (BalancingSettings_.PolicyParams.empty()) { - return selfLocation; - } else { - return BalancingSettings_.PolicyParams; - } - } - return {}; -} - -} // namespace NYdb + switch (BalancingSettings_.Policy) { + case EBalancingPolicy::UseAllNodes: + return {}; + case EBalancingPolicy::UsePreferableLocation: + if (BalancingSettings_.PolicyParams.empty()) { + return selfLocation; + } else { + return BalancingSettings_.PolicyParams; + } + } + return {}; +} + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/endpoint_pool.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/endpoint_pool.h index ad519257b1..2c81ee79ef 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/endpoint_pool.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/endpoint_pool.h @@ -1,68 +1,68 @@ -#pragma once - +#pragma once + #include <ydb/public/sdk/cpp/client/impl/ydb_internal/internal_header.h> - + #include <ydb/public/api/protos/ydb_discovery.pb.h> #include <ydb/public/sdk/cpp/client/impl/ydb_internal/internal_client/client.h> #include <ydb/public/sdk/cpp/client/impl/ydb_internal/plain_status/status.h> #include <ydb/public/sdk/cpp/client/impl/ydb_endpoints/endpoints.h> - -#include <library/cpp/threading/future/future.h> - + +#include <library/cpp/threading/future/future.h> + #include <mutex> -namespace NYdb { - -struct TListEndpointsResult { - Ydb::Discovery::ListEndpointsResult Result; - TPlainStatus DiscoveryStatus; -}; - -using TAsyncListEndpointsResult = NThreading::TFuture<TListEndpointsResult>; -using TListEndpointsResultProvider = std::function<TAsyncListEndpointsResult()>; - -struct TEndpointUpdateResult { +namespace NYdb { + +struct TListEndpointsResult { + Ydb::Discovery::ListEndpointsResult Result; + TPlainStatus DiscoveryStatus; +}; + +using TAsyncListEndpointsResult = NThreading::TFuture<TListEndpointsResult>; +using TListEndpointsResultProvider = std::function<TAsyncListEndpointsResult()>; + +struct TEndpointUpdateResult { std::vector<TStringType> Removed; - TPlainStatus DiscoveryStatus; -}; - -class TEndpointPool { -public: - TEndpointPool(TListEndpointsResultProvider&& provider, const IInternalClient* client); - ~TEndpointPool(); - std::pair<NThreading::TFuture<TEndpointUpdateResult>, bool> UpdateAsync(); + TPlainStatus DiscoveryStatus; +}; + +class TEndpointPool { +public: + TEndpointPool(TListEndpointsResultProvider&& provider, const IInternalClient* client); + ~TEndpointPool(); + std::pair<NThreading::TFuture<TEndpointUpdateResult>, bool> UpdateAsync(); TEndpointRecord GetEndpoint(const TStringType& preferredEndpoint) const; - TDuration TimeSinceLastUpdate() const; + TDuration TimeSinceLastUpdate() const; void BanEndpoint(const TStringType& endpoint); - int GetPessimizationRatio(); + int GetPessimizationRatio(); bool LinkObjToEndpoint(const TStringType& endpoint, TEndpointObj* obj, const void* tag); - void ForEachEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const; - void ForEachLocalEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const; - void ForEachForeignEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const; - EBalancingPolicy GetBalancingPolicy() const; - // TODO: Remove this mess - void SetStatCollector(NSdkStats::TStatCollector& statCollector); - static constexpr i32 GetLocalityShift(); - -private: + void ForEachEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const; + void ForEachLocalEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const; + void ForEachForeignEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const; + EBalancingPolicy GetBalancingPolicy() const; + // TODO: Remove this mess + void SetStatCollector(NSdkStats::TStatCollector& statCollector); + static constexpr i32 GetLocalityShift(); + +private: TStringType GetPreferedLocation(const TStringType& selfLocation); - -private: - TListEndpointsResultProvider Provider_; + +private: + TListEndpointsResultProvider Provider_; std::mutex Mutex_; - TEndpointElectorSafe Elector_; - NThreading::TPromise<TEndpointUpdateResult> DiscoveryPromise_; + TEndpointElectorSafe Elector_; + NThreading::TPromise<TEndpointUpdateResult> DiscoveryPromise_; std::atomic_uint64_t LastUpdateTime_; - const TBalancingSettings BalancingSettings_; - - NSdkStats::TStatCollector* StatCollector_ = nullptr; - - // Max, min load factor returned by discovery service - static constexpr float LoadMax = 100.0; - static constexpr float LoadMin = -100.0; - // Is used to convert float to integer load factor - // same integer values will be selected randomly. - static constexpr float Multiplicator = 10.0; -}; - -} // namespace NYdb + const TBalancingSettings BalancingSettings_; + + NSdkStats::TStatCollector* StatCollector_ = nullptr; + + // Max, min load factor returned by discovery service + static constexpr float LoadMax = 100.0; + static constexpr float LoadMin = -100.0; + // Is used to convert float to integer load factor + // same integer values will be selected randomly. + static constexpr float Multiplicator = 10.0; +}; + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/state.cpp b/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/state.cpp index 70c1842dbf..7364c34524 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/state.cpp +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/state.cpp @@ -1,201 +1,201 @@ -#define INCLUDE_YDB_INTERNAL_H -#include "state.h" - +#define INCLUDE_YDB_INTERNAL_H +#include "state.h" + #include <ydb/public/sdk/cpp/client/ydb_types/credentials/credentials.h> #include <ydb/public/sdk/cpp/client/impl/ydb_internal/logger/log.h> - -#include <library/cpp/string_utils/quote/quote.h> - + +#include <library/cpp/string_utils/quote/quote.h> + #include <thread> #include <unordered_map> -namespace NYdb { - -constexpr int PESSIMIZATION_DISCOVERY_THRESHOLD = 50; // percent of endpoints pessimized by transport error to start recheck -constexpr TDuration ENDPOINT_UPDATE_PERIOD = TDuration::Minutes(1); // period to perform endpoints update in "normal" case -constexpr TDuration DISCOVERY_RECHECK_PERIOD = TDuration::Seconds(5); // period to run periodic discovery task - -TDbDriverState::TDbDriverState( +namespace NYdb { + +constexpr int PESSIMIZATION_DISCOVERY_THRESHOLD = 50; // percent of endpoints pessimized by transport error to start recheck +constexpr TDuration ENDPOINT_UPDATE_PERIOD = TDuration::Minutes(1); // period to perform endpoints update in "normal" case +constexpr TDuration DISCOVERY_RECHECK_PERIOD = TDuration::Seconds(5); // period to run periodic discovery task + +TDbDriverState::TDbDriverState( const TStringType& database, const TStringType& discoveryEndpoint, - EDiscoveryMode discoveryMode, + EDiscoveryMode discoveryMode, bool enableSsl, - IInternalClient* client -) - : Database(database) - , DiscoveryEndpoint(discoveryEndpoint) - , DiscoveryMode(discoveryMode) + IInternalClient* client +) + : Database(database) + , DiscoveryEndpoint(discoveryEndpoint) + , DiscoveryMode(discoveryMode) , EnableSsl(enableSsl) - , Client(client) - , EndpointPool([this, client]() mutable { - // this callback will be called just after shared_ptr initialization - // so this call is safe - auto self = shared_from_this(); - return client->GetEndpoints(self); - }, client) - , StatCollector(database, client->GetMetricRegistry()) - , Log(Client->GetLog()) -{ + , Client(client) + , EndpointPool([this, client]() mutable { + // this callback will be called just after shared_ptr initialization + // so this call is safe + auto self = shared_from_this(); + return client->GetEndpoints(self); + }, client) + , StatCollector(database, client->GetMetricRegistry()) + , Log(Client->GetLog()) +{ EndpointPool.SetStatCollector(StatCollector); Log.SetFormatter(GetPrefixLogFormatter(GetDatabaseLogPrefix(Database))); } void TDbDriverState::SetCredentialsProvider(std::shared_ptr<ICredentialsProvider> credentialsProvider) { CredentialsProvider = std::move(credentialsProvider); -#ifndef YDB_GRPC_UNSECURE_AUTH - CallCredentials = grpc::MetadataCredentialsFromPlugin( - std::unique_ptr<grpc::MetadataCredentialsPlugin>(new TYdbAuthenticator(CredentialsProvider))); -#endif -} - -void TDbDriverState::AddCb(TCb&& cb, ENotifyType type) { +#ifndef YDB_GRPC_UNSECURE_AUTH + CallCredentials = grpc::MetadataCredentialsFromPlugin( + std::unique_ptr<grpc::MetadataCredentialsPlugin>(new TYdbAuthenticator(CredentialsProvider))); +#endif +} + +void TDbDriverState::AddCb(TCb&& cb, ENotifyType type) { std::lock_guard lock(NotifyCbsLock); NotifyCbs[static_cast<size_t>(type)].emplace_back(std::move(cb)); -} - -void TDbDriverState::ForEachEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const { - EndpointPool.ForEachEndpoint(cb, tag); -} - -void TDbDriverState::ForEachLocalEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const { - EndpointPool.ForEachLocalEndpoint(cb, tag); -} - -void TDbDriverState::ForEachForeignEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const { - EndpointPool.ForEachForeignEndpoint(cb, tag); -} - -EBalancingPolicy TDbDriverState::GetBalancingPolicy() const { - return EndpointPool.GetBalancingPolicy(); -} - +} + +void TDbDriverState::ForEachEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const { + EndpointPool.ForEachEndpoint(cb, tag); +} + +void TDbDriverState::ForEachLocalEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const { + EndpointPool.ForEachLocalEndpoint(cb, tag); +} + +void TDbDriverState::ForEachForeignEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const { + EndpointPool.ForEachForeignEndpoint(cb, tag); +} + +EBalancingPolicy TDbDriverState::GetBalancingPolicy() const { + return EndpointPool.GetBalancingPolicy(); +} + TStringType TDbDriverState::GetEndpoint() const { return EndpointPool.GetEndpoint(TStringType()).Endpoint; -} - -TPeriodicCb CreatePeriodicDiscoveryTask(TDbDriverState::TPtr driverState) { - auto weak = std::weak_ptr<TDbDriverState>(driverState); - return [weak](NYql::TIssues&&, EStatus status) { - if (status != EStatus::SUCCESS) { - return false; - } - - TDbDriverState::TPtr strong = weak.lock(); - if (!strong) { - return false; - } else { - - bool pessThreshold = strong->EndpointPool.GetPessimizationRatio() > PESSIMIZATION_DISCOVERY_THRESHOLD; - bool expiration = strong->EndpointPool.TimeSinceLastUpdate() > ENDPOINT_UPDATE_PERIOD; - - if (pessThreshold) { - strong->StatCollector.IncDiscoveryDuePessimization(); - } - if (expiration) { - strong->StatCollector.IncDiscoveryDueExpiration(); - } - - if (pessThreshold || expiration) { - auto asyncResult = strong->EndpointPool.UpdateAsync(); - // true - we were first who run UpdateAsync - if (asyncResult.second) { - auto cb = [strong](const NThreading::TFuture<TEndpointUpdateResult>& future) { - const auto& updateResult = future.GetValue(); -#ifndef YDB_GRPC_BYPASS_CHANNEL_POOL - strong->Client->DeleteChannels(updateResult.Removed); -#endif - if (strong->DiscoveryMode == EDiscoveryMode::Sync) { +} + +TPeriodicCb CreatePeriodicDiscoveryTask(TDbDriverState::TPtr driverState) { + auto weak = std::weak_ptr<TDbDriverState>(driverState); + return [weak](NYql::TIssues&&, EStatus status) { + if (status != EStatus::SUCCESS) { + return false; + } + + TDbDriverState::TPtr strong = weak.lock(); + if (!strong) { + return false; + } else { + + bool pessThreshold = strong->EndpointPool.GetPessimizationRatio() > PESSIMIZATION_DISCOVERY_THRESHOLD; + bool expiration = strong->EndpointPool.TimeSinceLastUpdate() > ENDPOINT_UPDATE_PERIOD; + + if (pessThreshold) { + strong->StatCollector.IncDiscoveryDuePessimization(); + } + if (expiration) { + strong->StatCollector.IncDiscoveryDueExpiration(); + } + + if (pessThreshold || expiration) { + auto asyncResult = strong->EndpointPool.UpdateAsync(); + // true - we were first who run UpdateAsync + if (asyncResult.second) { + auto cb = [strong](const NThreading::TFuture<TEndpointUpdateResult>& future) { + const auto& updateResult = future.GetValue(); +#ifndef YDB_GRPC_BYPASS_CHANNEL_POOL + strong->Client->DeleteChannels(updateResult.Removed); +#endif + if (strong->DiscoveryMode == EDiscoveryMode::Sync) { std::unique_lock guard(strong->LastDiscoveryStatusRWLock); - strong->LastDiscoveryStatus = updateResult.DiscoveryStatus; - } - }; - asyncResult.first.Subscribe(std::move(cb)); - } - } - } - return true; - }; -} - -TDbDriverStateTracker::TDbDriverStateTracker(IInternalClient* client) - : DiscoveryClient_(client) -{} - -TDbDriverStatePtr TDbDriverStateTracker::GetDriverState( + strong->LastDiscoveryStatus = updateResult.DiscoveryStatus; + } + }; + asyncResult.first.Subscribe(std::move(cb)); + } + } + } + return true; + }; +} + +TDbDriverStateTracker::TDbDriverStateTracker(IInternalClient* client) + : DiscoveryClient_(client) +{} + +TDbDriverStatePtr TDbDriverStateTracker::GetDriverState( TStringType database, TStringType discoveryEndpoint, - EDiscoveryMode discoveryMode, + EDiscoveryMode discoveryMode, bool enableSsl, - std::shared_ptr<ICredentialsProviderFactory> credentialsProviderFactory -) { + std::shared_ptr<ICredentialsProviderFactory> credentialsProviderFactory +) { TStringType clientIdentity; - if (credentialsProviderFactory) { - clientIdentity = credentialsProviderFactory->GetClientIdentity(); - } - Quote(database); + if (credentialsProviderFactory) { + clientIdentity = credentialsProviderFactory->GetClientIdentity(); + } + Quote(database); const TStateKey key{database, discoveryEndpoint, clientIdentity, discoveryMode, enableSsl}; - { + { std::shared_lock lock(Lock_); auto state = States_.find(key); if (state != States_.end()) { auto strong = state->second.lock(); - if (strong) { - return strong; - } - // If we can't promote to shared see bellow - } - } - TDbDriverStatePtr strongState; - for (;;) { + if (strong) { + return strong; + } + // If we can't promote to shared see bellow + } + } + TDbDriverStatePtr strongState; + for (;;) { std::unique_lock lock(Lock_); - { + { auto state = States_.find(key); if (state != States_.end()) { auto strong = state->second.lock(); - if (strong) { - return strong; - } else { - // We could find state record, but couldn't promote weak to shared - // this means weak ptr already expired but dtor hasn't been - // called yet. Likely other thread now is waiting on mutex to - // remove expired record from hashmap. So give him chance - // to do it after that we will be able to create new state + if (strong) { + return strong; + } else { + // We could find state record, but couldn't promote weak to shared + // this means weak ptr already expired but dtor hasn't been + // called yet. Likely other thread now is waiting on mutex to + // remove expired record from hashmap. So give him chance + // to do it after that we will be able to create new state lock.unlock(); std::this_thread::yield(); - continue; - } - } - } - { - auto deleter = [this, key](TDbDriverState* p) { - { + continue; + } + } + } + { + auto deleter = [this, key](TDbDriverState* p) { + { std::unique_lock lock(Lock_); - States_.erase(key); - } - delete p; - }; - strongState = std::shared_ptr<TDbDriverState>( - new TDbDriverState( - database, - discoveryEndpoint, - discoveryMode, + States_.erase(key); + } + delete p; + }; + strongState = std::shared_ptr<TDbDriverState>( + new TDbDriverState( + database, + discoveryEndpoint, + discoveryMode, enableSsl, - DiscoveryClient_), - deleter); + DiscoveryClient_), + deleter); strongState->SetCredentialsProvider( credentialsProviderFactory ? credentialsProviderFactory->CreateProvider(strongState) : CreateInsecureCredentialsProviderFactory()->CreateProvider(strongState)); - DiscoveryClient_->AddPeriodicTask(CreatePeriodicDiscoveryTask(strongState), DISCOVERY_RECHECK_PERIOD); - Y_VERIFY(States_.emplace(key, strongState).second); - break; - } - } - auto updateResult = strongState->EndpointPool.UpdateAsync(); + DiscoveryClient_->AddPeriodicTask(CreatePeriodicDiscoveryTask(strongState), DISCOVERY_RECHECK_PERIOD); + Y_VERIFY(States_.emplace(key, strongState).second); + break; + } + } + auto updateResult = strongState->EndpointPool.UpdateAsync(); - if (strongState->DiscoveryMode == EDiscoveryMode::Sync) { + if (strongState->DiscoveryMode == EDiscoveryMode::Sync) { const auto& discoveryStatus = updateResult.first.GetValueSync().DiscoveryStatus; // Almost always true, except the situation when the current thread was // preempted just before UpdateAsync call and other one get @@ -204,61 +204,61 @@ TDbDriverStatePtr TDbDriverStateTracker::GetDriverState( std::unique_lock guard(strongState->LastDiscoveryStatusRWLock); strongState->LastDiscoveryStatus = discoveryStatus; } - } + } + + return strongState; +} - return strongState; -} - void TDbDriverState::AddPeriodicTask(TPeriodicCb&& cb, TDuration period) { Client->AddPeriodicTask(std::move(cb), period); } -NThreading::TFuture<void> TDbDriverStateTracker::SendNotification( +NThreading::TFuture<void> TDbDriverStateTracker::SendNotification( TDbDriverState::ENotifyType type -) { +) { std::vector<std::weak_ptr<TDbDriverState>> states; - { + { std::shared_lock lock(Lock_); - states.reserve(States_.size()); - for (auto& weak : States_) { - states.push_back(weak.second); - } - } + states.reserve(States_.size()); + for (auto& weak : States_) { + states.push_back(weak.second); + } + } std::vector<NThreading::TFuture<void>> results; - for (auto& state : states) { - auto strong = state.lock(); - if (strong) { + for (auto& state : states) { + auto strong = state.lock(); + if (strong) { std::lock_guard lock(strong->NotifyCbsLock); for (auto& cb : strong->NotifyCbs[static_cast<size_t>(type)]) { if (cb) { auto future = cb(); if (!future.HasException()) { results.push_back(future); - } + } //TODO: Add loger - } - } - } - } - return NThreading::WaitExceptionOrAll(results); -} - -void TDbDriverStateTracker::SetMetricRegistry(NMonitoring::TMetricRegistry *sensorsRegistry) { + } + } + } + } + return NThreading::WaitExceptionOrAll(results); +} + +void TDbDriverStateTracker::SetMetricRegistry(NMonitoring::TMetricRegistry *sensorsRegistry) { std::vector<std::weak_ptr<TDbDriverState>> states; - { + { std::shared_lock lock(Lock_); - states.reserve(States_.size()); - for (auto& weak : States_) { - states.push_back(weak.second); - } - } - - for (auto& weak : states) { - if (auto strong = weak.lock()) { - strong->StatCollector.SetMetricRegistry(sensorsRegistry); - strong->EndpointPool.SetStatCollector(strong->StatCollector); - } - } -} - -} // namespace NYdb + states.reserve(States_.size()); + for (auto& weak : States_) { + states.push_back(weak.second); + } + } + + for (auto& weak : states) { + if (auto strong = weak.lock()) { + strong->StatCollector.SetMetricRegistry(sensorsRegistry); + strong->EndpointPool.SetStatCollector(strong->StatCollector); + } + } +} + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/state.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/state.h index 6df22f2d86..e0ce63e4b8 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/state.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/state.h @@ -1,71 +1,71 @@ -#pragma once - -#include "endpoint_pool.h" - +#pragma once + +#include "endpoint_pool.h" + #include <ydb/public/sdk/cpp/client/impl/ydb_internal/internal_header.h> - + #include <ydb/public/sdk/cpp/client/impl/ydb_internal/internal_client/client.h> #include <ydb/public/sdk/cpp/client/ydb_types/core_facility/core_facility.h> - -namespace NYdb { -class ICredentialsProvider; -class ICredentialsProviderFactory; - -// Represents state of driver for one particular database +namespace NYdb { + +class ICredentialsProvider; +class ICredentialsProviderFactory; + +// Represents state of driver for one particular database class TDbDriverState : public std::enable_shared_from_this<TDbDriverState> , public ICoreFacility { public: - enum class ENotifyType : size_t { - STOP = 0, - COUNT = 1 // types count - }; - + enum class ENotifyType : size_t { + STOP = 0, + COUNT = 1 // types count + }; + using TCb = std::function<NThreading::TFuture<void>()>; - using TPtr = std::shared_ptr<TDbDriverState>; - - TDbDriverState( + using TPtr = std::shared_ptr<TDbDriverState>; + + TDbDriverState( const TStringType& database, const TStringType& discoveryEndpoint, - EDiscoveryMode discoveryMode, + EDiscoveryMode discoveryMode, bool enableSsl, - IInternalClient* client - ); - + IInternalClient* client + ); + void AddPeriodicTask(TPeriodicCb&& cb, TDuration period) override; - void AddCb(TCb&& cb, ENotifyType type); - void ForEachEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const; - void ForEachLocalEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const; - void ForEachForeignEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const; - EBalancingPolicy GetBalancingPolicy() const; + void AddCb(TCb&& cb, ENotifyType type); + void ForEachEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const; + void ForEachLocalEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const; + void ForEachForeignEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const; + EBalancingPolicy GetBalancingPolicy() const; TStringType GetEndpoint() const; void SetCredentialsProvider(std::shared_ptr<ICredentialsProvider> credentialsProvider); - + const TStringType Database; const TStringType DiscoveryEndpoint; - const EDiscoveryMode DiscoveryMode; + const EDiscoveryMode DiscoveryMode; const bool EnableSsl; - std::shared_ptr<ICredentialsProvider> CredentialsProvider; - IInternalClient* Client; - TEndpointPool EndpointPool; - // StopCb allow client to subscribe for notifications from lower layer + std::shared_ptr<ICredentialsProvider> CredentialsProvider; + IInternalClient* Client; + TEndpointPool EndpointPool; + // StopCb allow client to subscribe for notifications from lower layer std::mutex NotifyCbsLock; std::array<std::vector<TCb>, static_cast<size_t>(ENotifyType::COUNT)> NotifyCbs; -#ifndef YDB_GRPC_UNSECURE_AUTH - std::shared_ptr<grpc::CallCredentials> CallCredentials; -#endif - // Status of last discovery call, used in sync mode, coresponding mutex +#ifndef YDB_GRPC_UNSECURE_AUTH + std::shared_ptr<grpc::CallCredentials> CallCredentials; +#endif + // Status of last discovery call, used in sync mode, coresponding mutex std::shared_mutex LastDiscoveryStatusRWLock; - TPlainStatus LastDiscoveryStatus; - NSdkStats::TStatCollector StatCollector; - TLog Log; -}; - -// Tracker allows to get driver state by database and credentials -class TDbDriverStateTracker { + TPlainStatus LastDiscoveryStatus; + NSdkStats::TStatCollector StatCollector; + TLog Log; +}; + +// Tracker allows to get driver state by database and credentials +class TDbDriverStateTracker { using TStateKey = std::tuple<TStringType, TStringType, TStringType, EDiscoveryMode, bool>; struct TStateKeyHash { size_t operator()(const TStateKey& k) const noexcept { @@ -77,24 +77,24 @@ class TDbDriverStateTracker { return (h0 ^ h1 ^ h2 ^ h3); } }; -public: - TDbDriverStateTracker(IInternalClient* client); - TDbDriverState::TPtr GetDriverState( +public: + TDbDriverStateTracker(IInternalClient* client); + TDbDriverState::TPtr GetDriverState( TStringType database, TStringType DiscoveryEndpoint, - EDiscoveryMode discoveryMode, + EDiscoveryMode discoveryMode, bool enableSsl, - std::shared_ptr<ICredentialsProviderFactory> credentialsProviderFactory - ); - NThreading::TFuture<void> SendNotification( + std::shared_ptr<ICredentialsProviderFactory> credentialsProviderFactory + ); + NThreading::TFuture<void> SendNotification( TDbDriverState::ENotifyType type); - void SetMetricRegistry(NMonitoring::TMetricRegistry *sensorsRegistry); -private: - IInternalClient* DiscoveryClient_; + void SetMetricRegistry(NMonitoring::TMetricRegistry *sensorsRegistry); +private: + IInternalClient* DiscoveryClient_; std::unordered_map<TStateKey, std::weak_ptr<TDbDriverState>, TStateKeyHash> States_; std::shared_mutex Lock_; -}; - -using TDbDriverStatePtr = TDbDriverState::TPtr; - -} // namespace NYdb +}; + +using TDbDriverStatePtr = TDbDriverState::TPtr; + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/ya.make b/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/ya.make index eac291f6c8..b9d984f4ba 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/ya.make +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/ya.make @@ -1,20 +1,20 @@ -LIBRARY() - +LIBRARY() + OWNER(g:kikimr) - -SRCS( - authenticator.cpp - endpoint_pool.cpp - state.cpp -) - -PEERDIR( - library/cpp/string_utils/quote - library/cpp/threading/future + +SRCS( + authenticator.cpp + endpoint_pool.cpp + state.cpp +) + +PEERDIR( + library/cpp/string_utils/quote + library/cpp/threading/future ydb/public/sdk/cpp/client/impl/ydb_endpoints ydb/public/sdk/cpp/client/impl/ydb_internal/logger ydb/public/sdk/cpp/client/impl/ydb_internal/plain_status ydb/public/sdk/cpp/client/ydb_types/credentials -) - -END() +) + +END() diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/driver/constants.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/driver/constants.h index 96e5586d37..3aa91d560d 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/driver/constants.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/driver/constants.h @@ -1,13 +1,13 @@ -#pragma once - +#pragma once + #include <ydb/public/sdk/cpp/client/impl/ydb_internal/internal_header.h> - -namespace NYdb { - -constexpr ui64 TCP_KEEPALIVE_IDLE = 30; // The time the connection needs to remain idle - // before TCP starts sending keepalive probes, seconds -constexpr ui64 TCP_KEEPALIVE_COUNT = 5; // The maximum number of keepalive probes TCP should send before - // dropping the connection -constexpr ui64 TCP_KEEPALIVE_INTERVAL = 10; // The time between individual keepalive probes, seconds - -} // namespace NYdb + +namespace NYdb { + +constexpr ui64 TCP_KEEPALIVE_IDLE = 30; // The time the connection needs to remain idle + // before TCP starts sending keepalive probes, seconds +constexpr ui64 TCP_KEEPALIVE_COUNT = 5; // The maximum number of keepalive probes TCP should send before + // dropping the connection +constexpr ui64 TCP_KEEPALIVE_INTERVAL = 10; // The time between individual keepalive probes, seconds + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/driver/interface.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/driver/interface.h index bafeca0eb1..d49b69d6e1 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/driver/interface.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/driver/interface.h @@ -1,12 +1,12 @@ -#pragma once - +#pragma once + #include <ydb/public/sdk/cpp/client/impl/ydb_internal/internal_header.h> - -namespace NYdb { - -class TDriver; -class TGRpcConnectionsImpl; - -std::shared_ptr<TGRpcConnectionsImpl> CreateInternalInterface(const TDriver connection); - -} // namespace NYdb + +namespace NYdb { + +class TDriver; +class TGRpcConnectionsImpl; + +std::shared_ptr<TGRpcConnectionsImpl> CreateInternalInterface(const TDriver connection); + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/actions.cpp b/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/actions.cpp index f6596583ad..f2db8c9416 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/actions.cpp +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/actions.cpp @@ -1,112 +1,112 @@ -#define INCLUDE_YDB_INTERNAL_H -#include "actions.h" -#include "grpc_connections.h" - +#define INCLUDE_YDB_INTERNAL_H +#include "actions.h" +#include "grpc_connections.h" + #include <ydb/public/api/grpc/ydb_operation_v1.grpc.pb.h> - -namespace NYdb { - -constexpr TDuration MAX_DEFERRED_CALL_DELAY = TDuration::Seconds(10); // The max delay between GetOperation calls for one operation - -TSimpleCbResult::TSimpleCbResult( - TSimpleCb&& cb, - TGRpcConnectionsImpl* connections, - std::shared_ptr<IQueueClientContext> context) - : TGenericCbHolder<TSimpleCb>(std::move(cb), connections, std::move(context)) -{ } - -void TSimpleCbResult::Process(void*) { - UserResponseCb_(); - delete this; -} - + +namespace NYdb { + +constexpr TDuration MAX_DEFERRED_CALL_DELAY = TDuration::Seconds(10); // The max delay between GetOperation calls for one operation + +TSimpleCbResult::TSimpleCbResult( + TSimpleCb&& cb, + TGRpcConnectionsImpl* connections, + std::shared_ptr<IQueueClientContext> context) + : TGenericCbHolder<TSimpleCb>(std::move(cb), connections, std::move(context)) +{ } + +void TSimpleCbResult::Process(void*) { + UserResponseCb_(); + delete this; +} + TDeferredAction::TDeferredAction(const TStringType& operationId, - TDeferredOperationCb&& userCb, - TGRpcConnectionsImpl* connection, - std::shared_ptr<IQueueClientContext> context, - TDuration delay, - TDbDriverStatePtr dbState, + TDeferredOperationCb&& userCb, + TGRpcConnectionsImpl* connection, + std::shared_ptr<IQueueClientContext> context, + TDuration delay, + TDbDriverStatePtr dbState, const TStringType& endpoint) - : TAlarmActionBase(std::move(userCb), connection, std::move(context)) - , NextDelay_(Min(delay * 2, MAX_DEFERRED_CALL_DELAY)) - , DbDriverState_(dbState) - , OperationId_(operationId) - , Endpoint_(endpoint) -{ - Deadline_ = gpr_time_add( - gpr_now(GPR_CLOCK_MONOTONIC), - gpr_time_from_micros(delay.MicroSeconds(), GPR_TIMESPAN)); -} - -void TDeferredAction::OnAlarm() { - Y_VERIFY(Connection_); - - Ydb::Operations::GetOperationRequest getOperationRequest; - getOperationRequest.set_id(OperationId_); - - Connection_->RunDeferred<Ydb::Operation::V1::OperationService, Ydb::Operations::GetOperationRequest, Ydb::Operations::GetOperationResponse>( - std::move(getOperationRequest), - std::move(UserResponseCb_), - &Ydb::Operation::V1::OperationService::Stub::AsyncGetOperation, - DbDriverState_, - NextDelay_, - {}, - TDuration::Zero(), - true, - Endpoint_, - std::move(Context_)); -} - -void TDeferredAction::OnError() { - Y_VERIFY(Connection_); - NGrpc::TGrpcStatus status = {"Deferred timer interrupted", -1, true}; - DbDriverState_->StatCollector.IncDiscoveryFailDueTransportError(); - - auto resp = new TGRpcErrorResponse<Ydb::Operations::Operation>( - std::move(status), - std::move(UserResponseCb_), - Connection_, - std::move(Context_), - Endpoint_); - Connection_->EnqueueResponse(resp); -} - -TPeriodicAction::TPeriodicAction( - TPeriodicCb&& userCb, - TGRpcConnectionsImpl* connection, - std::shared_ptr<NGrpc::IQueueClientContext> context, - TDuration period) - : TAlarmActionBase(std::move(userCb), connection, std::move(context)) - , Period_(period) -{ - Deadline_ = gpr_time_add( - gpr_now(GPR_CLOCK_MONOTONIC), - gpr_time_from_micros(Period_.MicroSeconds(), GPR_TIMESPAN)); -} - -void TPeriodicAction::OnAlarm() { - NYql::TIssues issues; - if (!UserResponseCb_(std::move(issues), EStatus::SUCCESS)) { - return; - } - - auto ctx = Connection_->CreateContext(); - if (!ctx) - return; - Context_ = ctx; - - auto action = MakeIntrusive<TPeriodicAction>( - std::move(UserResponseCb_), - Connection_, - Context_, - Period_); - action->Start(); -} - -void TPeriodicAction::OnError() { - NYql::TIssues issues; - issues.AddIssue(NYql::TIssue("Deferred timer interrupted")); - UserResponseCb_(std::move(issues), EStatus::CLIENT_INTERNAL_ERROR); -} - -} // namespace NYdb + : TAlarmActionBase(std::move(userCb), connection, std::move(context)) + , NextDelay_(Min(delay * 2, MAX_DEFERRED_CALL_DELAY)) + , DbDriverState_(dbState) + , OperationId_(operationId) + , Endpoint_(endpoint) +{ + Deadline_ = gpr_time_add( + gpr_now(GPR_CLOCK_MONOTONIC), + gpr_time_from_micros(delay.MicroSeconds(), GPR_TIMESPAN)); +} + +void TDeferredAction::OnAlarm() { + Y_VERIFY(Connection_); + + Ydb::Operations::GetOperationRequest getOperationRequest; + getOperationRequest.set_id(OperationId_); + + Connection_->RunDeferred<Ydb::Operation::V1::OperationService, Ydb::Operations::GetOperationRequest, Ydb::Operations::GetOperationResponse>( + std::move(getOperationRequest), + std::move(UserResponseCb_), + &Ydb::Operation::V1::OperationService::Stub::AsyncGetOperation, + DbDriverState_, + NextDelay_, + {}, + TDuration::Zero(), + true, + Endpoint_, + std::move(Context_)); +} + +void TDeferredAction::OnError() { + Y_VERIFY(Connection_); + NGrpc::TGrpcStatus status = {"Deferred timer interrupted", -1, true}; + DbDriverState_->StatCollector.IncDiscoveryFailDueTransportError(); + + auto resp = new TGRpcErrorResponse<Ydb::Operations::Operation>( + std::move(status), + std::move(UserResponseCb_), + Connection_, + std::move(Context_), + Endpoint_); + Connection_->EnqueueResponse(resp); +} + +TPeriodicAction::TPeriodicAction( + TPeriodicCb&& userCb, + TGRpcConnectionsImpl* connection, + std::shared_ptr<NGrpc::IQueueClientContext> context, + TDuration period) + : TAlarmActionBase(std::move(userCb), connection, std::move(context)) + , Period_(period) +{ + Deadline_ = gpr_time_add( + gpr_now(GPR_CLOCK_MONOTONIC), + gpr_time_from_micros(Period_.MicroSeconds(), GPR_TIMESPAN)); +} + +void TPeriodicAction::OnAlarm() { + NYql::TIssues issues; + if (!UserResponseCb_(std::move(issues), EStatus::SUCCESS)) { + return; + } + + auto ctx = Connection_->CreateContext(); + if (!ctx) + return; + Context_ = ctx; + + auto action = MakeIntrusive<TPeriodicAction>( + std::move(UserResponseCb_), + Connection_, + Context_, + Period_); + action->Start(); +} + +void TPeriodicAction::OnError() { + NYql::TIssues issues; + issues.AddIssue(NYql::TIssue("Deferred timer interrupted")); + UserResponseCb_(std::move(issues), EStatus::CLIENT_INTERNAL_ERROR); +} + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/actions.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/actions.h index 7be068bf50..ebd16c5a83 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/actions.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/actions.h @@ -1,232 +1,232 @@ -#pragma once - +#pragma once + #include <ydb/public/sdk/cpp/client/impl/ydb_internal/internal_header.h> - + #include <ydb/public/api/protos/ydb_operation.pb.h> #include <ydb/public/sdk/cpp/client/impl/ydb_internal/common/types.h> #include <ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/state.h> #include <ydb/public/sdk/cpp/client/impl/ydb_internal/plain_status/status.h> - -#include <library/cpp/grpc/client/grpc_client_low.h> - -#include <util/thread/pool.h> - -#include <grpc++/alarm.h> - -namespace NYdb { - -using NGrpc::IQueueClientContext; -using NGrpc::IQueueClientEvent; - -class TGRpcConnectionsImpl; -struct TPlainStatus; - - -template<typename TResponse> -using TResponseCb = std::function<void(TResponse*, TPlainStatus status)>; -using TDeferredOperationCb = std::function<void(Ydb::Operations::Operation*, TPlainStatus status)>; - -template<typename TCb> -class TGenericCbHolder { -protected: - TGenericCbHolder( - TCb&& userCb, - TGRpcConnectionsImpl* connections, - std::shared_ptr<IQueueClientContext> context) - : UserResponseCb_(std::move(userCb)) - , Connection_(connections) - , Context_(std::move(context)) - {} - - TCb UserResponseCb_; - TGRpcConnectionsImpl* Connection_; - std::shared_ptr<IQueueClientContext> Context_; -}; - -template<typename TCb> -class TAlarmActionBase - : public TThrRefBase - , public TGenericCbHolder<TCb> - , private IQueueClientEvent -{ -public: - using TPtr = TIntrusivePtr<TAlarmActionBase<TCb>>; - using TGenericCbHolder<TCb>::TGenericCbHolder; - - virtual void OnAlarm() = 0; - virtual void OnError() = 0; - - void Start() { - Y_VERIFY(this->Context_, "Missing shared context"); - auto context = this->Context_->CreateContext(); + +#include <library/cpp/grpc/client/grpc_client_low.h> + +#include <util/thread/pool.h> + +#include <grpc++/alarm.h> + +namespace NYdb { + +using NGrpc::IQueueClientContext; +using NGrpc::IQueueClientEvent; + +class TGRpcConnectionsImpl; +struct TPlainStatus; + + +template<typename TResponse> +using TResponseCb = std::function<void(TResponse*, TPlainStatus status)>; +using TDeferredOperationCb = std::function<void(Ydb::Operations::Operation*, TPlainStatus status)>; + +template<typename TCb> +class TGenericCbHolder { +protected: + TGenericCbHolder( + TCb&& userCb, + TGRpcConnectionsImpl* connections, + std::shared_ptr<IQueueClientContext> context) + : UserResponseCb_(std::move(userCb)) + , Connection_(connections) + , Context_(std::move(context)) + {} + + TCb UserResponseCb_; + TGRpcConnectionsImpl* Connection_; + std::shared_ptr<IQueueClientContext> Context_; +}; + +template<typename TCb> +class TAlarmActionBase + : public TThrRefBase + , public TGenericCbHolder<TCb> + , private IQueueClientEvent +{ +public: + using TPtr = TIntrusivePtr<TAlarmActionBase<TCb>>; + using TGenericCbHolder<TCb>::TGenericCbHolder; + + virtual void OnAlarm() = 0; + virtual void OnError() = 0; + + void Start() { + Y_VERIFY(this->Context_, "Missing shared context"); + auto context = this->Context_->CreateContext(); { std::lock_guard lock(Mutex_); - LocalContext_ = context; - Alarm_.Set(this->Context_->CompletionQueue(), Deadline_, PrepareTag()); - } - context->SubscribeStop([self = TPtr(this)] { - self->Stop(); - }); - } - - void Stop() { - Alarm_.Cancel(); - } - -private: - IQueueClientEvent* PrepareTag() { - Ref(); - return this; - } - - bool Execute(bool ok) override { + LocalContext_ = context; + Alarm_.Set(this->Context_->CompletionQueue(), Deadline_, PrepareTag()); + } + context->SubscribeStop([self = TPtr(this)] { + self->Stop(); + }); + } + + void Stop() { + Alarm_.Cancel(); + } + +private: + IQueueClientEvent* PrepareTag() { + Ref(); + return this; + } + + bool Execute(bool ok) override { { std::lock_guard lock(Mutex_); - LocalContext_.reset(); - } - - if (ok) { - OnAlarm(); - } else { - OnError(); - } - - return false; - } - - void Destroy() override { - UnRef(); - } - -protected: + LocalContext_.reset(); + } + + if (ok) { + OnAlarm(); + } else { + OnError(); + } + + return false; + } + + void Destroy() override { + UnRef(); + } + +protected: gpr_timespec Deadline_ = {}; - -private: + +private: std::mutex Mutex_; - grpc::Alarm Alarm_; - std::shared_ptr<IQueueClientContext> LocalContext_; -}; - -template<typename TResponse> -class TGRpcErrorResponse - : public TGenericCbHolder<TResponseCb<TResponse>> - , public IObjectInQueue -{ -public: - TGRpcErrorResponse( - NGrpc::TGrpcStatus&& status, - TResponseCb<TResponse>&& userCb, - TGRpcConnectionsImpl* connections, - std::shared_ptr<IQueueClientContext> context, + grpc::Alarm Alarm_; + std::shared_ptr<IQueueClientContext> LocalContext_; +}; + +template<typename TResponse> +class TGRpcErrorResponse + : public TGenericCbHolder<TResponseCb<TResponse>> + , public IObjectInQueue +{ +public: + TGRpcErrorResponse( + NGrpc::TGrpcStatus&& status, + TResponseCb<TResponse>&& userCb, + TGRpcConnectionsImpl* connections, + std::shared_ptr<IQueueClientContext> context, const TStringType& endpoint) - : TGenericCbHolder<TResponseCb<TResponse>>(std::move(userCb), connections, std::move(context)) - , GRpcStatus_(std::move(status)) - , Endpoint_(endpoint) - { } - - void Process(void*) override { - TPlainStatus status(GRpcStatus_, Endpoint_, {}); - + : TGenericCbHolder<TResponseCb<TResponse>>(std::move(userCb), connections, std::move(context)) + , GRpcStatus_(std::move(status)) + , Endpoint_(endpoint) + { } + + void Process(void*) override { + TPlainStatus status(GRpcStatus_, Endpoint_, {}); + if (!Endpoint_.empty()) { TStringType msg = "Grpc error response on endpoint "; - msg += Endpoint_; - status.Issues.AddIssue(NYql::TIssue(msg)); - } - - this->UserResponseCb_(nullptr, status); - delete this; - } - -private: - NGrpc::TGrpcStatus GRpcStatus_; + msg += Endpoint_; + status.Issues.AddIssue(NYql::TIssue(msg)); + } + + this->UserResponseCb_(nullptr, status); + delete this; + } + +private: + NGrpc::TGrpcStatus GRpcStatus_; TStringType Endpoint_; -}; - -template<typename TResponse> -class TResult - : public TGenericCbHolder<TResponseCb<TResponse>> - , public IObjectInQueue -{ -public: - TResult( - TResponse&& response, - NGrpc::TGrpcStatus&& status, - TResponseCb<TResponse>&& userCb, - TGRpcConnectionsImpl* connections, - std::shared_ptr<IQueueClientContext> context, +}; + +template<typename TResponse> +class TResult + : public TGenericCbHolder<TResponseCb<TResponse>> + , public IObjectInQueue +{ +public: + TResult( + TResponse&& response, + NGrpc::TGrpcStatus&& status, + TResponseCb<TResponse>&& userCb, + TGRpcConnectionsImpl* connections, + std::shared_ptr<IQueueClientContext> context, const TStringType& endpoint, std::multimap<TStringType, TStringType>&& metadata) - : TGenericCbHolder<TResponseCb<TResponse>>(std::move(userCb), connections, std::move(context)) - , Response_(std::move(response)) - , GRpcStatus_(std::move(status)) - , Endpoint_(endpoint) - , Metadata_(std::move(metadata)) {} - - void Process(void*) override { - this->UserResponseCb_(&Response_, TPlainStatus{GRpcStatus_, Endpoint_, std::move(Metadata_)}); - delete this; - } - -private: - TResponse Response_; - NGrpc::TGrpcStatus GRpcStatus_; + : TGenericCbHolder<TResponseCb<TResponse>>(std::move(userCb), connections, std::move(context)) + , Response_(std::move(response)) + , GRpcStatus_(std::move(status)) + , Endpoint_(endpoint) + , Metadata_(std::move(metadata)) {} + + void Process(void*) override { + this->UserResponseCb_(&Response_, TPlainStatus{GRpcStatus_, Endpoint_, std::move(Metadata_)}); + delete this; + } + +private: + TResponse Response_; + NGrpc::TGrpcStatus GRpcStatus_; const TStringType Endpoint_; std::multimap<TStringType, TStringType> Metadata_; -}; - -class TSimpleCbResult - : public TGenericCbHolder<TSimpleCb> - , public IObjectInQueue -{ -public: - TSimpleCbResult( - TSimpleCb&& cb, - TGRpcConnectionsImpl* connections, - std::shared_ptr<IQueueClientContext> context); - void Process(void*) override; -}; - -//////////////////////////////////////////////////////////////////////////////// - -class TDeferredAction - : public TAlarmActionBase<TDeferredOperationCb> -{ -public: - using TPtr = TIntrusivePtr<TDeferredAction>; - - TDeferredAction( +}; + +class TSimpleCbResult + : public TGenericCbHolder<TSimpleCb> + , public IObjectInQueue +{ +public: + TSimpleCbResult( + TSimpleCb&& cb, + TGRpcConnectionsImpl* connections, + std::shared_ptr<IQueueClientContext> context); + void Process(void*) override; +}; + +//////////////////////////////////////////////////////////////////////////////// + +class TDeferredAction + : public TAlarmActionBase<TDeferredOperationCb> +{ +public: + using TPtr = TIntrusivePtr<TDeferredAction>; + + TDeferredAction( const TStringType& operationId, - TDeferredOperationCb&& userCb, - TGRpcConnectionsImpl* connection, - std::shared_ptr<IQueueClientContext> context, - TDuration timeout, - TDbDriverStatePtr dbState, + TDeferredOperationCb&& userCb, + TGRpcConnectionsImpl* connection, + std::shared_ptr<IQueueClientContext> context, + TDuration timeout, + TDbDriverStatePtr dbState, const TStringType& endpoint); - - void OnAlarm() override; - void OnError() override; - -private: - TDuration NextDelay_; - TDbDriverStatePtr DbDriverState_; + + void OnAlarm() override; + void OnError() override; + +private: + TDuration NextDelay_; + TDbDriverStatePtr DbDriverState_; const TStringType OperationId_; const TStringType Endpoint_; -}; - -class TPeriodicAction - : public TAlarmActionBase<TPeriodicCb> -{ -public: - TPeriodicAction( - TPeriodicCb&& userCb, - TGRpcConnectionsImpl* connection, - std::shared_ptr<IQueueClientContext> context, - TDuration period); - - void OnAlarm() override; - void OnError() override; -private: - TDuration Period_; -}; - -} // namespace NYdb +}; + +class TPeriodicAction + : public TAlarmActionBase<TPeriodicCb> +{ +public: + TPeriodicAction( + TPeriodicCb&& userCb, + TGRpcConnectionsImpl* connection, + std::shared_ptr<IQueueClientContext> context, + TDuration period); + + void OnAlarm() override; + void OnError() override; +private: + TDuration Period_; +}; + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/grpc_connections.cpp b/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/grpc_connections.cpp index 9409c09870..8f2420043e 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/grpc_connections.cpp +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/grpc_connections.cpp @@ -1,162 +1,162 @@ -#define INCLUDE_YDB_INTERNAL_H -#include "grpc_connections.h" - +#define INCLUDE_YDB_INTERNAL_H +#include "grpc_connections.h" + #include <ydb/public/sdk/cpp/client/ydb_types/exceptions/exceptions.h> - -namespace NYdb { - + +namespace NYdb { + bool IsTokenCorrect(const TStringType& in) { - for (char c : in) { - if (!(IsAsciiAlnum(c) || IsAsciiPunct(c) || c == ' ')) - return false; - } - return true; -} - + for (char c : in) { + if (!(IsAsciiAlnum(c) || IsAsciiPunct(c) || c == ' ')) + return false; + } + return true; +} + TStringType GetAuthInfo(TDbDriverStatePtr p) { - auto token = p->CredentialsProvider->GetAuthInfo(); - if (!IsTokenCorrect(token)) { - throw TContractViolation("token is incorrect, iligal characters found"); - } - return token; -} - + auto token = p->CredentialsProvider->GetAuthInfo(); + if (!IsTokenCorrect(token)) { + throw TContractViolation("token is incorrect, iligal characters found"); + } + return token; +} + void SetDatabaseHeader(TCallMeta& meta, const TStringType& database) { - // See TDbDriverStateTracker::GetDriverState to find place where we do quote non ASCII characters - meta.Aux.push_back({YDB_DATABASE_HEADER, database}); -} - + // See TDbDriverStateTracker::GetDriverState to find place where we do quote non ASCII characters + meta.Aux.push_back({YDB_DATABASE_HEADER, database}); +} + TStringType CreateSDKBuildInfo() { return TStringType("ydb-cpp-sdk/") + GetSdkSemver(); -} - -template<class TDerived> -class TScheduledObject : public TThrRefBase { - using TSelf = TScheduledObject<TDerived>; - using TPtr = TIntrusivePtr<TSelf>; - - Y_FORCE_INLINE TDerived* Derived() { - return static_cast<TDerived*>(this); - } - -protected: - TScheduledObject() { } - - void Start(TDuration timeout, IQueueClientContextProvider* provider) { - auto context = provider->CreateContext(); - if (!context) { - Derived()->OnComplete(false); - return; - } - - auto deadline = gpr_time_add( - gpr_now(GPR_CLOCK_MONOTONIC), - gpr_time_from_micros(timeout.MicroSeconds(), GPR_TIMESPAN)); - +} + +template<class TDerived> +class TScheduledObject : public TThrRefBase { + using TSelf = TScheduledObject<TDerived>; + using TPtr = TIntrusivePtr<TSelf>; + + Y_FORCE_INLINE TDerived* Derived() { + return static_cast<TDerived*>(this); + } + +protected: + TScheduledObject() { } + + void Start(TDuration timeout, IQueueClientContextProvider* provider) { + auto context = provider->CreateContext(); + if (!context) { + Derived()->OnComplete(false); + return; + } + + auto deadline = gpr_time_add( + gpr_now(GPR_CLOCK_MONOTONIC), + gpr_time_from_micros(timeout.MicroSeconds(), GPR_TIMESPAN)); + { std::lock_guard guard(Mutex); - Context = context; - Alarm.Set(context->CompletionQueue(), deadline, OnAlarmTag.Prepare()); - } - - context->SubscribeCancel([self = TPtr(this)] { - self->Alarm.Cancel(); - }); - } - -private: - void OnAlarm(bool ok) { + Context = context; + Alarm.Set(context->CompletionQueue(), deadline, OnAlarmTag.Prepare()); + } + + context->SubscribeCancel([self = TPtr(this)] { + self->Alarm.Cancel(); + }); + } + +private: + void OnAlarm(bool ok) { { std::lock_guard guard(Mutex); - // Break circular dependencies - Context.reset(); - } - - Derived()->OnComplete(ok); - } - -private: + // Break circular dependencies + Context.reset(); + } + + Derived()->OnComplete(ok); + } + +private: std::mutex Mutex; - IQueueClientContextPtr Context; - grpc::Alarm Alarm; - -private: - using TFixedEvent = NGrpc::TQueueClientFixedEvent<TSelf>; - - TFixedEvent OnAlarmTag = { this, &TSelf::OnAlarm }; -}; - -class TScheduledCallback : public TScheduledObject<TScheduledCallback> { - using TBase = TScheduledObject<TScheduledCallback>; - -public: - using TCallback = std::function<void(bool)>; - - TScheduledCallback(TCallback&& callback) - : Callback(std::move(callback)) - { } - - void Start(TDuration timeout, IQueueClientContextProvider* provider) { - TBase::Start(timeout, provider); - } - - void OnComplete(bool ok) { - Callback(ok); - Callback = { }; - } - -private: - TCallback Callback; -}; - -class TScheduledFuture : public TScheduledObject<TScheduledFuture> { - using TBase = TScheduledObject<TScheduledFuture>; - -public: - TScheduledFuture() - : Promise(NThreading::NewPromise<bool>()) - { } - - NThreading::TFuture<bool> Start(TDuration timeout, IQueueClientContextProvider* provider) { - auto future = Promise.GetFuture(); - - TBase::Start(timeout, provider); - - return future; - } - - void OnComplete(bool ok) { - Promise.SetValue(ok); - Promise = { }; - } - -private: - NThreading::TPromise<bool> Promise; -}; - -TGRpcConnectionsImpl::TGRpcConnectionsImpl(std::shared_ptr<IConnectionsParams> params) - : MetricRegistryPtr_(nullptr) - , ResponseQueue_(CreateThreadPool(params->GetClientThreadsNum())) - , DefaultDiscoveryEndpoint_(params->GetEndpoint()) - , EnableSsl_(params->IsSslEnabled()) - , CaCert_(params->GetCaCert()) - , DefaultDatabase_(params->GetDatabase()) - , DefaultCredentialsProviderFactory_(params->GetCredentialsProviderFactory()) - , StateTracker_(this) - , DefaultDiscoveryMode_(params->GetDiscoveryMode()) - , MaxQueuedRequests_(params->GetMaxQueuedRequests()) - , DrainOnDtors_(params->GetDrinOnDtors()) - , BalancingSettings_(params->GetBalancingSettings()) - , GRpcKeepAliveTimeout_(params->GetGRpcKeepAliveTimeout()) - , GRpcKeepAlivePermitWithoutCalls_(params->GetGRpcKeepAlivePermitWithoutCalls()) + IQueueClientContextPtr Context; + grpc::Alarm Alarm; + +private: + using TFixedEvent = NGrpc::TQueueClientFixedEvent<TSelf>; + + TFixedEvent OnAlarmTag = { this, &TSelf::OnAlarm }; +}; + +class TScheduledCallback : public TScheduledObject<TScheduledCallback> { + using TBase = TScheduledObject<TScheduledCallback>; + +public: + using TCallback = std::function<void(bool)>; + + TScheduledCallback(TCallback&& callback) + : Callback(std::move(callback)) + { } + + void Start(TDuration timeout, IQueueClientContextProvider* provider) { + TBase::Start(timeout, provider); + } + + void OnComplete(bool ok) { + Callback(ok); + Callback = { }; + } + +private: + TCallback Callback; +}; + +class TScheduledFuture : public TScheduledObject<TScheduledFuture> { + using TBase = TScheduledObject<TScheduledFuture>; + +public: + TScheduledFuture() + : Promise(NThreading::NewPromise<bool>()) + { } + + NThreading::TFuture<bool> Start(TDuration timeout, IQueueClientContextProvider* provider) { + auto future = Promise.GetFuture(); + + TBase::Start(timeout, provider); + + return future; + } + + void OnComplete(bool ok) { + Promise.SetValue(ok); + Promise = { }; + } + +private: + NThreading::TPromise<bool> Promise; +}; + +TGRpcConnectionsImpl::TGRpcConnectionsImpl(std::shared_ptr<IConnectionsParams> params) + : MetricRegistryPtr_(nullptr) + , ResponseQueue_(CreateThreadPool(params->GetClientThreadsNum())) + , DefaultDiscoveryEndpoint_(params->GetEndpoint()) + , EnableSsl_(params->IsSslEnabled()) + , CaCert_(params->GetCaCert()) + , DefaultDatabase_(params->GetDatabase()) + , DefaultCredentialsProviderFactory_(params->GetCredentialsProviderFactory()) + , StateTracker_(this) + , DefaultDiscoveryMode_(params->GetDiscoveryMode()) + , MaxQueuedRequests_(params->GetMaxQueuedRequests()) + , DrainOnDtors_(params->GetDrinOnDtors()) + , BalancingSettings_(params->GetBalancingSettings()) + , GRpcKeepAliveTimeout_(params->GetGRpcKeepAliveTimeout()) + , GRpcKeepAlivePermitWithoutCalls_(params->GetGRpcKeepAlivePermitWithoutCalls()) , MemoryQuota_(params->GetMemoryQuota()) - , QueuedRequests_(0) -#ifndef YDB_GRPC_BYPASS_CHANNEL_POOL + , QueuedRequests_(0) +#ifndef YDB_GRPC_BYPASS_CHANNEL_POOL , ChannelPool_(params->GetTcpKeepAliveSettings(), params->GetSocketIdleTimeout()) -#endif - , GRpcClientLow_(params->GetNetworkThreadsNum()) - , Log(params->GetLog()) -{ +#endif + , GRpcClientLow_(params->GetNetworkThreadsNum()) + , Log(params->GetLog()) +{ #ifndef YDB_GRPC_BYPASS_CHANNEL_POOL if (params->GetSocketIdleTimeout() != TDuration::Max()) { auto channelPoolUpdateWrapper = [this] @@ -172,247 +172,247 @@ TGRpcConnectionsImpl::TGRpcConnectionsImpl(std::shared_ptr<IConnectionsParams> p AddPeriodicTask(channelPoolUpdateWrapper, params->GetSocketIdleTimeout() * 0.1); } #endif - //TAdaptiveThreadPool ignores params - ResponseQueue_->Start(params->GetClientThreadsNum(), params->GetMaxQueuedResponses()); + //TAdaptiveThreadPool ignores params + ResponseQueue_->Start(params->GetClientThreadsNum(), params->GetMaxQueuedResponses()); if (!DefaultDatabase_.empty()) { - DefaultState_ = StateTracker_.GetDriverState( - DefaultDatabase_, - DefaultDiscoveryEndpoint_, - DefaultDiscoveryMode_, + DefaultState_ = StateTracker_.GetDriverState( + DefaultDatabase_, + DefaultDiscoveryEndpoint_, + DefaultDiscoveryMode_, EnableSsl_, - DefaultCredentialsProviderFactory_ - ); - } -} - -TGRpcConnectionsImpl::~TGRpcConnectionsImpl() { - GRpcClientLow_.Stop(true); - ResponseQueue_->Stop(); -} - -void TGRpcConnectionsImpl::AddPeriodicTask(TPeriodicCb&& cb, TDuration period) { - std::shared_ptr<IQueueClientContext> context; - if (!TryCreateContext(context)) { - NYql::TIssues issues; - cb(std::move(issues), EStatus::CLIENT_INTERNAL_ERROR); - } else { - auto action = MakeIntrusive<TPeriodicAction>( - std::move(cb), - this, - std::move(context), - period); - action->Start(); - } -} - -void TGRpcConnectionsImpl::ScheduleOneTimeTask(TSimpleCb&& fn, TDuration timeout) { - auto cbLow = [this, fn = std::move(fn)](NYql::TIssues&&, EStatus status) mutable { - if (status != EStatus::SUCCESS) { - return false; - } - - std::shared_ptr<IQueueClientContext> context; - - if (!TryCreateContext(context)) { - // Shutting down, fn must handle it - fn(); - } else { - // Enqueue to user pool - auto resp = new TSimpleCbResult( - std::move(fn), - this, - std::move(context)); - EnqueueResponse(resp); - } - - return false; - }; - - if (timeout) { - AddPeriodicTask(std::move(cbLow), timeout); - } else { - cbLow(NYql::TIssues(), EStatus::SUCCESS); - } -} - -NThreading::TFuture<bool> TGRpcConnectionsImpl::ScheduleFuture( - TDuration timeout, - IQueueClientContextPtr context) -{ - IQueueClientContextProvider* provider = context.get(); - if (!provider) { - provider = &GRpcClientLow_; - } - - return MakeIntrusive<TScheduledFuture>() - ->Start(timeout, provider); -} - -void TGRpcConnectionsImpl::ScheduleCallback( - TDuration timeout, - std::function<void(bool)> callback, - IQueueClientContextPtr context) -{ - IQueueClientContextProvider* provider = context.get(); - if (!provider) { - provider = &GRpcClientLow_; - } - - return MakeIntrusive<TScheduledCallback>(std::move(callback)) - ->Start(timeout, provider); -} - -TDbDriverStatePtr TGRpcConnectionsImpl::GetDriverState( + DefaultCredentialsProviderFactory_ + ); + } +} + +TGRpcConnectionsImpl::~TGRpcConnectionsImpl() { + GRpcClientLow_.Stop(true); + ResponseQueue_->Stop(); +} + +void TGRpcConnectionsImpl::AddPeriodicTask(TPeriodicCb&& cb, TDuration period) { + std::shared_ptr<IQueueClientContext> context; + if (!TryCreateContext(context)) { + NYql::TIssues issues; + cb(std::move(issues), EStatus::CLIENT_INTERNAL_ERROR); + } else { + auto action = MakeIntrusive<TPeriodicAction>( + std::move(cb), + this, + std::move(context), + period); + action->Start(); + } +} + +void TGRpcConnectionsImpl::ScheduleOneTimeTask(TSimpleCb&& fn, TDuration timeout) { + auto cbLow = [this, fn = std::move(fn)](NYql::TIssues&&, EStatus status) mutable { + if (status != EStatus::SUCCESS) { + return false; + } + + std::shared_ptr<IQueueClientContext> context; + + if (!TryCreateContext(context)) { + // Shutting down, fn must handle it + fn(); + } else { + // Enqueue to user pool + auto resp = new TSimpleCbResult( + std::move(fn), + this, + std::move(context)); + EnqueueResponse(resp); + } + + return false; + }; + + if (timeout) { + AddPeriodicTask(std::move(cbLow), timeout); + } else { + cbLow(NYql::TIssues(), EStatus::SUCCESS); + } +} + +NThreading::TFuture<bool> TGRpcConnectionsImpl::ScheduleFuture( + TDuration timeout, + IQueueClientContextPtr context) +{ + IQueueClientContextProvider* provider = context.get(); + if (!provider) { + provider = &GRpcClientLow_; + } + + return MakeIntrusive<TScheduledFuture>() + ->Start(timeout, provider); +} + +void TGRpcConnectionsImpl::ScheduleCallback( + TDuration timeout, + std::function<void(bool)> callback, + IQueueClientContextPtr context) +{ + IQueueClientContextProvider* provider = context.get(); + if (!provider) { + provider = &GRpcClientLow_; + } + + return MakeIntrusive<TScheduledCallback>(std::move(callback)) + ->Start(timeout, provider); +} + +TDbDriverStatePtr TGRpcConnectionsImpl::GetDriverState( const TMaybe<TStringType>& database, const TMaybe<TStringType>& discoveryEndpoint, - const TMaybe<EDiscoveryMode>& discoveryMode, + const TMaybe<EDiscoveryMode>& discoveryMode, const TMaybe<bool>& enableSsl, - const TMaybe<std::shared_ptr<ICredentialsProviderFactory>>& credentialsProviderFactory -) { - return StateTracker_.GetDriverState( - database ? database.GetRef() : DefaultDatabase_, - discoveryEndpoint ? discoveryEndpoint.GetRef() : DefaultDiscoveryEndpoint_, - discoveryMode ? discoveryMode.GetRef() : DefaultDiscoveryMode_, + const TMaybe<std::shared_ptr<ICredentialsProviderFactory>>& credentialsProviderFactory +) { + return StateTracker_.GetDriverState( + database ? database.GetRef() : DefaultDatabase_, + discoveryEndpoint ? discoveryEndpoint.GetRef() : DefaultDiscoveryEndpoint_, + discoveryMode ? discoveryMode.GetRef() : DefaultDiscoveryMode_, enableSsl ? enableSsl.GetRef() : EnableSsl_, - credentialsProviderFactory ? credentialsProviderFactory.GetRef() : DefaultCredentialsProviderFactory_); -} - -IQueueClientContextPtr TGRpcConnectionsImpl::CreateContext() { - return GRpcClientLow_.CreateContext(); -} - -bool TGRpcConnectionsImpl::TryCreateContext(IQueueClientContextPtr& context) { - if (!context) { - // Keep CQ running until the request is complete - context = CreateContext(); - if (!context) { - return false; - } - } - return true; -} - -void TGRpcConnectionsImpl::WaitIdle() { - GRpcClientLow_.WaitIdle(); -} - -void TGRpcConnectionsImpl::Stop(bool wait) { + credentialsProviderFactory ? credentialsProviderFactory.GetRef() : DefaultCredentialsProviderFactory_); +} + +IQueueClientContextPtr TGRpcConnectionsImpl::CreateContext() { + return GRpcClientLow_.CreateContext(); +} + +bool TGRpcConnectionsImpl::TryCreateContext(IQueueClientContextPtr& context) { + if (!context) { + // Keep CQ running until the request is complete + context = CreateContext(); + if (!context) { + return false; + } + } + return true; +} + +void TGRpcConnectionsImpl::WaitIdle() { + GRpcClientLow_.WaitIdle(); +} + +void TGRpcConnectionsImpl::Stop(bool wait) { StateTracker_.SendNotification(TDbDriverState::ENotifyType::STOP).Wait(); - GRpcClientLow_.Stop(wait); -} - -void TGRpcConnectionsImpl::SetGrpcKeepAlive(NGrpc::TGRpcClientConfig& config, const TDuration& timeout, bool permitWithoutCalls) { - ui64 timeoutMs = timeout.MilliSeconds(); - config.IntChannelParams[GRPC_ARG_KEEPALIVE_TIME_MS] = timeoutMs >> 3; - config.IntChannelParams[GRPC_ARG_KEEPALIVE_TIMEOUT_MS] = timeoutMs; - config.IntChannelParams[GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA] = 0; - config.IntChannelParams[GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS] = permitWithoutCalls ? 1 : 0; -} - -TAsyncListEndpointsResult TGRpcConnectionsImpl::GetEndpoints(TDbDriverStatePtr dbState) { - Ydb::Discovery::ListEndpointsRequest request; - request.set_database(dbState->Database); - - auto promise = NThreading::NewPromise<TListEndpointsResult>(); - - auto extractor = [promise] - (google::protobuf::Any* any, TPlainStatus status) mutable { - Ydb::Discovery::ListEndpointsResult result; - if (any) { - any->UnpackTo(&result); - } - TListEndpointsResult val{result, status}; - promise.SetValue(std::move(val)); - }; - - RunDeferred<Ydb::Discovery::V1::DiscoveryService, Ydb::Discovery::ListEndpointsRequest, Ydb::Discovery::ListEndpointsResponse>( - std::move(request), - extractor, - &Ydb::Discovery::V1::DiscoveryService::Stub::AsyncListEndpoints, - dbState->shared_from_this(), - INITIAL_DEFERRED_CALL_DELAY, - TRpcRequestSettings(), - GET_ENDPOINTS_TIMEOUT); - - std::weak_ptr<TDbDriverState> weakState = dbState; - - return promise.GetFuture().Apply([this, weakState](NThreading::TFuture<TListEndpointsResult> future){ - auto strong = weakState.lock(); - auto result = future.ExtractValue(); - if (strong && result.DiscoveryStatus.IsTransportError()) { - strong->StatCollector.IncDiscoveryFailDueTransportError(); - } - return NThreading::MakeFuture<TListEndpointsResult>(MutateDiscovery(std::move(result), strong->Database)); - }); -} - + GRpcClientLow_.Stop(wait); +} + +void TGRpcConnectionsImpl::SetGrpcKeepAlive(NGrpc::TGRpcClientConfig& config, const TDuration& timeout, bool permitWithoutCalls) { + ui64 timeoutMs = timeout.MilliSeconds(); + config.IntChannelParams[GRPC_ARG_KEEPALIVE_TIME_MS] = timeoutMs >> 3; + config.IntChannelParams[GRPC_ARG_KEEPALIVE_TIMEOUT_MS] = timeoutMs; + config.IntChannelParams[GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA] = 0; + config.IntChannelParams[GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS] = permitWithoutCalls ? 1 : 0; +} + +TAsyncListEndpointsResult TGRpcConnectionsImpl::GetEndpoints(TDbDriverStatePtr dbState) { + Ydb::Discovery::ListEndpointsRequest request; + request.set_database(dbState->Database); + + auto promise = NThreading::NewPromise<TListEndpointsResult>(); + + auto extractor = [promise] + (google::protobuf::Any* any, TPlainStatus status) mutable { + Ydb::Discovery::ListEndpointsResult result; + if (any) { + any->UnpackTo(&result); + } + TListEndpointsResult val{result, status}; + promise.SetValue(std::move(val)); + }; + + RunDeferred<Ydb::Discovery::V1::DiscoveryService, Ydb::Discovery::ListEndpointsRequest, Ydb::Discovery::ListEndpointsResponse>( + std::move(request), + extractor, + &Ydb::Discovery::V1::DiscoveryService::Stub::AsyncListEndpoints, + dbState->shared_from_this(), + INITIAL_DEFERRED_CALL_DELAY, + TRpcRequestSettings(), + GET_ENDPOINTS_TIMEOUT); + + std::weak_ptr<TDbDriverState> weakState = dbState; + + return promise.GetFuture().Apply([this, weakState](NThreading::TFuture<TListEndpointsResult> future){ + auto strong = weakState.lock(); + auto result = future.ExtractValue(); + if (strong && result.DiscoveryStatus.IsTransportError()) { + strong->StatCollector.IncDiscoveryFailDueTransportError(); + } + return NThreading::MakeFuture<TListEndpointsResult>(MutateDiscovery(std::move(result), strong->Database)); + }); +} + TListEndpointsResult TGRpcConnectionsImpl::MutateDiscovery(TListEndpointsResult result, const TStringType& database) { std::lock_guard lock(ExtensionsLock_); - if (!DiscoveryMutatorCb) - return result; - - auto endpoint = result.DiscoveryStatus.Endpoint; + if (!DiscoveryMutatorCb) + return result; + + auto endpoint = result.DiscoveryStatus.Endpoint; auto ydbStatus = NYdb::TStatus(std::move(result.DiscoveryStatus)); - - ydbStatus = DiscoveryMutatorCb(&result.Result, std::move(ydbStatus), database); - - auto issues = ydbStatus.GetIssues(); - - auto plainStatus = TPlainStatus(ydbStatus.GetStatus(), std::move(issues), endpoint, {}); - result.DiscoveryStatus = plainStatus; - return result; -} - -bool TGRpcConnectionsImpl::GetDrainOnDtors() const { - return DrainOnDtors_; -} - -TBalancingSettings TGRpcConnectionsImpl::GetBalancingSettings() const { - return BalancingSettings_; -} - -bool TGRpcConnectionsImpl::StartStatCollecting(NMonitoring::IMetricRegistry* sensorsRegistry) { + + ydbStatus = DiscoveryMutatorCb(&result.Result, std::move(ydbStatus), database); + + auto issues = ydbStatus.GetIssues(); + + auto plainStatus = TPlainStatus(ydbStatus.GetStatus(), std::move(issues), endpoint, {}); + result.DiscoveryStatus = plainStatus; + return result; +} + +bool TGRpcConnectionsImpl::GetDrainOnDtors() const { + return DrainOnDtors_; +} + +TBalancingSettings TGRpcConnectionsImpl::GetBalancingSettings() const { + return BalancingSettings_; +} + +bool TGRpcConnectionsImpl::StartStatCollecting(NMonitoring::IMetricRegistry* sensorsRegistry) { { std::lock_guard lock(ExtensionsLock_); - if (MetricRegistryPtr_) { - return false; - } - if (auto ptr = dynamic_cast<NMonitoring::TMetricRegistry*>(sensorsRegistry)) { - MetricRegistryPtr_ = ptr; - } else { - Cerr << "Unknown IMetricRegistry impl" << Endl; - return false; - } - } - - StateTracker_.SetMetricRegistry(MetricRegistryPtr_); - return true; -} - -NMonitoring::TMetricRegistry* TGRpcConnectionsImpl::GetMetricRegistry() { + if (MetricRegistryPtr_) { + return false; + } + if (auto ptr = dynamic_cast<NMonitoring::TMetricRegistry*>(sensorsRegistry)) { + MetricRegistryPtr_ = ptr; + } else { + Cerr << "Unknown IMetricRegistry impl" << Endl; + return false; + } + } + + StateTracker_.SetMetricRegistry(MetricRegistryPtr_); + return true; +} + +NMonitoring::TMetricRegistry* TGRpcConnectionsImpl::GetMetricRegistry() { std::lock_guard lock(ExtensionsLock_); return MetricRegistryPtr_; -} - -void TGRpcConnectionsImpl::RegisterExtension(IExtension* extension) { - Extensions_.emplace_back(extension); -} - -void TGRpcConnectionsImpl::RegisterExtensionApi(IExtensionApi* api) { - ExtensionApis_.emplace_back(api); -} - -void TGRpcConnectionsImpl::SetDiscoveryMutator(IDiscoveryMutatorApi::TMutatorCb&& cb) { +} + +void TGRpcConnectionsImpl::RegisterExtension(IExtension* extension) { + Extensions_.emplace_back(extension); +} + +void TGRpcConnectionsImpl::RegisterExtensionApi(IExtensionApi* api) { + ExtensionApis_.emplace_back(api); +} + +void TGRpcConnectionsImpl::SetDiscoveryMutator(IDiscoveryMutatorApi::TMutatorCb&& cb) { std::lock_guard lock(ExtensionsLock_); - DiscoveryMutatorCb = std::move(cb); -} - -const TLog& TGRpcConnectionsImpl::GetLog() const { - return Log; -} - -void TGRpcConnectionsImpl::EnqueueResponse(IObjectInQueue* action) { - Y_ENSURE(ResponseQueue_->Add(action)); -} - -} // namespace NYdb + DiscoveryMutatorCb = std::move(cb); +} + +const TLog& TGRpcConnectionsImpl::GetLog() const { + return Log; +} + +void TGRpcConnectionsImpl::EnqueueResponse(IObjectInQueue* action) { + Y_ENSURE(ResponseQueue_->Add(action)); +} + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/grpc_connections.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/grpc_connections.h index 4aa2b8d0e4..bfc1b7ec38 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/grpc_connections.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/grpc_connections.h @@ -1,324 +1,324 @@ -#pragma once - +#pragma once + #include <ydb/public/sdk/cpp/client/impl/ydb_internal/internal_header.h> - -#include "actions.h" -#include "params.h" - + +#include "actions.h" +#include "params.h" + #include <ydb/public/api/grpc/ydb_discovery_v1.grpc.pb.h> #include <ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state/state.h> #include <ydb/public/sdk/cpp/client/impl/ydb_internal/rpc_request_settings/settings.h> #include <ydb/public/sdk/cpp/client/impl/ydb_internal/thread_pool/pool.h> #include <ydb/public/sdk/cpp/client/resources/ydb_resources.h> #include <ydb/public/sdk/cpp/client/ydb_extension/extension.h> - + #include <ydb/library/yql/public/issue/yql_issue_message.h> - -namespace NYdb { - -constexpr TDuration GRPC_KEEP_ALIVE_TIMEOUT_FOR_DISCOVERY = TDuration::Seconds(10); -constexpr TDuration INITIAL_DEFERRED_CALL_DELAY = TDuration::MilliSeconds(10); // The delay before first deferred service call -constexpr TDuration GET_ENDPOINTS_TIMEOUT = TDuration::Seconds(10); // Time wait for ListEndpoints request, after this time we pass error to client - -using NGrpc::TCallMeta; -using NGrpc::IQueueClientContextPtr; -using NGrpc::IQueueClientContextProvider; - -class ICredentialsProvider; - -// Deferred callbacks -using TDeferredResultCb = std::function<void(google::protobuf::Any*, TPlainStatus status)>; - + +namespace NYdb { + +constexpr TDuration GRPC_KEEP_ALIVE_TIMEOUT_FOR_DISCOVERY = TDuration::Seconds(10); +constexpr TDuration INITIAL_DEFERRED_CALL_DELAY = TDuration::MilliSeconds(10); // The delay before first deferred service call +constexpr TDuration GET_ENDPOINTS_TIMEOUT = TDuration::Seconds(10); // Time wait for ListEndpoints request, after this time we pass error to client + +using NGrpc::TCallMeta; +using NGrpc::IQueueClientContextPtr; +using NGrpc::IQueueClientContextProvider; + +class ICredentialsProvider; + +// Deferred callbacks +using TDeferredResultCb = std::function<void(google::protobuf::Any*, TPlainStatus status)>; + TStringType GetAuthInfo(TDbDriverStatePtr p); void SetDatabaseHeader(TCallMeta& meta, const TStringType& database); TStringType CreateSDKBuildInfo(); - -class TGRpcConnectionsImpl - : public IQueueClientContextProvider - , public IInternalClient -{ - friend class TDeferredAction; -public: - TGRpcConnectionsImpl(std::shared_ptr<IConnectionsParams> params); - ~TGRpcConnectionsImpl(); - - void AddPeriodicTask(TPeriodicCb&& cb, TDuration period) override; - void ScheduleOneTimeTask(TSimpleCb&& fn, TDuration timeout); - NThreading::TFuture<bool> ScheduleFuture( - TDuration timeout, - IQueueClientContextPtr token = nullptr - ); - void ScheduleCallback( - TDuration timeout, - std::function<void(bool)> callback, - IQueueClientContextPtr token = nullptr - ); - // The idea is: sometimes we need to work with multiple databases simultaneously - // This method returns DbDriverState (or just db state) for given database credentials pair - // this state is used to keep data related to particular database. - TDbDriverStatePtr GetDriverState( + +class TGRpcConnectionsImpl + : public IQueueClientContextProvider + , public IInternalClient +{ + friend class TDeferredAction; +public: + TGRpcConnectionsImpl(std::shared_ptr<IConnectionsParams> params); + ~TGRpcConnectionsImpl(); + + void AddPeriodicTask(TPeriodicCb&& cb, TDuration period) override; + void ScheduleOneTimeTask(TSimpleCb&& fn, TDuration timeout); + NThreading::TFuture<bool> ScheduleFuture( + TDuration timeout, + IQueueClientContextPtr token = nullptr + ); + void ScheduleCallback( + TDuration timeout, + std::function<void(bool)> callback, + IQueueClientContextPtr token = nullptr + ); + // The idea is: sometimes we need to work with multiple databases simultaneously + // This method returns DbDriverState (or just db state) for given database credentials pair + // this state is used to keep data related to particular database. + TDbDriverStatePtr GetDriverState( const TMaybe<TStringType>& database, const TMaybe<TStringType>& discoveryEndpoint, - const TMaybe<EDiscoveryMode>& discoveryMode, + const TMaybe<EDiscoveryMode>& discoveryMode, const TMaybe<bool>& enableSsl, - const TMaybe<std::shared_ptr<ICredentialsProviderFactory>>& credentialsProviderFactory - ); + const TMaybe<std::shared_ptr<ICredentialsProviderFactory>>& credentialsProviderFactory + ); IQueueClientContextPtr CreateContext() override; - bool TryCreateContext(IQueueClientContextPtr& context); - void WaitIdle(); - void Stop(bool wait = false); - - template<typename TService> - using TServiceConnection = NGrpc::TServiceConnection<TService>; - - static void SetGrpcKeepAlive(NGrpc::TGRpcClientConfig& config, const TDuration& timeout, bool permitWithoutCalls); - - template<typename TService> + bool TryCreateContext(IQueueClientContextPtr& context); + void WaitIdle(); + void Stop(bool wait = false); + + template<typename TService> + using TServiceConnection = NGrpc::TServiceConnection<TService>; + + static void SetGrpcKeepAlive(NGrpc::TGRpcClientConfig& config, const TDuration& timeout, bool permitWithoutCalls); + + template<typename TService> std::pair<std::unique_ptr<TServiceConnection<TService>>, TStringType> GetServiceConnection( TDbDriverStatePtr dbState, const TStringType& preferredEndpoint, TRpcRequestSettings::TEndpointPolicy endpointPolicy) - { - auto clientConfig = NGrpc::TGRpcClientConfig(dbState->DiscoveryEndpoint); + { + auto clientConfig = NGrpc::TGRpcClientConfig(dbState->DiscoveryEndpoint); clientConfig.EnableSsl = dbState->EnableSsl; - clientConfig.SslCaCert = CaCert_; + clientConfig.SslCaCert = CaCert_; clientConfig.MemQuota = MemoryQuota_; - + if (std::is_same<TService,Ydb::Discovery::V1::DiscoveryService>() || dbState->Database.empty() || endpointPolicy == TRpcRequestSettings::TEndpointPolicy::UseDiscoveryEndpoint) { SetGrpcKeepAlive(clientConfig, GRPC_KEEP_ALIVE_TIMEOUT_FOR_DISCOVERY, GRpcKeepAlivePermitWithoutCalls_); } else { - auto endpoint = dbState->EndpointPool.GetEndpoint(preferredEndpoint); + auto endpoint = dbState->EndpointPool.GetEndpoint(preferredEndpoint); if (!endpoint) { - return {nullptr, ""}; - } + return {nullptr, ""}; + } clientConfig.Locator = endpoint.Endpoint; clientConfig.SslTargetNameOverride = endpoint.SslTargetNameOverride; - if (GRpcKeepAliveTimeout_) { - SetGrpcKeepAlive(clientConfig, GRpcKeepAliveTimeout_, GRpcKeepAlivePermitWithoutCalls_); - } - } - - std::unique_ptr<TServiceConnection<TService>> conn; -#ifndef YDB_GRPC_BYPASS_CHANNEL_POOL - ChannelPool_.GetStubsHolderLocked( - clientConfig.Locator, clientConfig, [&conn, this](NGrpc::TStubsHolder& holder) mutable { - conn.reset(GRpcClientLow_.CreateGRpcServiceConnection<TService>(holder).release()); - }); -#else - conn = std::move(GRpcClientLow_.CreateGRpcServiceConnection<TService>(clientConfig)); -#endif - return {std::move(conn), clientConfig.Locator}; - } - - template<class TService, class TRequest, class TResponse> - using TSimpleRpc = - typename NGrpc::TSimpleRequestProcessor< - typename TService::Stub, - TRequest, - TResponse>::TAsyncRequest; - - template<typename TService, typename TRequest, typename TResponse> - void Run( - TRequest&& request, - TResponseCb<TResponse>&& userResponseCb, - TSimpleRpc<TService, TRequest, TResponse> rpc, - TDbDriverStatePtr dbState, - const TRpcRequestSettings& requestSettings, - TDuration clientTimeout, + if (GRpcKeepAliveTimeout_) { + SetGrpcKeepAlive(clientConfig, GRpcKeepAliveTimeout_, GRpcKeepAlivePermitWithoutCalls_); + } + } + + std::unique_ptr<TServiceConnection<TService>> conn; +#ifndef YDB_GRPC_BYPASS_CHANNEL_POOL + ChannelPool_.GetStubsHolderLocked( + clientConfig.Locator, clientConfig, [&conn, this](NGrpc::TStubsHolder& holder) mutable { + conn.reset(GRpcClientLow_.CreateGRpcServiceConnection<TService>(holder).release()); + }); +#else + conn = std::move(GRpcClientLow_.CreateGRpcServiceConnection<TService>(clientConfig)); +#endif + return {std::move(conn), clientConfig.Locator}; + } + + template<class TService, class TRequest, class TResponse> + using TSimpleRpc = + typename NGrpc::TSimpleRequestProcessor< + typename TService::Stub, + TRequest, + TResponse>::TAsyncRequest; + + template<typename TService, typename TRequest, typename TResponse> + void Run( + TRequest&& request, + TResponseCb<TResponse>&& userResponseCb, + TSimpleRpc<TService, TRequest, TResponse> rpc, + TDbDriverStatePtr dbState, + const TRpcRequestSettings& requestSettings, + TDuration clientTimeout, const TStringType& preferredEndpoint, - std::shared_ptr<IQueueClientContext> context = nullptr) - { - using NGrpc::TGrpcStatus; - using TConnection = std::unique_ptr<TServiceConnection<TService>>; - Y_VERIFY(dbState); - - if (!TryCreateContext(context)) { - TPlainStatus status(EStatus::CLIENT_CANCELLED, "Client is stopped"); - userResponseCb(nullptr, TPlainStatus{status.Status, std::move(status.Issues)}); - return; - } - - if (dbState->StatCollector.IsCollecting()) { - std::weak_ptr<TDbDriverState> weakState = dbState; - const auto startTime = TInstant::Now(); - userResponseCb = std::move([cb = std::move(userResponseCb), weakState, startTime](TResponse* response, TPlainStatus status) { - const auto resultSize = response ? response->ByteSizeLong() : 0; - cb(response, status); - - if (auto state = weakState.lock()) { - state->StatCollector.IncRequestLatency(TInstant::Now() - startTime); - state->StatCollector.IncResultSize(resultSize); - } - }); - } - + std::shared_ptr<IQueueClientContext> context = nullptr) + { + using NGrpc::TGrpcStatus; + using TConnection = std::unique_ptr<TServiceConnection<TService>>; + Y_VERIFY(dbState); + + if (!TryCreateContext(context)) { + TPlainStatus status(EStatus::CLIENT_CANCELLED, "Client is stopped"); + userResponseCb(nullptr, TPlainStatus{status.Status, std::move(status.Issues)}); + return; + } + + if (dbState->StatCollector.IsCollecting()) { + std::weak_ptr<TDbDriverState> weakState = dbState; + const auto startTime = TInstant::Now(); + userResponseCb = std::move([cb = std::move(userResponseCb), weakState, startTime](TResponse* response, TPlainStatus status) { + const auto resultSize = response ? response->ByteSizeLong() : 0; + cb(response, status); + + if (auto state = weakState.lock()) { + state->StatCollector.IncRequestLatency(TInstant::Now() - startTime); + state->StatCollector.IncResultSize(resultSize); + } + }); + } + auto endpointPolicy = requestSettings.EndpointPolicy; - WithServiceConnection<TService>( - [this, request = std::move(request), userResponseCb = std::move(userResponseCb), rpc, requestSettings, context = std::move(context), clientTimeout, dbState] + WithServiceConnection<TService>( + [this, request = std::move(request), userResponseCb = std::move(userResponseCb), rpc, requestSettings, context = std::move(context), clientTimeout, dbState] (TPlainStatus status, TConnection serviceConnection, TStringType endpoint) mutable -> void { - if (!status.Ok()) { - userResponseCb( - nullptr, - std::move(status)); - return; - } - - TCallMeta meta; - meta.Timeout = clientTimeout; - #ifndef YDB_GRPC_UNSECURE_AUTH - meta.CallCredentials = dbState->CallCredentials; - #else + if (!status.Ok()) { + userResponseCb( + nullptr, + std::move(status)); + return; + } + + TCallMeta meta; + meta.Timeout = clientTimeout; + #ifndef YDB_GRPC_UNSECURE_AUTH + meta.CallCredentials = dbState->CallCredentials; + #else if (requestSettings.UseAuth && dbState->CredentialsProvider && dbState->CredentialsProvider->IsValid()) { - try { - meta.Aux.push_back({ YDB_AUTH_TICKET_HEADER, GetAuthInfo(dbState) }); - } catch (const yexception& e) { - userResponseCb( - nullptr, - TPlainStatus( - EStatus::CLIENT_UNAUTHENTICATED, - TStringBuilder() << "Can't get Authentication info from CredentialsProvider. " << e.what() - ) - ); - return; - } - } - #endif - if (!requestSettings.TraceId.empty()) { - meta.Aux.push_back({YDB_TRACE_ID_HEADER, requestSettings.TraceId}); - } - - if (!requestSettings.RequestType.empty()) { - meta.Aux.push_back({YDB_REQUEST_TYPE_HEADER, requestSettings.RequestType}); - } - + try { + meta.Aux.push_back({ YDB_AUTH_TICKET_HEADER, GetAuthInfo(dbState) }); + } catch (const yexception& e) { + userResponseCb( + nullptr, + TPlainStatus( + EStatus::CLIENT_UNAUTHENTICATED, + TStringBuilder() << "Can't get Authentication info from CredentialsProvider. " << e.what() + ) + ); + return; + } + } + #endif + if (!requestSettings.TraceId.empty()) { + meta.Aux.push_back({YDB_TRACE_ID_HEADER, requestSettings.TraceId}); + } + + if (!requestSettings.RequestType.empty()) { + meta.Aux.push_back({YDB_REQUEST_TYPE_HEADER, requestSettings.RequestType}); + } + if (!dbState->Database.empty()) { - SetDatabaseHeader(meta, dbState->Database); - } - - if (std::is_same<TService,Ydb::Discovery::V1::DiscoveryService>()) { - meta.Aux.push_back({YDB_SDK_BUILD_INFO_HEADER, CreateSDKBuildInfo()}); - } - + SetDatabaseHeader(meta, dbState->Database); + } + + if (std::is_same<TService,Ydb::Discovery::V1::DiscoveryService>()) { + meta.Aux.push_back({YDB_SDK_BUILD_INFO_HEADER, CreateSDKBuildInfo()}); + } + meta.Aux.insert(meta.Aux.end(), requestSettings.Header.begin(), requestSettings.Header.end()); - dbState->StatCollector.IncGRpcInFlight(); - dbState->StatCollector.IncGRpcInFlightByHost(endpoint); - - NGrpc::TAdvancedResponseCallback<TResponse> responseCbLow = - [this, context, userResponseCb = std::move(userResponseCb), endpoint, dbState] - (const grpc::ClientContext& ctx, TGrpcStatus&& grpcStatus, TResponse&& response) mutable -> void { - dbState->StatCollector.DecGRpcInFlight(); - dbState->StatCollector.DecGRpcInFlightByHost(endpoint); - - if (NGrpc::IsGRpcStatusGood(grpcStatus)) { + dbState->StatCollector.IncGRpcInFlight(); + dbState->StatCollector.IncGRpcInFlightByHost(endpoint); + + NGrpc::TAdvancedResponseCallback<TResponse> responseCbLow = + [this, context, userResponseCb = std::move(userResponseCb), endpoint, dbState] + (const grpc::ClientContext& ctx, TGrpcStatus&& grpcStatus, TResponse&& response) mutable -> void { + dbState->StatCollector.DecGRpcInFlight(); + dbState->StatCollector.DecGRpcInFlightByHost(endpoint); + + if (NGrpc::IsGRpcStatusGood(grpcStatus)) { std::multimap<TStringType, TStringType> metadata; - - for (const auto& [name, value] : ctx.GetServerInitialMetadata()) { - metadata.emplace( + + for (const auto& [name, value] : ctx.GetServerInitialMetadata()) { + metadata.emplace( TStringType(name.begin(), name.end()), TStringType(value.begin(), value.end())); - } - for (const auto& [name, value] : ctx.GetServerTrailingMetadata()) { - metadata.emplace( + } + for (const auto& [name, value] : ctx.GetServerTrailingMetadata()) { + metadata.emplace( TStringType(name.begin(), name.end()), TStringType(value.begin(), value.end())); - } - - auto resp = new TResult<TResponse>( - std::move(response), - std::move(grpcStatus), - std::move(userResponseCb), - this, - std::move(context), - endpoint, - std::move(metadata)); - - EnqueueResponse(resp); - } else { - dbState->StatCollector.IncReqFailDueTransportError(); - dbState->StatCollector.IncTransportErrorsByHost(endpoint); - - auto resp = new TGRpcErrorResponse<TResponse>( - std::move(grpcStatus), - std::move(userResponseCb), - this, - std::move(context), - endpoint); - - dbState->EndpointPool.BanEndpoint(endpoint); - - EnqueueResponse(resp); - } - }; - - serviceConnection->DoAdvancedRequest(std::move(request), std::move(responseCbLow), rpc, meta, - context.get()); + } + + auto resp = new TResult<TResponse>( + std::move(response), + std::move(grpcStatus), + std::move(userResponseCb), + this, + std::move(context), + endpoint, + std::move(metadata)); + + EnqueueResponse(resp); + } else { + dbState->StatCollector.IncReqFailDueTransportError(); + dbState->StatCollector.IncTransportErrorsByHost(endpoint); + + auto resp = new TGRpcErrorResponse<TResponse>( + std::move(grpcStatus), + std::move(userResponseCb), + this, + std::move(context), + endpoint); + + dbState->EndpointPool.BanEndpoint(endpoint); + + EnqueueResponse(resp); + } + }; + + serviceConnection->DoAdvancedRequest(std::move(request), std::move(responseCbLow), rpc, meta, + context.get()); }, dbState, preferredEndpoint, endpointPolicy); - } - - template<typename TService, typename TRequest, typename TResponse> - void RunDeferred( - TRequest&& request, - TDeferredOperationCb&& userResponseCb, - TSimpleRpc<TService, TRequest, TResponse> rpc, - TDbDriverStatePtr dbState, - TDuration deferredTimeout, - const TRpcRequestSettings& requestSettings, - TDuration clientTimeout, - bool poll = false, + } + + template<typename TService, typename TRequest, typename TResponse> + void RunDeferred( + TRequest&& request, + TDeferredOperationCb&& userResponseCb, + TSimpleRpc<TService, TRequest, TResponse> rpc, + TDbDriverStatePtr dbState, + TDuration deferredTimeout, + const TRpcRequestSettings& requestSettings, + TDuration clientTimeout, + bool poll = false, const TStringType& preferredEndpoint = TStringType(), - std::shared_ptr<IQueueClientContext> context = nullptr) - { - if (!TryCreateContext(context)) { - TPlainStatus status(EStatus::CLIENT_CANCELLED, "Client is stopped"); - userResponseCb(nullptr, status); - return; - } - - auto responseCb = [this, userResponseCb = std::move(userResponseCb), dbState, deferredTimeout, poll, context] - (TResponse* response, TPlainStatus status) mutable - { - if (response) { - Ydb::Operations::Operation* operation = response->mutable_operation(); - if (!operation->ready() && poll) { - auto action = MakeIntrusive<TDeferredAction>( - operation->id(), - std::move(userResponseCb), - this, - std::move(context), - deferredTimeout, - dbState, - status.Endpoint); - - action->Start(); - } else { - NYql::TIssues opIssues; - NYql::IssuesFromMessage(operation->issues(), opIssues); - userResponseCb(operation, TPlainStatus{static_cast<EStatus>(operation->status()), std::move(opIssues), - status.Endpoint, std::move(status.Metadata)}); - } - } else { - userResponseCb(nullptr, status); - } - }; - - Run<TService, TRequest, TResponse>( - std::move(request), - responseCb, - rpc, - dbState, - requestSettings, - clientTimeout, - preferredEndpoint, - std::move(context)); - } - + std::shared_ptr<IQueueClientContext> context = nullptr) + { + if (!TryCreateContext(context)) { + TPlainStatus status(EStatus::CLIENT_CANCELLED, "Client is stopped"); + userResponseCb(nullptr, status); + return; + } + + auto responseCb = [this, userResponseCb = std::move(userResponseCb), dbState, deferredTimeout, poll, context] + (TResponse* response, TPlainStatus status) mutable + { + if (response) { + Ydb::Operations::Operation* operation = response->mutable_operation(); + if (!operation->ready() && poll) { + auto action = MakeIntrusive<TDeferredAction>( + operation->id(), + std::move(userResponseCb), + this, + std::move(context), + deferredTimeout, + dbState, + status.Endpoint); + + action->Start(); + } else { + NYql::TIssues opIssues; + NYql::IssuesFromMessage(operation->issues(), opIssues); + userResponseCb(operation, TPlainStatus{static_cast<EStatus>(operation->status()), std::move(opIssues), + status.Endpoint, std::move(status.Metadata)}); + } + } else { + userResponseCb(nullptr, status); + } + }; + + Run<TService, TRequest, TResponse>( + std::move(request), + responseCb, + rpc, + dbState, + requestSettings, + clientTimeout, + preferredEndpoint, + std::move(context)); + } + // Run request using discovery endpoint. // Mostly usefull to make calls from credential provider - template<typename TService, typename TRequest, typename TResponse> + template<typename TService, typename TRequest, typename TResponse> static void RunOnDiscoveryEndpoint( std::shared_ptr<ICoreFacility> facility, TRequest&& request, @@ -346,281 +346,281 @@ public: } template<typename TService, typename TRequest, typename TResponse> - void RunDeferred( - TRequest&& request, - TDeferredResultCb&& userResponseCb, - TSimpleRpc<TService, TRequest, TResponse> rpc, - TDbDriverStatePtr dbState, - TDuration deferredTimeout, - const TRpcRequestSettings& requestSettings, - TDuration clientTimeout, + void RunDeferred( + TRequest&& request, + TDeferredResultCb&& userResponseCb, + TSimpleRpc<TService, TRequest, TResponse> rpc, + TDbDriverStatePtr dbState, + TDuration deferredTimeout, + const TRpcRequestSettings& requestSettings, + TDuration clientTimeout, const TStringType& preferredEndpoint = TStringType(), - std::shared_ptr<IQueueClientContext> context = nullptr) - { - auto operationCb = [userResponseCb = std::move(userResponseCb)](Ydb::Operations::Operation* operation, TPlainStatus status) mutable { + std::shared_ptr<IQueueClientContext> context = nullptr) + { + auto operationCb = [userResponseCb = std::move(userResponseCb)](Ydb::Operations::Operation* operation, TPlainStatus status) mutable { if (operation) { status.SetCostInfo(std::move(*operation->mutable_cost_info())); userResponseCb(operation->mutable_result(), std::move(status)); } else { userResponseCb(nullptr, std::move(status)); } - }; - - RunDeferred<TService, TRequest, TResponse>( - std::move(request), - operationCb, - rpc, - dbState, - deferredTimeout, - requestSettings, - clientTimeout, - true, // poll - preferredEndpoint, - context); - } - - template<class TService, class TRequest, class TResponse, template<typename TA, typename TB, typename TC> class TStream> - using TStreamRpc = - typename TStream< - typename TService::Stub, - TRequest, - TResponse>::TAsyncRequest; - - template<class TService, class TRequest, class TResponse, class TCallback> - void StartReadStream( - const TRequest& request, - TCallback responseCb, - TStreamRpc<TService, TRequest, TResponse, NGrpc::TStreamRequestReadProcessor> rpc, - TDbDriverStatePtr dbState, - const TRpcRequestSettings& requestSettings, - std::shared_ptr<IQueueClientContext> context = nullptr) - { - using NGrpc::TGrpcStatus; - using TConnection = std::unique_ptr<TServiceConnection<TService>>; - using TProcessor = typename NGrpc::IStreamRequestReadProcessor<TResponse>::TPtr; - - if (!TryCreateContext(context)) { - responseCb(TPlainStatus(EStatus::CLIENT_CANCELLED, "Client is stopped"), nullptr); - return; - } - + }; + + RunDeferred<TService, TRequest, TResponse>( + std::move(request), + operationCb, + rpc, + dbState, + deferredTimeout, + requestSettings, + clientTimeout, + true, // poll + preferredEndpoint, + context); + } + + template<class TService, class TRequest, class TResponse, template<typename TA, typename TB, typename TC> class TStream> + using TStreamRpc = + typename TStream< + typename TService::Stub, + TRequest, + TResponse>::TAsyncRequest; + + template<class TService, class TRequest, class TResponse, class TCallback> + void StartReadStream( + const TRequest& request, + TCallback responseCb, + TStreamRpc<TService, TRequest, TResponse, NGrpc::TStreamRequestReadProcessor> rpc, + TDbDriverStatePtr dbState, + const TRpcRequestSettings& requestSettings, + std::shared_ptr<IQueueClientContext> context = nullptr) + { + using NGrpc::TGrpcStatus; + using TConnection = std::unique_ptr<TServiceConnection<TService>>; + using TProcessor = typename NGrpc::IStreamRequestReadProcessor<TResponse>::TPtr; + + if (!TryCreateContext(context)) { + responseCb(TPlainStatus(EStatus::CLIENT_CANCELLED, "Client is stopped"), nullptr); + return; + } + auto endpointPolicy = requestSettings.EndpointPolicy; - WithServiceConnection<TService>( - [request, responseCb = std::move(responseCb), rpc, requestSettings, context = std::move(context), dbState] + WithServiceConnection<TService>( + [request, responseCb = std::move(responseCb), rpc, requestSettings, context = std::move(context), dbState] (TPlainStatus status, TConnection serviceConnection, TStringType endpoint) mutable { - if (!status.Ok()) { - responseCb(std::move(status), nullptr); - return; - } - - TCallMeta meta; -#ifndef YDB_GRPC_UNSECURE_AUTH - meta.CallCredentials = dbState->CallCredentials; -#else + if (!status.Ok()) { + responseCb(std::move(status), nullptr); + return; + } + + TCallMeta meta; +#ifndef YDB_GRPC_UNSECURE_AUTH + meta.CallCredentials = dbState->CallCredentials; +#else if (requestSettings.UseAuth && dbState->CredentialsProvider && dbState->CredentialsProvider->IsValid()) { - try { - meta.Aux.push_back({ YDB_AUTH_TICKET_HEADER, GetAuthInfo(dbState) }); - } catch (const yexception& e) { - responseCb( - TPlainStatus( - EStatus::CLIENT_UNAUTHENTICATED, - TStringBuilder() << "Can't get Authentication info from CredentialsProvider. " << e.what() - ), - nullptr - ); - return; - } - } -#endif - if (!requestSettings.TraceId.empty()) { - meta.Aux.push_back({YDB_TRACE_ID_HEADER, requestSettings.TraceId}); - } - - if (!requestSettings.RequestType.empty()) { - meta.Aux.push_back({YDB_REQUEST_TYPE_HEADER, requestSettings.RequestType}); - } - + try { + meta.Aux.push_back({ YDB_AUTH_TICKET_HEADER, GetAuthInfo(dbState) }); + } catch (const yexception& e) { + responseCb( + TPlainStatus( + EStatus::CLIENT_UNAUTHENTICATED, + TStringBuilder() << "Can't get Authentication info from CredentialsProvider. " << e.what() + ), + nullptr + ); + return; + } + } +#endif + if (!requestSettings.TraceId.empty()) { + meta.Aux.push_back({YDB_TRACE_ID_HEADER, requestSettings.TraceId}); + } + + if (!requestSettings.RequestType.empty()) { + meta.Aux.push_back({YDB_REQUEST_TYPE_HEADER, requestSettings.RequestType}); + } + if (!dbState->Database.empty()) { - SetDatabaseHeader(meta, dbState->Database); - } - - dbState->StatCollector.IncGRpcInFlight(); - dbState->StatCollector.IncGRpcInFlightByHost(endpoint); - - auto lowCallback = [responseCb = std::move(responseCb), dbState, endpoint] - (TGrpcStatus grpcStatus, TProcessor processor) mutable { - dbState->StatCollector.DecGRpcInFlight(); - dbState->StatCollector.DecGRpcInFlightByHost(endpoint); - - if (grpcStatus.Ok()) { - Y_VERIFY(processor); - auto finishedCallback = [dbState, endpoint] (TGrpcStatus grpcStatus) { - if (!grpcStatus.Ok() && grpcStatus.GRpcStatusCode != grpc::StatusCode::CANCELLED) { - dbState->EndpointPool.BanEndpoint(endpoint); - } - }; - processor->AddFinishedCallback(std::move(finishedCallback)); - // TODO: Add headers for streaming calls. - TPlainStatus status(std::move(grpcStatus), endpoint, {}); - responseCb(std::move(status), std::move(processor)); - } else { - dbState->StatCollector.IncReqFailDueTransportError(); - dbState->StatCollector.IncTransportErrorsByHost(endpoint); - if (grpcStatus.GRpcStatusCode != grpc::StatusCode::CANCELLED) { - dbState->EndpointPool.BanEndpoint(endpoint); - } - // TODO: Add headers for streaming calls. - TPlainStatus status(std::move(grpcStatus), endpoint, {}); - responseCb(std::move(status), nullptr); - } - }; - - serviceConnection->template DoStreamRequest<TRequest, TResponse>( - request, - std::move(lowCallback), - std::move(rpc), - std::move(meta), - context.get()); + SetDatabaseHeader(meta, dbState->Database); + } + + dbState->StatCollector.IncGRpcInFlight(); + dbState->StatCollector.IncGRpcInFlightByHost(endpoint); + + auto lowCallback = [responseCb = std::move(responseCb), dbState, endpoint] + (TGrpcStatus grpcStatus, TProcessor processor) mutable { + dbState->StatCollector.DecGRpcInFlight(); + dbState->StatCollector.DecGRpcInFlightByHost(endpoint); + + if (grpcStatus.Ok()) { + Y_VERIFY(processor); + auto finishedCallback = [dbState, endpoint] (TGrpcStatus grpcStatus) { + if (!grpcStatus.Ok() && grpcStatus.GRpcStatusCode != grpc::StatusCode::CANCELLED) { + dbState->EndpointPool.BanEndpoint(endpoint); + } + }; + processor->AddFinishedCallback(std::move(finishedCallback)); + // TODO: Add headers for streaming calls. + TPlainStatus status(std::move(grpcStatus), endpoint, {}); + responseCb(std::move(status), std::move(processor)); + } else { + dbState->StatCollector.IncReqFailDueTransportError(); + dbState->StatCollector.IncTransportErrorsByHost(endpoint); + if (grpcStatus.GRpcStatusCode != grpc::StatusCode::CANCELLED) { + dbState->EndpointPool.BanEndpoint(endpoint); + } + // TODO: Add headers for streaming calls. + TPlainStatus status(std::move(grpcStatus), endpoint, {}); + responseCb(std::move(status), nullptr); + } + }; + + serviceConnection->template DoStreamRequest<TRequest, TResponse>( + request, + std::move(lowCallback), + std::move(rpc), + std::move(meta), + context.get()); }, dbState, TStringType(), endpointPolicy); - } - - template<class TService, class TRequest, class TResponse, class TCallback> - void StartBidirectionalStream( - TCallback connectedCallback, - TStreamRpc<TService, TRequest, TResponse, NGrpc::TStreamRequestReadWriteProcessor> rpc, - TDbDriverStatePtr dbState, - const TRpcRequestSettings& requestSettings, - std::shared_ptr<IQueueClientContext> context = nullptr) - { - using NGrpc::TGrpcStatus; - using TConnection = std::unique_ptr<TServiceConnection<TService>>; - using TProcessor = typename NGrpc::IStreamRequestReadWriteProcessor<TRequest, TResponse>::TPtr; - - if (!TryCreateContext(context)) { - connectedCallback(TPlainStatus(EStatus::CLIENT_CANCELLED, "Client is stopped"), nullptr); - return; - } - + } + + template<class TService, class TRequest, class TResponse, class TCallback> + void StartBidirectionalStream( + TCallback connectedCallback, + TStreamRpc<TService, TRequest, TResponse, NGrpc::TStreamRequestReadWriteProcessor> rpc, + TDbDriverStatePtr dbState, + const TRpcRequestSettings& requestSettings, + std::shared_ptr<IQueueClientContext> context = nullptr) + { + using NGrpc::TGrpcStatus; + using TConnection = std::unique_ptr<TServiceConnection<TService>>; + using TProcessor = typename NGrpc::IStreamRequestReadWriteProcessor<TRequest, TResponse>::TPtr; + + if (!TryCreateContext(context)) { + connectedCallback(TPlainStatus(EStatus::CLIENT_CANCELLED, "Client is stopped"), nullptr); + return; + } + auto endpointPolicy = requestSettings.EndpointPolicy; - WithServiceConnection<TService>( - [connectedCallback = std::move(connectedCallback), rpc, requestSettings, context = std::move(context), dbState] + WithServiceConnection<TService>( + [connectedCallback = std::move(connectedCallback), rpc, requestSettings, context = std::move(context), dbState] (TPlainStatus status, TConnection serviceConnection, TStringType endpoint) mutable { - if (!status.Ok()) { - connectedCallback(std::move(status), nullptr); - return; - } - - TCallMeta meta; - #ifndef YDB_GRPC_UNSECURE_AUTH - meta.CallCredentials = dbState->CallCredentials; - #else + if (!status.Ok()) { + connectedCallback(std::move(status), nullptr); + return; + } + + TCallMeta meta; + #ifndef YDB_GRPC_UNSECURE_AUTH + meta.CallCredentials = dbState->CallCredentials; + #else if (requestSettings.UseAuth && dbState->CredentialsProvider && dbState->CredentialsProvider->IsValid()) { - try { - meta.Aux.push_back({ YDB_AUTH_TICKET_HEADER, GetAuthInfo(dbState) }); - } catch (const yexception& e) { - connectedCallback( - TPlainStatus( - EStatus::CLIENT_UNAUTHENTICATED, - TStringBuilder() << "Can't get Authentication info from CredentialsProvider. " << e.what() - ), - nullptr - ); - return; - } - } - #endif - if (!requestSettings.TraceId.empty()) { - meta.Aux.push_back({YDB_TRACE_ID_HEADER, requestSettings.TraceId}); - } - - if (!requestSettings.RequestType.empty()) { - meta.Aux.push_back({YDB_REQUEST_TYPE_HEADER, requestSettings.RequestType}); - } - + try { + meta.Aux.push_back({ YDB_AUTH_TICKET_HEADER, GetAuthInfo(dbState) }); + } catch (const yexception& e) { + connectedCallback( + TPlainStatus( + EStatus::CLIENT_UNAUTHENTICATED, + TStringBuilder() << "Can't get Authentication info from CredentialsProvider. " << e.what() + ), + nullptr + ); + return; + } + } + #endif + if (!requestSettings.TraceId.empty()) { + meta.Aux.push_back({YDB_TRACE_ID_HEADER, requestSettings.TraceId}); + } + + if (!requestSettings.RequestType.empty()) { + meta.Aux.push_back({YDB_REQUEST_TYPE_HEADER, requestSettings.RequestType}); + } + if (!dbState->Database.empty()) { - SetDatabaseHeader(meta, dbState->Database); - } - - dbState->StatCollector.IncGRpcInFlight(); - dbState->StatCollector.IncGRpcInFlightByHost(endpoint); - - auto lowCallback = [connectedCallback = std::move(connectedCallback), dbState, endpoint] - (TGrpcStatus grpcStatus, TProcessor processor) { - dbState->StatCollector.DecGRpcInFlight(); - dbState->StatCollector.DecGRpcInFlightByHost(endpoint); - - if (grpcStatus.Ok()) { - Y_VERIFY(processor); - auto finishedCallback = [dbState, endpoint] (TGrpcStatus grpcStatus) { - if (!grpcStatus.Ok() && grpcStatus.GRpcStatusCode != grpc::StatusCode::CANCELLED) { - dbState->EndpointPool.BanEndpoint(endpoint); - } - }; - processor->AddFinishedCallback(std::move(finishedCallback)); - // TODO: Add headers for streaming calls. - TPlainStatus status(std::move(grpcStatus), endpoint, {}); - connectedCallback(std::move(status), std::move(processor)); - } else { - dbState->StatCollector.IncReqFailDueTransportError(); - dbState->StatCollector.IncTransportErrorsByHost(endpoint); - if (grpcStatus.GRpcStatusCode != grpc::StatusCode::CANCELLED) { - dbState->EndpointPool.BanEndpoint(endpoint); - } - // TODO: Add headers for streaming calls. - TPlainStatus status(std::move(grpcStatus), endpoint, {}); - connectedCallback(std::move(status), nullptr); - } - }; - - serviceConnection->template DoStreamRequest<TRequest, TResponse>( - std::move(lowCallback), - std::move(rpc), - std::move(meta), - context.get()); + SetDatabaseHeader(meta, dbState->Database); + } + + dbState->StatCollector.IncGRpcInFlight(); + dbState->StatCollector.IncGRpcInFlightByHost(endpoint); + + auto lowCallback = [connectedCallback = std::move(connectedCallback), dbState, endpoint] + (TGrpcStatus grpcStatus, TProcessor processor) { + dbState->StatCollector.DecGRpcInFlight(); + dbState->StatCollector.DecGRpcInFlightByHost(endpoint); + + if (grpcStatus.Ok()) { + Y_VERIFY(processor); + auto finishedCallback = [dbState, endpoint] (TGrpcStatus grpcStatus) { + if (!grpcStatus.Ok() && grpcStatus.GRpcStatusCode != grpc::StatusCode::CANCELLED) { + dbState->EndpointPool.BanEndpoint(endpoint); + } + }; + processor->AddFinishedCallback(std::move(finishedCallback)); + // TODO: Add headers for streaming calls. + TPlainStatus status(std::move(grpcStatus), endpoint, {}); + connectedCallback(std::move(status), std::move(processor)); + } else { + dbState->StatCollector.IncReqFailDueTransportError(); + dbState->StatCollector.IncTransportErrorsByHost(endpoint); + if (grpcStatus.GRpcStatusCode != grpc::StatusCode::CANCELLED) { + dbState->EndpointPool.BanEndpoint(endpoint); + } + // TODO: Add headers for streaming calls. + TPlainStatus status(std::move(grpcStatus), endpoint, {}); + connectedCallback(std::move(status), nullptr); + } + }; + + serviceConnection->template DoStreamRequest<TRequest, TResponse>( + std::move(lowCallback), + std::move(rpc), + std::move(meta), + context.get()); }, dbState, TStringType(), endpointPolicy); - } - + } + TAsyncListEndpointsResult GetEndpoints(TDbDriverStatePtr dbState) override; TListEndpointsResult MutateDiscovery(TListEndpointsResult result, const TStringType& database); - -#ifndef YDB_GRPC_BYPASS_CHANNEL_POOL + +#ifndef YDB_GRPC_BYPASS_CHANNEL_POOL void DeleteChannels(const std::vector<TStringType>& endpoints) override { - for (const auto& endpoint : endpoints) { - ChannelPool_.DeleteChannel(endpoint); - } - } -#endif - - bool GetDrainOnDtors() const; - TBalancingSettings GetBalancingSettings() const override; - bool StartStatCollecting(NMonitoring::IMetricRegistry* sensorsRegistry) override; - NMonitoring::TMetricRegistry* GetMetricRegistry() override; - void RegisterExtension(IExtension* extension); - void RegisterExtensionApi(IExtensionApi* api); - void SetDiscoveryMutator(IDiscoveryMutatorApi::TMutatorCb&& cb); - const TLog& GetLog() const override; - -private: - template <typename TService, typename TCallback> + for (const auto& endpoint : endpoints) { + ChannelPool_.DeleteChannel(endpoint); + } + } +#endif + + bool GetDrainOnDtors() const; + TBalancingSettings GetBalancingSettings() const override; + bool StartStatCollecting(NMonitoring::IMetricRegistry* sensorsRegistry) override; + NMonitoring::TMetricRegistry* GetMetricRegistry() override; + void RegisterExtension(IExtension* extension); + void RegisterExtensionApi(IExtensionApi* api); + void SetDiscoveryMutator(IDiscoveryMutatorApi::TMutatorCb&& cb); + const TLog& GetLog() const override; + +private: + template <typename TService, typename TCallback> void WithServiceConnection(TCallback callback, TDbDriverStatePtr dbState, const TStringType& preferredEndpoint, TRpcRequestSettings::TEndpointPolicy endpointPolicy) { - using TConnection = std::unique_ptr<TServiceConnection<TService>>; - TConnection serviceConnection; + using TConnection = std::unique_ptr<TServiceConnection<TService>>; + TConnection serviceConnection; TStringType endpoint; std::tie(serviceConnection, endpoint) = GetServiceConnection<TService>(dbState, preferredEndpoint, endpointPolicy); - if (!serviceConnection) { - if (dbState->DiscoveryMode == EDiscoveryMode::Sync) { - TStringStream errString; - errString << "Endpoint list is empty for database " << dbState->Database; - errString << ", cluster endpoint " << dbState->DiscoveryEndpoint; + if (!serviceConnection) { + if (dbState->DiscoveryMode == EDiscoveryMode::Sync) { + TStringStream errString; + errString << "Endpoint list is empty for database " << dbState->Database; + errString << ", cluster endpoint " << dbState->DiscoveryEndpoint; TPlainStatus discoveryStatus; - { + { std::shared_lock guard(dbState->LastDiscoveryStatusRWLock); discoveryStatus = dbState->LastDiscoveryStatus; - } + } // Discovery returns success but no serviceConnection - in this case we unable to continue processing if (discoveryStatus.Ok()) { @@ -630,96 +630,96 @@ private: errString <<"."; discoveryStatus.Issues.AddIssues({NYql::TIssue(errString.Str())}); } - dbState->StatCollector.IncReqFailNoEndpoint(); - callback( - discoveryStatus, - TConnection{nullptr}, + dbState->StatCollector.IncReqFailNoEndpoint(); + callback( + discoveryStatus, + TConnection{nullptr}, TStringType{ }); - } else { + } else { int64_t newVal; int64_t val; - do { + do { val = QueuedRequests_.load(); - if (val >= MaxQueuedRequests_) { - dbState->StatCollector.IncReqFailQueueOverflow(); - callback( - TPlainStatus(EStatus::CLIENT_LIMITS_REACHED, "Requests queue limit reached"), - TConnection{nullptr}, + if (val >= MaxQueuedRequests_) { + dbState->StatCollector.IncReqFailQueueOverflow(); + callback( + TPlainStatus(EStatus::CLIENT_LIMITS_REACHED, "Requests queue limit reached"), + TConnection{nullptr}, TStringType{ }); - return; - } - newVal = val + 1; + return; + } + newVal = val + 1; } while (!QueuedRequests_.compare_exchange_weak(val, newVal)); - - // UpdateAsync guarantee one update in progress for state - auto asyncResult = dbState->EndpointPool.UpdateAsync(); - const bool needUpdateChannels = asyncResult.second; + + // UpdateAsync guarantee one update in progress for state + auto asyncResult = dbState->EndpointPool.UpdateAsync(); + const bool needUpdateChannels = asyncResult.second; asyncResult.first.Subscribe([this, callback = std::move(callback), needUpdateChannels, dbState, preferredEndpoint, endpointPolicy] - (const NThreading::TFuture<TEndpointUpdateResult>& future) mutable { + (const NThreading::TFuture<TEndpointUpdateResult>& future) mutable { --QueuedRequests_; - const auto& updateResult = future.GetValue(); - if (needUpdateChannels) { -#ifndef YDB_GRPC_BYPASS_CHANNEL_POOL - DeleteChannels(updateResult.Removed); -#endif - } - auto discoveryStatus = updateResult.DiscoveryStatus; - if (discoveryStatus.Status == EStatus::SUCCESS) { + const auto& updateResult = future.GetValue(); + if (needUpdateChannels) { +#ifndef YDB_GRPC_BYPASS_CHANNEL_POOL + DeleteChannels(updateResult.Removed); +#endif + } + auto discoveryStatus = updateResult.DiscoveryStatus; + if (discoveryStatus.Status == EStatus::SUCCESS) { WithServiceConnection<TService>(std::move(callback), dbState, preferredEndpoint, endpointPolicy); - } else { - callback( - TPlainStatus(discoveryStatus.Status, std::move(discoveryStatus.Issues)), - TConnection{nullptr}, + } else { + callback( + TPlainStatus(discoveryStatus.Status, std::move(discoveryStatus.Issues)), + TConnection{nullptr}, TStringType{ }); - } - }); - } - return; - } - - callback( - TPlainStatus{ }, - std::move(serviceConnection), - std::move(endpoint)); - } - - void EnqueueResponse(IObjectInQueue* action); - -private: + } + }); + } + return; + } + + callback( + TPlainStatus{ }, + std::move(serviceConnection), + std::move(endpoint)); + } + + void EnqueueResponse(IObjectInQueue* action); + +private: std::mutex ExtensionsLock_; - NMonitoring::TMetricRegistry* MetricRegistryPtr_ = nullptr; - - std::unique_ptr<IThreadPool> ResponseQueue_; - + NMonitoring::TMetricRegistry* MetricRegistryPtr_ = nullptr; + + std::unique_ptr<IThreadPool> ResponseQueue_; + const TStringType DefaultDiscoveryEndpoint_; - const bool EnableSsl_; + const bool EnableSsl_; const TStringType CaCert_; const TStringType DefaultDatabase_; - std::shared_ptr<ICredentialsProviderFactory> DefaultCredentialsProviderFactory_; - TDbDriverStateTracker StateTracker_; - const EDiscoveryMode DefaultDiscoveryMode_; - const i64 MaxQueuedRequests_; - const bool DrainOnDtors_; - const TBalancingSettings BalancingSettings_; - const TDuration GRpcKeepAliveTimeout_; - const bool GRpcKeepAlivePermitWithoutCalls_; + std::shared_ptr<ICredentialsProviderFactory> DefaultCredentialsProviderFactory_; + TDbDriverStateTracker StateTracker_; + const EDiscoveryMode DefaultDiscoveryMode_; + const i64 MaxQueuedRequests_; + const bool DrainOnDtors_; + const TBalancingSettings BalancingSettings_; + const TDuration GRpcKeepAliveTimeout_; + const bool GRpcKeepAlivePermitWithoutCalls_; const ui64 MemoryQuota_; - + std::atomic_int64_t QueuedRequests_; -#ifndef YDB_GRPC_BYPASS_CHANNEL_POOL - NGrpc::TChannelPool ChannelPool_; -#endif - // State for default database, token pair - TDbDriverStatePtr DefaultState_; - +#ifndef YDB_GRPC_BYPASS_CHANNEL_POOL + NGrpc::TChannelPool ChannelPool_; +#endif + // State for default database, token pair + TDbDriverStatePtr DefaultState_; + std::vector<std::unique_ptr<IExtension>> Extensions_; std::vector<std::unique_ptr<IExtensionApi>> ExtensionApis_; - - IDiscoveryMutatorApi::TMutatorCb DiscoveryMutatorCb; - - // Must be the last member (first called destructor) - NGrpc::TGRpcClientLow GRpcClientLow_; - TLog Log; -}; - -} // namespace NYdb + + IDiscoveryMutatorApi::TMutatorCb DiscoveryMutatorCb; + + // Must be the last member (first called destructor) + NGrpc::TGRpcClientLow GRpcClientLow_; + TLog Log; +}; + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/params.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/params.h index 2425d6883b..7833c60947 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/params.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/params.h @@ -1,30 +1,30 @@ -#pragma once - +#pragma once + #include <ydb/public/sdk/cpp/client/impl/ydb_internal/internal_header.h> - -namespace NYdb { - -class IConnectionsParams { -public: - virtual ~IConnectionsParams() = default; + +namespace NYdb { + +class IConnectionsParams { +public: + virtual ~IConnectionsParams() = default; virtual TStringType GetEndpoint() const = 0; - virtual size_t GetNetworkThreadsNum() const = 0; - virtual size_t GetClientThreadsNum() const = 0; - virtual size_t GetMaxQueuedResponses() const = 0; - virtual bool IsSslEnabled() const = 0; + virtual size_t GetNetworkThreadsNum() const = 0; + virtual size_t GetClientThreadsNum() const = 0; + virtual size_t GetMaxQueuedResponses() const = 0; + virtual bool IsSslEnabled() const = 0; virtual TStringType GetCaCert() const = 0; virtual TStringType GetDatabase() const = 0; - virtual std::shared_ptr<ICredentialsProviderFactory> GetCredentialsProviderFactory() const = 0; - virtual EDiscoveryMode GetDiscoveryMode() const = 0; - virtual size_t GetMaxQueuedRequests() const = 0; - virtual NGrpc::TTcpKeepAliveSettings GetTcpKeepAliveSettings() const = 0; - virtual bool GetDrinOnDtors() const = 0; - virtual TBalancingSettings GetBalancingSettings() const = 0; - virtual TDuration GetGRpcKeepAliveTimeout() const = 0; - virtual bool GetGRpcKeepAlivePermitWithoutCalls() const = 0; + virtual std::shared_ptr<ICredentialsProviderFactory> GetCredentialsProviderFactory() const = 0; + virtual EDiscoveryMode GetDiscoveryMode() const = 0; + virtual size_t GetMaxQueuedRequests() const = 0; + virtual NGrpc::TTcpKeepAliveSettings GetTcpKeepAliveSettings() const = 0; + virtual bool GetDrinOnDtors() const = 0; + virtual TBalancingSettings GetBalancingSettings() const = 0; + virtual TDuration GetGRpcKeepAliveTimeout() const = 0; + virtual bool GetGRpcKeepAlivePermitWithoutCalls() const = 0; virtual TDuration GetSocketIdleTimeout() const = 0; - virtual const TLog& GetLog() const = 0; + virtual const TLog& GetLog() const = 0; virtual ui64 GetMemoryQuota() const = 0; -}; - -} // namespace NYdb +}; + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/ya.make b/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/ya.make index a23d6e5872..5d79153da3 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/ya.make +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/ya.make @@ -1,13 +1,13 @@ -LIBRARY() - +LIBRARY() + OWNER(g:kikimr) - -SRCS( - actions.cpp - grpc_connections.cpp -) - -PEERDIR( + +SRCS( + actions.cpp + grpc_connections.cpp +) + +PEERDIR( ydb/public/api/grpc ydb/public/api/protos ydb/public/sdk/cpp/client/impl/ydb_internal/db_driver_state @@ -16,6 +16,6 @@ PEERDIR( ydb/public/sdk/cpp/client/impl/ydb_stats ydb/public/sdk/cpp/client/resources ydb/public/sdk/cpp/client/ydb_types/exceptions -) - -END() +) + +END() diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/internal_client/client.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/internal_client/client.h index 87340dc021..07fbadf88b 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/internal_client/client.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/internal_client/client.h @@ -1,37 +1,37 @@ -#pragma once - +#pragma once + #include <ydb/public/sdk/cpp/client/impl/ydb_internal/internal_header.h> - + #include <ydb/public/sdk/cpp/client/impl/ydb_internal/common/type_switcher.h> #include <ydb/public/sdk/cpp/client/impl/ydb_internal/common/types.h> #include <ydb/public/sdk/cpp/client/ydb_types/ydb.h> #include <ydb/public/sdk/cpp/client/ydb_types/core_facility/core_facility.h> - -#include <library/cpp/threading/future/future.h> -#include <library/cpp/logger/log.h> - -namespace NMonitoring { - class IMetricRegistry; - class TMetricRegistry; -} - -namespace NYdb { - + +#include <library/cpp/threading/future/future.h> +#include <library/cpp/logger/log.h> + +namespace NMonitoring { + class IMetricRegistry; + class TMetricRegistry; +} + +namespace NYdb { + class TDbDriverState; -struct TListEndpointsResult; - -class IInternalClient { -public: - virtual NThreading::TFuture<TListEndpointsResult> GetEndpoints(std::shared_ptr<TDbDriverState> dbState) = 0; - virtual void AddPeriodicTask(TPeriodicCb&& cb, TDuration period) = 0; -#ifndef YDB_GRPC_BYPASS_CHANNEL_POOL +struct TListEndpointsResult; + +class IInternalClient { +public: + virtual NThreading::TFuture<TListEndpointsResult> GetEndpoints(std::shared_ptr<TDbDriverState> dbState) = 0; + virtual void AddPeriodicTask(TPeriodicCb&& cb, TDuration period) = 0; +#ifndef YDB_GRPC_BYPASS_CHANNEL_POOL virtual void DeleteChannels(const std::vector<TStringType>& endpoints) = 0; -#endif - virtual TBalancingSettings GetBalancingSettings() const = 0; - virtual bool StartStatCollecting(NMonitoring::IMetricRegistry* sensorsRegistry) = 0; - virtual NMonitoring::TMetricRegistry* GetMetricRegistry() = 0; - virtual const TLog& GetLog() const = 0; -}; - -} // namespace NYdb +#endif + virtual TBalancingSettings GetBalancingSettings() const = 0; + virtual bool StartStatCollecting(NMonitoring::IMetricRegistry* sensorsRegistry) = 0; + virtual NMonitoring::TMetricRegistry* GetMetricRegistry() = 0; + virtual const TLog& GetLog() const = 0; +}; + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/internal_header.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/internal_header.h index 159f034437..dce5d14301 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/internal_header.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/internal_header.h @@ -1,11 +1,11 @@ -#pragma once - -#if !defined(INCLUDE_YDB_INTERNAL_H) -#error "you are trying to use internal ydb header" -#endif // INCLUDE_YDB_INTERNAL_H - -// This macro allow to send credentials data via insecure channel -#define YDB_GRPC_UNSECURE_AUTH - -// This macro is used for grpc debug purpose only -//#define YDB_GRPC_BYPASS_CHANNEL_POOL +#pragma once + +#if !defined(INCLUDE_YDB_INTERNAL_H) +#error "you are trying to use internal ydb header" +#endif // INCLUDE_YDB_INTERNAL_H + +// This macro allow to send credentials data via insecure channel +#define YDB_GRPC_UNSECURE_AUTH + +// This macro is used for grpc debug purpose only +//#define YDB_GRPC_BYPASS_CHANNEL_POOL diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/logger/log.cpp b/ydb/public/sdk/cpp/client/impl/ydb_internal/logger/log.cpp index 6e0f44a3a1..da3751e1f8 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/logger/log.cpp +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/logger/log.cpp @@ -1,47 +1,47 @@ -#define INCLUDE_YDB_INTERNAL_H -#include "log.h" - -#include <util/datetime/base.h> -#include <util/string/builder.h> - -namespace NYdb { - -static const TStringBuf LogPrioritiesStrings[] = { - " :EMERG: ", - " :ALERT: ", - " :CRIT: ", - " :ERROR: ", - " :WARNING: ", - " :NOTICE: ", - " :INFO: ", - " :DEBUG: ", - " :TRACE: ", -}; - -TStringBuf LogPriorityToString(ELogPriority priority) { - constexpr int maxPriority = static_cast<int>(std::size(LogPrioritiesStrings) - 1); - const int index = static_cast<int>(priority); - return LogPrioritiesStrings[ClampVal(index, 0, maxPriority)]; -} - -TLogFormatter GetPrefixLogFormatter(const TString& prefix) { - return [prefix](ELogPriority priority, TStringBuf message) -> TString { - constexpr size_t timeLen = 27; // 2020-10-08T20:31:11.202588Z - constexpr size_t endlLen = 1; - - const TStringBuf priorityString = LogPriorityToString(priority); - TStringBuilder result; - const size_t toReserve = prefix.size() + message.Size() + timeLen + endlLen + priorityString.Size(); - result.reserve(toReserve); - - result << TInstant::Now() << priorityString << prefix << message << Endl; - Y_ASSERT(result.size() == toReserve); - return std::move(result); - }; -} - +#define INCLUDE_YDB_INTERNAL_H +#include "log.h" + +#include <util/datetime/base.h> +#include <util/string/builder.h> + +namespace NYdb { + +static const TStringBuf LogPrioritiesStrings[] = { + " :EMERG: ", + " :ALERT: ", + " :CRIT: ", + " :ERROR: ", + " :WARNING: ", + " :NOTICE: ", + " :INFO: ", + " :DEBUG: ", + " :TRACE: ", +}; + +TStringBuf LogPriorityToString(ELogPriority priority) { + constexpr int maxPriority = static_cast<int>(std::size(LogPrioritiesStrings) - 1); + const int index = static_cast<int>(priority); + return LogPrioritiesStrings[ClampVal(index, 0, maxPriority)]; +} + +TLogFormatter GetPrefixLogFormatter(const TString& prefix) { + return [prefix](ELogPriority priority, TStringBuf message) -> TString { + constexpr size_t timeLen = 27; // 2020-10-08T20:31:11.202588Z + constexpr size_t endlLen = 1; + + const TStringBuf priorityString = LogPriorityToString(priority); + TStringBuilder result; + const size_t toReserve = prefix.size() + message.Size() + timeLen + endlLen + priorityString.Size(); + result.reserve(toReserve); + + result << TInstant::Now() << priorityString << prefix << message << Endl; + Y_ASSERT(result.size() == toReserve); + return std::move(result); + }; +} + TStringType GetDatabaseLogPrefix(const TStringType& database) { return "[" + database + "] "; -} - -} // namespace NYdb +} + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/logger/log.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/logger/log.h index 7271ef4edb..dc1ede5d29 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/logger/log.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/logger/log.h @@ -1,13 +1,13 @@ -#pragma once - +#pragma once + #include <ydb/public/sdk/cpp/client/impl/ydb_internal/internal_header.h> #include <ydb/public/sdk/cpp/client/impl/ydb_internal/common/type_switcher.h> - -#include <library/cpp/logger/log.h> - -namespace NYdb { - -TLogFormatter GetPrefixLogFormatter(const TString& prefix); + +#include <library/cpp/logger/log.h> + +namespace NYdb { + +TLogFormatter GetPrefixLogFormatter(const TString& prefix); TStringType GetDatabaseLogPrefix(const TStringType& database); - -} // namespace NYdb + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/logger/ya.make b/ydb/public/sdk/cpp/client/impl/ydb_internal/logger/ya.make index 54a1e21d05..89a802f98d 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/logger/ya.make +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/logger/ya.make @@ -1,13 +1,13 @@ -LIBRARY() - +LIBRARY() + OWNER(g:kikimr) - -SRCS( - log.cpp -) - -PEERDIR( - library/cpp/logger -) - -END() + +SRCS( + log.cpp +) + +PEERDIR( + library/cpp/logger +) + +END() diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.cpp b/ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.cpp index e4eaf8e7ab..bfbf465529 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.cpp +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.cpp @@ -1,12 +1,12 @@ -#define INCLUDE_YDB_INTERNAL_H -#include "make.h" - - -namespace NYdb { - -void SetDuration(const TDuration& duration, google::protobuf::Duration& protoValue) { - protoValue.set_seconds(duration.Seconds()); - protoValue.set_nanos(duration.NanoSecondsOfSecond()); -} - -} // namespace NYdb +#define INCLUDE_YDB_INTERNAL_H +#include "make.h" + + +namespace NYdb { + +void SetDuration(const TDuration& duration, google::protobuf::Duration& protoValue) { + protoValue.set_seconds(duration.Seconds()); + protoValue.set_nanos(duration.NanoSecondsOfSecond()); +} + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.h index 3cf3a4cffc..c055ee9609 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.h @@ -1,45 +1,45 @@ -#pragma once - +#pragma once + #include <ydb/public/sdk/cpp/client/impl/ydb_internal/internal_header.h> #include <ydb/public/api/protos/ydb_common.pb.h> - -#include <util/datetime/base.h> - -#include <google/protobuf/duration.pb.h> - -namespace NYdb { - -void SetDuration(const TDuration& duration, google::protobuf::Duration& protoValue); - -template <typename TRequestSettings, typename TProtoRequest> -void FillOperationParams(const TRequestSettings& settings, TProtoRequest& params) { - auto& operationParams = *params.mutable_operation_params(); - - if (settings.CancelAfter_) { - SetDuration(settings.CancelAfter_, *operationParams.mutable_cancel_after()); - } - - if (settings.OperationTimeout_) { - SetDuration(settings.OperationTimeout_, *operationParams.mutable_operation_timeout()); - } else if (settings.ClientTimeout_ && settings.UseClientTimeoutForOperation_) { - SetDuration(settings.ClientTimeout_, *operationParams.mutable_operation_timeout()); - } + +#include <util/datetime/base.h> + +#include <google/protobuf/duration.pb.h> + +namespace NYdb { + +void SetDuration(const TDuration& duration, google::protobuf::Duration& protoValue); + +template <typename TRequestSettings, typename TProtoRequest> +void FillOperationParams(const TRequestSettings& settings, TProtoRequest& params) { + auto& operationParams = *params.mutable_operation_params(); + + if (settings.CancelAfter_) { + SetDuration(settings.CancelAfter_, *operationParams.mutable_cancel_after()); + } + + if (settings.OperationTimeout_) { + SetDuration(settings.OperationTimeout_, *operationParams.mutable_operation_timeout()); + } else if (settings.ClientTimeout_ && settings.UseClientTimeoutForOperation_) { + SetDuration(settings.ClientTimeout_, *operationParams.mutable_operation_timeout()); + } if (settings.ReportCostInfo_) { operationParams.set_report_cost_info(Ydb::FeatureFlag::ENABLED); } -} - -template <typename TProtoRequest> -TProtoRequest MakeRequest() { - return TProtoRequest(); -} - -template <typename TProtoRequest, typename TRequestSettings> -TProtoRequest MakeOperationRequest(const TRequestSettings& settings) { - auto request = MakeRequest<TProtoRequest>(); - FillOperationParams(settings, request); - return request; -} - -} // namespace NYdb +} + +template <typename TProtoRequest> +TProtoRequest MakeRequest() { + return TProtoRequest(); +} + +template <typename TProtoRequest, typename TRequestSettings> +TProtoRequest MakeOperationRequest(const TRequestSettings& settings) { + auto request = MakeRequest<TProtoRequest>(); + FillOperationParams(settings, request); + return request; +} + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/ya.make b/ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/ya.make index 46518f378a..87d7f3f272 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/ya.make +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/ya.make @@ -1,14 +1,14 @@ -LIBRARY() - +LIBRARY() + OWNER(g:kikimr) - -SRCS( - make.cpp -) - -PEERDIR( - contrib/libs/protobuf + +SRCS( + make.cpp +) + +PEERDIR( + contrib/libs/protobuf ydb/public/api/protos -) - -END() +) + +END() diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/plain_status/status.cpp b/ydb/public/sdk/cpp/client/impl/ydb_internal/plain_status/status.cpp index d86f3c1556..0449c78552 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/plain_status/status.cpp +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/plain_status/status.cpp @@ -1,63 +1,63 @@ -#define INCLUDE_YDB_INTERNAL_H -#include "status.h" - -#include <util/string/builder.h> - -namespace NYdb { - -TPlainStatus::TPlainStatus( - const NGrpc::TGrpcStatus& grpcStatus, +#define INCLUDE_YDB_INTERNAL_H +#include "status.h" + +#include <util/string/builder.h> + +namespace NYdb { + +TPlainStatus::TPlainStatus( + const NGrpc::TGrpcStatus& grpcStatus, const TStringType& endpoint, std::multimap<TStringType, TStringType>&& metadata) - : Endpoint(endpoint) - , Metadata(std::move(metadata)) -{ + : Endpoint(endpoint) + , Metadata(std::move(metadata)) +{ TStringType msg; - if (grpcStatus.InternalError) { - Status = EStatus::CLIENT_INTERNAL_ERROR; - if (grpcStatus.Msg) { - msg = TStringBuilder() << "Internal client error: " << grpcStatus.Msg; - } else { - msg = "Unknown internal client error"; - } - } else if (grpcStatus.GRpcStatusCode != grpc::StatusCode::OK) { - switch (grpcStatus.GRpcStatusCode) { - case grpc::StatusCode::UNAVAILABLE: - Status = EStatus::TRANSPORT_UNAVAILABLE; - break; - case grpc::StatusCode::CANCELLED: - Status = EStatus::CLIENT_CANCELLED; - break; - case grpc::StatusCode::UNAUTHENTICATED: - Status = EStatus::CLIENT_UNAUTHENTICATED; - break; - case grpc::StatusCode::UNIMPLEMENTED: - Status = EStatus::CLIENT_CALL_UNIMPLEMENTED; - break; - case grpc::StatusCode::RESOURCE_EXHAUSTED: - Status = EStatus::CLIENT_RESOURCE_EXHAUSTED; - break; - case grpc::StatusCode::DEADLINE_EXCEEDED: - Status = EStatus::CLIENT_DEADLINE_EXCEEDED; - break; - case grpc::StatusCode::OUT_OF_RANGE: - Status = EStatus::CLIENT_OUT_OF_RANGE; - break; - default: - Status = EStatus::CLIENT_INTERNAL_ERROR; - break; - } - msg = TStringBuilder() << "GRpc error: (" << grpcStatus.GRpcStatusCode << "): " << grpcStatus.Msg; - } else { - Status = EStatus::SUCCESS; - } - if (msg) { - Issues.AddIssue(NYql::TIssue(msg)); - } -} - + if (grpcStatus.InternalError) { + Status = EStatus::CLIENT_INTERNAL_ERROR; + if (grpcStatus.Msg) { + msg = TStringBuilder() << "Internal client error: " << grpcStatus.Msg; + } else { + msg = "Unknown internal client error"; + } + } else if (grpcStatus.GRpcStatusCode != grpc::StatusCode::OK) { + switch (grpcStatus.GRpcStatusCode) { + case grpc::StatusCode::UNAVAILABLE: + Status = EStatus::TRANSPORT_UNAVAILABLE; + break; + case grpc::StatusCode::CANCELLED: + Status = EStatus::CLIENT_CANCELLED; + break; + case grpc::StatusCode::UNAUTHENTICATED: + Status = EStatus::CLIENT_UNAUTHENTICATED; + break; + case grpc::StatusCode::UNIMPLEMENTED: + Status = EStatus::CLIENT_CALL_UNIMPLEMENTED; + break; + case grpc::StatusCode::RESOURCE_EXHAUSTED: + Status = EStatus::CLIENT_RESOURCE_EXHAUSTED; + break; + case grpc::StatusCode::DEADLINE_EXCEEDED: + Status = EStatus::CLIENT_DEADLINE_EXCEEDED; + break; + case grpc::StatusCode::OUT_OF_RANGE: + Status = EStatus::CLIENT_OUT_OF_RANGE; + break; + default: + Status = EStatus::CLIENT_INTERNAL_ERROR; + break; + } + msg = TStringBuilder() << "GRpc error: (" << grpcStatus.GRpcStatusCode << "): " << grpcStatus.Msg; + } else { + Status = EStatus::SUCCESS; + } + if (msg) { + Issues.AddIssue(NYql::TIssue(msg)); + } +} + TPlainStatus TPlainStatus::Internal(const TStringType& message) { return { EStatus::CLIENT_INTERNAL_ERROR, "Internal client error: " + message }; -} - -} // namespace NYdb +} + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/plain_status/status.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/plain_status/status.h index d1a5832566..084f667892 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/plain_status/status.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/plain_status/status.h @@ -1,68 +1,68 @@ -#pragma once - +#pragma once + #include <ydb/public/sdk/cpp/client/impl/ydb_internal/internal_header.h> #include <ydb/public/sdk/cpp/client/impl/ydb_internal/common/type_switcher.h> #include <ydb/public/sdk/cpp/client/ydb_types/status_codes.h> - + #include <ydb/public/api/protos/ydb_operation.pb.h> #include <ydb/library/yql/public/issue/yql_issue.h> - -#include <library/cpp/grpc/client/grpc_client_low.h> - -namespace NYdb { - -struct TPlainStatus { - EStatus Status; - NYql::TIssues Issues; + +#include <library/cpp/grpc/client/grpc_client_low.h> + +namespace NYdb { + +struct TPlainStatus { + EStatus Status; + NYql::TIssues Issues; TStringType Endpoint; std::multimap<TStringType, TStringType> Metadata; Ydb::CostInfo ConstInfo; - - TPlainStatus() - : Status(EStatus::SUCCESS) - { } - - TPlainStatus(EStatus status, NYql::TIssues&& issues) - : Status(status) - , Issues(std::move(issues)) - { } - + + TPlainStatus() + : Status(EStatus::SUCCESS) + { } + + TPlainStatus(EStatus status, NYql::TIssues&& issues) + : Status(status) + , Issues(std::move(issues)) + { } + TPlainStatus(EStatus status, NYql::TIssues&& issues, const TStringType& endpoint, std::multimap<TStringType, TStringType>&& metadata) - : Status(status) - , Issues(std::move(issues)) - , Endpoint(endpoint) - , Metadata(std::move(metadata)) - { } - + : Status(status) + , Issues(std::move(issues)) + , Endpoint(endpoint) + , Metadata(std::move(metadata)) + { } + TPlainStatus(EStatus status, const TStringType& message) - : Status(status) - { + : Status(status) + { if (!message.empty()) { - Issues.AddIssue(NYql::TIssue(message)); - } - } - - TPlainStatus( + Issues.AddIssue(NYql::TIssue(message)); + } + } + + TPlainStatus( const NGrpc::TGrpcStatus& grpcStatus, const TStringType& endpoint = TStringType(), std::multimap<TStringType, TStringType>&& metadata = {}); - + template<class T> void SetCostInfo(T&& costInfo) { ConstInfo = std::forward<T>(costInfo); } - bool Ok() const { - return Status == EStatus::SUCCESS; - } - + bool Ok() const { + return Status == EStatus::SUCCESS; + } + static TPlainStatus Internal(const TStringType& message); - - bool IsTransportError() const { - auto status = static_cast<size_t>(Status); - return TRANSPORT_STATUSES_FIRST <= status && status <= TRANSPORT_STATUSES_LAST; - } -}; - -} // namespace NYdb + + bool IsTransportError() const { + auto status = static_cast<size_t>(Status); + return TRANSPORT_STATUSES_FIRST <= status && status <= TRANSPORT_STATUSES_LAST; + } +}; + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/plain_status/ya.make b/ydb/public/sdk/cpp/client/impl/ydb_internal/plain_status/ya.make index 683105e939..93157bcf39 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/plain_status/ya.make +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/plain_status/ya.make @@ -1,15 +1,15 @@ -LIBRARY() - +LIBRARY() + OWNER(g:kikimr) - -SRCS( - status.cpp -) - -PEERDIR( + +SRCS( + status.cpp +) + +PEERDIR( contrib/libs/protobuf library/cpp/grpc/client ydb/library/yql/public/issue -) - -END() +) + +END() diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/rpc_request_settings/settings.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/rpc_request_settings/settings.h index dda9b5302f..2a2611794d 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/rpc_request_settings/settings.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/rpc_request_settings/settings.h @@ -1,11 +1,11 @@ -#pragma once - +#pragma once + #include <ydb/public/sdk/cpp/client/impl/ydb_internal/internal_header.h> #include <ydb/public/sdk/cpp/client/impl/ydb_internal/common/type_switcher.h> - -namespace NYdb { - -struct TRpcRequestSettings { + +namespace NYdb { + +struct TRpcRequestSettings { TStringType TraceId; TStringType RequestType; std::vector<std::pair<TStringType, TStringType>> Header; @@ -14,17 +14,17 @@ struct TRpcRequestSettings { UseDiscoveryEndpoint // Use single discovery endpoint for request } EndpointPolicy = TEndpointPolicy::UsePreferedEndpoint; bool UseAuth = true; - - template<typename TRequestSettings> - static TRpcRequestSettings Make(const TRequestSettings& settings) { - TRpcRequestSettings rpcSettings; - rpcSettings.TraceId = settings.TraceId_; - rpcSettings.RequestType = settings.RequestType_; + + template<typename TRequestSettings> + static TRpcRequestSettings Make(const TRequestSettings& settings) { + TRpcRequestSettings rpcSettings; + rpcSettings.TraceId = settings.TraceId_; + rpcSettings.RequestType = settings.RequestType_; rpcSettings.Header = settings.Header_; rpcSettings.EndpointPolicy = TEndpointPolicy::UsePreferedEndpoint; rpcSettings.UseAuth = true; - return rpcSettings; - } -}; - -} // namespace NYdb + return rpcSettings; + } +}; + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/stats_extractor/extractor.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/stats_extractor/extractor.h index 7ca0517f34..2a2186a0dc 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/stats_extractor/extractor.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/stats_extractor/extractor.h @@ -1,44 +1,44 @@ -#pragma once - +#pragma once + #include <ydb/public/sdk/cpp/client/impl/ydb_internal/internal_header.h> - + #include <ydb/public/sdk/cpp/client/impl/ydb_internal/internal_client/client.h> - + #include <ydb/public/sdk/cpp/client/ydb_extension/extension.h> - -#include <library/cpp/monlib/metrics/metric_registry.h> - -namespace NYdb { - -class TStatsExtractor: public NSdkStats::IStatApi { -public: - - TStatsExtractor(std::shared_ptr<IInternalClient> client) - : Client_(client) - { } - - virtual void SetMetricRegistry(NMonitoring::IMetricRegistry* sensorsRegistry) override { - auto strong = Client_.lock(); - if (strong) { - strong->StartStatCollecting(sensorsRegistry); - } else { - return; - } - } - - void Accept(NMonitoring::IMetricConsumer* consumer) const override { - - auto strong = Client_.lock(); - if (strong) { - auto sensorsRegistry = strong->GetMetricRegistry(); - Y_VERIFY(sensorsRegistry, "TMetricRegistry is null in Stats Extractor"); - sensorsRegistry->Accept(TInstant::Zero(), consumer); - } else { - throw NSdkStats::DestroyedClientException(); - } - } -private: - std::weak_ptr<IInternalClient> Client_; -}; - -} // namespace NYdb + +#include <library/cpp/monlib/metrics/metric_registry.h> + +namespace NYdb { + +class TStatsExtractor: public NSdkStats::IStatApi { +public: + + TStatsExtractor(std::shared_ptr<IInternalClient> client) + : Client_(client) + { } + + virtual void SetMetricRegistry(NMonitoring::IMetricRegistry* sensorsRegistry) override { + auto strong = Client_.lock(); + if (strong) { + strong->StartStatCollecting(sensorsRegistry); + } else { + return; + } + } + + void Accept(NMonitoring::IMetricConsumer* consumer) const override { + + auto strong = Client_.lock(); + if (strong) { + auto sensorsRegistry = strong->GetMetricRegistry(); + Y_VERIFY(sensorsRegistry, "TMetricRegistry is null in Stats Extractor"); + sensorsRegistry->Accept(TInstant::Zero(), consumer); + } else { + throw NSdkStats::DestroyedClientException(); + } + } +private: + std::weak_ptr<IInternalClient> Client_; +}; + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/table_helpers/helpers.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/table_helpers/helpers.h index 14bf8a2fb7..2ab4c68dcc 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/table_helpers/helpers.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/table_helpers/helpers.h @@ -1,38 +1,38 @@ -#pragma once - +#pragma once + #include <ydb/public/sdk/cpp/client/impl/ydb_internal/internal_header.h> - + #include <ydb/public/api/protos/ydb_table.pb.h> #include <ydb/public/sdk/cpp/client/ydb_scheme/scheme.h> #include <ydb/public/sdk/cpp/client/ydb_table/query_stats/stats.h> - -namespace NYdb { - -template<typename TFrom> -inline void PermissionToSchemeEntry(const TFrom& from, TVector<NScheme::TPermissions>* to) { - for (const auto& effPerm : from) { - to->push_back(NScheme::TPermissions{effPerm.subject(), TVector<TString>()}); - for (const auto& permName : effPerm.permission_names()) { - to->back().PermissionNames.push_back(permName); - } - } -} - -inline Ydb::Table::QueryStatsCollection::Mode GetStatsCollectionMode(TMaybe<NTable::ECollectQueryStatsMode> mode) { - if (mode) { - switch (*mode) { - case NTable::ECollectQueryStatsMode::None: - return Ydb::Table::QueryStatsCollection::STATS_COLLECTION_NONE; - case NTable::ECollectQueryStatsMode::Basic: - return Ydb::Table::QueryStatsCollection::STATS_COLLECTION_BASIC; - case NTable::ECollectQueryStatsMode::Full: - return Ydb::Table::QueryStatsCollection::STATS_COLLECTION_FULL; - default: - break; - } - } - - return Ydb::Table::QueryStatsCollection::STATS_COLLECTION_UNSPECIFIED; -} - -} // namespace NYdb + +namespace NYdb { + +template<typename TFrom> +inline void PermissionToSchemeEntry(const TFrom& from, TVector<NScheme::TPermissions>* to) { + for (const auto& effPerm : from) { + to->push_back(NScheme::TPermissions{effPerm.subject(), TVector<TString>()}); + for (const auto& permName : effPerm.permission_names()) { + to->back().PermissionNames.push_back(permName); + } + } +} + +inline Ydb::Table::QueryStatsCollection::Mode GetStatsCollectionMode(TMaybe<NTable::ECollectQueryStatsMode> mode) { + if (mode) { + switch (*mode) { + case NTable::ECollectQueryStatsMode::None: + return Ydb::Table::QueryStatsCollection::STATS_COLLECTION_NONE; + case NTable::ECollectQueryStatsMode::Basic: + return Ydb::Table::QueryStatsCollection::STATS_COLLECTION_BASIC; + case NTable::ECollectQueryStatsMode::Full: + return Ydb::Table::QueryStatsCollection::STATS_COLLECTION_FULL; + default: + break; + } + } + + return Ydb::Table::QueryStatsCollection::STATS_COLLECTION_UNSPECIFIED; +} + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/thread_pool/pool.cpp b/ydb/public/sdk/cpp/client/impl/ydb_internal/thread_pool/pool.cpp index 459573259b..7fbf33277f 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/thread_pool/pool.cpp +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/thread_pool/pool.cpp @@ -1,2 +1,2 @@ -#define INCLUDE_YDB_INTERNAL_H -#include "pool.h" +#define INCLUDE_YDB_INTERNAL_H +#include "pool.h" diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/thread_pool/pool.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/thread_pool/pool.h index 2480ce5a56..8917c83f25 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/thread_pool/pool.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/thread_pool/pool.h @@ -1,21 +1,21 @@ -#pragma once - +#pragma once + #include <ydb/public/sdk/cpp/client/impl/ydb_internal/internal_header.h> - -#include <util/thread/pool.h> - + +#include <util/thread/pool.h> + #include <memory> -namespace NYdb { - -inline std::unique_ptr<IThreadPool> CreateThreadPool(size_t threads) { - std::unique_ptr<IThreadPool> queue; - if (threads) { - queue.reset(new TThreadPool(TThreadPool::TParams().SetBlocking(true).SetCatching(false))); - } else { - queue.reset(new TAdaptiveThreadPool()); - } - return queue; -} - -} // namespace NYdb +namespace NYdb { + +inline std::unique_ptr<IThreadPool> CreateThreadPool(size_t threads) { + std::unique_ptr<IThreadPool> queue; + if (threads) { + queue.reset(new TThreadPool(TThreadPool::TParams().SetBlocking(true).SetCatching(false))); + } else { + queue.reset(new TAdaptiveThreadPool()); + } + return queue; +} + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/thread_pool/ya.make b/ydb/public/sdk/cpp/client/impl/ydb_internal/thread_pool/ya.make index fb23507c6f..cefbaddc23 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/thread_pool/ya.make +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/thread_pool/ya.make @@ -1,9 +1,9 @@ -LIBRARY() - +LIBRARY() + OWNER(g:kikimr) - -SRCS( - pool.cpp -) - -END() + +SRCS( + pool.cpp +) + +END() diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/value_helpers/helpers.cpp b/ydb/public/sdk/cpp/client/impl/ydb_internal/value_helpers/helpers.cpp index 4cb281383b..0048cd3ddf 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/value_helpers/helpers.cpp +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/value_helpers/helpers.cpp @@ -1,95 +1,95 @@ -#define INCLUDE_YDB_INTERNAL_H -#include "helpers.h" - -namespace NYdb { - -bool TypesEqual(const Ydb::Type& t1, const Ydb::Type& t2) { - if (t1.type_case() != t2.type_case()) { - return false; - } - - switch (t1.type_case()) { - case Ydb::Type::kTypeId: - return t1.type_id() == t2.type_id(); - case Ydb::Type::kDecimalType: - return t1.decimal_type().precision() == t2.decimal_type().precision() - && t1.decimal_type().scale() == t2.decimal_type().scale(); - case Ydb::Type::kOptionalType: - return TypesEqual(t1.optional_type().item(), t2.optional_type().item()); - case Ydb::Type::kListType: - return TypesEqual(t1.list_type().item(), t2.list_type().item()); - case Ydb::Type::kTupleType: - if (t1.tuple_type().elements_size() != t2.tuple_type().elements_size()) { - return false; - } - for (size_t i = 0; i < (size_t)t1.tuple_type().elements_size(); ++i) { - if (!TypesEqual(t1.tuple_type().elements(i), t2.tuple_type().elements(i))) { - return false; - } - } - return true; - case Ydb::Type::kStructType: - if (t1.struct_type().members_size() != t2.struct_type().members_size()) { - return false; - } - for (size_t i = 0; i < (size_t)t1.struct_type().members_size(); ++i) { - auto& m1 = t1.struct_type().members(i); - auto& m2 = t2.struct_type().members(i); - if (m1.name() != m2.name()) { - return false; - } - - if (!TypesEqual(m1.type(), m2.type())) { - return false; - } - } - return true; - case Ydb::Type::kDictType: - return TypesEqual(t1.dict_type().key(), t2.dict_type().key()) - && TypesEqual(t1.dict_type().payload(), t2.dict_type().payload()); - case Ydb::Type::kVariantType: { - const auto& v1 = t1.variant_type(); - const auto& v2 = t2.variant_type(); - if (v1.type_case() != v2.type_case()) { - return false; - } - switch (v1.type_case()) { - case Ydb::VariantType::kTupleItems: - if (v1.tuple_items().elements_size() != v2.tuple_items().elements_size()) { - return false; - } - for (size_t i = 0; i < (size_t)v1.tuple_items().elements_size(); ++i) { - if (!TypesEqual(v1.tuple_items().elements(i), v2.tuple_items().elements(i))) { - return false; - } - } - return true; - case Ydb::VariantType::kStructItems: - if (v1.struct_items().members_size() != v2.struct_items().members_size()) { - return false; - } - for (size_t i = 0; i < (size_t)v1.struct_items().members_size(); ++i) { - auto& m1 = v1.struct_items().members(i); - auto& m2 = v2.struct_items().members(i); - if (m1.name() != m2.name()) { - return false; - } - - if (!TypesEqual(m1.type(), m2.type())) { - return false; - } - } - return true; - default: - return false; - } - return false; - } - case Ydb::Type::kVoidType: - return true; - default: - return false; - } -} - -} // namespace NYdb +#define INCLUDE_YDB_INTERNAL_H +#include "helpers.h" + +namespace NYdb { + +bool TypesEqual(const Ydb::Type& t1, const Ydb::Type& t2) { + if (t1.type_case() != t2.type_case()) { + return false; + } + + switch (t1.type_case()) { + case Ydb::Type::kTypeId: + return t1.type_id() == t2.type_id(); + case Ydb::Type::kDecimalType: + return t1.decimal_type().precision() == t2.decimal_type().precision() + && t1.decimal_type().scale() == t2.decimal_type().scale(); + case Ydb::Type::kOptionalType: + return TypesEqual(t1.optional_type().item(), t2.optional_type().item()); + case Ydb::Type::kListType: + return TypesEqual(t1.list_type().item(), t2.list_type().item()); + case Ydb::Type::kTupleType: + if (t1.tuple_type().elements_size() != t2.tuple_type().elements_size()) { + return false; + } + for (size_t i = 0; i < (size_t)t1.tuple_type().elements_size(); ++i) { + if (!TypesEqual(t1.tuple_type().elements(i), t2.tuple_type().elements(i))) { + return false; + } + } + return true; + case Ydb::Type::kStructType: + if (t1.struct_type().members_size() != t2.struct_type().members_size()) { + return false; + } + for (size_t i = 0; i < (size_t)t1.struct_type().members_size(); ++i) { + auto& m1 = t1.struct_type().members(i); + auto& m2 = t2.struct_type().members(i); + if (m1.name() != m2.name()) { + return false; + } + + if (!TypesEqual(m1.type(), m2.type())) { + return false; + } + } + return true; + case Ydb::Type::kDictType: + return TypesEqual(t1.dict_type().key(), t2.dict_type().key()) + && TypesEqual(t1.dict_type().payload(), t2.dict_type().payload()); + case Ydb::Type::kVariantType: { + const auto& v1 = t1.variant_type(); + const auto& v2 = t2.variant_type(); + if (v1.type_case() != v2.type_case()) { + return false; + } + switch (v1.type_case()) { + case Ydb::VariantType::kTupleItems: + if (v1.tuple_items().elements_size() != v2.tuple_items().elements_size()) { + return false; + } + for (size_t i = 0; i < (size_t)v1.tuple_items().elements_size(); ++i) { + if (!TypesEqual(v1.tuple_items().elements(i), v2.tuple_items().elements(i))) { + return false; + } + } + return true; + case Ydb::VariantType::kStructItems: + if (v1.struct_items().members_size() != v2.struct_items().members_size()) { + return false; + } + for (size_t i = 0; i < (size_t)v1.struct_items().members_size(); ++i) { + auto& m1 = v1.struct_items().members(i); + auto& m2 = v2.struct_items().members(i); + if (m1.name() != m2.name()) { + return false; + } + + if (!TypesEqual(m1.type(), m2.type())) { + return false; + } + } + return true; + default: + return false; + } + return false; + } + case Ydb::Type::kVoidType: + return true; + default: + return false; + } +} + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/value_helpers/helpers.h b/ydb/public/sdk/cpp/client/impl/ydb_internal/value_helpers/helpers.h index 8a1ee45884..9b6fd6a275 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/value_helpers/helpers.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/value_helpers/helpers.h @@ -1,11 +1,11 @@ -#pragma once - +#pragma once + #include <ydb/public/sdk/cpp/client/impl/ydb_internal/internal_header.h> - + #include <ydb/public/api/protos/ydb_value.pb.h> - -namespace NYdb { - -bool TypesEqual(const Ydb::Type& t1, const Ydb::Type& t2); - -} // namespace NYdb + +namespace NYdb { + +bool TypesEqual(const Ydb::Type& t1, const Ydb::Type& t2); + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/value_helpers/ya.make b/ydb/public/sdk/cpp/client/impl/ydb_internal/value_helpers/ya.make index cdbf8d5039..a2054b8511 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/value_helpers/ya.make +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/value_helpers/ya.make @@ -1,13 +1,13 @@ -LIBRARY() - +LIBRARY() + OWNER(g:kikimr) - -SRCS( - helpers.cpp -) - -PEERDIR( + +SRCS( + helpers.cpp +) + +PEERDIR( ydb/public/api/protos -) - -END() +) + +END() diff --git a/ydb/public/sdk/cpp/client/impl/ydb_stats/stats.cpp b/ydb/public/sdk/cpp/client/impl/ydb_stats/stats.cpp index 4ef6f437b4..21a58b33cb 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_stats/stats.cpp +++ b/ydb/public/sdk/cpp/client/impl/ydb_stats/stats.cpp @@ -1,4 +1,4 @@ -#include "stats.h" +#include "stats.h" namespace NYdb { namespace NSdkStats { diff --git a/ydb/public/sdk/cpp/client/impl/ydb_stats/stats.h b/ydb/public/sdk/cpp/client/impl/ydb_stats/stats.h index 62c4033535..cae4099682 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_stats/stats.h +++ b/ydb/public/sdk/cpp/client/impl/ydb_stats/stats.h @@ -1,12 +1,12 @@ #pragma once - + #include <ydb/public/sdk/cpp/client/ydb_types/status_codes.h> #include <ydb/public/sdk/cpp/client/impl/ydb_internal/common/type_switcher.h> - + #include <library/cpp/grpc/client/grpc_client_low.h> #include <library/cpp/monlib/metrics/metric_registry.h> #include <library/cpp/monlib/metrics/histogram_collector.h> - + #include <util/string/builder.h> #include <atomic> diff --git a/ydb/public/sdk/cpp/client/impl/ydb_stats/ya.make b/ydb/public/sdk/cpp/client/impl/ydb_stats/ya.make index a805a72700..06e178b8c7 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_stats/ya.make +++ b/ydb/public/sdk/cpp/client/impl/ydb_stats/ya.make @@ -1,17 +1,17 @@ -LIBRARY() - -OWNER( - dcherednik - g:kikimr -) - -SRCS( - stats.cpp -) - -PEERDIR( - library/cpp/grpc/client - library/cpp/monlib/metrics -) - -END() +LIBRARY() + +OWNER( + dcherednik + g:kikimr +) + +SRCS( + stats.cpp +) + +PEERDIR( + library/cpp/grpc/client + library/cpp/monlib/metrics +) + +END() diff --git a/ydb/public/sdk/cpp/client/ydb_common_client/impl/client.cpp b/ydb/public/sdk/cpp/client/ydb_common_client/impl/client.cpp index 18c4c8c3ef..f679c0d5ec 100644 --- a/ydb/public/sdk/cpp/client/ydb_common_client/impl/client.cpp +++ b/ydb/public/sdk/cpp/client/ydb_common_client/impl/client.cpp @@ -1 +1 @@ -#include "client.h" +#include "client.h" diff --git a/ydb/public/sdk/cpp/client/ydb_common_client/impl/client.h b/ydb/public/sdk/cpp/client/ydb_common_client/impl/client.h index 0689504f1c..c43e9aab69 100644 --- a/ydb/public/sdk/cpp/client/ydb_common_client/impl/client.h +++ b/ydb/public/sdk/cpp/client/ydb_common_client/impl/client.h @@ -5,9 +5,9 @@ #undef INCLUDE_YDB_INTERNAL_H #include <ydb/public/sdk/cpp/client/ydb_types/exceptions/exceptions.h> - -#include <memory> - + +#include <memory> + namespace NYdb { template<typename T> diff --git a/ydb/public/sdk/cpp/client/ydb_common_client/impl/ya.make b/ydb/public/sdk/cpp/client/ydb_common_client/impl/ya.make index 9b7651f575..1908d735a0 100644 --- a/ydb/public/sdk/cpp/client/ydb_common_client/impl/ya.make +++ b/ydb/public/sdk/cpp/client/ydb_common_client/impl/ya.make @@ -1,16 +1,16 @@ -LIBRARY() - -OWNER( - dcherednik - g:kikimr -) - -SRCS( - client.cpp -) - -PEERDIR( +LIBRARY() + +OWNER( + dcherednik + g:kikimr +) + +SRCS( + client.cpp +) + +PEERDIR( ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections -) - -END() +) + +END() diff --git a/ydb/public/sdk/cpp/client/ydb_common_client/settings.cpp b/ydb/public/sdk/cpp/client/ydb_common_client/settings.cpp index 37eac01bad..8bffcd7a5a 100644 --- a/ydb/public/sdk/cpp/client/ydb_common_client/settings.cpp +++ b/ydb/public/sdk/cpp/client/ydb_common_client/settings.cpp @@ -1,13 +1,13 @@ -#include "settings.h" - +#include "settings.h" + #include <ydb/public/sdk/cpp/client/impl/ydb_internal/common/parser.h> -namespace NYdb { - +namespace NYdb { + TCommonClientSettings& TCommonClientSettings::AuthToken(const TMaybe<TStringType>& token) { - return CredentialsProviderFactory(CreateOAuthCredentialsProviderFactory(token.GetRef())); -} - + return CredentialsProviderFactory(CreateOAuthCredentialsProviderFactory(token.GetRef())); +} + TCommonClientSettings GetClientSettingsFromConnectionString(const TStringType& connectionString) { TCommonClientSettings settings; auto connectionInfo = ParseConnectionString(connectionString); @@ -17,4 +17,4 @@ TCommonClientSettings GetClientSettingsFromConnectionString(const TStringType& c return settings; } -} // namespace NYdb +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/ydb_common_client/settings.h b/ydb/public/sdk/cpp/client/ydb_common_client/settings.h index a0b234c524..4ada7c09d0 100644 --- a/ydb/public/sdk/cpp/client/ydb_common_client/settings.h +++ b/ydb/public/sdk/cpp/client/ydb_common_client/settings.h @@ -1,38 +1,38 @@ -#pragma once - +#pragma once + #include <ydb/public/sdk/cpp/client/ydb_types/credentials/credentials.h> #include <ydb/public/sdk/cpp/client/ydb_types/fluent_settings_helpers.h> #include <ydb/public/sdk/cpp/client/ydb_types/ydb.h> - + #include <ydb/public/sdk/cpp/client/impl/ydb_internal/common/type_switcher.h> -#include <functional> - -namespace NYdb { - -struct TCommonClientSettings { - using TSelf = TCommonClientSettings; - - //! NOTE: Options below are discovery state specific options. - //! If client overrides it and no state in cache (no client with same settings) - //! ctor may blocks during discovery (according to DiscoveryMode settings - //! The key of state cache is <Database, Credentials, DiscoveryEndpoint, DiscoveryMode> tuple - //! For performance reason (to avoid extra discovery requests) it is - //! always better to keep instance of client. - - //! Allows to override current database for client +#include <functional> + +namespace NYdb { + +struct TCommonClientSettings { + using TSelf = TCommonClientSettings; + + //! NOTE: Options below are discovery state specific options. + //! If client overrides it and no state in cache (no client with same settings) + //! ctor may blocks during discovery (according to DiscoveryMode settings + //! The key of state cache is <Database, Credentials, DiscoveryEndpoint, DiscoveryMode> tuple + //! For performance reason (to avoid extra discovery requests) it is + //! always better to keep instance of client. + + //! Allows to override current database for client FLUENT_SETTING_OPTIONAL(TStringType, Database); - //! Allows to override current discovery endpoint + //! Allows to override current discovery endpoint FLUENT_SETTING_OPTIONAL(TStringType, DiscoveryEndpoint); - //! Allows to override current token for client + //! Allows to override current token for client TSelf& AuthToken(const TMaybe<TStringType>& token); - //! Allows to override current credentials provider - FLUENT_SETTING_OPTIONAL(std::shared_ptr<ICredentialsProviderFactory>, CredentialsProviderFactory); - //! Allows to override discovery mode - FLUENT_SETTING_OPTIONAL(EDiscoveryMode, DiscoveryMode); + //! Allows to override current credentials provider + FLUENT_SETTING_OPTIONAL(std::shared_ptr<ICredentialsProviderFactory>, CredentialsProviderFactory); + //! Allows to override discovery mode + FLUENT_SETTING_OPTIONAL(EDiscoveryMode, DiscoveryMode); //! Allows to override current Ssl mode FLUENT_SETTING_OPTIONAL(bool, EnableSsl); -}; - +}; + template<class TDerived> struct TCommonClientSettingsBase : public TCommonClientSettings { @@ -55,4 +55,4 @@ struct TCommonClientSettingsBase : public TCommonClientSettings { TCommonClientSettings GetClientSettingsFromConnectionString(const TStringType& connectionString); -} // namespace NYdb +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/ydb_common_client/ya.make b/ydb/public/sdk/cpp/client/ydb_common_client/ya.make index b0f4d9275b..9667fbd3ec 100644 --- a/ydb/public/sdk/cpp/client/ydb_common_client/ya.make +++ b/ydb/public/sdk/cpp/client/ydb_common_client/ya.make @@ -1,17 +1,17 @@ -LIBRARY() - -OWNER( - dcherednik - g:kikimr -) - -SRCS( - settings.cpp -) - -PEERDIR( +LIBRARY() + +OWNER( + dcherednik + g:kikimr +) + +SRCS( + settings.cpp +) + +PEERDIR( ydb/public/sdk/cpp/client/impl/ydb_internal/common ydb/public/sdk/cpp/client/ydb_types/credentials -) - -END() +) + +END() diff --git a/ydb/public/sdk/cpp/client/ydb_coordination/coordination.cpp b/ydb/public/sdk/cpp/client/ydb_coordination/coordination.cpp index 402652d605..7a99c95214 100644 --- a/ydb/public/sdk/cpp/client/ydb_coordination/coordination.cpp +++ b/ydb/public/sdk/cpp/client/ydb_coordination/coordination.cpp @@ -1,10 +1,10 @@ -#include "coordination.h" - -#define INCLUDE_YDB_INTERNAL_H +#include "coordination.h" + +#define INCLUDE_YDB_INTERNAL_H #include <ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.h> #include <ydb/public/sdk/cpp/client/impl/ydb_internal/table_helpers/helpers.h> -#undef INCLUDE_YDB_INTERNAL_H - +#undef INCLUDE_YDB_INTERNAL_H + #include <ydb/public/api/grpc/ydb_coordination_v1.grpc.pb.h> #include <ydb/public/sdk/cpp/client/ydb_common_client/impl/client.h> diff --git a/ydb/public/sdk/cpp/client/ydb_coordination/coordination.h b/ydb/public/sdk/cpp/client/ydb_coordination/coordination.h index a7b9b48a8d..fa8ab572d0 100644 --- a/ydb/public/sdk/cpp/client/ydb_coordination/coordination.h +++ b/ydb/public/sdk/cpp/client/ydb_coordination/coordination.h @@ -13,11 +13,11 @@ namespace Coordination { } namespace NYdb { - -namespace NScheme { -struct TPermissions; -} - + +namespace NScheme { +struct TPermissions; +} + namespace NCoordination { //! Represents result of a call with status diff --git a/ydb/public/sdk/cpp/client/ydb_coordination/ut/ya.make b/ydb/public/sdk/cpp/client/ydb_coordination/ut/ya.make index 2617467d1a..23e9ae6a1b 100644 --- a/ydb/public/sdk/cpp/client/ydb_coordination/ut/ya.make +++ b/ydb/public/sdk/cpp/client/ydb_coordination/ut/ya.make @@ -1,27 +1,27 @@ UNITTEST_FOR(ydb/public/sdk/cpp/client/ydb_coordination) - -OWNER( - dcherednik - g:kikimr -) - -IF (SANITIZER_TYPE) - TIMEOUT(1200) - SIZE(LARGE) - TAG(ya:fat) -ELSE() - TIMEOUT(600) - SIZE(MEDIUM) -ENDIF() - -FORK_SUBTESTS() - -PEERDIR( + +OWNER( + dcherednik + g:kikimr +) + +IF (SANITIZER_TYPE) + TIMEOUT(1200) + SIZE(LARGE) + TAG(ya:fat) +ELSE() + TIMEOUT(600) + SIZE(MEDIUM) +ENDIF() + +FORK_SUBTESTS() + +PEERDIR( ydb/public/api/grpc -) - -SRCS( - coordination_ut.cpp -) - -END() +) + +SRCS( + coordination_ut.cpp +) + +END() diff --git a/ydb/public/sdk/cpp/client/ydb_coordination/ya.make b/ydb/public/sdk/cpp/client/ydb_coordination/ya.make index c499d066ff..ac7b926a07 100644 --- a/ydb/public/sdk/cpp/client/ydb_coordination/ya.make +++ b/ydb/public/sdk/cpp/client/ydb_coordination/ya.make @@ -1,17 +1,17 @@ -LIBRARY() - -OWNER( - dcherednik - g:kikimr -) - -SRCS( - coordination.cpp -) - -GENERATE_ENUM_SERIALIZATION(coordination.h) - -PEERDIR( +LIBRARY() + +OWNER( + dcherednik + g:kikimr +) + +SRCS( + coordination.cpp +) + +GENERATE_ENUM_SERIALIZATION(coordination.h) + +PEERDIR( ydb/public/api/grpc ydb/public/sdk/cpp/client/impl/ydb_internal/make_request ydb/public/sdk/cpp/client/ydb_common_client @@ -19,10 +19,10 @@ PEERDIR( ydb/public/sdk/cpp/client/ydb_driver ydb/public/sdk/cpp/client/ydb_types ydb/public/sdk/cpp/client/ydb_types/status -) - -END() - +) + +END() + RECURSE_FOR_TESTS( ut ) diff --git a/ydb/public/sdk/cpp/client/ydb_datastreams/ya.make b/ydb/public/sdk/cpp/client/ydb_datastreams/ya.make index f79749e18e..3e94f96757 100644 --- a/ydb/public/sdk/cpp/client/ydb_datastreams/ya.make +++ b/ydb/public/sdk/cpp/client/ydb_datastreams/ya.make @@ -1,18 +1,18 @@ -LIBRARY() - -OWNER(g:kikimr) - -SRCS( +LIBRARY() + +OWNER(g:kikimr) + +SRCS( datastreams.cpp -) - -PEERDIR( - library/cpp/grpc/client - library/cpp/string_utils/url +) + +PEERDIR( + library/cpp/grpc/client + library/cpp/string_utils/url ydb/public/api/grpc/draft ydb/public/lib/operation_id ydb/public/sdk/cpp/client/impl/ydb_internal/make_request ydb/public/sdk/cpp/client/ydb_driver -) - -END() +) + +END() diff --git a/ydb/public/sdk/cpp/client/ydb_discovery/discovery.cpp b/ydb/public/sdk/cpp/client/ydb_discovery/discovery.cpp index ba4de61513..6957a558ec 100644 --- a/ydb/public/sdk/cpp/client/ydb_discovery/discovery.cpp +++ b/ydb/public/sdk/cpp/client/ydb_discovery/discovery.cpp @@ -1,7 +1,7 @@ -#include "discovery.h" +#include "discovery.h" #include <ydb/public/sdk/cpp/client/ydb_common_client/impl/client.h> - + namespace NYdb { namespace NDiscovery { diff --git a/ydb/public/sdk/cpp/client/ydb_discovery/discovery.h b/ydb/public/sdk/cpp/client/ydb_discovery/discovery.h index 8162884a98..d81275b7f6 100644 --- a/ydb/public/sdk/cpp/client/ydb_discovery/discovery.h +++ b/ydb/public/sdk/cpp/client/ydb_discovery/discovery.h @@ -5,7 +5,7 @@ namespace Ydb { namespace Discovery { class ListEndpointsResult; - class WhoAmIResult; + class WhoAmIResult; } // namespace Discovery } // namespace Ydb diff --git a/ydb/public/sdk/cpp/client/ydb_discovery/ya.make b/ydb/public/sdk/cpp/client/ydb_discovery/ya.make index 7ca2f01b71..2cf49a926e 100644 --- a/ydb/public/sdk/cpp/client/ydb_discovery/ya.make +++ b/ydb/public/sdk/cpp/client/ydb_discovery/ya.make @@ -1,17 +1,17 @@ -LIBRARY() - -OWNER( - dcherednik - g:kikimr -) - -SRCS( - discovery.cpp -) - -PEERDIR( +LIBRARY() + +OWNER( + dcherednik + g:kikimr +) + +SRCS( + discovery.cpp +) + +PEERDIR( ydb/public/sdk/cpp/client/ydb_common_client/impl ydb/public/sdk/cpp/client/ydb_driver -) - -END() +) + +END() diff --git a/ydb/public/sdk/cpp/client/ydb_driver/driver.cpp b/ydb/public/sdk/cpp/client/ydb_driver/driver.cpp index 601ef3fbf7..de17c6e679 100644 --- a/ydb/public/sdk/cpp/client/ydb_driver/driver.cpp +++ b/ydb/public/sdk/cpp/client/ydb_driver/driver.cpp @@ -1,5 +1,5 @@ -#include "driver.h" - +#include "driver.h" + #define INCLUDE_YDB_INTERNAL_H #include <ydb/public/sdk/cpp/client/impl/ydb_internal/driver/constants.h> #include <ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/grpc_connections.h> diff --git a/ydb/public/sdk/cpp/client/ydb_driver/driver_ut.cpp b/ydb/public/sdk/cpp/client/ydb_driver/driver_ut.cpp index 7c67478c15..08172ee80a 100644 --- a/ydb/public/sdk/cpp/client/ydb_driver/driver_ut.cpp +++ b/ydb/public/sdk/cpp/client/ydb_driver/driver_ut.cpp @@ -1,6 +1,6 @@ #include <ydb/public/sdk/cpp/client/ydb_table/table.h> #include <ydb/public/sdk/cpp/client/ydb_types/exceptions/exceptions.h> - + #include <ydb/public/api/grpc/ydb_discovery_v1.grpc.pb.h> #include <ydb/public/api/grpc/ydb_table_v1.grpc.pb.h> @@ -8,16 +8,16 @@ #include <grpcpp/server_builder.h> #include <grpcpp/server_context.h> -#include <library/cpp/testing/unittest/registar.h> -#include <library/cpp/testing/unittest/tests_data.h> - +#include <library/cpp/testing/unittest/registar.h> +#include <library/cpp/testing/unittest/tests_data.h> + #include <atomic> - -#include <google/protobuf/text_format.h> - -using namespace NYdb; -using namespace NYdb::NTable; - + +#include <google/protobuf/text_format.h> + +using namespace NYdb; +using namespace NYdb::NTable; + namespace { class TMockDiscoveryService : public Ydb::Discovery::V1::DiscoveryService::Service { @@ -77,72 +77,72 @@ namespace { } // namespace -Y_UNIT_TEST_SUITE(CppGrpcClientSimpleTest) { - Y_UNIT_TEST(ConnectWrongPort) { - auto driver = TDriver( - TDriverConfig() - .SetEndpoint("localhost:100")); - auto client = NTable::TTableClient(driver); - auto sessionFuture = client.CreateSession(); - - UNIT_ASSERT(sessionFuture.Wait(TDuration::Seconds(10))); - } - - Y_UNIT_TEST(ConnectWrongPortRetry) { - auto driver = TDriver( - TDriverConfig() - .SetEndpoint("localhost:100")); - auto client = NTable::TTableClient(driver); - +Y_UNIT_TEST_SUITE(CppGrpcClientSimpleTest) { + Y_UNIT_TEST(ConnectWrongPort) { + auto driver = TDriver( + TDriverConfig() + .SetEndpoint("localhost:100")); + auto client = NTable::TTableClient(driver); + auto sessionFuture = client.CreateSession(); + + UNIT_ASSERT(sessionFuture.Wait(TDuration::Seconds(10))); + } + + Y_UNIT_TEST(ConnectWrongPortRetry) { + auto driver = TDriver( + TDriverConfig() + .SetEndpoint("localhost:100")); + auto client = NTable::TTableClient(driver); + std::atomic_int counter = 0; - std::function<void(const NTable::TAsyncCreateSessionResult& future)> handler = - [&handler, &counter, client] (const NTable::TAsyncCreateSessionResult& future) mutable { - UNIT_ASSERT_EQUAL(future.GetValue().GetStatus(), EStatus::TRANSPORT_UNAVAILABLE); - UNIT_ASSERT_EXCEPTION(future.GetValue().GetSession(), NYdb::TContractViolation); + std::function<void(const NTable::TAsyncCreateSessionResult& future)> handler = + [&handler, &counter, client] (const NTable::TAsyncCreateSessionResult& future) mutable { + UNIT_ASSERT_EQUAL(future.GetValue().GetStatus(), EStatus::TRANSPORT_UNAVAILABLE); + UNIT_ASSERT_EXCEPTION(future.GetValue().GetSession(), NYdb::TContractViolation); ++counter; if (counter.load() > 4) { - return; - } - - auto f = client.CreateSession(); - - f.Apply(handler).GetValueSync(); - }; - - client.CreateSession().Apply(handler).GetValueSync(); - UNIT_ASSERT_EQUAL(counter, 5); - } - - Y_UNIT_TEST(TokenCharacters) { - auto checkToken = [](const TString& token) { - auto driver = TDriver( - TDriverConfig() - .SetEndpoint("localhost:100") - .SetAuthToken(token)); - auto client = NTable::TTableClient(driver); - - auto result = client.CreateSession().GetValueSync(); - - return result.GetStatus(); - }; - - TVector<TString> InvalidTokens = { - TString('\t'), - TString('\n'), - TString('\r') - }; - for (auto& t : InvalidTokens) { - UNIT_ASSERT_EQUAL(checkToken(t), EStatus::CLIENT_UNAUTHENTICATED); - } - - TVector<TString> ValidTokens = { - TString("qwerty 1234 <>,.?/:;\"'\\|}{~`!@#$%^&*()_+=-"), - TString() - }; - for (auto& t : ValidTokens) { - UNIT_ASSERT_EQUAL(checkToken(t), EStatus::TRANSPORT_UNAVAILABLE); - } - } + return; + } + + auto f = client.CreateSession(); + + f.Apply(handler).GetValueSync(); + }; + + client.CreateSession().Apply(handler).GetValueSync(); + UNIT_ASSERT_EQUAL(counter, 5); + } + + Y_UNIT_TEST(TokenCharacters) { + auto checkToken = [](const TString& token) { + auto driver = TDriver( + TDriverConfig() + .SetEndpoint("localhost:100") + .SetAuthToken(token)); + auto client = NTable::TTableClient(driver); + + auto result = client.CreateSession().GetValueSync(); + + return result.GetStatus(); + }; + + TVector<TString> InvalidTokens = { + TString('\t'), + TString('\n'), + TString('\r') + }; + for (auto& t : InvalidTokens) { + UNIT_ASSERT_EQUAL(checkToken(t), EStatus::CLIENT_UNAUTHENTICATED); + } + + TVector<TString> ValidTokens = { + TString("qwerty 1234 <>,.?/:;\"'\\|}{~`!@#$%^&*()_+=-"), + TString() + }; + for (auto& t : ValidTokens) { + UNIT_ASSERT_EQUAL(checkToken(t), EStatus::TRANSPORT_UNAVAILABLE); + } + } Y_UNIT_TEST(UsingIpAddresses) { TPortManager pm; @@ -181,4 +181,4 @@ Y_UNIT_TEST_SUITE(CppGrpcClientSimpleTest) { auto session = sessionResult.GetSession(); UNIT_ASSERT_VALUES_EQUAL(session.GetId(), "my-session-id"); } -} +} diff --git a/ydb/public/sdk/cpp/client/ydb_driver/ut/ya.make b/ydb/public/sdk/cpp/client/ydb_driver/ut/ya.make index ef65759572..0cdbb9d72b 100644 --- a/ydb/public/sdk/cpp/client/ydb_driver/ut/ya.make +++ b/ydb/public/sdk/cpp/client/ydb_driver/ut/ya.make @@ -1,27 +1,27 @@ UNITTEST_FOR(ydb/public/sdk/cpp/client/ydb_driver) - -OWNER( - dcherednik - g:kikimr -) - -IF (SANITIZER_TYPE) - TIMEOUT(1200) - SIZE(LARGE) - TAG(ya:fat) -ELSE() - TIMEOUT(600) - SIZE(MEDIUM) -ENDIF() - -FORK_SUBTESTS() - -PEERDIR( + +OWNER( + dcherednik + g:kikimr +) + +IF (SANITIZER_TYPE) + TIMEOUT(1200) + SIZE(LARGE) + TAG(ya:fat) +ELSE() + TIMEOUT(600) + SIZE(MEDIUM) +ENDIF() + +FORK_SUBTESTS() + +PEERDIR( ydb/public/sdk/cpp/client/ydb_table -) - -SRCS( - driver_ut.cpp -) - -END() +) + +SRCS( + driver_ut.cpp +) + +END() diff --git a/ydb/public/sdk/cpp/client/ydb_driver/ya.make b/ydb/public/sdk/cpp/client/ydb_driver/ya.make index e3338852eb..ff187205bb 100644 --- a/ydb/public/sdk/cpp/client/ydb_driver/ya.make +++ b/ydb/public/sdk/cpp/client/ydb_driver/ya.make @@ -1,24 +1,24 @@ -LIBRARY() - -OWNER( - dcherednik - g:kikimr -) - -SRCS( - driver.cpp -) - -PEERDIR( +LIBRARY() + +OWNER( + dcherednik + g:kikimr +) + +SRCS( + driver.cpp +) + +PEERDIR( ydb/public/sdk/cpp/client/impl/ydb_internal/common ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections ydb/public/sdk/cpp/client/resources ydb/public/sdk/cpp/client/ydb_common_client ydb/public/sdk/cpp/client/ydb_types/status -) - -END() - +) + +END() + RECURSE_FOR_TESTS( ut ) diff --git a/ydb/public/sdk/cpp/client/ydb_export/export.cpp b/ydb/public/sdk/cpp/client/ydb_export/export.cpp index b967b6467a..c391c033e1 100644 --- a/ydb/public/sdk/cpp/client/ydb_export/export.cpp +++ b/ydb/public/sdk/cpp/client/ydb_export/export.cpp @@ -1,18 +1,18 @@ -#include "export.h" +#include "export.h" -#define INCLUDE_YDB_INTERNAL_H +#define INCLUDE_YDB_INTERNAL_H #include <ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.h> -#undef INCLUDE_YDB_INTERNAL_H - +#undef INCLUDE_YDB_INTERNAL_H + #include <ydb/public/api/grpc/ydb_discovery_v1.grpc.pb.h> #include <ydb/public/api/grpc/ydb_export_v1.grpc.pb.h> #include <ydb/public/api/protos/ydb_export.pb.h> #include <ydb/public/sdk/cpp/client/ydb_common_client/impl/client.h> #include <ydb/public/sdk/cpp/client/ydb_proto/accessor.h> - -#include <google/protobuf/repeated_field.h> -#include <google/protobuf/timestamp.pb.h> - + +#include <google/protobuf/repeated_field.h> +#include <google/protobuf/timestamp.pb.h> + namespace NYdb { namespace NExport { diff --git a/ydb/public/sdk/cpp/client/ydb_export/ya.make b/ydb/public/sdk/cpp/client/ydb_export/ya.make index 9a084f23b4..67680a6f9d 100644 --- a/ydb/public/sdk/cpp/client/ydb_export/ya.make +++ b/ydb/public/sdk/cpp/client/ydb_export/ya.make @@ -1,23 +1,23 @@ -LIBRARY() - -OWNER( - dcherednik - g:kikimr -) - -SRCS( - export.cpp -) - -GENERATE_ENUM_SERIALIZATION(export.h) - -PEERDIR( +LIBRARY() + +OWNER( + dcherednik + g:kikimr +) + +SRCS( + export.cpp +) + +GENERATE_ENUM_SERIALIZATION(export.h) + +PEERDIR( ydb/public/api/grpc ydb/public/api/protos ydb/public/sdk/cpp/client/ydb_common_client/impl ydb/public/sdk/cpp/client/ydb_driver ydb/public/sdk/cpp/client/ydb_proto ydb/public/sdk/cpp/client/ydb_types/operation -) - -END() +) + +END() diff --git a/ydb/public/sdk/cpp/client/ydb_extension/extension.cpp b/ydb/public/sdk/cpp/client/ydb_extension/extension.cpp index f03aacc007..4cb966a5ce 100644 --- a/ydb/public/sdk/cpp/client/ydb_extension/extension.cpp +++ b/ydb/public/sdk/cpp/client/ydb_extension/extension.cpp @@ -1,4 +1,4 @@ -#include "extension.h" +#include "extension.h" #define INCLUDE_YDB_INTERNAL_H #include <ydb/public/sdk/cpp/client/impl/ydb_internal/grpc_connections/grpc_connections.h> diff --git a/ydb/public/sdk/cpp/client/ydb_extension/extension.h b/ydb/public/sdk/cpp/client/ydb_extension/extension.h index 8551322710..3371bffddc 100644 --- a/ydb/public/sdk/cpp/client/ydb_extension/extension.h +++ b/ydb/public/sdk/cpp/client/ydb_extension/extension.h @@ -1,7 +1,7 @@ #pragma once #include <ydb/public/sdk/cpp/client/ydb_driver/driver.h> - + #include <library/cpp/monlib/metrics/metric_registry.h> namespace Ydb { @@ -14,23 +14,23 @@ class ListEndpointsResult; namespace NYdb { -class IExtensionApi { -public: - friend class TDriver; -public: - virtual ~IExtensionApi() = default; -private: - void SelfRegister(TDriver driver); -}; - -class IExtension { - friend class TDriver; -public: - virtual ~IExtension() = default; -private: - void SelfRegister(TDriver driver); -}; - +class IExtensionApi { +public: + friend class TDriver; +public: + virtual ~IExtensionApi() = default; +private: + void SelfRegister(TDriver driver); +}; + +class IExtension { + friend class TDriver; +public: + virtual ~IExtension() = default; +private: + void SelfRegister(TDriver driver); +}; + namespace NSdkStats { class IStatApi: public IExtensionApi { diff --git a/ydb/public/sdk/cpp/client/ydb_extension/ya.make b/ydb/public/sdk/cpp/client/ydb_extension/ya.make index 842bb6c8d2..31d367b6bf 100644 --- a/ydb/public/sdk/cpp/client/ydb_extension/ya.make +++ b/ydb/public/sdk/cpp/client/ydb_extension/ya.make @@ -1,17 +1,17 @@ -LIBRARY() - -OWNER( - dcherednik - g:kikimr -) - -SRCS( - extension.cpp -) - -PEERDIR( - library/cpp/monlib/metrics +LIBRARY() + +OWNER( + dcherednik + g:kikimr +) + +SRCS( + extension.cpp +) + +PEERDIR( + library/cpp/monlib/metrics ydb/public/sdk/cpp/client/ydb_driver -) - -END() +) + +END() diff --git a/ydb/public/sdk/cpp/client/ydb_import/import.cpp b/ydb/public/sdk/cpp/client/ydb_import/import.cpp index 7e2829486f..f0a55678ef 100644 --- a/ydb/public/sdk/cpp/client/ydb_import/import.cpp +++ b/ydb/public/sdk/cpp/client/ydb_import/import.cpp @@ -1,9 +1,9 @@ -#include "import.h" +#include "import.h" -#define INCLUDE_YDB_INTERNAL_H +#define INCLUDE_YDB_INTERNAL_H #include <ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.h> -#undef INCLUDE_YDB_INTERNAL_H - +#undef INCLUDE_YDB_INTERNAL_H + #include <ydb/public/api/grpc/ydb_discovery_v1.grpc.pb.h> #include <ydb/public/api/grpc/ydb_import_v1.grpc.pb.h> #include <ydb/public/api/protos/ydb_import.pb.h> diff --git a/ydb/public/sdk/cpp/client/ydb_import/ya.make b/ydb/public/sdk/cpp/client/ydb_import/ya.make index 943f8275b9..8e1ac730c8 100644 --- a/ydb/public/sdk/cpp/client/ydb_import/ya.make +++ b/ydb/public/sdk/cpp/client/ydb_import/ya.make @@ -1,23 +1,23 @@ -LIBRARY() - -OWNER( - dcherednik - g:kikimr -) - -SRCS( - import.cpp -) - -GENERATE_ENUM_SERIALIZATION(import.h) - -PEERDIR( +LIBRARY() + +OWNER( + dcherednik + g:kikimr +) + +SRCS( + import.cpp +) + +GENERATE_ENUM_SERIALIZATION(import.h) + +PEERDIR( ydb/public/api/grpc ydb/public/api/protos ydb/public/sdk/cpp/client/ydb_common_client/impl ydb/public/sdk/cpp/client/ydb_driver ydb/public/sdk/cpp/client/ydb_proto ydb/public/sdk/cpp/client/ydb_types/operation -) - -END() +) + +END() diff --git a/ydb/public/sdk/cpp/client/ydb_operation/operation.cpp b/ydb/public/sdk/cpp/client/ydb_operation/operation.cpp index 64d78d13d6..fac441c505 100644 --- a/ydb/public/sdk/cpp/client/ydb_operation/operation.cpp +++ b/ydb/public/sdk/cpp/client/ydb_operation/operation.cpp @@ -1,9 +1,9 @@ -#include "operation.h" +#include "operation.h" -#define INCLUDE_YDB_INTERNAL_H +#define INCLUDE_YDB_INTERNAL_H #include <ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.h> -#undef INCLUDE_YDB_INTERNAL_H - +#undef INCLUDE_YDB_INTERNAL_H + /* Headers below used to instantiate concrete 'Get' & 'List' methods */ #include <ydb/public/sdk/cpp/client/ydb_export/export.h> #include <ydb/public/sdk/cpp/client/ydb_import/import.h> diff --git a/ydb/public/sdk/cpp/client/ydb_operation/ya.make b/ydb/public/sdk/cpp/client/ydb_operation/ya.make index b33455bdf9..b07cd710ab 100644 --- a/ydb/public/sdk/cpp/client/ydb_operation/ya.make +++ b/ydb/public/sdk/cpp/client/ydb_operation/ya.make @@ -1,15 +1,15 @@ -LIBRARY() - -OWNER( - dcherednik - g:kikimr -) - -SRCS( - operation.cpp -) - -PEERDIR( +LIBRARY() + +OWNER( + dcherednik + g:kikimr +) + +SRCS( + operation.cpp +) + +PEERDIR( ydb/public/api/grpc ydb/public/lib/operation_id ydb/public/sdk/cpp/client/ydb_common_client/impl @@ -17,6 +17,6 @@ PEERDIR( ydb/public/sdk/cpp/client/ydb_export ydb/public/sdk/cpp/client/ydb_import ydb/public/sdk/cpp/client/ydb_types/operation -) - -END() +) + +END() diff --git a/ydb/public/sdk/cpp/client/ydb_params/impl.cpp b/ydb/public/sdk/cpp/client/ydb_params/impl.cpp index b1bef036f5..9e8c5e1e7d 100644 --- a/ydb/public/sdk/cpp/client/ydb_params/impl.cpp +++ b/ydb/public/sdk/cpp/client/ydb_params/impl.cpp @@ -1,5 +1,5 @@ -#include "impl.h" - +#include "impl.h" + #include <ydb/public/api/protos/ydb_scheme.pb.h> #include <ydb/public/api/protos/ydb_value.pb.h> @@ -42,7 +42,7 @@ TMaybe<TValue> TParams::TImpl::GetValue(const TString& name) const { return &ParamsMap_; } -const ::google::protobuf::Map<TString, Ydb::TypedValue>& TParams::TImpl::GetProtoMap() const { +const ::google::protobuf::Map<TString, Ydb::TypedValue>& TParams::TImpl::GetProtoMap() const { return ParamsMap_; } diff --git a/ydb/public/sdk/cpp/client/ydb_params/impl.h b/ydb/public/sdk/cpp/client/ydb_params/impl.h index 8c58fb7d72..9e38d2463d 100644 --- a/ydb/public/sdk/cpp/client/ydb_params/impl.h +++ b/ydb/public/sdk/cpp/client/ydb_params/impl.h @@ -1,6 +1,6 @@ #pragma once -#include "params.h" +#include "params.h" namespace NYdb { @@ -12,7 +12,7 @@ public: TMap<TString, TValue> GetValues() const; TMaybe<TValue> GetValue(const TString& name) const; ::google::protobuf::Map<TString, Ydb::TypedValue>* GetProtoMapPtr(); - const ::google::protobuf::Map<TString, Ydb::TypedValue>& GetProtoMap() const; + const ::google::protobuf::Map<TString, Ydb::TypedValue>& GetProtoMap() const; private: ::google::protobuf::Map<TString, Ydb::TypedValue> ParamsMap_; diff --git a/ydb/public/sdk/cpp/client/ydb_params/params.cpp b/ydb/public/sdk/cpp/client/ydb_params/params.cpp index 91b03f6b50..a69d29b2f6 100644 --- a/ydb/public/sdk/cpp/client/ydb_params/params.cpp +++ b/ydb/public/sdk/cpp/client/ydb_params/params.cpp @@ -1,5 +1,5 @@ -#include "params.h" -#include "impl.h" +#include "params.h" +#include "impl.h" #include <ydb/public/sdk/cpp/client/ydb_proto/accessor.h> #include <ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers/handlers.h> @@ -14,14 +14,14 @@ namespace NYdb { TParams::TParams(::google::protobuf::Map<TString, Ydb::TypedValue>&& protoMap) : Impl_(new TImpl(std::move(protoMap))) {} -::google::protobuf::Map<TString, Ydb::TypedValue>* TParams::GetProtoMapPtr() { - return Impl_->GetProtoMapPtr(); -} - -const ::google::protobuf::Map<TString, Ydb::TypedValue>& TParams::GetProtoMap() const { - return Impl_->GetProtoMap(); -} - +::google::protobuf::Map<TString, Ydb::TypedValue>* TParams::GetProtoMapPtr() { + return Impl_->GetProtoMapPtr(); +} + +const ::google::protobuf::Map<TString, Ydb::TypedValue>& TParams::GetProtoMap() const { + return Impl_->GetProtoMap(); +} + bool TParams::Empty() const { return Impl_->Empty(); } diff --git a/ydb/public/sdk/cpp/client/ydb_params/params.h b/ydb/public/sdk/cpp/client/ydb_params/params.h index c9ccb8071f..0f85f18329 100644 --- a/ydb/public/sdk/cpp/client/ydb_params/params.h +++ b/ydb/public/sdk/cpp/client/ydb_params/params.h @@ -2,8 +2,8 @@ #include <ydb/public/sdk/cpp/client/ydb_value/value.h> -#include <google/protobuf/map.h> - +#include <google/protobuf/map.h> + namespace Ydb { class TypedValue; } @@ -42,9 +42,9 @@ public: private: TParams(::google::protobuf::Map<TString, Ydb::TypedValue>&& protoMap); - ::google::protobuf::Map<TString, Ydb::TypedValue>* GetProtoMapPtr(); - const ::google::protobuf::Map<TString, Ydb::TypedValue>& GetProtoMap() const; - + ::google::protobuf::Map<TString, Ydb::TypedValue>* GetProtoMapPtr(); + const ::google::protobuf::Map<TString, Ydb::TypedValue>& GetProtoMap() const; + class TImpl; std::shared_ptr<TImpl> Impl_; }; diff --git a/ydb/public/sdk/cpp/client/ydb_params/params_ut.cpp b/ydb/public/sdk/cpp/client/ydb_params/params_ut.cpp index ce33b8be96..1b516de577 100644 --- a/ydb/public/sdk/cpp/client/ydb_params/params_ut.cpp +++ b/ydb/public/sdk/cpp/client/ydb_params/params_ut.cpp @@ -1,8 +1,8 @@ #include <ydb/public/lib/yson_value/ydb_yson_value.h> #include <ydb/public/sdk/cpp/client/ydb_params/params.h> -#include <library/cpp/testing/unittest/registar.h> -#include <library/cpp/testing/unittest/tests_data.h> +#include <library/cpp/testing/unittest/registar.h> +#include <library/cpp/testing/unittest/tests_data.h> using namespace NYdb; diff --git a/ydb/public/sdk/cpp/client/ydb_params/ut/ya.make b/ydb/public/sdk/cpp/client/ydb_params/ut/ya.make index 05a87bb748..cd5ee478c9 100644 --- a/ydb/public/sdk/cpp/client/ydb_params/ut/ya.make +++ b/ydb/public/sdk/cpp/client/ydb_params/ut/ya.make @@ -21,7 +21,7 @@ PEERDIR( ) SRCS( - params_ut.cpp + params_ut.cpp ) END() diff --git a/ydb/public/sdk/cpp/client/ydb_params/ya.make b/ydb/public/sdk/cpp/client/ydb_params/ya.make index 5ee6777789..b9f225d275 100644 --- a/ydb/public/sdk/cpp/client/ydb_params/ya.make +++ b/ydb/public/sdk/cpp/client/ydb_params/ya.make @@ -1,22 +1,22 @@ -LIBRARY() - -OWNER( - dcherednik - g:kikimr -) - -SRCS( - params.cpp - impl.cpp -) - -PEERDIR( +LIBRARY() + +OWNER( + dcherednik + g:kikimr +) + +SRCS( + params.cpp + impl.cpp +) + +PEERDIR( ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers ydb/public/sdk/cpp/client/ydb_value -) - -END() - +) + +END() + RECURSE_FOR_TESTS( ut ) diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/common.cpp b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/common.cpp index e82e48eb2a..dd42c8c4ed 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/common.cpp +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/common.cpp @@ -1,4 +1,4 @@ -#include "common.h" +#include "common.h" #include <util/charset/unidata.h> @@ -73,16 +73,16 @@ NYql::TIssues MakeIssueWithSubIssues(const TString& description, const NYql::TIs size_t CalcDataSize(const TReadSessionEvent::TEvent& event) { if (const TReadSessionEvent::TDataReceivedEvent* dataEvent = std::get_if<TReadSessionEvent::TDataReceivedEvent>(&event)) { size_t len = 0; - if (dataEvent->IsCompressedMessages()) { - for (const auto& msg : dataEvent->GetCompressedMessages()) { - len += msg.GetData().size(); - } - } else { - for (const auto& msg : dataEvent->GetMessages()) { - if (!msg.HasException()) { - len += msg.GetData().size(); - } - } + if (dataEvent->IsCompressedMessages()) { + for (const auto& msg : dataEvent->GetCompressedMessages()) { + len += msg.GetData().size(); + } + } else { + for (const auto& msg : dataEvent->GetMessages()) { + if (!msg.HasException()) { + len += msg.GetData().size(); + } + } } return len; } else { diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/common.h b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/common.h index 51f924c1a1..7287b69894 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/common.h +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/common.h @@ -3,7 +3,7 @@ #include <ydb/public/sdk/cpp/client/ydb_persqueue_core/persqueue.h> #include <ydb/public/sdk/cpp/client/ydb_common_client/impl/client.h> -#include <util/generic/queue.h> +#include <util/generic/queue.h> #include <util/system/condvar.h> #include <util/thread/pool.h> @@ -20,8 +20,8 @@ NYql::TIssues MakeIssueWithSubIssues(const TString& description, const NYql::TIs TString IssuesSingleLineString(const NYql::TIssues& issues); -size_t CalcDataSize(const TReadSessionEvent::TEvent& event); - +size_t CalcDataSize(const TReadSessionEvent::TEvent& event); + template <class TMessage> bool IsErrorMessage(const TMessage& serverMessage) { const Ydb::StatusIds::StatusCode status = serverMessage.status(); diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue.cpp b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue.cpp index 2e7dcb640d..a22ed23cea 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue.cpp +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue.cpp @@ -126,7 +126,7 @@ TDescribeTopicResult::TTopicSettings::TReadRule::TReadRule(const Ydb::PersQueue: SupportedCodecs_.push_back(static_cast<ECodec>(codec)); } Version_ = settings.version(); - ServiceType_ = settings.service_type(); + ServiceType_ = settings.service_type(); } TDescribeTopicResult::TTopicSettings::TRemoteMirrorRule::TRemoteMirrorRule(const Ydb::PersQueue::V1::TopicSettings::RemoteMirrorRule& settings) diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue_impl.cpp b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue_impl.cpp index f86e6cd599..bd466fd6f3 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue_impl.cpp +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue_impl.cpp @@ -1,6 +1,6 @@ #include "persqueue_impl.h" -#include "read_session.h" -#include "write_session.h" +#include "read_session.h" +#include "write_session.h" namespace NYdb::NPersQueue { diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue_impl.h b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue_impl.h index f4dfb247fe..57e9dd43bf 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue_impl.h +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue_impl.h @@ -1,11 +1,11 @@ #pragma once -#include "common.h" - -#define INCLUDE_YDB_INTERNAL_H +#include "common.h" + +#define INCLUDE_YDB_INTERNAL_H #include <ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.h> -#undef INCLUDE_YDB_INTERNAL_H - +#undef INCLUDE_YDB_INTERNAL_H + #include <ydb/public/api/grpc/draft/ydb_persqueue_v1.grpc.pb.h> #include <ydb/public/sdk/cpp/client/ydb_persqueue_core/persqueue.h> @@ -40,7 +40,7 @@ public: for (const auto& codec : readRule.SupportedCodecs_) { rrProps.add_supported_codecs((static_cast<Ydb::PersQueue::V1::Codec>(codec))); } - rrProps.set_service_type(readRule.ServiceType_); + rrProps.set_service_type(readRule.ServiceType_); } template <class TRequest, class TSettings> diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/read_session.cpp b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/read_session.cpp index 2a43f71a99..e7dd0a87e5 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/read_session.cpp +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/read_session.cpp @@ -1,11 +1,11 @@ #include "persqueue_impl.h" -#include "read_session.h" -#include "common.h" +#include "read_session.h" +#include "common.h" -#define INCLUDE_YDB_INTERNAL_H +#define INCLUDE_YDB_INTERNAL_H #include <ydb/public/sdk/cpp/client/impl/ydb_internal/logger/log.h> -#undef INCLUDE_YDB_INTERNAL_H - +#undef INCLUDE_YDB_INTERNAL_H + #include <library/cpp/containers/disjoint_interval_tree/disjoint_interval_tree.h> #include <util/generic/guid.h> #include <util/generic/size_literals.h> @@ -22,15 +22,15 @@ static const TString DRIVER_IS_STOPPING_DESCRIPTION = "Driver is stopping"; static const bool RangesMode = !GetEnv("PQ_OFFSET_RANGES_MODE").empty(); -std::pair<ui64, ui64> GetMessageOffsetRange(const TReadSessionEvent::TDataReceivedEvent& dataReceivedEvent, ui64 index) { - if (dataReceivedEvent.IsCompressedMessages()) { - const auto& msg = dataReceivedEvent.GetCompressedMessages()[index]; - return {msg.GetOffset(0), msg.GetOffset(msg.GetBlocksCount() - 1) + 1}; - } - const auto& msg = dataReceivedEvent.GetMessages()[index]; - return {msg.GetOffset(), msg.GetOffset() + 1}; -} - +std::pair<ui64, ui64> GetMessageOffsetRange(const TReadSessionEvent::TDataReceivedEvent& dataReceivedEvent, ui64 index) { + if (dataReceivedEvent.IsCompressedMessages()) { + const auto& msg = dataReceivedEvent.GetCompressedMessages()[index]; + return {msg.GetOffset(0), msg.GetOffset(msg.GetBlocksCount() - 1) + 1}; + } + const auto& msg = dataReceivedEvent.GetMessages()[index]; + return {msg.GetOffset(), msg.GetOffset() + 1}; +} + TString IssuesSingleLineString(const NYql::TIssues& issues) { return SubstGlobalCopy(issues.ToString(), '\n', ' '); } @@ -1023,14 +1023,14 @@ void TSingleClusterReadSessionImpl::RequestPartitionStreamStatus(const TPartitio void TSingleClusterReadSessionImpl::OnUserRetrievedEvent(const TReadSessionEvent::TEvent& event) { Log << TLOG_DEBUG << "Read session event " << DebugString(event); - const i64 bytesCount = static_cast<i64>(CalcDataSize(event)); + const i64 bytesCount = static_cast<i64>(CalcDataSize(event)); Y_ASSERT(bytesCount >= 0); if (!std::get_if<TReadSessionEvent::TDataReceivedEvent>(&event)) { // Event is not data event. return; } - *Settings.Counters_->MessagesInflight -= std::get<TReadSessionEvent::TDataReceivedEvent>(event).GetMessagesCount(); + *Settings.Counters_->MessagesInflight -= std::get<TReadSessionEvent::TDataReceivedEvent>(event).GetMessagesCount(); *Settings.Counters_->BytesInflightTotal -= bytesCount; *Settings.Counters_->BytesInflightUncompressed -= bytesCount; @@ -1580,18 +1580,18 @@ TReadSessionEvent::TPartitionStreamClosedEvent::TPartitionStreamClosedEvent(TPar } TReadSessionEvent::TDataReceivedEvent::TDataReceivedEvent(TVector<TMessage> messages, - TVector<TCompressedMessage> compressedMessages, + TVector<TCompressedMessage> compressedMessages, TPartitionStream::TPtr partitionStream) : Messages(std::move(messages)) - , CompressedMessages(std::move(compressedMessages)) + , CompressedMessages(std::move(compressedMessages)) , PartitionStream(std::move(partitionStream)) { - for (size_t i = 0; i < GetMessagesCount(); ++i) { - auto [from, to] = GetMessageOffsetRange(*this, i); - if (OffsetRanges.empty() || OffsetRanges.back().second != from) { - OffsetRanges.emplace_back(from, to); - } else { - OffsetRanges.back().second = to; + for (size_t i = 0; i < GetMessagesCount(); ++i) { + auto [from, to] = GetMessageOffsetRange(*this, i); + if (OffsetRanges.empty() || OffsetRanges.back().second != from) { + OffsetRanges.emplace_back(from, to); + } else { + OffsetRanges.back().second = to; } } } @@ -1616,14 +1616,14 @@ TString TReadSessionEvent::TDataReceivedEvent::DebugString(bool printData) const TStringBuilder ret; ret << "DataReceived { PartitionStreamId: " << GetPartitionStream()->GetPartitionStreamId() << " PartitionId: " << GetPartitionStream()->GetPartitionId(); - for (const auto& message : Messages) { + for (const auto& message : Messages) { + ret << " "; + message.DebugString(ret, printData); + } + for (const auto& message : CompressedMessages) { ret << " "; - message.DebugString(ret, printData); + message.DebugString(ret, printData); } - for (const auto& message : CompressedMessages) { - ret << " "; - message.DebugString(ret, printData); - } ret << " }"; return std::move(ret); } @@ -1696,16 +1696,16 @@ TReadSessionEventInfo::TReadSessionEventInfo(TIntrusivePtr<TPartitionStreamImpl> TReadSessionEventInfo::TReadSessionEventInfo(TIntrusivePtr<TPartitionStreamImpl> partitionStream, std::weak_ptr<IUserRetrievedEventCallback> session, - TVector<TReadSessionEvent::TDataReceivedEvent::TMessage> messages, - TVector<TReadSessionEvent::TDataReceivedEvent::TCompressedMessage> compressedMessages) + TVector<TReadSessionEvent::TDataReceivedEvent::TMessage> messages, + TVector<TReadSessionEvent::TDataReceivedEvent::TCompressedMessage> compressedMessages) : PartitionStream(std::move(partitionStream)) - , Event( - NMaybe::TInPlace(), - std::in_place_type_t<TReadSessionEvent::TDataReceivedEvent>(), - std::move(messages), - std::move(compressedMessages), - PartitionStream - ) + , Event( + NMaybe::TInPlace(), + std::in_place_type_t<TReadSessionEvent::TDataReceivedEvent>(), + std::move(messages), + std::move(compressedMessages), + PartitionStream + ) , Session(std::move(session)) { } @@ -1745,11 +1745,11 @@ void TReadSessionEventInfo::OnUserRetrievedEvent() { } } -bool TReadSessionEventInfo::TakeData(TVector<TReadSessionEvent::TDataReceivedEvent::TMessage>* messages, - TVector<TReadSessionEvent::TDataReceivedEvent::TCompressedMessage>* compressedMessages, - size_t* maxByteSize) -{ - return PartitionStream->TopEvent().GetData().TakeData(PartitionStream, messages, compressedMessages, maxByteSize); +bool TReadSessionEventInfo::TakeData(TVector<TReadSessionEvent::TDataReceivedEvent::TMessage>* messages, + TVector<TReadSessionEvent::TDataReceivedEvent::TCompressedMessage>* compressedMessages, + size_t* maxByteSize) +{ + return PartitionStream->TopEvent().GetData().TakeData(PartitionStream, messages, compressedMessages, maxByteSize); } TReadSessionEventsQueue::TReadSessionEventsQueue(const TSettings& settings, std::weak_ptr<IUserRetrievedEventCallback> session) @@ -1799,17 +1799,17 @@ TDataDecompressionInfo* TReadSessionEventsQueue::PushDataEvent(TIntrusivePtr<TPa } with_lock (Mutex) { - return &partitionStream->InsertDataEvent(std::move(msg), Settings.Decompress_); + return &partitionStream->InsertDataEvent(std::move(msg), Settings.Decompress_); } } TMaybe<TReadSessionEventsQueue::TEventInfo> TReadSessionEventsQueue::GetDataEventImpl(TEventInfo& srcDataEventInfo, size_t* maxByteSize) { // Assumes that we're under lock. TVector<TReadSessionEvent::TDataReceivedEvent::TMessage> messages; - TVector<TReadSessionEvent::TDataReceivedEvent::TCompressedMessage> compressedMessages; + TVector<TReadSessionEvent::TDataReceivedEvent::TCompressedMessage> compressedMessages; TIntrusivePtr<TPartitionStreamImpl> partitionStream = srcDataEventInfo.PartitionStream; bool messageExtracted = false; while (srcDataEventInfo.HasReadyUnreadData() && *maxByteSize > 0) { - const bool hasMoreUnpackedData = srcDataEventInfo.TakeData(&messages, &compressedMessages, maxByteSize); + const bool hasMoreUnpackedData = srcDataEventInfo.TakeData(&messages, &compressedMessages, maxByteSize); if (!hasMoreUnpackedData) { const bool messageIsFullyRead = !srcDataEventInfo.HasMoreData(); if (messageIsFullyRead) { @@ -1823,10 +1823,10 @@ TMaybe<TReadSessionEventsQueue::TEventInfo> TReadSessionEventsQueue::GetDataEven partitionStream->TopEvent().Signalled = false; } - if (messages.empty() && compressedMessages.empty()) { + if (messages.empty() && compressedMessages.empty()) { return Nothing(); } - return TEventInfo(partitionStream, partitionStream->GetSession(), std::move(messages), std::move(compressedMessages)); + return TEventInfo(partitionStream, partitionStream->GetSession(), std::move(messages), std::move(compressedMessages)); } void TReadSessionEventsQueue::SignalReadyEvents(TPartitionStreamImpl* partitionStream) { @@ -1938,14 +1938,14 @@ void TReadSessionEventsQueue::ClearAllEvents() { } } -TDataDecompressionInfo::TDataDecompressionInfo( - Ydb::PersQueue::V1::MigrationStreamingReadServerMessage::DataBatch::PartitionData&& msg, - std::weak_ptr<TSingleClusterReadSessionImpl> session, - bool doDecompress -) +TDataDecompressionInfo::TDataDecompressionInfo( + Ydb::PersQueue::V1::MigrationStreamingReadServerMessage::DataBatch::PartitionData&& msg, + std::weak_ptr<TSingleClusterReadSessionImpl> session, + bool doDecompress +) : ServerMessage(std::move(msg)) , Session(std::move(session)) - , DoDecompress(doDecompress) + , DoDecompress(doDecompress) { for (const Ydb::PersQueue::V1::MigrationStreamingReadServerMessage::DataBatch::Batch& batch : ServerMessage.batches()) { for (const Ydb::PersQueue::V1::MigrationStreamingReadServerMessage::DataBatch::MessageData& messageData : batch.message_data()) { @@ -2035,11 +2035,11 @@ i64 TDataDecompressionInfo::StartDecompressionTasks(const IExecutor::TPtr& execu return used; } -bool TDataDecompressionInfo::TakeData(const TIntrusivePtr<TPartitionStreamImpl>& partitionStream, - TVector<TReadSessionEvent::TDataReceivedEvent::TMessage>* messages, - TVector<TReadSessionEvent::TDataReceivedEvent::TCompressedMessage>* compressedMessages, - size_t* maxByteSize) -{ +bool TDataDecompressionInfo::TakeData(const TIntrusivePtr<TPartitionStreamImpl>& partitionStream, + TVector<TReadSessionEvent::TDataReceivedEvent::TMessage>* messages, + TVector<TReadSessionEvent::TDataReceivedEvent::TCompressedMessage>* compressedMessages, + size_t* maxByteSize) +{ TMaybe<std::pair<size_t, size_t>> readyThreshold = GetReadyThreshold(); Y_ASSERT(readyThreshold); auto& msg = GetServerMessage(); @@ -2054,35 +2054,35 @@ bool TDataDecompressionInfo::TakeData(const TIntrusivePtr<TPartitionStreamImpl>& auto& messageData = *batch.mutable_message_data(CurrentReadingMessage.second); minOffset = Min(minOffset, messageData.offset()); maxOffset = Max(maxOffset, messageData.offset()); - TReadSessionEvent::TDataReceivedEvent::TMessageInformation messageInfo( + TReadSessionEvent::TDataReceivedEvent::TMessageInformation messageInfo( messageData.offset(), batch.source_id(), messageData.seq_no(), TInstant::MilliSeconds(messageData.create_timestamp_ms()), batchWriteTimestamp, batch.ip(), - meta, - messageData.uncompressed_size() + meta, + messageData.uncompressed_size() ); - if (DoDecompress) { - messages->emplace_back( - messageData.data(), - GetDecompressionError(CurrentReadingMessage.first, CurrentReadingMessage.second), - messageInfo, - partitionStream, - messageData.partition_key(), - messageData.explicit_hash() - ); - } else { - compressedMessages->emplace_back( - static_cast<ECodec>(messageData.codec()), - messageData.data(), - TVector<TReadSessionEvent::TDataReceivedEvent::TMessageInformation>{messageInfo}, - partitionStream, - messageData.partition_key(), - messageData.explicit_hash() - ); - } + if (DoDecompress) { + messages->emplace_back( + messageData.data(), + GetDecompressionError(CurrentReadingMessage.first, CurrentReadingMessage.second), + messageInfo, + partitionStream, + messageData.partition_key(), + messageData.explicit_hash() + ); + } else { + compressedMessages->emplace_back( + static_cast<ECodec>(messageData.codec()), + messageData.data(), + TVector<TReadSessionEvent::TDataReceivedEvent::TMessageInformation>{messageInfo}, + partitionStream, + messageData.partition_key(), + messageData.explicit_hash() + ); + } *maxByteSize -= Min(*maxByteSize, messageData.data().size()); // Clear data to free internal session's memory. @@ -2154,15 +2154,15 @@ void TDataDecompressionInfo::TDecompressionTask::operator()() { maxOffset = Max(maxOffset, data.offset()); try { - if (Parent->DoDecompress - && data.codec() != Ydb::PersQueue::V1::CODEC_RAW - && data.codec() != Ydb::PersQueue::V1::CODEC_UNSPECIFIED - ) { - TString decompressed = NCompressionDetails::Decompress(data); - data.set_data(decompressed); - data.set_codec(Ydb::PersQueue::V1::CODEC_RAW); - } - DecompressedSize += data.data().size(); + if (Parent->DoDecompress + && data.codec() != Ydb::PersQueue::V1::CODEC_RAW + && data.codec() != Ydb::PersQueue::V1::CODEC_UNSPECIFIED + ) { + TString decompressed = NCompressionDetails::Decompress(data); + data.set_data(decompressed); + data.set_codec(Ydb::PersQueue::V1::CODEC_RAW); + } + DecompressedSize += data.data().size(); } catch (...) { Parent->PutDecompressionError(std::current_exception(), messages.Batch, i); data.clear_data(); // Free memory, because we don't count it. @@ -2303,17 +2303,17 @@ public: } void OnDataReceived(TReadSessionEvent::TDataReceivedEvent& event) { - Y_ASSERT(event.GetMessagesCount()); + Y_ASSERT(event.GetMessagesCount()); TDeferredCommit deferredCommit; with_lock (Lock) { auto& offsetSet = PartitionStreamToUncommittedOffsets[event.GetPartitionStream()->GetPartitionStreamId()]; // Messages could contain holes in offset, but later commit ack will tell us right border. // So we can easily insert the whole interval with holes included. // It will be removed from set by specifying proper right border. - auto firstMessageOffsets = GetMessageOffsetRange(event, 0); - auto lastMessageOffsets = GetMessageOffsetRange(event, event.GetMessagesCount() - 1); + auto firstMessageOffsets = GetMessageOffsetRange(event, 0); + auto lastMessageOffsets = GetMessageOffsetRange(event, event.GetMessagesCount() - 1); - offsetSet.InsertInterval(firstMessageOffsets.first, lastMessageOffsets.second); + offsetSet.InsertInterval(firstMessageOffsets.first, lastMessageOffsets.second); if (CommitAfterProcessing) { deferredCommit.Add(event); @@ -2457,7 +2457,7 @@ TDeferredCommit::~TDeferredCommit() { Impl = MakeHolder<TImpl>(); \ } \ Impl - + void TDeferredCommit::Add(const TPartitionStream::TPtr& partitionStream, ui64 startOffset, ui64 endOffset) { GET_IMPL()->Add(partitionStream, startOffset, endOffset); } @@ -2485,8 +2485,8 @@ void TDeferredCommit::Commit() { void TDeferredCommit::TImpl::Add(const TReadSessionEvent::TDataReceivedEvent::TMessage& message) { Y_ASSERT(message.GetPartitionStream()); Add(message.GetPartitionStream(), message.GetOffset()); -} - +} + void TDeferredCommit::TImpl::Add(const TPartitionStream::TPtr& partitionStream, TDisjointIntervalTree<ui64>& offsetSet, ui64 startOffset, ui64 endOffset) { if (offsetSet.Intersects(startOffset, endOffset)) { ThrowFatalError(TStringBuilder() << "Commit set already has some offsets from half-interval [" @@ -2517,15 +2517,15 @@ void TDeferredCommit::TImpl::Add(const TReadSessionEvent::TDataReceivedEvent& da const TPartitionStream::TPtr& partitionStream = dataReceivedEvent.GetPartitionStream(); Y_ASSERT(partitionStream); auto& offsetSet = Offsets[partitionStream]; - auto [startOffset, endOffset] = GetMessageOffsetRange(dataReceivedEvent, 0); - for (size_t i = 1; i < dataReceivedEvent.GetMessagesCount(); ++i) { - auto msgOffsetRange = GetMessageOffsetRange(dataReceivedEvent, i); - if (msgOffsetRange.first == endOffset) { - endOffset= msgOffsetRange.second; + auto [startOffset, endOffset] = GetMessageOffsetRange(dataReceivedEvent, 0); + for (size_t i = 1; i < dataReceivedEvent.GetMessagesCount(); ++i) { + auto msgOffsetRange = GetMessageOffsetRange(dataReceivedEvent, i); + if (msgOffsetRange.first == endOffset) { + endOffset= msgOffsetRange.second; } else { Add(partitionStream, offsetSet, startOffset, endOffset); - startOffset = msgOffsetRange.first; - endOffset = msgOffsetRange.second; + startOffset = msgOffsetRange.first; + endOffset = msgOffsetRange.second; } } Add(partitionStream, offsetSet, startOffset, endOffset); diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/read_session.h b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/read_session.h index 8d3dfde9a6..b5b874954f 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/read_session.h +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/read_session.h @@ -1,7 +1,7 @@ #pragma once - -#include "common.h" - + +#include "common.h" + #include <ydb/public/sdk/cpp/client/ydb_common_client/impl/client.h> #include <ydb/public/api/grpc/draft/ydb_persqueue_v1.grpc.pb.h> @@ -98,11 +98,11 @@ class TDataDecompressionInfo { public: TDataDecompressionInfo(const TDataDecompressionInfo&) = default; TDataDecompressionInfo(TDataDecompressionInfo&&) = default; - TDataDecompressionInfo( - Ydb::PersQueue::V1::MigrationStreamingReadServerMessage::DataBatch::PartitionData&& msg, - std::weak_ptr<TSingleClusterReadSessionImpl> session, - bool doDecompress - ); + TDataDecompressionInfo( + Ydb::PersQueue::V1::MigrationStreamingReadServerMessage::DataBatch::PartitionData&& msg, + std::weak_ptr<TSingleClusterReadSessionImpl> session, + bool doDecompress + ); i64 StartDecompressionTasks(const IExecutor::TPtr& executor, i64 availableMemory, @@ -155,10 +155,10 @@ public: } // Takes data. Returns true if event has more unpacked data. - bool TakeData(const TIntrusivePtr<TPartitionStreamImpl>& partitionStream, - TVector<TReadSessionEvent::TDataReceivedEvent::TMessage>* messages, - TVector<TReadSessionEvent::TDataReceivedEvent::TCompressedMessage>* compressedMessages, - size_t* maxByteSize); + bool TakeData(const TIntrusivePtr<TPartitionStreamImpl>& partitionStream, + TVector<TReadSessionEvent::TDataReceivedEvent::TMessage>* messages, + TVector<TReadSessionEvent::TDataReceivedEvent::TCompressedMessage>* compressedMessages, + size_t* maxByteSize); bool HasMoreData() const { return CurrentReadingMessage.first < static_cast<size_t>(GetServerMessage().batches_size()); @@ -212,7 +212,7 @@ private: Ydb::PersQueue::V1::MigrationStreamingReadServerMessage::DataBatch::PartitionData ServerMessage; std::vector<TWriteSessionMeta::TPtr> BatchesMeta; std::weak_ptr<TSingleClusterReadSessionImpl> Session; - bool DoDecompress; + bool DoDecompress; i64 CompressedDataSize = 0; std::atomic<i64> SourceDataNotProcessed = 0; std::pair<size_t, size_t> CurrentDecompressingMessage = {0, 0}; // (Batch, Message) @@ -256,16 +256,16 @@ struct TReadSessionEventInfo { TReadSessionEventInfo(TIntrusivePtr<TPartitionStreamImpl> partitionStream, std::weak_ptr<IUserRetrievedEventCallback> session, - TVector<TReadSessionEvent::TDataReceivedEvent::TMessage> messages, - TVector<TReadSessionEvent::TDataReceivedEvent::TCompressedMessage> compressedMessages); + TVector<TReadSessionEvent::TDataReceivedEvent::TMessage> messages, + TVector<TReadSessionEvent::TDataReceivedEvent::TCompressedMessage> compressedMessages); bool IsEmpty() const; bool IsDataEvent() const; // Takes data. Returns true if event has more unpacked data. - bool TakeData(TVector<TReadSessionEvent::TDataReceivedEvent::TMessage>* messages, - TVector<TReadSessionEvent::TDataReceivedEvent::TCompressedMessage>* comressedMessages, - size_t* maxByteSize); + bool TakeData(TVector<TReadSessionEvent::TDataReceivedEvent::TMessage>* messages, + TVector<TReadSessionEvent::TDataReceivedEvent::TCompressedMessage>* comressedMessages, + size_t* maxByteSize); TEvent& GetEvent() { Y_ASSERT(Event); @@ -295,12 +295,12 @@ struct TRawPartitionStreamEvent { TRawPartitionStreamEvent(const TRawPartitionStreamEvent&) = default; TRawPartitionStreamEvent(TRawPartitionStreamEvent&&) = default; - TRawPartitionStreamEvent( - Ydb::PersQueue::V1::MigrationStreamingReadServerMessage::DataBatch::PartitionData&& msg, - std::weak_ptr<TSingleClusterReadSessionImpl> session, - bool doDecompress - ) - : Event(std::in_place_type_t<TDataDecompressionInfo>(), std::move(msg), std::move(session), doDecompress) + TRawPartitionStreamEvent( + Ydb::PersQueue::V1::MigrationStreamingReadServerMessage::DataBatch::PartitionData&& msg, + std::weak_ptr<TSingleClusterReadSessionImpl> session, + bool doDecompress + ) + : Event(std::in_place_type_t<TDataDecompressionInfo>(), std::move(msg), std::move(session), doDecompress) { } @@ -413,12 +413,12 @@ public: EventsQueue.emplace_back(std::forward<T>(event)); } - TDataDecompressionInfo& InsertDataEvent( - Ydb::PersQueue::V1::MigrationStreamingReadServerMessage::DataBatch::PartitionData&& msg, - bool doDecompress - ) { + TDataDecompressionInfo& InsertDataEvent( + Ydb::PersQueue::V1::MigrationStreamingReadServerMessage::DataBatch::PartitionData&& msg, + bool doDecompress + ) { ++DataDecompressionEventsCount; - return EventsQueue.emplace_back(std::move(msg), Session, doDecompress).GetData(); + return EventsQueue.emplace_back(std::move(msg), Session, doDecompress).GetData(); } bool IsWaitingForDataDecompression() const { diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/read_session_messages.cpp b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/read_session_messages.cpp index f8636286ec..2f0fd93a8f 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/read_session_messages.cpp +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/read_session_messages.cpp @@ -1,242 +1,242 @@ -#include "read_session.h" - +#include "read_session.h" + #include <ydb/public/sdk/cpp/client/ydb_persqueue_core/persqueue.h> - -namespace NYdb::NPersQueue { - -TReadSessionEvent::TDataReceivedEvent::TMessageInformation::TMessageInformation( - ui64 offset, - TString messageGroupId, - ui64 seqNo, - TInstant createTime, - TInstant writeTime, - TString ip, - TWriteSessionMeta::TPtr meta, - ui64 uncompressedSize -) - : Offset(offset) - , MessageGroupId(messageGroupId) - , SeqNo(seqNo) - , CreateTime(createTime) - , WriteTime(writeTime) - , Ip(ip) - , Meta(meta) + +namespace NYdb::NPersQueue { + +TReadSessionEvent::TDataReceivedEvent::TMessageInformation::TMessageInformation( + ui64 offset, + TString messageGroupId, + ui64 seqNo, + TInstant createTime, + TInstant writeTime, + TString ip, + TWriteSessionMeta::TPtr meta, + ui64 uncompressedSize +) + : Offset(offset) + , MessageGroupId(messageGroupId) + , SeqNo(seqNo) + , CreateTime(createTime) + , WriteTime(writeTime) + , Ip(ip) + , Meta(meta) , UncompressedSize(uncompressedSize) -{} - -static void DebugStringImpl(const TReadSessionEvent::TDataReceivedEvent::TMessageInformation& info, TStringBuilder& ret) { +{} + +static void DebugStringImpl(const TReadSessionEvent::TDataReceivedEvent::TMessageInformation& info, TStringBuilder& ret) { ret << " Information: {" - << " Offset: " << info.Offset - << " SeqNo: " << info.SeqNo - << " MessageGroupId: \"" << info.MessageGroupId << "\"" - << " CreateTime: " << info.CreateTime - << " WriteTime: " << info.WriteTime - << " Ip: \"" << info.Ip << "\"" - << " UncompressedSize: " << info.UncompressedSize; - ret << " Meta: {"; - bool firstKey = true; - for (const auto& [k, v] : info.Meta->Fields) { - ret << (firstKey ? " \"" : ", \"") << k << "\": \"" << v << "\""; - firstKey = false; - } - ret << " } }"; -} - -template <class TSerializeInformationFunc> -static void DebugStringImpl(TStringBuilder& ret, - const TString& name, - const TReadSessionEvent::TDataReceivedEvent::IMessage& msg, - bool printData, - TSerializeInformationFunc serializeInformationFunc, - std::optional<ECodec> codec = std::nullopt) -{ - ret << name << " {"; - try { - const TString& data = msg.GetData(); - if (printData) { - ret << " Data: \"" << data << "\""; - } else { - ret << " Data: .." << data.size() << " bytes.."; - } - } catch (...) { - ret << " DataDecompressionError: \"" << CurrentExceptionMessage() << "\""; - } - auto partitionStream = msg.GetPartitionStream(); - ret << " Partition stream id: " << partitionStream->GetPartitionStreamId() - << " Cluster: \"" << partitionStream->GetCluster() << "\". Topic: \"" << partitionStream->GetTopicPath() << "\"" - << " Partition: " << partitionStream->GetPartitionId() - << " PartitionKey: \"" << msg.GetPartitionKey() << "\""; - if (codec) { - ret << " Codec: " << codec.value(); - } + << " Offset: " << info.Offset + << " SeqNo: " << info.SeqNo + << " MessageGroupId: \"" << info.MessageGroupId << "\"" + << " CreateTime: " << info.CreateTime + << " WriteTime: " << info.WriteTime + << " Ip: \"" << info.Ip << "\"" + << " UncompressedSize: " << info.UncompressedSize; + ret << " Meta: {"; + bool firstKey = true; + for (const auto& [k, v] : info.Meta->Fields) { + ret << (firstKey ? " \"" : ", \"") << k << "\": \"" << v << "\""; + firstKey = false; + } + ret << " } }"; +} + +template <class TSerializeInformationFunc> +static void DebugStringImpl(TStringBuilder& ret, + const TString& name, + const TReadSessionEvent::TDataReceivedEvent::IMessage& msg, + bool printData, + TSerializeInformationFunc serializeInformationFunc, + std::optional<ECodec> codec = std::nullopt) +{ + ret << name << " {"; + try { + const TString& data = msg.GetData(); + if (printData) { + ret << " Data: \"" << data << "\""; + } else { + ret << " Data: .." << data.size() << " bytes.."; + } + } catch (...) { + ret << " DataDecompressionError: \"" << CurrentExceptionMessage() << "\""; + } + auto partitionStream = msg.GetPartitionStream(); + ret << " Partition stream id: " << partitionStream->GetPartitionStreamId() + << " Cluster: \"" << partitionStream->GetCluster() << "\". Topic: \"" << partitionStream->GetTopicPath() << "\"" + << " Partition: " << partitionStream->GetPartitionId() + << " PartitionKey: \"" << msg.GetPartitionKey() << "\""; + if (codec) { + ret << " Codec: " << codec.value(); + } serializeInformationFunc(ret); - ret << " }"; -} - -const TString& TReadSessionEvent::TDataReceivedEvent::IMessage::GetData() const { - return Data; -} - -const TPartitionStream::TPtr& TReadSessionEvent::TDataReceivedEvent::IMessage::GetPartitionStream() const { - return PartitionStream; -} - -const TString& TReadSessionEvent::TDataReceivedEvent::IMessage::GetPartitionKey() const { - return PartitionKey; -} - -const TString TReadSessionEvent::TDataReceivedEvent::IMessage::GetExplicitHash() const { - return ExplicitHash; -} - -TString TReadSessionEvent::TDataReceivedEvent::IMessage::DebugString(bool printData) const { - TStringBuilder ret; - DebugString(ret, printData); - return std::move(ret); -} - -TReadSessionEvent::TDataReceivedEvent::IMessage::IMessage(const TString& data, - TPartitionStream::TPtr partitionStream, - const TString& partitionKey, - const TString& explicitHash) - : Data(data) - , PartitionStream(partitionStream) - , PartitionKey(partitionKey) - , ExplicitHash(explicitHash) -{} - -const TString& TReadSessionEvent::TDataReceivedEvent::TMessage::GetData() const { - if (DecompressionException) { - std::rethrow_exception(DecompressionException); - } - return IMessage::GetData(); -} - -bool TReadSessionEvent::TDataReceivedEvent::TMessage::HasException() const { - return DecompressionException != nullptr; -} - -ui64 TReadSessionEvent::TDataReceivedEvent::TMessage::GetOffset() const { - return Information.Offset; -} - -const TString& TReadSessionEvent::TDataReceivedEvent::TMessage::GetMessageGroupId() const { - return Information.MessageGroupId; -} - -ui64 TReadSessionEvent::TDataReceivedEvent::TMessage::GetSeqNo() const { - return Information.SeqNo; -} - -TInstant TReadSessionEvent::TDataReceivedEvent::TMessage::GetCreateTime() const { - return Information.CreateTime; -} - -TInstant TReadSessionEvent::TDataReceivedEvent::TMessage::GetWriteTime() const { - return Information.WriteTime; -} - -const TString& TReadSessionEvent::TDataReceivedEvent::TMessage::GetIp() const { - return Information.Ip; -} - -const TWriteSessionMeta::TPtr& TReadSessionEvent::TDataReceivedEvent::TMessage::GetMeta() const { - return Information.Meta; -} - -void TReadSessionEvent::TDataReceivedEvent::TMessage::DebugString(TStringBuilder& ret, bool printData) const { - DebugStringImpl(ret, "Message", *this, printData, [this](TStringBuilder& ret) { - DebugStringImpl(this->Information, ret); - }); -} - -TReadSessionEvent::TDataReceivedEvent::TMessage::TMessage(const TString& data, - std::exception_ptr decompressionException, - const TMessageInformation& information, - TPartitionStream::TPtr partitionStream, - const TString& partitionKey, - const TString& explicitHash) - : IMessage(data, partitionStream, partitionKey, explicitHash) - , DecompressionException(std::move(decompressionException)) - , Information(information) -{ -} - -void TReadSessionEvent::TDataReceivedEvent::TMessage::Commit() { - static_cast<TPartitionStreamImpl*>(PartitionStream.Get())->Commit(Information.Offset, Information.Offset + 1); -} - -ui64 TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::GetBlocksCount() const { - return Information.size(); -} - -ECodec TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::GetCodec() const { - return Codec; -} - -ui64 TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::GetOffset(ui64 index) const { - return Information.at(index).Offset; -} - -const TString& TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::GetMessageGroupId(ui64 index) const { - return Information.at(index).MessageGroupId; -} - -ui64 TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::GetSeqNo(ui64 index) const { - return Information.at(index).SeqNo; -} - -TInstant TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::GetCreateTime(ui64 index) const { - return Information.at(index).CreateTime; -} - -TInstant TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::GetWriteTime(ui64 index) const { - return Information.at(index).WriteTime; -} - -const TString& TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::GetIp(ui64 index) const { - return Information.at(index).Ip; -} - -const TWriteSessionMeta::TPtr& TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::GetMeta(ui64 index) const { - return Information.at(index).Meta; -} - -ui64 TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::GetUncompressedSize(ui64 index) const { - return Information.at(index).UncompressedSize; -} - -void TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::DebugString(TStringBuilder& ret, bool printData) const { - DebugStringImpl( - ret, - "CompressedMessage", - *this, - printData, - [this](TStringBuilder& ret) { - for (auto& info : this->Information) { - DebugStringImpl(info, ret); - } - }, - Codec - ); -} - -TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::TCompressedMessage(ECodec codec, - const TString& data, - const TVector<TMessageInformation>& information, - TPartitionStream::TPtr partitionStream, - const TString& partitionKey, - const TString& explicitHash) - : IMessage(data, partitionStream, partitionKey, explicitHash) - , Codec(codec) - , Information(information) -{} - -void TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::Commit() { - static_cast<TPartitionStreamImpl*>(PartitionStream.Get())->Commit( - Information.front().Offset, - Information.back().Offset + 1 - ); -} - -} // namespace NYdb::NPersQueue + ret << " }"; +} + +const TString& TReadSessionEvent::TDataReceivedEvent::IMessage::GetData() const { + return Data; +} + +const TPartitionStream::TPtr& TReadSessionEvent::TDataReceivedEvent::IMessage::GetPartitionStream() const { + return PartitionStream; +} + +const TString& TReadSessionEvent::TDataReceivedEvent::IMessage::GetPartitionKey() const { + return PartitionKey; +} + +const TString TReadSessionEvent::TDataReceivedEvent::IMessage::GetExplicitHash() const { + return ExplicitHash; +} + +TString TReadSessionEvent::TDataReceivedEvent::IMessage::DebugString(bool printData) const { + TStringBuilder ret; + DebugString(ret, printData); + return std::move(ret); +} + +TReadSessionEvent::TDataReceivedEvent::IMessage::IMessage(const TString& data, + TPartitionStream::TPtr partitionStream, + const TString& partitionKey, + const TString& explicitHash) + : Data(data) + , PartitionStream(partitionStream) + , PartitionKey(partitionKey) + , ExplicitHash(explicitHash) +{} + +const TString& TReadSessionEvent::TDataReceivedEvent::TMessage::GetData() const { + if (DecompressionException) { + std::rethrow_exception(DecompressionException); + } + return IMessage::GetData(); +} + +bool TReadSessionEvent::TDataReceivedEvent::TMessage::HasException() const { + return DecompressionException != nullptr; +} + +ui64 TReadSessionEvent::TDataReceivedEvent::TMessage::GetOffset() const { + return Information.Offset; +} + +const TString& TReadSessionEvent::TDataReceivedEvent::TMessage::GetMessageGroupId() const { + return Information.MessageGroupId; +} + +ui64 TReadSessionEvent::TDataReceivedEvent::TMessage::GetSeqNo() const { + return Information.SeqNo; +} + +TInstant TReadSessionEvent::TDataReceivedEvent::TMessage::GetCreateTime() const { + return Information.CreateTime; +} + +TInstant TReadSessionEvent::TDataReceivedEvent::TMessage::GetWriteTime() const { + return Information.WriteTime; +} + +const TString& TReadSessionEvent::TDataReceivedEvent::TMessage::GetIp() const { + return Information.Ip; +} + +const TWriteSessionMeta::TPtr& TReadSessionEvent::TDataReceivedEvent::TMessage::GetMeta() const { + return Information.Meta; +} + +void TReadSessionEvent::TDataReceivedEvent::TMessage::DebugString(TStringBuilder& ret, bool printData) const { + DebugStringImpl(ret, "Message", *this, printData, [this](TStringBuilder& ret) { + DebugStringImpl(this->Information, ret); + }); +} + +TReadSessionEvent::TDataReceivedEvent::TMessage::TMessage(const TString& data, + std::exception_ptr decompressionException, + const TMessageInformation& information, + TPartitionStream::TPtr partitionStream, + const TString& partitionKey, + const TString& explicitHash) + : IMessage(data, partitionStream, partitionKey, explicitHash) + , DecompressionException(std::move(decompressionException)) + , Information(information) +{ +} + +void TReadSessionEvent::TDataReceivedEvent::TMessage::Commit() { + static_cast<TPartitionStreamImpl*>(PartitionStream.Get())->Commit(Information.Offset, Information.Offset + 1); +} + +ui64 TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::GetBlocksCount() const { + return Information.size(); +} + +ECodec TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::GetCodec() const { + return Codec; +} + +ui64 TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::GetOffset(ui64 index) const { + return Information.at(index).Offset; +} + +const TString& TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::GetMessageGroupId(ui64 index) const { + return Information.at(index).MessageGroupId; +} + +ui64 TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::GetSeqNo(ui64 index) const { + return Information.at(index).SeqNo; +} + +TInstant TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::GetCreateTime(ui64 index) const { + return Information.at(index).CreateTime; +} + +TInstant TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::GetWriteTime(ui64 index) const { + return Information.at(index).WriteTime; +} + +const TString& TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::GetIp(ui64 index) const { + return Information.at(index).Ip; +} + +const TWriteSessionMeta::TPtr& TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::GetMeta(ui64 index) const { + return Information.at(index).Meta; +} + +ui64 TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::GetUncompressedSize(ui64 index) const { + return Information.at(index).UncompressedSize; +} + +void TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::DebugString(TStringBuilder& ret, bool printData) const { + DebugStringImpl( + ret, + "CompressedMessage", + *this, + printData, + [this](TStringBuilder& ret) { + for (auto& info : this->Information) { + DebugStringImpl(info, ret); + } + }, + Codec + ); +} + +TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::TCompressedMessage(ECodec codec, + const TString& data, + const TVector<TMessageInformation>& information, + TPartitionStream::TPtr partitionStream, + const TString& partitionKey, + const TString& explicitHash) + : IMessage(data, partitionStream, partitionKey, explicitHash) + , Codec(codec) + , Information(information) +{} + +void TReadSessionEvent::TDataReceivedEvent::TCompressedMessage::Commit() { + static_cast<TPartitionStreamImpl*>(PartitionStream.Get())->Commit( + Information.front().Offset, + Information.back().Offset + 1 + ); +} + +} // namespace NYdb::NPersQueue diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/write_session.cpp b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/write_session.cpp index 0b91e6acaf..af4e08b979 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/write_session.cpp +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/write_session.cpp @@ -1,4 +1,4 @@ -#include "write_session.h" +#include "write_session.h" #include <ydb/public/sdk/cpp/client/ydb_persqueue_core/persqueue.h> #include <library/cpp/string_utils/url/url.h> diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/write_session.h b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/write_session.h index 37529f48a2..324550bb02 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/write_session.h +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/write_session.h @@ -1,10 +1,10 @@ #pragma once -#include "common.h" +#include "common.h" #include "persqueue_impl.h" - + #include <ydb/public/sdk/cpp/client/ydb_persqueue_core/persqueue.h> - + #include <util/generic/buffer.h> diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/ya.make b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/ya.make index f6e6e8a5e8..21c92fa949 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/ya.make +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/ya.make @@ -1,29 +1,29 @@ -LIBRARY() - -OWNER( +LIBRARY() + +OWNER( g:kikimr - g:logbroker -) - -SRCS( - read_session_messages.cpp - common.cpp - write_session.cpp - read_session.cpp - persqueue.cpp + g:logbroker +) + +SRCS( + read_session_messages.cpp + common.cpp + write_session.cpp + read_session.cpp + persqueue.cpp persqueue_impl.cpp -) - -PEERDIR( - library/cpp/containers/disjoint_interval_tree - library/cpp/grpc/client +) + +PEERDIR( + library/cpp/containers/disjoint_interval_tree + library/cpp/grpc/client library/cpp/monlib/dynamic_counters - library/cpp/string_utils/url + library/cpp/string_utils/url ydb/library/persqueue/obfuscate ydb/public/api/grpc/draft ydb/public/sdk/cpp/client/impl/ydb_internal/make_request ydb/public/sdk/cpp/client/ydb_common_client/impl ydb/public/sdk/cpp/client/ydb_driver -) - -END() +) + +END() diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/persqueue.h b/ydb/public/sdk/cpp/client/ydb_persqueue_core/persqueue.h index 407eaf4999..ee9aa75a0a 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/persqueue.h +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/persqueue.h @@ -9,7 +9,7 @@ #include <util/generic/hash.h> #include <util/generic/maybe.h> #include <util/generic/ptr.h> -#include <util/string/builder.h> +#include <util/string/builder.h> #include <util/thread/pool.h> #include <exception> @@ -77,7 +77,7 @@ struct TDescribeTopicResult : public TStatus { return SupportedCodecs_; } GETTER(ui32, Version); - GETTER(TString, ServiceType); + GETTER(TString, ServiceType); private: TString ConsumerName_; @@ -86,7 +86,7 @@ struct TDescribeTopicResult : public TStatus { EFormat SupportedFormat_; TVector<ECodec> SupportedCodecs_; ui32 Version_; - TString ServiceType_; + TString ServiceType_; }; struct TRemoteMirrorRule { @@ -180,7 +180,7 @@ struct TReadRuleSettings { FLUENT_SETTING_DEFAULT(TVector<ECodec>, SupportedCodecs, GetDefaultCodecs()); FLUENT_SETTING_DEFAULT(ui32, Version, 0); - FLUENT_SETTING(TString, ServiceType); + FLUENT_SETTING(TString, ServiceType); TReadRuleSettings& SetSettings(const TDescribeTopicResult::TTopicSettings::TReadRule& settings) { ConsumerName_ = settings.ConsumerName(); @@ -192,7 +192,7 @@ struct TReadRuleSettings { SupportedCodecs_.push_back(codec); } Version_ = settings.Version(); - ServiceType_ = settings.ServiceType(); + ServiceType_ = settings.ServiceType(); return *this; } @@ -476,210 +476,210 @@ struct TReadSessionEvent { //! Contains batch of messages from single partition stream. struct TDataReceivedEvent { - struct TMessageInformation { - TMessageInformation(ui64 offset, - TString messageGroupId, - ui64 seqNo, - TInstant createTime, - TInstant writeTime, - TString ip, - TWriteSessionMeta::TPtr meta, - ui64 uncompressedSize); - ui64 Offset; - TString MessageGroupId; - ui64 SeqNo; - TInstant CreateTime; - TInstant WriteTime; - TString Ip; - TWriteSessionMeta::TPtr Meta; - ui64 UncompressedSize; - }; - - class IMessage { - public: - virtual const TString& GetData() const; - - //! Partition stream. Same as in batch. - const TPartitionStream::TPtr& GetPartitionStream() const; - - const TString& GetPartitionKey() const; - - const TString GetExplicitHash() const; - - virtual void Commit() = 0; - - TString DebugString(bool printData = false) const; - virtual void DebugString(TStringBuilder& ret, bool printData = false) const = 0; - - IMessage(const TString& data, - TPartitionStream::TPtr partitionStream, - const TString& partitionKey, - const TString& explicitHash); - + struct TMessageInformation { + TMessageInformation(ui64 offset, + TString messageGroupId, + ui64 seqNo, + TInstant createTime, + TInstant writeTime, + TString ip, + TWriteSessionMeta::TPtr meta, + ui64 uncompressedSize); + ui64 Offset; + TString MessageGroupId; + ui64 SeqNo; + TInstant CreateTime; + TInstant WriteTime; + TString Ip; + TWriteSessionMeta::TPtr Meta; + ui64 UncompressedSize; + }; + + class IMessage { + public: + virtual const TString& GetData() const; + + //! Partition stream. Same as in batch. + const TPartitionStream::TPtr& GetPartitionStream() const; + + const TString& GetPartitionKey() const; + + const TString GetExplicitHash() const; + + virtual void Commit() = 0; + + TString DebugString(bool printData = false) const; + virtual void DebugString(TStringBuilder& ret, bool printData = false) const = 0; + + IMessage(const TString& data, + TPartitionStream::TPtr partitionStream, + const TString& partitionKey, + const TString& explicitHash); + virtual ~IMessage() = default; - protected: - TString Data; - - TPartitionStream::TPtr PartitionStream; - TString PartitionKey; - TString ExplicitHash; - }; - + protected: + TString Data; + + TPartitionStream::TPtr PartitionStream; + TString PartitionKey; + TString ExplicitHash; + }; + //! Single message. - struct TMessage : public IMessage { + struct TMessage : public IMessage { //! User data. //! Throws decompressor exception if decompression failed. - const TString& GetData() const override; + const TString& GetData() const override; - bool HasException() const; + bool HasException() const; //! Message offset. - ui64 GetOffset() const; + ui64 GetOffset() const; //! Message group id. - const TString& GetMessageGroupId() const; + const TString& GetMessageGroupId() const; //! Sequence number. - ui64 GetSeqNo() const; + ui64 GetSeqNo() const; //! Message creation timestamp. - TInstant GetCreateTime() const; + TInstant GetCreateTime() const; //! Message write timestamp. - TInstant GetWriteTime() const; + TInstant GetWriteTime() const; //! Ip address of message source host. - const TString& GetIp() const; + const TString& GetIp() const; //! Metainfo. - const TWriteSessionMeta::TPtr& GetMeta() const; + const TWriteSessionMeta::TPtr& GetMeta() const; TMessage(const TString& data, std::exception_ptr decompressionException, - const TMessageInformation& information, + const TMessageInformation& information, TPartitionStream::TPtr partitionStream, const TString& partitionKey, const TString& explicitHash); - //! Commits single message. - void Commit() override; - - using IMessage::DebugString; - void DebugString(TStringBuilder& ret, bool printData = false) const override; - + //! Commits single message. + void Commit() override; + + using IMessage::DebugString; + void DebugString(TStringBuilder& ret, bool printData = false) const override; + private: std::exception_ptr DecompressionException; - TMessageInformation Information; + TMessageInformation Information; + }; + + struct TCompressedMessage : public IMessage { + //! Messages count in compressed data + ui64 GetBlocksCount() const; + + //! Message codec + ECodec GetCodec() const; + + //! Message offset. + ui64 GetOffset(ui64 index) const; + + //! Message group id. + const TString& GetMessageGroupId(ui64 index) const; + + //! Sequence number. + ui64 GetSeqNo(ui64 index) const; + + //! Message creation timestamp. + TInstant GetCreateTime(ui64 index) const; + + //! Message write timestamp. + TInstant GetWriteTime(ui64 index) const; + + //! Ip address of message source host. + const TString& GetIp(ui64 index) const; + + //! Metainfo. + const TWriteSessionMeta::TPtr& GetMeta(ui64 index) const; + + //! Uncompressed block size. + ui64 GetUncompressedSize(ui64 index) const; + + virtual ~TCompressedMessage() {} + TCompressedMessage(ECodec codec, + const TString& data, + const TVector<TMessageInformation>& information, + TPartitionStream::TPtr partitionStream, + const TString& partitionKey, + const TString& explicitHash); + + //! Commits all offsets in compressed message. + void Commit() override; + + using IMessage::DebugString; + void DebugString(TStringBuilder& ret, bool printData = false) const override; + + private: + ECodec Codec; + TVector<TMessageInformation> Information; }; - struct TCompressedMessage : public IMessage { - //! Messages count in compressed data - ui64 GetBlocksCount() const; - - //! Message codec - ECodec GetCodec() const; - - //! Message offset. - ui64 GetOffset(ui64 index) const; - - //! Message group id. - const TString& GetMessageGroupId(ui64 index) const; - - //! Sequence number. - ui64 GetSeqNo(ui64 index) const; - - //! Message creation timestamp. - TInstant GetCreateTime(ui64 index) const; - - //! Message write timestamp. - TInstant GetWriteTime(ui64 index) const; - - //! Ip address of message source host. - const TString& GetIp(ui64 index) const; - - //! Metainfo. - const TWriteSessionMeta::TPtr& GetMeta(ui64 index) const; - - //! Uncompressed block size. - ui64 GetUncompressedSize(ui64 index) const; - - virtual ~TCompressedMessage() {} - TCompressedMessage(ECodec codec, - const TString& data, - const TVector<TMessageInformation>& information, - TPartitionStream::TPtr partitionStream, - const TString& partitionKey, - const TString& explicitHash); - - //! Commits all offsets in compressed message. - void Commit() override; - - using IMessage::DebugString; - void DebugString(TStringBuilder& ret, bool printData = false) const override; - - private: - ECodec Codec; - TVector<TMessageInformation> Information; - }; - //! Partition stream. const TPartitionStream::TPtr& GetPartitionStream() const { return PartitionStream; } - bool IsCompressedMessages() const { - return !CompressedMessages.empty(); - } - - size_t GetMessagesCount() const { - return Messages.size() + CompressedMessages.size(); - } - + bool IsCompressedMessages() const { + return !CompressedMessages.empty(); + } + + size_t GetMessagesCount() const { + return Messages.size() + CompressedMessages.size(); + } + //! Get messages. TVector<TMessage>& GetMessages() { - CheckMessagesFilled(false); + CheckMessagesFilled(false); return Messages; } const TVector<TMessage>& GetMessages() const { - CheckMessagesFilled(false); + CheckMessagesFilled(false); return Messages; } - //! Get compressed messages. - TVector<TCompressedMessage>& GetCompressedMessages() { - CheckMessagesFilled(true); - return CompressedMessages; - } - - const TVector<TCompressedMessage>& GetCompressedMessages() const { - CheckMessagesFilled(true); - return CompressedMessages; - } - + //! Get compressed messages. + TVector<TCompressedMessage>& GetCompressedMessages() { + CheckMessagesFilled(true); + return CompressedMessages; + } + + const TVector<TCompressedMessage>& GetCompressedMessages() const { + CheckMessagesFilled(true); + return CompressedMessages; + } + //! Commits all messages in batch. void Commit(); TString DebugString(bool printData = false) const; TDataReceivedEvent(TVector<TMessage> messages, - TVector<TCompressedMessage> compressedMessages, + TVector<TCompressedMessage> compressedMessages, TPartitionStream::TPtr partitionStream); private: - void CheckMessagesFilled(bool compressed) const { - Y_VERIFY(!Messages.empty() || !CompressedMessages.empty()); - if (compressed && CompressedMessages.empty()) { - ythrow yexception() << "cannot get compressed messages, parameter decompress=true for read session"; - } - if (!compressed && Messages.empty()) { - ythrow yexception() << "cannot get decompressed messages, parameter decompress=false for read session"; - } - } - - private: + void CheckMessagesFilled(bool compressed) const { + Y_VERIFY(!Messages.empty() || !CompressedMessages.empty()); + if (compressed && CompressedMessages.empty()) { + ythrow yexception() << "cannot get compressed messages, parameter decompress=true for read session"; + } + if (!compressed && Messages.empty()) { + ythrow yexception() << "cannot get decompressed messages, parameter decompress=false for read session"; + } + } + + private: TVector<TMessage> Messages; - TVector<TCompressedMessage> CompressedMessages; + TVector<TCompressedMessage> CompressedMessages; TPartitionStream::TPtr PartitionStream; std::vector<std::pair<ui64, ui64>> OffsetRanges; }; @@ -853,12 +853,12 @@ public: //! Add all messages from dataReceivedEvent to set. void Add(const TReadSessionEvent::TDataReceivedEvent& dataReceivedEvent); - //! Add offsets range to set. - void Add(const TPartitionStream::TPtr& partitionStream, ui64 startOffset, ui64 endOffset); - - //! Add offset to set. - void Add(const TPartitionStream::TPtr& partitionStream, ui64 offset); - + //! Add offsets range to set. + void Add(const TPartitionStream::TPtr& partitionStream, ui64 startOffset, ui64 endOffset); + + //! Add offset to set. + void Add(const TPartitionStream::TPtr& partitionStream, ui64 offset); + //! Commit all added offsets. void Commit(); @@ -1296,9 +1296,9 @@ struct TReadSessionSettings : public TRequestSettings<TReadSessionSettings> { //! See description in TEventHandlers class. FLUENT_SETTING(TEventHandlers, EventHandlers); - //! Decompress messages - FLUENT_SETTING_DEFAULT(bool, Decompress, true); - + //! Decompress messages + FLUENT_SETTING_DEFAULT(bool, Decompress, true); + //! Executor for decompression tasks. //! If not set, default executor will be used. FLUENT_SETTING(IExecutor::TPtr, DecompressionExecutor); diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/basic_usage_ut.cpp b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/basic_usage_ut.cpp index d26bc6948c..7e0ac17404 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/basic_usage_ut.cpp +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/basic_usage_ut.cpp @@ -3,15 +3,15 @@ #include <library/cpp/testing/unittest/registar.h> #include <library/cpp/threading/future/future.h> #include <library/cpp/threading/future/async.h> -#include <library/cpp/threading/chunk_queue/queue.h> - +#include <library/cpp/threading/chunk_queue/queue.h> + #include <util/generic/ptr.h> #include <util/generic/queue.h> #include <util/string/join.h> #include <util/system/event.h> #include <util/stream/zlib.h> #include <util/stream/str.h> - + #include <atomic> using namespace NThreading; diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/compress_executor_ut.cpp b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/compress_executor_ut.cpp index 081f08da4a..04abfab569 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/compress_executor_ut.cpp +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/compress_executor_ut.cpp @@ -1,4 +1,4 @@ -#include "ut_utils.h" +#include "ut_utils.h" namespace NYdb::NPersQueue::NTests { diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/read_session_ut.cpp b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/read_session_ut.cpp index bf780ce78f..9b2367e3f3 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/read_session_ut.cpp +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/read_session_ut.cpp @@ -1,12 +1,12 @@ -#include "ut_utils.h" +#include "ut_utils.h" -#define INCLUDE_YDB_INTERNAL_H +#define INCLUDE_YDB_INTERNAL_H #include <ydb/public/sdk/cpp/client/impl/ydb_internal/logger/log.h> -#undef INCLUDE_YDB_INTERNAL_H - +#undef INCLUDE_YDB_INTERNAL_H + #include <ydb/public/sdk/cpp/client/ydb_persqueue_core/persqueue.h> #include <ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/read_session.h> - + #include <library/cpp/streams/zstd/zstd.h> #include <library/cpp/testing/gmock_in_unittest/gmock.h> #include <library/cpp/testing/unittest/registar.h> diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/retry_policy_ut.cpp b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/retry_policy_ut.cpp index 74ae538f06..50d050e973 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/retry_policy_ut.cpp +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/retry_policy_ut.cpp @@ -1,5 +1,5 @@ -#include "ut_utils.h" - +#include "ut_utils.h" + #include <library/cpp/threading/future/future.h> #include <library/cpp/testing/unittest/registar.h> diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/data_plane_helpers.cpp b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/data_plane_helpers.cpp index 0c9cd6d70e..3cf6e7f8b8 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/data_plane_helpers.cpp +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/data_plane_helpers.cpp @@ -1,4 +1,4 @@ -#include "data_plane_helpers.h" +#include "data_plane_helpers.h" namespace NKikimr::NPersQueueTests { diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/sdk_test_setup.h b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/sdk_test_setup.h index 3fe421d05b..677eb8c03d 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/sdk_test_setup.h +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/sdk_test_setup.h @@ -87,9 +87,9 @@ public: return LocalDC; } - ui16 GetGrpcPort() const { - return Server.GrpcPort; - } + ui16 GetGrpcPort() const { + return Server.GrpcPort; + } NGrpc::TServerOptions& GetGrpcServerOptions() { return Server.GrpcServerOptions; diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/ya.make b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/ya.make index 93643a245b..b5f051599c 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/ya.make +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/ya.make @@ -3,7 +3,7 @@ LIBRARY() OWNER(g:logbroker) SRCS( - data_plane_helpers.cpp + data_plane_helpers.cpp sdk_test_setup.h test_utils.h test_server.h diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/with_offset_ranges_mode_ut/ya.make b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/with_offset_ranges_mode_ut/ya.make index 7d02dd4644..fc27259e6e 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/with_offset_ranges_mode_ut/ya.make +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/with_offset_ranges_mode_ut/ya.make @@ -2,7 +2,7 @@ UNITTEST() OWNER( g:kikimr - g:logbroker + g:logbroker ) IF (SANITIZER_TYPE) diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ya.make b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ya.make index 2ddf7a1b78..a95f5de06f 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ya.make +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ya.make @@ -2,7 +2,7 @@ UNITTEST_FOR(ydb/public/sdk/cpp/client/ydb_persqueue_core) OWNER( g:kikimr - g:logbroker + g:logbroker ) IF (SANITIZER_TYPE) @@ -35,7 +35,7 @@ SRCS( read_session_ut.cpp basic_usage_ut.cpp compress_executor_ut.cpp - compression_ut.cpp + compression_ut.cpp retry_policy_ut.cpp ut_utils.cpp ) diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ya.make b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ya.make index 60235628ba..d19d4f6b2e 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ya.make +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ya.make @@ -1,24 +1,24 @@ -LIBRARY() - -OWNER( +LIBRARY() + +OWNER( g:kikimr - g:logbroker -) - + g:logbroker +) + GENERATE_ENUM_SERIALIZATION(ydb/public/sdk/cpp/client/ydb_persqueue_core/persqueue.h) - -SRCS( + +SRCS( persqueue.h proto_accessor.cpp -) - -PEERDIR( +) + +PEERDIR( ydb/public/sdk/cpp/client/ydb_persqueue_core/impl ydb/public/sdk/cpp/client/ydb_proto -) - -END() - -RECURSE_FOR_TESTS( - ut -) +) + +END() + +RECURSE_FOR_TESTS( + ut +) diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_public/codecs/codecs.cpp b/ydb/public/sdk/cpp/client/ydb_persqueue_public/codecs/codecs.cpp index a33205780b..9c3bcbd25c 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_public/codecs/codecs.cpp +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_public/codecs/codecs.cpp @@ -21,7 +21,7 @@ IInputStream* CreateDecompressorStream(TInputStreamVariant& inputStreamStorage, default: //case Ydb::PersQueue::V1::CODEC_RAW: //case Ydb::PersQueue::V1::CODEC_UNSPECIFIED: - throw yexception() << "unsupported codec value : " << ui64(codec); + throw yexception() << "unsupported codec value : " << ui64(codec); } } diff --git a/ydb/public/sdk/cpp/client/ydb_proto/accessor.cpp b/ydb/public/sdk/cpp/client/ydb_proto/accessor.cpp index a50b398b0f..dd02fb89db 100644 --- a/ydb/public/sdk/cpp/client/ydb_proto/accessor.cpp +++ b/ydb/public/sdk/cpp/client/ydb_proto/accessor.cpp @@ -1,7 +1,7 @@ -#include "accessor.h" +#include "accessor.h" #include <ydb/public/sdk/cpp/client/ydb_value/value.h> - + namespace NYdb { const Ydb::Type& TProtoAccessor::GetProto(const TType& type) { diff --git a/ydb/public/sdk/cpp/client/ydb_proto/accessor.h b/ydb/public/sdk/cpp/client/ydb_proto/accessor.h index 907e3ccc66..be0165d74b 100644 --- a/ydb/public/sdk/cpp/client/ydb_proto/accessor.h +++ b/ydb/public/sdk/cpp/client/ydb_proto/accessor.h @@ -11,19 +11,19 @@ #include <ydb/public/sdk/cpp/client/ydb_import/import.h> #include <ydb/public/sdk/cpp/client/ydb_table/table.h> #include <ydb/public/sdk/cpp/client/ydb_persqueue_public/persqueue.h> - + namespace NYdb { -class TResultSet; -class TValue; -class TType; - -namespace NTable { -class TQueryStats; -class TTableDescription; +class TResultSet; +class TValue; +class TType; + +namespace NTable { +class TQueryStats; +class TTableDescription; class TIndexDescription; -} - +} + //! Provides access to raw protobuf values of YDB API entities. It is not recommended to use this //! class in client applications as it add dependency on API protobuf format which is subject to //! change. Use functionality provided by YDB SDK classes. diff --git a/ydb/public/sdk/cpp/client/ydb_proto/ya.make b/ydb/public/sdk/cpp/client/ydb_proto/ya.make index 02f26c7674..68f3d76432 100644 --- a/ydb/public/sdk/cpp/client/ydb_proto/ya.make +++ b/ydb/public/sdk/cpp/client/ydb_proto/ya.make @@ -1,19 +1,19 @@ -LIBRARY() - -OWNER( - dcherednik - g:kikimr -) - -SRCS( - accessor.cpp -) - -PEERDIR( +LIBRARY() + +OWNER( + dcherednik + g:kikimr +) + +SRCS( + accessor.cpp +) + +PEERDIR( ydb/public/api/grpc/draft ydb/public/api/protos ydb/public/lib/operation_id/protos ydb/library/yql/public/issue/protos -) - -END() +) + +END() diff --git a/ydb/public/sdk/cpp/client/ydb_rate_limiter/rate_limiter.cpp b/ydb/public/sdk/cpp/client/ydb_rate_limiter/rate_limiter.cpp index 0946263a56..bea557812c 100644 --- a/ydb/public/sdk/cpp/client/ydb_rate_limiter/rate_limiter.cpp +++ b/ydb/public/sdk/cpp/client/ydb_rate_limiter/rate_limiter.cpp @@ -1,9 +1,9 @@ -#include "rate_limiter.h" +#include "rate_limiter.h" -#define INCLUDE_YDB_INTERNAL_H +#define INCLUDE_YDB_INTERNAL_H #include <ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.h> -#undef INCLUDE_YDB_INTERNAL_H - +#undef INCLUDE_YDB_INTERNAL_H + #include <ydb/public/api/grpc/ydb_rate_limiter_v1.grpc.pb.h> #include <ydb/public/sdk/cpp/client/ydb_common_client/impl/client.h> @@ -186,7 +186,7 @@ public: &Ydb::RateLimiter::V1::RateLimiterService::Stub::AsyncAcquireResource, TRpcRequestSettings::Make(settings), settings.ClientTimeout_); - } + } }; TRateLimiterClient::TRateLimiterClient(const TDriver& driver, const TCommonClientSettings& settings) diff --git a/ydb/public/sdk/cpp/client/ydb_rate_limiter/rate_limiter.h b/ydb/public/sdk/cpp/client/ydb_rate_limiter/rate_limiter.h index 4f91206c79..dca90fbe63 100644 --- a/ydb/public/sdk/cpp/client/ydb_rate_limiter/rate_limiter.h +++ b/ydb/public/sdk/cpp/client/ydb_rate_limiter/rate_limiter.h @@ -161,7 +161,7 @@ public: TAsyncDescribeResourceResult DescribeResource(const TString& coordinationNodePath, const TString& resourcePath, const TDescribeResourceSettings& = {}); // Acquire resources's units inside a coordination node. - TAsyncStatus AcquireResource(const TString& coordinationNodePath, const TString& resourcePath, const TAcquireResourceSettings& = {}); + TAsyncStatus AcquireResource(const TString& coordinationNodePath, const TString& resourcePath, const TAcquireResourceSettings& = {}); private: class TImpl; diff --git a/ydb/public/sdk/cpp/client/ydb_rate_limiter/ya.make b/ydb/public/sdk/cpp/client/ydb_rate_limiter/ya.make index 4c02b74c17..45f1d6d1f1 100644 --- a/ydb/public/sdk/cpp/client/ydb_rate_limiter/ya.make +++ b/ydb/public/sdk/cpp/client/ydb_rate_limiter/ya.make @@ -1,18 +1,18 @@ -LIBRARY() - -OWNER( - dcherednik - g:kikimr -) - -SRCS( - rate_limiter.cpp -) - -PEERDIR( +LIBRARY() + +OWNER( + dcherednik + g:kikimr +) + +SRCS( + rate_limiter.cpp +) + +PEERDIR( ydb/public/api/grpc ydb/public/sdk/cpp/client/ydb_common_client/impl ydb/public/sdk/cpp/client/ydb_driver -) - -END() +) + +END() diff --git a/ydb/public/sdk/cpp/client/ydb_result/proto_accessor.cpp b/ydb/public/sdk/cpp/client/ydb_result/proto_accessor.cpp index 3b85ce84f5..3ff390d4ce 100644 --- a/ydb/public/sdk/cpp/client/ydb_result/proto_accessor.cpp +++ b/ydb/public/sdk/cpp/client/ydb_result/proto_accessor.cpp @@ -1,11 +1,11 @@ -#include "result.h" - +#include "result.h" + #include <ydb/public/sdk/cpp/client/ydb_proto/accessor.h> - -namespace NYdb { - -const Ydb::ResultSet& TProtoAccessor::GetProto(const TResultSet& resultSet) { - return resultSet.GetProto(); -} - -} // namespace NYdb + +namespace NYdb { + +const Ydb::ResultSet& TProtoAccessor::GetProto(const TResultSet& resultSet) { + return resultSet.GetProto(); +} + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/ydb_result/result.cpp b/ydb/public/sdk/cpp/client/ydb_result/result.cpp index 0b1420a10f..dc7eb09c0a 100644 --- a/ydb/public/sdk/cpp/client/ydb_result/result.cpp +++ b/ydb/public/sdk/cpp/client/ydb_result/result.cpp @@ -1,11 +1,11 @@ -#include "result.h" +#include "result.h" #include <ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers/handlers.h> - + #include <ydb/public/api/protos/ydb_common.pb.h> #include <ydb/public/api/protos/ydb_value.pb.h> -#include <util/generic/map.h> +#include <util/generic/map.h> #include <util/string/builder.h> #include <google/protobuf/text_format.h> diff --git a/ydb/public/sdk/cpp/client/ydb_result/result_ut.cpp b/ydb/public/sdk/cpp/client/ydb_result/result_ut.cpp index 3c26543938..16c5fa5db2 100644 --- a/ydb/public/sdk/cpp/client/ydb_result/result_ut.cpp +++ b/ydb/public/sdk/cpp/client/ydb_result/result_ut.cpp @@ -1,196 +1,196 @@ #include <ydb/public/api/protos/ydb_value.pb.h> #include <ydb/public/sdk/cpp/client/ydb_result/result.h> - -#include <library/cpp/testing/unittest/registar.h> -#include <library/cpp/testing/unittest/tests_data.h> - -#include <google/protobuf/text_format.h> - -using namespace NYdb; - -Y_UNIT_TEST_SUITE(CppGrpcClientResultSetTest) { - Y_UNIT_TEST(ListResultSet) { - const TString resultSetString = - "columns {\n" - " name: \"colName\"\n" - " type {\n" - " list_type {\n" - " item {\n" - " type_id: INT32\n" - " }\n" - " }\n" - " }\n" - "}\n" - "rows {\n" - " items {\n" - " items {\n" - " int32_value: 42\n" - " }\n" - " items {\n" - " int32_value: 43\n" - " }\n" - " items {\n" - " int32_value: 44\n" - " }\n" - " }\n" - "}\n" - "rows {\n" - " items {\n" - " items {\n" - " int32_value: 45\n" - " }\n" - " items {\n" - " int32_value: 46\n" - " }\n" - " items {\n" - " int32_value: 47\n" - " }\n" - " }\n" - "}\n"; - Ydb::ResultSet rsProto; - google::protobuf::TextFormat::ParseFromString(resultSetString, &rsProto); - - NYdb::TResultSet rs(std::move(rsProto)); - NYdb::TResultSetParser rsParser(rs); - UNIT_ASSERT_EQUAL(rsParser.ColumnsCount(), 1); - UNIT_ASSERT_EQUAL(rsParser.RowsCount(), 2); - UNIT_ASSERT_EQUAL(rsParser.ColumnIndex("colName"), 0); - UNIT_ASSERT_EQUAL(rsParser.ColumnIndex("otherName"), -1); - int expectedVal = 42; - auto& column0 = rsParser.ColumnParser(0); - while (rsParser.TryNextRow()) { - column0.OpenList(); - while (column0.TryNextListItem()) { - UNIT_ASSERT_EQUAL(column0.GetInt32(), expectedVal++); - } - } - } - - Y_UNIT_TEST(OptionalDictResultSet) { - const TString resultSetString = - "columns {\n" - " name: \"colName\"\n" - " type {\n" - " optional_type {\n" - " item {\n" - " dict_type {\n" - " key {\n" - " type_id: INT32\n" - " }\n" - " payload {\n" - " type_id: STRING\n" - " }\n" - " }\n" - " }\n" - " }\n" - " }\n" - "}\n" - "rows {\n" - " items {\n" - " pairs {\n" - " key {\n" - " int32_value: 42\n" - " }\n" - " payload {\n" - " bytes_value: \"abc\"\n" - " }\n" - " }\n" - " }\n" - "}\n" - "rows {\n" - " items {\n" - " pairs {\n" - " key {\n" - " int32_value: 43\n" - " }\n" - " payload {\n" - " bytes_value: \"zxc\"\n" - " }\n" - " }\n" - " }\n" - "}\n"; - Ydb::ResultSet rsProto; - google::protobuf::TextFormat::ParseFromString(resultSetString, &rsProto); - - NYdb::TResultSet rs(std::move(rsProto)); - NYdb::TResultSetParser rsParser(rs); - UNIT_ASSERT_EQUAL(rsParser.ColumnsCount(), 1); - UNIT_ASSERT_EQUAL(rsParser.RowsCount(), 2); - UNIT_ASSERT_EQUAL(rsParser.ColumnIndex("colName"), 0); - UNIT_ASSERT_EQUAL(rsParser.ColumnIndex("otherName"), -1); - auto& column0 = rsParser.ColumnParser(0); - - UNIT_ASSERT(rsParser.TryNextRow()); - column0.OpenOptional(); - UNIT_ASSERT(!column0.IsNull()); - column0.OpenDict(); - UNIT_ASSERT(column0.TryNextDictItem()); - column0.DictKey(); - UNIT_ASSERT_EQUAL(column0.GetInt32(), 42); - column0.DictPayload(); - UNIT_ASSERT_EQUAL(column0.GetString(), "abc"); - - UNIT_ASSERT(rsParser.TryNextRow()); - column0.OpenOptional(); - UNIT_ASSERT(!column0.IsNull()); - column0.OpenDict(); - UNIT_ASSERT(column0.TryNextDictItem()); - column0.DictKey(); - UNIT_ASSERT_EQUAL(column0.GetInt32(), 43); - column0.DictPayload(); - UNIT_ASSERT_EQUAL(column0.GetString(), "zxc"); - column0.DictKey(); - column0.DictKey(); - UNIT_ASSERT_EQUAL(column0.GetInt32(), 43); - UNIT_ASSERT_EQUAL(column0.GetInt32(), 43); - column0.DictPayload(); - UNIT_ASSERT_EQUAL(column0.GetString(), "zxc"); - UNIT_ASSERT_EQUAL(column0.GetString(), "zxc"); - } - - Y_UNIT_TEST(Utf8OptionalResultSet) { - const TString resultSetString = - "columns {\n" - " name: \"colName\"\n" - " type {\n" - " optional_type {\n" - " item {\n" - " type_id: UTF8\n" - " }\n" - " }\n" - " }\n" - "}\n" - "rows {\n" - " items {\n" - " text_value: \"йцукен\"\n" - " }\n" - "}\n" - "rows {\n" - " items {\n" - " null_flag_value: NULL_VALUE\n" - " }\n" - "}\n"; - Ydb::ResultSet rsProto; - google::protobuf::TextFormat::ParseFromString(resultSetString, &rsProto); - - NYdb::TResultSet rs(std::move(rsProto)); - NYdb::TResultSetParser rsParser(rs); - UNIT_ASSERT_EQUAL(rsParser.ColumnsCount(), 1); - UNIT_ASSERT_EQUAL(rsParser.RowsCount(), 2); - auto& column0 = rsParser.ColumnParser(0); - - int row = 0; - while (rsParser.TryNextRow()) { - column0.OpenOptional(); - - if (row == 0) { - UNIT_ASSERT(!column0.IsNull()); - UNIT_ASSERT_EQUAL(column0.GetUtf8(), "йцукен"); - } else { - UNIT_ASSERT(column0.IsNull()); - } - - row++; - } - } -} + +#include <library/cpp/testing/unittest/registar.h> +#include <library/cpp/testing/unittest/tests_data.h> + +#include <google/protobuf/text_format.h> + +using namespace NYdb; + +Y_UNIT_TEST_SUITE(CppGrpcClientResultSetTest) { + Y_UNIT_TEST(ListResultSet) { + const TString resultSetString = + "columns {\n" + " name: \"colName\"\n" + " type {\n" + " list_type {\n" + " item {\n" + " type_id: INT32\n" + " }\n" + " }\n" + " }\n" + "}\n" + "rows {\n" + " items {\n" + " items {\n" + " int32_value: 42\n" + " }\n" + " items {\n" + " int32_value: 43\n" + " }\n" + " items {\n" + " int32_value: 44\n" + " }\n" + " }\n" + "}\n" + "rows {\n" + " items {\n" + " items {\n" + " int32_value: 45\n" + " }\n" + " items {\n" + " int32_value: 46\n" + " }\n" + " items {\n" + " int32_value: 47\n" + " }\n" + " }\n" + "}\n"; + Ydb::ResultSet rsProto; + google::protobuf::TextFormat::ParseFromString(resultSetString, &rsProto); + + NYdb::TResultSet rs(std::move(rsProto)); + NYdb::TResultSetParser rsParser(rs); + UNIT_ASSERT_EQUAL(rsParser.ColumnsCount(), 1); + UNIT_ASSERT_EQUAL(rsParser.RowsCount(), 2); + UNIT_ASSERT_EQUAL(rsParser.ColumnIndex("colName"), 0); + UNIT_ASSERT_EQUAL(rsParser.ColumnIndex("otherName"), -1); + int expectedVal = 42; + auto& column0 = rsParser.ColumnParser(0); + while (rsParser.TryNextRow()) { + column0.OpenList(); + while (column0.TryNextListItem()) { + UNIT_ASSERT_EQUAL(column0.GetInt32(), expectedVal++); + } + } + } + + Y_UNIT_TEST(OptionalDictResultSet) { + const TString resultSetString = + "columns {\n" + " name: \"colName\"\n" + " type {\n" + " optional_type {\n" + " item {\n" + " dict_type {\n" + " key {\n" + " type_id: INT32\n" + " }\n" + " payload {\n" + " type_id: STRING\n" + " }\n" + " }\n" + " }\n" + " }\n" + " }\n" + "}\n" + "rows {\n" + " items {\n" + " pairs {\n" + " key {\n" + " int32_value: 42\n" + " }\n" + " payload {\n" + " bytes_value: \"abc\"\n" + " }\n" + " }\n" + " }\n" + "}\n" + "rows {\n" + " items {\n" + " pairs {\n" + " key {\n" + " int32_value: 43\n" + " }\n" + " payload {\n" + " bytes_value: \"zxc\"\n" + " }\n" + " }\n" + " }\n" + "}\n"; + Ydb::ResultSet rsProto; + google::protobuf::TextFormat::ParseFromString(resultSetString, &rsProto); + + NYdb::TResultSet rs(std::move(rsProto)); + NYdb::TResultSetParser rsParser(rs); + UNIT_ASSERT_EQUAL(rsParser.ColumnsCount(), 1); + UNIT_ASSERT_EQUAL(rsParser.RowsCount(), 2); + UNIT_ASSERT_EQUAL(rsParser.ColumnIndex("colName"), 0); + UNIT_ASSERT_EQUAL(rsParser.ColumnIndex("otherName"), -1); + auto& column0 = rsParser.ColumnParser(0); + + UNIT_ASSERT(rsParser.TryNextRow()); + column0.OpenOptional(); + UNIT_ASSERT(!column0.IsNull()); + column0.OpenDict(); + UNIT_ASSERT(column0.TryNextDictItem()); + column0.DictKey(); + UNIT_ASSERT_EQUAL(column0.GetInt32(), 42); + column0.DictPayload(); + UNIT_ASSERT_EQUAL(column0.GetString(), "abc"); + + UNIT_ASSERT(rsParser.TryNextRow()); + column0.OpenOptional(); + UNIT_ASSERT(!column0.IsNull()); + column0.OpenDict(); + UNIT_ASSERT(column0.TryNextDictItem()); + column0.DictKey(); + UNIT_ASSERT_EQUAL(column0.GetInt32(), 43); + column0.DictPayload(); + UNIT_ASSERT_EQUAL(column0.GetString(), "zxc"); + column0.DictKey(); + column0.DictKey(); + UNIT_ASSERT_EQUAL(column0.GetInt32(), 43); + UNIT_ASSERT_EQUAL(column0.GetInt32(), 43); + column0.DictPayload(); + UNIT_ASSERT_EQUAL(column0.GetString(), "zxc"); + UNIT_ASSERT_EQUAL(column0.GetString(), "zxc"); + } + + Y_UNIT_TEST(Utf8OptionalResultSet) { + const TString resultSetString = + "columns {\n" + " name: \"colName\"\n" + " type {\n" + " optional_type {\n" + " item {\n" + " type_id: UTF8\n" + " }\n" + " }\n" + " }\n" + "}\n" + "rows {\n" + " items {\n" + " text_value: \"йцукен\"\n" + " }\n" + "}\n" + "rows {\n" + " items {\n" + " null_flag_value: NULL_VALUE\n" + " }\n" + "}\n"; + Ydb::ResultSet rsProto; + google::protobuf::TextFormat::ParseFromString(resultSetString, &rsProto); + + NYdb::TResultSet rs(std::move(rsProto)); + NYdb::TResultSetParser rsParser(rs); + UNIT_ASSERT_EQUAL(rsParser.ColumnsCount(), 1); + UNIT_ASSERT_EQUAL(rsParser.RowsCount(), 2); + auto& column0 = rsParser.ColumnParser(0); + + int row = 0; + while (rsParser.TryNextRow()) { + column0.OpenOptional(); + + if (row == 0) { + UNIT_ASSERT(!column0.IsNull()); + UNIT_ASSERT_EQUAL(column0.GetUtf8(), "йцукен"); + } else { + UNIT_ASSERT(column0.IsNull()); + } + + row++; + } + } +} diff --git a/ydb/public/sdk/cpp/client/ydb_result/ut/ya.make b/ydb/public/sdk/cpp/client/ydb_result/ut/ya.make index e9c668d542..d99231cee7 100644 --- a/ydb/public/sdk/cpp/client/ydb_result/ut/ya.make +++ b/ydb/public/sdk/cpp/client/ydb_result/ut/ya.make @@ -1,23 +1,23 @@ UNITTEST_FOR(ydb/public/sdk/cpp/client/ydb_result) - -OWNER( - dcherednik - g:kikimr -) - -IF (SANITIZER_TYPE) - TIMEOUT(1200) - SIZE(LARGE) - TAG(ya:fat) -ELSE() - TIMEOUT(600) - SIZE(MEDIUM) -ENDIF() - -FORK_SUBTESTS() - -SRCS( - result_ut.cpp -) - -END() + +OWNER( + dcherednik + g:kikimr +) + +IF (SANITIZER_TYPE) + TIMEOUT(1200) + SIZE(LARGE) + TAG(ya:fat) +ELSE() + TIMEOUT(600) + SIZE(MEDIUM) +ENDIF() + +FORK_SUBTESTS() + +SRCS( + result_ut.cpp +) + +END() diff --git a/ydb/public/sdk/cpp/client/ydb_result/ya.make b/ydb/public/sdk/cpp/client/ydb_result/ya.make index b67b5e0d55..5bccb8e3f4 100644 --- a/ydb/public/sdk/cpp/client/ydb_result/ya.make +++ b/ydb/public/sdk/cpp/client/ydb_result/ya.make @@ -1,23 +1,23 @@ -LIBRARY() - -OWNER( - dcherednik - g:kikimr -) - -SRCS( - proto_accessor.cpp - result.cpp -) - -PEERDIR( +LIBRARY() + +OWNER( + dcherednik + g:kikimr +) + +SRCS( + proto_accessor.cpp + result.cpp +) + +PEERDIR( ydb/public/api/protos ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers ydb/public/sdk/cpp/client/ydb_value -) - -END() - +) + +END() + RECURSE_FOR_TESTS( ut ) diff --git a/ydb/public/sdk/cpp/client/ydb_scheme/scheme.cpp b/ydb/public/sdk/cpp/client/ydb_scheme/scheme.cpp index 6b82d5a243..31f5535b96 100644 --- a/ydb/public/sdk/cpp/client/ydb_scheme/scheme.cpp +++ b/ydb/public/sdk/cpp/client/ydb_scheme/scheme.cpp @@ -1,14 +1,14 @@ -#include "scheme.h" +#include "scheme.h" -#define INCLUDE_YDB_INTERNAL_H +#define INCLUDE_YDB_INTERNAL_H #include <ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.h> #include <ydb/public/sdk/cpp/client/impl/ydb_internal/table_helpers/helpers.h> -#undef INCLUDE_YDB_INTERNAL_H - +#undef INCLUDE_YDB_INTERNAL_H + #include <ydb/public/api/grpc/ydb_scheme_v1.grpc.pb.h> #include <ydb/public/api/protos/ydb_scheme.pb.h> #include <ydb/public/sdk/cpp/client/ydb_common_client/impl/client.h> - + namespace NYdb { namespace NScheme { diff --git a/ydb/public/sdk/cpp/client/ydb_scheme/ya.make b/ydb/public/sdk/cpp/client/ydb_scheme/ya.make index c351b5ac71..fe5b7b5ae5 100644 --- a/ydb/public/sdk/cpp/client/ydb_scheme/ya.make +++ b/ydb/public/sdk/cpp/client/ydb_scheme/ya.make @@ -1,20 +1,20 @@ -LIBRARY() - -OWNER( - dcherednik - g:kikimr -) - -SRCS( - scheme.cpp -) - -GENERATE_ENUM_SERIALIZATION(scheme.h) - -PEERDIR( +LIBRARY() + +OWNER( + dcherednik + g:kikimr +) + +SRCS( + scheme.cpp +) + +GENERATE_ENUM_SERIALIZATION(scheme.h) + +PEERDIR( ydb/public/sdk/cpp/client/impl/ydb_internal/make_request ydb/public/sdk/cpp/client/ydb_common_client/impl ydb/public/sdk/cpp/client/ydb_driver -) - -END() +) + +END() diff --git a/ydb/public/sdk/cpp/client/ydb_table/impl/client_session.cpp b/ydb/public/sdk/cpp/client/ydb_table/impl/client_session.cpp index 091b325094..edfee60847 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/impl/client_session.cpp +++ b/ydb/public/sdk/cpp/client/ydb_table/impl/client_session.cpp @@ -1,43 +1,43 @@ -#include "client_session.h" -#include "data_query.h" - -namespace NYdb { -namespace NTable { - -TSession::TImpl::TImpl(const TString& sessionId, const TString& endpoint, bool useQueryCache, ui32 queryCacheSize) - : SessionId_(sessionId) - , Endpoint_(endpoint) - , State_(S_STANDALONE) - , UseQueryCache_(useQueryCache) - , QueryCache_(queryCacheSize) - , Lock_() - , TimeToTouch_(TInstant::Now()) - , TimeInPast_(TInstant::Now()) - , NeedUpdateActiveCounter_(false) -{} - -TSession::TImpl::~TImpl() { - Unlink(); -} - -const TString& TSession::TImpl::GetId() const { - return SessionId_; -} - -const TString& TSession::TImpl::GetEndpoint() const { - return Endpoint_; -} - -// Can be called from interceptor, need lock -void TSession::TImpl::MarkBroken() { - with_lock(Lock_) { - if (State_ == EState::S_ACTIVE) { - NeedUpdateActiveCounter_ = true; - } - State_ = EState::S_BROKEN; - } -} - +#include "client_session.h" +#include "data_query.h" + +namespace NYdb { +namespace NTable { + +TSession::TImpl::TImpl(const TString& sessionId, const TString& endpoint, bool useQueryCache, ui32 queryCacheSize) + : SessionId_(sessionId) + , Endpoint_(endpoint) + , State_(S_STANDALONE) + , UseQueryCache_(useQueryCache) + , QueryCache_(queryCacheSize) + , Lock_() + , TimeToTouch_(TInstant::Now()) + , TimeInPast_(TInstant::Now()) + , NeedUpdateActiveCounter_(false) +{} + +TSession::TImpl::~TImpl() { + Unlink(); +} + +const TString& TSession::TImpl::GetId() const { + return SessionId_; +} + +const TString& TSession::TImpl::GetEndpoint() const { + return Endpoint_; +} + +// Can be called from interceptor, need lock +void TSession::TImpl::MarkBroken() { + with_lock(Lock_) { + if (State_ == EState::S_ACTIVE) { + NeedUpdateActiveCounter_ = true; + } + State_ = EState::S_BROKEN; + } +} + void TSession::TImpl::MarkAsClosing() { with_lock(Lock_) { if (State_ == EState::S_ACTIVE) { @@ -48,149 +48,149 @@ void TSession::TImpl::MarkAsClosing() { } } -void TSession::TImpl::MarkStandalone() { - State_ = EState::S_STANDALONE; - NeedUpdateActiveCounter_ = false; -} - -void TSession::TImpl::MarkActive() { - State_ = EState::S_ACTIVE; - NeedUpdateActiveCounter_ = false; -} - -void TSession::TImpl::MarkIdle() { - State_ = EState::S_IDLE; - NeedUpdateActiveCounter_ = false; -} - -// Can be called from interceptor, need lock -void TSession::TImpl::MarkDisconnected() { - with_lock(Lock_) { - if (State_ == EState::S_ACTIVE) { - NeedUpdateActiveCounter_ = true; - } - State_ = EState::S_DISCONNECTED; - } -} - -TSession::TImpl::EState TSession::TImpl::GetState() const { - // See comments in InjectSessionStatusInterception about lock - with_lock(Lock_) { - return State_; - } -} - -void TSession::TImpl::SetNeedUpdateActiveCounter(bool flag) { - NeedUpdateActiveCounter_ = flag; -} - -bool TSession::TImpl::NeedUpdateActiveCounter() const { - return NeedUpdateActiveCounter_; -} - -void TSession::TImpl::InvalidateQueryInCache(const TString& key) { - if (!UseQueryCache_) { - return; - } - - with_lock(Lock_) { - auto it = QueryCache_.Find(key); - if (it != QueryCache_.End()) { - QueryCache_.Erase(it); - } - } -} - -void TSession::TImpl::InvalidateQueryCache() { - if (!UseQueryCache_) { - return; - } - - with_lock(Lock_) { - QueryCache_.Clear(); - } -} - -TMaybe<TSession::TImpl::TDataQueryInfo> TSession::TImpl::GetQueryFromCache(const TString& query, bool allowMigration) { - if (!UseQueryCache_) { - return {}; - } - - auto key = EncodeQuery(query, allowMigration); - - with_lock(Lock_) { - auto it = QueryCache_.Find(key); - if (it != QueryCache_.End()) { - return *it; - } - } - - return Nothing(); -} - -void TSession::TImpl::AddQueryToCache(const TDataQuery& query) { - if (!UseQueryCache_) { - return; - } - - const auto& id = query.Impl_->GetId(); - if (id.empty()) { - return; - } - - auto key = query.Impl_->GetTextHash(); - TDataQueryInfo queryInfo(id, query.Impl_->GetParameterTypes()); - - with_lock(Lock_) { - auto it = QueryCache_.Find(key); - if (it != QueryCache_.End()) { - *it = queryInfo; - } else { - QueryCache_.Insert(key, queryInfo); - } - } -} - -// We need lock here because this method can be called from different thread if client -// makes simultaneous calls on one session. It should be possible to rewrite this part. -void TSession::TImpl::ScheduleTimeToTouch(TDuration interval, bool updateTimeInPast) { - auto now = TInstant::Now(); - with_lock(Lock_) { - if (updateTimeInPast) { - TimeInPast_ = now; - } - TimeToTouch_ = now + interval; - } -} - -void TSession::TImpl::ScheduleTimeToTouchFast(TDuration interval, bool updateTimeInPast) { - auto now = TInstant::Now(); - if (updateTimeInPast) { - TimeInPast_ = now; - } - TimeToTouch_ = now + interval; -} - -TInstant TSession::TImpl::GetTimeToTouchFast() const { - return TimeToTouch_; -} - -TInstant TSession::TImpl::GetTimeInPastFast() const { - return TimeInPast_; -} - -// SetTimeInterval/GetTimeInterval, are not atomic! -void TSession::TImpl::SetTimeInterval(TDuration interval) { - TimeInterval_ = interval; -} - -TDuration TSession::TImpl::GetTimeInterval() const { - return TimeInterval_; -} - -const TLRUCache<TString, TSession::TImpl::TDataQueryInfo>& TSession::TImpl::GetQueryCacheUnsafe() const { - return QueryCache_; -} - -} // namespace NTable -} // namespace NYdb +void TSession::TImpl::MarkStandalone() { + State_ = EState::S_STANDALONE; + NeedUpdateActiveCounter_ = false; +} + +void TSession::TImpl::MarkActive() { + State_ = EState::S_ACTIVE; + NeedUpdateActiveCounter_ = false; +} + +void TSession::TImpl::MarkIdle() { + State_ = EState::S_IDLE; + NeedUpdateActiveCounter_ = false; +} + +// Can be called from interceptor, need lock +void TSession::TImpl::MarkDisconnected() { + with_lock(Lock_) { + if (State_ == EState::S_ACTIVE) { + NeedUpdateActiveCounter_ = true; + } + State_ = EState::S_DISCONNECTED; + } +} + +TSession::TImpl::EState TSession::TImpl::GetState() const { + // See comments in InjectSessionStatusInterception about lock + with_lock(Lock_) { + return State_; + } +} + +void TSession::TImpl::SetNeedUpdateActiveCounter(bool flag) { + NeedUpdateActiveCounter_ = flag; +} + +bool TSession::TImpl::NeedUpdateActiveCounter() const { + return NeedUpdateActiveCounter_; +} + +void TSession::TImpl::InvalidateQueryInCache(const TString& key) { + if (!UseQueryCache_) { + return; + } + + with_lock(Lock_) { + auto it = QueryCache_.Find(key); + if (it != QueryCache_.End()) { + QueryCache_.Erase(it); + } + } +} + +void TSession::TImpl::InvalidateQueryCache() { + if (!UseQueryCache_) { + return; + } + + with_lock(Lock_) { + QueryCache_.Clear(); + } +} + +TMaybe<TSession::TImpl::TDataQueryInfo> TSession::TImpl::GetQueryFromCache(const TString& query, bool allowMigration) { + if (!UseQueryCache_) { + return {}; + } + + auto key = EncodeQuery(query, allowMigration); + + with_lock(Lock_) { + auto it = QueryCache_.Find(key); + if (it != QueryCache_.End()) { + return *it; + } + } + + return Nothing(); +} + +void TSession::TImpl::AddQueryToCache(const TDataQuery& query) { + if (!UseQueryCache_) { + return; + } + + const auto& id = query.Impl_->GetId(); + if (id.empty()) { + return; + } + + auto key = query.Impl_->GetTextHash(); + TDataQueryInfo queryInfo(id, query.Impl_->GetParameterTypes()); + + with_lock(Lock_) { + auto it = QueryCache_.Find(key); + if (it != QueryCache_.End()) { + *it = queryInfo; + } else { + QueryCache_.Insert(key, queryInfo); + } + } +} + +// We need lock here because this method can be called from different thread if client +// makes simultaneous calls on one session. It should be possible to rewrite this part. +void TSession::TImpl::ScheduleTimeToTouch(TDuration interval, bool updateTimeInPast) { + auto now = TInstant::Now(); + with_lock(Lock_) { + if (updateTimeInPast) { + TimeInPast_ = now; + } + TimeToTouch_ = now + interval; + } +} + +void TSession::TImpl::ScheduleTimeToTouchFast(TDuration interval, bool updateTimeInPast) { + auto now = TInstant::Now(); + if (updateTimeInPast) { + TimeInPast_ = now; + } + TimeToTouch_ = now + interval; +} + +TInstant TSession::TImpl::GetTimeToTouchFast() const { + return TimeToTouch_; +} + +TInstant TSession::TImpl::GetTimeInPastFast() const { + return TimeInPast_; +} + +// SetTimeInterval/GetTimeInterval, are not atomic! +void TSession::TImpl::SetTimeInterval(TDuration interval) { + TimeInterval_ = interval; +} + +TDuration TSession::TImpl::GetTimeInterval() const { + return TimeInterval_; +} + +const TLRUCache<TString, TSession::TImpl::TDataQueryInfo>& TSession::TImpl::GetQueryCacheUnsafe() const { + return QueryCache_; +} + +} // namespace NTable +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/ydb_table/impl/client_session.h b/ydb/public/sdk/cpp/client/ydb_table/impl/client_session.h index ef12876a30..1c33edc333 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/impl/client_session.h +++ b/ydb/public/sdk/cpp/client/ydb_table/impl/client_session.h @@ -2,11 +2,11 @@ #include <ydb/public/sdk/cpp/client/ydb_table/table.h> #include <ydb/public/sdk/cpp/client/impl/ydb_endpoints/endpoints.h> - + #include <ydb/public/api/protos/ydb_table.pb.h> #include <library/cpp/cache/cache.h> - + #include <util/datetime/base.h> #include <functional> diff --git a/ydb/public/sdk/cpp/client/ydb_table/impl/data_query.cpp b/ydb/public/sdk/cpp/client/ydb_table/impl/data_query.cpp index 7d71abf864..ce6f4b5efb 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/impl/data_query.cpp +++ b/ydb/public/sdk/cpp/client/ydb_table/impl/data_query.cpp @@ -1,61 +1,61 @@ -#include "data_query.h" - -#include <contrib/libs/openssl/include/openssl/sha.h> - -namespace NYdb { -namespace NTable { - -TString GetQueryHash(const TString& text) { - SHA256_CTX sha; - SHA256_Init(&sha); - SHA256_Update(&sha, text.data(), text.size()); - unsigned char hash[SHA256_DIGEST_LENGTH]; - SHA256_Final(hash, &sha); - return TString(reinterpret_cast<char*>(hash), sizeof(hash)); -} - -TString EncodeQuery(const TString& text, bool reversible) { - if (reversible) { - //TODO: may be compress and decompress this query - return text; - } else { - return GetQueryHash(text); - } -} - -//////////////////////////////////////////////////////////////////////////////// - -TDataQuery::TImpl::TImpl(const TSession& session, const TString& text, bool keepText, const TString& id, bool allowMigration) - : Session_(session) - , Id_(id) - , TextHash_(EncodeQuery(text, allowMigration)) - , Text_(keepText ? text : TMaybe<TString>()) -{} - -TDataQuery::TImpl::TImpl(const TSession& session, const TString& text, bool keepText, const TString& id, bool allowMigration, - const ::google::protobuf::Map<TString, Ydb::Type>& types) - : Session_(session) - , Id_(id) - , ParameterTypes_(types) - , TextHash_(EncodeQuery(text, allowMigration)) - , Text_(keepText ? text : TMaybe<TString>()) -{} - -const TString& TDataQuery::TImpl::GetId() const { - return Id_; -} - -const ::google::protobuf::Map<TString, Ydb::Type>& TDataQuery::TImpl::GetParameterTypes() const { - return ParameterTypes_; -} - -const TString& TDataQuery::TImpl::GetTextHash() const { - return TextHash_; -} - -const TMaybe<TString>& TDataQuery::TImpl::GetText() const { - return Text_; -} - -} // namespace NTable -} // namespace NYdb +#include "data_query.h" + +#include <contrib/libs/openssl/include/openssl/sha.h> + +namespace NYdb { +namespace NTable { + +TString GetQueryHash(const TString& text) { + SHA256_CTX sha; + SHA256_Init(&sha); + SHA256_Update(&sha, text.data(), text.size()); + unsigned char hash[SHA256_DIGEST_LENGTH]; + SHA256_Final(hash, &sha); + return TString(reinterpret_cast<char*>(hash), sizeof(hash)); +} + +TString EncodeQuery(const TString& text, bool reversible) { + if (reversible) { + //TODO: may be compress and decompress this query + return text; + } else { + return GetQueryHash(text); + } +} + +//////////////////////////////////////////////////////////////////////////////// + +TDataQuery::TImpl::TImpl(const TSession& session, const TString& text, bool keepText, const TString& id, bool allowMigration) + : Session_(session) + , Id_(id) + , TextHash_(EncodeQuery(text, allowMigration)) + , Text_(keepText ? text : TMaybe<TString>()) +{} + +TDataQuery::TImpl::TImpl(const TSession& session, const TString& text, bool keepText, const TString& id, bool allowMigration, + const ::google::protobuf::Map<TString, Ydb::Type>& types) + : Session_(session) + , Id_(id) + , ParameterTypes_(types) + , TextHash_(EncodeQuery(text, allowMigration)) + , Text_(keepText ? text : TMaybe<TString>()) +{} + +const TString& TDataQuery::TImpl::GetId() const { + return Id_; +} + +const ::google::protobuf::Map<TString, Ydb::Type>& TDataQuery::TImpl::GetParameterTypes() const { + return ParameterTypes_; +} + +const TString& TDataQuery::TImpl::GetTextHash() const { + return TextHash_; +} + +const TMaybe<TString>& TDataQuery::TImpl::GetText() const { + return Text_; +} + +} // namespace NTable +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/ydb_table/impl/data_query.h b/ydb/public/sdk/cpp/client/ydb_table/impl/data_query.h index 462d356702..2ebbd5296e 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/impl/data_query.h +++ b/ydb/public/sdk/cpp/client/ydb_table/impl/data_query.h @@ -1,36 +1,36 @@ -#pragma once - +#pragma once + #include <ydb/public/sdk/cpp/client/ydb_table/table.h> #include <ydb/public/api/protos/ydb_table.pb.h> - -namespace NYdb { -namespace NTable { - -TString EncodeQuery(const TString& text, bool reversible); - -//////////////////////////////////////////////////////////////////////////////// - -class TDataQuery::TImpl { - friend class TDataQuery; - -public: - TImpl(const TSession& session, const TString& text, bool keepText, const TString& id, bool allowMigration); - - TImpl(const TSession& session, const TString& text, bool keepText, const TString& id, bool allowMigration, - const ::google::protobuf::Map<TString, Ydb::Type>& types); - - const TString& GetId() const; - const ::google::protobuf::Map<TString, Ydb::Type>& GetParameterTypes() const; - const TString& GetTextHash() const; - const TMaybe<TString>& GetText() const; - -private: - NTable::TSession Session_; - TString Id_; - ::google::protobuf::Map<TString, Ydb::Type> ParameterTypes_; - TString TextHash_; - TMaybe<TString> Text_; -}; - -} // namespace NTable -} // namespace NYdb + +namespace NYdb { +namespace NTable { + +TString EncodeQuery(const TString& text, bool reversible); + +//////////////////////////////////////////////////////////////////////////////// + +class TDataQuery::TImpl { + friend class TDataQuery; + +public: + TImpl(const TSession& session, const TString& text, bool keepText, const TString& id, bool allowMigration); + + TImpl(const TSession& session, const TString& text, bool keepText, const TString& id, bool allowMigration, + const ::google::protobuf::Map<TString, Ydb::Type>& types); + + const TString& GetId() const; + const ::google::protobuf::Map<TString, Ydb::Type>& GetParameterTypes() const; + const TString& GetTextHash() const; + const TMaybe<TString>& GetText() const; + +private: + NTable::TSession Session_; + TString Id_; + ::google::protobuf::Map<TString, Ydb::Type> ParameterTypes_; + TString TextHash_; + TMaybe<TString> Text_; +}; + +} // namespace NTable +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/ydb_table/impl/request_migrator.cpp b/ydb/public/sdk/cpp/client/ydb_table/impl/request_migrator.cpp index 6c1c6b0381..1484553c85 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/impl/request_migrator.cpp +++ b/ydb/public/sdk/cpp/client/ydb_table/impl/request_migrator.cpp @@ -1,4 +1,4 @@ -#include "request_migrator.h" +#include "request_migrator.h" #include <vector> #include <numeric> diff --git a/ydb/public/sdk/cpp/client/ydb_table/impl/request_migrator.h b/ydb/public/sdk/cpp/client/ydb_table/impl/request_migrator.h index ace3e0e04d..81b89392be 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/impl/request_migrator.h +++ b/ydb/public/sdk/cpp/client/ydb_table/impl/request_migrator.h @@ -1,9 +1,9 @@ #pragma once -#include "client_session.h" +#include "client_session.h" #include <ydb/public/sdk/cpp/client/ydb_table/table.h> - + #include <library/cpp/threading/future/future.h> #include <util/generic/string.h> diff --git a/ydb/public/sdk/cpp/client/ydb_table/impl/ut/ya.make b/ydb/public/sdk/cpp/client/ydb_table/impl/ut/ya.make index e139fec547..dc192877ea 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/impl/ut/ya.make +++ b/ydb/public/sdk/cpp/client/ydb_table/impl/ut/ya.make @@ -1,27 +1,27 @@ UNITTEST_FOR(ydb/public/sdk/cpp/client/ydb_table/impl) - -OWNER( - dcherednik - g:kikimr -) - -IF (SANITIZER_TYPE) - TIMEOUT(1200) - SIZE(LARGE) - TAG(ya:fat) -ELSE() - TIMEOUT(600) - SIZE(MEDIUM) -ENDIF() - -FORK_SUBTESTS() - -PEERDIR( + +OWNER( + dcherednik + g:kikimr +) + +IF (SANITIZER_TYPE) + TIMEOUT(1200) + SIZE(LARGE) + TAG(ya:fat) +ELSE() + TIMEOUT(600) + SIZE(MEDIUM) +ENDIF() + +FORK_SUBTESTS() + +PEERDIR( ydb/public/sdk/cpp/client/ydb_table -) - -SRCS( - request_migrator_ut.cpp -) - -END() +) + +SRCS( + request_migrator_ut.cpp +) + +END() diff --git a/ydb/public/sdk/cpp/client/ydb_table/impl/ya.make b/ydb/public/sdk/cpp/client/ydb_table/impl/ya.make index 0a76173697..25daf2f20d 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/impl/ya.make +++ b/ydb/public/sdk/cpp/client/ydb_table/impl/ya.make @@ -1,26 +1,26 @@ -LIBRARY() - -OWNER( - dcherednik - g:kikimr -) - -SRCS( - client_session.cpp - data_query.cpp - request_migrator.cpp -) - -PEERDIR( - library/cpp/threading/future +LIBRARY() + +OWNER( + dcherednik + g:kikimr +) + +SRCS( + client_session.cpp + data_query.cpp + request_migrator.cpp +) + +PEERDIR( + library/cpp/threading/future ydb/public/api/protos ydb/public/lib/operation_id/protos ydb/public/sdk/cpp/client/impl/ydb_endpoints ydb/library/yql/public/issue/protos -) - -END() - +) + +END() + RECURSE_FOR_TESTS( ut ) diff --git a/ydb/public/sdk/cpp/client/ydb_table/proto_accessor.cpp b/ydb/public/sdk/cpp/client/ydb_table/proto_accessor.cpp index 3ebae56f2e..40bfe74f67 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/proto_accessor.cpp +++ b/ydb/public/sdk/cpp/client/ydb_table/proto_accessor.cpp @@ -1,28 +1,28 @@ #include <ydb/public/sdk/cpp/client/ydb_proto/accessor.h> - -#include "table.h" - -namespace NYdb { - -const Ydb::TableStats::QueryStats& TProtoAccessor::GetProto(const NTable::TQueryStats& queryStats) { - return queryStats.GetProto(); -} - -const Ydb::Table::DescribeTableResult& TProtoAccessor::GetProto(const NTable::TTableDescription& tableDescription) { - return tableDescription.GetProto(); -} - -NTable::TQueryStats TProtoAccessor::FromProto(const Ydb::TableStats::QueryStats& queryStats) { - return NTable::TQueryStats(queryStats); -} - -NTable::TTableDescription TProtoAccessor::FromProto(const Ydb::Table::CreateTableRequest& request) { - return NTable::TTableDescription(request); -} - + +#include "table.h" + +namespace NYdb { + +const Ydb::TableStats::QueryStats& TProtoAccessor::GetProto(const NTable::TQueryStats& queryStats) { + return queryStats.GetProto(); +} + +const Ydb::Table::DescribeTableResult& TProtoAccessor::GetProto(const NTable::TTableDescription& tableDescription) { + return tableDescription.GetProto(); +} + +NTable::TQueryStats TProtoAccessor::FromProto(const Ydb::TableStats::QueryStats& queryStats) { + return NTable::TQueryStats(queryStats); +} + +NTable::TTableDescription TProtoAccessor::FromProto(const Ydb::Table::CreateTableRequest& request) { + return NTable::TTableDescription(request); +} + NTable::TIndexDescription TProtoAccessor::FromProto(const Ydb::Table::TableIndex& tableIndex) { return NTable::TIndexDescription(tableIndex); -} +} NTable::TIndexDescription TProtoAccessor::FromProto(const Ydb::Table::TableIndexDescription& tableIndexDesc) { return NTable::TIndexDescription(tableIndexDesc); diff --git a/ydb/public/sdk/cpp/client/ydb_table/query_stats/stats.cpp b/ydb/public/sdk/cpp/client/ydb_table/query_stats/stats.cpp index 3df4277638..106b848c83 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/query_stats/stats.cpp +++ b/ydb/public/sdk/cpp/client/ydb_table/query_stats/stats.cpp @@ -1,11 +1,11 @@ -#include "stats.h" +#include "stats.h" #include <ydb/public/api/protos/ydb_table.pb.h> #include <util/datetime/base.h> -#include <google/protobuf/text_format.h> - +#include <google/protobuf/text_format.h> + namespace NYdb { namespace NTable { diff --git a/ydb/public/sdk/cpp/client/ydb_table/query_stats/stats.h b/ydb/public/sdk/cpp/client/ydb_table/query_stats/stats.h index a8b1128069..355c271dc2 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/query_stats/stats.h +++ b/ydb/public/sdk/cpp/client/ydb_table/query_stats/stats.h @@ -1,10 +1,10 @@ #pragma once #include <util/generic/maybe.h> -#include <util/generic/string.h> +#include <util/generic/string.h> + +#include <memory> -#include <memory> - class TDuration; namespace Ydb { diff --git a/ydb/public/sdk/cpp/client/ydb_table/query_stats/ya.make b/ydb/public/sdk/cpp/client/ydb_table/query_stats/ya.make index 28f9270eb8..b0f884e23a 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/query_stats/ya.make +++ b/ydb/public/sdk/cpp/client/ydb_table/query_stats/ya.make @@ -1,17 +1,17 @@ -LIBRARY() - -OWNER( - dcherednik - g:kikimr -) - -SRCS( - stats.cpp -) - -PEERDIR( - contrib/libs/protobuf +LIBRARY() + +OWNER( + dcherednik + g:kikimr +) + +SRCS( + stats.cpp +) + +PEERDIR( + contrib/libs/protobuf ydb/public/api/protos -) - -END() +) + +END() diff --git a/ydb/public/sdk/cpp/client/ydb_table/table.cpp b/ydb/public/sdk/cpp/client/ydb_table/table.cpp index 0fb997781f..610fec3884 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/table.cpp +++ b/ydb/public/sdk/cpp/client/ydb_table/table.cpp @@ -1,10 +1,10 @@ -#include "table.h" +#include "table.h" -#define INCLUDE_YDB_INTERNAL_H +#define INCLUDE_YDB_INTERNAL_H #include <ydb/public/sdk/cpp/client/impl/ydb_internal/table_helpers/helpers.h> #include <ydb/public/sdk/cpp/client/impl/ydb_internal/make_request/make.h> -#undef INCLUDE_YDB_INTERNAL_H - +#undef INCLUDE_YDB_INTERNAL_H + #include <ydb/public/api/grpc/ydb_table_v1.grpc.pb.h> #include <ydb/public/api/protos/ydb_table.pb.h> #include <ydb/public/sdk/cpp/client/impl/ydb_stats/stats.h> @@ -14,7 +14,7 @@ #include <ydb/public/sdk/cpp/client/ydb_table/impl/client_session.h> #include <ydb/public/sdk/cpp/client/ydb_table/impl/data_query.h> #include <ydb/public/sdk/cpp/client/ydb_table/impl/request_migrator.h> - + #include <library/cpp/cache/cache.h> #include <util/generic/map.h> @@ -3549,7 +3549,7 @@ TAsyncBulkUpsertResult TTableClient::BulkUpsert(const TString& table, EDataForma TAsyncScanQueryPartIterator TTableClient::StreamExecuteScanQuery(const TString& query, const TParams& params, const TStreamExecScanQuerySettings& settings) { - return Impl_->StreamExecuteScanQuery(query, ¶ms.GetProtoMap(), settings); + return Impl_->StreamExecuteScanQuery(query, ¶ms.GetProtoMap(), settings); } TAsyncScanQueryPartIterator TTableClient::StreamExecuteScanQuery(const TString& query, @@ -3871,7 +3871,7 @@ TAsyncDataQueryResult TSession::ExecuteDataQuery(const TString& query, const TTx TAsyncDataQueryResult TSession::ExecuteDataQuery(const TString& query, const TTxControl& txControl, TParams&& params, const TExecDataQuerySettings& settings) { - auto paramsPtr = params.Empty() ? nullptr : params.GetProtoMapPtr(); + auto paramsPtr = params.Empty() ? nullptr : params.GetProtoMapPtr(); return Client_->ExecuteDataQuery(*this, query, txControl, paramsPtr, settings); } @@ -3891,7 +3891,7 @@ TAsyncDataQueryResult TSession::ExecuteDataQuery(const TString& query, const TTx *this, query, txControl, - params.GetProtoMap(), + params.GetProtoMap(), settings); } } @@ -4048,7 +4048,7 @@ TAsyncDataQueryResult TDataQuery::Execute(const TTxControl& txControl, TAsyncDataQueryResult TDataQuery::Execute(const TTxControl& txControl, TParams&& params, const TExecDataQuerySettings& settings) { - auto paramsPtr = params.Empty() ? nullptr : params.GetProtoMapPtr(); + auto paramsPtr = params.Empty() ? nullptr : params.GetProtoMapPtr(); return Impl_->Session_.Client_->ExecuteDataQuery( Impl_->Session_, *this, @@ -4075,7 +4075,7 @@ TAsyncDataQueryResult TDataQuery::Execute(const TTxControl& txControl, const TPa Impl_->Session_, *this, txControl, - params.GetProtoMap(), + params.GetProtoMap(), settings, false); } diff --git a/ydb/public/sdk/cpp/client/ydb_table/table.h b/ydb/public/sdk/cpp/client/ydb_table/table.h index 0a533b42b0..42d21d2fec 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/table.h +++ b/ydb/public/sdk/cpp/client/ydb_table/table.h @@ -1,13 +1,13 @@ #pragma once -#include "table_enum.h" +#include "table_enum.h" #include <ydb/public/sdk/cpp/client/ydb_driver/driver.h> #include <ydb/public/sdk/cpp/client/ydb_result/result.h> #include <ydb/public/sdk/cpp/client/ydb_table/query_stats/stats.h> #include <ydb/public/sdk/cpp/client/ydb_params/params.h> #include <ydb/public/sdk/cpp/client/ydb_types/operation/operation.h> - + #include <util/generic/hash.h> #include <util/generic/maybe.h> #include <util/generic/variant.h> @@ -30,11 +30,11 @@ class ValueSinceUnixEpochModeSettings; } namespace NYdb { - -namespace NScheme { -struct TPermissions; -} - + +namespace NScheme { +struct TPermissions; +} + namespace NTable { //////////////////////////////////////////////////////////////////////////////// diff --git a/ydb/public/sdk/cpp/client/ydb_table/ya.make b/ydb/public/sdk/cpp/client/ydb_table/ya.make index b427661365..901908d243 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/ya.make +++ b/ydb/public/sdk/cpp/client/ydb_table/ya.make @@ -1,18 +1,18 @@ -LIBRARY() - -OWNER( - dcherednik - g:kikimr -) - -SRCS( - table.cpp - proto_accessor.cpp -) - -GENERATE_ENUM_SERIALIZATION(table_enum.h) - -PEERDIR( +LIBRARY() + +OWNER( + dcherednik + g:kikimr +) + +SRCS( + table.cpp + proto_accessor.cpp +) + +GENERATE_ENUM_SERIALIZATION(table_enum.h) + +PEERDIR( ydb/public/api/protos ydb/public/sdk/cpp/client/impl/ydb_internal/make_request ydb/public/sdk/cpp/client/ydb_driver @@ -23,6 +23,6 @@ PEERDIR( ydb/public/sdk/cpp/client/ydb_table/query_stats ydb/public/sdk/cpp/client/ydb_types/operation ydb/public/sdk/cpp/client/ydb_value -) - -END() +) + +END() diff --git a/ydb/public/sdk/cpp/client/ydb_types/credentials/credentials.cpp b/ydb/public/sdk/cpp/client/ydb_types/credentials/credentials.cpp index ec00093baf..679634450f 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/credentials/credentials.cpp +++ b/ydb/public/sdk/cpp/client/ydb_types/credentials/credentials.cpp @@ -1,77 +1,77 @@ -#include "credentials.h" - -namespace NYdb { - -class TInsecureCredentialsProvider : public ICredentialsProvider { -public: - TInsecureCredentialsProvider() - {} - +#include "credentials.h" + +namespace NYdb { + +class TInsecureCredentialsProvider : public ICredentialsProvider { +public: + TInsecureCredentialsProvider() + {} + TStringType GetAuthInfo() const override { return TStringType(); - } - - bool IsValid() const override { - return false; - } -}; - -class TInsecureCredentialsProviderFactory : public ICredentialsProviderFactory { -public: - TInsecureCredentialsProviderFactory() - {} - - std::shared_ptr<ICredentialsProvider> CreateProvider() const override { - return std::make_shared<TInsecureCredentialsProvider>(); - } - + } + + bool IsValid() const override { + return false; + } +}; + +class TInsecureCredentialsProviderFactory : public ICredentialsProviderFactory { +public: + TInsecureCredentialsProviderFactory() + {} + + std::shared_ptr<ICredentialsProvider> CreateProvider() const override { + return std::make_shared<TInsecureCredentialsProvider>(); + } + TStringType GetClientIdentity() const override { return TStringType(); - } -}; - -class TOAuthCredentialsProvider : public ICredentialsProvider { -public: + } +}; + +class TOAuthCredentialsProvider : public ICredentialsProvider { +public: TOAuthCredentialsProvider(const TStringType& token) - : Token(token) - {} - + : Token(token) + {} + TStringType GetAuthInfo() const override { - return Token; - } - - bool IsValid() const override { - return !Token.empty(); - } - -private: + return Token; + } + + bool IsValid() const override { + return !Token.empty(); + } + +private: TStringType Token; -}; - -class TOAuthCredentialsProviderFactory : public ICredentialsProviderFactory { -public: +}; + +class TOAuthCredentialsProviderFactory : public ICredentialsProviderFactory { +public: TOAuthCredentialsProviderFactory(const TStringType& token) - : Token(token) - {} - - std::shared_ptr<ICredentialsProvider> CreateProvider() const override { - return std::make_shared<TOAuthCredentialsProvider>(Token); - } - + : Token(token) + {} + + std::shared_ptr<ICredentialsProvider> CreateProvider() const override { + return std::make_shared<TOAuthCredentialsProvider>(Token); + } + TStringType GetClientIdentity() const override { - return Token; - } - -private: + return Token; + } + +private: TStringType Token; -}; - -std::shared_ptr<ICredentialsProviderFactory> CreateInsecureCredentialsProviderFactory() { - return std::make_shared<TInsecureCredentialsProviderFactory>(); -} +}; + +std::shared_ptr<ICredentialsProviderFactory> CreateInsecureCredentialsProviderFactory() { + return std::make_shared<TInsecureCredentialsProviderFactory>(); +} std::shared_ptr<ICredentialsProviderFactory> CreateOAuthCredentialsProviderFactory(const TStringType& token) { - return std::make_shared<TOAuthCredentialsProviderFactory>(token); -} - -} // namespace NYdb - + return std::make_shared<TOAuthCredentialsProviderFactory>(token); +} + +} // namespace NYdb + diff --git a/ydb/public/sdk/cpp/client/ydb_types/credentials/credentials.h b/ydb/public/sdk/cpp/client/ydb_types/credentials/credentials.h index 17515b7ff1..75b86933ce 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/credentials/credentials.h +++ b/ydb/public/sdk/cpp/client/ydb_types/credentials/credentials.h @@ -1,24 +1,24 @@ -#pragma once - +#pragma once + #include <ydb/public/sdk/cpp/client/impl/ydb_internal/common/type_switcher.h> - + #include <memory> -namespace NYdb { - -class ICredentialsProvider { -public: - virtual ~ICredentialsProvider() = default; +namespace NYdb { + +class ICredentialsProvider { +public: + virtual ~ICredentialsProvider() = default; virtual TStringType GetAuthInfo() const = 0; - virtual bool IsValid() const = 0; -}; - + virtual bool IsValid() const = 0; +}; + using TCredentialsProviderPtr = std::shared_ptr<ICredentialsProvider>; class ICoreFacility; -class ICredentialsProviderFactory { -public: - virtual ~ICredentialsProviderFactory() = default; +class ICredentialsProviderFactory { +public: + virtual ~ICredentialsProviderFactory() = default; virtual TCredentialsProviderPtr CreateProvider() const = 0; // !!!Experimental!!! virtual TCredentialsProviderPtr CreateProvider(std::weak_ptr<ICoreFacility> facility) const { @@ -26,13 +26,13 @@ public: return CreateProvider(); } virtual TStringType GetClientIdentity() const = 0; -}; - +}; + using TCredentialsProviderFactoryPtr = std::shared_ptr<ICredentialsProviderFactory>; -std::shared_ptr<ICredentialsProviderFactory> CreateInsecureCredentialsProviderFactory(); +std::shared_ptr<ICredentialsProviderFactory> CreateInsecureCredentialsProviderFactory(); std::shared_ptr<ICredentialsProviderFactory> CreateOAuthCredentialsProviderFactory(const TStringType& token); - + struct TLoginCredentialsParams { TString User; TString Password; @@ -40,4 +40,4 @@ struct TLoginCredentialsParams { std::shared_ptr<ICredentialsProviderFactory> CreateLoginCredentialsProviderFactory(TLoginCredentialsParams params); -} // namespace NYdb +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/ydb_types/credentials/ya.make b/ydb/public/sdk/cpp/client/ydb_types/credentials/ya.make index 4a296ea6d9..c58b2112f8 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/credentials/ya.make +++ b/ydb/public/sdk/cpp/client/ydb_types/credentials/ya.make @@ -1,12 +1,12 @@ -LIBRARY() - +LIBRARY() + OWNER(g:kikimr) - -SRCS( - credentials.cpp + +SRCS( + credentials.cpp login.cpp -) - +) + PEERDIR( ydb/library/login ydb/public/api/grpc @@ -14,4 +14,4 @@ PEERDIR( ydb/library/yql/public/issue ) -END() +END() diff --git a/ydb/public/sdk/cpp/client/ydb_types/exceptions/exceptions.cpp b/ydb/public/sdk/cpp/client/ydb_types/exceptions/exceptions.cpp index e0dfd27684..f103a88ecc 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/exceptions/exceptions.cpp +++ b/ydb/public/sdk/cpp/client/ydb_types/exceptions/exceptions.cpp @@ -1,4 +1,4 @@ -#include "exceptions.h" +#include "exceptions.h" namespace NYdb { diff --git a/ydb/public/sdk/cpp/client/ydb_types/exceptions/ya.make b/ydb/public/sdk/cpp/client/ydb_types/exceptions/ya.make index ae043eedd2..8e9a1705e7 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/exceptions/ya.make +++ b/ydb/public/sdk/cpp/client/ydb_types/exceptions/ya.make @@ -1,9 +1,9 @@ -LIBRARY() - +LIBRARY() + OWNER(g:kikimr) - -SRCS( - exceptions.cpp -) - -END() + +SRCS( + exceptions.cpp +) + +END() diff --git a/ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers/handlers.cpp b/ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers/handlers.cpp index 766f910058..69b94f9b40 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers/handlers.cpp +++ b/ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers/handlers.cpp @@ -1,10 +1,10 @@ -#include "handlers.h" +#include "handlers.h" #include <ydb/public/sdk/cpp/client/ydb_types/exceptions/exceptions.h> - -namespace NYdb { - + +namespace NYdb { + void ThrowFatalError(const TString& str) { throw TContractViolation(str); -} - -} // namespace NYdb +} + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers/handlers.h b/ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers/handlers.h index 2ea990d417..6af54a52f2 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers/handlers.h +++ b/ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers/handlers.h @@ -1,11 +1,11 @@ -#pragma once - -#include <util/generic/string.h> - -#include <functional> - -namespace NYdb { - +#pragma once + +#include <util/generic/string.h> + +#include <functional> + +namespace NYdb { + void ThrowFatalError(const TString& str); - -} // namespace NYdb + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers/ya.make b/ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers/ya.make index 7e9d79e7e7..423aa103de 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers/ya.make +++ b/ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers/ya.make @@ -1,16 +1,16 @@ -LIBRARY() - -OWNER( - dcherednik - g:kikimr -) - -SRCS( - handlers.cpp -) - -PEERDIR( +LIBRARY() + +OWNER( + dcherednik + g:kikimr +) + +SRCS( + handlers.cpp +) + +PEERDIR( ydb/public/sdk/cpp/client/ydb_types/exceptions -) - -END() +) + +END() diff --git a/ydb/public/sdk/cpp/client/ydb_types/fluent_settings_helpers.h b/ydb/public/sdk/cpp/client/ydb_types/fluent_settings_helpers.h index a882840182..8747e049f5 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/fluent_settings_helpers.h +++ b/ydb/public/sdk/cpp/client/ydb_types/fluent_settings_helpers.h @@ -1,44 +1,44 @@ -#pragma once - -#include <util/generic/maybe.h> - -#define FLUENT_SETTING(type, name) \ - type name##_; \ - TSelf& name(const type& value) { \ - name##_ = value; \ - return static_cast<TSelf&>(*this); \ - } - -#define FLUENT_SETTING_OPTIONAL(type, name) \ - TMaybe<type> name##_; \ - TSelf& name(const TMaybe<type>& value) { \ - name##_ = value; \ - return static_cast<TSelf&>(*this); \ - } - -#define FLUENT_SETTING_DEFAULT(type, name, defaultValue) \ - type name##_ = defaultValue; \ - TSelf& name(const type& value) { \ - name##_ = value; \ - return static_cast<TSelf&>(*this); \ - } - -#define FLUENT_SETTING_FLAG(name) \ - bool name##_ = false; \ - TSelf& name(bool value = true) { \ - name##_ = value; \ - return static_cast<TSelf&>(*this); \ - } - -#define FLUENT_SETTING_FLAG_ALIAS(name, other, value) \ - TSelf& name() { \ - other##_ = value; \ - return static_cast<TSelf&>(*this); \ - } - -#define FLUENT_SETTING_VECTOR(type, name) \ - TVector<type> name##_; \ - TSelf& Append##name(const type& value) { \ - name##_.push_back(value); \ - return static_cast<TSelf&>(*this); \ - } +#pragma once + +#include <util/generic/maybe.h> + +#define FLUENT_SETTING(type, name) \ + type name##_; \ + TSelf& name(const type& value) { \ + name##_ = value; \ + return static_cast<TSelf&>(*this); \ + } + +#define FLUENT_SETTING_OPTIONAL(type, name) \ + TMaybe<type> name##_; \ + TSelf& name(const TMaybe<type>& value) { \ + name##_ = value; \ + return static_cast<TSelf&>(*this); \ + } + +#define FLUENT_SETTING_DEFAULT(type, name, defaultValue) \ + type name##_ = defaultValue; \ + TSelf& name(const type& value) { \ + name##_ = value; \ + return static_cast<TSelf&>(*this); \ + } + +#define FLUENT_SETTING_FLAG(name) \ + bool name##_ = false; \ + TSelf& name(bool value = true) { \ + name##_ = value; \ + return static_cast<TSelf&>(*this); \ + } + +#define FLUENT_SETTING_FLAG_ALIAS(name, other, value) \ + TSelf& name() { \ + other##_ = value; \ + return static_cast<TSelf&>(*this); \ + } + +#define FLUENT_SETTING_VECTOR(type, name) \ + TVector<type> name##_; \ + TSelf& Append##name(const type& value) { \ + name##_.push_back(value); \ + return static_cast<TSelf&>(*this); \ + } diff --git a/ydb/public/sdk/cpp/client/ydb_types/operation/operation.cpp b/ydb/public/sdk/cpp/client/ydb_types/operation/operation.cpp index 4d37a150ea..aef87cc9cc 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/operation/operation.cpp +++ b/ydb/public/sdk/cpp/client/ydb_types/operation/operation.cpp @@ -1,91 +1,91 @@ -#include "operation.h" - +#include "operation.h" + #include <ydb/public/api/protos/ydb_operation.pb.h> #include <ydb/public/sdk/cpp/client/ydb_types/status/status.h> - -namespace NYdb { - - -class TOperation::TImpl { -public: - TImpl(TStatus&& status) - : Status_(std::move(status)) - , Ready_(true) - { } - - TImpl(TStatus&& status, Ydb::Operations::Operation&& operation) - : Id_(operation.id(), true /* allowEmpty */) - , Status_(std::move(status)) - , Ready_(operation.ready()) - , Operation_(std::move(operation)) - { - } - - const TOperationId& Id() const { - return Id_; - } - - bool Ready() const { - return Ready_; - } - - const TStatus& Status() const { - return Status_; - } - - const Ydb::Operations::Operation& GetProto() const { - return Operation_; - } - -private: - const TOperationId Id_; - const TStatus Status_; - const bool Ready_; - const Ydb::Operations::Operation Operation_; -}; - -TOperation::TOperation(TStatus&& status) - : Impl_(std::make_shared<TImpl>(std::move(status))) -{ } - -TOperation::TOperation(TStatus&& status, Ydb::Operations::Operation&& operation) - : Impl_(std::make_shared<TImpl>(std::move(status), std::move(operation))) -{ } - -const TOperation::TOperationId& TOperation::Id() const { - return Impl_->Id(); -} - -bool TOperation::Ready() const { - return Impl_->Ready(); -} - -const TStatus& TOperation::Status() const { - return Impl_->Status(); -} - -TString TOperation::ToString() const { - TString result; - TStringOutput out(result); - Out(out); - return result; -} - -void TOperation::Out(IOutputStream& o) const { - o << GetProto().DebugString(); -} - -TString TOperation::ToJsonString() const { - using namespace google::protobuf::util; - - TString json; + +namespace NYdb { + + +class TOperation::TImpl { +public: + TImpl(TStatus&& status) + : Status_(std::move(status)) + , Ready_(true) + { } + + TImpl(TStatus&& status, Ydb::Operations::Operation&& operation) + : Id_(operation.id(), true /* allowEmpty */) + , Status_(std::move(status)) + , Ready_(operation.ready()) + , Operation_(std::move(operation)) + { + } + + const TOperationId& Id() const { + return Id_; + } + + bool Ready() const { + return Ready_; + } + + const TStatus& Status() const { + return Status_; + } + + const Ydb::Operations::Operation& GetProto() const { + return Operation_; + } + +private: + const TOperationId Id_; + const TStatus Status_; + const bool Ready_; + const Ydb::Operations::Operation Operation_; +}; + +TOperation::TOperation(TStatus&& status) + : Impl_(std::make_shared<TImpl>(std::move(status))) +{ } + +TOperation::TOperation(TStatus&& status, Ydb::Operations::Operation&& operation) + : Impl_(std::make_shared<TImpl>(std::move(status), std::move(operation))) +{ } + +const TOperation::TOperationId& TOperation::Id() const { + return Impl_->Id(); +} + +bool TOperation::Ready() const { + return Impl_->Ready(); +} + +const TStatus& TOperation::Status() const { + return Impl_->Status(); +} + +TString TOperation::ToString() const { + TString result; + TStringOutput out(result); + Out(out); + return result; +} + +void TOperation::Out(IOutputStream& o) const { + o << GetProto().DebugString(); +} + +TString TOperation::ToJsonString() const { + using namespace google::protobuf::util; + + TString json; auto status = MessageToJsonString(GetProto(), &json, JsonPrintOptions()); Y_VERIFY(status.ok()); - return json; -} - -const Ydb::Operations::Operation& TOperation::GetProto() const { - return Impl_->GetProto(); -} - -} // namespace NYdb + return json; +} + +const Ydb::Operations::Operation& TOperation::GetProto() const { + return Impl_->GetProto(); +} + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/ydb_types/operation/operation.h b/ydb/public/sdk/cpp/client/ydb_types/operation/operation.h index 83263c34a2..bc5dce05eb 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/operation/operation.h +++ b/ydb/public/sdk/cpp/client/ydb_types/operation/operation.h @@ -1,12 +1,12 @@ -#pragma once - +#pragma once + #include <ydb/public/lib/operation_id/operation_id.h> - -#include <library/cpp/threading/future/future.h> - -#include <google/protobuf/stubs/status.h> -#include <google/protobuf/util/json_util.h> - + +#include <library/cpp/threading/future/future.h> + +#include <google/protobuf/stubs/status.h> +#include <google/protobuf/util/json_util.h> + namespace Ydb { namespace Operations { @@ -15,39 +15,39 @@ class Operation; } // namespace Operations } // namespace Ydb -namespace NYdb { - -class TStatus; - -class TOperation { -public: - using TOperationId = NKikimr::NOperationId::TOperationId; - -public: - TOperation(TStatus&& status); - TOperation(TStatus&& status, Ydb::Operations::Operation&& operation); - virtual ~TOperation() = default; - - const TOperationId& Id() const; - bool Ready() const; - const TStatus& Status() const; - - TString ToString() const; - TString ToJsonString() const; - void Out(IOutputStream& o) const; - -protected: - const Ydb::Operations::Operation& GetProto() const; - -private: - class TImpl; - std::shared_ptr<TImpl> Impl_; -}; - -using TAsyncOperation = NThreading::TFuture<TOperation>; - -} // namespace NYdb - +namespace NYdb { + +class TStatus; + +class TOperation { +public: + using TOperationId = NKikimr::NOperationId::TOperationId; + +public: + TOperation(TStatus&& status); + TOperation(TStatus&& status, Ydb::Operations::Operation&& operation); + virtual ~TOperation() = default; + + const TOperationId& Id() const; + bool Ready() const; + const TStatus& Status() const; + + TString ToString() const; + TString ToJsonString() const; + void Out(IOutputStream& o) const; + +protected: + const Ydb::Operations::Operation& GetProto() const; + +private: + class TImpl; + std::shared_ptr<TImpl> Impl_; +}; + +using TAsyncOperation = NThreading::TFuture<TOperation>; + +} // namespace NYdb + Y_DECLARE_OUT_SPEC(inline, NYdb::TOperation, o, x) { - return x.Out(o); -} + return x.Out(o); +} diff --git a/ydb/public/sdk/cpp/client/ydb_types/operation/ya.make b/ydb/public/sdk/cpp/client/ydb_types/operation/ya.make index cae8f767c3..322463eca5 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/operation/ya.make +++ b/ydb/public/sdk/cpp/client/ydb_types/operation/ya.make @@ -1,16 +1,16 @@ -LIBRARY() - +LIBRARY() + OWNER(g:kikimr) - -SRCS( - operation.cpp -) - -PEERDIR( - contrib/libs/protobuf - library/cpp/threading/future + +SRCS( + operation.cpp +) + +PEERDIR( + contrib/libs/protobuf + library/cpp/threading/future ydb/public/lib/operation_id ydb/public/sdk/cpp/client/ydb_types -) - -END() +) + +END() diff --git a/ydb/public/sdk/cpp/client/ydb_types/request_settings.h b/ydb/public/sdk/cpp/client/ydb_types/request_settings.h index 3b7bf71285..fb8aa8ac68 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/request_settings.h +++ b/ydb/public/sdk/cpp/client/ydb_types/request_settings.h @@ -1,21 +1,21 @@ -#pragma once - -#include "fluent_settings_helpers.h" - -#include <util/datetime/base.h> - +#pragma once + +#include "fluent_settings_helpers.h" + +#include <util/datetime/base.h> + #include <vector> #include <utility> -namespace NYdb { - -template<typename TDerived> -struct TRequestSettings { - using TSelf = TDerived; +namespace NYdb { + +template<typename TDerived> +struct TRequestSettings { + using TSelf = TDerived; using THeader = std::vector<std::pair<TString, TString>>; - - FLUENT_SETTING(TString, TraceId); - FLUENT_SETTING(TString, RequestType); + + FLUENT_SETTING(TString, TraceId); + FLUENT_SETTING(TString, RequestType); FLUENT_SETTING(THeader, Header); TRequestSettings() = default; @@ -26,13 +26,13 @@ struct TRequestSettings { , RequestType_(other.RequestType_) , Header_(other.Header_) {} -}; - -template<typename TDerived> -struct TSimpleRequestSettings : public TRequestSettings<TDerived> { - using TSelf = TDerived; - - FLUENT_SETTING(TDuration, ClientTimeout); +}; + +template<typename TDerived> +struct TSimpleRequestSettings : public TRequestSettings<TDerived> { + using TSelf = TDerived; + + FLUENT_SETTING(TDuration, ClientTimeout); TSimpleRequestSettings() = default; @@ -41,18 +41,18 @@ struct TSimpleRequestSettings : public TRequestSettings<TDerived> { : TRequestSettings<TDerived>(other) , ClientTimeout_(other.ClientTimeout_) {} -}; - -template<typename TDerived> -struct TOperationRequestSettings : public TSimpleRequestSettings<TDerived> { - using TSelf = TDerived; - - /* Cancel/timeout operation settings available from 18-8 YDB server version */ - FLUENT_SETTING(TDuration, OperationTimeout); - FLUENT_SETTING(TDuration, CancelAfter); - FLUENT_SETTING_DEFAULT(bool, UseClientTimeoutForOperation, true); +}; + +template<typename TDerived> +struct TOperationRequestSettings : public TSimpleRequestSettings<TDerived> { + using TSelf = TDerived; + + /* Cancel/timeout operation settings available from 18-8 YDB server version */ + FLUENT_SETTING(TDuration, OperationTimeout); + FLUENT_SETTING(TDuration, CancelAfter); + FLUENT_SETTING_DEFAULT(bool, UseClientTimeoutForOperation, true); FLUENT_SETTING_DEFAULT(bool, ReportCostInfo, false); - + TOperationRequestSettings() = default; template <typename T> @@ -64,11 +64,11 @@ struct TOperationRequestSettings : public TSimpleRequestSettings<TDerived> { , ReportCostInfo_(other.ReportCostInfo_) {} - TSelf& CancelAfterWithTimeout(const TDuration& cancelAfter, const TDuration& operationTimeout) { - CancelAfter_ = cancelAfter; - OperationTimeout_ = operationTimeout; - return static_cast<TSelf&>(*this); - } -}; - -} // namespace NYdb + TSelf& CancelAfterWithTimeout(const TDuration& cancelAfter, const TDuration& operationTimeout) { + CancelAfter_ = cancelAfter; + OperationTimeout_ = operationTimeout; + return static_cast<TSelf&>(*this); + } +}; + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/ydb_types/status/status.cpp b/ydb/public/sdk/cpp/client/ydb_types/status/status.cpp index eed05872bf..d3f2ca05af 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/status/status.cpp +++ b/ydb/public/sdk/cpp/client/ydb_types/status/status.cpp @@ -1,73 +1,73 @@ -#include "status.h" - -#define INCLUDE_YDB_INTERNAL_H +#include "status.h" + +#define INCLUDE_YDB_INTERNAL_H #include <ydb/public/sdk/cpp/client/impl/ydb_internal/plain_status/status.h> -#undef INCLUDE_YDB_INTERNAL_H - +#undef INCLUDE_YDB_INTERNAL_H + #include <ydb/public/sdk/cpp/client/ydb_types/exceptions/exceptions.h> -namespace NYdb { - -class TStatus::TImpl { -public: - const TPlainStatus Status; - +namespace NYdb { + +class TStatus::TImpl { +public: + const TPlainStatus Status; + TImpl(TPlainStatus&& status) - : Status(std::move(status)) - { } - + : Status(std::move(status)) + { } + void CheckStatusOk(const TStringType& str) const { - if (!Status.Ok()) { + if (!Status.Ok()) { ThrowFatalError(TStringType("Attempt to use result with not successfull status. ") + str + "\n"); - } - } - + } + } + void RaiseError(const TStringType& str) const { ythrow TContractViolation(str); - } -}; - + } +}; + TStatus::TStatus(EStatus statusCode, NYql::TIssues&& issues) : Impl_(std::make_shared<TImpl>(TPlainStatus{statusCode, std::move(issues)})) -{ } - +{ } + TStatus::TStatus(TPlainStatus&& plain) : Impl_(std::make_shared<TImpl>(std::move(plain))) -{ } - -const NYql::TIssues& TStatus::GetIssues() const { - return Impl_->Status.Issues; -} - -EStatus TStatus::GetStatus() const { - return Impl_->Status.Status; -} - -bool TStatus::IsSuccess() const { - return Impl_->Status.Status == EStatus::SUCCESS; -} - -bool TStatus::IsTransportError() const { - return static_cast<size_t>(Impl_->Status.Status) >= TRANSPORT_STATUSES_FIRST - && static_cast<size_t>(Impl_->Status.Status) <= TRANSPORT_STATUSES_LAST; -} - +{ } + +const NYql::TIssues& TStatus::GetIssues() const { + return Impl_->Status.Issues; +} + +EStatus TStatus::GetStatus() const { + return Impl_->Status.Status; +} + +bool TStatus::IsSuccess() const { + return Impl_->Status.Status == EStatus::SUCCESS; +} + +bool TStatus::IsTransportError() const { + return static_cast<size_t>(Impl_->Status.Status) >= TRANSPORT_STATUSES_FIRST + && static_cast<size_t>(Impl_->Status.Status) <= TRANSPORT_STATUSES_LAST; +} + void TStatus::CheckStatusOk(const TStringType& str) const { - Impl_->CheckStatusOk(str); -} - + Impl_->CheckStatusOk(str); +} + void TStatus::RaiseError(const TStringType& str) const { - Impl_->RaiseError(str); -} - + Impl_->RaiseError(str); +} + const TStringType& TStatus::GetEndpoint() const { - return Impl_->Status.Endpoint; -} - + return Impl_->Status.Endpoint; +} + const std::multimap<TStringType, TStringType>& TStatus::GetResponseMetadata() const { - return Impl_->Status.Metadata; -} - + return Impl_->Status.Metadata; +} + float TStatus::GetConsumedRu() const { return Impl_->Status.ConstInfo.consumed_units(); } @@ -82,4 +82,4 @@ bool TStreamPartStatus::EOS() const { return GetStatus() == EStatus::CLIENT_OUT_OF_RANGE; } -} // namespace NYdb +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/ydb_types/status/status.h b/ydb/public/sdk/cpp/client/ydb_types/status/status.h index 2f5354a106..7185a03964 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/status/status.h +++ b/ydb/public/sdk/cpp/client/ydb_types/status/status.h @@ -1,46 +1,46 @@ -#pragma once - +#pragma once + #include <ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers/handlers.h> #include <ydb/public/sdk/cpp/client/impl/ydb_internal/common/type_switcher.h> #include <ydb/public/sdk/cpp/client/ydb_types/ydb.h> #include <ydb/library/yql/public/issue/yql_issue.h> - -#include <library/cpp/threading/future/future.h> - -namespace NYdb { - -//! Internal status representation -struct TPlainStatus; - -//! Represents status of call -class TStatus { -public: + +#include <library/cpp/threading/future/future.h> + +namespace NYdb { + +//! Internal status representation +struct TPlainStatus; + +//! Represents status of call +class TStatus { +public: TStatus(EStatus statusCode, NYql::TIssues&& issues); TStatus(TPlainStatus&& plain); - - EStatus GetStatus() const; - const NYql::TIssues& GetIssues() const; - bool IsSuccess() const; - bool IsTransportError() const; + + EStatus GetStatus() const; + const NYql::TIssues& GetIssues() const; + bool IsSuccess() const; + bool IsTransportError() const; const TStringType& GetEndpoint() const; const std::multimap<TStringType, TStringType>& GetResponseMetadata() const; float GetConsumedRu() const; - -protected: + +protected: void CheckStatusOk(const TStringType& str) const; void RaiseError(const TStringType& str) const; -private: - class TImpl; - std::shared_ptr<TImpl> Impl_; -}; - -using TAsyncStatus = NThreading::TFuture<TStatus>; - +private: + class TImpl; + std::shared_ptr<TImpl> Impl_; +}; + +using TAsyncStatus = NThreading::TFuture<TStatus>; + class TStreamPartStatus : public TStatus { public: TStreamPartStatus(TStatus&& status); bool EOS() const; }; -} // namespace NYdb +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/ydb_types/status/ya.make b/ydb/public/sdk/cpp/client/ydb_types/status/ya.make index 5a62725483..0a5bdd0da4 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/status/ya.make +++ b/ydb/public/sdk/cpp/client/ydb_types/status/ya.make @@ -1,17 +1,17 @@ -LIBRARY() - +LIBRARY() + OWNER(g:kikimr) - -SRCS( - status.cpp -) - -PEERDIR( + +SRCS( + status.cpp +) + +PEERDIR( library/cpp/threading/future ydb/public/sdk/cpp/client/impl/ydb_internal/plain_status ydb/public/sdk/cpp/client/ydb_types ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers ydb/library/yql/public/issue -) - -END() +) + +END() diff --git a/ydb/public/sdk/cpp/client/ydb_types/ya.make b/ydb/public/sdk/cpp/client/ydb_types/ya.make index 3e3bf15f23..d1419ecd34 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/ya.make +++ b/ydb/public/sdk/cpp/client/ydb_types/ya.make @@ -1,14 +1,14 @@ -LIBRARY() - +LIBRARY() + OWNER(g:kikimr) - -PEERDIR( - contrib/libs/protobuf - library/cpp/grpc/client + +PEERDIR( + contrib/libs/protobuf + library/cpp/grpc/client ydb/library/yql/public/issue -) - +) + GENERATE_ENUM_SERIALIZATION(s3_settings.h) -GENERATE_ENUM_SERIALIZATION(status_codes.h) - -END() +GENERATE_ENUM_SERIALIZATION(status_codes.h) + +END() diff --git a/ydb/public/sdk/cpp/client/ydb_types/ydb.h b/ydb/public/sdk/cpp/client/ydb_types/ydb.h index 284f158b13..2321d94c2a 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/ydb.h +++ b/ydb/public/sdk/cpp/client/ydb_types/ydb.h @@ -1,29 +1,29 @@ -#pragma once - -#include "status_codes.h" - -namespace NYdb { - -enum class EDiscoveryMode { - //! Block in ctor (driver or client if database and/or auth token is overridden by client settings) - //! until we get list of endpoints, if list of endpoints become empty during executing requests - //! corresponding error will be returned. - //! Note: Even in Sync mode SDK will perform lazy async update of endpoints list - Sync, - //! Do not block in ctor to get endpoint list. - //! If list of endpoints is empty in time of request (or becomes empty during execution) the request will be pending until - //! we got endpoint list. The error will be returned if the endpoint list - //! is empty and discovery failed - //! This method is a bit more "user friendly" but can produce additional hidden latency - Async -}; - -enum class EBalancingPolicy { - //! Use all available cluster nodes regardless datacenter locality - UseAllNodes, - //! Use preferable location, - //! params is a name of location (VLA, MAN), if params is empty local datacenter is used - UsePreferableLocation -}; - -} // namespace NYdb +#pragma once + +#include "status_codes.h" + +namespace NYdb { + +enum class EDiscoveryMode { + //! Block in ctor (driver or client if database and/or auth token is overridden by client settings) + //! until we get list of endpoints, if list of endpoints become empty during executing requests + //! corresponding error will be returned. + //! Note: Even in Sync mode SDK will perform lazy async update of endpoints list + Sync, + //! Do not block in ctor to get endpoint list. + //! If list of endpoints is empty in time of request (or becomes empty during execution) the request will be pending until + //! we got endpoint list. The error will be returned if the endpoint list + //! is empty and discovery failed + //! This method is a bit more "user friendly" but can produce additional hidden latency + Async +}; + +enum class EBalancingPolicy { + //! Use all available cluster nodes regardless datacenter locality + UseAllNodes, + //! Use preferable location, + //! params is a name of location (VLA, MAN), if params is empty local datacenter is used + UsePreferableLocation +}; + +} // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/ydb_value/ut/ya.make b/ydb/public/sdk/cpp/client/ydb_value/ut/ya.make index 59696f2b9a..aaf085d2bd 100644 --- a/ydb/public/sdk/cpp/client/ydb_value/ut/ya.make +++ b/ydb/public/sdk/cpp/client/ydb_value/ut/ya.make @@ -1,28 +1,28 @@ UNITTEST_FOR(ydb/public/sdk/cpp/client/ydb_value) - -OWNER( - dcherednik - g:kikimr -) - -IF (SANITIZER_TYPE) - TIMEOUT(1200) - SIZE(LARGE) - TAG(ya:fat) -ELSE() - TIMEOUT(600) - SIZE(MEDIUM) -ENDIF() - -FORK_SUBTESTS() - -PEERDIR( + +OWNER( + dcherednik + g:kikimr +) + +IF (SANITIZER_TYPE) + TIMEOUT(1200) + SIZE(LARGE) + TAG(ya:fat) +ELSE() + TIMEOUT(600) + SIZE(MEDIUM) +ENDIF() + +FORK_SUBTESTS() + +PEERDIR( ydb/public/lib/json_value ydb/public/lib/yson_value -) - -SRCS( - value_ut.cpp -) - -END() +) + +SRCS( + value_ut.cpp +) + +END() diff --git a/ydb/public/sdk/cpp/client/ydb_value/value.cpp b/ydb/public/sdk/cpp/client/ydb_value/value.cpp index 83dacb79c5..8c18d4d50f 100644 --- a/ydb/public/sdk/cpp/client/ydb_value/value.cpp +++ b/ydb/public/sdk/cpp/client/ydb_value/value.cpp @@ -1,5 +1,5 @@ -#include "value.h" - +#include "value.h" + #define INCLUDE_YDB_INTERNAL_H #include <ydb/public/sdk/cpp/client/impl/ydb_internal/value_helpers/helpers.h> #undef INCLUDE_YDB_INTERNAL_H @@ -7,15 +7,15 @@ #include <ydb/public/sdk/cpp/client/ydb_params/params.h> #include <ydb/public/sdk/cpp/client/ydb_proto/accessor.h> #include <ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers/handlers.h> - + #include <ydb/public/api/protos/ydb_value.pb.h> #include <library/cpp/containers/stack_vector/stack_vec.h> #include <ydb/library/yql/public/decimal/yql_decimal.h> -#include <util/generic/bitmap.h> -#include <util/generic/map.h> +#include <util/generic/bitmap.h> +#include <util/generic/map.h> #include <util/string/builder.h> namespace NYdb { diff --git a/ydb/public/sdk/cpp/client/ydb_value/value.h b/ydb/public/sdk/cpp/client/ydb_value/value.h index c935af67ad..4a163b9220 100644 --- a/ydb/public/sdk/cpp/client/ydb_value/value.h +++ b/ydb/public/sdk/cpp/client/ydb_value/value.h @@ -1,10 +1,10 @@ #pragma once -#include <util/datetime/base.h> +#include <util/datetime/base.h> #include <util/generic/maybe.h> -#include <memory> - +#include <memory> + namespace Ydb { class Type; class Value; diff --git a/ydb/public/sdk/cpp/client/ydb_value/value_ut.cpp b/ydb/public/sdk/cpp/client/ydb_value/value_ut.cpp index 2895acb944..d7508a5f9b 100644 --- a/ydb/public/sdk/cpp/client/ydb_value/value_ut.cpp +++ b/ydb/public/sdk/cpp/client/ydb_value/value_ut.cpp @@ -4,9 +4,9 @@ #include <ydb/public/lib/json_value/ydb_json_value.h> #include <ydb/public/lib/yson_value/ydb_yson_value.h> -#include <library/cpp/testing/unittest/registar.h> -#include <library/cpp/testing/unittest/tests_data.h> - +#include <library/cpp/testing/unittest/registar.h> +#include <library/cpp/testing/unittest/tests_data.h> + #include <google/protobuf/messagext.h> namespace NYdb { diff --git a/ydb/public/sdk/cpp/client/ydb_value/ya.make b/ydb/public/sdk/cpp/client/ydb_value/ya.make index 7d8df655b3..c7db140950 100644 --- a/ydb/public/sdk/cpp/client/ydb_value/ya.make +++ b/ydb/public/sdk/cpp/client/ydb_value/ya.make @@ -1,27 +1,27 @@ -LIBRARY() - -OWNER( - dcherednik - g:kikimr -) - -SRCS( - value.cpp -) - -GENERATE_ENUM_SERIALIZATION(value.h) - -PEERDIR( - library/cpp/containers/stack_vector +LIBRARY() + +OWNER( + dcherednik + g:kikimr +) + +SRCS( + value.cpp +) + +GENERATE_ENUM_SERIALIZATION(value.h) + +PEERDIR( + library/cpp/containers/stack_vector ydb/public/api/protos ydb/public/sdk/cpp/client/impl/ydb_internal/value_helpers ydb/public/sdk/cpp/client/ydb_proto ydb/public/sdk/cpp/client/ydb_types/fatal_error_handlers ydb/library/yql/public/decimal -) - -END() - +) + +END() + RECURSE_FOR_TESTS( ut ) diff --git a/ydb/public/sdk/python/ydb/resolver.py b/ydb/public/sdk/python/ydb/resolver.py index 5170227269..54712c0085 100644 --- a/ydb/public/sdk/python/ydb/resolver.py +++ b/ydb/public/sdk/python/ydb/resolver.py @@ -137,7 +137,7 @@ class DiscoveryEndpointsResolver(object): self._endpoints_iter = itertools.cycle(self._endpoints) def _add_debug_details(self, message, *args): - self.logger.debug(message, *args) + self.logger.debug(message, *args) message = message % args with self._lock: self._debug_details_items.append(message) diff --git a/ydb/public/tools/lib/cmds/ya.make b/ydb/public/tools/lib/cmds/ya.make index 9e1ca7c4e7..61698522e9 100644 --- a/ydb/public/tools/lib/cmds/ya.make +++ b/ydb/public/tools/lib/cmds/ya.make @@ -1,13 +1,13 @@ -PY23_LIBRARY() -OWNER(g:kikimr) - -PY_SRCS( - __init__.py -) - -PEERDIR( - ydb/tests/library - library/python/testing/recipe -) - -END() +PY23_LIBRARY() +OWNER(g:kikimr) + +PY_SRCS( + __init__.py +) + +PEERDIR( + ydb/tests/library + library/python/testing/recipe +) + +END() diff --git a/ydb/services/datastreams/datastreams_proxy.cpp b/ydb/services/datastreams/datastreams_proxy.cpp index e6e4685af1..03a6d06213 100644 --- a/ydb/services/datastreams/datastreams_proxy.cpp +++ b/ydb/services/datastreams/datastreams_proxy.cpp @@ -22,7 +22,7 @@ using namespace NKikimrClient; using grpc::Status; namespace NKikimr::NDataStreams::V1 { - const TString YDS_SERVICE_TYPE = "data-streams"; + const TString YDS_SERVICE_TYPE = "data-streams"; using namespace NGRpcService; using namespace NGRpcProxy::V1; @@ -263,7 +263,7 @@ namespace NKikimr::NDataStreams::V1 { } void Bootstrap(const TActorContext& ctx); - void ModifyPersqueueConfig(const TActorContext& ctx, + void ModifyPersqueueConfig(const TActorContext& ctx, NKikimrSchemeOp::TPersQueueGroupDescription& groupConfig, const NKikimrSchemeOp::TPersQueueGroupDescription& pqGroupDescription, const NKikimrSchemeOp::TDirEntry& selfInfo); @@ -275,14 +275,14 @@ namespace NKikimr::NDataStreams::V1 { Become(&TBase::StateWork); } - void TUpdateShardCountActor::ModifyPersqueueConfig( - const TActorContext& ctx, + void TUpdateShardCountActor::ModifyPersqueueConfig( + const TActorContext& ctx, NKikimrSchemeOp::TPersQueueGroupDescription& groupConfig, const NKikimrSchemeOp::TPersQueueGroupDescription& pqGroupDescription, const NKikimrSchemeOp::TDirEntry& selfInfo - ) { - Y_UNUSED(pqGroupDescription); - Y_UNUSED(selfInfo); + ) { + Y_UNUSED(pqGroupDescription); + Y_UNUSED(selfInfo); TString error; if (!ValidateShardsCount(*GetProtoRequest(), groupConfig, error)) { @@ -304,7 +304,7 @@ namespace NKikimr::NDataStreams::V1 { } void Bootstrap(const TActorContext& ctx); - void ModifyPersqueueConfig(const TActorContext& ctx, + void ModifyPersqueueConfig(const TActorContext& ctx, NKikimrSchemeOp::TPersQueueGroupDescription& groupConfig, const NKikimrSchemeOp::TPersQueueGroupDescription& pqGroupDescription, const NKikimrSchemeOp::TDirEntry& selfInfo); @@ -316,14 +316,14 @@ namespace NKikimr::NDataStreams::V1 { Become(&TBase::StateWork); } - void TUpdateStreamActor::ModifyPersqueueConfig( - const TActorContext& ctx, + void TUpdateStreamActor::ModifyPersqueueConfig( + const TActorContext& ctx, NKikimrSchemeOp::TPersQueueGroupDescription& groupConfig, const NKikimrSchemeOp::TPersQueueGroupDescription& pqGroupDescription, const NKikimrSchemeOp::TDirEntry& selfInfo - ) { - Y_UNUSED(pqGroupDescription); - Y_UNUSED(selfInfo); + ) { + Y_UNUSED(pqGroupDescription); + Y_UNUSED(selfInfo); TString error; if (!ValidateShardsCount(*GetProtoRequest(), groupConfig, error) @@ -351,7 +351,7 @@ namespace NKikimr::NDataStreams::V1 { } void Bootstrap(const TActorContext& ctx); - void ModifyPersqueueConfig(const TActorContext& ctx, + void ModifyPersqueueConfig(const TActorContext& ctx, NKikimrSchemeOp::TPersQueueGroupDescription& groupConfig, const NKikimrSchemeOp::TPersQueueGroupDescription& pqGroupDescription, const NKikimrSchemeOp::TDirEntry& selfInfo); @@ -363,14 +363,14 @@ namespace NKikimr::NDataStreams::V1 { Become(&TBase::StateWork); } - void TSetWriteQuotaActor::ModifyPersqueueConfig( - const TActorContext& ctx, + void TSetWriteQuotaActor::ModifyPersqueueConfig( + const TActorContext& ctx, NKikimrSchemeOp::TPersQueueGroupDescription& groupConfig, const NKikimrSchemeOp::TPersQueueGroupDescription& pqGroupDescription, const NKikimrSchemeOp::TDirEntry& selfInfo - ) { - Y_UNUSED(pqGroupDescription); - Y_UNUSED(selfInfo); + ) { + Y_UNUSED(pqGroupDescription); + Y_UNUSED(selfInfo); TString error; if (!ValidateWriteSpeedLimit(*GetProtoRequest(), error, ctx)) { @@ -399,14 +399,14 @@ namespace NKikimr::NDataStreams::V1 { TBase::Become(&TBase::StateWork); } - void ModifyPersqueueConfig( - const TActorContext& ctx, + void ModifyPersqueueConfig( + const TActorContext& ctx, NKikimrSchemeOp::TPersQueueGroupDescription& groupConfig, const NKikimrSchemeOp::TPersQueueGroupDescription& pqGroupDescription, const NKikimrSchemeOp::TDirEntry& selfInfo - ) { - Y_UNUSED(pqGroupDescription); - Y_UNUSED(selfInfo); + ) { + Y_UNUSED(pqGroupDescription); + Y_UNUSED(selfInfo); TString error; if (!ValidateRetentionPeriod(*this->GetProtoRequest(), groupConfig, ShouldIncrease, error)) { @@ -493,9 +493,9 @@ namespace NKikimr::NDataStreams::V1 { } void TDescribeStreamActor::HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) { - const NSchemeCache::TSchemeCacheNavigate* result = ev->Get()->Request.Get(); - Y_VERIFY(result->ResultSet.size() == 1); // describe only one topic - const auto& response = result->ResultSet.front(); + const NSchemeCache::TSchemeCacheNavigate* result = ev->Get()->Request.Get(); + Y_VERIFY(result->ResultSet.size() == 1); // describe only one topic + const auto& response = result->ResultSet.front(); const TString path = JoinSeq("/", response.Path); if (ReplyIfNotTopic(ev, ctx)) { @@ -535,29 +535,29 @@ namespace NKikimr::NDataStreams::V1 { void TDescribeStreamActor::ReplyAndDie(const TActorContext& ctx) { Ydb::DataStreams::V1::DescribeStreamResult result; - auto& pqConfig = PQGroup.GetPQTabletConfig(); + auto& pqConfig = PQGroup.GetPQTabletConfig(); ui32 writeSpeed = pqConfig.GetPartitionConfig().GetWriteSpeedInBytesPerSecond() / 1024; auto& description = *result.mutable_stream_description(); description.set_stream_name(GetProtoRequest()->stream_name()); ui32 retentionPeriodHours = TInstant::Seconds(pqConfig.GetPartitionConfig().GetLifetimeSeconds()).Hours(); description.set_retention_period_hours(retentionPeriodHours); description.set_write_quota_kb_per_sec(writeSpeed); - if (SelfInfo.GetCreateFinished()) { - description.set_stream_status(Ydb::DataStreams::V1::StreamDescription::ACTIVE); - } else { - description.set_stream_status(Ydb::DataStreams::V1::StreamDescription::CREATING); + if (SelfInfo.GetCreateFinished()) { + description.set_stream_status(Ydb::DataStreams::V1::StreamDescription::ACTIVE); + } else { + description.set_stream_status(Ydb::DataStreams::V1::StreamDescription::CREATING); } bool startShardFound = GetProtoRequest()->exclusive_start_shard_id().empty(); description.set_has_more_shards(false); - description.set_owner(SelfInfo.GetOwner()); - description.set_stream_creation_timestamp(TInstant::MilliSeconds(SelfInfo.GetCreateStep()).Seconds()); + description.set_owner(SelfInfo.GetOwner()); + description.set_stream_creation_timestamp(TInstant::MilliSeconds(SelfInfo.GetCreateStep()).Seconds()); int limit = GetProtoRequest()->limit() == 0 ? 100 : GetProtoRequest()->limit(); - for (uint32_t i = 0; i < (uint32_t)PQGroup.GetPartitions().size(); ++i) { - ui32 partitionId = PQGroup.GetPartitions(i).GetPartitionId(); + for (uint32_t i = 0; i < (uint32_t)PQGroup.GetPartitions().size(); ++i) { + ui32 partitionId = PQGroup.GetPartitions(i).GetPartitionId(); TString shardName = GetShardName(partitionId); if (shardName == GetProtoRequest()->exclusive_start_shard_id()) { startShardFound = true; @@ -569,7 +569,7 @@ namespace NKikimr::NDataStreams::V1 { auto* shard = description.add_shards(); shard->set_shard_id(shardName); auto* rangeProto = shard->mutable_hash_key_range(); - auto range = RangeFromShardNumber(partitionId, PQGroup.GetPartitions().size()); + auto range = RangeFromShardNumber(partitionId, PQGroup.GetPartitions().size()); rangeProto->set_starting_hash_key(Uint128ToDecimalString(range.Start)); rangeProto->set_ending_hash_key(Uint128ToDecimalString(range.End)); auto it = StartEndOffsetsPerPartition.find(partitionId); @@ -809,9 +809,9 @@ namespace NKikimr::NDataStreams::V1 { } void TListStreamConsumersActor::HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) { - const NSchemeCache::TSchemeCacheNavigate* result = ev->Get()->Request.Get(); - Y_VERIFY(result->ResultSet.size() == 1); // describe only one topic - + const NSchemeCache::TSchemeCacheNavigate* result = ev->Get()->Request.Get(); + Y_VERIFY(result->ResultSet.size() == 1); // describe only one topic + if (ReplyIfNotTopic(ev, ctx)) { return; } @@ -893,24 +893,24 @@ namespace NKikimr::NDataStreams::V1 { Become(&TRegisterStreamConsumerActor::StateWork); } - void TRegisterStreamConsumerActor::ModifyPersqueueConfig( - const TActorContext& ctx, + void TRegisterStreamConsumerActor::ModifyPersqueueConfig( + const TActorContext& ctx, NKikimrSchemeOp::TPersQueueGroupDescription& groupConfig, const NKikimrSchemeOp::TPersQueueGroupDescription& pqGroupDescription, const NKikimrSchemeOp::TDirEntry& selfInfo - ) { - Y_UNUSED(pqGroupDescription); - + ) { + Y_UNUSED(pqGroupDescription); + auto* pqConfig = groupConfig.MutablePQTabletConfig(); Ydb::PersQueue::V1::TopicSettings::ReadRule readRule; readRule.set_consumer_name(ConsumerName); readRule.set_supported_format(Ydb::PersQueue::V1::TopicSettings_Format_FORMAT_BASE); readRule.set_starting_message_timestamp_ms(TInstant::Now().MilliSeconds()); readRule.set_important(false); - readRule.set_service_type(YDS_SERVICE_TYPE); + readRule.set_service_type(YDS_SERVICE_TYPE); if (readRule.version() == 0) { - readRule.set_version(selfInfo.GetVersion().GetPQVersion()); + readRule.set_version(selfInfo.GetVersion().GetPQVersion()); } auto serviceTypes = GetSupportedClientServiceTypes(ctx); TString error = AddReadRuleToConfig(pqConfig, readRule, serviceTypes, ctx); @@ -967,19 +967,19 @@ namespace NKikimr::NDataStreams::V1 { Become(&TDeregisterStreamConsumerActor::StateWork); } - void TDeregisterStreamConsumerActor::ModifyPersqueueConfig( - const TActorContext& ctx, + void TDeregisterStreamConsumerActor::ModifyPersqueueConfig( + const TActorContext& ctx, NKikimrSchemeOp::TPersQueueGroupDescription& groupConfig, const NKikimrSchemeOp::TPersQueueGroupDescription& pqGroupDescription, const NKikimrSchemeOp::TDirEntry& selfInfo - ) { - Y_UNUSED(selfInfo); - auto error = RemoveReadRuleFromConfig( - groupConfig.MutablePQTabletConfig(), - pqGroupDescription.GetPQTabletConfig(), - GetProtoRequest()->consumer_name(), - ctx - ); + ) { + Y_UNUSED(selfInfo); + auto error = RemoveReadRuleFromConfig( + groupConfig.MutablePQTabletConfig(), + pqGroupDescription.GetPQTabletConfig(), + GetProtoRequest()->consumer_name(), + ctx + ); if (!error.Empty()) { return ReplyWithError(Ydb::StatusIds::NOT_FOUND, Ydb::PersQueue::ErrorCode::BAD_REQUEST, error, ctx); } diff --git a/ydb/services/datastreams/datastreams_ut.cpp b/ydb/services/datastreams/datastreams_ut.cpp index 4fedce232f..2ede2a2d05 100644 --- a/ydb/services/datastreams/datastreams_ut.cpp +++ b/ydb/services/datastreams/datastreams_ut.cpp @@ -42,7 +42,7 @@ public: appConfig.MutablePQConfig()->MutableQuotingConfig()->SetPartitionReadQuotaIsTwiceWriteQuota(true); appConfig.MutablePQConfig()->MutableBillingMeteringConfig()->SetEnabled(true); appConfig.MutablePQConfig()->MutableBillingMeteringConfig()->SetFlushIntervalSec(1); - appConfig.MutablePQConfig()->AddClientServiceType()->SetName("data-streams"); + appConfig.MutablePQConfig()->AddClientServiceType()->SetName("data-streams"); MeteringFile = MakeHolder<TTempFileHandle>("meteringData.txt"); appConfig.MutableMeteringConfig()->SetMeteringFilePath(MeteringFile->Name()); @@ -157,13 +157,13 @@ Y_UNIT_TEST_SUITE(DataStreams) { auto result = testServer.DataStreamsClient->DescribeStream(streamName).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - + UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_description().stream_status(), YDS_V1::StreamDescription::ACTIVE); UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_description().stream_name(), streamName); - UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_description().write_quota_kb_per_sec(), 1024); - UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_description().retention_period_hours(), 24); - + UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_description().write_quota_kb_per_sec(), 1024); + UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_description().retention_period_hours(), 24); + UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_description().shards().size(), 3); UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_description().shards(0).sequence_number_range().starting_sequence_number(), "0"); UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_description().shards(0).hash_key_range().starting_hash_key(), "0"); diff --git a/ydb/services/lib/actors/pq_schema_actor.cpp b/ydb/services/lib/actors/pq_schema_actor.cpp index ecbe921cae..f48e398488 100644 --- a/ydb/services/lib/actors/pq_schema_actor.cpp +++ b/ydb/services/lib/actors/pq_schema_actor.cpp @@ -22,54 +22,54 @@ namespace NKikimr::NGRpcProxy::V1 { TClientServiceTypes GetSupportedClientServiceTypes(const TActorContext& ctx) { TClientServiceTypes serviceTypes; - const auto& pqConfig = AppData(ctx)->PQConfig; + const auto& pqConfig = AppData(ctx)->PQConfig; ui32 count = pqConfig.GetDefaultClientServiceType().GetMaxReadRulesCountPerTopic(); if (count == 0) count = Max<ui32>(); TString name = pqConfig.GetDefaultClientServiceType().GetName(); serviceTypes.insert({name, {name, count}}); - for (const auto& serviceType : pqConfig.GetClientServiceType()) { + for (const auto& serviceType : pqConfig.GetClientServiceType()) { ui32 count = serviceType.GetMaxReadRulesCountPerTopic(); if (count == 0) count = Max<ui32>(); TString name = serviceType.GetName(); serviceTypes.insert({name, {name, count}}); - } - return serviceTypes; - } - - TString ReadRuleServiceTypeMigration(NKikimrPQ::TPQTabletConfig *config, const TActorContext& ctx) { - auto rrServiceTypes = config->MutableReadRuleServiceTypes(); - if (config->ReadRuleServiceTypesSize() > config->ReadRulesSize()) { - rrServiceTypes->Clear(); - } - if (config->ReadRuleServiceTypesSize() < config->ReadRulesSize()) { - rrServiceTypes->Reserve(config->ReadRulesSize()); - const auto& pqConfig = AppData(ctx)->PQConfig; - if (pqConfig.GetDisallowDefaultClientServiceType()) { - return "service type must be set for all read rules"; - } - for (ui64 i = rrServiceTypes->size(); i < config->ReadRulesSize(); ++i) { - *rrServiceTypes->Add() = pqConfig.GetDefaultClientServiceType().GetName(); - } - } - return ""; - } - - TString AddReadRuleToConfig( - NKikimrPQ::TPQTabletConfig* config, - const Ydb::PersQueue::V1::TopicSettings::ReadRule& rr, + } + return serviceTypes; + } + + TString ReadRuleServiceTypeMigration(NKikimrPQ::TPQTabletConfig *config, const TActorContext& ctx) { + auto rrServiceTypes = config->MutableReadRuleServiceTypes(); + if (config->ReadRuleServiceTypesSize() > config->ReadRulesSize()) { + rrServiceTypes->Clear(); + } + if (config->ReadRuleServiceTypesSize() < config->ReadRulesSize()) { + rrServiceTypes->Reserve(config->ReadRulesSize()); + const auto& pqConfig = AppData(ctx)->PQConfig; + if (pqConfig.GetDisallowDefaultClientServiceType()) { + return "service type must be set for all read rules"; + } + for (ui64 i = rrServiceTypes->size(); i < config->ReadRulesSize(); ++i) { + *rrServiceTypes->Add() = pqConfig.GetDefaultClientServiceType().GetName(); + } + } + return ""; + } + + TString AddReadRuleToConfig( + NKikimrPQ::TPQTabletConfig* config, + const Ydb::PersQueue::V1::TopicSettings::ReadRule& rr, const TClientServiceTypes& supportedClientServiceTypes, - const TActorContext& ctx - ) { + const TActorContext& ctx + ) { auto consumerName = NPersQueue::ConvertNewConsumerName(rr.consumer_name(), ctx); if(consumerName.find("/") != TString::npos || consumerName.find("|") != TString::npos) { return TStringBuilder() << "consumer '" << rr.consumer_name() << "' has illegal symbols"; } - { - TString migrationError = ReadRuleServiceTypeMigration(config, ctx); - if (migrationError) { - return migrationError; - } - } + { + TString migrationError = ReadRuleServiceTypeMigration(config, ctx); + if (migrationError) { + return migrationError; + } + } config->AddReadRules(consumerName); @@ -102,29 +102,29 @@ namespace NKikimr::NGRpcProxy::V1 { if (rr.important()) config->MutablePartitionConfig()->AddImportantClientId(consumerName); - if (!rr.service_type().empty()) { + if (!rr.service_type().empty()) { if (!supportedClientServiceTypes.contains(rr.service_type())) { - return TStringBuilder() << "Unknown read rule service type '" << rr.service_type() - << "' for consumer '" << rr.consumer_name() << "'"; - } - config->AddReadRuleServiceTypes(rr.service_type()); - } else { - const auto& pqConfig = AppData(ctx)->PQConfig; - if (pqConfig.GetDisallowDefaultClientServiceType()) { - return TStringBuilder() << "service type cannot be empty for consumer '" << rr.consumer_name() << "'"; - } - const auto& defaultCientServiceType = pqConfig.GetDefaultClientServiceType().GetName(); - config->AddReadRuleServiceTypes(defaultCientServiceType); - } + return TStringBuilder() << "Unknown read rule service type '" << rr.service_type() + << "' for consumer '" << rr.consumer_name() << "'"; + } + config->AddReadRuleServiceTypes(rr.service_type()); + } else { + const auto& pqConfig = AppData(ctx)->PQConfig; + if (pqConfig.GetDisallowDefaultClientServiceType()) { + return TStringBuilder() << "service type cannot be empty for consumer '" << rr.consumer_name() << "'"; + } + const auto& defaultCientServiceType = pqConfig.GetDefaultClientServiceType().GetName(); + config->AddReadRuleServiceTypes(defaultCientServiceType); + } return ""; } - TString RemoveReadRuleFromConfig( - NKikimrPQ::TPQTabletConfig* config, - const NKikimrPQ::TPQTabletConfig& originalConfig, - const TString& consumerName, - const TActorContext& ctx - ) { + TString RemoveReadRuleFromConfig( + NKikimrPQ::TPQTabletConfig* config, + const NKikimrPQ::TPQTabletConfig& originalConfig, + const TString& consumerName, + const TActorContext& ctx + ) { THashSet<TString> rulesToRemove; rulesToRemove.insert(consumerName); @@ -134,7 +134,7 @@ namespace NKikimr::NGRpcProxy::V1 { config->ClearConsumerFormatVersions(); config->ClearConsumerCodecs(); config->MutablePartitionConfig()->ClearImportantClientId(); - config->ClearReadRuleServiceTypes(); + config->ClearReadRuleServiceTypes(); for (const auto& importantConsumer : originalConfig.GetPartitionConfig().GetImportantClientId()) { if (rulesToRemove.find(importantConsumer) == rulesToRemove.end()) { @@ -142,7 +142,7 @@ namespace NKikimr::NGRpcProxy::V1 { } } - const auto& pqConfig = AppData(ctx)->PQConfig; + const auto& pqConfig = AppData(ctx)->PQConfig; for (size_t i = 0; i < originalConfig.ReadRulesSize(); i++) { if (auto it = rulesToRemove.find(originalConfig.GetReadRules(i)); it != rulesToRemove.end()) { rulesToRemove.erase(it); @@ -158,15 +158,15 @@ namespace NKikimr::NGRpcProxy::V1 { ct->AddCodecs(originalConfig.GetConsumerCodecs(i).GetCodecs(j)); ct->AddIds(originalConfig.GetConsumerCodecs(i).GetIds(j)); } - if (i < originalConfig.ReadRuleServiceTypesSize()) { - config->AddReadRuleServiceTypes(originalConfig.GetReadRuleServiceTypes(i)); - } else { - if (pqConfig.GetDisallowDefaultClientServiceType()) { + if (i < originalConfig.ReadRuleServiceTypesSize()) { + config->AddReadRuleServiceTypes(originalConfig.GetReadRuleServiceTypes(i)); + } else { + if (pqConfig.GetDisallowDefaultClientServiceType()) { return TStringBuilder() << "service type cannot be empty for consumer '" - << originalConfig.GetReadRules(i) << "'"; - } - config->AddReadRuleServiceTypes(pqConfig.GetDefaultClientServiceType().GetName()); - } + << originalConfig.GetReadRules(i) << "'"; + } + config->AddReadRuleServiceTypes(pqConfig.GetDefaultClientServiceType().GetName()); + } } if (rulesToRemove.size() > 0) { @@ -387,15 +387,15 @@ namespace NKikimr::NGRpcProxy::V1 { return Ydb::StatusIds::BAD_REQUEST; } - { - error = ReadRuleServiceTypeMigration(config, ctx); - if (error) { - return Ydb::StatusIds::INTERNAL_ERROR; - } - } - const auto& supportedClientServiceTypes = GetSupportedClientServiceTypes(ctx); + { + error = ReadRuleServiceTypeMigration(config, ctx); + if (error) { + return Ydb::StatusIds::INTERNAL_ERROR; + } + } + const auto& supportedClientServiceTypes = GetSupportedClientServiceTypes(ctx); for (const auto& rr : settings.read_rules()) { - error = AddReadRuleToConfig(config, rr, supportedClientServiceTypes, ctx); + error = AddReadRuleToConfig(config, rr, supportedClientServiceTypes, ctx); if (!error.Empty()) { return Ydb::StatusIds::BAD_REQUEST; } diff --git a/ydb/services/lib/actors/pq_schema_actor.h b/ydb/services/lib/actors/pq_schema_actor.h index 90761e978f..1be2e7fa24 100644 --- a/ydb/services/lib/actors/pq_schema_actor.h +++ b/ydb/services/lib/actors/pq_schema_actor.h @@ -12,37 +12,37 @@ namespace NKikimr::NGRpcProxy::V1 { - Ydb::StatusIds::StatusCode FillProposeRequestImpl( - const TString& name, - const Ydb::PersQueue::V1::TopicSettings& settings, + Ydb::StatusIds::StatusCode FillProposeRequestImpl( + const TString& name, + const Ydb::PersQueue::V1::TopicSettings& settings, NKikimrSchemeOp::TModifyScheme& modifyScheme, - const TActorContext& ctx, - bool alter, - TString& error - ); - + const TActorContext& ctx, + bool alter, + TString& error + ); + struct TClientServiceType { TString Name; ui32 MaxCount; }; typedef std::map<TString, TClientServiceType> TClientServiceTypes; TClientServiceTypes GetSupportedClientServiceTypes(const TActorContext& ctx); - + // Returns true if have duplicated read rules bool CheckReadRulesConfig(const NKikimrPQ::TPQTabletConfig& config, const TClientServiceTypes& supportedReadRuleServiceTypes, TString& error); - - TString AddReadRuleToConfig( - NKikimrPQ::TPQTabletConfig *config, - const Ydb::PersQueue::V1::TopicSettings::ReadRule& rr, + + TString AddReadRuleToConfig( + NKikimrPQ::TPQTabletConfig *config, + const Ydb::PersQueue::V1::TopicSettings::ReadRule& rr, const TClientServiceTypes& supportedReadRuleServiceTypes, - const TActorContext& ctx - ); - TString RemoveReadRuleFromConfig( - NKikimrPQ::TPQTabletConfig *config, + const TActorContext& ctx + ); + TString RemoveReadRuleFromConfig( + NKikimrPQ::TPQTabletConfig *config, const NKikimrPQ::TPQTabletConfig& originalConfig, - const TString& consumerName, - const TActorContext& ctx - ); + const TString& consumerName, + const TActorContext& ctx + ); NYql::TIssue FillIssue(const TString &errorReason, const Ydb::PersQueue::ErrorCode::ErrorCode errorCode); @@ -105,14 +105,14 @@ namespace NKikimr::NGRpcProxy::V1 { void SendDescribeProposeRequest(const NActors::TActorContext& ctx) { PrepareTopicPath(ctx); - auto navigateRequest = std::make_unique<NSchemeCache::TSchemeCacheNavigate>(); - navigateRequest->DatabaseName = CanonizePath(this->Request_->GetDatabaseName().GetOrElse("")); + auto navigateRequest = std::make_unique<NSchemeCache::TSchemeCacheNavigate>(); + navigateRequest->DatabaseName = CanonizePath(this->Request_->GetDatabaseName().GetOrElse("")); - NSchemeCache::TSchemeCacheNavigate::TEntry entry; - entry.Path = NKikimr::SplitPath(TopicPath); - entry.SyncVersion = true; - entry.Operation = NSchemeCache::TSchemeCacheNavigate::OpTopic; - navigateRequest->ResultSet.emplace_back(entry); + NSchemeCache::TSchemeCacheNavigate::TEntry entry; + entry.Path = NKikimr::SplitPath(TopicPath); + entry.SyncVersion = true; + entry.Operation = NSchemeCache::TSchemeCacheNavigate::OpTopic; + navigateRequest->ResultSet.emplace_back(entry); if (this->Request_->GetInternalToken().empty()) { if (AppData(ctx)->PQConfig.GetRequireCredentialsInNewProtocol()) { @@ -120,24 +120,24 @@ namespace NKikimr::NGRpcProxy::V1 { "Unauthenticated access is forbidden, please provide credentials", ctx); } } else { - navigateRequest->UserToken = new NACLib::TUserToken(this->Request_->GetInternalToken()); + navigateRequest->UserToken = new NACLib::TUserToken(this->Request_->GetInternalToken()); } if (!IsDead) { - ctx.Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvNavigateKeySet(navigateRequest.release())); + ctx.Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvNavigateKeySet(navigateRequest.release())); } } bool ReplyIfNotTopic(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) { - const NSchemeCache::TSchemeCacheNavigate* result = ev->Get()->Request.Get(); - Y_VERIFY(result->ResultSet.size() == 1); - const auto& response = result->ResultSet.front(); - const TString path = JoinPath(response.Path); + const NSchemeCache::TSchemeCacheNavigate* result = ev->Get()->Request.Get(); + Y_VERIFY(result->ResultSet.size() == 1); + const auto& response = result->ResultSet.front(); + const TString path = JoinPath(response.Path); if (ev->Get()->Request.Get()->ResultSet.size() != 1 || ev->Get()->Request.Get()->ResultSet.begin()->Kind != NSchemeCache::TSchemeCacheNavigate::KindTopic) { - this->Request_->RaiseIssue( - FillIssue( + this->Request_->RaiseIssue( + FillIssue( TStringBuilder() << "path '" << path << "' is not a stream", Ydb::PersQueue::ErrorCode::ERROR ) @@ -165,10 +165,10 @@ namespace NKikimr::NGRpcProxy::V1 { FillIssue( TStringBuilder() << "path '" << path << "' does not exist or you " << "do not have access rights", - Ydb::PersQueue::ErrorCode::ERROR - ) - ); - return TBase::Reply(Ydb::StatusIds::SCHEME_ERROR, ctx); + Ydb::PersQueue::ErrorCode::ERROR + ) + ); + return TBase::Reply(Ydb::StatusIds::SCHEME_ERROR, ctx); } case NSchemeCache::TSchemeCacheNavigate::EStatus::TableCreationNotComplete: { this->Request_->RaiseIssue( @@ -256,28 +256,28 @@ namespace NKikimr::NGRpcProxy::V1 { const TString& workingDir, const TString& name) { Y_UNUSED(name); - const auto& response = DescribeSchemeResult->Get()->Request.Get()->ResultSet.front(); + const auto& response = DescribeSchemeResult->Get()->Request.Get()->ResultSet.front(); NKikimrSchemeOp::TModifyScheme& modifyScheme(*proposal.Record.MutableTransaction()->MutableModifyScheme()); modifyScheme.SetOperationType(NKikimrSchemeOp::EOperationType::ESchemeOpAlterPersQueueGroup); modifyScheme.SetWorkingDir(workingDir); auto* config = modifyScheme.MutableAlterPersQueueGroup(); - Y_VERIFY(response.Self); - Y_VERIFY(response.PQGroupInfo); - config->CopyFrom(response.PQGroupInfo->Description); + Y_VERIFY(response.Self); + Y_VERIFY(response.PQGroupInfo); + config->CopyFrom(response.PQGroupInfo->Description); { auto applyIf = modifyScheme.AddApplyIf(); - applyIf->SetPathId(response.Self->Info.GetPathId()); - applyIf->SetPathVersion(response.Self->Info.GetPathVersion()); + applyIf->SetPathId(response.Self->Info.GetPathId()); + applyIf->SetPathVersion(response.Self->Info.GetPathVersion()); } - static_cast<TDerived*>(this)->ModifyPersqueueConfig( - ctx, - *config, - response.PQGroupInfo->Description, - response.Self->Info - ); + static_cast<TDerived*>(this)->ModifyPersqueueConfig( + ctx, + *config, + response.PQGroupInfo->Description, + response.Self->Info + ); this->DescribeSchemeResult.Reset(); } @@ -296,7 +296,7 @@ namespace NKikimr::NGRpcProxy::V1 { } private: - THolder<NActors::TEventHandle<TEvTxProxySchemeCache::TEvNavigateKeySetResult>> DescribeSchemeResult; + THolder<NActors::TEventHandle<TEvTxProxySchemeCache::TEvNavigateKeySetResult>> DescribeSchemeResult; }; } diff --git a/ydb/services/persqueue_cluster_discovery/cluster_ordering/ut/ya.make b/ydb/services/persqueue_cluster_discovery/cluster_ordering/ut/ya.make index fbcd07dfa8..c1e137b002 100644 --- a/ydb/services/persqueue_cluster_discovery/cluster_ordering/ut/ya.make +++ b/ydb/services/persqueue_cluster_discovery/cluster_ordering/ut/ya.make @@ -3,7 +3,7 @@ UNITTEST_FOR(ydb/services/persqueue_cluster_discovery/cluster_ordering) OWNER( radix g:kikimr - g:logbroker + g:logbroker ) SRCS( diff --git a/ydb/services/persqueue_cluster_discovery/cluster_ordering/ya.make b/ydb/services/persqueue_cluster_discovery/cluster_ordering/ya.make index ca9ec26420..6978da1d9b 100644 --- a/ydb/services/persqueue_cluster_discovery/cluster_ordering/ya.make +++ b/ydb/services/persqueue_cluster_discovery/cluster_ordering/ya.make @@ -1,7 +1,7 @@ OWNER( radix g:kikimr - g:logbroker + g:logbroker ) LIBRARY() diff --git a/ydb/services/persqueue_cluster_discovery/ut/ya.make b/ydb/services/persqueue_cluster_discovery/ut/ya.make index 28f0cbb64c..352988d2fe 100644 --- a/ydb/services/persqueue_cluster_discovery/ut/ya.make +++ b/ydb/services/persqueue_cluster_discovery/ut/ya.make @@ -1,10 +1,10 @@ UNITTEST_FOR(ydb/services/persqueue_cluster_discovery) -OWNER( - radix +OWNER( + radix g:kikimr - g:logbroker -) + g:logbroker +) FORK_SUBTESTS() diff --git a/ydb/services/persqueue_cluster_discovery/ya.make b/ydb/services/persqueue_cluster_discovery/ya.make index 629439e884..23917ae376 100644 --- a/ydb/services/persqueue_cluster_discovery/ya.make +++ b/ydb/services/persqueue_cluster_discovery/ya.make @@ -3,7 +3,7 @@ LIBRARY() OWNER( radix g:kikimr - g:logbroker + g:logbroker ) SRCS( diff --git a/ydb/services/persqueue_v1/grpc_pq_actor.h b/ydb/services/persqueue_v1/grpc_pq_actor.h index fd2c569730..65c8aca1db 100644 --- a/ydb/services/persqueue_v1/grpc_pq_actor.h +++ b/ydb/services/persqueue_v1/grpc_pq_actor.h @@ -280,17 +280,17 @@ struct TEvPQProxy { }; struct TEvReadResponse : public NActors::TEventLocal<TEvReadResponse, EvReadResponse> { - explicit TEvReadResponse(PersQueue::V1::MigrationStreamingReadServerMessage&& resp, ui64 nextReadOffset, bool fromDisk, TDuration waitQuotaTime) + explicit TEvReadResponse(PersQueue::V1::MigrationStreamingReadServerMessage&& resp, ui64 nextReadOffset, bool fromDisk, TDuration waitQuotaTime) : Response(std::move(resp)) , NextReadOffset(nextReadOffset) , FromDisk(fromDisk) - , WaitQuotaTime(waitQuotaTime) + , WaitQuotaTime(waitQuotaTime) { } PersQueue::V1::MigrationStreamingReadServerMessage Response; ui64 NextReadOffset; bool FromDisk; - TDuration WaitQuotaTime; + TDuration WaitQuotaTime; }; struct TCommitCookie { @@ -1009,7 +1009,7 @@ private: const TString Guid; TInstant Start; bool FromDisk; - TDuration WaitQuotaTime; + TDuration WaitQuotaTime; }; THashMap<TActorId, TFormedReadResponse::TPtr> PartitionToReadResponse; // Partition actor -> TFormedReadResponse answer that has this partition. @@ -1163,7 +1163,7 @@ public: TAddReadRuleActor(NKikimr::NGRpcService::TEvPQAddReadRuleRequest *request); void Bootstrap(const NActors::TActorContext &ctx); - void ModifyPersqueueConfig(const TActorContext& ctx, + void ModifyPersqueueConfig(const TActorContext& ctx, NKikimrSchemeOp::TPersQueueGroupDescription& groupConfig, const NKikimrSchemeOp::TPersQueueGroupDescription& pqGroupDescription, const NKikimrSchemeOp::TDirEntry& selfInfo); @@ -1176,7 +1176,7 @@ public: TRemoveReadRuleActor(NKikimr::NGRpcService::TEvPQRemoveReadRuleRequest* request); void Bootstrap(const NActors::TActorContext &ctx); - void ModifyPersqueueConfig(const TActorContext& ctx, + void ModifyPersqueueConfig(const TActorContext& ctx, NKikimrSchemeOp::TPersQueueGroupDescription& groupConfig, const NKikimrSchemeOp::TPersQueueGroupDescription& pqGroupDescription, const NKikimrSchemeOp::TDirEntry& selfInfo); diff --git a/ydb/services/persqueue_v1/grpc_pq_read_actor.cpp b/ydb/services/persqueue_v1/grpc_pq_read_actor.cpp index d597ec8396..654edfcfcd 100644 --- a/ydb/services/persqueue_v1/grpc_pq_read_actor.cpp +++ b/ydb/services/persqueue_v1/grpc_pq_read_actor.cpp @@ -1451,7 +1451,7 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvReadResponse::TPtr& ev, const TAct if (event->FromDisk) { formedResponse->FromDisk = true; } - formedResponse->WaitQuotaTime = Max(formedResponse->WaitQuotaTime, event->WaitQuotaTime); + formedResponse->WaitQuotaTime = Max(formedResponse->WaitQuotaTime, event->WaitQuotaTime); --formedResponse->RequestsInfly; BytesInflight_ += diff; @@ -1472,7 +1472,7 @@ bool TReadSessionActor::WriteResponse(PersQueue::V1::MigrationStreamingReadServe } void TReadSessionActor::ProcessAnswer(const TActorContext& ctx, TFormedReadResponse::TPtr formedResponse) { - ui32 readDurationMs = (ctx.Now() - formedResponse->Start - formedResponse->WaitQuotaTime).MilliSeconds(); + ui32 readDurationMs = (ctx.Now() - formedResponse->Start - formedResponse->WaitQuotaTime).MilliSeconds(); if (formedResponse->FromDisk) { ReadLatencyFromDisk.IncFor(readDurationMs, 1); } else { @@ -1988,7 +1988,7 @@ bool FillBatchedData(MigrationStreamingReadServerMessage::DataBatch * data, cons message->set_partition_key(r.GetPartitionKey()); if (proto.HasCodec()) { - message->set_codec(NPQ::ToV1Codec((NPersQueueCommon::ECodec)proto.GetCodec())); + message->set_codec(NPQ::ToV1Codec((NPersQueueCommon::ECodec)proto.GetCodec())); } message->set_uncompressed_size(r.GetUncompressedSize()); message->set_data(proto.GetData()); @@ -2166,13 +2166,13 @@ void TPartitionActor::Handle(TEvPersQueue::TEvResponse::TPtr& ev, const TActorCo << " EndOffset " << EndOffset << " ReadOffset " << ReadOffset << " ReadGuid " << ReadGuid << " has messages " << hasData); ReadGuid = TString(); - auto readResponse = MakeHolder<TEvPQProxy::TEvReadResponse>( - std::move(response), - ReadOffset, - res.GetBlobsFromDisk() > 0, - TDuration::MilliSeconds(res.GetWaitQuotaTimeMs()) - ); - ctx.Send(ParentId, readResponse.Release()); + auto readResponse = MakeHolder<TEvPQProxy::TEvReadResponse>( + std::move(response), + ReadOffset, + res.GetBlobsFromDisk() > 0, + TDuration::MilliSeconds(res.GetWaitQuotaTimeMs()) + ); + ctx.Send(ParentId, readResponse.Release()); CheckRelease(ctx); PipeGeneration = 0; //reset tries counter - all ok @@ -2858,12 +2858,12 @@ void TReadInfoActor::Handle(TEvPQProxy::TEvAuthResultOk::TPtr& ev, const TActorC proto.MutableMetaRequest()->MutableCmdGetReadSessionsInfo()->AddTopic(t.TopicNameConverter->GetClientsideName()); } - ctx.Register(NMsgBusProxy::CreateActorServerPersQueue( - ctx.SelfID, - proto, - SchemeCache, - std::make_shared<NMsgBusProxy::TPersQueueGetReadSessionsInfoWorkerFactory>() - )); + ctx.Register(NMsgBusProxy::CreateActorServerPersQueue( + ctx.SelfID, + proto, + SchemeCache, + std::make_shared<NMsgBusProxy::TPersQueueGetReadSessionsInfoWorkerFactory>() + )); } diff --git a/ydb/services/persqueue_v1/grpc_pq_schema.cpp b/ydb/services/persqueue_v1/grpc_pq_schema.cpp index f4b340de25..7756adc797 100644 --- a/ydb/services/persqueue_v1/grpc_pq_schema.cpp +++ b/ydb/services/persqueue_v1/grpc_pq_schema.cpp @@ -18,7 +18,7 @@ using grpc::Status; namespace NKikimr::NGRpcProxy::V1 { constexpr TStringBuf GRPCS_ENDPOINT_PREFIX = "grpcs://"; - + /////////////////////////////////////////////////////////////////////////////// using namespace PersQueue::V1; @@ -122,13 +122,13 @@ void TDescribeTopicActor::StateWork(TAutoPtr<IEventHandle>& ev, const TActorCont void TDescribeTopicActor::HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) { - Y_VERIFY(ev->Get()->Request.Get()->ResultSet.size() == 1); // describe for only one topic + Y_VERIFY(ev->Get()->Request.Get()->ResultSet.size() == 1); // describe for only one topic if (ReplyIfNotTopic(ev, ctx)) { return; } - const auto& response = ev->Get()->Request.Get()->ResultSet.front(); - + const auto& response = ev->Get()->Request.Get()->ResultSet.front(); + const TString path = JoinSeq("/", response.Path); Ydb::PersQueue::V1::DescribeTopicResult result; @@ -184,7 +184,7 @@ void TDescribeTopicActor::HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEv settings->add_supported_codecs((Ydb::PersQueue::V1::Codec) (codec + 1)); } - const auto& pqConfig = AppData(ctx)->PQConfig; + const auto& pqConfig = AppData(ctx)->PQConfig; for (ui32 i = 0; i < config.ReadRulesSize(); ++i) { auto rr = settings->add_read_rules(); auto consumerName = NPersQueue::ConvertOldConsumerName(config.GetReadRules(i), ctx); @@ -204,20 +204,20 @@ void TDescribeTopicActor::HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEv } } rr->set_important(important); - - if (i < config.ReadRuleServiceTypesSize()) { - rr->set_service_type(config.GetReadRuleServiceTypes(i)); - } else { - if (pqConfig.GetDisallowDefaultClientServiceType()) { - this->Request_->RaiseIssue(FillIssue( - "service type must be set for all read rules", - Ydb::PersQueue::ErrorCode::ERROR - )); - Reply(Ydb::StatusIds::INTERNAL_ERROR, ctx); - return; - } - rr->set_service_type(pqConfig.GetDefaultClientServiceType().GetName()); - } + + if (i < config.ReadRuleServiceTypesSize()) { + rr->set_service_type(config.GetReadRuleServiceTypes(i)); + } else { + if (pqConfig.GetDisallowDefaultClientServiceType()) { + this->Request_->RaiseIssue(FillIssue( + "service type must be set for all read rules", + Ydb::PersQueue::ErrorCode::ERROR + )); + Reply(Ydb::StatusIds::INTERNAL_ERROR, ctx); + return; + } + rr->set_service_type(pqConfig.GetDefaultClientServiceType().GetName()); + } } if (partConfig.HasMirrorFrom()) { auto rmr = settings->mutable_remote_mirror_rule(); @@ -250,7 +250,7 @@ void TDescribeTopicActor::HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEv NPersQueue::ObfuscateString( partConfig.GetMirrorFrom().GetCredentials().GetIam().GetServiceAccountKey()) ); - } + } } rmr->set_database(partConfig.GetMirrorFrom().GetDatabase()); } @@ -279,19 +279,19 @@ void TAddReadRuleActor::Bootstrap(const NActors::TActorContext& ctx) { Become(&TBase::StateWork); } -void TAddReadRuleActor::ModifyPersqueueConfig( - const TActorContext& ctx, +void TAddReadRuleActor::ModifyPersqueueConfig( + const TActorContext& ctx, NKikimrSchemeOp::TPersQueueGroupDescription& groupConfig, const NKikimrSchemeOp::TPersQueueGroupDescription& pqGroupDescription, const NKikimrSchemeOp::TDirEntry& selfInfo -) { - Y_UNUSED(pqGroupDescription); - +) { + Y_UNUSED(pqGroupDescription); + auto* pqConfig = groupConfig.MutablePQTabletConfig(); auto rule = GetProtoRequest()->read_rule(); if (rule.version() == 0) { - rule.set_version(selfInfo.GetVersion().GetPQVersion()); + rule.set_version(selfInfo.GetVersion().GetPQVersion()); } auto serviceTypes = GetSupportedClientServiceTypes(ctx); TString error = AddReadRuleToConfig(pqConfig, rule, serviceTypes, ctx); @@ -318,20 +318,20 @@ void TRemoveReadRuleActor::Bootstrap(const NActors::TActorContext& ctx) { Become(&TBase::StateWork); } -void TRemoveReadRuleActor::ModifyPersqueueConfig( - const TActorContext& ctx, +void TRemoveReadRuleActor::ModifyPersqueueConfig( + const TActorContext& ctx, NKikimrSchemeOp::TPersQueueGroupDescription& groupConfig, const NKikimrSchemeOp::TPersQueueGroupDescription& pqGroupDescription, const NKikimrSchemeOp::TDirEntry& selfInfo -) { - Y_UNUSED(selfInfo); - - auto error = RemoveReadRuleFromConfig( - groupConfig.MutablePQTabletConfig(), - pqGroupDescription.GetPQTabletConfig(), - GetProtoRequest()->consumer_name(), - ctx - ); +) { + Y_UNUSED(selfInfo); + + auto error = RemoveReadRuleFromConfig( + groupConfig.MutablePQTabletConfig(), + pqGroupDescription.GetPQTabletConfig(), + GetProtoRequest()->consumer_name(), + ctx + ); if (!error.Empty()) { return ReplyWithError(Ydb::StatusIds::NOT_FOUND, Ydb::PersQueue::ErrorCode::BAD_REQUEST, error, ctx); } @@ -372,14 +372,14 @@ void TCreateTopicActor::FillProposeRequest(TEvTxUserProxy::TEvProposeTransaction NKikimrSchemeOp::TModifyScheme& modifyScheme(*proposal.Record.MutableTransaction()->MutableModifyScheme()); modifyScheme.SetWorkingDir(workingDir); - { - TString error; - auto status = FillProposeRequestImpl(name, GetProtoRequest()->settings(), modifyScheme, ctx, false, error); - if (!error.empty()) { - Request_->RaiseIssue(FillIssue(error, PersQueue::ErrorCode::BAD_REQUEST)); - return ReplyWithResult(status, ctx); - } - } + { + TString error; + auto status = FillProposeRequestImpl(name, GetProtoRequest()->settings(), modifyScheme, ctx, false, error); + if (!error.empty()) { + Request_->RaiseIssue(FillIssue(error, PersQueue::ErrorCode::BAD_REQUEST)); + return ReplyWithResult(status, ctx); + } + } const auto& pqDescr = modifyScheme.GetCreatePersQueueGroup(); const auto& config = pqDescr.GetPQTabletConfig(); @@ -389,8 +389,8 @@ void TCreateTopicActor::FillProposeRequest(TEvTxUserProxy::TEvProposeTransaction << "' instead of " << LocalCluster, PersQueue::ErrorCode::BAD_REQUEST)); return ReplyWithResult(Ydb::StatusIds::BAD_REQUEST, ctx); } - if (Count(Clusters, config.GetDC()) == 0 && !Clusters.empty()) { - Request_->RaiseIssue(FillIssue(TStringBuilder() << "Unknown cluster '" << config.GetDC() << "'", PersQueue::ErrorCode::BAD_REQUEST)); + if (Count(Clusters, config.GetDC()) == 0 && !Clusters.empty()) { + Request_->RaiseIssue(FillIssue(TStringBuilder() << "Unknown cluster '" << config.GetDC() << "'", PersQueue::ErrorCode::BAD_REQUEST)); return ReplyWithResult(Ydb::StatusIds::BAD_REQUEST, ctx); } } diff --git a/ydb/services/persqueue_v1/grpc_pq_write_actor.cpp b/ydb/services/persqueue_v1/grpc_pq_write_actor.cpp index 3a91a67af5..3038cc82a6 100644 --- a/ydb/services/persqueue_v1/grpc_pq_write_actor.cpp +++ b/ydb/services/persqueue_v1/grpc_pq_write_actor.cpp @@ -1,6 +1,6 @@ #include "grpc_pq_actor.h" #include "grpc_pq_write.h" -#include "grpc_pq_codecs.h" +#include "grpc_pq_codecs.h" #include <ydb/core/persqueue/pq_database.h> #include <ydb/core/persqueue/write_meta.h> @@ -19,8 +19,8 @@ using namespace NActors; using namespace NKikimrClient; - - + + namespace NKikimr { using namespace NSchemeCache; @@ -35,44 +35,44 @@ Ydb::PersQueue::V1::Codec CodecByName(const TString& codec) { return codecIt != codecsByName.end() ? codecIt->second : Ydb::PersQueue::V1::CODEC_UNSPECIFIED; } -template <> -void FillExtraFieldsForDataChunk( - const Ydb::PersQueue::V1::StreamingWriteClientMessage::InitRequest& init, - NKikimrPQClient::TDataChunk& data, - TString& server, - TString& ident, - TString& logType, - TString& file -) { - for (const auto& item : init.session_meta()) { - if (item.first == "server") { - server = item.second; - } else if (item.first == "ident") { - ident = item.second; - } else if (item.first == "logtype") { - logType = item.second; - } else if (item.first == "file") { - file = item.second; - } else { - auto res = data.MutableExtraFields()->AddItems(); - res->SetKey(item.first); - res->SetValue(item.second); - } - } -} - -template <> -void FillChunkDataFromReq( - NKikimrPQClient::TDataChunk& proto, - const Ydb::PersQueue::V1::StreamingWriteClientMessage::WriteRequest& writeRequest, - const i32 messageIndex -) { - proto.SetSeqNo(writeRequest.sequence_numbers(messageIndex)); - proto.SetCreateTime(writeRequest.created_at_ms(messageIndex)); - proto.SetCodec(writeRequest.blocks_headers(messageIndex).front()); - proto.SetData(writeRequest.blocks_data(messageIndex)); -} - +template <> +void FillExtraFieldsForDataChunk( + const Ydb::PersQueue::V1::StreamingWriteClientMessage::InitRequest& init, + NKikimrPQClient::TDataChunk& data, + TString& server, + TString& ident, + TString& logType, + TString& file +) { + for (const auto& item : init.session_meta()) { + if (item.first == "server") { + server = item.second; + } else if (item.first == "ident") { + ident = item.second; + } else if (item.first == "logtype") { + logType = item.second; + } else if (item.first == "file") { + file = item.second; + } else { + auto res = data.MutableExtraFields()->AddItems(); + res->SetKey(item.first); + res->SetValue(item.second); + } + } +} + +template <> +void FillChunkDataFromReq( + NKikimrPQClient::TDataChunk& proto, + const Ydb::PersQueue::V1::StreamingWriteClientMessage::WriteRequest& writeRequest, + const i32 messageIndex +) { + proto.SetSeqNo(writeRequest.sequence_numbers(messageIndex)); + proto.SetCreateTime(writeRequest.created_at_ms(messageIndex)); + proto.SetCodec(writeRequest.blocks_headers(messageIndex).front()); + proto.SetData(writeRequest.blocks_data(messageIndex)); +} + namespace NGRpcProxy { namespace V1 { @@ -316,14 +316,14 @@ void TWriteSessionActor::Handle(TEvPQProxy::TEvWriteInit::TPtr& ev, const TActor PeerName = event->PeerName; SourceId = init.message_group_id(); - TString encodedSourceId; - try { + TString encodedSourceId; + try { encodedSourceId = NPQ::NSourceIdEncoding::Encode(SourceId); - } catch (yexception& e) { - CloseSession(TStringBuilder() << "incorrect sourceId \"" << SourceId << "\": " << e.what(), PersQueue::ErrorCode::BAD_REQUEST, ctx); - return; + } catch (yexception& e) { + CloseSession(TStringBuilder() << "incorrect sourceId \"" << SourceId << "\": " << e.what(), PersQueue::ErrorCode::BAD_REQUEST, ctx); + return; } - EscapedSourceId = HexEncode(encodedSourceId); + EscapedSourceId = HexEncode(encodedSourceId); TString s = TopicConverter->GetClientsideName() + encodedSourceId; Hash = MurmurHash<ui32>(s.c_str(), s.size(), MURMUR_ARRAY_SEED); @@ -950,7 +950,7 @@ void TWriteSessionActor::GenerateNextWriteRequest(const TActorContext& ctx) { i64 diff = 0; auto addData = [&](const StreamingWriteClientMessage::WriteRequest& writeRequest, const i32 messageIndex) { auto w = request.MutablePartitionRequest()->AddCmdWrite(); - w->SetData(GetSerializedData(InitMeta, writeRequest, messageIndex)); + w->SetData(GetSerializedData(InitMeta, writeRequest, messageIndex)); w->SetSeqNo(writeRequest.sequence_numbers(messageIndex)); w->SetSourceId(NPQ::NSourceIdEncoding::EncodeSimple(SourceId)); w->SetCreateTimeMS(writeRequest.created_at_ms(messageIndex)); diff --git a/ydb/services/persqueue_v1/persqueue_common_ut.cpp b/ydb/services/persqueue_v1/persqueue_common_ut.cpp index d80eb1672d..9d706d0656 100644 --- a/ydb/services/persqueue_v1/persqueue_common_ut.cpp +++ b/ydb/services/persqueue_v1/persqueue_common_ut.cpp @@ -26,7 +26,7 @@ #include <ydb/public/api/protos/persqueue_error_codes_v1.pb.h> #include <ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/data_plane_helpers.h> - + namespace NKikimr::NPersQueueTests { using namespace Tests; diff --git a/ydb/services/persqueue_v1/persqueue_ut.cpp b/ydb/services/persqueue_v1/persqueue_ut.cpp index 6176287f88..3bc9b21728 100644 --- a/ydb/services/persqueue_v1/persqueue_ut.cpp +++ b/ydb/services/persqueue_v1/persqueue_ut.cpp @@ -14,12 +14,12 @@ #include <ydb/library/aclib/aclib.h> #include <ydb/library/persqueue/obfuscate/obfuscate.h> -#include <ydb/library/persqueue/tests/counters.h> +#include <ydb/library/persqueue/tests/counters.h> #include <ydb/library/persqueue/topic_parser/topic_parser.h> #include <library/cpp/testing/unittest/tests_data.h> #include <library/cpp/testing/unittest/registar.h> -#include <library/cpp/json/json_reader.h> +#include <library/cpp/json/json_reader.h> #include <library/cpp/monlib/dynamic_counters/encode.h> #include <google/protobuf/text_format.h> #include <google/protobuf/util/message_differencer.h> @@ -433,9 +433,9 @@ namespace { server.AnnoyingClient->CreateTopic("rt3.dc1--topic", 1); auto driver = server.AnnoyingClient->GetDriver(); - + auto writer = CreateSimpleWriter(*driver, "topic", "test source ID"); - + bool res = true; ui32 messageCount = 1000; @@ -668,7 +668,7 @@ namespace { server.EnableLogs({ NKikimrServices::PERSQUEUE }); - // empty data and sourceId + // empty data and sourceId server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 1, "", 1, "", "", ETransport::MsgBus, NMsgBusProxy::MSTATUS_ERROR); server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 1, "a", 1, "", "", ETransport::MsgBus, NMsgBusProxy::MSTATUS_ERROR); server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 1, "", 1, "a", "", ETransport::MsgBus, NMsgBusProxy::MSTATUS_ERROR); @@ -1623,7 +1623,7 @@ namespace { auto pqLib = TPQLib::WithCerrLogger(); // Write nonempty data - NKikimr::NPersQueueTests::TRequestWritePQ writeReq(DEFAULT_TOPIC_NAME, 0, "src", 4); + NKikimr::NPersQueueTests::TRequestWritePQ writeReq(DEFAULT_TOPIC_NAME, 0, "src", 4); auto write = [&](const TString& data, bool empty = false) { NKikimrPQClient::TDataChunk dataChunk; @@ -1643,10 +1643,10 @@ namespace { write("data3"); } - ui32 maxCount = 1; - bool unpack = false; - ui32 maxInflyRequests = 1; - ui32 maxMemoryUsage = 1; + ui32 maxCount = 1; + bool unpack = false; + ui32 maxInflyRequests = 1; + ui32 maxMemoryUsage = 1; auto [consumer, ccResult] = CreateConsumer( pqLib, server.GrpcPort, "user", {SHORT_TOPIC_NAME, {}}, maxCount, unpack, {}, maxInflyRequests, maxMemoryUsage @@ -1707,10 +1707,10 @@ namespace { auto f = producer->Write(1, TString(1024, 'a')); f.Wait(); - ui32 maxCount = 1; - bool unpack = false; + ui32 maxCount = 1; + bool unpack = false; auto [consumer, ccResult] = CreateConsumer(pqLib, server.GrpcPort, "user", {SHORT_TOPIC_NAME, {}}, maxCount, unpack); - Cerr << ccResult.Response << "\n"; + Cerr << ccResult.Response << "\n"; auto msg = consumer->GetNextMessage(); msg.Wait(); @@ -1743,7 +1743,7 @@ namespace { { auto [producer, pcResult] = CreateProducer(pqLib, server.GrpcPort, "aaa/bbb/ccc/topic", "123"); - UNIT_ASSERT_C(pcResult.Response.server_message_case() == StreamingWriteServerMessage::kInitResponse, pcResult.Response); + UNIT_ASSERT_C(pcResult.Response.server_message_case() == StreamingWriteServerMessage::kInitResponse, pcResult.Response); for (ui32 i = 1; i <= 11; ++i) { auto f = producer->Write(i, TString(10, 'a')); f.Wait(); @@ -1751,8 +1751,8 @@ namespace { } } - ui32 maxCount = 1; - bool unpack = false; + ui32 maxCount = 1; + bool unpack = false; auto [consumer, ccResult] = CreateConsumer(pqLib, server.GrpcPort, "user", {"aaa/bbb/ccc/topic", {}}, maxCount, unpack); UNIT_ASSERT_C(ccResult.Response.response_case() == MigrationStreamingReadServerMessage::kInitResponse, ccResult.Response); @@ -1773,11 +1773,11 @@ namespace { TPQDataWriter writer("source", server); for (ui32 i = 1; i <= 3; ++i) { - TString sourceId = "123" + ToString<int>(i); - ui32 partitionGroup = i; + TString sourceId = "123" + ToString<int>(i); + ui32 partitionGroup = i; auto [producer, pcResult] = CreateProducer(pqLib, server.GrpcPort, SHORT_TOPIC_NAME, sourceId, partitionGroup); - UNIT_ASSERT(pcResult.Response.server_message_case() == StreamingWriteServerMessage::kInitResponse); + UNIT_ASSERT(pcResult.Response.server_message_case() == StreamingWriteServerMessage::kInitResponse); auto f = producer->Write(i, TString(10, 'a')); f.Wait(); } @@ -1791,7 +1791,7 @@ namespace { ss.Unpack = false; auto [consumer, ccResult] = CreateConsumer(pqLib, ss); - Cerr << ccResult.Response << "\n"; + Cerr << ccResult.Response << "\n"; for (ui32 i = 1; i <= 3; ++i) { auto msg = consumer->GetNextMessage(); @@ -1803,7 +1803,7 @@ namespace { } auto [consumer2, ccResult2] = CreateConsumer(pqLib, ss); - Cerr << ccResult2.Response << "\n"; + Cerr << ccResult2.Response << "\n"; auto msg = consumer->GetNextMessage(); auto msg2 = consumer2->GetNextMessage(); @@ -1838,12 +1838,12 @@ namespace { TVector<std::pair<ui64, ui64>> cookies; for (ui32 i = 1; i <= 3; ++i) { - TString sourceId = "123" + ToString<int>(i); - ui32 partitionGroup = i; + TString sourceId = "123" + ToString<int>(i); + ui32 partitionGroup = i; auto [producer, pcResult] = CreateProducer(pqLib, server.GrpcPort, SHORT_TOPIC_NAME, sourceId, partitionGroup); Cerr << "===Response: " << pcResult.Response << Endl; - UNIT_ASSERT(pcResult.Response.server_message_case() == StreamingWriteServerMessage::kInitResponse); + UNIT_ASSERT(pcResult.Response.server_message_case() == StreamingWriteServerMessage::kInitResponse); auto f = producer->Write(i, TString(10, 'a')); f.Wait(); } @@ -1857,7 +1857,7 @@ namespace { ss.Unpack = false; auto [consumer, ccResult] = CreateConsumer(pqLib, ss); - Cerr << ccResult.Response << "\n"; + Cerr << ccResult.Response << "\n"; for (ui32 i = 1; i <= 3; ++i) { auto msg = GetNextMessageSkipAssignment(consumer); @@ -1869,7 +1869,7 @@ namespace { } auto [consumer2, ccResult2] = CreateConsumer(pqLib, ss); - Cerr << ccResult2.Response << "\n"; + Cerr << ccResult2.Response << "\n"; auto msg = consumer->GetNextMessage(); auto msg2 = consumer2->GetNextMessage(); @@ -1910,7 +1910,7 @@ namespace { ss.Unpack = false; auto [consumer, ccResult] = CreateConsumer(pqLib, ss); - Cerr << ccResult.Response << "\n"; + Cerr << ccResult.Response << "\n"; auto msg = consumer->GetNextMessage(); msg.Wait(); @@ -1925,7 +1925,7 @@ namespace { THolder<IConsumer> consumer2; do { std::tie(consumer2, ccResult) = CreateConsumer(pqLib, ss); - Cerr << ccResult.Response << "\n"; + Cerr << ccResult.Response << "\n"; } while(!msg.Wait(TDuration::Seconds(1))); Cerr << msg.GetValue().Response << "\n"; @@ -2015,96 +2015,96 @@ namespace { } while (count < 100); } - Y_UNIT_TEST(TestReadQuotasSimple) { - TRateLimiterTestSetup setup(NKikimrPQ::TPQConfig::TQuotingConfig::USER_PAYLOAD_SIZE, 1000, 1000, true); + Y_UNIT_TEST(TestReadQuotasSimple) { + TRateLimiterTestSetup setup(NKikimrPQ::TPQConfig::TQuotingConfig::USER_PAYLOAD_SIZE, 1000, 1000, true); + + const TString topicPath = "acc/topic1"; + const TString consumerPath = "acc2/reader1"; + setup.CreateTopic(topicPath); + setup.CreateConsumer(consumerPath); + + THolder<IProducer> producer = setup.StartProducer(topicPath, true); - const TString topicPath = "acc/topic1"; - const TString consumerPath = "acc2/reader1"; - setup.CreateTopic(topicPath); - setup.CreateConsumer(consumerPath); - - THolder<IProducer> producer = setup.StartProducer(topicPath, true); - auto pqLib = TPQLib::WithCerrLogger(); auto [consumer, ccResult] = CreateConsumer( pqLib, setup.GetGrpcPort(), consumerPath, {topicPath , {}}, 1000, false ); - Cerr << ccResult.Response << "\n"; - - { - auto msg = consumer->GetNextMessage(); - msg.Wait(); + Cerr << ccResult.Response << "\n"; + + { + auto msg = consumer->GetNextMessage(); + msg.Wait(); Cerr << "consumer assign response: " << msg.GetValue().Response << "\n"; - UNIT_ASSERT(msg.GetValue().Type == EMT_ASSIGNED); - msg.GetValue().StartRead.SetValue(TAssignInfo()); - } - - TVector<NThreading::TFuture<Ydb::PersQueue::TProducerCommitResponse>> writeResults; - TVector<NThreading::TFuture<Ydb::PersQueue::TConsumerMessage>> readResults; - - for (ui32 readBatches = 0; readBatches < 10; ++readBatches) { - auto msg = consumer->GetNextMessage(); - while (!msg.HasValue()) { - producer->Write(TString(std::string(10000, 'A'))); - Sleep(TDuration::MilliSeconds(10)); - } - const auto& response = msg.GetValue().Response; - Cerr << "next read response: " << response << "\n"; - - for (auto& data : response.data_batch().partition_data()) { - for (auto& batch : data.batches()) { - UNIT_ASSERT(batch.message_data_size() > 0); - } - } - } - } - - Y_UNIT_TEST(TestReadWithQuoterWithoutResources) { + UNIT_ASSERT(msg.GetValue().Type == EMT_ASSIGNED); + msg.GetValue().StartRead.SetValue(TAssignInfo()); + } + + TVector<NThreading::TFuture<Ydb::PersQueue::TProducerCommitResponse>> writeResults; + TVector<NThreading::TFuture<Ydb::PersQueue::TConsumerMessage>> readResults; + + for (ui32 readBatches = 0; readBatches < 10; ++readBatches) { + auto msg = consumer->GetNextMessage(); + while (!msg.HasValue()) { + producer->Write(TString(std::string(10000, 'A'))); + Sleep(TDuration::MilliSeconds(10)); + } + const auto& response = msg.GetValue().Response; + Cerr << "next read response: " << response << "\n"; + + for (auto& data : response.data_batch().partition_data()) { + for (auto& batch : data.batches()) { + UNIT_ASSERT(batch.message_data_size() > 0); + } + } + } + } + + Y_UNIT_TEST(TestReadWithQuoterWithoutResources) { if (NSan::ASanIsOn()) { return; } - TRateLimiterTestSetup setup(NKikimrPQ::TPQConfig::TQuotingConfig::USER_PAYLOAD_SIZE, 1000, 1000, true); - - const TString topicPath = "acc/topic1"; - const TString consumerPath = "acc2/reader1"; // don't create kesus resources - setup.CreateTopic(topicPath); - - THolder<IProducer> producer = setup.StartProducer(topicPath, true); - - TPQLibSettings pqLibSettings({ .DefaultLogger = new TCerrLogger(DEBUG_LOG_LEVEL) }); - TPQLib PQLib(pqLibSettings); - auto [consumer, ccResult] = CreateConsumer(PQLib, setup.GetGrpcPort(), consumerPath, {topicPath , {}}, 1000, false); - Cerr << ccResult.Response << "\n"; - - { - auto msg = consumer->GetNextMessage(); - msg.Wait(); - Cerr << "consumer assign response: " << msg.GetValue().Response << "\n"; - UNIT_ASSERT(msg.GetValue().Type == EMT_ASSIGNED); - msg.GetValue().StartRead.SetValue(TAssignInfo()); - } - - TVector<NThreading::TFuture<Ydb::PersQueue::TProducerCommitResponse>> writeResults; - TVector<NThreading::TFuture<Ydb::PersQueue::TConsumerMessage>> readResults; - - for (ui32 readBatches = 0; readBatches < 10; ++readBatches) { - auto msg = consumer->GetNextMessage(); - while (!msg.HasValue()) { - producer->Write(TString(std::string(10000, 'A'))); - Sleep(TDuration::MilliSeconds(10)); - } - const auto& response = msg.GetValue().Response; - Cerr << "next read response: " << response << "\n"; - - for (auto& data : response.data_batch().partition_data()) { - for (auto& batch : data.batches()) { - UNIT_ASSERT(batch.message_data_size() > 0); - } - } - } - } - + TRateLimiterTestSetup setup(NKikimrPQ::TPQConfig::TQuotingConfig::USER_PAYLOAD_SIZE, 1000, 1000, true); + + const TString topicPath = "acc/topic1"; + const TString consumerPath = "acc2/reader1"; // don't create kesus resources + setup.CreateTopic(topicPath); + + THolder<IProducer> producer = setup.StartProducer(topicPath, true); + + TPQLibSettings pqLibSettings({ .DefaultLogger = new TCerrLogger(DEBUG_LOG_LEVEL) }); + TPQLib PQLib(pqLibSettings); + auto [consumer, ccResult] = CreateConsumer(PQLib, setup.GetGrpcPort(), consumerPath, {topicPath , {}}, 1000, false); + Cerr << ccResult.Response << "\n"; + + { + auto msg = consumer->GetNextMessage(); + msg.Wait(); + Cerr << "consumer assign response: " << msg.GetValue().Response << "\n"; + UNIT_ASSERT(msg.GetValue().Type == EMT_ASSIGNED); + msg.GetValue().StartRead.SetValue(TAssignInfo()); + } + + TVector<NThreading::TFuture<Ydb::PersQueue::TProducerCommitResponse>> writeResults; + TVector<NThreading::TFuture<Ydb::PersQueue::TConsumerMessage>> readResults; + + for (ui32 readBatches = 0; readBatches < 10; ++readBatches) { + auto msg = consumer->GetNextMessage(); + while (!msg.HasValue()) { + producer->Write(TString(std::string(10000, 'A'))); + Sleep(TDuration::MilliSeconds(10)); + } + const auto& response = msg.GetValue().Response; + Cerr << "next read response: " << response << "\n"; + + for (auto& data : response.data_batch().partition_data()) { + for (auto& batch : data.batches()) { + UNIT_ASSERT(batch.message_data_size() > 0); + } + } + } + } + Y_UNIT_TEST(TestDeletionOfTopic) { if (NSan::ASanIsOn()) { return; @@ -2118,8 +2118,8 @@ namespace { server.AnnoyingClient->DeleteTopic2(DEFAULT_TOPIC_NAME, NPersQueue::NErrorCode::OK, false); auto pqLib = TPQLib::WithCerrLogger(); - ui32 maxCount = 1; - bool unpack = false; + ui32 maxCount = 1; + bool unpack = false; auto [consumer, ccResult] = CreateConsumer(pqLib, server.GrpcPort, "user", {SHORT_TOPIC_NAME, {}}, maxCount, unpack); Cerr << "Consumer create response: " << ccResult.Response << "\n"; @@ -2375,10 +2375,10 @@ namespace { auto props = request.mutable_settings(); props->set_partitions_count(1); props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE); - props->set_retention_period_ms(TDuration::Days(1).MilliSeconds()); - props->set_max_partition_storage_size(1000); - props->set_max_partition_write_speed(1000); - props->set_max_partition_write_burst(1000); + props->set_retention_period_ms(TDuration::Days(1).MilliSeconds()); + props->set_max_partition_storage_size(1000); + props->set_max_partition_write_speed(1000); + props->set_max_partition_write_burst(1000); grpc::ClientContext rcontext; rcontext.AddMetadata("x-ydb-auth-ticket", "user@" BUILTIN_ACL_DOMAIN); @@ -2405,10 +2405,10 @@ namespace { auto props = request.mutable_settings(); props->set_partitions_count(2); props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE); - props->set_retention_period_ms(TDuration::Days(1).MilliSeconds()); - props->set_max_partition_storage_size(1000); - props->set_max_partition_write_speed(1000); - props->set_max_partition_write_burst(1000); + props->set_retention_period_ms(TDuration::Days(1).MilliSeconds()); + props->set_max_partition_storage_size(1000); + props->set_max_partition_write_speed(1000); + props->set_max_partition_write_burst(1000); grpc::ClientContext rcontext; @@ -2444,10 +2444,10 @@ namespace { auto props = request.mutable_settings(); props->set_partitions_count(1); props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE); - props->set_retention_period_ms(TDuration::Days(1).MilliSeconds()); - props->set_max_partition_storage_size(1000); - props->set_max_partition_write_speed(1000); - props->set_max_partition_write_burst(1000); + props->set_retention_period_ms(TDuration::Days(1).MilliSeconds()); + props->set_max_partition_storage_size(1000); + props->set_max_partition_write_speed(1000); + props->set_max_partition_write_burst(1000); alter(request, Ydb::StatusIds::UNAUTHORIZED, true); alter(request, Ydb::StatusIds::GENERIC_ERROR, false); props->set_partitions_count(3); @@ -2492,7 +2492,7 @@ namespace { PartitionConfig { MaxCountInPartition: 2147483647 MaxSizeInPartition: 234 - LifetimeSeconds: 86400 + LifetimeSeconds: 86400 ImportantClientId: "consumer" SourceIdLifetimeSeconds: 1382400 WriteSpeedInBytesPerSecond: 123 @@ -2553,7 +2553,7 @@ namespace { Codecs: "lzop" Codecs: "gzip" } - ReadRuleServiceTypes: "data-transfer" + ReadRuleServiceTypes: "data-transfer" FormatVersion: 0 Codecs { Ids: 0 @@ -2582,7 +2582,7 @@ namespace { DescribeTopicResult res; response.operation().result().UnpackTo(&res); Cerr << response << "\n" << res << "\n"; - UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::SCHEME_ERROR); // muts be Ydb::StatusIds::UNAUTHORIZED); + UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::SCHEME_ERROR); // muts be Ydb::StatusIds::UNAUTHORIZED); } { @@ -2684,226 +2684,226 @@ namespace { } - - Y_UNIT_TEST(SchemeOperationsCheckPropValues) { + + Y_UNIT_TEST(SchemeOperationsCheckPropValues) { NPersQueue::TTestServer server; server.EnableLogs({ NKikimrServices::PQ_READ_PROXY, NKikimrServices::BLACKBOX_VALIDATOR }); - + server.AnnoyingClient->CreateTopic("rt3.dc1--acc--topic1", 1); server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1); server.AnnoyingClient->CreateConsumer("user"); - - std::shared_ptr<grpc::Channel> Channel_; - std::unique_ptr<Ydb::PersQueue::V1::PersQueueService::Stub> StubP_; - - { + + std::shared_ptr<grpc::Channel> Channel_; + std::unique_ptr<Ydb::PersQueue::V1::PersQueueService::Stub> StubP_; + + { Channel_ = grpc::CreateChannel("localhost:" + ToString(server.GrpcPort), grpc::InsecureChannelCredentials()); - StubP_ = Ydb::PersQueue::V1::PersQueueService::NewStub(Channel_); - } - - { - // zero value is forbidden for: partitions_count - CreateTopicRequest request; - CreateTopicResponse response; - request.set_path("/Root/PQ/rt3.dc1--acc--topic1"); - auto props = request.mutable_settings(); - props->set_partitions_count(0); - props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE); - props->set_retention_period_ms(TDuration::Days(1).MilliSeconds()); - - grpc::ClientContext rcontext; - auto status = StubP_->CreateTopic(&rcontext, request, &response); - - UNIT_ASSERT(status.ok()); - CreateTopicResult res; - response.operation().result().UnpackTo(&res); - Cerr << response << "\n" << res << "\n"; - UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::BAD_REQUEST); - } - { - // zero value is forbidden for: retention_period_ms - CreateTopicRequest request; - CreateTopicResponse response; - request.set_path("/Root/PQ/rt3.dc1--acc--topic1"); - auto props = request.mutable_settings(); - props->set_partitions_count(1); - props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE); - props->set_retention_period_ms(0); - - grpc::ClientContext rcontext; - auto status = StubP_->CreateTopic(&rcontext, request, &response); - - UNIT_ASSERT(status.ok()); - CreateTopicResult res; - response.operation().result().UnpackTo(&res); - Cerr << response << "\n" << res << "\n"; - UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::BAD_REQUEST); - } - { - // zero value is allowed for: partition_storage_size, max_partition_write_speed, max_partition_write_burst - CreateTopicRequest request; - CreateTopicResponse response; - request.set_path("/Root/PQ/rt3.dc1--acc--topic1"); - auto props = request.mutable_settings(); - props->set_partitions_count(1); - props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE); - props->set_retention_period_ms(TDuration::Days(1).MilliSeconds()); - props->set_max_partition_storage_size(0); - props->set_max_partition_write_speed(0); - props->set_max_partition_write_burst(0); - - grpc::ClientContext rcontext; - auto status = StubP_->CreateTopic(&rcontext, request, &response); - - UNIT_ASSERT(status.ok()); - CreateTopicResult res; - response.operation().result().UnpackTo(&res); - Cerr << response << "\n" << res << "\n"; - UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::SUCCESS); - } - } - - Y_UNIT_TEST(ReadRuleServiceType) { + StubP_ = Ydb::PersQueue::V1::PersQueueService::NewStub(Channel_); + } + + { + // zero value is forbidden for: partitions_count + CreateTopicRequest request; + CreateTopicResponse response; + request.set_path("/Root/PQ/rt3.dc1--acc--topic1"); + auto props = request.mutable_settings(); + props->set_partitions_count(0); + props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE); + props->set_retention_period_ms(TDuration::Days(1).MilliSeconds()); + + grpc::ClientContext rcontext; + auto status = StubP_->CreateTopic(&rcontext, request, &response); + + UNIT_ASSERT(status.ok()); + CreateTopicResult res; + response.operation().result().UnpackTo(&res); + Cerr << response << "\n" << res << "\n"; + UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::BAD_REQUEST); + } + { + // zero value is forbidden for: retention_period_ms + CreateTopicRequest request; + CreateTopicResponse response; + request.set_path("/Root/PQ/rt3.dc1--acc--topic1"); + auto props = request.mutable_settings(); + props->set_partitions_count(1); + props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE); + props->set_retention_period_ms(0); + + grpc::ClientContext rcontext; + auto status = StubP_->CreateTopic(&rcontext, request, &response); + + UNIT_ASSERT(status.ok()); + CreateTopicResult res; + response.operation().result().UnpackTo(&res); + Cerr << response << "\n" << res << "\n"; + UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::BAD_REQUEST); + } + { + // zero value is allowed for: partition_storage_size, max_partition_write_speed, max_partition_write_burst + CreateTopicRequest request; + CreateTopicResponse response; + request.set_path("/Root/PQ/rt3.dc1--acc--topic1"); + auto props = request.mutable_settings(); + props->set_partitions_count(1); + props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE); + props->set_retention_period_ms(TDuration::Days(1).MilliSeconds()); + props->set_max_partition_storage_size(0); + props->set_max_partition_write_speed(0); + props->set_max_partition_write_burst(0); + + grpc::ClientContext rcontext; + auto status = StubP_->CreateTopic(&rcontext, request, &response); + + UNIT_ASSERT(status.ok()); + CreateTopicResult res; + response.operation().result().UnpackTo(&res); + Cerr << response << "\n" << res << "\n"; + UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::SUCCESS); + } + } + + Y_UNIT_TEST(ReadRuleServiceType) { TServerSettings settings = PQSettings(0); - { - settings.PQConfig.AddClientServiceType()->SetName("MyGreatType"); - settings.PQConfig.AddClientServiceType()->SetName("AnotherType"); - settings.PQConfig.AddClientServiceType()->SetName("SecondType"); - } + { + settings.PQConfig.AddClientServiceType()->SetName("MyGreatType"); + settings.PQConfig.AddClientServiceType()->SetName("AnotherType"); + settings.PQConfig.AddClientServiceType()->SetName("SecondType"); + } NPersQueue::TTestServer server(settings); server.EnableLogs({ NKikimrServices::PQ_READ_PROXY, NKikimrServices::BLACKBOX_VALIDATOR }); - - std::unique_ptr<Ydb::PersQueue::V1::PersQueueService::Stub> pqStub; - - { + + std::unique_ptr<Ydb::PersQueue::V1::PersQueueService::Stub> pqStub; + + { std::shared_ptr<grpc::Channel> channel = grpc::CreateChannel("localhost:" + ToString(server.GrpcPort), grpc::InsecureChannelCredentials()); - pqStub = Ydb::PersQueue::V1::PersQueueService::NewStub(channel); - } - auto checkDescribe = [&](const TVector<std::pair<TString, TString>>& readRules) { - DescribeTopicRequest request; - DescribeTopicResponse response; - request.set_path("/Root/PQ/rt3.dc1--acc--some-topic"); - grpc::ClientContext rcontext; - - auto status = pqStub->DescribeTopic(&rcontext, request, &response); - UNIT_ASSERT(status.ok()); - DescribeTopicResult res; - response.operation().result().UnpackTo(&res); - Cerr << response << "\n" << res << "\n"; - UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::SUCCESS); - - UNIT_ASSERT_VALUES_EQUAL(res.settings().read_rules().size(), readRules.size()); - for (ui64 i = 0; i < readRules.size(); ++i) { - const auto& rr = res.settings().read_rules(i); - UNIT_ASSERT_EQUAL(rr.consumer_name(), readRules[i].first); - UNIT_ASSERT_EQUAL(rr.service_type(), readRules[i].second); - } - }; - { - CreateTopicRequest request; - CreateTopicResponse response; - request.set_path("/Root/PQ/rt3.dc1--acc--some-topic"); - auto props = request.mutable_settings(); - props->set_partitions_count(1); - props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE); - props->set_retention_period_ms(TDuration::Days(1).MilliSeconds()); - { - auto rr = props->add_read_rules(); - rr->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1)); - rr->set_consumer_name("acc/consumer1"); - } - { - auto rr = props->add_read_rules(); - rr->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1)); - rr->set_consumer_name("acc/consumer2"); - rr->set_service_type("MyGreatType"); - } - - grpc::ClientContext rcontext; - auto status = pqStub->CreateTopic(&rcontext, request, &response); - - UNIT_ASSERT(status.ok()); - CreateTopicResult res; - response.operation().result().UnpackTo(&res); - Cerr << response << "\n" << res << "\n"; - UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::SUCCESS); - } - checkDescribe({ - {"acc/consumer1", "data-transfer"}, - {"acc/consumer2", "MyGreatType"} - }); - { - AlterTopicRequest request; - AlterTopicResponse response; - request.set_path("/Root/PQ/rt3.dc1--acc--some-topic"); - auto props = request.mutable_settings(); - props->set_partitions_count(1); - props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE); - props->set_retention_period_ms(TDuration::Days(1).MilliSeconds()); - { - auto rr = props->add_read_rules(); - rr->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1)); - rr->set_consumer_name("acc/consumer1"); - } - { - auto rr = props->add_read_rules(); - rr->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1)); - rr->set_consumer_name("acc/consumer2"); - rr->set_service_type("AnotherType"); - } - { - auto rr = props->add_read_rules(); - rr->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1)); - rr->set_consumer_name("acc/consumer3"); - rr->set_service_type("SecondType"); - } - - grpc::ClientContext rcontext; - auto status = pqStub->AlterTopic(&rcontext, request, &response); - - UNIT_ASSERT(status.ok()); - CreateTopicResult res; - response.operation().result().UnpackTo(&res); - Cerr << response << "\n" << res << "\n"; - UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::SUCCESS); - } - checkDescribe({ - {"acc/consumer1", "data-transfer"}, - {"acc/consumer2", "AnotherType"}, - {"acc/consumer3", "SecondType"} - }); - - { - AlterTopicRequest request; - AlterTopicResponse response; - request.set_path("/Root/PQ/rt3.dc1--acc--some-topic"); - auto props = request.mutable_settings(); - props->set_partitions_count(1); - props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE); - props->set_retention_period_ms(TDuration::Days(1).MilliSeconds()); - { - auto rr = props->add_read_rules(); - rr->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1)); - rr->set_consumer_name("acc/consumer1"); - rr->set_service_type("BadServiceType"); - } - - grpc::ClientContext rcontext; - auto status = pqStub->AlterTopic(&rcontext, request, &response); - - UNIT_ASSERT(status.ok()); - CreateTopicResult res; - response.operation().result().UnpackTo(&res); - Cerr << response << "\n" << res << "\n"; - UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::BAD_REQUEST); - } - checkDescribe({ - {"acc/consumer1", "data-transfer"}, - {"acc/consumer2", "AnotherType"}, - {"acc/consumer3", "SecondType"} - }); - } - + pqStub = Ydb::PersQueue::V1::PersQueueService::NewStub(channel); + } + auto checkDescribe = [&](const TVector<std::pair<TString, TString>>& readRules) { + DescribeTopicRequest request; + DescribeTopicResponse response; + request.set_path("/Root/PQ/rt3.dc1--acc--some-topic"); + grpc::ClientContext rcontext; + + auto status = pqStub->DescribeTopic(&rcontext, request, &response); + UNIT_ASSERT(status.ok()); + DescribeTopicResult res; + response.operation().result().UnpackTo(&res); + Cerr << response << "\n" << res << "\n"; + UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::SUCCESS); + + UNIT_ASSERT_VALUES_EQUAL(res.settings().read_rules().size(), readRules.size()); + for (ui64 i = 0; i < readRules.size(); ++i) { + const auto& rr = res.settings().read_rules(i); + UNIT_ASSERT_EQUAL(rr.consumer_name(), readRules[i].first); + UNIT_ASSERT_EQUAL(rr.service_type(), readRules[i].second); + } + }; + { + CreateTopicRequest request; + CreateTopicResponse response; + request.set_path("/Root/PQ/rt3.dc1--acc--some-topic"); + auto props = request.mutable_settings(); + props->set_partitions_count(1); + props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE); + props->set_retention_period_ms(TDuration::Days(1).MilliSeconds()); + { + auto rr = props->add_read_rules(); + rr->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1)); + rr->set_consumer_name("acc/consumer1"); + } + { + auto rr = props->add_read_rules(); + rr->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1)); + rr->set_consumer_name("acc/consumer2"); + rr->set_service_type("MyGreatType"); + } + + grpc::ClientContext rcontext; + auto status = pqStub->CreateTopic(&rcontext, request, &response); + + UNIT_ASSERT(status.ok()); + CreateTopicResult res; + response.operation().result().UnpackTo(&res); + Cerr << response << "\n" << res << "\n"; + UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::SUCCESS); + } + checkDescribe({ + {"acc/consumer1", "data-transfer"}, + {"acc/consumer2", "MyGreatType"} + }); + { + AlterTopicRequest request; + AlterTopicResponse response; + request.set_path("/Root/PQ/rt3.dc1--acc--some-topic"); + auto props = request.mutable_settings(); + props->set_partitions_count(1); + props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE); + props->set_retention_period_ms(TDuration::Days(1).MilliSeconds()); + { + auto rr = props->add_read_rules(); + rr->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1)); + rr->set_consumer_name("acc/consumer1"); + } + { + auto rr = props->add_read_rules(); + rr->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1)); + rr->set_consumer_name("acc/consumer2"); + rr->set_service_type("AnotherType"); + } + { + auto rr = props->add_read_rules(); + rr->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1)); + rr->set_consumer_name("acc/consumer3"); + rr->set_service_type("SecondType"); + } + + grpc::ClientContext rcontext; + auto status = pqStub->AlterTopic(&rcontext, request, &response); + + UNIT_ASSERT(status.ok()); + CreateTopicResult res; + response.operation().result().UnpackTo(&res); + Cerr << response << "\n" << res << "\n"; + UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::SUCCESS); + } + checkDescribe({ + {"acc/consumer1", "data-transfer"}, + {"acc/consumer2", "AnotherType"}, + {"acc/consumer3", "SecondType"} + }); + + { + AlterTopicRequest request; + AlterTopicResponse response; + request.set_path("/Root/PQ/rt3.dc1--acc--some-topic"); + auto props = request.mutable_settings(); + props->set_partitions_count(1); + props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE); + props->set_retention_period_ms(TDuration::Days(1).MilliSeconds()); + { + auto rr = props->add_read_rules(); + rr->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1)); + rr->set_consumer_name("acc/consumer1"); + rr->set_service_type("BadServiceType"); + } + + grpc::ClientContext rcontext; + auto status = pqStub->AlterTopic(&rcontext, request, &response); + + UNIT_ASSERT(status.ok()); + CreateTopicResult res; + response.operation().result().UnpackTo(&res); + Cerr << response << "\n" << res << "\n"; + UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::BAD_REQUEST); + } + checkDescribe({ + {"acc/consumer1", "data-transfer"}, + {"acc/consumer2", "AnotherType"}, + {"acc/consumer3", "SecondType"} + }); + } + Y_UNIT_TEST(ReadRuleServiceTypeLimit) { TServerSettings settings = PQSettings(0); @@ -3023,514 +3023,514 @@ namespace { } - Y_UNIT_TEST(ReadRuleDisallowDefaultServiceType) { - TServerSettings settings = PQSettings(0); - { - settings.PQConfig.AddClientServiceType()->SetName("MyGreatType"); - settings.PQConfig.AddClientServiceType()->SetName("AnotherType"); - settings.PQConfig.AddClientServiceType()->SetName("SecondType"); - settings.PQConfig.SetDisallowDefaultClientServiceType(true); - } - NPersQueue::TTestServer server(settings); - server.EnableLogs({ NKikimrServices::PQ_READ_PROXY, NKikimrServices::BLACKBOX_VALIDATOR }); - - std::unique_ptr<Ydb::PersQueue::V1::PersQueueService::Stub> pqStub; - - { - std::shared_ptr<grpc::Channel> channel = grpc::CreateChannel("localhost:" + ToString(server.GrpcPort), grpc::InsecureChannelCredentials()); - pqStub = Ydb::PersQueue::V1::PersQueueService::NewStub(channel); - } - auto checkDescribe = [&](const TVector<std::pair<TString, TString>>& readRules) { - DescribeTopicRequest request; - DescribeTopicResponse response; - request.set_path("/Root/PQ/rt3.dc1--acc--some-topic"); - grpc::ClientContext rcontext; - - auto status = pqStub->DescribeTopic(&rcontext, request, &response); - UNIT_ASSERT(status.ok()); - DescribeTopicResult res; - response.operation().result().UnpackTo(&res); - Cerr << response << "\n" << res << "\n"; - UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::SUCCESS); - - UNIT_ASSERT_VALUES_EQUAL(res.settings().read_rules().size(), readRules.size()); - for (ui64 i = 0; i < readRules.size(); ++i) { - const auto& rr = res.settings().read_rules(i); - UNIT_ASSERT_EQUAL(rr.consumer_name(), readRules[i].first); - UNIT_ASSERT_EQUAL(rr.service_type(), readRules[i].second); - } - }; - { - CreateTopicRequest request; - CreateTopicResponse response; - request.set_path("/Root/PQ/rt3.dc1--acc--some-topic"); - auto props = request.mutable_settings(); - props->set_partitions_count(1); - props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE); - props->set_retention_period_ms(TDuration::Days(1).MilliSeconds()); - { - auto rr = props->add_read_rules(); - rr->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1)); - rr->set_consumer_name("acc/consumer1"); - } - - grpc::ClientContext rcontext; - auto status = pqStub->CreateTopic(&rcontext, request, &response); - - UNIT_ASSERT(status.ok()); - CreateTopicResult res; - response.operation().result().UnpackTo(&res); - Cerr << response << "\n" << res << "\n"; - UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::BAD_REQUEST); - } - { - CreateTopicRequest request; - CreateTopicResponse response; - request.set_path("/Root/PQ/rt3.dc1--acc--some-topic"); - auto props = request.mutable_settings(); - props->set_partitions_count(1); - props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE); - props->set_retention_period_ms(TDuration::Days(1).MilliSeconds()); - { - auto rr = props->add_read_rules(); - rr->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1)); - rr->set_consumer_name("acc/consumer1"); - rr->set_service_type("MyGreatType"); - } - - grpc::ClientContext rcontext; - auto status = pqStub->CreateTopic(&rcontext, request, &response); - - UNIT_ASSERT(status.ok()); - CreateTopicResult res; - response.operation().result().UnpackTo(&res); - Cerr << response << "\n" << res << "\n"; - UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::SUCCESS); - } - checkDescribe({{"acc/consumer1", "MyGreatType"}}); - { - AlterTopicRequest request; - AlterTopicResponse response; - request.set_path("/Root/PQ/rt3.dc1--acc--some-topic"); - auto props = request.mutable_settings(); - props->set_partitions_count(1); - props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE); - props->set_retention_period_ms(TDuration::Days(1).MilliSeconds()); - { - auto rr = props->add_read_rules(); - rr->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1)); - rr->set_consumer_name("acc/consumer1"); - } - - grpc::ClientContext rcontext; - auto status = pqStub->AlterTopic(&rcontext, request, &response); - - UNIT_ASSERT(status.ok()); - CreateTopicResult res; - response.operation().result().UnpackTo(&res); - Cerr << response << "\n" << res << "\n"; - UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::BAD_REQUEST); - } - checkDescribe({{"acc/consumer1", "MyGreatType"}}); - - { - AlterTopicRequest request; - AlterTopicResponse response; - request.set_path("/Root/PQ/rt3.dc1--acc--some-topic"); - auto props = request.mutable_settings(); - props->set_partitions_count(1); - props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE); - props->set_retention_period_ms(TDuration::Days(1).MilliSeconds()); - { - auto rr = props->add_read_rules(); - rr->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1)); - rr->set_consumer_name("acc/consumer1"); - rr->set_service_type("AnotherType"); - } - { - auto rr = props->add_read_rules(); - rr->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1)); - rr->set_consumer_name("acc/consumer2"); - } - - grpc::ClientContext rcontext; - auto status = pqStub->AlterTopic(&rcontext, request, &response); - - UNIT_ASSERT(status.ok()); - CreateTopicResult res; - response.operation().result().UnpackTo(&res); - Cerr << response << "\n" << res << "\n"; - UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::BAD_REQUEST); - } - checkDescribe({{"acc/consumer1", "MyGreatType"}}); - } - - Y_UNIT_TEST(ReadRuleServiceTypeMigration) { + Y_UNIT_TEST(ReadRuleDisallowDefaultServiceType) { + TServerSettings settings = PQSettings(0); + { + settings.PQConfig.AddClientServiceType()->SetName("MyGreatType"); + settings.PQConfig.AddClientServiceType()->SetName("AnotherType"); + settings.PQConfig.AddClientServiceType()->SetName("SecondType"); + settings.PQConfig.SetDisallowDefaultClientServiceType(true); + } + NPersQueue::TTestServer server(settings); + server.EnableLogs({ NKikimrServices::PQ_READ_PROXY, NKikimrServices::BLACKBOX_VALIDATOR }); + + std::unique_ptr<Ydb::PersQueue::V1::PersQueueService::Stub> pqStub; + + { + std::shared_ptr<grpc::Channel> channel = grpc::CreateChannel("localhost:" + ToString(server.GrpcPort), grpc::InsecureChannelCredentials()); + pqStub = Ydb::PersQueue::V1::PersQueueService::NewStub(channel); + } + auto checkDescribe = [&](const TVector<std::pair<TString, TString>>& readRules) { + DescribeTopicRequest request; + DescribeTopicResponse response; + request.set_path("/Root/PQ/rt3.dc1--acc--some-topic"); + grpc::ClientContext rcontext; + + auto status = pqStub->DescribeTopic(&rcontext, request, &response); + UNIT_ASSERT(status.ok()); + DescribeTopicResult res; + response.operation().result().UnpackTo(&res); + Cerr << response << "\n" << res << "\n"; + UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::SUCCESS); + + UNIT_ASSERT_VALUES_EQUAL(res.settings().read_rules().size(), readRules.size()); + for (ui64 i = 0; i < readRules.size(); ++i) { + const auto& rr = res.settings().read_rules(i); + UNIT_ASSERT_EQUAL(rr.consumer_name(), readRules[i].first); + UNIT_ASSERT_EQUAL(rr.service_type(), readRules[i].second); + } + }; + { + CreateTopicRequest request; + CreateTopicResponse response; + request.set_path("/Root/PQ/rt3.dc1--acc--some-topic"); + auto props = request.mutable_settings(); + props->set_partitions_count(1); + props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE); + props->set_retention_period_ms(TDuration::Days(1).MilliSeconds()); + { + auto rr = props->add_read_rules(); + rr->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1)); + rr->set_consumer_name("acc/consumer1"); + } + + grpc::ClientContext rcontext; + auto status = pqStub->CreateTopic(&rcontext, request, &response); + + UNIT_ASSERT(status.ok()); + CreateTopicResult res; + response.operation().result().UnpackTo(&res); + Cerr << response << "\n" << res << "\n"; + UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::BAD_REQUEST); + } + { + CreateTopicRequest request; + CreateTopicResponse response; + request.set_path("/Root/PQ/rt3.dc1--acc--some-topic"); + auto props = request.mutable_settings(); + props->set_partitions_count(1); + props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE); + props->set_retention_period_ms(TDuration::Days(1).MilliSeconds()); + { + auto rr = props->add_read_rules(); + rr->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1)); + rr->set_consumer_name("acc/consumer1"); + rr->set_service_type("MyGreatType"); + } + + grpc::ClientContext rcontext; + auto status = pqStub->CreateTopic(&rcontext, request, &response); + + UNIT_ASSERT(status.ok()); + CreateTopicResult res; + response.operation().result().UnpackTo(&res); + Cerr << response << "\n" << res << "\n"; + UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::SUCCESS); + } + checkDescribe({{"acc/consumer1", "MyGreatType"}}); + { + AlterTopicRequest request; + AlterTopicResponse response; + request.set_path("/Root/PQ/rt3.dc1--acc--some-topic"); + auto props = request.mutable_settings(); + props->set_partitions_count(1); + props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE); + props->set_retention_period_ms(TDuration::Days(1).MilliSeconds()); + { + auto rr = props->add_read_rules(); + rr->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1)); + rr->set_consumer_name("acc/consumer1"); + } + + grpc::ClientContext rcontext; + auto status = pqStub->AlterTopic(&rcontext, request, &response); + + UNIT_ASSERT(status.ok()); + CreateTopicResult res; + response.operation().result().UnpackTo(&res); + Cerr << response << "\n" << res << "\n"; + UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::BAD_REQUEST); + } + checkDescribe({{"acc/consumer1", "MyGreatType"}}); + + { + AlterTopicRequest request; + AlterTopicResponse response; + request.set_path("/Root/PQ/rt3.dc1--acc--some-topic"); + auto props = request.mutable_settings(); + props->set_partitions_count(1); + props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE); + props->set_retention_period_ms(TDuration::Days(1).MilliSeconds()); + { + auto rr = props->add_read_rules(); + rr->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1)); + rr->set_consumer_name("acc/consumer1"); + rr->set_service_type("AnotherType"); + } + { + auto rr = props->add_read_rules(); + rr->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1)); + rr->set_consumer_name("acc/consumer2"); + } + + grpc::ClientContext rcontext; + auto status = pqStub->AlterTopic(&rcontext, request, &response); + + UNIT_ASSERT(status.ok()); + CreateTopicResult res; + response.operation().result().UnpackTo(&res); + Cerr << response << "\n" << res << "\n"; + UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::BAD_REQUEST); + } + checkDescribe({{"acc/consumer1", "MyGreatType"}}); + } + + Y_UNIT_TEST(ReadRuleServiceTypeMigration) { + TServerSettings settings = PQSettings(0); + { + settings.PQConfig.MutableDefaultClientServiceType()->SetName("default_type"); + settings.PQConfig.AddClientServiceType()->SetName("MyGreatType"); + settings.PQConfig.AddClientServiceType()->SetName("AnotherType"); + settings.PQConfig.AddClientServiceType()->SetName("SecondType"); + } + NPersQueue::TTestServer server(settings); + + server.EnableLogs({ NKikimrServices::PQ_READ_PROXY, NKikimrServices::BLACKBOX_VALIDATOR }); + + const ui32 topicsCount = 4; + for (ui32 i = 1; i <= topicsCount; ++i) { + TRequestCreatePQ createTopicRequest(TStringBuilder() << "rt3.dc1--topic_" << i, 1); + createTopicRequest.ReadRules.push_back("acc@user1"); + createTopicRequest.ReadRules.push_back("acc@user2"); + createTopicRequest.ReadRules.push_back("acc@user3"); + server.AnnoyingClient->CreateTopic(createTopicRequest); + } + + std::unique_ptr<Ydb::PersQueue::V1::PersQueueService::Stub> pqStub; + { + std::shared_ptr<grpc::Channel> channel = grpc::CreateChannel("localhost:" + ToString(server.GrpcPort), grpc::InsecureChannelCredentials()); + pqStub = Ydb::PersQueue::V1::PersQueueService::NewStub(channel); + } + auto doAlter = [&](const TString& topic, const TVector<std::pair<TString, TString>>& readRules) { + AlterTopicRequest request; + AlterTopicResponse response; + request.set_path(topic); + auto props = request.mutable_settings(); + props->set_partitions_count(1); + props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE); + props->set_retention_period_ms(TDuration::Days(1).MilliSeconds()); + for (auto rrInfo : readRules) { + auto rr = props->add_read_rules(); + rr->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1)); + rr->set_consumer_name(rrInfo.first); + rr->set_service_type(rrInfo.second); + } + + grpc::ClientContext rcontext; + auto status = pqStub->AlterTopic(&rcontext, request, &response); + + UNIT_ASSERT(status.ok()); + CreateTopicResult res; + response.operation().result().UnpackTo(&res); + Cerr << response << "\n" << res << "\n"; + UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::SUCCESS); + }; + + + auto checkDescribe = [&](const TString& topic, const TVector<std::pair<TString, TString>>& readRules) { + DescribeTopicRequest request; + DescribeTopicResponse response; + request.set_path(topic); + grpc::ClientContext rcontext; + + auto status = pqStub->DescribeTopic(&rcontext, request, &response); + UNIT_ASSERT(status.ok()); + DescribeTopicResult res; + response.operation().result().UnpackTo(&res); + Cerr << response << "\n" << res << "\n"; + UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::SUCCESS); + + UNIT_ASSERT_VALUES_EQUAL(res.settings().read_rules().size(), readRules.size()); + for (ui64 i = 0; i < readRules.size(); ++i) { + const auto& rr = res.settings().read_rules(i); + UNIT_ASSERT_EQUAL(rr.consumer_name(), readRules[i].first); + UNIT_ASSERT_EQUAL(rr.service_type(), readRules[i].second); + } + }; + checkDescribe( + "/Root/PQ/rt3.dc1--topic_1", + { + {"acc/user1", "default_type"}, + {"acc/user2", "default_type"}, + {"acc/user3", "default_type"} + } + ); + { + doAlter( + "/Root/PQ/rt3.dc1--topic_2", + { + {"acc/user1", ""}, + {"acc/new_user", "MyGreatType"}, + {"acc/user2", "default_type"}, + {"acc/user3", "default_type"}, + {"acc/user4", "AnotherType"} + } + ); + checkDescribe( + "/Root/PQ/rt3.dc1--topic_2", + { + {"acc/user1", "default_type"}, + {"acc/new_user", "MyGreatType"}, + {"acc/user2", "default_type"}, + {"acc/user3", "default_type"}, + {"acc/user4", "AnotherType"} + } + ); + } + { + AddReadRuleRequest request; + AddReadRuleResponse response; + request.set_path("/Root/PQ/rt3.dc1--topic_3"); + auto rr = request.mutable_read_rule(); + rr->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1)); + rr->set_consumer_name("acc/new_user"); + rr->set_service_type("MyGreatType"); + + grpc::ClientContext rcontext; + auto status = pqStub->AddReadRule(&rcontext, request, &response); + + UNIT_ASSERT(status.ok()); + CreateTopicResult res; + response.operation().result().UnpackTo(&res); + Cerr << response << "\n" << res << "\n"; + UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::SUCCESS); + + checkDescribe( + "/Root/PQ/rt3.dc1--topic_3", + { + {"acc/user1", "default_type"}, + {"acc/user2", "default_type"}, + {"acc/user3", "default_type"}, + {"acc/new_user", "MyGreatType"} + } + ); + } + + { + checkDescribe( + "/Root/PQ/rt3.dc1--topic_4", + { + {"acc/user1", "default_type"}, + {"acc/user2", "default_type"}, + {"acc/user3", "default_type"} + } + ); + + RemoveReadRuleRequest request; + RemoveReadRuleResponse response; + request.set_path("/Root/PQ/rt3.dc1--topic_4"); + request.set_consumer_name("acc@user2"); + + grpc::ClientContext rcontext; + auto status = pqStub->RemoveReadRule(&rcontext, request, &response); + + UNIT_ASSERT(status.ok()); + CreateTopicResult res; + response.operation().result().UnpackTo(&res); + Cerr << response << "\n" << res << "\n"; + UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::SUCCESS); + + checkDescribe( + "/Root/PQ/rt3.dc1--topic_4", + { + {"acc/user1", "default_type"}, + {"acc/user3", "default_type"} + } + ); + } + } + + Y_UNIT_TEST(ReadRuleServiceTypeMigrationWithDisallowDefault) { TServerSettings settings = PQSettings(0); - { - settings.PQConfig.MutableDefaultClientServiceType()->SetName("default_type"); - settings.PQConfig.AddClientServiceType()->SetName("MyGreatType"); - settings.PQConfig.AddClientServiceType()->SetName("AnotherType"); - settings.PQConfig.AddClientServiceType()->SetName("SecondType"); - } + { + settings.PQConfig.MutableDefaultClientServiceType()->SetName("default_type"); + settings.PQConfig.AddClientServiceType()->SetName("MyGreatType"); + settings.PQConfig.AddClientServiceType()->SetName("AnotherType"); + settings.PQConfig.AddClientServiceType()->SetName("SecondType"); + settings.PQConfig.SetDisallowDefaultClientServiceType(true); + } NPersQueue::TTestServer server(settings); - + server.EnableLogs({ NKikimrServices::PQ_READ_PROXY, NKikimrServices::BLACKBOX_VALIDATOR }); - - const ui32 topicsCount = 4; - for (ui32 i = 1; i <= topicsCount; ++i) { - TRequestCreatePQ createTopicRequest(TStringBuilder() << "rt3.dc1--topic_" << i, 1); - createTopicRequest.ReadRules.push_back("acc@user1"); - createTopicRequest.ReadRules.push_back("acc@user2"); - createTopicRequest.ReadRules.push_back("acc@user3"); - server.AnnoyingClient->CreateTopic(createTopicRequest); - } - - std::unique_ptr<Ydb::PersQueue::V1::PersQueueService::Stub> pqStub; - { + + const ui32 topicsCount = 4; + for (ui32 i = 1; i <= topicsCount; ++i) { + TRequestCreatePQ createTopicRequest(TStringBuilder() << "rt3.dc1--topic_" << i, 1); + createTopicRequest.ReadRules.push_back("acc@user1"); + createTopicRequest.ReadRules.push_back("acc@user2"); + createTopicRequest.ReadRules.push_back("acc@user3"); + server.AnnoyingClient->CreateTopic(createTopicRequest); + } + + std::unique_ptr<Ydb::PersQueue::V1::PersQueueService::Stub> pqStub; + { std::shared_ptr<grpc::Channel> channel = grpc::CreateChannel("localhost:" + ToString(server.GrpcPort), grpc::InsecureChannelCredentials()); - pqStub = Ydb::PersQueue::V1::PersQueueService::NewStub(channel); - } - auto doAlter = [&](const TString& topic, const TVector<std::pair<TString, TString>>& readRules) { - AlterTopicRequest request; - AlterTopicResponse response; - request.set_path(topic); - auto props = request.mutable_settings(); - props->set_partitions_count(1); - props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE); - props->set_retention_period_ms(TDuration::Days(1).MilliSeconds()); - for (auto rrInfo : readRules) { - auto rr = props->add_read_rules(); - rr->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1)); - rr->set_consumer_name(rrInfo.first); - rr->set_service_type(rrInfo.second); - } - - grpc::ClientContext rcontext; - auto status = pqStub->AlterTopic(&rcontext, request, &response); - - UNIT_ASSERT(status.ok()); - CreateTopicResult res; - response.operation().result().UnpackTo(&res); - Cerr << response << "\n" << res << "\n"; - UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::SUCCESS); - }; - - - auto checkDescribe = [&](const TString& topic, const TVector<std::pair<TString, TString>>& readRules) { - DescribeTopicRequest request; - DescribeTopicResponse response; - request.set_path(topic); - grpc::ClientContext rcontext; - - auto status = pqStub->DescribeTopic(&rcontext, request, &response); - UNIT_ASSERT(status.ok()); - DescribeTopicResult res; - response.operation().result().UnpackTo(&res); - Cerr << response << "\n" << res << "\n"; - UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::SUCCESS); - - UNIT_ASSERT_VALUES_EQUAL(res.settings().read_rules().size(), readRules.size()); - for (ui64 i = 0; i < readRules.size(); ++i) { - const auto& rr = res.settings().read_rules(i); - UNIT_ASSERT_EQUAL(rr.consumer_name(), readRules[i].first); - UNIT_ASSERT_EQUAL(rr.service_type(), readRules[i].second); - } - }; - checkDescribe( - "/Root/PQ/rt3.dc1--topic_1", - { - {"acc/user1", "default_type"}, - {"acc/user2", "default_type"}, - {"acc/user3", "default_type"} - } - ); - { - doAlter( - "/Root/PQ/rt3.dc1--topic_2", - { - {"acc/user1", ""}, - {"acc/new_user", "MyGreatType"}, - {"acc/user2", "default_type"}, - {"acc/user3", "default_type"}, - {"acc/user4", "AnotherType"} - } - ); - checkDescribe( - "/Root/PQ/rt3.dc1--topic_2", - { - {"acc/user1", "default_type"}, - {"acc/new_user", "MyGreatType"}, - {"acc/user2", "default_type"}, - {"acc/user3", "default_type"}, - {"acc/user4", "AnotherType"} - } - ); - } - { - AddReadRuleRequest request; - AddReadRuleResponse response; - request.set_path("/Root/PQ/rt3.dc1--topic_3"); - auto rr = request.mutable_read_rule(); - rr->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1)); - rr->set_consumer_name("acc/new_user"); - rr->set_service_type("MyGreatType"); - - grpc::ClientContext rcontext; - auto status = pqStub->AddReadRule(&rcontext, request, &response); - - UNIT_ASSERT(status.ok()); - CreateTopicResult res; - response.operation().result().UnpackTo(&res); - Cerr << response << "\n" << res << "\n"; - UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::SUCCESS); - - checkDescribe( - "/Root/PQ/rt3.dc1--topic_3", - { - {"acc/user1", "default_type"}, - {"acc/user2", "default_type"}, - {"acc/user3", "default_type"}, - {"acc/new_user", "MyGreatType"} - } - ); - } - - { - checkDescribe( - "/Root/PQ/rt3.dc1--topic_4", - { - {"acc/user1", "default_type"}, - {"acc/user2", "default_type"}, - {"acc/user3", "default_type"} - } - ); - - RemoveReadRuleRequest request; - RemoveReadRuleResponse response; - request.set_path("/Root/PQ/rt3.dc1--topic_4"); - request.set_consumer_name("acc@user2"); - - grpc::ClientContext rcontext; - auto status = pqStub->RemoveReadRule(&rcontext, request, &response); - - UNIT_ASSERT(status.ok()); - CreateTopicResult res; - response.operation().result().UnpackTo(&res); - Cerr << response << "\n" << res << "\n"; - UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::SUCCESS); - - checkDescribe( - "/Root/PQ/rt3.dc1--topic_4", - { - {"acc/user1", "default_type"}, - {"acc/user3", "default_type"} - } - ); - } - } - - Y_UNIT_TEST(ReadRuleServiceTypeMigrationWithDisallowDefault) { - TServerSettings settings = PQSettings(0); - { - settings.PQConfig.MutableDefaultClientServiceType()->SetName("default_type"); - settings.PQConfig.AddClientServiceType()->SetName("MyGreatType"); - settings.PQConfig.AddClientServiceType()->SetName("AnotherType"); - settings.PQConfig.AddClientServiceType()->SetName("SecondType"); - settings.PQConfig.SetDisallowDefaultClientServiceType(true); - } - NPersQueue::TTestServer server(settings); - - server.EnableLogs({ NKikimrServices::PQ_READ_PROXY, NKikimrServices::BLACKBOX_VALIDATOR }); - - const ui32 topicsCount = 4; - for (ui32 i = 1; i <= topicsCount; ++i) { - TRequestCreatePQ createTopicRequest(TStringBuilder() << "rt3.dc1--topic_" << i, 1); - createTopicRequest.ReadRules.push_back("acc@user1"); - createTopicRequest.ReadRules.push_back("acc@user2"); - createTopicRequest.ReadRules.push_back("acc@user3"); - server.AnnoyingClient->CreateTopic(createTopicRequest); - } - - std::unique_ptr<Ydb::PersQueue::V1::PersQueueService::Stub> pqStub; - { - std::shared_ptr<grpc::Channel> channel = grpc::CreateChannel("localhost:" + ToString(server.GrpcPort), grpc::InsecureChannelCredentials()); - pqStub = Ydb::PersQueue::V1::PersQueueService::NewStub(channel); - } - - auto doAlter = [&]( - const TString& topic, - const TVector<std::pair<TString, TString>>& readRules, - Ydb::StatusIds::StatusCode statusCode = Ydb::StatusIds::SUCCESS - ) { - AlterTopicRequest request; - AlterTopicResponse response; - request.set_path(topic); - auto props = request.mutable_settings(); - props->set_partitions_count(1); - props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE); - props->set_retention_period_ms(TDuration::Days(1).MilliSeconds()); - for (auto rrInfo : readRules) { - auto rr = props->add_read_rules(); - rr->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1)); - rr->set_consumer_name(rrInfo.first); - rr->set_service_type(rrInfo.second); - } - - grpc::ClientContext rcontext; - auto status = pqStub->AlterTopic(&rcontext, request, &response); - - UNIT_ASSERT(status.ok()); - CreateTopicResult res; - response.operation().result().UnpackTo(&res); - Cerr << response << "\n" << res << "\n"; - UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), statusCode); - }; - - auto checkDescribe = [&]( - const TString& topic, - const TVector<std::pair<TString, TString>>& readRules, - Ydb::StatusIds::StatusCode statusCode = Ydb::StatusIds::SUCCESS - ) { - DescribeTopicRequest request; - DescribeTopicResponse response; - request.set_path(topic); - grpc::ClientContext rcontext; - - auto status = pqStub->DescribeTopic(&rcontext, request, &response); - UNIT_ASSERT(status.ok()); - DescribeTopicResult res; - response.operation().result().UnpackTo(&res); - Cerr << response << "\n" << res << "\n"; - UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), statusCode); - if (statusCode == Ydb::StatusIds::SUCCESS) { - UNIT_ASSERT_VALUES_EQUAL(res.settings().read_rules().size(), readRules.size()); - for (ui64 i = 0; i < readRules.size(); ++i) { - const auto& rr = res.settings().read_rules(i); - UNIT_ASSERT_EQUAL(rr.consumer_name(), readRules[i].first); - UNIT_ASSERT_EQUAL(rr.service_type(), readRules[i].second); - } - } - }; - checkDescribe( - "/Root/PQ/rt3.dc1--topic_1", - {}, - Ydb::StatusIds::INTERNAL_ERROR - ); - { - doAlter( - "/Root/PQ/rt3.dc1--topic_2", - { - {"acc/new_user", "MyGreatType"}, - {"acc/user2", "SecondType"}, - {"acc/user3", "AnotherType"}, - {"acc/user4", "AnotherType"} - } - ); - checkDescribe( - "/Root/PQ/rt3.dc1--topic_2", - { - {"acc/new_user", "MyGreatType"}, - {"acc/user2", "SecondType"}, - {"acc/user3", "AnotherType"}, - {"acc/user4", "AnotherType"} - } - ); - } - } - - Y_UNIT_TEST(TClusterTrackerTest) { - APITestSetup setup{TEST_CASE_NAME}; - setup.GetPQConfig().SetClustersUpdateTimeoutSec(0); - const auto edgeActorID = setup.GetServer().GetRuntime()->AllocateEdgeActor(); - THashMap<TString, TPQTestClusterInfo> clusters = DEFAULT_CLUSTERS_LIST; - - auto compareInfo = [](const TString& name, const TPQTestClusterInfo& info, const NPQ::NClusterTracker::TClustersList::TCluster& trackerInfo) { - UNIT_ASSERT_EQUAL(name, trackerInfo.Name); - UNIT_ASSERT_EQUAL(name, trackerInfo.Datacenter); - UNIT_ASSERT_EQUAL(info.Balancer, trackerInfo.Balancer); - UNIT_ASSERT_EQUAL(info.Enabled, trackerInfo.IsEnabled); - UNIT_ASSERT_EQUAL(info.Weight, trackerInfo.Weight); - }; - - auto getClustersFromTracker = [&]() { - setup.GetServer().GetRuntime()->Send(new IEventHandle( - NPQ::NClusterTracker::MakeClusterTrackerID(), - edgeActorID, - new NPQ::NClusterTracker::TEvClusterTracker::TEvSubscribe - )); - return setup.GetServer().GetRuntime()->GrabEdgeEvent<NPQ::NClusterTracker::TEvClusterTracker::TEvClustersUpdate>(); - }; - - - { - auto trackerResponce = getClustersFromTracker(); - for (auto& clusterInfo : trackerResponce->ClustersList->Clusters) { - auto it = clusters.find(clusterInfo.Name); - UNIT_ASSERT(it != clusters.end()); - compareInfo(it->first, it->second, clusterInfo); - } - } - - UNIT_ASSERT_EQUAL(clusters.count("dc1"), 1); - UNIT_ASSERT_EQUAL(clusters.count("dc2"), 1); - clusters["dc1"].Weight = 666; - clusters["dc2"].Balancer = "newbalancer.net"; - setup.GetFlatMsgBusPQClient().InitDCs(clusters); - TInstant updateTime = TInstant::Now(); - - while (true) { - auto trackerResponce = getClustersFromTracker(); - if (trackerResponce->ClustersListUpdateTimestamp) { - if (trackerResponce->ClustersListUpdateTimestamp.GetRef() >= updateTime + TDuration::Seconds(5)) { - for (auto& clusterInfo : trackerResponce->ClustersList->Clusters) { - auto it = clusters.find(clusterInfo.Name); - UNIT_ASSERT(it != clusters.end()); - compareInfo(it->first, it->second, clusterInfo); - } - break; - } - } - Sleep(TDuration::MilliSeconds(100)); - } - } - - Y_UNIT_TEST(TestReadPartitionByGroupId) { - NPersQueue::TTestServer server; - - ui32 partitionsCount = 100; + pqStub = Ydb::PersQueue::V1::PersQueueService::NewStub(channel); + } + + auto doAlter = [&]( + const TString& topic, + const TVector<std::pair<TString, TString>>& readRules, + Ydb::StatusIds::StatusCode statusCode = Ydb::StatusIds::SUCCESS + ) { + AlterTopicRequest request; + AlterTopicResponse response; + request.set_path(topic); + auto props = request.mutable_settings(); + props->set_partitions_count(1); + props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE); + props->set_retention_period_ms(TDuration::Days(1).MilliSeconds()); + for (auto rrInfo : readRules) { + auto rr = props->add_read_rules(); + rr->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1)); + rr->set_consumer_name(rrInfo.first); + rr->set_service_type(rrInfo.second); + } + + grpc::ClientContext rcontext; + auto status = pqStub->AlterTopic(&rcontext, request, &response); + + UNIT_ASSERT(status.ok()); + CreateTopicResult res; + response.operation().result().UnpackTo(&res); + Cerr << response << "\n" << res << "\n"; + UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), statusCode); + }; + + auto checkDescribe = [&]( + const TString& topic, + const TVector<std::pair<TString, TString>>& readRules, + Ydb::StatusIds::StatusCode statusCode = Ydb::StatusIds::SUCCESS + ) { + DescribeTopicRequest request; + DescribeTopicResponse response; + request.set_path(topic); + grpc::ClientContext rcontext; + + auto status = pqStub->DescribeTopic(&rcontext, request, &response); + UNIT_ASSERT(status.ok()); + DescribeTopicResult res; + response.operation().result().UnpackTo(&res); + Cerr << response << "\n" << res << "\n"; + UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), statusCode); + if (statusCode == Ydb::StatusIds::SUCCESS) { + UNIT_ASSERT_VALUES_EQUAL(res.settings().read_rules().size(), readRules.size()); + for (ui64 i = 0; i < readRules.size(); ++i) { + const auto& rr = res.settings().read_rules(i); + UNIT_ASSERT_EQUAL(rr.consumer_name(), readRules[i].first); + UNIT_ASSERT_EQUAL(rr.service_type(), readRules[i].second); + } + } + }; + checkDescribe( + "/Root/PQ/rt3.dc1--topic_1", + {}, + Ydb::StatusIds::INTERNAL_ERROR + ); + { + doAlter( + "/Root/PQ/rt3.dc1--topic_2", + { + {"acc/new_user", "MyGreatType"}, + {"acc/user2", "SecondType"}, + {"acc/user3", "AnotherType"}, + {"acc/user4", "AnotherType"} + } + ); + checkDescribe( + "/Root/PQ/rt3.dc1--topic_2", + { + {"acc/new_user", "MyGreatType"}, + {"acc/user2", "SecondType"}, + {"acc/user3", "AnotherType"}, + {"acc/user4", "AnotherType"} + } + ); + } + } + + Y_UNIT_TEST(TClusterTrackerTest) { + APITestSetup setup{TEST_CASE_NAME}; + setup.GetPQConfig().SetClustersUpdateTimeoutSec(0); + const auto edgeActorID = setup.GetServer().GetRuntime()->AllocateEdgeActor(); + THashMap<TString, TPQTestClusterInfo> clusters = DEFAULT_CLUSTERS_LIST; + + auto compareInfo = [](const TString& name, const TPQTestClusterInfo& info, const NPQ::NClusterTracker::TClustersList::TCluster& trackerInfo) { + UNIT_ASSERT_EQUAL(name, trackerInfo.Name); + UNIT_ASSERT_EQUAL(name, trackerInfo.Datacenter); + UNIT_ASSERT_EQUAL(info.Balancer, trackerInfo.Balancer); + UNIT_ASSERT_EQUAL(info.Enabled, trackerInfo.IsEnabled); + UNIT_ASSERT_EQUAL(info.Weight, trackerInfo.Weight); + }; + + auto getClustersFromTracker = [&]() { + setup.GetServer().GetRuntime()->Send(new IEventHandle( + NPQ::NClusterTracker::MakeClusterTrackerID(), + edgeActorID, + new NPQ::NClusterTracker::TEvClusterTracker::TEvSubscribe + )); + return setup.GetServer().GetRuntime()->GrabEdgeEvent<NPQ::NClusterTracker::TEvClusterTracker::TEvClustersUpdate>(); + }; + + + { + auto trackerResponce = getClustersFromTracker(); + for (auto& clusterInfo : trackerResponce->ClustersList->Clusters) { + auto it = clusters.find(clusterInfo.Name); + UNIT_ASSERT(it != clusters.end()); + compareInfo(it->first, it->second, clusterInfo); + } + } + + UNIT_ASSERT_EQUAL(clusters.count("dc1"), 1); + UNIT_ASSERT_EQUAL(clusters.count("dc2"), 1); + clusters["dc1"].Weight = 666; + clusters["dc2"].Balancer = "newbalancer.net"; + setup.GetFlatMsgBusPQClient().InitDCs(clusters); + TInstant updateTime = TInstant::Now(); + + while (true) { + auto trackerResponce = getClustersFromTracker(); + if (trackerResponce->ClustersListUpdateTimestamp) { + if (trackerResponce->ClustersListUpdateTimestamp.GetRef() >= updateTime + TDuration::Seconds(5)) { + for (auto& clusterInfo : trackerResponce->ClustersList->Clusters) { + auto it = clusters.find(clusterInfo.Name); + UNIT_ASSERT(it != clusters.end()); + compareInfo(it->first, it->second, clusterInfo); + } + break; + } + } + Sleep(TDuration::MilliSeconds(100)); + } + } + + Y_UNIT_TEST(TestReadPartitionByGroupId) { + NPersQueue::TTestServer server; + + ui32 partitionsCount = 100; TString topic = "topic1"; TString topicFullName = "rt3.dc1--" + topic; - + server.AnnoyingClient->CreateTopic(topicFullName, partitionsCount); server.EnableLogs({ NKikimrServices::PQ_READ_PROXY}); - - auto driver = server.AnnoyingClient->GetDriver(); - - for (ui32 partition = 30; partition < partitionsCount; ++partition) { - auto reader = CreateReader( - *driver, - NYdb::NPersQueue::TReadSessionSettings() - .AppendTopics( - NYdb::NPersQueue::TTopicReadSettings(topic) - .AppendPartitionGroupIds(partition + 1) - ) - .ConsumerName("shared/user") - .ReadOnlyOriginal(true) - ); - - TMaybe<NYdb::NPersQueue::TReadSessionEvent::TEvent> event = reader->GetEvent(true, 1); - auto createStream = std::get_if<NYdb::NPersQueue::TReadSessionEvent::TCreatePartitionStreamEvent>(&*event); - UNIT_ASSERT(createStream); + + auto driver = server.AnnoyingClient->GetDriver(); + + for (ui32 partition = 30; partition < partitionsCount; ++partition) { + auto reader = CreateReader( + *driver, + NYdb::NPersQueue::TReadSessionSettings() + .AppendTopics( + NYdb::NPersQueue::TTopicReadSettings(topic) + .AppendPartitionGroupIds(partition + 1) + ) + .ConsumerName("shared/user") + .ReadOnlyOriginal(true) + ); + + TMaybe<NYdb::NPersQueue::TReadSessionEvent::TEvent> event = reader->GetEvent(true, 1); + auto createStream = std::get_if<NYdb::NPersQueue::TReadSessionEvent::TCreatePartitionStreamEvent>(&*event); + UNIT_ASSERT(createStream); TString stepDescription = TStringBuilder() << "create stream for partition=" << partition - << " : " << createStream->DebugString(); - Cerr << stepDescription << Endl; - UNIT_ASSERT_EQUAL_C( - partition, - createStream->GetPartitionStream()->GetPartitionId(), - stepDescription + << " : " << createStream->DebugString(); + Cerr << stepDescription << Endl; + UNIT_ASSERT_EQUAL_C( + partition, + createStream->GetPartitionStream()->GetPartitionId(), + stepDescription ); } - } + } } } diff --git a/ydb/services/persqueue_v1/ut/new_schemecache_ut/ya.make b/ydb/services/persqueue_v1/ut/new_schemecache_ut/ya.make index 4cffa3015d..2a8fc402a9 100644 --- a/ydb/services/persqueue_v1/ut/new_schemecache_ut/ya.make +++ b/ydb/services/persqueue_v1/ut/new_schemecache_ut/ya.make @@ -3,7 +3,7 @@ UNITTEST_FOR(ydb/services/persqueue_v1) OWNER( zhenyok g:kikimr - g:logbroker + g:logbroker ) CFLAGS( diff --git a/ydb/services/persqueue_v1/ut/rate_limiter_test_setup.cpp b/ydb/services/persqueue_v1/ut/rate_limiter_test_setup.cpp index c801eb7b4e..779d60e778 100644 --- a/ydb/services/persqueue_v1/ut/rate_limiter_test_setup.cpp +++ b/ydb/services/persqueue_v1/ut/rate_limiter_test_setup.cpp @@ -9,88 +9,88 @@ using namespace NKikimr::Tests; namespace NKikimr::NPersQueueTests { -TRateLimiterTestSetup::TRateLimiterTestSetup( - NKikimrPQ::TPQConfig::TQuotingConfig::ELimitedEntity limitedEntity, - double writeAccountQuota, - double readAccountQuota, - bool enableReadQuoting -) +TRateLimiterTestSetup::TRateLimiterTestSetup( + NKikimrPQ::TPQConfig::TQuotingConfig::ELimitedEntity limitedEntity, + double writeAccountQuota, + double readAccountQuota, + bool enableReadQuoting +) : Server(new NPersQueue::TTestServer(false)) , LimitedEntity(limitedEntity) - , WriteAccountQuota(writeAccountQuota) - , ReadAccountQuota(readAccountQuota) + , WriteAccountQuota(writeAccountQuota) + , ReadAccountQuota(readAccountQuota) { - Start(enableReadQuoting); + Start(enableReadQuoting); } -void TRateLimiterTestSetup::CreateTopic(const TString& path) { +void TRateLimiterTestSetup::CreateTopic(const TString& path) { const TString name = BuildFullTopicName(path, "dc1"); const TString account = GetAccount(name); Cerr << "Creating topic \"" << name << "\"" << Endl; Server->AnnoyingClient->CreateTopic(name, 1); - CreateKesus(account); - CreateQuotaResources(path, "write-quota", false); - CreateQuotaResources(path, "read-quota", true); -} + CreateKesus(account); + CreateQuotaResources(path, "write-quota", false); + CreateQuotaResources(path, "read-quota", true); +} -void TRateLimiterTestSetup::CreateConsumer(const TString& path) { - const TString account = GetAccount(path); +void TRateLimiterTestSetup::CreateConsumer(const TString& path) { + const TString account = GetAccount(path); - Cerr << "Creating consumer \"" << path << "\"" << Endl; + Cerr << "Creating consumer \"" << path << "\"" << Endl; Server->AnnoyingClient->CreateConsumer(path); - CreateKesus(account); - CreateQuotaResources(path, "write-quota", true); - CreateQuotaResources(path, "read-quota", false); -} - -void TRateLimiterTestSetup::CreateKesus(const TString& account) { + CreateKesus(account); + CreateQuotaResources(path, "write-quota", true); + CreateQuotaResources(path, "read-quota", false); +} + +void TRateLimiterTestSetup::CreateKesus(const TString& account) { const NMsgBusProxy::EResponseStatus createKesusResult = Server->AnnoyingClient->CreateKesus(QuotersRootPath, account); - UNIT_ASSERT_C(createKesusResult == NMsgBusProxy::MSTATUS_OK, createKesusResult); - - const TString kesusPath = TStringBuilder() << QuotersRootPath << "/" << account; + UNIT_ASSERT_C(createKesusResult == NMsgBusProxy::MSTATUS_OK, createKesusResult); - Cerr << "Creating kesus with path=" << kesusPath << Endl; - auto setAccountQuota = [&](const TString& quotaPrefix, double value) { + const TString kesusPath = TStringBuilder() << QuotersRootPath << "/" << account; + + Cerr << "Creating kesus with path=" << kesusPath << Endl; + auto setAccountQuota = [&](const TString& quotaPrefix, double value) { Cerr << "Adding quota for account kesus=" << kesusPath << " quota-path=" << quotaPrefix << " value=" << value << Endl; const auto statusCode = Server->AnnoyingClient->AddQuoterResource( Server->CleverServer->GetRuntime(), kesusPath, quotaPrefix, value ); - UNIT_ASSERT_C( - statusCode == Ydb::StatusIds::SUCCESS || statusCode == Ydb::StatusIds::ALREADY_EXISTS, - "Status: " << Ydb::StatusIds::StatusCode_Name(statusCode) - ); - }; - setAccountQuota("write-quota", WriteAccountQuota); - setAccountQuota("read-quota", ReadAccountQuota); -} - -void TRateLimiterTestSetup::CreateQuotaResources(const TString& path, const TString& quotaPrefix, bool excludeLastComponent) { + UNIT_ASSERT_C( + statusCode == Ydb::StatusIds::SUCCESS || statusCode == Ydb::StatusIds::ALREADY_EXISTS, + "Status: " << Ydb::StatusIds::StatusCode_Name(statusCode) + ); + }; + setAccountQuota("write-quota", WriteAccountQuota); + setAccountQuota("read-quota", ReadAccountQuota); +} + +void TRateLimiterTestSetup::CreateQuotaResources(const TString& path, const TString& quotaPrefix, bool excludeLastComponent) { TVector<TString> pathComponents = SplitPath(path); - if (pathComponents.size() <= 1) { - return; + if (pathComponents.size() <= 1) { + return; } - TStringBuilder prefixPath; - prefixPath << quotaPrefix; - - auto firstIt = pathComponents.begin() + 1; // resource path must be without account - auto lastIt = pathComponents.end() - (excludeLastComponent ? 1 : 0); - - const TString account = GetAccount(path); - const TString kesusPath = TStringBuilder() << QuotersRootPath << "/" << account; - for (auto currentComponent = firstIt; currentComponent != lastIt; ++currentComponent) { - prefixPath << "/" << *currentComponent; - Cerr << "Adding quoter resource: \"" << prefixPath << "\"" << Endl; + TStringBuilder prefixPath; + prefixPath << quotaPrefix; + + auto firstIt = pathComponents.begin() + 1; // resource path must be without account + auto lastIt = pathComponents.end() - (excludeLastComponent ? 1 : 0); + + const TString account = GetAccount(path); + const TString kesusPath = TStringBuilder() << QuotersRootPath << "/" << account; + for (auto currentComponent = firstIt; currentComponent != lastIt; ++currentComponent) { + prefixPath << "/" << *currentComponent; + Cerr << "Adding quoter resource: \"" << prefixPath << "\"" << Endl; const auto statusCode = Server->AnnoyingClient->AddQuoterResource( Server->CleverServer->GetRuntime(), kesusPath, prefixPath ); - UNIT_ASSERT_C( - statusCode == Ydb::StatusIds::SUCCESS || statusCode == Ydb::StatusIds::ALREADY_EXISTS, - "Status: " << Ydb::StatusIds::StatusCode_Name(statusCode) - ); - } + UNIT_ASSERT_C( + statusCode == Ydb::StatusIds::SUCCESS || statusCode == Ydb::StatusIds::ALREADY_EXISTS, + "Status: " << Ydb::StatusIds::StatusCode_Name(statusCode) + ); + } } /* @@ -107,13 +107,13 @@ THolder<Ydb::PersQueue::IProducer> TRateLimiterTestSetup::StartProducer(const TS } */ -void TRateLimiterTestSetup::Start(bool enableReadQuoting) { - InitServer(enableReadQuoting); +void TRateLimiterTestSetup::Start(bool enableReadQuoting) { + InitServer(enableReadQuoting); InitQuoting(); WaitWritePQServiceInitialization(); } -void TRateLimiterTestSetup::InitServer(bool enableReadQuoting) { +void TRateLimiterTestSetup::InitServer(bool enableReadQuoting) { auto& settings = Server->ServerSettings; settings.PQConfig.MutableQuotingConfig()->SetEnableQuoting(true); @@ -130,7 +130,7 @@ void TRateLimiterTestSetup::InitServer(bool enableReadQuoting) { NKikimrServices::QUOTER_SERVICE, NKikimrServices::QUOTER_PROXY, NKikimrServices::KESUS_TABLET, - NKikimrServices::PQ_READ_SPEED_LIMITER + NKikimrServices::PQ_READ_SPEED_LIMITER }, NActors::NLog::PRI_TRACE ); diff --git a/ydb/services/persqueue_v1/ut/rate_limiter_test_setup.h b/ydb/services/persqueue_v1/ut/rate_limiter_test_setup.h index dbb25e3769..01529818e1 100644 --- a/ydb/services/persqueue_v1/ut/rate_limiter_test_setup.h +++ b/ydb/services/persqueue_v1/ut/rate_limiter_test_setup.h @@ -8,15 +8,15 @@ namespace NKikimr::NPersQueueTests { class TRateLimiterTestSetup { public: - explicit TRateLimiterTestSetup( - NKikimrPQ::TPQConfig::TQuotingConfig::ELimitedEntity limitedEntity, - double writeAccountQuota = 1000.0, - double readAccountQuota = 1000.0, - bool enableReadQuoting = false - ); + explicit TRateLimiterTestSetup( + NKikimrPQ::TPQConfig::TQuotingConfig::ELimitedEntity limitedEntity, + double writeAccountQuota = 1000.0, + double readAccountQuota = 1000.0, + bool enableReadQuoting = false + ); - void CreateTopic(const TString& path); - void CreateConsumer(const TString& path); + void CreateTopic(const TString& path); + void CreateConsumer(const TString& path); // namespace NPersQueue = Ydb::PersQueue; // THolder<Ydb::PersQueue::IProducer> StartProducer(const TString& topicPath, bool compress = false); @@ -39,12 +39,12 @@ public: } private: - void CreateKesus(const TString& account); - void CreateQuotaResources(const TString& path, const TString& quotaPrefix, bool excludeLastComponent); + void CreateKesus(const TString& account); + void CreateQuotaResources(const TString& path, const TString& quotaPrefix, bool excludeLastComponent); - void Start(bool enableReadQuoting); - - void InitServer(bool enableReadQuoting); + void Start(bool enableReadQuoting); + + void InitServer(bool enableReadQuoting); void InitQuoting(); void WaitWritePQServiceInitialization(); @@ -52,8 +52,8 @@ private: THolder<NPersQueue::TTestServer> Server; THolder<TPQDataWriter> PQDataWriter; // For waiting for grpc writer service initialization. const NKikimrPQ::TPQConfig::TQuotingConfig::ELimitedEntity LimitedEntity; - double WriteAccountQuota; - double ReadAccountQuota; - const TString QuotersRootPath = "/Root/PersQueue/System/Quoters"; + double WriteAccountQuota; + double ReadAccountQuota; + const TString QuotersRootPath = "/Root/PersQueue/System/Quoters"; }; } diff --git a/ydb/services/persqueue_v1/ut/ya.make b/ydb/services/persqueue_v1/ut/ya.make index e349b2e85f..70c088383d 100644 --- a/ydb/services/persqueue_v1/ut/ya.make +++ b/ydb/services/persqueue_v1/ut/ya.make @@ -1,11 +1,11 @@ UNITTEST_FOR(ydb/services/persqueue_v1) -OWNER( - alexnick +OWNER( + alexnick g:kikimr - g:logbroker -) - + g:logbroker +) + CFLAGS( -DACTORLIB_HUGE_PB_SIZE ) diff --git a/ydb/services/persqueue_v1/ya.make b/ydb/services/persqueue_v1/ya.make index 79179d7463..ade0406fcc 100644 --- a/ydb/services/persqueue_v1/ya.make +++ b/ydb/services/persqueue_v1/ya.make @@ -3,12 +3,12 @@ LIBRARY() OWNER( alexnick g:kikimr - g:logbroker + g:logbroker ) SRCS( grpc_pq_actor.h - grpc_pq_codecs.cpp + grpc_pq_codecs.cpp grpc_pq_read_actor.cpp grpc_pq_read.cpp grpc_pq_read.h @@ -37,7 +37,7 @@ PEERDIR( ydb/core/ydb_convert ydb/library/aclib ydb/library/persqueue/obfuscate - ydb/library/persqueue/tests + ydb/library/persqueue/tests ydb/library/persqueue/topic_parser ydb/public/api/grpc/draft ydb/public/api/protos diff --git a/ydb/services/rate_limiter/rate_limiter_ut.cpp b/ydb/services/rate_limiter/rate_limiter_ut.cpp index 1be91c9454..4ac05a571c 100644 --- a/ydb/services/rate_limiter/rate_limiter_ut.cpp +++ b/ydb/services/rate_limiter/rate_limiter_ut.cpp @@ -76,7 +76,7 @@ public: void virtual CheckAcquireResource(const TString& coordinationNodePath, const TString& resourcePath, const NYdb::NRateLimiter::TAcquireResourceSettings& settings, NYdb::EStatus expected) { const auto acquireResultFuture = RateLimiterClient.AcquireResource(coordinationNodePath, resourcePath, settings); ASSERT_STATUS(acquireResultFuture, expected); - } + } static TString CoordinationNodePath; @@ -345,7 +345,7 @@ Y_UNIT_TEST_SUITE(TGRpcRateLimiterTest) { setup->CheckAcquireResource(TTestSetup::CoordinationNodePath, "res", TAcquireResourceSettings().Amount(1).OperationTimeout(TDuration::MilliSeconds(200)), NYdb::EStatus::TIMEOUT); setup->CheckAcquireResource(TTestSetup::CoordinationNodePath, "res", TAcquireResourceSettings().Amount(1).IsUsedAmount(true).OperationTimeout(TDuration::MilliSeconds(200)), NYdb::EStatus::SUCCESS); } - } + } Y_UNIT_TEST(AcquireResourceManyRequiredGrpcApi) { AcquireResourceManyRequired(false); diff --git a/ydb/services/ydb/ydb_stats_ut.cpp b/ydb/services/ydb/ydb_stats_ut.cpp index dcb7029cfe..b40e22fc42 100644 --- a/ydb/services/ydb/ydb_stats_ut.cpp +++ b/ydb/services/ydb/ydb_stats_ut.cpp @@ -1,15 +1,15 @@ -#include "ydb_common_ut.h" - +#include "ydb_common_ut.h" + #include <ydb/public/sdk/cpp/client/extensions/solomon_stats/pull_connector.h> #include <ydb/public/sdk/cpp/client/ydb_extension/extension.h> #include <ydb/public/sdk/cpp/client/ydb_table/table.h> - + #include <library/cpp/testing/unittest/registar.h> #include <library/cpp/testing/unittest/tests_data.h> -#include <library/cpp/monlib/encode/encoder.h> +#include <library/cpp/monlib/encode/encoder.h> #include <library/cpp/monlib/encode/json/json.h> - -#include <util/generic/ptr.h> + +#include <util/generic/ptr.h> #include <util/system/valgrind.h> struct TStatCounters { diff --git a/ydb/services/ydb/ydb_table_ut.cpp b/ydb/services/ydb/ydb_table_ut.cpp index e22d10131d..4861c06b92 100644 --- a/ydb/services/ydb/ydb_table_ut.cpp +++ b/ydb/services/ydb/ydb_table_ut.cpp @@ -1,4 +1,4 @@ -#include "ydb_common_ut.h" +#include "ydb_common_ut.h" #include <ydb/public/api/grpc/ydb_table_v1.grpc.pb.h> #include <ydb/public/sdk/cpp/client/ydb_proto/accessor.h> @@ -11,11 +11,11 @@ #include <ydb/public/sdk/cpp/client/ydb_operation/operation.h> #include <ydb/public/sdk/cpp/client/resources/ydb_resources.h> #include <ydb/public/lib/yson_value/ydb_yson_value.h> - + #include <ydb/library/yql/public/issue/yql_issue.h> #include <ydb/library/yql/public/issue/yql_issue_message.h> -#include <library/cpp/grpc/client/grpc_client_low.h> +#include <library/cpp/grpc/client/grpc_client_low.h> #include <util/thread/factory.h> diff --git a/ydb/tests/functional/sqs/sqs_test_base.py b/ydb/tests/functional/sqs/sqs_test_base.py index 26ebcb3bec..05d5303f64 100644 --- a/ydb/tests/functional/sqs/sqs_test_base.py +++ b/ydb/tests/functional/sqs/sqs_test_base.py @@ -15,7 +15,7 @@ import ydb.tests.library.common.yatest_common as yatest_common from ydb.tests.library.harness.kikimr_cluster import kikimr_cluster_factory from ydb.tests.library.harness.kikimr_config import KikimrConfigGenerator from ydb.tests.library.harness.util import LogLevels -from ydb.tests.library.sqs.tables import create_all_tables as create_all_sqs_tables +from ydb.tests.library.sqs.tables import create_all_tables as create_all_sqs_tables from sqs_requests_client import SqsHttpApi @@ -282,7 +282,7 @@ class KikimrSqsTestBase(object): driver.wait() with ydb.SessionPool(driver, size=1) as pool: with pool.checkout() as session: - create_all_sqs_tables(cls.sqs_root, driver, session) + create_all_sqs_tables(cls.sqs_root, driver, session) cls.create_metauser(cluster, config_generator) @classmethod diff --git a/ydb/tests/functional/sqs/test_multiplexing_tables_format.py b/ydb/tests/functional/sqs/test_multiplexing_tables_format.py index 44065f6e30..f5840333d3 100644 --- a/ydb/tests/functional/sqs/test_multiplexing_tables_format.py +++ b/ydb/tests/functional/sqs/test_multiplexing_tables_format.py @@ -1,56 +1,56 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import pytest -import yatest - -from sqs_test_base import KikimrSqsTestBase, get_test_with_sqs_installation_by_path, get_test_with_sqs_tenant_installation, IS_FIFO_PARAMS - - -class MultiplexingTablesFormatTest(KikimrSqsTestBase): - def _set_new_format_settings(self, username=None, tables_format=1): - if username is None: - username = self._username - self._execute_yql_query( - f'UPSERT INTO `{self.sqs_root}/.Settings` (Account, Name, Value) \ - VALUES ("{username}", "CreateQueuesWithTabletFormat", "{tables_format}")' - ) - - def create_queue(self, is_fifo): - if is_fifo: - self.queue_name = self.queue_name + '.fifo' - self._create_queue_and_assert(self.queue_name, is_fifo=is_fifo) - - def create_queue_must_fail(self, is_fifo): - try: - self.create_queue(is_fifo) - except yatest.common.process.ExecutionError: - return - assert(False) - - def create_queue_with_wrong_tables_format(self, tables_format): - self._set_new_format_settings(tables_format='qwerty') - self.create_queue_must_fail(True) - self.create_queue_must_fail(False) - - @pytest.mark.parametrize(**IS_FIFO_PARAMS) - def test_create_queue(self, is_fifo): - self._set_new_format_settings() - self.create_queue(is_fifo) - - def test_create_queue_with_incorrect_tables_format(self): - self.create_queue_with_wrong_tables_format('qwerty') - - def test_create_queue_with_empty_tables_format(self): - self.create_queue_with_wrong_tables_format('') - - def test_create_queue_with_unsupported_tables_format(self): - self.create_queue_with_wrong_tables_format(2) - - -class TestMultiplexingTablesFormatWithTenant(get_test_with_sqs_tenant_installation(MultiplexingTablesFormatTest)): - pass - - -class TestMultiplexingTablesFormatWithPath(get_test_with_sqs_installation_by_path(MultiplexingTablesFormatTest)): - pass +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import pytest +import yatest + +from sqs_test_base import KikimrSqsTestBase, get_test_with_sqs_installation_by_path, get_test_with_sqs_tenant_installation, IS_FIFO_PARAMS + + +class MultiplexingTablesFormatTest(KikimrSqsTestBase): + def _set_new_format_settings(self, username=None, tables_format=1): + if username is None: + username = self._username + self._execute_yql_query( + f'UPSERT INTO `{self.sqs_root}/.Settings` (Account, Name, Value) \ + VALUES ("{username}", "CreateQueuesWithTabletFormat", "{tables_format}")' + ) + + def create_queue(self, is_fifo): + if is_fifo: + self.queue_name = self.queue_name + '.fifo' + self._create_queue_and_assert(self.queue_name, is_fifo=is_fifo) + + def create_queue_must_fail(self, is_fifo): + try: + self.create_queue(is_fifo) + except yatest.common.process.ExecutionError: + return + assert(False) + + def create_queue_with_wrong_tables_format(self, tables_format): + self._set_new_format_settings(tables_format='qwerty') + self.create_queue_must_fail(True) + self.create_queue_must_fail(False) + + @pytest.mark.parametrize(**IS_FIFO_PARAMS) + def test_create_queue(self, is_fifo): + self._set_new_format_settings() + self.create_queue(is_fifo) + + def test_create_queue_with_incorrect_tables_format(self): + self.create_queue_with_wrong_tables_format('qwerty') + + def test_create_queue_with_empty_tables_format(self): + self.create_queue_with_wrong_tables_format('') + + def test_create_queue_with_unsupported_tables_format(self): + self.create_queue_with_wrong_tables_format(2) + + +class TestMultiplexingTablesFormatWithTenant(get_test_with_sqs_tenant_installation(MultiplexingTablesFormatTest)): + pass + + +class TestMultiplexingTablesFormatWithPath(get_test_with_sqs_installation_by_path(MultiplexingTablesFormatTest)): + pass diff --git a/ydb/tests/functional/sqs/ya.make b/ydb/tests/functional/sqs/ya.make index 8e8170d042..bbb0a57bb8 100644 --- a/ydb/tests/functional/sqs/ya.make +++ b/ydb/tests/functional/sqs/ya.make @@ -17,7 +17,7 @@ TEST_SRCS( test_generic_messaging.py test_fifo_messaging.py test_multinode_cluster.py - test_multiplexing_tables_format.py + test_multiplexing_tables_format.py test_ping.py test_polling.py test_queue_attributes_validation.py @@ -50,7 +50,7 @@ DEPENDS( PEERDIR( ydb/tests/library - ydb/tests/library/sqs + ydb/tests/library/sqs contrib/python/xmltodict contrib/python/boto3 contrib/python/botocore diff --git a/ydb/tests/library/harness/kikimr_client.py b/ydb/tests/library/harness/kikimr_client.py index 225a50487f..d74383ae23 100644 --- a/ydb/tests/library/harness/kikimr_client.py +++ b/ydb/tests/library/harness/kikimr_client.py @@ -74,14 +74,14 @@ class KiKiMRMessageBusClient(object): request.Options.ReturnPartitioningInfo = False return self.send(request, 'SchemeDescribe') - def _get_invoke_callee(self, method): - return getattr(self._stub, method) - + def _get_invoke_callee(self, method): + return getattr(self._stub, method) + def invoke(self, request, method): retry = self.__retry_count while True: try: - callee = self._get_invoke_callee(method) + callee = self._get_invoke_callee(method) return callee(request) except (RuntimeError, grpc.RpcError): retry -= 1 diff --git a/ydb/tests/library/ya.make b/ydb/tests/library/ya.make index 3ab351ae9b..33fbdd5d01 100644 --- a/ydb/tests/library/ya.make +++ b/ydb/tests/library/ya.make @@ -96,7 +96,7 @@ PEERDIR( ydb/public/api/grpc ydb/public/api/grpc/draft ydb/public/api/protos - ydb/tests/library/sqs + ydb/tests/library/sqs ) END() |