diff options
author | komels <komels@yandex-team.ru> | 2022-02-10 16:50:21 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:50:21 +0300 |
commit | 0dd632d6fc5676c75d7004172992cefaa2192db0 (patch) | |
tree | 5d5cb817648f650d76cf1076100726fd9b8448e8 | |
parent | 244466466f4f03621b0b399620f9094a16b511d2 (diff) | |
download | ydb-0dd632d6fc5676c75d7004172992cefaa2192db0.tar.gz |
Restoring authorship annotation for <komels@yandex-team.ru>. Commit 2 of 2.
147 files changed, 10519 insertions, 10519 deletions
diff --git a/library/cpp/grpc/server/grpc_request.h b/library/cpp/grpc/server/grpc_request.h index b61ac1a351..5bd8d3902b 100644 --- a/library/cpp/grpc/server/grpc_request.h +++ b/library/cpp/grpc/server/grpc_request.h @@ -113,10 +113,10 @@ public: return FinishPromise_.GetFuture(); } - TString GetPeer() const override { - return TString(this->Context.peer()); - } - + TString GetPeer() const override { + return TString(this->Context.peer()); + } + bool SslServer() const override { return Server_->SslServer(); } diff --git a/library/cpp/grpc/server/grpc_request_base.h b/library/cpp/grpc/server/grpc_request_base.h index c80c62cb60..fcfce1c181 100644 --- a/library/cpp/grpc/server/grpc_request_base.h +++ b/library/cpp/grpc/server/grpc_request_base.h @@ -105,9 +105,9 @@ public: //! Returns future to get cancel of finish notification virtual TAsyncFinishResult GetFinishFuture() = 0; - - //! Returns peer address - virtual TString GetPeer() const = 0; + + //! Returns peer address + virtual TString GetPeer() const = 0; //! Returns true if server is using ssl virtual bool SslServer() const = 0; diff --git a/ydb/core/base/appdata.h b/ydb/core/base/appdata.h index 976e99eaab..c666f7468c 100644 --- a/ydb/core/base/appdata.h +++ b/ydb/core/base/appdata.h @@ -34,10 +34,10 @@ namespace NMonitoring { class TBusNgMonPage; } -namespace NYdb { - class TDriver; -} - +namespace NYdb { + class TDriver; +} + namespace NKikimr { namespace NScheme { diff --git a/ydb/core/client/server/msgbus_server_persqueue.cpp b/ydb/core/client/server/msgbus_server_persqueue.cpp index f2e1d7ca83..78b83ef3a3 100644 --- a/ydb/core/client/server/msgbus_server_persqueue.cpp +++ b/ydb/core/client/server/msgbus_server_persqueue.cpp @@ -14,9 +14,9 @@ namespace NKikimr { namespace NMsgBusProxy { -using namespace NSchemeCache; -using namespace NPqMetaCacheV2; - +using namespace NSchemeCache; +using namespace NPqMetaCacheV2; + const TDuration TPersQueueBaseRequestProcessor::TIMEOUT = TDuration::MilliSeconds(90000); namespace { @@ -32,104 +32,104 @@ const TString& TopicPrefix(const TActorContext& ctx) { return topicPrefix; } -TProcessingResult ProcessMetaCacheAllTopicsResponse(TEvPqNewMetaCache::TEvDescribeAllTopicsResponse::TPtr& ev) { - auto *res = ev->Get()->Result.Get(); - const TString& path = ev->Get()->Path; - TProcessingResult result; - if (!ev->Get()->Success) { - return TProcessingResult{ - MSTATUS_ERROR, - NPersQueue::NErrorCode::UNKNOWN_TOPIC, - Sprintf("path '%s' has invalid/unknown root prefix, Marker# PQ14", path.c_str()), - true - }; - } - if (!res) { - return TProcessingResult{ - MSTATUS_ERROR, - NPersQueue::NErrorCode::ERROR, - Sprintf("path '%s' describe error, Status# no status, reason: no reason, Marker# PQ1", path.c_str()), - true - }; - } - return {}; -} - -TProcessingResult ProcessMetaCacheSingleTopicsResponse( - const TSchemeCacheNavigate::TEntry& entry -) { - auto fullPath = JoinPath(entry.Path); - switch (entry.Status) { - case TSchemeCacheNavigate::EStatus::RootUnknown : { - return TProcessingResult { - MSTATUS_ERROR, - NPersQueue::NErrorCode::UNKNOWN_TOPIC, - Sprintf("path '%s' has unknown/invalid root prefix '%s', Marker# PQ14", - fullPath.c_str(), entry.Path[0].c_str()), - true - }; - } - case TSchemeCacheNavigate::EStatus::PathErrorUnknown: { - return TProcessingResult { - MSTATUS_ERROR, - NPersQueue::NErrorCode::UNKNOWN_TOPIC, - Sprintf("no path '%s', Marker# PQ150", fullPath.c_str()), - true - }; - } - case TSchemeCacheNavigate::EStatus::Ok: - break; - default: { - return TProcessingResult { - MSTATUS_ERROR, - NPersQueue::NErrorCode::ERROR, - Sprintf("topic '%s' describe error, Status# %s, Marker# PQ1", - fullPath.c_str(), ToString(entry.Status).c_str()), - true - }; - } - } - - if (entry.Kind != TSchemeCacheNavigate::KindTopic) { - return TProcessingResult { - MSTATUS_ERROR, - NPersQueue::NErrorCode::UNKNOWN_TOPIC, - Sprintf("item '%s' is not a topic, Marker# PQ13", fullPath.c_str()), - true - }; - } - if (!entry.PQGroupInfo) { - return TProcessingResult { - MSTATUS_ERROR, - NPersQueue::NErrorCode::UNKNOWN_TOPIC, - Sprintf("topic '%s' describe error, reason: could not retrieve topic description, Marker# PQ2", - fullPath.c_str()), - true - }; - } - auto topicName = entry.PQGroupInfo->Description.GetName(); - if (topicName.empty()) { - topicName = entry.Path.back(); - } - if (!entry.PQGroupInfo->Description.HasBalancerTabletID()) { - return TProcessingResult { - MSTATUS_ERROR, - NPersQueue::NErrorCode::UNKNOWN_TOPIC, - Sprintf("topic '%s' has no balancer, Marker# PQ193", topicName.c_str()), - true - }; - } - if (entry.PQGroupInfo->Description.GetBalancerTabletID() == 0) { - return TProcessingResult { - MSTATUS_ERROR, - NPersQueue::NErrorCode::UNKNOWN_TOPIC, - Sprintf("topic '%s' is not created, Marker# PQ94", topicName.c_str()), - true - }; - } - return {}; -} - +TProcessingResult ProcessMetaCacheAllTopicsResponse(TEvPqNewMetaCache::TEvDescribeAllTopicsResponse::TPtr& ev) { + auto *res = ev->Get()->Result.Get(); + const TString& path = ev->Get()->Path; + TProcessingResult result; + if (!ev->Get()->Success) { + return TProcessingResult{ + MSTATUS_ERROR, + NPersQueue::NErrorCode::UNKNOWN_TOPIC, + Sprintf("path '%s' has invalid/unknown root prefix, Marker# PQ14", path.c_str()), + true + }; + } + if (!res) { + return TProcessingResult{ + MSTATUS_ERROR, + NPersQueue::NErrorCode::ERROR, + Sprintf("path '%s' describe error, Status# no status, reason: no reason, Marker# PQ1", path.c_str()), + true + }; + } + return {}; +} + +TProcessingResult ProcessMetaCacheSingleTopicsResponse( + const TSchemeCacheNavigate::TEntry& entry +) { + auto fullPath = JoinPath(entry.Path); + switch (entry.Status) { + case TSchemeCacheNavigate::EStatus::RootUnknown : { + return TProcessingResult { + MSTATUS_ERROR, + NPersQueue::NErrorCode::UNKNOWN_TOPIC, + Sprintf("path '%s' has unknown/invalid root prefix '%s', Marker# PQ14", + fullPath.c_str(), entry.Path[0].c_str()), + true + }; + } + case TSchemeCacheNavigate::EStatus::PathErrorUnknown: { + return TProcessingResult { + MSTATUS_ERROR, + NPersQueue::NErrorCode::UNKNOWN_TOPIC, + Sprintf("no path '%s', Marker# PQ150", fullPath.c_str()), + true + }; + } + case TSchemeCacheNavigate::EStatus::Ok: + break; + default: { + return TProcessingResult { + MSTATUS_ERROR, + NPersQueue::NErrorCode::ERROR, + Sprintf("topic '%s' describe error, Status# %s, Marker# PQ1", + fullPath.c_str(), ToString(entry.Status).c_str()), + true + }; + } + } + + if (entry.Kind != TSchemeCacheNavigate::KindTopic) { + return TProcessingResult { + MSTATUS_ERROR, + NPersQueue::NErrorCode::UNKNOWN_TOPIC, + Sprintf("item '%s' is not a topic, Marker# PQ13", fullPath.c_str()), + true + }; + } + if (!entry.PQGroupInfo) { + return TProcessingResult { + MSTATUS_ERROR, + NPersQueue::NErrorCode::UNKNOWN_TOPIC, + Sprintf("topic '%s' describe error, reason: could not retrieve topic description, Marker# PQ2", + fullPath.c_str()), + true + }; + } + auto topicName = entry.PQGroupInfo->Description.GetName(); + if (topicName.empty()) { + topicName = entry.Path.back(); + } + if (!entry.PQGroupInfo->Description.HasBalancerTabletID()) { + return TProcessingResult { + MSTATUS_ERROR, + NPersQueue::NErrorCode::UNKNOWN_TOPIC, + Sprintf("topic '%s' has no balancer, Marker# PQ193", topicName.c_str()), + true + }; + } + if (entry.PQGroupInfo->Description.GetBalancerTabletID() == 0) { + return TProcessingResult { + MSTATUS_ERROR, + NPersQueue::NErrorCode::UNKNOWN_TOPIC, + Sprintf("topic '%s' is not created, Marker# PQ94", topicName.c_str()), + true + }; + } + return {}; +} + NKikimrClient::TResponse CreateErrorReply(EResponseStatus status, NPersQueue::NErrorCode::EErrorCode code, const TString& errorReason) { NKikimrClient::TResponse rec; rec.SetStatus(status); @@ -143,10 +143,10 @@ struct TTopicInfo { THashMap<ui32, ui64> PartitionToTablet; ui64 BalancerTabletId = 0; - THolder<NKikimrPQ::TReadSessionsInfoResponse> ReadSessionsInfo; + THolder<NKikimrPQ::TReadSessionsInfoResponse> ReadSessionsInfo; NKikimrPQ::TPQTabletConfig Config; - TIntrusiveConstPtr<TSchemeCacheNavigate::TPQGroupInfo> PQInfo; + TIntrusiveConstPtr<TSchemeCacheNavigate::TPQGroupInfo> PQInfo; ui32 NumParts = 0; THashSet<ui32> PartitionsToRequest; @@ -164,10 +164,10 @@ struct TTabletInfo { TVector<NKikimrPQ::TStatusResponse::TPartResult> StatusResponses; }; -TPersQueueBaseRequestProcessor::TPersQueueBaseRequestProcessor(const NKikimrClient::TPersQueueRequest& request, const TActorId& pqMetaCacheId, bool listNodes) +TPersQueueBaseRequestProcessor::TPersQueueBaseRequestProcessor(const NKikimrClient::TPersQueueRequest& request, const TActorId& pqMetaCacheId, bool listNodes) : RequestProto(new NKikimrClient::TPersQueueRequest(request)) , RequestId(RequestProto->HasRequestId() ? RequestProto->GetRequestId() : "<none>") - , PqMetaCache(pqMetaCacheId) + , PqMetaCache(pqMetaCacheId) , ListNodes(listNodes) { } @@ -192,7 +192,7 @@ void TPersQueueBaseRequestProcessor::Bootstrap(const TActorContext& ctx) { LOG_INFO_S(ctx, NKikimrServices::PERSQUEUE, "proxy got request " << RequestId); StartTimestamp = ctx.Now(); - ctx.Send(PqMetaCache, new NPqMetaCacheV2::TEvPqNewMetaCache::TEvDescribeAllTopicsRequest(TopicPrefix(ctx))); + ctx.Send(PqMetaCache, new NPqMetaCacheV2::TEvPqNewMetaCache::TEvDescribeAllTopicsRequest(TopicPrefix(ctx))); if (ListNodes) { const TActorId nameserviceId = GetNameserviceActorId(); @@ -213,8 +213,8 @@ void TPersQueueBaseRequestProcessor::Die(const TActorContext& ctx) { STFUNC(TPersQueueBaseRequestProcessor::StateFunc) { switch (ev->GetTypeRewrite()) { HFunc(TEvInterconnect::TEvNodesInfo, Handle); - HFunc(NPqMetaCacheV2::TEvPqNewMetaCache::TEvDescribeTopicsResponse, Handle); - HFunc(NPqMetaCacheV2::TEvPqNewMetaCache::TEvDescribeAllTopicsResponse, Handle); + HFunc(NPqMetaCacheV2::TEvPqNewMetaCache::TEvDescribeTopicsResponse, Handle); + HFunc(NPqMetaCacheV2::TEvPqNewMetaCache::TEvDescribeAllTopicsResponse, Handle); HFunc(TEvPersQueue::TEvResponse, Handle); CFunc(TEvents::TSystem::Wakeup, HandleTimeout); CFunc(NActors::TEvents::TSystem::PoisonPill, Die); @@ -284,31 +284,31 @@ void TPersQueueBaseRequestProcessor::GetTopicsListOrThrow(const ::google::protob throw std::runtime_error(desc); } } - + +} + +void TPersQueueBaseRequestProcessor::Handle( + NPqMetaCacheV2::TEvPqNewMetaCache::TEvDescribeTopicsResponse::TPtr&, const TActorContext& +) { + Y_FAIL(); } -void TPersQueueBaseRequestProcessor::Handle( - NPqMetaCacheV2::TEvPqNewMetaCache::TEvDescribeTopicsResponse::TPtr&, const TActorContext& -) { - Y_FAIL(); -} - -void TPersQueueBaseRequestProcessor::Handle( - NPqMetaCacheV2::TEvPqNewMetaCache::TEvDescribeAllTopicsResponse::TPtr& ev, const TActorContext& ctx -) { - auto& path = ev->Get()->Path; - if (!ev->Get()->Success) { +void TPersQueueBaseRequestProcessor::Handle( + NPqMetaCacheV2::TEvPqNewMetaCache::TEvDescribeAllTopicsResponse::TPtr& ev, const TActorContext& ctx +) { + auto& path = ev->Get()->Path; + if (!ev->Get()->Success) { return SendErrorReplyAndDie(ctx, MSTATUS_ERROR, NPersQueue::NErrorCode::UNKNOWN_TOPIC, TStringBuilder() << "no path '" << path << "', Marker# PQ17"); } - if (path != TopicPrefix(ctx)) { - return SendErrorReplyAndDie(ctx, MSTATUS_ERROR, NPersQueue::NErrorCode::UNKNOWN_TOPIC, - TStringBuilder() << "path '" << path << "' has no correct root prefix '" << TopicPrefix(ctx) + if (path != TopicPrefix(ctx)) { + return SendErrorReplyAndDie(ctx, MSTATUS_ERROR, NPersQueue::NErrorCode::UNKNOWN_TOPIC, + TStringBuilder() << "path '" << path << "' has no correct root prefix '" << TopicPrefix(ctx) << "', Marker# PQ18"); } - SchemeCacheResponse = std::move(ev->Get()->Result); + SchemeCacheResponse = std::move(ev->Get()->Result); if (ReadyToCreateChildren()) { if (CreateChildren(ctx)) { return; @@ -321,13 +321,13 @@ bool TPersQueueBaseRequestProcessor::ReadyToCreateChildren() const { } bool TPersQueueBaseRequestProcessor::CreateChildren(const TActorContext& ctx) { - for (const auto& child : SchemeCacheResponse->ResultSet) { - if (child.Kind == TSchemeCacheNavigate::EKind::KindTopic) { - TString name = child.PQGroupInfo->Description.GetName(); - if (name.empty()) { - name = child.Path.back(); - } - if (!TopicsToRequest.empty() && !IsIn(TopicsToRequest, name)) { + for (const auto& child : SchemeCacheResponse->ResultSet) { + if (child.Kind == TSchemeCacheNavigate::EKind::KindTopic) { + TString name = child.PQGroupInfo->Description.GetName(); + if (name.empty()) { + name = child.Path.back(); + } + if (!TopicsToRequest.empty() && !IsIn(TopicsToRequest, name)) { continue; } ChildrenToCreate.emplace_back(new TPerTopicInfo(child)); @@ -357,12 +357,12 @@ bool TPersQueueBaseRequestProcessor::CreateChildrenIfNeeded(const TActorContext& while (!ChildrenToCreate.empty()) { THolder<TPerTopicInfo> perTopicInfo(ChildrenToCreate.front().Release()); ChildrenToCreate.pop_front(); - const auto& name = perTopicInfo->TopicEntry.PQGroupInfo->Description.GetName(); - THolder<IActor> childActor = CreateTopicSubactor(perTopicInfo->TopicEntry, name); + const auto& name = perTopicInfo->TopicEntry.PQGroupInfo->Description.GetName(); + THolder<IActor> childActor = CreateTopicSubactor(perTopicInfo->TopicEntry, name); if (childActor.Get() != nullptr) { const TActorId actorId = ctx.Register(childActor.Release()); perTopicInfo->ActorId = actorId; - topics.emplace(name); + topics.emplace(name); Children.emplace(actorId, std::move(perTopicInfo)); } } @@ -409,20 +409,20 @@ TPersQueueBaseRequestProcessor::TNodesInfo::TNodesInfo(THolder<TEvInterconnect:: } } -TTopicInfoBasedActor::TTopicInfoBasedActor(const TSchemeEntry& topicEntry, const TString& topicName) - : TActorBootstrapped<TTopicInfoBasedActor>() - , SchemeEntry(topicEntry) - , Name(topicName) - , ProcessingResult(ProcessMetaCacheSingleTopicsResponse(SchemeEntry)) +TTopicInfoBasedActor::TTopicInfoBasedActor(const TSchemeEntry& topicEntry, const TString& topicName) + : TActorBootstrapped<TTopicInfoBasedActor>() + , SchemeEntry(topicEntry) + , Name(topicName) + , ProcessingResult(ProcessMetaCacheSingleTopicsResponse(SchemeEntry)) { } -void TTopicInfoBasedActor::Bootstrap(const TActorContext &ctx) { - Become(&TTopicInfoBasedActor::StateFunc); - BootstrapImpl(ctx); +void TTopicInfoBasedActor::Bootstrap(const TActorContext &ctx) { + Become(&TTopicInfoBasedActor::StateFunc); + BootstrapImpl(ctx); } -STFUNC(TTopicInfoBasedActor::StateFunc) { +STFUNC(TTopicInfoBasedActor::StateFunc) { switch (ev->GetTypeRewrite()) { CFunc(NActors::TEvents::TSystem::PoisonPill, Die); default: @@ -431,15 +431,15 @@ STFUNC(TTopicInfoBasedActor::StateFunc) { } -class TMessageBusServerPersQueueImpl : public TActorBootstrapped<TMessageBusServerPersQueueImpl> { - using TEvAllTopicsDescribeRequest = NMsgBusProxy::NPqMetaCacheV2::TEvPqNewMetaCache::TEvDescribeAllTopicsRequest; - using TEvAllTopicsDescribeResponse = NMsgBusProxy::NPqMetaCacheV2::TEvPqNewMetaCache::TEvDescribeAllTopicsResponse; +class TMessageBusServerPersQueueImpl : public TActorBootstrapped<TMessageBusServerPersQueueImpl> { + using TEvAllTopicsDescribeRequest = NMsgBusProxy::NPqMetaCacheV2::TEvPqNewMetaCache::TEvDescribeAllTopicsRequest; + using TEvAllTopicsDescribeResponse = NMsgBusProxy::NPqMetaCacheV2::TEvPqNewMetaCache::TEvDescribeAllTopicsResponse; protected: NKikimrClient::TPersQueueRequest RequestProto; const TString RequestId; const bool IsMetaRequest; - const bool IsFetchRequest; + const bool IsFetchRequest; bool CanProcessFetchRequest; //any partitions answered that it has data or WaitMs timeout occured ui32 FetchRequestReadsDone; @@ -738,10 +738,10 @@ public: THashMap<ui32, ui32> partitionToResp; ui32 sz = 0; - Y_VERIFY(p.second.ReadSessionsInfo); - auto* sessionsInfo = p.second.ReadSessionsInfo.Get(); - for (ui32 i = 0; i < sessionsInfo->PartitionInfoSize(); ++i) { - const auto& resp = sessionsInfo->GetPartitionInfo(i); + Y_VERIFY(p.second.ReadSessionsInfo); + auto* sessionsInfo = p.second.ReadSessionsInfo.Get(); + for (ui32 i = 0; i < sessionsInfo->PartitionInfoSize(); ++i) { + const auto& resp = sessionsInfo->GetPartitionInfo(i); partitionToResp[resp.GetPartition()] = sz++; auto res = topicRes->AddPartitionResult(); res->SetPartition(resp.GetPartition()); @@ -846,7 +846,7 @@ public: auto jt = TopicInfo.find(it->second.Topic); Y_VERIFY(jt != TopicInfo.end()); - jt->second.ReadSessionsInfo = MakeHolder<NKikimrPQ::TReadSessionsInfoResponse>(std::move(response)); + jt->second.ReadSessionsInfo = MakeHolder<NKikimrPQ::TReadSessionsInfoResponse>(std::move(response)); AnswerIfCanForMeta(ctx); } @@ -862,7 +862,7 @@ public: it->second.StatusResponses.push_back(response.GetPartResult(i)); } TabletsAnswered.insert(it->first); - + AnswerIfCanForMeta(ctx); } @@ -872,37 +872,37 @@ public: } - void Handle(TEvAllTopicsDescribeResponse::TPtr& ev, const TActorContext& ctx) { + void Handle(TEvAllTopicsDescribeResponse::TPtr& ev, const TActorContext& ctx) { --DescribeRequests; - auto* res = ev->Get()->Result.Get(); - auto processResult = ProcessMetaCacheAllTopicsResponse(ev); - if (processResult.IsFatal) { - ErrorReason = processResult.Reason; - return SendReplyAndDie(CreateErrorReply(processResult.Status, processResult.ErrorCode, ctx), ctx); + auto* res = ev->Get()->Result.Get(); + auto processResult = ProcessMetaCacheAllTopicsResponse(ev); + if (processResult.IsFatal) { + ErrorReason = processResult.Reason; + return SendReplyAndDie(CreateErrorReply(processResult.Status, processResult.ErrorCode, ctx), ctx); } NoTopicsAtStart = TopicInfo.empty(); bool hasTopics = !NoTopicsAtStart; - for (const auto& entry : res->ResultSet) { - if (entry.Kind == TSchemeCacheNavigate::EKind::KindTopic && entry.PQGroupInfo) { - auto& description = entry.PQGroupInfo->Description; - if (!hasTopics || TopicInfo.find(description.GetName()) != TopicInfo.end()) { - auto& topicInfo = TopicInfo[description.GetName()]; - topicInfo.BalancerTabletId = description.GetBalancerTabletID(); - topicInfo.PQInfo = entry.PQGroupInfo; + for (const auto& entry : res->ResultSet) { + if (entry.Kind == TSchemeCacheNavigate::EKind::KindTopic && entry.PQGroupInfo) { + auto& description = entry.PQGroupInfo->Description; + if (!hasTopics || TopicInfo.find(description.GetName()) != TopicInfo.end()) { + auto& topicInfo = TopicInfo[description.GetName()]; + topicInfo.BalancerTabletId = description.GetBalancerTabletID(); + topicInfo.PQInfo = entry.PQGroupInfo; } } } - for (auto& p: TopicInfo) { + for (auto& p: TopicInfo) { const TString& topic = p.first; if (!p.second.BalancerTabletId) { ErrorReason = Sprintf("topic '%s' is not created, Marker# PQ94", topic.c_str()); return SendReplyAndDie(CreateErrorReply(MSTATUS_ERROR, NPersQueue::NErrorCode::UNKNOWN_TOPIC, ctx), ctx); } - ProcessMetadata(p.first, p.second, ctx); + ProcessMetadata(p.first, p.second, ctx); } if (RequestProto.HasMetaRequest()) { @@ -910,22 +910,22 @@ public: } } - void ProcessMetadata(const TString& name, TTopicInfo& info, const TActorContext& ctx) { - //const TString& name = info.PQInfo->Description.GetName(); - if (!info.PQInfo) { //not supposed to happen anymore + void ProcessMetadata(const TString& name, TTopicInfo& info, const TActorContext& ctx) { + //const TString& name = info.PQInfo->Description.GetName(); + if (!info.PQInfo) { //not supposed to happen anymore if (RequestProto.HasMetaRequest() && NoTopicsAtStart && !RequestProto.GetMetaRequest().HasCmdGetTopicMetadata()) { ++TopicsAnswered; AnswerIfCanForMeta(ctx); } else { - ErrorReason = Sprintf("topic '%s' is not ready, Marker# PQ85", name.c_str()); - SendReplyAndDie(CreateErrorReply(MSTATUS_ERROR, NPersQueue::NErrorCode::UNKNOWN_TOPIC, ctx), ctx); + ErrorReason = Sprintf("topic '%s' is not ready, Marker# PQ85", name.c_str()); + SendReplyAndDie(CreateErrorReply(MSTATUS_ERROR, NPersQueue::NErrorCode::UNKNOWN_TOPIC, ctx), ctx); } return; } - const auto& pqDescr = info.PQInfo->Description; - const TString& topic = pqDescr.GetName(); - Y_VERIFY(topic == name, "topic '%s' path '%s'", topic.c_str(), name.c_str()); + const auto& pqDescr = info.PQInfo->Description; + const TString& topic = pqDescr.GetName(); + Y_VERIFY(topic == name, "topic '%s' path '%s'", topic.c_str(), name.c_str()); bool mirrorerRequest = false; if (RequestProto.HasPartitionRequest()) { @@ -960,10 +960,10 @@ public: } if (AppData(ctx)->PQConfig.GetCheckACL() && operation && !mirrorerRequest) { - if (*operation == NKikimrPQ::EOperation::WRITE_OP && pqDescr.GetPQTabletConfig().GetRequireAuthWrite() || - *operation == NKikimrPQ::EOperation::READ_OP && pqDescr.GetPQTabletConfig().GetRequireAuthRead()) { + if (*operation == NKikimrPQ::EOperation::WRITE_OP && pqDescr.GetPQTabletConfig().GetRequireAuthWrite() || + *operation == NKikimrPQ::EOperation::READ_OP && pqDescr.GetPQTabletConfig().GetRequireAuthRead()) { ErrorReason = Sprintf("unauthenticated access to '%s' is denied, Marker# PQ419", topic.c_str()); - return SendReplyAndDie(CreateErrorReply(MSTATUS_ERROR, NPersQueue::NErrorCode::ACCESS_DENIED, ctx), ctx); + return SendReplyAndDie(CreateErrorReply(MSTATUS_ERROR, NPersQueue::NErrorCode::ACCESS_DENIED, ctx), ctx); } } @@ -973,9 +973,9 @@ public: Y_VERIFY(it != TopicInfo.end()); Y_VERIFY(it->second.PartitionsToRequest.size() == 1); ui32 partition = *(it->second.PartitionsToRequest.begin()); - for (ui32 i = 0; i < pqDescr.PartitionsSize(); ++i) { - const auto& pi = pqDescr.GetPartitions(i); - if (pi.GetPartitionId() == partition) { + for (ui32 i = 0; i < pqDescr.PartitionsSize(); ++i) { + const auto& pi = pqDescr.GetPartitions(i); + if (pi.GetPartitionId() == partition) { tabletId = pi.GetTabletId(); break; } @@ -1011,9 +1011,9 @@ public: ++TopicsAnswered; auto it = TopicInfo.find(topic); Y_VERIFY(it != TopicInfo.end(), "topic '%s'", topic.c_str()); - it->second.Config = pqDescr.GetPQTabletConfig(); - it->second.Config.SetVersion(pqDescr.GetAlterVersion()); - it->second.NumParts = pqDescr.PartitionsSize(); + it->second.Config = pqDescr.GetPQTabletConfig(); + it->second.Config.SetVersion(pqDescr.GetAlterVersion()); + it->second.NumParts = pqDescr.PartitionsSize(); if (metadataOnly) { AnswerIfCanForMeta(ctx); return; @@ -1041,9 +1041,9 @@ public: NTabletPipe::SendData(ctx, pipeClient, ev.Release()); } - for (ui32 i = 0; i < pqDescr.PartitionsSize(); ++i) { - ui32 part = pqDescr.GetPartitions(i).GetPartitionId(); - ui64 tabletId = pqDescr.GetPartitions(i).GetTabletId(); + for (ui32 i = 0; i < pqDescr.PartitionsSize(); ++i) { + ui32 part = pqDescr.GetPartitions(i).GetPartitionId(); + ui64 tabletId = pqDescr.GetPartitions(i).GetTabletId(); if (!it->second.PartitionsToRequest.empty() && !it->second.PartitionsToRequest.contains(part)) { continue; } @@ -1152,14 +1152,14 @@ public: ui32 nodeId = ev->Get()->ServerId.NodeId(); it->second.NodeId = nodeId; TabletsDiscovered.insert(tabletId); - + AnswerIfCanForMeta(ctx); } } } void Handle(TEvTabletPipe::TEvClientDestroyed::TPtr& ev, const TActorContext& ctx) { - + ui64 tabletId = ev->Get()->TabletId; if (HandlePipeError(tabletId, ctx)) return; @@ -1310,8 +1310,8 @@ public: ctx.Schedule(TDuration::MilliSeconds(Min<ui32>(RequestProto.GetFetchRequest().GetWaitMs(), 30000)), new TEvPersQueue::TEvHasDataInfoResponse); } - auto* request = new TEvAllTopicsDescribeRequest(TopicPrefix(ctx)); - ctx.Send(SchemeCache, request); + auto* request = new TEvAllTopicsDescribeRequest(TopicPrefix(ctx)); + ctx.Send(SchemeCache, request); ++DescribeRequests; if (RequestProto.HasMetaRequest() && (RequestProto.GetMetaRequest().HasCmdGetPartitionLocations() @@ -1324,9 +1324,9 @@ public: Become(&TMessageBusServerPersQueueImpl::StateFunc, ctx, TDuration::MilliSeconds(DefaultTimeout), new TEvents::TEvWakeup()); } - STRICT_STFUNC(StateFunc, + STRICT_STFUNC(StateFunc, HFunc(TEvInterconnect::TEvNodesInfo, Handle); - HFunc(TEvAllTopicsDescribeResponse, Handle); + HFunc(TEvAllTopicsDescribeResponse, Handle); HFunc(TEvTabletPipe::TEvClientDestroyed, Handle); HFunc(TEvTabletPipe::TEvClientConnected, Handle); HFunc(TEvPersQueue::TEvResponse, Handle); @@ -1336,7 +1336,7 @@ public: HFunc(TEvPersQueue::TEvReadSessionsInfoResponse, Handle); CFunc(TEvents::TSystem::Wakeup, HandleTimeout); CFunc(NActors::TEvents::TSystem::PoisonPill, Die); - ) + ) private: bool GetTopicsList(const ::google::protobuf::RepeatedPtrField<::NKikimrClient::TPersQueueMetaRequest::TTopicRequest>& requests) { for (auto ri = requests.begin(); ri != requests.end(); ++ri) { @@ -1355,12 +1355,12 @@ private: return false; } } - const auto& topic = topicRequest.GetTopic(); - if (TopicInfo.contains(topic)) { - ErrorReason = Sprintf("multiple TopicRequest for topic '%s'", topic.c_str()); + const auto& topic = topicRequest.GetTopic(); + if (TopicInfo.contains(topic)) { + ErrorReason = Sprintf("multiple TopicRequest for topic '%s'", topic.c_str()); return false; - } else { - TopicInfo[topic] = std::move(topicInfo); + } else { + TopicInfo[topic] = std::move(topicInfo); } } return true; diff --git a/ydb/core/client/server/msgbus_server_persqueue.h b/ydb/core/client/server/msgbus_server_persqueue.h index b74e51712e..52c67832e4 100644 --- a/ydb/core/client/server/msgbus_server_persqueue.h +++ b/ydb/core/client/server/msgbus_server_persqueue.h @@ -17,16 +17,16 @@ namespace NMsgBusProxy { const TString& TopicPrefix(const TActorContext& ctx); -struct TProcessingResult { - EResponseStatus Status = MSTATUS_OK; - NPersQueue::NErrorCode::EErrorCode ErrorCode; - TString Reason; - bool IsFatal = false; -}; - -TProcessingResult ProcessMetaCacheAllTopicsResponse(NPqMetaCacheV2::TEvPqNewMetaCache::TEvDescribeAllTopicsResponse::TPtr& response); -TProcessingResult ProcessMetaCacheSingleTopicsResponse(const NSchemeCache::TSchemeCacheNavigate::TEntry& entry); - +struct TProcessingResult { + EResponseStatus Status = MSTATUS_OK; + NPersQueue::NErrorCode::EErrorCode ErrorCode; + TString Reason; + bool IsFatal = false; +}; + +TProcessingResult ProcessMetaCacheAllTopicsResponse(NPqMetaCacheV2::TEvPqNewMetaCache::TEvDescribeAllTopicsResponse::TPtr& response); +TProcessingResult ProcessMetaCacheSingleTopicsResponse(const NSchemeCache::TSchemeCacheNavigate::TEntry& entry); + // Worker actor creation IActor* CreateMessageBusServerPersQueue( TBusMessageContext& msg, @@ -60,20 +60,20 @@ inline ui64 GetTabletId<TEvTabletPipe::TEvClientConnected>(const TEvTabletPipe:: // 2. Implement CreateTopicSubactor() and, optionally, MergeSubactorReplies() methods. class TPersQueueBaseRequestProcessor : public TActorBootstrapped<TPersQueueBaseRequestProcessor> { protected: - using TSchemeEntry = NSchemeCache::TSchemeCacheNavigate::TEntry; - using TPQGroupInfoPtr = TIntrusiveConstPtr<NSchemeCache::TSchemeCacheNavigate::TPQGroupInfo>; - using ESchemeStatus = NSchemeCache::TSchemeCacheNavigate::EStatus; - + using TSchemeEntry = NSchemeCache::TSchemeCacheNavigate::TEntry; + using TPQGroupInfoPtr = TIntrusiveConstPtr<NSchemeCache::TSchemeCacheNavigate::TPQGroupInfo>; + using ESchemeStatus = NSchemeCache::TSchemeCacheNavigate::EStatus; + struct TPerTopicInfo { TPerTopicInfo() { } - explicit TPerTopicInfo(const TSchemeEntry& topicEntry) + explicit TPerTopicInfo(const TSchemeEntry& topicEntry) : TopicEntry(topicEntry) { } TActorId ActorId; - TSchemeEntry TopicEntry; + TSchemeEntry TopicEntry; NKikimrClient::TResponse Response; bool ActorAnswered = false; }; @@ -101,7 +101,7 @@ public: } protected: - TPersQueueBaseRequestProcessor(const NKikimrClient::TPersQueueRequest& request, const TActorId& pqMetaCacheId, bool listNodes); + TPersQueueBaseRequestProcessor(const NKikimrClient::TPersQueueRequest& request, const TActorId& pqMetaCacheId, bool listNodes); ~TPersQueueBaseRequestProcessor(); @@ -111,7 +111,7 @@ public: protected: bool CreateChildrenIfNeeded(const TActorContext& ctx); - virtual THolder<IActor> CreateTopicSubactor(const TSchemeEntry& topicEntry, const TString& name) = 0; // Creates actor for processing one concrete topic. + virtual THolder<IActor> CreateTopicSubactor(const TSchemeEntry& topicEntry, const TString& name) = 0; // Creates actor for processing one concrete topic. virtual NKikimrClient::TResponse MergeSubactorReplies(); virtual void SendReplyAndDie(NKikimrClient::TResponse&& record, const TActorContext& ctx) = 0; @@ -129,8 +129,8 @@ protected: virtual STFUNC(StateFunc); void Handle(TEvInterconnect::TEvNodesInfo::TPtr& ev, const TActorContext& ctx); - void Handle(NPqMetaCacheV2::TEvPqNewMetaCache::TEvDescribeTopicsResponse::TPtr& ev, const TActorContext& ctx); - void Handle(NPqMetaCacheV2::TEvPqNewMetaCache::TEvDescribeAllTopicsResponse::TPtr& ev, const TActorContext& ctx); + void Handle(NPqMetaCacheV2::TEvPqNewMetaCache::TEvDescribeTopicsResponse::TPtr& ev, const TActorContext& ctx); + void Handle(NPqMetaCacheV2::TEvPqNewMetaCache::TEvDescribeAllTopicsResponse::TPtr& ev, const TActorContext& ctx); void Handle(TEvPersQueue::TEvResponse::TPtr& ev, const TActorContext& ctx); void HandleTimeout(const TActorContext& ctx); @@ -142,10 +142,10 @@ protected: const TString RequestId; THashSet<TString> TopicsToRequest; // Topics that we need to request. If this set id empty, we are interested in all existing topics. - const TActorId PqMetaCache; + const TActorId PqMetaCache; THashMap<TActorId, THolder<TPerTopicInfo>> Children; size_t ChildrenAnswered = 0; - THolder<NSchemeCache::TSchemeCacheNavigate> SchemeCacheResponse; + THolder<NSchemeCache::TSchemeCacheNavigate> SchemeCacheResponse; // Nodes info const bool ListNodes; @@ -153,15 +153,15 @@ protected: }; // Helper actor that sends TEvGetBalancerDescribe and checks ACL (ACL is not implemented yet). -class TTopicInfoBasedActor : public TActorBootstrapped<TTopicInfoBasedActor> { +class TTopicInfoBasedActor : public TActorBootstrapped<TTopicInfoBasedActor> { protected: - using TSchemeEntry = NSchemeCache::TSchemeCacheNavigate::TEntry; - using TPQGroupInfoPtr = TIntrusiveConstPtr<NSchemeCache::TSchemeCacheNavigate::TPQGroupInfo>; - using ESchemeStatus = NSchemeCache::TSchemeCacheNavigate::EStatus; + using TSchemeEntry = NSchemeCache::TSchemeCacheNavigate::TEntry; + using TPQGroupInfoPtr = TIntrusiveConstPtr<NSchemeCache::TSchemeCacheNavigate::TPQGroupInfo>; + using ESchemeStatus = NSchemeCache::TSchemeCacheNavigate::EStatus; - TTopicInfoBasedActor(const TSchemeEntry& topicEntry, const TString& topicName); + TTopicInfoBasedActor(const TSchemeEntry& topicEntry, const TString& topicName); - virtual void BootstrapImpl(const TActorContext& ctx) = 0; + virtual void BootstrapImpl(const TActorContext& ctx) = 0; virtual void Answer(const TActorContext& ctx, EResponseStatus status, NPersQueue::NErrorCode::EErrorCode code, const TString& errorReason) = 0; virtual void SendReplyAndDie(NKikimrClient::TResponse&& record, const TActorContext& ctx) = 0; @@ -170,11 +170,11 @@ protected: protected: TActorId SchemeCache; - TSchemeEntry SchemeEntry; - TString Name; - TProcessingResult ProcessingResult; -public: - void Bootstrap(const TActorContext& ctx); + TSchemeEntry SchemeEntry; + TString Name; + TProcessingResult ProcessingResult; +public: + void Bootstrap(const TActorContext& ctx); static constexpr NKikimrServices::TActivity::EType ActorActivityType() { return NKikimrServices::TActivity::PQ_BASE_REQUEST_PROCESSOR; diff --git a/ydb/core/client/server/msgbus_server_pq_metacache.cpp b/ydb/core/client/server/msgbus_server_pq_metacache.cpp index ae3ac12f10..f44ab6f480 100644 --- a/ydb/core/client/server/msgbus_server_pq_metacache.cpp +++ b/ydb/core/client/server/msgbus_server_pq_metacache.cpp @@ -1,5 +1,5 @@ #include "msgbus_server_persqueue.h" -#include "msgbus_server_pq_metacache.h" +#include "msgbus_server_pq_metacache.h" #include <ydb/public/api/protos/draft/persqueue_error_codes.pb.h> #include <ydb/library/persqueue/topic_parser/topic_parser.h> @@ -7,89 +7,89 @@ #include <ydb/core/tx/schemeshard/schemeshard.h> #include <ydb/core/tx/scheme_board/cache.h> #include <ydb/core/base/counters.h> - + #include <ydb/core/base/appdata.h> -namespace NKikimr::NMsgBusProxy { +namespace NKikimr::NMsgBusProxy { -using namespace NYdb::NTable; +using namespace NYdb::NTable; -namespace NPqMetaCacheV2 { +namespace NPqMetaCacheV2 { -IActor* CreateSchemeCache(NActors::TActorSystem* ActorSystem, TIntrusivePtr<NMonitoring::TDynamicCounters> counters) { - auto appData = ActorSystem->AppData<TAppData>(); +IActor* CreateSchemeCache(NActors::TActorSystem* ActorSystem, TIntrusivePtr<NMonitoring::TDynamicCounters> counters) { + auto appData = ActorSystem->AppData<TAppData>(); auto cacheCounters = GetServiceCounters(counters, "pqproxy|schemecache"); auto cacheConfig = MakeIntrusive<NSchemeCache::TSchemeCacheConfig>(appData, cacheCounters); - return CreateSchemeBoardSchemeCache(cacheConfig.Get()); -} + return CreateSchemeBoardSchemeCache(cacheConfig.Get()); +} -class TPersQueueMetaCacheActor : public TActorBootstrapped<TPersQueueMetaCacheActor> { - using TBase = TActorBootstrapped<TPersQueueMetaCacheActor>; -public: +class TPersQueueMetaCacheActor : public TActorBootstrapped<TPersQueueMetaCacheActor> { + using TBase = TActorBootstrapped<TPersQueueMetaCacheActor>; +public: TPersQueueMetaCacheActor(TPersQueueMetaCacheActor&&) = default; TPersQueueMetaCacheActor& operator=(TPersQueueMetaCacheActor&&) = default; TPersQueueMetaCacheActor(ui64 grpcPort, const NMonitoring::TDynamicCounterPtr& counters, const TDuration& versionCheckInterval) - : Counters(counters) + : Counters(counters) , ClientWrapper(std::move(std::make_unique<TClientWrapper>(grpcPort))) - , VersionCheckInterval(versionCheckInterval) + , VersionCheckInterval(versionCheckInterval) , Generation(std::make_shared<TAtomicCounter>()) - { - } - + { + } + TPersQueueMetaCacheActor(const NMonitoring::TDynamicCounterPtr& counters, const TDuration& versionCheckInterval) - : Counters(counters) - , VersionCheckInterval(versionCheckInterval) + : Counters(counters) + , VersionCheckInterval(versionCheckInterval) , Generation(std::make_shared<TAtomicCounter>()) { } - void Bootstrap(const TActorContext& ctx) { - if (ClientWrapper == nullptr) { + void Bootstrap(const TActorContext& ctx) { + if (ClientWrapper == nullptr) { auto* driver = AppData(ctx)->YdbDriver; - if (driver == nullptr) { - LOG_WARN_S( - ctx, NKikimrServices::PQ_METACACHE, - "Initialized without valid YDB driver - suppose misconfiguration. Refuse to work. Die." - ); - Die(ctx); - return; - } + if (driver == nullptr) { + LOG_WARN_S( + ctx, NKikimrServices::PQ_METACACHE, + "Initialized without valid YDB driver - suppose misconfiguration. Refuse to work. Die." + ); + Die(ctx); + return; + } ClientWrapper.reset(new TClientWrapper(driver)); - } - SkipVersionCheck = !AppData(ctx)->PQConfig.GetMetaCacheSkipVersionCheck(); - PathPrefix = TopicPrefix(ctx); - TopicsQuery = TStringBuilder() << "--!syntax_v1\n" - << "DECLARE $Path as Utf8; DECLARE $Cluster as Utf8; " - << "SELECT path, dc from `" << PathPrefix << "Config/V2/Topics` " - << "WHERE path > $Path OR (path = $Path AND dc > $Cluster);"; - - VersionQuery = TStringBuilder() << "--!syntax_v1\nSELECT version FROM `" << PathPrefix << "Config/V2/Versions` " - << "WHERE name = 'Topics';"; - PathPrefixParts = NKikimr::SplitPath(PathPrefix); - Become(&TPersQueueMetaCacheActor::StateFunc); - - if (!SchemeCacheId) { - SchemeCacheId = Register(CreateSchemeCache(GetActorSystem(), Counters)); - } - ActorSystem = GetActorSystem(); - Reset(ctx); - } - - ~TPersQueueMetaCacheActor() { - Generation->Inc(); - if (ClientWrapper) - ClientWrapper->Stop(); - } - -private: + } + SkipVersionCheck = !AppData(ctx)->PQConfig.GetMetaCacheSkipVersionCheck(); + PathPrefix = TopicPrefix(ctx); + TopicsQuery = TStringBuilder() << "--!syntax_v1\n" + << "DECLARE $Path as Utf8; DECLARE $Cluster as Utf8; " + << "SELECT path, dc from `" << PathPrefix << "Config/V2/Topics` " + << "WHERE path > $Path OR (path = $Path AND dc > $Cluster);"; + + VersionQuery = TStringBuilder() << "--!syntax_v1\nSELECT version FROM `" << PathPrefix << "Config/V2/Versions` " + << "WHERE name = 'Topics';"; + PathPrefixParts = NKikimr::SplitPath(PathPrefix); + Become(&TPersQueueMetaCacheActor::StateFunc); + + if (!SchemeCacheId) { + SchemeCacheId = Register(CreateSchemeCache(GetActorSystem(), Counters)); + } + ActorSystem = GetActorSystem(); + Reset(ctx); + } + + ~TPersQueueMetaCacheActor() { + Generation->Inc(); + if (ClientWrapper) + ClientWrapper->Stop(); + } + +private: template<class TEventType, class TFutureType, class... TArgs> void SubscribeEvent(const TFutureType& future, TArgs... args) { std::weak_ptr<TAtomicCounter> weakGeneration(Generation); - future.Subscribe( + future.Subscribe( [ id = SelfId(), originalGen = Generation->Val(), @@ -100,235 +100,235 @@ private: auto currentGen = weakGen.lock(); if (currentGen && originalGen == currentGen->Val()) { as->Send(id, new TEventType(args...)); - } + } } - ); - } - - void Reset(const TActorContext& ctx) { - LOG_DEBUG_S(ctx, NKikimrServices::PQ_METACACHE, "Metacache: reset"); - Y_VERIFY(ClientWrapper); - ClientWrapper->Reset(); - Generation->Inc(); - YdbSession = Nothing(); - PreparedTopicsQuery = Nothing(); - LastTopicKey = {}; - ctx.Schedule(QueryRetryInterval, new NActors::TEvents::TEvWakeup()); - } - - void StartSession(const TActorContext& ctx) { - LOG_DEBUG_S(ctx, NKikimrServices::PQ_METACACHE, "Start new session"); - SessionFuture = ClientWrapper->GetClient()->GetSession(); + ); + } + + void Reset(const TActorContext& ctx) { + LOG_DEBUG_S(ctx, NKikimrServices::PQ_METACACHE, "Metacache: reset"); + Y_VERIFY(ClientWrapper); + ClientWrapper->Reset(); + Generation->Inc(); + YdbSession = Nothing(); + PreparedTopicsQuery = Nothing(); + LastTopicKey = {}; + ctx.Schedule(QueryRetryInterval, new NActors::TEvents::TEvWakeup()); + } + + void StartSession(const TActorContext& ctx) { + LOG_DEBUG_S(ctx, NKikimrServices::PQ_METACACHE, "Start new session"); + SessionFuture = ClientWrapper->GetClient()->GetSession(); SubscribeEvent<TEvPqNewMetaCache::TEvSessionStarted>(SessionFuture); - } - - void HandleSessionStarted(const TActorContext& ctx) { - auto& value = SessionFuture.GetValue(); - if (!value.IsSuccess()) { - LOG_ERROR_S(ctx, NKikimrServices::PQ_METACACHE, "Session start failed: " << value.GetIssues().ToString()); - return Reset(ctx); - } - LOG_DEBUG_S(ctx, NKikimrServices::PQ_METACACHE, "Session started"); - YdbSession = value.GetSession(); - PrepareTopicsQuery(ctx); - } - - void PrepareTopicsQuery(const TActorContext& ctx) { - Y_UNUSED(ctx); - TopicsQueryFuture = YdbSession->PrepareDataQuery(TopicsQuery); + } + + void HandleSessionStarted(const TActorContext& ctx) { + auto& value = SessionFuture.GetValue(); + if (!value.IsSuccess()) { + LOG_ERROR_S(ctx, NKikimrServices::PQ_METACACHE, "Session start failed: " << value.GetIssues().ToString()); + return Reset(ctx); + } + LOG_DEBUG_S(ctx, NKikimrServices::PQ_METACACHE, "Session started"); + YdbSession = value.GetSession(); + PrepareTopicsQuery(ctx); + } + + void PrepareTopicsQuery(const TActorContext& ctx) { + Y_UNUSED(ctx); + TopicsQueryFuture = YdbSession->PrepareDataQuery(TopicsQuery); SubscribeEvent<TEvPqNewMetaCache::TEvQueryPrepared>(TopicsQueryFuture); - } - - void HandleTopicsQueryPrepared(const TActorContext& ctx) { - auto& value = TopicsQueryFuture.GetValue(); - if (!value.IsSuccess()) { - LOG_DEBUG_S(ctx, NKikimrServices::PQ_METACACHE, "Topics query prepare failed: " << value.GetIssues().ToString()); - return Reset(ctx); - } - PreparedTopicsQuery = value.GetQuery(); - if (NewTopicsVersion > CurrentTopicsVersion) { - LOG_DEBUG_S(ctx, NKikimrServices::PQ_METACACHE, "Start topics rescan"); - RunQuery(EQueryType::EGetTopics, ctx); - } else { - Y_VERIFY(NewTopicsVersion == CurrentTopicsVersion); - LOG_DEBUG_S(ctx, NKikimrServices::PQ_METACACHE, "Check version"); - RunQuery(EQueryType::ECheckVersion, ctx); - } - } - - void RunQuery(EQueryType type, const TActorContext& ctx) { - Y_UNUSED(ctx); - Y_VERIFY(YdbSession); - if (type == EQueryType::ECheckVersion) { - AsyncQueryResult = YdbSession->ExecuteDataQuery(VersionQuery, TTxControl::BeginTx().CommitTx()); - } else { - Y_VERIFY(PreparedTopicsQuery); - auto builder = PreparedTopicsQuery->GetParamsBuilder(); - { - auto& param = builder.AddParam("$Path"); - param.Utf8(LastTopicKey.Path); - param.Build(); - } - { - auto& param = builder.AddParam("$Cluster"); - param.Utf8(LastTopicKey.Cluster); - param.Build(); - } - AsyncQueryResult = PreparedTopicsQuery->Execute(TTxControl::BeginTx().CommitTx(), builder.Build()); - } + } + + void HandleTopicsQueryPrepared(const TActorContext& ctx) { + auto& value = TopicsQueryFuture.GetValue(); + if (!value.IsSuccess()) { + LOG_DEBUG_S(ctx, NKikimrServices::PQ_METACACHE, "Topics query prepare failed: " << value.GetIssues().ToString()); + return Reset(ctx); + } + PreparedTopicsQuery = value.GetQuery(); + if (NewTopicsVersion > CurrentTopicsVersion) { + LOG_DEBUG_S(ctx, NKikimrServices::PQ_METACACHE, "Start topics rescan"); + RunQuery(EQueryType::EGetTopics, ctx); + } else { + Y_VERIFY(NewTopicsVersion == CurrentTopicsVersion); + LOG_DEBUG_S(ctx, NKikimrServices::PQ_METACACHE, "Check version"); + RunQuery(EQueryType::ECheckVersion, ctx); + } + } + + void RunQuery(EQueryType type, const TActorContext& ctx) { + Y_UNUSED(ctx); + Y_VERIFY(YdbSession); + if (type == EQueryType::ECheckVersion) { + AsyncQueryResult = YdbSession->ExecuteDataQuery(VersionQuery, TTxControl::BeginTx().CommitTx()); + } else { + Y_VERIFY(PreparedTopicsQuery); + auto builder = PreparedTopicsQuery->GetParamsBuilder(); + { + auto& param = builder.AddParam("$Path"); + param.Utf8(LastTopicKey.Path); + param.Build(); + } + { + auto& param = builder.AddParam("$Cluster"); + param.Utf8(LastTopicKey.Cluster); + param.Build(); + } + AsyncQueryResult = PreparedTopicsQuery->Execute(TTxControl::BeginTx().CommitTx(), builder.Build()); + } SubscribeEvent<TEvPqNewMetaCache::TEvQueryComplete>(AsyncQueryResult, type); - } - - void HandleCheckVersionResult(const TActorContext& ctx) { - auto result = AsyncQueryResult.GetValue(); - if (!result.IsSuccess()) { - LOG_ERROR_S(ctx, NKikimrServices::PQ_METACACHE, - "Got error trying to update config version: " << result.GetIssues().ToString()); - ctx.Schedule(QueryRetryInterval, new TEvPqNewMetaCache::TEvRestartQuery(EQueryType::ECheckVersion)); - return; - } - Y_VERIFY(result.GetResultSets().size() == 1); - ui64 newVersion = 0; - { - auto versionQueryResult = result.GetResultSetParser(0); - while (versionQueryResult.TryNextRow()) { - newVersion = versionQueryResult.ColumnParser("version").GetOptionalInt64().GetOrElse(0); - } - } - LastVersionUpdate = ctx.Now(); - if (newVersion > CurrentTopicsVersion || CurrentTopicsVersion == 0 || SkipVersionCheck) { - LOG_DEBUG_S(ctx, NKikimrServices::PQ_METACACHE, "Got config version: " << newVersion); - NewTopicsVersion = newVersion; - RunQuery(EQueryType::EGetTopics, ctx); - } else { - ctx.Schedule(VersionCheckInterval, new TEvPqNewMetaCache::TEvRestartQuery(EQueryType::ECheckVersion)); - } - } - - void HandleGetTopicsResult(const TActorContext& ctx) { - auto result = AsyncQueryResult.GetValue(); - if (!result.IsSuccess()) { - LOG_ERROR_S(ctx, NKikimrServices::PQ_METACACHE, - "Got error trying to get topics: " << result.GetIssues().ToString()); - return Reset(ctx); - } - Y_VERIFY(result.GetResultSets().size() == 1); - auto versionQueryResult = result.GetResultSetParser(0); - TString path, dc; - while (versionQueryResult.TryNextRow()) { - path = *versionQueryResult.ColumnParser("path").GetOptionalUtf8(); - dc = *versionQueryResult.ColumnParser("dc").GetOptionalUtf8(); - NewTopics.emplace_back(NPersQueue::BuildFullTopicName(path, dc)); - } - if (result.GetResultSet(0).Truncated()) { - LastTopicKey = {path, dc}; - return RunQuery(EQueryType::EGetTopics, ctx); - } else { - LastTopicKey = {}; - CurrentTopics = std::move(NewTopics); - NewTopics.clear(); - EverGotTopics = true; - CurrentTopicsVersion = NewTopicsVersion; - FullTopicsCacheOutdated = true; - FullTopicsCache = nullptr; - while (!ListTopicsWaiters.empty()) { - auto& waiter = ListTopicsWaiters.front(); - switch (waiter.Type) { - case EWaiterType::ListTopics: - SendListTopicsResponse(waiter.WaiterId, ctx); - break; - case EWaiterType::DescribeAllTopics: - ProcessDescribeAllTopics(waiter.WaiterId, ctx); - break; - default: - Y_FAIL(); - break; - } - ListTopicsWaiters.pop_front(); - } - LOG_DEBUG_S(ctx, NKikimrServices::PQ_METACACHE, - "Updated topics list with : " << CurrentTopics.size() << " topics"); - ctx.Schedule(VersionCheckInterval, new TEvPqNewMetaCache::TEvRestartQuery(EQueryType::ECheckVersion)); - } - } - - void HandleQueryComplete(TEvPqNewMetaCache::TEvQueryComplete::TPtr& ev, const TActorContext& ctx) { - switch (ev->Get()->Type) { - case EQueryType::ECheckVersion: - return HandleCheckVersionResult(ctx); - case EQueryType::EGetTopics: - return HandleGetTopicsResult(ctx); - default: - Y_FAIL(); - } - } - - void HandleRestartQuery(TEvPqNewMetaCache::TEvRestartQuery::TPtr& ev, const TActorContext& ctx) { - Y_VERIFY(ev->Get()->Type == EQueryType::ECheckVersion); - RunQuery(ev->Get()->Type, ctx); - } - - void HandleListTopics(TEvPqNewMetaCache::TEvListTopicsRequest::TPtr& ev, const TActorContext& ctx) { - if (!LastVersionUpdate || !EverGotTopics) { - ListTopicsWaiters.emplace_back(TWaiter{ev->Sender, {}, EWaiterType::ListTopics}); - } else { - SendListTopicsResponse(ev->Sender, ctx); - } - } - - void HandleDescribeTopics(TEvPqNewMetaCache::TEvDescribeTopicsRequest::TPtr& ev, const TActorContext& ctx) { - LOG_DEBUG_S(ctx, NKikimrServices::PQ_METACACHE, "Handle describe topics with prefix: " << ev->Get()->PathPrefix); - - SendSchemeCacheRequest(ev->Get()->Topics, !ev->Get()->PathPrefix.empty(), ctx); - auto inserted = DescribeTopicsWaiters.insert(std::make_pair( - RequestId, - TWaiter{ev->Sender, std::move(ev->Get()->Topics), EWaiterType::DescribeCustomTopics} - )).second; - Y_VERIFY(inserted); - } - - void HandleDescribeAllTopics(TEvPqNewMetaCache::TEvDescribeAllTopicsRequest::TPtr& ev, const TActorContext& ctx) { - if (ev->Get()->PathPrefix && ev->Get()->PathPrefix != PathPrefix) { - auto* response = new TEvPqNewMetaCache::TEvDescribeAllTopicsResponse(ev->Get()->PathPrefix); - response->Success = false; - ctx.Send(ev->Sender, response); - return; - } - if (!EverGotTopics) { - ListTopicsWaiters.emplace_back(TWaiter{ev->Sender, {}, EWaiterType::DescribeAllTopics}); - return; - } - return ProcessDescribeAllTopics(ev->Sender, ctx); - } - - void ProcessDescribeAllTopics(const TActorId& waiterId, const TActorContext& ctx) { - if (EverGotTopics && CurrentTopics.empty()) { - SendDescribeAllTopicsResponse(waiterId, ctx, true); - return; - } - if (FullTopicsCache && !FullTopicsCacheOutdated) { - LOG_DEBUG_S(ctx, NKikimrServices::PQ_METACACHE, "Respond from cache"); - - return SendDescribeAllTopicsResponse(waiterId, ctx); - } - LOG_DEBUG_S(ctx, NKikimrServices::PQ_METACACHE, "Store waiter"); - SendSchemeCacheRequest(CurrentTopics, true, ctx); - auto inserted = DescribeTopicsWaiters.insert(std::make_pair( - RequestId, - TWaiter{waiterId, {}, EWaiterType::DescribeAllTopics} - )).second; - Y_VERIFY(inserted); - FullTopicsCacheOutdated = false; - FullTopicsCache = nullptr; - } - - void HandleSchemeCacheResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) { - auto* result = ev->Get()->Request.Release(); - auto waiterIter = DescribeTopicsWaiters.find(result->Instant); - Y_VERIFY(!waiterIter.IsEnd()); - auto& waiter = waiterIter->second; - - if (waiter.Type == EWaiterType::DescribeAllTopics) { + } + + void HandleCheckVersionResult(const TActorContext& ctx) { + auto result = AsyncQueryResult.GetValue(); + if (!result.IsSuccess()) { + LOG_ERROR_S(ctx, NKikimrServices::PQ_METACACHE, + "Got error trying to update config version: " << result.GetIssues().ToString()); + ctx.Schedule(QueryRetryInterval, new TEvPqNewMetaCache::TEvRestartQuery(EQueryType::ECheckVersion)); + return; + } + Y_VERIFY(result.GetResultSets().size() == 1); + ui64 newVersion = 0; + { + auto versionQueryResult = result.GetResultSetParser(0); + while (versionQueryResult.TryNextRow()) { + newVersion = versionQueryResult.ColumnParser("version").GetOptionalInt64().GetOrElse(0); + } + } + LastVersionUpdate = ctx.Now(); + if (newVersion > CurrentTopicsVersion || CurrentTopicsVersion == 0 || SkipVersionCheck) { + LOG_DEBUG_S(ctx, NKikimrServices::PQ_METACACHE, "Got config version: " << newVersion); + NewTopicsVersion = newVersion; + RunQuery(EQueryType::EGetTopics, ctx); + } else { + ctx.Schedule(VersionCheckInterval, new TEvPqNewMetaCache::TEvRestartQuery(EQueryType::ECheckVersion)); + } + } + + void HandleGetTopicsResult(const TActorContext& ctx) { + auto result = AsyncQueryResult.GetValue(); + if (!result.IsSuccess()) { + LOG_ERROR_S(ctx, NKikimrServices::PQ_METACACHE, + "Got error trying to get topics: " << result.GetIssues().ToString()); + return Reset(ctx); + } + Y_VERIFY(result.GetResultSets().size() == 1); + auto versionQueryResult = result.GetResultSetParser(0); + TString path, dc; + while (versionQueryResult.TryNextRow()) { + path = *versionQueryResult.ColumnParser("path").GetOptionalUtf8(); + dc = *versionQueryResult.ColumnParser("dc").GetOptionalUtf8(); + NewTopics.emplace_back(NPersQueue::BuildFullTopicName(path, dc)); + } + if (result.GetResultSet(0).Truncated()) { + LastTopicKey = {path, dc}; + return RunQuery(EQueryType::EGetTopics, ctx); + } else { + LastTopicKey = {}; + CurrentTopics = std::move(NewTopics); + NewTopics.clear(); + EverGotTopics = true; + CurrentTopicsVersion = NewTopicsVersion; + FullTopicsCacheOutdated = true; + FullTopicsCache = nullptr; + while (!ListTopicsWaiters.empty()) { + auto& waiter = ListTopicsWaiters.front(); + switch (waiter.Type) { + case EWaiterType::ListTopics: + SendListTopicsResponse(waiter.WaiterId, ctx); + break; + case EWaiterType::DescribeAllTopics: + ProcessDescribeAllTopics(waiter.WaiterId, ctx); + break; + default: + Y_FAIL(); + break; + } + ListTopicsWaiters.pop_front(); + } + LOG_DEBUG_S(ctx, NKikimrServices::PQ_METACACHE, + "Updated topics list with : " << CurrentTopics.size() << " topics"); + ctx.Schedule(VersionCheckInterval, new TEvPqNewMetaCache::TEvRestartQuery(EQueryType::ECheckVersion)); + } + } + + void HandleQueryComplete(TEvPqNewMetaCache::TEvQueryComplete::TPtr& ev, const TActorContext& ctx) { + switch (ev->Get()->Type) { + case EQueryType::ECheckVersion: + return HandleCheckVersionResult(ctx); + case EQueryType::EGetTopics: + return HandleGetTopicsResult(ctx); + default: + Y_FAIL(); + } + } + + void HandleRestartQuery(TEvPqNewMetaCache::TEvRestartQuery::TPtr& ev, const TActorContext& ctx) { + Y_VERIFY(ev->Get()->Type == EQueryType::ECheckVersion); + RunQuery(ev->Get()->Type, ctx); + } + + void HandleListTopics(TEvPqNewMetaCache::TEvListTopicsRequest::TPtr& ev, const TActorContext& ctx) { + if (!LastVersionUpdate || !EverGotTopics) { + ListTopicsWaiters.emplace_back(TWaiter{ev->Sender, {}, EWaiterType::ListTopics}); + } else { + SendListTopicsResponse(ev->Sender, ctx); + } + } + + void HandleDescribeTopics(TEvPqNewMetaCache::TEvDescribeTopicsRequest::TPtr& ev, const TActorContext& ctx) { + LOG_DEBUG_S(ctx, NKikimrServices::PQ_METACACHE, "Handle describe topics with prefix: " << ev->Get()->PathPrefix); + + SendSchemeCacheRequest(ev->Get()->Topics, !ev->Get()->PathPrefix.empty(), ctx); + auto inserted = DescribeTopicsWaiters.insert(std::make_pair( + RequestId, + TWaiter{ev->Sender, std::move(ev->Get()->Topics), EWaiterType::DescribeCustomTopics} + )).second; + Y_VERIFY(inserted); + } + + void HandleDescribeAllTopics(TEvPqNewMetaCache::TEvDescribeAllTopicsRequest::TPtr& ev, const TActorContext& ctx) { + if (ev->Get()->PathPrefix && ev->Get()->PathPrefix != PathPrefix) { + auto* response = new TEvPqNewMetaCache::TEvDescribeAllTopicsResponse(ev->Get()->PathPrefix); + response->Success = false; + ctx.Send(ev->Sender, response); + return; + } + if (!EverGotTopics) { + ListTopicsWaiters.emplace_back(TWaiter{ev->Sender, {}, EWaiterType::DescribeAllTopics}); + return; + } + return ProcessDescribeAllTopics(ev->Sender, ctx); + } + + void ProcessDescribeAllTopics(const TActorId& waiterId, const TActorContext& ctx) { + if (EverGotTopics && CurrentTopics.empty()) { + SendDescribeAllTopicsResponse(waiterId, ctx, true); + return; + } + if (FullTopicsCache && !FullTopicsCacheOutdated) { + LOG_DEBUG_S(ctx, NKikimrServices::PQ_METACACHE, "Respond from cache"); + + return SendDescribeAllTopicsResponse(waiterId, ctx); + } + LOG_DEBUG_S(ctx, NKikimrServices::PQ_METACACHE, "Store waiter"); + SendSchemeCacheRequest(CurrentTopics, true, ctx); + auto inserted = DescribeTopicsWaiters.insert(std::make_pair( + RequestId, + TWaiter{waiterId, {}, EWaiterType::DescribeAllTopics} + )).second; + Y_VERIFY(inserted); + FullTopicsCacheOutdated = false; + FullTopicsCache = nullptr; + } + + void HandleSchemeCacheResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) { + auto* result = ev->Get()->Request.Release(); + auto waiterIter = DescribeTopicsWaiters.find(result->Instant); + Y_VERIFY(!waiterIter.IsEnd()); + auto& waiter = waiterIter->second; + + if (waiter.Type == EWaiterType::DescribeAllTopics) { for (const auto& entry : result->ResultSet) { if (!entry.PQGroupInfo) { continue; @@ -342,198 +342,198 @@ private: FullTopicsCacheOutdated = true; } - FullTopicsCache.Reset(result); - LOG_DEBUG_S(ctx, NKikimrServices::PQ_METACACHE, "Updated topics cache with " << FullTopicsCache->ResultSet.size()); - SendDescribeAllTopicsResponse(waiter.WaiterId, ctx); - } else { - Y_VERIFY(waiter.Type == EWaiterType::DescribeCustomTopics); - Y_VERIFY(waiter.Topics.size() == result->ResultSet.size()); - auto *response = new TEvPqNewMetaCache::TEvDescribeTopicsResponse{ - std::move(waiter.Topics), result - }; - ctx.Send(waiter.WaiterId, response); - } - DescribeTopicsWaiters.erase(waiterIter); - } - - - void SendSchemeCacheRequest(const TVector<TString>& topics, bool addDefaultPathPrefix, const TActorContext& ctx) { + FullTopicsCache.Reset(result); + LOG_DEBUG_S(ctx, NKikimrServices::PQ_METACACHE, "Updated topics cache with " << FullTopicsCache->ResultSet.size()); + SendDescribeAllTopicsResponse(waiter.WaiterId, ctx); + } else { + Y_VERIFY(waiter.Type == EWaiterType::DescribeCustomTopics); + Y_VERIFY(waiter.Topics.size() == result->ResultSet.size()); + auto *response = new TEvPqNewMetaCache::TEvDescribeTopicsResponse{ + std::move(waiter.Topics), result + }; + ctx.Send(waiter.WaiterId, response); + } + DescribeTopicsWaiters.erase(waiterIter); + } + + + void SendSchemeCacheRequest(const TVector<TString>& topics, bool addDefaultPathPrefix, const TActorContext& ctx) { auto schemeCacheRequest = std::make_unique<NSchemeCache::TSchemeCacheNavigate>(++RequestId); - for (const auto& path : topics) { - auto split = NKikimr::SplitPath(path); - Y_VERIFY(!split.empty()); - NSchemeCache::TSchemeCacheNavigate::TEntry entry; - if (addDefaultPathPrefix) { - entry.Path.insert(entry.Path.end(), PathPrefixParts.begin(), PathPrefixParts.end()); - } - - entry.Path.insert(entry.Path.end(), split.begin(), split.end()); - entry.SyncVersion = true; - entry.Operation = NSchemeCache::TSchemeCacheNavigate::OpTopic; - schemeCacheRequest->ResultSet.emplace_back(std::move(entry)); - } + for (const auto& path : topics) { + auto split = NKikimr::SplitPath(path); + Y_VERIFY(!split.empty()); + NSchemeCache::TSchemeCacheNavigate::TEntry entry; + if (addDefaultPathPrefix) { + entry.Path.insert(entry.Path.end(), PathPrefixParts.begin(), PathPrefixParts.end()); + } + + entry.Path.insert(entry.Path.end(), split.begin(), split.end()); + entry.SyncVersion = true; + entry.Operation = NSchemeCache::TSchemeCacheNavigate::OpTopic; + schemeCacheRequest->ResultSet.emplace_back(std::move(entry)); + } ctx.Send(SchemeCacheId, new TEvTxProxySchemeCache::TEvNavigateKeySet(schemeCacheRequest.release())); - } - - void SendListTopicsResponse(const TActorId& recipient, const TActorContext& ctx) { - auto* response = new TEvPqNewMetaCache::TEvListTopicsResponse(); - response->Topics = CurrentTopics; - response->TopicsVersion = CurrentTopicsVersion; - ctx.Send(recipient, response); - } - - void SendDescribeAllTopicsResponse(const TActorId& recipient, const TActorContext& ctx, bool empty = false) { - NSchemeCache::TSchemeCacheNavigate* scResponse; - if (empty) { - scResponse = new NSchemeCache::TSchemeCacheNavigate(); - } else { - scResponse = new NSchemeCache::TSchemeCacheNavigate(*FullTopicsCache); - } - LOG_DEBUG_S(ctx, NKikimrServices::PQ_METACACHE, "Send describe all topics response with " << scResponse->ResultSet.size() << " topics"); - auto* response = new TEvPqNewMetaCache::TEvDescribeAllTopicsResponse( - PathPrefix, scResponse - ); - ctx.Send(recipient, response); - } - - static NActors::TActorSystem* GetActorSystem() { - return TActivationContext::ActorSystem(); - } - -public: - void Die(const TActorContext& ctx) { - Generation->Inc(); - TBase::Die(ctx); - } - - STRICT_STFUNC(StateFunc, - SFunc(NActors::TEvents::TEvWakeup, StartSession) - SFunc(TEvPqNewMetaCache::TEvSessionStarted, HandleSessionStarted) - SFunc(TEvPqNewMetaCache::TEvQueryPrepared, HandleTopicsQueryPrepared) - HFunc(TEvPqNewMetaCache::TEvQueryComplete, HandleQueryComplete) - HFunc(TEvPqNewMetaCache::TEvRestartQuery, HandleRestartQuery) - - HFunc(TEvPqNewMetaCache::TEvListTopicsRequest, HandleListTopics) - HFunc(TEvPqNewMetaCache::TEvDescribeTopicsRequest, HandleDescribeTopics) - HFunc(TEvPqNewMetaCache::TEvDescribeAllTopicsRequest, HandleDescribeAllTopics) - HFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, HandleSchemeCacheResponse) - ) - -private: - enum class EWaiterType { - ListTopics, - DescribeAllTopics, - DescribeCustomTopics - }; - - struct TWaiter { - TActorId WaiterId; - TVector<TString> Topics; - EWaiterType Type; - - static TWaiter AllTopicsWaiter(const TActorId& waiterId) { - return TWaiter{waiterId, {}, EWaiterType::DescribeAllTopics}; - } - }; - - struct TTopicKey { - TString Path; - TString Cluster; - }; - - - class TClientWrapper { - public: + } + + void SendListTopicsResponse(const TActorId& recipient, const TActorContext& ctx) { + auto* response = new TEvPqNewMetaCache::TEvListTopicsResponse(); + response->Topics = CurrentTopics; + response->TopicsVersion = CurrentTopicsVersion; + ctx.Send(recipient, response); + } + + void SendDescribeAllTopicsResponse(const TActorId& recipient, const TActorContext& ctx, bool empty = false) { + NSchemeCache::TSchemeCacheNavigate* scResponse; + if (empty) { + scResponse = new NSchemeCache::TSchemeCacheNavigate(); + } else { + scResponse = new NSchemeCache::TSchemeCacheNavigate(*FullTopicsCache); + } + LOG_DEBUG_S(ctx, NKikimrServices::PQ_METACACHE, "Send describe all topics response with " << scResponse->ResultSet.size() << " topics"); + auto* response = new TEvPqNewMetaCache::TEvDescribeAllTopicsResponse( + PathPrefix, scResponse + ); + ctx.Send(recipient, response); + } + + static NActors::TActorSystem* GetActorSystem() { + return TActivationContext::ActorSystem(); + } + +public: + void Die(const TActorContext& ctx) { + Generation->Inc(); + TBase::Die(ctx); + } + + STRICT_STFUNC(StateFunc, + SFunc(NActors::TEvents::TEvWakeup, StartSession) + SFunc(TEvPqNewMetaCache::TEvSessionStarted, HandleSessionStarted) + SFunc(TEvPqNewMetaCache::TEvQueryPrepared, HandleTopicsQueryPrepared) + HFunc(TEvPqNewMetaCache::TEvQueryComplete, HandleQueryComplete) + HFunc(TEvPqNewMetaCache::TEvRestartQuery, HandleRestartQuery) + + HFunc(TEvPqNewMetaCache::TEvListTopicsRequest, HandleListTopics) + HFunc(TEvPqNewMetaCache::TEvDescribeTopicsRequest, HandleDescribeTopics) + HFunc(TEvPqNewMetaCache::TEvDescribeAllTopicsRequest, HandleDescribeAllTopics) + HFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, HandleSchemeCacheResponse) + ) + +private: + enum class EWaiterType { + ListTopics, + DescribeAllTopics, + DescribeCustomTopics + }; + + struct TWaiter { + TActorId WaiterId; + TVector<TString> Topics; + EWaiterType Type; + + static TWaiter AllTopicsWaiter(const TActorId& waiterId) { + return TWaiter{waiterId, {}, EWaiterType::DescribeAllTopics}; + } + }; + + struct TTopicKey { + TString Path; + TString Cluster; + }; + + + class TClientWrapper { + public: TClientWrapper(const TClientWrapper&) = delete; TClientWrapper& operator=(const TClientWrapper&) = delete; TClientWrapper(TClientWrapper&&) = default; TClientWrapper& operator=(TClientWrapper&&) = default; - TClientWrapper(ui64 driverPort) - : DriverPort(driverPort) - {} - - TClientWrapper(NYdb::TDriver* driver) - : Driver(driver) - {} - - void Reset() { - if (DriverPort.Defined()) { // Own driver - if (DriverHolder == nullptr) { - TString endpoint = TStringBuilder() << "localhost:" << *DriverPort; - auto driverConfig = NYdb::TDriverConfig().SetEndpoint(endpoint); - DriverHolder.Reset(new NYdb::TDriver(driverConfig)); - Driver = DriverHolder.Get(); + TClientWrapper(ui64 driverPort) + : DriverPort(driverPort) + {} + + TClientWrapper(NYdb::TDriver* driver) + : Driver(driver) + {} + + void Reset() { + if (DriverPort.Defined()) { // Own driver + if (DriverHolder == nullptr) { + TString endpoint = TStringBuilder() << "localhost:" << *DriverPort; + auto driverConfig = NYdb::TDriverConfig().SetEndpoint(endpoint); + DriverHolder.Reset(new NYdb::TDriver(driverConfig)); + Driver = DriverHolder.Get(); TableClient.reset(new NYdb::NTable::TTableClient(*Driver)); - } - } else if (Driver != nullptr) { + } + } else if (Driver != nullptr) { TableClient.reset(new NYdb::NTable::TTableClient(*Driver)); - } - } + } + } - NYdb::NTable::TTableClient* GetClient() { - Y_VERIFY(TableClient); + NYdb::NTable::TTableClient* GetClient() { + Y_VERIFY(TableClient); return TableClient.get(); - } - - void Stop() { - if (DriverHolder != nullptr) { - TableClient->Stop(); - DriverHolder->Stop(); - } else if (Driver != nullptr) { - TableClient->Stop(); - } - } - - private: - THolder<NYdb::TDriver> DriverHolder; - NYdb::TDriver* Driver = nullptr; - TMaybe<ui64> DriverPort; + } + + void Stop() { + if (DriverHolder != nullptr) { + TableClient->Stop(); + DriverHolder->Stop(); + } else if (Driver != nullptr) { + TableClient->Stop(); + } + } + + private: + THolder<NYdb::TDriver> DriverHolder; + NYdb::TDriver* Driver = nullptr; + TMaybe<ui64> DriverPort; std::unique_ptr<NYdb::NTable::TTableClient> TableClient; }; - - NMonitoring::TDynamicCounterPtr Counters; - NActors::TActorSystem* ActorSystem; - TString VersionQuery; - TString TopicsQuery; - + + NMonitoring::TDynamicCounterPtr Counters; + NActors::TActorSystem* ActorSystem; + TString VersionQuery; + TString TopicsQuery; + std::unique_ptr<TClientWrapper> ClientWrapper; - TAsyncCreateSessionResult SessionFuture; - TMaybe<TSession> YdbSession; - TAsyncPrepareQueryResult TopicsQueryFuture; - TMaybe<TDataQuery> PreparedTopicsQuery; - TAsyncDataQueryResult AsyncQueryResult; - ui64 CurrentTopicsVersion = 0; - ui64 NewTopicsVersion = 0; - TTopicKey LastTopicKey = TTopicKey{}; - TVector<TString> NewTopics; - TVector<TString> CurrentTopics; - bool EverGotTopics = false; - TDuration QueryRetryInterval = TDuration::Seconds(2); - TDuration VersionCheckInterval = TDuration::Seconds(1); - TInstant LastVersionUpdate = TInstant::Zero(); - - TDeque<TWaiter> ListTopicsWaiters; - THashMap<ui64, TWaiter> DescribeTopicsWaiters; - ui64 RequestId = 1; - - THolder<NSchemeCache::TSchemeCacheNavigate> FullTopicsCache; - bool FullTopicsCacheOutdated = false; - NActors::TActorId SchemeCacheId; - TString PathPrefix; - TVector<TString> PathPrefixParts; + TAsyncCreateSessionResult SessionFuture; + TMaybe<TSession> YdbSession; + TAsyncPrepareQueryResult TopicsQueryFuture; + TMaybe<TDataQuery> PreparedTopicsQuery; + TAsyncDataQueryResult AsyncQueryResult; + ui64 CurrentTopicsVersion = 0; + ui64 NewTopicsVersion = 0; + TTopicKey LastTopicKey = TTopicKey{}; + TVector<TString> NewTopics; + TVector<TString> CurrentTopics; + bool EverGotTopics = false; + TDuration QueryRetryInterval = TDuration::Seconds(2); + TDuration VersionCheckInterval = TDuration::Seconds(1); + TInstant LastVersionUpdate = TInstant::Zero(); + + TDeque<TWaiter> ListTopicsWaiters; + THashMap<ui64, TWaiter> DescribeTopicsWaiters; + ui64 RequestId = 1; + + THolder<NSchemeCache::TSchemeCacheNavigate> FullTopicsCache; + bool FullTopicsCacheOutdated = false; + NActors::TActorId SchemeCacheId; + TString PathPrefix; + TVector<TString> PathPrefixParts; std::shared_ptr<TAtomicCounter> Generation; - bool SkipVersionCheck = false; -}; - -IActor* CreatePQMetaCache(const NMonitoring::TDynamicCounterPtr& counters, const TDuration& versionCheckInterval) { - return new TPersQueueMetaCacheActor(counters, versionCheckInterval); -} - -IActor* CreatePQMetaCache(ui64 grpcPort, const NMonitoring::TDynamicCounterPtr& counters, const TDuration& versionCheckInterval) { - return new TPersQueueMetaCacheActor(grpcPort, counters, versionCheckInterval); -} - -} // namespace NPqMetaCacheV2 - -} // namespace NKikimr::NMsgBusProxy + bool SkipVersionCheck = false; +}; + +IActor* CreatePQMetaCache(const NMonitoring::TDynamicCounterPtr& counters, const TDuration& versionCheckInterval) { + return new TPersQueueMetaCacheActor(counters, versionCheckInterval); +} + +IActor* CreatePQMetaCache(ui64 grpcPort, const NMonitoring::TDynamicCounterPtr& counters, const TDuration& versionCheckInterval) { + return new TPersQueueMetaCacheActor(grpcPort, counters, versionCheckInterval); +} + +} // namespace NPqMetaCacheV2 + +} // namespace NKikimr::NMsgBusProxy diff --git a/ydb/core/client/server/msgbus_server_pq_metacache.h b/ydb/core/client/server/msgbus_server_pq_metacache.h index 8a1822dab3..6c12b76b04 100644 --- a/ydb/core/client/server/msgbus_server_pq_metacache.h +++ b/ydb/core/client/server/msgbus_server_pq_metacache.h @@ -3,140 +3,140 @@ #include <library/cpp/actors/core/events.h> #include <library/cpp/actors/core/event_local.h> #include <ydb/core/tx/schemeshard/schemeshard.h> - + #include <ydb/core/tx/scheme_cache/scheme_cache.h> #include <ydb/public/api/protos/draft/persqueue_error_codes.pb.h> #include <ydb/public/sdk/cpp/client/ydb_table/table.h> #include <util/generic/string.h> #include <util/generic/vector.h> -namespace NKikimr::NMsgBusProxy { +namespace NKikimr::NMsgBusProxy { static const ui32 PQ_METACACHE_TIMEOUT_SECONDS = 120; -static const ui32 PQ_METACACHE_REFRESH_INTERVAL_SECONDS = 10; +static const ui32 PQ_METACACHE_REFRESH_INTERVAL_SECONDS = 10; -inline TActorId CreatePersQueueMetaCacheV2Id() { +inline TActorId CreatePersQueueMetaCacheV2Id() { return TActorId(0, "PQMetaCache"); } -namespace NPqMetaCacheV2 { - -enum class EQueryType { - ECheckVersion, - EGetTopics, -}; - -struct TTopicMetaRequest { - TString Path; - THolder<NSchemeCache::TSchemeCacheNavigate> Response; - bool Success = false; -}; - -using TMetaCacheRequest = TVector<TTopicMetaRequest>; - -struct TEvPqNewMetaCache { - enum EEv { - EvWakeup = EventSpaceBegin(TKikimrEvents::ES_PQ_META_CACHE), - EvSessionStarted, - EvQueryPrepared, - EvQueryComplete, - EvRestartQuery, - EvListTopicsRequest, - EvListTopicsResponse, - EvDescribeTopicsRequest, - EvDescribeTopicsResponse, - EvDescribeAllTopicsRequest, - EvDescribeAllTopicsResponse, - EvEnd - }; - - static_assert(EvEnd < EventSpaceEnd(TKikimrEvents::ES_PQ_META_CACHE), - "expect EvEnd < EventSpaceEnd(TKikimrEvents::ES_PQ_META_CACHE)"); - - struct TEvSessionStarted : public TEventLocal<TEvSessionStarted, EvSessionStarted> { - }; - - struct TEvQueryPrepared : public TEventLocal<TEvQueryPrepared, EvQueryPrepared> { - }; - - struct TEvQueryComplete : public TEventLocal<TEvQueryComplete, EvQueryComplete> { - EQueryType Type; - - explicit TEvQueryComplete(EQueryType type) - : Type(type) {} - }; - - struct TEvRestartQuery : public TEventLocal<TEvRestartQuery, EvRestartQuery> { - EQueryType Type; - - explicit TEvRestartQuery(EQueryType type) - : Type(type) {} - }; - - struct TEvListTopicsRequest : public TEventLocal<TEvListTopicsRequest, EvListTopicsRequest> { - }; - - struct TEvListTopicsResponse : public TEventLocal<TEvListTopicsResponse, EvListTopicsResponse> { - TVector<TString> Topics; - ui64 TopicsVersion; - }; - - struct TEvDescribeTopicsRequest : public TEventLocal<TEvDescribeTopicsRequest, EvDescribeTopicsRequest> { - TString PathPrefix; - TVector<TString> Topics; - - TEvDescribeTopicsRequest() = default; - - explicit TEvDescribeTopicsRequest(const TVector<TString>& topics) - : Topics(topics) - {} - - TEvDescribeTopicsRequest(const TVector<TString>& topics, const TString& pathPrefix) - : PathPrefix(pathPrefix) - , Topics(topics) - {} - }; - - struct TEvDescribeTopicsResponse : public TEventLocal<TEvDescribeTopicsResponse, EvDescribeTopicsResponse> { - TVector<TString> TopicsRequested; - THolder<NSchemeCache::TSchemeCacheNavigate> Result; - explicit TEvDescribeTopicsResponse(const TVector<TString>& topics, - NSchemeCache::TSchemeCacheNavigate* result) - - : TopicsRequested(topics) - , Result(result) - {} - }; - - struct TEvDescribeAllTopicsRequest : public TEventLocal<TEvDescribeAllTopicsRequest, EvDescribeAllTopicsRequest> { - TString PathPrefix; - TEvDescribeAllTopicsRequest() = default; - explicit TEvDescribeAllTopicsRequest(const TString& pathPrefix) - : PathPrefix(pathPrefix) - {} - }; - - struct TEvDescribeAllTopicsResponse : public TEventLocal<TEvDescribeAllTopicsResponse, EvDescribeAllTopicsResponse> { - bool Success = true; - TString Path; - THolder<NSchemeCache::TSchemeCacheNavigate> Result; - explicit TEvDescribeAllTopicsResponse(const TString& path) - : Path(path) - {} - TEvDescribeAllTopicsResponse(const TString& path, NSchemeCache::TSchemeCacheNavigate* result) - : Path(path) - , Result(result) - {} - }; -}; -IActor* CreatePQMetaCache(const NMonitoring::TDynamicCounterPtr& counters, - const TDuration& versionCheckInterval); - -IActor* CreatePQMetaCache(ui64 grpcPort, - const NMonitoring::TDynamicCounterPtr& counters, - const TDuration& versionCheckInterval = TDuration::Seconds(1)); - -} // namespace NPqMetaCacheV2 - -} //namespace NKikimr::NMsgBusProxy +namespace NPqMetaCacheV2 { + +enum class EQueryType { + ECheckVersion, + EGetTopics, +}; + +struct TTopicMetaRequest { + TString Path; + THolder<NSchemeCache::TSchemeCacheNavigate> Response; + bool Success = false; +}; + +using TMetaCacheRequest = TVector<TTopicMetaRequest>; + +struct TEvPqNewMetaCache { + enum EEv { + EvWakeup = EventSpaceBegin(TKikimrEvents::ES_PQ_META_CACHE), + EvSessionStarted, + EvQueryPrepared, + EvQueryComplete, + EvRestartQuery, + EvListTopicsRequest, + EvListTopicsResponse, + EvDescribeTopicsRequest, + EvDescribeTopicsResponse, + EvDescribeAllTopicsRequest, + EvDescribeAllTopicsResponse, + EvEnd + }; + + static_assert(EvEnd < EventSpaceEnd(TKikimrEvents::ES_PQ_META_CACHE), + "expect EvEnd < EventSpaceEnd(TKikimrEvents::ES_PQ_META_CACHE)"); + + struct TEvSessionStarted : public TEventLocal<TEvSessionStarted, EvSessionStarted> { + }; + + struct TEvQueryPrepared : public TEventLocal<TEvQueryPrepared, EvQueryPrepared> { + }; + + struct TEvQueryComplete : public TEventLocal<TEvQueryComplete, EvQueryComplete> { + EQueryType Type; + + explicit TEvQueryComplete(EQueryType type) + : Type(type) {} + }; + + struct TEvRestartQuery : public TEventLocal<TEvRestartQuery, EvRestartQuery> { + EQueryType Type; + + explicit TEvRestartQuery(EQueryType type) + : Type(type) {} + }; + + struct TEvListTopicsRequest : public TEventLocal<TEvListTopicsRequest, EvListTopicsRequest> { + }; + + struct TEvListTopicsResponse : public TEventLocal<TEvListTopicsResponse, EvListTopicsResponse> { + TVector<TString> Topics; + ui64 TopicsVersion; + }; + + struct TEvDescribeTopicsRequest : public TEventLocal<TEvDescribeTopicsRequest, EvDescribeTopicsRequest> { + TString PathPrefix; + TVector<TString> Topics; + + TEvDescribeTopicsRequest() = default; + + explicit TEvDescribeTopicsRequest(const TVector<TString>& topics) + : Topics(topics) + {} + + TEvDescribeTopicsRequest(const TVector<TString>& topics, const TString& pathPrefix) + : PathPrefix(pathPrefix) + , Topics(topics) + {} + }; + + struct TEvDescribeTopicsResponse : public TEventLocal<TEvDescribeTopicsResponse, EvDescribeTopicsResponse> { + TVector<TString> TopicsRequested; + THolder<NSchemeCache::TSchemeCacheNavigate> Result; + explicit TEvDescribeTopicsResponse(const TVector<TString>& topics, + NSchemeCache::TSchemeCacheNavigate* result) + + : TopicsRequested(topics) + , Result(result) + {} + }; + + struct TEvDescribeAllTopicsRequest : public TEventLocal<TEvDescribeAllTopicsRequest, EvDescribeAllTopicsRequest> { + TString PathPrefix; + TEvDescribeAllTopicsRequest() = default; + explicit TEvDescribeAllTopicsRequest(const TString& pathPrefix) + : PathPrefix(pathPrefix) + {} + }; + + struct TEvDescribeAllTopicsResponse : public TEventLocal<TEvDescribeAllTopicsResponse, EvDescribeAllTopicsResponse> { + bool Success = true; + TString Path; + THolder<NSchemeCache::TSchemeCacheNavigate> Result; + explicit TEvDescribeAllTopicsResponse(const TString& path) + : Path(path) + {} + TEvDescribeAllTopicsResponse(const TString& path, NSchemeCache::TSchemeCacheNavigate* result) + : Path(path) + , Result(result) + {} + }; +}; +IActor* CreatePQMetaCache(const NMonitoring::TDynamicCounterPtr& counters, + const TDuration& versionCheckInterval); + +IActor* CreatePQMetaCache(ui64 grpcPort, + const NMonitoring::TDynamicCounterPtr& counters, + const TDuration& versionCheckInterval = TDuration::Seconds(1)); + +} // namespace NPqMetaCacheV2 + +} //namespace NKikimr::NMsgBusProxy diff --git a/ydb/core/client/server/msgbus_server_pq_metarequest.cpp b/ydb/core/client/server/msgbus_server_pq_metarequest.cpp index 5aac260ffe..32f37251cc 100644 --- a/ydb/core/client/server/msgbus_server_pq_metarequest.cpp +++ b/ydb/core/client/server/msgbus_server_pq_metarequest.cpp @@ -3,16 +3,16 @@ namespace NKikimr { namespace NMsgBusProxy { -using namespace NSchemeCache; +using namespace NSchemeCache; template <class TTopicResult> -void SetErrorCode( - TTopicResult* topicResult, - const TSchemeCacheNavigate::TEntry& topicEntry -) { - NPersQueue::NErrorCode::EErrorCode code = NPersQueue::NErrorCode::OK; - if (topicEntry.Status != TSchemeCacheNavigate::EStatus::Ok || !topicEntry.PQGroupInfo) { - code = NPersQueue::NErrorCode::UNKNOWN_TOPIC; +void SetErrorCode( + TTopicResult* topicResult, + const TSchemeCacheNavigate::TEntry& topicEntry +) { + NPersQueue::NErrorCode::EErrorCode code = NPersQueue::NErrorCode::OK; + if (topicEntry.Status != TSchemeCacheNavigate::EStatus::Ok || !topicEntry.PQGroupInfo) { + code = NPersQueue::NErrorCode::UNKNOWN_TOPIC; } topicResult->SetErrorCode(code); if (code == NPersQueue::NErrorCode::UNKNOWN_TOPIC) { @@ -38,24 +38,24 @@ TPersQueueGetTopicMetadataProcessor::TPersQueueGetTopicMetadataProcessor(const N } } -THolder<IActor> TPersQueueGetTopicMetadataProcessor::CreateTopicSubactor( - const TSchemeEntry& topicEntry, const TString& name -) { - return MakeHolder<TPersQueueGetTopicMetadataTopicWorker>(SelfId(), topicEntry, name); +THolder<IActor> TPersQueueGetTopicMetadataProcessor::CreateTopicSubactor( + const TSchemeEntry& topicEntry, const TString& name +) { + return MakeHolder<TPersQueueGetTopicMetadataTopicWorker>(SelfId(), topicEntry, name); } -TPersQueueGetTopicMetadataTopicWorker::TPersQueueGetTopicMetadataTopicWorker( - const TActorId& parent, const TSchemeEntry& topicEntry, const TString& name -) - : TReplierToParent<TTopicInfoBasedActor>(parent, topicEntry, name) +TPersQueueGetTopicMetadataTopicWorker::TPersQueueGetTopicMetadataTopicWorker( + const TActorId& parent, const TSchemeEntry& topicEntry, const TString& name +) + : TReplierToParent<TTopicInfoBasedActor>(parent, topicEntry, name) { SetActivityType(NKikimrServices::TActivity::PQ_META_REQUEST_PROCESSOR); } -void TPersQueueGetTopicMetadataTopicWorker::BootstrapImpl(const TActorContext& ctx) { - auto processingResult = ProcessMetaCacheSingleTopicsResponse(SchemeEntry); - Answer(ctx, processingResult.Status, processingResult.ErrorCode, processingResult.Reason); +void TPersQueueGetTopicMetadataTopicWorker::BootstrapImpl(const TActorContext& ctx) { + auto processingResult = ProcessMetaCacheSingleTopicsResponse(SchemeEntry); + Answer(ctx, processingResult.Status, processingResult.ErrorCode, processingResult.Reason); } void TPersQueueGetTopicMetadataTopicWorker::Answer(const TActorContext& ctx, EResponseStatus status, NPersQueue::NErrorCode::EErrorCode code, const TString& errorReason) { @@ -66,13 +66,13 @@ void TPersQueueGetTopicMetadataTopicWorker::Answer(const TActorContext& ctx, ERe response.SetErrorReason(errorReason); if (code == NPersQueue::NErrorCode::OK) { auto* topicInfo = response.MutableMetaResponse()->MutableCmdGetTopicMetadataResult()->AddTopicInfo(); - SetErrorCode(topicInfo, SchemeEntry); - if (SchemeEntry.PQGroupInfo != nullptr) { - const auto& desc = SchemeEntry.PQGroupInfo->Description; - topicInfo->SetTopic(desc.GetName()); - topicInfo->MutableConfig()->CopyFrom(desc.GetPQTabletConfig()); - topicInfo->MutableConfig()->SetVersion(desc.GetAlterVersion()); - topicInfo->SetNumPartitions(desc.PartitionsSize()); + SetErrorCode(topicInfo, SchemeEntry); + if (SchemeEntry.PQGroupInfo != nullptr) { + const auto& desc = SchemeEntry.PQGroupInfo->Description; + topicInfo->SetTopic(desc.GetName()); + topicInfo->MutableConfig()->CopyFrom(desc.GetPQTabletConfig()); + topicInfo->MutableConfig()->SetVersion(desc.GetAlterVersion()); + topicInfo->SetNumPartitions(desc.PartitionsSize()); } } SendReplyAndDie(std::move(response), ctx); @@ -83,41 +83,41 @@ void TPersQueueGetTopicMetadataTopicWorker::Answer(const TActorContext& ctx, ERe // GetPartitionOffsets command // -TPersQueueGetPartitionOffsetsProcessor::TPersQueueGetPartitionOffsetsProcessor( - const NKikimrClient::TPersQueueRequest& request, const TActorId& metaCacheId -) - : TPersQueueBaseRequestProcessor(request, metaCacheId, false) +TPersQueueGetPartitionOffsetsProcessor::TPersQueueGetPartitionOffsetsProcessor( + const NKikimrClient::TPersQueueRequest& request, const TActorId& metaCacheId +) + : TPersQueueBaseRequestProcessor(request, metaCacheId, false) { GetTopicsListOrThrow(RequestProto->GetMetaRequest().GetCmdGetPartitionOffsets().GetTopicRequest(), PartitionsToRequest); } -THolder<IActor> TPersQueueGetPartitionOffsetsProcessor::CreateTopicSubactor( - const TSchemeEntry& topicEntry, const TString& name -) { - return MakeHolder<TPersQueueGetPartitionOffsetsTopicWorker>( - SelfId(), topicEntry, name, PartitionsToRequest[topicEntry.PQGroupInfo->Description.GetName()], RequestProto - ); +THolder<IActor> TPersQueueGetPartitionOffsetsProcessor::CreateTopicSubactor( + const TSchemeEntry& topicEntry, const TString& name +) { + return MakeHolder<TPersQueueGetPartitionOffsetsTopicWorker>( + SelfId(), topicEntry, name, PartitionsToRequest[topicEntry.PQGroupInfo->Description.GetName()], RequestProto + ); } -TPersQueueGetPartitionOffsetsTopicWorker::TPersQueueGetPartitionOffsetsTopicWorker( - const TActorId& parent, const TSchemeEntry& topicEntry, const TString& name, +TPersQueueGetPartitionOffsetsTopicWorker::TPersQueueGetPartitionOffsetsTopicWorker( + const TActorId& parent, const TSchemeEntry& topicEntry, const TString& name, const std::shared_ptr<THashSet<ui64>>& partitionsToRequest, const std::shared_ptr<const NKikimrClient::TPersQueueRequest>& requestProto -) - : TReplierToParent<TPipesWaiterActor<TTopicInfoBasedActor, TEvPersQueue::TEvOffsetsResponse>>(parent, topicEntry, name) +) + : TReplierToParent<TPipesWaiterActor<TTopicInfoBasedActor, TEvPersQueue::TEvOffsetsResponse>>(parent, topicEntry, name) , PartitionsToRequest(partitionsToRequest) , RequestProto(requestProto) { SetActivityType(NKikimrServices::TActivity::PQ_META_REQUEST_PROCESSOR); } -void TPersQueueGetPartitionOffsetsTopicWorker::BootstrapImpl(const TActorContext &ctx) { +void TPersQueueGetPartitionOffsetsTopicWorker::BootstrapImpl(const TActorContext &ctx) { size_t partitionsAsked = 0; THashSet<ui64> parts; - if (SchemeEntry.PQGroupInfo) { - const auto& pqDescr = SchemeEntry.PQGroupInfo->Description; - for (const auto& partition : pqDescr.GetPartitions()) { - const ui32 partIndex = partition.GetPartitionId(); + if (SchemeEntry.PQGroupInfo) { + const auto& pqDescr = SchemeEntry.PQGroupInfo->Description; + for (const auto& partition : pqDescr.GetPartitions()) { + const ui32 partIndex = partition.GetPartitionId(); const ui64 tabletId = partition.GetTabletId(); if (PartitionsToRequest.get() != nullptr && !PartitionsToRequest->empty() && !PartitionsToRequest->contains(partIndex)) { continue; @@ -137,22 +137,22 @@ void TPersQueueGetPartitionOffsetsTopicWorker::BootstrapImpl(const TActorContext } if (PartitionsToRequest.get() != nullptr && !PartitionsToRequest->empty() && PartitionsToRequest->size() != partitionsAsked) { SendErrorReplyAndDie(ctx, MSTATUS_ERROR, NPersQueue::NErrorCode::UNKNOWN_TOPIC, - TStringBuilder() << "no one of requested partitions in topic '" << Name << "', Marker# PQ96"); - return; + TStringBuilder() << "no one of requested partitions in topic '" << Name << "', Marker# PQ96"); + return; } if (!PartitionsToRequest.get() || PartitionsToRequest->empty()) { PartitionsToRequest.reset(new THashSet<ui64>()); PartitionsToRequest->swap(parts); } - if(WaitAllPipeEvents(ctx)) { - return; - } + if(WaitAllPipeEvents(ctx)) { + return; + } } bool TPersQueueGetPartitionOffsetsTopicWorker::OnPipeEventsAreReady(const TActorContext& ctx) { - auto processResult = ProcessMetaCacheSingleTopicsResponse(SchemeEntry); - Answer(ctx, processResult.Status, processResult.ErrorCode, processResult.Reason); - + auto processResult = ProcessMetaCacheSingleTopicsResponse(SchemeEntry); + Answer(ctx, processResult.Status, processResult.ErrorCode, processResult.Reason); + return true; } @@ -164,8 +164,8 @@ void TPersQueueGetPartitionOffsetsTopicWorker::Answer(const TActorContext& ctx, response.SetErrorReason(errorReason); if (code == NPersQueue::NErrorCode::OK) { auto& topicResult = *response.MutableMetaResponse()->MutableCmdGetPartitionOffsetsResult()->AddTopicResult(); - topicResult.SetTopic(Name); - SetErrorCode(&topicResult, SchemeEntry); + topicResult.SetTopic(Name); + SetErrorCode(&topicResult, SchemeEntry); THashSet<ui64> partitionsInserted; for (auto& ans : PipeAnswers) { if (ans.second.Get() != nullptr) { @@ -204,35 +204,35 @@ TPersQueueGetPartitionStatusProcessor::TPersQueueGetPartitionStatusProcessor(con GetTopicsListOrThrow(RequestProto->GetMetaRequest().GetCmdGetPartitionStatus().GetTopicRequest(), PartitionsToRequest); } -THolder<IActor> TPersQueueGetPartitionStatusProcessor::CreateTopicSubactor( - const TSchemeEntry& topicEntry, const TString& name -) { - return MakeHolder<TPersQueueGetPartitionStatusTopicWorker>( - SelfId(), topicEntry, name, PartitionsToRequest[topicEntry.PQGroupInfo->Description.GetName()], RequestProto - ); +THolder<IActor> TPersQueueGetPartitionStatusProcessor::CreateTopicSubactor( + const TSchemeEntry& topicEntry, const TString& name +) { + return MakeHolder<TPersQueueGetPartitionStatusTopicWorker>( + SelfId(), topicEntry, name, PartitionsToRequest[topicEntry.PQGroupInfo->Description.GetName()], RequestProto + ); } -TPersQueueGetPartitionStatusTopicWorker::TPersQueueGetPartitionStatusTopicWorker( - const TActorId& parent, const TSchemeEntry& topicEntry, const TString& name, +TPersQueueGetPartitionStatusTopicWorker::TPersQueueGetPartitionStatusTopicWorker( + const TActorId& parent, const TSchemeEntry& topicEntry, const TString& name, const std::shared_ptr<THashSet<ui64>>& partitionsToRequest, const std::shared_ptr<const NKikimrClient::TPersQueueRequest>& requestProto -) - : TReplierToParent<TPipesWaiterActor<TTopicInfoBasedActor, TEvPersQueue::TEvStatusResponse>>(parent, topicEntry, name) +) + : TReplierToParent<TPipesWaiterActor<TTopicInfoBasedActor, TEvPersQueue::TEvStatusResponse>>(parent, topicEntry, name) , PartitionsToRequest(partitionsToRequest) , RequestProto(requestProto) { SetActivityType(NKikimrServices::TActivity::PQ_META_REQUEST_PROCESSOR); } -void TPersQueueGetPartitionStatusTopicWorker::BootstrapImpl(const TActorContext &ctx) { +void TPersQueueGetPartitionStatusTopicWorker::BootstrapImpl(const TActorContext &ctx) { size_t partitionsAsked = 0; THashSet<ui64> parts; - if (!ProcessingResult.IsFatal) { - const auto& pqDescr = SchemeEntry.PQGroupInfo->Description; - for (const auto& partition : pqDescr.GetPartitions()) { - const ui32 partIndex = partition.GetPartitionId(); + if (!ProcessingResult.IsFatal) { + const auto& pqDescr = SchemeEntry.PQGroupInfo->Description; + for (const auto& partition : pqDescr.GetPartitions()) { + const ui32 partIndex = partition.GetPartitionId(); const ui64 tabletId = partition.GetTabletId(); - if (PartitionsToRequest != nullptr && !PartitionsToRequest->empty() && !PartitionsToRequest->contains(partIndex)) { + if (PartitionsToRequest != nullptr && !PartitionsToRequest->empty() && !PartitionsToRequest->contains(partIndex)) { continue; } parts.insert(partIndex); @@ -245,34 +245,34 @@ void TPersQueueGetPartitionStatusTopicWorker::BootstrapImpl(const TActorContext ev->Record.SetClientId(RequestProto->GetMetaRequest().GetCmdGetPartitionStatus().GetClientId()); CreatePipeAndSend(tabletId, ctx, std::move(ev)); } - } else { - SendErrorReplyAndDie(ctx, ProcessingResult.Status, ProcessingResult.ErrorCode, ProcessingResult.Reason); - return; + } else { + SendErrorReplyAndDie(ctx, ProcessingResult.Status, ProcessingResult.ErrorCode, ProcessingResult.Reason); + return; } - if (PartitionsToRequest != nullptr && !PartitionsToRequest->empty() && PartitionsToRequest->size() != partitionsAsked) { + if (PartitionsToRequest != nullptr && !PartitionsToRequest->empty() && PartitionsToRequest->size() != partitionsAsked) { SendErrorReplyAndDie(ctx, MSTATUS_ERROR, NPersQueue::NErrorCode::UNKNOWN_TOPIC, - TStringBuilder() << "no one of requested partitions in topic '" << Name << "', Marker# PQ97"); - return; + TStringBuilder() << "no one of requested partitions in topic '" << Name << "', Marker# PQ97"); + return; } if (!PartitionsToRequest.get() || PartitionsToRequest->empty()) { PartitionsToRequest.reset(new THashSet<ui64>()); PartitionsToRequest->swap(parts); } - if (WaitAllPipeEvents(ctx)) - return; + if (WaitAllPipeEvents(ctx)) + return; } bool TPersQueueGetPartitionStatusTopicWorker::OnPipeEventsAreReady(const TActorContext& ctx) { - auto processResult = ProcessMetaCacheSingleTopicsResponse(SchemeEntry); - Answer(ctx, processResult.Status, processResult.ErrorCode, processResult.Reason); + auto processResult = ProcessMetaCacheSingleTopicsResponse(SchemeEntry); + Answer(ctx, processResult.Status, processResult.ErrorCode, processResult.Reason); return true; } -void TPersQueueGetPartitionStatusTopicWorker::Answer( - const TActorContext& ctx, EResponseStatus status, NPersQueue::NErrorCode::EErrorCode code, - const TString& errorReason -) { +void TPersQueueGetPartitionStatusTopicWorker::Answer( + const TActorContext& ctx, EResponseStatus status, NPersQueue::NErrorCode::EErrorCode code, + const TString& errorReason +) { NKikimrClient::TResponse response; response.SetStatus(status); response.SetErrorCode(code); @@ -280,8 +280,8 @@ void TPersQueueGetPartitionStatusTopicWorker::Answer( response.SetErrorReason(errorReason); if (code == NPersQueue::NErrorCode::OK) { auto& topicResult = *response.MutableMetaResponse()->MutableCmdGetPartitionStatusResult()->AddTopicResult(); - topicResult.SetTopic(Name); - SetErrorCode(&topicResult, SchemeEntry); + topicResult.SetTopic(Name); + SetErrorCode(&topicResult, SchemeEntry); THashSet<ui64> partitionsInserted; for (auto& ans : PipeAnswers) { if (ans.second.Get() != nullptr) { @@ -315,32 +315,32 @@ void TPersQueueGetPartitionStatusTopicWorker::Answer( // GetPartitionLocations command // -TPersQueueGetPartitionLocationsProcessor::TPersQueueGetPartitionLocationsProcessor( - const NKikimrClient::TPersQueueRequest& request, const TActorId& schemeCache -) +TPersQueueGetPartitionLocationsProcessor::TPersQueueGetPartitionLocationsProcessor( + const NKikimrClient::TPersQueueRequest& request, const TActorId& schemeCache +) : TPersQueueBaseRequestProcessor(request, schemeCache, true) { GetTopicsListOrThrow(RequestProto->GetMetaRequest().GetCmdGetPartitionLocations().GetTopicRequest(), PartitionsToRequest); } -THolder<IActor> TPersQueueGetPartitionLocationsProcessor::CreateTopicSubactor( - const TSchemeEntry& topicEntry, const TString& name -) { +THolder<IActor> TPersQueueGetPartitionLocationsProcessor::CreateTopicSubactor( + const TSchemeEntry& topicEntry, const TString& name +) { Y_VERIFY(NodesInfo.get() != nullptr); - return MakeHolder<TPersQueueGetPartitionLocationsTopicWorker>( - SelfId(), topicEntry, name, - PartitionsToRequest[topicEntry.PQGroupInfo->Description.GetName()], RequestProto, NodesInfo - ); + return MakeHolder<TPersQueueGetPartitionLocationsTopicWorker>( + SelfId(), topicEntry, name, + PartitionsToRequest[topicEntry.PQGroupInfo->Description.GetName()], RequestProto, NodesInfo + ); } -TPersQueueGetPartitionLocationsTopicWorker::TPersQueueGetPartitionLocationsTopicWorker( - const TActorId& parent, - const TSchemeEntry& topicEntry, const TString& name, +TPersQueueGetPartitionLocationsTopicWorker::TPersQueueGetPartitionLocationsTopicWorker( + const TActorId& parent, + const TSchemeEntry& topicEntry, const TString& name, const std::shared_ptr<THashSet<ui64>>& partitionsToRequest, const std::shared_ptr<const NKikimrClient::TPersQueueRequest>& requestProto, std::shared_ptr<const TPersQueueBaseRequestProcessor::TNodesInfo> nodesInfo -) - : TReplierToParent<TPipesWaiterActor<TTopicInfoBasedActor, TEvTabletPipe::TEvClientConnected>>(parent, topicEntry, name) +) + : TReplierToParent<TPipesWaiterActor<TTopicInfoBasedActor, TEvTabletPipe::TEvClientConnected>>(parent, topicEntry, name) , PartitionsToRequest(partitionsToRequest) , RequestProto(requestProto) , NodesInfo(nodesInfo) @@ -348,13 +348,13 @@ TPersQueueGetPartitionLocationsTopicWorker::TPersQueueGetPartitionLocationsTopic SetActivityType(NKikimrServices::TActivity::PQ_META_REQUEST_PROCESSOR); } -void TPersQueueGetPartitionLocationsTopicWorker::BootstrapImpl(const TActorContext& ctx) { +void TPersQueueGetPartitionLocationsTopicWorker::BootstrapImpl(const TActorContext& ctx) { size_t partitionsAsked = 0; THashSet<ui64> parts; - if (SchemeEntry.PQGroupInfo) { - const auto& pqDescr = SchemeEntry.PQGroupInfo->Description; - for (const auto& partition : pqDescr.GetPartitions()) { - const ui32 partIndex = partition.GetPartitionId(); + if (SchemeEntry.PQGroupInfo) { + const auto& pqDescr = SchemeEntry.PQGroupInfo->Description; + for (const auto& partition : pqDescr.GetPartitions()) { + const ui32 partIndex = partition.GetPartitionId(); const ui64 tabletId = partition.GetTabletId(); if (PartitionsToRequest.get() != nullptr && !PartitionsToRequest->empty() && !PartitionsToRequest->contains(partIndex)) { continue; @@ -370,28 +370,28 @@ void TPersQueueGetPartitionLocationsTopicWorker::BootstrapImpl(const TActorConte } if (PartitionsToRequest.get() != nullptr && !PartitionsToRequest->empty() && PartitionsToRequest->size() != partitionsAsked) { SendErrorReplyAndDie(ctx, MSTATUS_ERROR, NPersQueue::NErrorCode::UNKNOWN_TOPIC, - TStringBuilder() << "no one of requested partitions in topic '" << Name << "', Marker# PQ98"); - return; + TStringBuilder() << "no one of requested partitions in topic '" << Name << "', Marker# PQ98"); + return; } if (!PartitionsToRequest.get() || PartitionsToRequest->empty()) { PartitionsToRequest.reset(new THashSet<ui64>()); PartitionsToRequest->swap(parts); } - if(WaitAllConnections(ctx)) - return; + if(WaitAllConnections(ctx)) + return; } bool TPersQueueGetPartitionLocationsTopicWorker::OnPipeEventsAreReady(const TActorContext& ctx) { - auto processResult = ProcessMetaCacheSingleTopicsResponse(SchemeEntry); - Answer(ctx, processResult.Status, processResult.ErrorCode, processResult.Reason); + auto processResult = ProcessMetaCacheSingleTopicsResponse(SchemeEntry); + Answer(ctx, processResult.Status, processResult.ErrorCode, processResult.Reason); return true; } -void TPersQueueGetPartitionLocationsTopicWorker::Answer( - const TActorContext& ctx, EResponseStatus status, NPersQueue::NErrorCode::EErrorCode code, - const TString& errorReason -) { +void TPersQueueGetPartitionLocationsTopicWorker::Answer( + const TActorContext& ctx, EResponseStatus status, NPersQueue::NErrorCode::EErrorCode code, + const TString& errorReason +) { NKikimrClient::TResponse response; response.SetStatus(status); response.SetErrorCode(code); @@ -400,8 +400,8 @@ void TPersQueueGetPartitionLocationsTopicWorker::Answer( if (code == NPersQueue::NErrorCode::OK) { auto& topicResult = *response.MutableMetaResponse()->MutableCmdGetPartitionLocationsResult()->AddTopicResult(); - topicResult.SetTopic(Name); - SetErrorCode(&topicResult, SchemeEntry); + topicResult.SetTopic(Name); + SetErrorCode(&topicResult, SchemeEntry); for (const auto& partitionToTablet : PartitionToTablet) { const ui32 partition = partitionToTablet.first; const ui64 tabletId = partitionToTablet.second; @@ -438,11 +438,11 @@ void TPersQueueGetPartitionLocationsTopicWorker::Answer( // GetReadSessionsInfo command // -TPersQueueGetReadSessionsInfoProcessor::TPersQueueGetReadSessionsInfoProcessor( +TPersQueueGetReadSessionsInfoProcessor::TPersQueueGetReadSessionsInfoProcessor( const NKikimrClient::TPersQueueRequest& request, const TActorId& schemeCache, std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory -) +) : TPersQueueBaseRequestProcessor(request, schemeCache, true) , PQReadSessionsInfoWorkerFactory(pqReadSessionsInfoWorkerFactory) { @@ -458,20 +458,20 @@ TPersQueueGetReadSessionsInfoProcessor::TPersQueueGetReadSessionsInfoProcessor( } } -THolder<IActor> TPersQueueGetReadSessionsInfoProcessor::CreateTopicSubactor( - const TSchemeEntry& topicEntry, const TString& name -) { +THolder<IActor> TPersQueueGetReadSessionsInfoProcessor::CreateTopicSubactor( + const TSchemeEntry& topicEntry, const TString& name +) { Y_VERIFY(NodesInfo.get() != nullptr); - return MakeHolder<TPersQueueGetReadSessionsInfoTopicWorker>( - SelfId(), topicEntry, name, RequestProto, NodesInfo); + return MakeHolder<TPersQueueGetReadSessionsInfoTopicWorker>( + SelfId(), topicEntry, name, RequestProto, NodesInfo); } -TPersQueueGetReadSessionsInfoTopicWorker::TPersQueueGetReadSessionsInfoTopicWorker( - const TActorId& parent, const TSchemeEntry& topicEntry, const TString& name, +TPersQueueGetReadSessionsInfoTopicWorker::TPersQueueGetReadSessionsInfoTopicWorker( + const TActorId& parent, const TSchemeEntry& topicEntry, const TString& name, const std::shared_ptr<const NKikimrClient::TPersQueueRequest>& requestProto, std::shared_ptr<const TPersQueueBaseRequestProcessor::TNodesInfo> nodesInfo -) - : TReplierToParent<TPipesWaiterActor<TTopicInfoBasedActor, TEvPersQueue::TEvOffsetsResponse>>(parent, topicEntry, name) +) + : TReplierToParent<TPipesWaiterActor<TTopicInfoBasedActor, TEvPersQueue::TEvOffsetsResponse>>(parent, topicEntry, name) , RequestProto(requestProto) , NodesInfo(nodesInfo) { @@ -487,20 +487,20 @@ void TPersQueueGetReadSessionsInfoTopicWorker::Die(const TActorContext& ctx) { void TPersQueueGetReadSessionsInfoTopicWorker::SendReadSessionsInfoToBalancer(const TActorContext& ctx) { NTabletPipe::TClientConfig clientConfig; - BalancerPipe = ctx.RegisterWithSameMailbox( - NTabletPipe::CreateClient(ctx.SelfID, SchemeEntry.PQGroupInfo->Description.GetBalancerTabletID(), clientConfig) - ); + BalancerPipe = ctx.RegisterWithSameMailbox( + NTabletPipe::CreateClient(ctx.SelfID, SchemeEntry.PQGroupInfo->Description.GetBalancerTabletID(), clientConfig) + ); THolder<TEvPersQueue::TEvGetReadSessionsInfo> ev(new TEvPersQueue::TEvGetReadSessionsInfo()); ev->Record.SetClientId(RequestProto->GetMetaRequest().GetCmdGetReadSessionsInfo().GetClientId()); NTabletPipe::SendData(ctx, BalancerPipe, ev.Release()); } -void TPersQueueGetReadSessionsInfoTopicWorker::BootstrapImpl(const TActorContext &ctx) { - if (!ProcessingResult.IsFatal) { - SendReadSessionsInfoToBalancer(ctx); - for (const auto& partition : SchemeEntry.PQGroupInfo->Description.GetPartitions()) { - const ui32 partitionIndex = partition.GetPartitionId(); +void TPersQueueGetReadSessionsInfoTopicWorker::BootstrapImpl(const TActorContext &ctx) { + if (!ProcessingResult.IsFatal) { + SendReadSessionsInfoToBalancer(ctx); + for (const auto& partition : SchemeEntry.PQGroupInfo->Description.GetPartitions()) { + const ui32 partitionIndex = partition.GetPartitionId(); const ui64 tabletId = partition.GetTabletId(); const bool inserted = PartitionToTablet.emplace(partitionIndex, tabletId).second; Y_VERIFY(inserted); @@ -516,11 +516,11 @@ void TPersQueueGetReadSessionsInfoTopicWorker::BootstrapImpl(const TActorContext } CreatePipeAndSend(tabletId, ctx, std::move(ev)); } - } else { - Answer(ctx, ProcessingResult.Status, ProcessingResult.ErrorCode, ProcessingResult.Reason); + } else { + Answer(ctx, ProcessingResult.Status, ProcessingResult.ErrorCode, ProcessingResult.Reason); } - if(WaitAllPipeEvents(ctx)) - return; + if(WaitAllPipeEvents(ctx)) + return; } bool TPersQueueGetReadSessionsInfoTopicWorker::WaitAllPipeEvents(const TActorContext& ctx) { @@ -565,7 +565,7 @@ bool TPersQueueGetReadSessionsInfoTopicWorker::HandleConnect(TEvTabletPipe::TEvC if (ev->Status != NKikimrProto::OK) { BalancerReplied = true; if (ReadyToAnswer()) { - Answer(ctx, ProcessingResult.Status, ProcessingResult.ErrorCode, ProcessingResult.Reason); + Answer(ctx, ProcessingResult.Status, ProcessingResult.ErrorCode, ProcessingResult.Reason); } } else { TabletNodes[GetTabletId(ev)] = ev->ServerId.NodeId(); @@ -579,7 +579,7 @@ bool TPersQueueGetReadSessionsInfoTopicWorker::HandleDestroy(TEvTabletPipe::TEvC } BalancerReplied = true; if (ReadyToAnswer()) { - Answer(ctx, ProcessingResult.Status, ProcessingResult.ErrorCode, ProcessingResult.Reason); + Answer(ctx, ProcessingResult.Status, ProcessingResult.ErrorCode, ProcessingResult.Reason); } return true; } @@ -597,7 +597,7 @@ void TPersQueueGetReadSessionsInfoTopicWorker::Handle(TEvPersQueue::TEvReadSessi bool TPersQueueGetReadSessionsInfoTopicWorker::OnPipeEventsAreReady(const TActorContext& ctx) { PipeEventsAreReady = true; if (ReadyToAnswer()) { - Answer(ctx, ProcessingResult.Status, ProcessingResult.ErrorCode, ProcessingResult.Reason); + Answer(ctx, ProcessingResult.Status, ProcessingResult.ErrorCode, ProcessingResult.Reason); return true; } return false; @@ -623,8 +623,8 @@ void TPersQueueGetReadSessionsInfoTopicWorker::Answer(const TActorContext& ctx, auto stat = response.MutableMetaResponse()->MutableCmdGetReadSessionsInfoResult(); auto topicRes = stat->AddTopicResult(); - topicRes->SetTopic(Name); - SetErrorCode(topicRes, SchemeEntry); + topicRes->SetTopic(Name); + SetErrorCode(topicRes, SchemeEntry); THashMap<ui32, ui32> partitionToResp; ui32 index = 0; if (BalancerResponse.Get() != nullptr) { @@ -646,8 +646,8 @@ void TPersQueueGetReadSessionsInfoTopicWorker::Answer(const TActorContext& ctx, ctx.Send(Parent, request.Release()); } else if (topicRes->GetErrorCode() == (ui32)NPersQueue::NErrorCode::OK) { - for (const auto& partition : SchemeEntry.PQGroupInfo->Description.GetPartitions()) { - const ui32 partitionIndex = partition.GetPartitionId(); + for (const auto& partition : SchemeEntry.PQGroupInfo->Description.GetPartitions()) { + const ui32 partitionIndex = partition.GetPartitionId(); partitionToResp[partitionIndex] = index++; auto res = topicRes->AddPartitionResult(); res->SetPartition(partitionIndex); diff --git a/ydb/core/client/server/msgbus_server_pq_metarequest.h b/ydb/core/client/server/msgbus_server_pq_metarequest.h index deb696913f..1d56c28dc0 100644 --- a/ydb/core/client/server/msgbus_server_pq_metarequest.h +++ b/ydb/core/client/server/msgbus_server_pq_metarequest.h @@ -7,20 +7,20 @@ namespace NMsgBusProxy { // // GetTopicMetadata command // -// +// class TPersQueueGetTopicMetadataProcessor : public TPersQueueBaseRequestProcessor { public: TPersQueueGetTopicMetadataProcessor(const NKikimrClient::TPersQueueRequest& request, const TActorId& schemeCache); private: - THolder<IActor> CreateTopicSubactor(const TSchemeEntry& topicEntry, const TString& name) override; + THolder<IActor> CreateTopicSubactor(const TSchemeEntry& topicEntry, const TString& name) override; }; -class TPersQueueGetTopicMetadataTopicWorker : public TReplierToParent<TTopicInfoBasedActor> { +class TPersQueueGetTopicMetadataTopicWorker : public TReplierToParent<TTopicInfoBasedActor> { public: - TPersQueueGetTopicMetadataTopicWorker(const TActorId& parent, const TSchemeEntry& topicEntry, const TString& name); + TPersQueueGetTopicMetadataTopicWorker(const TActorId& parent, const TSchemeEntry& topicEntry, const TString& name); - void BootstrapImpl(const TActorContext& ctx) override; + void BootstrapImpl(const TActorContext& ctx) override; void Answer(const TActorContext& ctx, EResponseStatus status, NPersQueue::NErrorCode::EErrorCode code, const TString& errorReason) override; }; @@ -31,23 +31,23 @@ public: class TPersQueueGetPartitionOffsetsProcessor : public TPersQueueBaseRequestProcessor { public: - TPersQueueGetPartitionOffsetsProcessor(const NKikimrClient::TPersQueueRequest& request, const TActorId& metaCacheId); + TPersQueueGetPartitionOffsetsProcessor(const NKikimrClient::TPersQueueRequest& request, const TActorId& metaCacheId); private: - THolder<IActor> CreateTopicSubactor(const TSchemeEntry& topicEntry, const TString& name) override; + THolder<IActor> CreateTopicSubactor(const TSchemeEntry& topicEntry, const TString& name) override; private: THashMap<TString, std::shared_ptr<THashSet<ui64>>> PartitionsToRequest; }; -class TPersQueueGetPartitionOffsetsTopicWorker : public TReplierToParent<TPipesWaiterActor<TTopicInfoBasedActor, TEvPersQueue::TEvOffsetsResponse>> { +class TPersQueueGetPartitionOffsetsTopicWorker : public TReplierToParent<TPipesWaiterActor<TTopicInfoBasedActor, TEvPersQueue::TEvOffsetsResponse>> { public: TPersQueueGetPartitionOffsetsTopicWorker(const TActorId& parent, - const TSchemeEntry& topicEntry, const TString& name, + const TSchemeEntry& topicEntry, const TString& name, const std::shared_ptr<THashSet<ui64>>& partitionsToRequest, const std::shared_ptr<const NKikimrClient::TPersQueueRequest>& requestProto); - void BootstrapImpl(const TActorContext& ctx) override; + void BootstrapImpl(const TActorContext& ctx) override; bool OnPipeEventsAreReady(const TActorContext& ctx) override; void Answer(const TActorContext& ctx, EResponseStatus status, NPersQueue::NErrorCode::EErrorCode code, const TString& errorReason) override; @@ -66,21 +66,21 @@ public: TPersQueueGetPartitionStatusProcessor(const NKikimrClient::TPersQueueRequest& request, const TActorId& schemeCache); private: - THolder<IActor> CreateTopicSubactor(const TSchemeEntry& topicEntry, const TString& name) override; + THolder<IActor> CreateTopicSubactor(const TSchemeEntry& topicEntry, const TString& name) override; private: THashMap<TString, std::shared_ptr<THashSet<ui64>>> PartitionsToRequest; }; -class TPersQueueGetPartitionStatusTopicWorker : public TReplierToParent<TPipesWaiterActor<TTopicInfoBasedActor, TEvPersQueue::TEvStatusResponse>> { +class TPersQueueGetPartitionStatusTopicWorker : public TReplierToParent<TPipesWaiterActor<TTopicInfoBasedActor, TEvPersQueue::TEvStatusResponse>> { public: TPersQueueGetPartitionStatusTopicWorker(const TActorId& parent, - const TTopicInfoBasedActor::TSchemeEntry& topicEntry, - const TString& name, + const TTopicInfoBasedActor::TSchemeEntry& topicEntry, + const TString& name, const std::shared_ptr<THashSet<ui64>>& partitionsToRequest, const std::shared_ptr<const NKikimrClient::TPersQueueRequest>& requestProto); - void BootstrapImpl(const TActorContext& ctx) override; + void BootstrapImpl(const TActorContext& ctx) override; bool OnPipeEventsAreReady(const TActorContext& ctx) override; void Answer(const TActorContext& ctx, EResponseStatus status, NPersQueue::NErrorCode::EErrorCode code, const TString& errorReason) override; @@ -99,21 +99,21 @@ public: TPersQueueGetPartitionLocationsProcessor(const NKikimrClient::TPersQueueRequest& request, const TActorId& schemeCache); private: - THolder<IActor> CreateTopicSubactor(const TSchemeEntry& topicEntry, const TString& name) override; + THolder<IActor> CreateTopicSubactor(const TSchemeEntry& topicEntry, const TString& name) override; private: THashMap<TString, std::shared_ptr<THashSet<ui64>>> PartitionsToRequest; }; -class TPersQueueGetPartitionLocationsTopicWorker : public TReplierToParent<TPipesWaiterActor<TTopicInfoBasedActor, TEvTabletPipe::TEvClientConnected>> { +class TPersQueueGetPartitionLocationsTopicWorker : public TReplierToParent<TPipesWaiterActor<TTopicInfoBasedActor, TEvTabletPipe::TEvClientConnected>> { public: TPersQueueGetPartitionLocationsTopicWorker(const TActorId& parent, - const TTopicInfoBasedActor::TSchemeEntry& topicEntry, const TString& name, + const TTopicInfoBasedActor::TSchemeEntry& topicEntry, const TString& name, const std::shared_ptr<THashSet<ui64>>& partitionsToRequest, const std::shared_ptr<const NKikimrClient::TPersQueueRequest>& requestProto, std::shared_ptr<const TPersQueueBaseRequestProcessor::TNodesInfo> nodesInfo); - void BootstrapImpl(const TActorContext& ctx) override; + void BootstrapImpl(const TActorContext& ctx) override; bool OnPipeEventsAreReady(const TActorContext& ctx) override; void Answer(const TActorContext& ctx, EResponseStatus status, NPersQueue::NErrorCode::EErrorCode code, const TString& errorReason) override; @@ -169,7 +169,7 @@ public: private: - THolder<IActor> CreateTopicSubactor(const TSchemeEntry& topicEntry, const TString& name) override; + THolder<IActor> CreateTopicSubactor(const TSchemeEntry& topicEntry, const TString& name) override; THolder<IActor> CreateSessionsSubactor(const THashMap<TString, TActorId>&& readSessions); std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> PQReadSessionsInfoWorkerFactory; @@ -178,14 +178,14 @@ private: }; -class TPersQueueGetReadSessionsInfoTopicWorker : public TReplierToParent<TPipesWaiterActor<TTopicInfoBasedActor, TEvPersQueue::TEvOffsetsResponse>> { +class TPersQueueGetReadSessionsInfoTopicWorker : public TReplierToParent<TPipesWaiterActor<TTopicInfoBasedActor, TEvPersQueue::TEvOffsetsResponse>> { public: TPersQueueGetReadSessionsInfoTopicWorker(const TActorId& parent, - const TTopicInfoBasedActor::TSchemeEntry& topicEntry, const TString& name, + const TTopicInfoBasedActor::TSchemeEntry& topicEntry, const TString& name, const std::shared_ptr<const NKikimrClient::TPersQueueRequest>& requestProto, std::shared_ptr<const TPersQueueBaseRequestProcessor::TNodesInfo> nodesInfo); - void BootstrapImpl(const TActorContext& ctx) override; + void BootstrapImpl(const TActorContext& ctx) override; void Answer(const TActorContext& ctx, EResponseStatus status, NPersQueue::NErrorCode::EErrorCode code, const TString& errorReason) override; bool OnPipeEventsAreReady(const TActorContext& ctx) override; diff --git a/ydb/core/client/server/msgbus_server_pq_metarequest_ut.cpp b/ydb/core/client/server/msgbus_server_pq_metarequest_ut.cpp index 5271d241f8..237bba147b 100644 --- a/ydb/core/client/server/msgbus_server_pq_metarequest_ut.cpp +++ b/ydb/core/client/server/msgbus_server_pq_metarequest_ut.cpp @@ -25,9 +25,9 @@ namespace NKikimr { namespace NMsgBusProxy { using namespace testing; -using namespace NSchemeCache; +using namespace NSchemeCache; -void FillValidTopicRequest(NProtoBuf::RepeatedPtrField<::NKikimrClient::TPersQueueMetaRequest::TTopicRequest>& request, ui64 topicsCount); +void FillValidTopicRequest(NProtoBuf::RepeatedPtrField<::NKikimrClient::TPersQueueMetaRequest::TTopicRequest>& request, ui64 topicsCount); void MakeEmptyTopic(NProtoBuf::RepeatedPtrField<::NKikimrClient::TPersQueueMetaRequest::TTopicRequest>& request); void MakeDuplicatedTopic(NProtoBuf::RepeatedPtrField<::NKikimrClient::TPersQueueMetaRequest::TTopicRequest>& request); void MakeDuplicatedPartition(NProtoBuf::RepeatedPtrField<::NKikimrClient::TPersQueueMetaRequest::TTopicRequest>& request); @@ -314,7 +314,7 @@ protected: void AssertFailedResponse(NPersQueue::NErrorCode::EErrorCode code, const THashSet<TString>& markers = {}, EResponseStatus status = MSTATUS_ERROR) { const TEvPersQueue::TEvResponse* resp = GetResponse(); - Cerr << "Assert failed: Check response: " << resp->Record << Endl; + Cerr << "Assert failed: Check response: " << resp->Record << Endl; UNIT_ASSERT(resp != nullptr); UNIT_ASSERT_C(resp->Record.HasStatus(), "Response: " << resp->Record); UNIT_ASSERT_UNEQUAL_C(resp->Record.GetStatus(), 1, "Response: " << resp->Record); @@ -437,91 +437,91 @@ class TMessageBusServerPersQueueRequestCommonTest: public TMessageBusServerPersQ public: #define COMMON_TESTS_LIST() \ UNIT_TEST(HandlesTimeout) \ - UNIT_TEST(FailsOnFailedGetAllTopicsRequest) \ - UNIT_TEST(FailsOnBadRootStatusInGetNodeRequest) \ - UNIT_TEST(FailesOnNotATopic) \ + UNIT_TEST(FailsOnFailedGetAllTopicsRequest) \ + UNIT_TEST(FailsOnBadRootStatusInGetNodeRequest) \ + UNIT_TEST(FailesOnNotATopic) \ UNIT_TEST(FailsOnNotOkStatusInGetNodeRequest) \ UNIT_TEST(FailsOnNoBalancerInGetNodeRequest) \ UNIT_TEST(FailsOnZeroBalancerTabletIdInGetNodeRequest) \ UNIT_TEST(FailsOnBalancerDescribeResultFailureWhenTopicsAreGivenExplicitly) \ /**/ - virtual NKikimrClient::TPersQueueRequest MakeValidRequest(ui64 topicsCount = 2) = 0; + virtual NKikimrClient::TPersQueueRequest MakeValidRequest(ui64 topicsCount = 2) = 0; - void SetBalancerId(TSchemeCacheNavigate::TResultSet& resultSet, ui64 index, const TMaybe<ui64>& tabletId) { - auto* newInfo = new TSchemeCacheNavigate::TPQGroupInfo(*resultSet[index].PQGroupInfo); - if (tabletId.Defined()) { - newInfo->Description.SetBalancerTabletID(*tabletId); - } else { - newInfo->Description.ClearBalancerTabletID(); + void SetBalancerId(TSchemeCacheNavigate::TResultSet& resultSet, ui64 index, const TMaybe<ui64>& tabletId) { + auto* newInfo = new TSchemeCacheNavigate::TPQGroupInfo(*resultSet[index].PQGroupInfo); + if (tabletId.Defined()) { + newInfo->Description.SetBalancerTabletID(*tabletId); + } else { + newInfo->Description.ClearBalancerTabletID(); } - resultSet[index].PQGroupInfo.Reset(newInfo); - } - - TSchemeCacheNavigate::TEntry MakeEntry( - ui64 topicId, - TSchemeCacheNavigate::EStatus status = TSchemeCacheNavigate::EStatus::Ok, - TSchemeCacheNavigate::EKind kind = TSchemeCacheNavigate::KindTopic, - bool makePQDescription = true - ) { - TSchemeCacheNavigate::TEntry entry; - entry.Status = status; - entry.Kind = kind; - entry.Path = {"Root", "PQ"}; - if (status != TSchemeCacheNavigate::EStatus::Ok || kind != TSchemeCacheNavigate::KindTopic - || !makePQDescription - ) { - return entry; + resultSet[index].PQGroupInfo.Reset(newInfo); + } + + TSchemeCacheNavigate::TEntry MakeEntry( + ui64 topicId, + TSchemeCacheNavigate::EStatus status = TSchemeCacheNavigate::EStatus::Ok, + TSchemeCacheNavigate::EKind kind = TSchemeCacheNavigate::KindTopic, + bool makePQDescription = true + ) { + TSchemeCacheNavigate::TEntry entry; + entry.Status = status; + entry.Kind = kind; + entry.Path = {"Root", "PQ"}; + if (status != TSchemeCacheNavigate::EStatus::Ok || kind != TSchemeCacheNavigate::KindTopic + || !makePQDescription + ) { + return entry; } - auto *pqInfo = new TSchemeCacheNavigate::TPQGroupInfo(); - pqInfo->Kind = TSchemeCacheNavigate::KindTopic; - auto &descr = pqInfo->Description; - switch (topicId) { - case 1: - descr.SetName("topic1"); - descr.SetBalancerTabletID(MakeTabletID(0, 0, 100)); - descr.SetAlterVersion(42); - descr.SetPartitionPerTablet(1); - break; - case 2: - descr.SetName("topic2"); - descr.SetBalancerTabletID(MakeTabletID(0, 0, 200)); - descr.SetAlterVersion(5); - descr.SetPartitionPerTablet(3); - break; - default: - UNIT_FAIL(""); - } - for (auto i = 0u; i < descr.GetPartitionPerTablet(); i++) { - auto* part = descr.AddPartitions(); - part->SetPartitionId(i); - part->SetTabletId(MakeTabletID(0, 0, topicId * 100 + 1 + i)); - } - entry.PQGroupInfo.Reset(pqInfo); - return entry; - } - - TSchemeCacheNavigate::TResultSet MakeResultSet(bool valid = true) { - TSchemeCacheNavigate::TResultSet resultSet; - if (valid) { - resultSet.emplace_back(std::move( - MakeEntry(1) - )); - resultSet.emplace_back(std::move( - MakeEntry(2) - )); + auto *pqInfo = new TSchemeCacheNavigate::TPQGroupInfo(); + pqInfo->Kind = TSchemeCacheNavigate::KindTopic; + auto &descr = pqInfo->Description; + switch (topicId) { + case 1: + descr.SetName("topic1"); + descr.SetBalancerTabletID(MakeTabletID(0, 0, 100)); + descr.SetAlterVersion(42); + descr.SetPartitionPerTablet(1); + break; + case 2: + descr.SetName("topic2"); + descr.SetBalancerTabletID(MakeTabletID(0, 0, 200)); + descr.SetAlterVersion(5); + descr.SetPartitionPerTablet(3); + break; + default: + UNIT_FAIL(""); + } + for (auto i = 0u; i < descr.GetPartitionPerTablet(); i++) { + auto* part = descr.AddPartitions(); + part->SetPartitionId(i); + part->SetTabletId(MakeTabletID(0, 0, topicId * 100 + 1 + i)); + } + entry.PQGroupInfo.Reset(pqInfo); + return entry; + } + + TSchemeCacheNavigate::TResultSet MakeResultSet(bool valid = true) { + TSchemeCacheNavigate::TResultSet resultSet; + if (valid) { + resultSet.emplace_back(std::move( + MakeEntry(1) + )); + resultSet.emplace_back(std::move( + MakeEntry(2) + )); } - return resultSet; + return resultSet; } void HandlesTimeout() { - EXPECT_CALL(GetMockPQMetaCache(), HandleDescribeAllTopics(_, _)); // gets request and doesn't reply + EXPECT_CALL(GetMockPQMetaCache(), HandleDescribeAllTopics(_, _)); // gets request and doesn't reply NKikimrClient::TPersQueueRequest request = MakeValidRequest(); RegisterActor(request); Runtime->EnableScheduleForActor(Actor->SelfId()); TDispatchOptions options; - options.FinalEvents.emplace_back([](IEventHandle& h) { return h.Type == TEvPqMetaCache::TEvDescribeAllTopicsRequest::EventType; }, 1); + options.FinalEvents.emplace_back([](IEventHandle& h) { return h.Type == TEvPqMetaCache::TEvDescribeAllTopicsRequest::EventType; }, 1); Runtime->DispatchEvents(options); Runtime->UpdateCurrentTime(Runtime->GetCurrentTime() + TDuration::MilliSeconds(90000 + 1)); @@ -530,8 +530,8 @@ public: AssertFailedResponse(NPersQueue::NErrorCode::ERROR, {"Marker# PQ11", "Marker# PQ16"}, MSTATUS_TIMEOUT); } - void FailsOnFailedGetAllTopicsRequest() { - GetMockPQMetaCache().SetAllTopicsAnswer(false); + void FailsOnFailedGetAllTopicsRequest() { + GetMockPQMetaCache().SetAllTopicsAnswer(false); NKikimrClient::TPersQueueRequest request = MakeValidRequest(); RegisterActor(request); @@ -541,61 +541,61 @@ public: } void FailsOnNotOkStatusInGetNodeRequest() { - auto entry = MakeEntry(1); - entry.Status = TSchemeCacheNavigate::EStatus::PathErrorUnknown; + auto entry = MakeEntry(1); + entry.Status = TSchemeCacheNavigate::EStatus::PathErrorUnknown; - GetMockPQMetaCache().SetAllTopicsAnswer(true, TSchemeCacheNavigate::TResultSet{entry}); - - NKikimrClient::TPersQueueRequest request = MakeValidRequest(1); + GetMockPQMetaCache().SetAllTopicsAnswer(true, TSchemeCacheNavigate::TResultSet{entry}); + + NKikimrClient::TPersQueueRequest request = MakeValidRequest(1); RegisterActor(request); GrabResponseEvent(); - AssertFailedResponse(NPersQueue::NErrorCode::UNKNOWN_TOPIC, "Marker# PQ150"); + AssertFailedResponse(NPersQueue::NErrorCode::UNKNOWN_TOPIC, "Marker# PQ150"); } - void FailsOnBadRootStatusInGetNodeRequest() { - auto resultSet = MakeResultSet(); - resultSet[0].Status = ESchemeStatus::RootUnknown; + void FailsOnBadRootStatusInGetNodeRequest() { + auto resultSet = MakeResultSet(); + resultSet[0].Status = ESchemeStatus::RootUnknown; + + GetMockPQMetaCache().SetAllTopicsAnswer(true, std::move(resultSet)); - GetMockPQMetaCache().SetAllTopicsAnswer(true, std::move(resultSet)); - NKikimrClient::TPersQueueRequest request = MakeValidRequest(); RegisterActor(request); GrabResponseEvent(); - AssertFailedResponse(NPersQueue::NErrorCode::UNKNOWN_TOPIC, {"Marker# PQ1", "Marker# PQ14"}); + AssertFailedResponse(NPersQueue::NErrorCode::UNKNOWN_TOPIC, {"Marker# PQ1", "Marker# PQ14"}); } - void FailesOnNotATopic() { - auto resultSet = MakeResultSet(); - resultSet[1].Kind = TSchemeCacheNavigate::KindPath; - resultSet[1].PQGroupInfo = nullptr; + void FailesOnNotATopic() { + auto resultSet = MakeResultSet(); + resultSet[1].Kind = TSchemeCacheNavigate::KindPath; + resultSet[1].PQGroupInfo = nullptr; + + GetMockPQMetaCache().SetAllTopicsAnswer(true, std::move(resultSet)); - GetMockPQMetaCache().SetAllTopicsAnswer(true, std::move(resultSet)); - NKikimrClient::TPersQueueRequest request = MakeValidRequest(); RegisterActor(request); GrabResponseEvent(); - AssertFailedResponse(NPersQueue::NErrorCode::UNKNOWN_TOPIC, {"Marker# PQ95", "Marker# PQ13"}); + AssertFailedResponse(NPersQueue::NErrorCode::UNKNOWN_TOPIC, {"Marker# PQ95", "Marker# PQ13"}); } void FailsOnNoBalancerInGetNodeRequest() { - auto resultSet = MakeResultSet(); - SetBalancerId(resultSet, 0, Nothing()); - GetMockPQMetaCache().SetAllTopicsAnswer(true, std::move(resultSet)); + auto resultSet = MakeResultSet(); + SetBalancerId(resultSet, 0, Nothing()); + GetMockPQMetaCache().SetAllTopicsAnswer(true, std::move(resultSet)); NKikimrClient::TPersQueueRequest request = MakeValidRequest(); RegisterActor(request); GrabResponseEvent(); - AssertFailedResponse(NPersQueue::NErrorCode::UNKNOWN_TOPIC, {"Marker# PQ93", "Marker# PQ193"}); + AssertFailedResponse(NPersQueue::NErrorCode::UNKNOWN_TOPIC, {"Marker# PQ93", "Marker# PQ193"}); } void FailsOnZeroBalancerTabletIdInGetNodeRequest() { - auto resultSet = MakeResultSet(); - SetBalancerId(resultSet, 0, 0); - GetMockPQMetaCache().SetAllTopicsAnswer(true, std::move(resultSet)); + auto resultSet = MakeResultSet(); + SetBalancerId(resultSet, 0, 0); + GetMockPQMetaCache().SetAllTopicsAnswer(true, std::move(resultSet)); NKikimrClient::TPersQueueRequest request = MakeValidRequest(); RegisterActor(request); @@ -605,17 +605,17 @@ public: } void FailsOnBalancerDescribeResultFailureWhenTopicsAreGivenExplicitly() { - auto resultSet = MakeResultSet(); - resultSet[1].Status = TSchemeCacheNavigate::EStatus::LookupError; - //SetBalancerId(resultSet, 1, 0); - GetMockPQMetaCache().SetAllTopicsAnswer(true, resultSet); + auto resultSet = MakeResultSet(); + resultSet[1].Status = TSchemeCacheNavigate::EStatus::LookupError; + //SetBalancerId(resultSet, 1, 0); + GetMockPQMetaCache().SetAllTopicsAnswer(true, resultSet); NKikimrClient::TPersQueueRequest request = MakeValidRequest(); RegisterActor(request); GrabResponseEvent(); - AssertFailedResponse(NPersQueue::NErrorCode::ERROR, "Marker# PQ1"); - + AssertFailedResponse(NPersQueue::NErrorCode::ERROR, "Marker# PQ1"); + } // Implementation details for test with pipe disconnection for inheritance @@ -627,7 +627,7 @@ public: template <class TResponseEvent> void HandlesPipeDisconnectionImpl(EDisconnectionMode disconnectionMode, std::function<void(EDisconnectionMode disconnectionMode)> dataValidationFunction, bool requestTheWholeTopic = false) { - GetMockPQMetaCache().SetAllTopicsAnswer(true, std::forward<TSchemeCacheNavigate::TResultSet>(MakeResultSet())); + GetMockPQMetaCache().SetAllTopicsAnswer(true, std::forward<TSchemeCacheNavigate::TResultSet>(MakeResultSet())); PrepareBalancer("topic1", MakeTabletID(0, 0, 100), {{1, MakeTabletID(0, 0, 101)}}); PreparePQTablet("topic1", MakeTabletID(0, 0, 101), {0}); @@ -725,14 +725,14 @@ public: UNIT_TEST(SuccessfullyReplies) UNIT_TEST_SUITE_END(); - NKikimrClient::TPersQueueRequest MakeValidRequest(ui64 topicsCount = 2) override { + NKikimrClient::TPersQueueRequest MakeValidRequest(ui64 topicsCount = 2) override { NKikimrClient::TPersQueueRequest persQueueRequest; persQueueRequest.SetTicket("client_id@" BUILTIN_ACL_DOMAIN); auto& req = *persQueueRequest.MutableMetaRequest()->MutableCmdGetTopicMetadata(); req.AddTopic("topic1"); - if (topicsCount > 1) - req.AddTopic("topic2"); + if (topicsCount > 1) + req.AddTopic("topic2"); return persQueueRequest; } @@ -746,7 +746,7 @@ public: } void SuccessfullyReplies() { - GetMockPQMetaCache().SetAllTopicsAnswer(true, MakeResultSet()); + GetMockPQMetaCache().SetAllTopicsAnswer(true, MakeResultSet()); NKikimrClient::TPersQueueRequest req = MakeValidRequest(); RegisterActor(req); @@ -777,13 +777,13 @@ public: } }; -void FillValidTopicRequest(NProtoBuf::RepeatedPtrField<::NKikimrClient::TPersQueueMetaRequest::TTopicRequest>& request, ui64 topicsCount = 2) { +void FillValidTopicRequest(NProtoBuf::RepeatedPtrField<::NKikimrClient::TPersQueueMetaRequest::TTopicRequest>& request, ui64 topicsCount = 2) { { auto& topic1 = *request.Add(); topic1.SetTopic("topic1"); } - if (topicsCount > 1){ + if (topicsCount > 1){ auto& topic2 = *request.Add(); topic2.SetTopic("topic2"); topic2.AddPartition(1); @@ -819,12 +819,12 @@ public: UNIT_TEST(HandlesPipeDisconnection_AnswerDoesNotArrive) UNIT_TEST_SUITE_END(); - NKikimrClient::TPersQueueRequest MakeValidRequest(ui64 topicsCount = 2) override { + NKikimrClient::TPersQueueRequest MakeValidRequest(ui64 topicsCount = 2) override { NKikimrClient::TPersQueueRequest persQueueRequest; persQueueRequest.SetTicket("client_id@" BUILTIN_ACL_DOMAIN); auto& req = *persQueueRequest.MutableMetaRequest()->MutableCmdGetPartitionLocations(); - FillValidTopicRequest(*req.MutableTopicRequest(), topicsCount); + FillValidTopicRequest(*req.MutableTopicRequest(), topicsCount); return persQueueRequest; } @@ -856,7 +856,7 @@ public: } void SuccessfullyPassesResponsesFromTablets() { - GetMockPQMetaCache().SetAllTopicsAnswer(true, MakeResultSet()); + GetMockPQMetaCache().SetAllTopicsAnswer(true, MakeResultSet()); PrepareBalancer("topic1", MakeTabletID(0, 0, 100), {{1, MakeTabletID(0, 0, 101)}}); PreparePQTablet("topic1", MakeTabletID(0, 0, 101), {0}); @@ -982,12 +982,12 @@ public: UNIT_TEST(HandlesPipeDisconnection_AnswerDoesNotArrive) UNIT_TEST_SUITE_END(); - NKikimrClient::TPersQueueRequest MakeValidRequest(ui64 topicsCount = 2) override { + NKikimrClient::TPersQueueRequest MakeValidRequest(ui64 topicsCount = 2) override { NKikimrClient::TPersQueueRequest persQueueRequest; persQueueRequest.SetTicket("client_id@" BUILTIN_ACL_DOMAIN); auto& req = *persQueueRequest.MutableMetaRequest()->MutableCmdGetPartitionOffsets(); - FillValidTopicRequest(*req.MutableTopicRequest(), topicsCount); + FillValidTopicRequest(*req.MutableTopicRequest(), topicsCount); return persQueueRequest; } @@ -1019,7 +1019,7 @@ public: } void SuccessfullyPassesResponsesFromTablets() { - GetMockPQMetaCache().SetAllTopicsAnswer(true, MakeResultSet()); + GetMockPQMetaCache().SetAllTopicsAnswer(true, MakeResultSet()); PrepareBalancer("topic1", MakeTabletID(0, 0, 100), {{1, MakeTabletID(0, 0, 101)}}); PreparePQTablet("topic1", MakeTabletID(0, 0, 101), {0}); @@ -1143,12 +1143,12 @@ public: UNIT_TEST(HandlesPipeDisconnection_AnswerDoesNotArrive) UNIT_TEST_SUITE_END(); - NKikimrClient::TPersQueueRequest MakeValidRequest(ui64 topicsCount = 2) override { + NKikimrClient::TPersQueueRequest MakeValidRequest(ui64 topicsCount = 2) override { NKikimrClient::TPersQueueRequest persQueueRequest; persQueueRequest.SetTicket("client_id@" BUILTIN_ACL_DOMAIN); auto& req = *persQueueRequest.MutableMetaRequest()->MutableCmdGetPartitionStatus(); - FillValidTopicRequest(*req.MutableTopicRequest(), topicsCount); + FillValidTopicRequest(*req.MutableTopicRequest(), topicsCount); return persQueueRequest; } @@ -1180,7 +1180,7 @@ public: } void SuccessfullyPassesResponsesFromTablets() { - GetMockPQMetaCache().SetAllTopicsAnswer(true, MakeResultSet()); + GetMockPQMetaCache().SetAllTopicsAnswer(true, MakeResultSet()); PrepareBalancer("topic1", MakeTabletID(0, 0, 100), {{1, MakeTabletID(0, 0, 101)}}); PreparePQTablet("topic1", MakeTabletID(0, 0, 101), {0}); @@ -1296,15 +1296,15 @@ public: UNIT_TEST(HandlesPipeDisconnection_AnswerDoesNotArrive) UNIT_TEST_SUITE_END(); - NKikimrClient::TPersQueueRequest MakeValidRequest(ui64 topicsCount = 2) override { + NKikimrClient::TPersQueueRequest MakeValidRequest(ui64 topicsCount = 2) override { NKikimrClient::TPersQueueRequest persQueueRequest; persQueueRequest.SetTicket("client_id@" BUILTIN_ACL_DOMAIN); auto& req = *persQueueRequest.MutableMetaRequest()->MutableCmdGetReadSessionsInfo(); req.SetClientId("client_id"); req.AddTopic("topic1"); - if (topicsCount > 1) - req.AddTopic("topic2"); + if (topicsCount > 1) + req.AddTopic("topic2"); return persQueueRequest; } @@ -1327,7 +1327,7 @@ public: } void SuccessfullyPassesResponsesFromTablets() { - GetMockPQMetaCache().SetAllTopicsAnswer(true, MakeResultSet()); + GetMockPQMetaCache().SetAllTopicsAnswer(true, MakeResultSet()); PrepareBalancer("topic1", MakeTabletID(0, 0, 100), {{1, MakeTabletID(0, 0, 101)}}); PreparePQTablet("topic1", MakeTabletID(0, 0, 101), {0}); diff --git a/ydb/core/client/server/msgbus_server_proxy.cpp b/ydb/core/client/server/msgbus_server_proxy.cpp index 63081b72a8..7f3146d21a 100644 --- a/ydb/core/client/server/msgbus_server_proxy.cpp +++ b/ydb/core/client/server/msgbus_server_proxy.cpp @@ -176,7 +176,7 @@ void TMessageBusServerProxy::Bootstrap(const TActorContext& ctx) { auto cacheConfig = MakeIntrusive<NSchemeCache::TSchemeCacheConfig>(AppData(ctx), SchemeCacheCounters); SchemeCache = ctx.ExecutorThread.RegisterActor(CreateSchemeBoardSchemeCache(cacheConfig.Get())); - PqMetaCache = CreatePersQueueMetaCacheV2Id(); + PqMetaCache = CreatePersQueueMetaCacheV2Id(); if (Server) { Server->InitSession(ctx.ExecutorThread.ActorSystem, ctx.SelfID); diff --git a/ydb/core/client/server/pq_metacache_v2_ut.cpp b/ydb/core/client/server/pq_metacache_v2_ut.cpp index de48476681..f02fd420c9 100644 --- a/ydb/core/client/server/pq_metacache_v2_ut.cpp +++ b/ydb/core/client/server/pq_metacache_v2_ut.cpp @@ -1,5 +1,5 @@ -#include "msgbus_server_pq_metacache.h" - +#include "msgbus_server_pq_metacache.h" + #include <ydb/core/testlib/basics/runtime.h> #include <ydb/core/testlib/test_client.h> #include <ydb/core/testlib/test_pq_client.h> @@ -7,274 +7,274 @@ #include <ydb/core/tx/scheme_board/cache.h> #include <ydb/core/tx/scheme_cache/scheme_cache.h> #include <ydb/library/persqueue/topic_parser/topic_parser.h> -#include <library/cpp/testing/unittest/registar.h> - -namespace NKikimr::NMsgBusProxy { -using namespace NPqMetaCacheV2; -using namespace Tests; -using namespace NYdb; -using namespace NYdb::NTable; -using namespace NKikimr::NSchemeCache; -namespace NTests { - -class TPqMetaCacheV2Test: public TTestBase { - void SetUp() override { - TTestBase::SetUp(); - TPortManager pm{}; - MainServerPort = pm.GetPort(2134); - GrpcServerPort = pm.GetPort(2135); - NKikimr::NPersQueueTests::TServerSettings settings = NKikimr::NPersQueueTests::PQSettings(MainServerPort); - settings.SetDomainName("Root"); - settings.SetDomain(0); - settings.SetUseRealThreads(true); - Server = new NKikimr::Tests::TServer(settings); - - Server->EnableGRpc(NGrpc::TServerOptions().SetHost("localhost").SetPort(GrpcServerPort)); - auto* runtime = Server->GetRuntime(); - //runtime->SetLogPriority(NKikimrServices::PERSQUEUE, NActors::NLog::PRI_DEBUG); -// runtime->SetLogPriority(NKikimrServices::KQP_PROXY, NActors::NLog::PRI_DEBUG); -// runtime->SetLogPriority(NKikimrServices::KQP_COMPILE_SERVICE, NActors::NLog::PRI_DEBUG); - runtime->SetLogPriority(NKikimrServices::PQ_METACACHE, NActors::NLog::PRI_DEBUG); - runtime->SetLogPriority(NKikimrServices::PQ_WRITE_PROXY, NActors::NLog::PRI_EMERG); - runtime->SetLogPriority(NKikimrServices::PERSQUEUE_CLUSTER_TRACKER, NActors::NLog::PRI_EMERG); - - Client = MakeHolder<NKikimr::NPersQueueTests::TFlatMsgBusPQClient>(settings, GrpcServerPort); - - Client->InitRootScheme(); - - NYdb::TDriverConfig driverCfg; - driverCfg.SetEndpoint(TStringBuilder() << "localhost:" << GrpcServerPort); - YdbDriver = new NYdb::TDriver(driverCfg); - TableClient = new NYdb::NTable::TTableClient(*YdbDriver); - PQClient = new NYdb::NPersQueue::TPersQueueClient(*YdbDriver); - - Client->MkDir("/Root", "PQ"); - Client->MkDir("/Root/PQ", "Config"); - Client->MkDir("/Root/PQ/Config", "V2"); - - auto tableDesc = TTableBuilder() - .AddNullableColumn("path", EPrimitiveType::Utf8) - .AddNullableColumn("dc", EPrimitiveType::Utf8) - .SetPrimaryKeyColumns({"path", "dc"}) - .Build(); - - auto session = CheckYdbResult(std::move(TableClient->CreateSession())).GetSession(); - CheckYdbResult(session.CreateTable("/Root/PQ/Config/V2/Topics", std::move(tableDesc))); - - tableDesc = TTableBuilder() - .AddNullableColumn("name", EPrimitiveType::String) - .AddNullableColumn("version", EPrimitiveType::Uint64) - .SetPrimaryKeyColumns({"name", "version"}) - .Build(); - CheckYdbResult(session.CreateTable("/Root/PQ/Config/V2/Versions", std::move(tableDesc))); - - EdgeActorId = runtime->AllocateEdgeActor(); - auto config = MakeIntrusive<TSchemeCacheConfig>(); - config->Counters = new NMonitoring::TDynamicCounters; +#include <library/cpp/testing/unittest/registar.h> + +namespace NKikimr::NMsgBusProxy { +using namespace NPqMetaCacheV2; +using namespace Tests; +using namespace NYdb; +using namespace NYdb::NTable; +using namespace NKikimr::NSchemeCache; +namespace NTests { + +class TPqMetaCacheV2Test: public TTestBase { + void SetUp() override { + TTestBase::SetUp(); + TPortManager pm{}; + MainServerPort = pm.GetPort(2134); + GrpcServerPort = pm.GetPort(2135); + NKikimr::NPersQueueTests::TServerSettings settings = NKikimr::NPersQueueTests::PQSettings(MainServerPort); + settings.SetDomainName("Root"); + settings.SetDomain(0); + settings.SetUseRealThreads(true); + Server = new NKikimr::Tests::TServer(settings); + + Server->EnableGRpc(NGrpc::TServerOptions().SetHost("localhost").SetPort(GrpcServerPort)); + auto* runtime = Server->GetRuntime(); + //runtime->SetLogPriority(NKikimrServices::PERSQUEUE, NActors::NLog::PRI_DEBUG); +// runtime->SetLogPriority(NKikimrServices::KQP_PROXY, NActors::NLog::PRI_DEBUG); +// runtime->SetLogPriority(NKikimrServices::KQP_COMPILE_SERVICE, NActors::NLog::PRI_DEBUG); + runtime->SetLogPriority(NKikimrServices::PQ_METACACHE, NActors::NLog::PRI_DEBUG); + runtime->SetLogPriority(NKikimrServices::PQ_WRITE_PROXY, NActors::NLog::PRI_EMERG); + runtime->SetLogPriority(NKikimrServices::PERSQUEUE_CLUSTER_TRACKER, NActors::NLog::PRI_EMERG); + + Client = MakeHolder<NKikimr::NPersQueueTests::TFlatMsgBusPQClient>(settings, GrpcServerPort); + + Client->InitRootScheme(); + + NYdb::TDriverConfig driverCfg; + driverCfg.SetEndpoint(TStringBuilder() << "localhost:" << GrpcServerPort); + YdbDriver = new NYdb::TDriver(driverCfg); + TableClient = new NYdb::NTable::TTableClient(*YdbDriver); + PQClient = new NYdb::NPersQueue::TPersQueueClient(*YdbDriver); + + Client->MkDir("/Root", "PQ"); + Client->MkDir("/Root/PQ", "Config"); + Client->MkDir("/Root/PQ/Config", "V2"); + + auto tableDesc = TTableBuilder() + .AddNullableColumn("path", EPrimitiveType::Utf8) + .AddNullableColumn("dc", EPrimitiveType::Utf8) + .SetPrimaryKeyColumns({"path", "dc"}) + .Build(); + + auto session = CheckYdbResult(std::move(TableClient->CreateSession())).GetSession(); + CheckYdbResult(session.CreateTable("/Root/PQ/Config/V2/Topics", std::move(tableDesc))); + + tableDesc = TTableBuilder() + .AddNullableColumn("name", EPrimitiveType::String) + .AddNullableColumn("version", EPrimitiveType::Uint64) + .SetPrimaryKeyColumns({"name", "version"}) + .Build(); + CheckYdbResult(session.CreateTable("/Root/PQ/Config/V2/Versions", std::move(tableDesc))); + + EdgeActorId = runtime->AllocateEdgeActor(); + auto config = MakeIntrusive<TSchemeCacheConfig>(); + config->Counters = new NMonitoring::TDynamicCounters; config->Roots.emplace_back(settings.Domain, 8716544, "Root"); - auto& domainsInfo = settings.AppConfig.GetDomainsConfig(); - for (const auto& domain : domainsInfo.GetDomain()) { + auto& domainsInfo = settings.AppConfig.GetDomainsConfig(); + for (const auto& domain : domainsInfo.GetDomain()) { config->Roots.emplace_back(domain.GetDomainId(), domain.GetSchemeRoot(), domain.GetName()); - } - SchemeCacheId = runtime->Register(CreateSchemeBoardSchemeCache(config.Get())); - MetaCacheId = runtime->Register( - NPqMetaCacheV2::CreatePQMetaCache(config->Counters, TDuration::MilliSeconds(50)) - ); - runtime->EnableScheduleForActor(SchemeCacheId, true); - runtime->EnableScheduleForActor(MetaCacheId, true); - } - - struct TTopicInfo { - TString Path; - TString Cluster; - bool DoCreate; - }; - - void AddTopics(const TVector<TTopicInfo>& topics, bool shiftVersion = true) { - auto session = TableClient->GetSession().GetValueSync().GetSession(); - auto topicsPrepared = CheckYdbResult(session.PrepareDataQuery(UpsertTopicQuery)).GetQuery(); - TVector<TAsyncStatus> createTopicStatus; - auto tx = session.BeginTransaction().GetValueSync().GetTransaction(); - auto txControl = TTxControl::Tx(tx); - for (const auto& topic : topics) { - auto fullPath = "/Root/PQ/" + ::NPersQueue::BuildFullTopicName(topic.Path, topic.Cluster); - if (topic.DoCreate) { - createTopicStatus.push_back(PQClient->CreateTopic(fullPath)); - } - auto builder = topicsPrepared.GetParamsBuilder(); - { - auto ¶m = builder.AddParam("$Path"); - param.Utf8(topic.Path); - param.Build(); - } - { - auto ¶m = builder.AddParam("$Cluster"); - param.Utf8(topic.Cluster); - param.Build(); - } - CheckYdbResult(topicsPrepared.Execute(txControl, builder.Build())); - } - if (shiftVersion) { - auto versionPrepared = CheckYdbResult(session.PrepareDataQuery(UpdateVersionQuery)).GetQuery(); - auto builder = versionPrepared.GetParamsBuilder(); - { - auto ¶m = builder.AddParam("$Version"); - param.Uint64(++Version); - param.Build(); - } - CheckYdbResult(versionPrepared.Execute(txControl, builder.Build())); - } - CheckYdbResult(tx.Commit()); - for (auto& status : createTopicStatus) { - CheckYdbResult(std::move(status)); - } - } - - THolder<TEvPqNewMetaCache::TEvDescribeTopicsResponse> DoMetaCacheRequest(const TVector<TTopicInfo>& topicList) { - IEventBase* ev; - if (topicList.empty()) { - ev = new TEvPqNewMetaCache::TEvDescribeAllTopicsRequest(); - } else { - TVector<TString> topicNames; - for (const auto &topic : topicList) { - topicNames.emplace_back(std::move(::NPersQueue::BuildFullTopicName(topic.Path, topic.Cluster))); - } - ev = new TEvPqNewMetaCache::TEvDescribeTopicsRequest(topicNames); - } - auto handle = new IEventHandle(MetaCacheId, EdgeActorId, ev); - Server->GetRuntime()->Send(handle); - auto response = Server->GetRuntime()->GrabEdgeEvent<TEvPqNewMetaCache::TEvDescribeTopicsResponse>(); - return std::move(response); - } - - void CheckTopicInfo(const TVector<TTopicInfo>& expected, TSchemeCacheNavigate* result) { - ui64 i = 0; - Cerr << "=== Got cache navigate response: \n" << result->ToString(NKikimr::NScheme::TTypeRegistry()) << Endl; - Cerr << "=== Expect to have " << expected.size() << " records, got: " << result->ResultSet.size() << " records" << Endl; - UNIT_ASSERT_VALUES_EQUAL_C(expected.size(), result->ResultSet.size(), "Result size mismatch"); - for (const auto& topic : expected) { - Cerr << "===Check topic: " << topic.Path << ":" << topic.Cluster << Endl; - auto& response = result->ResultSet[i++]; - if (topic.DoCreate) { - UNIT_ASSERT(response.Status == TSchemeCacheNavigate::EStatus::Ok); - UNIT_ASSERT(response.PQGroupInfo); - UNIT_ASSERT(response.PQGroupInfo->Kind == NSchemeCache::TSchemeCacheNavigate::KindTopic); - } else { - UNIT_ASSERT_VALUES_EQUAL(response.Status, TSchemeCacheNavigate::EStatus::PathErrorUnknown); - } - } - } - - void WaitForVersion(ui64 version, TDuration timeout = TDuration::Seconds(5)) { - ui64 currentVersion = 0; - auto endTime = TInstant::Now() + timeout; - auto* runtime = Server->GetRuntime(); - while (endTime > TInstant::Now()) { - auto handle = new IEventHandle(MetaCacheId, EdgeActorId, new TEvPqNewMetaCache::TEvListTopicsRequest()); - runtime->Send(handle); - auto response = runtime->GrabEdgeEvent<TEvPqNewMetaCache::TEvListTopicsResponse>(); - currentVersion = response->TopicsVersion; - if (currentVersion >= version) { - Cerr << "=== Got current topics version: " << currentVersion << Endl; - return; - } - Sleep(TDuration::MilliSeconds(100)); - } - UNIT_FAIL("Wait for topics version timed out"); - } - - void TestDescribeTopics() { - auto topics = TVector<TTopicInfo>({ - {"topic1", "man", true}, - {"topic2", "man", true}, - {"topic3", "man", false} - }); - AddTopics(topics); - auto ev = DoMetaCacheRequest(topics); - CheckTopicInfo(topics, ev->Result.Get()); - UNIT_ASSERT(ev); - } - void TestDescribeAllTopics() { - auto topics = TVector<TTopicInfo>({ - {"topic1", "man", true}, - {"topic2", "man", true}, - {"topic3", "man", false} - }); - AddTopics(topics); - auto ev = DoMetaCacheRequest({}); - CheckTopicInfo(topics, ev->Result.Get()); - UNIT_ASSERT(ev); - } - - void TestTopicsUpdate() { - auto topics = TVector<TTopicInfo>({ - {"topic1", "man", true}, - {"topic2", "man", true}, - {"topic3", "man", false} - }); - AddTopics(topics); - CheckTopicInfo(topics, DoMetaCacheRequest(topics)->Result.Get()); - WaitForVersion(Version); - - TTopicInfo topic{"topic1", "sas", true}; - AddTopics({topic}, false); - - CheckTopicInfo(topics, DoMetaCacheRequest({})->Result.Get()); - - AddTopics({}, true); - Cerr << "===Wait for version: " << Version << Endl; - WaitForVersion(Version); - topics.insert(topics.end() - 1, topic); - CheckTopicInfo(topics, DoMetaCacheRequest({})->Result.Get()); - } - - template<class T> - T CheckYdbResult(NThreading::TFuture<T>&& asyncResult) { - auto res = asyncResult.GetValueSync(); - UNIT_ASSERT_C(res.IsSuccess(), res.GetIssues().ToString()); - return res; - } - - TString GetUpsertTopicQuery(const TTopicInfo& info) { - return TStringBuilder() - << "--!syntax_v1\n" - << "UPSERT INTO `/Root/PQ/Config/V2/Topics` " - << "(path, dc) VALUES (" << info.Path << ", " << info.Cluster << ");"; - } - - UNIT_TEST_SUITE(TPqMetaCacheV2Test) - UNIT_TEST(TestDescribeTopics) - UNIT_TEST(TestDescribeAllTopics) - UNIT_TEST(TestTopicsUpdate) - UNIT_TEST_SUITE_END(); -private: - ui16 MainServerPort; - ui16 GrpcServerPort; - THolder<Tests::TServer> Server; - - TActorId EdgeActorId; - TActorId SchemeCacheId; - TActorId MetaCacheId; - ui64 Version = 1; + } + SchemeCacheId = runtime->Register(CreateSchemeBoardSchemeCache(config.Get())); + MetaCacheId = runtime->Register( + NPqMetaCacheV2::CreatePQMetaCache(config->Counters, TDuration::MilliSeconds(50)) + ); + runtime->EnableScheduleForActor(SchemeCacheId, true); + runtime->EnableScheduleForActor(MetaCacheId, true); + } + + struct TTopicInfo { + TString Path; + TString Cluster; + bool DoCreate; + }; + + void AddTopics(const TVector<TTopicInfo>& topics, bool shiftVersion = true) { + auto session = TableClient->GetSession().GetValueSync().GetSession(); + auto topicsPrepared = CheckYdbResult(session.PrepareDataQuery(UpsertTopicQuery)).GetQuery(); + TVector<TAsyncStatus> createTopicStatus; + auto tx = session.BeginTransaction().GetValueSync().GetTransaction(); + auto txControl = TTxControl::Tx(tx); + for (const auto& topic : topics) { + auto fullPath = "/Root/PQ/" + ::NPersQueue::BuildFullTopicName(topic.Path, topic.Cluster); + if (topic.DoCreate) { + createTopicStatus.push_back(PQClient->CreateTopic(fullPath)); + } + auto builder = topicsPrepared.GetParamsBuilder(); + { + auto ¶m = builder.AddParam("$Path"); + param.Utf8(topic.Path); + param.Build(); + } + { + auto ¶m = builder.AddParam("$Cluster"); + param.Utf8(topic.Cluster); + param.Build(); + } + CheckYdbResult(topicsPrepared.Execute(txControl, builder.Build())); + } + if (shiftVersion) { + auto versionPrepared = CheckYdbResult(session.PrepareDataQuery(UpdateVersionQuery)).GetQuery(); + auto builder = versionPrepared.GetParamsBuilder(); + { + auto ¶m = builder.AddParam("$Version"); + param.Uint64(++Version); + param.Build(); + } + CheckYdbResult(versionPrepared.Execute(txControl, builder.Build())); + } + CheckYdbResult(tx.Commit()); + for (auto& status : createTopicStatus) { + CheckYdbResult(std::move(status)); + } + } + + THolder<TEvPqNewMetaCache::TEvDescribeTopicsResponse> DoMetaCacheRequest(const TVector<TTopicInfo>& topicList) { + IEventBase* ev; + if (topicList.empty()) { + ev = new TEvPqNewMetaCache::TEvDescribeAllTopicsRequest(); + } else { + TVector<TString> topicNames; + for (const auto &topic : topicList) { + topicNames.emplace_back(std::move(::NPersQueue::BuildFullTopicName(topic.Path, topic.Cluster))); + } + ev = new TEvPqNewMetaCache::TEvDescribeTopicsRequest(topicNames); + } + auto handle = new IEventHandle(MetaCacheId, EdgeActorId, ev); + Server->GetRuntime()->Send(handle); + auto response = Server->GetRuntime()->GrabEdgeEvent<TEvPqNewMetaCache::TEvDescribeTopicsResponse>(); + return std::move(response); + } + + void CheckTopicInfo(const TVector<TTopicInfo>& expected, TSchemeCacheNavigate* result) { + ui64 i = 0; + Cerr << "=== Got cache navigate response: \n" << result->ToString(NKikimr::NScheme::TTypeRegistry()) << Endl; + Cerr << "=== Expect to have " << expected.size() << " records, got: " << result->ResultSet.size() << " records" << Endl; + UNIT_ASSERT_VALUES_EQUAL_C(expected.size(), result->ResultSet.size(), "Result size mismatch"); + for (const auto& topic : expected) { + Cerr << "===Check topic: " << topic.Path << ":" << topic.Cluster << Endl; + auto& response = result->ResultSet[i++]; + if (topic.DoCreate) { + UNIT_ASSERT(response.Status == TSchemeCacheNavigate::EStatus::Ok); + UNIT_ASSERT(response.PQGroupInfo); + UNIT_ASSERT(response.PQGroupInfo->Kind == NSchemeCache::TSchemeCacheNavigate::KindTopic); + } else { + UNIT_ASSERT_VALUES_EQUAL(response.Status, TSchemeCacheNavigate::EStatus::PathErrorUnknown); + } + } + } + + void WaitForVersion(ui64 version, TDuration timeout = TDuration::Seconds(5)) { + ui64 currentVersion = 0; + auto endTime = TInstant::Now() + timeout; + auto* runtime = Server->GetRuntime(); + while (endTime > TInstant::Now()) { + auto handle = new IEventHandle(MetaCacheId, EdgeActorId, new TEvPqNewMetaCache::TEvListTopicsRequest()); + runtime->Send(handle); + auto response = runtime->GrabEdgeEvent<TEvPqNewMetaCache::TEvListTopicsResponse>(); + currentVersion = response->TopicsVersion; + if (currentVersion >= version) { + Cerr << "=== Got current topics version: " << currentVersion << Endl; + return; + } + Sleep(TDuration::MilliSeconds(100)); + } + UNIT_FAIL("Wait for topics version timed out"); + } + + void TestDescribeTopics() { + auto topics = TVector<TTopicInfo>({ + {"topic1", "man", true}, + {"topic2", "man", true}, + {"topic3", "man", false} + }); + AddTopics(topics); + auto ev = DoMetaCacheRequest(topics); + CheckTopicInfo(topics, ev->Result.Get()); + UNIT_ASSERT(ev); + } + void TestDescribeAllTopics() { + auto topics = TVector<TTopicInfo>({ + {"topic1", "man", true}, + {"topic2", "man", true}, + {"topic3", "man", false} + }); + AddTopics(topics); + auto ev = DoMetaCacheRequest({}); + CheckTopicInfo(topics, ev->Result.Get()); + UNIT_ASSERT(ev); + } + + void TestTopicsUpdate() { + auto topics = TVector<TTopicInfo>({ + {"topic1", "man", true}, + {"topic2", "man", true}, + {"topic3", "man", false} + }); + AddTopics(topics); + CheckTopicInfo(topics, DoMetaCacheRequest(topics)->Result.Get()); + WaitForVersion(Version); + + TTopicInfo topic{"topic1", "sas", true}; + AddTopics({topic}, false); + + CheckTopicInfo(topics, DoMetaCacheRequest({})->Result.Get()); + + AddTopics({}, true); + Cerr << "===Wait for version: " << Version << Endl; + WaitForVersion(Version); + topics.insert(topics.end() - 1, topic); + CheckTopicInfo(topics, DoMetaCacheRequest({})->Result.Get()); + } + + template<class T> + T CheckYdbResult(NThreading::TFuture<T>&& asyncResult) { + auto res = asyncResult.GetValueSync(); + UNIT_ASSERT_C(res.IsSuccess(), res.GetIssues().ToString()); + return res; + } + + TString GetUpsertTopicQuery(const TTopicInfo& info) { + return TStringBuilder() + << "--!syntax_v1\n" + << "UPSERT INTO `/Root/PQ/Config/V2/Topics` " + << "(path, dc) VALUES (" << info.Path << ", " << info.Cluster << ");"; + } + + UNIT_TEST_SUITE(TPqMetaCacheV2Test) + UNIT_TEST(TestDescribeTopics) + UNIT_TEST(TestDescribeAllTopics) + UNIT_TEST(TestTopicsUpdate) + UNIT_TEST_SUITE_END(); +private: + ui16 MainServerPort; + ui16 GrpcServerPort; + THolder<Tests::TServer> Server; + + TActorId EdgeActorId; + TActorId SchemeCacheId; + TActorId MetaCacheId; + ui64 Version = 1; std::shared_ptr<NKikimr::NPersQueueTests::TFlatMsgBusPQClient> Client; - + std::shared_ptr<NYdb::TDriver> YdbDriver; std::shared_ptr<NYdb::NTable::TTableClient> TableClient; std::shared_ptr<NYdb::NPersQueue::TPersQueueClient> PQClient; - TString UpsertTopicQuery = TStringBuilder() - << "--!syntax_v1\n" - << "DECLARE $Path as Utf8; DECLARE $Cluster as Utf8; " - << "UPSERT INTO `/Root/PQ/Config/V2/Topics`" - << "(path, dc) VALUES ($Path, $Cluster);"; - TString UpdateVersionQuery = TStringBuilder() - << "--!syntax_v1\n" - << "DECLARE $Version as Uint64; " - << "UPSERT INTO `/Root/PQ/Config/V2/Version`" - << "(name, version) VALUES ('Topics', $Version);"; - - -}; -UNIT_TEST_SUITE_REGISTRATION(TPqMetaCacheV2Test); - -}// namespace NTests + TString UpsertTopicQuery = TStringBuilder() + << "--!syntax_v1\n" + << "DECLARE $Path as Utf8; DECLARE $Cluster as Utf8; " + << "UPSERT INTO `/Root/PQ/Config/V2/Topics`" + << "(path, dc) VALUES ($Path, $Cluster);"; + TString UpdateVersionQuery = TStringBuilder() + << "--!syntax_v1\n" + << "DECLARE $Version as Uint64; " + << "UPSERT INTO `/Root/PQ/Config/V2/Version`" + << "(name, version) VALUES ('Topics', $Version);"; + + +}; +UNIT_TEST_SUITE_REGISTRATION(TPqMetaCacheV2Test); + +}// namespace NTests } // namespace NKikimr::NMsgBusProxy diff --git a/ydb/core/driver_lib/run/config.h b/ydb/core/driver_lib/run/config.h index 5f9c8cca20..faf1797413 100644 --- a/ydb/core/driver_lib/run/config.h +++ b/ydb/core/driver_lib/run/config.h @@ -47,7 +47,7 @@ union TBasicKikimrServicesMask { bool EnableCms:1; bool EnableNodeTable:1; bool EnableGRpcProxyStatus:1; - bool EnablePQ:1; + bool EnablePQ:1; bool EnableSqs:1; bool EnableConfigsDispatcher:1; bool EnableSecurityServices:1; diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp index 73c167fbce..819c1478d1 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp @@ -1425,16 +1425,16 @@ void TMessageBusServicesInitializer::InitializeServices(NActors::TActorSystemSet if (!IsServiceInitialized(setup, NMsgBusProxy::CreateMsgBusProxyId()) && Config.HasMessageBusConfig() && Config.GetMessageBusConfig().GetStartBusProxy()) { if (IActor *proxy = BusServer.CreateProxy()) { - TDuration pqMetaRefresh = TDuration::MilliSeconds(appData->PQConfig.GetMetaCacheRefreshIntervalMilliSeconds()); - if (appData->PQConfig.GetEnabled()) { - setup->LocalServices.emplace_back( - NMsgBusProxy::CreatePersQueueMetaCacheV2Id(), - TActorSetupCmd( - NMsgBusProxy::NPqMetaCacheV2::CreatePQMetaCache(appData->Counters, pqMetaRefresh), - TMailboxType::ReadAsFilled, appData->UserPoolId - ) - ); - } + TDuration pqMetaRefresh = TDuration::MilliSeconds(appData->PQConfig.GetMetaCacheRefreshIntervalMilliSeconds()); + if (appData->PQConfig.GetEnabled()) { + setup->LocalServices.emplace_back( + NMsgBusProxy::CreatePersQueueMetaCacheV2Id(), + TActorSetupCmd( + NMsgBusProxy::NPqMetaCacheV2::CreatePQMetaCache(appData->Counters, pqMetaRefresh), + TMailboxType::ReadAsFilled, appData->UserPoolId + ) + ); + } } if (IActor* traceService = BusServer.CreateMessageBusTraceService()) { @@ -1496,9 +1496,9 @@ void TGRpcServicesInitializer::InitializeServices(NActors::TActorSystemSetup* se if (appData->PQConfig.GetEnabled()) { TDuration pqMetaRefresh = TDuration::Seconds(NMsgBusProxy::PQ_METACACHE_REFRESH_INTERVAL_SECONDS); - IActor * cache = NMsgBusProxy::NPqMetaCacheV2::CreatePQMetaCache( - appData->Counters, pqMetaRefresh - ); + IActor * cache = NMsgBusProxy::NPqMetaCacheV2::CreatePQMetaCache( + appData->Counters, pqMetaRefresh + ); Y_VERIFY(cache); setup->LocalServices.emplace_back( NMsgBusProxy::CreatePersQueueMetaCacheV2Id(), @@ -2092,11 +2092,11 @@ TSqsServiceInitializer::TSqsServiceInitializer(const TKikimrRunConfig& runConfig void TSqsServiceInitializer::InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) { if (Config.GetSqsConfig().GetEnableSqs()) { - ui32 grpcPort = 0; - if (Config.HasGRpcConfig()) - grpcPort = Config.GetGRpcConfig().GetPort(); + ui32 grpcPort = 0; + if (Config.HasGRpcConfig()) + grpcPort = Config.GetGRpcConfig().GetPort(); { - IActor* actor = NSQS::CreateSqsService(grpcPort); + IActor* actor = NSQS::CreateSqsService(grpcPort); setup->LocalServices.emplace_back( NSQS::MakeSqsServiceID(NodeId), TActorSetupCmd(actor, TMailboxType::HTSwap, appData->UserPoolId)); diff --git a/ydb/core/driver_lib/run/run.cpp b/ydb/core/driver_lib/run/run.cpp index b0c9e93416..a4f74aa4e0 100644 --- a/ydb/core/driver_lib/run/run.cpp +++ b/ydb/core/driver_lib/run/run.cpp @@ -606,7 +606,7 @@ void TKikimrRunner::InitializeGRpc(const TKikimrRunConfig& runConfig) { if (hasLegacy) { // start legacy service auto grpcService = new NGRpcProxy::TGRpcService(); - auto future = grpcService->Prepare(ActorSystem.Get(), NMsgBusProxy::CreatePersQueueMetaCacheV2Id(), NMsgBusProxy::CreateMsgBusProxyId(), Counters); + auto future = grpcService->Prepare(ActorSystem.Get(), NMsgBusProxy::CreatePersQueueMetaCacheV2Id(), NMsgBusProxy::CreateMsgBusProxyId(), Counters); auto startCb = [grpcService](NThreading::TFuture<void> result) { if (result.HasException()) { try { @@ -674,19 +674,19 @@ void TKikimrRunner::InitializeGRpc(const TKikimrRunConfig& runConfig) { } if (hasPQv1) { - server.AddService(new NGRpcService::V1::TGRpcPersQueueService(ActorSystem.Get(), Counters, NMsgBusProxy::CreatePersQueueMetaCacheV2Id(), grpcRequestProxyId)); + server.AddService(new NGRpcService::V1::TGRpcPersQueueService(ActorSystem.Get(), Counters, NMsgBusProxy::CreatePersQueueMetaCacheV2Id(), grpcRequestProxyId)); } if (hasPQCD) { // the service has its own flag since it should be capable of using custom grpc port - const auto& pqcdConfig = AppData->PQClusterDiscoveryConfig; - TMaybe<ui64> inflightLimit = Nothing(); - if (pqcdConfig.HasRequestInflightLimit() && pqcdConfig.GetRequestInflightLimit() > 0) { - inflightLimit = pqcdConfig.GetRequestInflightLimit(); - } - server.AddService(new NGRpcService::TGRpcPQClusterDiscoveryService( - ActorSystem.Get(), Counters, grpcRequestProxyId, inflightLimit - )); + const auto& pqcdConfig = AppData->PQClusterDiscoveryConfig; + TMaybe<ui64> inflightLimit = Nothing(); + if (pqcdConfig.HasRequestInflightLimit() && pqcdConfig.GetRequestInflightLimit() > 0) { + inflightLimit = pqcdConfig.GetRequestInflightLimit(); + } + server.AddService(new NGRpcService::TGRpcPQClusterDiscoveryService( + ActorSystem.Get(), Counters, grpcRequestProxyId, inflightLimit + )); } if (hasCms) { @@ -1109,19 +1109,19 @@ void TKikimrRunner::InitializeActorSystem( SqsHttp.Reset(new NSQS::TAsyncHttpServer(AppData->SqsConfig)); SqsHttp->Initialize(ActorSystem.Get(), GetServiceCounters(AppData->Counters, "sqs"), - GetServiceCounters(AppData->Counters, "ymq_public"), + GetServiceCounters(AppData->Counters, "ymq_public"), AppData->UserPoolId); } } if (runConfig.AppConfig.HasGRpcConfig()) { if (const ui32 grpcPort = runConfig.AppConfig.GetGRpcConfig().GetPort()) { - auto driverConfig = NYdb::TDriverConfig().SetEndpoint(TStringBuilder() << "localhost:" << grpcPort); + auto driverConfig = NYdb::TDriverConfig().SetEndpoint(TStringBuilder() << "localhost:" << grpcPort); YdbDriver.Reset(new NYdb::TDriver(driverConfig)); AppData->YdbDriver = YdbDriver.Get(); - } - } - + } + } + if (YqSharedResources) { YqSharedResources->Init(ActorSystem.Get()); } @@ -1439,8 +1439,8 @@ void TKikimrRunner::KikimrStop(bool graceful) { if (YdbDriver) { YdbDriver->Stop(true); - } - + } + if (Monitoring) { Monitoring->Stop(); } diff --git a/ydb/core/driver_lib/run/run.h b/ydb/core/driver_lib/run/run.h index 28b2683ba3..dfd93a24e7 100644 --- a/ydb/core/driver_lib/run/run.h +++ b/ydb/core/driver_lib/run/run.h @@ -44,7 +44,7 @@ protected: THolder<NSQS::TAsyncHttpServer> SqsHttp; THolder<NYdb::TDriver> YdbDriver; - + THolder<NKqp::TKqpShutdownController> KqpShutdownController; TIntrusivePtr<NInterconnect::TPollerThreads> PollerThreads; diff --git a/ydb/core/persqueue/metering_typedefs.h b/ydb/core/persqueue/metering_typedefs.h index e49e387a1b..0b05329dbe 100644 --- a/ydb/core/persqueue/metering_typedefs.h +++ b/ydb/core/persqueue/metering_typedefs.h @@ -1,18 +1,18 @@ -#pragma once -#include <library/cpp/json/writer/json_value.h> - -//ToDo: Remove after switching to new metering service - -namespace NKikimr::NMetering { -struct TMetricValue { - ui64 Quantity; - - /** Start time or moment when event occurred. usage -> start of billing json */ - TInstant StartTime; - - /** 'tags' in billing json. Concatenated with MetricMeta MetricTags (conflicting keys ore overridden)*/ - THashMap <TString, NJson::TJsonValue> MetricTags; - - TInstant EndTime = TInstant::Zero(); -}; -} +#pragma once +#include <library/cpp/json/writer/json_value.h> + +//ToDo: Remove after switching to new metering service + +namespace NKikimr::NMetering { +struct TMetricValue { + ui64 Quantity; + + /** Start time or moment when event occurred. usage -> start of billing json */ + TInstant StartTime; + + /** 'tags' in billing json. Concatenated with MetricMeta MetricTags (conflicting keys ore overridden)*/ + THashMap <TString, NJson::TJsonValue> MetricTags; + + TInstant EndTime = TInstant::Zero(); +}; +} diff --git a/ydb/core/persqueue/partition.cpp b/ydb/core/persqueue/partition.cpp index 11657f09c2..cc8e2419cf 100644 --- a/ydb/core/persqueue/partition.cpp +++ b/ydb/core/persqueue/partition.cpp @@ -4546,7 +4546,7 @@ void TPartition::HandleWrites(const TActorContext& ctx) ProcessReserveRequests(ctx); if (!haveData && !haveDrop && !haveCheckDisk) { //no data writed/deleted - if (!Requests.empty()) { //there could be change ownership requests that + if (!Requests.empty()) { //there could be change ownership requests that bool res = ProcessWrites(request.Get(), ctx); Y_VERIFY(!res); } diff --git a/ydb/core/persqueue/pq_impl.cpp b/ydb/core/persqueue/pq_impl.cpp index 6a0de22df0..9937dbe561 100644 --- a/ydb/core/persqueue/pq_impl.cpp +++ b/ydb/core/persqueue/pq_impl.cpp @@ -9,10 +9,10 @@ #include <ydb/core/protos/counters_keyvalue.pb.h> #include <ydb/core/metering/metering.h> #include <ydb/core/tablet/tablet_counters.h> -#include <library/cpp/json/json_writer.h> +#include <library/cpp/json/json_writer.h> + +#include <util/generic/strbuf.h> -#include <util/generic/strbuf.h> - //TODO: move this code to vieiwer #include <ydb/core/tablet/tablet_counters_aggregator.h> @@ -583,7 +583,7 @@ void TPersQueue::ApplyNewConfigAndReply(const TActorContext& ctx) Config = NewConfig; FillMeteringParams(ctx); - + if (!Config.PartitionsSize()) { for (const auto partitionId : Config.GetPartitionIds()) { Config.AddPartitions()->SetPartitionId(partitionId); @@ -1406,10 +1406,10 @@ void TPersQueue::HandleWriteRequest(const ui64 responseCookie, const TActorId& p const NKikimrClient::TPersQueuePartitionRequest& req, const TActorContext& ctx) { Y_VERIFY(req.CmdWriteSize()); - FlushMetrics(false, ctx); // To ensure hours' border; - if (req.HasPutUnitsSize()) { - CurrentPutUnitsQuantity += req.GetPutUnitsSize(); - } + FlushMetrics(false, ctx); // To ensure hours' border; + if (req.HasPutUnitsSize()) { + CurrentPutUnitsQuantity += req.GetPutUnitsSize(); + } TVector <TEvPQ::TEvWrite::TMsg> msgs; @@ -1949,8 +1949,8 @@ bool TPersQueue::OnRenderAppHtmlPage(NMon::TEvRemoteHttpInfo::TPtr ev, const TAc void TPersQueue::HandleDie(const TActorContext& ctx) { - FlushMetrics(true, ctx); - + FlushMetrics(true, ctx); + for (const auto& p : Partitions) { ctx.Send(p.second.Actor, new TEvents::TEvPoisonPill()); } @@ -2034,122 +2034,122 @@ void TPersQueue::HandleWakeup(const TActorContext& ctx) { for (auto& g : groups) { AggregateAndSendLabeledCountersFor(g, ctx); } - FlushMetrics(false, ctx); + FlushMetrics(false, ctx); ctx.Schedule(TDuration::Seconds(5), new TEvents::TEvWakeup()); } -TString TPersQueue::GetMeteringJson( - const TString& metricBillingId, const TString& schemeName, const THashMap<TString, ui64>& tags, - ui64 quantity, const TString& quantityUnit, - const TInstant& start, const TInstant& end, const TInstant& now -) { - TStringStream output; - NJson::TJsonWriter writer(&output, false); - - writer.OpenMap(); - - writer.Write("cloud_id", Config.GetYcCloudId()); - writer.Write("folder_id", Config.GetYcFolderId()); - writer.Write("resource_id", ResourceId); +TString TPersQueue::GetMeteringJson( + const TString& metricBillingId, const TString& schemeName, const THashMap<TString, ui64>& tags, + ui64 quantity, const TString& quantityUnit, + const TInstant& start, const TInstant& end, const TInstant& now +) { + TStringStream output; + NJson::TJsonWriter writer(&output, false); + + writer.OpenMap(); + + writer.Write("cloud_id", Config.GetYcCloudId()); + writer.Write("folder_id", Config.GetYcFolderId()); + writer.Write("resource_id", ResourceId); writer.Write("id", TStringBuilder() << metricBillingId << "-" << Config.GetYdbDatabaseId() << "-" << TabletID() << "-" << start.MilliSeconds() << "-" << (++MeteringCounter)); - - writer.Write("schema", schemeName); - - writer.OpenMap("tags"); - for (const auto& [tag, value] : tags) { - writer.Write(tag, value); - } - writer.CloseMap(); // "tags" - - writer.OpenMap("usage"); - writer.Write("quantity", quantity); - writer.Write("unit", quantityUnit); - writer.Write("start", start.Seconds()); + + writer.Write("schema", schemeName); + + writer.OpenMap("tags"); + for (const auto& [tag, value] : tags) { + writer.Write(tag, value); + } + writer.CloseMap(); // "tags" + + writer.OpenMap("usage"); + writer.Write("quantity", quantity); + writer.Write("unit", quantityUnit); + writer.Write("start", start.Seconds()); writer.Write("finish", end.Seconds()); - writer.CloseMap(); // "usage" - - writer.OpenMap("labels"); + writer.CloseMap(); // "usage" + + writer.OpenMap("labels"); writer.Write("datastreams_stream_name", StreamName); writer.Write("ydb_database", Config.GetYdbDatabaseId()); - writer.CloseMap(); // "labels" - - writer.Write("version", "v1"); - writer.Write("source_id", ToString(TabletID())); - writer.Write("source_wt", now.Seconds()); - writer.CloseMap(); - writer.Flush(); - output << Endl; - return output.Str(); -} - -void TPersQueue::FlushMetrics(bool force, const TActorContext &ctx) { - if (!MeteringEnabled) { - return; - } + writer.CloseMap(); // "labels" + + writer.Write("version", "v1"); + writer.Write("source_id", ToString(TabletID())); + writer.Write("source_wt", now.Seconds()); + writer.CloseMap(); + writer.Flush(); + output << Endl; + return output.Str(); +} + +void TPersQueue::FlushMetrics(bool force, const TActorContext &ctx) { + if (!MeteringEnabled) { + return; + } if (Config.PartitionsSize() == 0) return; - auto now = ctx.Now(); - bool needFlushRequests = force, needFlushShards = force; - - auto checkFlushInterval = [&](const TInstant& lastFlush, bool& flag) { - if ((now - lastFlush) >= MetricsFlushInterval) { - flag = true; - } - }; - checkFlushInterval(RequestsMetricsLastFlush, needFlushRequests); - checkFlushInterval(ShardsMetricsLastFlush, needFlushShards); - - auto requestsEndTime = now; - if (now.Hours() > RequestsMetricsLastFlush.Hours()) { - needFlushRequests = true; - // If we jump over a hour edge, report requests metrics for a previous hour; - requestsEndTime = TInstant::Hours(RequestsMetricsLastFlush.Hours() + 1); - Y_VERIFY(requestsEndTime < now); - } else { - Y_VERIFY(now.Hours() == RequestsMetricsLastFlush.Hours()); - } - - if (needFlushRequests) { - if (CurrentPutUnitsQuantity > 0) { - auto record = GetMeteringJson( + auto now = ctx.Now(); + bool needFlushRequests = force, needFlushShards = force; + + auto checkFlushInterval = [&](const TInstant& lastFlush, bool& flag) { + if ((now - lastFlush) >= MetricsFlushInterval) { + flag = true; + } + }; + checkFlushInterval(RequestsMetricsLastFlush, needFlushRequests); + checkFlushInterval(ShardsMetricsLastFlush, needFlushShards); + + auto requestsEndTime = now; + if (now.Hours() > RequestsMetricsLastFlush.Hours()) { + needFlushRequests = true; + // If we jump over a hour edge, report requests metrics for a previous hour; + requestsEndTime = TInstant::Hours(RequestsMetricsLastFlush.Hours() + 1); + Y_VERIFY(requestsEndTime < now); + } else { + Y_VERIFY(now.Hours() == RequestsMetricsLastFlush.Hours()); + } + + if (needFlushRequests) { + if (CurrentPutUnitsQuantity > 0) { + auto record = GetMeteringJson( "put_units", "yds.events.puts.v1", {}, CurrentPutUnitsQuantity, "put_events", - RequestsMetricsLastFlush, requestsEndTime, now - ); - NMetering::SendMeteringJson(ctx, record); - } - CurrentPutUnitsQuantity = 0; - RequestsMetricsLastFlush = now; - } - if (needFlushShards) { + RequestsMetricsLastFlush, requestsEndTime, now + ); + NMetering::SendMeteringJson(ctx, record); + } + CurrentPutUnitsQuantity = 0; + RequestsMetricsLastFlush = now; + } + if (needFlushShards) { ui64 writeQuota = Config.GetPartitionConfig().GetWriteSpeedInBytesPerSecond(); ui64 reservedSpace = Config.GetPartitionConfig().GetLifetimeSeconds() * writeQuota; ui64 consumersThroughput = Config.ReadRulesSize() * writeQuota; ui64 numPartitions = Config.PartitionsSize(); - THashMap<TString, ui64> tags = { - {"reserved_throughput_bps", writeQuota}, {"shard_enhanced_consumers_throughput", consumersThroughput}, - {"reserved_storage_bytes", reservedSpace} - }; - auto makeShardsMetricsJson = [&](TInstant& end) { - auto res = GetMeteringJson( + THashMap<TString, ui64> tags = { + {"reserved_throughput_bps", writeQuota}, {"shard_enhanced_consumers_throughput", consumersThroughput}, + {"reserved_storage_bytes", reservedSpace} + }; + auto makeShardsMetricsJson = [&](TInstant& end) { + auto res = GetMeteringJson( "reserved_resources", "yds.resources.reserved.v1", tags, numPartitions * (end - ShardsMetricsLastFlush).Seconds(), "second", - ShardsMetricsLastFlush, end, now - ); - ShardsMetricsLastFlush = end; - return res; - }; - auto interval = TInstant::Hours(ShardsMetricsLastFlush.Hours() + 1); - while (interval < now) { - NMetering::SendMeteringJson(ctx, makeShardsMetricsJson(interval)); - interval += TDuration::Hours(1); - } - if (ShardsMetricsLastFlush < now) { - NMetering::SendMeteringJson(ctx, makeShardsMetricsJson(now)); - } - Y_VERIFY(ShardsMetricsLastFlush == now); - } -} - + ShardsMetricsLastFlush, end, now + ); + ShardsMetricsLastFlush = end; + return res; + }; + auto interval = TInstant::Hours(ShardsMetricsLastFlush.Hours() + 1); + while (interval < now) { + NMetering::SendMeteringJson(ctx, makeShardsMetricsJson(interval)); + interval += TDuration::Hours(1); + } + if (ShardsMetricsLastFlush < now) { + NMetering::SendMeteringJson(ctx, makeShardsMetricsJson(now)); + } + Y_VERIFY(ShardsMetricsLastFlush == now); + } +} + bool TPersQueue::HandleHook(STFUNC_SIG) { SetActivityType(NKikimrServices::TActivity::PERSQUEUE_ACTOR); diff --git a/ydb/core/persqueue/pq_impl.h b/ydb/core/persqueue/pq_impl.h index 0203a544f2..bc3bfb10ba 100644 --- a/ydb/core/persqueue/pq_impl.h +++ b/ydb/core/persqueue/pq_impl.h @@ -80,14 +80,14 @@ class TPersQueue : public NKeyValue::TKeyValueFlat { void ReadState(const NKikimrClient::TKeyValueResponse::TReadResult& read, const TActorContext& ctx); void FillMeteringParams(const TActorContext& ctx); - TString GetMeteringJson(const TString& metricBillingId, const TString& schemeName, const THashMap<TString, ui64>& tags, - ui64 quantity, const TString& quantityUnit, - const TInstant& start, const TInstant& end, const TInstant& now); - - void FlushMetrics(bool force, const TActorContext& ctx); - void FlushRequests(bool force, const TActorContext& ctx); - void FlushShardData(bool force, const TActorContext& ctx); - + TString GetMeteringJson(const TString& metricBillingId, const TString& schemeName, const THashMap<TString, ui64>& tags, + ui64 quantity, const TString& quantityUnit, + const TInstant& start, const TInstant& end, const TInstant& now); + + void FlushMetrics(bool force, const TActorContext& ctx); + void FlushRequests(bool force, const TActorContext& ctx); + void FlushShardData(bool force, const TActorContext& ctx); + TMaybe<TEvPQ::TEvRegisterMessageGroup::TBody> MakeRegisterMessageGroup( const NKikimrClient::TPersQueuePartitionRequest::TCmdRegisterMessageGroup& cmd, NPersQueue::NErrorCode::EErrorCode& code, TString& error) const; @@ -176,14 +176,14 @@ private: THashMap<ui64, TAutoPtr<TResponseBuilder>> ResponseProxy; NMetrics::TResourceMetrics *ResourceMetrics; - - bool MeteringEnabled = false; - TDuration MetricsFlushInterval; - TInstant ShardsMetricsLastFlush = TInstant::Zero(); - TInstant RequestsMetricsLastFlush = TInstant::Zero(); - ui64 CurrentPutUnitsQuantity = 0; - TString ResourceId; - TString StreamName; + + bool MeteringEnabled = false; + TDuration MetricsFlushInterval; + TInstant ShardsMetricsLastFlush = TInstant::Zero(); + TInstant RequestsMetricsLastFlush = TInstant::Zero(); + ui64 CurrentPutUnitsQuantity = 0; + TString ResourceId; + TString StreamName; ui64 MeteringCounter = 0; }; diff --git a/ydb/core/persqueue/pq_ut.cpp b/ydb/core/persqueue/pq_ut.cpp index ac6d98a236..13a6122910 100644 --- a/ydb/core/persqueue/pq_ut.cpp +++ b/ydb/core/persqueue/pq_ut.cpp @@ -1327,7 +1327,7 @@ Y_UNIT_TEST(TestWriteSplit) { }, [&](const TString& dispatchName, std::function<void(TTestActorRuntime&)> setup, bool& activeZone) { TFinalizer finalizer(tc); tc.Prepare(dispatchName, setup, activeZone); - activeZone = false; + activeZone = false; tc.Runtime->SetScheduledLimit(200); PQTabletPrepare(20000000, 100 * 1024 * 1024, 0, {{"user1", true}}, tc); //never delete @@ -1336,10 +1336,10 @@ Y_UNIT_TEST(TestWriteSplit) { TVector<std::pair<ui64, TString>> data; data.push_back({1, TString{size, 'b'}}); data.push_back({2, TString{size, 'a'}}); - activeZone = PlainOrSoSlow(true, false); + activeZone = PlainOrSoSlow(true, false); CmdWrite(0, "sourceIdx", data, tc, false, {}, false, "", -1, 40000); RestartTablet(tc); - activeZone = false; + activeZone = false; PQGetPartInfo(40000, 40002, tc); }); } diff --git a/ydb/core/persqueue/pq_ut_slow.cpp b/ydb/core/persqueue/pq_ut_slow.cpp index 744fd3ba03..51041c1045 100644 --- a/ydb/core/persqueue/pq_ut_slow.cpp +++ b/ydb/core/persqueue/pq_ut_slow.cpp @@ -49,7 +49,7 @@ Y_UNIT_TEST(TestWriteVeryBigMessage) { data.push_back({2, TString{size, 'a'}}); CmdWrite(1, "sourceIdx", data, tc, false, {}, false, "", -1, 80000); CmdWrite(0, "sourceIdx", data, tc, false, {}, false, "", -1, 0); - activeZone = true; + activeZone = true; PQGetPartInfo(so, 1, tc); RestartTablet(tc); PQGetPartInfo(so, 1, tc); diff --git a/ydb/core/persqueue/read_balancer.cpp b/ydb/core/persqueue/read_balancer.cpp index 0de3728122..4eb9c7206d 100644 --- a/ydb/core/persqueue/read_balancer.cpp +++ b/ydb/core/persqueue/read_balancer.cpp @@ -348,7 +348,7 @@ void TPersQueueReadBalancer::RespondWithACL( const TActorContext &ctx) { THolder<TEvPersQueue::TEvCheckACLResponse> res{new TEvPersQueue::TEvCheckACLResponse}; res->Record.SetTopic(Topic); - res->Record.SetPath(Path); + res->Record.SetPath(Path); res->Record.SetAccess(access); res->Record.SetError(error); ctx.Send(request->Sender, res.Release()); diff --git a/ydb/core/protos/config.proto b/ydb/core/protos/config.proto index cee10b1e22..d64169d4fc 100644 --- a/ydb/core/protos/config.proto +++ b/ydb/core/protos/config.proto @@ -841,36 +841,36 @@ message TSqsConfig { optional uint64 MinimumGarbageAgeSeconds = 64 [default = 3600]; optional bool MeteringByNetClassifierOnly = 65 [default = false]; - - message TYcSearchEventsConfig { - optional bool EnableYcSearch = 1 [default = false]; - oneof OutputMethod { - string UnifiedAgentUri = 10; - string OutputFileName = 11; - } - optional uint64 RescanIntervalSeconds = 3 [default = 60]; - optional uint64 ReindexIntervalSeconds = 4 [default = 14400]; - optional bool TenantMode = 5 [default = false]; - } - - message TOauthTokenConfig { - required string TokenFile = 1; - } - - message TJwtConfig { - required string JwtFile = 1; - optional string IamEndpoint = 2; - } - - message TYdbAuthConfig { - oneof LocalAuthConfig { - TOauthTokenConfig OauthToken = 1; - TJwtConfig Jwt = 2; - } - } - - optional TYcSearchEventsConfig YcSearchEventsConfig = 66; - optional TYdbAuthConfig AuthConfig = 67; + + message TYcSearchEventsConfig { + optional bool EnableYcSearch = 1 [default = false]; + oneof OutputMethod { + string UnifiedAgentUri = 10; + string OutputFileName = 11; + } + optional uint64 RescanIntervalSeconds = 3 [default = 60]; + optional uint64 ReindexIntervalSeconds = 4 [default = 14400]; + optional bool TenantMode = 5 [default = false]; + } + + message TOauthTokenConfig { + required string TokenFile = 1; + } + + message TJwtConfig { + required string JwtFile = 1; + optional string IamEndpoint = 2; + } + + message TYdbAuthConfig { + oneof LocalAuthConfig { + TOauthTokenConfig OauthToken = 1; + TJwtConfig Jwt = 2; + } + } + + optional TYcSearchEventsConfig YcSearchEventsConfig = 66; + optional TYdbAuthConfig AuthConfig = 67; } message TConfigsDispatcherConfig { diff --git a/ydb/core/protos/msgbus_pq.proto b/ydb/core/protos/msgbus_pq.proto index b20743c665..849b292383 100644 --- a/ydb/core/protos/msgbus_pq.proto +++ b/ydb/core/protos/msgbus_pq.proto @@ -137,7 +137,7 @@ message TPersQueuePartitionRequest { optional TCmdUpdateWriteTimestamp CmdUpdateWriteTimestamp = 17; optional bool IsDirectWrite = 18 [default = false]; - optional uint64 PutUnitsSize = 19; + optional uint64 PutUnitsSize = 19; } message TPersQueueMetaRequest { diff --git a/ydb/core/protos/pqconfig.proto b/ydb/core/protos/pqconfig.proto index 8a3b59d60b..7c85927449 100644 --- a/ydb/core/protos/pqconfig.proto +++ b/ydb/core/protos/pqconfig.proto @@ -26,11 +26,11 @@ message TPQConfig { //32 => 1mb blobs at last level optional uint32 MaxBlobSize = 4 [default = 8388608]; //8mb optional uint32 ClustersUpdateTimeoutSec = 5 [default = 30]; - - optional bool Enabled = 6 [default = false]; // Enable PQ proxies - - optional uint32 MetaCacheTimeoutSec = 7 [default = 30]; - + + optional bool Enabled = 6 [default = false]; // Enable PQ proxies + + optional uint32 MetaCacheTimeoutSec = 7 [default = 30]; + optional uint32 MaxReadCookies = 8 [default = 100000]; optional bool CheckACL = 9 [default = false]; @@ -107,7 +107,7 @@ message TPQConfig { optional string SourceIdTablePath = 33 [default = "/Root/PQ/SourceIdMeta2"]; repeated uint32 ValidWriteSpeedLimitsKbPerSec = 34; - + message TReadMeteringConfig { message TPricing { optional string Name = 1 [default = "unknown"]; @@ -116,13 +116,13 @@ message TPQConfig { optional TPricing UnknownPricing = 3; } - message TBillingMeteringConfig { - optional bool Enabled = 1 [default = false]; - optional uint64 FlushIntervalSec = 2 [default = 30]; + message TBillingMeteringConfig { + optional bool Enabled = 1 [default = false]; + optional uint64 FlushIntervalSec = 2 [default = 30]; optional TReadMeteringConfig Read = 3; - } + } - optional TBillingMeteringConfig BillingMeteringConfig = 35; + optional TBillingMeteringConfig BillingMeteringConfig = 35; optional NKikimrNodeLimits.TNodeLimitsConfig.TPersQueueNodeConfig PersQueueNodeConfig = 36; @@ -139,9 +139,9 @@ message TPQConfig { repeated TClientServiceType ClientServiceType = 39; optional TClientServiceType DefaultClientServiceType = 40; optional bool DisallowDefaultClientServiceType = 43 [default = false]; - - optional uint32 MetaCacheRefreshIntervalMilliSeconds = 41 [default = 10000]; - optional bool MetaCacheSkipVersionCheck = 42 [default = false]; + + optional uint32 MetaCacheRefreshIntervalMilliSeconds = 41 [default = 10000]; + optional bool MetaCacheSkipVersionCheck = 42 [default = false]; } message TChannelProfile { @@ -259,11 +259,11 @@ message TPQTabletConfig { optional uint64 AbcId = 24; optional string AbcSlug = 25; - - optional string YcCloudId = 26 [default = ""]; - optional string YcFolderId = 27 [default = ""]; - optional string YdbDatabaseId = 28 [default = ""]; - optional string YdbDatabasePath = 29 [default = ""]; + + optional string YcCloudId = 26 [default = ""]; + optional string YcFolderId = 27 [default = ""]; + optional string YdbDatabaseId = 28 [default = ""]; + optional string YdbDatabasePath = 29 [default = ""]; message TKeyComponentSchema { optional string Name = 1; @@ -365,7 +365,7 @@ message TCheckACL { message TCheckACLResponse { optional EAccess Access = 1; optional string Topic = 2; - optional string Path = 3; + optional string Path = 3; optional string Error = 4; } @@ -680,7 +680,7 @@ message TPQClusterDiscoveryConfig { optional bool Enabled = 1; optional uint32 TimedCountersUpdateIntervalSeconds = 2 [default = 15]; optional NKikimrNetClassifier.TNetData CloudNetData = 3; - optional uint64 RequestInflightLimit = 4; + optional uint64 RequestInflightLimit = 4; } message TYdsNextToken { diff --git a/ydb/core/protos/services.proto b/ydb/core/protos/services.proto index 9fe4288ebe..c17c8a7dc3 100644 --- a/ydb/core/protos/services.proto +++ b/ydb/core/protos/services.proto @@ -169,7 +169,7 @@ enum EServiceKikimr { PERSQUEUE_READ_BALANCER = 448; CHOOSE_PROXY = 444; - LB_CONFIG_MANAGER = 445; + LB_CONFIG_MANAGER = 445; TOKEN_BUILDER = 450; TICKET_PARSER = 455; diff --git a/ydb/core/protos/ya.make b/ydb/core/protos/ya.make index 5b5d91da0c..70bb65514c 100644 --- a/ydb/core/protos/ya.make +++ b/ydb/core/protos/ya.make @@ -7,7 +7,7 @@ OWNER( vvvv g:kikimr ) - + IF (OS_WINDOWS) NO_OPTIMIZE_PY_PROTOS() ENDIF() diff --git a/ydb/core/testlib/mock_pq_metacache.h b/ydb/core/testlib/mock_pq_metacache.h index 60bd72be42..1280f87779 100644 --- a/ydb/core/testlib/mock_pq_metacache.h +++ b/ydb/core/testlib/mock_pq_metacache.h @@ -9,9 +9,9 @@ namespace NKikimr { -using TEvPqMetaCache = NMsgBusProxy::NPqMetaCacheV2::TEvPqNewMetaCache; -using TPQGroupInfoPtr = TIntrusiveConstPtr<NSchemeCache::TSchemeCacheNavigate::TPQGroupInfo>; -using ESchemeStatus = NSchemeCache::TSchemeCacheNavigate::EStatus; +using TEvPqMetaCache = NMsgBusProxy::NPqMetaCacheV2::TEvPqNewMetaCache; +using TPQGroupInfoPtr = TIntrusiveConstPtr<NSchemeCache::TSchemeCacheNavigate::TPQGroupInfo>; +using ESchemeStatus = NSchemeCache::TSchemeCacheNavigate::EStatus; // The functionality of this class is not full. // So anyone is welcome to improve it. class TMockPQMetaCache: public TActor<TMockPQMetaCache> { @@ -23,57 +23,57 @@ public: STFUNC(StateFunc) { switch (ev->GetTypeRewrite()) { - HFunc(TEvPqMetaCache::TEvDescribeTopicsRequest, HandleDescribeTopics); - HFunc(TEvPqMetaCache::TEvDescribeAllTopicsRequest, HandleDescribeAllTopics); + HFunc(TEvPqMetaCache::TEvDescribeTopicsRequest, HandleDescribeTopics); + HFunc(TEvPqMetaCache::TEvDescribeAllTopicsRequest, HandleDescribeAllTopics); default: UNIT_FAIL_NONFATAL("Unexpected event to PQ metacache: " << ev->GetTypeRewrite()); } } - MOCK_METHOD(void, HandleDescribeTopics, (TEvPqMetaCache::TEvDescribeTopicsRequest::TPtr& ev, const TActorContext& ctx), ()); - MOCK_METHOD(void, HandleDescribeAllTopics, (TEvPqMetaCache::TEvDescribeAllTopicsRequest::TPtr& ev, const TActorContext& ctx), ()); - //MOCK_METHOD4(HandleDescribeAllTopics, void(const TString& topic, ui64 balancerTabletId, NMsgBusProxy::TEvPqMetaCache::TEvGetBalancerDescribe::TPtr& ev, const TActorContext& ctx)); + MOCK_METHOD(void, HandleDescribeTopics, (TEvPqMetaCache::TEvDescribeTopicsRequest::TPtr& ev, const TActorContext& ctx), ()); + MOCK_METHOD(void, HandleDescribeAllTopics, (TEvPqMetaCache::TEvDescribeAllTopicsRequest::TPtr& ev, const TActorContext& ctx), ()); + //MOCK_METHOD4(HandleDescribeAllTopics, void(const TString& topic, ui64 balancerTabletId, NMsgBusProxy::TEvPqMetaCache::TEvGetBalancerDescribe::TPtr& ev, const TActorContext& ctx)); // // Helpers // - void SetDescribeCustomTopicsAnswer(const NSchemeCache::TSchemeCacheNavigate::TResultSet& resultSet = {}) { - // ToDo - !!! + void SetDescribeCustomTopicsAnswer(const NSchemeCache::TSchemeCacheNavigate::TResultSet& resultSet = {}) { + // ToDo - !!! using namespace testing; -// auto handle = [success, description](NMsgBusProxy::TEvPqMetaCache::TEvGetNode::TPtr& ev, const TActorContext& ctx) { -// auto& req = ev->Get()->Request; -// req->Description = description; -// req->Succeded = success; -// auto result = MakeHolder<NMsgBusProxy::TEvPqMetaCache::TEvGetNodeResult>(std::move(req)); -// -// ctx.Send(ev->Sender, std::move(result)); -// }; - auto handle = [=](TEvPqMetaCache::TEvDescribeTopicsRequest::TPtr& ev, const TActorContext& ctx) { - auto* result = new NSchemeCache::TSchemeCacheNavigate(); - result->ResultSet = resultSet; - auto* response = new TEvPqMetaCache::TEvDescribeTopicsResponse(ev->Get()->Topics, result); - ctx.Send(ev->Sender, response); +// auto handle = [success, description](NMsgBusProxy::TEvPqMetaCache::TEvGetNode::TPtr& ev, const TActorContext& ctx) { +// auto& req = ev->Get()->Request; +// req->Description = description; +// req->Succeded = success; +// auto result = MakeHolder<NMsgBusProxy::TEvPqMetaCache::TEvGetNodeResult>(std::move(req)); +// +// ctx.Send(ev->Sender, std::move(result)); +// }; + auto handle = [=](TEvPqMetaCache::TEvDescribeTopicsRequest::TPtr& ev, const TActorContext& ctx) { + auto* result = new NSchemeCache::TSchemeCacheNavigate(); + result->ResultSet = resultSet; + auto* response = new TEvPqMetaCache::TEvDescribeTopicsResponse(ev->Get()->Topics, result); + ctx.Send(ev->Sender, response); }; - EXPECT_CALL(*this, HandleDescribeTopics(_, _)) + EXPECT_CALL(*this, HandleDescribeTopics(_, _)) .WillOnce(Invoke(handle)); } - void SetAllTopicsAnswer( - bool success = true, const NSchemeCache::TSchemeCacheNavigate::TResultSet& resultSet = {} - ) { + void SetAllTopicsAnswer( + bool success = true, const NSchemeCache::TSchemeCacheNavigate::TResultSet& resultSet = {} + ) { using namespace testing; - auto handle = [=](TEvPqMetaCache::TEvDescribeAllTopicsRequest::TPtr& ev, const TActorContext& ctx) { - auto* response = new TEvPqMetaCache::TEvDescribeAllTopicsResponse("/Root/PQ/"); - response->Success = success; - auto* result = new NSchemeCache::TSchemeCacheNavigate(); - result->ResultSet = resultSet; - response->Result.Reset(result); - ctx.Send(ev->Sender, std::move(response)); + auto handle = [=](TEvPqMetaCache::TEvDescribeAllTopicsRequest::TPtr& ev, const TActorContext& ctx) { + auto* response = new TEvPqMetaCache::TEvDescribeAllTopicsResponse("/Root/PQ/"); + response->Success = success; + auto* result = new NSchemeCache::TSchemeCacheNavigate(); + result->ResultSet = resultSet; + response->Result.Reset(result); + ctx.Send(ev->Sender, std::move(response)); }; - EXPECT_CALL(*this, HandleDescribeAllTopics(_, _)) + EXPECT_CALL(*this, HandleDescribeAllTopics(_, _)) .WillOnce(Invoke(handle)); } }; diff --git a/ydb/core/testlib/test_client.cpp b/ydb/core/testlib/test_client.cpp index 1d1335c6ae..d4907c26f1 100644 --- a/ydb/core/testlib/test_client.cpp +++ b/ydb/core/testlib/test_client.cpp @@ -281,7 +281,7 @@ namespace Tests { auto future = grpcService->Prepare( system, - NMsgBusProxy::CreatePersQueueMetaCacheV2Id(), + NMsgBusProxy::CreatePersQueueMetaCacheV2Id(), NMsgBusProxy::CreateMsgBusProxyId(), counters ); @@ -306,7 +306,7 @@ namespace Tests { GRpcServer->AddService(new NGRpcService::TGRpcYdbTableService(system, counters, grpcRequestProxyId)); GRpcServer->AddService(new NGRpcService::TGRpcYdbScriptingService(system, counters, grpcRequestProxyId)); GRpcServer->AddService(new NGRpcService::TGRpcOperationService(system, counters, grpcRequestProxyId)); - GRpcServer->AddService(new NGRpcService::V1::TGRpcPersQueueService(system, counters, NMsgBusProxy::CreatePersQueueMetaCacheV2Id(), grpcRequestProxyId)); + GRpcServer->AddService(new NGRpcService::V1::TGRpcPersQueueService(system, counters, NMsgBusProxy::CreatePersQueueMetaCacheV2Id(), grpcRequestProxyId)); GRpcServer->AddService(new NGRpcService::TGRpcPQClusterDiscoveryService(system, counters, grpcRequestProxyId)); GRpcServer->AddService(new NKesus::TKesusGRpcService(system, counters, grpcRequestProxyId)); GRpcServer->AddService(new NGRpcService::TGRpcCmsService(system, counters, grpcRequestProxyId)); @@ -661,33 +661,33 @@ namespace Tests { } { - if (Settings->PQConfig.GetEnabled() == true) { - IActor *pqMetaCache = NMsgBusProxy::NPqMetaCacheV2::CreatePQMetaCache( - new NMonitoring::TDynamicCounters(), TDuration::Seconds(1) - ); - - TActorId pqMetaCacheId = Runtime->Register(pqMetaCache, nodeIdx); - Runtime->RegisterService(NMsgBusProxy::CreatePersQueueMetaCacheV2Id(), pqMetaCacheId, nodeIdx); - } + if (Settings->PQConfig.GetEnabled() == true) { + IActor *pqMetaCache = NMsgBusProxy::NPqMetaCacheV2::CreatePQMetaCache( + new NMonitoring::TDynamicCounters(), TDuration::Seconds(1) + ); + + TActorId pqMetaCacheId = Runtime->Register(pqMetaCache, nodeIdx); + Runtime->RegisterService(NMsgBusProxy::CreatePersQueueMetaCacheV2Id(), pqMetaCacheId, nodeIdx); + } + } + + { + if (Settings->EnableMetering) { + THolder<TFileLogBackend> fileBackend; + try { + fileBackend = MakeHolder<TFileLogBackend>(Settings->MeteringFilePath); + auto meteringActor = NMetering::CreateMeteringWriter(std::move(fileBackend)); + TActorId meteringId = Runtime->Register(meteringActor.Release(), nodeIdx); + Runtime->RegisterService(NMetering::MakeMeteringServiceID(), meteringId, nodeIdx); + + } catch (const TFileError &ex) { + Cerr << "TMeteringWriterInitializer: failed to open file '" << Settings->MeteringFilePath << "': " + << ex.what() << Endl; + } + } } { - if (Settings->EnableMetering) { - THolder<TFileLogBackend> fileBackend; - try { - fileBackend = MakeHolder<TFileLogBackend>(Settings->MeteringFilePath); - auto meteringActor = NMetering::CreateMeteringWriter(std::move(fileBackend)); - TActorId meteringId = Runtime->Register(meteringActor.Release(), nodeIdx); - Runtime->RegisterService(NMetering::MakeMeteringServiceID(), meteringId, nodeIdx); - - } catch (const TFileError &ex) { - Cerr << "TMeteringWriterInitializer: failed to open file '" << Settings->MeteringFilePath << "': " - << ex.what() << Endl; - } - } - } - - { IActor* kesusService = NKesus::CreateKesusProxyService(); TActorId kesusServiceId = Runtime->Register(kesusService, nodeIdx); Runtime->RegisterService(NKesus::MakeKesusProxyServiceId(), kesusServiceId, nodeIdx); diff --git a/ydb/core/testlib/test_client.h b/ydb/core/testlib/test_client.h index e4283d3d70..2064752ab4 100644 --- a/ydb/core/testlib/test_client.h +++ b/ydb/core/testlib/test_client.h @@ -122,9 +122,9 @@ namespace Tests { ui64 DomainPlanResolution = 0; std::shared_ptr<NKikimr::NMsgBusProxy::IPersQueueGetReadSessionsInfoWorkerFactory> PersQueueGetReadSessionsInfoWorkerFactory; - bool EnableMetering = false; - TString MeteringFilePath; - + bool EnableMetering = false; + TString MeteringFilePath; + std::function<IActor*(const NKikimrProto::TAuthConfig&)> CreateTicketParser = NKikimr::CreateTicketParser; std::shared_ptr<TGrpcServiceFactory> GrpcServiceFactory; @@ -158,7 +158,7 @@ namespace Tests { TServerSettings& SetKeepSnapshotTimeout(TDuration value) { KeepSnapshotTimeout = value; return *this; } TServerSettings& SetChangesQueueItemsLimit(ui64 value) { ChangesQueueItemsLimit = value; return *this; } TServerSettings& SetChangesQueueBytesLimit(ui64 value) { ChangesQueueBytesLimit = value; return *this; } - TServerSettings& SetMeteringFilePath(const TString& path) { EnableMetering = true; MeteringFilePath = path; return *this; } + TServerSettings& SetMeteringFilePath(const TString& path) { EnableMetering = true; MeteringFilePath = path; return *this; } TServerSettings& SetPersQueueGetReadSessionsInfoWorkerFactory( std::shared_ptr<NKikimr::NMsgBusProxy::IPersQueueGetReadSessionsInfoWorkerFactory> factory ) { @@ -252,7 +252,7 @@ namespace Tests { const bool UseStoragePools; THolder<TTestActorRuntime> Runtime; - THolder<NYdb::TDriver> Driver; + THolder<NYdb::TDriver> Driver; TIntrusivePtr<NBus::TBusMessageQueue> Bus; const NBus::TBusServerSessionConfig BusServerSessionConfig; //BusServer hold const & on config TAutoPtr<NMsgBusProxy::IMessageBusServer> BusServer; diff --git a/ydb/core/testlib/test_pq_client.h b/ydb/core/testlib/test_pq_client.h index 5211010e10..f059923250 100644 --- a/ydb/core/testlib/test_pq_client.h +++ b/ydb/core/testlib/test_pq_client.h @@ -53,8 +53,8 @@ inline Tests::TServerSettings PQSettings(ui16 port, ui32 nodesCount = 2, bool ro } Tests::TServerSettings settings(port, authConfig, pqConfig); - settings.SetDomainName("Root").SetNodeCount(nodesCount); - + settings.SetDomainName("Root").SetNodeCount(nodesCount); + TVector<NKikimrKqp::TKqpSetting> kqpSettings; NKikimrKqp::TKqpSetting kqpSetting; kqpSetting.SetName("_KqpQueryTimeout"); @@ -62,9 +62,9 @@ inline Tests::TServerSettings PQSettings(ui16 port, ui32 nodesCount = 2, bool ro kqpSetting.SetValue(yql_timeout); kqpSettings.push_back(kqpSetting); settings.SetKqpSettings(kqpSettings); - settings.PQClusterDiscoveryConfig.SetEnabled(true); - settings.PQClusterDiscoveryConfig.SetTimedCountersUpdateIntervalSeconds(1); - + settings.PQClusterDiscoveryConfig.SetEnabled(true); + settings.PQClusterDiscoveryConfig.SetTimedCountersUpdateIntervalSeconds(1); + if (netDataFile) settings.NetClassifierConfig.SetNetDataFilePath(netDataFile->Name()); @@ -72,7 +72,7 @@ inline Tests::TServerSettings PQSettings(ui16 port, ui32 nodesCount = 2, bool ro } const TString TopicPrefix = "/Root/PQ/"; -const static TString DEFAULT_SRC_IDS_PATH = "/Root/PQ/SourceIdMeta2"; +const static TString DEFAULT_SRC_IDS_PATH = "/Root/PQ/SourceIdMeta2"; struct TRequestCreatePQ { @@ -137,7 +137,7 @@ struct TRequestCreatePQ { config->MutablePartitionConfig()->SetSourceIdLifetimeSeconds(SourceIdLifetime); config->MutablePartitionConfig()->SetSourceIdMaxCounts(SourceIdMaxCount); config->MutablePartitionConfig()->SetLowWatermark(LowWatermark); - + config->SetLocalDC(true); auto codec = config->MutableCodecs(); @@ -161,9 +161,9 @@ struct TRequestCreatePQ { config->AddReadRuleVersions(0); config->AddConsumerCodecs()->AddIds(0); } - if (!ReadRules.empty()) { - config->SetRequireAuthRead(true); - } + if (!ReadRules.empty()) { + config->SetRequireAuthRead(true); + } if (!User.empty()) { auto rq = config->MutablePartitionConfig()->AddReadQuota(); rq->SetSpeedInBytesPerSecond(ReadSpeed); @@ -456,12 +456,12 @@ enum class ETransport { GRpc }; -struct TPQTestClusterInfo { - TString Balancer; - bool Enabled; +struct TPQTestClusterInfo { + TString Balancer; + bool Enabled; ui64 Weight = 1000; -}; - +}; + static THashMap<TString, TPQTestClusterInfo> DEFAULT_CLUSTERS_LIST = { {"dc1", {"localhost", true}}, {"dc2", {"dc2.logbroker.yandex.net", true}} @@ -479,8 +479,8 @@ private: const ui16 GRpcPort; NClient::TKikimr Kikimr; THolder<NYdb::TDriver> Driver; - ui64 TopicsVersion = 0; - bool UseConfigTables = true; + ui64 TopicsVersion = 0; + bool UseConfigTables = true; void RunYqlSchemeQuery(TString query) { auto tableClient = NYdb::NTable::TTableClient(*Driver); @@ -515,9 +515,9 @@ private: public: - TFlatMsgBusPQClient( - const Tests::TServerSettings& settings, ui16 grpc, TMaybe<TString> databaseName = Nothing() - ) + TFlatMsgBusPQClient( + const Tests::TServerSettings& settings, ui16 grpc, TMaybe<TString> databaseName = Nothing() + ) : TFlatMsgBusClient(settings) , Settings(settings) , GRpcPort(grpc) @@ -535,24 +535,24 @@ public: ~TFlatMsgBusPQClient() { Driver->Stop(true); } - - void SetNoConfigMode() { - UseConfigTables = false; - } - - void FullInit( - THashMap<TString, TPQTestClusterInfo> clusters = DEFAULT_CLUSTERS_LIST, - const TString& srcIdsPath = DEFAULT_SRC_IDS_PATH, - const TString& localCluster = TString() - ) { - InitRoot(); - if (UseConfigTables) { - InitSourceIds(srcIdsPath); - InitDCs(clusters, localCluster); - } else { - InitSourceIds({}); - } - } + + void SetNoConfigMode() { + UseConfigTables = false; + } + + void FullInit( + THashMap<TString, TPQTestClusterInfo> clusters = DEFAULT_CLUSTERS_LIST, + const TString& srcIdsPath = DEFAULT_SRC_IDS_PATH, + const TString& localCluster = TString() + ) { + InitRoot(); + if (UseConfigTables) { + InitSourceIds(srcIdsPath); + InitDCs(clusters, localCluster); + } else { + InitSourceIds({}); + } + } void InitRoot() { InitRootScheme(); MkDir("/Root", "PQ"); @@ -562,7 +562,7 @@ public: return Driver.Get(); } - void InitSourceIds(const TString& path = DEFAULT_SRC_IDS_PATH) { + void InitSourceIds(const TString& path = DEFAULT_SRC_IDS_PATH) { TFsPath fsPath(path); CreateTable(fsPath.Dirname(), "Name: \"" + fsPath.Basename() + "\"" @@ -618,11 +618,11 @@ public: weight Uint64, PRIMARY KEY (name) ); - CREATE TABLE [/Root/PQ/Config/V2/Topics] ( - path Utf8, - dc Utf8, - PRIMARY KEY (path, dc) - ); + CREATE TABLE [/Root/PQ/Config/V2/Topics] ( + path Utf8, + dc Utf8, + PRIMARY KEY (path, dc) + ); )___"); RunYqlSchemeQuery(R"___( @@ -633,34 +633,34 @@ public: ); )___"); - TStringBuilder upsertClusters; + TStringBuilder upsertClusters; upsertClusters << "UPSERT INTO [/Root/PQ/Config/V2/Cluster] (name, balancer, local, enabled, weight) VALUES "; - bool first = true; - for (auto& [cluster, info] : clusters) { - bool isLocal = localCluster.empty() ? first : localCluster == cluster; - if (!first) - upsertClusters << ", "; - upsertClusters << "(\"" << cluster << "\", \"" << info.Balancer << "\", " << (isLocal ? "true" : "false") + bool first = true; + for (auto& [cluster, info] : clusters) { + bool isLocal = localCluster.empty() ? first : localCluster == cluster; + if (!first) + upsertClusters << ", "; + upsertClusters << "(\"" << cluster << "\", \"" << info.Balancer << "\", " << (isLocal ? "true" : "false") << ", " << (info.Enabled ? "true" : "false") << ", " << info.Weight << ")"; - first = false; - } - upsertClusters << ";\n"; - TString clustersStr = clusters.empty() ? "" : TString(upsertClusters); - Cerr << "=== Init DC: " << clustersStr << Endl; + first = false; + } + upsertClusters << ";\n"; + TString clustersStr = clusters.empty() ? "" : TString(upsertClusters); + Cerr << "=== Init DC: " << clustersStr << Endl; RunYqlDataQuery(clustersStr + R"___( - UPSERT INTO [/Root/PQ/Config/V2/Versions] (name, version) VALUES ("Cluster", 1); - UPSERT INTO [/Root/PQ/Config/V2/Versions] (name, version) VALUES ("Topics", 0); + UPSERT INTO [/Root/PQ/Config/V2/Versions] (name, version) VALUES ("Cluster", 1); + UPSERT INTO [/Root/PQ/Config/V2/Versions] (name, version) VALUES ("Topics", 0); )___"); } - void UpdateDcEnabled(const TString& name, bool enabled) { - TStringBuilder query; - query << "UPDATE [/Root/PQ/Config/V2/Cluster] SET enabled = " << (enabled ? "true" : "false") + void UpdateDcEnabled(const TString& name, bool enabled) { + TStringBuilder query; + query << "UPDATE [/Root/PQ/Config/V2/Cluster] SET enabled = " << (enabled ? "true" : "false") << " where name = \"" << name << "\";"; - Cerr << "===Update clusters: " << query << Endl; + Cerr << "===Update clusters: " << query << Endl; RunYqlDataQuery(query); - } - + } + TPQTestClusterInfo GetDcInfo(const TString& name) { TStringBuilder query; query << "SELECT balancer, enabled, weight FROM [/Root/PQ/Config/V2/Cluster] where name = \"" << name << "\";"; @@ -736,9 +736,9 @@ public: TAutoPtr<NBus::TBusMessage> reply; NBus::EMessageStatus status = SyncCall(request, reply); - Cerr << "Topic created - response: " << PrintResult<NMsgBusProxy::TBusResponse>(reply.Get()) << Endl; + Cerr << "Topic created - response: " << PrintResult<NMsgBusProxy::TBusResponse>(reply.Get()) << Endl; if (status != NBus::MESSAGE_OK) - return 0; + return 0; UNIT_ASSERT_VALUES_EQUAL(status, NBus::MESSAGE_OK); const NMsgBusProxy::TBusResponse* response = dynamic_cast<NMsgBusProxy::TBusResponse*>(reply.Get()); UNIT_ASSERT(response); @@ -759,7 +759,7 @@ public: if (actualSize != cacheSize) return 0; } - Cerr << "=== Topic created, have version: " << topicInfo.GetConfig().GetVersion() << Endl; + Cerr << "=== Topic created, have version: " << topicInfo.GetConfig().GetVersion() << Endl; return topicInfo.GetConfig().GetVersion(); } @@ -807,12 +807,12 @@ public: TAutoPtr<NBus::TBusMessage> reply; NBus::EMessageStatus status = SyncCall(request, reply); - Cerr << "Topic deleted got reply with status: " << status << Endl; + Cerr << "Topic deleted got reply with status: " << status << Endl; if (status != NBus::MESSAGE_OK) return false; const NMsgBusProxy::TBusResponse* response = dynamic_cast<NMsgBusProxy::TBusResponse*>(reply.Get()); - if (response == nullptr) - return false; + if (response == nullptr) + return false; Cerr << "TopicDeleted response " << response->Record << "\n"; UNIT_ASSERT(response); UNIT_ASSERT(response->Record.HasErrorCode()); @@ -837,7 +837,7 @@ public: } void CreateConsumer(const TString& oldName) { - auto name = NPersQueue::ConvertOldConsumerName(oldName); + auto name = NPersQueue::ConvertOldConsumerName(oldName); RunYqlSchemeQuery("CREATE TABLE [/Root/PQ/" + name + "] (" + "Topic Utf8, Partition Uint32, Offset Uint64, PRIMARY KEY (Topic,Partition) );"); } @@ -845,35 +845,35 @@ public: NACLib::TDiffACL acl; acl.AddAccess(NACLib::EAccessType::Allow, NACLib::ReadAttributes, subj); acl.AddAccess(NACLib::EAccessType::Allow, NACLib::WriteAttributes, subj); - auto name = NPersQueue::ConvertOldConsumerName(oldName); + auto name = NPersQueue::ConvertOldConsumerName(oldName); auto pos = name.rfind("/"); Y_VERIFY(pos != TString::npos); auto pref = "/Root/PQ/" + name.substr(0, pos); ModifyACL(pref, name.substr(pos + 1), acl.SerializeAsString()); } - + void CreateTopicNoLegacy(const TString& name, ui32 partsCount, bool doWait = true, bool canWrite = true) { - TString path = name; - if (UseConfigTables) { - path = TStringBuilder() << "/Root/PQ/" << name; - } - - auto pqClient = NYdb::NPersQueue::TPersQueueClient(*Driver); + TString path = name; + if (UseConfigTables) { + path = TStringBuilder() << "/Root/PQ/" << name; + } + + auto pqClient = NYdb::NPersQueue::TPersQueueClient(*Driver); auto settings = NYdb::NPersQueue::TCreateTopicSettings().PartitionsCount(partsCount).ClientWriteDisabled(!canWrite); - Cerr << "===Create topic: " << path << Endl; - auto res = pqClient.CreateTopic(path, settings); - //ToDo - hack, cannot avoid legacy compat yet as PQv1 still uses RequestProcessor from core/client/server - if (UseConfigTables) { - AddTopic(name); - } - if (doWait) { - res.Wait(); + Cerr << "===Create topic: " << path << Endl; + auto res = pqClient.CreateTopic(path, settings); + //ToDo - hack, cannot avoid legacy compat yet as PQv1 still uses RequestProcessor from core/client/server + if (UseConfigTables) { + AddTopic(name); + } + if (doWait) { + res.Wait(); Cerr << "Create topic result: " << res.GetValue().IsSuccess() << " " << res.GetValue().GetIssues().ToString() << "\n"; UNIT_ASSERT(res.GetValue().IsSuccess()); - } - } - + } + } + void WaitTopicInit(const TString& topic) { auto pqClient = NYdb::NPersQueue::TPersQueueClient(*Driver); do { @@ -889,30 +889,30 @@ public: } while (true); } - void CreateTopic( - const TRequestCreatePQ& createRequest, - bool doWait = true - ) { - const TInstant start = TInstant::Now(); - - THolder<NMsgBusProxy::TBusPersQueue> request = createRequest.GetRequest(); - - ui32 prevVersion = TopicCreated(createRequest.Topic); - TAutoPtr<NBus::TBusMessage> reply; - const NMsgBusProxy::TBusResponse* response = SendAndGetReply(request, reply); - UNIT_ASSERT(response); - UNIT_ASSERT_VALUES_EQUAL_C((ui32)response->Record.GetErrorCode(), (ui32)NPersQueue::NErrorCode::OK, - "proxy failure"); - - AddTopic(createRequest.Topic); - while (doWait && TopicRealCreated(createRequest.Topic) != prevVersion + 1) { - Sleep(TDuration::MilliSeconds(500)); - UNIT_ASSERT(TInstant::Now() - start < ::DEFAULT_DISPATCH_TIMEOUT); + void CreateTopic( + const TRequestCreatePQ& createRequest, + bool doWait = true + ) { + const TInstant start = TInstant::Now(); + + THolder<NMsgBusProxy::TBusPersQueue> request = createRequest.GetRequest(); + + ui32 prevVersion = TopicCreated(createRequest.Topic); + TAutoPtr<NBus::TBusMessage> reply; + const NMsgBusProxy::TBusResponse* response = SendAndGetReply(request, reply); + UNIT_ASSERT(response); + UNIT_ASSERT_VALUES_EQUAL_C((ui32)response->Record.GetErrorCode(), (ui32)NPersQueue::NErrorCode::OK, + "proxy failure"); + + AddTopic(createRequest.Topic); + while (doWait && TopicRealCreated(createRequest.Topic) != prevVersion + 1) { + Sleep(TDuration::MilliSeconds(500)); + UNIT_ASSERT(TInstant::Now() - start < ::DEFAULT_DISPATCH_TIMEOUT); + } + while (doWait && TopicCreated(createRequest.Topic, prevVersion) != prevVersion + 1) { + Sleep(TDuration::MilliSeconds(500)); + UNIT_ASSERT(TInstant::Now() - start < ::DEFAULT_DISPATCH_TIMEOUT); } - while (doWait && TopicCreated(createRequest.Topic, prevVersion) != prevVersion + 1) { - Sleep(TDuration::MilliSeconds(500)); - UNIT_ASSERT(TInstant::Now() - start < ::DEFAULT_DISPATCH_TIMEOUT); - } } void CreateTopic( @@ -929,30 +929,30 @@ public: ui64 sourceIdMaxCount = 6000000, ui64 sourceIdLifetime = 86400 ) { - Y_VERIFY(name.StartsWith("rt3.")); - - Cerr << "PQ Client: create topic: " << name << " with " << nParts << " partitions" << Endl; + Y_VERIFY(name.StartsWith("rt3.")); + + Cerr << "PQ Client: create topic: " << name << " with " << nParts << " partitions" << Endl; auto request = TRequestCreatePQ( - name, nParts, 0, lifetimeS, lowWatermark, writeSpeed, user, readSpeed, rr, important, mirrorFrom, - sourceIdMaxCount, sourceIdLifetime + name, nParts, 0, lifetimeS, lowWatermark, writeSpeed, user, readSpeed, rr, important, mirrorFrom, + sourceIdMaxCount, sourceIdLifetime ); - return CreateTopic(request); - } - - void AlterTopicNoLegacy(const TString& name, ui32 nParts, ui64 lifetimeS = 86400) { - TString path = name; - if (!UseConfigTables) { - path = TStringBuilder() << "/Root/PQ/" << name; - } - auto settings = NYdb::NPersQueue::TAlterTopicSettings().PartitionsCount(nParts); - settings.RetentionPeriod(TDuration::Seconds(lifetimeS)); - auto pqClient = NYdb::NPersQueue::TPersQueueClient(*Driver); - auto res = pqClient.AlterTopic(path, settings); - if (UseConfigTables) { // ToDo - legacy - AlterTopic(); - } - res.Wait(); - Cerr << "Alter topic (" << path << ") response: " << res.GetValue().IsSuccess() << " " << res.GetValue().GetIssues().ToString() << Endl; + return CreateTopic(request); + } + + void AlterTopicNoLegacy(const TString& name, ui32 nParts, ui64 lifetimeS = 86400) { + TString path = name; + if (!UseConfigTables) { + path = TStringBuilder() << "/Root/PQ/" << name; + } + auto settings = NYdb::NPersQueue::TAlterTopicSettings().PartitionsCount(nParts); + settings.RetentionPeriod(TDuration::Seconds(lifetimeS)); + auto pqClient = NYdb::NPersQueue::TPersQueueClient(*Driver); + auto res = pqClient.AlterTopic(path, settings); + if (UseConfigTables) { // ToDo - legacy + AlterTopic(); + } + res.Wait(); + Cerr << "Alter topic (" << path << ") response: " << res.GetValue().IsSuccess() << " " << res.GetValue().GetIssues().ToString() << Endl; } void AlterTopic( @@ -963,7 +963,7 @@ public: bool fillPartitionConfig = false, std::optional<NKikimrPQ::TMirrorPartitionConfig> mirrorFrom = {} ) { - Y_VERIFY(name.StartsWith("rt3.")); + Y_VERIFY(name.StartsWith("rt3.")); TRequestAlterPQ requestDescr(name, nParts, cacheSize, lifetimeS, fillPartitionConfig, mirrorFrom); THolder<NMsgBusProxy::TBusPersQueue> request = requestDescr.GetRequest(); @@ -976,7 +976,7 @@ public: response->Record.DebugString().c_str()); const TInstant start = TInstant::Now(); - AlterTopic(); + AlterTopic(); while (TopicCreated(name, cacheSize) != prevVersion + 1) { Sleep(TDuration::MilliSeconds(500)); UNIT_ASSERT(TInstant::Now() - start < ::DEFAULT_DISPATCH_TIMEOUT); @@ -988,23 +988,23 @@ public: } - void DeleteTopicNoLegacy (const TString& name) { - TString path = name; - if (!UseConfigTables) { - path = TStringBuilder() << "/Root/PQ/" << name; - } else { - RemoveTopic(name); // ToDo - also legacy - } - auto settings = NYdb::NPersQueue::TDropTopicSettings(); - auto pqClient = NYdb::NPersQueue::TPersQueueClient(*Driver); - auto res = pqClient.DropTopic(path, settings); - res.Wait(); - Cerr << "Drop topic response: " << res.GetValue().IsSuccess() << " " << res.GetValue().GetIssues().ToString() << Endl; - return; - } - + void DeleteTopicNoLegacy (const TString& name) { + TString path = name; + if (!UseConfigTables) { + path = TStringBuilder() << "/Root/PQ/" << name; + } else { + RemoveTopic(name); // ToDo - also legacy + } + auto settings = NYdb::NPersQueue::TDropTopicSettings(); + auto pqClient = NYdb::NPersQueue::TPersQueueClient(*Driver); + auto res = pqClient.DropTopic(path, settings); + res.Wait(); + Cerr << "Drop topic response: " << res.GetValue().IsSuccess() << " " << res.GetValue().GetIssues().ToString() << Endl; + return; + } + void DeleteTopic2(const TString& name, NPersQueue::NErrorCode::EErrorCode expectedStatus = NPersQueue::NErrorCode::OK, bool waitForTopicDeletion = true) { - Y_VERIFY(name.StartsWith("rt3.")); + Y_VERIFY(name.StartsWith("rt3.")); THolder<NMsgBusProxy::TBusPersQueue> request = TRequestDeletePQ{name}.GetRequest(); TAutoPtr<NBus::TBusMessage> reply; @@ -1026,7 +1026,7 @@ public: } UNIT_ASSERT_C(i < 500, "Drop is taking too long"); //25 seconds } - RemoveTopic(name); + RemoveTopic(name); const TInstant start = TInstant::Now(); while (waitForTopicDeletion && !TopicDeleted(name)) { Sleep(TDuration::MilliSeconds(50)); @@ -1076,13 +1076,13 @@ public: } - void WriteToPQ( - const TRequestWritePQ& writeRequest, const TString& data, - const TString& ticket = "", - ETransport transport = ETransport::MsgBus, - NMsgBusProxy::EResponseStatus expectedStatus = NMsgBusProxy::MSTATUS_OK, - NMsgBusProxy::EResponseStatus expectedOwnerStatus = NMsgBusProxy::MSTATUS_OK - ) { + void WriteToPQ( + const TRequestWritePQ& writeRequest, const TString& data, + const TString& ticket = "", + ETransport transport = ETransport::MsgBus, + NMsgBusProxy::EResponseStatus expectedStatus = NMsgBusProxy::MSTATUS_OK, + NMsgBusProxy::EResponseStatus expectedOwnerStatus = NMsgBusProxy::MSTATUS_OK + ) { TString cookie = GetOwnership({writeRequest.Topic, writeRequest.Partition}, expectedOwnerStatus); @@ -1130,12 +1130,12 @@ public: TVector<TString> Values; }; - TReadDebugInfo ReadFromPQ( - const TRequestReadPQ& readRequest, ui32 readCount, - const TString& ticket = "", - NMsgBusProxy::EResponseStatus expectedStatus = NMsgBusProxy::MSTATUS_OK, - NPersQueue::NErrorCode::EErrorCode expectedError = NPersQueue::NErrorCode::OK - ) { + TReadDebugInfo ReadFromPQ( + const TRequestReadPQ& readRequest, ui32 readCount, + const TString& ticket = "", + NMsgBusProxy::EResponseStatus expectedStatus = NMsgBusProxy::MSTATUS_OK, + NPersQueue::NErrorCode::EErrorCode expectedError = NPersQueue::NErrorCode::OK + ) { THolder<NMsgBusProxy::TBusPersQueue> request = readRequest.GetRequest(); if (!ticket.empty()) { request.Get()->Record.SetTicket(ticket); @@ -1175,11 +1175,11 @@ public: return ReadFromPQ({topic, partition, startOffset, count, "user"}, readCount, ticket); } - void SetClientOffsetPQ( - const TRequestSetClientOffsetPQ& cmdRequest, const TString& ticket = "", - NMsgBusProxy::EResponseStatus expectedStatus = NMsgBusProxy::MSTATUS_OK, - NPersQueue::NErrorCode::EErrorCode expectedError = NPersQueue::NErrorCode::OK - ) { + void SetClientOffsetPQ( + const TRequestSetClientOffsetPQ& cmdRequest, const TString& ticket = "", + NMsgBusProxy::EResponseStatus expectedStatus = NMsgBusProxy::MSTATUS_OK, + NPersQueue::NErrorCode::EErrorCode expectedError = NPersQueue::NErrorCode::OK + ) { THolder<NMsgBusProxy::TBusPersQueue> request = cmdRequest.GetRequest(); if (!ticket.empty()) { request.Get()->Record.SetTicket(ticket); @@ -1372,51 +1372,51 @@ public: GetPartLocation(topicsAndParts, resCount, ok); GetPartStatus(topicsAndParts, resCount, ok); } - -private: - static TString GetAlterTopicsVersionQuery() { - return "UPSERT INTO [/Root/PQ/Config/V2/Versions] (name, version) VALUES (\"Topics\", $version);"; - } - -public: - void AddTopic(const TString& topic) { - return AddOrRemoveTopic(topic, true); - } - - void RemoveTopic(const TString& topic) { - return AddOrRemoveTopic(topic, false); - } - - void AddOrRemoveTopic(const TString& topic, bool add) { - TStringBuilder query; - query << "DECLARE $version as Int64; DECLARE $path AS Utf8; DECLARE $cluster as Utf8; "; - if (add) { - query << "UPSERT INTO [/Root/PQ/Config/V2/Topics] (path, dc) VALUES ($path, $cluster); "; - } else { - query << "DELETE FROM [/Root/PQ/Config/V2/Topics] WHERE path = $path AND dc = $cluster; "; - } - query << GetAlterTopicsVersionQuery(); - NYdb::TParamsBuilder builder; - auto params = builder - .AddParam("$version").Int64(++TopicsVersion).Build() - .AddParam("$path").Utf8(NPersQueue::GetTopicPath(topic)).Build() - .AddParam("$cluster").Utf8(NPersQueue::GetDC(topic)).Build() - .Build(); - - Cerr << "===Run query:``" << query << "`` with topic = " << NPersQueue::GetTopicPath(topic) << ", dc = " << NPersQueue::GetDC(topic) << Endl; - RunYqlDataQueryWithParams(query, params); - Cerr << "===Query complete" << Endl; - } - - void AlterTopic() { - NYdb::TParamsBuilder builder; - auto params = builder - .AddParam("$version").Int64(++TopicsVersion).Build() - .Build(); - TStringBuilder query; - query << "DECLARE $version as Int64; " << GetAlterTopicsVersionQuery(); - RunYqlDataQueryWithParams(query, params); - } + +private: + static TString GetAlterTopicsVersionQuery() { + return "UPSERT INTO [/Root/PQ/Config/V2/Versions] (name, version) VALUES (\"Topics\", $version);"; + } + +public: + void AddTopic(const TString& topic) { + return AddOrRemoveTopic(topic, true); + } + + void RemoveTopic(const TString& topic) { + return AddOrRemoveTopic(topic, false); + } + + void AddOrRemoveTopic(const TString& topic, bool add) { + TStringBuilder query; + query << "DECLARE $version as Int64; DECLARE $path AS Utf8; DECLARE $cluster as Utf8; "; + if (add) { + query << "UPSERT INTO [/Root/PQ/Config/V2/Topics] (path, dc) VALUES ($path, $cluster); "; + } else { + query << "DELETE FROM [/Root/PQ/Config/V2/Topics] WHERE path = $path AND dc = $cluster; "; + } + query << GetAlterTopicsVersionQuery(); + NYdb::TParamsBuilder builder; + auto params = builder + .AddParam("$version").Int64(++TopicsVersion).Build() + .AddParam("$path").Utf8(NPersQueue::GetTopicPath(topic)).Build() + .AddParam("$cluster").Utf8(NPersQueue::GetDC(topic)).Build() + .Build(); + + Cerr << "===Run query:``" << query << "`` with topic = " << NPersQueue::GetTopicPath(topic) << ", dc = " << NPersQueue::GetDC(topic) << Endl; + RunYqlDataQueryWithParams(query, params); + Cerr << "===Query complete" << Endl; + } + + void AlterTopic() { + NYdb::TParamsBuilder builder; + auto params = builder + .AddParam("$version").Int64(++TopicsVersion).Build() + .Build(); + TStringBuilder query; + query << "DECLARE $version as Int64; " << GetAlterTopicsVersionQuery(); + RunYqlDataQueryWithParams(query, params); + } }; } // namespace NPersQueueTests diff --git a/ydb/core/tx/scheme_cache/scheme_cache.h b/ydb/core/tx/scheme_cache/scheme_cache.h index e811d71865..26a07897a1 100644 --- a/ydb/core/tx/scheme_cache/scheme_cache.h +++ b/ydb/core/tx/scheme_cache/scheme_cache.h @@ -263,10 +263,10 @@ struct TSchemeCacheNavigate { : Instant(0) {} - explicit TSchemeCacheNavigate(ui64 instant) - : Instant(instant) - {} - + explicit TSchemeCacheNavigate(ui64 instant) + : Instant(instant) + {} + TString ToString(const NScheme::TTypeRegistry& typeRegistry) const; }; // TSchemeCacheNavigate diff --git a/ydb/core/ymq/actor/action.h b/ydb/core/ymq/actor/action.h index 04e319a5cd..1801c98d68 100644 --- a/ydb/core/ymq/actor/action.h +++ b/ydb/core/ymq/actor/action.h @@ -217,23 +217,23 @@ protected: RLOG_SQS_ERROR("Request timeout. User [" << UserName_ << "] Queue [" << GetQueueName() << "] Action [" << Action_ << "]. State: { " << DumpState() << " }"); if (QueueCounters_) { - INC_COUNTER_COUPLE(QueueCounters_, RequestTimeouts, request_timeouts_count_per_second); + INC_COUNTER_COUPLE(QueueCounters_, RequestTimeouts, request_timeouts_count_per_second); } else if (UserCounters_) { INC_COUNTER(UserCounters_, RequestTimeouts); } else { - TIntrusivePtrCntrCouple rootCounters { - SqsCoreCounters_ ? SqsCoreCounters_ : GetSqsServiceCounters(AppData()->Counters, "core"), - GetYmqPublicCounters(AppData()->Counters) - }; - auto [userCounters, queueCounters] = GetUserAndQueueCounters(rootCounters, TQueuePath(Cfg().GetRoot(), UserName_, GetQueueName())); - if (queueCounters.SqsCounters) { - queueCounters.SqsCounters->GetCounter("RequestTimeouts", true)->Inc(); - } else if (userCounters.SqsCounters) { - userCounters.SqsCounters->GetCounter("RequestTimeouts", true)->Inc(); + TIntrusivePtrCntrCouple rootCounters { + SqsCoreCounters_ ? SqsCoreCounters_ : GetSqsServiceCounters(AppData()->Counters, "core"), + GetYmqPublicCounters(AppData()->Counters) + }; + auto [userCounters, queueCounters] = GetUserAndQueueCounters(rootCounters, TQueuePath(Cfg().GetRoot(), UserName_, GetQueueName())); + if (queueCounters.SqsCounters) { + queueCounters.SqsCounters->GetCounter("RequestTimeouts", true)->Inc(); + } else if (userCounters.SqsCounters) { + userCounters.SqsCounters->GetCounter("RequestTimeouts", true)->Inc(); + } + if (queueCounters.YmqCounters) { + queueCounters.YmqCounters->GetCounter("RequestTimeouts", true)->Inc(); } - if (queueCounters.YmqCounters) { - queueCounters.YmqCounters->GetCounter("RequestTimeouts", true)->Inc(); - } } MakeError(MutableErrorDesc(), NErrors::TIMEOUT); @@ -260,30 +260,30 @@ protected: void SendReplyAndDie() { RLOG_SQS_TRACE("SendReplyAndDie from action actor " << Response_); - auto actionCountersCouple = GetActionCounters(); + auto actionCountersCouple = GetActionCounters(); auto* detailedCounters = UserCounters_ ? UserCounters_->GetDetailedCounters() : nullptr; const size_t errors = ErrorsCount(Response_, detailedCounters ? &detailedCounters->APIStatuses : nullptr); - if (actionCountersCouple.SqsCounters) { + if (actionCountersCouple.SqsCounters) { if (errors) { - ADD_COUNTER(actionCountersCouple.SqsCounters, Errors, errors); + ADD_COUNTER(actionCountersCouple.SqsCounters, Errors, errors); } else { - INC_COUNTER(actionCountersCouple.SqsCounters, Success); + INC_COUNTER(actionCountersCouple.SqsCounters, Success); + } + } + if (actionCountersCouple.YmqCounters) { + if (errors) { + ADD_COUNTER(actionCountersCouple.YmqCounters, Errors, errors); + } else { + INC_COUNTER(actionCountersCouple.YmqCounters, Success); } } - if (actionCountersCouple.YmqCounters) { - if (errors) { - ADD_COUNTER(actionCountersCouple.YmqCounters, Errors, errors); - } else { - INC_COUNTER(actionCountersCouple.YmqCounters, Success); - } - } FinishTs_ = TActivationContext::Now(); const TDuration workingDuration = GetRequestWorkingDuration(); RLOG_SQS_DEBUG("Request " << Action_ << " working duration: " << workingDuration.MilliSeconds() << "ms"); - if (actionCountersCouple.Defined()) { + if (actionCountersCouple.Defined()) { const TDuration duration = GetRequestDuration(); - COLLECT_HISTOGRAM_COUNTER_COUPLE(actionCountersCouple, Duration, duration.MilliSeconds()); - COLLECT_HISTOGRAM_COUNTER_COUPLE(actionCountersCouple, WorkingDuration, workingDuration.MilliSeconds()); + COLLECT_HISTOGRAM_COUNTER_COUPLE(actionCountersCouple, Duration, duration.MilliSeconds()); + COLLECT_HISTOGRAM_COUNTER_COUPLE(actionCountersCouple, WorkingDuration, workingDuration.MilliSeconds()); } if (IsRequestSlow()) { PrintSlowRequestWarning(); @@ -395,32 +395,32 @@ protected: bool IsForbiddenPath(const TString& path) const { return path.Contains("..") || path.Contains("//") || IsInternalResource(path); } - struct TActionCountersPack { - TActionCounters* CoreCounters = nullptr; - TActionCounters* YmqCounters = nullptr; - }; + struct TActionCountersPack { + TActionCounters* CoreCounters = nullptr; + TActionCounters* YmqCounters = nullptr; + }; - TCountersCouple<TActionCounters*> GetActionCounters() const { - TCountersCouple<TActionCounters*> result{nullptr, nullptr}; + TCountersCouple<TActionCounters*> GetActionCounters() const { + TCountersCouple<TActionCounters*> result{nullptr, nullptr}; if (IsActionForQueue(Action_) && QueueCounters_) { if (IsActionForMessage(Action_) || QueueCounters_->NeedToShowDetailedCounters()) { - result.SqsCounters = &QueueCounters_->SqsActionCounters[Action_]; + result.SqsCounters = &QueueCounters_->SqsActionCounters[Action_]; } } else if (IsActionForUser(Action_) && UserCounters_) { if (UserCounters_->NeedToShowDetailedCounters()) { - result.SqsCounters = &UserCounters_->SqsActionCounters[Action_]; + result.SqsCounters = &UserCounters_->SqsActionCounters[Action_]; + } + } + if (IsActionForQueueYMQ(Action_) && QueueCounters_) { + if (IsActionForMessage(Action_) || QueueCounters_->NeedToShowDetailedCounters()) { + result.YmqCounters = &QueueCounters_->YmqActionCounters[Action_]; + } + } else if (IsActionForUserYMQ(Action_) && UserCounters_) { + if (UserCounters_->NeedToShowDetailedCounters()) { + result.YmqCounters = &UserCounters_->YmqActionCounters[Action_]; } } - if (IsActionForQueueYMQ(Action_) && QueueCounters_) { - if (IsActionForMessage(Action_) || QueueCounters_->NeedToShowDetailedCounters()) { - result.YmqCounters = &QueueCounters_->YmqActionCounters[Action_]; - } - } else if (IsActionForUserYMQ(Action_) && UserCounters_) { - if (UserCounters_->NeedToShowDetailedCounters()) { - result.YmqCounters = &UserCounters_->YmqActionCounters[Action_]; - } - } - return result; + return result; } void RequestSchemeCache(const TString& path) { @@ -696,39 +696,39 @@ private: private: void Start() { - auto actionCountersCouple = GetActionCounters(); - if (actionCountersCouple.SqsCounters) { + auto actionCountersCouple = GetActionCounters(); + if (actionCountersCouple.SqsCounters) { if (IsActionForQueue(Action_) && QueueCounters_) { - NeedReportSqsActionInflyCounter = QueueCounters_->NeedToShowDetailedCounters(); + NeedReportSqsActionInflyCounter = QueueCounters_->NeedToShowDetailedCounters(); } else if (IsActionForUser(Action_) && UserCounters_) { - NeedReportSqsActionInflyCounter = UserCounters_->NeedToShowDetailedCounters(); + NeedReportSqsActionInflyCounter = UserCounters_->NeedToShowDetailedCounters(); } - if (NeedReportSqsActionInflyCounter) { - INC_COUNTER(actionCountersCouple.SqsCounters, Infly); + if (NeedReportSqsActionInflyCounter) { + INC_COUNTER(actionCountersCouple.SqsCounters, Infly); + } + } + if (actionCountersCouple.YmqCounters) { + if (IsActionForQueueYMQ(Action_) && QueueCounters_) { + NeedReportYmqActionInflyCounter = QueueCounters_->NeedToShowDetailedCounters(); + } else if (IsActionForUserYMQ(Action_) && UserCounters_) { + NeedReportYmqActionInflyCounter = UserCounters_->NeedToShowDetailedCounters(); + } + if (NeedReportYmqActionInflyCounter) { + INC_COUNTER(actionCountersCouple.YmqCounters, Infly); } } - if (actionCountersCouple.YmqCounters) { - if (IsActionForQueueYMQ(Action_) && QueueCounters_) { - NeedReportYmqActionInflyCounter = QueueCounters_->NeedToShowDetailedCounters(); - } else if (IsActionForUserYMQ(Action_) && UserCounters_) { - NeedReportYmqActionInflyCounter = UserCounters_->NeedToShowDetailedCounters(); - } - if (NeedReportYmqActionInflyCounter) { - INC_COUNTER(actionCountersCouple.YmqCounters, Infly); - } - } DoStart(); StartRequestWasCalled_ = true; } void Finish() { - auto actionCounters = GetActionCounters(); - if (NeedReportSqsActionInflyCounter) { - DEC_COUNTER(actionCounters.SqsCounters, Infly); + auto actionCounters = GetActionCounters(); + if (NeedReportSqsActionInflyCounter) { + DEC_COUNTER(actionCounters.SqsCounters, Infly); + } + if (NeedReportYmqActionInflyCounter && actionCounters.YmqCounters) { + DEC_COUNTER(actionCounters.YmqCounters, Infly); } - if (NeedReportYmqActionInflyCounter && actionCounters.YmqCounters) { - DEC_COUNTER(actionCounters.YmqCounters, Infly); - } if (StartRequestWasCalled_) { DoFinish(); FinishRequestWasCalled_ = true; @@ -756,7 +756,7 @@ protected: TInstant StartTs_; TInstant FinishTs_; TIntrusivePtr<NMonitoring::TDynamicCounters> SqsCoreCounters_; // Raw counters interface. Is is not prefered to use them - TIntrusivePtr<NMonitoring::TDynamicCounters> YmqRootCounters_; // Raw counters interface. Is is not prefered to use them + TIntrusivePtr<NMonitoring::TDynamicCounters> YmqRootCounters_; // Raw counters interface. Is is not prefered to use them TIntrusivePtr<TUserCounters> UserCounters_; TIntrusivePtr<TQueueCounters> QueueCounters_; TMaybe<TSqsEvents::TQueueAttributes> QueueAttributes_; @@ -767,8 +767,8 @@ protected: bool FinishRequestWasCalled_ = false; TInstant QuotaRequestTs_; TIntrusivePtr<TSqsEvents::TQuoterResourcesForActions> QuoterResources_; - bool NeedReportSqsActionInflyCounter = false; - bool NeedReportYmqActionInflyCounter = false; + bool NeedReportSqsActionInflyCounter = false; + bool NeedReportYmqActionInflyCounter = false; TSchedulerCookieHolder TimeoutCookie_ = ISchedulerCookie::Make2Way(); NKikimrClient::TSqsRequest SourceSqsRequest_; }; diff --git a/ydb/core/ymq/actor/delete_message.cpp b/ydb/core/ymq/actor/delete_message.cpp index 7e11e5b061..50b59ddcfb 100644 --- a/ydb/core/ymq/actor/delete_message.cpp +++ b/ydb/core/ymq/actor/delete_message.cpp @@ -94,7 +94,7 @@ private: // Calc metrics const TDuration processingDuration = TActivationContext::Now() - lockTimestamp; COLLECT_HISTOGRAM_COUNTER(QueueCounters_, ClientMessageProcessing_Duration, processingDuration.MilliSeconds()); - COLLECT_HISTOGRAM_COUNTER(QueueCounters_, client_processing_duration_milliseconds, processingDuration.MilliSeconds()); + COLLECT_HISTOGRAM_COUNTER(QueueCounters_, client_processing_duration_milliseconds, processingDuration.MilliSeconds()); } catch (...) { RLOG_SQS_WARN("Failed to process receipt handle " << entry.GetReceiptHandle() << ": " << CurrentExceptionMessage()); MakeError(resp, NErrors::RECEIPT_HANDLE_IS_INVALID); @@ -104,7 +104,7 @@ private: void ProcessAnswer(TDeleteMessageResponse* resp, const TSqsEvents::TEvDeleteMessageBatchResponse::TMessageResult& answer) { switch (answer.Status) { case TSqsEvents::TEvDeleteMessageBatchResponse::EDeleteMessageStatus::OK: { - INC_COUNTER_COUPLE(QueueCounters_, DeleteMessage_Count, deleted_count_per_second); + INC_COUNTER_COUPLE(QueueCounters_, DeleteMessage_Count, deleted_count_per_second); break; } case TSqsEvents::TEvDeleteMessageBatchResponse::EDeleteMessageStatus::NotFound: { diff --git a/ydb/core/ymq/actor/events.h b/ydb/core/ymq/actor/events.h index 0e5948556d..9fff62c7cf 100644 --- a/ydb/core/ymq/actor/events.h +++ b/ydb/core/ymq/actor/events.h @@ -124,8 +124,8 @@ struct TSqsEvents { EvGarbageSearchResult, - EvCleanupQueryComplete, - + EvCleanupQueryComplete, + EvEnd, }; @@ -876,15 +876,15 @@ struct TSqsEvents { bool Success = true; THashMap<TString, TGarbageHint> GarbageHints; }; - struct TEvCleanupQueryComplete : public NActors::TEventLocal<TEvCleanupQueryComplete, EvCleanupQueryComplete> { - explicit TEvCleanupQueryComplete(const TString& name, ui64 type) - : Name(name) - , Type(type) - { - } - TString Name; - ui64 Type; - }; + struct TEvCleanupQueryComplete : public NActors::TEventLocal<TEvCleanupQueryComplete, EvCleanupQueryComplete> { + explicit TEvCleanupQueryComplete(const TString& name, ui64 type) + : Name(name) + , Type(type) + { + } + TString Name; + ui64 Type; + }; }; } // namespace NKikimr::NSQS diff --git a/ydb/core/ymq/actor/index_events_processor.cpp b/ydb/core/ymq/actor/index_events_processor.cpp index 16ba23d32f..646e06420e 100644 --- a/ydb/core/ymq/actor/index_events_processor.cpp +++ b/ydb/core/ymq/actor/index_events_processor.cpp @@ -1,493 +1,493 @@ -#include "index_events_processor.h" +#include "index_events_processor.h" #include <ydb/public/sdk/cpp/client/ydb_types/ydb.h> - -namespace NKikimr::NSQS { -using namespace NActors; -using namespace NJson; -using namespace NYdb; - -constexpr TDuration DEFAULT_RETRY_TIMEOUT = TDuration::Seconds(10); -constexpr TDuration SHORT_RETRY_TIMEOUT = TDuration::Seconds(2); -constexpr TDuration DEFAULT_QUERY_TIMEOUT = TDuration::Seconds(30); - -TSearchEventsProcessor::TSearchEventsProcessor( - const TString& root, const TDuration& reindexInterval, const TDuration& rescanInterval, - const TSimpleSharedPtr<NTable::TTableClient>& tableClient, - IEventsWriterWrapper::TPtr eventsWriter, - bool waitForWake -) - : RescanInterval(rescanInterval) - , ReindexInterval(reindexInterval) - , TableClient(tableClient) - , EventsWriter(eventsWriter) - , WaitForWake(waitForWake) -{ - InitQueries(root); -} - -void TSearchEventsProcessor::InitQueries(const TString& root) { - auto getTableFullPath = [&](const TString& tableName) { - if (!root.empty()) { - return root + "/" + tableName; - } else { - return tableName; - } - }; - TString eventsTablePath = getTableFullPath(".Events"); - - SelectQueuesQuery = TStringBuilder() - << "--!syntax_v1\n" - << "DECLARE $Account as Utf8; DECLARE $QueueName as Utf8; " - << "SELECT Account, QueueName, CustomQueueName, CreatedTimestamp, FolderId from `" - << getTableFullPath(".Queues") << "` " - << "WHERE Account > $Account OR (Account = $Account AND QueueName > $QueueName);"; - - SelectEventsQuery = TStringBuilder() - << "--!syntax_v1\n" - << "SELECT Account, QueueName, EventType, CustomQueueName, EventTimestamp, FolderId " - << "FROM `"<< eventsTablePath << "`;"; - - DeleteEventQuery = TStringBuilder() - << "--!syntax_v1\n" - << "DECLARE $Events AS List<Struct<Account:Utf8, QueueName:Utf8, EventType:Uint64>>;\n" - << "$EventsSource = (SELECT item.Account AS Account, item.QueueName AS QueueName, item.EventType AS EventType \n" - << "FROM (SELECT $Events AS events) FLATTEN LIST BY events as item);\n" - << "DELETE FROM `" << eventsTablePath << "` ON SELECT * FROM $EventsSource;"; -} - -TSearchEventsProcessor::~TSearchEventsProcessor() { - EventsWriter->Close(); -} - -void TSearchEventsProcessor::Bootstrap(const TActorContext& ctx) { - Become(&TSearchEventsProcessor::StateFunc); - if (!WaitForWake) - ctx.Schedule(DEFAULT_RETRY_TIMEOUT, new TEvWakeup(StartQueuesListingTag)); -} - -STFUNC(TSearchEventsProcessor::StateFunc) { - switch (ev->GetTypeRewrite()) { - HFunc(TEvWakeup, HandleWakeup); - default: - Y_FAIL(); - }; -} - -void TSearchEventsProcessor::HandleWakeup(TEvWakeup::TPtr& ev, const TActorContext& ctx) { - if (Stopping) - return; - switch (ev->Get()->Tag) { - case StartQueuesListingTag: - StartQueuesListing(ctx); - break; - case QueuesListSessionStartedTag: - OnQueuesListSessionReady(ctx); - break; - case QueuesListPreparedTag: - OnQueuesListPrepared(ctx); - break; - case QueuesListQueryCompleteTag: - OnQueuesListQueryComplete(ctx); - break; - case RunEventsListingTag: - RunEventsListing(ctx); - break; - case EventsListingDoneTag: - OnEventsListingDone(ctx); - break; - case StartCleanupTag: - StartCleanupSession(ctx); - break; - case CleanupSessionReadyTag: - OnCleanupSessionReady(ctx); - break; - case CleanupTxReadyTag: - OnCleanupTxReady(ctx); - break; - case CleanupPreparedTag: - OnCleanupPrepared(ctx); - break; - case CleanupQueryCompleteTag: - OnCleanupQueryComplete(ctx); - break; - case CleanupTxCommittedTag: - OnCleanupTxCommitted(ctx); - break; - case StopAllTag: - Stopping = true; - EventsWriter->Close(); - break; - default: - Y_FAIL(); - } -} - -void TSearchEventsProcessor::StartQueuesListing(const TActorContext& ctx) { - LastQueuesKey = {}; - StartSession(QueuesListSessionStartedTag, ctx); -} - -void TSearchEventsProcessor::OnQueuesListSessionReady(const TActorContext &ctx) { - if (!SessionStarted(ctx)) { - return ctx.Schedule(DEFAULT_RETRY_TIMEOUT, new TEvWakeup(StartQueuesListingTag)); - } - PrepareDataQuery(SelectQueuesQuery, QueuesListPreparedTag, ctx); -} - -void TSearchEventsProcessor::OnQueuesListPrepared(const TActorContext &ctx) { - if (!QueryPrepared(ctx)) { - return ctx.Schedule(DEFAULT_RETRY_TIMEOUT, new TEvWakeup(StartQueuesListingTag)); - } - RunQueuesListQuery(ctx, true); -} - -void TSearchEventsProcessor::RunQueuesListQuery(const TActorContext& ctx, bool initial) { - if (initial) { - ExistingQueues.clear(); - } - Y_VERIFY(Session.Defined()); - Y_VERIFY(PreparedQuery.Defined()); - - auto builder = PreparedQuery.Get()->GetParamsBuilder(); - { - auto ¶m = builder.AddParam("$Account"); - param.Utf8(LastQueuesKey.Account); - param.Build(); - } - { - auto ¶m = builder.AddParam("$QueueName"); - param.Utf8(LastQueuesKey.QueueName); - param.Build(); - } - RunPrepared(builder.Build(), QueuesListQueryCompleteTag, ctx); -} - -void TSearchEventsProcessor::OnQueuesListQueryComplete(const TActorContext& ctx) { - if (!QueryComplete(ctx)) { - return ctx.Schedule(DEFAULT_RETRY_TIMEOUT, new TEvWakeup(StartQueuesListingTag)); - } - - const auto& result = QueryResult.Get()->GetResultSet(0); - - TResultSetParser parser(result); - while (parser.TryNextRow()) { - TString queueName = *parser.ColumnParser("QueueName").GetOptionalUtf8(); - TString cloudId = *parser.ColumnParser("Account").GetOptionalUtf8(); - TString customName = *parser.ColumnParser("CustomQueueName").GetOptionalUtf8(); - ui64 createTs = *parser.ColumnParser("CreatedTimestamp").GetOptionalUint64(); - TString folderId = *parser.ColumnParser("FolderId").GetOptionalUtf8(); - auto insResult = ExistingQueues.insert(std::make_pair( - queueName, TQueueEvent{EQueueEventType::Existed, createTs, customName, cloudId, folderId} - )); - Y_VERIFY(insResult.second); - LastQueuesKey.QueueName = queueName; - LastQueuesKey.Account = cloudId; - } - if (result.Truncated()) { - RunQueuesListQuery(ctx, false); - } else { - Send(ctx.SelfID, new TEvWakeup(RunEventsListingTag)); - } -} - -void TSearchEventsProcessor::RunEventsListing(const TActorContext& ctx) { - QueryResult = Nothing(); - Status = TableClient->RetryOperation<NTable::TDataQueryResult>([this, query = SelectEventsQuery](NTable::TSession session) { - auto tx = NTable::TTxControl::BeginTx().CommitTx(); - return session.ExecuteDataQuery( - query, tx, NTable::TExecDataQuerySettings().ClientTimeout(DEFAULT_QUERY_TIMEOUT) - ).Apply([this](const auto& future) mutable { - QueryResult = future.GetValue(); - return future; - }); - }); - Apply(&Status, EventsListingDoneTag, ctx); -} - -void TSearchEventsProcessor::OnEventsListingDone(const TActorContext& ctx) { - auto& status = Status.GetValue(); - if (!status.IsSuccess()) { - Schedule(DEFAULT_RETRY_TIMEOUT, new TEvWakeup(RunEventsListingTag)); - return; - } - - Y_VERIFY(QueryResult.Defined()); - Y_VERIFY(QueryResult.Get()->GetResultSets().size() == 1); - const auto& result = QueryResult.Get()->GetResultSet(0); - TResultSetParser parser(result); - QueuesEvents.clear(); - - while (parser.TryNextRow()) { - TString cloudId = *parser.ColumnParser("Account").GetOptionalUtf8(); - TString queueName = *parser.ColumnParser("QueueName").GetOptionalUtf8(); - ui64 evType = *parser.ColumnParser("EventType").GetOptionalUint64(); - TString customName = *parser.ColumnParser("CustomQueueName").GetOptionalUtf8(); - TString folderId = *parser.ColumnParser("FolderId").GetOptionalUtf8(); - ui64 timestamp = *parser.ColumnParser("EventTimestamp").GetOptionalUint64(); - auto& qEvents = QueuesEvents[queueName]; - auto insResult = qEvents.insert(std::make_pair( - timestamp, TQueueEvent{EQueueEventType(evType), timestamp, customName, cloudId, folderId} - )); - Y_VERIFY(insResult.second); - } - ProcessEventsQueue(ctx); -} - -void TSearchEventsProcessor::StartCleanupSession(const TActorContext& ctx) { - StartSession(CleanupSessionReadyTag, ctx); -} - -void TSearchEventsProcessor::OnCleanupSessionReady(const TActorContext &ctx) { - if (!SessionStarted(ctx)) { - return ctx.Schedule(SHORT_RETRY_TIMEOUT, new TEvWakeup(StartCleanupTag)); - } - StartTx(CleanupTxReadyTag, ctx); -} - -void TSearchEventsProcessor::OnCleanupTxReady(const TActorContext &ctx) { - if (!TxStarted(ctx)) { - return ctx.Schedule(SHORT_RETRY_TIMEOUT, new TEvWakeup(StartCleanupTag)); - } - PrepareDataQuery(DeleteEventQuery, CleanupPreparedTag, ctx); -} - -void TSearchEventsProcessor::OnCleanupPrepared(const TActorContext &ctx) { - if (!QueryPrepared(ctx)) { - return ctx.Schedule(DEFAULT_RETRY_TIMEOUT, new TEvWakeup(StartCleanupTag)); - } - RunEventsCleanup(ctx); -} - -void TSearchEventsProcessor::RunEventsCleanup(const TActorContext& ctx) { - Y_VERIFY(PreparedQuery.Defined()); - Y_VERIFY(CurrentTx.Defined()); - - auto builder = PreparedQuery.Get()->GetParamsBuilder(); - auto ¶m = builder.AddParam("$Events"); - param.BeginList(); - for (const auto&[qName, events] : QueuesEvents) { - for (const auto&[_, event]: events) { - param.AddListItem().BeginStruct().AddMember("Account").Utf8(event.CloudId) - .AddMember("QueueName").Utf8(qName) - .AddMember("EventType").Uint64(static_cast<ui64>(event.Type)) - .EndStruct(); - - } - } - param.EndList().Build(); - RunPrepared(builder.Build(), CleanupQueryCompleteTag, ctx); -} - -void TSearchEventsProcessor::OnCleanupQueryComplete(const TActorContext& ctx) { - if (!QueryComplete(ctx)) { - return ctx.Schedule(SHORT_RETRY_TIMEOUT, new TEvWakeup(StartCleanupTag)); - } - CommitTx(CleanupTxCommittedTag, ctx); -} - -void TSearchEventsProcessor::OnCleanupTxCommitted(const TActorContext &ctx) { - if (!TxCommitted(ctx)) { - return ctx.Schedule(DEFAULT_RETRY_TIMEOUT, new TEvWakeup(StartCleanupTag)); - } - QueuesEvents.clear(); - ProcessReindexIfRequired(ctx); -} - -void TSearchEventsProcessor::ProcessEventsQueue(const TActorContext& ctx) { - - for (const auto& [qName, events] : QueuesEvents) { - for (const auto& [ts, event]: events) { - auto existsIter = ExistingQueues.find(qName); - if (event.Type == EQueueEventType::Deleted) { - if (existsIter != ExistingQueues.end() && existsIter->second.CustomName == event.CustomName - && existsIter->second.Timestamp < ts) { - ExistingQueues.erase(existsIter); - } - } else if (event.Type == EQueueEventType::Created) { - if (existsIter == ExistingQueues.end()) { - auto iter = ExistingQueues.insert(std::make_pair( - qName, event - )).first; - iter->second.Type = EQueueEventType::Existed; - } - } - SaveQueueEvent(qName, event, ctx); - } - } - if (!QueuesEvents.empty()) { - Send(ctx.SelfID, new TEvWakeup(StartCleanupTag)); - } else { - ProcessReindexIfRequired(ctx); - } -} - -void TSearchEventsProcessor::ProcessReindexIfRequired(const TActorContext &ctx) { - if (LastReindex == TInstant::Zero() || LastReindex + ReindexInterval < ctx.Now()) { - if (ExistingQueues) { - TStringStream ss; - NJson::TJsonWriter writer(&ss, false); - for (const auto&[queue, event] : ExistingQueues) { - SaveQueueEvent(queue, event, ctx); - } - } - ProcessReindexResult(ctx); - } else { - WaitNextCycle(ctx); - } -} -void TSearchEventsProcessor::SendJson(const TString& json, const TActorContext &) { - EventsWriter->Write(json); -} - -void TSearchEventsProcessor::ProcessReindexResult(const TActorContext &ctx) { - ReindexComplete.Inc(); - LastReindex = ctx.Now(); - WaitNextCycle(ctx); -} - -void TSearchEventsProcessor::WaitNextCycle(const TActorContext &ctx) { - ctx.Schedule(RescanInterval, new TEvWakeup(StartQueuesListingTag)); -} - -ui64 TSearchEventsProcessor::GetReindexCount() const { - return ReindexComplete.Val(); -} - -NActors::TActorSystem* TSearchEventsProcessor::GetActorSystem() { - return TActivationContext::ActorSystem(); -} - -void TSearchEventsProcessor::StartSession(ui32 evTag, const TActorContext &ctx) { - Session = Nothing(); - CurrentTx = Nothing(); - PreparedQuery = Nothing(); - QueryResult = Nothing(); - SessionFuture = TableClient->GetSession(); - Apply(&SessionFuture, evTag, ctx); -} - -void TSearchEventsProcessor::PrepareDataQuery(const TString& query, ui32 evTag, const TActorContext& ctx) { - Y_VERIFY(Session.Defined()); - PreparedQuery = Nothing(); - QueryResult = Nothing(); - PrepQueryFuture = Session.Get()->PrepareDataQuery(query); - Apply(&PrepQueryFuture, evTag, ctx); -} - -void TSearchEventsProcessor::StartTx(ui32 evTag, const TActorContext &ctx) { - Y_VERIFY(Session.Defined()); - CurrentTx = Nothing(); - TxFuture = Session.Get()->BeginTransaction(); - Apply(&TxFuture, evTag, ctx); -} - -void TSearchEventsProcessor::CommitTx(ui32 evTag, const TActorContext &ctx) { - Y_VERIFY(Session.Defined()); - Y_VERIFY(CurrentTx.Defined()); - CommitTxFuture = CurrentTx.Get()->Commit(); - Apply(&CommitTxFuture, evTag, ctx); -} - -void TSearchEventsProcessor::RunPrepared(NYdb::TParams&& params, ui32 evTag, const TActorContext& ctx) { - Y_VERIFY(Session.Defined()); - Y_VERIFY(PreparedQuery.Defined()); - QueryResult = Nothing(); - NTable::TTxControl txControl = CurrentTx.Defined() ? NTable::TTxControl::Tx(*CurrentTx) - : NTable::TTxControl::BeginTx().CommitTx(); - QueryResultFuture = PreparedQuery.Get()->Execute(txControl, std::forward<NYdb::TParams>(params)); - Apply(&QueryResultFuture, evTag, ctx); -} - -bool TSearchEventsProcessor::SessionStarted(const TActorContext&) { - Y_VERIFY(SessionFuture.HasValue()); - auto& value = SessionFuture.GetValueSync(); - if (!value.IsSuccess()) - return false; - Session = value.GetSession(); - return true; -} - -bool TSearchEventsProcessor::TxStarted(const TActorContext&) { - Y_VERIFY(Session.Defined()); - Y_VERIFY(TxFuture.HasValue()); - auto& value = TxFuture.GetValueSync(); - if (!value.IsSuccess()) - return false; - CurrentTx = value.GetTransaction(); - return true; -} -bool TSearchEventsProcessor::TxCommitted(const TActorContext&) { - Y_VERIFY(Session.Defined()); - Y_VERIFY(CurrentTx.Defined()); - Y_VERIFY(CommitTxFuture.HasValue()); - auto& value = CommitTxFuture.GetValueSync(); - if (!value.IsSuccess()) - return false; - CurrentTx = Nothing(); - return true; -} - -bool TSearchEventsProcessor::QueryPrepared(const TActorContext&) { - Y_VERIFY(Session.Defined()); - Y_VERIFY(PrepQueryFuture.HasValue()); - auto& value = PrepQueryFuture.GetValueSync(); - if (!value.IsSuccess()) { - return false; - } - PreparedQuery = value.GetQuery(); - return true; -} -bool TSearchEventsProcessor::QueryComplete(const TActorContext&) { - Y_VERIFY(Session.Defined()); - Y_VERIFY(QueryResultFuture.HasValue()); - QueryResult = QueryResultFuture.GetValueSync(); - if (!QueryResult.Get()->IsSuccess()) { - return false; - } - return true; -} - - -void TSearchEventsProcessor::SaveQueueEvent( - - const TString& queueName, const TQueueEvent& event, const TActorContext& ctx -) { - auto tsIsoString = TInstant::MilliSeconds(event.Timestamp).ToIsoStringLocal(); - auto nowIsoString = TInstant::Now().ToIsoStringLocal(); - TStringStream ss; - NJson::TJsonWriter writer(&ss, false); - - writer.OpenMap(); - { - writer.Write("resource_type", "message-queue"); - writer.Write("timestamp", tsIsoString); - writer.Write("resource_id", queueName); - writer.Write("name", event.CustomName); - writer.Write("service", "message-queue"); - if (event.Type == EQueueEventType::Deleted) - writer.Write("deleted", tsIsoString); - if (event.Type == EQueueEventType::Existed) - writer.Write("reindex_timestamp", nowIsoString); - writer.Write("permission", "ymq.queues.list"); - writer.Write("cloud_id", event.CloudId); - writer.Write("folder_id", event.FolderId); - writer.OpenArray("resource_path"); - writer.OpenMap(); - writer.Write("resource_type", "resource-manager.folder"); - writer.Write("resource_id", event.FolderId); - writer.CloseMap(); - writer.CloseArray(); // resource_path - } - writer.CloseMap(); - writer.Flush(); - SendJson(ss.Str(), ctx); -} - -void IEventsWriterWrapper::Close() { - if (!Closed) { - Closed = true; - CloseImpl(); - } -} - + +namespace NKikimr::NSQS { +using namespace NActors; +using namespace NJson; +using namespace NYdb; + +constexpr TDuration DEFAULT_RETRY_TIMEOUT = TDuration::Seconds(10); +constexpr TDuration SHORT_RETRY_TIMEOUT = TDuration::Seconds(2); +constexpr TDuration DEFAULT_QUERY_TIMEOUT = TDuration::Seconds(30); + +TSearchEventsProcessor::TSearchEventsProcessor( + const TString& root, const TDuration& reindexInterval, const TDuration& rescanInterval, + const TSimpleSharedPtr<NTable::TTableClient>& tableClient, + IEventsWriterWrapper::TPtr eventsWriter, + bool waitForWake +) + : RescanInterval(rescanInterval) + , ReindexInterval(reindexInterval) + , TableClient(tableClient) + , EventsWriter(eventsWriter) + , WaitForWake(waitForWake) +{ + InitQueries(root); +} + +void TSearchEventsProcessor::InitQueries(const TString& root) { + auto getTableFullPath = [&](const TString& tableName) { + if (!root.empty()) { + return root + "/" + tableName; + } else { + return tableName; + } + }; + TString eventsTablePath = getTableFullPath(".Events"); + + SelectQueuesQuery = TStringBuilder() + << "--!syntax_v1\n" + << "DECLARE $Account as Utf8; DECLARE $QueueName as Utf8; " + << "SELECT Account, QueueName, CustomQueueName, CreatedTimestamp, FolderId from `" + << getTableFullPath(".Queues") << "` " + << "WHERE Account > $Account OR (Account = $Account AND QueueName > $QueueName);"; + + SelectEventsQuery = TStringBuilder() + << "--!syntax_v1\n" + << "SELECT Account, QueueName, EventType, CustomQueueName, EventTimestamp, FolderId " + << "FROM `"<< eventsTablePath << "`;"; + + DeleteEventQuery = TStringBuilder() + << "--!syntax_v1\n" + << "DECLARE $Events AS List<Struct<Account:Utf8, QueueName:Utf8, EventType:Uint64>>;\n" + << "$EventsSource = (SELECT item.Account AS Account, item.QueueName AS QueueName, item.EventType AS EventType \n" + << "FROM (SELECT $Events AS events) FLATTEN LIST BY events as item);\n" + << "DELETE FROM `" << eventsTablePath << "` ON SELECT * FROM $EventsSource;"; +} + +TSearchEventsProcessor::~TSearchEventsProcessor() { + EventsWriter->Close(); +} + +void TSearchEventsProcessor::Bootstrap(const TActorContext& ctx) { + Become(&TSearchEventsProcessor::StateFunc); + if (!WaitForWake) + ctx.Schedule(DEFAULT_RETRY_TIMEOUT, new TEvWakeup(StartQueuesListingTag)); +} + +STFUNC(TSearchEventsProcessor::StateFunc) { + switch (ev->GetTypeRewrite()) { + HFunc(TEvWakeup, HandleWakeup); + default: + Y_FAIL(); + }; +} + +void TSearchEventsProcessor::HandleWakeup(TEvWakeup::TPtr& ev, const TActorContext& ctx) { + if (Stopping) + return; + switch (ev->Get()->Tag) { + case StartQueuesListingTag: + StartQueuesListing(ctx); + break; + case QueuesListSessionStartedTag: + OnQueuesListSessionReady(ctx); + break; + case QueuesListPreparedTag: + OnQueuesListPrepared(ctx); + break; + case QueuesListQueryCompleteTag: + OnQueuesListQueryComplete(ctx); + break; + case RunEventsListingTag: + RunEventsListing(ctx); + break; + case EventsListingDoneTag: + OnEventsListingDone(ctx); + break; + case StartCleanupTag: + StartCleanupSession(ctx); + break; + case CleanupSessionReadyTag: + OnCleanupSessionReady(ctx); + break; + case CleanupTxReadyTag: + OnCleanupTxReady(ctx); + break; + case CleanupPreparedTag: + OnCleanupPrepared(ctx); + break; + case CleanupQueryCompleteTag: + OnCleanupQueryComplete(ctx); + break; + case CleanupTxCommittedTag: + OnCleanupTxCommitted(ctx); + break; + case StopAllTag: + Stopping = true; + EventsWriter->Close(); + break; + default: + Y_FAIL(); + } +} + +void TSearchEventsProcessor::StartQueuesListing(const TActorContext& ctx) { + LastQueuesKey = {}; + StartSession(QueuesListSessionStartedTag, ctx); +} + +void TSearchEventsProcessor::OnQueuesListSessionReady(const TActorContext &ctx) { + if (!SessionStarted(ctx)) { + return ctx.Schedule(DEFAULT_RETRY_TIMEOUT, new TEvWakeup(StartQueuesListingTag)); + } + PrepareDataQuery(SelectQueuesQuery, QueuesListPreparedTag, ctx); +} + +void TSearchEventsProcessor::OnQueuesListPrepared(const TActorContext &ctx) { + if (!QueryPrepared(ctx)) { + return ctx.Schedule(DEFAULT_RETRY_TIMEOUT, new TEvWakeup(StartQueuesListingTag)); + } + RunQueuesListQuery(ctx, true); +} + +void TSearchEventsProcessor::RunQueuesListQuery(const TActorContext& ctx, bool initial) { + if (initial) { + ExistingQueues.clear(); + } + Y_VERIFY(Session.Defined()); + Y_VERIFY(PreparedQuery.Defined()); + + auto builder = PreparedQuery.Get()->GetParamsBuilder(); + { + auto ¶m = builder.AddParam("$Account"); + param.Utf8(LastQueuesKey.Account); + param.Build(); + } + { + auto ¶m = builder.AddParam("$QueueName"); + param.Utf8(LastQueuesKey.QueueName); + param.Build(); + } + RunPrepared(builder.Build(), QueuesListQueryCompleteTag, ctx); +} + +void TSearchEventsProcessor::OnQueuesListQueryComplete(const TActorContext& ctx) { + if (!QueryComplete(ctx)) { + return ctx.Schedule(DEFAULT_RETRY_TIMEOUT, new TEvWakeup(StartQueuesListingTag)); + } + + const auto& result = QueryResult.Get()->GetResultSet(0); + + TResultSetParser parser(result); + while (parser.TryNextRow()) { + TString queueName = *parser.ColumnParser("QueueName").GetOptionalUtf8(); + TString cloudId = *parser.ColumnParser("Account").GetOptionalUtf8(); + TString customName = *parser.ColumnParser("CustomQueueName").GetOptionalUtf8(); + ui64 createTs = *parser.ColumnParser("CreatedTimestamp").GetOptionalUint64(); + TString folderId = *parser.ColumnParser("FolderId").GetOptionalUtf8(); + auto insResult = ExistingQueues.insert(std::make_pair( + queueName, TQueueEvent{EQueueEventType::Existed, createTs, customName, cloudId, folderId} + )); + Y_VERIFY(insResult.second); + LastQueuesKey.QueueName = queueName; + LastQueuesKey.Account = cloudId; + } + if (result.Truncated()) { + RunQueuesListQuery(ctx, false); + } else { + Send(ctx.SelfID, new TEvWakeup(RunEventsListingTag)); + } +} + +void TSearchEventsProcessor::RunEventsListing(const TActorContext& ctx) { + QueryResult = Nothing(); + Status = TableClient->RetryOperation<NTable::TDataQueryResult>([this, query = SelectEventsQuery](NTable::TSession session) { + auto tx = NTable::TTxControl::BeginTx().CommitTx(); + return session.ExecuteDataQuery( + query, tx, NTable::TExecDataQuerySettings().ClientTimeout(DEFAULT_QUERY_TIMEOUT) + ).Apply([this](const auto& future) mutable { + QueryResult = future.GetValue(); + return future; + }); + }); + Apply(&Status, EventsListingDoneTag, ctx); +} + +void TSearchEventsProcessor::OnEventsListingDone(const TActorContext& ctx) { + auto& status = Status.GetValue(); + if (!status.IsSuccess()) { + Schedule(DEFAULT_RETRY_TIMEOUT, new TEvWakeup(RunEventsListingTag)); + return; + } + + Y_VERIFY(QueryResult.Defined()); + Y_VERIFY(QueryResult.Get()->GetResultSets().size() == 1); + const auto& result = QueryResult.Get()->GetResultSet(0); + TResultSetParser parser(result); + QueuesEvents.clear(); + + while (parser.TryNextRow()) { + TString cloudId = *parser.ColumnParser("Account").GetOptionalUtf8(); + TString queueName = *parser.ColumnParser("QueueName").GetOptionalUtf8(); + ui64 evType = *parser.ColumnParser("EventType").GetOptionalUint64(); + TString customName = *parser.ColumnParser("CustomQueueName").GetOptionalUtf8(); + TString folderId = *parser.ColumnParser("FolderId").GetOptionalUtf8(); + ui64 timestamp = *parser.ColumnParser("EventTimestamp").GetOptionalUint64(); + auto& qEvents = QueuesEvents[queueName]; + auto insResult = qEvents.insert(std::make_pair( + timestamp, TQueueEvent{EQueueEventType(evType), timestamp, customName, cloudId, folderId} + )); + Y_VERIFY(insResult.second); + } + ProcessEventsQueue(ctx); +} + +void TSearchEventsProcessor::StartCleanupSession(const TActorContext& ctx) { + StartSession(CleanupSessionReadyTag, ctx); +} + +void TSearchEventsProcessor::OnCleanupSessionReady(const TActorContext &ctx) { + if (!SessionStarted(ctx)) { + return ctx.Schedule(SHORT_RETRY_TIMEOUT, new TEvWakeup(StartCleanupTag)); + } + StartTx(CleanupTxReadyTag, ctx); +} + +void TSearchEventsProcessor::OnCleanupTxReady(const TActorContext &ctx) { + if (!TxStarted(ctx)) { + return ctx.Schedule(SHORT_RETRY_TIMEOUT, new TEvWakeup(StartCleanupTag)); + } + PrepareDataQuery(DeleteEventQuery, CleanupPreparedTag, ctx); +} + +void TSearchEventsProcessor::OnCleanupPrepared(const TActorContext &ctx) { + if (!QueryPrepared(ctx)) { + return ctx.Schedule(DEFAULT_RETRY_TIMEOUT, new TEvWakeup(StartCleanupTag)); + } + RunEventsCleanup(ctx); +} + +void TSearchEventsProcessor::RunEventsCleanup(const TActorContext& ctx) { + Y_VERIFY(PreparedQuery.Defined()); + Y_VERIFY(CurrentTx.Defined()); + + auto builder = PreparedQuery.Get()->GetParamsBuilder(); + auto ¶m = builder.AddParam("$Events"); + param.BeginList(); + for (const auto&[qName, events] : QueuesEvents) { + for (const auto&[_, event]: events) { + param.AddListItem().BeginStruct().AddMember("Account").Utf8(event.CloudId) + .AddMember("QueueName").Utf8(qName) + .AddMember("EventType").Uint64(static_cast<ui64>(event.Type)) + .EndStruct(); + + } + } + param.EndList().Build(); + RunPrepared(builder.Build(), CleanupQueryCompleteTag, ctx); +} + +void TSearchEventsProcessor::OnCleanupQueryComplete(const TActorContext& ctx) { + if (!QueryComplete(ctx)) { + return ctx.Schedule(SHORT_RETRY_TIMEOUT, new TEvWakeup(StartCleanupTag)); + } + CommitTx(CleanupTxCommittedTag, ctx); +} + +void TSearchEventsProcessor::OnCleanupTxCommitted(const TActorContext &ctx) { + if (!TxCommitted(ctx)) { + return ctx.Schedule(DEFAULT_RETRY_TIMEOUT, new TEvWakeup(StartCleanupTag)); + } + QueuesEvents.clear(); + ProcessReindexIfRequired(ctx); +} + +void TSearchEventsProcessor::ProcessEventsQueue(const TActorContext& ctx) { + + for (const auto& [qName, events] : QueuesEvents) { + for (const auto& [ts, event]: events) { + auto existsIter = ExistingQueues.find(qName); + if (event.Type == EQueueEventType::Deleted) { + if (existsIter != ExistingQueues.end() && existsIter->second.CustomName == event.CustomName + && existsIter->second.Timestamp < ts) { + ExistingQueues.erase(existsIter); + } + } else if (event.Type == EQueueEventType::Created) { + if (existsIter == ExistingQueues.end()) { + auto iter = ExistingQueues.insert(std::make_pair( + qName, event + )).first; + iter->second.Type = EQueueEventType::Existed; + } + } + SaveQueueEvent(qName, event, ctx); + } + } + if (!QueuesEvents.empty()) { + Send(ctx.SelfID, new TEvWakeup(StartCleanupTag)); + } else { + ProcessReindexIfRequired(ctx); + } +} + +void TSearchEventsProcessor::ProcessReindexIfRequired(const TActorContext &ctx) { + if (LastReindex == TInstant::Zero() || LastReindex + ReindexInterval < ctx.Now()) { + if (ExistingQueues) { + TStringStream ss; + NJson::TJsonWriter writer(&ss, false); + for (const auto&[queue, event] : ExistingQueues) { + SaveQueueEvent(queue, event, ctx); + } + } + ProcessReindexResult(ctx); + } else { + WaitNextCycle(ctx); + } +} +void TSearchEventsProcessor::SendJson(const TString& json, const TActorContext &) { + EventsWriter->Write(json); +} + +void TSearchEventsProcessor::ProcessReindexResult(const TActorContext &ctx) { + ReindexComplete.Inc(); + LastReindex = ctx.Now(); + WaitNextCycle(ctx); +} + +void TSearchEventsProcessor::WaitNextCycle(const TActorContext &ctx) { + ctx.Schedule(RescanInterval, new TEvWakeup(StartQueuesListingTag)); +} + +ui64 TSearchEventsProcessor::GetReindexCount() const { + return ReindexComplete.Val(); +} + +NActors::TActorSystem* TSearchEventsProcessor::GetActorSystem() { + return TActivationContext::ActorSystem(); +} + +void TSearchEventsProcessor::StartSession(ui32 evTag, const TActorContext &ctx) { + Session = Nothing(); + CurrentTx = Nothing(); + PreparedQuery = Nothing(); + QueryResult = Nothing(); + SessionFuture = TableClient->GetSession(); + Apply(&SessionFuture, evTag, ctx); +} + +void TSearchEventsProcessor::PrepareDataQuery(const TString& query, ui32 evTag, const TActorContext& ctx) { + Y_VERIFY(Session.Defined()); + PreparedQuery = Nothing(); + QueryResult = Nothing(); + PrepQueryFuture = Session.Get()->PrepareDataQuery(query); + Apply(&PrepQueryFuture, evTag, ctx); +} + +void TSearchEventsProcessor::StartTx(ui32 evTag, const TActorContext &ctx) { + Y_VERIFY(Session.Defined()); + CurrentTx = Nothing(); + TxFuture = Session.Get()->BeginTransaction(); + Apply(&TxFuture, evTag, ctx); +} + +void TSearchEventsProcessor::CommitTx(ui32 evTag, const TActorContext &ctx) { + Y_VERIFY(Session.Defined()); + Y_VERIFY(CurrentTx.Defined()); + CommitTxFuture = CurrentTx.Get()->Commit(); + Apply(&CommitTxFuture, evTag, ctx); +} + +void TSearchEventsProcessor::RunPrepared(NYdb::TParams&& params, ui32 evTag, const TActorContext& ctx) { + Y_VERIFY(Session.Defined()); + Y_VERIFY(PreparedQuery.Defined()); + QueryResult = Nothing(); + NTable::TTxControl txControl = CurrentTx.Defined() ? NTable::TTxControl::Tx(*CurrentTx) + : NTable::TTxControl::BeginTx().CommitTx(); + QueryResultFuture = PreparedQuery.Get()->Execute(txControl, std::forward<NYdb::TParams>(params)); + Apply(&QueryResultFuture, evTag, ctx); +} + +bool TSearchEventsProcessor::SessionStarted(const TActorContext&) { + Y_VERIFY(SessionFuture.HasValue()); + auto& value = SessionFuture.GetValueSync(); + if (!value.IsSuccess()) + return false; + Session = value.GetSession(); + return true; +} + +bool TSearchEventsProcessor::TxStarted(const TActorContext&) { + Y_VERIFY(Session.Defined()); + Y_VERIFY(TxFuture.HasValue()); + auto& value = TxFuture.GetValueSync(); + if (!value.IsSuccess()) + return false; + CurrentTx = value.GetTransaction(); + return true; +} +bool TSearchEventsProcessor::TxCommitted(const TActorContext&) { + Y_VERIFY(Session.Defined()); + Y_VERIFY(CurrentTx.Defined()); + Y_VERIFY(CommitTxFuture.HasValue()); + auto& value = CommitTxFuture.GetValueSync(); + if (!value.IsSuccess()) + return false; + CurrentTx = Nothing(); + return true; +} + +bool TSearchEventsProcessor::QueryPrepared(const TActorContext&) { + Y_VERIFY(Session.Defined()); + Y_VERIFY(PrepQueryFuture.HasValue()); + auto& value = PrepQueryFuture.GetValueSync(); + if (!value.IsSuccess()) { + return false; + } + PreparedQuery = value.GetQuery(); + return true; +} +bool TSearchEventsProcessor::QueryComplete(const TActorContext&) { + Y_VERIFY(Session.Defined()); + Y_VERIFY(QueryResultFuture.HasValue()); + QueryResult = QueryResultFuture.GetValueSync(); + if (!QueryResult.Get()->IsSuccess()) { + return false; + } + return true; +} + + +void TSearchEventsProcessor::SaveQueueEvent( + + const TString& queueName, const TQueueEvent& event, const TActorContext& ctx +) { + auto tsIsoString = TInstant::MilliSeconds(event.Timestamp).ToIsoStringLocal(); + auto nowIsoString = TInstant::Now().ToIsoStringLocal(); + TStringStream ss; + NJson::TJsonWriter writer(&ss, false); + + writer.OpenMap(); + { + writer.Write("resource_type", "message-queue"); + writer.Write("timestamp", tsIsoString); + writer.Write("resource_id", queueName); + writer.Write("name", event.CustomName); + writer.Write("service", "message-queue"); + if (event.Type == EQueueEventType::Deleted) + writer.Write("deleted", tsIsoString); + if (event.Type == EQueueEventType::Existed) + writer.Write("reindex_timestamp", nowIsoString); + writer.Write("permission", "ymq.queues.list"); + writer.Write("cloud_id", event.CloudId); + writer.Write("folder_id", event.FolderId); + writer.OpenArray("resource_path"); + writer.OpenMap(); + writer.Write("resource_type", "resource-manager.folder"); + writer.Write("resource_id", event.FolderId); + writer.CloseMap(); + writer.CloseArray(); // resource_path + } + writer.CloseMap(); + writer.Flush(); + SendJson(ss.Str(), ctx); +} + +void IEventsWriterWrapper::Close() { + if (!Closed) { + Closed = true; + CloseImpl(); + } +} + }; // namespace NKikimr::NSQS diff --git a/ydb/core/ymq/actor/index_events_processor.h b/ydb/core/ymq/actor/index_events_processor.h index 3efde48f04..a8ae462064 100644 --- a/ydb/core/ymq/actor/index_events_processor.h +++ b/ydb/core/ymq/actor/index_events_processor.h @@ -1,148 +1,148 @@ -#pragma once - -#include "events.h" +#pragma once + +#include "events.h" #include <ydb/core/ymq/base/events_writer_iface.h> #include <ydb/public/sdk/cpp/client/ydb_table/table.h> -#include <library/cpp/actors/core/actor_bootstrapped.h> -#include <library/cpp/actors/core/hfunc.h> -#include <library/cpp/actors/core/actorsystem.h> -#include <library/cpp/json/json_writer.h> -#include <util/stream/file.h> - - - +#include <library/cpp/actors/core/actor_bootstrapped.h> +#include <library/cpp/actors/core/hfunc.h> +#include <library/cpp/actors/core/actorsystem.h> +#include <library/cpp/json/json_writer.h> +#include <util/stream/file.h> + + + namespace NKikimr::NSQS { - - - -class TSearchEventsProcessor : public NActors::TActorBootstrapped<TSearchEventsProcessor> { -using TActorContext = NActors::TActorContext; -friend class TIndexProcesorTests; -public: - - TSearchEventsProcessor(const TString& root, const TDuration& reindexInterval, const TDuration& rescanInterval, - const TSimpleSharedPtr<NYdb::NTable::TTableClient>& tableClient, - IEventsWriterWrapper::TPtr eventsWriter, - bool waitForWake = false); - - ~TSearchEventsProcessor(); - void Bootstrap(const TActorContext& ctx); - ui64 GetReindexCount() const; - -private: - TString SelectQueuesQuery; - TString SelectEventsQuery; - TString DeleteEventQuery; - TDuration RescanInterval; - TDuration ReindexInterval; - TInstant LastReindex = TInstant::Zero(); - TSimpleSharedPtr<NYdb::NTable::TTableClient> TableClient; - IEventsWriterWrapper::TPtr EventsWriter; - - NYdb::TAsyncStatus Status; - NYdb::NTable::TAsyncCreateSessionResult SessionFuture; - NYdb::NTable::TAsyncBeginTransactionResult TxFuture; - NYdb::NTable::TAsyncCommitTransactionResult CommitTxFuture; - NYdb::NTable::TAsyncPrepareQueryResult PrepQueryFuture; - NYdb::NTable::TAsyncDataQueryResult QueryResultFuture; - - TMaybe<NYdb::NTable::TSession> Session; - TMaybe<NYdb::NTable::TDataQuery> PreparedQuery; - TMaybe<NYdb::NTable::TTransaction> CurrentTx; - TMaybe<NYdb::NTable::TDataQueryResult> QueryResult; - - bool WaitForWake = false; - bool Stopping = false; - enum class EQueueEventType { - Deleted = 0, - Created = 1, - Existed = 2, - }; - struct TQueueEvent { - EQueueEventType Type; - ui64 Timestamp; - TString CustomName; - TString CloudId; - TString FolderId; - }; - struct TQueueTableKey { - TString Account; - TString QueueName; - }; - TQueueTableKey LastQueuesKey = {}; - - THashMap<TString, TMap<ui64, TQueueEvent>> QueuesEvents; - THashMap<TString, TQueueEvent> ExistingQueues; - - TAtomicCounter ReindexComplete = 0; - - STATEFN(StateFunc); - void HandleWakeup(NActors::TEvents::TEvWakeup::TPtr& ev, const TActorContext& ctx); - - void StartQueuesListing(const TActorContext& ctx); - void OnQueuesListSessionReady(const TActorContext& ctx); - void OnQueuesListPrepared(const TActorContext& ctx); - void RunQueuesListQuery(const TActorContext& ctx, bool initial = false); - void OnQueuesListQueryComplete(const TActorContext& ctx); - - void RunEventsListing(const TActorContext& ctx); - void OnEventsListingDone(const TActorContext& ctx); - void StartCleanupSession(const TActorContext& ctx); - void OnCleanupSessionReady(const TActorContext& ctx); - void OnCleanupTxReady(const TActorContext& ctx); - void OnCleanupPrepared(const TActorContext& ctx); - void RunEventsCleanup(const TActorContext& ctx); - void OnCleanupQueryComplete(const TActorContext&ctx); - void OnCleanupTxCommitted(const TActorContext&ctx); - - void ProcessEventsQueue(const TActorContext& ctx); - void ProcessReindexIfRequired(const TActorContext& ctx); - void ProcessReindexResult(const TActorContext& ctx); - void WaitNextCycle(const TActorContext& ctx); - - - void StartSession(ui32 evTag, const TActorContext& ctx); - void PrepareDataQuery(const TString& query, ui32 evTag, const TActorContext& ctx); - void StartTx(ui32 evTag, const TActorContext& ctx); - void CommitTx(ui32 evTag, const TActorContext& ctx); - void RunPrepared(NYdb::TParams&& params, ui32 evTag, const TActorContext& ctx); - bool SessionStarted(const TActorContext& ctx); - bool QueryPrepared(const TActorContext& ctx); - bool QueryComplete(const TActorContext& ctx); - bool TxStarted(const TActorContext& ctx); - bool TxCommitted(const TActorContext& ctx); - - - template<class TFutureType> - void Apply(TFutureType* future, ui32 evTag, const TActorContext& ctx) { - future->Subscribe( - [evTag, as = GetActorSystem(), selfId = ctx.SelfID](const auto&) - {as->Send(selfId, new TEvWakeup(evTag));} - ); - } - void InitQueries(const TString& root); - void SaveQueueEvent(const TString& queueName, const TQueueEvent& event, const TActorContext& ctx); - void SendJson(const TString& json, const TActorContext &ctx); - static NActors::TActorSystem* GetActorSystem(); - - - constexpr static ui32 StartQueuesListingTag = 1; - constexpr static ui32 QueuesListSessionStartedTag = 2; - constexpr static ui32 QueuesListPreparedTag = 3; - constexpr static ui32 QueuesListQueryCompleteTag = 4; - - constexpr static ui32 RunEventsListingTag = 10; - constexpr static ui32 EventsListingDoneTag = 11; - - constexpr static ui32 StartCleanupTag = 20; - constexpr static ui32 CleanupSessionReadyTag = 21; - constexpr static ui32 CleanupTxReadyTag = 22; - constexpr static ui32 CleanupPreparedTag = 23; - constexpr static ui32 CleanupQueryCompleteTag = 24; - constexpr static ui32 CleanupTxCommittedTag = 25; - - constexpr static ui32 StopAllTag = 99; - - -}; + + + +class TSearchEventsProcessor : public NActors::TActorBootstrapped<TSearchEventsProcessor> { +using TActorContext = NActors::TActorContext; +friend class TIndexProcesorTests; +public: + + TSearchEventsProcessor(const TString& root, const TDuration& reindexInterval, const TDuration& rescanInterval, + const TSimpleSharedPtr<NYdb::NTable::TTableClient>& tableClient, + IEventsWriterWrapper::TPtr eventsWriter, + bool waitForWake = false); + + ~TSearchEventsProcessor(); + void Bootstrap(const TActorContext& ctx); + ui64 GetReindexCount() const; + +private: + TString SelectQueuesQuery; + TString SelectEventsQuery; + TString DeleteEventQuery; + TDuration RescanInterval; + TDuration ReindexInterval; + TInstant LastReindex = TInstant::Zero(); + TSimpleSharedPtr<NYdb::NTable::TTableClient> TableClient; + IEventsWriterWrapper::TPtr EventsWriter; + + NYdb::TAsyncStatus Status; + NYdb::NTable::TAsyncCreateSessionResult SessionFuture; + NYdb::NTable::TAsyncBeginTransactionResult TxFuture; + NYdb::NTable::TAsyncCommitTransactionResult CommitTxFuture; + NYdb::NTable::TAsyncPrepareQueryResult PrepQueryFuture; + NYdb::NTable::TAsyncDataQueryResult QueryResultFuture; + + TMaybe<NYdb::NTable::TSession> Session; + TMaybe<NYdb::NTable::TDataQuery> PreparedQuery; + TMaybe<NYdb::NTable::TTransaction> CurrentTx; + TMaybe<NYdb::NTable::TDataQueryResult> QueryResult; + + bool WaitForWake = false; + bool Stopping = false; + enum class EQueueEventType { + Deleted = 0, + Created = 1, + Existed = 2, + }; + struct TQueueEvent { + EQueueEventType Type; + ui64 Timestamp; + TString CustomName; + TString CloudId; + TString FolderId; + }; + struct TQueueTableKey { + TString Account; + TString QueueName; + }; + TQueueTableKey LastQueuesKey = {}; + + THashMap<TString, TMap<ui64, TQueueEvent>> QueuesEvents; + THashMap<TString, TQueueEvent> ExistingQueues; + + TAtomicCounter ReindexComplete = 0; + + STATEFN(StateFunc); + void HandleWakeup(NActors::TEvents::TEvWakeup::TPtr& ev, const TActorContext& ctx); + + void StartQueuesListing(const TActorContext& ctx); + void OnQueuesListSessionReady(const TActorContext& ctx); + void OnQueuesListPrepared(const TActorContext& ctx); + void RunQueuesListQuery(const TActorContext& ctx, bool initial = false); + void OnQueuesListQueryComplete(const TActorContext& ctx); + + void RunEventsListing(const TActorContext& ctx); + void OnEventsListingDone(const TActorContext& ctx); + void StartCleanupSession(const TActorContext& ctx); + void OnCleanupSessionReady(const TActorContext& ctx); + void OnCleanupTxReady(const TActorContext& ctx); + void OnCleanupPrepared(const TActorContext& ctx); + void RunEventsCleanup(const TActorContext& ctx); + void OnCleanupQueryComplete(const TActorContext&ctx); + void OnCleanupTxCommitted(const TActorContext&ctx); + + void ProcessEventsQueue(const TActorContext& ctx); + void ProcessReindexIfRequired(const TActorContext& ctx); + void ProcessReindexResult(const TActorContext& ctx); + void WaitNextCycle(const TActorContext& ctx); + + + void StartSession(ui32 evTag, const TActorContext& ctx); + void PrepareDataQuery(const TString& query, ui32 evTag, const TActorContext& ctx); + void StartTx(ui32 evTag, const TActorContext& ctx); + void CommitTx(ui32 evTag, const TActorContext& ctx); + void RunPrepared(NYdb::TParams&& params, ui32 evTag, const TActorContext& ctx); + bool SessionStarted(const TActorContext& ctx); + bool QueryPrepared(const TActorContext& ctx); + bool QueryComplete(const TActorContext& ctx); + bool TxStarted(const TActorContext& ctx); + bool TxCommitted(const TActorContext& ctx); + + + template<class TFutureType> + void Apply(TFutureType* future, ui32 evTag, const TActorContext& ctx) { + future->Subscribe( + [evTag, as = GetActorSystem(), selfId = ctx.SelfID](const auto&) + {as->Send(selfId, new TEvWakeup(evTag));} + ); + } + void InitQueries(const TString& root); + void SaveQueueEvent(const TString& queueName, const TQueueEvent& event, const TActorContext& ctx); + void SendJson(const TString& json, const TActorContext &ctx); + static NActors::TActorSystem* GetActorSystem(); + + + constexpr static ui32 StartQueuesListingTag = 1; + constexpr static ui32 QueuesListSessionStartedTag = 2; + constexpr static ui32 QueuesListPreparedTag = 3; + constexpr static ui32 QueuesListQueryCompleteTag = 4; + + constexpr static ui32 RunEventsListingTag = 10; + constexpr static ui32 EventsListingDoneTag = 11; + + constexpr static ui32 StartCleanupTag = 20; + constexpr static ui32 CleanupSessionReadyTag = 21; + constexpr static ui32 CleanupTxReadyTag = 22; + constexpr static ui32 CleanupPreparedTag = 23; + constexpr static ui32 CleanupQueryCompleteTag = 24; + constexpr static ui32 CleanupTxCommittedTag = 25; + + constexpr static ui32 StopAllTag = 99; + + +}; } // namespace NKikimr::NSQS diff --git a/ydb/core/ymq/actor/proxy_actor.cpp b/ydb/core/ymq/actor/proxy_actor.cpp index 426977cb11..98d85210a5 100644 --- a/ydb/core/ymq/actor/proxy_actor.cpp +++ b/ydb/core/ymq/actor/proxy_actor.cpp @@ -145,19 +145,19 @@ void TProxyActor::HandleWakeup(TEvWakeup::TPtr&) { RLOG_SQS_ERROR("Proxy request timeout. User [" << UserName_ << "] Queue [" << QueueName_ << "] Action [" << actionName << "]"); if (QueueCounters_) { - INC_COUNTER_COUPLE(QueueCounters_, RequestTimeouts, request_timeouts_count_per_second); + INC_COUNTER_COUPLE(QueueCounters_, RequestTimeouts, request_timeouts_count_per_second); } else { - auto rootCounters = TIntrusivePtrCntrCouple{ - GetSqsServiceCounters(AppData()->Counters, "core"), - GetYmqPublicCounters(AppData()->Counters) - }; - auto [userCountersCouple, queueCountersCouple] = GetUserAndQueueCounters(rootCounters, TQueuePath(Cfg().GetRoot(), UserName_, QueueName_)); - if (queueCountersCouple.SqsCounters) { - queueCountersCouple.SqsCounters->GetCounter("RequestTimeouts", true)->Inc(); + auto rootCounters = TIntrusivePtrCntrCouple{ + GetSqsServiceCounters(AppData()->Counters, "core"), + GetYmqPublicCounters(AppData()->Counters) + }; + auto [userCountersCouple, queueCountersCouple] = GetUserAndQueueCounters(rootCounters, TQueuePath(Cfg().GetRoot(), UserName_, QueueName_)); + if (queueCountersCouple.SqsCounters) { + queueCountersCouple.SqsCounters->GetCounter("RequestTimeouts", true)->Inc(); + } + if (queueCountersCouple.YmqCounters) { + queueCountersCouple.YmqCounters->GetCounter("request_timeouts_count_per_second", true)->Inc(); } - if (queueCountersCouple.YmqCounters) { - queueCountersCouple.YmqCounters->GetCounter("request_timeouts_count_per_second", true)->Inc(); - } } SendErrorAndDie(NErrors::TIMEOUT); diff --git a/ydb/core/ymq/actor/proxy_service.cpp b/ydb/core/ymq/actor/proxy_service.cpp index f7aeb601a6..1eecd85103 100644 --- a/ydb/core/ymq/actor/proxy_service.cpp +++ b/ydb/core/ymq/actor/proxy_service.cpp @@ -60,8 +60,8 @@ void TSqsProxyService::Bootstrap() { LOG_SQS_INFO("Start SQS proxy service actor"); Become(&TThis::StateFunc); - SqsCounters_ = GetSqsServiceCounters(AppData()->Counters, "core"); - YmqPublicCounters_ = GetYmqPublicCounters(AppData()->Counters); + SqsCounters_ = GetSqsServiceCounters(AppData()->Counters, "core"); + YmqPublicCounters_ = GetYmqPublicCounters(AppData()->Counters); } void TSqsProxyService::HandleExecuted(TSqsEvents::TEvExecuted::TPtr& ev) { diff --git a/ydb/core/ymq/actor/proxy_service.h b/ydb/core/ymq/actor/proxy_service.h index c03db657fa..a3c991105f 100644 --- a/ydb/core/ymq/actor/proxy_service.h +++ b/ydb/core/ymq/actor/proxy_service.h @@ -85,8 +85,8 @@ private: void HandleGetLeaderNodeForQueueResponse(TSqsEvents::TEvGetLeaderNodeForQueueResponse::TPtr& ev); private: - TIntrusivePtr<NMonitoring::TDynamicCounters> SqsCounters_; - TIntrusivePtr<NMonitoring::TDynamicCounters> YmqPublicCounters_; + TIntrusivePtr<NMonitoring::TDynamicCounters> SqsCounters_; + TIntrusivePtr<NMonitoring::TDynamicCounters> YmqPublicCounters_; /// A map of node ids to TNodeIfno THashMap<ui32, TNodeInfoRef> NodesInfo_; diff --git a/ydb/core/ymq/actor/purge.cpp b/ydb/core/ymq/actor/purge.cpp index bf41126180..aff713d784 100644 --- a/ydb/core/ymq/actor/purge.cpp +++ b/ydb/core/ymq/actor/purge.cpp @@ -146,7 +146,7 @@ void TPurgeActor::MakeStage2Request(ui64 cleanupVersion, const TValue& messages, if (status == TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecComplete) { const TValue val(TValue::Create(ev.GetExecutionEngineEvaluatedResponse())); const ui64 messagesDeleted = val["messagesDeleted"]; - ADD_COUNTER_COUPLE(Counters_, MessagesPurged, purged_count_per_second, messagesDeleted); + ADD_COUNTER_COUPLE(Counters_, MessagesPurged, purged_count_per_second, messagesDeleted); RLOG_SQS_DEBUG("Purged " << messagesDeleted << " messages from queue [" << QueuePath_ << "]"); const bool versionIsSame = val["versionIsSame"]; if (versionIsSame) { diff --git a/ydb/core/ymq/actor/queue_leader.cpp b/ydb/core/ymq/actor/queue_leader.cpp index 335b4fa7f6..2c6e81d827 100644 --- a/ydb/core/ymq/actor/queue_leader.cpp +++ b/ydb/core/ymq/actor/queue_leader.cpp @@ -168,9 +168,9 @@ void TQueueLeader::PassAway() { } // Explicitly set absolute counters to zero for proper counting aggregated parent counters: - SET_COUNTER_COUPLE(Counters_, MessagesCount, stored_count, 0); - SET_COUNTER_COUPLE(Counters_, InflyMessagesCount, inflight_count, 0); - SET_COUNTER_COUPLE(Counters_, OldestMessageAgeSeconds, oldest_age_milliseconds, 0); + SET_COUNTER_COUPLE(Counters_, MessagesCount, stored_count, 0); + SET_COUNTER_COUPLE(Counters_, InflyMessagesCount, inflight_count, 0); + SET_COUNTER_COUPLE(Counters_, OldestMessageAgeSeconds, oldest_age_milliseconds, 0); TActorBootstrapped::PassAway(); } @@ -1214,7 +1214,7 @@ void TQueueLeader::AnswerGetConfiguration(TSqsEvents::TEvGetConfiguration::TPtr& auto resp = MakeHolder<TSqsEvents::TEvConfiguration>(); resp->RootUrl = RootUrl_; - resp->SqsCoreCounters = Counters_->RootCounters.SqsCounters; + resp->SqsCoreCounters = Counters_->RootCounters.SqsCounters; resp->QueueCounters = Counters_; resp->UserCounters = UserCounters_; resp->Shards = ShardsCount_; @@ -1236,7 +1236,7 @@ void TQueueLeader::AnswerGetConfiguration(TSqsEvents::TEvGetConfiguration::TPtr& void TQueueLeader::AnswerFailed(TSqsEvents::TEvGetConfiguration::TPtr& ev) { auto answer = MakeHolder<TSqsEvents::TEvConfiguration>(); answer->RootUrl = RootUrl_; - answer->SqsCoreCounters = Counters_->RootCounters.SqsCounters; + answer->SqsCoreCounters = Counters_->RootCounters.SqsCounters; answer->QueueCounters = Counters_; answer->UserCounters = UserCounters_; answer->Fail = true; @@ -1323,7 +1323,7 @@ void TQueueLeader::OnQueueConfiguration(const TSqsEvents::TEvExecuted::TRecord& answer->UserExists = true; answer->QueueExists = false; answer->RootUrl = RootUrl_; - answer->SqsCoreCounters = Counters_->RootCounters.SqsCounters; + answer->SqsCoreCounters = Counters_->RootCounters.SqsCounters; answer->QueueCounters = Counters_; answer->UserCounters = UserCounters_; answer->Fail = false; @@ -1602,9 +1602,9 @@ void TQueueLeader::ReportMessagesCountMetricsIfReady() { } if (Counters_) { - SET_COUNTER_COUPLE(Counters_, MessagesCount, stored_count, messagesCount); + SET_COUNTER_COUPLE(Counters_, MessagesCount, stored_count, messagesCount); if (!IsFifoQueue_) { // for fifo queues infly is always 0 - SET_COUNTER_COUPLE(Counters_, InflyMessagesCount, inflight_count, inflyMessagesCount); + SET_COUNTER_COUPLE(Counters_, InflyMessagesCount, inflight_count, inflyMessagesCount); } } } @@ -1619,10 +1619,10 @@ void TQueueLeader::ReportOldestTimestampMetricsIfReady() { } if (Counters_) { if (oldestMessagesTimestamp != Max<ui64>()) { - auto age = (TActivationContext::Now() - TInstant::MilliSeconds(oldestMessagesTimestamp)).Seconds(); - SET_COUNTER_COUPLE(Counters_, OldestMessageAgeSeconds, oldest_age_milliseconds, age); + auto age = (TActivationContext::Now() - TInstant::MilliSeconds(oldestMessagesTimestamp)).Seconds(); + SET_COUNTER_COUPLE(Counters_, OldestMessageAgeSeconds, oldest_age_milliseconds, age); } else { - SET_COUNTER_COUPLE(Counters_, OldestMessageAgeSeconds, oldest_age_milliseconds, 0); + SET_COUNTER_COUPLE(Counters_, OldestMessageAgeSeconds, oldest_age_milliseconds, 0); } } } diff --git a/ydb/core/ymq/actor/queue_schema.cpp b/ydb/core/ymq/actor/queue_schema.cpp index fe58018d74..91d3c5d3df 100644 --- a/ydb/core/ymq/actor/queue_schema.cpp +++ b/ydb/core/ymq/actor/queue_schema.cpp @@ -769,12 +769,12 @@ static const char* const CommitQueueParamsQuery = R"__( (let queuesRow '( '('Account userName) '('QueueName name))) - - (let eventsRow '( - '('Account userName) - '('QueueName name) - '('EventType (Uint64 '1)))) - + + (let eventsRow '( + '('Account userName) + '('QueueName name) + '('EventType (Uint64 '1)))) + (let queuesSelect '( 'QueueState 'QueueId @@ -836,11 +836,11 @@ static const char* const CommitQueueParamsQuery = R"__( '('MasterTabletId masterTabletId) '('TablesFormat tablesFormat))) - (let eventsUpdate '( - '('CustomQueueName customName) - '('EventTimestamp now) - '('FolderId folderId))) - + (let eventsUpdate '( + '('CustomQueueName customName) + '('EventTimestamp now) + '('FolderId folderId))) + (let attrRow '(%3$s)) (let attrUpdate '( @@ -884,7 +884,7 @@ static const char* const CommitQueueParamsQuery = R"__( (ListIf queueExists (SetResult 'meta queuesRead)) (ListIf willCommit (UpdateRow queuesTable queuesRow queuesUpdate)) - (ListIf willCommit (UpdateRow eventsTable eventsRow eventsUpdate)) + (ListIf willCommit (UpdateRow eventsTable eventsRow eventsUpdate)) (ListIf willCommit (UpdateRow attrsTable attrRow attrUpdate)) (If (Not willCommit) (AsList (Void)) @@ -1299,27 +1299,27 @@ static const char* EraseQueueRecordQuery = R"__( ( (let name (Parameter 'NAME (DataType 'Utf8String))) (let userName (Parameter 'USER_NAME (DataType 'Utf8String))) - (let now (Parameter 'NOW (DataType 'Uint64))) + (let now (Parameter 'NOW (DataType 'Uint64))) (let queuesTable '%2$s/.Queues) - (let eventsTable '%2$s/.Events) + (let eventsTable '%2$s/.Events) (let queuesRow '( '('Account userName) '('QueueName name))) - (let eventsRow '( - '('Account userName) - '('QueueName name) - '('EventType (Uint64 '0)))) - + (let eventsRow '( + '('Account userName) + '('QueueName name) + '('EventType (Uint64 '0)))) + (let queuesSelect '( 'QueueState 'Version 'FifoQueue - 'Shards - 'CustomQueueName - 'CreatedTimestamp - 'FolderId)) + 'Shards + 'CustomQueueName + 'CreatedTimestamp + 'FolderId)) (let queuesRead (SelectRow queuesTable queuesRow queuesSelect)) (let currentVersion @@ -1329,28 +1329,28 @@ static const char* EraseQueueRecordQuery = R"__( ) ) - (let queueCreateTs - (Coalesce - (Member queuesRead 'CreatedTimestamp) - (Uint64 '0) - ) - ) - (let folderId - (Coalesce - (Member queuesRead 'FolderId) - (Utf8String '"") - ) - ) - - (let customName - (Coalesce - (Member queuesRead 'CustomQueueName) - (Utf8String '"") - ) - ) - - (let eventTs (Max now (Add queueCreateTs (Uint64 '2)))) - + (let queueCreateTs + (Coalesce + (Member queuesRead 'CreatedTimestamp) + (Uint64 '0) + ) + ) + (let folderId + (Coalesce + (Member queuesRead 'FolderId) + (Utf8String '"") + ) + ) + + (let customName + (Coalesce + (Member queuesRead 'CustomQueueName) + (Utf8String '"") + ) + ) + + (let eventTs (Max now (Add queueCreateTs (Uint64 '2)))) + (let queueExists (Coalesce (Or @@ -1359,16 +1359,16 @@ static const char* EraseQueueRecordQuery = R"__( ) (Bool 'false))) - (let eventsUpdate '( - '('CustomQueueName customName) - '('EventTimestamp eventTs) - '('FolderId folderId))) - + (let eventsUpdate '( + '('CustomQueueName customName) + '('EventTimestamp eventTs) + '('FolderId folderId))) + (return (AsList (SetResult 'exists queueExists) (SetResult 'version currentVersion) (SetResult 'fields queuesRead) - (If queueExists (UpdateRow eventsTable eventsRow eventsUpdate) (Void)) + (If queueExists (UpdateRow eventsTable eventsRow eventsUpdate) (Void)) (If queueExists (EraseRow queuesTable queuesRow) (Void)))) ) )__"; @@ -1376,13 +1376,13 @@ static const char* EraseQueueRecordQuery = R"__( void TDeleteQueueSchemaActorV2::NextAction() { switch (EDeleting(SI_)) { case EDeleting::EraseQueueRecord: { - auto ev = MakeExecuteEvent(Sprintf(EraseQueueRecordQuery, QueuePath_.GetUserPath().c_str(), Cfg().GetRoot().c_str())); + auto ev = MakeExecuteEvent(Sprintf(EraseQueueRecordQuery, QueuePath_.GetUserPath().c_str(), Cfg().GetRoot().c_str())); auto* trans = ev->Record.MutableTransaction()->MutableMiniKQLTransaction(); - auto nowMs = TInstant::Now().MilliSeconds(); + auto nowMs = TInstant::Now().MilliSeconds(); TParameters(trans->MutableParams()->MutableProto()) .Utf8("NAME", QueuePath_.QueueName) - .Utf8("USER_NAME", QueuePath_.UserName) - .Uint64("NOW", nowMs); + .Utf8("USER_NAME", QueuePath_.UserName) + .Uint64("NOW", nowMs); Register(new TMiniKqlExecutionActor(SelfId(), RequestId_, std::move(ev), false, QueuePath_, GetTransactionCounters(UserCounters_))); break; diff --git a/ydb/core/ymq/actor/receive_message.cpp b/ydb/core/ymq/actor/receive_message.cpp index 31fe47600e..2b5bf0e517 100644 --- a/ydb/core/ymq/actor/receive_message.cpp +++ b/ydb/core/ymq/actor/receive_message.cpp @@ -209,14 +209,14 @@ private: // counters const TDuration messageResideDuration = TActivationContext::Now() - message.SentTimestamp; COLLECT_HISTOGRAM_COUNTER(QueueCounters_, MessageReceiveAttempts, message.ReceiveCount); - COLLECT_HISTOGRAM_COUNTER(QueueCounters_, receive_attempts_count_rate, message.ReceiveCount); + COLLECT_HISTOGRAM_COUNTER(QueueCounters_, receive_attempts_count_rate, message.ReceiveCount); COLLECT_HISTOGRAM_COUNTER(QueueCounters_, MessageReside_Duration, messageResideDuration.MilliSeconds()); - COLLECT_HISTOGRAM_COUNTER(QueueCounters_, reside_duration_milliseconds, messageResideDuration.MilliSeconds()); - INC_COUNTER_COUPLE(QueueCounters_, ReceiveMessage_Count, received_count_per_second); - ADD_COUNTER_COUPLE(QueueCounters_, ReceiveMessage_BytesRead, received_bytes_per_second, message.Data.size()); + COLLECT_HISTOGRAM_COUNTER(QueueCounters_, reside_duration_milliseconds, messageResideDuration.MilliSeconds()); + INC_COUNTER_COUPLE(QueueCounters_, ReceiveMessage_Count, received_count_per_second); + ADD_COUNTER_COUPLE(QueueCounters_, ReceiveMessage_BytesRead, received_bytes_per_second, message.Data.size()); } if (ev->Get()->Messages.empty()) { - INC_COUNTER_COUPLE(QueueCounters_, ReceiveMessage_EmptyCount, empty_receive_attempts_count_per_second); + INC_COUNTER_COUPLE(QueueCounters_, ReceiveMessage_EmptyCount, empty_receive_attempts_count_per_second); } } SendReplyAndDie(); diff --git a/ydb/core/ymq/actor/send_message.cpp b/ydb/core/ymq/actor/send_message.cpp index f2c84b5e59..e596b8f2e3 100644 --- a/ydb/core/ymq/actor/send_message.cpp +++ b/ydb/core/ymq/actor/send_message.cpp @@ -269,10 +269,10 @@ private: // counters if (status.Status == TSqsEvents::TEvSendMessageBatchResponse::ESendMessageStatus::AlreadySent) { - INC_COUNTER_COUPLE(QueueCounters_, SendMessage_DeduplicationCount, deduplicated_count_per_second); + INC_COUNTER_COUPLE(QueueCounters_, SendMessage_DeduplicationCount, deduplicated_count_per_second); } else { - INC_COUNTER_COUPLE(QueueCounters_, SendMessage_Count, sent_count_per_second); - ADD_COUNTER_COUPLE(QueueCounters_, SendMessage_BytesWritten, sent_bytes_per_second, CalculateMessageSize(*currentRequest)); + INC_COUNTER_COUPLE(QueueCounters_, SendMessage_Count, sent_count_per_second); + ADD_COUNTER_COUPLE(QueueCounters_, SendMessage_BytesWritten, sent_bytes_per_second, CalculateMessageSize(*currentRequest)); } } else { MakeError(currentResponse, NErrors::INTERNAL_FAILURE); diff --git a/ydb/core/ymq/actor/service.cpp b/ydb/core/ymq/actor/service.cpp index 0ebf998f33..34c9a4ae46 100644 --- a/ydb/core/ymq/actor/service.cpp +++ b/ydb/core/ymq/actor/service.cpp @@ -11,7 +11,7 @@ #include "queues_list_reader.h" #include "user_settings_names.h" #include "user_settings_reader.h" -#include "index_events_processor.h" +#include "index_events_processor.h" #include <ydb/public/lib/value/value.h> #include <ydb/public/sdk/cpp/client/ydb_types/credentials/credentials.h> @@ -28,11 +28,11 @@ #include <library/cpp/actors/core/events.h> #include <library/cpp/actors/core/hfunc.h> -#include <library/cpp/logger/global/global.h> +#include <library/cpp/logger/global/global.h> #include <util/generic/algorithm.h> #include <util/generic/hash_set.h> -#include <util/stream/file.h> +#include <util/stream/file.h> #include <util/string/builder.h> #include <util/string/cast.h> #include <util/system/hostname.h> @@ -63,17 +63,17 @@ constexpr ui64 LIST_QUEUES_WAKEUP_TAG = 2; constexpr size_t EARLY_REQUEST_USERS_LIST_MAX_BUDGET = 10; constexpr i64 EARLY_REQUEST_QUEUES_LIST_MAX_BUDGET = 5; // per user -bool IsInternalFolder(const TString& folder) { - return folder.StartsWith(".sys"); -} - +bool IsInternalFolder(const TString& folder) { + return folder.StartsWith(".sys"); +} + struct TSqsService::TQueueInfo : public TAtomicRefCount<TQueueInfo> { - TQueueInfo( + TQueueInfo( TString userName, TString queueName, TString rootUrl, ui64 leaderTabletId, TString customName, - TString folderId, ui64 version, ui64 shardsCount, const TIntrusivePtr<TUserCounters>& userCounters, - const TActorId& schemeCache, TIntrusivePtr<TSqsEvents::TQuoterResourcesForActions> quoterResourcesForUser, - bool insertCounters - ) + TString folderId, ui64 version, ui64 shardsCount, const TIntrusivePtr<TUserCounters>& userCounters, + const TActorId& schemeCache, TIntrusivePtr<TSqsEvents::TQuoterResourcesForActions> quoterResourcesForUser, + bool insertCounters + ) : UserName_(std::move(userName)) , QueueName_(std::move(queueName)) , CustomName_(std::move(customName)) @@ -128,16 +128,16 @@ struct TSqsService::TQueueInfo : public TAtomicRefCount<TQueueInfo> { LWPROBE(CreateLeader, UserName_, QueueName_, reason); LocalLeader_ = TActivationContext::Register(new TQueueLeader(UserName_, QueueName_, FolderId_, RootUrl_, Counters_, UserCounters_, SchemeCache_, QuoterResourcesForUser_)); LOG_SQS_INFO("Start local leader [" << UserName_ << "/" << QueueName_ << "] actor " << LocalLeader_); - - // ToDo: Should better make TFolderCounters struct and move it there. - // Will have to refactor TQueueCounters a bit, since it directly works with TUserCounters - if (FolderId_) { - auto folderCounters = GetFolderCounters(UserCounters_->UserCounters, FolderId_); - if (folderCounters.YmqCounters) { - auto counter = folderCounters.YmqCounters->GetCounter("queue.total_count", false); - (*counter)++; - } - } + + // ToDo: Should better make TFolderCounters struct and move it there. + // Will have to refactor TQueueCounters a bit, since it directly works with TUserCounters + if (FolderId_) { + auto folderCounters = GetFolderCounters(UserCounters_->UserCounters, FolderId_); + if (folderCounters.YmqCounters) { + auto counter = folderCounters.YmqCounters->GetCounter("queue.total_count", false); + (*counter)++; + } + } } } @@ -148,16 +148,16 @@ struct TSqsService::TQueueInfo : public TAtomicRefCount<TQueueInfo> { LOG_SQS_INFO("Stop local leader [" << UserName_ << "/" << QueueName_ << "] actor " << LocalLeader_); TActivationContext::Send(new IEventHandle(LocalLeader_, SelfId(), new TEvPoisonPill())); LocalLeader_ = TActorId(); - if (FolderId_) { - auto folderCounters = GetFolderCounters(UserCounters_->UserCounters, FolderId_); - if (folderCounters.YmqCounters) { - auto counter = folderCounters.YmqCounters->GetCounter("queue.total_count", false); - counter->Dec(); - if (counter->Val() == 0) { - RemoveFolderCounters(UserCounters_->UserCounters, FolderId_); - } - } - } + if (FolderId_) { + auto folderCounters = GetFolderCounters(UserCounters_->UserCounters, FolderId_); + if (folderCounters.YmqCounters) { + auto counter = folderCounters.YmqCounters->GetCounter("queue.total_count", false); + counter->Dec(); + if (counter->Val() == 0) { + RemoveFolderCounters(UserCounters_->UserCounters, FolderId_); + } + } + } } } @@ -275,10 +275,10 @@ static TString GetEndpoint(const NKikimrConfig::TSqsConfig& config) { } } -TSqsService::TSqsService(const TMaybe<ui32>& ydbPort) { - if (ydbPort.Defined()) { - YcSearchEventsConfig.GrpcPort = *ydbPort; - } +TSqsService::TSqsService(const TMaybe<ui32>& ydbPort) { + if (ydbPort.Defined()) { + YcSearchEventsConfig.GrpcPort = *ydbPort; + } DebugInfo->SqsServiceActorPtr = this; } TSqsService::~TSqsService() { @@ -298,11 +298,11 @@ void TSqsService::Bootstrap() { // Counters. SqsCoreCounters_ = GetSqsServiceCounters(AppData()->Counters, "core"); - YmqRootCounters_ = GetYmqPublicCounters(AppData()->Counters); + YmqRootCounters_ = GetYmqPublicCounters(AppData()->Counters); AllocPoolCounters_ = std::make_shared<TAlignedPagePoolCounters>(AppData()->Counters, "sqs"); - AggregatedUserCounters_ = MakeIntrusive<TUserCounters>( - Cfg(), SqsCoreCounters_, nullptr, AllocPoolCounters_, TOTAL_COUNTER_LABEL, nullptr, true - ); + AggregatedUserCounters_ = MakeIntrusive<TUserCounters>( + Cfg(), SqsCoreCounters_, nullptr, AllocPoolCounters_, TOTAL_COUNTER_LABEL, nullptr, true + ); AggregatedUserCounters_->ShowDetailedCounters(TInstant::Max()); InitSchemeCache(); @@ -314,29 +314,29 @@ void TSqsService::Bootstrap() { RequestSqsUsersList(); RequestSqsQueuesList(); - - if (Cfg().HasYcSearchEventsConfig() && YcSearchEventsConfig.GrpcPort) { - auto& ycSearchCfg = Cfg().GetYcSearchEventsConfig(); - YcSearchEventsConfig.Enabled = ycSearchCfg.GetEnableYcSearch(); - - YcSearchEventsConfig.ReindexInterval = TDuration::Seconds(ycSearchCfg.GetReindexIntervalSeconds()); - YcSearchEventsConfig.RescanInterval = TDuration::Seconds(ycSearchCfg.GetRescanIntervalSeconds()); - - auto driverConfig = NYdb::TDriverConfig().SetEndpoint( - TStringBuilder() << "localhost:" << YcSearchEventsConfig.GrpcPort); - if (ycSearchCfg.HasTenantMode() && ycSearchCfg.GetTenantMode()) { - driverConfig.SetDatabase(Cfg().GetRoot()); - YcSearchEventsConfig.TenantMode = true; - } + + if (Cfg().HasYcSearchEventsConfig() && YcSearchEventsConfig.GrpcPort) { + auto& ycSearchCfg = Cfg().GetYcSearchEventsConfig(); + YcSearchEventsConfig.Enabled = ycSearchCfg.GetEnableYcSearch(); + + YcSearchEventsConfig.ReindexInterval = TDuration::Seconds(ycSearchCfg.GetReindexIntervalSeconds()); + YcSearchEventsConfig.RescanInterval = TDuration::Seconds(ycSearchCfg.GetRescanIntervalSeconds()); + + auto driverConfig = NYdb::TDriverConfig().SetEndpoint( + TStringBuilder() << "localhost:" << YcSearchEventsConfig.GrpcPort); + if (ycSearchCfg.HasTenantMode() && ycSearchCfg.GetTenantMode()) { + driverConfig.SetDatabase(Cfg().GetRoot()); + YcSearchEventsConfig.TenantMode = true; + } auto factory = AppData()->SqsAuthFactory; Y_VERIFY(factory); - + driverConfig.SetCredentialsProviderFactory(factory->CreateCredentialsProviderFactory(Cfg())); - - YcSearchEventsConfig.Driver = MakeHolder<NYdb::TDriver>(driverConfig); - MakeAndRegisterYcEventsProcessor(); - } + + YcSearchEventsConfig.Driver = MakeHolder<NYdb::TDriver>(driverConfig); + MakeAndRegisterYcEventsProcessor(); + } } STATEFN(TSqsService::StateFunc) { @@ -947,18 +947,18 @@ TSqsService::TUsersMap::iterator TSqsService::MutableUserIter(const TString& use auto userIt = Users_.find(userName); if (userIt == Users_.end()) { LOG_SQS_INFO("Creating user info record for user [" << userName << "]"); - bool isInternal = IsInternalFolder(userName); - if (isInternal) { - LOG_SQS_INFO("[" << userName << "] is considered and internal service folder, will not create YMQ counters"); - } - TUserInfoPtr user = new TUserInfo( - userName, - new TUserCounters( - Cfg(), SqsCoreCounters_, - isInternal ? nullptr : YmqRootCounters_, - AllocPoolCounters_, userName, AggregatedUserCounters_, false - ) - ); + bool isInternal = IsInternalFolder(userName); + if (isInternal) { + LOG_SQS_INFO("[" << userName << "] is considered and internal service folder, will not create YMQ counters"); + } + TUserInfoPtr user = new TUserInfo( + userName, + new TUserCounters( + Cfg(), SqsCoreCounters_, + isInternal ? nullptr : YmqRootCounters_, + AllocPoolCounters_, userName, AggregatedUserCounters_, false + ) + ); user->InitQuoterResources(); userIt = Users_.emplace(userName, user).first; @@ -1024,7 +1024,7 @@ void TSqsService::RemoveQueue(const TString& userName, const TString& queue) { queuePtr->GetLeaderNodeRequests_.clear(); LeaderTabletIdToQueue_.erase(queuePtr->LeaderTabletId_); userIt->second->QueueByNameAndFolder_.erase(std::make_pair(queuePtr->CustomName_, queuePtr->FolderId_)); - + userIt->second->Queues_.erase(queueIt); queuePtr->Counters_->RemoveCounters(); } @@ -1290,13 +1290,13 @@ void TSqsService::AnswerErrorToRequests(const TUserInfoPtr& user, TMultimap& map } map.clear(); } - -void TSqsService::MakeAndRegisterYcEventsProcessor() { - if (!YcSearchEventsConfig.Enabled) - return; - - auto root = YcSearchEventsConfig.TenantMode ? TString() : Cfg().GetRoot(); - + +void TSqsService::MakeAndRegisterYcEventsProcessor() { + if (!YcSearchEventsConfig.Enabled) + return; + + auto root = YcSearchEventsConfig.TenantMode ? TString() : Cfg().GetRoot(); + auto factory = AppData()->SqsEventsWriterFactory; Y_VERIFY(factory); Register(new TSearchEventsProcessor( @@ -1304,14 +1304,14 @@ void TSqsService::MakeAndRegisterYcEventsProcessor() { MakeSimpleShared<NYdb::NTable::TTableClient>(*YcSearchEventsConfig.Driver), factory->CreateEventsWriter(Cfg(), GetSqsServiceCounters(AppData()->Counters, "yc_unified_agent")) )); -} -// -//IActor* CreateSqsService(const TYcSearchEventsConfig& ycSearchEventsConfig) { -// return new TSqsService(ycSearchEventsConfig); -//} - -IActor* CreateSqsService(TMaybe<ui32> ydbPort) { - return new TSqsService(ydbPort); +} +// +//IActor* CreateSqsService(const TYcSearchEventsConfig& ycSearchEventsConfig) { +// return new TSqsService(ycSearchEventsConfig); +//} + +IActor* CreateSqsService(TMaybe<ui32> ydbPort) { + return new TSqsService(ydbPort); } } // namespace NKikimr::NSQS diff --git a/ydb/core/ymq/actor/service.h b/ydb/core/ymq/actor/service.h index 5a0ad1294f..caf091c24b 100644 --- a/ydb/core/ymq/actor/service.h +++ b/ydb/core/ymq/actor/service.h @@ -3,7 +3,7 @@ #include "events.h" #include "log.h" #include "serviceid.h" -#include "index_events_processor.h" +#include "index_events_processor.h" #include <ydb/core/ymq/base/events_writer_iface.h> #include <ydb/core/base/tablet_pipe.h> @@ -14,7 +14,7 @@ #include <library/cpp/actors/core/actor_bootstrapped.h> #include <util/generic/hash.h> #include <util/generic/ptr.h> -#include <library/cpp/logger/log.h> +#include <library/cpp/logger/log.h> namespace NKikimr::NSQS { @@ -22,7 +22,7 @@ class TSqsService : public TActorBootstrapped<TSqsService> { public: - TSqsService(const TMaybe<ui32>& ydbPort); + TSqsService(const TMaybe<ui32>& ydbPort); ~TSqsService(); void Bootstrap(); @@ -129,12 +129,12 @@ private: void NotifyLocalDeadLetterQueuesLeaders(const std::vector<TSqsEvents::TEvQueuesList::TQueueRecord>& sortedQueues) const; - void MakeAndRegisterYcEventsProcessor(); - + void MakeAndRegisterYcEventsProcessor(); + private: TString RootUrl_; TIntrusivePtr<NMonitoring::TDynamicCounters> SqsCoreCounters_; - TIntrusivePtr<NMonitoring::TDynamicCounters> YmqRootCounters_; + TIntrusivePtr<NMonitoring::TDynamicCounters> YmqRootCounters_; std::shared_ptr<TAlignedPagePoolCounters> AllocPoolCounters_; TIntrusivePtr<TUserCounters> AggregatedUserCounters_; TUsersMap Users_; @@ -158,17 +158,17 @@ private: THashMultiMap<TString, TSqsEvents::TEvGetQueueId::TPtr> GetQueueIdRequests_; // user name -> request THashMultiMap<TString, TSqsEvents::TEvGetQueueFolderIdAndCustomName::TPtr> GetQueueFolderIdAndCustomNameRequests_; // user name -> request THashMultiMap<TString, TSqsEvents::TEvCountQueues::TPtr> CountQueuesRequests_; // user name -> request - - - struct TYcSearchEventsConfig { - THolder<NYdb::TDriver> Driver; - ui32 GrpcPort = 0; - bool Enabled = false; - bool TenantMode = false; - TDuration ReindexInterval = TDuration::Hours(4); - TDuration RescanInterval = TDuration::Minutes(1); - }; - TYcSearchEventsConfig YcSearchEventsConfig; + + + struct TYcSearchEventsConfig { + THolder<NYdb::TDriver> Driver; + ui32 GrpcPort = 0; + bool Enabled = false; + bool TenantMode = false; + TDuration ReindexInterval = TDuration::Hours(4); + TDuration RescanInterval = TDuration::Minutes(1); + }; + TYcSearchEventsConfig YcSearchEventsConfig; }; } // namespace NKikimr::NSQS diff --git a/ydb/core/ymq/actor/serviceid.h b/ydb/core/ymq/actor/serviceid.h index 741b094c9c..7219d918ba 100644 --- a/ydb/core/ymq/actor/serviceid.h +++ b/ydb/core/ymq/actor/serviceid.h @@ -31,7 +31,7 @@ inline TActorId MakeSqsMeteringServiceID() { return TActorId(0, TStringBuf("SQS_METER")); } -IActor* CreateSqsService(TMaybe<ui32> ydbPort = Nothing()); +IActor* CreateSqsService(TMaybe<ui32> ydbPort = Nothing()); IActor* CreateSqsProxyService(); IActor* CreateSqsAccessService(const TString& address, const TString& pathToRootCA); IActor* CreateSqsFolderService(const TString& address, const TString& pathToRootCA); diff --git a/ydb/core/ymq/actor/ya.make b/ydb/core/ymq/actor/ya.make index 5bbd51aa09..070653bb57 100644 --- a/ydb/core/ymq/actor/ya.make +++ b/ydb/core/ymq/actor/ya.make @@ -23,7 +23,7 @@ SRCS( garbage_collector.cpp get_queue_attributes.cpp get_queue_url.cpp - index_events_processor.cpp + index_events_processor.cpp infly.cpp log.cpp list_dead_letter_source_queues.cpp diff --git a/ydb/core/ymq/actor/yc_search_ut/index_events_processor_ut.cpp b/ydb/core/ymq/actor/yc_search_ut/index_events_processor_ut.cpp index 77863e5a7e..e484f117be 100644 --- a/ydb/core/ymq/actor/yc_search_ut/index_events_processor_ut.cpp +++ b/ydb/core/ymq/actor/yc_search_ut/index_events_processor_ut.cpp @@ -1,379 +1,379 @@ #include <ydb/core/testlib/test_client.h> #include <ydb/core/ymq/actor/index_events_processor.h> -#include <library/cpp/testing/unittest/registar.h> -#include <util/stream/file.h> -#include <library/cpp/json/json_reader.h> -#include "test_events_writer.h" - -namespace NKikimr::NSQS { - -using namespace Tests; -using namespace NYdb; - - -class TIndexProcesorTests : public TTestBase { - using EEvType = TSearchEventsProcessor::EQueueEventType; - -public: - TIndexProcesorTests() { - TPortManager portManager; - auto mbusPort = portManager.GetPort(2134); - auto grpcPort = portManager.GetPort(2135); - auto settings = TServerSettings(mbusPort); - settings.SetDomainName("Root"); - Server = MakeHolder<TServer>(settings); +#include <library/cpp/testing/unittest/registar.h> +#include <util/stream/file.h> +#include <library/cpp/json/json_reader.h> +#include "test_events_writer.h" + +namespace NKikimr::NSQS { + +using namespace Tests; +using namespace NYdb; + + +class TIndexProcesorTests : public TTestBase { + using EEvType = TSearchEventsProcessor::EQueueEventType; + +public: + TIndexProcesorTests() { + TPortManager portManager; + auto mbusPort = portManager.GetPort(2134); + auto grpcPort = portManager.GetPort(2135); + auto settings = TServerSettings(mbusPort); + settings.SetDomainName("Root"); + Server = MakeHolder<TServer>(settings); Server->EnableGRpc(NGrpc::TServerOptions().SetHost("localhost").SetPort(grpcPort)); - auto driverConfig = TDriverConfig().SetEndpoint(TStringBuilder() << "localhost:" << grpcPort); - - Driver = MakeHolder<TDriver>(driverConfig); + auto driverConfig = TDriverConfig().SetEndpoint(TStringBuilder() << "localhost:" << grpcPort); + + Driver = MakeHolder<TDriver>(driverConfig); TableClient = MakeSimpleShared<NYdb::NTable::TTableClient>(*Driver); - } - -private: - struct TTestRunner { - TString TestName; - TString SchemePath; - TSearchEventsProcessor* Processor; - TDispatchOptions DispatchOpts; - TInstant CurrTs; - TInstant PrevTs; - TMockSessionPtr EventsWriter; - TIndexProcesorTests* Parent; - TActorId ProcessorId; - TTestRunner(const TString& name, TIndexProcesorTests* parent) - : TestName(name) - , SchemePath(parent->Root + "/" + name) - , Processor() - , CurrTs(TInstant::Now()) - , PrevTs(TInstant::Seconds(CurrTs.Seconds() - 1)) - , Parent(parent) - { - EventsWriter = MakeIntrusive<TTestEventsWriter>(); - - auto* runtime = parent->Server->GetRuntime(); - Processor = new TSearchEventsProcessor( - SchemePath, - TDuration::Minutes(10), TDuration::MilliSeconds(100), - parent->TableClient, - EventsWriter, - true - ); - ProcessorId = runtime->Register(Processor); - runtime->EnableScheduleForActor(ProcessorId, true); - parent->Server->GetRuntime()->SetObserverFunc( - [&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>&) - { - if (Processor->GetReindexCount() >= 1) return TTestActorRuntimeBase::EEventAction::DROP; - else return TTestActorRuntimeBase::EEventAction::PROCESS; - } - ); - InitTables(); - } - ~TTestRunner() { - if (Processor != nullptr) { - auto handle = new IEventHandle(ProcessorId, TActorId(), new TEvWakeup(TSearchEventsProcessor::StopAllTag)); - Parent->Server->GetRuntime()->Send(handle); - } - } - void InitTables() { - TClient client(Parent->Server->GetSettings()); - client.MkDir(Parent->Root, SchemePath); - auto session = Parent->TableClient->CreateSession().GetValueSync().GetSession(); + } + +private: + struct TTestRunner { + TString TestName; + TString SchemePath; + TSearchEventsProcessor* Processor; + TDispatchOptions DispatchOpts; + TInstant CurrTs; + TInstant PrevTs; + TMockSessionPtr EventsWriter; + TIndexProcesorTests* Parent; + TActorId ProcessorId; + TTestRunner(const TString& name, TIndexProcesorTests* parent) + : TestName(name) + , SchemePath(parent->Root + "/" + name) + , Processor() + , CurrTs(TInstant::Now()) + , PrevTs(TInstant::Seconds(CurrTs.Seconds() - 1)) + , Parent(parent) + { + EventsWriter = MakeIntrusive<TTestEventsWriter>(); + + auto* runtime = parent->Server->GetRuntime(); + Processor = new TSearchEventsProcessor( + SchemePath, + TDuration::Minutes(10), TDuration::MilliSeconds(100), + parent->TableClient, + EventsWriter, + true + ); + ProcessorId = runtime->Register(Processor); + runtime->EnableScheduleForActor(ProcessorId, true); + parent->Server->GetRuntime()->SetObserverFunc( + [&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>&) + { + if (Processor->GetReindexCount() >= 1) return TTestActorRuntimeBase::EEventAction::DROP; + else return TTestActorRuntimeBase::EEventAction::PROCESS; + } + ); + InitTables(); + } + ~TTestRunner() { + if (Processor != nullptr) { + auto handle = new IEventHandle(ProcessorId, TActorId(), new TEvWakeup(TSearchEventsProcessor::StopAllTag)); + Parent->Server->GetRuntime()->Send(handle); + } + } + void InitTables() { + TClient client(Parent->Server->GetSettings()); + client.MkDir(Parent->Root, SchemePath); + auto session = Parent->TableClient->CreateSession().GetValueSync().GetSession(); auto desc = NYdb::NTable::TTableBuilder() - .AddNullableColumn("Account", EPrimitiveType::Utf8) - .AddNullableColumn("QueueName", EPrimitiveType::Utf8) - .AddNullableColumn("CreatedTimestamp", EPrimitiveType::Uint64) - .AddNullableColumn("FolderId", EPrimitiveType::Utf8) - .AddNullableColumn("CustomQueueName", EPrimitiveType::Utf8) - .SetPrimaryKeyColumns({"Account", "QueueName"}) - .Build(); + .AddNullableColumn("Account", EPrimitiveType::Utf8) + .AddNullableColumn("QueueName", EPrimitiveType::Utf8) + .AddNullableColumn("CreatedTimestamp", EPrimitiveType::Uint64) + .AddNullableColumn("FolderId", EPrimitiveType::Utf8) + .AddNullableColumn("CustomQueueName", EPrimitiveType::Utf8) + .SetPrimaryKeyColumns({"Account", "QueueName"}) + .Build(); auto eventsDesc = NYdb::NTable::TTableBuilder() - .AddNullableColumn("Account", EPrimitiveType::Utf8) - .AddNullableColumn("QueueName", EPrimitiveType::Utf8) - .AddNullableColumn("EventType", EPrimitiveType::Uint64) - .AddNullableColumn("EventTimestamp", EPrimitiveType::Uint64) - .AddNullableColumn("FolderId", EPrimitiveType::Utf8) - .AddNullableColumn("CustomQueueName", EPrimitiveType::Utf8) - .SetPrimaryKeyColumns({"Account", "QueueName", "EventType"}) - .Build(); - auto f1 = session.CreateTable(SchemePath + "/.Queues", std::move(desc)); - auto f2 = session.CreateTable(SchemePath + "/.Events", std::move(eventsDesc)); - - auto status = f1.GetValueSync(); - UNIT_ASSERT(status.IsSuccess()); - status = f2.GetValueSync(); - UNIT_ASSERT(status.IsSuccess()); - session.Close(); - } - - TAsyncStatus RunDataQuery(const TString& query) { + .AddNullableColumn("Account", EPrimitiveType::Utf8) + .AddNullableColumn("QueueName", EPrimitiveType::Utf8) + .AddNullableColumn("EventType", EPrimitiveType::Uint64) + .AddNullableColumn("EventTimestamp", EPrimitiveType::Uint64) + .AddNullableColumn("FolderId", EPrimitiveType::Utf8) + .AddNullableColumn("CustomQueueName", EPrimitiveType::Utf8) + .SetPrimaryKeyColumns({"Account", "QueueName", "EventType"}) + .Build(); + auto f1 = session.CreateTable(SchemePath + "/.Queues", std::move(desc)); + auto f2 = session.CreateTable(SchemePath + "/.Events", std::move(eventsDesc)); + + auto status = f1.GetValueSync(); + UNIT_ASSERT(status.IsSuccess()); + status = f2.GetValueSync(); + UNIT_ASSERT(status.IsSuccess()); + session.Close(); + } + + TAsyncStatus RunDataQuery(const TString& query) { auto status = Parent->TableClient->RetryOperation<NYdb::NTable::TDataQueryResult>( [query](NYdb::NTable::TSession session) { - return session.ExecuteDataQuery( + return session.ExecuteDataQuery( query, NYdb::NTable::TTxControl::BeginTx().CommitTx(), NYdb::NTable::TExecDataQuerySettings().ClientTimeout(TDuration::Seconds(10)) ).Apply([](const auto &future) mutable { - return future; - }); - }); - return status; - } - - void ExecDataQuery(const TString& query) { - Cerr << "===Execute query: " << query << Endl; - auto statusVal = RunDataQuery(query).GetValueSync(); - if (!statusVal.IsSuccess()) { - Cerr << "Query execution failed with error: " << statusVal.GetIssues().ToString() << Endl; - } - UNIT_ASSERT(statusVal.IsSuccess()); - } - void AddEvent( - const TString& account, const TString& queueName, const EEvType& type, TInstant ts = TInstant::Zero()) - { - if (ts == TInstant::Zero()) - ts = CurrTs; - TStringBuilder queryBuilder; - queryBuilder << "UPSERT INTO [" << SchemePath << "/.Events] (Account, QueueName, EventType, CustomQueueName, EventTimestamp, FolderId) " - << "VALUES (\"" << account << "\", \"" << queueName << "\", " << static_cast<ui64>(type) << ", \"myQueueCustomName\", " - << ts.MilliSeconds() << ", \"myFolder\");"; - ExecDataQuery(queryBuilder.c_str()); - } - - void AddQueue(const TString& account, const TString& queueName, TInstant ts = TInstant::Zero()) { - if (ts == TInstant::Zero()) - ts = CurrTs; - TStringBuilder queryBuilder; - queryBuilder << "UPSERT INTO [" << SchemePath << "/.Queues] (Account, QueueName, CustomQueueName, CreatedTimestamp, FolderId) " - << "VALUES (\"" << account << "\", \"" << queueName << "\", \"myQueueCustomName\", " - << ts.MilliSeconds() << ", \"myFolder\");"; - ExecDataQuery(queryBuilder.c_str()); - } - - void AddQueuesBatch(const TString& account, const TString& queueNameBase, ui64 count, ui64 startIndex = 0) { - Cerr << "===Started add queue batch\n"; + return future; + }); + }); + return status; + } + + void ExecDataQuery(const TString& query) { + Cerr << "===Execute query: " << query << Endl; + auto statusVal = RunDataQuery(query).GetValueSync(); + if (!statusVal.IsSuccess()) { + Cerr << "Query execution failed with error: " << statusVal.GetIssues().ToString() << Endl; + } + UNIT_ASSERT(statusVal.IsSuccess()); + } + void AddEvent( + const TString& account, const TString& queueName, const EEvType& type, TInstant ts = TInstant::Zero()) + { + if (ts == TInstant::Zero()) + ts = CurrTs; + TStringBuilder queryBuilder; + queryBuilder << "UPSERT INTO [" << SchemePath << "/.Events] (Account, QueueName, EventType, CustomQueueName, EventTimestamp, FolderId) " + << "VALUES (\"" << account << "\", \"" << queueName << "\", " << static_cast<ui64>(type) << ", \"myQueueCustomName\", " + << ts.MilliSeconds() << ", \"myFolder\");"; + ExecDataQuery(queryBuilder.c_str()); + } + + void AddQueue(const TString& account, const TString& queueName, TInstant ts = TInstant::Zero()) { + if (ts == TInstant::Zero()) + ts = CurrTs; + TStringBuilder queryBuilder; + queryBuilder << "UPSERT INTO [" << SchemePath << "/.Queues] (Account, QueueName, CustomQueueName, CreatedTimestamp, FolderId) " + << "VALUES (\"" << account << "\", \"" << queueName << "\", \"myQueueCustomName\", " + << ts.MilliSeconds() << ", \"myFolder\");"; + ExecDataQuery(queryBuilder.c_str()); + } + + void AddQueuesBatch(const TString& account, const TString& queueNameBase, ui64 count, ui64 startIndex = 0) { + Cerr << "===Started add queue batch\n"; TDeque<NYdb::NTable::TAsyncDataQueryResult> results; - ui64 maxInflight = 1; - auto getRessionResult = Parent->TableClient->GetSession().GetValueSync(); - UNIT_ASSERT(getRessionResult.IsSuccess()); - auto session = getRessionResult.GetSession(); - TStringBuilder queryBuilder; - queryBuilder << "DECLARE $QueueName as Utf8; " - << "UPSERT INTO [" << SchemePath << "/.Queues] " - << "(Account, QueueName, CustomQueueName, CreatedTimestamp, FolderId) " - << "VALUES (\"" << account << "\", $QueueName, \"myQueueCustomName\", " - << CurrTs.MilliSeconds() << ", \"myFolder\");"; - - auto preparedResult = session.PrepareDataQuery(queryBuilder.c_str()).GetValueSync(); - UNIT_ASSERT(preparedResult.IsSuccess()); - auto prepQuery = preparedResult.GetQuery(); - - auto txResult = session.BeginTransaction().GetValueSync(); - UNIT_ASSERT(txResult.IsSuccess()); - auto tx = txResult.GetTransaction(); + ui64 maxInflight = 1; + auto getRessionResult = Parent->TableClient->GetSession().GetValueSync(); + UNIT_ASSERT(getRessionResult.IsSuccess()); + auto session = getRessionResult.GetSession(); + TStringBuilder queryBuilder; + queryBuilder << "DECLARE $QueueName as Utf8; " + << "UPSERT INTO [" << SchemePath << "/.Queues] " + << "(Account, QueueName, CustomQueueName, CreatedTimestamp, FolderId) " + << "VALUES (\"" << account << "\", $QueueName, \"myQueueCustomName\", " + << CurrTs.MilliSeconds() << ", \"myFolder\");"; + + auto preparedResult = session.PrepareDataQuery(queryBuilder.c_str()).GetValueSync(); + UNIT_ASSERT(preparedResult.IsSuccess()); + auto prepQuery = preparedResult.GetQuery(); + + auto txResult = session.BeginTransaction().GetValueSync(); + UNIT_ASSERT(txResult.IsSuccess()); + auto tx = txResult.GetTransaction(); auto txControl = NYdb::NTable::TTxControl::Tx(tx); - - for (auto i = startIndex; i < startIndex + count; i++) { - while (results.size() < maxInflight) { - auto builder = prepQuery.GetParamsBuilder(); - builder.AddParam("$QueueName").Utf8(queueNameBase + ToString(i)).Build(); - - results.push_back( - prepQuery.Execute(txControl, builder.Build()) - ); - } - results.front().Wait(); - while(!results.empty() && results.front().HasValue()) { - UNIT_ASSERT_C(results.front().GetValueSync().IsSuccess(),results.front().GetValueSync().GetIssues().ToString().c_str()); - results.pop_front(); - } - - } - while (!results.empty()) { - auto statusVal = results.front().GetValueSync(); - UNIT_ASSERT(statusVal.IsSuccess()); - results.pop_front(); - } - auto commitStatus = tx.Commit().GetValueSync(); - UNIT_ASSERT(commitStatus.IsSuccess()); - } - - ui64 CountRecords(const TString& table) const { - TStringBuilder queryBuilder; - TMaybe<NYdb::NTable::TDataQueryResult> countEventsResult; - - queryBuilder << "select count(*) as Count from [" << SchemePath << "/." << table << "];"; + + for (auto i = startIndex; i < startIndex + count; i++) { + while (results.size() < maxInflight) { + auto builder = prepQuery.GetParamsBuilder(); + builder.AddParam("$QueueName").Utf8(queueNameBase + ToString(i)).Build(); + + results.push_back( + prepQuery.Execute(txControl, builder.Build()) + ); + } + results.front().Wait(); + while(!results.empty() && results.front().HasValue()) { + UNIT_ASSERT_C(results.front().GetValueSync().IsSuccess(),results.front().GetValueSync().GetIssues().ToString().c_str()); + results.pop_front(); + } + + } + while (!results.empty()) { + auto statusVal = results.front().GetValueSync(); + UNIT_ASSERT(statusVal.IsSuccess()); + results.pop_front(); + } + auto commitStatus = tx.Commit().GetValueSync(); + UNIT_ASSERT(commitStatus.IsSuccess()); + } + + ui64 CountRecords(const TString& table) const { + TStringBuilder queryBuilder; + TMaybe<NYdb::NTable::TDataQueryResult> countEventsResult; + + queryBuilder << "select count(*) as Count from [" << SchemePath << "/." << table << "];"; auto status = Parent->TableClient->RetryOperation<NYdb::NTable::TDataQueryResult>([query = TString(queryBuilder), &countEventsResult](NYdb::NTable::TSession session) { - return session.ExecuteDataQuery( + return session.ExecuteDataQuery( query, NYdb::NTable::TTxControl::BeginTx().CommitTx(), NYdb::NTable::TExecDataQuerySettings().ClientTimeout(TDuration::Seconds(5)) - ).Apply([&](const auto& future) mutable { - countEventsResult = future.GetValue(); - return future; - }); - }).GetValueSync(); - UNIT_ASSERT(status.IsSuccess()); - auto result = countEventsResult.Get()->GetResultSet(0); - TResultSetParser parser(result); - auto haveNext = parser.TryNextRow(); - UNIT_ASSERT(haveNext); - return parser.ColumnParser("Count").GetUint64(); - } - ui64 CountEvents() const { - return CountRecords("Events"); - } - ui64 CountQueues() const { - return CountRecords("Queues"); - } - void DispatchEvents() { - auto initialCount = Processor->GetReindexCount(); - DispatchOpts.CustomFinalCondition = [initialCount, processor = Processor]() { return processor->GetReindexCount() > initialCount; }; - if (!initialCount) { - auto handle = new IEventHandle(ProcessorId, TActorId(), new TEvWakeup(TSearchEventsProcessor::StartQueuesListingTag)); - Parent->Server->GetRuntime()->Send(handle); - } - Parent->Server->GetRuntime()->DispatchEvents(DispatchOpts, TDuration::Minutes(1)); - } - }; - void SetUp() override { - TClient client(Server->GetSettings()); + ).Apply([&](const auto& future) mutable { + countEventsResult = future.GetValue(); + return future; + }); + }).GetValueSync(); + UNIT_ASSERT(status.IsSuccess()); + auto result = countEventsResult.Get()->GetResultSet(0); + TResultSetParser parser(result); + auto haveNext = parser.TryNextRow(); + UNIT_ASSERT(haveNext); + return parser.ColumnParser("Count").GetUint64(); + } + ui64 CountEvents() const { + return CountRecords("Events"); + } + ui64 CountQueues() const { + return CountRecords("Queues"); + } + void DispatchEvents() { + auto initialCount = Processor->GetReindexCount(); + DispatchOpts.CustomFinalCondition = [initialCount, processor = Processor]() { return processor->GetReindexCount() > initialCount; }; + if (!initialCount) { + auto handle = new IEventHandle(ProcessorId, TActorId(), new TEvWakeup(TSearchEventsProcessor::StartQueuesListingTag)); + Parent->Server->GetRuntime()->Send(handle); + } + Parent->Server->GetRuntime()->DispatchEvents(DispatchOpts, TDuration::Minutes(1)); + } + }; + void SetUp() override { + TClient client(Server->GetSettings()); client.InitRootScheme(); - client.MkDir("/Root", "SQS"); - } + client.MkDir("/Root", "SQS"); + } -private: - THolder<TDriver> Driver; +private: + THolder<TDriver> Driver; TSimpleSharedPtr<NYdb::NTable::TTableClient> TableClient; - THolder<TServer> Server; - TString Root = "/Root/SQS"; - - UNIT_TEST_SUITE(TIndexProcesorTests) - UNIT_TEST(TestCreateIndexProcessor) - UNIT_TEST(TestSingleCreateQueueEvent) - UNIT_TEST(TestReindexSingleQueue) - UNIT_TEST(TestDeletedQueueNotReindexed) - UNIT_TEST(TestManyMessages) - UNIT_TEST(TestOver1000Queues) - UNIT_TEST_SUITE_END(); - - void CheckEventsLine( - const TString& line, EEvType type, const TString& queueName = TString()) - { - Cerr << "===CheckEventsLine: " << line << Endl; - NJson::TJsonValue json; - NJson::ReadJsonTree(line, &json, true); - auto &map = json.GetMap(); - if (!queueName.empty()) { - auto resId = map.find("resource_id")->second.GetString(); - UNIT_ASSERT_STRINGS_EQUAL(resId, queueName); - } - bool hasDeleted = map.find("deleted") != map.end(); - bool hasReindex = map.find("reindex_timestamp") != map.end(); - auto& res_path = map.find("resource_path")->second.GetArray(); - UNIT_ASSERT_VALUES_EQUAL(res_path.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(res_path[0].GetMap().size(), 2); - - - switch (type) { - case EEvType::Existed: - UNIT_ASSERT(!hasDeleted); - UNIT_ASSERT(hasReindex); - UNIT_ASSERT(!map.find("reindex_timestamp")->second.GetString().empty()); - break; - case EEvType::Created: - UNIT_ASSERT(!hasDeleted); - UNIT_ASSERT(!hasReindex); - break; - case EEvType::Deleted: - UNIT_ASSERT(hasDeleted); - UNIT_ASSERT(!map.find("deleted")->second.GetString().empty()); - UNIT_ASSERT(!hasReindex); - break; - } - } - - void TestCreateIndexProcessor() { - TTestRunner("CreateIndexProcessor", this); - } - - void TestSingleCreateQueueEvent() { - TTestRunner runner{"SingleCreateQueueEvent", this}; - runner.AddEvent( "cloud1", "queue1", EEvType::Created); - runner.DispatchEvents(); - auto messages = runner.EventsWriter->GetMessages(); - UNIT_ASSERT_VALUES_EQUAL(messages.size(), 2); // Events, reindex - CheckEventsLine(messages[0], EEvType::Created); - CheckEventsLine(messages[1], EEvType::Existed); - UNIT_ASSERT_VALUES_EQUAL(runner.CountEvents(), 0); - } - - void TestReindexSingleQueue() { - TTestRunner runner{"ReindexSingleQueue", this}; - runner.AddQueue("cloud1", "queue1"); - runner.DispatchEvents(); - auto messages = runner.EventsWriter->GetMessages(); - UNIT_ASSERT_VALUES_EQUAL(messages.size(), 1); - CheckEventsLine(messages[0], EEvType::Existed); - } - - void TestDeletedQueueNotReindexed() { - TTestRunner runner{"DeletedQueueNotReindexed", this}; - runner.AddQueue("cloud1", "queue2", runner.PrevTs); - runner.AddEvent("cloud1", "queue2", EEvType::Deleted); - Sleep(TDuration::Seconds(1)); - runner.DispatchEvents(); - auto messages = runner.EventsWriter->GetMessages(); - UNIT_ASSERT_VALUES_EQUAL(messages.size(), 1); - CheckEventsLine(messages[0], EEvType::Deleted); - } - - void TestManyMessages() { - TTestRunner runner{"TestManyMessages", this}; - runner.AddQueue("cloud1", "existing1", runner.PrevTs); - runner.AddQueue("cloud1", "existing2", runner.PrevTs); - runner.AddQueue("cloud1", "existing3", runner.PrevTs); - runner.AddQueue("cloud1", "deleting1", runner.PrevTs); - runner.AddQueue("cloud1", "deleting2", runner.PrevTs); - runner.AddEvent("cloud1", "deleting1", EEvType::Deleted); - runner.AddEvent("cloud1", "deleting2", EEvType::Deleted); - runner.AddEvent("cloud1", "creating1", EEvType::Created); - runner.AddEvent("cloud1", "creating2", EEvType::Created); - UNIT_ASSERT_VALUES_EQUAL(runner.CountEvents(), 4); - - runner.DispatchEvents(); - auto messages = runner.EventsWriter->GetMessages(); - UNIT_ASSERT_VALUES_EQUAL(messages.size(), 9); //4 events, 5 queues in reindex - for (auto i = 0u; i < 4; i++) { - if (messages[i].find("creating") != TString::npos) - CheckEventsLine(messages[i], EEvType::Created); - else { - UNIT_ASSERT(messages[i].find("deleting") != TString::npos); - UNIT_ASSERT(messages[i].find("existing") == TString::npos); - CheckEventsLine(messages[i], EEvType::Deleted); - } - } - for (auto i = 4u; i < messages.size(); i++) { - UNIT_ASSERT_C( - messages[i].find("existing") != TString::npos - || messages[i].find("creating") != TString::npos, - messages[i].c_str() - ); - CheckEventsLine(messages[i], EEvType::Existed); - } - UNIT_ASSERT_VALUES_EQUAL(runner.CountEvents(), 0); - } - - void TestOver1000Queues() { - TTestRunner runner{"TestOver1000Queues", this}; - ui64 queuesCount = 2010; - runner.AddQueuesBatch("cloud2", "queue-", queuesCount); - UNIT_ASSERT_VALUES_EQUAL(runner.CountQueues(), queuesCount); - Sleep(TDuration::Seconds(1)); - runner.EventsWriter->GetMessages(); //reset messages; - runner.DispatchEvents(); - auto messages = runner.EventsWriter->GetMessages(); - UNIT_ASSERT_VALUES_EQUAL(messages.size(), queuesCount); - - } -}; -UNIT_TEST_SUITE_REGISTRATION(TIndexProcesorTests); -} // NKikimr::NSQS + THolder<TServer> Server; + TString Root = "/Root/SQS"; + + UNIT_TEST_SUITE(TIndexProcesorTests) + UNIT_TEST(TestCreateIndexProcessor) + UNIT_TEST(TestSingleCreateQueueEvent) + UNIT_TEST(TestReindexSingleQueue) + UNIT_TEST(TestDeletedQueueNotReindexed) + UNIT_TEST(TestManyMessages) + UNIT_TEST(TestOver1000Queues) + UNIT_TEST_SUITE_END(); + + void CheckEventsLine( + const TString& line, EEvType type, const TString& queueName = TString()) + { + Cerr << "===CheckEventsLine: " << line << Endl; + NJson::TJsonValue json; + NJson::ReadJsonTree(line, &json, true); + auto &map = json.GetMap(); + if (!queueName.empty()) { + auto resId = map.find("resource_id")->second.GetString(); + UNIT_ASSERT_STRINGS_EQUAL(resId, queueName); + } + bool hasDeleted = map.find("deleted") != map.end(); + bool hasReindex = map.find("reindex_timestamp") != map.end(); + auto& res_path = map.find("resource_path")->second.GetArray(); + UNIT_ASSERT_VALUES_EQUAL(res_path.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(res_path[0].GetMap().size(), 2); + + + switch (type) { + case EEvType::Existed: + UNIT_ASSERT(!hasDeleted); + UNIT_ASSERT(hasReindex); + UNIT_ASSERT(!map.find("reindex_timestamp")->second.GetString().empty()); + break; + case EEvType::Created: + UNIT_ASSERT(!hasDeleted); + UNIT_ASSERT(!hasReindex); + break; + case EEvType::Deleted: + UNIT_ASSERT(hasDeleted); + UNIT_ASSERT(!map.find("deleted")->second.GetString().empty()); + UNIT_ASSERT(!hasReindex); + break; + } + } + + void TestCreateIndexProcessor() { + TTestRunner("CreateIndexProcessor", this); + } + + void TestSingleCreateQueueEvent() { + TTestRunner runner{"SingleCreateQueueEvent", this}; + runner.AddEvent( "cloud1", "queue1", EEvType::Created); + runner.DispatchEvents(); + auto messages = runner.EventsWriter->GetMessages(); + UNIT_ASSERT_VALUES_EQUAL(messages.size(), 2); // Events, reindex + CheckEventsLine(messages[0], EEvType::Created); + CheckEventsLine(messages[1], EEvType::Existed); + UNIT_ASSERT_VALUES_EQUAL(runner.CountEvents(), 0); + } + + void TestReindexSingleQueue() { + TTestRunner runner{"ReindexSingleQueue", this}; + runner.AddQueue("cloud1", "queue1"); + runner.DispatchEvents(); + auto messages = runner.EventsWriter->GetMessages(); + UNIT_ASSERT_VALUES_EQUAL(messages.size(), 1); + CheckEventsLine(messages[0], EEvType::Existed); + } + + void TestDeletedQueueNotReindexed() { + TTestRunner runner{"DeletedQueueNotReindexed", this}; + runner.AddQueue("cloud1", "queue2", runner.PrevTs); + runner.AddEvent("cloud1", "queue2", EEvType::Deleted); + Sleep(TDuration::Seconds(1)); + runner.DispatchEvents(); + auto messages = runner.EventsWriter->GetMessages(); + UNIT_ASSERT_VALUES_EQUAL(messages.size(), 1); + CheckEventsLine(messages[0], EEvType::Deleted); + } + + void TestManyMessages() { + TTestRunner runner{"TestManyMessages", this}; + runner.AddQueue("cloud1", "existing1", runner.PrevTs); + runner.AddQueue("cloud1", "existing2", runner.PrevTs); + runner.AddQueue("cloud1", "existing3", runner.PrevTs); + runner.AddQueue("cloud1", "deleting1", runner.PrevTs); + runner.AddQueue("cloud1", "deleting2", runner.PrevTs); + runner.AddEvent("cloud1", "deleting1", EEvType::Deleted); + runner.AddEvent("cloud1", "deleting2", EEvType::Deleted); + runner.AddEvent("cloud1", "creating1", EEvType::Created); + runner.AddEvent("cloud1", "creating2", EEvType::Created); + UNIT_ASSERT_VALUES_EQUAL(runner.CountEvents(), 4); + + runner.DispatchEvents(); + auto messages = runner.EventsWriter->GetMessages(); + UNIT_ASSERT_VALUES_EQUAL(messages.size(), 9); //4 events, 5 queues in reindex + for (auto i = 0u; i < 4; i++) { + if (messages[i].find("creating") != TString::npos) + CheckEventsLine(messages[i], EEvType::Created); + else { + UNIT_ASSERT(messages[i].find("deleting") != TString::npos); + UNIT_ASSERT(messages[i].find("existing") == TString::npos); + CheckEventsLine(messages[i], EEvType::Deleted); + } + } + for (auto i = 4u; i < messages.size(); i++) { + UNIT_ASSERT_C( + messages[i].find("existing") != TString::npos + || messages[i].find("creating") != TString::npos, + messages[i].c_str() + ); + CheckEventsLine(messages[i], EEvType::Existed); + } + UNIT_ASSERT_VALUES_EQUAL(runner.CountEvents(), 0); + } + + void TestOver1000Queues() { + TTestRunner runner{"TestOver1000Queues", this}; + ui64 queuesCount = 2010; + runner.AddQueuesBatch("cloud2", "queue-", queuesCount); + UNIT_ASSERT_VALUES_EQUAL(runner.CountQueues(), queuesCount); + Sleep(TDuration::Seconds(1)); + runner.EventsWriter->GetMessages(); //reset messages; + runner.DispatchEvents(); + auto messages = runner.EventsWriter->GetMessages(); + UNIT_ASSERT_VALUES_EQUAL(messages.size(), queuesCount); + + } +}; +UNIT_TEST_SUITE_REGISTRATION(TIndexProcesorTests); +} // NKikimr::NSQS diff --git a/ydb/core/ymq/actor/yc_search_ut/test_events_writer.cpp b/ydb/core/ymq/actor/yc_search_ut/test_events_writer.cpp index f643cce14b..21c0f887ba 100644 --- a/ydb/core/ymq/actor/yc_search_ut/test_events_writer.cpp +++ b/ydb/core/ymq/actor/yc_search_ut/test_events_writer.cpp @@ -1,20 +1,20 @@ -#include "test_events_writer.h" - -namespace NKikimr::NSQS { - -void TTestEventsWriter::Write(const TString& data) { - with_lock(Lock) { - Messages.emplace_back(data); - } -} - -TVector<TString> TTestEventsWriter::GetMessages() { - TVector<TString> result; - with_lock(Lock) { - result = std::move(Messages); - Messages.clear(); - } - return result; -} - -} //namespace NKikimr::NSQS +#include "test_events_writer.h" + +namespace NKikimr::NSQS { + +void TTestEventsWriter::Write(const TString& data) { + with_lock(Lock) { + Messages.emplace_back(data); + } +} + +TVector<TString> TTestEventsWriter::GetMessages() { + TVector<TString> result; + with_lock(Lock) { + result = std::move(Messages); + Messages.clear(); + } + return result; +} + +} //namespace NKikimr::NSQS diff --git a/ydb/core/ymq/actor/yc_search_ut/test_events_writer.h b/ydb/core/ymq/actor/yc_search_ut/test_events_writer.h index 3364007312..4ff876074f 100644 --- a/ydb/core/ymq/actor/yc_search_ut/test_events_writer.h +++ b/ydb/core/ymq/actor/yc_search_ut/test_events_writer.h @@ -1,22 +1,22 @@ -#pragma once - +#pragma once + #include <ydb/core/ymq/actor/index_events_processor.h> - -namespace NKikimr::NSQS { - -class TTestEventsWriter : public IEventsWriterWrapper { -public: - TTestEventsWriter() = default; - void Write(const TString& data) override; - TVector<TString> GetMessages(); - - void CloseImpl() override { - } - -private: - TAdaptiveLock Lock; - TVector<TString> Messages; -}; -using TMockSessionPtr = TIntrusivePtr<TTestEventsWriter>; - -} //namespace NKikimr::NSQS + +namespace NKikimr::NSQS { + +class TTestEventsWriter : public IEventsWriterWrapper { +public: + TTestEventsWriter() = default; + void Write(const TString& data) override; + TVector<TString> GetMessages(); + + void CloseImpl() override { + } + +private: + TAdaptiveLock Lock; + TVector<TString> Messages; +}; +using TMockSessionPtr = TIntrusivePtr<TTestEventsWriter>; + +} //namespace NKikimr::NSQS diff --git a/ydb/core/ymq/actor/yc_search_ut/ya.make b/ydb/core/ymq/actor/yc_search_ut/ya.make index 7465570487..19d200d2e2 100644 --- a/ydb/core/ymq/actor/yc_search_ut/ya.make +++ b/ydb/core/ymq/actor/yc_search_ut/ya.make @@ -1,25 +1,25 @@ -OWNER( - komels +OWNER( + komels g:kikimr - g:sqs -) - -UNITTEST() - -PEERDIR( + g:sqs +) + +UNITTEST() + +PEERDIR( ydb/core/testlib ydb/core/ymq/actor -) - -SRCS( - index_events_processor_ut.cpp - test_events_writer.cpp -) +) + +SRCS( + index_events_processor_ut.cpp + test_events_writer.cpp +) + +SIZE(MEDIUM) -SIZE(MEDIUM) +TIMEOUT(300) -TIMEOUT(300) +YQL_LAST_ABI_VERSION() -YQL_LAST_ABI_VERSION() - -END() +END() diff --git a/ydb/core/ymq/base/action.cpp b/ydb/core/ymq/base/action.cpp index b49c145146..b601d72d87 100644 --- a/ydb/core/ymq/base/action.cpp +++ b/ydb/core/ymq/base/action.cpp @@ -14,43 +14,43 @@ constexpr ui32 FOR_MESSAGE = 4; constexpr ui32 BATCH = 8; constexpr ui32 FAST = 16; constexpr ui32 PRIVATE = 32; -constexpr ui32 YMQ_FOR_QUEUE = 64; -constexpr ui32 YMQ_FOR_USER = 128; +constexpr ui32 YMQ_FOR_QUEUE = 64; +constexpr ui32 YMQ_FOR_USER = 128; struct TActionProps { TString StringName; - TString ConvMethodName; + TString ConvMethodName; EAction Action; ui32 Flags; EAction NonBatchAction; }; static const TActionProps ActionProps[] = { - {"Unknown", "unknown", EAction::Unknown, 0, EAction::Unknown}, - {"ChangeMessageVisibility", "change_message_visibility", EAction::ChangeMessageVisibility, FOR_QUEUE | FOR_MESSAGE | FAST, EAction::ChangeMessageVisibility}, - {"ChangeMessageVisibilityBatch", "change_message_visibility_batch", EAction::ChangeMessageVisibilityBatch, FOR_QUEUE | FOR_MESSAGE | BATCH | FAST, EAction::ChangeMessageVisibility}, - {"CreateQueue", "create_queue", EAction::CreateQueue, FOR_USER | YMQ_FOR_USER, EAction::CreateQueue}, - {"CreateUser", "create_user", EAction::CreateUser, FOR_USER, EAction::CreateUser}, - {"GetQueueAttributes", "get_queue_attributes", EAction::GetQueueAttributes, FOR_QUEUE | YMQ_FOR_QUEUE | FAST, EAction::GetQueueAttributes}, - {"GetQueueAttributesBatch", "get_queue_attributes_batch", EAction::GetQueueAttributesBatch, FOR_USER | BATCH | FAST | PRIVATE, EAction::GetQueueAttributes}, - {"GetQueueUrl", "get_queue_url", EAction::GetQueueUrl, FOR_QUEUE | YMQ_FOR_QUEUE | FAST, EAction::GetQueueUrl}, - {"DeleteMessage", "delete_message", EAction::DeleteMessage, FOR_QUEUE | YMQ_FOR_QUEUE | FOR_MESSAGE | FAST, EAction::DeleteMessage}, - {"DeleteMessageBatch", "delete_message_batch", EAction::DeleteMessageBatch, FOR_QUEUE | YMQ_FOR_QUEUE | FOR_MESSAGE | BATCH | FAST, EAction::DeleteMessage}, - {"DeleteQueue", "delete_queue", EAction::DeleteQueue, FOR_USER | YMQ_FOR_USER, EAction::DeleteQueue}, - {"DeleteQueueBatch", "delete_queue_batch", EAction::DeleteQueueBatch, FOR_USER | BATCH | PRIVATE, EAction::DeleteQueue}, - {"DeleteUser", "delete_user", EAction::DeleteUser, FOR_USER, EAction::DeleteUser}, - {"ListQueues", "list_queues", EAction::ListQueues, FOR_USER | YMQ_FOR_USER | FAST, EAction::ListQueues}, - {"ListUsers", "list_users", EAction::ListUsers, FOR_USER | FAST, EAction::ListUsers}, - {"PurgeQueue", "purge_queue", EAction::PurgeQueue, FOR_QUEUE | YMQ_FOR_QUEUE | FAST, EAction::PurgeQueue}, - {"PurgeQueueBatch", "purge_queue_batch", EAction::PurgeQueueBatch, FOR_USER | BATCH | FAST | PRIVATE, EAction::PurgeQueue}, - {"ReceiveMessage", "receive_message", EAction::ReceiveMessage, FOR_QUEUE | YMQ_FOR_QUEUE | FOR_MESSAGE | FAST, EAction::ReceiveMessage}, - {"SendMessage", "send_message", EAction::SendMessage, FOR_QUEUE | YMQ_FOR_QUEUE | FOR_MESSAGE | FAST, EAction::SendMessage}, - {"SendMessageBatch", "send_message_batch", EAction::SendMessageBatch, FOR_QUEUE | YMQ_FOR_QUEUE | FOR_MESSAGE | BATCH | FAST, EAction::SendMessage}, - {"SetQueueAttributes", "set_queue_attributes", EAction::SetQueueAttributes, FOR_QUEUE | YMQ_FOR_QUEUE | FAST, EAction::SetQueueAttributes}, - {"ModifyPermissions", "modify_permissions", EAction::ModifyPermissions, FOR_USER | FAST, EAction::ModifyPermissions}, - {"ListPermissions", "list_permissions", EAction::ListPermissions, FOR_USER | FAST, EAction::ListPermissions}, - {"ListDeadLetterSourceQueues", "list_dead_letter_source_queues", EAction::ListDeadLetterSourceQueues, FOR_QUEUE | FAST, EAction::ListDeadLetterSourceQueues}, - {"CountQueues", "count_queues", EAction::CountQueues, FOR_USER | FAST | PRIVATE, EAction::CountQueues}, + {"Unknown", "unknown", EAction::Unknown, 0, EAction::Unknown}, + {"ChangeMessageVisibility", "change_message_visibility", EAction::ChangeMessageVisibility, FOR_QUEUE | FOR_MESSAGE | FAST, EAction::ChangeMessageVisibility}, + {"ChangeMessageVisibilityBatch", "change_message_visibility_batch", EAction::ChangeMessageVisibilityBatch, FOR_QUEUE | FOR_MESSAGE | BATCH | FAST, EAction::ChangeMessageVisibility}, + {"CreateQueue", "create_queue", EAction::CreateQueue, FOR_USER | YMQ_FOR_USER, EAction::CreateQueue}, + {"CreateUser", "create_user", EAction::CreateUser, FOR_USER, EAction::CreateUser}, + {"GetQueueAttributes", "get_queue_attributes", EAction::GetQueueAttributes, FOR_QUEUE | YMQ_FOR_QUEUE | FAST, EAction::GetQueueAttributes}, + {"GetQueueAttributesBatch", "get_queue_attributes_batch", EAction::GetQueueAttributesBatch, FOR_USER | BATCH | FAST | PRIVATE, EAction::GetQueueAttributes}, + {"GetQueueUrl", "get_queue_url", EAction::GetQueueUrl, FOR_QUEUE | YMQ_FOR_QUEUE | FAST, EAction::GetQueueUrl}, + {"DeleteMessage", "delete_message", EAction::DeleteMessage, FOR_QUEUE | YMQ_FOR_QUEUE | FOR_MESSAGE | FAST, EAction::DeleteMessage}, + {"DeleteMessageBatch", "delete_message_batch", EAction::DeleteMessageBatch, FOR_QUEUE | YMQ_FOR_QUEUE | FOR_MESSAGE | BATCH | FAST, EAction::DeleteMessage}, + {"DeleteQueue", "delete_queue", EAction::DeleteQueue, FOR_USER | YMQ_FOR_USER, EAction::DeleteQueue}, + {"DeleteQueueBatch", "delete_queue_batch", EAction::DeleteQueueBatch, FOR_USER | BATCH | PRIVATE, EAction::DeleteQueue}, + {"DeleteUser", "delete_user", EAction::DeleteUser, FOR_USER, EAction::DeleteUser}, + {"ListQueues", "list_queues", EAction::ListQueues, FOR_USER | YMQ_FOR_USER | FAST, EAction::ListQueues}, + {"ListUsers", "list_users", EAction::ListUsers, FOR_USER | FAST, EAction::ListUsers}, + {"PurgeQueue", "purge_queue", EAction::PurgeQueue, FOR_QUEUE | YMQ_FOR_QUEUE | FAST, EAction::PurgeQueue}, + {"PurgeQueueBatch", "purge_queue_batch", EAction::PurgeQueueBatch, FOR_USER | BATCH | FAST | PRIVATE, EAction::PurgeQueue}, + {"ReceiveMessage", "receive_message", EAction::ReceiveMessage, FOR_QUEUE | YMQ_FOR_QUEUE | FOR_MESSAGE | FAST, EAction::ReceiveMessage}, + {"SendMessage", "send_message", EAction::SendMessage, FOR_QUEUE | YMQ_FOR_QUEUE | FOR_MESSAGE | FAST, EAction::SendMessage}, + {"SendMessageBatch", "send_message_batch", EAction::SendMessageBatch, FOR_QUEUE | YMQ_FOR_QUEUE | FOR_MESSAGE | BATCH | FAST, EAction::SendMessage}, + {"SetQueueAttributes", "set_queue_attributes", EAction::SetQueueAttributes, FOR_QUEUE | YMQ_FOR_QUEUE | FAST, EAction::SetQueueAttributes}, + {"ModifyPermissions", "modify_permissions", EAction::ModifyPermissions, FOR_USER | FAST, EAction::ModifyPermissions}, + {"ListPermissions", "list_permissions", EAction::ListPermissions, FOR_USER | FAST, EAction::ListPermissions}, + {"ListDeadLetterSourceQueues", "list_dead_letter_source_queues", EAction::ListDeadLetterSourceQueues, FOR_QUEUE | FAST, EAction::ListDeadLetterSourceQueues}, + {"CountQueues", "count_queues", EAction::CountQueues, FOR_USER | FAST | PRIVATE, EAction::CountQueues}, }; static_assert(Y_ARRAY_SIZE(ActionProps) == EAction::ActionsArraySize); @@ -88,10 +88,10 @@ const TString& ActionToString(EAction action) { return GetProps(action).StringName; } -const TString& ActionToCloudConvMethod(EAction action) { - return GetProps(action).ConvMethodName; -} - +const TString& ActionToCloudConvMethod(EAction action) { + return GetProps(action).ConvMethodName; +} + bool IsBatchAction(EAction action) { return GetProps(action).Flags & BATCH; } @@ -104,18 +104,18 @@ bool IsActionForQueue(EAction action) { return GetProps(action).Flags & FOR_QUEUE; } -bool IsActionForQueueYMQ(EAction action) { - return GetProps(action).Flags & YMQ_FOR_QUEUE; -} - +bool IsActionForQueueYMQ(EAction action) { + return GetProps(action).Flags & YMQ_FOR_QUEUE; +} + bool IsActionForUser(EAction action) { return GetProps(action).Flags & FOR_USER; } -bool IsActionForUserYMQ(EAction action) { - return GetProps(action).Flags & YMQ_FOR_USER; -} - +bool IsActionForUserYMQ(EAction action) { + return GetProps(action).Flags & YMQ_FOR_USER; +} + bool IsActionForMessage(EAction action) { return GetProps(action).Flags & FOR_MESSAGE; } diff --git a/ydb/core/ymq/base/action.h b/ydb/core/ymq/base/action.h index c7e87c7203..ede7922dd1 100644 --- a/ydb/core/ymq/base/action.h +++ b/ydb/core/ymq/base/action.h @@ -36,13 +36,13 @@ enum EAction { EAction ActionFromString(const TString& name); const TString& ActionToString(EAction action); -const TString& ActionToCloudConvMethod(EAction action); +const TString& ActionToCloudConvMethod(EAction action); bool IsBatchAction(EAction action); bool IsActionForQueue(EAction action); -bool IsActionForQueueYMQ(EAction action); +bool IsActionForQueueYMQ(EAction action); bool IsActionForUser(EAction action); -bool IsActionForUserYMQ(EAction action); +bool IsActionForUserYMQ(EAction action); bool IsProxyAction(EAction action); bool IsActionForMessage(EAction action); bool IsFastAction(EAction action); diff --git a/ydb/core/ymq/base/counters.cpp b/ydb/core/ymq/base/counters.cpp index b8a77d62fb..90a9f5000f 100644 --- a/ydb/core/ymq/base/counters.cpp +++ b/ydb/core/ymq/base/counters.cpp @@ -14,12 +14,12 @@ namespace NKikimr::NSQS { extern const TString DEFAULT_COUNTER_NAME = "sensor"; -extern const TString DEFAULT_YMQ_COUNTER_NAME = "name"; -extern const TString ACTION_CNTR_PREFIX = "api.http."; -extern const TString QUEUE_CNTR_PREFIX = "queue.messages."; -extern const TString METHOD_LABLE = "method"; +extern const TString DEFAULT_YMQ_COUNTER_NAME = "name"; +extern const TString ACTION_CNTR_PREFIX = "api.http."; +extern const TString QUEUE_CNTR_PREFIX = "queue.messages."; +extern const TString METHOD_LABLE = "method"; static const TString USER_LABEL = "user"; -static const TString CLOUD_LABEL = "cloud"; +static const TString CLOUD_LABEL = "cloud"; static const TString FOLDER_LABEL = "folder"; static const TString QUEUE_LABEL = "queue"; extern const TString TOTAL_COUNTER_LABEL = "total"; @@ -40,21 +40,21 @@ static const NMonitoring::TBucketBounds FastActionsDurationBucketsMs = { 5000, }; -static const NMonitoring::TBucketBounds YmqFastActionsDurationBucketsMs = { - 5, - 10, - 25, - 50, - 75, - 100, - 150, - 300, - 500, - 1000, - 5000, -}; - - +static const NMonitoring::TBucketBounds YmqFastActionsDurationBucketsMs = { + 5, + 10, + 25, + 50, + 75, + 100, + 150, + 300, + 500, + 1000, + 5000, +}; + + static const NMonitoring::TBucketBounds SlowActionsDurationBucketsMs = { 100, 150, @@ -64,18 +64,18 @@ static const NMonitoring::TBucketBounds SlowActionsDurationBucketsMs = { 5000, }; -static const NMonitoring::TBucketBounds YmqSlowActionsDurationBucketsMs = { - 100, - 150, - 300, - 500, - 1000, - 5000, - 10'000, - 20'000, - 50'000 -}; - +static const NMonitoring::TBucketBounds YmqSlowActionsDurationBucketsMs = { + 100, + 150, + 300, + 500, + 1000, + 5000, + 10'000, + 20'000, + 50'000 +}; + static const NMonitoring::TBucketBounds DurationBucketsMs = { 5, 10, @@ -118,22 +118,22 @@ static const NMonitoring::TBucketBounds ClientDurationBucketsMs = { 25'000'000, 50'000'000, }; -static const NMonitoring::TBucketBounds YmqClientDurationBucketsMs = { - 100, - 200, - 500, - 1'000, - 2'000, - 5'000, - 10'000, - 20'000, - 60'000, - 120'000, - 300'000, - 600'000, - 3'600'000, - 7'200'000, -}; +static const NMonitoring::TBucketBounds YmqClientDurationBucketsMs = { + 100, + 200, + 500, + 1'000, + 2'000, + 5'000, + 10'000, + 20'000, + 60'000, + 120'000, + 300'000, + 600'000, + 3'600'000, + 7'200'000, +}; static const NMonitoring::TBucketBounds GetQuotaDurationBucketsMs = { 1, @@ -301,46 +301,46 @@ static_assert(AbsDiffLessThanCounter(HttpUserCountersDescriptor.SizeOfCounters() TIntrusivePtr<NMonitoring::TDynamicCounters> GetSqsServiceCounters(const TIntrusivePtr<NMonitoring::TDynamicCounters>& countersRoot, const TString& subgroup) { return GetServiceCounters(countersRoot, "sqs")->GetSubgroup("subsystem", subgroup); } -TIntrusivePtr<NMonitoring::TDynamicCounters> GetYmqPublicCounters(const TIntrusivePtr<NMonitoring::TDynamicCounters>& countersRoot) { - // Remove subgroup and don't have subsystem (is this correct - ?) - return GetServiceCounters(countersRoot, "ymq_public"); -} - -static TIntrusivePtrCntrCouple GetUserCounters( - const TIntrusivePtrCntrCouple & sqsCoreCounters, const TString& userName -) { - return { - sqsCoreCounters.SqsCounters->GetSubgroup(USER_LABEL, userName), - sqsCoreCounters.YmqCounters ? sqsCoreCounters.YmqCounters->GetSubgroup(CLOUD_LABEL, userName) : nullptr - }; -} - -TIntrusivePtrCntrCouple GetFolderCounters(const TIntrusivePtrCntrCouple& userCounters, const TString& folderId) { - return { - userCounters.SqsCounters->GetSubgroup(FOLDER_LABEL, folderId), - userCounters.YmqCounters ? userCounters.YmqCounters->GetSubgroup(FOLDER_LABEL, folderId) : nullptr - }; -} - -void RemoveFolderCounters(const TIntrusivePtrCntrCouple& userCounters, const TString& folderId) { - if (userCounters.YmqCounters) { - userCounters.YmqCounters->RemoveSubgroup(FOLDER_LABEL, folderId); - } -} - -static TIntrusivePtrCntrCouple GetQueueCounters(const TIntrusivePtrCntrCouple & userOrFolderCounters, const TString& queueName) { - return { - userOrFolderCounters.SqsCounters->GetSubgroup(QUEUE_LABEL, queueName), - userOrFolderCounters.YmqCounters ? userOrFolderCounters.YmqCounters->GetSubgroup(QUEUE_LABEL, queueName) : nullptr - }; -} - -std::pair<TIntrusivePtrCntrCouple, TIntrusivePtrCntrCouple> GetUserAndQueueCounters( - const TIntrusivePtrCntrCouple& sqsCounters, const TQueuePath& queuePath -) { - TIntrusivePtrCntrCouple userCounters; - TIntrusivePtrCntrCouple queueCounters; - if (queuePath.UserName && sqsCounters.SqsCounters) { +TIntrusivePtr<NMonitoring::TDynamicCounters> GetYmqPublicCounters(const TIntrusivePtr<NMonitoring::TDynamicCounters>& countersRoot) { + // Remove subgroup and don't have subsystem (is this correct - ?) + return GetServiceCounters(countersRoot, "ymq_public"); +} + +static TIntrusivePtrCntrCouple GetUserCounters( + const TIntrusivePtrCntrCouple & sqsCoreCounters, const TString& userName +) { + return { + sqsCoreCounters.SqsCounters->GetSubgroup(USER_LABEL, userName), + sqsCoreCounters.YmqCounters ? sqsCoreCounters.YmqCounters->GetSubgroup(CLOUD_LABEL, userName) : nullptr + }; +} + +TIntrusivePtrCntrCouple GetFolderCounters(const TIntrusivePtrCntrCouple& userCounters, const TString& folderId) { + return { + userCounters.SqsCounters->GetSubgroup(FOLDER_LABEL, folderId), + userCounters.YmqCounters ? userCounters.YmqCounters->GetSubgroup(FOLDER_LABEL, folderId) : nullptr + }; +} + +void RemoveFolderCounters(const TIntrusivePtrCntrCouple& userCounters, const TString& folderId) { + if (userCounters.YmqCounters) { + userCounters.YmqCounters->RemoveSubgroup(FOLDER_LABEL, folderId); + } +} + +static TIntrusivePtrCntrCouple GetQueueCounters(const TIntrusivePtrCntrCouple & userOrFolderCounters, const TString& queueName) { + return { + userOrFolderCounters.SqsCounters->GetSubgroup(QUEUE_LABEL, queueName), + userOrFolderCounters.YmqCounters ? userOrFolderCounters.YmqCounters->GetSubgroup(QUEUE_LABEL, queueName) : nullptr + }; +} + +std::pair<TIntrusivePtrCntrCouple, TIntrusivePtrCntrCouple> GetUserAndQueueCounters( + const TIntrusivePtrCntrCouple& sqsCounters, const TQueuePath& queuePath +) { + TIntrusivePtrCntrCouple userCounters; + TIntrusivePtrCntrCouple queueCounters; + if (queuePath.UserName && sqsCounters.SqsCounters) { userCounters = GetUserCounters(sqsCounters, queuePath.UserName); if (queuePath.QueueName) { queueCounters = GetQueueCounters(userCounters, queuePath.QueueName); @@ -349,19 +349,19 @@ std::pair<TIntrusivePtrCntrCouple, TIntrusivePtrCntrCouple> GetUserAndQueueCount return { std::move(userCounters), std::move(queueCounters) }; } -TIntrusivePtr<NMonitoring::TDynamicCounters> GetAggregatedCountersFromSqsCoreCounters( - const TIntrusivePtrCntrCouple& rootCounters, const NKikimrConfig::TSqsConfig& cfg -) { - return GetAggregatedCountersFromUserCounters(GetUserCounters(rootCounters, TOTAL_COUNTER_LABEL), cfg); +TIntrusivePtr<NMonitoring::TDynamicCounters> GetAggregatedCountersFromSqsCoreCounters( + const TIntrusivePtrCntrCouple& rootCounters, const NKikimrConfig::TSqsConfig& cfg +) { + return GetAggregatedCountersFromUserCounters(GetUserCounters(rootCounters, TOTAL_COUNTER_LABEL), cfg); } -TIntrusivePtr<NMonitoring::TDynamicCounters> GetAggregatedCountersFromUserCounters( - const TIntrusivePtrCntrCouple& userCounters, const NKikimrConfig::TSqsConfig& cfg -) { +TIntrusivePtr<NMonitoring::TDynamicCounters> GetAggregatedCountersFromUserCounters( + const TIntrusivePtrCntrCouple& userCounters, const NKikimrConfig::TSqsConfig& cfg +) { if (cfg.GetYandexCloudMode()) { - return GetQueueCounters(GetFolderCounters(userCounters, TOTAL_COUNTER_LABEL), TOTAL_COUNTER_LABEL).SqsCounters; + return GetQueueCounters(GetFolderCounters(userCounters, TOTAL_COUNTER_LABEL), TOTAL_COUNTER_LABEL).SqsCounters; } else { - return GetQueueCounters(userCounters, TOTAL_COUNTER_LABEL).SqsCounters; + return GetQueueCounters(userCounters, TOTAL_COUNTER_LABEL).SqsCounters; } } @@ -371,33 +371,33 @@ ELaziness Lazy(const NKikimrConfig::TSqsConfig& cfg) { #define INIT_COUNTER_WITH_NAME(rootCounters, variable, name, expiring, valueType, lazy) \ variable.Init(rootCounters, expiring, valueType, name, lazy) -#define INIT_COUNTER_WITH_NAME_AND_LABEL(rootCounters, variable, labelName, name, expiring, valueType, lazy) \ - variable.Init(rootCounters, expiring, valueType, labelName, name, lazy) - -#define INIT_COUNTERS_COUPLE_WITH_NAMES(rootCounters, sqsCounter, ymqCounter, sqsName, ymqName, expiring, valueType, lazy, aggr) \ - sqsCounter.Init(rootCounters.SqsCounters, expiring, valueType, sqsName, lazy); \ - if (rootCounters.YmqCounters && !aggr) { \ - ymqCounter.Init(rootCounters.YmqCounters, expiring, valueType, DEFAULT_YMQ_COUNTER_NAME, ymqName, ELaziness::OnStart); \ - } - +#define INIT_COUNTER_WITH_NAME_AND_LABEL(rootCounters, variable, labelName, name, expiring, valueType, lazy) \ + variable.Init(rootCounters, expiring, valueType, labelName, name, lazy) + +#define INIT_COUNTERS_COUPLE_WITH_NAMES(rootCounters, sqsCounter, ymqCounter, sqsName, ymqName, expiring, valueType, lazy, aggr) \ + sqsCounter.Init(rootCounters.SqsCounters, expiring, valueType, sqsName, lazy); \ + if (rootCounters.YmqCounters && !aggr) { \ + ymqCounter.Init(rootCounters.YmqCounters, expiring, valueType, DEFAULT_YMQ_COUNTER_NAME, ymqName, ELaziness::OnStart); \ + } + #define INIT_COUNTER(rootCounters, variable, expiring, valueType, lazy) \ INIT_COUNTER_WITH_NAME(rootCounters, variable, Y_STRINGIZE(variable), expiring, valueType, lazy) - -#define INIT_COUNTERS_COUPLE(rootCounters, sqsCounter, ymqCounter, expiring, valueType, lazy, aggr) \ - INIT_COUNTERS_COUPLE_WITH_NAMES(rootCounters, sqsCounter, ymqCounter, Y_STRINGIZE(sqsCounter), TString(QUEUE_CNTR_PREFIX) + Y_STRINGIZE(ymqCounter), expiring, valueType, lazy, aggr) - -#define INIT_HISTOGRAMS_COUPLE_WITH_NAMES(rootCounters, sqsHistogram, ymqHistogram, sqsName, ymqName, expiring, sqsBuckets, ymqBuckets, lazy, aggr) \ - sqsHistogram.Init(rootCounters.SqsCounters, expiring, sqsBuckets, sqsName, lazy); \ - if (rootCounters.YmqCounters && !aggr) { \ - ymqHistogram.Init(rootCounters.YmqCounters, expiring, ymqBuckets, DEFAULT_YMQ_COUNTER_NAME, ymqName, lazy); \ - } - -#define INIT_HISTOGRAMS_COUPLE_WITH_BUCKETS(rootCounters, sqsHistogram, ymqHistogram, expiring, sqsBuckets, ymqBuckets, lazy, aggr) \ - INIT_HISTOGRAMS_COUPLE_WITH_NAMES(rootCounters, sqsHistogram, ymqHistogram, Y_STRINGIZE(sqsHistogram), TString(QUEUE_CNTR_PREFIX) + Y_STRINGIZE(ymqHistogram), expiring, sqsBuckets, ymqBuckets, lazy, aggr) - -#define INIT_HISTOGRAMS_COUPLE(rootCounters, sqsHistogram, ymqHistogram, expiring, buckets, lazy, aggr) \ - INIT_HISTOGRAMS_COUPLE_WITH_BUCKETS(rootCounters, sqsHistogram, ymqHistogram, expiring, buckets, Y_CAT(Ymq, buckets), lazy, aggr) - + +#define INIT_COUNTERS_COUPLE(rootCounters, sqsCounter, ymqCounter, expiring, valueType, lazy, aggr) \ + INIT_COUNTERS_COUPLE_WITH_NAMES(rootCounters, sqsCounter, ymqCounter, Y_STRINGIZE(sqsCounter), TString(QUEUE_CNTR_PREFIX) + Y_STRINGIZE(ymqCounter), expiring, valueType, lazy, aggr) + +#define INIT_HISTOGRAMS_COUPLE_WITH_NAMES(rootCounters, sqsHistogram, ymqHistogram, sqsName, ymqName, expiring, sqsBuckets, ymqBuckets, lazy, aggr) \ + sqsHistogram.Init(rootCounters.SqsCounters, expiring, sqsBuckets, sqsName, lazy); \ + if (rootCounters.YmqCounters && !aggr) { \ + ymqHistogram.Init(rootCounters.YmqCounters, expiring, ymqBuckets, DEFAULT_YMQ_COUNTER_NAME, ymqName, lazy); \ + } + +#define INIT_HISTOGRAMS_COUPLE_WITH_BUCKETS(rootCounters, sqsHistogram, ymqHistogram, expiring, sqsBuckets, ymqBuckets, lazy, aggr) \ + INIT_HISTOGRAMS_COUPLE_WITH_NAMES(rootCounters, sqsHistogram, ymqHistogram, Y_STRINGIZE(sqsHistogram), TString(QUEUE_CNTR_PREFIX) + Y_STRINGIZE(ymqHistogram), expiring, sqsBuckets, ymqBuckets, lazy, aggr) + +#define INIT_HISTOGRAMS_COUPLE(rootCounters, sqsHistogram, ymqHistogram, expiring, buckets, lazy, aggr) \ + INIT_HISTOGRAMS_COUPLE_WITH_BUCKETS(rootCounters, sqsHistogram, ymqHistogram, expiring, buckets, Y_CAT(Ymq, buckets), lazy, aggr) + #define INIT_HISTOGRAM_COUNTER_WITH_NAME(rootCounters, variable, name, expiring, buckets, lazy) \ variable.Init(rootCounters, expiring, buckets, name, lazy) #define INIT_HISTOGRAM_COUNTER(rootCounters, variable, expiring, buckets, lazy) \ @@ -417,28 +417,28 @@ void TActionCounters::Init(const NKikimrConfig::TSqsConfig& cfg, const TIntrusiv } } -void TYmqActionCounters::Init( - const NKikimrConfig::TSqsConfig&, const TIntrusivePtr<NMonitoring::TDynamicCounters>& rootCounters, - EAction action, const TString& labelName, const TString& namePrefix, ELifetime lifetime -) { - const auto& methodName = ActionToCloudConvMethod(action); - SubGroup = rootCounters->GetSubgroup(labelName, methodName); - INIT_COUNTER_WITH_NAME_AND_LABEL( - SubGroup, Success, DEFAULT_YMQ_COUNTER_NAME, TStringBuilder() << namePrefix << "requests_count_per_second", - lifetime,EValueType::Derivative, ELaziness::OnStart); - INIT_COUNTER_WITH_NAME_AND_LABEL( - SubGroup, Errors, DEFAULT_YMQ_COUNTER_NAME, TStringBuilder() << namePrefix << "errors_count_per_second", - lifetime,EValueType::Derivative, ELaziness::OnStart - ); - // ! - No inflight counter - - Duration.Init( - SubGroup, lifetime, IsFastAction(action) ? YmqFastActionsDurationBucketsMs : YmqSlowActionsDurationBucketsMs, - DEFAULT_YMQ_COUNTER_NAME, TStringBuilder() << namePrefix << "request_duration_milliseconds", - ELaziness::OnStart - ); -} - +void TYmqActionCounters::Init( + const NKikimrConfig::TSqsConfig&, const TIntrusivePtr<NMonitoring::TDynamicCounters>& rootCounters, + EAction action, const TString& labelName, const TString& namePrefix, ELifetime lifetime +) { + const auto& methodName = ActionToCloudConvMethod(action); + SubGroup = rootCounters->GetSubgroup(labelName, methodName); + INIT_COUNTER_WITH_NAME_AND_LABEL( + SubGroup, Success, DEFAULT_YMQ_COUNTER_NAME, TStringBuilder() << namePrefix << "requests_count_per_second", + lifetime,EValueType::Derivative, ELaziness::OnStart); + INIT_COUNTER_WITH_NAME_AND_LABEL( + SubGroup, Errors, DEFAULT_YMQ_COUNTER_NAME, TStringBuilder() << namePrefix << "errors_count_per_second", + lifetime,EValueType::Derivative, ELaziness::OnStart + ); + // ! - No inflight counter + + Duration.Init( + SubGroup, lifetime, IsFastAction(action) ? YmqFastActionsDurationBucketsMs : YmqSlowActionsDurationBucketsMs, + DEFAULT_YMQ_COUNTER_NAME, TStringBuilder() << namePrefix << "request_duration_milliseconds", + ELaziness::OnStart + ); +} + void TActionCounters::SetAggregatedParent(TActionCounters* parent) { ActionCountersDescriptor.SetAggregatedParent(this, parent); } @@ -514,15 +514,15 @@ void TAPIStatusesCounters::SetAggregatedParent(TAPIStatusesCounters* parent) { } TQueueCounters::TQueueCounters(const NKikimrConfig::TSqsConfig& cfg, - const TIntrusivePtrCntrCouple& rootCounters, + const TIntrusivePtrCntrCouple& rootCounters, const TUserCounters* userCounters, const TString& queueName, const TString& folderId, bool insertCounters, bool aggregated) - : RootCounters(rootCounters) + : RootCounters(rootCounters) , UserCounters(userCounters->UserCounters) - , FolderCounters(folderId ? GetFolderCounters(UserCounters, folderId) : TIntrusivePtrCntrCouple{}) + , FolderCounters(folderId ? GetFolderCounters(UserCounters, folderId) : TIntrusivePtrCntrCouple{}) , Cfg(&cfg) , QueueName(queueName) , AggregatedCounters(aggregated) @@ -530,120 +530,120 @@ TQueueCounters::TQueueCounters(const NKikimrConfig::TSqsConfig& cfg, , AllocPoolCounters(userCounters->DetailedCounters.TransactionCounters->AllocPoolCounters) { if (insertCounters) { - QueueCounters = GetQueueCounters(FolderCounters.Defined() ? FolderCounters : UserCounters, queueName); + QueueCounters = GetQueueCounters(FolderCounters.Defined() ? FolderCounters : UserCounters, queueName); } else { - QueueCounters = {new NMonitoring::TDynamicCounters(), new NMonitoring::TDynamicCounters()}; + QueueCounters = {new NMonitoring::TDynamicCounters(), new NMonitoring::TDynamicCounters()}; } InitCounters(); } void TQueueCounters::InitCounters(bool forLeaderNode) { if (!RequestTimeouts) { - INIT_COUNTERS_COUPLE( - QueueCounters, - RequestTimeouts, request_timeouts_count_per_second, - ELifetime::Persistent, EValueType::Derivative, - Lazy(*Cfg), AggregatedCounters - ); + INIT_COUNTERS_COUPLE( + QueueCounters, + RequestTimeouts, request_timeouts_count_per_second, + ELifetime::Persistent, EValueType::Derivative, + Lazy(*Cfg), AggregatedCounters + ); } - + if (forLeaderNode) { - INIT_COUNTER(QueueCounters.SqsCounters, RequestsThrottled, ELifetime::Expiring, EValueType::Derivative, ELaziness::OnStart); + INIT_COUNTER(QueueCounters.SqsCounters, RequestsThrottled, ELifetime::Expiring, EValueType::Derivative, ELaziness::OnStart); - INIT_COUNTER(QueueCounters.SqsCounters, QueueMasterStartProblems, ELifetime::Persistent, EValueType::Derivative, ELaziness::OnStart); + INIT_COUNTER(QueueCounters.SqsCounters, QueueMasterStartProblems, ELifetime::Persistent, EValueType::Derivative, ELaziness::OnStart); INIT_COUNTER(QueueCounters.SqsCounters, QueueLeaderStartProblems, ELifetime::Persistent, EValueType::Derivative, ELaziness::OnStart); - INIT_COUNTERS_COUPLE( - QueueCounters, - MessagesPurged, purged_count_per_second, - ELifetime::Expiring, EValueType::Derivative, - ELaziness::OnStart, AggregatedCounters - ); - - INIT_HISTOGRAMS_COUPLE_WITH_BUCKETS( - QueueCounters, MessageReceiveAttempts, receive_attempts_count_rate, - ELifetime::Expiring, MessageReceiveAttemptsBuckets, MessageReceiveAttemptsBuckets, - Lazy(*Cfg), AggregatedCounters - ); - INIT_HISTOGRAMS_COUPLE( - QueueCounters, - ClientMessageProcessing_Duration, client_processing_duration_milliseconds, - ELifetime::Expiring, ClientDurationBucketsMs, - Lazy(*Cfg), AggregatedCounters - ); - INIT_HISTOGRAMS_COUPLE( - QueueCounters, - MessageReside_Duration, reside_duration_milliseconds, - ELifetime::Expiring, ClientDurationBucketsMs, - Lazy(*Cfg), AggregatedCounters - ); - INIT_COUNTERS_COUPLE( - QueueCounters, - DeleteMessage_Count, deleted_count_per_second, - ELifetime::Expiring, EValueType::Derivative, - Lazy(*Cfg), AggregatedCounters - ); - - INIT_COUNTERS_COUPLE( - QueueCounters, - ReceiveMessage_EmptyCount, empty_receive_attempts_count_per_second, - ELifetime::Expiring, EValueType::Derivative, - Lazy(*Cfg), AggregatedCounters - ); - INIT_COUNTERS_COUPLE( - QueueCounters, - ReceiveMessage_Count, received_count_per_second, - ELifetime::Expiring, EValueType::Derivative, - Lazy(*Cfg), AggregatedCounters - ); - INIT_COUNTERS_COUPLE( - QueueCounters, - ReceiveMessage_BytesRead, received_bytes_per_second, - ELifetime::Expiring, EValueType::Derivative, - Lazy(*Cfg), AggregatedCounters - ); - - INIT_COUNTER(QueueCounters.SqsCounters, MessagesMovedToDLQ, ELifetime::Expiring, EValueType::Derivative, ELaziness::OnStart); - - INIT_COUNTERS_COUPLE( - QueueCounters, - SendMessage_DeduplicationCount, deduplicated_count_per_second, - ELifetime::Expiring, EValueType::Derivative, - Lazy(*Cfg), AggregatedCounters - ); - INIT_COUNTERS_COUPLE( - QueueCounters, - SendMessage_Count, sent_count_per_second, - ELifetime::Expiring, EValueType::Derivative, - Lazy(*Cfg), AggregatedCounters - ); - INIT_COUNTERS_COUPLE( - QueueCounters, - SendMessage_BytesWritten, sent_bytes_per_second, - ELifetime::Expiring, EValueType::Derivative, - Lazy(*Cfg), AggregatedCounters - ); - - INIT_COUNTERS_COUPLE( - QueueCounters, - MessagesCount, stored_count, - ELifetime::Expiring, EValueType::Absolute, - Lazy(*Cfg), AggregatedCounters - ); - INIT_COUNTERS_COUPLE( - QueueCounters, - InflyMessagesCount, inflight_count, - ELifetime::Expiring, EValueType::Absolute, - Lazy(*Cfg), AggregatedCounters - ); + INIT_COUNTERS_COUPLE( + QueueCounters, + MessagesPurged, purged_count_per_second, + ELifetime::Expiring, EValueType::Derivative, + ELaziness::OnStart, AggregatedCounters + ); + + INIT_HISTOGRAMS_COUPLE_WITH_BUCKETS( + QueueCounters, MessageReceiveAttempts, receive_attempts_count_rate, + ELifetime::Expiring, MessageReceiveAttemptsBuckets, MessageReceiveAttemptsBuckets, + Lazy(*Cfg), AggregatedCounters + ); + INIT_HISTOGRAMS_COUPLE( + QueueCounters, + ClientMessageProcessing_Duration, client_processing_duration_milliseconds, + ELifetime::Expiring, ClientDurationBucketsMs, + Lazy(*Cfg), AggregatedCounters + ); + INIT_HISTOGRAMS_COUPLE( + QueueCounters, + MessageReside_Duration, reside_duration_milliseconds, + ELifetime::Expiring, ClientDurationBucketsMs, + Lazy(*Cfg), AggregatedCounters + ); + INIT_COUNTERS_COUPLE( + QueueCounters, + DeleteMessage_Count, deleted_count_per_second, + ELifetime::Expiring, EValueType::Derivative, + Lazy(*Cfg), AggregatedCounters + ); + + INIT_COUNTERS_COUPLE( + QueueCounters, + ReceiveMessage_EmptyCount, empty_receive_attempts_count_per_second, + ELifetime::Expiring, EValueType::Derivative, + Lazy(*Cfg), AggregatedCounters + ); + INIT_COUNTERS_COUPLE( + QueueCounters, + ReceiveMessage_Count, received_count_per_second, + ELifetime::Expiring, EValueType::Derivative, + Lazy(*Cfg), AggregatedCounters + ); + INIT_COUNTERS_COUPLE( + QueueCounters, + ReceiveMessage_BytesRead, received_bytes_per_second, + ELifetime::Expiring, EValueType::Derivative, + Lazy(*Cfg), AggregatedCounters + ); + + INIT_COUNTER(QueueCounters.SqsCounters, MessagesMovedToDLQ, ELifetime::Expiring, EValueType::Derivative, ELaziness::OnStart); + + INIT_COUNTERS_COUPLE( + QueueCounters, + SendMessage_DeduplicationCount, deduplicated_count_per_second, + ELifetime::Expiring, EValueType::Derivative, + Lazy(*Cfg), AggregatedCounters + ); + INIT_COUNTERS_COUPLE( + QueueCounters, + SendMessage_Count, sent_count_per_second, + ELifetime::Expiring, EValueType::Derivative, + Lazy(*Cfg), AggregatedCounters + ); + INIT_COUNTERS_COUPLE( + QueueCounters, + SendMessage_BytesWritten, sent_bytes_per_second, + ELifetime::Expiring, EValueType::Derivative, + Lazy(*Cfg), AggregatedCounters + ); + + INIT_COUNTERS_COUPLE( + QueueCounters, + MessagesCount, stored_count, + ELifetime::Expiring, EValueType::Absolute, + Lazy(*Cfg), AggregatedCounters + ); + INIT_COUNTERS_COUPLE( + QueueCounters, + InflyMessagesCount, inflight_count, + ELifetime::Expiring, EValueType::Absolute, + Lazy(*Cfg), AggregatedCounters + ); if (!AggregatedCounters) { // OldestMessageAgeSeconds will not be aggregated properly. - INIT_COUNTERS_COUPLE( - QueueCounters, - OldestMessageAgeSeconds, oldest_age_milliseconds, - ELifetime::Expiring, EValueType::Absolute, - ELaziness::OnStart, false - ); + INIT_COUNTERS_COUPLE( + QueueCounters, + OldestMessageAgeSeconds, oldest_age_milliseconds, + ELifetime::Expiring, EValueType::Absolute, + ELaziness::OnStart, false + ); } } @@ -653,14 +653,14 @@ void TQueueCounters::InitCounters(bool forLeaderNode) { SqsActionCounters[action].Init(*Cfg, QueueCounters.SqsCounters, action, forLeaderNode ? ELifetime::Expiring : ELifetime::Persistent); } } - if (IsActionForQueueYMQ(action) && QueueCounters.YmqCounters && !AggregatedCounters) { + if (IsActionForQueueYMQ(action) && QueueCounters.YmqCounters && !AggregatedCounters) { if (forLeaderNode && IsProxyAction(action) || !forLeaderNode && !IsProxyAction(action)) { - YmqActionCounters[action].Init( - *Cfg, QueueCounters.YmqCounters, action, METHOD_LABLE, ACTION_CNTR_PREFIX, + YmqActionCounters[action].Init( + *Cfg, QueueCounters.YmqCounters, action, METHOD_LABLE, ACTION_CNTR_PREFIX, forLeaderNode ? ELifetime::Expiring : ELifetime::Persistent - ); - } - } + ); + } + } } DetailedCounters.Init(QueueCounters.SqsCounters, AllocPoolCounters, forLeaderNode); @@ -691,17 +691,17 @@ void TQueueCounters::TDetailedCounters::SetAggregatedParent(TQueueCounters::TDet void TQueueCounters::InsertCounters() { auto insert = [&](auto& parent) { - if (!parent.SqsCounters->FindSubgroup(QUEUE_LABEL, QueueName)) { - QueueCounters.SqsCounters->ResetCounters(); - parent.SqsCounters->RegisterSubgroup(QUEUE_LABEL, QueueName, QueueCounters.SqsCounters); + if (!parent.SqsCounters->FindSubgroup(QUEUE_LABEL, QueueName)) { + QueueCounters.SqsCounters->ResetCounters(); + parent.SqsCounters->RegisterSubgroup(QUEUE_LABEL, QueueName, QueueCounters.SqsCounters); + } + if (parent.YmqCounters && !parent.YmqCounters->FindSubgroup(QUEUE_LABEL, QueueName)) { + QueueCounters.YmqCounters->ResetCounters(); + parent.YmqCounters->RegisterSubgroup(QUEUE_LABEL, QueueName, QueueCounters.YmqCounters); } - if (parent.YmqCounters && !parent.YmqCounters->FindSubgroup(QUEUE_LABEL, QueueName)) { - QueueCounters.YmqCounters->ResetCounters(); - parent.YmqCounters->RegisterSubgroup(QUEUE_LABEL, QueueName, QueueCounters.YmqCounters); - } }; - if (FolderCounters.Defined()) { + if (FolderCounters.Defined()) { insert(FolderCounters); } else { insert(UserCounters); @@ -713,15 +713,15 @@ void TQueueCounters::SetAggregatedParent(const TIntrusivePtr<TQueueCounters>& pa QueueCountersDescriptor.SetAggregatedParent(this, parent.Get()); DetailedCounters.SetAggregatedParent(parent ? &parent->DetailedCounters : nullptr); for (size_t i = 0; i < EAction::ActionsArraySize; ++i) { - SqsActionCounters[i].SetAggregatedParent(parent ? &parent->SqsActionCounters[i] : nullptr); + SqsActionCounters[i].SetAggregatedParent(parent ? &parent->SqsActionCounters[i] : nullptr); } } void TQueueCounters::RemoveCounters() { - auto couple = FolderCounters.Defined() ? FolderCounters : UserCounters; - couple.SqsCounters->RemoveSubgroup(QUEUE_LABEL, QueueName); - if (couple.YmqCounters) - couple.YmqCounters->RemoveSubgroup(QUEUE_LABEL, QueueName); + auto couple = FolderCounters.Defined() ? FolderCounters : UserCounters; + couple.SqsCounters->RemoveSubgroup(QUEUE_LABEL, QueueName); + if (couple.YmqCounters) + couple.YmqCounters->RemoveSubgroup(QUEUE_LABEL, QueueName); } TIntrusivePtr<TQueueCounters> TQueueCounters::GetCountersForLeaderNode() { @@ -741,22 +741,22 @@ TIntrusivePtr<TQueueCounters> TQueueCounters::GetCountersForNotLeaderNode() { void TUserCounters::InitCounters(const TString& userName, const std::shared_ptr<TAlignedPagePoolCounters>& allocPoolCounters) { UserCounters = GetUserCounters(SqsCoreCounters, userName); - INIT_COUNTER(UserCounters.SqsCounters, RequestTimeouts, ELifetime::Persistent, EValueType::Derivative, Lazy(*Cfg)); + INIT_COUNTER(UserCounters.SqsCounters, RequestTimeouts, ELifetime::Persistent, EValueType::Derivative, Lazy(*Cfg)); if (Cfg->GetForceAccessControl() && Cfg->AccountsWithoutMandatoryAuthSize() && (userName == TOTAL_COUNTER_LABEL || IsIn(Cfg->GetAccountsWithoutMandatoryAuth(), userName))) { - INIT_COUNTER(UserCounters.SqsCounters, UnauthenticatedAccess, ELifetime::Persistent, EValueType::Derivative, ELaziness::OnStart); + INIT_COUNTER(UserCounters.SqsCounters, UnauthenticatedAccess, ELifetime::Persistent, EValueType::Derivative, ELaziness::OnStart); } for (EAction action = static_cast<EAction>(EAction::Unknown + 1); action < EAction::ActionsArraySize; action = static_cast<EAction>(action + 1)) { if (IsActionForUser(action)) { - SqsActionCounters[action].Init(*Cfg, UserCounters.SqsCounters, action); + SqsActionCounters[action].Init(*Cfg, UserCounters.SqsCounters, action); + } + if (IsActionForUserYMQ(action) && UserCounters.YmqCounters && !IsAggregatedCounters) { + YmqActionCounters[action].Init(*Cfg, UserCounters.YmqCounters, action, METHOD_LABLE, ACTION_CNTR_PREFIX); } - if (IsActionForUserYMQ(action) && UserCounters.YmqCounters && !IsAggregatedCounters) { - YmqActionCounters[action].Init(*Cfg, UserCounters.YmqCounters, action, METHOD_LABLE, ACTION_CNTR_PREFIX); - } } - // ToDo. Errors codes here. Will probably need this in Ymq counters further + // ToDo. Errors codes here. Will probably need this in Ymq counters further DetailedCounters.Init(UserCounters, allocPoolCounters, *Cfg); AggregatedQueueCounters = CreateQueueCountersImpl(TOTAL_COUNTER_LABEL, Cfg->GetYandexCloudMode() ? TOTAL_COUNTER_LABEL : TString(), true, true)->GetCountersForLeaderNode(); @@ -766,7 +766,7 @@ void TUserCounters::InitCounters(const TString& userName, const std::shared_ptr< UserCountersDescriptor.SetAggregatedParent(this, AggregatedParent.Get()); DetailedCounters.SetAggregatedParent(&AggregatedParent->DetailedCounters); for (size_t i = 0; i < EAction::ActionsArraySize; ++i) { - SqsActionCounters[i].SetAggregatedParent(&AggregatedParent->SqsActionCounters[i]); + SqsActionCounters[i].SetAggregatedParent(&AggregatedParent->SqsActionCounters[i]); } } } @@ -776,13 +776,13 @@ void TUserCounters::TDetailedCounters::Init(const TIntrusivePtrCntrCouple& userC TransactionCounters = new TTransactionCounters(); TransactionCounters->Init(GetAggregatedCountersFromUserCounters(userCounters, cfg), allocPoolCounters, false); - APIStatuses.Init(userCounters.SqsCounters); + APIStatuses.Init(userCounters.SqsCounters); - INIT_HISTOGRAM_COUNTER(userCounters.SqsCounters, GetConfiguration_Duration, ELifetime::Persistent, DurationBucketsMs, ELaziness::OnDemand); - INIT_HISTOGRAM_COUNTER(userCounters.SqsCounters, GetQuota_Duration, ELifetime::Persistent, GetQuotaDurationBucketsMs, ELaziness::OnDemand); + INIT_HISTOGRAM_COUNTER(userCounters.SqsCounters, GetConfiguration_Duration, ELifetime::Persistent, DurationBucketsMs, ELaziness::OnDemand); + INIT_HISTOGRAM_COUNTER(userCounters.SqsCounters, GetQuota_Duration, ELifetime::Persistent, GetQuotaDurationBucketsMs, ELaziness::OnDemand); - INIT_COUNTER(userCounters.SqsCounters, CreateAccountOnTheFly_Success, ELifetime::Persistent, EValueType::Derivative, ELaziness::OnDemand); - INIT_COUNTER(userCounters.SqsCounters, CreateAccountOnTheFly_Errors, ELifetime::Persistent, EValueType::Derivative, ELaziness::OnDemand); + INIT_COUNTER(userCounters.SqsCounters, CreateAccountOnTheFly_Success, ELifetime::Persistent, EValueType::Derivative, ELaziness::OnDemand); + INIT_COUNTER(userCounters.SqsCounters, CreateAccountOnTheFly_Errors, ELifetime::Persistent, EValueType::Derivative, ELaziness::OnDemand); } void TUserCounters::TDetailedCounters::SetAggregatedParent(TUserCounters::TDetailedCounters* parent) { @@ -791,7 +791,7 @@ void TUserCounters::TDetailedCounters::SetAggregatedParent(TUserCounters::TDetai } TIntrusivePtr<TQueueCounters> TUserCounters::CreateQueueCounters(const TString& queueName, const TString& folderId, bool insertCounters) { - auto counters = CreateQueueCountersImpl(queueName, folderId, insertCounters, IsAggregatedCounters); + auto counters = CreateQueueCountersImpl(queueName, folderId, insertCounters, IsAggregatedCounters); counters->SetAggregatedParent(AggregatedQueueCounters); return counters; } @@ -801,25 +801,25 @@ TIntrusivePtr<TQueueCounters> TUserCounters::CreateQueueCountersImpl(const TStri } void TUserCounters::RemoveCounters() { - SqsCoreCounters.SqsCounters->RemoveSubgroup(USER_LABEL, UserName); - if (SqsCoreCounters.YmqCounters) - SqsCoreCounters.YmqCounters->RemoveSubgroup(CLOUD_LABEL, UserName); + SqsCoreCounters.SqsCounters->RemoveSubgroup(USER_LABEL, UserName); + if (SqsCoreCounters.YmqCounters) + SqsCoreCounters.YmqCounters->RemoveSubgroup(CLOUD_LABEL, UserName); } void TUserCounters::DisableCounters(bool disable) { if (disable) { - SqsCoreCounters.SqsCounters->RemoveSubgroup(USER_LABEL, UserName); - if (SqsCoreCounters.YmqCounters) - SqsCoreCounters.YmqCounters->RemoveSubgroup(CLOUD_LABEL, UserName); + SqsCoreCounters.SqsCounters->RemoveSubgroup(USER_LABEL, UserName); + if (SqsCoreCounters.YmqCounters) + SqsCoreCounters.YmqCounters->RemoveSubgroup(CLOUD_LABEL, UserName); } else { - if (!SqsCoreCounters.SqsCounters->FindSubgroup(USER_LABEL, UserName)) { - UserCounters.SqsCounters->ResetCounters(); - SqsCoreCounters.SqsCounters->RegisterSubgroup(USER_LABEL, UserName, UserCounters.SqsCounters); + if (!SqsCoreCounters.SqsCounters->FindSubgroup(USER_LABEL, UserName)) { + UserCounters.SqsCounters->ResetCounters(); + SqsCoreCounters.SqsCounters->RegisterSubgroup(USER_LABEL, UserName, UserCounters.SqsCounters); + } + if (SqsCoreCounters.YmqCounters && !SqsCoreCounters.YmqCounters->FindSubgroup(CLOUD_LABEL, UserName)) { + UserCounters.YmqCounters->ResetCounters(); + SqsCoreCounters.YmqCounters->RegisterSubgroup(CLOUD_LABEL, UserName, UserCounters.YmqCounters); } - if (SqsCoreCounters.YmqCounters && !SqsCoreCounters.YmqCounters->FindSubgroup(CLOUD_LABEL, UserName)) { - UserCounters.YmqCounters->ResetCounters(); - SqsCoreCounters.YmqCounters->RegisterSubgroup(CLOUD_LABEL, UserName, UserCounters.YmqCounters); - } } } diff --git a/ydb/core/ymq/base/counters.h b/ydb/core/ymq/base/counters.h index ecd9777af2..3c2667b241 100644 --- a/ydb/core/ymq/base/counters.h +++ b/ydb/core/ymq/base/counters.h @@ -29,12 +29,12 @@ struct TCloudAuthCounters; ++*countersPack->counter; \ } -#define INC_COUNTER_COUPLE(countersPack, sqsCounter, ymqCounter) \ - if (countersPack) { \ - ++*countersPack->sqsCounter; \ - ++*countersPack->ymqCounter; \ - } - +#define INC_COUNTER_COUPLE(countersPack, sqsCounter, ymqCounter) \ + if (countersPack) { \ + ++*countersPack->sqsCounter; \ + ++*countersPack->ymqCounter; \ + } + #define DEC_COUNTER(countersPack, counter) \ if (countersPack) { \ --*countersPack->counter; \ @@ -45,31 +45,31 @@ struct TCloudAuthCounters; *countersPack->counter += (count); \ } -#define ADD_COUNTER_COUPLE(countersPack, sqsCounter, ymqCounter, count) \ - if (countersPack) { \ - *countersPack->sqsCounter += (count); \ - *countersPack->ymqCounter += (count); \ - } - -#define SET_COUNTER_COUPLE(countersPack, sqsCounter, ymqCounter, count) \ - if (countersPack) { \ - *countersPack->sqsCounter = (count); \ - *countersPack->ymqCounter = (count); \ - } - +#define ADD_COUNTER_COUPLE(countersPack, sqsCounter, ymqCounter, count) \ + if (countersPack) { \ + *countersPack->sqsCounter += (count); \ + *countersPack->ymqCounter += (count); \ + } + +#define SET_COUNTER_COUPLE(countersPack, sqsCounter, ymqCounter, count) \ + if (countersPack) { \ + *countersPack->sqsCounter = (count); \ + *countersPack->ymqCounter = (count); \ + } + #define COLLECT_HISTOGRAM_COUNTER(countersPack, counter, count) \ if (countersPack) { \ countersPack->counter->Collect(count); \ } -#define COLLECT_HISTOGRAM_COUNTER_COUPLE(countersCouple, counter, count) \ - if (countersCouple.SqsCounters) { \ - countersCouple.SqsCounters->counter->Collect(count); \ - } \ - if (countersCouple.YmqCounters) { \ - countersCouple.YmqCounters->counter->Collect(count); \ - } - +#define COLLECT_HISTOGRAM_COUNTER_COUPLE(countersCouple, counter, count) \ + if (countersCouple.SqsCounters) { \ + countersCouple.SqsCounters->counter->Collect(count); \ + } \ + if (countersCouple.YmqCounters) { \ + countersCouple.YmqCounters->counter->Collect(count); \ + } + // Enums for code readability and for static types guarantee that all parameters passed correctly. enum class ELaziness { OnStart, @@ -87,36 +87,36 @@ enum class EValueType { }; extern const TString TOTAL_COUNTER_LABEL; -//constexpr static std::array<int, 10> RESPONSE_CODES = {200, 400, 403, 404, 500, 503, 504}; - -template<typename TCounterPtrType> -struct TCountersCouple { - TCounterPtrType SqsCounters = nullptr; - TCounterPtrType YmqCounters = nullptr; - bool Defined() { - return SqsCounters != nullptr; - } -}; - -using TIntrusivePtrCntrCouple = TCountersCouple<TIntrusivePtr<NMonitoring::TDynamicCounters>>; - -TIntrusivePtr<NMonitoring::TDynamicCounters> GetSqsServiceCounters( - const TIntrusivePtr<NMonitoring::TDynamicCounters>& countersRoot, const TString& subgroup); -TIntrusivePtr<NMonitoring::TDynamicCounters> GetYmqPublicCounters( - const TIntrusivePtr<NMonitoring::TDynamicCounters>& countersRoot); - -TIntrusivePtrCntrCouple GetFolderCounters(const TIntrusivePtrCntrCouple& userCounters, const TString& folderId); -void RemoveFolderCounters(const TIntrusivePtrCntrCouple& userCounters, const TString& folderId); -std::pair<TIntrusivePtrCntrCouple, TIntrusivePtrCntrCouple> GetUserAndQueueCounters( - const TIntrusivePtrCntrCouple& sqsRootCounters, const TQueuePath& queuePath); -TIntrusivePtr<NMonitoring::TDynamicCounters> GetAggregatedCountersFromSqsCoreCounters( - const TIntrusivePtrCntrCouple& sqsCoreCounters, const NKikimrConfig::TSqsConfig& cfg); -TIntrusivePtr<NMonitoring::TDynamicCounters> GetAggregatedCountersFromUserCounters( - const TIntrusivePtrCntrCouple& sqsCoreCounters, const NKikimrConfig::TSqsConfig& cfg); - +//constexpr static std::array<int, 10> RESPONSE_CODES = {200, 400, 403, 404, 500, 503, 504}; + +template<typename TCounterPtrType> +struct TCountersCouple { + TCounterPtrType SqsCounters = nullptr; + TCounterPtrType YmqCounters = nullptr; + bool Defined() { + return SqsCounters != nullptr; + } +}; + +using TIntrusivePtrCntrCouple = TCountersCouple<TIntrusivePtr<NMonitoring::TDynamicCounters>>; + +TIntrusivePtr<NMonitoring::TDynamicCounters> GetSqsServiceCounters( + const TIntrusivePtr<NMonitoring::TDynamicCounters>& countersRoot, const TString& subgroup); +TIntrusivePtr<NMonitoring::TDynamicCounters> GetYmqPublicCounters( + const TIntrusivePtr<NMonitoring::TDynamicCounters>& countersRoot); + +TIntrusivePtrCntrCouple GetFolderCounters(const TIntrusivePtrCntrCouple& userCounters, const TString& folderId); +void RemoveFolderCounters(const TIntrusivePtrCntrCouple& userCounters, const TString& folderId); +std::pair<TIntrusivePtrCntrCouple, TIntrusivePtrCntrCouple> GetUserAndQueueCounters( + const TIntrusivePtrCntrCouple& sqsRootCounters, const TQueuePath& queuePath); +TIntrusivePtr<NMonitoring::TDynamicCounters> GetAggregatedCountersFromSqsCoreCounters( + const TIntrusivePtrCntrCouple& sqsCoreCounters, const NKikimrConfig::TSqsConfig& cfg); +TIntrusivePtr<NMonitoring::TDynamicCounters> GetAggregatedCountersFromUserCounters( + const TIntrusivePtrCntrCouple& sqsCoreCounters, const NKikimrConfig::TSqsConfig& cfg); + extern const TString DEFAULT_COUNTER_NAME; -extern const TString DEFAULT_YMQ_COUNTER_NAME; -//extern const TString ACTION_CNTR_PREFIX; +extern const TString DEFAULT_YMQ_COUNTER_NAME; +//extern const TString ACTION_CNTR_PREFIX; namespace NDetails { @@ -342,7 +342,7 @@ struct TLazyCachedHistogram : public NDetails::TLazyCachedCounterBase<NMonitorin } void Init(const TIntrusivePtr<NMonitoring::TDynamicCounters>& rootCounters, ELifetime lifetime, const NMonitoring::TBucketBounds& buckets, const TString& name, const TString& value, ELaziness laziness) { - Buckets = &buckets; + Buckets = &buckets; TLazyCachedCounterBase::Init(rootCounters, lifetime, name, value, ELaziness::OnDemand); if (laziness == ELaziness::OnStart) { EnsureCreated(); @@ -368,7 +368,7 @@ private: // Counters for actions (like SendMessage, CreateQueue or GetQueueUrl). struct TActionCounters { -public: +public: TLazyCachedCounter Success; TLazyCachedCounter Errors; // User metric for cloud console (SendMessage/ReceiveMessage/DeleteMessage). TLazyCachedCounter Infly; @@ -379,27 +379,27 @@ public: TLazyCachedHistogram WorkingDuration; // Special duration except wait time for ReceiveMessage action (== 18 counters). // User metric for cloud console (ReceiveMessage). public: - void Init(const NKikimrConfig::TSqsConfig& cfg, const TIntrusivePtr<NMonitoring::TDynamicCounters>& rootCounters, - EAction action, ELifetime lifetime = ELifetime::Persistent); + void Init(const NKikimrConfig::TSqsConfig& cfg, const TIntrusivePtr<NMonitoring::TDynamicCounters>& rootCounters, + EAction action, ELifetime lifetime = ELifetime::Persistent); - virtual void SetAggregatedParent(TActionCounters* parent); + virtual void SetAggregatedParent(TActionCounters* parent); virtual ~TActionCounters() {} }; -struct TYmqActionCounters : public TActionCounters { - void Init(const NKikimrConfig::TSqsConfig& cfg, const TIntrusivePtr<NMonitoring::TDynamicCounters>& rootCounters, - EAction action, const TString& labelName, const TString& namePrefix, - ELifetime lifetime = ELifetime::Persistent); -public: +struct TYmqActionCounters : public TActionCounters { + void Init(const NKikimrConfig::TSqsConfig& cfg, const TIntrusivePtr<NMonitoring::TDynamicCounters>& rootCounters, + EAction action, const TString& labelName, const TString& namePrefix, + ELifetime lifetime = ELifetime::Persistent); +public: virtual ~TYmqActionCounters() {} - void SetAggregatedParent(TActionCounters*) override { - return; - } - -private: - NMonitoring::TDynamicCounterPtr SubGroup; -}; - + void SetAggregatedParent(TActionCounters*) override { + return; + } + +private: + NMonitoring::TDynamicCounterPtr SubGroup; +}; + // Counters for typed queries (WRITE_MESSAGE_ID, PURGE_QUEUE_ID and etc). struct TQueryTypeCounters { TLazyCachedCounter TransactionsCount; @@ -453,7 +453,7 @@ private: struct TUserCounters : public TAtomicRefCount<TUserCounters> { // Action types are declared in ydb/core/ymq/base/action.h. TActionCounters SqsActionCounters[EAction::ActionsArraySize]; // 11 actions. See IsActionForUser() function in ydb/core/ymq/base/action.cpp. - TYmqActionCounters YmqActionCounters[EAction::ActionsArraySize]; // see IsActionForUserYMQ() function. + TYmqActionCounters YmqActionCounters[EAction::ActionsArraySize]; // see IsActionForUserYMQ() function. TLazyCachedCounter RequestTimeouts; // Requests that weren't processed in 10 minutes. They are almost sure hanged. @@ -470,7 +470,7 @@ struct TUserCounters : public TAtomicRefCount<TUserCounters> { TLazyCachedCounter CreateAccountOnTheFly_Success; // Account created on the fly (Yandex Cloud mode). TLazyCachedCounter CreateAccountOnTheFly_Errors; // Account that were failed to create on the fly (Yandex Cloud mode). - void Init(const TIntrusivePtrCntrCouple& userCounters, + void Init(const TIntrusivePtrCntrCouple& userCounters, const std::shared_ptr<TAlignedPagePoolCounters>& allocPoolCounters, const NKikimrConfig::TSqsConfig& cfg); void SetAggregatedParent(TDetailedCounters* parent); @@ -488,21 +488,21 @@ struct TUserCounters : public TAtomicRefCount<TUserCounters> { // Raw counters interface // Don't use counters by name! - TIntrusivePtrCntrCouple SqsCoreCounters; // Sqs core subsystem - TIntrusivePtrCntrCouple UserCounters; // User tree in core subsystem + TIntrusivePtrCntrCouple SqsCoreCounters; // Sqs core subsystem + TIntrusivePtrCntrCouple UserCounters; // User tree in core subsystem - TUserCounters( - const NKikimrConfig::TSqsConfig& cfg, const TIntrusivePtr<NMonitoring::TDynamicCounters>& sqsCoreCounters, - const TIntrusivePtr<NMonitoring::TDynamicCounters>& ymqRootCounters, + TUserCounters( + const NKikimrConfig::TSqsConfig& cfg, const TIntrusivePtr<NMonitoring::TDynamicCounters>& sqsCoreCounters, + const TIntrusivePtr<NMonitoring::TDynamicCounters>& ymqRootCounters, const std::shared_ptr<TAlignedPagePoolCounters>& allocPoolCounters, const TString& userName, - const TIntrusivePtr<TUserCounters>& aggregatedParent, - bool isAggregatedCounters = false - ) - : SqsCoreCounters{sqsCoreCounters, ymqRootCounters} + const TIntrusivePtr<TUserCounters>& aggregatedParent, + bool isAggregatedCounters = false + ) + : SqsCoreCounters{sqsCoreCounters, ymqRootCounters} , Cfg(&cfg) , UserName(userName) , AggregatedParent(aggregatedParent) - , IsAggregatedCounters(isAggregatedCounters) + , IsAggregatedCounters(isAggregatedCounters) { InitCounters(userName, allocPoolCounters); } @@ -551,7 +551,7 @@ private: TDetailedCounters DetailedCounters; TIntrusivePtr<TUserCounters> AggregatedParent; TIntrusivePtr<TQueueCounters> AggregatedQueueCounters; - bool IsAggregatedCounters; + bool IsAggregatedCounters; }; // Queue counters in SQS core subsystem. @@ -561,46 +561,46 @@ struct TQueueCounters : public TAtomicRefCount<TQueueCounters> { TYmqActionCounters YmqActionCounters[EAction::ActionsArraySize]; // See IsActionForQueueYMQ() function in ydb/core/ymq/sqs/base/action.cpp. TLazyCachedCounter RequestTimeouts; // Requests that weren't processed in 10 minutes. They are almost sure hanged. - TLazyCachedCounter request_timeouts_count_per_second; // Requests that weren't processed in 10 minutes. They are almost sure hanged. + TLazyCachedCounter request_timeouts_count_per_second; // Requests that weren't processed in 10 minutes. They are almost sure hanged. TLazyCachedCounter RequestsThrottled; // Request that ended with ThrottlingException TLazyCachedCounter QueueMasterStartProblems; // TODO: remove after migration TLazyCachedCounter QueueLeaderStartProblems; // Critical problems during leader start. TLazyCachedCounter MessagesPurged; - TLazyCachedCounter purged_count_per_second; + TLazyCachedCounter purged_count_per_second; TLazyCachedHistogram MessageReceiveAttempts; // User attempts for receive messages. Histogram with buckets for receive attempts (== 4 counters). // User metric for cloud console. - TLazyCachedHistogram receive_attempts_count_rate; + TLazyCachedHistogram receive_attempts_count_rate; TLazyCachedHistogram ClientMessageProcessing_Duration; // Time between receive and delete for deleted message. Histogram with buckets for client processing (== 21 counters). // User metric for cloud console. - TLazyCachedHistogram client_processing_duration_milliseconds; + TLazyCachedHistogram client_processing_duration_milliseconds; TLazyCachedHistogram MessageReside_Duration; // Time between send and receive for received messages. Histogram with buckets for client processing (== 21 counters). // User metric for cloud console. - TLazyCachedHistogram reside_duration_milliseconds; + TLazyCachedHistogram reside_duration_milliseconds; TLazyCachedCounter DeleteMessage_Count; // Messages count that were deleted. // User metric for cloud console. - TLazyCachedCounter deleted_count_per_second; + TLazyCachedCounter deleted_count_per_second; TLazyCachedCounter ReceiveMessage_EmptyCount; // Receive message requests count that returned empty results. - TLazyCachedCounter empty_receive_attempts_count_per_second; + TLazyCachedCounter empty_receive_attempts_count_per_second; TLazyCachedCounter ReceiveMessage_Count; // Messages count that were received. // User metric for cloud console. - TLazyCachedCounter received_count_per_second; + TLazyCachedCounter received_count_per_second; TLazyCachedCounter ReceiveMessage_BytesRead; // Bytes of message bodies that were received. // User metric for cloud console. - TLazyCachedCounter received_bytes_per_second; + TLazyCachedCounter received_bytes_per_second; TLazyCachedCounter MessagesMovedToDLQ; // Count of messages that were moved to DLQ. TLazyCachedCounter SendMessage_DeduplicationCount; // Count of messages that were deduplicated (for fifo only). - TLazyCachedCounter deduplicated_count_per_second; + TLazyCachedCounter deduplicated_count_per_second; TLazyCachedCounter SendMessage_Count; // Count of messages that were sent. // User metric for cloud console. - TLazyCachedCounter sent_count_per_second; + TLazyCachedCounter sent_count_per_second; TLazyCachedCounter SendMessage_BytesWritten; // Bytes of message bodies that were sent. - TLazyCachedCounter sent_bytes_per_second; + TLazyCachedCounter sent_bytes_per_second; TLazyCachedCounter MessagesCount; // Messages count in queue. // User metric for cloud console. - TLazyCachedCounter stored_count; + TLazyCachedCounter stored_count; TLazyCachedCounter InflyMessagesCount; // Messages count in queue that are inflight. // User metric for cloud console. - TLazyCachedCounter inflight_count; + TLazyCachedCounter inflight_count; TLazyCachedCounter OldestMessageAgeSeconds; // Age of the oldest message in queue. // User metric for cloud console. - TLazyCachedCounter oldest_age_milliseconds; + TLazyCachedCounter oldest_age_milliseconds; struct TDetailedCounters { TIntrusivePtr<TTransactionCounters> TransactionCounters; @@ -626,16 +626,16 @@ struct TQueueCounters : public TAtomicRefCount<TQueueCounters> { } } - // Raw counters interface.RootCounters + // Raw counters interface.RootCounters // Don't use counters by name! - TIntrusivePtrCntrCouple RootCounters; // Sqs core subsystem. - TIntrusivePtrCntrCouple UserCounters; // User tree in core subsystem - TIntrusivePtrCntrCouple FolderCounters; // Folder subtree in user tree (only for Yandex Cloud). - TIntrusivePtrCntrCouple QueueCounters; // Queue subtree in user (or folder) tree. + TIntrusivePtrCntrCouple RootCounters; // Sqs core subsystem. + TIntrusivePtrCntrCouple UserCounters; // User tree in core subsystem + TIntrusivePtrCntrCouple FolderCounters; // Folder subtree in user tree (only for Yandex Cloud). + TIntrusivePtrCntrCouple QueueCounters; // Queue subtree in user (or folder) tree. // Creates counters for not leader node. TQueueCounters(const NKikimrConfig::TSqsConfig& cfg, - const TIntrusivePtrCntrCouple& sqsCoreCounters, + const TIntrusivePtrCntrCouple& sqsCoreCounters, const TUserCounters* userCounters, const TString& queueName, const TString& folderId, diff --git a/ydb/core/ymq/base/ut/counters_ut.cpp b/ydb/core/ymq/base/ut/counters_ut.cpp index f2189504bc..5ff3d26e36 100644 --- a/ydb/core/ymq/base/ut/counters_ut.cpp +++ b/ydb/core/ymq/base/ut/counters_ut.cpp @@ -195,99 +195,99 @@ Y_UNIT_TEST_SUITE(LazyCounterTest) { } Y_UNIT_TEST_SUITE(UserCountersTest) { -#define ASSERT_STR_COUPLE_CONTAINS(string1, string2, what) \ - UNIT_ASSERT_STRING_CONTAINS(string1, what); \ - UNIT_ASSERT_STRING_CONTAINS(string2, what); - -#define ASSERT_FIRST_OF_COUPLE_CONTAINS(string1, string2, what) \ - UNIT_ASSERT_STRING_CONTAINS(string1, what); \ - UNIT_ASSERT(!string2.Contains(what)); - -#define ASSERT_STR_COUPLE_DONT_CONTAIN(string1, string2, what) \ - UNIT_ASSERT(string1.find(what) == TString::npos); \ - UNIT_ASSERT(string2.find(what) == TString::npos); - -#define ASSERT_USER_PRESENT(user) \ - ASSERT_FIRST_OF_COUPLE_CONTAINS(CountersString(core), CountersString(ymqCounters), TString("user=") + user); \ - ASSERT_FIRST_OF_COUPLE_CONTAINS(CountersString(ymqCounters), CountersString(core), TString("cloud=") + user); - -#define ASSERT_USER_ABSENT(user) \ - ASSERT_STR_COUPLE_DONT_CONTAIN(CountersString(core), CountersString(ymqCounters), TString("user=") + user); \ - ASSERT_STR_COUPLE_DONT_CONTAIN(CountersString(core), CountersString(ymqCounters), TString("cloud=") + user); - +#define ASSERT_STR_COUPLE_CONTAINS(string1, string2, what) \ + UNIT_ASSERT_STRING_CONTAINS(string1, what); \ + UNIT_ASSERT_STRING_CONTAINS(string2, what); + +#define ASSERT_FIRST_OF_COUPLE_CONTAINS(string1, string2, what) \ + UNIT_ASSERT_STRING_CONTAINS(string1, what); \ + UNIT_ASSERT(!string2.Contains(what)); + +#define ASSERT_STR_COUPLE_DONT_CONTAIN(string1, string2, what) \ + UNIT_ASSERT(string1.find(what) == TString::npos); \ + UNIT_ASSERT(string2.find(what) == TString::npos); + +#define ASSERT_USER_PRESENT(user) \ + ASSERT_FIRST_OF_COUPLE_CONTAINS(CountersString(core), CountersString(ymqCounters), TString("user=") + user); \ + ASSERT_FIRST_OF_COUPLE_CONTAINS(CountersString(ymqCounters), CountersString(core), TString("cloud=") + user); + +#define ASSERT_USER_ABSENT(user) \ + ASSERT_STR_COUPLE_DONT_CONTAIN(CountersString(core), CountersString(ymqCounters), TString("user=") + user); \ + ASSERT_STR_COUPLE_DONT_CONTAIN(CountersString(core), CountersString(ymqCounters), TString("cloud=") + user); + Y_UNIT_TEST(DisableCountersTest) { NKikimrConfig::TSqsConfig cfg; cfg.SetCreateLazyCounters(false); TIntrusivePtr<NMonitoring::TDynamicCounters> core = new NMonitoring::TDynamicCounters(); - TIntrusivePtr<NMonitoring::TDynamicCounters> ymqCounters = new NMonitoring::TDynamicCounters(); - TIntrusivePtr<TUserCounters> user1 = new TUserCounters(cfg, core, ymqCounters, nullptr, "user1", nullptr); - TIntrusivePtr<TUserCounters> user2 = new TUserCounters(cfg, core, ymqCounters, nullptr, "user2", nullptr); - ASSERT_USER_PRESENT("user1"); - ASSERT_USER_PRESENT("user2"); + TIntrusivePtr<NMonitoring::TDynamicCounters> ymqCounters = new NMonitoring::TDynamicCounters(); + TIntrusivePtr<TUserCounters> user1 = new TUserCounters(cfg, core, ymqCounters, nullptr, "user1", nullptr); + TIntrusivePtr<TUserCounters> user2 = new TUserCounters(cfg, core, ymqCounters, nullptr, "user2", nullptr); + ASSERT_USER_PRESENT("user1"); + ASSERT_USER_PRESENT("user2"); user1->DisableCounters(false); - ASSERT_USER_PRESENT("user1"); + ASSERT_USER_PRESENT("user1"); user1->DisableCounters(true); - TString sqsCntrText = CountersString(core); - TString ymqCntrText = CountersString(ymqCounters); - ASSERT_USER_ABSENT("user1"); - ASSERT_USER_PRESENT("user2"); + TString sqsCntrText = CountersString(core); + TString ymqCntrText = CountersString(ymqCounters); + ASSERT_USER_ABSENT("user1"); + ASSERT_USER_PRESENT("user2"); // again user1->DisableCounters(true); - sqsCntrText = CountersString(core); - ymqCntrText = CountersString(ymqCounters); - ASSERT_USER_ABSENT("user1"); - ASSERT_USER_PRESENT("user2"); + sqsCntrText = CountersString(core); + ymqCntrText = CountersString(ymqCounters); + ASSERT_USER_ABSENT("user1"); + ASSERT_USER_PRESENT("user2"); *user1->RequestTimeouts = 1; *user2->RequestTimeouts = 2; // return back user1->DisableCounters(false); - sqsCntrText = CountersString(core); - ymqCntrText = CountersString(ymqCounters); - ASSERT_USER_PRESENT("user1"); - ASSERT_USER_PRESENT("user2"); - ASSERT_STR_COUPLE_DONT_CONTAIN(sqsCntrText,ymqCntrText, "RequestTimeouts: 1"); - ASSERT_FIRST_OF_COUPLE_CONTAINS(sqsCntrText, ymqCntrText, "RequestTimeouts: 2"); - ASSERT_FIRST_OF_COUPLE_CONTAINS(sqsCntrText, ymqCntrText, "RequestTimeouts: 2"); - ASSERT_FIRST_OF_COUPLE_CONTAINS(sqsCntrText, ymqCntrText, "RequestTimeouts: 0"); + sqsCntrText = CountersString(core); + ymqCntrText = CountersString(ymqCounters); + ASSERT_USER_PRESENT("user1"); + ASSERT_USER_PRESENT("user2"); + ASSERT_STR_COUPLE_DONT_CONTAIN(sqsCntrText,ymqCntrText, "RequestTimeouts: 1"); + ASSERT_FIRST_OF_COUPLE_CONTAINS(sqsCntrText, ymqCntrText, "RequestTimeouts: 2"); + ASSERT_FIRST_OF_COUPLE_CONTAINS(sqsCntrText, ymqCntrText, "RequestTimeouts: 2"); + ASSERT_FIRST_OF_COUPLE_CONTAINS(sqsCntrText, ymqCntrText, "RequestTimeouts: 0"); // again *user1->RequestTimeouts = 3; user1->DisableCounters(false); - sqsCntrText = CountersString(core); - ymqCntrText = CountersString(ymqCounters); - ASSERT_USER_PRESENT("user1"); - ASSERT_USER_PRESENT("user2"); - ASSERT_FIRST_OF_COUPLE_CONTAINS(sqsCntrText, ymqCntrText, "RequestTimeouts: 2"); - ASSERT_FIRST_OF_COUPLE_CONTAINS(sqsCntrText, ymqCntrText, "RequestTimeouts: 3"); + sqsCntrText = CountersString(core); + ymqCntrText = CountersString(ymqCounters); + ASSERT_USER_PRESENT("user1"); + ASSERT_USER_PRESENT("user2"); + ASSERT_FIRST_OF_COUPLE_CONTAINS(sqsCntrText, ymqCntrText, "RequestTimeouts: 2"); + ASSERT_FIRST_OF_COUPLE_CONTAINS(sqsCntrText, ymqCntrText, "RequestTimeouts: 3"); } Y_UNIT_TEST(RemoveUserCountersTest) { NKikimrConfig::TSqsConfig cfg; cfg.SetCreateLazyCounters(false); TIntrusivePtr<NMonitoring::TDynamicCounters> core = new NMonitoring::TDynamicCounters(); - TIntrusivePtr<NMonitoring::TDynamicCounters> ymqCounters = new NMonitoring::TDynamicCounters(); + TIntrusivePtr<NMonitoring::TDynamicCounters> ymqCounters = new NMonitoring::TDynamicCounters(); + + TIntrusivePtr<TUserCounters> user = new TUserCounters(cfg, core, ymqCounters, nullptr, "my_user", nullptr); + ASSERT_USER_PRESENT("my_user"); - TIntrusivePtr<TUserCounters> user = new TUserCounters(cfg, core, ymqCounters, nullptr, "my_user", nullptr); - ASSERT_USER_PRESENT("my_user"); - user->RemoveCounters(); - ASSERT_USER_ABSENT("my_user"); + ASSERT_USER_ABSENT("my_user"); } Y_UNIT_TEST(CountersAggregationTest) { NKikimrConfig::TSqsConfig cfg; cfg.SetCreateLazyCounters(false); TIntrusivePtr<NMonitoring::TDynamicCounters> core = new NMonitoring::TDynamicCounters(); - TIntrusivePtr<NMonitoring::TDynamicCounters> ymqCounters = new NMonitoring::TDynamicCounters(); - TIntrusivePtr<TUserCounters> total = new TUserCounters(cfg, core, ymqCounters, nullptr, TOTAL_COUNTER_LABEL, nullptr); + TIntrusivePtr<NMonitoring::TDynamicCounters> ymqCounters = new NMonitoring::TDynamicCounters(); + TIntrusivePtr<TUserCounters> total = new TUserCounters(cfg, core, ymqCounters, nullptr, TOTAL_COUNTER_LABEL, nullptr); total->ShowDetailedCounters(TInstant::Max()); - TIntrusivePtr<TUserCounters> user = new TUserCounters(cfg, core, ymqCounters, nullptr, "my_user", total); + TIntrusivePtr<TUserCounters> user = new TUserCounters(cfg, core, ymqCounters, nullptr, "my_user", total); UNIT_ASSERT_STRING_CONTAINS(CountersString(core), "user=my_user"); UNIT_ASSERT_STRING_CONTAINS(CountersString(core), "user=total"); @@ -300,7 +300,7 @@ Y_UNIT_TEST_SUITE(UserCountersTest) { // Try different types of member counters: detailed/usual/arrays/api statuses. ++*user->RequestTimeouts; - ++*user->SqsActionCounters[EAction::CreateQueue].Errors; + ++*user->SqsActionCounters[EAction::CreateQueue].Errors; ++*user->GetDetailedCounters()->CreateAccountOnTheFly_Success; user->GetDetailedCounters()->APIStatuses.AddOk(3); user->GetDetailedCounters()->APIStatuses.AddError("AccessDeniedException", 2); @@ -308,8 +308,8 @@ Y_UNIT_TEST_SUITE(UserCountersTest) { ++*user->GetTransactionCounters()->TransactionsCount; ++*user->GetTransactionCounters()->QueryTypeCounters[EQueryId::WRITE_MESSAGE_ID].TransactionsFailed; - ++*user->YmqActionCounters[EAction::CreateQueue].Errors; - + ++*user->YmqActionCounters[EAction::CreateQueue].Errors; + AssertCounterValues(core, { { "user=total/RequestTimeouts", 1 }, @@ -331,14 +331,14 @@ Y_UNIT_TEST_SUITE(UserCountersTest) { { "user=total/queue=total/TransactionsFailedByType/query_type=WRITE_MESSAGE_ID", 1 }, { "user=my_user/queue=total/TransactionsFailedByType/query_type=WRITE_MESSAGE_ID", 1 }, }); - -// auto queue = user->CreateQueueCounters("my_queue", "folder", true); - - AssertCounterValues(ymqCounters, - { - { "cloud=my_user/method=create_queue/name=api.http.errors_count_per_second", 1 }, - } - ); + +// auto queue = user->CreateQueueCounters("my_queue", "folder", true); + + AssertCounterValues(ymqCounters, + { + { "cloud=my_user/method=create_queue/name=api.http.errors_count_per_second", 1 }, + } + ); } } @@ -347,44 +347,44 @@ Y_UNIT_TEST_SUITE(QueueCountersTest) { NKikimrConfig::TSqsConfig cfg; cfg.SetCreateLazyCounters(false); TIntrusivePtr<NMonitoring::TDynamicCounters> core = new NMonitoring::TDynamicCounters(); - TIntrusivePtr<NMonitoring::TDynamicCounters> ymqCounters = new NMonitoring::TDynamicCounters(); - TIntrusivePtr<TUserCounters> user = new TUserCounters(cfg, core, ymqCounters, nullptr, "my_user", nullptr); - ASSERT_USER_PRESENT("my_user"); + TIntrusivePtr<NMonitoring::TDynamicCounters> ymqCounters = new NMonitoring::TDynamicCounters(); + TIntrusivePtr<TUserCounters> user = new TUserCounters(cfg, core, ymqCounters, nullptr, "my_user", nullptr); + ASSERT_USER_PRESENT("my_user"); - auto queue = user->CreateQueueCounters("my_queue", "folder", false); - ASSERT_STR_COUPLE_DONT_CONTAIN(CountersString(core), CountersString(ymqCounters), "my_queue"); + auto queue = user->CreateQueueCounters("my_queue", "folder", false); + ASSERT_STR_COUPLE_DONT_CONTAIN(CountersString(core), CountersString(ymqCounters), "my_queue"); queue->InsertCounters(); - ASSERT_STR_COUPLE_CONTAINS(CountersString(core), CountersString(ymqCounters), "my_queue"); - - ++*queue->RequestTimeouts; - auto sqsCntrText = CountersString(core); - auto ymqCntrText = CountersString(ymqCounters); - ASSERT_FIRST_OF_COUPLE_CONTAINS(sqsCntrText, ymqCntrText, "RequestTimeouts: 1"); - - ++*queue->request_timeouts_count_per_second; - sqsCntrText = CountersString(core); - ymqCntrText = CountersString(ymqCounters); - ASSERT_FIRST_OF_COUPLE_CONTAINS(ymqCntrText, sqsCntrText, "request_timeouts_count_per_second: 1"); - + ASSERT_STR_COUPLE_CONTAINS(CountersString(core), CountersString(ymqCounters), "my_queue"); + + ++*queue->RequestTimeouts; + auto sqsCntrText = CountersString(core); + auto ymqCntrText = CountersString(ymqCounters); + ASSERT_FIRST_OF_COUPLE_CONTAINS(sqsCntrText, ymqCntrText, "RequestTimeouts: 1"); + + ++*queue->request_timeouts_count_per_second; + sqsCntrText = CountersString(core); + ymqCntrText = CountersString(ymqCounters); + ASSERT_FIRST_OF_COUPLE_CONTAINS(ymqCntrText, sqsCntrText, "request_timeouts_count_per_second: 1"); + // Second time: queue->InsertCounters(); - ASSERT_STR_COUPLE_CONTAINS(CountersString(core), CountersString(ymqCounters), "my_queue"); + ASSERT_STR_COUPLE_CONTAINS(CountersString(core), CountersString(ymqCounters), "my_queue"); } void RemoveQueueCountersTest(bool leader, const TString& folderId) { NKikimrConfig::TSqsConfig cfg; cfg.SetCreateLazyCounters(false); TIntrusivePtr<NMonitoring::TDynamicCounters> core = new NMonitoring::TDynamicCounters(); - TIntrusivePtr<NMonitoring::TDynamicCounters> ymqCounters = new NMonitoring::TDynamicCounters(); - TIntrusivePtr<TUserCounters> user = new TUserCounters(cfg, core, ymqCounters, nullptr, "my_user", nullptr); + TIntrusivePtr<NMonitoring::TDynamicCounters> ymqCounters = new NMonitoring::TDynamicCounters(); + TIntrusivePtr<TUserCounters> user = new TUserCounters(cfg, core, ymqCounters, nullptr, "my_user", nullptr); TIntrusivePtr<TQueueCounters> queue = user->CreateQueueCounters("my_queue", folderId, true); if (leader) { queue = queue->GetCountersForLeaderNode(); } - ASSERT_STR_COUPLE_CONTAINS(CountersString(core), CountersString(ymqCounters), "queue=my_queue"); + ASSERT_STR_COUPLE_CONTAINS(CountersString(core), CountersString(ymqCounters), "queue=my_queue"); queue->RemoveCounters(); - ASSERT_STR_COUPLE_DONT_CONTAIN(CountersString(core), CountersString(ymqCounters), "queue=my_queue"); + ASSERT_STR_COUPLE_DONT_CONTAIN(CountersString(core), CountersString(ymqCounters), "queue=my_queue"); } Y_UNIT_TEST(RemoveQueueCountersNonLeaderWithoutFolderTest) { @@ -408,10 +408,10 @@ Y_UNIT_TEST_SUITE(QueueCountersTest) { cfg.SetYandexCloudMode(cloudMode); cfg.SetCreateLazyCounters(false); TIntrusivePtr<NMonitoring::TDynamicCounters> core = new NMonitoring::TDynamicCounters(); - TIntrusivePtr<NMonitoring::TDynamicCounters> ymqCounters = new NMonitoring::TDynamicCounters(); - TIntrusivePtr<TUserCounters> total = new TUserCounters(cfg, core, ymqCounters, nullptr, TOTAL_COUNTER_LABEL, nullptr); + TIntrusivePtr<NMonitoring::TDynamicCounters> ymqCounters = new NMonitoring::TDynamicCounters(); + TIntrusivePtr<TUserCounters> total = new TUserCounters(cfg, core, ymqCounters, nullptr, TOTAL_COUNTER_LABEL, nullptr); total->ShowDetailedCounters(TInstant::Max()); - TIntrusivePtr<TUserCounters> user = new TUserCounters(cfg, core, ymqCounters, nullptr, "my_user", total); + TIntrusivePtr<TUserCounters> user = new TUserCounters(cfg, core, ymqCounters, nullptr, "my_user", total); TIntrusivePtr<TQueueCounters> queue = user->CreateQueueCounters("my_queue", cloudMode ? "my_folder" : "", true)->GetCountersForLeaderNode(); UNIT_ASSERT_STRING_CONTAINS(CountersString(core), "queue=my_queue"); @@ -419,9 +419,9 @@ Y_UNIT_TEST_SUITE(QueueCountersTest) { // Try different types of member counters: detailed/usual/arrays/api statuses. ++*queue->MessagesPurged; - ++*queue->purged_count_per_second; - ++*queue->SqsActionCounters[EAction::SendMessageBatch].Errors; - ++*queue->YmqActionCounters[EAction::SendMessageBatch].Errors; + ++*queue->purged_count_per_second; + ++*queue->SqsActionCounters[EAction::SendMessageBatch].Errors; + ++*queue->YmqActionCounters[EAction::SendMessageBatch].Errors; ++*queue->GetDetailedCounters()->ReceiveMessage_KeysInvalidated; ++*queue->GetTransactionCounters()->TransactionRetryTimeouts; ++*queue->GetTransactionCounters()->QueryTypeCounters[EQueryId::DELETE_MESSAGE_ID].TransactionsCount; @@ -459,13 +459,13 @@ Y_UNIT_TEST_SUITE(QueueCountersTest) { { x("user=my_user/queue=total/TransactionsByType/query_type=DELETE_MESSAGE_ID"), 1 }, { x("user=my_user/queue=my_queue/TransactionsByType/query_type=DELETE_MESSAGE_ID"), 1 }, }); - - AssertCounterValues(ymqCounters, - { - { x("cloud=my_user/queue=my_queue/name=queue.messages.purged_count_per_second"), 1 }, - - { x("cloud=my_user/queue=my_queue/method=send_message_batch/name=api.http.errors_count_per_second"), 1 }, - }); + + AssertCounterValues(ymqCounters, + { + { x("cloud=my_user/queue=my_queue/name=queue.messages.purged_count_per_second"), 1 }, + + { x("cloud=my_user/queue=my_queue/method=send_message_batch/name=api.http.errors_count_per_second"), 1 }, + }); } Y_UNIT_TEST(CountersAggregationTest) { diff --git a/ydb/core/ymq/http/http.cpp b/ydb/core/ymq/http/http.cpp index 1418127cd5..5d0963bfb2 100644 --- a/ydb/core/ymq/http/http.cpp +++ b/ydb/core/ymq/http/http.cpp @@ -233,12 +233,12 @@ bool THttpRequest::DoReply(const TReplyParams& p) { const TDuration parseTime = TInstant::Now() - StartTime_; RLOG_SQS_BASE_DEBUG(*Parent_->ActorSystem_, "Parse time: [" << parseTime.MilliSeconds() << "ms]"); - RLOG_SQS_BASE_INFO( - *Parent_->ActorSystem_, - "Start request. User [" << UserName_ << "] Queue [" << QueueName_ << "], Cloud [" << AccountName_ - << "], Folder [" << FolderId_ << "] Action [" << ActionToString(Action_) - << "] IP [" << SourceAddress_ << "]" - ); + RLOG_SQS_BASE_INFO( + *Parent_->ActorSystem_, + "Start request. User [" << UserName_ << "] Queue [" << QueueName_ << "], Cloud [" << AccountName_ + << "], Folder [" << FolderId_ << "] Action [" << ActionToString(Action_) + << "] IP [" << SourceAddress_ << "]" + ); if (!Parent_->Config.GetYandexCloudMode() && UserName_.empty()) { WriteResponse(p, MakeErrorXmlResponse(NErrors::MISSING_PARAMETER, Parent_->AggregatedUserCounters_.Get(), "No user name was provided.")); @@ -996,19 +996,19 @@ TAsyncHttpServer::~TAsyncHttpServer() { DebugInfo->HttpServer = nullptr; } -void TAsyncHttpServer::Initialize( - NActors::TActorSystem* as, TIntrusivePtr<NMonitoring::TDynamicCounters> sqsCounters, - TIntrusivePtr<NMonitoring::TDynamicCounters> ymqCounters, ui32 poolId -) { +void TAsyncHttpServer::Initialize( + NActors::TActorSystem* as, TIntrusivePtr<NMonitoring::TDynamicCounters> sqsCounters, + TIntrusivePtr<NMonitoring::TDynamicCounters> ymqCounters, ui32 poolId +) { ActorSystem_ = as; - HttpCounters_ = new THttpCounters(Config, sqsCounters->GetSubgroup("subsystem", "http")); + HttpCounters_ = new THttpCounters(Config, sqsCounters->GetSubgroup("subsystem", "http")); if (Config.GetYandexCloudMode()) { - CloudAuthCounters_ = MakeHolder<TCloudAuthCounters>(Config, sqsCounters->GetSubgroup("subsystem", "cloud_auth")); + CloudAuthCounters_ = MakeHolder<TCloudAuthCounters>(Config, sqsCounters->GetSubgroup("subsystem", "cloud_auth")); } - AggregatedUserCounters_ = MakeIntrusive<TUserCounters>( - Config, sqsCounters->GetSubgroup("subsystem", "core"), ymqCounters, - nullptr, TOTAL_COUNTER_LABEL, nullptr, true - ); + AggregatedUserCounters_ = MakeIntrusive<TUserCounters>( + Config, sqsCounters->GetSubgroup("subsystem", "core"), ymqCounters, + nullptr, TOTAL_COUNTER_LABEL, nullptr, true + ); AggregatedUserCounters_->ShowDetailedCounters(TInstant::Max()); PoolId_ = poolId; } diff --git a/ydb/core/ymq/http/http.h b/ydb/core/ymq/http/http.h index 28b140174c..01a2b0edc4 100644 --- a/ydb/core/ymq/http/http.h +++ b/ydb/core/ymq/http/http.h @@ -145,11 +145,11 @@ public: TAsyncHttpServer(const NKikimrConfig::TSqsConfig& config); ~TAsyncHttpServer(); - void Initialize( - NActors::TActorSystem* as, - TIntrusivePtr<NMonitoring::TDynamicCounters> sqsCounters, - TIntrusivePtr<NMonitoring::TDynamicCounters> ymqCounters, - ui32 poolId); + void Initialize( + NActors::TActorSystem* as, + TIntrusivePtr<NMonitoring::TDynamicCounters> sqsCounters, + TIntrusivePtr<NMonitoring::TDynamicCounters> ymqCounters, + ui32 poolId); void Start(); diff --git a/ydb/library/persqueue/topic_parser/topic_parser.cpp b/ydb/library/persqueue/topic_parser/topic_parser.cpp index 46b20618eb..a9598b25b2 100644 --- a/ydb/library/persqueue/topic_parser/topic_parser.cpp +++ b/ydb/library/persqueue/topic_parser/topic_parser.cpp @@ -44,79 +44,79 @@ TString ConvertOldConsumerName(const TString& consumer, const NActors::TActorCon } } -TString MakeConsumerPath(const TString& consumer) { - TStringBuilder res; - res.reserve(consumer.size()); - for (ui32 i = 0; i < consumer.size(); ++i) { - if (consumer[i] == '@') { - res << "/"; - } else { - res << consumer[i]; - } - } - if (res.find("/") == TString::npos) { - return TStringBuilder() << "shared/" << res; - } - return res; -} - -TString NormalizeFullPath(const TString& fullPath) { - if (fullPath.StartsWith("/")) - return fullPath.substr(1); - else { - return fullPath; - } -} - -TTopicsListController::TTopicsListController( - const std::shared_ptr<TTopicNamesConverterFactory>& converterFactory, - bool haveClusters, const TVector<TString>& clusters, const TString& localCluster -) - : ConverterFactory(converterFactory) - , HaveClusters(haveClusters) -{ - UpdateClusters(clusters, localCluster); -} - -void TTopicsListController::UpdateClusters(const TVector<TString>& clusters, const TString& localCluster) { - if (!HaveClusters) - return; - - Clusters = clusters; - LocalCluster = localCluster; -} - -TTopicsToConverter TTopicsListController::GetReadTopicsList( - const THashSet<TString>& clientTopics, bool onlyLocal, const TString& database) const -{ - TTopicsToConverter result; - auto PutTopic = [&] (const TString& topic, const TString& dc) { - auto converter = ConverterFactory->MakeTopicNameConverter(topic, dc, database); - result[converter->GetPrimaryPath()] = converter; - }; - - for (const auto& t : clientTopics) { - if (onlyLocal) { - PutTopic(t, LocalCluster); - } else if (HaveClusters){ - for(const auto& c : Clusters) { - PutTopic(t, c); - } - } else { - PutTopic(t, TString()); - } - } - return result; -} -TConverterPtr TTopicsListController::GetWriteTopicConverter( - const TString& clientName, const TString& database -) { - return ConverterFactory->MakeTopicNameConverter(clientName, LocalCluster, database); -} - -TConverterFactoryPtr TTopicsListController::GetConverterFactory() const { - return ConverterFactory; -}; - +TString MakeConsumerPath(const TString& consumer) { + TStringBuilder res; + res.reserve(consumer.size()); + for (ui32 i = 0; i < consumer.size(); ++i) { + if (consumer[i] == '@') { + res << "/"; + } else { + res << consumer[i]; + } + } + if (res.find("/") == TString::npos) { + return TStringBuilder() << "shared/" << res; + } + return res; +} + +TString NormalizeFullPath(const TString& fullPath) { + if (fullPath.StartsWith("/")) + return fullPath.substr(1); + else { + return fullPath; + } +} + +TTopicsListController::TTopicsListController( + const std::shared_ptr<TTopicNamesConverterFactory>& converterFactory, + bool haveClusters, const TVector<TString>& clusters, const TString& localCluster +) + : ConverterFactory(converterFactory) + , HaveClusters(haveClusters) +{ + UpdateClusters(clusters, localCluster); +} + +void TTopicsListController::UpdateClusters(const TVector<TString>& clusters, const TString& localCluster) { + if (!HaveClusters) + return; + + Clusters = clusters; + LocalCluster = localCluster; +} + +TTopicsToConverter TTopicsListController::GetReadTopicsList( + const THashSet<TString>& clientTopics, bool onlyLocal, const TString& database) const +{ + TTopicsToConverter result; + auto PutTopic = [&] (const TString& topic, const TString& dc) { + auto converter = ConverterFactory->MakeTopicNameConverter(topic, dc, database); + result[converter->GetPrimaryPath()] = converter; + }; + + for (const auto& t : clientTopics) { + if (onlyLocal) { + PutTopic(t, LocalCluster); + } else if (HaveClusters){ + for(const auto& c : Clusters) { + PutTopic(t, c); + } + } else { + PutTopic(t, TString()); + } + } + return result; +} +TConverterPtr TTopicsListController::GetWriteTopicConverter( + const TString& clientName, const TString& database +) { + return ConverterFactory->MakeTopicNameConverter(clientName, LocalCluster, database); +} + +TConverterFactoryPtr TTopicsListController::GetConverterFactory() const { + return ConverterFactory; +}; + } // namespace NPersQueue - + diff --git a/ydb/library/persqueue/topic_parser/topic_parser.h b/ydb/library/persqueue/topic_parser/topic_parser.h index 18c55177f8..eb5a69cc37 100644 --- a/ydb/library/persqueue/topic_parser/topic_parser.h +++ b/ydb/library/persqueue/topic_parser/topic_parser.h @@ -3,352 +3,352 @@ #include <library/cpp/actors/core/actor.h> #include <util/generic/string.h> -#include <util/generic/hash.h> -#include <util/string/builder.h> +#include <util/generic/hash.h> +#include <util/string/builder.h> #include <ydb/core/base/path.h> #include <ydb/library/persqueue/topic_parser_public/topic_parser.h> - + namespace NPersQueue { TString GetFullTopicPath(const NActors::TActorContext& ctx, TMaybe<TString> database, const TString& topicPath); TString ConvertNewConsumerName(const TString& consumer, const NActors::TActorContext& ctx); TString ConvertOldConsumerName(const TString& consumer, const NActors::TActorContext& ctx); -TString MakeConsumerPath(const TString& consumer); - - -#define CHECK_SET_VALID(cond, reason, statement) \ - if (!(cond)) { \ - Valid = false; \ - Reason = Reason + (reason) + ". "; \ - statement; \ - } - - -#define CHECK_VALID_AND_RETURN(result) \ - if (!Valid) { \ - return TString(); \ - } else { \ - Y_VERIFY(!result.empty()); \ - return result; \ - } - -class TTopicNamesConverterFactory; - -class TTopicNameConverter { - friend class TTopicNamesConverterFactory; - -public: - TTopicNameConverter() = default; - -protected: - TTopicNameConverter(bool noDcMode, TStringBuf& pqNormalizedRootPrefix) - : NoDcMode(noDcMode) - , PQRootPrefix(pqNormalizedRootPrefix) - {} - - TTopicNameConverter& ApplyToTopic( - const TString& topic, const TString& dc = TString(), const TString& database = TString() - ) { - OriginalTopic = topic; - - TStringBuf topicNormalized = topic; - topicNormalized.SkipPrefix("/"); - - TStringBuf dbNormalized = database; - dbNormalized.SkipPrefix("/"); - dbNormalized.ChopSuffix("/"); - - TStringBuf root; - if (NoDcMode || !PQRootPrefix.StartsWith(dbNormalized)) { - Database = database; - root = dbNormalized; - } else { - root = PQRootPrefix; - } - topicNormalized.SkipPrefix(root); - topicNormalized.SkipPrefix("/"); - Dc = dc; - bool supposeLegacyMode = Database.empty() && !NoDcMode; - auto hasDcInName = IsLegacyTopicNameWithDc(topicNormalized); - - CHECK_SET_VALID(!hasDcInName || dc.Empty(), "May not specify DC with legacy-style topic name containing dc", return *this); - if (NoDcMode) { - CHECK_SET_VALID(!Database.empty(), "Database is mandatory in FirstClass mode", return *this); - } else { - HasDc = hasDcInName || !dc.empty(); - } - - if (hasDcInName) { - supposeLegacyMode = true; - SplitFullLegacyName(topicNormalized); - return *this; - } else if (IsLegacyTopicName(topicNormalized)) { // Also must get here if 'SplitFullLegacyName' performed - supposeLegacyMode = true; - SplitShortLegacyName(topicNormalized); - return *this; - } else { - SplitModernName(topicNormalized); - - BuildModernNames(TString(root)); - if (supposeLegacyMode) { - ClientsideName = FullLegacyName; - PrimaryFullPath = FullLegacyPath; - } else { - ClientsideName = ModernName; - PrimaryFullPath = FullModernPath; - } - } - return *this; - } - -public: - bool IsValid() const { - return Valid; - } - - const TString& GetReason() { - return Reason; - } - - TString GetClientsideName() { - CHECK_VALID_AND_RETURN(ClientsideName); - } - - TString GetPrimaryPath() { - CHECK_VALID_AND_RETURN(PrimaryFullPath); - } - - TString GetSecondaryPath() { - Y_FAIL("UNIMPLEMENTED"); - CHECK_VALID_AND_RETURN(SecondaryFullPath); - } - - TString GetOriginalPath() const { - CHECK_VALID_AND_RETURN(OriginalTopic); - } - - TString GetShortLegacyName() const { - CHECK_VALID_AND_RETURN(LegacyName); - } - - TString GetFullLegacyName() const { - CHECK_VALID_AND_RETURN(FullLegacyName); - } - - TString GetFullLegacyPath() const { - CHECK_VALID_AND_RETURN(FullLegacyPath); - } - TString GetLegacyProducer() const { - CHECK_VALID_AND_RETURN(Producer); - } - - TString GetModernName() const { - CHECK_VALID_AND_RETURN(ModernName); - } - - TString GetModernPath() const { - CHECK_VALID_AND_RETURN(FullModernPath); - } - TString GetCluster() const { - return Dc; - } - - TString GetAccount() const { - if (!Valid) - return {}; - if (!Database.empty()) { - return Database; - } - TStringBuf buf(ModernName); - TStringBuf fst, snd; - auto res = buf.TrySplit("/", fst, snd); - if (!res) { - fst = buf; - } - return TString{fst}; - } - -private: - bool IsLegacyTopicName(const TStringBuf& path) { - if (path.find("--") != TStringBuf::npos) - return true; - if (Database.empty() && !NoDcMode && path.find("/") == TStringBuf::npos) - return true; - return false; - } - - bool IsLegacyTopicNameWithDc(const TStringBuf& path) { - return path.StartsWith("rt3."); - } - - void SplitFullLegacyName(TStringBuf& name) { - // rt3.sas--accountName[@dirName]--topicName - FullLegacyName = name; - TStringBuf fst, snd; - name.SkipPrefix("rt3."); - auto res = name.TrySplit("--", fst, snd); - CHECK_SET_VALID(res, Sprintf("topic name %s is corrupted", OriginalTopic.c_str()), return); - Dc = fst; - LegacyName = snd; - FullLegacyPath = NKikimr::JoinPath({TString(PQRootPrefix), FullLegacyName}); - SplitShortLegacyName(snd); - } - - void SplitShortLegacyName(const TStringBuf& name) { - if (LegacyName.empty()) - LegacyName = name; - CHECK_SET_VALID(!NoDcMode, "legacy-style topic name as first-class citizen", return); - TStringBuf buf, logtype; - TStringBuilder topicPath; - auto res = name.TryRSplit("--", buf, logtype); - if (!res) { - topicPath << name; - Producer = name; - } else { - Producer = buf; - while (true) { - TStringBuf fst, snd; - if (!buf.TrySplit("@", fst, snd)) - break; - topicPath << fst << "/"; - buf = fst; - } - topicPath << buf << "/" << logtype; - } - ModernName = topicPath; - BuildFullLegacyName(); - PrimaryFullPath = FullLegacyPath; - if (ClientsideName.empty() && !NoDcMode && !FullLegacyName.empty()) { - ClientsideName = FullLegacyName; - } - } - - void SplitModernName(TStringBuf& name) { - if (ModernName.empty()) { - ModernName = name; - } - TStringBuf fst, snd; - - auto res = name.TrySplit("/", fst, snd); - if (!res) { - LegacyName = name; - Producer = name; - } else { - TStringBuilder legacyName; - legacyName << fst; - while (true) { - name = snd; - res = name.TrySplit("/", fst, snd); - if (!res) - break; - legacyName << "@" << fst; - } - Producer = legacyName; - legacyName << "--" << name; - LegacyName = legacyName; - } - BuildFullLegacyName(); - } - - void BuildModernNames(const TString& prefix) { - if (!NoDcMode && !Dc.empty()) { - // ToDo[migration]: build full modern name, full modern path - } else { - FullModernName = ModernName; - FullModernPath = NKikimr::JoinPath({prefix, FullModernName}); - } - } - void BuildFullLegacyName() { - if (FullLegacyName.empty() || !Dc.empty()) { - TStringBuilder builder; - builder << "rt3." << Dc << "--" << LegacyName; - FullLegacyName = builder; - } - if (FullLegacyPath.empty() && !FullLegacyName.empty()) { - FullLegacyPath = NKikimr::JoinPath({PQRootPrefix, FullLegacyName}); - } - } - - - bool NoDcMode; - TString PQRootPrefix; - - TString OriginalTopic; - TString Database; - TString Dc; - - TString LegacyName; - TString FullLegacyName; - TString FullLegacyPath; - - TString ModernName; - TString FullModernName; - TString FullModernPath; - - TString PrimaryFullPath; - TString SecondaryFullPath; - TString ClientsideName; - - TString Producer; - TString Account; - bool HasDc = false; - - bool Valid = true; - TString Reason; -}; - -using TConverterPtr = std::shared_ptr<TTopicNameConverter>; - -class TTopicNamesConverterFactory { -public: - TTopicNamesConverterFactory(bool noDcMode, const TString& pqRootPrefix) - : NoDcMode(noDcMode) - , PQRootPrefix(pqRootPrefix) - , NormalizedPrefix(PQRootPrefix) - { - NormalizedPrefix.SkipPrefix("/"); - NormalizedPrefix.ChopSuffix("/"); - } - - TConverterPtr MakeTopicNameConverter( - const TString& topic, const TString& dc = TString(), const TString& database = TString() - ) { - auto* converter = new TTopicNameConverter(NoDcMode, NormalizedPrefix); - converter->ApplyToTopic(topic, dc, database); - return TConverterPtr(converter); - } - -private: - bool NoDcMode; - TString PQRootPrefix; - TStringBuf NormalizedPrefix; -}; - -using TConverterFactoryPtr = std::shared_ptr<NPersQueue::TTopicNamesConverterFactory>; -using TTopicsToConverter = THashMap<TString, TConverterPtr>; - -class TTopicsListController { -public: - TTopicsListController(const TConverterFactoryPtr& converterFactory, - bool haveClusters, const TVector<TString>& clusters = {}, const TString& localCluster = {}); - - void UpdateClusters(const TVector<TString>& clusters, const TString& localCluster); - TTopicsToConverter GetReadTopicsList(const THashSet<TString>& clientTopics, bool onlyLocal, const TString& database) const; - TConverterPtr GetWriteTopicConverter(const TString& clientName, const TString& database); - TConverterFactoryPtr GetConverterFactory() const; - - bool GetHaveClusters() const { return HaveClusters;}; - -private: - TConverterFactoryPtr ConverterFactory; - bool HaveClusters; - TVector<TString> Clusters; - TString LocalCluster; - -}; - -TString NormalizeFullPath(const TString& fullPath); - - +TString MakeConsumerPath(const TString& consumer); + + +#define CHECK_SET_VALID(cond, reason, statement) \ + if (!(cond)) { \ + Valid = false; \ + Reason = Reason + (reason) + ". "; \ + statement; \ + } + + +#define CHECK_VALID_AND_RETURN(result) \ + if (!Valid) { \ + return TString(); \ + } else { \ + Y_VERIFY(!result.empty()); \ + return result; \ + } + +class TTopicNamesConverterFactory; + +class TTopicNameConverter { + friend class TTopicNamesConverterFactory; + +public: + TTopicNameConverter() = default; + +protected: + TTopicNameConverter(bool noDcMode, TStringBuf& pqNormalizedRootPrefix) + : NoDcMode(noDcMode) + , PQRootPrefix(pqNormalizedRootPrefix) + {} + + TTopicNameConverter& ApplyToTopic( + const TString& topic, const TString& dc = TString(), const TString& database = TString() + ) { + OriginalTopic = topic; + + TStringBuf topicNormalized = topic; + topicNormalized.SkipPrefix("/"); + + TStringBuf dbNormalized = database; + dbNormalized.SkipPrefix("/"); + dbNormalized.ChopSuffix("/"); + + TStringBuf root; + if (NoDcMode || !PQRootPrefix.StartsWith(dbNormalized)) { + Database = database; + root = dbNormalized; + } else { + root = PQRootPrefix; + } + topicNormalized.SkipPrefix(root); + topicNormalized.SkipPrefix("/"); + Dc = dc; + bool supposeLegacyMode = Database.empty() && !NoDcMode; + auto hasDcInName = IsLegacyTopicNameWithDc(topicNormalized); + + CHECK_SET_VALID(!hasDcInName || dc.Empty(), "May not specify DC with legacy-style topic name containing dc", return *this); + if (NoDcMode) { + CHECK_SET_VALID(!Database.empty(), "Database is mandatory in FirstClass mode", return *this); + } else { + HasDc = hasDcInName || !dc.empty(); + } + + if (hasDcInName) { + supposeLegacyMode = true; + SplitFullLegacyName(topicNormalized); + return *this; + } else if (IsLegacyTopicName(topicNormalized)) { // Also must get here if 'SplitFullLegacyName' performed + supposeLegacyMode = true; + SplitShortLegacyName(topicNormalized); + return *this; + } else { + SplitModernName(topicNormalized); + + BuildModernNames(TString(root)); + if (supposeLegacyMode) { + ClientsideName = FullLegacyName; + PrimaryFullPath = FullLegacyPath; + } else { + ClientsideName = ModernName; + PrimaryFullPath = FullModernPath; + } + } + return *this; + } + +public: + bool IsValid() const { + return Valid; + } + + const TString& GetReason() { + return Reason; + } + + TString GetClientsideName() { + CHECK_VALID_AND_RETURN(ClientsideName); + } + + TString GetPrimaryPath() { + CHECK_VALID_AND_RETURN(PrimaryFullPath); + } + + TString GetSecondaryPath() { + Y_FAIL("UNIMPLEMENTED"); + CHECK_VALID_AND_RETURN(SecondaryFullPath); + } + + TString GetOriginalPath() const { + CHECK_VALID_AND_RETURN(OriginalTopic); + } + + TString GetShortLegacyName() const { + CHECK_VALID_AND_RETURN(LegacyName); + } + + TString GetFullLegacyName() const { + CHECK_VALID_AND_RETURN(FullLegacyName); + } + + TString GetFullLegacyPath() const { + CHECK_VALID_AND_RETURN(FullLegacyPath); + } + TString GetLegacyProducer() const { + CHECK_VALID_AND_RETURN(Producer); + } + + TString GetModernName() const { + CHECK_VALID_AND_RETURN(ModernName); + } + + TString GetModernPath() const { + CHECK_VALID_AND_RETURN(FullModernPath); + } + TString GetCluster() const { + return Dc; + } + + TString GetAccount() const { + if (!Valid) + return {}; + if (!Database.empty()) { + return Database; + } + TStringBuf buf(ModernName); + TStringBuf fst, snd; + auto res = buf.TrySplit("/", fst, snd); + if (!res) { + fst = buf; + } + return TString{fst}; + } + +private: + bool IsLegacyTopicName(const TStringBuf& path) { + if (path.find("--") != TStringBuf::npos) + return true; + if (Database.empty() && !NoDcMode && path.find("/") == TStringBuf::npos) + return true; + return false; + } + + bool IsLegacyTopicNameWithDc(const TStringBuf& path) { + return path.StartsWith("rt3."); + } + + void SplitFullLegacyName(TStringBuf& name) { + // rt3.sas--accountName[@dirName]--topicName + FullLegacyName = name; + TStringBuf fst, snd; + name.SkipPrefix("rt3."); + auto res = name.TrySplit("--", fst, snd); + CHECK_SET_VALID(res, Sprintf("topic name %s is corrupted", OriginalTopic.c_str()), return); + Dc = fst; + LegacyName = snd; + FullLegacyPath = NKikimr::JoinPath({TString(PQRootPrefix), FullLegacyName}); + SplitShortLegacyName(snd); + } + + void SplitShortLegacyName(const TStringBuf& name) { + if (LegacyName.empty()) + LegacyName = name; + CHECK_SET_VALID(!NoDcMode, "legacy-style topic name as first-class citizen", return); + TStringBuf buf, logtype; + TStringBuilder topicPath; + auto res = name.TryRSplit("--", buf, logtype); + if (!res) { + topicPath << name; + Producer = name; + } else { + Producer = buf; + while (true) { + TStringBuf fst, snd; + if (!buf.TrySplit("@", fst, snd)) + break; + topicPath << fst << "/"; + buf = fst; + } + topicPath << buf << "/" << logtype; + } + ModernName = topicPath; + BuildFullLegacyName(); + PrimaryFullPath = FullLegacyPath; + if (ClientsideName.empty() && !NoDcMode && !FullLegacyName.empty()) { + ClientsideName = FullLegacyName; + } + } + + void SplitModernName(TStringBuf& name) { + if (ModernName.empty()) { + ModernName = name; + } + TStringBuf fst, snd; + + auto res = name.TrySplit("/", fst, snd); + if (!res) { + LegacyName = name; + Producer = name; + } else { + TStringBuilder legacyName; + legacyName << fst; + while (true) { + name = snd; + res = name.TrySplit("/", fst, snd); + if (!res) + break; + legacyName << "@" << fst; + } + Producer = legacyName; + legacyName << "--" << name; + LegacyName = legacyName; + } + BuildFullLegacyName(); + } + + void BuildModernNames(const TString& prefix) { + if (!NoDcMode && !Dc.empty()) { + // ToDo[migration]: build full modern name, full modern path + } else { + FullModernName = ModernName; + FullModernPath = NKikimr::JoinPath({prefix, FullModernName}); + } + } + void BuildFullLegacyName() { + if (FullLegacyName.empty() || !Dc.empty()) { + TStringBuilder builder; + builder << "rt3." << Dc << "--" << LegacyName; + FullLegacyName = builder; + } + if (FullLegacyPath.empty() && !FullLegacyName.empty()) { + FullLegacyPath = NKikimr::JoinPath({PQRootPrefix, FullLegacyName}); + } + } + + + bool NoDcMode; + TString PQRootPrefix; + + TString OriginalTopic; + TString Database; + TString Dc; + + TString LegacyName; + TString FullLegacyName; + TString FullLegacyPath; + + TString ModernName; + TString FullModernName; + TString FullModernPath; + + TString PrimaryFullPath; + TString SecondaryFullPath; + TString ClientsideName; + + TString Producer; + TString Account; + bool HasDc = false; + + bool Valid = true; + TString Reason; +}; + +using TConverterPtr = std::shared_ptr<TTopicNameConverter>; + +class TTopicNamesConverterFactory { +public: + TTopicNamesConverterFactory(bool noDcMode, const TString& pqRootPrefix) + : NoDcMode(noDcMode) + , PQRootPrefix(pqRootPrefix) + , NormalizedPrefix(PQRootPrefix) + { + NormalizedPrefix.SkipPrefix("/"); + NormalizedPrefix.ChopSuffix("/"); + } + + TConverterPtr MakeTopicNameConverter( + const TString& topic, const TString& dc = TString(), const TString& database = TString() + ) { + auto* converter = new TTopicNameConverter(NoDcMode, NormalizedPrefix); + converter->ApplyToTopic(topic, dc, database); + return TConverterPtr(converter); + } + +private: + bool NoDcMode; + TString PQRootPrefix; + TStringBuf NormalizedPrefix; +}; + +using TConverterFactoryPtr = std::shared_ptr<NPersQueue::TTopicNamesConverterFactory>; +using TTopicsToConverter = THashMap<TString, TConverterPtr>; + +class TTopicsListController { +public: + TTopicsListController(const TConverterFactoryPtr& converterFactory, + bool haveClusters, const TVector<TString>& clusters = {}, const TString& localCluster = {}); + + void UpdateClusters(const TVector<TString>& clusters, const TString& localCluster); + TTopicsToConverter GetReadTopicsList(const THashSet<TString>& clientTopics, bool onlyLocal, const TString& database) const; + TConverterPtr GetWriteTopicConverter(const TString& clientName, const TString& database); + TConverterFactoryPtr GetConverterFactory() const; + + bool GetHaveClusters() const { return HaveClusters;}; + +private: + TConverterFactoryPtr ConverterFactory; + bool HaveClusters; + TVector<TString> Clusters; + TString LocalCluster; + +}; + +TString NormalizeFullPath(const TString& fullPath); + + } // namespace NPersQueue - + diff --git a/ydb/library/persqueue/topic_parser/ut/topic_names_converter_ut.cpp b/ydb/library/persqueue/topic_parser/ut/topic_names_converter_ut.cpp index a50bcf47c8..d684d43172 100644 --- a/ydb/library/persqueue/topic_parser/ut/topic_names_converter_ut.cpp +++ b/ydb/library/persqueue/topic_parser/ut/topic_names_converter_ut.cpp @@ -1,176 +1,176 @@ #include <ydb/library/persqueue/topic_parser/topic_parser.h> -#include <library/cpp/testing/unittest/registar.h> - -using namespace NPersQueue; -namespace NTests { - -void BasicLegacyModeChecks(TTopicNameConverter* converter) { - UNIT_ASSERT(converter->IsValid()); - UNIT_ASSERT_VALUES_EQUAL(converter->GetPrimaryPath(), converter->GetFullLegacyPath()); - UNIT_ASSERT_VALUES_EQUAL(converter->GetClientsideName(), converter->GetFullLegacyName()); -} - -void BasicPathModeChecks(TTopicNameConverter* converter, const TString& fullPath) { - UNIT_ASSERT_C(converter->IsValid(), converter->GetReason()); - UNIT_ASSERT_VALUES_EQUAL(converter->GetClientsideName(), converter->GetModernName()); - UNIT_ASSERT_VALUES_EQUAL(converter->GetPrimaryPath(), fullPath); - UNIT_ASSERT_VALUES_EQUAL(converter->GetModernPath(), fullPath); -} - -Y_UNIT_TEST_SUITE(TopicNamesConverterTest) { - Y_UNIT_TEST(LegacyModeWithMinimalName) { - TTopicNamesConverterFactory factory(false, "/Root/PQ"); - - auto converter = factory.MakeTopicNameConverter("topic1", "dc1", ""); - UNIT_ASSERT_VALUES_EQUAL(converter->GetShortLegacyName(), "topic1"); - UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyName(), "rt3.dc1--topic1"); - UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyPath(), "Root/PQ/rt3.dc1--topic1"); - BasicLegacyModeChecks(converter.get()); - - converter = factory.MakeTopicNameConverter("topic2", "dc1", ""); - UNIT_ASSERT_VALUES_EQUAL(converter->GetShortLegacyName(), "topic2"); - UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyName(), "rt3.dc1--topic2"); - UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyPath(), "Root/PQ/rt3.dc1--topic2"); - BasicLegacyModeChecks(converter.get()); - } - - Y_UNIT_TEST(LegacyModeWithReset) { - TTopicNamesConverterFactory factory(false, "/Root/PQ"); - auto converter = factory.MakeTopicNameConverter("topic1", "dc1", ""); - UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyName(), "rt3.dc1--topic1"); - BasicLegacyModeChecks(converter.get()); - - converter = factory.MakeTopicNameConverter("topic1", "dc2", ""); - UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyName(), "rt3.dc2--topic1"); - BasicLegacyModeChecks(converter.get()); - - } - - Y_UNIT_TEST(LegacyModeWithShortNameAndDc) { - TTopicNamesConverterFactory factory(false, "/Root/PQ"); - auto converter = factory.MakeTopicNameConverter("rt3.dc2--topic2", "", ""); - UNIT_ASSERT_VALUES_EQUAL(converter->GetShortLegacyName(), "topic2"); - UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyName(), "rt3.dc2--topic2"); - UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyPath(), "Root/PQ/rt3.dc2--topic2"); - BasicLegacyModeChecks(converter.get()); - } - - Y_UNIT_TEST(LegacyModeWithNormalNameAndRootDB) { - TTopicNamesConverterFactory factory(false, "/Root/PQ"); - auto converter = factory.MakeTopicNameConverter("account--topic1", "dc2", "/Root"); - UNIT_ASSERT_VALUES_EQUAL(converter->GetShortLegacyName(), "account--topic1"); - UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyName(), "rt3.dc2--account--topic1"); - UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyPath(), "Root/PQ/rt3.dc2--account--topic1"); - BasicLegacyModeChecks(converter.get()); - - converter = factory.MakeTopicNameConverter("/Root/PQ/rt3.dc3--account3--topic3", "", ""); - UNIT_ASSERT_VALUES_EQUAL(converter->GetShortLegacyName(), "account3--topic3"); - UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyName(), "rt3.dc3--account3--topic3"); - UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyPath(), "Root/PQ/rt3.dc3--account3--topic3"); - BasicLegacyModeChecks(converter.get()); - } - - Y_UNIT_TEST(LegacyModeWithFullNameAndRootDB) { - TTopicNamesConverterFactory factory(false, "/Root/PQ"); - auto converter = factory.MakeTopicNameConverter("rt3.sas--accountX@dir--topicX", "", "/Root"); - UNIT_ASSERT_VALUES_EQUAL(converter->GetShortLegacyName(), "accountX@dir--topicX"); - UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyName(), "rt3.sas--accountX@dir--topicX"); - UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyPath(), "Root/PQ/rt3.sas--accountX@dir--topicX"); - BasicLegacyModeChecks(converter.get()); - } - - Y_UNIT_TEST(PathModeWithConversionToLegacy) { - TTopicNamesConverterFactory factory(false, "/Root/PQ"); - auto converter = factory.MakeTopicNameConverter("account1/topic1", "dc", "/Root"); - UNIT_ASSERT_VALUES_EQUAL(converter->GetShortLegacyName(), "account1--topic1"); - UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyName(), "rt3.dc--account1--topic1"); - UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyPath(), "Root/PQ/rt3.dc--account1--topic1"); - BasicLegacyModeChecks(converter.get()); - } - - Y_UNIT_TEST(LegacyModeWithErrors) { - TTopicNamesConverterFactory factory(false, "/Root/PQ"); - //UNIT_ASSERT(!factory.MakeTopicNameConverter("topic", "", "")->IsValid()); - //UNIT_ASSERT(!factory.MakeTopicNameConverter("topic", "", "/database")->IsValid()); - UNIT_ASSERT(!factory.MakeTopicNameConverter("rt3.man--account2--topic", "man", "")->IsValid()); - UNIT_ASSERT(!factory.MakeTopicNameConverter("rt3.man--account2--topic", "man", "")->IsValid()); - UNIT_ASSERT(!factory.MakeTopicNameConverter("rt3.man--account2--topic", "man", "")->IsValid()); - - } - - Y_UNIT_TEST(PathWithNoDc) { - TTopicNamesConverterFactory factory(false, "/Root/PQ"); - auto converter = factory.MakeTopicNameConverter("account/topic1", "", ""); - UNIT_ASSERT(converter->IsValid()); - UNIT_ASSERT_VALUES_EQUAL(converter->GetModernName(), "account/topic1"); - } - - Y_UNIT_TEST(FstClassModeWithShortTopic) { - TTopicNamesConverterFactory factory(true, ""); - { - auto converter = factory.MakeTopicNameConverter("topic1", "", "/database"); - UNIT_ASSERT_VALUES_EQUAL(converter->GetClientsideName(), "topic1"); - BasicPathModeChecks(converter.get(), "database/topic1"); - } - { - auto converter = factory.MakeTopicNameConverter("topic2", "", "database2"); - UNIT_ASSERT_VALUES_EQUAL(converter->GetClientsideName(), "topic2"); - BasicPathModeChecks(converter.get(), "database2/topic2"); - } - { - auto converter = factory.MakeTopicNameConverter("topic3", "", "database3/"); - BasicPathModeChecks(converter.get(), "database3/topic3"); - UNIT_ASSERT_VALUES_EQUAL(converter->GetClientsideName(), "topic3"); - } - } - Y_UNIT_TEST(FstClassModeWithLegacyNames) { - { - TTopicNamesConverterFactory factory(true, "Root/PQ"); - auto converter = factory.MakeTopicNameConverter("/Root/PQ/rt3.dc1--topic3", "", "/Root"); - UNIT_ASSERT(!converter->IsValid()); - converter = factory.MakeTopicNameConverter("/Root/PQ/rt3.dc1--topic3", "", ""); - UNIT_ASSERT(!converter->IsValid()); - } - { - TTopicNamesConverterFactory factory(true, ""); - auto converter = factory.MakeTopicNameConverter("/Root/PQ/rt3.dc1--topic3", "", "/Root"); - UNIT_ASSERT(!converter->IsValid()); - } - } - - Y_UNIT_TEST(FstClassModeWithRoot) { - TVector<TTopicNamesConverterFactory> factories{ - {true, ""}, - {true, "/Root/PQ"} - }; - for (auto& factory: factories) { - { - auto converter = factory.MakeTopicNameConverter("account/topic1", "", "/Root"); - BasicPathModeChecks(converter.get(), "Root/account/topic1"); - UNIT_ASSERT_VALUES_EQUAL(converter->GetClientsideName(), "account/topic1"); - } - { - auto converter = factory.MakeTopicNameConverter("/Root/account2/topic2", "", "/Root"); - BasicPathModeChecks(converter.get(), "Root/account2/topic2"); - UNIT_ASSERT_VALUES_EQUAL(converter->GetClientsideName(), "account2/topic2"); - } - } - } - - Y_UNIT_TEST(FstClassModeWithLongTopic) { - TTopicNamesConverterFactory factory(true, ""); - - auto converter = factory.MakeTopicNameConverter("dir4/topic4", "", "/database4"); - BasicPathModeChecks(converter.get(), "database4/dir4/topic4"); - UNIT_ASSERT_VALUES_EQUAL(converter->GetClientsideName(), "dir4/topic4"); - - } - - Y_UNIT_TEST(FstClassModeWithBadTopics) { - TTopicNamesConverterFactory factory(true, "/Root/PQ"); - UNIT_ASSERT(!factory.MakeTopicNameConverter("topic1", "", "")->IsValid()); - } - -} - -} // NTests
\ No newline at end of file +#include <library/cpp/testing/unittest/registar.h> + +using namespace NPersQueue; +namespace NTests { + +void BasicLegacyModeChecks(TTopicNameConverter* converter) { + UNIT_ASSERT(converter->IsValid()); + UNIT_ASSERT_VALUES_EQUAL(converter->GetPrimaryPath(), converter->GetFullLegacyPath()); + UNIT_ASSERT_VALUES_EQUAL(converter->GetClientsideName(), converter->GetFullLegacyName()); +} + +void BasicPathModeChecks(TTopicNameConverter* converter, const TString& fullPath) { + UNIT_ASSERT_C(converter->IsValid(), converter->GetReason()); + UNIT_ASSERT_VALUES_EQUAL(converter->GetClientsideName(), converter->GetModernName()); + UNIT_ASSERT_VALUES_EQUAL(converter->GetPrimaryPath(), fullPath); + UNIT_ASSERT_VALUES_EQUAL(converter->GetModernPath(), fullPath); +} + +Y_UNIT_TEST_SUITE(TopicNamesConverterTest) { + Y_UNIT_TEST(LegacyModeWithMinimalName) { + TTopicNamesConverterFactory factory(false, "/Root/PQ"); + + auto converter = factory.MakeTopicNameConverter("topic1", "dc1", ""); + UNIT_ASSERT_VALUES_EQUAL(converter->GetShortLegacyName(), "topic1"); + UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyName(), "rt3.dc1--topic1"); + UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyPath(), "Root/PQ/rt3.dc1--topic1"); + BasicLegacyModeChecks(converter.get()); + + converter = factory.MakeTopicNameConverter("topic2", "dc1", ""); + UNIT_ASSERT_VALUES_EQUAL(converter->GetShortLegacyName(), "topic2"); + UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyName(), "rt3.dc1--topic2"); + UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyPath(), "Root/PQ/rt3.dc1--topic2"); + BasicLegacyModeChecks(converter.get()); + } + + Y_UNIT_TEST(LegacyModeWithReset) { + TTopicNamesConverterFactory factory(false, "/Root/PQ"); + auto converter = factory.MakeTopicNameConverter("topic1", "dc1", ""); + UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyName(), "rt3.dc1--topic1"); + BasicLegacyModeChecks(converter.get()); + + converter = factory.MakeTopicNameConverter("topic1", "dc2", ""); + UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyName(), "rt3.dc2--topic1"); + BasicLegacyModeChecks(converter.get()); + + } + + Y_UNIT_TEST(LegacyModeWithShortNameAndDc) { + TTopicNamesConverterFactory factory(false, "/Root/PQ"); + auto converter = factory.MakeTopicNameConverter("rt3.dc2--topic2", "", ""); + UNIT_ASSERT_VALUES_EQUAL(converter->GetShortLegacyName(), "topic2"); + UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyName(), "rt3.dc2--topic2"); + UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyPath(), "Root/PQ/rt3.dc2--topic2"); + BasicLegacyModeChecks(converter.get()); + } + + Y_UNIT_TEST(LegacyModeWithNormalNameAndRootDB) { + TTopicNamesConverterFactory factory(false, "/Root/PQ"); + auto converter = factory.MakeTopicNameConverter("account--topic1", "dc2", "/Root"); + UNIT_ASSERT_VALUES_EQUAL(converter->GetShortLegacyName(), "account--topic1"); + UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyName(), "rt3.dc2--account--topic1"); + UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyPath(), "Root/PQ/rt3.dc2--account--topic1"); + BasicLegacyModeChecks(converter.get()); + + converter = factory.MakeTopicNameConverter("/Root/PQ/rt3.dc3--account3--topic3", "", ""); + UNIT_ASSERT_VALUES_EQUAL(converter->GetShortLegacyName(), "account3--topic3"); + UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyName(), "rt3.dc3--account3--topic3"); + UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyPath(), "Root/PQ/rt3.dc3--account3--topic3"); + BasicLegacyModeChecks(converter.get()); + } + + Y_UNIT_TEST(LegacyModeWithFullNameAndRootDB) { + TTopicNamesConverterFactory factory(false, "/Root/PQ"); + auto converter = factory.MakeTopicNameConverter("rt3.sas--accountX@dir--topicX", "", "/Root"); + UNIT_ASSERT_VALUES_EQUAL(converter->GetShortLegacyName(), "accountX@dir--topicX"); + UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyName(), "rt3.sas--accountX@dir--topicX"); + UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyPath(), "Root/PQ/rt3.sas--accountX@dir--topicX"); + BasicLegacyModeChecks(converter.get()); + } + + Y_UNIT_TEST(PathModeWithConversionToLegacy) { + TTopicNamesConverterFactory factory(false, "/Root/PQ"); + auto converter = factory.MakeTopicNameConverter("account1/topic1", "dc", "/Root"); + UNIT_ASSERT_VALUES_EQUAL(converter->GetShortLegacyName(), "account1--topic1"); + UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyName(), "rt3.dc--account1--topic1"); + UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyPath(), "Root/PQ/rt3.dc--account1--topic1"); + BasicLegacyModeChecks(converter.get()); + } + + Y_UNIT_TEST(LegacyModeWithErrors) { + TTopicNamesConverterFactory factory(false, "/Root/PQ"); + //UNIT_ASSERT(!factory.MakeTopicNameConverter("topic", "", "")->IsValid()); + //UNIT_ASSERT(!factory.MakeTopicNameConverter("topic", "", "/database")->IsValid()); + UNIT_ASSERT(!factory.MakeTopicNameConverter("rt3.man--account2--topic", "man", "")->IsValid()); + UNIT_ASSERT(!factory.MakeTopicNameConverter("rt3.man--account2--topic", "man", "")->IsValid()); + UNIT_ASSERT(!factory.MakeTopicNameConverter("rt3.man--account2--topic", "man", "")->IsValid()); + + } + + Y_UNIT_TEST(PathWithNoDc) { + TTopicNamesConverterFactory factory(false, "/Root/PQ"); + auto converter = factory.MakeTopicNameConverter("account/topic1", "", ""); + UNIT_ASSERT(converter->IsValid()); + UNIT_ASSERT_VALUES_EQUAL(converter->GetModernName(), "account/topic1"); + } + + Y_UNIT_TEST(FstClassModeWithShortTopic) { + TTopicNamesConverterFactory factory(true, ""); + { + auto converter = factory.MakeTopicNameConverter("topic1", "", "/database"); + UNIT_ASSERT_VALUES_EQUAL(converter->GetClientsideName(), "topic1"); + BasicPathModeChecks(converter.get(), "database/topic1"); + } + { + auto converter = factory.MakeTopicNameConverter("topic2", "", "database2"); + UNIT_ASSERT_VALUES_EQUAL(converter->GetClientsideName(), "topic2"); + BasicPathModeChecks(converter.get(), "database2/topic2"); + } + { + auto converter = factory.MakeTopicNameConverter("topic3", "", "database3/"); + BasicPathModeChecks(converter.get(), "database3/topic3"); + UNIT_ASSERT_VALUES_EQUAL(converter->GetClientsideName(), "topic3"); + } + } + Y_UNIT_TEST(FstClassModeWithLegacyNames) { + { + TTopicNamesConverterFactory factory(true, "Root/PQ"); + auto converter = factory.MakeTopicNameConverter("/Root/PQ/rt3.dc1--topic3", "", "/Root"); + UNIT_ASSERT(!converter->IsValid()); + converter = factory.MakeTopicNameConverter("/Root/PQ/rt3.dc1--topic3", "", ""); + UNIT_ASSERT(!converter->IsValid()); + } + { + TTopicNamesConverterFactory factory(true, ""); + auto converter = factory.MakeTopicNameConverter("/Root/PQ/rt3.dc1--topic3", "", "/Root"); + UNIT_ASSERT(!converter->IsValid()); + } + } + + Y_UNIT_TEST(FstClassModeWithRoot) { + TVector<TTopicNamesConverterFactory> factories{ + {true, ""}, + {true, "/Root/PQ"} + }; + for (auto& factory: factories) { + { + auto converter = factory.MakeTopicNameConverter("account/topic1", "", "/Root"); + BasicPathModeChecks(converter.get(), "Root/account/topic1"); + UNIT_ASSERT_VALUES_EQUAL(converter->GetClientsideName(), "account/topic1"); + } + { + auto converter = factory.MakeTopicNameConverter("/Root/account2/topic2", "", "/Root"); + BasicPathModeChecks(converter.get(), "Root/account2/topic2"); + UNIT_ASSERT_VALUES_EQUAL(converter->GetClientsideName(), "account2/topic2"); + } + } + } + + Y_UNIT_TEST(FstClassModeWithLongTopic) { + TTopicNamesConverterFactory factory(true, ""); + + auto converter = factory.MakeTopicNameConverter("dir4/topic4", "", "/database4"); + BasicPathModeChecks(converter.get(), "database4/dir4/topic4"); + UNIT_ASSERT_VALUES_EQUAL(converter->GetClientsideName(), "dir4/topic4"); + + } + + Y_UNIT_TEST(FstClassModeWithBadTopics) { + TTopicNamesConverterFactory factory(true, "/Root/PQ"); + UNIT_ASSERT(!factory.MakeTopicNameConverter("topic1", "", "")->IsValid()); + } + +} + +} // NTests
\ No newline at end of file diff --git a/ydb/library/persqueue/topic_parser/ut/ya.make b/ydb/library/persqueue/topic_parser/ut/ya.make index bfc6ca3074..74fdbf0bcd 100644 --- a/ydb/library/persqueue/topic_parser/ut/ya.make +++ b/ydb/library/persqueue/topic_parser/ut/ya.make @@ -1,24 +1,24 @@ UNITTEST_FOR(ydb/library/persqueue/topic_parser) - + OWNER( komels g:kikimr ) - -FORK_SUBTESTS() -TIMEOUT(30) +FORK_SUBTESTS() + +TIMEOUT(30) -SIZE(SMALL) - -PEERDIR( - library/cpp/getopt - library/cpp/svnversion +SIZE(SMALL) + +PEERDIR( + library/cpp/getopt + library/cpp/svnversion ydb/library/persqueue/topic_parser -) - -SRCS( - topic_names_converter_ut.cpp -) - -END() +) + +SRCS( + topic_names_converter_ut.cpp +) + +END() diff --git a/ydb/library/persqueue/topic_parser/ya.make b/ydb/library/persqueue/topic_parser/ya.make index 0078e83431..2b4f126a59 100644 --- a/ydb/library/persqueue/topic_parser/ya.make +++ b/ydb/library/persqueue/topic_parser/ya.make @@ -1,5 +1,5 @@ OWNER( - komels + komels g:kikimr g:logbroker ) @@ -18,7 +18,7 @@ SRCS( ) END() - + RECURSE_FOR_TESTS( - ut -) + ut +) diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/common.cpp b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/common.cpp index b939a1cfb7..dd42c8c4ed 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/common.cpp +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/common.cpp @@ -2,8 +2,8 @@ #include <util/charset/unidata.h> -namespace NYdb::NPersQueue { - +namespace NYdb::NPersQueue { + IRetryPolicy::ERetryErrorClass GetRetryErrorClass(EStatus status) { switch (status) { case EStatus::SUCCESS: @@ -29,7 +29,7 @@ IRetryPolicy::ERetryErrorClass GetRetryErrorClass(EStatus status) { case EStatus::CLIENT_LIMITS_REACHED: case EStatus::CLIENT_DISCOVERY_FAILED: return IRetryPolicy::ERetryErrorClass::LongRetry; - + case EStatus::SCHEME_ERROR: case EStatus::STATUS_UNDEFINED: case EStatus::BAD_REQUEST: @@ -41,38 +41,38 @@ IRetryPolicy::ERetryErrorClass GetRetryErrorClass(EStatus status) { case EStatus::CLIENT_UNAUTHENTICATED: case EStatus::CLIENT_CALL_UNIMPLEMENTED: return IRetryPolicy::ERetryErrorClass::NoRetry; - } -} - -IRetryPolicy::ERetryErrorClass GetRetryErrorClassV2(EStatus status) { - switch (status) { - case EStatus::SCHEME_ERROR: - return IRetryPolicy::ERetryErrorClass::NoRetry; - default: - return GetRetryErrorClass(status); - - } -} - -void Cancel(NGrpc::IQueueClientContextPtr& context) { - if (context) { - context->Cancel(); - } -} - -NYql::TIssues MakeIssueWithSubIssues(const TString& description, const NYql::TIssues& subissues) { - NYql::TIssues issues; - NYql::TIssue issue(description); - for (const NYql::TIssue& i : subissues) { - issue.AddSubIssue(MakeIntrusive<NYql::TIssue>(i)); - } - issues.AddIssue(std::move(issue)); - return issues; -} - -size_t CalcDataSize(const TReadSessionEvent::TEvent& event) { + } +} + +IRetryPolicy::ERetryErrorClass GetRetryErrorClassV2(EStatus status) { + switch (status) { + case EStatus::SCHEME_ERROR: + return IRetryPolicy::ERetryErrorClass::NoRetry; + default: + return GetRetryErrorClass(status); + + } +} + +void Cancel(NGrpc::IQueueClientContextPtr& context) { + if (context) { + context->Cancel(); + } +} + +NYql::TIssues MakeIssueWithSubIssues(const TString& description, const NYql::TIssues& subissues) { + NYql::TIssues issues; + NYql::TIssue issue(description); + for (const NYql::TIssue& i : subissues) { + issue.AddSubIssue(MakeIntrusive<NYql::TIssue>(i)); + } + issues.AddIssue(std::move(issue)); + return issues; +} + +size_t CalcDataSize(const TReadSessionEvent::TEvent& event) { if (const TReadSessionEvent::TDataReceivedEvent* dataEvent = std::get_if<TReadSessionEvent::TDataReceivedEvent>(&event)) { - size_t len = 0; + size_t len = 0; if (dataEvent->IsCompressedMessages()) { for (const auto& msg : dataEvent->GetCompressedMessages()) { len += msg.GetData().size(); @@ -83,13 +83,13 @@ size_t CalcDataSize(const TReadSessionEvent::TEvent& event) { len += msg.GetData().size(); } } - } - return len; - } else { - return 0; - } -} - + } + return len; + } else { + return 0; + } +} + static TStringBuf SplitPort(TStringBuf endpoint) { for (int i = endpoint.Size() - 1; i >= 0; --i) { if (endpoint[i] == ':') { @@ -121,90 +121,90 @@ TString ApplyClusterEndpoint(TStringBuf driverEndpoint, const TString& clusterDi } } -void IAsyncExecutor::Post(TFunction&& f) { - PostImpl(std::move(f)); -} - -IAsyncExecutor::TPtr CreateDefaultExecutor() { +void IAsyncExecutor::Post(TFunction&& f) { + PostImpl(std::move(f)); +} + +IAsyncExecutor::TPtr CreateDefaultExecutor() { return CreateThreadPoolExecutor(1); -} - -void TThreadPoolExecutor::PostImpl(TVector<TFunction>&& fs) { - for (auto& f : fs) { - ThreadPool->SafeAddFunc(std::move(f)); - } -} - -void TThreadPoolExecutor::PostImpl(TFunction&& f) { - ThreadPool->SafeAddFunc(std::move(f)); -} - -TSerialExecutor::TSerialExecutor(IAsyncExecutor::TPtr executor) +} + +void TThreadPoolExecutor::PostImpl(TVector<TFunction>&& fs) { + for (auto& f : fs) { + ThreadPool->SafeAddFunc(std::move(f)); + } +} + +void TThreadPoolExecutor::PostImpl(TFunction&& f) { + ThreadPool->SafeAddFunc(std::move(f)); +} + +TSerialExecutor::TSerialExecutor(IAsyncExecutor::TPtr executor) : Executor(executor) { - Y_VERIFY(executor); -} - -void TSerialExecutor::PostImpl(TVector<TFunction>&& fs) { - for (auto& f : fs) { - PostImpl(std::move(f)); - } -} - -void TSerialExecutor::PostImpl(TFunction&& f) { - with_lock(Mutex) { - ExecutionQueue.push(std::move(f)); - if (Busy) { - return; - } - PostNext(); - } -} - -void TSerialExecutor::PostNext() { - Y_VERIFY(!Busy); - - if (ExecutionQueue.empty()) { - return; - } - - auto weakThis = weak_from_this(); - Executor->Post([weakThis, f = std::move(ExecutionQueue.front())]() { - if (auto sharedThis = weakThis.lock()) { - f(); - with_lock(sharedThis->Mutex) { - sharedThis->Busy = false; - sharedThis->PostNext(); - } - } - }); - ExecutionQueue.pop(); - Busy = true; -} + Y_VERIFY(executor); +} + +void TSerialExecutor::PostImpl(TVector<TFunction>&& fs) { + for (auto& f : fs) { + PostImpl(std::move(f)); + } +} + +void TSerialExecutor::PostImpl(TFunction&& f) { + with_lock(Mutex) { + ExecutionQueue.push(std::move(f)); + if (Busy) { + return; + } + PostNext(); + } +} + +void TSerialExecutor::PostNext() { + Y_VERIFY(!Busy); + + if (ExecutionQueue.empty()) { + return; + } + + auto weakThis = weak_from_this(); + Executor->Post([weakThis, f = std::move(ExecutionQueue.front())]() { + if (auto sharedThis = weakThis.lock()) { + f(); + with_lock(sharedThis->Mutex) { + sharedThis->Busy = false; + sharedThis->PostNext(); + } + } + }); + ExecutionQueue.pop(); + Busy = true; +} IExecutor::TPtr CreateThreadPoolExecutor(size_t threads) { return MakeIntrusive<TThreadPoolExecutor>(threads); -} - +} + IExecutor::TPtr CreateGenericExecutor() { return CreateThreadPoolExecutor(1); -} +} IExecutor::TPtr CreateThreadPoolExecutorAdapter(std::shared_ptr<IThreadPool> threadPool) { return MakeIntrusive<TThreadPoolExecutor>(std::move(threadPool)); -} - +} + TThreadPoolExecutor::TThreadPoolExecutor(std::shared_ptr<IThreadPool> threadPool) : ThreadPool(std::move(threadPool)) { IsFakeThreadPool = dynamic_cast<TFakeThreadPool*>(ThreadPool.get()) != nullptr; -} - +} + TThreadPoolExecutor::TThreadPoolExecutor(size_t threadsCount) : TThreadPoolExecutor(CreateThreadPool(threadsCount)) { Y_VERIFY(threadsCount > 0); ThreadsCount = threadsCount; -} +} } diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/common.h b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/common.h index 96b671dee8..7287b69894 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/common.h +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/common.h @@ -5,36 +5,36 @@ #include <util/generic/queue.h> #include <util/system/condvar.h> -#include <util/thread/pool.h> +#include <util/thread/pool.h> #include <queue> namespace NYdb::NPersQueue { IRetryPolicy::ERetryErrorClass GetRetryErrorClass(EStatus status); -IRetryPolicy::ERetryErrorClass GetRetryErrorClassV2(EStatus status); - -void Cancel(NGrpc::IQueueClientContextPtr& context); - -NYql::TIssues MakeIssueWithSubIssues(const TString& description, const NYql::TIssues& subissues); - +IRetryPolicy::ERetryErrorClass GetRetryErrorClassV2(EStatus status); + +void Cancel(NGrpc::IQueueClientContextPtr& context); + +NYql::TIssues MakeIssueWithSubIssues(const TString& description, const NYql::TIssues& subissues); + TString IssuesSingleLineString(const NYql::TIssues& issues); size_t CalcDataSize(const TReadSessionEvent::TEvent& event); -template <class TMessage> -bool IsErrorMessage(const TMessage& serverMessage) { - const Ydb::StatusIds::StatusCode status = serverMessage.status(); - return status != Ydb::StatusIds::SUCCESS && status != Ydb::StatusIds::STATUS_CODE_UNSPECIFIED; -} - -template <class TMessage> -TPlainStatus MakeErrorFromProto(const TMessage& serverMessage) { - NYql::TIssues issues; - NYql::IssuesFromMessage(serverMessage.issues(), issues); - return TPlainStatus(static_cast<EStatus>(serverMessage.status()), std::move(issues)); -} - +template <class TMessage> +bool IsErrorMessage(const TMessage& serverMessage) { + const Ydb::StatusIds::StatusCode status = serverMessage.status(); + return status != Ydb::StatusIds::SUCCESS && status != Ydb::StatusIds::STATUS_CODE_UNSPECIFIED; +} + +template <class TMessage> +TPlainStatus MakeErrorFromProto(const TMessage& serverMessage) { + NYql::TIssues issues; + NYql::IssuesFromMessage(serverMessage.issues(), issues); + return TPlainStatus(static_cast<EStatus>(serverMessage.status()), std::move(issues)); +} + // Gets source endpoint for the whole driver (or persqueue client) // and endpoint that was given us by the cluster discovery service // and gives endpoint for the current LB cluster. @@ -189,22 +189,22 @@ std::shared_ptr<ISessionConnectionProcessorFactory<TRequest, TResponse>> template <class TEvent_> struct TBaseEventInfo { using TEvent = TEvent_; - + TEvent Event; - + TEvent& GetEvent() { return Event; - } - + } + void OnUserRetrievedEvent() { - } - + } + template <class T> TBaseEventInfo(T&& event) : Event(std::forward<T>(event)) {} }; - + class ISignalable { public: @@ -278,27 +278,27 @@ protected: using TSettings = TSettings_; using TEvent = TEvent_; using TEventInfo = TEventInfo_; - - + + // Template for visitor implementation. struct TBaseHandlersVisitor { TBaseHandlersVisitor(const TSettings& settings, TEventInfo& eventInfo) : Settings(settings) , EventInfo(eventInfo) - {} - + {} + template <class TEventType, class TFunc, class TCommonFunc> bool PushHandler(TEventInfo&& eventInfo, const TFunc& specific, const TCommonFunc& common) { if (specific) { PushSpecificHandler<TEventType>(std::move(eventInfo), specific); return true; - } + } if (common) { PushCommonHandler(std::move(eventInfo), common); return true; } return false; - } + } template <class TEventType, class TFunc> void PushSpecificHandler(TEventInfo&& eventInfo, const TFunc& f) { @@ -320,17 +320,17 @@ protected: executor->Post(std::move(f)); } - const TSettings& Settings; - TEventInfo& EventInfo; - }; - + const TSettings& Settings; + TEventInfo& EventInfo; + }; + public: TBaseSessionEventsQueue(const TSettings& settings) : Settings(settings) , Waiter(NThreading::NewPromise<void>(), this) {} - + virtual ~TBaseSessionEventsQueue() = default; @@ -338,11 +338,11 @@ public: CondVar.Signal(); } -protected: +protected: virtual bool HasEventsImpl() const { // Assumes that we're under lock. - return !Events.empty() || CloseEvent; - } - + return !Events.empty() || CloseEvent; + } + TWaiter PopWaiterImpl() { // Assumes that we're under lock. TWaiter waiter(Waiter.ExtractPromise(), this); return std::move(waiter); @@ -353,7 +353,7 @@ protected: CondVar.WaitI(Mutex); } } - + void RenewWaiterImpl() { if (Events.empty() && Waiter.GetFuture().HasValue()) { Waiter = TWaiter(NThreading::NewPromise<void>(), this); @@ -373,41 +373,41 @@ public: } } -protected: - const TSettings& Settings; +protected: + const TSettings& Settings; TWaiter Waiter; - std::queue<TEventInfo> Events; - TCondVar CondVar; - TMutex Mutex; - TMaybe<TSessionClosedEvent> CloseEvent; - std::atomic<bool> Closed = false; -}; - + std::queue<TEventInfo> Events; + TCondVar CondVar; + TMutex Mutex; + TMaybe<TSessionClosedEvent> CloseEvent; + std::atomic<bool> Closed = false; +}; + class IAsyncExecutor : public IExecutor { -private: - virtual void PostImpl(TVector<std::function<void()>>&&) = 0; - virtual void PostImpl(std::function<void()>&&) = 0; - -public: +private: + virtual void PostImpl(TVector<std::function<void()>>&&) = 0; + virtual void PostImpl(std::function<void()>&&) = 0; + +public: bool IsAsync() const override { - return true; - } - // Post Implementation MUST NOT run f before it returns - void Post(TFunction&& f) final; -}; - + return true; + } + // Post Implementation MUST NOT run f before it returns + void Post(TFunction&& f) final; +}; + IExecutor::TPtr CreateDefaultExecutor(); - - -class TThreadPoolExecutor : public IAsyncExecutor { -private: + + +class TThreadPoolExecutor : public IAsyncExecutor { +private: std::shared_ptr<IThreadPool> ThreadPool; - -public: + +public: TThreadPoolExecutor(std::shared_ptr<IThreadPool> threadPool); TThreadPoolExecutor(size_t threadsCount); - ~TThreadPoolExecutor() = default; - + ~TThreadPoolExecutor() = default; + bool IsAsync() const override { return !IsFakeThreadPool; } @@ -418,44 +418,44 @@ public: } } -private: - void PostImpl(TVector<TFunction>&& fs) override; - void PostImpl(TFunction&& f) override; +private: + void PostImpl(TVector<TFunction>&& fs) override; + void PostImpl(TFunction&& f) override; private: bool IsFakeThreadPool = false; size_t ThreadsCount = 0; -}; - -class TSerialExecutor : public IAsyncExecutor, public std::enable_shared_from_this<TSerialExecutor> { -private: - IAsyncExecutor::TPtr Executor; //!< Wrapped executor that is actually doing the job - bool Busy = false; //!< Set if some closure was scheduled for execution and did not finish yet - TMutex Mutex = {}; - TQueue<TFunction> ExecutionQueue = {}; - -public: - TSerialExecutor(IAsyncExecutor::TPtr executor); - ~TSerialExecutor() = default; - -private: - void PostImpl(TVector<TFunction>&& fs) override; - void PostImpl(TFunction&& f) override; - void PostNext(); -}; - +}; + +class TSerialExecutor : public IAsyncExecutor, public std::enable_shared_from_this<TSerialExecutor> { +private: + IAsyncExecutor::TPtr Executor; //!< Wrapped executor that is actually doing the job + bool Busy = false; //!< Set if some closure was scheduled for execution and did not finish yet + TMutex Mutex = {}; + TQueue<TFunction> ExecutionQueue = {}; + +public: + TSerialExecutor(IAsyncExecutor::TPtr executor); + ~TSerialExecutor() = default; + +private: + void PostImpl(TVector<TFunction>&& fs) override; + void PostImpl(TFunction&& f) override; + void PostNext(); +}; + class TSyncExecutor : public IExecutor { -public: - void Post(TFunction&& f) final { - return f(); - } +public: + void Post(TFunction&& f) final { + return f(); + } bool IsAsync() const final { - return false; - } + return false; + } void DoStart() override { } -}; - +}; + IExecutor::TPtr CreateGenericExecutor(); - + } // namespace NYdb::NPersQueue diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue.cpp b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue.cpp index 673d4bc7fb..a22ed23cea 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue.cpp +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue.cpp @@ -380,13 +380,13 @@ std::shared_ptr<IReadSession> TPersQueueClient::CreateReadSession(const TReadSes } std::shared_ptr<IWriteSession> TPersQueueClient::CreateWriteSession(const TWriteSessionSettings& settings) { - return Impl_->CreateWriteSession(settings); -} + return Impl_->CreateWriteSession(settings); +} std::shared_ptr<ISimpleBlockingWriteSession> TPersQueueClient::CreateSimpleBlockingWriteSession( - const TWriteSessionSettings& settings -) { - return Impl_->CreateSimpleWriteSession(settings); -} + const TWriteSessionSettings& settings +) { + return Impl_->CreateSimpleWriteSession(settings); +} } // namespace NYdb::NPersQueue diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue_impl.cpp b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue_impl.cpp index 9541e92152..bd466fd6f3 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue_impl.cpp +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue_impl.cpp @@ -23,8 +23,8 @@ std::shared_ptr<IReadSession> TPersQueueClient::TImpl::CreateReadSession(const T } std::shared_ptr<IWriteSession> TPersQueueClient::TImpl::CreateWriteSession( - const TWriteSessionSettings& settings -) { + const TWriteSessionSettings& settings +) { TMaybe<TWriteSessionSettings> maybeSettings; if (!settings.CompressionExecutor_ || !settings.EventHandlers_.HandlersExecutor_ || !settings.ClusterDiscoveryMode_) { maybeSettings = settings; @@ -40,30 +40,30 @@ std::shared_ptr<IWriteSession> TPersQueueClient::TImpl::CreateWriteSession( } } } - auto session = std::make_shared<TWriteSession>( + auto session = std::make_shared<TWriteSession>( maybeSettings.GetOrElse(settings), shared_from_this(), Connections_, DbDriverState_ - ); - session->Start(TDuration::Zero()); - return std::move(session); -} - + ); + session->Start(TDuration::Zero()); + return std::move(session); +} + std::shared_ptr<ISimpleBlockingWriteSession> TPersQueueClient::TImpl::CreateSimpleWriteSession( - const TWriteSessionSettings& settings -) { - auto alteredSettings = settings; - with_lock (Lock) { - alteredSettings.EventHandlers_.HandlersExecutor(Settings.DefaultHandlersExecutor_); - if (!settings.CompressionExecutor_) { - alteredSettings.CompressionExecutor(Settings.DefaultCompressionExecutor_); - } - } - - auto session = std::make_shared<TSimpleBlockingWriteSession>( + const TWriteSessionSettings& settings +) { + auto alteredSettings = settings; + with_lock (Lock) { + alteredSettings.EventHandlers_.HandlersExecutor(Settings.DefaultHandlersExecutor_); + if (!settings.CompressionExecutor_) { + alteredSettings.CompressionExecutor(Settings.DefaultCompressionExecutor_); + } + } + + auto session = std::make_shared<TSimpleBlockingWriteSession>( alteredSettings, shared_from_this(), Connections_, DbDriverState_ - ); - return std::move(session); -} - + ); + return std::move(session); +} + std::shared_ptr<TPersQueueClient::TImpl> TPersQueueClient::TImpl::GetClientForEndpoint(const TString& clusterEndoint) { with_lock (Lock) { Y_VERIFY(!CustomEndpoint); @@ -83,10 +83,10 @@ std::shared_ptr<TPersQueueClient::TImpl::IReadSessionConnectionProcessorFactory> } std::shared_ptr<TPersQueueClient::TImpl::IWriteSessionConnectionProcessorFactory> TPersQueueClient::TImpl::CreateWriteSessionConnectionProcessorFactory() { - using TService = Ydb::PersQueue::V1::PersQueueService; - using TRequest = Ydb::PersQueue::V1::StreamingWriteClientMessage; - using TResponse = Ydb::PersQueue::V1::StreamingWriteServerMessage; - return CreateConnectionProcessorFactory<TService, TRequest, TResponse>(&TService::Stub::AsyncStreamingWrite, Connections_, DbDriverState_); -} - + using TService = Ydb::PersQueue::V1::PersQueueService; + using TRequest = Ydb::PersQueue::V1::StreamingWriteClientMessage; + using TResponse = Ydb::PersQueue::V1::StreamingWriteServerMessage; + return CreateConnectionProcessorFactory<TService, TRequest, TResponse>(&TService::Stub::AsyncStreamingWrite, Connections_, DbDriverState_); +} + } // namespace NYdb::NPersQueue diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue_impl.h b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue_impl.h index f97469ac96..57e9dd43bf 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue_impl.h +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue_impl.h @@ -195,8 +195,8 @@ public: // Runtime API. std::shared_ptr<IReadSession> CreateReadSession(const TReadSessionSettings& settings); - std::shared_ptr<ISimpleBlockingWriteSession> CreateSimpleWriteSession(const TWriteSessionSettings& settings); - std::shared_ptr<IWriteSession> CreateWriteSession(const TWriteSessionSettings& settings); + std::shared_ptr<ISimpleBlockingWriteSession> CreateSimpleWriteSession(const TWriteSessionSettings& settings); + std::shared_ptr<IWriteSession> CreateWriteSession(const TWriteSessionSettings& settings); std::shared_ptr<TImpl> GetClientForEndpoint(const TString& clusterEndoint); @@ -204,12 +204,12 @@ public: std::shared_ptr<IReadSessionConnectionProcessorFactory> CreateReadSessionConnectionProcessorFactory(); - using IWriteSessionConnectionProcessorFactory = ISessionConnectionProcessorFactory< - Ydb::PersQueue::V1::StreamingWriteClientMessage, - Ydb::PersQueue::V1::StreamingWriteServerMessage>; - - std::shared_ptr<IWriteSessionConnectionProcessorFactory> CreateWriteSessionConnectionProcessorFactory(); - + using IWriteSessionConnectionProcessorFactory = ISessionConnectionProcessorFactory< + Ydb::PersQueue::V1::StreamingWriteClientMessage, + Ydb::PersQueue::V1::StreamingWriteServerMessage>; + + std::shared_ptr<IWriteSessionConnectionProcessorFactory> CreateWriteSessionConnectionProcessorFactory(); + NGrpc::IQueueClientContextPtr CreateContext() { return Connections_->CreateContext(); } diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/read_session.h b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/read_session.h index 7d77159d75..b5b874954f 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/read_session.h +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/read_session.h @@ -103,7 +103,7 @@ public: std::weak_ptr<TSingleClusterReadSessionImpl> session, bool doDecompress ); - + i64 StartDecompressionTasks(const IExecutor::TPtr& executor, i64 availableMemory, double averageCompressionRatio, diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/write_session.cpp b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/write_session.cpp index a469a2a8be..af4e08b979 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/write_session.cpp +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/write_session.cpp @@ -4,354 +4,354 @@ #include <util/generic/store_policy.h> #include <util/generic/utility.h> -#include <util/stream/buffer.h> - +#include <util/stream/buffer.h> + + +namespace NYdb::NPersQueue { +using NMonitoring::TDynamicCounterPtr; +using TCounterPtr = NMonitoring::TDynamicCounters::TCounterPtr; -namespace NYdb::NPersQueue { -using NMonitoring::TDynamicCounterPtr; -using TCounterPtr = NMonitoring::TDynamicCounters::TCounterPtr; - const TDuration UPDATE_TOKEN_PERIOD = TDuration::Hours(1); namespace NCompressionDetails { THolder<IOutputStream> CreateCoder(ECodec codec, TBuffer& result, int quality); -} - -#define HISTOGRAM_SETUP NMonitoring::ExplicitHistogram({0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100}) +} + +#define HISTOGRAM_SETUP NMonitoring::ExplicitHistogram({0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100}) TWriterCounters::TWriterCounters(const TIntrusivePtr<NMonitoring::TDynamicCounters>& counters) { - Errors = counters->GetCounter("errors", true); - CurrentSessionLifetimeMs = counters->GetCounter("currentSessionLifetimeMs", false); - BytesWritten = counters->GetCounter("bytesWritten", true); - MessagesWritten = counters->GetCounter("messagesWritten", true); - BytesWrittenCompressed = counters->GetCounter("bytesWrittenCompressed", true); - BytesInflightUncompressed = counters->GetCounter("bytesInflightUncompressed", false); - BytesInflightCompressed = counters->GetCounter("bytesInflightCompressed", false); - BytesInflightTotal = counters->GetCounter("bytesInflightTotal", false); - MessagesInflight = counters->GetCounter("messagesInflight", false); - - TotalBytesInflightUsageByTime = counters->GetHistogram("totalBytesInflightUsageByTime", HISTOGRAM_SETUP); - UncompressedBytesInflightUsageByTime = counters->GetHistogram("uncompressedBytesInflightUsageByTime", HISTOGRAM_SETUP); - CompressedBytesInflightUsageByTime = counters->GetHistogram("compressedBytesInflightUsageByTime", HISTOGRAM_SETUP); -} -#undef HISTOGRAM_SETUP - -TWriteSession::TWriteSession( - const TWriteSessionSettings& settings, + Errors = counters->GetCounter("errors", true); + CurrentSessionLifetimeMs = counters->GetCounter("currentSessionLifetimeMs", false); + BytesWritten = counters->GetCounter("bytesWritten", true); + MessagesWritten = counters->GetCounter("messagesWritten", true); + BytesWrittenCompressed = counters->GetCounter("bytesWrittenCompressed", true); + BytesInflightUncompressed = counters->GetCounter("bytesInflightUncompressed", false); + BytesInflightCompressed = counters->GetCounter("bytesInflightCompressed", false); + BytesInflightTotal = counters->GetCounter("bytesInflightTotal", false); + MessagesInflight = counters->GetCounter("messagesInflight", false); + + TotalBytesInflightUsageByTime = counters->GetHistogram("totalBytesInflightUsageByTime", HISTOGRAM_SETUP); + UncompressedBytesInflightUsageByTime = counters->GetHistogram("uncompressedBytesInflightUsageByTime", HISTOGRAM_SETUP); + CompressedBytesInflightUsageByTime = counters->GetHistogram("compressedBytesInflightUsageByTime", HISTOGRAM_SETUP); +} +#undef HISTOGRAM_SETUP + +TWriteSession::TWriteSession( + const TWriteSessionSettings& settings, std::shared_ptr<TPersQueueClient::TImpl> client, - std::shared_ptr<TGRpcConnectionsImpl> connections, - TDbDriverStatePtr dbDriverState) - : Settings(settings) - , Client(std::move(client)) - , Connections(std::move(connections)) - , DbDriverState(std::move(dbDriverState)) + std::shared_ptr<TGRpcConnectionsImpl> connections, + TDbDriverStatePtr dbDriverState) + : Settings(settings) + , Client(std::move(client)) + , Connections(std::move(connections)) + , DbDriverState(std::move(dbDriverState)) , PrevToken(DbDriverState->CredentialsProvider ? DbDriverState->CredentialsProvider->GetAuthInfo() : "") , EventsQueue(std::make_shared<TWriteSessionEventsQueue>(Settings)) , InitSeqNoPromise(NThreading::NewPromise<ui64>()) - , WakeupInterval( - Settings.BatchFlushInterval_.GetOrElse(TDuration::Zero()) ? - std::min(Settings.BatchFlushInterval_.GetOrElse(TDuration::Seconds(1)) / 5, TDuration::MilliSeconds(100)) - : - TDuration::MilliSeconds(100) - ) -{ - if (!Settings.RetryPolicy_) { - Settings.RetryPolicy_ = IRetryPolicy::GetDefaultPolicy(); - } + , WakeupInterval( + Settings.BatchFlushInterval_.GetOrElse(TDuration::Zero()) ? + std::min(Settings.BatchFlushInterval_.GetOrElse(TDuration::Seconds(1)) / 5, TDuration::MilliSeconds(100)) + : + TDuration::MilliSeconds(100) + ) +{ + if (!Settings.RetryPolicy_) { + Settings.RetryPolicy_ = IRetryPolicy::GetDefaultPolicy(); + } if (Settings.PreferredCluster_ && !Settings.AllowFallbackToOtherClusters_) { TargetCluster = *Settings.PreferredCluster_; - TargetCluster.to_lower(); - } - if (Settings.Counters_.Defined()) { - Counters = *Settings.Counters_; - } else { - Counters = MakeIntrusive<TWriterCounters>(new NMonitoring::TDynamicCounters()); - } - -} - -void TWriteSession::Start(const TDuration& delay) { - ++ConnectionAttemptsDone; - if (!Started) { - HandleWakeUpImpl(); - InitWriter(); - } - Started = true; + TargetCluster.to_lower(); + } + if (Settings.Counters_.Defined()) { + Counters = *Settings.Counters_; + } else { + Counters = MakeIntrusive<TWriterCounters>(new NMonitoring::TDynamicCounters()); + } + +} + +void TWriteSession::Start(const TDuration& delay) { + ++ConnectionAttemptsDone; + if (!Started) { + HandleWakeUpImpl(); + InitWriter(); + } + Started = true; DoCdsRequest(delay); -} - -// Only called under lock -TWriteSession::THandleResult TWriteSession::RestartImpl(const TPlainStatus& status) { - THandleResult result; - if (AtomicGet(Aborting)) { +} + +// Only called under lock +TWriteSession::THandleResult TWriteSession::RestartImpl(const TPlainStatus& status) { + THandleResult result; + if (AtomicGet(Aborting)) { DbDriverState->Log << TLOG_DEBUG << LogPrefix() << "Write session is aborting and will not restart"; - return result; - } + return result; + } DbDriverState->Log << TLOG_INFO << LogPrefix() << "Got error. Status: " << status.Status << ". Description: " << IssuesSingleLineString(status.Issues); - SessionEstablished = false; - TMaybe<TDuration> nextDelay = TDuration::Zero(); - if (!RetryState) { - RetryState = Settings.RetryPolicy_->CreateRetryState(); - } + SessionEstablished = false; + TMaybe<TDuration> nextDelay = TDuration::Zero(); + if (!RetryState) { + RetryState = Settings.RetryPolicy_->CreateRetryState(); + } nextDelay = RetryState->GetNextRetryDelay(TPlainStatus(status)); - - if (nextDelay) { - result.StartDelay = *nextDelay; - result.DoRestart = true; + + if (nextDelay) { + result.StartDelay = *nextDelay; + result.DoRestart = true; DbDriverState->Log << TLOG_DEBUG << LogPrefix() << "Write session will restart in " << result.StartDelay.MilliSeconds() << " ms"; - ResetForRetryImpl(); - - } else { + ResetForRetryImpl(); + + } else { DbDriverState->Log << TLOG_INFO << LogPrefix() << "Write session will not restart after a fatal error"; - result.DoStop = true; - CheckHandleResultImpl(result); - } - return result; -} - + result.DoStop = true; + CheckHandleResultImpl(result); + } + return result; +} + bool IsFederation(const TString& endpoint) { TStringBuf host = GetHost(endpoint); return host == "logbroker.yandex.net" || host == "logbroker-prestable.yandex.net"; } -void TWriteSession::DoCdsRequest(TDuration delay) { +void TWriteSession::DoCdsRequest(TDuration delay) { DbDriverState->Log << TLOG_INFO << LogPrefix() << "Write session: Do CDS request"; auto weakThis = weak_from_this(); - - if ( - Settings.ClusterDiscoveryMode_ == EClusterDiscoveryMode::Off || - (Settings.ClusterDiscoveryMode_ == EClusterDiscoveryMode::Auto && !IsFederation(DbDriverState->DiscoveryEndpoint)) - ) { + + if ( + Settings.ClusterDiscoveryMode_ == EClusterDiscoveryMode::Off || + (Settings.ClusterDiscoveryMode_ == EClusterDiscoveryMode::Auto && !IsFederation(DbDriverState->DiscoveryEndpoint)) + ) { DoConnect(delay, DbDriverState->DiscoveryEndpoint); return; } auto extractor = [weakThis] - (google::protobuf::Any* any, TPlainStatus status) mutable { - Ydb::PersQueue::ClusterDiscovery::DiscoverClustersResult result; - if (any) { - any->UnpackTo(&result); - } + (google::protobuf::Any* any, TPlainStatus status) mutable { + Ydb::PersQueue::ClusterDiscovery::DiscoverClustersResult result; + if (any) { + any->UnpackTo(&result); + } TStatus st(std::move(status)); if (auto sharedThis = weakThis.lock()) { sharedThis->OnCdsResponse(st, result); } - }; - - Ydb::PersQueue::ClusterDiscovery::DiscoverClustersRequest req; - auto* params = req.add_write_sessions(); - params->set_topic(Settings.Path_); - params->set_source_id(Settings.MessageGroupId_); - if (Settings.PartitionGroupId_.Defined()) - params->set_partition_group(*Settings.PartitionGroupId_); + }; + + Ydb::PersQueue::ClusterDiscovery::DiscoverClustersRequest req; + auto* params = req.add_write_sessions(); + params->set_topic(Settings.Path_); + params->set_source_id(Settings.MessageGroupId_); + if (Settings.PartitionGroupId_.Defined()) + params->set_partition_group(*Settings.PartitionGroupId_); if (Settings.PreferredCluster_.Defined()) params->set_preferred_cluster_name(*Settings.PreferredCluster_); - + auto weakConnections = std::weak_ptr<TGRpcConnectionsImpl>(Connections); DbDriverState->Log << TLOG_INFO << LogPrefix() << "Do schedule cds request after " << delay.MilliSeconds() << " ms\n"; auto cdsRequestCall = [req_=std::move(req), extr=std::move(extractor), weakConnections, dbState=DbDriverState, settings=Settings]() mutable { if (auto connections = weakConnections.lock()) { dbState->Log << TLOG_INFO << "MessageGroupId [" << settings.MessageGroupId_ << "] Running cds request ms\n"; connections->RunDeferred<Ydb::PersQueue::V1::ClusterDiscoveryService, - Ydb::PersQueue::ClusterDiscovery::DiscoverClustersRequest, - Ydb::PersQueue::ClusterDiscovery::DiscoverClustersResponse>( - std::move(req_), - std::move(extr), - &Ydb::PersQueue::V1::ClusterDiscoveryService::Stub::AsyncDiscoverClusters, - dbState, - INITIAL_DEFERRED_CALL_DELAY, - TRpcRequestSettings::Make(settings), + Ydb::PersQueue::ClusterDiscovery::DiscoverClustersRequest, + Ydb::PersQueue::ClusterDiscovery::DiscoverClustersResponse>( + std::move(req_), + std::move(extr), + &Ydb::PersQueue::V1::ClusterDiscoveryService::Stub::AsyncDiscoverClusters, + dbState, + INITIAL_DEFERRED_CALL_DELAY, + TRpcRequestSettings::Make(settings), settings.ConnectTimeout_); // TODO: make client timeout setting } - }; - Connections->ScheduleOneTimeTask(std::move(cdsRequestCall), delay); -} - -void TWriteSession::OnCdsResponse( - TStatus& status, const Ydb::PersQueue::ClusterDiscovery::DiscoverClustersResult& result -) { + }; + Connections->ScheduleOneTimeTask(std::move(cdsRequestCall), delay); +} + +void TWriteSession::OnCdsResponse( + TStatus& status, const Ydb::PersQueue::ClusterDiscovery::DiscoverClustersResult& result +) { DbDriverState->Log << TLOG_INFO << LogPrefix() << "Got CDS response: \n" << result.ShortDebugString(); - TString endpoint, name; - THandleResult handleResult; - if (!status.IsSuccess()) { - with_lock (Lock) { - handleResult = OnErrorImpl({ - status.GetStatus(), - MakeIssueWithSubIssues("Failed to discover clusters", status.GetIssues()) - }); - } - ProcessHandleResult(handleResult); - return; - } - - NYql::TIssues issues; - EStatus errorStatus = EStatus::INTERNAL_ERROR; - with_lock (Lock) { - const Ydb::PersQueue::ClusterDiscovery::WriteSessionClusters& wsClusters = result.write_sessions_clusters(0); + TString endpoint, name; + THandleResult handleResult; + if (!status.IsSuccess()) { + with_lock (Lock) { + handleResult = OnErrorImpl({ + status.GetStatus(), + MakeIssueWithSubIssues("Failed to discover clusters", status.GetIssues()) + }); + } + ProcessHandleResult(handleResult); + return; + } + + NYql::TIssues issues; + EStatus errorStatus = EStatus::INTERNAL_ERROR; + with_lock (Lock) { + const Ydb::PersQueue::ClusterDiscovery::WriteSessionClusters& wsClusters = result.write_sessions_clusters(0); bool isFirst = true; - for (const auto& clusterInfo : wsClusters.clusters()) { - TString normalizedName = clusterInfo.name(); - normalizedName.to_lower(); - + for (const auto& clusterInfo : wsClusters.clusters()) { + TString normalizedName = clusterInfo.name(); + normalizedName.to_lower(); + if(isFirst) { isFirst = false; PreferredClusterByCDS = clusterInfo.name(); } - if (!clusterInfo.available()) { - if (TargetCluster && TargetCluster == normalizedName) { - errorStatus = EStatus::UNAVAILABLE; - issues.AddIssue(TStringBuilder() << "Selected destination cluster: " << normalizedName - << " is currently disabled"); - break; - } - continue; - } - if (clusterInfo.endpoint().empty()) { - issues.AddIssue(TStringBuilder() << "Unexpected reply from cluster discovery. Empty endpoint for cluster " - << normalizedName); - } else { - name = clusterInfo.name(); + if (!clusterInfo.available()) { + if (TargetCluster && TargetCluster == normalizedName) { + errorStatus = EStatus::UNAVAILABLE; + issues.AddIssue(TStringBuilder() << "Selected destination cluster: " << normalizedName + << " is currently disabled"); + break; + } + continue; + } + if (clusterInfo.endpoint().empty()) { + issues.AddIssue(TStringBuilder() << "Unexpected reply from cluster discovery. Empty endpoint for cluster " + << normalizedName); + } else { + name = clusterInfo.name(); endpoint = ApplyClusterEndpoint(DbDriverState->DiscoveryEndpoint, clusterInfo.endpoint()); - break; + break; } } - if (endpoint.empty()) { - errorStatus = EStatus::GENERIC_ERROR; - issues.AddIssue(TStringBuilder() << "Could not get valid endpoint from cluster discovery"); - } - } - if (issues) { - with_lock(Lock) { - handleResult = OnErrorImpl({errorStatus, std::move(issues)}); - } - ProcessHandleResult(handleResult); - return; - } - with_lock(Lock) { - if (!InitialCluster) { - InitialCluster = name; - } else if (CurrentCluster != name) { // Switched to another cluster - Y_VERIFY(CurrentCluster); - if (name == InitialCluster) { // Returned to initial cluster, disabled SeqNo Shift - SeqNoShift = 0; - OnSeqNoShift = false; - } else { // Switched from initial cluster to second one; - Y_VERIFY(CurrentCluster == InitialCluster); - if (!AutoSeqNoMode.Defined() || !(*AutoSeqNoMode)) - OnSeqNoShift = true; - } - - } - CurrentCluster = name; - } + if (endpoint.empty()) { + errorStatus = EStatus::GENERIC_ERROR; + issues.AddIssue(TStringBuilder() << "Could not get valid endpoint from cluster discovery"); + } + } + if (issues) { + with_lock(Lock) { + handleResult = OnErrorImpl({errorStatus, std::move(issues)}); + } + ProcessHandleResult(handleResult); + return; + } + with_lock(Lock) { + if (!InitialCluster) { + InitialCluster = name; + } else if (CurrentCluster != name) { // Switched to another cluster + Y_VERIFY(CurrentCluster); + if (name == InitialCluster) { // Returned to initial cluster, disabled SeqNo Shift + SeqNoShift = 0; + OnSeqNoShift = false; + } else { // Switched from initial cluster to second one; + Y_VERIFY(CurrentCluster == InitialCluster); + if (!AutoSeqNoMode.Defined() || !(*AutoSeqNoMode)) + OnSeqNoShift = true; + } + + } + CurrentCluster = name; + } DoConnect(TDuration::Zero(), endpoint); - -} - -void TWriteSession::InitWriter() { // No Lock, very initial start - no race yet as well. - CompressionExecutor = Settings.CompressionExecutor_; - IExecutor::TPtr executor; - executor = new TSyncExecutor(); - executor->Start(); - Executor = std::move(executor); - - Settings.CompressionExecutor_->Start(); - Settings.EventHandlers_.HandlersExecutor_->Start(); - -} -// Client method -NThreading::TFuture<ui64> TWriteSession::GetInitSeqNo() { + +} + +void TWriteSession::InitWriter() { // No Lock, very initial start - no race yet as well. + CompressionExecutor = Settings.CompressionExecutor_; + IExecutor::TPtr executor; + executor = new TSyncExecutor(); + executor->Start(); + Executor = std::move(executor); + + Settings.CompressionExecutor_->Start(); + Settings.EventHandlers_.HandlersExecutor_->Start(); + +} +// Client method +NThreading::TFuture<ui64> TWriteSession::GetInitSeqNo() { if (Settings.ValidateSeqNo_) { - if (AutoSeqNoMode.Defined() && *AutoSeqNoMode) { + if (AutoSeqNoMode.Defined() && *AutoSeqNoMode) { DbDriverState->Log << TLOG_ERR << LogPrefix() << "Cannot call GetInitSeqNo in Auto SeqNo mode"; ThrowFatalError("Cannot call GetInitSeqNo in Auto SeqNo mode"); - } + } else AutoSeqNoMode = false; } - return InitSeqNoPromise.GetFuture(); -} - + return InitSeqNoPromise.GetFuture(); +} + TString DebugString(const TWriteSessionEvent::TEvent& event) { return std::visit([](const auto& ev) { return ev.DebugString(); }, event); } -// Client method -TMaybe<TWriteSessionEvent::TEvent> TWriteSession::GetEvent(bool block) { - return EventsQueue->GetEvent(block); -} - -// Client method -TVector<TWriteSessionEvent::TEvent> TWriteSession::GetEvents(bool block, TMaybe<size_t> maxEventsCount) { +// Client method +TMaybe<TWriteSessionEvent::TEvent> TWriteSession::GetEvent(bool block) { + return EventsQueue->GetEvent(block); +} + +// Client method +TVector<TWriteSessionEvent::TEvent> TWriteSession::GetEvents(bool block, TMaybe<size_t> maxEventsCount) { return EventsQueue->GetEvents(block, maxEventsCount); -} - -// Only called under lock -ui64 TWriteSession::GetNextSeqNoImpl(const TMaybe<ui64>& seqNo) { - ui64 seqNoValue = LastSeqNo + 1; - if (!AutoSeqNoMode.Defined()) { - AutoSeqNoMode = !seqNo.Defined(); - //! Disable SeqNo shift for manual SeqNo mode; - if (seqNo.Defined()) { - OnSeqNoShift = false; - SeqNoShift = 0; - } +} + +// Only called under lock +ui64 TWriteSession::GetNextSeqNoImpl(const TMaybe<ui64>& seqNo) { + ui64 seqNoValue = LastSeqNo + 1; + if (!AutoSeqNoMode.Defined()) { + AutoSeqNoMode = !seqNo.Defined(); + //! Disable SeqNo shift for manual SeqNo mode; + if (seqNo.Defined()) { + OnSeqNoShift = false; + SeqNoShift = 0; + } } if (seqNo.Defined()) { if (*AutoSeqNoMode) { DbDriverState->Log << TLOG_ERR << LogPrefix() << "Cannot call write() with defined SeqNo on WriteSession running in auto-seqNo mode"; ThrowFatalError( - "Cannot call write() with defined SeqNo on WriteSession running in auto-seqNo mode" - ); - } else { - seqNoValue = *seqNo; - } + "Cannot call write() with defined SeqNo on WriteSession running in auto-seqNo mode" + ); + } else { + seqNoValue = *seqNo; + } } else if (!(*AutoSeqNoMode)) { DbDriverState->Log << TLOG_ERR << LogPrefix() << "Cannot call write() without defined SeqNo on WriteSession running in manual-seqNo mode"; ThrowFatalError( - "Cannot call write() without defined SeqNo on WriteSession running in manual-seqNo mode" - ); - } - LastSeqNo = seqNoValue; - return seqNoValue; -} -inline void TWriteSession::CheckHandleResultImpl(THandleResult& result) { - result.DoSetSeqNo = result.DoStop && !InitSeqNoSetDone && (InitSeqNoSetDone = true); -} - -void TWriteSession::ProcessHandleResult(THandleResult& result) { - if (result.DoRestart) { - Start(result.StartDelay); - } else if (result.DoSetSeqNo) { - InitSeqNoPromise.SetException("session closed"); - } -} - -NThreading::TFuture<void> TWriteSession::WaitEvent() { - return EventsQueue->WaitEvent(); -} - -// Client method. + "Cannot call write() without defined SeqNo on WriteSession running in manual-seqNo mode" + ); + } + LastSeqNo = seqNoValue; + return seqNoValue; +} +inline void TWriteSession::CheckHandleResultImpl(THandleResult& result) { + result.DoSetSeqNo = result.DoStop && !InitSeqNoSetDone && (InitSeqNoSetDone = true); +} + +void TWriteSession::ProcessHandleResult(THandleResult& result) { + if (result.DoRestart) { + Start(result.StartDelay); + } else if (result.DoSetSeqNo) { + InitSeqNoPromise.SetException("session closed"); + } +} + +NThreading::TFuture<void> TWriteSession::WaitEvent() { + return EventsQueue->WaitEvent(); +} + +// Client method. void TWriteSession::WriteInternal( TContinuationToken&&, TStringBuf data, TMaybe<ECodec> codec, ui32 originalSize, TMaybe<ui64> seqNo, TMaybe<TInstant> createTimestamp ) { - TInstant createdAtValue = createTimestamp.Defined() ? *createTimestamp : TInstant::Now(); - bool readyToAccept = false; - size_t bufferSize = data.size(); - with_lock(Lock) { + TInstant createdAtValue = createTimestamp.Defined() ? *createTimestamp : TInstant::Now(); + bool readyToAccept = false; + size_t bufferSize = data.size(); + with_lock(Lock) { CurrentBatch.Add(GetNextSeqNoImpl(seqNo), createdAtValue, data, codec, originalSize); - - FlushWriteIfRequiredImpl(); - readyToAccept = OnMemoryUsageChangedImpl(bufferSize).NowOk; - } - if (readyToAccept) { - EventsQueue->PushEvent(TWriteSessionEvent::TReadyToAcceptEvent{TContinuationToken{}}); - } -} - + + FlushWriteIfRequiredImpl(); + readyToAccept = OnMemoryUsageChangedImpl(bufferSize).NowOk; + } + if (readyToAccept) { + EventsQueue->PushEvent(TWriteSessionEvent::TReadyToAcceptEvent{TContinuationToken{}}); + } +} + // Client method. void TWriteSession::WriteEncoded( TContinuationToken&& token, TStringBuf data, ECodec codec, ui32 originalSize, TMaybe<ui64> seqNo, TMaybe<TInstant> createTimestamp @@ -366,275 +366,275 @@ void TWriteSession::Write( } -// Only called under lock. -TWriteSession::THandleResult TWriteSession::OnErrorImpl(NYdb::TPlainStatus&& status) { - (*Counters->Errors)++; - auto result = RestartImpl(status); - if (result.DoStop) { - CloseImpl(status.Status, std::move(status.Issues)); - } - return result; -} - -// No lock +// Only called under lock. +TWriteSession::THandleResult TWriteSession::OnErrorImpl(NYdb::TPlainStatus&& status) { + (*Counters->Errors)++; + auto result = RestartImpl(status); + if (result.DoStop) { + CloseImpl(status.Status, std::move(status.Issues)); + } + return result; +} + +// No lock void TWriteSession::DoConnect(const TDuration& delay, const TString& endpoint) { DbDriverState->Log << TLOG_INFO << LogPrefix() << "Start write session. Will connect to endpoint: " << endpoint; - - NGrpc::IQueueClientContextPtr prevConnectContext; - NGrpc::IQueueClientContextPtr prevConnectTimeoutContext; - NGrpc::IQueueClientContextPtr prevConnectDelayContext; - NGrpc::IQueueClientContextPtr connectContext = nullptr; + + NGrpc::IQueueClientContextPtr prevConnectContext; + NGrpc::IQueueClientContextPtr prevConnectTimeoutContext; + NGrpc::IQueueClientContextPtr prevConnectDelayContext; + NGrpc::IQueueClientContextPtr connectContext = nullptr; NGrpc::IQueueClientContextPtr connectDelayContext = nullptr; - NGrpc::IQueueClientContextPtr connectTimeoutContext = nullptr; - TRpcRequestSettings reqSettings; - std::shared_ptr<IWriteSessionConnectionProcessorFactory> connectionFactory; - with_lock(Lock) { - ++ConnectionGeneration; - auto subclient = Client->GetClientForEndpoint(endpoint); - connectionFactory = subclient->CreateWriteSessionConnectionProcessorFactory(); - auto clientContext = subclient->CreateContext(); - ConnectionFactory = connectionFactory; - - ClientContext = std::move(clientContext); - ServerMessage = std::make_shared<TServerMessage>(); - + NGrpc::IQueueClientContextPtr connectTimeoutContext = nullptr; + TRpcRequestSettings reqSettings; + std::shared_ptr<IWriteSessionConnectionProcessorFactory> connectionFactory; + with_lock(Lock) { + ++ConnectionGeneration; + auto subclient = Client->GetClientForEndpoint(endpoint); + connectionFactory = subclient->CreateWriteSessionConnectionProcessorFactory(); + auto clientContext = subclient->CreateContext(); + ConnectionFactory = connectionFactory; + + ClientContext = std::move(clientContext); + ServerMessage = std::make_shared<TServerMessage>(); + if (!ClientContext) { AbortImpl(); // Grpc and WriteSession is closing right now. return; } - connectContext = ClientContext->CreateContext(); + connectContext = ClientContext->CreateContext(); if (delay) connectDelayContext = ClientContext->CreateContext(); - connectTimeoutContext = ClientContext->CreateContext(); - - // Previous operations contexts. - - // Set new context - prevConnectContext = std::exchange(ConnectContext, connectContext); - prevConnectTimeoutContext = std::exchange(ConnectTimeoutContext, connectTimeoutContext); + connectTimeoutContext = ClientContext->CreateContext(); + + // Previous operations contexts. + + // Set new context + prevConnectContext = std::exchange(ConnectContext, connectContext); + prevConnectTimeoutContext = std::exchange(ConnectTimeoutContext, connectTimeoutContext); prevConnectDelayContext = std::exchange(ConnectDelayContext, connectDelayContext); - Y_ASSERT(ConnectContext); - Y_ASSERT(ConnectTimeoutContext); - - // Cancel previous operations. - Cancel(prevConnectContext); + Y_ASSERT(ConnectContext); + Y_ASSERT(ConnectTimeoutContext); + + // Cancel previous operations. + Cancel(prevConnectContext); if (prevConnectDelayContext) Cancel(prevConnectDelayContext); - Cancel(prevConnectTimeoutContext); - Y_ASSERT(connectContext); - Y_ASSERT(connectTimeoutContext); - reqSettings = TRpcRequestSettings::Make(Settings); - } - auto connectCallback = [weakThis = weak_from_this(), connectContext = connectContext] - (TPlainStatus&& st, typename IProcessor::TPtr&& processor) { - if (auto sharedThis = weakThis.lock()) { - sharedThis->OnConnect(std::move(st), std::move(processor), connectContext); - } - }; - - auto connectTimeoutCallback = [weakThis = weak_from_this(), connectTimeoutContext = connectTimeoutContext] - (bool ok) { - if (ok) { - if (auto sharedThis = weakThis.lock()) { - sharedThis->OnConnectTimeout(connectTimeoutContext); - } - } - }; - - connectionFactory->CreateProcessor( - std::move(connectCallback), - reqSettings, - std::move(connectContext), - TDuration::Seconds(30) /* connect timeout */, // TODO: make connect timeout setting. - std::move(connectTimeoutContext), + Cancel(prevConnectTimeoutContext); + Y_ASSERT(connectContext); + Y_ASSERT(connectTimeoutContext); + reqSettings = TRpcRequestSettings::Make(Settings); + } + auto connectCallback = [weakThis = weak_from_this(), connectContext = connectContext] + (TPlainStatus&& st, typename IProcessor::TPtr&& processor) { + if (auto sharedThis = weakThis.lock()) { + sharedThis->OnConnect(std::move(st), std::move(processor), connectContext); + } + }; + + auto connectTimeoutCallback = [weakThis = weak_from_this(), connectTimeoutContext = connectTimeoutContext] + (bool ok) { + if (ok) { + if (auto sharedThis = weakThis.lock()) { + sharedThis->OnConnectTimeout(connectTimeoutContext); + } + } + }; + + connectionFactory->CreateProcessor( + std::move(connectCallback), + reqSettings, + std::move(connectContext), + TDuration::Seconds(30) /* connect timeout */, // TODO: make connect timeout setting. + std::move(connectTimeoutContext), std::move(connectTimeoutCallback), delay, std::move(connectDelayContext) - ); -} - -// RPC callback. -void TWriteSession::OnConnectTimeout(const NGrpc::IQueueClientContextPtr& connectTimeoutContext) { + ); +} + +// RPC callback. +void TWriteSession::OnConnectTimeout(const NGrpc::IQueueClientContextPtr& connectTimeoutContext) { DbDriverState->Log << TLOG_ERR << LogPrefix() << "Write session: connect timeout"; - THandleResult handleResult; - with_lock (Lock) { - if (ConnectTimeoutContext == connectTimeoutContext) { - Cancel(ConnectContext); - ConnectContext = nullptr; - ConnectTimeoutContext = nullptr; - ConnectDelayContext = nullptr; - } else { - return; - } + THandleResult handleResult; + with_lock (Lock) { + if (ConnectTimeoutContext == connectTimeoutContext) { + Cancel(ConnectContext); + ConnectContext = nullptr; + ConnectTimeoutContext = nullptr; + ConnectDelayContext = nullptr; + } else { + return; + } TStringBuilder description; description << "Failed to establish connection to server. Attempts done: " << ConnectionAttemptsDone; - handleResult = RestartImpl(TPlainStatus(EStatus::TIMEOUT, description)); - if (handleResult.DoStop) { - CloseImpl( - EStatus::TIMEOUT, + handleResult = RestartImpl(TPlainStatus(EStatus::TIMEOUT, description)); + if (handleResult.DoStop) { + CloseImpl( + EStatus::TIMEOUT, description - ); - } - } - ProcessHandleResult(handleResult); -} - -// RPC callback. -void TWriteSession::OnConnect( - TPlainStatus&& st, typename IProcessor::TPtr&& processor, const NGrpc::IQueueClientContextPtr& connectContext -) { - THandleResult handleResult; - with_lock (Lock) { - if (ConnectContext == connectContext) { - Cancel(ConnectTimeoutContext); - ConnectContext = nullptr; - ConnectTimeoutContext = nullptr; - ConnectDelayContext = nullptr; - - if (st.Ok()) { - Processor = std::move(processor); - InitImpl(); - // Still should call ReadFromProcessor(); - } - } else { - return; - } - if (!st.Ok()) { - handleResult = RestartImpl(st); - if (handleResult.DoStop) { - CloseImpl( - st.Status, - MakeIssueWithSubIssues( - TStringBuilder() << "Failed to establish connection to server \"" << st.Endpoint - << "\". Attempts done: " << ConnectionAttemptsDone, - st.Issues - ) - ); - } - } - } - if (st.Ok()) - ReadFromProcessor(); // Out of Init - ProcessHandleResult(handleResult); -} - -// Produce init request for session. -// Only called under lock. -void TWriteSession::InitImpl() { - Ydb::PersQueue::V1::StreamingWriteClientMessage req; - auto* init = req.mutable_init_request(); - init->set_topic(Settings.Path_); - init->set_message_group_id(Settings.MessageGroupId_); - if (Settings.PartitionGroupId_) { - init->set_partition_group_id(*Settings.PartitionGroupId_); - } - init->set_max_supported_block_format_version(0); + ); + } + } + ProcessHandleResult(handleResult); +} + +// RPC callback. +void TWriteSession::OnConnect( + TPlainStatus&& st, typename IProcessor::TPtr&& processor, const NGrpc::IQueueClientContextPtr& connectContext +) { + THandleResult handleResult; + with_lock (Lock) { + if (ConnectContext == connectContext) { + Cancel(ConnectTimeoutContext); + ConnectContext = nullptr; + ConnectTimeoutContext = nullptr; + ConnectDelayContext = nullptr; + + if (st.Ok()) { + Processor = std::move(processor); + InitImpl(); + // Still should call ReadFromProcessor(); + } + } else { + return; + } + if (!st.Ok()) { + handleResult = RestartImpl(st); + if (handleResult.DoStop) { + CloseImpl( + st.Status, + MakeIssueWithSubIssues( + TStringBuilder() << "Failed to establish connection to server \"" << st.Endpoint + << "\". Attempts done: " << ConnectionAttemptsDone, + st.Issues + ) + ); + } + } + } + if (st.Ok()) + ReadFromProcessor(); // Out of Init + ProcessHandleResult(handleResult); +} + +// Produce init request for session. +// Only called under lock. +void TWriteSession::InitImpl() { + Ydb::PersQueue::V1::StreamingWriteClientMessage req; + auto* init = req.mutable_init_request(); + init->set_topic(Settings.Path_); + init->set_message_group_id(Settings.MessageGroupId_); + if (Settings.PartitionGroupId_) { + init->set_partition_group_id(*Settings.PartitionGroupId_); + } + init->set_max_supported_block_format_version(0); init->set_preferred_cluster(PreferredClusterByCDS); - - for (const auto& attr : Settings.Meta_.Fields) { - (*init->mutable_session_meta())[attr.first] = attr.second; - } + + for (const auto& attr : Settings.Meta_.Fields) { + (*init->mutable_session_meta())[attr.first] = attr.second; + } DbDriverState->Log << TLOG_DEBUG << LogPrefix() << "Write session: send init request: "<< req.ShortDebugString(); - WriteToProcessorImpl(std::move(req)); -} - -// Called under lock. Invokes Processor->Write, which is assumed to be deadlock-safe -void TWriteSession::WriteToProcessorImpl(TWriteSession::TClientMessage&& req) { - Y_ASSERT(Processor); - if (Aborting) - return; - auto callback = [weakThis = weak_from_this(), connectionGeneration = ConnectionGeneration](NGrpc::TGrpcStatus&& grpcStatus) { - if (auto sharedThis = weakThis.lock()) { - sharedThis->OnWriteDone(std::move(grpcStatus), connectionGeneration); - } - }; - - Processor->Write(std::move(req), callback); -} - -void TWriteSession::ReadFromProcessor() { - Y_ASSERT(Processor); - IProcessor::TPtr prc; - ui64 generation; - with_lock(Lock) { - prc = Processor; - generation = ConnectionGeneration; - } - auto callback = [weakThis = weak_from_this(), connectionGeneration = generation, processor = prc, serverMessage = ServerMessage] - (NGrpc::TGrpcStatus&& grpcStatus) { - if (auto sharedThis = weakThis.lock()) { - sharedThis->OnReadDone(std::move(grpcStatus), connectionGeneration); - } - }; - prc->Read(ServerMessage.get(), std::move(callback)); -} - -void TWriteSession::OnWriteDone(NGrpc::TGrpcStatus&& status, size_t connectionGeneration) { - THandleResult handleResult; - with_lock (Lock) { - if (connectionGeneration != ConnectionGeneration) { - return; // Message from previous connection. Ignore. - } - if (Aborting) { - return; - } - if(!status.Ok()) { - handleResult = OnErrorImpl(status); - } - } - ProcessHandleResult(handleResult); -} - -void TWriteSession::OnReadDone(NGrpc::TGrpcStatus&& grpcStatus, size_t connectionGeneration) { - TPlainStatus errorStatus; - TProcessSrvMessageResult processResult; - bool needSetValue = false; - if (!grpcStatus.Ok()) { - errorStatus = TPlainStatus(std::move(grpcStatus)); - } - bool doRead = false; - with_lock (Lock) { - UpdateTimedCountersImpl(); - if (connectionGeneration != ConnectionGeneration) { - return; // Message from previous connection. Ignore. - } - if (errorStatus.Ok()) { - if (IsErrorMessage(*ServerMessage)) { - errorStatus = MakeErrorFromProto(*ServerMessage); - } else { - processResult = ProcessServerMessageImpl(); - needSetValue = !InitSeqNoSetDone && processResult.InitSeqNo.Defined() && (InitSeqNoSetDone = true); - if (errorStatus.Ok() && processResult.Ok) { - doRead = true; - } - } - } - } - if (doRead) - ReadFromProcessor(); - - with_lock(Lock) { - if (!errorStatus.Ok()) { - if (processResult.Ok) { // Otherwise, OnError was already called - processResult.HandleResult = RestartImpl(errorStatus); - } - } - if (processResult.HandleResult.DoStop) { - CloseImpl(std::move(errorStatus)); - } - } - for (auto& event : processResult.Events) { - EventsQueue->PushEvent(std::move(event)); - } - if (needSetValue) { - InitSeqNoPromise.SetValue(*processResult.InitSeqNo); - processResult.HandleResult.DoSetSeqNo = false; // Redundant. Just in case. - } - ProcessHandleResult(processResult.HandleResult); -} - + WriteToProcessorImpl(std::move(req)); +} + +// Called under lock. Invokes Processor->Write, which is assumed to be deadlock-safe +void TWriteSession::WriteToProcessorImpl(TWriteSession::TClientMessage&& req) { + Y_ASSERT(Processor); + if (Aborting) + return; + auto callback = [weakThis = weak_from_this(), connectionGeneration = ConnectionGeneration](NGrpc::TGrpcStatus&& grpcStatus) { + if (auto sharedThis = weakThis.lock()) { + sharedThis->OnWriteDone(std::move(grpcStatus), connectionGeneration); + } + }; + + Processor->Write(std::move(req), callback); +} + +void TWriteSession::ReadFromProcessor() { + Y_ASSERT(Processor); + IProcessor::TPtr prc; + ui64 generation; + with_lock(Lock) { + prc = Processor; + generation = ConnectionGeneration; + } + auto callback = [weakThis = weak_from_this(), connectionGeneration = generation, processor = prc, serverMessage = ServerMessage] + (NGrpc::TGrpcStatus&& grpcStatus) { + if (auto sharedThis = weakThis.lock()) { + sharedThis->OnReadDone(std::move(grpcStatus), connectionGeneration); + } + }; + prc->Read(ServerMessage.get(), std::move(callback)); +} + +void TWriteSession::OnWriteDone(NGrpc::TGrpcStatus&& status, size_t connectionGeneration) { + THandleResult handleResult; + with_lock (Lock) { + if (connectionGeneration != ConnectionGeneration) { + return; // Message from previous connection. Ignore. + } + if (Aborting) { + return; + } + if(!status.Ok()) { + handleResult = OnErrorImpl(status); + } + } + ProcessHandleResult(handleResult); +} + +void TWriteSession::OnReadDone(NGrpc::TGrpcStatus&& grpcStatus, size_t connectionGeneration) { + TPlainStatus errorStatus; + TProcessSrvMessageResult processResult; + bool needSetValue = false; + if (!grpcStatus.Ok()) { + errorStatus = TPlainStatus(std::move(grpcStatus)); + } + bool doRead = false; + with_lock (Lock) { + UpdateTimedCountersImpl(); + if (connectionGeneration != ConnectionGeneration) { + return; // Message from previous connection. Ignore. + } + if (errorStatus.Ok()) { + if (IsErrorMessage(*ServerMessage)) { + errorStatus = MakeErrorFromProto(*ServerMessage); + } else { + processResult = ProcessServerMessageImpl(); + needSetValue = !InitSeqNoSetDone && processResult.InitSeqNo.Defined() && (InitSeqNoSetDone = true); + if (errorStatus.Ok() && processResult.Ok) { + doRead = true; + } + } + } + } + if (doRead) + ReadFromProcessor(); + + with_lock(Lock) { + if (!errorStatus.Ok()) { + if (processResult.Ok) { // Otherwise, OnError was already called + processResult.HandleResult = RestartImpl(errorStatus); + } + } + if (processResult.HandleResult.DoStop) { + CloseImpl(std::move(errorStatus)); + } + } + for (auto& event : processResult.Events) { + EventsQueue->PushEvent(std::move(event)); + } + if (needSetValue) { + InitSeqNoPromise.SetValue(*processResult.InitSeqNo); + processResult.HandleResult.DoSetSeqNo = false; // Redundant. Just in case. + } + ProcessHandleResult(processResult.HandleResult); +} + TString TWriteSession::LogPrefix() const { return TStringBuilder() << "MessageGroupId [" << Settings.MessageGroupId_ << "] SessionId [" << SessionId << "] "; } @@ -662,46 +662,46 @@ TString TWriteSessionEvent::TReadyToAcceptEvent::DebugString() const { } -TWriteSession::TProcessSrvMessageResult TWriteSession::ProcessServerMessageImpl() { - TProcessSrvMessageResult result; - switch (ServerMessage->GetServerMessageCase()) { - case TServerMessage::SERVER_MESSAGE_NOT_SET: { - SessionEstablished = false; - result.HandleResult = OnErrorImpl({ - static_cast<NYdb::EStatus>(ServerMessage->status()), - {NYql::TIssue{ServerMessage->DebugString()}} - }); - result.Ok = false; - break; - } - case TServerMessage::kInitResponse: { - const auto& initResponse = ServerMessage->init_response(); +TWriteSession::TProcessSrvMessageResult TWriteSession::ProcessServerMessageImpl() { + TProcessSrvMessageResult result; + switch (ServerMessage->GetServerMessageCase()) { + case TServerMessage::SERVER_MESSAGE_NOT_SET: { + SessionEstablished = false; + result.HandleResult = OnErrorImpl({ + static_cast<NYdb::EStatus>(ServerMessage->status()), + {NYql::TIssue{ServerMessage->DebugString()}} + }); + result.Ok = false; + break; + } + case TServerMessage::kInitResponse: { + const auto& initResponse = ServerMessage->init_response(); DbDriverState->Log << TLOG_INFO << LogPrefix() << "Write session established. Init response: " << initResponse.ShortDebugString(); - SessionId = initResponse.session_id(); - PartitionId = initResponse.partition_id(); - ui64 newLastSeqNo = initResponse.last_sequence_number(); - // SeqNo increased, so there's a risk of loss, apply SeqNo shift. - // MinUnsentSeqNo must be > 0 if anything was ever sent yet - if (MinUnsentSeqNo && OnSeqNoShift && newLastSeqNo > MinUnsentSeqNo) { - SeqNoShift = newLastSeqNo - MinUnsentSeqNo; - } - result.InitSeqNo = newLastSeqNo; - LastSeqNo = newLastSeqNo; - - SessionEstablished = true; - LastCountersUpdateTs = TInstant::Now(); - SessionStartedTs = TInstant::Now(); - OnErrorResolved(); - - //EventsQueue->PushEvent(TWriteSessionEvent::TReadyToAcceptEvent{TContinuationToken{}}); - result.Events.emplace_back(TWriteSessionEvent::TReadyToAcceptEvent{TContinuationToken{}}); - // Kickstart send after session reestablishment - SendImpl(); - break; - } - case TServerMessage::kBatchWriteResponse: { - TWriteSessionEvent::TAcksEvent acksEvent; - const auto& batchWriteResponse = ServerMessage->batch_write_response(); + SessionId = initResponse.session_id(); + PartitionId = initResponse.partition_id(); + ui64 newLastSeqNo = initResponse.last_sequence_number(); + // SeqNo increased, so there's a risk of loss, apply SeqNo shift. + // MinUnsentSeqNo must be > 0 if anything was ever sent yet + if (MinUnsentSeqNo && OnSeqNoShift && newLastSeqNo > MinUnsentSeqNo) { + SeqNoShift = newLastSeqNo - MinUnsentSeqNo; + } + result.InitSeqNo = newLastSeqNo; + LastSeqNo = newLastSeqNo; + + SessionEstablished = true; + LastCountersUpdateTs = TInstant::Now(); + SessionStartedTs = TInstant::Now(); + OnErrorResolved(); + + //EventsQueue->PushEvent(TWriteSessionEvent::TReadyToAcceptEvent{TContinuationToken{}}); + result.Events.emplace_back(TWriteSessionEvent::TReadyToAcceptEvent{TContinuationToken{}}); + // Kickstart send after session reestablishment + SendImpl(); + break; + } + case TServerMessage::kBatchWriteResponse: { + TWriteSessionEvent::TAcksEvent acksEvent; + const auto& batchWriteResponse = ServerMessage->batch_write_response(); DbDriverState->Log << TLOG_DEBUG << LogPrefix() << "Write session got write response: " << batchWriteResponse.ShortDebugString(); TWriteStat::TPtr writeStat = new TWriteStat{}; const auto& stat = batchWriteResponse.write_statistics(); @@ -710,285 +710,285 @@ TWriteSession::TProcessSrvMessageResult TWriteSession::ProcessServerMessageImpl( writeStat->PartitionQuotedTime = TDuration::MilliSeconds(stat.throttled_on_partition_duration_ms()); writeStat->TopicQuotedTime = TDuration::MilliSeconds(stat.throttled_on_topic_duration_ms()); - for (size_t messageIndex = 0, endIndex = batchWriteResponse.sequence_numbers_size(); messageIndex != endIndex; ++messageIndex) { - // TODO: Fill writer statistics - ui64 sequenceNumber = batchWriteResponse.sequence_numbers(messageIndex); - - acksEvent.Acks.push_back(TWriteSessionEvent::TWriteAck{ - sequenceNumber - SeqNoShift, - batchWriteResponse.already_written(messageIndex) ? TWriteSessionEvent::TWriteAck::EES_ALREADY_WRITTEN: - TWriteSessionEvent::TWriteAck::EES_WRITTEN, - TWriteSessionEvent::TWriteAck::TWrittenMessageDetails { - static_cast<ui64>(batchWriteResponse.offsets(messageIndex)), - PartitionId, - }, + for (size_t messageIndex = 0, endIndex = batchWriteResponse.sequence_numbers_size(); messageIndex != endIndex; ++messageIndex) { + // TODO: Fill writer statistics + ui64 sequenceNumber = batchWriteResponse.sequence_numbers(messageIndex); + + acksEvent.Acks.push_back(TWriteSessionEvent::TWriteAck{ + sequenceNumber - SeqNoShift, + batchWriteResponse.already_written(messageIndex) ? TWriteSessionEvent::TWriteAck::EES_ALREADY_WRITTEN: + TWriteSessionEvent::TWriteAck::EES_WRITTEN, + TWriteSessionEvent::TWriteAck::TWrittenMessageDetails { + static_cast<ui64>(batchWriteResponse.offsets(messageIndex)), + PartitionId, + }, writeStat, - }); - - if (CleanupOnAcknowledged(sequenceNumber - SeqNoShift)) { - result.Events.emplace_back(TWriteSessionEvent::TReadyToAcceptEvent{TContinuationToken{}}); - } - } - //EventsQueue->PushEvent(std::move(acksEvent)); - result.Events.emplace_back(std::move(acksEvent)); - break; - } - case TServerMessage::kUpdateTokenResponse: { + }); + + if (CleanupOnAcknowledged(sequenceNumber - SeqNoShift)) { + result.Events.emplace_back(TWriteSessionEvent::TReadyToAcceptEvent{TContinuationToken{}}); + } + } + //EventsQueue->PushEvent(std::move(acksEvent)); + result.Events.emplace_back(std::move(acksEvent)); + break; + } + case TServerMessage::kUpdateTokenResponse: { UpdateTokenInProgress = false; DbDriverState->Log << TLOG_DEBUG << LogPrefix() << "Write session: token updated successfully"; UpdateTokenIfNeededImpl(); - break; - } - } - return result; -} - -bool TWriteSession::CleanupOnAcknowledged(ui64 sequenceNumber) { - bool result = false; + break; + } + } + return result; +} + +bool TWriteSession::CleanupOnAcknowledged(ui64 sequenceNumber) { + bool result = false; DbDriverState->Log << TLOG_DEBUG << LogPrefix() << "Write session: acknoledged message " << sequenceNumber; - UpdateTimedCountersImpl(); - const auto& sentFront = SentOriginalMessages.front(); - ui64 size = 0; - ui64 compressedSize = 0; - if(!SentPackedMessage.empty() && SentPackedMessage.front().Offset == sequenceNumber) { - auto memoryUsage = OnMemoryUsageChangedImpl(-SentPackedMessage.front().Data.size()); - result = memoryUsage.NowOk && !memoryUsage.WasOk; - //EventsQueue->PushEvent(TWriteSessionEvent::TReadyToAcceptEvent{TContinuationToken{}}); + UpdateTimedCountersImpl(); + const auto& sentFront = SentOriginalMessages.front(); + ui64 size = 0; + ui64 compressedSize = 0; + if(!SentPackedMessage.empty() && SentPackedMessage.front().Offset == sequenceNumber) { + auto memoryUsage = OnMemoryUsageChangedImpl(-SentPackedMessage.front().Data.size()); + result = memoryUsage.NowOk && !memoryUsage.WasOk; + //EventsQueue->PushEvent(TWriteSessionEvent::TReadyToAcceptEvent{TContinuationToken{}}); const auto& front = SentPackedMessage.front(); - if (front.Compressed) { - compressedSize = front.Data.size(); - } else { - size = front.Data.size(); - } - - (*Counters->MessagesWritten) += front.MessageCount; - (*Counters->MessagesInflight) -= front.MessageCount; + if (front.Compressed) { + compressedSize = front.Data.size(); + } else { + size = front.Data.size(); + } + + (*Counters->MessagesWritten) += front.MessageCount; + (*Counters->MessagesInflight) -= front.MessageCount; (*Counters->BytesWritten) += front.OriginalSize; - - SentPackedMessage.pop(); - } else { - size = sentFront.Size; - (*Counters->BytesWritten) += sentFront.Size; - (*Counters->MessagesWritten)++; - (*Counters->MessagesInflight)--; - } - - (*Counters->BytesInflightCompressed) -= compressedSize; - (*Counters->BytesWrittenCompressed) += compressedSize; - (*Counters->BytesInflightUncompressed) -= size; - - Y_VERIFY(Counters->BytesInflightCompressed->Val() >= 0); - Y_VERIFY(Counters->BytesInflightUncompressed->Val() >= 0); - - Y_VERIFY(sentFront.SeqNo == sequenceNumber); - - (*Counters->BytesInflightTotal) = MemoryUsage; - SentOriginalMessages.pop(); - return result; -} - -// Only called under Lock -TMemoryUsageChange TWriteSession::OnMemoryUsageChangedImpl(i64 diff) { - bool wasOk = MemoryUsage <= Settings.MaxMemoryUsage_; + + SentPackedMessage.pop(); + } else { + size = sentFront.Size; + (*Counters->BytesWritten) += sentFront.Size; + (*Counters->MessagesWritten)++; + (*Counters->MessagesInflight)--; + } + + (*Counters->BytesInflightCompressed) -= compressedSize; + (*Counters->BytesWrittenCompressed) += compressedSize; + (*Counters->BytesInflightUncompressed) -= size; + + Y_VERIFY(Counters->BytesInflightCompressed->Val() >= 0); + Y_VERIFY(Counters->BytesInflightUncompressed->Val() >= 0); + + Y_VERIFY(sentFront.SeqNo == sequenceNumber); + + (*Counters->BytesInflightTotal) = MemoryUsage; + SentOriginalMessages.pop(); + return result; +} + +// Only called under Lock +TMemoryUsageChange TWriteSession::OnMemoryUsageChangedImpl(i64 diff) { + bool wasOk = MemoryUsage <= Settings.MaxMemoryUsage_; //if (diff < 0) { // Y_VERIFY(MemoryUsage >= static_cast<size_t>(std::abs(diff))); //} - MemoryUsage += diff; - bool nowOk = MemoryUsage <= Settings.MaxMemoryUsage_; - if (wasOk != nowOk) { - if (wasOk) { + MemoryUsage += diff; + bool nowOk = MemoryUsage <= Settings.MaxMemoryUsage_; + if (wasOk != nowOk) { + if (wasOk) { DbDriverState->Log << TLOG_DEBUG << LogPrefix() << "Estimated memory usage " << MemoryUsage << "[B] reached maximum (" << Settings.MaxMemoryUsage_ << "[B])"; - } - else { + } + else { DbDriverState->Log << TLOG_DEBUG << LogPrefix() << "Estimated memory usage got back to normal " << MemoryUsage << "[B]"; - } - } - return {wasOk, nowOk}; -} - + } + } + return {wasOk, nowOk}; +} + TBuffer CompressBuffer(TVector<TStringBuf>& data, ECodec codec, i32 level) { - TBuffer result; + TBuffer result; THolder<IOutputStream> coder = NCompressionDetails::CreateCoder(codec, result, level); - for (auto& buffer : data) { - coder->Write(buffer.data(), buffer.size()); - } - coder->Finish(); - return result; -} - -// May call OnCompressed with sync executor. No external lock. + for (auto& buffer : data) { + coder->Write(buffer.data(), buffer.size()); + } + coder->Finish(); + return result; +} + +// May call OnCompressed with sync executor. No external lock. void TWriteSession::CompressImpl(TBlock&& block_) { - auto weakThis = weak_from_this(); + auto weakThis = weak_from_this(); bool isSyncCompression = !CompressionExecutor->IsAsync(); Y_VERIFY(block_.Valid); - + std::shared_ptr<TBlock> blockPtr(std::make_shared<TBlock>()); blockPtr->Move(block_); auto lambda = [weakThis, codec = Settings.Codec_, level = Settings.CompressionLevel_, isSyncCompression, blockPtr]() mutable - { - if (auto sharedThis = weakThis.lock()) { + { + if (auto sharedThis = weakThis.lock()) { Y_VERIFY(!blockPtr->Compressed); - auto compressedData = CompressBuffer( + auto compressedData = CompressBuffer( blockPtr->OriginalDataRefs, codec, level - ); - Y_VERIFY(!compressedData.Empty()); + ); + Y_VERIFY(!compressedData.Empty()); blockPtr->Data = std::move(compressedData); blockPtr->Compressed = true; blockPtr->CodecID = GetCodecId(sharedThis->Settings.Codec_); sharedThis->OnCompressed(std::move(*blockPtr), isSyncCompression); - } + } }; CompressionExecutor->Post(lambda); -} - -void TWriteSession::OnCompressed(TBlock&& block, bool isSyncCompression) { - TMemoryUsageChange memoryUsage; - if (!isSyncCompression) { - with_lock(Lock) { - memoryUsage = OnCompressedImpl(std::move(block)); - } - } else { - memoryUsage = OnCompressedImpl(std::move(block)); - } - if (memoryUsage.NowOk && !memoryUsage.WasOk) { - EventsQueue->PushEvent(TWriteSessionEvent::TReadyToAcceptEvent{TContinuationToken{}}); - } -} - -//Called under lock or synchronously if compression is sync -TMemoryUsageChange TWriteSession::OnCompressedImpl(TBlock&& block) { - UpdateTimedCountersImpl(); +} + +void TWriteSession::OnCompressed(TBlock&& block, bool isSyncCompression) { + TMemoryUsageChange memoryUsage; + if (!isSyncCompression) { + with_lock(Lock) { + memoryUsage = OnCompressedImpl(std::move(block)); + } + } else { + memoryUsage = OnCompressedImpl(std::move(block)); + } + if (memoryUsage.NowOk && !memoryUsage.WasOk) { + EventsQueue->PushEvent(TWriteSessionEvent::TReadyToAcceptEvent{TContinuationToken{}}); + } +} + +//Called under lock or synchronously if compression is sync +TMemoryUsageChange TWriteSession::OnCompressedImpl(TBlock&& block) { + UpdateTimedCountersImpl(); Y_VERIFY(block.Valid); - auto memoryUsage = OnMemoryUsageChangedImpl(static_cast<i64>(block.Data.size()) - block.OriginalMemoryUsage); - (*Counters->BytesInflightUncompressed) -= block.OriginalSize; - (*Counters->BytesInflightCompressed) += block.Data.size(); - + auto memoryUsage = OnMemoryUsageChangedImpl(static_cast<i64>(block.Data.size()) - block.OriginalMemoryUsage); + (*Counters->BytesInflightUncompressed) -= block.OriginalSize; + (*Counters->BytesInflightCompressed) += block.Data.size(); + PackedMessagesToSend.emplace(std::move(block)); - SendImpl(); - return memoryUsage; -} - -// Only called under lock -void TWriteSession::ResetForRetryImpl() { - SessionEstablished = false; - const size_t totalPackedMessages = PackedMessagesToSend.size() + SentPackedMessage.size(); - const size_t totalOriginalMessages = OriginalMessagesToSend.size() + SentOriginalMessages.size(); - while (!SentPackedMessage.empty()) { + SendImpl(); + return memoryUsage; +} + +// Only called under lock +void TWriteSession::ResetForRetryImpl() { + SessionEstablished = false; + const size_t totalPackedMessages = PackedMessagesToSend.size() + SentPackedMessage.size(); + const size_t totalOriginalMessages = OriginalMessagesToSend.size() + SentOriginalMessages.size(); + while (!SentPackedMessage.empty()) { PackedMessagesToSend.emplace(std::move(SentPackedMessage.front())); - SentPackedMessage.pop(); - } - ui64 minSeqNo = PackedMessagesToSend.empty() ? LastSeqNo + 1 : PackedMessagesToSend.top().Offset; + SentPackedMessage.pop(); + } + ui64 minSeqNo = PackedMessagesToSend.empty() ? LastSeqNo + 1 : PackedMessagesToSend.top().Offset; std::queue<TOriginalMessage> freshOriginalMessagesToSend; OriginalMessagesToSend.swap(freshOriginalMessagesToSend); - while (!SentOriginalMessages.empty()) { + while (!SentOriginalMessages.empty()) { OriginalMessagesToSend.emplace(std::move(SentOriginalMessages.front())); - SentOriginalMessages.pop(); - } - while (!freshOriginalMessagesToSend.empty()) { + SentOriginalMessages.pop(); + } + while (!freshOriginalMessagesToSend.empty()) { OriginalMessagesToSend.emplace(std::move(freshOriginalMessagesToSend.front())); - freshOriginalMessagesToSend.pop(); - } - if (!OriginalMessagesToSend.empty() && OriginalMessagesToSend.front().SeqNo < minSeqNo) - minSeqNo = OriginalMessagesToSend.front().SeqNo; - MinUnsentSeqNo = minSeqNo; - Y_VERIFY(PackedMessagesToSend.size() == totalPackedMessages); - Y_VERIFY(OriginalMessagesToSend.size() == totalOriginalMessages); -} - -// Called from client Write() methods. With lock -void TWriteSession::FlushWriteIfRequiredImpl() { - - if (!CurrentBatch.Empty() && !CurrentBatch.FlushRequested) { - MessagesAcquired += static_cast<ui64>(CurrentBatch.Acquire()); - if (TInstant::Now() - CurrentBatch.StartedAt >= Settings.BatchFlushInterval_.GetOrElse(TDuration::Zero()) - || CurrentBatch.CurrentSize >= Settings.BatchFlushSizeBytes_.GetOrElse(0) - || CurrentBatch.CurrentSize >= MaxBlockSize - || CurrentBatch.Messages.size() >= MaxBlockMessageCount + freshOriginalMessagesToSend.pop(); + } + if (!OriginalMessagesToSend.empty() && OriginalMessagesToSend.front().SeqNo < minSeqNo) + minSeqNo = OriginalMessagesToSend.front().SeqNo; + MinUnsentSeqNo = minSeqNo; + Y_VERIFY(PackedMessagesToSend.size() == totalPackedMessages); + Y_VERIFY(OriginalMessagesToSend.size() == totalOriginalMessages); +} + +// Called from client Write() methods. With lock +void TWriteSession::FlushWriteIfRequiredImpl() { + + if (!CurrentBatch.Empty() && !CurrentBatch.FlushRequested) { + MessagesAcquired += static_cast<ui64>(CurrentBatch.Acquire()); + if (TInstant::Now() - CurrentBatch.StartedAt >= Settings.BatchFlushInterval_.GetOrElse(TDuration::Zero()) + || CurrentBatch.CurrentSize >= Settings.BatchFlushSizeBytes_.GetOrElse(0) + || CurrentBatch.CurrentSize >= MaxBlockSize + || CurrentBatch.Messages.size() >= MaxBlockMessageCount || CurrentBatch.HasCodec() - ) { - WriteBatchImpl(); - return; - } - } -} - - -// Involves compression, but still called under lock. -size_t TWriteSession::WriteBatchImpl() { + ) { + WriteBatchImpl(); + return; + } + } +} + + +// Involves compression, but still called under lock. +size_t TWriteSession::WriteBatchImpl() { DbDriverState->Log << TLOG_DEBUG << LogPrefix() << "write " << CurrentBatch.Messages.size() << " messages with seqNo from " - << CurrentBatch.Messages.begin()->SeqNo << " to " << CurrentBatch.Messages.back().SeqNo; - - Y_VERIFY(CurrentBatch.Messages.size() <= MaxBlockMessageCount); - + << CurrentBatch.Messages.begin()->SeqNo << " to " << CurrentBatch.Messages.back().SeqNo; + + Y_VERIFY(CurrentBatch.Messages.size() <= MaxBlockMessageCount); + const bool skipCompression = Settings.Codec_ == ECodec::RAW || CurrentBatch.HasCodec(); - if (!skipCompression && Settings.CompressionExecutor_->IsAsync()) { - MessagesAcquired += static_cast<ui64>(CurrentBatch.Acquire()); - } - - size_t size = 0; - for (size_t i = 0; i != CurrentBatch.Messages.size();) { - TBlock block{}; - for (; block.OriginalSize < MaxBlockSize && i != CurrentBatch.Messages.size(); ++i) { - auto sequenceNumber = CurrentBatch.Messages[i].SeqNo; - auto createTs = CurrentBatch.Messages[i].CreatedAt; - - if (!block.MessageCount) { - block.Offset = sequenceNumber; - } - - block.MessageCount += 1; - const auto& datum = CurrentBatch.Messages[i].DataRef; - block.OriginalSize += datum.size(); - block.OriginalMemoryUsage = CurrentBatch.Data.size(); - block.OriginalDataRefs.emplace_back(datum); + if (!skipCompression && Settings.CompressionExecutor_->IsAsync()) { + MessagesAcquired += static_cast<ui64>(CurrentBatch.Acquire()); + } + + size_t size = 0; + for (size_t i = 0; i != CurrentBatch.Messages.size();) { + TBlock block{}; + for (; block.OriginalSize < MaxBlockSize && i != CurrentBatch.Messages.size(); ++i) { + auto sequenceNumber = CurrentBatch.Messages[i].SeqNo; + auto createTs = CurrentBatch.Messages[i].CreatedAt; + + if (!block.MessageCount) { + block.Offset = sequenceNumber; + } + + block.MessageCount += 1; + const auto& datum = CurrentBatch.Messages[i].DataRef; + block.OriginalSize += datum.size(); + block.OriginalMemoryUsage = CurrentBatch.Data.size(); + block.OriginalDataRefs.emplace_back(datum); if (CurrentBatch.Messages[i].Codec.Defined()) { Y_VERIFY(CurrentBatch.Messages.size() == 1); block.CodecID = GetCodecId(*CurrentBatch.Messages[i].Codec); block.OriginalSize = CurrentBatch.Messages[i].OriginalSize; block.Compressed = false; } - size += datum.size(); - UpdateTimedCountersImpl(); - (*Counters->BytesInflightUncompressed) += datum.size(); - (*Counters->MessagesInflight)++; - OriginalMessagesToSend.emplace(sequenceNumber, createTs, datum.size()); - } - block.Data = std::move(CurrentBatch.Data); - if (skipCompression) { + size += datum.size(); + UpdateTimedCountersImpl(); + (*Counters->BytesInflightUncompressed) += datum.size(); + (*Counters->MessagesInflight)++; + OriginalMessagesToSend.emplace(sequenceNumber, createTs, datum.size()); + } + block.Data = std::move(CurrentBatch.Data); + if (skipCompression) { PackedMessagesToSend.emplace(std::move(block)); - } else { - CompressImpl(std::move(block)); - } - } - CurrentBatch.Reset(); - if (skipCompression) { - SendImpl(); - } - return size; -} - -size_t GetMaxGrpcMessageSize() { - return 120 * 1024 * 1024; -} - -bool TWriteSession::IsReadyToSendNextImpl() const { - if (!SessionEstablished) { - return false; - } + } else { + CompressImpl(std::move(block)); + } + } + CurrentBatch.Reset(); + if (skipCompression) { + SendImpl(); + } + return size; +} + +size_t GetMaxGrpcMessageSize() { + return 120 * 1024 * 1024; +} + +bool TWriteSession::IsReadyToSendNextImpl() const { + if (!SessionEstablished) { + return false; + } if (Aborting) return false; - if (PackedMessagesToSend.empty()) { - return false; - } - Y_VERIFY(!OriginalMessagesToSend.empty(), "There are packed messages but no original messages"); - Y_VERIFY(OriginalMessagesToSend.front().SeqNo <= PackedMessagesToSend.top().Offset, "Lost original message(s)"); - - return PackedMessagesToSend.top().Offset == OriginalMessagesToSend.front().SeqNo; -} - + if (PackedMessagesToSend.empty()) { + return false; + } + Y_VERIFY(!OriginalMessagesToSend.empty(), "There are packed messages but no original messages"); + Y_VERIFY(OriginalMessagesToSend.front().SeqNo <= PackedMessagesToSend.top().Offset, "Lost original message(s)"); + + return PackedMessagesToSend.top().Offset == OriginalMessagesToSend.front().SeqNo; +} + void TWriteSession::UpdateTokenIfNeededImpl() { DbDriverState->Log << TLOG_DEBUG << LogPrefix() << "Write session: try to update token"; @@ -1009,120 +1009,120 @@ void TWriteSession::UpdateTokenIfNeededImpl() { Processor->Write(std::move(clientMessage)); } -void TWriteSession::SendImpl() { - // External cycle splits ready blocks into multiple gRPC messages. Current gRPC message size hard limit is 64MiB - while(IsReadyToSendNextImpl()) { - TClientMessage clientMessage; - auto* writeRequest = clientMessage.mutable_write_request(); - auto sentAtMs = TInstant::Now().MilliSeconds(); - - // Sent blocks while we can without messages reordering - while (IsReadyToSendNextImpl() && clientMessage.ByteSizeLong() < GetMaxGrpcMessageSize()) { +void TWriteSession::SendImpl() { + // External cycle splits ready blocks into multiple gRPC messages. Current gRPC message size hard limit is 64MiB + while(IsReadyToSendNextImpl()) { + TClientMessage clientMessage; + auto* writeRequest = clientMessage.mutable_write_request(); + auto sentAtMs = TInstant::Now().MilliSeconds(); + + // Sent blocks while we can without messages reordering + while (IsReadyToSendNextImpl() && clientMessage.ByteSizeLong() < GetMaxGrpcMessageSize()) { const auto& block = PackedMessagesToSend.top(); Y_VERIFY(block.Valid); - for (size_t i = 0; i != block.MessageCount; ++i) { - Y_VERIFY(!OriginalMessagesToSend.empty()); - - auto& message = OriginalMessagesToSend.front(); - - writeRequest->add_sent_at_ms(sentAtMs); - writeRequest->add_sequence_numbers(message.SeqNo + SeqNoShift); - writeRequest->add_message_sizes(message.Size); - writeRequest->add_created_at_ms(message.CreatedAt.MilliSeconds()); - + for (size_t i = 0; i != block.MessageCount; ++i) { + Y_VERIFY(!OriginalMessagesToSend.empty()); + + auto& message = OriginalMessagesToSend.front(); + + writeRequest->add_sent_at_ms(sentAtMs); + writeRequest->add_sequence_numbers(message.SeqNo + SeqNoShift); + writeRequest->add_message_sizes(message.Size); + writeRequest->add_created_at_ms(message.CreatedAt.MilliSeconds()); + SentOriginalMessages.emplace(std::move(message)); - OriginalMessagesToSend.pop(); - } - - writeRequest->add_blocks_offsets(block.Offset); - writeRequest->add_blocks_message_counts(block.MessageCount); - writeRequest->add_blocks_part_numbers(block.PartNumber); - writeRequest->add_blocks_uncompressed_sizes(block.OriginalSize); - writeRequest->add_blocks_headers(block.CodecID); - if (block.Compressed) - writeRequest->add_blocks_data(block.Data.data(), block.Data.size()); - else { - for (auto& buffer: block.OriginalDataRefs) { - writeRequest->add_blocks_data(buffer.data(), buffer.size()); - } - } - + OriginalMessagesToSend.pop(); + } + + writeRequest->add_blocks_offsets(block.Offset); + writeRequest->add_blocks_message_counts(block.MessageCount); + writeRequest->add_blocks_part_numbers(block.PartNumber); + writeRequest->add_blocks_uncompressed_sizes(block.OriginalSize); + writeRequest->add_blocks_headers(block.CodecID); + if (block.Compressed) + writeRequest->add_blocks_data(block.Data.data(), block.Data.size()); + else { + for (auto& buffer: block.OriginalDataRefs) { + writeRequest->add_blocks_data(buffer.data(), buffer.size()); + } + } + TBlock moveBlock; moveBlock.Move(block); SentPackedMessage.emplace(std::move(moveBlock)); - PackedMessagesToSend.pop(); - } + PackedMessagesToSend.pop(); + } UpdateTokenIfNeededImpl(); DbDriverState->Log << TLOG_DEBUG << LogPrefix() << "Send " << writeRequest->sequence_numbers_size() << " message(s) (" - << OriginalMessagesToSend.size() << " left), first sequence number is " - << writeRequest->sequence_numbers(0); - Processor->Write(std::move(clientMessage)); - } -} - -// Client method, no Lock -bool TWriteSession::Close(TDuration closeTimeout) { - if (AtomicGet(Aborting)) + << OriginalMessagesToSend.size() << " left), first sequence number is " + << writeRequest->sequence_numbers(0); + Processor->Write(std::move(clientMessage)); + } +} + +// Client method, no Lock +bool TWriteSession::Close(TDuration closeTimeout) { + if (AtomicGet(Aborting)) return false; DbDriverState->Log << TLOG_INFO << LogPrefix() << "Write session: close. Timeout = " << closeTimeout.MilliSeconds() << " ms"; - auto startTime = TInstant::Now(); - auto remaining = closeTimeout; - bool ready = false; - bool needSetSeqNoValue = false; - while (remaining > TDuration::Zero()) { - with_lock(Lock) { - if (OriginalMessagesToSend.empty() && SentOriginalMessages.empty()) { - ready = true; - } + auto startTime = TInstant::Now(); + auto remaining = closeTimeout; + bool ready = false; + bool needSetSeqNoValue = false; + while (remaining > TDuration::Zero()) { + with_lock(Lock) { + if (OriginalMessagesToSend.empty() && SentOriginalMessages.empty()) { + ready = true; + } if (AtomicGet(Aborting)) break; - } - if (ready) { - break; - } - remaining = closeTimeout - (TInstant::Now() - startTime); - Sleep(Min(TDuration::MilliSeconds(100), remaining)); - } - with_lock(Lock) { + } + if (ready) { + break; + } + remaining = closeTimeout - (TInstant::Now() - startTime); + Sleep(Min(TDuration::MilliSeconds(100), remaining)); + } + with_lock(Lock) { ready = (OriginalMessagesToSend.empty() && SentOriginalMessages.empty()) && !AtomicGet(Aborting); - } + } with_lock(Lock) { CloseImpl(EStatus::SUCCESS, NYql::TIssues{}); - needSetSeqNoValue = !InitSeqNoSetDone && (InitSeqNoSetDone = true); + needSetSeqNoValue = !InitSeqNoSetDone && (InitSeqNoSetDone = true); + } + if (needSetSeqNoValue) { + InitSeqNoPromise.SetException("session closed"); } - if (needSetSeqNoValue) { - InitSeqNoPromise.SetException("session closed"); - } if (ready) { DbDriverState->Log << TLOG_INFO << LogPrefix() << "Write session: gracefully shut down, all writes complete"; } else { DbDriverState->Log << TLOG_WARNING << LogPrefix() << "Write session: could not confirm all writes in time or session aborted, perform hard shutdown"; } - return ready; -} - -void TWriteSession::HandleWakeUpImpl() { - FlushWriteIfRequiredImpl(); - if (AtomicGet(Aborting)) { - return; - } - auto callback = [weakThis = this->weak_from_this()] (bool ok) - { - if (!ok) - return; - if (auto sharedThis = weakThis.lock()) { - with_lock(sharedThis->Lock) { - sharedThis->HandleWakeUpImpl(); - } - } - }; - auto enqueueTokenCallback = [weakThis = this->weak_from_this()] (bool ok) { - if (!ok) - return; - if (auto sharedThis = weakThis.lock()) { - sharedThis->EventsQueue->PushEvent(TWriteSessionEvent::TReadyToAcceptEvent{TContinuationToken{}}); - } - }; + return ready; +} + +void TWriteSession::HandleWakeUpImpl() { + FlushWriteIfRequiredImpl(); + if (AtomicGet(Aborting)) { + return; + } + auto callback = [weakThis = this->weak_from_this()] (bool ok) + { + if (!ok) + return; + if (auto sharedThis = weakThis.lock()) { + with_lock(sharedThis->Lock) { + sharedThis->HandleWakeUpImpl(); + } + } + }; + auto enqueueTokenCallback = [weakThis = this->weak_from_this()] (bool ok) { + if (!ok) + return; + if (auto sharedThis = weakThis.lock()) { + sharedThis->EventsQueue->PushEvent(TWriteSessionEvent::TReadyToAcceptEvent{TContinuationToken{}}); + } + }; if (TInstant::Now() - LastTokenUpdate > UPDATE_TOKEN_PERIOD) { LastTokenUpdate = TInstant::Now(); UpdateTokenIfNeededImpl(); @@ -1132,179 +1132,179 @@ void TWriteSession::HandleWakeUpImpl() { ? WakeupInterval : WakeupInterval - Min(Now() - CurrentBatch.StartedAt, WakeupInterval); Connections->ScheduleCallback(flushAfter, std::move(callback)); -} - -void TWriteSession::UpdateTimedCountersImpl() { - auto now = TInstant::Now(); - auto delta = (now - LastCountersUpdateTs).MilliSeconds(); - double percent = 100.0 / Settings.MaxMemoryUsage_; - - Counters->TotalBytesInflightUsageByTime->Collect(*Counters->BytesInflightTotal * percent, delta); - Counters->UncompressedBytesInflightUsageByTime->Collect(*Counters->BytesInflightUncompressed * percent, delta); - Counters->CompressedBytesInflightUsageByTime->Collect(*Counters->BytesInflightCompressed * percent, delta); - - *Counters->CurrentSessionLifetimeMs = (TInstant::Now() - SessionStartedTs).MilliSeconds(); - LastCountersUpdateTs = now; - if (LastCountersLogTs == TInstant::Zero() || TInstant::Now() - LastCountersLogTs > TDuration::Seconds(60)) { - LastCountersLogTs = TInstant::Now(); - -#define LOG_COUNTER(counter) \ - << " " Y_STRINGIZE(counter) ": " \ - << Counters->counter->Val() \ - /**/ - +} + +void TWriteSession::UpdateTimedCountersImpl() { + auto now = TInstant::Now(); + auto delta = (now - LastCountersUpdateTs).MilliSeconds(); + double percent = 100.0 / Settings.MaxMemoryUsage_; + + Counters->TotalBytesInflightUsageByTime->Collect(*Counters->BytesInflightTotal * percent, delta); + Counters->UncompressedBytesInflightUsageByTime->Collect(*Counters->BytesInflightUncompressed * percent, delta); + Counters->CompressedBytesInflightUsageByTime->Collect(*Counters->BytesInflightCompressed * percent, delta); + + *Counters->CurrentSessionLifetimeMs = (TInstant::Now() - SessionStartedTs).MilliSeconds(); + LastCountersUpdateTs = now; + if (LastCountersLogTs == TInstant::Zero() || TInstant::Now() - LastCountersLogTs > TDuration::Seconds(60)) { + LastCountersLogTs = TInstant::Now(); + +#define LOG_COUNTER(counter) \ + << " " Y_STRINGIZE(counter) ": " \ + << Counters->counter->Val() \ + /**/ + DbDriverState->Log << TLOG_INFO << LogPrefix() - << "Counters: {" - LOG_COUNTER(Errors) - LOG_COUNTER(CurrentSessionLifetimeMs) - LOG_COUNTER(BytesWritten) - LOG_COUNTER(MessagesWritten) - LOG_COUNTER(BytesWrittenCompressed) - LOG_COUNTER(BytesInflightUncompressed) - LOG_COUNTER(BytesInflightCompressed) - LOG_COUNTER(BytesInflightTotal) - LOG_COUNTER(MessagesInflight) - << " }"; - -#undef LOG_COUNTER - } -} - + << "Counters: {" + LOG_COUNTER(Errors) + LOG_COUNTER(CurrentSessionLifetimeMs) + LOG_COUNTER(BytesWritten) + LOG_COUNTER(MessagesWritten) + LOG_COUNTER(BytesWrittenCompressed) + LOG_COUNTER(BytesInflightUncompressed) + LOG_COUNTER(BytesInflightCompressed) + LOG_COUNTER(BytesInflightTotal) + LOG_COUNTER(MessagesInflight) + << " }"; + +#undef LOG_COUNTER + } +} + void TWriteSession::AbortImpl() { - if (!AtomicGet(Aborting)) { + if (!AtomicGet(Aborting)) { DbDriverState->Log << TLOG_DEBUG << LogPrefix() << "Write session: aborting"; - AtomicSet(Aborting, 1); - Cancel(ConnectContext); - Cancel(ConnectTimeoutContext); - Cancel(ConnectDelayContext); - ///Cancel(ClientContext); - if (Processor) - Processor->Cancel(); - } -} - -void TWriteSession::CloseImpl(EStatus statusCode, NYql::TIssues&& issues) { + AtomicSet(Aborting, 1); + Cancel(ConnectContext); + Cancel(ConnectTimeoutContext); + Cancel(ConnectDelayContext); + ///Cancel(ClientContext); + if (Processor) + Processor->Cancel(); + } +} + +void TWriteSession::CloseImpl(EStatus statusCode, NYql::TIssues&& issues) { DbDriverState->Log << TLOG_INFO << LogPrefix() << "Write session will now close"; EventsQueue->Close(TSessionClosedEvent(statusCode, std::move(issues))); - AbortImpl(); -} - -void TWriteSession::CloseImpl(EStatus statusCode, const TString& message) { - NYql::TIssues issues; - issues.AddIssue(message); - CloseImpl(statusCode, std::move(issues)); -} - -void TWriteSession::CloseImpl(TPlainStatus&& status) { + AbortImpl(); +} + +void TWriteSession::CloseImpl(EStatus statusCode, const TString& message) { + NYql::TIssues issues; + issues.AddIssue(message); + CloseImpl(statusCode, std::move(issues)); +} + +void TWriteSession::CloseImpl(TPlainStatus&& status) { DbDriverState->Log << TLOG_INFO << LogPrefix() << "Write session will now close"; EventsQueue->Close(TSessionClosedEvent(std::move(status))); - AbortImpl(); -} - -TWriteSession::~TWriteSession() { + AbortImpl(); +} + +TWriteSession::~TWriteSession() { DbDriverState->Log << TLOG_DEBUG << LogPrefix() << "Write session: destroy"; - bool needClose = false; - with_lock(Lock) { - if (!AtomicGet(Aborting)) { - CloseImpl(EStatus::SUCCESS, NYql::TIssues{}); - - needClose = !InitSeqNoSetDone && (InitSeqNoSetDone = true); - } - } - if (needClose) - InitSeqNoPromise.SetException("session closed"); -} - -TSimpleBlockingWriteSession::TSimpleBlockingWriteSession( - const TWriteSessionSettings& settings, + bool needClose = false; + with_lock(Lock) { + if (!AtomicGet(Aborting)) { + CloseImpl(EStatus::SUCCESS, NYql::TIssues{}); + + needClose = !InitSeqNoSetDone && (InitSeqNoSetDone = true); + } + } + if (needClose) + InitSeqNoPromise.SetException("session closed"); +} + +TSimpleBlockingWriteSession::TSimpleBlockingWriteSession( + const TWriteSessionSettings& settings, std::shared_ptr<TPersQueueClient::TImpl> client, - std::shared_ptr<TGRpcConnectionsImpl> connections, - TDbDriverStatePtr dbDriverState -) { - auto alteredSettings = settings; - alteredSettings.EventHandlers_.AcksHandler_ = [this](TWriteSessionEvent::TAcksEvent& event) {this->HandleAck(event); }; - alteredSettings.EventHandlers_.ReadyToAcceptHander_ = [this](TWriteSessionEvent::TReadyToAcceptEvent& event) - {this->HandleReady(event); }; - alteredSettings.EventHandlers_.SessionClosedHandler_ = [this](const TSessionClosedEvent& event) {this->HandleClosed(event); }; - - Writer = std::make_shared<TWriteSession>( + std::shared_ptr<TGRpcConnectionsImpl> connections, + TDbDriverStatePtr dbDriverState +) { + auto alteredSettings = settings; + alteredSettings.EventHandlers_.AcksHandler_ = [this](TWriteSessionEvent::TAcksEvent& event) {this->HandleAck(event); }; + alteredSettings.EventHandlers_.ReadyToAcceptHander_ = [this](TWriteSessionEvent::TReadyToAcceptEvent& event) + {this->HandleReady(event); }; + alteredSettings.EventHandlers_.SessionClosedHandler_ = [this](const TSessionClosedEvent& event) {this->HandleClosed(event); }; + + Writer = std::make_shared<TWriteSession>( alteredSettings, client, connections, dbDriverState - ); - Writer->Start(TDuration::Max()); -} - -ui64 TSimpleBlockingWriteSession::GetInitSeqNo() { - return Writer->GetInitSeqNo().GetValueSync(); -} - -bool TSimpleBlockingWriteSession::Write( - TStringBuf data, TMaybe<ui64> seqNo, TMaybe<TInstant> createTimestamp, const TDuration& blockTimeout -) { + ); + Writer->Start(TDuration::Max()); +} + +ui64 TSimpleBlockingWriteSession::GetInitSeqNo() { + return Writer->GetInitSeqNo().GetValueSync(); +} + +bool TSimpleBlockingWriteSession::Write( + TStringBuf data, TMaybe<ui64> seqNo, TMaybe<TInstant> createTimestamp, const TDuration& blockTimeout +) { if (!IsAlive()) return false; - auto continuationToken = WaitForToken(blockTimeout); - if (continuationToken.Defined()) { - Writer->Write(std::move(*continuationToken), std::move(data), seqNo, createTimestamp); - return true; - } - return false; -} - -TMaybe<TContinuationToken> TSimpleBlockingWriteSession::WaitForToken(const TDuration& timeout) { - auto startTime = TInstant::Now(); - TDuration remainingTime = timeout; - TMaybe<TContinuationToken> token = Nothing(); - while(!token.Defined() && remainingTime > TDuration::Zero()) { - with_lock(Lock) { - if (!ContinueTokens.empty()) { - token = std::move(ContinueTokens.front()); - ContinueTokens.pop(); - } - } + auto continuationToken = WaitForToken(blockTimeout); + if (continuationToken.Defined()) { + Writer->Write(std::move(*continuationToken), std::move(data), seqNo, createTimestamp); + return true; + } + return false; +} + +TMaybe<TContinuationToken> TSimpleBlockingWriteSession::WaitForToken(const TDuration& timeout) { + auto startTime = TInstant::Now(); + TDuration remainingTime = timeout; + TMaybe<TContinuationToken> token = Nothing(); + while(!token.Defined() && remainingTime > TDuration::Zero()) { + with_lock(Lock) { + if (!ContinueTokens.empty()) { + token = std::move(ContinueTokens.front()); + ContinueTokens.pop(); + } + } if (!IsAlive()) return Nothing(); - if (token.Defined()) { - return std::move(*token); - } - else { - remainingTime = timeout - (TInstant::Now() - startTime); - Sleep(Min(remainingTime, TDuration::MilliSeconds(100))); - } - } - return Nothing(); -} - -TWriterCounters::TPtr TSimpleBlockingWriteSession::GetCounters() { - return Writer->GetCounters(); -} - - -bool TSimpleBlockingWriteSession::IsAlive() const { - bool closed = false; - with_lock(Lock) { - closed = Closed; - } - return !closed; -} - -void TSimpleBlockingWriteSession::HandleAck(TWriteSessionEvent::TAcksEvent& event) { - Y_UNUSED(event); -} - -void TSimpleBlockingWriteSession::HandleReady(TWriteSessionEvent::TReadyToAcceptEvent& event) { - with_lock(Lock) { + if (token.Defined()) { + return std::move(*token); + } + else { + remainingTime = timeout - (TInstant::Now() - startTime); + Sleep(Min(remainingTime, TDuration::MilliSeconds(100))); + } + } + return Nothing(); +} + +TWriterCounters::TPtr TSimpleBlockingWriteSession::GetCounters() { + return Writer->GetCounters(); +} + + +bool TSimpleBlockingWriteSession::IsAlive() const { + bool closed = false; + with_lock(Lock) { + closed = Closed; + } + return !closed; +} + +void TSimpleBlockingWriteSession::HandleAck(TWriteSessionEvent::TAcksEvent& event) { + Y_UNUSED(event); +} + +void TSimpleBlockingWriteSession::HandleReady(TWriteSessionEvent::TReadyToAcceptEvent& event) { + with_lock(Lock) { ContinueTokens.emplace(std::move(event.ContinuationToken)); - } -} -void TSimpleBlockingWriteSession::HandleClosed(const TSessionClosedEvent&) { - with_lock(Lock) { - Closed = true; - } -} -bool TSimpleBlockingWriteSession::Close(TDuration closeTimeout) { - return Writer->Close(std::move(closeTimeout)); -} - -}; // namespace NYdb::NPersQueue + } +} +void TSimpleBlockingWriteSession::HandleClosed(const TSessionClosedEvent&) { + with_lock(Lock) { + Closed = true; + } +} +bool TSimpleBlockingWriteSession::Close(TDuration closeTimeout) { + return Writer->Close(std::move(closeTimeout)); +} + +}; // namespace NYdb::NPersQueue diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/write_session.h b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/write_session.h index 9b3d8f906e..324550bb02 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/write_session.h +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/write_session.h @@ -1,5 +1,5 @@ -#pragma once - +#pragma once + #include "common.h" #include "persqueue_impl.h" @@ -7,27 +7,27 @@ #include <util/generic/buffer.h> - -namespace NYdb::NPersQueue { - -inline const TString& GetCodecId(const ECodec codec) { - static THashMap<ECodec, TString> idByCodec{ - {ECodec::RAW, TString(1, '\0')}, - {ECodec::GZIP, "\1"}, - {ECodec::LZOP, "\2"}, - {ECodec::ZSTD, "\3"} - }; - Y_VERIFY(idByCodec.contains(codec)); - return idByCodec[codec]; -} - + +namespace NYdb::NPersQueue { + +inline const TString& GetCodecId(const ECodec codec) { + static THashMap<ECodec, TString> idByCodec{ + {ECodec::RAW, TString(1, '\0')}, + {ECodec::GZIP, "\1"}, + {ECodec::LZOP, "\2"}, + {ECodec::ZSTD, "\3"} + }; + Y_VERIFY(idByCodec.contains(codec)); + return idByCodec[codec]; +} + class TWriteSessionEventsQueue : public TBaseSessionEventsQueue<TWriteSessionSettings, TWriteSessionEvent::TEvent> { using TParent = TBaseSessionEventsQueue<TWriteSessionSettings, TWriteSessionEvent::TEvent>; -public: - TWriteSessionEventsQueue(const TWriteSessionSettings& settings) - : TParent(settings) - {} - +public: + TWriteSessionEventsQueue(const TWriteSessionSettings& settings) + : TParent(settings) + {} + void PushEvent(TEventInfo eventInfo) { if (Closed || ApplyHandler(eventInfo)) { return; @@ -98,33 +98,33 @@ public: waiter.Signal(); } -private: +private: struct THandlersVisitor : public TParent::TBaseHandlersVisitor { using TParent::TBaseHandlersVisitor::TBaseHandlersVisitor; -#define DECLARE_HANDLER(type, handler, answer) \ - bool operator()(type& event) { \ - if (Settings.EventHandlers_.handler) { \ - Settings.EventHandlers_.handler(event); \ - return answer; \ - } \ - return false; \ - } \ - /**/ - DECLARE_HANDLER(TWriteSessionEvent::TAcksEvent, AcksHandler_, true); - DECLARE_HANDLER(TWriteSessionEvent::TReadyToAcceptEvent, ReadyToAcceptHander_, true); - DECLARE_HANDLER(TSessionClosedEvent, SessionClosedHandler_, false); // Not applied - -#undef DECLARE_HANDLER +#define DECLARE_HANDLER(type, handler, answer) \ + bool operator()(type& event) { \ + if (Settings.EventHandlers_.handler) { \ + Settings.EventHandlers_.handler(event); \ + return answer; \ + } \ + return false; \ + } \ + /**/ + DECLARE_HANDLER(TWriteSessionEvent::TAcksEvent, AcksHandler_, true); + DECLARE_HANDLER(TWriteSessionEvent::TReadyToAcceptEvent, ReadyToAcceptHander_, true); + DECLARE_HANDLER(TSessionClosedEvent, SessionClosedHandler_, false); // Not applied + +#undef DECLARE_HANDLER bool Visit() { - return std::visit(*this, EventInfo.Event); - } - - }; - + return std::visit(*this, EventInfo.Event); + } + + }; + bool ApplyHandler(TEventInfo& eventInfo) { THandlersVisitor visitor(Settings, eventInfo); - return visitor.Visit(); - } + return visitor.Visit(); + } TEventInfo GetEventImpl() { // Assumes that we're under lock and that the event queue has events. Y_ASSERT(HasEventsImpl()); @@ -137,98 +137,98 @@ private: Y_ASSERT(CloseEvent); return {*CloseEvent}; } -}; - -struct TMemoryUsageChange { - bool WasOk; //!< MemoryUsage <= Config.MaxMemoryUsage_ before update - bool NowOk; //!< Same, only after update -}; - -namespace NTests { - class TSimpleWriteSessionTestAdapter; -} - -class TWriteSession : public IWriteSession, +}; + +struct TMemoryUsageChange { + bool WasOk; //!< MemoryUsage <= Config.MaxMemoryUsage_ before update + bool NowOk; //!< Same, only after update +}; + +namespace NTests { + class TSimpleWriteSessionTestAdapter; +} + +class TWriteSession : public IWriteSession, public std::enable_shared_from_this<TWriteSession> { -private: - friend class TSimpleBlockingWriteSession; +private: + friend class TSimpleBlockingWriteSession; friend class TPersQueueClient; - friend class NTests::TSimpleWriteSessionTestAdapter; - - using TClientMessage = Ydb::PersQueue::V1::StreamingWriteClientMessage; - using TServerMessage = Ydb::PersQueue::V1::StreamingWriteServerMessage; - using IWriteSessionConnectionProcessorFactory = + friend class NTests::TSimpleWriteSessionTestAdapter; + + using TClientMessage = Ydb::PersQueue::V1::StreamingWriteClientMessage; + using TServerMessage = Ydb::PersQueue::V1::StreamingWriteServerMessage; + using IWriteSessionConnectionProcessorFactory = TPersQueueClient::TImpl::IWriteSessionConnectionProcessorFactory; - using IProcessor = IWriteSessionConnectionProcessorFactory::IProcessor; - - struct TMessage { - ui64 SeqNo; - TInstant CreatedAt; - TStringBuf DataRef; + using IProcessor = IWriteSessionConnectionProcessorFactory::IProcessor; + + struct TMessage { + ui64 SeqNo; + TInstant CreatedAt; + TStringBuf DataRef; TMaybe<ECodec> Codec; ui32 OriginalSize; // only for coded messages TMessage(ui64 seqNo, const TInstant& createdAt, TStringBuf data, TMaybe<ECodec> codec = {}, ui32 originalSize = 0) - : SeqNo(seqNo) - , CreatedAt(createdAt) - , DataRef(data) + : SeqNo(seqNo) + , CreatedAt(createdAt) + , DataRef(data) , Codec(codec) , OriginalSize(originalSize) - {} - }; - - struct TMessageBatch { - TBuffer Data; - TVector<TMessage> Messages; - ui64 CurrentSize = 0; - TInstant StartedAt = TInstant::Zero(); - bool Acquired = false; - bool FlushRequested = false; + {} + }; + + struct TMessageBatch { + TBuffer Data; + TVector<TMessage> Messages; + ui64 CurrentSize = 0; + TInstant StartedAt = TInstant::Zero(); + bool Acquired = false; + bool FlushRequested = false; void Add(ui64 seqNo, const TInstant& createdAt, TStringBuf data, TMaybe<ECodec> codec, ui32 originalSize) { - if (StartedAt == TInstant::Zero()) - StartedAt = TInstant::Now(); + if (StartedAt == TInstant::Zero()) + StartedAt = TInstant::Now(); CurrentSize += codec ? originalSize : data.size(); Messages.emplace_back(seqNo, createdAt, data, codec, originalSize); - Acquired = false; - } - + Acquired = false; + } + bool HasCodec() const { return Messages.empty() ? false : Messages.front().Codec.Defined(); } - bool Acquire() { - if (Acquired || Messages.empty()) - return false; - auto currSize = Data.size(); - Data.Append(Messages.back().DataRef.data(), Messages.back().DataRef.size()); - Messages.back().DataRef = TStringBuf(Data.data() + currSize, Data.size() - currSize); - Acquired = true; - return true; - } - - bool Empty() const noexcept { - return CurrentSize == 0 && Messages.empty(); - } - - void Reset() { - StartedAt = TInstant::Zero(); - Messages.clear(); - Data.Clear(); - Acquired = false; - CurrentSize = 0; - FlushRequested = false; - } - }; - - struct TBlock { - size_t Offset = 0; //!< First message sequence number in the block - size_t MessageCount = 0; - size_t PartNumber = 0; - size_t OriginalSize = 0; - size_t OriginalMemoryUsage = 0; - TString CodecID = GetCodecId(ECodec::RAW); + bool Acquire() { + if (Acquired || Messages.empty()) + return false; + auto currSize = Data.size(); + Data.Append(Messages.back().DataRef.data(), Messages.back().DataRef.size()); + Messages.back().DataRef = TStringBuf(Data.data() + currSize, Data.size() - currSize); + Acquired = true; + return true; + } + + bool Empty() const noexcept { + return CurrentSize == 0 && Messages.empty(); + } + + void Reset() { + StartedAt = TInstant::Zero(); + Messages.clear(); + Data.Clear(); + Acquired = false; + CurrentSize = 0; + FlushRequested = false; + } + }; + + struct TBlock { + size_t Offset = 0; //!< First message sequence number in the block + size_t MessageCount = 0; + size_t PartNumber = 0; + size_t OriginalSize = 0; + size_t OriginalMemoryUsage = 0; + TString CodecID = GetCodecId(ECodec::RAW); mutable TVector<TStringBuf> OriginalDataRefs; mutable TBuffer Data; - bool Compressed = false; + bool Compressed = false; mutable bool Valid = true; TBlock& operator=(TBlock&&) = default; @@ -250,69 +250,69 @@ private: rhs.Data.Clear(); rhs.OriginalDataRefs.clear(); } - }; - - struct TOriginalMessage { - ui64 SeqNo; - TInstant CreatedAt; - size_t Size; - TOriginalMessage(const ui64 sequenceNumber, const TInstant createdAt, const size_t size) - : SeqNo(sequenceNumber) - , CreatedAt(createdAt) - , Size(size) - {} - }; - - //! Block comparer, makes block with smallest offset (first sequence number) appear on top of the PackedMessagesToSend priority queue - struct Greater { - bool operator() (const TBlock& lhs, const TBlock& rhs) { - return lhs.Offset > rhs.Offset; - } - }; - - struct THandleResult { - bool DoRestart = false; - TDuration StartDelay = TDuration::Zero(); - bool DoStop = false; - bool DoSetSeqNo = false; - }; - struct TProcessSrvMessageResult { - THandleResult HandleResult; - TMaybe<ui64> InitSeqNo; - TVector<TWriteSessionEvent::TEvent> Events; - bool Ok = true; - }; - - THandleResult OnErrorImpl(NYdb::TPlainStatus&& status); // true - should Start(), false - should Close(), empty - no action - -public: - TWriteSession(const TWriteSessionSettings& settings, + }; + + struct TOriginalMessage { + ui64 SeqNo; + TInstant CreatedAt; + size_t Size; + TOriginalMessage(const ui64 sequenceNumber, const TInstant createdAt, const size_t size) + : SeqNo(sequenceNumber) + , CreatedAt(createdAt) + , Size(size) + {} + }; + + //! Block comparer, makes block with smallest offset (first sequence number) appear on top of the PackedMessagesToSend priority queue + struct Greater { + bool operator() (const TBlock& lhs, const TBlock& rhs) { + return lhs.Offset > rhs.Offset; + } + }; + + struct THandleResult { + bool DoRestart = false; + TDuration StartDelay = TDuration::Zero(); + bool DoStop = false; + bool DoSetSeqNo = false; + }; + struct TProcessSrvMessageResult { + THandleResult HandleResult; + TMaybe<ui64> InitSeqNo; + TVector<TWriteSessionEvent::TEvent> Events; + bool Ok = true; + }; + + THandleResult OnErrorImpl(NYdb::TPlainStatus&& status); // true - should Start(), false - should Close(), empty - no action + +public: + TWriteSession(const TWriteSessionSettings& settings, std::shared_ptr<TPersQueueClient::TImpl> client, - std::shared_ptr<TGRpcConnectionsImpl> connections, - TDbDriverStatePtr dbDriverState); - - TMaybe<TWriteSessionEvent::TEvent> GetEvent(bool block = false) override; - TVector<TWriteSessionEvent::TEvent> GetEvents(bool block = false, - TMaybe<size_t> maxEventsCount = Nothing()) override; - NThreading::TFuture<ui64> GetInitSeqNo() override; - - void Write(TContinuationToken&& continuationToken, TStringBuf data, - TMaybe<ui64> seqNo = Nothing(), TMaybe<TInstant> createTimestamp = Nothing()) override; - + std::shared_ptr<TGRpcConnectionsImpl> connections, + TDbDriverStatePtr dbDriverState); + + TMaybe<TWriteSessionEvent::TEvent> GetEvent(bool block = false) override; + TVector<TWriteSessionEvent::TEvent> GetEvents(bool block = false, + TMaybe<size_t> maxEventsCount = Nothing()) override; + NThreading::TFuture<ui64> GetInitSeqNo() override; + + void Write(TContinuationToken&& continuationToken, TStringBuf data, + TMaybe<ui64> seqNo = Nothing(), TMaybe<TInstant> createTimestamp = Nothing()) override; + void WriteEncoded(TContinuationToken&& continuationToken, TStringBuf data, ECodec codec, ui32 originalSize, TMaybe<ui64> seqNo = Nothing(), TMaybe<TInstant> createTimestamp = Nothing()) override; - NThreading::TFuture<void> WaitEvent() override; - - // Empty maybe - block till all work is done. Otherwise block at most at closeTimeout duration. - bool Close(TDuration closeTimeout = TDuration::Max()) override; - + NThreading::TFuture<void> WaitEvent() override; + + // Empty maybe - block till all work is done. Otherwise block at most at closeTimeout duration. + bool Close(TDuration closeTimeout = TDuration::Max()) override; + TWriterCounters::TPtr GetCounters() override {Y_FAIL("Unimplemented"); } //ToDo - unimplemented; - - ~TWriteSession(); // will not call close - destroy everything without acks - -private: + + ~TWriteSession(); // will not call close - destroy everything without acks + +private: TString LogPrefix() const; @@ -321,152 +321,152 @@ private: void WriteInternal(TContinuationToken&& continuationToken, TStringBuf data, TMaybe<ECodec> codec, ui32 originalSize, TMaybe<ui64> seqNo = Nothing(), TMaybe<TInstant> createTimestamp = Nothing()); - void FlushWriteIfRequiredImpl(); - size_t WriteBatchImpl(); - void Start(const TDuration& delay); - void InitWriter(); - - void DoCdsRequest(TDuration delay = TDuration::Zero()); - void OnCdsResponse(TStatus& status, const Ydb::PersQueue::ClusterDiscovery::DiscoverClustersResult& result); - void OnConnect(TPlainStatus&& st, typename IProcessor::TPtr&& processor, - const NGrpc::IQueueClientContextPtr& connectContext); - void OnConnectTimeout(const NGrpc::IQueueClientContextPtr& connectTimeoutContext); - void ResetForRetryImpl(); - THandleResult RestartImpl(const TPlainStatus& status); + void FlushWriteIfRequiredImpl(); + size_t WriteBatchImpl(); + void Start(const TDuration& delay); + void InitWriter(); + + void DoCdsRequest(TDuration delay = TDuration::Zero()); + void OnCdsResponse(TStatus& status, const Ydb::PersQueue::ClusterDiscovery::DiscoverClustersResult& result); + void OnConnect(TPlainStatus&& st, typename IProcessor::TPtr&& processor, + const NGrpc::IQueueClientContextPtr& connectContext); + void OnConnectTimeout(const NGrpc::IQueueClientContextPtr& connectTimeoutContext); + void ResetForRetryImpl(); + THandleResult RestartImpl(const TPlainStatus& status); void DoConnect(const TDuration& delay, const TString& endpoint); - void InitImpl(); - void ReadFromProcessor(); // Assumes that we're under lock. - void WriteToProcessorImpl(TClientMessage&& req); // Assumes that we're under lock. - void OnReadDone(NGrpc::TGrpcStatus&& grpcStatus, size_t connectionGeneration); - void OnWriteDone(NGrpc::TGrpcStatus&& status, size_t connectionGeneration); - TProcessSrvMessageResult ProcessServerMessageImpl(); - TMemoryUsageChange OnMemoryUsageChangedImpl(i64 diff); - void CompressImpl(TBlock&& block); - void OnCompressed(TBlock&& block, bool isSyncCompression=false); - TMemoryUsageChange OnCompressedImpl(TBlock&& block); - - //TString GetDebugIdentity() const; - Ydb::PersQueue::V1::StreamingWriteClientMessage GetInitClientMessage(); - bool CleanupOnAcknowledged(ui64 sequenceNumber); - bool IsReadyToSendNextImpl() const; - ui64 GetNextSeqNoImpl(const TMaybe<ui64>& seqNo); - void SendImpl(); + void InitImpl(); + void ReadFromProcessor(); // Assumes that we're under lock. + void WriteToProcessorImpl(TClientMessage&& req); // Assumes that we're under lock. + void OnReadDone(NGrpc::TGrpcStatus&& grpcStatus, size_t connectionGeneration); + void OnWriteDone(NGrpc::TGrpcStatus&& status, size_t connectionGeneration); + TProcessSrvMessageResult ProcessServerMessageImpl(); + TMemoryUsageChange OnMemoryUsageChangedImpl(i64 diff); + void CompressImpl(TBlock&& block); + void OnCompressed(TBlock&& block, bool isSyncCompression=false); + TMemoryUsageChange OnCompressedImpl(TBlock&& block); + + //TString GetDebugIdentity() const; + Ydb::PersQueue::V1::StreamingWriteClientMessage GetInitClientMessage(); + bool CleanupOnAcknowledged(ui64 sequenceNumber); + bool IsReadyToSendNextImpl() const; + ui64 GetNextSeqNoImpl(const TMaybe<ui64>& seqNo); + void SendImpl(); void AbortImpl(); - void CloseImpl(EStatus statusCode, NYql::TIssues&& issues); - void CloseImpl(EStatus statusCode, const TString& message); - void CloseImpl(TPlainStatus&& status); - - void OnErrorResolved() { - RetryState = nullptr; - } - void CheckHandleResultImpl(THandleResult& result); - void ProcessHandleResult(THandleResult& result); - void HandleWakeUpImpl(); - void UpdateTimedCountersImpl(); - -private: - TWriteSessionSettings Settings; + void CloseImpl(EStatus statusCode, NYql::TIssues&& issues); + void CloseImpl(EStatus statusCode, const TString& message); + void CloseImpl(TPlainStatus&& status); + + void OnErrorResolved() { + RetryState = nullptr; + } + void CheckHandleResultImpl(THandleResult& result); + void ProcessHandleResult(THandleResult& result); + void HandleWakeUpImpl(); + void UpdateTimedCountersImpl(); + +private: + TWriteSessionSettings Settings; std::shared_ptr<TPersQueueClient::TImpl> Client; - std::shared_ptr<TGRpcConnectionsImpl> Connections; - TString TargetCluster; - TString InitialCluster; - TString CurrentCluster; - bool OnSeqNoShift = false; + std::shared_ptr<TGRpcConnectionsImpl> Connections; + TString TargetCluster; + TString InitialCluster; + TString CurrentCluster; + bool OnSeqNoShift = false; TString PreferredClusterByCDS; - std::shared_ptr<IWriteSessionConnectionProcessorFactory> ConnectionFactory; - TDbDriverStatePtr DbDriverState; + std::shared_ptr<IWriteSessionConnectionProcessorFactory> ConnectionFactory; + TDbDriverStatePtr DbDriverState; TStringType PrevToken; bool UpdateTokenInProgress = false; TInstant LastTokenUpdate = TInstant::Zero(); - std::shared_ptr<TWriteSessionEventsQueue> EventsQueue; - NGrpc::IQueueClientContextPtr ClientContext; // Common client context. - NGrpc::IQueueClientContextPtr ConnectContext; - NGrpc::IQueueClientContextPtr ConnectTimeoutContext; - NGrpc::IQueueClientContextPtr ConnectDelayContext; - size_t ConnectionGeneration = 0; - size_t ConnectionAttemptsDone = 0; - TAdaptiveLock Lock; - IProcessor::TPtr Processor; - IRetryState::TPtr RetryState; // Current retry state (if now we are (re)connecting). - std::shared_ptr<TServerMessage> ServerMessage; // Server message to write server response to. - - TString SessionId; + std::shared_ptr<TWriteSessionEventsQueue> EventsQueue; + NGrpc::IQueueClientContextPtr ClientContext; // Common client context. + NGrpc::IQueueClientContextPtr ConnectContext; + NGrpc::IQueueClientContextPtr ConnectTimeoutContext; + NGrpc::IQueueClientContextPtr ConnectDelayContext; + size_t ConnectionGeneration = 0; + size_t ConnectionAttemptsDone = 0; + TAdaptiveLock Lock; + IProcessor::TPtr Processor; + IRetryState::TPtr RetryState; // Current retry state (if now we are (re)connecting). + std::shared_ptr<TServerMessage> ServerMessage; // Server message to write server response to. + + TString SessionId; IExecutor::TPtr Executor; IExecutor::TPtr CompressionExecutor; - size_t MemoryUsage = 0; //!< Estimated amount of memory used - - TMessageBatch CurrentBatch; - + size_t MemoryUsage = 0; //!< Estimated amount of memory used + + TMessageBatch CurrentBatch; + std::queue<TOriginalMessage> OriginalMessagesToSend; std::priority_queue<TBlock, std::vector<TBlock>, Greater> PackedMessagesToSend; - //! Messages that are sent but yet not acknowledged + //! Messages that are sent but yet not acknowledged std::queue<TOriginalMessage> SentOriginalMessages; std::queue<TBlock> SentPackedMessage; - - const size_t MaxBlockSize = std::numeric_limits<size_t>::max(); - const size_t MaxBlockMessageCount = 1; //!< Max message count that can be packed into a single block. In block version 0 is equal to 1 for compatibility - bool Connected = false; - bool Started = false; - TAtomic Aborting = 0; - bool SessionEstablished = false; - ui32 PartitionId = 0; - ui64 LastSeqNo = 0; - ui64 MinUnsentSeqNo = 0; - ui64 SeqNoShift = 0; - TMaybe<bool> AutoSeqNoMode; + + const size_t MaxBlockSize = std::numeric_limits<size_t>::max(); + const size_t MaxBlockMessageCount = 1; //!< Max message count that can be packed into a single block. In block version 0 is equal to 1 for compatibility + bool Connected = false; + bool Started = false; + TAtomic Aborting = 0; + bool SessionEstablished = false; + ui32 PartitionId = 0; + ui64 LastSeqNo = 0; + ui64 MinUnsentSeqNo = 0; + ui64 SeqNoShift = 0; + TMaybe<bool> AutoSeqNoMode; bool ValidateSeqNoMode = false; - - NThreading::TPromise<ui64> InitSeqNoPromise; - bool InitSeqNoSetDone = false; - TInstant SessionStartedTs; - TInstant LastCountersUpdateTs = TInstant::Zero(); - TInstant LastCountersLogTs; - TWriterCounters::TPtr Counters; - TDuration WakeupInterval; - -protected: - ui64 MessagesAcquired = 0; -}; - -class TSimpleBlockingWriteSession : public ISimpleBlockingWriteSession { - friend class NTests::TSimpleWriteSessionTestAdapter; - -private: - using TClientMessage = TWriteSession::TClientMessage; - using TServerMessage = TWriteSession::TServerMessage; - using IWriteSessionConnectionProcessorFactory = TWriteSession::IWriteSessionConnectionProcessorFactory; - using IProcessor = TWriteSession::IProcessor; - -public: - TSimpleBlockingWriteSession( - const TWriteSessionSettings& settings, + + NThreading::TPromise<ui64> InitSeqNoPromise; + bool InitSeqNoSetDone = false; + TInstant SessionStartedTs; + TInstant LastCountersUpdateTs = TInstant::Zero(); + TInstant LastCountersLogTs; + TWriterCounters::TPtr Counters; + TDuration WakeupInterval; + +protected: + ui64 MessagesAcquired = 0; +}; + +class TSimpleBlockingWriteSession : public ISimpleBlockingWriteSession { + friend class NTests::TSimpleWriteSessionTestAdapter; + +private: + using TClientMessage = TWriteSession::TClientMessage; + using TServerMessage = TWriteSession::TServerMessage; + using IWriteSessionConnectionProcessorFactory = TWriteSession::IWriteSessionConnectionProcessorFactory; + using IProcessor = TWriteSession::IProcessor; + +public: + TSimpleBlockingWriteSession( + const TWriteSessionSettings& settings, std::shared_ptr<TPersQueueClient::TImpl> client, - std::shared_ptr<TGRpcConnectionsImpl> connections, - TDbDriverStatePtr dbDriverState); - - bool Write(TStringBuf data, TMaybe<ui64> seqNo = Nothing(), TMaybe<TInstant> createTimestamp = Nothing(), - const TDuration& blockTimeout = TDuration::Max()) override; - - ui64 GetInitSeqNo() override; - - bool Close(TDuration closeTimeout = TDuration::Max()) override; - bool IsAlive() const override; - - TWriterCounters::TPtr GetCounters() override; - -protected: - std::shared_ptr<TWriteSession> Writer; - -private: - TMaybe<TContinuationToken> WaitForToken(const TDuration& timeout); - void HandleAck(TWriteSessionEvent::TAcksEvent&); - void HandleReady(TWriteSessionEvent::TReadyToAcceptEvent&); - void HandleClosed(const TSessionClosedEvent&); - - TAdaptiveLock Lock; + std::shared_ptr<TGRpcConnectionsImpl> connections, + TDbDriverStatePtr dbDriverState); + + bool Write(TStringBuf data, TMaybe<ui64> seqNo = Nothing(), TMaybe<TInstant> createTimestamp = Nothing(), + const TDuration& blockTimeout = TDuration::Max()) override; + + ui64 GetInitSeqNo() override; + + bool Close(TDuration closeTimeout = TDuration::Max()) override; + bool IsAlive() const override; + + TWriterCounters::TPtr GetCounters() override; + +protected: + std::shared_ptr<TWriteSession> Writer; + +private: + TMaybe<TContinuationToken> WaitForToken(const TDuration& timeout); + void HandleAck(TWriteSessionEvent::TAcksEvent&); + void HandleReady(TWriteSessionEvent::TReadyToAcceptEvent&); + void HandleClosed(const TSessionClosedEvent&); + + TAdaptiveLock Lock; std::queue<TContinuationToken> ContinueTokens; - bool Closed = false; -}; - - -}; // namespace NYdb::NPersQueue + bool Closed = false; +}; + + +}; // namespace NYdb::NPersQueue diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/persqueue.h b/ydb/public/sdk/cpp/client/ydb_persqueue_core/persqueue.h index 161a4074ee..ee9aa75a0a 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/persqueue.h +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/persqueue.h @@ -2,9 +2,9 @@ #include <ydb/public/api/grpc/draft/ydb_persqueue_v1.grpc.pb.h> #include <ydb/public/sdk/cpp/client/ydb_driver/driver.h> -#include <library/cpp/monlib/dynamic_counters/counters.h> -#include <library/cpp/logger/log.h> - +#include <library/cpp/monlib/dynamic_counters/counters.h> +#include <library/cpp/logger/log.h> + #include <util/datetime/base.h> #include <util/generic/hash.h> #include <util/generic/maybe.h> @@ -320,22 +320,22 @@ struct TSessionClosedEvent : public TStatus { TString DebugString() const; }; -struct TWriteStat : public TThrRefBase { - TDuration WriteTime; - TDuration TotalTimeInPartitionQueue; - TDuration PartitionQuotedTime; - TDuration TopicQuotedTime; - using TPtr = TIntrusivePtr<TWriteStat>; -}; - -//enum class ECodec { -// EWC_RAW, -// EWC_GZIP, -// EWC_LZOP, -// EWC_ZSTD -//}; - - +struct TWriteStat : public TThrRefBase { + TDuration WriteTime; + TDuration TotalTimeInPartitionQueue; + TDuration PartitionQuotedTime; + TDuration TopicQuotedTime; + using TPtr = TIntrusivePtr<TWriteStat>; +}; + +//enum class ECodec { +// EWC_RAW, +// EWC_GZIP, +// EWC_LZOP, +// EWC_ZSTD +//}; + + enum class EClusterDiscoveryMode { Auto = 0, // enables cluster discovery only for hostname "logbroker.yandex.net" and "logbroker-prestable.yandex.net" On, @@ -343,43 +343,43 @@ enum class EClusterDiscoveryMode { }; class TContinuationToken : public TMoveOnly { - friend class TWriteSession; -private: + friend class TWriteSession; +private: TContinuationToken() = default; -}; - -struct TWriterCounters : public TThrRefBase { - using TSelf = TWriterCounters; - using TPtr = TIntrusivePtr<TSelf>; - +}; + +struct TWriterCounters : public TThrRefBase { + using TSelf = TWriterCounters; + using TPtr = TIntrusivePtr<TSelf>; + explicit TWriterCounters(const TIntrusivePtr<NMonitoring::TDynamicCounters>& counters); - - NMonitoring::TDynamicCounters::TCounterPtr Errors; - NMonitoring::TDynamicCounters::TCounterPtr CurrentSessionLifetimeMs; - - NMonitoring::TDynamicCounters::TCounterPtr BytesWritten; - NMonitoring::TDynamicCounters::TCounterPtr MessagesWritten; - NMonitoring::TDynamicCounters::TCounterPtr BytesWrittenCompressed; - - NMonitoring::TDynamicCounters::TCounterPtr BytesInflightUncompressed; - NMonitoring::TDynamicCounters::TCounterPtr BytesInflightCompressed; - NMonitoring::TDynamicCounters::TCounterPtr BytesInflightTotal; - NMonitoring::TDynamicCounters::TCounterPtr MessagesInflight; - - //! Histograms reporting % usage of memory limit in time. - //! Provides a histogram looking like: 10% : 100ms, 20%: 300ms, ... 50%: 200ms, ... 100%: 50ms - //! Which means that < 10% memory usage was observed for 100ms during the period and 50% usage was observed for 200ms - //! Used to monitor if the writer successfully deals with data flow provided. Larger values in higher buckets - //! mean that writer is close to overflow (or being overflown) for major periods of time - //! 3 histograms stand for: - //! Total memory usage: - NMonitoring::THistogramPtr TotalBytesInflightUsageByTime; - //! Memory usage by messages waiting for comression: - NMonitoring::THistogramPtr UncompressedBytesInflightUsageByTime; - //! Memory usage by compressed messages pending for write: - NMonitoring::THistogramPtr CompressedBytesInflightUsageByTime; -}; - + + NMonitoring::TDynamicCounters::TCounterPtr Errors; + NMonitoring::TDynamicCounters::TCounterPtr CurrentSessionLifetimeMs; + + NMonitoring::TDynamicCounters::TCounterPtr BytesWritten; + NMonitoring::TDynamicCounters::TCounterPtr MessagesWritten; + NMonitoring::TDynamicCounters::TCounterPtr BytesWrittenCompressed; + + NMonitoring::TDynamicCounters::TCounterPtr BytesInflightUncompressed; + NMonitoring::TDynamicCounters::TCounterPtr BytesInflightCompressed; + NMonitoring::TDynamicCounters::TCounterPtr BytesInflightTotal; + NMonitoring::TDynamicCounters::TCounterPtr MessagesInflight; + + //! Histograms reporting % usage of memory limit in time. + //! Provides a histogram looking like: 10% : 100ms, 20%: 300ms, ... 50%: 200ms, ... 100%: 50ms + //! Which means that < 10% memory usage was observed for 100ms during the period and 50% usage was observed for 200ms + //! Used to monitor if the writer successfully deals with data flow provided. Larger values in higher buckets + //! mean that writer is close to overflow (or being overflown) for major periods of time + //! 3 histograms stand for: + //! Total memory usage: + NMonitoring::THistogramPtr TotalBytesInflightUsageByTime; + //! Memory usage by messages waiting for comression: + NMonitoring::THistogramPtr UncompressedBytesInflightUsageByTime; + //! Memory usage by compressed messages pending for write: + NMonitoring::THistogramPtr CompressedBytesInflightUsageByTime; +}; + struct TReaderCounters : public TThrRefBase { using TSelf = TReaderCounters; using TPtr = TIntrusivePtr<TSelf>; @@ -946,15 +946,15 @@ struct IRetryPolicy { }; class IExecutor : public TThrRefBase { -public: +public: using TPtr = TIntrusivePtr<IExecutor>; using TFunction = std::function<void()>; - + // Is executor asynchronous. virtual bool IsAsync() const = 0; - + // Post function to execute. - virtual void Post(TFunction&& f) = 0; + virtual void Post(TFunction&& f) = 0; // Start method. // This method is idempotent. @@ -974,156 +974,156 @@ private: private: bool Started = false; TAdaptiveLock StartLock; -}; +}; IExecutor::TPtr CreateThreadPoolExecutorAdapter(std::shared_ptr<IThreadPool> threadPool); // Thread pool is expected to have been started. IExecutor::TPtr CreateThreadPoolExecutor(size_t threads); - -//! Events for write session. -struct TWriteSessionEvent { - - //! Event with acknowledge for written messages. - struct TWriteAck { - //! Write result. - enum EEventState { - EES_WRITTEN, //! Successfully written. - EES_ALREADY_WRITTEN, //! Skipped on SeqNo deduplication. - EES_DISCARDED //! In case of destruction of writer or retry policy discarded future retries in this writer. - }; - //! Details of successfully written message. - struct TWrittenMessageDetails { - ui64 Offset; - ui64 PartitionId; - }; - //! Same SeqNo as provided on write. - ui64 SeqNo; - EEventState State; - //! Filled only for EES_WRITTEN. Empty for ALREADY and DISCARDED. - TMaybe<TWrittenMessageDetails> Details; - //! Write stats from server. See TWriteStat. nullptr for DISCARDED event. - TWriteStat::TPtr Stat; - - }; - - struct TAcksEvent { - //! Acks could be batched from several WriteBatch/Write requests. - //! Acks for messages from one WriteBatch request could be emitted as several TAcksEvents - - //! they are provided to client as soon as possible. - TVector<TWriteAck> Acks; + +//! Events for write session. +struct TWriteSessionEvent { + + //! Event with acknowledge for written messages. + struct TWriteAck { + //! Write result. + enum EEventState { + EES_WRITTEN, //! Successfully written. + EES_ALREADY_WRITTEN, //! Skipped on SeqNo deduplication. + EES_DISCARDED //! In case of destruction of writer or retry policy discarded future retries in this writer. + }; + //! Details of successfully written message. + struct TWrittenMessageDetails { + ui64 Offset; + ui64 PartitionId; + }; + //! Same SeqNo as provided on write. + ui64 SeqNo; + EEventState State; + //! Filled only for EES_WRITTEN. Empty for ALREADY and DISCARDED. + TMaybe<TWrittenMessageDetails> Details; + //! Write stats from server. See TWriteStat. nullptr for DISCARDED event. + TWriteStat::TPtr Stat; + + }; + + struct TAcksEvent { + //! Acks could be batched from several WriteBatch/Write requests. + //! Acks for messages from one WriteBatch request could be emitted as several TAcksEvents - + //! they are provided to client as soon as possible. + TVector<TWriteAck> Acks; TString DebugString() const; - }; - - //! Indicates that a writer is ready to accept new message(s). + }; + + //! Indicates that a writer is ready to accept new message(s). //! Continuation token should be kept and then used in write methods. - struct TReadyToAcceptEvent { + struct TReadyToAcceptEvent { TContinuationToken ContinuationToken; TString DebugString() const; - }; - + }; + using TEvent = std::variant<TAcksEvent, TReadyToAcceptEvent, TSessionClosedEvent>; -}; - +}; + //! Event debug string. TString DebugString(const TWriteSessionEvent::TEvent& event); -using TSessionClosedHandler = std::function<void(const TSessionClosedEvent&)>; - -//! Settings for write session. -struct TWriteSessionSettings : public TRequestSettings<TWriteSessionSettings> { - using TSelf = TWriteSessionSettings; - - TWriteSessionSettings() = default; - TWriteSessionSettings(const TWriteSessionSettings&) = default; - TWriteSessionSettings(TWriteSessionSettings&&) = default; - TWriteSessionSettings(const TString& path, const TString& messageGroupId) { - Path(path); - MessageGroupId(messageGroupId); - } - - TWriteSessionSettings& operator=(const TWriteSessionSettings&) = default; - TWriteSessionSettings& operator=(TWriteSessionSettings&&) = default; - - //! Path of topic to write. - FLUENT_SETTING(TString, Path); - - //! MessageGroupId (aka SourceId) to use. - FLUENT_SETTING(TString, MessageGroupId); - - //! Write to an exact partition group. Generally server assigns partition group automatically. - //! Using this option is not recommended unless you know for sure why you need it. - FLUENT_SETTING_OPTIONAL(ui32, PartitionGroupId); - +using TSessionClosedHandler = std::function<void(const TSessionClosedEvent&)>; + +//! Settings for write session. +struct TWriteSessionSettings : public TRequestSettings<TWriteSessionSettings> { + using TSelf = TWriteSessionSettings; + + TWriteSessionSettings() = default; + TWriteSessionSettings(const TWriteSessionSettings&) = default; + TWriteSessionSettings(TWriteSessionSettings&&) = default; + TWriteSessionSettings(const TString& path, const TString& messageGroupId) { + Path(path); + MessageGroupId(messageGroupId); + } + + TWriteSessionSettings& operator=(const TWriteSessionSettings&) = default; + TWriteSessionSettings& operator=(TWriteSessionSettings&&) = default; + + //! Path of topic to write. + FLUENT_SETTING(TString, Path); + + //! MessageGroupId (aka SourceId) to use. + FLUENT_SETTING(TString, MessageGroupId); + + //! Write to an exact partition group. Generally server assigns partition group automatically. + //! Using this option is not recommended unless you know for sure why you need it. + FLUENT_SETTING_OPTIONAL(ui32, PartitionGroupId); + //! Preferred LB cluster. Used for multi-cluster installation. //! If specified cluster is unavailable, session will write to other cluster. FLUENT_SETTING_OPTIONAL(TString, PreferredCluster); - + //! Write to other clusters if there are problems with connection //! to the first one. FLUENT_SETTING_DEFAULT(bool, AllowFallbackToOtherClusters, true); - //! codec and level to use for data compression prior to write. - FLUENT_SETTING_DEFAULT(ECodec, Codec, ECodec::GZIP); - FLUENT_SETTING_DEFAULT(i32, CompressionLevel, 4); - - //! Writer will not accept new messages if memory usage exceeds this limit. - //! Memory usage consists of raw data pending compression and compressed messages being sent. - FLUENT_SETTING_DEFAULT(ui64, MaxMemoryUsage, 20 * 1024 * 1024); - - //! Maximum messages accepted by writer but not written (with confirmation from server). - //! Writer will not accept new messages after reaching the limit. - FLUENT_SETTING_DEFAULT(ui32, MaxInflightCount, 100000); - - //! Retry policy enables automatic retries for non-fatal errors. + //! codec and level to use for data compression prior to write. + FLUENT_SETTING_DEFAULT(ECodec, Codec, ECodec::GZIP); + FLUENT_SETTING_DEFAULT(i32, CompressionLevel, 4); + + //! Writer will not accept new messages if memory usage exceeds this limit. + //! Memory usage consists of raw data pending compression and compressed messages being sent. + FLUENT_SETTING_DEFAULT(ui64, MaxMemoryUsage, 20 * 1024 * 1024); + + //! Maximum messages accepted by writer but not written (with confirmation from server). + //! Writer will not accept new messages after reaching the limit. + FLUENT_SETTING_DEFAULT(ui32, MaxInflightCount, 100000); + + //! Retry policy enables automatic retries for non-fatal errors. //! IRetryPolicy::GetDefaultPolicy() if null (not set). - FLUENT_SETTING(IRetryPolicy::TPtr, RetryPolicy); - - //! User metadata that may be attached to write session. + FLUENT_SETTING(IRetryPolicy::TPtr, RetryPolicy); + + //! User metadata that may be attached to write session. TWriteSessionSettings& AppendSessionMeta(const TString& key, const TString& value) { Meta_.Fields[key] = value; return *this; }; - NYdb::NPersQueue::TWriteSessionMeta Meta_; - - //! Writer will accumulate messages until reaching up to BatchFlushSize bytes - //! but for no longer than BatchFlushInterval. - //! Upon reaching FlushInterval or FlushSize limit, all messages will be written with one batch. - //! Greatly increases performance for small messages. - //! Setting either value to zero means immediate write with no batching. (Unrecommended, especially for clients - //! sending small messages at high rate). - FLUENT_SETTING_OPTIONAL(TDuration, BatchFlushInterval); - FLUENT_SETTING_OPTIONAL(ui64, BatchFlushSizeBytes); - + NYdb::NPersQueue::TWriteSessionMeta Meta_; + + //! Writer will accumulate messages until reaching up to BatchFlushSize bytes + //! but for no longer than BatchFlushInterval. + //! Upon reaching FlushInterval or FlushSize limit, all messages will be written with one batch. + //! Greatly increases performance for small messages. + //! Setting either value to zero means immediate write with no batching. (Unrecommended, especially for clients + //! sending small messages at high rate). + FLUENT_SETTING_OPTIONAL(TDuration, BatchFlushInterval); + FLUENT_SETTING_OPTIONAL(ui64, BatchFlushSizeBytes); + FLUENT_SETTING_DEFAULT(TDuration, ConnectTimeout, TDuration::Seconds(30)); - - FLUENT_SETTING_OPTIONAL(TWriterCounters::TPtr, Counters); - + + FLUENT_SETTING_OPTIONAL(TWriterCounters::TPtr, Counters); + //! Executor for compression tasks. //! If not set, default executor will be used. FLUENT_SETTING(IExecutor::TPtr, CompressionExecutor); - struct TEventHandlers { - using TSelf = TEventHandlers; - using TWriteAckHandler = std::function<void(TWriteSessionEvent::TAcksEvent&)>; - using TReadyToAcceptHandler = std::function<void(TWriteSessionEvent::TReadyToAcceptEvent&)>; - - //! Function to handle Acks events. - //! If this handler is set, write ack events will be handled by handler, - //! otherwise sent to TWriteSession::GetEvent(). - FLUENT_SETTING(TWriteAckHandler, AcksHandler); - - //! Function to handle ReadyToAccept event. - //! If this handler is set, write these events will be handled by handler, - //! otherwise sent to TWriteSession::GetEvent(). - FLUENT_SETTING(TReadyToAcceptHandler, ReadyToAcceptHander); - - //! Function to handle close session events. - //! If this handler is set, close session events will be handled by handler - //! and then sent to TWriteSession::GetEvent(). - FLUENT_SETTING(TSessionClosedHandler, SessionClosedHandler); + struct TEventHandlers { + using TSelf = TEventHandlers; + using TWriteAckHandler = std::function<void(TWriteSessionEvent::TAcksEvent&)>; + using TReadyToAcceptHandler = std::function<void(TWriteSessionEvent::TReadyToAcceptEvent&)>; + + //! Function to handle Acks events. + //! If this handler is set, write ack events will be handled by handler, + //! otherwise sent to TWriteSession::GetEvent(). + FLUENT_SETTING(TWriteAckHandler, AcksHandler); + + //! Function to handle ReadyToAccept event. + //! If this handler is set, write these events will be handled by handler, + //! otherwise sent to TWriteSession::GetEvent(). + FLUENT_SETTING(TReadyToAcceptHandler, ReadyToAcceptHander); + + //! Function to handle close session events. + //! If this handler is set, close session events will be handled by handler + //! and then sent to TWriteSession::GetEvent(). + FLUENT_SETTING(TSessionClosedHandler, SessionClosedHandler); //! Function to handle all event types. //! If event with current type has no handler for this type of event, @@ -1134,10 +1134,10 @@ struct TWriteSessionSettings : public TRequestSettings<TWriteSessionSettings> { //! Executor for handlers. //! If not set, default single threaded executor will be used. FLUENT_SETTING(IExecutor::TPtr, HandlersExecutor); - }; - - //! Event handlers. - FLUENT_SETTING(TEventHandlers, EventHandlers); + }; + + //! Event handlers. + FLUENT_SETTING(TEventHandlers, EventHandlers); //! Enables validation of SeqNo. If enabled, then writer will check writing with seqNo and without it and throws exception. FLUENT_SETTING_DEFAULT(bool, ValidateSeqNo, true); @@ -1145,8 +1145,8 @@ struct TWriteSessionSettings : public TRequestSettings<TWriteSessionSettings> { //! Manages cluster discovery mode. FLUENT_SETTING_OPTIONAL(EClusterDiscoveryMode, ClusterDiscoveryMode); -}; - +}; + //! Read settings for single topic. struct TTopicReadSettings { using TSelf = TTopicReadSettings; @@ -1237,7 +1237,7 @@ struct TReadSessionSettings : public TRequestSettings<TReadSessionSettings> { //! If this handler is set, close session events will be handled by handler //! and then sent to TReadSession::GetEvent(). //! Default value is empty function (not set). - FLUENT_SETTING(TSessionClosedHandler, SessionClosedHandler); + FLUENT_SETTING(TSessionClosedHandler, SessionClosedHandler); //! Function to handle all event types. //! If event with current type has no handler for this type of event, @@ -1329,71 +1329,71 @@ struct TReadSessionSettings : public TRequestSettings<TReadSessionSettings> { }; //! Simple write session. Does not need event handlers. Does not provide Events, ContinuationTokens, write Acks. -class ISimpleBlockingWriteSession : public TThrRefBase { -public: - //! Write single message. Blocks for up to blockTimeout if inflight is full or memoryUsage is exceeded; - //! return - true if write succeeded, false if message was not enqueued for write within blockTimeout. - //! no Ack is provided. - virtual bool Write(TStringBuf data, TMaybe<ui64> seqNo = Nothing(), TMaybe<TInstant> createTimestamp = Nothing(), - const TDuration& blockTimeout = TDuration::Max()) = 0; - - //! Blocks till SeqNo is discovered from server. Returns 0 in case of failure on init. - virtual ui64 GetInitSeqNo() = 0; - - //! Complete all active writes, wait for ack from server and close. - //! closeTimeout - max time to wait. Empty Maybe means infinity. - //! return - true if all writes were completed and acked. false if timeout was reached and some writes were aborted. - - virtual bool Close(TDuration closeTimeout = TDuration::Max()) = 0; - - //! Returns true if write session is alive and acitve. False if session was closed. - virtual bool IsAlive() const = 0; - - virtual TWriterCounters::TPtr GetCounters() = 0; - - //! Close immediately and destroy, don't wait for anything. - virtual ~ISimpleBlockingWriteSession() = default; -}; - -//! Generic write session with all capabilities. -class IWriteSession { -public: - //! Future that is set when next event is available. - virtual NThreading::TFuture<void> WaitEvent() = 0; - +class ISimpleBlockingWriteSession : public TThrRefBase { +public: + //! Write single message. Blocks for up to blockTimeout if inflight is full or memoryUsage is exceeded; + //! return - true if write succeeded, false if message was not enqueued for write within blockTimeout. + //! no Ack is provided. + virtual bool Write(TStringBuf data, TMaybe<ui64> seqNo = Nothing(), TMaybe<TInstant> createTimestamp = Nothing(), + const TDuration& blockTimeout = TDuration::Max()) = 0; + + //! Blocks till SeqNo is discovered from server. Returns 0 in case of failure on init. + virtual ui64 GetInitSeqNo() = 0; + + //! Complete all active writes, wait for ack from server and close. + //! closeTimeout - max time to wait. Empty Maybe means infinity. + //! return - true if all writes were completed and acked. false if timeout was reached and some writes were aborted. + + virtual bool Close(TDuration closeTimeout = TDuration::Max()) = 0; + + //! Returns true if write session is alive and acitve. False if session was closed. + virtual bool IsAlive() const = 0; + + virtual TWriterCounters::TPtr GetCounters() = 0; + + //! Close immediately and destroy, don't wait for anything. + virtual ~ISimpleBlockingWriteSession() = default; +}; + +//! Generic write session with all capabilities. +class IWriteSession { +public: + //! Future that is set when next event is available. + virtual NThreading::TFuture<void> WaitEvent() = 0; + //! Wait and return next event. Use WaitEvent() for non-blocking wait. virtual TMaybe<TWriteSessionEvent::TEvent> GetEvent(bool block = false) = 0; - - //! Get several events in one call. - //! If blocking = false, instantly returns up to maxEventsCount available events. - //! If blocking = true, blocks till maxEventsCount events are available. - //! If maxEventsCount is unset, write session decides the count to return itself. + + //! Get several events in one call. + //! If blocking = false, instantly returns up to maxEventsCount available events. + //! If blocking = true, blocks till maxEventsCount events are available. + //! If maxEventsCount is unset, write session decides the count to return itself. virtual TVector<TWriteSessionEvent::TEvent> GetEvents(bool block = false, TMaybe<size_t> maxEventsCount = Nothing()) = 0; - - //! Future that is set when initial SeqNo is available. - virtual NThreading::TFuture<ui64> GetInitSeqNo() = 0; - - //! Write single message. + + //! Future that is set when initial SeqNo is available. + virtual NThreading::TFuture<ui64> GetInitSeqNo() = 0; + + //! Write single message. //! continuationToken - a token earlier provided to client with ReadyToAccept event. - virtual void Write(TContinuationToken&& continuationToken, TStringBuf data, TMaybe<ui64> seqNo = Nothing(), TMaybe<TInstant> createTimestamp = Nothing()) = 0; - + virtual void Write(TContinuationToken&& continuationToken, TStringBuf data, TMaybe<ui64> seqNo = Nothing(), TMaybe<TInstant> createTimestamp = Nothing()) = 0; + //! Write single message that is already coded by codec. Codec from settings does not apply to this message. //! continuationToken - a token earlier provided to client with ReadyToAccept event. //! originalSize - size of unpacked message virtual void WriteEncoded(TContinuationToken&& continuationToken, TStringBuf data, ECodec codec, ui32 originalSize, TMaybe<ui64> seqNo = Nothing(), TMaybe<TInstant> createTimestamp = Nothing()) = 0; - //! Wait for all writes to complete (no more that closeTimeout()), than close. Empty maybe - means infinite timeout. - //! return - true if all writes were completed and acked. false if timeout was reached and some writes were aborted. - virtual bool Close(TDuration closeTimeout = TDuration::Max()) = 0; - - //! Writer counters with different stats (see TWriterConuters). - virtual TWriterCounters::TPtr GetCounters() = 0; - - //! Close() with timeout = 0 and destroy everything instantly. - virtual ~IWriteSession() = default; -}; - + //! Wait for all writes to complete (no more that closeTimeout()), than close. Empty maybe - means infinite timeout. + //! return - true if all writes were completed and acked. false if timeout was reached and some writes were aborted. + virtual bool Close(TDuration closeTimeout = TDuration::Max()) = 0; + + //! Writer counters with different stats (see TWriterConuters). + virtual TWriterCounters::TPtr GetCounters() = 0; + + //! Close() with timeout = 0 and destroy everything instantly. + virtual ~IWriteSession() = default; +}; + class IReadSession { public: //! Main reader loop. @@ -1493,8 +1493,8 @@ public: std::shared_ptr<IReadSession> CreateReadSession(const TReadSessionSettings& settings); //! Create write session. - std::shared_ptr<ISimpleBlockingWriteSession> CreateSimpleBlockingWriteSession(const TWriteSessionSettings& settings); - std::shared_ptr<IWriteSession> CreateWriteSession(const TWriteSessionSettings& settings); + std::shared_ptr<ISimpleBlockingWriteSession> CreateSimpleBlockingWriteSession(const TWriteSessionSettings& settings); + std::shared_ptr<IWriteSession> CreateWriteSession(const TWriteSessionSettings& settings); private: std::shared_ptr<TImpl> Impl_; diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/basic_usage_ut.cpp b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/basic_usage_ut.cpp index a3505e77fb..7e0ac17404 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/basic_usage_ut.cpp +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/basic_usage_ut.cpp @@ -1,223 +1,223 @@ -#include "ut_utils.h" - -#include <library/cpp/testing/unittest/registar.h> -#include <library/cpp/threading/future/future.h> -#include <library/cpp/threading/future/async.h> +#include "ut_utils.h" + +#include <library/cpp/testing/unittest/registar.h> +#include <library/cpp/threading/future/future.h> +#include <library/cpp/threading/future/async.h> #include <library/cpp/threading/chunk_queue/queue.h> -#include <util/generic/ptr.h> -#include <util/generic/queue.h> -#include <util/string/join.h> -#include <util/system/event.h> +#include <util/generic/ptr.h> +#include <util/generic/queue.h> +#include <util/string/join.h> +#include <util/system/event.h> #include <util/stream/zlib.h> #include <util/stream/str.h> -#include <atomic> - -using namespace NThreading; -using namespace NKikimr; -using namespace NKikimr::NPersQueueTests; -using namespace NPersQueue; - -namespace NYdb::NPersQueue::NTests { -// This suite tests basic Logbroker usage scenario: write and read a bunch of messages -Y_UNIT_TEST_SUITE(BasicUsage) { - template<typename TResponse> - void AssertStreamingMessageCase(const typename TResponse::ResponseCase& expectedCase, const TResponse& actualResponse) { - UNIT_ASSERT_EQUAL_C(expectedCase, actualResponse.GetResponseCase(), "Got unexpected streaming message or error:\n" + actualResponse.DebugString()); - } - - using TWriteCallable = std::function<TFuture<TWriteResult>(TString& data, ui64 sequenceNumber, TInstant createdAt)>; - +#include <atomic> + +using namespace NThreading; +using namespace NKikimr; +using namespace NKikimr::NPersQueueTests; +using namespace NPersQueue; + +namespace NYdb::NPersQueue::NTests { +// This suite tests basic Logbroker usage scenario: write and read a bunch of messages +Y_UNIT_TEST_SUITE(BasicUsage) { + template<typename TResponse> + void AssertStreamingMessageCase(const typename TResponse::ResponseCase& expectedCase, const TResponse& actualResponse) { + UNIT_ASSERT_EQUAL_C(expectedCase, actualResponse.GetResponseCase(), "Got unexpected streaming message or error:\n" + actualResponse.DebugString()); + } + + using TWriteCallable = std::function<TFuture<TWriteResult>(TString& data, ui64 sequenceNumber, TInstant createdAt)>; + void WriteAndReadAndCommitRandomMessages(TPersQueueYdbSdkTestSetup* setup, TWriteCallable write, bool disableClusterDiscovery = false) { - auto log = setup->GetLog(); - const TInstant start = TInstant::Now(); - TVector<TString> messages; - const ui32 messageCount = 5; - ui32 totalSize = 0; - std::queue<NThreading::TFuture<TWriteResult>> writeFutures; - for (ui32 i = 1; i <= messageCount; ++i) { - const size_t size = (1 + (i%100))*1_KB; - - const auto timestamp = TInstant::MilliSeconds(i); - const auto message = NUnitTest::RandomString(size, std::rand()); - - messages.emplace_back(std::move(message)); - - auto writeFuture = write(messages.back(), i, timestamp); - writeFutures.push(writeFuture); - totalSize += size; - } - - while (!writeFutures.empty()) { - const auto& oldestWriteFuture = writeFutures.front(); - const auto& result = oldestWriteFuture.GetValueSync(); - UNIT_ASSERT_C(result.Ok || result.NoWait, result.ResponseDebugString); - writeFutures.pop(); - } - - log.Write(TLOG_INFO, "All messages are written"); - - std::shared_ptr<IReadSession> readSession = setup->GetPersQueueClient().CreateReadSession( + auto log = setup->GetLog(); + const TInstant start = TInstant::Now(); + TVector<TString> messages; + const ui32 messageCount = 5; + ui32 totalSize = 0; + std::queue<NThreading::TFuture<TWriteResult>> writeFutures; + for (ui32 i = 1; i <= messageCount; ++i) { + const size_t size = (1 + (i%100))*1_KB; + + const auto timestamp = TInstant::MilliSeconds(i); + const auto message = NUnitTest::RandomString(size, std::rand()); + + messages.emplace_back(std::move(message)); + + auto writeFuture = write(messages.back(), i, timestamp); + writeFutures.push(writeFuture); + totalSize += size; + } + + while (!writeFutures.empty()) { + const auto& oldestWriteFuture = writeFutures.front(); + const auto& result = oldestWriteFuture.GetValueSync(); + UNIT_ASSERT_C(result.Ok || result.NoWait, result.ResponseDebugString); + writeFutures.pop(); + } + + log.Write(TLOG_INFO, "All messages are written"); + + std::shared_ptr<IReadSession> readSession = setup->GetPersQueueClient().CreateReadSession( setup->GetReadSessionSettings().DisableClusterDiscovery(disableClusterDiscovery) - ); -// auto isStarted = consumer->Start().ExtractValueSync(); -// AssertStreamingMessageCase(TReadResponse::kInit, isStarted.Response); - - ui32 readMessageCount = 0; - TMaybe<ui32> committedOffset; - ui32 previousOffset = 0; - bool closed = false; + ); +// auto isStarted = consumer->Start().ExtractValueSync(); +// AssertStreamingMessageCase(TReadResponse::kInit, isStarted.Response); + + ui32 readMessageCount = 0; + TMaybe<ui32> committedOffset; + ui32 previousOffset = 0; + bool closed = false; while ((readMessageCount < messageCount || committedOffset <= previousOffset) && !closed) { - Cerr << "Get event on client\n"; - auto event = *readSession->GetEvent(true); - std::visit(TOverloaded { - [&](TReadSessionEvent::TDataReceivedEvent& event) { - for (auto& message: event.GetMessages()) { - TString sourceId = message.GetMessageGroupId(); - ui32 seqNo = message.GetSeqNo(); - UNIT_ASSERT_VALUES_EQUAL(readMessageCount + 1, seqNo); - ++readMessageCount; - auto offset = message.GetOffset(); - if (readMessageCount == 1) { - UNIT_ASSERT_VALUES_EQUAL(offset, 0); - } else { - UNIT_ASSERT_VALUES_EQUAL(offset, previousOffset + 1); - } - previousOffset = offset; - message.Commit(); - } - }, - [&](TReadSessionEvent::TCommitAcknowledgementEvent& event) { - auto offset = event.GetCommittedOffset(); - log << TLOG_INFO << "Got commit ack with offset " << offset; - if (committedOffset.Defined()) { - UNIT_ASSERT_GT(offset, *committedOffset); - } - committedOffset = offset; - }, - [&](TReadSessionEvent::TCreatePartitionStreamEvent& event) { - //UNIT_FAIL("Test does not support lock sessions yet"); - event.Confirm(); - }, - [&](TReadSessionEvent::TDestroyPartitionStreamEvent& event) { -// UNIT_FAIL("Test does not support lock sessions yet"); - event.Confirm(); - }, - [&](TReadSessionEvent::TPartitionStreamStatusEvent&) { - UNIT_FAIL("Test does not support lock sessions yet"); - }, - [&](TReadSessionEvent::TPartitionStreamClosedEvent&) { - UNIT_FAIL("Test does not support lock sessions yet"); - }, - [&](TSessionClosedEvent& event) { - log << TLOG_INFO << "Got close event: " << event.DebugString(); - //UNIT_FAIL("Session closed"); - closed = true; - } - - }, event); - log << TLOG_INFO << "Read message count is " << readMessageCount << " (from " << messageCount - << " in total), committed offset count is " - << (committedOffset.Defined() ? * committedOffset : 0) << " (maximum offset: " - << previousOffset << ")"; - } - + Cerr << "Get event on client\n"; + auto event = *readSession->GetEvent(true); + std::visit(TOverloaded { + [&](TReadSessionEvent::TDataReceivedEvent& event) { + for (auto& message: event.GetMessages()) { + TString sourceId = message.GetMessageGroupId(); + ui32 seqNo = message.GetSeqNo(); + UNIT_ASSERT_VALUES_EQUAL(readMessageCount + 1, seqNo); + ++readMessageCount; + auto offset = message.GetOffset(); + if (readMessageCount == 1) { + UNIT_ASSERT_VALUES_EQUAL(offset, 0); + } else { + UNIT_ASSERT_VALUES_EQUAL(offset, previousOffset + 1); + } + previousOffset = offset; + message.Commit(); + } + }, + [&](TReadSessionEvent::TCommitAcknowledgementEvent& event) { + auto offset = event.GetCommittedOffset(); + log << TLOG_INFO << "Got commit ack with offset " << offset; + if (committedOffset.Defined()) { + UNIT_ASSERT_GT(offset, *committedOffset); + } + committedOffset = offset; + }, + [&](TReadSessionEvent::TCreatePartitionStreamEvent& event) { + //UNIT_FAIL("Test does not support lock sessions yet"); + event.Confirm(); + }, + [&](TReadSessionEvent::TDestroyPartitionStreamEvent& event) { +// UNIT_FAIL("Test does not support lock sessions yet"); + event.Confirm(); + }, + [&](TReadSessionEvent::TPartitionStreamStatusEvent&) { + UNIT_FAIL("Test does not support lock sessions yet"); + }, + [&](TReadSessionEvent::TPartitionStreamClosedEvent&) { + UNIT_FAIL("Test does not support lock sessions yet"); + }, + [&](TSessionClosedEvent& event) { + log << TLOG_INFO << "Got close event: " << event.DebugString(); + //UNIT_FAIL("Session closed"); + closed = true; + } + + }, event); + log << TLOG_INFO << "Read message count is " << readMessageCount << " (from " << messageCount + << " in total), committed offset count is " + << (committedOffset.Defined() ? * committedOffset : 0) << " (maximum offset: " + << previousOffset << ")"; + } + UNIT_ASSERT_VALUES_EQUAL(previousOffset + 1, committedOffset); - UNIT_ASSERT_VALUES_EQUAL(readMessageCount, messageCount); - log.Write(TLOG_INFO, Sprintf("Time took to write and read %u messages, %u [MiB] in total is %lu [s]", messageCount, (totalSize / 1024 / 1024), (TInstant::Now() - start).Seconds())); - } - - - void SimpleWriteAndValidateData( - TPersQueueYdbSdkTestSetup* setup, TWriteSessionSettings& writeSettings, ui64 count, - TMaybe<bool> shouldCaptureData = Nothing() - ) { - auto& client = setup->GetPersQueueClient(); - auto session = client.CreateSimpleBlockingWriteSession(writeSettings); - TString messageBase = "message-"; - TVector<TString> sentMessages; - - for (auto i = 0u; i < count; i++) { - sentMessages.emplace_back(messageBase * (i+1) + ToString(i)); - auto res = session->Write(sentMessages.back()); - UNIT_ASSERT(res); - } - { - auto sessionAdapter = TSimpleWriteSessionTestAdapter( - dynamic_cast<TSimpleBlockingWriteSession *>(session.get())); - if (shouldCaptureData.Defined()) { - TStringBuilder msg; - msg << "Session has captured " << sessionAdapter.GetAcquiredMessagesCount() - << " messages, capturing was expected: " << *shouldCaptureData << Endl; - UNIT_ASSERT_VALUES_EQUAL_C(sessionAdapter.GetAcquiredMessagesCount() > 0, *shouldCaptureData, msg.c_str()); - } - } - session->Close(); - - auto readSettings = setup->GetReadSessionSettings(); - NThreading::TPromise<void> checkedPromise = NThreading::NewPromise<void>(); - auto totalReceived = 0u; - readSettings.EventHandlers_.SimpleDataHandlers([&](NYdb::NPersQueue::TReadSessionEvent::TDataReceivedEvent& ev) { - auto& messages = ev.GetMessages(); - for (size_t i = 0u; i < messages.size(); ++i) { - auto& message = messages[i]; - UNIT_ASSERT_VALUES_EQUAL(message.GetData(), sentMessages[totalReceived]); - totalReceived++; - } - if (totalReceived == sentMessages.size()) - checkedPromise.SetValue(); - }); - auto readSession = client.CreateReadSession(readSettings); - checkedPromise.GetFuture().GetValueSync(); - readSession->Close(TDuration::Zero()); - } - - Y_UNIT_TEST(WriteAndReadSomeMessagesWithAsyncCompression) { - auto setup = std::make_shared<TPersQueueYdbSdkTestSetup>(TEST_CASE_NAME); - TWriteSessionSettings writeSettings; - writeSettings.Path(setup->GetTestTopic()).MessageGroupId("src_id"); - SimpleWriteAndValidateData(setup.get(), writeSettings, 100u, true); - } - - Y_UNIT_TEST(WriteAndReadSomeMessagesWithSyncCompression) { - auto setup = std::make_shared<TPersQueueYdbSdkTestSetup>(TEST_CASE_NAME); - TWriteSessionSettings writeSettings; - writeSettings.Path(setup->GetTestTopic()).MessageGroupId("src_id"); - IExecutor::TPtr executor = new TSyncExecutor(); - writeSettings.CompressionExecutor(executor); - // LOGBROKER-7189 - //SimpleWriteAndValidateData(setup.get(), writeSettings, 100u, false); - SimpleWriteAndValidateData(setup.get(), writeSettings, 100u, true); - } - - Y_UNIT_TEST(WriteAndReadSomeMessagesWithNoCompression) { - auto setup = std::make_shared<TPersQueueYdbSdkTestSetup>(TEST_CASE_NAME); - TWriteSessionSettings writeSettings; - writeSettings.Path(setup->GetTestTopic()).MessageGroupId("src_id"); - writeSettings.Codec(ECodec::RAW); - // LOGBROKER-7189 - //SimpleWriteAndValidateData(setup.get(), writeSettings, 100u, false); - SimpleWriteAndValidateData(setup.get(), writeSettings, 100u, true); - } - - Y_UNIT_TEST(TWriteSession_WriteAndReadAndCommitRandomMessages) { - auto setup = std::make_shared<TPersQueueYdbSdkTestSetup>(TEST_CASE_NAME); - auto log = setup->GetLog(); - TYDBClientEventLoop clientEventLoop{setup}; - - TAutoEvent messagesWrittenToBuffer; - auto clientWrite = [&](TString& message, ui64 sequenceNumber, TInstant createdAt) { - auto promise = NewPromise<TWriteResult>(); - //log << TLOG_INFO << "Enqueue message with sequence number " << sequenceNumber; - clientEventLoop.MessageBuffer.Enqueue(TAcknowledgableMessage{message, sequenceNumber, createdAt, promise}); - messagesWrittenToBuffer.Signal(); - return promise.GetFuture(); - }; - - WriteAndReadAndCommitRandomMessages(setup.get(), std::move(clientWrite)); - } - + UNIT_ASSERT_VALUES_EQUAL(readMessageCount, messageCount); + log.Write(TLOG_INFO, Sprintf("Time took to write and read %u messages, %u [MiB] in total is %lu [s]", messageCount, (totalSize / 1024 / 1024), (TInstant::Now() - start).Seconds())); + } + + + void SimpleWriteAndValidateData( + TPersQueueYdbSdkTestSetup* setup, TWriteSessionSettings& writeSettings, ui64 count, + TMaybe<bool> shouldCaptureData = Nothing() + ) { + auto& client = setup->GetPersQueueClient(); + auto session = client.CreateSimpleBlockingWriteSession(writeSettings); + TString messageBase = "message-"; + TVector<TString> sentMessages; + + for (auto i = 0u; i < count; i++) { + sentMessages.emplace_back(messageBase * (i+1) + ToString(i)); + auto res = session->Write(sentMessages.back()); + UNIT_ASSERT(res); + } + { + auto sessionAdapter = TSimpleWriteSessionTestAdapter( + dynamic_cast<TSimpleBlockingWriteSession *>(session.get())); + if (shouldCaptureData.Defined()) { + TStringBuilder msg; + msg << "Session has captured " << sessionAdapter.GetAcquiredMessagesCount() + << " messages, capturing was expected: " << *shouldCaptureData << Endl; + UNIT_ASSERT_VALUES_EQUAL_C(sessionAdapter.GetAcquiredMessagesCount() > 0, *shouldCaptureData, msg.c_str()); + } + } + session->Close(); + + auto readSettings = setup->GetReadSessionSettings(); + NThreading::TPromise<void> checkedPromise = NThreading::NewPromise<void>(); + auto totalReceived = 0u; + readSettings.EventHandlers_.SimpleDataHandlers([&](NYdb::NPersQueue::TReadSessionEvent::TDataReceivedEvent& ev) { + auto& messages = ev.GetMessages(); + for (size_t i = 0u; i < messages.size(); ++i) { + auto& message = messages[i]; + UNIT_ASSERT_VALUES_EQUAL(message.GetData(), sentMessages[totalReceived]); + totalReceived++; + } + if (totalReceived == sentMessages.size()) + checkedPromise.SetValue(); + }); + auto readSession = client.CreateReadSession(readSettings); + checkedPromise.GetFuture().GetValueSync(); + readSession->Close(TDuration::Zero()); + } + + Y_UNIT_TEST(WriteAndReadSomeMessagesWithAsyncCompression) { + auto setup = std::make_shared<TPersQueueYdbSdkTestSetup>(TEST_CASE_NAME); + TWriteSessionSettings writeSettings; + writeSettings.Path(setup->GetTestTopic()).MessageGroupId("src_id"); + SimpleWriteAndValidateData(setup.get(), writeSettings, 100u, true); + } + + Y_UNIT_TEST(WriteAndReadSomeMessagesWithSyncCompression) { + auto setup = std::make_shared<TPersQueueYdbSdkTestSetup>(TEST_CASE_NAME); + TWriteSessionSettings writeSettings; + writeSettings.Path(setup->GetTestTopic()).MessageGroupId("src_id"); + IExecutor::TPtr executor = new TSyncExecutor(); + writeSettings.CompressionExecutor(executor); + // LOGBROKER-7189 + //SimpleWriteAndValidateData(setup.get(), writeSettings, 100u, false); + SimpleWriteAndValidateData(setup.get(), writeSettings, 100u, true); + } + + Y_UNIT_TEST(WriteAndReadSomeMessagesWithNoCompression) { + auto setup = std::make_shared<TPersQueueYdbSdkTestSetup>(TEST_CASE_NAME); + TWriteSessionSettings writeSettings; + writeSettings.Path(setup->GetTestTopic()).MessageGroupId("src_id"); + writeSettings.Codec(ECodec::RAW); + // LOGBROKER-7189 + //SimpleWriteAndValidateData(setup.get(), writeSettings, 100u, false); + SimpleWriteAndValidateData(setup.get(), writeSettings, 100u, true); + } + + Y_UNIT_TEST(TWriteSession_WriteAndReadAndCommitRandomMessages) { + auto setup = std::make_shared<TPersQueueYdbSdkTestSetup>(TEST_CASE_NAME); + auto log = setup->GetLog(); + TYDBClientEventLoop clientEventLoop{setup}; + + TAutoEvent messagesWrittenToBuffer; + auto clientWrite = [&](TString& message, ui64 sequenceNumber, TInstant createdAt) { + auto promise = NewPromise<TWriteResult>(); + //log << TLOG_INFO << "Enqueue message with sequence number " << sequenceNumber; + clientEventLoop.MessageBuffer.Enqueue(TAcknowledgableMessage{message, sequenceNumber, createdAt, promise}); + messagesWrittenToBuffer.Signal(); + return promise.GetFuture(); + }; + + WriteAndReadAndCommitRandomMessages(setup.get(), std::move(clientWrite)); + } + Y_UNIT_TEST(TWriteSession_WriteAndReadAndCommitRandomMessagesNoClusterDiscovery) { auto setup = std::make_shared<TPersQueueYdbSdkTestSetup>(TEST_CASE_NAME); auto log = setup->GetLog(); @@ -235,49 +235,49 @@ Y_UNIT_TEST_SUITE(BasicUsage) { WriteAndReadAndCommitRandomMessages(setup.get(), std::move(clientWrite), true); } - Y_UNIT_TEST(TSimpleWriteSession_AutoSeqNo_BasicUsage) { - auto setup = std::make_shared<TPersQueueYdbSdkTestSetup>(TEST_CASE_NAME); - auto& client = setup->GetPersQueueClient(); - auto settings = setup->GetWriteSessionSettings(); - auto simpleSession = client.CreateSimpleBlockingWriteSession(settings); - - TString msg = "message"; - auto clientWrite = [&](TString& message, ui64 seqNo, TInstant createdAt) { - auto promise = NewPromise<TWriteResult>(); - //log << TLOG_INFO << "Enqueue message with sequence number " << sequenceNumber; - simpleSession->Write(message, seqNo, createdAt); - promise.SetValue(TWriteResult{true}); - return promise.GetFuture(); - }; - WriteAndReadAndCommitRandomMessages(setup.get(), std::move(clientWrite)); - auto res = simpleSession->Close(); - UNIT_ASSERT(res); - UNIT_ASSERT(res); - } - - - - Y_UNIT_TEST(TWriteSession_AutoBatching) { - // ToDo: Re-enable once batching takes more than 1 message at once - return; - auto setup = std::make_shared<TPersQueueYdbSdkTestSetup>(TEST_CASE_NAME); - auto& client = setup->GetPersQueueClient(); - auto settings = setup->GetWriteSessionSettings(); - size_t batchSize = 100; - settings.BatchFlushInterval(TDuration::Seconds(1000)); // Batch on size, not on time. - settings.BatchFlushSizeBytes(batchSize); - auto writer = client.CreateWriteSession(settings); - TString message = "message"; - size_t totalSize = 0, seqNo = 0; - while (totalSize < batchSize) { - auto event = *writer->GetEvent(true); - UNIT_ASSERT(std::holds_alternative<TWriteSessionEvent::TReadyToAcceptEvent>(event)); - auto continueToken = std::move(std::get<TWriteSessionEvent::TReadyToAcceptEvent>(event).ContinuationToken); - writer->Write(std::move(continueToken), message, ++seqNo); - totalSize += message.size(); - } - WaitMessagesAcked(writer, 1, seqNo); - } + Y_UNIT_TEST(TSimpleWriteSession_AutoSeqNo_BasicUsage) { + auto setup = std::make_shared<TPersQueueYdbSdkTestSetup>(TEST_CASE_NAME); + auto& client = setup->GetPersQueueClient(); + auto settings = setup->GetWriteSessionSettings(); + auto simpleSession = client.CreateSimpleBlockingWriteSession(settings); + + TString msg = "message"; + auto clientWrite = [&](TString& message, ui64 seqNo, TInstant createdAt) { + auto promise = NewPromise<TWriteResult>(); + //log << TLOG_INFO << "Enqueue message with sequence number " << sequenceNumber; + simpleSession->Write(message, seqNo, createdAt); + promise.SetValue(TWriteResult{true}); + return promise.GetFuture(); + }; + WriteAndReadAndCommitRandomMessages(setup.get(), std::move(clientWrite)); + auto res = simpleSession->Close(); + UNIT_ASSERT(res); + UNIT_ASSERT(res); + } + + + + Y_UNIT_TEST(TWriteSession_AutoBatching) { + // ToDo: Re-enable once batching takes more than 1 message at once + return; + auto setup = std::make_shared<TPersQueueYdbSdkTestSetup>(TEST_CASE_NAME); + auto& client = setup->GetPersQueueClient(); + auto settings = setup->GetWriteSessionSettings(); + size_t batchSize = 100; + settings.BatchFlushInterval(TDuration::Seconds(1000)); // Batch on size, not on time. + settings.BatchFlushSizeBytes(batchSize); + auto writer = client.CreateWriteSession(settings); + TString message = "message"; + size_t totalSize = 0, seqNo = 0; + while (totalSize < batchSize) { + auto event = *writer->GetEvent(true); + UNIT_ASSERT(std::holds_alternative<TWriteSessionEvent::TReadyToAcceptEvent>(event)); + auto continueToken = std::move(std::get<TWriteSessionEvent::TReadyToAcceptEvent>(event).ContinuationToken); + writer->Write(std::move(continueToken), message, ++seqNo); + totalSize += message.size(); + } + WaitMessagesAcked(writer, 1, seqNo); + } Y_UNIT_TEST(TWriteSession_WriteEncoded) { auto setup = std::make_shared<TPersQueueYdbSdkTestSetup>(TEST_CASE_NAME); @@ -379,36 +379,36 @@ Y_UNIT_TEST_SUITE(BasicUsage) { } - Y_UNIT_TEST(TWriteSession_BatchingProducesContinueTokens) { - // ToDo: Re-enable once batching takes more than 1 message at once - return; - auto setup = std::make_shared<TPersQueueYdbSdkTestSetup>(TEST_CASE_NAME); - auto& client = setup->GetPersQueueClient(); - auto settings = setup->GetWriteSessionSettings(); - auto batchInterval = TDuration::Seconds(10); - settings.BatchFlushInterval(batchInterval); // Batch on size, not on time. - settings.BatchFlushSizeBytes(9999999999); - settings.MaxMemoryUsage(99); - auto writer = client.CreateWriteSession(settings); - TString message = "0123456789"; - auto seqNo = 10u; - - auto event = *writer->GetEvent(true); - UNIT_ASSERT(std::holds_alternative<TWriteSessionEvent::TReadyToAcceptEvent>(event)); - auto continueToken = std::move(std::get<TWriteSessionEvent::TReadyToAcceptEvent>(event).ContinuationToken); - writer->Write(std::move(continueToken), message * 5, ++seqNo); - auto eventF = writer->WaitEvent(); - eventF.Wait(batchInterval / 2); - UNIT_ASSERT(!eventF.HasValue()); - while (true) { - event = *writer->GetEvent(true); - if (std::holds_alternative<TWriteSessionEvent::TReadyToAcceptEvent>(event)) { - break; - } else { - UNIT_ASSERT(std::holds_alternative<TWriteSessionEvent::TAcksEvent>(event)); - } - } - writer->Close(); - } -} -} // namespace NYdb::NPersQueue::NTests + Y_UNIT_TEST(TWriteSession_BatchingProducesContinueTokens) { + // ToDo: Re-enable once batching takes more than 1 message at once + return; + auto setup = std::make_shared<TPersQueueYdbSdkTestSetup>(TEST_CASE_NAME); + auto& client = setup->GetPersQueueClient(); + auto settings = setup->GetWriteSessionSettings(); + auto batchInterval = TDuration::Seconds(10); + settings.BatchFlushInterval(batchInterval); // Batch on size, not on time. + settings.BatchFlushSizeBytes(9999999999); + settings.MaxMemoryUsage(99); + auto writer = client.CreateWriteSession(settings); + TString message = "0123456789"; + auto seqNo = 10u; + + auto event = *writer->GetEvent(true); + UNIT_ASSERT(std::holds_alternative<TWriteSessionEvent::TReadyToAcceptEvent>(event)); + auto continueToken = std::move(std::get<TWriteSessionEvent::TReadyToAcceptEvent>(event).ContinuationToken); + writer->Write(std::move(continueToken), message * 5, ++seqNo); + auto eventF = writer->WaitEvent(); + eventF.Wait(batchInterval / 2); + UNIT_ASSERT(!eventF.HasValue()); + while (true) { + event = *writer->GetEvent(true); + if (std::holds_alternative<TWriteSessionEvent::TReadyToAcceptEvent>(event)) { + break; + } else { + UNIT_ASSERT(std::holds_alternative<TWriteSessionEvent::TAcksEvent>(event)); + } + } + writer->Close(); + } +} +} // namespace NYdb::NPersQueue::NTests diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/compress_executor_ut.cpp b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/compress_executor_ut.cpp index 47c6a96562..04abfab569 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/compress_executor_ut.cpp +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/compress_executor_ut.cpp @@ -1,95 +1,95 @@ #include "ut_utils.h" - -namespace NYdb::NPersQueue::NTests { - -Y_UNIT_TEST_SUITE(CompressExecutor) { - Y_UNIT_TEST(TestReorderedExecutor) { - auto queue = std::make_shared<TLockFreeQueue<ui64>>(); - TYdbPqWriterTestHelper helper(TEST_CASE_NAME, queue); - - auto f1 = helper.Write(); - auto f2 = helper.Write(); - f1.Wait(TDuration::Seconds(1)); - UNIT_ASSERT(!f1.HasValue()); - UNIT_ASSERT(!f2.HasValue()); - - queue->Enqueue(2); - f2.Wait(TDuration::Seconds(1)); - UNIT_ASSERT(!f1.HasValue()); - UNIT_ASSERT(!f2.HasValue()); - queue->Enqueue(1); - Cerr << "Waiting for writes to complete.\n"; - f1.Wait(); - f2.Wait(); - - } - Y_UNIT_TEST(TestExecutorMemUsage) { - auto queue = std::make_shared<TLockFreeQueue<ui64>>(); - auto setup = std::make_shared<TPersQueueYdbSdkTestSetup>(TEST_CASE_NAME); - auto executor = MakeIntrusive<TYdbPqTestExecutor>(queue); - auto config = setup->GetWriteSessionSettings(); - auto memUsageLimit = 1_KB; - config.MaxMemoryUsage(memUsageLimit); - config.CompressionExecutor(executor); - auto retryPolicy = std::make_shared<TYdbPqTestRetryPolicy>(); - - config.RetryPolicy(retryPolicy); - - auto writer = setup->GetPersQueueClient().CreateWriteSession(config); - - TMaybe<TContinuationToken> continueToken; - auto event = *writer->GetEvent(true); - UNIT_ASSERT(std::holds_alternative<TWriteSessionEvent::TReadyToAcceptEvent>(event)); - continueToken = std::move(std::get<TWriteSessionEvent::TReadyToAcceptEvent>(event).ContinuationToken); - - auto waitEventFuture = writer->WaitEvent(); - UNIT_ASSERT(!waitEventFuture.HasValue()); - - TStringBuilder msgBuilder; - while (msgBuilder.size() < 100 * 1024) { - msgBuilder << "0123456789abcdefghijk"; - } - const ui64 COMPRESSED_SIZE = 305; - TString message = TString(msgBuilder); - ui64 seqNo = 1; - auto doWrite = [&] { - writer->Write(std::move(*continueToken), message, seqNo); - ++seqNo; - }; - doWrite(); - waitEventFuture = writer->WaitEvent(); - waitEventFuture.Wait(TDuration::Seconds(3)); - UNIT_ASSERT(!waitEventFuture.HasValue()); - queue->Enqueue(1); - event = *writer->GetEvent(true); - UNIT_ASSERT(std::holds_alternative<TWriteSessionEvent::TReadyToAcceptEvent>(event)); - continueToken = std::move(std::get<TWriteSessionEvent::TReadyToAcceptEvent>(event).ContinuationToken); - event = *writer->GetEvent(true); - UNIT_ASSERT(std::holds_alternative<TWriteSessionEvent::TAcksEvent>(event)); - - Cerr << "===Will now kick tablets\n"; - setup->KickTablets(); - retryPolicy->ExpectBreakDown(); - retryPolicy->WaitForRetriesSync(1); - ui64 currentUsageEstimate = 0; - while (currentUsageEstimate <= memUsageLimit) { - doWrite(); - queue->Enqueue(seqNo - 1); - currentUsageEstimate += COMPRESSED_SIZE; - if (currentUsageEstimate <= memUsageLimit) { - event = *writer->GetEvent(true); - UNIT_ASSERT(std::holds_alternative<TWriteSessionEvent::TReadyToAcceptEvent>(event)); - continueToken = std::move(std::get<TWriteSessionEvent::TReadyToAcceptEvent>(event).ContinuationToken); - } - } - Cerr << "============ UT - Wait event...\n"; - waitEventFuture = writer->WaitEvent(); - waitEventFuture.Wait(TDuration::Seconds(3)); - UNIT_ASSERT(!waitEventFuture.HasValue()); - - writer = nullptr; - retryPolicy = nullptr; - } -} -}; - + +namespace NYdb::NPersQueue::NTests { + +Y_UNIT_TEST_SUITE(CompressExecutor) { + Y_UNIT_TEST(TestReorderedExecutor) { + auto queue = std::make_shared<TLockFreeQueue<ui64>>(); + TYdbPqWriterTestHelper helper(TEST_CASE_NAME, queue); + + auto f1 = helper.Write(); + auto f2 = helper.Write(); + f1.Wait(TDuration::Seconds(1)); + UNIT_ASSERT(!f1.HasValue()); + UNIT_ASSERT(!f2.HasValue()); + + queue->Enqueue(2); + f2.Wait(TDuration::Seconds(1)); + UNIT_ASSERT(!f1.HasValue()); + UNIT_ASSERT(!f2.HasValue()); + queue->Enqueue(1); + Cerr << "Waiting for writes to complete.\n"; + f1.Wait(); + f2.Wait(); + + } + Y_UNIT_TEST(TestExecutorMemUsage) { + auto queue = std::make_shared<TLockFreeQueue<ui64>>(); + auto setup = std::make_shared<TPersQueueYdbSdkTestSetup>(TEST_CASE_NAME); + auto executor = MakeIntrusive<TYdbPqTestExecutor>(queue); + auto config = setup->GetWriteSessionSettings(); + auto memUsageLimit = 1_KB; + config.MaxMemoryUsage(memUsageLimit); + config.CompressionExecutor(executor); + auto retryPolicy = std::make_shared<TYdbPqTestRetryPolicy>(); + + config.RetryPolicy(retryPolicy); + + auto writer = setup->GetPersQueueClient().CreateWriteSession(config); + + TMaybe<TContinuationToken> continueToken; + auto event = *writer->GetEvent(true); + UNIT_ASSERT(std::holds_alternative<TWriteSessionEvent::TReadyToAcceptEvent>(event)); + continueToken = std::move(std::get<TWriteSessionEvent::TReadyToAcceptEvent>(event).ContinuationToken); + + auto waitEventFuture = writer->WaitEvent(); + UNIT_ASSERT(!waitEventFuture.HasValue()); + + TStringBuilder msgBuilder; + while (msgBuilder.size() < 100 * 1024) { + msgBuilder << "0123456789abcdefghijk"; + } + const ui64 COMPRESSED_SIZE = 305; + TString message = TString(msgBuilder); + ui64 seqNo = 1; + auto doWrite = [&] { + writer->Write(std::move(*continueToken), message, seqNo); + ++seqNo; + }; + doWrite(); + waitEventFuture = writer->WaitEvent(); + waitEventFuture.Wait(TDuration::Seconds(3)); + UNIT_ASSERT(!waitEventFuture.HasValue()); + queue->Enqueue(1); + event = *writer->GetEvent(true); + UNIT_ASSERT(std::holds_alternative<TWriteSessionEvent::TReadyToAcceptEvent>(event)); + continueToken = std::move(std::get<TWriteSessionEvent::TReadyToAcceptEvent>(event).ContinuationToken); + event = *writer->GetEvent(true); + UNIT_ASSERT(std::holds_alternative<TWriteSessionEvent::TAcksEvent>(event)); + + Cerr << "===Will now kick tablets\n"; + setup->KickTablets(); + retryPolicy->ExpectBreakDown(); + retryPolicy->WaitForRetriesSync(1); + ui64 currentUsageEstimate = 0; + while (currentUsageEstimate <= memUsageLimit) { + doWrite(); + queue->Enqueue(seqNo - 1); + currentUsageEstimate += COMPRESSED_SIZE; + if (currentUsageEstimate <= memUsageLimit) { + event = *writer->GetEvent(true); + UNIT_ASSERT(std::holds_alternative<TWriteSessionEvent::TReadyToAcceptEvent>(event)); + continueToken = std::move(std::get<TWriteSessionEvent::TReadyToAcceptEvent>(event).ContinuationToken); + } + } + Cerr << "============ UT - Wait event...\n"; + waitEventFuture = writer->WaitEvent(); + waitEventFuture.Wait(TDuration::Seconds(3)); + UNIT_ASSERT(!waitEventFuture.HasValue()); + + writer = nullptr; + retryPolicy = nullptr; + } +} +}; + diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/read_session_ut.cpp b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/read_session_ut.cpp index 34b64e2aab..9b2367e3f3 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/read_session_ut.cpp +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/read_session_ut.cpp @@ -665,10 +665,10 @@ void TReadSessionImplTestSetup::AssertNoEvents() { UNIT_ASSERT(!event); } -using NYdb::NPersQueue::NTests::TPersQueueYdbSdkTestSetup; +using NYdb::NPersQueue::NTests::TPersQueueYdbSdkTestSetup; Y_UNIT_TEST_SUITE(PersQueueSdkReadSessionTest) { void ReadSessionImpl(bool close, bool commit, bool explicitlySpecifiedPartitions = false) { - NYdb::NPersQueue::NTests::TPersQueueYdbSdkTestSetup setup("ReadSession"); + NYdb::NPersQueue::NTests::TPersQueueYdbSdkTestSetup setup("ReadSession"); setup.WriteToTopic({"message1", "message2"}); auto settings = setup.GetReadSessionSettings(); if (explicitlySpecifiedPartitions) { diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/retry_policy_ut.cpp b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/retry_policy_ut.cpp index 7070d9e5be..50d050e973 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/retry_policy_ut.cpp +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/retry_policy_ut.cpp @@ -1,22 +1,22 @@ #include "ut_utils.h" -#include <library/cpp/threading/future/future.h> -#include <library/cpp/testing/unittest/registar.h> - - -using namespace NThreading; -using namespace NKikimr; -using namespace NKikimr::NPersQueueTests; -using namespace NPersQueue; - -namespace NYdb::NPersQueue::NTests { - -Y_UNIT_TEST_SUITE(RetryPolicy) { - Y_UNIT_TEST(TWriteSession_TestPolicy) { - TYdbPqWriterTestHelper helper(TEST_CASE_NAME); - helper.Write(true); - helper.Policy->Initialized(); // Thus ignoring possible early retries on "cluster initializing" - auto doBreakDown = [&] () { +#include <library/cpp/threading/future/future.h> +#include <library/cpp/testing/unittest/registar.h> + + +using namespace NThreading; +using namespace NKikimr; +using namespace NKikimr::NPersQueueTests; +using namespace NPersQueue; + +namespace NYdb::NPersQueue::NTests { + +Y_UNIT_TEST_SUITE(RetryPolicy) { + Y_UNIT_TEST(TWriteSession_TestPolicy) { + TYdbPqWriterTestHelper helper(TEST_CASE_NAME); + helper.Write(true); + helper.Policy->Initialized(); // Thus ignoring possible early retries on "cluster initializing" + auto doBreakDown = [&] () { helper.Policy->ExpectBreakDown(); NThreading::TPromise<void> retriesPromise = NThreading::NewPromise(); Cerr << "WAIT for retries...\n"; @@ -24,16 +24,16 @@ Y_UNIT_TEST_SUITE(RetryPolicy) { Cerr << "KICK tablets\n"; helper.Setup->KickTablets(); - auto f1 = helper.Write(false); - auto f2 = helper.Write(); + auto f1 = helper.Write(false); + auto f2 = helper.Write(); - auto retriesFuture = retriesPromise.GetFuture(); - retriesFuture.Wait(); + auto retriesFuture = retriesPromise.GetFuture(); + retriesFuture.Wait(); Cerr << "WAIT for retries done\n"; - NThreading::TPromise<void> repairPromise = NThreading::NewPromise(); - auto repairFuture = repairPromise.GetFuture(); - helper.Policy->WaitForRepair(repairPromise); + NThreading::TPromise<void> repairPromise = NThreading::NewPromise(); + auto repairFuture = repairPromise.GetFuture(); + helper.Policy->WaitForRepair(repairPromise); Cerr << "ALLOW tablets\n"; @@ -42,85 +42,85 @@ Y_UNIT_TEST_SUITE(RetryPolicy) { Cerr << "WAIT for repair\n"; repairFuture.Wait(); Cerr << "REPAIR done\n"; - f1.Wait(); - f2.Wait(); - helper.Write(true); - }; - doBreakDown(); - doBreakDown(); - - } - Y_UNIT_TEST(TWriteSession_TestBrokenPolicy) { - TYdbPqWriterTestHelper helper(TEST_CASE_NAME); - helper.Write(); - helper.Policy->Initialized(); - helper.Policy->ExpectFatalBreakDown(); - helper.EventLoop->AllowStop(); - auto f1 = helper.Write(false); - helper.Setup->KickTablets(); - helper.Write(false); - - helper.EventLoop->WaitForStop(); - UNIT_ASSERT(!f1.HasValue()); - helper.Setup = nullptr; - - }; - + f1.Wait(); + f2.Wait(); + helper.Write(true); + }; + doBreakDown(); + doBreakDown(); + + } + Y_UNIT_TEST(TWriteSession_TestBrokenPolicy) { + TYdbPqWriterTestHelper helper(TEST_CASE_NAME); + helper.Write(); + helper.Policy->Initialized(); + helper.Policy->ExpectFatalBreakDown(); + helper.EventLoop->AllowStop(); + auto f1 = helper.Write(false); + helper.Setup->KickTablets(); + helper.Write(false); + + helper.EventLoop->WaitForStop(); + UNIT_ASSERT(!f1.HasValue()); + helper.Setup = nullptr; + + }; + Y_UNIT_TEST(TWriteSession_RetryOnTargetCluster) { - auto setup1 = std::make_shared<TPersQueueYdbSdkTestSetup>(TEST_CASE_NAME, false); + auto setup1 = std::make_shared<TPersQueueYdbSdkTestSetup>(TEST_CASE_NAME, false); SDKTestSetup setup2("RetryOnTargetCluster_Dc2"); - setup1->AddDataCenter("dc2", setup2, false); - setup1->Start(); - auto retryPolicy = std::make_shared<TYdbPqTestRetryPolicy>(); - auto settings = setup1->GetWriteSessionSettings(); + setup1->AddDataCenter("dc2", setup2, false); + setup1->Start(); + auto retryPolicy = std::make_shared<TYdbPqTestRetryPolicy>(); + auto settings = setup1->GetWriteSessionSettings(); settings.PreferredCluster("dc2"); settings.AllowFallbackToOtherClusters(false); - settings.RetryPolicy(retryPolicy); - - retryPolicy->Initialized(); - retryPolicy->ExpectBreakDown(); - - auto& client = setup1->GetPersQueueClient(); - Cerr << "=== Create write session \n"; - auto writer = client.CreateWriteSession(settings); - - NThreading::TPromise<void> retriesPromise = NThreading::NewPromise(); - auto retriesFuture = retriesPromise.GetFuture(); - retryPolicy->WaitForRetries(3, retriesPromise); - Cerr << "=== Wait retries\n"; - retriesFuture.Wait(); - - Cerr << "=== Enable dc2\n"; - setup1->EnableDataCenter("dc2"); - - NThreading::TPromise<void> repairPromise = NThreading::NewPromise(); - auto repairFuture = repairPromise.GetFuture(); - retryPolicy->WaitForRepair(repairPromise); - Cerr << "=== Wait for repair\n"; - repairFuture.Wait(); - Cerr << "=== Close writer\n"; - writer->Close(); - } - - Y_UNIT_TEST(TWriteSession_SwitchBackToLocalCluster) { - Cerr << "====Start test\n"; - - auto setup1 = std::make_shared<TPersQueueYdbSdkTestSetup>(TEST_CASE_NAME, false); - SDKTestSetup setup2("SwitchBackToLocalCluster", false); - setup2.SetSingleDataCenter("dc2"); - setup2.AddDataCenter("dc1", *setup1, true); - setup1->AddDataCenter("dc2", setup2, true); - setup1->Start(); + settings.RetryPolicy(retryPolicy); + + retryPolicy->Initialized(); + retryPolicy->ExpectBreakDown(); + + auto& client = setup1->GetPersQueueClient(); + Cerr << "=== Create write session \n"; + auto writer = client.CreateWriteSession(settings); + + NThreading::TPromise<void> retriesPromise = NThreading::NewPromise(); + auto retriesFuture = retriesPromise.GetFuture(); + retryPolicy->WaitForRetries(3, retriesPromise); + Cerr << "=== Wait retries\n"; + retriesFuture.Wait(); + + Cerr << "=== Enable dc2\n"; + setup1->EnableDataCenter("dc2"); + + NThreading::TPromise<void> repairPromise = NThreading::NewPromise(); + auto repairFuture = repairPromise.GetFuture(); + retryPolicy->WaitForRepair(repairPromise); + Cerr << "=== Wait for repair\n"; + repairFuture.Wait(); + Cerr << "=== Close writer\n"; + writer->Close(); + } + + Y_UNIT_TEST(TWriteSession_SwitchBackToLocalCluster) { + Cerr << "====Start test\n"; + + auto setup1 = std::make_shared<TPersQueueYdbSdkTestSetup>(TEST_CASE_NAME, false); + SDKTestSetup setup2("SwitchBackToLocalCluster", false); + setup2.SetSingleDataCenter("dc2"); + setup2.AddDataCenter("dc1", *setup1, true); + setup1->AddDataCenter("dc2", setup2, true); + setup1->Start(); setup2.Start(false); - Cerr << "=== Start session 1\n"; - auto helper = MakeHolder<TYdbPqWriterTestHelper>("", nullptr, TString(), setup1); - helper->Write(true); - auto retryPolicy = helper->Policy; - retryPolicy->Initialized(); - - auto waitForReconnect = [&](bool enable) { - Cerr << "=== Expect breakdown\n"; - retryPolicy->ExpectBreakDown(); + Cerr << "=== Start session 1\n"; + auto helper = MakeHolder<TYdbPqWriterTestHelper>("", nullptr, TString(), setup1); + helper->Write(true); + auto retryPolicy = helper->Policy; + retryPolicy->Initialized(); + + auto waitForReconnect = [&](bool enable) { + Cerr << "=== Expect breakdown\n"; + retryPolicy->ExpectBreakDown(); NThreading::TPromise<void> retriesPromise = NThreading::NewPromise(); auto retriesFuture = retriesPromise.GetFuture(); @@ -130,246 +130,246 @@ Y_UNIT_TEST_SUITE(RetryPolicy) { auto repairFuture = repairPromise.GetFuture(); retryPolicy->WaitForRepair(repairPromise); - if (enable) { - Cerr << "===Enabled DC1\n"; - setup1->EnableDataCenter("dc1"); - setup2.EnableDataCenter("dc1"); - } else { - Cerr << "===Disabled DC1\n"; - setup1->DisableDataCenter("dc1"); - setup2.DisableDataCenter("dc1"); - } + if (enable) { + Cerr << "===Enabled DC1\n"; + setup1->EnableDataCenter("dc1"); + setup2.EnableDataCenter("dc1"); + } else { + Cerr << "===Disabled DC1\n"; + setup1->DisableDataCenter("dc1"); + setup2.DisableDataCenter("dc1"); + } Sleep(TDuration::Seconds(5)); retriesFuture.Wait(); repairFuture.Wait(); - }; - Cerr << "===Wait for 1st reconnect\n"; - waitForReconnect(false); - Cerr << "===Wait for 2nd reconnect\n"; - waitForReconnect(true); - } - - Y_UNIT_TEST(TWriteSession_SeqNoShift) { - auto setup1 = std::make_shared<TPersQueueYdbSdkTestSetup>(TEST_CASE_NAME, false); - SDKTestSetup setup2("SeqNoShift_Dc2", false); - setup2.SetSingleDataCenter("dc2"); + }; + Cerr << "===Wait for 1st reconnect\n"; + waitForReconnect(false); + Cerr << "===Wait for 2nd reconnect\n"; + waitForReconnect(true); + } + + Y_UNIT_TEST(TWriteSession_SeqNoShift) { + auto setup1 = std::make_shared<TPersQueueYdbSdkTestSetup>(TEST_CASE_NAME, false); + SDKTestSetup setup2("SeqNoShift_Dc2", false); + setup2.SetSingleDataCenter("dc2"); setup2.AddDataCenter("dc1", *setup1, true); - setup2.Start(); - setup1->AddDataCenter("dc2", setup2, true); - setup1->Start(); - TString tmpSrcId = "tmp-src-seqno-shift", noShiftSrcId = "tmp-src-no-seqno-shift"; - auto helper = MakeHolder<TYdbPqWriterTestHelper>("", nullptr, "dc1", setup1); - auto helper2 = MakeHolder<TYdbPqWriterTestHelper>("", nullptr, "dc1", setup1, tmpSrcId); - //auto helperNoShift = MakeHolder<TYdbPqWriterTestHelper>("", nullptr, "dc1", setup1, tmpSrcId); - - auto settings = setup1->GetWriteSessionSettings(); - auto& client = setup1->GetPersQueueClient(); - //! Fill data in dc1 1 with SeqNo = 1..10 for 2 different SrcId - for (auto i = 0; i != 10; i++) { - helper->Write(true); // 1 - helper2->Write(true); // 1 - } - Cerr << "===Disable dc1\n"; - //! Leave only dc2 available - setup1->DisableDataCenter("dc1"); - helper->Policy->ExpectBreakDown(); - helper->Policy->WaitForRetriesSync(1); - - //! Re-create helpers, kill previous sessions. New sessions will connect to dc2. - helper = MakeHolder<TYdbPqWriterTestHelper>("", nullptr, TString(), setup1); - helper2 = MakeHolder<TYdbPqWriterTestHelper>("", nullptr, TString(), setup1, tmpSrcId); - - //! Write some data and await confirmation - just to ensure sessions are started. - helper->Write(true); - helper2->Write(true); - - helper->Policy->ExpectBreakDown(); - Cerr << "===Disable dc2\n"; - //! Leave no available DCs - setup1->DisableDataCenter("dc2"); - Cerr << "=== Wait for retries after initial dc2 shutdown\n"; - helper->Policy->WaitForRetriesSync(1); - - //! Put some data inflight. It cannot be written now, but SeqNo will be assigned. - for (auto i = 0; i != 4; i++) { - helper->Write(false); - helper2->Write(false); - } - auto f = helper->Write(false); - auto f2 = helper2->Write(false); - //! Enable DC1. Now writers gonna write collected data to DC1 having LastSeqNo = 10 - //! (because of data written and the very beginning), and inflight data has SeqNo assingned = 2..5, - //! so the SeqNo shift takes place. - setup1->EnableDataCenter("dc1"); - - Cerr << "=====Wait for writes to complete\n"; - f.Wait(); - f2.Wait(); - //! Writer1 is not used any more. - helper->EventLoop->AllowStop(); - helper = nullptr; - - helper2->Policy->Initialized(); - helper2->Policy->ExpectBreakDown(); - //! For the second writer, do switchback to dc2. - setup1->DisableDataCenter("dc1"); - helper2->Policy->WaitForRetriesSync(1); - //! Put some data inflight again; - for (auto i = 0; i != 4; i++) { - helper2->Write(false); - } - f = helper2->Write(false); - Cerr << "===Enable dc2\n"; - setup1->EnableDataCenter("dc2"); - f.Wait(); - helper2->EventLoop->AllowStop(); - helper2->Policy->ExpectBreakDown(); - - Cerr << "===Enable dc1\n"; - setup1->EnableDataCenter("dc1"); - auto CheckSeqNo = [&] (const TString& dcName, ui64 expectedSeqNo) { + setup2.Start(); + setup1->AddDataCenter("dc2", setup2, true); + setup1->Start(); + TString tmpSrcId = "tmp-src-seqno-shift", noShiftSrcId = "tmp-src-no-seqno-shift"; + auto helper = MakeHolder<TYdbPqWriterTestHelper>("", nullptr, "dc1", setup1); + auto helper2 = MakeHolder<TYdbPqWriterTestHelper>("", nullptr, "dc1", setup1, tmpSrcId); + //auto helperNoShift = MakeHolder<TYdbPqWriterTestHelper>("", nullptr, "dc1", setup1, tmpSrcId); + + auto settings = setup1->GetWriteSessionSettings(); + auto& client = setup1->GetPersQueueClient(); + //! Fill data in dc1 1 with SeqNo = 1..10 for 2 different SrcId + for (auto i = 0; i != 10; i++) { + helper->Write(true); // 1 + helper2->Write(true); // 1 + } + Cerr << "===Disable dc1\n"; + //! Leave only dc2 available + setup1->DisableDataCenter("dc1"); + helper->Policy->ExpectBreakDown(); + helper->Policy->WaitForRetriesSync(1); + + //! Re-create helpers, kill previous sessions. New sessions will connect to dc2. + helper = MakeHolder<TYdbPqWriterTestHelper>("", nullptr, TString(), setup1); + helper2 = MakeHolder<TYdbPqWriterTestHelper>("", nullptr, TString(), setup1, tmpSrcId); + + //! Write some data and await confirmation - just to ensure sessions are started. + helper->Write(true); + helper2->Write(true); + + helper->Policy->ExpectBreakDown(); + Cerr << "===Disable dc2\n"; + //! Leave no available DCs + setup1->DisableDataCenter("dc2"); + Cerr << "=== Wait for retries after initial dc2 shutdown\n"; + helper->Policy->WaitForRetriesSync(1); + + //! Put some data inflight. It cannot be written now, but SeqNo will be assigned. + for (auto i = 0; i != 4; i++) { + helper->Write(false); + helper2->Write(false); + } + auto f = helper->Write(false); + auto f2 = helper2->Write(false); + //! Enable DC1. Now writers gonna write collected data to DC1 having LastSeqNo = 10 + //! (because of data written and the very beginning), and inflight data has SeqNo assingned = 2..5, + //! so the SeqNo shift takes place. + setup1->EnableDataCenter("dc1"); + + Cerr << "=====Wait for writes to complete\n"; + f.Wait(); + f2.Wait(); + //! Writer1 is not used any more. + helper->EventLoop->AllowStop(); + helper = nullptr; + + helper2->Policy->Initialized(); + helper2->Policy->ExpectBreakDown(); + //! For the second writer, do switchback to dc2. + setup1->DisableDataCenter("dc1"); + helper2->Policy->WaitForRetriesSync(1); + //! Put some data inflight again; + for (auto i = 0; i != 4; i++) { + helper2->Write(false); + } + f = helper2->Write(false); + Cerr << "===Enable dc2\n"; + setup1->EnableDataCenter("dc2"); + f.Wait(); + helper2->EventLoop->AllowStop(); + helper2->Policy->ExpectBreakDown(); + + Cerr << "===Enable dc1\n"; + setup1->EnableDataCenter("dc1"); + auto CheckSeqNo = [&] (const TString& dcName, ui64 expectedSeqNo) { settings.PreferredCluster(dcName); settings.AllowFallbackToOtherClusters(false); settings.RetryPolicy(nullptr); //switch to default policy; - auto writer = client.CreateWriteSession(settings); - auto seqNo = writer->GetInitSeqNo().GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL(seqNo, expectedSeqNo); - writer->Close(TDuration::Zero()); - }; - - //!check SeqNo in both DC. For writer1 We expect 14 messages in DC1 - //! (10 written initially + 4 wriiten after recoonect) and 1 message in DC2 (only initial message). - Cerr << "===Check SeqNo writer1, dc2\n"; - CheckSeqNo("dc2", 1); - Cerr << "===Check SeqNo writer1, dc1\n"; - CheckSeqNo("dc1", 14); - - //! Check SeqNo for writer 2; Expect to have 6 messages on DC2 with MaxSeqNo = 6; - helper2 = nullptr; - settings.MessageGroupId(tmpSrcId); - Cerr << "===Check SeqNo writer2 dc2\n"; - //!DC2 has no a ap in SeqNo since 5 messages were written ot dc 1. - CheckSeqNo("dc2", 11); - Cerr << "===Check SeqNo writer2 dc1\n"; - CheckSeqNo("dc1", 14); - - - auto readSession = client.CreateReadSession(setup1->GetReadSessionSettings()); - - bool stop = false; - THashMap<TString, ui64> seqNoByClusterSrc1 = { - {"dc1", 0}, - {"dc2", 0} - }; - auto SeqNoByClusterSrc2 = seqNoByClusterSrc1; - - THashMap<TString, ui64> MsgCountByClusterSrc1 = { - {"dc1", 14}, - {"dc2", 1} - }; - THashMap<TString, ui64> MsgCountByClusterSrc2 = { - {"dc1", 14}, - {"dc2", 6} - }; - ui32 clustersPendingSrc1 = 2; - ui32 clustersPendingSrc2 = 2; - - while (!stop && (clustersPendingSrc2 || clustersPendingSrc1)) { - Cerr << "===Get event on client\n"; - auto event = *readSession->GetEvent(true); - std::visit(TOverloaded { - [&](TReadSessionEvent::TDataReceivedEvent& event) { - Cerr << "===Data event\n"; - auto& clusterName = event.GetPartitionStream()->GetCluster(); - for (auto& message: event.GetMessages()) { - TString sourceId = message.GetMessageGroupId(); - ui32 seqNo = message.GetSeqNo(); - if (sourceId == setup1->GetTestMessageGroupId()) { - UNIT_ASSERT_VALUES_EQUAL(seqNo, seqNoByClusterSrc1[clusterName] + 1); - seqNoByClusterSrc1[clusterName]++; - auto& msgRemaining = MsgCountByClusterSrc1[clusterName]; - UNIT_ASSERT(msgRemaining > 0); - msgRemaining--; - if (!msgRemaining) - clustersPendingSrc1--; - } else { - UNIT_ASSERT_VALUES_EQUAL(sourceId, tmpSrcId); - auto& prevSeqNo = SeqNoByClusterSrc2[clusterName]; - if (clusterName == "dc1" || prevSeqNo != 1) { - UNIT_ASSERT_VALUES_EQUAL(seqNo, prevSeqNo + 1); - prevSeqNo++; - } else { - UNIT_ASSERT_VALUES_EQUAL(clusterName, "dc2"); - UNIT_ASSERT_VALUES_EQUAL(seqNo, prevSeqNo + 6); - prevSeqNo+= 6; - } - auto& msgRemaining = MsgCountByClusterSrc2[clusterName]; - UNIT_ASSERT(msgRemaining > 0); - msgRemaining--; - if (!msgRemaining) - clustersPendingSrc2--; - } - message.Commit(); - } - }, - [&](TReadSessionEvent::TCommitAcknowledgementEvent&) { - }, - [&](TReadSessionEvent::TCreatePartitionStreamEvent& event) { - event.Confirm(); - }, - [&](TReadSessionEvent::TDestroyPartitionStreamEvent& event) { - event.Confirm(); - }, - [&](TReadSessionEvent::TPartitionStreamStatusEvent&) { - Cerr << "===Status event\n"; - UNIT_FAIL("Test does not support lock sessions yet"); - }, - [&](TReadSessionEvent::TPartitionStreamClosedEvent&) { - Cerr << "===Stream closed event\n"; - UNIT_FAIL("Test does not support lock sessions yet"); - }, - [&](TSessionClosedEvent& event) { - Cerr << "===Got close event: " << event.DebugString(); - stop = true; - } - - }, event); - } - UNIT_ASSERT_VALUES_EQUAL(clustersPendingSrc1 || clustersPendingSrc2, 0); - } - Y_UNIT_TEST(RetryWithBatching) { - auto setup = std::make_shared<TPersQueueYdbSdkTestSetup>(TEST_CASE_NAME); - auto settings = setup->GetWriteSessionSettings(); - auto retryPolicy = std::make_shared<TYdbPqTestRetryPolicy>(); - settings.BatchFlushInterval(TDuration::Seconds(1000)); // Batch on size, not on time. - settings.BatchFlushSizeBytes(100); - settings.RetryPolicy(retryPolicy); - auto& client = setup->GetPersQueueClient(); - auto writer = client.CreateWriteSession(settings); - auto event = *writer->GetEvent(true); + auto writer = client.CreateWriteSession(settings); + auto seqNo = writer->GetInitSeqNo().GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL(seqNo, expectedSeqNo); + writer->Close(TDuration::Zero()); + }; + + //!check SeqNo in both DC. For writer1 We expect 14 messages in DC1 + //! (10 written initially + 4 wriiten after recoonect) and 1 message in DC2 (only initial message). + Cerr << "===Check SeqNo writer1, dc2\n"; + CheckSeqNo("dc2", 1); + Cerr << "===Check SeqNo writer1, dc1\n"; + CheckSeqNo("dc1", 14); + + //! Check SeqNo for writer 2; Expect to have 6 messages on DC2 with MaxSeqNo = 6; + helper2 = nullptr; + settings.MessageGroupId(tmpSrcId); + Cerr << "===Check SeqNo writer2 dc2\n"; + //!DC2 has no a ap in SeqNo since 5 messages were written ot dc 1. + CheckSeqNo("dc2", 11); + Cerr << "===Check SeqNo writer2 dc1\n"; + CheckSeqNo("dc1", 14); + + + auto readSession = client.CreateReadSession(setup1->GetReadSessionSettings()); + + bool stop = false; + THashMap<TString, ui64> seqNoByClusterSrc1 = { + {"dc1", 0}, + {"dc2", 0} + }; + auto SeqNoByClusterSrc2 = seqNoByClusterSrc1; + + THashMap<TString, ui64> MsgCountByClusterSrc1 = { + {"dc1", 14}, + {"dc2", 1} + }; + THashMap<TString, ui64> MsgCountByClusterSrc2 = { + {"dc1", 14}, + {"dc2", 6} + }; + ui32 clustersPendingSrc1 = 2; + ui32 clustersPendingSrc2 = 2; + + while (!stop && (clustersPendingSrc2 || clustersPendingSrc1)) { + Cerr << "===Get event on client\n"; + auto event = *readSession->GetEvent(true); + std::visit(TOverloaded { + [&](TReadSessionEvent::TDataReceivedEvent& event) { + Cerr << "===Data event\n"; + auto& clusterName = event.GetPartitionStream()->GetCluster(); + for (auto& message: event.GetMessages()) { + TString sourceId = message.GetMessageGroupId(); + ui32 seqNo = message.GetSeqNo(); + if (sourceId == setup1->GetTestMessageGroupId()) { + UNIT_ASSERT_VALUES_EQUAL(seqNo, seqNoByClusterSrc1[clusterName] + 1); + seqNoByClusterSrc1[clusterName]++; + auto& msgRemaining = MsgCountByClusterSrc1[clusterName]; + UNIT_ASSERT(msgRemaining > 0); + msgRemaining--; + if (!msgRemaining) + clustersPendingSrc1--; + } else { + UNIT_ASSERT_VALUES_EQUAL(sourceId, tmpSrcId); + auto& prevSeqNo = SeqNoByClusterSrc2[clusterName]; + if (clusterName == "dc1" || prevSeqNo != 1) { + UNIT_ASSERT_VALUES_EQUAL(seqNo, prevSeqNo + 1); + prevSeqNo++; + } else { + UNIT_ASSERT_VALUES_EQUAL(clusterName, "dc2"); + UNIT_ASSERT_VALUES_EQUAL(seqNo, prevSeqNo + 6); + prevSeqNo+= 6; + } + auto& msgRemaining = MsgCountByClusterSrc2[clusterName]; + UNIT_ASSERT(msgRemaining > 0); + msgRemaining--; + if (!msgRemaining) + clustersPendingSrc2--; + } + message.Commit(); + } + }, + [&](TReadSessionEvent::TCommitAcknowledgementEvent&) { + }, + [&](TReadSessionEvent::TCreatePartitionStreamEvent& event) { + event.Confirm(); + }, + [&](TReadSessionEvent::TDestroyPartitionStreamEvent& event) { + event.Confirm(); + }, + [&](TReadSessionEvent::TPartitionStreamStatusEvent&) { + Cerr << "===Status event\n"; + UNIT_FAIL("Test does not support lock sessions yet"); + }, + [&](TReadSessionEvent::TPartitionStreamClosedEvent&) { + Cerr << "===Stream closed event\n"; + UNIT_FAIL("Test does not support lock sessions yet"); + }, + [&](TSessionClosedEvent& event) { + Cerr << "===Got close event: " << event.DebugString(); + stop = true; + } + + }, event); + } + UNIT_ASSERT_VALUES_EQUAL(clustersPendingSrc1 || clustersPendingSrc2, 0); + } + Y_UNIT_TEST(RetryWithBatching) { + auto setup = std::make_shared<TPersQueueYdbSdkTestSetup>(TEST_CASE_NAME); + auto settings = setup->GetWriteSessionSettings(); + auto retryPolicy = std::make_shared<TYdbPqTestRetryPolicy>(); + settings.BatchFlushInterval(TDuration::Seconds(1000)); // Batch on size, not on time. + settings.BatchFlushSizeBytes(100); + settings.RetryPolicy(retryPolicy); + auto& client = setup->GetPersQueueClient(); + auto writer = client.CreateWriteSession(settings); + auto event = *writer->GetEvent(true); Cerr << NYdb::NPersQueue::DebugString(event) << "\n"; UNIT_ASSERT(std::holds_alternative<TWriteSessionEvent::TReadyToAcceptEvent>(event)); - auto continueToken = std::move(std::get<TWriteSessionEvent::TReadyToAcceptEvent>(event).ContinuationToken); - TString message = "1234567890"; - ui64 seqNo = 0; + auto continueToken = std::move(std::get<TWriteSessionEvent::TReadyToAcceptEvent>(event).ContinuationToken); + TString message = "1234567890"; + ui64 seqNo = 0; setup->KickTablets(); - writer->Write(std::move(continueToken), message, ++seqNo); - retryPolicy->ExpectBreakDown(); - retryPolicy->WaitForRetriesSync(3); + writer->Write(std::move(continueToken), message, ++seqNo); + retryPolicy->ExpectBreakDown(); + retryPolicy->WaitForRetriesSync(3); while (seqNo < 10) { auto event = *writer->GetEvent(true); Cerr << NYdb::NPersQueue::DebugString(event) << "\n"; UNIT_ASSERT(std::holds_alternative<TWriteSessionEvent::TReadyToAcceptEvent>(event)); - writer->Write( + writer->Write( std::move(std::get<TWriteSessionEvent::TReadyToAcceptEvent>(event).ContinuationToken), - message, ++seqNo - ); - } - - setup->AllowTablets(); - retryPolicy->WaitForRepairSync(); - WaitMessagesAcked(writer, 1, seqNo); - } -}; -}; //NYdb::NPersQueue::NTests + message, ++seqNo + ); + } + + setup->AllowTablets(); + retryPolicy->WaitForRepairSync(); + WaitMessagesAcked(writer, 1, seqNo); + } +}; +}; //NYdb::NPersQueue::NTests diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils.cpp b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils.cpp index d9208d2b6c..6a853b6fc5 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils.cpp +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils.cpp @@ -1,33 +1,33 @@ -#include "ut_utils.h" - -namespace NYdb::NPersQueue::NTests { - -void WaitMessagesAcked(std::shared_ptr<IWriteSession> writer, ui64 startSeqNo, ui64 endSeqNo) { - THashSet<ui64> ackedSeqNo; - while (ackedSeqNo.size() < endSeqNo - startSeqNo + 1) { - auto event = *writer->GetEvent(true); - if (std::holds_alternative<TWriteSessionEvent::TReadyToAcceptEvent>(event)) { - continue; - } else { - UNIT_ASSERT(std::holds_alternative<TWriteSessionEvent::TAcksEvent>(event)); - for (auto& ack : std::get<TWriteSessionEvent::TAcksEvent>(event).Acks) { - UNIT_ASSERT(!ackedSeqNo.contains(ack.SeqNo)); - UNIT_ASSERT(ack.SeqNo >= startSeqNo && ack.SeqNo <= endSeqNo); - ackedSeqNo.insert(ack.SeqNo); - } - } - } -} - -TSimpleWriteSessionTestAdapter::TSimpleWriteSessionTestAdapter(TSimpleBlockingWriteSession* session) - : Session(session) -{} - -ui64 TSimpleWriteSessionTestAdapter::GetAcquiredMessagesCount() const { - if (Session->Writer) - return Session->Writer->MessagesAcquired; - else - return 0; -} - -} +#include "ut_utils.h" + +namespace NYdb::NPersQueue::NTests { + +void WaitMessagesAcked(std::shared_ptr<IWriteSession> writer, ui64 startSeqNo, ui64 endSeqNo) { + THashSet<ui64> ackedSeqNo; + while (ackedSeqNo.size() < endSeqNo - startSeqNo + 1) { + auto event = *writer->GetEvent(true); + if (std::holds_alternative<TWriteSessionEvent::TReadyToAcceptEvent>(event)) { + continue; + } else { + UNIT_ASSERT(std::holds_alternative<TWriteSessionEvent::TAcksEvent>(event)); + for (auto& ack : std::get<TWriteSessionEvent::TAcksEvent>(event).Acks) { + UNIT_ASSERT(!ackedSeqNo.contains(ack.SeqNo)); + UNIT_ASSERT(ack.SeqNo >= startSeqNo && ack.SeqNo <= endSeqNo); + ackedSeqNo.insert(ack.SeqNo); + } + } + } +} + +TSimpleWriteSessionTestAdapter::TSimpleWriteSessionTestAdapter(TSimpleBlockingWriteSession* session) + : Session(session) +{} + +ui64 TSimpleWriteSessionTestAdapter::GetAcquiredMessagesCount() const { + if (Session->Writer) + return Session->Writer->MessagesAcquired; + else + return 0; +} + +} diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils.h b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils.h index 8d70b6af4d..7fb219757b 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils.h +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils.h @@ -1,25 +1,25 @@ -#pragma once - +#pragma once + #include <ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/test_utils.h> #include <ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/sdk_test_setup.h> #include <ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/common.h> #include <ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/write_session.h> - -using namespace NKikimr; -using namespace NKikimr::NPersQueueTests; - -namespace NYdb::NPersQueue::NTests { - -class TPersQueueYdbSdkTestSetup : public ::NPersQueue::SDKTestSetup { - THolder<NYdb::TDriver> Driver; + +using namespace NKikimr; +using namespace NKikimr::NPersQueueTests; + +namespace NYdb::NPersQueue::NTests { + +class TPersQueueYdbSdkTestSetup : public ::NPersQueue::SDKTestSetup { + THolder<NYdb::TDriver> Driver; THolder<NYdb::NPersQueue::TPersQueueClient> PersQueueClient; - - TAdaptiveLock Lock; -public: - TPersQueueYdbSdkTestSetup(const TString& testCaseName, bool start = true) - : SDKTestSetup(testCaseName, start) - {} - + + TAdaptiveLock Lock; +public: + TPersQueueYdbSdkTestSetup(const TString& testCaseName, bool start = true) + : SDKTestSetup(testCaseName, start) + {} + ~TPersQueueYdbSdkTestSetup() { if (PersQueueClient) { PersQueueClient = nullptr; @@ -31,387 +31,387 @@ public: } } - NYdb::TDriver& GetDriver() { - if (!Driver) { - NYdb::TDriverConfig cfg; - cfg.SetEndpoint(TStringBuilder() << "localhost:" << Server.GrpcPort); - cfg.SetDatabase("/Root"); + NYdb::TDriver& GetDriver() { + if (!Driver) { + NYdb::TDriverConfig cfg; + cfg.SetEndpoint(TStringBuilder() << "localhost:" << Server.GrpcPort); + cfg.SetDatabase("/Root"); cfg.SetLog(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG)); - Driver = MakeHolder<NYdb::TDriver>(cfg); - } - return *Driver; - } - + Driver = MakeHolder<NYdb::TDriver>(cfg); + } + return *Driver; + } + NYdb::NPersQueue::TPersQueueClient& GetPersQueueClient() { - with_lock(Lock) { - if (!PersQueueClient) { + with_lock(Lock) { + if (!PersQueueClient) { PersQueueClient = MakeHolder<NYdb::NPersQueue::TPersQueueClient>(GetDriver()); - } - return *PersQueueClient; - } - } - - NYdb::NPersQueue::TReadSessionSettings GetReadSessionSettings() { - NYdb::NPersQueue::TReadSessionSettings settings; - settings - .ConsumerName(GetTestClient()) - .AppendTopics(GetTestTopic()); - return settings; - } - - NYdb::NPersQueue::TWriteSessionSettings GetWriteSessionSettings() { - TWriteSessionSettings settings; + } + return *PersQueueClient; + } + } + + NYdb::NPersQueue::TReadSessionSettings GetReadSessionSettings() { + NYdb::NPersQueue::TReadSessionSettings settings; + settings + .ConsumerName(GetTestClient()) + .AppendTopics(GetTestTopic()); + return settings; + } + + NYdb::NPersQueue::TWriteSessionSettings GetWriteSessionSettings() { + TWriteSessionSettings settings; settings .Path(GetTestTopic()) .MessageGroupId(GetTestMessageGroupId()); - return settings; - } -}; - -struct TYDBClientEventLoop : public ::NPersQueue::IClientEventLoop { -public: - std::shared_ptr<TPersQueueYdbSdkTestSetup> Setup; - using TAcksCallback = std::function<void (const TVector<ui64>&)>; - - TYDBClientEventLoop( - std::shared_ptr<TPersQueueYdbSdkTestSetup> setup, - IRetryPolicy::TPtr retryPolicy = nullptr, + return settings; + } +}; + +struct TYDBClientEventLoop : public ::NPersQueue::IClientEventLoop { +public: + std::shared_ptr<TPersQueueYdbSdkTestSetup> Setup; + using TAcksCallback = std::function<void (const TVector<ui64>&)>; + + TYDBClientEventLoop( + std::shared_ptr<TPersQueueYdbSdkTestSetup> setup, + IRetryPolicy::TPtr retryPolicy = nullptr, IExecutor::TPtr compressExecutor = nullptr, const TString& preferredCluster = TString(), - const TString& sourceId = TString() - ) - : IClientEventLoop() - , Setup(setup) - { - Log = Setup->GetLog(); + const TString& sourceId = TString() + ) + : IClientEventLoop() + , Setup(setup) + { + Log = Setup->GetLog(); Thread = std::make_unique<TThread>([setup, retryPolicy, compressExecutor, preferredCluster, sourceId, this]() { - auto writerConfig = Setup->GetWriteSessionSettings(); - writerConfig.MaxMemoryUsage(100_MB); - if (!sourceId.empty()) { - writerConfig.MessageGroupId(sourceId); - } - if (retryPolicy != nullptr) - writerConfig.RetryPolicy(retryPolicy); - if (compressExecutor != nullptr) - writerConfig.CompressionExecutor(compressExecutor); + auto writerConfig = Setup->GetWriteSessionSettings(); + writerConfig.MaxMemoryUsage(100_MB); + if (!sourceId.empty()) { + writerConfig.MessageGroupId(sourceId); + } + if (retryPolicy != nullptr) + writerConfig.RetryPolicy(retryPolicy); + if (compressExecutor != nullptr) + writerConfig.CompressionExecutor(compressExecutor); if (preferredCluster) writerConfig.PreferredCluster(preferredCluster); - auto writer = setup->GetPersQueueClient().CreateWriteSession(writerConfig); - - TMaybe<TContinuationToken> continueToken; - NThreading::TFuture<void> waitEventFuture = writer->WaitEvent(); - THashMap<ui64, NThreading::TPromise<::NPersQueue::TWriteResult>> ackPromiseBySequenceNumber; - while (!MustStop) { - if (!continueToken) { - Log << TLOG_INFO << "Wait for writer event"; - waitEventFuture.Wait(); - } - - bool closed = false; - while (waitEventFuture.HasValue() && !closed) { - TWriteSessionEvent::TEvent event = *writer->GetEvent(true); - waitEventFuture = writer->WaitEvent(); - std::visit(TOverloaded { - [&](const TWriteSessionEvent::TAcksEvent& event) { - TVector<ui64> sequenceNumbers; - for (const auto& ack : event.Acks) { - UNIT_ASSERT(ackPromiseBySequenceNumber.contains(ack.SeqNo)); - sequenceNumbers.push_back(ack.SeqNo); - ackPromiseBySequenceNumber[ack.SeqNo].SetValue({true, false}); - ackPromiseBySequenceNumber.erase(ack.SeqNo); - } - }, - [&](TWriteSessionEvent::TReadyToAcceptEvent& event) { - Log << TLOG_INFO << "Got new continue token"; - continueToken = std::move(event.ContinuationToken); - }, - [&](const TSessionClosedEvent& event) { - Log << TLOG_INFO << "===Got close event: " << event.DebugString() << Endl; - if (!MayStop) { - UNIT_ASSERT(MustStop); - UNIT_ASSERT(MessageBuffer.IsEmpty()); - UNIT_ASSERT(ackPromiseBySequenceNumber.empty()); - } else { - MustStop = true; - closed = true; - } - } - }, event); - } - - if (continueToken && !MessageBuffer.IsEmpty()) { - ::NPersQueue::TAcknowledgableMessage acknowledgeableMessage; - Y_VERIFY(MessageBuffer.Dequeue(acknowledgeableMessage)); - ackPromiseBySequenceNumber.emplace(acknowledgeableMessage.SequenceNumber, acknowledgeableMessage.AckPromise); - Y_VERIFY(continueToken); - Log << TLOG_INFO << "Write messages with sequence numbers " << acknowledgeableMessage.SequenceNumber; - writer->Write( - std::move(*continueToken), - std::move(acknowledgeableMessage.Value), - acknowledgeableMessage.SequenceNumber, - acknowledgeableMessage.CreatedAt - ); - continueToken = Nothing(); - } - } - Log << TLOG_DEBUG << "Close writer (stop)"; - writer->Close(TDuration::Zero()); - writer = nullptr; - Log << TLOG_DEBUG << "Writer closed"; - }); - Thread->Start(); - } -}; - -struct TYdbPqTestRetryState : IRetryState { - TYdbPqTestRetryState( - std::function<void ()> retryCallback, std::function<void ()> destroyCallback, const TDuration& delay - ) - : RetryDone(retryCallback) - , DestroyDone(destroyCallback) - , Delay(delay) - {} - + auto writer = setup->GetPersQueueClient().CreateWriteSession(writerConfig); + + TMaybe<TContinuationToken> continueToken; + NThreading::TFuture<void> waitEventFuture = writer->WaitEvent(); + THashMap<ui64, NThreading::TPromise<::NPersQueue::TWriteResult>> ackPromiseBySequenceNumber; + while (!MustStop) { + if (!continueToken) { + Log << TLOG_INFO << "Wait for writer event"; + waitEventFuture.Wait(); + } + + bool closed = false; + while (waitEventFuture.HasValue() && !closed) { + TWriteSessionEvent::TEvent event = *writer->GetEvent(true); + waitEventFuture = writer->WaitEvent(); + std::visit(TOverloaded { + [&](const TWriteSessionEvent::TAcksEvent& event) { + TVector<ui64> sequenceNumbers; + for (const auto& ack : event.Acks) { + UNIT_ASSERT(ackPromiseBySequenceNumber.contains(ack.SeqNo)); + sequenceNumbers.push_back(ack.SeqNo); + ackPromiseBySequenceNumber[ack.SeqNo].SetValue({true, false}); + ackPromiseBySequenceNumber.erase(ack.SeqNo); + } + }, + [&](TWriteSessionEvent::TReadyToAcceptEvent& event) { + Log << TLOG_INFO << "Got new continue token"; + continueToken = std::move(event.ContinuationToken); + }, + [&](const TSessionClosedEvent& event) { + Log << TLOG_INFO << "===Got close event: " << event.DebugString() << Endl; + if (!MayStop) { + UNIT_ASSERT(MustStop); + UNIT_ASSERT(MessageBuffer.IsEmpty()); + UNIT_ASSERT(ackPromiseBySequenceNumber.empty()); + } else { + MustStop = true; + closed = true; + } + } + }, event); + } + + if (continueToken && !MessageBuffer.IsEmpty()) { + ::NPersQueue::TAcknowledgableMessage acknowledgeableMessage; + Y_VERIFY(MessageBuffer.Dequeue(acknowledgeableMessage)); + ackPromiseBySequenceNumber.emplace(acknowledgeableMessage.SequenceNumber, acknowledgeableMessage.AckPromise); + Y_VERIFY(continueToken); + Log << TLOG_INFO << "Write messages with sequence numbers " << acknowledgeableMessage.SequenceNumber; + writer->Write( + std::move(*continueToken), + std::move(acknowledgeableMessage.Value), + acknowledgeableMessage.SequenceNumber, + acknowledgeableMessage.CreatedAt + ); + continueToken = Nothing(); + } + } + Log << TLOG_DEBUG << "Close writer (stop)"; + writer->Close(TDuration::Zero()); + writer = nullptr; + Log << TLOG_DEBUG << "Writer closed"; + }); + Thread->Start(); + } +}; + +struct TYdbPqTestRetryState : IRetryState { + TYdbPqTestRetryState( + std::function<void ()> retryCallback, std::function<void ()> destroyCallback, const TDuration& delay + ) + : RetryDone(retryCallback) + , DestroyDone(destroyCallback) + , Delay(delay) + {} + TMaybe<TDuration> GetNextRetryDelay(const TStatus&) override { - Cerr << "Test retry state: get retry delay\n"; - RetryDone(); - return Delay; - } - std::function<void ()> RetryDone; - std::function<void ()> DestroyDone; - TDuration Delay; - - ~TYdbPqTestRetryState() { - DestroyDone(); - } -}; -struct TYdbPqNoRetryState : IRetryState { - TAtomic DelayCalled = 0; + Cerr << "Test retry state: get retry delay\n"; + RetryDone(); + return Delay; + } + std::function<void ()> RetryDone; + std::function<void ()> DestroyDone; + TDuration Delay; + + ~TYdbPqTestRetryState() { + DestroyDone(); + } +}; +struct TYdbPqNoRetryState : IRetryState { + TAtomic DelayCalled = 0; TMaybe<TDuration> GetNextRetryDelay(const TStatus&) override { - auto res = AtomicSwap(&DelayCalled, 0); - UNIT_ASSERT(!res); - return Nothing(); - } -}; - -struct TYdbPqTestRetryPolicy : IRetryPolicy { - TYdbPqTestRetryPolicy(const TDuration& delay = TDuration::MilliSeconds(2000)) - : Delay(delay) - {} - - IRetryState::TPtr CreateRetryState() const override { - if (AtomicSwap(&OnFatalBreakDown, 0)) { - return std::make_unique<TYdbPqNoRetryState>(); - } - if (AtomicGet(Initialized_)) { - auto res = AtomicSwap(&OnBreakDown, 0); - UNIT_ASSERT(res); - UNIT_ASSERT(AtomicGet(CurrentRetries) == 0); - } - auto retryCb = [this]() mutable {this->RetryDone();}; - auto destroyCb = [this]() mutable {this->StateDestroyed();}; - return std::make_unique<TYdbPqTestRetryState>(retryCb, destroyCb, Delay); - } - - void RetryDone() const { - AtomicAdd(CurrentRetries, 1); - auto expected = AtomicGet(RetriesExpected); - if (expected > 0 && AtomicGet(CurrentRetries) >= expected) { - with_lock(Lock) { - RetryPromise.SetValue(); - } - AtomicSet(RetriesExpected, 0); - } - } - void StateDestroyed() const { - auto expected = AtomicSwap(&RepairExpected, 0); - if (expected) { - with_lock(Lock) { - RepairPromise.SetValue(); - } - } - } - void ExpectBreakDown() { - UNIT_ASSERT(AtomicGet(OnBreakDown) == 0); - AtomicSet(CurrentRetries, 0); - AtomicSet(OnBreakDown, 1); - } - void ExpectFatalBreakDown() { - AtomicSet(OnFatalBreakDown, 1); - } - - void WaitForRetries(ui64 retryCount, NThreading::TPromise<void>& promise) { - AtomicSet(RetriesExpected, retryCount); - with_lock(Lock) { - RetryPromise = promise; - } - } - void WaitForRetriesSync(ui64 retryCount) { - NThreading::TPromise<void> retriesPromise = NThreading::NewPromise(); - auto retriesFuture = retriesPromise.GetFuture(); - WaitForRetries(retryCount, retriesPromise); - retriesFuture.Wait(); - } - - void WaitForRepair(NThreading::TPromise<void>& promise) { - AtomicSet(RepairExpected, 1 ); - with_lock(Lock) { - RepairPromise = promise; - } - } - - void WaitForRepairSync() { - NThreading::TPromise<void> repairPromise = NThreading::NewPromise(); - auto repairFuture = repairPromise.GetFuture(); - WaitForRepair(repairPromise); - repairFuture.Wait(); - } - - void Initialized() { - AtomicSet(Initialized_, 1); - AtomicSet(CurrentRetries, 0); - } -private: - TDuration Delay; - mutable TAtomic CurrentRetries = 0; - mutable TAtomic Initialized_ = 0; - mutable TAtomic OnBreakDown = 0; - mutable TAtomic OnFatalBreakDown = 0; - mutable NThreading::TPromise<void> RetryPromise; - mutable NThreading::TPromise<void> RepairPromise; - mutable TAtomic RetriesExpected = 0; - mutable TAtomic RepairExpected = 0; - mutable TAdaptiveLock Lock; -}; - -class TYdbPqTestExecutor : public IAsyncExecutor { -public: - TYdbPqTestExecutor(std::shared_ptr<TLockFreeQueue<ui64>> idsQueue) - : Stop() - , ExecIdsQueue(idsQueue) - , Thread([idsQueue, this]() { - while(!Stop) { - TFunction f; - while (TasksQueue.Dequeue(&f)) { - ++CurrentTaskId; - Cerr << "Enqueue task with id " << CurrentTaskId << Endl; - Tasks[CurrentTaskId] = f; - } - ui64 id = 0; - while (ExecIdsQueue->Dequeue(&id)) { - ExecIds.push(id); - Cerr << "Got ok to execute task with id " << id << Endl; - - } - while (!ExecIds.empty()) { - auto id = ExecIds.front(); - auto iter = Tasks.find(id); - if (iter == Tasks.end()) - break; - Cerr << "Executing compression of " << id << Endl; - ExecIds.pop(); - try { - (iter->second)(); - } catch (...) { - Cerr << "Failed on compression call: " << CurrentExceptionMessage() << Endl; - Y_FAIL(); - } - Cerr << "Compression of " << id << " Done\n"; - Tasks.erase(iter); - } - - } - }) - { - } - ~TYdbPqTestExecutor() { - Stop = true; - Thread.Join(); - } - void PostImpl(TVector<TFunction>&& fs) override { - for (auto& f : fs) { - TasksQueue.Enqueue(std::move(f)); - } - } - - void PostImpl(TFunction&& f) override { - TasksQueue.Enqueue(std::move(f)); - } - + auto res = AtomicSwap(&DelayCalled, 0); + UNIT_ASSERT(!res); + return Nothing(); + } +}; + +struct TYdbPqTestRetryPolicy : IRetryPolicy { + TYdbPqTestRetryPolicy(const TDuration& delay = TDuration::MilliSeconds(2000)) + : Delay(delay) + {} + + IRetryState::TPtr CreateRetryState() const override { + if (AtomicSwap(&OnFatalBreakDown, 0)) { + return std::make_unique<TYdbPqNoRetryState>(); + } + if (AtomicGet(Initialized_)) { + auto res = AtomicSwap(&OnBreakDown, 0); + UNIT_ASSERT(res); + UNIT_ASSERT(AtomicGet(CurrentRetries) == 0); + } + auto retryCb = [this]() mutable {this->RetryDone();}; + auto destroyCb = [this]() mutable {this->StateDestroyed();}; + return std::make_unique<TYdbPqTestRetryState>(retryCb, destroyCb, Delay); + } + + void RetryDone() const { + AtomicAdd(CurrentRetries, 1); + auto expected = AtomicGet(RetriesExpected); + if (expected > 0 && AtomicGet(CurrentRetries) >= expected) { + with_lock(Lock) { + RetryPromise.SetValue(); + } + AtomicSet(RetriesExpected, 0); + } + } + void StateDestroyed() const { + auto expected = AtomicSwap(&RepairExpected, 0); + if (expected) { + with_lock(Lock) { + RepairPromise.SetValue(); + } + } + } + void ExpectBreakDown() { + UNIT_ASSERT(AtomicGet(OnBreakDown) == 0); + AtomicSet(CurrentRetries, 0); + AtomicSet(OnBreakDown, 1); + } + void ExpectFatalBreakDown() { + AtomicSet(OnFatalBreakDown, 1); + } + + void WaitForRetries(ui64 retryCount, NThreading::TPromise<void>& promise) { + AtomicSet(RetriesExpected, retryCount); + with_lock(Lock) { + RetryPromise = promise; + } + } + void WaitForRetriesSync(ui64 retryCount) { + NThreading::TPromise<void> retriesPromise = NThreading::NewPromise(); + auto retriesFuture = retriesPromise.GetFuture(); + WaitForRetries(retryCount, retriesPromise); + retriesFuture.Wait(); + } + + void WaitForRepair(NThreading::TPromise<void>& promise) { + AtomicSet(RepairExpected, 1 ); + with_lock(Lock) { + RepairPromise = promise; + } + } + + void WaitForRepairSync() { + NThreading::TPromise<void> repairPromise = NThreading::NewPromise(); + auto repairFuture = repairPromise.GetFuture(); + WaitForRepair(repairPromise); + repairFuture.Wait(); + } + + void Initialized() { + AtomicSet(Initialized_, 1); + AtomicSet(CurrentRetries, 0); + } +private: + TDuration Delay; + mutable TAtomic CurrentRetries = 0; + mutable TAtomic Initialized_ = 0; + mutable TAtomic OnBreakDown = 0; + mutable TAtomic OnFatalBreakDown = 0; + mutable NThreading::TPromise<void> RetryPromise; + mutable NThreading::TPromise<void> RepairPromise; + mutable TAtomic RetriesExpected = 0; + mutable TAtomic RepairExpected = 0; + mutable TAdaptiveLock Lock; +}; + +class TYdbPqTestExecutor : public IAsyncExecutor { +public: + TYdbPqTestExecutor(std::shared_ptr<TLockFreeQueue<ui64>> idsQueue) + : Stop() + , ExecIdsQueue(idsQueue) + , Thread([idsQueue, this]() { + while(!Stop) { + TFunction f; + while (TasksQueue.Dequeue(&f)) { + ++CurrentTaskId; + Cerr << "Enqueue task with id " << CurrentTaskId << Endl; + Tasks[CurrentTaskId] = f; + } + ui64 id = 0; + while (ExecIdsQueue->Dequeue(&id)) { + ExecIds.push(id); + Cerr << "Got ok to execute task with id " << id << Endl; + + } + while (!ExecIds.empty()) { + auto id = ExecIds.front(); + auto iter = Tasks.find(id); + if (iter == Tasks.end()) + break; + Cerr << "Executing compression of " << id << Endl; + ExecIds.pop(); + try { + (iter->second)(); + } catch (...) { + Cerr << "Failed on compression call: " << CurrentExceptionMessage() << Endl; + Y_FAIL(); + } + Cerr << "Compression of " << id << " Done\n"; + Tasks.erase(iter); + } + + } + }) + { + } + ~TYdbPqTestExecutor() { + Stop = true; + Thread.Join(); + } + void PostImpl(TVector<TFunction>&& fs) override { + for (auto& f : fs) { + TasksQueue.Enqueue(std::move(f)); + } + } + + void PostImpl(TFunction&& f) override { + TasksQueue.Enqueue(std::move(f)); + } + void DoStart() override { Thread.Start(); } -private: - std::atomic_bool Stop; - TLockFreeQueue<TFunction> TasksQueue; - std::shared_ptr<TLockFreeQueue<ui64>> ExecIdsQueue; - THashMap<ui64, TFunction> Tasks; - TQueue<ui64> ExecIds; - ui64 CurrentTaskId = 0; - TThread Thread; - -}; - - -struct TYdbPqWriterTestHelper { - std::shared_ptr<TPersQueueYdbSdkTestSetup> Setup; - std::shared_ptr<TYdbPqTestRetryPolicy> Policy; - std::unique_ptr<TYDBClientEventLoop> EventLoop; - TIntrusivePtr<TYdbPqTestExecutor> CompressExecutor; - - TAutoEvent MessagesWrittenToBuffer; - ui64 SeqNo = 1; - TString Message = "message"; -public: - TYdbPqWriterTestHelper( - const TString& name, - std::shared_ptr<TLockFreeQueue<ui64>> executorQueue = nullptr, +private: + std::atomic_bool Stop; + TLockFreeQueue<TFunction> TasksQueue; + std::shared_ptr<TLockFreeQueue<ui64>> ExecIdsQueue; + THashMap<ui64, TFunction> Tasks; + TQueue<ui64> ExecIds; + ui64 CurrentTaskId = 0; + TThread Thread; + +}; + + +struct TYdbPqWriterTestHelper { + std::shared_ptr<TPersQueueYdbSdkTestSetup> Setup; + std::shared_ptr<TYdbPqTestRetryPolicy> Policy; + std::unique_ptr<TYDBClientEventLoop> EventLoop; + TIntrusivePtr<TYdbPqTestExecutor> CompressExecutor; + + TAutoEvent MessagesWrittenToBuffer; + ui64 SeqNo = 1; + TString Message = "message"; +public: + TYdbPqWriterTestHelper( + const TString& name, + std::shared_ptr<TLockFreeQueue<ui64>> executorQueue = nullptr, const TString& preferredCluster = TString(), - std::shared_ptr<TPersQueueYdbSdkTestSetup> setup = nullptr, - const TString& sourceId = TString() - ) - : Setup(setup ? setup : std::make_shared<TPersQueueYdbSdkTestSetup>(name)) - , Policy(std::make_shared<TYdbPqTestRetryPolicy>()) - { - if (executorQueue) - CompressExecutor = MakeIntrusive<TYdbPqTestExecutor>(executorQueue); + std::shared_ptr<TPersQueueYdbSdkTestSetup> setup = nullptr, + const TString& sourceId = TString() + ) + : Setup(setup ? setup : std::make_shared<TPersQueueYdbSdkTestSetup>(name)) + , Policy(std::make_shared<TYdbPqTestRetryPolicy>()) + { + if (executorQueue) + CompressExecutor = MakeIntrusive<TYdbPqTestExecutor>(executorQueue); EventLoop = std::make_unique<TYDBClientEventLoop>(Setup, Policy, CompressExecutor, preferredCluster, sourceId); - } - - NThreading::TFuture<::NPersQueue::TWriteResult> Write(bool doWait = false, const TString& message = TString()) { - //auto f = ClientWrite(Message, SeqNo, TInstant::Now()); - auto promise = NThreading::NewPromise<::NPersQueue::TWriteResult>(); - auto log = Setup->GetLog(); - log << TLOG_INFO << "Enqueue message with sequence number " << SeqNo; - EventLoop->MessageBuffer.Enqueue(::NPersQueue::TAcknowledgableMessage{ - message.Empty() ? Message : message, - SeqNo, TInstant::Now(), promise - }); - MessagesWrittenToBuffer.Signal(); - auto f = promise.GetFuture(); - ++SeqNo; - if (doWait) - f.Wait(); - return f; - } - ~TYdbPqWriterTestHelper() { - EventLoop = nullptr; - Setup = nullptr; - CompressExecutor = nullptr; - Policy = nullptr; - } -}; - -class TSimpleWriteSessionTestAdapter { -public: - TSimpleWriteSessionTestAdapter(TSimpleBlockingWriteSession* session); - ui64 GetAcquiredMessagesCount() const; - -private: - TSimpleBlockingWriteSession* Session; -}; - - -void WaitMessagesAcked(std::shared_ptr<IWriteSession> writer, ui64 startSeqNo, ui64 endSeqNo); -} // namespace NYdb::NPersQueue::NTests + } + + NThreading::TFuture<::NPersQueue::TWriteResult> Write(bool doWait = false, const TString& message = TString()) { + //auto f = ClientWrite(Message, SeqNo, TInstant::Now()); + auto promise = NThreading::NewPromise<::NPersQueue::TWriteResult>(); + auto log = Setup->GetLog(); + log << TLOG_INFO << "Enqueue message with sequence number " << SeqNo; + EventLoop->MessageBuffer.Enqueue(::NPersQueue::TAcknowledgableMessage{ + message.Empty() ? Message : message, + SeqNo, TInstant::Now(), promise + }); + MessagesWrittenToBuffer.Signal(); + auto f = promise.GetFuture(); + ++SeqNo; + if (doWait) + f.Wait(); + return f; + } + ~TYdbPqWriterTestHelper() { + EventLoop = nullptr; + Setup = nullptr; + CompressExecutor = nullptr; + Policy = nullptr; + } +}; + +class TSimpleWriteSessionTestAdapter { +public: + TSimpleWriteSessionTestAdapter(TSimpleBlockingWriteSession* session); + ui64 GetAcquiredMessagesCount() const; + +private: + TSimpleBlockingWriteSession* Session; +}; + + +void WaitMessagesAcked(std::shared_ptr<IWriteSession> writer, ui64 startSeqNo, ui64 endSeqNo); +} // namespace NYdb::NPersQueue::NTests diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/sdk_test_setup.h b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/sdk_test_setup.h index 81d915bd83..677eb8c03d 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/sdk_test_setup.h +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/sdk_test_setup.h @@ -2,7 +2,7 @@ #include "test_server.h" #include <ydb/library/persqueue/topic_parser_public/topic_parser.h> #include <library/cpp/logger/log.h> -#include <util/system/tempfile.h> +#include <util/system/tempfile.h> #define TEST_CASE_NAME (this->Name_) @@ -13,8 +13,8 @@ protected: TString TestCaseName; THolder<TTempFileHandle> NetDataFile; - THashMap<TString, NKikimr::NPersQueueTests::TPQTestClusterInfo> DataCenters; - TString LocalDC = "dc1"; + THashMap<TString, NKikimr::NPersQueueTests::TPQTestClusterInfo> DataCenters; + TString LocalDC = "dc1"; TTestServer Server = TTestServer(false /* don't start */); TLog Log = TLog("cerr"); @@ -39,10 +39,10 @@ public: // 2. 'got more than 10 uncommitted reads' Server.ServerSettings.PQConfig.Clear(); Server.ServerSettings.PQConfig.SetEnabled(true); - Server.ServerSettings.PQConfig.SetRemoteClusterEnabledDelaySec(1); - Server.ServerSettings.PQConfig.SetCloseClientSessionWithEnabledRemotePreferredClusterDelaySec(1); + Server.ServerSettings.PQConfig.SetRemoteClusterEnabledDelaySec(1); + Server.ServerSettings.PQConfig.SetCloseClientSessionWithEnabledRemotePreferredClusterDelaySec(1); Server.ServerSettings.PQClusterDiscoveryConfig.SetEnabled(true); - SetNetDataViaFile("::1/128\t" + GetLocalCluster()); + SetNetDataViaFile("::1/128\t" + GetLocalCluster()); auto seed = TInstant::Now().MicroSeconds(); // This makes failing randomized tests (for example with NUnitTest::RandomString(size, std::rand()) calls) reproducable @@ -51,7 +51,7 @@ public: } void Start(bool waitInit = true, bool addBrokenDatacenter = false) { - Server.StartServer(false); + Server.StartServer(false); //Server.EnableLogs({NKikimrServices::PQ_WRITE_PROXY, NKikimrServices::PQ_READ_PROXY}); Server.AnnoyingClient->InitRoot(); if (DataCenters.empty()) { @@ -62,10 +62,10 @@ public: } Server.AnnoyingClient->InitDCs(dataCenters); } else { - Server.AnnoyingClient->InitDCs(DataCenters, LocalDC); + Server.AnnoyingClient->InitDCs(DataCenters, LocalDC); } Server.AnnoyingClient->InitSourceIds(); - CreateTopic(GetTestTopic(), GetLocalCluster()); + CreateTopic(GetTestTopic(), GetLocalCluster()); if (waitInit) { Server.WaitInit(GetTestTopic()); } @@ -84,7 +84,7 @@ public: } TString GetLocalCluster() const { - return LocalDC; + return LocalDC; } ui16 GetGrpcPort() const { @@ -95,13 +95,13 @@ public: return Server.GrpcServerOptions; } - void SetNetDataViaFile(const TString& netDataTsv) { - NetDataFile = MakeHolder<TTempFileHandle>("netData.tsv"); - NetDataFile->Write(netDataTsv.Data(), netDataTsv.Size()); - NetDataFile->FlushData(); - Server.ServerSettings.NetClassifierConfig.SetNetDataFilePath(NetDataFile->Name()); - } - + void SetNetDataViaFile(const TString& netDataTsv) { + NetDataFile = MakeHolder<TTempFileHandle>("netData.tsv"); + NetDataFile->Write(netDataTsv.Data(), netDataTsv.Size()); + NetDataFile->FlushData(); + Server.ServerSettings.NetClassifierConfig.SetNetDataFilePath(NetDataFile->Name()); + } + TLog& GetLog() { return Log; @@ -131,40 +131,40 @@ public: } void SetSingleDataCenter(const TString& name = "dc1") { - UNIT_ASSERT( - DataCenters.insert(std::make_pair( - name, - NKikimr::NPersQueueTests::TPQTestClusterInfo{TStringBuilder() << "localhost:" << Server.GrpcPort, true} - )).second - ); - LocalDC = name; + UNIT_ASSERT( + DataCenters.insert(std::make_pair( + name, + NKikimr::NPersQueueTests::TPQTestClusterInfo{TStringBuilder() << "localhost:" << Server.GrpcPort, true} + )).second + ); + LocalDC = name; } void AddDataCenter(const TString& name, const TString& address, bool enabled = true, bool setSelfAsDc = true) { if (DataCenters.empty() && setSelfAsDc) { SetSingleDataCenter(); } - NKikimr::NPersQueueTests::TPQTestClusterInfo info{ + NKikimr::NPersQueueTests::TPQTestClusterInfo info{ address, - enabled - }; - UNIT_ASSERT(DataCenters.insert(std::make_pair(name, info)).second); + enabled + }; + UNIT_ASSERT(DataCenters.insert(std::make_pair(name, info)).second); } void AddDataCenter(const TString& name, const SDKTestSetup& cluster, bool enabled = true, bool setSelfAsDc = true) { AddDataCenter(name, TStringBuilder() << "localhost:" << cluster.Server.GrpcPort, enabled, setSelfAsDc); } - void EnableDataCenter(const TString& name) { - auto iter = DataCenters.find(name); - UNIT_ASSERT(iter != DataCenters.end()); - Server.AnnoyingClient->UpdateDcEnabled(name, true); - } - void DisableDataCenter(const TString& name) { - auto iter = DataCenters.find(name); - UNIT_ASSERT(iter != DataCenters.end()); - Server.AnnoyingClient->UpdateDcEnabled(name, false); - } + void EnableDataCenter(const TString& name) { + auto iter = DataCenters.find(name); + UNIT_ASSERT(iter != DataCenters.end()); + Server.AnnoyingClient->UpdateDcEnabled(name, true); + } + void DisableDataCenter(const TString& name) { + auto iter = DataCenters.find(name); + UNIT_ASSERT(iter != DataCenters.end()); + Server.AnnoyingClient->UpdateDcEnabled(name, false); + } void ShutdownGRpc() { Server.ShutdownGRpc(); @@ -174,21 +174,21 @@ public: Server.EnableGRpc(); Server.WaitInit(GetTestTopic()); } - - void KickTablets() { + + void KickTablets() { for (ui32 i = 0; i < Server.CleverServer->StaticNodes() + Server.CleverServer->DynamicNodes(); i++) { - Server.AnnoyingClient->MarkNodeInHive(Server.CleverServer->GetRuntime(), i, false); + Server.AnnoyingClient->MarkNodeInHive(Server.CleverServer->GetRuntime(), i, false); } for (ui32 i = 0; i < Server.CleverServer->StaticNodes() + Server.CleverServer->DynamicNodes(); i++) { - Server.AnnoyingClient->KickNodeInHive(Server.CleverServer->GetRuntime(), i); - } - } - void AllowTablets() { + Server.AnnoyingClient->KickNodeInHive(Server.CleverServer->GetRuntime(), i); + } + } + void AllowTablets() { for (ui32 i = 0; i < Server.CleverServer->StaticNodes() + Server.CleverServer->DynamicNodes(); i++) { - Server.AnnoyingClient->MarkNodeInHive(Server.CleverServer->GetRuntime(), i, true); - } - } - + Server.AnnoyingClient->MarkNodeInHive(Server.CleverServer->GetRuntime(), i, true); + } + } + void CreateTopic(const TString& topic, const TString& cluster, size_t partitionsCount = 1) { Server.AnnoyingClient->CreateTopic(BuildFullTopicName(topic, cluster), partitionsCount); } diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/test_server.h b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/test_server.h index 0b4a58d483..250ce03620 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/test_server.h +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/test_server.h @@ -11,43 +11,43 @@ namespace NPersQueue { -static constexpr int DEBUG_LOG_LEVEL = 7; - +static constexpr int DEBUG_LOG_LEVEL = 7; + class TTestServer { public: - TTestServer(bool start = true, TMaybe<TSimpleSharedPtr<TPortManager>> portManager = Nothing()) - : PortManager(portManager.GetOrElse(MakeSimpleShared<TPortManager>())) - , Port(PortManager->GetPort(2134)) - , GrpcPort(PortManager->GetPort(2135)) - , ServerSettings(NKikimr::NPersQueueTests::PQSettings(Port).SetGrpcPort(GrpcPort)) + TTestServer(bool start = true, TMaybe<TSimpleSharedPtr<TPortManager>> portManager = Nothing()) + : PortManager(portManager.GetOrElse(MakeSimpleShared<TPortManager>())) + , Port(PortManager->GetPort(2134)) + , GrpcPort(PortManager->GetPort(2135)) + , ServerSettings(NKikimr::NPersQueueTests::PQSettings(Port).SetGrpcPort(GrpcPort)) , GrpcServerOptions(NGrpc::TServerOptions().SetHost("[::1]").SetPort(GrpcPort)) { if (start) { StartServer(); } } - TTestServer(const NKikimr::Tests::TServerSettings& settings, bool start = true) - : PortManager(MakeSimpleShared<TPortManager>()) - , Port(PortManager->GetPort(2134)) - , GrpcPort(PortManager->GetPort(2135)) - , ServerSettings(settings) - , GrpcServerOptions(NGrpc::TServerOptions().SetHost("[::1]").SetPort(GrpcPort)) - { - ServerSettings.Port = Port; - ServerSettings.SetGrpcPort(GrpcPort); - if (start) - StartServer(); - } - - void StartServer(bool doClientInit = true) { - PrepareNetDataFile(); + TTestServer(const NKikimr::Tests::TServerSettings& settings, bool start = true) + : PortManager(MakeSimpleShared<TPortManager>()) + , Port(PortManager->GetPort(2134)) + , GrpcPort(PortManager->GetPort(2135)) + , ServerSettings(settings) + , GrpcServerOptions(NGrpc::TServerOptions().SetHost("[::1]").SetPort(GrpcPort)) + { + ServerSettings.Port = Port; + ServerSettings.SetGrpcPort(GrpcPort); + if (start) + StartServer(); + } + + void StartServer(bool doClientInit = true) { + PrepareNetDataFile(); CleverServer = MakeHolder<NKikimr::Tests::TServer>(ServerSettings); CleverServer->EnableGRpc(GrpcServerOptions); AnnoyingClient = MakeHolder<NKikimr::NPersQueueTests::TFlatMsgBusPQClient>(ServerSettings, GrpcPort); EnableLogs(LOGGED_SERVICES); - if (doClientInit) { - AnnoyingClient->FullInit(); - } + if (doClientInit) { + AnnoyingClient->FullInit(); + } } void ShutdownGRpc() { @@ -68,7 +68,7 @@ public: } void EnableLogs(const TVector<NKikimrServices::EServiceKikimr> services, - NActors::NLog::EPriority prio = NActors::NLog::PRI_DEBUG) { + NActors::NLog::EPriority prio = NActors::NLog::PRI_DEBUG) { Y_VERIFY(CleverServer != nullptr, "Start server before enabling logs"); for (auto s : services) { CleverServer->GetRuntime()->SetLogPriority(s, prio); @@ -79,14 +79,14 @@ public: AnnoyingClient->WaitTopicInit(topic); } - bool PrepareNetDataFile(const TString& content = "::1/128\tdc1") { - if (NetDataFile) - return false; + bool PrepareNetDataFile(const TString& content = "::1/128\tdc1") { + if (NetDataFile) + return false; NetDataFile = MakeHolder<TTempFileHandle>("netData.tsv"); NetDataFile->Write(content.Data(), content.Size()); NetDataFile->FlushData(); ServerSettings.NetClassifierConfig.SetNetDataFilePath(NetDataFile->Name()); - return true; + return true; } void UpdateDC(const TString& name, bool local, bool enabled) { @@ -94,7 +94,7 @@ public: } public: - TSimpleSharedPtr<TPortManager> PortManager; + TSimpleSharedPtr<TPortManager> PortManager; ui16 Port; ui16 GrpcPort; @@ -105,7 +105,7 @@ public: THolder<NKikimr::NPersQueueTests::TFlatMsgBusPQClient> AnnoyingClient; - + static const TVector<NKikimrServices::EServiceKikimr> LOGGED_SERVICES; }; diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/test_utils.h b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/test_utils.h index cbb4d38886..f5841d6277 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/test_utils.h +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/test_utils.h @@ -1,74 +1,74 @@ #pragma once #include <util/generic/ptr.h> -#include <util/generic/size_literals.h> -#include <library/cpp/threading/chunk_queue/queue.h> +#include <util/generic/size_literals.h> +#include <library/cpp/threading/chunk_queue/queue.h> #include <util/generic/overloaded.h> -#include <library/cpp/testing/unittest/registar.h> +#include <library/cpp/testing/unittest/registar.h> #include "sdk_test_setup.h" - + namespace NPersQueue { -using namespace NThreading; -using namespace NYdb::NPersQueue; -using namespace NKikimr; -using namespace NKikimr::NPersQueueTests; - +using namespace NThreading; +using namespace NYdb::NPersQueue; +using namespace NKikimr; +using namespace NKikimr::NPersQueueTests; + + +struct TWriteResult { + bool Ok = false; + // No acknowledgement is expected from a writer under test + bool NoWait = false; + TString ResponseDebugString = TString(); +}; + +struct TAcknowledgableMessage { + TString Value; + ui64 SequenceNumber; + TInstant CreatedAt; + TPromise<TWriteResult> AckPromise; +}; + +class IClientEventLoop { +protected: + std::atomic_bool MayStop; + std::atomic_bool MustStop; + bool Stopped = false; + std::unique_ptr<TThread> Thread; + TLog Log; + +public: + IClientEventLoop() + : MayStop() + , MustStop() + , MessageBuffer() + {} + + void AllowStop() { + MayStop = true; + } + + void WaitForStop() { + if (!Stopped) { + Log << TLOG_INFO << "Wait for writer to die on itself"; + Thread->Join(); + Log << TLOG_INFO << "Client write event loop stopped"; + } + Stopped = true; + } + + virtual ~IClientEventLoop() { + MustStop = true; + if (!Stopped) { + Log << TLOG_INFO << "Wait for client write event loop to stop"; + Thread->Join(); + Log << TLOG_INFO << "Client write event loop stopped"; + } + Stopped = true; + } + + TManyOneQueue<TAcknowledgableMessage> MessageBuffer; + +}; -struct TWriteResult { - bool Ok = false; - // No acknowledgement is expected from a writer under test - bool NoWait = false; - TString ResponseDebugString = TString(); -}; - -struct TAcknowledgableMessage { - TString Value; - ui64 SequenceNumber; - TInstant CreatedAt; - TPromise<TWriteResult> AckPromise; -}; - -class IClientEventLoop { -protected: - std::atomic_bool MayStop; - std::atomic_bool MustStop; - bool Stopped = false; - std::unique_ptr<TThread> Thread; - TLog Log; - -public: - IClientEventLoop() - : MayStop() - , MustStop() - , MessageBuffer() - {} - - void AllowStop() { - MayStop = true; - } - - void WaitForStop() { - if (!Stopped) { - Log << TLOG_INFO << "Wait for writer to die on itself"; - Thread->Join(); - Log << TLOG_INFO << "Client write event loop stopped"; - } - Stopped = true; - } - - virtual ~IClientEventLoop() { - MustStop = true; - if (!Stopped) { - Log << TLOG_INFO << "Wait for client write event loop to stop"; - Thread->Join(); - Log << TLOG_INFO << "Client write event loop stopped"; - } - Stopped = true; - } - - TManyOneQueue<TAcknowledgableMessage> MessageBuffer; - -}; - } // namespace NPersQueue diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/with_offset_ranges_mode_ut/ya.make b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/with_offset_ranges_mode_ut/ya.make index a08cab9ece..fc27259e6e 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/with_offset_ranges_mode_ut/ya.make +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/with_offset_ranges_mode_ut/ya.make @@ -39,7 +39,7 @@ SRCS( basic_usage_ut.cpp compress_executor_ut.cpp retry_policy_ut.cpp - ut_utils.cpp + ut_utils.cpp ) END() diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ya.make b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ya.make index 178452d4e0..a95f5de06f 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ya.make +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ya.make @@ -34,10 +34,10 @@ SRCS( common_ut.cpp read_session_ut.cpp basic_usage_ut.cpp - compress_executor_ut.cpp + compress_executor_ut.cpp compression_ut.cpp - retry_policy_ut.cpp - ut_utils.cpp + retry_policy_ut.cpp + ut_utils.cpp ) END() diff --git a/ydb/public/sdk/python/ydb/connection.py b/ydb/public/sdk/python/ydb/connection.py index 5bbffcc759..573b276e20 100644 --- a/ydb/public/sdk/python/ydb/connection.py +++ b/ydb/public/sdk/python/ydb/connection.py @@ -206,11 +206,11 @@ def _construct_channel_options(driver_config, endpoint_options=None): if driver_config.channel_options is None: return _default_connect_options channel_options = copy.deepcopy(driver_config.channel_options) - custom_options_keys = set(i[0] for i in driver_config.channel_options) + custom_options_keys = set(i[0] for i in driver_config.channel_options) for item in filter( lambda x: x[0] not in custom_options_keys, _default_connect_options ): - channel_options.append(item) + channel_options.append(item) return channel_options @@ -297,8 +297,8 @@ def channel_factory( ): channel_provider = channel_provider if channel_provider is not None else grpc options = _construct_channel_options(driver_config, endpoint_options) - logger.debug("Channel options: {}".format(options)) - + logger.debug("Channel options: {}".format(options)) + if driver_config.root_certificates is None and not driver_config.secure_channel: return channel_provider.insecure_channel(endpoint, options) root_certificates = driver_config.root_certificates diff --git a/ydb/services/datastreams/datastreams_ut.cpp b/ydb/services/datastreams/datastreams_ut.cpp index b0940265d2..2ede2a2d05 100644 --- a/ydb/services/datastreams/datastreams_ut.cpp +++ b/ydb/services/datastreams/datastreams_ut.cpp @@ -10,9 +10,9 @@ #include <ydb/public/sdk/cpp/client/ydb_scheme/scheme.h> #include <ydb/public/api/grpc/draft/ydb_datastreams_v1.grpc.pb.h> -#include <library/cpp/json/json_reader.h> +#include <library/cpp/json/json_reader.h> #include <library/cpp/digest/md5/md5.h> - + #include <random> @@ -30,22 +30,22 @@ using TKikimrWithGrpcAndRootSchemaSecure = NYdb::TBasicKikimrWithGrpcAndRootSche template<class TKikimr, bool secure> -class TDatastreamsTestServer { +class TDatastreamsTestServer { public: - TDatastreamsTestServer() { + TDatastreamsTestServer() { NKikimrConfig::TAppConfig appConfig; appConfig.MutablePQConfig()->SetTopicsAreFirstClassCitizen(true); appConfig.MutablePQConfig()->SetEnabled(true); - appConfig.MutablePQConfig()->SetMetaCacheRefreshIntervalMilliSeconds(30000); + appConfig.MutablePQConfig()->SetMetaCacheRefreshIntervalMilliSeconds(30000); appConfig.MutablePQConfig()->MutableQuotingConfig()->SetEnableQuoting(true); appConfig.MutablePQConfig()->MutableQuotingConfig()->SetQuotaWaitDurationMs(300); appConfig.MutablePQConfig()->MutableQuotingConfig()->SetPartitionReadQuotaIsTwiceWriteQuota(true); - appConfig.MutablePQConfig()->MutableBillingMeteringConfig()->SetEnabled(true); - appConfig.MutablePQConfig()->MutableBillingMeteringConfig()->SetFlushIntervalSec(1); + appConfig.MutablePQConfig()->MutableBillingMeteringConfig()->SetEnabled(true); + appConfig.MutablePQConfig()->MutableBillingMeteringConfig()->SetFlushIntervalSec(1); appConfig.MutablePQConfig()->AddClientServiceType()->SetName("data-streams"); - - MeteringFile = MakeHolder<TTempFileHandle>("meteringData.txt"); - appConfig.MutableMeteringConfig()->SetMeteringFilePath(MeteringFile->Name()); + + MeteringFile = MakeHolder<TTempFileHandle>("meteringData.txt"); + appConfig.MutableMeteringConfig()->SetMeteringFilePath(MeteringFile->Name()); if (secure) { appConfig.MutablePQConfig()->SetRequireCredentialsInNewProtocol(true); @@ -86,11 +86,11 @@ public: std::unique_ptr<TDriver> Driver; std::unique_ptr<NYDS_V1::TDataStreamsClient> DataStreamsClient; std::unique_ptr<NYDS_V1::TDataStreamsClient> UnauthenticatedClient; - THolder<TTempFileHandle> MeteringFile; + THolder<TTempFileHandle> MeteringFile; }; -using TInsecureDatastreamsTestServer = TDatastreamsTestServer<TKikimrWithGrpcAndRootSchema, false>; -using TSecureDatastreamsTestServer = TDatastreamsTestServer<TKikimrWithGrpcAndRootSchemaSecure, true>; +using TInsecureDatastreamsTestServer = TDatastreamsTestServer<TKikimrWithGrpcAndRootSchema, false>; +using TSecureDatastreamsTestServer = TDatastreamsTestServer<TKikimrWithGrpcAndRootSchemaSecure, true>; void CheckMeteringFile(TTempFileHandle* meteringFile, const TString& streamPath) { Sleep(TDuration::Seconds(1)); @@ -133,8 +133,8 @@ void CheckMeteringFile(TTempFileHandle* meteringFile, const TString& streamPath) Y_UNIT_TEST_SUITE(DataStreams) { - Y_UNIT_TEST(TestControlPlaneAndMeteringData) { - TInsecureDatastreamsTestServer testServer; + Y_UNIT_TEST(TestControlPlaneAndMeteringData) { + TInsecureDatastreamsTestServer testServer; const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; // Trying to delete stream that doesn't exist yet { @@ -206,7 +206,7 @@ Y_UNIT_TEST_SUITE(DataStreams) { } { - auto result = testServer.DataStreamsClient->ListStreams().ExtractValueSync(); + auto result = testServer.DataStreamsClient->ListStreams().ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_names().size(), 1); @@ -245,8 +245,8 @@ Y_UNIT_TEST_SUITE(DataStreams) { CheckMeteringFile(testServer.MeteringFile.Get(), "/Root/" + streamName); } - Y_UNIT_TEST(TestCreateExistingStream) { - TInsecureDatastreamsTestServer testServer; + Y_UNIT_TEST(TestCreateExistingStream) { + TInsecureDatastreamsTestServer testServer; const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; { auto result = testServer.DataStreamsClient->CreateStream(streamName, @@ -264,8 +264,8 @@ Y_UNIT_TEST_SUITE(DataStreams) { } - Y_UNIT_TEST(TestStreamPagination) { - TInsecureDatastreamsTestServer testServer; + Y_UNIT_TEST(TestStreamPagination) { + TInsecureDatastreamsTestServer testServer; const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; for (ui32 folderIdx = 0; folderIdx < 4; folderIdx++) { for (ui32 streamIdx = 0; streamIdx < 5; streamIdx++) { @@ -389,8 +389,8 @@ Y_UNIT_TEST_SUITE(DataStreams) { } } - Y_UNIT_TEST(TestUpdateStream) { - TInsecureDatastreamsTestServer testServer; + Y_UNIT_TEST(TestUpdateStream) { + TInsecureDatastreamsTestServer testServer; const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; { auto result = testServer.DataStreamsClient->CreateStream(streamName, @@ -419,8 +419,8 @@ Y_UNIT_TEST_SUITE(DataStreams) { } } - Y_UNIT_TEST(TestStreamRetention) { - TInsecureDatastreamsTestServer testServer; + Y_UNIT_TEST(TestStreamRetention) { + TInsecureDatastreamsTestServer testServer; const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; { auto result = testServer.DataStreamsClient->CreateStream(streamName, @@ -463,8 +463,8 @@ Y_UNIT_TEST_SUITE(DataStreams) { } - Y_UNIT_TEST(TestShardPagination) { - TInsecureDatastreamsTestServer testServer; + Y_UNIT_TEST(TestShardPagination) { + TInsecureDatastreamsTestServer testServer; const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; { auto result = testServer.DataStreamsClient->CreateStream("/Root/" + streamName, NYDS_V1::TCreateStreamSettings().ShardCount(9)).ExtractValueSync(); @@ -516,26 +516,26 @@ Y_UNIT_TEST_SUITE(DataStreams) { } - -#define SET_YDS_LOCALS \ - auto& kikimr = testServer.KikimrServer->Server_; \ - Y_UNUSED(kikimr); \ - auto& driver = testServer.Driver; \ - Y_UNUSED(driver); \ - - - Y_UNIT_TEST(TestPutRecordsWithRead) { - TInsecureDatastreamsTestServer testServer; + +#define SET_YDS_LOCALS \ + auto& kikimr = testServer.KikimrServer->Server_; \ + Y_UNUSED(kikimr); \ + auto& driver = testServer.Driver; \ + Y_UNUSED(driver); \ + + + Y_UNIT_TEST(TestPutRecordsWithRead) { + TInsecureDatastreamsTestServer testServer; const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; - SET_YDS_LOCALS; + SET_YDS_LOCALS; { auto result = testServer.DataStreamsClient->CreateStream(streamName, NYDS_V1::TCreateStreamSettings().ShardCount(5)).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); } - kikimr->GetRuntime()->SetLogPriority(NKikimrServices::PQ_READ_PROXY, NLog::EPriority::PRI_DEBUG); - kikimr->GetRuntime()->SetLogPriority(NKikimrServices::PQ_WRITE_PROXY, NLog::EPriority::PRI_DEBUG); + kikimr->GetRuntime()->SetLogPriority(NKikimrServices::PQ_READ_PROXY, NLog::EPriority::PRI_DEBUG); + kikimr->GetRuntime()->SetLogPriority(NKikimrServices::PQ_WRITE_PROXY, NLog::EPriority::PRI_DEBUG); NYDS_V1::TDataStreamsClient client(*driver, TCommonClientSettings().AuthToken("user2@builtin")); @@ -559,7 +559,7 @@ Y_UNIT_TEST_SUITE(DataStreams) { UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); } - NYdb::NPersQueue::TPersQueueClient pqClient(*driver); + NYdb::NPersQueue::TPersQueueClient pqClient(*driver); { auto result = testServer.DataStreamsClient->RegisterStreamConsumer(streamName, "user1", NYDS_V1::TRegisterStreamConsumerSettings()).ExtractValueSync(); @@ -601,20 +601,20 @@ Y_UNIT_TEST_SUITE(DataStreams) { UNIT_ASSERT_VALUES_EQUAL(readCount, 31); } - Y_UNIT_TEST(TestPutRecordsCornerCases) { - TInsecureDatastreamsTestServer testServer; + Y_UNIT_TEST(TestPutRecordsCornerCases) { + TInsecureDatastreamsTestServer testServer; const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; const TString streamPath = "/Root/" + streamName; - SET_YDS_LOCALS; + SET_YDS_LOCALS; { - auto result = testServer.DataStreamsClient->CreateStream( + auto result = testServer.DataStreamsClient->CreateStream( streamName, NYDS_V1::TCreateStreamSettings().ShardCount(5) ).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); } - kikimr->GetRuntime()->SetLogPriority(NKikimrServices::PQ_READ_PROXY, NLog::EPriority::PRI_DEBUG); + kikimr->GetRuntime()->SetLogPriority(NKikimrServices::PQ_READ_PROXY, NLog::EPriority::PRI_DEBUG); NYDS_V1::TDataStreamsClient client(*driver, TCommonClientSettings().AuthToken("user2@builtin")); // Test for too long partition key @@ -679,8 +679,8 @@ Y_UNIT_TEST_SUITE(DataStreams) { UNIT_ASSERT(result.IsSuccess()); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - kikimr->GetRuntime()->SetLogPriority(NKikimrServices::PQ_READ_PROXY, NLog::EPriority::PRI_INFO); - kikimr->GetRuntime()->SetLogPriority(NKikimrServices::PERSQUEUE, NLog::EPriority::PRI_INFO); + kikimr->GetRuntime()->SetLogPriority(NKikimrServices::PQ_READ_PROXY, NLog::EPriority::PRI_INFO); + kikimr->GetRuntime()->SetLogPriority(NKikimrServices::PERSQUEUE, NLog::EPriority::PRI_INFO); { std::vector<NYDS_V1::TDataRecord> records; @@ -722,7 +722,7 @@ Y_UNIT_TEST_SUITE(DataStreams) { } - NYdb::NPersQueue::TPersQueueClient pqClient(*driver); + NYdb::NPersQueue::TPersQueueClient pqClient(*driver); { NYdb::NPersQueue::TAddReadRuleSettings addReadRuleSettings; @@ -756,10 +756,10 @@ Y_UNIT_TEST_SUITE(DataStreams) { UNIT_ASSERT_VALUES_EQUAL(readCount, 14); } - Y_UNIT_TEST(TestPutRecords) { - TSecureDatastreamsTestServer testServer; + Y_UNIT_TEST(TestPutRecords) { + TSecureDatastreamsTestServer testServer; const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; - SET_YDS_LOCALS; + SET_YDS_LOCALS; const TString streamPath = "/Root/" + streamName; { auto result = testServer.DataStreamsClient->CreateStream(streamPath, @@ -769,14 +769,14 @@ Y_UNIT_TEST_SUITE(DataStreams) { } NYDS_V1::TDataStreamsClient client(*driver, TCommonClientSettings().AuthToken("user2@builtin")); - NYdb::NScheme::TSchemeClient schemeClient(*driver); + NYdb::NScheme::TSchemeClient schemeClient(*driver); { std::vector<NYDS_V1::TDataRecord> records; for (ui32 i = 1; i <= 30; ++i) { TString data = Sprintf("%04u", i); records.push_back({data, data, ""}); } - auto result = client.PutRecords(streamPath, records).ExtractValueSync(); + auto result = client.PutRecords(streamPath, records).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); if (result.GetStatus() != EStatus::SUCCESS) { result.GetIssues().PrintTo(Cerr); @@ -790,7 +790,7 @@ Y_UNIT_TEST_SUITE(DataStreams) { UNIT_ASSERT(result.IsSuccess()); } - result = client.PutRecords(streamPath, records).ExtractValueSync(); + result = client.PutRecords(streamPath, records).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); Cerr << "PutRecordsResponse = " << result.GetResult().DebugString() << Endl; @@ -799,7 +799,7 @@ Y_UNIT_TEST_SUITE(DataStreams) { UNIT_ASSERT_VALUES_EQUAL(result.GetResult().encryption_type(), YDS_V1::EncryptionType::NONE); TString dataStr = "9876543210"; - auto putRecordResult = client.PutRecord(streamPath, {dataStr, dataStr, ""}).ExtractValueSync(); + auto putRecordResult = client.PutRecord(streamPath, {dataStr, dataStr, ""}).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(putRecordResult.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(putRecordResult.GetStatus(), EStatus::SUCCESS); Cerr << "PutRecord response = " << putRecordResult.GetResult().DebugString() << Endl; @@ -811,18 +811,18 @@ Y_UNIT_TEST_SUITE(DataStreams) { } } - Y_UNIT_TEST(TestPutEmptyMessage) { - TInsecureDatastreamsTestServer testServer; + Y_UNIT_TEST(TestPutEmptyMessage) { + TInsecureDatastreamsTestServer testServer; const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; - SET_YDS_LOCALS; + SET_YDS_LOCALS; { auto result = testServer.DataStreamsClient->CreateStream(streamName, NYDS_V1::TCreateStreamSettings().ShardCount(5)).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); } - kikimr->GetRuntime()->SetLogPriority(NKikimrServices::PQ_READ_PROXY, NLog::EPriority::PRI_DEBUG); - kikimr->GetRuntime()->SetLogPriority(NKikimrServices::PQ_WRITE_PROXY, NLog::EPriority::PRI_DEBUG); + kikimr->GetRuntime()->SetLogPriority(NKikimrServices::PQ_READ_PROXY, NLog::EPriority::PRI_DEBUG); + kikimr->GetRuntime()->SetLogPriority(NKikimrServices::PQ_WRITE_PROXY, NLog::EPriority::PRI_DEBUG); NYDS_V1::TDataStreamsClient client(*driver, TCommonClientSettings().AuthToken("user2@builtin")); @@ -834,7 +834,7 @@ Y_UNIT_TEST_SUITE(DataStreams) { UNIT_ASSERT_VALUES_EQUAL(putRecordResult.GetStatus(), EStatus::SUCCESS); UNIT_ASSERT_VALUES_EQUAL(putRecordResult.GetResult().sequence_number(), "0"); - NYdb::NPersQueue::TPersQueueClient pqClient(*driver); + NYdb::NPersQueue::TPersQueueClient pqClient(*driver); { auto result = testServer.DataStreamsClient->RegisterStreamConsumer(streamName, "user1", NYDS_V1::TRegisterStreamConsumerSettings()).ExtractValueSync(); @@ -873,8 +873,8 @@ Y_UNIT_TEST_SUITE(DataStreams) { } } - Y_UNIT_TEST(TestListStreamConsumers) { - TInsecureDatastreamsTestServer testServer; + Y_UNIT_TEST(TestListStreamConsumers) { + TInsecureDatastreamsTestServer testServer; const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; // Create stream -> OK { @@ -1070,8 +1070,8 @@ Y_UNIT_TEST_SUITE(DataStreams) { } } - Y_UNIT_TEST(TestGetShardIterator) { - TInsecureDatastreamsTestServer testServer; + Y_UNIT_TEST(TestGetShardIterator) { + TInsecureDatastreamsTestServer testServer; const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; { auto result = testServer.DataStreamsClient->CreateStream(streamName, \ @@ -1179,8 +1179,8 @@ Y_UNIT_TEST_SUITE(DataStreams) { } } - Y_UNIT_TEST(TestGetRecordsStreamWithSingleShard) { - TInsecureDatastreamsTestServer testServer; + Y_UNIT_TEST(TestGetRecordsStreamWithSingleShard) { + TInsecureDatastreamsTestServer testServer; const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; { auto result = testServer.DataStreamsClient->CreateStream(streamName, NYDS_V1::TCreateStreamSettings().ShardCount(1)).ExtractValueSync(); @@ -1213,7 +1213,7 @@ Y_UNIT_TEST_SUITE(DataStreams) { } { - auto result = testServer.DataStreamsClient->GetRecords(shardIterator).ExtractValueSync(); + auto result = testServer.DataStreamsClient->GetRecords(shardIterator).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), 0); @@ -1228,7 +1228,7 @@ Y_UNIT_TEST_SUITE(DataStreams) { } { - auto result = testServer.DataStreamsClient->GetRecords(shardIterator).ExtractValueSync(); + auto result = testServer.DataStreamsClient->GetRecords(shardIterator).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), recordsCount); @@ -1245,7 +1245,7 @@ Y_UNIT_TEST_SUITE(DataStreams) { } { - auto result = testServer.DataStreamsClient->GetRecords(shardIterator).ExtractValueSync(); + auto result = testServer.DataStreamsClient->GetRecords(shardIterator).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), recordsCount - 2); @@ -1262,7 +1262,7 @@ Y_UNIT_TEST_SUITE(DataStreams) { } { - auto result = testServer.DataStreamsClient->GetRecords(shardIterator).ExtractValueSync(); + auto result = testServer.DataStreamsClient->GetRecords(shardIterator).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), recordsCount - 3); @@ -1286,7 +1286,7 @@ Y_UNIT_TEST_SUITE(DataStreams) { } { - auto result = testServer.DataStreamsClient->GetRecords(shardIterator).ExtractValueSync(); + auto result = testServer.DataStreamsClient->GetRecords(shardIterator).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), recordsCount); @@ -1294,10 +1294,10 @@ Y_UNIT_TEST_SUITE(DataStreams) { } - Y_UNIT_TEST(TestGetRecordsWithoutPermission) { - TInsecureDatastreamsTestServer testServer; + Y_UNIT_TEST(TestGetRecordsWithoutPermission) { + TInsecureDatastreamsTestServer testServer; const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; - SET_YDS_LOCALS; + SET_YDS_LOCALS; { auto result = testServer.DataStreamsClient->CreateStream("/Root/" + streamName, NYDS_V1::TCreateStreamSettings().ShardCount(1)).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); @@ -1316,7 +1316,7 @@ Y_UNIT_TEST_SUITE(DataStreams) { } NYDS_V1::TDataStreamsClient client(*driver, TCommonClientSettings().AuthToken("user2@builtin")); - NYdb::NScheme::TSchemeClient schemeClient(*driver); + NYdb::NScheme::TSchemeClient schemeClient(*driver); TString shardIterator; { @@ -1365,8 +1365,8 @@ Y_UNIT_TEST_SUITE(DataStreams) { } } - Y_UNIT_TEST(TestGetRecords1MBMessagesOneByOneBySeqNo) { - TInsecureDatastreamsTestServer testServer; + Y_UNIT_TEST(TestGetRecords1MBMessagesOneByOneBySeqNo) { + TInsecureDatastreamsTestServer testServer; const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; { auto result = testServer.DataStreamsClient->CreateStream(streamName, @@ -1410,7 +1410,7 @@ Y_UNIT_TEST_SUITE(DataStreams) { } { - auto result = testServer.DataStreamsClient->GetRecords(shardIterator, + auto result = testServer.DataStreamsClient->GetRecords(shardIterator, NYDS_V1::TGetRecordsSettings().Limit(1)).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); @@ -1423,8 +1423,8 @@ Y_UNIT_TEST_SUITE(DataStreams) { } } - Y_UNIT_TEST(TestGetRecords1MBMessagesOneByOneByTS) { - TInsecureDatastreamsTestServer testServer; + Y_UNIT_TEST(TestGetRecords1MBMessagesOneByOneByTS) { + TInsecureDatastreamsTestServer testServer; const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; { auto result = testServer.DataStreamsClient->CreateStream(streamName, @@ -1469,7 +1469,7 @@ Y_UNIT_TEST_SUITE(DataStreams) { } { - auto result = testServer.DataStreamsClient->GetRecords(shardIterator, + auto result = testServer.DataStreamsClient->GetRecords(shardIterator, NYDS_V1::TGetRecordsSettings().Limit(1)).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); @@ -1480,8 +1480,8 @@ Y_UNIT_TEST_SUITE(DataStreams) { } - Y_UNIT_TEST(TestGetRecordsStreamWithMultipleShards) { - TInsecureDatastreamsTestServer testServer; + Y_UNIT_TEST(TestGetRecordsStreamWithMultipleShards) { + TInsecureDatastreamsTestServer testServer; const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; { auto result = testServer.DataStreamsClient->CreateStream(streamName, NYDS_V1::TCreateStreamSettings().ShardCount(5)).ExtractValueSync(); @@ -1514,7 +1514,7 @@ Y_UNIT_TEST_SUITE(DataStreams) { } { - auto result = testServer.DataStreamsClient->GetRecords(shardIterator).ExtractValueSync(); + auto result = testServer.DataStreamsClient->GetRecords(shardIterator).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), 0); @@ -1529,7 +1529,7 @@ Y_UNIT_TEST_SUITE(DataStreams) { } { - auto result = testServer.DataStreamsClient->GetRecords(shardIterator).ExtractValueSync(); + auto result = testServer.DataStreamsClient->GetRecords(shardIterator).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), 0); @@ -1544,7 +1544,7 @@ Y_UNIT_TEST_SUITE(DataStreams) { } { - auto result = testServer.DataStreamsClient->GetRecords(shardIterator).ExtractValueSync(); + auto result = testServer.DataStreamsClient->GetRecords(shardIterator).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), 0); @@ -1559,15 +1559,15 @@ Y_UNIT_TEST_SUITE(DataStreams) { } { - auto result = testServer.DataStreamsClient->GetRecords(shardIterator).ExtractValueSync(); + auto result = testServer.DataStreamsClient->GetRecords(shardIterator).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), 0); } } - Y_UNIT_TEST(TestListShards1Shard) { - TInsecureDatastreamsTestServer testServer; + Y_UNIT_TEST(TestListShards1Shard) { + TInsecureDatastreamsTestServer testServer; const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; { auto result = testServer.DataStreamsClient->CreateStream(streamName, @@ -1626,70 +1626,70 @@ Y_UNIT_TEST_SUITE(DataStreams) { } } - Y_UNIT_TEST(TestUnsupported) { - TInsecureDatastreamsTestServer testServer; + Y_UNIT_TEST(TestUnsupported) { + TInsecureDatastreamsTestServer testServer; { - auto result = testServer.DataStreamsClient->DescribeLimits().ExtractValueSync(); + auto result = testServer.DataStreamsClient->DescribeLimits().ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNSUPPORTED); } { - auto result = testServer.DataStreamsClient->DescribeStreamConsumer().ExtractValueSync(); + auto result = testServer.DataStreamsClient->DescribeStreamConsumer().ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNSUPPORTED); } { - auto result = testServer.DataStreamsClient->AddTagsToStream().ExtractValueSync(); + auto result = testServer.DataStreamsClient->AddTagsToStream().ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNSUPPORTED); } { - auto result = testServer.DataStreamsClient->DisableEnhancedMonitoring().ExtractValueSync(); + auto result = testServer.DataStreamsClient->DisableEnhancedMonitoring().ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNSUPPORTED); } { - auto result = testServer.DataStreamsClient->EnableEnhancedMonitoring().ExtractValueSync(); + auto result = testServer.DataStreamsClient->EnableEnhancedMonitoring().ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNSUPPORTED); } { - auto result = testServer.DataStreamsClient->ListTagsForStream().ExtractValueSync(); + auto result = testServer.DataStreamsClient->ListTagsForStream().ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNSUPPORTED); } { - auto result = testServer.DataStreamsClient->MergeShards().ExtractValueSync(); + auto result = testServer.DataStreamsClient->MergeShards().ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNSUPPORTED); } { - auto result = testServer.DataStreamsClient->RemoveTagsFromStream().ExtractValueSync(); + auto result = testServer.DataStreamsClient->RemoveTagsFromStream().ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNSUPPORTED); } { - auto result = testServer.DataStreamsClient->SplitShard().ExtractValueSync(); + auto result = testServer.DataStreamsClient->SplitShard().ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNSUPPORTED); } { - auto result = testServer.DataStreamsClient->StartStreamEncryption().ExtractValueSync(); + auto result = testServer.DataStreamsClient->StartStreamEncryption().ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNSUPPORTED); } { - auto result = testServer.DataStreamsClient->StopStreamEncryption().ExtractValueSync(); + auto result = testServer.DataStreamsClient->StopStreamEncryption().ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNSUPPORTED); } diff --git a/ydb/services/datastreams/put_records_actor.h b/ydb/services/datastreams/put_records_actor.h index 5fde6ee96f..8f06a99157 100644 --- a/ydb/services/datastreams/put_records_actor.h +++ b/ydb/services/datastreams/put_records_actor.h @@ -13,12 +13,12 @@ #include <library/cpp/digest/md5/md5.h> -#define PUT_UNIT_SIZE 40960u // 40Kb +#define PUT_UNIT_SIZE 40960u // 40Kb namespace NKikimr::NDataStreams::V1 { - + struct TPutRecordsItem { TString Data; TString Key; @@ -98,7 +98,7 @@ namespace NKikimr::NDataStreams::V1 { request.MutablePartitionRequest()->SetPartition(Partition); request.MutablePartitionRequest()->SetIsDirectWrite(true); ActorIdToProto(PipeClient, request.MutablePartitionRequest()->MutablePipeClient()); - ui64 totalSize = 0; + ui64 totalSize = 0; for (const auto& item : DataToWrite) { auto w = request.MutablePartitionRequest()->AddCmdWrite(); w->SetData(GetSerializedData(item)); @@ -108,12 +108,12 @@ namespace NKikimr::NDataStreams::V1 { w->SetCreateTimeMS(TInstant::Now().MilliSeconds()); w->SetUncompressedSize(item.Data.size()); w->SetExternalOperation(true); - totalSize += (item.Data.size() + item.Key.size() + item.ExplicitHash.size()); + totalSize += (item.Data.size() + item.Key.size() + item.ExplicitHash.size()); } - ui64 putUnitsCount = totalSize / PUT_UNIT_SIZE; - if (totalSize % PUT_UNIT_SIZE != 0) - putUnitsCount++; - request.MutablePartitionRequest()->SetPutUnitsSize(putUnitsCount); + ui64 putUnitsCount = totalSize / PUT_UNIT_SIZE; + if (totalSize % PUT_UNIT_SIZE != 0) + putUnitsCount++; + request.MutablePartitionRequest()->SetPutUnitsSize(putUnitsCount); TAutoPtr<TEvPersQueue::TEvRequest> req(new TEvPersQueue::TEvRequest); req->Record.Swap(&request); diff --git a/ydb/services/lib/actors/pq_schema_actor.cpp b/ydb/services/lib/actors/pq_schema_actor.cpp index 38aa8f8d0a..f48e398488 100644 --- a/ydb/services/lib/actors/pq_schema_actor.cpp +++ b/ydb/services/lib/actors/pq_schema_actor.cpp @@ -11,8 +11,8 @@ namespace NKikimr::NGRpcProxy::V1 { -#define DEFAULT_PARTITION_SPEED 1048576 // 1Mb - +#define DEFAULT_PARTITION_SPEED 1048576 // 1Mb + constexpr i32 MAX_READ_RULES_COUNT = 3000; constexpr TStringBuf GRPCS_ENDPOINT_PREFIX = "grpcs://"; @@ -337,23 +337,23 @@ namespace NKikimr::NGRpcProxy::V1 { } if (local) { - auto partSpeed = settings.max_partition_write_speed(); - if (partSpeed < 0) { - error = TStringBuilder() << "max_partition_write_speed can't be negative, provided " << partSpeed; + auto partSpeed = settings.max_partition_write_speed(); + if (partSpeed < 0) { + error = TStringBuilder() << "max_partition_write_speed can't be negative, provided " << partSpeed; return Ydb::StatusIds::BAD_REQUEST; - } else if (partSpeed == 0) { - partSpeed = DEFAULT_PARTITION_SPEED; + } else if (partSpeed == 0) { + partSpeed = DEFAULT_PARTITION_SPEED; } - partConfig->SetWriteSpeedInBytesPerSecond(partSpeed); + partConfig->SetWriteSpeedInBytesPerSecond(partSpeed); - const auto& burstSpeed = settings.max_partition_write_burst(); - if (burstSpeed < 0) { - error = TStringBuilder() << "max_partition_write_burst can't be negative, provided " << burstSpeed; + const auto& burstSpeed = settings.max_partition_write_burst(); + if (burstSpeed < 0) { + error = TStringBuilder() << "max_partition_write_burst can't be negative, provided " << burstSpeed; return Ydb::StatusIds::BAD_REQUEST; - } else if (burstSpeed == 0) { - partConfig->SetBurstSize(partSpeed); - } else { - partConfig->SetBurstSize(burstSpeed); + } else if (burstSpeed == 0) { + partConfig->SetBurstSize(partSpeed); + } else { + partConfig->SetBurstSize(burstSpeed); } } diff --git a/ydb/services/lib/actors/pq_schema_actor.h b/ydb/services/lib/actors/pq_schema_actor.h index 6d77b205ee..1be2e7fa24 100644 --- a/ydb/services/lib/actors/pq_schema_actor.h +++ b/ydb/services/lib/actors/pq_schema_actor.h @@ -58,7 +58,7 @@ namespace NKikimr::NGRpcProxy::V1 { { } - void PrepareTopicPath(const NActors::TActorContext &ctx) { // ToDo !! + void PrepareTopicPath(const NActors::TActorContext &ctx) { // ToDo !! TopicPath = NPersQueue::GetFullTopicPath(ctx, this->Request_->GetDatabaseName(), TopicPath); } diff --git a/ydb/services/lib/actors/type_definitions.h b/ydb/services/lib/actors/type_definitions.h index 5f77f3f843..127e7d987b 100644 --- a/ydb/services/lib/actors/type_definitions.h +++ b/ydb/services/lib/actors/type_definitions.h @@ -1,39 +1,39 @@ -#pragma once - -#include <ydb/library/persqueue/topic_parser/topic_parser.h> -#include <library/cpp/actors/core/actor.h> -#include <library/cpp/actors/core/event_local.h> - - -namespace NKikimr::NGRpcProxy { - - struct TTopicHolder { - ui64 TabletID; - TActorId PipeClient; - bool ACLRequestInfly; - TString CloudId; - TString DbId; - TString FolderId; - NPersQueue::TConverterPtr TopicNameConverter; - - TVector<ui32> Groups; - TMap<ui64, ui64> Partitions; - - TTopicHolder() - : TabletID(0) - , PipeClient() - , ACLRequestInfly(false) - {} - }; - - struct TTopicInitInfo { - NPersQueue::TConverterPtr TopicNameConverter; - ui64 TabletID; - TString CloudId; - TString DbId; - TString FolderId; - }; - - using TTopicTabletsPairs = TVector<TTopicInitInfo>; - -} // namespace NKikimr::NGRpcProxy +#pragma once + +#include <ydb/library/persqueue/topic_parser/topic_parser.h> +#include <library/cpp/actors/core/actor.h> +#include <library/cpp/actors/core/event_local.h> + + +namespace NKikimr::NGRpcProxy { + + struct TTopicHolder { + ui64 TabletID; + TActorId PipeClient; + bool ACLRequestInfly; + TString CloudId; + TString DbId; + TString FolderId; + NPersQueue::TConverterPtr TopicNameConverter; + + TVector<ui32> Groups; + TMap<ui64, ui64> Partitions; + + TTopicHolder() + : TabletID(0) + , PipeClient() + , ACLRequestInfly(false) + {} + }; + + struct TTopicInitInfo { + NPersQueue::TConverterPtr TopicNameConverter; + ui64 TabletID; + TString CloudId; + TString DbId; + TString FolderId; + }; + + using TTopicTabletsPairs = TVector<TTopicInitInfo>; + +} // namespace NKikimr::NGRpcProxy diff --git a/ydb/services/persqueue_cluster_discovery/grpc_service.cpp b/ydb/services/persqueue_cluster_discovery/grpc_service.cpp index 0dc27831f3..086d162443 100644 --- a/ydb/services/persqueue_cluster_discovery/grpc_service.cpp +++ b/ydb/services/persqueue_cluster_discovery/grpc_service.cpp @@ -13,18 +13,18 @@ namespace NKikimr::NGRpcService { -TGRpcPQClusterDiscoveryService::TGRpcPQClusterDiscoveryService( - NActors::TActorSystem* system, TIntrusivePtr<NMonitoring::TDynamicCounters> counters, NActors::TActorId id, - const TMaybe<ui64>& requestsInflightLimit -) +TGRpcPQClusterDiscoveryService::TGRpcPQClusterDiscoveryService( + NActors::TActorSystem* system, TIntrusivePtr<NMonitoring::TDynamicCounters> counters, NActors::TActorId id, + const TMaybe<ui64>& requestsInflightLimit +) : ActorSystem_(system) , Counters_(counters) , GRpcRequestProxyId_(id) -{ - if (requestsInflightLimit.Defined()) { - Limiter = MakeHolder<NGrpc::TGlobalLimiter>(requestsInflightLimit.GetRef()); - } -} +{ + if (requestsInflightLimit.Defined()) { + Limiter = MakeHolder<NGrpc::TGlobalLimiter>(requestsInflightLimit.GetRef()); + } +} void TGRpcPQClusterDiscoveryService::InitService(grpc::ServerCompletionQueue *cq, NGrpc::TLoggerPtr logger) { CQ_ = cq; @@ -42,16 +42,16 @@ void TGRpcPQClusterDiscoveryService::SetGlobalLimiterHandle(NGrpc::TGlobalLimite } bool TGRpcPQClusterDiscoveryService::IncRequest() { - if (Limiter) { - return Limiter->Inc(); - } - return true; + if (Limiter) { + return Limiter->Inc(); + } + return true; } void TGRpcPQClusterDiscoveryService::DecRequest() { - if (Limiter) { - Limiter->Dec(); - } + if (Limiter) { + Limiter->Dec(); + } } void TGRpcPQClusterDiscoveryService::SetupIncomingRequests(NGrpc::TLoggerPtr logger) { diff --git a/ydb/services/persqueue_cluster_discovery/grpc_service.h b/ydb/services/persqueue_cluster_discovery/grpc_service.h index 6909f05bd6..7bc17ba2f5 100644 --- a/ydb/services/persqueue_cluster_discovery/grpc_service.h +++ b/ydb/services/persqueue_cluster_discovery/grpc_service.h @@ -9,8 +9,8 @@ namespace NKikimr::NGRpcService { class TGRpcPQClusterDiscoveryService : public NGrpc::TGrpcServiceBase<Ydb::PersQueue::V1::ClusterDiscoveryService> { public: - TGRpcPQClusterDiscoveryService(NActors::TActorSystem* system, TIntrusivePtr<NMonitoring::TDynamicCounters> counters, - NActors::TActorId id, const TMaybe<ui64>& requestsInflightLimit = Nothing()); + TGRpcPQClusterDiscoveryService(NActors::TActorSystem* system, TIntrusivePtr<NMonitoring::TDynamicCounters> counters, + NActors::TActorId id, const TMaybe<ui64>& requestsInflightLimit = Nothing()); void InitService(grpc::ServerCompletionQueue* cq, NGrpc::TLoggerPtr logger) override; void SetGlobalLimiterHandle(NGrpc::TGlobalLimiter* limiter) override; @@ -27,7 +27,7 @@ private: TIntrusivePtr<NMonitoring::TDynamicCounters> Counters_; NActors::TActorId GRpcRequestProxyId_; - THolder<NGrpc::TGlobalLimiter> Limiter; + THolder<NGrpc::TGlobalLimiter> Limiter; }; } diff --git a/ydb/services/persqueue_v1/grpc_pq_actor.h b/ydb/services/persqueue_v1/grpc_pq_actor.h index 87c0040147..65c8aca1db 100644 --- a/ydb/services/persqueue_v1/grpc_pq_actor.h +++ b/ydb/services/persqueue_v1/grpc_pq_actor.h @@ -76,13 +76,13 @@ static inline bool InternalErrorCode(PersQueue::ErrorCode::ErrorCode errorCode) } struct TPartitionId { - NPersQueue::TConverterPtr TopicConverter; + NPersQueue::TConverterPtr TopicConverter; ui64 Partition; ui64 AssignId; bool operator < (const TPartitionId& rhs) const { - return std::make_tuple(AssignId, Partition, TopicConverter->GetClientsideName()) < - std::make_tuple(rhs.AssignId, rhs.Partition, rhs.TopicConverter->GetClientsideName()); + return std::make_tuple(AssignId, Partition, TopicConverter->GetClientsideName()) < + std::make_tuple(rhs.AssignId, rhs.Partition, rhs.TopicConverter->GetClientsideName()); } }; @@ -141,11 +141,11 @@ struct TEvPQProxy { struct TEvAuthResultOk : public NActors::TEventLocal<TEvAuthResultOk, EvAuthResultOk> { - TEvAuthResultOk(const TTopicTabletsPairs&& topicAndTablets) + TEvAuthResultOk(const TTopicTabletsPairs&& topicAndTablets) : TopicAndTablets(std::move(topicAndTablets)) { } - TTopicTabletsPairs TopicAndTablets; + TTopicTabletsPairs TopicAndTablets; }; struct TEvSessionSetPreferredCluster : public NActors::TEventLocal<TEvSessionSetPreferredCluster, EvSessionSetPreferredCluster> { @@ -454,14 +454,14 @@ struct TEvPQProxy { /// WRITE ACTOR class TWriteSessionActor : public NActors::TActorBootstrapped<TWriteSessionActor> { using IContext = NGRpcServer::IGRpcStreamingContext<PersQueue::V1::StreamingWriteClientMessage, PersQueue::V1::StreamingWriteServerMessage>; -using TEvDescribeTopicsResponse = NMsgBusProxy::NPqMetaCacheV2::TEvPqNewMetaCache::TEvDescribeTopicsResponse; -using TEvDescribeTopicsRequest = NMsgBusProxy::NPqMetaCacheV2::TEvPqNewMetaCache::TEvDescribeTopicsRequest; +using TEvDescribeTopicsResponse = NMsgBusProxy::NPqMetaCacheV2::TEvPqNewMetaCache::TEvDescribeTopicsResponse; +using TEvDescribeTopicsRequest = NMsgBusProxy::NPqMetaCacheV2::TEvPqNewMetaCache::TEvDescribeTopicsRequest; public: - TWriteSessionActor(NKikimr::NGRpcService::TEvStreamPQWriteRequest* request, const ui64 cookie, - const NActors::TActorId& schemeCache, const NActors::TActorId& newSchemeCache, - TIntrusivePtr<NMonitoring::TDynamicCounters> counters, const TMaybe<TString> clientDC, - const NPersQueue::TTopicsListController& topicsController); + TWriteSessionActor(NKikimr::NGRpcService::TEvStreamPQWriteRequest* request, const ui64 cookie, + const NActors::TActorId& schemeCache, const NActors::TActorId& newSchemeCache, + TIntrusivePtr<NMonitoring::TDynamicCounters> counters, const TMaybe<TString> clientDC, + const NPersQueue::TTopicsListController& topicsController); ~TWriteSessionActor(); void Bootstrap(const NActors::TActorContext& ctx); @@ -486,7 +486,7 @@ private: HFunc(TEvPQProxy::TEvDone, Handle) HFunc(TEvPersQueue::TEvGetPartitionIdForWriteResponse, Handle) - HFunc(TEvDescribeTopicsResponse, Handle) + HFunc(TEvDescribeTopicsResponse, Handle) HFunc(NPQ::TEvPartitionWriter::TEvInitResult, Handle); HFunc(NPQ::TEvPartitionWriter::TEvWriteAccepted, Handle); @@ -522,7 +522,7 @@ private: void Handle(TEvPQProxy::TEvUpdateToken::TPtr& ev, const NActors::TActorContext& ctx); void Handle(TEvPQProxy::TEvDone::TPtr& ev, const NActors::TActorContext& ctx); void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx); - void Handle(TEvDescribeTopicsResponse::TPtr& ev, const TActorContext& ctx); + void Handle(TEvDescribeTopicsResponse::TPtr& ev, const TActorContext& ctx); void LogSession(const TActorContext& ctx); void DiscoverPartition(const NActors::TActorContext& ctx); @@ -530,7 +530,7 @@ private: void RequestNextPartition(const NActors::TActorContext& ctx); void ProceedPartition(const ui32 partition, const NActors::TActorContext& ctx); - THolder<NKqp::TEvKqp::TEvQueryRequest> MakeUpdateSourceIdMetadataRequest(const NActors::TActorContext& ctx); + THolder<NKqp::TEvKqp::TEvQueryRequest> MakeUpdateSourceIdMetadataRequest(const NActors::TActorContext& ctx); void InitCheckACL(const TActorContext& ctx); void Handle(NPQ::TEvPartitionWriter::TEvInitResult::TPtr& ev, const TActorContext& ctx); @@ -577,8 +577,8 @@ private: TString PeerName; ui64 Cookie; - NPersQueue::TTopicsListController TopicsController; - NPersQueue::TConverterPtr TopicConverter; + NPersQueue::TTopicsListController TopicsController; + NPersQueue::TConverterPtr TopicConverter; ui32 Partition; ui32 PreferedPartition; // 'SourceId' is called 'MessageGroupId' since gRPC data plane API v1 @@ -673,15 +673,15 @@ private: class TReadInitAndAuthActor : public NActors::TActorBootstrapped<TReadInitAndAuthActor> { -using TEvDescribeTopicsResponse = NMsgBusProxy::NPqMetaCacheV2::TEvPqNewMetaCache::TEvDescribeTopicsResponse; -using TEvDescribeTopicsRequest = NMsgBusProxy::NPqMetaCacheV2::TEvPqNewMetaCache::TEvDescribeTopicsRequest; - +using TEvDescribeTopicsResponse = NMsgBusProxy::NPqMetaCacheV2::TEvPqNewMetaCache::TEvDescribeTopicsResponse; +using TEvDescribeTopicsRequest = NMsgBusProxy::NPqMetaCacheV2::TEvPqNewMetaCache::TEvDescribeTopicsRequest; + public: - TReadInitAndAuthActor(const TActorContext& ctx, const TActorId& parentId, const TString& clientId, const ui64 cookie, - const TString& session, const NActors::TActorId& schemeCache, const NActors::TActorId& newSchemeCache, - TIntrusivePtr<NMonitoring::TDynamicCounters> counters, TIntrusivePtr<NACLib::TUserToken> token, - const NPersQueue::TTopicsToConverter& topics); - + TReadInitAndAuthActor(const TActorContext& ctx, const TActorId& parentId, const TString& clientId, const ui64 cookie, + const TString& session, const NActors::TActorId& schemeCache, const NActors::TActorId& newSchemeCache, + TIntrusivePtr<NMonitoring::TDynamicCounters> counters, TIntrusivePtr<NACLib::TUserToken> token, + const NPersQueue::TTopicsToConverter& topics); + ~TReadInitAndAuthActor(); void Bootstrap(const NActors::TActorContext& ctx); @@ -691,11 +691,11 @@ public: private: - STRICT_STFUNC(StateFunc, - HFunc(TEvDescribeTopicsResponse, HandleTopicsDescribeResponse) - HFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, HandleClientSchemeCacheResponse) - HFunc(NActors::TEvents::TEvPoisonPill, HandlePoison) - ); + STRICT_STFUNC(StateFunc, + HFunc(TEvDescribeTopicsResponse, HandleTopicsDescribeResponse) + HFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, HandleClientSchemeCacheResponse) + HFunc(NActors::TEvents::TEvPoisonPill, HandlePoison) + ); void HandlePoison(NActors::TEvents::TEvPoisonPill::TPtr&, const NActors::TActorContext& ctx) { Die(ctx); @@ -704,27 +704,27 @@ private: void CloseSession(const TString& errorReason, const PersQueue::ErrorCode::ErrorCode code, const TActorContext& ctx); - bool ProcessTopicSchemeCacheResponse(const NSchemeCache::TSchemeCacheNavigate::TEntry& entry, - THashMap<TString, TTopicHolder>::iterator topicsIter, const TActorContext& ctx); - void HandleClientSchemeCacheResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx); + bool ProcessTopicSchemeCacheResponse(const NSchemeCache::TSchemeCacheNavigate::TEntry& entry, + THashMap<TString, TTopicHolder>::iterator topicsIter, const TActorContext& ctx); + void HandleClientSchemeCacheResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx); void SendCacheNavigateRequest(const TActorContext& ctx, const TString& path); - void HandleTopicsDescribeResponse(TEvDescribeTopicsResponse::TPtr& ev, const NActors::TActorContext& ctx); + void HandleTopicsDescribeResponse(TEvDescribeTopicsResponse::TPtr& ev, const NActors::TActorContext& ctx); void FinishInitialization(const NActors::TActorContext& ctx); - bool CheckTopicACL(const NSchemeCache::TSchemeCacheNavigate::TEntry& entry, const TString& topic, const TActorContext& ctx); + bool CheckTopicACL(const NSchemeCache::TSchemeCacheNavigate::TEntry& entry, const TString& topic, const TActorContext& ctx); void CheckClientACL(const TActorContext& ctx); - bool CheckACLPermissionsForNavigate(const TIntrusivePtr<TSecurityObject>& secObject, - const TString& path, NACLib::EAccessRights rights, - const TString& errorTextWhenAccessDenied, - const TActorContext& ctx); + bool CheckACLPermissionsForNavigate(const TIntrusivePtr<TSecurityObject>& secObject, + const TString& path, NACLib::EAccessRights rights, + const TString& errorTextWhenAccessDenied, + const TActorContext& ctx); private: const TActorId ParentId; const ui64 Cookie; const TString Session; - const TActorId MetaCacheId; + const TActorId MetaCacheId; const TActorId NewSchemeCache; const TString ClientId; @@ -735,17 +735,17 @@ private: THashMap<TString, TTopicHolder> Topics; // topic -> info TIntrusivePtr<NMonitoring::TDynamicCounters> Counters; - bool DoCheckACL; + bool DoCheckACL; }; class TReadSessionActor : public TActorBootstrapped<TReadSessionActor> { using IContext = NGRpcServer::IGRpcStreamingContext<PersQueue::V1::MigrationStreamingReadClientMessage, PersQueue::V1::MigrationStreamingReadServerMessage>; public: - TReadSessionActor(NKikimr::NGRpcService::TEvStreamPQReadRequest* request, const ui64 cookie, - const NActors::TActorId& schemeCache, const NActors::TActorId& newSchemeCache, - TIntrusivePtr<NMonitoring::TDynamicCounters> counters, const TMaybe<TString> clientDC, - const NPersQueue::TTopicsListController& topicsHandler); + TReadSessionActor(NKikimr::NGRpcService::TEvStreamPQReadRequest* request, const ui64 cookie, + const NActors::TActorId& schemeCache, const NActors::TActorId& newSchemeCache, + TIntrusivePtr<NMonitoring::TDynamicCounters> counters, const TMaybe<TString> clientDC, + const NPersQueue::TTopicsListController& topicsHandler); ~TReadSessionActor(); void Bootstrap(const NActors::TActorContext& ctx); @@ -961,7 +961,7 @@ private: THashMap<ui64, TPartitionActorInfo> Partitions; //assignId -> info THashMap<TString, TTopicHolder> Topics; // topic -> info - THashMap<TString, NPersQueue::TConverterPtr> FullPathToConverter; // PrimaryFullPath -> Converter, for balancer replies matching + THashMap<TString, NPersQueue::TConverterPtr> FullPathToConverter; // PrimaryFullPath -> Converter, for balancer replies matching THashSet<TString> TopicsToResolve; THashMap<TString, TVector<ui32>> TopicGroups; THashMap<TString, ui64> ReadFromTimestamp; @@ -1063,19 +1063,19 @@ private: NKikimr::NPQ::TMultiCounter SLIBigLatency; NKikimr::NPQ::TMultiCounter SLIBigReadLatency; NKikimr::NPQ::TMultiCounter ReadsTotal; - - NPersQueue::TTopicsListController TopicsHandler; + + NPersQueue::TTopicsListController TopicsHandler; }; class TReadInfoActor : public TRpcOperationRequestActor<TReadInfoActor, TEvPQReadInfoRequest> { using TBase = TRpcOperationRequestActor<TReadInfoActor, TEvPQReadInfoRequest>; public: - TReadInfoActor( - NKikimr::NGRpcService::TEvPQReadInfoRequest* request, const NPersQueue::TTopicsListController& topicsHandler, - const NActors::TActorId& schemeCache, const NActors::TActorId& newSchemeCache, - TIntrusivePtr<NMonitoring::TDynamicCounters> counters - ); + TReadInfoActor( + NKikimr::NGRpcService::TEvPQReadInfoRequest* request, const NPersQueue::TTopicsListController& topicsHandler, + const NActors::TActorId& schemeCache, const NActors::TActorId& newSchemeCache, + TIntrusivePtr<NMonitoring::TDynamicCounters> counters + ); ~TReadInfoActor(); void Bootstrap(const NActors::TActorContext& ctx); @@ -1117,12 +1117,12 @@ private: TActorId AuthInitActor; - TTopicTabletsPairs TopicAndTablets; + TTopicTabletsPairs TopicAndTablets; TIntrusivePtr<NMonitoring::TDynamicCounters> Counters; TString ClientId; - NPersQueue::TTopicsListController TopicsHandler; + NPersQueue::TTopicsListController TopicsHandler; }; diff --git a/ydb/services/persqueue_v1/grpc_pq_read.cpp b/ydb/services/persqueue_v1/grpc_pq_read.cpp index 705cabf08f..6b7e72fdd0 100644 --- a/ydb/services/persqueue_v1/grpc_pq_read.cpp +++ b/ydb/services/persqueue_v1/grpc_pq_read.cpp @@ -40,18 +40,18 @@ TPQReadService::TPQReadService(const TActorId& schemeCache, const TActorId& newS void TPQReadService::Bootstrap(const TActorContext& ctx) { - HaveClusters = !AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen(); // ToDo[migration] - proper condition - if (HaveClusters) { + HaveClusters = !AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen(); // ToDo[migration] - proper condition + if (HaveClusters) { ctx.Send(NPQ::NClusterTracker::MakeClusterTrackerID(), new NPQ::NClusterTracker::TEvClusterTracker::TEvSubscribe); } ctx.Send(NNetClassifier::MakeNetClassifierID(), new NNetClassifier::TEvNetClassifier::TEvSubscribe); - TopicConverterFactory = std::make_shared<NPersQueue::TTopicNamesConverterFactory>( - AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen(), AppData(ctx)->PQConfig.GetRoot() - ); - TopicsHandler = std::make_unique<NPersQueue::TTopicsListController>( - TopicConverterFactory, HaveClusters, Clusters, LocalCluster - ); + TopicConverterFactory = std::make_shared<NPersQueue::TTopicNamesConverterFactory>( + AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen(), AppData(ctx)->PQConfig.GetRoot() + ); + TopicsHandler = std::make_unique<NPersQueue::TTopicsListController>( + TopicConverterFactory, HaveClusters, Clusters, LocalCluster + ); Become(&TThis::StateFunc); } @@ -89,7 +89,7 @@ void TPQReadService::Handle(NPQ::NClusterTracker::TEvClusterTracker::TEvClusters for (size_t i = 0; i < clusters.size(); ++i) { Clusters[i] = clusters[i].Name; } - TopicsHandler->UpdateClusters(Clusters, LocalCluster); + TopicsHandler->UpdateClusters(Clusters, LocalCluster); } @@ -122,8 +122,8 @@ void TPQReadService::Handle(NKikimr::NGRpcService::TEvStreamPQReadRequest::TPtr& ev->Get()->GetStreamCtx()->WriteAndFinish(FillReadResponse("proxy overloaded", PersQueue::ErrorCode::OVERLOAD), grpc::Status::OK); //CANCELLED return; } - if (HaveClusters - && (Clusters.empty() || LocalCluster.empty())) { + if (HaveClusters + && (Clusters.empty() || LocalCluster.empty())) { LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, "new grpc connection failed - cluster is not known yet"); ev->Get()->GetStreamCtx()->Attach(ctx.SelfID); @@ -137,11 +137,11 @@ void TPQReadService::Handle(NKikimr::NGRpcService::TEvStreamPQReadRequest::TPtr& auto ip = ev->Get()->GetStreamCtx()->GetPeerName(); - TActorId worker = ctx.Register(new TReadSessionActor( - ev->Release().Release(), cookie, SchemeCache, NewSchemeCache, Counters, - DatacenterClassifier ? DatacenterClassifier->ClassifyAddress(NAddressClassifier::ExtractAddress(ip)) : "unknown", - *TopicsHandler - )); + TActorId worker = ctx.Register(new TReadSessionActor( + ev->Release().Release(), cookie, SchemeCache, NewSchemeCache, Counters, + DatacenterClassifier ? DatacenterClassifier->ClassifyAddress(NAddressClassifier::ExtractAddress(ip)) : "unknown", + *TopicsHandler + )); Sessions[cookie] = worker; } @@ -158,8 +158,8 @@ void TPQReadService::Handle(NKikimr::NGRpcService::TEvPQReadInfoRequest::TPtr& e ev->Get()->SendResult(ConvertPersQueueInternalCodeToStatus(PersQueue::ErrorCode::INITIALIZING), FillInfoResponse("cluster initializing", PersQueue::ErrorCode::INITIALIZING)); //CANCELLED return; } else { - //ctx.Register(new TReadInfoActor(ev->Release().Release(), Clusters, LocalCluster, SchemeCache, NewSchemeCache, Counters)); - ctx.Register(new TReadInfoActor(ev->Release().Release(), *TopicsHandler, SchemeCache, NewSchemeCache, Counters)); + //ctx.Register(new TReadInfoActor(ev->Release().Release(), Clusters, LocalCluster, SchemeCache, NewSchemeCache, Counters)); + ctx.Register(new TReadInfoActor(ev->Release().Release(), *TopicsHandler, SchemeCache, NewSchemeCache, Counters)); } } diff --git a/ydb/services/persqueue_v1/grpc_pq_read.h b/ydb/services/persqueue_v1/grpc_pq_read.h index 2c685fbb18..558063ed13 100644 --- a/ydb/services/persqueue_v1/grpc_pq_read.h +++ b/ydb/services/persqueue_v1/grpc_pq_read.h @@ -74,9 +74,9 @@ private: TString LocalCluster; NAddressClassifier::TLabeledAddressClassifier::TConstPtr DatacenterClassifier; // Detects client's datacenter by IP. May be null - std::shared_ptr<NPersQueue::TTopicNamesConverterFactory> TopicConverterFactory; - std::unique_ptr<NPersQueue::TTopicsListController> TopicsHandler; - bool HaveClusters; + std::shared_ptr<NPersQueue::TTopicNamesConverterFactory> TopicConverterFactory; + std::unique_ptr<NPersQueue::TTopicsListController> TopicsHandler; + bool HaveClusters; }; diff --git a/ydb/services/persqueue_v1/grpc_pq_read_actor.cpp b/ydb/services/persqueue_v1/grpc_pq_read_actor.cpp index c7e2c11407..654edfcfcd 100644 --- a/ydb/services/persqueue_v1/grpc_pq_read_actor.cpp +++ b/ydb/services/persqueue_v1/grpc_pq_read_actor.cpp @@ -71,7 +71,7 @@ static const double LAG_GROW_MULTIPLIER = 1.2; //assume that 20% more data arriv IOutputStream& operator <<(IOutputStream& out, const TPartitionId& partId) { - out << "TopicId: " << partId.TopicConverter->GetClientsideName() << ":" << partId.Partition << "(assignId:" << partId.AssignId << ")"; + out << "TopicId: " << partId.TopicConverter->GetClientsideName() << ":" << partId.Partition << "(assignId:" << partId.AssignId << ")"; return out; } @@ -142,8 +142,8 @@ private: void Handle(TEvPQProxy::TEvReleasePartition::TPtr& ev, const NActors::TActorContext& ctx); void Handle(TEvPQProxy::TEvLockPartition::TPtr& ev, const NActors::TActorContext& ctx); - void Handle(TEvPQProxy::TEvGetStatus:: - TPtr& ev, const NActors::TActorContext& ctx); + void Handle(TEvPQProxy::TEvGetStatus:: + TPtr& ev, const NActors::TActorContext& ctx); void Handle(TEvPQProxy::TEvDeadlineExceeded::TPtr& ev, const NActors::TActorContext& ctx); @@ -236,11 +236,11 @@ private: }; -TReadSessionActor::TReadSessionActor( - NKikimr::NGRpcService::TEvStreamPQReadRequest* request, const ui64 cookie, const TActorId& schemeCache, const TActorId& newSchemeCache, - TIntrusivePtr<NMonitoring::TDynamicCounters> counters, const TMaybe<TString> clientDC, - const NPersQueue::TTopicsListController& topicsHandler -) +TReadSessionActor::TReadSessionActor( + NKikimr::NGRpcService::TEvStreamPQReadRequest* request, const ui64 cookie, const TActorId& schemeCache, const TActorId& newSchemeCache, + TIntrusivePtr<NMonitoring::TDynamicCounters> counters, const TMaybe<TString> clientDC, + const NPersQueue::TTopicsListController& topicsHandler +) : Request(request) , ClientDC(clientDC ? *clientDC : "other") , StartTimestamp(TInstant::Now()) @@ -267,7 +267,7 @@ TReadSessionActor::TReadSessionActor( , BytesInflight_(0) , RequestedBytes(0) , ReadsInfly(0) - , TopicsHandler(topicsHandler) + , TopicsHandler(topicsHandler) { Y_ASSERT(Request); ++(*GetServiceCounters(Counters, "pqproxy|readSession")->GetCounter("SessionsCreatedTotal", true)); @@ -285,9 +285,9 @@ void TReadSessionActor::Bootstrap(const TActorContext& ctx) { Die(ctx); return; } - //auto& pqConfig = AppData(ctx)->PQConfig; - //bool noDcMode = !pqConfig.GetTopicsAreFirstClassCitizen(); // ToDo[migration] - add multicluster mode - //ConverterFactory = MakeHolder<NPersQueue::TTopicNamesConverterFactory>(noDcMode, pqConfig.GetRoot()); + //auto& pqConfig = AppData(ctx)->PQConfig; + //bool noDcMode = !pqConfig.GetTopicsAreFirstClassCitizen(); // ToDo[migration] - add multicluster mode + //ConverterFactory = MakeHolder<NPersQueue::TTopicNamesConverterFactory>(noDcMode, pqConfig.GetRoot()); StartTime = ctx.Now(); Become(&TThis::StateFunc); @@ -317,29 +317,29 @@ void TReadSessionActor::Handle(IContext::TEvReadFinished::TPtr& ev, const TActor ctx.Send(ctx.SelfID, new TEvPQProxy::TEvDone()); return; } - auto converterFactory = TopicsHandler.GetConverterFactory(); - auto MakePartitionId = [&](auto& request) { - auto converter = converterFactory->MakeTopicNameConverter( - request.topic().path(), request.cluster(), Request->GetDatabaseName().GetOrElse(TString()) - ); - if (!converter->IsValid()) { - CloseSession( - TStringBuilder() << "Topic " << request.topic().path() << " not recognized: " << converter->GetReason(), - PersQueue::ErrorCode::BAD_REQUEST, ctx - ); - } - const ui32 partition = request.partition(); - const ui64 assignId = request.assign_id(); - return TPartitionId{converter, partition, assignId}; - }; + auto converterFactory = TopicsHandler.GetConverterFactory(); + auto MakePartitionId = [&](auto& request) { + auto converter = converterFactory->MakeTopicNameConverter( + request.topic().path(), request.cluster(), Request->GetDatabaseName().GetOrElse(TString()) + ); + if (!converter->IsValid()) { + CloseSession( + TStringBuilder() << "Topic " << request.topic().path() << " not recognized: " << converter->GetReason(), + PersQueue::ErrorCode::BAD_REQUEST, ctx + ); + } + const ui32 partition = request.partition(); + const ui64 assignId = request.assign_id(); + return TPartitionId{converter, partition, assignId}; + }; switch (request.request_case()) { case MigrationStreamingReadClientMessage::kInitRequest: { ctx.Send(ctx.SelfID, new TEvPQProxy::TEvReadInit(request, Request->GetStreamCtx()->GetPeerName())); break; } case MigrationStreamingReadClientMessage::kStatus: { - //const auto& req = request.status(); - ctx.Send(ctx.SelfID, new TEvPQProxy::TEvGetStatus(MakePartitionId(request.status()))); + //const auto& req = request.status(); + ctx.Send(ctx.SelfID, new TEvPQProxy::TEvGetStatus(MakePartitionId(request.status()))); if (!Request->GetStreamCtx()->Read()) { LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " grpc read failed at start"); Die(ctx); @@ -353,8 +353,8 @@ void TReadSessionActor::Handle(IContext::TEvReadFinished::TPtr& ev, const TActor break; } case MigrationStreamingReadClientMessage::kReleased: { - //const auto& req = request.released(); - ctx.Send(ctx.SelfID, new TEvPQProxy::TEvReleased(MakePartitionId(request.released()))); + //const auto& req = request.released(); + ctx.Send(ctx.SelfID, new TEvPQProxy::TEvReleased(MakePartitionId(request.released()))); if (!Request->GetStreamCtx()->Read()) { LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " grpc read failed at start"); Die(ctx); @@ -370,7 +370,7 @@ void TReadSessionActor::Handle(IContext::TEvReadFinished::TPtr& ev, const TActor const ui64 commitOffset = req.commit_offset(); const bool verifyReadOffset = req.verify_read_offset(); - ctx.Send(ctx.SelfID, new TEvPQProxy::TEvStartRead(MakePartitionId(request.start_read()), readOffset, commitOffset, verifyReadOffset)); + ctx.Send(ctx.SelfID, new TEvPQProxy::TEvStartRead(MakePartitionId(request.start_read()), readOffset, commitOffset, verifyReadOffset)); if (!Request->GetStreamCtx()->Read()) { LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " grpc read failed at start"); Die(ctx); @@ -449,7 +449,7 @@ void TReadSessionActor::Die(const TActorContext& ctx) { ctx.Send(p.second.Actor, new TEvents::TEvPoisonPill()); if (!p.second.Released) { - auto it = TopicCounters.find(p.second.Partition.TopicConverter->GetClientsideName()); + auto it = TopicCounters.find(p.second.Partition.TopicConverter->GetClientsideName()); Y_VERIFY(it != TopicCounters.end()); it->second.PartitionsInfly.Dec(); it->second.PartitionsReleased.Inc(); @@ -600,8 +600,8 @@ void TReadSessionActor::DropPartition(THashMap<ui64, TPartitionActorInfo>::itera bool res = ActualPartitionActors.erase(it->second.Actor); Y_VERIFY(res); - if (--NumPartitionsFromTopic[it->second.Partition.TopicConverter->GetClientsideName()] == 0) { - bool res = TopicCounters.erase(it->second.Partition.TopicConverter->GetClientsideName()); + if (--NumPartitionsFromTopic[it->second.Partition.TopicConverter->GetClientsideName()] == 0) { + bool res = TopicCounters.erase(it->second.Partition.TopicConverter->GetClientsideName()); Y_VERIFY(res); } @@ -662,7 +662,7 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvReadSessionStatus::TPtr& ev, const THolder<TEvPQProxy::TEvReadSessionStatusResponse> result(new TEvPQProxy::TEvReadSessionStatusResponse()); for (auto& p : Partitions) { auto part = result->Record.AddPartition(); - part->SetTopic(p.second.Partition.TopicConverter->GetPrimaryPath()); + part->SetTopic(p.second.Partition.TopicConverter->GetPrimaryPath()); part->SetPartition(p.second.Partition.Partition); part->SetAssignId(p.second.Partition.AssignId); for (auto& c : p.second.NextCommits) { @@ -730,11 +730,11 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvReadInit::TPtr& ev, const TActorCo ReadOnlyLocal = init.read_only_original(); for (const auto& topic : init.topics_read_settings()) { - auto converter = TopicsHandler.GetConverterFactory()->MakeTopicNameConverter( - topic.topic(), TString(), Request->GetDatabaseName().GetOrElse(TString()) - ); - const auto topicName = converter->GetModernName(); - if (topicName.empty()) { + auto converter = TopicsHandler.GetConverterFactory()->MakeTopicNameConverter( + topic.topic(), TString(), Request->GetDatabaseName().GetOrElse(TString()) + ); + const auto topicName = converter->GetModernName(); + if (topicName.empty()) { CloseSession("empty topic in init request", PersQueue::ErrorCode::BAD_REQUEST, ctx); return; } @@ -742,7 +742,7 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvReadInit::TPtr& ev, const TActorCo CloseSession("start_from_written_at_ms must be nonnegative number", PersQueue::ErrorCode::BAD_REQUEST, ctx); return; } - TopicsToResolve.insert(topicName); + TopicsToResolve.insert(topicName); for (i64 pg : topic.partition_group_ids()) { if (pg < 0) { CloseSession("partition group id must be nonnegative number", PersQueue::ErrorCode::BAD_REQUEST, ctx); @@ -752,9 +752,9 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvReadInit::TPtr& ev, const TActorCo CloseSession(TStringBuilder() << "partition group id " << pg << " is too big for partition group id", PersQueue::ErrorCode::BAD_REQUEST, ctx); return; } - TopicGroups[topicName].push_back(static_cast<ui32>(pg)); + TopicGroups[topicName].push_back(static_cast<ui32>(pg)); } - ReadFromTimestamp[topicName] = topic.start_from_written_at_ms(); + ReadFromTimestamp[topicName] = topic.start_from_written_at_ms(); } LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " init: " << event->Request << " from " << PeerName); @@ -770,10 +770,10 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvReadInit::TPtr& ev, const TActorCo Auth = *(Request->GetYdbToken()); Token = new NACLib::TUserToken(Request->GetInternalToken()); } - AuthInitActor = ctx.Register(new TReadInitAndAuthActor( - ctx, ctx.SelfID, ClientId, Cookie, Session, SchemeCache, NewSchemeCache, Counters, Token, - TopicsHandler.GetReadTopicsList(TopicsToResolve, ReadOnlyLocal, Request->GetDatabaseName().GetOrElse(TString())) - )); + AuthInitActor = ctx.Register(new TReadInitAndAuthActor( + ctx, ctx.SelfID, ClientId, Cookie, Session, SchemeCache, NewSchemeCache, Counters, Token, + TopicsHandler.GetReadTopicsList(TopicsToResolve, ReadOnlyLocal, Request->GetDatabaseName().GetOrElse(TString())) + )); auto subGroup = GetServiceCounters(Counters, "pqproxy|SLI"); @@ -811,7 +811,7 @@ void TReadSessionActor::RegisterSessions(const TActorContext& ctx) { for (auto& t : Topics) { auto& topic = t.first; RegisterSession(t.second.PipeClient, topic, t.second.Groups, ctx); - NumPartitionsFromTopic[t.second.TopicNameConverter->GetClientsideName()] = 0; + NumPartitionsFromTopic[t.second.TopicNameConverter->GetClientsideName()] = 0; } } @@ -887,11 +887,11 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvAuthResultOk::TPtr& ev, const TAct LastACLCheckTimestamp = ctx.Now(); - LOG_INFO_S( - ctx, - NKikimrServices::PQ_READ_PROXY, - PQ_LOG_PREFIX << " auth ok, got " << ev->Get()->TopicAndTablets.size() << " topics, init done " << InitDone - ); + LOG_INFO_S( + ctx, + NKikimrServices::PQ_READ_PROXY, + PQ_LOG_PREFIX << " auth ok, got " << ev->Get()->TopicAndTablets.size() << " topics, init done " << InitDone + ); AuthInitActor = TActorId(); @@ -935,11 +935,11 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvAuthResultOk::TPtr& ev, const TAct Y_VERIFY(!BalancersInitStarted); BalancersInitStarted = true; - for (auto& t : ev->Get()->TopicAndTablets) { // ToDo - return something from Init and Auth Actor (Full Path - ?) - auto& topicHolder = Topics[t.TopicNameConverter->GetClientsideName()]; - topicHolder.TabletID = t.TabletID; - topicHolder.TopicNameConverter = t.TopicNameConverter; - FullPathToConverter[t.TopicNameConverter->GetPrimaryPath()] = t.TopicNameConverter; + for (auto& t : ev->Get()->TopicAndTablets) { // ToDo - return something from Init and Auth Actor (Full Path - ?) + auto& topicHolder = Topics[t.TopicNameConverter->GetClientsideName()]; + topicHolder.TabletID = t.TabletID; + topicHolder.TopicNameConverter = t.TopicNameConverter; + FullPathToConverter[t.TopicNameConverter->GetPrimaryPath()] = t.TopicNameConverter; } for (auto& t : Topics) { @@ -949,8 +949,8 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvAuthResultOk::TPtr& ev, const TAct clientConfig.RetryPolicy = RetryPolicyForPipes; t.second.PipeClient = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, t.second.TabletID, clientConfig)); - - auto it = TopicGroups.find(t.second.TopicNameConverter->GetModernName()); + + auto it = TopicGroups.find(t.second.TopicNameConverter->GetModernName()); if (it != TopicGroups.end()) { t.second.Groups = it->second; } @@ -961,12 +961,12 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvAuthResultOk::TPtr& ev, const TAct ctx.Schedule(CHECK_ACL_DELAY, new TEvents::TEvWakeup()); } else { for (auto& t : ev->Get()->TopicAndTablets) { - if (Topics.find(t.TopicNameConverter->GetClientsideName()) == Topics.end()) { - CloseSession( - TStringBuilder() << "list of topics changed - new topic '" - << t.TopicNameConverter->GetClientsideName() << "' found", - PersQueue::ErrorCode::BAD_REQUEST, ctx - ); + if (Topics.find(t.TopicNameConverter->GetClientsideName()) == Topics.end()) { + CloseSession( + TStringBuilder() << "list of topics changed - new topic '" + << t.TopicNameConverter->GetClientsideName() << "' found", + PersQueue::ErrorCode::BAD_REQUEST, ctx + ); return; } } @@ -981,23 +981,23 @@ void TReadSessionActor::Handle(TEvPersQueue::TEvLockPartition::TPtr& ev, const T Y_VERIFY(record.GetClientId() == ClientId); TActorId pipe = ActorIdFromProto(record.GetPipeClient()); - auto converterIter = FullPathToConverter.find(NPersQueue::NormalizeFullPath(record.GetPath())); - - if (converterIter.IsEnd()) { - LOG_DEBUG_S( - ctx, NKikimrServices::PQ_READ_PROXY, - PQ_LOG_PREFIX << " ignored ev lock for path = " << record.GetPath() << ", path not recognized" - ); - return; - } - const auto& topic = converterIter->second->GetClientsideName(); - auto jt = Topics.find(topic); // ToDo - Check + auto converterIter = FullPathToConverter.find(NPersQueue::NormalizeFullPath(record.GetPath())); + + if (converterIter.IsEnd()) { + LOG_DEBUG_S( + ctx, NKikimrServices::PQ_READ_PROXY, + PQ_LOG_PREFIX << " ignored ev lock for path = " << record.GetPath() << ", path not recognized" + ); + return; + } + const auto& topic = converterIter->second->GetClientsideName(); + auto jt = Topics.find(topic); // ToDo - Check if (jt == Topics.end() || pipe != jt->second.PipeClient) { //this is message from old version of pipe - LOG_ALERT_S( - ctx, NKikimrServices::PQ_READ_PROXY, - PQ_LOG_PREFIX << " ignored ev lock for topic = " << topic - << " path recognized, but topic is unknown, this is unexpected" - ); + LOG_ALERT_S( + ctx, NKikimrServices::PQ_READ_PROXY, + PQ_LOG_PREFIX << " ignored ev lock for topic = " << topic + << " path recognized, but topic is unknown, this is unexpected" + ); return; } @@ -1014,7 +1014,7 @@ void TReadSessionActor::Handle(TEvPersQueue::TEvLockPartition::TPtr& ev, const T ui64 assignId = NextAssignId++; BalancerGeneration[assignId] = {record.GetGeneration(), record.GetStep()}; - TPartitionId partitionId{converterIter->second, record.GetPartition(), assignId}; + TPartitionId partitionId{converterIter->second, record.GetPartition(), assignId}; IActor* partitionActor = new TPartitionActor(ctx.SelfID, ClientId, ClientPath, Cookie, Session, partitionId, record.GetGeneration(), record.GetStep(), record.GetTabletId(), it->second, CommitsDisabled, ClientDC); @@ -1054,8 +1054,8 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvPartitionStatus::TPtr& ev, const T MigrationStreamingReadServerMessage result; result.set_status(Ydb::StatusIds::SUCCESS); - result.mutable_assigned()->mutable_topic()->set_path(ev->Get()->Partition.TopicConverter->GetModernName()); - result.mutable_assigned()->set_cluster(ev->Get()->Partition.TopicConverter->GetCluster()); + result.mutable_assigned()->mutable_topic()->set_path(ev->Get()->Partition.TopicConverter->GetModernName()); + result.mutable_assigned()->set_cluster(ev->Get()->Partition.TopicConverter->GetCluster()); result.mutable_assigned()->set_partition(ev->Get()->Partition.Partition); result.mutable_assigned()->set_assign_id(it->first); @@ -1083,8 +1083,8 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvPartitionStatus::TPtr& ev, const T MigrationStreamingReadServerMessage result; result.set_status(Ydb::StatusIds::SUCCESS); - result.mutable_partition_status()->mutable_topic()->set_path(ev->Get()->Partition.TopicConverter->GetModernName()); - result.mutable_partition_status()->set_cluster(ev->Get()->Partition.TopicConverter->GetCluster()); + result.mutable_partition_status()->mutable_topic()->set_path(ev->Get()->Partition.TopicConverter->GetModernName()); + result.mutable_partition_status()->set_cluster(ev->Get()->Partition.TopicConverter->GetCluster()); result.mutable_partition_status()->set_partition(ev->Get()->Partition.Partition); result.mutable_partition_status()->set_assign_id(it->first); @@ -1118,8 +1118,8 @@ void TReadSessionActor::SendReleaseSignalToClient(const THashMap<ui64, TPartitio MigrationStreamingReadServerMessage result; result.set_status(Ydb::StatusIds::SUCCESS); - result.mutable_release()->mutable_topic()->set_path(it->second.Partition.TopicConverter->GetModernName()); - result.mutable_release()->set_cluster(it->second.Partition.TopicConverter->GetCluster()); + result.mutable_release()->mutable_topic()->set_path(it->second.Partition.TopicConverter->GetModernName()); + result.mutable_release()->set_cluster(it->second.Partition.TopicConverter->GetCluster()); result.mutable_release()->set_partition(it->second.Partition.Partition); result.mutable_release()->set_assign_id(it->second.Partition.AssignId); result.mutable_release()->set_forceful_release(kill); @@ -1147,14 +1147,14 @@ void TReadSessionActor::Handle(TEvPersQueue::TEvReleasePartition::TPtr& ev, cons auto& record = ev->Get()->Record; Y_VERIFY(record.GetSession() == Session); Y_VERIFY(record.GetClientId() == ClientId); - TString topicPath = NPersQueue::NormalizeFullPath(record.GetPath()); + TString topicPath = NPersQueue::NormalizeFullPath(record.GetPath()); ui32 group = record.HasGroup() ? record.GetGroup() : 0; - auto pathIter = FullPathToConverter.find(topicPath); - Y_VERIFY(!pathIter.IsEnd()); - auto it = Topics.find(pathIter->second->GetClientsideName()); - Y_VERIFY(!it.IsEnd()); - auto& converter = it->second.TopicNameConverter; + auto pathIter = FullPathToConverter.find(topicPath); + Y_VERIFY(!pathIter.IsEnd()); + auto it = Topics.find(pathIter->second->GetClientsideName()); + Y_VERIFY(!it.IsEnd()); + auto& converter = it->second.TopicNameConverter; TActorId pipe = ActorIdFromProto(record.GetPipeClient()); @@ -1169,10 +1169,10 @@ void TReadSessionActor::Handle(TEvPersQueue::TEvReleasePartition::TPtr& ev, cons auto jt = Partitions.begin(); ui32 i = 0; for (auto it = Partitions.begin(); it != Partitions.end(); ++it) { - if (it->second.Partition.TopicConverter->GetPrimaryPath() == converter->GetPrimaryPath() - && !it->second.Releasing - && (group == 0 || it->second.Partition.Partition + 1 == group) - ) { + if (it->second.Partition.TopicConverter->GetPrimaryPath() == converter->GetPrimaryPath() + && !it->second.Releasing + && (group == 0 || it->second.Partition.Partition + 1 == group) + ) { ++i; if (rand() % i == 0) { //will lead to 1/n probability for each of n partitions actorId = it->second.Actor; @@ -1183,7 +1183,7 @@ void TReadSessionActor::Handle(TEvPersQueue::TEvReleasePartition::TPtr& ev, cons Y_VERIFY(actorId); { - auto it = TopicCounters.find(converter->GetClientsideName()); + auto it = TopicCounters.find(converter->GetClientsideName()); Y_VERIFY(it != TopicCounters.end()); it->second.PartitionsToBeReleased.Inc(); } @@ -1218,14 +1218,14 @@ void TReadSessionActor::InformBalancerAboutRelease(const THashMap<ui64, TPartiti request.Reset(new TEvPersQueue::TEvPartitionReleased); auto& req = request->Record; - const auto& converter = it->second.Partition.TopicConverter; - auto jt = Topics.find(converter->GetClientsideName()); + const auto& converter = it->second.Partition.TopicConverter; + auto jt = Topics.find(converter->GetClientsideName()); Y_VERIFY(jt != Topics.end()); req.SetSession(Session); ActorIdToProto(jt->second.PipeClient, req.MutablePipeClient()); req.SetClientId(ClientId); - req.SetTopic(converter->GetPrimaryPath()); + req.SetTopic(converter->GetPrimaryPath()); req.SetPartition(it->second.Partition.Partition); LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " released: " << it->second.Partition); @@ -1300,7 +1300,7 @@ void TReadSessionActor::ReleasePartition(const THashMap<ui64, TPartitionActorInf bool couldBeReads, const TActorContext& ctx) { { - auto jt = TopicCounters.find(it->second.Partition.TopicConverter->GetClientsideName()); + auto jt = TopicCounters.find(it->second.Partition.TopicConverter->GetClientsideName()); Y_VERIFY(jt != TopicCounters.end()); jt->second.PartitionsReleased.Inc(); jt->second.PartitionsInfly.Dec(); @@ -1338,7 +1338,7 @@ bool TReadSessionActor::ProcessBalancerDead(const ui64 tablet, const TActorConte //Drop all partitions from this topic for (auto it = Partitions.begin(); it != Partitions.end();) { - if (it->second.Partition.TopicConverter->GetClientsideName() == t.first) { //partition from this topic + if (it->second.Partition.TopicConverter->GetClientsideName() == t.first) { //partition from this topic // kill actor auto jt = it; ++it; @@ -1580,7 +1580,7 @@ void TReadSessionActor::ProcessReads(const TActorContext& ctx) { size -= csize; Y_VERIFY(csize < Max<i32>()); - auto jt = ReadFromTimestamp.find(it->second.Partition.TopicConverter->GetModernName()); + auto jt = ReadFromTimestamp.find(it->second.Partition.TopicConverter->GetModernName()); Y_VERIFY(jt != ReadFromTimestamp.end()); ui64 readTimestampMs = Max(ReadTimestampMs, jt->second); @@ -1659,10 +1659,10 @@ void TReadSessionActor::HandleWakeup(const TActorContext& ctx) { Y_VERIFY(!AuthInitActor); LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " checking auth because of timeout"); - AuthInitActor = ctx.Register(new TReadInitAndAuthActor( - ctx, ctx.SelfID, ClientId, Cookie, Session, SchemeCache, NewSchemeCache, Counters, Token, - TopicsHandler.GetReadTopicsList(TopicsToResolve, ReadOnlyLocal, Request->GetDatabaseName().GetOrElse(TString())) - )); + AuthInitActor = ctx.Register(new TReadInitAndAuthActor( + ctx, ctx.SelfID, ClientId, Cookie, Session, SchemeCache, NewSchemeCache, Counters, Token, + TopicsHandler.GetReadTopicsList(TopicsToResolve, ReadOnlyLocal, Request->GetDatabaseName().GetOrElse(TString())) + )); } } @@ -1811,7 +1811,7 @@ void TPartitionActor::CheckRelease(const TActorContext& ctx) { void TPartitionActor::SendCommit(const ui64 readId, const ui64 offset, const TActorContext& ctx) { NKikimrClient::TPersQueueRequest request; - request.MutablePartitionRequest()->SetTopic(Partition.TopicConverter->GetPrimaryPath()); + request.MutablePartitionRequest()->SetTopic(Partition.TopicConverter->GetPrimaryPath()); request.MutablePartitionRequest()->SetPartition(Partition.Partition); request.MutablePartitionRequest()->SetCookie(readId); @@ -1903,10 +1903,10 @@ void TPartitionActor::Handle(const TEvPQProxy::TEvRestartPipe::TPtr&, const TAct bool FillBatchedData(MigrationStreamingReadServerMessage::DataBatch * data, const NKikimrClient::TCmdReadResult& res, const TPartitionId& Partition, ui64 ReadIdToResponse, ui64& ReadOffset, ui64& WTime, ui64 EndOffset, const TActorContext& ctx) { auto* partitionData = data->add_partition_data(); - partitionData->mutable_topic()->set_path(Partition.TopicConverter->GetModernName()); - partitionData->set_cluster(Partition.TopicConverter->GetCluster()); + partitionData->mutable_topic()->set_path(Partition.TopicConverter->GetModernName()); + partitionData->set_cluster(Partition.TopicConverter->GetCluster()); partitionData->set_partition(Partition.Partition); - partitionData->set_deprecated_topic(Partition.TopicConverter->GetFullLegacyName()); + partitionData->set_deprecated_topic(Partition.TopicConverter->GetFullLegacyName()); partitionData->mutable_cookie()->set_assign_id(Partition.AssignId); partitionData->mutable_cookie()->set_partition_cookie(ReadIdToResponse); @@ -2318,7 +2318,7 @@ void TPartitionActor::InitLockPartition(const TActorContext& ctx) { NKikimrClient::TPersQueueRequest request; - request.MutablePartitionRequest()->SetTopic(Partition.TopicConverter->GetPrimaryPath()); + request.MutablePartitionRequest()->SetTopic(Partition.TopicConverter->GetPrimaryPath()); request.MutablePartitionRequest()->SetPartition(Partition.Partition); request.MutablePartitionRequest()->SetCookie(INIT_COOKIE); @@ -2438,7 +2438,7 @@ void TPartitionActor::Handle(TEvPQProxy::TEvRead::TPtr& ev, const TActorContext& NKikimrClient::TPersQueueRequest request; - request.MutablePartitionRequest()->SetTopic(Partition.TopicConverter->GetPrimaryPath()); + request.MutablePartitionRequest()->SetTopic(Partition.TopicConverter->GetPrimaryPath()); request.MutablePartitionRequest()->SetPartition(Partition.Partition); request.MutablePartitionRequest()->SetCookie((ui64)ReadOffset); @@ -2565,16 +2565,16 @@ void TPartitionActor::HandleWakeup(const TActorContext& ctx) { } ///////////////// AuthAndInitActor -TReadInitAndAuthActor::TReadInitAndAuthActor( - const TActorContext& ctx, const TActorId& parentId, const TString& clientId, const ui64 cookie, - const TString& session, const NActors::TActorId& metaCache, const NActors::TActorId& newSchemeCache, - TIntrusivePtr<NMonitoring::TDynamicCounters> counters, TIntrusivePtr<NACLib::TUserToken> token, - const NPersQueue::TTopicsToConverter& topics -) +TReadInitAndAuthActor::TReadInitAndAuthActor( + const TActorContext& ctx, const TActorId& parentId, const TString& clientId, const ui64 cookie, + const TString& session, const NActors::TActorId& metaCache, const NActors::TActorId& newSchemeCache, + TIntrusivePtr<NMonitoring::TDynamicCounters> counters, TIntrusivePtr<NACLib::TUserToken> token, + const NPersQueue::TTopicsToConverter& topics +) : ParentId(parentId) , Cookie(cookie) , Session(session) - , MetaCacheId(metaCache) + , MetaCacheId(metaCache) , NewSchemeCache(newSchemeCache) , ClientId(clientId) , ClientPath(NPersQueue::ConvertOldConsumerName(ClientId, ctx)) @@ -2582,7 +2582,7 @@ TReadInitAndAuthActor::TReadInitAndAuthActor( , Counters(counters) { for (const auto& t : topics) { - Topics[t.first].TopicNameConverter = t.second; + Topics[t.first].TopicNameConverter = t.second; } } @@ -2592,13 +2592,13 @@ TReadInitAndAuthActor::~TReadInitAndAuthActor() = default; void TReadInitAndAuthActor::Bootstrap(const TActorContext &ctx) { LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " auth for : " << ClientId); - Become(&TThis::StateFunc); - TVector<TString> topicNames; - for (const auto& topic : Topics) { - topicNames.emplace_back(topic.second.TopicNameConverter->GetPrimaryPath()); + Become(&TThis::StateFunc); + TVector<TString> topicNames; + for (const auto& topic : Topics) { + topicNames.emplace_back(topic.second.TopicNameConverter->GetPrimaryPath()); } - DoCheckACL = AppData(ctx)->PQConfig.GetCheckACL() && Token; - ctx.Send(MetaCacheId, new TEvDescribeTopicsRequest(topicNames)); + DoCheckACL = AppData(ctx)->PQConfig.GetCheckACL() && Token; + ctx.Send(MetaCacheId, new TEvDescribeTopicsRequest(topicNames)); } void TReadInitAndAuthActor::Die(const TActorContext& ctx) { @@ -2625,89 +2625,89 @@ void TReadInitAndAuthActor::SendCacheNavigateRequest(const TActorContext& ctx, c entry.SyncVersion = true; entry.Operation = NSchemeCache::TSchemeCacheNavigate::OpPath; schemeCacheRequest->ResultSet.emplace_back(entry); - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " Send client acl request"); + LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " Send client acl request"); ctx.Send(NewSchemeCache, new TEvTxProxySchemeCache::TEvNavigateKeySet(schemeCacheRequest.Release())); } -bool TReadInitAndAuthActor::ProcessTopicSchemeCacheResponse( - const NSchemeCache::TSchemeCacheNavigate::TEntry& entry, THashMap<TString, TTopicHolder>::iterator topicsIter, - const TActorContext& ctx -) { - auto& pqDescr = entry.PQGroupInfo->Description; - Y_VERIFY(entry.PQGroupInfo); - topicsIter->second.TabletID = pqDescr.GetBalancerTabletID(); - return CheckTopicACL(entry, topicsIter->first, ctx); -} - - -void TReadInitAndAuthActor::HandleTopicsDescribeResponse(TEvDescribeTopicsResponse::TPtr& ev, const TActorContext& ctx) { - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " Handle describe topics response"); - for (const auto& entry : ev->Get()->Result->ResultSet) { - auto processResult = ProcessMetaCacheTopicResponse(entry); - auto path = JoinPath(entry.Path); - auto it = Topics.find(path); - Y_VERIFY(it != Topics.end()); - - if (processResult.IsFatal) { - Topics.erase(it); - if (Topics.empty()) { - TStringBuilder reason; - reason << "Discovery for all topics failed. The last error was: " << processResult.Reason; - return CloseSession(reason, processResult.ErrorCode, ctx); - } else { - continue; - } - } - if (!ProcessTopicSchemeCacheResponse(entry, it, ctx)) - return; - - } - if (Topics.empty()) { - CloseSession("no topics found", PersQueue::ErrorCode::BAD_REQUEST, ctx); - return; - } - // ToDo[migration] - separate option - ? - bool doCheckClientAcl = DoCheckACL && !AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen(); - if (doCheckClientAcl) { - CheckClientACL(ctx); - } else { - FinishInitialization(ctx); - } -} - - -bool TReadInitAndAuthActor::CheckTopicACL( - const NSchemeCache::TSchemeCacheNavigate::TEntry& entry, const TString& topic, const TActorContext& ctx -) { - auto& pqDescr = entry.PQGroupInfo->Description; - //ToDo[migration] - proper auth setup - bool alwaysCheckPermissions = AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen(); - bool reqAuthRead = DoCheckACL && ( - pqDescr.GetPQTabletConfig().GetRequireAuthRead() || alwaysCheckPermissions - ); - - if (reqAuthRead && !CheckACLPermissionsForNavigate( - entry.SecurityObject, topic, NACLib::EAccessRights::SelectRow, - "No ReadTopic permissions", ctx - )) { - return false; - } - //ToDo[migration] - proper auth setup - bool doCheckReadRules = AppData(ctx)->PQConfig.GetCheckACL() && (Token || alwaysCheckPermissions); - if (doCheckReadRules) { - bool found = false; - for (auto& cons : pqDescr.GetPQTabletConfig().GetReadRules() ) { - if (cons == ClientId) { - found = true; - break; +bool TReadInitAndAuthActor::ProcessTopicSchemeCacheResponse( + const NSchemeCache::TSchemeCacheNavigate::TEntry& entry, THashMap<TString, TTopicHolder>::iterator topicsIter, + const TActorContext& ctx +) { + auto& pqDescr = entry.PQGroupInfo->Description; + Y_VERIFY(entry.PQGroupInfo); + topicsIter->second.TabletID = pqDescr.GetBalancerTabletID(); + return CheckTopicACL(entry, topicsIter->first, ctx); +} + + +void TReadInitAndAuthActor::HandleTopicsDescribeResponse(TEvDescribeTopicsResponse::TPtr& ev, const TActorContext& ctx) { + LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " Handle describe topics response"); + for (const auto& entry : ev->Get()->Result->ResultSet) { + auto processResult = ProcessMetaCacheTopicResponse(entry); + auto path = JoinPath(entry.Path); + auto it = Topics.find(path); + Y_VERIFY(it != Topics.end()); + + if (processResult.IsFatal) { + Topics.erase(it); + if (Topics.empty()) { + TStringBuilder reason; + reason << "Discovery for all topics failed. The last error was: " << processResult.Reason; + return CloseSession(reason, processResult.ErrorCode, ctx); + } else { + continue; + } + } + if (!ProcessTopicSchemeCacheResponse(entry, it, ctx)) + return; + + } + if (Topics.empty()) { + CloseSession("no topics found", PersQueue::ErrorCode::BAD_REQUEST, ctx); + return; + } + // ToDo[migration] - separate option - ? + bool doCheckClientAcl = DoCheckACL && !AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen(); + if (doCheckClientAcl) { + CheckClientACL(ctx); + } else { + FinishInitialization(ctx); + } +} + + +bool TReadInitAndAuthActor::CheckTopicACL( + const NSchemeCache::TSchemeCacheNavigate::TEntry& entry, const TString& topic, const TActorContext& ctx +) { + auto& pqDescr = entry.PQGroupInfo->Description; + //ToDo[migration] - proper auth setup + bool alwaysCheckPermissions = AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen(); + bool reqAuthRead = DoCheckACL && ( + pqDescr.GetPQTabletConfig().GetRequireAuthRead() || alwaysCheckPermissions + ); + + if (reqAuthRead && !CheckACLPermissionsForNavigate( + entry.SecurityObject, topic, NACLib::EAccessRights::SelectRow, + "No ReadTopic permissions", ctx + )) { + return false; + } + //ToDo[migration] - proper auth setup + bool doCheckReadRules = AppData(ctx)->PQConfig.GetCheckACL() && (Token || alwaysCheckPermissions); + if (doCheckReadRules) { + bool found = false; + for (auto& cons : pqDescr.GetPQTabletConfig().GetReadRules() ) { + if (cons == ClientId) { + found = true; + break; } } - if (!found) { - CloseSession( - TStringBuilder() << "no read rule provided for consumer '" << ClientPath << "' in topic '" << topic << "'", - PersQueue::ErrorCode::BAD_REQUEST, ctx - ); + if (!found) { + CloseSession( + TStringBuilder() << "no read rule provided for consumer '" << ClientPath << "' in topic '" << topic << "'", + PersQueue::ErrorCode::BAD_REQUEST, ctx + ); return false; } } @@ -2715,79 +2715,79 @@ bool TReadInitAndAuthActor::CheckTopicACL( } -void TReadInitAndAuthActor::CheckClientACL(const TActorContext& ctx) { - // ToDo[migration] - Through converter/metacache - ? - SendCacheNavigateRequest(ctx, AppData(ctx)->PQConfig.GetRoot() + "/" + ClientPath); -} +void TReadInitAndAuthActor::CheckClientACL(const TActorContext& ctx) { + // ToDo[migration] - Through converter/metacache - ? + SendCacheNavigateRequest(ctx, AppData(ctx)->PQConfig.GetRoot() + "/" + ClientPath); +} -void TReadInitAndAuthActor::HandleClientSchemeCacheResponse( - TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx -) { - TEvTxProxySchemeCache::TEvNavigateKeySetResult* msg = ev->Get(); - const NSchemeCache::TSchemeCacheNavigate* navigate = msg->Request.Get(); +void TReadInitAndAuthActor::HandleClientSchemeCacheResponse( + TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx +) { + TEvTxProxySchemeCache::TEvNavigateKeySetResult* msg = ev->Get(); + const NSchemeCache::TSchemeCacheNavigate* navigate = msg->Request.Get(); - Y_VERIFY(navigate->ResultSet.size() == 1); - auto& entry = navigate->ResultSet.front(); - auto path = "/" + JoinPath(entry.Path); // ToDo [migration] - through converter ? - if (navigate->ErrorCount > 0) { - const NSchemeCache::TSchemeCacheNavigate::EStatus status = navigate->ResultSet.front().Status; - CloseSession(TStringBuilder() << "Failed to read ACL for '" << path << "' Scheme cache error : " << status, PersQueue::ErrorCode::ERROR, ctx); - return; + Y_VERIFY(navigate->ResultSet.size() == 1); + auto& entry = navigate->ResultSet.front(); + auto path = "/" + JoinPath(entry.Path); // ToDo [migration] - through converter ? + if (navigate->ErrorCount > 0) { + const NSchemeCache::TSchemeCacheNavigate::EStatus status = navigate->ResultSet.front().Status; + CloseSession(TStringBuilder() << "Failed to read ACL for '" << path << "' Scheme cache error : " << status, PersQueue::ErrorCode::ERROR, ctx); + return; } - NACLib::EAccessRights rights = (NACLib::EAccessRights)(NACLib::EAccessRights::ReadAttributes + NACLib::EAccessRights::WriteAttributes); - if ( - !CheckACLPermissionsForNavigate(entry.SecurityObject, path, rights, "No ReadAsConsumer permissions", ctx) - ) { + NACLib::EAccessRights rights = (NACLib::EAccessRights)(NACLib::EAccessRights::ReadAttributes + NACLib::EAccessRights::WriteAttributes); + if ( + !CheckACLPermissionsForNavigate(entry.SecurityObject, path, rights, "No ReadAsConsumer permissions", ctx) + ) { return; - } - FinishInitialization(ctx); -} - - -bool TReadInitAndAuthActor::CheckACLPermissionsForNavigate( - const TIntrusivePtr<TSecurityObject>& secObject, const TString& path, - NACLib::EAccessRights rights, const TString& errorTextWhenAccessDenied, const TActorContext& ctx -) { - // TODO: SCHEME_ERROR если нет топика/консумера - // TODO: если AccessDenied на корень, то надо ACCESS_DENIED, а не SCHEME_ERROR - - if (DoCheckACL && !secObject->CheckAccess(rights, *Token)) { - CloseSession( - TStringBuilder() << errorTextWhenAccessDenied << " for '" << path - << "' for subject '" << Token->GetUserSID() << "'", - PersQueue::ErrorCode::ACCESS_DENIED, ctx - ); - return false; - } - return true; -} - - -void TReadInitAndAuthActor::FinishInitialization(const TActorContext& ctx) { - TTopicTabletsPairs res; - for (auto& t : Topics) { - res.emplace_back(decltype(res)::value_type({ - t.second.TopicNameConverter, t.second.TabletID, t.second.CloudId, t.second.DbId, t.second.FolderId - })); - } - ctx.Send(ParentId, new TEvPQProxy::TEvAuthResultOk(std::move(res))); - Die(ctx); + } + FinishInitialization(ctx); +} + + +bool TReadInitAndAuthActor::CheckACLPermissionsForNavigate( + const TIntrusivePtr<TSecurityObject>& secObject, const TString& path, + NACLib::EAccessRights rights, const TString& errorTextWhenAccessDenied, const TActorContext& ctx +) { + // TODO: SCHEME_ERROR если нет топика/консумера + // TODO: если AccessDenied на корень, то надо ACCESS_DENIED, а не SCHEME_ERROR + + if (DoCheckACL && !secObject->CheckAccess(rights, *Token)) { + CloseSession( + TStringBuilder() << errorTextWhenAccessDenied << " for '" << path + << "' for subject '" << Token->GetUserSID() << "'", + PersQueue::ErrorCode::ACCESS_DENIED, ctx + ); + return false; + } + return true; +} + + +void TReadInitAndAuthActor::FinishInitialization(const TActorContext& ctx) { + TTopicTabletsPairs res; + for (auto& t : Topics) { + res.emplace_back(decltype(res)::value_type({ + t.second.TopicNameConverter, t.second.TabletID, t.second.CloudId, t.second.DbId, t.second.FolderId + })); + } + ctx.Send(ParentId, new TEvPQProxy::TEvAuthResultOk(std::move(res))); + Die(ctx); } // READINFOACTOR -TReadInfoActor::TReadInfoActor( - TEvPQReadInfoRequest* request, const NPersQueue::TTopicsListController& topicsHandler, - const TActorId& schemeCache, const TActorId& newSchemeCache, - TIntrusivePtr<NMonitoring::TDynamicCounters> counters -) +TReadInfoActor::TReadInfoActor( + TEvPQReadInfoRequest* request, const NPersQueue::TTopicsListController& topicsHandler, + const TActorId& schemeCache, const TActorId& newSchemeCache, + TIntrusivePtr<NMonitoring::TDynamicCounters> counters +) : TBase(request) , SchemeCache(schemeCache) , NewSchemeCache(newSchemeCache) , AuthInitActor() , Counters(counters) - , TopicsHandler(topicsHandler) + , TopicsHandler(topicsHandler) { Y_ASSERT(request); } @@ -2820,18 +2820,18 @@ void TReadInfoActor::Bootstrap(const TActorContext& ctx) { THashSet<TString> topicsToResolve; for (auto& t : request->topics()) { - if (t.path().empty()) { + if (t.path().empty()) { AnswerError("empty topic in init request", PersQueue::ErrorCode::BAD_REQUEST, ctx); return; } - topicsToResolve.insert(t.path()); + topicsToResolve.insert(t.path()); } - AuthInitActor = ctx.Register(new TReadInitAndAuthActor( - ctx, ctx.SelfID, ClientId, 0, TString("read_info:") + Request().GetPeerName(), - SchemeCache, NewSchemeCache, Counters, token, - TopicsHandler.GetReadTopicsList(topicsToResolve, readOnlyLocal, Request().GetDatabaseName().GetOrElse(TString())) - )); + AuthInitActor = ctx.Register(new TReadInitAndAuthActor( + ctx, ctx.SelfID, ClientId, 0, TString("read_info:") + Request().GetPeerName(), + SchemeCache, NewSchemeCache, Counters, token, + TopicsHandler.GetReadTopicsList(topicsToResolve, readOnlyLocal, Request().GetDatabaseName().GetOrElse(TString())) + )); } @@ -2846,7 +2846,7 @@ void TReadInfoActor::Die(const TActorContext& ctx) { void TReadInfoActor::Handle(TEvPQProxy::TEvAuthResultOk::TPtr& ev, const TActorContext& ctx) { LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, "GetReadInfo auth ok fo read info, got " << ev->Get()->TopicAndTablets.size() << " topics"); - TopicAndTablets = std::move(ev->Get()->TopicAndTablets); + TopicAndTablets = std::move(ev->Get()->TopicAndTablets); if (TopicAndTablets.empty()) { AnswerError("empty list of topics", PersQueue::ErrorCode::UNKNOWN_TOPIC, ctx); return; @@ -2855,7 +2855,7 @@ void TReadInfoActor::Handle(TEvPQProxy::TEvAuthResultOk::TPtr& ev, const TActorC NKikimrClient::TPersQueueRequest proto; proto.MutableMetaRequest()->MutableCmdGetReadSessionsInfo()->SetClientId(ClientId); for (auto& t : TopicAndTablets) { - proto.MutableMetaRequest()->MutableCmdGetReadSessionsInfo()->AddTopic(t.TopicNameConverter->GetClientsideName()); + proto.MutableMetaRequest()->MutableCmdGetReadSessionsInfo()->AddTopic(t.TopicNameConverter->GetClientsideName()); } ctx.Register(NMsgBusProxy::CreateActorServerPersQueue( diff --git a/ydb/services/persqueue_v1/grpc_pq_schema.cpp b/ydb/services/persqueue_v1/grpc_pq_schema.cpp index c844a712d3..7756adc797 100644 --- a/ydb/services/persqueue_v1/grpc_pq_schema.cpp +++ b/ydb/services/persqueue_v1/grpc_pq_schema.cpp @@ -40,7 +40,7 @@ TPQSchemaService::TPQSchemaService(const TActorId& schemeCache, void TPQSchemaService::Bootstrap(const TActorContext& ctx) { - if (!AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen()) { // ToDo[migration]: switch to haveClusters + if (!AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen()) { // ToDo[migration]: switch to haveClusters ctx.Send(NPQ::NClusterTracker::MakeClusterTrackerID(), new NPQ::NClusterTracker::TEvClusterTracker::TEvSubscribe); } @@ -136,7 +136,7 @@ void TDescribeTopicActor::HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEv auto settings = result.mutable_settings(); Ydb::Scheme::Entry *selfEntry = result.mutable_self(); const auto& selfInfo = response.Self->Info; - selfEntry->set_name(path + "/" + selfInfo.GetName()); + selfEntry->set_name(path + "/" + selfInfo.GetName()); selfEntry->set_type(static_cast<Ydb::Scheme::Entry::Type>(selfInfo.GetPathType())); ConvertDirectoryEntry(selfInfo, selfEntry, true); if (response.PQGroupInfo) { diff --git a/ydb/services/persqueue_v1/grpc_pq_write.cpp b/ydb/services/persqueue_v1/grpc_pq_write.cpp index dc9f9eaf27..5f17a2f3ab 100644 --- a/ydb/services/persqueue_v1/grpc_pq_write.cpp +++ b/ydb/services/persqueue_v1/grpc_pq_write.cpp @@ -37,15 +37,15 @@ TPQWriteService::TPQWriteService(const TActorId& schemeCache, const TActorId& ne void TPQWriteService::Bootstrap(const TActorContext& ctx) { - HaveClusters = !AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen(); // ToDo[migration]: switch to proper option - if (HaveClusters) { + HaveClusters = !AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen(); // ToDo[migration]: switch to proper option + if (HaveClusters) { ctx.Send(NPQ::NClusterTracker::MakeClusterTrackerID(), new NPQ::NClusterTracker::TEvClusterTracker::TEvSubscribe); } ctx.Send(NNetClassifier::MakeNetClassifierID(), new NNetClassifier::TEvNetClassifier::TEvSubscribe); - ConverterFactory = std::make_shared<NPersQueue::TTopicNamesConverterFactory>( - AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen(), AppData(ctx)->PQConfig.GetRoot() - ); + ConverterFactory = std::make_shared<NPersQueue::TTopicNamesConverterFactory>( + AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen(), AppData(ctx)->PQConfig.GetRoot() + ); Become(&TThis::StateFunc); } @@ -178,7 +178,7 @@ void TPQWriteService::Handle(NKikimr::NGRpcService::TEvStreamPQWriteRequest::TPt TString localCluster = AvailableLocalCluster(ctx); - if (HaveClusters && localCluster.empty()) { + if (HaveClusters && localCluster.empty()) { ev->Get()->GetStreamCtx()->Attach(ctx.SelfID); if (LocalCluster) { LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "new grpc connection failed - cluster disabled"); @@ -189,19 +189,19 @@ void TPQWriteService::Handle(NKikimr::NGRpcService::TEvStreamPQWriteRequest::TPt } return; } else { - TopicsHandler = std::make_unique<NPersQueue::TTopicsListController>( - ConverterFactory, HaveClusters, TVector<TString>{}, localCluster - ); + TopicsHandler = std::make_unique<NPersQueue::TTopicsListController>( + ConverterFactory, HaveClusters, TVector<TString>{}, localCluster + ); const ui64 cookie = NextCookie(); LOG_DEBUG_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "new session created cookie " << cookie); auto ip = ev->Get()->GetStreamCtx()->GetPeerName(); - TActorId worker = ctx.Register(new TWriteSessionActor( - ev->Release().Release(), cookie, SchemeCache, NewSchemeCache, Counters, - DatacenterClassifier ? DatacenterClassifier->ClassifyAddress(NAddressClassifier::ExtractAddress(ip)) : "unknown", - *TopicsHandler - )); + TActorId worker = ctx.Register(new TWriteSessionActor( + ev->Release().Release(), cookie, SchemeCache, NewSchemeCache, Counters, + DatacenterClassifier ? DatacenterClassifier->ClassifyAddress(NAddressClassifier::ExtractAddress(ip)) : "unknown", + *TopicsHandler + )); Sessions[cookie] = worker; } @@ -212,8 +212,8 @@ bool TPQWriteService::TooMuchSessions() { } -TString TPQWriteService::AvailableLocalCluster(const TActorContext&) const { - return HaveClusters && Enabled ? *LocalCluster : ""; +TString TPQWriteService::AvailableLocalCluster(const TActorContext&) const { + return HaveClusters && Enabled ? *LocalCluster : ""; } diff --git a/ydb/services/persqueue_v1/grpc_pq_write.h b/ydb/services/persqueue_v1/grpc_pq_write.h index bb0af21dee..73148388b2 100644 --- a/ydb/services/persqueue_v1/grpc_pq_write.h +++ b/ydb/services/persqueue_v1/grpc_pq_write.h @@ -81,9 +81,9 @@ private: TString DeleteSourceIdQuery; NAddressClassifier::TLabeledAddressClassifier::TConstPtr DatacenterClassifier; // Detects client's datacenter by IP. May be null - bool HaveClusters; - NPersQueue::TConverterFactoryPtr ConverterFactory; - std::unique_ptr<NPersQueue::TTopicsListController> TopicsHandler; + bool HaveClusters; + NPersQueue::TConverterFactoryPtr ConverterFactory; + std::unique_ptr<NPersQueue::TTopicsListController> TopicsHandler; }; diff --git a/ydb/services/persqueue_v1/grpc_pq_write_actor.cpp b/ydb/services/persqueue_v1/grpc_pq_write_actor.cpp index 5fba4e8686..3038cc82a6 100644 --- a/ydb/services/persqueue_v1/grpc_pq_write_actor.cpp +++ b/ydb/services/persqueue_v1/grpc_pq_write_actor.cpp @@ -22,7 +22,7 @@ using namespace NKikimrClient; namespace NKikimr { -using namespace NSchemeCache; +using namespace NSchemeCache; Ydb::PersQueue::V1::Codec CodecByName(const TString& codec) { static const THashMap<TString, Ydb::PersQueue::V1::Codec> codecsByName = { @@ -108,19 +108,19 @@ static const TString UPDATE_SOURCEID_QUERY2 = "` (Hash, Topic, SourceId, CreateT //TODO: add here tracking of bytes in/out -TWriteSessionActor::TWriteSessionActor( - NKikimr::NGRpcService::TEvStreamPQWriteRequest* request, const ui64 cookie, - const NActors::TActorId& schemeCache, const NActors::TActorId& newSchemeCache, - TIntrusivePtr<NMonitoring::TDynamicCounters> counters, const TMaybe<TString> clientDC, - const NPersQueue::TTopicsListController& topicsController -) +TWriteSessionActor::TWriteSessionActor( + NKikimr::NGRpcService::TEvStreamPQWriteRequest* request, const ui64 cookie, + const NActors::TActorId& schemeCache, const NActors::TActorId& newSchemeCache, + TIntrusivePtr<NMonitoring::TDynamicCounters> counters, const TMaybe<TString> clientDC, + const NPersQueue::TTopicsListController& topicsController +) : Request(request) , State(ES_CREATED) , SchemeCache(schemeCache) , NewSchemeCache(newSchemeCache) , PeerName("") , Cookie(cookie) - , TopicsController(topicsController) + , TopicsController(topicsController) , Partition(0) , PreferedPartition(Max<ui32>()) , NumReserveBytesRequests(0) @@ -174,23 +174,23 @@ void TWriteSessionActor::HandleDone(const TActorContext& ctx) { Die(ctx); } -TString WriteRequestToLog(const Ydb::PersQueue::V1::StreamingWriteClientMessage& proto) { - switch (proto.client_message_case()) { - case StreamingWriteClientMessage::kInitRequest: - return proto.ShortDebugString(); - break; - case StreamingWriteClientMessage::kWriteRequest: - return " write_request[data omitted]"; - break; - case StreamingWriteClientMessage::kUpdateTokenRequest: - return " update_token_request [content omitted]"; - default: - return TString(); - } -} - +TString WriteRequestToLog(const Ydb::PersQueue::V1::StreamingWriteClientMessage& proto) { + switch (proto.client_message_case()) { + case StreamingWriteClientMessage::kInitRequest: + return proto.ShortDebugString(); + break; + case StreamingWriteClientMessage::kWriteRequest: + return " write_request[data omitted]"; + break; + case StreamingWriteClientMessage::kUpdateTokenRequest: + return " update_token_request [content omitted]"; + default: + return TString(); + } +} + void TWriteSessionActor::Handle(IContext::TEvReadFinished::TPtr& ev, const TActorContext& ctx) { - LOG_DEBUG_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "session v1 cookie: " << Cookie << " sessionId: " << OwnerCookie << " grpc read done: success: " << ev->Get()->Success << " data: " << WriteRequestToLog(ev->Get()->Record)); + LOG_DEBUG_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "session v1 cookie: " << Cookie << " sessionId: " << OwnerCookie << " grpc read done: success: " << ev->Get()->Success << " data: " << WriteRequestToLog(ev->Get()->Record)); if (!ev->Get()->Success) { LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "session v1 cookie: " << Cookie << " sessionId: " << OwnerCookie << " grpc read failed"); ctx.Send(ctx.SelfID, new TEvPQProxy::TEvDone()); @@ -260,7 +260,7 @@ void TWriteSessionActor::Handle(TEvPQProxy::TEvDone::TPtr&, const TActorContext& } void TWriteSessionActor::CheckACL(const TActorContext& ctx) { - //Y_VERIFY(ACLCheckInProgress); + //Y_VERIFY(ACLCheckInProgress); NACLib::EAccessRights rights = NACLib::EAccessRights::UpdateRow; @@ -283,7 +283,7 @@ void TWriteSessionActor::CheckACL(const TActorContext& ctx) { } } else { TString errorReason = Sprintf("access to topic '%s' denied for '%s' due to 'no WriteTopic rights', Marker# PQ1125", - TopicConverter->GetClientsideName().c_str(), + TopicConverter->GetClientsideName().c_str(), Token->GetUserSID().c_str()); CloseSession(errorReason, PersQueue::ErrorCode::ACCESS_DENIED, ctx); } @@ -304,15 +304,15 @@ void TWriteSessionActor::Handle(TEvPQProxy::TEvWriteInit::TPtr& ev, const TActor return; } - TopicConverter = TopicsController.GetWriteTopicConverter(init.topic(), Request->GetDatabaseName().GetOrElse("/Root")); - if (!TopicConverter->IsValid()) { - CloseSession( - TStringBuilder() << "topic " << init.topic() << " could not be recognized: " << TopicConverter->GetReason(), - PersQueue::ErrorCode::BAD_REQUEST, ctx - ); - return; + TopicConverter = TopicsController.GetWriteTopicConverter(init.topic(), Request->GetDatabaseName().GetOrElse("/Root")); + if (!TopicConverter->IsValid()) { + CloseSession( + TStringBuilder() << "topic " << init.topic() << " could not be recognized: " << TopicConverter->GetReason(), + PersQueue::ErrorCode::BAD_REQUEST, ctx + ); + return; } - + PeerName = event->PeerName; SourceId = init.message_group_id(); @@ -325,7 +325,7 @@ void TWriteSessionActor::Handle(TEvPQProxy::TEvWriteInit::TPtr& ev, const TActor } EscapedSourceId = HexEncode(encodedSourceId); - TString s = TopicConverter->GetClientsideName() + encodedSourceId; + TString s = TopicConverter->GetClientsideName() + encodedSourceId; Hash = MurmurHash<ui32>(s.c_str(), s.size(), MURMUR_ARRAY_SEED); LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "session request cookie: " << Cookie << " " << init << " from " << PeerName); @@ -345,10 +345,10 @@ void TWriteSessionActor::Handle(TEvPQProxy::TEvWriteInit::TPtr& ev, const TActor PreferedPartition = init.partition_group_id() > 0 ? init.partition_group_id() - 1 : Max<ui32>(); - InitMeta = GetInitialDataChunk(init, TopicConverter->GetFullLegacyName(), PeerName); // ToDo[migration] - check? + InitMeta = GetInitialDataChunk(init, TopicConverter->GetFullLegacyName(), PeerName); // ToDo[migration] - check? auto subGroup = GetServiceCounters(Counters, "pqproxy|SLI"); - Aggr = {{{{"Account", TopicConverter->GetAccount()}}, {"total"}}}; + Aggr = {{{{"Account", TopicConverter->GetAccount()}}, {"total"}}}; SLITotal = NKikimr::NPQ::TMultiCounter(subGroup, Aggr, {}, {"RequestsTotal"}, true, "sensor", false); SLIErrors = NKikimr::NPQ::TMultiCounter(subGroup, Aggr, {}, {"RequestsError"}, true, "sensor", false); @@ -364,7 +364,7 @@ void TWriteSessionActor::SetupCounters() { //now topic is checked, can create group for real topic, not garbage auto subGroup = GetServiceCounters(Counters, "pqproxy|writeSession"); - TVector<NPQ::TLabelsInfo> aggr = NKikimr::NPQ::GetLabels(LocalDC, TopicConverter->GetClientsideName()); + TVector<NPQ::TLabelsInfo> aggr = NKikimr::NPQ::GetLabels(LocalDC, TopicConverter->GetClientsideName()); BytesInflight = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"BytesInflight"}, false); BytesInflightTotal = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"BytesInflightTotal"}, false); @@ -381,7 +381,7 @@ void TWriteSessionActor::SetupCounters(const TString& cloudId, const TString& db { //now topic is checked, can create group for real topic, not garbage auto subGroup = NKikimr::NPQ::GetCountersForStream(Counters, "writeSession"); - TVector<NPQ::TLabelsInfo> aggr = NKikimr::NPQ::GetLabelsForStream(TopicConverter->GetClientsideName(), cloudId, dbId, folderId); + TVector<NPQ::TLabelsInfo> aggr = NKikimr::NPQ::GetLabelsForStream(TopicConverter->GetClientsideName(), cloudId, dbId, folderId); BytesInflight = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"stream.internal_write.bytes_proceeding"}, false); BytesInflightTotal = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"stream.internal_write.bytes_proceeding_total"}, false); @@ -396,35 +396,35 @@ void TWriteSessionActor::SetupCounters(const TString& cloudId, const TString& db void TWriteSessionActor::InitCheckSchema(const TActorContext& ctx, bool needWaitSchema) { LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "init check schema"); - if (!needWaitSchema) { - ACLCheckInProgress = true; + if (!needWaitSchema) { + ACLCheckInProgress = true; } - ctx.Send(SchemeCache, new TEvDescribeTopicsRequest({TopicConverter->GetPrimaryPath()})); + ctx.Send(SchemeCache, new TEvDescribeTopicsRequest({TopicConverter->GetPrimaryPath()})); if (needWaitSchema) { State = ES_WAIT_SCHEME_2; } } -void TWriteSessionActor::Handle(TEvDescribeTopicsResponse::TPtr& ev, const TActorContext& ctx) { - auto* res = ev->Get()->Result.Get(); - Y_VERIFY(res->ResultSet.size() == 1); +void TWriteSessionActor::Handle(TEvDescribeTopicsResponse::TPtr& ev, const TActorContext& ctx) { + auto* res = ev->Get()->Result.Get(); + Y_VERIFY(res->ResultSet.size() == 1); - auto& entry = res->ResultSet[0]; + auto& entry = res->ResultSet[0]; TString errorReason; - auto processResult = ProcessMetaCacheTopicResponse(entry); - if (processResult.IsFatal) { - CloseSession(processResult.Reason, processResult.ErrorCode, ctx); + auto processResult = ProcessMetaCacheTopicResponse(entry); + if (processResult.IsFatal) { + CloseSession(processResult.Reason, processResult.ErrorCode, ctx); return; } - auto& description = entry.PQGroupInfo->Description; - Y_VERIFY(description.PartitionsSize() > 0); - Y_VERIFY(description.HasPQTabletConfig()); - InitialPQTabletConfig = description.GetPQTabletConfig(); + auto& description = entry.PQGroupInfo->Description; + Y_VERIFY(description.PartitionsSize() > 0); + Y_VERIFY(description.HasPQTabletConfig()); + InitialPQTabletConfig = description.GetPQTabletConfig(); - BalancerTabletId = description.GetBalancerTabletID(); + BalancerTabletId = description.GetBalancerTabletID(); - for (ui32 i = 0; i < description.PartitionsSize(); ++i) { - const auto& pi = description.GetPartitions(i); + for (ui32 i = 0; i < description.PartitionsSize(); ++i) { + const auto& pi = description.GetPartitions(i); PartitionToTablet[pi.GetPartitionId()] = pi.GetTabletId(); } @@ -436,10 +436,10 @@ void TWriteSessionActor::Handle(TEvDescribeTopicsResponse::TPtr& ev, const TActo SetupCounters(); } - Y_VERIFY (entry.SecurityObject); - ACL.Reset(new TAclWrapper(entry.SecurityObject)); - LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "session v1 cookie: " << Cookie << " sessionId: " << OwnerCookie << " describe result for acl check"); - + Y_VERIFY (entry.SecurityObject); + ACL.Reset(new TAclWrapper(entry.SecurityObject)); + LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "session v1 cookie: " << Cookie << " sessionId: " << OwnerCookie << " describe result for acl check"); + if (Request->GetInternalToken().empty()) { // session without auth if (AppData(ctx)->PQConfig.GetRequireCredentialsInNewProtocol()) { Request->ReplyUnauthenticated("Unauthenticated access is forbidden, please provide credentials"); @@ -454,7 +454,7 @@ void TWriteSessionActor::Handle(TEvDescribeTopicsResponse::TPtr& ev, const TActo Auth = *Request->GetYdbToken(); Token = new NACLib::TUserToken(Request->GetInternalToken()); - CheckACL(ctx); + CheckACL(ctx); } } @@ -464,11 +464,11 @@ void TWriteSessionActor::Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult:: Y_VERIFY(navigate->ResultSet.size() == 1); if (navigate->ErrorCount > 0) { const NSchemeCache::TSchemeCacheNavigate::EStatus status = navigate->ResultSet.front().Status; - return CloseSession( - TStringBuilder() << "Failed to read ACL for '" << TopicConverter->GetClientsideName() - << "' Scheme cache error : " << status, - PersQueue::ErrorCode::ERROR, ctx - ); + return CloseSession( + TStringBuilder() << "Failed to read ACL for '" << TopicConverter->GetClientsideName() + << "' Scheme cache error : " << status, + PersQueue::ErrorCode::ERROR, ctx + ); } const auto& pqDescription = navigate->ResultSet.front().PQGroupInfo->Description; @@ -478,7 +478,7 @@ void TWriteSessionActor::Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult:: InitialPQTabletConfig = pqDescription.GetPQTabletConfig(); if (!pqDescription.HasBalancerTabletID()) { - TString errorReason = Sprintf("topic '%s' has no balancer, Marker# PQ93", TopicConverter->GetClientsideName().c_str()); + TString errorReason = Sprintf("topic '%s' has no balancer, Marker# PQ93", TopicConverter->GetClientsideName().c_str()); CloseSession(errorReason, PersQueue::ErrorCode::UNKNOWN_TOPIC, ctx); return; } @@ -515,7 +515,7 @@ void TWriteSessionActor::Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult:: void TWriteSessionActor::DiscoverPartition(const NActors::TActorContext& ctx) { - if (AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen()) { // ToDo[migration] - separate flag for having config tables + if (AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen()) { // ToDo[migration] - separate flag for having config tables auto partitionId = PreferedPartition < Max<ui32>() ? PreferedPartition : NKikimr::NDataStreams::V1::ShardFromDecimal(NKikimr::NDataStreams::V1::HexBytesToDecimal(MD5::Calc(SourceId)), PartitionToTablet.size()); ProceedPartition(partitionId, ctx); @@ -528,7 +528,7 @@ void TWriteSessionActor::DiscoverPartition(const NActors::TActorContext& ctx) { ev->Record.MutableRequest()->SetType(NKikimrKqp::QUERY_TYPE_SQL_DML); ev->Record.MutableRequest()->SetKeepSession(false); ev->Record.MutableRequest()->SetQuery(SelectSourceIdQuery); - ev->Record.MutableRequest()->SetDatabase(NKikimr::NPQ::GetDatabaseFromConfig(AppData(ctx)->PQConfig)); + ev->Record.MutableRequest()->SetDatabase(NKikimr::NPQ::GetDatabaseFromConfig(AppData(ctx)->PQConfig)); // fill tx settings: set commit tx flag & begin new serializable tx. ev->Record.MutableRequest()->MutableTxControl()->set_commit_tx(true); ev->Record.MutableRequest()->MutableTxControl()->mutable_begin_tx()->mutable_serializable_read_write(); @@ -536,7 +536,7 @@ void TWriteSessionActor::DiscoverPartition(const NActors::TActorContext& ctx) { ev->Record.MutableRequest()->MutableQueryCachePolicy()->set_keep_in_cache(true); NClient::TParameters parameters; parameters["$Hash"] = Hash; - parameters["$Topic"] = TopicConverter->GetClientsideName(); + parameters["$Topic"] = TopicConverter->GetClientsideName(); parameters["$SourceId"] = EscapedSourceId; ev->Record.MutableRequest()->MutableParameters()->Swap(¶meters); ctx.Send(NKqp::MakeKqpProxyID(ctx.SelfID.NodeId()), ev.Release()); @@ -546,7 +546,7 @@ void TWriteSessionActor::DiscoverPartition(const NActors::TActorContext& ctx) { void TWriteSessionActor::UpdatePartition(const TActorContext& ctx) { Y_VERIFY(State == ES_WAIT_TABLE_REQUEST_1 || State == ES_WAIT_NEXT_PARTITION); - auto ev = MakeUpdateSourceIdMetadataRequest(ctx); + auto ev = MakeUpdateSourceIdMetadataRequest(ctx); ctx.Send(NKqp::MakeKqpProxyID(ctx.SelfID.NodeId()), ev.Release()); State = ES_WAIT_TABLE_REQUEST_2; } @@ -644,16 +644,16 @@ void TWriteSessionActor::Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr &ev, const } } -THolder<NKqp::TEvKqp::TEvQueryRequest> TWriteSessionActor::MakeUpdateSourceIdMetadataRequest( - const NActors::TActorContext& ctx -) { - +THolder<NKqp::TEvKqp::TEvQueryRequest> TWriteSessionActor::MakeUpdateSourceIdMetadataRequest( + const NActors::TActorContext& ctx +) { + auto ev = MakeHolder<NKqp::TEvKqp::TEvQueryRequest>(); ev->Record.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_EXECUTE); ev->Record.MutableRequest()->SetType(NKikimrKqp::QUERY_TYPE_SQL_DML); ev->Record.MutableRequest()->SetQuery(UpdateSourceIdQuery); - ev->Record.MutableRequest()->SetDatabase(NKikimr::NPQ::GetDatabaseFromConfig(AppData(ctx)->PQConfig)); + ev->Record.MutableRequest()->SetDatabase(NKikimr::NPQ::GetDatabaseFromConfig(AppData(ctx)->PQConfig)); ev->Record.MutableRequest()->SetKeepSession(false); // fill tx settings: set commit tx flag & begin new serializable tx. ev->Record.MutableRequest()->MutableTxControl()->set_commit_tx(true); @@ -663,7 +663,7 @@ THolder<NKqp::TEvKqp::TEvQueryRequest> TWriteSessionActor::MakeUpdateSourceIdMet NClient::TParameters parameters; parameters["$Hash"] = Hash; - parameters["$Topic"] = TopicConverter->GetClientsideName(); + parameters["$Topic"] = TopicConverter->GetClientsideName(); parameters["$SourceId"] = EscapedSourceId; parameters["$CreateTime"] = SourceIdCreateTime; parameters["$AccessTime"] = TInstant::Now().MilliSeconds(); @@ -691,10 +691,10 @@ void TWriteSessionActor::ProceedPartition(const ui32 partition, const TActorCont ui64 tabletId = it != PartitionToTablet.end() ? it->second : 0; if (!tabletId) { - CloseSession( - Sprintf("no partition %u in topic '%s', Marker# PQ4", Partition, TopicConverter->GetClientsideName().c_str()), - PersQueue::ErrorCode::UNKNOWN_TOPIC, ctx - ); + CloseSession( + Sprintf("no partition %u in topic '%s', Marker# PQ4", Partition, TopicConverter->GetClientsideName().c_str()), + PersQueue::ErrorCode::UNKNOWN_TOPIC, ctx + ); return; } @@ -770,8 +770,8 @@ void TWriteSessionActor::Handle(NPQ::TEvPartitionWriter::TEvInitResult::TPtr& ev init->set_session_id(EscapeC(OwnerCookie)); init->set_last_sequence_number(maxSeqNo); init->set_partition_id(Partition); - init->set_topic(TopicConverter->GetModernName()); - init->set_cluster(TopicConverter->GetCluster()); + init->set_topic(TopicConverter->GetModernName()); + init->set_cluster(TopicConverter->GetCluster()); init->set_block_format_version(0); if (InitialPQTabletConfig.HasCodecs()) { for (const auto& codecName : InitialPQTabletConfig.GetCodecs().GetCodecs()) { @@ -1146,7 +1146,7 @@ void TWriteSessionActor::HandlePoison(TEvPQProxy::TEvDieCommand::TPtr& ev, const void TWriteSessionActor::LogSession(const TActorContext& ctx) { LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "write session: cookie=" << Cookie << " sessionId=" << OwnerCookie << " userAgent=\"" << UserAgent << "\" ip=" << PeerName << " proto=v1 " - << " topic=" << TopicConverter->GetModernName() << " durationSec=" << (ctx.Now() - StartTime).Seconds()); + << " topic=" << TopicConverter->GetModernName() << " durationSec=" << (ctx.Now() - StartTime).Seconds()); LogSessionDeadline = ctx.Now() + TDuration::Hours(1) + TDuration::Seconds(rand() % 60); } @@ -1158,9 +1158,9 @@ void TWriteSessionActor::HandleWakeup(const TActorContext& ctx) { RequestNotChecked = false; InitCheckSchema(ctx); } - // ToDo[migration] - separate flag for having config tables + // ToDo[migration] - separate flag for having config tables if (!AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen() && !SourceIdUpdateInfly && ctx.Now() - LastSourceIdUpdate > SOURCEID_UPDATE_PERIOD) { - auto ev = MakeUpdateSourceIdMetadataRequest(ctx); + auto ev = MakeUpdateSourceIdMetadataRequest(ctx); SourceIdUpdateInfly = true; ctx.Send(NKqp::MakeKqpProxyID(ctx.SelfID.NodeId()), ev.Release()); } diff --git a/ydb/services/persqueue_v1/persqueue_common_ut.cpp b/ydb/services/persqueue_v1/persqueue_common_ut.cpp index a4d32e199d..9d706d0656 100644 --- a/ydb/services/persqueue_v1/persqueue_common_ut.cpp +++ b/ydb/services/persqueue_v1/persqueue_common_ut.cpp @@ -36,32 +36,32 @@ using namespace Ydb::PersQueue::V1; using namespace NThreading; using namespace NNetClassifier; - -#define MAKE_WRITE_STREAM(TOKEN) \ - grpc::ClientContext context; \ - context.AddMetadata(NYdb::YDB_AUTH_TICKET_HEADER, TOKEN); \ - auto stream = server.ServiceStub->StreamingWrite(&context); \ - - + +#define MAKE_WRITE_STREAM(TOKEN) \ + grpc::ClientContext context; \ + context.AddMetadata(NYdb::YDB_AUTH_TICKET_HEADER, TOKEN); \ + auto stream = server.ServiceStub->StreamingWrite(&context); \ + + Y_UNIT_TEST_SUITE(TPersQueueCommonTest) { // Auth* tests are for both authentication and authorization - Y_UNIT_TEST(Auth_CreateGrpcStreamWithInvalidTokenInInitialMetadata_SessionClosedWithUnauthenticatedError) { - TPersQueueV1TestServer server; - SET_LOCALS; - - runtime->GetAppData().PQConfig.SetRequireCredentialsInNewProtocol(true); - runtime->GetAppData().EnforceUserTokenRequirement = true; + Y_UNIT_TEST(Auth_CreateGrpcStreamWithInvalidTokenInInitialMetadata_SessionClosedWithUnauthenticatedError) { + TPersQueueV1TestServer server; + SET_LOCALS; + + runtime->GetAppData().PQConfig.SetRequireCredentialsInNewProtocol(true); + runtime->GetAppData().EnforceUserTokenRequirement = true; TVector<TString> invalidTokens = {TString(), "test_user", "test_user@invalid_domain"}; for (const auto &invalidToken : invalidTokens) { Cerr << "Invalid token under test is '" << invalidToken << "'" << Endl; - MAKE_WRITE_STREAM(invalidToken); + MAKE_WRITE_STREAM(invalidToken); // TODO: Message should be written to gRPC in order to get error. Fix gRPC data plane API code if our expectations are different. // Note that I check that initial metadata is sent during gRPC stream constructor. StreamingWriteClientMessage clientMessage; StreamingWriteServerMessage serverMessage; - clientMessage.mutable_init_request()->set_topic(server.GetTopicPath()); + clientMessage.mutable_init_request()->set_topic(server.GetTopicPath()); clientMessage.mutable_init_request()->set_message_group_id("test-group-id"); AssertSuccessfullStreamingOperation(stream->Write(clientMessage), stream, &clientMessage); @@ -76,22 +76,22 @@ Y_UNIT_TEST_SUITE(TPersQueueCommonTest) { return "test_user_" + ToString(i) + "@" + BUILTIN_ACL_DOMAIN; } - Y_UNIT_TEST(Auth_MultipleUpdateTokenRequestIterationsWithValidToken_GotUpdateTokenResponseForEachRequest) { - TPersQueueV1TestServer server; - + Y_UNIT_TEST(Auth_MultipleUpdateTokenRequestIterationsWithValidToken_GotUpdateTokenResponseForEachRequest) { + TPersQueueV1TestServer server; + const int iterations = 10; NACLib::TDiffACL acl; for (int i = 0; i != iterations; ++i) { acl.AddAccess(NACLib::EAccessType::Allow, NACLib::UpdateRow, GenerateValidToken(i)); } - server.ModifyTopicACL(server.GetTopic(), acl); + server.ModifyTopicACL(server.GetTopic(), acl); - MAKE_WRITE_STREAM(GenerateValidToken(0)); + MAKE_WRITE_STREAM(GenerateValidToken(0)); StreamingWriteClientMessage clientMessage; StreamingWriteServerMessage serverMessage; - clientMessage.mutable_init_request()->set_topic(server.GetTopicPath()); + clientMessage.mutable_init_request()->set_topic(server.GetTopicPath()); clientMessage.mutable_init_request()->set_message_group_id("test-group-id"); AssertSuccessfullStreamingOperation(stream->Write(clientMessage), stream, &clientMessage); AssertSuccessfullStreamingOperation(stream->Read(&serverMessage), stream); @@ -109,22 +109,22 @@ Y_UNIT_TEST_SUITE(TPersQueueCommonTest) { } } - Y_UNIT_TEST( - Auth_WriteSessionWithValidTokenAndACEAndThenRemoveACEAndSendWriteRequest_SessionClosedWithUnauthorizedErrorAfterSuccessfullWriteResponse - ) { - TPersQueueV1TestServer server; - + Y_UNIT_TEST( + Auth_WriteSessionWithValidTokenAndACEAndThenRemoveACEAndSendWriteRequest_SessionClosedWithUnauthorizedErrorAfterSuccessfullWriteResponse + ) { + TPersQueueV1TestServer server; + //setup.GetPQConfig().SetACLRetryTimeoutSec(0); NACLib::TDiffACL acl; const auto token = GenerateValidToken(); acl.AddAccess(NACLib::EAccessType::Allow, NACLib::UpdateRow, token); - server.ModifyTopicACL(server.GetTopic(), acl); + server.ModifyTopicACL(server.GetTopic(), acl); - MAKE_WRITE_STREAM(token); + MAKE_WRITE_STREAM(token); StreamingWriteClientMessage clientMessage; StreamingWriteServerMessage serverMessage; - clientMessage.mutable_init_request()->set_topic(server.GetTopicPath()); + clientMessage.mutable_init_request()->set_topic(server.GetTopicPath()); clientMessage.mutable_init_request()->set_message_group_id("test-group-id"); AssertSuccessfullStreamingOperation(stream->Write(clientMessage), stream, &clientMessage); AssertSuccessfullStreamingOperation(stream->Read(&serverMessage), stream); @@ -132,7 +132,7 @@ Y_UNIT_TEST_SUITE(TPersQueueCommonTest) { serverMessage); acl.ClearAccess(); - server.ModifyTopicACL(server.GetTopic(), acl); + server.ModifyTopicACL(server.GetTopic(), acl); clientMessage = StreamingWriteClientMessage(); auto *writeRequest = clientMessage.mutable_write_request(); @@ -158,23 +158,23 @@ Y_UNIT_TEST_SUITE(TPersQueueCommonTest) { } // TODO: Replace this test with a unit-test of TWriteSessionActor - Y_UNIT_TEST( - Auth_MultipleInflightWriteUpdateTokenRequestWithDifferentValidToken_SessionClosedWithOverloadedError - ) { - TPersQueueV1TestServer server; - SET_LOCALS; + Y_UNIT_TEST( + Auth_MultipleInflightWriteUpdateTokenRequestWithDifferentValidToken_SessionClosedWithOverloadedError + ) { + TPersQueueV1TestServer server; + SET_LOCALS; const int iterations = 3; NACLib::TDiffACL acl; for (int i = 0; i != iterations; ++i) { acl.AddAccess(NACLib::EAccessType::Allow, NACLib::UpdateRow, GenerateValidToken(i)); } - server.ModifyTopicACL(server.GetTopic(), acl); - MAKE_WRITE_STREAM(GenerateValidToken(0)); + server.ModifyTopicACL(server.GetTopic(), acl); + MAKE_WRITE_STREAM(GenerateValidToken(0)); StreamingWriteClientMessage clientMessage; StreamingWriteServerMessage serverMessage; - clientMessage.mutable_init_request()->set_topic(server.GetTopicPath()); + clientMessage.mutable_init_request()->set_topic(server.GetTopicPath()); clientMessage.mutable_init_request()->set_message_group_id("test-group-id"); AssertSuccessfullStreamingOperation(stream->Write(clientMessage), stream, &clientMessage); AssertSuccessfullStreamingOperation(stream->Read(&serverMessage), stream); @@ -183,9 +183,9 @@ Y_UNIT_TEST_SUITE(TPersQueueCommonTest) { // TWriteSessionActor uses GRpcRequestProxy for authentication. This will make next update token procedure stuck indefinetely auto noopActorID = TActorId(); - for (size_t i = 0; i != runtime->GetNodeCount(); ++i) { + for (size_t i = 0; i != runtime->GetNodeCount(); ++i) { // Destroy GRpcRequestProxy - runtime->RegisterService(NGRpcService::CreateGRpcRequestProxyId(), noopActorID); + runtime->RegisterService(NGRpcService::CreateGRpcRequestProxyId(), noopActorID); } clientMessage.mutable_update_token_request()->set_token(GenerateValidToken(1)); @@ -203,24 +203,24 @@ Y_UNIT_TEST_SUITE(TPersQueueCommonTest) { serverMessage); } - Y_UNIT_TEST(Auth_WriteUpdateTokenRequestWithInvalidToken_SessionClosedWithUnauthenticatedError) { - TPersQueueV1TestServer server; - SET_LOCALS; - runtime->GetAppData().PQConfig.SetRequireCredentialsInNewProtocol(true); + Y_UNIT_TEST(Auth_WriteUpdateTokenRequestWithInvalidToken_SessionClosedWithUnauthenticatedError) { + TPersQueueV1TestServer server; + SET_LOCALS; + runtime->GetAppData().PQConfig.SetRequireCredentialsInNewProtocol(true); const TString validToken = "test_user@" BUILTIN_ACL_DOMAIN; // TODO: Why test fails with 'BUILTIN_ACL_DOMAIN' as domain in invalid token? TVector<TString> invalidTokens = {TString(), "test_user", "test_user@invalid_domain"}; NACLib::TDiffACL acl; acl.AddAccess(NACLib::EAccessType::Allow, NACLib::UpdateRow, validToken); - server.ModifyTopicACL(server.GetTopic(), acl); + server.ModifyTopicACL(server.GetTopic(), acl); for (const auto &invalidToken : invalidTokens) { Cerr << "Invalid token under test is '" << invalidToken << "'" << Endl; - MAKE_WRITE_STREAM(validToken); + MAKE_WRITE_STREAM(validToken); StreamingWriteClientMessage clientMessage; StreamingWriteServerMessage serverMessage; - clientMessage.mutable_init_request()->set_topic(server.GetTopicPath()); + clientMessage.mutable_init_request()->set_topic(server.GetTopicPath()); clientMessage.mutable_init_request()->set_message_group_id("test-group-id"); AssertSuccessfullStreamingOperation(stream->Write(clientMessage), stream, &clientMessage); AssertSuccessfullStreamingOperation(stream->Read(&serverMessage), stream); @@ -238,9 +238,9 @@ Y_UNIT_TEST_SUITE(TPersQueueCommonTest) { } } - Y_UNIT_TEST(Auth_WriteUpdateTokenRequestWithValidTokenButWithoutACL_SessionClosedWithUnauthorizedError) { - TPersQueueV1TestServer server; - + Y_UNIT_TEST(Auth_WriteUpdateTokenRequestWithValidTokenButWithoutACL_SessionClosedWithUnauthorizedError) { + TPersQueueV1TestServer server; + const TString validToken = "test_user@" BUILTIN_ACL_DOMAIN; const TString invalidToken = "test_user_2@" @@ -248,13 +248,13 @@ Y_UNIT_TEST_SUITE(TPersQueueCommonTest) { NACLib::TDiffACL acl; acl.AddAccess(NACLib::EAccessType::Allow, NACLib::UpdateRow, validToken); - server.ModifyTopicACL(server.GetTopic(), acl); + server.ModifyTopicACL(server.GetTopic(), acl); - MAKE_WRITE_STREAM(validToken); + MAKE_WRITE_STREAM(validToken); StreamingWriteClientMessage clientMessage; StreamingWriteServerMessage serverMessage; - clientMessage.mutable_init_request()->set_topic(server.GetTopicPath()); + clientMessage.mutable_init_request()->set_topic(server.GetTopicPath()); clientMessage.mutable_init_request()->set_message_group_id("test-message-group"); AssertSuccessfullStreamingOperation(stream->Write(clientMessage), stream, &clientMessage); AssertSuccessfullStreamingOperation(stream->Read(&serverMessage), stream); @@ -272,7 +272,7 @@ Y_UNIT_TEST_SUITE(TPersQueueCommonTest) { serverMessage.server_message_case(), serverMessage); } - void TestWriteWithRateLimiter(TPersQueueV1TestServerWithRateLimiter& server) { + void TestWriteWithRateLimiter(TPersQueueV1TestServerWithRateLimiter& server) { const std::vector<TString> differentTopicPathsTypes = { "account1/topic", // without folder "account2/folder/topic", // with folder @@ -280,7 +280,7 @@ Y_UNIT_TEST_SUITE(TPersQueueCommonTest) { }; const TString data = TString("12345") * 100; for (const TString &topicPath : differentTopicPathsTypes) { - server.CreateTopicWithQuota(topicPath); + server.CreateTopicWithQuota(topicPath); auto driver = server.Server->AnnoyingClient->GetDriver(); @@ -293,22 +293,22 @@ Y_UNIT_TEST_SUITE(TPersQueueCommonTest) { } } - Y_UNIT_TEST(TestWriteWithRateLimiterWithBlobsRateLimit) { - TPersQueueV1TestServerWithRateLimiter server; - server.InitAll(NKikimrPQ::TPQConfig::TQuotingConfig::WRITTEN_BLOB_SIZE); - TestWriteWithRateLimiter(server); + Y_UNIT_TEST(TestWriteWithRateLimiterWithBlobsRateLimit) { + TPersQueueV1TestServerWithRateLimiter server; + server.InitAll(NKikimrPQ::TPQConfig::TQuotingConfig::WRITTEN_BLOB_SIZE); + TestWriteWithRateLimiter(server); } - Y_UNIT_TEST(TestWriteWithRateLimiterWithUserPayloadRateLimit) { - TPersQueueV1TestServerWithRateLimiter server; - server.InitAll(NKikimrPQ::TPQConfig::TQuotingConfig::USER_PAYLOAD_SIZE); - TestWriteWithRateLimiter(server); + Y_UNIT_TEST(TestWriteWithRateLimiterWithUserPayloadRateLimit) { + TPersQueueV1TestServerWithRateLimiter server; + server.InitAll(NKikimrPQ::TPQConfig::TQuotingConfig::USER_PAYLOAD_SIZE); + TestWriteWithRateLimiter(server); } - void TestRateLimiterLimitsWrite(TPersQueueV1TestServerWithRateLimiter& server) { + void TestRateLimiterLimitsWrite(TPersQueueV1TestServerWithRateLimiter& server) { const TString topicPath = "account/topic"; - server.CreateTopicWithQuota(topicPath, true, 100.0); + server.CreateTopicWithQuota(topicPath, true, 100.0); const TString data = TString("123") * 100; // 300 bytes // 3 seconds auto driver = server.Server->AnnoyingClient->GetDriver(); @@ -348,7 +348,7 @@ Y_UNIT_TEST_SUITE(TPersQueueCommonTest) { // Check write time with quota const TDuration writeTime = endWrite - startWrite; // in new scheme cache rate limiting is turned off - if (server.TenantModeEnabled()) { + if (server.TenantModeEnabled()) { UNIT_ASSERT_GE_C(TDuration::Seconds(3), writeTime, "Write time: " << writeTime); } else { UNIT_ASSERT_GE_C(writeTime, TDuration::Seconds(3), "Write time: " << writeTime); diff --git a/ydb/services/persqueue_v1/persqueue_new_schemecache_ut.cpp b/ydb/services/persqueue_v1/persqueue_new_schemecache_ut.cpp index 23e45478d1..8e209dbd9d 100644 --- a/ydb/services/persqueue_v1/persqueue_new_schemecache_ut.cpp +++ b/ydb/services/persqueue_v1/persqueue_new_schemecache_ut.cpp @@ -24,13 +24,13 @@ #include <ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/data_plane_helpers.h> namespace { - const static TString DEFAULT_TOPIC_NAME = "rt3.dc1--topic1"; - const static TString DEFAULT_TOPIC_PATH = "/Root/PQ/rt3.dc1--topic1"; - - const static TString SHORT_TOPIC_NAME = "topic1"; + const static TString DEFAULT_TOPIC_NAME = "rt3.dc1--topic1"; + const static TString DEFAULT_TOPIC_PATH = "/Root/PQ/rt3.dc1--topic1"; + + const static TString SHORT_TOPIC_NAME = "topic1"; } - + namespace NKikimr::NPersQueueTests { using namespace Tests; @@ -40,7 +40,7 @@ namespace NKikimr::NPersQueueTests { using namespace NThreading; using namespace NNetClassifier; using namespace NYdb::NPersQueue; - using namespace NPersQueue; + using namespace NPersQueue; NJson::TJsonValue GetCountersNewSchemeCache(ui16 port, const TString& counters, const TString& subsystem, const TString& topicPath) { TString escapedPath = "%2F" + JoinStrings(SplitString(topicPath, "/"), "%2F"); @@ -69,28 +69,28 @@ namespace NKikimr::NPersQueueTests { Y_UNIT_TEST_SUITE(TPersQueueNewSchemeCacheTest) { void PrepareForGrpcNoDC(TFlatMsgBusPQClient& annoyingClient) { - annoyingClient.SetNoConfigMode(); - annoyingClient.FullInit(); + annoyingClient.SetNoConfigMode(); + annoyingClient.FullInit(); annoyingClient.InitUserRegistry(); annoyingClient.MkDir("/Root", "account1"); annoyingClient.MkDir("/Root/PQ", "account1"); - annoyingClient.CreateTopicNoLegacy(DEFAULT_TOPIC_PATH, 5, false); - annoyingClient.CreateTopicNoLegacy("/Root/PQ/account1/topic1", 5, false); + annoyingClient.CreateTopicNoLegacy(DEFAULT_TOPIC_PATH, 5, false); + annoyingClient.CreateTopicNoLegacy("/Root/PQ/account1/topic1", 5, false); annoyingClient.CreateTopicNoLegacy("/Root/account2/topic2", 5); } Y_UNIT_TEST(CheckGrpcWriteNoDC) { - TTestServer server(false); - server.ServerSettings.PQConfig.SetTopicsAreFirstClassCitizen(true); + TTestServer server(false); + server.ServerSettings.PQConfig.SetTopicsAreFirstClassCitizen(true); - server.StartServer(); - server.EnableLogs({ - NKikimrServices::PQ_WRITE_PROXY, NKikimrServices::TX_PROXY_SCHEME_CACHE, - NKikimrServices::FLAT_TX_SCHEMESHARD, NKikimrServices::PQ_METACACHE} - ); - PrepareForGrpcNoDC(*server.AnnoyingClient); + server.StartServer(); + server.EnableLogs({ + NKikimrServices::PQ_WRITE_PROXY, NKikimrServices::TX_PROXY_SCHEME_CACHE, + NKikimrServices::FLAT_TX_SCHEMESHARD, NKikimrServices::PQ_METACACHE} + ); + PrepareForGrpcNoDC(*server.AnnoyingClient); - TPQDataWriter writer("source1", server, DEFAULT_TOPIC_PATH); + TPQDataWriter writer("source1", server, DEFAULT_TOPIC_PATH); writer.Write("/Root/account2/topic2", {"valuevaluevalue1"}, true, "topic1@" BUILTIN_ACL_DOMAIN); writer.Write("/Root/PQ/account1/topic1", {"valuevaluevalue1"}, true, "topic1@" BUILTIN_ACL_DOMAIN); @@ -98,7 +98,7 @@ namespace NKikimr::NPersQueueTests { NACLib::TDiffACL acl; acl.AddAccess(NACLib::EAccessType::Allow, NACLib::UpdateRow, "topic1@" BUILTIN_ACL_DOMAIN); server.AnnoyingClient->ModifyACL("/Root/account2", "topic2", acl.SerializeAsString()); - server.AnnoyingClient->ModifyACL("/Root/PQ/account1", "topic1", acl.SerializeAsString()); + server.AnnoyingClient->ModifyACL("/Root/PQ/account1", "topic1", acl.SerializeAsString()); WaitACLModification(); writer.Write("/Root/account2/topic2", {"valuevaluevalue1"}, false, "topic1@" BUILTIN_ACL_DOMAIN); @@ -109,17 +109,17 @@ namespace NKikimr::NPersQueueTests { } Y_UNIT_TEST(CheckGrpcReadNoDC) { - TTestServer server(false); - server.ServerSettings.PQConfig.SetTopicsAreFirstClassCitizen(true); - server.StartServer(); - server.EnableLogs({NKikimrServices::PQ_READ_PROXY, NKikimrServices::TX_PROXY_SCHEME_CACHE}); - PrepareForGrpcNoDC(*server.AnnoyingClient); - NYdb::TDriverConfig driverCfg; + TTestServer server(false); + server.ServerSettings.PQConfig.SetTopicsAreFirstClassCitizen(true); + server.StartServer(); + server.EnableLogs({NKikimrServices::PQ_READ_PROXY, NKikimrServices::TX_PROXY_SCHEME_CACHE}); + PrepareForGrpcNoDC(*server.AnnoyingClient); + NYdb::TDriverConfig driverCfg; driverCfg.SetEndpoint(TStringBuilder() << "localhost:" << server.GrpcPort).SetLog(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG)).SetDatabase("/Root"); - auto ydbDriver = MakeHolder<NYdb::TDriver>(driverCfg); - auto persQueueClient = MakeHolder<NYdb::NPersQueue::TPersQueueClient>(*ydbDriver); + auto ydbDriver = MakeHolder<NYdb::TDriver>(driverCfg); + auto persQueueClient = MakeHolder<NYdb::NPersQueue::TPersQueueClient>(*ydbDriver); { auto res = persQueueClient->AddReadRule("/Root/account2/topic2", TAddReadRuleSettings().ReadRule(TReadRuleSettings().ConsumerName("user1"))); @@ -165,25 +165,25 @@ namespace NKikimr::NPersQueueTests { Y_UNIT_TEST_SUITE(TPersqueueDataPlaneTestSuite) { - Y_UNIT_TEST(WriteSession) { - TPersQueueV1TestServer server(true); + Y_UNIT_TEST(WriteSession) { + TPersQueueV1TestServer server(true); TString topic = "/Root/account1/write_topic"; TString consumer = "consumer_aba"; { - auto res = server.PersQueueClient->CreateTopic(topic); + auto res = server.PersQueueClient->CreateTopic(topic); res.Wait(); UNIT_ASSERT(res.GetValue().IsSuccess()); } { - auto res = server.PersQueueClient->AddReadRule(topic, TAddReadRuleSettings().ReadRule(TReadRuleSettings().ConsumerName(consumer))); + auto res = server.PersQueueClient->AddReadRule(topic, TAddReadRuleSettings().ReadRule(TReadRuleSettings().ConsumerName(consumer))); res.Wait(); UNIT_ASSERT(res.GetValue().IsSuccess()); } { - auto writer = server.PersQueueClient->CreateSimpleBlockingWriteSession(TWriteSessionSettings() + auto writer = server.PersQueueClient->CreateSimpleBlockingWriteSession(TWriteSessionSettings() .Path(topic).MessageGroupId("my_group_1") .ClusterDiscoveryMode(EClusterDiscoveryMode::Off) .RetryPolicy(IRetryPolicy::GetNoRetryPolicy())); @@ -192,7 +192,7 @@ namespace NKikimr::NPersQueueTests { writer->Close(); } { - auto reader = server.PersQueueClient->CreateReadSession(TReadSessionSettings().ConsumerName("non_existing") + auto reader = server.PersQueueClient->CreateReadSession(TReadSessionSettings().ConsumerName("non_existing") .AppendTopics(topic).DisableClusterDiscovery(true) .RetryPolicy(IRetryPolicy::GetNoRetryPolicy())); @@ -209,7 +209,7 @@ namespace NKikimr::NPersQueueTests { UNIT_ASSERT(std::get_if<TSessionClosedEvent>(&*event)); } { - auto reader = server.PersQueueClient->CreateReadSession(TReadSessionSettings().ConsumerName(consumer) + auto reader = server.PersQueueClient->CreateReadSession(TReadSessionSettings().ConsumerName(consumer) .AppendTopics(topic).DisableClusterDiscovery(true) .RetryPolicy(IRetryPolicy::GetNoRetryPolicy())); @@ -229,18 +229,18 @@ namespace NKikimr::NPersQueueTests { } Y_UNIT_TEST_SUITE(TPersqueueControlPlaneTestSuite) { - Y_UNIT_TEST(SetupReadLockSessionWithDatabase) { - TPersQueueV1TestServer server; + Y_UNIT_TEST(SetupReadLockSessionWithDatabase) { + TPersQueueV1TestServer server; { - auto res = server.PersQueueClient->AddReadRule("/Root/acc/topic1", TAddReadRuleSettings().ReadRule(TReadRuleSettings().ConsumerName("user"))); + auto res = server.PersQueueClient->AddReadRule("/Root/acc/topic1", TAddReadRuleSettings().ReadRule(TReadRuleSettings().ConsumerName("user"))); res.Wait(); Cerr << "ADD RESULT " << res.GetValue().GetIssues().ToString() << "\n"; UNIT_ASSERT(res.GetValue().IsSuccess()); } - auto stub = Ydb::PersQueue::V1::PersQueueService::NewStub(server.InsecureChannel); + auto stub = Ydb::PersQueue::V1::PersQueueService::NewStub(server.InsecureChannel); grpc::ClientContext grpcContext; grpcContext.AddMetadata("x-ydb-database", "/Root/acc"); auto readStream = stub->MigrationStreamingRead(&grpcContext); @@ -265,10 +265,10 @@ namespace NKikimr::NPersQueueTests { } } - Y_UNIT_TEST(SetupWriteLockSessionWithDatabase) { - TPersQueueV1TestServer server; - - auto stub = Ydb::PersQueue::V1::PersQueueService::NewStub(server.InsecureChannel); + Y_UNIT_TEST(SetupWriteLockSessionWithDatabase) { + TPersQueueV1TestServer server; + + auto stub = Ydb::PersQueue::V1::PersQueueService::NewStub(server.InsecureChannel); grpc::ClientContext grpcContext; grpcContext.AddMetadata("x-ydb-database", "/Root/acc"); @@ -290,19 +290,19 @@ namespace NKikimr::NPersQueueTests { } } - Y_UNIT_TEST(TestAddRemoveReadRule) { - TPersQueueV1TestServer server; - SET_LOCALS; + Y_UNIT_TEST(TestAddRemoveReadRule) { + TPersQueueV1TestServer server; + SET_LOCALS; + + pqClient->CreateConsumer("goodUser"); - pqClient->CreateConsumer("goodUser"); + TString path = server.GetTopicPath(); - TString path = server.GetTopicPath(); - Ydb::PersQueue::V1::AddReadRuleRequest addRuleRequest; Ydb::PersQueue::V1::AddReadRuleResponse addRuleResponse; addRuleRequest.set_path(path); - auto stub = Ydb::PersQueue::V1::PersQueueService::NewStub(server.InsecureChannel); + auto stub = Ydb::PersQueue::V1::PersQueueService::NewStub(server.InsecureChannel); { grpc::ClientContext grpcContext; grpcContext.AddMetadata("x-ydb-database", "/Root/acc"); diff --git a/ydb/services/persqueue_v1/persqueue_ut.cpp b/ydb/services/persqueue_v1/persqueue_ut.cpp index f795097918..3bc9b21728 100644 --- a/ydb/services/persqueue_v1/persqueue_ut.cpp +++ b/ydb/services/persqueue_v1/persqueue_ut.cpp @@ -78,27 +78,27 @@ static TString FormNetData() { "::1/128\tVLA\n"; } -namespace { - const static TString DEFAULT_TOPIC_NAME = "rt3.dc1--topic1"; - const static TString SHORT_TOPIC_NAME = "topic1"; -} - -#define MAKE_INSECURE_STUB \ - std::shared_ptr<grpc::Channel> Channel_; \ - std::unique_ptr<Ydb::PersQueue::V1::PersQueueService::Stub> StubP_; \ - \ - { \ - Channel_ = grpc::CreateChannel( \ - "localhost:" + ToString(server.Server->GrpcPort), \ - grpc::InsecureChannelCredentials() \ - ); \ - StubP_ = Ydb::PersQueue::V1::PersQueueService::NewStub(Channel_); \ - } \ - grpc::ClientContext rcontext; - - - - Y_UNIT_TEST_SUITE(TPersQueueTest) { +namespace { + const static TString DEFAULT_TOPIC_NAME = "rt3.dc1--topic1"; + const static TString SHORT_TOPIC_NAME = "topic1"; +} + +#define MAKE_INSECURE_STUB \ + std::shared_ptr<grpc::Channel> Channel_; \ + std::unique_ptr<Ydb::PersQueue::V1::PersQueueService::Stub> StubP_; \ + \ + { \ + Channel_ = grpc::CreateChannel( \ + "localhost:" + ToString(server.Server->GrpcPort), \ + grpc::InsecureChannelCredentials() \ + ); \ + StubP_ = Ydb::PersQueue::V1::PersQueueService::NewStub(Channel_); \ + } \ + grpc::ClientContext rcontext; + + + + Y_UNIT_TEST_SUITE(TPersQueueTest) { Y_UNIT_TEST(AllEqual) { using NGRpcProxy::V1::AllEqual; @@ -115,10 +115,10 @@ namespace { UNIT_ASSERT(!AllEqual(1, 1, 1, 1, 1, 0)); } - Y_UNIT_TEST(SetupLockSession2) { - TPersQueueV1TestServer server; - SET_LOCALS; - const TString topicPath = server.GetTopicPathMultipleDC(); + Y_UNIT_TEST(SetupLockSession2) { + TPersQueueV1TestServer server; + SET_LOCALS; + const TString topicPath = server.GetTopicPathMultipleDC(); auto driver = server.Server->AnnoyingClient->GetDriver(); @@ -130,7 +130,7 @@ namespace { auto msg = reader->GetEvent(true, 1); UNIT_ASSERT(msg); - Cerr << "Got message: " << NYdb::NPersQueue::DebugString(*msg) << "\n"; + Cerr << "Got message: " << NYdb::NPersQueue::DebugString(*msg) << "\n"; auto ev = std::get_if<NYdb::NPersQueue::TReadSessionEvent::TCreatePartitionStreamEvent>(&*msg); @@ -144,8 +144,8 @@ namespace { auto wait = reader->WaitEvent(); UNIT_ASSERT(!wait.Wait(TDuration::Seconds(1))); - pqClient->AlterTopicNoLegacy("/Root/PQ/rt3.dc2--acc--topic2dc", 2); - Cerr << "======Alter topic done\n"; + pqClient->AlterTopicNoLegacy("/Root/PQ/rt3.dc2--acc--topic2dc", 2); + Cerr << "======Alter topic done\n"; UNIT_ASSERT(wait.Wait(TDuration::Seconds(5))); auto msg = reader->GetEvent(true, 1); @@ -163,10 +163,10 @@ namespace { } - Y_UNIT_TEST(SetupLockSession) { - TPersQueueV1TestServer server; - SET_LOCALS; - MAKE_INSECURE_STUB; + Y_UNIT_TEST(SetupLockSession) { + TPersQueueV1TestServer server; + SET_LOCALS; + MAKE_INSECURE_STUB; auto readStream = StubP_->MigrationStreamingRead(&rcontext); UNIT_ASSERT(readStream); @@ -178,14 +178,14 @@ namespace { req.mutable_init_request()->add_topics_read_settings()->set_topic("acc/topic1"); req.mutable_init_request()->set_consumer("user"); - req.mutable_init_request()->set_read_only_original(true); + req.mutable_init_request()->set_read_only_original(true); req.mutable_init_request()->mutable_read_params()->set_max_read_messages_count(1); if (!readStream->Write(req)) { ythrow yexception() << "write fail"; } UNIT_ASSERT(readStream->Read(&resp)); - Cerr << "===Got response: " << resp.ShortDebugString() << Endl; + Cerr << "===Got response: " << resp.ShortDebugString() << Endl; UNIT_ASSERT(resp.response_case() == MigrationStreamingReadServerMessage::kInitResponse); //send some reads req.Clear(); @@ -198,19 +198,19 @@ namespace { } { - Sleep(TDuration::Seconds(10)); + Sleep(TDuration::Seconds(10)); ReadInfoRequest request; ReadInfoResponse response; request.mutable_consumer()->set_path("user"); - request.set_get_only_original(true); + request.set_get_only_original(true); request.add_topics()->set_path("acc/topic1"); grpc::ClientContext rcontext; auto status = StubP_->GetReadSessionsInfo(&rcontext, request, &response); UNIT_ASSERT(status.ok()); ReadInfoResult res; response.operation().result().UnpackTo(&res); - Cerr << "Read info response: " << response << Endl << res << Endl; - UNIT_ASSERT_VALUES_EQUAL(res.topics_size(), 1); + Cerr << "Read info response: " << response << Endl << res << Endl; + UNIT_ASSERT_VALUES_EQUAL(res.topics_size(), 1); UNIT_ASSERT(res.topics(0).status() == Ydb::StatusIds::SUCCESS); } ui64 assignId = 0; @@ -279,7 +279,7 @@ namespace { } - pqClient->AlterTopic("rt3.dc1--acc--topic1", 10); + pqClient->AlterTopic("rt3.dc1--acc--topic1", 10); { ReadInfoRequest request; ReadInfoResponse response; @@ -291,10 +291,10 @@ namespace { UNIT_ASSERT(status.ok()); ReadInfoResult res; response.operation().result().UnpackTo(&res); - Cerr << "Get read session info response: " << response << "\n" << res << "\n"; + Cerr << "Get read session info response: " << response << "\n" << res << "\n"; // UNIT_ASSERT(res.sessions_size() == 1); - UNIT_ASSERT_VALUES_EQUAL(res.topics_size(), 1); - UNIT_ASSERT_VALUES_EQUAL(res.topics(0).partitions_size(), 10); + UNIT_ASSERT_VALUES_EQUAL(res.topics_size(), 1); + UNIT_ASSERT_VALUES_EQUAL(res.topics(0).partitions_size(), 10); } { @@ -305,39 +305,39 @@ namespace { request.add_topics()->set_path("acc/topic1"); grpc::ClientContext rcontext; - pqClient->MarkNodeInHive(runtime, 0, false); - pqClient->MarkNodeInHive(runtime, 1, false); + pqClient->MarkNodeInHive(runtime, 0, false); + pqClient->MarkNodeInHive(runtime, 1, false); - pqClient->RestartBalancerTablet(runtime, "rt3.dc1--acc--topic1"); + pqClient->RestartBalancerTablet(runtime, "rt3.dc1--acc--topic1"); auto status = StubP_->GetReadSessionsInfo(&rcontext, request, &response); UNIT_ASSERT(status.ok()); ReadInfoResult res; response.operation().result().UnpackTo(&res); - Cerr << "Read sessions info response: " << response << "\nResult: " << res << "\n"; + Cerr << "Read sessions info response: " << response << "\nResult: " << res << "\n"; UNIT_ASSERT(res.topics().size() == 1); - UNIT_ASSERT(res.topics(0).partitions(0).status() == Ydb::StatusIds::UNAVAILABLE); + UNIT_ASSERT(res.topics(0).partitions(0).status() == Ydb::StatusIds::UNAVAILABLE); } } void SetupWriteSessionImpl(bool rr) { - NPersQueue::TTestServer server{PQSettings(0, 2, rr), false}; - server.ServerSettings.SetEnableSystemViews(false); - server.StartServer(); + NPersQueue::TTestServer server{PQSettings(0, 2, rr), false}; + server.ServerSettings.SetEnableSystemViews(false); + server.StartServer(); - server.EnableLogs({ NKikimrServices::PERSQUEUE }); - server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 10); + server.EnableLogs({ NKikimrServices::PERSQUEUE }); + server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 10); - TPQDataWriter writer("source", server); + TPQDataWriter writer("source", server); - ui32 p = writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue1"}); + ui32 p = writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue1"}); - server.AnnoyingClient->AlterTopic(DEFAULT_TOPIC_NAME, 15); + server.AnnoyingClient->AlterTopic(DEFAULT_TOPIC_NAME, 15); - ui32 pp = writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue2"}); + ui32 pp = writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue2"}); UNIT_ASSERT_VALUES_EQUAL(p, pp); - writer.Write(SHORT_TOPIC_NAME, {"1", "2", "3", "4", "5"}); + writer.Write(SHORT_TOPIC_NAME, {"1", "2", "3", "4", "5"}); writer.Write("topic2", {"valuevaluevalue1"}, true); @@ -377,9 +377,9 @@ namespace { Y_UNIT_TEST(StoreNoMoreThanXSourceIDs) { ui16 X = 4; ui64 SOURCEID_COUNT_DELETE_BATCH_SIZE = 100; - NPersQueue::TTestServer server; - server.EnableLogs({ NKikimrServices::PERSQUEUE, NKikimrServices::PQ_WRITE_PROXY }); - server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1, 8*1024*1024, 86400, 20000000, "", 200000000, {}, {}, {}, X, 86400); + NPersQueue::TTestServer server; + server.EnableLogs({ NKikimrServices::PERSQUEUE, NKikimrServices::PQ_WRITE_PROXY }); + server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1, 8*1024*1024, 86400, 20000000, "", 200000000, {}, {}, {}, X, 86400); auto driver = server.AnnoyingClient->GetDriver(); @@ -429,8 +429,8 @@ namespace { } Y_UNIT_TEST(EachMessageGetsExactlyOneAcknowledgementInCorrectOrder) { - NPersQueue::TTestServer server; - server.AnnoyingClient->CreateTopic("rt3.dc1--topic", 1); + NPersQueue::TTestServer server; + server.AnnoyingClient->CreateTopic("rt3.dc1--topic", 1); auto driver = server.AnnoyingClient->GetDriver(); @@ -449,24 +449,24 @@ namespace { UNIT_ASSERT(res); } - Y_UNIT_TEST(SetupWriteSessionOnDisabledCluster) { - TPersQueueV1TestServer server; - SET_LOCALS; + Y_UNIT_TEST(SetupWriteSessionOnDisabledCluster) { + TPersQueueV1TestServer server; + SET_LOCALS; - TPQDataWriter writer("source", *server.Server); + TPQDataWriter writer("source", *server.Server); - pqClient->DisableDC(); + pqClient->DisableDC(); Sleep(TDuration::Seconds(5)); - writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue1"}, true); + writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue1"}, true); } Y_UNIT_TEST(CloseActiveWriteSessionOnClusterDisable) { - NPersQueue::TTestServer server; + NPersQueue::TTestServer server; - server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 10); + server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 10); - server.EnableLogs({ NKikimrServices::PQ_WRITE_PROXY }); + server.EnableLogs({ NKikimrServices::PQ_WRITE_PROXY }); TPQDataWriter writer2("source", server); @@ -480,7 +480,7 @@ namespace { Cerr << DebugString(*msg) << "\n"; - server.AnnoyingClient->DisableDC(); + server.AnnoyingClient->DisableDC(); UNIT_ASSERT(writer->WaitEvent().Wait(TDuration::Seconds(30))); msg = writer->GetEvent(true); @@ -498,11 +498,11 @@ namespace { } Y_UNIT_TEST(BadSids) { - NPersQueue::TTestServer server; - server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 10); - server.EnableLogs({ NKikimrServices::PQ_WRITE_PROXY }); + NPersQueue::TTestServer server; + server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 10); + server.EnableLogs({ NKikimrServices::PQ_WRITE_PROXY }); TPQDataWriter writer2("source", server); - TString topic = SHORT_TOPIC_NAME; + TString topic = SHORT_TOPIC_NAME; auto driver = server.AnnoyingClient->GetDriver(); @@ -518,26 +518,26 @@ namespace { } Y_UNIT_TEST(ReadFromSeveralPartitions) { - NPersQueue::TTestServer server; - server.EnableLogs({ NKikimrServices::PQ_READ_PROXY, NKikimrServices::PQ_METACACHE }); + NPersQueue::TTestServer server; + server.EnableLogs({ NKikimrServices::PQ_READ_PROXY, NKikimrServices::PQ_METACACHE }); - server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 10); + server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 10); - TPQDataWriter writer("source1", server); - Cerr << "===Writer started\n"; + TPQDataWriter writer("source1", server); + Cerr << "===Writer started\n"; std::shared_ptr<grpc::Channel> Channel_; std::unique_ptr<Ydb::PersQueue::V1::PersQueueService::Stub> StubP_; - Channel_ = grpc::CreateChannel("localhost:" + ToString(server.GrpcPort), grpc::InsecureChannelCredentials()); + Channel_ = grpc::CreateChannel("localhost:" + ToString(server.GrpcPort), grpc::InsecureChannelCredentials()); StubP_ = Ydb::PersQueue::V1::PersQueueService::NewStub(Channel_); //Write some data - writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue1"}); + writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue1"}); - TPQDataWriter writer2("source2", server); - writer2.Write(SHORT_TOPIC_NAME, {"valuevaluevalue2"}); - Cerr << "===Writer - writes done\n"; + TPQDataWriter writer2("source2", server); + writer2.Write(SHORT_TOPIC_NAME, {"valuevaluevalue2"}); + Cerr << "===Writer - writes done\n"; grpc::ClientContext rcontext; auto readStream = StubP_->MigrationStreamingRead(&rcontext); @@ -548,20 +548,20 @@ namespace { MigrationStreamingReadClientMessage req; MigrationStreamingReadServerMessage resp; - req.mutable_init_request()->add_topics_read_settings()->set_topic(SHORT_TOPIC_NAME); + req.mutable_init_request()->add_topics_read_settings()->set_topic(SHORT_TOPIC_NAME); req.mutable_init_request()->set_consumer("user"); - req.mutable_init_request()->set_read_only_original(true); - + req.mutable_init_request()->set_read_only_original(true); + req.mutable_init_request()->mutable_read_params()->set_max_read_messages_count(1000); if (!readStream->Write(req)) { ythrow yexception() << "write fail"; } - Cerr << "===Try to get read response\n"; - + Cerr << "===Try to get read response\n"; + UNIT_ASSERT(readStream->Read(&resp)); - Cerr << "Read server response: " << resp.ShortDebugString() << Endl; + Cerr << "Read server response: " << resp.ShortDebugString() << Endl; UNIT_ASSERT(resp.response_case() == MigrationStreamingReadServerMessage::kInitResponse); //send some reads @@ -584,7 +584,7 @@ namespace { if (resp.response_case() == MigrationStreamingReadServerMessage::kAssigned) { auto assignId = resp.assigned().assign_id(); MigrationStreamingReadClientMessage req; - req.mutable_start_read()->mutable_topic()->set_path(SHORT_TOPIC_NAME); + req.mutable_start_read()->mutable_topic()->set_path(SHORT_TOPIC_NAME); req.mutable_start_read()->set_cluster("dc1"); req.mutable_start_read()->set_assign_id(assignId); UNIT_ASSERT(readStream->Write(req)); @@ -598,26 +598,26 @@ namespace { void SetupReadSessionTest() { - NPersQueue::TTestServer server; - server.EnableLogs({ NKikimrServices::PQ_READ_PROXY }); + NPersQueue::TTestServer server; + server.EnableLogs({ NKikimrServices::PQ_READ_PROXY }); - server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 2); + server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 2); server.AnnoyingClient->CreateTopicNoLegacy("rt3.dc2--topic1", 2, true, false); - TPQDataWriter writer("source1", server); + TPQDataWriter writer("source1", server); - writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue0"}); - writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue1"}); - writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue2"}); - writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue3"}); - writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue4"}); - writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue5"}); - writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue6"}); - writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue7"}); - writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue8"}); - writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue9"}); + writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue0"}); + writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue1"}); + writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue2"}); + writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue3"}); + writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue4"}); + writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue5"}); + writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue6"}); + writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue7"}); + writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue8"}); + writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue9"}); - writer.Read(SHORT_TOPIC_NAME, "user1", "", false, false); + writer.Read(SHORT_TOPIC_NAME, "user1", "", false, false); } Y_UNIT_TEST(SetupReadSession) { @@ -626,17 +626,17 @@ namespace { Y_UNIT_TEST(WriteExisting) { - NPersQueue::TTestServer server; - server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD, NKikimrServices::PERSQUEUE }); + NPersQueue::TTestServer server; + server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD, NKikimrServices::PERSQUEUE }); - server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 2); + server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 2); { THolder<NMsgBusProxy::TBusPersQueue> request = TRequestDescribePQ().GetRequest({}); NKikimrClient::TResponse response; - auto channel = grpc::CreateChannel("localhost:"+ToString(server.GrpcPort), grpc::InsecureChannelCredentials()); + auto channel = grpc::CreateChannel("localhost:"+ToString(server.GrpcPort), grpc::InsecureChannelCredentials()); auto stub(NKikimrClient::TGRpcServer::NewStub(channel)); grpc::ClientContext context; auto status = stub->PersQueueRequest(&context, request->Record, &response); @@ -644,177 +644,177 @@ namespace { UNIT_ASSERT(status.ok()); } - server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 1, "abacaba", 1, "valuevaluevalue1", "", ETransport::GRpc); - server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 1, "abacaba", 2, "valuevaluevalue1", "", ETransport::GRpc); + server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 1, "abacaba", 1, "valuevaluevalue1", "", ETransport::GRpc); + server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 1, "abacaba", 2, "valuevaluevalue1", "", ETransport::GRpc); } Y_UNIT_TEST(WriteExistingBigValue) { - NPersQueue::TTestServer server(PQSettings(0).SetDomainName("Root").SetNodeCount(2)); - server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD, NKikimrServices::PERSQUEUE }); - server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 2, 8*1024*1024, 86400, 100000); + NPersQueue::TTestServer server(PQSettings(0).SetDomainName("Root").SetNodeCount(2)); + server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD, NKikimrServices::PERSQUEUE }); + server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 2, 8*1024*1024, 86400, 100000); TInstant now(Now()); - server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 1, "abacaba", 1, TString(1000000, 'a')); - server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 1, "abacaba", 2, TString(1, 'a')); + server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 1, "abacaba", 1, TString(1000000, 'a')); + server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 1, "abacaba", 2, TString(1, 'a')); UNIT_ASSERT(TInstant::Now() - now > TDuration::MilliSeconds(5990)); //speed limit is 200kb/s and burst is 200kb, so to write 1mb it will take at least 4 seconds } Y_UNIT_TEST(WriteEmptyData) { - NPersQueue::TTestServer server{PQSettings(0).SetDomainName("Root").SetNodeCount(2)}; + NPersQueue::TTestServer server{PQSettings(0).SetDomainName("Root").SetNodeCount(2)}; - server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 2); + server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 2); - server.EnableLogs({ NKikimrServices::PERSQUEUE }); + server.EnableLogs({ NKikimrServices::PERSQUEUE }); // empty data and sourceId - server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 1, "", 1, "", "", ETransport::MsgBus, NMsgBusProxy::MSTATUS_ERROR); - server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 1, "a", 1, "", "", ETransport::MsgBus, NMsgBusProxy::MSTATUS_ERROR); - server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 1, "", 1, "a", "", ETransport::MsgBus, NMsgBusProxy::MSTATUS_ERROR); - server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 1, "a", 1, "a", "", ETransport::MsgBus, NMsgBusProxy::MSTATUS_OK); + server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 1, "", 1, "", "", ETransport::MsgBus, NMsgBusProxy::MSTATUS_ERROR); + server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 1, "a", 1, "", "", ETransport::MsgBus, NMsgBusProxy::MSTATUS_ERROR); + server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 1, "", 1, "a", "", ETransport::MsgBus, NMsgBusProxy::MSTATUS_ERROR); + server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 1, "a", 1, "a", "", ETransport::MsgBus, NMsgBusProxy::MSTATUS_OK); } Y_UNIT_TEST(WriteNonExistingPartition) { - NPersQueue::TTestServer server{PQSettings(0).SetDomainName("Root").SetNodeCount(2)}; - server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 2); + NPersQueue::TTestServer server{PQSettings(0).SetDomainName("Root").SetNodeCount(2)}; + server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 2); - server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD, NKikimrServices::PERSQUEUE }); + server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD, NKikimrServices::PERSQUEUE }); - server.AnnoyingClient->WriteToPQ( - DEFAULT_TOPIC_NAME, 100500, "abacaba", 1, "valuevaluevalue1", "", - ETransport::MsgBus, NMsgBusProxy::MSTATUS_ERROR, NMsgBusProxy::MSTATUS_ERROR - ); + server.AnnoyingClient->WriteToPQ( + DEFAULT_TOPIC_NAME, 100500, "abacaba", 1, "valuevaluevalue1", "", + ETransport::MsgBus, NMsgBusProxy::MSTATUS_ERROR, NMsgBusProxy::MSTATUS_ERROR + ); } Y_UNIT_TEST(WriteNonExistingTopic) { - NPersQueue::TTestServer server(PQSettings(0).SetDomainName("Root").SetNodeCount(2)); - server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 2); - server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD, NKikimrServices::PERSQUEUE }); + NPersQueue::TTestServer server(PQSettings(0).SetDomainName("Root").SetNodeCount(2)); + server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 2); + server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD, NKikimrServices::PERSQUEUE }); - server.AnnoyingClient->WriteToPQ( - DEFAULT_TOPIC_NAME + "000", 1, "abacaba", 1, "valuevaluevalue1", "", - ETransport::MsgBus, NMsgBusProxy::MSTATUS_ERROR, NMsgBusProxy::MSTATUS_ERROR - ); + server.AnnoyingClient->WriteToPQ( + DEFAULT_TOPIC_NAME + "000", 1, "abacaba", 1, "valuevaluevalue1", "", + ETransport::MsgBus, NMsgBusProxy::MSTATUS_ERROR, NMsgBusProxy::MSTATUS_ERROR + ); } Y_UNIT_TEST(SchemeshardRestart) { - NPersQueue::TTestServer server(PQSettings(0).SetDomainName("Root").SetNodeCount(1)); - server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 2); - TString secondTopic = "rt3.dc1--topic2"; - server.AnnoyingClient->CreateTopic(secondTopic, 2); + NPersQueue::TTestServer server(PQSettings(0).SetDomainName("Root").SetNodeCount(1)); + server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 2); + TString secondTopic = "rt3.dc1--topic2"; + server.AnnoyingClient->CreateTopic(secondTopic, 2); // force topic1 into cache and establish pipe from cache to schemeshard - server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 1, "abacaba", 1, "valuevaluevalue1"); + server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 1, "abacaba", 1, "valuevaluevalue1"); - server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD, + server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD, NKikimrServices::PERSQUEUE, NKikimrServices::PQ_METACACHE }); - server.AnnoyingClient->RestartSchemeshard(server.CleverServer->GetRuntime()); + server.AnnoyingClient->RestartSchemeshard(server.CleverServer->GetRuntime()); - server.AnnoyingClient->WriteToPQ(secondTopic, 1, "abacaba", 1, "valuevaluevalue1"); + server.AnnoyingClient->WriteToPQ(secondTopic, 1, "abacaba", 1, "valuevaluevalue1"); } Y_UNIT_TEST(WriteAfterAlter) { - NPersQueue::TTestServer server(PQSettings(0).SetDomainName("Root").SetNodeCount(2)); - server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 2); + NPersQueue::TTestServer server(PQSettings(0).SetDomainName("Root").SetNodeCount(2)); + server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 2); - server.AnnoyingClient->WriteToPQ( - DEFAULT_TOPIC_NAME, 5, "abacaba", 1, "valuevaluevalue1", "", - ETransport::MsgBus, NMsgBusProxy::MSTATUS_ERROR, NMsgBusProxy::MSTATUS_ERROR - ); + server.AnnoyingClient->WriteToPQ( + DEFAULT_TOPIC_NAME, 5, "abacaba", 1, "valuevaluevalue1", "", + ETransport::MsgBus, NMsgBusProxy::MSTATUS_ERROR, NMsgBusProxy::MSTATUS_ERROR + ); - server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD, + server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD, NKikimrServices::PERSQUEUE, NKikimrServices::PQ_METACACHE }); - server.AnnoyingClient->AlterTopic(DEFAULT_TOPIC_NAME, 10); + server.AnnoyingClient->AlterTopic(DEFAULT_TOPIC_NAME, 10); Sleep(TDuration::Seconds(1)); - server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 5, "abacaba", 1, "valuevaluevalue1"); - server.AnnoyingClient->WriteToPQ( - DEFAULT_TOPIC_NAME, 15, "abacaba", 1, "valuevaluevalue1", "", - ETransport::MsgBus, NMsgBusProxy::MSTATUS_ERROR, NMsgBusProxy::MSTATUS_ERROR - ); + server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 5, "abacaba", 1, "valuevaluevalue1"); + server.AnnoyingClient->WriteToPQ( + DEFAULT_TOPIC_NAME, 15, "abacaba", 1, "valuevaluevalue1", "", + ETransport::MsgBus, NMsgBusProxy::MSTATUS_ERROR, NMsgBusProxy::MSTATUS_ERROR + ); - server.AnnoyingClient->AlterTopic(DEFAULT_TOPIC_NAME, 20); + server.AnnoyingClient->AlterTopic(DEFAULT_TOPIC_NAME, 20); Sleep(TDuration::Seconds(1)); - server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 5, "abacaba", 1, "valuevaluevalue1"); - server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 15, "abacaba", 1, "valuevaluevalue1"); + server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 5, "abacaba", 1, "valuevaluevalue1"); + server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 15, "abacaba", 1, "valuevaluevalue1"); } Y_UNIT_TEST(Delete) { - NPersQueue::TTestServer server(PQSettings(0).SetDomainName("Root").SetNodeCount(2)); + NPersQueue::TTestServer server(PQSettings(0).SetDomainName("Root").SetNodeCount(2)); - server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD, NKikimrServices::PERSQUEUE}); + server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD, NKikimrServices::PERSQUEUE}); // Delete non-existing - server.AnnoyingClient->DeleteTopic2(DEFAULT_TOPIC_NAME, NPersQueue::NErrorCode::UNKNOWN_TOPIC); + server.AnnoyingClient->DeleteTopic2(DEFAULT_TOPIC_NAME, NPersQueue::NErrorCode::UNKNOWN_TOPIC); - server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 2); + server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 2); // Delete existing - server.AnnoyingClient->DeleteTopic2(DEFAULT_TOPIC_NAME); + server.AnnoyingClient->DeleteTopic2(DEFAULT_TOPIC_NAME); // Double delete - "What Is Dead May Never Die" - server.AnnoyingClient->DeleteTopic2(DEFAULT_TOPIC_NAME, NPersQueue::NErrorCode::UNKNOWN_TOPIC); + server.AnnoyingClient->DeleteTopic2(DEFAULT_TOPIC_NAME, NPersQueue::NErrorCode::UNKNOWN_TOPIC); // Resurrect deleted topic - server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 2); - server.AnnoyingClient->DeleteTopic2(DEFAULT_TOPIC_NAME); + server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 2); + server.AnnoyingClient->DeleteTopic2(DEFAULT_TOPIC_NAME); } Y_UNIT_TEST(BigRead) { - NPersQueue::TTestServer server(PQSettings(0).SetDomainName("Root")); + NPersQueue::TTestServer server(PQSettings(0).SetDomainName("Root")); server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1, 8*1024*1024, 86400, 20000000, "user1", 2000000); - server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD, NKikimrServices::PERSQUEUE }); + server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD, NKikimrServices::PERSQUEUE }); TString value(1024*1024, 'x'); for (ui32 i = 0; i < 32; ++i) - server.AnnoyingClient->WriteToPQ({DEFAULT_TOPIC_NAME, 0, "source1", i}, value); + server.AnnoyingClient->WriteToPQ({DEFAULT_TOPIC_NAME, 0, "source1", i}, value); // trying to read small PQ messages in a big messagebus event - auto info = server.AnnoyingClient->ReadFromPQ({DEFAULT_TOPIC_NAME, 0, 0, 32, "user1"}, 23, "", NMsgBusProxy::MSTATUS_OK); //will read 21mb + auto info = server.AnnoyingClient->ReadFromPQ({DEFAULT_TOPIC_NAME, 0, 0, 32, "user1"}, 23, "", NMsgBusProxy::MSTATUS_OK); //will read 21mb UNIT_ASSERT_VALUES_EQUAL(info.BlobsFromDisk, 0); UNIT_ASSERT_VALUES_EQUAL(info.BlobsFromCache, 4); TInstant now(TInstant::Now()); - info = server.AnnoyingClient->ReadFromPQ({DEFAULT_TOPIC_NAME, 0, 0, 32, "user1"}, 23, "", NMsgBusProxy::MSTATUS_OK); //will read 21mb + info = server.AnnoyingClient->ReadFromPQ({DEFAULT_TOPIC_NAME, 0, 0, 32, "user1"}, 23, "", NMsgBusProxy::MSTATUS_OK); //will read 21mb TDuration dur = TInstant::Now() - now; UNIT_ASSERT_C(dur > TDuration::Seconds(7) && dur < TDuration::Seconds(20), "dur = " << dur); //speed limit is 2000kb/s and burst is 2000kb, so to read 24mb it will take at least 11 seconds - server.AnnoyingClient->GetPartStatus({}, 1, true); + server.AnnoyingClient->GetPartStatus({}, 1, true); } // expects that L2 size is 32Mb Y_UNIT_TEST(Cache) { - NPersQueue::TTestServer server(PQSettings(0).SetDomainName("Root")); - server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1, 8*1024*1024, 86400); + NPersQueue::TTestServer server(PQSettings(0).SetDomainName("Root")); + server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1, 8*1024*1024, 86400); - server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD, NKikimrServices::PERSQUEUE }); + server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD, NKikimrServices::PERSQUEUE }); TString value(1024*1024, 'x'); for (ui32 i = 0; i < 32; ++i) - server.AnnoyingClient->WriteToPQ({DEFAULT_TOPIC_NAME, 0, "source1", i}, value); + server.AnnoyingClient->WriteToPQ({DEFAULT_TOPIC_NAME, 0, "source1", i}, value); - auto info0 = server.AnnoyingClient->ReadFromPQ({DEFAULT_TOPIC_NAME, 0, 0, 16, "user1"}, 16); - auto info16 = server.AnnoyingClient->ReadFromPQ({DEFAULT_TOPIC_NAME, 0, 16, 16, "user1"}, 16); + auto info0 = server.AnnoyingClient->ReadFromPQ({DEFAULT_TOPIC_NAME, 0, 0, 16, "user1"}, 16); + auto info16 = server.AnnoyingClient->ReadFromPQ({DEFAULT_TOPIC_NAME, 0, 16, 16, "user1"}, 16); UNIT_ASSERT_VALUES_EQUAL(info0.BlobsFromCache, 3); UNIT_ASSERT_VALUES_EQUAL(info16.BlobsFromCache, 2); UNIT_ASSERT_VALUES_EQUAL(info0.BlobsFromDisk + info16.BlobsFromDisk, 0); for (ui32 i = 0; i < 8; ++i) - server.AnnoyingClient->WriteToPQ({DEFAULT_TOPIC_NAME, 0, "source1", 32+i}, value); + server.AnnoyingClient->WriteToPQ({DEFAULT_TOPIC_NAME, 0, "source1", 32+i}, value); - info0 = server.AnnoyingClient->ReadFromPQ({DEFAULT_TOPIC_NAME, 0, 0, 16, "user1"}, 16); - info16 = server.AnnoyingClient->ReadFromPQ({DEFAULT_TOPIC_NAME, 0, 16, 16, "user1"}, 16); + info0 = server.AnnoyingClient->ReadFromPQ({DEFAULT_TOPIC_NAME, 0, 0, 16, "user1"}, 16); + info16 = server.AnnoyingClient->ReadFromPQ({DEFAULT_TOPIC_NAME, 0, 16, 16, "user1"}, 16); ui32 fromDisk = info0.BlobsFromDisk + info16.BlobsFromDisk; ui32 fromCache = info0.BlobsFromCache + info16.BlobsFromCache; @@ -825,10 +825,10 @@ namespace { } Y_UNIT_TEST(CacheHead) { - NPersQueue::TTestServer server(PQSettings(0).SetDomainName("Root")); - server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1, 6*1024*1024, 86400); + NPersQueue::TTestServer server(PQSettings(0).SetDomainName("Root")); + server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1, 6*1024*1024, 86400); - server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD, NKikimrServices::PERSQUEUE }); + server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD, NKikimrServices::PERSQUEUE }); ui64 seqNo = 0; for (ui32 blobSizeKB = 256; blobSizeKB < 4096; blobSizeKB *= 2) { @@ -841,10 +841,10 @@ namespace { ui64 offset = seqNo; TString value(blobSize, 'a'); for (ui32 i = 0; i < count; ++i) - server.AnnoyingClient->WriteToPQ({DEFAULT_TOPIC_NAME, 0, "source1", seqNo++}, value); + server.AnnoyingClient->WriteToPQ({DEFAULT_TOPIC_NAME, 0, "source1", seqNo++}, value); - auto info_half1 = server.AnnoyingClient->ReadFromPQ({DEFAULT_TOPIC_NAME, 0, offset, half, "user1"}, half); - auto info_half2 = server.AnnoyingClient->ReadFromPQ({DEFAULT_TOPIC_NAME, 0, offset, half, "user1"}, half); + auto info_half1 = server.AnnoyingClient->ReadFromPQ({DEFAULT_TOPIC_NAME, 0, offset, half, "user1"}, half); + auto info_half2 = server.AnnoyingClient->ReadFromPQ({DEFAULT_TOPIC_NAME, 0, offset, half, "user1"}, half); UNIT_ASSERT(info_half1.BlobsFromCache > 0); UNIT_ASSERT(info_half2.BlobsFromCache > 0); @@ -854,28 +854,28 @@ namespace { } Y_UNIT_TEST(SameOffset) { - NPersQueue::TTestServer server(PQSettings(0).SetDomainName("Root")); - server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1, 6*1024*1024, 86400); - TString secondTopic = DEFAULT_TOPIC_NAME + "2"; - server.AnnoyingClient->CreateTopic(secondTopic, 1, 6*1024*1024, 86400); + NPersQueue::TTestServer server(PQSettings(0).SetDomainName("Root")); + server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1, 6*1024*1024, 86400); + TString secondTopic = DEFAULT_TOPIC_NAME + "2"; + server.AnnoyingClient->CreateTopic(secondTopic, 1, 6*1024*1024, 86400); - server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD, NKikimrServices::PERSQUEUE }); + server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD, NKikimrServices::PERSQUEUE }); ui32 valueSize = 128; TString value1(valueSize, 'a'); TString value2(valueSize, 'b'); - server.AnnoyingClient->WriteToPQ({DEFAULT_TOPIC_NAME, 0, "source1", 0}, value1); - server.AnnoyingClient->WriteToPQ({secondTopic, 0, "source1", 0}, value2); + server.AnnoyingClient->WriteToPQ({DEFAULT_TOPIC_NAME, 0, "source1", 0}, value1); + server.AnnoyingClient->WriteToPQ({secondTopic, 0, "source1", 0}, value2); // avoid reading from head TString mb(1024*1024, 'x'); for (ui32 i = 1; i < 16; ++i) { - server.AnnoyingClient->WriteToPQ({DEFAULT_TOPIC_NAME, 0, "source1", i}, mb); - server.AnnoyingClient->WriteToPQ({secondTopic, 0, "source1", i}, mb); + server.AnnoyingClient->WriteToPQ({DEFAULT_TOPIC_NAME, 0, "source1", i}, mb); + server.AnnoyingClient->WriteToPQ({secondTopic, 0, "source1", i}, mb); } - auto info1 = server.AnnoyingClient->ReadFromPQ({DEFAULT_TOPIC_NAME, 0, 0, 1, "user1"}, 1); - auto info2 = server.AnnoyingClient->ReadFromPQ({secondTopic, 0, 0, 1, "user1"}, 1); + auto info1 = server.AnnoyingClient->ReadFromPQ({DEFAULT_TOPIC_NAME, 0, 0, 1, "user1"}, 1); + auto info2 = server.AnnoyingClient->ReadFromPQ({secondTopic, 0, 0, 1, "user1"}, 1); UNIT_ASSERT_VALUES_EQUAL(info1.BlobsFromCache, 1); UNIT_ASSERT_VALUES_EQUAL(info2.BlobsFromCache, 1); @@ -889,34 +889,34 @@ namespace { Y_UNIT_TEST(FetchRequest) { - NPersQueue::TTestServer server(PQSettings(0).SetDomainName("Root")); - TString secondTopic = DEFAULT_TOPIC_NAME + "2"; + NPersQueue::TTestServer server(PQSettings(0).SetDomainName("Root")); + TString secondTopic = DEFAULT_TOPIC_NAME + "2"; - server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 10); - server.AnnoyingClient->CreateTopic(secondTopic, 10); + server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 10); + server.AnnoyingClient->CreateTopic(secondTopic, 10); ui32 valueSize = 128; TString value1(valueSize, 'a'); TString value2(valueSize, 'b'); - server.AnnoyingClient->WriteToPQ({secondTopic, 5, "source1", 0}, value2); - server.AnnoyingClient->WriteToPQ({DEFAULT_TOPIC_NAME, 1, "source1", 0}, value1); - server.AnnoyingClient->WriteToPQ({DEFAULT_TOPIC_NAME, 1, "source1", 1}, value2); + server.AnnoyingClient->WriteToPQ({secondTopic, 5, "source1", 0}, value2); + server.AnnoyingClient->WriteToPQ({DEFAULT_TOPIC_NAME, 1, "source1", 0}, value1); + server.AnnoyingClient->WriteToPQ({DEFAULT_TOPIC_NAME, 1, "source1", 1}, value2); - server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD, NKikimrServices::PERSQUEUE }); + server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD, NKikimrServices::PERSQUEUE }); TInstant tm(TInstant::Now()); - server.AnnoyingClient->FetchRequestPQ({{secondTopic, 5, 0, 400},{DEFAULT_TOPIC_NAME, 1, 0, 400},{DEFAULT_TOPIC_NAME, 3, 0, 400}}, 400, 1000000); + server.AnnoyingClient->FetchRequestPQ({{secondTopic, 5, 0, 400},{DEFAULT_TOPIC_NAME, 1, 0, 400},{DEFAULT_TOPIC_NAME, 3, 0, 400}}, 400, 1000000); UNIT_ASSERT((TInstant::Now() - tm).Seconds() < 1); tm = TInstant::Now(); - server.AnnoyingClient->FetchRequestPQ({{secondTopic, 5, 1, 400}}, 400, 5000); + server.AnnoyingClient->FetchRequestPQ({{secondTopic, 5, 1, 400}}, 400, 5000); UNIT_ASSERT((TInstant::Now() - tm).Seconds() > 2); - server.AnnoyingClient->FetchRequestPQ({{secondTopic, 5, 0, 400},{DEFAULT_TOPIC_NAME, 1, 0, 400},{DEFAULT_TOPIC_NAME, 3, 0, 400}}, 1, 1000000); - server.AnnoyingClient->FetchRequestPQ({{secondTopic, 5, 500, 400},{secondTopic, 4, 0, 400},{DEFAULT_TOPIC_NAME, 1, 0, 400}}, 400, 1000000); + server.AnnoyingClient->FetchRequestPQ({{secondTopic, 5, 0, 400},{DEFAULT_TOPIC_NAME, 1, 0, 400},{DEFAULT_TOPIC_NAME, 3, 0, 400}}, 1, 1000000); + server.AnnoyingClient->FetchRequestPQ({{secondTopic, 5, 500, 400},{secondTopic, 4, 0, 400},{DEFAULT_TOPIC_NAME, 1, 0, 400}}, 400, 1000000); } Y_UNIT_TEST(Init) { - NPersQueue::TTestServer server(PQSettings(0).SetDomainName("Root").SetNodeCount(2)); + NPersQueue::TTestServer server(PQSettings(0).SetDomainName("Root").SetNodeCount(2)); if (!true) { - server.EnableLogs( { + server.EnableLogs( { NKikimrServices::FLAT_TX_SCHEMESHARD, NKikimrServices::TX_DATASHARD, NKikimrServices::HIVE, @@ -927,38 +927,38 @@ namespace { NKikimrServices::PQ_METACACHE }); } - server.AnnoyingClient->DescribeTopic({}); - server.AnnoyingClient->TestCase({}, 0, 0, true); + server.AnnoyingClient->DescribeTopic({}); + server.AnnoyingClient->TestCase({}, 0, 0, true); - server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 10); - server.AnnoyingClient->AlterTopic(DEFAULT_TOPIC_NAME, 20); - TString secondTopic = DEFAULT_TOPIC_NAME + "2"; - TString thirdTopic = DEFAULT_TOPIC_NAME + "3"; - server.AnnoyingClient->CreateTopic(secondTopic, 25); + server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 10); + server.AnnoyingClient->AlterTopic(DEFAULT_TOPIC_NAME, 20); + TString secondTopic = DEFAULT_TOPIC_NAME + "2"; + TString thirdTopic = DEFAULT_TOPIC_NAME + "3"; + server.AnnoyingClient->CreateTopic(secondTopic, 25); - server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 5, "abacaba", 1, "valuevaluevalue1"); - server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 5, "abacaba", 2, "valuevaluevalue2"); - server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 5, "abacabae", 1, "valuevaluevalue3"); - server.AnnoyingClient->ReadFromPQ(DEFAULT_TOPIC_NAME, 5, 0, 10, 3); + server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 5, "abacaba", 1, "valuevaluevalue1"); + server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 5, "abacaba", 2, "valuevaluevalue2"); + server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 5, "abacabae", 1, "valuevaluevalue3"); + server.AnnoyingClient->ReadFromPQ(DEFAULT_TOPIC_NAME, 5, 0, 10, 3); - server.AnnoyingClient->SetClientOffsetPQ(DEFAULT_TOPIC_NAME, 5, 2); + server.AnnoyingClient->SetClientOffsetPQ(DEFAULT_TOPIC_NAME, 5, 2); - server.AnnoyingClient->TestCase({{DEFAULT_TOPIC_NAME, {5}}}, 1, 1, true); - server.AnnoyingClient->TestCase({{DEFAULT_TOPIC_NAME, {0}}}, 1, 0, true); - server.AnnoyingClient->TestCase({{DEFAULT_TOPIC_NAME, {}}}, 20, 1, true); - server.AnnoyingClient->TestCase({{DEFAULT_TOPIC_NAME, {5, 5}}}, 0, 0, false); - server.AnnoyingClient->TestCase({{DEFAULT_TOPIC_NAME, {111}}}, 0, 0, false); - server.AnnoyingClient->TestCase({}, 45, 1, true); - server.AnnoyingClient->TestCase({{thirdTopic, {}}}, 0, 0, false); - server.AnnoyingClient->TestCase({{DEFAULT_TOPIC_NAME, {}}, {thirdTopic, {}}}, 0, 0, false); - server.AnnoyingClient->TestCase({{DEFAULT_TOPIC_NAME, {}}, {secondTopic, {}}}, 45, 1, true); - server.AnnoyingClient->TestCase({{DEFAULT_TOPIC_NAME, {0, 3, 5}}, {secondTopic, {1, 4, 6, 8}}}, 7, 1, true); + server.AnnoyingClient->TestCase({{DEFAULT_TOPIC_NAME, {5}}}, 1, 1, true); + server.AnnoyingClient->TestCase({{DEFAULT_TOPIC_NAME, {0}}}, 1, 0, true); + server.AnnoyingClient->TestCase({{DEFAULT_TOPIC_NAME, {}}}, 20, 1, true); + server.AnnoyingClient->TestCase({{DEFAULT_TOPIC_NAME, {5, 5}}}, 0, 0, false); + server.AnnoyingClient->TestCase({{DEFAULT_TOPIC_NAME, {111}}}, 0, 0, false); + server.AnnoyingClient->TestCase({}, 45, 1, true); + server.AnnoyingClient->TestCase({{thirdTopic, {}}}, 0, 0, false); + server.AnnoyingClient->TestCase({{DEFAULT_TOPIC_NAME, {}}, {thirdTopic, {}}}, 0, 0, false); + server.AnnoyingClient->TestCase({{DEFAULT_TOPIC_NAME, {}}, {secondTopic, {}}}, 45, 1, true); + server.AnnoyingClient->TestCase({{DEFAULT_TOPIC_NAME, {0, 3, 5}}, {secondTopic, {1, 4, 6, 8}}}, 7, 1, true); - server.AnnoyingClient->DescribeTopic({DEFAULT_TOPIC_NAME}); - server.AnnoyingClient->DescribeTopic({secondTopic}); - server.AnnoyingClient->DescribeTopic({secondTopic, DEFAULT_TOPIC_NAME}); - server.AnnoyingClient->DescribeTopic({}); - server.AnnoyingClient->DescribeTopic({thirdTopic}, true); + server.AnnoyingClient->DescribeTopic({DEFAULT_TOPIC_NAME}); + server.AnnoyingClient->DescribeTopic({secondTopic}); + server.AnnoyingClient->DescribeTopic({secondTopic, DEFAULT_TOPIC_NAME}); + server.AnnoyingClient->DescribeTopic({}); + server.AnnoyingClient->DescribeTopic({thirdTopic}, true); } void WaitResolveSuccess(TFlatMsgBusPQClient& annoyingClient, TString topic, ui32 numParts) { @@ -985,28 +985,28 @@ namespace { } Y_UNIT_TEST(WhenDisableNodeAndCreateTopic_ThenAllPartitionsAreOnOtherNode) { - NPersQueue::TTestServer server(PQSettings(0).SetDomainName("Root").SetNodeCount(2)); - server.EnableLogs({ NKikimrServices::PERSQUEUE, NKikimrServices::HIVE }); - TString unusedTopic = "rt3.dc1--unusedtopic"; - server.AnnoyingClient->CreateTopic(unusedTopic, 1); - WaitResolveSuccess(*server.AnnoyingClient, unusedTopic, 1); + NPersQueue::TTestServer server(PQSettings(0).SetDomainName("Root").SetNodeCount(2)); + server.EnableLogs({ NKikimrServices::PERSQUEUE, NKikimrServices::HIVE }); + TString unusedTopic = "rt3.dc1--unusedtopic"; + server.AnnoyingClient->CreateTopic(unusedTopic, 1); + WaitResolveSuccess(*server.AnnoyingClient, unusedTopic, 1); // Act // Disable node #0 - server.AnnoyingClient->MarkNodeInHive(server.CleverServer->GetRuntime(), 0, false); - server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 3); - WaitResolveSuccess(*server.AnnoyingClient, DEFAULT_TOPIC_NAME, 3); + server.AnnoyingClient->MarkNodeInHive(server.CleverServer->GetRuntime(), 0, false); + server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 3); + WaitResolveSuccess(*server.AnnoyingClient, DEFAULT_TOPIC_NAME, 3); // Assert that all partitions are on node #1 - const ui32 node1Id = server.CleverServer->GetRuntime()->GetNodeId(1); + const ui32 node1Id = server.CleverServer->GetRuntime()->GetNodeId(1); UNIT_ASSERT_VALUES_EQUAL( - server.AnnoyingClient->GetPartLocation({{DEFAULT_TOPIC_NAME, {0, 1}}}, 2, true), + server.AnnoyingClient->GetPartLocation({{DEFAULT_TOPIC_NAME, {0, 1}}}, 2, true), TVector<ui32>({node1Id, node1Id}) ); } - void PrepareForGrpc(NPersQueue::TTestServer& server) { - server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 2); + void PrepareForGrpc(NPersQueue::TTestServer& server) { + server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 2); } class TTestCredentialsProvider : public NYdb::ICredentialsProvider { @@ -1064,25 +1064,25 @@ namespace { Y_UNIT_TEST(CheckACLForGrpcWrite) { - NPersQueue::TTestServer server(PQSettings(0, 1)); - server.EnableLogs({ NKikimrServices::PQ_WRITE_PROXY }); - PrepareForGrpc(server); + NPersQueue::TTestServer server(PQSettings(0, 1)); + server.EnableLogs({ NKikimrServices::PQ_WRITE_PROXY }); + PrepareForGrpc(server); - TPQDataWriter writer("source1", server); - TPQDataWriter writer2("source1", server); + TPQDataWriter writer("source1", server); + TPQDataWriter writer2("source1", server); - server.CleverServer->GetRuntime()->GetAppData().PQConfig.SetRequireCredentialsInNewProtocol(true); + server.CleverServer->GetRuntime()->GetAppData().PQConfig.SetRequireCredentialsInNewProtocol(true); - writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue1"}, true, TString()); // Fail if user set empty token - writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue1"}, true, "topic1@" BUILTIN_ACL_DOMAIN); + writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue1"}, true, TString()); // Fail if user set empty token + writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue1"}, true, "topic1@" BUILTIN_ACL_DOMAIN); NACLib::TDiffACL acl; acl.AddAccess(NACLib::EAccessType::Allow, NACLib::UpdateRow, "topic1@" BUILTIN_ACL_DOMAIN); - server.AnnoyingClient->ModifyACL("/Root/PQ", DEFAULT_TOPIC_NAME, acl.SerializeAsString()); + server.AnnoyingClient->ModifyACL("/Root/PQ", DEFAULT_TOPIC_NAME, acl.SerializeAsString()); WaitACLModification(); - writer2.Write(SHORT_TOPIC_NAME, {"valuevaluevalue1"}, false, "topic1@" BUILTIN_ACL_DOMAIN); - writer2.Write(SHORT_TOPIC_NAME, {"valuevaluevalue1"}, true, "invalid_ticket"); + writer2.Write(SHORT_TOPIC_NAME, {"valuevaluevalue1"}, false, "topic1@" BUILTIN_ACL_DOMAIN); + writer2.Write(SHORT_TOPIC_NAME, {"valuevaluevalue1"}, true, "invalid_ticket"); auto driver = server.AnnoyingClient->GetDriver(); @@ -1136,67 +1136,67 @@ namespace { Y_UNIT_TEST(CheckACLForGrpcRead) { - NPersQueue::TTestServer server(PQSettings(0, 1)); - server.EnableLogs({ NKikimrServices::PQ_READ_PROXY }); - server.EnableLogs({ NKikimrServices::PERSQUEUE }, NActors::NLog::PRI_INFO); - TString topic2 = DEFAULT_TOPIC_NAME + "2"; - TString shortTopic2Name = "topic12"; - PrepareForGrpc(server); - - server.AnnoyingClient->CreateTopic(topic2, 1, 8*1024*1024, 86400, 20000000, "", 200000000, {"user1", "user2"}); - server.WaitInit(shortTopic2Name); - server.AnnoyingClient->CreateConsumer("user1"); - server.AnnoyingClient->CreateConsumer("user2"); - server.AnnoyingClient->CreateConsumer("user5"); - server.AnnoyingClient->GrantConsumerAccess("user1", "user2@" BUILTIN_ACL_DOMAIN); - server.AnnoyingClient->GrantConsumerAccess("user1", "user3@" BUILTIN_ACL_DOMAIN); + NPersQueue::TTestServer server(PQSettings(0, 1)); + server.EnableLogs({ NKikimrServices::PQ_READ_PROXY }); + server.EnableLogs({ NKikimrServices::PERSQUEUE }, NActors::NLog::PRI_INFO); + TString topic2 = DEFAULT_TOPIC_NAME + "2"; + TString shortTopic2Name = "topic12"; + PrepareForGrpc(server); + + server.AnnoyingClient->CreateTopic(topic2, 1, 8*1024*1024, 86400, 20000000, "", 200000000, {"user1", "user2"}); + server.WaitInit(shortTopic2Name); + server.AnnoyingClient->CreateConsumer("user1"); + server.AnnoyingClient->CreateConsumer("user2"); + server.AnnoyingClient->CreateConsumer("user5"); + server.AnnoyingClient->GrantConsumerAccess("user1", "user2@" BUILTIN_ACL_DOMAIN); + server.AnnoyingClient->GrantConsumerAccess("user1", "user3@" BUILTIN_ACL_DOMAIN); server.AnnoyingClient->GrantConsumerAccess("user1", "1@" BUILTIN_ACL_DOMAIN); server.AnnoyingClient->GrantConsumerAccess("user2", "2@" BUILTIN_ACL_DOMAIN); server.AnnoyingClient->GrantConsumerAccess("user5", "1@" BUILTIN_ACL_DOMAIN); server.AnnoyingClient->GrantConsumerAccess("user5", "2@" BUILTIN_ACL_DOMAIN); - Cerr << "=== Create writer\n"; - TPQDataWriter writer("source1", server); + Cerr << "=== Create writer\n"; + TPQDataWriter writer("source1", server); - server.CleverServer->GetRuntime()->GetAppData().PQConfig.SetRequireCredentialsInNewProtocol(true); + server.CleverServer->GetRuntime()->GetAppData().PQConfig.SetRequireCredentialsInNewProtocol(true); NACLib::TDiffACL acl; acl.AddAccess(NACLib::EAccessType::Allow, NACLib::SelectRow, "1@" BUILTIN_ACL_DOMAIN); acl.AddAccess(NACLib::EAccessType::Allow, NACLib::SelectRow, "2@" BUILTIN_ACL_DOMAIN); acl.AddAccess(NACLib::EAccessType::Allow, NACLib::SelectRow, "user1@" BUILTIN_ACL_DOMAIN); acl.AddAccess(NACLib::EAccessType::Allow, NACLib::SelectRow, "user2@" BUILTIN_ACL_DOMAIN); - server.AnnoyingClient->ModifyACL("/Root/PQ", topic2, acl.SerializeAsString()); + server.AnnoyingClient->ModifyACL("/Root/PQ", topic2, acl.SerializeAsString()); WaitACLModification(); auto ticket1 = "1@" BUILTIN_ACL_DOMAIN; auto ticket2 = "2@" BUILTIN_ACL_DOMAIN; - Cerr << "=== Writer - do reads\n"; - writer.Read(shortTopic2Name, "user1", ticket1, false, false, true); + Cerr << "=== Writer - do reads\n"; + writer.Read(shortTopic2Name, "user1", ticket1, false, false, true); - writer.Read(shortTopic2Name, "user1", "user2@" BUILTIN_ACL_DOMAIN, false, false, true); - writer.Read(shortTopic2Name, "user1", "user3@" BUILTIN_ACL_DOMAIN, true, false, true); //for topic - writer.Read(shortTopic2Name, "user1", "user1@" BUILTIN_ACL_DOMAIN, true, false, true); //for consumer - writer.Read(shortTopic2Name, "user2", ticket1, true, false, true); - writer.Read(shortTopic2Name, "user2", ticket2, false, false, true); + writer.Read(shortTopic2Name, "user1", "user2@" BUILTIN_ACL_DOMAIN, false, false, true); + writer.Read(shortTopic2Name, "user1", "user3@" BUILTIN_ACL_DOMAIN, true, false, true); //for topic + writer.Read(shortTopic2Name, "user1", "user1@" BUILTIN_ACL_DOMAIN, true, false, true); //for consumer + writer.Read(shortTopic2Name, "user2", ticket1, true, false, true); + writer.Read(shortTopic2Name, "user2", ticket2, false, false, true); - writer.Read(shortTopic2Name, "user5", ticket1, true, false, true); - writer.Read(shortTopic2Name, "user5", ticket2, true, false, true); + writer.Read(shortTopic2Name, "user5", ticket1, true, false, true); + writer.Read(shortTopic2Name, "user5", ticket2, true, false, true); acl.Clear(); acl.AddAccess(NACLib::EAccessType::Allow, NACLib::SelectRow, "user3@" BUILTIN_ACL_DOMAIN); - server.AnnoyingClient->ModifyACL("/Root/PQ", topic2, acl.SerializeAsString()); + server.AnnoyingClient->ModifyACL("/Root/PQ", topic2, acl.SerializeAsString()); WaitACLModification(); - Cerr << "==== Writer - read\n"; - writer.Read(shortTopic2Name, "user1", "user3@" BUILTIN_ACL_DOMAIN, false, true, true); + Cerr << "==== Writer - read\n"; + writer.Read(shortTopic2Name, "user1", "user3@" BUILTIN_ACL_DOMAIN, false, true, true); - auto Channel_ = grpc::CreateChannel("localhost:" + ToString(server.GrpcPort), grpc::InsecureChannelCredentials()); + auto Channel_ = grpc::CreateChannel("localhost:" + ToString(server.GrpcPort), grpc::InsecureChannelCredentials()); auto StubP_ = Ydb::PersQueue::V1::PersQueueService::NewStub(Channel_); /* auto driver = server.AnnoyingClient->GetDriver(); - Cerr << "==== Start consuming loop\n"; + Cerr << "==== Start consuming loop\n"; for (ui32 i = 0; i < 2; ++i) { std::shared_ptr<NYdb::ICredentialsProviderFactory> creds = std::make_shared<TTestCredentialsProviderFactory>(NYdb::TStringType("user3@" BUILTIN_ACL_DOMAIN)); @@ -1217,7 +1217,7 @@ namespace { ev->Confirm(); - Cerr << "=== Wait for consumer death (" << i << ")" << Endl; + Cerr << "=== Wait for consumer death (" << i << ")" << Endl; msg = reader->GetEvent(true, 1); UNIT_ASSERT(msg); @@ -1229,15 +1229,15 @@ namespace { UNIT_ASSERT(closeEv); } */ - Cerr << "==== Start second loop\n"; + Cerr << "==== Start second loop\n"; for (ui32 i = 0; i < 3; ++i){ - server.AnnoyingClient->GetClientInfo({topic2}, "user1", true); + server.AnnoyingClient->GetClientInfo({topic2}, "user1", true); ReadInfoRequest request; ReadInfoResponse response; request.mutable_consumer()->set_path("user1"); request.set_get_only_original(true); - request.add_topics()->set_path(shortTopic2Name); + request.add_topics()->set_path(shortTopic2Name); grpc::ClientContext rcontext; if (i == 0) { rcontext.AddMetadata("x-ydb-auth-ticket", "user_without_rights@" BUILTIN_ACL_DOMAIN); @@ -1259,11 +1259,11 @@ namespace { } Y_UNIT_TEST(CheckKillBalancer) { - NPersQueue::TTestServer server; - server.EnableLogs({ NKikimrServices::PQ_WRITE_PROXY, NKikimrServices::PQ_READ_PROXY}); - PrepareForGrpc(server); + NPersQueue::TTestServer server; + server.EnableLogs({ NKikimrServices::PQ_WRITE_PROXY, NKikimrServices::PQ_READ_PROXY}); + PrepareForGrpc(server); - TPQDataWriter writer("source1", server); + TPQDataWriter writer("source1", server); auto driver = server.AnnoyingClient->GetDriver(); @@ -1284,7 +1284,7 @@ namespace { ev->Confirm(); } - server.AnnoyingClient->RestartBalancerTablet(server.CleverServer->GetRuntime(), "rt3.dc1--topic1"); + server.AnnoyingClient->RestartBalancerTablet(server.CleverServer->GetRuntime(), "rt3.dc1--topic1"); Cerr << "Balancer killed\n"; @@ -1316,20 +1316,20 @@ namespace { Y_UNIT_TEST(TestWriteStat) { - NPersQueue::TTestServer server(PQSettings(0, 1, true, "10"), false); - auto netDataUpdated = server.PrepareNetDataFile(FormNetData()); - UNIT_ASSERT(netDataUpdated); - server.StartServer(); + NPersQueue::TTestServer server(PQSettings(0, 1, true, "10"), false); + auto netDataUpdated = server.PrepareNetDataFile(FormNetData()); + UNIT_ASSERT(netDataUpdated); + server.StartServer(); - server.EnableLogs({ NKikimrServices::PQ_WRITE_PROXY, NKikimrServices::NET_CLASSIFIER }); - server.EnableLogs({ NKikimrServices::PERSQUEUE }, NActors::NLog::PRI_ERROR); + server.EnableLogs({ NKikimrServices::PQ_WRITE_PROXY, NKikimrServices::NET_CLASSIFIER }); + server.EnableLogs({ NKikimrServices::PERSQUEUE }, NActors::NLog::PRI_ERROR); - auto sender = server.CleverServer->GetRuntime()->AllocateEdgeActor(); + auto sender = server.CleverServer->GetRuntime()->AllocateEdgeActor(); - GetClassifierUpdate(*server.CleverServer, sender); //wait for initializing + GetClassifierUpdate(*server.CleverServer, sender); //wait for initializing - server.AnnoyingClient->CreateTopic("rt3.dc1--account--topic1", 10, 10000, 10000, 2000); - server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 10, 10000, 10000, 2000); + server.AnnoyingClient->CreateTopic("rt3.dc1--account--topic1", 10, 10000, 10000, 2000); + server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 10, 10000, 10000, 2000); auto driver = server.AnnoyingClient->GetDriver(); @@ -1366,7 +1366,7 @@ namespace { UNIT_ASSERT(ack->Acks.back().Stat->TotalTimeInPartitionQueue <= ack->Acks.back().Stat->PartitionQuotedTime + TDuration::Seconds(10)); } } - GetCounters(server.CleverServer->GetRuntime()->GetMonPort(), "pqproxy", "writeSession", "account/topic1", "Vla"); + GetCounters(server.CleverServer->GetRuntime()->GetMonPort(), "pqproxy", "writeSession", "account/topic1", "Vla"); { NYdb::NPersQueue::TReadSessionSettings settings; settings.ConsumerName("shared/user").AppendTopics(TString("account/topic1")).ReadOriginal({"dc1"}); @@ -1378,14 +1378,14 @@ namespace { Cerr << NYdb::NPersQueue::DebugString(*msg) << "\n"; } - GetCounters(server.CleverServer->GetRuntime()->GetMonPort(), "pqproxy", "readSession", "account/topic1", "Vla"); + GetCounters(server.CleverServer->GetRuntime()->GetMonPort(), "pqproxy", "readSession", "account/topic1", "Vla"); } Y_UNIT_TEST(TestWriteSessionsConflicts) { - NPersQueue::TTestServer server; - server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1); + NPersQueue::TTestServer server; + server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1); - TPQDataWriter writer("source", server); + TPQDataWriter writer("source", server); server.EnableLogs({ NKikimrServices::PQ_WRITE_PROXY }); @@ -1433,13 +1433,13 @@ namespace { Y_UNIT_TEST(TestLockErrors) { return; // Test is ignored. FIX: KIKIMR-7881 - NPersQueue::TTestServer server; - server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1); - server.EnableLogs({ NKikimrServices::PQ_READ_PROXY }); - auto pqLib = TPQLib::WithCerrLogger(); + NPersQueue::TTestServer server; + server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1); + server.EnableLogs({ NKikimrServices::PQ_READ_PROXY }); + auto pqLib = TPQLib::WithCerrLogger(); { - auto [producer, pcResult] = CreateProducer(pqLib, server.GrpcPort, SHORT_TOPIC_NAME, "123"); + auto [producer, pcResult] = CreateProducer(pqLib, server.GrpcPort, SHORT_TOPIC_NAME, "123"); for (ui32 i = 1; i <= 11; ++i) { auto f = producer->Write(i, TString(10, 'a')); f.Wait(); @@ -1448,17 +1448,17 @@ namespace { TConsumerSettings ss; ss.Consumer = "user"; - ss.Server = TServerSetting{"localhost", server.GrpcPort}; - ss.Topics.push_back({SHORT_TOPIC_NAME, {}}); + ss.Server = TServerSetting{"localhost", server.GrpcPort}; + ss.Topics.push_back({SHORT_TOPIC_NAME, {}}); ss.MaxCount = 1; ss.Unpack = false; - auto [consumer, ccResult] = CreateConsumer(pqLib, ss); + auto [consumer, ccResult] = CreateConsumer(pqLib, ss); auto msg = consumer->GetNextMessage(); msg.Wait(); Cerr << msg.GetValue().Response << "\n"; UNIT_ASSERT(msg.GetValue().Response.response_case() == MigrationStreamingReadServerMessage::kAssigned); - UNIT_ASSERT(msg.GetValue().Response.assigned().topic().path() == SHORT_TOPIC_NAME); + UNIT_ASSERT(msg.GetValue().Response.assigned().topic().path() == SHORT_TOPIC_NAME); UNIT_ASSERT(msg.GetValue().Response.assigned().cluster() == "dc1"); UNIT_ASSERT(msg.GetValue().Response.assigned().partition() == 0); UNIT_ASSERT(msg.GetValue().Response.assigned().read_offset() == 0); @@ -1470,12 +1470,12 @@ namespace { future.Wait(); Cerr << future.GetValue() << "\n"; - std::tie(consumer, ccResult) = CreateConsumer(pqLib, ss); + std::tie(consumer, ccResult) = CreateConsumer(pqLib, ss); msg = consumer->GetNextMessage(); msg.Wait(); Cerr << msg.GetValue().Response << "\n"; UNIT_ASSERT(msg.GetValue().Response.response_case() == MigrationStreamingReadServerMessage::kAssigned); - UNIT_ASSERT(msg.GetValue().Response.assigned().topic().path() == SHORT_TOPIC_NAME); + UNIT_ASSERT(msg.GetValue().Response.assigned().topic().path() == SHORT_TOPIC_NAME); UNIT_ASSERT(msg.GetValue().Response.assigned().cluster() == "dc1"); UNIT_ASSERT(msg.GetValue().Response.assigned().partition() == 0); UNIT_ASSERT(msg.GetValue().Response.assigned().read_offset() == 0); @@ -1487,12 +1487,12 @@ namespace { future.Wait(); Cerr << future.GetValue() << "\n"; - std::tie(consumer, ccResult) = CreateConsumer(pqLib, ss); + std::tie(consumer, ccResult) = CreateConsumer(pqLib, ss); msg = consumer->GetNextMessage(); msg.Wait(); Cerr << msg.GetValue().Response << "\n"; UNIT_ASSERT(msg.GetValue().Response.response_case() == MigrationStreamingReadServerMessage::kAssigned); - UNIT_ASSERT(msg.GetValue().Response.assigned().topic().path() == SHORT_TOPIC_NAME); + UNIT_ASSERT(msg.GetValue().Response.assigned().topic().path() == SHORT_TOPIC_NAME); UNIT_ASSERT(msg.GetValue().Response.assigned().cluster() == "dc1"); UNIT_ASSERT(msg.GetValue().Response.assigned().partition() == 0); UNIT_ASSERT(msg.GetValue().Response.assigned().read_offset() == 0); @@ -1504,12 +1504,12 @@ namespace { future.Wait(); Cerr << future.GetValue() << "\n"; - std::tie(consumer, ccResult) = CreateConsumer(pqLib, ss); + std::tie(consumer, ccResult) = CreateConsumer(pqLib, ss); msg = consumer->GetNextMessage(); msg.Wait(); Cerr << msg.GetValue().Response << "\n"; UNIT_ASSERT(msg.GetValue().Response.response_case() == MigrationStreamingReadServerMessage::kAssigned); - UNIT_ASSERT(msg.GetValue().Response.assigned().topic().path() == SHORT_TOPIC_NAME); + UNIT_ASSERT(msg.GetValue().Response.assigned().topic().path() == SHORT_TOPIC_NAME); UNIT_ASSERT(msg.GetValue().Response.assigned().cluster() == "dc1"); UNIT_ASSERT(msg.GetValue().Response.assigned().partition() == 0); UNIT_ASSERT(msg.GetValue().Response.assigned().read_offset() == 0); @@ -1530,12 +1530,12 @@ namespace { } } - std::tie(consumer, ccResult) = CreateConsumer(pqLib, ss); + std::tie(consumer, ccResult) = CreateConsumer(pqLib, ss); msg = consumer->GetNextMessage(); msg.Wait(); Cerr << msg.GetValue().Response << "\n"; UNIT_ASSERT(msg.GetValue().Response.response_case() == MigrationStreamingReadServerMessage::kAssigned); - UNIT_ASSERT(msg.GetValue().Response.assigned().topic().path() == SHORT_TOPIC_NAME); + UNIT_ASSERT(msg.GetValue().Response.assigned().topic().path() == SHORT_TOPIC_NAME); UNIT_ASSERT(msg.GetValue().Response.assigned().cluster() == "dc1"); UNIT_ASSERT(msg.GetValue().Response.assigned().partition() == 0); UNIT_ASSERT(msg.GetValue().Response.assigned().read_offset() == 5); @@ -1545,12 +1545,12 @@ namespace { pp.SetValue(TAssignInfo{11, 11, false}); Sleep(TDuration::Seconds(5)); - std::tie(consumer, ccResult) = CreateConsumer(pqLib, ss); + std::tie(consumer, ccResult) = CreateConsumer(pqLib, ss); msg = consumer->GetNextMessage(); msg.Wait(); Cerr << msg.GetValue().Response << "\n"; UNIT_ASSERT(msg.GetValue().Response.response_case() == MigrationStreamingReadServerMessage::kAssigned); - UNIT_ASSERT(msg.GetValue().Response.assigned().topic().path() == SHORT_TOPIC_NAME); + UNIT_ASSERT(msg.GetValue().Response.assigned().topic().path() == SHORT_TOPIC_NAME); UNIT_ASSERT(msg.GetValue().Response.assigned().cluster() == "dc1"); UNIT_ASSERT(msg.GetValue().Response.assigned().partition() == 0); UNIT_ASSERT(msg.GetValue().Response.assigned().read_offset() == 11); @@ -1562,12 +1562,12 @@ namespace { future.Wait(); Cerr << future.GetValue() << "\n"; - std::tie(consumer, ccResult) = CreateConsumer(pqLib, ss); + std::tie(consumer, ccResult) = CreateConsumer(pqLib, ss); msg = consumer->GetNextMessage(); msg.Wait(); Cerr << msg.GetValue().Response << "\n"; UNIT_ASSERT(msg.GetValue().Response.response_case() == MigrationStreamingReadServerMessage::kAssigned); - UNIT_ASSERT(msg.GetValue().Response.assigned().topic().path() == SHORT_TOPIC_NAME); + UNIT_ASSERT(msg.GetValue().Response.assigned().topic().path() == SHORT_TOPIC_NAME); UNIT_ASSERT(msg.GetValue().Response.assigned().cluster() == "dc1"); UNIT_ASSERT(msg.GetValue().Response.assigned().partition() == 0); UNIT_ASSERT(msg.GetValue().Response.assigned().read_offset() == 11); @@ -1581,15 +1581,15 @@ namespace { */ Y_UNIT_TEST(TestBigMessage) { - NPersQueue::TTestServer server; - server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1); + NPersQueue::TTestServer server; + server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1); server.EnableLogs({ NKikimrServices::PQ_WRITE_PROXY }); TPQDataWriter writer2("source", server); auto driver = server.AnnoyingClient->GetDriver(); - + auto writer = CreateWriter(*driver, SHORT_TOPIC_NAME, "123", 0, "raw"); auto msg = writer->GetEvent(true); @@ -1599,28 +1599,28 @@ namespace { UNIT_ASSERT(ev); writer->Write(std::move(ev->ContinuationToken), TString((1 << 20) * 60, 'a')); //TODO: Increase GRPC_ARG_MAX_SEND_MESSAGE_LENGTH - { - msg = writer->GetEvent(true); - UNIT_ASSERT(msg); // ReadyToAcceptEvent - auto ev2 = std::get_if<NYdb::NPersQueue::TWriteSessionEvent::TReadyToAcceptEvent>(&*msg); - UNIT_ASSERT(ev2); - } - { - msg = writer->GetEvent(true); - UNIT_ASSERT(msg); // Ack on write - auto ev2 = std::get_if<NYdb::NPersQueue::TWriteSessionEvent::TAcksEvent>(&*msg); - UNIT_ASSERT(ev2); - } + { + msg = writer->GetEvent(true); + UNIT_ASSERT(msg); // ReadyToAcceptEvent + auto ev2 = std::get_if<NYdb::NPersQueue::TWriteSessionEvent::TReadyToAcceptEvent>(&*msg); + UNIT_ASSERT(ev2); + } + { + msg = writer->GetEvent(true); + UNIT_ASSERT(msg); // Ack on write + auto ev2 = std::get_if<NYdb::NPersQueue::TWriteSessionEvent::TAcksEvent>(&*msg); + UNIT_ASSERT(ev2); + } } /* void TestRereadsWhenDataIsEmptyImpl(bool withWait) { - NPersQueue::TTestServer server; - server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1); + NPersQueue::TTestServer server; + server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1); - server.EnableLogs({ NKikimrServices::PQ_READ_PROXY }); - TPQDataWriter writer("source", server); - auto pqLib = TPQLib::WithCerrLogger(); + server.EnableLogs({ NKikimrServices::PQ_READ_PROXY }); + TPQDataWriter writer("source", server); + auto pqLib = TPQLib::WithCerrLogger(); // Write nonempty data NKikimr::NPersQueueTests::TRequestWritePQ writeReq(DEFAULT_TOPIC_NAME, 0, "src", 4); @@ -1635,7 +1635,7 @@ namespace { } TString serialized; UNIT_ASSERT(dataChunk.SerializeToString(&serialized)); - server.AnnoyingClient->WriteToPQ(writeReq, serialized); + server.AnnoyingClient->WriteToPQ(writeReq, serialized); }; write("data1"); write("data2", true); @@ -1647,10 +1647,10 @@ namespace { bool unpack = false; ui32 maxInflyRequests = 1; ui32 maxMemoryUsage = 1; - auto [consumer, ccResult] = CreateConsumer( - pqLib, server.GrpcPort, "user", {SHORT_TOPIC_NAME, {}}, - maxCount, unpack, {}, maxInflyRequests, maxMemoryUsage - ); + auto [consumer, ccResult] = CreateConsumer( + pqLib, server.GrpcPort, "user", {SHORT_TOPIC_NAME, {}}, + maxCount, unpack, {}, maxInflyRequests, maxMemoryUsage + ); UNIT_ASSERT_C(ccResult.Response.response_case() == MigrationStreamingReadServerMessage::kInitResponse, ccResult.Response); auto msg1 = GetNextMessageSkipAssignment(consumer).GetValueSync().Response; @@ -1695,32 +1695,32 @@ namespace { Y_UNIT_TEST(TestLockAfterDrop) { - NPersQueue::TTestServer server{false}; - server.GrpcServerOptions.SetMaxMessageSize(130*1024*1024); - server.StartServer(); - server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1); - server.WaitInit(SHORT_TOPIC_NAME); - server.EnableLogs({ NKikimrServices::PQ_READ_PROXY }); - auto pqLib = TPQLib::WithCerrLogger(); - - auto [producer, pcResult] = CreateProducer(pqLib, server.GrpcPort, SHORT_TOPIC_NAME, "123"); + NPersQueue::TTestServer server{false}; + server.GrpcServerOptions.SetMaxMessageSize(130*1024*1024); + server.StartServer(); + server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1); + server.WaitInit(SHORT_TOPIC_NAME); + server.EnableLogs({ NKikimrServices::PQ_READ_PROXY }); + auto pqLib = TPQLib::WithCerrLogger(); + + auto [producer, pcResult] = CreateProducer(pqLib, server.GrpcPort, SHORT_TOPIC_NAME, "123"); auto f = producer->Write(1, TString(1024, 'a')); f.Wait(); ui32 maxCount = 1; bool unpack = false; - auto [consumer, ccResult] = CreateConsumer(pqLib, server.GrpcPort, "user", {SHORT_TOPIC_NAME, {}}, maxCount, unpack); + auto [consumer, ccResult] = CreateConsumer(pqLib, server.GrpcPort, "user", {SHORT_TOPIC_NAME, {}}, maxCount, unpack); Cerr << ccResult.Response << "\n"; auto msg = consumer->GetNextMessage(); msg.Wait(); UNIT_ASSERT_C(msg.GetValue().Response.response_case() == MigrationStreamingReadServerMessage::kAssigned, msg.GetValue().Response); - UNIT_ASSERT(msg.GetValue().Response.assigned().topic().path() == SHORT_TOPIC_NAME); + UNIT_ASSERT(msg.GetValue().Response.assigned().topic().path() == SHORT_TOPIC_NAME); UNIT_ASSERT(msg.GetValue().Response.assigned().cluster() == "dc1"); UNIT_ASSERT(msg.GetValue().Response.assigned().partition() == 0); - server.CleverServer->GetRuntime()->ResetScheduledCount(); - server.AnnoyingClient->RestartPartitionTablets(server.CleverServer->GetRuntime(), DEFAULT_TOPIC_NAME); + server.CleverServer->GetRuntime()->ResetScheduledCount(); + server.AnnoyingClient->RestartPartitionTablets(server.CleverServer->GetRuntime(), DEFAULT_TOPIC_NAME); msg.GetValue().StartRead.SetValue({0,0,false}); @@ -1733,16 +1733,16 @@ namespace { Y_UNIT_TEST(TestMaxNewTopicModel) { - NPersQueue::TTestServer server; - server.AnnoyingClient->AlterUserAttributes("/", "Root", {{"__extra_path_symbols_allowed", "@"}}); - server.AnnoyingClient->CreateTopic("rt3.dc1--aaa@bbb@ccc--topic", 1); - server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1); + NPersQueue::TTestServer server; + server.AnnoyingClient->AlterUserAttributes("/", "Root", {{"__extra_path_symbols_allowed", "@"}}); + server.AnnoyingClient->CreateTopic("rt3.dc1--aaa@bbb@ccc--topic", 1); + server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1); - server.EnableLogs({ NKikimrServices::PQ_READ_PROXY }); - auto pqLib = TPQLib::WithCerrLogger(); + server.EnableLogs({ NKikimrServices::PQ_READ_PROXY }); + auto pqLib = TPQLib::WithCerrLogger(); { - auto [producer, pcResult] = CreateProducer(pqLib, server.GrpcPort, "aaa/bbb/ccc/topic", "123"); + auto [producer, pcResult] = CreateProducer(pqLib, server.GrpcPort, "aaa/bbb/ccc/topic", "123"); UNIT_ASSERT_C(pcResult.Response.server_message_case() == StreamingWriteServerMessage::kInitResponse, pcResult.Response); for (ui32 i = 1; i <= 11; ++i) { auto f = producer->Write(i, TString(10, 'a')); @@ -1753,7 +1753,7 @@ namespace { ui32 maxCount = 1; bool unpack = false; - auto [consumer, ccResult] = CreateConsumer(pqLib, server.GrpcPort, "user", {"aaa/bbb/ccc/topic", {}}, maxCount, unpack); + auto [consumer, ccResult] = CreateConsumer(pqLib, server.GrpcPort, "user", {"aaa/bbb/ccc/topic", {}}, maxCount, unpack); UNIT_ASSERT_C(ccResult.Response.response_case() == MigrationStreamingReadServerMessage::kInitResponse, ccResult.Response); auto msg = consumer->GetNextMessage(); @@ -1764,18 +1764,18 @@ namespace { Y_UNIT_TEST(TestReleaseWithAssigns) { - NPersQueue::TTestServer server; - server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 3); + NPersQueue::TTestServer server; + server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 3); - server.EnableLogs({ NKikimrServices::PQ_READ_PROXY }); - auto pqLib = TPQLib::WithCerrLogger(); + server.EnableLogs({ NKikimrServices::PQ_READ_PROXY }); + auto pqLib = TPQLib::WithCerrLogger(); - TPQDataWriter writer("source", server); + TPQDataWriter writer("source", server); for (ui32 i = 1; i <= 3; ++i) { TString sourceId = "123" + ToString<int>(i); ui32 partitionGroup = i; - auto [producer, pcResult] = CreateProducer(pqLib, server.GrpcPort, SHORT_TOPIC_NAME, sourceId, partitionGroup); + auto [producer, pcResult] = CreateProducer(pqLib, server.GrpcPort, SHORT_TOPIC_NAME, sourceId, partitionGroup); UNIT_ASSERT(pcResult.Response.server_message_case() == StreamingWriteServerMessage::kInitResponse); auto f = producer->Write(i, TString(10, 'a')); @@ -1784,13 +1784,13 @@ namespace { TConsumerSettings ss; ss.Consumer = "user"; - ss.Server = TServerSetting{"localhost", server.GrpcPort}; - ss.Topics.push_back({SHORT_TOPIC_NAME, {}}); + ss.Server = TServerSetting{"localhost", server.GrpcPort}; + ss.Topics.push_back({SHORT_TOPIC_NAME, {}}); ss.ReadMirroredPartitions = false; ss.MaxCount = 3; ss.Unpack = false; - auto [consumer, ccResult] = CreateConsumer(pqLib, ss); + auto [consumer, ccResult] = CreateConsumer(pqLib, ss); Cerr << ccResult.Response << "\n"; for (ui32 i = 1; i <= 3; ++i) { @@ -1798,11 +1798,11 @@ namespace { msg.Wait(); Cerr << msg.GetValue().Response << "\n"; UNIT_ASSERT(msg.GetValue().Response.response_case() == MigrationStreamingReadServerMessage::kAssigned); - UNIT_ASSERT(msg.GetValue().Response.assigned().topic().path() == SHORT_TOPIC_NAME); + UNIT_ASSERT(msg.GetValue().Response.assigned().topic().path() == SHORT_TOPIC_NAME); UNIT_ASSERT(msg.GetValue().Response.assigned().cluster() == "dc1"); } - auto [consumer2, ccResult2] = CreateConsumer(pqLib, ss); + auto [consumer2, ccResult2] = CreateConsumer(pqLib, ss); Cerr << ccResult2.Response << "\n"; auto msg = consumer->GetNextMessage(); @@ -1811,7 +1811,7 @@ namespace { msg.Wait(); Cerr << msg.GetValue().Response << "\n"; UNIT_ASSERT(msg.GetValue().Response.response_case() == MigrationStreamingReadServerMessage::kRelease); - UNIT_ASSERT(msg.GetValue().Response.release().topic().path() == SHORT_TOPIC_NAME); + UNIT_ASSERT(msg.GetValue().Response.release().topic().path() == SHORT_TOPIC_NAME); UNIT_ASSERT(msg.GetValue().Response.release().cluster() == "dc1"); UNIT_ASSERT(!msg2.Wait(TDuration::Seconds(1))); @@ -1822,18 +1822,18 @@ namespace { Cerr << msg2.GetValue().Response << "\n"; UNIT_ASSERT(msg2.GetValue().Response.response_case() == MigrationStreamingReadServerMessage::kAssigned); - UNIT_ASSERT(msg2.GetValue().Response.assigned().topic().path() == SHORT_TOPIC_NAME); + UNIT_ASSERT(msg2.GetValue().Response.assigned().topic().path() == SHORT_TOPIC_NAME); UNIT_ASSERT(msg2.GetValue().Response.assigned().cluster() == "dc1"); } Y_UNIT_TEST(TestSilentRelease) { - NPersQueue::TTestServer server; - server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 3); + NPersQueue::TTestServer server; + server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 3); - server.EnableLogs({ NKikimrServices::PQ_READ_PROXY }); - auto pqLib = TPQLib::WithCerrLogger(); + server.EnableLogs({ NKikimrServices::PQ_READ_PROXY }); + auto pqLib = TPQLib::WithCerrLogger(); - TPQDataWriter writer("source", server); + TPQDataWriter writer("source", server); TVector<std::pair<ui64, ui64>> cookies; @@ -1841,8 +1841,8 @@ namespace { TString sourceId = "123" + ToString<int>(i); ui32 partitionGroup = i; - auto [producer, pcResult] = CreateProducer(pqLib, server.GrpcPort, SHORT_TOPIC_NAME, sourceId, partitionGroup); - Cerr << "===Response: " << pcResult.Response << Endl; + auto [producer, pcResult] = CreateProducer(pqLib, server.GrpcPort, SHORT_TOPIC_NAME, sourceId, partitionGroup); + Cerr << "===Response: " << pcResult.Response << Endl; UNIT_ASSERT(pcResult.Response.server_message_case() == StreamingWriteServerMessage::kInitResponse); auto f = producer->Write(i, TString(10, 'a')); f.Wait(); @@ -1850,13 +1850,13 @@ namespace { TConsumerSettings ss; ss.Consumer = "user"; - ss.Server = TServerSetting{"localhost", server.GrpcPort}; - ss.Topics.push_back({SHORT_TOPIC_NAME, {}}); + ss.Server = TServerSetting{"localhost", server.GrpcPort}; + ss.Topics.push_back({SHORT_TOPIC_NAME, {}}); ss.ReadMirroredPartitions = false; ss.MaxCount = 1; ss.Unpack = false; - auto [consumer, ccResult] = CreateConsumer(pqLib, ss); + auto [consumer, ccResult] = CreateConsumer(pqLib, ss); Cerr << ccResult.Response << "\n"; for (ui32 i = 1; i <= 3; ++i) { @@ -1868,7 +1868,7 @@ namespace { } } - auto [consumer2, ccResult2] = CreateConsumer(pqLib, ss); + auto [consumer2, ccResult2] = CreateConsumer(pqLib, ss); Cerr << ccResult2.Response << "\n"; auto msg = consumer->GetNextMessage(); @@ -1884,7 +1884,7 @@ namespace { Cerr << msg2.GetValue().Response << "\n"; UNIT_ASSERT(msg2.GetValue().Response.response_case() == MigrationStreamingReadServerMessage::kAssigned); - UNIT_ASSERT(msg2.GetValue().Response.assigned().topic().path() == SHORT_TOPIC_NAME); + UNIT_ASSERT(msg2.GetValue().Response.assigned().topic().path() == SHORT_TOPIC_NAME); UNIT_ASSERT(msg2.GetValue().Response.assigned().cluster() == "dc1"); UNIT_ASSERT(msg2.GetValue().Response.assigned().read_offset() == 1); } @@ -1893,30 +1893,30 @@ namespace { /* Y_UNIT_TEST(TestDoubleRelease) { - NPersQueue::TTestServer server; - server.AnnoyingClient->CreateTopic( + NPersQueue::TTestServer server; + server.AnnoyingClient->CreateTopic( DEFAULT_TOPIC_NAME, 1, 8000000, 86400, 50000000, "", 50000000, {"user1"}, {"user1", "user2"} - ); - server.EnableLogs({ NKikimrServices::PQ_READ_PROXY, NKikimrServices::TABLET_AGGREGATOR }); - auto pqLib = TPQLib::WithCerrLogger(); - TPQDataWriter writer("source", server); + ); + server.EnableLogs({ NKikimrServices::PQ_READ_PROXY, NKikimrServices::TABLET_AGGREGATOR }); + auto pqLib = TPQLib::WithCerrLogger(); + TPQDataWriter writer("source", server); TConsumerSettings ss; ss.Consumer = "user"; - ss.Server = TServerSetting{"localhost", server.GrpcPort}; - ss.Topics.push_back({SHORT_TOPIC_NAME, {}}); + ss.Server = TServerSetting{"localhost", server.GrpcPort}; + ss.Topics.push_back({SHORT_TOPIC_NAME, {}}); ss.ReadMirroredPartitions = false; ss.MaxCount = 3; ss.Unpack = false; - auto [consumer, ccResult] = CreateConsumer(pqLib, ss); + auto [consumer, ccResult] = CreateConsumer(pqLib, ss); Cerr << ccResult.Response << "\n"; auto msg = consumer->GetNextMessage(); msg.Wait(); Cerr << msg.GetValue().Response << "\n"; UNIT_ASSERT(msg.GetValue().Response.response_case() == MigrationStreamingReadServerMessage::kAssigned); - UNIT_ASSERT(msg.GetValue().Response.assigned().topic().path() == SHORT_TOPIC_NAME); + UNIT_ASSERT(msg.GetValue().Response.assigned().topic().path() == SHORT_TOPIC_NAME); UNIT_ASSERT(msg.GetValue().Response.assigned().cluster() == "dc1"); msg = consumer->GetNextMessage(); @@ -1924,36 +1924,36 @@ namespace { THolder<IConsumer> consumer2; do { - std::tie(consumer2, ccResult) = CreateConsumer(pqLib, ss); + std::tie(consumer2, ccResult) = CreateConsumer(pqLib, ss); Cerr << ccResult.Response << "\n"; } while(!msg.Wait(TDuration::Seconds(1))); Cerr << msg.GetValue().Response << "\n"; UNIT_ASSERT(msg.GetValue().Response.response_case() == MigrationStreamingReadServerMessage::kRelease); - UNIT_ASSERT(msg.GetValue().Response.release().topic().path() == SHORT_TOPIC_NAME); + UNIT_ASSERT(msg.GetValue().Response.release().topic().path() == SHORT_TOPIC_NAME); UNIT_ASSERT(msg.GetValue().Response.release().cluster() == "dc1"); UNIT_ASSERT(msg.GetValue().Response.release().forceful_release() == false); msg = consumer->GetNextMessage(); UNIT_ASSERT(!msg.Wait(TDuration::Seconds(1))); - server.AnnoyingClient->RestartBalancerTablet(server.CleverServer->GetRuntime(), DEFAULT_TOPIC_NAME); + server.AnnoyingClient->RestartBalancerTablet(server.CleverServer->GetRuntime(), DEFAULT_TOPIC_NAME); UNIT_ASSERT(msg.Wait(TDuration::Seconds(1))); Cerr << msg.GetValue().Response << "\n"; UNIT_ASSERT(msg.GetValue().Response.response_case() == MigrationStreamingReadServerMessage::kRelease); - UNIT_ASSERT(msg.GetValue().Response.release().topic().path() == SHORT_TOPIC_NAME); + UNIT_ASSERT(msg.GetValue().Response.release().topic().path() == SHORT_TOPIC_NAME); UNIT_ASSERT(msg.GetValue().Response.release().cluster() == "dc1"); UNIT_ASSERT(msg.GetValue().Response.release().forceful_release() == true); THolder<TEvTabletCounters::TEvTabletLabeledCountersResponse> response; - TActorId edge = server.CleverServer->GetRuntime()->AllocateEdgeActor(); + TActorId edge = server.CleverServer->GetRuntime()->AllocateEdgeActor(); do { auto actorId = NKikimr::CreateClusterLabeledCountersAggregatorActor(edge, TTabletTypes::PersQueue, 3, "rt3.*--*,user*!!/!!*!!/rt3.*--*", 0); // remove !! - server.CleverServer->GetRuntime()->Register(actorId); - response = server.CleverServer->GetRuntime()-> + server.CleverServer->GetRuntime()->Register(actorId); + response = server.CleverServer->GetRuntime()-> GrabEdgeEvent<TEvTabletCounters::TEvTabletLabeledCountersResponse>(); Cerr << "FINAL RESPONSE :\n" << response->Record.DebugString() << Endl; @@ -1962,8 +1962,8 @@ namespace { Cerr << "MULITREQUEST\n"; auto actorId = NKikimr::CreateClusterLabeledCountersAggregatorActor(edge, TTabletTypes::PersQueue, 3, "rt3.*--*,user*!!/!!*!!/rt3.*--*", 3); // remove !! - server.CleverServer->GetRuntime()->Register(actorId); - response = server.CleverServer->GetRuntime()-> + server.CleverServer->GetRuntime()->Register(actorId); + response = server.CleverServer->GetRuntime()-> GrabEdgeEvent<TEvTabletCounters::TEvTabletLabeledCountersResponse>(); Cerr << "FINAL RESPONSE2 :\n" << response->Record.DebugString() << Endl; @@ -1986,9 +1986,9 @@ namespace { const auto& writeResponse = writeResult.GetValueSync().Response; UNIT_ASSERT_EQUAL_C(Ydb::StatusIds::SUCCESS, writeResponse.status(), "Response: " << writeResponse); - auto pqLib = TPQLib::WithCerrLogger(); + auto pqLib = TPQLib::WithCerrLogger(); using namespace NPersQueue; - auto [consumer, ccResult] = CreateConsumer(pqLib, setup.GetGrpcPort(), "shared/user", {"/account/topic" , {}}, 1000, false); + auto [consumer, ccResult] = CreateConsumer(pqLib, setup.GetGrpcPort(), "shared/user", {"/account/topic" , {}}, 1000, false); Cerr << ccResult.Response << "\n"; auto msg = consumer->GetNextMessage(); @@ -2025,17 +2025,17 @@ namespace { THolder<IProducer> producer = setup.StartProducer(topicPath, true); - auto pqLib = TPQLib::WithCerrLogger(); - - auto [consumer, ccResult] = CreateConsumer( - pqLib, setup.GetGrpcPort(), consumerPath, {topicPath , {}}, 1000, false - ); + auto pqLib = TPQLib::WithCerrLogger(); + + auto [consumer, ccResult] = CreateConsumer( + pqLib, setup.GetGrpcPort(), consumerPath, {topicPath , {}}, 1000, false + ); Cerr << ccResult.Response << "\n"; { auto msg = consumer->GetNextMessage(); msg.Wait(); - Cerr << "consumer assign response: " << msg.GetValue().Response << "\n"; + Cerr << "consumer assign response: " << msg.GetValue().Response << "\n"; UNIT_ASSERT(msg.GetValue().Type == EMT_ASSIGNED); msg.GetValue().StartRead.SetValue(TAssignInfo()); } @@ -2110,23 +2110,23 @@ namespace { return; } - NPersQueue::TTestServer server; - server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1); - server.WaitInit(SHORT_TOPIC_NAME); - server.EnableLogs({ NKikimrServices::PQ_READ_PROXY }); + NPersQueue::TTestServer server; + server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1); + server.WaitInit(SHORT_TOPIC_NAME); + server.EnableLogs({ NKikimrServices::PQ_READ_PROXY }); - server.AnnoyingClient->DeleteTopic2(DEFAULT_TOPIC_NAME, NPersQueue::NErrorCode::OK, false); - auto pqLib = TPQLib::WithCerrLogger(); + server.AnnoyingClient->DeleteTopic2(DEFAULT_TOPIC_NAME, NPersQueue::NErrorCode::OK, false); + auto pqLib = TPQLib::WithCerrLogger(); ui32 maxCount = 1; bool unpack = false; - auto [consumer, ccResult] = CreateConsumer(pqLib, server.GrpcPort, "user", {SHORT_TOPIC_NAME, {}}, maxCount, unpack); - Cerr << "Consumer create response: " << ccResult.Response << "\n"; + auto [consumer, ccResult] = CreateConsumer(pqLib, server.GrpcPort, "user", {SHORT_TOPIC_NAME, {}}, maxCount, unpack); + Cerr << "Consumer create response: " << ccResult.Response << "\n"; auto isDead = consumer->IsDead(); isDead.Wait(); - Cerr << "Is dead future value: " << isDead.GetValue() << "\n"; - UNIT_ASSERT_EQUAL(ccResult.Response.Getissues(0).issue_code(), Ydb::PersQueue::ErrorCode::UNKNOWN_TOPIC); + Cerr << "Is dead future value: " << isDead.GetValue() << "\n"; + UNIT_ASSERT_EQUAL(ccResult.Response.Getissues(0).issue_code(), Ydb::PersQueue::ErrorCode::UNKNOWN_TOPIC); } */ Y_UNIT_TEST(Codecs_InitWriteSession_DefaultTopicSupportedCodecsInInitResponse) { @@ -2352,26 +2352,26 @@ namespace { } Y_UNIT_TEST(SchemeOperationsTest) { - NPersQueue::TTestServer server; - server.EnableLogs({NKikimrServices::PQ_READ_PROXY, NKikimrServices::BLACKBOX_VALIDATOR }); - TString topic1 = "rt3.dc1--acc--topic1"; - TString topic3 = "rt3.dc1--acc--topic3"; - server.AnnoyingClient->CreateTopic(topic1, 1); - server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1); - server.AnnoyingClient->CreateConsumer("user"); + NPersQueue::TTestServer server; + server.EnableLogs({NKikimrServices::PQ_READ_PROXY, NKikimrServices::BLACKBOX_VALIDATOR }); + TString topic1 = "rt3.dc1--acc--topic1"; + TString topic3 = "rt3.dc1--acc--topic3"; + server.AnnoyingClient->CreateTopic(topic1, 1); + server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1); + server.AnnoyingClient->CreateConsumer("user"); std::shared_ptr<grpc::Channel> Channel_; std::unique_ptr<Ydb::PersQueue::V1::PersQueueService::Stub> StubP_; { - Channel_ = grpc::CreateChannel("localhost:" + ToString(server.GrpcPort), grpc::InsecureChannelCredentials()); + Channel_ = grpc::CreateChannel("localhost:" + ToString(server.GrpcPort), grpc::InsecureChannelCredentials()); StubP_ = Ydb::PersQueue::V1::PersQueueService::NewStub(Channel_); } do { CreateTopicRequest request; CreateTopicResponse response; - request.set_path(TStringBuilder() << "/Root/PQ/" << topic3); + request.set_path(TStringBuilder() << "/Root/PQ/" << topic3); auto props = request.mutable_settings(); props->set_partitions_count(1); props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE); @@ -2401,7 +2401,7 @@ namespace { // local cluster CreateTopicRequest request; CreateTopicResponse response; - request.set_path(TStringBuilder() << "/Root/PQ/" << topic3); + request.set_path(TStringBuilder() << "/Root/PQ/" << topic3); auto props = request.mutable_settings(); props->set_partitions_count(2); props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE); @@ -2419,7 +2419,7 @@ namespace { response.operation().result().UnpackTo(&res); Cerr << response << "\n" << res << "\n"; UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::SUCCESS); - server.AnnoyingClient->AddTopic(topic3); + server.AnnoyingClient->AddTopic(topic3); } auto alter =[&StubP_](const AlterTopicRequest& request, Ydb::StatusIds::StatusCode statusCode, bool auth) @@ -2440,7 +2440,7 @@ namespace { }; AlterTopicRequest request; - request.set_path(TStringBuilder() << "/Root/PQ/" << topic3); + request.set_path(TStringBuilder() << "/Root/PQ/" << topic3); auto props = request.mutable_settings(); props->set_partitions_count(1); props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE); @@ -2457,7 +2457,7 @@ namespace { props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1)); auto rr = props->add_read_rules(); alter(request, Ydb::StatusIds::BAD_REQUEST, false); - server.AnnoyingClient->AlterTopic(); + server.AnnoyingClient->AlterTopic(); props->add_supported_codecs(Ydb::PersQueue::V1::CODEC_RAW); props->add_supported_codecs(Ydb::PersQueue::V1::CODEC_ZSTD); @@ -2481,9 +2481,9 @@ namespace { (*props->mutable_attributes())["_partitions_per_tablet"] = "5"; alter(request, Ydb::StatusIds::SUCCESS, false); - TString topic4 = "rt3.dc1--acc--topic4"; - server.AnnoyingClient->CreateTopic(topic4, 1); //ensure creation - auto res = server.AnnoyingClient->DescribeTopic({topic3}); + TString topic4 = "rt3.dc1--acc--topic4"; + server.AnnoyingClient->CreateTopic(topic4, 1); //ensure creation + auto res = server.AnnoyingClient->DescribeTopic({topic3}); Cerr << res.DebugString(); TString resultDescribe = R"___(TopicInfo { Topic: "rt3.dc1--acc--topic3" @@ -2572,7 +2572,7 @@ namespace { { DescribeTopicRequest request; DescribeTopicResponse response; - request.set_path(TStringBuilder() << "/Root/PQ/" << topic3); + request.set_path(TStringBuilder() << "/Root/PQ/" << topic3); grpc::ClientContext rcontext; rcontext.AddMetadata("x-ydb-auth-ticket", "user@" BUILTIN_ACL_DOMAIN); @@ -2603,7 +2603,7 @@ namespace { { DescribeTopicRequest request; DescribeTopicResponse response; - request.set_path(TStringBuilder() << "/Root/PQ/" << topic3); + request.set_path(TStringBuilder() << "/Root/PQ/" << topic3); grpc::ClientContext rcontext; auto status = StubP_->DescribeTopic(&rcontext, request, &response); @@ -2623,7 +2623,7 @@ namespace { { DescribeTopicRequest request; DescribeTopicResponse response; - request.set_path(TStringBuilder() << "/Root/PQ/" << topic3); + request.set_path(TStringBuilder() << "/Root/PQ/" << topic3); grpc::ClientContext rcontext; auto status = StubP_->DescribeTopic(&rcontext, request, &response); @@ -2639,7 +2639,7 @@ namespace { { DropTopicRequest request; DropTopicResponse response; - request.set_path(TStringBuilder() << "/Root/PQ/" << topic3); + request.set_path(TStringBuilder() << "/Root/PQ/" << topic3); grpc::ClientContext rcontext; auto status = StubP_->DropTopic(&rcontext, request, &response); @@ -2648,15 +2648,15 @@ namespace { response.operation().result().UnpackTo(&res); Cerr << response << "\n" << res << "\n"; UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::SUCCESS); - server.AnnoyingClient->RemoveTopic(topic3); + server.AnnoyingClient->RemoveTopic(topic3); } { DropTopicRequest request; DropTopicResponse response; - request.set_path(TStringBuilder() << "/Root/PQ/" << topic3); - + request.set_path(TStringBuilder() << "/Root/PQ/" << topic3); + grpc::ClientContext rcontext; auto status = StubP_->DropTopic(&rcontext, request, &response); @@ -2667,13 +2667,13 @@ namespace { UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::SCHEME_ERROR); } - server.AnnoyingClient->CreateTopic("rt3.dc1--acc--topic5", 1); //ensure creation - server.AnnoyingClient->DescribeTopic({topic3}, true); + server.AnnoyingClient->CreateTopic("rt3.dc1--acc--topic5", 1); //ensure creation + server.AnnoyingClient->DescribeTopic({topic3}, true); { NYdb::TDriverConfig driverCfg; - driverCfg.SetEndpoint(TStringBuilder() << "localhost:" << server.GrpcPort); + driverCfg.SetEndpoint(TStringBuilder() << "localhost:" << server.GrpcPort); std::shared_ptr<NYdb::TDriver> ydbDriver(new NYdb::TDriver(driverCfg)); auto pqClient = NYdb::NPersQueue::TPersQueueClient(*ydbDriver); @@ -2686,18 +2686,18 @@ namespace { Y_UNIT_TEST(SchemeOperationsCheckPropValues) { - NPersQueue::TTestServer server; - server.EnableLogs({ NKikimrServices::PQ_READ_PROXY, NKikimrServices::BLACKBOX_VALIDATOR }); + NPersQueue::TTestServer server; + server.EnableLogs({ NKikimrServices::PQ_READ_PROXY, NKikimrServices::BLACKBOX_VALIDATOR }); - server.AnnoyingClient->CreateTopic("rt3.dc1--acc--topic1", 1); - server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1); - server.AnnoyingClient->CreateConsumer("user"); + server.AnnoyingClient->CreateTopic("rt3.dc1--acc--topic1", 1); + server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1); + server.AnnoyingClient->CreateConsumer("user"); std::shared_ptr<grpc::Channel> Channel_; std::unique_ptr<Ydb::PersQueue::V1::PersQueueService::Stub> StubP_; { - Channel_ = grpc::CreateChannel("localhost:" + ToString(server.GrpcPort), grpc::InsecureChannelCredentials()); + Channel_ = grpc::CreateChannel("localhost:" + ToString(server.GrpcPort), grpc::InsecureChannelCredentials()); StubP_ = Ydb::PersQueue::V1::PersQueueService::NewStub(Channel_); } @@ -2764,19 +2764,19 @@ namespace { } Y_UNIT_TEST(ReadRuleServiceType) { - TServerSettings settings = PQSettings(0); + TServerSettings settings = PQSettings(0); { settings.PQConfig.AddClientServiceType()->SetName("MyGreatType"); settings.PQConfig.AddClientServiceType()->SetName("AnotherType"); settings.PQConfig.AddClientServiceType()->SetName("SecondType"); } - NPersQueue::TTestServer server(settings); - server.EnableLogs({ NKikimrServices::PQ_READ_PROXY, NKikimrServices::BLACKBOX_VALIDATOR }); + NPersQueue::TTestServer server(settings); + server.EnableLogs({ NKikimrServices::PQ_READ_PROXY, NKikimrServices::BLACKBOX_VALIDATOR }); std::unique_ptr<Ydb::PersQueue::V1::PersQueueService::Stub> pqStub; { - std::shared_ptr<grpc::Channel> channel = grpc::CreateChannel("localhost:" + ToString(server.GrpcPort), grpc::InsecureChannelCredentials()); + std::shared_ptr<grpc::Channel> channel = grpc::CreateChannel("localhost:" + ToString(server.GrpcPort), grpc::InsecureChannelCredentials()); pqStub = Ydb::PersQueue::V1::PersQueueService::NewStub(channel); } auto checkDescribe = [&](const TVector<std::pair<TString, TString>>& readRules) { @@ -3166,16 +3166,16 @@ namespace { } Y_UNIT_TEST(ReadRuleServiceTypeMigration) { - TServerSettings settings = PQSettings(0); + TServerSettings settings = PQSettings(0); { settings.PQConfig.MutableDefaultClientServiceType()->SetName("default_type"); settings.PQConfig.AddClientServiceType()->SetName("MyGreatType"); settings.PQConfig.AddClientServiceType()->SetName("AnotherType"); settings.PQConfig.AddClientServiceType()->SetName("SecondType"); } - NPersQueue::TTestServer server(settings); + NPersQueue::TTestServer server(settings); - server.EnableLogs({ NKikimrServices::PQ_READ_PROXY, NKikimrServices::BLACKBOX_VALIDATOR }); + server.EnableLogs({ NKikimrServices::PQ_READ_PROXY, NKikimrServices::BLACKBOX_VALIDATOR }); const ui32 topicsCount = 4; for (ui32 i = 1; i <= topicsCount; ++i) { @@ -3188,7 +3188,7 @@ namespace { std::unique_ptr<Ydb::PersQueue::V1::PersQueueService::Stub> pqStub; { - std::shared_ptr<grpc::Channel> channel = grpc::CreateChannel("localhost:" + ToString(server.GrpcPort), grpc::InsecureChannelCredentials()); + std::shared_ptr<grpc::Channel> channel = grpc::CreateChannel("localhost:" + ToString(server.GrpcPort), grpc::InsecureChannelCredentials()); pqStub = Ydb::PersQueue::V1::PersQueueService::NewStub(channel); } auto doAlter = [&](const TString& topic, const TVector<std::pair<TString, TString>>& readRules) { diff --git a/ydb/services/persqueue_v1/persqueue_utils.cpp b/ydb/services/persqueue_v1/persqueue_utils.cpp index 7d21e2314a..51d7f38045 100644 --- a/ydb/services/persqueue_v1/persqueue_utils.cpp +++ b/ydb/services/persqueue_v1/persqueue_utils.cpp @@ -2,85 +2,85 @@ #include <ydb/core/base/path.h> -namespace NKikimr::NGRpcProxy::V1 { +namespace NKikimr::NGRpcProxy::V1 { -TAclWrapper::TAclWrapper(THolder<NACLib::TSecurityObject> acl) - : AclOldSchemeCache(std::move(acl)) -{ - Y_VERIFY(AclOldSchemeCache); -} +TAclWrapper::TAclWrapper(THolder<NACLib::TSecurityObject> acl) + : AclOldSchemeCache(std::move(acl)) +{ + Y_VERIFY(AclOldSchemeCache); +} + +TAclWrapper::TAclWrapper(TIntrusivePtr<TSecurityObject> acl) + : AclNewSchemeCache(std::move(acl)) +{ + Y_VERIFY(AclNewSchemeCache); +} -TAclWrapper::TAclWrapper(TIntrusivePtr<TSecurityObject> acl) - : AclNewSchemeCache(std::move(acl)) -{ - Y_VERIFY(AclNewSchemeCache); -} - -bool TAclWrapper::CheckAccess(NACLib::EAccessRights rights, const NACLib::TUserToken& userToken) { - if (AclOldSchemeCache) { - return AclOldSchemeCache->CheckAccess(rights, userToken); - } else { - return AclNewSchemeCache->CheckAccess(rights, userToken); +bool TAclWrapper::CheckAccess(NACLib::EAccessRights rights, const NACLib::TUserToken& userToken) { + if (AclOldSchemeCache) { + return AclOldSchemeCache->CheckAccess(rights, userToken); + } else { + return AclNewSchemeCache->CheckAccess(rights, userToken); } -} +} + +using namespace NSchemeCache; -using namespace NSchemeCache; - -TProcessingResult ProcessMetaCacheTopicResponse(const TSchemeCacheNavigate::TEntry& entry) { - auto fullPath = JoinPath(entry.Path); - auto& topicName = entry.Path.back(); - switch (entry.Status) { - case TSchemeCacheNavigate::EStatus::RootUnknown : { - return TProcessingResult { - Ydb::PersQueue::ErrorCode::ErrorCode::BAD_REQUEST, - Sprintf("path '%s' has unknown/invalid root prefix '%s', Marker# PQ14", - fullPath.c_str(), entry.Path[0].c_str()), - true - }; +TProcessingResult ProcessMetaCacheTopicResponse(const TSchemeCacheNavigate::TEntry& entry) { + auto fullPath = JoinPath(entry.Path); + auto& topicName = entry.Path.back(); + switch (entry.Status) { + case TSchemeCacheNavigate::EStatus::RootUnknown : { + return TProcessingResult { + Ydb::PersQueue::ErrorCode::ErrorCode::BAD_REQUEST, + Sprintf("path '%s' has unknown/invalid root prefix '%s', Marker# PQ14", + fullPath.c_str(), entry.Path[0].c_str()), + true + }; + } + case TSchemeCacheNavigate::EStatus::PathErrorUnknown: { + return TProcessingResult { + Ydb::PersQueue::ErrorCode::ErrorCode::UNKNOWN_TOPIC, + Sprintf("no path '%s', Marker# PQ15", fullPath.c_str()), + true + }; + } + case TSchemeCacheNavigate::EStatus::Ok: + break; + default: { + return TProcessingResult { + Ydb::PersQueue::ErrorCode::ErrorCode::ERROR, + Sprintf("topic '%s' describe error, Status# %s, Marker# PQ1", + topicName.c_str(), ToString(entry.Status).c_str()), + true + }; } - case TSchemeCacheNavigate::EStatus::PathErrorUnknown: { - return TProcessingResult { - Ydb::PersQueue::ErrorCode::ErrorCode::UNKNOWN_TOPIC, - Sprintf("no path '%s', Marker# PQ15", fullPath.c_str()), - true - }; - } - case TSchemeCacheNavigate::EStatus::Ok: - break; - default: { - return TProcessingResult { - Ydb::PersQueue::ErrorCode::ErrorCode::ERROR, - Sprintf("topic '%s' describe error, Status# %s, Marker# PQ1", - topicName.c_str(), ToString(entry.Status).c_str()), - true - }; - } } - if (entry.Kind != TSchemeCacheNavigate::KindTopic) { - return TProcessingResult { - Ydb::PersQueue::ErrorCode::ErrorCode::UNKNOWN_TOPIC, - Sprintf("item '%s' is not a topic, Marker# PQ13", fullPath.c_str()), - true - }; - } - if (!entry.PQGroupInfo) { - return TProcessingResult { - Ydb::PersQueue::ErrorCode::ErrorCode::ERROR, - Sprintf("topic '%s' describe error, reason: could not retrieve topic description, Marker# PQ99", - topicName.c_str()), - true - }; - } - auto& description = entry.PQGroupInfo->Description; - if (!description.HasBalancerTabletID() || description.GetBalancerTabletID() == 0) { - return TProcessingResult { - Ydb::PersQueue::ErrorCode::ErrorCode::UNKNOWN_TOPIC, - Sprintf("topic '%s' has no balancer, Marker# PQ193", topicName.c_str()), - true - }; - } - return {}; + if (entry.Kind != TSchemeCacheNavigate::KindTopic) { + return TProcessingResult { + Ydb::PersQueue::ErrorCode::ErrorCode::UNKNOWN_TOPIC, + Sprintf("item '%s' is not a topic, Marker# PQ13", fullPath.c_str()), + true + }; + } + if (!entry.PQGroupInfo) { + return TProcessingResult { + Ydb::PersQueue::ErrorCode::ErrorCode::ERROR, + Sprintf("topic '%s' describe error, reason: could not retrieve topic description, Marker# PQ99", + topicName.c_str()), + true + }; + } + auto& description = entry.PQGroupInfo->Description; + if (!description.HasBalancerTabletID() || description.GetBalancerTabletID() == 0) { + return TProcessingResult { + Ydb::PersQueue::ErrorCode::ErrorCode::UNKNOWN_TOPIC, + Sprintf("topic '%s' has no balancer, Marker# PQ193", topicName.c_str()), + true + }; + } + return {}; } - -} // namespace NKikimr::NGRpcProxy::V1 + +} // namespace NKikimr::NGRpcProxy::V1 diff --git a/ydb/services/persqueue_v1/persqueue_utils.h b/ydb/services/persqueue_v1/persqueue_utils.h index 188e47a1af..7860067982 100644 --- a/ydb/services/persqueue_v1/persqueue_utils.h +++ b/ydb/services/persqueue_v1/persqueue_utils.h @@ -1,32 +1,32 @@ #pragma once #include <ydb/public/api/protos/persqueue_error_codes_v1.pb.h> - + #include <ydb/library/aclib/aclib.h> #include <ydb/core/scheme/scheme_tabledefs.h> #include <ydb/core/base/counters.h> #include <ydb/core/tx/scheme_cache/scheme_cache.h> -namespace NKikimr::NGRpcProxy::V1 { - -class TAclWrapper { -public: - TAclWrapper(THolder<NACLib::TSecurityObject>); - TAclWrapper(TIntrusivePtr<TSecurityObject>); - - bool CheckAccess(NACLib::EAccessRights, const NACLib::TUserToken& userToken); - -private: - THolder<NACLib::TSecurityObject> AclOldSchemeCache; - TIntrusivePtr<TSecurityObject> AclNewSchemeCache; -}; - -struct TProcessingResult { - Ydb::PersQueue::ErrorCode::ErrorCode ErrorCode; - TString Reason; - bool IsFatal = false; -}; - -TProcessingResult ProcessMetaCacheTopicResponse(const NSchemeCache::TSchemeCacheNavigate::TEntry& entry); - -} //namespace NKikimr::NGRpcProxy::V1 +namespace NKikimr::NGRpcProxy::V1 { + +class TAclWrapper { +public: + TAclWrapper(THolder<NACLib::TSecurityObject>); + TAclWrapper(TIntrusivePtr<TSecurityObject>); + + bool CheckAccess(NACLib::EAccessRights, const NACLib::TUserToken& userToken); + +private: + THolder<NACLib::TSecurityObject> AclOldSchemeCache; + TIntrusivePtr<TSecurityObject> AclNewSchemeCache; +}; + +struct TProcessingResult { + Ydb::PersQueue::ErrorCode::ErrorCode ErrorCode; + TString Reason; + bool IsFatal = false; +}; + +TProcessingResult ProcessMetaCacheTopicResponse(const NSchemeCache::TSchemeCacheNavigate::TEntry& entry); + +} //namespace NKikimr::NGRpcProxy::V1 diff --git a/ydb/services/persqueue_v1/ut/api_test_setup.h b/ydb/services/persqueue_v1/ut/api_test_setup.h index 1e904b8ff9..8d5d82b33d 100644 --- a/ydb/services/persqueue_v1/ut/api_test_setup.h +++ b/ydb/services/persqueue_v1/ut/api_test_setup.h @@ -16,23 +16,23 @@ private: using TStreamingWriteClientMessage = Ydb::PersQueue::V1::StreamingWriteClientMessage; using TStreamingWriteServerMessage = Ydb::PersQueue::V1::StreamingWriteServerMessage; TLog Log; - NPersQueue::TTestServer server; + NPersQueue::TTestServer server; std::shared_ptr<::grpc::Channel> channel; std::unique_ptr<Ydb::PersQueue::V1::PersQueueService::Stub> service; public: APITestSetup(const TString& testCaseName) : Log("cerr") - , server(NPersQueue::TTestServer()) - , channel(grpc::CreateChannel("localhost:" + ToString(server.GrpcPort), grpc::InsecureChannelCredentials())) + , server(NPersQueue::TTestServer()) + , channel(grpc::CreateChannel("localhost:" + ToString(server.GrpcPort), grpc::InsecureChannelCredentials())) , service(Ydb::PersQueue::V1::PersQueueService::NewStub(channel)) { Log.SetFormatter([testCaseName](ELogPriority priority, TStringBuf message) { return TStringBuilder() << TInstant::Now() << " :" << testCaseName << " " << priority << ": " << message << Endl; }); - server.CleverServer->GetRuntime()->SetLogPriority(NKikimrServices::PQ_WRITE_PROXY, NActors::NLog::PRI_DEBUG); - server.CleverServer->GetRuntime()->SetLogPriority(NKikimrServices::PQ_READ_PROXY, NActors::NLog::PRI_DEBUG); - server.AnnoyingClient->CreateTopic("rt3.dc1--" + TString(GetTestTopic()), 1); - NKikimr::NPersQueueTests::TPQDataWriter writer(GetTestMessageGroupId(), server); + server.CleverServer->GetRuntime()->SetLogPriority(NKikimrServices::PQ_WRITE_PROXY, NActors::NLog::PRI_DEBUG); + server.CleverServer->GetRuntime()->SetLogPriority(NKikimrServices::PQ_READ_PROXY, NActors::NLog::PRI_DEBUG); + server.AnnoyingClient->CreateTopic("rt3.dc1--" + TString(GetTestTopic()), 1); + NKikimr::NPersQueueTests::TPQDataWriter writer(GetTestMessageGroupId(), server); auto seed = TInstant::Now().MicroSeconds(); // This makes failing randomized tests (for example with NUnitTest::RandomString(size, std::rand()) calls) reproducable @@ -65,7 +65,7 @@ public: } NKikimr::Tests::TServer& GetServer() { - return *server.CleverServer; + return *server.CleverServer; } TString GetPQRoot() { @@ -73,11 +73,11 @@ public: } NKikimrPQ::TPQConfig& GetPQConfig() { - return server.CleverServer->GetRuntime()->GetAppData().PQConfig; + return server.CleverServer->GetRuntime()->GetAppData().PQConfig; } NKikimr::NPersQueueTests::TFlatMsgBusPQClient& GetFlatMsgBusPQClient() { - return *server.AnnoyingClient; + return *server.AnnoyingClient; } std::unique_ptr<Ydb::PersQueue::V1::PersQueueService::Stub>& GetPersQueueService() { @@ -112,7 +112,7 @@ public: TServerMessage serverMessage; Log << TLOG_INFO << "Wait for \"init_response\""; AssertSuccessfullStreamingOperation(stream->Read(&serverMessage), stream); - Cerr << "Init response: " << serverMessage.ShortDebugString() << Endl; + Cerr << "Init response: " << serverMessage.ShortDebugString() << Endl; UNIT_ASSERT_C(serverMessage.server_message_case() == TServerMessage::kInitResponse, serverMessage); Log << TLOG_INFO << "Session ID is " << serverMessage.init_response().session_id().Quote(); return serverMessage; diff --git a/ydb/services/persqueue_v1/ut/persqueue_test_fixture.h b/ydb/services/persqueue_v1/ut/persqueue_test_fixture.h index 37432a7f7a..2933d1f3a1 100644 --- a/ydb/services/persqueue_v1/ut/persqueue_test_fixture.h +++ b/ydb/services/persqueue_v1/ut/persqueue_test_fixture.h @@ -16,15 +16,15 @@ namespace NKikimr::NPersQueueTests { - - -#define SET_LOCALS \ - auto& pqClient = server.Server->AnnoyingClient; \ - Y_UNUSED(pqClient); \ - auto* runtime = server.Server->CleverServer->GetRuntime(); \ - Y_UNUSED(runtime); \ - - + + +#define SET_LOCALS \ + auto& pqClient = server.Server->AnnoyingClient; \ + Y_UNUSED(pqClient); \ + auto* runtime = server.Server->CleverServer->GetRuntime(); \ + Y_UNUSED(runtime); \ + + using namespace Tests; using namespace NKikimrClient; using namespace Ydb::PersQueue; @@ -32,46 +32,46 @@ namespace NKikimr::NPersQueueTests { using namespace NThreading; using namespace NNetClassifier; - class TPersQueueV1TestServerBase { + class TPersQueueV1TestServerBase { public: - virtual void AlterSettings(NKikimr::Tests::TServerSettings& settings) { - Y_UNUSED(settings); - } - void InitializePQ() { - Cerr << "===Init PQ - create server\n"; - Y_VERIFY(Server == nullptr); - PortManager = new TPortManager(); - Server = MakeHolder<NPersQueue::TTestServer>(false, PortManager); - Server->ServerSettings.PQConfig.SetTopicsAreFirstClassCitizen(TenantModeEnabled()); - Server->ServerSettings.PQConfig.SetMetaCacheTimeoutSec(30); - AlterSettings(Server->ServerSettings); - Server->StartServer(false); + virtual void AlterSettings(NKikimr::Tests::TServerSettings& settings) { + Y_UNUSED(settings); + } + void InitializePQ() { + Cerr << "===Init PQ - create server\n"; + Y_VERIFY(Server == nullptr); + PortManager = new TPortManager(); + Server = MakeHolder<NPersQueue::TTestServer>(false, PortManager); + Server->ServerSettings.PQConfig.SetTopicsAreFirstClassCitizen(TenantModeEnabled()); + Server->ServerSettings.PQConfig.SetMetaCacheTimeoutSec(30); + AlterSettings(Server->ServerSettings); + Server->StartServer(false); if (TenantModeEnabled()) { - Server->AnnoyingClient->SetNoConfigMode(); - Server->ServerSettings.PQConfig.SetSourceIdTablePath("some unused path"); + Server->AnnoyingClient->SetNoConfigMode(); + Server->ServerSettings.PQConfig.SetSourceIdTablePath("some unused path"); } - Cerr << "Init PQ - start server on port " << Server->GrpcPort << Endl; - Server->GrpcServerOptions.SetMaxMessageSize(130 * 1024 * 1024); + Cerr << "Init PQ - start server on port " << Server->GrpcPort << Endl; + Server->GrpcServerOptions.SetMaxMessageSize(130 * 1024 * 1024); EnablePQLogs({NKikimrServices::PQ_READ_PROXY, NKikimrServices::PQ_WRITE_PROXY, NKikimrServices::FLAT_TX_SCHEMESHARD}); - EnablePQLogs({NKikimrServices::PERSQUEUE}, NLog::EPriority::PRI_INFO); + EnablePQLogs({NKikimrServices::PERSQUEUE}, NLog::EPriority::PRI_INFO); - Server->AnnoyingClient->FullInit(); - Server->AnnoyingClient->CreateConsumer("user"); + Server->AnnoyingClient->FullInit(); + Server->AnnoyingClient->CreateConsumer("user"); if (TenantModeEnabled()) { - Cerr << "=== Will create fst-class topics\n"; - Server->AnnoyingClient->CreateTopicNoLegacy("/Root/acc/topic1", 1); - Server->AnnoyingClient->CreateTopicNoLegacy("/Root/PQ/acc/topic1", 1); + Cerr << "=== Will create fst-class topics\n"; + Server->AnnoyingClient->CreateTopicNoLegacy("/Root/acc/topic1", 1); + Server->AnnoyingClient->CreateTopicNoLegacy("/Root/PQ/acc/topic1", 1); } else { - Cerr << "=== Will create legacy-style topics\n"; + Cerr << "=== Will create legacy-style topics\n"; Server->AnnoyingClient->CreateTopicNoLegacy("rt3.dc1--acc--topic2dc", 1); Server->AnnoyingClient->CreateTopicNoLegacy("rt3.dc2--acc--topic2dc", 1, true, false); Server->AnnoyingClient->CreateTopicNoLegacy("rt3.dc1--topic1", 1); - Server->AnnoyingClient->CreateTopicNoLegacy("rt3.dc1--acc--topic1", 1); - Server->WaitInit("topic1"); - Sleep(TDuration::Seconds(10)); + Server->AnnoyingClient->CreateTopicNoLegacy("rt3.dc1--acc--topic1", 1); + Server->WaitInit("topic1"); + Sleep(TDuration::Seconds(10)); } - InsecureChannel = grpc::CreateChannel("localhost:" + ToString(Server->GrpcPort), grpc::InsecureChannelCredentials()); + InsecureChannel = grpc::CreateChannel("localhost:" + ToString(Server->GrpcPort), grpc::InsecureChannelCredentials()); ServiceStub = Ydb::PersQueue::V1::PersQueueService::NewStub(InsecureChannel); InitializeWritePQService(TenantModeEnabled() ? "Root/acc/topic1" : "topic1"); @@ -85,7 +85,7 @@ namespace NKikimr::NPersQueueTests { NActors::NLog::EPriority prio = NActors::NLog::PRI_DEBUG) { for (auto s : services) { - Server->CleverServer->GetRuntime()->SetLogPriority(s, prio); + Server->CleverServer->GetRuntime()->SetLogPriority(s, prio); } } @@ -132,9 +132,9 @@ namespace NKikimr::NPersQueueTests { void ModifyTopicACL(const TString& topic, const NACLib::TDiffACL& acl) { if (TenantModeEnabled()) { TFsPath path(topic); - Server->AnnoyingClient->ModifyACL(path.Dirname(), path.Basename(), acl.SerializeAsString()); + Server->AnnoyingClient->ModifyACL(path.Dirname(), path.Basename(), acl.SerializeAsString()); } else { - Server->AnnoyingClient->ModifyACL("/Root/PQ", topic, acl.SerializeAsString()); + Server->AnnoyingClient->ModifyACL("/Root/PQ", topic, acl.SerializeAsString()); } WaitACLModification(); @@ -157,8 +157,8 @@ namespace NKikimr::NPersQueueTests { } public: - THolder<NPersQueue::TTestServer> Server; - TSimpleSharedPtr<TPortManager> PortManager; + THolder<NPersQueue::TTestServer> Server; + TSimpleSharedPtr<TPortManager> PortManager; std::shared_ptr<grpc::Channel> InsecureChannel; std::unique_ptr<Ydb::PersQueue::V1::PersQueueService::Stub> ServiceStub; @@ -166,61 +166,61 @@ namespace NKikimr::NPersQueueTests { THolder<NYdb::NPersQueue::TPersQueueClient> PersQueueClient; }; - class TPersQueueV1TestServer : public TPersQueueV1TestServerBase { + class TPersQueueV1TestServer : public TPersQueueV1TestServerBase { public: - TPersQueueV1TestServer(bool checkAcl = false) - : CheckACL(checkAcl) - { + TPersQueueV1TestServer(bool checkAcl = false) + : CheckACL(checkAcl) + { InitAll(); } void InitAll() { - InitializePQ(); + InitializePQ(); } - - void AlterSettings(NKikimr::Tests::TServerSettings& settings) override { - if (CheckACL) - settings.PQConfig.SetCheckACL(true); - } - private: - bool CheckACL; + + void AlterSettings(NKikimr::Tests::TServerSettings& settings) override { + if (CheckACL) + settings.PQConfig.SetCheckACL(true); + } + private: + bool CheckACL; }; - class TPersQueueV1TestServerWithRateLimiter : public TPersQueueV1TestServerBase { - private: - NKikimrPQ::TPQConfig::TQuotingConfig::ELimitedEntity LimitedEntity; + class TPersQueueV1TestServerWithRateLimiter : public TPersQueueV1TestServerBase { + private: + NKikimrPQ::TPQConfig::TQuotingConfig::ELimitedEntity LimitedEntity; public: - TPersQueueV1TestServerWithRateLimiter() - : TPersQueueV1TestServerBase() - {} - - void AlterSettings(NKikimr::Tests::TServerSettings& settings) override { - settings.PQConfig.MutableQuotingConfig()->SetEnableQuoting(true); - settings.PQConfig.MutableQuotingConfig()->SetTopicWriteQuotaEntityToLimit(LimitedEntity); - } - + TPersQueueV1TestServerWithRateLimiter() + : TPersQueueV1TestServerBase() + {} + + void AlterSettings(NKikimr::Tests::TServerSettings& settings) override { + settings.PQConfig.MutableQuotingConfig()->SetEnableQuoting(true); + settings.PQConfig.MutableQuotingConfig()->SetTopicWriteQuotaEntityToLimit(LimitedEntity); + } + void InitAll(NKikimrPQ::TPQConfig::TQuotingConfig::ELimitedEntity limitedEntity) { - LimitedEntity = limitedEntity; - InitializePQ(); + LimitedEntity = limitedEntity; + InitializePQ(); InitQuotingPaths(); } void InitQuotingPaths() { - Server->AnnoyingClient->MkDir("/Root", "PersQueue"); - Server->AnnoyingClient->MkDir("/Root/PersQueue", "System"); - Server->AnnoyingClient->MkDir("/Root/PersQueue/System", "Quoters"); + Server->AnnoyingClient->MkDir("/Root", "PersQueue"); + Server->AnnoyingClient->MkDir("/Root/PersQueue", "System"); + Server->AnnoyingClient->MkDir("/Root/PersQueue/System", "Quoters"); } void CreateTopicWithQuota(const TString& path, bool createKesus = true, double writeQuota = 1000.0) { TVector<TString> pathComponents = SplitPath(path); const TString account = pathComponents[0]; - const TString name = NPersQueue::BuildFullTopicName(path, "dc1"); + const TString name = NPersQueue::BuildFullTopicName(path, "dc1"); if (TenantModeEnabled()) { - Server->AnnoyingClient->CreateTopicNoLegacy("/Root/PQ/" + path, 1); + Server->AnnoyingClient->CreateTopicNoLegacy("/Root/PQ/" + path, 1); } else { Cerr << "Creating topic \"" << name << "\"" << Endl; - Server->AnnoyingClient->CreateTopicNoLegacy(name, 1); + Server->AnnoyingClient->CreateTopicNoLegacy(name, 1); } const TString rootPath = "/Root/PersQueue/System/Quoters"; @@ -228,12 +228,12 @@ namespace NKikimr::NPersQueueTests { if (createKesus) { Cerr << "Creating kesus \"" << account << "\"" << Endl; - const NMsgBusProxy::EResponseStatus createKesusResult = Server->AnnoyingClient->CreateKesus(rootPath, account); + const NMsgBusProxy::EResponseStatus createKesusResult = Server->AnnoyingClient->CreateKesus(rootPath, account); UNIT_ASSERT_C(createKesusResult == NMsgBusProxy::MSTATUS_OK, createKesusResult); - const auto statusCode = Server->AnnoyingClient->AddQuoterResource( - Server->CleverServer->GetRuntime(), kesusPath, "write-quota", writeQuota - ); + const auto statusCode = Server->AnnoyingClient->AddQuoterResource( + Server->CleverServer->GetRuntime(), kesusPath, "write-quota", writeQuota + ); UNIT_ASSERT_EQUAL_C(statusCode, Ydb::StatusIds::SUCCESS, "Status: " << Ydb::StatusIds::StatusCode_Name(statusCode)); } @@ -243,9 +243,9 @@ namespace NKikimr::NPersQueueTests { for (auto currentComponent = pathComponents.begin() + 1; currentComponent != pathComponents.end(); ++currentComponent) { prefixPath << "/" << *currentComponent; Cerr << "Adding quoter resource: \"" << prefixPath << "\"" << Endl; - const auto statusCode = Server->AnnoyingClient->AddQuoterResource( - Server->CleverServer->GetRuntime(), kesusPath, prefixPath - ); + const auto statusCode = Server->AnnoyingClient->AddQuoterResource( + Server->CleverServer->GetRuntime(), kesusPath, prefixPath + ); UNIT_ASSERT_C(statusCode == Ydb::StatusIds::SUCCESS || statusCode == Ydb::StatusIds::ALREADY_EXISTS, "Status: " << Ydb::StatusIds::StatusCode_Name(statusCode)); } } @@ -254,7 +254,7 @@ namespace NKikimr::NPersQueueTests { THolder<IProducer> StartProducer(const TString& topicPath, bool compress = false) { TString fullPath = TenantModeEnabled() ? "/Root/PQ/" + topicPath : topicPath; TProducerSettings producerSettings; - producerSettings.Server = TServerSetting("localhost", Server->GrpcPort); + producerSettings.Server = TServerSetting("localhost", Server->GrpcPort); producerSettings.Topic = fullPath; producerSettings.SourceId = "TRateLimiterTestSetupSourceId"; producerSettings.Codec = compress ? "gzip" : "raw"; diff --git a/ydb/services/persqueue_v1/ut/pq_data_writer.h b/ydb/services/persqueue_v1/ut/pq_data_writer.h index 79195ef0f7..caef894060 100644 --- a/ydb/services/persqueue_v1/ut/pq_data_writer.h +++ b/ydb/services/persqueue_v1/ut/pq_data_writer.h @@ -13,11 +13,11 @@ namespace NKikimr::NPersQueueTests { class TPQDataWriter { public: - TPQDataWriter(const TString& sourceId, NPersQueue::TTestServer& server, const TString& testTopicPath = "topic1") + TPQDataWriter(const TString& sourceId, NPersQueue::TTestServer& server, const TString& testTopicPath = "topic1") : SourceId_(sourceId) - , Port_(server.GrpcPort) - , Client(*server.AnnoyingClient) - , Runtime(server.CleverServer->GetRuntime()) + , Port_(server.GrpcPort) + , Client(*server.AnnoyingClient) + , Runtime(server.CleverServer->GetRuntime()) { InitializeChannel(); WaitWritePQServiceInitialization(testTopicPath); @@ -42,7 +42,7 @@ public: req.mutable_init_request()->add_topics_read_settings()->set_topic(topic); req.mutable_init_request()->mutable_read_params()->set_max_read_messages_count(1); - req.mutable_init_request()->set_read_only_original(true); + req.mutable_init_request()->set_read_only_original(true); req.mutable_init_request()->set_consumer(clientId); @@ -58,7 +58,7 @@ public: Y_FAIL(""); UNIT_ASSERT(error); } - Cerr << "=== Got response: " << resp.ShortDebugString() << Endl; + Cerr << "=== Got response: " << resp.ShortDebugString() << Endl; if (error) { UNIT_ASSERT(resp.response_case() == Ydb::PersQueue::V1::MigrationStreamingReadServerMessage::RESPONSE_NOT_SET); return; diff --git a/ydb/services/persqueue_v1/ut/rate_limiter_test_setup.cpp b/ydb/services/persqueue_v1/ut/rate_limiter_test_setup.cpp index 08145ddd57..779d60e778 100644 --- a/ydb/services/persqueue_v1/ut/rate_limiter_test_setup.cpp +++ b/ydb/services/persqueue_v1/ut/rate_limiter_test_setup.cpp @@ -6,7 +6,7 @@ using namespace NPersQueue; using namespace Ydb::PersQueue; using namespace NKikimr::Tests; - + namespace NKikimr::NPersQueueTests { TRateLimiterTestSetup::TRateLimiterTestSetup( @@ -15,7 +15,7 @@ TRateLimiterTestSetup::TRateLimiterTestSetup( double readAccountQuota, bool enableReadQuoting ) - : Server(new NPersQueue::TTestServer(false)) + : Server(new NPersQueue::TTestServer(false)) , LimitedEntity(limitedEntity) , WriteAccountQuota(writeAccountQuota) , ReadAccountQuota(readAccountQuota) @@ -28,7 +28,7 @@ void TRateLimiterTestSetup::CreateTopic(const TString& path) { const TString account = GetAccount(name); Cerr << "Creating topic \"" << name << "\"" << Endl; - Server->AnnoyingClient->CreateTopic(name, 1); + Server->AnnoyingClient->CreateTopic(name, 1); CreateKesus(account); CreateQuotaResources(path, "write-quota", false); @@ -39,7 +39,7 @@ void TRateLimiterTestSetup::CreateConsumer(const TString& path) { const TString account = GetAccount(path); Cerr << "Creating consumer \"" << path << "\"" << Endl; - Server->AnnoyingClient->CreateConsumer(path); + Server->AnnoyingClient->CreateConsumer(path); CreateKesus(account); CreateQuotaResources(path, "write-quota", true); @@ -47,7 +47,7 @@ void TRateLimiterTestSetup::CreateConsumer(const TString& path) { } void TRateLimiterTestSetup::CreateKesus(const TString& account) { - const NMsgBusProxy::EResponseStatus createKesusResult = Server->AnnoyingClient->CreateKesus(QuotersRootPath, account); + const NMsgBusProxy::EResponseStatus createKesusResult = Server->AnnoyingClient->CreateKesus(QuotersRootPath, account); UNIT_ASSERT_C(createKesusResult == NMsgBusProxy::MSTATUS_OK, createKesusResult); const TString kesusPath = TStringBuilder() << QuotersRootPath << "/" << account; @@ -55,9 +55,9 @@ void TRateLimiterTestSetup::CreateKesus(const TString& account) { Cerr << "Creating kesus with path=" << kesusPath << Endl; auto setAccountQuota = [&](const TString& quotaPrefix, double value) { Cerr << "Adding quota for account kesus=" << kesusPath << " quota-path=" << quotaPrefix << " value=" << value << Endl; - const auto statusCode = Server->AnnoyingClient->AddQuoterResource( - Server->CleverServer->GetRuntime(), kesusPath, quotaPrefix, value - ); + const auto statusCode = Server->AnnoyingClient->AddQuoterResource( + Server->CleverServer->GetRuntime(), kesusPath, quotaPrefix, value + ); UNIT_ASSERT_C( statusCode == Ydb::StatusIds::SUCCESS || statusCode == Ydb::StatusIds::ALREADY_EXISTS, "Status: " << Ydb::StatusIds::StatusCode_Name(statusCode) @@ -83,9 +83,9 @@ void TRateLimiterTestSetup::CreateQuotaResources(const TString& path, const TStr for (auto currentComponent = firstIt; currentComponent != lastIt; ++currentComponent) { prefixPath << "/" << *currentComponent; Cerr << "Adding quoter resource: \"" << prefixPath << "\"" << Endl; - const auto statusCode = Server->AnnoyingClient->AddQuoterResource( - Server->CleverServer->GetRuntime(), kesusPath, prefixPath - ); + const auto statusCode = Server->AnnoyingClient->AddQuoterResource( + Server->CleverServer->GetRuntime(), kesusPath, prefixPath + ); UNIT_ASSERT_C( statusCode == Ydb::StatusIds::SUCCESS || statusCode == Ydb::StatusIds::ALREADY_EXISTS, "Status: " << Ydb::StatusIds::StatusCode_Name(statusCode) @@ -94,13 +94,13 @@ void TRateLimiterTestSetup::CreateQuotaResources(const TString& path, const TStr } /* -THolder<Ydb::PersQueue::IProducer> TRateLimiterTestSetup::StartProducer(const TString& topicPath, bool compress) { - Ydb::PersQueue::TProducerSettings producerSettings; - producerSettings.Server = Ydb::PersQueue::TServerSetting("localhost", Server->GrpcPort); +THolder<Ydb::PersQueue::IProducer> TRateLimiterTestSetup::StartProducer(const TString& topicPath, bool compress) { + Ydb::PersQueue::TProducerSettings producerSettings; + producerSettings.Server = Ydb::PersQueue::TServerSetting("localhost", Server->GrpcPort); producerSettings.Topic = topicPath; producerSettings.SourceId = "TRateLimiterTestSetupSourceId"; producerSettings.Codec = compress ? "gzip" : "raw"; - THolder<Ydb::PersQueue::IProducer> producer = PQLib->CreateProducer(producerSettings); + THolder<Ydb::PersQueue::IProducer> producer = PQLib->CreateProducer(producerSettings); auto startResult = producer->Start(); UNIT_ASSERT_EQUAL_C(Ydb::StatusIds::SUCCESS, startResult.GetValueSync().Response.status(), "Response: " << startResult.GetValueSync().Response); return producer; @@ -114,15 +114,15 @@ void TRateLimiterTestSetup::Start(bool enableReadQuoting) { } void TRateLimiterTestSetup::InitServer(bool enableReadQuoting) { - auto& settings = Server->ServerSettings; + auto& settings = Server->ServerSettings; - settings.PQConfig.MutableQuotingConfig()->SetEnableQuoting(true); - settings.PQConfig.MutableQuotingConfig()->SetEnableReadQuoting(enableReadQuoting); - settings.PQConfig.MutableQuotingConfig()->SetTopicWriteQuotaEntityToLimit(LimitedEntity); + settings.PQConfig.MutableQuotingConfig()->SetEnableQuoting(true); + settings.PQConfig.MutableQuotingConfig()->SetEnableReadQuoting(enableReadQuoting); + settings.PQConfig.MutableQuotingConfig()->SetTopicWriteQuotaEntityToLimit(LimitedEntity); - Server->GrpcServerOptions.SetMaxMessageSize(130 * 1024 * 1024); - Server->StartServer(); - Server->EnableLogs( + Server->GrpcServerOptions.SetMaxMessageSize(130 * 1024 * 1024); + Server->StartServer(); + Server->EnableLogs( { NKikimrServices::PQ_READ_PROXY, NKikimrServices::PQ_WRITE_PROXY, @@ -137,13 +137,13 @@ void TRateLimiterTestSetup::InitServer(bool enableReadQuoting) { } void TRateLimiterTestSetup::InitQuoting() { - Server->AnnoyingClient->MkDir("/Root", "PersQueue"); - Server->AnnoyingClient->MkDir("/Root/PersQueue", "System"); - Server->AnnoyingClient->MkDir("/Root/PersQueue/System", "Quoters"); + Server->AnnoyingClient->MkDir("/Root", "PersQueue"); + Server->AnnoyingClient->MkDir("/Root/PersQueue", "System"); + Server->AnnoyingClient->MkDir("/Root/PersQueue/System", "Quoters"); } void TRateLimiterTestSetup::WaitWritePQServiceInitialization() { - PQDataWriter = MakeHolder<TPQDataWriter>("writer_source_id", *Server); + PQDataWriter = MakeHolder<TPQDataWriter>("writer_source_id", *Server); } } diff --git a/ydb/services/persqueue_v1/ut/rate_limiter_test_setup.h b/ydb/services/persqueue_v1/ut/rate_limiter_test_setup.h index ecd7ce9299..01529818e1 100644 --- a/ydb/services/persqueue_v1/ut/rate_limiter_test_setup.h +++ b/ydb/services/persqueue_v1/ut/rate_limiter_test_setup.h @@ -23,19 +23,19 @@ public: // Getters ui16 GetGrpcPort() const { - return Server->GrpcPort; + return Server->GrpcPort; } ui16 GetMsgBusPort() const { - return Server->Port; + return Server->Port; } NKikimr::Tests::TServer& GetServer() { - return *Server->CleverServer; + return *Server->CleverServer; } NPersQueueTests::TFlatMsgBusPQClient& GetClient() { - return *Server->AnnoyingClient; + return *Server->AnnoyingClient; } private: @@ -49,7 +49,7 @@ private: void WaitWritePQServiceInitialization(); private: - THolder<NPersQueue::TTestServer> Server; + THolder<NPersQueue::TTestServer> Server; THolder<TPQDataWriter> PQDataWriter; // For waiting for grpc writer service initialization. const NKikimrPQ::TPQConfig::TQuotingConfig::ELimitedEntity LimitedEntity; double WriteAccountQuota; diff --git a/ydb/services/persqueue_v1/ut/test_utils.h b/ydb/services/persqueue_v1/ut/test_utils.h index f4d95edca3..c02308b0d7 100644 --- a/ydb/services/persqueue_v1/ut/test_utils.h +++ b/ydb/services/persqueue_v1/ut/test_utils.h @@ -33,7 +33,7 @@ inline void WaitACLModification() { TServer server = TServer(settings);\ server.EnableGRpc(NGrpc::TServerOptions().SetHost("localhost").SetPort(grpc));\ TFlatMsgBusPQClient client(settings, grpc);\ - client.FullInit();\ + client.FullInit();\ client.CreateTopic("rt3.dc1--" + TString(topic), 1);\ EnableLogs(server, { NKikimrServices::PQ_WRITE_PROXY });\ TPQDataWriter writer(messageGroupId, grpc, client, server.GetRuntime());\ diff --git a/ydb/services/ydb/ydb_common_ut.h b/ydb/services/ydb/ydb_common_ut.h index 9aef73f1bd..c067928c78 100644 --- a/ydb/services/ydb/ydb_common_ut.h +++ b/ydb/services/ydb/ydb_common_ut.h @@ -80,9 +80,9 @@ public: ServerSettings->SetEnableYq(enableYq); ServerSettings->Formats = new TFormatFactory; ServerSettings->PQConfig = appConfig.GetPQConfig(); - if (appConfig.HasMeteringConfig() && appConfig.GetMeteringConfig().HasMeteringFilePath()) { - ServerSettings->SetMeteringFilePath(appConfig.GetMeteringConfig().GetMeteringFilePath()); - } + if (appConfig.HasMeteringConfig() && appConfig.GetMeteringConfig().HasMeteringFilePath()) { + ServerSettings->SetMeteringFilePath(appConfig.GetMeteringConfig().GetMeteringFilePath()); + } ServerSettings->RegisterGrpcService<NKikimr::NGRpcService::TGRpcYdbDummyService>("dummy"); if (udfFrFactory) { ServerSettings->SetFrFactory(udfFrFactory); diff --git a/ydb/tests/functional/blobstorage/replication.py b/ydb/tests/functional/blobstorage/replication.py index 49a67190ed..cdf6df6f3f 100644 --- a/ydb/tests/functional/blobstorage/replication.py +++ b/ydb/tests/functional/blobstorage/replication.py @@ -1,18 +1,18 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -import pytest +#!/usr/bin/env python +# -*- coding: utf-8 -*- +import pytest from hamcrest import is_ - + from ydb.tests.library.common.types import Erasure from ydb.tests.library.common.wait_for import wait_for_and_assert from ydb.tests.library.harness.kikimr_cluster import kikimr_cluster_factory from ydb.tests.library.harness.kikimr_config import KikimrConfigGenerator from ydb.tests.library.predicates import blobstorage - - + + TIMEOUT_SECONDS = 480 - - + + class TestReplicationAfterNodesRestart(object): def __setup_cluster(self, erasure): self.cluster = kikimr_cluster_factory(configurator=KikimrConfigGenerator(erasure=erasure)) @@ -23,15 +23,15 @@ class TestReplicationAfterNodesRestart(object): self.cluster.stop() @pytest.mark.parametrize('erasure', Erasure.common_used(), ids=str) - def test_replication(self, erasure): - # Arrange + def test_replication(self, erasure): + # Arrange self.__setup_cluster(erasure) - - # Act + + # Act for node in self.cluster.nodes.values(): - node.stop() - node.start() - + node.stop() + node.start() + wait_for_and_assert( lambda: blobstorage.cluster_has_no_unsynced_vdisks(self.cluster), is_(True), timeout_seconds=TIMEOUT_SECONDS, message='All vdisks are sync' diff --git a/ydb/tests/functional/blobstorage/ya.make b/ydb/tests/functional/blobstorage/ya.make index aed9c64738..3248849d57 100644 --- a/ydb/tests/functional/blobstorage/ya.make +++ b/ydb/tests/functional/blobstorage/ya.make @@ -1,7 +1,7 @@ OWNER(g:kikimr) - + PY3TEST() - + ENV(YDB_DRIVER_BINARY="ydb/apps/ydbd/ydbd") TEST_SRCS( pdisk_format_info.py @@ -9,7 +9,7 @@ TEST_SRCS( self_heal.py tablet_channel_migration.py ) - + IF (SANITIZER_TYPE) REQUIREMENTS( cpu:4 @@ -26,19 +26,19 @@ ELSE() TIMEOUT(600) SIZE(MEDIUM) ENDIF() - + SPLIT_FACTOR(20) DEPENDS( ydb/apps/ydbd ) - + PEERDIR( ydb/tests/library contrib/python/PyHamcrest ) - + FORK_SUBTESTS() FORK_TEST_FILES() - + END() diff --git a/ydb/tests/functional/hive/test_kill_tablets.py b/ydb/tests/functional/hive/test_kill_tablets.py index 3ed6aa6d0c..08dbd5c0b1 100644 --- a/ydb/tests/functional/hive/test_kill_tablets.py +++ b/ydb/tests/functional/hive/test_kill_tablets.py @@ -32,7 +32,7 @@ class TestKillTablets(object): self.cluster.client.tablet_kill(tablet_id) generations = {info.TabletId: info.Generation for info in actual_tablet_info} - wait_tablets_state_by_id( + wait_tablets_state_by_id( self.cluster.client, TabletStates.Active, tablet_ids=all_created_tablet_ids, @@ -57,7 +57,7 @@ class TestKillTablets(object): is_ok_response() ) - wait_tablets_state_by_id( + wait_tablets_state_by_id( self.cluster.client, TabletStates.Active, tablet_ids=[hive_id], @@ -67,7 +67,7 @@ class TestKillTablets(object): all_tablet_ids = create_tablets_and_wait_for_start(self.cluster.client, NUMBER_OF_TABLETS) # Assert - wait_tablets_state_by_id( + wait_tablets_state_by_id( self.cluster.client, TabletStates.Active, tablet_ids=all_tablet_ids, diff --git a/ydb/tests/functional/hive/ya.make b/ydb/tests/functional/hive/ya.make index b5838a7262..60c5fdbf13 100644 --- a/ydb/tests/functional/hive/ya.make +++ b/ydb/tests/functional/hive/ya.make @@ -37,7 +37,7 @@ DEPENDS( PEERDIR( ydb/tests/library ) - + FORK_SUBTESTS() FORK_TEST_FILES() diff --git a/ydb/tests/functional/sqs/sqs_matchers.py b/ydb/tests/functional/sqs/sqs_matchers.py index 1e0295e168..a8bb8508fc 100644 --- a/ydb/tests/functional/sqs/sqs_matchers.py +++ b/ydb/tests/functional/sqs/sqs_matchers.py @@ -1,147 +1,147 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -from hamcrest import equal_to, less_than_or_equal_to, contains_inanyorder, contains, greater_than_or_equal_to, all_of, \ - has_item -from hamcrest.core.base_matcher import BaseMatcher - +#!/usr/bin/env python +# -*- coding: utf-8 -*- +from hamcrest import equal_to, less_than_or_equal_to, contains_inanyorder, contains, greater_than_or_equal_to, all_of, \ + has_item +from hamcrest.core.base_matcher import BaseMatcher + from ydb.tests.library.matchers.collection import IsSublistOf - -_author__ = 'komels@yandex-team.ru' - - -def extract_field(read_response, field): - if read_response is None: - return [] - return [i[field] for i in read_response] - - -def extract_message_ids(read_response): - return extract_field(read_response, 'MessageId') - - -def extract_message_data(read_response): - return extract_field(read_response, 'Body') - - -class ReadResponseMatcher(BaseMatcher): - def __init__(self): - self.__message_count_matcher = None - self.__message_ids_matcher = None - self.__msg_id_to_data = {} - self.__message_ids = None - - self.__messages_data = None - self.__absent_message_id = None - self.__description = "read result " - self.__mismatch_reason = "" - - @staticmethod - def __list_startswith(full_list, sublist): - for i, v in enumerate(sublist): - if full_list[i] != v: - return False - return True - - def with_n_messages(self, messages_count): - # self.__description += " with {} messages".format(messages_count) - self.__message_count_matcher = equal_to(messages_count) - return self - - def with_n_of_fewer_messages(self, messages_count): - # self.__description += " with {} or fewer messages".format(messages_count) - self.__message_count_matcher = less_than_or_equal_to(messages_count) - return self - + +_author__ = 'komels@yandex-team.ru' + + +def extract_field(read_response, field): + if read_response is None: + return [] + return [i[field] for i in read_response] + + +def extract_message_ids(read_response): + return extract_field(read_response, 'MessageId') + + +def extract_message_data(read_response): + return extract_field(read_response, 'Body') + + +class ReadResponseMatcher(BaseMatcher): + def __init__(self): + self.__message_count_matcher = None + self.__message_ids_matcher = None + self.__msg_id_to_data = {} + self.__message_ids = None + + self.__messages_data = None + self.__absent_message_id = None + self.__description = "read result " + self.__mismatch_reason = "" + + @staticmethod + def __list_startswith(full_list, sublist): + for i, v in enumerate(sublist): + if full_list[i] != v: + return False + return True + + def with_n_messages(self, messages_count): + # self.__description += " with {} messages".format(messages_count) + self.__message_count_matcher = equal_to(messages_count) + return self + + def with_n_of_fewer_messages(self, messages_count): + # self.__description += " with {} or fewer messages".format(messages_count) + self.__message_count_matcher = less_than_or_equal_to(messages_count) + return self + def with_n_or_more_messages(self, messages_count): - self.__message_count_matcher = greater_than_or_equal_to(messages_count) - return self - - def with_message_ids(self, message_ids): - self.__message_ids_matcher = contains_inanyorder(*message_ids) - self.__message_ids = message_ids - return self - - def with_some_of_message_ids(self, message_ids): - self.__message_ids = message_ids - self.__message_ids_matcher = IsSublistOf(message_ids) - return self - - def with_these_or_more_message_ids(self, message_ids): - self.__message_ids = message_ids - self.__message_ids_matcher = all_of( - *[has_item(m_id) for m_id in message_ids] - ) - return self - - def in_order(self): - self.__description += " with exact message_id order:\n {}\n" - if not self.__message_ids: - raise ValueError("Called in_order, but message_ids to match are not set") - self.__message_ids_matcher = contains(self.__message_ids) - return self - - def with_messages_data(self, messages_data): - if self.__message_ids is None: - raise ValueError("Must set message_ids to verify message data") - if len(self.__message_ids) != len(messages_data): - raise ValueError("Must provide message data for same count of messages as message ids") - for i, msg_id in enumerate(self.__message_ids): - self.__msg_id_to_data[msg_id] = messages_data[i] - # self.__description += " with messages containing following data:\n {}\n".format(';'.join(messages_data)) - return self - - def without_message_id(self, message_id): - self.__description += " without message_id: {}".format(message_id) - self.__absent_message_id = message_id - return self - - def describe_to(self, description): - description.append_text(self.__description) - if self.__message_count_matcher is not None: - description.append_text("with messages count: ") - self.__message_count_matcher.describe_to(description) - description.append_text('\n') - if self.__message_ids_matcher is not None: - description.append_text("with message_ids: ") - self.__message_ids_matcher.describe_to(description) - description.append_text('\n') - if self.__msg_id_to_data: - description.append_text("with message_data:\n{}\n".format(self.__msg_id_to_data.values())) - - def describe_mismatch(self, actual_response, mismatch_description): - mismatch_description.append_text("actual result was: messages\n{}\n".format( - extract_message_ids(actual_response) - )) - # message_ids = extract_message_ids(actual_response) - # if self.__message_count_matcher is not None: - # self.__message_count_matcher.describe_mismatch(message_ids, mismatch_description) - # if self.__message_ids_matcher is not None: - # self.__message_ids_matcher.describe_mismatch(len(message_ids), mismatch_description) - if self.__msg_id_to_data: - mismatch_description.append_text("and data: {}\n".format(self.__msg_id_to_data.values())) - if self.__mismatch_reason: - mismatch_description.append_text("Mismatch reason was: {}".format(self.__mismatch_reason)) - - def _matches(self, actual_response): - actual_message_ids = extract_message_ids(actual_response) - if len(actual_message_ids) != len(set(actual_message_ids)): - self.__mismatch_reason += "Duplicate messages appear in result" - return False - if self.__message_count_matcher is not None: - messages_count = len(actual_message_ids) - if not self.__message_count_matcher.matches(messages_count): - self.__mismatch_reason += "Messages count didn't match\n" - return False - if self.__absent_message_id is not None and self.__absent_message_id is actual_message_ids: - self.__mismatch_reason += "Message with {} appeared in results" - return False - if self.__message_ids_matcher is not None and not self.__message_ids_matcher.matches(actual_message_ids): - self.__mismatch_reason += "Message ids didn't match" - return False - if self.__msg_id_to_data: - actual_data = extract_message_data(actual_response) - expected_data = [self.__msg_id_to_data[i] for i in actual_message_ids] - if sorted(actual_data) != sorted(expected_data): - self.__mismatch_reason += "Message data didn't match" - return False - return True + self.__message_count_matcher = greater_than_or_equal_to(messages_count) + return self + + def with_message_ids(self, message_ids): + self.__message_ids_matcher = contains_inanyorder(*message_ids) + self.__message_ids = message_ids + return self + + def with_some_of_message_ids(self, message_ids): + self.__message_ids = message_ids + self.__message_ids_matcher = IsSublistOf(message_ids) + return self + + def with_these_or_more_message_ids(self, message_ids): + self.__message_ids = message_ids + self.__message_ids_matcher = all_of( + *[has_item(m_id) for m_id in message_ids] + ) + return self + + def in_order(self): + self.__description += " with exact message_id order:\n {}\n" + if not self.__message_ids: + raise ValueError("Called in_order, but message_ids to match are not set") + self.__message_ids_matcher = contains(self.__message_ids) + return self + + def with_messages_data(self, messages_data): + if self.__message_ids is None: + raise ValueError("Must set message_ids to verify message data") + if len(self.__message_ids) != len(messages_data): + raise ValueError("Must provide message data for same count of messages as message ids") + for i, msg_id in enumerate(self.__message_ids): + self.__msg_id_to_data[msg_id] = messages_data[i] + # self.__description += " with messages containing following data:\n {}\n".format(';'.join(messages_data)) + return self + + def without_message_id(self, message_id): + self.__description += " without message_id: {}".format(message_id) + self.__absent_message_id = message_id + return self + + def describe_to(self, description): + description.append_text(self.__description) + if self.__message_count_matcher is not None: + description.append_text("with messages count: ") + self.__message_count_matcher.describe_to(description) + description.append_text('\n') + if self.__message_ids_matcher is not None: + description.append_text("with message_ids: ") + self.__message_ids_matcher.describe_to(description) + description.append_text('\n') + if self.__msg_id_to_data: + description.append_text("with message_data:\n{}\n".format(self.__msg_id_to_data.values())) + + def describe_mismatch(self, actual_response, mismatch_description): + mismatch_description.append_text("actual result was: messages\n{}\n".format( + extract_message_ids(actual_response) + )) + # message_ids = extract_message_ids(actual_response) + # if self.__message_count_matcher is not None: + # self.__message_count_matcher.describe_mismatch(message_ids, mismatch_description) + # if self.__message_ids_matcher is not None: + # self.__message_ids_matcher.describe_mismatch(len(message_ids), mismatch_description) + if self.__msg_id_to_data: + mismatch_description.append_text("and data: {}\n".format(self.__msg_id_to_data.values())) + if self.__mismatch_reason: + mismatch_description.append_text("Mismatch reason was: {}".format(self.__mismatch_reason)) + + def _matches(self, actual_response): + actual_message_ids = extract_message_ids(actual_response) + if len(actual_message_ids) != len(set(actual_message_ids)): + self.__mismatch_reason += "Duplicate messages appear in result" + return False + if self.__message_count_matcher is not None: + messages_count = len(actual_message_ids) + if not self.__message_count_matcher.matches(messages_count): + self.__mismatch_reason += "Messages count didn't match\n" + return False + if self.__absent_message_id is not None and self.__absent_message_id is actual_message_ids: + self.__mismatch_reason += "Message with {} appeared in results" + return False + if self.__message_ids_matcher is not None and not self.__message_ids_matcher.matches(actual_message_ids): + self.__mismatch_reason += "Message ids didn't match" + return False + if self.__msg_id_to_data: + actual_data = extract_message_data(actual_response) + expected_data = [self.__msg_id_to_data[i] for i in actual_message_ids] + if sorted(actual_data) != sorted(expected_data): + self.__mismatch_reason += "Message data didn't match" + return False + return True diff --git a/ydb/tests/functional/sqs/sqs_requests_client.py b/ydb/tests/functional/sqs/sqs_requests_client.py index c32598a4e0..6b7545503f 100644 --- a/ydb/tests/functional/sqs/sqs_requests_client.py +++ b/ydb/tests/functional/sqs/sqs_requests_client.py @@ -3,9 +3,9 @@ import itertools import logging -import urllib -import requests -import xmltodict +import urllib +import requests +import xmltodict import six from ydb.tests.library.common.helpers import wrap_in_list @@ -83,7 +83,7 @@ class SqsHttpApi(object): self.__auth_headers = auth_headers(user, security_token, iam_token) if not server.startswith('http'): server = 'http://' + server - self.__request = requests.Request( + self.__request = requests.Request( 'POST', "{}:{}".format(server, port), headers=auth_headers(user, security_token, iam_token) @@ -93,8 +93,8 @@ class SqsHttpApi(object): "{}:{}/private".format(server, port), headers=auth_headers(user, security_token, iam_token) ) - self.__session = requests.Session() - self.__raise_on_error = raise_on_error + self.__session = requests.Session() + self.__raise_on_error = raise_on_error self.__user = user self.__timeout = timeout self.__security_token = security_token @@ -117,9 +117,9 @@ class SqsHttpApi(object): request.data = urllib.parse.urlencode(params) logger.debug("Execute request {} {} from user {} with params: {}".format( request.method, request.url, self.__user, request.data) - ) + ) prep = request.prepare() - try: + try: response = self.__session.send(prep, timeout=self.__timeout) except (requests.ConnectionError, requests.exceptions.Timeout) as ex: logging.debug("Request failed with connection exception {}: {}".format(type(ex), ex)) @@ -127,44 +127,44 @@ class SqsHttpApi(object): raise else: response = None - return self._process_response( - response, - extract_result_method, default - ) + return self._process_response( + response, + extract_result_method, default + ) - def _process_response(self, response, extract_method, default=None): - if response is None: + def _process_response(self, response, extract_method, default=None): + if response is None: logging.debug('Returning {} by default'.format(default)) - return default - if response.status_code != 200: + return default + if response.status_code != 200: logger.warn("Last request failed with code {}, reason '{}' and text '{}'".format( response.status_code, response.reason, response.text )) # Assert that no internal info will be given to user assert response.text.find('.cpp:') == -1, 'No internal info should be given to user' - if self.__raise_on_error: - raise RuntimeError( - "Request {} failed with status {} and text {}".format( - self.__request.data, response.status_code, response.text - ) - ) - return default + if self.__raise_on_error: + raise RuntimeError( + "Request {} failed with status {} and text {}".format( + self.__request.data, response.status_code, response.text + ) + ) + return default logging.debug('Parsing response: {}'.format(response.text)) - result = xmltodict.parse(response.text) + result = xmltodict.parse(response.text) try: - return extract_method(result) + return extract_method(result) except (KeyError, TypeError) as ex: logger.error("Could not process response from SQS: {}. {}: {}".format(result, type(ex), ex)) - return default + return default - def create_user(self, username): + def create_user(self, username): return self.execute_request( - action='CreateUser', - extract_result_method=lambda x: x['CreateUserResponse']['ResponseMetadata'], - UserName=username - ) - + action='CreateUser', + extract_result_method=lambda x: x['CreateUserResponse']['ResponseMetadata'], + UserName=username + ) + def delete_user(self, username): return self.execute_request( action='DeleteUser', @@ -172,24 +172,24 @@ class SqsHttpApi(object): UserName=username ) - def list_users(self): + def list_users(self): return self.execute_request( - action='ListUsers', + action='ListUsers', extract_result_method=lambda x: wrap_in_list(x['ListUsersResponse']['ListUsersResult']['UserName']) - ) - - def list_queues(self, name_prefix=None): - if name_prefix is not None: + ) + + def list_queues(self, name_prefix=None): + if name_prefix is not None: return self.execute_request( - action='ListQueues', + action='ListQueues', extract_result_method=lambda x: wrap_in_list(x['ListQueuesResponse']['ListQueuesResult']['QueueUrl']), default=(), - QueueNamePrefix=name_prefix - ) - else: + QueueNamePrefix=name_prefix + ) + else: return self.execute_request( - action='ListQueues', + action='ListQueues', extract_result_method=lambda x: wrap_in_list(x['ListQueuesResponse']['ListQueuesResult']['QueueUrl']), default=(), - ) + ) def private_count_queues(self): return self.execute_request( @@ -198,32 +198,32 @@ class SqsHttpApi(object): extract_result_method=lambda x: x['CountQueuesResponse']['CountQueuesResult']['Count'], ) - def create_queue(self, queue_name, is_fifo=False, attributes=None): - # if is_fifo and not queue_name.endswith('.fifo'): - # return None - if attributes is None: + def create_queue(self, queue_name, is_fifo=False, attributes=None): + # if is_fifo and not queue_name.endswith('.fifo'): + # return None + if attributes is None: attributes = dict() - if is_fifo: + if is_fifo: attributes = dict(attributes) # copy - attributes['FifoQueue'] = 'true' - params = {} - for i, (k, v) in enumerate(attributes.items()): - params['Attribute.{id}.Name'.format(id=i+1)] = k - params['Attribute.{id}.Value'.format(id=i + 1)] = v - + attributes['FifoQueue'] = 'true' + params = {} + for i, (k, v) in enumerate(attributes.items()): + params['Attribute.{id}.Name'.format(id=i+1)] = k + params['Attribute.{id}.Value'.format(id=i + 1)] = v + return self.execute_request( - action='CreateQueue', - extract_result_method=lambda x: x['CreateQueueResponse']['CreateQueueResult']['QueueUrl'], - QueueName=queue_name, - **params - ) + action='CreateQueue', + extract_result_method=lambda x: x['CreateQueueResponse']['CreateQueueResult']['QueueUrl'], + QueueName=queue_name, + **params + ) - def delete_queue(self, queue_url): + def delete_queue(self, queue_url): return self.execute_request( - action='DeleteQueue', - extract_result_method=lambda x: x['DeleteQueueResponse']['ResponseMetadata']['RequestId'], - QueueUrl=queue_url - ) + action='DeleteQueue', + extract_result_method=lambda x: x['DeleteQueueResponse']['ResponseMetadata']['RequestId'], + QueueUrl=queue_url + ) def private_delete_queue_batch(self, queue_urls): args = {} @@ -260,10 +260,10 @@ class SqsHttpApi(object): def get_queue_url(self, queue_name): return self.execute_request( - action='GetQueueUrl', - extract_result_method=lambda x: x['GetQueueUrlResponse']['GetQueueUrlResult']['QueueUrl'], - QueueName=queue_name - ) + action='GetQueueUrl', + extract_result_method=lambda x: x['GetQueueUrlResponse']['GetQueueUrlResult']['QueueUrl'], + QueueName=queue_name + ) def list_dead_letter_source_queues(self, queue_url): return self.execute_request( @@ -383,10 +383,10 @@ class SqsHttpApi(object): )] = attr.value return self.execute_request( - action='SendMessage', - extract_result_method=lambda x: x['SendMessageResponse']['SendMessageResult']['MessageId'], - **params - ) + action='SendMessage', + extract_result_method=lambda x: x['SendMessageResponse']['SendMessageResult']['MessageId'], + **params + ) def send_message_batch(self, queue_url, send_message_params_list): params = { @@ -470,31 +470,31 @@ class SqsHttpApi(object): params['ReceiveRequestAttemptId'] = receive_request_attempt_id return self.execute_request( - action='ReceiveMessage', - extract_result_method=lambda x: wrap_in_list( - x['ReceiveMessageResponse']['ReceiveMessageResult']['Message'] - ), - **params + action='ReceiveMessage', + extract_result_method=lambda x: wrap_in_list( + x['ReceiveMessageResponse']['ReceiveMessageResult']['Message'] + ), + **params ) def delete_message(self, queue_url, handle): return self.execute_request( - action='DeleteMessage', - extract_result_method=lambda x: x['DeleteMessageResponse']['ResponseMetadata']['RequestId'], - QueueUrl=queue_url, ReceiptHandle=handle - ) - - def delete_message_batch(self, queue_url, message_handles): - args = {} - for i, handle in enumerate(message_handles): + action='DeleteMessage', + extract_result_method=lambda x: x['DeleteMessageResponse']['ResponseMetadata']['RequestId'], + QueueUrl=queue_url, ReceiptHandle=handle + ) + + def delete_message_batch(self, queue_url, message_handles): + args = {} + for i, handle in enumerate(message_handles): args["DeleteMessageBatchRequestEntry.{}.Id".format(i+1)] = str(i) - args["DeleteMessageBatchRequestEntry.{}.ReceiptHandle".format(i+1)] = handle - + args["DeleteMessageBatchRequestEntry.{}.ReceiptHandle".format(i+1)] = handle + resp = self.execute_request( - action='DeleteMessageBatch', + action='DeleteMessageBatch', extract_result_method=lambda x: x['DeleteMessageBatchResponse'], - QueueUrl=queue_url, **args - ) + QueueUrl=queue_url, **args + ) result = [dict() for i in six.moves.range(len(message_handles))] results = resp.get('DeleteMessageBatchResult') logging.debug('results: {}'.format(results)) @@ -506,7 +506,7 @@ class SqsHttpApi(object): i = int(res['Id']) assert i < len(result) and i >= 0 result[i]['DeleteMessageBatchResultEntry'] = res - + errors = results.get('BatchResultErrorEntry') logging.debug('errors: {}'.format(errors)) if errors: @@ -515,7 +515,7 @@ class SqsHttpApi(object): assert i < len(result) and i >= 0 result[i]['BatchResultErrorEntry'] = err return result - + def change_message_visibility(self, queue_url, handle, visibility_timeout): return self.execute_request( action='ChangeMessageVisibility', @@ -557,19 +557,19 @@ class SqsHttpApi(object): return result -class SqsHttpMinigunApi(SqsHttpApi): - def _process_response(self, response, extract_method, default=None): - if response is None: - return -1, default - try: - parsed = xmltodict.parse(response.text) - except xmltodict.expat.ExpatError: - return response.status_code, default - - if response.status_code != 200: - extract_method = lambda x: x['ErrorResponse']['Error']['Message'] - try: - result = extract_method(parsed) - except (KeyError, TypeError): - result = default - return response.status_code, result +class SqsHttpMinigunApi(SqsHttpApi): + def _process_response(self, response, extract_method, default=None): + if response is None: + return -1, default + try: + parsed = xmltodict.parse(response.text) + except xmltodict.expat.ExpatError: + return response.status_code, default + + if response.status_code != 200: + extract_method = lambda x: x['ErrorResponse']['Error']['Message'] + try: + result = extract_method(parsed) + except (KeyError, TypeError): + result = default + return response.status_code, result diff --git a/ydb/tests/functional/sqs/sqs_test_base.py b/ydb/tests/functional/sqs/sqs_test_base.py index 12b98ad1b8..05d5303f64 100644 --- a/ydb/tests/functional/sqs/sqs_test_base.py +++ b/ydb/tests/functional/sqs/sqs_test_base.py @@ -24,7 +24,7 @@ from concurrent import futures from sqs_matchers import ReadResponseMatcher -DEFAULT_VISIBILITY_TIMEOUT = 30 +DEFAULT_VISIBILITY_TIMEOUT = 30 logger = logging.getLogger(__name__) @@ -138,15 +138,15 @@ def get_test_with_sqs_installation_by_path(base_test_class): return TestWithPath -class KikimrSqsTestBase(object): +class KikimrSqsTestBase(object): erasure = None slot_count = 0 database = '/Root' sqs_root = '/Root/SQS' use_in_memory_pdisks = True - - @classmethod - def setup_class(cls): + + @classmethod + def setup_class(cls): cls.cluster, cls.config_generator = cls._setup_cluster() cls.sqs_ports = [] if cls.slot_count: @@ -158,8 +158,8 @@ class KikimrSqsTestBase(object): cls.sqs_ports.append(node.sqs_port) cls.sqs_port = cls.sqs_ports[0] - cls.server_fqdn = get_fqdn() - + cls.server_fqdn = get_fqdn() + def setup_method(self, method=None): logging.debug('Test started: {}'.format(str(method.__name__))) logging.debug("Kikimr logs dir: {}".format(self.cluster.slots[1].cwd if self.slot_count else self.cluster.nodes[1].cwd)) @@ -216,8 +216,8 @@ class KikimrSqsTestBase(object): self.message_ids = [] self.read_result = [] - self.seq_no = 0 - + self.seq_no = 0 + def teardown_method(self, method=None): self.check_no_queues_table(self._username) self._driver.stop() @@ -246,14 +246,14 @@ class KikimrSqsTestBase(object): assert_that(raised) - @classmethod - def teardown_class(cls): - if hasattr(cls, 'cluster'): - cls.cluster.stop() - + @classmethod + def teardown_class(cls): + if hasattr(cls, 'cluster'): + cls.cluster.stop() + @classmethod def _setup_config_generator(cls): - config_generator = KikimrConfigGenerator( + config_generator = KikimrConfigGenerator( erasure=cls.erasure, use_in_memory_pdisks=cls.use_in_memory_pdisks, additional_log_configs={'SQS': LogLevels.INFO}, @@ -268,7 +268,7 @@ class KikimrSqsTestBase(object): config_generator.yaml_config['sqs_config']['check_all_shards_in_receive_message'] = True config_generator.yaml_config['sqs_config']['create_legacy_duration_counters'] = False config_generator.yaml_config['sqs_config']['validate_message_body'] = True - + return config_generator @classmethod @@ -313,17 +313,17 @@ class KikimrSqsTestBase(object): '-u', 'metauser', '-n', _username, ] + self._sqs_server_opts - while retries_count: + while retries_count: logging.debug("Running {}".format(' '.join(cmd))) - try: + try: yatest_common.execute(cmd) except yatest_common.ExecutionError as ex: logging.debug("Create user failed: {}. Retrying".format(ex)) - retries_count -= 1 + retries_count -= 1 time.sleep(3) - else: + else: return - raise RuntimeError("Failed to create SQS user") + raise RuntimeError("Failed to create SQS user") def _create_api_for_user(self, user_name, raise_on_error=True, security_token=None, force_private=False, iam_token=None, folder_id=None): api = SqsHttpApi(self.cluster.nodes[1].host, @@ -368,7 +368,7 @@ class KikimrSqsTestBase(object): else: raise - def _queue_url_matcher(self, queue_name): + def _queue_url_matcher(self, queue_name): urls_matchers = [ equal_to( to_bytes( @@ -388,9 +388,9 @@ class KikimrSqsTestBase(object): logging.debug('Create queue. Attributes: {}. Use http: {}. Is fifo: {}'.format(attributes, use_http, is_fifo)) assert (len(attributes.keys()) == 0 or use_http), 'Attributes are supported only for http queue creation' assert (shards is None or not use_http), 'Custom shards number is only supported in non-http mode' - while retries: + while retries: retries -= 1 - try: + try: if use_http: self.queue_url = self._sqs_api.create_queue(queue_name, is_fifo=is_fifo, attributes=attributes) else: @@ -412,52 +412,52 @@ class KikimrSqsTestBase(object): else: raise if self.queue_url is not None: # queue_url will be None in case of connection error - break + break assert_that( to_bytes(self.queue_url), - self._queue_url_matcher(queue_name) + self._queue_url_matcher(queue_name) ) return self.queue_url def _send_message_and_assert(self, queue_url, msg_body, seq_no=None, group_id=None, attributes=None, delay_seconds=None): attributes = {} if attributes is None else attributes - send_msg_result = self._sqs_api.send_message( + send_msg_result = self._sqs_api.send_message( queue_url, msg_body, deduplication_id=seq_no, group_id=group_id, attributes=attributes, delay_seconds=delay_seconds - ) + ) assert_that( send_msg_result, not_none() ) return send_msg_result - def _send_messages(self, queue_url, message_count, msg_body_template=None, is_fifo=False, group_id=None): - if msg_body_template is None: - msg_body_template = self._msg_body_template + def _send_messages(self, queue_url, message_count, msg_body_template=None, is_fifo=False, group_id=None): + if msg_body_template is None: + msg_body_template = self._msg_body_template ret = [] for _ in range(message_count): - if is_fifo: - result = self._send_message_and_assert( + if is_fifo: + result = self._send_message_and_assert( queue_url, msg_body_template.format(next(self.counter)), seq_no=self.seq_no, group_id=group_id - ) - self.seq_no += 1 - else: + ) + self.seq_no += 1 + else: result = self._send_message_and_assert(queue_url, msg_body_template.format(next(self.counter))) ret.append(result) return ret def _read_while_not_empty(self, queue_url, messages_count, visibility_timeout=None, wait_timeout=1, max_empty_reads=1): - ret = [] - messages_by_time = {} - actual_vis_timeout = visibility_timeout if visibility_timeout is not None else DEFAULT_VISIBILITY_TIMEOUT + ret = [] + messages_by_time = {} + actual_vis_timeout = visibility_timeout if visibility_timeout is not None else DEFAULT_VISIBILITY_TIMEOUT empty_reads_count = 0 max_batch_to_read = self.config_generator.yaml_config['sqs_config']['max_number_of_receive_messages'] - while len(ret) < messages_count: - # noinspection PyTypeChecker + while len(ret) < messages_count: + # noinspection PyTypeChecker request_start = time.time() - read_result = self._sqs_api.receive_message( + read_result = self._sqs_api.receive_message( queue_url, max_number_of_messages=min(messages_count - len(ret), max_batch_to_read), visibility_timeout=visibility_timeout, wait_timeout=wait_timeout - ) - if not read_result: + ) + if not read_result: empty_reads_count += 1 if empty_reads_count == max_empty_reads: break @@ -477,38 +477,38 @@ class KikimrSqsTestBase(object): ) else: raise AssertionError("Message {} appeared twice before visibility timeout expired".format(msg_id)) - return ret - - def _read_messages_and_assert( + return ret + + def _read_messages_and_assert( self, queue_url, messages_count, matcher=None, visibility_timeout=None, wait_timeout=1 ): read_result = self._read_while_not_empty( - queue_url, messages_count=messages_count, + queue_url, messages_count=messages_count, visibility_timeout=visibility_timeout, wait_timeout=wait_timeout ) - if matcher is not None: - assert_that( - read_result, matcher - ) + if matcher is not None: + assert_that( + read_result, matcher + ) return read_result - def _create_queue_send_x_messages_read_y_messages( - self, queue_name, send_count, read_count, msg_body_template, + def _create_queue_send_x_messages_read_y_messages( + self, queue_name, send_count, read_count, msg_body_template, is_fifo=False, visibility_timeout=None, wait_timeout=1, group_id="1" ): self._create_queue_and_assert(queue_name, is_fifo) if is_fifo: - self.message_ids = self._send_messages( - self.queue_url, send_count, msg_body_template, is_fifo=True, group_id=group_id - ) - else: - self.message_ids = self._send_messages( - self.queue_url, send_count, msg_body_template - ) - self.read_result = self._read_messages_and_assert( - self.queue_url, read_count, - ReadResponseMatcher().with_some_of_message_ids(self.message_ids).with_n_messages(read_count), - visibility_timeout, wait_timeout + self.message_ids = self._send_messages( + self.queue_url, send_count, msg_body_template, is_fifo=True, group_id=group_id + ) + else: + self.message_ids = self._send_messages( + self.queue_url, send_count, msg_body_template + ) + self.read_result = self._read_messages_and_assert( + self.queue_url, read_count, + ReadResponseMatcher().with_some_of_message_ids(self.message_ids).with_n_messages(read_count), + visibility_timeout, wait_timeout ) def _other_node(self, node_index): @@ -516,7 +516,7 @@ class KikimrSqsTestBase(object): return 1 else: return node_index - 1 - + def _get_live_node_index(self): for i in range(self.cluster_nodes_count): if self.slot_count: @@ -534,12 +534,12 @@ class KikimrSqsTestBase(object): return self.config_generator.port_allocator.get_node_port_allocator(node_index + 1).mon_port def _get_sqs_counters(self, node_index=0, counters_format='json'): - return self._get_counters(node_index, "sqs", counters_format) - - def _get_ymq_counters(self, cloud, folder, node_index=0, counters_format='json'): - return self._get_counters(node_index, "ymq_public", counters_format, cloud=cloud, folder=folder) - - def _get_counters(self, node_index, component, counters_format, cloud=None, folder=None): + return self._get_counters(node_index, "sqs", counters_format) + + def _get_ymq_counters(self, cloud, folder, node_index=0, counters_format='json'): + return self._get_counters(node_index, "ymq_public", counters_format, cloud=cloud, folder=folder) + + def _get_counters(self, node_index, component, counters_format, cloud=None, folder=None): mon_port = self._get_mon_port(node_index) if counters_format == 'json': @@ -549,13 +549,13 @@ class KikimrSqsTestBase(object): else: raise Exception('Unknown counters format: \"{}\"'.format(counters_format)) - if folder is not None: - labels="/cloud%3D{cloud}/folder%3D{folder}".format(cloud=cloud, folder=folder) - else: - labels = '' - counters_url = 'http://localhost:{port}/counters/counters%3D{component}{labels}{suffix}'.format( - port=mon_port, component=component, suffix=format_suffix, labels=labels - ) + if folder is not None: + labels="/cloud%3D{cloud}/folder%3D{folder}".format(cloud=cloud, folder=folder) + else: + labels = '' + counters_url = 'http://localhost:{port}/counters/counters%3D{component}{labels}{suffix}'.format( + port=mon_port, component=component, suffix=format_suffix, labels=labels + ) reply = requests.get(counters_url) assert_that(reply.status_code, equal_to(200)) diff --git a/ydb/tests/functional/sqs/test_fifo_messaging.py b/ydb/tests/functional/sqs/test_fifo_messaging.py index 1fd35885e7..6410b21966 100644 --- a/ydb/tests/functional/sqs/test_fifo_messaging.py +++ b/ydb/tests/functional/sqs/test_fifo_messaging.py @@ -3,10 +3,10 @@ import logging import time -import pytest +import pytest from hamcrest import assert_that, equal_to, not_none, greater_than, less_than_or_equal_to, has_items, raises -from sqs_matchers import ReadResponseMatcher, extract_message_ids +from sqs_matchers import ReadResponseMatcher, extract_message_ids from sqs_test_base import KikimrSqsTestBase, get_test_with_sqs_installation_by_path, get_test_with_sqs_tenant_installation, VISIBILITY_CHANGE_METHOD_PARAMS @@ -41,41 +41,41 @@ class TestSqsFifoMicroBatchesWithPath(get_test_with_sqs_installation_by_path(Sqs class SqsFifoMessagingTest(KikimrSqsTestBase): - def setup_method(self, method=None): + def setup_method(self, method=None): super(SqsFifoMessagingTest, self).setup_method(method) - self.queue_name = self.queue_name + ".fifo" - + self.queue_name = self.queue_name + ".fifo" + @classmethod def _setup_config_generator(cls): config_generator = super(SqsFifoMessagingTest, cls)._setup_config_generator() config_generator.yaml_config['sqs_config']['group_selection_batch_size'] = 100 return config_generator - def test_only_single_read_infly_from_fifo(self): - self._create_queue_send_x_messages_read_y_messages( + def test_only_single_read_infly_from_fifo(self): + self._create_queue_send_x_messages_read_y_messages( self.queue_name, send_count=10, read_count=1, visibility_timeout=1000, msg_body_template=self._msg_body_template, is_fifo=True - ) - self._read_messages_and_assert( + ) + self._read_messages_and_assert( self.queue_url, messages_count=10, matcher=ReadResponseMatcher().with_n_messages(0) - ) - - def test_fifo_read_delete_single_message(self): - created_queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=True) - message_ids = self._send_messages( + ) + + def test_fifo_read_delete_single_message(self): + created_queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=True) + message_ids = self._send_messages( created_queue_url, message_count=10, msg_body_template=self._msg_body_template, is_fifo=True, group_id='group' - ) - read_result = self._read_messages_and_assert( + ) + read_result = self._read_messages_and_assert( self.queue_url, messages_count=1, matcher=ReadResponseMatcher().with_message_ids(message_ids[:1]) - ) - handle = read_result[0]['ReceiptHandle'] - assert_that( - self._sqs_api.delete_message(self.queue_url, handle), not_none() - ) - self._read_messages_and_assert( + ) + handle = read_result[0]['ReceiptHandle'] + assert_that( + self._sqs_api.delete_message(self.queue_url, handle), not_none() + ) + self._read_messages_and_assert( self.queue_url, messages_count=1, matcher=ReadResponseMatcher().with_message_ids(message_ids[1:2]) - ) - + ) + counters = self._get_sqs_counters() delete_counter_labels = { 'subsystem': 'core', @@ -85,129 +85,129 @@ class SqsFifoMessagingTest(KikimrSqsTestBase): } assert_that(self._get_counter_value(counters, delete_counter_labels, 0), equal_to(1)) - def test_write_and_read_to_different_groups(self): - seq_no = 1 - self._create_queue_and_assert(self.queue_name, is_fifo=True) - message_ids = [] - for group_id in range(10): - msg_id = self._send_message_and_assert( - self.queue_url, self._msg_body_template.format('1'), seq_no, str(group_id) - ) - seq_no += 1 - message_ids.append(msg_id) - - result = self._read_messages_and_assert( + def test_write_and_read_to_different_groups(self): + seq_no = 1 + self._create_queue_and_assert(self.queue_name, is_fifo=True) + message_ids = [] + for group_id in range(10): + msg_id = self._send_message_and_assert( + self.queue_url, self._msg_body_template.format('1'), seq_no, str(group_id) + ) + seq_no += 1 + message_ids.append(msg_id) + + result = self._read_messages_and_assert( self.queue_url, 10, visibility_timeout=1000, matcher=ReadResponseMatcher().with_n_messages(10) - ) - received_message_ids = extract_message_ids(result) - assert_that( - sorted(received_message_ids), equal_to(sorted(message_ids)) - ) - - def test_can_read_from_different_groups(self): - seq_no = 1 - self._create_queue_and_assert(self.queue_name, is_fifo=True) - message_ids = [] - for group_id in range(20): - msg_id = self._send_message_and_assert( - self.queue_url, self._msg_body_template.format(group_id), seq_no, str(group_id) - ) - seq_no += 1 - message_ids.append(msg_id) - - first_message_ids = extract_message_ids( - self._read_messages_and_assert( + ) + received_message_ids = extract_message_ids(result) + assert_that( + sorted(received_message_ids), equal_to(sorted(message_ids)) + ) + + def test_can_read_from_different_groups(self): + seq_no = 1 + self._create_queue_and_assert(self.queue_name, is_fifo=True) + message_ids = [] + for group_id in range(20): + msg_id = self._send_message_and_assert( + self.queue_url, self._msg_body_template.format(group_id), seq_no, str(group_id) + ) + seq_no += 1 + message_ids.append(msg_id) + + first_message_ids = extract_message_ids( + self._read_messages_and_assert( self.queue_url, 10, ReadResponseMatcher().with_n_messages(10), visibility_timeout=1000 - ) - ) - second_message_ids = extract_message_ids( - self._read_messages_and_assert( + ) + ) + second_message_ids = extract_message_ids( + self._read_messages_and_assert( self.queue_url, 10, ReadResponseMatcher().with_n_messages(10), visibility_timeout=1000 - ) - ) - - assert_that( - len(set(first_message_ids + second_message_ids)), - equal_to(len(first_message_ids) + len(second_message_ids)) - ) - self._read_messages_and_assert( + ) + ) + + assert_that( + len(set(first_message_ids + second_message_ids)), + equal_to(len(first_message_ids) + len(second_message_ids)) + ) + self._read_messages_and_assert( self.queue_url, 10, visibility_timeout=1000, matcher=ReadResponseMatcher().with_n_messages(0) - ) - - def test_send_and_read_multiple_messages(self): - queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=True) - first_message_id = self._send_message_and_assert( + ) + + def test_send_and_read_multiple_messages(self): + queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=True) + first_message_id = self._send_message_and_assert( queue_url, self._msg_body_template.format('0'), seq_no=1, group_id='group' - ) + ) time.sleep(5) - self._send_message_and_assert( + self._send_message_and_assert( queue_url, self._msg_body_template.format('1'), seq_no=2, group_id='group' - ) - self._read_messages_and_assert( + ) + self._read_messages_and_assert( queue_url, messages_count=1, - matcher=ReadResponseMatcher().with_message_ids([first_message_id, ]) - ) - - def test_read_dont_stall(self): - queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=True) - pack_size = 5 - first_pack_ids = self._send_messages( - queue_url, message_count=pack_size, msg_body_template=self._msg_body_template, is_fifo=True, - group_id='1', - ) + matcher=ReadResponseMatcher().with_message_ids([first_message_id, ]) + ) + + def test_read_dont_stall(self): + queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=True) + pack_size = 5 + first_pack_ids = self._send_messages( + queue_url, message_count=pack_size, msg_body_template=self._msg_body_template, is_fifo=True, + group_id='1', + ) time.sleep(5) - second_pack_ids = self._send_messages( - queue_url, message_count=pack_size, msg_body_template=self._msg_body_template, is_fifo=True, - group_id='2' - ) + second_pack_ids = self._send_messages( + queue_url, message_count=pack_size, msg_body_template=self._msg_body_template, is_fifo=True, + group_id='2' + ) time.sleep(5) - self._read_messages_and_assert( + self._read_messages_and_assert( queue_url, messages_count=10, visibility_timeout=1000, - matcher=ReadResponseMatcher().with_message_ids( - [first_pack_ids[0], second_pack_ids[0]] - ) - ) - third_pack_ids = self._send_messages( - queue_url, message_count=pack_size, msg_body_template=self._msg_body_template, is_fifo=True, - group_id='3' - ) - self._read_messages_and_assert( + matcher=ReadResponseMatcher().with_message_ids( + [first_pack_ids[0], second_pack_ids[0]] + ) + ) + third_pack_ids = self._send_messages( + queue_url, message_count=pack_size, msg_body_template=self._msg_body_template, is_fifo=True, + group_id='3' + ) + self._read_messages_and_assert( queue_url, messages_count=10, visibility_timeout=1000, - matcher=ReadResponseMatcher().with_message_ids(third_pack_ids[:1]) - ) - - def test_visibility_timeout_works(self): - self._create_queue_send_x_messages_read_y_messages( - self.queue_name, send_count=5, read_count=1, visibility_timeout=10, + matcher=ReadResponseMatcher().with_message_ids(third_pack_ids[:1]) + ) + + def test_visibility_timeout_works(self): + self._create_queue_send_x_messages_read_y_messages( + self.queue_name, send_count=5, read_count=1, visibility_timeout=10, msg_body_template=self._msg_body_template, is_fifo=True, group_id='1' - ) - second_pack_ids = self._send_messages(self.queue_url, 5, group_id='2', is_fifo=True) - self._read_messages_and_assert( + ) + second_pack_ids = self._send_messages(self.queue_url, 5, group_id='2', is_fifo=True) + self._read_messages_and_assert( self.queue_url, messages_count=5, matcher=ReadResponseMatcher().with_these_or_more_message_ids(second_pack_ids[:1]), - visibility_timeout=10 - ) + visibility_timeout=10 + ) time.sleep(12) - self._read_messages_and_assert( + self._read_messages_and_assert( self.queue_url, messages_count=5, visibility_timeout=1000, matcher=ReadResponseMatcher().with_these_or_more_message_ids( - [self.message_ids[0], second_pack_ids[0]] - ) - ) - - def test_delete_message_works(self): - self._create_queue_send_x_messages_read_y_messages( - self.queue_name, send_count=10, read_count=1, visibility_timeout=1, + [self.message_ids[0], second_pack_ids[0]] + ) + ) + + def test_delete_message_works(self): + self._create_queue_send_x_messages_read_y_messages( + self.queue_name, send_count=10, read_count=1, visibility_timeout=1, msg_body_template=self._msg_body_template, is_fifo=True - ) - - handle = self.read_result[0]['ReceiptHandle'] - assert_that( - self._sqs_api.delete_message(self.queue_url, handle), not_none() - ) + ) + + handle = self.read_result[0]['ReceiptHandle'] + assert_that( + self._sqs_api.delete_message(self.queue_url, handle), not_none() + ) time.sleep(1) - self._read_messages_and_assert( + self._read_messages_and_assert( self.queue_url, messages_count=5, visibility_timeout=1000, matcher=ReadResponseMatcher().with_message_ids(self.message_ids[1:2]) - ) - + ) + counters = self._get_sqs_counters() delete_counter_labels = { 'subsystem': 'core', @@ -230,32 +230,32 @@ class SqsFifoMessagingTest(KikimrSqsTestBase): raises(RuntimeError, pattern='InternalFailure') ) - def test_write_read_delete_many_groups(self): - queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=True) - message_ids = {} - for i in range(10): - message_ids[i] = self._send_messages(queue_url, 5, is_fifo=True, group_id=str(i)) - - matcher = ReadResponseMatcher().with_n_messages(10).with_message_ids( - [i[0] for i in message_ids.values()] - ).with_messages_data( - [self._msg_body_template.format(5*i) for i in range(10)] - ) - result = self._read_messages_and_assert(queue_url, 10, matcher=matcher, visibility_timeout=5) - # Delete message from group 0 - for msg in result: - if msg['MessageId'] in message_ids[0]: - assert_that( - self._sqs_api.delete_message(self.queue_url, msg['ReceiptHandle']), not_none() - ) - message_ids[0] = message_ids[0][1:] - break + def test_write_read_delete_many_groups(self): + queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=True) + message_ids = {} + for i in range(10): + message_ids[i] = self._send_messages(queue_url, 5, is_fifo=True, group_id=str(i)) + + matcher = ReadResponseMatcher().with_n_messages(10).with_message_ids( + [i[0] for i in message_ids.values()] + ).with_messages_data( + [self._msg_body_template.format(5*i) for i in range(10)] + ) + result = self._read_messages_and_assert(queue_url, 10, matcher=matcher, visibility_timeout=5) + # Delete message from group 0 + for msg in result: + if msg['MessageId'] in message_ids[0]: + assert_that( + self._sqs_api.delete_message(self.queue_url, msg['ReceiptHandle']), not_none() + ) + message_ids[0] = message_ids[0][1:] + break time.sleep(5) - matcher = ReadResponseMatcher().with_n_messages(10).with_message_ids( - [i[0] for i in message_ids.values()] - ).with_messages_data( - [self._msg_body_template.format(1)] + [self._msg_body_template.format(i*5) for i in range(1, 10)] - ) + matcher = ReadResponseMatcher().with_n_messages(10).with_message_ids( + [i[0] for i in message_ids.values()] + ).with_messages_data( + [self._msg_body_template.format(1)] + [self._msg_body_template.format(i*5) for i in range(1, 10)] + ) self._read_messages_and_assert(queue_url, 10, matcher=matcher, visibility_timeout=1000) def test_queue_attributes(self): diff --git a/ydb/tests/functional/sqs/test_garbage_collection.py b/ydb/tests/functional/sqs/test_garbage_collection.py index bc09e02951..992c8a6513 100644 --- a/ydb/tests/functional/sqs/test_garbage_collection.py +++ b/ydb/tests/functional/sqs/test_garbage_collection.py @@ -5,7 +5,7 @@ import time import multiprocessing import random -import pytest +import pytest from hamcrest import assert_that, equal_to, less_than_or_equal_to from sqs_requests_client import SqsHttpApi diff --git a/ydb/tests/functional/sqs/test_generic_messaging.py b/ydb/tests/functional/sqs/test_generic_messaging.py index 8ea67f99dd..3f6b2371ba 100644 --- a/ydb/tests/functional/sqs/test_generic_messaging.py +++ b/ydb/tests/functional/sqs/test_generic_messaging.py @@ -5,22 +5,22 @@ import logging import time from collections import OrderedDict -import pytest +import pytest from hamcrest import assert_that, equal_to, not_none, greater_than, has_item, has_items, raises, is_not, not_, empty, instance_of from sqs_requests_client import SqsMessageAttribute, SqsSendMessageParams, SqsChangeMessageVisibilityParams -from sqs_matchers import ReadResponseMatcher, extract_message_ids +from sqs_matchers import ReadResponseMatcher, extract_message_ids from sqs_test_base import to_bytes from sqs_test_base import KikimrSqsTestBase, get_test_with_sqs_installation_by_path, get_test_with_sqs_tenant_installation, IS_FIFO_PARAMS class SqsGenericMessagingTest(KikimrSqsTestBase): @pytest.mark.parametrize(**IS_FIFO_PARAMS) - def test_send_message(self, is_fifo): - if is_fifo: - self.queue_name = self.queue_name + '.fifo' - created_queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=is_fifo) + def test_send_message(self, is_fifo): + if is_fifo: + self.queue_name = self.queue_name + '.fifo' + created_queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=is_fifo) self.seq_no += 1 self._send_message_and_assert(created_queue_url, 'test_send_message', seq_no=self.seq_no if is_fifo else None, group_id='group' if is_fifo else None) @@ -130,11 +130,11 @@ class SqsGenericMessagingTest(KikimrSqsTestBase): ) @pytest.mark.parametrize(**IS_FIFO_PARAMS) - def test_send_and_read_message(self, is_fifo): - if is_fifo: - self.queue_name = self.queue_name + '.fifo' - - created_queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=is_fifo) + def test_send_and_read_message(self, is_fifo): + if is_fifo: + self.queue_name = self.queue_name + '.fifo' + + created_queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=is_fifo) body = '<' + self._msg_body_template.format('trololo') + '<' # to ensure that we have correct xml attributes = { SqsMessageAttribute('a', 'String', 'xyz'), @@ -304,89 +304,89 @@ class SqsGenericMessagingTest(KikimrSqsTestBase): ) @pytest.mark.parametrize(**IS_FIFO_PARAMS) - def test_create_q_twice(self, is_fifo): - if is_fifo: - self.queue_name = self.queue_name + '.fifo' - created_queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=is_fifo) + def test_create_q_twice(self, is_fifo): + if is_fifo: + self.queue_name = self.queue_name + '.fifo' + created_queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=is_fifo) self.seq_no += 1 message_id = self._send_message_and_assert(created_queue_url, self._msg_body_template, seq_no=self.seq_no if is_fifo else None, group_id='group' if is_fifo else None) - second_url = self._create_queue_and_assert(self.queue_name, is_fifo=is_fifo) + second_url = self._create_queue_and_assert(self.queue_name, is_fifo=is_fifo) assert_that(second_url, equal_to(created_queue_url)) - self._read_messages_and_assert( + self._read_messages_and_assert( created_queue_url, messages_count=1, visibility_timeout=1000, matcher=ReadResponseMatcher().with_message_ids([message_id, ]) - ) + ) - def test_send_and_read_multiple_messages(self): - queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=False) - first_message_id = self._send_message_and_assert( - queue_url, self._msg_body_template.format('0') - ) + def test_send_and_read_multiple_messages(self): + queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=False) + first_message_id = self._send_message_and_assert( + queue_url, self._msg_body_template.format('0') + ) time.sleep(5) - second_message_id = self._send_message_and_assert( - queue_url, self._msg_body_template.format('1') - ) - self._read_messages_and_assert( + second_message_id = self._send_message_and_assert( + queue_url, self._msg_body_template.format('1') + ) + self._read_messages_and_assert( queue_url, messages_count=2, visibility_timeout=1000, - matcher=ReadResponseMatcher().with_message_ids( - [first_message_id, second_message_id] - ).with_messages_data([self._msg_body_template.format('0'), self._msg_body_template.format('1')]) - ) - - def test_read_dont_stall(self): - queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=False) - first_pack_size = 5 - second_pack_size = 5 - first_pack_ids = self._send_messages( - queue_url, message_count=first_pack_size, msg_body_template=self._msg_body_template - ) - self._send_messages(queue_url, message_count=second_pack_size, msg_body_template=self._msg_body_template) - self._read_messages_and_assert( + matcher=ReadResponseMatcher().with_message_ids( + [first_message_id, second_message_id] + ).with_messages_data([self._msg_body_template.format('0'), self._msg_body_template.format('1')]) + ) + + def test_read_dont_stall(self): + queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=False) + first_pack_size = 5 + second_pack_size = 5 + first_pack_ids = self._send_messages( + queue_url, message_count=first_pack_size, msg_body_template=self._msg_body_template + ) + self._send_messages(queue_url, message_count=second_pack_size, msg_body_template=self._msg_body_template) + self._read_messages_and_assert( queue_url, messages_count=10, visibility_timeout=1000, matcher=ReadResponseMatcher( ).with_n_or_more_messages(first_pack_size + 1).with_these_or_more_message_ids(first_pack_ids) - ) - - def test_multi_read_dont_stall(self): - queue_url = self._create_queue_and_assert(self.queue_name) - pack_size = 7 - total_packs = 3 - all_ids = set() - read_ids = set() - for _ in range(total_packs): - all_ids.update(self._send_messages( - queue_url, message_count=pack_size, msg_body_template=self._msg_body_template - )) - result = self._read_messages_and_assert( + ) + + def test_multi_read_dont_stall(self): + queue_url = self._create_queue_and_assert(self.queue_name) + pack_size = 7 + total_packs = 3 + all_ids = set() + read_ids = set() + for _ in range(total_packs): + all_ids.update(self._send_messages( + queue_url, message_count=pack_size, msg_body_template=self._msg_body_template + )) + result = self._read_messages_and_assert( queue_url, messages_count=10, visibility_timeout=1000, - matcher=ReadResponseMatcher().with_some_of_message_ids(all_ids - read_ids) - ) - if result: - read_ids.update(extract_message_ids(result)) - - assert_that( - len(read_ids), greater_than(pack_size), - "Wrote {packs} packs of size {size}, but got only {total_read} messages after all read attempts".format( - packs=total_packs, size=pack_size, total_read=len(read_ids) - ) - ) - - def test_visibility_timeout_works(self): - total_msg_count = 10 + matcher=ReadResponseMatcher().with_some_of_message_ids(all_ids - read_ids) + ) + if result: + read_ids.update(extract_message_ids(result)) + + assert_that( + len(read_ids), greater_than(pack_size), + "Wrote {packs} packs of size {size}, but got only {total_read} messages after all read attempts".format( + packs=total_packs, size=pack_size, total_read=len(read_ids) + ) + ) + + def test_visibility_timeout_works(self): + total_msg_count = 10 visibility_timeout = 5 before_read_time = time.time() - self._create_queue_send_x_messages_read_y_messages( + self._create_queue_send_x_messages_read_y_messages( self.queue_name, send_count=total_msg_count, read_count=1, visibility_timeout=visibility_timeout, msg_body_template=self._msg_body_template, is_fifo=False ) - msg_data = [self._msg_body_template.format(i) for i in range(total_msg_count)] - - already_read_id = extract_message_ids(self.read_result)[0] + msg_data = [self._msg_body_template.format(i) for i in range(total_msg_count)] + + already_read_id = extract_message_ids(self.read_result)[0] read_result = self._read_messages_and_assert( self.queue_url, messages_count=5, visibility_timeout=visibility_timeout, wait_timeout=1, - matcher=ReadResponseMatcher().with_some_of_message_ids(self.message_ids) + matcher=ReadResponseMatcher().with_some_of_message_ids(self.message_ids) .with_n_messages(5) ) @@ -396,25 +396,25 @@ class SqsGenericMessagingTest(KikimrSqsTestBase): assert_that(read_ids, not_(has_item(already_read_id))) time.sleep(visibility_timeout + 0.1) - self._read_messages_and_assert( + self._read_messages_and_assert( self.queue_url, messages_count=10, visibility_timeout=1000, - matcher=ReadResponseMatcher().with_message_ids(self.message_ids).with_messages_data(msg_data) - ) - - def test_visibility_timeout_expires_on_wait_timeout(self): - queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=False) - message_ids = self._send_messages( - queue_url, message_count=10, msg_body_template=self._msg_body_template - ) - # noinspection PyTypeChecker - self._read_messages_and_assert( - queue_url, messages_count=10, matcher=ReadResponseMatcher().with_message_ids(message_ids), - visibility_timeout=9 - ) - self._read_messages_and_assert( + matcher=ReadResponseMatcher().with_message_ids(self.message_ids).with_messages_data(msg_data) + ) + + def test_visibility_timeout_expires_on_wait_timeout(self): + queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=False) + message_ids = self._send_messages( + queue_url, message_count=10, msg_body_template=self._msg_body_template + ) + # noinspection PyTypeChecker + self._read_messages_and_assert( + queue_url, messages_count=10, matcher=ReadResponseMatcher().with_message_ids(message_ids), + visibility_timeout=9 + ) + self._read_messages_and_assert( queue_url, messages_count=10, visibility_timeout=1000, matcher=ReadResponseMatcher().with_some_of_message_ids(message_ids), wait_timeout=10, - ) - + ) + @pytest.mark.parametrize(**IS_FIFO_PARAMS) def test_zero_visibility_timeout_works(self, is_fifo): if is_fifo: @@ -633,24 +633,24 @@ class SqsGenericMessagingTest(KikimrSqsTestBase): raises(RuntimeError, pattern='InvalidParameterValue') ) - def test_delete_message_works(self): - self._create_queue_send_x_messages_read_y_messages( + def test_delete_message_works(self): + self._create_queue_send_x_messages_read_y_messages( self.queue_name, send_count=10, read_count=2, visibility_timeout=0, - msg_body_template=self._msg_body_template + msg_body_template=self._msg_body_template ) handle = self.read_result[0]['ReceiptHandle'] - self.message_ids.remove(self.read_result[0]['MessageId']) - + self.message_ids.remove(self.read_result[0]['MessageId']) + assert_that( - self._sqs_api.delete_message(self.queue_url, handle), not_none() + self._sqs_api.delete_message(self.queue_url, handle), not_none() ) # check double deletion assert_that( self._sqs_api.delete_message(self.queue_url, handle), not_none() ) - self._read_messages_and_assert( + self._read_messages_and_assert( self.queue_url, messages_count=10, visibility_timeout=1000, - matcher=ReadResponseMatcher().with_message_ids(self.message_ids) + matcher=ReadResponseMatcher().with_message_ids(self.message_ids) ) counters = self._get_sqs_counters() @@ -774,37 +774,37 @@ class SqsGenericMessagingTest(KikimrSqsTestBase): else: assert_that(messages_count_metric, equal_to(0)) - def test_can_read_new_written_data_on_visibility_timeout(self): - visibility_timeout = 15 - self._create_queue_send_x_messages_read_y_messages( - self.queue_name, send_count=7, read_count=4, visibility_timeout=visibility_timeout, - msg_body_template=self._msg_body_template + def test_can_read_new_written_data_on_visibility_timeout(self): + visibility_timeout = 15 + self._create_queue_send_x_messages_read_y_messages( + self.queue_name, send_count=7, read_count=4, visibility_timeout=visibility_timeout, + msg_body_template=self._msg_body_template ) begin_time = time.time() - first_pack_ids = self.message_ids - second_pack_ids = self._send_messages( - self.queue_url, message_count=3, msg_body_template=self._msg_body_template - ) - message_ids = set(first_pack_ids) - set(extract_message_ids(self.read_result)) - message_ids.update(second_pack_ids) + first_pack_ids = self.message_ids + second_pack_ids = self._send_messages( + self.queue_url, message_count=3, msg_body_template=self._msg_body_template + ) + message_ids = set(first_pack_ids) - set(extract_message_ids(self.read_result)) + message_ids.update(second_pack_ids) visibility_timeout_2 = 15 - self._read_messages_and_assert( + self._read_messages_and_assert( self.queue_url, messages_count=10, visibility_timeout=visibility_timeout_2, matcher=ReadResponseMatcher().with_these_or_more_message_ids(message_ids).with_n_or_more_messages(6), - ) + ) remaining_time = visibility_timeout - (time.time() - begin_time) # for first pack read time.sleep(max(remaining_time + 0.1, visibility_timeout_2 + 0.1, 0)) - self._read_messages_and_assert( + self._read_messages_and_assert( self.queue_url, messages_count=10, visibility_timeout=1000, matcher=ReadResponseMatcher().with_message_ids( - first_pack_ids + second_pack_ids - ) - ) + first_pack_ids + second_pack_ids + ) + ) - def test_partial_delete_works(self): - self._create_queue_send_x_messages_read_y_messages( - self.queue_name, send_count=10, read_count=5, visibility_timeout=5, - msg_body_template=self._msg_body_template + def test_partial_delete_works(self): + self._create_queue_send_x_messages_read_y_messages( + self.queue_name, send_count=10, read_count=5, visibility_timeout=5, + msg_body_template=self._msg_body_template ) handle = self.read_result[4]['ReceiptHandle'] # select the last read message to avoid race with visibility timeout while reading self.message_ids.remove(self.read_result[4]['MessageId']) @@ -813,18 +813,18 @@ class SqsGenericMessagingTest(KikimrSqsTestBase): not_none() ) time.sleep(6) - self._read_messages_and_assert( + self._read_messages_and_assert( self.queue_url, messages_count=10, visibility_timeout=1000, - matcher=ReadResponseMatcher().with_message_ids(self.message_ids) + matcher=ReadResponseMatcher().with_message_ids(self.message_ids) ) - + def test_wrong_delete_fails(self): self._create_queue_send_x_messages_read_y_messages( self.queue_name, send_count=1, read_count=1, visibility_timeout=5, msg_body_template=self._msg_body_template ) handle = self.read_result[0]['ReceiptHandle'] - + def call_delete_invalid_handle(): self._sqs_api.delete_message(self.queue_url, handle + handle), # wrong handle assert_that( diff --git a/ydb/tests/functional/sqs/test_multinode_cluster.py b/ydb/tests/functional/sqs/test_multinode_cluster.py index 6622e0ce19..047ae56198 100644 --- a/ydb/tests/functional/sqs/test_multinode_cluster.py +++ b/ydb/tests/functional/sqs/test_multinode_cluster.py @@ -4,7 +4,7 @@ import logging import time import threading -import pytest +import pytest from hamcrest import assert_that, equal_to, not_none, raises, not_ from ydb.tests.library.common.types import Erasure diff --git a/ydb/tests/functional/sqs/test_polling.py b/ydb/tests/functional/sqs/test_polling.py index f8586dacca..dba0556e7d 100644 --- a/ydb/tests/functional/sqs/test_polling.py +++ b/ydb/tests/functional/sqs/test_polling.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -import pytest +import pytest from hamcrest import assert_that, equal_to from sqs_matchers import ReadResponseMatcher diff --git a/ydb/tests/functional/sqs/test_queue_attributes_validation.py b/ydb/tests/functional/sqs/test_queue_attributes_validation.py index 8990056ef7..27ec79c9ee 100644 --- a/ydb/tests/functional/sqs/test_queue_attributes_validation.py +++ b/ydb/tests/functional/sqs/test_queue_attributes_validation.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -import pytest +import pytest from hamcrest import assert_that, equal_to from sqs_test_base import KikimrSqsTestBase, IS_FIFO_PARAMS diff --git a/ydb/tests/functional/sqs/test_queues_managing.py b/ydb/tests/functional/sqs/test_queues_managing.py index 1e0d060d79..bcfd91ca67 100644 --- a/ydb/tests/functional/sqs/test_queues_managing.py +++ b/ydb/tests/functional/sqs/test_queues_managing.py @@ -3,8 +3,8 @@ import time from collections import OrderedDict -import pytest -from hamcrest import assert_that, equal_to, greater_than, not_none, none, has_item, has_items, raises, empty, instance_of +import pytest +from hamcrest import assert_that, equal_to, greater_than, not_none, none, has_item, has_items, raises, empty, instance_of from sqs_matchers import ReadResponseMatcher @@ -21,10 +21,10 @@ class QueuesManagingTest(KikimrSqsTestBase): return config_generator @pytest.mark.parametrize(**IS_FIFO_PARAMS) - def test_create_queue(self, is_fifo): + def test_create_queue(self, is_fifo): attributes = {} - if is_fifo: - self.queue_name = self.queue_name + '.fifo' + if is_fifo: + self.queue_name = self.queue_name + '.fifo' attributes['ContentBasedDeduplication'] = 'true' attributes['DelaySeconds'] = '506' attributes['MaximumMessageSize'] = '10003' @@ -32,11 +32,11 @@ class QueuesManagingTest(KikimrSqsTestBase): attributes['ReceiveMessageWaitTimeSeconds'] = '11' attributes['VisibilityTimeout'] = '42' created_queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=is_fifo, use_http=True) - existing_queues = self._sqs_api.list_queues() + existing_queues = self._sqs_api.list_queues() assert_that( created_queue_url in existing_queues ) - got_queue_url = self._sqs_api.get_queue_url(self.queue_name) + got_queue_url = self._sqs_api.get_queue_url(self.queue_name) assert_that( got_queue_url, equal_to(created_queue_url) ) @@ -50,38 +50,38 @@ class QueuesManagingTest(KikimrSqsTestBase): if is_fifo: assert_that(created_attributes.get('ContentBasedDeduplication'), 'true') - def test_create_fifo_queue_wo_postfix(self): + def test_create_fifo_queue_wo_postfix(self): def call_create(): self.called = True self._sqs_api.create_queue(self.queue_name, is_fifo=True) - assert_that( + assert_that( call_create, raises( RuntimeError, pattern='failed with status 400.*\n.*FIFO queue should end with "\\.fifo"' ) - ) - - def test_create_queue_generates_event(self): - pytest.skip("Outdated") - self._create_queue_and_assert(self.queue_name, is_fifo=False) + ) + + def test_create_queue_generates_event(self): + pytest.skip("Outdated") + self._create_queue_and_assert(self.queue_name, is_fifo=False) table_path = '{}/.Queues'.format(self.sqs_root) - assert_that(self._get_table_lines_count(table_path), equal_to(1)) - + assert_that(self._get_table_lines_count(table_path), equal_to(1)) + table_path = '{}/.Events'.format(self.sqs_root) - assert_that(self._get_table_lines_count(table_path), equal_to(1)) - - def test_remove_queue_generates_event(self): - pytest.skip("Outdated") - queue_url = self._create_queue_and_assert(self.queue_name) + assert_that(self._get_table_lines_count(table_path), equal_to(1)) + + def test_remove_queue_generates_event(self): + pytest.skip("Outdated") + queue_url = self._create_queue_and_assert(self.queue_name) table_path = '{}/.Events'.format(self.sqs_root) - lines_count = self._get_table_lines_count(table_path) - assert_that(lines_count, greater_than(0)) - - self._sqs_api.delete_queue(queue_url) - assert_that(self._get_table_lines_count(table_path), greater_than(lines_count)) - + lines_count = self._get_table_lines_count(table_path) + assert_that(lines_count, greater_than(0)) + + self._sqs_api.delete_queue(queue_url) + assert_that(self._get_table_lines_count(table_path), greater_than(lines_count)) + def test_create_queue_with_invalid_name(self): def call_create(): self._sqs_api.create_queue('invalid_queue_name!') @@ -95,11 +95,11 @@ class QueuesManagingTest(KikimrSqsTestBase): ) @pytest.mark.parametrize(**IS_FIFO_PARAMS) - def test_delete_queue(self, is_fifo): - if is_fifo: - self.queue_name = self.queue_name + '.fifo' - created_queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=is_fifo) - self._sqs_api.list_queues() + def test_delete_queue(self, is_fifo): + if is_fifo: + self.queue_name = self.queue_name + '.fifo' + created_queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=is_fifo) + self._sqs_api.list_queues() self._sqs_api.send_message(created_queue_url, 'body', group_id='group' if is_fifo else None, deduplication_id='123' if is_fifo else None) send_message_labels = { @@ -112,12 +112,12 @@ class QueuesManagingTest(KikimrSqsTestBase): sends = self._get_counter_value(counters, send_message_labels) assert_that(sends, equal_to(1)) - delete_result = self._sqs_api.delete_queue(created_queue_url) + delete_result = self._sqs_api.delete_queue(created_queue_url) assert_that( delete_result, not_none() ) - existing_queues = self._sqs_api.list_queues() + existing_queues = self._sqs_api.list_queues() assert_that( created_queue_url not in existing_queues, "Deleted queue appears in list_queues()" @@ -238,16 +238,16 @@ class QueuesManagingTest(KikimrSqsTestBase): check_purged_queue(created_queue_url2) @pytest.mark.parametrize(**IS_FIFO_PARAMS) - def test_delete_and_create_queue(self, is_fifo): - if is_fifo: - self.queue_name = self.queue_name + '.fifo' + def test_delete_and_create_queue(self, is_fifo): + if is_fifo: + self.queue_name = self.queue_name + '.fifo' created_queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=is_fifo, use_http=True) self.seq_no += 1 self._send_message_and_assert(created_queue_url, self._msg_body_template.format(1), seq_no=self.seq_no if is_fifo else None, group_id='group' if is_fifo else None) - delete_result = self._sqs_api.delete_queue(created_queue_url) - assert_that( - delete_result, not_none() - ) + delete_result = self._sqs_api.delete_queue(created_queue_url) + assert_that( + delete_result, not_none() + ) created_queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=is_fifo, use_http=True) master_is_updated = False @@ -265,9 +265,9 @@ class QueuesManagingTest(KikimrSqsTestBase): assert_that(master_is_updated) self.seq_no += 1 msg_id = self._send_message_and_assert(created_queue_url, self._msg_body_template.format(2), seq_no=self.seq_no if is_fifo else None, group_id='group' if is_fifo else None) - self._read_messages_and_assert( + self._read_messages_and_assert( created_queue_url, 10, ReadResponseMatcher().with_message_ids([msg_id, ]) - ) + ) def test_ya_count_queues(self): assert_that(self._sqs_api.private_count_queues(), equal_to('0')) @@ -336,5 +336,5 @@ class TestQueuesManagingWithTenant(get_test_with_sqs_tenant_installation(QueuesM pass -class TestQueuesManagingWithPathTestQueuesManagingWithPath(get_test_with_sqs_installation_by_path(QueuesManagingTest)): +class TestQueuesManagingWithPathTestQueuesManagingWithPath(get_test_with_sqs_installation_by_path(QueuesManagingTest)): pass diff --git a/ydb/tests/functional/sqs/test_recompiles_requests.py b/ydb/tests/functional/sqs/test_recompiles_requests.py index 8053131116..367003323f 100644 --- a/ydb/tests/functional/sqs/test_recompiles_requests.py +++ b/ydb/tests/functional/sqs/test_recompiles_requests.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -import pytest +import pytest from hamcrest import assert_that, not_none from ydb.tests.library.common.types import Erasure diff --git a/ydb/tests/functional/sqs/ya.make b/ydb/tests/functional/sqs/ya.make index c9fb9f0623..bbb0a57bb8 100644 --- a/ydb/tests/functional/sqs/ya.make +++ b/ydb/tests/functional/sqs/ya.make @@ -27,7 +27,7 @@ TEST_SRCS( ) IF (SANITIZER_TYPE) - TIMEOUT(2400) + TIMEOUT(2400) SIZE(LARGE) TAG(ya:fat) REQUIREMENTS( diff --git a/ydb/tests/library/common/composite_assert.py b/ydb/tests/library/common/composite_assert.py index 4c8c1075c6..38087494cb 100644 --- a/ydb/tests/library/common/composite_assert.py +++ b/ydb/tests/library/common/composite_assert.py @@ -44,28 +44,28 @@ class CompositeAssert(object): return False -# ToDo: Refactor to use CompositeAssert instead of this. -class CompositeCheckResult(object): - def __init__(self): - self.__result = True - self.__reason = [] - - @property - def result(self): - return self.__result - - @property - def description(self): - return '\n'.join(self.__reason) - - def check(self, condition, description=""): - if not condition: - self.set_failed(description) - - def set_failed(self, description=""): - self.__result = False - if description: - self.__reason.append(description) - - def __nonzero__(self): - return self.__result +# ToDo: Refactor to use CompositeAssert instead of this. +class CompositeCheckResult(object): + def __init__(self): + self.__result = True + self.__reason = [] + + @property + def result(self): + return self.__result + + @property + def description(self): + return '\n'.join(self.__reason) + + def check(self, condition, description=""): + if not condition: + self.set_failed(description) + + def set_failed(self, description=""): + self.__result = False + if description: + self.__reason.append(description) + + def __nonzero__(self): + return self.__result diff --git a/ydb/tests/library/common/generators.py b/ydb/tests/library/common/generators.py index ba99d3c6fc..8261535a30 100644 --- a/ydb/tests/library/common/generators.py +++ b/ydb/tests/library/common/generators.py @@ -1,49 +1,49 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import random -import string - -__author__ = 'asatarin@yandex-team.ru' - - -__hex_digits = '0123456789ABCDEF' - - -def int_between(lo, hi): - def closure(): - while True: - yield random.randint(lo, hi) - return closure - - -def one_of(list_of_values): - def closure(): - while True: - yield random.choice(list_of_values) - return closure - - -def float_in(lo, hi): - def closure(): - while True: - yield random.uniform(lo, hi) - return closure - - -def float_as_hex(byte_length=4): - def closure(): - while True: - yield 'x"' + ''.join([random.choice(__hex_digits) for _ in range(byte_length * 2)]) + '"' - return closure - - -# TODO use string.printable for character generation or even wider range (e.g. UTF-8) -def string_with_length(length): - def closure(): - while True: - yield ''.join([random.choice(string.ascii_letters) for _ in range(length)]) - return closure +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import random +import string + +__author__ = 'asatarin@yandex-team.ru' + + +__hex_digits = '0123456789ABCDEF' + + +def int_between(lo, hi): + def closure(): + while True: + yield random.randint(lo, hi) + return closure + + +def one_of(list_of_values): + def closure(): + while True: + yield random.choice(list_of_values) + return closure + + +def float_in(lo, hi): + def closure(): + while True: + yield random.uniform(lo, hi) + return closure + + +def float_as_hex(byte_length=4): + def closure(): + while True: + yield 'x"' + ''.join([random.choice(__hex_digits) for _ in range(byte_length * 2)]) + '"' + return closure + + +# TODO use string.printable for character generation or even wider range (e.g. UTF-8) +def string_with_length(length): + def closure(): + while True: + yield ''.join([random.choice(string.ascii_letters) for _ in range(length)]) + return closure def actor_id(): diff --git a/ydb/tests/library/common/protobuf_ss.py b/ydb/tests/library/common/protobuf_ss.py index 65045bc8c3..90b0638eb4 100644 --- a/ydb/tests/library/common/protobuf_ss.py +++ b/ydb/tests/library/common/protobuf_ss.py @@ -9,9 +9,9 @@ from ydb.core.protos import flat_scheme_op_pb2 from ydb.tests.library.common.protobuf import AbstractProtobufBuilder, build_protobuf_if_necessary -DEFAULT_SIZE_TO_SPLIT = 10 ** 6 - - +DEFAULT_SIZE_TO_SPLIT = 10 ** 6 + + class TPartitionConfig(AbstractProtobufBuilder): """ See /arcadia/ydb/core/protos/flat_scheme_op_pb2.proto @@ -19,7 +19,7 @@ class TPartitionConfig(AbstractProtobufBuilder): def __init__(self): super(TPartitionConfig, self).__init__(flat_scheme_op_pb2.TPartitionConfig()) - self.with_partitioning_policy(DEFAULT_SIZE_TO_SPLIT) + self.with_partitioning_policy(DEFAULT_SIZE_TO_SPLIT) def __ensure_has_compaction_policy(self): if not self.protobuf.HasField('CompactionPolicy'): diff --git a/ydb/tests/library/common/yatest_common.py b/ydb/tests/library/common/yatest_common.py index ba7c3b9ce2..0b83695155 100644 --- a/ydb/tests/library/common/yatest_common.py +++ b/ydb/tests/library/common/yatest_common.py @@ -30,9 +30,9 @@ binary_path = wrap(ya_common.binary_path, lambda x: x) output_path = wrap(ya_common.output_path, lambda x: x) work_path = wrap(ya_common.work_path, lambda x: x) -get_param = wrap(ya_common.get_param, lambda x, y=None: y) +get_param = wrap(ya_common.get_param, lambda x, y=None: y) get_param_dict_copy = wrap(ya_common.get_param_dict_copy, lambda: dict()) - + def get_bool_param(key, default): val = get_param(key, default) @@ -67,4 +67,4 @@ def plain_or_under_sanitizer(plain, sanitized): :return: plain if no sanitizer enabled or sanitized otherwise """ - return plain if not context.sanitize else sanitized + return plain if not context.sanitize else sanitized diff --git a/ydb/tests/library/harness/daemon.py b/ydb/tests/library/harness/daemon.py index 5c0055bf3c..edabe07262 100644 --- a/ydb/tests/library/harness/daemon.py +++ b/ydb/tests/library/harness/daemon.py @@ -178,7 +178,7 @@ class Daemon(object): if not is_killed: exit_code = self.__daemon.exit_code self.__check_before_fail() - + if exit_code not in self._acceptable_exit_codes: raise DaemonError( "Bad exit_code.", diff --git a/ydb/tests/library/harness/kikimr_client.py b/ydb/tests/library/harness/kikimr_client.py index c89add5a68..d74383ae23 100644 --- a/ydb/tests/library/harness/kikimr_client.py +++ b/ydb/tests/library/harness/kikimr_client.py @@ -91,9 +91,9 @@ class KiKiMRMessageBusClient(object): time.sleep(self.__retry_sleep_seconds) - def close(self): + def close(self): self._channel.close() - + def update_self_heal(self, enable, domain=1): request = msgbus.TBlobStorageConfigRequest() request.Domain = domain @@ -392,5 +392,5 @@ class KiKiMRMessageBusClient(object): request.Alive = True return self.invoke(request, 'TabletStateRequest') - def __del__(self): - self.close() + def __del__(self): + self.close() diff --git a/ydb/tests/library/harness/kikimr_cluster.py b/ydb/tests/library/harness/kikimr_cluster.py index 253e415d69..942e1588ca 100644 --- a/ydb/tests/library/harness/kikimr_cluster.py +++ b/ydb/tests/library/harness/kikimr_cluster.py @@ -1,13 +1,13 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- import os -import itertools -import logging +import itertools +import logging import subprocess import time import pprint from concurrent import futures - + import ydb.tests.library.common.yatest_common as yatest_common from . import param_constants diff --git a/ydb/tests/library/harness/kikimr_config.py b/ydb/tests/library/harness/kikimr_config.py index 2ffa0d6b3a..57c282b9f4 100644 --- a/ydb/tests/library/harness/kikimr_config.py +++ b/ydb/tests/library/harness/kikimr_config.py @@ -368,7 +368,7 @@ class KikimrConfigGenerator(object): {"vdisk_locations": [{"node_id": node_id, "pdisk_id": pdisk_id, "pdisk_guid": pdisk_id, 'vdisk_slot_id': 0}]} ) - def __build(self): + def __build(self): datacenter_id_generator = itertools.cycle(self._dcs) self.yaml_config["blob_storage_config"] = {} self.yaml_config["blob_storage_config"]["service_set"] = {} diff --git a/ydb/tests/library/harness/kikimr_http_client.py b/ydb/tests/library/harness/kikimr_http_client.py index fe023bedb3..68bbfe7375 100644 --- a/ydb/tests/library/harness/kikimr_http_client.py +++ b/ydb/tests/library/harness/kikimr_http_client.py @@ -55,40 +55,40 @@ class HiveClient(object): url_lst.append('channel=' + ','.join(map(str, channels))) self.__get('&'.join(url_lst), timeout=TIMEOUT) - def kick_tablets_from_node(self, node_id): + def kick_tablets_from_node(self, node_id): self.__get( - self.__url + "&page=KickNode&node={node}".format(node=node_id), - timeout=TIMEOUT - ) + self.__url + "&page=KickNode&node={node}".format(node=node_id), + timeout=TIMEOUT + ) - def block_node(self, node_id, block=True): - block = '1' if block else '0' + def block_node(self, node_id, block=True): + block = '1' if block else '0' self.__get( - self.__url + "&page=SetDown&node={node}&down={down}".format(node=node_id, down=block), - timeout=TIMEOUT - ) - - def unblock_node(self, node_id): + self.__url + "&page=SetDown&node={node}&down={down}".format(node=node_id, down=block), + timeout=TIMEOUT + ) + + def unblock_node(self, node_id): self.block_node(node_id, block=False) - - def change_tablet_weight(self, tablet_id, tablet_weight): + + def change_tablet_weight(self, tablet_id, tablet_weight): self.__get( - self.__url + "&page=UpdateResources&tablet={tablet}&kv={size}".format( - tablet=tablet_id, - size=tablet_weight - ), - timeout=TIMEOUT - ) - - def change_tablet_cpu_usage(self, tablet_id, cpu_usage): + self.__url + "&page=UpdateResources&tablet={tablet}&kv={size}".format( + tablet=tablet_id, + size=tablet_weight + ), + timeout=TIMEOUT + ) + + def change_tablet_cpu_usage(self, tablet_id, cpu_usage): self.__get( - self.__url + "&page=UpdateResources&tablet={tablet}&cpu={size}".format( - tablet=tablet_id, - size=cpu_usage - ), - timeout=TIMEOUT - ) - + self.__url + "&page=UpdateResources&tablet={tablet}&cpu={size}".format( + tablet=tablet_id, + size=cpu_usage + ), + timeout=TIMEOUT + ) + def set_min_scatter_to_balance(self, min_scatter_to_balance=101): """ min_scatter_to_balance=101 -- effectively disables auto rebalancing @@ -97,7 +97,7 @@ class HiveClient(object): self.__url + "&page=Settings&minScatterToBalance={min_scatter_to_balance}".format( min_scatter_to_balance=min_scatter_to_balance), timeout=TIMEOUT ) - + def set_max_scheduled_tablets(self, max_scheduled_tablets=10): self.__get( self.__url + "&page=Settings&maxScheduledTablets={max_scheduled_tablets}".format( @@ -158,17 +158,17 @@ class SwaggerClient(object): "/json/pdiskinfo", node_id=str(node_id), enums='true', timeout=self.__timeout ) - def vdisk_info(self): - return self.__http_get_and_parse_json( - "/json/vdiskinfo", timeout=self.__timeout - ) - - def tablet_info(self, tablet_type=None): - if tablet_type is not None: - return self.__http_get_and_parse_json( - "/json/tabletinfo", timeout=self.__timeout, filter='Type={}'.format(tablet_type) - ) - else: - return self.__http_get_and_parse_json( - "/json/tabletinfo", timeout=self.__timeout - ) + def vdisk_info(self): + return self.__http_get_and_parse_json( + "/json/vdiskinfo", timeout=self.__timeout + ) + + def tablet_info(self, tablet_type=None): + if tablet_type is not None: + return self.__http_get_and_parse_json( + "/json/tabletinfo", timeout=self.__timeout, filter='Type={}'.format(tablet_type) + ) + else: + return self.__http_get_and_parse_json( + "/json/tabletinfo", timeout=self.__timeout + ) diff --git a/ydb/tests/library/harness/kikimr_runner.py b/ydb/tests/library/harness/kikimr_runner.py index 25803842c3..251f485b5d 100644 --- a/ydb/tests/library/harness/kikimr_runner.py +++ b/ydb/tests/library/harness/kikimr_runner.py @@ -30,7 +30,7 @@ def get_unique_path_for_current_test(output_path, sub_folder): return os.path.join(output_path, test_name, sub_folder) - + def ensure_path_exists(path): if not os.path.isdir(path): os.makedirs(path) @@ -119,20 +119,20 @@ class KiKiMRNode(daemon.Daemon, kikimr_node_interface.NodeInterface): if self.__configurator.suppress_version_check: command.append("--suppress-version-check") - if self.__node_broker_port is not None: + if self.__node_broker_port is not None: command.append("--node-broker=%s%s:%d" % ( "grpcs://" if self.__configurator.grpc_ssl_enable else "", self.host, self.__node_broker_port)) else: - command.append("--node=%d" % self.node_id) + command.append("--node=%d" % self.node_id) if self.__configurator.grpc_ssl_enable: command.append( "--ca=%s" % self.__configurator.grpc_tls_ca_path ) - if self.__role == 'slot': + if self.__role == 'slot': command.append( "--tenant=%s" % self._tenant_affiliation ) @@ -165,12 +165,12 @@ class KiKiMRNode(daemon.Daemon, kikimr_node_interface.NodeInterface): logger.info("Final command: %s", ' '.join(command).replace(self.__config_path, '$CFG_DIR_PATH')) return command - def stop(self): + def stop(self): try: super(KiKiMRNode, self).stop() finally: logger.info("Stopped node %s", self) - + def kill(self): try: super(KiKiMRNode, self).kill() @@ -431,7 +431,7 @@ class KiKiMR(kikimr_cluster_interface.KiKiMRClusterInterface): self.nodes[node_id].format_pdisk(**pdisk) def __add_bs_box(self): - request = bs.TConfigRequest() + request = bs.TConfigRequest() for node_id in self.__configurator.all_node_ids(): cmd = request.Command.add() @@ -483,7 +483,7 @@ class KiKiMR(kikimr_cluster_interface.KiKiMRClusterInterface): def add_storage_pool(self, name=None, kind="rot", pdisk_user_kind=0, erasure=None): if erasure is None: erasure = self.__configurator.static_erasure - request = bs.TConfigRequest() + request = bs.TConfigRequest() cmd = request.Command.add() cmd.DefineStoragePool.BoxId = 1 @@ -505,16 +505,16 @@ class KiKiMR(kikimr_cluster_interface.KiKiMRClusterInterface): def __wait_for_bs_controller_to_start(self): monitors = [node.monitor for node in self.nodes.values()] - - def predicate(): - return blobstorage_controller_has_started_on_some_node(monitors) - + + def predicate(): + return blobstorage_controller_has_started_on_some_node(monitors) + timeout_seconds = yatest_common.plain_or_under_sanitizer(120, 240) bs_controller_started = wait_for( predicate=predicate, timeout_seconds=timeout_seconds, step_seconds=1.0, multiply=1.3 ) assert bs_controller_started - + class KikimrExternalNode(daemon.ExternalNodeDaemon, kikimr_node_interface.NodeInterface): def __init__( diff --git a/ydb/tests/library/matchers/response.py b/ydb/tests/library/matchers/response.py index 2fe460597d..d492116b4b 100644 --- a/ydb/tests/library/matchers/response.py +++ b/ydb/tests/library/matchers/response.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- import six -from hamcrest import all_of, has_property, has_properties +from hamcrest import all_of, has_property, has_properties from hamcrest.core.base_matcher import BaseMatcher import ydb.core.protos.msgbus_kv_pb2 as msgbus_kv @@ -26,56 +26,56 @@ def to_bytes(v): class KeyValueResponseProtobufMatcher(BaseMatcher): - def __init__(self, status=MessageBusStatus.MSTATUS_OK, do_describe=True): + def __init__(self, status=MessageBusStatus.MSTATUS_OK, do_describe=True): self.__read_range_matchers = [] self.__read_keys_matchers = [] - self.__write_matchers = [] - self.__rename_matchers = [] - self.__delete_matcher = None + self.__write_matchers = [] + self.__rename_matchers = [] + self.__delete_matcher = None self.__copy_range_matchers = [] self.__concat_keys_result_matchers = [] self.__inc_gen_result_matcher = None self.__storage_channel_result_matchers = [] - self.__do_describe = do_describe + self.__do_describe = do_describe self.__describe_actual_proto_fields = set() self.__status = status def no_describe(self, describe_fields=()): - self.__do_describe = False + self.__do_describe = False self.__describe_actual_proto_fields = set(describe_fields) - return self - + return self + def describe_to(self, description): description.append_text('Valid response with Status: %i' % self.__status) - if self.__do_describe: - if self.__inc_gen_result_matcher is not None: - description.append_text(' and IncrementGenerationResult = \n') - self.__inc_gen_result_matcher.describe_to(description) - if self.__write_matchers: - description.append_text(' and WriteResult = \n') - contains(*self.__write_matchers).describe_to(description) - if self.__delete_matcher is not None: - description.append_text(' and DeleteRangeResult = \n') - self.__delete_matcher.describe_to(description) - if self.__read_keys_matchers: - description.append_text(' and ReadResult = \n') - contains(*self.__read_keys_matchers).describe_to(description) - if self.__read_range_matchers: - description.append_text(' and ReadRangeResult = \n') - contains(*self.__read_range_matchers).describe_to(description) - if self.__copy_range_matchers: - description.append_text(' and CopyRangeResult = \n') - contains(*self.__copy_range_matchers).describe_to(description) - if self.__concat_keys_result_matchers: - description.append_text(' and ConcatResult = \n') - contains(*self.__concat_keys_result_matchers).describe_to(description) - else: - description.append_text("(Verbose mismatch description was disabled for this test)") + if self.__do_describe: + if self.__inc_gen_result_matcher is not None: + description.append_text(' and IncrementGenerationResult = \n') + self.__inc_gen_result_matcher.describe_to(description) + if self.__write_matchers: + description.append_text(' and WriteResult = \n') + contains(*self.__write_matchers).describe_to(description) + if self.__delete_matcher is not None: + description.append_text(' and DeleteRangeResult = \n') + self.__delete_matcher.describe_to(description) + if self.__read_keys_matchers: + description.append_text(' and ReadResult = \n') + contains(*self.__read_keys_matchers).describe_to(description) + if self.__read_range_matchers: + description.append_text(' and ReadRangeResult = \n') + contains(*self.__read_range_matchers).describe_to(description) + if self.__copy_range_matchers: + description.append_text(' and CopyRangeResult = \n') + contains(*self.__copy_range_matchers).describe_to(description) + if self.__concat_keys_result_matchers: + description.append_text(' and ConcatResult = \n') + contains(*self.__concat_keys_result_matchers).describe_to(description) + else: + description.append_text("(Verbose mismatch description was disabled for this test)") def describe_mismatch(self, actual_protobuf, mismatch_description): if self.__do_describe and not self.__describe_actual_proto_fields: actual_protobuf_str = str(actual_protobuf) - else: + else: lst = [] field_names = [f.name for f in actual_protobuf.DESCRIPTOR.fields] for f in field_names: @@ -89,7 +89,7 @@ class KeyValueResponseProtobufMatcher(BaseMatcher): lst.append("{f} = {v}".format(f=f, v=value)) lst.append('\n') actual_protobuf_str = '\n'.join(lst) - + mismatch_description.append_text("Actual protobuf = \n") mismatch_description.append_text("=" * 60 + "\n") mismatch_description.append_text(actual_protobuf_str) @@ -99,45 +99,45 @@ class KeyValueResponseProtobufMatcher(BaseMatcher): if not hasattr(actual_protobuf, 'Status') or actual_protobuf.Status != self.__status: return False - if self.__inc_gen_result_matcher is not None\ - and not self.__inc_gen_result_matcher.matches(actual_protobuf.IncrementGenerationResult): + if self.__inc_gen_result_matcher is not None\ + and not self.__inc_gen_result_matcher.matches(actual_protobuf.IncrementGenerationResult): return False if ( - (self.__read_range_matchers and - not contains(*self.__read_range_matchers).matches(actual_protobuf.ReadRangeResult) - ) or - (self.__rename_matchers and - not contains(*self.__rename_matchers).matches(actual_protobuf.RenameResult) - ) or - (self.__write_matchers and - not contains(*self.__write_matchers).matches(actual_protobuf.WriteResult) - ) or - (self.__delete_matcher is not None and - not self.__delete_matcher.matches(actual_protobuf.DeleteRangeResult) - ) or - (self.__copy_range_matchers and - not contains(*self.__copy_range_matchers).matches(actual_protobuf.CopyRangeResult) - ) or - (self.__concat_keys_result_matchers and - not contains(*self.__concat_keys_result_matchers).matches(actual_protobuf.ConcatResult) - ) or - (self.__read_keys_matchers and - not contains(*self.__read_keys_matchers).matches(actual_protobuf.ReadResult) + (self.__read_range_matchers and + not contains(*self.__read_range_matchers).matches(actual_protobuf.ReadRangeResult) + ) or + (self.__rename_matchers and + not contains(*self.__rename_matchers).matches(actual_protobuf.RenameResult) + ) or + (self.__write_matchers and + not contains(*self.__write_matchers).matches(actual_protobuf.WriteResult) + ) or + (self.__delete_matcher is not None and + not self.__delete_matcher.matches(actual_protobuf.DeleteRangeResult) + ) or + (self.__copy_range_matchers and + not contains(*self.__copy_range_matchers).matches(actual_protobuf.CopyRangeResult) + ) or + (self.__concat_keys_result_matchers and + not contains(*self.__concat_keys_result_matchers).matches(actual_protobuf.ConcatResult) + ) or + (self.__read_keys_matchers and + not contains(*self.__read_keys_matchers).matches(actual_protobuf.ReadResult) ) or (self.__storage_channel_result_matchers and not contains(*self.__storage_channel_result_matchers).matches(actual_protobuf.GetStatusResult) - ) + ) ): return False return True - def read_range(self, key_value_pairs, status=EReplyStatus.OK): + def read_range(self, key_value_pairs, status=EReplyStatus.OK): key_value_pair_matchers = tuple(kv_pair.protobuf_matcher() for kv_pair in key_value_pairs) self.__read_range_matchers.append( has_properties( - Status=status, + Status=status, Pair=contains(*tuple( key_value_pair_matchers )) @@ -145,37 +145,37 @@ class KeyValueResponseProtobufMatcher(BaseMatcher): ) return self - def read_key(self, value, status=EReplyStatus.OK): + def read_key(self, value, status=EReplyStatus.OK): self.__read_keys_matchers.append( has_properties( - Status=status, + Status=status, Value=to_bytes(value) ) ) return self def write(self, num_of_writes=1): - self.__write_matchers.extend( + self.__write_matchers.extend( has_properties(Status=int(EReplyStatus.OK), StatusFlags=int(TStorageStatusFlags.StatusIsValid)) for _ in range(num_of_writes) - ) - return self - + ) + return self + def add_write_result_with_status(self, status, flags): self.__write_matchers.append( has_properties(Status=int(status), StatusFlags=int(flags)) ) return self - def rename(self, num_of_renames=1): - self.__rename_matchers.extend( - has_property('Status', int(EReplyStatus.OK)) for _ in range(num_of_renames) - ) - return self - - def delete(self, num_of_keys=1): - self.__delete_matcher = contains(*tuple( - has_property('Status', EReplyStatus.OK) for _ in range(num_of_keys) + def rename(self, num_of_renames=1): + self.__rename_matchers.extend( + has_property('Status', int(EReplyStatus.OK)) for _ in range(num_of_renames) + ) + return self + + def delete(self, num_of_keys=1): + self.__delete_matcher = contains(*tuple( + has_property('Status', EReplyStatus.OK) for _ in range(num_of_keys) )) return self @@ -236,10 +236,10 @@ def is_response_with_status_and_fields(status, **kwargs): ) -def is_valid_read_range_response(key_value_pairs, - main_status=MessageBusStatus.MSTATUS_OK, result_status=EReplyStatus.OK): - return KeyValueResponseProtobufMatcher(main_status).read_range(key_value_pairs, status=result_status) +def is_valid_read_range_response(key_value_pairs, + main_status=MessageBusStatus.MSTATUS_OK, result_status=EReplyStatus.OK): + return KeyValueResponseProtobufMatcher(main_status).read_range(key_value_pairs, status=result_status) -def is_valid_keyvalue_protobuf_response(): +def is_valid_keyvalue_protobuf_response(): return KeyValueResponseProtobufMatcher() diff --git a/ydb/tests/library/matchers/tablets.py b/ydb/tests/library/matchers/tablets.py index 925c1f8200..c73b8e4297 100644 --- a/ydb/tests/library/matchers/tablets.py +++ b/ydb/tests/library/matchers/tablets.py @@ -1,106 +1,106 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from hamcrest import has_property, all_of, has_length, has_items, anything -from hamcrest.core.base_matcher import BaseMatcher -from hamcrest.core.helpers.wrap_matcher import wrap_matcher +from hamcrest import has_property, all_of, has_length, has_items, anything +from hamcrest.core.base_matcher import BaseMatcher +from hamcrest.core.helpers.wrap_matcher import wrap_matcher from ydb.tests.library.common.msgbus_types import EMessageStatus, MessageBusStatus def all_tablets_are_created(tablets_to_create): return all_of( - has_property('Status', MessageBusStatus.MSTATUS_OK), + has_property('Status', MessageBusStatus.MSTATUS_OK), has_property( - 'CreateTabletResult', + 'CreateTabletResult', all_of( has_length(len(tablets_to_create)), - has_items(has_property('Status', EMessageStatus.MESSAGE_OK)) + has_items(has_property('Status', EMessageStatus.MESSAGE_OK)) ) ) ) - - -def is_balanced_list(lst, exact_values): - _min, _max = min(lst), max(lst) - if exact_values is not None: - return _min == _max and wrap_matcher(exact_values).matches(_min) - else: - return _max - _min <= 1 - - -class ExpectedTablets(object): - def __init__(self, tablets_count=anything(), tablet_ids=()): - self.tablets_count = wrap_matcher(tablets_count) - self.tablet_ids = list(tablet_ids) - - -class TabletsBalanceMatcher(BaseMatcher): - def __init__(self): - super(TabletsBalanceMatcher, self).__init__() - self.__all_balanced = False - self.__special_nodes = {} - self.__exact_count_on_all = None - - def all_balanced(self): - self.__all_balanced = True - return self - - def with_exact_count_tablets_on_all(self, exact_count): - self.__exact_count_on_all = exact_count - return self - - def with_x_tablets_on_node(self, node_id, num_tablets): - self.__special_nodes.setdefault(node_id, ExpectedTablets()) - self.__special_nodes[node_id].tablets_count = num_tablets - return self - - def with_exact_tablet_id_on_node(self, node_id, tablet_id): - self.__special_nodes.setdefault(node_id, ExpectedTablets()) - self.__special_nodes[node_id].tablet_ids.append(tablet_id) - return self - - def _matches(self, actual_tablets_per_node): - not_special_nodes = actual_tablets_per_node.keys() - - for node_id, expected_tablets in self.__special_nodes.items(): - if node_id not in actual_tablets_per_node: - return False - if not expected_tablets.tablets_count.matches(len(actual_tablets_per_node[node_id])): - return False - - for tablet_id in expected_tablets.tablet_ids: - if tablet_id not in actual_tablets_per_node[node_id]: - return False - not_special_nodes.remove(node_id) - - tablets_by_node = [len(actual_tablets_per_node[i]) for i in not_special_nodes] - if self.__all_balanced: - return is_balanced_list( - tablets_by_node, - self.__exact_count_on_all - ) - elif self.__exact_count_on_all is not None: - return self.__exact_count_on_all.matches( - max(tablets_by_node) - ) and self.__exact_count_on_all.matches( - min(tablets_by_node) - ) - - def describe_to(self, description): - if self.__all_balanced: - description.append_text('Equally spread tablets across all nodes\n') - if self.__exact_count_on_all is not None: - description.append_text('With each node having \n') - self.__exact_count_on_all.describe_to(description) - for node_id, expected_tablets in self.__special_nodes.items(): - description.append_text('Except node %s which is expected to have\n' % str(node_id)) - description.append_text('\t num tablets: ') - expected_tablets.tablets_count.describe_to(description) - description.append_text('.') - for tablet_id in expected_tablets.tablet_ids: - description.append_text('\t tablet with id %s on it' % str(tablet_id)) - - -def are_equally_spread_tablets(): - return TabletsBalanceMatcher().all_balanced() + + +def is_balanced_list(lst, exact_values): + _min, _max = min(lst), max(lst) + if exact_values is not None: + return _min == _max and wrap_matcher(exact_values).matches(_min) + else: + return _max - _min <= 1 + + +class ExpectedTablets(object): + def __init__(self, tablets_count=anything(), tablet_ids=()): + self.tablets_count = wrap_matcher(tablets_count) + self.tablet_ids = list(tablet_ids) + + +class TabletsBalanceMatcher(BaseMatcher): + def __init__(self): + super(TabletsBalanceMatcher, self).__init__() + self.__all_balanced = False + self.__special_nodes = {} + self.__exact_count_on_all = None + + def all_balanced(self): + self.__all_balanced = True + return self + + def with_exact_count_tablets_on_all(self, exact_count): + self.__exact_count_on_all = exact_count + return self + + def with_x_tablets_on_node(self, node_id, num_tablets): + self.__special_nodes.setdefault(node_id, ExpectedTablets()) + self.__special_nodes[node_id].tablets_count = num_tablets + return self + + def with_exact_tablet_id_on_node(self, node_id, tablet_id): + self.__special_nodes.setdefault(node_id, ExpectedTablets()) + self.__special_nodes[node_id].tablet_ids.append(tablet_id) + return self + + def _matches(self, actual_tablets_per_node): + not_special_nodes = actual_tablets_per_node.keys() + + for node_id, expected_tablets in self.__special_nodes.items(): + if node_id not in actual_tablets_per_node: + return False + if not expected_tablets.tablets_count.matches(len(actual_tablets_per_node[node_id])): + return False + + for tablet_id in expected_tablets.tablet_ids: + if tablet_id not in actual_tablets_per_node[node_id]: + return False + not_special_nodes.remove(node_id) + + tablets_by_node = [len(actual_tablets_per_node[i]) for i in not_special_nodes] + if self.__all_balanced: + return is_balanced_list( + tablets_by_node, + self.__exact_count_on_all + ) + elif self.__exact_count_on_all is not None: + return self.__exact_count_on_all.matches( + max(tablets_by_node) + ) and self.__exact_count_on_all.matches( + min(tablets_by_node) + ) + + def describe_to(self, description): + if self.__all_balanced: + description.append_text('Equally spread tablets across all nodes\n') + if self.__exact_count_on_all is not None: + description.append_text('With each node having \n') + self.__exact_count_on_all.describe_to(description) + for node_id, expected_tablets in self.__special_nodes.items(): + description.append_text('Except node %s which is expected to have\n' % str(node_id)) + description.append_text('\t num tablets: ') + expected_tablets.tablets_count.describe_to(description) + description.append_text('.') + for tablet_id in expected_tablets.tablet_ids: + description.append_text('\t tablet with id %s on it' % str(tablet_id)) + + +def are_equally_spread_tablets(): + return TabletsBalanceMatcher().all_balanced() diff --git a/ydb/tests/library/nemesis/nemesis_core.py b/ydb/tests/library/nemesis/nemesis_core.py index c992f7f7bf..6886982c23 100644 --- a/ydb/tests/library/nemesis/nemesis_core.py +++ b/ydb/tests/library/nemesis/nemesis_core.py @@ -1,80 +1,80 @@ -# -*- coding: utf-8 -*- +# -*- coding: utf-8 -*- import abc import heapq import itertools -import logging -import random -import time +import logging +import random +import time import threading import six.moves from six.moves import collections_abc from ydb.tests.library.nemesis import remote_execution - - -def wrap_in_list(item): - if isinstance(item, list): - return item - else: - return [item] - - -logger = logging.getLogger(__name__) - - -class RunUnderNemesisContext(object): - def __init__(self, nemesis_process): - self.__nemesis = nemesis_process - - def __enter__(self): - if self.__nemesis is not None: - self.__nemesis.start() - - def __exit__(self, type, value, traceback): - if self.__nemesis is not None: - self.__nemesis.stop() + + +def wrap_in_list(item): + if isinstance(item, list): + return item + else: + return [item] + + +logger = logging.getLogger(__name__) + + +class RunUnderNemesisContext(object): + def __init__(self, nemesis_process): + self.__nemesis = nemesis_process + + def __enter__(self): + if self.__nemesis is not None: + self.__nemesis.start() + + def __exit__(self, type, value, traceback): + if self.__nemesis is not None: + self.__nemesis.stop() return False - + class NemesisProcess(threading.Thread): - def __init__(self, nemesis_factory, initial_sleep=10): - super(NemesisProcess, self).__init__(name='Nemesis') + def __init__(self, nemesis_factory, initial_sleep=10): + super(NemesisProcess, self).__init__(name='Nemesis') self.__nemesis_factory = nemesis_factory self.__private_nemesis_list = wrap_in_list(nemesis_factory) - self.__pq = PriorityQueue() + self.__pq = PriorityQueue() self.__is_running = threading.Event() self.__finished_running = threading.Event() - self.__initial_sleep = initial_sleep - - self.daemon = True + self.__initial_sleep = initial_sleep + + self.daemon = True self.__logger = logger.getChild(self.__class__.__name__) - - @property + + @property def __nemesis_list(self): if self.__private_nemesis_list is None: self.__private_nemesis_list = wrap_in_list(self.__nemesis_factory()) return self.__private_nemesis_list - - def stop(self): - if self.__is_running.is_set(): - self.__is_running.clear() - else: - return - + + def stop(self): + if self.__is_running.is_set(): + self.__is_running.clear() + else: + return + start_time = time.time() - self.__logger.info("Stopping Nemesis") - - # wait for Nemesis to stop - finish_time = time.time() + 480 - while not self.__finished_running.is_set() and time.time() < finish_time: - time.sleep(1) + self.__logger.info("Stopping Nemesis") + + # wait for Nemesis to stop + finish_time = time.time() + 480 + while not self.__finished_running.is_set() and time.time() < finish_time: + time.sleep(1) finish_time = time.time() self.__logger.info( "Stopped Nemesis successfully in {} seconds".format( int(finish_time - start_time) ) ) - - def run(self): + + def run(self): try: self.__run() except Exception: @@ -85,13 +85,13 @@ class NemesisProcess(threading.Thread): def __run(self): random.seed() - self.__pq.clear() - self.__is_running.set() - self.__finished_running.clear() - - time.sleep(self.__initial_sleep) + self.__pq.clear() + self.__is_running.set() + self.__finished_running.clear() + + time.sleep(self.__initial_sleep) self.__init_pq() - + while self.__is_running.is_set() and self.__pq: while self.__is_running.is_set() and time.time() < self.__pq.peek_priority(): time.sleep(1) @@ -101,7 +101,7 @@ class NemesisProcess(threading.Thread): nemesis = self.__pq.pop() try: nemesis.inject_fault() - except Exception: + except Exception: self.__logger.exception( 'Inject fault for nemesis = {nemesis} failed.'.format( nemesis=nemesis, @@ -112,87 +112,87 @@ class NemesisProcess(threading.Thread): self.__pq.add_task(task=nemesis, priority=priority) def __init_pq(self): - self.__logger.info("NemesisProcess started") + self.__logger.info("NemesisProcess started") self.__logger.info("self.nemesis_list = " + str(self.__nemesis_list)) - - # noinspection PyTypeChecker + + # noinspection PyTypeChecker for nemesis in self.__nemesis_list: - prepared = False - while not prepared: - try: - self.__logger.info("Preparing nemesis = " + str(nemesis)) - nemesis.prepare_state() - prepared = True - self.__logger.info("Preparation succeeded nemesis = " + str(nemesis)) - except Exception: - self.__logger.exception("Preparation failed for nemesis = " + str(nemesis)) - time.sleep(1) - - # noinspection PyTypeChecker + prepared = False + while not prepared: + try: + self.__logger.info("Preparing nemesis = " + str(nemesis)) + nemesis.prepare_state() + prepared = True + self.__logger.info("Preparation succeeded nemesis = " + str(nemesis)) + except Exception: + self.__logger.exception("Preparation failed for nemesis = " + str(nemesis)) + time.sleep(1) + + # noinspection PyTypeChecker for nemesis in self.__nemesis_list: - priority = time.time() + nemesis.next_schedule() - self.__pq.add_task(nemesis, priority=priority) - + priority = time.time() + nemesis.next_schedule() + self.__pq.add_task(nemesis, priority=priority) + self.__logger.debug("Initial PriorityQueue = " + str(self.__pq)) - + def __stop_nemesis(self): - # Stopping Nemesis - self.__logger.info("Stopping Nemesis in run()") - # noinspection PyTypeChecker + # Stopping Nemesis + self.__logger.info("Stopping Nemesis in run()") + # noinspection PyTypeChecker for nemesis in self.__nemesis_list: - self.__logger.info("Extracting fault for Nemesis = " + str(nemesis)) + self.__logger.info("Extracting fault for Nemesis = " + str(nemesis)) try: nemesis.extract_fault() except Exception: logger.exception('Nemesis = {nemesis} extract_fault() failed with exception = '.format(nemesis=nemesis)) - - self.__finished_running.set() - self.__logger.info("Stopped Nemesis successfully in run()") - - -class Nemesis(object): + + self.__finished_running.set() + self.__logger.info("Stopped Nemesis successfully in run()") + + +class Nemesis(object): __metaclass__ = abc.ABCMeta - - def __init__(self, schedule): - self.__schedule = Schedule.from_tuple_or_int(schedule) + + def __init__(self, schedule): + self.__schedule = Schedule.from_tuple_or_int(schedule) self.__logger = logging.getLogger(self.__class__.__name__) - - def next_schedule(self): + + def next_schedule(self): """ Return amount of second from current time this Nemesis should be called with `inject_fault()` :return: amount of seconds to schedule this Nemesis next time """ - return next(self.__schedule) - + return next(self.__schedule) + @abc.abstractmethod - def prepare_state(self): + def prepare_state(self): """ Prepare state of your Nemesis. Called only once on start. :return: not specified """ - pass - + pass + @abc.abstractmethod - def inject_fault(self): + def inject_fault(self): """ Inject some fault into running cluster. :return: not specified """ - pass - + pass + @abc.abstractmethod - def extract_fault(self): + def extract_fault(self): """ Cancel all injected fault if this is possible. Some faults can't be canceled (e.g. node formatting). Usually called at the end of the Nemesis run. :return: not specified """ - pass - + pass + @property def logger(self): """ @@ -205,179 +205,179 @@ class Nemesis(object): def __str__(self): return self.__repr__() - def __repr__(self): + def __repr__(self): return '{ntype}'.format(ntype=type(self)) - - -class AbstractNemesisNodeTerrorist(Nemesis): + + +class AbstractNemesisNodeTerrorist(Nemesis): __metaclass__ = abc.ABCMeta - + def __init__(self, node_list, act_interval, ssh_user=None, remote_sudo_user=None, timeout=60): - super(AbstractNemesisNodeTerrorist, self).__init__(act_interval) + super(AbstractNemesisNodeTerrorist, self).__init__(act_interval) self._remote_exec_method = remote_execution.execute_command - self.__nodes = node_list - self.__ssh_user = ssh_user - self.__sudo_user = remote_sudo_user - + self.__nodes = node_list + self.__ssh_user = ssh_user + self.__sudo_user = remote_sudo_user + self.__timeout = timeout - - def _get_victim_node(self): - return random.choice(self.__nodes) - + + def _get_victim_node(self): + return random.choice(self.__nodes) + @property def timeout(self): return self.__timeout @abc.abstractmethod - def _commands_on_node(self, node=None): - pass - - def _pre_inject_fault(self, node, commands): - pass - - def _post_inject_fault(self, node, commands): - pass - - def prepare_state(self): - pass - - def extract_fault(self): - pass - - def inject_fault(self): - success = True - node = random.choice(self.__nodes) - node_commands = wrap_in_list(self._commands_on_node(node)) - - self._pre_inject_fault(node, node_commands) - - for command in node_commands: + def _commands_on_node(self, node=None): + pass + + def _pre_inject_fault(self, node, commands): + pass + + def _post_inject_fault(self, node, commands): + pass + + def prepare_state(self): + pass + + def extract_fault(self): + pass + + def inject_fault(self): + success = True + node = random.choice(self.__nodes) + node_commands = wrap_in_list(self._commands_on_node(node)) + + self._pre_inject_fault(node, node_commands) + + for command in node_commands: cmd_result = self._remote_exec_method(self._full_command(node, command)) - success = cmd_result == 0 and success - - self._post_inject_fault(node, node_commands) - - return success - - def _full_command(self, node, node_command): - if self.__ssh_user is not None: - address = "%s@%s" % (self.__ssh_user, node) - else: - address = node - if self.__sudo_user is not None: - if self.__sudo_user == 'root': - return ["ssh", address, "sudo"] + list(node_command) - else: - return [ - "ssh", address, - "sudo", "-u", self.__sudo_user - ] + list(node_command) - else: - return ["ssh", address] + list(node_command) - - + success = cmd_result == 0 and success + + self._post_inject_fault(node, node_commands) + + return success + + def _full_command(self, node, node_command): + if self.__ssh_user is not None: + address = "%s@%s" % (self.__ssh_user, node) + else: + address = node + if self.__sudo_user is not None: + if self.__sudo_user == 'root': + return ["ssh", address, "sudo"] + list(node_command) + else: + return [ + "ssh", address, + "sudo", "-u", self.__sudo_user + ] + list(node_command) + else: + return ["ssh", address] + list(node_command) + + class Schedule(collections_abc.Iterator): - def __init__(self, schedule_iterator): - self.__iterator = schedule_iterator - - def __iter__(self): - return self - + def __init__(self, schedule_iterator): + self.__iterator = schedule_iterator + + def __iter__(self): + return self + def __next__(self): return next(self.__iterator) - def next(self): - return next(self.__iterator) - - @staticmethod - def combine(list_of_iterators): - return Schedule(itertools.chain.from_iterable(list_of_iterators)) - - @staticmethod - def cycle(factory_of_list_of_iterators): - def iterable(): - while True: - list_of_iterators = factory_of_list_of_iterators() - for item in list_of_iterators: - yield item - - return Schedule(itertools.chain.from_iterable(iterable())) - - @staticmethod - def from_tuple_or_int(value): - if isinstance(value, Schedule): - return value - elif isinstance(value, int): - return Schedule(itertools.repeat(value)) - else: - a, b = value + def next(self): + return next(self.__iterator) + + @staticmethod + def combine(list_of_iterators): + return Schedule(itertools.chain.from_iterable(list_of_iterators)) + + @staticmethod + def cycle(factory_of_list_of_iterators): + def iterable(): + while True: + list_of_iterators = factory_of_list_of_iterators() + for item in list_of_iterators: + yield item + + return Schedule(itertools.chain.from_iterable(iterable())) + + @staticmethod + def from_tuple_or_int(value): + if isinstance(value, Schedule): + return value + elif isinstance(value, int): + return Schedule(itertools.repeat(value)) + else: + a, b = value return Schedule(six.moves.map(lambda x: random.randint(a, b), itertools.count())) - - + + class RunOnceSchedule(Schedule): def __init__(self, time_at): super(RunOnceSchedule, self).__init__(iter([time_at, 2**50])) -class PriorityQueue(object): - def __init__(self): - super(PriorityQueue, self).__init__() - self.__heap = [] - self.__count = itertools.count() - - def add_task(self, task, priority): - entry = priority, next(self.__count), task - heapq.heappush(self.__heap, entry) - - def pop(self): - _, _, task = heapq.heappop(self.__heap) - return task - - def peek(self): - _, _, task = self.__heap[0] - return task - - def peek_priority(self): - priority, _, _ = self.__heap[0] - return priority - - def clear(self): - self.__heap = [] - - def __len__(self): - return len(self.__heap) - - def __str__(self): +class PriorityQueue(object): + def __init__(self): + super(PriorityQueue, self).__init__() + self.__heap = [] + self.__count = itertools.count() + + def add_task(self, task, priority): + entry = priority, next(self.__count), task + heapq.heappush(self.__heap, entry) + + def pop(self): + _, _, task = heapq.heappop(self.__heap) + return task + + def peek(self): + _, _, task = self.__heap[0] + return task + + def peek_priority(self): + priority, _, _ = self.__heap[0] + return priority + + def clear(self): + self.__heap = [] + + def __len__(self): + return len(self.__heap) + + def __str__(self): # return str(self.__heap) return '\n'.join(map(str, sorted([(entry[0], entry[1]) for entry in self.__heap]))) - - -class FakeNemesis(Nemesis): + + +class FakeNemesis(Nemesis): """ >>> n = FakeNemesis(None) >>> print(n.logger.name) FakeNemesis """ - + def __init__(self, out_queue, schedule=1): super(FakeNemesis, self).__init__(schedule=schedule) self.__out_queue = out_queue - - def inject_fault(self): + + def inject_fault(self): self.__out_queue.put('inject_fault') - - def prepare_state(self): + + def prepare_state(self): self.__out_queue.put('prepare_state') - - def extract_fault(self): + + def extract_fault(self): self.__out_queue.put('extract_fault') - - def next_schedule(self): + + def next_schedule(self): n = super(FakeNemesis, self).next_schedule() self.__out_queue.put('next_schedule = ' + str(n)) return n - - + + class ExceptionThrowingNemesis(Nemesis): def __init__(self, out_queue, schedule=1): @@ -400,54 +400,54 @@ class ExceptionThrowingNemesis(Nemesis): return n -def test_priority_queue(): - """ - >>> pq = PriorityQueue() +def test_priority_queue(): + """ + >>> pq = PriorityQueue() >>> pq.add_task('last', time.time() + 10) >>> pq.add_task('first', time.time()) >>> pq.add_task('task 4', time.time() + 4) >>> pq.add_task('task 3', time.time() + 3) >>> pq.add_task('task 2', time.time() + 2) - >>> pq.pop() + >>> pq.pop() 'first' - >>> pq.peek() + >>> pq.peek() 'task 2' - >>> len(pq) - 4 - >>> pq.pop() + >>> len(pq) + 4 + >>> pq.pop() 'task 2' - >>> len(pq) - 3 - >>> bool(pq) - True + >>> len(pq) + 3 + >>> bool(pq) + True >>> pq.add_task('first', time.time()) - >>> pq.pop() + >>> pq.pop() 'first' - >>> pq.pop() + >>> pq.pop() 'task 3' - >>> pq.pop() + >>> pq.pop() 'task 4' >>> pq.pop() 'last' - >>> bool(pq) - False - """ - - -def test_nemesis_process(): - """ - >>> from multiprocessing import Queue + >>> bool(pq) + False + """ + + +def test_nemesis_process(): + """ + >>> from multiprocessing import Queue >>> timeout = 10 - >>> out_queue = Queue() + >>> out_queue = Queue() >>> np = NemesisProcess(FakeNemesis(out_queue, schedule=1), initial_sleep=0) - >>> np.start() - >>> time.sleep(1) + >>> np.start() + >>> time.sleep(1) >>> out_queue.get(timeout=timeout) - 'prepare_state' + 'prepare_state' >>> out_queue.get(timeout=timeout) 'next_schedule = 1' >>> out_queue.get(timeout=timeout) - 'inject_fault' + 'inject_fault' >>> out_queue.get(timeout=timeout) 'next_schedule = 1' >>> out_queue.get(timeout=timeout) @@ -457,25 +457,25 @@ def test_nemesis_process(): >>> np.stop() >>> out_queue.get(timeout=timeout) - 'extract_fault' + 'extract_fault' >>> np = NemesisProcess(ExceptionThrowingNemesis(out_queue, schedule=1), initial_sleep=0) >>> np.start() >>> time.sleep(3) >>> np.stop() - """ - - -def test_schedule(): - """ - >>> random.seed(123) - >>> s = Schedule.from_tuple_or_int((10, 20)) - >>> [next(s) for _ in range(10)] + """ + + +def test_schedule(): + """ + >>> random.seed(123) + >>> s = Schedule.from_tuple_or_int((10, 20)) + >>> [next(s) for _ in range(10)] [10, 14, 11, 16, 14, 11, 10, 16, 18, 18] - >>> s = Schedule.from_tuple_or_int(15) - >>> [next(s) for _ in range(10)] - [15, 15, 15, 15, 15, 15, 15, 15, 15, 15] - >>> q = Schedule.combine(iter([[1, 2], Schedule.from_tuple_or_int(5)])) - >>> [next(q) for _ in range(10)] - [1, 2, 5, 5, 5, 5, 5, 5, 5, 5] - """ + >>> s = Schedule.from_tuple_or_int(15) + >>> [next(s) for _ in range(10)] + [15, 15, 15, 15, 15, 15, 15, 15, 15, 15] + >>> q = Schedule.combine(iter([[1, 2], Schedule.from_tuple_or_int(5)])) + >>> [next(q) for _ in range(10)] + [1, 2, 5, 5, 5, 5, 5, 5, 5, 5] + """ diff --git a/ydb/tests/library/nemesis/nemesis_process_killers.py b/ydb/tests/library/nemesis/nemesis_process_killers.py index df8e88be1b..2ec73a7b35 100644 --- a/ydb/tests/library/nemesis/nemesis_process_killers.py +++ b/ydb/tests/library/nemesis/nemesis_process_killers.py @@ -1,69 +1,69 @@ # -*- coding: utf-8 -*- -import logging +import logging import random from ydb.tests.library.nemesis.nemesis_core import AbstractNemesisNodeTerrorist from ydb.tests.library.nemesis.remote_execution import execute_command_with_output - + logger = logging.getLogger(__name__) - -DEFAULT_KILL_ALL_CHANCE = 0.3 - - -def kill_cmd(process, signal): - return ( - "killall", - "-s", - str(signal), - str(process) - ) - - -def kill_pid_cmd(pid, signal): - return ( - 'kill', - '-s', - str(signal), - str(pid) - ) - - -class NemesisProcessKiller(AbstractNemesisNodeTerrorist): - def __init__( - self, cluster, processes_affected, act_interval, ssh_user=None, remote_sudo_user=None, - kill_everything_chance=DEFAULT_KILL_ALL_CHANCE, signal='SIGKILL', processes_filters=None, - timeout=3.0 - ): - super(NemesisProcessKiller, self).__init__(cluster, act_interval, timeout=timeout, - ssh_user=ssh_user, remote_sudo_user=remote_sudo_user) - self.__victims_procs = processes_affected + +DEFAULT_KILL_ALL_CHANCE = 0.3 + + +def kill_cmd(process, signal): + return ( + "killall", + "-s", + str(signal), + str(process) + ) + + +def kill_pid_cmd(pid, signal): + return ( + 'kill', + '-s', + str(signal), + str(pid) + ) + + +class NemesisProcessKiller(AbstractNemesisNodeTerrorist): + def __init__( + self, cluster, processes_affected, act_interval, ssh_user=None, remote_sudo_user=None, + kill_everything_chance=DEFAULT_KILL_ALL_CHANCE, signal='SIGKILL', processes_filters=None, + timeout=3.0 + ): + super(NemesisProcessKiller, self).__init__(cluster, act_interval, timeout=timeout, + ssh_user=ssh_user, remote_sudo_user=remote_sudo_user) + self.__victims_procs = processes_affected self.__kill_all_chance = kill_everything_chance - self._signal = signal - - self._command_for_victim = kill_cmd if processes_filters is None else kill_pid_cmd - self.__process_filters = processes_filters - - def _commands_on_node(self, node=None): - ret = [] - victim_candidates = self._get_victim(node) - - if random.random() < self.__kill_all_chance or not victim_candidates: - real_victims = victim_candidates - else: - real_victims = [random.choice(victim_candidates), ] - - for prc in real_victims: - ret.append( - self._command_for_victim(prc, self._signal) - ) - return ret - - def _get_victim(self, node=None): - if self.__process_filters is None: - return self.__victims_procs - else: - return self.__get_victims_by_filter(node) - - def __get_victims_by_filter(self, node): + self._signal = signal + + self._command_for_victim = kill_cmd if processes_filters is None else kill_pid_cmd + self.__process_filters = processes_filters + + def _commands_on_node(self, node=None): + ret = [] + victim_candidates = self._get_victim(node) + + if random.random() < self.__kill_all_chance or not victim_candidates: + real_victims = victim_candidates + else: + real_victims = [random.choice(victim_candidates), ] + + for prc in real_victims: + ret.append( + self._command_for_victim(prc, self._signal) + ) + return ret + + def _get_victim(self, node=None): + if self.__process_filters is None: + return self.__victims_procs + else: + return self.__get_victims_by_filter(node) + + def __get_victims_by_filter(self, node): ps_command = ['ps', 'ax'] result, output = execute_command_with_output(self._full_command(node, ps_command), timeout=self.timeout) if result: @@ -74,80 +74,80 @@ class NemesisProcessKiller(AbstractNemesisNodeTerrorist): lambda x: x.strip().split(' ')[0], self.__filter_output(output) ) - - def __filter_output(self, output): - lines_match = {} - for i in range(len(output)): - lines_match[i] = True - - for filt in self.__process_filters: - for line_number, still_matching in lines_match.items(): - if still_matching: - if filt not in output[line_number]: - lines_match[line_number] = False - - return [output[i] for i in lines_match if lines_match[i]] - - -class NemesisProcessSuspender(NemesisProcessKiller): - def __init__( - self, cluster, processes_affected, act_interval, ssh_user=None, remote_sudo_user=None, - kill_everything_chance=DEFAULT_KILL_ALL_CHANCE, processes_filters=None - ): - + + def __filter_output(self, output): + lines_match = {} + for i in range(len(output)): + lines_match[i] = True + + for filt in self.__process_filters: + for line_number, still_matching in lines_match.items(): + if still_matching: + if filt not in output[line_number]: + lines_match[line_number] = False + + return [output[i] for i in lines_match if lines_match[i]] + + +class NemesisProcessSuspender(NemesisProcessKiller): + def __init__( + self, cluster, processes_affected, act_interval, ssh_user=None, remote_sudo_user=None, + kill_everything_chance=DEFAULT_KILL_ALL_CHANCE, processes_filters=None + ): + super(NemesisProcessSuspender, self).__init__( - cluster, processes_affected, act_interval, ssh_user, remote_sudo_user, - kill_everything_chance, signal='SIGSTOP', processes_filters=processes_filters + cluster, processes_affected, act_interval, ssh_user, remote_sudo_user, + kill_everything_chance, signal='SIGSTOP', processes_filters=processes_filters ) - self.__to_unsuspend = [] - - def __wake_all_suspended(self): - for node, commands in self.__to_unsuspend: - for command in commands: + self.__to_unsuspend = [] + + def __wake_all_suspended(self): + for node, commands in self.__to_unsuspend: + for command in commands: self._remote_exec_method(self._full_command(node, command), timeout=self.timeout) - self.__to_unsuspend = [] - - def _pre_inject_fault(self, node, commands): - self.__wake_all_suspended() - wake_commands = [self._command_for_victim(i[-1], signal="SIGCONT") for i in commands] - self.__to_unsuspend.append((node, wake_commands)) - - def extract_fault(self): - self.__wake_all_suspended() - - -def fake_execute_command(command): - print("{}".format(command)) - - + self.__to_unsuspend = [] + + def _pre_inject_fault(self, node, commands): + self.__wake_all_suspended() + wake_commands = [self._command_for_victim(i[-1], signal="SIGCONT") for i in commands] + self.__to_unsuspend.append((node, wake_commands)) + + def extract_fault(self): + self.__wake_all_suspended() + + +def fake_execute_command(command): + print("{}".format(command)) + + class TestNemesisProcessSuspender(NemesisProcessSuspender): - _remote_exec_method = fake_execute_command + _remote_exec_method = fake_execute_command """ >>> random.seed(123) >>> n = TestNemesisProcessSuspender(range(8), range(2), 1) >>> n.prepare_state() >>> n.inject_fault() - ['ssh', 0, 'killall', '-s', 'SIGSTOP', '0'] - ['ssh', 0, 'killall', '-s', 'SIGSTOP', '1'] + ['ssh', 0, 'killall', '-s', 'SIGSTOP', '0'] + ['ssh', 0, 'killall', '-s', 'SIGSTOP', '1'] >>> n.inject_fault() - ['ssh', 0, 'killall', '-s', 'SIGCONT', '0'] - ['ssh', 0, 'killall', '-s', 'SIGCONT', '1'] - ['ssh', 3, 'killall', '-s', 'SIGSTOP', '0'] - ['ssh', 3, 'killall', '-s', 'SIGSTOP', '1'] + ['ssh', 0, 'killall', '-s', 'SIGCONT', '0'] + ['ssh', 0, 'killall', '-s', 'SIGCONT', '1'] + ['ssh', 3, 'killall', '-s', 'SIGSTOP', '0'] + ['ssh', 3, 'killall', '-s', 'SIGSTOP', '1'] >>> n.inject_fault() - ['ssh', 3, 'killall', '-s', 'SIGCONT', '0'] - ['ssh', 3, 'killall', '-s', 'SIGCONT', '1'] - ['ssh', 7, 'killall', '-s', 'SIGSTOP', '0'] - ['ssh', 7, 'killall', '-s', 'SIGSTOP', '1'] + ['ssh', 3, 'killall', '-s', 'SIGCONT', '0'] + ['ssh', 3, 'killall', '-s', 'SIGCONT', '1'] + ['ssh', 7, 'killall', '-s', 'SIGSTOP', '0'] + ['ssh', 7, 'killall', '-s', 'SIGSTOP', '1'] >>> n.inject_fault() - ['ssh', 7, 'killall', '-s', 'SIGCONT', '0'] - ['ssh', 7, 'killall', '-s', 'SIGCONT', '1'] - ['ssh', 4, 'killall', '-s', 'SIGSTOP', '1'] + ['ssh', 7, 'killall', '-s', 'SIGCONT', '0'] + ['ssh', 7, 'killall', '-s', 'SIGCONT', '1'] + ['ssh', 4, 'killall', '-s', 'SIGSTOP', '1'] >>> n.inject_fault() - ['ssh', 4, 'killall', '-s', 'SIGCONT', '1'] - ['ssh', 1, 'killall', '-s', 'SIGSTOP', '0'] + ['ssh', 4, 'killall', '-s', 'SIGCONT', '1'] + ['ssh', 1, 'killall', '-s', 'SIGSTOP', '0'] >>> n.inject_fault() - ['ssh', 1, 'killall', '-s', 'SIGCONT', '0'] - ['ssh', 1, 'killall', '-s', 'SIGSTOP', '0'] - ['ssh', 1, 'killall', '-s', 'SIGSTOP', '1'] + ['ssh', 1, 'killall', '-s', 'SIGCONT', '0'] + ['ssh', 1, 'killall', '-s', 'SIGSTOP', '0'] + ['ssh', 1, 'killall', '-s', 'SIGSTOP', '1'] """ diff --git a/ydb/tests/library/nemesis/nemesis_time_terrorist.py b/ydb/tests/library/nemesis/nemesis_time_terrorist.py index fb32d01c5f..564b721bfb 100644 --- a/ydb/tests/library/nemesis/nemesis_time_terrorist.py +++ b/ydb/tests/library/nemesis/nemesis_time_terrorist.py @@ -1,46 +1,46 @@ -# -*- coding: utf-8 -*- -from datetime import datetime, timedelta -import logging -from random import randint - +# -*- coding: utf-8 -*- +from datetime import datetime, timedelta +import logging +from random import randint + from ydb.tests.library.nemesis.nemesis_core import AbstractNemesisNodeTerrorist from ydb.tests.library.nemesis.remote_execution import execute_command_with_output - -logger = logging.getLogger(__name__) - - -class NemesisTimeChanger(AbstractNemesisNodeTerrorist): - def __init__( - self, cluster, act_interval, ssh_user=None, remote_sudo_user=None, - time_change_range=(-60, 60), timeout=3.0 - ): - super(NemesisTimeChanger, self).__init__( - cluster, act_interval, ssh_user=ssh_user, remote_sudo_user=remote_sudo_user, timeout=timeout - ) - self.__change_range = time_change_range - self.__get_time_cmd = ('date', '+%s') - - @staticmethod - def set_time_cmd(time_to_set): - time_str = '{:02d}:{:02d}:{:02d}'.format(time_to_set.hour, time_to_set.minute, time_to_set.second) - return ( - 'date', '+%T', '-s', time_str - ) - - def _commands_on_node(self, node=None): - current_time = self.__get_curr_time(node) - if current_time is None: - return [] - else: - delta = timedelta(seconds=randint(*self.__change_range)) - return self.set_time_cmd(current_time + delta) - - def __get_curr_time(self, node): - result, output = execute_command_with_output( + +logger = logging.getLogger(__name__) + + +class NemesisTimeChanger(AbstractNemesisNodeTerrorist): + def __init__( + self, cluster, act_interval, ssh_user=None, remote_sudo_user=None, + time_change_range=(-60, 60), timeout=3.0 + ): + super(NemesisTimeChanger, self).__init__( + cluster, act_interval, ssh_user=ssh_user, remote_sudo_user=remote_sudo_user, timeout=timeout + ) + self.__change_range = time_change_range + self.__get_time_cmd = ('date', '+%s') + + @staticmethod + def set_time_cmd(time_to_set): + time_str = '{:02d}:{:02d}:{:02d}'.format(time_to_set.hour, time_to_set.minute, time_to_set.second) + return ( + 'date', '+%T', '-s', time_str + ) + + def _commands_on_node(self, node=None): + current_time = self.__get_curr_time(node) + if current_time is None: + return [] + else: + delta = timedelta(seconds=randint(*self.__change_range)) + return self.set_time_cmd(current_time + delta) + + def __get_curr_time(self, node): + result, output = execute_command_with_output( self._full_command(node, self.__get_time_cmd), timeout=self.timeout - ) - if result: - logger.error("Failed to fetch victim process on node {node}. Will not act this time".format(node=node)) - return None - else: - return datetime.fromtimestamp(int(output[0].strip())) + ) + if result: + logger.error("Failed to fetch victim process on node {node}. Will not act this time".format(node=node)) + return None + else: + return datetime.fromtimestamp(int(output[0].strip())) diff --git a/ydb/tests/library/nemesis/remote_execution.py b/ydb/tests/library/nemesis/remote_execution.py index 15e6827424..b7b5be9c68 100644 --- a/ydb/tests/library/nemesis/remote_execution.py +++ b/ydb/tests/library/nemesis/remote_execution.py @@ -9,19 +9,19 @@ from concurrent import futures logger = logging.getLogger(__name__) -def execute_command(command, timeout=60): +def execute_command(command, timeout=60): logger.info("Running: {}".format(command)) process = subprocess.Popen(command) - wait_timeout(process, timeout) - return process.returncode - - + wait_timeout(process, timeout) + return process.returncode + + def execute_command_with_output(command, timeout=60): logger.info("Running command = {}".format(command)) list_of_lines = [] with tempfile.TemporaryFile() as f_out, tempfile.TemporaryFile() as f_err: process = subprocess.Popen(command, stdout=f_out, stderr=f_err) - wait_timeout(process, timeout) + wait_timeout(process, timeout) process_return_code = process.returncode f_err.flush() f_err.seek(0) @@ -30,22 +30,22 @@ def execute_command_with_output(command, timeout=60): f_out.flush() f_out.seek(0) list_of_lines = list(f_out.readlines()) - - logger.info("Finished execution command = {}".format(command)) + + logger.info("Finished execution command = {}".format(command)) if logger.isEnabledFor(logging.DEBUG): logger.debug('Command output = \n{}'.format('\n'.join(list_of_lines))) logger.debug('Stderr = \n{}'.format('\n'.join(std_err_lines))) return process_return_code, list_of_lines - -def wait_timeout(process, timeout): - start_time = time.time() - while time.time() < start_time + timeout: - process.poll() - if process.returncode is not None: - return - time.sleep(0.5) - process.kill() + +def wait_timeout(process, timeout): + start_time = time.time() + while time.time() < start_time + timeout: + process.poll() + if process.returncode is not None: + return + time.sleep(0.5) + process.kill() process.wait() diff --git a/ydb/tests/library/nemesis/safety_warden.py b/ydb/tests/library/nemesis/safety_warden.py index 51822ed0a1..e9d43a0c38 100644 --- a/ydb/tests/library/nemesis/safety_warden.py +++ b/ydb/tests/library/nemesis/safety_warden.py @@ -1,48 +1,48 @@ -# -*- coding: utf-8 -*- - +# -*- coding: utf-8 -*- + import functools import itertools -import logging -from abc import ABCMeta, abstractmethod +import logging +from abc import ABCMeta, abstractmethod from ydb.tests.library.nemesis.remote_execution import execute_command_with_output_on_hosts - - -logger = logging.getLogger() - - -class SafetyWarden(object): - __metaclass__ = ABCMeta - + + +logger = logging.getLogger() + + +class SafetyWarden(object): + __metaclass__ = ABCMeta + def __init__(self, name): super(SafetyWarden, self).__init__() self.__name = name - @abstractmethod - def list_of_safety_violations(self): - pass - + @abstractmethod + def list_of_safety_violations(self): + pass + @property - def name(self): + def name(self): return self.__name - + def __str__(self): return self.name - -class AggregateSafetyWarden(SafetyWarden): - def __init__(self, list_of_safety_wardens): + +class AggregateSafetyWarden(SafetyWarden): + def __init__(self, list_of_safety_wardens): super(AggregateSafetyWarden, self).__init__('AggregateSafetyWarden') self.__list_of_safety_wardens = list(list_of_safety_wardens) - - def list_of_safety_violations(self): - all_safety_violations = [] + + def list_of_safety_violations(self): + all_safety_violations = [] for warden in self.__list_of_safety_wardens: - all_safety_violations.extend( - warden.list_of_safety_violations() - ) - return all_safety_violations - - + all_safety_violations.extend( + warden.list_of_safety_violations() + ) + return all_safety_violations + + def split_in_chunks(list_of_lines, chunk_size): """ >>> a = list(map(str, range(6))) @@ -132,7 +132,7 @@ class GrepLogFileForMarkers(AbstractRemoteCommandExecutionSafetyWarden): + [ log_file_name, ] - ) + ) remote_command.extend( [ |