diff options
author | mokhotskii <mokhotskii@yandex-team.ru> | 2022-02-10 16:49:07 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:49:07 +0300 |
commit | edfa737d405b60f2d9f05f230c1e97118c14f5bb (patch) | |
tree | 9a35074610afa0306a528c0086a939233c0ca06b | |
parent | 72941fe161cb3d63209386c3229b4b877b1f6199 (diff) | |
download | ydb-edfa737d405b60f2d9f05f230c1e97118c14f5bb.tar.gz |
Restoring authorship annotation for <mokhotskii@yandex-team.ru>. Commit 1 of 2.
40 files changed, 3211 insertions, 3211 deletions
diff --git a/contrib/python/botocore/ya.make b/contrib/python/botocore/ya.make index dc02d4a419..9f7b34b17d 100644 --- a/contrib/python/botocore/ya.make +++ b/contrib/python/botocore/ya.make @@ -568,10 +568,10 @@ RESOURCE_FILES( #botocore/data/kinesis-video-media/2017-09-30/service-2.json #botocore/data/kinesis-video-signaling/2019-12-04/paginators-1.json #botocore/data/kinesis-video-signaling/2019-12-04/service-2.json - botocore/data/kinesis/2013-12-02/examples-1.json - botocore/data/kinesis/2013-12-02/paginators-1.json - botocore/data/kinesis/2013-12-02/service-2.json - botocore/data/kinesis/2013-12-02/waiters-2.json + botocore/data/kinesis/2013-12-02/examples-1.json + botocore/data/kinesis/2013-12-02/paginators-1.json + botocore/data/kinesis/2013-12-02/service-2.json + botocore/data/kinesis/2013-12-02/waiters-2.json #botocore/data/kinesisanalytics/2015-08-14/examples-1.json #botocore/data/kinesisanalytics/2015-08-14/paginators-1.json #botocore/data/kinesisanalytics/2015-08-14/service-2.json diff --git a/ydb/core/client/server/msgbus_server_pq_metacache.cpp b/ydb/core/client/server/msgbus_server_pq_metacache.cpp index f44ab6f480..a88fe52091 100644 --- a/ydb/core/client/server/msgbus_server_pq_metacache.cpp +++ b/ydb/core/client/server/msgbus_server_pq_metacache.cpp @@ -26,21 +26,21 @@ IActor* CreateSchemeCache(NActors::TActorSystem* ActorSystem, TIntrusivePtr<NMon class TPersQueueMetaCacheActor : public TActorBootstrapped<TPersQueueMetaCacheActor> { using TBase = TActorBootstrapped<TPersQueueMetaCacheActor>; public: - TPersQueueMetaCacheActor(TPersQueueMetaCacheActor&&) = default; - TPersQueueMetaCacheActor& operator=(TPersQueueMetaCacheActor&&) = default; - - TPersQueueMetaCacheActor(ui64 grpcPort, - const NMonitoring::TDynamicCounterPtr& counters, - const TDuration& versionCheckInterval) + TPersQueueMetaCacheActor(TPersQueueMetaCacheActor&&) = default; + TPersQueueMetaCacheActor& operator=(TPersQueueMetaCacheActor&&) = default; + + TPersQueueMetaCacheActor(ui64 grpcPort, + const NMonitoring::TDynamicCounterPtr& counters, + const TDuration& versionCheckInterval) : Counters(counters) - , ClientWrapper(std::move(std::make_unique<TClientWrapper>(grpcPort))) + , ClientWrapper(std::move(std::make_unique<TClientWrapper>(grpcPort))) , VersionCheckInterval(versionCheckInterval) , Generation(std::make_shared<TAtomicCounter>()) { } - TPersQueueMetaCacheActor(const NMonitoring::TDynamicCounterPtr& counters, - const TDuration& versionCheckInterval) + TPersQueueMetaCacheActor(const NMonitoring::TDynamicCounterPtr& counters, + const TDuration& versionCheckInterval) : Counters(counters) , VersionCheckInterval(versionCheckInterval) , Generation(std::make_shared<TAtomicCounter>()) @@ -58,7 +58,7 @@ public: Die(ctx); return; } - ClientWrapper.reset(new TClientWrapper(driver)); + ClientWrapper.reset(new TClientWrapper(driver)); } SkipVersionCheck = !AppData(ctx)->PQConfig.GetMetaCacheSkipVersionCheck(); PathPrefix = TopicPrefix(ctx); @@ -358,7 +358,7 @@ private: void SendSchemeCacheRequest(const TVector<TString>& topics, bool addDefaultPathPrefix, const TActorContext& ctx) { - auto schemeCacheRequest = std::make_unique<NSchemeCache::TSchemeCacheNavigate>(++RequestId); + auto schemeCacheRequest = std::make_unique<NSchemeCache::TSchemeCacheNavigate>(++RequestId); for (const auto& path : topics) { auto split = NKikimr::SplitPath(path); Y_VERIFY(!split.empty()); @@ -372,7 +372,7 @@ private: entry.Operation = NSchemeCache::TSchemeCacheNavigate::OpTopic; schemeCacheRequest->ResultSet.emplace_back(std::move(entry)); } - ctx.Send(SchemeCacheId, new TEvTxProxySchemeCache::TEvNavigateKeySet(schemeCacheRequest.release())); + ctx.Send(SchemeCacheId, new TEvTxProxySchemeCache::TEvNavigateKeySet(schemeCacheRequest.release())); } void SendListTopicsResponse(const TActorId& recipient, const TActorContext& ctx) { @@ -444,11 +444,11 @@ private: class TClientWrapper { public: - TClientWrapper(const TClientWrapper&) = delete; - TClientWrapper& operator=(const TClientWrapper&) = delete; - TClientWrapper(TClientWrapper&&) = default; - TClientWrapper& operator=(TClientWrapper&&) = default; - + TClientWrapper(const TClientWrapper&) = delete; + TClientWrapper& operator=(const TClientWrapper&) = delete; + TClientWrapper(TClientWrapper&&) = default; + TClientWrapper& operator=(TClientWrapper&&) = default; + TClientWrapper(ui64 driverPort) : DriverPort(driverPort) {} @@ -464,18 +464,18 @@ private: auto driverConfig = NYdb::TDriverConfig().SetEndpoint(endpoint); DriverHolder.Reset(new NYdb::TDriver(driverConfig)); Driver = DriverHolder.Get(); - TableClient.reset(new NYdb::NTable::TTableClient(*Driver)); + TableClient.reset(new NYdb::NTable::TTableClient(*Driver)); } } else if (Driver != nullptr) { - TableClient.reset(new NYdb::NTable::TTableClient(*Driver)); + TableClient.reset(new NYdb::NTable::TTableClient(*Driver)); } } - + NYdb::NTable::TTableClient* GetClient() { Y_VERIFY(TableClient); - return TableClient.get(); + return TableClient.get(); } - + void Stop() { if (DriverHolder != nullptr) { TableClient->Stop(); @@ -489,15 +489,15 @@ private: THolder<NYdb::TDriver> DriverHolder; NYdb::TDriver* Driver = nullptr; TMaybe<ui64> DriverPort; - std::unique_ptr<NYdb::NTable::TTableClient> TableClient; - }; + std::unique_ptr<NYdb::NTable::TTableClient> TableClient; + }; NMonitoring::TDynamicCounterPtr Counters; NActors::TActorSystem* ActorSystem; TString VersionQuery; TString TopicsQuery; - std::unique_ptr<TClientWrapper> ClientWrapper; + std::unique_ptr<TClientWrapper> ClientWrapper; TAsyncCreateSessionResult SessionFuture; TMaybe<TSession> YdbSession; TAsyncPrepareQueryResult TopicsQueryFuture; diff --git a/ydb/core/grpc_services/rpc_scheme_base.h b/ydb/core/grpc_services/rpc_scheme_base.h index 0be68e802e..444e56a556 100644 --- a/ydb/core/grpc_services/rpc_scheme_base.h +++ b/ydb/core/grpc_services/rpc_scheme_base.h @@ -10,7 +10,7 @@ namespace NGRpcService { template <typename TDerived, typename TRequest> class TRpcSchemeRequestActor : public TRpcOperationRequestActor<TDerived, TRequest> { -protected: +protected: using TBase = TRpcOperationRequestActor<TDerived, TRequest>; public: @@ -128,17 +128,17 @@ protected: void Handle(NSchemeShard::TEvSchemeShard::TEvNotifyTxCompletionResult::TPtr& ev, const TActorContext& ctx) { NTabletPipe::CloseClient(ctx, SchemePipeActorId_); - return this->ReplyNotifyTxCompletionResult(ev, ctx); + return this->ReplyNotifyTxCompletionResult(ev, ctx); } void Handle(NSchemeShard::TEvSchemeShard::TEvNotifyTxCompletionRegistered::TPtr&, const TActorContext&) { } virtual void ReplyNotifyTxCompletionResult(NSchemeShard::TEvSchemeShard::TEvNotifyTxCompletionResult::TPtr& ev, const TActorContext& ctx) { - Y_UNUSED(ev); - return this->Reply(Ydb::StatusIds::SUCCESS, ctx); - } - + Y_UNUSED(ev); + return this->Reply(Ydb::StatusIds::SUCCESS, ctx); + } + private: TActorId SchemePipeActorId_; }; diff --git a/ydb/core/persqueue/event_helpers.cpp b/ydb/core/persqueue/event_helpers.cpp index 320002a936..7038d31579 100644 --- a/ydb/core/persqueue/event_helpers.cpp +++ b/ydb/core/persqueue/event_helpers.cpp @@ -20,8 +20,8 @@ void ReplyPersQueueError( NKikimrServices::EServiceKikimr service, const ui64 responseCookie, NPersQueue::NErrorCode::EErrorCode errorCode, - const TString& error, - bool logDebug + const TString& error, + bool logDebug ) { if (errorCode == NPersQueue::NErrorCode::BAD_REQUEST) { counters.Cumulative()[COUNTER_PQ_BAD_REQUEST].Increment(1); @@ -36,11 +36,11 @@ void ReplyPersQueueError( } logStr << " error: " << error; - if (logDebug) { - LOG_DEBUG_S(ctx, service, logStr); - } else { - LOG_WARN_S(ctx, service, logStr); - } + if (logDebug) { + LOG_DEBUG_S(ctx, service, logStr); + } else { + LOG_WARN_S(ctx, service, logStr); + } ctx.Send(dstActor, new TEvPQ::TEvError(errorCode, error, responseCookie)); } diff --git a/ydb/core/persqueue/event_helpers.h b/ydb/core/persqueue/event_helpers.h index 51bb814c92..b8e39fbe8a 100644 --- a/ydb/core/persqueue/event_helpers.h +++ b/ydb/core/persqueue/event_helpers.h @@ -19,8 +19,8 @@ void ReplyPersQueueError( NKikimrServices::EServiceKikimr service, const ui64 responseCookie, NPersQueue::NErrorCode::EErrorCode errorCode, - const TString& error, - bool logDebug = false + const TString& error, + bool logDebug = false ); }// NPQ diff --git a/ydb/core/persqueue/events/internal.h b/ydb/core/persqueue/events/internal.h index bc892988b4..aaee13d586 100644 --- a/ydb/core/persqueue/events/internal.h +++ b/ydb/core/persqueue/events/internal.h @@ -131,7 +131,7 @@ struct TEvPQ { ui32 UncompressedSize; TString PartitionKey; TString ExplicitHashKey; - bool External; + bool External; }; TEvWrite(const ui64 cookie, const ui64 messageNo, const TString& ownerCookie, const TMaybe<ui64> offset, TVector<TMsg> &&msgs, bool isDirectWrite) @@ -149,7 +149,7 @@ struct TEvPQ { TMaybe<ui64> Offset; TVector<TMsg> Msgs; bool IsDirectWrite; - + }; struct TEvReadTimeout : public TEventLocal<TEvReadTimeout, EvReadTimeout> { @@ -161,10 +161,10 @@ struct TEvPQ { }; struct TEvRead : public TEventLocal<TEvRead, EvRead> { - TEvRead(const ui64 cookie, const ui64 offset, const ui16 partNo, const ui32 count, - const TString& sessionId, const TString& clientId, const ui32 timeout, const ui32 size, - const ui32 maxTimeLagMs, const ui64 readTimestampMs, const TString& clientDC, - bool externalOperation) + TEvRead(const ui64 cookie, const ui64 offset, const ui16 partNo, const ui32 count, + const TString& sessionId, const TString& clientId, const ui32 timeout, const ui32 size, + const ui32 maxTimeLagMs, const ui64 readTimestampMs, const TString& clientDC, + bool externalOperation) : Cookie(cookie) , Offset(offset) , PartNo(partNo) @@ -176,7 +176,7 @@ struct TEvPQ { , MaxTimeLagMs(maxTimeLagMs) , ReadTimestampMs(readTimestampMs) , ClientDC(clientDC) - , ExternalOperation(externalOperation) + , ExternalOperation(externalOperation) {} ui64 Cookie; @@ -190,7 +190,7 @@ struct TEvPQ { ui32 MaxTimeLagMs; ui64 ReadTimestampMs; TString ClientDC; - bool ExternalOperation; + bool ExternalOperation; }; struct TEvMonRequest : public TEventLocal<TEvMonRequest, EvMonRequest> { diff --git a/ydb/core/persqueue/mirrorer.cpp b/ydb/core/persqueue/mirrorer.cpp index 37aea95ad1..d05b5fc228 100644 --- a/ydb/core/persqueue/mirrorer.cpp +++ b/ydb/core/persqueue/mirrorer.cpp @@ -61,7 +61,7 @@ void TMirrorer::Bootstrap(const TActorContext& ctx) { TString suffix = LocalDC ? "Remote" : "Internal"; MirrorerErrors = NKikimr::NPQ::TMultiCounter( GetServiceCounters(counters, "pqproxy|writeSession"), - GetLabels(TopicName), {}, {"MirrorerErrors" + suffix}, true + GetLabels(TopicName), {}, {"MirrorerErrors" + suffix}, true ); MirrorerTimeLags = THolder<TPercentileCounter>(new TPercentileCounter( GetServiceCounters(counters, "pqproxy|mirrorWriteTimeLag"), @@ -71,11 +71,11 @@ void TMirrorer::Bootstrap(const TActorContext& ctx) { )); InitTimeoutCounter = NKikimr::NPQ::TMultiCounter( GetServiceCounters(counters, "pqproxy|writeSession"), - GetLabels(TopicName), {}, {"MirrorerInitTimeout" + suffix}, true + GetLabels(TopicName), {}, {"MirrorerInitTimeout" + suffix}, true ); WriteTimeoutCounter = NKikimr::NPQ::TMultiCounter( GetServiceCounters(counters, "pqproxy|writeSession"), - {}, {}, {"MirrorerWriteTimeout"}, true, "sensor", false + {}, {}, {"MirrorerWriteTimeout"}, true, "sensor", false ); } } diff --git a/ydb/core/persqueue/partition.cpp b/ydb/core/persqueue/partition.cpp index cc8e2419cf..c9aa61be49 100644 --- a/ydb/core/persqueue/partition.cpp +++ b/ydb/core/persqueue/partition.cpp @@ -14,11 +14,11 @@ #include <ydb/core/protos/msgbus.pb.h> #include <ydb/library/persqueue/topic_parser/topic_parser.h> #include <ydb/public/lib/base/msgbus.h> -#include <library/cpp/html/pcdata/pcdata.h> -#include <library/cpp/monlib/service/pages/templates.h> +#include <library/cpp/html/pcdata/pcdata.h> +#include <library/cpp/monlib/service/pages/templates.h> #include <library/cpp/time_provider/time_provider.h> -#include <util/folder/path.h> -#include <util/string/escape.h> +#include <util/folder/path.h> +#include <util/string/escape.h> #include <util/system/byteorder.h> #define VERIFY_RESULT_BLOB(blob, pos) \ @@ -243,7 +243,7 @@ void TPartition::ReplyError(const TActorContext& ctx, const ui64 dst, NPersQueue { ReplyPersQueueError( dst == 0 ? ctx.SelfID : Tablet, ctx, TabletID, TopicName, Partition, Counters, NKikimrServices::PERSQUEUE, - dst, errorCode, error, true + dst, errorCode, error, true ); } @@ -474,10 +474,10 @@ TPartition::TPartition(ui64 tabletId, ui32 partition, const TActorId& tablet, co , BodySize(0) , MaxWriteResponsesSize(0) , GapSize(0) - , CloudId(config.GetYcCloudId()) - , DbId(config.GetYdbDatabaseId()) - , FolderId(config.GetYcFolderId()) - , UsersInfoStorage(DCId, TabletID, TopicName, Partition, counters, Config, CloudId, DbId, FolderId) + , CloudId(config.GetYcCloudId()) + , DbId(config.GetYdbDatabaseId()) + , FolderId(config.GetYcFolderId()) + , UsersInfoStorage(DCId, TabletID, TopicName, Partition, counters, Config, CloudId, DbId, FolderId) , ReadingTimestamp(false) , SetOffsetCookie(0) , Cookie(0) @@ -490,10 +490,10 @@ TPartition::TPartition(ui64 tabletId, ui32 partition, const TActorId& tablet, co , WriteCycleStartTime(ctx.Now()) , WriteCycleSize(0) , WriteNewSize(0) - , WriteNewSizeInternal(0) + , WriteNewSizeInternal(0) , WriteNewSizeUncompressed(0) , WriteNewMessages(0) - , WriteNewMessagesInternal(0) + , WriteNewMessagesInternal(0) , DiskIsFull(false) , HasDataReqNum(0) , WriteQuota(Config.GetPartitionConfig().GetBurstSize(), Config.GetPartitionConfig().GetWriteSpeedInBytesPerSecond(), ctx.Now()) @@ -749,7 +749,7 @@ void TPartition::Bootstrap(const TActorContext& ctx) } LOG_INFO_S(ctx, NKikimrServices::PERSQUEUE, "boostrapping " << Partition << " " << ctx.SelfID); - + if (NewPartition) { InitComplete(ctx); } else { @@ -757,164 +757,164 @@ void TPartition::Bootstrap(const TActorContext& ctx) RequestDiskStatus(ctx, Tablet, Config.GetPartitionConfig().GetNumChannels()); Become(&TThis::StateInit); } - + if (AppData(ctx)->Counters) { - TVector<NPQ::TLabelsInfo> labels; - if (AppData()->PQConfig.GetTopicsAreFirstClassCitizen()) { - SetupStreamCounters(ctx); - } else { - if (TopicName.find("--") == TString::npos) - return; - SetupTopicCounters(ctx); - } - } -} - -void TPartition::SetupTopicCounters(const TActorContext& ctx) { - auto counters = AppData(ctx)->Counters; - auto labels = NKikimr::NPQ::GetLabels(TopicName); - const TString suffix = LocalDC ? "Original" : "Mirrored"; - - WriteBufferIsFullCounter.SetCounter( - GetCounters(counters, "writingTime", TopicName), - {{"host", DCId}, - {"Partition", ToString<ui32>(Partition)}}, - {"sensor", "BufferFullTime" + suffix, true}); - - InputTimeLag = THolder<NKikimr::NPQ::TPercentileCounter>(new NKikimr::NPQ::TPercentileCounter( - GetServiceCounters(counters, "pqproxy|writeTimeLag"), GetLabels(TopicName), - {{"sensor", "TimeLags" + suffix}}, "Interval", - TVector<std::pair<ui64, TString>>{ - {100, "100ms"}, {200, "200ms"}, {500, "500ms"}, {1000, "1000ms"}, - {2000, "2000ms"}, {5000, "5000ms"}, {10'000, "10000ms"}, {30'000, "30000ms"}, - {60'000, "60000ms"}, {180'000,"180000ms"}, {9'999'999, "999999ms"}}, true)); - - - MessageSize = THolder<NKikimr::NPQ::TPercentileCounter>(new NKikimr::NPQ::TPercentileCounter( - GetServiceCounters(counters, "pqproxy|writeInfo"), GetLabels(TopicName), - {{"sensor", "MessageSize" + suffix}}, "Size", - TVector<std::pair<ui64, TString>>{ - {1024, "1kb"}, {5120, "5kb"}, {10240, "10kb"}, - {20'480, "20kb"}, {51'200, "50kb"}, {102'400, "100kb"}, {204'800, "200kb"}, - {524'288, "512kb"},{1'048'576, "1024kb"}, {2'097'152,"2048kb"}, {5'242'880, "5120kb"}, - {10'485'760, "10240kb"}, {67'108'864, "65536kb"}, {999'999'999, "99999999kb"}}, true)); - - BytesWritten = NKikimr::NPQ::TMultiCounter(GetServiceCounters(counters, "pqproxy|writeSession"), - GetLabels(TopicName), {}, {"BytesWritten" + suffix}, true); - BytesWrittenUncompressed = NKikimr::NPQ::TMultiCounter(GetServiceCounters(counters, "pqproxy|writeSession"), - GetLabels(TopicName), {}, {"UncompressedBytesWritten" + suffix}, true); - - BytesWrittenComp = NKikimr::NPQ::TMultiCounter(GetServiceCounters(counters, "pqproxy|writeSession"), - GetLabels(TopicName), {}, {"CompactedBytesWritten" + suffix}, true); - - MsgsWritten = NKikimr::NPQ::TMultiCounter(GetServiceCounters(counters, "pqproxy|writeSession"), - GetLabels(TopicName), {}, {"MessagesWritten" + suffix}, true); - - TVector<NPQ::TLabelsInfo> aggr = {{{{"Account", NPersQueue::GetAccount(TopicName)}}, {"total"}}}; - ui32 border = AppData(ctx)->PQConfig.GetWriteLatencyBigMs(); - auto subGroup = GetServiceCounters(counters, "pqproxy|SLI"); - WriteLatency = NKikimr::NPQ::CreateSLIDurationCounter(subGroup, aggr, "Write", border, - {100, 200, 500, 1000, 1500, 2000, - 5000, 10'000, 30'000, 99'999'999}); - SLIBigLatency = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"WriteBigLatency"}, true, "sensor", false); - WritesTotal = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"WritesTotal"}, true, "sensor", false); - if (IsQuotingEnabled() && !TopicWriteQuotaResourcePath.empty()) { - TopicWriteQuotaWaitCounter = THolder<NKikimr::NPQ::TPercentileCounter>( - new NKikimr::NPQ::TPercentileCounter( - GetServiceCounters(counters, "pqproxy|topicWriteQuotaWait"), GetLabels(TopicName), - {{"sensor", "TopicWriteQuotaWait" + suffix}}, "Interval", - TVector<std::pair<ui64, TString>>{ - {0, "0ms"}, {1, "1ms"}, {5, "5ms"}, {10, "10ms"}, - {20, "20ms"}, {50, "50ms"}, {100, "100ms"}, {500, "500ms"}, - {1000, "1000ms"}, {2500, "2500ms"}, {5000, "5000ms"}, - {10'000, "10000ms"}, {9'999'999, "999999ms"}}, true)); - } - - PartitionWriteQuotaWaitCounter = THolder<NKikimr::NPQ::TPercentileCounter>( - new NKikimr::NPQ::TPercentileCounter(GetServiceCounters(counters, "pqproxy|partitionWriteQuotaWait"), - GetLabels(TopicName), {{"sensor", "PartitionWriteQuotaWait" + suffix}}, "Interval", - TVector<std::pair<ui64, TString>>{ - {0, "0ms"}, {1, "1ms"}, {5, "5ms"}, {10, "10ms"}, - {20, "20ms"}, {50, "50ms"}, {100, "100ms"}, {500, "500ms"}, - {1000, "1000ms"}, {2500, "2500ms"}, {5000, "5000ms"}, - {10'000, "10000ms"}, {9'999'999, "999999ms"}}, true)); -} - -void TPartition::SetupStreamCounters(const TActorContext& ctx) { - auto counters = AppData(ctx)->Counters; - auto labels = NKikimr::NPQ::GetLabelsForStream(TopicName, CloudId, DbId, FolderId); - - WriteBufferIsFullCounter.SetCounter( - GetCountersForStream(counters, "writingTime"), - {{"host", DCId}, - {"partition", ToString<ui32>(Partition)}, - {"stream", TopicName}}, - {"name", "stream.internal_write.buffer_brimmed_duration_ms", true}); - - InputTimeLag = THolder<NKikimr::NPQ::TPercentileCounter>(new NKikimr::NPQ::TPercentileCounter( - NKikimr::NPQ::GetCountersForStream(counters, "writeTimeLag"), labels, - {{"name", "stream.internal_write.time_lags_milliseconds"}}, "bin", - TVector<std::pair<ui64, TString>>{ - {100, "100"}, {200, "200"}, {500, "500"}, - {1000, "1000"}, {2000, "2000"}, {5000, "5000"}, - {10'000, "10000"}, {30'000, "30000"}, {60'000, "60000"}, - {180'000,"180000"}, {9'999'999, "999999"}}, true)); - - MessageSize = THolder<NKikimr::NPQ::TPercentileCounter>(new NKikimr::NPQ::TPercentileCounter( - NKikimr::NPQ::GetCountersForStream(counters, "writeInfo"), labels, - {{"name", "stream.internal_write.record_size_bytes"}}, "bin", - TVector<std::pair<ui64, TString>>{ - {1024, "1024"}, {5120, "5120"}, {10'240, "10240"}, - {20'480, "20480"}, {51'200, "51200"}, {102'400, "102400"}, - {204'800, "204800"}, {524'288, "524288"},{1'048'576, "1048576"}, - {2'097'152,"2097152"}, {5'242'880, "5242880"}, {10'485'760, "10485760"}, - {67'108'864, "67108864"}, {999'999'999, "99999999"}}, true)); - - BytesWritten = NKikimr::NPQ::TMultiCounter( - NKikimr::NPQ::GetCountersForStream(counters, "writeSession"), labels, {}, - {"stream.internal_write.bytes_per_second", - "stream.incoming_bytes_per_second"} , true, "name"); - MsgsWritten = NKikimr::NPQ::TMultiCounter( - NKikimr::NPQ::GetCountersForStream(counters, "writeSession"), labels, {}, - {"stream.internal_write.records_per_second", - "stream.incoming_records_per_second"}, true, "name"); - - BytesWrittenUncompressed = NKikimr::NPQ::TMultiCounter( - NKikimr::NPQ::GetCountersForStream(counters, "writeSession"), labels, {}, - {"stream.internal_write.uncompressed_bytes_per_second"}, true, "name"); - BytesWrittenComp = NKikimr::NPQ::TMultiCounter( - NKikimr::NPQ::GetCountersForStream(counters, "writeSession"), labels, {}, - {"stream.internal_write.compacted_bytes_per_second"}, true, "name"); - - TVector<NPQ::TLabelsInfo> aggr = {{{{"Account", NPersQueue::GetAccount(TopicName)}}, {"total"}}}; - ui32 border = AppData(ctx)->PQConfig.GetWriteLatencyBigMs(); - auto subGroup = GetServiceCounters(counters, "pqproxy|SLI"); - WriteLatency = NKikimr::NPQ::CreateSLIDurationCounter(subGroup, aggr, "Write", border, - {100, 200, 500, 1000, 1500, 2000, - 5000, 10'000, 30'000, 99'999'999}); - SLIBigLatency = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"WriteBigLatency"}, true, "name", false); - WritesTotal = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"WritesTotal"}, true, "name", false); - if (IsQuotingEnabled() && !TopicWriteQuotaResourcePath.empty()) { - TopicWriteQuotaWaitCounter = THolder<NKikimr::NPQ::TPercentileCounter>( - new NKikimr::NPQ::TPercentileCounter( - GetCountersForStream(counters, "topicWriteQuotaWait"), labels, - {{"name", "stream.internal_write.topic_write_quota_wait_milliseconds"}}, "bin", - TVector<std::pair<ui64, TString>>{ - {0, "0"}, {1, "1"}, {5, "5"}, {10, "10"}, - {20, "20"}, {50, "50"}, {100, "100"}, {500, "500"}, - {1000, "1000"}, {2500, "2500"}, {5000, "5000"}, - {10'000, "10000"}, {9'999'999, "999999"}}, true)); - } - - PartitionWriteQuotaWaitCounter = THolder<NKikimr::NPQ::TPercentileCounter>( - new NKikimr::NPQ::TPercentileCounter( - GetCountersForStream(counters, "partitionWriteQuotaWait"), labels, - {{"name", "stream.internal_write.partition_write_quota_wait_milliseconds"}}, "bin", - TVector<std::pair<ui64, TString>>{ - {0, "0"}, {1, "1"}, {5, "5"}, {10, "10"}, - {20, "20"}, {50, "50"}, {100, "100"}, {500, "500"}, - {1000, "1000"}, {2500, "2500"}, {5000, "5000"}, - {10'000, "10000"}, {9'999'999, "999999"}}, true)); + TVector<NPQ::TLabelsInfo> labels; + if (AppData()->PQConfig.GetTopicsAreFirstClassCitizen()) { + SetupStreamCounters(ctx); + } else { + if (TopicName.find("--") == TString::npos) + return; + SetupTopicCounters(ctx); + } + } +} + +void TPartition::SetupTopicCounters(const TActorContext& ctx) { + auto counters = AppData(ctx)->Counters; + auto labels = NKikimr::NPQ::GetLabels(TopicName); + const TString suffix = LocalDC ? "Original" : "Mirrored"; + + WriteBufferIsFullCounter.SetCounter( + GetCounters(counters, "writingTime", TopicName), + {{"host", DCId}, + {"Partition", ToString<ui32>(Partition)}}, + {"sensor", "BufferFullTime" + suffix, true}); + + InputTimeLag = THolder<NKikimr::NPQ::TPercentileCounter>(new NKikimr::NPQ::TPercentileCounter( + GetServiceCounters(counters, "pqproxy|writeTimeLag"), GetLabels(TopicName), + {{"sensor", "TimeLags" + suffix}}, "Interval", + TVector<std::pair<ui64, TString>>{ + {100, "100ms"}, {200, "200ms"}, {500, "500ms"}, {1000, "1000ms"}, + {2000, "2000ms"}, {5000, "5000ms"}, {10'000, "10000ms"}, {30'000, "30000ms"}, + {60'000, "60000ms"}, {180'000,"180000ms"}, {9'999'999, "999999ms"}}, true)); + + + MessageSize = THolder<NKikimr::NPQ::TPercentileCounter>(new NKikimr::NPQ::TPercentileCounter( + GetServiceCounters(counters, "pqproxy|writeInfo"), GetLabels(TopicName), + {{"sensor", "MessageSize" + suffix}}, "Size", + TVector<std::pair<ui64, TString>>{ + {1024, "1kb"}, {5120, "5kb"}, {10240, "10kb"}, + {20'480, "20kb"}, {51'200, "50kb"}, {102'400, "100kb"}, {204'800, "200kb"}, + {524'288, "512kb"},{1'048'576, "1024kb"}, {2'097'152,"2048kb"}, {5'242'880, "5120kb"}, + {10'485'760, "10240kb"}, {67'108'864, "65536kb"}, {999'999'999, "99999999kb"}}, true)); + + BytesWritten = NKikimr::NPQ::TMultiCounter(GetServiceCounters(counters, "pqproxy|writeSession"), + GetLabels(TopicName), {}, {"BytesWritten" + suffix}, true); + BytesWrittenUncompressed = NKikimr::NPQ::TMultiCounter(GetServiceCounters(counters, "pqproxy|writeSession"), + GetLabels(TopicName), {}, {"UncompressedBytesWritten" + suffix}, true); + + BytesWrittenComp = NKikimr::NPQ::TMultiCounter(GetServiceCounters(counters, "pqproxy|writeSession"), + GetLabels(TopicName), {}, {"CompactedBytesWritten" + suffix}, true); + + MsgsWritten = NKikimr::NPQ::TMultiCounter(GetServiceCounters(counters, "pqproxy|writeSession"), + GetLabels(TopicName), {}, {"MessagesWritten" + suffix}, true); + + TVector<NPQ::TLabelsInfo> aggr = {{{{"Account", NPersQueue::GetAccount(TopicName)}}, {"total"}}}; + ui32 border = AppData(ctx)->PQConfig.GetWriteLatencyBigMs(); + auto subGroup = GetServiceCounters(counters, "pqproxy|SLI"); + WriteLatency = NKikimr::NPQ::CreateSLIDurationCounter(subGroup, aggr, "Write", border, + {100, 200, 500, 1000, 1500, 2000, + 5000, 10'000, 30'000, 99'999'999}); + SLIBigLatency = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"WriteBigLatency"}, true, "sensor", false); + WritesTotal = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"WritesTotal"}, true, "sensor", false); + if (IsQuotingEnabled() && !TopicWriteQuotaResourcePath.empty()) { + TopicWriteQuotaWaitCounter = THolder<NKikimr::NPQ::TPercentileCounter>( + new NKikimr::NPQ::TPercentileCounter( + GetServiceCounters(counters, "pqproxy|topicWriteQuotaWait"), GetLabels(TopicName), + {{"sensor", "TopicWriteQuotaWait" + suffix}}, "Interval", + TVector<std::pair<ui64, TString>>{ + {0, "0ms"}, {1, "1ms"}, {5, "5ms"}, {10, "10ms"}, + {20, "20ms"}, {50, "50ms"}, {100, "100ms"}, {500, "500ms"}, + {1000, "1000ms"}, {2500, "2500ms"}, {5000, "5000ms"}, + {10'000, "10000ms"}, {9'999'999, "999999ms"}}, true)); + } + + PartitionWriteQuotaWaitCounter = THolder<NKikimr::NPQ::TPercentileCounter>( + new NKikimr::NPQ::TPercentileCounter(GetServiceCounters(counters, "pqproxy|partitionWriteQuotaWait"), + GetLabels(TopicName), {{"sensor", "PartitionWriteQuotaWait" + suffix}}, "Interval", + TVector<std::pair<ui64, TString>>{ + {0, "0ms"}, {1, "1ms"}, {5, "5ms"}, {10, "10ms"}, + {20, "20ms"}, {50, "50ms"}, {100, "100ms"}, {500, "500ms"}, + {1000, "1000ms"}, {2500, "2500ms"}, {5000, "5000ms"}, + {10'000, "10000ms"}, {9'999'999, "999999ms"}}, true)); +} + +void TPartition::SetupStreamCounters(const TActorContext& ctx) { + auto counters = AppData(ctx)->Counters; + auto labels = NKikimr::NPQ::GetLabelsForStream(TopicName, CloudId, DbId, FolderId); + + WriteBufferIsFullCounter.SetCounter( + GetCountersForStream(counters, "writingTime"), + {{"host", DCId}, + {"partition", ToString<ui32>(Partition)}, + {"stream", TopicName}}, + {"name", "stream.internal_write.buffer_brimmed_duration_ms", true}); + + InputTimeLag = THolder<NKikimr::NPQ::TPercentileCounter>(new NKikimr::NPQ::TPercentileCounter( + NKikimr::NPQ::GetCountersForStream(counters, "writeTimeLag"), labels, + {{"name", "stream.internal_write.time_lags_milliseconds"}}, "bin", + TVector<std::pair<ui64, TString>>{ + {100, "100"}, {200, "200"}, {500, "500"}, + {1000, "1000"}, {2000, "2000"}, {5000, "5000"}, + {10'000, "10000"}, {30'000, "30000"}, {60'000, "60000"}, + {180'000,"180000"}, {9'999'999, "999999"}}, true)); + + MessageSize = THolder<NKikimr::NPQ::TPercentileCounter>(new NKikimr::NPQ::TPercentileCounter( + NKikimr::NPQ::GetCountersForStream(counters, "writeInfo"), labels, + {{"name", "stream.internal_write.record_size_bytes"}}, "bin", + TVector<std::pair<ui64, TString>>{ + {1024, "1024"}, {5120, "5120"}, {10'240, "10240"}, + {20'480, "20480"}, {51'200, "51200"}, {102'400, "102400"}, + {204'800, "204800"}, {524'288, "524288"},{1'048'576, "1048576"}, + {2'097'152,"2097152"}, {5'242'880, "5242880"}, {10'485'760, "10485760"}, + {67'108'864, "67108864"}, {999'999'999, "99999999"}}, true)); + + BytesWritten = NKikimr::NPQ::TMultiCounter( + NKikimr::NPQ::GetCountersForStream(counters, "writeSession"), labels, {}, + {"stream.internal_write.bytes_per_second", + "stream.incoming_bytes_per_second"} , true, "name"); + MsgsWritten = NKikimr::NPQ::TMultiCounter( + NKikimr::NPQ::GetCountersForStream(counters, "writeSession"), labels, {}, + {"stream.internal_write.records_per_second", + "stream.incoming_records_per_second"}, true, "name"); + + BytesWrittenUncompressed = NKikimr::NPQ::TMultiCounter( + NKikimr::NPQ::GetCountersForStream(counters, "writeSession"), labels, {}, + {"stream.internal_write.uncompressed_bytes_per_second"}, true, "name"); + BytesWrittenComp = NKikimr::NPQ::TMultiCounter( + NKikimr::NPQ::GetCountersForStream(counters, "writeSession"), labels, {}, + {"stream.internal_write.compacted_bytes_per_second"}, true, "name"); + + TVector<NPQ::TLabelsInfo> aggr = {{{{"Account", NPersQueue::GetAccount(TopicName)}}, {"total"}}}; + ui32 border = AppData(ctx)->PQConfig.GetWriteLatencyBigMs(); + auto subGroup = GetServiceCounters(counters, "pqproxy|SLI"); + WriteLatency = NKikimr::NPQ::CreateSLIDurationCounter(subGroup, aggr, "Write", border, + {100, 200, 500, 1000, 1500, 2000, + 5000, 10'000, 30'000, 99'999'999}); + SLIBigLatency = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"WriteBigLatency"}, true, "name", false); + WritesTotal = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"WritesTotal"}, true, "name", false); + if (IsQuotingEnabled() && !TopicWriteQuotaResourcePath.empty()) { + TopicWriteQuotaWaitCounter = THolder<NKikimr::NPQ::TPercentileCounter>( + new NKikimr::NPQ::TPercentileCounter( + GetCountersForStream(counters, "topicWriteQuotaWait"), labels, + {{"name", "stream.internal_write.topic_write_quota_wait_milliseconds"}}, "bin", + TVector<std::pair<ui64, TString>>{ + {0, "0"}, {1, "1"}, {5, "5"}, {10, "10"}, + {20, "20"}, {50, "50"}, {100, "100"}, {500, "500"}, + {1000, "1000"}, {2500, "2500"}, {5000, "5000"}, + {10'000, "10000"}, {9'999'999, "999999"}}, true)); + } + + PartitionWriteQuotaWaitCounter = THolder<NKikimr::NPQ::TPercentileCounter>( + new NKikimr::NPQ::TPercentileCounter( + GetCountersForStream(counters, "partitionWriteQuotaWait"), labels, + {{"name", "stream.internal_write.partition_write_quota_wait_milliseconds"}}, "bin", + TVector<std::pair<ui64, TString>>{ + {0, "0"}, {1, "1"}, {5, "5"}, {10, "10"}, + {20, "20"}, {50, "50"}, {100, "100"}, {500, "500"}, + {1000, "1000"}, {2500, "2500"}, {5000, "5000"}, + {10'000, "10000"}, {9'999'999, "999999"}}, true)); } void TPartition::ProcessHasDataRequests(const TActorContext& ctx) { @@ -2330,11 +2330,11 @@ TReadAnswer TReadInfo::FormAnswer( ui16 internalPartsCount = blobs[pos].InternalPartsCount; const TString& blobValue = blobs[pos].Value; - if (blobValue.empty()) { // this is ok. Means that someone requested too much data + if (blobValue.empty()) { // this is ok. Means that someone requested too much data LOG_DEBUG(ctx, NKikimrServices::PERSQUEUE, "Not full answer here!"); ui64 answerSize = answer->Response.ByteSize(); if (userInfo && Destination != 0) { - userInfo->ReadDone(ctx, ctx.Now(), answerSize, cnt, ClientDC); + userInfo->ReadDone(ctx, ctx.Now(), answerSize, cnt, ClientDC); } readResult->SetSizeLag(sizeLag - size); return {answerSize, std::move(answer)}; @@ -2342,9 +2342,9 @@ TReadAnswer TReadInfo::FormAnswer( Y_VERIFY(blobValue.size() == blobs[pos].Size, "value for offset %" PRIu64 " count %u size must be %u, but got %u", offset, count, blobs[pos].Size, (ui32)blobValue.size()); - if (offset > Offset || (offset == Offset && partNo > PartNo)) { // got gap + if (offset > Offset || (offset == Offset && partNo > PartNo)) { // got gap Offset = offset; - PartNo = partNo; + PartNo = partNo; } Y_VERIFY(offset <= Offset); Y_VERIFY(offset < Offset || partNo <= PartNo); @@ -2362,7 +2362,7 @@ TReadAnswer TReadInfo::FormAnswer( } offset += header.GetCount(); - if (pos == Max<ui32>()) // this batch does not contain data to read, skip it + if (pos == Max<ui32>()) // this batch does not contain data to read, skip it continue; @@ -2375,31 +2375,31 @@ TReadAnswer TReadInfo::FormAnswer( psize = size; TClientBlob &res = batch.Blobs[i]; VERIFY_RESULT_BLOB(res, i); - bool messageSkippingBehaviour = AppData()->PQConfig.GetTopicsAreFirstClassCitizen() && - ReadTimestampMs > res.WriteTimestamp.MilliSeconds(); - if (!messageSkippingBehaviour) { - size += res.GetBlobSize(); - Y_VERIFY(PartNo == res.GetPartNo(), "pos %" PRIu32 " i %" PRIu32 " Offset %" PRIu64 " PartNo %" PRIu16 " offset %" PRIu64 " partNo %" PRIu16, - pos, i, Offset, PartNo, offset, res.GetPartNo()); - - if (userInfo) { - userInfo->AddTimestampToCache( - Offset, res.WriteTimestamp, res.CreateTimestamp, - Destination != 0, ctx.Now() - ); - } - - AddResultBlob(readResult, res, Offset); - if (res.IsLastPart()) { - ++cnt; - } + bool messageSkippingBehaviour = AppData()->PQConfig.GetTopicsAreFirstClassCitizen() && + ReadTimestampMs > res.WriteTimestamp.MilliSeconds(); + if (!messageSkippingBehaviour) { + size += res.GetBlobSize(); + Y_VERIFY(PartNo == res.GetPartNo(), "pos %" PRIu32 " i %" PRIu32 " Offset %" PRIu64 " PartNo %" PRIu16 " offset %" PRIu64 " partNo %" PRIu16, + pos, i, Offset, PartNo, offset, res.GetPartNo()); + + if (userInfo) { + userInfo->AddTimestampToCache( + Offset, res.WriteTimestamp, res.CreateTimestamp, + Destination != 0, ctx.Now() + ); + } + + AddResultBlob(readResult, res, Offset); + if (res.IsLastPart()) { + ++cnt; + } } - if (res.IsLastPart()) { + if (res.IsLastPart()) { PartNo = 0; ++Offset; - } else { - ++PartNo; + } else { + ++PartNo; } } @@ -2444,7 +2444,7 @@ TReadAnswer TReadInfo::FormAnswer( Y_VERIFY(Offset <= (ui64)Max<i64>(), "Offset is too big: %" PRIu64, Offset); ui64 answerSize = answer->Response.ByteSize(); if (userInfo && Destination != 0) { - userInfo->ReadDone(ctx, ctx.Now(), answerSize, cnt, ClientDC); + userInfo->ReadDone(ctx, ctx.Now(), answerSize, cnt, ClientDC); } readResult->SetSizeLag(sizeLag - size); return {answerSize, std::move(answer)}; @@ -2472,7 +2472,7 @@ void TPartition::Handle(TEvPQ::TEvReadTimeout::TPtr& ev, const TActorContext& ct } -TVector<TRequestedBlob> TPartition::GetReadRequestFromBody(const ui64 startOffset, const ui16 partNo, const ui32 maxCount, const ui32 maxSize, ui32* rcount, ui32* rsize) +TVector<TRequestedBlob> TPartition::GetReadRequestFromBody(const ui64 startOffset, const ui16 partNo, const ui32 maxCount, const ui32 maxSize, ui32* rcount, ui32* rsize) { Y_VERIFY(rcount && rsize); ui32& count = *rcount; @@ -2502,12 +2502,12 @@ TVector<TRequestedBlob> TPartition::GetReadRequestFromBody(const ui64 startOffse sz = (cnt == it->Key.GetCount() ? it->Size : 0); //not readed client blobs can be of ~8Mb, so don't count this size at all } while (it != DataKeysBody.end() && size < maxSize && count < maxCount) { - size += sz; - count += cnt; - TRequestedBlob reqBlob(it->Key.GetOffset(), it->Key.GetPartNo(), it->Key.GetCount(), - it->Key.GetInternalPartsCount(), it->Size, TString()); - blobs.push_back(reqBlob); - + size += sz; + count += cnt; + TRequestedBlob reqBlob(it->Key.GetOffset(), it->Key.GetPartNo(), it->Key.GetCount(), + it->Key.GetInternalPartsCount(), it->Size, TString()); + blobs.push_back(reqBlob); + ++it; if (it == DataKeysBody.end()) break; @@ -2520,12 +2520,12 @@ TVector<TRequestedBlob> TPartition::GetReadRequestFromBody(const ui64 startOffse -TVector<TClientBlob> TPartition::GetReadRequestFromHead(const ui64 startOffset, const ui16 partNo, const ui32 maxCount, const ui32 maxSize, const ui64 readTimestampMs, ui32* rcount, ui32* rsize, ui64* insideHeadOffset) +TVector<TClientBlob> TPartition::GetReadRequestFromHead(const ui64 startOffset, const ui16 partNo, const ui32 maxCount, const ui32 maxSize, const ui64 readTimestampMs, ui32* rcount, ui32* rsize, ui64* insideHeadOffset) { ui32& count = *rcount; ui32& size = *rsize; TVector<TClientBlob> res; - std::optional<ui64> firstAddedBlobOffset{}; + std::optional<ui64> firstAddedBlobOffset{}; ui32 pos = 0; if (startOffset > Head.Offset || startOffset == Head.Offset && partNo > Head.PartNo) { pos = Head.FindPos(startOffset, partNo); @@ -2542,10 +2542,10 @@ TVector<TClientBlob> TPartition::GetReadRequestFromHead(const ui64 startOffset, for (; i < blobs.size(); ++i) { Y_VERIFY(pno == blobs[i].GetPartNo()); - bool messageSkippingBehaviour = AppData()->PQConfig.GetTopicsAreFirstClassCitizen() && - readTimestampMs > blobs[i].WriteTimestamp.MilliSeconds(); - bool skip = offset < startOffset || offset == startOffset && - blobs[i].GetPartNo() < partNo || messageSkippingBehaviour; + bool messageSkippingBehaviour = AppData()->PQConfig.GetTopicsAreFirstClassCitizen() && + readTimestampMs > blobs[i].WriteTimestamp.MilliSeconds(); + bool skip = offset < startOffset || offset == startOffset && + blobs[i].GetPartNo() < partNo || messageSkippingBehaviour; if (blobs[i].IsLastPart()) { ++offset; pno = 0; @@ -2556,19 +2556,19 @@ TVector<TClientBlob> TPartition::GetReadRequestFromHead(const ui64 startOffset, } if (skip) continue; - if (count > maxCount) // blob is counted already + if (count > maxCount) // blob is counted already break; if (size >= maxSize) break; - size += blobs[i].GetBlobSize(); - res.push_back(blobs[i]); - if (!firstAddedBlobOffset && AppData()->PQConfig.GetTopicsAreFirstClassCitizen()) - firstAddedBlobOffset = offset > 0 ? offset - 1 : 0; + size += blobs[i].GetBlobSize(); + res.push_back(blobs[i]); + if (!firstAddedBlobOffset && AppData()->PQConfig.GetTopicsAreFirstClassCitizen()) + firstAddedBlobOffset = offset > 0 ? offset - 1 : 0; } if (i < blobs.size()) // already got limit break; } - *insideHeadOffset = firstAddedBlobOffset.value_or(*insideHeadOffset); + *insideHeadOffset = firstAddedBlobOffset.value_or(*insideHeadOffset); return res; } @@ -2847,9 +2847,9 @@ void TPartition::ReadTimestampForOffset(const TString& user, TUserInfo& userInfo << " user " << user << " send read request for offset " << userInfo.Offset << " initiated " << " queuesize " << UpdateUserInfoTimestamp.size() << " startOffset " << StartOffset << " ReadingTimestamp " << ReadingTimestamp); - THolder<TEvPQ::TEvRead> event = MakeHolder<TEvPQ::TEvRead>(0, userInfo.Offset, 0, 1, "", - user, 0, MAX_BLOB_PART_SIZE * 2, 0, 0, "", - userInfo.DoExternalRead); + THolder<TEvPQ::TEvRead> event = MakeHolder<TEvPQ::TEvRead>(0, userInfo.Offset, 0, 1, "", + user, 0, MAX_BLOB_PART_SIZE * 2, 0, 0, "", + userInfo.DoExternalRead); ctx.Send(ctx.SelfID, event.Release()); Counters.Cumulative()[COUNTER_PQ_WRITE_TIMESTAMP_CACHE_MISS].Increment(1); } @@ -3314,8 +3314,8 @@ void TPartition::Handle(TEvKeyValue::TEvResponse::TPtr& ev, const TActorContext& //check correctness of response if (response.GetStatus() != NMsgBusProxy::MSTATUS_OK) { - LOG_ERROR_S(ctx, NKikimrServices::PERSQUEUE, "OnWrite topic '" << TopicName << "' partition " << - Partition << " commands are not processed at all, reason: " << response.DebugString()); + LOG_ERROR_S(ctx, NKikimrServices::PERSQUEUE, "OnWrite topic '" << TopicName << "' partition " << + Partition << " commands are not processed at all, reason: " << response.DebugString()); ctx.Send(Tablet, new TEvents::TEvPoisonPill()); //TODO: if status is DISK IS FULL, is global status MSTATUS_OK? it will be good if it is true return; @@ -3500,13 +3500,13 @@ void TPartition::HandleWriteResponse(const TActorContext& ctx) { Counters.Percentile()[COUNTER_PQ_WRITE_CYCLE_BYTES].IncrementFor(WriteCycleSize); Counters.Percentile()[COUNTER_PQ_WRITE_NEW_BYTES].IncrementFor(WriteNewSize); if (BytesWritten) - BytesWritten.Inc(WriteNewSizeInternal); + BytesWritten.Inc(WriteNewSizeInternal); if (BytesWrittenUncompressed) BytesWrittenUncompressed.Inc(WriteNewSizeUncompressed); if (BytesWrittenComp) BytesWrittenComp.Inc(WriteCycleSize); if (MsgsWritten) - MsgsWritten.Inc(WriteNewMessagesInternal); + MsgsWritten.Inc(WriteNewMessagesInternal); //All ok auto now = ctx.Now(); @@ -3525,10 +3525,10 @@ void TPartition::HandleWriteResponse(const TActorContext& ctx) { WriteCycleSize = 0; WriteNewSize = 0; - WriteNewSizeInternal = 0; + WriteNewSizeInternal = 0; WriteNewSizeUncompressed = 0; WriteNewMessages = 0; - WriteNewMessagesInternal = 0; + WriteNewMessagesInternal = 0; UpdateWriteBufferIsFullState(now); AnswerCurrentWrites(ctx); @@ -3958,8 +3958,8 @@ void TPartition::CancelAllWritesOnWrite(const TActorContext& ctx, TEvKeyValue::T } -bool TPartition::AppendHeadWithNewWrites(TEvKeyValue::TEvRequest* request, const TActorContext& ctx, - TSourceIdWriter& sourceIdWriter) { +bool TPartition::AppendHeadWithNewWrites(TEvKeyValue::TEvRequest* request, const TActorContext& ctx, + TSourceIdWriter& sourceIdWriter) { ui64 curOffset = PartitionedBlob.IsInited() ? PartitionedBlob.GetOffset() : EndOffset; @@ -4112,14 +4112,14 @@ bool TPartition::AppendHeadWithNewWrites(TEvKeyValue::TEvRequest* request, const } } } - PartitionedBlob = TPartitionedBlob(Partition, curOffset, p.Msg.SourceId, p.Msg.SeqNo, - p.Msg.TotalParts, p.Msg.TotalSize, Head, NewHead, - headCleared, needCompactHead, MaxBlobSize); + PartitionedBlob = TPartitionedBlob(Partition, curOffset, p.Msg.SourceId, p.Msg.SeqNo, + p.Msg.TotalParts, p.Msg.TotalSize, Head, NewHead, + headCleared, needCompactHead, MaxBlobSize); } LOG_DEBUG_S(ctx, NKikimrServices::PERSQUEUE, "Topic '" << TopicName << "' partition " << Partition - << " part blob processing sourceId '" << EscapeC(p.Msg.SourceId) << - "' seqNo " << p.Msg.SeqNo << " partNo " << p.Msg.PartNo); + << " part blob processing sourceId '" << EscapeC(p.Msg.SourceId) << + "' seqNo " << p.Msg.SeqNo << " partNo " << p.Msg.PartNo); TString s; if (!PartitionedBlob.IsNextPart(p.Msg.SourceId, p.Msg.SeqNo, p.Msg.PartNo, &s)) { //this must not be happen - client sends gaps, fail this client till the end @@ -4129,13 +4129,13 @@ bool TPartition::AppendHeadWithNewWrites(TEvKeyValue::TEvRequest* request, const } WriteNewSize += p.Msg.SourceId.size() + p.Msg.Data.size(); - WriteNewSizeInternal = p.Msg.External ? 0 : WriteNewSize; + WriteNewSizeInternal = p.Msg.External ? 0 : WriteNewSize; WriteNewSizeUncompressed += p.Msg.UncompressedSize + p.Msg.SourceId.size(); - if (p.Msg.PartNo == 0) { - ++WriteNewMessages; - if (!p.Msg.External) - ++WriteNewMessagesInternal; - } + if (p.Msg.PartNo == 0) { + ++WriteNewMessages; + if (!p.Msg.External) + ++WriteNewMessagesInternal; + } TMaybe<TPartData> partData; if (p.Msg.TotalParts > 1) { //this is multi-part message @@ -4175,12 +4175,12 @@ bool TPartition::AppendHeadWithNewWrites(TEvKeyValue::TEvRequest* request, const write->SetKeyToCache(resKey.Data(), resKey.Size()); WriteCycleSize += newWrite.second.size(); - LOG_DEBUG_S(ctx, NKikimrServices::PERSQUEUE, "Topic '" << TopicName << - "' partition " << Partition << - " part blob sourceId '" << EscapeC(p.Msg.SourceId) << - "' seqNo " << p.Msg.SeqNo << " partNo " << p.Msg.PartNo << - " result is " << TStringBuf(newWrite.first.Data(), newWrite.first.Size()) << - " size " << newWrite.second.size()); + LOG_DEBUG_S(ctx, NKikimrServices::PERSQUEUE, "Topic '" << TopicName << + "' partition " << Partition << + " part blob sourceId '" << EscapeC(p.Msg.SourceId) << + "' seqNo " << p.Msg.SeqNo << " partNo " << p.Msg.PartNo << + " result is " << TStringBuf(newWrite.first.Data(), newWrite.first.Size()) << + " size " << newWrite.second.size()); } if (lastBlobPart) { @@ -4578,15 +4578,15 @@ void TPartition::ProcessRead(const TActorContext& ctx, TReadInfo&& info, const u userInfo.UpdateReadingTimeAndState(ctx.Now()); return; } - TVector<TRequestedBlob> blobs = GetReadRequestFromBody(info.Offset, info.PartNo, info.Count, info.Size, &count, &size); + TVector<TRequestedBlob> blobs = GetReadRequestFromBody(info.Offset, info.PartNo, info.Count, info.Size, &count, &size); info.Blobs = blobs; ui64 lastOffset = info.Offset + Min(count, info.Count); LOG_DEBUG_S(ctx, NKikimrServices::PERSQUEUE, "read cookie " << cookie << " added " << info.Blobs.size() << " blobs, size " << size << " count " << count << " last offset " << lastOffset); - ui64 insideHeadOffset{0}; - info.Cached = GetReadRequestFromHead(info.Offset, info.PartNo, info.Count, info.Size, info.ReadTimestampMs, &count, &size, &insideHeadOffset); - info.CachedOffset = Head.Offset > 0 ? Head.Offset : insideHeadOffset; + ui64 insideHeadOffset{0}; + info.Cached = GetReadRequestFromHead(info.Offset, info.PartNo, info.Count, info.Size, info.ReadTimestampMs, &count, &size, &insideHeadOffset); + info.CachedOffset = Head.Offset > 0 ? Head.Offset : insideHeadOffset; if (info.Destination != 0) { ++userInfo.ActiveReads; @@ -4618,7 +4618,7 @@ void TPartition::ProcessRead(const TActorContext& ctx, TReadInfo&& info, const u Y_VERIFY(res); THolder<TEvPQ::TEvBlobRequest> request(new TEvPQ::TEvBlobRequest(user, cookie, Partition, - lastOffset, std::move(blobs))); + lastOffset, std::move(blobs))); ctx.Send(BlobCache, request.Release()); } diff --git a/ydb/core/persqueue/partition.h b/ydb/core/persqueue/partition.h index 2eec26f251..2f9bc11db1 100644 --- a/ydb/core/persqueue/partition.h +++ b/ydb/core/persqueue/partition.h @@ -141,8 +141,8 @@ private: void OnReadRequestFinished(TReadInfo&& info, ui64 answerSize); // will return rcount and rsize also - TVector<TRequestedBlob> GetReadRequestFromBody(const ui64 startOffset, const ui16 partNo, const ui32 maxCount, const ui32 maxSize, ui32* rcount, ui32* rsize); - TVector<TClientBlob> GetReadRequestFromHead(const ui64 startOffset, const ui16 partNo, const ui32 maxCount, const ui32 maxSize, const ui64 readTimestampMs, ui32* rcount, ui32* rsize, ui64* insideHeadOffset); + TVector<TRequestedBlob> GetReadRequestFromBody(const ui64 startOffset, const ui16 partNo, const ui32 maxCount, const ui32 maxSize, ui32* rcount, ui32* rsize); + TVector<TClientBlob> GetReadRequestFromHead(const ui64 startOffset, const ui16 partNo, const ui32 maxCount, const ui32 maxSize, const ui64 readTimestampMs, ui32* rcount, ui32* rsize, ui64* insideHeadOffset); void ProcessRead(const TActorContext& ctx, TReadInfo&& info, const ui64 cookie, bool subscription); void HandleOnIdle(TEvPQ::TEvWrite::TPtr& ev, const TActorContext& ctx); @@ -210,9 +210,9 @@ private: void CreateMirrorerActor(); bool IsQuotingEnabled() const; - void SetupTopicCounters(const TActorContext& ctx); - void SetupStreamCounters(const TActorContext& ctx); - + void SetupTopicCounters(const TActorContext& ctx); + void SetupStreamCounters(const TActorContext& ctx); + public: static constexpr NKikimrServices::TActivity::EType ActorActivityType() { return NKikimrServices::TActivity::PERSQUEUE_PARTITION_ACTOR; @@ -564,10 +564,10 @@ private: std::deque<std::pair<ui64,ui64>> GapOffsets; ui64 GapSize; - TString CloudId; - TString DbId; - TString FolderId; - + TString CloudId; + TString DbId; + TString FolderId; + TUsersInfoStorage UsersInfoStorage; std::deque<TString> UpdateUserInfoTimestamp; @@ -601,10 +601,10 @@ private: TInstant WriteCycleStartTime; ui32 WriteCycleSize; ui32 WriteNewSize; - ui32 WriteNewSizeInternal; + ui32 WriteNewSizeInternal; ui64 WriteNewSizeUncompressed; ui32 WriteNewMessages; - ui32 WriteNewMessagesInternal; + ui32 WriteNewMessagesInternal; TInstant CurrentTimestamp; diff --git a/ydb/core/persqueue/percentile_counter.cpp b/ydb/core/persqueue/percentile_counter.cpp index a6414e70bd..780c417e18 100644 --- a/ydb/core/persqueue/percentile_counter.cpp +++ b/ydb/core/persqueue/percentile_counter.cpp @@ -7,11 +7,11 @@ namespace NKikimr { namespace NPQ { -NMonitoring::TDynamicCounterPtr GetCounters(NMonitoring::TDynamicCounterPtr counters, - const TString& subsystem, const TString& topic) +NMonitoring::TDynamicCounterPtr GetCounters(NMonitoring::TDynamicCounterPtr counters, + const TString& subsystem, const TString& topic) { auto pos = topic.find("--"); - Y_VERIFY(pos != TString::npos); + Y_VERIFY(pos != TString::npos); TString origDC = topic.substr(4, pos - 4); origDC.to_title(); @@ -21,20 +21,20 @@ NMonitoring::TDynamicCounterPtr GetCounters(NMonitoring::TDynamicCounterPtr coun TString topicPath = NPersQueue::ConvertOldTopicName(realTopic); TString account = topicPath.substr(0, topicPath.find("/")); return GetServiceCounters(counters, "pqproxy|" + subsystem) - ->GetSubgroup("OriginDC", origDC) - ->GetSubgroup("Producer", producer) - ->GetSubgroup("TopicPath", topicPath) - ->GetSubgroup("Account", account) - ->GetSubgroup("Topic", realTopic); -} - -NMonitoring::TDynamicCounterPtr GetCountersForStream(NMonitoring::TDynamicCounterPtr counters, - const TString& subsystem) -{ - return counters->GetSubgroup("counters", "pqproxy") - ->GetSubgroup("subsystem", subsystem); + ->GetSubgroup("OriginDC", origDC) + ->GetSubgroup("Producer", producer) + ->GetSubgroup("TopicPath", topicPath) + ->GetSubgroup("Account", account) + ->GetSubgroup("Topic", realTopic); } +NMonitoring::TDynamicCounterPtr GetCountersForStream(NMonitoring::TDynamicCounterPtr counters, + const TString& subsystem) +{ + return counters->GetSubgroup("counters", "pqproxy") + ->GetSubgroup("subsystem", subsystem); +} + TVector<TLabelsInfo> GetLabels(const TString& topic) { auto pos = topic.find("--"); @@ -66,71 +66,71 @@ TVector<TLabelsInfo> GetLabels(const TString& cluster, const TString& realTopic) return res; } -TVector<TLabelsInfo> GetLabelsForStream(const TString& topic, const TString& cloudId, - const TString& dbId, const TString& folderId) { - TVector<TLabelsInfo> res = { - {{{"database", dbId}}, {dbId}}, - {{{"cloud", cloudId}}, {cloudId}}, - {{{"folder", folderId}}, {folderId}}, - {{{"stream", topic}}, {topic}}}; - return res; -} - -TMultiCounter::TMultiCounter(NMonitoring::TDynamicCounterPtr counters, - const TVector<TLabelsInfo>& labels, - const TVector<std::pair<TString, TString>>& subgroups, - const TVector<TString>& counter_names, - bool deriv, - const TString& name, - bool expiring) - : Value(0) -{ - Y_VERIFY(counters); - - for (const auto& counter : counter_names) { - for (ui32 i = 0; i <= labels.size(); ++i) { - auto cc = counters; - for (ui32 j = 0; j < labels.size(); ++j) { - Y_VERIFY(!labels[j].Labels.empty()); - for (ui32 k = 0; k < labels[j].Labels.size(); ++k) { - Y_VERIFY(labels[j].Labels.size() == labels[j].AggrNames.size()); - const TString& res = (j < i) ? labels[j].Labels[k].second : labels[j].AggrNames[k]; - cc = cc->GetSubgroup(labels[j].Labels[k].first, res); - } - } - for (const auto& g: subgroups) { - cc = cc->GetSubgroup(g.first, g.second); - } - if (expiring) { - Counters.push_back(cc->GetExpiringNamedCounter(name, counter, deriv)); - } else { - Counters.push_back(cc->GetNamedCounter(name, counter, deriv)); - } - } - } -} - -void TMultiCounter::Inc(ui64 val) -{ - for (auto& c : Counters) (*c) += val; - Value += val; -} - -void TMultiCounter::Dec(ui64 val) { - for (auto& c : Counters) (*c) -= val; - Value -= val; -} - -void TMultiCounter::Set(ui64 value) { - auto diff = value - Value; - Inc(diff); -} - -TMultiCounter::operator bool() { - return !Counters.empty(); -} - - +TVector<TLabelsInfo> GetLabelsForStream(const TString& topic, const TString& cloudId, + const TString& dbId, const TString& folderId) { + TVector<TLabelsInfo> res = { + {{{"database", dbId}}, {dbId}}, + {{{"cloud", cloudId}}, {cloudId}}, + {{{"folder", folderId}}, {folderId}}, + {{{"stream", topic}}, {topic}}}; + return res; +} + +TMultiCounter::TMultiCounter(NMonitoring::TDynamicCounterPtr counters, + const TVector<TLabelsInfo>& labels, + const TVector<std::pair<TString, TString>>& subgroups, + const TVector<TString>& counter_names, + bool deriv, + const TString& name, + bool expiring) + : Value(0) +{ + Y_VERIFY(counters); + + for (const auto& counter : counter_names) { + for (ui32 i = 0; i <= labels.size(); ++i) { + auto cc = counters; + for (ui32 j = 0; j < labels.size(); ++j) { + Y_VERIFY(!labels[j].Labels.empty()); + for (ui32 k = 0; k < labels[j].Labels.size(); ++k) { + Y_VERIFY(labels[j].Labels.size() == labels[j].AggrNames.size()); + const TString& res = (j < i) ? labels[j].Labels[k].second : labels[j].AggrNames[k]; + cc = cc->GetSubgroup(labels[j].Labels[k].first, res); + } + } + for (const auto& g: subgroups) { + cc = cc->GetSubgroup(g.first, g.second); + } + if (expiring) { + Counters.push_back(cc->GetExpiringNamedCounter(name, counter, deriv)); + } else { + Counters.push_back(cc->GetNamedCounter(name, counter, deriv)); + } + } + } +} + +void TMultiCounter::Inc(ui64 val) +{ + for (auto& c : Counters) (*c) += val; + Value += val; +} + +void TMultiCounter::Dec(ui64 val) { + for (auto& c : Counters) (*c) -= val; + Value -= val; +} + +void TMultiCounter::Set(ui64 value) { + auto diff = value - Value; + Inc(diff); +} + +TMultiCounter::operator bool() { + return !Counters.empty(); +} + + TPercentileCounter::TPercentileCounter(TIntrusivePtr<NMonitoring::TDynamicCounters> counters, const TVector<TLabelsInfo>& labels, const TVector<std::pair<TString, TString>>& subgroups, const TString& sensor, const TVector<std::pair<ui64, TString>>& intervals, const bool deriv, bool expiring) { @@ -139,7 +139,7 @@ TPercentileCounter::TPercentileCounter(TIntrusivePtr<NMonitoring::TDynamicCounte Ranges.reserve(intervals.size()); for (auto& interval : intervals) { Ranges.push_back(interval.first); - Counters.push_back(TMultiCounter(counters, labels, subgroups, {interval.second}, deriv, sensor, expiring)); + Counters.push_back(TMultiCounter(counters, labels, subgroups, {interval.second}, deriv, sensor, expiring)); } Ranges.back() = Max<ui64>(); } diff --git a/ydb/core/persqueue/percentile_counter.h b/ydb/core/persqueue/percentile_counter.h index 74ee43d287..ce2730ba7a 100644 --- a/ydb/core/persqueue/percentile_counter.h +++ b/ydb/core/persqueue/percentile_counter.h @@ -5,11 +5,11 @@ namespace NKikimr { namespace NPQ { -NMonitoring::TDynamicCounterPtr GetCounters(NMonitoring::TDynamicCounterPtr counters, - const TString& subsystem, - const TString& topic); -NMonitoring::TDynamicCounterPtr GetCountersForStream(NMonitoring::TDynamicCounterPtr counters, - const TString& subsystem); +NMonitoring::TDynamicCounterPtr GetCounters(NMonitoring::TDynamicCounterPtr counters, + const TString& subsystem, + const TString& topic); +NMonitoring::TDynamicCounterPtr GetCountersForStream(NMonitoring::TDynamicCounterPtr counters, + const TString& subsystem); struct TLabelsInfo { TVector<std::pair<TString,TString>> Labels; @@ -18,26 +18,26 @@ struct TLabelsInfo { TVector<TLabelsInfo> GetLabels(const TString& topic); TVector<TLabelsInfo> GetLabels(const TString& cluster, const TString& oldTopic); -TVector<TLabelsInfo> GetLabelsForStream(const TString& topic, const TString& cloudId, - const TString& dbId, const TString& folderId); +TVector<TLabelsInfo> GetLabelsForStream(const TString& topic, const TString& cloudId, + const TString& dbId, const TString& folderId); class TMultiCounter { public: - TMultiCounter() = default; + TMultiCounter() = default; - TMultiCounter(NMonitoring::TDynamicCounterPtr counters, - const TVector<TLabelsInfo>& labels, - const TVector<std::pair<TString, TString>>& subgroups, - const TVector<TString>& counter_names, - bool deriv, - const TString& name = "sensor", - bool expiring = true); + TMultiCounter(NMonitoring::TDynamicCounterPtr counters, + const TVector<TLabelsInfo>& labels, + const TVector<std::pair<TString, TString>>& subgroups, + const TVector<TString>& counter_names, + bool deriv, + const TString& name = "sensor", + bool expiring = true); - void Inc(ui64 val = 1); - void Dec(ui64 val = 1); - void Set(ui64 value); + void Inc(ui64 val = 1); + void Dec(ui64 val = 1); + void Set(ui64 value); - operator bool(); + operator bool(); private: ui64 Value = 0; @@ -46,18 +46,18 @@ private: class TPercentileCounter { public: - TPercentileCounter() = default; - TPercentileCounter(TIntrusivePtr<NMonitoring::TDynamicCounters> counters, - const TVector<TLabelsInfo>& labels, - const TVector<std::pair<TString, TString>>& subgroups, - const TString& sensor, - const TVector<std::pair<ui64, TString>>& intervals, - const bool deriv, - bool expiring = true); + TPercentileCounter() = default; + TPercentileCounter(TIntrusivePtr<NMonitoring::TDynamicCounters> counters, + const TVector<TLabelsInfo>& labels, + const TVector<std::pair<TString, TString>>& subgroups, + const TString& sensor, + const TVector<std::pair<ui64, TString>>& intervals, + const bool deriv, + bool expiring = true); void IncFor(ui64 key, ui64 value = 1); void DecFor(ui64 key, ui64 value = 1); - + private: TVector<TMultiCounter> Counters; TVector<ui64> Ranges; diff --git a/ydb/core/persqueue/pq_impl.cpp b/ydb/core/persqueue/pq_impl.cpp index 9937dbe561..0fafca79b8 100644 --- a/ydb/core/persqueue/pq_impl.cpp +++ b/ydb/core/persqueue/pq_impl.cpp @@ -1449,7 +1449,7 @@ void TPersQueue::HandleWriteRequest(const ui64 responseCookie, const TActorId& p labels.pop_back(); } it = BytesWrittenFromDC.emplace(clientDC, NKikimr::NPQ::TMultiCounter(GetServiceCounters(counters, "pqproxy|writeSession"), - labels, {{"ClientDC", clientDC}}, {"BytesWrittenFromDC"}, true)).first; + labels, {{"ClientDC", clientDC}}, {"BytesWrittenFromDC"}, true)).first; } } if (it != BytesWrittenFromDC.end()) @@ -1527,7 +1527,7 @@ void TPersQueue::HandleWriteRequest(const ui64 responseCookie, const TActorId& p msgs.push_back({cmd.GetSourceId(), static_cast<ui64>(cmd.GetSeqNo()), partNo, totalParts, totalSize, createTimestampMs, receiveTimestampMs, disableDeduplication, writeTimestampMs, data, uncompressedSize, - cmd.GetPartitionKey(), cmd.GetExplicitHash(), cmd.GetExternalOperation() + cmd.GetPartitionKey(), cmd.GetExplicitHash(), cmd.GetExternalOperation() }); partNo++; uncompressedSize = 0; @@ -1542,23 +1542,23 @@ void TPersQueue::HandleWriteRequest(const ui64 responseCookie, const TActorId& p static_cast<ui16>(cmd.HasPartNo() ? cmd.GetTotalParts() : 1), static_cast<ui32>(cmd.HasTotalSize() ? cmd.GetTotalSize() : cmd.GetData().Size()), createTimestampMs, receiveTimestampMs, disableDeduplication, writeTimestampMs, cmd.GetData(), - cmd.HasUncompressedSize() ? cmd.GetUncompressedSize() : 0u, cmd.GetPartitionKey(), cmd.GetExplicitHash(), - cmd.GetExternalOperation() + cmd.HasUncompressedSize() ? cmd.GetUncompressedSize() : 0u, cmd.GetPartitionKey(), cmd.GetExplicitHash(), + cmd.GetExternalOperation() }); } - LOG_DEBUG_S(ctx, NKikimrServices::PERSQUEUE, "got client message topic: " << TopicName << - " partition: " << req.GetPartition() << - " SourceId: \'" << EscapeC(msgs.back().SourceId) << - "\' SeqNo: " << msgs.back().SeqNo << " partNo : " << msgs.back().PartNo << - " messageNo: " << req.GetMessageNo() << " size " << msgs.back().Data.size() << - " offset: " << (req.HasCmdWriteOffset() ? (req.GetCmdWriteOffset() + i) : -1)); + LOG_DEBUG_S(ctx, NKikimrServices::PERSQUEUE, "got client message topic: " << TopicName << + " partition: " << req.GetPartition() << + " SourceId: \'" << EscapeC(msgs.back().SourceId) << + "\' SeqNo: " << msgs.back().SeqNo << " partNo : " << msgs.back().PartNo << + " messageNo: " << req.GetMessageNo() << " size " << msgs.back().Data.size() << + " offset: " << (req.HasCmdWriteOffset() ? (req.GetCmdWriteOffset() + i) : -1)); } InitResponseBuilder(responseCookie, msgs.size(), COUNTER_LATENCY_PQ_WRITE); - THolder<TEvPQ::TEvWrite> event = - MakeHolder<TEvPQ::TEvWrite>(responseCookie, req.GetMessageNo(), - req.HasOwnerCookie() ? req.GetOwnerCookie() : "", - req.HasCmdWriteOffset() ? req.GetCmdWriteOffset() : TMaybe<ui64>(), - std::move(msgs), req.GetIsDirectWrite()); + THolder<TEvPQ::TEvWrite> event = + MakeHolder<TEvPQ::TEvWrite>(responseCookie, req.GetMessageNo(), + req.HasOwnerCookie() ? req.GetOwnerCookie() : "", + req.HasCmdWriteOffset() ? req.GetCmdWriteOffset() : TMaybe<ui64>(), + std::move(msgs), req.GetIsDirectWrite()); ctx.Send(partActor, event.Release()); } @@ -1659,16 +1659,16 @@ void TPersQueue::HandleReadRequest(const ui64 responseCookie, const TActorId& pa ui32 bytes = Min<ui32>(MAX_BYTES, cmd.HasBytes() ? cmd.GetBytes() : MAX_BYTES); auto clientDC = cmd.HasClientDC() ? to_lower(cmd.GetClientDC()) : "unknown"; clientDC.to_title(); - THolder<TEvPQ::TEvRead> event = - MakeHolder<TEvPQ::TEvRead>(responseCookie, cmd.GetOffset(), - cmd.HasPartNo() ? cmd.GetPartNo() : 0, - count, - cmd.HasSessionId() ? cmd.GetSessionId() : "", - cmd.GetClientId(), - cmd.HasTimeoutMs() ? cmd.GetTimeoutMs() : 0, bytes, - cmd.HasMaxTimeLagMs() ? cmd.GetMaxTimeLagMs() : 0, - cmd.HasReadTimestampMs() ? cmd.GetReadTimestampMs() : 0, clientDC, - cmd.GetExternalOperation()); + THolder<TEvPQ::TEvRead> event = + MakeHolder<TEvPQ::TEvRead>(responseCookie, cmd.GetOffset(), + cmd.HasPartNo() ? cmd.GetPartNo() : 0, + count, + cmd.HasSessionId() ? cmd.GetSessionId() : "", + cmd.GetClientId(), + cmd.HasTimeoutMs() ? cmd.GetTimeoutMs() : 0, bytes, + cmd.HasMaxTimeLagMs() ? cmd.GetMaxTimeLagMs() : 0, + cmd.HasReadTimestampMs() ? cmd.GetReadTimestampMs() : 0, clientDC, + cmd.GetExternalOperation()); ctx.Send(partActor, event.Release()); } } diff --git a/ydb/core/persqueue/pq_ut.cpp b/ydb/core/persqueue/pq_ut.cpp index 13a6122910..09d33224fe 100644 --- a/ydb/core/persqueue/pq_ut.cpp +++ b/ydb/core/persqueue/pq_ut.cpp @@ -19,7 +19,7 @@ #include <util/system/sanitizers.h> #include <util/system/valgrind.h> - + namespace NKikimr { Y_UNIT_TEST_SUITE(TPQTest) { @@ -1002,8 +1002,8 @@ Y_UNIT_TEST(TestAlreadyWrittenWithoutDeduplication) { CmdRead(0, 0, Max<i32>(), Max<i32>(), 2, false, tc, {0, 1}); }); } - - + + Y_UNIT_TEST(TestWritePQCompact) { TTestContext tc; RunTestWithReboots(tc.TabletIds, [&]() { @@ -1500,7 +1500,7 @@ Y_UNIT_TEST(TestPQRead) { }); } - + Y_UNIT_TEST(TestPQSmallRead) { TTestContext tc; RunTestWithReboots(tc.TabletIds, [&]() { @@ -1954,7 +1954,7 @@ Y_UNIT_TEST(TestMaxTimeLagRewind) { CmdRead(0, 0, 1, Max<i32>(), 1, false, tc, {21}, 0, ts.MilliSeconds() - 3 * 60 * 1000); CmdRead(0, 22, 1, Max<i32>(), 1, false, tc, {22}, 0, ts.MilliSeconds() - 3 * 60 * 1000); - CmdRead(0, 4, 1, Max<i32>(), 1, false, tc, {34}, 0, ts.MilliSeconds() - 1000); + CmdRead(0, 4, 1, Max<i32>(), 1, false, tc, {34}, 0, ts.MilliSeconds() - 1000); PQTabletPrepare(20000000, 100 * 1024 * 1024, 0, {{"aaa", true}}, tc, 2, 6*1024*1024, true, ts.MilliSeconds() - 1000); CmdRead(0, 0, 1, Max<i32>(), 1, false, tc, {34}); diff --git a/ydb/core/persqueue/pq_ut.h b/ydb/core/persqueue/pq_ut.h index 3db93bc104..8eecbe15a8 100644 --- a/ydb/core/persqueue/pq_ut.h +++ b/ydb/core/persqueue/pq_ut.h @@ -605,11 +605,11 @@ void WriteData(const ui32 partition, const TString& sourceId, const TVector<std: tc.Runtime->SendToPipe(tc.TabletId, tc.Edge, request.Release(), 0, GetPipeConfigWithRetries()); } -void CmdWrite(const ui32 partition, const TString& sourceId, const TVector<std::pair<ui64, TString>> data, - TTestContext& tc, bool error = false, const THashSet<ui32>& alreadyWrittenSeqNo = {}, - bool isFirst = false, const TString& ownerCookie = "", i32 msn = -1, i64 offset = -1, - bool treatWrongCookieAsError = false, bool treatBadOffsetAsError = true, - bool disableDeduplication = false) { +void CmdWrite(const ui32 partition, const TString& sourceId, const TVector<std::pair<ui64, TString>> data, + TTestContext& tc, bool error = false, const THashSet<ui32>& alreadyWrittenSeqNo = {}, + bool isFirst = false, const TString& ownerCookie = "", i32 msn = -1, i64 offset = -1, + bool treatWrongCookieAsError = false, bool treatBadOffsetAsError = true, + bool disableDeduplication = false) { TAutoPtr<IEventHandle> handle; TEvPersQueue::TEvResponse *result; @@ -1046,9 +1046,9 @@ void CmdRead(const ui32 partition, const ui64 offset, const ui32 count, const ui auto r = res.GetResult(i); if (offsets.empty()) { - if (readTimestampMs == 0) { - UNIT_ASSERT_EQUAL((ui64)r.GetOffset(), off); - } + if (readTimestampMs == 0) { + UNIT_ASSERT_EQUAL((ui64)r.GetOffset(), off); + } UNIT_ASSERT(r.GetSourceId().size() == 9 && r.GetSourceId().StartsWith("sourceid")); UNIT_ASSERT_EQUAL(ui32(r.GetData()[0]), off); UNIT_ASSERT_EQUAL(ui32((unsigned char)r.GetData().back()), r.GetSeqNo() % 256); diff --git a/ydb/core/persqueue/subscriber.h b/ydb/core/persqueue/subscriber.h index 7bbfe41e1f..3162f9c051 100644 --- a/ydb/core/persqueue/subscriber.h +++ b/ydb/core/persqueue/subscriber.h @@ -26,7 +26,7 @@ struct TReadInfo { ui32 Size; ui64 Destination; TInstant Timestamp; - ui64 ReadTimestampMs; + ui64 ReadTimestampMs; TDuration WaitQuotaTime; bool IsSubscription; @@ -70,7 +70,7 @@ struct TReadInfo { const ui64 dst, const ui64 sizeLag ); - + TReadAnswer FormAnswer( const TActorContext& ctx, const ui64 endOffset, diff --git a/ydb/core/persqueue/user_info.cpp b/ydb/core/persqueue/user_info.cpp index 26811765e2..d08c581efd 100644 --- a/ydb/core/persqueue/user_info.cpp +++ b/ydb/core/persqueue/user_info.cpp @@ -35,18 +35,18 @@ TUsersInfoStorage::TUsersInfoStorage( ui32 partition, const TTabletCountersBase& counters, const NKikimrPQ::TPQTabletConfig& config, - const TString& cloudId, - const TString& dbId, - const TString& folderId + const TString& cloudId, + const TString& dbId, + const TString& folderId ) : DCId(std::move(dcId)) , TabletId(tabletId) , TopicName(topicName) , Partition(partition) , Config(config) - , CloudId(cloudId) - , DbId(dbId) - , FolderId(folderId) + , CloudId(cloudId) + , DbId(dbId) + , FolderId(folderId) { Counters.Populate(counters); } @@ -154,7 +154,7 @@ TUserInfo& TUsersInfoStorage::Create( ui32 gen, ui32 step, i64 offset, ui64 readOffsetRewindSum, TInstant readFromTimestamp ) { - ui64 burst = 1'000'000'000, speed = 1'000'000'000; + ui64 burst = 1'000'000'000, speed = 1'000'000'000; if (AppData(ctx)->PQConfig.GetQuotingConfig().GetPartitionReadQuotaIsTwiceWriteQuota()) { burst = Config.GetPartitionConfig().GetBurstSize() * 2; speed = Config.GetPartitionConfig().GetWriteSpeedInBytesPerSecond() * 2; diff --git a/ydb/core/persqueue/user_info.h b/ydb/core/persqueue/user_info.h index 258e947c46..65384078ea 100644 --- a/ydb/core/persqueue/user_info.h +++ b/ydb/core/persqueue/user_info.h @@ -24,9 +24,9 @@ namespace NDeprecatedUserData { void Parse(const TString& data, ui64& offset, ui32& gen, ui32& step, TString& session); } // NDeprecatedUserInfo -static const ui32 MAX_USER_TS_CACHE_SIZE = 10'000; -static const ui64 MIN_TIMESTAMP_MS = 1'000'000'000'000ll; // around 2002 year -static const TString CLIENTID_TO_READ_INTERNALLY = "$without_consumer"; +static const ui32 MAX_USER_TS_CACHE_SIZE = 10'000; +static const ui64 MIN_TIMESTAMP_MS = 1'000'000'000'000ll; // around 2002 year +static const TString CLIENTID_TO_READ_INTERNALLY = "$without_consumer"; typedef TProtobufTabletLabeledCounters<EClientLabeledCounters_descriptor> TUserLabeledCounters; @@ -82,7 +82,7 @@ public: , SpeedPerSecond(speedPerSecond) , LastUpdateTime(timestamp) , MaxBurst(maxBurst) - , AvgMin(60'000) //avg avail in bytes per sec for last minute + , AvgMin(60'000) //avg avail in bytes per sec for last minute , AvgSec(1000) //avg avail in bytes per sec , QuotedTime(0) {} @@ -199,7 +199,7 @@ struct TUserInfo { NSlidingWindow::TSlidingWindow<NSlidingWindow::TMaxOperation<ui64>> WriteLagMs; std::shared_ptr<TPercentileCounter> ReadTimeLag; - bool DoExternalRead = false; + bool DoExternalRead = false; bool WriteInProgress = false; @@ -208,7 +208,7 @@ struct TUserInfo { TUserInfo(THolder<TReadSpeedLimiterHolder> readSpeedLimiter, const TString& user, const ui64 readRuleGeneration, bool important, const TString& topic, const ui32 partition, bool doExternalRead, - ui64 burst = 1'000'000'000, ui64 speed = 1'000'000'000) + ui64 burst = 1'000'000'000, ui64 speed = 1'000'000'000) : ReadSpeedLimiter(std::move(readSpeedLimiter)) , Important(important) , LabeledCounters(user + "/" + (important ? "1" : "0") + "/" + topic, partition) @@ -223,7 +223,7 @@ struct TUserInfo { , Subscriptions(0) , EndOffset(0) , WriteLagMs(TDuration::Minutes(1), 100) - , DoExternalRead(doExternalRead) + , DoExternalRead(doExternalRead) { } @@ -249,13 +249,13 @@ struct TUserInfo { } } - void ReadDone(const TActorContext& ctx, const TInstant& now, ui64 readSize, ui32 readCount, - const TString& clientDC) { + void ReadDone(const TActorContext& ctx, const TInstant& now, ui64 readSize, ui32 readCount, + const TString& clientDC) { if (BytesRead && !clientDC.empty()) { - if (BytesRead) - BytesRead.Inc(readSize); - if (MsgsRead) - MsgsRead.Inc(readCount); + if (BytesRead) + BytesRead.Inc(readSize); + if (MsgsRead) + MsgsRead.Inc(readCount); auto it = BytesReadFromDC.find(clientDC); if (it == BytesReadFromDC.end()) { auto pos = Topic.find("--"); @@ -264,12 +264,12 @@ struct TUserInfo { if (!labels.empty()) { labels.pop_back(); } - it = BytesReadFromDC.emplace(clientDC, - TMultiCounter(GetServiceCounters(AppData(ctx)->Counters, "pqproxy|readSession"), - labels, {{"ClientDC", clientDC}, - {"Client", User}, - {"ConsumerPath", NPersQueue::ConvertOldConsumerName(User, ctx)}}, - {"BytesReadFromDC"}, true)).first; + it = BytesReadFromDC.emplace(clientDC, + TMultiCounter(GetServiceCounters(AppData(ctx)->Counters, "pqproxy|readSession"), + labels, {{"ClientDC", clientDC}, + {"Client", User}, + {"ConsumerPath", NPersQueue::ConvertOldConsumerName(User, ctx)}}, + {"BytesReadFromDC"}, true)).first; } } if (it != BytesReadFromDC.end()) @@ -286,11 +286,11 @@ struct TUserInfo { } TUserInfo( - const TActorContext& ctx, THolder<TReadSpeedLimiterHolder> readSpeedLimiter, const TString& user, + const TActorContext& ctx, THolder<TReadSpeedLimiterHolder> readSpeedLimiter, const TString& user, const ui64 readRuleGeneration, const bool important, const TString& topic, const ui32 partition, const TString &session, - ui32 gen, ui32 step, i64 offset, const ui64 readOffsetRewindSum, const TString& dcId, - TInstant readFromTimestamp, const TString& cloudId, const TString& dbId, const TString& folderId, - ui64 burst = 1'000'000'000, ui64 speed = 1'000'000'000 + ui32 gen, ui32 step, i64 offset, const ui64 readOffsetRewindSum, const TString& dcId, + TInstant readFromTimestamp, const TString& cloudId, const TString& dbId, const TString& folderId, + ui64 burst = 1'000'000'000, ui64 speed = 1'000'000'000 ) : ReadSpeedLimiter(std::move(readSpeedLimiter)) , Session(session) @@ -318,97 +318,97 @@ struct TUserInfo { , ActiveReads(0) , Subscriptions(0) , EndOffset(0) - , AvgReadBytes{{TDuration::Seconds(1), 1000}, {TDuration::Minutes(1), 1000}, - {TDuration::Hours(1), 2000}, {TDuration::Days(1), 2000}} + , AvgReadBytes{{TDuration::Seconds(1), 1000}, {TDuration::Minutes(1), 1000}, + {TDuration::Hours(1), 2000}, {TDuration::Days(1), 2000}} , WriteLagMs(TDuration::Minutes(1), 100) { - if (AppData(ctx)->Counters) { - if (AppData()->PQConfig.GetTopicsAreFirstClassCitizen()) { - SetupStreamCounters(ctx, dcId, ToString<ui32>(partition), topic, cloudId, dbId, folderId); - } else { - if (topic.find("--") == TString::npos) - return; - SetupTopicCounters(ctx, dcId, ToString<ui32>(partition), topic); - } - } - } - - void SetupStreamCounters(const TActorContext& ctx, const TString& dcId, const TString& partition, - const TString& topic, const TString& cloudId, - const TString& dbId, const TString& folderId) { - auto subgroup = [&](const TString& subsystem) { - return NKikimr::NPQ::GetCountersForStream(AppData(ctx)->Counters, subsystem); - }; - auto additionalLabels = [&](const TVector<std::pair<TString, TString>>& subgroups = {}) { - TVector<std::pair<TString, TString>> result; - if (User != CLIENTID_TO_READ_INTERNALLY) - result.push_back({"consumer", User}); - - for (const auto& sb : subgroups) { - result.push_back(sb); - } - return result; - }; - const TVector<NPQ::TLabelsInfo> aggregates = - NKikimr::NPQ::GetLabelsForStream(topic, cloudId, dbId, folderId); - - if (DoExternalRead) { - BytesRead = TMultiCounter(subgroup("readSession"), aggregates, additionalLabels(), - {"stream.internal_read.bytes_per_second", - "stream.outcoming_bytes_per_second"}, true); - MsgsRead = TMultiCounter(subgroup("readSession"), aggregates, additionalLabels(), - {"stream.internal_read.records_per_second", - "stream.outcoming_records_per_second"}, true); + if (AppData(ctx)->Counters) { + if (AppData()->PQConfig.GetTopicsAreFirstClassCitizen()) { + SetupStreamCounters(ctx, dcId, ToString<ui32>(partition), topic, cloudId, dbId, folderId); + } else { + if (topic.find("--") == TString::npos) + return; + SetupTopicCounters(ctx, dcId, ToString<ui32>(partition), topic); + } } - - Counter.SetCounter(subgroup("readingTime"), - additionalLabels({{"host", dcId}, {"shard", partition}, {"stream", topic}}), - {"name", "stream.await_operating_milliseconds", true}); - - ReadTimeLag.reset(new TPercentileCounter(subgroup("readTimeLag"), aggregates, - additionalLabels({{"name", "stream.internal_read.time_lags_milliseconds"}}), "bin", - TVector<std::pair<ui64, TString>>{{100, "100"}, {200, "200"}, {500, "500"}, - {1000, "1000"}, {2000, "2000"}, - {5000, "5000"}, {10'000, "10000"}, - {30'000, "30000"}, {60'000, "60000"}, - {180'000,"180000"}, {9'999'999, "999999"}}, - true)); - } - - void SetupTopicCounters(const TActorContext& ctx, const TString& dcId, const TString& partition, - const TString& topic) { - auto subgroup = [&](const TString& subsystem) { - return GetServiceCounters(AppData(ctx)->Counters, subsystem); - }; - const TVector<NPQ::TLabelsInfo> aggr = NKikimr::NPQ::GetLabels(topic); - TVector<std::pair<TString, TString>> additional_labels = {{"Client", User}, - {"ConsumerPath", NPersQueue::ConvertOldConsumerName(User, ctx)} - }; - - Counter.SetCounter(subgroup("readingTime"), - {{"Client", User}, - {"ConsumerPath", NPersQueue::ConvertOldConsumerName(User, ctx)}, - {"host", dcId}, - {"Partition", partition}}, - {"sensor", "ReadTime", true}); - - BytesRead = TMultiCounter(subgroup("pqproxy|readSession"), aggr, additional_labels, - {"BytesRead"}, true); - MsgsRead = TMultiCounter(subgroup("pqproxy|readSession"), aggr, additional_labels, - {"MessagesRead"}, true); - - additional_labels.push_back({"sensor", "TimeLags"}); - ReadTimeLag.reset(new TPercentileCounter(subgroup("pqproxy|readTimeLag"), aggr, - additional_labels, "Interval", - TVector<std::pair<ui64, TString>>{{100, "100ms"}, {200, "200ms"}, {500, "500ms"}, - {1000, "1000ms"}, {2000, "2000ms"}, - {5000, "5000ms"}, {10'000, "10000ms"}, - {30'000, "30000ms"}, {60'000, "60000ms"}, - {180'000,"180000ms"}, {9'999'999, "999999ms"}}, - true)); - } + void SetupStreamCounters(const TActorContext& ctx, const TString& dcId, const TString& partition, + const TString& topic, const TString& cloudId, + const TString& dbId, const TString& folderId) { + auto subgroup = [&](const TString& subsystem) { + return NKikimr::NPQ::GetCountersForStream(AppData(ctx)->Counters, subsystem); + }; + auto additionalLabels = [&](const TVector<std::pair<TString, TString>>& subgroups = {}) { + TVector<std::pair<TString, TString>> result; + if (User != CLIENTID_TO_READ_INTERNALLY) + result.push_back({"consumer", User}); + + for (const auto& sb : subgroups) { + result.push_back(sb); + } + return result; + }; + const TVector<NPQ::TLabelsInfo> aggregates = + NKikimr::NPQ::GetLabelsForStream(topic, cloudId, dbId, folderId); + + if (DoExternalRead) { + BytesRead = TMultiCounter(subgroup("readSession"), aggregates, additionalLabels(), + {"stream.internal_read.bytes_per_second", + "stream.outcoming_bytes_per_second"}, true); + MsgsRead = TMultiCounter(subgroup("readSession"), aggregates, additionalLabels(), + {"stream.internal_read.records_per_second", + "stream.outcoming_records_per_second"}, true); + } + + Counter.SetCounter(subgroup("readingTime"), + additionalLabels({{"host", dcId}, {"shard", partition}, {"stream", topic}}), + {"name", "stream.await_operating_milliseconds", true}); + + ReadTimeLag.reset(new TPercentileCounter(subgroup("readTimeLag"), aggregates, + additionalLabels({{"name", "stream.internal_read.time_lags_milliseconds"}}), "bin", + TVector<std::pair<ui64, TString>>{{100, "100"}, {200, "200"}, {500, "500"}, + {1000, "1000"}, {2000, "2000"}, + {5000, "5000"}, {10'000, "10000"}, + {30'000, "30000"}, {60'000, "60000"}, + {180'000,"180000"}, {9'999'999, "999999"}}, + true)); + } + + void SetupTopicCounters(const TActorContext& ctx, const TString& dcId, const TString& partition, + const TString& topic) { + auto subgroup = [&](const TString& subsystem) { + return GetServiceCounters(AppData(ctx)->Counters, subsystem); + }; + const TVector<NPQ::TLabelsInfo> aggr = NKikimr::NPQ::GetLabels(topic); + TVector<std::pair<TString, TString>> additional_labels = {{"Client", User}, + {"ConsumerPath", NPersQueue::ConvertOldConsumerName(User, ctx)} + }; + + Counter.SetCounter(subgroup("readingTime"), + {{"Client", User}, + {"ConsumerPath", NPersQueue::ConvertOldConsumerName(User, ctx)}, + {"host", dcId}, + {"Partition", partition}}, + {"sensor", "ReadTime", true}); + + BytesRead = TMultiCounter(subgroup("pqproxy|readSession"), aggr, additional_labels, + {"BytesRead"}, true); + MsgsRead = TMultiCounter(subgroup("pqproxy|readSession"), aggr, additional_labels, + {"MessagesRead"}, true); + + additional_labels.push_back({"sensor", "TimeLags"}); + ReadTimeLag.reset(new TPercentileCounter(subgroup("pqproxy|readTimeLag"), aggr, + additional_labels, "Interval", + TVector<std::pair<ui64, TString>>{{100, "100ms"}, {200, "200ms"}, {500, "500ms"}, + {1000, "1000ms"}, {2000, "2000ms"}, + {5000, "5000ms"}, {10'000, "10000ms"}, + {30'000, "30000ms"}, {60'000, "60000ms"}, + {180'000,"180000ms"}, {9'999'999, "999999ms"}}, + true)); + + } + void SetQuota(const ui64 maxBurst, const ui64 speed) { ReadQuota.UpdateConfig(maxBurst, speed); } @@ -507,9 +507,9 @@ struct TUserInfo { class TUsersInfoStorage { public: - TUsersInfoStorage(TString dcId, ui64 tabletId, const TString& topicName, ui32 partition, + TUsersInfoStorage(TString dcId, ui64 tabletId, const TString& topicName, ui32 partition, const TTabletCountersBase& counters, const NKikimrPQ::TPQTabletConfig& config, - const TString& CloudId, const TString& DbId, const TString& FolderId); + const TString& CloudId, const TString& DbId, const TString& FolderId); void Init(TActorId tabletActor, TActorId partitionActor); @@ -549,10 +549,10 @@ private: TMaybe<TActorId> TabletActor; TMaybe<TActorId> PartitionActor; NKikimrPQ::TPQTabletConfig Config; - - TString CloudId; - TString DbId; - TString FolderId; + + TString CloudId; + TString DbId; + TString FolderId; }; } //NPQ diff --git a/ydb/core/persqueue/working_time_counter.h b/ydb/core/persqueue/working_time_counter.h index 4f4d81d263..964bb4ce8a 100644 --- a/ydb/core/persqueue/working_time_counter.h +++ b/ydb/core/persqueue/working_time_counter.h @@ -7,7 +7,7 @@ namespace NKikimr { namespace NPQ { - + class TWorkingTimeCounter { private: bool IsInWorkingState; @@ -36,16 +36,16 @@ public: } } - void SetCounter(NMonitoring::TDynamicCounterPtr counter, - const TVector<std::pair<TString, TString>>& subgroups, - const std::tuple<TString, TString, bool>& expiring) { - for (const auto& subgroup : subgroups) { - counter = counter->GetSubgroup(subgroup.first, subgroup.second); - } - - WorkingTimeMicroSec = counter->GetExpiringNamedCounter(std::get<0>(expiring), - std::get<1>(expiring), - std::get<2>(expiring)); + void SetCounter(NMonitoring::TDynamicCounterPtr counter, + const TVector<std::pair<TString, TString>>& subgroups, + const std::tuple<TString, TString, bool>& expiring) { + for (const auto& subgroup : subgroups) { + counter = counter->GetSubgroup(subgroup.first, subgroup.second); + } + + WorkingTimeMicroSec = counter->GetExpiringNamedCounter(std::get<0>(expiring), + std::get<1>(expiring), + std::get<2>(expiring)); } }; diff --git a/ydb/core/protos/msgbus_pq.proto b/ydb/core/protos/msgbus_pq.proto index 849b292383..879ba6d4a3 100644 --- a/ydb/core/protos/msgbus_pq.proto +++ b/ydb/core/protos/msgbus_pq.proto @@ -24,8 +24,8 @@ message TPersQueuePartitionRequest { optional string PartitionKey = 12; optional string ExplicitHash = 13; - - optional bool ExternalOperation = 14 [default = false]; + + optional bool ExternalOperation = 14 [default = false]; } message TCmdCreateSession { @@ -68,8 +68,8 @@ message TPersQueuePartitionRequest { optional string PartitionKey = 13; optional bytes ExplicitHash = 14; - - optional bool ExternalOperation = 15 [ default = false ]; + + optional bool ExternalOperation = 15 [ default = false ]; } message TCmdUpdateWriteTimestamp { diff --git a/ydb/core/protos/pqconfig.proto b/ydb/core/protos/pqconfig.proto index 7c85927449..d21ee01e20 100644 --- a/ydb/core/protos/pqconfig.proto +++ b/ydb/core/protos/pqconfig.proto @@ -682,19 +682,19 @@ message TPQClusterDiscoveryConfig { optional NKikimrNetClassifier.TNetData CloudNetData = 3; optional uint64 RequestInflightLimit = 4; } - -message TYdsNextToken { - required uint64 CreationTimestamp = 1; - required uint32 MaxResults = 2; - required uint32 AlreadyRead = 3; - required string StreamArn = 4; -} - -message TYdsShardIterator { - required string StreamName = 1; - required string StreamArn = 2; - required uint32 ShardId = 3; - required uint64 ReadTimestampMs = 4; - required uint32 SequenceNumber = 5; - required uint64 CreationTimestampMs = 6; -} + +message TYdsNextToken { + required uint64 CreationTimestamp = 1; + required uint32 MaxResults = 2; + required uint32 AlreadyRead = 3; + required string StreamArn = 4; +} + +message TYdsShardIterator { + required string StreamName = 1; + required string StreamArn = 2; + required uint32 ShardId = 3; + required uint64 ReadTimestampMs = 4; + required uint32 SequenceNumber = 5; + required uint64 CreationTimestampMs = 6; +} diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_alter_pq.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_alter_pq.cpp index f7a47459b9..1842f6317e 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_alter_pq.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_alter_pq.cpp @@ -75,7 +75,7 @@ public: } TPersQueueGroupInfo::TPtr ParseParams( - TOperationContext& context, + TOperationContext& context, NKikimrPQ::TPQTabletConfig* tabletConfig, const NKikimrSchemeOp::TPersQueueGroupDescription& alter, TString& errStr) @@ -139,22 +139,22 @@ public: return nullptr; } - const TPathElement::TPtr dbRootEl = context.SS->PathsById.at(context.SS->RootPathId()); - if (dbRootEl->UserAttrs->Attrs.contains("cloud_id")) { - auto cloudId = dbRootEl->UserAttrs->Attrs.at("cloud_id"); - tabletConfig->SetYcCloudId(cloudId); - } - if (dbRootEl->UserAttrs->Attrs.contains("folder_id")) { - auto folderId = dbRootEl->UserAttrs->Attrs.at("folder_id"); - tabletConfig->SetYcFolderId(folderId); - } - if (dbRootEl->UserAttrs->Attrs.contains("database_id")) { - auto databaseId = dbRootEl->UserAttrs->Attrs.at("database_id"); - tabletConfig->SetYdbDatabaseId(databaseId); - } - const TString databasePath = TPath::Init(context.SS->RootPathId(), context.SS).PathString(); - tabletConfig->SetYdbDatabasePath(databasePath); - + const TPathElement::TPtr dbRootEl = context.SS->PathsById.at(context.SS->RootPathId()); + if (dbRootEl->UserAttrs->Attrs.contains("cloud_id")) { + auto cloudId = dbRootEl->UserAttrs->Attrs.at("cloud_id"); + tabletConfig->SetYcCloudId(cloudId); + } + if (dbRootEl->UserAttrs->Attrs.contains("folder_id")) { + auto folderId = dbRootEl->UserAttrs->Attrs.at("folder_id"); + tabletConfig->SetYcFolderId(folderId); + } + if (dbRootEl->UserAttrs->Attrs.contains("database_id")) { + auto databaseId = dbRootEl->UserAttrs->Attrs.at("database_id"); + tabletConfig->SetYdbDatabaseId(databaseId); + } + const TString databasePath = TPath::Init(context.SS->RootPathId(), context.SS).PathString(); + tabletConfig->SetYdbDatabasePath(databasePath); + alterConfig.MutablePartitionKeySchema()->Swap(tabletConfig->MutablePartitionKeySchema()); Y_PROTOBUF_SUPPRESS_NODISCARD alterConfig.SerializeToString(¶ms->TabletConfig); alterConfig.Swap(tabletConfig); @@ -453,7 +453,7 @@ public: newTabletConfig = tabletConfig; - TPersQueueGroupInfo::TPtr alterData = ParseParams(context, &newTabletConfig, alter, errStr); + TPersQueueGroupInfo::TPtr alterData = ParseParams(context, &newTabletConfig, alter, errStr); if (!alterData) { result->SetError(NKikimrScheme::StatusInvalidParameter, errStr); return result; diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_pq.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_pq.cpp index 992fd3175c..7e58e43847 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_pq.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_pq.cpp @@ -12,8 +12,8 @@ namespace { using namespace NKikimr; using namespace NSchemeShard; -TPersQueueGroupInfo::TPtr CreatePersQueueGroup(TOperationContext& context, - const NKikimrSchemeOp::TPersQueueGroupDescription& op, +TPersQueueGroupInfo::TPtr CreatePersQueueGroup(TOperationContext& context, + const NKikimrSchemeOp::TPersQueueGroupDescription& op, TEvSchemeShard::EStatus& status, TString& errStr) { TPersQueueGroupInfo::TPtr pqGroupInfo = new TPersQueueGroupInfo; @@ -147,22 +147,22 @@ TPersQueueGroupInfo::TPtr CreatePersQueueGroup(TOperationContext& context, return nullptr; } - const TPathElement::TPtr dbRootEl = context.SS->PathsById.at(context.SS->RootPathId()); - if (dbRootEl->UserAttrs->Attrs.contains("cloud_id")) { - auto cloudId = dbRootEl->UserAttrs->Attrs.at("cloud_id"); - tabletConfig.SetYcCloudId(cloudId); - } - if (dbRootEl->UserAttrs->Attrs.contains("folder_id")) { - auto folderId = dbRootEl->UserAttrs->Attrs.at("folder_id"); - tabletConfig.SetYcFolderId(folderId); - } - if (dbRootEl->UserAttrs->Attrs.contains("database_id")) { - auto databaseId = dbRootEl->UserAttrs->Attrs.at("database_id"); - tabletConfig.SetYdbDatabaseId(databaseId); - } - const TString databasePath = TPath::Init(context.SS->RootPathId(), context.SS).PathString(); - tabletConfig.SetYdbDatabasePath(databasePath); - + const TPathElement::TPtr dbRootEl = context.SS->PathsById.at(context.SS->RootPathId()); + if (dbRootEl->UserAttrs->Attrs.contains("cloud_id")) { + auto cloudId = dbRootEl->UserAttrs->Attrs.at("cloud_id"); + tabletConfig.SetYcCloudId(cloudId); + } + if (dbRootEl->UserAttrs->Attrs.contains("folder_id")) { + auto folderId = dbRootEl->UserAttrs->Attrs.at("folder_id"); + tabletConfig.SetYcFolderId(folderId); + } + if (dbRootEl->UserAttrs->Attrs.contains("database_id")) { + auto databaseId = dbRootEl->UserAttrs->Attrs.at("database_id"); + tabletConfig.SetYdbDatabaseId(databaseId); + } + const TString databasePath = TPath::Init(context.SS->RootPathId(), context.SS).PathString(); + tabletConfig.SetYdbDatabasePath(databasePath); + Y_PROTOBUF_SUPPRESS_NODISCARD tabletConfig.SerializeToString(&pqGroupInfo->TabletConfig); if (op.HasBootstrapConfig()) { @@ -377,7 +377,7 @@ public: } TPersQueueGroupInfo::TPtr pqGroup = CreatePersQueueGroup( - context, createDEscription, status, errStr); + context, createDEscription, status, errStr); if (!pqGroup.Get()) { result->SetError(status, errStr); @@ -390,7 +390,7 @@ public: auto tabletConfig = pqGroup->TabletConfig; NKikimrPQ::TPQTabletConfig config; Y_VERIFY(!tabletConfig.empty()); - + bool parseOk = ParseFromStringNoSizeLimit(config, tabletConfig); Y_VERIFY(parseOk); diff --git a/ydb/public/api/protos/draft/datastreams.proto b/ydb/public/api/protos/draft/datastreams.proto index 5c1354288a..5543e964cf 100644 --- a/ydb/public/api/protos/draft/datastreams.proto +++ b/ydb/public/api/protos/draft/datastreams.proto @@ -12,8 +12,8 @@ option java_package = "com.yandex.ydb.datastreams.v1"; enum EFieldTransformationType { TRANSFORM_NONE = 0; TRANSFORM_BASE64 = 1; - TRANSFORM_DOUBLE_S_TO_INT_MS = 2; - TRANSFORM_EMPTY_TO_NOTHING = 3; + TRANSFORM_DOUBLE_S_TO_INT_MS = 2; + TRANSFORM_EMPTY_TO_NOTHING = 3; } extend google.protobuf.FieldOptions { @@ -45,7 +45,7 @@ message ChildShard { // Represents details of consumer message Consumer { string consumer_arn = 1; - int64 consumer_creation_timestamp = 2 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS]; + int64 consumer_creation_timestamp = 2 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS]; string consumer_name = 3; ConsumerDescription.ConsumerStatus consumer_status = 4; } @@ -58,9 +58,9 @@ message HashKeyRange { message Record { // Timestamp that the record was inserted into the stream - int64 timestamp = 1 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS]; + int64 timestamp = 1 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS]; // Data blob - bytes data = 2 [(FieldTransformer) = TRANSFORM_BASE64]; + bytes data = 2 [(FieldTransformer) = TRANSFORM_BASE64]; // Encryption type used on record EncryptionType encryption = 3; // Identifies shard in the stream the record is assigned to @@ -98,7 +98,7 @@ message StreamDescription { repeated Shard shards = 6; string stream_arn = 7; // Timestamp that the stream was created - int64 stream_creation_timestamp = 8 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS]; + int64 stream_creation_timestamp = 8 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS]; // Current status of the stream StreamStatus stream_status = 9; // Name of the stream @@ -116,11 +116,11 @@ message SequenceNumberRange { // Represents shard details message Shard { // Id of the shard adjacent to the shard's parent - string adjacent_parent_shard_id = 1 [(FieldTransformer) = TRANSFORM_EMPTY_TO_NOTHING]; + string adjacent_parent_shard_id = 1 [(FieldTransformer) = TRANSFORM_EMPTY_TO_NOTHING]; // The range of possible hash key values for the shard HashKeyRange hash_key_range = 2; // Id of the shard's parent - string parent_shard_id = 3 [(FieldTransformer) = TRANSFORM_EMPTY_TO_NOTHING]; + string parent_shard_id = 3 [(FieldTransformer) = TRANSFORM_EMPTY_TO_NOTHING]; // The range of possible sequence numbers for the shard SequenceNumberRange sequence_number_range = 4; // Unique id of the shard within stream @@ -138,7 +138,7 @@ message ConsumerDescription { string consumer_arn = 1; // Timestamp that the consumer was created - int64 consumer_creation_timestamp = 2 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS]; + int64 consumer_creation_timestamp = 2 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS]; // Name of the consumer string consumer_name = 3; ConsumerStatus consumer_status = 4; @@ -181,14 +181,14 @@ message ShardFilter { // Exclusive id. Can only be used if AFTER_SHARD_ID is specified string shard_id = 1; // Can only be used if AT_TIMESTAMP or FROM_TIMESTAMP are specified. - int64 timestamp = 2 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS]; + int64 timestamp = 2 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS]; ShardFilterType type = 3; } // Represents starting position in the stream from which to start reading message StartingPosition { // Timestamp of the record from which to start reading - int64 timestamp = 1 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS]; + int64 timestamp = 1 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS]; // Sequence number of the record from which to start reading string sequence_number = 2; ShardIteratorType type = 3; @@ -209,7 +209,7 @@ message StreamDescriptionSummary { int32 retention_period_hours = 6; string stream_arn = 7; // Timestamp that the stream was created - int64 stream_creation_timestamp = 8 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS]; + int64 stream_creation_timestamp = 8 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS]; // Name of the stream string stream_name = 9; // Current status of the stream @@ -322,7 +322,7 @@ message ListShardsRequest { // Filter out response ShardFilter shard_filter = 5; // Used to distinguish streams that have the same name - int64 stream_creation_timestamp = 6 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS]; + int64 stream_creation_timestamp = 6 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS]; // Name of the stream string stream_name = 7; } @@ -381,7 +381,7 @@ message ListStreamConsumersRequest { string next_token = 3; string stream_arn = 4; // Used to distinguish streams that have the same name - int64 stream_creation_timestamp = 5 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS]; + int64 stream_creation_timestamp = 5 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS]; } message ListStreamConsumersResponse { @@ -475,8 +475,8 @@ message PutRecordsRequestEntry { // Represents result of an individual record message PutRecordsResultEntry { - string error_message = 2 [(FieldTransformer) = TRANSFORM_EMPTY_TO_NOTHING]; - string error_code = 3 [(FieldTransformer) = TRANSFORM_EMPTY_TO_NOTHING]; + string error_message = 2 [(FieldTransformer) = TRANSFORM_EMPTY_TO_NOTHING]; + string error_code = 3 [(FieldTransformer) = TRANSFORM_EMPTY_TO_NOTHING]; string sequence_number = 4; string shard_id = 5; } @@ -484,7 +484,7 @@ message PutRecordsResultEntry { message GetRecordsRequest { Ydb.Operations.OperationParams operation_params = 1; // Max number of records to return - int32 limit = 2; + int32 limit = 2; // Iterator to shard string shard_iterator = 3; } @@ -513,7 +513,7 @@ message PutRecordRequest { string explicit_hash_key = 3; // Used as input to hash function that maps partition key to a specific shard string partition_key = 4; - string sequence_number_for_ordering = 5; + string sequence_number_for_ordering = 5; // Name of the stream to put record to string stream_name = 6; } @@ -564,7 +564,7 @@ message GetShardIteratorRequest { // Name of the stream string stream_name = 5; // Used with shard iterator type AT_TIMESTAMP - int64 timestamp = 6 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS]; + int64 timestamp = 6 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS]; } message GetShardIteratorResponse { @@ -573,7 +573,7 @@ message GetShardIteratorResponse { } message GetShardIteratorResult { - string shard_iterator = 1; + string shard_iterator = 1; } message SubscribeToShardRequest { @@ -653,8 +653,8 @@ message DescribeStreamSummaryResponse { } message DescribeStreamSummaryResult { - // Stream description sumary - StreamDescriptionSummary stream_description_summary = 1; + // Stream description sumary + StreamDescriptionSummary stream_description_summary = 1; } message DisableEnhancedMonitoringRequest { diff --git a/ydb/public/sdk/cpp/client/ydb_datastreams/datastreams.cpp b/ydb/public/sdk/cpp/client/ydb_datastreams/datastreams.cpp index f8297f4323..79552c06ef 100644 --- a/ydb/public/sdk/cpp/client/ydb_datastreams/datastreams.cpp +++ b/ydb/public/sdk/cpp/client/ydb_datastreams/datastreams.cpp @@ -94,21 +94,21 @@ namespace NYdb::NDataStreams::V1 { Ydb::DataStreams::V1::DescribeStreamResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDescribeStream); } - TAsyncListShardsResult ListShards(const TString &path, - const Ydb::DataStreams::V1::ShardFilter& shardFilter, - TListShardsSettings settings) { + TAsyncListShardsResult ListShards(const TString &path, + const Ydb::DataStreams::V1::ShardFilter& shardFilter, + TListShardsSettings settings) { return CallImpl<Ydb::DataStreams::V1::DataStreamsService, Ydb::DataStreams::V1::ListShardsRequest, Ydb::DataStreams::V1::ListShardsResponse, - Ydb::DataStreams::V1::ListShardsResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncListShards, - [&](Ydb::DataStreams::V1::ListShardsRequest& req) { - req.set_exclusive_start_shard_id(settings.ExclusiveStartShardId_); - req.set_max_results(settings.MaxResults_); - req.set_next_token(settings.NextToken_); - req.mutable_shard_filter()->CopyFrom(shardFilter); - req.set_stream_creation_timestamp(settings.StreamCreationTimestamp_); - req.set_stream_name(path); - }); + Ydb::DataStreams::V1::ListShardsResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncListShards, + [&](Ydb::DataStreams::V1::ListShardsRequest& req) { + req.set_exclusive_start_shard_id(settings.ExclusiveStartShardId_); + req.set_max_results(settings.MaxResults_); + req.set_next_token(settings.NextToken_); + req.mutable_shard_filter()->CopyFrom(shardFilter); + req.set_stream_creation_timestamp(settings.StreamCreationTimestamp_); + req.set_stream_name(path); + }); } TAsyncPutRecordsResult PutRecords(const TString& path, const std::vector<TDataRecord>& records, TPutRecordsSettings settings) { @@ -127,30 +127,30 @@ namespace NYdb::NDataStreams::V1 { }); } - TAsyncGetRecordsResult GetRecords(const TString& shardIterator, TGetRecordsSettings settings) { + TAsyncGetRecordsResult GetRecords(const TString& shardIterator, TGetRecordsSettings settings) { return CallImpl<Ydb::DataStreams::V1::DataStreamsService, Ydb::DataStreams::V1::GetRecordsRequest, Ydb::DataStreams::V1::GetRecordsResponse, - Ydb::DataStreams::V1::GetRecordsResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncGetRecords, - [&](Ydb::DataStreams::V1::GetRecordsRequest& req) { - req.set_shard_iterator(shardIterator); - req.set_limit(settings.Limit_); - }); + Ydb::DataStreams::V1::GetRecordsResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncGetRecords, + [&](Ydb::DataStreams::V1::GetRecordsRequest& req) { + req.set_shard_iterator(shardIterator); + req.set_limit(settings.Limit_); + }); } - TAsyncGetShardIteratorResult GetShardIterator(const TString& path, const TString& shardId, - Ydb::DataStreams::V1::ShardIteratorType shardIteratorType, - TGetShardIteratorSettings settings) { + TAsyncGetShardIteratorResult GetShardIterator(const TString& path, const TString& shardId, + Ydb::DataStreams::V1::ShardIteratorType shardIteratorType, + TGetShardIteratorSettings settings) { return CallImpl<Ydb::DataStreams::V1::DataStreamsService, Ydb::DataStreams::V1::GetShardIteratorRequest, Ydb::DataStreams::V1::GetShardIteratorResponse, Ydb::DataStreams::V1::GetShardIteratorResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncGetShardIterator, [&](Ydb::DataStreams::V1::GetShardIteratorRequest& req) { req.set_stream_name(path); - req.set_shard_id(shardId); - req.set_shard_iterator_type(shardIteratorType); - req.set_starting_sequence_number(settings.StartingSequenceNumber_); - req.set_timestamp(settings.Timestamp_); + req.set_shard_id(shardId); + req.set_shard_iterator_type(shardIteratorType); + req.set_starting_sequence_number(settings.StartingSequenceNumber_); + req.set_timestamp(settings.Timestamp_); }); } @@ -168,14 +168,14 @@ namespace NYdb::NDataStreams::V1 { Ydb::DataStreams::V1::DescribeLimitsResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDescribeLimits); } - TAsyncDescribeStreamSummaryResult DescribeStreamSummary(const TString& path, TDescribeStreamSummarySettings settings) { + TAsyncDescribeStreamSummaryResult DescribeStreamSummary(const TString& path, TDescribeStreamSummarySettings settings) { return CallImpl<Ydb::DataStreams::V1::DataStreamsService, Ydb::DataStreams::V1::DescribeStreamSummaryRequest, Ydb::DataStreams::V1::DescribeStreamSummaryResponse, - Ydb::DataStreams::V1::DescribeStreamSummaryResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDescribeStreamSummary, - [&](Ydb::DataStreams::V1::DescribeStreamSummaryRequest& req) { - req.set_stream_name(path); - }); + Ydb::DataStreams::V1::DescribeStreamSummaryResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDescribeStreamSummary, + [&](Ydb::DataStreams::V1::DescribeStreamSummaryRequest& req) { + req.set_stream_name(path); + }); } TAsyncDecreaseStreamRetentionPeriodResult DecreaseStreamRetentionPeriod(const TString& path, TDecreaseStreamRetentionPeriodSettings settings) { @@ -213,26 +213,26 @@ namespace NYdb::NDataStreams::V1 { }); } - TAsyncRegisterStreamConsumerResult RegisterStreamConsumer(const TString& path, const TString& consumer_name, TRegisterStreamConsumerSettings settings) { + TAsyncRegisterStreamConsumerResult RegisterStreamConsumer(const TString& path, const TString& consumer_name, TRegisterStreamConsumerSettings settings) { return CallImpl<Ydb::DataStreams::V1::DataStreamsService, Ydb::DataStreams::V1::RegisterStreamConsumerRequest, Ydb::DataStreams::V1::RegisterStreamConsumerResponse, - Ydb::DataStreams::V1::RegisterStreamConsumerResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncRegisterStreamConsumer, - [&](Ydb::DataStreams::V1::RegisterStreamConsumerRequest& req) { - req.set_stream_arn(path); - req.set_consumer_name(consumer_name); - }); + Ydb::DataStreams::V1::RegisterStreamConsumerResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncRegisterStreamConsumer, + [&](Ydb::DataStreams::V1::RegisterStreamConsumerRequest& req) { + req.set_stream_arn(path); + req.set_consumer_name(consumer_name); + }); } - TAsyncDeregisterStreamConsumerResult DeregisterStreamConsumer(const TString& path, const TString& consumer_name, TDeregisterStreamConsumerSettings settings) { + TAsyncDeregisterStreamConsumerResult DeregisterStreamConsumer(const TString& path, const TString& consumer_name, TDeregisterStreamConsumerSettings settings) { return CallImpl<Ydb::DataStreams::V1::DataStreamsService, Ydb::DataStreams::V1::DeregisterStreamConsumerRequest, Ydb::DataStreams::V1::DeregisterStreamConsumerResponse, - Ydb::DataStreams::V1::DeregisterStreamConsumerResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDeregisterStreamConsumer, - [&](Ydb::DataStreams::V1::DeregisterStreamConsumerRequest& req) { - req.set_stream_arn(path); - req.set_consumer_name(consumer_name); - }); + Ydb::DataStreams::V1::DeregisterStreamConsumerResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDeregisterStreamConsumer, + [&](Ydb::DataStreams::V1::DeregisterStreamConsumerRequest& req) { + req.set_stream_arn(path); + req.set_consumer_name(consumer_name); + }); } TAsyncDescribeStreamConsumerResult DescribeStreamConsumer(TDescribeStreamConsumerSettings settings) { @@ -242,15 +242,15 @@ namespace NYdb::NDataStreams::V1 { Ydb::DataStreams::V1::DescribeStreamConsumerResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDescribeStreamConsumer); } - TAsyncListStreamConsumersResult ListStreamConsumers(const TString& path, TListStreamConsumersSettings settings) { + TAsyncListStreamConsumersResult ListStreamConsumers(const TString& path, TListStreamConsumersSettings settings) { return CallImpl<Ydb::DataStreams::V1::DataStreamsService, Ydb::DataStreams::V1::ListStreamConsumersRequest, Ydb::DataStreams::V1::ListStreamConsumersResponse, - Ydb::DataStreams::V1::ListStreamConsumersResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncListStreamConsumers, [&](Ydb::DataStreams::V1::ListStreamConsumersRequest& req) { - req.set_stream_arn(path); - req.set_next_token(settings.NextToken_); - req.set_max_results(settings.MaxResults_); - }); + Ydb::DataStreams::V1::ListStreamConsumersResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncListStreamConsumers, [&](Ydb::DataStreams::V1::ListStreamConsumersRequest& req) { + req.set_stream_arn(path); + req.set_next_token(settings.NextToken_); + req.set_max_results(settings.MaxResults_); + }); } TAsyncAddTagsToStreamResult AddTagsToStream(TAddTagsToStreamSettings settings) { @@ -333,12 +333,12 @@ namespace NYdb::NDataStreams::V1 { return CallImpl<Ydb::DataStreams::V1::DataStreamsService, Ydb::DataStreams::V1::DeleteStreamRequest, Ydb::DataStreams::V1::DeleteStreamResponse, - Ydb::DataStreams::V1::DeleteStreamResult>(settings, - &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDeleteStream, - [&](Ydb::DataStreams::V1::DeleteStreamRequest& req) { - req.set_stream_name(path); - req.set_enforce_consumer_deletion(settings.EnforceConsumerDeletion_); - }); + Ydb::DataStreams::V1::DeleteStreamResult>(settings, + &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDeleteStream, + [&](Ydb::DataStreams::V1::DeleteStreamRequest& req) { + req.set_stream_name(path); + req.set_enforce_consumer_deletion(settings.EnforceConsumerDeletion_); + }); } TAsyncDescribeStreamResult DescribeStream(const TString &path, TDescribeStreamSettings settings) { @@ -400,22 +400,22 @@ namespace NYdb::NDataStreams::V1 { return Impl_->ListStreams(settings); } - TAsyncListShardsResult TDataStreamsClient::ListShards(const TString& path, - const Ydb::DataStreams::V1::ShardFilter& shardFilter, - TListShardsSettings settings) { - return Impl_->ListShards(path, shardFilter, settings); + TAsyncListShardsResult TDataStreamsClient::ListShards(const TString& path, + const Ydb::DataStreams::V1::ShardFilter& shardFilter, + TListShardsSettings settings) { + return Impl_->ListShards(path, shardFilter, settings); } TAsyncPutRecordsResult TDataStreamsClient::PutRecords(const TString& path, const std::vector<TDataRecord>& records, TPutRecordsSettings settings) { return Impl_->PutRecords(path, records, settings); } - TAsyncGetRecordsResult TDataStreamsClient::GetRecords(const TString& shardIterator, TGetRecordsSettings settings) { - return Impl_->GetRecords(shardIterator, settings); + TAsyncGetRecordsResult TDataStreamsClient::GetRecords(const TString& shardIterator, TGetRecordsSettings settings) { + return Impl_->GetRecords(shardIterator, settings); } - TAsyncGetShardIteratorResult TDataStreamsClient::GetShardIterator(const TString& path, const TString& shardId, Ydb::DataStreams::V1::ShardIteratorType shardIteratorType, TGetShardIteratorSettings settings) { - return Impl_->GetShardIterator(path, shardId, shardIteratorType, settings); + TAsyncGetShardIteratorResult TDataStreamsClient::GetShardIterator(const TString& path, const TString& shardId, Ydb::DataStreams::V1::ShardIteratorType shardIteratorType, TGetShardIteratorSettings settings) { + return Impl_->GetShardIterator(path, shardId, shardIteratorType, settings); } /* TAsyncSubscribeToShardResult TDataStreamsClient::SubscribeToShard(TSubscribeToShardSettings settings) { @@ -426,8 +426,8 @@ namespace NYdb::NDataStreams::V1 { return Impl_->DescribeLimits(settings); } - TAsyncDescribeStreamSummaryResult TDataStreamsClient::DescribeStreamSummary(const TString& path, TDescribeStreamSummarySettings settings) { - return Impl_->DescribeStreamSummary(path, settings); + TAsyncDescribeStreamSummaryResult TDataStreamsClient::DescribeStreamSummary(const TString& path, TDescribeStreamSummarySettings settings) { + return Impl_->DescribeStreamSummary(path, settings); } TAsyncDecreaseStreamRetentionPeriodResult TDataStreamsClient::DecreaseStreamRetentionPeriod(const TString& path, TDecreaseStreamRetentionPeriodSettings settings) { @@ -442,20 +442,20 @@ namespace NYdb::NDataStreams::V1 { return Impl_->UpdateShardCount(path, settings); } - TAsyncRegisterStreamConsumerResult TDataStreamsClient::RegisterStreamConsumer(const TString& path, const TString& consumer_name, const TRegisterStreamConsumerSettings settings) { - return Impl_->RegisterStreamConsumer(path, consumer_name, settings); + TAsyncRegisterStreamConsumerResult TDataStreamsClient::RegisterStreamConsumer(const TString& path, const TString& consumer_name, const TRegisterStreamConsumerSettings settings) { + return Impl_->RegisterStreamConsumer(path, consumer_name, settings); } - TAsyncDeregisterStreamConsumerResult TDataStreamsClient::DeregisterStreamConsumer(const TString& path, const TString& consumer_name, TDeregisterStreamConsumerSettings settings) { - return Impl_->DeregisterStreamConsumer(path, consumer_name, settings); + TAsyncDeregisterStreamConsumerResult TDataStreamsClient::DeregisterStreamConsumer(const TString& path, const TString& consumer_name, TDeregisterStreamConsumerSettings settings) { + return Impl_->DeregisterStreamConsumer(path, consumer_name, settings); } TAsyncDescribeStreamConsumerResult TDataStreamsClient::DescribeStreamConsumer(TDescribeStreamConsumerSettings settings) { return Impl_->DescribeStreamConsumer(settings); } - TAsyncListStreamConsumersResult TDataStreamsClient::ListStreamConsumers(const TString& path, TListStreamConsumersSettings settings) { - return Impl_->ListStreamConsumers(path, settings); + TAsyncListStreamConsumersResult TDataStreamsClient::ListStreamConsumers(const TString& path, TListStreamConsumersSettings settings) { + return Impl_->ListStreamConsumers(path, settings); } TAsyncAddTagsToStreamResult TDataStreamsClient::AddTagsToStream(TAddTagsToStreamSettings settings) { diff --git a/ydb/public/sdk/cpp/client/ydb_datastreams/datastreams.h b/ydb/public/sdk/cpp/client/ydb_datastreams/datastreams.h index 783f51080d..937e819607 100644 --- a/ydb/public/sdk/cpp/client/ydb_datastreams/datastreams.h +++ b/ydb/public/sdk/cpp/client/ydb_datastreams/datastreams.h @@ -104,26 +104,26 @@ namespace NYdb::NDataStreams::V1 { FLUENT_SETTING(TString, ExclusiveStartStreamName); FLUENT_SETTING_DEFAULT(bool, Recurse, false); }; - struct TDeleteStreamSettings : public NYdb::TOperationRequestSettings<TDeleteStreamSettings> { - FLUENT_SETTING_DEFAULT(bool, EnforceConsumerDeletion, false); - }; + struct TDeleteStreamSettings : public NYdb::TOperationRequestSettings<TDeleteStreamSettings> { + FLUENT_SETTING_DEFAULT(bool, EnforceConsumerDeletion, false); + }; struct TDescribeStreamSettings : public NYdb::TOperationRequestSettings<TDescribeStreamSettings> { FLUENT_SETTING(ui32, Limit); FLUENT_SETTING(TString, ExclusiveStartShardId); }; - struct TListShardsSettings : public NYdb::TOperationRequestSettings<TListShardsSettings> { - FLUENT_SETTING(TString, ExclusiveStartShardId); - FLUENT_SETTING(ui32, MaxResults); - FLUENT_SETTING(TString, NextToken); - FLUENT_SETTING(ui64, StreamCreationTimestamp); - }; - struct TGetRecordsSettings : public NYdb::TOperationRequestSettings<TGetRecordsSettings> { - FLUENT_SETTING_DEFAULT(ui32, Limit, 10000); - }; - struct TGetShardIteratorSettings : public NYdb::TOperationRequestSettings<TGetShardIteratorSettings> { - FLUENT_SETTING(TString, StartingSequenceNumber); - FLUENT_SETTING(ui64, Timestamp); - }; + struct TListShardsSettings : public NYdb::TOperationRequestSettings<TListShardsSettings> { + FLUENT_SETTING(TString, ExclusiveStartShardId); + FLUENT_SETTING(ui32, MaxResults); + FLUENT_SETTING(TString, NextToken); + FLUENT_SETTING(ui64, StreamCreationTimestamp); + }; + struct TGetRecordsSettings : public NYdb::TOperationRequestSettings<TGetRecordsSettings> { + FLUENT_SETTING_DEFAULT(ui32, Limit, 10000); + }; + struct TGetShardIteratorSettings : public NYdb::TOperationRequestSettings<TGetShardIteratorSettings> { + FLUENT_SETTING(TString, StartingSequenceNumber); + FLUENT_SETTING(ui64, Timestamp); + }; struct TSubscribeToShardSettings : public NYdb::TOperationRequestSettings<TSubscribeToShardSettings> {}; struct TDescribeLimitsSettings : public NYdb::TOperationRequestSettings<TDescribeLimitsSettings> {}; struct TDescribeStreamSummarySettings : public NYdb::TOperationRequestSettings<TDescribeStreamSummarySettings> {}; @@ -146,10 +146,10 @@ namespace NYdb::NDataStreams::V1 { struct TRegisterStreamConsumerSettings : public NYdb::TOperationRequestSettings<TRegisterStreamConsumerSettings> {}; struct TDeregisterStreamConsumerSettings : public NYdb::TOperationRequestSettings<TDeregisterStreamConsumerSettings> {}; struct TDescribeStreamConsumerSettings : public NYdb::TOperationRequestSettings<TDescribeStreamConsumerSettings> {}; - struct TListStreamConsumersSettings : public NYdb::TOperationRequestSettings<TListStreamConsumersSettings> { - FLUENT_SETTING(ui32, MaxResults); - FLUENT_SETTING(TString, NextToken); - }; + struct TListStreamConsumersSettings : public NYdb::TOperationRequestSettings<TListStreamConsumersSettings> { + FLUENT_SETTING(ui32, MaxResults); + FLUENT_SETTING(TString, NextToken); + }; struct TAddTagsToStreamSettings : public NYdb::TOperationRequestSettings<TAddTagsToStreamSettings> {}; struct TDisableEnhancedMonitoringSettings : public NYdb::TOperationRequestSettings<TDisableEnhancedMonitoringSettings> {}; struct TEnableEnhancedMonitoringSettings : public NYdb::TOperationRequestSettings<TEnableEnhancedMonitoringSettings> {}; @@ -172,21 +172,21 @@ namespace NYdb::NDataStreams::V1 { TAsyncDescribeStreamResult DescribeStream(const TString& path, TDescribeStreamSettings settings = TDescribeStreamSettings()); TAsyncPutRecordResult PutRecord(const TString& path, const TDataRecord& record, TPutRecordSettings settings = TPutRecordSettings()); TAsyncListStreamsResult ListStreams(TListStreamsSettings settings = TListStreamsSettings()); - TAsyncListShardsResult ListShards(const TString& path, const Ydb::DataStreams::V1::ShardFilter& shardFilter, TListShardsSettings settings = TListShardsSettings()); + TAsyncListShardsResult ListShards(const TString& path, const Ydb::DataStreams::V1::ShardFilter& shardFilter, TListShardsSettings settings = TListShardsSettings()); TAsyncPutRecordsResult PutRecords(const TString& path, const std::vector<TDataRecord>& records, TPutRecordsSettings settings = TPutRecordsSettings()); - TAsyncGetRecordsResult GetRecords(const TString& shardIterator, TGetRecordsSettings settings = TGetRecordsSettings()); - TAsyncGetShardIteratorResult GetShardIterator(const TString& path, const TString& shardId, Ydb::DataStreams::V1::ShardIteratorType shardIteratorTypeStr, - TGetShardIteratorSettings settings = TGetShardIteratorSettings()); + TAsyncGetRecordsResult GetRecords(const TString& shardIterator, TGetRecordsSettings settings = TGetRecordsSettings()); + TAsyncGetShardIteratorResult GetShardIterator(const TString& path, const TString& shardId, Ydb::DataStreams::V1::ShardIteratorType shardIteratorTypeStr, + TGetShardIteratorSettings settings = TGetShardIteratorSettings()); // TAsyncSubscribeToShardResult SubscribeToShard(TSubscribeToShardSettings settings = TSubscribeToShardSettings()); TAsyncDescribeLimitsResult DescribeLimits(TDescribeLimitsSettings settings = TDescribeLimitsSettings()); - TAsyncDescribeStreamSummaryResult DescribeStreamSummary(const TString& path, TDescribeStreamSummarySettings settings = TDescribeStreamSummarySettings()); + TAsyncDescribeStreamSummaryResult DescribeStreamSummary(const TString& path, TDescribeStreamSummarySettings settings = TDescribeStreamSummarySettings()); TAsyncDecreaseStreamRetentionPeriodResult DecreaseStreamRetentionPeriod(const TString& path, TDecreaseStreamRetentionPeriodSettings settings = TDecreaseStreamRetentionPeriodSettings()); TAsyncIncreaseStreamRetentionPeriodResult IncreaseStreamRetentionPeriod(const TString& path, TIncreaseStreamRetentionPeriodSettings settings = TIncreaseStreamRetentionPeriodSettings()); TAsyncUpdateShardCountResult UpdateShardCount(const TString& path, TUpdateShardCountSettings settings = TUpdateShardCountSettings()); - TAsyncRegisterStreamConsumerResult RegisterStreamConsumer(const TString& path, const TString& consumer_name, TRegisterStreamConsumerSettings settings = TRegisterStreamConsumerSettings()); - TAsyncDeregisterStreamConsumerResult DeregisterStreamConsumer(const TString& path, const TString& consumer_name, TDeregisterStreamConsumerSettings settings = TDeregisterStreamConsumerSettings()); + TAsyncRegisterStreamConsumerResult RegisterStreamConsumer(const TString& path, const TString& consumer_name, TRegisterStreamConsumerSettings settings = TRegisterStreamConsumerSettings()); + TAsyncDeregisterStreamConsumerResult DeregisterStreamConsumer(const TString& path, const TString& consumer_name, TDeregisterStreamConsumerSettings settings = TDeregisterStreamConsumerSettings()); TAsyncDescribeStreamConsumerResult DescribeStreamConsumer(TDescribeStreamConsumerSettings settings = TDescribeStreamConsumerSettings()); - TAsyncListStreamConsumersResult ListStreamConsumers(const TString& path, TListStreamConsumersSettings settings = TListStreamConsumersSettings()); + TAsyncListStreamConsumersResult ListStreamConsumers(const TString& path, TListStreamConsumersSettings settings = TListStreamConsumersSettings()); TAsyncAddTagsToStreamResult AddTagsToStream(TAddTagsToStreamSettings settings = TAddTagsToStreamSettings()); TAsyncDisableEnhancedMonitoringResult DisableEnhancedMonitoring(TDisableEnhancedMonitoringSettings settings = TDisableEnhancedMonitoringSettings()); TAsyncEnableEnhancedMonitoringResult EnableEnhancedMonitoring(TEnableEnhancedMonitoringSettings settings = TEnableEnhancedMonitoringSettings()); diff --git a/ydb/services/datastreams/datastreams_proxy.cpp b/ydb/services/datastreams/datastreams_proxy.cpp index 03a6d06213..336df8d874 100644 --- a/ydb/services/datastreams/datastreams_proxy.cpp +++ b/ydb/services/datastreams/datastreams_proxy.cpp @@ -1,21 +1,21 @@ #include "datastreams_proxy.h" #include "put_records_actor.h" -#include "shard_iterator.h" -#include "next_token.h" +#include "shard_iterator.h" +#include "next_token.h" #include <ydb/core/grpc_services/grpc_request_proxy.h> #include <ydb/core/grpc_services/rpc_deferrable.h> #include <ydb/core/grpc_services/rpc_scheme_base.h> #include <ydb/core/persqueue/partition.h> #include <ydb/core/persqueue/write_meta.h> - + #include <ydb/services/lib/actors/pq_schema_actor.h> #include <ydb/services/lib/sharding/sharding.h> #include <util/folder/path.h> -#include <iterator> - +#include <iterator> + using namespace NActors; using namespace NKikimrClient; @@ -111,7 +111,7 @@ namespace NKikimr::NDataStreams::V1 { void FillProposeRequest(TEvTxUserProxy::TEvProposeTransaction& proposal, const TActorContext& ctx, const TString& workingDir, const TString& name); void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx); - void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx); + void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx); void Handle(TEvTxUserProxy::TEvProposeTransactionStatus::TPtr& ev, const TActorContext& ctx); }; @@ -129,12 +129,12 @@ namespace NKikimr::NDataStreams::V1 { Become(&TCreateStreamActor::StateWork); } - void TCreateStreamActor::HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) { - Y_UNUSED(ev); - Y_UNUSED(ctx); - } - + void TCreateStreamActor::HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) { + Y_UNUSED(ev); + Y_UNUSED(ctx); + } + void TCreateStreamActor::FillProposeRequest(TEvTxUserProxy::TEvProposeTransaction& proposal, const TActorContext& ctx, const TString& workingDir, const TString& name) { @@ -179,7 +179,7 @@ namespace NKikimr::NDataStreams::V1 { TStringBuilder() << "Stream with name " << GetProtoRequest()->stream_name() << " is already exists", ctx); } - return TBase::TBase::Handle(ev, ctx); + return TBase::TBase::Handle(ev, ctx); } void TCreateStreamActor::StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) { @@ -202,29 +202,29 @@ namespace NKikimr::NDataStreams::V1 { void FillProposeRequest(TEvTxUserProxy::TEvProposeTransaction& proposal, const TActorContext& ctx, const TString& workingDir, const TString& name); - - void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, - const TActorContext& ctx); - - private: - bool EnforceDeletion; + + void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, + const TActorContext& ctx); + + private: + bool EnforceDeletion; }; TDeleteStreamActor::TDeleteStreamActor(NKikimr::NGRpcService::TEvDataStreamsDeleteStreamRequest* request) : TBase(request, request->GetProtoRequest()->stream_name()) - , EnforceDeletion{request->GetProtoRequest()->enforce_consumer_deletion()} + , EnforceDeletion{request->GetProtoRequest()->enforce_consumer_deletion()} { } void TDeleteStreamActor::Bootstrap(const NActors::TActorContext& ctx) { TBase::Bootstrap(ctx); - SendDescribeProposeRequest(ctx); + SendDescribeProposeRequest(ctx); Become(&TDeleteStreamActor::StateWork); } - void TDeleteStreamActor::FillProposeRequest(TEvTxUserProxy::TEvProposeTransaction& proposal, - const TActorContext& ctx, const TString& workingDir, - const TString& name) + void TDeleteStreamActor::FillProposeRequest(TEvTxUserProxy::TEvProposeTransaction& proposal, + const TActorContext& ctx, const TString& workingDir, + const TString& name) { LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, "WorkingDir = " << workingDir << ", name = " << name); NKikimrSchemeOp::TModifyScheme& modifyScheme(*proposal.Record.MutableTransaction()->MutableModifyScheme()); @@ -233,24 +233,24 @@ namespace NKikimr::NDataStreams::V1 { modifyScheme.MutableDrop()->SetName(name); } - void TDeleteStreamActor::HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, - const TActorContext& ctx) { - if (ReplyIfNotTopic(ev, ctx)) { - return; - } - - const auto& response = ev->Get()->Request.Get()->ResultSet.front(); - const auto& pqGroupDescription = response.PQGroupInfo->Description; - const auto& readRules = pqGroupDescription.GetPQTabletConfig().GetReadRules(); - - if (readRules.size() > 0 && EnforceDeletion == false) { - return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::ERROR, - TStringBuilder() << "Stream has registered consumers" << - "and EnforceConsumerDeletion flag is false", ctx); - } - - SendProposeRequest(ctx); - } + void TDeleteStreamActor::HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, + const TActorContext& ctx) { + if (ReplyIfNotTopic(ev, ctx)) { + return; + } + + const auto& response = ev->Get()->Request.Get()->ResultSet.front(); + const auto& pqGroupDescription = response.PQGroupInfo->Description; + const auto& readRules = pqGroupDescription.GetPQTabletConfig().GetReadRules(); + + if (readRules.size() > 0 && EnforceDeletion == false) { + return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::ERROR, + TStringBuilder() << "Stream has registered consumers" << + "and EnforceConsumerDeletion flag is false", ctx); + } + + SendProposeRequest(ctx); + } //----------------------------------------------------------------------------------------------------------- class TUpdateShardCountActor : public TUpdateSchemeActor<TUpdateShardCountActor, TEvDataStreamsUpdateShardCountRequest> { @@ -431,7 +431,7 @@ namespace NKikimr::NDataStreams::V1 { void Bootstrap(const NActors::TActorContext& ctx); void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx); - void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx); + void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx); void Handle(TEvTabletPipe::TEvClientConnected::TPtr& ev, const TActorContext& ctx) { if (ev->Get()->Status != NKikimrProto::EReplyStatus::OK) { @@ -454,7 +454,7 @@ namespace NKikimr::NDataStreams::V1 { } } - void Die(const TActorContext& ctx) override { + void Die(const TActorContext& ctx) override { //close all pipes for (auto& pipe : Pipes) { NTabletPipe::CloseClient(ctx, pipe); @@ -492,43 +492,43 @@ namespace NKikimr::NDataStreams::V1 { } } - void TDescribeStreamActor::HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) { + void TDescribeStreamActor::HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) { const NSchemeCache::TSchemeCacheNavigate* result = ev->Get()->Request.Get(); Y_VERIFY(result->ResultSet.size() == 1); // describe only one topic const auto& response = result->ResultSet.front(); - const TString path = JoinSeq("/", response.Path); - - if (ReplyIfNotTopic(ev, ctx)) { - return; - } - - Y_VERIFY(response.PQGroupInfo); - - PQGroup = response.PQGroupInfo->Description; - SelfInfo = response.Self->Info; - std::set<ui64> tabletIds; - for (auto& partition : PQGroup.GetPartitions()) { - tabletIds.insert(partition.GetTabletId()); - } - if (tabletIds.size() == 0) { - ReplyAndDie(ctx); - } - - RequestsInfly = tabletIds.size(); - - NTabletPipe::TClientConfig clientConfig; - clientConfig.RetryPolicy = { - .RetryLimitCount = 6, - .MinRetryTime = TDuration::MilliSeconds(10), - .MaxRetryTime = TDuration::MilliSeconds(100), - .BackoffMultiplier = 2, - .DoFirstRetryInstantly = true - }; - - for (auto& tabletId : tabletIds) { - Pipes.push_back(ctx.Register(NTabletPipe::CreateClient(ctx.SelfID, tabletId, clientConfig))); - TAutoPtr<TEvPersQueue::TEvOffsets> req(new TEvPersQueue::TEvOffsets); - NTabletPipe::SendData(ctx, Pipes.back(), req.Release()); + const TString path = JoinSeq("/", response.Path); + + if (ReplyIfNotTopic(ev, ctx)) { + return; + } + + Y_VERIFY(response.PQGroupInfo); + + PQGroup = response.PQGroupInfo->Description; + SelfInfo = response.Self->Info; + std::set<ui64> tabletIds; + for (auto& partition : PQGroup.GetPartitions()) { + tabletIds.insert(partition.GetTabletId()); + } + if (tabletIds.size() == 0) { + ReplyAndDie(ctx); + } + + RequestsInfly = tabletIds.size(); + + NTabletPipe::TClientConfig clientConfig; + clientConfig.RetryPolicy = { + .RetryLimitCount = 6, + .MinRetryTime = TDuration::MilliSeconds(10), + .MaxRetryTime = TDuration::MilliSeconds(100), + .BackoffMultiplier = 2, + .DoFirstRetryInstantly = true + }; + + for (auto& tabletId : tabletIds) { + Pipes.push_back(ctx.Register(NTabletPipe::CreateClient(ctx.SelfID, tabletId, clientConfig))); + TAutoPtr<TEvPersQueue::TEvOffsets> req(new TEvPersQueue::TEvOffsets); + NTabletPipe::SendData(ctx, Pipes.back(), req.Release()); } } @@ -744,155 +744,155 @@ namespace NKikimr::NDataStreams::V1 { //----------------------------------------------------------------------------------- - class TListStreamConsumersActor : public TPQGrpcSchemaBase<TListStreamConsumersActor, NKikimr::NGRpcService::TEvDataStreamsListStreamConsumersRequest> { - using TBase = TPQGrpcSchemaBase<TListStreamConsumersActor, TEvDataStreamsListStreamConsumersRequest>; - - public: - TListStreamConsumersActor(NKikimr::NGRpcService::TEvDataStreamsListStreamConsumersRequest* request); - ~TListStreamConsumersActor() = default; - - void Bootstrap(const NActors::TActorContext& ctx); - void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx); - void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx); - - protected: - void SendResponse(const TActorContext& ctx, const std::vector<std::pair<TString, ui64>>& readRules, ui32 leftToRead); - - private: - static constexpr ui32 MAX_MAX_RESULTS = 10000; - static constexpr ui32 MIN_MAX_RESULTS = 1; - static constexpr ui32 DEFAULT_MAX_RESULTS = 100; - - TString StreamArn; - ui32 MaxResults = DEFAULT_MAX_RESULTS; - TNextToken NextToken; - }; - TListStreamConsumersActor::TListStreamConsumersActor(NKikimr::NGRpcService::TEvDataStreamsListStreamConsumersRequest* request) - : TBase(request, TNextToken(request->GetProtoRequest()->next_token()).IsValid() ? - TNextToken(request->GetProtoRequest()->next_token()).GetStreamArn() : - request->GetProtoRequest()->stream_arn()) - , NextToken(request->GetProtoRequest()->next_token()) - { - if (request->GetProtoRequest()->next_token().empty()) { - StreamArn = request->GetProtoRequest()->stream_arn(); - MaxResults = request->GetProtoRequest()->max_results(); - NextToken = TNextToken(StreamArn, 0, MaxResults, TInstant::Now().MilliSeconds()); - } else { - StreamArn = NextToken.GetStreamArn(); - MaxResults = NextToken.GetMaxResults(); - } - } - - void TListStreamConsumersActor::Bootstrap(const NActors::TActorContext& ctx) { - TBase::Bootstrap(ctx); - - auto maxResultsInRange = MIN_MAX_RESULTS <= MaxResults && MaxResults <= MAX_MAX_RESULTS; - if (!maxResultsInRange) { - return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::ERROR, - TStringBuilder() << "Requested max_result value '" << MaxResults << - "' is out of range [" << MIN_MAX_RESULTS << ", " << MAX_MAX_RESULTS << - "]", ctx); - } - - if (!NextToken.IsValid()) { - return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::ERROR, - TStringBuilder() << "Provided NextToken has expired or malformed", ctx); - } - SendDescribeProposeRequest(ctx); - Become(&TListStreamConsumersActor::StateWork); - } - - void TListStreamConsumersActor::StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) { - switch (ev->GetTypeRewrite()) { - default: TBase::StateWork(ev, ctx); - } - } - - void TListStreamConsumersActor::HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) { + class TListStreamConsumersActor : public TPQGrpcSchemaBase<TListStreamConsumersActor, NKikimr::NGRpcService::TEvDataStreamsListStreamConsumersRequest> { + using TBase = TPQGrpcSchemaBase<TListStreamConsumersActor, TEvDataStreamsListStreamConsumersRequest>; + + public: + TListStreamConsumersActor(NKikimr::NGRpcService::TEvDataStreamsListStreamConsumersRequest* request); + ~TListStreamConsumersActor() = default; + + void Bootstrap(const NActors::TActorContext& ctx); + void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx); + void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx); + + protected: + void SendResponse(const TActorContext& ctx, const std::vector<std::pair<TString, ui64>>& readRules, ui32 leftToRead); + + private: + static constexpr ui32 MAX_MAX_RESULTS = 10000; + static constexpr ui32 MIN_MAX_RESULTS = 1; + static constexpr ui32 DEFAULT_MAX_RESULTS = 100; + + TString StreamArn; + ui32 MaxResults = DEFAULT_MAX_RESULTS; + TNextToken NextToken; + }; + TListStreamConsumersActor::TListStreamConsumersActor(NKikimr::NGRpcService::TEvDataStreamsListStreamConsumersRequest* request) + : TBase(request, TNextToken(request->GetProtoRequest()->next_token()).IsValid() ? + TNextToken(request->GetProtoRequest()->next_token()).GetStreamArn() : + request->GetProtoRequest()->stream_arn()) + , NextToken(request->GetProtoRequest()->next_token()) + { + if (request->GetProtoRequest()->next_token().empty()) { + StreamArn = request->GetProtoRequest()->stream_arn(); + MaxResults = request->GetProtoRequest()->max_results(); + NextToken = TNextToken(StreamArn, 0, MaxResults, TInstant::Now().MilliSeconds()); + } else { + StreamArn = NextToken.GetStreamArn(); + MaxResults = NextToken.GetMaxResults(); + } + } + + void TListStreamConsumersActor::Bootstrap(const NActors::TActorContext& ctx) { + TBase::Bootstrap(ctx); + + auto maxResultsInRange = MIN_MAX_RESULTS <= MaxResults && MaxResults <= MAX_MAX_RESULTS; + if (!maxResultsInRange) { + return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::ERROR, + TStringBuilder() << "Requested max_result value '" << MaxResults << + "' is out of range [" << MIN_MAX_RESULTS << ", " << MAX_MAX_RESULTS << + "]", ctx); + } + + if (!NextToken.IsValid()) { + return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::ERROR, + TStringBuilder() << "Provided NextToken has expired or malformed", ctx); + } + SendDescribeProposeRequest(ctx); + Become(&TListStreamConsumersActor::StateWork); + } + + void TListStreamConsumersActor::StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) { + switch (ev->GetTypeRewrite()) { + default: TBase::StateWork(ev, ctx); + } + } + + void TListStreamConsumersActor::HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) { const NSchemeCache::TSchemeCacheNavigate* result = ev->Get()->Request.Get(); Y_VERIFY(result->ResultSet.size() == 1); // describe only one topic - if (ReplyIfNotTopic(ev, ctx)) { - return; - } - - std::vector<std::pair<TString, ui64>> readRules; - ui32 leftToRead{0}; - const auto& response = result->ResultSet.front(); - const auto& pqGroupDescription = response.PQGroupInfo->Description; - const auto& streamReadRulesNames = pqGroupDescription.GetPQTabletConfig().GetReadRules(); - const auto& streamReadRulesReadFromTimestamps = pqGroupDescription.GetPQTabletConfig().GetReadFromTimestampsMs(); - const auto alreadyRead = NextToken.GetAlreadyRead(); - - if (alreadyRead > (ui32)streamReadRulesNames.size()) { - return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::ERROR, - TStringBuilder() << "Provided next_token is malformed - " << - "everything is already read", ctx); - } - - const auto rulesToRead = std::min(streamReadRulesNames.size() - alreadyRead, MaxResults); - readRules.reserve(rulesToRead); - auto itName = streamReadRulesNames.begin() + alreadyRead; - auto itTs = streamReadRulesReadFromTimestamps.begin() + alreadyRead; - for (auto i = rulesToRead; i > 0; --i, ++itName, ++itTs) { - readRules.push_back({*itName, *itTs}); - } - leftToRead = streamReadRulesNames.size() - alreadyRead - rulesToRead; - - SendResponse(ctx, readRules, leftToRead); - } - - void TListStreamConsumersActor::SendResponse(const TActorContext& ctx, const std::vector<std::pair<TString, ui64>>& readRules, ui32 leftToRead) { - Ydb::DataStreams::V1::ListStreamConsumersResult result; - - for (auto& readRule : readRules) { - auto consumer = result.Addconsumers(); - consumer->set_consumer_name(readRule.first); - consumer->set_consumer_creation_timestamp(readRule.second); - consumer->set_consumer_status(Ydb::DataStreams::V1::ConsumerDescription_ConsumerStatus_ACTIVE); - // TODO: consumer->set_consumer_arn(); - } - - if (leftToRead > 0) { - TNextToken token(StreamArn, NextToken.GetAlreadyRead() + readRules.size(), MaxResults, TInstant::Now().MilliSeconds()); - result.set_next_token(token.Serialize()); - } - - Request_->SendResult(result, Ydb::StatusIds::SUCCESS); - Die(ctx); - } - - //----------------------------------------------------------------------------------------- - - class TRegisterStreamConsumerActor : public TUpdateSchemeActor<TRegisterStreamConsumerActor, NKikimr::NGRpcService::TEvDataStreamsRegisterStreamConsumerRequest> { - using TBase = TUpdateSchemeActor<TRegisterStreamConsumerActor, TEvDataStreamsRegisterStreamConsumerRequest>; - - public: - TRegisterStreamConsumerActor(NKikimr::NGRpcService::TEvDataStreamsRegisterStreamConsumerRequest* request); - ~TRegisterStreamConsumerActor() = default; - - void Bootstrap(const NActors::TActorContext& ctx); - void ModifyPersqueueConfig(const TActorContext& ctx, + if (ReplyIfNotTopic(ev, ctx)) { + return; + } + + std::vector<std::pair<TString, ui64>> readRules; + ui32 leftToRead{0}; + const auto& response = result->ResultSet.front(); + const auto& pqGroupDescription = response.PQGroupInfo->Description; + const auto& streamReadRulesNames = pqGroupDescription.GetPQTabletConfig().GetReadRules(); + const auto& streamReadRulesReadFromTimestamps = pqGroupDescription.GetPQTabletConfig().GetReadFromTimestampsMs(); + const auto alreadyRead = NextToken.GetAlreadyRead(); + + if (alreadyRead > (ui32)streamReadRulesNames.size()) { + return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::ERROR, + TStringBuilder() << "Provided next_token is malformed - " << + "everything is already read", ctx); + } + + const auto rulesToRead = std::min(streamReadRulesNames.size() - alreadyRead, MaxResults); + readRules.reserve(rulesToRead); + auto itName = streamReadRulesNames.begin() + alreadyRead; + auto itTs = streamReadRulesReadFromTimestamps.begin() + alreadyRead; + for (auto i = rulesToRead; i > 0; --i, ++itName, ++itTs) { + readRules.push_back({*itName, *itTs}); + } + leftToRead = streamReadRulesNames.size() - alreadyRead - rulesToRead; + + SendResponse(ctx, readRules, leftToRead); + } + + void TListStreamConsumersActor::SendResponse(const TActorContext& ctx, const std::vector<std::pair<TString, ui64>>& readRules, ui32 leftToRead) { + Ydb::DataStreams::V1::ListStreamConsumersResult result; + + for (auto& readRule : readRules) { + auto consumer = result.Addconsumers(); + consumer->set_consumer_name(readRule.first); + consumer->set_consumer_creation_timestamp(readRule.second); + consumer->set_consumer_status(Ydb::DataStreams::V1::ConsumerDescription_ConsumerStatus_ACTIVE); + // TODO: consumer->set_consumer_arn(); + } + + if (leftToRead > 0) { + TNextToken token(StreamArn, NextToken.GetAlreadyRead() + readRules.size(), MaxResults, TInstant::Now().MilliSeconds()); + result.set_next_token(token.Serialize()); + } + + Request_->SendResult(result, Ydb::StatusIds::SUCCESS); + Die(ctx); + } + + //----------------------------------------------------------------------------------------- + + class TRegisterStreamConsumerActor : public TUpdateSchemeActor<TRegisterStreamConsumerActor, NKikimr::NGRpcService::TEvDataStreamsRegisterStreamConsumerRequest> { + using TBase = TUpdateSchemeActor<TRegisterStreamConsumerActor, TEvDataStreamsRegisterStreamConsumerRequest>; + + public: + TRegisterStreamConsumerActor(NKikimr::NGRpcService::TEvDataStreamsRegisterStreamConsumerRequest* request); + ~TRegisterStreamConsumerActor() = default; + + void Bootstrap(const NActors::TActorContext& ctx); + void ModifyPersqueueConfig(const TActorContext& ctx, NKikimrSchemeOp::TPersQueueGroupDescription& groupConfig, const NKikimrSchemeOp::TPersQueueGroupDescription& pqGroupDescription, const NKikimrSchemeOp::TDirEntry& selfInfo); void ReplyNotifyTxCompletionResult(NSchemeShard::TEvSchemeShard::TEvNotifyTxCompletionResult::TPtr& ev, const TActorContext& ctx) override; - - private: - TString ConsumerName; - }; - TRegisterStreamConsumerActor::TRegisterStreamConsumerActor(NKikimr::NGRpcService::TEvDataStreamsRegisterStreamConsumerRequest* request) - : TBase(request, request->GetProtoRequest()->stream_arn()) - , ConsumerName(request->GetProtoRequest()->consumer_name()) - { - } - - void TRegisterStreamConsumerActor::Bootstrap(const NActors::TActorContext& ctx) { - TBase::Bootstrap(ctx); - SendDescribeProposeRequest(ctx); - Become(&TRegisterStreamConsumerActor::StateWork); - } - + + private: + TString ConsumerName; + }; + TRegisterStreamConsumerActor::TRegisterStreamConsumerActor(NKikimr::NGRpcService::TEvDataStreamsRegisterStreamConsumerRequest* request) + : TBase(request, request->GetProtoRequest()->stream_arn()) + , ConsumerName(request->GetProtoRequest()->consumer_name()) + { + } + + void TRegisterStreamConsumerActor::Bootstrap(const NActors::TActorContext& ctx) { + TBase::Bootstrap(ctx); + SendDescribeProposeRequest(ctx); + Become(&TRegisterStreamConsumerActor::StateWork); + } + void TRegisterStreamConsumerActor::ModifyPersqueueConfig( const TActorContext& ctx, NKikimrSchemeOp::TPersQueueGroupDescription& groupConfig, @@ -901,72 +901,72 @@ namespace NKikimr::NDataStreams::V1 { ) { Y_UNUSED(pqGroupDescription); - auto* pqConfig = groupConfig.MutablePQTabletConfig(); - Ydb::PersQueue::V1::TopicSettings::ReadRule readRule; - readRule.set_consumer_name(ConsumerName); - readRule.set_supported_format(Ydb::PersQueue::V1::TopicSettings_Format_FORMAT_BASE); - readRule.set_starting_message_timestamp_ms(TInstant::Now().MilliSeconds()); - readRule.set_important(false); + auto* pqConfig = groupConfig.MutablePQTabletConfig(); + Ydb::PersQueue::V1::TopicSettings::ReadRule readRule; + readRule.set_consumer_name(ConsumerName); + readRule.set_supported_format(Ydb::PersQueue::V1::TopicSettings_Format_FORMAT_BASE); + readRule.set_starting_message_timestamp_ms(TInstant::Now().MilliSeconds()); + readRule.set_important(false); readRule.set_service_type(YDS_SERVICE_TYPE); - - if (readRule.version() == 0) { + + if (readRule.version() == 0) { readRule.set_version(selfInfo.GetVersion().GetPQVersion()); - } + } auto serviceTypes = GetSupportedClientServiceTypes(ctx); TString error = AddReadRuleToConfig(pqConfig, readRule, serviceTypes, ctx); bool hasDuplicates = false; - if (error.Empty()) { + if (error.Empty()) { hasDuplicates = CheckReadRulesConfig(*pqConfig, serviceTypes, error); - } - - if (!error.Empty()) { - return ReplyWithError(hasDuplicates ? Ydb::StatusIds::ALREADY_EXISTS : Ydb::StatusIds::BAD_REQUEST, + } + + if (!error.Empty()) { + return ReplyWithError(hasDuplicates ? Ydb::StatusIds::ALREADY_EXISTS : Ydb::StatusIds::BAD_REQUEST, hasDuplicates ? Ydb::PersQueue::ErrorCode::OK : Ydb::PersQueue::ErrorCode::BAD_REQUEST, error, ctx); - } - } - + } + } + void TRegisterStreamConsumerActor::ReplyNotifyTxCompletionResult(NSchemeShard::TEvSchemeShard::TEvNotifyTxCompletionResult::TPtr& ev, const TActorContext& ctx) { - Y_UNUSED(ev); - Ydb::DataStreams::V1::RegisterStreamConsumerResult result; - auto consumer = result.Mutableconsumer(); - consumer->set_consumer_name(ConsumerName); - consumer->set_consumer_creation_timestamp(std::chrono::seconds(std::time(nullptr)).count()); - // TODO: consumer->set_consumer_arn(); - consumer->set_consumer_status(Ydb::DataStreams::V1::ConsumerDescription_ConsumerStatus_ACTIVE); - Request_->SendResult(result, Ydb::StatusIds::SUCCESS); - Die(ctx); - } - - //----------------------------------------------------------------------------------------- - - class TDeregisterStreamConsumerActor : public TUpdateSchemeActor<TDeregisterStreamConsumerActor, NKikimr::NGRpcService::TEvDataStreamsDeregisterStreamConsumerRequest> { - using TBase = TUpdateSchemeActor<TDeregisterStreamConsumerActor, TEvDataStreamsDeregisterStreamConsumerRequest>; - - public: - TDeregisterStreamConsumerActor(NKikimr::NGRpcService::TEvDataStreamsDeregisterStreamConsumerRequest* request); - ~TDeregisterStreamConsumerActor() = default; - - void Bootstrap(const NActors::TActorContext& ctx); - void ModifyPersqueueConfig(const TActorContext& ctx, + Y_UNUSED(ev); + Ydb::DataStreams::V1::RegisterStreamConsumerResult result; + auto consumer = result.Mutableconsumer(); + consumer->set_consumer_name(ConsumerName); + consumer->set_consumer_creation_timestamp(std::chrono::seconds(std::time(nullptr)).count()); + // TODO: consumer->set_consumer_arn(); + consumer->set_consumer_status(Ydb::DataStreams::V1::ConsumerDescription_ConsumerStatus_ACTIVE); + Request_->SendResult(result, Ydb::StatusIds::SUCCESS); + Die(ctx); + } + + //----------------------------------------------------------------------------------------- + + class TDeregisterStreamConsumerActor : public TUpdateSchemeActor<TDeregisterStreamConsumerActor, NKikimr::NGRpcService::TEvDataStreamsDeregisterStreamConsumerRequest> { + using TBase = TUpdateSchemeActor<TDeregisterStreamConsumerActor, TEvDataStreamsDeregisterStreamConsumerRequest>; + + public: + TDeregisterStreamConsumerActor(NKikimr::NGRpcService::TEvDataStreamsDeregisterStreamConsumerRequest* request); + ~TDeregisterStreamConsumerActor() = default; + + void Bootstrap(const NActors::TActorContext& ctx); + void ModifyPersqueueConfig(const TActorContext& ctx, NKikimrSchemeOp::TPersQueueGroupDescription& groupConfig, const NKikimrSchemeOp::TPersQueueGroupDescription& pqGroupDescription, const NKikimrSchemeOp::TDirEntry& selfInfo); - - private: - TString ConsumerName; - }; - TDeregisterStreamConsumerActor::TDeregisterStreamConsumerActor(NKikimr::NGRpcService::TEvDataStreamsDeregisterStreamConsumerRequest* request) - : TBase(request, request->GetProtoRequest()->stream_arn()) - , ConsumerName(request->GetProtoRequest()->consumer_name()) - { - } - - void TDeregisterStreamConsumerActor::Bootstrap(const NActors::TActorContext& ctx) { - TBase::Bootstrap(ctx); - SendDescribeProposeRequest(ctx); - Become(&TDeregisterStreamConsumerActor::StateWork); - } - + + private: + TString ConsumerName; + }; + TDeregisterStreamConsumerActor::TDeregisterStreamConsumerActor(NKikimr::NGRpcService::TEvDataStreamsDeregisterStreamConsumerRequest* request) + : TBase(request, request->GetProtoRequest()->stream_arn()) + , ConsumerName(request->GetProtoRequest()->consumer_name()) + { + } + + void TDeregisterStreamConsumerActor::Bootstrap(const NActors::TActorContext& ctx) { + TBase::Bootstrap(ctx); + SendDescribeProposeRequest(ctx); + Become(&TDeregisterStreamConsumerActor::StateWork); + } + void TDeregisterStreamConsumerActor::ModifyPersqueueConfig( const TActorContext& ctx, NKikimrSchemeOp::TPersQueueGroupDescription& groupConfig, @@ -980,683 +980,683 @@ namespace NKikimr::NDataStreams::V1 { GetProtoRequest()->consumer_name(), ctx ); - if (!error.Empty()) { - return ReplyWithError(Ydb::StatusIds::NOT_FOUND, Ydb::PersQueue::ErrorCode::BAD_REQUEST, error, ctx); - } - } - - //----------------------------------------------------------------------------------------- - - class TGetShardIteratorActor : public TPQGrpcSchemaBase<TGetShardIteratorActor, NKikimr::NGRpcService::TEvDataStreamsGetShardIteratorRequest> { - using TBase = TPQGrpcSchemaBase<TGetShardIteratorActor, TEvDataStreamsGetShardIteratorRequest>; - - public: - TGetShardIteratorActor(NKikimr::NGRpcService::TEvDataStreamsGetShardIteratorRequest* request, NActors::TActorId newSchemeCache); - ~TGetShardIteratorActor() = default; - - void Bootstrap(const NActors::TActorContext& ctx); - void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx); - void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx); - - - private: - using TIteratorType = Ydb::DataStreams::V1::ShardIteratorType; - - void SendResponse(const TActorContext& ctx, const TShardIterator& shardIt); - std::optional<ui32> SequenceNumberToInt(const TString& sequenceNumberStr); - - TActorId NewSchemeCache; - TString StreamName; - TString ShardId; - TIteratorType IteratorType; - ui32 SequenceNumber; - ui64 ReadTimestampMs; - }; - - TGetShardIteratorActor::TGetShardIteratorActor(NKikimr::NGRpcService::TEvDataStreamsGetShardIteratorRequest* request, NActors::TActorId newSchemeCache) - : TBase(request, request->GetProtoRequest()->stream_name()) - , NewSchemeCache(std::move(newSchemeCache)) - , StreamName{request->GetProtoRequest()->stream_name()} - , ShardId{request->GetProtoRequest()->shard_id()} - , IteratorType{request->GetProtoRequest()->shard_iterator_type()} - , SequenceNumber{0} - , ReadTimestampMs{0} - { - } - - void TGetShardIteratorActor::Bootstrap(const NActors::TActorContext& ctx) { - TBase::Bootstrap(ctx); - - switch (IteratorType) { - case TIteratorType::AFTER_SEQUENCE_NUMBER: - case TIteratorType::AT_SEQUENCE_NUMBER: { - auto sn = SequenceNumberToInt(GetProtoRequest()->starting_sequence_number()); - if (!sn) { - return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::BAD_REQUEST, - TStringBuilder() << "Malformed sequence number", ctx); - } - SequenceNumber = sn.value() + (IteratorType == TIteratorType::AFTER_SEQUENCE_NUMBER ? 1u : 0u); - } - break; - case TIteratorType::AT_TIMESTAMP: - if (GetProtoRequest()->timestamp() == 0 || - GetProtoRequest()->timestamp() > static_cast<i64>(TInstant::Now().MilliSeconds())) { - return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::BAD_REQUEST, - TStringBuilder() << "Shard iterator type is AT_TIMESTAMP, " << - "but timestamp is either missed or too old or in future", ctx); - } - ReadTimestampMs = GetProtoRequest()->timestamp(); - break; - case TIteratorType::TRIM_HORIZON: - ReadTimestampMs = 0; - break; - case TIteratorType::LATEST: - ReadTimestampMs = TInstant::Now().MilliSeconds(); - break; - default: - return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::BAD_REQUEST, - TStringBuilder() << "Shard iterator type '" << - (ui32)IteratorType << "' is not known", ctx); - - } - - SendDescribeProposeRequest(ctx); - Become(&TGetShardIteratorActor::StateWork); - } - - void TGetShardIteratorActor::StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) { - switch (ev->GetTypeRewrite()) { - default: TBase::StateWork(ev, ctx); - } - } - - void TGetShardIteratorActor::HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) { - if (ReplyIfNotTopic(ev, ctx)) { - return; - } - - const NSchemeCache::TSchemeCacheNavigate* navigate = ev->Get()->Request.Get(); - auto topicInfo = navigate->ResultSet.begin(); - StreamName = NKikimr::CanonizePath(topicInfo->Path); - if (AppData(ctx)->PQConfig.GetRequireCredentialsInNewProtocol()) { - if (!topicInfo->SecurityObject->CheckAccess(NACLib::EAccessRights::SelectRow, - this->Request_->GetInternalToken())) { - return this->ReplyWithError(Ydb::StatusIds::UNAUTHORIZED, - Ydb::PersQueue::ErrorCode::ACCESS_DENIED, - TStringBuilder() << "Access to stream " - << this->GetProtoRequest()->stream_name() - << " is denied for subject " - << this->Request_->GetInternalToken(), ctx); - } - } - - const auto& partitions = topicInfo->PQGroupInfo->Description.GetPartitions(); - for (auto& partition : partitions) { - auto partitionId = partition.GetPartitionId(); - TString shardName = GetShardName(partitionId); - if (shardName == ShardId) { - TShardIterator it(StreamName, StreamName, partitionId, ReadTimestampMs, SequenceNumber); - SendResponse(ctx, it); - return; - } - } - - ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::ERROR, - TStringBuilder() << "No such shard: " << ShardId, ctx); - } - - void TGetShardIteratorActor::SendResponse(const TActorContext& ctx, const TShardIterator& shardIt) { - Ydb::DataStreams::V1::GetShardIteratorResult result; - result.set_shard_iterator(shardIt.Serialize()); - Request_->SendResult(result, Ydb::StatusIds::SUCCESS); - Die(ctx); - } - - std::optional<ui32> TGetShardIteratorActor::SequenceNumberToInt(const TString& sequenceNumberStr) { - try { - return std::stoi(sequenceNumberStr.c_str()); - } catch(...) { - return std::nullopt; - } - } - - //----------------------------------------------------------------------------------- - - class TGetRecordsActor : public TPQGrpcSchemaBase<TGetRecordsActor, TEvDataStreamsGetRecordsRequest> { - using TBase = TPQGrpcSchemaBase<TGetRecordsActor, TEvDataStreamsGetRecordsRequest>; - - static constexpr ui32 READ_TIMEOUT_MS = 150; - static constexpr i32 MAX_LIMIT = 10000; - - public: - TGetRecordsActor(TEvDataStreamsGetRecordsRequest* request, const TActorId& newSchemeCache); - ~TGetRecordsActor() = default; - - void Bootstrap(const NActors::TActorContext& ctx); - void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx); - void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx); - void Handle(TEvPersQueue::TEvResponse::TPtr& ev, const TActorContext& ctx); - - private: - void SendReadRequest(const TActorContext& ctx); - void SendResponse(const TActorContext& ctx, - const std::vector<Ydb::DataStreams::V1::Record>& records, - ui64 millisBehindLatestMs); - - TShardIterator ShardIterator; - TString StreamName; - ui64 TabletId; - i32 Limit; - TActorId NewSchemeCache; - }; - - TGetRecordsActor::TGetRecordsActor(TEvDataStreamsGetRecordsRequest* request, - const TActorId& newSchemeCache) - : TBase(request, TShardIterator(request->GetProtoRequest()->shard_iterator()).IsValid() - ? TShardIterator(request->GetProtoRequest()->shard_iterator()).GetStreamName() - : "undefined") - , ShardIterator{request->GetProtoRequest()->shard_iterator()} - , StreamName{ShardIterator.IsValid() ? ShardIterator.GetStreamName() : "undefined"} - , TabletId{0} - , Limit{request->GetProtoRequest()->limit()} - , NewSchemeCache{newSchemeCache} - { - } - - void TGetRecordsActor::Bootstrap(const NActors::TActorContext& ctx) { - TBase::Bootstrap(ctx); - - if (!ShardIterator.IsValid()) { - return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::ERROR, - TStringBuilder() << "Provided shard iterator is malformed or expired", ctx); - } - - Limit = Limit == 0 ? MAX_LIMIT : Limit; - if (Limit < 1 || Limit > MAX_LIMIT) { - return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::ERROR, - TStringBuilder() << "Limit '" << Limit << "' is out of bounds [1; " << MAX_LIMIT << "]", ctx); - } - - SendDescribeProposeRequest(ctx); - Become(&TGetRecordsActor::StateWork); - } - - void TGetRecordsActor::SendReadRequest(const TActorContext& ctx) { - NTabletPipe::TClientConfig clientConfig; - clientConfig.RetryPolicy = { - .RetryLimitCount = 6, - .MinRetryTime = TDuration::MilliSeconds(10), - .MaxRetryTime = TDuration::MilliSeconds(100), - .BackoffMultiplier = 2, - .DoFirstRetryInstantly = true - }; - auto PipeClient = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, TabletId, clientConfig)); - - NKikimrClient::TPersQueueRequest request; - request.MutablePartitionRequest()->SetTopic(this->GetTopicPath(ctx)); - request.MutablePartitionRequest()->SetPartition(ShardIterator.GetShardId()); - ActorIdToProto(PipeClient, request.MutablePartitionRequest()->MutablePipeClient()); - - auto cmdRead = request.MutablePartitionRequest()->MutableCmdRead(); - cmdRead->SetClientId(NKikimr::NPQ::CLIENTID_TO_READ_INTERNALLY); - cmdRead->SetCount(Limit); - cmdRead->SetOffset(ShardIterator.GetSequenceNumber()); - cmdRead->SetReadTimestampMs(ShardIterator.GetReadTimestamp()); - cmdRead->SetTimeoutMs(READ_TIMEOUT_MS); - cmdRead->SetExternalOperation(true); - - TAutoPtr<TEvPersQueue::TEvRequest> req(new TEvPersQueue::TEvRequest); - req->Record.Swap(&request); - NTabletPipe::SendData(ctx, PipeClient, req.Release()); - } - - void TGetRecordsActor::StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) { - switch (ev->GetTypeRewrite()) { - HFunc(TEvPersQueue::TEvResponse, Handle); - default: TBase::StateWork(ev, ctx); - } - } - - void TGetRecordsActor::HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, - const TActorContext& ctx) { - const auto &result = ev->Get()->Request.Get(); - const auto response = result->ResultSet.front(); - - if (AppData(ctx)->PQConfig.GetRequireCredentialsInNewProtocol()) { - if (!response.SecurityObject->CheckAccess(NACLib::EAccessRights::SelectRow, - this->Request_->GetInternalToken())) { - return ReplyWithError(Ydb::StatusIds::UNAUTHORIZED, - Ydb::PersQueue::ErrorCode::ACCESS_DENIED, - TStringBuilder() << "Access to stream " - << ShardIterator.GetStreamName() - << " is denied for subject " - << this->Request_->GetInternalToken(), ctx); - } - } - - + if (!error.Empty()) { + return ReplyWithError(Ydb::StatusIds::NOT_FOUND, Ydb::PersQueue::ErrorCode::BAD_REQUEST, error, ctx); + } + } + + //----------------------------------------------------------------------------------------- + + class TGetShardIteratorActor : public TPQGrpcSchemaBase<TGetShardIteratorActor, NKikimr::NGRpcService::TEvDataStreamsGetShardIteratorRequest> { + using TBase = TPQGrpcSchemaBase<TGetShardIteratorActor, TEvDataStreamsGetShardIteratorRequest>; + + public: + TGetShardIteratorActor(NKikimr::NGRpcService::TEvDataStreamsGetShardIteratorRequest* request, NActors::TActorId newSchemeCache); + ~TGetShardIteratorActor() = default; + + void Bootstrap(const NActors::TActorContext& ctx); + void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx); + void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx); + + + private: + using TIteratorType = Ydb::DataStreams::V1::ShardIteratorType; + + void SendResponse(const TActorContext& ctx, const TShardIterator& shardIt); + std::optional<ui32> SequenceNumberToInt(const TString& sequenceNumberStr); + + TActorId NewSchemeCache; + TString StreamName; + TString ShardId; + TIteratorType IteratorType; + ui32 SequenceNumber; + ui64 ReadTimestampMs; + }; + + TGetShardIteratorActor::TGetShardIteratorActor(NKikimr::NGRpcService::TEvDataStreamsGetShardIteratorRequest* request, NActors::TActorId newSchemeCache) + : TBase(request, request->GetProtoRequest()->stream_name()) + , NewSchemeCache(std::move(newSchemeCache)) + , StreamName{request->GetProtoRequest()->stream_name()} + , ShardId{request->GetProtoRequest()->shard_id()} + , IteratorType{request->GetProtoRequest()->shard_iterator_type()} + , SequenceNumber{0} + , ReadTimestampMs{0} + { + } + + void TGetShardIteratorActor::Bootstrap(const NActors::TActorContext& ctx) { + TBase::Bootstrap(ctx); + + switch (IteratorType) { + case TIteratorType::AFTER_SEQUENCE_NUMBER: + case TIteratorType::AT_SEQUENCE_NUMBER: { + auto sn = SequenceNumberToInt(GetProtoRequest()->starting_sequence_number()); + if (!sn) { + return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::BAD_REQUEST, + TStringBuilder() << "Malformed sequence number", ctx); + } + SequenceNumber = sn.value() + (IteratorType == TIteratorType::AFTER_SEQUENCE_NUMBER ? 1u : 0u); + } + break; + case TIteratorType::AT_TIMESTAMP: + if (GetProtoRequest()->timestamp() == 0 || + GetProtoRequest()->timestamp() > static_cast<i64>(TInstant::Now().MilliSeconds())) { + return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::BAD_REQUEST, + TStringBuilder() << "Shard iterator type is AT_TIMESTAMP, " << + "but timestamp is either missed or too old or in future", ctx); + } + ReadTimestampMs = GetProtoRequest()->timestamp(); + break; + case TIteratorType::TRIM_HORIZON: + ReadTimestampMs = 0; + break; + case TIteratorType::LATEST: + ReadTimestampMs = TInstant::Now().MilliSeconds(); + break; + default: + return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::BAD_REQUEST, + TStringBuilder() << "Shard iterator type '" << + (ui32)IteratorType << "' is not known", ctx); + + } + + SendDescribeProposeRequest(ctx); + Become(&TGetShardIteratorActor::StateWork); + } + + void TGetShardIteratorActor::StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) { + switch (ev->GetTypeRewrite()) { + default: TBase::StateWork(ev, ctx); + } + } + + void TGetShardIteratorActor::HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) { + if (ReplyIfNotTopic(ev, ctx)) { + return; + } + + const NSchemeCache::TSchemeCacheNavigate* navigate = ev->Get()->Request.Get(); + auto topicInfo = navigate->ResultSet.begin(); + StreamName = NKikimr::CanonizePath(topicInfo->Path); + if (AppData(ctx)->PQConfig.GetRequireCredentialsInNewProtocol()) { + if (!topicInfo->SecurityObject->CheckAccess(NACLib::EAccessRights::SelectRow, + this->Request_->GetInternalToken())) { + return this->ReplyWithError(Ydb::StatusIds::UNAUTHORIZED, + Ydb::PersQueue::ErrorCode::ACCESS_DENIED, + TStringBuilder() << "Access to stream " + << this->GetProtoRequest()->stream_name() + << " is denied for subject " + << this->Request_->GetInternalToken(), ctx); + } + } + + const auto& partitions = topicInfo->PQGroupInfo->Description.GetPartitions(); + for (auto& partition : partitions) { + auto partitionId = partition.GetPartitionId(); + TString shardName = GetShardName(partitionId); + if (shardName == ShardId) { + TShardIterator it(StreamName, StreamName, partitionId, ReadTimestampMs, SequenceNumber); + SendResponse(ctx, it); + return; + } + } + + ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::ERROR, + TStringBuilder() << "No such shard: " << ShardId, ctx); + } + + void TGetShardIteratorActor::SendResponse(const TActorContext& ctx, const TShardIterator& shardIt) { + Ydb::DataStreams::V1::GetShardIteratorResult result; + result.set_shard_iterator(shardIt.Serialize()); + Request_->SendResult(result, Ydb::StatusIds::SUCCESS); + Die(ctx); + } + + std::optional<ui32> TGetShardIteratorActor::SequenceNumberToInt(const TString& sequenceNumberStr) { + try { + return std::stoi(sequenceNumberStr.c_str()); + } catch(...) { + return std::nullopt; + } + } + + //----------------------------------------------------------------------------------- + + class TGetRecordsActor : public TPQGrpcSchemaBase<TGetRecordsActor, TEvDataStreamsGetRecordsRequest> { + using TBase = TPQGrpcSchemaBase<TGetRecordsActor, TEvDataStreamsGetRecordsRequest>; + + static constexpr ui32 READ_TIMEOUT_MS = 150; + static constexpr i32 MAX_LIMIT = 10000; + + public: + TGetRecordsActor(TEvDataStreamsGetRecordsRequest* request, const TActorId& newSchemeCache); + ~TGetRecordsActor() = default; + + void Bootstrap(const NActors::TActorContext& ctx); + void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx); + void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx); + void Handle(TEvPersQueue::TEvResponse::TPtr& ev, const TActorContext& ctx); + + private: + void SendReadRequest(const TActorContext& ctx); + void SendResponse(const TActorContext& ctx, + const std::vector<Ydb::DataStreams::V1::Record>& records, + ui64 millisBehindLatestMs); + + TShardIterator ShardIterator; + TString StreamName; + ui64 TabletId; + i32 Limit; + TActorId NewSchemeCache; + }; + + TGetRecordsActor::TGetRecordsActor(TEvDataStreamsGetRecordsRequest* request, + const TActorId& newSchemeCache) + : TBase(request, TShardIterator(request->GetProtoRequest()->shard_iterator()).IsValid() + ? TShardIterator(request->GetProtoRequest()->shard_iterator()).GetStreamName() + : "undefined") + , ShardIterator{request->GetProtoRequest()->shard_iterator()} + , StreamName{ShardIterator.IsValid() ? ShardIterator.GetStreamName() : "undefined"} + , TabletId{0} + , Limit{request->GetProtoRequest()->limit()} + , NewSchemeCache{newSchemeCache} + { + } + + void TGetRecordsActor::Bootstrap(const NActors::TActorContext& ctx) { + TBase::Bootstrap(ctx); + + if (!ShardIterator.IsValid()) { + return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::ERROR, + TStringBuilder() << "Provided shard iterator is malformed or expired", ctx); + } + + Limit = Limit == 0 ? MAX_LIMIT : Limit; + if (Limit < 1 || Limit > MAX_LIMIT) { + return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::ERROR, + TStringBuilder() << "Limit '" << Limit << "' is out of bounds [1; " << MAX_LIMIT << "]", ctx); + } + + SendDescribeProposeRequest(ctx); + Become(&TGetRecordsActor::StateWork); + } + + void TGetRecordsActor::SendReadRequest(const TActorContext& ctx) { + NTabletPipe::TClientConfig clientConfig; + clientConfig.RetryPolicy = { + .RetryLimitCount = 6, + .MinRetryTime = TDuration::MilliSeconds(10), + .MaxRetryTime = TDuration::MilliSeconds(100), + .BackoffMultiplier = 2, + .DoFirstRetryInstantly = true + }; + auto PipeClient = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, TabletId, clientConfig)); + + NKikimrClient::TPersQueueRequest request; + request.MutablePartitionRequest()->SetTopic(this->GetTopicPath(ctx)); + request.MutablePartitionRequest()->SetPartition(ShardIterator.GetShardId()); + ActorIdToProto(PipeClient, request.MutablePartitionRequest()->MutablePipeClient()); + + auto cmdRead = request.MutablePartitionRequest()->MutableCmdRead(); + cmdRead->SetClientId(NKikimr::NPQ::CLIENTID_TO_READ_INTERNALLY); + cmdRead->SetCount(Limit); + cmdRead->SetOffset(ShardIterator.GetSequenceNumber()); + cmdRead->SetReadTimestampMs(ShardIterator.GetReadTimestamp()); + cmdRead->SetTimeoutMs(READ_TIMEOUT_MS); + cmdRead->SetExternalOperation(true); + + TAutoPtr<TEvPersQueue::TEvRequest> req(new TEvPersQueue::TEvRequest); + req->Record.Swap(&request); + NTabletPipe::SendData(ctx, PipeClient, req.Release()); + } + + void TGetRecordsActor::StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) { + switch (ev->GetTypeRewrite()) { + HFunc(TEvPersQueue::TEvResponse, Handle); + default: TBase::StateWork(ev, ctx); + } + } + + void TGetRecordsActor::HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, + const TActorContext& ctx) { + const auto &result = ev->Get()->Request.Get(); + const auto response = result->ResultSet.front(); + + if (AppData(ctx)->PQConfig.GetRequireCredentialsInNewProtocol()) { + if (!response.SecurityObject->CheckAccess(NACLib::EAccessRights::SelectRow, + this->Request_->GetInternalToken())) { + return ReplyWithError(Ydb::StatusIds::UNAUTHORIZED, + Ydb::PersQueue::ErrorCode::ACCESS_DENIED, + TStringBuilder() << "Access to stream " + << ShardIterator.GetStreamName() + << " is denied for subject " + << this->Request_->GetInternalToken(), ctx); + } + } + + if (response.Self->Info.GetPathType() == NKikimrSchemeOp::EPathTypePersQueueGroup) { - const auto& partitions = response.PQGroupInfo->Description.GetPartitions(); - for (auto& partition : partitions) { - auto partitionId = partition.GetPartitionId(); - if (partitionId == ShardIterator.GetShardId()) { - TabletId = partition.GetTabletId(); - return SendReadRequest(ctx); - } - } - } - - ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::ERROR, - TStringBuilder() << "No such shard: " << ShardIterator.GetShardId(), ctx); - } - - void TGetRecordsActor::Handle(TEvPersQueue::TEvResponse::TPtr& ev, const TActorContext& ctx) { - const auto& record = ev->Get()->Record; - Y_VERIFY(ev->Get()->Record.HasPartitionResponse()); - - ui64 millisBehindLatestMs = 0; - std::vector<Ydb::DataStreams::V1::Record> records; - const auto& response = record.GetPartitionResponse(); - if (response.HasCmdReadResult()) { - const auto& results = response.GetCmdReadResult().GetResult(); - records.reserve(results.size()); - for (auto& r : results) { - auto proto(NKikimr::GetDeserializedData(r.GetData())); - Ydb::DataStreams::V1::Record record; - record.set_data(proto.GetData()); - record.set_timestamp(r.GetCreateTimestampMS()); - record.set_encryption(Ydb::DataStreams::V1::EncryptionType::NONE); - record.set_partition_key(r.GetPartitionKey()); - record.set_sequence_number(std::to_string(r.GetOffset()).c_str()); - records.push_back(record); - } - millisBehindLatestMs = records.size() > 0 ? TInstant::Now().MilliSeconds() - results.rbegin()->GetWriteTimestampMS() : 0; - } - - SendResponse(ctx, records, millisBehindLatestMs); - } - - void TGetRecordsActor::SendResponse(const TActorContext& ctx, - const std::vector<Ydb::DataStreams::V1::Record>& records, - ui64 millisBehindLatestMs) { - Ydb::DataStreams::V1::GetRecordsResult result; - for (auto& r : records) { - auto record = result.add_records(); - *record = r; - } - - auto timestamp = records.size() > 0 ? records.back().Gettimestamp() + 1 - : ShardIterator.GetReadTimestamp(); - auto seqNo = records.size() > 0 ? std::stoi(records.back().Getsequence_number()) + 1 - : ShardIterator.GetSequenceNumber(); - TShardIterator shardIterator(ShardIterator.GetStreamName(), - ShardIterator.GetStreamArn(), - ShardIterator.GetShardId(), - timestamp, seqNo); - result.set_next_shard_iterator(shardIterator.Serialize()); - result.set_millis_behind_latest(millisBehindLatestMs); - - Request_->SendResult(result, Ydb::StatusIds::SUCCESS); - Die(ctx); - } - - //----------------------------------------------------------------------------------------- - - class TListShardsActor : public TPQGrpcSchemaBase<TListShardsActor, NKikimr::NGRpcService::TEvDataStreamsListShardsRequest> { - using TBase = TPQGrpcSchemaBase<TListShardsActor, TEvDataStreamsListShardsRequest>; - - public: - TListShardsActor(NKikimr::NGRpcService::TEvDataStreamsListShardsRequest* request, NActors::TActorId newSchemeCache); - ~TListShardsActor() = default; - - void Bootstrap(const NActors::TActorContext& ctx); - void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx); - void Handle(TEvPersQueue::TEvOffsetsResponse::TPtr& ev, const TActorContext& ctx); - void Handle(TEvTabletPipe::TEvClientConnected::TPtr& ev, const TActorContext& ctx); - void Handle(TEvTabletPipe::TEvClientDestroyed::TPtr& ev, const TActorContext& ctx); - void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, - const TActorContext& ctx); - void Die(const TActorContext& ctx) override; - - private: - using TShardFilter = Ydb::DataStreams::V1::ShardFilter; - - void SendResponse(const TActorContext& ctx); - - static constexpr ui32 MAX_MAX_RESULTS = 10000; - static constexpr ui32 MIN_MAX_RESULTS = 1; - static constexpr ui32 DEFAULT_MAX_RESULTS = 100; - - TActorId NewSchemeCache; - TString StreamName; - TShardFilter ShardFilter; - TNextToken NextToken; - ui32 MaxResults = DEFAULT_MAX_RESULTS; - std::map<ui64, std::pair<ui64, ui64>> StartEndOffsetsPerPartition; + const auto& partitions = response.PQGroupInfo->Description.GetPartitions(); + for (auto& partition : partitions) { + auto partitionId = partition.GetPartitionId(); + if (partitionId == ShardIterator.GetShardId()) { + TabletId = partition.GetTabletId(); + return SendReadRequest(ctx); + } + } + } + + ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::ERROR, + TStringBuilder() << "No such shard: " << ShardIterator.GetShardId(), ctx); + } + + void TGetRecordsActor::Handle(TEvPersQueue::TEvResponse::TPtr& ev, const TActorContext& ctx) { + const auto& record = ev->Get()->Record; + Y_VERIFY(ev->Get()->Record.HasPartitionResponse()); + + ui64 millisBehindLatestMs = 0; + std::vector<Ydb::DataStreams::V1::Record> records; + const auto& response = record.GetPartitionResponse(); + if (response.HasCmdReadResult()) { + const auto& results = response.GetCmdReadResult().GetResult(); + records.reserve(results.size()); + for (auto& r : results) { + auto proto(NKikimr::GetDeserializedData(r.GetData())); + Ydb::DataStreams::V1::Record record; + record.set_data(proto.GetData()); + record.set_timestamp(r.GetCreateTimestampMS()); + record.set_encryption(Ydb::DataStreams::V1::EncryptionType::NONE); + record.set_partition_key(r.GetPartitionKey()); + record.set_sequence_number(std::to_string(r.GetOffset()).c_str()); + records.push_back(record); + } + millisBehindLatestMs = records.size() > 0 ? TInstant::Now().MilliSeconds() - results.rbegin()->GetWriteTimestampMS() : 0; + } + + SendResponse(ctx, records, millisBehindLatestMs); + } + + void TGetRecordsActor::SendResponse(const TActorContext& ctx, + const std::vector<Ydb::DataStreams::V1::Record>& records, + ui64 millisBehindLatestMs) { + Ydb::DataStreams::V1::GetRecordsResult result; + for (auto& r : records) { + auto record = result.add_records(); + *record = r; + } + + auto timestamp = records.size() > 0 ? records.back().Gettimestamp() + 1 + : ShardIterator.GetReadTimestamp(); + auto seqNo = records.size() > 0 ? std::stoi(records.back().Getsequence_number()) + 1 + : ShardIterator.GetSequenceNumber(); + TShardIterator shardIterator(ShardIterator.GetStreamName(), + ShardIterator.GetStreamArn(), + ShardIterator.GetShardId(), + timestamp, seqNo); + result.set_next_shard_iterator(shardIterator.Serialize()); + result.set_millis_behind_latest(millisBehindLatestMs); + + Request_->SendResult(result, Ydb::StatusIds::SUCCESS); + Die(ctx); + } + + //----------------------------------------------------------------------------------------- + + class TListShardsActor : public TPQGrpcSchemaBase<TListShardsActor, NKikimr::NGRpcService::TEvDataStreamsListShardsRequest> { + using TBase = TPQGrpcSchemaBase<TListShardsActor, TEvDataStreamsListShardsRequest>; + + public: + TListShardsActor(NKikimr::NGRpcService::TEvDataStreamsListShardsRequest* request, NActors::TActorId newSchemeCache); + ~TListShardsActor() = default; + + void Bootstrap(const NActors::TActorContext& ctx); + void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx); + void Handle(TEvPersQueue::TEvOffsetsResponse::TPtr& ev, const TActorContext& ctx); + void Handle(TEvTabletPipe::TEvClientConnected::TPtr& ev, const TActorContext& ctx); + void Handle(TEvTabletPipe::TEvClientDestroyed::TPtr& ev, const TActorContext& ctx); + void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, + const TActorContext& ctx); + void Die(const TActorContext& ctx) override; + + private: + using TShardFilter = Ydb::DataStreams::V1::ShardFilter; + + void SendResponse(const TActorContext& ctx); + + static constexpr ui32 MAX_MAX_RESULTS = 10000; + static constexpr ui32 MIN_MAX_RESULTS = 1; + static constexpr ui32 DEFAULT_MAX_RESULTS = 100; + + TActorId NewSchemeCache; + TString StreamName; + TShardFilter ShardFilter; + TNextToken NextToken; + ui32 MaxResults = DEFAULT_MAX_RESULTS; + std::map<ui64, std::pair<ui64, ui64>> StartEndOffsetsPerPartition; std::vector<NKikimrSchemeOp::TPersQueueGroupDescription::TPartition> Shards; ui32 LeftToRead = 0; ui32 AllShardsCount = 0; - std::atomic<ui32> GotOffsetResponds; - std::vector<TActorId> Pipes; - }; - - TListShardsActor::TListShardsActor(NKikimr::NGRpcService::TEvDataStreamsListShardsRequest* request, NActors::TActorId newSchemeCache) - : TBase(request, request->GetProtoRequest()->stream_name()) - , NewSchemeCache(std::move(newSchemeCache)) - , StreamName{request->GetProtoRequest()->stream_name()} - , ShardFilter{request->GetProtoRequest()->shard_filter()} - , NextToken{request->GetProtoRequest()->next_token()} - , GotOffsetResponds{0} - { - if (request->GetProtoRequest()->next_token().empty()) { - StreamName = request->GetProtoRequest()->stream_name(); - MaxResults = request->GetProtoRequest()->max_results(); - NextToken = TNextToken(StreamName, 0, MaxResults, TInstant::Now().MilliSeconds()); - } else { - StreamName = NextToken.GetStreamArn(); - MaxResults = NextToken.GetMaxResults(); - } - } - - void TListShardsActor::Bootstrap(const NActors::TActorContext& ctx) { - TBase::Bootstrap(ctx); - - if (!TShardFilter::ShardFilterType_IsValid(ShardFilter.type())) { - return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::BAD_REQUEST, - TStringBuilder() << "Shard filter '" << - (ui32)ShardFilter.type() << "' is not known", ctx); - } - - MaxResults = MaxResults == 0 ? DEFAULT_MAX_RESULTS : MaxResults; + std::atomic<ui32> GotOffsetResponds; + std::vector<TActorId> Pipes; + }; + + TListShardsActor::TListShardsActor(NKikimr::NGRpcService::TEvDataStreamsListShardsRequest* request, NActors::TActorId newSchemeCache) + : TBase(request, request->GetProtoRequest()->stream_name()) + , NewSchemeCache(std::move(newSchemeCache)) + , StreamName{request->GetProtoRequest()->stream_name()} + , ShardFilter{request->GetProtoRequest()->shard_filter()} + , NextToken{request->GetProtoRequest()->next_token()} + , GotOffsetResponds{0} + { + if (request->GetProtoRequest()->next_token().empty()) { + StreamName = request->GetProtoRequest()->stream_name(); + MaxResults = request->GetProtoRequest()->max_results(); + NextToken = TNextToken(StreamName, 0, MaxResults, TInstant::Now().MilliSeconds()); + } else { + StreamName = NextToken.GetStreamArn(); + MaxResults = NextToken.GetMaxResults(); + } + } + + void TListShardsActor::Bootstrap(const NActors::TActorContext& ctx) { + TBase::Bootstrap(ctx); + + if (!TShardFilter::ShardFilterType_IsValid(ShardFilter.type())) { + return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::BAD_REQUEST, + TStringBuilder() << "Shard filter '" << + (ui32)ShardFilter.type() << "' is not known", ctx); + } + + MaxResults = MaxResults == 0 ? DEFAULT_MAX_RESULTS : MaxResults; if (MaxResults > MAX_MAX_RESULTS) { - return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::BAD_REQUEST, - TStringBuilder() << "Max results '" << MaxResults << - "' is out of bound [" << MIN_MAX_RESULTS << "; " << - MAX_MAX_RESULTS << "]", ctx); - } - - if (ShardFilter.type() == TShardFilter::AFTER_SHARD_ID && ShardFilter.shard_id() == "") { - return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::BAD_REQUEST, - TStringBuilder() << "Shard filter type is AFTER_SHARD_ID," << - " but no ShardId provided", ctx); - } - - SendDescribeProposeRequest(ctx); - Become(&TListShardsActor::StateWork); - } - - void TListShardsActor::StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) { - switch (ev->GetTypeRewrite()) { - HFunc(TEvPersQueue::TEvOffsetsResponse, Handle); - HFunc(TEvTabletPipe::TEvClientDestroyed, Handle); - HFunc(TEvTabletPipe::TEvClientConnected, Handle); - default: TBase::StateWork(ev, ctx); - } - } - - void TListShardsActor::HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) { - if (ReplyIfNotTopic(ev, ctx)) { - return; - } - - const NSchemeCache::TSchemeCacheNavigate* navigate = ev->Get()->Request.Get(); - auto topicInfo = navigate->ResultSet.front(); - if (AppData(ctx)->PQConfig.GetRequireCredentialsInNewProtocol()) { - if (!topicInfo.SecurityObject->CheckAccess(NACLib::EAccessRights::SelectRow, - this->Request_->GetInternalToken())) { - return this->ReplyWithError(Ydb::StatusIds::UNAUTHORIZED, - Ydb::PersQueue::ErrorCode::ACCESS_DENIED, - TStringBuilder() << "Access to stream " - << this->GetProtoRequest()->stream_name() - << " is denied for subject " - << this->Request_->GetInternalToken(), ctx); - } - } - + return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::BAD_REQUEST, + TStringBuilder() << "Max results '" << MaxResults << + "' is out of bound [" << MIN_MAX_RESULTS << "; " << + MAX_MAX_RESULTS << "]", ctx); + } + + if (ShardFilter.type() == TShardFilter::AFTER_SHARD_ID && ShardFilter.shard_id() == "") { + return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::BAD_REQUEST, + TStringBuilder() << "Shard filter type is AFTER_SHARD_ID," << + " but no ShardId provided", ctx); + } + + SendDescribeProposeRequest(ctx); + Become(&TListShardsActor::StateWork); + } + + void TListShardsActor::StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) { + switch (ev->GetTypeRewrite()) { + HFunc(TEvPersQueue::TEvOffsetsResponse, Handle); + HFunc(TEvTabletPipe::TEvClientDestroyed, Handle); + HFunc(TEvTabletPipe::TEvClientConnected, Handle); + default: TBase::StateWork(ev, ctx); + } + } + + void TListShardsActor::HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) { + if (ReplyIfNotTopic(ev, ctx)) { + return; + } + + const NSchemeCache::TSchemeCacheNavigate* navigate = ev->Get()->Request.Get(); + auto topicInfo = navigate->ResultSet.front(); + if (AppData(ctx)->PQConfig.GetRequireCredentialsInNewProtocol()) { + if (!topicInfo.SecurityObject->CheckAccess(NACLib::EAccessRights::SelectRow, + this->Request_->GetInternalToken())) { + return this->ReplyWithError(Ydb::StatusIds::UNAUTHORIZED, + Ydb::PersQueue::ErrorCode::ACCESS_DENIED, + TStringBuilder() << "Access to stream " + << this->GetProtoRequest()->stream_name() + << " is denied for subject " + << this->Request_->GetInternalToken(), ctx); + } + } + using TPartition = NKikimrSchemeOp::TPersQueueGroupDescription::TPartition; - const auto& partitions = topicInfo.PQGroupInfo->Description.GetPartitions(); - TString startingShardId = this->GetProtoRequest()->Getexclusive_start_shard_id(); - ui64 startingTimepoint{0}; - bool onlyOpenShards{true}; - - std::map<TShardFilter::ShardFilterType, std::function<bool(const TPartition&)>> filters = { - {TShardFilter::SHARD_TYPE_UNDEFINED, [&](const TPartition& p) { - onlyOpenShards = false; - return GetShardName(p.GetPartitionId()) >= startingShardId; - }}, - {TShardFilter::AFTER_SHARD_ID, [&](const TPartition& p) { - startingShardId = ShardFilter.shard_id(); - startingTimepoint = 0; - onlyOpenShards = false; - return GetShardName(p.GetPartitionId()) > startingShardId; - }}, - {TShardFilter::AT_TRIM_HORIZON, [&](const TPartition& p) { - startingShardId = "0"; - startingTimepoint = 0; - onlyOpenShards = true; - return GetShardName(p.GetPartitionId()) >= startingShardId; - }}, - { TShardFilter::FROM_TRIM_HORIZON, [&](const TPartition& p) { - startingShardId = "0"; - startingTimepoint = 0; - onlyOpenShards = false; - return GetShardName(p.GetPartitionId()) >= startingShardId; - }}, - {TShardFilter::AT_LATEST, [&](const TPartition& p) { - startingTimepoint = TInstant::Now().MilliSeconds(); - startingShardId = "0"; - onlyOpenShards = true; - return GetShardName(p.GetPartitionId()) >= startingShardId; - }}, - {TShardFilter::AT_TIMESTAMP, [&](const TPartition& p) { - startingTimepoint = ShardFilter.timestamp(); - startingShardId = "0"; - onlyOpenShards = true; - return GetShardName(p.GetPartitionId()) >= startingShardId; - }}, - {TShardFilter::FROM_TIMESTAMP, [&](const TPartition& p) { - startingTimepoint = ShardFilter.timestamp(); - startingShardId = "0"; - onlyOpenShards = false; - return GetShardName(p.GetPartitionId()) >= startingShardId; - }} - }; - - const auto alreadyRead = NextToken.GetAlreadyRead(); - if (alreadyRead > (ui32)partitions.size()) { - return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::ERROR, - TStringBuilder() << "Provided next_token is malformed - " - "everything is already read", ctx); - } - - const auto shardsToGet = std::min(partitions.size() - alreadyRead, MaxResults); - i32 lastCopied{0}; - Shards.reserve(shardsToGet); - AllShardsCount = partitions.size(); - for (auto partition = partitions.begin() + alreadyRead; partition != partitions.end(); ++partition) { - if (Shards.size() == shardsToGet) { - break; - } - if (filters[ShardFilter.type()](*partition)) { - Shards.push_back(*partition); - } - ++lastCopied; - } - auto actuallyRead = lastCopied - alreadyRead; - LeftToRead = partitions.size() - alreadyRead - actuallyRead; - - if (Shards.size() == 0) { - return SendResponse(ctx); - } - - // Send OffsetRequests - std::set<ui64> tabletIds; - for (const auto& shard : Shards) { - tabletIds.insert(shard.GetTabletId()); - } - - NTabletPipe::TClientConfig clientConfig; - clientConfig.RetryPolicy = { - .RetryLimitCount = 6, - .MinRetryTime = TDuration::MilliSeconds(10), - .MaxRetryTime = TDuration::MilliSeconds(100), - .BackoffMultiplier = 2, - .DoFirstRetryInstantly = true - }; - - for (auto& tabletId : tabletIds) { - Pipes.push_back(ctx.Register(NTabletPipe::CreateClient(ctx.SelfID, tabletId, clientConfig))); - TAutoPtr<TEvPersQueue::TEvOffsets> req(new TEvPersQueue::TEvOffsets); - NTabletPipe::SendData(ctx, Pipes.back(), req.Release()); - } - } - - void TListShardsActor::Handle(TEvPersQueue::TEvOffsetsResponse::TPtr& ev, const TActorContext& ctx) { - for (auto& part : ev->Get()->Record.GetPartResult()) { - StartEndOffsetsPerPartition[part.GetPartition()] = - std::make_pair<ui64, ui64>(part.GetStartOffset(), part.GetEndOffset()); - ++GotOffsetResponds; - } - if (GotOffsetResponds == Shards.size()) { - SendResponse(ctx); - } - } - - void TListShardsActor::Handle(TEvTabletPipe::TEvClientConnected::TPtr& ev, const TActorContext& ctx) { - if (ev->Get()->Status != NKikimrProto::EReplyStatus::OK) { - ReplyWithError(Ydb::StatusIds::SCHEME_ERROR, Ydb::PersQueue::ErrorCode::ERROR, - TStringBuilder() << "Cannot connect to tablet " << ev->Get()->TabletId, ctx); - } - } - - void TListShardsActor::Handle(TEvTabletPipe::TEvClientDestroyed::TPtr& ev, const TActorContext& ctx) { - ReplyWithError(Ydb::StatusIds::SCHEME_ERROR, Ydb::PersQueue::ErrorCode::ERROR, - TStringBuilder() << "Cannot connect to tablet " << ev->Get()->TabletId, ctx); - } - - void TListShardsActor::SendResponse(const TActorContext& ctx) { - Ydb::DataStreams::V1::ListShardsResult result; - for (auto& shard : Shards) { - auto awsShard = result.Addshards(); - // TODO: - // awsShard->set_parent_shard_id(""); - // awsShard->set_adjacent_parent_shard_id(prevShardName); - auto range = RangeFromShardNumber(shard.GetPartitionId(), AllShardsCount); - awsShard->mutable_hash_key_range()->set_starting_hash_key( - Uint128ToDecimalString(range.Start)); - awsShard->mutable_hash_key_range()->set_ending_hash_key( - Uint128ToDecimalString(range.End)); - awsShard->mutable_sequence_number_range()->set_starting_sequence_number( - std::to_string(StartEndOffsetsPerPartition[shard.GetPartitionId()].first)); - awsShard->mutable_sequence_number_range()->set_ending_sequence_number( - std::to_string(StartEndOffsetsPerPartition[shard.GetPartitionId()].second)); - awsShard->set_shard_id(GetShardName(shard.GetPartitionId())); - } - if (LeftToRead > 0) { - TNextToken token(StreamName, NextToken.GetAlreadyRead() + Shards.size(), MaxResults, TInstant::Now().MilliSeconds()); - result.set_next_token(token.Serialize()); - } - Request_->SendResult(result, Ydb::StatusIds::SUCCESS); - Die(ctx); - } - - void TListShardsActor::Die(const TActorContext& ctx) { - //close all pipes - for (auto& pipe : Pipes) { - NTabletPipe::CloseClient(ctx, pipe); - } - TBase::Die(ctx); - } - - //----------------------------------------------------------------------------------- - - class TDescribeStreamSummaryActor : public TPQGrpcSchemaBase<TDescribeStreamSummaryActor, TEvDataStreamsDescribeStreamSummaryRequest> { - using TBase = TPQGrpcSchemaBase<TDescribeStreamSummaryActor, TEvDataStreamsDescribeStreamSummaryRequest>; - - public: - TDescribeStreamSummaryActor(NKikimr::NGRpcService::TEvDataStreamsDescribeStreamSummaryRequest* request); - ~TDescribeStreamSummaryActor() = default; - - void Bootstrap(const NActors::TActorContext& ctx); - - void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx); - void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, - const TActorContext& ctx); - - private: - void SendResponse(const TActorContext& ctx); - + const auto& partitions = topicInfo.PQGroupInfo->Description.GetPartitions(); + TString startingShardId = this->GetProtoRequest()->Getexclusive_start_shard_id(); + ui64 startingTimepoint{0}; + bool onlyOpenShards{true}; + + std::map<TShardFilter::ShardFilterType, std::function<bool(const TPartition&)>> filters = { + {TShardFilter::SHARD_TYPE_UNDEFINED, [&](const TPartition& p) { + onlyOpenShards = false; + return GetShardName(p.GetPartitionId()) >= startingShardId; + }}, + {TShardFilter::AFTER_SHARD_ID, [&](const TPartition& p) { + startingShardId = ShardFilter.shard_id(); + startingTimepoint = 0; + onlyOpenShards = false; + return GetShardName(p.GetPartitionId()) > startingShardId; + }}, + {TShardFilter::AT_TRIM_HORIZON, [&](const TPartition& p) { + startingShardId = "0"; + startingTimepoint = 0; + onlyOpenShards = true; + return GetShardName(p.GetPartitionId()) >= startingShardId; + }}, + { TShardFilter::FROM_TRIM_HORIZON, [&](const TPartition& p) { + startingShardId = "0"; + startingTimepoint = 0; + onlyOpenShards = false; + return GetShardName(p.GetPartitionId()) >= startingShardId; + }}, + {TShardFilter::AT_LATEST, [&](const TPartition& p) { + startingTimepoint = TInstant::Now().MilliSeconds(); + startingShardId = "0"; + onlyOpenShards = true; + return GetShardName(p.GetPartitionId()) >= startingShardId; + }}, + {TShardFilter::AT_TIMESTAMP, [&](const TPartition& p) { + startingTimepoint = ShardFilter.timestamp(); + startingShardId = "0"; + onlyOpenShards = true; + return GetShardName(p.GetPartitionId()) >= startingShardId; + }}, + {TShardFilter::FROM_TIMESTAMP, [&](const TPartition& p) { + startingTimepoint = ShardFilter.timestamp(); + startingShardId = "0"; + onlyOpenShards = false; + return GetShardName(p.GetPartitionId()) >= startingShardId; + }} + }; + + const auto alreadyRead = NextToken.GetAlreadyRead(); + if (alreadyRead > (ui32)partitions.size()) { + return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::ERROR, + TStringBuilder() << "Provided next_token is malformed - " + "everything is already read", ctx); + } + + const auto shardsToGet = std::min(partitions.size() - alreadyRead, MaxResults); + i32 lastCopied{0}; + Shards.reserve(shardsToGet); + AllShardsCount = partitions.size(); + for (auto partition = partitions.begin() + alreadyRead; partition != partitions.end(); ++partition) { + if (Shards.size() == shardsToGet) { + break; + } + if (filters[ShardFilter.type()](*partition)) { + Shards.push_back(*partition); + } + ++lastCopied; + } + auto actuallyRead = lastCopied - alreadyRead; + LeftToRead = partitions.size() - alreadyRead - actuallyRead; + + if (Shards.size() == 0) { + return SendResponse(ctx); + } + + // Send OffsetRequests + std::set<ui64> tabletIds; + for (const auto& shard : Shards) { + tabletIds.insert(shard.GetTabletId()); + } + + NTabletPipe::TClientConfig clientConfig; + clientConfig.RetryPolicy = { + .RetryLimitCount = 6, + .MinRetryTime = TDuration::MilliSeconds(10), + .MaxRetryTime = TDuration::MilliSeconds(100), + .BackoffMultiplier = 2, + .DoFirstRetryInstantly = true + }; + + for (auto& tabletId : tabletIds) { + Pipes.push_back(ctx.Register(NTabletPipe::CreateClient(ctx.SelfID, tabletId, clientConfig))); + TAutoPtr<TEvPersQueue::TEvOffsets> req(new TEvPersQueue::TEvOffsets); + NTabletPipe::SendData(ctx, Pipes.back(), req.Release()); + } + } + + void TListShardsActor::Handle(TEvPersQueue::TEvOffsetsResponse::TPtr& ev, const TActorContext& ctx) { + for (auto& part : ev->Get()->Record.GetPartResult()) { + StartEndOffsetsPerPartition[part.GetPartition()] = + std::make_pair<ui64, ui64>(part.GetStartOffset(), part.GetEndOffset()); + ++GotOffsetResponds; + } + if (GotOffsetResponds == Shards.size()) { + SendResponse(ctx); + } + } + + void TListShardsActor::Handle(TEvTabletPipe::TEvClientConnected::TPtr& ev, const TActorContext& ctx) { + if (ev->Get()->Status != NKikimrProto::EReplyStatus::OK) { + ReplyWithError(Ydb::StatusIds::SCHEME_ERROR, Ydb::PersQueue::ErrorCode::ERROR, + TStringBuilder() << "Cannot connect to tablet " << ev->Get()->TabletId, ctx); + } + } + + void TListShardsActor::Handle(TEvTabletPipe::TEvClientDestroyed::TPtr& ev, const TActorContext& ctx) { + ReplyWithError(Ydb::StatusIds::SCHEME_ERROR, Ydb::PersQueue::ErrorCode::ERROR, + TStringBuilder() << "Cannot connect to tablet " << ev->Get()->TabletId, ctx); + } + + void TListShardsActor::SendResponse(const TActorContext& ctx) { + Ydb::DataStreams::V1::ListShardsResult result; + for (auto& shard : Shards) { + auto awsShard = result.Addshards(); + // TODO: + // awsShard->set_parent_shard_id(""); + // awsShard->set_adjacent_parent_shard_id(prevShardName); + auto range = RangeFromShardNumber(shard.GetPartitionId(), AllShardsCount); + awsShard->mutable_hash_key_range()->set_starting_hash_key( + Uint128ToDecimalString(range.Start)); + awsShard->mutable_hash_key_range()->set_ending_hash_key( + Uint128ToDecimalString(range.End)); + awsShard->mutable_sequence_number_range()->set_starting_sequence_number( + std::to_string(StartEndOffsetsPerPartition[shard.GetPartitionId()].first)); + awsShard->mutable_sequence_number_range()->set_ending_sequence_number( + std::to_string(StartEndOffsetsPerPartition[shard.GetPartitionId()].second)); + awsShard->set_shard_id(GetShardName(shard.GetPartitionId())); + } + if (LeftToRead > 0) { + TNextToken token(StreamName, NextToken.GetAlreadyRead() + Shards.size(), MaxResults, TInstant::Now().MilliSeconds()); + result.set_next_token(token.Serialize()); + } + Request_->SendResult(result, Ydb::StatusIds::SUCCESS); + Die(ctx); + } + + void TListShardsActor::Die(const TActorContext& ctx) { + //close all pipes + for (auto& pipe : Pipes) { + NTabletPipe::CloseClient(ctx, pipe); + } + TBase::Die(ctx); + } + + //----------------------------------------------------------------------------------- + + class TDescribeStreamSummaryActor : public TPQGrpcSchemaBase<TDescribeStreamSummaryActor, TEvDataStreamsDescribeStreamSummaryRequest> { + using TBase = TPQGrpcSchemaBase<TDescribeStreamSummaryActor, TEvDataStreamsDescribeStreamSummaryRequest>; + + public: + TDescribeStreamSummaryActor(NKikimr::NGRpcService::TEvDataStreamsDescribeStreamSummaryRequest* request); + ~TDescribeStreamSummaryActor() = default; + + void Bootstrap(const NActors::TActorContext& ctx); + + void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx); + void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, + const TActorContext& ctx); + + private: + void SendResponse(const TActorContext& ctx); + NKikimrSchemeOp::TDirEntry SelfInfo; NKikimrSchemeOp::TPersQueueGroupDescription PQGroup; - }; - - TDescribeStreamSummaryActor::TDescribeStreamSummaryActor( - NKikimr::NGRpcService::TEvDataStreamsDescribeStreamSummaryRequest* request - ) - : TBase(request, request->GetProtoRequest()->stream_name()) - { - } - - void TDescribeStreamSummaryActor::Bootstrap(const NActors::TActorContext& ctx) { - TBase::Bootstrap(ctx); - SendDescribeProposeRequest(ctx); - Become(&TDescribeStreamSummaryActor::StateWork); - } - - void TDescribeStreamSummaryActor::StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) { - switch (ev->GetTypeRewrite()) { - default: TBase::StateWork(ev, ctx); - } - } - - void TDescribeStreamSummaryActor::HandleCacheNavigateResponse( - TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx - ) { - if (ReplyIfNotTopic(ev, ctx)) { - return; - } - - const NSchemeCache::TSchemeCacheNavigate* result = ev->Get()->Request.Get(); - Y_VERIFY(result->ResultSet.size() == 1); // describe only one topic - const auto& response = result->ResultSet.front(); - Y_VERIFY(response.PQGroupInfo); - const TString path = JoinSeq("/", response.Path); - - PQGroup = response.PQGroupInfo->Description; - SelfInfo = response.Self->Info; - - SendResponse(ctx); - } - - void TDescribeStreamSummaryActor::SendResponse(const TActorContext& ctx) { - Ydb::DataStreams::V1::DescribeStreamSummaryResult result; - - auto& pqConfig = PQGroup.GetPQTabletConfig(); - auto& descriptionSummary = *result.mutable_stream_description_summary(); - - descriptionSummary.set_stream_name(GetProtoRequest()->stream_name()); - descriptionSummary.set_stream_arn(GetProtoRequest()->stream_name()); - descriptionSummary.set_key_id(""); - descriptionSummary.set_retention_period_hours( - TInstant::Seconds(pqConfig.GetPartitionConfig().GetLifetimeSeconds()).Hours() - ); - descriptionSummary.set_stream_creation_timestamp( - TInstant::MilliSeconds(SelfInfo.GetCreateStep()).Seconds() - ); - descriptionSummary.set_stream_status( - SelfInfo.GetCreateFinished() ? Ydb::DataStreams::V1::StreamDescription::ACTIVE - : Ydb::DataStreams::V1::StreamDescription::CREATING - ); - descriptionSummary.set_open_shard_count(PQGroup.GetPartitions().size()); - descriptionSummary.set_consumer_count(PQGroup.MutablePQTabletConfig()->GetReadRules().size()); - descriptionSummary.set_encryption_type(Ydb::DataStreams::V1::EncryptionType::NONE); - - Request_->SendResult(result, Ydb::StatusIds::SUCCESS); - Die(ctx); - } - - //----------------------------------------------------------------------------------------- - + }; + + TDescribeStreamSummaryActor::TDescribeStreamSummaryActor( + NKikimr::NGRpcService::TEvDataStreamsDescribeStreamSummaryRequest* request + ) + : TBase(request, request->GetProtoRequest()->stream_name()) + { + } + + void TDescribeStreamSummaryActor::Bootstrap(const NActors::TActorContext& ctx) { + TBase::Bootstrap(ctx); + SendDescribeProposeRequest(ctx); + Become(&TDescribeStreamSummaryActor::StateWork); + } + + void TDescribeStreamSummaryActor::StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) { + switch (ev->GetTypeRewrite()) { + default: TBase::StateWork(ev, ctx); + } + } + + void TDescribeStreamSummaryActor::HandleCacheNavigateResponse( + TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx + ) { + if (ReplyIfNotTopic(ev, ctx)) { + return; + } + + const NSchemeCache::TSchemeCacheNavigate* result = ev->Get()->Request.Get(); + Y_VERIFY(result->ResultSet.size() == 1); // describe only one topic + const auto& response = result->ResultSet.front(); + Y_VERIFY(response.PQGroupInfo); + const TString path = JoinSeq("/", response.Path); + + PQGroup = response.PQGroupInfo->Description; + SelfInfo = response.Self->Info; + + SendResponse(ctx); + } + + void TDescribeStreamSummaryActor::SendResponse(const TActorContext& ctx) { + Ydb::DataStreams::V1::DescribeStreamSummaryResult result; + + auto& pqConfig = PQGroup.GetPQTabletConfig(); + auto& descriptionSummary = *result.mutable_stream_description_summary(); + + descriptionSummary.set_stream_name(GetProtoRequest()->stream_name()); + descriptionSummary.set_stream_arn(GetProtoRequest()->stream_name()); + descriptionSummary.set_key_id(""); + descriptionSummary.set_retention_period_hours( + TInstant::Seconds(pqConfig.GetPartitionConfig().GetLifetimeSeconds()).Hours() + ); + descriptionSummary.set_stream_creation_timestamp( + TInstant::MilliSeconds(SelfInfo.GetCreateStep()).Seconds() + ); + descriptionSummary.set_stream_status( + SelfInfo.GetCreateFinished() ? Ydb::DataStreams::V1::StreamDescription::ACTIVE + : Ydb::DataStreams::V1::StreamDescription::CREATING + ); + descriptionSummary.set_open_shard_count(PQGroup.GetPartitions().size()); + descriptionSummary.set_consumer_count(PQGroup.MutablePQTabletConfig()->GetReadRules().size()); + descriptionSummary.set_encryption_type(Ydb::DataStreams::V1::EncryptionType::NONE); + + Request_->SendResult(result, Ydb::StatusIds::SUCCESS); + Die(ctx); + } + + //----------------------------------------------------------------------------------------- + template<class TEvRequest> class TNotImplementedRequestActor : public TRpcSchemeRequestActor<TNotImplementedRequestActor<TEvRequest>, TEvRequest> { using TBase = TRpcSchemeRequestActor<TNotImplementedRequestActor, TEvRequest>; @@ -1676,7 +1676,7 @@ namespace NKikimr::NDataStreams::V1 { } }; - //----------------------------------------------------------------------------------- + //----------------------------------------------------------------------------------- IActor* CreateDataStreamsService(TIntrusivePtr<NMonitoring::TDynamicCounters> counters, TActorId newSchemeCache) { return new TDataStreamsService(counters, newSchemeCache); @@ -1705,11 +1705,11 @@ namespace NKikimr::NDataStreams::V1 { } void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsRegisterStreamConsumerRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Register(new TRegisterStreamConsumerActor(ev->Release().Release())); + ctx.Register(new TRegisterStreamConsumerActor(ev->Release().Release())); } void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsDeregisterStreamConsumerRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Register(new TDeregisterStreamConsumerActor(ev->Release().Release())); + ctx.Register(new TDeregisterStreamConsumerActor(ev->Release().Release())); } void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsDescribeStreamConsumerRequest::TPtr& ev, const TActorContext& ctx) { @@ -1725,7 +1725,7 @@ namespace NKikimr::NDataStreams::V1 { } void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsListShardsRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Register(new TListShardsActor(ev->Release().Release(), NewSchemeCache)); + ctx.Register(new TListShardsActor(ev->Release().Release(), NewSchemeCache)); } void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsPutRecordsRequest::TPtr& ev, const TActorContext& ctx) { @@ -1733,11 +1733,11 @@ namespace NKikimr::NDataStreams::V1 { } void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsGetRecordsRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Register(new TGetRecordsActor(ev->Release().Release(), NewSchemeCache)); + ctx.Register(new TGetRecordsActor(ev->Release().Release(), NewSchemeCache)); } void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsGetShardIteratorRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Register(new TGetShardIteratorActor(ev->Release().Release(), NewSchemeCache)); + ctx.Register(new TGetShardIteratorActor(ev->Release().Release(), NewSchemeCache)); } void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsSubscribeToShardRequest::TPtr& ev, const TActorContext& ctx) { @@ -1749,7 +1749,7 @@ namespace NKikimr::NDataStreams::V1 { } void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsDescribeStreamSummaryRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Register(new TDescribeStreamSummaryActor(ev->Release().Release())); + ctx.Register(new TDescribeStreamSummaryActor(ev->Release().Release())); } void TDataStreamsService::Handle(TEvDataStreamsDecreaseStreamRetentionPeriodRequest::TPtr& ev, const TActorContext& ctx) { @@ -1769,7 +1769,7 @@ namespace NKikimr::NDataStreams::V1 { } void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsListStreamConsumersRequest::TPtr& ev, const TActorContext& ctx) { - ctx.Register(new TListStreamConsumersActor(ev->Release().Release())); + ctx.Register(new TListStreamConsumersActor(ev->Release().Release())); } void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsAddTagsToStreamRequest::TPtr& ev, const TActorContext& ctx) { diff --git a/ydb/services/datastreams/datastreams_ut.cpp b/ydb/services/datastreams/datastreams_ut.cpp index 2ede2a2d05..9ab384b916 100644 --- a/ydb/services/datastreams/datastreams_ut.cpp +++ b/ydb/services/datastreams/datastreams_ut.cpp @@ -13,15 +13,15 @@ #include <library/cpp/json/json_reader.h> #include <library/cpp/digest/md5/md5.h> -#include <random> - - +#include <random> + + using namespace NYdb; using namespace NYdb::NTable; using namespace NKikimr::NPersQueueTests; using namespace NKikimr::NDataStreams::V1; -namespace YDS_V1 = Ydb::DataStreams::V1; -namespace NYDS_V1 = NYdb::NDataStreams::V1; +namespace YDS_V1 = Ydb::DataStreams::V1; +namespace NYDS_V1 = NYdb::NDataStreams::V1; struct WithSslAndAuth : TKikimrTestSettings { static constexpr bool SSL = true; static constexpr bool AUTH = true; @@ -61,7 +61,7 @@ public: } Driver = std::make_unique<TDriver>(std::move(driverConfig)); - DataStreamsClient = std::make_unique<NYDS_V1::TDataStreamsClient>(*Driver, + DataStreamsClient = std::make_unique<NYDS_V1::TDataStreamsClient>(*Driver, TCommonClientSettings() .AuthToken("user@builtin")); @@ -70,7 +70,7 @@ public: NYdb::NScheme::TPermissions permissions("user@builtin", {"ydb.generic.read", "ydb.generic.write"}); auto result = schemeClient.ModifyPermissions("/Root", - NYdb::NScheme::TModifyPermissionsSettings().AddGrantPermissions(permissions) + NYdb::NScheme::TModifyPermissionsSettings().AddGrantPermissions(permissions) ).ExtractValueSync(); Cerr << result.GetIssues().ToString() << "\n"; UNIT_ASSERT(result.IsSuccess()); @@ -84,8 +84,8 @@ public: public: std::unique_ptr<TKikimr> KikimrServer; std::unique_ptr<TDriver> Driver; - std::unique_ptr<NYDS_V1::TDataStreamsClient> DataStreamsClient; - std::unique_ptr<NYDS_V1::TDataStreamsClient> UnauthenticatedClient; + std::unique_ptr<NYDS_V1::TDataStreamsClient> DataStreamsClient; + std::unique_ptr<NYDS_V1::TDataStreamsClient> UnauthenticatedClient; THolder<TTempFileHandle> MeteringFile; }; @@ -116,7 +116,7 @@ void CheckMeteringFile(TTempFileHandle* meteringFile, const TString& streamPath) UNIT_ASSERT(map.contains("source_wt")); UNIT_ASSERT(map.find("cloud_id")->second.GetString() == "somecloud"); UNIT_ASSERT(map.find("folder_id")->second.GetString() == "somefolder"); - UNIT_ASSERT(map.find("resource_id")->second.GetString() == streamPath); + UNIT_ASSERT(map.find("resource_id")->second.GetString() == streamPath); auto& tags = map.find("tags")->second.GetMap(); if (!tags.empty()) { UNIT_ASSERT_VALUES_EQUAL(tags.size(), 3); @@ -129,23 +129,23 @@ void CheckMeteringFile(TTempFileHandle* meteringFile, const TString& streamPath) } -#define Y_UNIT_TEST_NAME this->Name_; - +#define Y_UNIT_TEST_NAME this->Name_; + Y_UNIT_TEST_SUITE(DataStreams) { Y_UNIT_TEST(TestControlPlaneAndMeteringData) { TInsecureDatastreamsTestServer testServer; - const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; + const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; // Trying to delete stream that doesn't exist yet { - auto result = testServer.DataStreamsClient->DeleteStream("testfolder/" + streamName).ExtractValueSync(); + auto result = testServer.DataStreamsClient->DeleteStream("testfolder/" + streamName).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR); } { - auto result = testServer.DataStreamsClient->CreateStream(streamName, - NYDS_V1::TCreateStreamSettings().ShardCount(3)).ExtractValueSync(); + auto result = testServer.DataStreamsClient->CreateStream(streamName, + NYDS_V1::TCreateStreamSettings().ShardCount(3)).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); if (result.GetStatus() != EStatus::SUCCESS) { result.GetIssues().PrintTo(Cerr); @@ -154,13 +154,13 @@ Y_UNIT_TEST_SUITE(DataStreams) { } { - auto result = testServer.DataStreamsClient->DescribeStream(streamName).ExtractValueSync(); + auto result = testServer.DataStreamsClient->DescribeStream(streamName).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_description().stream_status(), - YDS_V1::StreamDescription::ACTIVE); - UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_description().stream_name(), streamName); + UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_description().stream_status(), + YDS_V1::StreamDescription::ACTIVE); + UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_description().stream_name(), streamName); UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_description().write_quota_kb_per_sec(), 1024); UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_description().retention_period_hours(), 24); @@ -174,32 +174,32 @@ Y_UNIT_TEST_SUITE(DataStreams) { UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_description().shards(2).hash_key_range().ending_hash_key(), "340282366920938463463374607431768211455"); } - { - auto result = testServer.DataStreamsClient->DescribeStreamSummary(streamName).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - - UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_description_summary().stream_status(), - YDS_V1::StreamDescription::ACTIVE); - UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_description_summary().stream_name(), streamName); - UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_description_summary().stream_arn(), streamName); - UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_description_summary().retention_period_hours(), 24); - UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_description_summary().open_shard_count(), 3); - } + { + auto result = testServer.DataStreamsClient->DescribeStreamSummary(streamName).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_description_summary().stream_status(), + YDS_V1::StreamDescription::ACTIVE); + UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_description_summary().stream_name(), streamName); + UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_description_summary().stream_arn(), streamName); + UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_description_summary().retention_period_hours(), 24); + UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_description_summary().open_shard_count(), 3); + } + { - auto result = testServer.DataStreamsClient->CreateStream("testfolder/" + streamName).ExtractValueSync(); + auto result = testServer.DataStreamsClient->CreateStream("testfolder/" + streamName).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST); } { // for metering purposes - std::vector<NYDS_V1::TDataRecord> records; + std::vector<NYDS_V1::TDataRecord> records; for (ui32 i = 1; i <= 30; ++i) { TString data = Sprintf("%04u", i); records.push_back({data, data, ""}); } - auto result = testServer.DataStreamsClient->PutRecords(streamName, records).ExtractValueSync(); + auto result = testServer.DataStreamsClient->PutRecords(streamName, records).ExtractValueSync(); Cerr << result.GetResult().DebugString() << Endl; UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); @@ -210,20 +210,20 @@ Y_UNIT_TEST_SUITE(DataStreams) { UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_names().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_names(0), streamName); + UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_names(0), streamName); } // cannot decrease the number of shards { - auto result = testServer.DataStreamsClient->UpdateShardCount(streamName, - NYDS_V1::TUpdateShardCountSettings().TargetShardCount(2)).ExtractValueSync(); + auto result = testServer.DataStreamsClient->UpdateShardCount(streamName, + NYDS_V1::TUpdateShardCountSettings().TargetShardCount(2)).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST); } { - auto result = testServer.DataStreamsClient->UpdateShardCount(streamName, - NYDS_V1::TUpdateShardCountSettings().TargetShardCount(15)).ExtractValueSync(); + auto result = testServer.DataStreamsClient->UpdateShardCount(streamName, + NYDS_V1::TUpdateShardCountSettings().TargetShardCount(15)).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); } @@ -231,33 +231,33 @@ Y_UNIT_TEST_SUITE(DataStreams) { // now when stream is created delete should work fine { - auto result = testServer.DataStreamsClient->DeleteStream(streamName).ExtractValueSync(); + auto result = testServer.DataStreamsClient->DeleteStream(streamName).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); } // Describe should fail after delete { - auto result = testServer.DataStreamsClient->DescribeStream(streamName).ExtractValueSync(); + auto result = testServer.DataStreamsClient->DescribeStream(streamName).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR); } - CheckMeteringFile(testServer.MeteringFile.Get(), "/Root/" + streamName); + CheckMeteringFile(testServer.MeteringFile.Get(), "/Root/" + streamName); } Y_UNIT_TEST(TestCreateExistingStream) { TInsecureDatastreamsTestServer testServer; - const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; + const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; { - auto result = testServer.DataStreamsClient->CreateStream(streamName, - NYDS_V1::TCreateStreamSettings().ShardCount(10)).ExtractValueSync(); + auto result = testServer.DataStreamsClient->CreateStream(streamName, + NYDS_V1::TCreateStreamSettings().ShardCount(10)).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); } { - auto result = testServer.DataStreamsClient->CreateStream(streamName, - NYDS_V1::TCreateStreamSettings().ShardCount(10)).ExtractValueSync(); + auto result = testServer.DataStreamsClient->CreateStream(streamName, + NYDS_V1::TCreateStreamSettings().ShardCount(10)).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::ALREADY_EXISTS); } @@ -266,11 +266,11 @@ Y_UNIT_TEST_SUITE(DataStreams) { Y_UNIT_TEST(TestStreamPagination) { TInsecureDatastreamsTestServer testServer; - const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; + const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; for (ui32 folderIdx = 0; folderIdx < 4; folderIdx++) { for (ui32 streamIdx = 0; streamIdx < 5; streamIdx++) { - TStringBuilder streamNameX = TStringBuilder() << folderIdx << streamName << streamIdx; - auto result = testServer.DataStreamsClient->CreateStream(streamNameX, NYDS_V1::TCreateStreamSettings().ShardCount(10)).ExtractValueSync(); + TStringBuilder streamNameX = TStringBuilder() << folderIdx << streamName << streamIdx; + auto result = testServer.DataStreamsClient->CreateStream(streamNameX, NYDS_V1::TCreateStreamSettings().ShardCount(10)).ExtractValueSync(); Cerr << result.GetIssues().ToString() << "\n"; UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); @@ -280,7 +280,7 @@ Y_UNIT_TEST_SUITE(DataStreams) { TString startStream; THashSet<TString> streams; for (int i = 0; i < 3; i++) { - auto result = testServer.DataStreamsClient->ListStreams(NYDS_V1::TListStreamsSettings().Limit(6).ExclusiveStartStreamName(startStream)).ExtractValueSync(); + auto result = testServer.DataStreamsClient->ListStreams(NYDS_V1::TListStreamsSettings().Limit(6).ExclusiveStartStreamName(startStream)).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_names().size(), 6); @@ -290,7 +290,7 @@ Y_UNIT_TEST_SUITE(DataStreams) { } { - auto result = testServer.DataStreamsClient->ListStreams(NYDS_V1::TListStreamsSettings().Limit(6).ExclusiveStartStreamName(startStream)).ExtractValueSync(); + auto result = testServer.DataStreamsClient->ListStreams(NYDS_V1::TListStreamsSettings().Limit(6).ExclusiveStartStreamName(startStream)).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_names().size(), 2); @@ -301,114 +301,114 @@ Y_UNIT_TEST_SUITE(DataStreams) { UNIT_ASSERT_VALUES_EQUAL(streams.size(), 20); } - Y_UNIT_TEST(TestDeleteStream) { - TInsecureDatastreamsTestServer testServer; - const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; - { - auto result = testServer.DataStreamsClient->CreateStream(streamName, - NYDS_V1::TCreateStreamSettings().ShardCount(3)).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - { - auto result = testServer.DataStreamsClient->DeleteStream(streamName).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - } - - Y_UNIT_TEST(TestDeleteStreamWithEnforceFlag) { - TInsecureDatastreamsTestServer testServer; - const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; - { - auto result = testServer.DataStreamsClient->CreateStream(streamName, - NYDS_V1::TCreateStreamSettings().ShardCount(3)).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - { - auto result = testServer.DataStreamsClient->RegisterStreamConsumer(streamName, "user1", - NYDS_V1::TRegisterStreamConsumerSettings()).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - { - auto result = testServer.DataStreamsClient->DeleteStream(streamName).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST); - } - - { - auto result = testServer.DataStreamsClient->DeleteStream(streamName, - NYDS_V1::TDeleteStreamSettings().EnforceConsumerDeletion(true)).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - } - - - Y_UNIT_TEST(TestDeleteStreamWithEnforceFlagFalse) { - TInsecureDatastreamsTestServer testServer; - const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; - { - auto result = testServer.DataStreamsClient->CreateStream(streamName, - NYDS_V1::TCreateStreamSettings().ShardCount(3)).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - { - auto result = testServer.DataStreamsClient->RegisterStreamConsumer(streamName, "user1", - NYDS_V1::TRegisterStreamConsumerSettings()).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - { - auto result = testServer.DataStreamsClient->DeleteStream(streamName, - NYDS_V1::TDeleteStreamSettings().EnforceConsumerDeletion(false)).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST); - } - - { - auto result = testServer.DataStreamsClient->DeregisterStreamConsumer(streamName, "user1", - NYDS_V1::TDeregisterStreamConsumerSettings()).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - { - auto result = testServer.DataStreamsClient->DeleteStream(streamName, - NYDS_V1::TDeleteStreamSettings().EnforceConsumerDeletion(false)).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - } - + Y_UNIT_TEST(TestDeleteStream) { + TInsecureDatastreamsTestServer testServer; + const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; + { + auto result = testServer.DataStreamsClient->CreateStream(streamName, + NYDS_V1::TCreateStreamSettings().ShardCount(3)).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + + { + auto result = testServer.DataStreamsClient->DeleteStream(streamName).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + } + + Y_UNIT_TEST(TestDeleteStreamWithEnforceFlag) { + TInsecureDatastreamsTestServer testServer; + const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; + { + auto result = testServer.DataStreamsClient->CreateStream(streamName, + NYDS_V1::TCreateStreamSettings().ShardCount(3)).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + + { + auto result = testServer.DataStreamsClient->RegisterStreamConsumer(streamName, "user1", + NYDS_V1::TRegisterStreamConsumerSettings()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + + { + auto result = testServer.DataStreamsClient->DeleteStream(streamName).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST); + } + + { + auto result = testServer.DataStreamsClient->DeleteStream(streamName, + NYDS_V1::TDeleteStreamSettings().EnforceConsumerDeletion(true)).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + } + + + Y_UNIT_TEST(TestDeleteStreamWithEnforceFlagFalse) { + TInsecureDatastreamsTestServer testServer; + const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; + { + auto result = testServer.DataStreamsClient->CreateStream(streamName, + NYDS_V1::TCreateStreamSettings().ShardCount(3)).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + + { + auto result = testServer.DataStreamsClient->RegisterStreamConsumer(streamName, "user1", + NYDS_V1::TRegisterStreamConsumerSettings()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + + { + auto result = testServer.DataStreamsClient->DeleteStream(streamName, + NYDS_V1::TDeleteStreamSettings().EnforceConsumerDeletion(false)).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST); + } + + { + auto result = testServer.DataStreamsClient->DeregisterStreamConsumer(streamName, "user1", + NYDS_V1::TDeregisterStreamConsumerSettings()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + + { + auto result = testServer.DataStreamsClient->DeleteStream(streamName, + NYDS_V1::TDeleteStreamSettings().EnforceConsumerDeletion(false)).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + } + Y_UNIT_TEST(TestUpdateStream) { TInsecureDatastreamsTestServer testServer; - const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; + const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; { - auto result = testServer.DataStreamsClient->CreateStream(streamName, - NYDS_V1::TCreateStreamSettings().ShardCount(10)).ExtractValueSync(); + auto result = testServer.DataStreamsClient->CreateStream(streamName, + NYDS_V1::TCreateStreamSettings().ShardCount(10)).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); } for (ui32 i = 0; i < 2; ++i) { - auto result = testServer.DataStreamsClient->UpdateStream(streamName, - NYDS_V1::TUpdateStreamSettings().RetentionPeriodHours(5).TargetShardCount(20).WriteQuotaKbPerSec(128) - ).ExtractValueSync(); + auto result = testServer.DataStreamsClient->UpdateStream(streamName, + NYDS_V1::TUpdateStreamSettings().RetentionPeriodHours(5).TargetShardCount(20).WriteQuotaKbPerSec(128) + ).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); } { - auto result = testServer.DataStreamsClient->DescribeStream(streamName).ExtractValueSync(); + auto result = testServer.DataStreamsClient->DescribeStream(streamName).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_description().shards_size(), 20); @@ -421,42 +421,42 @@ Y_UNIT_TEST_SUITE(DataStreams) { Y_UNIT_TEST(TestStreamRetention) { TInsecureDatastreamsTestServer testServer; - const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; + const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; { - auto result = testServer.DataStreamsClient->CreateStream(streamName, - NYDS_V1::TCreateStreamSettings().ShardCount(10)).ExtractValueSync(); + auto result = testServer.DataStreamsClient->CreateStream(streamName, + NYDS_V1::TCreateStreamSettings().ShardCount(10)).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); } { - auto result = testServer.DataStreamsClient->IncreaseStreamRetentionPeriod(streamName, - NYDS_V1::TIncreaseStreamRetentionPeriodSettings().RetentionPeriodHours(50) - ).ExtractValueSync(); + auto result = testServer.DataStreamsClient->IncreaseStreamRetentionPeriod(streamName, + NYDS_V1::TIncreaseStreamRetentionPeriodSettings().RetentionPeriodHours(50) + ).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST); } { - auto result = testServer.DataStreamsClient->DecreaseStreamRetentionPeriod(streamName, - NYDS_V1::TDecreaseStreamRetentionPeriodSettings().RetentionPeriodHours(8) - ).ExtractValueSync(); + auto result = testServer.DataStreamsClient->DecreaseStreamRetentionPeriod(streamName, + NYDS_V1::TDecreaseStreamRetentionPeriodSettings().RetentionPeriodHours(8) + ).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); } { - auto result = testServer.DataStreamsClient->IncreaseStreamRetentionPeriod(streamName, - NYDS_V1::TIncreaseStreamRetentionPeriodSettings().RetentionPeriodHours(4) - ).ExtractValueSync(); + auto result = testServer.DataStreamsClient->IncreaseStreamRetentionPeriod(streamName, + NYDS_V1::TIncreaseStreamRetentionPeriodSettings().RetentionPeriodHours(4) + ).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST); } { - auto result = testServer.DataStreamsClient->IncreaseStreamRetentionPeriod(streamName, - NYDS_V1::TIncreaseStreamRetentionPeriodSettings().RetentionPeriodHours(15) - ).ExtractValueSync(); + auto result = testServer.DataStreamsClient->IncreaseStreamRetentionPeriod(streamName, + NYDS_V1::TIncreaseStreamRetentionPeriodSettings().RetentionPeriodHours(15) + ).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); } @@ -465,9 +465,9 @@ Y_UNIT_TEST_SUITE(DataStreams) { Y_UNIT_TEST(TestShardPagination) { TInsecureDatastreamsTestServer testServer; - const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; + const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; { - auto result = testServer.DataStreamsClient->CreateStream("/Root/" + streamName, NYDS_V1::TCreateStreamSettings().ShardCount(9)).ExtractValueSync(); + auto result = testServer.DataStreamsClient->CreateStream("/Root/" + streamName, NYDS_V1::TCreateStreamSettings().ShardCount(9)).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); } @@ -477,8 +477,8 @@ Y_UNIT_TEST_SUITE(DataStreams) { TString exclusiveStartShardId; THashSet<TString> describedShards; for (int i = 0; i < 8; i += 2) { - auto result = testServer.DataStreamsClient->DescribeStream(streamName, - NYDS_V1::TDescribeStreamSettings() + auto result = testServer.DataStreamsClient->DescribeStream(streamName, + NYDS_V1::TDescribeStreamSettings() .Limit(2) .ExclusiveStartShardId(exclusiveStartShardId) ).ExtractValueSync(); @@ -494,12 +494,12 @@ Y_UNIT_TEST_SUITE(DataStreams) { } { - auto result = testServer.DataStreamsClient->DescribeStream(streamName, - NYDS_V1::TDescribeStreamSettings() + auto result = testServer.DataStreamsClient->DescribeStream(streamName, + NYDS_V1::TDescribeStreamSettings() .Limit(2) .ExclusiveStartShardId( - exclusiveStartShardId) - ).ExtractValueSync(); + exclusiveStartShardId) + ).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); auto &description = result.GetResult().stream_description(); @@ -526,34 +526,34 @@ Y_UNIT_TEST_SUITE(DataStreams) { Y_UNIT_TEST(TestPutRecordsWithRead) { TInsecureDatastreamsTestServer testServer; - const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; + const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; SET_YDS_LOCALS; { - auto result = testServer.DataStreamsClient->CreateStream(streamName, - NYDS_V1::TCreateStreamSettings().ShardCount(5)).ExtractValueSync(); + auto result = testServer.DataStreamsClient->CreateStream(streamName, + NYDS_V1::TCreateStreamSettings().ShardCount(5)).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); } kikimr->GetRuntime()->SetLogPriority(NKikimrServices::PQ_READ_PROXY, NLog::EPriority::PRI_DEBUG); kikimr->GetRuntime()->SetLogPriority(NKikimrServices::PQ_WRITE_PROXY, NLog::EPriority::PRI_DEBUG); - NYDS_V1::TDataStreamsClient client(*driver, TCommonClientSettings().AuthToken("user2@builtin")); + NYDS_V1::TDataStreamsClient client(*driver, TCommonClientSettings().AuthToken("user2@builtin")); TString dataStr = "9876543210"; - auto putRecordResult = client.PutRecord("/Root/" + streamName, {dataStr, dataStr, dataStr}).ExtractValueSync(); + auto putRecordResult = client.PutRecord("/Root/" + streamName, {dataStr, dataStr, dataStr}).ExtractValueSync(); Cerr << putRecordResult.GetResult().DebugString() << Endl; UNIT_ASSERT_VALUES_EQUAL(putRecordResult.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(putRecordResult.GetStatus(), EStatus::SUCCESS); { - std::vector<NYDS_V1::TDataRecord> records; + std::vector<NYDS_V1::TDataRecord> records; for (ui32 i = 1; i <= 30; ++i) { TString data = Sprintf("%04u", i); records.push_back({data, data, ""}); } - auto result = client.PutRecords(streamName, records).ExtractValueSync(); + auto result = client.PutRecords(streamName, records).ExtractValueSync(); Cerr << result.GetResult().DebugString() << Endl; UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); @@ -562,18 +562,18 @@ Y_UNIT_TEST_SUITE(DataStreams) { NYdb::NPersQueue::TPersQueueClient pqClient(*driver); { - auto result = testServer.DataStreamsClient->RegisterStreamConsumer(streamName, "user1", NYDS_V1::TRegisterStreamConsumerSettings()).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT_VALUES_EQUAL(result.GetResult().consumer().consumer_name(), "user1"); - UNIT_ASSERT_VALUES_EQUAL(result.GetResult().consumer().consumer_status(), - YDS_V1::ConsumerDescription_ConsumerStatus_ACTIVE); + auto result = testServer.DataStreamsClient->RegisterStreamConsumer(streamName, "user1", NYDS_V1::TRegisterStreamConsumerSettings()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + UNIT_ASSERT_VALUES_EQUAL(result.GetResult().consumer().consumer_name(), "user1"); + UNIT_ASSERT_VALUES_EQUAL(result.GetResult().consumer().consumer_status(), + YDS_V1::ConsumerDescription_ConsumerStatus_ACTIVE); } auto session = pqClient.CreateReadSession(NYdb::NPersQueue::TReadSessionSettings() .ConsumerName("user1") .DisableClusterDiscovery(true) - .AppendTopics(NYdb::NPersQueue::TTopicReadSettings().Path("/Root/" + streamName))); + .AppendTopics(NYdb::NPersQueue::TTopicReadSettings().Path("/Root/" + streamName))); ui32 readCount = 0; while (readCount < 31) { auto event = session->GetEvent(true); @@ -603,32 +603,32 @@ Y_UNIT_TEST_SUITE(DataStreams) { Y_UNIT_TEST(TestPutRecordsCornerCases) { TInsecureDatastreamsTestServer testServer; - const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; - const TString streamPath = "/Root/" + streamName; + const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; + const TString streamPath = "/Root/" + streamName; SET_YDS_LOCALS; { auto result = testServer.DataStreamsClient->CreateStream( - streamName, - NYDS_V1::TCreateStreamSettings().ShardCount(5) - ).ExtractValueSync(); + streamName, + NYDS_V1::TCreateStreamSettings().ShardCount(5) + ).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); } kikimr->GetRuntime()->SetLogPriority(NKikimrServices::PQ_READ_PROXY, NLog::EPriority::PRI_DEBUG); - NYDS_V1::TDataStreamsClient client(*driver, TCommonClientSettings().AuthToken("user2@builtin")); + NYDS_V1::TDataStreamsClient client(*driver, TCommonClientSettings().AuthToken("user2@builtin")); // Test for too long partition key TString longKey = TString(257, '1'); TString shortEnoughKey = TString(256, '1'); - auto result = client.PutRecords(streamName, - {{longKey, longKey, ""}, - {shortEnoughKey, shortEnoughKey, ""}}).ExtractValueSync(); + auto result = client.PutRecords(streamName, + {{longKey, longKey, ""}, + {shortEnoughKey, shortEnoughKey, ""}}).ExtractValueSync(); UNIT_ASSERT(!result.IsSuccess()); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST); - result = client.PutRecords(streamName, - {{shortEnoughKey, shortEnoughKey, ""}, - {shortEnoughKey, shortEnoughKey, ""}}).ExtractValueSync(); + result = client.PutRecords(streamName, + {{shortEnoughKey, shortEnoughKey, ""}, + {shortEnoughKey, shortEnoughKey, ""}}).ExtractValueSync(); UNIT_ASSERT(result.IsSuccess()); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); @@ -636,15 +636,15 @@ Y_UNIT_TEST_SUITE(DataStreams) { TString longData = TString(1048577, '1'); TString shortEnoughData = TString(1048576, '1'); - result = client.PutRecords(streamName, - {{longData, shortEnoughKey, ""}, - {shortEnoughData, shortEnoughKey, ""}}).ExtractValueSync(); + result = client.PutRecords(streamName, + {{longData, shortEnoughKey, ""}, + {shortEnoughData, shortEnoughKey, ""}}).ExtractValueSync(); UNIT_ASSERT(!result.IsSuccess()); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST); - result = client.PutRecords(streamName, - {{shortEnoughData, shortEnoughKey, ""}, - {"", shortEnoughKey, ""}}).ExtractValueSync(); + result = client.PutRecords(streamName, + {{shortEnoughData, shortEnoughKey, ""}, + {"", shortEnoughKey, ""}}).ExtractValueSync(); UNIT_ASSERT(result.IsSuccess()); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); @@ -652,27 +652,27 @@ Y_UNIT_TEST_SUITE(DataStreams) { TString shortEnoughExplicitHash = "340282366920938463463374607431768211455"; TString badExplicitHash = "-439025493205215"; - result = client.PutRecords(streamName, - {{"", shortEnoughKey, longExplicitHash}, - {"", shortEnoughKey, longExplicitHash}}).ExtractValueSync(); + result = client.PutRecords(streamName, + {{"", shortEnoughKey, longExplicitHash}, + {"", shortEnoughKey, longExplicitHash}}).ExtractValueSync(); UNIT_ASSERT(!result.IsSuccess()); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST); - result = client.PutRecords(streamName, - {{"", shortEnoughKey, badExplicitHash}, - {"", shortEnoughKey, badExplicitHash}}).ExtractValueSync(); + result = client.PutRecords(streamName, + {{"", shortEnoughKey, badExplicitHash}, + {"", shortEnoughKey, badExplicitHash}}).ExtractValueSync(); UNIT_ASSERT(!result.IsSuccess()); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST); - result = client.PutRecords(streamName, - {{"", shortEnoughKey, shortEnoughExplicitHash}, - {"", shortEnoughKey, shortEnoughExplicitHash}}).ExtractValueSync(); + result = client.PutRecords(streamName, + {{"", shortEnoughKey, shortEnoughExplicitHash}, + {"", shortEnoughKey, shortEnoughExplicitHash}}).ExtractValueSync(); UNIT_ASSERT(result.IsSuccess()); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - result = client.PutRecords(streamName, - {{"", shortEnoughKey, "0"}, - {"", shortEnoughKey, "0"}}).ExtractValueSync(); + result = client.PutRecords(streamName, + {{"", shortEnoughKey, "0"}, + {"", shortEnoughKey, "0"}}).ExtractValueSync(); UNIT_ASSERT(result.IsSuccess()); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); @@ -683,7 +683,7 @@ Y_UNIT_TEST_SUITE(DataStreams) { kikimr->GetRuntime()->SetLogPriority(NKikimrServices::PERSQUEUE, NLog::EPriority::PRI_INFO); { - std::vector<NYDS_V1::TDataRecord> records; + std::vector<NYDS_V1::TDataRecord> records; TString data = TString(1024*1024, 'a'); records.push_back({data, "key", ""}); records.push_back({data, "key", ""}); @@ -691,7 +691,7 @@ Y_UNIT_TEST_SUITE(DataStreams) { records.push_back({data, "key", ""}); Cerr << "First put records\n"; - auto result = client.PutRecords(streamPath, records).ExtractValueSync(); + auto result = client.PutRecords(streamPath, records).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); if (result.GetStatus() != EStatus::SUCCESS) { result.GetIssues().PrintTo(Cerr); @@ -700,7 +700,7 @@ Y_UNIT_TEST_SUITE(DataStreams) { Cerr << result.GetResult().DebugString() << Endl; Cerr << "Second put records\n"; - result = client.PutRecords(streamPath, records).ExtractValueSync(); + result = client.PutRecords(streamPath, records).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); if (result.GetStatus() != EStatus::SUCCESS) { result.GetIssues().PrintTo(Cerr); @@ -712,7 +712,7 @@ Y_UNIT_TEST_SUITE(DataStreams) { Sleep(TDuration::Seconds(4)); Cerr << "Third put records\n"; - result = client.PutRecords(streamPath, records).ExtractValueSync(); + result = client.PutRecords(streamPath, records).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); if (result.GetStatus() != EStatus::SUCCESS) { result.GetIssues().PrintTo(Cerr); @@ -727,7 +727,7 @@ Y_UNIT_TEST_SUITE(DataStreams) { { NYdb::NPersQueue::TAddReadRuleSettings addReadRuleSettings; addReadRuleSettings.ReadRule(NYdb::NPersQueue::TReadRuleSettings().Version(1).ConsumerName("user1")); - auto res = pqClient.AddReadRule(streamPath, addReadRuleSettings); + auto res = pqClient.AddReadRule(streamPath, addReadRuleSettings); res.Wait(); UNIT_ASSERT(res.GetValue().IsSuccess()); } @@ -735,7 +735,7 @@ Y_UNIT_TEST_SUITE(DataStreams) { .ConsumerName("user1") .DisableClusterDiscovery(true) .RetryPolicy(NYdb::NPersQueue::IRetryPolicy::GetNoRetryPolicy()) - .AppendTopics(NYdb::NPersQueue::TTopicReadSettings().Path(streamPath))); + .AppendTopics(NYdb::NPersQueue::TTopicReadSettings().Path(streamPath))); ui32 readCount = 0; while (readCount < 14) { auto event = session->GetEvent(true); @@ -758,20 +758,20 @@ Y_UNIT_TEST_SUITE(DataStreams) { Y_UNIT_TEST(TestPutRecords) { TSecureDatastreamsTestServer testServer; - const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; + const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; SET_YDS_LOCALS; - const TString streamPath = "/Root/" + streamName; + const TString streamPath = "/Root/" + streamName; { - auto result = testServer.DataStreamsClient->CreateStream(streamPath, - NYDS_V1::TCreateStreamSettings().ShardCount(5)).ExtractValueSync(); + auto result = testServer.DataStreamsClient->CreateStream(streamPath, + NYDS_V1::TCreateStreamSettings().ShardCount(5)).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); } - NYDS_V1::TDataStreamsClient client(*driver, TCommonClientSettings().AuthToken("user2@builtin")); + NYDS_V1::TDataStreamsClient client(*driver, TCommonClientSettings().AuthToken("user2@builtin")); NYdb::NScheme::TSchemeClient schemeClient(*driver); { - std::vector<NYDS_V1::TDataRecord> records; + std::vector<NYDS_V1::TDataRecord> records; for (ui32 i = 1; i <= 30; ++i) { TString data = Sprintf("%04u", i); records.push_back({data, data, ""}); @@ -783,12 +783,12 @@ Y_UNIT_TEST_SUITE(DataStreams) { } UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNAUTHORIZED); - { - NYdb::NScheme::TPermissions permissions("user2@builtin", {"ydb.generic.read", "ydb.generic.write"}); - auto result = schemeClient.ModifyPermissions(streamPath, - NYdb::NScheme::TModifyPermissionsSettings().AddGrantPermissions(permissions)).ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - } + { + NYdb::NScheme::TPermissions permissions("user2@builtin", {"ydb.generic.read", "ydb.generic.write"}); + auto result = schemeClient.ModifyPermissions(streamPath, + NYdb::NScheme::TModifyPermissionsSettings().AddGrantPermissions(permissions)).ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + } result = client.PutRecords(streamPath, records).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); @@ -796,7 +796,7 @@ Y_UNIT_TEST_SUITE(DataStreams) { Cerr << "PutRecordsResponse = " << result.GetResult().DebugString() << Endl; UNIT_ASSERT_VALUES_EQUAL(result.GetResult().failed_record_count(), 0); UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records_size(), records.size()); - UNIT_ASSERT_VALUES_EQUAL(result.GetResult().encryption_type(), YDS_V1::EncryptionType::NONE); + UNIT_ASSERT_VALUES_EQUAL(result.GetResult().encryption_type(), YDS_V1::EncryptionType::NONE); TString dataStr = "9876543210"; auto putRecordResult = client.PutRecord(streamPath, {dataStr, dataStr, ""}).ExtractValueSync(); @@ -805,29 +805,29 @@ Y_UNIT_TEST_SUITE(DataStreams) { Cerr << "PutRecord response = " << putRecordResult.GetResult().DebugString() << Endl; UNIT_ASSERT_VALUES_EQUAL(putRecordResult.GetResult().shard_id(), "shard-000004"); UNIT_ASSERT_VALUES_EQUAL(putRecordResult.GetResult().sequence_number(), "7"); - UNIT_ASSERT_VALUES_EQUAL(putRecordResult.GetResult().encryption_type(), - YDS_V1::EncryptionType::NONE); + UNIT_ASSERT_VALUES_EQUAL(putRecordResult.GetResult().encryption_type(), + YDS_V1::EncryptionType::NONE); } } Y_UNIT_TEST(TestPutEmptyMessage) { TInsecureDatastreamsTestServer testServer; - const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; + const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; SET_YDS_LOCALS; { - auto result = testServer.DataStreamsClient->CreateStream(streamName, - NYDS_V1::TCreateStreamSettings().ShardCount(5)).ExtractValueSync(); + auto result = testServer.DataStreamsClient->CreateStream(streamName, + NYDS_V1::TCreateStreamSettings().ShardCount(5)).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); } kikimr->GetRuntime()->SetLogPriority(NKikimrServices::PQ_READ_PROXY, NLog::EPriority::PRI_DEBUG); kikimr->GetRuntime()->SetLogPriority(NKikimrServices::PQ_WRITE_PROXY, NLog::EPriority::PRI_DEBUG); - NYDS_V1::TDataStreamsClient client(*driver, TCommonClientSettings().AuthToken("user2@builtin")); + NYDS_V1::TDataStreamsClient client(*driver, TCommonClientSettings().AuthToken("user2@builtin")); - auto putRecordResult = client.PutRecord("/Root/" + streamName, {"", "key", ""}).ExtractValueSync(); + auto putRecordResult = client.PutRecord("/Root/" + streamName, {"", "key", ""}).ExtractValueSync(); Cerr << putRecordResult.GetResult().DebugString() << Endl; UNIT_ASSERT_VALUES_EQUAL(putRecordResult.IsTransportError(), false); @@ -837,18 +837,18 @@ Y_UNIT_TEST_SUITE(DataStreams) { NYdb::NPersQueue::TPersQueueClient pqClient(*driver); { - auto result = testServer.DataStreamsClient->RegisterStreamConsumer(streamName, "user1", NYDS_V1::TRegisterStreamConsumerSettings()).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT_VALUES_EQUAL(result.GetResult().consumer().consumer_name(), "user1"); - UNIT_ASSERT_VALUES_EQUAL(result.GetResult().consumer().consumer_status(), - YDS_V1::ConsumerDescription_ConsumerStatus_ACTIVE); + auto result = testServer.DataStreamsClient->RegisterStreamConsumer(streamName, "user1", NYDS_V1::TRegisterStreamConsumerSettings()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + UNIT_ASSERT_VALUES_EQUAL(result.GetResult().consumer().consumer_name(), "user1"); + UNIT_ASSERT_VALUES_EQUAL(result.GetResult().consumer().consumer_status(), + YDS_V1::ConsumerDescription_ConsumerStatus_ACTIVE); } auto session = pqClient.CreateReadSession(NYdb::NPersQueue::TReadSessionSettings() .ConsumerName("user1") .DisableClusterDiscovery(true) - .AppendTopics(NYdb::NPersQueue::TTopicReadSettings().Path("/Root/" + streamName))); + .AppendTopics(NYdb::NPersQueue::TTopicReadSettings().Path("/Root/" + streamName))); while (true) { auto event = session->GetEvent(true); @@ -875,760 +875,760 @@ Y_UNIT_TEST_SUITE(DataStreams) { Y_UNIT_TEST(TestListStreamConsumers) { TInsecureDatastreamsTestServer testServer; - const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; - // Create stream -> OK - { - auto result = testServer.DataStreamsClient->CreateStream(streamName, - NYDS_V1::TCreateStreamSettings().ShardCount(5)).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - // List stream consumers -> OK - { - auto result = testServer.DataStreamsClient->ListStreamConsumers(streamName, - NYDS_V1::TListStreamConsumersSettings().MaxResults(100)).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT_VALUES_EQUAL(result.GetResult().consumers().size(), 0); - } - - // List stream consumers more than allowed -> get BAD_REQUEST - { - auto result = testServer.DataStreamsClient->ListStreamConsumers(streamName, - NYDS_V1::TListStreamConsumersSettings().MaxResults(10001)).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST); - } - - // List stream consumers less than allowed -> get BAD_REQUEST - { - auto result = testServer.DataStreamsClient->ListStreamConsumers(streamName, - NYDS_V1::TListStreamConsumersSettings().MaxResults(0)).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST); - } - - // List not created stream -> get SCHEME_ERROR - { - auto result = testServer.DataStreamsClient->ListStreamConsumers(streamName + "_XXX", - NYDS_V1::TListStreamConsumersSettings().MaxResults(100)).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR); - } - - // Deregister unregistered consumer -> get NOT_FOUND - { - auto result = testServer.DataStreamsClient->DeregisterStreamConsumer(streamName, "user1", - NYDS_V1::TDeregisterStreamConsumerSettings()).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::NOT_FOUND); - } - - // Register consumer 1 -> OK - { - auto result = testServer.DataStreamsClient->RegisterStreamConsumer(streamName, "user1", - NYDS_V1::TRegisterStreamConsumerSettings()).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT_VALUES_EQUAL(result.GetResult().consumer().consumer_name(), "user1"); - UNIT_ASSERT_VALUES_EQUAL(result.GetResult().consumer().consumer_status(), - YDS_V1::ConsumerDescription_ConsumerStatus_ACTIVE); - } - - // Register consumer 2 -> OK - { - auto result = testServer.DataStreamsClient->RegisterStreamConsumer(streamName, "user2", - NYDS_V1::TRegisterStreamConsumerSettings()).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT_VALUES_EQUAL(result.GetResult().consumer().consumer_name(), "user2"); - } - - // List stream consumers -> OK, get 2 - { - auto result = testServer.DataStreamsClient->ListStreamConsumers(streamName, - NYDS_V1::TListStreamConsumersSettings().MaxResults(100)).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT_VALUES_EQUAL(result.GetResult().consumers().size(), 2); - } - - // Deregister consumer 2 -> OK - { - auto result = testServer.DataStreamsClient->DeregisterStreamConsumer(streamName, "user2", - NYDS_V1::TDeregisterStreamConsumerSettings()).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - // List stream consumers -> OK, there's only 1 now - { - auto result = testServer.DataStreamsClient->ListStreamConsumers(streamName, - NYDS_V1::TListStreamConsumersSettings().MaxResults(100)).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT_VALUES_EQUAL(result.GetResult().consumers().size(), 1); - } - - // Register stream consumer -> OK - { - auto result = testServer.DataStreamsClient->RegisterStreamConsumer(streamName, "user2", - NYDS_V1::TRegisterStreamConsumerSettings()).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT_VALUES_EQUAL(result.GetResult().consumer().consumer_name(), "user2"); - } - - // Add already added consumer -> get ALREADY_EXISTS - { - auto result = testServer.DataStreamsClient->RegisterStreamConsumer(streamName, "user2", - NYDS_V1::TRegisterStreamConsumerSettings()).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::ALREADY_EXISTS); - } - - // List stream consumers with pages of size 1 -> OK - { - auto result = testServer.DataStreamsClient->ListStreamConsumers(streamName, - NYDS_V1::TListStreamConsumersSettings().MaxResults(1)).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT_VALUES_EQUAL(result.GetResult().consumers().size(), 1); - UNIT_ASSERT_VALUES_UNEQUAL(result.GetResult().next_token().size(), 0); - - auto nextToken = result.GetResult().next_token(); - result = testServer.DataStreamsClient->ListStreamConsumers(streamName, - NYDS_V1::TListStreamConsumersSettings().NextToken(nextToken)).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT_VALUES_EQUAL(result.GetResult().consumers().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(result.GetResult().next_token().size(), 0); - } - - // List stream consumers with page size 1, but then provide corrupted or expired token - { - auto result = testServer.DataStreamsClient->ListStreamConsumers(streamName, - NYDS_V1::TListStreamConsumersSettings().MaxResults(1)).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT_VALUES_EQUAL(result.GetResult().consumers().size(), 1); - UNIT_ASSERT_VALUES_UNEQUAL(result.GetResult().next_token().size(), 0); - - auto makeNextToken = [streamName](ui64 timestamp) { - NKikimrPQ::TYdsNextToken protoNextToken; - protoNextToken.SetStreamArn(streamName); - protoNextToken.SetAlreadyRead(1); - protoNextToken.SetMaxResults(1); - protoNextToken.SetCreationTimestamp(timestamp); - TString stringNextToken; + const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; + // Create stream -> OK + { + auto result = testServer.DataStreamsClient->CreateStream(streamName, + NYDS_V1::TCreateStreamSettings().ShardCount(5)).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + + // List stream consumers -> OK + { + auto result = testServer.DataStreamsClient->ListStreamConsumers(streamName, + NYDS_V1::TListStreamConsumersSettings().MaxResults(100)).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + UNIT_ASSERT_VALUES_EQUAL(result.GetResult().consumers().size(), 0); + } + + // List stream consumers more than allowed -> get BAD_REQUEST + { + auto result = testServer.DataStreamsClient->ListStreamConsumers(streamName, + NYDS_V1::TListStreamConsumersSettings().MaxResults(10001)).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST); + } + + // List stream consumers less than allowed -> get BAD_REQUEST + { + auto result = testServer.DataStreamsClient->ListStreamConsumers(streamName, + NYDS_V1::TListStreamConsumersSettings().MaxResults(0)).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST); + } + + // List not created stream -> get SCHEME_ERROR + { + auto result = testServer.DataStreamsClient->ListStreamConsumers(streamName + "_XXX", + NYDS_V1::TListStreamConsumersSettings().MaxResults(100)).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR); + } + + // Deregister unregistered consumer -> get NOT_FOUND + { + auto result = testServer.DataStreamsClient->DeregisterStreamConsumer(streamName, "user1", + NYDS_V1::TDeregisterStreamConsumerSettings()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::NOT_FOUND); + } + + // Register consumer 1 -> OK + { + auto result = testServer.DataStreamsClient->RegisterStreamConsumer(streamName, "user1", + NYDS_V1::TRegisterStreamConsumerSettings()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + UNIT_ASSERT_VALUES_EQUAL(result.GetResult().consumer().consumer_name(), "user1"); + UNIT_ASSERT_VALUES_EQUAL(result.GetResult().consumer().consumer_status(), + YDS_V1::ConsumerDescription_ConsumerStatus_ACTIVE); + } + + // Register consumer 2 -> OK + { + auto result = testServer.DataStreamsClient->RegisterStreamConsumer(streamName, "user2", + NYDS_V1::TRegisterStreamConsumerSettings()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + UNIT_ASSERT_VALUES_EQUAL(result.GetResult().consumer().consumer_name(), "user2"); + } + + // List stream consumers -> OK, get 2 + { + auto result = testServer.DataStreamsClient->ListStreamConsumers(streamName, + NYDS_V1::TListStreamConsumersSettings().MaxResults(100)).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + UNIT_ASSERT_VALUES_EQUAL(result.GetResult().consumers().size(), 2); + } + + // Deregister consumer 2 -> OK + { + auto result = testServer.DataStreamsClient->DeregisterStreamConsumer(streamName, "user2", + NYDS_V1::TDeregisterStreamConsumerSettings()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + + // List stream consumers -> OK, there's only 1 now + { + auto result = testServer.DataStreamsClient->ListStreamConsumers(streamName, + NYDS_V1::TListStreamConsumersSettings().MaxResults(100)).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + UNIT_ASSERT_VALUES_EQUAL(result.GetResult().consumers().size(), 1); + } + + // Register stream consumer -> OK + { + auto result = testServer.DataStreamsClient->RegisterStreamConsumer(streamName, "user2", + NYDS_V1::TRegisterStreamConsumerSettings()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + UNIT_ASSERT_VALUES_EQUAL(result.GetResult().consumer().consumer_name(), "user2"); + } + + // Add already added consumer -> get ALREADY_EXISTS + { + auto result = testServer.DataStreamsClient->RegisterStreamConsumer(streamName, "user2", + NYDS_V1::TRegisterStreamConsumerSettings()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::ALREADY_EXISTS); + } + + // List stream consumers with pages of size 1 -> OK + { + auto result = testServer.DataStreamsClient->ListStreamConsumers(streamName, + NYDS_V1::TListStreamConsumersSettings().MaxResults(1)).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + UNIT_ASSERT_VALUES_EQUAL(result.GetResult().consumers().size(), 1); + UNIT_ASSERT_VALUES_UNEQUAL(result.GetResult().next_token().size(), 0); + + auto nextToken = result.GetResult().next_token(); + result = testServer.DataStreamsClient->ListStreamConsumers(streamName, + NYDS_V1::TListStreamConsumersSettings().NextToken(nextToken)).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + UNIT_ASSERT_VALUES_EQUAL(result.GetResult().consumers().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(result.GetResult().next_token().size(), 0); + } + + // List stream consumers with page size 1, but then provide corrupted or expired token + { + auto result = testServer.DataStreamsClient->ListStreamConsumers(streamName, + NYDS_V1::TListStreamConsumersSettings().MaxResults(1)).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + UNIT_ASSERT_VALUES_EQUAL(result.GetResult().consumers().size(), 1); + UNIT_ASSERT_VALUES_UNEQUAL(result.GetResult().next_token().size(), 0); + + auto makeNextToken = [streamName](ui64 timestamp) { + NKikimrPQ::TYdsNextToken protoNextToken; + protoNextToken.SetStreamArn(streamName); + protoNextToken.SetAlreadyRead(1); + protoNextToken.SetMaxResults(1); + protoNextToken.SetCreationTimestamp(timestamp); + TString stringNextToken; Y_PROTOBUF_SUPPRESS_NODISCARD protoNextToken.SerializeToString(&stringNextToken); - TString encodedNextToken; - Base64Encode(stringNextToken, encodedNextToken); - return encodedNextToken; - }; - - auto nextToken = makeNextToken(TInstant::Now().MilliSeconds() - 300001); - result = testServer.DataStreamsClient->ListStreamConsumers(streamName, - NYDS_V1::TListStreamConsumersSettings().NextToken(nextToken)).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST); - - nextToken = makeNextToken(TInstant::Now().MilliSeconds() + 1000); - result = testServer.DataStreamsClient->ListStreamConsumers(streamName, - NYDS_V1::TListStreamConsumersSettings().NextToken(nextToken)).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST); - - nextToken = makeNextToken(0); - result = testServer.DataStreamsClient->ListStreamConsumers(streamName, - NYDS_V1::TListStreamConsumersSettings().NextToken(nextToken)).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST); - - result = testServer.DataStreamsClient->ListStreamConsumers(streamName, - NYDS_V1::TListStreamConsumersSettings().NextToken("some_garbage")).ExtractValueSync(); - result = testServer.DataStreamsClient->ListStreamConsumers(streamName, - NYDS_V1::TListStreamConsumersSettings().NextToken("some_garbage")).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST); - } - - // Delete stream -> OK - { - auto result = testServer.DataStreamsClient->DeleteStream(streamName, - NYDS_V1::TDeleteStreamSettings().EnforceConsumerDeletion(true)).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - // List consumers of deleted stream -> get BAD_REQUEST - { - auto result = testServer.DataStreamsClient->ListStreamConsumers(streamName).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST); - } - } - + TString encodedNextToken; + Base64Encode(stringNextToken, encodedNextToken); + return encodedNextToken; + }; + + auto nextToken = makeNextToken(TInstant::Now().MilliSeconds() - 300001); + result = testServer.DataStreamsClient->ListStreamConsumers(streamName, + NYDS_V1::TListStreamConsumersSettings().NextToken(nextToken)).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST); + + nextToken = makeNextToken(TInstant::Now().MilliSeconds() + 1000); + result = testServer.DataStreamsClient->ListStreamConsumers(streamName, + NYDS_V1::TListStreamConsumersSettings().NextToken(nextToken)).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST); + + nextToken = makeNextToken(0); + result = testServer.DataStreamsClient->ListStreamConsumers(streamName, + NYDS_V1::TListStreamConsumersSettings().NextToken(nextToken)).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST); + + result = testServer.DataStreamsClient->ListStreamConsumers(streamName, + NYDS_V1::TListStreamConsumersSettings().NextToken("some_garbage")).ExtractValueSync(); + result = testServer.DataStreamsClient->ListStreamConsumers(streamName, + NYDS_V1::TListStreamConsumersSettings().NextToken("some_garbage")).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST); + } + + // Delete stream -> OK + { + auto result = testServer.DataStreamsClient->DeleteStream(streamName, + NYDS_V1::TDeleteStreamSettings().EnforceConsumerDeletion(true)).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + + // List consumers of deleted stream -> get BAD_REQUEST + { + auto result = testServer.DataStreamsClient->ListStreamConsumers(streamName).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST); + } + } + Y_UNIT_TEST(TestGetShardIterator) { TInsecureDatastreamsTestServer testServer; - const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; - { - auto result = testServer.DataStreamsClient->CreateStream(streamName, \ - NYDS_V1::TCreateStreamSettings().ShardCount(5) - ).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - { - std::vector<NYDS_V1::TDataRecord> records; - for (ui32 i = 1; i <= 30; ++i) { - TString data = Sprintf("%04u", i); - records.push_back({data, data, ""}); - } - - auto result = testServer.DataStreamsClient->PutRecords(streamName, records).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - if (result.GetStatus() != EStatus::SUCCESS) { - result.GetIssues().PrintTo(Cerr); - } - } - - { - auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000001", - YDS_V1::ShardIteratorType::LATEST).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - { - auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000010", - YDS_V1::ShardIteratorType::LATEST).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST); - } - - { - auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000001", - YDS_V1::ShardIteratorType::AT_SEQUENCE_NUMBER, - NYDS_V1::TGetShardIteratorSettings().StartingSequenceNumber("0") - ).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - { - auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000001", - YDS_V1::ShardIteratorType::AFTER_SEQUENCE_NUMBER, - NYDS_V1::TGetShardIteratorSettings().StartingSequenceNumber("0") - ).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - { - auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000001", - YDS_V1::ShardIteratorType::AT_SEQUENCE_NUMBER, - NYDS_V1::TGetShardIteratorSettings().StartingSequenceNumber("100") - ).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - { - auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000001", - YDS_V1::ShardIteratorType::AT_SEQUENCE_NUMBER, - NYDS_V1::TGetShardIteratorSettings().StartingSequenceNumber("001122") - ).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - { - auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000001", - YDS_V1::ShardIteratorType::AT_SEQUENCE_NUMBER, - NYDS_V1::TGetShardIteratorSettings().StartingSequenceNumber("garbage_value") - ).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST); - } - - { - auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000001", - YDS_V1::ShardIteratorType::AT_TIMESTAMP, - NYDS_V1::TGetShardIteratorSettings().Timestamp(TInstant::Now().MilliSeconds() * 0.99) - ).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - - NKikimrPQ::TYdsShardIterator protoShardIterator; - TString decoded; - Base64Decode(result.GetResult().shard_iterator(), decoded); - Y_VERIFY(protoShardIterator.ParseFromString(decoded)); - UNIT_ASSERT_VALUES_EQUAL(protoShardIterator.GetStreamName(), "/Root/" + streamName); - } - - { - auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000001", - YDS_V1::ShardIteratorType::AT_TIMESTAMP, - NYDS_V1::TGetShardIteratorSettings().Timestamp(TInstant::Now().MilliSeconds() * 1.3) - ).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST); - } - } - + const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; + { + auto result = testServer.DataStreamsClient->CreateStream(streamName, \ + NYDS_V1::TCreateStreamSettings().ShardCount(5) + ).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + + { + std::vector<NYDS_V1::TDataRecord> records; + for (ui32 i = 1; i <= 30; ++i) { + TString data = Sprintf("%04u", i); + records.push_back({data, data, ""}); + } + + auto result = testServer.DataStreamsClient->PutRecords(streamName, records).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + if (result.GetStatus() != EStatus::SUCCESS) { + result.GetIssues().PrintTo(Cerr); + } + } + + { + auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000001", + YDS_V1::ShardIteratorType::LATEST).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + + { + auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000010", + YDS_V1::ShardIteratorType::LATEST).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST); + } + + { + auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000001", + YDS_V1::ShardIteratorType::AT_SEQUENCE_NUMBER, + NYDS_V1::TGetShardIteratorSettings().StartingSequenceNumber("0") + ).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + + { + auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000001", + YDS_V1::ShardIteratorType::AFTER_SEQUENCE_NUMBER, + NYDS_V1::TGetShardIteratorSettings().StartingSequenceNumber("0") + ).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + + { + auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000001", + YDS_V1::ShardIteratorType::AT_SEQUENCE_NUMBER, + NYDS_V1::TGetShardIteratorSettings().StartingSequenceNumber("100") + ).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + + { + auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000001", + YDS_V1::ShardIteratorType::AT_SEQUENCE_NUMBER, + NYDS_V1::TGetShardIteratorSettings().StartingSequenceNumber("001122") + ).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + + { + auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000001", + YDS_V1::ShardIteratorType::AT_SEQUENCE_NUMBER, + NYDS_V1::TGetShardIteratorSettings().StartingSequenceNumber("garbage_value") + ).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST); + } + + { + auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000001", + YDS_V1::ShardIteratorType::AT_TIMESTAMP, + NYDS_V1::TGetShardIteratorSettings().Timestamp(TInstant::Now().MilliSeconds() * 0.99) + ).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + + NKikimrPQ::TYdsShardIterator protoShardIterator; + TString decoded; + Base64Decode(result.GetResult().shard_iterator(), decoded); + Y_VERIFY(protoShardIterator.ParseFromString(decoded)); + UNIT_ASSERT_VALUES_EQUAL(protoShardIterator.GetStreamName(), "/Root/" + streamName); + } + + { + auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000001", + YDS_V1::ShardIteratorType::AT_TIMESTAMP, + NYDS_V1::TGetShardIteratorSettings().Timestamp(TInstant::Now().MilliSeconds() * 1.3) + ).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST); + } + } + Y_UNIT_TEST(TestGetRecordsStreamWithSingleShard) { TInsecureDatastreamsTestServer testServer; - const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; - { - auto result = testServer.DataStreamsClient->CreateStream(streamName, NYDS_V1::TCreateStreamSettings().ShardCount(1)).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - const ui32 recordsCount = 30; - std::vector<NYDS_V1::TDataRecord> records; - for (ui32 i = 1; i <= recordsCount; ++i) { - TString data = Sprintf("%04u", i); - records.push_back({data, data, ""}); - } - - { - auto result = testServer.DataStreamsClient->PutRecords(streamName, records).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - if (result.GetStatus() != EStatus::SUCCESS) { - result.GetIssues().PrintTo(Cerr); - } - } - - TString shardIterator; - { - auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000000", - YDS_V1::ShardIteratorType::LATEST).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - shardIterator = result.GetResult().shard_iterator(); - } - - { + const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; + { + auto result = testServer.DataStreamsClient->CreateStream(streamName, NYDS_V1::TCreateStreamSettings().ShardCount(1)).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + + const ui32 recordsCount = 30; + std::vector<NYDS_V1::TDataRecord> records; + for (ui32 i = 1; i <= recordsCount; ++i) { + TString data = Sprintf("%04u", i); + records.push_back({data, data, ""}); + } + + { + auto result = testServer.DataStreamsClient->PutRecords(streamName, records).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + if (result.GetStatus() != EStatus::SUCCESS) { + result.GetIssues().PrintTo(Cerr); + } + } + + TString shardIterator; + { + auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000000", + YDS_V1::ShardIteratorType::LATEST).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + shardIterator = result.GetResult().shard_iterator(); + } + + { auto result = testServer.DataStreamsClient->GetRecords(shardIterator).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), 0); - } - - { - auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000000", - YDS_V1::ShardIteratorType::TRIM_HORIZON).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - shardIterator = result.GetResult().shard_iterator(); - } - - { + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), 0); + } + + { + auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000000", + YDS_V1::ShardIteratorType::TRIM_HORIZON).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + shardIterator = result.GetResult().shard_iterator(); + } + + { auto result = testServer.DataStreamsClient->GetRecords(shardIterator).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), recordsCount); - } - - { - auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000000", - YDS_V1::ShardIteratorType::AT_SEQUENCE_NUMBER, - NYDS_V1::TGetShardIteratorSettings().StartingSequenceNumber("0002") - ).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - shardIterator = result.GetResult().shard_iterator(); - } - - { + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), recordsCount); + } + + { + auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000000", + YDS_V1::ShardIteratorType::AT_SEQUENCE_NUMBER, + NYDS_V1::TGetShardIteratorSettings().StartingSequenceNumber("0002") + ).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + shardIterator = result.GetResult().shard_iterator(); + } + + { auto result = testServer.DataStreamsClient->GetRecords(shardIterator).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), recordsCount - 2); - } - - { - auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000000", - YDS_V1::ShardIteratorType::AFTER_SEQUENCE_NUMBER, - NYDS_V1::TGetShardIteratorSettings().StartingSequenceNumber("0002") - ).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - shardIterator = result.GetResult().shard_iterator(); - } - - { + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), recordsCount - 2); + } + + { + auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000000", + YDS_V1::ShardIteratorType::AFTER_SEQUENCE_NUMBER, + NYDS_V1::TGetShardIteratorSettings().StartingSequenceNumber("0002") + ).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + shardIterator = result.GetResult().shard_iterator(); + } + + { auto result = testServer.DataStreamsClient->GetRecords(shardIterator).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), recordsCount - 3); - } - - - { - auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000000", - YDS_V1::ShardIteratorType::LATEST).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - shardIterator = result.GetResult().shard_iterator(); - } - - { - auto result = testServer.DataStreamsClient->PutRecords(streamName, records).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - if (result.GetStatus() != EStatus::SUCCESS) { - result.GetIssues().PrintTo(Cerr); - } - } - - { + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), recordsCount - 3); + } + + + { + auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000000", + YDS_V1::ShardIteratorType::LATEST).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + shardIterator = result.GetResult().shard_iterator(); + } + + { + auto result = testServer.DataStreamsClient->PutRecords(streamName, records).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + if (result.GetStatus() != EStatus::SUCCESS) { + result.GetIssues().PrintTo(Cerr); + } + } + + { auto result = testServer.DataStreamsClient->GetRecords(shardIterator).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), recordsCount); - } - - } - + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), recordsCount); + } + + } + Y_UNIT_TEST(TestGetRecordsWithoutPermission) { TInsecureDatastreamsTestServer testServer; - const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; + const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; SET_YDS_LOCALS; - { - auto result = testServer.DataStreamsClient->CreateStream("/Root/" + streamName, NYDS_V1::TCreateStreamSettings().ShardCount(1)).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - std::string data; - data.resize(1 << 10); - std::iota(data.begin(), data.end(), 1); - TString id{"0000"}; - { - NYDS_V1::TDataRecord dataRecord{{data.begin(), data.end()}, id, ""}; - auto result = testServer.DataStreamsClient->PutRecord(streamName, dataRecord).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - NYDS_V1::TDataStreamsClient client(*driver, TCommonClientSettings().AuthToken("user2@builtin")); + { + auto result = testServer.DataStreamsClient->CreateStream("/Root/" + streamName, NYDS_V1::TCreateStreamSettings().ShardCount(1)).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + + std::string data; + data.resize(1 << 10); + std::iota(data.begin(), data.end(), 1); + TString id{"0000"}; + { + NYDS_V1::TDataRecord dataRecord{{data.begin(), data.end()}, id, ""}; + auto result = testServer.DataStreamsClient->PutRecord(streamName, dataRecord).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + + NYDS_V1::TDataStreamsClient client(*driver, TCommonClientSettings().AuthToken("user2@builtin")); NYdb::NScheme::TSchemeClient schemeClient(*driver); - - TString shardIterator; - { - auto result = client.GetShardIterator( - streamName, "shard-000000", - YDS_V1::ShardIteratorType::TRIM_HORIZON - ).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR); - } - - { - auto result = client.GetShardIterator( - "/Root/" + streamName, "shard-000000", - YDS_V1::ShardIteratorType::TRIM_HORIZON - ).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR); - } - - { - NYdb::NScheme::TPermissions permissions("user2@builtin", {"ydb.generic.read"}); - auto result = schemeClient.ModifyPermissions("/Root", - NYdb::NScheme::TModifyPermissionsSettings().AddGrantPermissions(permissions) - ).ExtractValueSync(); - UNIT_ASSERT(result.IsSuccess()); - } - - { - auto result = client.GetShardIterator( - streamName, "shard-000000", - YDS_V1::ShardIteratorType::TRIM_HORIZON - ).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - shardIterator = result.GetResult().shard_iterator(); - } - - { - auto result = client.GetRecords(shardIterator, - NYDS_V1::TGetRecordsSettings().Limit(2)).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(std::stoi(result.GetResult().records().begin()->sequence_number()), 0); - } - } - + + TString shardIterator; + { + auto result = client.GetShardIterator( + streamName, "shard-000000", + YDS_V1::ShardIteratorType::TRIM_HORIZON + ).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR); + } + + { + auto result = client.GetShardIterator( + "/Root/" + streamName, "shard-000000", + YDS_V1::ShardIteratorType::TRIM_HORIZON + ).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR); + } + + { + NYdb::NScheme::TPermissions permissions("user2@builtin", {"ydb.generic.read"}); + auto result = schemeClient.ModifyPermissions("/Root", + NYdb::NScheme::TModifyPermissionsSettings().AddGrantPermissions(permissions) + ).ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + } + + { + auto result = client.GetShardIterator( + streamName, "shard-000000", + YDS_V1::ShardIteratorType::TRIM_HORIZON + ).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + shardIterator = result.GetResult().shard_iterator(); + } + + { + auto result = client.GetRecords(shardIterator, + NYDS_V1::TGetRecordsSettings().Limit(2)).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(std::stoi(result.GetResult().records().begin()->sequence_number()), 0); + } + } + Y_UNIT_TEST(TestGetRecords1MBMessagesOneByOneBySeqNo) { TInsecureDatastreamsTestServer testServer; - const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; - { - auto result = testServer.DataStreamsClient->CreateStream(streamName, - NYDS_V1::TCreateStreamSettings().ShardCount(1)).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - std::vector<std::string> put_data; - const ui32 recordsCount = 10; - { - std::string data; - data.resize(1 << 15); - std::iota(data.begin(), data.end(), 1); - std::random_device rd; - std::mt19937 generator{rd()}; - for (ui32 i = 1; i <= recordsCount; ++i) { - std::shuffle(data.begin(), data.end(), generator); - put_data.push_back(data); - { - TString id = Sprintf("%04u", i); - NYDS_V1::TDataRecord dataRecord{{data.begin(), data.end()}, id, ""}; - auto result = testServer.DataStreamsClient->PutRecord(streamName, dataRecord).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - Sleep(TDuration::Seconds(1)); - } - } - - for (ui32 i = 0; i < recordsCount; ++i) { - TString shardIterator; - { - TString id = Sprintf("%04u", i); - auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000000", - YDS_V1::ShardIteratorType::AT_SEQUENCE_NUMBER, - NYDS_V1::TGetShardIteratorSettings().StartingSequenceNumber(id)).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - shardIterator = result.GetResult().shard_iterator(); - } - - { + const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; + { + auto result = testServer.DataStreamsClient->CreateStream(streamName, + NYDS_V1::TCreateStreamSettings().ShardCount(1)).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + + std::vector<std::string> put_data; + const ui32 recordsCount = 10; + { + std::string data; + data.resize(1 << 15); + std::iota(data.begin(), data.end(), 1); + std::random_device rd; + std::mt19937 generator{rd()}; + for (ui32 i = 1; i <= recordsCount; ++i) { + std::shuffle(data.begin(), data.end(), generator); + put_data.push_back(data); + { + TString id = Sprintf("%04u", i); + NYDS_V1::TDataRecord dataRecord{{data.begin(), data.end()}, id, ""}; + auto result = testServer.DataStreamsClient->PutRecord(streamName, dataRecord).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + Sleep(TDuration::Seconds(1)); + } + } + + for (ui32 i = 0; i < recordsCount; ++i) { + TString shardIterator; + { + TString id = Sprintf("%04u", i); + auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000000", + YDS_V1::ShardIteratorType::AT_SEQUENCE_NUMBER, + NYDS_V1::TGetShardIteratorSettings().StartingSequenceNumber(id)).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + shardIterator = result.GetResult().shard_iterator(); + } + + { auto result = testServer.DataStreamsClient->GetRecords(shardIterator, - NYDS_V1::TGetRecordsSettings().Limit(1)).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), 1); - UNIT_ASSERT_VALUES_EQUAL( - std::stoi(result.GetResult().records().begin()->sequence_number()), i); - UNIT_ASSERT_VALUES_EQUAL(MD5::Calc(result.GetResult().records().begin()->data()), - MD5::Calc(put_data[i])); - } - } - } - + NYDS_V1::TGetRecordsSettings().Limit(1)).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), 1); + UNIT_ASSERT_VALUES_EQUAL( + std::stoi(result.GetResult().records().begin()->sequence_number()), i); + UNIT_ASSERT_VALUES_EQUAL(MD5::Calc(result.GetResult().records().begin()->data()), + MD5::Calc(put_data[i])); + } + } + } + Y_UNIT_TEST(TestGetRecords1MBMessagesOneByOneByTS) { TInsecureDatastreamsTestServer testServer; - const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; - { - auto result = testServer.DataStreamsClient->CreateStream(streamName, - NYDS_V1::TCreateStreamSettings().ShardCount(1)).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - const ui32 recordsCount = 24; - std::vector<ui64> timestamps; - { - std::string data; - // data.resize((1 << 21) + 2); - data.resize(1 << 15); - std::iota(data.begin(), data.end(), 1); - std::random_device rd; - std::mt19937 generator{rd()}; - for (ui32 i = 1; i <= recordsCount; ++i) { - std::shuffle(data.begin(), data.end(), generator); - { - TString id = Sprintf("%04u", i); - NYDS_V1::TDataRecord dataRecord{{data.begin(), data.end()}, id, ""}; - timestamps.push_back(TInstant::Now().MilliSeconds()); - auto result = testServer.DataStreamsClient->PutRecord(streamName, dataRecord).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - Sleep(TDuration::Seconds(1)); - } - } - - for (ui32 i = 0; i < recordsCount; ++i) { - TString shardIterator; - { - TString id = Sprintf("%04u", i); - auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000000", - YDS_V1::ShardIteratorType::AT_TIMESTAMP, - NYDS_V1::TGetShardIteratorSettings().Timestamp(timestamps[i])).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - shardIterator = result.GetResult().shard_iterator(); - } - - { + const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; + { + auto result = testServer.DataStreamsClient->CreateStream(streamName, + NYDS_V1::TCreateStreamSettings().ShardCount(1)).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + + const ui32 recordsCount = 24; + std::vector<ui64> timestamps; + { + std::string data; + // data.resize((1 << 21) + 2); + data.resize(1 << 15); + std::iota(data.begin(), data.end(), 1); + std::random_device rd; + std::mt19937 generator{rd()}; + for (ui32 i = 1; i <= recordsCount; ++i) { + std::shuffle(data.begin(), data.end(), generator); + { + TString id = Sprintf("%04u", i); + NYDS_V1::TDataRecord dataRecord{{data.begin(), data.end()}, id, ""}; + timestamps.push_back(TInstant::Now().MilliSeconds()); + auto result = testServer.DataStreamsClient->PutRecord(streamName, dataRecord).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + Sleep(TDuration::Seconds(1)); + } + } + + for (ui32 i = 0; i < recordsCount; ++i) { + TString shardIterator; + { + TString id = Sprintf("%04u", i); + auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000000", + YDS_V1::ShardIteratorType::AT_TIMESTAMP, + NYDS_V1::TGetShardIteratorSettings().Timestamp(timestamps[i])).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + shardIterator = result.GetResult().shard_iterator(); + } + + { auto result = testServer.DataStreamsClient->GetRecords(shardIterator, - NYDS_V1::TGetRecordsSettings().Limit(1)).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(std::stoi(result.GetResult().records().begin()->sequence_number()), i); - } - } - - } - + NYDS_V1::TGetRecordsSettings().Limit(1)).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(std::stoi(result.GetResult().records().begin()->sequence_number()), i); + } + } + + } + Y_UNIT_TEST(TestGetRecordsStreamWithMultipleShards) { TInsecureDatastreamsTestServer testServer; - const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; - { - auto result = testServer.DataStreamsClient->CreateStream(streamName, NYDS_V1::TCreateStreamSettings().ShardCount(5)).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - const ui32 recordsCount = 30; - { - std::vector<NYDS_V1::TDataRecord> records; - for (ui32 i = 1; i <= recordsCount; ++i) { - TString data = Sprintf("%04u", i); - records.push_back({data, data, ""}); - } - - auto result = testServer.DataStreamsClient->PutRecords(streamName, records).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - if (result.GetStatus() != EStatus::SUCCESS) { - result.GetIssues().PrintTo(Cerr); - } - } - - TString shardIterator; - { - auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000001", - YDS_V1::ShardIteratorType::LATEST).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - shardIterator = result.GetResult().shard_iterator(); - } - - { + const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; + { + auto result = testServer.DataStreamsClient->CreateStream(streamName, NYDS_V1::TCreateStreamSettings().ShardCount(5)).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + + const ui32 recordsCount = 30; + { + std::vector<NYDS_V1::TDataRecord> records; + for (ui32 i = 1; i <= recordsCount; ++i) { + TString data = Sprintf("%04u", i); + records.push_back({data, data, ""}); + } + + auto result = testServer.DataStreamsClient->PutRecords(streamName, records).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + if (result.GetStatus() != EStatus::SUCCESS) { + result.GetIssues().PrintTo(Cerr); + } + } + + TString shardIterator; + { + auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000001", + YDS_V1::ShardIteratorType::LATEST).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + shardIterator = result.GetResult().shard_iterator(); + } + + { auto result = testServer.DataStreamsClient->GetRecords(shardIterator).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), 0); - } - - { - auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000002", - YDS_V1::ShardIteratorType::LATEST).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - shardIterator = result.GetResult().shard_iterator(); - } - - { + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), 0); + } + + { + auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000002", + YDS_V1::ShardIteratorType::LATEST).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + shardIterator = result.GetResult().shard_iterator(); + } + + { auto result = testServer.DataStreamsClient->GetRecords(shardIterator).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), 0); - } - - { - auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000003", - YDS_V1::ShardIteratorType::LATEST).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - shardIterator = result.GetResult().shard_iterator(); - } - - { + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), 0); + } + + { + auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000003", + YDS_V1::ShardIteratorType::LATEST).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + shardIterator = result.GetResult().shard_iterator(); + } + + { auto result = testServer.DataStreamsClient->GetRecords(shardIterator).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), 0); - } - - { - auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000002", - YDS_V1::ShardIteratorType::LATEST).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - shardIterator = result.GetResult().shard_iterator(); - } - - { + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), 0); + } + + { + auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000002", + YDS_V1::ShardIteratorType::LATEST).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + shardIterator = result.GetResult().shard_iterator(); + } + + { auto result = testServer.DataStreamsClient->GetRecords(shardIterator).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), 0); - } - } - + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), 0); + } + } + Y_UNIT_TEST(TestListShards1Shard) { TInsecureDatastreamsTestServer testServer; - const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; - { - auto result = testServer.DataStreamsClient->CreateStream(streamName, - NYDS_V1::TCreateStreamSettings().ShardCount(1)).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - { - auto result = testServer.DataStreamsClient->ListShards(streamName, {}, - NYDS_V1::TListShardsSettings()).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - { - YDS_V1::ShardFilter filter; - filter.set_type(YDS_V1::ShardFilter::FROM_TRIM_HORIZON); - auto result = testServer.DataStreamsClient->ListShards(streamName, filter, - NYDS_V1::TListShardsSettings()).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT_VALUES_EQUAL(result.GetResult().shards().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(result.GetResult().shards().begin()->shard_id(), "shard-000000"); - UNIT_ASSERT_VALUES_UNEQUAL(result.GetResult().shards().begin()->shard_id(), "shard-000001"); - } - - { - YDS_V1::ShardFilter filter; - filter.set_type(YDS_V1::ShardFilter::AT_TRIM_HORIZON); - auto result = testServer.DataStreamsClient->ListShards(streamName, filter, - NYDS_V1::TListShardsSettings()).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT_VALUES_EQUAL(result.GetResult().shards().size(), 1); - } - - { - YDS_V1::ShardFilter filter; - filter.set_type(YDS_V1::ShardFilter::AFTER_SHARD_ID); - filter.set_shard_id("00000"); - auto result = testServer.DataStreamsClient->ListShards(streamName, filter, - NYDS_V1::TListShardsSettings()).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - UNIT_ASSERT_VALUES_EQUAL(result.GetResult().shards().size(), 1); - } - - { - YDS_V1::ShardFilter filter; - filter.set_type(YDS_V1::ShardFilter::AFTER_SHARD_ID); - auto result = testServer.DataStreamsClient->ListShards(streamName, filter, - NYDS_V1::TListShardsSettings()).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST); - } - } - + const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME; + { + auto result = testServer.DataStreamsClient->CreateStream(streamName, + NYDS_V1::TCreateStreamSettings().ShardCount(1)).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + + { + auto result = testServer.DataStreamsClient->ListShards(streamName, {}, + NYDS_V1::TListShardsSettings()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + + { + YDS_V1::ShardFilter filter; + filter.set_type(YDS_V1::ShardFilter::FROM_TRIM_HORIZON); + auto result = testServer.DataStreamsClient->ListShards(streamName, filter, + NYDS_V1::TListShardsSettings()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + UNIT_ASSERT_VALUES_EQUAL(result.GetResult().shards().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(result.GetResult().shards().begin()->shard_id(), "shard-000000"); + UNIT_ASSERT_VALUES_UNEQUAL(result.GetResult().shards().begin()->shard_id(), "shard-000001"); + } + + { + YDS_V1::ShardFilter filter; + filter.set_type(YDS_V1::ShardFilter::AT_TRIM_HORIZON); + auto result = testServer.DataStreamsClient->ListShards(streamName, filter, + NYDS_V1::TListShardsSettings()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + UNIT_ASSERT_VALUES_EQUAL(result.GetResult().shards().size(), 1); + } + + { + YDS_V1::ShardFilter filter; + filter.set_type(YDS_V1::ShardFilter::AFTER_SHARD_ID); + filter.set_shard_id("00000"); + auto result = testServer.DataStreamsClient->ListShards(streamName, filter, + NYDS_V1::TListShardsSettings()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + UNIT_ASSERT_VALUES_EQUAL(result.GetResult().shards().size(), 1); + } + + { + YDS_V1::ShardFilter filter; + filter.set_type(YDS_V1::ShardFilter::AFTER_SHARD_ID); + auto result = testServer.DataStreamsClient->ListShards(streamName, filter, + NYDS_V1::TListShardsSettings()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST); + } + } + Y_UNIT_TEST(TestUnsupported) { TInsecureDatastreamsTestServer testServer; - { + { auto result = testServer.DataStreamsClient->DescribeLimits().ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNSUPPORTED); diff --git a/ydb/services/datastreams/next_token.cpp b/ydb/services/datastreams/next_token.cpp index f3edad9197..ac8d24514c 100644 --- a/ydb/services/datastreams/next_token.cpp +++ b/ydb/services/datastreams/next_token.cpp @@ -1 +1 @@ -#include "next_token.h" +#include "next_token.h" diff --git a/ydb/services/datastreams/next_token.h b/ydb/services/datastreams/next_token.h index 1fe61997bc..5a14b50d19 100644 --- a/ydb/services/datastreams/next_token.h +++ b/ydb/services/datastreams/next_token.h @@ -1,82 +1,82 @@ -#pragma once - +#pragma once + #include <ydb/core/protos/pqconfig.pb.h> -#include <library/cpp/string_utils/base64/base64.h> -#include <util/datetime/base.h> - -namespace NKikimr::NDataStreams::V1 { - - class TNextToken { - public: - static constexpr ui32 LIFETIME_MS = 300*1000; - - TNextToken(const TString& nextToken) - : Valid{true} - { - try { - TString decoded; - Base64Decode(nextToken, decoded); - auto ok = Proto.ParseFromString(decoded); - if (ok) { - Valid = IsAlive(TInstant::Now().MilliSeconds()); - } else { - Valid = false; - } - } catch (std::exception&) { - Valid = false; - } - } - - TNextToken(const TString& streamArn, ui32 alreadyRead, ui32 maxResults, ui64 creationTimestamp) - : Valid{true} - { - Proto.SetStreamArn(streamArn); - Proto.SetAlreadyRead(alreadyRead); - Proto.SetMaxResults(maxResults); - Proto.SetCreationTimestamp(creationTimestamp); - } - - TString Serialize() const { - TString data; - bool result = Proto.SerializeToString(&data); - Y_VERIFY(result); - TString encoded; - Base64Encode(data, encoded); - return encoded; - } - - ui32 GetAlreadyRead() const { - return Proto.GetAlreadyRead(); - } - - TString GetStreamArn() const { - return Proto.GetStreamArn(); - } - - TString GetStreamName() const { - return Proto.GetStreamArn(); - } - - ui32 GetMaxResults() const { - return Proto.GetMaxResults(); - } - - ui32 GetCreationTimestamp() const { - return Proto.GetCreationTimestamp(); - } - - bool IsAlive(ui64 now) const { - return now >= Proto.GetCreationTimestamp() && - (now - Proto.GetCreationTimestamp()) < LIFETIME_MS; - } - - bool IsValid() const { - return Valid && Proto.GetStreamArn().size() > 0; - } - - private: - bool Valid; - NKikimrPQ::TYdsNextToken Proto; - }; - -} // namespace NKikimr::NDataStreams::V1 +#include <library/cpp/string_utils/base64/base64.h> +#include <util/datetime/base.h> + +namespace NKikimr::NDataStreams::V1 { + + class TNextToken { + public: + static constexpr ui32 LIFETIME_MS = 300*1000; + + TNextToken(const TString& nextToken) + : Valid{true} + { + try { + TString decoded; + Base64Decode(nextToken, decoded); + auto ok = Proto.ParseFromString(decoded); + if (ok) { + Valid = IsAlive(TInstant::Now().MilliSeconds()); + } else { + Valid = false; + } + } catch (std::exception&) { + Valid = false; + } + } + + TNextToken(const TString& streamArn, ui32 alreadyRead, ui32 maxResults, ui64 creationTimestamp) + : Valid{true} + { + Proto.SetStreamArn(streamArn); + Proto.SetAlreadyRead(alreadyRead); + Proto.SetMaxResults(maxResults); + Proto.SetCreationTimestamp(creationTimestamp); + } + + TString Serialize() const { + TString data; + bool result = Proto.SerializeToString(&data); + Y_VERIFY(result); + TString encoded; + Base64Encode(data, encoded); + return encoded; + } + + ui32 GetAlreadyRead() const { + return Proto.GetAlreadyRead(); + } + + TString GetStreamArn() const { + return Proto.GetStreamArn(); + } + + TString GetStreamName() const { + return Proto.GetStreamArn(); + } + + ui32 GetMaxResults() const { + return Proto.GetMaxResults(); + } + + ui32 GetCreationTimestamp() const { + return Proto.GetCreationTimestamp(); + } + + bool IsAlive(ui64 now) const { + return now >= Proto.GetCreationTimestamp() && + (now - Proto.GetCreationTimestamp()) < LIFETIME_MS; + } + + bool IsValid() const { + return Valid && Proto.GetStreamArn().size() > 0; + } + + private: + bool Valid; + NKikimrPQ::TYdsNextToken Proto; + }; + +} // namespace NKikimr::NDataStreams::V1 diff --git a/ydb/services/datastreams/put_records_actor.h b/ydb/services/datastreams/put_records_actor.h index 8f06a99157..b3cdb66855 100644 --- a/ydb/services/datastreams/put_records_actor.h +++ b/ydb/services/datastreams/put_records_actor.h @@ -107,7 +107,7 @@ namespace NKikimr::NDataStreams::V1 { w->SetDisableDeduplication(true); w->SetCreateTimeMS(TInstant::Now().MilliSeconds()); w->SetUncompressedSize(item.Data.size()); - w->SetExternalOperation(true); + w->SetExternalOperation(true); totalSize += (item.Data.size() + item.Key.size() + item.ExplicitHash.size()); } ui64 putUnitsCount = totalSize / PUT_UNIT_SIZE; @@ -217,7 +217,7 @@ namespace NKikimr::NDataStreams::V1 { void Bootstrap(const NActors::TActorContext &ctx); void PreparePartitionActors(const NActors::TActorContext& ctx); - void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx); + void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx); protected: void AddRecord(THashMap<ui32, TVector<TPutRecordsItem>>& items, ui32 totalShardsCount, int index); @@ -244,7 +244,7 @@ namespace NKikimr::NDataStreams::V1 { STFUNC(StateFunc) { switch (ev->GetTypeRewrite()) { HFunc(NDataStreams::V1::TEvDataStreams::TEvPartitionActorResult, Handle); - default: TBase::StateWork(ev, ctx); + default: TBase::StateWork(ev, ctx); }; } }; @@ -289,10 +289,10 @@ namespace NKikimr::NDataStreams::V1 { } template<class TDerived, class TProto> - void TPutRecordsActorBase<TDerived, TProto>::HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) { - if (TBase::ReplyIfNotTopic(ev, ctx)) { - return; - } + void TPutRecordsActorBase<TDerived, TProto>::HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) { + if (TBase::ReplyIfNotTopic(ev, ctx)) { + return; + } const NSchemeCache::TSchemeCacheNavigate* navigate = ev->Get()->Request.Get(); auto topicInfo = navigate->ResultSet.begin(); @@ -301,9 +301,9 @@ namespace NKikimr::NDataStreams::V1 { this->Request_->GetInternalToken())) { return this->ReplyWithError(Ydb::StatusIds::UNAUTHORIZED, Ydb::PersQueue::ErrorCode::ACCESS_DENIED, TStringBuilder() << "Access for stream " - << this->GetProtoRequest()->stream_name() - << " is denied for subject " - << this->Request_->GetInternalToken(), ctx); + << this->GetProtoRequest()->stream_name() + << " is denied for subject " + << this->Request_->GetInternalToken(), ctx); } } @@ -319,8 +319,8 @@ namespace NKikimr::NDataStreams::V1 { auto part = partition.GetPartitionId(); if (items[part].empty()) continue; PartitionToActor[part].ActorId = ctx.Register( - new TDatastreamsPartitionActor(ctx.SelfID, partition.GetTabletId(), part, this->GetTopicPath(ctx), std::move(items[part])) - ); + new TDatastreamsPartitionActor(ctx.SelfID, partition.GetTabletId(), part, this->GetTopicPath(ctx), std::move(items[part])) + ); } this->CheckFinish(ctx); } diff --git a/ydb/services/datastreams/shard_iterator.cpp b/ydb/services/datastreams/shard_iterator.cpp index 2855ec2d52..9f0fcec7f6 100644 --- a/ydb/services/datastreams/shard_iterator.cpp +++ b/ydb/services/datastreams/shard_iterator.cpp @@ -1 +1 @@ -#include "shard_iterator.h" +#include "shard_iterator.h" diff --git a/ydb/services/datastreams/shard_iterator.h b/ydb/services/datastreams/shard_iterator.h index 7b6ce9c5cd..1061256b4f 100644 --- a/ydb/services/datastreams/shard_iterator.h +++ b/ydb/services/datastreams/shard_iterator.h @@ -1,77 +1,77 @@ -#pragma once - +#pragma once + #include <ydb/core/protos/pqconfig.pb.h> -#include <library/cpp/string_utils/base64/base64.h> -#include <util/datetime/base.h> - -namespace NKikimr::NDataStreams::V1 { - - class TShardIterator { - public: - static constexpr ui64 LIFETIME_MS = 5*60*1000; - - TShardIterator(const TString& iteratorStr) { - try { - TString decoded; - Base64Decode(iteratorStr, decoded); - Valid = Proto.ParseFromString(decoded) && IsAlive(TInstant::Now().MilliSeconds()); - } catch (std::exception&) { - Valid = false; - } - } - - TShardIterator(const TString& streamName, const TString& streamArn, - ui32 shardId, ui64 readTimestamp, ui32 sequenceNumber) { - Proto.SetStreamName(streamName); - Proto.SetStreamArn(streamArn); - Proto.SetShardId(shardId); - Proto.SetReadTimestampMs(readTimestamp); - Proto.SetSequenceNumber(sequenceNumber); - Proto.SetCreationTimestampMs(TInstant::Now().MilliSeconds()); - Valid = true; - } - - TString Serialize() const { - TString data; - bool result = Proto.SerializeToString(&data); - Y_VERIFY(result); - TString encoded; - Base64Encode(data, encoded); - return encoded; - } - - TString GetStreamName() const { - return Proto.GetStreamName(); - } - - TString GetStreamArn() const { - return Proto.GetStreamArn(); - } - - ui32 GetShardId() const { - return Proto.GetShardId(); - } - - ui64 GetReadTimestamp() const { - return Proto.GetReadTimestampMs(); - } - - ui32 GetSequenceNumber() const { - return Proto.GetSequenceNumber(); - } - - bool IsAlive(ui64 now) const { - return now >= Proto.GetCreationTimestampMs() && now - - Proto.GetCreationTimestampMs() < LIFETIME_MS; - } - - bool IsValid() const { - return Valid; - } - - private: - bool Valid; - NKikimrPQ::TYdsShardIterator Proto; - }; - -} // namespace NKikimr::NDataStreams::V1 +#include <library/cpp/string_utils/base64/base64.h> +#include <util/datetime/base.h> + +namespace NKikimr::NDataStreams::V1 { + + class TShardIterator { + public: + static constexpr ui64 LIFETIME_MS = 5*60*1000; + + TShardIterator(const TString& iteratorStr) { + try { + TString decoded; + Base64Decode(iteratorStr, decoded); + Valid = Proto.ParseFromString(decoded) && IsAlive(TInstant::Now().MilliSeconds()); + } catch (std::exception&) { + Valid = false; + } + } + + TShardIterator(const TString& streamName, const TString& streamArn, + ui32 shardId, ui64 readTimestamp, ui32 sequenceNumber) { + Proto.SetStreamName(streamName); + Proto.SetStreamArn(streamArn); + Proto.SetShardId(shardId); + Proto.SetReadTimestampMs(readTimestamp); + Proto.SetSequenceNumber(sequenceNumber); + Proto.SetCreationTimestampMs(TInstant::Now().MilliSeconds()); + Valid = true; + } + + TString Serialize() const { + TString data; + bool result = Proto.SerializeToString(&data); + Y_VERIFY(result); + TString encoded; + Base64Encode(data, encoded); + return encoded; + } + + TString GetStreamName() const { + return Proto.GetStreamName(); + } + + TString GetStreamArn() const { + return Proto.GetStreamArn(); + } + + ui32 GetShardId() const { + return Proto.GetShardId(); + } + + ui64 GetReadTimestamp() const { + return Proto.GetReadTimestampMs(); + } + + ui32 GetSequenceNumber() const { + return Proto.GetSequenceNumber(); + } + + bool IsAlive(ui64 now) const { + return now >= Proto.GetCreationTimestampMs() && now - + Proto.GetCreationTimestampMs() < LIFETIME_MS; + } + + bool IsValid() const { + return Valid; + } + + private: + bool Valid; + NKikimrPQ::TYdsShardIterator Proto; + }; + +} // namespace NKikimr::NDataStreams::V1 diff --git a/ydb/services/datastreams/ya.make b/ydb/services/datastreams/ya.make index 0a26e96de9..d2635df981 100644 --- a/ydb/services/datastreams/ya.make +++ b/ydb/services/datastreams/ya.make @@ -5,9 +5,9 @@ OWNER(g:kikimr) SRCS( datastreams_proxy.cpp grpc_service.cpp - next_token.cpp + next_token.cpp put_records_actor.cpp - shard_iterator.cpp + shard_iterator.cpp ) PEERDIR( diff --git a/ydb/services/lib/actors/pq_schema_actor.cpp b/ydb/services/lib/actors/pq_schema_actor.cpp index f48e398488..fc2e779e39 100644 --- a/ydb/services/lib/actors/pq_schema_actor.cpp +++ b/ydb/services/lib/actors/pq_schema_actor.cpp @@ -125,39 +125,39 @@ namespace NKikimr::NGRpcProxy::V1 { const TString& consumerName, const TActorContext& ctx ) { - THashSet<TString> rulesToRemove; - rulesToRemove.insert(consumerName); - - config->ClearReadRuleVersions(); - config->ClearReadRules(); - config->ClearReadFromTimestampsMs(); - config->ClearConsumerFormatVersions(); - config->ClearConsumerCodecs(); - config->MutablePartitionConfig()->ClearImportantClientId(); + THashSet<TString> rulesToRemove; + rulesToRemove.insert(consumerName); + + config->ClearReadRuleVersions(); + config->ClearReadRules(); + config->ClearReadFromTimestampsMs(); + config->ClearConsumerFormatVersions(); + config->ClearConsumerCodecs(); + config->MutablePartitionConfig()->ClearImportantClientId(); config->ClearReadRuleServiceTypes(); - - for (const auto& importantConsumer : originalConfig.GetPartitionConfig().GetImportantClientId()) { - if (rulesToRemove.find(importantConsumer) == rulesToRemove.end()) { - config->MutablePartitionConfig()->AddImportantClientId(importantConsumer); - } - } - + + for (const auto& importantConsumer : originalConfig.GetPartitionConfig().GetImportantClientId()) { + if (rulesToRemove.find(importantConsumer) == rulesToRemove.end()) { + config->MutablePartitionConfig()->AddImportantClientId(importantConsumer); + } + } + const auto& pqConfig = AppData(ctx)->PQConfig; - for (size_t i = 0; i < originalConfig.ReadRulesSize(); i++) { - if (auto it = rulesToRemove.find(originalConfig.GetReadRules(i)); it != rulesToRemove.end()) { - rulesToRemove.erase(it); - continue; - } - - config->AddReadRuleVersions(originalConfig.GetReadRuleVersions(i)); - config->AddReadRules(originalConfig.GetReadRules(i)); - config->AddReadFromTimestampsMs(originalConfig.GetReadFromTimestampsMs(i)); - config->AddConsumerFormatVersions(originalConfig.GetConsumerFormatVersions(i)); - auto ct = config->AddConsumerCodecs(); - for (size_t j = 0; j < originalConfig.GetConsumerCodecs(i).CodecsSize(); j++) { - ct->AddCodecs(originalConfig.GetConsumerCodecs(i).GetCodecs(j)); - ct->AddIds(originalConfig.GetConsumerCodecs(i).GetIds(j)); - } + for (size_t i = 0; i < originalConfig.ReadRulesSize(); i++) { + if (auto it = rulesToRemove.find(originalConfig.GetReadRules(i)); it != rulesToRemove.end()) { + rulesToRemove.erase(it); + continue; + } + + config->AddReadRuleVersions(originalConfig.GetReadRuleVersions(i)); + config->AddReadRules(originalConfig.GetReadRules(i)); + config->AddReadFromTimestampsMs(originalConfig.GetReadFromTimestampsMs(i)); + config->AddConsumerFormatVersions(originalConfig.GetConsumerFormatVersions(i)); + auto ct = config->AddConsumerCodecs(); + for (size_t j = 0; j < originalConfig.GetConsumerCodecs(i).CodecsSize(); j++) { + ct->AddCodecs(originalConfig.GetConsumerCodecs(i).GetCodecs(j)); + ct->AddIds(originalConfig.GetConsumerCodecs(i).GetIds(j)); + } if (i < originalConfig.ReadRuleServiceTypesSize()) { config->AddReadRuleServiceTypes(originalConfig.GetReadRuleServiceTypes(i)); } else { @@ -167,15 +167,15 @@ namespace NKikimr::NGRpcProxy::V1 { } config->AddReadRuleServiceTypes(pqConfig.GetDefaultClientServiceType().GetName()); } - } - - if (rulesToRemove.size() > 0) { - return TStringBuilder() << "Rule for consumer " << *rulesToRemove.begin() << " doesn't exist"; - } - - return ""; - } - + } + + if (rulesToRemove.size() > 0) { + return TStringBuilder() << "Rule for consumer " << *rulesToRemove.begin() << " doesn't exist"; + } + + return ""; + } + bool CheckReadRulesConfig(const NKikimrPQ::TPQTabletConfig& config, const TClientServiceTypes& supportedClientServiceTypes, TString& error) { diff --git a/ydb/services/lib/actors/pq_schema_actor.h b/ydb/services/lib/actors/pq_schema_actor.h index 1be2e7fa24..d798f6f9ad 100644 --- a/ydb/services/lib/actors/pq_schema_actor.h +++ b/ydb/services/lib/actors/pq_schema_actor.h @@ -48,7 +48,7 @@ namespace NKikimr::NGRpcProxy::V1 { template<class TDerived, class TRequest> class TPQGrpcSchemaBase : public NKikimr::NGRpcService::TRpcSchemeRequestActor<TDerived, TRequest> { - protected: + protected: using TBase = NKikimr::NGRpcService::TRpcSchemeRequestActor<TDerived, TRequest>; public: @@ -127,88 +127,88 @@ namespace NKikimr::NGRpcProxy::V1 { } } - bool ReplyIfNotTopic(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) { - const NSchemeCache::TSchemeCacheNavigate* result = ev->Get()->Request.Get(); - Y_VERIFY(result->ResultSet.size() == 1); - const auto& response = result->ResultSet.front(); - const TString path = JoinPath(response.Path); - - if (ev->Get()->Request.Get()->ResultSet.size() != 1 || - ev->Get()->Request.Get()->ResultSet.begin()->Kind != - NSchemeCache::TSchemeCacheNavigate::KindTopic) { - this->Request_->RaiseIssue( - FillIssue( - TStringBuilder() << "path '" << path << "' is not a stream", - Ydb::PersQueue::ErrorCode::ERROR - ) - ); - TBase::Reply(Ydb::StatusIds::SCHEME_ERROR, ctx); - return true; - } - - return false; - } - - void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) { + bool ReplyIfNotTopic(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) { const NSchemeCache::TSchemeCacheNavigate* result = ev->Get()->Request.Get(); Y_VERIFY(result->ResultSet.size() == 1); const auto& response = result->ResultSet.front(); const TString path = JoinPath(response.Path); - - switch (response.Status) { - case NSchemeCache::TSchemeCacheNavigate::EStatus::Ok: { - return static_cast<TDerived*>(this)->HandleCacheNavigateResponse(ev, ctx); - } - break; - case NSchemeCache::TSchemeCacheNavigate::EStatus::PathErrorUnknown: { - this->Request_->RaiseIssue( - FillIssue( - TStringBuilder() << "path '" << path << "' does not exist or you " << - "do not have access rights", - Ydb::PersQueue::ErrorCode::ERROR - ) - ); - return TBase::Reply(Ydb::StatusIds::SCHEME_ERROR, ctx); - } - case NSchemeCache::TSchemeCacheNavigate::EStatus::TableCreationNotComplete: { - this->Request_->RaiseIssue( - FillIssue( - TStringBuilder() << "table creation is not completed", - Ydb::PersQueue::ErrorCode::ERROR - ) - ); - return TBase::Reply(Ydb::StatusIds::SCHEME_ERROR, ctx); - } - break; - case NSchemeCache::TSchemeCacheNavigate::EStatus::PathNotTable: { - this->Request_->RaiseIssue( - FillIssue( - TStringBuilder() << "path '" << path << "' is not a table", - Ydb::PersQueue::ErrorCode::ERROR - ) - ); - return TBase::Reply(Ydb::StatusIds::SCHEME_ERROR, ctx); - } - break; - case NSchemeCache::TSchemeCacheNavigate::EStatus::RootUnknown: { + + if (ev->Get()->Request.Get()->ResultSet.size() != 1 || + ev->Get()->Request.Get()->ResultSet.begin()->Kind != + NSchemeCache::TSchemeCacheNavigate::KindTopic) { this->Request_->RaiseIssue( FillIssue( - TStringBuilder() << "unknown database root", + TStringBuilder() << "path '" << path << "' is not a stream", + Ydb::PersQueue::ErrorCode::ERROR + ) + ); + TBase::Reply(Ydb::StatusIds::SCHEME_ERROR, ctx); + return true; + } + + return false; + } + + void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) { + const NSchemeCache::TSchemeCacheNavigate* result = ev->Get()->Request.Get(); + Y_VERIFY(result->ResultSet.size() == 1); + const auto& response = result->ResultSet.front(); + const TString path = JoinPath(response.Path); + + switch (response.Status) { + case NSchemeCache::TSchemeCacheNavigate::EStatus::Ok: { + return static_cast<TDerived*>(this)->HandleCacheNavigateResponse(ev, ctx); + } + break; + case NSchemeCache::TSchemeCacheNavigate::EStatus::PathErrorUnknown: { + this->Request_->RaiseIssue( + FillIssue( + TStringBuilder() << "path '" << path << "' does not exist or you " << + "do not have access rights", Ydb::PersQueue::ErrorCode::ERROR ) ); return TBase::Reply(Ydb::StatusIds::SCHEME_ERROR, ctx); } - break; - - default: - return TBase::Reply(Ydb::StatusIds::GENERIC_ERROR, ctx); - } + case NSchemeCache::TSchemeCacheNavigate::EStatus::TableCreationNotComplete: { + this->Request_->RaiseIssue( + FillIssue( + TStringBuilder() << "table creation is not completed", + Ydb::PersQueue::ErrorCode::ERROR + ) + ); + return TBase::Reply(Ydb::StatusIds::SCHEME_ERROR, ctx); + } + break; + case NSchemeCache::TSchemeCacheNavigate::EStatus::PathNotTable: { + this->Request_->RaiseIssue( + FillIssue( + TStringBuilder() << "path '" << path << "' is not a table", + Ydb::PersQueue::ErrorCode::ERROR + ) + ); + return TBase::Reply(Ydb::StatusIds::SCHEME_ERROR, ctx); + } + break; + case NSchemeCache::TSchemeCacheNavigate::EStatus::RootUnknown: { + this->Request_->RaiseIssue( + FillIssue( + TStringBuilder() << "unknown database root", + Ydb::PersQueue::ErrorCode::ERROR + ) + ); + return TBase::Reply(Ydb::StatusIds::SCHEME_ERROR, ctx); + } + break; + + default: + return TBase::Reply(Ydb::StatusIds::GENERIC_ERROR, ctx); + } } - void ReplyWithError(Ydb::StatusIds::StatusCode status, - Ydb::PersQueue::ErrorCode::ErrorCode pqStatus, - const TString& messageText, const NActors::TActorContext& ctx) { + void ReplyWithError(Ydb::StatusIds::StatusCode status, + Ydb::PersQueue::ErrorCode::ErrorCode pqStatus, + const TString& messageText, const NActors::TActorContext& ctx) { this->Request_->RaiseIssue(FillIssue(messageText, pqStatus)); this->Request_->ReplyWithYdbStatus(status); this->Die(ctx); @@ -228,13 +228,13 @@ namespace NKikimr::NGRpcProxy::V1 { IsDead = true; } - void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) { - switch (ev->GetTypeRewrite()) { - HFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, Handle); - default: TBase::StateWork(ev, ctx); - } - } - + void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) { + switch (ev->GetTypeRewrite()) { + HFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, Handle); + default: TBase::StateWork(ev, ctx); + } + } + private: bool IsDead = false; TString TopicPath; @@ -282,16 +282,16 @@ namespace NKikimr::NGRpcProxy::V1 { this->DescribeSchemeResult.Reset(); } - void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) { - const NSchemeCache::TSchemeCacheNavigate* result = ev->Get()->Request.Get(); - Y_VERIFY(result->ResultSet.size() == 1); - DescribeSchemeResult = std::move(ev); - return this->SendProposeRequest(ctx); - } - + void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) { + const NSchemeCache::TSchemeCacheNavigate* result = ev->Get()->Request.Get(); + Y_VERIFY(result->ResultSet.size() == 1); + DescribeSchemeResult = std::move(ev); + return this->SendProposeRequest(ctx); + } + void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) { switch (ev->GetTypeRewrite()) { - default: TBase::StateWork(ev, ctx); + default: TBase::StateWork(ev, ctx); } } @@ -299,4 +299,4 @@ namespace NKikimr::NGRpcProxy::V1 { THolder<NActors::TEventHandle<TEvTxProxySchemeCache::TEvNavigateKeySetResult>> DescribeSchemeResult; }; -} +} diff --git a/ydb/services/persqueue_v1/grpc_pq_actor.h b/ydb/services/persqueue_v1/grpc_pq_actor.h index 65c8aca1db..95be559053 100644 --- a/ydb/services/persqueue_v1/grpc_pq_actor.h +++ b/ydb/services/persqueue_v1/grpc_pq_actor.h @@ -550,8 +550,8 @@ private: void GenerateNextWriteRequest(const NActors::TActorContext& ctx); - void SetupCounters(); - void SetupCounters(const TString& cloudId, const TString& dbId, const TString& folderId); + void SetupCounters(); + void SetupCounters(const TString& cloudId, const TString& dbId, const TString& folderId); private: std::unique_ptr<NKikimr::NGRpcService::TEvStreamPQWriteRequest> Request; @@ -854,13 +854,13 @@ private: void HandleWakeup(const NActors::TActorContext& ctx); void Handle(TEvPQProxy::TEvAuthResultOk::TPtr& ev, const NActors::TActorContext& ctx); - void CloseSession(const TString& errorReason, const PersQueue::ErrorCode::ErrorCode errorCode, - const NActors::TActorContext& ctx); + void CloseSession(const TString& errorReason, const PersQueue::ErrorCode::ErrorCode errorCode, + const NActors::TActorContext& ctx); void SetupCounters(); void SetupTopicCounters(const TString& topic); - void SetupTopicCounters(const TString& topic, const TString& cloudId, const TString& dbId, - const TString& folderId); + void SetupTopicCounters(const TString& topic, const TString& cloudId, const TString& dbId, + const TString& folderId); void ProcessReads(const NActors::TActorContext& ctx); // returns false if actor died struct TFormedReadResponse; @@ -1137,8 +1137,8 @@ public: const TString& workingDir, const TString& name); void Bootstrap(const NActors::TActorContext& ctx); - - void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx){ Y_UNUSED(ev); Y_UNUSED(ctx); } + + void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx){ Y_UNUSED(ev); Y_UNUSED(ctx); } }; class TDescribeTopicActor : public TPQGrpcSchemaBase<TDescribeTopicActor, NKikimr::NGRpcService::TEvPQDescribeTopicRequest> { @@ -1152,7 +1152,7 @@ public: void Bootstrap(const NActors::TActorContext& ctx); - void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx); + void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx); }; @@ -1195,8 +1195,8 @@ public: void Bootstrap(const NActors::TActorContext& ctx); - void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx){ Y_UNUSED(ev); Y_UNUSED(ctx); } - + void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx){ Y_UNUSED(ev); Y_UNUSED(ctx); } + private: TString LocalCluster; TVector<TString> Clusters; @@ -1213,8 +1213,8 @@ public: const TString& workingDir, const TString& name); void Bootstrap(const NActors::TActorContext& ctx); - - void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx){ Y_UNUSED(ev); Y_UNUSED(ctx); } + + void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx){ Y_UNUSED(ev); Y_UNUSED(ctx); } }; diff --git a/ydb/services/persqueue_v1/grpc_pq_read_actor.cpp b/ydb/services/persqueue_v1/grpc_pq_read_actor.cpp index 654edfcfcd..d48604b087 100644 --- a/ydb/services/persqueue_v1/grpc_pq_read_actor.cpp +++ b/ydb/services/persqueue_v1/grpc_pq_read_actor.cpp @@ -779,8 +779,8 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvReadInit::TPtr& ev, const TActorCo auto subGroup = GetServiceCounters(Counters, "pqproxy|SLI"); Aggr = {{{{"Account", ClientPath.substr(0, ClientPath.find("/"))}}, {"total"}}}; - SLIErrors = NKikimr::NPQ::TMultiCounter(subGroup, Aggr, {}, {"RequestsError"}, true, "sensor", false); - SLITotal = NKikimr::NPQ::TMultiCounter(subGroup, Aggr, {}, {"RequestsTotal"}, true, "sensor", false); + SLIErrors = NKikimr::NPQ::TMultiCounter(subGroup, Aggr, {}, {"RequestsError"}, true, "sensor", false); + SLITotal = NKikimr::NPQ::TMultiCounter(subGroup, Aggr, {}, {"RequestsTotal"}, true, "sensor", false); SLITotal.Inc(); } @@ -841,48 +841,48 @@ void TReadSessionActor::SetupCounters() void TReadSessionActor::SetupTopicCounters(const TString& topic) { auto& topicCounters = TopicCounters[topic]; - auto subGroup = GetServiceCounters(Counters, "pqproxy|readSession"); + auto subGroup = GetServiceCounters(Counters, "pqproxy|readSession"); //client/consumerPath Account/Producer OriginDC Topic/TopicPath TVector<NPQ::TLabelsInfo> aggr = NKikimr::NPQ::GetLabels(topic); TVector<std::pair<TString, TString>> cons = {{"Client", ClientId}, {"ConsumerPath", ClientPath}}; - topicCounters.PartitionsLocked = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"PartitionsLocked"}, true); - topicCounters.PartitionsReleased = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"PartitionsReleased"}, true); - topicCounters.PartitionsToBeReleased = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"PartitionsToBeReleased"}, false); - topicCounters.PartitionsToBeLocked = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"PartitionsToBeLocked"}, false); - topicCounters.PartitionsInfly = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"PartitionsInfly"}, false); - topicCounters.Errors = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"PartitionsErrors"}, true); - topicCounters.Commits = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"Commits"}, true); - topicCounters.WaitsForData = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"WaitsForData"}, true); - - topicCounters.CommitLatency = CommitLatency; - topicCounters.SLIBigLatency = SLIBigLatency; - topicCounters.SLITotal = SLITotal; -} - -void TReadSessionActor::SetupTopicCounters(const TString& topic, const TString& cloudId, - const TString& dbId, const TString& folderId) -{ - auto& topicCounters = TopicCounters[topic]; - auto subGroup = NKikimr::NPQ::GetCountersForStream(Counters, "readSession"); -//client/consumerPath Account/Producer OriginDC Topic/TopicPath - TVector<NPQ::TLabelsInfo> aggr = NKikimr::NPQ::GetLabelsForStream(topic, cloudId, dbId, folderId); - TVector<std::pair<TString, TString>> cons{}; - - topicCounters.PartitionsLocked = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"stream.internal_read.partitions_locked_per_second"}, true); - topicCounters.PartitionsReleased = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"stream.internal_read.partitions_released_per_second"}, true); - topicCounters.PartitionsToBeReleased = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"stream.internal_read.partitions_to_be_released"}, false); - topicCounters.PartitionsToBeLocked = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"stream.internal_read.partitions_to_be_locked"}, false); - topicCounters.PartitionsInfly = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"stream.internal_read.partitions_locked"}, false); - topicCounters.Errors = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"stream.internal_read.partitions_errors_per_second"}, true); - topicCounters.Commits = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"stream.internal_read.commits_per_second"}, true); - topicCounters.WaitsForData = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"stream.internal_read.waits_for_data"}, true); + topicCounters.PartitionsLocked = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"PartitionsLocked"}, true); + topicCounters.PartitionsReleased = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"PartitionsReleased"}, true); + topicCounters.PartitionsToBeReleased = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"PartitionsToBeReleased"}, false); + topicCounters.PartitionsToBeLocked = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"PartitionsToBeLocked"}, false); + topicCounters.PartitionsInfly = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"PartitionsInfly"}, false); + topicCounters.Errors = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"PartitionsErrors"}, true); + topicCounters.Commits = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"Commits"}, true); + topicCounters.WaitsForData = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"WaitsForData"}, true); topicCounters.CommitLatency = CommitLatency; topicCounters.SLIBigLatency = SLIBigLatency; topicCounters.SLITotal = SLITotal; } +void TReadSessionActor::SetupTopicCounters(const TString& topic, const TString& cloudId, + const TString& dbId, const TString& folderId) +{ + auto& topicCounters = TopicCounters[topic]; + auto subGroup = NKikimr::NPQ::GetCountersForStream(Counters, "readSession"); +//client/consumerPath Account/Producer OriginDC Topic/TopicPath + TVector<NPQ::TLabelsInfo> aggr = NKikimr::NPQ::GetLabelsForStream(topic, cloudId, dbId, folderId); + TVector<std::pair<TString, TString>> cons{}; + + topicCounters.PartitionsLocked = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"stream.internal_read.partitions_locked_per_second"}, true); + topicCounters.PartitionsReleased = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"stream.internal_read.partitions_released_per_second"}, true); + topicCounters.PartitionsToBeReleased = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"stream.internal_read.partitions_to_be_released"}, false); + topicCounters.PartitionsToBeLocked = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"stream.internal_read.partitions_to_be_locked"}, false); + topicCounters.PartitionsInfly = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"stream.internal_read.partitions_locked"}, false); + topicCounters.Errors = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"stream.internal_read.partitions_errors_per_second"}, true); + topicCounters.Commits = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"stream.internal_read.commits_per_second"}, true); + topicCounters.WaitsForData = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"stream.internal_read.waits_for_data"}, true); + + topicCounters.CommitLatency = CommitLatency; + topicCounters.SLIBigLatency = SLIBigLatency; + topicCounters.SLITotal = SLITotal; +} + void TReadSessionActor::Handle(TEvPQProxy::TEvAuthResultOk::TPtr& ev, const TActorContext& ctx) { LastACLCheckTimestamp = ctx.Now(); @@ -903,11 +903,11 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvAuthResultOk::TPtr& ev, const TAct auto subGroup = GetServiceCounters(Counters, "pqproxy|SLI"); InitLatency = NKikimr::NPQ::CreateSLIDurationCounter(subGroup, Aggr, "ReadInit", initBorder, {100, 200, 500, 1000, 1500, 2000, 5000, 10000, 30000, 99999999}); CommitLatency = NKikimr::NPQ::CreateSLIDurationCounter(subGroup, Aggr, "Commit", AppData(ctx)->PQConfig.GetCommitLatencyBigMs(), {100, 200, 500, 1000, 1500, 2000, 5000, 10000, 30000, 99999999}); - SLIBigLatency = NKikimr::NPQ::TMultiCounter(subGroup, Aggr, {}, {"RequestsBigLatency"}, true, "sensor", false); + SLIBigLatency = NKikimr::NPQ::TMultiCounter(subGroup, Aggr, {}, {"RequestsBigLatency"}, true, "sensor", false); ReadLatency = NKikimr::NPQ::CreateSLIDurationCounter(subGroup, Aggr, "Read", readBorder, {100, 200, 500, 1000, 1500, 2000, 5000, 10000, 30000, 99999999}); ReadLatencyFromDisk = NKikimr::NPQ::CreateSLIDurationCounter(subGroup, Aggr, "ReadFromDisk", readBorderFromDisk, {100, 200, 500, 1000, 1500, 2000, 5000, 10000, 30000, 99999999}); - SLIBigReadLatency = NKikimr::NPQ::TMultiCounter(subGroup, Aggr, {}, {"ReadBigLatency"}, true, "sensor", false); - ReadsTotal = NKikimr::NPQ::TMultiCounter(subGroup, Aggr, {}, {"ReadsTotal"}, true, "sensor", false); + SLIBigReadLatency = NKikimr::NPQ::TMultiCounter(subGroup, Aggr, {}, {"ReadBigLatency"}, true, "sensor", false); + ReadsTotal = NKikimr::NPQ::TMultiCounter(subGroup, Aggr, {}, {"ReadsTotal"}, true, "sensor", false); ui32 initDurationMs = (ctx.Now() - StartTime).MilliSeconds(); InitLatency.IncFor(initDurationMs, 1); @@ -1002,11 +1002,11 @@ void TReadSessionActor::Handle(TEvPersQueue::TEvLockPartition::TPtr& ev, const T } if (NumPartitionsFromTopic[topic]++ == 0) { - if (AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen()) { - SetupTopicCounters(topic, jt->second.CloudId, jt->second.DbId, jt->second.FolderId); - } else { - SetupTopicCounters(topic); - } + if (AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen()) { + SetupTopicCounters(topic, jt->second.CloudId, jt->second.DbId, jt->second.FolderId); + } else { + SetupTopicCounters(topic); + } } auto it = TopicCounters.find(topic); diff --git a/ydb/services/persqueue_v1/grpc_pq_schema.cpp b/ydb/services/persqueue_v1/grpc_pq_schema.cpp index 7756adc797..7f54ea8c64 100644 --- a/ydb/services/persqueue_v1/grpc_pq_schema.cpp +++ b/ydb/services/persqueue_v1/grpc_pq_schema.cpp @@ -121,89 +121,89 @@ void TDescribeTopicActor::StateWork(TAutoPtr<IEventHandle>& ev, const TActorCont } -void TDescribeTopicActor::HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) { +void TDescribeTopicActor::HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) { Y_VERIFY(ev->Get()->Request.Get()->ResultSet.size() == 1); // describe for only one topic - if (ReplyIfNotTopic(ev, ctx)) { - return; - } - + if (ReplyIfNotTopic(ev, ctx)) { + return; + } + const auto& response = ev->Get()->Request.Get()->ResultSet.front(); - const TString path = JoinSeq("/", response.Path); + const TString path = JoinSeq("/", response.Path); - Ydb::PersQueue::V1::DescribeTopicResult result; + Ydb::PersQueue::V1::DescribeTopicResult result; - auto settings = result.mutable_settings(); - Ydb::Scheme::Entry *selfEntry = result.mutable_self(); - const auto& selfInfo = response.Self->Info; + auto settings = result.mutable_settings(); + Ydb::Scheme::Entry *selfEntry = result.mutable_self(); + const auto& selfInfo = response.Self->Info; selfEntry->set_name(path + "/" + selfInfo.GetName()); - selfEntry->set_type(static_cast<Ydb::Scheme::Entry::Type>(selfInfo.GetPathType())); - ConvertDirectoryEntry(selfInfo, selfEntry, true); - if (response.PQGroupInfo) { - const auto &pqDescr = response.PQGroupInfo->Description; - settings->set_partitions_count(pqDescr.GetTotalGroupCount()); - - const auto &config = pqDescr.GetPQTabletConfig(); - if (!config.GetRequireAuthWrite()) { - (*settings->mutable_attributes())["_allow_unauthenticated_write"] = "true"; - } - - if (!config.GetRequireAuthRead()) { - (*settings->mutable_attributes())["_allow_unauthenticated_read"] = "true"; - } - - if (pqDescr.GetPartitionPerTablet() != 2) { - (*settings->mutable_attributes())["_partitions_per_tablet"] = - TStringBuilder() << pqDescr.GetPartitionPerTablet(); - } - if (config.HasAbcId()) { - (*settings->mutable_attributes())["_abc_id"] = TStringBuilder() << config.GetAbcId(); - } - if (config.HasAbcSlug()) { - (*settings->mutable_attributes())["_abc_slug"] = config.GetAbcSlug(); - } - bool local = config.GetLocalDC(); - settings->set_client_write_disabled(!local); - const auto &partConfig = config.GetPartitionConfig(); - i64 msip = partConfig.GetMaxSizeInPartition(); - if (msip != Max<i64>()) - settings->set_max_partition_storage_size(msip); - settings->set_retention_period_ms(partConfig.GetLifetimeSeconds() * 1000); - settings->set_message_group_seqno_retention_period_ms(partConfig.GetSourceIdLifetimeSeconds() * 1000); - settings->set_max_partition_message_groups_seqno_stored(partConfig.GetSourceIdMaxCounts()); - - if (local) { - settings->set_max_partition_write_speed(partConfig.GetWriteSpeedInBytesPerSecond()); - settings->set_max_partition_write_burst(partConfig.GetBurstSize()); - } - - settings->set_supported_format( - (Ydb::PersQueue::V1::TopicSettings::Format) (config.GetFormatVersion() + 1)); - - for (const auto &codec : config.GetCodecs().GetIds()) { - settings->add_supported_codecs((Ydb::PersQueue::V1::Codec) (codec + 1)); - } - + selfEntry->set_type(static_cast<Ydb::Scheme::Entry::Type>(selfInfo.GetPathType())); + ConvertDirectoryEntry(selfInfo, selfEntry, true); + if (response.PQGroupInfo) { + const auto &pqDescr = response.PQGroupInfo->Description; + settings->set_partitions_count(pqDescr.GetTotalGroupCount()); + + const auto &config = pqDescr.GetPQTabletConfig(); + if (!config.GetRequireAuthWrite()) { + (*settings->mutable_attributes())["_allow_unauthenticated_write"] = "true"; + } + + if (!config.GetRequireAuthRead()) { + (*settings->mutable_attributes())["_allow_unauthenticated_read"] = "true"; + } + + if (pqDescr.GetPartitionPerTablet() != 2) { + (*settings->mutable_attributes())["_partitions_per_tablet"] = + TStringBuilder() << pqDescr.GetPartitionPerTablet(); + } + if (config.HasAbcId()) { + (*settings->mutable_attributes())["_abc_id"] = TStringBuilder() << config.GetAbcId(); + } + if (config.HasAbcSlug()) { + (*settings->mutable_attributes())["_abc_slug"] = config.GetAbcSlug(); + } + bool local = config.GetLocalDC(); + settings->set_client_write_disabled(!local); + const auto &partConfig = config.GetPartitionConfig(); + i64 msip = partConfig.GetMaxSizeInPartition(); + if (msip != Max<i64>()) + settings->set_max_partition_storage_size(msip); + settings->set_retention_period_ms(partConfig.GetLifetimeSeconds() * 1000); + settings->set_message_group_seqno_retention_period_ms(partConfig.GetSourceIdLifetimeSeconds() * 1000); + settings->set_max_partition_message_groups_seqno_stored(partConfig.GetSourceIdMaxCounts()); + + if (local) { + settings->set_max_partition_write_speed(partConfig.GetWriteSpeedInBytesPerSecond()); + settings->set_max_partition_write_burst(partConfig.GetBurstSize()); + } + + settings->set_supported_format( + (Ydb::PersQueue::V1::TopicSettings::Format) (config.GetFormatVersion() + 1)); + + for (const auto &codec : config.GetCodecs().GetIds()) { + settings->add_supported_codecs((Ydb::PersQueue::V1::Codec) (codec + 1)); + } + const auto& pqConfig = AppData(ctx)->PQConfig; - for (ui32 i = 0; i < config.ReadRulesSize(); ++i) { - auto rr = settings->add_read_rules(); - auto consumerName = NPersQueue::ConvertOldConsumerName(config.GetReadRules(i), ctx); - rr->set_consumer_name(consumerName); - rr->set_starting_message_timestamp_ms(config.GetReadFromTimestampsMs(i)); - rr->set_supported_format( - (Ydb::PersQueue::V1::TopicSettings::Format) (config.GetConsumerFormatVersions(i) + 1)); - rr->set_version(config.GetReadRuleVersions(i)); - for (const auto &codec : config.GetConsumerCodecs(i).GetIds()) { - rr->add_supported_codecs((Ydb::PersQueue::V1::Codec) (codec + 1)); - } - bool important = false; - for (const auto &c : partConfig.GetImportantClientId()) { - if (c == config.GetReadRules(i)) { - important = true; - break; + for (ui32 i = 0; i < config.ReadRulesSize(); ++i) { + auto rr = settings->add_read_rules(); + auto consumerName = NPersQueue::ConvertOldConsumerName(config.GetReadRules(i), ctx); + rr->set_consumer_name(consumerName); + rr->set_starting_message_timestamp_ms(config.GetReadFromTimestampsMs(i)); + rr->set_supported_format( + (Ydb::PersQueue::V1::TopicSettings::Format) (config.GetConsumerFormatVersions(i) + 1)); + rr->set_version(config.GetReadRuleVersions(i)); + for (const auto &codec : config.GetConsumerCodecs(i).GetIds()) { + rr->add_supported_codecs((Ydb::PersQueue::V1::Codec) (codec + 1)); + } + bool important = false; + for (const auto &c : partConfig.GetImportantClientId()) { + if (c == config.GetReadRules(i)) { + important = true; + break; } - } - rr->set_important(important); + } + rr->set_important(important); if (i < config.ReadRuleServiceTypesSize()) { rr->set_service_type(config.GetReadRuleServiceTypes(i)); @@ -218,44 +218,44 @@ void TDescribeTopicActor::HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEv } rr->set_service_type(pqConfig.GetDefaultClientServiceType().GetName()); } - } - if (partConfig.HasMirrorFrom()) { - auto rmr = settings->mutable_remote_mirror_rule(); - TStringBuilder endpoint; - if (partConfig.GetMirrorFrom().GetUseSecureConnection()) { - endpoint << GRPCS_ENDPOINT_PREFIX; - } - endpoint << partConfig.GetMirrorFrom().GetEndpoint() << ":" - << partConfig.GetMirrorFrom().GetEndpointPort(); - rmr->set_endpoint(endpoint); - rmr->set_topic_path(partConfig.GetMirrorFrom().GetTopic()); - rmr->set_consumer_name(partConfig.GetMirrorFrom().GetConsumer()); - rmr->set_starting_message_timestamp_ms(partConfig.GetMirrorFrom().GetReadFromTimestampsMs()); - if (partConfig.GetMirrorFrom().HasCredentials()) { - if (partConfig.GetMirrorFrom().GetCredentials().HasOauthToken()) { - rmr->mutable_credentials()->set_oauth_token( - NPersQueue::ObfuscateString( - partConfig.GetMirrorFrom().GetCredentials().GetOauthToken()) - ); - } else if (partConfig.GetMirrorFrom().GetCredentials().HasJwtParams()) { - rmr->mutable_credentials()->set_jwt_params( - NPersQueue::ObfuscateString( - partConfig.GetMirrorFrom().GetCredentials().GetJwtParams()) - ); - } else if (partConfig.GetMirrorFrom().GetCredentials().HasIam()) { - rmr->mutable_credentials()->mutable_iam()->set_endpoint( - partConfig.GetMirrorFrom().GetCredentials().GetIam().GetEndpoint() - ); - rmr->mutable_credentials()->mutable_iam()->set_service_account_key( - NPersQueue::ObfuscateString( - partConfig.GetMirrorFrom().GetCredentials().GetIam().GetServiceAccountKey()) - ); + } + if (partConfig.HasMirrorFrom()) { + auto rmr = settings->mutable_remote_mirror_rule(); + TStringBuilder endpoint; + if (partConfig.GetMirrorFrom().GetUseSecureConnection()) { + endpoint << GRPCS_ENDPOINT_PREFIX; + } + endpoint << partConfig.GetMirrorFrom().GetEndpoint() << ":" + << partConfig.GetMirrorFrom().GetEndpointPort(); + rmr->set_endpoint(endpoint); + rmr->set_topic_path(partConfig.GetMirrorFrom().GetTopic()); + rmr->set_consumer_name(partConfig.GetMirrorFrom().GetConsumer()); + rmr->set_starting_message_timestamp_ms(partConfig.GetMirrorFrom().GetReadFromTimestampsMs()); + if (partConfig.GetMirrorFrom().HasCredentials()) { + if (partConfig.GetMirrorFrom().GetCredentials().HasOauthToken()) { + rmr->mutable_credentials()->set_oauth_token( + NPersQueue::ObfuscateString( + partConfig.GetMirrorFrom().GetCredentials().GetOauthToken()) + ); + } else if (partConfig.GetMirrorFrom().GetCredentials().HasJwtParams()) { + rmr->mutable_credentials()->set_jwt_params( + NPersQueue::ObfuscateString( + partConfig.GetMirrorFrom().GetCredentials().GetJwtParams()) + ); + } else if (partConfig.GetMirrorFrom().GetCredentials().HasIam()) { + rmr->mutable_credentials()->mutable_iam()->set_endpoint( + partConfig.GetMirrorFrom().GetCredentials().GetIam().GetEndpoint() + ); + rmr->mutable_credentials()->mutable_iam()->set_service_account_key( + NPersQueue::ObfuscateString( + partConfig.GetMirrorFrom().GetCredentials().GetIam().GetServiceAccountKey()) + ); } } - rmr->set_database(partConfig.GetMirrorFrom().GetDatabase()); + rmr->set_database(partConfig.GetMirrorFrom().GetDatabase()); } } - return ReplyWithResult(Ydb::StatusIds::SUCCESS, result, ctx); + return ReplyWithResult(Ydb::StatusIds::SUCCESS, result, ctx); } @@ -332,8 +332,8 @@ void TRemoveReadRuleActor::ModifyPersqueueConfig( GetProtoRequest()->consumer_name(), ctx ); - if (!error.Empty()) { - return ReplyWithError(Ydb::StatusIds::NOT_FOUND, Ydb::PersQueue::ErrorCode::BAD_REQUEST, error, ctx); + if (!error.Empty()) { + return ReplyWithError(Ydb::StatusIds::NOT_FOUND, Ydb::PersQueue::ErrorCode::BAD_REQUEST, error, ctx); } } diff --git a/ydb/services/persqueue_v1/grpc_pq_write_actor.cpp b/ydb/services/persqueue_v1/grpc_pq_write_actor.cpp index 3038cc82a6..5d211b3d88 100644 --- a/ydb/services/persqueue_v1/grpc_pq_write_actor.cpp +++ b/ydb/services/persqueue_v1/grpc_pq_write_actor.cpp @@ -350,8 +350,8 @@ void TWriteSessionActor::Handle(TEvPQProxy::TEvWriteInit::TPtr& ev, const TActor auto subGroup = GetServiceCounters(Counters, "pqproxy|SLI"); Aggr = {{{{"Account", TopicConverter->GetAccount()}}, {"total"}}}; - SLITotal = NKikimr::NPQ::TMultiCounter(subGroup, Aggr, {}, {"RequestsTotal"}, true, "sensor", false); - SLIErrors = NKikimr::NPQ::TMultiCounter(subGroup, Aggr, {}, {"RequestsError"}, true, "sensor", false); + SLITotal = NKikimr::NPQ::TMultiCounter(subGroup, Aggr, {}, {"RequestsTotal"}, true, "sensor", false); + SLIErrors = NKikimr::NPQ::TMultiCounter(subGroup, Aggr, {}, {"RequestsError"}, true, "sensor", false); SLITotal.Inc(); const auto& preferredCluster = init.preferred_cluster(); @@ -360,39 +360,39 @@ void TWriteSessionActor::Handle(TEvPQProxy::TEvWriteInit::TPtr& ev, const TActor } } -void TWriteSessionActor::SetupCounters() +void TWriteSessionActor::SetupCounters() { //now topic is checked, can create group for real topic, not garbage auto subGroup = GetServiceCounters(Counters, "pqproxy|writeSession"); TVector<NPQ::TLabelsInfo> aggr = NKikimr::NPQ::GetLabels(LocalDC, TopicConverter->GetClientsideName()); - BytesInflight = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"BytesInflight"}, false); - BytesInflightTotal = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"BytesInflightTotal"}, false); - SessionsCreated = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"SessionsCreated"}, true); - SessionsActive = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"SessionsActive"}, false); - Errors = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"Errors"}, true); + BytesInflight = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"BytesInflight"}, false); + BytesInflightTotal = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"BytesInflightTotal"}, false); + SessionsCreated = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"SessionsCreated"}, true); + SessionsActive = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"SessionsActive"}, false); + Errors = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"Errors"}, true); SessionsCreated.Inc(); SessionsActive.Inc(); } -void TWriteSessionActor::SetupCounters(const TString& cloudId, const TString& dbId, - const TString& folderId) -{ - //now topic is checked, can create group for real topic, not garbage - auto subGroup = NKikimr::NPQ::GetCountersForStream(Counters, "writeSession"); +void TWriteSessionActor::SetupCounters(const TString& cloudId, const TString& dbId, + const TString& folderId) +{ + //now topic is checked, can create group for real topic, not garbage + auto subGroup = NKikimr::NPQ::GetCountersForStream(Counters, "writeSession"); TVector<NPQ::TLabelsInfo> aggr = NKikimr::NPQ::GetLabelsForStream(TopicConverter->GetClientsideName(), cloudId, dbId, folderId); - - BytesInflight = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"stream.internal_write.bytes_proceeding"}, false); - BytesInflightTotal = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"stream.internal_write.bytes_proceeding_total"}, false); - SessionsCreated = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"stream.internal_write.sessions_created_per_second"}, true); - SessionsActive = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"stream.internal_write.sessions_active"}, false); - Errors = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"stream.internal_write.errors_per_second"}, true); - - SessionsCreated.Inc(); - SessionsActive.Inc(); -} - + + BytesInflight = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"stream.internal_write.bytes_proceeding"}, false); + BytesInflightTotal = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"stream.internal_write.bytes_proceeding_total"}, false); + SessionsCreated = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"stream.internal_write.sessions_created_per_second"}, true); + SessionsActive = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"stream.internal_write.sessions_active"}, false); + Errors = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"stream.internal_write.errors_per_second"}, true); + + SessionsCreated.Inc(); + SessionsActive.Inc(); +} + void TWriteSessionActor::InitCheckSchema(const TActorContext& ctx, bool needWaitSchema) { LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "init check schema"); @@ -428,13 +428,13 @@ void TWriteSessionActor::Handle(TEvDescribeTopicsResponse::TPtr& ev, const TActo PartitionToTablet[pi.GetPartitionId()] = pi.GetTabletId(); } - if (AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen()) { - const auto& tabletConfig = description.GetPQTabletConfig(); - SetupCounters(tabletConfig.GetYcCloudId(), tabletConfig.GetYdbDatabaseId(), - tabletConfig.GetYcFolderId()); - } else { - SetupCounters(); - } + if (AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen()) { + const auto& tabletConfig = description.GetPQTabletConfig(); + SetupCounters(tabletConfig.GetYcCloudId(), tabletConfig.GetYdbDatabaseId(), + tabletConfig.GetYcFolderId()); + } else { + SetupCounters(); + } Y_VERIFY (entry.SecurityObject); ACL.Reset(new TAclWrapper(entry.SecurityObject)); @@ -471,32 +471,32 @@ void TWriteSessionActor::Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult:: ); } - const auto& pqDescription = navigate->ResultSet.front().PQGroupInfo->Description; + const auto& pqDescription = navigate->ResultSet.front().PQGroupInfo->Description; - Y_VERIFY(pqDescription.PartitionsSize() > 0); - Y_VERIFY(pqDescription.HasPQTabletConfig()); - InitialPQTabletConfig = pqDescription.GetPQTabletConfig(); + Y_VERIFY(pqDescription.PartitionsSize() > 0); + Y_VERIFY(pqDescription.HasPQTabletConfig()); + InitialPQTabletConfig = pqDescription.GetPQTabletConfig(); - if (!pqDescription.HasBalancerTabletID()) { + if (!pqDescription.HasBalancerTabletID()) { TString errorReason = Sprintf("topic '%s' has no balancer, Marker# PQ93", TopicConverter->GetClientsideName().c_str()); CloseSession(errorReason, PersQueue::ErrorCode::UNKNOWN_TOPIC, ctx); return; } - BalancerTabletId = pqDescription.GetBalancerTabletID(); + BalancerTabletId = pqDescription.GetBalancerTabletID(); - for (ui32 i = 0; i < pqDescription.PartitionsSize(); ++i) { - const auto& pi = pqDescription.GetPartitions(i); + for (ui32 i = 0; i < pqDescription.PartitionsSize(); ++i) { + const auto& pi = pqDescription.GetPartitions(i); PartitionToTablet[pi.GetPartitionId()] = pi.GetTabletId(); } - if (AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen()) { - const auto& tabletConfig = pqDescription.GetPQTabletConfig(); - SetupCounters(tabletConfig.GetYcCloudId(), tabletConfig.GetYdbDatabaseId(), - tabletConfig.GetYcFolderId()); - } else { - SetupCounters(); - } + if (AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen()) { + const auto& tabletConfig = pqDescription.GetPQTabletConfig(); + SetupCounters(tabletConfig.GetYcCloudId(), tabletConfig.GetYdbDatabaseId(), + tabletConfig.GetYcFolderId()); + } else { + SetupCounters(); + } Y_VERIFY(!navigate->ResultSet.empty()); ACL.Reset(new TAclWrapper(navigate->ResultSet.front().SecurityObject)); @@ -705,7 +705,7 @@ void TWriteSessionActor::ProceedPartition(const ui32 partition, const TActorCont auto subGroup = GetServiceCounters(Counters, "pqproxy|SLI"); InitLatency = NKikimr::NPQ::CreateSLIDurationCounter(subGroup, Aggr, "WriteInit", border, {100, 200, 500, 1000, 1500, 2000, 5000, 10000, 30000, 99999999}); - SLIBigLatency = NKikimr::NPQ::TMultiCounter(subGroup, Aggr, {}, {"RequestsBigLatency"}, true, "sesnor", false); + SLIBigLatency = NKikimr::NPQ::TMultiCounter(subGroup, Aggr, {}, {"RequestsBigLatency"}, true, "sesnor", false); ui32 initDurationMs = (ctx.Now() - StartTime).MilliSeconds(); InitLatency.IncFor(initDurationMs, 1); diff --git a/ydb/services/persqueue_v1/persqueue_ut.cpp b/ydb/services/persqueue_v1/persqueue_ut.cpp index 3bc9b21728..98db74149a 100644 --- a/ydb/services/persqueue_v1/persqueue_ut.cpp +++ b/ydb/services/persqueue_v1/persqueue_ut.cpp @@ -1362,8 +1362,8 @@ namespace { // TODO: Describe this assert in comment UNIT_ASSERT(!ack->Acks.empty()); UNIT_ASSERT(ack->Acks.back().Stat); - UNIT_ASSERT(ack->Acks.back().Stat->TotalTimeInPartitionQueue >= ack->Acks.back().Stat->PartitionQuotedTime); - UNIT_ASSERT(ack->Acks.back().Stat->TotalTimeInPartitionQueue <= ack->Acks.back().Stat->PartitionQuotedTime + TDuration::Seconds(10)); + UNIT_ASSERT(ack->Acks.back().Stat->TotalTimeInPartitionQueue >= ack->Acks.back().Stat->PartitionQuotedTime); + UNIT_ASSERT(ack->Acks.back().Stat->TotalTimeInPartitionQueue <= ack->Acks.back().Stat->PartitionQuotedTime + TDuration::Seconds(10)); } } GetCounters(server.CleverServer->GetRuntime()->GetMonPort(), "pqproxy", "writeSession", "account/topic1", "Vla"); |