aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authormokhotskii <mokhotskii@yandex-team.ru>2022-02-10 16:49:07 +0300
committerDaniil Cherednik <dcherednik@yandex-team.ru>2022-02-10 16:49:07 +0300
commit9c5581106c4ee0da7c7cb99f2d05b4a27431ee1e (patch)
tree5d5cb817648f650d76cf1076100726fd9b8448e8
parentedfa737d405b60f2d9f05f230c1e97118c14f5bb (diff)
downloadydb-9c5581106c4ee0da7c7cb99f2d05b4a27431ee1e.tar.gz
Restoring authorship annotation for <mokhotskii@yandex-team.ru>. Commit 2 of 2.
-rw-r--r--contrib/python/botocore/ya.make8
-rw-r--r--ydb/core/client/server/msgbus_server_pq_metacache.cpp50
-rw-r--r--ydb/core/grpc_services/rpc_scheme_base.h12
-rw-r--r--ydb/core/persqueue/event_helpers.cpp14
-rw-r--r--ydb/core/persqueue/event_helpers.h4
-rw-r--r--ydb/core/persqueue/events/internal.h16
-rw-r--r--ydb/core/persqueue/mirrorer.cpp6
-rw-r--r--ydb/core/persqueue/partition.cpp496
-rw-r--r--ydb/core/persqueue/partition.h22
-rw-r--r--ydb/core/persqueue/percentile_counter.cpp162
-rw-r--r--ydb/core/persqueue/percentile_counter.h56
-rw-r--r--ydb/core/persqueue/pq_impl.cpp50
-rw-r--r--ydb/core/persqueue/pq_ut.cpp10
-rw-r--r--ydb/core/persqueue/pq_ut.h16
-rw-r--r--ydb/core/persqueue/subscriber.h4
-rw-r--r--ydb/core/persqueue/user_info.cpp14
-rw-r--r--ydb/core/persqueue/user_info.h230
-rw-r--r--ydb/core/persqueue/working_time_counter.h22
-rw-r--r--ydb/core/protos/msgbus_pq.proto8
-rw-r--r--ydb/core/protos/pqconfig.proto32
-rw-r--r--ydb/core/tx/schemeshard/schemeshard__operation_alter_pq.cpp36
-rw-r--r--ydb/core/tx/schemeshard/schemeshard__operation_create_pq.cpp40
-rw-r--r--ydb/public/api/protos/draft/datastreams.proto44
-rw-r--r--ydb/public/sdk/cpp/client/ydb_datastreams/datastreams.cpp140
-rw-r--r--ydb/public/sdk/cpp/client/ydb_datastreams/datastreams.h56
-rw-r--r--ydb/services/datastreams/datastreams_proxy.cpp1902
-rw-r--r--ydb/services/datastreams/datastreams_ut.cpp1944
-rw-r--r--ydb/services/datastreams/next_token.cpp2
-rw-r--r--ydb/services/datastreams/next_token.h162
-rw-r--r--ydb/services/datastreams/put_records_actor.h24
-rw-r--r--ydb/services/datastreams/shard_iterator.cpp2
-rw-r--r--ydb/services/datastreams/shard_iterator.h152
-rw-r--r--ydb/services/datastreams/ya.make4
-rw-r--r--ydb/services/lib/actors/pq_schema_actor.cpp80
-rw-r--r--ydb/services/lib/actors/pq_schema_actor.h172
-rw-r--r--ydb/services/persqueue_v1/grpc_pq_actor.h26
-rw-r--r--ydb/services/persqueue_v1/grpc_pq_read_actor.cpp84
-rw-r--r--ydb/services/persqueue_v1/grpc_pq_schema.cpp222
-rw-r--r--ydb/services/persqueue_v1/grpc_pq_write_actor.cpp94
-rw-r--r--ydb/services/persqueue_v1/persqueue_ut.cpp4
40 files changed, 3211 insertions, 3211 deletions
diff --git a/contrib/python/botocore/ya.make b/contrib/python/botocore/ya.make
index 9f7b34b17d..dc02d4a419 100644
--- a/contrib/python/botocore/ya.make
+++ b/contrib/python/botocore/ya.make
@@ -568,10 +568,10 @@ RESOURCE_FILES(
#botocore/data/kinesis-video-media/2017-09-30/service-2.json
#botocore/data/kinesis-video-signaling/2019-12-04/paginators-1.json
#botocore/data/kinesis-video-signaling/2019-12-04/service-2.json
- botocore/data/kinesis/2013-12-02/examples-1.json
- botocore/data/kinesis/2013-12-02/paginators-1.json
- botocore/data/kinesis/2013-12-02/service-2.json
- botocore/data/kinesis/2013-12-02/waiters-2.json
+ botocore/data/kinesis/2013-12-02/examples-1.json
+ botocore/data/kinesis/2013-12-02/paginators-1.json
+ botocore/data/kinesis/2013-12-02/service-2.json
+ botocore/data/kinesis/2013-12-02/waiters-2.json
#botocore/data/kinesisanalytics/2015-08-14/examples-1.json
#botocore/data/kinesisanalytics/2015-08-14/paginators-1.json
#botocore/data/kinesisanalytics/2015-08-14/service-2.json
diff --git a/ydb/core/client/server/msgbus_server_pq_metacache.cpp b/ydb/core/client/server/msgbus_server_pq_metacache.cpp
index a88fe52091..f44ab6f480 100644
--- a/ydb/core/client/server/msgbus_server_pq_metacache.cpp
+++ b/ydb/core/client/server/msgbus_server_pq_metacache.cpp
@@ -26,21 +26,21 @@ IActor* CreateSchemeCache(NActors::TActorSystem* ActorSystem, TIntrusivePtr<NMon
class TPersQueueMetaCacheActor : public TActorBootstrapped<TPersQueueMetaCacheActor> {
using TBase = TActorBootstrapped<TPersQueueMetaCacheActor>;
public:
- TPersQueueMetaCacheActor(TPersQueueMetaCacheActor&&) = default;
- TPersQueueMetaCacheActor& operator=(TPersQueueMetaCacheActor&&) = default;
-
- TPersQueueMetaCacheActor(ui64 grpcPort,
- const NMonitoring::TDynamicCounterPtr& counters,
- const TDuration& versionCheckInterval)
+ TPersQueueMetaCacheActor(TPersQueueMetaCacheActor&&) = default;
+ TPersQueueMetaCacheActor& operator=(TPersQueueMetaCacheActor&&) = default;
+
+ TPersQueueMetaCacheActor(ui64 grpcPort,
+ const NMonitoring::TDynamicCounterPtr& counters,
+ const TDuration& versionCheckInterval)
: Counters(counters)
- , ClientWrapper(std::move(std::make_unique<TClientWrapper>(grpcPort)))
+ , ClientWrapper(std::move(std::make_unique<TClientWrapper>(grpcPort)))
, VersionCheckInterval(versionCheckInterval)
, Generation(std::make_shared<TAtomicCounter>())
{
}
- TPersQueueMetaCacheActor(const NMonitoring::TDynamicCounterPtr& counters,
- const TDuration& versionCheckInterval)
+ TPersQueueMetaCacheActor(const NMonitoring::TDynamicCounterPtr& counters,
+ const TDuration& versionCheckInterval)
: Counters(counters)
, VersionCheckInterval(versionCheckInterval)
, Generation(std::make_shared<TAtomicCounter>())
@@ -58,7 +58,7 @@ public:
Die(ctx);
return;
}
- ClientWrapper.reset(new TClientWrapper(driver));
+ ClientWrapper.reset(new TClientWrapper(driver));
}
SkipVersionCheck = !AppData(ctx)->PQConfig.GetMetaCacheSkipVersionCheck();
PathPrefix = TopicPrefix(ctx);
@@ -358,7 +358,7 @@ private:
void SendSchemeCacheRequest(const TVector<TString>& topics, bool addDefaultPathPrefix, const TActorContext& ctx) {
- auto schemeCacheRequest = std::make_unique<NSchemeCache::TSchemeCacheNavigate>(++RequestId);
+ auto schemeCacheRequest = std::make_unique<NSchemeCache::TSchemeCacheNavigate>(++RequestId);
for (const auto& path : topics) {
auto split = NKikimr::SplitPath(path);
Y_VERIFY(!split.empty());
@@ -372,7 +372,7 @@ private:
entry.Operation = NSchemeCache::TSchemeCacheNavigate::OpTopic;
schemeCacheRequest->ResultSet.emplace_back(std::move(entry));
}
- ctx.Send(SchemeCacheId, new TEvTxProxySchemeCache::TEvNavigateKeySet(schemeCacheRequest.release()));
+ ctx.Send(SchemeCacheId, new TEvTxProxySchemeCache::TEvNavigateKeySet(schemeCacheRequest.release()));
}
void SendListTopicsResponse(const TActorId& recipient, const TActorContext& ctx) {
@@ -444,11 +444,11 @@ private:
class TClientWrapper {
public:
- TClientWrapper(const TClientWrapper&) = delete;
- TClientWrapper& operator=(const TClientWrapper&) = delete;
- TClientWrapper(TClientWrapper&&) = default;
- TClientWrapper& operator=(TClientWrapper&&) = default;
-
+ TClientWrapper(const TClientWrapper&) = delete;
+ TClientWrapper& operator=(const TClientWrapper&) = delete;
+ TClientWrapper(TClientWrapper&&) = default;
+ TClientWrapper& operator=(TClientWrapper&&) = default;
+
TClientWrapper(ui64 driverPort)
: DriverPort(driverPort)
{}
@@ -464,18 +464,18 @@ private:
auto driverConfig = NYdb::TDriverConfig().SetEndpoint(endpoint);
DriverHolder.Reset(new NYdb::TDriver(driverConfig));
Driver = DriverHolder.Get();
- TableClient.reset(new NYdb::NTable::TTableClient(*Driver));
+ TableClient.reset(new NYdb::NTable::TTableClient(*Driver));
}
} else if (Driver != nullptr) {
- TableClient.reset(new NYdb::NTable::TTableClient(*Driver));
+ TableClient.reset(new NYdb::NTable::TTableClient(*Driver));
}
}
-
+
NYdb::NTable::TTableClient* GetClient() {
Y_VERIFY(TableClient);
- return TableClient.get();
+ return TableClient.get();
}
-
+
void Stop() {
if (DriverHolder != nullptr) {
TableClient->Stop();
@@ -489,15 +489,15 @@ private:
THolder<NYdb::TDriver> DriverHolder;
NYdb::TDriver* Driver = nullptr;
TMaybe<ui64> DriverPort;
- std::unique_ptr<NYdb::NTable::TTableClient> TableClient;
- };
+ std::unique_ptr<NYdb::NTable::TTableClient> TableClient;
+ };
NMonitoring::TDynamicCounterPtr Counters;
NActors::TActorSystem* ActorSystem;
TString VersionQuery;
TString TopicsQuery;
- std::unique_ptr<TClientWrapper> ClientWrapper;
+ std::unique_ptr<TClientWrapper> ClientWrapper;
TAsyncCreateSessionResult SessionFuture;
TMaybe<TSession> YdbSession;
TAsyncPrepareQueryResult TopicsQueryFuture;
diff --git a/ydb/core/grpc_services/rpc_scheme_base.h b/ydb/core/grpc_services/rpc_scheme_base.h
index 444e56a556..0be68e802e 100644
--- a/ydb/core/grpc_services/rpc_scheme_base.h
+++ b/ydb/core/grpc_services/rpc_scheme_base.h
@@ -10,7 +10,7 @@ namespace NGRpcService {
template <typename TDerived, typename TRequest>
class TRpcSchemeRequestActor : public TRpcOperationRequestActor<TDerived, TRequest> {
-protected:
+protected:
using TBase = TRpcOperationRequestActor<TDerived, TRequest>;
public:
@@ -128,17 +128,17 @@ protected:
void Handle(NSchemeShard::TEvSchemeShard::TEvNotifyTxCompletionResult::TPtr& ev, const TActorContext& ctx) {
NTabletPipe::CloseClient(ctx, SchemePipeActorId_);
- return this->ReplyNotifyTxCompletionResult(ev, ctx);
+ return this->ReplyNotifyTxCompletionResult(ev, ctx);
}
void Handle(NSchemeShard::TEvSchemeShard::TEvNotifyTxCompletionRegistered::TPtr&, const TActorContext&) {
}
virtual void ReplyNotifyTxCompletionResult(NSchemeShard::TEvSchemeShard::TEvNotifyTxCompletionResult::TPtr& ev, const TActorContext& ctx) {
- Y_UNUSED(ev);
- return this->Reply(Ydb::StatusIds::SUCCESS, ctx);
- }
-
+ Y_UNUSED(ev);
+ return this->Reply(Ydb::StatusIds::SUCCESS, ctx);
+ }
+
private:
TActorId SchemePipeActorId_;
};
diff --git a/ydb/core/persqueue/event_helpers.cpp b/ydb/core/persqueue/event_helpers.cpp
index 7038d31579..320002a936 100644
--- a/ydb/core/persqueue/event_helpers.cpp
+++ b/ydb/core/persqueue/event_helpers.cpp
@@ -20,8 +20,8 @@ void ReplyPersQueueError(
NKikimrServices::EServiceKikimr service,
const ui64 responseCookie,
NPersQueue::NErrorCode::EErrorCode errorCode,
- const TString& error,
- bool logDebug
+ const TString& error,
+ bool logDebug
) {
if (errorCode == NPersQueue::NErrorCode::BAD_REQUEST) {
counters.Cumulative()[COUNTER_PQ_BAD_REQUEST].Increment(1);
@@ -36,11 +36,11 @@ void ReplyPersQueueError(
}
logStr << " error: " << error;
- if (logDebug) {
- LOG_DEBUG_S(ctx, service, logStr);
- } else {
- LOG_WARN_S(ctx, service, logStr);
- }
+ if (logDebug) {
+ LOG_DEBUG_S(ctx, service, logStr);
+ } else {
+ LOG_WARN_S(ctx, service, logStr);
+ }
ctx.Send(dstActor, new TEvPQ::TEvError(errorCode, error, responseCookie));
}
diff --git a/ydb/core/persqueue/event_helpers.h b/ydb/core/persqueue/event_helpers.h
index b8e39fbe8a..51bb814c92 100644
--- a/ydb/core/persqueue/event_helpers.h
+++ b/ydb/core/persqueue/event_helpers.h
@@ -19,8 +19,8 @@ void ReplyPersQueueError(
NKikimrServices::EServiceKikimr service,
const ui64 responseCookie,
NPersQueue::NErrorCode::EErrorCode errorCode,
- const TString& error,
- bool logDebug = false
+ const TString& error,
+ bool logDebug = false
);
}// NPQ
diff --git a/ydb/core/persqueue/events/internal.h b/ydb/core/persqueue/events/internal.h
index aaee13d586..bc892988b4 100644
--- a/ydb/core/persqueue/events/internal.h
+++ b/ydb/core/persqueue/events/internal.h
@@ -131,7 +131,7 @@ struct TEvPQ {
ui32 UncompressedSize;
TString PartitionKey;
TString ExplicitHashKey;
- bool External;
+ bool External;
};
TEvWrite(const ui64 cookie, const ui64 messageNo, const TString& ownerCookie, const TMaybe<ui64> offset, TVector<TMsg> &&msgs, bool isDirectWrite)
@@ -149,7 +149,7 @@ struct TEvPQ {
TMaybe<ui64> Offset;
TVector<TMsg> Msgs;
bool IsDirectWrite;
-
+
};
struct TEvReadTimeout : public TEventLocal<TEvReadTimeout, EvReadTimeout> {
@@ -161,10 +161,10 @@ struct TEvPQ {
};
struct TEvRead : public TEventLocal<TEvRead, EvRead> {
- TEvRead(const ui64 cookie, const ui64 offset, const ui16 partNo, const ui32 count,
- const TString& sessionId, const TString& clientId, const ui32 timeout, const ui32 size,
- const ui32 maxTimeLagMs, const ui64 readTimestampMs, const TString& clientDC,
- bool externalOperation)
+ TEvRead(const ui64 cookie, const ui64 offset, const ui16 partNo, const ui32 count,
+ const TString& sessionId, const TString& clientId, const ui32 timeout, const ui32 size,
+ const ui32 maxTimeLagMs, const ui64 readTimestampMs, const TString& clientDC,
+ bool externalOperation)
: Cookie(cookie)
, Offset(offset)
, PartNo(partNo)
@@ -176,7 +176,7 @@ struct TEvPQ {
, MaxTimeLagMs(maxTimeLagMs)
, ReadTimestampMs(readTimestampMs)
, ClientDC(clientDC)
- , ExternalOperation(externalOperation)
+ , ExternalOperation(externalOperation)
{}
ui64 Cookie;
@@ -190,7 +190,7 @@ struct TEvPQ {
ui32 MaxTimeLagMs;
ui64 ReadTimestampMs;
TString ClientDC;
- bool ExternalOperation;
+ bool ExternalOperation;
};
struct TEvMonRequest : public TEventLocal<TEvMonRequest, EvMonRequest> {
diff --git a/ydb/core/persqueue/mirrorer.cpp b/ydb/core/persqueue/mirrorer.cpp
index d05b5fc228..37aea95ad1 100644
--- a/ydb/core/persqueue/mirrorer.cpp
+++ b/ydb/core/persqueue/mirrorer.cpp
@@ -61,7 +61,7 @@ void TMirrorer::Bootstrap(const TActorContext& ctx) {
TString suffix = LocalDC ? "Remote" : "Internal";
MirrorerErrors = NKikimr::NPQ::TMultiCounter(
GetServiceCounters(counters, "pqproxy|writeSession"),
- GetLabels(TopicName), {}, {"MirrorerErrors" + suffix}, true
+ GetLabels(TopicName), {}, {"MirrorerErrors" + suffix}, true
);
MirrorerTimeLags = THolder<TPercentileCounter>(new TPercentileCounter(
GetServiceCounters(counters, "pqproxy|mirrorWriteTimeLag"),
@@ -71,11 +71,11 @@ void TMirrorer::Bootstrap(const TActorContext& ctx) {
));
InitTimeoutCounter = NKikimr::NPQ::TMultiCounter(
GetServiceCounters(counters, "pqproxy|writeSession"),
- GetLabels(TopicName), {}, {"MirrorerInitTimeout" + suffix}, true
+ GetLabels(TopicName), {}, {"MirrorerInitTimeout" + suffix}, true
);
WriteTimeoutCounter = NKikimr::NPQ::TMultiCounter(
GetServiceCounters(counters, "pqproxy|writeSession"),
- {}, {}, {"MirrorerWriteTimeout"}, true, "sensor", false
+ {}, {}, {"MirrorerWriteTimeout"}, true, "sensor", false
);
}
}
diff --git a/ydb/core/persqueue/partition.cpp b/ydb/core/persqueue/partition.cpp
index c9aa61be49..cc8e2419cf 100644
--- a/ydb/core/persqueue/partition.cpp
+++ b/ydb/core/persqueue/partition.cpp
@@ -14,11 +14,11 @@
#include <ydb/core/protos/msgbus.pb.h>
#include <ydb/library/persqueue/topic_parser/topic_parser.h>
#include <ydb/public/lib/base/msgbus.h>
-#include <library/cpp/html/pcdata/pcdata.h>
-#include <library/cpp/monlib/service/pages/templates.h>
+#include <library/cpp/html/pcdata/pcdata.h>
+#include <library/cpp/monlib/service/pages/templates.h>
#include <library/cpp/time_provider/time_provider.h>
-#include <util/folder/path.h>
-#include <util/string/escape.h>
+#include <util/folder/path.h>
+#include <util/string/escape.h>
#include <util/system/byteorder.h>
#define VERIFY_RESULT_BLOB(blob, pos) \
@@ -243,7 +243,7 @@ void TPartition::ReplyError(const TActorContext& ctx, const ui64 dst, NPersQueue
{
ReplyPersQueueError(
dst == 0 ? ctx.SelfID : Tablet, ctx, TabletID, TopicName, Partition, Counters, NKikimrServices::PERSQUEUE,
- dst, errorCode, error, true
+ dst, errorCode, error, true
);
}
@@ -474,10 +474,10 @@ TPartition::TPartition(ui64 tabletId, ui32 partition, const TActorId& tablet, co
, BodySize(0)
, MaxWriteResponsesSize(0)
, GapSize(0)
- , CloudId(config.GetYcCloudId())
- , DbId(config.GetYdbDatabaseId())
- , FolderId(config.GetYcFolderId())
- , UsersInfoStorage(DCId, TabletID, TopicName, Partition, counters, Config, CloudId, DbId, FolderId)
+ , CloudId(config.GetYcCloudId())
+ , DbId(config.GetYdbDatabaseId())
+ , FolderId(config.GetYcFolderId())
+ , UsersInfoStorage(DCId, TabletID, TopicName, Partition, counters, Config, CloudId, DbId, FolderId)
, ReadingTimestamp(false)
, SetOffsetCookie(0)
, Cookie(0)
@@ -490,10 +490,10 @@ TPartition::TPartition(ui64 tabletId, ui32 partition, const TActorId& tablet, co
, WriteCycleStartTime(ctx.Now())
, WriteCycleSize(0)
, WriteNewSize(0)
- , WriteNewSizeInternal(0)
+ , WriteNewSizeInternal(0)
, WriteNewSizeUncompressed(0)
, WriteNewMessages(0)
- , WriteNewMessagesInternal(0)
+ , WriteNewMessagesInternal(0)
, DiskIsFull(false)
, HasDataReqNum(0)
, WriteQuota(Config.GetPartitionConfig().GetBurstSize(), Config.GetPartitionConfig().GetWriteSpeedInBytesPerSecond(), ctx.Now())
@@ -749,7 +749,7 @@ void TPartition::Bootstrap(const TActorContext& ctx)
}
LOG_INFO_S(ctx, NKikimrServices::PERSQUEUE, "boostrapping " << Partition << " " << ctx.SelfID);
-
+
if (NewPartition) {
InitComplete(ctx);
} else {
@@ -757,164 +757,164 @@ void TPartition::Bootstrap(const TActorContext& ctx)
RequestDiskStatus(ctx, Tablet, Config.GetPartitionConfig().GetNumChannels());
Become(&TThis::StateInit);
}
-
+
if (AppData(ctx)->Counters) {
- TVector<NPQ::TLabelsInfo> labels;
- if (AppData()->PQConfig.GetTopicsAreFirstClassCitizen()) {
- SetupStreamCounters(ctx);
- } else {
- if (TopicName.find("--") == TString::npos)
- return;
- SetupTopicCounters(ctx);
- }
- }
-}
-
-void TPartition::SetupTopicCounters(const TActorContext& ctx) {
- auto counters = AppData(ctx)->Counters;
- auto labels = NKikimr::NPQ::GetLabels(TopicName);
- const TString suffix = LocalDC ? "Original" : "Mirrored";
-
- WriteBufferIsFullCounter.SetCounter(
- GetCounters(counters, "writingTime", TopicName),
- {{"host", DCId},
- {"Partition", ToString<ui32>(Partition)}},
- {"sensor", "BufferFullTime" + suffix, true});
-
- InputTimeLag = THolder<NKikimr::NPQ::TPercentileCounter>(new NKikimr::NPQ::TPercentileCounter(
- GetServiceCounters(counters, "pqproxy|writeTimeLag"), GetLabels(TopicName),
- {{"sensor", "TimeLags" + suffix}}, "Interval",
- TVector<std::pair<ui64, TString>>{
- {100, "100ms"}, {200, "200ms"}, {500, "500ms"}, {1000, "1000ms"},
- {2000, "2000ms"}, {5000, "5000ms"}, {10'000, "10000ms"}, {30'000, "30000ms"},
- {60'000, "60000ms"}, {180'000,"180000ms"}, {9'999'999, "999999ms"}}, true));
-
-
- MessageSize = THolder<NKikimr::NPQ::TPercentileCounter>(new NKikimr::NPQ::TPercentileCounter(
- GetServiceCounters(counters, "pqproxy|writeInfo"), GetLabels(TopicName),
- {{"sensor", "MessageSize" + suffix}}, "Size",
- TVector<std::pair<ui64, TString>>{
- {1024, "1kb"}, {5120, "5kb"}, {10240, "10kb"},
- {20'480, "20kb"}, {51'200, "50kb"}, {102'400, "100kb"}, {204'800, "200kb"},
- {524'288, "512kb"},{1'048'576, "1024kb"}, {2'097'152,"2048kb"}, {5'242'880, "5120kb"},
- {10'485'760, "10240kb"}, {67'108'864, "65536kb"}, {999'999'999, "99999999kb"}}, true));
-
- BytesWritten = NKikimr::NPQ::TMultiCounter(GetServiceCounters(counters, "pqproxy|writeSession"),
- GetLabels(TopicName), {}, {"BytesWritten" + suffix}, true);
- BytesWrittenUncompressed = NKikimr::NPQ::TMultiCounter(GetServiceCounters(counters, "pqproxy|writeSession"),
- GetLabels(TopicName), {}, {"UncompressedBytesWritten" + suffix}, true);
-
- BytesWrittenComp = NKikimr::NPQ::TMultiCounter(GetServiceCounters(counters, "pqproxy|writeSession"),
- GetLabels(TopicName), {}, {"CompactedBytesWritten" + suffix}, true);
-
- MsgsWritten = NKikimr::NPQ::TMultiCounter(GetServiceCounters(counters, "pqproxy|writeSession"),
- GetLabels(TopicName), {}, {"MessagesWritten" + suffix}, true);
-
- TVector<NPQ::TLabelsInfo> aggr = {{{{"Account", NPersQueue::GetAccount(TopicName)}}, {"total"}}};
- ui32 border = AppData(ctx)->PQConfig.GetWriteLatencyBigMs();
- auto subGroup = GetServiceCounters(counters, "pqproxy|SLI");
- WriteLatency = NKikimr::NPQ::CreateSLIDurationCounter(subGroup, aggr, "Write", border,
- {100, 200, 500, 1000, 1500, 2000,
- 5000, 10'000, 30'000, 99'999'999});
- SLIBigLatency = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"WriteBigLatency"}, true, "sensor", false);
- WritesTotal = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"WritesTotal"}, true, "sensor", false);
- if (IsQuotingEnabled() && !TopicWriteQuotaResourcePath.empty()) {
- TopicWriteQuotaWaitCounter = THolder<NKikimr::NPQ::TPercentileCounter>(
- new NKikimr::NPQ::TPercentileCounter(
- GetServiceCounters(counters, "pqproxy|topicWriteQuotaWait"), GetLabels(TopicName),
- {{"sensor", "TopicWriteQuotaWait" + suffix}}, "Interval",
- TVector<std::pair<ui64, TString>>{
- {0, "0ms"}, {1, "1ms"}, {5, "5ms"}, {10, "10ms"},
- {20, "20ms"}, {50, "50ms"}, {100, "100ms"}, {500, "500ms"},
- {1000, "1000ms"}, {2500, "2500ms"}, {5000, "5000ms"},
- {10'000, "10000ms"}, {9'999'999, "999999ms"}}, true));
- }
-
- PartitionWriteQuotaWaitCounter = THolder<NKikimr::NPQ::TPercentileCounter>(
- new NKikimr::NPQ::TPercentileCounter(GetServiceCounters(counters, "pqproxy|partitionWriteQuotaWait"),
- GetLabels(TopicName), {{"sensor", "PartitionWriteQuotaWait" + suffix}}, "Interval",
- TVector<std::pair<ui64, TString>>{
- {0, "0ms"}, {1, "1ms"}, {5, "5ms"}, {10, "10ms"},
- {20, "20ms"}, {50, "50ms"}, {100, "100ms"}, {500, "500ms"},
- {1000, "1000ms"}, {2500, "2500ms"}, {5000, "5000ms"},
- {10'000, "10000ms"}, {9'999'999, "999999ms"}}, true));
-}
-
-void TPartition::SetupStreamCounters(const TActorContext& ctx) {
- auto counters = AppData(ctx)->Counters;
- auto labels = NKikimr::NPQ::GetLabelsForStream(TopicName, CloudId, DbId, FolderId);
-
- WriteBufferIsFullCounter.SetCounter(
- GetCountersForStream(counters, "writingTime"),
- {{"host", DCId},
- {"partition", ToString<ui32>(Partition)},
- {"stream", TopicName}},
- {"name", "stream.internal_write.buffer_brimmed_duration_ms", true});
-
- InputTimeLag = THolder<NKikimr::NPQ::TPercentileCounter>(new NKikimr::NPQ::TPercentileCounter(
- NKikimr::NPQ::GetCountersForStream(counters, "writeTimeLag"), labels,
- {{"name", "stream.internal_write.time_lags_milliseconds"}}, "bin",
- TVector<std::pair<ui64, TString>>{
- {100, "100"}, {200, "200"}, {500, "500"},
- {1000, "1000"}, {2000, "2000"}, {5000, "5000"},
- {10'000, "10000"}, {30'000, "30000"}, {60'000, "60000"},
- {180'000,"180000"}, {9'999'999, "999999"}}, true));
-
- MessageSize = THolder<NKikimr::NPQ::TPercentileCounter>(new NKikimr::NPQ::TPercentileCounter(
- NKikimr::NPQ::GetCountersForStream(counters, "writeInfo"), labels,
- {{"name", "stream.internal_write.record_size_bytes"}}, "bin",
- TVector<std::pair<ui64, TString>>{
- {1024, "1024"}, {5120, "5120"}, {10'240, "10240"},
- {20'480, "20480"}, {51'200, "51200"}, {102'400, "102400"},
- {204'800, "204800"}, {524'288, "524288"},{1'048'576, "1048576"},
- {2'097'152,"2097152"}, {5'242'880, "5242880"}, {10'485'760, "10485760"},
- {67'108'864, "67108864"}, {999'999'999, "99999999"}}, true));
-
- BytesWritten = NKikimr::NPQ::TMultiCounter(
- NKikimr::NPQ::GetCountersForStream(counters, "writeSession"), labels, {},
- {"stream.internal_write.bytes_per_second",
- "stream.incoming_bytes_per_second"} , true, "name");
- MsgsWritten = NKikimr::NPQ::TMultiCounter(
- NKikimr::NPQ::GetCountersForStream(counters, "writeSession"), labels, {},
- {"stream.internal_write.records_per_second",
- "stream.incoming_records_per_second"}, true, "name");
-
- BytesWrittenUncompressed = NKikimr::NPQ::TMultiCounter(
- NKikimr::NPQ::GetCountersForStream(counters, "writeSession"), labels, {},
- {"stream.internal_write.uncompressed_bytes_per_second"}, true, "name");
- BytesWrittenComp = NKikimr::NPQ::TMultiCounter(
- NKikimr::NPQ::GetCountersForStream(counters, "writeSession"), labels, {},
- {"stream.internal_write.compacted_bytes_per_second"}, true, "name");
-
- TVector<NPQ::TLabelsInfo> aggr = {{{{"Account", NPersQueue::GetAccount(TopicName)}}, {"total"}}};
- ui32 border = AppData(ctx)->PQConfig.GetWriteLatencyBigMs();
- auto subGroup = GetServiceCounters(counters, "pqproxy|SLI");
- WriteLatency = NKikimr::NPQ::CreateSLIDurationCounter(subGroup, aggr, "Write", border,
- {100, 200, 500, 1000, 1500, 2000,
- 5000, 10'000, 30'000, 99'999'999});
- SLIBigLatency = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"WriteBigLatency"}, true, "name", false);
- WritesTotal = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"WritesTotal"}, true, "name", false);
- if (IsQuotingEnabled() && !TopicWriteQuotaResourcePath.empty()) {
- TopicWriteQuotaWaitCounter = THolder<NKikimr::NPQ::TPercentileCounter>(
- new NKikimr::NPQ::TPercentileCounter(
- GetCountersForStream(counters, "topicWriteQuotaWait"), labels,
- {{"name", "stream.internal_write.topic_write_quota_wait_milliseconds"}}, "bin",
- TVector<std::pair<ui64, TString>>{
- {0, "0"}, {1, "1"}, {5, "5"}, {10, "10"},
- {20, "20"}, {50, "50"}, {100, "100"}, {500, "500"},
- {1000, "1000"}, {2500, "2500"}, {5000, "5000"},
- {10'000, "10000"}, {9'999'999, "999999"}}, true));
- }
-
- PartitionWriteQuotaWaitCounter = THolder<NKikimr::NPQ::TPercentileCounter>(
- new NKikimr::NPQ::TPercentileCounter(
- GetCountersForStream(counters, "partitionWriteQuotaWait"), labels,
- {{"name", "stream.internal_write.partition_write_quota_wait_milliseconds"}}, "bin",
- TVector<std::pair<ui64, TString>>{
- {0, "0"}, {1, "1"}, {5, "5"}, {10, "10"},
- {20, "20"}, {50, "50"}, {100, "100"}, {500, "500"},
- {1000, "1000"}, {2500, "2500"}, {5000, "5000"},
- {10'000, "10000"}, {9'999'999, "999999"}}, true));
+ TVector<NPQ::TLabelsInfo> labels;
+ if (AppData()->PQConfig.GetTopicsAreFirstClassCitizen()) {
+ SetupStreamCounters(ctx);
+ } else {
+ if (TopicName.find("--") == TString::npos)
+ return;
+ SetupTopicCounters(ctx);
+ }
+ }
+}
+
+void TPartition::SetupTopicCounters(const TActorContext& ctx) {
+ auto counters = AppData(ctx)->Counters;
+ auto labels = NKikimr::NPQ::GetLabels(TopicName);
+ const TString suffix = LocalDC ? "Original" : "Mirrored";
+
+ WriteBufferIsFullCounter.SetCounter(
+ GetCounters(counters, "writingTime", TopicName),
+ {{"host", DCId},
+ {"Partition", ToString<ui32>(Partition)}},
+ {"sensor", "BufferFullTime" + suffix, true});
+
+ InputTimeLag = THolder<NKikimr::NPQ::TPercentileCounter>(new NKikimr::NPQ::TPercentileCounter(
+ GetServiceCounters(counters, "pqproxy|writeTimeLag"), GetLabels(TopicName),
+ {{"sensor", "TimeLags" + suffix}}, "Interval",
+ TVector<std::pair<ui64, TString>>{
+ {100, "100ms"}, {200, "200ms"}, {500, "500ms"}, {1000, "1000ms"},
+ {2000, "2000ms"}, {5000, "5000ms"}, {10'000, "10000ms"}, {30'000, "30000ms"},
+ {60'000, "60000ms"}, {180'000,"180000ms"}, {9'999'999, "999999ms"}}, true));
+
+
+ MessageSize = THolder<NKikimr::NPQ::TPercentileCounter>(new NKikimr::NPQ::TPercentileCounter(
+ GetServiceCounters(counters, "pqproxy|writeInfo"), GetLabels(TopicName),
+ {{"sensor", "MessageSize" + suffix}}, "Size",
+ TVector<std::pair<ui64, TString>>{
+ {1024, "1kb"}, {5120, "5kb"}, {10240, "10kb"},
+ {20'480, "20kb"}, {51'200, "50kb"}, {102'400, "100kb"}, {204'800, "200kb"},
+ {524'288, "512kb"},{1'048'576, "1024kb"}, {2'097'152,"2048kb"}, {5'242'880, "5120kb"},
+ {10'485'760, "10240kb"}, {67'108'864, "65536kb"}, {999'999'999, "99999999kb"}}, true));
+
+ BytesWritten = NKikimr::NPQ::TMultiCounter(GetServiceCounters(counters, "pqproxy|writeSession"),
+ GetLabels(TopicName), {}, {"BytesWritten" + suffix}, true);
+ BytesWrittenUncompressed = NKikimr::NPQ::TMultiCounter(GetServiceCounters(counters, "pqproxy|writeSession"),
+ GetLabels(TopicName), {}, {"UncompressedBytesWritten" + suffix}, true);
+
+ BytesWrittenComp = NKikimr::NPQ::TMultiCounter(GetServiceCounters(counters, "pqproxy|writeSession"),
+ GetLabels(TopicName), {}, {"CompactedBytesWritten" + suffix}, true);
+
+ MsgsWritten = NKikimr::NPQ::TMultiCounter(GetServiceCounters(counters, "pqproxy|writeSession"),
+ GetLabels(TopicName), {}, {"MessagesWritten" + suffix}, true);
+
+ TVector<NPQ::TLabelsInfo> aggr = {{{{"Account", NPersQueue::GetAccount(TopicName)}}, {"total"}}};
+ ui32 border = AppData(ctx)->PQConfig.GetWriteLatencyBigMs();
+ auto subGroup = GetServiceCounters(counters, "pqproxy|SLI");
+ WriteLatency = NKikimr::NPQ::CreateSLIDurationCounter(subGroup, aggr, "Write", border,
+ {100, 200, 500, 1000, 1500, 2000,
+ 5000, 10'000, 30'000, 99'999'999});
+ SLIBigLatency = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"WriteBigLatency"}, true, "sensor", false);
+ WritesTotal = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"WritesTotal"}, true, "sensor", false);
+ if (IsQuotingEnabled() && !TopicWriteQuotaResourcePath.empty()) {
+ TopicWriteQuotaWaitCounter = THolder<NKikimr::NPQ::TPercentileCounter>(
+ new NKikimr::NPQ::TPercentileCounter(
+ GetServiceCounters(counters, "pqproxy|topicWriteQuotaWait"), GetLabels(TopicName),
+ {{"sensor", "TopicWriteQuotaWait" + suffix}}, "Interval",
+ TVector<std::pair<ui64, TString>>{
+ {0, "0ms"}, {1, "1ms"}, {5, "5ms"}, {10, "10ms"},
+ {20, "20ms"}, {50, "50ms"}, {100, "100ms"}, {500, "500ms"},
+ {1000, "1000ms"}, {2500, "2500ms"}, {5000, "5000ms"},
+ {10'000, "10000ms"}, {9'999'999, "999999ms"}}, true));
+ }
+
+ PartitionWriteQuotaWaitCounter = THolder<NKikimr::NPQ::TPercentileCounter>(
+ new NKikimr::NPQ::TPercentileCounter(GetServiceCounters(counters, "pqproxy|partitionWriteQuotaWait"),
+ GetLabels(TopicName), {{"sensor", "PartitionWriteQuotaWait" + suffix}}, "Interval",
+ TVector<std::pair<ui64, TString>>{
+ {0, "0ms"}, {1, "1ms"}, {5, "5ms"}, {10, "10ms"},
+ {20, "20ms"}, {50, "50ms"}, {100, "100ms"}, {500, "500ms"},
+ {1000, "1000ms"}, {2500, "2500ms"}, {5000, "5000ms"},
+ {10'000, "10000ms"}, {9'999'999, "999999ms"}}, true));
+}
+
+void TPartition::SetupStreamCounters(const TActorContext& ctx) {
+ auto counters = AppData(ctx)->Counters;
+ auto labels = NKikimr::NPQ::GetLabelsForStream(TopicName, CloudId, DbId, FolderId);
+
+ WriteBufferIsFullCounter.SetCounter(
+ GetCountersForStream(counters, "writingTime"),
+ {{"host", DCId},
+ {"partition", ToString<ui32>(Partition)},
+ {"stream", TopicName}},
+ {"name", "stream.internal_write.buffer_brimmed_duration_ms", true});
+
+ InputTimeLag = THolder<NKikimr::NPQ::TPercentileCounter>(new NKikimr::NPQ::TPercentileCounter(
+ NKikimr::NPQ::GetCountersForStream(counters, "writeTimeLag"), labels,
+ {{"name", "stream.internal_write.time_lags_milliseconds"}}, "bin",
+ TVector<std::pair<ui64, TString>>{
+ {100, "100"}, {200, "200"}, {500, "500"},
+ {1000, "1000"}, {2000, "2000"}, {5000, "5000"},
+ {10'000, "10000"}, {30'000, "30000"}, {60'000, "60000"},
+ {180'000,"180000"}, {9'999'999, "999999"}}, true));
+
+ MessageSize = THolder<NKikimr::NPQ::TPercentileCounter>(new NKikimr::NPQ::TPercentileCounter(
+ NKikimr::NPQ::GetCountersForStream(counters, "writeInfo"), labels,
+ {{"name", "stream.internal_write.record_size_bytes"}}, "bin",
+ TVector<std::pair<ui64, TString>>{
+ {1024, "1024"}, {5120, "5120"}, {10'240, "10240"},
+ {20'480, "20480"}, {51'200, "51200"}, {102'400, "102400"},
+ {204'800, "204800"}, {524'288, "524288"},{1'048'576, "1048576"},
+ {2'097'152,"2097152"}, {5'242'880, "5242880"}, {10'485'760, "10485760"},
+ {67'108'864, "67108864"}, {999'999'999, "99999999"}}, true));
+
+ BytesWritten = NKikimr::NPQ::TMultiCounter(
+ NKikimr::NPQ::GetCountersForStream(counters, "writeSession"), labels, {},
+ {"stream.internal_write.bytes_per_second",
+ "stream.incoming_bytes_per_second"} , true, "name");
+ MsgsWritten = NKikimr::NPQ::TMultiCounter(
+ NKikimr::NPQ::GetCountersForStream(counters, "writeSession"), labels, {},
+ {"stream.internal_write.records_per_second",
+ "stream.incoming_records_per_second"}, true, "name");
+
+ BytesWrittenUncompressed = NKikimr::NPQ::TMultiCounter(
+ NKikimr::NPQ::GetCountersForStream(counters, "writeSession"), labels, {},
+ {"stream.internal_write.uncompressed_bytes_per_second"}, true, "name");
+ BytesWrittenComp = NKikimr::NPQ::TMultiCounter(
+ NKikimr::NPQ::GetCountersForStream(counters, "writeSession"), labels, {},
+ {"stream.internal_write.compacted_bytes_per_second"}, true, "name");
+
+ TVector<NPQ::TLabelsInfo> aggr = {{{{"Account", NPersQueue::GetAccount(TopicName)}}, {"total"}}};
+ ui32 border = AppData(ctx)->PQConfig.GetWriteLatencyBigMs();
+ auto subGroup = GetServiceCounters(counters, "pqproxy|SLI");
+ WriteLatency = NKikimr::NPQ::CreateSLIDurationCounter(subGroup, aggr, "Write", border,
+ {100, 200, 500, 1000, 1500, 2000,
+ 5000, 10'000, 30'000, 99'999'999});
+ SLIBigLatency = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"WriteBigLatency"}, true, "name", false);
+ WritesTotal = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"WritesTotal"}, true, "name", false);
+ if (IsQuotingEnabled() && !TopicWriteQuotaResourcePath.empty()) {
+ TopicWriteQuotaWaitCounter = THolder<NKikimr::NPQ::TPercentileCounter>(
+ new NKikimr::NPQ::TPercentileCounter(
+ GetCountersForStream(counters, "topicWriteQuotaWait"), labels,
+ {{"name", "stream.internal_write.topic_write_quota_wait_milliseconds"}}, "bin",
+ TVector<std::pair<ui64, TString>>{
+ {0, "0"}, {1, "1"}, {5, "5"}, {10, "10"},
+ {20, "20"}, {50, "50"}, {100, "100"}, {500, "500"},
+ {1000, "1000"}, {2500, "2500"}, {5000, "5000"},
+ {10'000, "10000"}, {9'999'999, "999999"}}, true));
+ }
+
+ PartitionWriteQuotaWaitCounter = THolder<NKikimr::NPQ::TPercentileCounter>(
+ new NKikimr::NPQ::TPercentileCounter(
+ GetCountersForStream(counters, "partitionWriteQuotaWait"), labels,
+ {{"name", "stream.internal_write.partition_write_quota_wait_milliseconds"}}, "bin",
+ TVector<std::pair<ui64, TString>>{
+ {0, "0"}, {1, "1"}, {5, "5"}, {10, "10"},
+ {20, "20"}, {50, "50"}, {100, "100"}, {500, "500"},
+ {1000, "1000"}, {2500, "2500"}, {5000, "5000"},
+ {10'000, "10000"}, {9'999'999, "999999"}}, true));
}
void TPartition::ProcessHasDataRequests(const TActorContext& ctx) {
@@ -2330,11 +2330,11 @@ TReadAnswer TReadInfo::FormAnswer(
ui16 internalPartsCount = blobs[pos].InternalPartsCount;
const TString& blobValue = blobs[pos].Value;
- if (blobValue.empty()) { // this is ok. Means that someone requested too much data
+ if (blobValue.empty()) { // this is ok. Means that someone requested too much data
LOG_DEBUG(ctx, NKikimrServices::PERSQUEUE, "Not full answer here!");
ui64 answerSize = answer->Response.ByteSize();
if (userInfo && Destination != 0) {
- userInfo->ReadDone(ctx, ctx.Now(), answerSize, cnt, ClientDC);
+ userInfo->ReadDone(ctx, ctx.Now(), answerSize, cnt, ClientDC);
}
readResult->SetSizeLag(sizeLag - size);
return {answerSize, std::move(answer)};
@@ -2342,9 +2342,9 @@ TReadAnswer TReadInfo::FormAnswer(
Y_VERIFY(blobValue.size() == blobs[pos].Size, "value for offset %" PRIu64 " count %u size must be %u, but got %u",
offset, count, blobs[pos].Size, (ui32)blobValue.size());
- if (offset > Offset || (offset == Offset && partNo > PartNo)) { // got gap
+ if (offset > Offset || (offset == Offset && partNo > PartNo)) { // got gap
Offset = offset;
- PartNo = partNo;
+ PartNo = partNo;
}
Y_VERIFY(offset <= Offset);
Y_VERIFY(offset < Offset || partNo <= PartNo);
@@ -2362,7 +2362,7 @@ TReadAnswer TReadInfo::FormAnswer(
}
offset += header.GetCount();
- if (pos == Max<ui32>()) // this batch does not contain data to read, skip it
+ if (pos == Max<ui32>()) // this batch does not contain data to read, skip it
continue;
@@ -2375,31 +2375,31 @@ TReadAnswer TReadInfo::FormAnswer(
psize = size;
TClientBlob &res = batch.Blobs[i];
VERIFY_RESULT_BLOB(res, i);
- bool messageSkippingBehaviour = AppData()->PQConfig.GetTopicsAreFirstClassCitizen() &&
- ReadTimestampMs > res.WriteTimestamp.MilliSeconds();
- if (!messageSkippingBehaviour) {
- size += res.GetBlobSize();
- Y_VERIFY(PartNo == res.GetPartNo(), "pos %" PRIu32 " i %" PRIu32 " Offset %" PRIu64 " PartNo %" PRIu16 " offset %" PRIu64 " partNo %" PRIu16,
- pos, i, Offset, PartNo, offset, res.GetPartNo());
-
- if (userInfo) {
- userInfo->AddTimestampToCache(
- Offset, res.WriteTimestamp, res.CreateTimestamp,
- Destination != 0, ctx.Now()
- );
- }
-
- AddResultBlob(readResult, res, Offset);
- if (res.IsLastPart()) {
- ++cnt;
- }
+ bool messageSkippingBehaviour = AppData()->PQConfig.GetTopicsAreFirstClassCitizen() &&
+ ReadTimestampMs > res.WriteTimestamp.MilliSeconds();
+ if (!messageSkippingBehaviour) {
+ size += res.GetBlobSize();
+ Y_VERIFY(PartNo == res.GetPartNo(), "pos %" PRIu32 " i %" PRIu32 " Offset %" PRIu64 " PartNo %" PRIu16 " offset %" PRIu64 " partNo %" PRIu16,
+ pos, i, Offset, PartNo, offset, res.GetPartNo());
+
+ if (userInfo) {
+ userInfo->AddTimestampToCache(
+ Offset, res.WriteTimestamp, res.CreateTimestamp,
+ Destination != 0, ctx.Now()
+ );
+ }
+
+ AddResultBlob(readResult, res, Offset);
+ if (res.IsLastPart()) {
+ ++cnt;
+ }
}
- if (res.IsLastPart()) {
+ if (res.IsLastPart()) {
PartNo = 0;
++Offset;
- } else {
- ++PartNo;
+ } else {
+ ++PartNo;
}
}
@@ -2444,7 +2444,7 @@ TReadAnswer TReadInfo::FormAnswer(
Y_VERIFY(Offset <= (ui64)Max<i64>(), "Offset is too big: %" PRIu64, Offset);
ui64 answerSize = answer->Response.ByteSize();
if (userInfo && Destination != 0) {
- userInfo->ReadDone(ctx, ctx.Now(), answerSize, cnt, ClientDC);
+ userInfo->ReadDone(ctx, ctx.Now(), answerSize, cnt, ClientDC);
}
readResult->SetSizeLag(sizeLag - size);
return {answerSize, std::move(answer)};
@@ -2472,7 +2472,7 @@ void TPartition::Handle(TEvPQ::TEvReadTimeout::TPtr& ev, const TActorContext& ct
}
-TVector<TRequestedBlob> TPartition::GetReadRequestFromBody(const ui64 startOffset, const ui16 partNo, const ui32 maxCount, const ui32 maxSize, ui32* rcount, ui32* rsize)
+TVector<TRequestedBlob> TPartition::GetReadRequestFromBody(const ui64 startOffset, const ui16 partNo, const ui32 maxCount, const ui32 maxSize, ui32* rcount, ui32* rsize)
{
Y_VERIFY(rcount && rsize);
ui32& count = *rcount;
@@ -2502,12 +2502,12 @@ TVector<TRequestedBlob> TPartition::GetReadRequestFromBody(const ui64 startOffse
sz = (cnt == it->Key.GetCount() ? it->Size : 0); //not readed client blobs can be of ~8Mb, so don't count this size at all
}
while (it != DataKeysBody.end() && size < maxSize && count < maxCount) {
- size += sz;
- count += cnt;
- TRequestedBlob reqBlob(it->Key.GetOffset(), it->Key.GetPartNo(), it->Key.GetCount(),
- it->Key.GetInternalPartsCount(), it->Size, TString());
- blobs.push_back(reqBlob);
-
+ size += sz;
+ count += cnt;
+ TRequestedBlob reqBlob(it->Key.GetOffset(), it->Key.GetPartNo(), it->Key.GetCount(),
+ it->Key.GetInternalPartsCount(), it->Size, TString());
+ blobs.push_back(reqBlob);
+
++it;
if (it == DataKeysBody.end())
break;
@@ -2520,12 +2520,12 @@ TVector<TRequestedBlob> TPartition::GetReadRequestFromBody(const ui64 startOffse
-TVector<TClientBlob> TPartition::GetReadRequestFromHead(const ui64 startOffset, const ui16 partNo, const ui32 maxCount, const ui32 maxSize, const ui64 readTimestampMs, ui32* rcount, ui32* rsize, ui64* insideHeadOffset)
+TVector<TClientBlob> TPartition::GetReadRequestFromHead(const ui64 startOffset, const ui16 partNo, const ui32 maxCount, const ui32 maxSize, const ui64 readTimestampMs, ui32* rcount, ui32* rsize, ui64* insideHeadOffset)
{
ui32& count = *rcount;
ui32& size = *rsize;
TVector<TClientBlob> res;
- std::optional<ui64> firstAddedBlobOffset{};
+ std::optional<ui64> firstAddedBlobOffset{};
ui32 pos = 0;
if (startOffset > Head.Offset || startOffset == Head.Offset && partNo > Head.PartNo) {
pos = Head.FindPos(startOffset, partNo);
@@ -2542,10 +2542,10 @@ TVector<TClientBlob> TPartition::GetReadRequestFromHead(const ui64 startOffset,
for (; i < blobs.size(); ++i) {
Y_VERIFY(pno == blobs[i].GetPartNo());
- bool messageSkippingBehaviour = AppData()->PQConfig.GetTopicsAreFirstClassCitizen() &&
- readTimestampMs > blobs[i].WriteTimestamp.MilliSeconds();
- bool skip = offset < startOffset || offset == startOffset &&
- blobs[i].GetPartNo() < partNo || messageSkippingBehaviour;
+ bool messageSkippingBehaviour = AppData()->PQConfig.GetTopicsAreFirstClassCitizen() &&
+ readTimestampMs > blobs[i].WriteTimestamp.MilliSeconds();
+ bool skip = offset < startOffset || offset == startOffset &&
+ blobs[i].GetPartNo() < partNo || messageSkippingBehaviour;
if (blobs[i].IsLastPart()) {
++offset;
pno = 0;
@@ -2556,19 +2556,19 @@ TVector<TClientBlob> TPartition::GetReadRequestFromHead(const ui64 startOffset,
}
if (skip)
continue;
- if (count > maxCount) // blob is counted already
+ if (count > maxCount) // blob is counted already
break;
if (size >= maxSize)
break;
- size += blobs[i].GetBlobSize();
- res.push_back(blobs[i]);
- if (!firstAddedBlobOffset && AppData()->PQConfig.GetTopicsAreFirstClassCitizen())
- firstAddedBlobOffset = offset > 0 ? offset - 1 : 0;
+ size += blobs[i].GetBlobSize();
+ res.push_back(blobs[i]);
+ if (!firstAddedBlobOffset && AppData()->PQConfig.GetTopicsAreFirstClassCitizen())
+ firstAddedBlobOffset = offset > 0 ? offset - 1 : 0;
}
if (i < blobs.size()) // already got limit
break;
}
- *insideHeadOffset = firstAddedBlobOffset.value_or(*insideHeadOffset);
+ *insideHeadOffset = firstAddedBlobOffset.value_or(*insideHeadOffset);
return res;
}
@@ -2847,9 +2847,9 @@ void TPartition::ReadTimestampForOffset(const TString& user, TUserInfo& userInfo
<< " user " << user << " send read request for offset " << userInfo.Offset << " initiated " << " queuesize " << UpdateUserInfoTimestamp.size() << " startOffset " << StartOffset << " ReadingTimestamp " << ReadingTimestamp);
- THolder<TEvPQ::TEvRead> event = MakeHolder<TEvPQ::TEvRead>(0, userInfo.Offset, 0, 1, "",
- user, 0, MAX_BLOB_PART_SIZE * 2, 0, 0, "",
- userInfo.DoExternalRead);
+ THolder<TEvPQ::TEvRead> event = MakeHolder<TEvPQ::TEvRead>(0, userInfo.Offset, 0, 1, "",
+ user, 0, MAX_BLOB_PART_SIZE * 2, 0, 0, "",
+ userInfo.DoExternalRead);
ctx.Send(ctx.SelfID, event.Release());
Counters.Cumulative()[COUNTER_PQ_WRITE_TIMESTAMP_CACHE_MISS].Increment(1);
}
@@ -3314,8 +3314,8 @@ void TPartition::Handle(TEvKeyValue::TEvResponse::TPtr& ev, const TActorContext&
//check correctness of response
if (response.GetStatus() != NMsgBusProxy::MSTATUS_OK) {
- LOG_ERROR_S(ctx, NKikimrServices::PERSQUEUE, "OnWrite topic '" << TopicName << "' partition " <<
- Partition << " commands are not processed at all, reason: " << response.DebugString());
+ LOG_ERROR_S(ctx, NKikimrServices::PERSQUEUE, "OnWrite topic '" << TopicName << "' partition " <<
+ Partition << " commands are not processed at all, reason: " << response.DebugString());
ctx.Send(Tablet, new TEvents::TEvPoisonPill());
//TODO: if status is DISK IS FULL, is global status MSTATUS_OK? it will be good if it is true
return;
@@ -3500,13 +3500,13 @@ void TPartition::HandleWriteResponse(const TActorContext& ctx) {
Counters.Percentile()[COUNTER_PQ_WRITE_CYCLE_BYTES].IncrementFor(WriteCycleSize);
Counters.Percentile()[COUNTER_PQ_WRITE_NEW_BYTES].IncrementFor(WriteNewSize);
if (BytesWritten)
- BytesWritten.Inc(WriteNewSizeInternal);
+ BytesWritten.Inc(WriteNewSizeInternal);
if (BytesWrittenUncompressed)
BytesWrittenUncompressed.Inc(WriteNewSizeUncompressed);
if (BytesWrittenComp)
BytesWrittenComp.Inc(WriteCycleSize);
if (MsgsWritten)
- MsgsWritten.Inc(WriteNewMessagesInternal);
+ MsgsWritten.Inc(WriteNewMessagesInternal);
//All ok
auto now = ctx.Now();
@@ -3525,10 +3525,10 @@ void TPartition::HandleWriteResponse(const TActorContext& ctx) {
WriteCycleSize = 0;
WriteNewSize = 0;
- WriteNewSizeInternal = 0;
+ WriteNewSizeInternal = 0;
WriteNewSizeUncompressed = 0;
WriteNewMessages = 0;
- WriteNewMessagesInternal = 0;
+ WriteNewMessagesInternal = 0;
UpdateWriteBufferIsFullState(now);
AnswerCurrentWrites(ctx);
@@ -3958,8 +3958,8 @@ void TPartition::CancelAllWritesOnWrite(const TActorContext& ctx, TEvKeyValue::T
}
-bool TPartition::AppendHeadWithNewWrites(TEvKeyValue::TEvRequest* request, const TActorContext& ctx,
- TSourceIdWriter& sourceIdWriter) {
+bool TPartition::AppendHeadWithNewWrites(TEvKeyValue::TEvRequest* request, const TActorContext& ctx,
+ TSourceIdWriter& sourceIdWriter) {
ui64 curOffset = PartitionedBlob.IsInited() ? PartitionedBlob.GetOffset() : EndOffset;
@@ -4112,14 +4112,14 @@ bool TPartition::AppendHeadWithNewWrites(TEvKeyValue::TEvRequest* request, const
}
}
}
- PartitionedBlob = TPartitionedBlob(Partition, curOffset, p.Msg.SourceId, p.Msg.SeqNo,
- p.Msg.TotalParts, p.Msg.TotalSize, Head, NewHead,
- headCleared, needCompactHead, MaxBlobSize);
+ PartitionedBlob = TPartitionedBlob(Partition, curOffset, p.Msg.SourceId, p.Msg.SeqNo,
+ p.Msg.TotalParts, p.Msg.TotalSize, Head, NewHead,
+ headCleared, needCompactHead, MaxBlobSize);
}
LOG_DEBUG_S(ctx, NKikimrServices::PERSQUEUE, "Topic '" << TopicName << "' partition " << Partition
- << " part blob processing sourceId '" << EscapeC(p.Msg.SourceId) <<
- "' seqNo " << p.Msg.SeqNo << " partNo " << p.Msg.PartNo);
+ << " part blob processing sourceId '" << EscapeC(p.Msg.SourceId) <<
+ "' seqNo " << p.Msg.SeqNo << " partNo " << p.Msg.PartNo);
TString s;
if (!PartitionedBlob.IsNextPart(p.Msg.SourceId, p.Msg.SeqNo, p.Msg.PartNo, &s)) {
//this must not be happen - client sends gaps, fail this client till the end
@@ -4129,13 +4129,13 @@ bool TPartition::AppendHeadWithNewWrites(TEvKeyValue::TEvRequest* request, const
}
WriteNewSize += p.Msg.SourceId.size() + p.Msg.Data.size();
- WriteNewSizeInternal = p.Msg.External ? 0 : WriteNewSize;
+ WriteNewSizeInternal = p.Msg.External ? 0 : WriteNewSize;
WriteNewSizeUncompressed += p.Msg.UncompressedSize + p.Msg.SourceId.size();
- if (p.Msg.PartNo == 0) {
- ++WriteNewMessages;
- if (!p.Msg.External)
- ++WriteNewMessagesInternal;
- }
+ if (p.Msg.PartNo == 0) {
+ ++WriteNewMessages;
+ if (!p.Msg.External)
+ ++WriteNewMessagesInternal;
+ }
TMaybe<TPartData> partData;
if (p.Msg.TotalParts > 1) { //this is multi-part message
@@ -4175,12 +4175,12 @@ bool TPartition::AppendHeadWithNewWrites(TEvKeyValue::TEvRequest* request, const
write->SetKeyToCache(resKey.Data(), resKey.Size());
WriteCycleSize += newWrite.second.size();
- LOG_DEBUG_S(ctx, NKikimrServices::PERSQUEUE, "Topic '" << TopicName <<
- "' partition " << Partition <<
- " part blob sourceId '" << EscapeC(p.Msg.SourceId) <<
- "' seqNo " << p.Msg.SeqNo << " partNo " << p.Msg.PartNo <<
- " result is " << TStringBuf(newWrite.first.Data(), newWrite.first.Size()) <<
- " size " << newWrite.second.size());
+ LOG_DEBUG_S(ctx, NKikimrServices::PERSQUEUE, "Topic '" << TopicName <<
+ "' partition " << Partition <<
+ " part blob sourceId '" << EscapeC(p.Msg.SourceId) <<
+ "' seqNo " << p.Msg.SeqNo << " partNo " << p.Msg.PartNo <<
+ " result is " << TStringBuf(newWrite.first.Data(), newWrite.first.Size()) <<
+ " size " << newWrite.second.size());
}
if (lastBlobPart) {
@@ -4578,15 +4578,15 @@ void TPartition::ProcessRead(const TActorContext& ctx, TReadInfo&& info, const u
userInfo.UpdateReadingTimeAndState(ctx.Now());
return;
}
- TVector<TRequestedBlob> blobs = GetReadRequestFromBody(info.Offset, info.PartNo, info.Count, info.Size, &count, &size);
+ TVector<TRequestedBlob> blobs = GetReadRequestFromBody(info.Offset, info.PartNo, info.Count, info.Size, &count, &size);
info.Blobs = blobs;
ui64 lastOffset = info.Offset + Min(count, info.Count);
LOG_DEBUG_S(ctx, NKikimrServices::PERSQUEUE, "read cookie " << cookie << " added " << info.Blobs.size()
<< " blobs, size " << size << " count " << count << " last offset " << lastOffset);
- ui64 insideHeadOffset{0};
- info.Cached = GetReadRequestFromHead(info.Offset, info.PartNo, info.Count, info.Size, info.ReadTimestampMs, &count, &size, &insideHeadOffset);
- info.CachedOffset = Head.Offset > 0 ? Head.Offset : insideHeadOffset;
+ ui64 insideHeadOffset{0};
+ info.Cached = GetReadRequestFromHead(info.Offset, info.PartNo, info.Count, info.Size, info.ReadTimestampMs, &count, &size, &insideHeadOffset);
+ info.CachedOffset = Head.Offset > 0 ? Head.Offset : insideHeadOffset;
if (info.Destination != 0) {
++userInfo.ActiveReads;
@@ -4618,7 +4618,7 @@ void TPartition::ProcessRead(const TActorContext& ctx, TReadInfo&& info, const u
Y_VERIFY(res);
THolder<TEvPQ::TEvBlobRequest> request(new TEvPQ::TEvBlobRequest(user, cookie, Partition,
- lastOffset, std::move(blobs)));
+ lastOffset, std::move(blobs)));
ctx.Send(BlobCache, request.Release());
}
diff --git a/ydb/core/persqueue/partition.h b/ydb/core/persqueue/partition.h
index 2f9bc11db1..2eec26f251 100644
--- a/ydb/core/persqueue/partition.h
+++ b/ydb/core/persqueue/partition.h
@@ -141,8 +141,8 @@ private:
void OnReadRequestFinished(TReadInfo&& info, ui64 answerSize);
// will return rcount and rsize also
- TVector<TRequestedBlob> GetReadRequestFromBody(const ui64 startOffset, const ui16 partNo, const ui32 maxCount, const ui32 maxSize, ui32* rcount, ui32* rsize);
- TVector<TClientBlob> GetReadRequestFromHead(const ui64 startOffset, const ui16 partNo, const ui32 maxCount, const ui32 maxSize, const ui64 readTimestampMs, ui32* rcount, ui32* rsize, ui64* insideHeadOffset);
+ TVector<TRequestedBlob> GetReadRequestFromBody(const ui64 startOffset, const ui16 partNo, const ui32 maxCount, const ui32 maxSize, ui32* rcount, ui32* rsize);
+ TVector<TClientBlob> GetReadRequestFromHead(const ui64 startOffset, const ui16 partNo, const ui32 maxCount, const ui32 maxSize, const ui64 readTimestampMs, ui32* rcount, ui32* rsize, ui64* insideHeadOffset);
void ProcessRead(const TActorContext& ctx, TReadInfo&& info, const ui64 cookie, bool subscription);
void HandleOnIdle(TEvPQ::TEvWrite::TPtr& ev, const TActorContext& ctx);
@@ -210,9 +210,9 @@ private:
void CreateMirrorerActor();
bool IsQuotingEnabled() const;
- void SetupTopicCounters(const TActorContext& ctx);
- void SetupStreamCounters(const TActorContext& ctx);
-
+ void SetupTopicCounters(const TActorContext& ctx);
+ void SetupStreamCounters(const TActorContext& ctx);
+
public:
static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
return NKikimrServices::TActivity::PERSQUEUE_PARTITION_ACTOR;
@@ -564,10 +564,10 @@ private:
std::deque<std::pair<ui64,ui64>> GapOffsets;
ui64 GapSize;
- TString CloudId;
- TString DbId;
- TString FolderId;
-
+ TString CloudId;
+ TString DbId;
+ TString FolderId;
+
TUsersInfoStorage UsersInfoStorage;
std::deque<TString> UpdateUserInfoTimestamp;
@@ -601,10 +601,10 @@ private:
TInstant WriteCycleStartTime;
ui32 WriteCycleSize;
ui32 WriteNewSize;
- ui32 WriteNewSizeInternal;
+ ui32 WriteNewSizeInternal;
ui64 WriteNewSizeUncompressed;
ui32 WriteNewMessages;
- ui32 WriteNewMessagesInternal;
+ ui32 WriteNewMessagesInternal;
TInstant CurrentTimestamp;
diff --git a/ydb/core/persqueue/percentile_counter.cpp b/ydb/core/persqueue/percentile_counter.cpp
index 780c417e18..a6414e70bd 100644
--- a/ydb/core/persqueue/percentile_counter.cpp
+++ b/ydb/core/persqueue/percentile_counter.cpp
@@ -7,11 +7,11 @@ namespace NKikimr {
namespace NPQ {
-NMonitoring::TDynamicCounterPtr GetCounters(NMonitoring::TDynamicCounterPtr counters,
- const TString& subsystem, const TString& topic)
+NMonitoring::TDynamicCounterPtr GetCounters(NMonitoring::TDynamicCounterPtr counters,
+ const TString& subsystem, const TString& topic)
{
auto pos = topic.find("--");
- Y_VERIFY(pos != TString::npos);
+ Y_VERIFY(pos != TString::npos);
TString origDC = topic.substr(4, pos - 4);
origDC.to_title();
@@ -21,20 +21,20 @@ NMonitoring::TDynamicCounterPtr GetCounters(NMonitoring::TDynamicCounterPtr coun
TString topicPath = NPersQueue::ConvertOldTopicName(realTopic);
TString account = topicPath.substr(0, topicPath.find("/"));
return GetServiceCounters(counters, "pqproxy|" + subsystem)
- ->GetSubgroup("OriginDC", origDC)
- ->GetSubgroup("Producer", producer)
- ->GetSubgroup("TopicPath", topicPath)
- ->GetSubgroup("Account", account)
- ->GetSubgroup("Topic", realTopic);
+ ->GetSubgroup("OriginDC", origDC)
+ ->GetSubgroup("Producer", producer)
+ ->GetSubgroup("TopicPath", topicPath)
+ ->GetSubgroup("Account", account)
+ ->GetSubgroup("Topic", realTopic);
+}
+
+NMonitoring::TDynamicCounterPtr GetCountersForStream(NMonitoring::TDynamicCounterPtr counters,
+ const TString& subsystem)
+{
+ return counters->GetSubgroup("counters", "pqproxy")
+ ->GetSubgroup("subsystem", subsystem);
}
-NMonitoring::TDynamicCounterPtr GetCountersForStream(NMonitoring::TDynamicCounterPtr counters,
- const TString& subsystem)
-{
- return counters->GetSubgroup("counters", "pqproxy")
- ->GetSubgroup("subsystem", subsystem);
-}
-
TVector<TLabelsInfo> GetLabels(const TString& topic)
{
auto pos = topic.find("--");
@@ -66,71 +66,71 @@ TVector<TLabelsInfo> GetLabels(const TString& cluster, const TString& realTopic)
return res;
}
-TVector<TLabelsInfo> GetLabelsForStream(const TString& topic, const TString& cloudId,
- const TString& dbId, const TString& folderId) {
- TVector<TLabelsInfo> res = {
- {{{"database", dbId}}, {dbId}},
- {{{"cloud", cloudId}}, {cloudId}},
- {{{"folder", folderId}}, {folderId}},
- {{{"stream", topic}}, {topic}}};
- return res;
-}
-
-TMultiCounter::TMultiCounter(NMonitoring::TDynamicCounterPtr counters,
- const TVector<TLabelsInfo>& labels,
- const TVector<std::pair<TString, TString>>& subgroups,
- const TVector<TString>& counter_names,
- bool deriv,
- const TString& name,
- bool expiring)
- : Value(0)
-{
- Y_VERIFY(counters);
-
- for (const auto& counter : counter_names) {
- for (ui32 i = 0; i <= labels.size(); ++i) {
- auto cc = counters;
- for (ui32 j = 0; j < labels.size(); ++j) {
- Y_VERIFY(!labels[j].Labels.empty());
- for (ui32 k = 0; k < labels[j].Labels.size(); ++k) {
- Y_VERIFY(labels[j].Labels.size() == labels[j].AggrNames.size());
- const TString& res = (j < i) ? labels[j].Labels[k].second : labels[j].AggrNames[k];
- cc = cc->GetSubgroup(labels[j].Labels[k].first, res);
- }
- }
- for (const auto& g: subgroups) {
- cc = cc->GetSubgroup(g.first, g.second);
- }
- if (expiring) {
- Counters.push_back(cc->GetExpiringNamedCounter(name, counter, deriv));
- } else {
- Counters.push_back(cc->GetNamedCounter(name, counter, deriv));
- }
- }
- }
-}
-
-void TMultiCounter::Inc(ui64 val)
-{
- for (auto& c : Counters) (*c) += val;
- Value += val;
-}
-
-void TMultiCounter::Dec(ui64 val) {
- for (auto& c : Counters) (*c) -= val;
- Value -= val;
-}
-
-void TMultiCounter::Set(ui64 value) {
- auto diff = value - Value;
- Inc(diff);
-}
-
-TMultiCounter::operator bool() {
- return !Counters.empty();
-}
-
-
+TVector<TLabelsInfo> GetLabelsForStream(const TString& topic, const TString& cloudId,
+ const TString& dbId, const TString& folderId) {
+ TVector<TLabelsInfo> res = {
+ {{{"database", dbId}}, {dbId}},
+ {{{"cloud", cloudId}}, {cloudId}},
+ {{{"folder", folderId}}, {folderId}},
+ {{{"stream", topic}}, {topic}}};
+ return res;
+}
+
+TMultiCounter::TMultiCounter(NMonitoring::TDynamicCounterPtr counters,
+ const TVector<TLabelsInfo>& labels,
+ const TVector<std::pair<TString, TString>>& subgroups,
+ const TVector<TString>& counter_names,
+ bool deriv,
+ const TString& name,
+ bool expiring)
+ : Value(0)
+{
+ Y_VERIFY(counters);
+
+ for (const auto& counter : counter_names) {
+ for (ui32 i = 0; i <= labels.size(); ++i) {
+ auto cc = counters;
+ for (ui32 j = 0; j < labels.size(); ++j) {
+ Y_VERIFY(!labels[j].Labels.empty());
+ for (ui32 k = 0; k < labels[j].Labels.size(); ++k) {
+ Y_VERIFY(labels[j].Labels.size() == labels[j].AggrNames.size());
+ const TString& res = (j < i) ? labels[j].Labels[k].second : labels[j].AggrNames[k];
+ cc = cc->GetSubgroup(labels[j].Labels[k].first, res);
+ }
+ }
+ for (const auto& g: subgroups) {
+ cc = cc->GetSubgroup(g.first, g.second);
+ }
+ if (expiring) {
+ Counters.push_back(cc->GetExpiringNamedCounter(name, counter, deriv));
+ } else {
+ Counters.push_back(cc->GetNamedCounter(name, counter, deriv));
+ }
+ }
+ }
+}
+
+void TMultiCounter::Inc(ui64 val)
+{
+ for (auto& c : Counters) (*c) += val;
+ Value += val;
+}
+
+void TMultiCounter::Dec(ui64 val) {
+ for (auto& c : Counters) (*c) -= val;
+ Value -= val;
+}
+
+void TMultiCounter::Set(ui64 value) {
+ auto diff = value - Value;
+ Inc(diff);
+}
+
+TMultiCounter::operator bool() {
+ return !Counters.empty();
+}
+
+
TPercentileCounter::TPercentileCounter(TIntrusivePtr<NMonitoring::TDynamicCounters> counters, const TVector<TLabelsInfo>& labels, const TVector<std::pair<TString, TString>>& subgroups, const TString& sensor,
const TVector<std::pair<ui64, TString>>& intervals, const bool deriv, bool expiring)
{
@@ -139,7 +139,7 @@ TPercentileCounter::TPercentileCounter(TIntrusivePtr<NMonitoring::TDynamicCounte
Ranges.reserve(intervals.size());
for (auto& interval : intervals) {
Ranges.push_back(interval.first);
- Counters.push_back(TMultiCounter(counters, labels, subgroups, {interval.second}, deriv, sensor, expiring));
+ Counters.push_back(TMultiCounter(counters, labels, subgroups, {interval.second}, deriv, sensor, expiring));
}
Ranges.back() = Max<ui64>();
}
diff --git a/ydb/core/persqueue/percentile_counter.h b/ydb/core/persqueue/percentile_counter.h
index ce2730ba7a..74ee43d287 100644
--- a/ydb/core/persqueue/percentile_counter.h
+++ b/ydb/core/persqueue/percentile_counter.h
@@ -5,11 +5,11 @@
namespace NKikimr {
namespace NPQ {
-NMonitoring::TDynamicCounterPtr GetCounters(NMonitoring::TDynamicCounterPtr counters,
- const TString& subsystem,
- const TString& topic);
-NMonitoring::TDynamicCounterPtr GetCountersForStream(NMonitoring::TDynamicCounterPtr counters,
- const TString& subsystem);
+NMonitoring::TDynamicCounterPtr GetCounters(NMonitoring::TDynamicCounterPtr counters,
+ const TString& subsystem,
+ const TString& topic);
+NMonitoring::TDynamicCounterPtr GetCountersForStream(NMonitoring::TDynamicCounterPtr counters,
+ const TString& subsystem);
struct TLabelsInfo {
TVector<std::pair<TString,TString>> Labels;
@@ -18,26 +18,26 @@ struct TLabelsInfo {
TVector<TLabelsInfo> GetLabels(const TString& topic);
TVector<TLabelsInfo> GetLabels(const TString& cluster, const TString& oldTopic);
-TVector<TLabelsInfo> GetLabelsForStream(const TString& topic, const TString& cloudId,
- const TString& dbId, const TString& folderId);
+TVector<TLabelsInfo> GetLabelsForStream(const TString& topic, const TString& cloudId,
+ const TString& dbId, const TString& folderId);
class TMultiCounter {
public:
- TMultiCounter() = default;
+ TMultiCounter() = default;
- TMultiCounter(NMonitoring::TDynamicCounterPtr counters,
- const TVector<TLabelsInfo>& labels,
- const TVector<std::pair<TString, TString>>& subgroups,
- const TVector<TString>& counter_names,
- bool deriv,
- const TString& name = "sensor",
- bool expiring = true);
+ TMultiCounter(NMonitoring::TDynamicCounterPtr counters,
+ const TVector<TLabelsInfo>& labels,
+ const TVector<std::pair<TString, TString>>& subgroups,
+ const TVector<TString>& counter_names,
+ bool deriv,
+ const TString& name = "sensor",
+ bool expiring = true);
- void Inc(ui64 val = 1);
- void Dec(ui64 val = 1);
- void Set(ui64 value);
+ void Inc(ui64 val = 1);
+ void Dec(ui64 val = 1);
+ void Set(ui64 value);
- operator bool();
+ operator bool();
private:
ui64 Value = 0;
@@ -46,18 +46,18 @@ private:
class TPercentileCounter {
public:
- TPercentileCounter() = default;
- TPercentileCounter(TIntrusivePtr<NMonitoring::TDynamicCounters> counters,
- const TVector<TLabelsInfo>& labels,
- const TVector<std::pair<TString, TString>>& subgroups,
- const TString& sensor,
- const TVector<std::pair<ui64, TString>>& intervals,
- const bool deriv,
- bool expiring = true);
+ TPercentileCounter() = default;
+ TPercentileCounter(TIntrusivePtr<NMonitoring::TDynamicCounters> counters,
+ const TVector<TLabelsInfo>& labels,
+ const TVector<std::pair<TString, TString>>& subgroups,
+ const TString& sensor,
+ const TVector<std::pair<ui64, TString>>& intervals,
+ const bool deriv,
+ bool expiring = true);
void IncFor(ui64 key, ui64 value = 1);
void DecFor(ui64 key, ui64 value = 1);
-
+
private:
TVector<TMultiCounter> Counters;
TVector<ui64> Ranges;
diff --git a/ydb/core/persqueue/pq_impl.cpp b/ydb/core/persqueue/pq_impl.cpp
index 0fafca79b8..9937dbe561 100644
--- a/ydb/core/persqueue/pq_impl.cpp
+++ b/ydb/core/persqueue/pq_impl.cpp
@@ -1449,7 +1449,7 @@ void TPersQueue::HandleWriteRequest(const ui64 responseCookie, const TActorId& p
labels.pop_back();
}
it = BytesWrittenFromDC.emplace(clientDC, NKikimr::NPQ::TMultiCounter(GetServiceCounters(counters, "pqproxy|writeSession"),
- labels, {{"ClientDC", clientDC}}, {"BytesWrittenFromDC"}, true)).first;
+ labels, {{"ClientDC", clientDC}}, {"BytesWrittenFromDC"}, true)).first;
}
}
if (it != BytesWrittenFromDC.end())
@@ -1527,7 +1527,7 @@ void TPersQueue::HandleWriteRequest(const ui64 responseCookie, const TActorId& p
msgs.push_back({cmd.GetSourceId(), static_cast<ui64>(cmd.GetSeqNo()), partNo,
totalParts, totalSize, createTimestampMs, receiveTimestampMs,
disableDeduplication, writeTimestampMs, data, uncompressedSize,
- cmd.GetPartitionKey(), cmd.GetExplicitHash(), cmd.GetExternalOperation()
+ cmd.GetPartitionKey(), cmd.GetExplicitHash(), cmd.GetExternalOperation()
});
partNo++;
uncompressedSize = 0;
@@ -1542,23 +1542,23 @@ void TPersQueue::HandleWriteRequest(const ui64 responseCookie, const TActorId& p
static_cast<ui16>(cmd.HasPartNo() ? cmd.GetTotalParts() : 1),
static_cast<ui32>(cmd.HasTotalSize() ? cmd.GetTotalSize() : cmd.GetData().Size()),
createTimestampMs, receiveTimestampMs, disableDeduplication, writeTimestampMs, cmd.GetData(),
- cmd.HasUncompressedSize() ? cmd.GetUncompressedSize() : 0u, cmd.GetPartitionKey(), cmd.GetExplicitHash(),
- cmd.GetExternalOperation()
+ cmd.HasUncompressedSize() ? cmd.GetUncompressedSize() : 0u, cmd.GetPartitionKey(), cmd.GetExplicitHash(),
+ cmd.GetExternalOperation()
});
}
- LOG_DEBUG_S(ctx, NKikimrServices::PERSQUEUE, "got client message topic: " << TopicName <<
- " partition: " << req.GetPartition() <<
- " SourceId: \'" << EscapeC(msgs.back().SourceId) <<
- "\' SeqNo: " << msgs.back().SeqNo << " partNo : " << msgs.back().PartNo <<
- " messageNo: " << req.GetMessageNo() << " size " << msgs.back().Data.size() <<
- " offset: " << (req.HasCmdWriteOffset() ? (req.GetCmdWriteOffset() + i) : -1));
+ LOG_DEBUG_S(ctx, NKikimrServices::PERSQUEUE, "got client message topic: " << TopicName <<
+ " partition: " << req.GetPartition() <<
+ " SourceId: \'" << EscapeC(msgs.back().SourceId) <<
+ "\' SeqNo: " << msgs.back().SeqNo << " partNo : " << msgs.back().PartNo <<
+ " messageNo: " << req.GetMessageNo() << " size " << msgs.back().Data.size() <<
+ " offset: " << (req.HasCmdWriteOffset() ? (req.GetCmdWriteOffset() + i) : -1));
}
InitResponseBuilder(responseCookie, msgs.size(), COUNTER_LATENCY_PQ_WRITE);
- THolder<TEvPQ::TEvWrite> event =
- MakeHolder<TEvPQ::TEvWrite>(responseCookie, req.GetMessageNo(),
- req.HasOwnerCookie() ? req.GetOwnerCookie() : "",
- req.HasCmdWriteOffset() ? req.GetCmdWriteOffset() : TMaybe<ui64>(),
- std::move(msgs), req.GetIsDirectWrite());
+ THolder<TEvPQ::TEvWrite> event =
+ MakeHolder<TEvPQ::TEvWrite>(responseCookie, req.GetMessageNo(),
+ req.HasOwnerCookie() ? req.GetOwnerCookie() : "",
+ req.HasCmdWriteOffset() ? req.GetCmdWriteOffset() : TMaybe<ui64>(),
+ std::move(msgs), req.GetIsDirectWrite());
ctx.Send(partActor, event.Release());
}
@@ -1659,16 +1659,16 @@ void TPersQueue::HandleReadRequest(const ui64 responseCookie, const TActorId& pa
ui32 bytes = Min<ui32>(MAX_BYTES, cmd.HasBytes() ? cmd.GetBytes() : MAX_BYTES);
auto clientDC = cmd.HasClientDC() ? to_lower(cmd.GetClientDC()) : "unknown";
clientDC.to_title();
- THolder<TEvPQ::TEvRead> event =
- MakeHolder<TEvPQ::TEvRead>(responseCookie, cmd.GetOffset(),
- cmd.HasPartNo() ? cmd.GetPartNo() : 0,
- count,
- cmd.HasSessionId() ? cmd.GetSessionId() : "",
- cmd.GetClientId(),
- cmd.HasTimeoutMs() ? cmd.GetTimeoutMs() : 0, bytes,
- cmd.HasMaxTimeLagMs() ? cmd.GetMaxTimeLagMs() : 0,
- cmd.HasReadTimestampMs() ? cmd.GetReadTimestampMs() : 0, clientDC,
- cmd.GetExternalOperation());
+ THolder<TEvPQ::TEvRead> event =
+ MakeHolder<TEvPQ::TEvRead>(responseCookie, cmd.GetOffset(),
+ cmd.HasPartNo() ? cmd.GetPartNo() : 0,
+ count,
+ cmd.HasSessionId() ? cmd.GetSessionId() : "",
+ cmd.GetClientId(),
+ cmd.HasTimeoutMs() ? cmd.GetTimeoutMs() : 0, bytes,
+ cmd.HasMaxTimeLagMs() ? cmd.GetMaxTimeLagMs() : 0,
+ cmd.HasReadTimestampMs() ? cmd.GetReadTimestampMs() : 0, clientDC,
+ cmd.GetExternalOperation());
ctx.Send(partActor, event.Release());
}
}
diff --git a/ydb/core/persqueue/pq_ut.cpp b/ydb/core/persqueue/pq_ut.cpp
index 09d33224fe..13a6122910 100644
--- a/ydb/core/persqueue/pq_ut.cpp
+++ b/ydb/core/persqueue/pq_ut.cpp
@@ -19,7 +19,7 @@
#include <util/system/sanitizers.h>
#include <util/system/valgrind.h>
-
+
namespace NKikimr {
Y_UNIT_TEST_SUITE(TPQTest) {
@@ -1002,8 +1002,8 @@ Y_UNIT_TEST(TestAlreadyWrittenWithoutDeduplication) {
CmdRead(0, 0, Max<i32>(), Max<i32>(), 2, false, tc, {0, 1});
});
}
-
-
+
+
Y_UNIT_TEST(TestWritePQCompact) {
TTestContext tc;
RunTestWithReboots(tc.TabletIds, [&]() {
@@ -1500,7 +1500,7 @@ Y_UNIT_TEST(TestPQRead) {
});
}
-
+
Y_UNIT_TEST(TestPQSmallRead) {
TTestContext tc;
RunTestWithReboots(tc.TabletIds, [&]() {
@@ -1954,7 +1954,7 @@ Y_UNIT_TEST(TestMaxTimeLagRewind) {
CmdRead(0, 0, 1, Max<i32>(), 1, false, tc, {21}, 0, ts.MilliSeconds() - 3 * 60 * 1000);
CmdRead(0, 22, 1, Max<i32>(), 1, false, tc, {22}, 0, ts.MilliSeconds() - 3 * 60 * 1000);
- CmdRead(0, 4, 1, Max<i32>(), 1, false, tc, {34}, 0, ts.MilliSeconds() - 1000);
+ CmdRead(0, 4, 1, Max<i32>(), 1, false, tc, {34}, 0, ts.MilliSeconds() - 1000);
PQTabletPrepare(20000000, 100 * 1024 * 1024, 0, {{"aaa", true}}, tc, 2, 6*1024*1024, true, ts.MilliSeconds() - 1000);
CmdRead(0, 0, 1, Max<i32>(), 1, false, tc, {34});
diff --git a/ydb/core/persqueue/pq_ut.h b/ydb/core/persqueue/pq_ut.h
index 8eecbe15a8..3db93bc104 100644
--- a/ydb/core/persqueue/pq_ut.h
+++ b/ydb/core/persqueue/pq_ut.h
@@ -605,11 +605,11 @@ void WriteData(const ui32 partition, const TString& sourceId, const TVector<std:
tc.Runtime->SendToPipe(tc.TabletId, tc.Edge, request.Release(), 0, GetPipeConfigWithRetries());
}
-void CmdWrite(const ui32 partition, const TString& sourceId, const TVector<std::pair<ui64, TString>> data,
- TTestContext& tc, bool error = false, const THashSet<ui32>& alreadyWrittenSeqNo = {},
- bool isFirst = false, const TString& ownerCookie = "", i32 msn = -1, i64 offset = -1,
- bool treatWrongCookieAsError = false, bool treatBadOffsetAsError = true,
- bool disableDeduplication = false) {
+void CmdWrite(const ui32 partition, const TString& sourceId, const TVector<std::pair<ui64, TString>> data,
+ TTestContext& tc, bool error = false, const THashSet<ui32>& alreadyWrittenSeqNo = {},
+ bool isFirst = false, const TString& ownerCookie = "", i32 msn = -1, i64 offset = -1,
+ bool treatWrongCookieAsError = false, bool treatBadOffsetAsError = true,
+ bool disableDeduplication = false) {
TAutoPtr<IEventHandle> handle;
TEvPersQueue::TEvResponse *result;
@@ -1046,9 +1046,9 @@ void CmdRead(const ui32 partition, const ui64 offset, const ui32 count, const ui
auto r = res.GetResult(i);
if (offsets.empty()) {
- if (readTimestampMs == 0) {
- UNIT_ASSERT_EQUAL((ui64)r.GetOffset(), off);
- }
+ if (readTimestampMs == 0) {
+ UNIT_ASSERT_EQUAL((ui64)r.GetOffset(), off);
+ }
UNIT_ASSERT(r.GetSourceId().size() == 9 && r.GetSourceId().StartsWith("sourceid"));
UNIT_ASSERT_EQUAL(ui32(r.GetData()[0]), off);
UNIT_ASSERT_EQUAL(ui32((unsigned char)r.GetData().back()), r.GetSeqNo() % 256);
diff --git a/ydb/core/persqueue/subscriber.h b/ydb/core/persqueue/subscriber.h
index 3162f9c051..7bbfe41e1f 100644
--- a/ydb/core/persqueue/subscriber.h
+++ b/ydb/core/persqueue/subscriber.h
@@ -26,7 +26,7 @@ struct TReadInfo {
ui32 Size;
ui64 Destination;
TInstant Timestamp;
- ui64 ReadTimestampMs;
+ ui64 ReadTimestampMs;
TDuration WaitQuotaTime;
bool IsSubscription;
@@ -70,7 +70,7 @@ struct TReadInfo {
const ui64 dst,
const ui64 sizeLag
);
-
+
TReadAnswer FormAnswer(
const TActorContext& ctx,
const ui64 endOffset,
diff --git a/ydb/core/persqueue/user_info.cpp b/ydb/core/persqueue/user_info.cpp
index d08c581efd..26811765e2 100644
--- a/ydb/core/persqueue/user_info.cpp
+++ b/ydb/core/persqueue/user_info.cpp
@@ -35,18 +35,18 @@ TUsersInfoStorage::TUsersInfoStorage(
ui32 partition,
const TTabletCountersBase& counters,
const NKikimrPQ::TPQTabletConfig& config,
- const TString& cloudId,
- const TString& dbId,
- const TString& folderId
+ const TString& cloudId,
+ const TString& dbId,
+ const TString& folderId
)
: DCId(std::move(dcId))
, TabletId(tabletId)
, TopicName(topicName)
, Partition(partition)
, Config(config)
- , CloudId(cloudId)
- , DbId(dbId)
- , FolderId(folderId)
+ , CloudId(cloudId)
+ , DbId(dbId)
+ , FolderId(folderId)
{
Counters.Populate(counters);
}
@@ -154,7 +154,7 @@ TUserInfo& TUsersInfoStorage::Create(
ui32 gen, ui32 step, i64 offset, ui64 readOffsetRewindSum, TInstant readFromTimestamp
) {
- ui64 burst = 1'000'000'000, speed = 1'000'000'000;
+ ui64 burst = 1'000'000'000, speed = 1'000'000'000;
if (AppData(ctx)->PQConfig.GetQuotingConfig().GetPartitionReadQuotaIsTwiceWriteQuota()) {
burst = Config.GetPartitionConfig().GetBurstSize() * 2;
speed = Config.GetPartitionConfig().GetWriteSpeedInBytesPerSecond() * 2;
diff --git a/ydb/core/persqueue/user_info.h b/ydb/core/persqueue/user_info.h
index 65384078ea..258e947c46 100644
--- a/ydb/core/persqueue/user_info.h
+++ b/ydb/core/persqueue/user_info.h
@@ -24,9 +24,9 @@ namespace NDeprecatedUserData {
void Parse(const TString& data, ui64& offset, ui32& gen, ui32& step, TString& session);
} // NDeprecatedUserInfo
-static const ui32 MAX_USER_TS_CACHE_SIZE = 10'000;
-static const ui64 MIN_TIMESTAMP_MS = 1'000'000'000'000ll; // around 2002 year
-static const TString CLIENTID_TO_READ_INTERNALLY = "$without_consumer";
+static const ui32 MAX_USER_TS_CACHE_SIZE = 10'000;
+static const ui64 MIN_TIMESTAMP_MS = 1'000'000'000'000ll; // around 2002 year
+static const TString CLIENTID_TO_READ_INTERNALLY = "$without_consumer";
typedef TProtobufTabletLabeledCounters<EClientLabeledCounters_descriptor> TUserLabeledCounters;
@@ -82,7 +82,7 @@ public:
, SpeedPerSecond(speedPerSecond)
, LastUpdateTime(timestamp)
, MaxBurst(maxBurst)
- , AvgMin(60'000) //avg avail in bytes per sec for last minute
+ , AvgMin(60'000) //avg avail in bytes per sec for last minute
, AvgSec(1000) //avg avail in bytes per sec
, QuotedTime(0)
{}
@@ -199,7 +199,7 @@ struct TUserInfo {
NSlidingWindow::TSlidingWindow<NSlidingWindow::TMaxOperation<ui64>> WriteLagMs;
std::shared_ptr<TPercentileCounter> ReadTimeLag;
- bool DoExternalRead = false;
+ bool DoExternalRead = false;
bool WriteInProgress = false;
@@ -208,7 +208,7 @@ struct TUserInfo {
TUserInfo(THolder<TReadSpeedLimiterHolder> readSpeedLimiter, const TString& user,
const ui64 readRuleGeneration, bool important, const TString& topic,
const ui32 partition, bool doExternalRead,
- ui64 burst = 1'000'000'000, ui64 speed = 1'000'000'000)
+ ui64 burst = 1'000'000'000, ui64 speed = 1'000'000'000)
: ReadSpeedLimiter(std::move(readSpeedLimiter))
, Important(important)
, LabeledCounters(user + "/" + (important ? "1" : "0") + "/" + topic, partition)
@@ -223,7 +223,7 @@ struct TUserInfo {
, Subscriptions(0)
, EndOffset(0)
, WriteLagMs(TDuration::Minutes(1), 100)
- , DoExternalRead(doExternalRead)
+ , DoExternalRead(doExternalRead)
{
}
@@ -249,13 +249,13 @@ struct TUserInfo {
}
}
- void ReadDone(const TActorContext& ctx, const TInstant& now, ui64 readSize, ui32 readCount,
- const TString& clientDC) {
+ void ReadDone(const TActorContext& ctx, const TInstant& now, ui64 readSize, ui32 readCount,
+ const TString& clientDC) {
if (BytesRead && !clientDC.empty()) {
- if (BytesRead)
- BytesRead.Inc(readSize);
- if (MsgsRead)
- MsgsRead.Inc(readCount);
+ if (BytesRead)
+ BytesRead.Inc(readSize);
+ if (MsgsRead)
+ MsgsRead.Inc(readCount);
auto it = BytesReadFromDC.find(clientDC);
if (it == BytesReadFromDC.end()) {
auto pos = Topic.find("--");
@@ -264,12 +264,12 @@ struct TUserInfo {
if (!labels.empty()) {
labels.pop_back();
}
- it = BytesReadFromDC.emplace(clientDC,
- TMultiCounter(GetServiceCounters(AppData(ctx)->Counters, "pqproxy|readSession"),
- labels, {{"ClientDC", clientDC},
- {"Client", User},
- {"ConsumerPath", NPersQueue::ConvertOldConsumerName(User, ctx)}},
- {"BytesReadFromDC"}, true)).first;
+ it = BytesReadFromDC.emplace(clientDC,
+ TMultiCounter(GetServiceCounters(AppData(ctx)->Counters, "pqproxy|readSession"),
+ labels, {{"ClientDC", clientDC},
+ {"Client", User},
+ {"ConsumerPath", NPersQueue::ConvertOldConsumerName(User, ctx)}},
+ {"BytesReadFromDC"}, true)).first;
}
}
if (it != BytesReadFromDC.end())
@@ -286,11 +286,11 @@ struct TUserInfo {
}
TUserInfo(
- const TActorContext& ctx, THolder<TReadSpeedLimiterHolder> readSpeedLimiter, const TString& user,
+ const TActorContext& ctx, THolder<TReadSpeedLimiterHolder> readSpeedLimiter, const TString& user,
const ui64 readRuleGeneration, const bool important, const TString& topic, const ui32 partition, const TString &session,
- ui32 gen, ui32 step, i64 offset, const ui64 readOffsetRewindSum, const TString& dcId,
- TInstant readFromTimestamp, const TString& cloudId, const TString& dbId, const TString& folderId,
- ui64 burst = 1'000'000'000, ui64 speed = 1'000'000'000
+ ui32 gen, ui32 step, i64 offset, const ui64 readOffsetRewindSum, const TString& dcId,
+ TInstant readFromTimestamp, const TString& cloudId, const TString& dbId, const TString& folderId,
+ ui64 burst = 1'000'000'000, ui64 speed = 1'000'000'000
)
: ReadSpeedLimiter(std::move(readSpeedLimiter))
, Session(session)
@@ -318,97 +318,97 @@ struct TUserInfo {
, ActiveReads(0)
, Subscriptions(0)
, EndOffset(0)
- , AvgReadBytes{{TDuration::Seconds(1), 1000}, {TDuration::Minutes(1), 1000},
- {TDuration::Hours(1), 2000}, {TDuration::Days(1), 2000}}
+ , AvgReadBytes{{TDuration::Seconds(1), 1000}, {TDuration::Minutes(1), 1000},
+ {TDuration::Hours(1), 2000}, {TDuration::Days(1), 2000}}
, WriteLagMs(TDuration::Minutes(1), 100)
{
- if (AppData(ctx)->Counters) {
- if (AppData()->PQConfig.GetTopicsAreFirstClassCitizen()) {
- SetupStreamCounters(ctx, dcId, ToString<ui32>(partition), topic, cloudId, dbId, folderId);
- } else {
- if (topic.find("--") == TString::npos)
- return;
- SetupTopicCounters(ctx, dcId, ToString<ui32>(partition), topic);
- }
+ if (AppData(ctx)->Counters) {
+ if (AppData()->PQConfig.GetTopicsAreFirstClassCitizen()) {
+ SetupStreamCounters(ctx, dcId, ToString<ui32>(partition), topic, cloudId, dbId, folderId);
+ } else {
+ if (topic.find("--") == TString::npos)
+ return;
+ SetupTopicCounters(ctx, dcId, ToString<ui32>(partition), topic);
+ }
+ }
+ }
+
+ void SetupStreamCounters(const TActorContext& ctx, const TString& dcId, const TString& partition,
+ const TString& topic, const TString& cloudId,
+ const TString& dbId, const TString& folderId) {
+ auto subgroup = [&](const TString& subsystem) {
+ return NKikimr::NPQ::GetCountersForStream(AppData(ctx)->Counters, subsystem);
+ };
+ auto additionalLabels = [&](const TVector<std::pair<TString, TString>>& subgroups = {}) {
+ TVector<std::pair<TString, TString>> result;
+ if (User != CLIENTID_TO_READ_INTERNALLY)
+ result.push_back({"consumer", User});
+
+ for (const auto& sb : subgroups) {
+ result.push_back(sb);
+ }
+ return result;
+ };
+ const TVector<NPQ::TLabelsInfo> aggregates =
+ NKikimr::NPQ::GetLabelsForStream(topic, cloudId, dbId, folderId);
+
+ if (DoExternalRead) {
+ BytesRead = TMultiCounter(subgroup("readSession"), aggregates, additionalLabels(),
+ {"stream.internal_read.bytes_per_second",
+ "stream.outcoming_bytes_per_second"}, true);
+ MsgsRead = TMultiCounter(subgroup("readSession"), aggregates, additionalLabels(),
+ {"stream.internal_read.records_per_second",
+ "stream.outcoming_records_per_second"}, true);
}
+
+ Counter.SetCounter(subgroup("readingTime"),
+ additionalLabels({{"host", dcId}, {"shard", partition}, {"stream", topic}}),
+ {"name", "stream.await_operating_milliseconds", true});
+
+ ReadTimeLag.reset(new TPercentileCounter(subgroup("readTimeLag"), aggregates,
+ additionalLabels({{"name", "stream.internal_read.time_lags_milliseconds"}}), "bin",
+ TVector<std::pair<ui64, TString>>{{100, "100"}, {200, "200"}, {500, "500"},
+ {1000, "1000"}, {2000, "2000"},
+ {5000, "5000"}, {10'000, "10000"},
+ {30'000, "30000"}, {60'000, "60000"},
+ {180'000,"180000"}, {9'999'999, "999999"}},
+ true));
+ }
+
+ void SetupTopicCounters(const TActorContext& ctx, const TString& dcId, const TString& partition,
+ const TString& topic) {
+ auto subgroup = [&](const TString& subsystem) {
+ return GetServiceCounters(AppData(ctx)->Counters, subsystem);
+ };
+ const TVector<NPQ::TLabelsInfo> aggr = NKikimr::NPQ::GetLabels(topic);
+ TVector<std::pair<TString, TString>> additional_labels = {{"Client", User},
+ {"ConsumerPath", NPersQueue::ConvertOldConsumerName(User, ctx)}
+ };
+
+ Counter.SetCounter(subgroup("readingTime"),
+ {{"Client", User},
+ {"ConsumerPath", NPersQueue::ConvertOldConsumerName(User, ctx)},
+ {"host", dcId},
+ {"Partition", partition}},
+ {"sensor", "ReadTime", true});
+
+ BytesRead = TMultiCounter(subgroup("pqproxy|readSession"), aggr, additional_labels,
+ {"BytesRead"}, true);
+ MsgsRead = TMultiCounter(subgroup("pqproxy|readSession"), aggr, additional_labels,
+ {"MessagesRead"}, true);
+
+ additional_labels.push_back({"sensor", "TimeLags"});
+ ReadTimeLag.reset(new TPercentileCounter(subgroup("pqproxy|readTimeLag"), aggr,
+ additional_labels, "Interval",
+ TVector<std::pair<ui64, TString>>{{100, "100ms"}, {200, "200ms"}, {500, "500ms"},
+ {1000, "1000ms"}, {2000, "2000ms"},
+ {5000, "5000ms"}, {10'000, "10000ms"},
+ {30'000, "30000ms"}, {60'000, "60000ms"},
+ {180'000,"180000ms"}, {9'999'999, "999999ms"}},
+ true));
+
}
- void SetupStreamCounters(const TActorContext& ctx, const TString& dcId, const TString& partition,
- const TString& topic, const TString& cloudId,
- const TString& dbId, const TString& folderId) {
- auto subgroup = [&](const TString& subsystem) {
- return NKikimr::NPQ::GetCountersForStream(AppData(ctx)->Counters, subsystem);
- };
- auto additionalLabels = [&](const TVector<std::pair<TString, TString>>& subgroups = {}) {
- TVector<std::pair<TString, TString>> result;
- if (User != CLIENTID_TO_READ_INTERNALLY)
- result.push_back({"consumer", User});
-
- for (const auto& sb : subgroups) {
- result.push_back(sb);
- }
- return result;
- };
- const TVector<NPQ::TLabelsInfo> aggregates =
- NKikimr::NPQ::GetLabelsForStream(topic, cloudId, dbId, folderId);
-
- if (DoExternalRead) {
- BytesRead = TMultiCounter(subgroup("readSession"), aggregates, additionalLabels(),
- {"stream.internal_read.bytes_per_second",
- "stream.outcoming_bytes_per_second"}, true);
- MsgsRead = TMultiCounter(subgroup("readSession"), aggregates, additionalLabels(),
- {"stream.internal_read.records_per_second",
- "stream.outcoming_records_per_second"}, true);
- }
-
- Counter.SetCounter(subgroup("readingTime"),
- additionalLabels({{"host", dcId}, {"shard", partition}, {"stream", topic}}),
- {"name", "stream.await_operating_milliseconds", true});
-
- ReadTimeLag.reset(new TPercentileCounter(subgroup("readTimeLag"), aggregates,
- additionalLabels({{"name", "stream.internal_read.time_lags_milliseconds"}}), "bin",
- TVector<std::pair<ui64, TString>>{{100, "100"}, {200, "200"}, {500, "500"},
- {1000, "1000"}, {2000, "2000"},
- {5000, "5000"}, {10'000, "10000"},
- {30'000, "30000"}, {60'000, "60000"},
- {180'000,"180000"}, {9'999'999, "999999"}},
- true));
- }
-
- void SetupTopicCounters(const TActorContext& ctx, const TString& dcId, const TString& partition,
- const TString& topic) {
- auto subgroup = [&](const TString& subsystem) {
- return GetServiceCounters(AppData(ctx)->Counters, subsystem);
- };
- const TVector<NPQ::TLabelsInfo> aggr = NKikimr::NPQ::GetLabels(topic);
- TVector<std::pair<TString, TString>> additional_labels = {{"Client", User},
- {"ConsumerPath", NPersQueue::ConvertOldConsumerName(User, ctx)}
- };
-
- Counter.SetCounter(subgroup("readingTime"),
- {{"Client", User},
- {"ConsumerPath", NPersQueue::ConvertOldConsumerName(User, ctx)},
- {"host", dcId},
- {"Partition", partition}},
- {"sensor", "ReadTime", true});
-
- BytesRead = TMultiCounter(subgroup("pqproxy|readSession"), aggr, additional_labels,
- {"BytesRead"}, true);
- MsgsRead = TMultiCounter(subgroup("pqproxy|readSession"), aggr, additional_labels,
- {"MessagesRead"}, true);
-
- additional_labels.push_back({"sensor", "TimeLags"});
- ReadTimeLag.reset(new TPercentileCounter(subgroup("pqproxy|readTimeLag"), aggr,
- additional_labels, "Interval",
- TVector<std::pair<ui64, TString>>{{100, "100ms"}, {200, "200ms"}, {500, "500ms"},
- {1000, "1000ms"}, {2000, "2000ms"},
- {5000, "5000ms"}, {10'000, "10000ms"},
- {30'000, "30000ms"}, {60'000, "60000ms"},
- {180'000,"180000ms"}, {9'999'999, "999999ms"}},
- true));
-
- }
-
void SetQuota(const ui64 maxBurst, const ui64 speed) {
ReadQuota.UpdateConfig(maxBurst, speed);
}
@@ -507,9 +507,9 @@ struct TUserInfo {
class TUsersInfoStorage {
public:
- TUsersInfoStorage(TString dcId, ui64 tabletId, const TString& topicName, ui32 partition,
+ TUsersInfoStorage(TString dcId, ui64 tabletId, const TString& topicName, ui32 partition,
const TTabletCountersBase& counters, const NKikimrPQ::TPQTabletConfig& config,
- const TString& CloudId, const TString& DbId, const TString& FolderId);
+ const TString& CloudId, const TString& DbId, const TString& FolderId);
void Init(TActorId tabletActor, TActorId partitionActor);
@@ -549,10 +549,10 @@ private:
TMaybe<TActorId> TabletActor;
TMaybe<TActorId> PartitionActor;
NKikimrPQ::TPQTabletConfig Config;
-
- TString CloudId;
- TString DbId;
- TString FolderId;
+
+ TString CloudId;
+ TString DbId;
+ TString FolderId;
};
} //NPQ
diff --git a/ydb/core/persqueue/working_time_counter.h b/ydb/core/persqueue/working_time_counter.h
index 964bb4ce8a..4f4d81d263 100644
--- a/ydb/core/persqueue/working_time_counter.h
+++ b/ydb/core/persqueue/working_time_counter.h
@@ -7,7 +7,7 @@
namespace NKikimr {
namespace NPQ {
-
+
class TWorkingTimeCounter {
private:
bool IsInWorkingState;
@@ -36,16 +36,16 @@ public:
}
}
- void SetCounter(NMonitoring::TDynamicCounterPtr counter,
- const TVector<std::pair<TString, TString>>& subgroups,
- const std::tuple<TString, TString, bool>& expiring) {
- for (const auto& subgroup : subgroups) {
- counter = counter->GetSubgroup(subgroup.first, subgroup.second);
- }
-
- WorkingTimeMicroSec = counter->GetExpiringNamedCounter(std::get<0>(expiring),
- std::get<1>(expiring),
- std::get<2>(expiring));
+ void SetCounter(NMonitoring::TDynamicCounterPtr counter,
+ const TVector<std::pair<TString, TString>>& subgroups,
+ const std::tuple<TString, TString, bool>& expiring) {
+ for (const auto& subgroup : subgroups) {
+ counter = counter->GetSubgroup(subgroup.first, subgroup.second);
+ }
+
+ WorkingTimeMicroSec = counter->GetExpiringNamedCounter(std::get<0>(expiring),
+ std::get<1>(expiring),
+ std::get<2>(expiring));
}
};
diff --git a/ydb/core/protos/msgbus_pq.proto b/ydb/core/protos/msgbus_pq.proto
index 879ba6d4a3..849b292383 100644
--- a/ydb/core/protos/msgbus_pq.proto
+++ b/ydb/core/protos/msgbus_pq.proto
@@ -24,8 +24,8 @@ message TPersQueuePartitionRequest {
optional string PartitionKey = 12;
optional string ExplicitHash = 13;
-
- optional bool ExternalOperation = 14 [default = false];
+
+ optional bool ExternalOperation = 14 [default = false];
}
message TCmdCreateSession {
@@ -68,8 +68,8 @@ message TPersQueuePartitionRequest {
optional string PartitionKey = 13;
optional bytes ExplicitHash = 14;
-
- optional bool ExternalOperation = 15 [ default = false ];
+
+ optional bool ExternalOperation = 15 [ default = false ];
}
message TCmdUpdateWriteTimestamp {
diff --git a/ydb/core/protos/pqconfig.proto b/ydb/core/protos/pqconfig.proto
index d21ee01e20..7c85927449 100644
--- a/ydb/core/protos/pqconfig.proto
+++ b/ydb/core/protos/pqconfig.proto
@@ -682,19 +682,19 @@ message TPQClusterDiscoveryConfig {
optional NKikimrNetClassifier.TNetData CloudNetData = 3;
optional uint64 RequestInflightLimit = 4;
}
-
-message TYdsNextToken {
- required uint64 CreationTimestamp = 1;
- required uint32 MaxResults = 2;
- required uint32 AlreadyRead = 3;
- required string StreamArn = 4;
-}
-
-message TYdsShardIterator {
- required string StreamName = 1;
- required string StreamArn = 2;
- required uint32 ShardId = 3;
- required uint64 ReadTimestampMs = 4;
- required uint32 SequenceNumber = 5;
- required uint64 CreationTimestampMs = 6;
-}
+
+message TYdsNextToken {
+ required uint64 CreationTimestamp = 1;
+ required uint32 MaxResults = 2;
+ required uint32 AlreadyRead = 3;
+ required string StreamArn = 4;
+}
+
+message TYdsShardIterator {
+ required string StreamName = 1;
+ required string StreamArn = 2;
+ required uint32 ShardId = 3;
+ required uint64 ReadTimestampMs = 4;
+ required uint32 SequenceNumber = 5;
+ required uint64 CreationTimestampMs = 6;
+}
diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_alter_pq.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_alter_pq.cpp
index 1842f6317e..f7a47459b9 100644
--- a/ydb/core/tx/schemeshard/schemeshard__operation_alter_pq.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard__operation_alter_pq.cpp
@@ -75,7 +75,7 @@ public:
}
TPersQueueGroupInfo::TPtr ParseParams(
- TOperationContext& context,
+ TOperationContext& context,
NKikimrPQ::TPQTabletConfig* tabletConfig,
const NKikimrSchemeOp::TPersQueueGroupDescription& alter,
TString& errStr)
@@ -139,22 +139,22 @@ public:
return nullptr;
}
- const TPathElement::TPtr dbRootEl = context.SS->PathsById.at(context.SS->RootPathId());
- if (dbRootEl->UserAttrs->Attrs.contains("cloud_id")) {
- auto cloudId = dbRootEl->UserAttrs->Attrs.at("cloud_id");
- tabletConfig->SetYcCloudId(cloudId);
- }
- if (dbRootEl->UserAttrs->Attrs.contains("folder_id")) {
- auto folderId = dbRootEl->UserAttrs->Attrs.at("folder_id");
- tabletConfig->SetYcFolderId(folderId);
- }
- if (dbRootEl->UserAttrs->Attrs.contains("database_id")) {
- auto databaseId = dbRootEl->UserAttrs->Attrs.at("database_id");
- tabletConfig->SetYdbDatabaseId(databaseId);
- }
- const TString databasePath = TPath::Init(context.SS->RootPathId(), context.SS).PathString();
- tabletConfig->SetYdbDatabasePath(databasePath);
-
+ const TPathElement::TPtr dbRootEl = context.SS->PathsById.at(context.SS->RootPathId());
+ if (dbRootEl->UserAttrs->Attrs.contains("cloud_id")) {
+ auto cloudId = dbRootEl->UserAttrs->Attrs.at("cloud_id");
+ tabletConfig->SetYcCloudId(cloudId);
+ }
+ if (dbRootEl->UserAttrs->Attrs.contains("folder_id")) {
+ auto folderId = dbRootEl->UserAttrs->Attrs.at("folder_id");
+ tabletConfig->SetYcFolderId(folderId);
+ }
+ if (dbRootEl->UserAttrs->Attrs.contains("database_id")) {
+ auto databaseId = dbRootEl->UserAttrs->Attrs.at("database_id");
+ tabletConfig->SetYdbDatabaseId(databaseId);
+ }
+ const TString databasePath = TPath::Init(context.SS->RootPathId(), context.SS).PathString();
+ tabletConfig->SetYdbDatabasePath(databasePath);
+
alterConfig.MutablePartitionKeySchema()->Swap(tabletConfig->MutablePartitionKeySchema());
Y_PROTOBUF_SUPPRESS_NODISCARD alterConfig.SerializeToString(&params->TabletConfig);
alterConfig.Swap(tabletConfig);
@@ -453,7 +453,7 @@ public:
newTabletConfig = tabletConfig;
- TPersQueueGroupInfo::TPtr alterData = ParseParams(context, &newTabletConfig, alter, errStr);
+ TPersQueueGroupInfo::TPtr alterData = ParseParams(context, &newTabletConfig, alter, errStr);
if (!alterData) {
result->SetError(NKikimrScheme::StatusInvalidParameter, errStr);
return result;
diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_pq.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_pq.cpp
index 7e58e43847..992fd3175c 100644
--- a/ydb/core/tx/schemeshard/schemeshard__operation_create_pq.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_pq.cpp
@@ -12,8 +12,8 @@ namespace {
using namespace NKikimr;
using namespace NSchemeShard;
-TPersQueueGroupInfo::TPtr CreatePersQueueGroup(TOperationContext& context,
- const NKikimrSchemeOp::TPersQueueGroupDescription& op,
+TPersQueueGroupInfo::TPtr CreatePersQueueGroup(TOperationContext& context,
+ const NKikimrSchemeOp::TPersQueueGroupDescription& op,
TEvSchemeShard::EStatus& status, TString& errStr)
{
TPersQueueGroupInfo::TPtr pqGroupInfo = new TPersQueueGroupInfo;
@@ -147,22 +147,22 @@ TPersQueueGroupInfo::TPtr CreatePersQueueGroup(TOperationContext& context,
return nullptr;
}
- const TPathElement::TPtr dbRootEl = context.SS->PathsById.at(context.SS->RootPathId());
- if (dbRootEl->UserAttrs->Attrs.contains("cloud_id")) {
- auto cloudId = dbRootEl->UserAttrs->Attrs.at("cloud_id");
- tabletConfig.SetYcCloudId(cloudId);
- }
- if (dbRootEl->UserAttrs->Attrs.contains("folder_id")) {
- auto folderId = dbRootEl->UserAttrs->Attrs.at("folder_id");
- tabletConfig.SetYcFolderId(folderId);
- }
- if (dbRootEl->UserAttrs->Attrs.contains("database_id")) {
- auto databaseId = dbRootEl->UserAttrs->Attrs.at("database_id");
- tabletConfig.SetYdbDatabaseId(databaseId);
- }
- const TString databasePath = TPath::Init(context.SS->RootPathId(), context.SS).PathString();
- tabletConfig.SetYdbDatabasePath(databasePath);
-
+ const TPathElement::TPtr dbRootEl = context.SS->PathsById.at(context.SS->RootPathId());
+ if (dbRootEl->UserAttrs->Attrs.contains("cloud_id")) {
+ auto cloudId = dbRootEl->UserAttrs->Attrs.at("cloud_id");
+ tabletConfig.SetYcCloudId(cloudId);
+ }
+ if (dbRootEl->UserAttrs->Attrs.contains("folder_id")) {
+ auto folderId = dbRootEl->UserAttrs->Attrs.at("folder_id");
+ tabletConfig.SetYcFolderId(folderId);
+ }
+ if (dbRootEl->UserAttrs->Attrs.contains("database_id")) {
+ auto databaseId = dbRootEl->UserAttrs->Attrs.at("database_id");
+ tabletConfig.SetYdbDatabaseId(databaseId);
+ }
+ const TString databasePath = TPath::Init(context.SS->RootPathId(), context.SS).PathString();
+ tabletConfig.SetYdbDatabasePath(databasePath);
+
Y_PROTOBUF_SUPPRESS_NODISCARD tabletConfig.SerializeToString(&pqGroupInfo->TabletConfig);
if (op.HasBootstrapConfig()) {
@@ -377,7 +377,7 @@ public:
}
TPersQueueGroupInfo::TPtr pqGroup = CreatePersQueueGroup(
- context, createDEscription, status, errStr);
+ context, createDEscription, status, errStr);
if (!pqGroup.Get()) {
result->SetError(status, errStr);
@@ -390,7 +390,7 @@ public:
auto tabletConfig = pqGroup->TabletConfig;
NKikimrPQ::TPQTabletConfig config;
Y_VERIFY(!tabletConfig.empty());
-
+
bool parseOk = ParseFromStringNoSizeLimit(config, tabletConfig);
Y_VERIFY(parseOk);
diff --git a/ydb/public/api/protos/draft/datastreams.proto b/ydb/public/api/protos/draft/datastreams.proto
index 5543e964cf..5c1354288a 100644
--- a/ydb/public/api/protos/draft/datastreams.proto
+++ b/ydb/public/api/protos/draft/datastreams.proto
@@ -12,8 +12,8 @@ option java_package = "com.yandex.ydb.datastreams.v1";
enum EFieldTransformationType {
TRANSFORM_NONE = 0;
TRANSFORM_BASE64 = 1;
- TRANSFORM_DOUBLE_S_TO_INT_MS = 2;
- TRANSFORM_EMPTY_TO_NOTHING = 3;
+ TRANSFORM_DOUBLE_S_TO_INT_MS = 2;
+ TRANSFORM_EMPTY_TO_NOTHING = 3;
}
extend google.protobuf.FieldOptions {
@@ -45,7 +45,7 @@ message ChildShard {
// Represents details of consumer
message Consumer {
string consumer_arn = 1;
- int64 consumer_creation_timestamp = 2 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS];
+ int64 consumer_creation_timestamp = 2 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS];
string consumer_name = 3;
ConsumerDescription.ConsumerStatus consumer_status = 4;
}
@@ -58,9 +58,9 @@ message HashKeyRange {
message Record {
// Timestamp that the record was inserted into the stream
- int64 timestamp = 1 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS];
+ int64 timestamp = 1 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS];
// Data blob
- bytes data = 2 [(FieldTransformer) = TRANSFORM_BASE64];
+ bytes data = 2 [(FieldTransformer) = TRANSFORM_BASE64];
// Encryption type used on record
EncryptionType encryption = 3;
// Identifies shard in the stream the record is assigned to
@@ -98,7 +98,7 @@ message StreamDescription {
repeated Shard shards = 6;
string stream_arn = 7;
// Timestamp that the stream was created
- int64 stream_creation_timestamp = 8 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS];
+ int64 stream_creation_timestamp = 8 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS];
// Current status of the stream
StreamStatus stream_status = 9;
// Name of the stream
@@ -116,11 +116,11 @@ message SequenceNumberRange {
// Represents shard details
message Shard {
// Id of the shard adjacent to the shard's parent
- string adjacent_parent_shard_id = 1 [(FieldTransformer) = TRANSFORM_EMPTY_TO_NOTHING];
+ string adjacent_parent_shard_id = 1 [(FieldTransformer) = TRANSFORM_EMPTY_TO_NOTHING];
// The range of possible hash key values for the shard
HashKeyRange hash_key_range = 2;
// Id of the shard's parent
- string parent_shard_id = 3 [(FieldTransformer) = TRANSFORM_EMPTY_TO_NOTHING];
+ string parent_shard_id = 3 [(FieldTransformer) = TRANSFORM_EMPTY_TO_NOTHING];
// The range of possible sequence numbers for the shard
SequenceNumberRange sequence_number_range = 4;
// Unique id of the shard within stream
@@ -138,7 +138,7 @@ message ConsumerDescription {
string consumer_arn = 1;
// Timestamp that the consumer was created
- int64 consumer_creation_timestamp = 2 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS];
+ int64 consumer_creation_timestamp = 2 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS];
// Name of the consumer
string consumer_name = 3;
ConsumerStatus consumer_status = 4;
@@ -181,14 +181,14 @@ message ShardFilter {
// Exclusive id. Can only be used if AFTER_SHARD_ID is specified
string shard_id = 1;
// Can only be used if AT_TIMESTAMP or FROM_TIMESTAMP are specified.
- int64 timestamp = 2 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS];
+ int64 timestamp = 2 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS];
ShardFilterType type = 3;
}
// Represents starting position in the stream from which to start reading
message StartingPosition {
// Timestamp of the record from which to start reading
- int64 timestamp = 1 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS];
+ int64 timestamp = 1 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS];
// Sequence number of the record from which to start reading
string sequence_number = 2;
ShardIteratorType type = 3;
@@ -209,7 +209,7 @@ message StreamDescriptionSummary {
int32 retention_period_hours = 6;
string stream_arn = 7;
// Timestamp that the stream was created
- int64 stream_creation_timestamp = 8 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS];
+ int64 stream_creation_timestamp = 8 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS];
// Name of the stream
string stream_name = 9;
// Current status of the stream
@@ -322,7 +322,7 @@ message ListShardsRequest {
// Filter out response
ShardFilter shard_filter = 5;
// Used to distinguish streams that have the same name
- int64 stream_creation_timestamp = 6 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS];
+ int64 stream_creation_timestamp = 6 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS];
// Name of the stream
string stream_name = 7;
}
@@ -381,7 +381,7 @@ message ListStreamConsumersRequest {
string next_token = 3;
string stream_arn = 4;
// Used to distinguish streams that have the same name
- int64 stream_creation_timestamp = 5 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS];
+ int64 stream_creation_timestamp = 5 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS];
}
message ListStreamConsumersResponse {
@@ -475,8 +475,8 @@ message PutRecordsRequestEntry {
// Represents result of an individual record
message PutRecordsResultEntry {
- string error_message = 2 [(FieldTransformer) = TRANSFORM_EMPTY_TO_NOTHING];
- string error_code = 3 [(FieldTransformer) = TRANSFORM_EMPTY_TO_NOTHING];
+ string error_message = 2 [(FieldTransformer) = TRANSFORM_EMPTY_TO_NOTHING];
+ string error_code = 3 [(FieldTransformer) = TRANSFORM_EMPTY_TO_NOTHING];
string sequence_number = 4;
string shard_id = 5;
}
@@ -484,7 +484,7 @@ message PutRecordsResultEntry {
message GetRecordsRequest {
Ydb.Operations.OperationParams operation_params = 1;
// Max number of records to return
- int32 limit = 2;
+ int32 limit = 2;
// Iterator to shard
string shard_iterator = 3;
}
@@ -513,7 +513,7 @@ message PutRecordRequest {
string explicit_hash_key = 3;
// Used as input to hash function that maps partition key to a specific shard
string partition_key = 4;
- string sequence_number_for_ordering = 5;
+ string sequence_number_for_ordering = 5;
// Name of the stream to put record to
string stream_name = 6;
}
@@ -564,7 +564,7 @@ message GetShardIteratorRequest {
// Name of the stream
string stream_name = 5;
// Used with shard iterator type AT_TIMESTAMP
- int64 timestamp = 6 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS];
+ int64 timestamp = 6 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS];
}
message GetShardIteratorResponse {
@@ -573,7 +573,7 @@ message GetShardIteratorResponse {
}
message GetShardIteratorResult {
- string shard_iterator = 1;
+ string shard_iterator = 1;
}
message SubscribeToShardRequest {
@@ -653,8 +653,8 @@ message DescribeStreamSummaryResponse {
}
message DescribeStreamSummaryResult {
- // Stream description sumary
- StreamDescriptionSummary stream_description_summary = 1;
+ // Stream description sumary
+ StreamDescriptionSummary stream_description_summary = 1;
}
message DisableEnhancedMonitoringRequest {
diff --git a/ydb/public/sdk/cpp/client/ydb_datastreams/datastreams.cpp b/ydb/public/sdk/cpp/client/ydb_datastreams/datastreams.cpp
index 79552c06ef..f8297f4323 100644
--- a/ydb/public/sdk/cpp/client/ydb_datastreams/datastreams.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_datastreams/datastreams.cpp
@@ -94,21 +94,21 @@ namespace NYdb::NDataStreams::V1 {
Ydb::DataStreams::V1::DescribeStreamResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDescribeStream);
}
- TAsyncListShardsResult ListShards(const TString &path,
- const Ydb::DataStreams::V1::ShardFilter& shardFilter,
- TListShardsSettings settings) {
+ TAsyncListShardsResult ListShards(const TString &path,
+ const Ydb::DataStreams::V1::ShardFilter& shardFilter,
+ TListShardsSettings settings) {
return CallImpl<Ydb::DataStreams::V1::DataStreamsService,
Ydb::DataStreams::V1::ListShardsRequest,
Ydb::DataStreams::V1::ListShardsResponse,
- Ydb::DataStreams::V1::ListShardsResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncListShards,
- [&](Ydb::DataStreams::V1::ListShardsRequest& req) {
- req.set_exclusive_start_shard_id(settings.ExclusiveStartShardId_);
- req.set_max_results(settings.MaxResults_);
- req.set_next_token(settings.NextToken_);
- req.mutable_shard_filter()->CopyFrom(shardFilter);
- req.set_stream_creation_timestamp(settings.StreamCreationTimestamp_);
- req.set_stream_name(path);
- });
+ Ydb::DataStreams::V1::ListShardsResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncListShards,
+ [&](Ydb::DataStreams::V1::ListShardsRequest& req) {
+ req.set_exclusive_start_shard_id(settings.ExclusiveStartShardId_);
+ req.set_max_results(settings.MaxResults_);
+ req.set_next_token(settings.NextToken_);
+ req.mutable_shard_filter()->CopyFrom(shardFilter);
+ req.set_stream_creation_timestamp(settings.StreamCreationTimestamp_);
+ req.set_stream_name(path);
+ });
}
TAsyncPutRecordsResult PutRecords(const TString& path, const std::vector<TDataRecord>& records, TPutRecordsSettings settings) {
@@ -127,30 +127,30 @@ namespace NYdb::NDataStreams::V1 {
});
}
- TAsyncGetRecordsResult GetRecords(const TString& shardIterator, TGetRecordsSettings settings) {
+ TAsyncGetRecordsResult GetRecords(const TString& shardIterator, TGetRecordsSettings settings) {
return CallImpl<Ydb::DataStreams::V1::DataStreamsService,
Ydb::DataStreams::V1::GetRecordsRequest,
Ydb::DataStreams::V1::GetRecordsResponse,
- Ydb::DataStreams::V1::GetRecordsResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncGetRecords,
- [&](Ydb::DataStreams::V1::GetRecordsRequest& req) {
- req.set_shard_iterator(shardIterator);
- req.set_limit(settings.Limit_);
- });
+ Ydb::DataStreams::V1::GetRecordsResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncGetRecords,
+ [&](Ydb::DataStreams::V1::GetRecordsRequest& req) {
+ req.set_shard_iterator(shardIterator);
+ req.set_limit(settings.Limit_);
+ });
}
- TAsyncGetShardIteratorResult GetShardIterator(const TString& path, const TString& shardId,
- Ydb::DataStreams::V1::ShardIteratorType shardIteratorType,
- TGetShardIteratorSettings settings) {
+ TAsyncGetShardIteratorResult GetShardIterator(const TString& path, const TString& shardId,
+ Ydb::DataStreams::V1::ShardIteratorType shardIteratorType,
+ TGetShardIteratorSettings settings) {
return CallImpl<Ydb::DataStreams::V1::DataStreamsService,
Ydb::DataStreams::V1::GetShardIteratorRequest,
Ydb::DataStreams::V1::GetShardIteratorResponse,
Ydb::DataStreams::V1::GetShardIteratorResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncGetShardIterator,
[&](Ydb::DataStreams::V1::GetShardIteratorRequest& req) {
req.set_stream_name(path);
- req.set_shard_id(shardId);
- req.set_shard_iterator_type(shardIteratorType);
- req.set_starting_sequence_number(settings.StartingSequenceNumber_);
- req.set_timestamp(settings.Timestamp_);
+ req.set_shard_id(shardId);
+ req.set_shard_iterator_type(shardIteratorType);
+ req.set_starting_sequence_number(settings.StartingSequenceNumber_);
+ req.set_timestamp(settings.Timestamp_);
});
}
@@ -168,14 +168,14 @@ namespace NYdb::NDataStreams::V1 {
Ydb::DataStreams::V1::DescribeLimitsResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDescribeLimits);
}
- TAsyncDescribeStreamSummaryResult DescribeStreamSummary(const TString& path, TDescribeStreamSummarySettings settings) {
+ TAsyncDescribeStreamSummaryResult DescribeStreamSummary(const TString& path, TDescribeStreamSummarySettings settings) {
return CallImpl<Ydb::DataStreams::V1::DataStreamsService,
Ydb::DataStreams::V1::DescribeStreamSummaryRequest,
Ydb::DataStreams::V1::DescribeStreamSummaryResponse,
- Ydb::DataStreams::V1::DescribeStreamSummaryResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDescribeStreamSummary,
- [&](Ydb::DataStreams::V1::DescribeStreamSummaryRequest& req) {
- req.set_stream_name(path);
- });
+ Ydb::DataStreams::V1::DescribeStreamSummaryResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDescribeStreamSummary,
+ [&](Ydb::DataStreams::V1::DescribeStreamSummaryRequest& req) {
+ req.set_stream_name(path);
+ });
}
TAsyncDecreaseStreamRetentionPeriodResult DecreaseStreamRetentionPeriod(const TString& path, TDecreaseStreamRetentionPeriodSettings settings) {
@@ -213,26 +213,26 @@ namespace NYdb::NDataStreams::V1 {
});
}
- TAsyncRegisterStreamConsumerResult RegisterStreamConsumer(const TString& path, const TString& consumer_name, TRegisterStreamConsumerSettings settings) {
+ TAsyncRegisterStreamConsumerResult RegisterStreamConsumer(const TString& path, const TString& consumer_name, TRegisterStreamConsumerSettings settings) {
return CallImpl<Ydb::DataStreams::V1::DataStreamsService,
Ydb::DataStreams::V1::RegisterStreamConsumerRequest,
Ydb::DataStreams::V1::RegisterStreamConsumerResponse,
- Ydb::DataStreams::V1::RegisterStreamConsumerResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncRegisterStreamConsumer,
- [&](Ydb::DataStreams::V1::RegisterStreamConsumerRequest& req) {
- req.set_stream_arn(path);
- req.set_consumer_name(consumer_name);
- });
+ Ydb::DataStreams::V1::RegisterStreamConsumerResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncRegisterStreamConsumer,
+ [&](Ydb::DataStreams::V1::RegisterStreamConsumerRequest& req) {
+ req.set_stream_arn(path);
+ req.set_consumer_name(consumer_name);
+ });
}
- TAsyncDeregisterStreamConsumerResult DeregisterStreamConsumer(const TString& path, const TString& consumer_name, TDeregisterStreamConsumerSettings settings) {
+ TAsyncDeregisterStreamConsumerResult DeregisterStreamConsumer(const TString& path, const TString& consumer_name, TDeregisterStreamConsumerSettings settings) {
return CallImpl<Ydb::DataStreams::V1::DataStreamsService,
Ydb::DataStreams::V1::DeregisterStreamConsumerRequest,
Ydb::DataStreams::V1::DeregisterStreamConsumerResponse,
- Ydb::DataStreams::V1::DeregisterStreamConsumerResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDeregisterStreamConsumer,
- [&](Ydb::DataStreams::V1::DeregisterStreamConsumerRequest& req) {
- req.set_stream_arn(path);
- req.set_consumer_name(consumer_name);
- });
+ Ydb::DataStreams::V1::DeregisterStreamConsumerResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDeregisterStreamConsumer,
+ [&](Ydb::DataStreams::V1::DeregisterStreamConsumerRequest& req) {
+ req.set_stream_arn(path);
+ req.set_consumer_name(consumer_name);
+ });
}
TAsyncDescribeStreamConsumerResult DescribeStreamConsumer(TDescribeStreamConsumerSettings settings) {
@@ -242,15 +242,15 @@ namespace NYdb::NDataStreams::V1 {
Ydb::DataStreams::V1::DescribeStreamConsumerResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDescribeStreamConsumer);
}
- TAsyncListStreamConsumersResult ListStreamConsumers(const TString& path, TListStreamConsumersSettings settings) {
+ TAsyncListStreamConsumersResult ListStreamConsumers(const TString& path, TListStreamConsumersSettings settings) {
return CallImpl<Ydb::DataStreams::V1::DataStreamsService,
Ydb::DataStreams::V1::ListStreamConsumersRequest,
Ydb::DataStreams::V1::ListStreamConsumersResponse,
- Ydb::DataStreams::V1::ListStreamConsumersResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncListStreamConsumers, [&](Ydb::DataStreams::V1::ListStreamConsumersRequest& req) {
- req.set_stream_arn(path);
- req.set_next_token(settings.NextToken_);
- req.set_max_results(settings.MaxResults_);
- });
+ Ydb::DataStreams::V1::ListStreamConsumersResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncListStreamConsumers, [&](Ydb::DataStreams::V1::ListStreamConsumersRequest& req) {
+ req.set_stream_arn(path);
+ req.set_next_token(settings.NextToken_);
+ req.set_max_results(settings.MaxResults_);
+ });
}
TAsyncAddTagsToStreamResult AddTagsToStream(TAddTagsToStreamSettings settings) {
@@ -333,12 +333,12 @@ namespace NYdb::NDataStreams::V1 {
return CallImpl<Ydb::DataStreams::V1::DataStreamsService,
Ydb::DataStreams::V1::DeleteStreamRequest,
Ydb::DataStreams::V1::DeleteStreamResponse,
- Ydb::DataStreams::V1::DeleteStreamResult>(settings,
- &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDeleteStream,
- [&](Ydb::DataStreams::V1::DeleteStreamRequest& req) {
- req.set_stream_name(path);
- req.set_enforce_consumer_deletion(settings.EnforceConsumerDeletion_);
- });
+ Ydb::DataStreams::V1::DeleteStreamResult>(settings,
+ &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDeleteStream,
+ [&](Ydb::DataStreams::V1::DeleteStreamRequest& req) {
+ req.set_stream_name(path);
+ req.set_enforce_consumer_deletion(settings.EnforceConsumerDeletion_);
+ });
}
TAsyncDescribeStreamResult DescribeStream(const TString &path, TDescribeStreamSettings settings) {
@@ -400,22 +400,22 @@ namespace NYdb::NDataStreams::V1 {
return Impl_->ListStreams(settings);
}
- TAsyncListShardsResult TDataStreamsClient::ListShards(const TString& path,
- const Ydb::DataStreams::V1::ShardFilter& shardFilter,
- TListShardsSettings settings) {
- return Impl_->ListShards(path, shardFilter, settings);
+ TAsyncListShardsResult TDataStreamsClient::ListShards(const TString& path,
+ const Ydb::DataStreams::V1::ShardFilter& shardFilter,
+ TListShardsSettings settings) {
+ return Impl_->ListShards(path, shardFilter, settings);
}
TAsyncPutRecordsResult TDataStreamsClient::PutRecords(const TString& path, const std::vector<TDataRecord>& records, TPutRecordsSettings settings) {
return Impl_->PutRecords(path, records, settings);
}
- TAsyncGetRecordsResult TDataStreamsClient::GetRecords(const TString& shardIterator, TGetRecordsSettings settings) {
- return Impl_->GetRecords(shardIterator, settings);
+ TAsyncGetRecordsResult TDataStreamsClient::GetRecords(const TString& shardIterator, TGetRecordsSettings settings) {
+ return Impl_->GetRecords(shardIterator, settings);
}
- TAsyncGetShardIteratorResult TDataStreamsClient::GetShardIterator(const TString& path, const TString& shardId, Ydb::DataStreams::V1::ShardIteratorType shardIteratorType, TGetShardIteratorSettings settings) {
- return Impl_->GetShardIterator(path, shardId, shardIteratorType, settings);
+ TAsyncGetShardIteratorResult TDataStreamsClient::GetShardIterator(const TString& path, const TString& shardId, Ydb::DataStreams::V1::ShardIteratorType shardIteratorType, TGetShardIteratorSettings settings) {
+ return Impl_->GetShardIterator(path, shardId, shardIteratorType, settings);
}
/* TAsyncSubscribeToShardResult TDataStreamsClient::SubscribeToShard(TSubscribeToShardSettings settings) {
@@ -426,8 +426,8 @@ namespace NYdb::NDataStreams::V1 {
return Impl_->DescribeLimits(settings);
}
- TAsyncDescribeStreamSummaryResult TDataStreamsClient::DescribeStreamSummary(const TString& path, TDescribeStreamSummarySettings settings) {
- return Impl_->DescribeStreamSummary(path, settings);
+ TAsyncDescribeStreamSummaryResult TDataStreamsClient::DescribeStreamSummary(const TString& path, TDescribeStreamSummarySettings settings) {
+ return Impl_->DescribeStreamSummary(path, settings);
}
TAsyncDecreaseStreamRetentionPeriodResult TDataStreamsClient::DecreaseStreamRetentionPeriod(const TString& path, TDecreaseStreamRetentionPeriodSettings settings) {
@@ -442,20 +442,20 @@ namespace NYdb::NDataStreams::V1 {
return Impl_->UpdateShardCount(path, settings);
}
- TAsyncRegisterStreamConsumerResult TDataStreamsClient::RegisterStreamConsumer(const TString& path, const TString& consumer_name, const TRegisterStreamConsumerSettings settings) {
- return Impl_->RegisterStreamConsumer(path, consumer_name, settings);
+ TAsyncRegisterStreamConsumerResult TDataStreamsClient::RegisterStreamConsumer(const TString& path, const TString& consumer_name, const TRegisterStreamConsumerSettings settings) {
+ return Impl_->RegisterStreamConsumer(path, consumer_name, settings);
}
- TAsyncDeregisterStreamConsumerResult TDataStreamsClient::DeregisterStreamConsumer(const TString& path, const TString& consumer_name, TDeregisterStreamConsumerSettings settings) {
- return Impl_->DeregisterStreamConsumer(path, consumer_name, settings);
+ TAsyncDeregisterStreamConsumerResult TDataStreamsClient::DeregisterStreamConsumer(const TString& path, const TString& consumer_name, TDeregisterStreamConsumerSettings settings) {
+ return Impl_->DeregisterStreamConsumer(path, consumer_name, settings);
}
TAsyncDescribeStreamConsumerResult TDataStreamsClient::DescribeStreamConsumer(TDescribeStreamConsumerSettings settings) {
return Impl_->DescribeStreamConsumer(settings);
}
- TAsyncListStreamConsumersResult TDataStreamsClient::ListStreamConsumers(const TString& path, TListStreamConsumersSettings settings) {
- return Impl_->ListStreamConsumers(path, settings);
+ TAsyncListStreamConsumersResult TDataStreamsClient::ListStreamConsumers(const TString& path, TListStreamConsumersSettings settings) {
+ return Impl_->ListStreamConsumers(path, settings);
}
TAsyncAddTagsToStreamResult TDataStreamsClient::AddTagsToStream(TAddTagsToStreamSettings settings) {
diff --git a/ydb/public/sdk/cpp/client/ydb_datastreams/datastreams.h b/ydb/public/sdk/cpp/client/ydb_datastreams/datastreams.h
index 937e819607..783f51080d 100644
--- a/ydb/public/sdk/cpp/client/ydb_datastreams/datastreams.h
+++ b/ydb/public/sdk/cpp/client/ydb_datastreams/datastreams.h
@@ -104,26 +104,26 @@ namespace NYdb::NDataStreams::V1 {
FLUENT_SETTING(TString, ExclusiveStartStreamName);
FLUENT_SETTING_DEFAULT(bool, Recurse, false);
};
- struct TDeleteStreamSettings : public NYdb::TOperationRequestSettings<TDeleteStreamSettings> {
- FLUENT_SETTING_DEFAULT(bool, EnforceConsumerDeletion, false);
- };
+ struct TDeleteStreamSettings : public NYdb::TOperationRequestSettings<TDeleteStreamSettings> {
+ FLUENT_SETTING_DEFAULT(bool, EnforceConsumerDeletion, false);
+ };
struct TDescribeStreamSettings : public NYdb::TOperationRequestSettings<TDescribeStreamSettings> {
FLUENT_SETTING(ui32, Limit);
FLUENT_SETTING(TString, ExclusiveStartShardId);
};
- struct TListShardsSettings : public NYdb::TOperationRequestSettings<TListShardsSettings> {
- FLUENT_SETTING(TString, ExclusiveStartShardId);
- FLUENT_SETTING(ui32, MaxResults);
- FLUENT_SETTING(TString, NextToken);
- FLUENT_SETTING(ui64, StreamCreationTimestamp);
- };
- struct TGetRecordsSettings : public NYdb::TOperationRequestSettings<TGetRecordsSettings> {
- FLUENT_SETTING_DEFAULT(ui32, Limit, 10000);
- };
- struct TGetShardIteratorSettings : public NYdb::TOperationRequestSettings<TGetShardIteratorSettings> {
- FLUENT_SETTING(TString, StartingSequenceNumber);
- FLUENT_SETTING(ui64, Timestamp);
- };
+ struct TListShardsSettings : public NYdb::TOperationRequestSettings<TListShardsSettings> {
+ FLUENT_SETTING(TString, ExclusiveStartShardId);
+ FLUENT_SETTING(ui32, MaxResults);
+ FLUENT_SETTING(TString, NextToken);
+ FLUENT_SETTING(ui64, StreamCreationTimestamp);
+ };
+ struct TGetRecordsSettings : public NYdb::TOperationRequestSettings<TGetRecordsSettings> {
+ FLUENT_SETTING_DEFAULT(ui32, Limit, 10000);
+ };
+ struct TGetShardIteratorSettings : public NYdb::TOperationRequestSettings<TGetShardIteratorSettings> {
+ FLUENT_SETTING(TString, StartingSequenceNumber);
+ FLUENT_SETTING(ui64, Timestamp);
+ };
struct TSubscribeToShardSettings : public NYdb::TOperationRequestSettings<TSubscribeToShardSettings> {};
struct TDescribeLimitsSettings : public NYdb::TOperationRequestSettings<TDescribeLimitsSettings> {};
struct TDescribeStreamSummarySettings : public NYdb::TOperationRequestSettings<TDescribeStreamSummarySettings> {};
@@ -146,10 +146,10 @@ namespace NYdb::NDataStreams::V1 {
struct TRegisterStreamConsumerSettings : public NYdb::TOperationRequestSettings<TRegisterStreamConsumerSettings> {};
struct TDeregisterStreamConsumerSettings : public NYdb::TOperationRequestSettings<TDeregisterStreamConsumerSettings> {};
struct TDescribeStreamConsumerSettings : public NYdb::TOperationRequestSettings<TDescribeStreamConsumerSettings> {};
- struct TListStreamConsumersSettings : public NYdb::TOperationRequestSettings<TListStreamConsumersSettings> {
- FLUENT_SETTING(ui32, MaxResults);
- FLUENT_SETTING(TString, NextToken);
- };
+ struct TListStreamConsumersSettings : public NYdb::TOperationRequestSettings<TListStreamConsumersSettings> {
+ FLUENT_SETTING(ui32, MaxResults);
+ FLUENT_SETTING(TString, NextToken);
+ };
struct TAddTagsToStreamSettings : public NYdb::TOperationRequestSettings<TAddTagsToStreamSettings> {};
struct TDisableEnhancedMonitoringSettings : public NYdb::TOperationRequestSettings<TDisableEnhancedMonitoringSettings> {};
struct TEnableEnhancedMonitoringSettings : public NYdb::TOperationRequestSettings<TEnableEnhancedMonitoringSettings> {};
@@ -172,21 +172,21 @@ namespace NYdb::NDataStreams::V1 {
TAsyncDescribeStreamResult DescribeStream(const TString& path, TDescribeStreamSettings settings = TDescribeStreamSettings());
TAsyncPutRecordResult PutRecord(const TString& path, const TDataRecord& record, TPutRecordSettings settings = TPutRecordSettings());
TAsyncListStreamsResult ListStreams(TListStreamsSettings settings = TListStreamsSettings());
- TAsyncListShardsResult ListShards(const TString& path, const Ydb::DataStreams::V1::ShardFilter& shardFilter, TListShardsSettings settings = TListShardsSettings());
+ TAsyncListShardsResult ListShards(const TString& path, const Ydb::DataStreams::V1::ShardFilter& shardFilter, TListShardsSettings settings = TListShardsSettings());
TAsyncPutRecordsResult PutRecords(const TString& path, const std::vector<TDataRecord>& records, TPutRecordsSettings settings = TPutRecordsSettings());
- TAsyncGetRecordsResult GetRecords(const TString& shardIterator, TGetRecordsSettings settings = TGetRecordsSettings());
- TAsyncGetShardIteratorResult GetShardIterator(const TString& path, const TString& shardId, Ydb::DataStreams::V1::ShardIteratorType shardIteratorTypeStr,
- TGetShardIteratorSettings settings = TGetShardIteratorSettings());
+ TAsyncGetRecordsResult GetRecords(const TString& shardIterator, TGetRecordsSettings settings = TGetRecordsSettings());
+ TAsyncGetShardIteratorResult GetShardIterator(const TString& path, const TString& shardId, Ydb::DataStreams::V1::ShardIteratorType shardIteratorTypeStr,
+ TGetShardIteratorSettings settings = TGetShardIteratorSettings());
// TAsyncSubscribeToShardResult SubscribeToShard(TSubscribeToShardSettings settings = TSubscribeToShardSettings());
TAsyncDescribeLimitsResult DescribeLimits(TDescribeLimitsSettings settings = TDescribeLimitsSettings());
- TAsyncDescribeStreamSummaryResult DescribeStreamSummary(const TString& path, TDescribeStreamSummarySettings settings = TDescribeStreamSummarySettings());
+ TAsyncDescribeStreamSummaryResult DescribeStreamSummary(const TString& path, TDescribeStreamSummarySettings settings = TDescribeStreamSummarySettings());
TAsyncDecreaseStreamRetentionPeriodResult DecreaseStreamRetentionPeriod(const TString& path, TDecreaseStreamRetentionPeriodSettings settings = TDecreaseStreamRetentionPeriodSettings());
TAsyncIncreaseStreamRetentionPeriodResult IncreaseStreamRetentionPeriod(const TString& path, TIncreaseStreamRetentionPeriodSettings settings = TIncreaseStreamRetentionPeriodSettings());
TAsyncUpdateShardCountResult UpdateShardCount(const TString& path, TUpdateShardCountSettings settings = TUpdateShardCountSettings());
- TAsyncRegisterStreamConsumerResult RegisterStreamConsumer(const TString& path, const TString& consumer_name, TRegisterStreamConsumerSettings settings = TRegisterStreamConsumerSettings());
- TAsyncDeregisterStreamConsumerResult DeregisterStreamConsumer(const TString& path, const TString& consumer_name, TDeregisterStreamConsumerSettings settings = TDeregisterStreamConsumerSettings());
+ TAsyncRegisterStreamConsumerResult RegisterStreamConsumer(const TString& path, const TString& consumer_name, TRegisterStreamConsumerSettings settings = TRegisterStreamConsumerSettings());
+ TAsyncDeregisterStreamConsumerResult DeregisterStreamConsumer(const TString& path, const TString& consumer_name, TDeregisterStreamConsumerSettings settings = TDeregisterStreamConsumerSettings());
TAsyncDescribeStreamConsumerResult DescribeStreamConsumer(TDescribeStreamConsumerSettings settings = TDescribeStreamConsumerSettings());
- TAsyncListStreamConsumersResult ListStreamConsumers(const TString& path, TListStreamConsumersSettings settings = TListStreamConsumersSettings());
+ TAsyncListStreamConsumersResult ListStreamConsumers(const TString& path, TListStreamConsumersSettings settings = TListStreamConsumersSettings());
TAsyncAddTagsToStreamResult AddTagsToStream(TAddTagsToStreamSettings settings = TAddTagsToStreamSettings());
TAsyncDisableEnhancedMonitoringResult DisableEnhancedMonitoring(TDisableEnhancedMonitoringSettings settings = TDisableEnhancedMonitoringSettings());
TAsyncEnableEnhancedMonitoringResult EnableEnhancedMonitoring(TEnableEnhancedMonitoringSettings settings = TEnableEnhancedMonitoringSettings());
diff --git a/ydb/services/datastreams/datastreams_proxy.cpp b/ydb/services/datastreams/datastreams_proxy.cpp
index 336df8d874..03a6d06213 100644
--- a/ydb/services/datastreams/datastreams_proxy.cpp
+++ b/ydb/services/datastreams/datastreams_proxy.cpp
@@ -1,21 +1,21 @@
#include "datastreams_proxy.h"
#include "put_records_actor.h"
-#include "shard_iterator.h"
-#include "next_token.h"
+#include "shard_iterator.h"
+#include "next_token.h"
#include <ydb/core/grpc_services/grpc_request_proxy.h>
#include <ydb/core/grpc_services/rpc_deferrable.h>
#include <ydb/core/grpc_services/rpc_scheme_base.h>
#include <ydb/core/persqueue/partition.h>
#include <ydb/core/persqueue/write_meta.h>
-
+
#include <ydb/services/lib/actors/pq_schema_actor.h>
#include <ydb/services/lib/sharding/sharding.h>
#include <util/folder/path.h>
-#include <iterator>
-
+#include <iterator>
+
using namespace NActors;
using namespace NKikimrClient;
@@ -111,7 +111,7 @@ namespace NKikimr::NDataStreams::V1 {
void FillProposeRequest(TEvTxUserProxy::TEvProposeTransaction& proposal, const TActorContext& ctx,
const TString& workingDir, const TString& name);
void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx);
- void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx);
+ void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx);
void Handle(TEvTxUserProxy::TEvProposeTransactionStatus::TPtr& ev, const TActorContext& ctx);
};
@@ -129,12 +129,12 @@ namespace NKikimr::NDataStreams::V1 {
Become(&TCreateStreamActor::StateWork);
}
- void TCreateStreamActor::HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) {
- Y_UNUSED(ev);
- Y_UNUSED(ctx);
- }
+ void TCreateStreamActor::HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) {
+ Y_UNUSED(ev);
+ Y_UNUSED(ctx);
+ }
+
-
void TCreateStreamActor::FillProposeRequest(TEvTxUserProxy::TEvProposeTransaction& proposal,
const TActorContext& ctx, const TString& workingDir, const TString& name)
{
@@ -179,7 +179,7 @@ namespace NKikimr::NDataStreams::V1 {
TStringBuilder() << "Stream with name " << GetProtoRequest()->stream_name() << " is already exists",
ctx);
}
- return TBase::TBase::Handle(ev, ctx);
+ return TBase::TBase::Handle(ev, ctx);
}
void TCreateStreamActor::StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) {
@@ -202,29 +202,29 @@ namespace NKikimr::NDataStreams::V1 {
void FillProposeRequest(TEvTxUserProxy::TEvProposeTransaction& proposal, const TActorContext& ctx,
const TString& workingDir, const TString& name);
-
- void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev,
- const TActorContext& ctx);
-
- private:
- bool EnforceDeletion;
+
+ void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev,
+ const TActorContext& ctx);
+
+ private:
+ bool EnforceDeletion;
};
TDeleteStreamActor::TDeleteStreamActor(NKikimr::NGRpcService::TEvDataStreamsDeleteStreamRequest* request)
: TBase(request, request->GetProtoRequest()->stream_name())
- , EnforceDeletion{request->GetProtoRequest()->enforce_consumer_deletion()}
+ , EnforceDeletion{request->GetProtoRequest()->enforce_consumer_deletion()}
{
}
void TDeleteStreamActor::Bootstrap(const NActors::TActorContext& ctx) {
TBase::Bootstrap(ctx);
- SendDescribeProposeRequest(ctx);
+ SendDescribeProposeRequest(ctx);
Become(&TDeleteStreamActor::StateWork);
}
- void TDeleteStreamActor::FillProposeRequest(TEvTxUserProxy::TEvProposeTransaction& proposal,
- const TActorContext& ctx, const TString& workingDir,
- const TString& name)
+ void TDeleteStreamActor::FillProposeRequest(TEvTxUserProxy::TEvProposeTransaction& proposal,
+ const TActorContext& ctx, const TString& workingDir,
+ const TString& name)
{
LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, "WorkingDir = " << workingDir << ", name = " << name);
NKikimrSchemeOp::TModifyScheme& modifyScheme(*proposal.Record.MutableTransaction()->MutableModifyScheme());
@@ -233,24 +233,24 @@ namespace NKikimr::NDataStreams::V1 {
modifyScheme.MutableDrop()->SetName(name);
}
- void TDeleteStreamActor::HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev,
- const TActorContext& ctx) {
- if (ReplyIfNotTopic(ev, ctx)) {
- return;
- }
-
- const auto& response = ev->Get()->Request.Get()->ResultSet.front();
- const auto& pqGroupDescription = response.PQGroupInfo->Description;
- const auto& readRules = pqGroupDescription.GetPQTabletConfig().GetReadRules();
-
- if (readRules.size() > 0 && EnforceDeletion == false) {
- return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::ERROR,
- TStringBuilder() << "Stream has registered consumers" <<
- "and EnforceConsumerDeletion flag is false", ctx);
- }
-
- SendProposeRequest(ctx);
- }
+ void TDeleteStreamActor::HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev,
+ const TActorContext& ctx) {
+ if (ReplyIfNotTopic(ev, ctx)) {
+ return;
+ }
+
+ const auto& response = ev->Get()->Request.Get()->ResultSet.front();
+ const auto& pqGroupDescription = response.PQGroupInfo->Description;
+ const auto& readRules = pqGroupDescription.GetPQTabletConfig().GetReadRules();
+
+ if (readRules.size() > 0 && EnforceDeletion == false) {
+ return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::ERROR,
+ TStringBuilder() << "Stream has registered consumers" <<
+ "and EnforceConsumerDeletion flag is false", ctx);
+ }
+
+ SendProposeRequest(ctx);
+ }
//-----------------------------------------------------------------------------------------------------------
class TUpdateShardCountActor : public TUpdateSchemeActor<TUpdateShardCountActor, TEvDataStreamsUpdateShardCountRequest> {
@@ -431,7 +431,7 @@ namespace NKikimr::NDataStreams::V1 {
void Bootstrap(const NActors::TActorContext& ctx);
void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx);
- void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx);
+ void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx);
void Handle(TEvTabletPipe::TEvClientConnected::TPtr& ev, const TActorContext& ctx) {
if (ev->Get()->Status != NKikimrProto::EReplyStatus::OK) {
@@ -454,7 +454,7 @@ namespace NKikimr::NDataStreams::V1 {
}
}
- void Die(const TActorContext& ctx) override {
+ void Die(const TActorContext& ctx) override {
//close all pipes
for (auto& pipe : Pipes) {
NTabletPipe::CloseClient(ctx, pipe);
@@ -492,43 +492,43 @@ namespace NKikimr::NDataStreams::V1 {
}
}
- void TDescribeStreamActor::HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) {
+ void TDescribeStreamActor::HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) {
const NSchemeCache::TSchemeCacheNavigate* result = ev->Get()->Request.Get();
Y_VERIFY(result->ResultSet.size() == 1); // describe only one topic
const auto& response = result->ResultSet.front();
- const TString path = JoinSeq("/", response.Path);
-
- if (ReplyIfNotTopic(ev, ctx)) {
- return;
- }
-
- Y_VERIFY(response.PQGroupInfo);
-
- PQGroup = response.PQGroupInfo->Description;
- SelfInfo = response.Self->Info;
- std::set<ui64> tabletIds;
- for (auto& partition : PQGroup.GetPartitions()) {
- tabletIds.insert(partition.GetTabletId());
- }
- if (tabletIds.size() == 0) {
- ReplyAndDie(ctx);
- }
-
- RequestsInfly = tabletIds.size();
-
- NTabletPipe::TClientConfig clientConfig;
- clientConfig.RetryPolicy = {
- .RetryLimitCount = 6,
- .MinRetryTime = TDuration::MilliSeconds(10),
- .MaxRetryTime = TDuration::MilliSeconds(100),
- .BackoffMultiplier = 2,
- .DoFirstRetryInstantly = true
- };
-
- for (auto& tabletId : tabletIds) {
- Pipes.push_back(ctx.Register(NTabletPipe::CreateClient(ctx.SelfID, tabletId, clientConfig)));
- TAutoPtr<TEvPersQueue::TEvOffsets> req(new TEvPersQueue::TEvOffsets);
- NTabletPipe::SendData(ctx, Pipes.back(), req.Release());
+ const TString path = JoinSeq("/", response.Path);
+
+ if (ReplyIfNotTopic(ev, ctx)) {
+ return;
+ }
+
+ Y_VERIFY(response.PQGroupInfo);
+
+ PQGroup = response.PQGroupInfo->Description;
+ SelfInfo = response.Self->Info;
+ std::set<ui64> tabletIds;
+ for (auto& partition : PQGroup.GetPartitions()) {
+ tabletIds.insert(partition.GetTabletId());
+ }
+ if (tabletIds.size() == 0) {
+ ReplyAndDie(ctx);
+ }
+
+ RequestsInfly = tabletIds.size();
+
+ NTabletPipe::TClientConfig clientConfig;
+ clientConfig.RetryPolicy = {
+ .RetryLimitCount = 6,
+ .MinRetryTime = TDuration::MilliSeconds(10),
+ .MaxRetryTime = TDuration::MilliSeconds(100),
+ .BackoffMultiplier = 2,
+ .DoFirstRetryInstantly = true
+ };
+
+ for (auto& tabletId : tabletIds) {
+ Pipes.push_back(ctx.Register(NTabletPipe::CreateClient(ctx.SelfID, tabletId, clientConfig)));
+ TAutoPtr<TEvPersQueue::TEvOffsets> req(new TEvPersQueue::TEvOffsets);
+ NTabletPipe::SendData(ctx, Pipes.back(), req.Release());
}
}
@@ -744,155 +744,155 @@ namespace NKikimr::NDataStreams::V1 {
//-----------------------------------------------------------------------------------
- class TListStreamConsumersActor : public TPQGrpcSchemaBase<TListStreamConsumersActor, NKikimr::NGRpcService::TEvDataStreamsListStreamConsumersRequest> {
- using TBase = TPQGrpcSchemaBase<TListStreamConsumersActor, TEvDataStreamsListStreamConsumersRequest>;
-
- public:
- TListStreamConsumersActor(NKikimr::NGRpcService::TEvDataStreamsListStreamConsumersRequest* request);
- ~TListStreamConsumersActor() = default;
-
- void Bootstrap(const NActors::TActorContext& ctx);
- void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx);
- void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx);
-
- protected:
- void SendResponse(const TActorContext& ctx, const std::vector<std::pair<TString, ui64>>& readRules, ui32 leftToRead);
-
- private:
- static constexpr ui32 MAX_MAX_RESULTS = 10000;
- static constexpr ui32 MIN_MAX_RESULTS = 1;
- static constexpr ui32 DEFAULT_MAX_RESULTS = 100;
-
- TString StreamArn;
- ui32 MaxResults = DEFAULT_MAX_RESULTS;
- TNextToken NextToken;
- };
- TListStreamConsumersActor::TListStreamConsumersActor(NKikimr::NGRpcService::TEvDataStreamsListStreamConsumersRequest* request)
- : TBase(request, TNextToken(request->GetProtoRequest()->next_token()).IsValid() ?
- TNextToken(request->GetProtoRequest()->next_token()).GetStreamArn() :
- request->GetProtoRequest()->stream_arn())
- , NextToken(request->GetProtoRequest()->next_token())
- {
- if (request->GetProtoRequest()->next_token().empty()) {
- StreamArn = request->GetProtoRequest()->stream_arn();
- MaxResults = request->GetProtoRequest()->max_results();
- NextToken = TNextToken(StreamArn, 0, MaxResults, TInstant::Now().MilliSeconds());
- } else {
- StreamArn = NextToken.GetStreamArn();
- MaxResults = NextToken.GetMaxResults();
- }
- }
-
- void TListStreamConsumersActor::Bootstrap(const NActors::TActorContext& ctx) {
- TBase::Bootstrap(ctx);
-
- auto maxResultsInRange = MIN_MAX_RESULTS <= MaxResults && MaxResults <= MAX_MAX_RESULTS;
- if (!maxResultsInRange) {
- return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::ERROR,
- TStringBuilder() << "Requested max_result value '" << MaxResults <<
- "' is out of range [" << MIN_MAX_RESULTS << ", " << MAX_MAX_RESULTS <<
- "]", ctx);
- }
-
- if (!NextToken.IsValid()) {
- return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::ERROR,
- TStringBuilder() << "Provided NextToken has expired or malformed", ctx);
- }
- SendDescribeProposeRequest(ctx);
- Become(&TListStreamConsumersActor::StateWork);
- }
-
- void TListStreamConsumersActor::StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) {
- switch (ev->GetTypeRewrite()) {
- default: TBase::StateWork(ev, ctx);
- }
- }
-
- void TListStreamConsumersActor::HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) {
+ class TListStreamConsumersActor : public TPQGrpcSchemaBase<TListStreamConsumersActor, NKikimr::NGRpcService::TEvDataStreamsListStreamConsumersRequest> {
+ using TBase = TPQGrpcSchemaBase<TListStreamConsumersActor, TEvDataStreamsListStreamConsumersRequest>;
+
+ public:
+ TListStreamConsumersActor(NKikimr::NGRpcService::TEvDataStreamsListStreamConsumersRequest* request);
+ ~TListStreamConsumersActor() = default;
+
+ void Bootstrap(const NActors::TActorContext& ctx);
+ void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx);
+ void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx);
+
+ protected:
+ void SendResponse(const TActorContext& ctx, const std::vector<std::pair<TString, ui64>>& readRules, ui32 leftToRead);
+
+ private:
+ static constexpr ui32 MAX_MAX_RESULTS = 10000;
+ static constexpr ui32 MIN_MAX_RESULTS = 1;
+ static constexpr ui32 DEFAULT_MAX_RESULTS = 100;
+
+ TString StreamArn;
+ ui32 MaxResults = DEFAULT_MAX_RESULTS;
+ TNextToken NextToken;
+ };
+ TListStreamConsumersActor::TListStreamConsumersActor(NKikimr::NGRpcService::TEvDataStreamsListStreamConsumersRequest* request)
+ : TBase(request, TNextToken(request->GetProtoRequest()->next_token()).IsValid() ?
+ TNextToken(request->GetProtoRequest()->next_token()).GetStreamArn() :
+ request->GetProtoRequest()->stream_arn())
+ , NextToken(request->GetProtoRequest()->next_token())
+ {
+ if (request->GetProtoRequest()->next_token().empty()) {
+ StreamArn = request->GetProtoRequest()->stream_arn();
+ MaxResults = request->GetProtoRequest()->max_results();
+ NextToken = TNextToken(StreamArn, 0, MaxResults, TInstant::Now().MilliSeconds());
+ } else {
+ StreamArn = NextToken.GetStreamArn();
+ MaxResults = NextToken.GetMaxResults();
+ }
+ }
+
+ void TListStreamConsumersActor::Bootstrap(const NActors::TActorContext& ctx) {
+ TBase::Bootstrap(ctx);
+
+ auto maxResultsInRange = MIN_MAX_RESULTS <= MaxResults && MaxResults <= MAX_MAX_RESULTS;
+ if (!maxResultsInRange) {
+ return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::ERROR,
+ TStringBuilder() << "Requested max_result value '" << MaxResults <<
+ "' is out of range [" << MIN_MAX_RESULTS << ", " << MAX_MAX_RESULTS <<
+ "]", ctx);
+ }
+
+ if (!NextToken.IsValid()) {
+ return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::ERROR,
+ TStringBuilder() << "Provided NextToken has expired or malformed", ctx);
+ }
+ SendDescribeProposeRequest(ctx);
+ Become(&TListStreamConsumersActor::StateWork);
+ }
+
+ void TListStreamConsumersActor::StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) {
+ switch (ev->GetTypeRewrite()) {
+ default: TBase::StateWork(ev, ctx);
+ }
+ }
+
+ void TListStreamConsumersActor::HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) {
const NSchemeCache::TSchemeCacheNavigate* result = ev->Get()->Request.Get();
Y_VERIFY(result->ResultSet.size() == 1); // describe only one topic
- if (ReplyIfNotTopic(ev, ctx)) {
- return;
- }
-
- std::vector<std::pair<TString, ui64>> readRules;
- ui32 leftToRead{0};
- const auto& response = result->ResultSet.front();
- const auto& pqGroupDescription = response.PQGroupInfo->Description;
- const auto& streamReadRulesNames = pqGroupDescription.GetPQTabletConfig().GetReadRules();
- const auto& streamReadRulesReadFromTimestamps = pqGroupDescription.GetPQTabletConfig().GetReadFromTimestampsMs();
- const auto alreadyRead = NextToken.GetAlreadyRead();
-
- if (alreadyRead > (ui32)streamReadRulesNames.size()) {
- return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::ERROR,
- TStringBuilder() << "Provided next_token is malformed - " <<
- "everything is already read", ctx);
- }
-
- const auto rulesToRead = std::min(streamReadRulesNames.size() - alreadyRead, MaxResults);
- readRules.reserve(rulesToRead);
- auto itName = streamReadRulesNames.begin() + alreadyRead;
- auto itTs = streamReadRulesReadFromTimestamps.begin() + alreadyRead;
- for (auto i = rulesToRead; i > 0; --i, ++itName, ++itTs) {
- readRules.push_back({*itName, *itTs});
- }
- leftToRead = streamReadRulesNames.size() - alreadyRead - rulesToRead;
-
- SendResponse(ctx, readRules, leftToRead);
- }
-
- void TListStreamConsumersActor::SendResponse(const TActorContext& ctx, const std::vector<std::pair<TString, ui64>>& readRules, ui32 leftToRead) {
- Ydb::DataStreams::V1::ListStreamConsumersResult result;
-
- for (auto& readRule : readRules) {
- auto consumer = result.Addconsumers();
- consumer->set_consumer_name(readRule.first);
- consumer->set_consumer_creation_timestamp(readRule.second);
- consumer->set_consumer_status(Ydb::DataStreams::V1::ConsumerDescription_ConsumerStatus_ACTIVE);
- // TODO: consumer->set_consumer_arn();
- }
-
- if (leftToRead > 0) {
- TNextToken token(StreamArn, NextToken.GetAlreadyRead() + readRules.size(), MaxResults, TInstant::Now().MilliSeconds());
- result.set_next_token(token.Serialize());
- }
-
- Request_->SendResult(result, Ydb::StatusIds::SUCCESS);
- Die(ctx);
- }
-
- //-----------------------------------------------------------------------------------------
-
- class TRegisterStreamConsumerActor : public TUpdateSchemeActor<TRegisterStreamConsumerActor, NKikimr::NGRpcService::TEvDataStreamsRegisterStreamConsumerRequest> {
- using TBase = TUpdateSchemeActor<TRegisterStreamConsumerActor, TEvDataStreamsRegisterStreamConsumerRequest>;
-
- public:
- TRegisterStreamConsumerActor(NKikimr::NGRpcService::TEvDataStreamsRegisterStreamConsumerRequest* request);
- ~TRegisterStreamConsumerActor() = default;
-
- void Bootstrap(const NActors::TActorContext& ctx);
- void ModifyPersqueueConfig(const TActorContext& ctx,
+ if (ReplyIfNotTopic(ev, ctx)) {
+ return;
+ }
+
+ std::vector<std::pair<TString, ui64>> readRules;
+ ui32 leftToRead{0};
+ const auto& response = result->ResultSet.front();
+ const auto& pqGroupDescription = response.PQGroupInfo->Description;
+ const auto& streamReadRulesNames = pqGroupDescription.GetPQTabletConfig().GetReadRules();
+ const auto& streamReadRulesReadFromTimestamps = pqGroupDescription.GetPQTabletConfig().GetReadFromTimestampsMs();
+ const auto alreadyRead = NextToken.GetAlreadyRead();
+
+ if (alreadyRead > (ui32)streamReadRulesNames.size()) {
+ return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::ERROR,
+ TStringBuilder() << "Provided next_token is malformed - " <<
+ "everything is already read", ctx);
+ }
+
+ const auto rulesToRead = std::min(streamReadRulesNames.size() - alreadyRead, MaxResults);
+ readRules.reserve(rulesToRead);
+ auto itName = streamReadRulesNames.begin() + alreadyRead;
+ auto itTs = streamReadRulesReadFromTimestamps.begin() + alreadyRead;
+ for (auto i = rulesToRead; i > 0; --i, ++itName, ++itTs) {
+ readRules.push_back({*itName, *itTs});
+ }
+ leftToRead = streamReadRulesNames.size() - alreadyRead - rulesToRead;
+
+ SendResponse(ctx, readRules, leftToRead);
+ }
+
+ void TListStreamConsumersActor::SendResponse(const TActorContext& ctx, const std::vector<std::pair<TString, ui64>>& readRules, ui32 leftToRead) {
+ Ydb::DataStreams::V1::ListStreamConsumersResult result;
+
+ for (auto& readRule : readRules) {
+ auto consumer = result.Addconsumers();
+ consumer->set_consumer_name(readRule.first);
+ consumer->set_consumer_creation_timestamp(readRule.second);
+ consumer->set_consumer_status(Ydb::DataStreams::V1::ConsumerDescription_ConsumerStatus_ACTIVE);
+ // TODO: consumer->set_consumer_arn();
+ }
+
+ if (leftToRead > 0) {
+ TNextToken token(StreamArn, NextToken.GetAlreadyRead() + readRules.size(), MaxResults, TInstant::Now().MilliSeconds());
+ result.set_next_token(token.Serialize());
+ }
+
+ Request_->SendResult(result, Ydb::StatusIds::SUCCESS);
+ Die(ctx);
+ }
+
+ //-----------------------------------------------------------------------------------------
+
+ class TRegisterStreamConsumerActor : public TUpdateSchemeActor<TRegisterStreamConsumerActor, NKikimr::NGRpcService::TEvDataStreamsRegisterStreamConsumerRequest> {
+ using TBase = TUpdateSchemeActor<TRegisterStreamConsumerActor, TEvDataStreamsRegisterStreamConsumerRequest>;
+
+ public:
+ TRegisterStreamConsumerActor(NKikimr::NGRpcService::TEvDataStreamsRegisterStreamConsumerRequest* request);
+ ~TRegisterStreamConsumerActor() = default;
+
+ void Bootstrap(const NActors::TActorContext& ctx);
+ void ModifyPersqueueConfig(const TActorContext& ctx,
NKikimrSchemeOp::TPersQueueGroupDescription& groupConfig,
const NKikimrSchemeOp::TPersQueueGroupDescription& pqGroupDescription,
const NKikimrSchemeOp::TDirEntry& selfInfo);
void ReplyNotifyTxCompletionResult(NSchemeShard::TEvSchemeShard::TEvNotifyTxCompletionResult::TPtr& ev, const TActorContext& ctx) override;
-
- private:
- TString ConsumerName;
- };
- TRegisterStreamConsumerActor::TRegisterStreamConsumerActor(NKikimr::NGRpcService::TEvDataStreamsRegisterStreamConsumerRequest* request)
- : TBase(request, request->GetProtoRequest()->stream_arn())
- , ConsumerName(request->GetProtoRequest()->consumer_name())
- {
- }
-
- void TRegisterStreamConsumerActor::Bootstrap(const NActors::TActorContext& ctx) {
- TBase::Bootstrap(ctx);
- SendDescribeProposeRequest(ctx);
- Become(&TRegisterStreamConsumerActor::StateWork);
- }
-
+
+ private:
+ TString ConsumerName;
+ };
+ TRegisterStreamConsumerActor::TRegisterStreamConsumerActor(NKikimr::NGRpcService::TEvDataStreamsRegisterStreamConsumerRequest* request)
+ : TBase(request, request->GetProtoRequest()->stream_arn())
+ , ConsumerName(request->GetProtoRequest()->consumer_name())
+ {
+ }
+
+ void TRegisterStreamConsumerActor::Bootstrap(const NActors::TActorContext& ctx) {
+ TBase::Bootstrap(ctx);
+ SendDescribeProposeRequest(ctx);
+ Become(&TRegisterStreamConsumerActor::StateWork);
+ }
+
void TRegisterStreamConsumerActor::ModifyPersqueueConfig(
const TActorContext& ctx,
NKikimrSchemeOp::TPersQueueGroupDescription& groupConfig,
@@ -901,72 +901,72 @@ namespace NKikimr::NDataStreams::V1 {
) {
Y_UNUSED(pqGroupDescription);
- auto* pqConfig = groupConfig.MutablePQTabletConfig();
- Ydb::PersQueue::V1::TopicSettings::ReadRule readRule;
- readRule.set_consumer_name(ConsumerName);
- readRule.set_supported_format(Ydb::PersQueue::V1::TopicSettings_Format_FORMAT_BASE);
- readRule.set_starting_message_timestamp_ms(TInstant::Now().MilliSeconds());
- readRule.set_important(false);
+ auto* pqConfig = groupConfig.MutablePQTabletConfig();
+ Ydb::PersQueue::V1::TopicSettings::ReadRule readRule;
+ readRule.set_consumer_name(ConsumerName);
+ readRule.set_supported_format(Ydb::PersQueue::V1::TopicSettings_Format_FORMAT_BASE);
+ readRule.set_starting_message_timestamp_ms(TInstant::Now().MilliSeconds());
+ readRule.set_important(false);
readRule.set_service_type(YDS_SERVICE_TYPE);
-
- if (readRule.version() == 0) {
+
+ if (readRule.version() == 0) {
readRule.set_version(selfInfo.GetVersion().GetPQVersion());
- }
+ }
auto serviceTypes = GetSupportedClientServiceTypes(ctx);
TString error = AddReadRuleToConfig(pqConfig, readRule, serviceTypes, ctx);
bool hasDuplicates = false;
- if (error.Empty()) {
+ if (error.Empty()) {
hasDuplicates = CheckReadRulesConfig(*pqConfig, serviceTypes, error);
- }
-
- if (!error.Empty()) {
- return ReplyWithError(hasDuplicates ? Ydb::StatusIds::ALREADY_EXISTS : Ydb::StatusIds::BAD_REQUEST,
+ }
+
+ if (!error.Empty()) {
+ return ReplyWithError(hasDuplicates ? Ydb::StatusIds::ALREADY_EXISTS : Ydb::StatusIds::BAD_REQUEST,
hasDuplicates ? Ydb::PersQueue::ErrorCode::OK : Ydb::PersQueue::ErrorCode::BAD_REQUEST, error, ctx);
- }
- }
-
+ }
+ }
+
void TRegisterStreamConsumerActor::ReplyNotifyTxCompletionResult(NSchemeShard::TEvSchemeShard::TEvNotifyTxCompletionResult::TPtr& ev, const TActorContext& ctx) {
- Y_UNUSED(ev);
- Ydb::DataStreams::V1::RegisterStreamConsumerResult result;
- auto consumer = result.Mutableconsumer();
- consumer->set_consumer_name(ConsumerName);
- consumer->set_consumer_creation_timestamp(std::chrono::seconds(std::time(nullptr)).count());
- // TODO: consumer->set_consumer_arn();
- consumer->set_consumer_status(Ydb::DataStreams::V1::ConsumerDescription_ConsumerStatus_ACTIVE);
- Request_->SendResult(result, Ydb::StatusIds::SUCCESS);
- Die(ctx);
- }
-
- //-----------------------------------------------------------------------------------------
-
- class TDeregisterStreamConsumerActor : public TUpdateSchemeActor<TDeregisterStreamConsumerActor, NKikimr::NGRpcService::TEvDataStreamsDeregisterStreamConsumerRequest> {
- using TBase = TUpdateSchemeActor<TDeregisterStreamConsumerActor, TEvDataStreamsDeregisterStreamConsumerRequest>;
-
- public:
- TDeregisterStreamConsumerActor(NKikimr::NGRpcService::TEvDataStreamsDeregisterStreamConsumerRequest* request);
- ~TDeregisterStreamConsumerActor() = default;
-
- void Bootstrap(const NActors::TActorContext& ctx);
- void ModifyPersqueueConfig(const TActorContext& ctx,
+ Y_UNUSED(ev);
+ Ydb::DataStreams::V1::RegisterStreamConsumerResult result;
+ auto consumer = result.Mutableconsumer();
+ consumer->set_consumer_name(ConsumerName);
+ consumer->set_consumer_creation_timestamp(std::chrono::seconds(std::time(nullptr)).count());
+ // TODO: consumer->set_consumer_arn();
+ consumer->set_consumer_status(Ydb::DataStreams::V1::ConsumerDescription_ConsumerStatus_ACTIVE);
+ Request_->SendResult(result, Ydb::StatusIds::SUCCESS);
+ Die(ctx);
+ }
+
+ //-----------------------------------------------------------------------------------------
+
+ class TDeregisterStreamConsumerActor : public TUpdateSchemeActor<TDeregisterStreamConsumerActor, NKikimr::NGRpcService::TEvDataStreamsDeregisterStreamConsumerRequest> {
+ using TBase = TUpdateSchemeActor<TDeregisterStreamConsumerActor, TEvDataStreamsDeregisterStreamConsumerRequest>;
+
+ public:
+ TDeregisterStreamConsumerActor(NKikimr::NGRpcService::TEvDataStreamsDeregisterStreamConsumerRequest* request);
+ ~TDeregisterStreamConsumerActor() = default;
+
+ void Bootstrap(const NActors::TActorContext& ctx);
+ void ModifyPersqueueConfig(const TActorContext& ctx,
NKikimrSchemeOp::TPersQueueGroupDescription& groupConfig,
const NKikimrSchemeOp::TPersQueueGroupDescription& pqGroupDescription,
const NKikimrSchemeOp::TDirEntry& selfInfo);
-
- private:
- TString ConsumerName;
- };
- TDeregisterStreamConsumerActor::TDeregisterStreamConsumerActor(NKikimr::NGRpcService::TEvDataStreamsDeregisterStreamConsumerRequest* request)
- : TBase(request, request->GetProtoRequest()->stream_arn())
- , ConsumerName(request->GetProtoRequest()->consumer_name())
- {
- }
-
- void TDeregisterStreamConsumerActor::Bootstrap(const NActors::TActorContext& ctx) {
- TBase::Bootstrap(ctx);
- SendDescribeProposeRequest(ctx);
- Become(&TDeregisterStreamConsumerActor::StateWork);
- }
-
+
+ private:
+ TString ConsumerName;
+ };
+ TDeregisterStreamConsumerActor::TDeregisterStreamConsumerActor(NKikimr::NGRpcService::TEvDataStreamsDeregisterStreamConsumerRequest* request)
+ : TBase(request, request->GetProtoRequest()->stream_arn())
+ , ConsumerName(request->GetProtoRequest()->consumer_name())
+ {
+ }
+
+ void TDeregisterStreamConsumerActor::Bootstrap(const NActors::TActorContext& ctx) {
+ TBase::Bootstrap(ctx);
+ SendDescribeProposeRequest(ctx);
+ Become(&TDeregisterStreamConsumerActor::StateWork);
+ }
+
void TDeregisterStreamConsumerActor::ModifyPersqueueConfig(
const TActorContext& ctx,
NKikimrSchemeOp::TPersQueueGroupDescription& groupConfig,
@@ -980,683 +980,683 @@ namespace NKikimr::NDataStreams::V1 {
GetProtoRequest()->consumer_name(),
ctx
);
- if (!error.Empty()) {
- return ReplyWithError(Ydb::StatusIds::NOT_FOUND, Ydb::PersQueue::ErrorCode::BAD_REQUEST, error, ctx);
- }
- }
-
- //-----------------------------------------------------------------------------------------
-
- class TGetShardIteratorActor : public TPQGrpcSchemaBase<TGetShardIteratorActor, NKikimr::NGRpcService::TEvDataStreamsGetShardIteratorRequest> {
- using TBase = TPQGrpcSchemaBase<TGetShardIteratorActor, TEvDataStreamsGetShardIteratorRequest>;
-
- public:
- TGetShardIteratorActor(NKikimr::NGRpcService::TEvDataStreamsGetShardIteratorRequest* request, NActors::TActorId newSchemeCache);
- ~TGetShardIteratorActor() = default;
-
- void Bootstrap(const NActors::TActorContext& ctx);
- void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx);
- void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx);
-
-
- private:
- using TIteratorType = Ydb::DataStreams::V1::ShardIteratorType;
-
- void SendResponse(const TActorContext& ctx, const TShardIterator& shardIt);
- std::optional<ui32> SequenceNumberToInt(const TString& sequenceNumberStr);
-
- TActorId NewSchemeCache;
- TString StreamName;
- TString ShardId;
- TIteratorType IteratorType;
- ui32 SequenceNumber;
- ui64 ReadTimestampMs;
- };
-
- TGetShardIteratorActor::TGetShardIteratorActor(NKikimr::NGRpcService::TEvDataStreamsGetShardIteratorRequest* request, NActors::TActorId newSchemeCache)
- : TBase(request, request->GetProtoRequest()->stream_name())
- , NewSchemeCache(std::move(newSchemeCache))
- , StreamName{request->GetProtoRequest()->stream_name()}
- , ShardId{request->GetProtoRequest()->shard_id()}
- , IteratorType{request->GetProtoRequest()->shard_iterator_type()}
- , SequenceNumber{0}
- , ReadTimestampMs{0}
- {
- }
-
- void TGetShardIteratorActor::Bootstrap(const NActors::TActorContext& ctx) {
- TBase::Bootstrap(ctx);
-
- switch (IteratorType) {
- case TIteratorType::AFTER_SEQUENCE_NUMBER:
- case TIteratorType::AT_SEQUENCE_NUMBER: {
- auto sn = SequenceNumberToInt(GetProtoRequest()->starting_sequence_number());
- if (!sn) {
- return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::BAD_REQUEST,
- TStringBuilder() << "Malformed sequence number", ctx);
- }
- SequenceNumber = sn.value() + (IteratorType == TIteratorType::AFTER_SEQUENCE_NUMBER ? 1u : 0u);
- }
- break;
- case TIteratorType::AT_TIMESTAMP:
- if (GetProtoRequest()->timestamp() == 0 ||
- GetProtoRequest()->timestamp() > static_cast<i64>(TInstant::Now().MilliSeconds())) {
- return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::BAD_REQUEST,
- TStringBuilder() << "Shard iterator type is AT_TIMESTAMP, " <<
- "but timestamp is either missed or too old or in future", ctx);
- }
- ReadTimestampMs = GetProtoRequest()->timestamp();
- break;
- case TIteratorType::TRIM_HORIZON:
- ReadTimestampMs = 0;
- break;
- case TIteratorType::LATEST:
- ReadTimestampMs = TInstant::Now().MilliSeconds();
- break;
- default:
- return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::BAD_REQUEST,
- TStringBuilder() << "Shard iterator type '" <<
- (ui32)IteratorType << "' is not known", ctx);
-
- }
-
- SendDescribeProposeRequest(ctx);
- Become(&TGetShardIteratorActor::StateWork);
- }
-
- void TGetShardIteratorActor::StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) {
- switch (ev->GetTypeRewrite()) {
- default: TBase::StateWork(ev, ctx);
- }
- }
-
- void TGetShardIteratorActor::HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) {
- if (ReplyIfNotTopic(ev, ctx)) {
- return;
- }
-
- const NSchemeCache::TSchemeCacheNavigate* navigate = ev->Get()->Request.Get();
- auto topicInfo = navigate->ResultSet.begin();
- StreamName = NKikimr::CanonizePath(topicInfo->Path);
- if (AppData(ctx)->PQConfig.GetRequireCredentialsInNewProtocol()) {
- if (!topicInfo->SecurityObject->CheckAccess(NACLib::EAccessRights::SelectRow,
- this->Request_->GetInternalToken())) {
- return this->ReplyWithError(Ydb::StatusIds::UNAUTHORIZED,
- Ydb::PersQueue::ErrorCode::ACCESS_DENIED,
- TStringBuilder() << "Access to stream "
- << this->GetProtoRequest()->stream_name()
- << " is denied for subject "
- << this->Request_->GetInternalToken(), ctx);
- }
- }
-
- const auto& partitions = topicInfo->PQGroupInfo->Description.GetPartitions();
- for (auto& partition : partitions) {
- auto partitionId = partition.GetPartitionId();
- TString shardName = GetShardName(partitionId);
- if (shardName == ShardId) {
- TShardIterator it(StreamName, StreamName, partitionId, ReadTimestampMs, SequenceNumber);
- SendResponse(ctx, it);
- return;
- }
- }
-
- ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::ERROR,
- TStringBuilder() << "No such shard: " << ShardId, ctx);
- }
-
- void TGetShardIteratorActor::SendResponse(const TActorContext& ctx, const TShardIterator& shardIt) {
- Ydb::DataStreams::V1::GetShardIteratorResult result;
- result.set_shard_iterator(shardIt.Serialize());
- Request_->SendResult(result, Ydb::StatusIds::SUCCESS);
- Die(ctx);
- }
-
- std::optional<ui32> TGetShardIteratorActor::SequenceNumberToInt(const TString& sequenceNumberStr) {
- try {
- return std::stoi(sequenceNumberStr.c_str());
- } catch(...) {
- return std::nullopt;
- }
- }
-
- //-----------------------------------------------------------------------------------
-
- class TGetRecordsActor : public TPQGrpcSchemaBase<TGetRecordsActor, TEvDataStreamsGetRecordsRequest> {
- using TBase = TPQGrpcSchemaBase<TGetRecordsActor, TEvDataStreamsGetRecordsRequest>;
-
- static constexpr ui32 READ_TIMEOUT_MS = 150;
- static constexpr i32 MAX_LIMIT = 10000;
-
- public:
- TGetRecordsActor(TEvDataStreamsGetRecordsRequest* request, const TActorId& newSchemeCache);
- ~TGetRecordsActor() = default;
-
- void Bootstrap(const NActors::TActorContext& ctx);
- void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx);
- void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx);
- void Handle(TEvPersQueue::TEvResponse::TPtr& ev, const TActorContext& ctx);
-
- private:
- void SendReadRequest(const TActorContext& ctx);
- void SendResponse(const TActorContext& ctx,
- const std::vector<Ydb::DataStreams::V1::Record>& records,
- ui64 millisBehindLatestMs);
-
- TShardIterator ShardIterator;
- TString StreamName;
- ui64 TabletId;
- i32 Limit;
- TActorId NewSchemeCache;
- };
-
- TGetRecordsActor::TGetRecordsActor(TEvDataStreamsGetRecordsRequest* request,
- const TActorId& newSchemeCache)
- : TBase(request, TShardIterator(request->GetProtoRequest()->shard_iterator()).IsValid()
- ? TShardIterator(request->GetProtoRequest()->shard_iterator()).GetStreamName()
- : "undefined")
- , ShardIterator{request->GetProtoRequest()->shard_iterator()}
- , StreamName{ShardIterator.IsValid() ? ShardIterator.GetStreamName() : "undefined"}
- , TabletId{0}
- , Limit{request->GetProtoRequest()->limit()}
- , NewSchemeCache{newSchemeCache}
- {
- }
-
- void TGetRecordsActor::Bootstrap(const NActors::TActorContext& ctx) {
- TBase::Bootstrap(ctx);
-
- if (!ShardIterator.IsValid()) {
- return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::ERROR,
- TStringBuilder() << "Provided shard iterator is malformed or expired", ctx);
- }
-
- Limit = Limit == 0 ? MAX_LIMIT : Limit;
- if (Limit < 1 || Limit > MAX_LIMIT) {
- return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::ERROR,
- TStringBuilder() << "Limit '" << Limit << "' is out of bounds [1; " << MAX_LIMIT << "]", ctx);
- }
-
- SendDescribeProposeRequest(ctx);
- Become(&TGetRecordsActor::StateWork);
- }
-
- void TGetRecordsActor::SendReadRequest(const TActorContext& ctx) {
- NTabletPipe::TClientConfig clientConfig;
- clientConfig.RetryPolicy = {
- .RetryLimitCount = 6,
- .MinRetryTime = TDuration::MilliSeconds(10),
- .MaxRetryTime = TDuration::MilliSeconds(100),
- .BackoffMultiplier = 2,
- .DoFirstRetryInstantly = true
- };
- auto PipeClient = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, TabletId, clientConfig));
-
- NKikimrClient::TPersQueueRequest request;
- request.MutablePartitionRequest()->SetTopic(this->GetTopicPath(ctx));
- request.MutablePartitionRequest()->SetPartition(ShardIterator.GetShardId());
- ActorIdToProto(PipeClient, request.MutablePartitionRequest()->MutablePipeClient());
-
- auto cmdRead = request.MutablePartitionRequest()->MutableCmdRead();
- cmdRead->SetClientId(NKikimr::NPQ::CLIENTID_TO_READ_INTERNALLY);
- cmdRead->SetCount(Limit);
- cmdRead->SetOffset(ShardIterator.GetSequenceNumber());
- cmdRead->SetReadTimestampMs(ShardIterator.GetReadTimestamp());
- cmdRead->SetTimeoutMs(READ_TIMEOUT_MS);
- cmdRead->SetExternalOperation(true);
-
- TAutoPtr<TEvPersQueue::TEvRequest> req(new TEvPersQueue::TEvRequest);
- req->Record.Swap(&request);
- NTabletPipe::SendData(ctx, PipeClient, req.Release());
- }
-
- void TGetRecordsActor::StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) {
- switch (ev->GetTypeRewrite()) {
- HFunc(TEvPersQueue::TEvResponse, Handle);
- default: TBase::StateWork(ev, ctx);
- }
- }
-
- void TGetRecordsActor::HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev,
- const TActorContext& ctx) {
- const auto &result = ev->Get()->Request.Get();
- const auto response = result->ResultSet.front();
-
- if (AppData(ctx)->PQConfig.GetRequireCredentialsInNewProtocol()) {
- if (!response.SecurityObject->CheckAccess(NACLib::EAccessRights::SelectRow,
- this->Request_->GetInternalToken())) {
- return ReplyWithError(Ydb::StatusIds::UNAUTHORIZED,
- Ydb::PersQueue::ErrorCode::ACCESS_DENIED,
- TStringBuilder() << "Access to stream "
- << ShardIterator.GetStreamName()
- << " is denied for subject "
- << this->Request_->GetInternalToken(), ctx);
- }
- }
-
-
+ if (!error.Empty()) {
+ return ReplyWithError(Ydb::StatusIds::NOT_FOUND, Ydb::PersQueue::ErrorCode::BAD_REQUEST, error, ctx);
+ }
+ }
+
+ //-----------------------------------------------------------------------------------------
+
+ class TGetShardIteratorActor : public TPQGrpcSchemaBase<TGetShardIteratorActor, NKikimr::NGRpcService::TEvDataStreamsGetShardIteratorRequest> {
+ using TBase = TPQGrpcSchemaBase<TGetShardIteratorActor, TEvDataStreamsGetShardIteratorRequest>;
+
+ public:
+ TGetShardIteratorActor(NKikimr::NGRpcService::TEvDataStreamsGetShardIteratorRequest* request, NActors::TActorId newSchemeCache);
+ ~TGetShardIteratorActor() = default;
+
+ void Bootstrap(const NActors::TActorContext& ctx);
+ void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx);
+ void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx);
+
+
+ private:
+ using TIteratorType = Ydb::DataStreams::V1::ShardIteratorType;
+
+ void SendResponse(const TActorContext& ctx, const TShardIterator& shardIt);
+ std::optional<ui32> SequenceNumberToInt(const TString& sequenceNumberStr);
+
+ TActorId NewSchemeCache;
+ TString StreamName;
+ TString ShardId;
+ TIteratorType IteratorType;
+ ui32 SequenceNumber;
+ ui64 ReadTimestampMs;
+ };
+
+ TGetShardIteratorActor::TGetShardIteratorActor(NKikimr::NGRpcService::TEvDataStreamsGetShardIteratorRequest* request, NActors::TActorId newSchemeCache)
+ : TBase(request, request->GetProtoRequest()->stream_name())
+ , NewSchemeCache(std::move(newSchemeCache))
+ , StreamName{request->GetProtoRequest()->stream_name()}
+ , ShardId{request->GetProtoRequest()->shard_id()}
+ , IteratorType{request->GetProtoRequest()->shard_iterator_type()}
+ , SequenceNumber{0}
+ , ReadTimestampMs{0}
+ {
+ }
+
+ void TGetShardIteratorActor::Bootstrap(const NActors::TActorContext& ctx) {
+ TBase::Bootstrap(ctx);
+
+ switch (IteratorType) {
+ case TIteratorType::AFTER_SEQUENCE_NUMBER:
+ case TIteratorType::AT_SEQUENCE_NUMBER: {
+ auto sn = SequenceNumberToInt(GetProtoRequest()->starting_sequence_number());
+ if (!sn) {
+ return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::BAD_REQUEST,
+ TStringBuilder() << "Malformed sequence number", ctx);
+ }
+ SequenceNumber = sn.value() + (IteratorType == TIteratorType::AFTER_SEQUENCE_NUMBER ? 1u : 0u);
+ }
+ break;
+ case TIteratorType::AT_TIMESTAMP:
+ if (GetProtoRequest()->timestamp() == 0 ||
+ GetProtoRequest()->timestamp() > static_cast<i64>(TInstant::Now().MilliSeconds())) {
+ return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::BAD_REQUEST,
+ TStringBuilder() << "Shard iterator type is AT_TIMESTAMP, " <<
+ "but timestamp is either missed or too old or in future", ctx);
+ }
+ ReadTimestampMs = GetProtoRequest()->timestamp();
+ break;
+ case TIteratorType::TRIM_HORIZON:
+ ReadTimestampMs = 0;
+ break;
+ case TIteratorType::LATEST:
+ ReadTimestampMs = TInstant::Now().MilliSeconds();
+ break;
+ default:
+ return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::BAD_REQUEST,
+ TStringBuilder() << "Shard iterator type '" <<
+ (ui32)IteratorType << "' is not known", ctx);
+
+ }
+
+ SendDescribeProposeRequest(ctx);
+ Become(&TGetShardIteratorActor::StateWork);
+ }
+
+ void TGetShardIteratorActor::StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) {
+ switch (ev->GetTypeRewrite()) {
+ default: TBase::StateWork(ev, ctx);
+ }
+ }
+
+ void TGetShardIteratorActor::HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) {
+ if (ReplyIfNotTopic(ev, ctx)) {
+ return;
+ }
+
+ const NSchemeCache::TSchemeCacheNavigate* navigate = ev->Get()->Request.Get();
+ auto topicInfo = navigate->ResultSet.begin();
+ StreamName = NKikimr::CanonizePath(topicInfo->Path);
+ if (AppData(ctx)->PQConfig.GetRequireCredentialsInNewProtocol()) {
+ if (!topicInfo->SecurityObject->CheckAccess(NACLib::EAccessRights::SelectRow,
+ this->Request_->GetInternalToken())) {
+ return this->ReplyWithError(Ydb::StatusIds::UNAUTHORIZED,
+ Ydb::PersQueue::ErrorCode::ACCESS_DENIED,
+ TStringBuilder() << "Access to stream "
+ << this->GetProtoRequest()->stream_name()
+ << " is denied for subject "
+ << this->Request_->GetInternalToken(), ctx);
+ }
+ }
+
+ const auto& partitions = topicInfo->PQGroupInfo->Description.GetPartitions();
+ for (auto& partition : partitions) {
+ auto partitionId = partition.GetPartitionId();
+ TString shardName = GetShardName(partitionId);
+ if (shardName == ShardId) {
+ TShardIterator it(StreamName, StreamName, partitionId, ReadTimestampMs, SequenceNumber);
+ SendResponse(ctx, it);
+ return;
+ }
+ }
+
+ ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::ERROR,
+ TStringBuilder() << "No such shard: " << ShardId, ctx);
+ }
+
+ void TGetShardIteratorActor::SendResponse(const TActorContext& ctx, const TShardIterator& shardIt) {
+ Ydb::DataStreams::V1::GetShardIteratorResult result;
+ result.set_shard_iterator(shardIt.Serialize());
+ Request_->SendResult(result, Ydb::StatusIds::SUCCESS);
+ Die(ctx);
+ }
+
+ std::optional<ui32> TGetShardIteratorActor::SequenceNumberToInt(const TString& sequenceNumberStr) {
+ try {
+ return std::stoi(sequenceNumberStr.c_str());
+ } catch(...) {
+ return std::nullopt;
+ }
+ }
+
+ //-----------------------------------------------------------------------------------
+
+ class TGetRecordsActor : public TPQGrpcSchemaBase<TGetRecordsActor, TEvDataStreamsGetRecordsRequest> {
+ using TBase = TPQGrpcSchemaBase<TGetRecordsActor, TEvDataStreamsGetRecordsRequest>;
+
+ static constexpr ui32 READ_TIMEOUT_MS = 150;
+ static constexpr i32 MAX_LIMIT = 10000;
+
+ public:
+ TGetRecordsActor(TEvDataStreamsGetRecordsRequest* request, const TActorId& newSchemeCache);
+ ~TGetRecordsActor() = default;
+
+ void Bootstrap(const NActors::TActorContext& ctx);
+ void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx);
+ void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx);
+ void Handle(TEvPersQueue::TEvResponse::TPtr& ev, const TActorContext& ctx);
+
+ private:
+ void SendReadRequest(const TActorContext& ctx);
+ void SendResponse(const TActorContext& ctx,
+ const std::vector<Ydb::DataStreams::V1::Record>& records,
+ ui64 millisBehindLatestMs);
+
+ TShardIterator ShardIterator;
+ TString StreamName;
+ ui64 TabletId;
+ i32 Limit;
+ TActorId NewSchemeCache;
+ };
+
+ TGetRecordsActor::TGetRecordsActor(TEvDataStreamsGetRecordsRequest* request,
+ const TActorId& newSchemeCache)
+ : TBase(request, TShardIterator(request->GetProtoRequest()->shard_iterator()).IsValid()
+ ? TShardIterator(request->GetProtoRequest()->shard_iterator()).GetStreamName()
+ : "undefined")
+ , ShardIterator{request->GetProtoRequest()->shard_iterator()}
+ , StreamName{ShardIterator.IsValid() ? ShardIterator.GetStreamName() : "undefined"}
+ , TabletId{0}
+ , Limit{request->GetProtoRequest()->limit()}
+ , NewSchemeCache{newSchemeCache}
+ {
+ }
+
+ void TGetRecordsActor::Bootstrap(const NActors::TActorContext& ctx) {
+ TBase::Bootstrap(ctx);
+
+ if (!ShardIterator.IsValid()) {
+ return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::ERROR,
+ TStringBuilder() << "Provided shard iterator is malformed or expired", ctx);
+ }
+
+ Limit = Limit == 0 ? MAX_LIMIT : Limit;
+ if (Limit < 1 || Limit > MAX_LIMIT) {
+ return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::ERROR,
+ TStringBuilder() << "Limit '" << Limit << "' is out of bounds [1; " << MAX_LIMIT << "]", ctx);
+ }
+
+ SendDescribeProposeRequest(ctx);
+ Become(&TGetRecordsActor::StateWork);
+ }
+
+ void TGetRecordsActor::SendReadRequest(const TActorContext& ctx) {
+ NTabletPipe::TClientConfig clientConfig;
+ clientConfig.RetryPolicy = {
+ .RetryLimitCount = 6,
+ .MinRetryTime = TDuration::MilliSeconds(10),
+ .MaxRetryTime = TDuration::MilliSeconds(100),
+ .BackoffMultiplier = 2,
+ .DoFirstRetryInstantly = true
+ };
+ auto PipeClient = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, TabletId, clientConfig));
+
+ NKikimrClient::TPersQueueRequest request;
+ request.MutablePartitionRequest()->SetTopic(this->GetTopicPath(ctx));
+ request.MutablePartitionRequest()->SetPartition(ShardIterator.GetShardId());
+ ActorIdToProto(PipeClient, request.MutablePartitionRequest()->MutablePipeClient());
+
+ auto cmdRead = request.MutablePartitionRequest()->MutableCmdRead();
+ cmdRead->SetClientId(NKikimr::NPQ::CLIENTID_TO_READ_INTERNALLY);
+ cmdRead->SetCount(Limit);
+ cmdRead->SetOffset(ShardIterator.GetSequenceNumber());
+ cmdRead->SetReadTimestampMs(ShardIterator.GetReadTimestamp());
+ cmdRead->SetTimeoutMs(READ_TIMEOUT_MS);
+ cmdRead->SetExternalOperation(true);
+
+ TAutoPtr<TEvPersQueue::TEvRequest> req(new TEvPersQueue::TEvRequest);
+ req->Record.Swap(&request);
+ NTabletPipe::SendData(ctx, PipeClient, req.Release());
+ }
+
+ void TGetRecordsActor::StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) {
+ switch (ev->GetTypeRewrite()) {
+ HFunc(TEvPersQueue::TEvResponse, Handle);
+ default: TBase::StateWork(ev, ctx);
+ }
+ }
+
+ void TGetRecordsActor::HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev,
+ const TActorContext& ctx) {
+ const auto &result = ev->Get()->Request.Get();
+ const auto response = result->ResultSet.front();
+
+ if (AppData(ctx)->PQConfig.GetRequireCredentialsInNewProtocol()) {
+ if (!response.SecurityObject->CheckAccess(NACLib::EAccessRights::SelectRow,
+ this->Request_->GetInternalToken())) {
+ return ReplyWithError(Ydb::StatusIds::UNAUTHORIZED,
+ Ydb::PersQueue::ErrorCode::ACCESS_DENIED,
+ TStringBuilder() << "Access to stream "
+ << ShardIterator.GetStreamName()
+ << " is denied for subject "
+ << this->Request_->GetInternalToken(), ctx);
+ }
+ }
+
+
if (response.Self->Info.GetPathType() == NKikimrSchemeOp::EPathTypePersQueueGroup) {
- const auto& partitions = response.PQGroupInfo->Description.GetPartitions();
- for (auto& partition : partitions) {
- auto partitionId = partition.GetPartitionId();
- if (partitionId == ShardIterator.GetShardId()) {
- TabletId = partition.GetTabletId();
- return SendReadRequest(ctx);
- }
- }
- }
-
- ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::ERROR,
- TStringBuilder() << "No such shard: " << ShardIterator.GetShardId(), ctx);
- }
-
- void TGetRecordsActor::Handle(TEvPersQueue::TEvResponse::TPtr& ev, const TActorContext& ctx) {
- const auto& record = ev->Get()->Record;
- Y_VERIFY(ev->Get()->Record.HasPartitionResponse());
-
- ui64 millisBehindLatestMs = 0;
- std::vector<Ydb::DataStreams::V1::Record> records;
- const auto& response = record.GetPartitionResponse();
- if (response.HasCmdReadResult()) {
- const auto& results = response.GetCmdReadResult().GetResult();
- records.reserve(results.size());
- for (auto& r : results) {
- auto proto(NKikimr::GetDeserializedData(r.GetData()));
- Ydb::DataStreams::V1::Record record;
- record.set_data(proto.GetData());
- record.set_timestamp(r.GetCreateTimestampMS());
- record.set_encryption(Ydb::DataStreams::V1::EncryptionType::NONE);
- record.set_partition_key(r.GetPartitionKey());
- record.set_sequence_number(std::to_string(r.GetOffset()).c_str());
- records.push_back(record);
- }
- millisBehindLatestMs = records.size() > 0 ? TInstant::Now().MilliSeconds() - results.rbegin()->GetWriteTimestampMS() : 0;
- }
-
- SendResponse(ctx, records, millisBehindLatestMs);
- }
-
- void TGetRecordsActor::SendResponse(const TActorContext& ctx,
- const std::vector<Ydb::DataStreams::V1::Record>& records,
- ui64 millisBehindLatestMs) {
- Ydb::DataStreams::V1::GetRecordsResult result;
- for (auto& r : records) {
- auto record = result.add_records();
- *record = r;
- }
-
- auto timestamp = records.size() > 0 ? records.back().Gettimestamp() + 1
- : ShardIterator.GetReadTimestamp();
- auto seqNo = records.size() > 0 ? std::stoi(records.back().Getsequence_number()) + 1
- : ShardIterator.GetSequenceNumber();
- TShardIterator shardIterator(ShardIterator.GetStreamName(),
- ShardIterator.GetStreamArn(),
- ShardIterator.GetShardId(),
- timestamp, seqNo);
- result.set_next_shard_iterator(shardIterator.Serialize());
- result.set_millis_behind_latest(millisBehindLatestMs);
-
- Request_->SendResult(result, Ydb::StatusIds::SUCCESS);
- Die(ctx);
- }
-
- //-----------------------------------------------------------------------------------------
-
- class TListShardsActor : public TPQGrpcSchemaBase<TListShardsActor, NKikimr::NGRpcService::TEvDataStreamsListShardsRequest> {
- using TBase = TPQGrpcSchemaBase<TListShardsActor, TEvDataStreamsListShardsRequest>;
-
- public:
- TListShardsActor(NKikimr::NGRpcService::TEvDataStreamsListShardsRequest* request, NActors::TActorId newSchemeCache);
- ~TListShardsActor() = default;
-
- void Bootstrap(const NActors::TActorContext& ctx);
- void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx);
- void Handle(TEvPersQueue::TEvOffsetsResponse::TPtr& ev, const TActorContext& ctx);
- void Handle(TEvTabletPipe::TEvClientConnected::TPtr& ev, const TActorContext& ctx);
- void Handle(TEvTabletPipe::TEvClientDestroyed::TPtr& ev, const TActorContext& ctx);
- void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev,
- const TActorContext& ctx);
- void Die(const TActorContext& ctx) override;
-
- private:
- using TShardFilter = Ydb::DataStreams::V1::ShardFilter;
-
- void SendResponse(const TActorContext& ctx);
-
- static constexpr ui32 MAX_MAX_RESULTS = 10000;
- static constexpr ui32 MIN_MAX_RESULTS = 1;
- static constexpr ui32 DEFAULT_MAX_RESULTS = 100;
-
- TActorId NewSchemeCache;
- TString StreamName;
- TShardFilter ShardFilter;
- TNextToken NextToken;
- ui32 MaxResults = DEFAULT_MAX_RESULTS;
- std::map<ui64, std::pair<ui64, ui64>> StartEndOffsetsPerPartition;
+ const auto& partitions = response.PQGroupInfo->Description.GetPartitions();
+ for (auto& partition : partitions) {
+ auto partitionId = partition.GetPartitionId();
+ if (partitionId == ShardIterator.GetShardId()) {
+ TabletId = partition.GetTabletId();
+ return SendReadRequest(ctx);
+ }
+ }
+ }
+
+ ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::ERROR,
+ TStringBuilder() << "No such shard: " << ShardIterator.GetShardId(), ctx);
+ }
+
+ void TGetRecordsActor::Handle(TEvPersQueue::TEvResponse::TPtr& ev, const TActorContext& ctx) {
+ const auto& record = ev->Get()->Record;
+ Y_VERIFY(ev->Get()->Record.HasPartitionResponse());
+
+ ui64 millisBehindLatestMs = 0;
+ std::vector<Ydb::DataStreams::V1::Record> records;
+ const auto& response = record.GetPartitionResponse();
+ if (response.HasCmdReadResult()) {
+ const auto& results = response.GetCmdReadResult().GetResult();
+ records.reserve(results.size());
+ for (auto& r : results) {
+ auto proto(NKikimr::GetDeserializedData(r.GetData()));
+ Ydb::DataStreams::V1::Record record;
+ record.set_data(proto.GetData());
+ record.set_timestamp(r.GetCreateTimestampMS());
+ record.set_encryption(Ydb::DataStreams::V1::EncryptionType::NONE);
+ record.set_partition_key(r.GetPartitionKey());
+ record.set_sequence_number(std::to_string(r.GetOffset()).c_str());
+ records.push_back(record);
+ }
+ millisBehindLatestMs = records.size() > 0 ? TInstant::Now().MilliSeconds() - results.rbegin()->GetWriteTimestampMS() : 0;
+ }
+
+ SendResponse(ctx, records, millisBehindLatestMs);
+ }
+
+ void TGetRecordsActor::SendResponse(const TActorContext& ctx,
+ const std::vector<Ydb::DataStreams::V1::Record>& records,
+ ui64 millisBehindLatestMs) {
+ Ydb::DataStreams::V1::GetRecordsResult result;
+ for (auto& r : records) {
+ auto record = result.add_records();
+ *record = r;
+ }
+
+ auto timestamp = records.size() > 0 ? records.back().Gettimestamp() + 1
+ : ShardIterator.GetReadTimestamp();
+ auto seqNo = records.size() > 0 ? std::stoi(records.back().Getsequence_number()) + 1
+ : ShardIterator.GetSequenceNumber();
+ TShardIterator shardIterator(ShardIterator.GetStreamName(),
+ ShardIterator.GetStreamArn(),
+ ShardIterator.GetShardId(),
+ timestamp, seqNo);
+ result.set_next_shard_iterator(shardIterator.Serialize());
+ result.set_millis_behind_latest(millisBehindLatestMs);
+
+ Request_->SendResult(result, Ydb::StatusIds::SUCCESS);
+ Die(ctx);
+ }
+
+ //-----------------------------------------------------------------------------------------
+
+ class TListShardsActor : public TPQGrpcSchemaBase<TListShardsActor, NKikimr::NGRpcService::TEvDataStreamsListShardsRequest> {
+ using TBase = TPQGrpcSchemaBase<TListShardsActor, TEvDataStreamsListShardsRequest>;
+
+ public:
+ TListShardsActor(NKikimr::NGRpcService::TEvDataStreamsListShardsRequest* request, NActors::TActorId newSchemeCache);
+ ~TListShardsActor() = default;
+
+ void Bootstrap(const NActors::TActorContext& ctx);
+ void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx);
+ void Handle(TEvPersQueue::TEvOffsetsResponse::TPtr& ev, const TActorContext& ctx);
+ void Handle(TEvTabletPipe::TEvClientConnected::TPtr& ev, const TActorContext& ctx);
+ void Handle(TEvTabletPipe::TEvClientDestroyed::TPtr& ev, const TActorContext& ctx);
+ void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev,
+ const TActorContext& ctx);
+ void Die(const TActorContext& ctx) override;
+
+ private:
+ using TShardFilter = Ydb::DataStreams::V1::ShardFilter;
+
+ void SendResponse(const TActorContext& ctx);
+
+ static constexpr ui32 MAX_MAX_RESULTS = 10000;
+ static constexpr ui32 MIN_MAX_RESULTS = 1;
+ static constexpr ui32 DEFAULT_MAX_RESULTS = 100;
+
+ TActorId NewSchemeCache;
+ TString StreamName;
+ TShardFilter ShardFilter;
+ TNextToken NextToken;
+ ui32 MaxResults = DEFAULT_MAX_RESULTS;
+ std::map<ui64, std::pair<ui64, ui64>> StartEndOffsetsPerPartition;
std::vector<NKikimrSchemeOp::TPersQueueGroupDescription::TPartition> Shards;
ui32 LeftToRead = 0;
ui32 AllShardsCount = 0;
- std::atomic<ui32> GotOffsetResponds;
- std::vector<TActorId> Pipes;
- };
-
- TListShardsActor::TListShardsActor(NKikimr::NGRpcService::TEvDataStreamsListShardsRequest* request, NActors::TActorId newSchemeCache)
- : TBase(request, request->GetProtoRequest()->stream_name())
- , NewSchemeCache(std::move(newSchemeCache))
- , StreamName{request->GetProtoRequest()->stream_name()}
- , ShardFilter{request->GetProtoRequest()->shard_filter()}
- , NextToken{request->GetProtoRequest()->next_token()}
- , GotOffsetResponds{0}
- {
- if (request->GetProtoRequest()->next_token().empty()) {
- StreamName = request->GetProtoRequest()->stream_name();
- MaxResults = request->GetProtoRequest()->max_results();
- NextToken = TNextToken(StreamName, 0, MaxResults, TInstant::Now().MilliSeconds());
- } else {
- StreamName = NextToken.GetStreamArn();
- MaxResults = NextToken.GetMaxResults();
- }
- }
-
- void TListShardsActor::Bootstrap(const NActors::TActorContext& ctx) {
- TBase::Bootstrap(ctx);
-
- if (!TShardFilter::ShardFilterType_IsValid(ShardFilter.type())) {
- return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::BAD_REQUEST,
- TStringBuilder() << "Shard filter '" <<
- (ui32)ShardFilter.type() << "' is not known", ctx);
- }
-
- MaxResults = MaxResults == 0 ? DEFAULT_MAX_RESULTS : MaxResults;
+ std::atomic<ui32> GotOffsetResponds;
+ std::vector<TActorId> Pipes;
+ };
+
+ TListShardsActor::TListShardsActor(NKikimr::NGRpcService::TEvDataStreamsListShardsRequest* request, NActors::TActorId newSchemeCache)
+ : TBase(request, request->GetProtoRequest()->stream_name())
+ , NewSchemeCache(std::move(newSchemeCache))
+ , StreamName{request->GetProtoRequest()->stream_name()}
+ , ShardFilter{request->GetProtoRequest()->shard_filter()}
+ , NextToken{request->GetProtoRequest()->next_token()}
+ , GotOffsetResponds{0}
+ {
+ if (request->GetProtoRequest()->next_token().empty()) {
+ StreamName = request->GetProtoRequest()->stream_name();
+ MaxResults = request->GetProtoRequest()->max_results();
+ NextToken = TNextToken(StreamName, 0, MaxResults, TInstant::Now().MilliSeconds());
+ } else {
+ StreamName = NextToken.GetStreamArn();
+ MaxResults = NextToken.GetMaxResults();
+ }
+ }
+
+ void TListShardsActor::Bootstrap(const NActors::TActorContext& ctx) {
+ TBase::Bootstrap(ctx);
+
+ if (!TShardFilter::ShardFilterType_IsValid(ShardFilter.type())) {
+ return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::BAD_REQUEST,
+ TStringBuilder() << "Shard filter '" <<
+ (ui32)ShardFilter.type() << "' is not known", ctx);
+ }
+
+ MaxResults = MaxResults == 0 ? DEFAULT_MAX_RESULTS : MaxResults;
if (MaxResults > MAX_MAX_RESULTS) {
- return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::BAD_REQUEST,
- TStringBuilder() << "Max results '" << MaxResults <<
- "' is out of bound [" << MIN_MAX_RESULTS << "; " <<
- MAX_MAX_RESULTS << "]", ctx);
- }
-
- if (ShardFilter.type() == TShardFilter::AFTER_SHARD_ID && ShardFilter.shard_id() == "") {
- return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::BAD_REQUEST,
- TStringBuilder() << "Shard filter type is AFTER_SHARD_ID," <<
- " but no ShardId provided", ctx);
- }
-
- SendDescribeProposeRequest(ctx);
- Become(&TListShardsActor::StateWork);
- }
-
- void TListShardsActor::StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) {
- switch (ev->GetTypeRewrite()) {
- HFunc(TEvPersQueue::TEvOffsetsResponse, Handle);
- HFunc(TEvTabletPipe::TEvClientDestroyed, Handle);
- HFunc(TEvTabletPipe::TEvClientConnected, Handle);
- default: TBase::StateWork(ev, ctx);
- }
- }
-
- void TListShardsActor::HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) {
- if (ReplyIfNotTopic(ev, ctx)) {
- return;
- }
-
- const NSchemeCache::TSchemeCacheNavigate* navigate = ev->Get()->Request.Get();
- auto topicInfo = navigate->ResultSet.front();
- if (AppData(ctx)->PQConfig.GetRequireCredentialsInNewProtocol()) {
- if (!topicInfo.SecurityObject->CheckAccess(NACLib::EAccessRights::SelectRow,
- this->Request_->GetInternalToken())) {
- return this->ReplyWithError(Ydb::StatusIds::UNAUTHORIZED,
- Ydb::PersQueue::ErrorCode::ACCESS_DENIED,
- TStringBuilder() << "Access to stream "
- << this->GetProtoRequest()->stream_name()
- << " is denied for subject "
- << this->Request_->GetInternalToken(), ctx);
- }
- }
-
+ return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::BAD_REQUEST,
+ TStringBuilder() << "Max results '" << MaxResults <<
+ "' is out of bound [" << MIN_MAX_RESULTS << "; " <<
+ MAX_MAX_RESULTS << "]", ctx);
+ }
+
+ if (ShardFilter.type() == TShardFilter::AFTER_SHARD_ID && ShardFilter.shard_id() == "") {
+ return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::BAD_REQUEST,
+ TStringBuilder() << "Shard filter type is AFTER_SHARD_ID," <<
+ " but no ShardId provided", ctx);
+ }
+
+ SendDescribeProposeRequest(ctx);
+ Become(&TListShardsActor::StateWork);
+ }
+
+ void TListShardsActor::StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) {
+ switch (ev->GetTypeRewrite()) {
+ HFunc(TEvPersQueue::TEvOffsetsResponse, Handle);
+ HFunc(TEvTabletPipe::TEvClientDestroyed, Handle);
+ HFunc(TEvTabletPipe::TEvClientConnected, Handle);
+ default: TBase::StateWork(ev, ctx);
+ }
+ }
+
+ void TListShardsActor::HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) {
+ if (ReplyIfNotTopic(ev, ctx)) {
+ return;
+ }
+
+ const NSchemeCache::TSchemeCacheNavigate* navigate = ev->Get()->Request.Get();
+ auto topicInfo = navigate->ResultSet.front();
+ if (AppData(ctx)->PQConfig.GetRequireCredentialsInNewProtocol()) {
+ if (!topicInfo.SecurityObject->CheckAccess(NACLib::EAccessRights::SelectRow,
+ this->Request_->GetInternalToken())) {
+ return this->ReplyWithError(Ydb::StatusIds::UNAUTHORIZED,
+ Ydb::PersQueue::ErrorCode::ACCESS_DENIED,
+ TStringBuilder() << "Access to stream "
+ << this->GetProtoRequest()->stream_name()
+ << " is denied for subject "
+ << this->Request_->GetInternalToken(), ctx);
+ }
+ }
+
using TPartition = NKikimrSchemeOp::TPersQueueGroupDescription::TPartition;
- const auto& partitions = topicInfo.PQGroupInfo->Description.GetPartitions();
- TString startingShardId = this->GetProtoRequest()->Getexclusive_start_shard_id();
- ui64 startingTimepoint{0};
- bool onlyOpenShards{true};
-
- std::map<TShardFilter::ShardFilterType, std::function<bool(const TPartition&)>> filters = {
- {TShardFilter::SHARD_TYPE_UNDEFINED, [&](const TPartition& p) {
- onlyOpenShards = false;
- return GetShardName(p.GetPartitionId()) >= startingShardId;
- }},
- {TShardFilter::AFTER_SHARD_ID, [&](const TPartition& p) {
- startingShardId = ShardFilter.shard_id();
- startingTimepoint = 0;
- onlyOpenShards = false;
- return GetShardName(p.GetPartitionId()) > startingShardId;
- }},
- {TShardFilter::AT_TRIM_HORIZON, [&](const TPartition& p) {
- startingShardId = "0";
- startingTimepoint = 0;
- onlyOpenShards = true;
- return GetShardName(p.GetPartitionId()) >= startingShardId;
- }},
- { TShardFilter::FROM_TRIM_HORIZON, [&](const TPartition& p) {
- startingShardId = "0";
- startingTimepoint = 0;
- onlyOpenShards = false;
- return GetShardName(p.GetPartitionId()) >= startingShardId;
- }},
- {TShardFilter::AT_LATEST, [&](const TPartition& p) {
- startingTimepoint = TInstant::Now().MilliSeconds();
- startingShardId = "0";
- onlyOpenShards = true;
- return GetShardName(p.GetPartitionId()) >= startingShardId;
- }},
- {TShardFilter::AT_TIMESTAMP, [&](const TPartition& p) {
- startingTimepoint = ShardFilter.timestamp();
- startingShardId = "0";
- onlyOpenShards = true;
- return GetShardName(p.GetPartitionId()) >= startingShardId;
- }},
- {TShardFilter::FROM_TIMESTAMP, [&](const TPartition& p) {
- startingTimepoint = ShardFilter.timestamp();
- startingShardId = "0";
- onlyOpenShards = false;
- return GetShardName(p.GetPartitionId()) >= startingShardId;
- }}
- };
-
- const auto alreadyRead = NextToken.GetAlreadyRead();
- if (alreadyRead > (ui32)partitions.size()) {
- return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::ERROR,
- TStringBuilder() << "Provided next_token is malformed - "
- "everything is already read", ctx);
- }
-
- const auto shardsToGet = std::min(partitions.size() - alreadyRead, MaxResults);
- i32 lastCopied{0};
- Shards.reserve(shardsToGet);
- AllShardsCount = partitions.size();
- for (auto partition = partitions.begin() + alreadyRead; partition != partitions.end(); ++partition) {
- if (Shards.size() == shardsToGet) {
- break;
- }
- if (filters[ShardFilter.type()](*partition)) {
- Shards.push_back(*partition);
- }
- ++lastCopied;
- }
- auto actuallyRead = lastCopied - alreadyRead;
- LeftToRead = partitions.size() - alreadyRead - actuallyRead;
-
- if (Shards.size() == 0) {
- return SendResponse(ctx);
- }
-
- // Send OffsetRequests
- std::set<ui64> tabletIds;
- for (const auto& shard : Shards) {
- tabletIds.insert(shard.GetTabletId());
- }
-
- NTabletPipe::TClientConfig clientConfig;
- clientConfig.RetryPolicy = {
- .RetryLimitCount = 6,
- .MinRetryTime = TDuration::MilliSeconds(10),
- .MaxRetryTime = TDuration::MilliSeconds(100),
- .BackoffMultiplier = 2,
- .DoFirstRetryInstantly = true
- };
-
- for (auto& tabletId : tabletIds) {
- Pipes.push_back(ctx.Register(NTabletPipe::CreateClient(ctx.SelfID, tabletId, clientConfig)));
- TAutoPtr<TEvPersQueue::TEvOffsets> req(new TEvPersQueue::TEvOffsets);
- NTabletPipe::SendData(ctx, Pipes.back(), req.Release());
- }
- }
-
- void TListShardsActor::Handle(TEvPersQueue::TEvOffsetsResponse::TPtr& ev, const TActorContext& ctx) {
- for (auto& part : ev->Get()->Record.GetPartResult()) {
- StartEndOffsetsPerPartition[part.GetPartition()] =
- std::make_pair<ui64, ui64>(part.GetStartOffset(), part.GetEndOffset());
- ++GotOffsetResponds;
- }
- if (GotOffsetResponds == Shards.size()) {
- SendResponse(ctx);
- }
- }
-
- void TListShardsActor::Handle(TEvTabletPipe::TEvClientConnected::TPtr& ev, const TActorContext& ctx) {
- if (ev->Get()->Status != NKikimrProto::EReplyStatus::OK) {
- ReplyWithError(Ydb::StatusIds::SCHEME_ERROR, Ydb::PersQueue::ErrorCode::ERROR,
- TStringBuilder() << "Cannot connect to tablet " << ev->Get()->TabletId, ctx);
- }
- }
-
- void TListShardsActor::Handle(TEvTabletPipe::TEvClientDestroyed::TPtr& ev, const TActorContext& ctx) {
- ReplyWithError(Ydb::StatusIds::SCHEME_ERROR, Ydb::PersQueue::ErrorCode::ERROR,
- TStringBuilder() << "Cannot connect to tablet " << ev->Get()->TabletId, ctx);
- }
-
- void TListShardsActor::SendResponse(const TActorContext& ctx) {
- Ydb::DataStreams::V1::ListShardsResult result;
- for (auto& shard : Shards) {
- auto awsShard = result.Addshards();
- // TODO:
- // awsShard->set_parent_shard_id("");
- // awsShard->set_adjacent_parent_shard_id(prevShardName);
- auto range = RangeFromShardNumber(shard.GetPartitionId(), AllShardsCount);
- awsShard->mutable_hash_key_range()->set_starting_hash_key(
- Uint128ToDecimalString(range.Start));
- awsShard->mutable_hash_key_range()->set_ending_hash_key(
- Uint128ToDecimalString(range.End));
- awsShard->mutable_sequence_number_range()->set_starting_sequence_number(
- std::to_string(StartEndOffsetsPerPartition[shard.GetPartitionId()].first));
- awsShard->mutable_sequence_number_range()->set_ending_sequence_number(
- std::to_string(StartEndOffsetsPerPartition[shard.GetPartitionId()].second));
- awsShard->set_shard_id(GetShardName(shard.GetPartitionId()));
- }
- if (LeftToRead > 0) {
- TNextToken token(StreamName, NextToken.GetAlreadyRead() + Shards.size(), MaxResults, TInstant::Now().MilliSeconds());
- result.set_next_token(token.Serialize());
- }
- Request_->SendResult(result, Ydb::StatusIds::SUCCESS);
- Die(ctx);
- }
-
- void TListShardsActor::Die(const TActorContext& ctx) {
- //close all pipes
- for (auto& pipe : Pipes) {
- NTabletPipe::CloseClient(ctx, pipe);
- }
- TBase::Die(ctx);
- }
-
- //-----------------------------------------------------------------------------------
-
- class TDescribeStreamSummaryActor : public TPQGrpcSchemaBase<TDescribeStreamSummaryActor, TEvDataStreamsDescribeStreamSummaryRequest> {
- using TBase = TPQGrpcSchemaBase<TDescribeStreamSummaryActor, TEvDataStreamsDescribeStreamSummaryRequest>;
-
- public:
- TDescribeStreamSummaryActor(NKikimr::NGRpcService::TEvDataStreamsDescribeStreamSummaryRequest* request);
- ~TDescribeStreamSummaryActor() = default;
-
- void Bootstrap(const NActors::TActorContext& ctx);
-
- void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx);
- void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev,
- const TActorContext& ctx);
-
- private:
- void SendResponse(const TActorContext& ctx);
-
+ const auto& partitions = topicInfo.PQGroupInfo->Description.GetPartitions();
+ TString startingShardId = this->GetProtoRequest()->Getexclusive_start_shard_id();
+ ui64 startingTimepoint{0};
+ bool onlyOpenShards{true};
+
+ std::map<TShardFilter::ShardFilterType, std::function<bool(const TPartition&)>> filters = {
+ {TShardFilter::SHARD_TYPE_UNDEFINED, [&](const TPartition& p) {
+ onlyOpenShards = false;
+ return GetShardName(p.GetPartitionId()) >= startingShardId;
+ }},
+ {TShardFilter::AFTER_SHARD_ID, [&](const TPartition& p) {
+ startingShardId = ShardFilter.shard_id();
+ startingTimepoint = 0;
+ onlyOpenShards = false;
+ return GetShardName(p.GetPartitionId()) > startingShardId;
+ }},
+ {TShardFilter::AT_TRIM_HORIZON, [&](const TPartition& p) {
+ startingShardId = "0";
+ startingTimepoint = 0;
+ onlyOpenShards = true;
+ return GetShardName(p.GetPartitionId()) >= startingShardId;
+ }},
+ { TShardFilter::FROM_TRIM_HORIZON, [&](const TPartition& p) {
+ startingShardId = "0";
+ startingTimepoint = 0;
+ onlyOpenShards = false;
+ return GetShardName(p.GetPartitionId()) >= startingShardId;
+ }},
+ {TShardFilter::AT_LATEST, [&](const TPartition& p) {
+ startingTimepoint = TInstant::Now().MilliSeconds();
+ startingShardId = "0";
+ onlyOpenShards = true;
+ return GetShardName(p.GetPartitionId()) >= startingShardId;
+ }},
+ {TShardFilter::AT_TIMESTAMP, [&](const TPartition& p) {
+ startingTimepoint = ShardFilter.timestamp();
+ startingShardId = "0";
+ onlyOpenShards = true;
+ return GetShardName(p.GetPartitionId()) >= startingShardId;
+ }},
+ {TShardFilter::FROM_TIMESTAMP, [&](const TPartition& p) {
+ startingTimepoint = ShardFilter.timestamp();
+ startingShardId = "0";
+ onlyOpenShards = false;
+ return GetShardName(p.GetPartitionId()) >= startingShardId;
+ }}
+ };
+
+ const auto alreadyRead = NextToken.GetAlreadyRead();
+ if (alreadyRead > (ui32)partitions.size()) {
+ return ReplyWithError(Ydb::StatusIds::BAD_REQUEST, Ydb::PersQueue::ErrorCode::ERROR,
+ TStringBuilder() << "Provided next_token is malformed - "
+ "everything is already read", ctx);
+ }
+
+ const auto shardsToGet = std::min(partitions.size() - alreadyRead, MaxResults);
+ i32 lastCopied{0};
+ Shards.reserve(shardsToGet);
+ AllShardsCount = partitions.size();
+ for (auto partition = partitions.begin() + alreadyRead; partition != partitions.end(); ++partition) {
+ if (Shards.size() == shardsToGet) {
+ break;
+ }
+ if (filters[ShardFilter.type()](*partition)) {
+ Shards.push_back(*partition);
+ }
+ ++lastCopied;
+ }
+ auto actuallyRead = lastCopied - alreadyRead;
+ LeftToRead = partitions.size() - alreadyRead - actuallyRead;
+
+ if (Shards.size() == 0) {
+ return SendResponse(ctx);
+ }
+
+ // Send OffsetRequests
+ std::set<ui64> tabletIds;
+ for (const auto& shard : Shards) {
+ tabletIds.insert(shard.GetTabletId());
+ }
+
+ NTabletPipe::TClientConfig clientConfig;
+ clientConfig.RetryPolicy = {
+ .RetryLimitCount = 6,
+ .MinRetryTime = TDuration::MilliSeconds(10),
+ .MaxRetryTime = TDuration::MilliSeconds(100),
+ .BackoffMultiplier = 2,
+ .DoFirstRetryInstantly = true
+ };
+
+ for (auto& tabletId : tabletIds) {
+ Pipes.push_back(ctx.Register(NTabletPipe::CreateClient(ctx.SelfID, tabletId, clientConfig)));
+ TAutoPtr<TEvPersQueue::TEvOffsets> req(new TEvPersQueue::TEvOffsets);
+ NTabletPipe::SendData(ctx, Pipes.back(), req.Release());
+ }
+ }
+
+ void TListShardsActor::Handle(TEvPersQueue::TEvOffsetsResponse::TPtr& ev, const TActorContext& ctx) {
+ for (auto& part : ev->Get()->Record.GetPartResult()) {
+ StartEndOffsetsPerPartition[part.GetPartition()] =
+ std::make_pair<ui64, ui64>(part.GetStartOffset(), part.GetEndOffset());
+ ++GotOffsetResponds;
+ }
+ if (GotOffsetResponds == Shards.size()) {
+ SendResponse(ctx);
+ }
+ }
+
+ void TListShardsActor::Handle(TEvTabletPipe::TEvClientConnected::TPtr& ev, const TActorContext& ctx) {
+ if (ev->Get()->Status != NKikimrProto::EReplyStatus::OK) {
+ ReplyWithError(Ydb::StatusIds::SCHEME_ERROR, Ydb::PersQueue::ErrorCode::ERROR,
+ TStringBuilder() << "Cannot connect to tablet " << ev->Get()->TabletId, ctx);
+ }
+ }
+
+ void TListShardsActor::Handle(TEvTabletPipe::TEvClientDestroyed::TPtr& ev, const TActorContext& ctx) {
+ ReplyWithError(Ydb::StatusIds::SCHEME_ERROR, Ydb::PersQueue::ErrorCode::ERROR,
+ TStringBuilder() << "Cannot connect to tablet " << ev->Get()->TabletId, ctx);
+ }
+
+ void TListShardsActor::SendResponse(const TActorContext& ctx) {
+ Ydb::DataStreams::V1::ListShardsResult result;
+ for (auto& shard : Shards) {
+ auto awsShard = result.Addshards();
+ // TODO:
+ // awsShard->set_parent_shard_id("");
+ // awsShard->set_adjacent_parent_shard_id(prevShardName);
+ auto range = RangeFromShardNumber(shard.GetPartitionId(), AllShardsCount);
+ awsShard->mutable_hash_key_range()->set_starting_hash_key(
+ Uint128ToDecimalString(range.Start));
+ awsShard->mutable_hash_key_range()->set_ending_hash_key(
+ Uint128ToDecimalString(range.End));
+ awsShard->mutable_sequence_number_range()->set_starting_sequence_number(
+ std::to_string(StartEndOffsetsPerPartition[shard.GetPartitionId()].first));
+ awsShard->mutable_sequence_number_range()->set_ending_sequence_number(
+ std::to_string(StartEndOffsetsPerPartition[shard.GetPartitionId()].second));
+ awsShard->set_shard_id(GetShardName(shard.GetPartitionId()));
+ }
+ if (LeftToRead > 0) {
+ TNextToken token(StreamName, NextToken.GetAlreadyRead() + Shards.size(), MaxResults, TInstant::Now().MilliSeconds());
+ result.set_next_token(token.Serialize());
+ }
+ Request_->SendResult(result, Ydb::StatusIds::SUCCESS);
+ Die(ctx);
+ }
+
+ void TListShardsActor::Die(const TActorContext& ctx) {
+ //close all pipes
+ for (auto& pipe : Pipes) {
+ NTabletPipe::CloseClient(ctx, pipe);
+ }
+ TBase::Die(ctx);
+ }
+
+ //-----------------------------------------------------------------------------------
+
+ class TDescribeStreamSummaryActor : public TPQGrpcSchemaBase<TDescribeStreamSummaryActor, TEvDataStreamsDescribeStreamSummaryRequest> {
+ using TBase = TPQGrpcSchemaBase<TDescribeStreamSummaryActor, TEvDataStreamsDescribeStreamSummaryRequest>;
+
+ public:
+ TDescribeStreamSummaryActor(NKikimr::NGRpcService::TEvDataStreamsDescribeStreamSummaryRequest* request);
+ ~TDescribeStreamSummaryActor() = default;
+
+ void Bootstrap(const NActors::TActorContext& ctx);
+
+ void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx);
+ void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev,
+ const TActorContext& ctx);
+
+ private:
+ void SendResponse(const TActorContext& ctx);
+
NKikimrSchemeOp::TDirEntry SelfInfo;
NKikimrSchemeOp::TPersQueueGroupDescription PQGroup;
- };
-
- TDescribeStreamSummaryActor::TDescribeStreamSummaryActor(
- NKikimr::NGRpcService::TEvDataStreamsDescribeStreamSummaryRequest* request
- )
- : TBase(request, request->GetProtoRequest()->stream_name())
- {
- }
-
- void TDescribeStreamSummaryActor::Bootstrap(const NActors::TActorContext& ctx) {
- TBase::Bootstrap(ctx);
- SendDescribeProposeRequest(ctx);
- Become(&TDescribeStreamSummaryActor::StateWork);
- }
-
- void TDescribeStreamSummaryActor::StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) {
- switch (ev->GetTypeRewrite()) {
- default: TBase::StateWork(ev, ctx);
- }
- }
-
- void TDescribeStreamSummaryActor::HandleCacheNavigateResponse(
- TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx
- ) {
- if (ReplyIfNotTopic(ev, ctx)) {
- return;
- }
-
- const NSchemeCache::TSchemeCacheNavigate* result = ev->Get()->Request.Get();
- Y_VERIFY(result->ResultSet.size() == 1); // describe only one topic
- const auto& response = result->ResultSet.front();
- Y_VERIFY(response.PQGroupInfo);
- const TString path = JoinSeq("/", response.Path);
-
- PQGroup = response.PQGroupInfo->Description;
- SelfInfo = response.Self->Info;
-
- SendResponse(ctx);
- }
-
- void TDescribeStreamSummaryActor::SendResponse(const TActorContext& ctx) {
- Ydb::DataStreams::V1::DescribeStreamSummaryResult result;
-
- auto& pqConfig = PQGroup.GetPQTabletConfig();
- auto& descriptionSummary = *result.mutable_stream_description_summary();
-
- descriptionSummary.set_stream_name(GetProtoRequest()->stream_name());
- descriptionSummary.set_stream_arn(GetProtoRequest()->stream_name());
- descriptionSummary.set_key_id("");
- descriptionSummary.set_retention_period_hours(
- TInstant::Seconds(pqConfig.GetPartitionConfig().GetLifetimeSeconds()).Hours()
- );
- descriptionSummary.set_stream_creation_timestamp(
- TInstant::MilliSeconds(SelfInfo.GetCreateStep()).Seconds()
- );
- descriptionSummary.set_stream_status(
- SelfInfo.GetCreateFinished() ? Ydb::DataStreams::V1::StreamDescription::ACTIVE
- : Ydb::DataStreams::V1::StreamDescription::CREATING
- );
- descriptionSummary.set_open_shard_count(PQGroup.GetPartitions().size());
- descriptionSummary.set_consumer_count(PQGroup.MutablePQTabletConfig()->GetReadRules().size());
- descriptionSummary.set_encryption_type(Ydb::DataStreams::V1::EncryptionType::NONE);
-
- Request_->SendResult(result, Ydb::StatusIds::SUCCESS);
- Die(ctx);
- }
-
- //-----------------------------------------------------------------------------------------
-
+ };
+
+ TDescribeStreamSummaryActor::TDescribeStreamSummaryActor(
+ NKikimr::NGRpcService::TEvDataStreamsDescribeStreamSummaryRequest* request
+ )
+ : TBase(request, request->GetProtoRequest()->stream_name())
+ {
+ }
+
+ void TDescribeStreamSummaryActor::Bootstrap(const NActors::TActorContext& ctx) {
+ TBase::Bootstrap(ctx);
+ SendDescribeProposeRequest(ctx);
+ Become(&TDescribeStreamSummaryActor::StateWork);
+ }
+
+ void TDescribeStreamSummaryActor::StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) {
+ switch (ev->GetTypeRewrite()) {
+ default: TBase::StateWork(ev, ctx);
+ }
+ }
+
+ void TDescribeStreamSummaryActor::HandleCacheNavigateResponse(
+ TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx
+ ) {
+ if (ReplyIfNotTopic(ev, ctx)) {
+ return;
+ }
+
+ const NSchemeCache::TSchemeCacheNavigate* result = ev->Get()->Request.Get();
+ Y_VERIFY(result->ResultSet.size() == 1); // describe only one topic
+ const auto& response = result->ResultSet.front();
+ Y_VERIFY(response.PQGroupInfo);
+ const TString path = JoinSeq("/", response.Path);
+
+ PQGroup = response.PQGroupInfo->Description;
+ SelfInfo = response.Self->Info;
+
+ SendResponse(ctx);
+ }
+
+ void TDescribeStreamSummaryActor::SendResponse(const TActorContext& ctx) {
+ Ydb::DataStreams::V1::DescribeStreamSummaryResult result;
+
+ auto& pqConfig = PQGroup.GetPQTabletConfig();
+ auto& descriptionSummary = *result.mutable_stream_description_summary();
+
+ descriptionSummary.set_stream_name(GetProtoRequest()->stream_name());
+ descriptionSummary.set_stream_arn(GetProtoRequest()->stream_name());
+ descriptionSummary.set_key_id("");
+ descriptionSummary.set_retention_period_hours(
+ TInstant::Seconds(pqConfig.GetPartitionConfig().GetLifetimeSeconds()).Hours()
+ );
+ descriptionSummary.set_stream_creation_timestamp(
+ TInstant::MilliSeconds(SelfInfo.GetCreateStep()).Seconds()
+ );
+ descriptionSummary.set_stream_status(
+ SelfInfo.GetCreateFinished() ? Ydb::DataStreams::V1::StreamDescription::ACTIVE
+ : Ydb::DataStreams::V1::StreamDescription::CREATING
+ );
+ descriptionSummary.set_open_shard_count(PQGroup.GetPartitions().size());
+ descriptionSummary.set_consumer_count(PQGroup.MutablePQTabletConfig()->GetReadRules().size());
+ descriptionSummary.set_encryption_type(Ydb::DataStreams::V1::EncryptionType::NONE);
+
+ Request_->SendResult(result, Ydb::StatusIds::SUCCESS);
+ Die(ctx);
+ }
+
+ //-----------------------------------------------------------------------------------------
+
template<class TEvRequest>
class TNotImplementedRequestActor : public TRpcSchemeRequestActor<TNotImplementedRequestActor<TEvRequest>, TEvRequest> {
using TBase = TRpcSchemeRequestActor<TNotImplementedRequestActor, TEvRequest>;
@@ -1676,7 +1676,7 @@ namespace NKikimr::NDataStreams::V1 {
}
};
- //-----------------------------------------------------------------------------------
+ //-----------------------------------------------------------------------------------
IActor* CreateDataStreamsService(TIntrusivePtr<NMonitoring::TDynamicCounters> counters, TActorId newSchemeCache) {
return new TDataStreamsService(counters, newSchemeCache);
@@ -1705,11 +1705,11 @@ namespace NKikimr::NDataStreams::V1 {
}
void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsRegisterStreamConsumerRequest::TPtr& ev, const TActorContext& ctx) {
- ctx.Register(new TRegisterStreamConsumerActor(ev->Release().Release()));
+ ctx.Register(new TRegisterStreamConsumerActor(ev->Release().Release()));
}
void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsDeregisterStreamConsumerRequest::TPtr& ev, const TActorContext& ctx) {
- ctx.Register(new TDeregisterStreamConsumerActor(ev->Release().Release()));
+ ctx.Register(new TDeregisterStreamConsumerActor(ev->Release().Release()));
}
void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsDescribeStreamConsumerRequest::TPtr& ev, const TActorContext& ctx) {
@@ -1725,7 +1725,7 @@ namespace NKikimr::NDataStreams::V1 {
}
void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsListShardsRequest::TPtr& ev, const TActorContext& ctx) {
- ctx.Register(new TListShardsActor(ev->Release().Release(), NewSchemeCache));
+ ctx.Register(new TListShardsActor(ev->Release().Release(), NewSchemeCache));
}
void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsPutRecordsRequest::TPtr& ev, const TActorContext& ctx) {
@@ -1733,11 +1733,11 @@ namespace NKikimr::NDataStreams::V1 {
}
void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsGetRecordsRequest::TPtr& ev, const TActorContext& ctx) {
- ctx.Register(new TGetRecordsActor(ev->Release().Release(), NewSchemeCache));
+ ctx.Register(new TGetRecordsActor(ev->Release().Release(), NewSchemeCache));
}
void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsGetShardIteratorRequest::TPtr& ev, const TActorContext& ctx) {
- ctx.Register(new TGetShardIteratorActor(ev->Release().Release(), NewSchemeCache));
+ ctx.Register(new TGetShardIteratorActor(ev->Release().Release(), NewSchemeCache));
}
void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsSubscribeToShardRequest::TPtr& ev, const TActorContext& ctx) {
@@ -1749,7 +1749,7 @@ namespace NKikimr::NDataStreams::V1 {
}
void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsDescribeStreamSummaryRequest::TPtr& ev, const TActorContext& ctx) {
- ctx.Register(new TDescribeStreamSummaryActor(ev->Release().Release()));
+ ctx.Register(new TDescribeStreamSummaryActor(ev->Release().Release()));
}
void TDataStreamsService::Handle(TEvDataStreamsDecreaseStreamRetentionPeriodRequest::TPtr& ev, const TActorContext& ctx) {
@@ -1769,7 +1769,7 @@ namespace NKikimr::NDataStreams::V1 {
}
void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsListStreamConsumersRequest::TPtr& ev, const TActorContext& ctx) {
- ctx.Register(new TListStreamConsumersActor(ev->Release().Release()));
+ ctx.Register(new TListStreamConsumersActor(ev->Release().Release()));
}
void TDataStreamsService::Handle(NKikimr::NGRpcService::TEvDataStreamsAddTagsToStreamRequest::TPtr& ev, const TActorContext& ctx) {
diff --git a/ydb/services/datastreams/datastreams_ut.cpp b/ydb/services/datastreams/datastreams_ut.cpp
index 9ab384b916..2ede2a2d05 100644
--- a/ydb/services/datastreams/datastreams_ut.cpp
+++ b/ydb/services/datastreams/datastreams_ut.cpp
@@ -13,15 +13,15 @@
#include <library/cpp/json/json_reader.h>
#include <library/cpp/digest/md5/md5.h>
-#include <random>
-
-
+#include <random>
+
+
using namespace NYdb;
using namespace NYdb::NTable;
using namespace NKikimr::NPersQueueTests;
using namespace NKikimr::NDataStreams::V1;
-namespace YDS_V1 = Ydb::DataStreams::V1;
-namespace NYDS_V1 = NYdb::NDataStreams::V1;
+namespace YDS_V1 = Ydb::DataStreams::V1;
+namespace NYDS_V1 = NYdb::NDataStreams::V1;
struct WithSslAndAuth : TKikimrTestSettings {
static constexpr bool SSL = true;
static constexpr bool AUTH = true;
@@ -61,7 +61,7 @@ public:
}
Driver = std::make_unique<TDriver>(std::move(driverConfig));
- DataStreamsClient = std::make_unique<NYDS_V1::TDataStreamsClient>(*Driver,
+ DataStreamsClient = std::make_unique<NYDS_V1::TDataStreamsClient>(*Driver,
TCommonClientSettings()
.AuthToken("user@builtin"));
@@ -70,7 +70,7 @@ public:
NYdb::NScheme::TPermissions permissions("user@builtin", {"ydb.generic.read", "ydb.generic.write"});
auto result = schemeClient.ModifyPermissions("/Root",
- NYdb::NScheme::TModifyPermissionsSettings().AddGrantPermissions(permissions)
+ NYdb::NScheme::TModifyPermissionsSettings().AddGrantPermissions(permissions)
).ExtractValueSync();
Cerr << result.GetIssues().ToString() << "\n";
UNIT_ASSERT(result.IsSuccess());
@@ -84,8 +84,8 @@ public:
public:
std::unique_ptr<TKikimr> KikimrServer;
std::unique_ptr<TDriver> Driver;
- std::unique_ptr<NYDS_V1::TDataStreamsClient> DataStreamsClient;
- std::unique_ptr<NYDS_V1::TDataStreamsClient> UnauthenticatedClient;
+ std::unique_ptr<NYDS_V1::TDataStreamsClient> DataStreamsClient;
+ std::unique_ptr<NYDS_V1::TDataStreamsClient> UnauthenticatedClient;
THolder<TTempFileHandle> MeteringFile;
};
@@ -116,7 +116,7 @@ void CheckMeteringFile(TTempFileHandle* meteringFile, const TString& streamPath)
UNIT_ASSERT(map.contains("source_wt"));
UNIT_ASSERT(map.find("cloud_id")->second.GetString() == "somecloud");
UNIT_ASSERT(map.find("folder_id")->second.GetString() == "somefolder");
- UNIT_ASSERT(map.find("resource_id")->second.GetString() == streamPath);
+ UNIT_ASSERT(map.find("resource_id")->second.GetString() == streamPath);
auto& tags = map.find("tags")->second.GetMap();
if (!tags.empty()) {
UNIT_ASSERT_VALUES_EQUAL(tags.size(), 3);
@@ -129,23 +129,23 @@ void CheckMeteringFile(TTempFileHandle* meteringFile, const TString& streamPath)
}
-#define Y_UNIT_TEST_NAME this->Name_;
-
+#define Y_UNIT_TEST_NAME this->Name_;
+
Y_UNIT_TEST_SUITE(DataStreams) {
Y_UNIT_TEST(TestControlPlaneAndMeteringData) {
TInsecureDatastreamsTestServer testServer;
- const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME;
+ const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME;
// Trying to delete stream that doesn't exist yet
{
- auto result = testServer.DataStreamsClient->DeleteStream("testfolder/" + streamName).ExtractValueSync();
+ auto result = testServer.DataStreamsClient->DeleteStream("testfolder/" + streamName).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR);
}
{
- auto result = testServer.DataStreamsClient->CreateStream(streamName,
- NYDS_V1::TCreateStreamSettings().ShardCount(3)).ExtractValueSync();
+ auto result = testServer.DataStreamsClient->CreateStream(streamName,
+ NYDS_V1::TCreateStreamSettings().ShardCount(3)).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
if (result.GetStatus() != EStatus::SUCCESS) {
result.GetIssues().PrintTo(Cerr);
@@ -154,13 +154,13 @@ Y_UNIT_TEST_SUITE(DataStreams) {
}
{
- auto result = testServer.DataStreamsClient->DescribeStream(streamName).ExtractValueSync();
+ auto result = testServer.DataStreamsClient->DescribeStream(streamName).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_description().stream_status(),
- YDS_V1::StreamDescription::ACTIVE);
- UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_description().stream_name(), streamName);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_description().stream_status(),
+ YDS_V1::StreamDescription::ACTIVE);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_description().stream_name(), streamName);
UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_description().write_quota_kb_per_sec(), 1024);
UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_description().retention_period_hours(), 24);
@@ -174,32 +174,32 @@ Y_UNIT_TEST_SUITE(DataStreams) {
UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_description().shards(2).hash_key_range().ending_hash_key(), "340282366920938463463374607431768211455");
}
- {
- auto result = testServer.DataStreamsClient->DescribeStreamSummary(streamName).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ {
+ auto result = testServer.DataStreamsClient->DescribeStreamSummary(streamName).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+
+ UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_description_summary().stream_status(),
+ YDS_V1::StreamDescription::ACTIVE);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_description_summary().stream_name(), streamName);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_description_summary().stream_arn(), streamName);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_description_summary().retention_period_hours(), 24);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_description_summary().open_shard_count(), 3);
+ }
- UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_description_summary().stream_status(),
- YDS_V1::StreamDescription::ACTIVE);
- UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_description_summary().stream_name(), streamName);
- UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_description_summary().stream_arn(), streamName);
- UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_description_summary().retention_period_hours(), 24);
- UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_description_summary().open_shard_count(), 3);
- }
-
{
- auto result = testServer.DataStreamsClient->CreateStream("testfolder/" + streamName).ExtractValueSync();
+ auto result = testServer.DataStreamsClient->CreateStream("testfolder/" + streamName).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST);
}
{ // for metering purposes
- std::vector<NYDS_V1::TDataRecord> records;
+ std::vector<NYDS_V1::TDataRecord> records;
for (ui32 i = 1; i <= 30; ++i) {
TString data = Sprintf("%04u", i);
records.push_back({data, data, ""});
}
- auto result = testServer.DataStreamsClient->PutRecords(streamName, records).ExtractValueSync();
+ auto result = testServer.DataStreamsClient->PutRecords(streamName, records).ExtractValueSync();
Cerr << result.GetResult().DebugString() << Endl;
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
@@ -210,20 +210,20 @@ Y_UNIT_TEST_SUITE(DataStreams) {
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_names().size(), 1);
- UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_names(0), streamName);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_names(0), streamName);
}
// cannot decrease the number of shards
{
- auto result = testServer.DataStreamsClient->UpdateShardCount(streamName,
- NYDS_V1::TUpdateShardCountSettings().TargetShardCount(2)).ExtractValueSync();
+ auto result = testServer.DataStreamsClient->UpdateShardCount(streamName,
+ NYDS_V1::TUpdateShardCountSettings().TargetShardCount(2)).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST);
}
{
- auto result = testServer.DataStreamsClient->UpdateShardCount(streamName,
- NYDS_V1::TUpdateShardCountSettings().TargetShardCount(15)).ExtractValueSync();
+ auto result = testServer.DataStreamsClient->UpdateShardCount(streamName,
+ NYDS_V1::TUpdateShardCountSettings().TargetShardCount(15)).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
}
@@ -231,33 +231,33 @@ Y_UNIT_TEST_SUITE(DataStreams) {
// now when stream is created delete should work fine
{
- auto result = testServer.DataStreamsClient->DeleteStream(streamName).ExtractValueSync();
+ auto result = testServer.DataStreamsClient->DeleteStream(streamName).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
}
// Describe should fail after delete
{
- auto result = testServer.DataStreamsClient->DescribeStream(streamName).ExtractValueSync();
+ auto result = testServer.DataStreamsClient->DescribeStream(streamName).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR);
}
- CheckMeteringFile(testServer.MeteringFile.Get(), "/Root/" + streamName);
+ CheckMeteringFile(testServer.MeteringFile.Get(), "/Root/" + streamName);
}
Y_UNIT_TEST(TestCreateExistingStream) {
TInsecureDatastreamsTestServer testServer;
- const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME;
+ const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME;
{
- auto result = testServer.DataStreamsClient->CreateStream(streamName,
- NYDS_V1::TCreateStreamSettings().ShardCount(10)).ExtractValueSync();
+ auto result = testServer.DataStreamsClient->CreateStream(streamName,
+ NYDS_V1::TCreateStreamSettings().ShardCount(10)).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
}
{
- auto result = testServer.DataStreamsClient->CreateStream(streamName,
- NYDS_V1::TCreateStreamSettings().ShardCount(10)).ExtractValueSync();
+ auto result = testServer.DataStreamsClient->CreateStream(streamName,
+ NYDS_V1::TCreateStreamSettings().ShardCount(10)).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::ALREADY_EXISTS);
}
@@ -266,11 +266,11 @@ Y_UNIT_TEST_SUITE(DataStreams) {
Y_UNIT_TEST(TestStreamPagination) {
TInsecureDatastreamsTestServer testServer;
- const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME;
+ const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME;
for (ui32 folderIdx = 0; folderIdx < 4; folderIdx++) {
for (ui32 streamIdx = 0; streamIdx < 5; streamIdx++) {
- TStringBuilder streamNameX = TStringBuilder() << folderIdx << streamName << streamIdx;
- auto result = testServer.DataStreamsClient->CreateStream(streamNameX, NYDS_V1::TCreateStreamSettings().ShardCount(10)).ExtractValueSync();
+ TStringBuilder streamNameX = TStringBuilder() << folderIdx << streamName << streamIdx;
+ auto result = testServer.DataStreamsClient->CreateStream(streamNameX, NYDS_V1::TCreateStreamSettings().ShardCount(10)).ExtractValueSync();
Cerr << result.GetIssues().ToString() << "\n";
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
@@ -280,7 +280,7 @@ Y_UNIT_TEST_SUITE(DataStreams) {
TString startStream;
THashSet<TString> streams;
for (int i = 0; i < 3; i++) {
- auto result = testServer.DataStreamsClient->ListStreams(NYDS_V1::TListStreamsSettings().Limit(6).ExclusiveStartStreamName(startStream)).ExtractValueSync();
+ auto result = testServer.DataStreamsClient->ListStreams(NYDS_V1::TListStreamsSettings().Limit(6).ExclusiveStartStreamName(startStream)).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_names().size(), 6);
@@ -290,7 +290,7 @@ Y_UNIT_TEST_SUITE(DataStreams) {
}
{
- auto result = testServer.DataStreamsClient->ListStreams(NYDS_V1::TListStreamsSettings().Limit(6).ExclusiveStartStreamName(startStream)).ExtractValueSync();
+ auto result = testServer.DataStreamsClient->ListStreams(NYDS_V1::TListStreamsSettings().Limit(6).ExclusiveStartStreamName(startStream)).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_names().size(), 2);
@@ -301,114 +301,114 @@ Y_UNIT_TEST_SUITE(DataStreams) {
UNIT_ASSERT_VALUES_EQUAL(streams.size(), 20);
}
- Y_UNIT_TEST(TestDeleteStream) {
- TInsecureDatastreamsTestServer testServer;
- const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME;
- {
- auto result = testServer.DataStreamsClient->CreateStream(streamName,
- NYDS_V1::TCreateStreamSettings().ShardCount(3)).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- }
-
- {
- auto result = testServer.DataStreamsClient->DeleteStream(streamName).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- }
- }
-
- Y_UNIT_TEST(TestDeleteStreamWithEnforceFlag) {
- TInsecureDatastreamsTestServer testServer;
- const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME;
- {
- auto result = testServer.DataStreamsClient->CreateStream(streamName,
- NYDS_V1::TCreateStreamSettings().ShardCount(3)).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- }
-
- {
- auto result = testServer.DataStreamsClient->RegisterStreamConsumer(streamName, "user1",
- NYDS_V1::TRegisterStreamConsumerSettings()).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- }
-
- {
- auto result = testServer.DataStreamsClient->DeleteStream(streamName).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST);
- }
-
- {
- auto result = testServer.DataStreamsClient->DeleteStream(streamName,
- NYDS_V1::TDeleteStreamSettings().EnforceConsumerDeletion(true)).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- }
- }
-
-
- Y_UNIT_TEST(TestDeleteStreamWithEnforceFlagFalse) {
- TInsecureDatastreamsTestServer testServer;
- const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME;
- {
- auto result = testServer.DataStreamsClient->CreateStream(streamName,
- NYDS_V1::TCreateStreamSettings().ShardCount(3)).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- }
-
- {
- auto result = testServer.DataStreamsClient->RegisterStreamConsumer(streamName, "user1",
- NYDS_V1::TRegisterStreamConsumerSettings()).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- }
-
- {
- auto result = testServer.DataStreamsClient->DeleteStream(streamName,
- NYDS_V1::TDeleteStreamSettings().EnforceConsumerDeletion(false)).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST);
- }
-
- {
- auto result = testServer.DataStreamsClient->DeregisterStreamConsumer(streamName, "user1",
- NYDS_V1::TDeregisterStreamConsumerSettings()).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- }
-
- {
- auto result = testServer.DataStreamsClient->DeleteStream(streamName,
- NYDS_V1::TDeleteStreamSettings().EnforceConsumerDeletion(false)).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- }
- }
-
+ Y_UNIT_TEST(TestDeleteStream) {
+ TInsecureDatastreamsTestServer testServer;
+ const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME;
+ {
+ auto result = testServer.DataStreamsClient->CreateStream(streamName,
+ NYDS_V1::TCreateStreamSettings().ShardCount(3)).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ }
+
+ {
+ auto result = testServer.DataStreamsClient->DeleteStream(streamName).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ }
+ }
+
+ Y_UNIT_TEST(TestDeleteStreamWithEnforceFlag) {
+ TInsecureDatastreamsTestServer testServer;
+ const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME;
+ {
+ auto result = testServer.DataStreamsClient->CreateStream(streamName,
+ NYDS_V1::TCreateStreamSettings().ShardCount(3)).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ }
+
+ {
+ auto result = testServer.DataStreamsClient->RegisterStreamConsumer(streamName, "user1",
+ NYDS_V1::TRegisterStreamConsumerSettings()).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ }
+
+ {
+ auto result = testServer.DataStreamsClient->DeleteStream(streamName).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST);
+ }
+
+ {
+ auto result = testServer.DataStreamsClient->DeleteStream(streamName,
+ NYDS_V1::TDeleteStreamSettings().EnforceConsumerDeletion(true)).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ }
+ }
+
+
+ Y_UNIT_TEST(TestDeleteStreamWithEnforceFlagFalse) {
+ TInsecureDatastreamsTestServer testServer;
+ const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME;
+ {
+ auto result = testServer.DataStreamsClient->CreateStream(streamName,
+ NYDS_V1::TCreateStreamSettings().ShardCount(3)).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ }
+
+ {
+ auto result = testServer.DataStreamsClient->RegisterStreamConsumer(streamName, "user1",
+ NYDS_V1::TRegisterStreamConsumerSettings()).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ }
+
+ {
+ auto result = testServer.DataStreamsClient->DeleteStream(streamName,
+ NYDS_V1::TDeleteStreamSettings().EnforceConsumerDeletion(false)).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST);
+ }
+
+ {
+ auto result = testServer.DataStreamsClient->DeregisterStreamConsumer(streamName, "user1",
+ NYDS_V1::TDeregisterStreamConsumerSettings()).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ }
+
+ {
+ auto result = testServer.DataStreamsClient->DeleteStream(streamName,
+ NYDS_V1::TDeleteStreamSettings().EnforceConsumerDeletion(false)).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ }
+ }
+
Y_UNIT_TEST(TestUpdateStream) {
TInsecureDatastreamsTestServer testServer;
- const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME;
+ const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME;
{
- auto result = testServer.DataStreamsClient->CreateStream(streamName,
- NYDS_V1::TCreateStreamSettings().ShardCount(10)).ExtractValueSync();
+ auto result = testServer.DataStreamsClient->CreateStream(streamName,
+ NYDS_V1::TCreateStreamSettings().ShardCount(10)).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
}
for (ui32 i = 0; i < 2; ++i) {
- auto result = testServer.DataStreamsClient->UpdateStream(streamName,
- NYDS_V1::TUpdateStreamSettings().RetentionPeriodHours(5).TargetShardCount(20).WriteQuotaKbPerSec(128)
- ).ExtractValueSync();
+ auto result = testServer.DataStreamsClient->UpdateStream(streamName,
+ NYDS_V1::TUpdateStreamSettings().RetentionPeriodHours(5).TargetShardCount(20).WriteQuotaKbPerSec(128)
+ ).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
}
{
- auto result = testServer.DataStreamsClient->DescribeStream(streamName).ExtractValueSync();
+ auto result = testServer.DataStreamsClient->DescribeStream(streamName).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_description().shards_size(), 20);
@@ -421,42 +421,42 @@ Y_UNIT_TEST_SUITE(DataStreams) {
Y_UNIT_TEST(TestStreamRetention) {
TInsecureDatastreamsTestServer testServer;
- const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME;
+ const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME;
{
- auto result = testServer.DataStreamsClient->CreateStream(streamName,
- NYDS_V1::TCreateStreamSettings().ShardCount(10)).ExtractValueSync();
+ auto result = testServer.DataStreamsClient->CreateStream(streamName,
+ NYDS_V1::TCreateStreamSettings().ShardCount(10)).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
}
{
- auto result = testServer.DataStreamsClient->IncreaseStreamRetentionPeriod(streamName,
- NYDS_V1::TIncreaseStreamRetentionPeriodSettings().RetentionPeriodHours(50)
- ).ExtractValueSync();
+ auto result = testServer.DataStreamsClient->IncreaseStreamRetentionPeriod(streamName,
+ NYDS_V1::TIncreaseStreamRetentionPeriodSettings().RetentionPeriodHours(50)
+ ).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST);
}
{
- auto result = testServer.DataStreamsClient->DecreaseStreamRetentionPeriod(streamName,
- NYDS_V1::TDecreaseStreamRetentionPeriodSettings().RetentionPeriodHours(8)
- ).ExtractValueSync();
+ auto result = testServer.DataStreamsClient->DecreaseStreamRetentionPeriod(streamName,
+ NYDS_V1::TDecreaseStreamRetentionPeriodSettings().RetentionPeriodHours(8)
+ ).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
}
{
- auto result = testServer.DataStreamsClient->IncreaseStreamRetentionPeriod(streamName,
- NYDS_V1::TIncreaseStreamRetentionPeriodSettings().RetentionPeriodHours(4)
- ).ExtractValueSync();
+ auto result = testServer.DataStreamsClient->IncreaseStreamRetentionPeriod(streamName,
+ NYDS_V1::TIncreaseStreamRetentionPeriodSettings().RetentionPeriodHours(4)
+ ).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST);
}
{
- auto result = testServer.DataStreamsClient->IncreaseStreamRetentionPeriod(streamName,
- NYDS_V1::TIncreaseStreamRetentionPeriodSettings().RetentionPeriodHours(15)
- ).ExtractValueSync();
+ auto result = testServer.DataStreamsClient->IncreaseStreamRetentionPeriod(streamName,
+ NYDS_V1::TIncreaseStreamRetentionPeriodSettings().RetentionPeriodHours(15)
+ ).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
}
@@ -465,9 +465,9 @@ Y_UNIT_TEST_SUITE(DataStreams) {
Y_UNIT_TEST(TestShardPagination) {
TInsecureDatastreamsTestServer testServer;
- const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME;
+ const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME;
{
- auto result = testServer.DataStreamsClient->CreateStream("/Root/" + streamName, NYDS_V1::TCreateStreamSettings().ShardCount(9)).ExtractValueSync();
+ auto result = testServer.DataStreamsClient->CreateStream("/Root/" + streamName, NYDS_V1::TCreateStreamSettings().ShardCount(9)).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
}
@@ -477,8 +477,8 @@ Y_UNIT_TEST_SUITE(DataStreams) {
TString exclusiveStartShardId;
THashSet<TString> describedShards;
for (int i = 0; i < 8; i += 2) {
- auto result = testServer.DataStreamsClient->DescribeStream(streamName,
- NYDS_V1::TDescribeStreamSettings()
+ auto result = testServer.DataStreamsClient->DescribeStream(streamName,
+ NYDS_V1::TDescribeStreamSettings()
.Limit(2)
.ExclusiveStartShardId(exclusiveStartShardId)
).ExtractValueSync();
@@ -494,12 +494,12 @@ Y_UNIT_TEST_SUITE(DataStreams) {
}
{
- auto result = testServer.DataStreamsClient->DescribeStream(streamName,
- NYDS_V1::TDescribeStreamSettings()
+ auto result = testServer.DataStreamsClient->DescribeStream(streamName,
+ NYDS_V1::TDescribeStreamSettings()
.Limit(2)
.ExclusiveStartShardId(
- exclusiveStartShardId)
- ).ExtractValueSync();
+ exclusiveStartShardId)
+ ).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
auto &description = result.GetResult().stream_description();
@@ -526,34 +526,34 @@ Y_UNIT_TEST_SUITE(DataStreams) {
Y_UNIT_TEST(TestPutRecordsWithRead) {
TInsecureDatastreamsTestServer testServer;
- const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME;
+ const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME;
SET_YDS_LOCALS;
{
- auto result = testServer.DataStreamsClient->CreateStream(streamName,
- NYDS_V1::TCreateStreamSettings().ShardCount(5)).ExtractValueSync();
+ auto result = testServer.DataStreamsClient->CreateStream(streamName,
+ NYDS_V1::TCreateStreamSettings().ShardCount(5)).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
}
kikimr->GetRuntime()->SetLogPriority(NKikimrServices::PQ_READ_PROXY, NLog::EPriority::PRI_DEBUG);
kikimr->GetRuntime()->SetLogPriority(NKikimrServices::PQ_WRITE_PROXY, NLog::EPriority::PRI_DEBUG);
- NYDS_V1::TDataStreamsClient client(*driver, TCommonClientSettings().AuthToken("user2@builtin"));
+ NYDS_V1::TDataStreamsClient client(*driver, TCommonClientSettings().AuthToken("user2@builtin"));
TString dataStr = "9876543210";
- auto putRecordResult = client.PutRecord("/Root/" + streamName, {dataStr, dataStr, dataStr}).ExtractValueSync();
+ auto putRecordResult = client.PutRecord("/Root/" + streamName, {dataStr, dataStr, dataStr}).ExtractValueSync();
Cerr << putRecordResult.GetResult().DebugString() << Endl;
UNIT_ASSERT_VALUES_EQUAL(putRecordResult.IsTransportError(), false);
UNIT_ASSERT_VALUES_EQUAL(putRecordResult.GetStatus(), EStatus::SUCCESS);
{
- std::vector<NYDS_V1::TDataRecord> records;
+ std::vector<NYDS_V1::TDataRecord> records;
for (ui32 i = 1; i <= 30; ++i) {
TString data = Sprintf("%04u", i);
records.push_back({data, data, ""});
}
- auto result = client.PutRecords(streamName, records).ExtractValueSync();
+ auto result = client.PutRecords(streamName, records).ExtractValueSync();
Cerr << result.GetResult().DebugString() << Endl;
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
@@ -562,18 +562,18 @@ Y_UNIT_TEST_SUITE(DataStreams) {
NYdb::NPersQueue::TPersQueueClient pqClient(*driver);
{
- auto result = testServer.DataStreamsClient->RegisterStreamConsumer(streamName, "user1", NYDS_V1::TRegisterStreamConsumerSettings()).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- UNIT_ASSERT_VALUES_EQUAL(result.GetResult().consumer().consumer_name(), "user1");
- UNIT_ASSERT_VALUES_EQUAL(result.GetResult().consumer().consumer_status(),
- YDS_V1::ConsumerDescription_ConsumerStatus_ACTIVE);
+ auto result = testServer.DataStreamsClient->RegisterStreamConsumer(streamName, "user1", NYDS_V1::TRegisterStreamConsumerSettings()).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetResult().consumer().consumer_name(), "user1");
+ UNIT_ASSERT_VALUES_EQUAL(result.GetResult().consumer().consumer_status(),
+ YDS_V1::ConsumerDescription_ConsumerStatus_ACTIVE);
}
auto session = pqClient.CreateReadSession(NYdb::NPersQueue::TReadSessionSettings()
.ConsumerName("user1")
.DisableClusterDiscovery(true)
- .AppendTopics(NYdb::NPersQueue::TTopicReadSettings().Path("/Root/" + streamName)));
+ .AppendTopics(NYdb::NPersQueue::TTopicReadSettings().Path("/Root/" + streamName)));
ui32 readCount = 0;
while (readCount < 31) {
auto event = session->GetEvent(true);
@@ -603,32 +603,32 @@ Y_UNIT_TEST_SUITE(DataStreams) {
Y_UNIT_TEST(TestPutRecordsCornerCases) {
TInsecureDatastreamsTestServer testServer;
- const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME;
- const TString streamPath = "/Root/" + streamName;
+ const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME;
+ const TString streamPath = "/Root/" + streamName;
SET_YDS_LOCALS;
{
auto result = testServer.DataStreamsClient->CreateStream(
- streamName,
- NYDS_V1::TCreateStreamSettings().ShardCount(5)
- ).ExtractValueSync();
+ streamName,
+ NYDS_V1::TCreateStreamSettings().ShardCount(5)
+ ).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
}
kikimr->GetRuntime()->SetLogPriority(NKikimrServices::PQ_READ_PROXY, NLog::EPriority::PRI_DEBUG);
- NYDS_V1::TDataStreamsClient client(*driver, TCommonClientSettings().AuthToken("user2@builtin"));
+ NYDS_V1::TDataStreamsClient client(*driver, TCommonClientSettings().AuthToken("user2@builtin"));
// Test for too long partition key
TString longKey = TString(257, '1');
TString shortEnoughKey = TString(256, '1');
- auto result = client.PutRecords(streamName,
- {{longKey, longKey, ""},
- {shortEnoughKey, shortEnoughKey, ""}}).ExtractValueSync();
+ auto result = client.PutRecords(streamName,
+ {{longKey, longKey, ""},
+ {shortEnoughKey, shortEnoughKey, ""}}).ExtractValueSync();
UNIT_ASSERT(!result.IsSuccess());
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST);
- result = client.PutRecords(streamName,
- {{shortEnoughKey, shortEnoughKey, ""},
- {shortEnoughKey, shortEnoughKey, ""}}).ExtractValueSync();
+ result = client.PutRecords(streamName,
+ {{shortEnoughKey, shortEnoughKey, ""},
+ {shortEnoughKey, shortEnoughKey, ""}}).ExtractValueSync();
UNIT_ASSERT(result.IsSuccess());
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
@@ -636,15 +636,15 @@ Y_UNIT_TEST_SUITE(DataStreams) {
TString longData = TString(1048577, '1');
TString shortEnoughData = TString(1048576, '1');
- result = client.PutRecords(streamName,
- {{longData, shortEnoughKey, ""},
- {shortEnoughData, shortEnoughKey, ""}}).ExtractValueSync();
+ result = client.PutRecords(streamName,
+ {{longData, shortEnoughKey, ""},
+ {shortEnoughData, shortEnoughKey, ""}}).ExtractValueSync();
UNIT_ASSERT(!result.IsSuccess());
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST);
- result = client.PutRecords(streamName,
- {{shortEnoughData, shortEnoughKey, ""},
- {"", shortEnoughKey, ""}}).ExtractValueSync();
+ result = client.PutRecords(streamName,
+ {{shortEnoughData, shortEnoughKey, ""},
+ {"", shortEnoughKey, ""}}).ExtractValueSync();
UNIT_ASSERT(result.IsSuccess());
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
@@ -652,27 +652,27 @@ Y_UNIT_TEST_SUITE(DataStreams) {
TString shortEnoughExplicitHash = "340282366920938463463374607431768211455";
TString badExplicitHash = "-439025493205215";
- result = client.PutRecords(streamName,
- {{"", shortEnoughKey, longExplicitHash},
- {"", shortEnoughKey, longExplicitHash}}).ExtractValueSync();
+ result = client.PutRecords(streamName,
+ {{"", shortEnoughKey, longExplicitHash},
+ {"", shortEnoughKey, longExplicitHash}}).ExtractValueSync();
UNIT_ASSERT(!result.IsSuccess());
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST);
- result = client.PutRecords(streamName,
- {{"", shortEnoughKey, badExplicitHash},
- {"", shortEnoughKey, badExplicitHash}}).ExtractValueSync();
+ result = client.PutRecords(streamName,
+ {{"", shortEnoughKey, badExplicitHash},
+ {"", shortEnoughKey, badExplicitHash}}).ExtractValueSync();
UNIT_ASSERT(!result.IsSuccess());
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST);
- result = client.PutRecords(streamName,
- {{"", shortEnoughKey, shortEnoughExplicitHash},
- {"", shortEnoughKey, shortEnoughExplicitHash}}).ExtractValueSync();
+ result = client.PutRecords(streamName,
+ {{"", shortEnoughKey, shortEnoughExplicitHash},
+ {"", shortEnoughKey, shortEnoughExplicitHash}}).ExtractValueSync();
UNIT_ASSERT(result.IsSuccess());
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- result = client.PutRecords(streamName,
- {{"", shortEnoughKey, "0"},
- {"", shortEnoughKey, "0"}}).ExtractValueSync();
+ result = client.PutRecords(streamName,
+ {{"", shortEnoughKey, "0"},
+ {"", shortEnoughKey, "0"}}).ExtractValueSync();
UNIT_ASSERT(result.IsSuccess());
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
@@ -683,7 +683,7 @@ Y_UNIT_TEST_SUITE(DataStreams) {
kikimr->GetRuntime()->SetLogPriority(NKikimrServices::PERSQUEUE, NLog::EPriority::PRI_INFO);
{
- std::vector<NYDS_V1::TDataRecord> records;
+ std::vector<NYDS_V1::TDataRecord> records;
TString data = TString(1024*1024, 'a');
records.push_back({data, "key", ""});
records.push_back({data, "key", ""});
@@ -691,7 +691,7 @@ Y_UNIT_TEST_SUITE(DataStreams) {
records.push_back({data, "key", ""});
Cerr << "First put records\n";
- auto result = client.PutRecords(streamPath, records).ExtractValueSync();
+ auto result = client.PutRecords(streamPath, records).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
if (result.GetStatus() != EStatus::SUCCESS) {
result.GetIssues().PrintTo(Cerr);
@@ -700,7 +700,7 @@ Y_UNIT_TEST_SUITE(DataStreams) {
Cerr << result.GetResult().DebugString() << Endl;
Cerr << "Second put records\n";
- result = client.PutRecords(streamPath, records).ExtractValueSync();
+ result = client.PutRecords(streamPath, records).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
if (result.GetStatus() != EStatus::SUCCESS) {
result.GetIssues().PrintTo(Cerr);
@@ -712,7 +712,7 @@ Y_UNIT_TEST_SUITE(DataStreams) {
Sleep(TDuration::Seconds(4));
Cerr << "Third put records\n";
- result = client.PutRecords(streamPath, records).ExtractValueSync();
+ result = client.PutRecords(streamPath, records).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
if (result.GetStatus() != EStatus::SUCCESS) {
result.GetIssues().PrintTo(Cerr);
@@ -727,7 +727,7 @@ Y_UNIT_TEST_SUITE(DataStreams) {
{
NYdb::NPersQueue::TAddReadRuleSettings addReadRuleSettings;
addReadRuleSettings.ReadRule(NYdb::NPersQueue::TReadRuleSettings().Version(1).ConsumerName("user1"));
- auto res = pqClient.AddReadRule(streamPath, addReadRuleSettings);
+ auto res = pqClient.AddReadRule(streamPath, addReadRuleSettings);
res.Wait();
UNIT_ASSERT(res.GetValue().IsSuccess());
}
@@ -735,7 +735,7 @@ Y_UNIT_TEST_SUITE(DataStreams) {
.ConsumerName("user1")
.DisableClusterDiscovery(true)
.RetryPolicy(NYdb::NPersQueue::IRetryPolicy::GetNoRetryPolicy())
- .AppendTopics(NYdb::NPersQueue::TTopicReadSettings().Path(streamPath)));
+ .AppendTopics(NYdb::NPersQueue::TTopicReadSettings().Path(streamPath)));
ui32 readCount = 0;
while (readCount < 14) {
auto event = session->GetEvent(true);
@@ -758,20 +758,20 @@ Y_UNIT_TEST_SUITE(DataStreams) {
Y_UNIT_TEST(TestPutRecords) {
TSecureDatastreamsTestServer testServer;
- const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME;
+ const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME;
SET_YDS_LOCALS;
- const TString streamPath = "/Root/" + streamName;
+ const TString streamPath = "/Root/" + streamName;
{
- auto result = testServer.DataStreamsClient->CreateStream(streamPath,
- NYDS_V1::TCreateStreamSettings().ShardCount(5)).ExtractValueSync();
+ auto result = testServer.DataStreamsClient->CreateStream(streamPath,
+ NYDS_V1::TCreateStreamSettings().ShardCount(5)).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
}
- NYDS_V1::TDataStreamsClient client(*driver, TCommonClientSettings().AuthToken("user2@builtin"));
+ NYDS_V1::TDataStreamsClient client(*driver, TCommonClientSettings().AuthToken("user2@builtin"));
NYdb::NScheme::TSchemeClient schemeClient(*driver);
{
- std::vector<NYDS_V1::TDataRecord> records;
+ std::vector<NYDS_V1::TDataRecord> records;
for (ui32 i = 1; i <= 30; ++i) {
TString data = Sprintf("%04u", i);
records.push_back({data, data, ""});
@@ -783,12 +783,12 @@ Y_UNIT_TEST_SUITE(DataStreams) {
}
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNAUTHORIZED);
- {
- NYdb::NScheme::TPermissions permissions("user2@builtin", {"ydb.generic.read", "ydb.generic.write"});
- auto result = schemeClient.ModifyPermissions(streamPath,
- NYdb::NScheme::TModifyPermissionsSettings().AddGrantPermissions(permissions)).ExtractValueSync();
- UNIT_ASSERT(result.IsSuccess());
- }
+ {
+ NYdb::NScheme::TPermissions permissions("user2@builtin", {"ydb.generic.read", "ydb.generic.write"});
+ auto result = schemeClient.ModifyPermissions(streamPath,
+ NYdb::NScheme::TModifyPermissionsSettings().AddGrantPermissions(permissions)).ExtractValueSync();
+ UNIT_ASSERT(result.IsSuccess());
+ }
result = client.PutRecords(streamPath, records).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
@@ -796,7 +796,7 @@ Y_UNIT_TEST_SUITE(DataStreams) {
Cerr << "PutRecordsResponse = " << result.GetResult().DebugString() << Endl;
UNIT_ASSERT_VALUES_EQUAL(result.GetResult().failed_record_count(), 0);
UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records_size(), records.size());
- UNIT_ASSERT_VALUES_EQUAL(result.GetResult().encryption_type(), YDS_V1::EncryptionType::NONE);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetResult().encryption_type(), YDS_V1::EncryptionType::NONE);
TString dataStr = "9876543210";
auto putRecordResult = client.PutRecord(streamPath, {dataStr, dataStr, ""}).ExtractValueSync();
@@ -805,29 +805,29 @@ Y_UNIT_TEST_SUITE(DataStreams) {
Cerr << "PutRecord response = " << putRecordResult.GetResult().DebugString() << Endl;
UNIT_ASSERT_VALUES_EQUAL(putRecordResult.GetResult().shard_id(), "shard-000004");
UNIT_ASSERT_VALUES_EQUAL(putRecordResult.GetResult().sequence_number(), "7");
- UNIT_ASSERT_VALUES_EQUAL(putRecordResult.GetResult().encryption_type(),
- YDS_V1::EncryptionType::NONE);
+ UNIT_ASSERT_VALUES_EQUAL(putRecordResult.GetResult().encryption_type(),
+ YDS_V1::EncryptionType::NONE);
}
}
Y_UNIT_TEST(TestPutEmptyMessage) {
TInsecureDatastreamsTestServer testServer;
- const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME;
+ const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME;
SET_YDS_LOCALS;
{
- auto result = testServer.DataStreamsClient->CreateStream(streamName,
- NYDS_V1::TCreateStreamSettings().ShardCount(5)).ExtractValueSync();
+ auto result = testServer.DataStreamsClient->CreateStream(streamName,
+ NYDS_V1::TCreateStreamSettings().ShardCount(5)).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
}
kikimr->GetRuntime()->SetLogPriority(NKikimrServices::PQ_READ_PROXY, NLog::EPriority::PRI_DEBUG);
kikimr->GetRuntime()->SetLogPriority(NKikimrServices::PQ_WRITE_PROXY, NLog::EPriority::PRI_DEBUG);
- NYDS_V1::TDataStreamsClient client(*driver, TCommonClientSettings().AuthToken("user2@builtin"));
+ NYDS_V1::TDataStreamsClient client(*driver, TCommonClientSettings().AuthToken("user2@builtin"));
- auto putRecordResult = client.PutRecord("/Root/" + streamName, {"", "key", ""}).ExtractValueSync();
+ auto putRecordResult = client.PutRecord("/Root/" + streamName, {"", "key", ""}).ExtractValueSync();
Cerr << putRecordResult.GetResult().DebugString() << Endl;
UNIT_ASSERT_VALUES_EQUAL(putRecordResult.IsTransportError(), false);
@@ -837,18 +837,18 @@ Y_UNIT_TEST_SUITE(DataStreams) {
NYdb::NPersQueue::TPersQueueClient pqClient(*driver);
{
- auto result = testServer.DataStreamsClient->RegisterStreamConsumer(streamName, "user1", NYDS_V1::TRegisterStreamConsumerSettings()).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- UNIT_ASSERT_VALUES_EQUAL(result.GetResult().consumer().consumer_name(), "user1");
- UNIT_ASSERT_VALUES_EQUAL(result.GetResult().consumer().consumer_status(),
- YDS_V1::ConsumerDescription_ConsumerStatus_ACTIVE);
+ auto result = testServer.DataStreamsClient->RegisterStreamConsumer(streamName, "user1", NYDS_V1::TRegisterStreamConsumerSettings()).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetResult().consumer().consumer_name(), "user1");
+ UNIT_ASSERT_VALUES_EQUAL(result.GetResult().consumer().consumer_status(),
+ YDS_V1::ConsumerDescription_ConsumerStatus_ACTIVE);
}
auto session = pqClient.CreateReadSession(NYdb::NPersQueue::TReadSessionSettings()
.ConsumerName("user1")
.DisableClusterDiscovery(true)
- .AppendTopics(NYdb::NPersQueue::TTopicReadSettings().Path("/Root/" + streamName)));
+ .AppendTopics(NYdb::NPersQueue::TTopicReadSettings().Path("/Root/" + streamName)));
while (true) {
auto event = session->GetEvent(true);
@@ -875,760 +875,760 @@ Y_UNIT_TEST_SUITE(DataStreams) {
Y_UNIT_TEST(TestListStreamConsumers) {
TInsecureDatastreamsTestServer testServer;
- const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME;
- // Create stream -> OK
- {
- auto result = testServer.DataStreamsClient->CreateStream(streamName,
- NYDS_V1::TCreateStreamSettings().ShardCount(5)).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- }
-
- // List stream consumers -> OK
- {
- auto result = testServer.DataStreamsClient->ListStreamConsumers(streamName,
- NYDS_V1::TListStreamConsumersSettings().MaxResults(100)).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- UNIT_ASSERT_VALUES_EQUAL(result.GetResult().consumers().size(), 0);
- }
-
- // List stream consumers more than allowed -> get BAD_REQUEST
- {
- auto result = testServer.DataStreamsClient->ListStreamConsumers(streamName,
- NYDS_V1::TListStreamConsumersSettings().MaxResults(10001)).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST);
- }
-
- // List stream consumers less than allowed -> get BAD_REQUEST
- {
- auto result = testServer.DataStreamsClient->ListStreamConsumers(streamName,
- NYDS_V1::TListStreamConsumersSettings().MaxResults(0)).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST);
- }
-
- // List not created stream -> get SCHEME_ERROR
- {
- auto result = testServer.DataStreamsClient->ListStreamConsumers(streamName + "_XXX",
- NYDS_V1::TListStreamConsumersSettings().MaxResults(100)).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR);
- }
-
- // Deregister unregistered consumer -> get NOT_FOUND
- {
- auto result = testServer.DataStreamsClient->DeregisterStreamConsumer(streamName, "user1",
- NYDS_V1::TDeregisterStreamConsumerSettings()).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::NOT_FOUND);
- }
-
- // Register consumer 1 -> OK
- {
- auto result = testServer.DataStreamsClient->RegisterStreamConsumer(streamName, "user1",
- NYDS_V1::TRegisterStreamConsumerSettings()).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- UNIT_ASSERT_VALUES_EQUAL(result.GetResult().consumer().consumer_name(), "user1");
- UNIT_ASSERT_VALUES_EQUAL(result.GetResult().consumer().consumer_status(),
- YDS_V1::ConsumerDescription_ConsumerStatus_ACTIVE);
- }
-
- // Register consumer 2 -> OK
- {
- auto result = testServer.DataStreamsClient->RegisterStreamConsumer(streamName, "user2",
- NYDS_V1::TRegisterStreamConsumerSettings()).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- UNIT_ASSERT_VALUES_EQUAL(result.GetResult().consumer().consumer_name(), "user2");
- }
-
- // List stream consumers -> OK, get 2
- {
- auto result = testServer.DataStreamsClient->ListStreamConsumers(streamName,
- NYDS_V1::TListStreamConsumersSettings().MaxResults(100)).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- UNIT_ASSERT_VALUES_EQUAL(result.GetResult().consumers().size(), 2);
- }
-
- // Deregister consumer 2 -> OK
- {
- auto result = testServer.DataStreamsClient->DeregisterStreamConsumer(streamName, "user2",
- NYDS_V1::TDeregisterStreamConsumerSettings()).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- }
-
- // List stream consumers -> OK, there's only 1 now
- {
- auto result = testServer.DataStreamsClient->ListStreamConsumers(streamName,
- NYDS_V1::TListStreamConsumersSettings().MaxResults(100)).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- UNIT_ASSERT_VALUES_EQUAL(result.GetResult().consumers().size(), 1);
- }
-
- // Register stream consumer -> OK
- {
- auto result = testServer.DataStreamsClient->RegisterStreamConsumer(streamName, "user2",
- NYDS_V1::TRegisterStreamConsumerSettings()).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- UNIT_ASSERT_VALUES_EQUAL(result.GetResult().consumer().consumer_name(), "user2");
- }
-
- // Add already added consumer -> get ALREADY_EXISTS
- {
- auto result = testServer.DataStreamsClient->RegisterStreamConsumer(streamName, "user2",
- NYDS_V1::TRegisterStreamConsumerSettings()).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::ALREADY_EXISTS);
- }
-
- // List stream consumers with pages of size 1 -> OK
- {
- auto result = testServer.DataStreamsClient->ListStreamConsumers(streamName,
- NYDS_V1::TListStreamConsumersSettings().MaxResults(1)).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- UNIT_ASSERT_VALUES_EQUAL(result.GetResult().consumers().size(), 1);
- UNIT_ASSERT_VALUES_UNEQUAL(result.GetResult().next_token().size(), 0);
-
- auto nextToken = result.GetResult().next_token();
- result = testServer.DataStreamsClient->ListStreamConsumers(streamName,
- NYDS_V1::TListStreamConsumersSettings().NextToken(nextToken)).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- UNIT_ASSERT_VALUES_EQUAL(result.GetResult().consumers().size(), 1);
- UNIT_ASSERT_VALUES_EQUAL(result.GetResult().next_token().size(), 0);
- }
-
- // List stream consumers with page size 1, but then provide corrupted or expired token
- {
- auto result = testServer.DataStreamsClient->ListStreamConsumers(streamName,
- NYDS_V1::TListStreamConsumersSettings().MaxResults(1)).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- UNIT_ASSERT_VALUES_EQUAL(result.GetResult().consumers().size(), 1);
- UNIT_ASSERT_VALUES_UNEQUAL(result.GetResult().next_token().size(), 0);
-
- auto makeNextToken = [streamName](ui64 timestamp) {
- NKikimrPQ::TYdsNextToken protoNextToken;
- protoNextToken.SetStreamArn(streamName);
- protoNextToken.SetAlreadyRead(1);
- protoNextToken.SetMaxResults(1);
- protoNextToken.SetCreationTimestamp(timestamp);
- TString stringNextToken;
+ const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME;
+ // Create stream -> OK
+ {
+ auto result = testServer.DataStreamsClient->CreateStream(streamName,
+ NYDS_V1::TCreateStreamSettings().ShardCount(5)).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ }
+
+ // List stream consumers -> OK
+ {
+ auto result = testServer.DataStreamsClient->ListStreamConsumers(streamName,
+ NYDS_V1::TListStreamConsumersSettings().MaxResults(100)).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetResult().consumers().size(), 0);
+ }
+
+ // List stream consumers more than allowed -> get BAD_REQUEST
+ {
+ auto result = testServer.DataStreamsClient->ListStreamConsumers(streamName,
+ NYDS_V1::TListStreamConsumersSettings().MaxResults(10001)).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST);
+ }
+
+ // List stream consumers less than allowed -> get BAD_REQUEST
+ {
+ auto result = testServer.DataStreamsClient->ListStreamConsumers(streamName,
+ NYDS_V1::TListStreamConsumersSettings().MaxResults(0)).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST);
+ }
+
+ // List not created stream -> get SCHEME_ERROR
+ {
+ auto result = testServer.DataStreamsClient->ListStreamConsumers(streamName + "_XXX",
+ NYDS_V1::TListStreamConsumersSettings().MaxResults(100)).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR);
+ }
+
+ // Deregister unregistered consumer -> get NOT_FOUND
+ {
+ auto result = testServer.DataStreamsClient->DeregisterStreamConsumer(streamName, "user1",
+ NYDS_V1::TDeregisterStreamConsumerSettings()).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::NOT_FOUND);
+ }
+
+ // Register consumer 1 -> OK
+ {
+ auto result = testServer.DataStreamsClient->RegisterStreamConsumer(streamName, "user1",
+ NYDS_V1::TRegisterStreamConsumerSettings()).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetResult().consumer().consumer_name(), "user1");
+ UNIT_ASSERT_VALUES_EQUAL(result.GetResult().consumer().consumer_status(),
+ YDS_V1::ConsumerDescription_ConsumerStatus_ACTIVE);
+ }
+
+ // Register consumer 2 -> OK
+ {
+ auto result = testServer.DataStreamsClient->RegisterStreamConsumer(streamName, "user2",
+ NYDS_V1::TRegisterStreamConsumerSettings()).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetResult().consumer().consumer_name(), "user2");
+ }
+
+ // List stream consumers -> OK, get 2
+ {
+ auto result = testServer.DataStreamsClient->ListStreamConsumers(streamName,
+ NYDS_V1::TListStreamConsumersSettings().MaxResults(100)).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetResult().consumers().size(), 2);
+ }
+
+ // Deregister consumer 2 -> OK
+ {
+ auto result = testServer.DataStreamsClient->DeregisterStreamConsumer(streamName, "user2",
+ NYDS_V1::TDeregisterStreamConsumerSettings()).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ }
+
+ // List stream consumers -> OK, there's only 1 now
+ {
+ auto result = testServer.DataStreamsClient->ListStreamConsumers(streamName,
+ NYDS_V1::TListStreamConsumersSettings().MaxResults(100)).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetResult().consumers().size(), 1);
+ }
+
+ // Register stream consumer -> OK
+ {
+ auto result = testServer.DataStreamsClient->RegisterStreamConsumer(streamName, "user2",
+ NYDS_V1::TRegisterStreamConsumerSettings()).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetResult().consumer().consumer_name(), "user2");
+ }
+
+ // Add already added consumer -> get ALREADY_EXISTS
+ {
+ auto result = testServer.DataStreamsClient->RegisterStreamConsumer(streamName, "user2",
+ NYDS_V1::TRegisterStreamConsumerSettings()).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::ALREADY_EXISTS);
+ }
+
+ // List stream consumers with pages of size 1 -> OK
+ {
+ auto result = testServer.DataStreamsClient->ListStreamConsumers(streamName,
+ NYDS_V1::TListStreamConsumersSettings().MaxResults(1)).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetResult().consumers().size(), 1);
+ UNIT_ASSERT_VALUES_UNEQUAL(result.GetResult().next_token().size(), 0);
+
+ auto nextToken = result.GetResult().next_token();
+ result = testServer.DataStreamsClient->ListStreamConsumers(streamName,
+ NYDS_V1::TListStreamConsumersSettings().NextToken(nextToken)).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetResult().consumers().size(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetResult().next_token().size(), 0);
+ }
+
+ // List stream consumers with page size 1, but then provide corrupted or expired token
+ {
+ auto result = testServer.DataStreamsClient->ListStreamConsumers(streamName,
+ NYDS_V1::TListStreamConsumersSettings().MaxResults(1)).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetResult().consumers().size(), 1);
+ UNIT_ASSERT_VALUES_UNEQUAL(result.GetResult().next_token().size(), 0);
+
+ auto makeNextToken = [streamName](ui64 timestamp) {
+ NKikimrPQ::TYdsNextToken protoNextToken;
+ protoNextToken.SetStreamArn(streamName);
+ protoNextToken.SetAlreadyRead(1);
+ protoNextToken.SetMaxResults(1);
+ protoNextToken.SetCreationTimestamp(timestamp);
+ TString stringNextToken;
Y_PROTOBUF_SUPPRESS_NODISCARD protoNextToken.SerializeToString(&stringNextToken);
- TString encodedNextToken;
- Base64Encode(stringNextToken, encodedNextToken);
- return encodedNextToken;
- };
-
- auto nextToken = makeNextToken(TInstant::Now().MilliSeconds() - 300001);
- result = testServer.DataStreamsClient->ListStreamConsumers(streamName,
- NYDS_V1::TListStreamConsumersSettings().NextToken(nextToken)).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST);
-
- nextToken = makeNextToken(TInstant::Now().MilliSeconds() + 1000);
- result = testServer.DataStreamsClient->ListStreamConsumers(streamName,
- NYDS_V1::TListStreamConsumersSettings().NextToken(nextToken)).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST);
-
- nextToken = makeNextToken(0);
- result = testServer.DataStreamsClient->ListStreamConsumers(streamName,
- NYDS_V1::TListStreamConsumersSettings().NextToken(nextToken)).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST);
-
- result = testServer.DataStreamsClient->ListStreamConsumers(streamName,
- NYDS_V1::TListStreamConsumersSettings().NextToken("some_garbage")).ExtractValueSync();
- result = testServer.DataStreamsClient->ListStreamConsumers(streamName,
- NYDS_V1::TListStreamConsumersSettings().NextToken("some_garbage")).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST);
- }
-
- // Delete stream -> OK
- {
- auto result = testServer.DataStreamsClient->DeleteStream(streamName,
- NYDS_V1::TDeleteStreamSettings().EnforceConsumerDeletion(true)).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- }
-
- // List consumers of deleted stream -> get BAD_REQUEST
- {
- auto result = testServer.DataStreamsClient->ListStreamConsumers(streamName).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST);
- }
- }
-
+ TString encodedNextToken;
+ Base64Encode(stringNextToken, encodedNextToken);
+ return encodedNextToken;
+ };
+
+ auto nextToken = makeNextToken(TInstant::Now().MilliSeconds() - 300001);
+ result = testServer.DataStreamsClient->ListStreamConsumers(streamName,
+ NYDS_V1::TListStreamConsumersSettings().NextToken(nextToken)).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST);
+
+ nextToken = makeNextToken(TInstant::Now().MilliSeconds() + 1000);
+ result = testServer.DataStreamsClient->ListStreamConsumers(streamName,
+ NYDS_V1::TListStreamConsumersSettings().NextToken(nextToken)).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST);
+
+ nextToken = makeNextToken(0);
+ result = testServer.DataStreamsClient->ListStreamConsumers(streamName,
+ NYDS_V1::TListStreamConsumersSettings().NextToken(nextToken)).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST);
+
+ result = testServer.DataStreamsClient->ListStreamConsumers(streamName,
+ NYDS_V1::TListStreamConsumersSettings().NextToken("some_garbage")).ExtractValueSync();
+ result = testServer.DataStreamsClient->ListStreamConsumers(streamName,
+ NYDS_V1::TListStreamConsumersSettings().NextToken("some_garbage")).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST);
+ }
+
+ // Delete stream -> OK
+ {
+ auto result = testServer.DataStreamsClient->DeleteStream(streamName,
+ NYDS_V1::TDeleteStreamSettings().EnforceConsumerDeletion(true)).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ }
+
+ // List consumers of deleted stream -> get BAD_REQUEST
+ {
+ auto result = testServer.DataStreamsClient->ListStreamConsumers(streamName).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST);
+ }
+ }
+
Y_UNIT_TEST(TestGetShardIterator) {
TInsecureDatastreamsTestServer testServer;
- const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME;
- {
- auto result = testServer.DataStreamsClient->CreateStream(streamName, \
- NYDS_V1::TCreateStreamSettings().ShardCount(5)
- ).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- }
-
- {
- std::vector<NYDS_V1::TDataRecord> records;
- for (ui32 i = 1; i <= 30; ++i) {
- TString data = Sprintf("%04u", i);
- records.push_back({data, data, ""});
- }
-
- auto result = testServer.DataStreamsClient->PutRecords(streamName, records).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- if (result.GetStatus() != EStatus::SUCCESS) {
- result.GetIssues().PrintTo(Cerr);
- }
- }
-
- {
- auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000001",
- YDS_V1::ShardIteratorType::LATEST).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- }
-
- {
- auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000010",
- YDS_V1::ShardIteratorType::LATEST).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST);
- }
-
- {
- auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000001",
- YDS_V1::ShardIteratorType::AT_SEQUENCE_NUMBER,
- NYDS_V1::TGetShardIteratorSettings().StartingSequenceNumber("0")
- ).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- }
-
- {
- auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000001",
- YDS_V1::ShardIteratorType::AFTER_SEQUENCE_NUMBER,
- NYDS_V1::TGetShardIteratorSettings().StartingSequenceNumber("0")
- ).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- }
-
- {
- auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000001",
- YDS_V1::ShardIteratorType::AT_SEQUENCE_NUMBER,
- NYDS_V1::TGetShardIteratorSettings().StartingSequenceNumber("100")
- ).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- }
-
- {
- auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000001",
- YDS_V1::ShardIteratorType::AT_SEQUENCE_NUMBER,
- NYDS_V1::TGetShardIteratorSettings().StartingSequenceNumber("001122")
- ).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- }
-
- {
- auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000001",
- YDS_V1::ShardIteratorType::AT_SEQUENCE_NUMBER,
- NYDS_V1::TGetShardIteratorSettings().StartingSequenceNumber("garbage_value")
- ).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST);
- }
-
- {
- auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000001",
- YDS_V1::ShardIteratorType::AT_TIMESTAMP,
- NYDS_V1::TGetShardIteratorSettings().Timestamp(TInstant::Now().MilliSeconds() * 0.99)
- ).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
-
- NKikimrPQ::TYdsShardIterator protoShardIterator;
- TString decoded;
- Base64Decode(result.GetResult().shard_iterator(), decoded);
- Y_VERIFY(protoShardIterator.ParseFromString(decoded));
- UNIT_ASSERT_VALUES_EQUAL(protoShardIterator.GetStreamName(), "/Root/" + streamName);
- }
-
- {
- auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000001",
- YDS_V1::ShardIteratorType::AT_TIMESTAMP,
- NYDS_V1::TGetShardIteratorSettings().Timestamp(TInstant::Now().MilliSeconds() * 1.3)
- ).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST);
- }
- }
-
+ const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME;
+ {
+ auto result = testServer.DataStreamsClient->CreateStream(streamName, \
+ NYDS_V1::TCreateStreamSettings().ShardCount(5)
+ ).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ }
+
+ {
+ std::vector<NYDS_V1::TDataRecord> records;
+ for (ui32 i = 1; i <= 30; ++i) {
+ TString data = Sprintf("%04u", i);
+ records.push_back({data, data, ""});
+ }
+
+ auto result = testServer.DataStreamsClient->PutRecords(streamName, records).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ if (result.GetStatus() != EStatus::SUCCESS) {
+ result.GetIssues().PrintTo(Cerr);
+ }
+ }
+
+ {
+ auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000001",
+ YDS_V1::ShardIteratorType::LATEST).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ }
+
+ {
+ auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000010",
+ YDS_V1::ShardIteratorType::LATEST).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST);
+ }
+
+ {
+ auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000001",
+ YDS_V1::ShardIteratorType::AT_SEQUENCE_NUMBER,
+ NYDS_V1::TGetShardIteratorSettings().StartingSequenceNumber("0")
+ ).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ }
+
+ {
+ auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000001",
+ YDS_V1::ShardIteratorType::AFTER_SEQUENCE_NUMBER,
+ NYDS_V1::TGetShardIteratorSettings().StartingSequenceNumber("0")
+ ).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ }
+
+ {
+ auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000001",
+ YDS_V1::ShardIteratorType::AT_SEQUENCE_NUMBER,
+ NYDS_V1::TGetShardIteratorSettings().StartingSequenceNumber("100")
+ ).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ }
+
+ {
+ auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000001",
+ YDS_V1::ShardIteratorType::AT_SEQUENCE_NUMBER,
+ NYDS_V1::TGetShardIteratorSettings().StartingSequenceNumber("001122")
+ ).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ }
+
+ {
+ auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000001",
+ YDS_V1::ShardIteratorType::AT_SEQUENCE_NUMBER,
+ NYDS_V1::TGetShardIteratorSettings().StartingSequenceNumber("garbage_value")
+ ).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST);
+ }
+
+ {
+ auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000001",
+ YDS_V1::ShardIteratorType::AT_TIMESTAMP,
+ NYDS_V1::TGetShardIteratorSettings().Timestamp(TInstant::Now().MilliSeconds() * 0.99)
+ ).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+
+ NKikimrPQ::TYdsShardIterator protoShardIterator;
+ TString decoded;
+ Base64Decode(result.GetResult().shard_iterator(), decoded);
+ Y_VERIFY(protoShardIterator.ParseFromString(decoded));
+ UNIT_ASSERT_VALUES_EQUAL(protoShardIterator.GetStreamName(), "/Root/" + streamName);
+ }
+
+ {
+ auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000001",
+ YDS_V1::ShardIteratorType::AT_TIMESTAMP,
+ NYDS_V1::TGetShardIteratorSettings().Timestamp(TInstant::Now().MilliSeconds() * 1.3)
+ ).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST);
+ }
+ }
+
Y_UNIT_TEST(TestGetRecordsStreamWithSingleShard) {
TInsecureDatastreamsTestServer testServer;
- const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME;
- {
- auto result = testServer.DataStreamsClient->CreateStream(streamName, NYDS_V1::TCreateStreamSettings().ShardCount(1)).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- }
-
- const ui32 recordsCount = 30;
- std::vector<NYDS_V1::TDataRecord> records;
- for (ui32 i = 1; i <= recordsCount; ++i) {
- TString data = Sprintf("%04u", i);
- records.push_back({data, data, ""});
- }
-
- {
- auto result = testServer.DataStreamsClient->PutRecords(streamName, records).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- if (result.GetStatus() != EStatus::SUCCESS) {
- result.GetIssues().PrintTo(Cerr);
- }
- }
-
- TString shardIterator;
- {
- auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000000",
- YDS_V1::ShardIteratorType::LATEST).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- shardIterator = result.GetResult().shard_iterator();
- }
-
- {
+ const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME;
+ {
+ auto result = testServer.DataStreamsClient->CreateStream(streamName, NYDS_V1::TCreateStreamSettings().ShardCount(1)).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ }
+
+ const ui32 recordsCount = 30;
+ std::vector<NYDS_V1::TDataRecord> records;
+ for (ui32 i = 1; i <= recordsCount; ++i) {
+ TString data = Sprintf("%04u", i);
+ records.push_back({data, data, ""});
+ }
+
+ {
+ auto result = testServer.DataStreamsClient->PutRecords(streamName, records).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ if (result.GetStatus() != EStatus::SUCCESS) {
+ result.GetIssues().PrintTo(Cerr);
+ }
+ }
+
+ TString shardIterator;
+ {
+ auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000000",
+ YDS_V1::ShardIteratorType::LATEST).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ shardIterator = result.GetResult().shard_iterator();
+ }
+
+ {
auto result = testServer.DataStreamsClient->GetRecords(shardIterator).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), 0);
- }
-
- {
- auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000000",
- YDS_V1::ShardIteratorType::TRIM_HORIZON).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- shardIterator = result.GetResult().shard_iterator();
- }
-
- {
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), 0);
+ }
+
+ {
+ auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000000",
+ YDS_V1::ShardIteratorType::TRIM_HORIZON).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ shardIterator = result.GetResult().shard_iterator();
+ }
+
+ {
auto result = testServer.DataStreamsClient->GetRecords(shardIterator).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), recordsCount);
- }
-
- {
- auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000000",
- YDS_V1::ShardIteratorType::AT_SEQUENCE_NUMBER,
- NYDS_V1::TGetShardIteratorSettings().StartingSequenceNumber("0002")
- ).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- shardIterator = result.GetResult().shard_iterator();
- }
-
- {
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), recordsCount);
+ }
+
+ {
+ auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000000",
+ YDS_V1::ShardIteratorType::AT_SEQUENCE_NUMBER,
+ NYDS_V1::TGetShardIteratorSettings().StartingSequenceNumber("0002")
+ ).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ shardIterator = result.GetResult().shard_iterator();
+ }
+
+ {
auto result = testServer.DataStreamsClient->GetRecords(shardIterator).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), recordsCount - 2);
- }
-
- {
- auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000000",
- YDS_V1::ShardIteratorType::AFTER_SEQUENCE_NUMBER,
- NYDS_V1::TGetShardIteratorSettings().StartingSequenceNumber("0002")
- ).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- shardIterator = result.GetResult().shard_iterator();
- }
-
- {
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), recordsCount - 2);
+ }
+
+ {
+ auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000000",
+ YDS_V1::ShardIteratorType::AFTER_SEQUENCE_NUMBER,
+ NYDS_V1::TGetShardIteratorSettings().StartingSequenceNumber("0002")
+ ).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ shardIterator = result.GetResult().shard_iterator();
+ }
+
+ {
auto result = testServer.DataStreamsClient->GetRecords(shardIterator).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), recordsCount - 3);
- }
-
-
- {
- auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000000",
- YDS_V1::ShardIteratorType::LATEST).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- shardIterator = result.GetResult().shard_iterator();
- }
-
- {
- auto result = testServer.DataStreamsClient->PutRecords(streamName, records).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- if (result.GetStatus() != EStatus::SUCCESS) {
- result.GetIssues().PrintTo(Cerr);
- }
- }
-
- {
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), recordsCount - 3);
+ }
+
+
+ {
+ auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000000",
+ YDS_V1::ShardIteratorType::LATEST).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ shardIterator = result.GetResult().shard_iterator();
+ }
+
+ {
+ auto result = testServer.DataStreamsClient->PutRecords(streamName, records).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ if (result.GetStatus() != EStatus::SUCCESS) {
+ result.GetIssues().PrintTo(Cerr);
+ }
+ }
+
+ {
auto result = testServer.DataStreamsClient->GetRecords(shardIterator).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), recordsCount);
- }
-
- }
-
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), recordsCount);
+ }
+
+ }
+
Y_UNIT_TEST(TestGetRecordsWithoutPermission) {
TInsecureDatastreamsTestServer testServer;
- const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME;
+ const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME;
SET_YDS_LOCALS;
- {
- auto result = testServer.DataStreamsClient->CreateStream("/Root/" + streamName, NYDS_V1::TCreateStreamSettings().ShardCount(1)).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- }
-
- std::string data;
- data.resize(1 << 10);
- std::iota(data.begin(), data.end(), 1);
- TString id{"0000"};
- {
- NYDS_V1::TDataRecord dataRecord{{data.begin(), data.end()}, id, ""};
- auto result = testServer.DataStreamsClient->PutRecord(streamName, dataRecord).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- }
-
- NYDS_V1::TDataStreamsClient client(*driver, TCommonClientSettings().AuthToken("user2@builtin"));
+ {
+ auto result = testServer.DataStreamsClient->CreateStream("/Root/" + streamName, NYDS_V1::TCreateStreamSettings().ShardCount(1)).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ }
+
+ std::string data;
+ data.resize(1 << 10);
+ std::iota(data.begin(), data.end(), 1);
+ TString id{"0000"};
+ {
+ NYDS_V1::TDataRecord dataRecord{{data.begin(), data.end()}, id, ""};
+ auto result = testServer.DataStreamsClient->PutRecord(streamName, dataRecord).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ }
+
+ NYDS_V1::TDataStreamsClient client(*driver, TCommonClientSettings().AuthToken("user2@builtin"));
NYdb::NScheme::TSchemeClient schemeClient(*driver);
-
- TString shardIterator;
- {
- auto result = client.GetShardIterator(
- streamName, "shard-000000",
- YDS_V1::ShardIteratorType::TRIM_HORIZON
- ).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR);
- }
-
- {
- auto result = client.GetShardIterator(
- "/Root/" + streamName, "shard-000000",
- YDS_V1::ShardIteratorType::TRIM_HORIZON
- ).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR);
- }
-
- {
- NYdb::NScheme::TPermissions permissions("user2@builtin", {"ydb.generic.read"});
- auto result = schemeClient.ModifyPermissions("/Root",
- NYdb::NScheme::TModifyPermissionsSettings().AddGrantPermissions(permissions)
- ).ExtractValueSync();
- UNIT_ASSERT(result.IsSuccess());
- }
-
- {
- auto result = client.GetShardIterator(
- streamName, "shard-000000",
- YDS_V1::ShardIteratorType::TRIM_HORIZON
- ).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- shardIterator = result.GetResult().shard_iterator();
- }
-
- {
- auto result = client.GetRecords(shardIterator,
- NYDS_V1::TGetRecordsSettings().Limit(2)).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), 1);
- UNIT_ASSERT_VALUES_EQUAL(std::stoi(result.GetResult().records().begin()->sequence_number()), 0);
- }
- }
-
+
+ TString shardIterator;
+ {
+ auto result = client.GetShardIterator(
+ streamName, "shard-000000",
+ YDS_V1::ShardIteratorType::TRIM_HORIZON
+ ).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR);
+ }
+
+ {
+ auto result = client.GetShardIterator(
+ "/Root/" + streamName, "shard-000000",
+ YDS_V1::ShardIteratorType::TRIM_HORIZON
+ ).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SCHEME_ERROR);
+ }
+
+ {
+ NYdb::NScheme::TPermissions permissions("user2@builtin", {"ydb.generic.read"});
+ auto result = schemeClient.ModifyPermissions("/Root",
+ NYdb::NScheme::TModifyPermissionsSettings().AddGrantPermissions(permissions)
+ ).ExtractValueSync();
+ UNIT_ASSERT(result.IsSuccess());
+ }
+
+ {
+ auto result = client.GetShardIterator(
+ streamName, "shard-000000",
+ YDS_V1::ShardIteratorType::TRIM_HORIZON
+ ).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ shardIterator = result.GetResult().shard_iterator();
+ }
+
+ {
+ auto result = client.GetRecords(shardIterator,
+ NYDS_V1::TGetRecordsSettings().Limit(2)).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(std::stoi(result.GetResult().records().begin()->sequence_number()), 0);
+ }
+ }
+
Y_UNIT_TEST(TestGetRecords1MBMessagesOneByOneBySeqNo) {
TInsecureDatastreamsTestServer testServer;
- const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME;
- {
- auto result = testServer.DataStreamsClient->CreateStream(streamName,
- NYDS_V1::TCreateStreamSettings().ShardCount(1)).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- }
-
- std::vector<std::string> put_data;
- const ui32 recordsCount = 10;
- {
- std::string data;
- data.resize(1 << 15);
- std::iota(data.begin(), data.end(), 1);
- std::random_device rd;
- std::mt19937 generator{rd()};
- for (ui32 i = 1; i <= recordsCount; ++i) {
- std::shuffle(data.begin(), data.end(), generator);
- put_data.push_back(data);
- {
- TString id = Sprintf("%04u", i);
- NYDS_V1::TDataRecord dataRecord{{data.begin(), data.end()}, id, ""};
- auto result = testServer.DataStreamsClient->PutRecord(streamName, dataRecord).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- }
- Sleep(TDuration::Seconds(1));
- }
- }
-
- for (ui32 i = 0; i < recordsCount; ++i) {
- TString shardIterator;
- {
- TString id = Sprintf("%04u", i);
- auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000000",
- YDS_V1::ShardIteratorType::AT_SEQUENCE_NUMBER,
- NYDS_V1::TGetShardIteratorSettings().StartingSequenceNumber(id)).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- shardIterator = result.GetResult().shard_iterator();
- }
-
- {
+ const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME;
+ {
+ auto result = testServer.DataStreamsClient->CreateStream(streamName,
+ NYDS_V1::TCreateStreamSettings().ShardCount(1)).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ }
+
+ std::vector<std::string> put_data;
+ const ui32 recordsCount = 10;
+ {
+ std::string data;
+ data.resize(1 << 15);
+ std::iota(data.begin(), data.end(), 1);
+ std::random_device rd;
+ std::mt19937 generator{rd()};
+ for (ui32 i = 1; i <= recordsCount; ++i) {
+ std::shuffle(data.begin(), data.end(), generator);
+ put_data.push_back(data);
+ {
+ TString id = Sprintf("%04u", i);
+ NYDS_V1::TDataRecord dataRecord{{data.begin(), data.end()}, id, ""};
+ auto result = testServer.DataStreamsClient->PutRecord(streamName, dataRecord).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ }
+ Sleep(TDuration::Seconds(1));
+ }
+ }
+
+ for (ui32 i = 0; i < recordsCount; ++i) {
+ TString shardIterator;
+ {
+ TString id = Sprintf("%04u", i);
+ auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000000",
+ YDS_V1::ShardIteratorType::AT_SEQUENCE_NUMBER,
+ NYDS_V1::TGetShardIteratorSettings().StartingSequenceNumber(id)).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ shardIterator = result.GetResult().shard_iterator();
+ }
+
+ {
auto result = testServer.DataStreamsClient->GetRecords(shardIterator,
- NYDS_V1::TGetRecordsSettings().Limit(1)).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), 1);
- UNIT_ASSERT_VALUES_EQUAL(
- std::stoi(result.GetResult().records().begin()->sequence_number()), i);
- UNIT_ASSERT_VALUES_EQUAL(MD5::Calc(result.GetResult().records().begin()->data()),
- MD5::Calc(put_data[i]));
- }
- }
- }
-
+ NYDS_V1::TGetRecordsSettings().Limit(1)).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(
+ std::stoi(result.GetResult().records().begin()->sequence_number()), i);
+ UNIT_ASSERT_VALUES_EQUAL(MD5::Calc(result.GetResult().records().begin()->data()),
+ MD5::Calc(put_data[i]));
+ }
+ }
+ }
+
Y_UNIT_TEST(TestGetRecords1MBMessagesOneByOneByTS) {
TInsecureDatastreamsTestServer testServer;
- const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME;
- {
- auto result = testServer.DataStreamsClient->CreateStream(streamName,
- NYDS_V1::TCreateStreamSettings().ShardCount(1)).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- }
-
- const ui32 recordsCount = 24;
- std::vector<ui64> timestamps;
- {
- std::string data;
- // data.resize((1 << 21) + 2);
- data.resize(1 << 15);
- std::iota(data.begin(), data.end(), 1);
- std::random_device rd;
- std::mt19937 generator{rd()};
- for (ui32 i = 1; i <= recordsCount; ++i) {
- std::shuffle(data.begin(), data.end(), generator);
- {
- TString id = Sprintf("%04u", i);
- NYDS_V1::TDataRecord dataRecord{{data.begin(), data.end()}, id, ""};
- timestamps.push_back(TInstant::Now().MilliSeconds());
- auto result = testServer.DataStreamsClient->PutRecord(streamName, dataRecord).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- }
- Sleep(TDuration::Seconds(1));
- }
- }
-
- for (ui32 i = 0; i < recordsCount; ++i) {
- TString shardIterator;
- {
- TString id = Sprintf("%04u", i);
- auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000000",
- YDS_V1::ShardIteratorType::AT_TIMESTAMP,
- NYDS_V1::TGetShardIteratorSettings().Timestamp(timestamps[i])).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- shardIterator = result.GetResult().shard_iterator();
- }
-
- {
+ const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME;
+ {
+ auto result = testServer.DataStreamsClient->CreateStream(streamName,
+ NYDS_V1::TCreateStreamSettings().ShardCount(1)).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ }
+
+ const ui32 recordsCount = 24;
+ std::vector<ui64> timestamps;
+ {
+ std::string data;
+ // data.resize((1 << 21) + 2);
+ data.resize(1 << 15);
+ std::iota(data.begin(), data.end(), 1);
+ std::random_device rd;
+ std::mt19937 generator{rd()};
+ for (ui32 i = 1; i <= recordsCount; ++i) {
+ std::shuffle(data.begin(), data.end(), generator);
+ {
+ TString id = Sprintf("%04u", i);
+ NYDS_V1::TDataRecord dataRecord{{data.begin(), data.end()}, id, ""};
+ timestamps.push_back(TInstant::Now().MilliSeconds());
+ auto result = testServer.DataStreamsClient->PutRecord(streamName, dataRecord).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ }
+ Sleep(TDuration::Seconds(1));
+ }
+ }
+
+ for (ui32 i = 0; i < recordsCount; ++i) {
+ TString shardIterator;
+ {
+ TString id = Sprintf("%04u", i);
+ auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000000",
+ YDS_V1::ShardIteratorType::AT_TIMESTAMP,
+ NYDS_V1::TGetShardIteratorSettings().Timestamp(timestamps[i])).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ shardIterator = result.GetResult().shard_iterator();
+ }
+
+ {
auto result = testServer.DataStreamsClient->GetRecords(shardIterator,
- NYDS_V1::TGetRecordsSettings().Limit(1)).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), 1);
- UNIT_ASSERT_VALUES_EQUAL(std::stoi(result.GetResult().records().begin()->sequence_number()), i);
- }
- }
-
- }
-
+ NYDS_V1::TGetRecordsSettings().Limit(1)).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(std::stoi(result.GetResult().records().begin()->sequence_number()), i);
+ }
+ }
+
+ }
+
Y_UNIT_TEST(TestGetRecordsStreamWithMultipleShards) {
TInsecureDatastreamsTestServer testServer;
- const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME;
- {
- auto result = testServer.DataStreamsClient->CreateStream(streamName, NYDS_V1::TCreateStreamSettings().ShardCount(5)).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- }
-
- const ui32 recordsCount = 30;
- {
- std::vector<NYDS_V1::TDataRecord> records;
- for (ui32 i = 1; i <= recordsCount; ++i) {
- TString data = Sprintf("%04u", i);
- records.push_back({data, data, ""});
- }
-
- auto result = testServer.DataStreamsClient->PutRecords(streamName, records).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- if (result.GetStatus() != EStatus::SUCCESS) {
- result.GetIssues().PrintTo(Cerr);
- }
- }
-
- TString shardIterator;
- {
- auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000001",
- YDS_V1::ShardIteratorType::LATEST).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- shardIterator = result.GetResult().shard_iterator();
- }
-
- {
+ const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME;
+ {
+ auto result = testServer.DataStreamsClient->CreateStream(streamName, NYDS_V1::TCreateStreamSettings().ShardCount(5)).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ }
+
+ const ui32 recordsCount = 30;
+ {
+ std::vector<NYDS_V1::TDataRecord> records;
+ for (ui32 i = 1; i <= recordsCount; ++i) {
+ TString data = Sprintf("%04u", i);
+ records.push_back({data, data, ""});
+ }
+
+ auto result = testServer.DataStreamsClient->PutRecords(streamName, records).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ if (result.GetStatus() != EStatus::SUCCESS) {
+ result.GetIssues().PrintTo(Cerr);
+ }
+ }
+
+ TString shardIterator;
+ {
+ auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000001",
+ YDS_V1::ShardIteratorType::LATEST).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ shardIterator = result.GetResult().shard_iterator();
+ }
+
+ {
auto result = testServer.DataStreamsClient->GetRecords(shardIterator).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), 0);
- }
-
- {
- auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000002",
- YDS_V1::ShardIteratorType::LATEST).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- shardIterator = result.GetResult().shard_iterator();
- }
-
- {
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), 0);
+ }
+
+ {
+ auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000002",
+ YDS_V1::ShardIteratorType::LATEST).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ shardIterator = result.GetResult().shard_iterator();
+ }
+
+ {
auto result = testServer.DataStreamsClient->GetRecords(shardIterator).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), 0);
- }
-
- {
- auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000003",
- YDS_V1::ShardIteratorType::LATEST).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- shardIterator = result.GetResult().shard_iterator();
- }
-
- {
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), 0);
+ }
+
+ {
+ auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000003",
+ YDS_V1::ShardIteratorType::LATEST).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ shardIterator = result.GetResult().shard_iterator();
+ }
+
+ {
auto result = testServer.DataStreamsClient->GetRecords(shardIterator).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), 0);
- }
-
- {
- auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000002",
- YDS_V1::ShardIteratorType::LATEST).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- shardIterator = result.GetResult().shard_iterator();
- }
-
- {
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), 0);
+ }
+
+ {
+ auto result = testServer.DataStreamsClient->GetShardIterator(streamName, "shard-000002",
+ YDS_V1::ShardIteratorType::LATEST).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ shardIterator = result.GetResult().shard_iterator();
+ }
+
+ {
auto result = testServer.DataStreamsClient->GetRecords(shardIterator).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), 0);
- }
- }
-
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), 0);
+ }
+ }
+
Y_UNIT_TEST(TestListShards1Shard) {
TInsecureDatastreamsTestServer testServer;
- const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME;
- {
- auto result = testServer.DataStreamsClient->CreateStream(streamName,
- NYDS_V1::TCreateStreamSettings().ShardCount(1)).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- }
-
- {
- auto result = testServer.DataStreamsClient->ListShards(streamName, {},
- NYDS_V1::TListShardsSettings()).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- }
-
- {
- YDS_V1::ShardFilter filter;
- filter.set_type(YDS_V1::ShardFilter::FROM_TRIM_HORIZON);
- auto result = testServer.DataStreamsClient->ListShards(streamName, filter,
- NYDS_V1::TListShardsSettings()).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- UNIT_ASSERT_VALUES_EQUAL(result.GetResult().shards().size(), 1);
- UNIT_ASSERT_VALUES_EQUAL(result.GetResult().shards().begin()->shard_id(), "shard-000000");
- UNIT_ASSERT_VALUES_UNEQUAL(result.GetResult().shards().begin()->shard_id(), "shard-000001");
- }
-
- {
- YDS_V1::ShardFilter filter;
- filter.set_type(YDS_V1::ShardFilter::AT_TRIM_HORIZON);
- auto result = testServer.DataStreamsClient->ListShards(streamName, filter,
- NYDS_V1::TListShardsSettings()).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- UNIT_ASSERT_VALUES_EQUAL(result.GetResult().shards().size(), 1);
- }
-
- {
- YDS_V1::ShardFilter filter;
- filter.set_type(YDS_V1::ShardFilter::AFTER_SHARD_ID);
- filter.set_shard_id("00000");
- auto result = testServer.DataStreamsClient->ListShards(streamName, filter,
- NYDS_V1::TListShardsSettings()).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- UNIT_ASSERT_VALUES_EQUAL(result.GetResult().shards().size(), 1);
- }
-
- {
- YDS_V1::ShardFilter filter;
- filter.set_type(YDS_V1::ShardFilter::AFTER_SHARD_ID);
- auto result = testServer.DataStreamsClient->ListShards(streamName, filter,
- NYDS_V1::TListShardsSettings()).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
- UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST);
- }
- }
-
+ const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME;
+ {
+ auto result = testServer.DataStreamsClient->CreateStream(streamName,
+ NYDS_V1::TCreateStreamSettings().ShardCount(1)).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ }
+
+ {
+ auto result = testServer.DataStreamsClient->ListShards(streamName, {},
+ NYDS_V1::TListShardsSettings()).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ }
+
+ {
+ YDS_V1::ShardFilter filter;
+ filter.set_type(YDS_V1::ShardFilter::FROM_TRIM_HORIZON);
+ auto result = testServer.DataStreamsClient->ListShards(streamName, filter,
+ NYDS_V1::TListShardsSettings()).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetResult().shards().size(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetResult().shards().begin()->shard_id(), "shard-000000");
+ UNIT_ASSERT_VALUES_UNEQUAL(result.GetResult().shards().begin()->shard_id(), "shard-000001");
+ }
+
+ {
+ YDS_V1::ShardFilter filter;
+ filter.set_type(YDS_V1::ShardFilter::AT_TRIM_HORIZON);
+ auto result = testServer.DataStreamsClient->ListShards(streamName, filter,
+ NYDS_V1::TListShardsSettings()).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetResult().shards().size(), 1);
+ }
+
+ {
+ YDS_V1::ShardFilter filter;
+ filter.set_type(YDS_V1::ShardFilter::AFTER_SHARD_ID);
+ filter.set_shard_id("00000");
+ auto result = testServer.DataStreamsClient->ListShards(streamName, filter,
+ NYDS_V1::TListShardsSettings()).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetResult().shards().size(), 1);
+ }
+
+ {
+ YDS_V1::ShardFilter filter;
+ filter.set_type(YDS_V1::ShardFilter::AFTER_SHARD_ID);
+ auto result = testServer.DataStreamsClient->ListShards(streamName, filter,
+ NYDS_V1::TListShardsSettings()).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST);
+ }
+ }
+
Y_UNIT_TEST(TestUnsupported) {
TInsecureDatastreamsTestServer testServer;
- {
+ {
auto result = testServer.DataStreamsClient->DescribeLimits().ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNSUPPORTED);
diff --git a/ydb/services/datastreams/next_token.cpp b/ydb/services/datastreams/next_token.cpp
index ac8d24514c..f3edad9197 100644
--- a/ydb/services/datastreams/next_token.cpp
+++ b/ydb/services/datastreams/next_token.cpp
@@ -1 +1 @@
-#include "next_token.h"
+#include "next_token.h"
diff --git a/ydb/services/datastreams/next_token.h b/ydb/services/datastreams/next_token.h
index 5a14b50d19..1fe61997bc 100644
--- a/ydb/services/datastreams/next_token.h
+++ b/ydb/services/datastreams/next_token.h
@@ -1,82 +1,82 @@
-#pragma once
-
+#pragma once
+
#include <ydb/core/protos/pqconfig.pb.h>
-#include <library/cpp/string_utils/base64/base64.h>
-#include <util/datetime/base.h>
-
-namespace NKikimr::NDataStreams::V1 {
-
- class TNextToken {
- public:
- static constexpr ui32 LIFETIME_MS = 300*1000;
-
- TNextToken(const TString& nextToken)
- : Valid{true}
- {
- try {
- TString decoded;
- Base64Decode(nextToken, decoded);
- auto ok = Proto.ParseFromString(decoded);
- if (ok) {
- Valid = IsAlive(TInstant::Now().MilliSeconds());
- } else {
- Valid = false;
- }
- } catch (std::exception&) {
- Valid = false;
- }
- }
-
- TNextToken(const TString& streamArn, ui32 alreadyRead, ui32 maxResults, ui64 creationTimestamp)
- : Valid{true}
- {
- Proto.SetStreamArn(streamArn);
- Proto.SetAlreadyRead(alreadyRead);
- Proto.SetMaxResults(maxResults);
- Proto.SetCreationTimestamp(creationTimestamp);
- }
-
- TString Serialize() const {
- TString data;
- bool result = Proto.SerializeToString(&data);
- Y_VERIFY(result);
- TString encoded;
- Base64Encode(data, encoded);
- return encoded;
- }
-
- ui32 GetAlreadyRead() const {
- return Proto.GetAlreadyRead();
- }
-
- TString GetStreamArn() const {
- return Proto.GetStreamArn();
- }
-
- TString GetStreamName() const {
- return Proto.GetStreamArn();
- }
-
- ui32 GetMaxResults() const {
- return Proto.GetMaxResults();
- }
-
- ui32 GetCreationTimestamp() const {
- return Proto.GetCreationTimestamp();
- }
-
- bool IsAlive(ui64 now) const {
- return now >= Proto.GetCreationTimestamp() &&
- (now - Proto.GetCreationTimestamp()) < LIFETIME_MS;
- }
-
- bool IsValid() const {
- return Valid && Proto.GetStreamArn().size() > 0;
- }
-
- private:
- bool Valid;
- NKikimrPQ::TYdsNextToken Proto;
- };
-
-} // namespace NKikimr::NDataStreams::V1
+#include <library/cpp/string_utils/base64/base64.h>
+#include <util/datetime/base.h>
+
+namespace NKikimr::NDataStreams::V1 {
+
+ class TNextToken {
+ public:
+ static constexpr ui32 LIFETIME_MS = 300*1000;
+
+ TNextToken(const TString& nextToken)
+ : Valid{true}
+ {
+ try {
+ TString decoded;
+ Base64Decode(nextToken, decoded);
+ auto ok = Proto.ParseFromString(decoded);
+ if (ok) {
+ Valid = IsAlive(TInstant::Now().MilliSeconds());
+ } else {
+ Valid = false;
+ }
+ } catch (std::exception&) {
+ Valid = false;
+ }
+ }
+
+ TNextToken(const TString& streamArn, ui32 alreadyRead, ui32 maxResults, ui64 creationTimestamp)
+ : Valid{true}
+ {
+ Proto.SetStreamArn(streamArn);
+ Proto.SetAlreadyRead(alreadyRead);
+ Proto.SetMaxResults(maxResults);
+ Proto.SetCreationTimestamp(creationTimestamp);
+ }
+
+ TString Serialize() const {
+ TString data;
+ bool result = Proto.SerializeToString(&data);
+ Y_VERIFY(result);
+ TString encoded;
+ Base64Encode(data, encoded);
+ return encoded;
+ }
+
+ ui32 GetAlreadyRead() const {
+ return Proto.GetAlreadyRead();
+ }
+
+ TString GetStreamArn() const {
+ return Proto.GetStreamArn();
+ }
+
+ TString GetStreamName() const {
+ return Proto.GetStreamArn();
+ }
+
+ ui32 GetMaxResults() const {
+ return Proto.GetMaxResults();
+ }
+
+ ui32 GetCreationTimestamp() const {
+ return Proto.GetCreationTimestamp();
+ }
+
+ bool IsAlive(ui64 now) const {
+ return now >= Proto.GetCreationTimestamp() &&
+ (now - Proto.GetCreationTimestamp()) < LIFETIME_MS;
+ }
+
+ bool IsValid() const {
+ return Valid && Proto.GetStreamArn().size() > 0;
+ }
+
+ private:
+ bool Valid;
+ NKikimrPQ::TYdsNextToken Proto;
+ };
+
+} // namespace NKikimr::NDataStreams::V1
diff --git a/ydb/services/datastreams/put_records_actor.h b/ydb/services/datastreams/put_records_actor.h
index b3cdb66855..8f06a99157 100644
--- a/ydb/services/datastreams/put_records_actor.h
+++ b/ydb/services/datastreams/put_records_actor.h
@@ -107,7 +107,7 @@ namespace NKikimr::NDataStreams::V1 {
w->SetDisableDeduplication(true);
w->SetCreateTimeMS(TInstant::Now().MilliSeconds());
w->SetUncompressedSize(item.Data.size());
- w->SetExternalOperation(true);
+ w->SetExternalOperation(true);
totalSize += (item.Data.size() + item.Key.size() + item.ExplicitHash.size());
}
ui64 putUnitsCount = totalSize / PUT_UNIT_SIZE;
@@ -217,7 +217,7 @@ namespace NKikimr::NDataStreams::V1 {
void Bootstrap(const NActors::TActorContext &ctx);
void PreparePartitionActors(const NActors::TActorContext& ctx);
- void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx);
+ void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx);
protected:
void AddRecord(THashMap<ui32, TVector<TPutRecordsItem>>& items, ui32 totalShardsCount, int index);
@@ -244,7 +244,7 @@ namespace NKikimr::NDataStreams::V1 {
STFUNC(StateFunc) {
switch (ev->GetTypeRewrite()) {
HFunc(NDataStreams::V1::TEvDataStreams::TEvPartitionActorResult, Handle);
- default: TBase::StateWork(ev, ctx);
+ default: TBase::StateWork(ev, ctx);
};
}
};
@@ -289,10 +289,10 @@ namespace NKikimr::NDataStreams::V1 {
}
template<class TDerived, class TProto>
- void TPutRecordsActorBase<TDerived, TProto>::HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) {
- if (TBase::ReplyIfNotTopic(ev, ctx)) {
- return;
- }
+ void TPutRecordsActorBase<TDerived, TProto>::HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) {
+ if (TBase::ReplyIfNotTopic(ev, ctx)) {
+ return;
+ }
const NSchemeCache::TSchemeCacheNavigate* navigate = ev->Get()->Request.Get();
auto topicInfo = navigate->ResultSet.begin();
@@ -301,9 +301,9 @@ namespace NKikimr::NDataStreams::V1 {
this->Request_->GetInternalToken())) {
return this->ReplyWithError(Ydb::StatusIds::UNAUTHORIZED, Ydb::PersQueue::ErrorCode::ACCESS_DENIED,
TStringBuilder() << "Access for stream "
- << this->GetProtoRequest()->stream_name()
- << " is denied for subject "
- << this->Request_->GetInternalToken(), ctx);
+ << this->GetProtoRequest()->stream_name()
+ << " is denied for subject "
+ << this->Request_->GetInternalToken(), ctx);
}
}
@@ -319,8 +319,8 @@ namespace NKikimr::NDataStreams::V1 {
auto part = partition.GetPartitionId();
if (items[part].empty()) continue;
PartitionToActor[part].ActorId = ctx.Register(
- new TDatastreamsPartitionActor(ctx.SelfID, partition.GetTabletId(), part, this->GetTopicPath(ctx), std::move(items[part]))
- );
+ new TDatastreamsPartitionActor(ctx.SelfID, partition.GetTabletId(), part, this->GetTopicPath(ctx), std::move(items[part]))
+ );
}
this->CheckFinish(ctx);
}
diff --git a/ydb/services/datastreams/shard_iterator.cpp b/ydb/services/datastreams/shard_iterator.cpp
index 9f0fcec7f6..2855ec2d52 100644
--- a/ydb/services/datastreams/shard_iterator.cpp
+++ b/ydb/services/datastreams/shard_iterator.cpp
@@ -1 +1 @@
-#include "shard_iterator.h"
+#include "shard_iterator.h"
diff --git a/ydb/services/datastreams/shard_iterator.h b/ydb/services/datastreams/shard_iterator.h
index 1061256b4f..7b6ce9c5cd 100644
--- a/ydb/services/datastreams/shard_iterator.h
+++ b/ydb/services/datastreams/shard_iterator.h
@@ -1,77 +1,77 @@
-#pragma once
-
+#pragma once
+
#include <ydb/core/protos/pqconfig.pb.h>
-#include <library/cpp/string_utils/base64/base64.h>
-#include <util/datetime/base.h>
-
-namespace NKikimr::NDataStreams::V1 {
-
- class TShardIterator {
- public:
- static constexpr ui64 LIFETIME_MS = 5*60*1000;
-
- TShardIterator(const TString& iteratorStr) {
- try {
- TString decoded;
- Base64Decode(iteratorStr, decoded);
- Valid = Proto.ParseFromString(decoded) && IsAlive(TInstant::Now().MilliSeconds());
- } catch (std::exception&) {
- Valid = false;
- }
- }
-
- TShardIterator(const TString& streamName, const TString& streamArn,
- ui32 shardId, ui64 readTimestamp, ui32 sequenceNumber) {
- Proto.SetStreamName(streamName);
- Proto.SetStreamArn(streamArn);
- Proto.SetShardId(shardId);
- Proto.SetReadTimestampMs(readTimestamp);
- Proto.SetSequenceNumber(sequenceNumber);
- Proto.SetCreationTimestampMs(TInstant::Now().MilliSeconds());
- Valid = true;
- }
-
- TString Serialize() const {
- TString data;
- bool result = Proto.SerializeToString(&data);
- Y_VERIFY(result);
- TString encoded;
- Base64Encode(data, encoded);
- return encoded;
- }
-
- TString GetStreamName() const {
- return Proto.GetStreamName();
- }
-
- TString GetStreamArn() const {
- return Proto.GetStreamArn();
- }
-
- ui32 GetShardId() const {
- return Proto.GetShardId();
- }
-
- ui64 GetReadTimestamp() const {
- return Proto.GetReadTimestampMs();
- }
-
- ui32 GetSequenceNumber() const {
- return Proto.GetSequenceNumber();
- }
-
- bool IsAlive(ui64 now) const {
- return now >= Proto.GetCreationTimestampMs() && now -
- Proto.GetCreationTimestampMs() < LIFETIME_MS;
- }
-
- bool IsValid() const {
- return Valid;
- }
-
- private:
- bool Valid;
- NKikimrPQ::TYdsShardIterator Proto;
- };
-
-} // namespace NKikimr::NDataStreams::V1
+#include <library/cpp/string_utils/base64/base64.h>
+#include <util/datetime/base.h>
+
+namespace NKikimr::NDataStreams::V1 {
+
+ class TShardIterator {
+ public:
+ static constexpr ui64 LIFETIME_MS = 5*60*1000;
+
+ TShardIterator(const TString& iteratorStr) {
+ try {
+ TString decoded;
+ Base64Decode(iteratorStr, decoded);
+ Valid = Proto.ParseFromString(decoded) && IsAlive(TInstant::Now().MilliSeconds());
+ } catch (std::exception&) {
+ Valid = false;
+ }
+ }
+
+ TShardIterator(const TString& streamName, const TString& streamArn,
+ ui32 shardId, ui64 readTimestamp, ui32 sequenceNumber) {
+ Proto.SetStreamName(streamName);
+ Proto.SetStreamArn(streamArn);
+ Proto.SetShardId(shardId);
+ Proto.SetReadTimestampMs(readTimestamp);
+ Proto.SetSequenceNumber(sequenceNumber);
+ Proto.SetCreationTimestampMs(TInstant::Now().MilliSeconds());
+ Valid = true;
+ }
+
+ TString Serialize() const {
+ TString data;
+ bool result = Proto.SerializeToString(&data);
+ Y_VERIFY(result);
+ TString encoded;
+ Base64Encode(data, encoded);
+ return encoded;
+ }
+
+ TString GetStreamName() const {
+ return Proto.GetStreamName();
+ }
+
+ TString GetStreamArn() const {
+ return Proto.GetStreamArn();
+ }
+
+ ui32 GetShardId() const {
+ return Proto.GetShardId();
+ }
+
+ ui64 GetReadTimestamp() const {
+ return Proto.GetReadTimestampMs();
+ }
+
+ ui32 GetSequenceNumber() const {
+ return Proto.GetSequenceNumber();
+ }
+
+ bool IsAlive(ui64 now) const {
+ return now >= Proto.GetCreationTimestampMs() && now -
+ Proto.GetCreationTimestampMs() < LIFETIME_MS;
+ }
+
+ bool IsValid() const {
+ return Valid;
+ }
+
+ private:
+ bool Valid;
+ NKikimrPQ::TYdsShardIterator Proto;
+ };
+
+} // namespace NKikimr::NDataStreams::V1
diff --git a/ydb/services/datastreams/ya.make b/ydb/services/datastreams/ya.make
index d2635df981..0a26e96de9 100644
--- a/ydb/services/datastreams/ya.make
+++ b/ydb/services/datastreams/ya.make
@@ -5,9 +5,9 @@ OWNER(g:kikimr)
SRCS(
datastreams_proxy.cpp
grpc_service.cpp
- next_token.cpp
+ next_token.cpp
put_records_actor.cpp
- shard_iterator.cpp
+ shard_iterator.cpp
)
PEERDIR(
diff --git a/ydb/services/lib/actors/pq_schema_actor.cpp b/ydb/services/lib/actors/pq_schema_actor.cpp
index fc2e779e39..f48e398488 100644
--- a/ydb/services/lib/actors/pq_schema_actor.cpp
+++ b/ydb/services/lib/actors/pq_schema_actor.cpp
@@ -125,39 +125,39 @@ namespace NKikimr::NGRpcProxy::V1 {
const TString& consumerName,
const TActorContext& ctx
) {
- THashSet<TString> rulesToRemove;
- rulesToRemove.insert(consumerName);
-
- config->ClearReadRuleVersions();
- config->ClearReadRules();
- config->ClearReadFromTimestampsMs();
- config->ClearConsumerFormatVersions();
- config->ClearConsumerCodecs();
- config->MutablePartitionConfig()->ClearImportantClientId();
+ THashSet<TString> rulesToRemove;
+ rulesToRemove.insert(consumerName);
+
+ config->ClearReadRuleVersions();
+ config->ClearReadRules();
+ config->ClearReadFromTimestampsMs();
+ config->ClearConsumerFormatVersions();
+ config->ClearConsumerCodecs();
+ config->MutablePartitionConfig()->ClearImportantClientId();
config->ClearReadRuleServiceTypes();
-
- for (const auto& importantConsumer : originalConfig.GetPartitionConfig().GetImportantClientId()) {
- if (rulesToRemove.find(importantConsumer) == rulesToRemove.end()) {
- config->MutablePartitionConfig()->AddImportantClientId(importantConsumer);
- }
- }
-
+
+ for (const auto& importantConsumer : originalConfig.GetPartitionConfig().GetImportantClientId()) {
+ if (rulesToRemove.find(importantConsumer) == rulesToRemove.end()) {
+ config->MutablePartitionConfig()->AddImportantClientId(importantConsumer);
+ }
+ }
+
const auto& pqConfig = AppData(ctx)->PQConfig;
- for (size_t i = 0; i < originalConfig.ReadRulesSize(); i++) {
- if (auto it = rulesToRemove.find(originalConfig.GetReadRules(i)); it != rulesToRemove.end()) {
- rulesToRemove.erase(it);
- continue;
- }
-
- config->AddReadRuleVersions(originalConfig.GetReadRuleVersions(i));
- config->AddReadRules(originalConfig.GetReadRules(i));
- config->AddReadFromTimestampsMs(originalConfig.GetReadFromTimestampsMs(i));
- config->AddConsumerFormatVersions(originalConfig.GetConsumerFormatVersions(i));
- auto ct = config->AddConsumerCodecs();
- for (size_t j = 0; j < originalConfig.GetConsumerCodecs(i).CodecsSize(); j++) {
- ct->AddCodecs(originalConfig.GetConsumerCodecs(i).GetCodecs(j));
- ct->AddIds(originalConfig.GetConsumerCodecs(i).GetIds(j));
- }
+ for (size_t i = 0; i < originalConfig.ReadRulesSize(); i++) {
+ if (auto it = rulesToRemove.find(originalConfig.GetReadRules(i)); it != rulesToRemove.end()) {
+ rulesToRemove.erase(it);
+ continue;
+ }
+
+ config->AddReadRuleVersions(originalConfig.GetReadRuleVersions(i));
+ config->AddReadRules(originalConfig.GetReadRules(i));
+ config->AddReadFromTimestampsMs(originalConfig.GetReadFromTimestampsMs(i));
+ config->AddConsumerFormatVersions(originalConfig.GetConsumerFormatVersions(i));
+ auto ct = config->AddConsumerCodecs();
+ for (size_t j = 0; j < originalConfig.GetConsumerCodecs(i).CodecsSize(); j++) {
+ ct->AddCodecs(originalConfig.GetConsumerCodecs(i).GetCodecs(j));
+ ct->AddIds(originalConfig.GetConsumerCodecs(i).GetIds(j));
+ }
if (i < originalConfig.ReadRuleServiceTypesSize()) {
config->AddReadRuleServiceTypes(originalConfig.GetReadRuleServiceTypes(i));
} else {
@@ -167,15 +167,15 @@ namespace NKikimr::NGRpcProxy::V1 {
}
config->AddReadRuleServiceTypes(pqConfig.GetDefaultClientServiceType().GetName());
}
- }
-
- if (rulesToRemove.size() > 0) {
- return TStringBuilder() << "Rule for consumer " << *rulesToRemove.begin() << " doesn't exist";
- }
-
- return "";
- }
-
+ }
+
+ if (rulesToRemove.size() > 0) {
+ return TStringBuilder() << "Rule for consumer " << *rulesToRemove.begin() << " doesn't exist";
+ }
+
+ return "";
+ }
+
bool CheckReadRulesConfig(const NKikimrPQ::TPQTabletConfig& config, const TClientServiceTypes& supportedClientServiceTypes,
TString& error) {
diff --git a/ydb/services/lib/actors/pq_schema_actor.h b/ydb/services/lib/actors/pq_schema_actor.h
index d798f6f9ad..1be2e7fa24 100644
--- a/ydb/services/lib/actors/pq_schema_actor.h
+++ b/ydb/services/lib/actors/pq_schema_actor.h
@@ -48,7 +48,7 @@ namespace NKikimr::NGRpcProxy::V1 {
template<class TDerived, class TRequest>
class TPQGrpcSchemaBase : public NKikimr::NGRpcService::TRpcSchemeRequestActor<TDerived, TRequest> {
- protected:
+ protected:
using TBase = NKikimr::NGRpcService::TRpcSchemeRequestActor<TDerived, TRequest>;
public:
@@ -127,88 +127,88 @@ namespace NKikimr::NGRpcProxy::V1 {
}
}
- bool ReplyIfNotTopic(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) {
+ bool ReplyIfNotTopic(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) {
+ const NSchemeCache::TSchemeCacheNavigate* result = ev->Get()->Request.Get();
+ Y_VERIFY(result->ResultSet.size() == 1);
+ const auto& response = result->ResultSet.front();
+ const TString path = JoinPath(response.Path);
+
+ if (ev->Get()->Request.Get()->ResultSet.size() != 1 ||
+ ev->Get()->Request.Get()->ResultSet.begin()->Kind !=
+ NSchemeCache::TSchemeCacheNavigate::KindTopic) {
+ this->Request_->RaiseIssue(
+ FillIssue(
+ TStringBuilder() << "path '" << path << "' is not a stream",
+ Ydb::PersQueue::ErrorCode::ERROR
+ )
+ );
+ TBase::Reply(Ydb::StatusIds::SCHEME_ERROR, ctx);
+ return true;
+ }
+
+ return false;
+ }
+
+ void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) {
const NSchemeCache::TSchemeCacheNavigate* result = ev->Get()->Request.Get();
Y_VERIFY(result->ResultSet.size() == 1);
const auto& response = result->ResultSet.front();
const TString path = JoinPath(response.Path);
-
- if (ev->Get()->Request.Get()->ResultSet.size() != 1 ||
- ev->Get()->Request.Get()->ResultSet.begin()->Kind !=
- NSchemeCache::TSchemeCacheNavigate::KindTopic) {
+
+ switch (response.Status) {
+ case NSchemeCache::TSchemeCacheNavigate::EStatus::Ok: {
+ return static_cast<TDerived*>(this)->HandleCacheNavigateResponse(ev, ctx);
+ }
+ break;
+ case NSchemeCache::TSchemeCacheNavigate::EStatus::PathErrorUnknown: {
+ this->Request_->RaiseIssue(
+ FillIssue(
+ TStringBuilder() << "path '" << path << "' does not exist or you " <<
+ "do not have access rights",
+ Ydb::PersQueue::ErrorCode::ERROR
+ )
+ );
+ return TBase::Reply(Ydb::StatusIds::SCHEME_ERROR, ctx);
+ }
+ case NSchemeCache::TSchemeCacheNavigate::EStatus::TableCreationNotComplete: {
+ this->Request_->RaiseIssue(
+ FillIssue(
+ TStringBuilder() << "table creation is not completed",
+ Ydb::PersQueue::ErrorCode::ERROR
+ )
+ );
+ return TBase::Reply(Ydb::StatusIds::SCHEME_ERROR, ctx);
+ }
+ break;
+ case NSchemeCache::TSchemeCacheNavigate::EStatus::PathNotTable: {
+ this->Request_->RaiseIssue(
+ FillIssue(
+ TStringBuilder() << "path '" << path << "' is not a table",
+ Ydb::PersQueue::ErrorCode::ERROR
+ )
+ );
+ return TBase::Reply(Ydb::StatusIds::SCHEME_ERROR, ctx);
+ }
+ break;
+ case NSchemeCache::TSchemeCacheNavigate::EStatus::RootUnknown: {
this->Request_->RaiseIssue(
FillIssue(
- TStringBuilder() << "path '" << path << "' is not a stream",
- Ydb::PersQueue::ErrorCode::ERROR
- )
- );
- TBase::Reply(Ydb::StatusIds::SCHEME_ERROR, ctx);
- return true;
- }
-
- return false;
- }
-
- void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) {
- const NSchemeCache::TSchemeCacheNavigate* result = ev->Get()->Request.Get();
- Y_VERIFY(result->ResultSet.size() == 1);
- const auto& response = result->ResultSet.front();
- const TString path = JoinPath(response.Path);
-
- switch (response.Status) {
- case NSchemeCache::TSchemeCacheNavigate::EStatus::Ok: {
- return static_cast<TDerived*>(this)->HandleCacheNavigateResponse(ev, ctx);
- }
- break;
- case NSchemeCache::TSchemeCacheNavigate::EStatus::PathErrorUnknown: {
- this->Request_->RaiseIssue(
- FillIssue(
- TStringBuilder() << "path '" << path << "' does not exist or you " <<
- "do not have access rights",
+ TStringBuilder() << "unknown database root",
Ydb::PersQueue::ErrorCode::ERROR
)
);
return TBase::Reply(Ydb::StatusIds::SCHEME_ERROR, ctx);
}
- case NSchemeCache::TSchemeCacheNavigate::EStatus::TableCreationNotComplete: {
- this->Request_->RaiseIssue(
- FillIssue(
- TStringBuilder() << "table creation is not completed",
- Ydb::PersQueue::ErrorCode::ERROR
- )
- );
- return TBase::Reply(Ydb::StatusIds::SCHEME_ERROR, ctx);
- }
- break;
- case NSchemeCache::TSchemeCacheNavigate::EStatus::PathNotTable: {
- this->Request_->RaiseIssue(
- FillIssue(
- TStringBuilder() << "path '" << path << "' is not a table",
- Ydb::PersQueue::ErrorCode::ERROR
- )
- );
- return TBase::Reply(Ydb::StatusIds::SCHEME_ERROR, ctx);
- }
- break;
- case NSchemeCache::TSchemeCacheNavigate::EStatus::RootUnknown: {
- this->Request_->RaiseIssue(
- FillIssue(
- TStringBuilder() << "unknown database root",
- Ydb::PersQueue::ErrorCode::ERROR
- )
- );
- return TBase::Reply(Ydb::StatusIds::SCHEME_ERROR, ctx);
- }
- break;
-
- default:
- return TBase::Reply(Ydb::StatusIds::GENERIC_ERROR, ctx);
- }
+ break;
+
+ default:
+ return TBase::Reply(Ydb::StatusIds::GENERIC_ERROR, ctx);
+ }
}
- void ReplyWithError(Ydb::StatusIds::StatusCode status,
- Ydb::PersQueue::ErrorCode::ErrorCode pqStatus,
- const TString& messageText, const NActors::TActorContext& ctx) {
+ void ReplyWithError(Ydb::StatusIds::StatusCode status,
+ Ydb::PersQueue::ErrorCode::ErrorCode pqStatus,
+ const TString& messageText, const NActors::TActorContext& ctx) {
this->Request_->RaiseIssue(FillIssue(messageText, pqStatus));
this->Request_->ReplyWithYdbStatus(status);
this->Die(ctx);
@@ -228,13 +228,13 @@ namespace NKikimr::NGRpcProxy::V1 {
IsDead = true;
}
- void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) {
- switch (ev->GetTypeRewrite()) {
- HFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, Handle);
- default: TBase::StateWork(ev, ctx);
- }
- }
-
+ void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) {
+ switch (ev->GetTypeRewrite()) {
+ HFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, Handle);
+ default: TBase::StateWork(ev, ctx);
+ }
+ }
+
private:
bool IsDead = false;
TString TopicPath;
@@ -282,16 +282,16 @@ namespace NKikimr::NGRpcProxy::V1 {
this->DescribeSchemeResult.Reset();
}
- void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) {
- const NSchemeCache::TSchemeCacheNavigate* result = ev->Get()->Request.Get();
- Y_VERIFY(result->ResultSet.size() == 1);
- DescribeSchemeResult = std::move(ev);
- return this->SendProposeRequest(ctx);
- }
-
+ void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) {
+ const NSchemeCache::TSchemeCacheNavigate* result = ev->Get()->Request.Get();
+ Y_VERIFY(result->ResultSet.size() == 1);
+ DescribeSchemeResult = std::move(ev);
+ return this->SendProposeRequest(ctx);
+ }
+
void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) {
switch (ev->GetTypeRewrite()) {
- default: TBase::StateWork(ev, ctx);
+ default: TBase::StateWork(ev, ctx);
}
}
@@ -299,4 +299,4 @@ namespace NKikimr::NGRpcProxy::V1 {
THolder<NActors::TEventHandle<TEvTxProxySchemeCache::TEvNavigateKeySetResult>> DescribeSchemeResult;
};
-}
+}
diff --git a/ydb/services/persqueue_v1/grpc_pq_actor.h b/ydb/services/persqueue_v1/grpc_pq_actor.h
index 95be559053..65c8aca1db 100644
--- a/ydb/services/persqueue_v1/grpc_pq_actor.h
+++ b/ydb/services/persqueue_v1/grpc_pq_actor.h
@@ -550,8 +550,8 @@ private:
void GenerateNextWriteRequest(const NActors::TActorContext& ctx);
- void SetupCounters();
- void SetupCounters(const TString& cloudId, const TString& dbId, const TString& folderId);
+ void SetupCounters();
+ void SetupCounters(const TString& cloudId, const TString& dbId, const TString& folderId);
private:
std::unique_ptr<NKikimr::NGRpcService::TEvStreamPQWriteRequest> Request;
@@ -854,13 +854,13 @@ private:
void HandleWakeup(const NActors::TActorContext& ctx);
void Handle(TEvPQProxy::TEvAuthResultOk::TPtr& ev, const NActors::TActorContext& ctx);
- void CloseSession(const TString& errorReason, const PersQueue::ErrorCode::ErrorCode errorCode,
- const NActors::TActorContext& ctx);
+ void CloseSession(const TString& errorReason, const PersQueue::ErrorCode::ErrorCode errorCode,
+ const NActors::TActorContext& ctx);
void SetupCounters();
void SetupTopicCounters(const TString& topic);
- void SetupTopicCounters(const TString& topic, const TString& cloudId, const TString& dbId,
- const TString& folderId);
+ void SetupTopicCounters(const TString& topic, const TString& cloudId, const TString& dbId,
+ const TString& folderId);
void ProcessReads(const NActors::TActorContext& ctx); // returns false if actor died
struct TFormedReadResponse;
@@ -1137,8 +1137,8 @@ public:
const TString& workingDir, const TString& name);
void Bootstrap(const NActors::TActorContext& ctx);
-
- void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx){ Y_UNUSED(ev); Y_UNUSED(ctx); }
+
+ void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx){ Y_UNUSED(ev); Y_UNUSED(ctx); }
};
class TDescribeTopicActor : public TPQGrpcSchemaBase<TDescribeTopicActor, NKikimr::NGRpcService::TEvPQDescribeTopicRequest> {
@@ -1152,7 +1152,7 @@ public:
void Bootstrap(const NActors::TActorContext& ctx);
- void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx);
+ void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx);
};
@@ -1195,8 +1195,8 @@ public:
void Bootstrap(const NActors::TActorContext& ctx);
- void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx){ Y_UNUSED(ev); Y_UNUSED(ctx); }
-
+ void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx){ Y_UNUSED(ev); Y_UNUSED(ctx); }
+
private:
TString LocalCluster;
TVector<TString> Clusters;
@@ -1213,8 +1213,8 @@ public:
const TString& workingDir, const TString& name);
void Bootstrap(const NActors::TActorContext& ctx);
-
- void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx){ Y_UNUSED(ev); Y_UNUSED(ctx); }
+
+ void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx){ Y_UNUSED(ev); Y_UNUSED(ctx); }
};
diff --git a/ydb/services/persqueue_v1/grpc_pq_read_actor.cpp b/ydb/services/persqueue_v1/grpc_pq_read_actor.cpp
index d48604b087..654edfcfcd 100644
--- a/ydb/services/persqueue_v1/grpc_pq_read_actor.cpp
+++ b/ydb/services/persqueue_v1/grpc_pq_read_actor.cpp
@@ -779,8 +779,8 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvReadInit::TPtr& ev, const TActorCo
auto subGroup = GetServiceCounters(Counters, "pqproxy|SLI");
Aggr = {{{{"Account", ClientPath.substr(0, ClientPath.find("/"))}}, {"total"}}};
- SLIErrors = NKikimr::NPQ::TMultiCounter(subGroup, Aggr, {}, {"RequestsError"}, true, "sensor", false);
- SLITotal = NKikimr::NPQ::TMultiCounter(subGroup, Aggr, {}, {"RequestsTotal"}, true, "sensor", false);
+ SLIErrors = NKikimr::NPQ::TMultiCounter(subGroup, Aggr, {}, {"RequestsError"}, true, "sensor", false);
+ SLITotal = NKikimr::NPQ::TMultiCounter(subGroup, Aggr, {}, {"RequestsTotal"}, true, "sensor", false);
SLITotal.Inc();
}
@@ -841,48 +841,48 @@ void TReadSessionActor::SetupCounters()
void TReadSessionActor::SetupTopicCounters(const TString& topic)
{
auto& topicCounters = TopicCounters[topic];
- auto subGroup = GetServiceCounters(Counters, "pqproxy|readSession");
+ auto subGroup = GetServiceCounters(Counters, "pqproxy|readSession");
//client/consumerPath Account/Producer OriginDC Topic/TopicPath
TVector<NPQ::TLabelsInfo> aggr = NKikimr::NPQ::GetLabels(topic);
TVector<std::pair<TString, TString>> cons = {{"Client", ClientId}, {"ConsumerPath", ClientPath}};
- topicCounters.PartitionsLocked = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"PartitionsLocked"}, true);
- topicCounters.PartitionsReleased = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"PartitionsReleased"}, true);
- topicCounters.PartitionsToBeReleased = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"PartitionsToBeReleased"}, false);
- topicCounters.PartitionsToBeLocked = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"PartitionsToBeLocked"}, false);
- topicCounters.PartitionsInfly = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"PartitionsInfly"}, false);
- topicCounters.Errors = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"PartitionsErrors"}, true);
- topicCounters.Commits = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"Commits"}, true);
- topicCounters.WaitsForData = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"WaitsForData"}, true);
+ topicCounters.PartitionsLocked = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"PartitionsLocked"}, true);
+ topicCounters.PartitionsReleased = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"PartitionsReleased"}, true);
+ topicCounters.PartitionsToBeReleased = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"PartitionsToBeReleased"}, false);
+ topicCounters.PartitionsToBeLocked = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"PartitionsToBeLocked"}, false);
+ topicCounters.PartitionsInfly = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"PartitionsInfly"}, false);
+ topicCounters.Errors = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"PartitionsErrors"}, true);
+ topicCounters.Commits = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"Commits"}, true);
+ topicCounters.WaitsForData = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"WaitsForData"}, true);
+
+ topicCounters.CommitLatency = CommitLatency;
+ topicCounters.SLIBigLatency = SLIBigLatency;
+ topicCounters.SLITotal = SLITotal;
+}
+
+void TReadSessionActor::SetupTopicCounters(const TString& topic, const TString& cloudId,
+ const TString& dbId, const TString& folderId)
+{
+ auto& topicCounters = TopicCounters[topic];
+ auto subGroup = NKikimr::NPQ::GetCountersForStream(Counters, "readSession");
+//client/consumerPath Account/Producer OriginDC Topic/TopicPath
+ TVector<NPQ::TLabelsInfo> aggr = NKikimr::NPQ::GetLabelsForStream(topic, cloudId, dbId, folderId);
+ TVector<std::pair<TString, TString>> cons{};
+
+ topicCounters.PartitionsLocked = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"stream.internal_read.partitions_locked_per_second"}, true);
+ topicCounters.PartitionsReleased = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"stream.internal_read.partitions_released_per_second"}, true);
+ topicCounters.PartitionsToBeReleased = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"stream.internal_read.partitions_to_be_released"}, false);
+ topicCounters.PartitionsToBeLocked = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"stream.internal_read.partitions_to_be_locked"}, false);
+ topicCounters.PartitionsInfly = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"stream.internal_read.partitions_locked"}, false);
+ topicCounters.Errors = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"stream.internal_read.partitions_errors_per_second"}, true);
+ topicCounters.Commits = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"stream.internal_read.commits_per_second"}, true);
+ topicCounters.WaitsForData = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"stream.internal_read.waits_for_data"}, true);
topicCounters.CommitLatency = CommitLatency;
topicCounters.SLIBigLatency = SLIBigLatency;
topicCounters.SLITotal = SLITotal;
}
-void TReadSessionActor::SetupTopicCounters(const TString& topic, const TString& cloudId,
- const TString& dbId, const TString& folderId)
-{
- auto& topicCounters = TopicCounters[topic];
- auto subGroup = NKikimr::NPQ::GetCountersForStream(Counters, "readSession");
-//client/consumerPath Account/Producer OriginDC Topic/TopicPath
- TVector<NPQ::TLabelsInfo> aggr = NKikimr::NPQ::GetLabelsForStream(topic, cloudId, dbId, folderId);
- TVector<std::pair<TString, TString>> cons{};
-
- topicCounters.PartitionsLocked = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"stream.internal_read.partitions_locked_per_second"}, true);
- topicCounters.PartitionsReleased = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"stream.internal_read.partitions_released_per_second"}, true);
- topicCounters.PartitionsToBeReleased = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"stream.internal_read.partitions_to_be_released"}, false);
- topicCounters.PartitionsToBeLocked = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"stream.internal_read.partitions_to_be_locked"}, false);
- topicCounters.PartitionsInfly = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"stream.internal_read.partitions_locked"}, false);
- topicCounters.Errors = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"stream.internal_read.partitions_errors_per_second"}, true);
- topicCounters.Commits = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"stream.internal_read.commits_per_second"}, true);
- topicCounters.WaitsForData = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"stream.internal_read.waits_for_data"}, true);
-
- topicCounters.CommitLatency = CommitLatency;
- topicCounters.SLIBigLatency = SLIBigLatency;
- topicCounters.SLITotal = SLITotal;
-}
-
void TReadSessionActor::Handle(TEvPQProxy::TEvAuthResultOk::TPtr& ev, const TActorContext& ctx) {
LastACLCheckTimestamp = ctx.Now();
@@ -903,11 +903,11 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvAuthResultOk::TPtr& ev, const TAct
auto subGroup = GetServiceCounters(Counters, "pqproxy|SLI");
InitLatency = NKikimr::NPQ::CreateSLIDurationCounter(subGroup, Aggr, "ReadInit", initBorder, {100, 200, 500, 1000, 1500, 2000, 5000, 10000, 30000, 99999999});
CommitLatency = NKikimr::NPQ::CreateSLIDurationCounter(subGroup, Aggr, "Commit", AppData(ctx)->PQConfig.GetCommitLatencyBigMs(), {100, 200, 500, 1000, 1500, 2000, 5000, 10000, 30000, 99999999});
- SLIBigLatency = NKikimr::NPQ::TMultiCounter(subGroup, Aggr, {}, {"RequestsBigLatency"}, true, "sensor", false);
+ SLIBigLatency = NKikimr::NPQ::TMultiCounter(subGroup, Aggr, {}, {"RequestsBigLatency"}, true, "sensor", false);
ReadLatency = NKikimr::NPQ::CreateSLIDurationCounter(subGroup, Aggr, "Read", readBorder, {100, 200, 500, 1000, 1500, 2000, 5000, 10000, 30000, 99999999});
ReadLatencyFromDisk = NKikimr::NPQ::CreateSLIDurationCounter(subGroup, Aggr, "ReadFromDisk", readBorderFromDisk, {100, 200, 500, 1000, 1500, 2000, 5000, 10000, 30000, 99999999});
- SLIBigReadLatency = NKikimr::NPQ::TMultiCounter(subGroup, Aggr, {}, {"ReadBigLatency"}, true, "sensor", false);
- ReadsTotal = NKikimr::NPQ::TMultiCounter(subGroup, Aggr, {}, {"ReadsTotal"}, true, "sensor", false);
+ SLIBigReadLatency = NKikimr::NPQ::TMultiCounter(subGroup, Aggr, {}, {"ReadBigLatency"}, true, "sensor", false);
+ ReadsTotal = NKikimr::NPQ::TMultiCounter(subGroup, Aggr, {}, {"ReadsTotal"}, true, "sensor", false);
ui32 initDurationMs = (ctx.Now() - StartTime).MilliSeconds();
InitLatency.IncFor(initDurationMs, 1);
@@ -1002,11 +1002,11 @@ void TReadSessionActor::Handle(TEvPersQueue::TEvLockPartition::TPtr& ev, const T
}
if (NumPartitionsFromTopic[topic]++ == 0) {
- if (AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen()) {
- SetupTopicCounters(topic, jt->second.CloudId, jt->second.DbId, jt->second.FolderId);
- } else {
- SetupTopicCounters(topic);
- }
+ if (AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen()) {
+ SetupTopicCounters(topic, jt->second.CloudId, jt->second.DbId, jt->second.FolderId);
+ } else {
+ SetupTopicCounters(topic);
+ }
}
auto it = TopicCounters.find(topic);
diff --git a/ydb/services/persqueue_v1/grpc_pq_schema.cpp b/ydb/services/persqueue_v1/grpc_pq_schema.cpp
index 7f54ea8c64..7756adc797 100644
--- a/ydb/services/persqueue_v1/grpc_pq_schema.cpp
+++ b/ydb/services/persqueue_v1/grpc_pq_schema.cpp
@@ -121,89 +121,89 @@ void TDescribeTopicActor::StateWork(TAutoPtr<IEventHandle>& ev, const TActorCont
}
-void TDescribeTopicActor::HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) {
+void TDescribeTopicActor::HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) {
Y_VERIFY(ev->Get()->Request.Get()->ResultSet.size() == 1); // describe for only one topic
- if (ReplyIfNotTopic(ev, ctx)) {
- return;
- }
-
+ if (ReplyIfNotTopic(ev, ctx)) {
+ return;
+ }
+
const auto& response = ev->Get()->Request.Get()->ResultSet.front();
- const TString path = JoinSeq("/", response.Path);
+ const TString path = JoinSeq("/", response.Path);
- Ydb::PersQueue::V1::DescribeTopicResult result;
+ Ydb::PersQueue::V1::DescribeTopicResult result;
- auto settings = result.mutable_settings();
- Ydb::Scheme::Entry *selfEntry = result.mutable_self();
- const auto& selfInfo = response.Self->Info;
+ auto settings = result.mutable_settings();
+ Ydb::Scheme::Entry *selfEntry = result.mutable_self();
+ const auto& selfInfo = response.Self->Info;
selfEntry->set_name(path + "/" + selfInfo.GetName());
- selfEntry->set_type(static_cast<Ydb::Scheme::Entry::Type>(selfInfo.GetPathType()));
- ConvertDirectoryEntry(selfInfo, selfEntry, true);
- if (response.PQGroupInfo) {
- const auto &pqDescr = response.PQGroupInfo->Description;
- settings->set_partitions_count(pqDescr.GetTotalGroupCount());
-
- const auto &config = pqDescr.GetPQTabletConfig();
- if (!config.GetRequireAuthWrite()) {
- (*settings->mutable_attributes())["_allow_unauthenticated_write"] = "true";
- }
-
- if (!config.GetRequireAuthRead()) {
- (*settings->mutable_attributes())["_allow_unauthenticated_read"] = "true";
- }
-
- if (pqDescr.GetPartitionPerTablet() != 2) {
- (*settings->mutable_attributes())["_partitions_per_tablet"] =
- TStringBuilder() << pqDescr.GetPartitionPerTablet();
- }
- if (config.HasAbcId()) {
- (*settings->mutable_attributes())["_abc_id"] = TStringBuilder() << config.GetAbcId();
- }
- if (config.HasAbcSlug()) {
- (*settings->mutable_attributes())["_abc_slug"] = config.GetAbcSlug();
- }
- bool local = config.GetLocalDC();
- settings->set_client_write_disabled(!local);
- const auto &partConfig = config.GetPartitionConfig();
- i64 msip = partConfig.GetMaxSizeInPartition();
- if (msip != Max<i64>())
- settings->set_max_partition_storage_size(msip);
- settings->set_retention_period_ms(partConfig.GetLifetimeSeconds() * 1000);
- settings->set_message_group_seqno_retention_period_ms(partConfig.GetSourceIdLifetimeSeconds() * 1000);
- settings->set_max_partition_message_groups_seqno_stored(partConfig.GetSourceIdMaxCounts());
-
- if (local) {
- settings->set_max_partition_write_speed(partConfig.GetWriteSpeedInBytesPerSecond());
- settings->set_max_partition_write_burst(partConfig.GetBurstSize());
- }
-
- settings->set_supported_format(
- (Ydb::PersQueue::V1::TopicSettings::Format) (config.GetFormatVersion() + 1));
-
- for (const auto &codec : config.GetCodecs().GetIds()) {
- settings->add_supported_codecs((Ydb::PersQueue::V1::Codec) (codec + 1));
- }
-
+ selfEntry->set_type(static_cast<Ydb::Scheme::Entry::Type>(selfInfo.GetPathType()));
+ ConvertDirectoryEntry(selfInfo, selfEntry, true);
+ if (response.PQGroupInfo) {
+ const auto &pqDescr = response.PQGroupInfo->Description;
+ settings->set_partitions_count(pqDescr.GetTotalGroupCount());
+
+ const auto &config = pqDescr.GetPQTabletConfig();
+ if (!config.GetRequireAuthWrite()) {
+ (*settings->mutable_attributes())["_allow_unauthenticated_write"] = "true";
+ }
+
+ if (!config.GetRequireAuthRead()) {
+ (*settings->mutable_attributes())["_allow_unauthenticated_read"] = "true";
+ }
+
+ if (pqDescr.GetPartitionPerTablet() != 2) {
+ (*settings->mutable_attributes())["_partitions_per_tablet"] =
+ TStringBuilder() << pqDescr.GetPartitionPerTablet();
+ }
+ if (config.HasAbcId()) {
+ (*settings->mutable_attributes())["_abc_id"] = TStringBuilder() << config.GetAbcId();
+ }
+ if (config.HasAbcSlug()) {
+ (*settings->mutable_attributes())["_abc_slug"] = config.GetAbcSlug();
+ }
+ bool local = config.GetLocalDC();
+ settings->set_client_write_disabled(!local);
+ const auto &partConfig = config.GetPartitionConfig();
+ i64 msip = partConfig.GetMaxSizeInPartition();
+ if (msip != Max<i64>())
+ settings->set_max_partition_storage_size(msip);
+ settings->set_retention_period_ms(partConfig.GetLifetimeSeconds() * 1000);
+ settings->set_message_group_seqno_retention_period_ms(partConfig.GetSourceIdLifetimeSeconds() * 1000);
+ settings->set_max_partition_message_groups_seqno_stored(partConfig.GetSourceIdMaxCounts());
+
+ if (local) {
+ settings->set_max_partition_write_speed(partConfig.GetWriteSpeedInBytesPerSecond());
+ settings->set_max_partition_write_burst(partConfig.GetBurstSize());
+ }
+
+ settings->set_supported_format(
+ (Ydb::PersQueue::V1::TopicSettings::Format) (config.GetFormatVersion() + 1));
+
+ for (const auto &codec : config.GetCodecs().GetIds()) {
+ settings->add_supported_codecs((Ydb::PersQueue::V1::Codec) (codec + 1));
+ }
+
const auto& pqConfig = AppData(ctx)->PQConfig;
- for (ui32 i = 0; i < config.ReadRulesSize(); ++i) {
- auto rr = settings->add_read_rules();
- auto consumerName = NPersQueue::ConvertOldConsumerName(config.GetReadRules(i), ctx);
- rr->set_consumer_name(consumerName);
- rr->set_starting_message_timestamp_ms(config.GetReadFromTimestampsMs(i));
- rr->set_supported_format(
- (Ydb::PersQueue::V1::TopicSettings::Format) (config.GetConsumerFormatVersions(i) + 1));
- rr->set_version(config.GetReadRuleVersions(i));
- for (const auto &codec : config.GetConsumerCodecs(i).GetIds()) {
- rr->add_supported_codecs((Ydb::PersQueue::V1::Codec) (codec + 1));
- }
- bool important = false;
- for (const auto &c : partConfig.GetImportantClientId()) {
- if (c == config.GetReadRules(i)) {
- important = true;
- break;
+ for (ui32 i = 0; i < config.ReadRulesSize(); ++i) {
+ auto rr = settings->add_read_rules();
+ auto consumerName = NPersQueue::ConvertOldConsumerName(config.GetReadRules(i), ctx);
+ rr->set_consumer_name(consumerName);
+ rr->set_starting_message_timestamp_ms(config.GetReadFromTimestampsMs(i));
+ rr->set_supported_format(
+ (Ydb::PersQueue::V1::TopicSettings::Format) (config.GetConsumerFormatVersions(i) + 1));
+ rr->set_version(config.GetReadRuleVersions(i));
+ for (const auto &codec : config.GetConsumerCodecs(i).GetIds()) {
+ rr->add_supported_codecs((Ydb::PersQueue::V1::Codec) (codec + 1));
+ }
+ bool important = false;
+ for (const auto &c : partConfig.GetImportantClientId()) {
+ if (c == config.GetReadRules(i)) {
+ important = true;
+ break;
}
- }
- rr->set_important(important);
+ }
+ rr->set_important(important);
if (i < config.ReadRuleServiceTypesSize()) {
rr->set_service_type(config.GetReadRuleServiceTypes(i));
@@ -218,44 +218,44 @@ void TDescribeTopicActor::HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEv
}
rr->set_service_type(pqConfig.GetDefaultClientServiceType().GetName());
}
- }
- if (partConfig.HasMirrorFrom()) {
- auto rmr = settings->mutable_remote_mirror_rule();
- TStringBuilder endpoint;
- if (partConfig.GetMirrorFrom().GetUseSecureConnection()) {
- endpoint << GRPCS_ENDPOINT_PREFIX;
- }
- endpoint << partConfig.GetMirrorFrom().GetEndpoint() << ":"
- << partConfig.GetMirrorFrom().GetEndpointPort();
- rmr->set_endpoint(endpoint);
- rmr->set_topic_path(partConfig.GetMirrorFrom().GetTopic());
- rmr->set_consumer_name(partConfig.GetMirrorFrom().GetConsumer());
- rmr->set_starting_message_timestamp_ms(partConfig.GetMirrorFrom().GetReadFromTimestampsMs());
- if (partConfig.GetMirrorFrom().HasCredentials()) {
- if (partConfig.GetMirrorFrom().GetCredentials().HasOauthToken()) {
- rmr->mutable_credentials()->set_oauth_token(
- NPersQueue::ObfuscateString(
- partConfig.GetMirrorFrom().GetCredentials().GetOauthToken())
- );
- } else if (partConfig.GetMirrorFrom().GetCredentials().HasJwtParams()) {
- rmr->mutable_credentials()->set_jwt_params(
- NPersQueue::ObfuscateString(
- partConfig.GetMirrorFrom().GetCredentials().GetJwtParams())
- );
- } else if (partConfig.GetMirrorFrom().GetCredentials().HasIam()) {
- rmr->mutable_credentials()->mutable_iam()->set_endpoint(
- partConfig.GetMirrorFrom().GetCredentials().GetIam().GetEndpoint()
- );
- rmr->mutable_credentials()->mutable_iam()->set_service_account_key(
- NPersQueue::ObfuscateString(
- partConfig.GetMirrorFrom().GetCredentials().GetIam().GetServiceAccountKey())
- );
+ }
+ if (partConfig.HasMirrorFrom()) {
+ auto rmr = settings->mutable_remote_mirror_rule();
+ TStringBuilder endpoint;
+ if (partConfig.GetMirrorFrom().GetUseSecureConnection()) {
+ endpoint << GRPCS_ENDPOINT_PREFIX;
+ }
+ endpoint << partConfig.GetMirrorFrom().GetEndpoint() << ":"
+ << partConfig.GetMirrorFrom().GetEndpointPort();
+ rmr->set_endpoint(endpoint);
+ rmr->set_topic_path(partConfig.GetMirrorFrom().GetTopic());
+ rmr->set_consumer_name(partConfig.GetMirrorFrom().GetConsumer());
+ rmr->set_starting_message_timestamp_ms(partConfig.GetMirrorFrom().GetReadFromTimestampsMs());
+ if (partConfig.GetMirrorFrom().HasCredentials()) {
+ if (partConfig.GetMirrorFrom().GetCredentials().HasOauthToken()) {
+ rmr->mutable_credentials()->set_oauth_token(
+ NPersQueue::ObfuscateString(
+ partConfig.GetMirrorFrom().GetCredentials().GetOauthToken())
+ );
+ } else if (partConfig.GetMirrorFrom().GetCredentials().HasJwtParams()) {
+ rmr->mutable_credentials()->set_jwt_params(
+ NPersQueue::ObfuscateString(
+ partConfig.GetMirrorFrom().GetCredentials().GetJwtParams())
+ );
+ } else if (partConfig.GetMirrorFrom().GetCredentials().HasIam()) {
+ rmr->mutable_credentials()->mutable_iam()->set_endpoint(
+ partConfig.GetMirrorFrom().GetCredentials().GetIam().GetEndpoint()
+ );
+ rmr->mutable_credentials()->mutable_iam()->set_service_account_key(
+ NPersQueue::ObfuscateString(
+ partConfig.GetMirrorFrom().GetCredentials().GetIam().GetServiceAccountKey())
+ );
}
}
- rmr->set_database(partConfig.GetMirrorFrom().GetDatabase());
+ rmr->set_database(partConfig.GetMirrorFrom().GetDatabase());
}
}
- return ReplyWithResult(Ydb::StatusIds::SUCCESS, result, ctx);
+ return ReplyWithResult(Ydb::StatusIds::SUCCESS, result, ctx);
}
@@ -332,8 +332,8 @@ void TRemoveReadRuleActor::ModifyPersqueueConfig(
GetProtoRequest()->consumer_name(),
ctx
);
- if (!error.Empty()) {
- return ReplyWithError(Ydb::StatusIds::NOT_FOUND, Ydb::PersQueue::ErrorCode::BAD_REQUEST, error, ctx);
+ if (!error.Empty()) {
+ return ReplyWithError(Ydb::StatusIds::NOT_FOUND, Ydb::PersQueue::ErrorCode::BAD_REQUEST, error, ctx);
}
}
diff --git a/ydb/services/persqueue_v1/grpc_pq_write_actor.cpp b/ydb/services/persqueue_v1/grpc_pq_write_actor.cpp
index 5d211b3d88..3038cc82a6 100644
--- a/ydb/services/persqueue_v1/grpc_pq_write_actor.cpp
+++ b/ydb/services/persqueue_v1/grpc_pq_write_actor.cpp
@@ -350,8 +350,8 @@ void TWriteSessionActor::Handle(TEvPQProxy::TEvWriteInit::TPtr& ev, const TActor
auto subGroup = GetServiceCounters(Counters, "pqproxy|SLI");
Aggr = {{{{"Account", TopicConverter->GetAccount()}}, {"total"}}};
- SLITotal = NKikimr::NPQ::TMultiCounter(subGroup, Aggr, {}, {"RequestsTotal"}, true, "sensor", false);
- SLIErrors = NKikimr::NPQ::TMultiCounter(subGroup, Aggr, {}, {"RequestsError"}, true, "sensor", false);
+ SLITotal = NKikimr::NPQ::TMultiCounter(subGroup, Aggr, {}, {"RequestsTotal"}, true, "sensor", false);
+ SLIErrors = NKikimr::NPQ::TMultiCounter(subGroup, Aggr, {}, {"RequestsError"}, true, "sensor", false);
SLITotal.Inc();
const auto& preferredCluster = init.preferred_cluster();
@@ -360,39 +360,39 @@ void TWriteSessionActor::Handle(TEvPQProxy::TEvWriteInit::TPtr& ev, const TActor
}
}
-void TWriteSessionActor::SetupCounters()
+void TWriteSessionActor::SetupCounters()
{
//now topic is checked, can create group for real topic, not garbage
auto subGroup = GetServiceCounters(Counters, "pqproxy|writeSession");
TVector<NPQ::TLabelsInfo> aggr = NKikimr::NPQ::GetLabels(LocalDC, TopicConverter->GetClientsideName());
- BytesInflight = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"BytesInflight"}, false);
- BytesInflightTotal = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"BytesInflightTotal"}, false);
- SessionsCreated = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"SessionsCreated"}, true);
- SessionsActive = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"SessionsActive"}, false);
- Errors = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"Errors"}, true);
+ BytesInflight = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"BytesInflight"}, false);
+ BytesInflightTotal = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"BytesInflightTotal"}, false);
+ SessionsCreated = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"SessionsCreated"}, true);
+ SessionsActive = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"SessionsActive"}, false);
+ Errors = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"Errors"}, true);
SessionsCreated.Inc();
SessionsActive.Inc();
}
-void TWriteSessionActor::SetupCounters(const TString& cloudId, const TString& dbId,
- const TString& folderId)
-{
- //now topic is checked, can create group for real topic, not garbage
- auto subGroup = NKikimr::NPQ::GetCountersForStream(Counters, "writeSession");
+void TWriteSessionActor::SetupCounters(const TString& cloudId, const TString& dbId,
+ const TString& folderId)
+{
+ //now topic is checked, can create group for real topic, not garbage
+ auto subGroup = NKikimr::NPQ::GetCountersForStream(Counters, "writeSession");
TVector<NPQ::TLabelsInfo> aggr = NKikimr::NPQ::GetLabelsForStream(TopicConverter->GetClientsideName(), cloudId, dbId, folderId);
-
- BytesInflight = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"stream.internal_write.bytes_proceeding"}, false);
- BytesInflightTotal = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"stream.internal_write.bytes_proceeding_total"}, false);
- SessionsCreated = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"stream.internal_write.sessions_created_per_second"}, true);
- SessionsActive = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"stream.internal_write.sessions_active"}, false);
- Errors = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"stream.internal_write.errors_per_second"}, true);
-
- SessionsCreated.Inc();
- SessionsActive.Inc();
-}
-
+
+ BytesInflight = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"stream.internal_write.bytes_proceeding"}, false);
+ BytesInflightTotal = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"stream.internal_write.bytes_proceeding_total"}, false);
+ SessionsCreated = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"stream.internal_write.sessions_created_per_second"}, true);
+ SessionsActive = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"stream.internal_write.sessions_active"}, false);
+ Errors = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"stream.internal_write.errors_per_second"}, true);
+
+ SessionsCreated.Inc();
+ SessionsActive.Inc();
+}
+
void TWriteSessionActor::InitCheckSchema(const TActorContext& ctx, bool needWaitSchema) {
LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "init check schema");
@@ -428,13 +428,13 @@ void TWriteSessionActor::Handle(TEvDescribeTopicsResponse::TPtr& ev, const TActo
PartitionToTablet[pi.GetPartitionId()] = pi.GetTabletId();
}
- if (AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen()) {
- const auto& tabletConfig = description.GetPQTabletConfig();
- SetupCounters(tabletConfig.GetYcCloudId(), tabletConfig.GetYdbDatabaseId(),
- tabletConfig.GetYcFolderId());
- } else {
- SetupCounters();
- }
+ if (AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen()) {
+ const auto& tabletConfig = description.GetPQTabletConfig();
+ SetupCounters(tabletConfig.GetYcCloudId(), tabletConfig.GetYdbDatabaseId(),
+ tabletConfig.GetYcFolderId());
+ } else {
+ SetupCounters();
+ }
Y_VERIFY (entry.SecurityObject);
ACL.Reset(new TAclWrapper(entry.SecurityObject));
@@ -471,32 +471,32 @@ void TWriteSessionActor::Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::
);
}
- const auto& pqDescription = navigate->ResultSet.front().PQGroupInfo->Description;
+ const auto& pqDescription = navigate->ResultSet.front().PQGroupInfo->Description;
- Y_VERIFY(pqDescription.PartitionsSize() > 0);
- Y_VERIFY(pqDescription.HasPQTabletConfig());
- InitialPQTabletConfig = pqDescription.GetPQTabletConfig();
+ Y_VERIFY(pqDescription.PartitionsSize() > 0);
+ Y_VERIFY(pqDescription.HasPQTabletConfig());
+ InitialPQTabletConfig = pqDescription.GetPQTabletConfig();
- if (!pqDescription.HasBalancerTabletID()) {
+ if (!pqDescription.HasBalancerTabletID()) {
TString errorReason = Sprintf("topic '%s' has no balancer, Marker# PQ93", TopicConverter->GetClientsideName().c_str());
CloseSession(errorReason, PersQueue::ErrorCode::UNKNOWN_TOPIC, ctx);
return;
}
- BalancerTabletId = pqDescription.GetBalancerTabletID();
+ BalancerTabletId = pqDescription.GetBalancerTabletID();
- for (ui32 i = 0; i < pqDescription.PartitionsSize(); ++i) {
- const auto& pi = pqDescription.GetPartitions(i);
+ for (ui32 i = 0; i < pqDescription.PartitionsSize(); ++i) {
+ const auto& pi = pqDescription.GetPartitions(i);
PartitionToTablet[pi.GetPartitionId()] = pi.GetTabletId();
}
- if (AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen()) {
- const auto& tabletConfig = pqDescription.GetPQTabletConfig();
- SetupCounters(tabletConfig.GetYcCloudId(), tabletConfig.GetYdbDatabaseId(),
- tabletConfig.GetYcFolderId());
- } else {
- SetupCounters();
- }
+ if (AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen()) {
+ const auto& tabletConfig = pqDescription.GetPQTabletConfig();
+ SetupCounters(tabletConfig.GetYcCloudId(), tabletConfig.GetYdbDatabaseId(),
+ tabletConfig.GetYcFolderId());
+ } else {
+ SetupCounters();
+ }
Y_VERIFY(!navigate->ResultSet.empty());
ACL.Reset(new TAclWrapper(navigate->ResultSet.front().SecurityObject));
@@ -705,7 +705,7 @@ void TWriteSessionActor::ProceedPartition(const ui32 partition, const TActorCont
auto subGroup = GetServiceCounters(Counters, "pqproxy|SLI");
InitLatency = NKikimr::NPQ::CreateSLIDurationCounter(subGroup, Aggr, "WriteInit", border, {100, 200, 500, 1000, 1500, 2000, 5000, 10000, 30000, 99999999});
- SLIBigLatency = NKikimr::NPQ::TMultiCounter(subGroup, Aggr, {}, {"RequestsBigLatency"}, true, "sesnor", false);
+ SLIBigLatency = NKikimr::NPQ::TMultiCounter(subGroup, Aggr, {}, {"RequestsBigLatency"}, true, "sesnor", false);
ui32 initDurationMs = (ctx.Now() - StartTime).MilliSeconds();
InitLatency.IncFor(initDurationMs, 1);
diff --git a/ydb/services/persqueue_v1/persqueue_ut.cpp b/ydb/services/persqueue_v1/persqueue_ut.cpp
index 98db74149a..3bc9b21728 100644
--- a/ydb/services/persqueue_v1/persqueue_ut.cpp
+++ b/ydb/services/persqueue_v1/persqueue_ut.cpp
@@ -1362,8 +1362,8 @@ namespace {
// TODO: Describe this assert in comment
UNIT_ASSERT(!ack->Acks.empty());
UNIT_ASSERT(ack->Acks.back().Stat);
- UNIT_ASSERT(ack->Acks.back().Stat->TotalTimeInPartitionQueue >= ack->Acks.back().Stat->PartitionQuotedTime);
- UNIT_ASSERT(ack->Acks.back().Stat->TotalTimeInPartitionQueue <= ack->Acks.back().Stat->PartitionQuotedTime + TDuration::Seconds(10));
+ UNIT_ASSERT(ack->Acks.back().Stat->TotalTimeInPartitionQueue >= ack->Acks.back().Stat->PartitionQuotedTime);
+ UNIT_ASSERT(ack->Acks.back().Stat->TotalTimeInPartitionQueue <= ack->Acks.back().Stat->PartitionQuotedTime + TDuration::Seconds(10));
}
}
GetCounters(server.CleverServer->GetRuntime()->GetMonPort(), "pqproxy", "writeSession", "account/topic1", "Vla");