summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authormokhotskii <[email protected]>2022-08-08 11:25:34 +0300
committermokhotskii <[email protected]>2022-08-08 11:25:34 +0300
commitdaf6368efef886e1b7894bd79df067b93130c781 (patch)
tree6be19e8d86da7ef2ed8c9afb2caf8ed7683117b9
parentf883bcc54069779989e8952de483dda899d7d6ab (diff)
Add TPartition LabeledCounters tests
Add LabeledCounters unit tests
-rw-r--r--ydb/core/persqueue/counters_ut.cpp339
-rw-r--r--ydb/core/persqueue/partition.cpp105
-rw-r--r--ydb/core/persqueue/partition.h6
-rw-r--r--ydb/core/persqueue/pq_impl.cpp13
-rw-r--r--ydb/core/persqueue/pq_ut.cpp141
-rw-r--r--ydb/core/persqueue/pq_ut.h23
-rw-r--r--ydb/core/persqueue/pq_ut_impl.cpp8
-rw-r--r--ydb/core/persqueue/ut/CMakeLists.darwin.txt16
-rw-r--r--ydb/core/persqueue/ut/CMakeLists.linux.txt16
-rw-r--r--ydb/core/persqueue/ut/counters_datastreams.html94
-rw-r--r--ydb/core/persqueue/ut/counters_labeled.html247
-rw-r--r--ydb/core/persqueue/ut/counters_pqproxy.html654
-rw-r--r--ydb/core/persqueue/ut/counters_pqproxy_firstclass.html39
13 files changed, 1496 insertions, 205 deletions
diff --git a/ydb/core/persqueue/counters_ut.cpp b/ydb/core/persqueue/counters_ut.cpp
new file mode 100644
index 00000000000..64e4d3ce315
--- /dev/null
+++ b/ydb/core/persqueue/counters_ut.cpp
@@ -0,0 +1,339 @@
+#include <library/cpp/testing/unittest/registar.h>
+#include <ydb/core/mon/sync_http_mon.h>
+
+#include "pq_ut.h"
+
+namespace NKikimr {
+
+Y_UNIT_TEST_SUITE(PQCountersSimple) {
+
+Y_UNIT_TEST(Partition) {
+ TTestContext tc;
+ TFinalizer finalizer(tc);
+ bool activeZone{false};
+ tc.Prepare("", [](TTestActorRuntime&) {}, activeZone, false, true);
+ tc.Runtime->SetScheduledLimit(100);
+
+ TVector<std::pair<ui64, TString>> data;
+ TString s{32, 'c'};
+ ui32 pp = 8 + 4 + 2 + 9;
+ for (ui32 i = 0; i < 10; ++i) {
+ data.push_back({i + 1, s.substr(pp)});
+ }
+ PQTabletPrepare({}, {}, tc);
+ CmdWrite(0, "sourceid0", data, tc, false, {}, true);
+ CmdWrite(0, "sourceid1", data, tc, false);
+ CmdWrite(0, "sourceid2", data, tc, false);
+
+ {
+ auto counters = tc.Runtime->GetAppData(0).Counters;
+ auto dbGroup = GetServiceCounters(counters, "pqproxy");
+ TStringStream countersStr;
+ dbGroup->OutputHtml(countersStr);
+ Cerr << "ASDFGS: " << countersStr.Str() << Endl;
+ TString referenceCounters = NResource::Find(TStringBuf("counters_pqproxy.html"));
+ UNIT_ASSERT_EQUAL(countersStr.Str() + "\n", referenceCounters);
+ }
+
+ {
+ auto counters = tc.Runtime->GetAppData(0).Counters;
+ auto dbGroup = GetServiceCounters(counters, "datastreams");
+ TStringStream countersStr;
+ dbGroup->OutputHtml(countersStr);
+ UNIT_ASSERT_EQUAL(countersStr.Str(), "<pre></pre>");
+ }
+}
+
+Y_UNIT_TEST(PartitionFirstClass) {
+ TTestContext tc;
+ TFinalizer finalizer(tc);
+ bool activeZone{false};
+ tc.Prepare("", [](TTestActorRuntime&){}, activeZone, true, true);
+ tc.Runtime->SetScheduledLimit(100);
+
+ TVector<std::pair<ui64, TString>> data;
+ TString s{32, 'c'};
+ ui32 pp = 8 + 4 + 2 + 9;
+ for (ui32 i = 0; i < 10; ++i) {
+ data.push_back({i + 1, s.substr(pp)});
+ }
+ PQTabletPrepare({}, {}, tc);
+ CmdWrite(0, "sourceid0", data, tc, false, {}, true);
+ CmdWrite(0, "sourceid1", data, tc, false);
+ CmdWrite(0, "sourceid2", data, tc, false);
+
+ {
+ auto counters = tc.Runtime->GetAppData(0).Counters;
+ auto dbGroup = GetServiceCounters(counters, "pqproxy");
+ TStringStream countersStr;
+ dbGroup->OutputHtml(countersStr);
+ TString referenceCounters = NResource::Find(TStringBuf("counters_pqproxy_firstclass.html"));
+ UNIT_ASSERT_EQUAL(countersStr.Str() + "\n", referenceCounters);
+ }
+
+ {
+ auto counters = tc.Runtime->GetAppData(0).Counters;
+ auto dbGroup = GetServiceCounters(counters, "datastreams");
+ TStringStream countersStr;
+ dbGroup->OutputHtml(countersStr);
+ const TString referenceCounters = NResource::Find(TStringBuf("counters_datastreams.html"));
+ UNIT_ASSERT_EQUAL(countersStr.Str() + "\n", referenceCounters);
+ }
+}
+
+} // Y_UNIT_TEST_SUITE(PQCountersSimple)
+
+Y_UNIT_TEST_SUITE(PQCountersLabeled) {
+
+struct THttpRequest : NMonitoring::IHttpRequest {
+ HTTP_METHOD Method;
+ TCgiParameters CgiParameters;
+ THttpHeaders HttpHeaders;
+
+ THttpRequest(HTTP_METHOD method)
+ : Method(method)
+ {
+ CgiParameters.emplace("type", TTabletTypes::TypeToStr(TTabletTypes::PersQueue));
+ }
+
+ ~THttpRequest() {}
+
+ const char* GetURI() const override {
+ return "";
+ }
+
+ const char* GetPath() const override {
+ return "";
+ }
+
+ const TCgiParameters& GetParams() const override {
+ return CgiParameters;
+ }
+
+ const TCgiParameters& GetPostParams() const override {
+ return CgiParameters;
+ }
+
+ TStringBuf GetPostContent() const override {
+ return TString();
+ }
+
+ HTTP_METHOD GetMethod() const override {
+ return Method;
+ }
+
+ const THttpHeaders& GetHeaders() const override {
+ return HttpHeaders;
+ }
+
+ TString GetRemoteAddr() const override {
+ return TString();
+ }
+};
+
+Y_UNIT_TEST(Partition) {
+ TTestContext tc;
+ RunTestWithReboots({}, [&]() {
+ return tc.InitialEventsFilter.Prepare();
+ }, [&](const TString& dispatchName, std::function<void(TTestActorRuntime&)> setup, bool& activeZone) {
+ TFinalizer finalizer(tc);
+ tc.Prepare(dispatchName, setup, activeZone, false, true);
+ tc.Runtime->SetScheduledLimit(100);
+
+ TVector<std::pair<ui64, TString>> data;
+ TString s{32, 'c'};
+ ui32 pp = 8 + 4 + 2 + 9;
+ for (ui32 i = 0; i < 10; ++i) {
+ data.push_back({i + 1, s.substr(pp)});
+ }
+ PQTabletPrepare({}, {}, tc);
+
+ IActor* actor = CreateTabletCountersAggregator(false);
+ auto aggregatorId = tc.Runtime->Register(actor);
+ tc.Runtime->EnableScheduleForActor(aggregatorId);
+
+ CmdWrite(0, "sourceid0", data, tc, false, {}, true);
+ CmdWrite(0, "sourceid1", data, tc, false);
+ CmdWrite(0, "sourceid2", data, tc, false);
+ PQGetPartInfo(0, 30, tc);
+
+ {
+ TDispatchOptions options;
+ options.FinalEvents.emplace_back(TEvTabletCounters::EvTabletAddLabeledCounters);
+ tc.Runtime->DispatchEvents(options);
+ }
+
+ IActor* actorX = CreateClusterLabeledCountersAggregatorActor(tc.Edge, TTabletTypes::PersQueue);
+ tc.Runtime->Register(actorX);
+
+ TAutoPtr<IEventHandle> handle;
+ TEvTabletCounters::TEvTabletLabeledCountersResponse *result;
+ result = tc.Runtime->GrabEdgeEvent<TEvTabletCounters::TEvTabletLabeledCountersResponse>(handle);
+ UNIT_ASSERT(result);
+
+ THttpRequest httpReq(HTTP_METHOD_GET);
+ NMonitoring::TMonService2HttpRequest monReq(nullptr, &httpReq, nullptr, nullptr, "", nullptr);
+ tc.Runtime->Send(new IEventHandle(aggregatorId, tc.Edge, new NMon::TEvHttpInfo(monReq)));
+
+ TAutoPtr<IEventHandle> handle1;
+ auto resp = tc.Runtime->GrabEdgeEvent<NMon::TEvHttpInfoRes>(handle1);
+ const TString countersStr = ((NMon::TEvHttpInfoRes*) resp)->Answer;
+
+ const TString referenceCounters = NResource::Find(TStringBuf("counters_datastreams.html"));
+ UNIT_ASSERT_VALUES_EQUAL(countersStr, referenceCounters);
+ }, 1);
+}
+
+Y_UNIT_TEST(PartitionFirstClass) {
+ TTestContext tc;
+ RunTestWithReboots({}, [&]() {
+ return tc.InitialEventsFilter.Prepare();
+ }, [&](const TString& dispatchName, std::function<void(TTestActorRuntime&)> setup, bool& activeZone) {
+ TFinalizer finalizer(tc);
+ activeZone = false;
+
+ tc.Prepare(dispatchName, setup, activeZone, true, true);
+ tc.Runtime->SetScheduledLimit(100);
+
+ TVector<std::pair<ui64, TString>> data;
+ TString s{32, 'c'};
+ ui32 pp = 8 + 4 + 2 + 9;
+ for (ui32 i = 0; i < 10; ++i) {
+ data.push_back({i + 1, s.substr(pp)});
+ }
+ PQTabletPrepare({}, {}, tc);
+
+ IActor* actor = CreateTabletCountersAggregator(false);
+ auto aggregatorId = tc.Runtime->Register(actor);
+ tc.Runtime->EnableScheduleForActor(aggregatorId);
+
+ CmdWrite(0, "sourceid0", data, tc, false, {}, true);
+ CmdWrite(0, "sourceid1", data, tc, false);
+ CmdWrite(0, "sourceid2", data, tc, false);
+ PQGetPartInfo(0, 30, tc);
+
+ {
+ TDispatchOptions options;
+ options.FinalEvents.emplace_back(TEvTabletCounters::EvTabletAddLabeledCounters);
+ auto processedCountersEvent = tc.Runtime->DispatchEvents(options, TDuration::MilliSeconds(500));
+ UNIT_ASSERT_VALUES_EQUAL(processedCountersEvent, false);
+ }
+ });
+}
+
+void CheckLabeledCountersResponse(TTestContext& tc, ui32 count, TVector<TString> mustHave = {}) {
+ IActor* actor = CreateClusterLabeledCountersAggregatorActor(tc.Edge, TTabletTypes::PersQueue);
+ tc.Runtime->Register(actor);
+
+ TAutoPtr<IEventHandle> handle;
+ TEvTabletCounters::TEvTabletLabeledCountersResponse *result;
+ result = tc.Runtime->GrabEdgeEvent<TEvTabletCounters::TEvTabletLabeledCountersResponse>(handle);
+ UNIT_ASSERT(result);
+
+ THashSet<TString> groups;
+
+ for (ui32 i = 0; i < result->Record.LabeledCountersByGroupSize(); ++i) {
+ auto& c = result->Record.GetLabeledCountersByGroup(i);
+ groups.insert(c.GetGroup());
+ }
+ UNIT_ASSERT_VALUES_EQUAL(groups.size(), count);
+ for (auto& g : mustHave) {
+ UNIT_ASSERT(groups.contains(g));
+ }
+}
+
+Y_UNIT_TEST(ImportantFlagSwitching) {
+ const static TString TOPIC_NAME = "rt3.dc1--asdfgs--topic";
+ TTestContext tc;
+ RunTestWithReboots(tc.TabletIds, [&]() {
+ return tc.InitialEventsFilter.Prepare();
+ }, [&](const TString& dispatchName, std::function<void(TTestActorRuntime&)> setup, bool& activeZone) {
+ TFinalizer finalizer(tc);
+ tc.Prepare(dispatchName, setup, activeZone);
+ activeZone = false;
+ tc.Runtime->SetScheduledLimit(600);
+
+ auto MakeTopics = [&] (const TVector<TString>& users) {
+ TVector<TString> res;
+ for (const auto& u : users) {
+ res.emplace_back(NKikimr::JoinPath({u, TOPIC_NAME}));
+ }
+ return res;
+ };
+
+ PQTabletPrepare({}, {}, tc);
+ {
+ TDispatchOptions options;
+ options.FinalEvents.emplace_back(TEvTabletCounters::EvTabletAddLabeledCounters);
+ tc.Runtime->DispatchEvents(options);
+ }
+ // Topic counters only
+ CheckLabeledCountersResponse(tc, 8);
+
+ // Topic counters + important
+ PQTabletPrepare({}, {{"user", true}}, tc);
+ {
+ TDispatchOptions options;
+ options.FinalEvents.emplace_back(TEvTabletCounters::EvTabletAddLabeledCounters);
+ tc.Runtime->DispatchEvents(options);
+ }
+ CheckLabeledCountersResponse(tc, 8, {NKikimr::JoinPath({"user/1", TOPIC_NAME})});
+
+ PQTabletPrepare({}, {}, tc);
+ {
+ TDispatchOptions options;
+ options.FinalEvents.emplace_back(TEvTabletCounters::EvTabletAddLabeledCounters);
+ tc.Runtime->DispatchEvents(options);
+ }
+ {
+ TDispatchOptions options;
+ options.FinalEvents.emplace_back(TEvTabletCounters::EvTabletAddLabeledCounters);
+ tc.Runtime->DispatchEvents(options);
+ }
+ // Topic counters + not important
+ CheckLabeledCountersResponse(tc, 8, MakeTopics({"user/0"}));
+
+ // Topic counters + not important
+ PQTabletPrepare({}, {{"user", true}, {"user2", true}}, tc);
+ {
+ TDispatchOptions options;
+ options.FinalEvents.emplace_back(TEvTabletCounters::EvTabletAddLabeledCounters);
+ tc.Runtime->DispatchEvents(options);
+ }
+ {
+ TDispatchOptions options;
+ options.FinalEvents.emplace_back(TEvTabletCounters::EvTabletAddLabeledCounters);
+ tc.Runtime->DispatchEvents(options);
+ }
+ CheckLabeledCountersResponse(tc, 11, MakeTopics({"user/1", "user2/1"}));
+
+ PQTabletPrepare({}, {{"user", true}, {"user2", false}}, tc);
+ {
+ TDispatchOptions options;
+ options.FinalEvents.emplace_back(TEvTabletCounters::EvTabletAddLabeledCounters);
+ tc.Runtime->DispatchEvents(options);
+ }
+ {
+ TDispatchOptions options;
+ options.FinalEvents.emplace_back(TEvTabletCounters::EvTabletAddLabeledCounters);
+ tc.Runtime->DispatchEvents(options);
+ }
+ CheckLabeledCountersResponse(tc, 12, MakeTopics({"user/1", "user2/0"}));
+
+ PQTabletPrepare({}, {{"user", true}}, tc);
+ {
+ TDispatchOptions options;
+ options.FinalEvents.emplace_back(TEvTabletCounters::EvTabletAddLabeledCounters);
+ tc.Runtime->DispatchEvents(options);
+ }
+ {
+ TDispatchOptions options;
+ options.FinalEvents.emplace_back(TEvTabletCounters::EvTabletAddLabeledCounters);
+ tc.Runtime->DispatchEvents(options);
+ }
+ CheckLabeledCountersResponse(tc, 8, MakeTopics({"user/1"}));
+ });
+}
+} // Y_UNIT_TEST_SUITE(PQCountersLabeled)
+
+} // namespace NKikimr
diff --git a/ydb/core/persqueue/partition.cpp b/ydb/core/persqueue/partition.cpp
index a4151cb566c..14e98aa2996 100644
--- a/ydb/core/persqueue/partition.cpp
+++ b/ydb/core/persqueue/partition.cpp
@@ -41,8 +41,7 @@ struct TPartition::THasDataReq {
TMaybe<ui64> Cookie;
TString ClientId;
- bool operator < (const THasDataReq& req) const
- {
+ bool operator < (const THasDataReq& req) const {
return Num < req.Num;
}
};
@@ -51,8 +50,7 @@ struct TPartition::THasDataDeadline {
TInstant Deadline;
TPartition::THasDataReq Request;
- bool operator < (const THasDataDeadline& dl) const
- {
+ bool operator < (const THasDataDeadline& dl) const {
return Deadline < dl.Deadline || Deadline == dl.Deadline && Request < dl.Request;
}
};
@@ -460,10 +458,6 @@ TPartition::TPartition(ui64 tabletId, ui32 partition, const TActorId& tablet, co
, InitDuration(TDuration::Zero())
, InitDone(false)
, NewPartition(newPartition)
- // TODO: ToReview - Which name to use here? It verifies in tablet_counters_protobuf.h:633 on proper path
- , PartitionCounters(
- topicConverter->IsFirstClass() ? nullptr
- : new TPartitionLabeledCounters(topicConverter->GetClientsideName(), partition))
, Subscriber(partition, TabletCounters, Tablet)
, WriteCycleStartTime(ctx.Now())
, WriteCycleSize(0)
@@ -494,6 +488,12 @@ TPartition::TPartition(ui64 tabletId, ui32 partition, const TActorId& tablet, co
CalcTopicWriteQuotaParams();
TabletCounters.Populate(counters);
+
+ if (topicConverter->IsFirstClass()) {
+ PartitionCountersLabeled = THolder<TPartitionLabeledCounters>(nullptr);
+ } else {
+ PartitionCountersLabeled = THolder<TPartitionLabeledCounters>(new TPartitionLabeledCounters(topicConverter->GetClientsideName(), partition));
+ }
}
void TPartition::HandleMonitoring(TEvPQ::TEvMonRequest::TPtr& ev, const TActorContext& ctx) {
@@ -1699,11 +1699,11 @@ void TPartition::InitComplete(const TActorContext& ctx) {
Y_VERIFY(userInfoPair.second.Offset >= 0);
ReadTimestampForOffset(userInfoPair.first, userInfoPair.second, ctx);
}
- if (PartitionCounters) {
- PartitionCounters->GetCounters()[METRIC_INIT_TIME] = InitDuration.MilliSeconds();
- PartitionCounters->GetCounters()[METRIC_LIFE_TIME] = CreationTime.MilliSeconds();
- PartitionCounters->GetCounters()[METRIC_PARTITIONS] = 1;
- ctx.Send(Tablet, new TEvPQ::TEvPartitionLabeledCounters(Partition, *PartitionCounters));
+ if (PartitionCountersLabeled) {
+ PartitionCountersLabeled->GetCounters()[METRIC_INIT_TIME] = InitDuration.MilliSeconds();
+ PartitionCountersLabeled->GetCounters()[METRIC_LIFE_TIME] = CreationTime.MilliSeconds();
+ PartitionCountersLabeled->GetCounters()[METRIC_PARTITIONS] = 1;
+ ctx.Send(Tablet, new TEvPQ::TEvPartitionLabeledCounters(Partition, *PartitionCountersLabeled));
}
UpdateUserInfoEndOffset(ctx.Now());
@@ -3151,7 +3151,7 @@ ui64 TPartition::GetSizeLag(i64 offset) {
void TPartition::ReportCounters(const TActorContext& ctx) {
- if (!PartitionCounters) {
+ if (!PartitionCountersLabeled) {
return;
}
//per client counters
@@ -3287,58 +3287,58 @@ void TPartition::ReportCounters(const TActorContext& ctx) {
}
}
bool haveChanges = false;
- if (SourceIdStorage.GetInMemorySourceIds().size() != PartitionCounters->GetCounters()[METRIC_MAX_NUM_SIDS].Get()) {
+ if (SourceIdStorage.GetInMemorySourceIds().size() != PartitionCountersLabeled->GetCounters()[METRIC_MAX_NUM_SIDS].Get()) {
haveChanges = true;
- PartitionCounters->GetCounters()[METRIC_MAX_NUM_SIDS].Set(SourceIdStorage.GetInMemorySourceIds().size());
- PartitionCounters->GetCounters()[METRIC_NUM_SIDS].Set(SourceIdStorage.GetInMemorySourceIds().size());
+ PartitionCountersLabeled->GetCounters()[METRIC_MAX_NUM_SIDS].Set(SourceIdStorage.GetInMemorySourceIds().size());
+ PartitionCountersLabeled->GetCounters()[METRIC_NUM_SIDS].Set(SourceIdStorage.GetInMemorySourceIds().size());
}
TDuration lifetimeNow = ctx.Now() - SourceIdStorage.MinAvailableTimestamp(ctx.Now());
- if (lifetimeNow.MilliSeconds() != PartitionCounters->GetCounters()[METRIC_MIN_SID_LIFETIME].Get()) {
+ if (lifetimeNow.MilliSeconds() != PartitionCountersLabeled->GetCounters()[METRIC_MIN_SID_LIFETIME].Get()) {
haveChanges = true;
- PartitionCounters->GetCounters()[METRIC_MIN_SID_LIFETIME].Set(lifetimeNow.MilliSeconds());
+ PartitionCountersLabeled->GetCounters()[METRIC_MIN_SID_LIFETIME].Set(lifetimeNow.MilliSeconds());
}
const ui64 headGapSize = DataKeysBody.empty() ? 0 : (Head.Offset - (DataKeysBody.back().Key.GetOffset() + DataKeysBody.back().Key.GetCount()));
const ui64 gapSize = GapSize + headGapSize;
- if (gapSize != PartitionCounters->GetCounters()[METRIC_GAPS_SIZE].Get()) {
+ if (gapSize != PartitionCountersLabeled->GetCounters()[METRIC_GAPS_SIZE].Get()) {
haveChanges = true;
- PartitionCounters->GetCounters()[METRIC_MAX_GAPS_SIZE].Set(gapSize);
- PartitionCounters->GetCounters()[METRIC_GAPS_SIZE].Set(gapSize);
+ PartitionCountersLabeled->GetCounters()[METRIC_MAX_GAPS_SIZE].Set(gapSize);
+ PartitionCountersLabeled->GetCounters()[METRIC_GAPS_SIZE].Set(gapSize);
}
const ui32 gapsCount = GapOffsets.size() + (headGapSize ? 1 : 0);
- if (gapsCount != PartitionCounters->GetCounters()[METRIC_GAPS_COUNT].Get()) {
+ if (gapsCount != PartitionCountersLabeled->GetCounters()[METRIC_GAPS_COUNT].Get()) {
haveChanges = true;
- PartitionCounters->GetCounters()[METRIC_MAX_GAPS_COUNT].Set(gapsCount);
- PartitionCounters->GetCounters()[METRIC_GAPS_COUNT].Set(gapsCount);
+ PartitionCountersLabeled->GetCounters()[METRIC_MAX_GAPS_COUNT].Set(gapsCount);
+ PartitionCountersLabeled->GetCounters()[METRIC_GAPS_COUNT].Set(gapsCount);
}
ui64 speed = WriteQuota.GetTotalSpeed();
- if (speed != PartitionCounters->GetCounters()[METRIC_WRITE_QUOTA_BYTES].Get()) {
+ if (speed != PartitionCountersLabeled->GetCounters()[METRIC_WRITE_QUOTA_BYTES].Get()) {
haveChanges = true;
- PartitionCounters->GetCounters()[METRIC_WRITE_QUOTA_BYTES].Set(speed);
+ PartitionCountersLabeled->GetCounters()[METRIC_WRITE_QUOTA_BYTES].Set(speed);
}
ui64 availSec = WriteQuota.GetAvailableAvgSec(ctx.Now());
- if (availSec != PartitionCounters->GetCounters()[METRIC_MIN_WRITE_QUOTA_BYTES_AVAIL_SEC].Get()) {
+ if (availSec != PartitionCountersLabeled->GetCounters()[METRIC_MIN_WRITE_QUOTA_BYTES_AVAIL_SEC].Get()) {
haveChanges = true;
- PartitionCounters->GetCounters()[METRIC_MIN_WRITE_QUOTA_BYTES_AVAIL_SEC].Set(availSec);
+ PartitionCountersLabeled->GetCounters()[METRIC_MIN_WRITE_QUOTA_BYTES_AVAIL_SEC].Set(availSec);
}
ui64 availMin = WriteQuota.GetAvailableAvgMin(ctx.Now());
- if (availMin != PartitionCounters->GetCounters()[METRIC_MIN_WRITE_QUOTA_BYTES_AVAIL_MIN].Get()) {
+ if (availMin != PartitionCountersLabeled->GetCounters()[METRIC_MIN_WRITE_QUOTA_BYTES_AVAIL_MIN].Get()) {
haveChanges = true;
- PartitionCounters->GetCounters()[METRIC_MIN_WRITE_QUOTA_BYTES_AVAIL_MIN].Set(availMin);
+ PartitionCountersLabeled->GetCounters()[METRIC_MIN_WRITE_QUOTA_BYTES_AVAIL_MIN].Set(availMin);
}
ui32 id = METRIC_TOTAL_WRITE_SPEED_1;
for (ui32 i = 0; i < AvgWriteBytes.size(); ++i) {
ui64 avg = AvgWriteBytes[i].GetValue();
- if (avg != PartitionCounters->GetCounters()[id].Get()) {
+ if (avg != PartitionCountersLabeled->GetCounters()[id].Get()) {
haveChanges = true;
- PartitionCounters->GetCounters()[id].Set(avg); //total
- PartitionCounters->GetCounters()[id + 1].Set(avg); //max
+ PartitionCountersLabeled->GetCounters()[id].Set(avg); //total
+ PartitionCountersLabeled->GetCounters()[id + 1].Set(avg); //max
}
id += 2;
}
@@ -3348,10 +3348,10 @@ void TPartition::ReportCounters(const TActorContext& ctx) {
id = METRIC_TOTAL_QUOTA_SPEED_1;
for (ui32 i = 0; i < AvgQuotaBytes.size(); ++i) {
ui64 avg = AvgQuotaBytes[i].GetValue();
- if (avg != PartitionCounters->GetCounters()[id].Get()) {
+ if (avg != PartitionCountersLabeled->GetCounters()[id].Get()) {
haveChanges = true;
- PartitionCounters->GetCounters()[id].Set(avg); //total
- PartitionCounters->GetCounters()[id + 1].Set(avg); //max
+ PartitionCountersLabeled->GetCounters()[id].Set(avg); //total
+ PartitionCountersLabeled->GetCounters()[id + 1].Set(avg); //max
}
id += 2;
}
@@ -3359,39 +3359,38 @@ void TPartition::ReportCounters(const TActorContext& ctx) {
if (WriteQuota.GetTotalSpeed()) {
ui64 quotaUsage = ui64(AvgQuotaBytes[1].GetValue()) * 1000000 / WriteQuota.GetTotalSpeed() / 60;
- if (quotaUsage != PartitionCounters->GetCounters()[METRIC_WRITE_QUOTA_USAGE].Get()) {
+ if (quotaUsage != PartitionCountersLabeled->GetCounters()[METRIC_WRITE_QUOTA_USAGE].Get()) {
haveChanges = true;
- PartitionCounters->GetCounters()[METRIC_WRITE_QUOTA_USAGE].Set(quotaUsage);
+ PartitionCountersLabeled->GetCounters()[METRIC_WRITE_QUOTA_USAGE].Set(quotaUsage);
}
}
ui64 partSize = BodySize + Head.PackedSize;
- if (partSize != PartitionCounters->GetCounters()[METRIC_TOTAL_PART_SIZE].Get()) {
+ if (partSize != PartitionCountersLabeled->GetCounters()[METRIC_TOTAL_PART_SIZE].Get()) {
haveChanges = true;
- PartitionCounters->GetCounters()[METRIC_MAX_PART_SIZE].Set(partSize);
- PartitionCounters->GetCounters()[METRIC_TOTAL_PART_SIZE].Set(partSize);
+ PartitionCountersLabeled->GetCounters()[METRIC_MAX_PART_SIZE].Set(partSize);
+ PartitionCountersLabeled->GetCounters()[METRIC_TOTAL_PART_SIZE].Set(partSize);
}
ui64 ts = (WriteTimestamp.MilliSeconds() < MIN_TIMESTAMP_MS) ? Max<i64>() : WriteTimestamp.MilliSeconds();
- if (PartitionCounters->GetCounters()[METRIC_LAST_WRITE_TIME].Get() != ts) {
+ if (PartitionCountersLabeled->GetCounters()[METRIC_LAST_WRITE_TIME].Get() != ts) {
haveChanges = true;
- PartitionCounters->GetCounters()[METRIC_LAST_WRITE_TIME].Set(ts);
+ PartitionCountersLabeled->GetCounters()[METRIC_LAST_WRITE_TIME].Set(ts);
}
ui64 timeLag = WriteLagMs.GetValue();
- if (PartitionCounters->GetCounters()[METRIC_WRITE_TIME_LAG_MS].Get() != timeLag) {
+ if (PartitionCountersLabeled->GetCounters()[METRIC_WRITE_TIME_LAG_MS].Get() != timeLag) {
haveChanges = true;
- PartitionCounters->GetCounters()[METRIC_WRITE_TIME_LAG_MS].Set(timeLag);
+ PartitionCountersLabeled->GetCounters()[METRIC_WRITE_TIME_LAG_MS].Set(timeLag);
}
if (haveChanges) {
- ctx.Send(Tablet, new TEvPQ::TEvPartitionLabeledCounters(Partition, *PartitionCounters));
+ ctx.Send(Tablet, new TEvPQ::TEvPartitionLabeledCounters(Partition, *PartitionCountersLabeled));
}
}
void TPartition::Handle(TEvKeyValue::TEvResponse::TPtr& ev, const TActorContext& ctx) {
-
auto& response = ev->Get()->Record;
//check correctness of response
@@ -4002,11 +4001,14 @@ void TPartition::WriteClientInfo(const ui64 cookie, TUserInfo& userInfo, const T
);
offset = EndOffset;
ev->Offset = offset;
-/* TabletCounters.Cumulative()[COUNTER_PQ_SET_CLIENT_OFFSET_ERROR].Increment(1);
+/*
+ TODO:
+ TabletCounters.Cumulative()[COUNTER_PQ_SET_CLIENT_OFFSET_ERROR].Increment(1);
ReplyError(ctx, ev->Cookie, NPersQueue::NErrorCode::SET_OFFSET_ERROR_COMMIT_TO_FUTURE,
TStringBuilder() << "can't commit to future. Offset " << offset << " EndOffset " << EndOffset);
userInfo.UserActrs.pop_front();
- continue;*/
+ continue;
+*/
}
TBuffer idata;
@@ -4270,7 +4272,8 @@ bool TPartition::AppendHeadWithNewWrites(TEvKeyValue::TEvRequest* request, const
TInstant::MilliSeconds(p.Msg.CreateTimestamp == 0 ? curOffset : p.Msg.CreateTimestamp),
p.Msg.UncompressedSize, p.Msg.PartitionKey, p.Msg.ExplicitHashKey); //remove curOffset when LB will report CTime
- ui64 writeLagMs = (WriteTimestamp - TInstant::MilliSeconds(p.Msg.CreateTimestamp)).MilliSeconds();
+ const ui64 writeLagMs =
+ (WriteTimestamp - TInstant::MilliSeconds(p.Msg.CreateTimestamp)).MilliSeconds();
WriteLagMs.Update(writeLagMs, WriteTimestamp);
if (InputTimeLag) {
InputTimeLag->IncFor(writeLagMs, 1);
diff --git a/ydb/core/persqueue/partition.h b/ydb/core/persqueue/partition.h
index bdd7ed90db6..c25b1f9c7f1 100644
--- a/ydb/core/persqueue/partition.h
+++ b/ydb/core/persqueue/partition.h
@@ -21,9 +21,9 @@ namespace NKikimr::NPQ {
static const ui32 MAX_BLOB_PART_SIZE = 500_KB;
-ui64 GetOffsetEstimate(const std::deque<TDataKey>& container, TInstant timestamp, ui64 headOffset);
+using TPartitionLabeledCounters = TProtobufTabletLabeledCounters<EPartitionLabeledCounters_descriptor>;
-typedef TProtobufTabletLabeledCounters<EPartitionLabeledCounters_descriptor> TPartitionLabeledCounters;
+ui64 GetOffsetEstimate(const std::deque<TDataKey>& container, TInstant timestamp, ui64 headOffset);
class TKeyLevel;
struct TMirrorerInfo;
@@ -408,7 +408,7 @@ private:
std::deque<THolder<TEvPQ::TEvChangeOwner>> WaitToChangeOwner;
TTabletCountersBase TabletCounters;
- THolder<TPartitionLabeledCounters> PartitionCounters;
+ THolder<TPartitionLabeledCounters> PartitionCountersLabeled;
TSubscriber Subscriber;
diff --git a/ydb/core/persqueue/pq_impl.cpp b/ydb/core/persqueue/pq_impl.cpp
index 75b325d5409..c66c6af4faf 100644
--- a/ydb/core/persqueue/pq_impl.cpp
+++ b/ydb/core/persqueue/pq_impl.cpp
@@ -652,8 +652,8 @@ void TPersQueue::ApplyNewConfigAndReply(const TActorContext& ctx)
const auto partitionId = partition.GetPartitionId();
if (Partitions.find(partitionId) == Partitions.end()) {
Partitions.emplace(partitionId, TPartitionInfo(
- ctx.Register(new TPartition(TabletID(), partitionId, ctx.SelfID, CacheActor, TopicConverter, IsLocalDC,
- DCId, Config, *Counters, ctx, true)),
+ ctx.Register(new TPartition(TabletID(), partitionId, ctx.SelfID, CacheActor, TopicConverter,
+ IsLocalDC, DCId, Config, *Counters, ctx, true)),
GetPartitionKeyRange(partition),
true,
*Counters
@@ -987,8 +987,10 @@ void TPersQueue::AggregateAndSendLabeledCountersFor(const TString& group, const
Y_VERIFY(aggr->HasCounters());
TActorId countersAggregator = MakeTabletCountersAggregatorID(ctx.SelfID.NodeId());
- ctx.Send(countersAggregator, new TEvTabletCounters::TEvTabletAddLabeledCounters(
- CounterEventsInflight[group], TabletID(), TTabletTypes::PersQueue, aggr));
+ ctx.Send(countersAggregator,
+ new TEvTabletCounters::TEvTabletAddLabeledCounters(
+ CounterEventsInflight[group], TabletID(), TTabletTypes::PersQueue, aggr)
+ );
}
}
@@ -1535,7 +1537,7 @@ void TPersQueue::HandleWriteRequest(const ui64 responseCookie, const TActorId& p
for (ui32 i = 0; i < req.CmdWriteSize(); ++i) {
const auto& cmd = req.GetCmdWrite(i);
- if (AppData(ctx)->Counters) {
+ if (AppData(ctx)->Counters && !AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen()) {
auto counters = AppData(ctx)->Counters;
TString clientDC = to_lower(cmd.HasClientDC() ? cmd.GetClientDC() : "unknown");
clientDC.to_title();
@@ -2117,6 +2119,7 @@ void TPersQueue::Handle(TEvInterconnect::TEvNodeInfo::TPtr& ev, const TActorCont
}
void TPersQueue::HandleWakeup(const TActorContext& ctx) {
+ // TIP: Send LabeledCounters here
THashSet<TString> groups;
for (auto& p : Partitions) {
for (auto& m : p.second.LabeledCounters) {
diff --git a/ydb/core/persqueue/pq_ut.cpp b/ydb/core/persqueue/pq_ut.cpp
index d68710b1562..7986bb63cd6 100644
--- a/ydb/core/persqueue/pq_ut.cpp
+++ b/ydb/core/persqueue/pq_ut.cpp
@@ -1,14 +1,9 @@
#include "pq_ut.h"
-#include <ydb/core/testlib/basics/runtime.h>
-#include <ydb/core/tablet_flat/tablet_flat_executed.h>
#include <ydb/core/tx/schemeshard/schemeshard.h>
-#include <ydb/public/lib/base/msgbus.h>
#include <ydb/core/keyvalue/keyvalue_events.h>
#include <ydb/core/persqueue/events/global.h>
-#include <ydb/core/tablet/tablet_counters_aggregator.h>
#include <ydb/core/persqueue/partition.h>
-#include <ydb/core/engine/minikql/flat_local_tx_factory.h>
#include <ydb/core/security/ticket_parser.h>
#include <ydb/core/testlib/fake_scheme_shard.h>
@@ -484,136 +479,6 @@ Y_UNIT_TEST(TestCheckACL) {
}
-void CheckLabeledCountersResponse(ui32 count, TTestContext& tc, TVector<TString> mustHave = {}) {
- IActor* actor = CreateClusterLabeledCountersAggregatorActor(tc.Edge, TTabletTypes::PersQueue);
- tc.Runtime->Register(actor);
-
- TAutoPtr<IEventHandle> handle;
- TEvTabletCounters::TEvTabletLabeledCountersResponse *result;
- result = tc.Runtime->GrabEdgeEvent<TEvTabletCounters::TEvTabletLabeledCountersResponse>(handle);
- UNIT_ASSERT(result);
- THashSet<TString> groups;
-
- Cerr << "Checking with " << count << " groups:\n";
-
- for (ui32 i = 0; i < result->Record.LabeledCountersByGroupSize(); ++i) {
- auto& c = result->Record.GetLabeledCountersByGroup(i);
- groups.insert(c.GetGroup());
- Cerr << "Has " << c.GetGroup() << "\n";
- }
- UNIT_ASSERT_VALUES_EQUAL(groups.size(), count);
- for (auto& g : mustHave) {
- UNIT_ASSERT(groups.contains(g));
- }
-}
-
-Y_UNIT_TEST(TestSwitchOffImportantFlag) {
- TTestContext tc;
- RunTestWithReboots(tc.TabletIds, [&]() {
- return tc.InitialEventsFilter.Prepare();
- }, [&](const TString& dispatchName, std::function<void(TTestActorRuntime&)> setup, bool& activeZone) {
- TFinalizer finalizer(tc);
- tc.Prepare(dispatchName, setup, activeZone);
- activeZone = false;
- tc.Runtime->SetScheduledLimit(600);
- PQTabletPrepare({}, {}, tc);
-
- {
- TDispatchOptions options;
- options.FinalEvents.emplace_back(TEvTabletCounters::EvTabletAddLabeledCounters);
- tc.Runtime->DispatchEvents(options);
- }
-
- CheckLabeledCountersResponse(8, tc); //only topic counters
-
- PQTabletPrepare({}, {{"user", true}}, tc);
-
- {
- TDispatchOptions options;
- options.FinalEvents.emplace_back(TEvTabletCounters::EvTabletAddLabeledCounters);
- tc.Runtime->DispatchEvents(options);
- }
-
-
- CheckLabeledCountersResponse(8, tc, {NKikimr::JoinPath({"user/1", TOPIC_NAME})}); //topic counters + important
-
- PQTabletPrepare({}, {}, tc);
-
- {
- TDispatchOptions options;
- options.FinalEvents.emplace_back(TEvTabletCounters::EvTabletAddLabeledCounters);
- tc.Runtime->DispatchEvents(options);
- }
-
- {
- TDispatchOptions options;
- options.FinalEvents.emplace_back(TEvTabletCounters::EvTabletAddLabeledCounters);
- tc.Runtime->DispatchEvents(options);
- }
-
- auto MakeTopics = [&] (const TVector<TString>& users) {
- TVector<TString> res;
- for (const auto& u : users) {
- res.emplace_back(NKikimr::JoinPath({u, TOPIC_NAME}));
- }
- return res;
- };
- CheckLabeledCountersResponse(8, tc, MakeTopics({"user/0"})); //topic counters + not important
-
- PQTabletPrepare({}, {{"user", true}, {"user2", true}}, tc);
-
- {
- TDispatchOptions options;
- options.FinalEvents.emplace_back(TEvTabletCounters::EvTabletAddLabeledCounters);
- tc.Runtime->DispatchEvents(options);
- }
-
- {
- TDispatchOptions options;
- options.FinalEvents.emplace_back(TEvTabletCounters::EvTabletAddLabeledCounters);
- tc.Runtime->DispatchEvents(options);
- }
-
- CheckLabeledCountersResponse(11, tc, MakeTopics({"user/1", "user2/1"})); //topic counters + not important
-
- PQTabletPrepare({}, {{"user", true}, {"user2", false}}, tc);
-
- {
- TDispatchOptions options;
- options.FinalEvents.emplace_back(TEvTabletCounters::EvTabletAddLabeledCounters);
- tc.Runtime->DispatchEvents(options);
- }
-
- {
- TDispatchOptions options;
- options.FinalEvents.emplace_back(TEvTabletCounters::EvTabletAddLabeledCounters);
- tc.Runtime->DispatchEvents(options);
- }
-
- CheckLabeledCountersResponse(12, tc, MakeTopics({"user/1", "user2/0"}));
-
-
- PQTabletPrepare({}, {{"user", true}}, tc);
-
- {
- TDispatchOptions options;
- options.FinalEvents.emplace_back(TEvTabletCounters::EvTabletAddLabeledCounters);
- tc.Runtime->DispatchEvents(options);
- }
-
- {
- TDispatchOptions options;
- options.FinalEvents.emplace_back(TEvTabletCounters::EvTabletAddLabeledCounters);
- tc.Runtime->DispatchEvents(options);
- }
-
- CheckLabeledCountersResponse(8, tc, MakeTopics({"user/1"}));
-
-
- });
-}
-
-
Y_UNIT_TEST(TestSeveralOwners) {
TTestContext tc;
RunTestWithReboots(tc.TabletIds, [&]() {
@@ -624,7 +489,8 @@ Y_UNIT_TEST(TestSeveralOwners) {
activeZone = false;
tc.Runtime->SetScheduledLimit(200);
- PQTabletPrepare({}, {}, tc); //no important clients, lifetimeseconds=0 - delete all right now, except last datablob
+ // No important clients, lifetimeseconds=0 - delete all right now, except last datablob
+ PQTabletPrepare({}, {}, tc);
TVector<std::pair<ui64, TString>> data;
@@ -656,7 +522,8 @@ Y_UNIT_TEST(TestWaitInOwners) {
activeZone = false;
tc.Runtime->SetScheduledLimit(200);
- PQTabletPrepare({}, {}, tc); //no important clients, lifetimeseconds=0 - delete all right now, except last datablob
+ // No important clients, lifetimeseconds=0 - delete all right now, except last datablob
+ PQTabletPrepare({}, {}, tc);
TVector<std::pair<ui64, TString>> data;
diff --git a/ydb/core/persqueue/pq_ut.h b/ydb/core/persqueue/pq_ut.h
index 753613f528a..13e671885f4 100644
--- a/ydb/core/persqueue/pq_ut.h
+++ b/ydb/core/persqueue/pq_ut.h
@@ -29,6 +29,7 @@ void SetupLogging(TTestActorRuntime& runtime) {
NActors::NLog::EPriority otherPriority = NLog::PRI_INFO;
runtime.SetLogPriority(NKikimrServices::PERSQUEUE, pqPriority);
+ runtime.SetLogPriority(NKikimrServices::SYSTEM_VIEWS, pqPriority);
runtime.SetLogPriority(NKikimrServices::KEYVALUE, priority);
runtime.SetLogPriority(NKikimrServices::BOOTSTRAPPER, priority);
runtime.SetLogPriority(NKikimrServices::TABLET_MAIN, priority);
@@ -115,10 +116,14 @@ struct TTestContext {
return RequestTimeoutFilter(runtime, event, duration, deadline);
}
- void Prepare(const TString& dispatchName, std::function<void(TTestActorRuntime&)> setup, bool& outActiveZone) {
+ void Prepare(const TString& dispatchName, std::function<void(TTestActorRuntime&)> setup, bool& outActiveZone, bool isFirstClass = false, bool enableMonitoring = false) {
Y_UNUSED(dispatchName);
outActiveZone = false;
- Runtime.Reset(new TTestBasicRuntime);
+ TTestBasicRuntime* runtime = new TTestBasicRuntime;
+ if (enableMonitoring) {
+ runtime->SetupMonitoring();
+ }
+ Runtime.Reset(runtime);
Runtime->SetScheduledLimit(200);
SetupLogging(*Runtime);
@@ -128,18 +133,17 @@ struct TTestContext {
CreateTestTabletInfo(TabletId, TabletType, TErasureType::ErasureNone),
&CreatePersQueue);
- TDispatchOptions options;
- options.FinalEvents.push_back(TDispatchOptions::TFinalEventCondition(TEvTablet::EvBoot));
Runtime->GetAppData(0).PQConfig.SetEnabled(true);
// NOTE(shmel1k@): KIKIMR-14221
- Runtime->GetAppData(0).PQConfig.SetTopicsAreFirstClassCitizen(false);
+ Runtime->GetAppData(0).PQConfig.SetTopicsAreFirstClassCitizen(isFirstClass);
Runtime->GetAppData(0).PQConfig.SetRequireCredentialsInNewProtocol(false);
Runtime->GetAppData(0).PQConfig.SetClusterTablePath("/Root/PQ/Config/V2/Cluster");
Runtime->GetAppData(0).PQConfig.SetVersionTablePath("/Root/PQ/Config/V2/Versions");
- Runtime->GetAppData(0).PQConfig.SetTopicsAreFirstClassCitizen(false);
Runtime->GetAppData(0).PQConfig.SetRoot("/Root/PQ");
Runtime->GetAppData(0).PQConfig.MutableQuotingConfig()->SetEnableQuoting(false);
+ TDispatchOptions options;
+ options.FinalEvents.push_back(TDispatchOptions::TFinalEventCondition(TEvTablet::EvBoot));
Runtime->DispatchEvents(options);
CreateTestBootstrapper(*Runtime,
@@ -165,6 +169,8 @@ struct TTestContext {
CreateTestTabletInfo(TabletId, TabletType, TErasureType::ErasureNone),
&CreatePersQueue);
+ Runtime->GetAppData(0).PQConfig.SetEnabled(true);
+
TDispatchOptions options;
options.FinalEvents.push_back(TDispatchOptions::TFinalEventCondition(TEvTablet::EvBoot));
Runtime->DispatchEvents(options);
@@ -179,7 +185,6 @@ struct TTestContext {
Edge = Runtime->AllocateEdgeActor();
Runtime->SetScheduledEventFilter(&RequestTimeoutFilter);
- Runtime->GetAppData(0).PQConfig.SetEnabled(true);
}
@@ -215,6 +220,10 @@ struct TTabletPreparationParameters {
ui64 sidMaxCount{0};
ui32 specVersion{0};
i32 storageLimitBytes{0};
+ TString folderId{"somefolder"};
+ TString cloudId{"somecloud"};
+ TString databaseId{"root"};
+ TString account{"federationAccount"};
};
void PQTabletPrepare(
const TTabletPreparationParameters& parameters,
diff --git a/ydb/core/persqueue/pq_ut_impl.cpp b/ydb/core/persqueue/pq_ut_impl.cpp
index b7c39bed47d..8153d9bc598 100644
--- a/ydb/core/persqueue/pq_ut_impl.cpp
+++ b/ydb/core/persqueue/pq_ut_impl.cpp
@@ -40,9 +40,13 @@ void PQTabletPrepare(const TTabletPreparationParameters& parameters,
if (tc.Runtime->GetAppData().PQConfig.GetTopicsAreFirstClassCitizen()) {
tabletConfig->SetTopicName("topic");
tabletConfig->SetTopicPath(tc.Runtime->GetAppData().PQConfig.GetDatabase() + "/topic");
+ tabletConfig->SetYcCloudId(parameters.cloudId);
+ tabletConfig->SetYcFolderId(parameters.folderId);
+ tabletConfig->SetYdbDatabaseId(parameters.databaseId);
+ tabletConfig->SetFederationAccount(parameters.account);
} else {
- tabletConfig->SetTopicName("rt3.dc1--topic");
- tabletConfig->SetTopicPath("/Root/PQ/rt3.dc1--topic");
+ tabletConfig->SetTopicName("rt3.dc1--asdfgs--topic");
+ tabletConfig->SetTopicPath("/Root/PQ/rt3.dc1--asdfgs--topic");
}
tabletConfig->SetTopic("topic");
tabletConfig->SetVersion(version);
diff --git a/ydb/core/persqueue/ut/CMakeLists.darwin.txt b/ydb/core/persqueue/ut/CMakeLists.darwin.txt
index 2ab8a0cbeb0..fe97d95f459 100644
--- a/ydb/core/persqueue/ut/CMakeLists.darwin.txt
+++ b/ydb/core/persqueue/ut/CMakeLists.darwin.txt
@@ -25,6 +25,7 @@ target_link_libraries(ydb-core-persqueue-ut PUBLIC
library-cpp-svnversion
ydb-core-testlib
ydb_persqueue_core-ut-ut_utils
+ library-cpp-resource
)
target_link_options(ydb-core-persqueue-ut PRIVATE
-Wl,-no_deduplicate
@@ -35,6 +36,7 @@ target_link_options(ydb-core-persqueue-ut PRIVATE
CoreFoundation
)
target_sources(ydb-core-persqueue-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/core/persqueue/counters_ut.cpp
${CMAKE_SOURCE_DIR}/ydb/core/persqueue/internals_ut.cpp
${CMAKE_SOURCE_DIR}/ydb/core/persqueue/metering_sink_ut.cpp
${CMAKE_SOURCE_DIR}/ydb/core/persqueue/mirrorer_ut.cpp
@@ -43,6 +45,7 @@ target_sources(ydb-core-persqueue-ut PRIVATE
${CMAKE_SOURCE_DIR}/ydb/core/persqueue/sourceid_ut.cpp
${CMAKE_SOURCE_DIR}/ydb/core/persqueue/type_codecs_ut.cpp
${CMAKE_SOURCE_DIR}/ydb/core/persqueue/user_info_ut.cpp
+ ${CMAKE_BINARY_DIR}/ydb/core/persqueue/ut/58e1cbb1dd06a8375ced22e9e6f7699d.cpp
)
add_test(
NAME
@@ -55,4 +58,17 @@ add_test(
--print-times
--show-fails
)
+resources(ydb-core-persqueue-ut
+ ${CMAKE_BINARY_DIR}/ydb/core/persqueue/ut/58e1cbb1dd06a8375ced22e9e6f7699d.cpp
+ INPUTS
+ ${CMAKE_SOURCE_DIR}/ydb/core/persqueue/ut/counters_datastreams.html
+ ${CMAKE_SOURCE_DIR}/ydb/core/persqueue/ut/counters_pqproxy_firstclass.html
+ ${CMAKE_SOURCE_DIR}/ydb/core/persqueue/ut/counters_pqproxy.html
+ ${CMAKE_SOURCE_DIR}/ydb/core/persqueue/ut/counters_labeled.html
+ KEYS
+ counters_datastreams.html
+ counters_pqproxy_firstclass.html
+ counters_pqproxy.html
+ counters_labeled.html
+)
vcs_info(ydb-core-persqueue-ut)
diff --git a/ydb/core/persqueue/ut/CMakeLists.linux.txt b/ydb/core/persqueue/ut/CMakeLists.linux.txt
index bd73cffc77b..e0e57717d3d 100644
--- a/ydb/core/persqueue/ut/CMakeLists.linux.txt
+++ b/ydb/core/persqueue/ut/CMakeLists.linux.txt
@@ -27,6 +27,7 @@ target_link_libraries(ydb-core-persqueue-ut PUBLIC
library-cpp-svnversion
ydb-core-testlib
ydb_persqueue_core-ut-ut_utils
+ library-cpp-resource
)
target_link_options(ydb-core-persqueue-ut PRIVATE
-ldl
@@ -39,6 +40,7 @@ target_link_options(ydb-core-persqueue-ut PRIVATE
-ldl
)
target_sources(ydb-core-persqueue-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/core/persqueue/counters_ut.cpp
${CMAKE_SOURCE_DIR}/ydb/core/persqueue/internals_ut.cpp
${CMAKE_SOURCE_DIR}/ydb/core/persqueue/metering_sink_ut.cpp
${CMAKE_SOURCE_DIR}/ydb/core/persqueue/mirrorer_ut.cpp
@@ -47,6 +49,7 @@ target_sources(ydb-core-persqueue-ut PRIVATE
${CMAKE_SOURCE_DIR}/ydb/core/persqueue/sourceid_ut.cpp
${CMAKE_SOURCE_DIR}/ydb/core/persqueue/type_codecs_ut.cpp
${CMAKE_SOURCE_DIR}/ydb/core/persqueue/user_info_ut.cpp
+ ${CMAKE_BINARY_DIR}/ydb/core/persqueue/ut/58e1cbb1dd06a8375ced22e9e6f7699d.cpp
)
add_test(
NAME
@@ -59,4 +62,17 @@ add_test(
--print-times
--show-fails
)
+resources(ydb-core-persqueue-ut
+ ${CMAKE_BINARY_DIR}/ydb/core/persqueue/ut/58e1cbb1dd06a8375ced22e9e6f7699d.cpp
+ INPUTS
+ ${CMAKE_SOURCE_DIR}/ydb/core/persqueue/ut/counters_datastreams.html
+ ${CMAKE_SOURCE_DIR}/ydb/core/persqueue/ut/counters_pqproxy_firstclass.html
+ ${CMAKE_SOURCE_DIR}/ydb/core/persqueue/ut/counters_pqproxy.html
+ ${CMAKE_SOURCE_DIR}/ydb/core/persqueue/ut/counters_labeled.html
+ KEYS
+ counters_datastreams.html
+ counters_pqproxy_firstclass.html
+ counters_pqproxy.html
+ counters_labeled.html
+)
vcs_info(ydb-core-persqueue-ut)
diff --git a/ydb/core/persqueue/ut/counters_datastreams.html b/ydb/core/persqueue/ut/counters_datastreams.html
new file mode 100644
index 00000000000..f01415c7556
--- /dev/null
+++ b/ydb/core/persqueue/ut/counters_datastreams.html
@@ -0,0 +1,94 @@
+<pre>
+cloud=somecloud:
+
+ folder=somefolder:
+
+ database=root:
+
+ stream=topic:
+ name=stream.incoming_bytes_per_second: 2700
+ name=stream.incoming_records_per_second: 150
+ name=stream.internal_write.bytes_per_second: 2700
+ name=stream.internal_write.compacted_bytes_per_second: 3720
+ name=stream.internal_write.records_per_second: 150
+ name=stream.internal_write.uncompressed_bytes_per_second: 1350
+
+ consumer=user:
+ name=stream.internal_read.bytes_per_second: 0
+ name=stream.internal_read.records_per_second: 0
+ name=stream.outgoing_bytes_per_second: 0
+ name=stream.outgoing_records_per_second: 0
+
+ host=1:
+
+ shard=0:
+ name=stream.await_operating_milliseconds: 0
+
+ shard=1:
+ name=stream.await_operating_milliseconds: 0
+
+ name=stream.internal_read.time_lags_milliseconds:
+ bin=100: 0
+ bin=1000: 0
+ bin=10000: 0
+ bin=180000: 0
+ bin=200: 0
+ bin=2000: 0
+ bin=30000: 0
+ bin=500: 0
+ bin=5000: 0
+ bin=60000: 0
+ bin=999999: 0
+
+ host=1:
+
+ shard=0:
+ name=stream.internal_write.buffer_brimmed_duration_ms: 0
+
+ shard=1:
+ name=stream.internal_write.buffer_brimmed_duration_ms: 0
+
+ name=stream.internal_write.partition_write_quota_wait_milliseconds:
+ bin=0: 150
+ bin=1: 0
+ bin=10: 0
+ bin=100: 0
+ bin=1000: 0
+ bin=10000: 0
+ bin=20: 0
+ bin=2500: 0
+ bin=5: 0
+ bin=50: 0
+ bin=500: 0
+ bin=5000: 0
+ bin=999999: 0
+
+ name=stream.internal_write.record_size_bytes:
+ bin=1024: 150
+ bin=10240: 0
+ bin=102400: 0
+ bin=1048576: 0
+ bin=10485760: 0
+ bin=20480: 0
+ bin=204800: 0
+ bin=2097152: 0
+ bin=5120: 0
+ bin=51200: 0
+ bin=524288: 0
+ bin=5242880: 0
+ bin=67108864: 0
+ bin=99999999: 0
+
+ name=stream.internal_write.time_lags_milliseconds:
+ bin=100: 150
+ bin=1000: 0
+ bin=10000: 0
+ bin=180000: 0
+ bin=200: 0
+ bin=2000: 0
+ bin=30000: 0
+ bin=500: 0
+ bin=5000: 0
+ bin=60000: 0
+ bin=999999: 0
+</pre>
diff --git a/ydb/core/persqueue/ut/counters_labeled.html b/ydb/core/persqueue/ut/counters_labeled.html
new file mode 100644
index 00000000000..86176ab9e27
--- /dev/null
+++ b/ydb/core/persqueue/ut/counters_labeled.html
@@ -0,0 +1,247 @@
+<pre>
+user_counters=PersQueue:
+
+ client=total:
+
+ important=0:
+
+ topic=rt3.dc1--topic:
+ sensor=PQ/MessageLagByCommitted: 30
+ sensor=PQ/MessageLagByLastRead: 29
+ sensor=PQ/PartitionMaxReadQuotaUsage: 0
+ sensor=PQ/ReadBytesAvailAvgMin: 1000000000
+ sensor=PQ/ReadBytesAvailAvgSec: 1000000000
+ sensor=PQ/ReadBytesMaxPerDay: 0
+ sensor=PQ/ReadBytesMaxPerHour: 0
+ sensor=PQ/ReadBytesMaxPerMin: 0
+ sensor=PQ/ReadBytesMaxPerSec: 0
+ sensor=PQ/ReadBytesPerDay: 0
+ sensor=PQ/ReadBytesPerHour: 0
+ sensor=PQ/ReadBytesPerMin: 0
+ sensor=PQ/ReadBytesPerSec: 0
+ sensor=PQ/ReadBytesQuota: 1000000000
+ sensor=PQ/ReadOffsetRewindSum: 0
+ sensor=PQ/ReadTimeLagMs: 0
+ sensor=PQ/SizeLagByCommitted: 744
+ sensor=PQ/SizeLagByLastRead: 744
+ sensor=PQ/TimeSinceLastReadMs: 4989
+ sensor=PQ/TotalMessageLagByLastRead: 29
+ sensor=PQ/TotalSizeLagByLastRead: 744
+ sensor=PQ/TotalTimeLagMsByLastRead: 4929
+ sensor=PQ/UserPartitionsAnswered: 2
+ sensor=PQ/WriteTimeLagMsByLastRead: 29
+ sensor=PQ/WriteTimeLagMsByLastReadOld: 4989
+
+ important=total:
+
+ topic=rt3.dc1--topic:
+ sensor=PQ/MessageLagByCommitted: 30
+ sensor=PQ/MessageLagByLastRead: 29
+ sensor=PQ/PartitionMaxReadQuotaUsage: 0
+ sensor=PQ/ReadBytesAvailAvgMin: 1000000000
+ sensor=PQ/ReadBytesAvailAvgSec: 1000000000
+ sensor=PQ/ReadBytesMaxPerDay: 0
+ sensor=PQ/ReadBytesMaxPerHour: 0
+ sensor=PQ/ReadBytesMaxPerMin: 0
+ sensor=PQ/ReadBytesMaxPerSec: 0
+ sensor=PQ/ReadBytesPerDay: 0
+ sensor=PQ/ReadBytesPerHour: 0
+ sensor=PQ/ReadBytesPerMin: 0
+ sensor=PQ/ReadBytesPerSec: 0
+ sensor=PQ/ReadBytesQuota: 1000000000
+ sensor=PQ/ReadOffsetRewindSum: 0
+ sensor=PQ/ReadTimeLagMs: 0
+ sensor=PQ/SizeLagByCommitted: 744
+ sensor=PQ/SizeLagByLastRead: 744
+ sensor=PQ/TimeSinceLastReadMs: 4989
+ sensor=PQ/TotalMessageLagByLastRead: 29
+ sensor=PQ/TotalSizeLagByLastRead: 744
+ sensor=PQ/TotalTimeLagMsByLastRead: 4929
+ sensor=PQ/UserPartitionsAnswered: 2
+ sensor=PQ/WriteTimeLagMsByLastRead: 29
+ sensor=PQ/WriteTimeLagMsByLastReadOld: 4989
+
+ topic=total:
+ sensor=PQ/MessageLagByCommitted: 30
+ sensor=PQ/MessageLagByLastRead: 29
+ sensor=PQ/PartitionMaxReadQuotaUsage: 0
+ sensor=PQ/ReadBytesAvailAvgMin: 1000000000
+ sensor=PQ/ReadBytesAvailAvgSec: 1000000000
+ sensor=PQ/ReadBytesMaxPerDay: 0
+ sensor=PQ/ReadBytesMaxPerHour: 0
+ sensor=PQ/ReadBytesMaxPerMin: 0
+ sensor=PQ/ReadBytesMaxPerSec: 0
+ sensor=PQ/ReadBytesPerDay: 0
+ sensor=PQ/ReadBytesPerHour: 0
+ sensor=PQ/ReadBytesPerMin: 0
+ sensor=PQ/ReadBytesPerSec: 0
+ sensor=PQ/ReadBytesQuota: 1000000000
+ sensor=PQ/ReadOffsetRewindSum: 0
+ sensor=PQ/ReadTimeLagMs: 0
+ sensor=PQ/SizeLagByCommitted: 744
+ sensor=PQ/SizeLagByLastRead: 744
+ sensor=PQ/TimeSinceLastReadMs: 4989
+ sensor=PQ/TotalMessageLagByLastRead: 29
+ sensor=PQ/TotalSizeLagByLastRead: 744
+ sensor=PQ/TotalTimeLagMsByLastRead: 4929
+ sensor=PQ/UserPartitionsAnswered: 2
+ sensor=PQ/WriteTimeLagMsByLastRead: 29
+ sensor=PQ/WriteTimeLagMsByLastReadOld: 4989
+
+ client=user:
+
+ important=0:
+
+ topic=rt3.dc1--topic:
+ sensor=PQ/MessageLagByCommitted: 30
+ sensor=PQ/MessageLagByLastRead: 29
+ sensor=PQ/PartitionMaxReadQuotaUsage: 0
+ sensor=PQ/ReadBytesAvailAvgMin: 1000000000
+ sensor=PQ/ReadBytesAvailAvgSec: 1000000000
+ sensor=PQ/ReadBytesMaxPerDay: 0
+ sensor=PQ/ReadBytesMaxPerHour: 0
+ sensor=PQ/ReadBytesMaxPerMin: 0
+ sensor=PQ/ReadBytesMaxPerSec: 0
+ sensor=PQ/ReadBytesPerDay: 0
+ sensor=PQ/ReadBytesPerHour: 0
+ sensor=PQ/ReadBytesPerMin: 0
+ sensor=PQ/ReadBytesPerSec: 0
+ sensor=PQ/ReadBytesQuota: 1000000000
+ sensor=PQ/ReadOffsetRewindSum: 0
+ sensor=PQ/ReadTimeLagMs: 0
+ sensor=PQ/SizeLagByCommitted: 744
+ sensor=PQ/SizeLagByLastRead: 744
+ sensor=PQ/TimeSinceLastReadMs: 4989
+ sensor=PQ/TotalMessageLagByLastRead: 29
+ sensor=PQ/TotalSizeLagByLastRead: 744
+ sensor=PQ/TotalTimeLagMsByLastRead: 4929
+ sensor=PQ/UserPartitionsAnswered: 2
+ sensor=PQ/WriteTimeLagMsByLastRead: 29
+ sensor=PQ/WriteTimeLagMsByLastReadOld: 4989
+
+ topic=total:
+ sensor=PQ/MessageLagByCommitted: 30
+ sensor=PQ/MessageLagByLastRead: 29
+ sensor=PQ/PartitionMaxReadQuotaUsage: 0
+ sensor=PQ/ReadBytesAvailAvgMin: 1000000000
+ sensor=PQ/ReadBytesAvailAvgSec: 1000000000
+ sensor=PQ/ReadBytesMaxPerDay: 0
+ sensor=PQ/ReadBytesMaxPerHour: 0
+ sensor=PQ/ReadBytesMaxPerMin: 0
+ sensor=PQ/ReadBytesMaxPerSec: 0
+ sensor=PQ/ReadBytesPerDay: 0
+ sensor=PQ/ReadBytesPerHour: 0
+ sensor=PQ/ReadBytesPerMin: 0
+ sensor=PQ/ReadBytesPerSec: 0
+ sensor=PQ/ReadBytesQuota: 1000000000
+ sensor=PQ/ReadOffsetRewindSum: 0
+ sensor=PQ/ReadTimeLagMs: 0
+ sensor=PQ/SizeLagByCommitted: 744
+ sensor=PQ/SizeLagByLastRead: 744
+ sensor=PQ/TimeSinceLastReadMs: 4989
+ sensor=PQ/TotalMessageLagByLastRead: 29
+ sensor=PQ/TotalSizeLagByLastRead: 744
+ sensor=PQ/TotalTimeLagMsByLastRead: 4929
+ sensor=PQ/UserPartitionsAnswered: 2
+ sensor=PQ/WriteTimeLagMsByLastRead: 29
+ sensor=PQ/WriteTimeLagMsByLastReadOld: 4989
+
+ important=total:
+
+ topic=total:
+ sensor=PQ/MessageLagByCommitted: 30
+ sensor=PQ/MessageLagByLastRead: 29
+ sensor=PQ/PartitionMaxReadQuotaUsage: 0
+ sensor=PQ/ReadBytesAvailAvgMin: 1000000000
+ sensor=PQ/ReadBytesAvailAvgSec: 1000000000
+ sensor=PQ/ReadBytesMaxPerDay: 0
+ sensor=PQ/ReadBytesMaxPerHour: 0
+ sensor=PQ/ReadBytesMaxPerMin: 0
+ sensor=PQ/ReadBytesMaxPerSec: 0
+ sensor=PQ/ReadBytesPerDay: 0
+ sensor=PQ/ReadBytesPerHour: 0
+ sensor=PQ/ReadBytesPerMin: 0
+ sensor=PQ/ReadBytesPerSec: 0
+ sensor=PQ/ReadBytesQuota: 1000000000
+ sensor=PQ/ReadOffsetRewindSum: 0
+ sensor=PQ/ReadTimeLagMs: 0
+ sensor=PQ/SizeLagByCommitted: 744
+ sensor=PQ/SizeLagByLastRead: 744
+ sensor=PQ/TimeSinceLastReadMs: 4989
+ sensor=PQ/TotalMessageLagByLastRead: 29
+ sensor=PQ/TotalSizeLagByLastRead: 744
+ sensor=PQ/TotalTimeLagMsByLastRead: 4929
+ sensor=PQ/UserPartitionsAnswered: 2
+ sensor=PQ/WriteTimeLagMsByLastRead: 29
+ sensor=PQ/WriteTimeLagMsByLastReadOld: 4989
+
+ topic=rt3.dc1--topic:
+ sensor=PQ/GapsCount: 0
+ sensor=PQ/GapsMaxCount: 0
+ sensor=PQ/GapsMaxSize: 0
+ sensor=PQ/GapsSize: 0
+ sensor=PQ/MaxPartSize: 744
+ sensor=PQ/PartitionInitTimeMs: 0
+ sensor=PQ/PartitionLifeTimeMs: 4989
+ sensor=PQ/PartitionMaxWriteQuotaUsage: 0
+ sensor=PQ/PartitionsAnswered: 2
+ sensor=PQ/QuotaBytesMaxPerDay: 540
+ sensor=PQ/QuotaBytesMaxPerHour: 540
+ sensor=PQ/QuotaBytesMaxPerMin: 540
+ sensor=PQ/QuotaBytesMaxPerSec: 540
+ sensor=PQ/QuotaBytesPerDay: 540
+ sensor=PQ/QuotaBytesPerHour: 540
+ sensor=PQ/QuotaBytesPerMin: 540
+ sensor=PQ/QuotaBytesPerSec: 540
+ sensor=PQ/SourceIdCount: 3
+ sensor=PQ/SourceIdMaxCount: 3
+ sensor=PQ/SourceIdMinLifetimeMs: 0
+ sensor=PQ/TotalPartSize: 744
+ sensor=PQ/WriteBytesAvailAvgMin: 49999998
+ sensor=PQ/WriteBytesAvailAvgSec: 50000000
+ sensor=PQ/WriteBytesMaxPerDay: 540
+ sensor=PQ/WriteBytesMaxPerHour: 540
+ sensor=PQ/WriteBytesMaxPerMin: 540
+ sensor=PQ/WriteBytesMaxPerSec: 540
+ sensor=PQ/WriteBytesPerDay: 540
+ sensor=PQ/WriteBytesPerHour: 540
+ sensor=PQ/WriteBytesPerMin: 540
+ sensor=PQ/WriteBytesPerSec: 540
+ sensor=PQ/WriteBytesQuota: 50000000
+ sensor=PQ/WriteTimeLagMsByLastWrite: 32
+
+ topic=total:
+ sensor=PQ/GapsCount: 0
+ sensor=PQ/GapsMaxCount: 0
+ sensor=PQ/GapsMaxSize: 0
+ sensor=PQ/GapsSize: 0
+ sensor=PQ/MaxPartSize: 744
+ sensor=PQ/PartitionInitTimeMs: 0
+ sensor=PQ/PartitionLifeTimeMs: 4989
+ sensor=PQ/PartitionMaxWriteQuotaUsage: 0
+ sensor=PQ/PartitionsAnswered: 2
+ sensor=PQ/QuotaBytesMaxPerDay: 540
+ sensor=PQ/QuotaBytesMaxPerHour: 540
+ sensor=PQ/QuotaBytesMaxPerMin: 540
+ sensor=PQ/QuotaBytesMaxPerSec: 540
+ sensor=PQ/QuotaBytesPerDay: 540
+ sensor=PQ/QuotaBytesPerHour: 540
+ sensor=PQ/QuotaBytesPerMin: 540
+ sensor=PQ/QuotaBytesPerSec: 540
+ sensor=PQ/SourceIdCount: 3
+ sensor=PQ/SourceIdMaxCount: 3
+ sensor=PQ/SourceIdMinLifetimeMs: 0
+ sensor=PQ/TotalPartSize: 744
+ sensor=PQ/WriteBytesAvailAvgMin: 49999998
+ sensor=PQ/WriteBytesAvailAvgSec: 50000000
+ sensor=PQ/WriteBytesMaxPerDay: 540
+ sensor=PQ/WriteBytesMaxPerHour: 540
+ sensor=PQ/WriteBytesMaxPerMin: 540
+ sensor=PQ/WriteBytesMaxPerSec: 540
+ sensor=PQ/WriteBytesPerDay: 540
+ sensor=PQ/WriteBytesPerHour: 540
+ sensor=PQ/WriteBytesPerMin: 540
+ sensor=PQ/WriteBytesPerSec: 540
+ sensor=PQ/WriteBytesQuota: 50000000
+ sensor=PQ/WriteTimeLagMsByLastWrite: 32
+</pre>
diff --git a/ydb/core/persqueue/ut/counters_pqproxy.html b/ydb/core/persqueue/ut/counters_pqproxy.html
new file mode 100644
index 00000000000..2f390ee8edc
--- /dev/null
+++ b/ydb/core/persqueue/ut/counters_pqproxy.html
@@ -0,0 +1,654 @@
+<pre>
+subsystem=SLI:
+
+ Account=asdfgs:
+ sensor=WriteBigLatency: 0
+ sensor=WritesTotal: 3
+
+ Account=total:
+ sensor=WriteBigLatency: 0
+ sensor=WritesTotal: 3
+
+ sensor=Write:
+
+ Account=asdfgs:
+ Duration=10000ms: 0
+ Duration=1000ms: 0
+ Duration=100ms: 3
+ Duration=1500ms: 0
+ Duration=2000ms: 0
+ Duration=200ms: 0
+ Duration=30000ms: 0
+ Duration=5000ms: 0
+ Duration=500ms: 0
+ Duration=550ms: 0
+ Duration=99999999ms: 0
+
+ Account=total:
+ Duration=10000ms: 0
+ Duration=1000ms: 0
+ Duration=100ms: 3
+ Duration=1500ms: 0
+ Duration=2000ms: 0
+ Duration=200ms: 0
+ Duration=30000ms: 0
+ Duration=5000ms: 0
+ Duration=500ms: 0
+ Duration=550ms: 0
+ Duration=99999999ms: 0
+
+subsystem=partitionWriteQuotaWait:
+
+ Account=asdfgs:
+
+ Producer=asdfgs:
+
+ Topic=asdfgs--topic:
+
+ TopicPath=asdfgs/topic:
+
+ OriginDC=Dc1:
+
+ sensor=PartitionWriteQuotaWaitOriginal:
+ Interval=0ms: 30
+ Interval=10000ms: 0
+ Interval=1000ms: 0
+ Interval=100ms: 0
+ Interval=10ms: 0
+ Interval=1ms: 0
+ Interval=20ms: 0
+ Interval=2500ms: 0
+ Interval=5000ms: 0
+ Interval=500ms: 0
+ Interval=50ms: 0
+ Interval=5ms: 0
+ Interval=999999ms: 0
+
+ OriginDC=cluster:
+
+ sensor=PartitionWriteQuotaWaitOriginal:
+ Interval=0ms: 30
+ Interval=10000ms: 0
+ Interval=1000ms: 0
+ Interval=100ms: 0
+ Interval=10ms: 0
+ Interval=1ms: 0
+ Interval=20ms: 0
+ Interval=2500ms: 0
+ Interval=5000ms: 0
+ Interval=500ms: 0
+ Interval=50ms: 0
+ Interval=5ms: 0
+ Interval=999999ms: 0
+
+ Topic=total:
+
+ TopicPath=total:
+
+ OriginDC=cluster:
+
+ sensor=PartitionWriteQuotaWaitOriginal:
+ Interval=0ms: 30
+ Interval=10000ms: 0
+ Interval=1000ms: 0
+ Interval=100ms: 0
+ Interval=10ms: 0
+ Interval=1ms: 0
+ Interval=20ms: 0
+ Interval=2500ms: 0
+ Interval=5000ms: 0
+ Interval=500ms: 0
+ Interval=50ms: 0
+ Interval=5ms: 0
+ Interval=999999ms: 0
+
+ Producer=total:
+
+ Topic=total:
+
+ TopicPath=total:
+
+ OriginDC=cluster:
+
+ sensor=PartitionWriteQuotaWaitOriginal:
+ Interval=0ms: 30
+ Interval=10000ms: 0
+ Interval=1000ms: 0
+ Interval=100ms: 0
+ Interval=10ms: 0
+ Interval=1ms: 0
+ Interval=20ms: 0
+ Interval=2500ms: 0
+ Interval=5000ms: 0
+ Interval=500ms: 0
+ Interval=50ms: 0
+ Interval=5ms: 0
+ Interval=999999ms: 0
+
+ Account=total:
+
+ Producer=total:
+
+ Topic=total:
+
+ TopicPath=total:
+
+ OriginDC=cluster:
+
+ sensor=PartitionWriteQuotaWaitOriginal:
+ Interval=0ms: 30
+ Interval=10000ms: 0
+ Interval=1000ms: 0
+ Interval=100ms: 0
+ Interval=10ms: 0
+ Interval=1ms: 0
+ Interval=20ms: 0
+ Interval=2500ms: 0
+ Interval=5000ms: 0
+ Interval=500ms: 0
+ Interval=50ms: 0
+ Interval=5ms: 0
+ Interval=999999ms: 0
+
+subsystem=readSession:
+
+ Account=asdfgs:
+
+ Producer=asdfgs:
+
+ Topic=asdfgs--topic:
+
+ TopicPath=asdfgs/topic:
+
+ OriginDC=Dc1:
+
+ Client=user:
+
+ ConsumerPath=shared/user:
+ sensor=BytesRead: 0
+ sensor=MessagesRead: 0
+
+ OriginDC=cluster:
+
+ Client=user:
+
+ ConsumerPath=shared/user:
+ sensor=BytesRead: 0
+ sensor=MessagesRead: 0
+
+ Topic=total:
+
+ TopicPath=total:
+
+ OriginDC=cluster:
+
+ Client=user:
+
+ ConsumerPath=shared/user:
+ sensor=BytesRead: 0
+ sensor=MessagesRead: 0
+
+ Producer=total:
+
+ Topic=total:
+
+ TopicPath=total:
+
+ OriginDC=cluster:
+
+ Client=user:
+
+ ConsumerPath=shared/user:
+ sensor=BytesRead: 0
+ sensor=MessagesRead: 0
+
+ Account=total:
+
+ Producer=total:
+
+ Topic=total:
+
+ TopicPath=total:
+
+ OriginDC=cluster:
+
+ Client=user:
+
+ ConsumerPath=shared/user:
+ sensor=BytesRead: 0
+ sensor=MessagesRead: 0
+
+subsystem=readTimeLag:
+
+ Account=asdfgs:
+
+ Producer=asdfgs:
+
+ Topic=asdfgs--topic:
+
+ TopicPath=asdfgs/topic:
+
+ OriginDC=Dc1:
+
+ Client=user:
+
+ ConsumerPath=shared/user:
+
+ sensor=TimeLags:
+ Interval=10000ms: 0
+ Interval=1000ms: 0
+ Interval=100ms: 0
+ Interval=180000ms: 0
+ Interval=2000ms: 0
+ Interval=200ms: 0
+ Interval=30000ms: 0
+ Interval=5000ms: 0
+ Interval=500ms: 0
+ Interval=60000ms: 0
+ Interval=999999ms: 0
+
+ OriginDC=cluster:
+
+ Client=user:
+
+ ConsumerPath=shared/user:
+
+ sensor=TimeLags:
+ Interval=10000ms: 0
+ Interval=1000ms: 0
+ Interval=100ms: 0
+ Interval=180000ms: 0
+ Interval=2000ms: 0
+ Interval=200ms: 0
+ Interval=30000ms: 0
+ Interval=5000ms: 0
+ Interval=500ms: 0
+ Interval=60000ms: 0
+ Interval=999999ms: 0
+
+ Topic=total:
+
+ TopicPath=total:
+
+ OriginDC=cluster:
+
+ Client=user:
+
+ ConsumerPath=shared/user:
+
+ sensor=TimeLags:
+ Interval=10000ms: 0
+ Interval=1000ms: 0
+ Interval=100ms: 0
+ Interval=180000ms: 0
+ Interval=2000ms: 0
+ Interval=200ms: 0
+ Interval=30000ms: 0
+ Interval=5000ms: 0
+ Interval=500ms: 0
+ Interval=60000ms: 0
+ Interval=999999ms: 0
+
+ Producer=total:
+
+ Topic=total:
+
+ TopicPath=total:
+
+ OriginDC=cluster:
+
+ Client=user:
+
+ ConsumerPath=shared/user:
+
+ sensor=TimeLags:
+ Interval=10000ms: 0
+ Interval=1000ms: 0
+ Interval=100ms: 0
+ Interval=180000ms: 0
+ Interval=2000ms: 0
+ Interval=200ms: 0
+ Interval=30000ms: 0
+ Interval=5000ms: 0
+ Interval=500ms: 0
+ Interval=60000ms: 0
+ Interval=999999ms: 0
+
+ Account=total:
+
+ Producer=total:
+
+ Topic=total:
+
+ TopicPath=total:
+
+ OriginDC=cluster:
+
+ Client=user:
+
+ ConsumerPath=shared/user:
+
+ sensor=TimeLags:
+ Interval=10000ms: 0
+ Interval=1000ms: 0
+ Interval=100ms: 0
+ Interval=180000ms: 0
+ Interval=2000ms: 0
+ Interval=200ms: 0
+ Interval=30000ms: 0
+ Interval=5000ms: 0
+ Interval=500ms: 0
+ Interval=60000ms: 0
+ Interval=999999ms: 0
+
+subsystem=writeInfo:
+
+ Account=asdfgs:
+
+ Producer=asdfgs:
+
+ Topic=asdfgs--topic:
+
+ TopicPath=asdfgs/topic:
+
+ OriginDC=Dc1:
+
+ sensor=MessageSizeOriginal:
+ Size=100kb: 0
+ Size=10240kb: 0
+ Size=1024kb: 0
+ Size=10kb: 0
+ Size=1kb: 30
+ Size=200kb: 0
+ Size=2048kb: 0
+ Size=20kb: 0
+ Size=50kb: 0
+ Size=5120kb: 0
+ Size=512kb: 0
+ Size=5kb: 0
+ Size=65536kb: 0
+ Size=99999999kb: 0
+
+ OriginDC=cluster:
+
+ sensor=MessageSizeOriginal:
+ Size=100kb: 0
+ Size=10240kb: 0
+ Size=1024kb: 0
+ Size=10kb: 0
+ Size=1kb: 30
+ Size=200kb: 0
+ Size=2048kb: 0
+ Size=20kb: 0
+ Size=50kb: 0
+ Size=5120kb: 0
+ Size=512kb: 0
+ Size=5kb: 0
+ Size=65536kb: 0
+ Size=99999999kb: 0
+
+ Topic=total:
+
+ TopicPath=total:
+
+ OriginDC=cluster:
+
+ sensor=MessageSizeOriginal:
+ Size=100kb: 0
+ Size=10240kb: 0
+ Size=1024kb: 0
+ Size=10kb: 0
+ Size=1kb: 30
+ Size=200kb: 0
+ Size=2048kb: 0
+ Size=20kb: 0
+ Size=50kb: 0
+ Size=5120kb: 0
+ Size=512kb: 0
+ Size=5kb: 0
+ Size=65536kb: 0
+ Size=99999999kb: 0
+
+ Producer=total:
+
+ Topic=total:
+
+ TopicPath=total:
+
+ OriginDC=cluster:
+
+ sensor=MessageSizeOriginal:
+ Size=100kb: 0
+ Size=10240kb: 0
+ Size=1024kb: 0
+ Size=10kb: 0
+ Size=1kb: 30
+ Size=200kb: 0
+ Size=2048kb: 0
+ Size=20kb: 0
+ Size=50kb: 0
+ Size=5120kb: 0
+ Size=512kb: 0
+ Size=5kb: 0
+ Size=65536kb: 0
+ Size=99999999kb: 0
+
+ Account=total:
+
+ Producer=total:
+
+ Topic=total:
+
+ TopicPath=total:
+
+ OriginDC=cluster:
+
+ sensor=MessageSizeOriginal:
+ Size=100kb: 0
+ Size=10240kb: 0
+ Size=1024kb: 0
+ Size=10kb: 0
+ Size=1kb: 30
+ Size=200kb: 0
+ Size=2048kb: 0
+ Size=20kb: 0
+ Size=50kb: 0
+ Size=5120kb: 0
+ Size=512kb: 0
+ Size=5kb: 0
+ Size=65536kb: 0
+ Size=99999999kb: 0
+
+subsystem=writeSession:
+
+ Account=asdfgs:
+
+ Producer=asdfgs:
+
+ Topic=asdfgs--topic:
+
+ TopicPath=asdfgs/topic:
+
+ ClientDC=Unknown:
+ sensor=BytesWrittenFromDC: 1560
+
+ OriginDC=Dc1:
+ sensor=BytesWrittenOriginal: 540
+ sensor=CompactedBytesWrittenOriginal: 744
+ sensor=MessagesWrittenOriginal: 30
+ sensor=UncompressedBytesWrittenOriginal: 270
+
+ OriginDC=cluster:
+ sensor=BytesWrittenOriginal: 540
+ sensor=CompactedBytesWrittenOriginal: 744
+ sensor=MessagesWrittenOriginal: 30
+ sensor=UncompressedBytesWrittenOriginal: 270
+
+ Topic=total:
+
+ TopicPath=total:
+
+ ClientDC=Unknown:
+ sensor=BytesWrittenFromDC: 1560
+
+ OriginDC=cluster:
+ sensor=BytesWrittenOriginal: 540
+ sensor=CompactedBytesWrittenOriginal: 744
+ sensor=MessagesWrittenOriginal: 30
+ sensor=UncompressedBytesWrittenOriginal: 270
+
+ Producer=total:
+
+ Topic=total:
+
+ TopicPath=total:
+
+ ClientDC=Unknown:
+ sensor=BytesWrittenFromDC: 1560
+
+ OriginDC=cluster:
+ sensor=BytesWrittenOriginal: 540
+ sensor=CompactedBytesWrittenOriginal: 744
+ sensor=MessagesWrittenOriginal: 30
+ sensor=UncompressedBytesWrittenOriginal: 270
+
+ Account=total:
+
+ Producer=total:
+
+ Topic=total:
+
+ TopicPath=total:
+
+ ClientDC=Unknown:
+ sensor=BytesWrittenFromDC: 1560
+
+ OriginDC=cluster:
+ sensor=BytesWrittenOriginal: 540
+ sensor=CompactedBytesWrittenOriginal: 744
+ sensor=MessagesWrittenOriginal: 30
+ sensor=UncompressedBytesWrittenOriginal: 270
+
+subsystem=writeTimeLag:
+
+ Account=asdfgs:
+
+ Producer=asdfgs:
+
+ Topic=asdfgs--topic:
+
+ TopicPath=asdfgs/topic:
+
+ OriginDC=Dc1:
+
+ sensor=TimeLagsOriginal:
+ Interval=10000ms: 0
+ Interval=1000ms: 0
+ Interval=100ms: 30
+ Interval=180000ms: 0
+ Interval=2000ms: 0
+ Interval=200ms: 0
+ Interval=30000ms: 0
+ Interval=5000ms: 0
+ Interval=500ms: 0
+ Interval=60000ms: 0
+ Interval=999999ms: 0
+
+ OriginDC=cluster:
+
+ sensor=TimeLagsOriginal:
+ Interval=10000ms: 0
+ Interval=1000ms: 0
+ Interval=100ms: 30
+ Interval=180000ms: 0
+ Interval=2000ms: 0
+ Interval=200ms: 0
+ Interval=30000ms: 0
+ Interval=5000ms: 0
+ Interval=500ms: 0
+ Interval=60000ms: 0
+ Interval=999999ms: 0
+
+ Topic=total:
+
+ TopicPath=total:
+
+ OriginDC=cluster:
+
+ sensor=TimeLagsOriginal:
+ Interval=10000ms: 0
+ Interval=1000ms: 0
+ Interval=100ms: 30
+ Interval=180000ms: 0
+ Interval=2000ms: 0
+ Interval=200ms: 0
+ Interval=30000ms: 0
+ Interval=5000ms: 0
+ Interval=500ms: 0
+ Interval=60000ms: 0
+ Interval=999999ms: 0
+
+ Producer=total:
+
+ Topic=total:
+
+ TopicPath=total:
+
+ OriginDC=cluster:
+
+ sensor=TimeLagsOriginal:
+ Interval=10000ms: 0
+ Interval=1000ms: 0
+ Interval=100ms: 30
+ Interval=180000ms: 0
+ Interval=2000ms: 0
+ Interval=200ms: 0
+ Interval=30000ms: 0
+ Interval=5000ms: 0
+ Interval=500ms: 0
+ Interval=60000ms: 0
+ Interval=999999ms: 0
+
+ Account=total:
+
+ Producer=total:
+
+ Topic=total:
+
+ TopicPath=total:
+
+ OriginDC=cluster:
+
+ sensor=TimeLagsOriginal:
+ Interval=10000ms: 0
+ Interval=1000ms: 0
+ Interval=100ms: 30
+ Interval=180000ms: 0
+ Interval=2000ms: 0
+ Interval=200ms: 0
+ Interval=30000ms: 0
+ Interval=5000ms: 0
+ Interval=500ms: 0
+ Interval=60000ms: 0
+ Interval=999999ms: 0
+
+subsystem=writingTime:
+
+ OriginDC=Dc1:
+
+ Producer=asdfgs:
+
+ TopicPath=asdfgs/topic:
+
+ Account=asdfgs:
+
+ Topic=asdfgs--topic:
+
+ host=1:
+
+ Partition=0:
+ sensor=BufferFullTimeOriginal: 0
+
+ Partition=1:
+ sensor=BufferFullTimeOriginal: 0
+</pre>
diff --git a/ydb/core/persqueue/ut/counters_pqproxy_firstclass.html b/ydb/core/persqueue/ut/counters_pqproxy_firstclass.html
new file mode 100644
index 00000000000..6fbbfec01a6
--- /dev/null
+++ b/ydb/core/persqueue/ut/counters_pqproxy_firstclass.html
@@ -0,0 +1,39 @@
+<pre>
+subsystem=SLI:
+
+ Account=federationAccount:
+ name=WriteBigLatency: 0
+ name=WritesTotal: 3
+
+ Account=total:
+ name=WriteBigLatency: 0
+ name=WritesTotal: 3
+
+ sensor=Write:
+
+ Account=federationAccount:
+ Duration=10000ms: 0
+ Duration=1000ms: 0
+ Duration=100ms: 3
+ Duration=1500ms: 0
+ Duration=2000ms: 0
+ Duration=200ms: 0
+ Duration=30000ms: 0
+ Duration=5000ms: 0
+ Duration=500ms: 0
+ Duration=550ms: 0
+ Duration=99999999ms: 0
+
+ Account=total:
+ Duration=10000ms: 0
+ Duration=1000ms: 0
+ Duration=100ms: 3
+ Duration=1500ms: 0
+ Duration=2000ms: 0
+ Duration=200ms: 0
+ Duration=30000ms: 0
+ Duration=5000ms: 0
+ Duration=500ms: 0
+ Duration=550ms: 0
+ Duration=99999999ms: 0
+</pre>