diff options
author | mokhotskii <[email protected]> | 2022-08-08 11:25:34 +0300 |
---|---|---|
committer | mokhotskii <[email protected]> | 2022-08-08 11:25:34 +0300 |
commit | daf6368efef886e1b7894bd79df067b93130c781 (patch) | |
tree | 6be19e8d86da7ef2ed8c9afb2caf8ed7683117b9 | |
parent | f883bcc54069779989e8952de483dda899d7d6ab (diff) |
Add TPartition LabeledCounters tests
Add LabeledCounters unit tests
-rw-r--r-- | ydb/core/persqueue/counters_ut.cpp | 339 | ||||
-rw-r--r-- | ydb/core/persqueue/partition.cpp | 105 | ||||
-rw-r--r-- | ydb/core/persqueue/partition.h | 6 | ||||
-rw-r--r-- | ydb/core/persqueue/pq_impl.cpp | 13 | ||||
-rw-r--r-- | ydb/core/persqueue/pq_ut.cpp | 141 | ||||
-rw-r--r-- | ydb/core/persqueue/pq_ut.h | 23 | ||||
-rw-r--r-- | ydb/core/persqueue/pq_ut_impl.cpp | 8 | ||||
-rw-r--r-- | ydb/core/persqueue/ut/CMakeLists.darwin.txt | 16 | ||||
-rw-r--r-- | ydb/core/persqueue/ut/CMakeLists.linux.txt | 16 | ||||
-rw-r--r-- | ydb/core/persqueue/ut/counters_datastreams.html | 94 | ||||
-rw-r--r-- | ydb/core/persqueue/ut/counters_labeled.html | 247 | ||||
-rw-r--r-- | ydb/core/persqueue/ut/counters_pqproxy.html | 654 | ||||
-rw-r--r-- | ydb/core/persqueue/ut/counters_pqproxy_firstclass.html | 39 |
13 files changed, 1496 insertions, 205 deletions
diff --git a/ydb/core/persqueue/counters_ut.cpp b/ydb/core/persqueue/counters_ut.cpp new file mode 100644 index 00000000000..64e4d3ce315 --- /dev/null +++ b/ydb/core/persqueue/counters_ut.cpp @@ -0,0 +1,339 @@ +#include <library/cpp/testing/unittest/registar.h> +#include <ydb/core/mon/sync_http_mon.h> + +#include "pq_ut.h" + +namespace NKikimr { + +Y_UNIT_TEST_SUITE(PQCountersSimple) { + +Y_UNIT_TEST(Partition) { + TTestContext tc; + TFinalizer finalizer(tc); + bool activeZone{false}; + tc.Prepare("", [](TTestActorRuntime&) {}, activeZone, false, true); + tc.Runtime->SetScheduledLimit(100); + + TVector<std::pair<ui64, TString>> data; + TString s{32, 'c'}; + ui32 pp = 8 + 4 + 2 + 9; + for (ui32 i = 0; i < 10; ++i) { + data.push_back({i + 1, s.substr(pp)}); + } + PQTabletPrepare({}, {}, tc); + CmdWrite(0, "sourceid0", data, tc, false, {}, true); + CmdWrite(0, "sourceid1", data, tc, false); + CmdWrite(0, "sourceid2", data, tc, false); + + { + auto counters = tc.Runtime->GetAppData(0).Counters; + auto dbGroup = GetServiceCounters(counters, "pqproxy"); + TStringStream countersStr; + dbGroup->OutputHtml(countersStr); + Cerr << "ASDFGS: " << countersStr.Str() << Endl; + TString referenceCounters = NResource::Find(TStringBuf("counters_pqproxy.html")); + UNIT_ASSERT_EQUAL(countersStr.Str() + "\n", referenceCounters); + } + + { + auto counters = tc.Runtime->GetAppData(0).Counters; + auto dbGroup = GetServiceCounters(counters, "datastreams"); + TStringStream countersStr; + dbGroup->OutputHtml(countersStr); + UNIT_ASSERT_EQUAL(countersStr.Str(), "<pre></pre>"); + } +} + +Y_UNIT_TEST(PartitionFirstClass) { + TTestContext tc; + TFinalizer finalizer(tc); + bool activeZone{false}; + tc.Prepare("", [](TTestActorRuntime&){}, activeZone, true, true); + tc.Runtime->SetScheduledLimit(100); + + TVector<std::pair<ui64, TString>> data; + TString s{32, 'c'}; + ui32 pp = 8 + 4 + 2 + 9; + for (ui32 i = 0; i < 10; ++i) { + data.push_back({i + 1, s.substr(pp)}); + } + PQTabletPrepare({}, {}, tc); + CmdWrite(0, "sourceid0", data, tc, false, {}, true); + CmdWrite(0, "sourceid1", data, tc, false); + CmdWrite(0, "sourceid2", data, tc, false); + + { + auto counters = tc.Runtime->GetAppData(0).Counters; + auto dbGroup = GetServiceCounters(counters, "pqproxy"); + TStringStream countersStr; + dbGroup->OutputHtml(countersStr); + TString referenceCounters = NResource::Find(TStringBuf("counters_pqproxy_firstclass.html")); + UNIT_ASSERT_EQUAL(countersStr.Str() + "\n", referenceCounters); + } + + { + auto counters = tc.Runtime->GetAppData(0).Counters; + auto dbGroup = GetServiceCounters(counters, "datastreams"); + TStringStream countersStr; + dbGroup->OutputHtml(countersStr); + const TString referenceCounters = NResource::Find(TStringBuf("counters_datastreams.html")); + UNIT_ASSERT_EQUAL(countersStr.Str() + "\n", referenceCounters); + } +} + +} // Y_UNIT_TEST_SUITE(PQCountersSimple) + +Y_UNIT_TEST_SUITE(PQCountersLabeled) { + +struct THttpRequest : NMonitoring::IHttpRequest { + HTTP_METHOD Method; + TCgiParameters CgiParameters; + THttpHeaders HttpHeaders; + + THttpRequest(HTTP_METHOD method) + : Method(method) + { + CgiParameters.emplace("type", TTabletTypes::TypeToStr(TTabletTypes::PersQueue)); + } + + ~THttpRequest() {} + + const char* GetURI() const override { + return ""; + } + + const char* GetPath() const override { + return ""; + } + + const TCgiParameters& GetParams() const override { + return CgiParameters; + } + + const TCgiParameters& GetPostParams() const override { + return CgiParameters; + } + + TStringBuf GetPostContent() const override { + return TString(); + } + + HTTP_METHOD GetMethod() const override { + return Method; + } + + const THttpHeaders& GetHeaders() const override { + return HttpHeaders; + } + + TString GetRemoteAddr() const override { + return TString(); + } +}; + +Y_UNIT_TEST(Partition) { + TTestContext tc; + RunTestWithReboots({}, [&]() { + return tc.InitialEventsFilter.Prepare(); + }, [&](const TString& dispatchName, std::function<void(TTestActorRuntime&)> setup, bool& activeZone) { + TFinalizer finalizer(tc); + tc.Prepare(dispatchName, setup, activeZone, false, true); + tc.Runtime->SetScheduledLimit(100); + + TVector<std::pair<ui64, TString>> data; + TString s{32, 'c'}; + ui32 pp = 8 + 4 + 2 + 9; + for (ui32 i = 0; i < 10; ++i) { + data.push_back({i + 1, s.substr(pp)}); + } + PQTabletPrepare({}, {}, tc); + + IActor* actor = CreateTabletCountersAggregator(false); + auto aggregatorId = tc.Runtime->Register(actor); + tc.Runtime->EnableScheduleForActor(aggregatorId); + + CmdWrite(0, "sourceid0", data, tc, false, {}, true); + CmdWrite(0, "sourceid1", data, tc, false); + CmdWrite(0, "sourceid2", data, tc, false); + PQGetPartInfo(0, 30, tc); + + { + TDispatchOptions options; + options.FinalEvents.emplace_back(TEvTabletCounters::EvTabletAddLabeledCounters); + tc.Runtime->DispatchEvents(options); + } + + IActor* actorX = CreateClusterLabeledCountersAggregatorActor(tc.Edge, TTabletTypes::PersQueue); + tc.Runtime->Register(actorX); + + TAutoPtr<IEventHandle> handle; + TEvTabletCounters::TEvTabletLabeledCountersResponse *result; + result = tc.Runtime->GrabEdgeEvent<TEvTabletCounters::TEvTabletLabeledCountersResponse>(handle); + UNIT_ASSERT(result); + + THttpRequest httpReq(HTTP_METHOD_GET); + NMonitoring::TMonService2HttpRequest monReq(nullptr, &httpReq, nullptr, nullptr, "", nullptr); + tc.Runtime->Send(new IEventHandle(aggregatorId, tc.Edge, new NMon::TEvHttpInfo(monReq))); + + TAutoPtr<IEventHandle> handle1; + auto resp = tc.Runtime->GrabEdgeEvent<NMon::TEvHttpInfoRes>(handle1); + const TString countersStr = ((NMon::TEvHttpInfoRes*) resp)->Answer; + + const TString referenceCounters = NResource::Find(TStringBuf("counters_datastreams.html")); + UNIT_ASSERT_VALUES_EQUAL(countersStr, referenceCounters); + }, 1); +} + +Y_UNIT_TEST(PartitionFirstClass) { + TTestContext tc; + RunTestWithReboots({}, [&]() { + return tc.InitialEventsFilter.Prepare(); + }, [&](const TString& dispatchName, std::function<void(TTestActorRuntime&)> setup, bool& activeZone) { + TFinalizer finalizer(tc); + activeZone = false; + + tc.Prepare(dispatchName, setup, activeZone, true, true); + tc.Runtime->SetScheduledLimit(100); + + TVector<std::pair<ui64, TString>> data; + TString s{32, 'c'}; + ui32 pp = 8 + 4 + 2 + 9; + for (ui32 i = 0; i < 10; ++i) { + data.push_back({i + 1, s.substr(pp)}); + } + PQTabletPrepare({}, {}, tc); + + IActor* actor = CreateTabletCountersAggregator(false); + auto aggregatorId = tc.Runtime->Register(actor); + tc.Runtime->EnableScheduleForActor(aggregatorId); + + CmdWrite(0, "sourceid0", data, tc, false, {}, true); + CmdWrite(0, "sourceid1", data, tc, false); + CmdWrite(0, "sourceid2", data, tc, false); + PQGetPartInfo(0, 30, tc); + + { + TDispatchOptions options; + options.FinalEvents.emplace_back(TEvTabletCounters::EvTabletAddLabeledCounters); + auto processedCountersEvent = tc.Runtime->DispatchEvents(options, TDuration::MilliSeconds(500)); + UNIT_ASSERT_VALUES_EQUAL(processedCountersEvent, false); + } + }); +} + +void CheckLabeledCountersResponse(TTestContext& tc, ui32 count, TVector<TString> mustHave = {}) { + IActor* actor = CreateClusterLabeledCountersAggregatorActor(tc.Edge, TTabletTypes::PersQueue); + tc.Runtime->Register(actor); + + TAutoPtr<IEventHandle> handle; + TEvTabletCounters::TEvTabletLabeledCountersResponse *result; + result = tc.Runtime->GrabEdgeEvent<TEvTabletCounters::TEvTabletLabeledCountersResponse>(handle); + UNIT_ASSERT(result); + + THashSet<TString> groups; + + for (ui32 i = 0; i < result->Record.LabeledCountersByGroupSize(); ++i) { + auto& c = result->Record.GetLabeledCountersByGroup(i); + groups.insert(c.GetGroup()); + } + UNIT_ASSERT_VALUES_EQUAL(groups.size(), count); + for (auto& g : mustHave) { + UNIT_ASSERT(groups.contains(g)); + } +} + +Y_UNIT_TEST(ImportantFlagSwitching) { + const static TString TOPIC_NAME = "rt3.dc1--asdfgs--topic"; + TTestContext tc; + RunTestWithReboots(tc.TabletIds, [&]() { + return tc.InitialEventsFilter.Prepare(); + }, [&](const TString& dispatchName, std::function<void(TTestActorRuntime&)> setup, bool& activeZone) { + TFinalizer finalizer(tc); + tc.Prepare(dispatchName, setup, activeZone); + activeZone = false; + tc.Runtime->SetScheduledLimit(600); + + auto MakeTopics = [&] (const TVector<TString>& users) { + TVector<TString> res; + for (const auto& u : users) { + res.emplace_back(NKikimr::JoinPath({u, TOPIC_NAME})); + } + return res; + }; + + PQTabletPrepare({}, {}, tc); + { + TDispatchOptions options; + options.FinalEvents.emplace_back(TEvTabletCounters::EvTabletAddLabeledCounters); + tc.Runtime->DispatchEvents(options); + } + // Topic counters only + CheckLabeledCountersResponse(tc, 8); + + // Topic counters + important + PQTabletPrepare({}, {{"user", true}}, tc); + { + TDispatchOptions options; + options.FinalEvents.emplace_back(TEvTabletCounters::EvTabletAddLabeledCounters); + tc.Runtime->DispatchEvents(options); + } + CheckLabeledCountersResponse(tc, 8, {NKikimr::JoinPath({"user/1", TOPIC_NAME})}); + + PQTabletPrepare({}, {}, tc); + { + TDispatchOptions options; + options.FinalEvents.emplace_back(TEvTabletCounters::EvTabletAddLabeledCounters); + tc.Runtime->DispatchEvents(options); + } + { + TDispatchOptions options; + options.FinalEvents.emplace_back(TEvTabletCounters::EvTabletAddLabeledCounters); + tc.Runtime->DispatchEvents(options); + } + // Topic counters + not important + CheckLabeledCountersResponse(tc, 8, MakeTopics({"user/0"})); + + // Topic counters + not important + PQTabletPrepare({}, {{"user", true}, {"user2", true}}, tc); + { + TDispatchOptions options; + options.FinalEvents.emplace_back(TEvTabletCounters::EvTabletAddLabeledCounters); + tc.Runtime->DispatchEvents(options); + } + { + TDispatchOptions options; + options.FinalEvents.emplace_back(TEvTabletCounters::EvTabletAddLabeledCounters); + tc.Runtime->DispatchEvents(options); + } + CheckLabeledCountersResponse(tc, 11, MakeTopics({"user/1", "user2/1"})); + + PQTabletPrepare({}, {{"user", true}, {"user2", false}}, tc); + { + TDispatchOptions options; + options.FinalEvents.emplace_back(TEvTabletCounters::EvTabletAddLabeledCounters); + tc.Runtime->DispatchEvents(options); + } + { + TDispatchOptions options; + options.FinalEvents.emplace_back(TEvTabletCounters::EvTabletAddLabeledCounters); + tc.Runtime->DispatchEvents(options); + } + CheckLabeledCountersResponse(tc, 12, MakeTopics({"user/1", "user2/0"})); + + PQTabletPrepare({}, {{"user", true}}, tc); + { + TDispatchOptions options; + options.FinalEvents.emplace_back(TEvTabletCounters::EvTabletAddLabeledCounters); + tc.Runtime->DispatchEvents(options); + } + { + TDispatchOptions options; + options.FinalEvents.emplace_back(TEvTabletCounters::EvTabletAddLabeledCounters); + tc.Runtime->DispatchEvents(options); + } + CheckLabeledCountersResponse(tc, 8, MakeTopics({"user/1"})); + }); +} +} // Y_UNIT_TEST_SUITE(PQCountersLabeled) + +} // namespace NKikimr diff --git a/ydb/core/persqueue/partition.cpp b/ydb/core/persqueue/partition.cpp index a4151cb566c..14e98aa2996 100644 --- a/ydb/core/persqueue/partition.cpp +++ b/ydb/core/persqueue/partition.cpp @@ -41,8 +41,7 @@ struct TPartition::THasDataReq { TMaybe<ui64> Cookie; TString ClientId; - bool operator < (const THasDataReq& req) const - { + bool operator < (const THasDataReq& req) const { return Num < req.Num; } }; @@ -51,8 +50,7 @@ struct TPartition::THasDataDeadline { TInstant Deadline; TPartition::THasDataReq Request; - bool operator < (const THasDataDeadline& dl) const - { + bool operator < (const THasDataDeadline& dl) const { return Deadline < dl.Deadline || Deadline == dl.Deadline && Request < dl.Request; } }; @@ -460,10 +458,6 @@ TPartition::TPartition(ui64 tabletId, ui32 partition, const TActorId& tablet, co , InitDuration(TDuration::Zero()) , InitDone(false) , NewPartition(newPartition) - // TODO: ToReview - Which name to use here? It verifies in tablet_counters_protobuf.h:633 on proper path - , PartitionCounters( - topicConverter->IsFirstClass() ? nullptr - : new TPartitionLabeledCounters(topicConverter->GetClientsideName(), partition)) , Subscriber(partition, TabletCounters, Tablet) , WriteCycleStartTime(ctx.Now()) , WriteCycleSize(0) @@ -494,6 +488,12 @@ TPartition::TPartition(ui64 tabletId, ui32 partition, const TActorId& tablet, co CalcTopicWriteQuotaParams(); TabletCounters.Populate(counters); + + if (topicConverter->IsFirstClass()) { + PartitionCountersLabeled = THolder<TPartitionLabeledCounters>(nullptr); + } else { + PartitionCountersLabeled = THolder<TPartitionLabeledCounters>(new TPartitionLabeledCounters(topicConverter->GetClientsideName(), partition)); + } } void TPartition::HandleMonitoring(TEvPQ::TEvMonRequest::TPtr& ev, const TActorContext& ctx) { @@ -1699,11 +1699,11 @@ void TPartition::InitComplete(const TActorContext& ctx) { Y_VERIFY(userInfoPair.second.Offset >= 0); ReadTimestampForOffset(userInfoPair.first, userInfoPair.second, ctx); } - if (PartitionCounters) { - PartitionCounters->GetCounters()[METRIC_INIT_TIME] = InitDuration.MilliSeconds(); - PartitionCounters->GetCounters()[METRIC_LIFE_TIME] = CreationTime.MilliSeconds(); - PartitionCounters->GetCounters()[METRIC_PARTITIONS] = 1; - ctx.Send(Tablet, new TEvPQ::TEvPartitionLabeledCounters(Partition, *PartitionCounters)); + if (PartitionCountersLabeled) { + PartitionCountersLabeled->GetCounters()[METRIC_INIT_TIME] = InitDuration.MilliSeconds(); + PartitionCountersLabeled->GetCounters()[METRIC_LIFE_TIME] = CreationTime.MilliSeconds(); + PartitionCountersLabeled->GetCounters()[METRIC_PARTITIONS] = 1; + ctx.Send(Tablet, new TEvPQ::TEvPartitionLabeledCounters(Partition, *PartitionCountersLabeled)); } UpdateUserInfoEndOffset(ctx.Now()); @@ -3151,7 +3151,7 @@ ui64 TPartition::GetSizeLag(i64 offset) { void TPartition::ReportCounters(const TActorContext& ctx) { - if (!PartitionCounters) { + if (!PartitionCountersLabeled) { return; } //per client counters @@ -3287,58 +3287,58 @@ void TPartition::ReportCounters(const TActorContext& ctx) { } } bool haveChanges = false; - if (SourceIdStorage.GetInMemorySourceIds().size() != PartitionCounters->GetCounters()[METRIC_MAX_NUM_SIDS].Get()) { + if (SourceIdStorage.GetInMemorySourceIds().size() != PartitionCountersLabeled->GetCounters()[METRIC_MAX_NUM_SIDS].Get()) { haveChanges = true; - PartitionCounters->GetCounters()[METRIC_MAX_NUM_SIDS].Set(SourceIdStorage.GetInMemorySourceIds().size()); - PartitionCounters->GetCounters()[METRIC_NUM_SIDS].Set(SourceIdStorage.GetInMemorySourceIds().size()); + PartitionCountersLabeled->GetCounters()[METRIC_MAX_NUM_SIDS].Set(SourceIdStorage.GetInMemorySourceIds().size()); + PartitionCountersLabeled->GetCounters()[METRIC_NUM_SIDS].Set(SourceIdStorage.GetInMemorySourceIds().size()); } TDuration lifetimeNow = ctx.Now() - SourceIdStorage.MinAvailableTimestamp(ctx.Now()); - if (lifetimeNow.MilliSeconds() != PartitionCounters->GetCounters()[METRIC_MIN_SID_LIFETIME].Get()) { + if (lifetimeNow.MilliSeconds() != PartitionCountersLabeled->GetCounters()[METRIC_MIN_SID_LIFETIME].Get()) { haveChanges = true; - PartitionCounters->GetCounters()[METRIC_MIN_SID_LIFETIME].Set(lifetimeNow.MilliSeconds()); + PartitionCountersLabeled->GetCounters()[METRIC_MIN_SID_LIFETIME].Set(lifetimeNow.MilliSeconds()); } const ui64 headGapSize = DataKeysBody.empty() ? 0 : (Head.Offset - (DataKeysBody.back().Key.GetOffset() + DataKeysBody.back().Key.GetCount())); const ui64 gapSize = GapSize + headGapSize; - if (gapSize != PartitionCounters->GetCounters()[METRIC_GAPS_SIZE].Get()) { + if (gapSize != PartitionCountersLabeled->GetCounters()[METRIC_GAPS_SIZE].Get()) { haveChanges = true; - PartitionCounters->GetCounters()[METRIC_MAX_GAPS_SIZE].Set(gapSize); - PartitionCounters->GetCounters()[METRIC_GAPS_SIZE].Set(gapSize); + PartitionCountersLabeled->GetCounters()[METRIC_MAX_GAPS_SIZE].Set(gapSize); + PartitionCountersLabeled->GetCounters()[METRIC_GAPS_SIZE].Set(gapSize); } const ui32 gapsCount = GapOffsets.size() + (headGapSize ? 1 : 0); - if (gapsCount != PartitionCounters->GetCounters()[METRIC_GAPS_COUNT].Get()) { + if (gapsCount != PartitionCountersLabeled->GetCounters()[METRIC_GAPS_COUNT].Get()) { haveChanges = true; - PartitionCounters->GetCounters()[METRIC_MAX_GAPS_COUNT].Set(gapsCount); - PartitionCounters->GetCounters()[METRIC_GAPS_COUNT].Set(gapsCount); + PartitionCountersLabeled->GetCounters()[METRIC_MAX_GAPS_COUNT].Set(gapsCount); + PartitionCountersLabeled->GetCounters()[METRIC_GAPS_COUNT].Set(gapsCount); } ui64 speed = WriteQuota.GetTotalSpeed(); - if (speed != PartitionCounters->GetCounters()[METRIC_WRITE_QUOTA_BYTES].Get()) { + if (speed != PartitionCountersLabeled->GetCounters()[METRIC_WRITE_QUOTA_BYTES].Get()) { haveChanges = true; - PartitionCounters->GetCounters()[METRIC_WRITE_QUOTA_BYTES].Set(speed); + PartitionCountersLabeled->GetCounters()[METRIC_WRITE_QUOTA_BYTES].Set(speed); } ui64 availSec = WriteQuota.GetAvailableAvgSec(ctx.Now()); - if (availSec != PartitionCounters->GetCounters()[METRIC_MIN_WRITE_QUOTA_BYTES_AVAIL_SEC].Get()) { + if (availSec != PartitionCountersLabeled->GetCounters()[METRIC_MIN_WRITE_QUOTA_BYTES_AVAIL_SEC].Get()) { haveChanges = true; - PartitionCounters->GetCounters()[METRIC_MIN_WRITE_QUOTA_BYTES_AVAIL_SEC].Set(availSec); + PartitionCountersLabeled->GetCounters()[METRIC_MIN_WRITE_QUOTA_BYTES_AVAIL_SEC].Set(availSec); } ui64 availMin = WriteQuota.GetAvailableAvgMin(ctx.Now()); - if (availMin != PartitionCounters->GetCounters()[METRIC_MIN_WRITE_QUOTA_BYTES_AVAIL_MIN].Get()) { + if (availMin != PartitionCountersLabeled->GetCounters()[METRIC_MIN_WRITE_QUOTA_BYTES_AVAIL_MIN].Get()) { haveChanges = true; - PartitionCounters->GetCounters()[METRIC_MIN_WRITE_QUOTA_BYTES_AVAIL_MIN].Set(availMin); + PartitionCountersLabeled->GetCounters()[METRIC_MIN_WRITE_QUOTA_BYTES_AVAIL_MIN].Set(availMin); } ui32 id = METRIC_TOTAL_WRITE_SPEED_1; for (ui32 i = 0; i < AvgWriteBytes.size(); ++i) { ui64 avg = AvgWriteBytes[i].GetValue(); - if (avg != PartitionCounters->GetCounters()[id].Get()) { + if (avg != PartitionCountersLabeled->GetCounters()[id].Get()) { haveChanges = true; - PartitionCounters->GetCounters()[id].Set(avg); //total - PartitionCounters->GetCounters()[id + 1].Set(avg); //max + PartitionCountersLabeled->GetCounters()[id].Set(avg); //total + PartitionCountersLabeled->GetCounters()[id + 1].Set(avg); //max } id += 2; } @@ -3348,10 +3348,10 @@ void TPartition::ReportCounters(const TActorContext& ctx) { id = METRIC_TOTAL_QUOTA_SPEED_1; for (ui32 i = 0; i < AvgQuotaBytes.size(); ++i) { ui64 avg = AvgQuotaBytes[i].GetValue(); - if (avg != PartitionCounters->GetCounters()[id].Get()) { + if (avg != PartitionCountersLabeled->GetCounters()[id].Get()) { haveChanges = true; - PartitionCounters->GetCounters()[id].Set(avg); //total - PartitionCounters->GetCounters()[id + 1].Set(avg); //max + PartitionCountersLabeled->GetCounters()[id].Set(avg); //total + PartitionCountersLabeled->GetCounters()[id + 1].Set(avg); //max } id += 2; } @@ -3359,39 +3359,38 @@ void TPartition::ReportCounters(const TActorContext& ctx) { if (WriteQuota.GetTotalSpeed()) { ui64 quotaUsage = ui64(AvgQuotaBytes[1].GetValue()) * 1000000 / WriteQuota.GetTotalSpeed() / 60; - if (quotaUsage != PartitionCounters->GetCounters()[METRIC_WRITE_QUOTA_USAGE].Get()) { + if (quotaUsage != PartitionCountersLabeled->GetCounters()[METRIC_WRITE_QUOTA_USAGE].Get()) { haveChanges = true; - PartitionCounters->GetCounters()[METRIC_WRITE_QUOTA_USAGE].Set(quotaUsage); + PartitionCountersLabeled->GetCounters()[METRIC_WRITE_QUOTA_USAGE].Set(quotaUsage); } } ui64 partSize = BodySize + Head.PackedSize; - if (partSize != PartitionCounters->GetCounters()[METRIC_TOTAL_PART_SIZE].Get()) { + if (partSize != PartitionCountersLabeled->GetCounters()[METRIC_TOTAL_PART_SIZE].Get()) { haveChanges = true; - PartitionCounters->GetCounters()[METRIC_MAX_PART_SIZE].Set(partSize); - PartitionCounters->GetCounters()[METRIC_TOTAL_PART_SIZE].Set(partSize); + PartitionCountersLabeled->GetCounters()[METRIC_MAX_PART_SIZE].Set(partSize); + PartitionCountersLabeled->GetCounters()[METRIC_TOTAL_PART_SIZE].Set(partSize); } ui64 ts = (WriteTimestamp.MilliSeconds() < MIN_TIMESTAMP_MS) ? Max<i64>() : WriteTimestamp.MilliSeconds(); - if (PartitionCounters->GetCounters()[METRIC_LAST_WRITE_TIME].Get() != ts) { + if (PartitionCountersLabeled->GetCounters()[METRIC_LAST_WRITE_TIME].Get() != ts) { haveChanges = true; - PartitionCounters->GetCounters()[METRIC_LAST_WRITE_TIME].Set(ts); + PartitionCountersLabeled->GetCounters()[METRIC_LAST_WRITE_TIME].Set(ts); } ui64 timeLag = WriteLagMs.GetValue(); - if (PartitionCounters->GetCounters()[METRIC_WRITE_TIME_LAG_MS].Get() != timeLag) { + if (PartitionCountersLabeled->GetCounters()[METRIC_WRITE_TIME_LAG_MS].Get() != timeLag) { haveChanges = true; - PartitionCounters->GetCounters()[METRIC_WRITE_TIME_LAG_MS].Set(timeLag); + PartitionCountersLabeled->GetCounters()[METRIC_WRITE_TIME_LAG_MS].Set(timeLag); } if (haveChanges) { - ctx.Send(Tablet, new TEvPQ::TEvPartitionLabeledCounters(Partition, *PartitionCounters)); + ctx.Send(Tablet, new TEvPQ::TEvPartitionLabeledCounters(Partition, *PartitionCountersLabeled)); } } void TPartition::Handle(TEvKeyValue::TEvResponse::TPtr& ev, const TActorContext& ctx) { - auto& response = ev->Get()->Record; //check correctness of response @@ -4002,11 +4001,14 @@ void TPartition::WriteClientInfo(const ui64 cookie, TUserInfo& userInfo, const T ); offset = EndOffset; ev->Offset = offset; -/* TabletCounters.Cumulative()[COUNTER_PQ_SET_CLIENT_OFFSET_ERROR].Increment(1); +/* + TODO: + TabletCounters.Cumulative()[COUNTER_PQ_SET_CLIENT_OFFSET_ERROR].Increment(1); ReplyError(ctx, ev->Cookie, NPersQueue::NErrorCode::SET_OFFSET_ERROR_COMMIT_TO_FUTURE, TStringBuilder() << "can't commit to future. Offset " << offset << " EndOffset " << EndOffset); userInfo.UserActrs.pop_front(); - continue;*/ + continue; +*/ } TBuffer idata; @@ -4270,7 +4272,8 @@ bool TPartition::AppendHeadWithNewWrites(TEvKeyValue::TEvRequest* request, const TInstant::MilliSeconds(p.Msg.CreateTimestamp == 0 ? curOffset : p.Msg.CreateTimestamp), p.Msg.UncompressedSize, p.Msg.PartitionKey, p.Msg.ExplicitHashKey); //remove curOffset when LB will report CTime - ui64 writeLagMs = (WriteTimestamp - TInstant::MilliSeconds(p.Msg.CreateTimestamp)).MilliSeconds(); + const ui64 writeLagMs = + (WriteTimestamp - TInstant::MilliSeconds(p.Msg.CreateTimestamp)).MilliSeconds(); WriteLagMs.Update(writeLagMs, WriteTimestamp); if (InputTimeLag) { InputTimeLag->IncFor(writeLagMs, 1); diff --git a/ydb/core/persqueue/partition.h b/ydb/core/persqueue/partition.h index bdd7ed90db6..c25b1f9c7f1 100644 --- a/ydb/core/persqueue/partition.h +++ b/ydb/core/persqueue/partition.h @@ -21,9 +21,9 @@ namespace NKikimr::NPQ { static const ui32 MAX_BLOB_PART_SIZE = 500_KB; -ui64 GetOffsetEstimate(const std::deque<TDataKey>& container, TInstant timestamp, ui64 headOffset); +using TPartitionLabeledCounters = TProtobufTabletLabeledCounters<EPartitionLabeledCounters_descriptor>; -typedef TProtobufTabletLabeledCounters<EPartitionLabeledCounters_descriptor> TPartitionLabeledCounters; +ui64 GetOffsetEstimate(const std::deque<TDataKey>& container, TInstant timestamp, ui64 headOffset); class TKeyLevel; struct TMirrorerInfo; @@ -408,7 +408,7 @@ private: std::deque<THolder<TEvPQ::TEvChangeOwner>> WaitToChangeOwner; TTabletCountersBase TabletCounters; - THolder<TPartitionLabeledCounters> PartitionCounters; + THolder<TPartitionLabeledCounters> PartitionCountersLabeled; TSubscriber Subscriber; diff --git a/ydb/core/persqueue/pq_impl.cpp b/ydb/core/persqueue/pq_impl.cpp index 75b325d5409..c66c6af4faf 100644 --- a/ydb/core/persqueue/pq_impl.cpp +++ b/ydb/core/persqueue/pq_impl.cpp @@ -652,8 +652,8 @@ void TPersQueue::ApplyNewConfigAndReply(const TActorContext& ctx) const auto partitionId = partition.GetPartitionId(); if (Partitions.find(partitionId) == Partitions.end()) { Partitions.emplace(partitionId, TPartitionInfo( - ctx.Register(new TPartition(TabletID(), partitionId, ctx.SelfID, CacheActor, TopicConverter, IsLocalDC, - DCId, Config, *Counters, ctx, true)), + ctx.Register(new TPartition(TabletID(), partitionId, ctx.SelfID, CacheActor, TopicConverter, + IsLocalDC, DCId, Config, *Counters, ctx, true)), GetPartitionKeyRange(partition), true, *Counters @@ -987,8 +987,10 @@ void TPersQueue::AggregateAndSendLabeledCountersFor(const TString& group, const Y_VERIFY(aggr->HasCounters()); TActorId countersAggregator = MakeTabletCountersAggregatorID(ctx.SelfID.NodeId()); - ctx.Send(countersAggregator, new TEvTabletCounters::TEvTabletAddLabeledCounters( - CounterEventsInflight[group], TabletID(), TTabletTypes::PersQueue, aggr)); + ctx.Send(countersAggregator, + new TEvTabletCounters::TEvTabletAddLabeledCounters( + CounterEventsInflight[group], TabletID(), TTabletTypes::PersQueue, aggr) + ); } } @@ -1535,7 +1537,7 @@ void TPersQueue::HandleWriteRequest(const ui64 responseCookie, const TActorId& p for (ui32 i = 0; i < req.CmdWriteSize(); ++i) { const auto& cmd = req.GetCmdWrite(i); - if (AppData(ctx)->Counters) { + if (AppData(ctx)->Counters && !AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen()) { auto counters = AppData(ctx)->Counters; TString clientDC = to_lower(cmd.HasClientDC() ? cmd.GetClientDC() : "unknown"); clientDC.to_title(); @@ -2117,6 +2119,7 @@ void TPersQueue::Handle(TEvInterconnect::TEvNodeInfo::TPtr& ev, const TActorCont } void TPersQueue::HandleWakeup(const TActorContext& ctx) { + // TIP: Send LabeledCounters here THashSet<TString> groups; for (auto& p : Partitions) { for (auto& m : p.second.LabeledCounters) { diff --git a/ydb/core/persqueue/pq_ut.cpp b/ydb/core/persqueue/pq_ut.cpp index d68710b1562..7986bb63cd6 100644 --- a/ydb/core/persqueue/pq_ut.cpp +++ b/ydb/core/persqueue/pq_ut.cpp @@ -1,14 +1,9 @@ #include "pq_ut.h" -#include <ydb/core/testlib/basics/runtime.h> -#include <ydb/core/tablet_flat/tablet_flat_executed.h> #include <ydb/core/tx/schemeshard/schemeshard.h> -#include <ydb/public/lib/base/msgbus.h> #include <ydb/core/keyvalue/keyvalue_events.h> #include <ydb/core/persqueue/events/global.h> -#include <ydb/core/tablet/tablet_counters_aggregator.h> #include <ydb/core/persqueue/partition.h> -#include <ydb/core/engine/minikql/flat_local_tx_factory.h> #include <ydb/core/security/ticket_parser.h> #include <ydb/core/testlib/fake_scheme_shard.h> @@ -484,136 +479,6 @@ Y_UNIT_TEST(TestCheckACL) { } -void CheckLabeledCountersResponse(ui32 count, TTestContext& tc, TVector<TString> mustHave = {}) { - IActor* actor = CreateClusterLabeledCountersAggregatorActor(tc.Edge, TTabletTypes::PersQueue); - tc.Runtime->Register(actor); - - TAutoPtr<IEventHandle> handle; - TEvTabletCounters::TEvTabletLabeledCountersResponse *result; - result = tc.Runtime->GrabEdgeEvent<TEvTabletCounters::TEvTabletLabeledCountersResponse>(handle); - UNIT_ASSERT(result); - THashSet<TString> groups; - - Cerr << "Checking with " << count << " groups:\n"; - - for (ui32 i = 0; i < result->Record.LabeledCountersByGroupSize(); ++i) { - auto& c = result->Record.GetLabeledCountersByGroup(i); - groups.insert(c.GetGroup()); - Cerr << "Has " << c.GetGroup() << "\n"; - } - UNIT_ASSERT_VALUES_EQUAL(groups.size(), count); - for (auto& g : mustHave) { - UNIT_ASSERT(groups.contains(g)); - } -} - -Y_UNIT_TEST(TestSwitchOffImportantFlag) { - TTestContext tc; - RunTestWithReboots(tc.TabletIds, [&]() { - return tc.InitialEventsFilter.Prepare(); - }, [&](const TString& dispatchName, std::function<void(TTestActorRuntime&)> setup, bool& activeZone) { - TFinalizer finalizer(tc); - tc.Prepare(dispatchName, setup, activeZone); - activeZone = false; - tc.Runtime->SetScheduledLimit(600); - PQTabletPrepare({}, {}, tc); - - { - TDispatchOptions options; - options.FinalEvents.emplace_back(TEvTabletCounters::EvTabletAddLabeledCounters); - tc.Runtime->DispatchEvents(options); - } - - CheckLabeledCountersResponse(8, tc); //only topic counters - - PQTabletPrepare({}, {{"user", true}}, tc); - - { - TDispatchOptions options; - options.FinalEvents.emplace_back(TEvTabletCounters::EvTabletAddLabeledCounters); - tc.Runtime->DispatchEvents(options); - } - - - CheckLabeledCountersResponse(8, tc, {NKikimr::JoinPath({"user/1", TOPIC_NAME})}); //topic counters + important - - PQTabletPrepare({}, {}, tc); - - { - TDispatchOptions options; - options.FinalEvents.emplace_back(TEvTabletCounters::EvTabletAddLabeledCounters); - tc.Runtime->DispatchEvents(options); - } - - { - TDispatchOptions options; - options.FinalEvents.emplace_back(TEvTabletCounters::EvTabletAddLabeledCounters); - tc.Runtime->DispatchEvents(options); - } - - auto MakeTopics = [&] (const TVector<TString>& users) { - TVector<TString> res; - for (const auto& u : users) { - res.emplace_back(NKikimr::JoinPath({u, TOPIC_NAME})); - } - return res; - }; - CheckLabeledCountersResponse(8, tc, MakeTopics({"user/0"})); //topic counters + not important - - PQTabletPrepare({}, {{"user", true}, {"user2", true}}, tc); - - { - TDispatchOptions options; - options.FinalEvents.emplace_back(TEvTabletCounters::EvTabletAddLabeledCounters); - tc.Runtime->DispatchEvents(options); - } - - { - TDispatchOptions options; - options.FinalEvents.emplace_back(TEvTabletCounters::EvTabletAddLabeledCounters); - tc.Runtime->DispatchEvents(options); - } - - CheckLabeledCountersResponse(11, tc, MakeTopics({"user/1", "user2/1"})); //topic counters + not important - - PQTabletPrepare({}, {{"user", true}, {"user2", false}}, tc); - - { - TDispatchOptions options; - options.FinalEvents.emplace_back(TEvTabletCounters::EvTabletAddLabeledCounters); - tc.Runtime->DispatchEvents(options); - } - - { - TDispatchOptions options; - options.FinalEvents.emplace_back(TEvTabletCounters::EvTabletAddLabeledCounters); - tc.Runtime->DispatchEvents(options); - } - - CheckLabeledCountersResponse(12, tc, MakeTopics({"user/1", "user2/0"})); - - - PQTabletPrepare({}, {{"user", true}}, tc); - - { - TDispatchOptions options; - options.FinalEvents.emplace_back(TEvTabletCounters::EvTabletAddLabeledCounters); - tc.Runtime->DispatchEvents(options); - } - - { - TDispatchOptions options; - options.FinalEvents.emplace_back(TEvTabletCounters::EvTabletAddLabeledCounters); - tc.Runtime->DispatchEvents(options); - } - - CheckLabeledCountersResponse(8, tc, MakeTopics({"user/1"})); - - - }); -} - - Y_UNIT_TEST(TestSeveralOwners) { TTestContext tc; RunTestWithReboots(tc.TabletIds, [&]() { @@ -624,7 +489,8 @@ Y_UNIT_TEST(TestSeveralOwners) { activeZone = false; tc.Runtime->SetScheduledLimit(200); - PQTabletPrepare({}, {}, tc); //no important clients, lifetimeseconds=0 - delete all right now, except last datablob + // No important clients, lifetimeseconds=0 - delete all right now, except last datablob + PQTabletPrepare({}, {}, tc); TVector<std::pair<ui64, TString>> data; @@ -656,7 +522,8 @@ Y_UNIT_TEST(TestWaitInOwners) { activeZone = false; tc.Runtime->SetScheduledLimit(200); - PQTabletPrepare({}, {}, tc); //no important clients, lifetimeseconds=0 - delete all right now, except last datablob + // No important clients, lifetimeseconds=0 - delete all right now, except last datablob + PQTabletPrepare({}, {}, tc); TVector<std::pair<ui64, TString>> data; diff --git a/ydb/core/persqueue/pq_ut.h b/ydb/core/persqueue/pq_ut.h index 753613f528a..13e671885f4 100644 --- a/ydb/core/persqueue/pq_ut.h +++ b/ydb/core/persqueue/pq_ut.h @@ -29,6 +29,7 @@ void SetupLogging(TTestActorRuntime& runtime) { NActors::NLog::EPriority otherPriority = NLog::PRI_INFO; runtime.SetLogPriority(NKikimrServices::PERSQUEUE, pqPriority); + runtime.SetLogPriority(NKikimrServices::SYSTEM_VIEWS, pqPriority); runtime.SetLogPriority(NKikimrServices::KEYVALUE, priority); runtime.SetLogPriority(NKikimrServices::BOOTSTRAPPER, priority); runtime.SetLogPriority(NKikimrServices::TABLET_MAIN, priority); @@ -115,10 +116,14 @@ struct TTestContext { return RequestTimeoutFilter(runtime, event, duration, deadline); } - void Prepare(const TString& dispatchName, std::function<void(TTestActorRuntime&)> setup, bool& outActiveZone) { + void Prepare(const TString& dispatchName, std::function<void(TTestActorRuntime&)> setup, bool& outActiveZone, bool isFirstClass = false, bool enableMonitoring = false) { Y_UNUSED(dispatchName); outActiveZone = false; - Runtime.Reset(new TTestBasicRuntime); + TTestBasicRuntime* runtime = new TTestBasicRuntime; + if (enableMonitoring) { + runtime->SetupMonitoring(); + } + Runtime.Reset(runtime); Runtime->SetScheduledLimit(200); SetupLogging(*Runtime); @@ -128,18 +133,17 @@ struct TTestContext { CreateTestTabletInfo(TabletId, TabletType, TErasureType::ErasureNone), &CreatePersQueue); - TDispatchOptions options; - options.FinalEvents.push_back(TDispatchOptions::TFinalEventCondition(TEvTablet::EvBoot)); Runtime->GetAppData(0).PQConfig.SetEnabled(true); // NOTE(shmel1k@): KIKIMR-14221 - Runtime->GetAppData(0).PQConfig.SetTopicsAreFirstClassCitizen(false); + Runtime->GetAppData(0).PQConfig.SetTopicsAreFirstClassCitizen(isFirstClass); Runtime->GetAppData(0).PQConfig.SetRequireCredentialsInNewProtocol(false); Runtime->GetAppData(0).PQConfig.SetClusterTablePath("/Root/PQ/Config/V2/Cluster"); Runtime->GetAppData(0).PQConfig.SetVersionTablePath("/Root/PQ/Config/V2/Versions"); - Runtime->GetAppData(0).PQConfig.SetTopicsAreFirstClassCitizen(false); Runtime->GetAppData(0).PQConfig.SetRoot("/Root/PQ"); Runtime->GetAppData(0).PQConfig.MutableQuotingConfig()->SetEnableQuoting(false); + TDispatchOptions options; + options.FinalEvents.push_back(TDispatchOptions::TFinalEventCondition(TEvTablet::EvBoot)); Runtime->DispatchEvents(options); CreateTestBootstrapper(*Runtime, @@ -165,6 +169,8 @@ struct TTestContext { CreateTestTabletInfo(TabletId, TabletType, TErasureType::ErasureNone), &CreatePersQueue); + Runtime->GetAppData(0).PQConfig.SetEnabled(true); + TDispatchOptions options; options.FinalEvents.push_back(TDispatchOptions::TFinalEventCondition(TEvTablet::EvBoot)); Runtime->DispatchEvents(options); @@ -179,7 +185,6 @@ struct TTestContext { Edge = Runtime->AllocateEdgeActor(); Runtime->SetScheduledEventFilter(&RequestTimeoutFilter); - Runtime->GetAppData(0).PQConfig.SetEnabled(true); } @@ -215,6 +220,10 @@ struct TTabletPreparationParameters { ui64 sidMaxCount{0}; ui32 specVersion{0}; i32 storageLimitBytes{0}; + TString folderId{"somefolder"}; + TString cloudId{"somecloud"}; + TString databaseId{"root"}; + TString account{"federationAccount"}; }; void PQTabletPrepare( const TTabletPreparationParameters& parameters, diff --git a/ydb/core/persqueue/pq_ut_impl.cpp b/ydb/core/persqueue/pq_ut_impl.cpp index b7c39bed47d..8153d9bc598 100644 --- a/ydb/core/persqueue/pq_ut_impl.cpp +++ b/ydb/core/persqueue/pq_ut_impl.cpp @@ -40,9 +40,13 @@ void PQTabletPrepare(const TTabletPreparationParameters& parameters, if (tc.Runtime->GetAppData().PQConfig.GetTopicsAreFirstClassCitizen()) { tabletConfig->SetTopicName("topic"); tabletConfig->SetTopicPath(tc.Runtime->GetAppData().PQConfig.GetDatabase() + "/topic"); + tabletConfig->SetYcCloudId(parameters.cloudId); + tabletConfig->SetYcFolderId(parameters.folderId); + tabletConfig->SetYdbDatabaseId(parameters.databaseId); + tabletConfig->SetFederationAccount(parameters.account); } else { - tabletConfig->SetTopicName("rt3.dc1--topic"); - tabletConfig->SetTopicPath("/Root/PQ/rt3.dc1--topic"); + tabletConfig->SetTopicName("rt3.dc1--asdfgs--topic"); + tabletConfig->SetTopicPath("/Root/PQ/rt3.dc1--asdfgs--topic"); } tabletConfig->SetTopic("topic"); tabletConfig->SetVersion(version); diff --git a/ydb/core/persqueue/ut/CMakeLists.darwin.txt b/ydb/core/persqueue/ut/CMakeLists.darwin.txt index 2ab8a0cbeb0..fe97d95f459 100644 --- a/ydb/core/persqueue/ut/CMakeLists.darwin.txt +++ b/ydb/core/persqueue/ut/CMakeLists.darwin.txt @@ -25,6 +25,7 @@ target_link_libraries(ydb-core-persqueue-ut PUBLIC library-cpp-svnversion ydb-core-testlib ydb_persqueue_core-ut-ut_utils + library-cpp-resource ) target_link_options(ydb-core-persqueue-ut PRIVATE -Wl,-no_deduplicate @@ -35,6 +36,7 @@ target_link_options(ydb-core-persqueue-ut PRIVATE CoreFoundation ) target_sources(ydb-core-persqueue-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/persqueue/counters_ut.cpp ${CMAKE_SOURCE_DIR}/ydb/core/persqueue/internals_ut.cpp ${CMAKE_SOURCE_DIR}/ydb/core/persqueue/metering_sink_ut.cpp ${CMAKE_SOURCE_DIR}/ydb/core/persqueue/mirrorer_ut.cpp @@ -43,6 +45,7 @@ target_sources(ydb-core-persqueue-ut PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/persqueue/sourceid_ut.cpp ${CMAKE_SOURCE_DIR}/ydb/core/persqueue/type_codecs_ut.cpp ${CMAKE_SOURCE_DIR}/ydb/core/persqueue/user_info_ut.cpp + ${CMAKE_BINARY_DIR}/ydb/core/persqueue/ut/58e1cbb1dd06a8375ced22e9e6f7699d.cpp ) add_test( NAME @@ -55,4 +58,17 @@ add_test( --print-times --show-fails ) +resources(ydb-core-persqueue-ut + ${CMAKE_BINARY_DIR}/ydb/core/persqueue/ut/58e1cbb1dd06a8375ced22e9e6f7699d.cpp + INPUTS + ${CMAKE_SOURCE_DIR}/ydb/core/persqueue/ut/counters_datastreams.html + ${CMAKE_SOURCE_DIR}/ydb/core/persqueue/ut/counters_pqproxy_firstclass.html + ${CMAKE_SOURCE_DIR}/ydb/core/persqueue/ut/counters_pqproxy.html + ${CMAKE_SOURCE_DIR}/ydb/core/persqueue/ut/counters_labeled.html + KEYS + counters_datastreams.html + counters_pqproxy_firstclass.html + counters_pqproxy.html + counters_labeled.html +) vcs_info(ydb-core-persqueue-ut) diff --git a/ydb/core/persqueue/ut/CMakeLists.linux.txt b/ydb/core/persqueue/ut/CMakeLists.linux.txt index bd73cffc77b..e0e57717d3d 100644 --- a/ydb/core/persqueue/ut/CMakeLists.linux.txt +++ b/ydb/core/persqueue/ut/CMakeLists.linux.txt @@ -27,6 +27,7 @@ target_link_libraries(ydb-core-persqueue-ut PUBLIC library-cpp-svnversion ydb-core-testlib ydb_persqueue_core-ut-ut_utils + library-cpp-resource ) target_link_options(ydb-core-persqueue-ut PRIVATE -ldl @@ -39,6 +40,7 @@ target_link_options(ydb-core-persqueue-ut PRIVATE -ldl ) target_sources(ydb-core-persqueue-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/persqueue/counters_ut.cpp ${CMAKE_SOURCE_DIR}/ydb/core/persqueue/internals_ut.cpp ${CMAKE_SOURCE_DIR}/ydb/core/persqueue/metering_sink_ut.cpp ${CMAKE_SOURCE_DIR}/ydb/core/persqueue/mirrorer_ut.cpp @@ -47,6 +49,7 @@ target_sources(ydb-core-persqueue-ut PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/persqueue/sourceid_ut.cpp ${CMAKE_SOURCE_DIR}/ydb/core/persqueue/type_codecs_ut.cpp ${CMAKE_SOURCE_DIR}/ydb/core/persqueue/user_info_ut.cpp + ${CMAKE_BINARY_DIR}/ydb/core/persqueue/ut/58e1cbb1dd06a8375ced22e9e6f7699d.cpp ) add_test( NAME @@ -59,4 +62,17 @@ add_test( --print-times --show-fails ) +resources(ydb-core-persqueue-ut + ${CMAKE_BINARY_DIR}/ydb/core/persqueue/ut/58e1cbb1dd06a8375ced22e9e6f7699d.cpp + INPUTS + ${CMAKE_SOURCE_DIR}/ydb/core/persqueue/ut/counters_datastreams.html + ${CMAKE_SOURCE_DIR}/ydb/core/persqueue/ut/counters_pqproxy_firstclass.html + ${CMAKE_SOURCE_DIR}/ydb/core/persqueue/ut/counters_pqproxy.html + ${CMAKE_SOURCE_DIR}/ydb/core/persqueue/ut/counters_labeled.html + KEYS + counters_datastreams.html + counters_pqproxy_firstclass.html + counters_pqproxy.html + counters_labeled.html +) vcs_info(ydb-core-persqueue-ut) diff --git a/ydb/core/persqueue/ut/counters_datastreams.html b/ydb/core/persqueue/ut/counters_datastreams.html new file mode 100644 index 00000000000..f01415c7556 --- /dev/null +++ b/ydb/core/persqueue/ut/counters_datastreams.html @@ -0,0 +1,94 @@ +<pre> +cloud=somecloud: + + folder=somefolder: + + database=root: + + stream=topic: + name=stream.incoming_bytes_per_second: 2700 + name=stream.incoming_records_per_second: 150 + name=stream.internal_write.bytes_per_second: 2700 + name=stream.internal_write.compacted_bytes_per_second: 3720 + name=stream.internal_write.records_per_second: 150 + name=stream.internal_write.uncompressed_bytes_per_second: 1350 + + consumer=user: + name=stream.internal_read.bytes_per_second: 0 + name=stream.internal_read.records_per_second: 0 + name=stream.outgoing_bytes_per_second: 0 + name=stream.outgoing_records_per_second: 0 + + host=1: + + shard=0: + name=stream.await_operating_milliseconds: 0 + + shard=1: + name=stream.await_operating_milliseconds: 0 + + name=stream.internal_read.time_lags_milliseconds: + bin=100: 0 + bin=1000: 0 + bin=10000: 0 + bin=180000: 0 + bin=200: 0 + bin=2000: 0 + bin=30000: 0 + bin=500: 0 + bin=5000: 0 + bin=60000: 0 + bin=999999: 0 + + host=1: + + shard=0: + name=stream.internal_write.buffer_brimmed_duration_ms: 0 + + shard=1: + name=stream.internal_write.buffer_brimmed_duration_ms: 0 + + name=stream.internal_write.partition_write_quota_wait_milliseconds: + bin=0: 150 + bin=1: 0 + bin=10: 0 + bin=100: 0 + bin=1000: 0 + bin=10000: 0 + bin=20: 0 + bin=2500: 0 + bin=5: 0 + bin=50: 0 + bin=500: 0 + bin=5000: 0 + bin=999999: 0 + + name=stream.internal_write.record_size_bytes: + bin=1024: 150 + bin=10240: 0 + bin=102400: 0 + bin=1048576: 0 + bin=10485760: 0 + bin=20480: 0 + bin=204800: 0 + bin=2097152: 0 + bin=5120: 0 + bin=51200: 0 + bin=524288: 0 + bin=5242880: 0 + bin=67108864: 0 + bin=99999999: 0 + + name=stream.internal_write.time_lags_milliseconds: + bin=100: 150 + bin=1000: 0 + bin=10000: 0 + bin=180000: 0 + bin=200: 0 + bin=2000: 0 + bin=30000: 0 + bin=500: 0 + bin=5000: 0 + bin=60000: 0 + bin=999999: 0 +</pre> diff --git a/ydb/core/persqueue/ut/counters_labeled.html b/ydb/core/persqueue/ut/counters_labeled.html new file mode 100644 index 00000000000..86176ab9e27 --- /dev/null +++ b/ydb/core/persqueue/ut/counters_labeled.html @@ -0,0 +1,247 @@ +<pre> +user_counters=PersQueue: + + client=total: + + important=0: + + topic=rt3.dc1--topic: + sensor=PQ/MessageLagByCommitted: 30 + sensor=PQ/MessageLagByLastRead: 29 + sensor=PQ/PartitionMaxReadQuotaUsage: 0 + sensor=PQ/ReadBytesAvailAvgMin: 1000000000 + sensor=PQ/ReadBytesAvailAvgSec: 1000000000 + sensor=PQ/ReadBytesMaxPerDay: 0 + sensor=PQ/ReadBytesMaxPerHour: 0 + sensor=PQ/ReadBytesMaxPerMin: 0 + sensor=PQ/ReadBytesMaxPerSec: 0 + sensor=PQ/ReadBytesPerDay: 0 + sensor=PQ/ReadBytesPerHour: 0 + sensor=PQ/ReadBytesPerMin: 0 + sensor=PQ/ReadBytesPerSec: 0 + sensor=PQ/ReadBytesQuota: 1000000000 + sensor=PQ/ReadOffsetRewindSum: 0 + sensor=PQ/ReadTimeLagMs: 0 + sensor=PQ/SizeLagByCommitted: 744 + sensor=PQ/SizeLagByLastRead: 744 + sensor=PQ/TimeSinceLastReadMs: 4989 + sensor=PQ/TotalMessageLagByLastRead: 29 + sensor=PQ/TotalSizeLagByLastRead: 744 + sensor=PQ/TotalTimeLagMsByLastRead: 4929 + sensor=PQ/UserPartitionsAnswered: 2 + sensor=PQ/WriteTimeLagMsByLastRead: 29 + sensor=PQ/WriteTimeLagMsByLastReadOld: 4989 + + important=total: + + topic=rt3.dc1--topic: + sensor=PQ/MessageLagByCommitted: 30 + sensor=PQ/MessageLagByLastRead: 29 + sensor=PQ/PartitionMaxReadQuotaUsage: 0 + sensor=PQ/ReadBytesAvailAvgMin: 1000000000 + sensor=PQ/ReadBytesAvailAvgSec: 1000000000 + sensor=PQ/ReadBytesMaxPerDay: 0 + sensor=PQ/ReadBytesMaxPerHour: 0 + sensor=PQ/ReadBytesMaxPerMin: 0 + sensor=PQ/ReadBytesMaxPerSec: 0 + sensor=PQ/ReadBytesPerDay: 0 + sensor=PQ/ReadBytesPerHour: 0 + sensor=PQ/ReadBytesPerMin: 0 + sensor=PQ/ReadBytesPerSec: 0 + sensor=PQ/ReadBytesQuota: 1000000000 + sensor=PQ/ReadOffsetRewindSum: 0 + sensor=PQ/ReadTimeLagMs: 0 + sensor=PQ/SizeLagByCommitted: 744 + sensor=PQ/SizeLagByLastRead: 744 + sensor=PQ/TimeSinceLastReadMs: 4989 + sensor=PQ/TotalMessageLagByLastRead: 29 + sensor=PQ/TotalSizeLagByLastRead: 744 + sensor=PQ/TotalTimeLagMsByLastRead: 4929 + sensor=PQ/UserPartitionsAnswered: 2 + sensor=PQ/WriteTimeLagMsByLastRead: 29 + sensor=PQ/WriteTimeLagMsByLastReadOld: 4989 + + topic=total: + sensor=PQ/MessageLagByCommitted: 30 + sensor=PQ/MessageLagByLastRead: 29 + sensor=PQ/PartitionMaxReadQuotaUsage: 0 + sensor=PQ/ReadBytesAvailAvgMin: 1000000000 + sensor=PQ/ReadBytesAvailAvgSec: 1000000000 + sensor=PQ/ReadBytesMaxPerDay: 0 + sensor=PQ/ReadBytesMaxPerHour: 0 + sensor=PQ/ReadBytesMaxPerMin: 0 + sensor=PQ/ReadBytesMaxPerSec: 0 + sensor=PQ/ReadBytesPerDay: 0 + sensor=PQ/ReadBytesPerHour: 0 + sensor=PQ/ReadBytesPerMin: 0 + sensor=PQ/ReadBytesPerSec: 0 + sensor=PQ/ReadBytesQuota: 1000000000 + sensor=PQ/ReadOffsetRewindSum: 0 + sensor=PQ/ReadTimeLagMs: 0 + sensor=PQ/SizeLagByCommitted: 744 + sensor=PQ/SizeLagByLastRead: 744 + sensor=PQ/TimeSinceLastReadMs: 4989 + sensor=PQ/TotalMessageLagByLastRead: 29 + sensor=PQ/TotalSizeLagByLastRead: 744 + sensor=PQ/TotalTimeLagMsByLastRead: 4929 + sensor=PQ/UserPartitionsAnswered: 2 + sensor=PQ/WriteTimeLagMsByLastRead: 29 + sensor=PQ/WriteTimeLagMsByLastReadOld: 4989 + + client=user: + + important=0: + + topic=rt3.dc1--topic: + sensor=PQ/MessageLagByCommitted: 30 + sensor=PQ/MessageLagByLastRead: 29 + sensor=PQ/PartitionMaxReadQuotaUsage: 0 + sensor=PQ/ReadBytesAvailAvgMin: 1000000000 + sensor=PQ/ReadBytesAvailAvgSec: 1000000000 + sensor=PQ/ReadBytesMaxPerDay: 0 + sensor=PQ/ReadBytesMaxPerHour: 0 + sensor=PQ/ReadBytesMaxPerMin: 0 + sensor=PQ/ReadBytesMaxPerSec: 0 + sensor=PQ/ReadBytesPerDay: 0 + sensor=PQ/ReadBytesPerHour: 0 + sensor=PQ/ReadBytesPerMin: 0 + sensor=PQ/ReadBytesPerSec: 0 + sensor=PQ/ReadBytesQuota: 1000000000 + sensor=PQ/ReadOffsetRewindSum: 0 + sensor=PQ/ReadTimeLagMs: 0 + sensor=PQ/SizeLagByCommitted: 744 + sensor=PQ/SizeLagByLastRead: 744 + sensor=PQ/TimeSinceLastReadMs: 4989 + sensor=PQ/TotalMessageLagByLastRead: 29 + sensor=PQ/TotalSizeLagByLastRead: 744 + sensor=PQ/TotalTimeLagMsByLastRead: 4929 + sensor=PQ/UserPartitionsAnswered: 2 + sensor=PQ/WriteTimeLagMsByLastRead: 29 + sensor=PQ/WriteTimeLagMsByLastReadOld: 4989 + + topic=total: + sensor=PQ/MessageLagByCommitted: 30 + sensor=PQ/MessageLagByLastRead: 29 + sensor=PQ/PartitionMaxReadQuotaUsage: 0 + sensor=PQ/ReadBytesAvailAvgMin: 1000000000 + sensor=PQ/ReadBytesAvailAvgSec: 1000000000 + sensor=PQ/ReadBytesMaxPerDay: 0 + sensor=PQ/ReadBytesMaxPerHour: 0 + sensor=PQ/ReadBytesMaxPerMin: 0 + sensor=PQ/ReadBytesMaxPerSec: 0 + sensor=PQ/ReadBytesPerDay: 0 + sensor=PQ/ReadBytesPerHour: 0 + sensor=PQ/ReadBytesPerMin: 0 + sensor=PQ/ReadBytesPerSec: 0 + sensor=PQ/ReadBytesQuota: 1000000000 + sensor=PQ/ReadOffsetRewindSum: 0 + sensor=PQ/ReadTimeLagMs: 0 + sensor=PQ/SizeLagByCommitted: 744 + sensor=PQ/SizeLagByLastRead: 744 + sensor=PQ/TimeSinceLastReadMs: 4989 + sensor=PQ/TotalMessageLagByLastRead: 29 + sensor=PQ/TotalSizeLagByLastRead: 744 + sensor=PQ/TotalTimeLagMsByLastRead: 4929 + sensor=PQ/UserPartitionsAnswered: 2 + sensor=PQ/WriteTimeLagMsByLastRead: 29 + sensor=PQ/WriteTimeLagMsByLastReadOld: 4989 + + important=total: + + topic=total: + sensor=PQ/MessageLagByCommitted: 30 + sensor=PQ/MessageLagByLastRead: 29 + sensor=PQ/PartitionMaxReadQuotaUsage: 0 + sensor=PQ/ReadBytesAvailAvgMin: 1000000000 + sensor=PQ/ReadBytesAvailAvgSec: 1000000000 + sensor=PQ/ReadBytesMaxPerDay: 0 + sensor=PQ/ReadBytesMaxPerHour: 0 + sensor=PQ/ReadBytesMaxPerMin: 0 + sensor=PQ/ReadBytesMaxPerSec: 0 + sensor=PQ/ReadBytesPerDay: 0 + sensor=PQ/ReadBytesPerHour: 0 + sensor=PQ/ReadBytesPerMin: 0 + sensor=PQ/ReadBytesPerSec: 0 + sensor=PQ/ReadBytesQuota: 1000000000 + sensor=PQ/ReadOffsetRewindSum: 0 + sensor=PQ/ReadTimeLagMs: 0 + sensor=PQ/SizeLagByCommitted: 744 + sensor=PQ/SizeLagByLastRead: 744 + sensor=PQ/TimeSinceLastReadMs: 4989 + sensor=PQ/TotalMessageLagByLastRead: 29 + sensor=PQ/TotalSizeLagByLastRead: 744 + sensor=PQ/TotalTimeLagMsByLastRead: 4929 + sensor=PQ/UserPartitionsAnswered: 2 + sensor=PQ/WriteTimeLagMsByLastRead: 29 + sensor=PQ/WriteTimeLagMsByLastReadOld: 4989 + + topic=rt3.dc1--topic: + sensor=PQ/GapsCount: 0 + sensor=PQ/GapsMaxCount: 0 + sensor=PQ/GapsMaxSize: 0 + sensor=PQ/GapsSize: 0 + sensor=PQ/MaxPartSize: 744 + sensor=PQ/PartitionInitTimeMs: 0 + sensor=PQ/PartitionLifeTimeMs: 4989 + sensor=PQ/PartitionMaxWriteQuotaUsage: 0 + sensor=PQ/PartitionsAnswered: 2 + sensor=PQ/QuotaBytesMaxPerDay: 540 + sensor=PQ/QuotaBytesMaxPerHour: 540 + sensor=PQ/QuotaBytesMaxPerMin: 540 + sensor=PQ/QuotaBytesMaxPerSec: 540 + sensor=PQ/QuotaBytesPerDay: 540 + sensor=PQ/QuotaBytesPerHour: 540 + sensor=PQ/QuotaBytesPerMin: 540 + sensor=PQ/QuotaBytesPerSec: 540 + sensor=PQ/SourceIdCount: 3 + sensor=PQ/SourceIdMaxCount: 3 + sensor=PQ/SourceIdMinLifetimeMs: 0 + sensor=PQ/TotalPartSize: 744 + sensor=PQ/WriteBytesAvailAvgMin: 49999998 + sensor=PQ/WriteBytesAvailAvgSec: 50000000 + sensor=PQ/WriteBytesMaxPerDay: 540 + sensor=PQ/WriteBytesMaxPerHour: 540 + sensor=PQ/WriteBytesMaxPerMin: 540 + sensor=PQ/WriteBytesMaxPerSec: 540 + sensor=PQ/WriteBytesPerDay: 540 + sensor=PQ/WriteBytesPerHour: 540 + sensor=PQ/WriteBytesPerMin: 540 + sensor=PQ/WriteBytesPerSec: 540 + sensor=PQ/WriteBytesQuota: 50000000 + sensor=PQ/WriteTimeLagMsByLastWrite: 32 + + topic=total: + sensor=PQ/GapsCount: 0 + sensor=PQ/GapsMaxCount: 0 + sensor=PQ/GapsMaxSize: 0 + sensor=PQ/GapsSize: 0 + sensor=PQ/MaxPartSize: 744 + sensor=PQ/PartitionInitTimeMs: 0 + sensor=PQ/PartitionLifeTimeMs: 4989 + sensor=PQ/PartitionMaxWriteQuotaUsage: 0 + sensor=PQ/PartitionsAnswered: 2 + sensor=PQ/QuotaBytesMaxPerDay: 540 + sensor=PQ/QuotaBytesMaxPerHour: 540 + sensor=PQ/QuotaBytesMaxPerMin: 540 + sensor=PQ/QuotaBytesMaxPerSec: 540 + sensor=PQ/QuotaBytesPerDay: 540 + sensor=PQ/QuotaBytesPerHour: 540 + sensor=PQ/QuotaBytesPerMin: 540 + sensor=PQ/QuotaBytesPerSec: 540 + sensor=PQ/SourceIdCount: 3 + sensor=PQ/SourceIdMaxCount: 3 + sensor=PQ/SourceIdMinLifetimeMs: 0 + sensor=PQ/TotalPartSize: 744 + sensor=PQ/WriteBytesAvailAvgMin: 49999998 + sensor=PQ/WriteBytesAvailAvgSec: 50000000 + sensor=PQ/WriteBytesMaxPerDay: 540 + sensor=PQ/WriteBytesMaxPerHour: 540 + sensor=PQ/WriteBytesMaxPerMin: 540 + sensor=PQ/WriteBytesMaxPerSec: 540 + sensor=PQ/WriteBytesPerDay: 540 + sensor=PQ/WriteBytesPerHour: 540 + sensor=PQ/WriteBytesPerMin: 540 + sensor=PQ/WriteBytesPerSec: 540 + sensor=PQ/WriteBytesQuota: 50000000 + sensor=PQ/WriteTimeLagMsByLastWrite: 32 +</pre> diff --git a/ydb/core/persqueue/ut/counters_pqproxy.html b/ydb/core/persqueue/ut/counters_pqproxy.html new file mode 100644 index 00000000000..2f390ee8edc --- /dev/null +++ b/ydb/core/persqueue/ut/counters_pqproxy.html @@ -0,0 +1,654 @@ +<pre> +subsystem=SLI: + + Account=asdfgs: + sensor=WriteBigLatency: 0 + sensor=WritesTotal: 3 + + Account=total: + sensor=WriteBigLatency: 0 + sensor=WritesTotal: 3 + + sensor=Write: + + Account=asdfgs: + Duration=10000ms: 0 + Duration=1000ms: 0 + Duration=100ms: 3 + Duration=1500ms: 0 + Duration=2000ms: 0 + Duration=200ms: 0 + Duration=30000ms: 0 + Duration=5000ms: 0 + Duration=500ms: 0 + Duration=550ms: 0 + Duration=99999999ms: 0 + + Account=total: + Duration=10000ms: 0 + Duration=1000ms: 0 + Duration=100ms: 3 + Duration=1500ms: 0 + Duration=2000ms: 0 + Duration=200ms: 0 + Duration=30000ms: 0 + Duration=5000ms: 0 + Duration=500ms: 0 + Duration=550ms: 0 + Duration=99999999ms: 0 + +subsystem=partitionWriteQuotaWait: + + Account=asdfgs: + + Producer=asdfgs: + + Topic=asdfgs--topic: + + TopicPath=asdfgs/topic: + + OriginDC=Dc1: + + sensor=PartitionWriteQuotaWaitOriginal: + Interval=0ms: 30 + Interval=10000ms: 0 + Interval=1000ms: 0 + Interval=100ms: 0 + Interval=10ms: 0 + Interval=1ms: 0 + Interval=20ms: 0 + Interval=2500ms: 0 + Interval=5000ms: 0 + Interval=500ms: 0 + Interval=50ms: 0 + Interval=5ms: 0 + Interval=999999ms: 0 + + OriginDC=cluster: + + sensor=PartitionWriteQuotaWaitOriginal: + Interval=0ms: 30 + Interval=10000ms: 0 + Interval=1000ms: 0 + Interval=100ms: 0 + Interval=10ms: 0 + Interval=1ms: 0 + Interval=20ms: 0 + Interval=2500ms: 0 + Interval=5000ms: 0 + Interval=500ms: 0 + Interval=50ms: 0 + Interval=5ms: 0 + Interval=999999ms: 0 + + Topic=total: + + TopicPath=total: + + OriginDC=cluster: + + sensor=PartitionWriteQuotaWaitOriginal: + Interval=0ms: 30 + Interval=10000ms: 0 + Interval=1000ms: 0 + Interval=100ms: 0 + Interval=10ms: 0 + Interval=1ms: 0 + Interval=20ms: 0 + Interval=2500ms: 0 + Interval=5000ms: 0 + Interval=500ms: 0 + Interval=50ms: 0 + Interval=5ms: 0 + Interval=999999ms: 0 + + Producer=total: + + Topic=total: + + TopicPath=total: + + OriginDC=cluster: + + sensor=PartitionWriteQuotaWaitOriginal: + Interval=0ms: 30 + Interval=10000ms: 0 + Interval=1000ms: 0 + Interval=100ms: 0 + Interval=10ms: 0 + Interval=1ms: 0 + Interval=20ms: 0 + Interval=2500ms: 0 + Interval=5000ms: 0 + Interval=500ms: 0 + Interval=50ms: 0 + Interval=5ms: 0 + Interval=999999ms: 0 + + Account=total: + + Producer=total: + + Topic=total: + + TopicPath=total: + + OriginDC=cluster: + + sensor=PartitionWriteQuotaWaitOriginal: + Interval=0ms: 30 + Interval=10000ms: 0 + Interval=1000ms: 0 + Interval=100ms: 0 + Interval=10ms: 0 + Interval=1ms: 0 + Interval=20ms: 0 + Interval=2500ms: 0 + Interval=5000ms: 0 + Interval=500ms: 0 + Interval=50ms: 0 + Interval=5ms: 0 + Interval=999999ms: 0 + +subsystem=readSession: + + Account=asdfgs: + + Producer=asdfgs: + + Topic=asdfgs--topic: + + TopicPath=asdfgs/topic: + + OriginDC=Dc1: + + Client=user: + + ConsumerPath=shared/user: + sensor=BytesRead: 0 + sensor=MessagesRead: 0 + + OriginDC=cluster: + + Client=user: + + ConsumerPath=shared/user: + sensor=BytesRead: 0 + sensor=MessagesRead: 0 + + Topic=total: + + TopicPath=total: + + OriginDC=cluster: + + Client=user: + + ConsumerPath=shared/user: + sensor=BytesRead: 0 + sensor=MessagesRead: 0 + + Producer=total: + + Topic=total: + + TopicPath=total: + + OriginDC=cluster: + + Client=user: + + ConsumerPath=shared/user: + sensor=BytesRead: 0 + sensor=MessagesRead: 0 + + Account=total: + + Producer=total: + + Topic=total: + + TopicPath=total: + + OriginDC=cluster: + + Client=user: + + ConsumerPath=shared/user: + sensor=BytesRead: 0 + sensor=MessagesRead: 0 + +subsystem=readTimeLag: + + Account=asdfgs: + + Producer=asdfgs: + + Topic=asdfgs--topic: + + TopicPath=asdfgs/topic: + + OriginDC=Dc1: + + Client=user: + + ConsumerPath=shared/user: + + sensor=TimeLags: + Interval=10000ms: 0 + Interval=1000ms: 0 + Interval=100ms: 0 + Interval=180000ms: 0 + Interval=2000ms: 0 + Interval=200ms: 0 + Interval=30000ms: 0 + Interval=5000ms: 0 + Interval=500ms: 0 + Interval=60000ms: 0 + Interval=999999ms: 0 + + OriginDC=cluster: + + Client=user: + + ConsumerPath=shared/user: + + sensor=TimeLags: + Interval=10000ms: 0 + Interval=1000ms: 0 + Interval=100ms: 0 + Interval=180000ms: 0 + Interval=2000ms: 0 + Interval=200ms: 0 + Interval=30000ms: 0 + Interval=5000ms: 0 + Interval=500ms: 0 + Interval=60000ms: 0 + Interval=999999ms: 0 + + Topic=total: + + TopicPath=total: + + OriginDC=cluster: + + Client=user: + + ConsumerPath=shared/user: + + sensor=TimeLags: + Interval=10000ms: 0 + Interval=1000ms: 0 + Interval=100ms: 0 + Interval=180000ms: 0 + Interval=2000ms: 0 + Interval=200ms: 0 + Interval=30000ms: 0 + Interval=5000ms: 0 + Interval=500ms: 0 + Interval=60000ms: 0 + Interval=999999ms: 0 + + Producer=total: + + Topic=total: + + TopicPath=total: + + OriginDC=cluster: + + Client=user: + + ConsumerPath=shared/user: + + sensor=TimeLags: + Interval=10000ms: 0 + Interval=1000ms: 0 + Interval=100ms: 0 + Interval=180000ms: 0 + Interval=2000ms: 0 + Interval=200ms: 0 + Interval=30000ms: 0 + Interval=5000ms: 0 + Interval=500ms: 0 + Interval=60000ms: 0 + Interval=999999ms: 0 + + Account=total: + + Producer=total: + + Topic=total: + + TopicPath=total: + + OriginDC=cluster: + + Client=user: + + ConsumerPath=shared/user: + + sensor=TimeLags: + Interval=10000ms: 0 + Interval=1000ms: 0 + Interval=100ms: 0 + Interval=180000ms: 0 + Interval=2000ms: 0 + Interval=200ms: 0 + Interval=30000ms: 0 + Interval=5000ms: 0 + Interval=500ms: 0 + Interval=60000ms: 0 + Interval=999999ms: 0 + +subsystem=writeInfo: + + Account=asdfgs: + + Producer=asdfgs: + + Topic=asdfgs--topic: + + TopicPath=asdfgs/topic: + + OriginDC=Dc1: + + sensor=MessageSizeOriginal: + Size=100kb: 0 + Size=10240kb: 0 + Size=1024kb: 0 + Size=10kb: 0 + Size=1kb: 30 + Size=200kb: 0 + Size=2048kb: 0 + Size=20kb: 0 + Size=50kb: 0 + Size=5120kb: 0 + Size=512kb: 0 + Size=5kb: 0 + Size=65536kb: 0 + Size=99999999kb: 0 + + OriginDC=cluster: + + sensor=MessageSizeOriginal: + Size=100kb: 0 + Size=10240kb: 0 + Size=1024kb: 0 + Size=10kb: 0 + Size=1kb: 30 + Size=200kb: 0 + Size=2048kb: 0 + Size=20kb: 0 + Size=50kb: 0 + Size=5120kb: 0 + Size=512kb: 0 + Size=5kb: 0 + Size=65536kb: 0 + Size=99999999kb: 0 + + Topic=total: + + TopicPath=total: + + OriginDC=cluster: + + sensor=MessageSizeOriginal: + Size=100kb: 0 + Size=10240kb: 0 + Size=1024kb: 0 + Size=10kb: 0 + Size=1kb: 30 + Size=200kb: 0 + Size=2048kb: 0 + Size=20kb: 0 + Size=50kb: 0 + Size=5120kb: 0 + Size=512kb: 0 + Size=5kb: 0 + Size=65536kb: 0 + Size=99999999kb: 0 + + Producer=total: + + Topic=total: + + TopicPath=total: + + OriginDC=cluster: + + sensor=MessageSizeOriginal: + Size=100kb: 0 + Size=10240kb: 0 + Size=1024kb: 0 + Size=10kb: 0 + Size=1kb: 30 + Size=200kb: 0 + Size=2048kb: 0 + Size=20kb: 0 + Size=50kb: 0 + Size=5120kb: 0 + Size=512kb: 0 + Size=5kb: 0 + Size=65536kb: 0 + Size=99999999kb: 0 + + Account=total: + + Producer=total: + + Topic=total: + + TopicPath=total: + + OriginDC=cluster: + + sensor=MessageSizeOriginal: + Size=100kb: 0 + Size=10240kb: 0 + Size=1024kb: 0 + Size=10kb: 0 + Size=1kb: 30 + Size=200kb: 0 + Size=2048kb: 0 + Size=20kb: 0 + Size=50kb: 0 + Size=5120kb: 0 + Size=512kb: 0 + Size=5kb: 0 + Size=65536kb: 0 + Size=99999999kb: 0 + +subsystem=writeSession: + + Account=asdfgs: + + Producer=asdfgs: + + Topic=asdfgs--topic: + + TopicPath=asdfgs/topic: + + ClientDC=Unknown: + sensor=BytesWrittenFromDC: 1560 + + OriginDC=Dc1: + sensor=BytesWrittenOriginal: 540 + sensor=CompactedBytesWrittenOriginal: 744 + sensor=MessagesWrittenOriginal: 30 + sensor=UncompressedBytesWrittenOriginal: 270 + + OriginDC=cluster: + sensor=BytesWrittenOriginal: 540 + sensor=CompactedBytesWrittenOriginal: 744 + sensor=MessagesWrittenOriginal: 30 + sensor=UncompressedBytesWrittenOriginal: 270 + + Topic=total: + + TopicPath=total: + + ClientDC=Unknown: + sensor=BytesWrittenFromDC: 1560 + + OriginDC=cluster: + sensor=BytesWrittenOriginal: 540 + sensor=CompactedBytesWrittenOriginal: 744 + sensor=MessagesWrittenOriginal: 30 + sensor=UncompressedBytesWrittenOriginal: 270 + + Producer=total: + + Topic=total: + + TopicPath=total: + + ClientDC=Unknown: + sensor=BytesWrittenFromDC: 1560 + + OriginDC=cluster: + sensor=BytesWrittenOriginal: 540 + sensor=CompactedBytesWrittenOriginal: 744 + sensor=MessagesWrittenOriginal: 30 + sensor=UncompressedBytesWrittenOriginal: 270 + + Account=total: + + Producer=total: + + Topic=total: + + TopicPath=total: + + ClientDC=Unknown: + sensor=BytesWrittenFromDC: 1560 + + OriginDC=cluster: + sensor=BytesWrittenOriginal: 540 + sensor=CompactedBytesWrittenOriginal: 744 + sensor=MessagesWrittenOriginal: 30 + sensor=UncompressedBytesWrittenOriginal: 270 + +subsystem=writeTimeLag: + + Account=asdfgs: + + Producer=asdfgs: + + Topic=asdfgs--topic: + + TopicPath=asdfgs/topic: + + OriginDC=Dc1: + + sensor=TimeLagsOriginal: + Interval=10000ms: 0 + Interval=1000ms: 0 + Interval=100ms: 30 + Interval=180000ms: 0 + Interval=2000ms: 0 + Interval=200ms: 0 + Interval=30000ms: 0 + Interval=5000ms: 0 + Interval=500ms: 0 + Interval=60000ms: 0 + Interval=999999ms: 0 + + OriginDC=cluster: + + sensor=TimeLagsOriginal: + Interval=10000ms: 0 + Interval=1000ms: 0 + Interval=100ms: 30 + Interval=180000ms: 0 + Interval=2000ms: 0 + Interval=200ms: 0 + Interval=30000ms: 0 + Interval=5000ms: 0 + Interval=500ms: 0 + Interval=60000ms: 0 + Interval=999999ms: 0 + + Topic=total: + + TopicPath=total: + + OriginDC=cluster: + + sensor=TimeLagsOriginal: + Interval=10000ms: 0 + Interval=1000ms: 0 + Interval=100ms: 30 + Interval=180000ms: 0 + Interval=2000ms: 0 + Interval=200ms: 0 + Interval=30000ms: 0 + Interval=5000ms: 0 + Interval=500ms: 0 + Interval=60000ms: 0 + Interval=999999ms: 0 + + Producer=total: + + Topic=total: + + TopicPath=total: + + OriginDC=cluster: + + sensor=TimeLagsOriginal: + Interval=10000ms: 0 + Interval=1000ms: 0 + Interval=100ms: 30 + Interval=180000ms: 0 + Interval=2000ms: 0 + Interval=200ms: 0 + Interval=30000ms: 0 + Interval=5000ms: 0 + Interval=500ms: 0 + Interval=60000ms: 0 + Interval=999999ms: 0 + + Account=total: + + Producer=total: + + Topic=total: + + TopicPath=total: + + OriginDC=cluster: + + sensor=TimeLagsOriginal: + Interval=10000ms: 0 + Interval=1000ms: 0 + Interval=100ms: 30 + Interval=180000ms: 0 + Interval=2000ms: 0 + Interval=200ms: 0 + Interval=30000ms: 0 + Interval=5000ms: 0 + Interval=500ms: 0 + Interval=60000ms: 0 + Interval=999999ms: 0 + +subsystem=writingTime: + + OriginDC=Dc1: + + Producer=asdfgs: + + TopicPath=asdfgs/topic: + + Account=asdfgs: + + Topic=asdfgs--topic: + + host=1: + + Partition=0: + sensor=BufferFullTimeOriginal: 0 + + Partition=1: + sensor=BufferFullTimeOriginal: 0 +</pre> diff --git a/ydb/core/persqueue/ut/counters_pqproxy_firstclass.html b/ydb/core/persqueue/ut/counters_pqproxy_firstclass.html new file mode 100644 index 00000000000..6fbbfec01a6 --- /dev/null +++ b/ydb/core/persqueue/ut/counters_pqproxy_firstclass.html @@ -0,0 +1,39 @@ +<pre> +subsystem=SLI: + + Account=federationAccount: + name=WriteBigLatency: 0 + name=WritesTotal: 3 + + Account=total: + name=WriteBigLatency: 0 + name=WritesTotal: 3 + + sensor=Write: + + Account=federationAccount: + Duration=10000ms: 0 + Duration=1000ms: 0 + Duration=100ms: 3 + Duration=1500ms: 0 + Duration=2000ms: 0 + Duration=200ms: 0 + Duration=30000ms: 0 + Duration=5000ms: 0 + Duration=500ms: 0 + Duration=550ms: 0 + Duration=99999999ms: 0 + + Account=total: + Duration=10000ms: 0 + Duration=1000ms: 0 + Duration=100ms: 3 + Duration=1500ms: 0 + Duration=2000ms: 0 + Duration=200ms: 0 + Duration=30000ms: 0 + Duration=5000ms: 0 + Duration=500ms: 0 + Duration=550ms: 0 + Duration=99999999ms: 0 +</pre> |