aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorEvgeniy Ivanov <i@eivanov.com>2022-04-15 15:51:58 +0300
committerEvgeniy Ivanov <i@eivanov.com>2022-04-15 15:51:58 +0300
commit798d0697ce54b4160e1e0a164cc88b4124a7c843 (patch)
tree9517a0b31fa0cb21040ab726f1a755e0baad57bd
parentbb1523598e756290b6e6888596c13320857d5dab (diff)
downloadydb-798d0697ce54b4160e1e0a164cc88b4124a7c843.tar.gz
KIKIMR-14646: aggregate tablet histograms; avoid overflow in hist buckets
ref:e2fc7b63e94a6f21fc05b425213586bd6c4d0115
-rw-r--r--library/cpp/monlib/dynamic_counters/counters.h4
-rw-r--r--library/cpp/monlib/metrics/atomics_array.h2
-rw-r--r--library/cpp/monlib/metrics/histogram_collector.h2
-rw-r--r--library/cpp/monlib/metrics/histogram_collector_explicit.cpp2
-rw-r--r--library/cpp/monlib/metrics/histogram_collector_exponential.cpp2
-rw-r--r--library/cpp/monlib/metrics/histogram_collector_linear.cpp2
-rw-r--r--ydb/core/tablet/tablet_counters.h11
-rw-r--r--ydb/core/tablet/tablet_counters_aggregator.cpp427
-rw-r--r--ydb/core/tablet/tablet_counters_aggregator_ut.cpp512
-rw-r--r--ydb/core/tablet_flat/flat_executor_counters.h3
10 files changed, 818 insertions, 149 deletions
diff --git a/library/cpp/monlib/dynamic_counters/counters.h b/library/cpp/monlib/dynamic_counters/counters.h
index dc178cfbe0..3bf89c5098 100644
--- a/library/cpp/monlib/dynamic_counters/counters.h
+++ b/library/cpp/monlib/dynamic_counters/counters.h
@@ -141,11 +141,11 @@ namespace NMonitoring {
Collector_->Collect(value);
}
- void Collect(i64 value, ui32 count) {
+ void Collect(i64 value, ui64 count) {
Collector_->Collect(value, count);
}
- void Collect(double value, ui32 count) {
+ void Collect(double value, ui64 count) {
Collector_->Collect(value, count);
}
diff --git a/library/cpp/monlib/metrics/atomics_array.h b/library/cpp/monlib/metrics/atomics_array.h
index f19aebf291..4a26c8186a 100644
--- a/library/cpp/monlib/metrics/atomics_array.h
+++ b/library/cpp/monlib/metrics/atomics_array.h
@@ -26,7 +26,7 @@ namespace NMonitoring {
return Size_;
}
- void Add(size_t index, ui32 count) noexcept {
+ void Add(size_t index, ui64 count) noexcept {
Y_VERIFY_DEBUG(index < Size_);
Values_[index].fetch_add(count, std::memory_order_relaxed);
}
diff --git a/library/cpp/monlib/metrics/histogram_collector.h b/library/cpp/monlib/metrics/histogram_collector.h
index 9f6bbbdfb7..177d779634 100644
--- a/library/cpp/monlib/metrics/histogram_collector.h
+++ b/library/cpp/monlib/metrics/histogram_collector.h
@@ -14,7 +14,7 @@ namespace NMonitoring {
/**
* Store {@code count} times given {@code value} in this collector.
*/
- virtual void Collect(double value, ui32 count) = 0;
+ virtual void Collect(double value, ui64 count) = 0;
/**
* Store given {@code value} in this collector.
diff --git a/library/cpp/monlib/metrics/histogram_collector_explicit.cpp b/library/cpp/monlib/metrics/histogram_collector_explicit.cpp
index 377fc233ef..11368525cb 100644
--- a/library/cpp/monlib/metrics/histogram_collector_explicit.cpp
+++ b/library/cpp/monlib/metrics/histogram_collector_explicit.cpp
@@ -21,7 +21,7 @@ namespace NMonitoring {
Bounds_.push_back(Max<TBucketBound>());
}
- void Collect(double value, ui32 count) override {
+ void Collect(double value, ui64 count) override {
auto it = LowerBound(Bounds_.begin(), Bounds_.end(), value);
auto index = std::distance(Bounds_.begin(), it);
Values_.Add(index, count);
diff --git a/library/cpp/monlib/metrics/histogram_collector_exponential.cpp b/library/cpp/monlib/metrics/histogram_collector_exponential.cpp
index 2f8a50a5f9..c6bbfcfc69 100644
--- a/library/cpp/monlib/metrics/histogram_collector_exponential.cpp
+++ b/library/cpp/monlib/metrics/histogram_collector_exponential.cpp
@@ -22,7 +22,7 @@ namespace NMonitoring {
{
}
- void Collect(double value, ui32 count) override {
+ void Collect(double value, ui64 count) override {
ui32 index = Max<ui32>();
if (value <= MinValue_) {
index = 0;
diff --git a/library/cpp/monlib/metrics/histogram_collector_linear.cpp b/library/cpp/monlib/metrics/histogram_collector_linear.cpp
index f8ad86f3a4..8342485320 100644
--- a/library/cpp/monlib/metrics/histogram_collector_linear.cpp
+++ b/library/cpp/monlib/metrics/histogram_collector_linear.cpp
@@ -23,7 +23,7 @@ namespace NMonitoring {
{
}
- void Collect(double value, ui32 count) override {
+ void Collect(double value, ui64 count) override {
ui32 index = Max<ui32>();
if (value <= StartValue_) {
index = 0;
diff --git a/ydb/core/tablet/tablet_counters.h b/ydb/core/tablet/tablet_counters.h
index a8cc27e9ed..22dcc330e0 100644
--- a/ydb/core/tablet/tablet_counters.h
+++ b/ydb/core/tablet/tablet_counters.h
@@ -155,6 +155,17 @@ public:
Initialize(rangeCount, ranges, integral);
}
+ void Initialize(const TRangeDef* ranges, size_t rangeCount, bool integral) {
+ Initialize(rangeCount, ranges, integral);
+ }
+
+ // mainly for use in tests
+ TTabletPercentileCounter& AddFor(ui64 what, ui64 value) {
+ ui32 index = FindSlot(what);
+ Values[index] += value;
+ return *this;
+ }
+
TTabletPercentileCounter& IncrementFor(ui64 what) {
ui32 index = FindSlot(what);
Values[index] += 1;
diff --git a/ydb/core/tablet/tablet_counters_aggregator.cpp b/ydb/core/tablet/tablet_counters_aggregator.cpp
index b1bf49382c..d90b1771d8 100644
--- a/ydb/core/tablet/tablet_counters_aggregator.cpp
+++ b/ydb/core/tablet/tablet_counters_aggregator.cpp
@@ -63,15 +63,15 @@ using TCountersVector = TVector<NMonitoring::TDynamicCounters::TCounterPtr>;
struct THistogramCounter {
TVector<TTabletPercentileCounter::TRangeDef> Ranges;
- TVector<NMonitoring::TDynamicCounters::TCounterPtr> Values;
+ TCountersVector Values;
NMonitoring::THistogramPtr Histogram;
THistogramCounter(
const TVector<TTabletPercentileCounter::TRangeDef>& ranges,
- TVector<NMonitoring::TDynamicCounters::TCounterPtr>&& values,
+ TCountersVector&& values,
NMonitoring::THistogramPtr histogram)
: Ranges(ranges)
- , Values(values)
+ , Values(std::move(values))
, Histogram(histogram)
{
Y_VERIFY(!Ranges.empty() && Ranges.size() == Values.size());
@@ -102,8 +102,14 @@ public:
: CounterGroup(counterGroup)
{}
+ void Reserve(size_t hint) {
+ CountersByTabletID.reserve(hint);
+ ChangedCounters.reserve(hint);
+ MaxSimpleCounters.reserve(hint);
+ }
+
void AddSimpleCounter(const char* name, THolder<THistogramCounter> percentileAggregate = THolder<THistogramCounter>()) {
- auto fnAddCounter = [this](const char* name, TVector<NMonitoring::TDynamicCounters::TCounterPtr>& container) {
+ auto fnAddCounter = [this](const char* name, TCountersVector& container) {
auto counter = CounterGroup->GetCounter(name, false);
container.push_back(counter);
};
@@ -220,8 +226,14 @@ public:
: CounterGroup(counterGroup)
{}
+ void Reserve(size_t hint) {
+ CountersByTabletID.reserve(hint);
+ ChangedCounters.reserve(hint);
+ MaxCumulativeCounters.reserve(hint);
+ }
+
void AddCumulativeCounter(const char* name, THolder<THistogramCounter> percentileAggregate = THolder<THistogramCounter>()) {
- auto fnAddCounter = [this](const char* name, TVector<NMonitoring::TDynamicCounters::TCounterPtr>& container) {
+ auto fnAddCounter = [this](const char* name, TCountersVector& container) {
auto counter = CounterGroup->GetCounter(name, false);
container.push_back(counter);
};
@@ -312,6 +324,234 @@ private:
}
};
+class TAggregatedHistogramCounters {
+public:
+ //
+
+ TAggregatedHistogramCounters(NMonitoring::TDynamicCounterPtr counterGroup)
+ : CounterGroup(counterGroup)
+ {}
+
+ void Reserve(size_t hint) {
+ PercentileCounters.reserve(hint);
+ Histograms.reserve(hint);
+ IsDerivative.reserve(hint);
+ ShiftedBucketBounds.reserve(hint);
+ CountersByTabletID.reserve(hint);
+ }
+
+ void AddCounter(
+ const char* name,
+ const NKikimr::TTabletPercentileCounter& percentileCounter,
+ THashMap<TString, THolder<THistogramCounter>>& histogramAggregates)
+ {
+ // old style
+ PercentileCounters.push_back(TCountersVector());
+ auto& rangeCounters = PercentileCounters.back();
+
+ TStringBuf counterName(name);
+ TStringBuf simpleCounterName = GetHistogramAggregateSimpleName(counterName);
+ bool histogramAggregate = !simpleCounterName.empty();
+ bool isDerivative = !histogramAggregate && !percentileCounter.GetIntegral();
+ IsDerivative.push_back(isDerivative);
+
+ auto rangeCount = percentileCounter.GetRangeCount();
+ Y_VERIFY_DEBUG(rangeCount > 0);
+
+ for (ui32 r = 0; r < rangeCount; ++r) {
+ const char* rangeName = percentileCounter.GetRangeName(r);
+ auto subgroup = CounterGroup->GetSubgroup("range", rangeName);
+ auto counter = subgroup->GetCounter(name, isDerivative);
+ rangeCounters.push_back(counter);
+ }
+
+ // Note that:
+ // 1. PercentileCounters always start from 0 range
+ // 2. Ranges in PercentileCounters are left inclusive, i.e. for ranges 0, 1, 2 buckets will be
+ // [0; 1), [1; 2), [2; +inf);
+ // 3. In monitoring's histogram buckets are right inclusive and can be negative, i.e. for ranges 0, 1, 2
+ // buckets will be: (-inf; 0], (0; 1], (1; 2], (2; +inf).
+ // 4. Currently we shift PercentileCounters ranges so that original ranges 0, 1, 2 become 1, 2:
+ // (-inf; 1], (1; 2], (2; +inf). This is because values in proto are lower bounds
+
+ // new style
+ NMonitoring::TBucketBounds bucketBounds;
+ for (ui32 r = 1; r < rangeCount; ++r) {
+ bucketBounds.push_back(percentileCounter.GetRangeBound(r));
+ }
+
+ // since we shift we need hack for hists with single bucket (though they are meaningless anyway),
+ // hist will be (-inf; range0], (range0; +inf).
+ if (bucketBounds.empty()) {
+ bucketBounds.push_back(percentileCounter.GetRangeBound(0));
+ }
+
+ auto histogram = CounterGroup->GetHistogram(
+ name, NMonitoring::ExplicitHistogram(bucketBounds), isDerivative);
+
+ if (histogramAggregate) {
+ // either simple or cumulative aggregate will handle this histogram,
+ // it is a special case for hists name HIST(name), which have corresponding
+ // simple or cumulative counter updated by tablet (tablet doesn't update hist itself,
+ // hist is updated here by aggregated values)
+ histogramAggregates.emplace(simpleCounterName, new THistogramCounter(
+ percentileCounter.GetRanges(), std::move(rangeCounters), histogram));
+
+ // we need this hack to access PercentileCounters by index easily skipping
+ // hists we moved to simple/cumulative aggregates
+ TCountersVector().swap(rangeCounters);
+ ShiftedBucketBounds.emplace_back();
+ } else {
+ // note that this bound in histogram is implicit
+ bucketBounds.push_back(Max<NMonitoring::TBucketBound>());
+ ShiftedBucketBounds.emplace_back(std::move(bucketBounds));
+ }
+
+ // note that in case of histogramAggregate it will contain reference
+ // on the histogram updated outside
+ Histograms.push_back(histogram);
+
+ CountersByTabletID.emplace_back(TCountersByTabletIDMap());
+ }
+
+ void SetValue(
+ ui64 tabletID,
+ ui32 counterIndex,
+ const NKikimr::TTabletPercentileCounter& percentileCounter,
+ const char* name,
+ TTabletTypes::EType tabletType)
+ {
+ Y_VERIFY(counterIndex < CountersByTabletID.size(),
+ "inconsistent counters for tablet type %s, counter %s",
+ TTabletTypes::TypeToStr(tabletType),
+ name);
+
+ Y_VERIFY(counterIndex < PercentileCounters.size(),
+ "inconsistent counters for tablet type %s, counter %s",
+ TTabletTypes::TypeToStr(tabletType),
+ name);
+
+ auto& percentileRanges = PercentileCounters[counterIndex];
+
+ // see comment in AddCounter() related to histogramAggregate
+ if (percentileRanges.empty())
+ return;
+
+ // just sanity check, normally should not happen
+ const auto rangeCount = percentileCounter.GetRangeCount();
+ if (rangeCount == 0)
+ return;
+
+ Y_VERIFY(rangeCount <= percentileRanges.size(),
+ "inconsistent counters for tablet type %s, counter %s",
+ TTabletTypes::TypeToStr(tabletType),
+ name);
+
+ if (IsDerivative[counterIndex]) {
+ AddValues(counterIndex, percentileCounter);
+ return;
+ }
+
+ // integral histogram
+
+ TValuesVec newValues;
+ newValues.reserve(rangeCount);
+ for (auto i: xrange(rangeCount))
+ newValues.push_back(percentileCounter.GetRangeValue(i));
+
+ TCountersByTabletIDMap::insert_ctx insertCtx;
+ auto it = CountersByTabletID[counterIndex].find(tabletID, insertCtx);
+ if (it != CountersByTabletID[counterIndex].end()) {
+ const auto& oldValues = it->second;
+ if (newValues != oldValues) {
+ SubValues(counterIndex, oldValues);
+ AddValues(counterIndex, newValues);
+ }
+ } else {
+ AddValues(counterIndex, newValues);
+ CountersByTabletID[counterIndex].insert_direct(std::make_pair(tabletID, std::move(newValues)), insertCtx);
+ }
+ }
+
+ void ForgetTablet(ui64 tabletId) {
+ for (auto idx : xrange(CountersByTabletID.size())) {
+ auto &tabletToCounters = CountersByTabletID[idx];
+ auto it = tabletToCounters.find(tabletId);
+ if (it == tabletToCounters.end())
+ continue;
+
+ auto values = std::move(it->second);
+ tabletToCounters.erase(it);
+
+ if (IsDerivative[idx])
+ continue;
+
+ SubValues(idx, values);
+ }
+ }
+
+ NMonitoring::THistogramPtr GetHistogram(size_t i) {
+ Y_VERIFY(i < Histograms.size());
+ return Histograms[i];
+ }
+
+private:
+ using TValuesVec = TVector<ui64>;
+
+ void SubValues(size_t counterIndex, const TValuesVec& values) {
+ auto& percentileRanges = PercentileCounters[counterIndex];
+ auto& histogram = Histograms[counterIndex];
+ auto snapshot = histogram->Snapshot();
+ histogram->Reset();
+ for (auto i: xrange(values.size())) {
+ Y_VERIFY_DEBUG(static_cast<ui64>(*percentileRanges[i]) >= values[i]);
+ *percentileRanges[i] -= values[i];
+
+ ui64 oldValue = snapshot->Value(i);
+ ui64 negValue = 0UL - values[i];
+ ui64 newValue = oldValue + negValue;
+ histogram->Collect(ShiftedBucketBounds[counterIndex][i], newValue);
+ }
+ }
+
+ void AddValues(size_t counterIndex, const TValuesVec& values) {
+ auto& percentileRanges = PercentileCounters[counterIndex];
+ auto& histogram = Histograms[counterIndex];
+ for (auto i: xrange(values.size())) {
+ *percentileRanges[i] += values[i];
+ histogram->Collect(ShiftedBucketBounds[counterIndex][i], values[i]);
+ }
+ }
+
+ void AddValues(size_t counterIndex, const NKikimr::TTabletPercentileCounter& percentileCounter) {
+ auto& percentileRanges = PercentileCounters[counterIndex];
+ auto& histogram = Histograms[counterIndex];
+ for (auto i: xrange(percentileCounter.GetRangeCount())) {
+ auto value = percentileCounter.GetRangeValue(i);
+ *percentileRanges[i] += value;
+ histogram->Collect(ShiftedBucketBounds[counterIndex][i], value);
+ }
+ }
+
+private:
+ NMonitoring::TDynamicCounterPtr CounterGroup;
+
+ // monitoring counters holders, updated only during recalculation
+ TVector<TCountersVector> PercentileCounters; // old style (ranges)
+ TVector<NMonitoring::THistogramPtr> Histograms; // new style (bins)
+ TVector<bool> IsDerivative;
+
+ // per percentile counter bounds. Note the shift: index0 is range1,
+ // hence array size is 1 less than original ranges count
+ TVector<NMonitoring::TBucketBounds> ShiftedBucketBounds;
+
+ // tabletId -> values
+ using TCountersByTabletIDMap = THashMap<ui64, TValuesVec>;
+
+ // counter values (not "real" monitoring counters)
+ TVector<TCountersByTabletIDMap> CountersByTabletID; // each index is map from tablet to counter value
+};
+
struct TTabletLabeledCountersResponseContext {
NKikimrTabletCountersAggregator::TEvTabletLabeledCountersResponse& Response;
THashMap<TStringBuf, ui32> NamesToId;
@@ -474,7 +714,7 @@ public:
//
TTabletMon(NMonitoring::TDynamicCounterPtr counters, bool isFollower, TActorId dbWatcherActorId)
: Counters(GetServiceCounters(counters, isFollower ? "followers" : "tablets"))
- , AllTypes(Counters.Get(), "type", "all", true)
+ , AllTypes(Counters.Get(), "type", "all")
, IsFollower(isFollower)
, DbWatcherActorId(dbWatcherActorId)
{
@@ -705,12 +945,12 @@ private:
class TTabletCountersForTabletType {
public:
//
- TTabletCountersForTabletType(NMonitoring::TDynamicCounters* owner, const char* category, const char* name, bool doAggregateSimpleCountrers)
+ TTabletCountersForTabletType(NMonitoring::TDynamicCounters* owner, const char* category, const char* name)
: TabletCountersSection(owner->GetSubgroup(category, name))
, TabletExecutorCountersSection(TabletCountersSection->GetSubgroup("category", "executor"))
, TabletAppCountersSection(TabletCountersSection->GetSubgroup("category", "app"))
- , TabletExecutorCounters(TabletExecutorCountersSection, doAggregateSimpleCountrers)
- , TabletAppCounters(TabletAppCountersSection, doAggregateSimpleCountrers)
+ , TabletExecutorCounters(TabletExecutorCountersSection)
+ , TabletAppCounters(TabletAppCountersSection)
{}
void Apply(ui64 tabletID,
@@ -803,12 +1043,11 @@ private:
//
bool IsInitialized;
- TSolomonCounters(NMonitoring::TDynamicCounterPtr counterGroup, bool doAggregateCounters)
+ TSolomonCounters(NMonitoring::TDynamicCounterPtr counterGroup)
: IsInitialized(false)
- , DoAggregateSimpleCounters(doAggregateCounters)
, AggregatedSimpleCounters(counterGroup)
- , DoAggregateCumulativeCounters(doAggregateCounters)
, AggregatedCumulativeCounters(counterGroup)
+ , AggregatedHistogramCounters(counterGroup)
, CounterGroup(counterGroup)
{}
@@ -820,83 +1059,52 @@ private:
// percentile counters
FullSizePercentile = counters->Percentile().Size();
+ AggregatedHistogramCounters.Reserve(FullSizePercentile);
for (ui32 i = 0; i < FullSizePercentile; ++i) {
if (!counters->PercentileCounterName(i)) {
DeprecatedPercentile.insert(i);
continue;
}
- // old style
- PercentileCounters.push_back(TVector<NMonitoring::TDynamicCounters::TCounterPtr>());
- auto counterRBeginIter = PercentileCounters.rbegin();
-
auto& percentileCounter = counters->Percentile()[i];
const char* percentileCounterName = counters->PercentileCounterName(i);
- TStringBuf counterName(percentileCounterName);
- TStringBuf simpleCounterName = GetHistogramAggregateSimpleName(counterName);
- bool histogramAggregate = !simpleCounterName.empty();
-
- bool isDerivative = !histogramAggregate && !percentileCounter.GetIntegral();
-
- auto rangeCount = percentileCounter.GetRangeCount();
- for (ui32 r = 0; r < rangeCount; ++r) {
- const char* rangeName = percentileCounter.GetRangeName(r);
- auto subgroup = CounterGroup->GetSubgroup("range", rangeName);
- auto counter = subgroup->GetCounter(percentileCounterName, isDerivative);
- counterRBeginIter->push_back(counter);
- }
-
- // new style
- NMonitoring::TBucketBounds bucketBounds;
- for (ui32 r = 1; r < rangeCount; ++r) { // values in proto are lower bounds, thus shift
- bucketBounds.push_back(percentileCounter.GetRangeBound(r));
- }
- auto histogram = CounterGroup->GetHistogram(
- percentileCounterName, NMonitoring::ExplicitHistogram(bucketBounds), isDerivative);
- Histograms.push_back(histogram);
-
- if (histogramAggregate) {
- histogramAggregates.emplace(simpleCounterName, new THistogramCounter(
- percentileCounter.GetRanges(), std::move(*counterRBeginIter), histogram));
- }
+ AggregatedHistogramCounters.AddCounter(
+ percentileCounterName,
+ percentileCounter,
+ histogramAggregates);
}
// simple counters
FullSizeSimple = counters->Simple().Size();
+ AggregatedSimpleCounters.Reserve(FullSizeSimple);
for (ui32 i = 0; i < FullSizeSimple; ++i) {
const char* name = counters->SimpleCounterName(i);
if (!name) {
DeprecatedSimple.insert(i);
continue;
}
- if (DoAggregateSimpleCounters) {
- auto itHistogramAggregate = histogramAggregates.find(name);
- if (itHistogramAggregate != histogramAggregates.end()) {
- AggregatedSimpleCounters.AddSimpleCounter(name, std::move(itHistogramAggregate->second));
- } else {
- AggregatedSimpleCounters.AddSimpleCounter(name);
- }
+ auto itHistogramAggregate = histogramAggregates.find(name);
+ if (itHistogramAggregate != histogramAggregates.end()) {
+ AggregatedSimpleCounters.AddSimpleCounter(name, std::move(itHistogramAggregate->second));
} else {
- auto counter = CounterGroup->GetCounter(name, false);
- SimpleCounters.push_back(counter);
+ AggregatedSimpleCounters.AddSimpleCounter(name);
}
}
// cumulative counters
FullSizeCumulative = counters->Cumulative().Size();
+ AggregatedCumulativeCounters.Reserve(FullSizeSimple);
for (ui32 i = 0; i < FullSizeCumulative; ++i) {
const char* name = counters->CumulativeCounterName(i);
if (!name) {
DeprecatedCumulative.insert(i);
continue;
}
- if (DoAggregateCumulativeCounters) {
- auto itHistogramAggregate = histogramAggregates.find(name);
- if (itHistogramAggregate != histogramAggregates.end()) {
- AggregatedCumulativeCounters.AddCumulativeCounter(name, std::move(itHistogramAggregate->second));
- } else {
- AggregatedCumulativeCounters.AddCumulativeCounter(name);
- }
+ auto itHistogramAggregate = histogramAggregates.find(name);
+ if (itHistogramAggregate != histogramAggregates.end()) {
+ AggregatedCumulativeCounters.AddCumulativeCounter(name, std::move(itHistogramAggregate->second));
+ } else {
+ AggregatedCumulativeCounters.AddCumulativeCounter(name);
}
auto counter = CounterGroup->GetCounter(name, true);
CumulativeCounters.push_back(counter);
@@ -928,12 +1136,7 @@ private:
}
const ui32 offset = nextSimpleOffset++;
const ui64 value = counters->Simple()[i].Get();
- if (DoAggregateSimpleCounters) {
- AggregatedSimpleCounters.SetValue(tabletID, offset, value, tabletType);
- } else {
- Y_VERIFY(offset < SimpleCounters.size(), "inconsistent counters for tablet type %s", TTabletTypes::TypeToStr(tabletType));
- *SimpleCounters[offset] = value;
- }
+ AggregatedSimpleCounters.SetValue(tabletID, offset, value, tabletType);
}
// cumulative counters
@@ -944,11 +1147,9 @@ private:
}
const ui32 offset = nextCumulativeOffset++;
const ui64 valueDiff = counters->Cumulative()[i].Get();
- if (DoAggregateCumulativeCounters) {
- if (diff) {
- const ui64 diffValue = valueDiff * 1000000 / diff.MicroSeconds(); // differentiate value to per second rate
- AggregatedCumulativeCounters.SetValue(tabletID, offset, diffValue, tabletType);
- }
+ if (diff) {
+ const ui64 diffValue = valueDiff * 1000000 / diff.MicroSeconds(); // differentiate value to per second rate
+ AggregatedCumulativeCounters.SetValue(tabletID, offset, diffValue, tabletType);
}
Y_VERIFY(offset < CumulativeCounters.size(), "inconsistent counters for tablet type %s", TTabletTypes::TypeToStr(tabletType));
*CumulativeCounters[offset] += valueDiff;
@@ -962,78 +1163,33 @@ private:
}
const ui32 offset = nextPercentileOffset++;
- Y_VERIFY(offset < PercentileCounters.size(), "inconsistent counters for tablet type %s", TTabletTypes::TypeToStr(tabletType));
-
- auto &pcx = PercentileCounters[offset];
- if (pcx.empty()) {
- continue;
- }
-
- auto&& percentileCounter = counters->Percentile()[i];
- auto rangeCount = percentileCounter.GetRangeCount();
- Y_VERIFY(rangeCount <= pcx.size(),
- "inconsistent counters for tablet type %s", TTabletTypes::TypeToStr(tabletType));
-
- for (ui32 r = 0; r < rangeCount; ++r) {
- if (percentileCounter.GetIntegral()) {
- *pcx[r] = percentileCounter.GetRangeValue(r);
- } else {
- *pcx[r] += percentileCounter.GetRangeValue(r);
- }
- }
-
- if (rangeCount < 2) {
- continue;
- }
-
- auto& histogram = Histograms[offset];
- if (percentileCounter.GetIntegral()) {
- histogram->Reset();
- }
- for (ui32 r = 0; r < rangeCount - 1; ++r) {
- histogram->Collect(
- (NMonitoring::TBucketBound)percentileCounter.GetRangeBound(r + 1),
- percentileCounter.GetRangeValue(r));
- }
- histogram->Collect(
- Max<NMonitoring::TBucketBound>(),
- percentileCounter.GetRangeValue(rangeCount - 1));
+ AggregatedHistogramCounters.SetValue(
+ tabletID,
+ offset,
+ counters->Percentile()[i],
+ counters->PercentileCounterName(i),
+ tabletType);
}
}
void Forget(ui64 tabletId) {
Y_VERIFY(IsInitialized);
- if (DoAggregateSimpleCounters || DoAggregateCumulativeCounters) {
- if (DoAggregateSimpleCounters) {
- AggregatedSimpleCounters.ForgetTablet(tabletId);
- }
- if (DoAggregateCumulativeCounters) {
- AggregatedCumulativeCounters.ForgetTablet(tabletId);
- LastAggregateUpdateTime.erase(tabletId);
- }
- } else {
- for (auto &x : SimpleCounters)
- x = 0;
- }
+ AggregatedSimpleCounters.ForgetTablet(tabletId);
+ AggregatedCumulativeCounters.ForgetTablet(tabletId);
+ AggregatedHistogramCounters.ForgetTablet(tabletId);
+ LastAggregateUpdateTime.erase(tabletId);
}
void RecalcAll() {
- if (DoAggregateSimpleCounters) {
- AggregatedSimpleCounters.RecalcAll();
- }
- if (DoAggregateCumulativeCounters) {
- AggregatedCumulativeCounters.RecalcAll();
- }
+ AggregatedSimpleCounters.RecalcAll();
+ AggregatedCumulativeCounters.RecalcAll();
}
template <bool IsSaving>
void Convert(NKikimrSysView::TDbCounters& sumCounters,
NKikimrSysView::TDbCounters& maxCounters)
{
- if (!DoAggregateSimpleCounters || !DoAggregateCumulativeCounters) {
- return;
- }
// simple counters
auto* simpleSum = sumCounters.MutableSimple();
auto* simpleMax = maxCounters.MutableSimple();
@@ -1097,7 +1253,7 @@ private:
}
auto* buckets = (*histogramSum)[i].MutableBuckets();
const ui32 offset = nextPercentileOffset++;
- auto& histogram = Histograms[offset];
+ auto histogram = AggregatedHistogramCounters.GetHistogram(offset);
auto snapshot = histogram->Snapshot();
auto count = snapshot->Count();
buckets->Resize(count, 0);
@@ -1134,17 +1290,14 @@ private:
ui32 FullSizePercentile = 0;
THashSet<ui32> DeprecatedPercentile;
//
- bool DoAggregateSimpleCounters;
- TCountersVector SimpleCounters;
TAggregatedSimpleCounters AggregatedSimpleCounters;
- bool DoAggregateCumulativeCounters;
TCountersVector CumulativeCounters;
TAggregatedCumulativeCounters AggregatedCumulativeCounters;
- THashMap<ui64, TInstant> LastAggregateUpdateTime;
- TVector<TCountersVector> PercentileCounters; // old style
- TVector<NMonitoring::THistogramPtr> Histograms; // new style
+ TAggregatedHistogramCounters AggregatedHistogramCounters;
+
+ THashMap<ui64, TInstant> LastAggregateUpdateTime;
NMonitoring::TDynamicCounterPtr CounterGroup;
};
@@ -1181,7 +1334,7 @@ private:
if (!typeCounters) {
TString tabletTypeStr = TTabletTypes::TypeToStr(tabletType);
typeCounters = new TTabletCountersForTabletType(
- counters.Get(), "type", tabletTypeStr.data(), true);
+ counters.Get(), "type", tabletTypeStr.data());
countersByTabletType.emplace(tabletType, typeCounters);
}
return typeCounters;
@@ -1412,7 +1565,7 @@ public:
public:
TTabletCountersForDb()
: SolomonCounters(new NMonitoring::TDynamicCounters)
- , AllTypes(SolomonCounters.Get(), "type", "all", true)
+ , AllTypes(SolomonCounters.Get(), "type", "all")
{}
TTabletCountersForDb(NMonitoring::TDynamicCounterPtr externalGroup,
@@ -1420,7 +1573,7 @@ public:
THolder<TTabletCountersBase> executorCounters)
: SolomonCounters(internalGroup->GetSubgroup("group", "tablets"))
, ExecutorCounters(std::move(executorCounters))
- , AllTypes(SolomonCounters.Get(), "type", "all", true)
+ , AllTypes(SolomonCounters.Get(), "type", "all")
{
YdbCounters = MakeIntrusive<TYdbTabletCounters>(externalGroup);
}
diff --git a/ydb/core/tablet/tablet_counters_aggregator_ut.cpp b/ydb/core/tablet/tablet_counters_aggregator_ut.cpp
index 707ffb977e..69d80f7572 100644
--- a/ydb/core/tablet/tablet_counters_aggregator_ut.cpp
+++ b/ydb/core/tablet/tablet_counters_aggregator_ut.cpp
@@ -1,8 +1,10 @@
-#include <library/cpp/testing/unittest/registar.h>
-#include <library/cpp/actors/core/interconnect.h>
+#include "tablet_counters_aggregator.h"
+
#include <ydb/core/testlib/basics/runtime.h>
#include <ydb/core/testlib/basics/appdata.h>
-#include "tablet_counters_aggregator.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+#include <library/cpp/actors/core/interconnect.h>
namespace NKikimr {
@@ -84,8 +86,509 @@ void TestHeavy(const ui32 v, ui32 numWorkers) {
Cerr << "TEST " << v << " " << numWorkers << " duration " << TInstant::Now() - t << "\n";
}
-
Y_UNIT_TEST_SUITE(TTabletCountersAggregator) {
+
+ struct TTabletWithHist {
+ TTabletWithHist(ui64 tabletId)
+ : TabletId(tabletId)
+ , TenantPathId(1113, 1001)
+ , CounterEventsInFlight(new TEvTabletCounters::TInFlightCookie)
+ , ExecutorCounters(new TTabletCountersBase)
+ {
+ auto simpleCount = sizeof(SimpleCountersMetaInfo) / sizeof(SimpleCountersMetaInfo[0]);
+ auto percentileCount = sizeof(PercentileCountersMetaInfo) / sizeof(PercentileCountersMetaInfo[0]);
+ AppCounters.reset(new TTabletCountersBase(
+ simpleCount,
+ 0, // cumulativeCnt
+ percentileCount,
+ SimpleCountersMetaInfo,
+ nullptr, // cumulative meta
+ PercentileCountersMetaInfo));
+
+ for (auto i: xrange(percentileCount))
+ AppCounters->Percentile()[i].Initialize(RangeDefs[i].first, RangeDefs[i].second, true);
+
+ AppCountersBaseline.reset(new TTabletCountersBase());
+ AppCounters->RememberCurrentStateAsBaseline(*AppCountersBaseline);
+
+ ExecutorCountersBaseline.reset(new TTabletCountersBase());
+ ExecutorCounters->RememberCurrentStateAsBaseline(*ExecutorCountersBaseline);
+ }
+
+ void SendUpdate(TTestBasicRuntime& runtime, const TActorId& aggregatorId, const TActorId& sender) {
+ auto executorCounters = ExecutorCounters->MakeDiffForAggr(*ExecutorCountersBaseline);
+ ExecutorCounters->RememberCurrentStateAsBaseline(*ExecutorCountersBaseline);
+
+ auto appCounters = AppCounters->MakeDiffForAggr(*AppCountersBaseline);
+ AppCounters->RememberCurrentStateAsBaseline(*AppCountersBaseline);
+
+ runtime.Send(new IEventHandle(aggregatorId, sender, new TEvTabletCounters::TEvTabletAddCounters(
+ CounterEventsInFlight, TabletId, TabletType, TenantPathId, executorCounters, appCounters)));
+
+ // force recalc
+ runtime.Send(new IEventHandle(aggregatorId, sender, new NActors::TEvents::TEvWakeup()));
+ }
+
+ void ForgetTablet(TTestBasicRuntime& runtime, const TActorId& aggregatorId, const TActorId& sender) {
+ runtime.Send(new IEventHandle(
+ aggregatorId,
+ sender,
+ new TEvTabletCounters::TEvTabletCountersForgetTablet(TabletId, TabletType, TenantPathId)));
+
+ // force recalc
+ runtime.Send(new IEventHandle(aggregatorId, sender, new NActors::TEvents::TEvWakeup()));
+ }
+
+ void SetSimpleCount(const char* name, ui64 count) {
+ size_t index = SimpleNameToIndex(name);
+ AppCounters->Simple()[index].Set(count);
+ }
+
+ void UpdatePercentile(const char* name, ui64 what) {
+ size_t index = PercentileNameToIndex(name);
+ AppCounters->Percentile()[index].IncrementFor(what);
+ }
+
+ void UpdatePercentile(const char* name, ui64 what, ui64 value) {
+ size_t index = PercentileNameToIndex(name);
+ AppCounters->Percentile()[index].AddFor(what, value);
+ }
+
+ public:
+ static NMonitoring::TDynamicCounterPtr GetAppCounters(TTestBasicRuntime& runtime) {
+ NMonitoring::TDynamicCounterPtr counters = runtime.GetAppData(0).Counters;
+ UNIT_ASSERT(counters);
+
+ TString tabletTypeStr = TTabletTypes::TypeToStr(TabletType);
+ auto dsCounters = counters->GetSubgroup("counters", "tablets")->GetSubgroup("type", tabletTypeStr);
+ return dsCounters->GetSubgroup("category", "app");
+ }
+
+ template <typename TArray>
+ static size_t StringToIndex(const char* name, const TArray& array) {
+ size_t i = 0;
+ for (const auto& s: array) {
+ if (TStringBuf(name) == TStringBuf(s))
+ return i;
+ ++i;
+ }
+ return i;
+ }
+
+ static size_t SimpleNameToIndex(const char* name) {
+ return StringToIndex(name, SimpleCountersMetaInfo);
+ }
+
+ static size_t PercentileNameToIndex(const char* name) {
+ return StringToIndex(name, PercentileCountersMetaInfo);
+ }
+
+ static NMonitoring::THistogramPtr GetHistogram(TTestBasicRuntime& runtime, const char* name) {
+ size_t index = PercentileNameToIndex(name);
+ return GetAppCounters(runtime)->FindHistogram(PercentileCountersMetaInfo[index]);
+ }
+
+ static std::vector<ui64> GetOldHistogram(TTestBasicRuntime& runtime, const char* name) {
+ size_t index = PercentileNameToIndex(name);
+ const auto ranges = RangeDefs[index].first;
+ const auto rangeCount = RangeDefs[index].second;
+
+ auto appCounters = GetAppCounters(runtime);
+ std::vector<ui64> buckets;
+ for (auto i: xrange(rangeCount)) {
+ auto subGroup = appCounters->GetSubgroup("range", ranges[i].RangeName);
+ auto sensor = subGroup->FindCounter(PercentileCountersMetaInfo[index]);
+ if (sensor) {
+ buckets.push_back(sensor->Val());
+ }
+ }
+
+ return buckets;
+ }
+
+ static void CheckHistogram(
+ TTestBasicRuntime& runtime,
+ const char* name,
+ const std::vector<ui64>& goldValuesNew,
+ const std::vector<ui64>& goldValuesOld)
+ {
+ // new stype histogram
+ auto histogram = TTabletWithHist::GetHistogram(runtime, name);
+ UNIT_ASSERT(histogram);
+ auto snapshot = histogram->Snapshot();
+ UNIT_ASSERT(snapshot);
+
+ UNIT_ASSERT_VALUES_EQUAL(snapshot->Count(), goldValuesNew.size());
+ {
+ // for pretty printing the diff
+ std::vector<ui64> values;
+ values.reserve(goldValuesNew.size());
+ for (auto i: xrange(goldValuesNew.size()))
+ values.push_back(snapshot->Value(i));
+ UNIT_ASSERT_VALUES_EQUAL(values, goldValuesNew);
+ }
+
+ // old histogram
+ auto values = TTabletWithHist::GetOldHistogram(runtime, name);
+ UNIT_ASSERT_VALUES_EQUAL(values.size(), goldValuesOld.size());
+ UNIT_ASSERT_VALUES_EQUAL(values, goldValuesOld);
+ }
+
+ public:
+ ui64 TabletId;
+ TPathId TenantPathId;
+ TIntrusivePtr<TEvTabletCounters::TInFlightCookie> CounterEventsInFlight;
+
+ std::unique_ptr<TTabletCountersBase> ExecutorCounters;
+ std::unique_ptr<TTabletCountersBase> ExecutorCountersBaseline;
+
+ std::unique_ptr<TTabletCountersBase> AppCounters;
+ std::unique_ptr<TTabletCountersBase> AppCountersBaseline;
+
+ public:
+ static constexpr TTabletTypes::EType TabletType = TTabletTypes::FLAT_DATASHARD;
+
+ static constexpr TTabletPercentileCounter::TRangeDef RangeDefs1[] = {
+ {0, "0"}
+ };
+
+ static constexpr TTabletPercentileCounter::TRangeDef RangeDefs4[] = {
+ {0, "0"},
+ {1, "1"},
+ {13, "13"},
+ {29, "29"}
+ };
+
+ static constexpr std::pair<const TTabletPercentileCounter::TRangeDef*, size_t> RangeDefs[] = {
+ {RangeDefs1, 1},
+ {RangeDefs4, 4},
+ {RangeDefs1, 1},
+ {RangeDefs4, 4},
+ };
+
+ static constexpr const char* PercentileCountersMetaInfo[] = {
+ "MyHistSingleBucket",
+ "HIST(Count)",
+ "HIST(CountSingleBucket)",
+ "MyHist",
+ };
+
+ static constexpr const char* SimpleCountersMetaInfo[] = {
+ "JustCount1",
+ "Count",
+ "CountSingleBucket",
+ "JustCount2",
+ };
+ };
+
+ Y_UNIT_TEST(IntegralPercentileAggregationHistNamedSingleBucket) {
+ // test case when only 1 range in hist
+ // histogram with name "HIST(CountSingleBucket)" and
+ // associated corresponding simple counter "CountSingleBucket"
+ TTestBasicRuntime runtime(1);
+
+ runtime.Initialize(TAppPrepare().Unwrap());
+ TActorId edge = runtime.AllocateEdgeActor();
+
+ auto aggregator = CreateTabletCountersAggregator(false);
+ auto aggregatorId = runtime.Register(aggregator);
+ runtime.EnableScheduleForActor(aggregatorId);
+
+ TDispatchOptions options;
+ options.FinalEvents.emplace_back(TEvents::TSystem::Bootstrap, 1);
+ runtime.DispatchEvents(options);
+
+ TTabletWithHist tablet1(1);
+
+ tablet1.SetSimpleCount("CountSingleBucket", 1);
+ tablet1.SendUpdate(runtime, aggregatorId, edge);
+
+ TTabletWithHist tablet2(2);
+ tablet2.SetSimpleCount("CountSingleBucket", 13);
+ tablet2.SendUpdate(runtime, aggregatorId, edge);
+
+ TTabletWithHist::CheckHistogram(
+ runtime,
+ "HIST(CountSingleBucket)",
+ {0, 2},
+ {2}
+ );
+
+ // sanity check we didn't mess other histograms
+
+ TTabletWithHist::CheckHistogram(
+ runtime,
+ "MyHist",
+ {0, 0, 0, 0},
+ {0, 0, 0, 0}
+ );
+
+ TTabletWithHist::CheckHistogram(
+ runtime,
+ "HIST(Count)",
+ {2, 0, 0, 0},
+ {2, 0, 0, 0}
+ );
+
+ TTabletWithHist::CheckHistogram(
+ runtime,
+ "MyHistSingleBucket",
+ {0, 0},
+ {0}
+ );
+ }
+
+ Y_UNIT_TEST(IntegralPercentileAggregationHistNamed) {
+ // test special histogram with name "HIST(Count)" and
+ // associated corresponding simple counter "Count"
+ TTestBasicRuntime runtime(1);
+
+ runtime.Initialize(TAppPrepare().Unwrap());
+ TActorId edge = runtime.AllocateEdgeActor();
+
+ auto aggregator = CreateTabletCountersAggregator(false);
+ auto aggregatorId = runtime.Register(aggregator);
+ runtime.EnableScheduleForActor(aggregatorId);
+
+ TDispatchOptions options;
+ options.FinalEvents.emplace_back(TEvents::TSystem::Bootstrap, 1);
+ runtime.DispatchEvents(options);
+
+ TTabletWithHist tablet1(1);
+
+ tablet1.SetSimpleCount("Count", 1);
+ tablet1.SendUpdate(runtime, aggregatorId, edge);
+
+ TTabletWithHist::CheckHistogram(
+ runtime,
+ "HIST(Count)",
+ {1, 0, 0, 0},
+ {0, 1, 0, 0}
+ );
+
+ TTabletWithHist tablet2(2);
+ tablet2.SetSimpleCount("Count", 13);
+ tablet2.SendUpdate(runtime, aggregatorId, edge);
+
+ TTabletWithHist::CheckHistogram(
+ runtime,
+ "HIST(Count)",
+ {1, 1, 0, 0},
+ {0, 1, 1, 0}
+ );
+
+ TTabletWithHist tablet3(3);
+ tablet3.SetSimpleCount("Count", 1);
+ tablet3.SendUpdate(runtime, aggregatorId, edge);
+
+ TTabletWithHist::CheckHistogram(
+ runtime,
+ "HIST(Count)",
+ {2, 1, 0, 0},
+ {0, 2, 1, 0}
+ );
+
+ tablet3.SetSimpleCount("Count", 13);
+ tablet3.SendUpdate(runtime, aggregatorId, edge);
+
+ TTabletWithHist::CheckHistogram(
+ runtime,
+ "HIST(Count)",
+ {1, 2, 0, 0},
+ {0, 1, 2, 0}
+ );
+
+ tablet3.ForgetTablet(runtime, aggregatorId, edge);
+
+ TTabletWithHist::CheckHistogram(
+ runtime,
+ "HIST(Count)",
+ {1, 1, 0, 0},
+ {0, 1, 1, 0}
+ );
+
+ // sanity check we didn't mess other histograms
+
+ TTabletWithHist::CheckHistogram(
+ runtime,
+ "MyHist",
+ {0, 0, 0, 0},
+ {0, 0, 0, 0}
+ );
+
+ TTabletWithHist::CheckHistogram(
+ runtime,
+ "HIST(CountSingleBucket)",
+ {2, 0},
+ {2}
+ );
+
+ TTabletWithHist::CheckHistogram(
+ runtime,
+ "MyHistSingleBucket",
+ {0, 0},
+ {0}
+ );
+ }
+
+ Y_UNIT_TEST(IntegralPercentileAggregationHistNamedNoOverflowCheck) {
+ // test special histogram with name "HIST(Count)" and
+ // associated corresponding simple counter "Count"
+ //
+ // test just for extra sanity, because for Max<ui32> in bucket we
+ // will need Max<ui32> tablets. So just check simple count behaviour
+ TTestBasicRuntime runtime(1);
+
+ runtime.Initialize(TAppPrepare().Unwrap());
+ TActorId edge = runtime.AllocateEdgeActor();
+
+ auto aggregator = CreateTabletCountersAggregator(false);
+ auto aggregatorId = runtime.Register(aggregator);
+ runtime.EnableScheduleForActor(aggregatorId);
+
+ TDispatchOptions options;
+ options.FinalEvents.emplace_back(TEvents::TSystem::Bootstrap, 1);
+ runtime.DispatchEvents(options);
+
+ TTabletWithHist tablet1(1);
+
+ tablet1.SetSimpleCount("Count", Max<i64>() - 100UL);
+ tablet1.SendUpdate(runtime, aggregatorId, edge);
+
+ TTabletWithHist::CheckHistogram(
+ runtime,
+ "HIST(Count)",
+ {0, 0, 0, 1},
+ {0, 0, 0, 1}
+ );
+
+ TTabletWithHist tablet2(2);
+ tablet2.SetSimpleCount("Count", 100);
+ tablet2.SendUpdate(runtime, aggregatorId, edge);
+
+ TTabletWithHist::CheckHistogram(
+ runtime,
+ "HIST(Count)",
+ {0, 0, 0, 2},
+ {0, 0, 0, 2}
+ );
+ }
+
+ // Regression test for KIKIMR-13457
+ Y_UNIT_TEST(IntegralPercentileAggregationRegular) {
+ // test regular histogram, i.e. not named "HIST"
+ TTestBasicRuntime runtime(1);
+
+ runtime.Initialize(TAppPrepare().Unwrap());
+ TActorId edge = runtime.AllocateEdgeActor();
+
+ auto aggregator = CreateTabletCountersAggregator(false);
+ auto aggregatorId = runtime.Register(aggregator);
+ runtime.EnableScheduleForActor(aggregatorId);
+
+ TDispatchOptions options;
+ options.FinalEvents.emplace_back(TEvents::TSystem::Bootstrap, 1);
+ runtime.DispatchEvents(options);
+
+ TTabletWithHist tablet1(1);
+ tablet1.UpdatePercentile("MyHist", 1);
+ tablet1.SendUpdate(runtime, aggregatorId, edge);
+
+ TTabletWithHist tablet2(2);
+ tablet2.UpdatePercentile("MyHist", 1);
+ tablet2.SendUpdate(runtime, aggregatorId, edge);
+
+ TTabletWithHist tablet3(3);
+ tablet3.UpdatePercentile("MyHist", 1);
+ tablet3.UpdatePercentile("MyHist", 13);
+ tablet3.SendUpdate(runtime, aggregatorId, edge);
+
+ TTabletWithHist::CheckHistogram(
+ runtime,
+ "MyHist",
+ {0, 3, 1, 0}, // XXX
+ {0, 3, 1, 0}
+ );
+
+ tablet3.ForgetTablet(runtime, aggregatorId, edge);
+
+ TTabletWithHist::CheckHistogram(
+ runtime,
+ "MyHist",
+ {0, 2, 0, 0}, // XXX
+ {0, 2, 0, 0}
+ );
+
+ // sanity check we didn't mess other histograms
+
+ TTabletWithHist::CheckHistogram(
+ runtime,
+ "HIST(Count)",
+ {2, 0, 0, 0},
+ {2, 0, 0, 0}
+ );
+
+ TTabletWithHist::CheckHistogram(
+ runtime,
+ "MyHistSingleBucket",
+ {0, 0},
+ {0}
+ );
+
+ TTabletWithHist::CheckHistogram(
+ runtime,
+ "HIST(CountSingleBucket)",
+ {2, 0},
+ {2}
+ );
+ }
+
+ Y_UNIT_TEST(IntegralPercentileAggregationRegularNoOverflowCheck) {
+ // test regular histogram, i.e. not named "HIST"
+ TTestBasicRuntime runtime(1);
+
+ runtime.Initialize(TAppPrepare().Unwrap());
+ TActorId edge = runtime.AllocateEdgeActor();
+
+ auto aggregator = CreateTabletCountersAggregator(false);
+ auto aggregatorId = runtime.Register(aggregator);
+ runtime.EnableScheduleForActor(aggregatorId);
+
+ TDispatchOptions options;
+ options.FinalEvents.emplace_back(TEvents::TSystem::Bootstrap, 1);
+ runtime.DispatchEvents(options);
+
+ TTabletWithHist tablet1(1);
+ tablet1.UpdatePercentile("MyHist", 10, Max<i64>() - 100);
+ tablet1.SendUpdate(runtime, aggregatorId, edge);
+
+ TTabletWithHist tablet2(2);
+ tablet2.UpdatePercentile("MyHist", 10, 25);
+ tablet2.SendUpdate(runtime, aggregatorId, edge);
+
+ TTabletWithHist tablet3(3);
+ tablet3.UpdatePercentile("MyHist", 10, 5);
+ tablet3.SendUpdate(runtime, aggregatorId, edge);
+
+ ui64 v = Max<i64>() - 70;
+ TTabletWithHist::CheckHistogram(
+ runtime,
+ "MyHist",
+ {0, v, 0, 0},
+ {0, v, 0, 0}
+ );
+
+ tablet1.ForgetTablet(runtime, aggregatorId, edge);
+ TTabletWithHist::CheckHistogram(
+ runtime,
+ "MyHist",
+ {0, 30, 0, 0},
+ {0, 30, 0, 0}
+ );
+ }
+}
+
+Y_UNIT_TEST_SUITE(TTabletLabeledCountersAggregator) {
Y_UNIT_TEST(SimpleAggregation) {
TVector<TActorId> cc;
TActorId aggregatorId;
@@ -176,7 +679,6 @@ Y_UNIT_TEST_SUITE(TTabletCountersAggregator) {
UNIT_ASSERT_VALUES_EQUAL(counter1.GetValue(), 39);
}
-
Y_UNIT_TEST(HeavyAggregation) {
TestHeavy(2, 10);
TestHeavy(2, 20);
diff --git a/ydb/core/tablet_flat/flat_executor_counters.h b/ydb/core/tablet_flat/flat_executor_counters.h
index 2423410491..912369bd96 100644
--- a/ydb/core/tablet_flat/flat_executor_counters.h
+++ b/ydb/core/tablet_flat/flat_executor_counters.h
@@ -5,6 +5,7 @@
namespace NKikimr {
namespace NTabletFlatExecutor {
+// don't change order!
#define FLAT_EXECUTOR_SIMPLE_COUNTERS_MAP(XX) \
XX(DB_TX_IN_FLY, "ExecutorTxInFly") \
XX(LOG_REDO_COUNT, "LogRedoItems") \
@@ -62,6 +63,7 @@ namespace NTabletFlatExecutor {
XX(CONSUMED_MEMORY, "ConsumedMemory") \
XX(COMPACTION_READ_IN_FLY, "CompactionReadInFly") \
+// don't change order!
#define FLAT_EXECUTOR_CUMULATIVE_COUNTERS_MAP(XX) \
XX(LOG_COMMITS, "LogCommits") \
XX(LOG_WRITTEN, "LogWritten") \
@@ -112,6 +114,7 @@ namespace NTabletFlatExecutor {
XX(COMPACTION_READ_LOAD_BYTES, "CompactionReadLoadBytes") \
XX(COMPACTION_READ_LOAD_PAGES, "CompactionReadLoadPages") \
+// don't change order!
#define FLAT_EXECUTOR_PERCENTILE_COUNTERS_MAP(XX) \
XX(TX_PERCENTILE_LATENCY_RO, "TxRoLatency") \
XX(TX_PERCENTILE_LATENCY_RW, "TxRwLatency") \