diff options
author | msherbakov <msherbakov@yandex-team.ru> | 2022-02-10 16:49:17 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:49:17 +0300 |
commit | a0ffafe83b7d6229709a32fa942c71d672ac989c (patch) | |
tree | 5d5cb817648f650d76cf1076100726fd9b8448e8 /library/cpp/monlib/dynamic_counters | |
parent | c224a621661ddd69699f9476922eb316607ef57e (diff) | |
download | ydb-a0ffafe83b7d6229709a32fa942c71d672ac989c.tar.gz |
Restoring authorship annotation for <msherbakov@yandex-team.ru>. Commit 2 of 2.
Diffstat (limited to 'library/cpp/monlib/dynamic_counters')
-rw-r--r-- | library/cpp/monlib/dynamic_counters/counters.cpp | 132 | ||||
-rw-r--r-- | library/cpp/monlib/dynamic_counters/counters.h | 232 | ||||
-rw-r--r-- | library/cpp/monlib/dynamic_counters/counters_ut.cpp | 76 | ||||
-rw-r--r-- | library/cpp/monlib/dynamic_counters/encode.cpp | 16 | ||||
-rw-r--r-- | library/cpp/monlib/dynamic_counters/encode.h | 10 | ||||
-rw-r--r-- | library/cpp/monlib/dynamic_counters/encode_ut.cpp | 84 | ||||
-rw-r--r-- | library/cpp/monlib/dynamic_counters/golovan_page.cpp | 4 | ||||
-rw-r--r-- | library/cpp/monlib/dynamic_counters/golovan_page.h | 2 | ||||
-rw-r--r-- | library/cpp/monlib/dynamic_counters/page.cpp | 102 | ||||
-rw-r--r-- | library/cpp/monlib/dynamic_counters/page.h | 32 | ||||
-rw-r--r-- | library/cpp/monlib/dynamic_counters/percentile/percentile.h | 114 | ||||
-rw-r--r-- | library/cpp/monlib/dynamic_counters/percentile/percentile_lg.h | 342 | ||||
-rw-r--r-- | library/cpp/monlib/dynamic_counters/percentile/percentile_ut.cpp | 152 | ||||
-rw-r--r-- | library/cpp/monlib/dynamic_counters/percentile/ut/ya.make | 16 | ||||
-rw-r--r-- | library/cpp/monlib/dynamic_counters/percentile/ya.make | 26 |
15 files changed, 670 insertions, 670 deletions
diff --git a/library/cpp/monlib/dynamic_counters/counters.cpp b/library/cpp/monlib/dynamic_counters/counters.cpp index 68bf4d6edf..3635d87d0d 100644 --- a/library/cpp/monlib/dynamic_counters/counters.cpp +++ b/library/cpp/monlib/dynamic_counters/counters.cpp @@ -6,68 +6,68 @@ using namespace NMonitoring; -namespace { - TDynamicCounters* AsDynamicCounters(const TIntrusivePtr<TCountableBase>& ptr) { - return dynamic_cast<TDynamicCounters*>(ptr.Get()); - } - - TCounterForPtr* AsCounter(const TIntrusivePtr<TCountableBase>& ptr) { - return dynamic_cast<TCounterForPtr*>(ptr.Get()); - } - - TExpiringCounter* AsExpiringCounter(const TIntrusivePtr<TCountableBase>& ptr) { - return dynamic_cast<TExpiringCounter*>(ptr.Get()); - } - +namespace { + TDynamicCounters* AsDynamicCounters(const TIntrusivePtr<TCountableBase>& ptr) { + return dynamic_cast<TDynamicCounters*>(ptr.Get()); + } + + TCounterForPtr* AsCounter(const TIntrusivePtr<TCountableBase>& ptr) { + return dynamic_cast<TCounterForPtr*>(ptr.Get()); + } + + TExpiringCounter* AsExpiringCounter(const TIntrusivePtr<TCountableBase>& ptr) { + return dynamic_cast<TExpiringCounter*>(ptr.Get()); + } + TExpiringHistogramCounter* AsExpiringHistogramCounter(const TIntrusivePtr<TCountableBase>& ptr) { return dynamic_cast<TExpiringHistogramCounter*>(ptr.Get()); } - THistogramCounter* AsHistogram(const TIntrusivePtr<TCountableBase>& ptr) { - return dynamic_cast<THistogramCounter*>(ptr.Get()); - } - - TIntrusivePtr<TCounterForPtr> AsCounterRef(const TIntrusivePtr<TCountableBase>& ptr) { - return VerifyDynamicCast<TCounterForPtr*>(ptr.Get()); - } - - TIntrusivePtr<TDynamicCounters> AsGroupRef(const TIntrusivePtr<TCountableBase>& ptr) { - return VerifyDynamicCast<TDynamicCounters*>(ptr.Get()); - } - - THistogramPtr AsHistogramRef(const TIntrusivePtr<TCountableBase>& ptr) { - return VerifyDynamicCast<THistogramCounter*>(ptr.Get()); - } + THistogramCounter* AsHistogram(const TIntrusivePtr<TCountableBase>& ptr) { + return dynamic_cast<THistogramCounter*>(ptr.Get()); + } + + TIntrusivePtr<TCounterForPtr> AsCounterRef(const TIntrusivePtr<TCountableBase>& ptr) { + return VerifyDynamicCast<TCounterForPtr*>(ptr.Get()); + } + + TIntrusivePtr<TDynamicCounters> AsGroupRef(const TIntrusivePtr<TCountableBase>& ptr) { + return VerifyDynamicCast<TDynamicCounters*>(ptr.Get()); + } + + THistogramPtr AsHistogramRef(const TIntrusivePtr<TCountableBase>& ptr) { + return VerifyDynamicCast<THistogramCounter*>(ptr.Get()); + } bool IsExpiringCounter(const TIntrusivePtr<TCountableBase>& ptr) { return AsExpiringCounter(ptr) != nullptr || AsExpiringHistogramCounter(ptr) != nullptr; } -} - +} + static constexpr TStringBuf INDENT = " "; -TDynamicCounters::TDynamicCounters(EVisibility vis) -{ - Visibility_ = vis; +TDynamicCounters::TDynamicCounters(EVisibility vis) +{ + Visibility_ = vis; } - + TDynamicCounters::~TDynamicCounters() { } -TDynamicCounters::TCounterPtr TDynamicCounters::GetExpiringCounter(const TString& value, bool derivative, EVisibility vis) { - return GetExpiringNamedCounter("sensor", value, derivative, vis); -} - -TDynamicCounters::TCounterPtr TDynamicCounters::GetExpiringNamedCounter(const TString& name, const TString& value, bool derivative, EVisibility vis) { - return AsCounterRef(GetNamedCounterImpl<true, TExpiringCounter>(name, value, derivative, vis)); -} - -TDynamicCounters::TCounterPtr TDynamicCounters::GetCounter(const TString& value, bool derivative, EVisibility vis) { - return GetNamedCounter("sensor", value, derivative, vis); +TDynamicCounters::TCounterPtr TDynamicCounters::GetExpiringCounter(const TString& value, bool derivative, EVisibility vis) { + return GetExpiringNamedCounter("sensor", value, derivative, vis); +} + +TDynamicCounters::TCounterPtr TDynamicCounters::GetExpiringNamedCounter(const TString& name, const TString& value, bool derivative, EVisibility vis) { + return AsCounterRef(GetNamedCounterImpl<true, TExpiringCounter>(name, value, derivative, vis)); +} + +TDynamicCounters::TCounterPtr TDynamicCounters::GetCounter(const TString& value, bool derivative, EVisibility vis) { + return GetNamedCounter("sensor", value, derivative, vis); } -TDynamicCounters::TCounterPtr TDynamicCounters::GetNamedCounter(const TString& name, const TString& value, bool derivative, EVisibility vis) { - return AsCounterRef(GetNamedCounterImpl<false, TCounterForPtr>(name, value, derivative, vis)); +TDynamicCounters::TCounterPtr TDynamicCounters::GetNamedCounter(const TString& name, const TString& value, bool derivative, EVisibility vis) { + return AsCounterRef(GetNamedCounterImpl<false, TCounterForPtr>(name, value, derivative, vis)); } THistogramPtr TDynamicCounters::GetHistogram(const TString& value, IHistogramCollectorPtr collector, bool derivative, EVisibility vis) { @@ -110,7 +110,7 @@ void TDynamicCounters::RemoveNamedCounter(const TString& name, const TString &va auto g = LockForUpdate("RemoveNamedCounter", name, value); if (const auto it = Counters.find({name, value}); it != Counters.end() && AsCounter(it->second)) { Counters.erase(it); - } + } } TIntrusivePtr<TDynamicCounters> TDynamicCounters::GetSubgroup(const TString& name, const TString& value) { @@ -172,15 +172,15 @@ void TDynamicCounters::ResetCounters(bool derivOnly) { } } -void TDynamicCounters::RegisterCountable(const TString& name, const TString& value, TCountablePtr countable) { - Y_VERIFY(countable); +void TDynamicCounters::RegisterCountable(const TString& name, const TString& value, TCountablePtr countable) { + Y_VERIFY(countable); auto g = LockForUpdate("RegisterCountable", name, value); const bool inserted = Counters.emplace(TChildId(name, value), std::move(countable)).second; Y_VERIFY(inserted); } void TDynamicCounters::RegisterSubgroup(const TString& name, const TString& value, TIntrusivePtr<TDynamicCounters> subgroup) { - RegisterCountable(name, value, subgroup); + RegisterCountable(name, value, subgroup); } void TDynamicCounters::OutputHtml(IOutputStream& os) const { @@ -245,36 +245,36 @@ void TDynamicCounters::OutputPlainText(IOutputStream& os, const TString& indent) } void TDynamicCounters::Accept(const TString& labelName, const TString& labelValue, ICountableConsumer& consumer) const { - if (!IsVisible(Visibility(), consumer.Visibility())) { - return; - } - + if (!IsVisible(Visibility(), consumer.Visibility())) { + return; + } + consumer.OnGroupBegin(labelName, labelValue, this); for (auto& [key, value] : ReadSnapshot()) { value->Accept(key.LabelName, key.LabelValue, consumer); } consumer.OnGroupEnd(labelName, labelValue, this); } - -void TDynamicCounters::RemoveExpired() const { + +void TDynamicCounters::RemoveExpired() const { if (AtomicGet(ExpiringCount) == 0) { - return; - } - + return; + } + TWriteGuard g(Lock); TAtomicBase count = 0; - for (auto it = Counters.begin(); it != Counters.end();) { + for (auto it = Counters.begin(); it != Counters.end();) { if (IsExpiringCounter(it->second) && it->second->RefCount() == 1) { - it = Counters.erase(it); + it = Counters.erase(it); ++count; - } else { - ++it; - } - } + } else { + ++it; + } + } AtomicSub(ExpiringCount, count); -} +} template <bool expiring, class TCounterType, class... TArgs> TDynamicCounters::TCountablePtr TDynamicCounters::GetNamedCounterImpl(const TString& name, const TString& value, TArgs&&... args) { diff --git a/library/cpp/monlib/dynamic_counters/counters.h b/library/cpp/monlib/dynamic_counters/counters.h index d644575d1b..dc178cfbe0 100644 --- a/library/cpp/monlib/dynamic_counters/counters.h +++ b/library/cpp/monlib/dynamic_counters/counters.h @@ -6,7 +6,7 @@ #include <library/cpp/threading/light_rw_lock/lightrwlock.h> #include <library/cpp/containers/stack_vector/stack_vec.h> -#include <util/generic/cast.h> +#include <util/generic/cast.h> #include <util/generic/map.h> #include <util/generic/ptr.h> #include <util/string/cast.h> @@ -19,43 +19,43 @@ namespace NMonitoring { struct TDynamicCounters; struct ICountableConsumer; - + struct TCountableBase: public TAtomicRefCount<TCountableBase> { - // Private means that the object must not be serialized unless the consumer - // has explicitly specified this by setting its Visibility to Private. - // - // Works only for the methods that accept ICountableConsumer - enum class EVisibility: ui8 { - Unspecified, - Public, - Private, - }; - + // Private means that the object must not be serialized unless the consumer + // has explicitly specified this by setting its Visibility to Private. + // + // Works only for the methods that accept ICountableConsumer + enum class EVisibility: ui8 { + Unspecified, + Public, + Private, + }; + virtual ~TCountableBase() { } virtual void Accept( const TString& labelName, const TString& labelValue, ICountableConsumer& consumer) const = 0; - - virtual EVisibility Visibility() const { - return Visibility_; - } - - protected: - EVisibility Visibility_{EVisibility::Unspecified}; + + virtual EVisibility Visibility() const { + return Visibility_; + } + + protected: + EVisibility Visibility_{EVisibility::Unspecified}; }; - inline bool IsVisible(TCountableBase::EVisibility myLevel, TCountableBase::EVisibility consumerLevel) { - if (myLevel == TCountableBase::EVisibility::Private - && consumerLevel != TCountableBase::EVisibility::Private) { - - return false; - } - - return true; - } - + inline bool IsVisible(TCountableBase::EVisibility myLevel, TCountableBase::EVisibility consumerLevel) { + if (myLevel == TCountableBase::EVisibility::Private + && consumerLevel != TCountableBase::EVisibility::Private) { + + return false; + } + + return true; + } + struct ICountableConsumer { virtual ~ICountableConsumer() { } @@ -75,10 +75,10 @@ namespace NMonitoring { virtual void OnGroupEnd( const TString& labelName, const TString& labelValue, const TDynamicCounters* group) = 0; - - virtual TCountableBase::EVisibility Visibility() const { - return TCountableBase::EVisibility::Unspecified; - } + + virtual TCountableBase::EVisibility Visibility() const { + return TCountableBase::EVisibility::Unspecified; + } }; #ifdef _MSC_VER @@ -87,10 +87,10 @@ namespace NMonitoring { #endif // _MSC_VER struct TCounterForPtr: public TDeprecatedCounter, public TCountableBase { - TCounterForPtr(bool derivative = false, EVisibility vis = EVisibility::Public) + TCounterForPtr(bool derivative = false, EVisibility vis = EVisibility::Public) : TDeprecatedCounter(0ULL, derivative) { - Visibility_ = vis; + Visibility_ = vis; } TCounterForPtr(const TCounterForPtr&) = delete; @@ -99,15 +99,15 @@ namespace NMonitoring { void Accept( const TString& labelName, const TString& labelValue, ICountableConsumer& consumer) const override { - if (IsVisible(Visibility(), consumer.Visibility())) { - consumer.OnCounter(labelName, labelValue, this); - } + if (IsVisible(Visibility(), consumer.Visibility())) { + consumer.OnCounter(labelName, labelValue, this); + } + } + + TCountableBase::EVisibility Visibility() const override { + return Visibility_; } - TCountableBase::EVisibility Visibility() const override { - return Visibility_; - } - using TDeprecatedCounter::operator++; using TDeprecatedCounter::operator--; using TDeprecatedCounter::operator+=; @@ -116,25 +116,25 @@ namespace NMonitoring { using TDeprecatedCounter::operator!; }; - struct TExpiringCounter: public TCounterForPtr { - explicit TExpiringCounter(bool derivative = false, EVisibility vis = EVisibility::Public) - : TCounterForPtr{derivative} - { - Visibility_ = vis; - } - - void Reset() { + struct TExpiringCounter: public TCounterForPtr { + explicit TExpiringCounter(bool derivative = false, EVisibility vis = EVisibility::Public) + : TCounterForPtr{derivative} + { + Visibility_ = vis; + } + + void Reset() { TDeprecatedCounter::operator=(0); - } - }; - + } + }; + struct THistogramCounter: public TCountableBase { explicit THistogramCounter( IHistogramCollectorPtr collector, bool derivative = true, EVisibility vis = EVisibility::Public) : Collector_(std::move(collector)) , Derivative_(derivative) { - Visibility_ = vis; + Visibility_ = vis; } void Collect(i64 value) { @@ -157,9 +157,9 @@ namespace NMonitoring { const TString& labelName, const TString& labelValue, ICountableConsumer& consumer) const override { - if (IsVisible(Visibility(), consumer.Visibility())) { + if (IsVisible(Visibility(), consumer.Visibility())) { consumer.OnHistogram(labelName, labelValue, Collector_->Snapshot(), Derivative_); - } + } } void Reset() { @@ -190,7 +190,7 @@ namespace NMonitoring { typedef TIntrusivePtr<TDynamicCounters> TDynamicCounterPtr; struct TDynamicCounters: public TCountableBase { public: - using TCounterPtr = TIntrusivePtr<TCounterForPtr>; + using TCounterPtr = TIntrusivePtr<TCounterForPtr>; using TOnLookupPtr = void (*)(const char *methodName, const TString &name, const TString &value); private: @@ -225,14 +225,14 @@ namespace NMonitoring { }; using TCounters = TMap<TChildId, TCountablePtr>; - using TLabels = TVector<TChildId>; - - /// XXX: hack for deferred removal of expired counters. Remove once Output* functions are not used for serialization - mutable TCounters Counters; + using TLabels = TVector<TChildId>; + + /// XXX: hack for deferred removal of expired counters. Remove once Output* functions are not used for serialization + mutable TCounters Counters; mutable TAtomic ExpiringCount = 0; public: - TDynamicCounters(TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public); + TDynamicCounters(TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public); TDynamicCounters(const TDynamicCounters *origin) : LookupCounter(origin->LookupCounter) @@ -270,56 +270,56 @@ namespace NMonitoring { return items; } - TCounterPtr GetCounter( - const TString& value, - bool derivative = false, - TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public); - - TCounterPtr GetNamedCounter( - const TString& name, - const TString& value, - bool derivative = false, - TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public); - - THistogramPtr GetHistogram( - const TString& value, - IHistogramCollectorPtr collector, + TCounterPtr GetCounter( + const TString& value, + bool derivative = false, + TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public); + + TCounterPtr GetNamedCounter( + const TString& name, + const TString& value, + bool derivative = false, + TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public); + + THistogramPtr GetHistogram( + const TString& value, + IHistogramCollectorPtr collector, bool derivative = true, - TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public); - - THistogramPtr GetNamedHistogram( - const TString& name, - const TString& value, - IHistogramCollectorPtr collector, + TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public); + + THistogramPtr GetNamedHistogram( + const TString& name, + const TString& value, + IHistogramCollectorPtr collector, bool derivative = true, - TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public); - - // These counters will be automatically removed from the registry - // when last reference to the counter expires. - TCounterPtr GetExpiringCounter( - const TString& value, - bool derivative = false, - TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public); - - TCounterPtr GetExpiringNamedCounter( - const TString& name, - const TString& value, - bool derivative = false, - TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public); - - THistogramPtr GetExpiringHistogram( - const TString& value, - IHistogramCollectorPtr collector, + TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public); + + // These counters will be automatically removed from the registry + // when last reference to the counter expires. + TCounterPtr GetExpiringCounter( + const TString& value, + bool derivative = false, + TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public); + + TCounterPtr GetExpiringNamedCounter( + const TString& name, + const TString& value, + bool derivative = false, + TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public); + + THistogramPtr GetExpiringHistogram( + const TString& value, + IHistogramCollectorPtr collector, bool derivative = true, - TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public); - - THistogramPtr GetExpiringNamedHistogram( - const TString& name, - const TString& value, - IHistogramCollectorPtr collector, + TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public); + + THistogramPtr GetExpiringNamedHistogram( + const TString& name, + const TString& value, + IHistogramCollectorPtr collector, bool derivative = true, - TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public); - + TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public); + TCounterPtr FindCounter(const TString& value) const; TCounterPtr FindNamedCounter(const TString& name, const TString& value) const; @@ -339,14 +339,14 @@ namespace NMonitoring { // Recursively reset all/deriv counters to 0. void ResetCounters(bool derivOnly = false); - void RegisterSubgroup(const TString& name, - const TString& value, - TIntrusivePtr<TDynamicCounters> subgroup); - + void RegisterSubgroup(const TString& name, + const TString& value, + TIntrusivePtr<TDynamicCounters> subgroup); + void OutputHtml(IOutputStream& os) const; void EnumerateSubgroups(const std::function<void(const TString& name, const TString& value)>& output) const; - - // mostly for debugging purposes -- use accept with encoder instead + + // mostly for debugging purposes -- use accept with encoder instead void OutputPlainText(IOutputStream& os, const TString& indent = "") const; void Accept( @@ -361,8 +361,8 @@ namespace NMonitoring { return counters; } - void RegisterCountable(const TString& name, const TString& value, TCountablePtr countable); - void RemoveExpired() const; + void RegisterCountable(const TString& name, const TString& value, TCountablePtr countable); + void RemoveExpired() const; template <bool expiring, class TCounterType, class... TArgs> TCountablePtr GetNamedCounterImpl(const TString& name, const TString& value, TArgs&&... args); diff --git a/library/cpp/monlib/dynamic_counters/counters_ut.cpp b/library/cpp/monlib/dynamic_counters/counters_ut.cpp index eb66b15fd7..3591037e0a 100644 --- a/library/cpp/monlib/dynamic_counters/counters_ut.cpp +++ b/library/cpp/monlib/dynamic_counters/counters_ut.cpp @@ -224,48 +224,48 @@ Y_UNIT_TEST_SUITE(TDynamicCountersTest) { " sensor:2 = 0\n" "}\n"); } - - Y_UNIT_TEST(ExpiringCounters) { - TDynamicCounterPtr rootGroup{new TDynamicCounters()}; - - { - auto c = rootGroup->GetExpiringCounter("foo"); + + Y_UNIT_TEST(ExpiringCounters) { + TDynamicCounterPtr rootGroup{new TDynamicCounters()}; + + { + auto c = rootGroup->GetExpiringCounter("foo"); auto h = rootGroup->GetExpiringHistogram("bar", ExplicitHistogram({1, 42})); h->Collect(15); - - TStringStream ss; - TCountersPrinter printer(&ss); - rootGroup->Accept("root", "counters", printer); - UNIT_ASSERT_STRINGS_EQUAL(ss.Str(), - "root:counters {\n" + + TStringStream ss; + TCountersPrinter printer(&ss); + rootGroup->Accept("root", "counters", printer); + UNIT_ASSERT_STRINGS_EQUAL(ss.Str(), + "root:counters {\n" " sensor:bar = {1: 0, 42: 1, inf: 0}\n" - " sensor:foo = 0\n" - "}\n"); - } - - TStringStream ss; - TCountersPrinter printer(&ss); - rootGroup->Accept("root", "counters", printer); - UNIT_ASSERT_STRINGS_EQUAL(ss.Str(), - "root:counters {\n" - "}\n"); + " sensor:foo = 0\n" + "}\n"); + } + + TStringStream ss; + TCountersPrinter printer(&ss); + rootGroup->Accept("root", "counters", printer); + UNIT_ASSERT_STRINGS_EQUAL(ss.Str(), + "root:counters {\n" + "}\n"); } - - Y_UNIT_TEST(ExpiringCountersDiesAfterRegistry) { - TDynamicCounters::TCounterPtr ptr; - - { - TDynamicCounterPtr rootGroup{new TDynamicCounters()}; - ptr = rootGroup->GetExpiringCounter("foo"); - - TStringStream ss; - TCountersPrinter printer(&ss); - rootGroup->Accept("root", "counters", printer); - UNIT_ASSERT_STRINGS_EQUAL(ss.Str(), - "root:counters {\n" - " sensor:foo = 0\n" - "}\n"); - } + + Y_UNIT_TEST(ExpiringCountersDiesAfterRegistry) { + TDynamicCounters::TCounterPtr ptr; + + { + TDynamicCounterPtr rootGroup{new TDynamicCounters()}; + ptr = rootGroup->GetExpiringCounter("foo"); + + TStringStream ss; + TCountersPrinter printer(&ss); + rootGroup->Accept("root", "counters", printer); + UNIT_ASSERT_STRINGS_EQUAL(ss.Str(), + "root:counters {\n" + " sensor:foo = 0\n" + "}\n"); + } } Y_UNIT_TEST(HistogramCounter) { diff --git a/library/cpp/monlib/dynamic_counters/encode.cpp b/library/cpp/monlib/dynamic_counters/encode.cpp index 47927c0e65..ffa48d276e 100644 --- a/library/cpp/monlib/dynamic_counters/encode.cpp +++ b/library/cpp/monlib/dynamic_counters/encode.cpp @@ -17,7 +17,7 @@ namespace NMonitoring { public: explicit TConsumer(NMonitoring::IMetricEncoderPtr encoderImpl, TCountableBase::EVisibility vis) : EncoderImpl_(std::move(encoderImpl)) - , Visibility_{vis} + , Visibility_{vis} { } @@ -73,10 +73,10 @@ namespace NMonitoring { } } - TCountableBase::EVisibility Visibility() const override { - return Visibility_; - } - + TCountableBase::EVisibility Visibility() const override { + return Visibility_; + } + private: void EncodeLabels(const TString& labelName, const TString& labelValue) { EncoderImpl_->OnLabelsBegin(); @@ -90,12 +90,12 @@ namespace NMonitoring { private: NMonitoring::IMetricEncoderPtr EncoderImpl_; TVector<TLabel> ParentLabels_; - TCountableBase::EVisibility Visibility_; + TCountableBase::EVisibility Visibility_; }; } - THolder<ICountableConsumer> CreateEncoder(IOutputStream* out, EFormat format, TCountableBase::EVisibility vis) { + THolder<ICountableConsumer> CreateEncoder(IOutputStream* out, EFormat format, TCountableBase::EVisibility vis) { switch (format) { case EFormat::JSON: return MakeHolder<TConsumer>(NMonitoring::EncoderJson(out), vis); @@ -103,7 +103,7 @@ namespace NMonitoring { return MakeHolder<TConsumer>(NMonitoring::EncoderSpackV1( out, NMonitoring::ETimePrecision::SECONDS, - NMonitoring::ECompression::ZSTD), vis); + NMonitoring::ECompression::ZSTD), vis); case EFormat::PROMETHEUS: return MakeHolder<TConsumer>(NMonitoring::EncoderPrometheus( out), vis); diff --git a/library/cpp/monlib/dynamic_counters/encode.h b/library/cpp/monlib/dynamic_counters/encode.h index c6e190c729..c79964d7cb 100644 --- a/library/cpp/monlib/dynamic_counters/encode.h +++ b/library/cpp/monlib/dynamic_counters/encode.h @@ -7,11 +7,11 @@ namespace NMonitoring { - THolder<ICountableConsumer> CreateEncoder( - IOutputStream* out, - EFormat format, - TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public - ); + THolder<ICountableConsumer> CreateEncoder( + IOutputStream* out, + EFormat format, + TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public + ); THolder<ICountableConsumer> AsCountableConsumer( NMonitoring::IMetricEncoderPtr encoder, diff --git a/library/cpp/monlib/dynamic_counters/encode_ut.cpp b/library/cpp/monlib/dynamic_counters/encode_ut.cpp index 0e48009ee2..52d77b6b41 100644 --- a/library/cpp/monlib/dynamic_counters/encode_ut.cpp +++ b/library/cpp/monlib/dynamic_counters/encode_ut.cpp @@ -167,48 +167,48 @@ namespace NMonitoring { AssertResult(samples); } - - Y_UNIT_TEST(PrivateSubgroupIsNotSerialized) { - TBuffer result; - auto subGroup = MakeIntrusive<TDynamicCounters>(TCountableBase::EVisibility::Private); - subGroup->GetCounter("hello"); - Data.RegisterSubgroup("foo", "bar", subGroup); - - { - TBufferOutput out(result); - auto encoder = CreateEncoder(&out, EFormat::SPACK); - Data.Accept(TString(), TString(), *encoder); - } - - NProto::TSingleSamplesList samples; - { - auto e = EncoderProtobuf(&samples); - TBufferInput in(result); - DecodeSpackV1(&in, e.Get()); - } - - AssertResult(samples); - } - - Y_UNIT_TEST(PrivateCounterIsNotSerialized) { - TBuffer result; - Data.GetCounter("foo", false, TCountableBase::EVisibility::Private); - - { - TBufferOutput out(result); - auto encoder = CreateEncoder(&out, EFormat::SPACK); - Data.Accept(TString(), TString(), *encoder); - } - - NProto::TSingleSamplesList samples; - { - auto e = EncoderProtobuf(&samples); - TBufferInput in(result); - DecodeSpackV1(&in, e.Get()); - } - - AssertResult(samples); - } + + Y_UNIT_TEST(PrivateSubgroupIsNotSerialized) { + TBuffer result; + auto subGroup = MakeIntrusive<TDynamicCounters>(TCountableBase::EVisibility::Private); + subGroup->GetCounter("hello"); + Data.RegisterSubgroup("foo", "bar", subGroup); + + { + TBufferOutput out(result); + auto encoder = CreateEncoder(&out, EFormat::SPACK); + Data.Accept(TString(), TString(), *encoder); + } + + NProto::TSingleSamplesList samples; + { + auto e = EncoderProtobuf(&samples); + TBufferInput in(result); + DecodeSpackV1(&in, e.Get()); + } + + AssertResult(samples); + } + + Y_UNIT_TEST(PrivateCounterIsNotSerialized) { + TBuffer result; + Data.GetCounter("foo", false, TCountableBase::EVisibility::Private); + + { + TBufferOutput out(result); + auto encoder = CreateEncoder(&out, EFormat::SPACK); + Data.Accept(TString(), TString(), *encoder); + } + + NProto::TSingleSamplesList samples; + { + auto e = EncoderProtobuf(&samples); + TBufferInput in(result); + DecodeSpackV1(&in, e.Get()); + } + + AssertResult(samples); + } Y_UNIT_TEST(ToJson) { TString result = ToJson(Data); diff --git a/library/cpp/monlib/dynamic_counters/golovan_page.cpp b/library/cpp/monlib/dynamic_counters/golovan_page.cpp index 3fc3152365..49cf2d39bb 100644 --- a/library/cpp/monlib/dynamic_counters/golovan_page.cpp +++ b/library/cpp/monlib/dynamic_counters/golovan_page.cpp @@ -72,8 +72,8 @@ TGolovanCountersPage::TGolovanCountersPage(const TString& path, TIntrusivePtr<NM { } -void TGolovanCountersPage::Output(IMonHttpRequest& request) { - TGolovanCountableConsumer consumer(request.Output(), OutputCallback); +void TGolovanCountersPage::Output(IMonHttpRequest& request) { + TGolovanCountableConsumer consumer(request.Output(), OutputCallback); Counters->Accept("", "", consumer); consumer.Flush(); } diff --git a/library/cpp/monlib/dynamic_counters/golovan_page.h b/library/cpp/monlib/dynamic_counters/golovan_page.h index b323094c1b..e1772c7734 100644 --- a/library/cpp/monlib/dynamic_counters/golovan_page.h +++ b/library/cpp/monlib/dynamic_counters/golovan_page.h @@ -18,7 +18,7 @@ public: TGolovanCountersPage(const TString& path, TIntrusivePtr<NMonitoring::TDynamicCounters> counters, TOutputCallback outputCallback = nullptr); - void Output(NMonitoring::IMonHttpRequest& request) override; + void Output(NMonitoring::IMonHttpRequest& request) override; private: TOutputCallback OutputCallback; diff --git a/library/cpp/monlib/dynamic_counters/page.cpp b/library/cpp/monlib/dynamic_counters/page.cpp index 85214ac883..5124a47bb3 100644 --- a/library/cpp/monlib/dynamic_counters/page.cpp +++ b/library/cpp/monlib/dynamic_counters/page.cpp @@ -14,7 +14,7 @@ namespace { currentCounters(nullptr); } -TMaybe<EFormat> ParseFormat(TStringBuf str) { +TMaybe<EFormat> ParseFormat(TStringBuf str) { if (str == TStringBuf("json")) { return EFormat::JSON; } else if (str == TStringBuf("spack")) { @@ -26,20 +26,20 @@ TMaybe<EFormat> ParseFormat(TStringBuf str) { } } -void TDynamicCountersPage::Output(NMonitoring::IMonHttpRequest& request) { +void TDynamicCountersPage::Output(NMonitoring::IMonHttpRequest& request) { if (OutputCallback) { OutputCallback(); } - TCountableBase::EVisibility visibility{ - TCountableBase::EVisibility::Public - }; - - TVector<TStringBuf> parts; - StringSplitter(request.GetPathInfo()) - .Split('/') - .SkipEmpty() - .Collect(&parts); + TCountableBase::EVisibility visibility{ + TCountableBase::EVisibility::Public + }; + + TVector<TStringBuf> parts; + StringSplitter(request.GetPathInfo()) + .Split('/') + .SkipEmpty() + .Collect(&parts); TMaybe<EFormat> format = !parts.empty() ? ParseFormat(parts.back()) : Nothing(); if (format) { @@ -47,29 +47,29 @@ void TDynamicCountersPage::Output(NMonitoring::IMonHttpRequest& request) { } if (!parts.empty() && parts.back() == TStringBuf("private")) { - visibility = TCountableBase::EVisibility::Private; - parts.pop_back(); - } - + visibility = TCountableBase::EVisibility::Private; + parts.pop_back(); + } + auto counters = Counters; - for (const auto& escaped : parts) { - const auto part = CGIUnescapeRet(escaped); - + for (const auto& escaped : parts) { + const auto part = CGIUnescapeRet(escaped); + TVector<TString> labels; - StringSplitter(part).Split('=').SkipEmpty().Collect(&labels); + StringSplitter(part).Split('=').SkipEmpty().Collect(&labels); if (labels.size() != 2U) return NotFound(request); - if (const auto child = counters->FindSubgroup( - labels.front(), - labels.back())) { - + if (const auto child = counters->FindSubgroup( + labels.front(), + labels.back())) { + counters = child; - } else { - return HandleAbsentSubgroup(request); - } + } else { + return HandleAbsentSubgroup(request); + } } if (!format) { @@ -95,21 +95,21 @@ void TDynamicCountersPage::Output(NMonitoring::IMonHttpRequest& request) { out.Flush(); } -void TDynamicCountersPage::HandleAbsentSubgroup(IMonHttpRequest& request) { - if (UnknownGroupPolicy == EUnknownGroupPolicy::Error) { - NotFound(request); - } else if (UnknownGroupPolicy == EUnknownGroupPolicy::Ignore) { - NoContent(request); - } else { - Y_FAIL("Unsupported policy set"); - } -} - -void TDynamicCountersPage::BeforePre(IMonHttpRequest& request) { - IOutputStream& out = request.Output(); +void TDynamicCountersPage::HandleAbsentSubgroup(IMonHttpRequest& request) { + if (UnknownGroupPolicy == EUnknownGroupPolicy::Error) { + NotFound(request); + } else if (UnknownGroupPolicy == EUnknownGroupPolicy::Ignore) { + NoContent(request); + } else { + Y_FAIL("Unsupported policy set"); + } +} + +void TDynamicCountersPage::BeforePre(IMonHttpRequest& request) { + IOutputStream& out = request.Output(); HTML(out) { DIV() { - out << "<a href='" << request.GetPath() << "/json'>Counters as JSON</a>"; + out << "<a href='" << request.GetPath() << "/json'>Counters as JSON</a>"; out << " for <a href='https://wiki.yandex-team.ru/solomon/'>Solomon</a>"; } @@ -117,13 +117,13 @@ void TDynamicCountersPage::BeforePre(IMonHttpRequest& request) { out << "Counters subgroups"; } UL() { - currentCounters->EnumerateSubgroups([&](const TString& name, const TString& value) { - LI() { - TString pathPart = name + "=" + value; - Quote(pathPart, ""); - out << "\n<a href='" << request.GetPath() << "/" << pathPart << "'>" << name << " " << value << "</a>"; - } - }); + currentCounters->EnumerateSubgroups([&](const TString& name, const TString& value) { + LI() { + TString pathPart = name + "=" + value; + Quote(pathPart, ""); + out << "\n<a href='" << request.GetPath() << "/" << pathPart << "'>" << name << " " << value << "</a>"; + } + }); } H4() { @@ -132,10 +132,10 @@ void TDynamicCountersPage::BeforePre(IMonHttpRequest& request) { } } -void TDynamicCountersPage::OutputText(IOutputStream& out, IMonHttpRequest&) { +void TDynamicCountersPage::OutputText(IOutputStream& out, IMonHttpRequest&) { currentCounters->OutputPlainText(out); } - -void TDynamicCountersPage::SetUnknownGroupPolicy(EUnknownGroupPolicy value) { - UnknownGroupPolicy = value; -} + +void TDynamicCountersPage::SetUnknownGroupPolicy(EUnknownGroupPolicy value) { + UnknownGroupPolicy = value; +} diff --git a/library/cpp/monlib/dynamic_counters/page.h b/library/cpp/monlib/dynamic_counters/page.h index 470d427b40..1f0ef6a5ea 100644 --- a/library/cpp/monlib/dynamic_counters/page.h +++ b/library/cpp/monlib/dynamic_counters/page.h @@ -9,11 +9,11 @@ #include <functional> namespace NMonitoring { - enum class EUnknownGroupPolicy { - Error, // send 404 - Ignore, // send 204 - }; - + enum class EUnknownGroupPolicy { + Error, // send 404 + Ignore, // send 204 + }; + struct TDynamicCountersPage: public TPreMonPage { public: using TOutputCallback = std::function<void()>; @@ -21,11 +21,11 @@ namespace NMonitoring { private: const TIntrusivePtr<TDynamicCounters> Counters; TOutputCallback OutputCallback; - EUnknownGroupPolicy UnknownGroupPolicy {EUnknownGroupPolicy::Error}; + EUnknownGroupPolicy UnknownGroupPolicy {EUnknownGroupPolicy::Error}; + + private: + void HandleAbsentSubgroup(IMonHttpRequest& request); - private: - void HandleAbsentSubgroup(IMonHttpRequest& request); - public: TDynamicCountersPage(const TString& path, const TString& title, @@ -37,14 +37,14 @@ namespace NMonitoring { { } - void Output(NMonitoring::IMonHttpRequest& request) override; + void Output(NMonitoring::IMonHttpRequest& request) override; + + void BeforePre(NMonitoring::IMonHttpRequest& request) override; - void BeforePre(NMonitoring::IMonHttpRequest& request) override; + void OutputText(IOutputStream& out, NMonitoring::IMonHttpRequest&) override; - void OutputText(IOutputStream& out, NMonitoring::IMonHttpRequest&) override; - - /// If set to Error, responds with 404 if the requested subgroup is not found. This is the default. - /// If set to Ignore, responds with 204 if the requested subgroup is not found - void SetUnknownGroupPolicy(EUnknownGroupPolicy value); + /// If set to Error, responds with 404 if the requested subgroup is not found. This is the default. + /// If set to Ignore, responds with 204 if the requested subgroup is not found + void SetUnknownGroupPolicy(EUnknownGroupPolicy value); }; } diff --git a/library/cpp/monlib/dynamic_counters/percentile/percentile.h b/library/cpp/monlib/dynamic_counters/percentile/percentile.h index 08b7ec5f0f..73c482bce9 100644 --- a/library/cpp/monlib/dynamic_counters/percentile/percentile.h +++ b/library/cpp/monlib/dynamic_counters/percentile/percentile.h @@ -1,59 +1,59 @@ -#pragma once - +#pragma once + #include "percentile_base.h" - -namespace NMonitoring { - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -// Percentile tracker for monitoring -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -template <size_t BUCKET_SIZE, size_t BUCKET_COUNT, size_t FRAME_COUNT> + +namespace NMonitoring { + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Percentile tracker for monitoring +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +template <size_t BUCKET_SIZE, size_t BUCKET_COUNT, size_t FRAME_COUNT> struct TPercentileTracker : public TPercentileBase { - TAtomic Items[BUCKET_COUNT]; - TAtomicBase Frame[FRAME_COUNT][BUCKET_COUNT]; - size_t CurrentFrame; - - TPercentileTracker() - : CurrentFrame(0) - { - for (size_t i = 0; i < BUCKET_COUNT; ++i) { - AtomicSet(Items[i], 0); - } - for (size_t frame = 0; frame < FRAME_COUNT; ++frame) { - for (size_t bucket = 0; bucket < BUCKET_COUNT; ++bucket) { - Frame[frame][bucket] = 0; - } - } - } - - void Increment(size_t value) { - AtomicIncrement(Items[Min((value + BUCKET_SIZE - 1) / BUCKET_SIZE, BUCKET_COUNT - 1)]); - } - - // shift frame (call periodically) - void Update() { - TVector<TAtomicBase> totals(BUCKET_COUNT); - totals.resize(BUCKET_COUNT); - TAtomicBase total = 0; - for (size_t i = 0; i < BUCKET_COUNT; ++i) { - TAtomicBase item = AtomicGet(Items[i]); - TAtomicBase prevItem = Frame[CurrentFrame][i]; - Frame[CurrentFrame][i] = item; - total += item - prevItem; - totals[i] = total; - } - - for (size_t i = 0; i < Percentiles.size(); ++i) { - TPercentile &percentile = Percentiles[i]; - auto threshold = (TAtomicBase)(percentile.first * (float)total); - threshold = Min(threshold, total); - auto it = LowerBound(totals.begin(), totals.end(), threshold); - size_t index = it - totals.begin(); - (*percentile.second) = index * BUCKET_SIZE; - } - CurrentFrame = (CurrentFrame + 1) % FRAME_COUNT; - } -}; - -} // NMonitoring + TAtomic Items[BUCKET_COUNT]; + TAtomicBase Frame[FRAME_COUNT][BUCKET_COUNT]; + size_t CurrentFrame; + + TPercentileTracker() + : CurrentFrame(0) + { + for (size_t i = 0; i < BUCKET_COUNT; ++i) { + AtomicSet(Items[i], 0); + } + for (size_t frame = 0; frame < FRAME_COUNT; ++frame) { + for (size_t bucket = 0; bucket < BUCKET_COUNT; ++bucket) { + Frame[frame][bucket] = 0; + } + } + } + + void Increment(size_t value) { + AtomicIncrement(Items[Min((value + BUCKET_SIZE - 1) / BUCKET_SIZE, BUCKET_COUNT - 1)]); + } + + // shift frame (call periodically) + void Update() { + TVector<TAtomicBase> totals(BUCKET_COUNT); + totals.resize(BUCKET_COUNT); + TAtomicBase total = 0; + for (size_t i = 0; i < BUCKET_COUNT; ++i) { + TAtomicBase item = AtomicGet(Items[i]); + TAtomicBase prevItem = Frame[CurrentFrame][i]; + Frame[CurrentFrame][i] = item; + total += item - prevItem; + totals[i] = total; + } + + for (size_t i = 0; i < Percentiles.size(); ++i) { + TPercentile &percentile = Percentiles[i]; + auto threshold = (TAtomicBase)(percentile.first * (float)total); + threshold = Min(threshold, total); + auto it = LowerBound(totals.begin(), totals.end(), threshold); + size_t index = it - totals.begin(); + (*percentile.second) = index * BUCKET_SIZE; + } + CurrentFrame = (CurrentFrame + 1) % FRAME_COUNT; + } +}; + +} // NMonitoring diff --git a/library/cpp/monlib/dynamic_counters/percentile/percentile_lg.h b/library/cpp/monlib/dynamic_counters/percentile/percentile_lg.h index b39c53e3df..0042cd9a6a 100644 --- a/library/cpp/monlib/dynamic_counters/percentile/percentile_lg.h +++ b/library/cpp/monlib/dynamic_counters/percentile/percentile_lg.h @@ -1,182 +1,182 @@ -#pragma once - +#pragma once + #include <library/cpp/containers/stack_vector/stack_vec.h> - -#include <util/generic/bitops.h> - -#include <cmath> - + +#include <util/generic/bitops.h> + +#include <cmath> + #include "percentile_base.h" -namespace NMonitoring { - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -// Percentile tracker for monitoring -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -template <size_t BASE_BITS, size_t EXP_BITS, size_t FRAME_COUNT> +namespace NMonitoring { + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Percentile tracker for monitoring +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +template <size_t BASE_BITS, size_t EXP_BITS, size_t FRAME_COUNT> struct TPercentileTrackerLg : public TPercentileBase { static constexpr size_t BUCKET_COUNT = size_t(1) << EXP_BITS; static constexpr size_t BUCKET_SIZE = size_t(1) << BASE_BITS; - static constexpr size_t ITEMS_COUNT = BUCKET_COUNT * BUCKET_SIZE; + static constexpr size_t ITEMS_COUNT = BUCKET_COUNT * BUCKET_SIZE; static constexpr size_t TRACKER_LIMIT = BUCKET_SIZE * ((size_t(1) << BUCKET_COUNT) - 1) - (size_t(1) << (BUCKET_COUNT - 1)); static constexpr size_t MAX_GRANULARITY = size_t(1) << (BUCKET_COUNT - 1); - size_t Borders[BUCKET_COUNT]; - TAtomic Items[ITEMS_COUNT]; - TAtomicBase Frame[FRAME_COUNT][ITEMS_COUNT]; - size_t CurrentFrame; - - TPercentileTrackerLg() - : CurrentFrame(0) - { - Borders[0] = 0; - for (size_t i = 1; i < BUCKET_COUNT; ++i) { - Borders[i] = Borders[i-1] + (BUCKET_SIZE << (i - 1)); - } - for (size_t i = 0; i < ITEMS_COUNT; ++i) { - AtomicSet(Items[i], 0); - } - for (size_t frame = 0; frame < FRAME_COUNT; ++frame) { - for (size_t bucket = 0; bucket < ITEMS_COUNT; ++bucket) { - Frame[frame][bucket] = 0; - } - } - } - - size_t inline BucketIdxIf(size_t value) { - static_assert(BASE_BITS == 5, "if-based bucket calculation cannot be used if BASE_BITS != 5"); - size_t bucket_idx; - if (value < 8160) { - if (value < 480) { - if (value < 96) { - if (value < 32) { - bucket_idx = 0; - } else { - bucket_idx = 1; - } - } else { - if (value < 224) { - bucket_idx = 2; - } else { - bucket_idx = 3; - } - } - } else { - if (value < 2016) { - if (value < 992) { - bucket_idx = 4; - } else { - bucket_idx = 5; - } - } else { - if (value < 4064) { - bucket_idx = 6; - } else { - bucket_idx = 7; - } - } - } - } else { - if (value < 131040) { - if (value < 32736) { - if (value < 16352) { - bucket_idx = 8; - } else { - bucket_idx = 9; - } - } else { - if (value < 65504) { - bucket_idx = 10; - } else { - bucket_idx = 11; - } - } - } else { - if (value < 524256) { - if (value < 262112) { - bucket_idx = 12; - } else { - bucket_idx = 13; - } - } else { - if (value < 1048544) { - bucket_idx = 14; - } else { - bucket_idx = 15; - } - } - } - } - return Min(bucket_idx, BUCKET_COUNT - 1); - } - - size_t inline BucketIdxBinarySearch(size_t value) { - size_t l = 0; - size_t r = BUCKET_COUNT; - while (l < r - 1) { - size_t mid = (l + r) / 2; - if (value < Borders[mid]) { - r = mid; - } else { - l = mid; - } - } - return l; - } - - size_t inline BucketIdxMostSignificantBit(size_t value) { - size_t bucket_idx = MostSignificantBit(value + BUCKET_SIZE) - BASE_BITS; - return Min(bucket_idx, BUCKET_COUNT - 1); - } - - void Increment(size_t value) { + size_t Borders[BUCKET_COUNT]; + TAtomic Items[ITEMS_COUNT]; + TAtomicBase Frame[FRAME_COUNT][ITEMS_COUNT]; + size_t CurrentFrame; + + TPercentileTrackerLg() + : CurrentFrame(0) + { + Borders[0] = 0; + for (size_t i = 1; i < BUCKET_COUNT; ++i) { + Borders[i] = Borders[i-1] + (BUCKET_SIZE << (i - 1)); + } + for (size_t i = 0; i < ITEMS_COUNT; ++i) { + AtomicSet(Items[i], 0); + } + for (size_t frame = 0; frame < FRAME_COUNT; ++frame) { + for (size_t bucket = 0; bucket < ITEMS_COUNT; ++bucket) { + Frame[frame][bucket] = 0; + } + } + } + + size_t inline BucketIdxIf(size_t value) { + static_assert(BASE_BITS == 5, "if-based bucket calculation cannot be used if BASE_BITS != 5"); + size_t bucket_idx; + if (value < 8160) { + if (value < 480) { + if (value < 96) { + if (value < 32) { + bucket_idx = 0; + } else { + bucket_idx = 1; + } + } else { + if (value < 224) { + bucket_idx = 2; + } else { + bucket_idx = 3; + } + } + } else { + if (value < 2016) { + if (value < 992) { + bucket_idx = 4; + } else { + bucket_idx = 5; + } + } else { + if (value < 4064) { + bucket_idx = 6; + } else { + bucket_idx = 7; + } + } + } + } else { + if (value < 131040) { + if (value < 32736) { + if (value < 16352) { + bucket_idx = 8; + } else { + bucket_idx = 9; + } + } else { + if (value < 65504) { + bucket_idx = 10; + } else { + bucket_idx = 11; + } + } + } else { + if (value < 524256) { + if (value < 262112) { + bucket_idx = 12; + } else { + bucket_idx = 13; + } + } else { + if (value < 1048544) { + bucket_idx = 14; + } else { + bucket_idx = 15; + } + } + } + } + return Min(bucket_idx, BUCKET_COUNT - 1); + } + + size_t inline BucketIdxBinarySearch(size_t value) { + size_t l = 0; + size_t r = BUCKET_COUNT; + while (l < r - 1) { + size_t mid = (l + r) / 2; + if (value < Borders[mid]) { + r = mid; + } else { + l = mid; + } + } + return l; + } + + size_t inline BucketIdxMostSignificantBit(size_t value) { + size_t bucket_idx = MostSignificantBit(value + BUCKET_SIZE) - BASE_BITS; + return Min(bucket_idx, BUCKET_COUNT - 1); + } + + void Increment(size_t value) { size_t bucket_idx = BucketIdxMostSignificantBit(value); - size_t inside_bucket_idx = (value - Borders[bucket_idx] + (1 << bucket_idx) - 1) >> bucket_idx; - size_t idx = bucket_idx * BUCKET_SIZE + inside_bucket_idx; - AtomicIncrement(Items[Min(idx, ITEMS_COUNT - 1)]); - } - - // Needed only for tests - size_t GetPercentile(float threshold) { - TStackVec<TAtomicBase, ITEMS_COUNT> totals(ITEMS_COUNT); - TAtomicBase total = 0; - for (size_t i = 0; i < ITEMS_COUNT; ++i) { - total += AtomicGet(Items[i]); - totals[i] = total; - } - TAtomicBase item_threshold = std::llround(threshold * (float)total); - item_threshold = Min(item_threshold, total); - auto it = LowerBound(totals.begin(), totals.end(), item_threshold); - size_t index = it - totals.begin(); - size_t bucket_idx = index / BUCKET_SIZE; - return Borders[bucket_idx] + ((index % BUCKET_SIZE) << bucket_idx); - } - - // shift frame (call periodically) - void Update() { - TStackVec<TAtomicBase, ITEMS_COUNT> totals(ITEMS_COUNT); - TAtomicBase total = 0; - for (size_t i = 0; i < ITEMS_COUNT; ++i) { - TAtomicBase item = AtomicGet(Items[i]); - TAtomicBase prevItem = Frame[CurrentFrame][i]; - Frame[CurrentFrame][i] = item; - total += item - prevItem; - totals[i] = total; - } - - for (size_t i = 0; i < Percentiles.size(); ++i) { - TPercentile &percentile = Percentiles[i]; - TAtomicBase threshold = std::llround(percentile.first * (float)total); - threshold = Min(threshold, total); - auto it = LowerBound(totals.begin(), totals.end(), threshold); - size_t index = it - totals.begin(); - size_t bucket_idx = index / BUCKET_SIZE; - (*percentile.second) = Borders[bucket_idx] + ((index % BUCKET_SIZE) << bucket_idx); - } - CurrentFrame = (CurrentFrame + 1) % FRAME_COUNT; - } -}; - -} // NMonitoring + size_t inside_bucket_idx = (value - Borders[bucket_idx] + (1 << bucket_idx) - 1) >> bucket_idx; + size_t idx = bucket_idx * BUCKET_SIZE + inside_bucket_idx; + AtomicIncrement(Items[Min(idx, ITEMS_COUNT - 1)]); + } + + // Needed only for tests + size_t GetPercentile(float threshold) { + TStackVec<TAtomicBase, ITEMS_COUNT> totals(ITEMS_COUNT); + TAtomicBase total = 0; + for (size_t i = 0; i < ITEMS_COUNT; ++i) { + total += AtomicGet(Items[i]); + totals[i] = total; + } + TAtomicBase item_threshold = std::llround(threshold * (float)total); + item_threshold = Min(item_threshold, total); + auto it = LowerBound(totals.begin(), totals.end(), item_threshold); + size_t index = it - totals.begin(); + size_t bucket_idx = index / BUCKET_SIZE; + return Borders[bucket_idx] + ((index % BUCKET_SIZE) << bucket_idx); + } + + // shift frame (call periodically) + void Update() { + TStackVec<TAtomicBase, ITEMS_COUNT> totals(ITEMS_COUNT); + TAtomicBase total = 0; + for (size_t i = 0; i < ITEMS_COUNT; ++i) { + TAtomicBase item = AtomicGet(Items[i]); + TAtomicBase prevItem = Frame[CurrentFrame][i]; + Frame[CurrentFrame][i] = item; + total += item - prevItem; + totals[i] = total; + } + + for (size_t i = 0; i < Percentiles.size(); ++i) { + TPercentile &percentile = Percentiles[i]; + TAtomicBase threshold = std::llround(percentile.first * (float)total); + threshold = Min(threshold, total); + auto it = LowerBound(totals.begin(), totals.end(), threshold); + size_t index = it - totals.begin(); + size_t bucket_idx = index / BUCKET_SIZE; + (*percentile.second) = Borders[bucket_idx] + ((index % BUCKET_SIZE) << bucket_idx); + } + CurrentFrame = (CurrentFrame + 1) % FRAME_COUNT; + } +}; + +} // NMonitoring diff --git a/library/cpp/monlib/dynamic_counters/percentile/percentile_ut.cpp b/library/cpp/monlib/dynamic_counters/percentile/percentile_ut.cpp index b09004bd7f..6c8bb54ec9 100644 --- a/library/cpp/monlib/dynamic_counters/percentile/percentile_ut.cpp +++ b/library/cpp/monlib/dynamic_counters/percentile/percentile_ut.cpp @@ -1,10 +1,10 @@ -#include "percentile_lg.h" +#include "percentile_lg.h" #include <library/cpp/testing/unittest/registar.h> - -using namespace NMonitoring; - -Y_UNIT_TEST_SUITE(PercentileTest) { - + +using namespace NMonitoring; + +Y_UNIT_TEST_SUITE(PercentileTest) { + template<size_t A, size_t B, size_t B_BEGIN> void printSizeAndLimit() { using TPerc = TPercentileTrackerLg<A, B, 15>; @@ -57,73 +57,73 @@ void printSizeAndLimit() { } } - Y_UNIT_TEST(BucketIdxIfvsBucketIdxBinarySearch) { - for (size_t var = 0; var < 5; var++) { - if (var == 0) { - TPercentileTrackerLg<3, 2, 15> tracker; - for (size_t i = 0; i < 3000000; i += 1) { - size_t num1 = tracker.BucketIdxMostSignificantBit(i); - size_t num2 = tracker.BucketIdxBinarySearch(i); - UNIT_ASSERT_EQUAL(num1, num2); - } - } else if (var == 1) { - TPercentileTrackerLg<4, 4, 15> tracker; - for (size_t i = 0; i < 3000000; i += 1) { - size_t num1 = tracker.BucketIdxMostSignificantBit(i); - size_t num2 = tracker.BucketIdxBinarySearch(i); - UNIT_ASSERT_EQUAL(num1, num2); - } - } else if (var == 2) { - TPercentileTrackerLg<5, 3, 15> tracker; - for (size_t i = 0; i < 3000000; i += 1) { - size_t num1 = tracker.BucketIdxMostSignificantBit(i); - size_t num2 = tracker.BucketIdxBinarySearch(i); - size_t num3 = tracker.BucketIdxIf(i); - UNIT_ASSERT_EQUAL(num1, num2); - UNIT_ASSERT_EQUAL(num2, num3); - } - } else if (var == 3) { - TPercentileTrackerLg<5, 4, 15> tracker; - for (size_t i = 0; i < 3000000; i += 1) { - size_t num1 = tracker.BucketIdxMostSignificantBit(i); - size_t num2 = tracker.BucketIdxBinarySearch(i); - size_t num3 = tracker.BucketIdxIf(i); - UNIT_ASSERT_EQUAL(num1, num2); - UNIT_ASSERT_EQUAL(num2, num3); - } - } else if (var == 4) { - TPercentileTrackerLg<6, 5, 15> tracker; - for (size_t i = 0; i < 3000000; i += 1) { - size_t num1 = tracker.BucketIdxMostSignificantBit(i); - size_t num2 = tracker.BucketIdxBinarySearch(i); - UNIT_ASSERT_EQUAL(num1, num2); - } - for (size_t i = 0; i < 400000000000ul; i += 1303) { - size_t num1 = tracker.BucketIdxMostSignificantBit(i); - size_t num2 = tracker.BucketIdxBinarySearch(i); - UNIT_ASSERT_EQUAL(num1, num2); - } - } - } - } - - Y_UNIT_TEST(DifferentPercentiles) { - TPercentileTrackerLg<5, 4, 15> tracker; - TVector<size_t> values({0, 115, 1216, 15, 3234567, 1234567, 216546, 263421, 751654, 96, 224, 223, 225}); - TVector<size_t> percentiles50({0, 0, 116, 15, 116, 116, 1216, 1216, 217056, 1216, 1216, 224, 232}); - TVector<size_t> percentiles75({0, 116, 116, 116, 1216, 1245152, 217056, 270304, 753632, 753632, - 270304, 270304, 270304}); - TVector<size_t> percentiles90({ 0, 116, 1216, 1216, 2064352, 1245152, 1245152, 1245152, 1245152, - 1245152, 1245152, 1245152, 1245152}); - TVector<size_t> percentiles100({ 0, 116, 1216, 1216, 2064352, 2064352, 2064352, 2064352, 2064352, - 2064352, 2064352, 2064352, 2064352 }); - - for (size_t i = 0; i < values.size(); ++i) { - tracker.Increment(values[i]); - UNIT_ASSERT_EQUAL(tracker.GetPercentile(0.5), percentiles50[i]); - UNIT_ASSERT_EQUAL(tracker.GetPercentile(0.75), percentiles75[i]); - UNIT_ASSERT_EQUAL(tracker.GetPercentile(0.90), percentiles90[i]); - UNIT_ASSERT_EQUAL(tracker.GetPercentile(1.0), percentiles100[i]); - } - } -} + Y_UNIT_TEST(BucketIdxIfvsBucketIdxBinarySearch) { + for (size_t var = 0; var < 5; var++) { + if (var == 0) { + TPercentileTrackerLg<3, 2, 15> tracker; + for (size_t i = 0; i < 3000000; i += 1) { + size_t num1 = tracker.BucketIdxMostSignificantBit(i); + size_t num2 = tracker.BucketIdxBinarySearch(i); + UNIT_ASSERT_EQUAL(num1, num2); + } + } else if (var == 1) { + TPercentileTrackerLg<4, 4, 15> tracker; + for (size_t i = 0; i < 3000000; i += 1) { + size_t num1 = tracker.BucketIdxMostSignificantBit(i); + size_t num2 = tracker.BucketIdxBinarySearch(i); + UNIT_ASSERT_EQUAL(num1, num2); + } + } else if (var == 2) { + TPercentileTrackerLg<5, 3, 15> tracker; + for (size_t i = 0; i < 3000000; i += 1) { + size_t num1 = tracker.BucketIdxMostSignificantBit(i); + size_t num2 = tracker.BucketIdxBinarySearch(i); + size_t num3 = tracker.BucketIdxIf(i); + UNIT_ASSERT_EQUAL(num1, num2); + UNIT_ASSERT_EQUAL(num2, num3); + } + } else if (var == 3) { + TPercentileTrackerLg<5, 4, 15> tracker; + for (size_t i = 0; i < 3000000; i += 1) { + size_t num1 = tracker.BucketIdxMostSignificantBit(i); + size_t num2 = tracker.BucketIdxBinarySearch(i); + size_t num3 = tracker.BucketIdxIf(i); + UNIT_ASSERT_EQUAL(num1, num2); + UNIT_ASSERT_EQUAL(num2, num3); + } + } else if (var == 4) { + TPercentileTrackerLg<6, 5, 15> tracker; + for (size_t i = 0; i < 3000000; i += 1) { + size_t num1 = tracker.BucketIdxMostSignificantBit(i); + size_t num2 = tracker.BucketIdxBinarySearch(i); + UNIT_ASSERT_EQUAL(num1, num2); + } + for (size_t i = 0; i < 400000000000ul; i += 1303) { + size_t num1 = tracker.BucketIdxMostSignificantBit(i); + size_t num2 = tracker.BucketIdxBinarySearch(i); + UNIT_ASSERT_EQUAL(num1, num2); + } + } + } + } + + Y_UNIT_TEST(DifferentPercentiles) { + TPercentileTrackerLg<5, 4, 15> tracker; + TVector<size_t> values({0, 115, 1216, 15, 3234567, 1234567, 216546, 263421, 751654, 96, 224, 223, 225}); + TVector<size_t> percentiles50({0, 0, 116, 15, 116, 116, 1216, 1216, 217056, 1216, 1216, 224, 232}); + TVector<size_t> percentiles75({0, 116, 116, 116, 1216, 1245152, 217056, 270304, 753632, 753632, + 270304, 270304, 270304}); + TVector<size_t> percentiles90({ 0, 116, 1216, 1216, 2064352, 1245152, 1245152, 1245152, 1245152, + 1245152, 1245152, 1245152, 1245152}); + TVector<size_t> percentiles100({ 0, 116, 1216, 1216, 2064352, 2064352, 2064352, 2064352, 2064352, + 2064352, 2064352, 2064352, 2064352 }); + + for (size_t i = 0; i < values.size(); ++i) { + tracker.Increment(values[i]); + UNIT_ASSERT_EQUAL(tracker.GetPercentile(0.5), percentiles50[i]); + UNIT_ASSERT_EQUAL(tracker.GetPercentile(0.75), percentiles75[i]); + UNIT_ASSERT_EQUAL(tracker.GetPercentile(0.90), percentiles90[i]); + UNIT_ASSERT_EQUAL(tracker.GetPercentile(1.0), percentiles100[i]); + } + } +} diff --git a/library/cpp/monlib/dynamic_counters/percentile/ut/ya.make b/library/cpp/monlib/dynamic_counters/percentile/ut/ya.make index 94faeee43d..f9f3564101 100644 --- a/library/cpp/monlib/dynamic_counters/percentile/ut/ya.make +++ b/library/cpp/monlib/dynamic_counters/percentile/ut/ya.make @@ -1,9 +1,9 @@ UNITTEST_FOR(library/cpp/monlib/dynamic_counters/percentile) - - OWNER(alexvru g:kikimr g:solomon) - - SRCS( - percentile_ut.cpp - ) - -END() + + OWNER(alexvru g:kikimr g:solomon) + + SRCS( + percentile_ut.cpp + ) + +END() diff --git a/library/cpp/monlib/dynamic_counters/percentile/ya.make b/library/cpp/monlib/dynamic_counters/percentile/ya.make index b255fc21fc..cb52cdd9ad 100644 --- a/library/cpp/monlib/dynamic_counters/percentile/ya.make +++ b/library/cpp/monlib/dynamic_counters/percentile/ya.make @@ -1,15 +1,15 @@ -LIBRARY() - - OWNER(alexvru g:kikimr g:solomon) - - SRCS( - percentile.h - percentile_lg.h - ) - - PEERDIR( +LIBRARY() + + OWNER(alexvru g:kikimr g:solomon) + + SRCS( + percentile.h + percentile_lg.h + ) + + PEERDIR( library/cpp/containers/stack_vector library/cpp/monlib/dynamic_counters - ) - -END() + ) + +END() |