diff options
author | Devtools Arcadia <arcadia-devtools@yandex-team.ru> | 2022-02-07 18:08:42 +0300 |
---|---|---|
committer | Devtools Arcadia <arcadia-devtools@mous.vla.yp-c.yandex.net> | 2022-02-07 18:08:42 +0300 |
commit | 1110808a9d39d4b808aef724c861a2e1a38d2a69 (patch) | |
tree | e26c9fed0de5d9873cce7e00bc214573dc2195b7 /library/cpp/monlib | |
download | ydb-1110808a9d39d4b808aef724c861a2e1a38d2a69.tar.gz |
intermediate changes
ref:cde9a383711a11544ce7e107a78147fb96cc4029
Diffstat (limited to 'library/cpp/monlib')
231 files changed, 24288 insertions, 0 deletions
diff --git a/library/cpp/monlib/counters/counters.cpp b/library/cpp/monlib/counters/counters.cpp new file mode 100644 index 0000000000..50dca4c577 --- /dev/null +++ b/library/cpp/monlib/counters/counters.cpp @@ -0,0 +1,49 @@ + +#include "counters.h" + +namespace NMonitoring { + char* PrettyNumShort(i64 val, char* buf, size_t size) { + static const char shorts[] = {' ', 'K', 'M', 'G', 'T', 'P', 'E'}; + unsigned i = 0; + i64 major = val; + i64 minor = 0; + const unsigned imax = sizeof(shorts) / sizeof(char); + for (i = 0; i < imax; i++) { + if (major >> 10 == 0) + break; + else { + minor = major - (major >> 10 << 10); + major = major >> 10; + } + } + minor = (minor * 10) >> 10; + + if (i == 0 || i >= imax) + *buf = '\0'; + else + snprintf(buf, size, "%" PRId64 ".%" PRId64 "%c", major, minor, shorts[i]); + + return buf; + } + + char* PrettyNum(i64 val, char* buf, size_t size) { + Y_ASSERT(buf); + if (size < 4) { + buf[0] = 0; + return buf; + } + PrettyNumShort(val, buf + 2, size - 3); + if (buf[2] == 0) { + *buf = '\0'; + } else { + size_t len = 2 + strnlen(buf + 2, size - 4); + Y_ASSERT(len < size); + buf[0] = ' '; + buf[1] = '('; + buf[len] = ')'; + buf[len + 1] = '\0'; + } + + return buf; + } +} diff --git a/library/cpp/monlib/counters/counters.h b/library/cpp/monlib/counters/counters.h new file mode 100644 index 0000000000..038b55f0c8 --- /dev/null +++ b/library/cpp/monlib/counters/counters.h @@ -0,0 +1,350 @@ +#pragma once + +#include <util/datetime/base.h> +#include <util/generic/algorithm.h> +#include <util/generic/list.h> +#include <util/generic/map.h> +#include <util/generic/ptr.h> +#include <util/generic/singleton.h> +#include <util/generic/vector.h> +#include <util/str_stl.h> +#include <util/stream/output.h> +#include <util/string/util.h> +#include <util/system/atomic.h> +#include <util/system/defaults.h> +#include <util/system/guard.h> +#include <util/system/sem.h> +#include <util/system/spinlock.h> + +#include <array> + +namespace NMonitoring { +#define BEGIN_OUTPUT_COUNTERS \ + void OutputImpl(IOutputStream& out) { \ + char prettyBuf[32]; +#define END_OUTPUT_COUNTERS \ + out.Flush(); \ + } + +#define OUTPUT_NAMED_COUNTER(var, name) out << name << ": \t" << var << NMonitoring::PrettyNum(var, prettyBuf, 32) << '\n' +#define OUTPUT_COUNTER(var) OUTPUT_NAMED_COUNTER(var, #var); + + char* PrettyNumShort(i64 val, char* buf, size_t size); + char* PrettyNum(i64 val, char* buf, size_t size); + + // This class is deprecated. Please consider to use + // library/cpp/monlib/metrics instead. See more info at + // https://wiki.yandex-team.ru/solomon/libs/monlib_cpp/ + class TDeprecatedCounter { + public: + using TValue = TAtomic; + using TValueBase = TAtomicBase; + + TDeprecatedCounter() + : Value() + , Derivative(false) + { + } + + TDeprecatedCounter(TValue value, bool derivative = false) + : Value(value) + , Derivative(derivative) + { + } + + bool ForDerivative() const { + return Derivative; + } + + operator TValueBase() const { + return AtomicGet(Value); + } + TValueBase Val() const { + return AtomicGet(Value); + } + + void Set(TValue val) { + AtomicSet(Value, val); + } + + TValueBase Inc() { + return AtomicIncrement(Value); + } + TValueBase Dec() { + return AtomicDecrement(Value); + } + + TValueBase Add(const TValue val) { + return AtomicAdd(Value, val); + } + TValueBase Sub(const TValue val) { + return AtomicAdd(Value, -val); + } + + // operator overloads convinient + void operator++() { + Inc(); + } + void operator++(int) { + Inc(); + } + + void operator--() { + Dec(); + } + void operator--(int) { + Dec(); + } + + void operator+=(TValue rhs) { + Add(rhs); + } + void operator-=(TValue rhs) { + Sub(rhs); + } + + TValueBase operator=(TValue rhs) { + AtomicSwap(&Value, rhs); + return rhs; + } + + bool operator!() const { + return AtomicGet(Value) == 0; + } + + TAtomic& GetAtomic() { + return Value; + } + + private: + TAtomic Value; + bool Derivative; + }; + + template <typename T> + struct TDeprecatedCountersBase { + virtual ~TDeprecatedCountersBase() { + } + + virtual void OutputImpl(IOutputStream&) = 0; + + static T& Instance() { + return *Singleton<T>(); + } + + static void Output(IOutputStream& out) { + Instance().OutputImpl(out); + } + }; + + // This class is deprecated. Please consider to use + // library/cpp/monlib/metrics instead. See more info at + // https://wiki.yandex-team.ru/solomon/libs/monlib_cpp/ + // + // Groups of G counters, defined by T type. + // Less(a,b) returns true, if a < b. + // It's threadsafe. + template <typename T, typename G, typename TL = TLess<T>> + class TDeprecatedCounterGroups { + public: + typedef TMap<T, G*> TGroups; + typedef TVector<T> TGroupsNames; + typedef THolder<TGroupsNames> TGroupsNamesPtr; + + private: + class TCollection { + struct TElement { + T* Name; + G* Counters; + + public: + static bool Compare(const TElement& a, const TElement& b) { + return Less(*(a.Name), *(b.Name)); + } + }; // TElement + private: + TArrayHolder<TElement> Elements; + size_t Size; + + public: + TCollection() + : Size(0) + { + } + + TCollection(const TCollection& collection) + : Elements(new TElement[collection.Size]) + , Size(collection.Size) + { + for (int i = 0; i < Size; ++i) { + Elements[i] = collection.Elements[i]; + } + } + + TCollection(const TCollection& collection, T* name, G* counters) + : Elements(new TElement[collection.Size + 1]) + , Size(collection.Size + 1) + { + for (size_t i = 0; i < Size - 1; ++i) { + Elements[i] = collection.Elements[i]; + } + Elements[Size - 1].Name = name; + Elements[Size - 1].Counters = counters; + for (size_t i = 1; i < Size; ++i) { + size_t j = i; + while (j > 0 && + TElement::Compare(Elements[j], Elements[j - 1])) { + std::swap(Elements[j], Elements[j - 1]); + --j; + } + } + } + + G* Find(const T& name) const { + G* result = nullptr; + if (Size == 0) { + return nullptr; + } + size_t l = 0; + size_t r = Size - 1; + while (l < r) { + size_t m = (l + r) / 2; + if (Less(*(Elements[m].Name), name)) { + l = m + 1; + } else { + r = m; + } + } + if (!Less(*(Elements[l].Name), name) && !Less(name, *(Elements[l].Name))) { + result = Elements[l].Counters; + } + return result; + } + + void Free() { + for (size_t i = 0; i < Size; ++i) { + T* name = Elements[i].Name; + G* counters = Elements[i].Counters; + Elements[i].Name = nullptr; + Elements[i].Counters = nullptr; + delete name; + delete counters; + } + Size = 0; + } + + TGroupsNamesPtr GetNames() const { + TGroupsNamesPtr result(new TGroupsNames()); + for (size_t i = 0; i < Size; ++i) { + result->push_back(*(Elements[i].Name)); + } + return result; + } + }; // TCollection + struct TOldGroup { + TCollection* Collection; + ui64 Time; + }; + + private: + TCollection* Groups; + TList<TOldGroup> OldGroups; + TSpinLock AddMutex; + + ui64 Timeout; + + static TL Less; + + private: + G* Add(const T& name) { + TGuard<TSpinLock> guard(AddMutex); + G* result = Groups->Find(name); + if (result == nullptr) { + T* newName = new T(name); + G* newCounters = new G(); + TCollection* newGroups = + new TCollection(*Groups, newName, newCounters); + ui64 now = ::Now().MicroSeconds(); + TOldGroup group; + group.Collection = Groups; + group.Time = now; + OldGroups.push_back(group); + for (ui32 i = 0; i < 5; ++i) { + if (OldGroups.front().Time + Timeout < now) { + delete OldGroups.front().Collection; + OldGroups.front().Collection = nullptr; + OldGroups.pop_front(); + } else { + break; + } + } + Groups = newGroups; + result = Groups->Find(name); + } + return result; + } + + public: + TDeprecatedCounterGroups(ui64 timeout = 5 * 1000000L) { + Groups = new TCollection(); + Timeout = timeout; + } + + virtual ~TDeprecatedCounterGroups() { + TGuard<TSpinLock> guard(AddMutex); + Groups->Free(); + delete Groups; + Groups = nullptr; + typename TList<TOldGroup>::iterator i; + for (i = OldGroups.begin(); i != OldGroups.end(); ++i) { + delete i->Collection; + i->Collection = nullptr; + } + OldGroups.clear(); + } + + bool Has(const T& name) const { + TCollection* groups = Groups; + return groups->Find(name) != nullptr; + } + + G* Find(const T& name) const { + TCollection* groups = Groups; + return groups->Find(name); + } + + // Get group with the name, if it exists. + // If there is no group with the name, add new group. + G& Get(const T& name) { + G* result = Find(name); + if (result == nullptr) { + result = Add(name); + Y_ASSERT(result != nullptr); + } + return *result; + } + + // Get copy of groups names array. + TGroupsNamesPtr GetGroupsNames() const { + TCollection* groups = Groups; + TGroupsNamesPtr result = groups->GetNames(); + return result; + } + }; // TDeprecatedCounterGroups + + template <typename T, typename G, typename TL> + TL TDeprecatedCounterGroups<T, G, TL>::Less; +} + +static inline IOutputStream& operator<<(IOutputStream& o, const NMonitoring::TDeprecatedCounter& rhs) { + return o << rhs.Val(); +} + +template <size_t N> +static inline IOutputStream& operator<<(IOutputStream& o, const std::array<NMonitoring::TDeprecatedCounter, N>& rhs) { + for (typename std::array<NMonitoring::TDeprecatedCounter, N>::const_iterator it = rhs.begin(); it != rhs.end(); ++it) { + if (!!*it) + o << *it << Endl; + } + return o; +} diff --git a/library/cpp/monlib/counters/counters_ut.cpp b/library/cpp/monlib/counters/counters_ut.cpp new file mode 100644 index 0000000000..2845efb97b --- /dev/null +++ b/library/cpp/monlib/counters/counters_ut.cpp @@ -0,0 +1,49 @@ +#include "counters.h" + +#include <library/cpp/testing/unittest/registar.h> + +#include <util/generic/set.h> +#include <util/thread/pool.h> + +using namespace NMonitoring; + +Y_UNIT_TEST_SUITE(TDeprecatedCountersTest) { + Y_UNIT_TEST(CounterGroupsAreThreadSafe) { + const static ui32 GROUPS_COUNT = 1000; + const static ui32 THREADS_COUNT = 10; + + TDeprecatedCounterGroups<ui32, ui32> groups; + + auto adder = [&groups]() { + for (ui32 id = 0; id < GROUPS_COUNT; id++) { + groups.Get(id); + + // adds contention + ::NanoSleep(42); + } + }; + + TThreadPool q; + q.Start(THREADS_COUNT); + for (ui32 i = 0; i < THREADS_COUNT; i++) { + q.SafeAddFunc(adder); + } + q.Stop(); + + // each group id is present + for (ui32 id = 0; id < GROUPS_COUNT; id++) { + UNIT_ASSERT(groups.Has(id)); + } + + // group names contains only appropriate ids + auto ids = groups.GetGroupsNames(); + for (ui32 id : *ids) { + UNIT_ASSERT(id < GROUPS_COUNT); + } + + // no duplication in group names + TSet<ui32> uniqueIds(ids->begin(), ids->end()); + UNIT_ASSERT_EQUAL(ids->size(), uniqueIds.size()); + UNIT_ASSERT_EQUAL(ids->size(), GROUPS_COUNT); + } +} diff --git a/library/cpp/monlib/counters/histogram.cpp b/library/cpp/monlib/counters/histogram.cpp new file mode 100644 index 0000000000..46cf4e6ec8 --- /dev/null +++ b/library/cpp/monlib/counters/histogram.cpp @@ -0,0 +1,40 @@ +#include "histogram.h" + +namespace NMonitoring { + void THistogramSnapshot::Print(IOutputStream* out) const { + (*out) << "mean: " << Mean + << ", stddev: " << StdDeviation + << ", min: " << Min + << ", max: " << Max + << ", 50%: " << Percentile50 + << ", 75%: " << Percentile75 + << ", 90%: " << Percentile90 + << ", 95%: " << Percentile95 + << ", 98%: " << Percentile98 + << ", 99%: " << Percentile99 + << ", 99.9%: " << Percentile999 + << ", count: " << TotalCount; + } + + void THdrHistogram::TakeSnaphot(THistogramSnapshot* snapshot) { + with_lock (Lock_) { + // TODO: get data with single traverse + snapshot->Mean = Data_.GetMean(); + snapshot->StdDeviation = Data_.GetStdDeviation(); + snapshot->Min = Data_.GetMin(); + snapshot->Max = Data_.GetMax(); + snapshot->Percentile50 = Data_.GetValueAtPercentile(50.0); + snapshot->Percentile75 = Data_.GetValueAtPercentile(75.0); + snapshot->Percentile90 = Data_.GetValueAtPercentile(90.0); + snapshot->Percentile95 = Data_.GetValueAtPercentile(95.0); + snapshot->Percentile98 = Data_.GetValueAtPercentile(98.0); + snapshot->Percentile99 = Data_.GetValueAtPercentile(99.0); + snapshot->Percentile999 = Data_.GetValueAtPercentile(99.9); + snapshot->TotalCount = Data_.GetTotalCount(); + + // cleanup histogram data + Data_.Reset(); + } + } + +} diff --git a/library/cpp/monlib/counters/histogram.h b/library/cpp/monlib/counters/histogram.h new file mode 100644 index 0000000000..96361b0023 --- /dev/null +++ b/library/cpp/monlib/counters/histogram.h @@ -0,0 +1,189 @@ +#pragma once + +#include <library/cpp/histogram/hdr/histogram.h> + +#include <util/system/spinlock.h> +#include <util/stream/output.h> + +namespace NMonitoring { + /** + * A statistical snapshot of values recorded in histogram. + */ + struct THistogramSnapshot { + double Mean; + double StdDeviation; + i64 Min; + i64 Max; + i64 Percentile50; + i64 Percentile75; + i64 Percentile90; + i64 Percentile95; + i64 Percentile98; + i64 Percentile99; + i64 Percentile999; + i64 TotalCount; + + void Print(IOutputStream* out) const; + }; + + /** + * Special counter which calculates the distribution of a value. + */ + class THdrHistogram { + public: + /** + * Construct a histogram given the Lowest and Highest values to be tracked + * and a number of significant decimal digits. Providing a + * lowestDiscernibleValue is useful in situations where the units used for + * the histogram's values are much smaller that the minimal accuracy + * required. E.g. when tracking time values stated in nanosecond units, + * where the minimal accuracy required is a microsecond, the proper value + * for lowestDiscernibleValue would be 1000. + * + * @param lowestDiscernibleValue The lowest value that can be discerned + * (distinguished from 0) by the histogram. Must be a positive + * integer that is >= 1. May be internally rounded down to nearest + * power of 2. + * + * @param highestTrackableValue The highest value to be tracked by the + * histogram. Must be a positive integer that is + * >= (2 * lowestDiscernibleValue). + * + * @param numberOfSignificantValueDigits Specifies the precision to use. + * This is the number of significant decimal digits to which the + * histogram will maintain value resolution and separation. Must be + * a non-negative integer between 0 and 5. + */ + THdrHistogram(i64 lowestDiscernibleValue, i64 highestTrackableValue, + i32 numberOfSignificantValueDigits) + : Data_(lowestDiscernibleValue, highestTrackableValue, + numberOfSignificantValueDigits) { + } + + /** + * Records a value in the histogram, will round this value of to a + * precision at or better than the NumberOfSignificantValueDigits specified + * at construction time. + * + * @param value Value to add to the histogram + * @return false if the value is larger than the HighestTrackableValue + * and can't be recorded, true otherwise. + */ + bool RecordValue(i64 value) { + with_lock (Lock_) { + return Data_.RecordValue(value); + } + } + + /** + * Records count values in the histogram, will round this value of to a + * precision at or better than the NumberOfSignificantValueDigits specified + * at construction time. + * + * @param value Value to add to the histogram + * @param count Number of values to add to the histogram + * @return false if the value is larger than the HighestTrackableValue + * and can't be recorded, true otherwise. + */ + bool RecordValues(i64 value, i64 count) { + with_lock (Lock_) { + return Data_.RecordValues(value, count); + } + } + + /** + * Records a value in the histogram and backfill based on an expected + * interval. Value will be rounded this to a precision at or better + * than the NumberOfSignificantValueDigits specified at contruction time. + * This is specifically used for recording latency. If the value is larger + * than the expectedInterval then the latency recording system has + * experienced co-ordinated omission. This method fills in the values that + * would have occured had the client providing the load not been blocked. + * + * @param value Value to add to the histogram + * @param expectedInterval The delay between recording values + * @return false if the value is larger than the HighestTrackableValue + * and can't be recorded, true otherwise. + */ + bool RecordValueWithExpectedInterval(i64 value, i64 expectedInterval) { + with_lock (Lock_) { + return Data_.RecordValueWithExpectedInterval(value, expectedInterval); + } + } + + /** + * Record a value in the histogram count times. Applies the same correcting + * logic as {@link THdrHistogram::RecordValueWithExpectedInterval}. + * + * @param value Value to add to the histogram + * @param count Number of values to add to the histogram + * @param expectedInterval The delay between recording values. + * @return false if the value is larger than the HighestTrackableValue + * and can't be recorded, true otherwise. + */ + bool RecordValuesWithExpectedInterval( + i64 value, i64 count, i64 expectedInterval) { + with_lock (Lock_) { + return Data_.RecordValuesWithExpectedInterval( + value, count, expectedInterval); + } + } + + /** + * @return The configured lowestDiscernibleValue + */ + i64 GetLowestDiscernibleValue() const { + with_lock (Lock_) { + return Data_.GetLowestDiscernibleValue(); + } + } + + /** + * @return The configured highestTrackableValue + */ + i64 GetHighestTrackableValue() const { + with_lock (Lock_) { + return Data_.GetHighestTrackableValue(); + } + } + + /** + * @return The configured numberOfSignificantValueDigits + */ + i32 GetNumberOfSignificantValueDigits() const { + with_lock (Lock_) { + return Data_.GetNumberOfSignificantValueDigits(); + } + } + + /** + * @return The total count of all recorded values in the histogram + */ + i64 GetTotalCount() const { + with_lock (Lock_) { + return Data_.GetTotalCount(); + } + } + + /** + * Place a copy of the value counts accumulated since the last snapshot + * was taken into {@code snapshot}. Calling this member-function will + * reset the value counts, and start accumulating value counts for the + * next interval. + * + * @param snapshot the structure into which the values should be copied. + */ + void TakeSnaphot(THistogramSnapshot* snapshot); + + private: + mutable TSpinLock Lock_; + NHdr::THistogram Data_; + }; + +} + +template <> +inline void Out<NMonitoring::THistogramSnapshot>( + IOutputStream& out, const NMonitoring::THistogramSnapshot& snapshot) { + snapshot.Print(&out); +} diff --git a/library/cpp/monlib/counters/histogram_ut.cpp b/library/cpp/monlib/counters/histogram_ut.cpp new file mode 100644 index 0000000000..5a0800505a --- /dev/null +++ b/library/cpp/monlib/counters/histogram_ut.cpp @@ -0,0 +1,47 @@ +#include "histogram.h" + +#include <library/cpp/testing/unittest/registar.h> + +using namespace NMonitoring; + +Y_UNIT_TEST_SUITE(THistorgamTest) { + Y_UNIT_TEST(TakeSnapshot) { + THdrHistogram h(1, 10, 3); + UNIT_ASSERT(h.RecordValue(1)); + UNIT_ASSERT(h.RecordValue(2)); + UNIT_ASSERT(h.RecordValues(3, 10)); + UNIT_ASSERT(h.RecordValue(4)); + UNIT_ASSERT(h.RecordValue(5)); + + UNIT_ASSERT_EQUAL(h.GetTotalCount(), 14); + + THistogramSnapshot snapshot; + h.TakeSnaphot(&snapshot); + + UNIT_ASSERT_EQUAL(h.GetTotalCount(), 0); + + UNIT_ASSERT_EQUAL(snapshot.Min, 1); + UNIT_ASSERT_EQUAL(snapshot.Max, 5); + + // >>> a = [1, 2] + [3 for i in range(10)] + [4, 5] + // >>> numpy.mean(a) + // 3.0 + UNIT_ASSERT_DOUBLES_EQUAL(snapshot.Mean, 3.0, 1e-6); + + // >>> numpy.std(a) + // 0.84515425472851657 + UNIT_ASSERT_DOUBLES_EQUAL(snapshot.StdDeviation, 0.84515425472851657, 1e-6); + + // >>> [(p, round(numpy.percentile(a, p))) for p in [50, 75, 90, 95, 98, 99, 99.9, 100]] + // [(50, 3.0), (75, 3.0), (90, 4.0), (95, 4.0), (98, 5.0), (99, 5.0), (99.9, 5.0), (100, 5.0)] + UNIT_ASSERT_EQUAL(snapshot.Percentile50, 3); + UNIT_ASSERT_EQUAL(snapshot.Percentile75, 3); + UNIT_ASSERT_EQUAL(snapshot.Percentile90, 4); + UNIT_ASSERT_EQUAL(snapshot.Percentile95, 4); + UNIT_ASSERT_EQUAL(snapshot.Percentile98, 5); + UNIT_ASSERT_EQUAL(snapshot.Percentile99, 5); + UNIT_ASSERT_EQUAL(snapshot.Percentile999, 5); + + UNIT_ASSERT_EQUAL(snapshot.TotalCount, 14); + } +} diff --git a/library/cpp/monlib/counters/meter.cpp b/library/cpp/monlib/counters/meter.cpp new file mode 100644 index 0000000000..6f15f173d1 --- /dev/null +++ b/library/cpp/monlib/counters/meter.cpp @@ -0,0 +1,4 @@ +#include "meter.h" + +namespace NMonitoring { +} diff --git a/library/cpp/monlib/counters/meter.h b/library/cpp/monlib/counters/meter.h new file mode 100644 index 0000000000..1219f95c4d --- /dev/null +++ b/library/cpp/monlib/counters/meter.h @@ -0,0 +1,285 @@ +#pragma once + +#include <util/system/types.h> +#include <util/generic/noncopyable.h> +#include <util/system/atomic.h> + +#include <chrono> +#include <cstdlib> +#include <cmath> + +namespace NMonitoring { + /** + * An exponentially-weighted moving average. + * + * @see <a href="http://www.teamquest.com/pdfs/whitepaper/ldavg1.pdf"> + * UNIX Load Average Part 1: How It Works</a> + * @see <a href="http://www.teamquest.com/pdfs/whitepaper/ldavg2.pdf"> + * UNIX Load Average Part 2: Not Your Average Average</a> + * @see <a href="http://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average">EMA</a> + */ + class TMovingAverage { + public: + enum { + INTERVAL = 5 // in seconds + }; + + public: + /** + * Creates a new EWMA which is equivalent to the UNIX one minute load + * average and which expects to be ticked every 5 seconds. + * + * @return a one-minute EWMA + */ + static TMovingAverage OneMinute() { + static const double M1_ALPHA = 1 - std::exp(-INTERVAL / 60.0 / 1); + return {M1_ALPHA, std::chrono::seconds(INTERVAL)}; + } + + /** + * Creates a new EWMA which is equivalent to the UNIX five minute load + * average and which expects to be ticked every 5 seconds. + * + * @return a five-minute EWMA + */ + static TMovingAverage FiveMinutes() { + static const double M5_ALPHA = 1 - std::exp(-INTERVAL / 60.0 / 5); + return {M5_ALPHA, std::chrono::seconds(INTERVAL)}; + } + + /** + * Creates a new EWMA which is equivalent to the UNIX fifteen minute load + * average and which expects to be ticked every 5 seconds. + * + * @return a fifteen-minute EWMA + */ + static TMovingAverage FifteenMinutes() { + static const double M15_ALPHA = 1 - std::exp(-INTERVAL / 60.0 / 15); + return {M15_ALPHA, std::chrono::seconds(INTERVAL)}; + } + + /** + * Create a new EWMA with a specific smoothing constant. + * + * @param alpha the smoothing constant + * @param interval the expected tick interval + */ + TMovingAverage(double alpha, std::chrono::seconds interval) + : Initialized_(0) + , Rate_(0) + , Uncounted_(0) + , Alpha_(alpha) + , Interval_(std::chrono::nanoseconds(interval).count()) + { + } + + TMovingAverage(const TMovingAverage& rhs) + : Initialized_(AtomicGet(rhs.Initialized_)) + , Rate_(AtomicGet(rhs.Rate_)) + , Uncounted_(AtomicGet(rhs.Uncounted_)) + , Alpha_(rhs.Alpha_) + , Interval_(rhs.Interval_) + { + } + + TMovingAverage& operator=(const TMovingAverage& rhs) { + AtomicSet(Initialized_, AtomicGet(Initialized_)); + AtomicSet(Rate_, AtomicGet(rhs.Rate_)); + AtomicSet(Uncounted_, AtomicGet(rhs.Uncounted_)); + Alpha_ = rhs.Alpha_; + Interval_ = rhs.Interval_; + return *this; + } + + /** + * Update the moving average with a new value. + * + * @param n the new value + */ + void Update(ui64 n = 1) { + AtomicAdd(Uncounted_, n); + } + + /** + * Mark the passage of time and decay the current rate accordingly. + */ + void Tick() { + double instantRate = AtomicSwap(&Uncounted_, 0) / Interval_; + if (AtomicGet(Initialized_)) { + double rate = AsDouble(AtomicGet(Rate_)); + rate += (Alpha_ * (instantRate - rate)); + AtomicSet(Rate_, AsAtomic(rate)); + } else { + AtomicSet(Rate_, AsAtomic(instantRate)); + AtomicSet(Initialized_, 1); + } + } + + /** + * @return the rate in the seconds + */ + double GetRate() const { + double rate = AsDouble(AtomicGet(Rate_)); + return rate * std::nano::den; + } + + private: + static double AsDouble(TAtomicBase val) { + union { + double D; + TAtomicBase A; + } doubleAtomic; + doubleAtomic.A = val; + return doubleAtomic.D; + } + + static TAtomicBase AsAtomic(double val) { + union { + double D; + TAtomicBase A; + } doubleAtomic; + doubleAtomic.D = val; + return doubleAtomic.A; + } + + private: + TAtomic Initialized_; + TAtomic Rate_; + TAtomic Uncounted_; + double Alpha_; + double Interval_; + }; + + /** + * A meter metric which measures mean throughput and one-, five-, and + * fifteen-minute exponentially-weighted moving average throughputs. + */ + template <typename TClock> + class TMeterImpl: private TNonCopyable { + public: + TMeterImpl() + : StartTime_(TClock::now()) + , LastTick_(StartTime_.time_since_epoch().count()) + , Count_(0) + , OneMinuteRate_(TMovingAverage::OneMinute()) + , FiveMinutesRate_(TMovingAverage::FiveMinutes()) + , FifteenMinutesRate_(TMovingAverage::FifteenMinutes()) + { + } + + /** + * Mark the occurrence of events. + * + * @param n the number of events + */ + void Mark(ui64 n = 1) { + TickIfNecessary(); + AtomicAdd(Count_, n); + OneMinuteRate_.Update(n); + FiveMinutesRate_.Update(n); + FifteenMinutesRate_.Update(n); + } + + /** + * Returns the one-minute exponentially-weighted moving average rate at + * which events have occurred since the meter was created. + * + * This rate has the same exponential decay factor as the one-minute load + * average in the top Unix command. + * + * @return the one-minute exponentially-weighted moving average rate at + * which events have occurred since the meter was created + */ + double GetOneMinuteRate() const { + return OneMinuteRate_.GetRate(); + } + + /** + * Returns the five-minute exponentially-weighted moving average rate at + * which events have occurred since the meter was created. + * + * This rate has the same exponential decay factor as the five-minute load + * average in the top Unix command. + * + * @return the five-minute exponentially-weighted moving average rate at + * which events have occurred since the meter was created + */ + double GetFiveMinutesRate() const { + return FiveMinutesRate_.GetRate(); + } + + /** + * Returns the fifteen-minute exponentially-weighted moving average rate + * at which events have occurred since the meter was created. + * + * This rate has the same exponential decay factor as the fifteen-minute + * load average in the top Unix command. + * + * @return the fifteen-minute exponentially-weighted moving average rate + * at which events have occurred since the meter was created + */ + double GetFifteenMinutesRate() const { + return FifteenMinutesRate_.GetRate(); + } + + /** + * @return the mean rate at which events have occurred since the meter + * was created + */ + double GetMeanRate() const { + if (GetCount() == 0) { + return 0.0; + } + + auto now = TClock::now(); + std::chrono::duration<double> elapsedSeconds = now - StartTime_; + return GetCount() / elapsedSeconds.count(); + } + + /** + * @return the number of events which have been marked + */ + ui64 GetCount() const { + return AtomicGet(Count_); + } + + private: + void TickIfNecessary() { + static ui64 TICK_INTERVAL_NS = + std::chrono::nanoseconds( + std::chrono::seconds(TMovingAverage::INTERVAL)) + .count(); + + auto oldTickNs = AtomicGet(LastTick_); + auto newTickNs = TClock::now().time_since_epoch().count(); + ui64 elapsedNs = std::abs(newTickNs - oldTickNs); + + if (elapsedNs > TICK_INTERVAL_NS) { + // adjust to interval begining + newTickNs -= elapsedNs % TICK_INTERVAL_NS; + if (AtomicCas(&LastTick_, newTickNs, oldTickNs)) { + ui64 requiredTicks = elapsedNs / TICK_INTERVAL_NS; + for (ui64 i = 0; i < requiredTicks; ++i) { + OneMinuteRate_.Tick(); + FiveMinutesRate_.Tick(); + FifteenMinutesRate_.Tick(); + } + } + } + } + + private: + const typename TClock::time_point StartTime_; + TAtomic LastTick_; + TAtomic Count_; + TMovingAverage OneMinuteRate_; + TMovingAverage FiveMinutesRate_; + TMovingAverage FifteenMinutesRate_; + }; + + using TSystemMeter = TMeterImpl<std::chrono::system_clock>; + using TSteadyMeter = TMeterImpl<std::chrono::steady_clock>; + using THighResMeter = TMeterImpl<std::chrono::high_resolution_clock>; + using TMeter = THighResMeter; + +} diff --git a/library/cpp/monlib/counters/meter_ut.cpp b/library/cpp/monlib/counters/meter_ut.cpp new file mode 100644 index 0000000000..b507d16fbd --- /dev/null +++ b/library/cpp/monlib/counters/meter_ut.cpp @@ -0,0 +1,41 @@ +#include "meter.h" + +#include <library/cpp/testing/unittest/registar.h> + +using namespace NMonitoring; + +struct TMockClock { + using duration = std::chrono::nanoseconds; + using rep = duration::rep; + using period = duration::period; + using time_point = std::chrono::time_point<TMockClock, duration>; + + static time_point now() noexcept { + static int index = 0; + return index++ < 2 ? time_point() : time_point(std::chrono::seconds(10)); + } +}; + +using TMockMeter = TMeterImpl<TMockClock>; + +Y_UNIT_TEST_SUITE(TMeterTest) { + Y_UNIT_TEST(StartsOutWithNoRatesOrCount) { + TMeter meter; + UNIT_ASSERT_EQUAL(meter.GetCount(), 0L); + UNIT_ASSERT_DOUBLES_EQUAL(meter.GetMeanRate(), 0.0, 0.0001); + UNIT_ASSERT_DOUBLES_EQUAL(meter.GetOneMinuteRate(), 0.0, 0.0001); + UNIT_ASSERT_DOUBLES_EQUAL(meter.GetFiveMinutesRate(), 0.0, 0.0001); + UNIT_ASSERT_DOUBLES_EQUAL(meter.GetFifteenMinutesRate(), 0.0, 0.0001); + } + + Y_UNIT_TEST(MarksEventsAndUpdatesRatesAndCount) { + TMockMeter meter; + meter.Mark(); + meter.Mark(2); + UNIT_ASSERT_EQUAL(meter.GetCount(), 3L); + UNIT_ASSERT_DOUBLES_EQUAL(meter.GetMeanRate(), 0.3, 0.001); + UNIT_ASSERT_DOUBLES_EQUAL(meter.GetOneMinuteRate(), 0.1840, 0.0001); + UNIT_ASSERT_DOUBLES_EQUAL(meter.GetFiveMinutesRate(), 0.1966, 0.0001); + UNIT_ASSERT_DOUBLES_EQUAL(meter.GetFifteenMinutesRate(), 0.1988, 0.0001); + } +} diff --git a/library/cpp/monlib/counters/timer.h b/library/cpp/monlib/counters/timer.h new file mode 100644 index 0000000000..03dfb35337 --- /dev/null +++ b/library/cpp/monlib/counters/timer.h @@ -0,0 +1,176 @@ +#pragma once + +#include "histogram.h" + +#include <util/generic/scope.h> + +#include <chrono> + +namespace NMonitoring { + /** + * A timer counter which aggregates timing durations and provides duration + * statistics in selected time resolution. + */ + template <typename TResolution> + class TTimerImpl { + public: + /** + * Construct a timer given the Lowest and Highest values to be tracked + * and a number of significant decimal digits. Providing a + * lowestDiscernibleValue is useful in situations where the units used for + * the timer's values are much smaller that the minimal accuracy + * required. E.g. when tracking time values stated in nanosecond units, + * where the minimal accuracy required is a microsecond, the proper value + * for lowestDiscernibleValue would be 1000. + * + * @param min The lowest value that can be discerned (distinguished from + * 0) by the timer. Must be a positive integer that is >= 1. + * May be internally rounded down to nearest power of 2. + * + * @param max The highest value to be tracked by the timer. Must be a + * positive integer that is >= (2 * min). + * + * @param numberOfSignificantValueDigits Specifies the precision to use. + * This is the number of significant decimal digits to which the + * timer will maintain value resolution and separation. Must be + * a non-negative integer between 0 and 5. + */ + TTimerImpl(ui64 min, ui64 max, i32 numberOfSignificantValueDigits = 3) + : TTimerImpl(TResolution(min), TResolution(max), + numberOfSignificantValueDigits) { + } + + /** + * Construct a timer given the Lowest and Highest values to be tracked + * and a number of significant decimal digits. + * + * @param min The lowest value that can be discerned (distinguished from + * 0) by the timer. + * + * @param max The highest value to be tracked by the histogram. Must be a + * positive integer that is >= (2 * min). + * + * @param numberOfSignificantValueDigits Specifies the precision to use. + */ + template <typename TDurationMin, typename TDurationMax> + TTimerImpl(TDurationMin min, TDurationMax max, + i32 numberOfSignificantValueDigits = 3) + : Histogram_(std::chrono::duration_cast<TResolution>(min).count(), + std::chrono::duration_cast<TResolution>(max).count(), + numberOfSignificantValueDigits) { + } + + /** + * Records a value in the timer with timer resulution. Recorded value will + * be rounded to a precision at or better than the + * NumberOfSignificantValueDigits specified at construction time. + * + * @param duration duration to add to the timer + * @return false if the value is larger than the max and can't be recorded, + * true otherwise. + */ + bool RecordValue(ui64 duration) { + return Histogram_.RecordValue(duration); + } + + /** + * Records a duration in the timer. Recorded value will be converted to + * the timer resulution and rounded to a precision at or better than the + * NumberOfSignificantValueDigits specified at construction time. + * + * @param duration duration to add to the timer + * @return false if the value is larger than the max and can't be recorded, + * true otherwise. + */ + template <typename TDuration> + bool RecordValue(TDuration duration) { + auto count = static_cast<ui64>( + std::chrono::duration_cast<TResolution>(duration).count()); + return RecordValue(count); + } + + /** + * Records count values in the timer with timer resulution. Recorded value will + * be rounded to a precision at or better than the + * NumberOfSignificantValueDigits specified at construction time. + * + * @param duration duration to add to the timer + * @param count number of values to add to the histogram + * @return false if the value is larger than the max and can't be recorded, + * true otherwise. + */ + bool RecordValues(ui64 duration, ui64 count) { + return Histogram_.RecordValues(duration, count); + } + + /** + * Measures a time of functor execution. + * + * @param fn functor whose duration should be timed + */ + template <typename TFunc> + void Measure(TFunc&& fn) { + using TClock = std::chrono::high_resolution_clock; + + auto start = TClock::now(); + + Y_SCOPE_EXIT(this, start) { + RecordValue(TClock::now() - start); + }; + + fn(); + } + + /** + * Place a copy of the value counts accumulated since the last snapshot + * was taken into {@code snapshot}. Calling this member-function will + * reset the value counts, and start accumulating value counts for the + * next interval. + * + * @param snapshot the structure into which the values should be copied. + */ + void TakeSnapshot(THistogramSnapshot* snapshot) { + Histogram_.TakeSnaphot(snapshot); + } + + private: + THdrHistogram Histogram_; + }; + + /** + * Timer template instantiations for certain time resolutions. + */ + using TTimerNs = TTimerImpl<std::chrono::nanoseconds>; + using TTimerUs = TTimerImpl<std::chrono::microseconds>; + using TTimerMs = TTimerImpl<std::chrono::milliseconds>; + using TTimerS = TTimerImpl<std::chrono::seconds>; + + /** + * A timing scope to record elapsed time since creation. + */ + template <typename TTimer, typename TFunc = std::function<void(std::chrono::high_resolution_clock::duration)>> + class TTimerScope { + using TClock = std::chrono::high_resolution_clock; + + public: + explicit TTimerScope(TTimer* timer, TFunc* callback = nullptr) + : Timer_(timer) + , StartTime_(TClock::now()) + , Callback_(callback) + { + } + + ~TTimerScope() { + TClock::duration duration = TClock::now() - StartTime_; + if (Callback_) { + (*Callback_)(duration); + } + Timer_->RecordValue(duration); + } + + private: + TTimer* Timer_; + TClock::time_point StartTime_; + TFunc* Callback_; + }; +} diff --git a/library/cpp/monlib/counters/timer_ut.cpp b/library/cpp/monlib/counters/timer_ut.cpp new file mode 100644 index 0000000000..c5cd07e89d --- /dev/null +++ b/library/cpp/monlib/counters/timer_ut.cpp @@ -0,0 +1,81 @@ +#include "timer.h" + +#include <library/cpp/testing/unittest/registar.h> + +using namespace NMonitoring; +using namespace std::literals::chrono_literals; + +class TCallback { +public: + explicit TCallback(int value) + : Value_(value){}; + void operator()(std::chrono::high_resolution_clock::duration duration) { + Value_ = duration.count(); + }; + + int Value_; +}; + +Y_UNIT_TEST_SUITE(TTimerTest) { + Y_UNIT_TEST(RecordValue) { + TTimerNs timerNs(1ns, 1s); + UNIT_ASSERT(timerNs.RecordValue(10us)); + + TTimerUs timerUs(1us, 1s); + UNIT_ASSERT(timerUs.RecordValue(10us)); + + THistogramSnapshot snapshot; + timerNs.TakeSnapshot(&snapshot); + UNIT_ASSERT_EQUAL(snapshot.Min, 10000); + UNIT_ASSERT_EQUAL(snapshot.Max, 10007); + UNIT_ASSERT_DOUBLES_EQUAL(snapshot.StdDeviation, 0.0, 1e-6); + + timerUs.TakeSnapshot(&snapshot); + UNIT_ASSERT_EQUAL(snapshot.Min, 10); + UNIT_ASSERT_EQUAL(snapshot.Max, 10); + UNIT_ASSERT_DOUBLES_EQUAL(snapshot.StdDeviation, 0.0, 1e-6); + } + + Y_UNIT_TEST(Measure) { + TTimerNs timer(1ns, 1s); + timer.Measure([]() { + Sleep(TDuration::MilliSeconds(1)); + }); + THistogramSnapshot snapshot; + timer.TakeSnapshot(&snapshot); + + UNIT_ASSERT(snapshot.Min > std::chrono::nanoseconds(1ms).count()); + UNIT_ASSERT(snapshot.Max > std::chrono::nanoseconds(1ms).count()); + UNIT_ASSERT_DOUBLES_EQUAL(snapshot.StdDeviation, 0.0, 1e-6); + } + + Y_UNIT_TEST(TimerScope) { + TTimerUs timer(1us, 1000s); + { + TTimerScope<TTimerUs> scope(&timer); + Sleep(TDuration::MilliSeconds(10)); + } + THistogramSnapshot snapshot; + timer.TakeSnapshot(&snapshot); + + UNIT_ASSERT(snapshot.Min > std::chrono::microseconds(10ms).count()); + UNIT_ASSERT(snapshot.Max > std::chrono::microseconds(10ms).count()); + UNIT_ASSERT_DOUBLES_EQUAL(snapshot.StdDeviation, 0.0, 1e-6); + } + + Y_UNIT_TEST(TimerScopeWithCallback) { + TCallback callback(0); + TTimerUs timer(1us, 1000s); + { + TTimerScope<TTimerUs, TCallback> scope(&timer, &callback); + Sleep(TDuration::MilliSeconds(10)); + } + THistogramSnapshot snapshot; + timer.TakeSnapshot(&snapshot); + + UNIT_ASSERT(snapshot.Min > std::chrono::microseconds(10ms).count()); + UNIT_ASSERT(snapshot.Max > std::chrono::microseconds(10ms).count()); + UNIT_ASSERT_DOUBLES_EQUAL(snapshot.StdDeviation, 0.0, 1e-6); + UNIT_ASSERT(callback.Value_ > std::chrono::microseconds(10ms).count()); + } +} diff --git a/library/cpp/monlib/counters/ut/ya.make b/library/cpp/monlib/counters/ut/ya.make new file mode 100644 index 0000000000..999dadb199 --- /dev/null +++ b/library/cpp/monlib/counters/ut/ya.make @@ -0,0 +1,12 @@ +UNITTEST_FOR(library/cpp/monlib/counters) + +OWNER(jamel) + +SRCS( + counters_ut.cpp + histogram_ut.cpp + meter_ut.cpp + timer_ut.cpp +) + +END() diff --git a/library/cpp/monlib/counters/ya.make b/library/cpp/monlib/counters/ya.make new file mode 100644 index 0000000000..aa1a671bf8 --- /dev/null +++ b/library/cpp/monlib/counters/ya.make @@ -0,0 +1,15 @@ +LIBRARY() + +OWNER(jamel) + +SRCS( + counters.cpp + histogram.cpp + meter.cpp +) + +PEERDIR( + library/cpp/histogram/hdr +) + +END() diff --git a/library/cpp/monlib/deprecated/json/ut/ya.make b/library/cpp/monlib/deprecated/json/ut/ya.make new file mode 100644 index 0000000000..18315993b5 --- /dev/null +++ b/library/cpp/monlib/deprecated/json/ut/ya.make @@ -0,0 +1,12 @@ +UNITTEST_FOR(library/cpp/monlib/deprecated/json) + +OWNER( + jamel + g:solomon +) + +SRCS( + writer_ut.cpp +) + +END() diff --git a/library/cpp/monlib/deprecated/json/writer.cpp b/library/cpp/monlib/deprecated/json/writer.cpp new file mode 100644 index 0000000000..a581f2e07a --- /dev/null +++ b/library/cpp/monlib/deprecated/json/writer.cpp @@ -0,0 +1,100 @@ +#include "writer.h" + +namespace NMonitoring { + TDeprecatedJsonWriter::TDeprecatedJsonWriter(IOutputStream* out) + : JsonWriter(out, false) + , State(STATE_ROOT) + { + } + + void TDeprecatedJsonWriter::TransitionState(EState current, EState next) { + if (State != current) { + ythrow yexception() << "wrong state"; + } + State = next; + } + + void TDeprecatedJsonWriter::OpenDocument() { + TransitionState(STATE_ROOT, STATE_DOCUMENT); + JsonWriter.OpenMap(); + } + + void TDeprecatedJsonWriter::CloseDocument() { + TransitionState(STATE_DOCUMENT, STATE_ROOT); + JsonWriter.CloseMap(); + JsonWriter.Flush(); + } + + void TDeprecatedJsonWriter::OpenCommonLabels() { + TransitionState(STATE_DOCUMENT, STATE_COMMON_LABELS); + JsonWriter.Write("commonLabels"); + JsonWriter.OpenMap(); + } + + void TDeprecatedJsonWriter::CloseCommonLabels() { + TransitionState(STATE_COMMON_LABELS, STATE_DOCUMENT); + JsonWriter.CloseMap(); + } + + void TDeprecatedJsonWriter::WriteCommonLabel(TStringBuf name, TStringBuf value) { + TransitionState(STATE_COMMON_LABELS, STATE_COMMON_LABELS); + JsonWriter.Write(name, value); + } + + void TDeprecatedJsonWriter::OpenMetrics() { + TransitionState(STATE_DOCUMENT, STATE_METRICS); + JsonWriter.Write("sensors"); + JsonWriter.OpenArray(); + } + + void TDeprecatedJsonWriter::CloseMetrics() { + TransitionState(STATE_METRICS, STATE_DOCUMENT); + JsonWriter.CloseArray(); + } + + void TDeprecatedJsonWriter::OpenMetric() { + TransitionState(STATE_METRICS, STATE_METRIC); + JsonWriter.OpenMap(); + } + + void TDeprecatedJsonWriter::CloseMetric() { + TransitionState(STATE_METRIC, STATE_METRICS); + JsonWriter.CloseMap(); + } + + void TDeprecatedJsonWriter::OpenLabels() { + TransitionState(STATE_METRIC, STATE_LABELS); + JsonWriter.Write("labels"); + JsonWriter.OpenMap(); + } + + void TDeprecatedJsonWriter::CloseLabels() { + TransitionState(STATE_LABELS, STATE_METRIC); + JsonWriter.CloseMap(); + } + + void TDeprecatedJsonWriter::WriteLabel(TStringBuf name, TStringBuf value) { + TransitionState(STATE_LABELS, STATE_LABELS); + JsonWriter.Write(name, value); + } + + void TDeprecatedJsonWriter::WriteModeDeriv() { + TransitionState(STATE_METRIC, STATE_METRIC); + JsonWriter.Write("mode", "deriv"); + } + + void TDeprecatedJsonWriter::WriteValue(long long value) { + TransitionState(STATE_METRIC, STATE_METRIC); + JsonWriter.Write("value", value); + } + + void TDeprecatedJsonWriter::WriteDoubleValue(double value) { + TransitionState(STATE_METRIC, STATE_METRIC); + JsonWriter.Write("value", value); + } + + void TDeprecatedJsonWriter::WriteTs(ui64 ts) { + TransitionState(STATE_METRIC, STATE_METRIC); + JsonWriter.Write("ts", ts); + } +} diff --git a/library/cpp/monlib/deprecated/json/writer.h b/library/cpp/monlib/deprecated/json/writer.h new file mode 100644 index 0000000000..183288143c --- /dev/null +++ b/library/cpp/monlib/deprecated/json/writer.h @@ -0,0 +1,76 @@ +#pragma once + +#include <library/cpp/json/json_writer.h> + +namespace NMonitoring { + /** + * Deprecated writer of Solomon JSON format + * https://wiki.yandex-team.ru/solomon/api/dataformat/json + * + * This writer will be deleted soon, so please consider to use + * high level library library/cpp/monlib/encode which is decoupled from the + * particular format. + */ + class TDeprecatedJsonWriter { + private: + NJson::TJsonWriter JsonWriter; + enum EState { + STATE_ROOT, + STATE_DOCUMENT, + STATE_COMMON_LABELS, + STATE_METRICS, + STATE_METRIC, + STATE_LABELS, + }; + EState State; + + public: + explicit TDeprecatedJsonWriter(IOutputStream* out); + + void OpenDocument(); + void CloseDocument(); + + void OpenCommonLabels(); + void CloseCommonLabels(); + + void WriteCommonLabel(TStringBuf name, TStringBuf value); + + void OpenMetrics(); + void CloseMetrics(); + + void OpenMetric(); + void CloseMetric(); + + void OpenLabels(); + void CloseLabels(); + + void WriteLabel(TStringBuf name, TStringBuf value); + + template <typename... T> + void WriteLabels(T... pairs) { + OpenLabels(); + WriteLabelsInner(pairs...); + CloseLabels(); + } + + void WriteModeDeriv(); + + void WriteValue(long long value); + void WriteDoubleValue(double d); + + void WriteTs(ui64 ts); + + private: + void WriteLabelsInner(TStringBuf name, TStringBuf value) { + WriteLabel(name, value); + } + + template <typename... T> + void WriteLabelsInner(TStringBuf name, TStringBuf value, T... pairs) { + WriteLabel(name, value); + WriteLabelsInner(pairs...); + } + + inline void TransitionState(EState current, EState next); + }; +} diff --git a/library/cpp/monlib/deprecated/json/writer_ut.cpp b/library/cpp/monlib/deprecated/json/writer_ut.cpp new file mode 100644 index 0000000000..1f9fc8f393 --- /dev/null +++ b/library/cpp/monlib/deprecated/json/writer_ut.cpp @@ -0,0 +1,32 @@ +#include "writer.h" + +#include <library/cpp/testing/unittest/registar.h> + +using namespace NMonitoring; + +Y_UNIT_TEST_SUITE(JsonWriterTests) { + Y_UNIT_TEST(One) { + TStringStream ss; + TDeprecatedJsonWriter w(&ss); + w.OpenDocument(); + w.OpenMetrics(); + + for (int i = 0; i < 5; ++i) { + w.OpenMetric(); + w.OpenLabels(); + w.WriteLabel("user", TString("") + (char)('a' + i)); + w.WriteLabel("name", "NWrites"); + w.CloseLabels(); + if (i % 2 == 0) { + w.WriteModeDeriv(); + } + w.WriteValue(10l); + w.CloseMetric(); + } + + w.CloseMetrics(); + w.CloseDocument(); + + //Cout << ss.Str() << "\n"; + } +} diff --git a/library/cpp/monlib/deprecated/json/ya.make b/library/cpp/monlib/deprecated/json/ya.make new file mode 100644 index 0000000000..0ca903ee62 --- /dev/null +++ b/library/cpp/monlib/deprecated/json/ya.make @@ -0,0 +1,26 @@ +LIBRARY() + +# Deprecated writer of Solomon JSON format +# https://wiki.yandex-team.ru/solomon/api/dataformat/json +# +# This writer will be deleted soon, so please consider to use +# high level library library/cpp/monlib/encode which is decoupled from the +# particular format. + +OWNER( + jamel + g:solomon +) + +SRCS( + writer.h + writer.cpp +) + +PEERDIR( + library/cpp/json +) + +END() + +RECURSE_FOR_TESTS(ut) diff --git a/library/cpp/monlib/deprecated/ya.make b/library/cpp/monlib/deprecated/ya.make new file mode 100644 index 0000000000..9345139aee --- /dev/null +++ b/library/cpp/monlib/deprecated/ya.make @@ -0,0 +1,8 @@ +OWNER( + g:solomon + jamel +) + +RECURSE( + json +) diff --git a/library/cpp/monlib/dynamic_counters/contention_ut.cpp b/library/cpp/monlib/dynamic_counters/contention_ut.cpp new file mode 100644 index 0000000000..8798044ee3 --- /dev/null +++ b/library/cpp/monlib/dynamic_counters/contention_ut.cpp @@ -0,0 +1,61 @@ +#include "counters.h" +#include <library/cpp/testing/unittest/registar.h> +#include <util/system/event.h> +#include <util/system/thread.h> + +using namespace NMonitoring; + +Y_UNIT_TEST_SUITE(TDynamicCountersContentionTest) { + + Y_UNIT_TEST(EnsureNonlocking) { + TDynamicCounterPtr counters = MakeIntrusive<TDynamicCounters>(); + + class TConsumer : public ICountableConsumer { + TAutoEvent Ev; + TAutoEvent Response; + TDynamicCounterPtr Counters; + TThread Thread; + + public: + TConsumer(TDynamicCounterPtr counters) + : Counters(counters) + , Thread(std::bind(&TConsumer::ThreadFunc, this)) + { + Thread.Start(); + } + + ~TConsumer() override { + Thread.Join(); + } + + void OnCounter(const TString& /*labelName*/, const TString& /*labelValue*/, const TCounterForPtr* /*counter*/) override { + Ev.Signal(); + Response.Wait(); + } + + void OnHistogram(const TString& /*labelName*/, const TString& /*labelValue*/, IHistogramSnapshotPtr /*snapshot*/, bool /*derivative*/) override { + } + + void OnGroupBegin(const TString& /*labelName*/, const TString& /*labelValue*/, const TDynamicCounters* /*group*/) override { + } + + void OnGroupEnd(const TString& /*labelName*/, const TString& /*labelValue*/, const TDynamicCounters* /*group*/) override { + } + + private: + void ThreadFunc() { + // acts like a coroutine + Ev.Wait(); + auto ctr = Counters->GetSubgroup("label", "value")->GetCounter("name"); + Y_VERIFY(*ctr == 42); + Response.Signal(); + } + }; + + auto ctr = counters->GetSubgroup("label", "value")->GetCounter("name"); + *ctr = 42; + TConsumer consumer(counters); + counters->Accept({}, {}, consumer); + } + +} diff --git a/library/cpp/monlib/dynamic_counters/counters.cpp b/library/cpp/monlib/dynamic_counters/counters.cpp new file mode 100644 index 0000000000..3635d87d0d --- /dev/null +++ b/library/cpp/monlib/dynamic_counters/counters.cpp @@ -0,0 +1,308 @@ +#include "counters.h" + +#include <library/cpp/monlib/service/pages/templates.h> + +#include <util/generic/cast.h> + +using namespace NMonitoring; + +namespace { + TDynamicCounters* AsDynamicCounters(const TIntrusivePtr<TCountableBase>& ptr) { + return dynamic_cast<TDynamicCounters*>(ptr.Get()); + } + + TCounterForPtr* AsCounter(const TIntrusivePtr<TCountableBase>& ptr) { + return dynamic_cast<TCounterForPtr*>(ptr.Get()); + } + + TExpiringCounter* AsExpiringCounter(const TIntrusivePtr<TCountableBase>& ptr) { + return dynamic_cast<TExpiringCounter*>(ptr.Get()); + } + + TExpiringHistogramCounter* AsExpiringHistogramCounter(const TIntrusivePtr<TCountableBase>& ptr) { + return dynamic_cast<TExpiringHistogramCounter*>(ptr.Get()); + } + + THistogramCounter* AsHistogram(const TIntrusivePtr<TCountableBase>& ptr) { + return dynamic_cast<THistogramCounter*>(ptr.Get()); + } + + TIntrusivePtr<TCounterForPtr> AsCounterRef(const TIntrusivePtr<TCountableBase>& ptr) { + return VerifyDynamicCast<TCounterForPtr*>(ptr.Get()); + } + + TIntrusivePtr<TDynamicCounters> AsGroupRef(const TIntrusivePtr<TCountableBase>& ptr) { + return VerifyDynamicCast<TDynamicCounters*>(ptr.Get()); + } + + THistogramPtr AsHistogramRef(const TIntrusivePtr<TCountableBase>& ptr) { + return VerifyDynamicCast<THistogramCounter*>(ptr.Get()); + } + + bool IsExpiringCounter(const TIntrusivePtr<TCountableBase>& ptr) { + return AsExpiringCounter(ptr) != nullptr || AsExpiringHistogramCounter(ptr) != nullptr; + } +} + +static constexpr TStringBuf INDENT = " "; + +TDynamicCounters::TDynamicCounters(EVisibility vis) +{ + Visibility_ = vis; +} + +TDynamicCounters::~TDynamicCounters() { +} + +TDynamicCounters::TCounterPtr TDynamicCounters::GetExpiringCounter(const TString& value, bool derivative, EVisibility vis) { + return GetExpiringNamedCounter("sensor", value, derivative, vis); +} + +TDynamicCounters::TCounterPtr TDynamicCounters::GetExpiringNamedCounter(const TString& name, const TString& value, bool derivative, EVisibility vis) { + return AsCounterRef(GetNamedCounterImpl<true, TExpiringCounter>(name, value, derivative, vis)); +} + +TDynamicCounters::TCounterPtr TDynamicCounters::GetCounter(const TString& value, bool derivative, EVisibility vis) { + return GetNamedCounter("sensor", value, derivative, vis); +} + +TDynamicCounters::TCounterPtr TDynamicCounters::GetNamedCounter(const TString& name, const TString& value, bool derivative, EVisibility vis) { + return AsCounterRef(GetNamedCounterImpl<false, TCounterForPtr>(name, value, derivative, vis)); +} + +THistogramPtr TDynamicCounters::GetHistogram(const TString& value, IHistogramCollectorPtr collector, bool derivative, EVisibility vis) { + return GetNamedHistogram("sensor", value, std::move(collector), derivative, vis); +} + +THistogramPtr TDynamicCounters::GetNamedHistogram(const TString& name, const TString& value, IHistogramCollectorPtr collector, bool derivative, EVisibility vis) { + return AsHistogramRef(GetNamedCounterImpl<false, THistogramCounter>(name, value, std::move(collector), derivative, vis)); +} + +THistogramPtr TDynamicCounters::GetExpiringHistogram(const TString& value, IHistogramCollectorPtr collector, bool derivative, EVisibility vis) { + return GetExpiringNamedHistogram("sensor", value, std::move(collector), derivative, vis); +} + +THistogramPtr TDynamicCounters::GetExpiringNamedHistogram(const TString& name, const TString& value, IHistogramCollectorPtr collector, bool derivative, EVisibility vis) { + return AsHistogramRef(GetNamedCounterImpl<true, TExpiringHistogramCounter>(name, value, std::move(collector), derivative, vis)); +} + +TDynamicCounters::TCounterPtr TDynamicCounters::FindCounter(const TString& value) const { + return FindNamedCounter("sensor", value); +} + +TDynamicCounters::TCounterPtr TDynamicCounters::FindNamedCounter(const TString& name, const TString& value) const { + return AsCounterRef(FindNamedCounterImpl<TCounterForPtr>(name, value)); +} + +THistogramPtr TDynamicCounters::FindHistogram(const TString& value) const { + return FindNamedHistogram("sensor", value); +} + +THistogramPtr TDynamicCounters::FindNamedHistogram(const TString& name,const TString& value) const { + return AsHistogramRef(FindNamedCounterImpl<THistogramCounter>(name, value)); +} + +void TDynamicCounters::RemoveCounter(const TString &value) { + RemoveNamedCounter("sensor", value); +} + +void TDynamicCounters::RemoveNamedCounter(const TString& name, const TString &value) { + auto g = LockForUpdate("RemoveNamedCounter", name, value); + if (const auto it = Counters.find({name, value}); it != Counters.end() && AsCounter(it->second)) { + Counters.erase(it); + } +} + +TIntrusivePtr<TDynamicCounters> TDynamicCounters::GetSubgroup(const TString& name, const TString& value) { + auto res = FindSubgroup(name, value); + if (!res) { + auto g = LockForUpdate("GetSubgroup", name, value); + const TChildId key(name, value); + if (const auto it = Counters.lower_bound(key); it != Counters.end() && it->first == key) { + res = AsGroupRef(it->second); + } else { + res = MakeIntrusive<TDynamicCounters>(this); + Counters.emplace_hint(it, key, res); + } + } + return res; +} + +TIntrusivePtr<TDynamicCounters> TDynamicCounters::FindSubgroup(const TString& name, const TString& value) const { + TReadGuard g(Lock); + const auto it = Counters.find({name, value}); + return it != Counters.end() ? AsDynamicCounters(it->second) : nullptr; +} + +void TDynamicCounters::RemoveSubgroup(const TString& name, const TString& value) { + auto g = LockForUpdate("RemoveSubgroup", name, value); + if (const auto it = Counters.find({name, value}); it != Counters.end() && AsDynamicCounters(it->second)) { + Counters.erase(it); + } +} + +void TDynamicCounters::ReplaceSubgroup(const TString& name, const TString& value, TIntrusivePtr<TDynamicCounters> subgroup) { + auto g = LockForUpdate("ReplaceSubgroup", name, value); + const auto it = Counters.find({name, value}); + Y_VERIFY(it != Counters.end() && AsDynamicCounters(it->second)); + it->second = std::move(subgroup); +} + +void TDynamicCounters::MergeWithSubgroup(const TString& name, const TString& value) { + auto g = LockForUpdate("MergeWithSubgroup", name, value); + auto it = Counters.find({name, value}); + Y_VERIFY(it != Counters.end()); + TIntrusivePtr<TDynamicCounters> subgroup = AsDynamicCounters(it->second); + Y_VERIFY(subgroup); + Counters.erase(it); + Counters.merge(subgroup->Resign()); + AtomicAdd(ExpiringCount, AtomicSwap(&subgroup->ExpiringCount, 0)); +} + +void TDynamicCounters::ResetCounters(bool derivOnly) { + TReadGuard g(Lock); + for (auto& [key, value] : Counters) { + if (auto counter = AsCounter(value)) { + if (!derivOnly || counter->ForDerivative()) { + *counter = 0; + } + } else if (auto subgroup = AsDynamicCounters(value)) { + subgroup->ResetCounters(derivOnly); + } + } +} + +void TDynamicCounters::RegisterCountable(const TString& name, const TString& value, TCountablePtr countable) { + Y_VERIFY(countable); + auto g = LockForUpdate("RegisterCountable", name, value); + const bool inserted = Counters.emplace(TChildId(name, value), std::move(countable)).second; + Y_VERIFY(inserted); +} + +void TDynamicCounters::RegisterSubgroup(const TString& name, const TString& value, TIntrusivePtr<TDynamicCounters> subgroup) { + RegisterCountable(name, value, subgroup); +} + +void TDynamicCounters::OutputHtml(IOutputStream& os) const { + HTML(os) { + PRE() { + OutputPlainText(os); + } + } +} + +void TDynamicCounters::EnumerateSubgroups(const std::function<void(const TString& name, const TString& value)>& output) const { + TReadGuard g(Lock); + for (const auto& [key, value] : Counters) { + if (AsDynamicCounters(value)) { + output(key.LabelName, key.LabelValue); + } + } +} + +void TDynamicCounters::OutputPlainText(IOutputStream& os, const TString& indent) const { + auto snap = ReadSnapshot(); + // mark private records in plain text output + auto outputVisibilityMarker = [] (EVisibility vis) { + return vis == EVisibility::Private ? "\t[PRIVATE]" : ""; + }; + + for (const auto& [key, value] : snap) { + if (const auto counter = AsCounter(value)) { + os << indent + << key.LabelName << '=' << key.LabelValue + << ": " << counter->Val() + << outputVisibilityMarker(counter->Visibility()) + << '\n'; + } else if (const auto histogram = AsHistogram(value)) { + os << indent + << key.LabelName << '=' << key.LabelValue + << ":" + << outputVisibilityMarker(histogram->Visibility()) + << "\n"; + + auto snapshot = histogram->Snapshot(); + for (ui32 i = 0, count = snapshot->Count(); i < count; i++) { + os << indent << INDENT << TStringBuf("bin="); + TBucketBound bound = snapshot->UpperBound(i); + if (bound == Max<TBucketBound>()) { + os << TStringBuf("inf"); + } else { + os << bound; + } + os << ": " << snapshot->Value(i) << '\n'; + } + } + } + + for (const auto& [key, value] : snap) { + if (const auto subgroup = AsDynamicCounters(value)) { + os << "\n"; + os << indent << key.LabelName << "=" << key.LabelValue << ":\n"; + subgroup->OutputPlainText(os, indent + INDENT); + } + } +} + +void TDynamicCounters::Accept(const TString& labelName, const TString& labelValue, ICountableConsumer& consumer) const { + if (!IsVisible(Visibility(), consumer.Visibility())) { + return; + } + + consumer.OnGroupBegin(labelName, labelValue, this); + for (auto& [key, value] : ReadSnapshot()) { + value->Accept(key.LabelName, key.LabelValue, consumer); + } + consumer.OnGroupEnd(labelName, labelValue, this); +} + +void TDynamicCounters::RemoveExpired() const { + if (AtomicGet(ExpiringCount) == 0) { + return; + } + + TWriteGuard g(Lock); + TAtomicBase count = 0; + + for (auto it = Counters.begin(); it != Counters.end();) { + if (IsExpiringCounter(it->second) && it->second->RefCount() == 1) { + it = Counters.erase(it); + ++count; + } else { + ++it; + } + } + + AtomicSub(ExpiringCount, count); +} + +template <bool expiring, class TCounterType, class... TArgs> +TDynamicCounters::TCountablePtr TDynamicCounters::GetNamedCounterImpl(const TString& name, const TString& value, TArgs&&... args) { + { + TReadGuard g(Lock); + auto it = Counters.find({name, value}); + if (it != Counters.end()) { + return it->second; + } + } + + auto g = LockForUpdate("GetNamedCounterImpl", name, value); + const TChildId key(name, value); + auto it = Counters.lower_bound(key); + if (it == Counters.end() || it->first != key) { + auto value = MakeIntrusive<TCounterType>(std::forward<TArgs>(args)...); + it = Counters.emplace_hint(it, key, value); + if constexpr (expiring) { + AtomicIncrement(ExpiringCount); + } + } + return it->second; +} + +template <class TCounterType> +TDynamicCounters::TCountablePtr TDynamicCounters::FindNamedCounterImpl(const TString& name, const TString& value) const { + TReadGuard g(Lock); + auto it = Counters.find({name, value}); + return it != Counters.end() ? it->second : nullptr; +} + diff --git a/library/cpp/monlib/dynamic_counters/counters.h b/library/cpp/monlib/dynamic_counters/counters.h new file mode 100644 index 0000000000..dc178cfbe0 --- /dev/null +++ b/library/cpp/monlib/dynamic_counters/counters.h @@ -0,0 +1,374 @@ +#pragma once + +#include <library/cpp/monlib/counters/counters.h> +#include <library/cpp/monlib/metrics/histogram_collector.h> + +#include <library/cpp/threading/light_rw_lock/lightrwlock.h> +#include <library/cpp/containers/stack_vector/stack_vec.h> + +#include <util/generic/cast.h> +#include <util/generic/map.h> +#include <util/generic/ptr.h> +#include <util/string/cast.h> +#include <util/system/rwlock.h> + +#include <functional> + +namespace NMonitoring { + struct TCounterForPtr; + struct TDynamicCounters; + struct ICountableConsumer; + + + struct TCountableBase: public TAtomicRefCount<TCountableBase> { + // Private means that the object must not be serialized unless the consumer + // has explicitly specified this by setting its Visibility to Private. + // + // Works only for the methods that accept ICountableConsumer + enum class EVisibility: ui8 { + Unspecified, + Public, + Private, + }; + + virtual ~TCountableBase() { + } + + virtual void Accept( + const TString& labelName, const TString& labelValue, + ICountableConsumer& consumer) const = 0; + + virtual EVisibility Visibility() const { + return Visibility_; + } + + protected: + EVisibility Visibility_{EVisibility::Unspecified}; + }; + + inline bool IsVisible(TCountableBase::EVisibility myLevel, TCountableBase::EVisibility consumerLevel) { + if (myLevel == TCountableBase::EVisibility::Private + && consumerLevel != TCountableBase::EVisibility::Private) { + + return false; + } + + return true; + } + + struct ICountableConsumer { + virtual ~ICountableConsumer() { + } + + virtual void OnCounter( + const TString& labelName, const TString& labelValue, + const TCounterForPtr* counter) = 0; + + virtual void OnHistogram( + const TString& labelName, const TString& labelValue, + IHistogramSnapshotPtr snapshot, bool derivative) = 0; + + virtual void OnGroupBegin( + const TString& labelName, const TString& labelValue, + const TDynamicCounters* group) = 0; + + virtual void OnGroupEnd( + const TString& labelName, const TString& labelValue, + const TDynamicCounters* group) = 0; + + virtual TCountableBase::EVisibility Visibility() const { + return TCountableBase::EVisibility::Unspecified; + } + }; + +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable : 4522) // multiple assignment operators specified +#endif // _MSC_VER + + struct TCounterForPtr: public TDeprecatedCounter, public TCountableBase { + TCounterForPtr(bool derivative = false, EVisibility vis = EVisibility::Public) + : TDeprecatedCounter(0ULL, derivative) + { + Visibility_ = vis; + } + + TCounterForPtr(const TCounterForPtr&) = delete; + TCounterForPtr& operator=(const TCounterForPtr& other) = delete; + + void Accept( + const TString& labelName, const TString& labelValue, + ICountableConsumer& consumer) const override { + if (IsVisible(Visibility(), consumer.Visibility())) { + consumer.OnCounter(labelName, labelValue, this); + } + } + + TCountableBase::EVisibility Visibility() const override { + return Visibility_; + } + + using TDeprecatedCounter::operator++; + using TDeprecatedCounter::operator--; + using TDeprecatedCounter::operator+=; + using TDeprecatedCounter::operator-=; + using TDeprecatedCounter::operator=; + using TDeprecatedCounter::operator!; + }; + + struct TExpiringCounter: public TCounterForPtr { + explicit TExpiringCounter(bool derivative = false, EVisibility vis = EVisibility::Public) + : TCounterForPtr{derivative} + { + Visibility_ = vis; + } + + void Reset() { + TDeprecatedCounter::operator=(0); + } + }; + + struct THistogramCounter: public TCountableBase { + explicit THistogramCounter( + IHistogramCollectorPtr collector, bool derivative = true, EVisibility vis = EVisibility::Public) + : Collector_(std::move(collector)) + , Derivative_(derivative) + { + Visibility_ = vis; + } + + void Collect(i64 value) { + Collector_->Collect(value); + } + + void Collect(i64 value, ui32 count) { + Collector_->Collect(value, count); + } + + void Collect(double value, ui32 count) { + Collector_->Collect(value, count); + } + + void Collect(const IHistogramSnapshot& snapshot) { + Collector_->Collect(snapshot); + } + + void Accept( + const TString& labelName, const TString& labelValue, + ICountableConsumer& consumer) const override + { + if (IsVisible(Visibility(), consumer.Visibility())) { + consumer.OnHistogram(labelName, labelValue, Collector_->Snapshot(), Derivative_); + } + } + + void Reset() { + Collector_->Reset(); + } + + IHistogramSnapshotPtr Snapshot() const { + return Collector_->Snapshot(); + } + + private: + IHistogramCollectorPtr Collector_; + bool Derivative_; + }; + + struct TExpiringHistogramCounter: public THistogramCounter { + using THistogramCounter::THistogramCounter; + }; + + using THistogramPtr = TIntrusivePtr<THistogramCounter>; + +#ifdef _MSC_VER +#pragma warning(pop) +#endif + + struct TDynamicCounters; + + typedef TIntrusivePtr<TDynamicCounters> TDynamicCounterPtr; + struct TDynamicCounters: public TCountableBase { + public: + using TCounterPtr = TIntrusivePtr<TCounterForPtr>; + using TOnLookupPtr = void (*)(const char *methodName, const TString &name, const TString &value); + + private: + TRWMutex Lock; + TCounterPtr LookupCounter; // Counts lookups by name + TOnLookupPtr OnLookup = nullptr; // Called on each lookup if not nullptr, intended for lightweight tracing. + + typedef TIntrusivePtr<TCountableBase> TCountablePtr; + + struct TChildId { + TString LabelName; + TString LabelValue; + TChildId() { + } + TChildId(const TString& labelName, const TString& labelValue) + : LabelName(labelName) + , LabelValue(labelValue) + { + } + auto AsTuple() const { + return std::make_tuple(std::cref(LabelName), std::cref(LabelValue)); + } + friend bool operator <(const TChildId& x, const TChildId& y) { + return x.AsTuple() < y.AsTuple(); + } + friend bool operator ==(const TChildId& x, const TChildId& y) { + return x.AsTuple() == y.AsTuple(); + } + friend bool operator !=(const TChildId& x, const TChildId& y) { + return x.AsTuple() != y.AsTuple(); + } + }; + + using TCounters = TMap<TChildId, TCountablePtr>; + using TLabels = TVector<TChildId>; + + /// XXX: hack for deferred removal of expired counters. Remove once Output* functions are not used for serialization + mutable TCounters Counters; + mutable TAtomic ExpiringCount = 0; + + public: + TDynamicCounters(TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public); + + TDynamicCounters(const TDynamicCounters *origin) + : LookupCounter(origin->LookupCounter) + , OnLookup(origin->OnLookup) + {} + + ~TDynamicCounters() override; + + // This counter allows to track lookups by name within the whole subtree + void SetLookupCounter(TCounterPtr lookupCounter) { + TWriteGuard g(Lock); + LookupCounter = lookupCounter; + } + + void SetOnLookup(TOnLookupPtr onLookup) { + TWriteGuard g(Lock); + OnLookup = onLookup; + } + + TWriteGuard LockForUpdate(const char *method, const TString& name, const TString& value) { + auto res = TWriteGuard(Lock); + if (LookupCounter) { + ++*LookupCounter; + } + if (OnLookup) { + OnLookup(method, name, value); + } + return res; + } + + TStackVec<TCounters::value_type, 256> ReadSnapshot() const { + RemoveExpired(); + TReadGuard g(Lock); + TStackVec<TCounters::value_type, 256> items(Counters.begin(), Counters.end()); + return items; + } + + TCounterPtr GetCounter( + const TString& value, + bool derivative = false, + TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public); + + TCounterPtr GetNamedCounter( + const TString& name, + const TString& value, + bool derivative = false, + TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public); + + THistogramPtr GetHistogram( + const TString& value, + IHistogramCollectorPtr collector, + bool derivative = true, + TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public); + + THistogramPtr GetNamedHistogram( + const TString& name, + const TString& value, + IHistogramCollectorPtr collector, + bool derivative = true, + TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public); + + // These counters will be automatically removed from the registry + // when last reference to the counter expires. + TCounterPtr GetExpiringCounter( + const TString& value, + bool derivative = false, + TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public); + + TCounterPtr GetExpiringNamedCounter( + const TString& name, + const TString& value, + bool derivative = false, + TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public); + + THistogramPtr GetExpiringHistogram( + const TString& value, + IHistogramCollectorPtr collector, + bool derivative = true, + TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public); + + THistogramPtr GetExpiringNamedHistogram( + const TString& name, + const TString& value, + IHistogramCollectorPtr collector, + bool derivative = true, + TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public); + + TCounterPtr FindCounter(const TString& value) const; + TCounterPtr FindNamedCounter(const TString& name, const TString& value) const; + + THistogramPtr FindHistogram(const TString& value) const; + THistogramPtr FindNamedHistogram(const TString& name,const TString& value) const; + + void RemoveCounter(const TString &value); + void RemoveNamedCounter(const TString& name, const TString &value); + + TIntrusivePtr<TDynamicCounters> GetSubgroup(const TString& name, const TString& value); + TIntrusivePtr<TDynamicCounters> FindSubgroup(const TString& name, const TString& value) const; + void RemoveSubgroup(const TString& name, const TString& value); + void ReplaceSubgroup(const TString& name, const TString& value, TIntrusivePtr<TDynamicCounters> subgroup); + + // Move all counters from specified subgroup and remove the subgroup. + void MergeWithSubgroup(const TString& name, const TString& value); + // Recursively reset all/deriv counters to 0. + void ResetCounters(bool derivOnly = false); + + void RegisterSubgroup(const TString& name, + const TString& value, + TIntrusivePtr<TDynamicCounters> subgroup); + + void OutputHtml(IOutputStream& os) const; + void EnumerateSubgroups(const std::function<void(const TString& name, const TString& value)>& output) const; + + // mostly for debugging purposes -- use accept with encoder instead + void OutputPlainText(IOutputStream& os, const TString& indent = "") const; + + void Accept( + const TString& labelName, const TString& labelValue, + ICountableConsumer& consumer) const override; + + private: + TCounters Resign() { + TCounters counters; + TWriteGuard g(Lock); + Counters.swap(counters); + return counters; + } + + void RegisterCountable(const TString& name, const TString& value, TCountablePtr countable); + void RemoveExpired() const; + + template <bool expiring, class TCounterType, class... TArgs> + TCountablePtr GetNamedCounterImpl(const TString& name, const TString& value, TArgs&&... args); + + template <class TCounterType> + TCountablePtr FindNamedCounterImpl(const TString& name, const TString& value) const; + }; + +} diff --git a/library/cpp/monlib/dynamic_counters/counters_ut.cpp b/library/cpp/monlib/dynamic_counters/counters_ut.cpp new file mode 100644 index 0000000000..3591037e0a --- /dev/null +++ b/library/cpp/monlib/dynamic_counters/counters_ut.cpp @@ -0,0 +1,342 @@ +#include "counters.h" + +#include <library/cpp/testing/unittest/registar.h> + +using namespace NMonitoring; + +class TCountersPrinter: public ICountableConsumer { +public: + TCountersPrinter(IOutputStream* out) + : Out_(out) + , Level_(0) + { + } + +private: + void OnCounter( + const TString& labelName, const TString& labelValue, + const TCounterForPtr* counter) override { + Indent(Out_, Level_) + << labelName << ':' << labelValue + << " = " << counter->Val() << '\n'; + } + + void OnHistogram( + const TString& labelName, const TString& labelValue, + IHistogramSnapshotPtr snapshot, bool /*derivative*/) override { + Indent(Out_, Level_) + << labelName << ':' << labelValue + << " = " << *snapshot << '\n'; + } + + void OnGroupBegin( + const TString& labelName, const TString& labelValue, + const TDynamicCounters*) override { + Indent(Out_, Level_++) << labelName << ':' << labelValue << " {\n"; + } + + void OnGroupEnd( + const TString&, const TString&, + const TDynamicCounters*) override { + Indent(Out_, --Level_) << "}\n"; + } + + static IOutputStream& Indent(IOutputStream* out, int level) { + for (int i = 0; i < level; i++) { + out->Write(" "); + } + return *out; + } + +private: + IOutputStream* Out_; + int Level_ = 0; +}; + +Y_UNIT_TEST_SUITE(TDynamicCountersTest) { + Y_UNIT_TEST(CountersConsumer) { + TDynamicCounterPtr rootGroup(new TDynamicCounters()); + + auto usersCounter = rootGroup->GetNamedCounter("users", "count"); + *usersCounter = 7; + + auto hostGroup = rootGroup->GetSubgroup("counters", "resources"); + auto cpuCounter = hostGroup->GetNamedCounter("resource", "cpu"); + *cpuCounter = 30; + + auto memGroup = hostGroup->GetSubgroup("resource", "mem"); + auto usedCounter = memGroup->GetCounter("used"); + auto freeCounter = memGroup->GetCounter("free"); + *usedCounter = 100; + *freeCounter = 28; + + auto netGroup = hostGroup->GetSubgroup("resource", "net"); + auto rxCounter = netGroup->GetCounter("rx", true); + auto txCounter = netGroup->GetCounter("tx", true); + *rxCounter = 8; + *txCounter = 9; + + TStringStream ss; + TCountersPrinter printer(&ss); + rootGroup->Accept("root", "counters", printer); + + UNIT_ASSERT_STRINGS_EQUAL(ss.Str(), + "root:counters {\n" + " counters:resources {\n" + " resource:cpu = 30\n" + " resource:mem {\n" + " sensor:free = 28\n" + " sensor:used = 100\n" + " }\n" + " resource:net {\n" + " sensor:rx = 8\n" + " sensor:tx = 9\n" + " }\n" + " }\n" + " users:count = 7\n" + "}\n"); + } + + Y_UNIT_TEST(MergeSubgroup) { + TDynamicCounterPtr rootGroup(new TDynamicCounters()); + + auto sensor1 = rootGroup->GetNamedCounter("sensor", "1"); + *sensor1 = 1; + + auto group1 = rootGroup->GetSubgroup("group", "1"); + auto sensor2 = group1->GetNamedCounter("sensor", "2"); + *sensor2 = 2; + + auto group2 = group1->GetSubgroup("group", "2"); + auto sensor3 = group2->GetNamedCounter("sensor", "3"); + *sensor3 = 3; + + rootGroup->MergeWithSubgroup("group", "1"); + + TStringStream ss; + TCountersPrinter printer(&ss); + rootGroup->Accept("root", "counters", printer); + + UNIT_ASSERT_STRINGS_EQUAL(ss.Str(), + "root:counters {\n" + " group:2 {\n" + " sensor:3 = 3\n" + " }\n" + " sensor:1 = 1\n" + " sensor:2 = 2\n" + "}\n"); + } + + Y_UNIT_TEST(ResetCounters) { + TDynamicCounterPtr rootGroup(new TDynamicCounters()); + + auto sensor1 = rootGroup->GetNamedCounter("sensor", "1"); + *sensor1 = 1; + + auto group1 = rootGroup->GetSubgroup("group", "1"); + auto sensor2 = group1->GetNamedCounter("sensor", "2"); + *sensor2 = 2; + + auto group2 = group1->GetSubgroup("group", "2"); + auto sensor3 = group2->GetNamedCounter("sensor", "3", true); + *sensor3 = 3; + + rootGroup->ResetCounters(true); + + TStringStream ss1; + TCountersPrinter printer1(&ss1); + rootGroup->Accept("root", "counters", printer1); + + UNIT_ASSERT_STRINGS_EQUAL(ss1.Str(), + "root:counters {\n" + " group:1 {\n" + " group:2 {\n" + " sensor:3 = 0\n" + " }\n" + " sensor:2 = 2\n" + " }\n" + " sensor:1 = 1\n" + "}\n"); + + rootGroup->ResetCounters(); + + TStringStream ss2; + TCountersPrinter printer2(&ss2); + rootGroup->Accept("root", "counters", printer2); + + UNIT_ASSERT_STRINGS_EQUAL(ss2.Str(), + "root:counters {\n" + " group:1 {\n" + " group:2 {\n" + " sensor:3 = 0\n" + " }\n" + " sensor:2 = 0\n" + " }\n" + " sensor:1 = 0\n" + "}\n"); + } + + Y_UNIT_TEST(RemoveCounter) { + TDynamicCounterPtr rootGroup(new TDynamicCounters()); + + rootGroup->GetNamedCounter("label", "1"); + rootGroup->GetCounter("2"); + rootGroup->GetCounter("3"); + rootGroup->GetSubgroup("group", "1"); + + rootGroup->RemoveNamedCounter("label", "1"); + rootGroup->RemoveNamedCounter("label", "5"); + rootGroup->RemoveNamedCounter("group", "1"); + rootGroup->RemoveCounter("2"); + rootGroup->RemoveCounter("5"); + + TStringStream ss; + TCountersPrinter printer(&ss); + rootGroup->Accept("root", "counters", printer); + + UNIT_ASSERT_STRINGS_EQUAL(ss.Str(), + "root:counters {\n" + " group:1 {\n" + " }\n" + " sensor:3 = 0\n" + "}\n"); + } + + Y_UNIT_TEST(RemoveSubgroup) { + TDynamicCounterPtr rootGroup(new TDynamicCounters()); + + rootGroup->GetSubgroup("group", "1"); + rootGroup->GetSubgroup("group", "2"); + rootGroup->GetCounter("2"); + + rootGroup->RemoveSubgroup("group", "1"); + rootGroup->RemoveSubgroup("group", "3"); + rootGroup->RemoveSubgroup("sensor", "2"); + + TStringStream ss; + TCountersPrinter printer(&ss); + rootGroup->Accept("root", "counters", printer); + + UNIT_ASSERT_STRINGS_EQUAL(ss.Str(), + "root:counters {\n" + " group:2 {\n" + " }\n" + " sensor:2 = 0\n" + "}\n"); + } + + Y_UNIT_TEST(ExpiringCounters) { + TDynamicCounterPtr rootGroup{new TDynamicCounters()}; + + { + auto c = rootGroup->GetExpiringCounter("foo"); + auto h = rootGroup->GetExpiringHistogram("bar", ExplicitHistogram({1, 42})); + h->Collect(15); + + TStringStream ss; + TCountersPrinter printer(&ss); + rootGroup->Accept("root", "counters", printer); + UNIT_ASSERT_STRINGS_EQUAL(ss.Str(), + "root:counters {\n" + " sensor:bar = {1: 0, 42: 1, inf: 0}\n" + " sensor:foo = 0\n" + "}\n"); + } + + TStringStream ss; + TCountersPrinter printer(&ss); + rootGroup->Accept("root", "counters", printer); + UNIT_ASSERT_STRINGS_EQUAL(ss.Str(), + "root:counters {\n" + "}\n"); + } + + Y_UNIT_TEST(ExpiringCountersDiesAfterRegistry) { + TDynamicCounters::TCounterPtr ptr; + + { + TDynamicCounterPtr rootGroup{new TDynamicCounters()}; + ptr = rootGroup->GetExpiringCounter("foo"); + + TStringStream ss; + TCountersPrinter printer(&ss); + rootGroup->Accept("root", "counters", printer); + UNIT_ASSERT_STRINGS_EQUAL(ss.Str(), + "root:counters {\n" + " sensor:foo = 0\n" + "}\n"); + } + } + + Y_UNIT_TEST(HistogramCounter) { + TDynamicCounterPtr rootGroup(new TDynamicCounters()); + + auto h = rootGroup->GetHistogram("timeMillis", ExponentialHistogram(4, 2)); + for (i64 i = 1; i < 100; i++) { + h->Collect(i); + } + + TStringStream ss; + TCountersPrinter printer(&ss); + rootGroup->Accept("root", "counters", printer); + UNIT_ASSERT_STRINGS_EQUAL(ss.Str(), + "root:counters {\n" + " sensor:timeMillis = {1: 1, 2: 1, 4: 2, inf: 95}\n" + "}\n"); + } + + Y_UNIT_TEST(CounterLookupCounter) { + TDynamicCounterPtr rootGroup(new TDynamicCounters()); + TDynamicCounters::TCounterPtr lookups = rootGroup->GetCounter("Lookups", true); + rootGroup->SetLookupCounter(lookups); + + // Create subtree and check that counter is inherited + TDynamicCounterPtr serviceGroup = rootGroup->GetSubgroup("service", "MyService"); + UNIT_ASSERT_VALUES_EQUAL(lookups->Val(), 1); + + TDynamicCounterPtr subGroup = serviceGroup->GetSubgroup("component", "MyComponent"); + UNIT_ASSERT_VALUES_EQUAL(lookups->Val(), 2); + + auto counter = subGroup->GetNamedCounter("range", "20 msec", true); + UNIT_ASSERT_VALUES_EQUAL(lookups->Val(), 3); + + auto hist = subGroup->GetHistogram("timeMsec", ExponentialHistogram(4, 2)); + UNIT_ASSERT_VALUES_EQUAL(lookups->Val(), 4); + + // Replace the counter for subGroup + auto subGroupLookups = rootGroup->GetCounter("LookupsInMyComponent", true); + UNIT_ASSERT_VALUES_EQUAL(lookups->Val(), 5); + subGroup->SetLookupCounter(subGroupLookups); + auto counter2 = subGroup->GetNamedCounter("range", "30 msec", true); + UNIT_ASSERT_VALUES_EQUAL(subGroupLookups->Val(), 1); + UNIT_ASSERT_VALUES_EQUAL(lookups->Val(), 5); + } + + Y_UNIT_TEST(FindCounters) { + TDynamicCounterPtr rootGroup(new TDynamicCounters()); + + auto counter = rootGroup->FindCounter("counter1"); + UNIT_ASSERT(!counter); + rootGroup->GetCounter("counter1"); + counter = rootGroup->FindCounter("counter1"); + UNIT_ASSERT(counter); + + counter = rootGroup->FindNamedCounter("name", "counter2"); + UNIT_ASSERT(!counter); + rootGroup->GetNamedCounter("name", "counter2"); + counter = rootGroup->FindNamedCounter("name", "counter2"); + UNIT_ASSERT(counter); + + auto histogram = rootGroup->FindHistogram("histogram1"); + UNIT_ASSERT(!histogram); + rootGroup->GetHistogram("histogram1", ExponentialHistogram(4, 2)); + histogram = rootGroup->FindHistogram("histogram1"); + UNIT_ASSERT(histogram); + + histogram = rootGroup->FindNamedHistogram("name", "histogram2"); + UNIT_ASSERT(!histogram); + rootGroup->GetNamedHistogram("name", "histogram2", ExponentialHistogram(4, 2)); + histogram = rootGroup->FindNamedHistogram("name", "histogram2"); + UNIT_ASSERT(histogram); + } +} diff --git a/library/cpp/monlib/dynamic_counters/encode.cpp b/library/cpp/monlib/dynamic_counters/encode.cpp new file mode 100644 index 0000000000..ffa48d276e --- /dev/null +++ b/library/cpp/monlib/dynamic_counters/encode.cpp @@ -0,0 +1,131 @@ +#include "encode.h" + +#include <library/cpp/monlib/encode/encoder.h> +#include <library/cpp/monlib/encode/json/json.h> +#include <library/cpp/monlib/encode/spack/spack_v1.h> +#include <library/cpp/monlib/encode/prometheus/prometheus.h> + +#include <util/stream/str.h> + +namespace NMonitoring { + namespace { + constexpr TInstant ZERO_TIME = TInstant::Zero(); + + class TConsumer final: public ICountableConsumer { + using TLabel = std::pair<TString, TString>; // name, value + + public: + explicit TConsumer(NMonitoring::IMetricEncoderPtr encoderImpl, TCountableBase::EVisibility vis) + : EncoderImpl_(std::move(encoderImpl)) + , Visibility_{vis} + { + } + + void OnCounter( + const TString& labelName, const TString& labelValue, + const TCounterForPtr* counter) override { + NMonitoring::EMetricType metricType = counter->ForDerivative() + ? NMonitoring::EMetricType::RATE + : NMonitoring::EMetricType::GAUGE; + EncoderImpl_->OnMetricBegin(metricType); + EncodeLabels(labelName, labelValue); + + if (metricType == NMonitoring::EMetricType::GAUGE) { + EncoderImpl_->OnDouble(ZERO_TIME, static_cast<double>(counter->Val())); + } else { + EncoderImpl_->OnUint64(ZERO_TIME, counter->Val()); + } + + EncoderImpl_->OnMetricEnd(); + } + + void OnHistogram( + const TString& labelName, const TString& labelValue, + IHistogramSnapshotPtr snapshot, bool derivative) override { + NMonitoring::EMetricType metricType = derivative ? EMetricType::HIST_RATE : EMetricType::HIST; + + EncoderImpl_->OnMetricBegin(metricType); + EncodeLabels(labelName, labelValue); + EncoderImpl_->OnHistogram(ZERO_TIME, snapshot); + EncoderImpl_->OnMetricEnd(); + } + + void OnGroupBegin( + const TString& labelName, const TString& labelValue, + const TDynamicCounters*) override { + if (labelName.empty() && labelValue.empty()) { + // root group has empty label name and value + EncoderImpl_->OnStreamBegin(); + } else { + ParentLabels_.emplace_back(labelName, labelValue); + } + } + + void OnGroupEnd( + const TString& labelName, const TString& labelValue, + const TDynamicCounters*) override { + if (labelName.empty() && labelValue.empty()) { + // root group has empty label name and value + EncoderImpl_->OnStreamEnd(); + EncoderImpl_->Close(); + } else { + ParentLabels_.pop_back(); + } + } + + TCountableBase::EVisibility Visibility() const override { + return Visibility_; + } + + private: + void EncodeLabels(const TString& labelName, const TString& labelValue) { + EncoderImpl_->OnLabelsBegin(); + for (const auto& label : ParentLabels_) { + EncoderImpl_->OnLabel(label.first, label.second); + } + EncoderImpl_->OnLabel(labelName, labelValue); + EncoderImpl_->OnLabelsEnd(); + } + + private: + NMonitoring::IMetricEncoderPtr EncoderImpl_; + TVector<TLabel> ParentLabels_; + TCountableBase::EVisibility Visibility_; + }; + + } + + THolder<ICountableConsumer> CreateEncoder(IOutputStream* out, EFormat format, TCountableBase::EVisibility vis) { + switch (format) { + case EFormat::JSON: + return MakeHolder<TConsumer>(NMonitoring::EncoderJson(out), vis); + case EFormat::SPACK: + return MakeHolder<TConsumer>(NMonitoring::EncoderSpackV1( + out, + NMonitoring::ETimePrecision::SECONDS, + NMonitoring::ECompression::ZSTD), vis); + case EFormat::PROMETHEUS: + return MakeHolder<TConsumer>(NMonitoring::EncoderPrometheus( + out), vis); + default: + ythrow yexception() << "unsupported metric encoding format: " << format; + break; + } + } + + THolder<ICountableConsumer> AsCountableConsumer(IMetricEncoderPtr encoder, TCountableBase::EVisibility visibility) { + return MakeHolder<TConsumer>(std::move(encoder), visibility); + } + + void ToJson(const TDynamicCounters& counters, IOutputStream* out) { + TConsumer consumer{EncoderJson(out), TCountableBase::EVisibility::Public}; + counters.Accept(TString{}, TString{}, consumer); + } + + TString ToJson(const TDynamicCounters& counters) { + TStringStream ss; + ToJson(counters, &ss); + return ss.Str(); + } + +} diff --git a/library/cpp/monlib/dynamic_counters/encode.h b/library/cpp/monlib/dynamic_counters/encode.h new file mode 100644 index 0000000000..c79964d7cb --- /dev/null +++ b/library/cpp/monlib/dynamic_counters/encode.h @@ -0,0 +1,23 @@ +#pragma once + +#include "counters.h" + +#include <library/cpp/monlib/encode/encoder.h> +#include <library/cpp/monlib/encode/format.h> + +namespace NMonitoring { + + THolder<ICountableConsumer> CreateEncoder( + IOutputStream* out, + EFormat format, + TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public + ); + + THolder<ICountableConsumer> AsCountableConsumer( + NMonitoring::IMetricEncoderPtr encoder, + TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public); + + void ToJson(const TDynamicCounters& counters, IOutputStream* out); + + TString ToJson(const TDynamicCounters& counters); +} diff --git a/library/cpp/monlib/dynamic_counters/encode_ut.cpp b/library/cpp/monlib/dynamic_counters/encode_ut.cpp new file mode 100644 index 0000000000..52d77b6b41 --- /dev/null +++ b/library/cpp/monlib/dynamic_counters/encode_ut.cpp @@ -0,0 +1,226 @@ +#include "encode.h" + +#include <library/cpp/monlib/encode/json/json.h> +#include <library/cpp/monlib/encode/spack/spack_v1.h> +#include <library/cpp/monlib/encode/protobuf/protobuf.h> + +#include <library/cpp/monlib/encode/protobuf/protos/samples.pb.h> +#include <library/cpp/testing/unittest/registar.h> + +#include <util/generic/buffer.h> +#include <util/stream/buffer.h> + +namespace NMonitoring { + struct TTestData: public TDynamicCounters { + TTestData() { + auto hostGroup = GetSubgroup("counters", "resources"); + { + auto cpuCounter = hostGroup->GetNamedCounter("resource", "cpu"); + *cpuCounter = 30; + + auto memGroup = hostGroup->GetSubgroup("resource", "mem"); + auto usedCounter = memGroup->GetCounter("used"); + auto freeCounter = memGroup->GetCounter("free"); + *usedCounter = 100; + *freeCounter = 28; + + auto netGroup = hostGroup->GetSubgroup("resource", "net"); + auto rxCounter = netGroup->GetCounter("rx", true); + auto txCounter = netGroup->GetCounter("tx", true); + *rxCounter = 8; + *txCounter = 9; + } + + auto usersCounter = GetNamedCounter("users", "count"); + *usersCounter = 7; + + auto responseTimeMillis = GetHistogram("responseTimeMillis", ExplicitHistogram({1, 5, 10, 15, 20, 100, 200})); + for (i64 i = 0; i < 400; i++) { + responseTimeMillis->Collect(i); + } + } + }; + + void AssertLabelsEqual(const NProto::TLabel& l, TStringBuf name, TStringBuf value) { + UNIT_ASSERT_STRINGS_EQUAL(l.GetName(), name); + UNIT_ASSERT_STRINGS_EQUAL(l.GetValue(), value); + } + + void AssertResult(const NProto::TSingleSamplesList& samples) { + UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 7); + + { + auto s = samples.GetSamples(0); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2); + AssertLabelsEqual(s.GetLabels(0), "counters", "resources"); + AssertLabelsEqual(s.GetLabels(1), "resource", "cpu"); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_DOUBLES_EQUAL(s.GetFloat64(), 30.0, Min<double>()); + } + { + auto s = samples.GetSamples(1); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 3); + AssertLabelsEqual(s.GetLabels(0), "counters", "resources"); + AssertLabelsEqual(s.GetLabels(1), "resource", "mem"); + AssertLabelsEqual(s.GetLabels(2), "sensor", "free"); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_DOUBLES_EQUAL(s.GetFloat64(), 28.0, Min<double>()); + } + { + auto s = samples.GetSamples(2); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 3); + AssertLabelsEqual(s.GetLabels(0), "counters", "resources"); + AssertLabelsEqual(s.GetLabels(1), "resource", "mem"); + AssertLabelsEqual(s.GetLabels(2), "sensor", "used"); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_DOUBLES_EQUAL(s.GetFloat64(), 100.0, Min<double>()); + } + { + auto s = samples.GetSamples(3); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 3); + AssertLabelsEqual(s.GetLabels(0), "counters", "resources"); + AssertLabelsEqual(s.GetLabels(1), "resource", "net"); + AssertLabelsEqual(s.GetLabels(2), "sensor", "rx"); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::RATE); + UNIT_ASSERT_VALUES_EQUAL(s.GetUint64(), 8); + } + { + auto s = samples.GetSamples(4); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 3); + AssertLabelsEqual(s.GetLabels(0), "counters", "resources"); + AssertLabelsEqual(s.GetLabels(1), "resource", "net"); + AssertLabelsEqual(s.GetLabels(2), "sensor", "tx"); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::RATE); + UNIT_ASSERT_VALUES_EQUAL(s.GetUint64(), 9); + } + { + auto s = samples.GetSamples(5); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelsEqual(s.GetLabels(0), "sensor", "responseTimeMillis"); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::HIST_RATE); + + const NProto::THistogram& h = s.GetHistogram(); + + UNIT_ASSERT_EQUAL(h.BoundsSize(), 8); + UNIT_ASSERT_DOUBLES_EQUAL(h.GetBounds(0), 1, Min<double>()); + UNIT_ASSERT_DOUBLES_EQUAL(h.GetBounds(1), 5, Min<double>()); + UNIT_ASSERT_DOUBLES_EQUAL(h.GetBounds(2), 10, Min<double>()); + UNIT_ASSERT_DOUBLES_EQUAL(h.GetBounds(3), 15, Min<double>()); + UNIT_ASSERT_DOUBLES_EQUAL(h.GetBounds(4), 20, Min<double>()); + UNIT_ASSERT_DOUBLES_EQUAL(h.GetBounds(5), 100, Min<double>()); + UNIT_ASSERT_DOUBLES_EQUAL(h.GetBounds(6), 200, Min<double>()); + UNIT_ASSERT_DOUBLES_EQUAL(h.GetBounds(7), Max<double>(), Min<double>()); + + UNIT_ASSERT_EQUAL(h.ValuesSize(), 8); + UNIT_ASSERT_EQUAL(h.GetValues(0), 2); + UNIT_ASSERT_EQUAL(h.GetValues(1), 4); + UNIT_ASSERT_EQUAL(h.GetValues(2), 5); + UNIT_ASSERT_EQUAL(h.GetValues(3), 5); + UNIT_ASSERT_EQUAL(h.GetValues(4), 5); + UNIT_ASSERT_EQUAL(h.GetValues(5), 80); + UNIT_ASSERT_EQUAL(h.GetValues(6), 100); + UNIT_ASSERT_EQUAL(h.GetValues(7), 199); + } + { + auto s = samples.GetSamples(6); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelsEqual(s.GetLabels(0), "users", "count"); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_DOUBLES_EQUAL(s.GetFloat64(), 7, Min<double>()); + } + } + + Y_UNIT_TEST_SUITE(TDynamicCountersEncodeTest) { + TTestData Data; + + Y_UNIT_TEST(Json) { + TString result; + { + TStringOutput out(result); + auto encoder = CreateEncoder(&out, EFormat::JSON); + Data.Accept(TString(), TString(), *encoder); + } + + NProto::TSingleSamplesList samples; + { + auto e = EncoderProtobuf(&samples); + DecodeJson(result, e.Get()); + } + + AssertResult(samples); + } + + Y_UNIT_TEST(Spack) { + TBuffer result; + { + TBufferOutput out(result); + auto encoder = CreateEncoder(&out, EFormat::SPACK); + Data.Accept(TString(), TString(), *encoder); + } + + NProto::TSingleSamplesList samples; + { + auto e = EncoderProtobuf(&samples); + TBufferInput in(result); + DecodeSpackV1(&in, e.Get()); + } + + AssertResult(samples); + } + + Y_UNIT_TEST(PrivateSubgroupIsNotSerialized) { + TBuffer result; + auto subGroup = MakeIntrusive<TDynamicCounters>(TCountableBase::EVisibility::Private); + subGroup->GetCounter("hello"); + Data.RegisterSubgroup("foo", "bar", subGroup); + + { + TBufferOutput out(result); + auto encoder = CreateEncoder(&out, EFormat::SPACK); + Data.Accept(TString(), TString(), *encoder); + } + + NProto::TSingleSamplesList samples; + { + auto e = EncoderProtobuf(&samples); + TBufferInput in(result); + DecodeSpackV1(&in, e.Get()); + } + + AssertResult(samples); + } + + Y_UNIT_TEST(PrivateCounterIsNotSerialized) { + TBuffer result; + Data.GetCounter("foo", false, TCountableBase::EVisibility::Private); + + { + TBufferOutput out(result); + auto encoder = CreateEncoder(&out, EFormat::SPACK); + Data.Accept(TString(), TString(), *encoder); + } + + NProto::TSingleSamplesList samples; + { + auto e = EncoderProtobuf(&samples); + TBufferInput in(result); + DecodeSpackV1(&in, e.Get()); + } + + AssertResult(samples); + } + + Y_UNIT_TEST(ToJson) { + TString result = ToJson(Data); + + NProto::TSingleSamplesList samples; + { + auto e = EncoderProtobuf(&samples); + DecodeJson(result, e.Get()); + } + + AssertResult(samples); + } + } + +} diff --git a/library/cpp/monlib/dynamic_counters/golovan_page.cpp b/library/cpp/monlib/dynamic_counters/golovan_page.cpp new file mode 100644 index 0000000000..49cf2d39bb --- /dev/null +++ b/library/cpp/monlib/dynamic_counters/golovan_page.cpp @@ -0,0 +1,79 @@ +#include "golovan_page.h" + +#include <library/cpp/monlib/service/pages/templates.h> + +#include <util/string/split.h> +#include <util/system/tls.h> + +using namespace NMonitoring; + +class TGolovanCountableConsumer: public ICountableConsumer { +public: + using TOutputCallback = std::function<void()>; + + TGolovanCountableConsumer(IOutputStream& out, TOutputCallback& OutputCallback) + : out(out) + { + if (OutputCallback) { + OutputCallback(); + } + + out << HTTPOKJSON << "["; + FirstCounter = true; + } + + void OnCounter(const TString&, const TString& value, const TCounterForPtr* counter) override { + if (FirstCounter) { + FirstCounter = false; + } else { + out << ","; + } + + out << "[\"" << prefix + value; + if (counter->ForDerivative()) { + out << "_dmmm"; + } else { + out << "_ahhh"; + } + + out << "\"," << counter->Val() << "]"; + } + + void OnHistogram(const TString&, const TString&, IHistogramSnapshotPtr, bool) override { + } + + void OnGroupBegin(const TString&, const TString& value, const TDynamicCounters*) override { + prefix += value; + if (!value.empty()) { + prefix += "_"; + } + } + + void OnGroupEnd(const TString&, const TString&, const TDynamicCounters*) override { + prefix = ""; + } + + void Flush() { + out << "]"; + out.Flush(); + } + +private: + IOutputStream& out; + bool FirstCounter; + TString prefix; +}; + +TGolovanCountersPage::TGolovanCountersPage(const TString& path, TIntrusivePtr<NMonitoring::TDynamicCounters> counters, + TOutputCallback outputCallback) + : IMonPage(path) + , Counters(counters) + , OutputCallback(outputCallback) +{ +} + +void TGolovanCountersPage::Output(IMonHttpRequest& request) { + TGolovanCountableConsumer consumer(request.Output(), OutputCallback); + Counters->Accept("", "", consumer); + consumer.Flush(); +} diff --git a/library/cpp/monlib/dynamic_counters/golovan_page.h b/library/cpp/monlib/dynamic_counters/golovan_page.h new file mode 100644 index 0000000000..e1772c7734 --- /dev/null +++ b/library/cpp/monlib/dynamic_counters/golovan_page.h @@ -0,0 +1,25 @@ +#pragma once + +#include "counters.h" + +#include <library/cpp/monlib/service/pages/mon_page.h> + +#include <util/generic/ptr.h> + +#include <functional> + +// helper class to output json for Golovan. +class TGolovanCountersPage: public NMonitoring::IMonPage { +public: + using TOutputCallback = std::function<void()>; + + const TIntrusivePtr<NMonitoring::TDynamicCounters> Counters; + + TGolovanCountersPage(const TString& path, TIntrusivePtr<NMonitoring::TDynamicCounters> counters, + TOutputCallback outputCallback = nullptr); + + void Output(NMonitoring::IMonHttpRequest& request) override; + +private: + TOutputCallback OutputCallback; +}; diff --git a/library/cpp/monlib/dynamic_counters/page.cpp b/library/cpp/monlib/dynamic_counters/page.cpp new file mode 100644 index 0000000000..5124a47bb3 --- /dev/null +++ b/library/cpp/monlib/dynamic_counters/page.cpp @@ -0,0 +1,141 @@ +#include "page.h" +#include "encode.h" + +#include <library/cpp/monlib/service/pages/templates.h> +#include <library/cpp/string_utils/quote/quote.h> + +#include <util/string/split.h> +#include <util/system/tls.h> + +using namespace NMonitoring; + +namespace { + Y_POD_STATIC_THREAD(TDynamicCounters*) + currentCounters(nullptr); +} + +TMaybe<EFormat> ParseFormat(TStringBuf str) { + if (str == TStringBuf("json")) { + return EFormat::JSON; + } else if (str == TStringBuf("spack")) { + return EFormat::SPACK; + } else if (str == TStringBuf("prometheus")) { + return EFormat::PROMETHEUS; + } else { + return Nothing(); + } +} + +void TDynamicCountersPage::Output(NMonitoring::IMonHttpRequest& request) { + if (OutputCallback) { + OutputCallback(); + } + + TCountableBase::EVisibility visibility{ + TCountableBase::EVisibility::Public + }; + + TVector<TStringBuf> parts; + StringSplitter(request.GetPathInfo()) + .Split('/') + .SkipEmpty() + .Collect(&parts); + + TMaybe<EFormat> format = !parts.empty() ? ParseFormat(parts.back()) : Nothing(); + if (format) { + parts.pop_back(); + } + + if (!parts.empty() && parts.back() == TStringBuf("private")) { + visibility = TCountableBase::EVisibility::Private; + parts.pop_back(); + } + + auto counters = Counters; + + for (const auto& escaped : parts) { + const auto part = CGIUnescapeRet(escaped); + + TVector<TString> labels; + StringSplitter(part).Split('=').SkipEmpty().Collect(&labels); + + if (labels.size() != 2U) + return NotFound(request); + + if (const auto child = counters->FindSubgroup( + labels.front(), + labels.back())) { + + counters = child; + } else { + return HandleAbsentSubgroup(request); + } + } + + if (!format) { + currentCounters = counters.Get(); + THtmlMonPage::Output(request); + currentCounters = nullptr; + return; + } + + IOutputStream& out = request.Output(); + if (*format == EFormat::JSON) { + out << HTTPOKJSON; + } else if (*format == EFormat::SPACK) { + out << HTTPOKSPACK; + } else if (*format == EFormat::PROMETHEUS) { + out << HTTPOKPROMETHEUS; + } else { + ythrow yexception() << "unsupported metric encoding format: " << *format; + } + + auto encoder = CreateEncoder(&out, *format, visibility); + counters->Accept(TString(), TString(), *encoder); + out.Flush(); +} + +void TDynamicCountersPage::HandleAbsentSubgroup(IMonHttpRequest& request) { + if (UnknownGroupPolicy == EUnknownGroupPolicy::Error) { + NotFound(request); + } else if (UnknownGroupPolicy == EUnknownGroupPolicy::Ignore) { + NoContent(request); + } else { + Y_FAIL("Unsupported policy set"); + } +} + +void TDynamicCountersPage::BeforePre(IMonHttpRequest& request) { + IOutputStream& out = request.Output(); + HTML(out) { + DIV() { + out << "<a href='" << request.GetPath() << "/json'>Counters as JSON</a>"; + out << " for <a href='https://wiki.yandex-team.ru/solomon/'>Solomon</a>"; + } + + H5() { + out << "Counters subgroups"; + } + UL() { + currentCounters->EnumerateSubgroups([&](const TString& name, const TString& value) { + LI() { + TString pathPart = name + "=" + value; + Quote(pathPart, ""); + out << "\n<a href='" << request.GetPath() << "/" << pathPart << "'>" << name << " " << value << "</a>"; + } + }); + } + + H4() { + out << "Counters as text"; + } + } +} + +void TDynamicCountersPage::OutputText(IOutputStream& out, IMonHttpRequest&) { + currentCounters->OutputPlainText(out); +} + +void TDynamicCountersPage::SetUnknownGroupPolicy(EUnknownGroupPolicy value) { + UnknownGroupPolicy = value; +} diff --git a/library/cpp/monlib/dynamic_counters/page.h b/library/cpp/monlib/dynamic_counters/page.h new file mode 100644 index 0000000000..1f0ef6a5ea --- /dev/null +++ b/library/cpp/monlib/dynamic_counters/page.h @@ -0,0 +1,50 @@ +#pragma once + +#include "counters.h" + +#include <library/cpp/monlib/service/pages/pre_mon_page.h> + +#include <util/generic/ptr.h> + +#include <functional> + +namespace NMonitoring { + enum class EUnknownGroupPolicy { + Error, // send 404 + Ignore, // send 204 + }; + + struct TDynamicCountersPage: public TPreMonPage { + public: + using TOutputCallback = std::function<void()>; + + private: + const TIntrusivePtr<TDynamicCounters> Counters; + TOutputCallback OutputCallback; + EUnknownGroupPolicy UnknownGroupPolicy {EUnknownGroupPolicy::Error}; + + private: + void HandleAbsentSubgroup(IMonHttpRequest& request); + + public: + TDynamicCountersPage(const TString& path, + const TString& title, + TIntrusivePtr<TDynamicCounters> counters, + TOutputCallback outputCallback = nullptr) + : TPreMonPage(path, title) + , Counters(counters) + , OutputCallback(outputCallback) + { + } + + void Output(NMonitoring::IMonHttpRequest& request) override; + + void BeforePre(NMonitoring::IMonHttpRequest& request) override; + + void OutputText(IOutputStream& out, NMonitoring::IMonHttpRequest&) override; + + /// If set to Error, responds with 404 if the requested subgroup is not found. This is the default. + /// If set to Ignore, responds with 204 if the requested subgroup is not found + void SetUnknownGroupPolicy(EUnknownGroupPolicy value); + }; +} diff --git a/library/cpp/monlib/dynamic_counters/percentile/percentile.h b/library/cpp/monlib/dynamic_counters/percentile/percentile.h new file mode 100644 index 0000000000..73c482bce9 --- /dev/null +++ b/library/cpp/monlib/dynamic_counters/percentile/percentile.h @@ -0,0 +1,59 @@ +#pragma once + +#include "percentile_base.h" + +namespace NMonitoring { + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Percentile tracker for monitoring +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +template <size_t BUCKET_SIZE, size_t BUCKET_COUNT, size_t FRAME_COUNT> +struct TPercentileTracker : public TPercentileBase { + TAtomic Items[BUCKET_COUNT]; + TAtomicBase Frame[FRAME_COUNT][BUCKET_COUNT]; + size_t CurrentFrame; + + TPercentileTracker() + : CurrentFrame(0) + { + for (size_t i = 0; i < BUCKET_COUNT; ++i) { + AtomicSet(Items[i], 0); + } + for (size_t frame = 0; frame < FRAME_COUNT; ++frame) { + for (size_t bucket = 0; bucket < BUCKET_COUNT; ++bucket) { + Frame[frame][bucket] = 0; + } + } + } + + void Increment(size_t value) { + AtomicIncrement(Items[Min((value + BUCKET_SIZE - 1) / BUCKET_SIZE, BUCKET_COUNT - 1)]); + } + + // shift frame (call periodically) + void Update() { + TVector<TAtomicBase> totals(BUCKET_COUNT); + totals.resize(BUCKET_COUNT); + TAtomicBase total = 0; + for (size_t i = 0; i < BUCKET_COUNT; ++i) { + TAtomicBase item = AtomicGet(Items[i]); + TAtomicBase prevItem = Frame[CurrentFrame][i]; + Frame[CurrentFrame][i] = item; + total += item - prevItem; + totals[i] = total; + } + + for (size_t i = 0; i < Percentiles.size(); ++i) { + TPercentile &percentile = Percentiles[i]; + auto threshold = (TAtomicBase)(percentile.first * (float)total); + threshold = Min(threshold, total); + auto it = LowerBound(totals.begin(), totals.end(), threshold); + size_t index = it - totals.begin(); + (*percentile.second) = index * BUCKET_SIZE; + } + CurrentFrame = (CurrentFrame + 1) % FRAME_COUNT; + } +}; + +} // NMonitoring diff --git a/library/cpp/monlib/dynamic_counters/percentile/percentile_base.h b/library/cpp/monlib/dynamic_counters/percentile/percentile_base.h new file mode 100644 index 0000000000..d3c825c43d --- /dev/null +++ b/library/cpp/monlib/dynamic_counters/percentile/percentile_base.h @@ -0,0 +1,36 @@ +#pragma once + +#include <library/cpp/monlib/dynamic_counters/counters.h> + +#include <util/string/printf.h> + +namespace NMonitoring { + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Percentile tracker for monitoring +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +struct TPercentileBase : public TThrRefBase { + using TPercentile = std::pair<float, NMonitoring::TDynamicCounters::TCounterPtr>; + using TPercentiles = TVector<TPercentile>; + + TPercentiles Percentiles; + + void Initialize(const TIntrusivePtr<NMonitoring::TDynamicCounters> &counters, const TVector<float> &thresholds, + TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public) { + Percentiles.reserve(thresholds.size()); + for (size_t i = 0; i < thresholds.size(); ++i) { + Percentiles.emplace_back(thresholds[i], + counters->GetNamedCounter("percentile", Sprintf("%.1f", thresholds[i] * 100.f), false, visibility)); + } + } + + void Initialize(const TIntrusivePtr<NMonitoring::TDynamicCounters> &counters, TString group, TString subgroup, + TString name, const TVector<float> &thresholds, + TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public) { + auto subCounters = counters->GetSubgroup(group, subgroup)->GetSubgroup("sensor", name); + Initialize(subCounters, thresholds, visibility); + } +}; + +} // NMonitoring diff --git a/library/cpp/monlib/dynamic_counters/percentile/percentile_lg.h b/library/cpp/monlib/dynamic_counters/percentile/percentile_lg.h new file mode 100644 index 0000000000..0042cd9a6a --- /dev/null +++ b/library/cpp/monlib/dynamic_counters/percentile/percentile_lg.h @@ -0,0 +1,182 @@ +#pragma once + +#include <library/cpp/containers/stack_vector/stack_vec.h> + +#include <util/generic/bitops.h> + +#include <cmath> + +#include "percentile_base.h" + +namespace NMonitoring { + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Percentile tracker for monitoring +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +template <size_t BASE_BITS, size_t EXP_BITS, size_t FRAME_COUNT> +struct TPercentileTrackerLg : public TPercentileBase { + static constexpr size_t BUCKET_COUNT = size_t(1) << EXP_BITS; + static constexpr size_t BUCKET_SIZE = size_t(1) << BASE_BITS; + static constexpr size_t ITEMS_COUNT = BUCKET_COUNT * BUCKET_SIZE; + static constexpr size_t TRACKER_LIMIT = BUCKET_SIZE * ((size_t(1) << BUCKET_COUNT) - 1) + - (size_t(1) << (BUCKET_COUNT - 1)); + static constexpr size_t MAX_GRANULARITY = size_t(1) << (BUCKET_COUNT - 1); + + size_t Borders[BUCKET_COUNT]; + TAtomic Items[ITEMS_COUNT]; + TAtomicBase Frame[FRAME_COUNT][ITEMS_COUNT]; + size_t CurrentFrame; + + TPercentileTrackerLg() + : CurrentFrame(0) + { + Borders[0] = 0; + for (size_t i = 1; i < BUCKET_COUNT; ++i) { + Borders[i] = Borders[i-1] + (BUCKET_SIZE << (i - 1)); + } + for (size_t i = 0; i < ITEMS_COUNT; ++i) { + AtomicSet(Items[i], 0); + } + for (size_t frame = 0; frame < FRAME_COUNT; ++frame) { + for (size_t bucket = 0; bucket < ITEMS_COUNT; ++bucket) { + Frame[frame][bucket] = 0; + } + } + } + + size_t inline BucketIdxIf(size_t value) { + static_assert(BASE_BITS == 5, "if-based bucket calculation cannot be used if BASE_BITS != 5"); + size_t bucket_idx; + if (value < 8160) { + if (value < 480) { + if (value < 96) { + if (value < 32) { + bucket_idx = 0; + } else { + bucket_idx = 1; + } + } else { + if (value < 224) { + bucket_idx = 2; + } else { + bucket_idx = 3; + } + } + } else { + if (value < 2016) { + if (value < 992) { + bucket_idx = 4; + } else { + bucket_idx = 5; + } + } else { + if (value < 4064) { + bucket_idx = 6; + } else { + bucket_idx = 7; + } + } + } + } else { + if (value < 131040) { + if (value < 32736) { + if (value < 16352) { + bucket_idx = 8; + } else { + bucket_idx = 9; + } + } else { + if (value < 65504) { + bucket_idx = 10; + } else { + bucket_idx = 11; + } + } + } else { + if (value < 524256) { + if (value < 262112) { + bucket_idx = 12; + } else { + bucket_idx = 13; + } + } else { + if (value < 1048544) { + bucket_idx = 14; + } else { + bucket_idx = 15; + } + } + } + } + return Min(bucket_idx, BUCKET_COUNT - 1); + } + + size_t inline BucketIdxBinarySearch(size_t value) { + size_t l = 0; + size_t r = BUCKET_COUNT; + while (l < r - 1) { + size_t mid = (l + r) / 2; + if (value < Borders[mid]) { + r = mid; + } else { + l = mid; + } + } + return l; + } + + size_t inline BucketIdxMostSignificantBit(size_t value) { + size_t bucket_idx = MostSignificantBit(value + BUCKET_SIZE) - BASE_BITS; + return Min(bucket_idx, BUCKET_COUNT - 1); + } + + void Increment(size_t value) { + size_t bucket_idx = BucketIdxMostSignificantBit(value); + size_t inside_bucket_idx = (value - Borders[bucket_idx] + (1 << bucket_idx) - 1) >> bucket_idx; + size_t idx = bucket_idx * BUCKET_SIZE + inside_bucket_idx; + AtomicIncrement(Items[Min(idx, ITEMS_COUNT - 1)]); + } + + // Needed only for tests + size_t GetPercentile(float threshold) { + TStackVec<TAtomicBase, ITEMS_COUNT> totals(ITEMS_COUNT); + TAtomicBase total = 0; + for (size_t i = 0; i < ITEMS_COUNT; ++i) { + total += AtomicGet(Items[i]); + totals[i] = total; + } + TAtomicBase item_threshold = std::llround(threshold * (float)total); + item_threshold = Min(item_threshold, total); + auto it = LowerBound(totals.begin(), totals.end(), item_threshold); + size_t index = it - totals.begin(); + size_t bucket_idx = index / BUCKET_SIZE; + return Borders[bucket_idx] + ((index % BUCKET_SIZE) << bucket_idx); + } + + // shift frame (call periodically) + void Update() { + TStackVec<TAtomicBase, ITEMS_COUNT> totals(ITEMS_COUNT); + TAtomicBase total = 0; + for (size_t i = 0; i < ITEMS_COUNT; ++i) { + TAtomicBase item = AtomicGet(Items[i]); + TAtomicBase prevItem = Frame[CurrentFrame][i]; + Frame[CurrentFrame][i] = item; + total += item - prevItem; + totals[i] = total; + } + + for (size_t i = 0; i < Percentiles.size(); ++i) { + TPercentile &percentile = Percentiles[i]; + TAtomicBase threshold = std::llround(percentile.first * (float)total); + threshold = Min(threshold, total); + auto it = LowerBound(totals.begin(), totals.end(), threshold); + size_t index = it - totals.begin(); + size_t bucket_idx = index / BUCKET_SIZE; + (*percentile.second) = Borders[bucket_idx] + ((index % BUCKET_SIZE) << bucket_idx); + } + CurrentFrame = (CurrentFrame + 1) % FRAME_COUNT; + } +}; + +} // NMonitoring diff --git a/library/cpp/monlib/dynamic_counters/percentile/percentile_ut.cpp b/library/cpp/monlib/dynamic_counters/percentile/percentile_ut.cpp new file mode 100644 index 0000000000..6c8bb54ec9 --- /dev/null +++ b/library/cpp/monlib/dynamic_counters/percentile/percentile_ut.cpp @@ -0,0 +1,129 @@ +#include "percentile_lg.h" +#include <library/cpp/testing/unittest/registar.h> + +using namespace NMonitoring; + +Y_UNIT_TEST_SUITE(PercentileTest) { + +template<size_t A, size_t B, size_t B_BEGIN> +void printSizeAndLimit() { + using TPerc = TPercentileTrackerLg<A, B, 15>; + Cout << "TPercentileTrackerLg<" << A << ", " << B << ", 15>" + << "; sizeof# " << LeftPad(HumanReadableSize(sizeof(TPerc), SF_BYTES), 7) + << "; max_granularity# " << LeftPad(HumanReadableSize(TPerc::MAX_GRANULARITY, SF_QUANTITY), 5) + << "; limit# " << LeftPad(HumanReadableSize(TPerc::TRACKER_LIMIT , SF_QUANTITY), 5) << Endl; + if constexpr (B > 1) { + printSizeAndLimit<A, B - 1, B_BEGIN>(); + } else if constexpr (A > 1) { + Cout << Endl; + printSizeAndLimit<A - 1, B_BEGIN, B_BEGIN>(); + } +} + + Y_UNIT_TEST(PrintTrackerLgSizeAndLimits) { + printSizeAndLimit<10, 5, 5>(); + } + + Y_UNIT_TEST(TrackerLimitTest) { + { + using TPerc = TPercentileTrackerLg<1, 0, 1>; + TPerc tracker; + tracker.Increment(Max<size_t>()); + UNIT_ASSERT_EQUAL(TPerc::TRACKER_LIMIT, tracker.GetPercentile(1.0)); + } + { + using TPerc = TPercentileTrackerLg<1, 1, 1>; + TPerc tracker; + tracker.Increment(Max<size_t>()); + UNIT_ASSERT_EQUAL(TPerc::TRACKER_LIMIT, tracker.GetPercentile(1.0)); + } + { + using TPerc = TPercentileTrackerLg<1, 5, 1>; + TPerc tracker; + tracker.Increment(Max<size_t>()); + UNIT_ASSERT_EQUAL(TPerc::TRACKER_LIMIT, tracker.GetPercentile(1.0)); + } + { + using TPerc = TPercentileTrackerLg<2, 1, 1>; + TPerc tracker; + tracker.Increment(Max<size_t>()); + UNIT_ASSERT_EQUAL(TPerc::TRACKER_LIMIT, tracker.GetPercentile(1.0)); + } + { + using TPerc = TPercentileTrackerLg<5, 4, 1>; + TPerc tracker; + tracker.Increment(Max<size_t>()); + UNIT_ASSERT_EQUAL(TPerc::TRACKER_LIMIT, tracker.GetPercentile(1.0)); + } + } + + Y_UNIT_TEST(BucketIdxIfvsBucketIdxBinarySearch) { + for (size_t var = 0; var < 5; var++) { + if (var == 0) { + TPercentileTrackerLg<3, 2, 15> tracker; + for (size_t i = 0; i < 3000000; i += 1) { + size_t num1 = tracker.BucketIdxMostSignificantBit(i); + size_t num2 = tracker.BucketIdxBinarySearch(i); + UNIT_ASSERT_EQUAL(num1, num2); + } + } else if (var == 1) { + TPercentileTrackerLg<4, 4, 15> tracker; + for (size_t i = 0; i < 3000000; i += 1) { + size_t num1 = tracker.BucketIdxMostSignificantBit(i); + size_t num2 = tracker.BucketIdxBinarySearch(i); + UNIT_ASSERT_EQUAL(num1, num2); + } + } else if (var == 2) { + TPercentileTrackerLg<5, 3, 15> tracker; + for (size_t i = 0; i < 3000000; i += 1) { + size_t num1 = tracker.BucketIdxMostSignificantBit(i); + size_t num2 = tracker.BucketIdxBinarySearch(i); + size_t num3 = tracker.BucketIdxIf(i); + UNIT_ASSERT_EQUAL(num1, num2); + UNIT_ASSERT_EQUAL(num2, num3); + } + } else if (var == 3) { + TPercentileTrackerLg<5, 4, 15> tracker; + for (size_t i = 0; i < 3000000; i += 1) { + size_t num1 = tracker.BucketIdxMostSignificantBit(i); + size_t num2 = tracker.BucketIdxBinarySearch(i); + size_t num3 = tracker.BucketIdxIf(i); + UNIT_ASSERT_EQUAL(num1, num2); + UNIT_ASSERT_EQUAL(num2, num3); + } + } else if (var == 4) { + TPercentileTrackerLg<6, 5, 15> tracker; + for (size_t i = 0; i < 3000000; i += 1) { + size_t num1 = tracker.BucketIdxMostSignificantBit(i); + size_t num2 = tracker.BucketIdxBinarySearch(i); + UNIT_ASSERT_EQUAL(num1, num2); + } + for (size_t i = 0; i < 400000000000ul; i += 1303) { + size_t num1 = tracker.BucketIdxMostSignificantBit(i); + size_t num2 = tracker.BucketIdxBinarySearch(i); + UNIT_ASSERT_EQUAL(num1, num2); + } + } + } + } + + Y_UNIT_TEST(DifferentPercentiles) { + TPercentileTrackerLg<5, 4, 15> tracker; + TVector<size_t> values({0, 115, 1216, 15, 3234567, 1234567, 216546, 263421, 751654, 96, 224, 223, 225}); + TVector<size_t> percentiles50({0, 0, 116, 15, 116, 116, 1216, 1216, 217056, 1216, 1216, 224, 232}); + TVector<size_t> percentiles75({0, 116, 116, 116, 1216, 1245152, 217056, 270304, 753632, 753632, + 270304, 270304, 270304}); + TVector<size_t> percentiles90({ 0, 116, 1216, 1216, 2064352, 1245152, 1245152, 1245152, 1245152, + 1245152, 1245152, 1245152, 1245152}); + TVector<size_t> percentiles100({ 0, 116, 1216, 1216, 2064352, 2064352, 2064352, 2064352, 2064352, + 2064352, 2064352, 2064352, 2064352 }); + + for (size_t i = 0; i < values.size(); ++i) { + tracker.Increment(values[i]); + UNIT_ASSERT_EQUAL(tracker.GetPercentile(0.5), percentiles50[i]); + UNIT_ASSERT_EQUAL(tracker.GetPercentile(0.75), percentiles75[i]); + UNIT_ASSERT_EQUAL(tracker.GetPercentile(0.90), percentiles90[i]); + UNIT_ASSERT_EQUAL(tracker.GetPercentile(1.0), percentiles100[i]); + } + } +} diff --git a/library/cpp/monlib/dynamic_counters/percentile/ut/ya.make b/library/cpp/monlib/dynamic_counters/percentile/ut/ya.make new file mode 100644 index 0000000000..f9f3564101 --- /dev/null +++ b/library/cpp/monlib/dynamic_counters/percentile/ut/ya.make @@ -0,0 +1,9 @@ +UNITTEST_FOR(library/cpp/monlib/dynamic_counters/percentile) + + OWNER(alexvru g:kikimr g:solomon) + + SRCS( + percentile_ut.cpp + ) + +END() diff --git a/library/cpp/monlib/dynamic_counters/percentile/ya.make b/library/cpp/monlib/dynamic_counters/percentile/ya.make new file mode 100644 index 0000000000..cb52cdd9ad --- /dev/null +++ b/library/cpp/monlib/dynamic_counters/percentile/ya.make @@ -0,0 +1,15 @@ +LIBRARY() + + OWNER(alexvru g:kikimr g:solomon) + + SRCS( + percentile.h + percentile_lg.h + ) + + PEERDIR( + library/cpp/containers/stack_vector + library/cpp/monlib/dynamic_counters + ) + +END() diff --git a/library/cpp/monlib/dynamic_counters/ut/ya.make b/library/cpp/monlib/dynamic_counters/ut/ya.make new file mode 100644 index 0000000000..8242f2fe30 --- /dev/null +++ b/library/cpp/monlib/dynamic_counters/ut/ya.make @@ -0,0 +1,16 @@ +UNITTEST_FOR(library/cpp/monlib/dynamic_counters) + +OWNER(jamel) + +SRCS( + contention_ut.cpp + counters_ut.cpp + encode_ut.cpp +) + +PEERDIR( + library/cpp/monlib/encode/protobuf + library/cpp/monlib/encode/json +) + +END() diff --git a/library/cpp/monlib/dynamic_counters/ya.make b/library/cpp/monlib/dynamic_counters/ya.make new file mode 100644 index 0000000000..aafe1c34be --- /dev/null +++ b/library/cpp/monlib/dynamic_counters/ya.make @@ -0,0 +1,27 @@ +LIBRARY() + +OWNER( + g:solomon + jamel +) + +NO_WSHADOW() + +SRCS( + counters.cpp + encode.cpp + golovan_page.cpp + page.cpp +) + +PEERDIR( + library/cpp/containers/stack_vector + library/cpp/monlib/encode/json + library/cpp/monlib/encode/spack + library/cpp/monlib/encode/prometheus + library/cpp/monlib/service/pages + library/cpp/string_utils/quote + library/cpp/threading/light_rw_lock +) + +END() diff --git a/library/cpp/monlib/encode/buffered/buffered_encoder_base.cpp b/library/cpp/monlib/encode/buffered/buffered_encoder_base.cpp new file mode 100644 index 0000000000..87c832d642 --- /dev/null +++ b/library/cpp/monlib/encode/buffered/buffered_encoder_base.cpp @@ -0,0 +1,170 @@ +#include "buffered_encoder_base.h" + +#include <util/string/join.h> +#include <util/string/builder.h> + +namespace NMonitoring { + +void TBufferedEncoderBase::OnStreamBegin() { + State_.Expect(TEncoderState::EState::ROOT); +} + +void TBufferedEncoderBase::OnStreamEnd() { + State_.Expect(TEncoderState::EState::ROOT); +} + +void TBufferedEncoderBase::OnCommonTime(TInstant time) { + State_.Expect(TEncoderState::EState::ROOT); + CommonTime_ = time; +} + +void TBufferedEncoderBase::OnMetricBegin(EMetricType type) { + State_.Switch(TEncoderState::EState::ROOT, TEncoderState::EState::METRIC); + Metrics_.emplace_back(); + Metrics_.back().MetricType = type; +} + +void TBufferedEncoderBase::OnMetricEnd() { + State_.Switch(TEncoderState::EState::METRIC, TEncoderState::EState::ROOT); + + switch (MetricsMergingMode_) { + case EMetricsMergingMode::MERGE_METRICS: { + auto& metric = Metrics_.back(); + Sort(metric.Labels, [] (const TPooledLabel& lhs, const TPooledLabel& rhs) { + return std::tie(lhs.Key, lhs.Value) < std::tie(rhs.Key, rhs.Value); + }); + + auto it = MetricMap_.find(metric.Labels); + if (it == std::end(MetricMap_)) { + MetricMap_.emplace(metric.Labels, Metrics_.size() - 1); + } else { + auto& existing = Metrics_[it->second].TimeSeries; + + Y_ENSURE(existing.GetValueType() == metric.TimeSeries.GetValueType(), + "Time series point type mismatch: expected " << existing.GetValueType() + << " but found " << metric.TimeSeries.GetValueType() + << ", labels '" << FormatLabels(metric.Labels) << "'"); + + existing.CopyFrom(metric.TimeSeries); + Metrics_.pop_back(); + } + + break; + } + case EMetricsMergingMode::DEFAULT: + break; + } +} + +void TBufferedEncoderBase::OnLabelsBegin() { + if (State_ == TEncoderState::EState::METRIC) { + State_ = TEncoderState::EState::METRIC_LABELS; + } else if (State_ == TEncoderState::EState::ROOT) { + State_ = TEncoderState::EState::COMMON_LABELS; + } else { + State_.ThrowInvalid("expected METRIC or ROOT"); + } +} + +void TBufferedEncoderBase::OnLabelsEnd() { + if (State_ == TEncoderState::EState::METRIC_LABELS) { + State_ = TEncoderState::EState::METRIC; + } else if (State_ == TEncoderState::EState::COMMON_LABELS) { + State_ = TEncoderState::EState::ROOT; + } else { + State_.ThrowInvalid("expected LABELS or COMMON_LABELS"); + } +} + +void TBufferedEncoderBase::OnLabel(TStringBuf name, TStringBuf value) { + TPooledLabels* labels; + if (State_ == TEncoderState::EState::METRIC_LABELS) { + labels = &Metrics_.back().Labels; + } else if (State_ == TEncoderState::EState::COMMON_LABELS) { + labels = &CommonLabels_; + } else { + State_.ThrowInvalid("expected LABELS or COMMON_LABELS"); + } + + labels->emplace_back(LabelNamesPool_.PutIfAbsent(name), LabelValuesPool_.PutIfAbsent(value)); +} + +void TBufferedEncoderBase::OnLabel(ui32 name, ui32 value) { + TPooledLabels* labels; + if (State_ == TEncoderState::EState::METRIC_LABELS) { + labels = &Metrics_.back().Labels; + } else if (State_ == TEncoderState::EState::COMMON_LABELS) { + labels = &CommonLabels_; + } else { + State_.ThrowInvalid("expected LABELS or COMMON_LABELS"); + } + + labels->emplace_back(LabelNamesPool_.GetByIndex(name), LabelValuesPool_.GetByIndex(value)); +} + +std::pair<ui32, ui32> TBufferedEncoderBase::PrepareLabel(TStringBuf name, TStringBuf value) { + auto nameLabel = LabelNamesPool_.PutIfAbsent(name); + auto valueLabel = LabelValuesPool_.PutIfAbsent(value); + return std::make_pair(nameLabel->Index, valueLabel->Index); +} + +void TBufferedEncoderBase::OnDouble(TInstant time, double value) { + State_.Expect(TEncoderState::EState::METRIC); + TMetric& metric = Metrics_.back(); + metric.TimeSeries.Add(time, value); +} + +void TBufferedEncoderBase::OnInt64(TInstant time, i64 value) { + State_.Expect(TEncoderState::EState::METRIC); + TMetric& metric = Metrics_.back(); + metric.TimeSeries.Add(time, value); +} + +void TBufferedEncoderBase::OnUint64(TInstant time, ui64 value) { + State_.Expect(TEncoderState::EState::METRIC); + TMetric& metric = Metrics_.back(); + metric.TimeSeries.Add(time, value); +} + +void TBufferedEncoderBase::OnHistogram(TInstant time, IHistogramSnapshotPtr s) { + State_.Expect(TEncoderState::EState::METRIC); + TMetric& metric = Metrics_.back(); + metric.TimeSeries.Add(time, s.Get()); +} + +void TBufferedEncoderBase::OnSummaryDouble(TInstant time, ISummaryDoubleSnapshotPtr s) { + State_.Expect(TEncoderState::EState::METRIC); + TMetric& metric = Metrics_.back(); + metric.TimeSeries.Add(time, s.Get()); +} + +void TBufferedEncoderBase::OnLogHistogram(TInstant time, TLogHistogramSnapshotPtr s) { + State_.Expect(TEncoderState::EState::METRIC); + TMetric& metric = Metrics_.back(); + metric.TimeSeries.Add(time, s.Get()); +} + +TString TBufferedEncoderBase::FormatLabels(const TPooledLabels& labels) const { + auto formattedLabels = TVector<TString>(Reserve(labels.size() + CommonLabels_.size())); + auto addLabel = [&](const TPooledLabel& l) { + auto formattedLabel = TStringBuilder() << LabelNamesPool_.Get(l.Key) << '=' << LabelValuesPool_.Get(l.Value); + formattedLabels.push_back(std::move(formattedLabel)); + }; + + for (const auto& l: labels) { + addLabel(l); + } + for (const auto& l: CommonLabels_) { + const auto it = FindIf(labels, [&](const TPooledLabel& label) { + return label.Key == l.Key; + }); + if (it == labels.end()) { + addLabel(l); + } + } + Sort(formattedLabels); + + return TStringBuilder() << "{" << JoinSeq(", ", formattedLabels) << "}"; +} + +} // namespace NMonitoring diff --git a/library/cpp/monlib/encode/buffered/buffered_encoder_base.h b/library/cpp/monlib/encode/buffered/buffered_encoder_base.h new file mode 100644 index 0000000000..fe3714e58f --- /dev/null +++ b/library/cpp/monlib/encode/buffered/buffered_encoder_base.h @@ -0,0 +1,100 @@ +#pragma once + +#include "string_pool.h" + +#include <library/cpp/monlib/encode/encoder.h> +#include <library/cpp/monlib/encode/encoder_state.h> +#include <library/cpp/monlib/encode/format.h> +#include <library/cpp/monlib/metrics/metric_value.h> + +#include <util/datetime/base.h> +#include <util/digest/numeric.h> + + +namespace NMonitoring { + +class TBufferedEncoderBase : public IMetricEncoder { +public: + void OnStreamBegin() override; + void OnStreamEnd() override; + + void OnCommonTime(TInstant time) override; + + void OnMetricBegin(EMetricType type) override; + void OnMetricEnd() override; + + void OnLabelsBegin() override; + void OnLabelsEnd() override; + void OnLabel(TStringBuf name, TStringBuf value) override; + void OnLabel(ui32 name, ui32 value) override; + std::pair<ui32, ui32> PrepareLabel(TStringBuf name, TStringBuf value) override; + + void OnDouble(TInstant time, double value) override; + void OnInt64(TInstant time, i64 value) override; + void OnUint64(TInstant time, ui64 value) override; + + void OnHistogram(TInstant time, IHistogramSnapshotPtr snapshot) override; + void OnSummaryDouble(TInstant time, ISummaryDoubleSnapshotPtr snapshot) override; + void OnLogHistogram(TInstant, TLogHistogramSnapshotPtr) override; + +protected: + using TPooledStr = TStringPoolBuilder::TValue; + + struct TPooledLabel { + TPooledLabel(const TPooledStr* key, const TPooledStr* value) + : Key{key} + , Value{value} + { + } + + bool operator==(const TPooledLabel& other) const { + return std::tie(Key, Value) == std::tie(other.Key, other.Value); + } + + bool operator!=(const TPooledLabel& other) const { + return !(*this == other); + } + + const TPooledStr* Key; + const TPooledStr* Value; + }; + + using TPooledLabels = TVector<TPooledLabel>; + + struct TPooledLabelsHash { + size_t operator()(const TPooledLabels& val) const { + size_t hash{0}; + + for (auto v : val) { + hash = CombineHashes<size_t>(hash, reinterpret_cast<size_t>(v.Key)); + hash = CombineHashes<size_t>(hash, reinterpret_cast<size_t>(v.Value)); + } + + return hash; + } + }; + + using TMetricMap = THashMap<TPooledLabels, size_t, TPooledLabelsHash>; + + struct TMetric { + EMetricType MetricType = EMetricType::UNKNOWN; + TPooledLabels Labels; + TMetricTimeSeries TimeSeries; + }; + +protected: + TString FormatLabels(const TPooledLabels& labels) const; + +protected: + TEncoderState State_; + + TStringPoolBuilder LabelNamesPool_; + TStringPoolBuilder LabelValuesPool_; + TInstant CommonTime_ = TInstant::Zero(); + TPooledLabels CommonLabels_; + TVector<TMetric> Metrics_; + TMetricMap MetricMap_; + EMetricsMergingMode MetricsMergingMode_ = EMetricsMergingMode::DEFAULT; +}; + +} diff --git a/library/cpp/monlib/encode/buffered/string_pool.cpp b/library/cpp/monlib/encode/buffered/string_pool.cpp new file mode 100644 index 0000000000..b4c7988ba3 --- /dev/null +++ b/library/cpp/monlib/encode/buffered/string_pool.cpp @@ -0,0 +1,58 @@ +#include "string_pool.h" + +namespace NMonitoring { + //////////////////////////////////////////////////////////////////////////////// + // TStringPoolBuilder + //////////////////////////////////////////////////////////////////////////////// + const TStringPoolBuilder::TValue* TStringPoolBuilder::PutIfAbsent(TStringBuf str) { + Y_ENSURE(!IsBuilt_, "Cannot add more values after string has been built"); + + auto [it, isInserted] = StrMap_.try_emplace(str, Max<ui32>(), 0); + if (isInserted) { + BytesSize_ += str.size(); + it->second.Index = StrVector_.size(); + StrVector_.emplace_back(it->first, &it->second); + } + + TValue* value = &it->second; + ++value->Frequency; + return value; + } + + const TStringPoolBuilder::TValue* TStringPoolBuilder::GetByIndex(ui32 index) const { + return StrVector_.at(index).second; + } + + TStringPoolBuilder& TStringPoolBuilder::Build() { + if (RequiresSorting_) { + // sort in reversed order + std::sort(StrVector_.begin(), StrVector_.end(), [](auto& a, auto& b) { + return a.second->Frequency > b.second->Frequency; + }); + + ui32 i = 0; + for (auto& value : StrVector_) { + value.second->Index = i++; + } + } + + IsBuilt_ = true; + + return *this; + } + + //////////////////////////////////////////////////////////////////////////////// + // TStringPool + //////////////////////////////////////////////////////////////////////////////// + void TStringPool::InitIndex(const char* data, ui32 size) { + const char* begin = data; + const char* end = begin + size; + for (const char* p = begin; p != end; ++p) { + if (*p == '\0') { + Index_.push_back(TStringBuf(begin, p)); + begin = p + 1; + } + } + } + +} diff --git a/library/cpp/monlib/encode/buffered/string_pool.h b/library/cpp/monlib/encode/buffered/string_pool.h new file mode 100644 index 0000000000..00e5644608 --- /dev/null +++ b/library/cpp/monlib/encode/buffered/string_pool.h @@ -0,0 +1,92 @@ +#pragma once + +#include <util/generic/hash.h> +#include <util/generic/vector.h> + +namespace NMonitoring { + //////////////////////////////////////////////////////////////////////////////// + // TStringPoolBuilder + //////////////////////////////////////////////////////////////////////////////// + class TStringPoolBuilder { + public: + struct TValue: TNonCopyable { + TValue(ui32 idx, ui32 freq) + : Index{idx} + , Frequency{freq} + { + } + + ui32 Index; + ui32 Frequency; + }; + + public: + const TValue* PutIfAbsent(TStringBuf str); + const TValue* GetByIndex(ui32 index) const; + + /// Determines whether pool must be sorted by value frequencies + TStringPoolBuilder& SetSorted(bool sorted) { + RequiresSorting_ = sorted; + return *this; + } + + TStringPoolBuilder& Build(); + + TStringBuf Get(ui32 index) const { + Y_ENSURE(IsBuilt_, "Pool must be sorted first"); + return StrVector_.at(index).first; + } + + TStringBuf Get(const TValue* value) const { + return StrVector_.at(value->Index).first; + } + + template <typename TConsumer> + void ForEach(TConsumer&& c) { + Y_ENSURE(IsBuilt_, "Pool must be sorted first"); + for (const auto& value : StrVector_) { + c(value.first, value.second->Index, value.second->Frequency); + } + } + + size_t BytesSize() const noexcept { + return BytesSize_; + } + + size_t Count() const noexcept { + return StrMap_.size(); + } + + private: + THashMap<TString, TValue> StrMap_; + TVector<std::pair<TStringBuf, TValue*>> StrVector_; + bool RequiresSorting_ = false; + bool IsBuilt_ = false; + size_t BytesSize_ = 0; + }; + + //////////////////////////////////////////////////////////////////////////////// + // TStringPool + //////////////////////////////////////////////////////////////////////////////// + class TStringPool { + public: + TStringPool(const char* data, ui32 size) { + InitIndex(data, size); + } + + TStringBuf Get(ui32 i) const { + return Index_.at(i); + } + + size_t Size() const { + return Index_.size(); + } + + private: + void InitIndex(const char* data, ui32 size); + + private: + TVector<TStringBuf> Index_; + }; + +} diff --git a/library/cpp/monlib/encode/buffered/string_pool_ut.cpp b/library/cpp/monlib/encode/buffered/string_pool_ut.cpp new file mode 100644 index 0000000000..9fc3421d0b --- /dev/null +++ b/library/cpp/monlib/encode/buffered/string_pool_ut.cpp @@ -0,0 +1,84 @@ +#include "string_pool.h" + +#include <library/cpp/testing/unittest/registar.h> + +using namespace NMonitoring; + +Y_UNIT_TEST_SUITE(TStringPoolTest) { + Y_UNIT_TEST(PutIfAbsent) { + TStringPoolBuilder strPool; + strPool.SetSorted(true); + + auto* h1 = strPool.PutIfAbsent("one"); + auto* h2 = strPool.PutIfAbsent("two"); + auto* h3 = strPool.PutIfAbsent("two"); + UNIT_ASSERT(h1 != h2); + UNIT_ASSERT(h2 == h3); + + UNIT_ASSERT_VALUES_EQUAL(h1->Frequency, 1); + UNIT_ASSERT_VALUES_EQUAL(h1->Index, 0); + + UNIT_ASSERT_VALUES_EQUAL(h2->Frequency, 2); + UNIT_ASSERT_VALUES_EQUAL(h2->Index, 1); + + UNIT_ASSERT_VALUES_EQUAL(strPool.BytesSize(), 6); + UNIT_ASSERT_VALUES_EQUAL(strPool.Count(), 2); + } + + Y_UNIT_TEST(SortByFrequency) { + TStringPoolBuilder strPool; + strPool.SetSorted(true); + + auto* h1 = strPool.PutIfAbsent("one"); + auto* h2 = strPool.PutIfAbsent("two"); + auto* h3 = strPool.PutIfAbsent("two"); + UNIT_ASSERT(h1 != h2); + UNIT_ASSERT(h2 == h3); + + strPool.Build(); + + UNIT_ASSERT_VALUES_EQUAL(h1->Frequency, 1); + UNIT_ASSERT_VALUES_EQUAL(h1->Index, 1); + + UNIT_ASSERT_VALUES_EQUAL(h2->Frequency, 2); + UNIT_ASSERT_VALUES_EQUAL(h2->Index, 0); + + UNIT_ASSERT_VALUES_EQUAL(strPool.BytesSize(), 6); + UNIT_ASSERT_VALUES_EQUAL(strPool.Count(), 2); + } + + Y_UNIT_TEST(ForEach) { + TStringPoolBuilder strPool; + strPool.SetSorted(true); + + strPool.PutIfAbsent("one"); + strPool.PutIfAbsent("two"); + strPool.PutIfAbsent("two"); + strPool.PutIfAbsent("three"); + strPool.PutIfAbsent("three"); + strPool.PutIfAbsent("three"); + + UNIT_ASSERT_VALUES_EQUAL(strPool.BytesSize(), 11); + UNIT_ASSERT_VALUES_EQUAL(strPool.Count(), 3); + + strPool.Build(); + + TVector<TString> strings; + TVector<ui32> indexes; + TVector<ui32> frequences; + strPool.ForEach([&](TStringBuf str, ui32 index, ui32 freq) { + strings.emplace_back(str); + indexes.push_back(index); + frequences.push_back(freq); + }); + + TVector<TString> expectedStrings = {"three", "two", "one"}; + UNIT_ASSERT_EQUAL(strings, expectedStrings); + + TVector<ui32> expectedIndexes = {0, 1, 2}; + UNIT_ASSERT_EQUAL(indexes, expectedIndexes); + + TVector<ui32> expectedFrequences = {3, 2, 1}; + UNIT_ASSERT_EQUAL(frequences, expectedFrequences); + } +} diff --git a/library/cpp/monlib/encode/buffered/ut/ya.make b/library/cpp/monlib/encode/buffered/ut/ya.make new file mode 100644 index 0000000000..2157ac1490 --- /dev/null +++ b/library/cpp/monlib/encode/buffered/ut/ya.make @@ -0,0 +1,12 @@ +UNITTEST_FOR(library/cpp/monlib/encode/buffered) + +OWNER( + g:solomon + jamel +) + +SRCS( + string_pool_ut.cpp +) + +END() diff --git a/library/cpp/monlib/encode/buffered/ya.make b/library/cpp/monlib/encode/buffered/ya.make new file mode 100644 index 0000000000..81b6a78b93 --- /dev/null +++ b/library/cpp/monlib/encode/buffered/ya.make @@ -0,0 +1,19 @@ +LIBRARY() + +OWNER( + g:solomon + jamel + msherbakov +) + +SRCS( + buffered_encoder_base.cpp + string_pool.cpp +) + +PEERDIR( + library/cpp/monlib/encode + library/cpp/monlib/metrics +) + +END() diff --git a/library/cpp/monlib/encode/encoder.cpp b/library/cpp/monlib/encode/encoder.cpp new file mode 100644 index 0000000000..becf932689 --- /dev/null +++ b/library/cpp/monlib/encode/encoder.cpp @@ -0,0 +1,6 @@ +#include "encoder.h" + +namespace NMonitoring { + IMetricEncoder::~IMetricEncoder() { + } +} diff --git a/library/cpp/monlib/encode/encoder.h b/library/cpp/monlib/encode/encoder.h new file mode 100644 index 0000000000..a26a133d16 --- /dev/null +++ b/library/cpp/monlib/encode/encoder.h @@ -0,0 +1,17 @@ +#pragma once + +#include <util/generic/ptr.h> + +#include <library/cpp/monlib/metrics/metric_consumer.h> + +namespace NMonitoring { + class IMetricEncoder: public IMetricConsumer { + public: + virtual ~IMetricEncoder(); + + virtual void Close() = 0; + }; + + using IMetricEncoderPtr = THolder<IMetricEncoder>; + +} diff --git a/library/cpp/monlib/encode/encoder_state.cpp b/library/cpp/monlib/encode/encoder_state.cpp new file mode 100644 index 0000000000..0ece696b1a --- /dev/null +++ b/library/cpp/monlib/encode/encoder_state.cpp @@ -0,0 +1 @@ +#include "encoder_state.h" diff --git a/library/cpp/monlib/encode/encoder_state.h b/library/cpp/monlib/encode/encoder_state.h new file mode 100644 index 0000000000..e6a098f404 --- /dev/null +++ b/library/cpp/monlib/encode/encoder_state.h @@ -0,0 +1,62 @@ +#pragma once + +#include "encoder_state_enum.h" + +#include <util/generic/serialized_enum.h> +#include <util/generic/yexception.h> + + +namespace NMonitoring { + + template <typename EEncoderState> + class TEncoderStateImpl { + public: + using EState = EEncoderState; + + explicit TEncoderStateImpl(EEncoderState state = EEncoderState::ROOT) + : State_(state) + { + } + + TEncoderStateImpl& operator=(EEncoderState rhs) noexcept { + State_ = rhs; + return *this; + } + + inline bool operator==(EEncoderState rhs) const noexcept { + return State_ == rhs; + } + + inline bool operator!=(EEncoderState rhs) const noexcept { + return !operator==(rhs); + } + + [[noreturn]] inline void ThrowInvalid(TStringBuf message) const { + ythrow yexception() << "invalid encoder state: " + << ToStr() << ", " << message; + } + + inline void Expect(EEncoderState expected) const { + if (Y_UNLIKELY(State_ != expected)) { + ythrow yexception() + << "invalid encoder state: " << ToStr() + << ", expected: " << TEncoderStateImpl(expected).ToStr(); + } + } + + inline void Switch(EEncoderState from, EEncoderState to) { + Expect(from); + State_ = to; + } + + TStringBuf ToStr() const noexcept { + return NEnumSerializationRuntime::GetEnumNamesImpl<EEncoderState>().at(State_); + } + + private: + EEncoderState State_; + }; + + using TEncoderState = TEncoderStateImpl<EEncoderState>; + +} // namespace NMonitoring diff --git a/library/cpp/monlib/encode/encoder_state_enum.h b/library/cpp/monlib/encode/encoder_state_enum.h new file mode 100644 index 0000000000..471604f91d --- /dev/null +++ b/library/cpp/monlib/encode/encoder_state_enum.h @@ -0,0 +1,12 @@ +#pragma once + +namespace NMonitoring { + + enum class EEncoderState { + ROOT, + COMMON_LABELS, + METRIC, + METRIC_LABELS, + }; + +} // namespace NMonitoring diff --git a/library/cpp/monlib/encode/fake/fake.cpp b/library/cpp/monlib/encode/fake/fake.cpp new file mode 100644 index 0000000000..69d691361a --- /dev/null +++ b/library/cpp/monlib/encode/fake/fake.cpp @@ -0,0 +1,51 @@ +#include "fake.h" + +#include <util/datetime/base.h> + +namespace NMonitoring { + class TFakeEncoder: public IMetricEncoder { + public: + void OnStreamBegin() override { + } + void OnStreamEnd() override { + } + + void OnCommonTime(TInstant) override { + } + + void OnMetricBegin(EMetricType) override { + } + void OnMetricEnd() override { + } + + void OnLabelsBegin() override { + } + void OnLabelsEnd() override { + } + void OnLabel(const TStringBuf, const TStringBuf) override { + } + + void OnDouble(TInstant, double) override { + } + void OnInt64(TInstant, i64) override { + } + void OnUint64(TInstant, ui64) override { + } + + void OnHistogram(TInstant, IHistogramSnapshotPtr) override { + } + + void OnSummaryDouble(TInstant, ISummaryDoubleSnapshotPtr) override { + } + + void OnLogHistogram(TInstant, TLogHistogramSnapshotPtr) override { + } + + void Close() override { + } + }; + + IMetricEncoderPtr EncoderFake() { + return MakeHolder<TFakeEncoder>(); + } +} diff --git a/library/cpp/monlib/encode/fake/fake.h b/library/cpp/monlib/encode/fake/fake.h new file mode 100644 index 0000000000..8109326987 --- /dev/null +++ b/library/cpp/monlib/encode/fake/fake.h @@ -0,0 +1,10 @@ +#pragma once + +#include <library/cpp/monlib/encode/encoder.h> + +class IOutputStream; + +namespace NMonitoring { + // Does nothing: just implements IMetricEncoder interface with stubs + IMetricEncoderPtr EncoderFake(); +} diff --git a/library/cpp/monlib/encode/fake/ya.make b/library/cpp/monlib/encode/fake/ya.make new file mode 100644 index 0000000000..ae96f45782 --- /dev/null +++ b/library/cpp/monlib/encode/fake/ya.make @@ -0,0 +1,12 @@ +LIBRARY() + +OWNER( + g:solomon + msherbakov +) + +SRCS( + fake.cpp +) + +END() diff --git a/library/cpp/monlib/encode/format.cpp b/library/cpp/monlib/encode/format.cpp new file mode 100644 index 0000000000..400ce5a643 --- /dev/null +++ b/library/cpp/monlib/encode/format.cpp @@ -0,0 +1,202 @@ +#include "format.h" + +#include <util/string/ascii.h> +#include <util/string/split.h> +#include <util/string/strip.h> +#include <util/stream/output.h> +#include <util/string/cast.h> + +namespace NMonitoring { + static ECompression CompressionFromHeader(TStringBuf value) { + if (value.empty()) { + return ECompression::UNKNOWN; + } + + for (const auto& it : StringSplitter(value).Split(',').SkipEmpty()) { + TStringBuf token = StripString(it.Token()); + + if (AsciiEqualsIgnoreCase(token, NFormatContentEncoding::IDENTITY)) { + return ECompression::IDENTITY; + } else if (AsciiEqualsIgnoreCase(token, NFormatContentEncoding::ZLIB)) { + return ECompression::ZLIB; + } else if (AsciiEqualsIgnoreCase(token, NFormatContentEncoding::LZ4)) { + return ECompression::LZ4; + } else if (AsciiEqualsIgnoreCase(token, NFormatContentEncoding::ZSTD)) { + return ECompression::ZSTD; + } + } + + return ECompression::UNKNOWN; + } + + static EFormat FormatFromHttpMedia(TStringBuf value) { + if (AsciiEqualsIgnoreCase(value, NFormatContenType::SPACK)) { + return EFormat::SPACK; + } else if (AsciiEqualsIgnoreCase(value, NFormatContenType::JSON)) { + return EFormat::JSON; + } else if (AsciiEqualsIgnoreCase(value, NFormatContenType::PROTOBUF)) { + return EFormat::PROTOBUF; + } else if (AsciiEqualsIgnoreCase(value, NFormatContenType::TEXT)) { + return EFormat::TEXT; + } else if (AsciiEqualsIgnoreCase(value, NFormatContenType::PROMETHEUS)) { + return EFormat::PROMETHEUS; + } + + return EFormat::UNKNOWN; + } + + EFormat FormatFromAcceptHeader(TStringBuf value) { + EFormat result{EFormat::UNKNOWN}; + + for (const auto& it : StringSplitter(value).Split(',').SkipEmpty()) { + TStringBuf token = StripString(it.Token()); + + result = FormatFromHttpMedia(token); + if (result != EFormat::UNKNOWN) { + break; + } + } + + return result; + } + + EFormat FormatFromContentType(TStringBuf value) { + value = value.NextTok(';'); + + return FormatFromHttpMedia(value); + } + + TStringBuf ContentTypeByFormat(EFormat format) { + switch (format) { + case EFormat::SPACK: + return NFormatContenType::SPACK; + case EFormat::JSON: + return NFormatContenType::JSON; + case EFormat::PROTOBUF: + return NFormatContenType::PROTOBUF; + case EFormat::TEXT: + return NFormatContenType::TEXT; + case EFormat::PROMETHEUS: + return NFormatContenType::PROMETHEUS; + case EFormat::UNKNOWN: + return TStringBuf(); + } + + Y_FAIL(); // for GCC + } + + ECompression CompressionFromAcceptEncodingHeader(TStringBuf value) { + return CompressionFromHeader(value); + } + + ECompression CompressionFromContentEncodingHeader(TStringBuf value) { + return CompressionFromHeader(value); + } + + TStringBuf ContentEncodingByCompression(ECompression compression) { + switch (compression) { + case ECompression::IDENTITY: + return NFormatContentEncoding::IDENTITY; + case ECompression::ZLIB: + return NFormatContentEncoding::ZLIB; + case ECompression::LZ4: + return NFormatContentEncoding::LZ4; + case ECompression::ZSTD: + return NFormatContentEncoding::ZSTD; + case ECompression::UNKNOWN: + return TStringBuf(); + } + + Y_FAIL(); // for GCC + } + +} + +template <> +NMonitoring::EFormat FromStringImpl<NMonitoring::EFormat>(const char* str, size_t len) { + using NMonitoring::EFormat; + TStringBuf value(str, len); + if (value == TStringBuf("SPACK")) { + return EFormat::SPACK; + } else if (value == TStringBuf("JSON")) { + return EFormat::JSON; + } else if (value == TStringBuf("PROTOBUF")) { + return EFormat::PROTOBUF; + } else if (value == TStringBuf("TEXT")) { + return EFormat::TEXT; + } else if (value == TStringBuf("PROMETHEUS")) { + return EFormat::PROMETHEUS; + } else if (value == TStringBuf("UNKNOWN")) { + return EFormat::UNKNOWN; + } + ythrow yexception() << "unknown format: " << value; +} + +template <> +void Out<NMonitoring::EFormat>(IOutputStream& o, NMonitoring::EFormat f) { + using NMonitoring::EFormat; + switch (f) { + case EFormat::SPACK: + o << TStringBuf("SPACK"); + return; + case EFormat::JSON: + o << TStringBuf("JSON"); + return; + case EFormat::PROTOBUF: + o << TStringBuf("PROTOBUF"); + return; + case EFormat::TEXT: + o << TStringBuf("TEXT"); + return; + case EFormat::PROMETHEUS: + o << TStringBuf("PROMETHEUS"); + return; + case EFormat::UNKNOWN: + o << TStringBuf("UNKNOWN"); + return; + } + + Y_FAIL(); // for GCC +} + +template <> +NMonitoring::ECompression FromStringImpl<NMonitoring::ECompression>(const char* str, size_t len) { + using NMonitoring::ECompression; + TStringBuf value(str, len); + if (value == TStringBuf("IDENTITY")) { + return ECompression::IDENTITY; + } else if (value == TStringBuf("ZLIB")) { + return ECompression::ZLIB; + } else if (value == TStringBuf("LZ4")) { + return ECompression::LZ4; + } else if (value == TStringBuf("ZSTD")) { + return ECompression::ZSTD; + } else if (value == TStringBuf("UNKNOWN")) { + return ECompression::UNKNOWN; + } + ythrow yexception() << "unknown compression: " << value; +} + +template <> +void Out<NMonitoring::ECompression>(IOutputStream& o, NMonitoring::ECompression c) { + using NMonitoring::ECompression; + switch (c) { + case ECompression::IDENTITY: + o << TStringBuf("IDENTITY"); + return; + case ECompression::ZLIB: + o << TStringBuf("ZLIB"); + return; + case ECompression::LZ4: + o << TStringBuf("LZ4"); + return; + case ECompression::ZSTD: + o << TStringBuf("ZSTD"); + return; + case ECompression::UNKNOWN: + o << TStringBuf("UNKNOWN"); + return; + } + + Y_FAIL(); // for GCC +} diff --git a/library/cpp/monlib/encode/format.h b/library/cpp/monlib/encode/format.h new file mode 100644 index 0000000000..495d42d786 --- /dev/null +++ b/library/cpp/monlib/encode/format.h @@ -0,0 +1,166 @@ +#pragma once + +#include <util/generic/strbuf.h> + +namespace NMonitoring { + namespace NFormatContenType { + constexpr TStringBuf TEXT = "application/x-solomon-txt"; + constexpr TStringBuf JSON = "application/json"; + constexpr TStringBuf PROTOBUF = "application/x-solomon-pb"; + constexpr TStringBuf SPACK = "application/x-solomon-spack"; + constexpr TStringBuf PROMETHEUS = "text/plain"; + } + + namespace NFormatContentEncoding { + constexpr TStringBuf IDENTITY = "identity"; + constexpr TStringBuf ZLIB = "zlib"; + constexpr TStringBuf LZ4 = "lz4"; + constexpr TStringBuf ZSTD = "zstd"; + } + + /** + * Defines format types for metric encoders. + */ + enum class EFormat { + /** + * Special case when it was not possible to determine format. + */ + UNKNOWN, + + /** + * Read more https://wiki.yandex-team.ru/solomon/api/dataformat/spackv1 + */ + SPACK, + + /** + * Read more https://wiki.yandex-team.ru/solomon/api/dataformat/json + */ + JSON, + + /** + * Simple protobuf format, only for testing purposes. + */ + PROTOBUF, + + /** + * Simple text representation, only for debug purposes. + */ + TEXT, + + /** + * Prometheus text-based format + */ + PROMETHEUS, + }; + + /** + * Defines compression algorithms for metric encoders. + */ + enum class ECompression { + /** + * Special case when it was not possible to determine compression. + */ + UNKNOWN, + + /** + * Means no compression. + */ + IDENTITY, + + /** + * Using the zlib structure (defined in RFC 1950), with the + * deflate compression algorithm and Adler32 checkums. + */ + ZLIB, + + /** + * Using LZ4 compression algorithm (read http://lz4.org for more info) + * with XxHash32 checksums. + */ + LZ4, + + /** + * Using Zstandard compression algorithm (read http://zstd.net for more + * info) with XxHash32 checksums. + */ + ZSTD, + }; + + enum class EMetricsMergingMode { + /** + * Do not merge metric batches. If several points of the same metric were + * added multiple times accross different writes, paste them as + * separate metrics. + * + * Example: + * COUNTER [(ts1, val1)] | COUNTER [(ts1, val1)] + * COUNTER [(ts2, val2)] | --> COUNTER [(ts2, val2)] + * COUNTER [(ts3, val3)] | COUNTER [(ts3, val3)] + */ + DEFAULT, + + /** + * If several points of the same metric were added multiple times across + * different writes, merge all values to one timeseries. + * + * Example: + * COUNTER [(ts1, val1)] | + * COUNTER [(ts2, val2)] | --> COUNTER [(ts1, val1), (ts2, val2), (ts3, val3)] + * COUNTER [(ts3, val3)] | + */ + MERGE_METRICS, + }; + + /** + * Matches serialization format by the given "Accept" header value. + * + * @param value value of the "Accept" header. + * @return most preffered serialization format type + */ + EFormat FormatFromAcceptHeader(TStringBuf value); + + /** + * Matches serialization format by the given "Content-Type" header value + * + * @param value value of the "Content-Type" header + * @return message format + */ + EFormat FormatFromContentType(TStringBuf value); + + /** + * Returns value for "Content-Type" header determined by the given + * format type. + * + * @param format serialization format type + * @return mime-type indentificator + * or empty string if format is UNKNOWN + */ + TStringBuf ContentTypeByFormat(EFormat format); + + /** + * Matches compression algorithm by the given "Accept-Encoding" header value. + * + * @param value value of the "Accept-Encoding" header. + * @return most preffered compression algorithm + */ + ECompression CompressionFromAcceptEncodingHeader(TStringBuf value); + + /** + * Matches compression algorithm by the given "Content-Encoding" header value. + * + * @param value value of the "Accept-Encoding" header. + * @return most preffered compression algorithm + */ + ECompression CompressionFromContentEncodingHeader(TStringBuf value); + + /** + * Returns value for "Content-Encoding" header determined by the given + * compression algorithm. + * + * @param compression encoding compression alg + * @return media-type compresion algorithm + * or empty string if compression is UNKNOWN + */ + TStringBuf ContentEncodingByCompression(ECompression compression); + +} diff --git a/library/cpp/monlib/encode/format_ut.cpp b/library/cpp/monlib/encode/format_ut.cpp new file mode 100644 index 0000000000..22a0e30c03 --- /dev/null +++ b/library/cpp/monlib/encode/format_ut.cpp @@ -0,0 +1,136 @@ +#include "format.h" + +#include <library/cpp/testing/unittest/registar.h> + +#include <util/generic/string.h> +#include <util/string/builder.h> + +#include <array> + +using namespace NMonitoring; + +Y_UNIT_TEST_SUITE(TFormatTest) { + Y_UNIT_TEST(ContentTypeHeader) { + UNIT_ASSERT_EQUAL(FormatFromContentType(""), EFormat::UNKNOWN); + UNIT_ASSERT_EQUAL(FormatFromContentType("application/json;some=stuff"), EFormat::JSON); + UNIT_ASSERT_EQUAL(FormatFromContentType("application/x-solomon-spack"), EFormat::SPACK); + UNIT_ASSERT_EQUAL(FormatFromContentType("application/xml"), EFormat::UNKNOWN); + UNIT_ASSERT_EQUAL(FormatFromContentType(";application/xml"), EFormat::UNKNOWN); + } + + Y_UNIT_TEST(AcceptHeader) { + UNIT_ASSERT_EQUAL(FormatFromAcceptHeader(""), EFormat::UNKNOWN); + UNIT_ASSERT_EQUAL(FormatFromAcceptHeader("*/*"), EFormat::UNKNOWN); + + UNIT_ASSERT_EQUAL( + FormatFromAcceptHeader("application/xml"), + EFormat::UNKNOWN); + + UNIT_ASSERT_EQUAL( + FormatFromAcceptHeader("application/json"), + EFormat::JSON); + + UNIT_ASSERT_EQUAL( + FormatFromAcceptHeader("application/x-solomon-spack"), + EFormat::SPACK); + + UNIT_ASSERT_EQUAL( + FormatFromAcceptHeader("application/x-solomon-pb"), + EFormat::PROTOBUF); + + UNIT_ASSERT_EQUAL( + FormatFromAcceptHeader("application/x-solomon-txt"), + EFormat::TEXT); + + UNIT_ASSERT_EQUAL( + FormatFromAcceptHeader("application/json, text/plain"), + EFormat::JSON); + + UNIT_ASSERT_EQUAL( + FormatFromAcceptHeader("application/x-solomon-spack, application/json, text/plain"), + EFormat::SPACK); + + UNIT_ASSERT_EQUAL( + FormatFromAcceptHeader(" , application/x-solomon-spack ,, application/json , text/plain"), + EFormat::SPACK); + + UNIT_ASSERT_EQUAL( + FormatFromAcceptHeader("application/xml, application/x-solomon-spack, text/plain"), + EFormat::SPACK); + + UNIT_ASSERT_EQUAL( + FormatFromAcceptHeader("text/plain"), + EFormat::PROMETHEUS); + } + + Y_UNIT_TEST(FormatToStrFromStr) { + const std::array<EFormat, 6> formats = {{ + EFormat::UNKNOWN, + EFormat::SPACK, + EFormat::JSON, + EFormat::PROTOBUF, + EFormat::TEXT, + EFormat::PROMETHEUS, + }}; + + for (EFormat f : formats) { + TString str = (TStringBuilder() << f); + EFormat g = FromString<EFormat>(str); + UNIT_ASSERT_EQUAL(f, g); + } + } + + Y_UNIT_TEST(AcceptEncodingHeader) { + UNIT_ASSERT_EQUAL( + CompressionFromAcceptEncodingHeader(""), + ECompression::UNKNOWN); + + UNIT_ASSERT_EQUAL( + CompressionFromAcceptEncodingHeader("br"), + ECompression::UNKNOWN); + + UNIT_ASSERT_EQUAL( + CompressionFromAcceptEncodingHeader("identity"), + ECompression::IDENTITY); + + UNIT_ASSERT_EQUAL( + CompressionFromAcceptEncodingHeader("zlib"), + ECompression::ZLIB); + + UNIT_ASSERT_EQUAL( + CompressionFromAcceptEncodingHeader("lz4"), + ECompression::LZ4); + + UNIT_ASSERT_EQUAL( + CompressionFromAcceptEncodingHeader("zstd"), + ECompression::ZSTD); + + UNIT_ASSERT_EQUAL( + CompressionFromAcceptEncodingHeader("zstd, zlib"), + ECompression::ZSTD); + + UNIT_ASSERT_EQUAL( + CompressionFromAcceptEncodingHeader(" ,, , zstd , zlib"), + ECompression::ZSTD); + + UNIT_ASSERT_EQUAL( + CompressionFromAcceptEncodingHeader("br, deflate,lz4, zlib"), + ECompression::LZ4); + } + + Y_UNIT_TEST(CompressionToStrFromStr) { + const std::array<ECompression, 5> algs = {{ + ECompression::UNKNOWN, + ECompression::IDENTITY, + ECompression::ZLIB, + ECompression::LZ4, + ECompression::ZSTD, + }}; + + for (ECompression a : algs) { + TString str = (TStringBuilder() << a); + ECompression b = FromString<ECompression>(str); + UNIT_ASSERT_EQUAL(a, b); + } + } +} diff --git a/library/cpp/monlib/encode/fuzz/ya.make b/library/cpp/monlib/encode/fuzz/ya.make new file mode 100644 index 0000000000..d9ca172bae --- /dev/null +++ b/library/cpp/monlib/encode/fuzz/ya.make @@ -0,0 +1,5 @@ +RECURSE_ROOT_RELATIVE( + library/cpp/monlib/encode/json/fuzz + library/cpp/monlib/encode/prometheus/fuzz + library/cpp/monlib/encode/spack/fuzz +) diff --git a/library/cpp/monlib/encode/json/fuzz/main.cpp b/library/cpp/monlib/encode/json/fuzz/main.cpp new file mode 100644 index 0000000000..4f40310e06 --- /dev/null +++ b/library/cpp/monlib/encode/json/fuzz/main.cpp @@ -0,0 +1,16 @@ +#include <library/cpp/monlib/encode/json/json.h> +#include <library/cpp/monlib/encode/fake/fake.h> + +#include <util/generic/strbuf.h> + + +extern "C" int LLVMFuzzerTestOneInput(const ui8* data, size_t size) { + auto encoder = NMonitoring::EncoderFake(); + + try { + NMonitoring::DecodeJson({reinterpret_cast<const char*>(data), size}, encoder.Get()); + } catch (...) { + } + + return 0; +} diff --git a/library/cpp/monlib/encode/json/fuzz/ya.make b/library/cpp/monlib/encode/json/fuzz/ya.make new file mode 100644 index 0000000000..75baa77716 --- /dev/null +++ b/library/cpp/monlib/encode/json/fuzz/ya.make @@ -0,0 +1,19 @@ +FUZZ() + +OWNER( + g:solomon + msherbakov +) + +PEERDIR( + library/cpp/monlib/encode/json + library/cpp/monlib/encode/fake +) + +SIZE(MEDIUM) + +SRCS( + main.cpp +) + +END() diff --git a/library/cpp/monlib/encode/json/json.h b/library/cpp/monlib/encode/json/json.h new file mode 100644 index 0000000000..21530f20c3 --- /dev/null +++ b/library/cpp/monlib/encode/json/json.h @@ -0,0 +1,29 @@ +#pragma once + +#include <library/cpp/monlib/encode/encoder.h> +#include <library/cpp/monlib/encode/format.h> + + +class IOutputStream; + +namespace NMonitoring { + + class TJsonDecodeError: public yexception { + }; + + IMetricEncoderPtr EncoderJson(IOutputStream* out, int indentation = 0); + + /// Buffered encoder will merge series with same labels into one. + IMetricEncoderPtr BufferedEncoderJson(IOutputStream* out, int indentation = 0); + + IMetricEncoderPtr EncoderCloudJson(IOutputStream* out, + int indentation = 0, + TStringBuf metricNameLabel = "name"); + + IMetricEncoderPtr BufferedEncoderCloudJson(IOutputStream* out, + int indentation = 0, + TStringBuf metricNameLabel = "name"); + + void DecodeJson(TStringBuf data, IMetricConsumer* c, TStringBuf metricNameLabel = "name"); + +} diff --git a/library/cpp/monlib/encode/json/json_decoder.cpp b/library/cpp/monlib/encode/json/json_decoder.cpp new file mode 100644 index 0000000000..d44ff5fd28 --- /dev/null +++ b/library/cpp/monlib/encode/json/json_decoder.cpp @@ -0,0 +1,1162 @@ +#include "json.h" +#include "typed_point.h" + + +#include <library/cpp/monlib/exception/exception.h> +#include <library/cpp/monlib/metrics/labels.h> +#include <library/cpp/monlib/metrics/metric_value.h> + +#include <library/cpp/json/json_reader.h> + +#include <util/datetime/base.h> +#include <util/string/cast.h> + +#include <limits> + +namespace NMonitoring { + +#define DECODE_ENSURE(COND, ...) MONLIB_ENSURE_EX(COND, TJsonDecodeError() << __VA_ARGS__) + +namespace { + +/////////////////////////////////////////////////////////////////////// +// THistogramBuilder +/////////////////////////////////////////////////////////////////////// +class THistogramBuilder { +public: + void AddBound(TBucketBound bound) { + if (!Bounds_.empty()) { + DECODE_ENSURE(Bounds_.back() < bound, + "non sorted bounds, " << Bounds_.back() << + " >= " << bound); + } + Bounds_.push_back(bound); + } + + void AddValue(TBucketValue value) { + Values_.push_back(value); + } + + void AddInf(TBucketValue value) { + InfPresented_ = true; + InfValue_ = value; + } + + IHistogramSnapshotPtr Build() { + if (InfPresented_) { + Bounds_.push_back(Max<TBucketBound>()); + Values_.push_back(InfValue_); + } + + auto snapshot = ExplicitHistogramSnapshot(Bounds_, Values_); + + Bounds_.clear(); + Values_.clear(); + InfPresented_ = false; + + return snapshot; + } + + bool Empty() const noexcept { + return Bounds_.empty() && Values_.empty(); + } + + void Clear() { + Bounds_.clear(); + Values_.clear(); + } + +private: + TBucketBounds Bounds_; + TBucketValues Values_; + + bool InfPresented_ = false; + TBucketValue InfValue_; +}; + +class TSummaryDoubleBuilder { +public: + ISummaryDoubleSnapshotPtr Build() const { + return MakeIntrusive<TSummaryDoubleSnapshot>(Sum_, Min_, Max_, Last_, Count_); + } + + void SetSum(double sum) { + Empty_ = false; + Sum_ = sum; + } + + void SetMin(double min) { + Empty_ = false; + Min_ = min; + } + + void SetMax(double max) { + Empty_ = false; + Max_ = max; + } + + void SetLast(double last) { + Empty_ = false; + Last_ = last; + } + + void SetCount(ui64 count) { + Empty_ = false; + Count_ = count; + } + + void Clear() { + Empty_ = true; + Sum_ = 0; + Min_ = 0; + Max_ = 0; + Last_ = 0; + Count_ = 0; + } + + bool Empty() const { + return Empty_; + } + +private: + double Sum_ = 0; + double Min_ = 0; + double Max_ = 0; + double Last_ = 0; + ui64 Count_ = 0; + bool Empty_ = true; +}; + +class TLogHistogramBuilder { +public: + void SetBase(double base) { + DECODE_ENSURE(base > 0, "base must be positive"); + Base_ = base; + } + + void SetZerosCount(ui64 zerosCount) { + DECODE_ENSURE(zerosCount >= 0, "zeros count must be positive"); + ZerosCount_ = zerosCount; + } + + void SetStartPower(int startPower) { + StartPower_ = startPower; + } + + void AddBucketValue(double value) { + DECODE_ENSURE(value > 0.0, "bucket values must be positive"); + DECODE_ENSURE(value < std::numeric_limits<double>::max(), "bucket values must be finite"); + Buckets_.push_back(value); + } + + void Clear() { + Buckets_.clear(); + Base_ = 1.5; + ZerosCount_ = 0; + StartPower_ = 0; + } + + bool Empty() const { + return Buckets_.empty() && ZerosCount_ == 0; + } + + TLogHistogramSnapshotPtr Build() { + return MakeIntrusive<TLogHistogramSnapshot>(Base_, ZerosCount_, StartPower_, std::move(Buckets_)); + } + +private: + double Base_ = 1.5; + ui64 ZerosCount_ = 0; + int StartPower_ = 0; + TVector<double> Buckets_; +}; + +std::pair<double, bool> ParseSpecDouble(TStringBuf string) { + if (string == TStringBuf("nan") || string == TStringBuf("NaN")) { + return {std::numeric_limits<double>::quiet_NaN(), true}; + } else if (string == TStringBuf("inf") || string == TStringBuf("Infinity")) { + return {std::numeric_limits<double>::infinity(), true}; + } else if (string == TStringBuf("-inf") || string == TStringBuf("-Infinity")) { + return {-std::numeric_limits<double>::infinity(), true}; + } else { + return {0, false}; + } +} + +/////////////////////////////////////////////////////////////////////// +// TMetricCollector +/////////////////////////////////////////////////////////////////////// +struct TMetricCollector { + EMetricType Type = EMetricType::UNKNOWN; + TLabels Labels; + THistogramBuilder HistogramBuilder; + TSummaryDoubleBuilder SummaryBuilder; + TLogHistogramBuilder LogHistBuilder; + TTypedPoint LastPoint; + TVector<TTypedPoint> TimeSeries; + + bool SeenTsOrValue = false; + bool SeenTimeseries = false; + + void Clear() { + Type = EMetricType::UNKNOWN; + Labels.Clear(); + SeenTsOrValue = false; + SeenTimeseries = false; + TimeSeries.clear(); + LastPoint = {}; + HistogramBuilder.Clear(); + SummaryBuilder.Clear(); + LogHistBuilder.Clear(); + } + + void AddLabel(const TLabel& label) { + Labels.Add(label.Name(), label.Value()); + } + + void SetLastTime(TInstant time) { + LastPoint.SetTime(time); + } + + template <typename T> + void SetLastValue(T value) { + LastPoint.SetValue(value); + } + + void SaveLastPoint() { + DECODE_ENSURE(LastPoint.GetTime() != TInstant::Zero(), + "cannot add point without or zero timestamp"); + if (!HistogramBuilder.Empty()) { + auto histogram = HistogramBuilder.Build(); + TimeSeries.emplace_back(LastPoint.GetTime(), histogram.Get()); + } else if (!SummaryBuilder.Empty()) { + auto summary = SummaryBuilder.Build(); + TimeSeries.emplace_back(LastPoint.GetTime(), summary.Get()); + } else if (!LogHistBuilder.Empty()) { + auto logHist = LogHistBuilder.Build(); + TimeSeries.emplace_back(LastPoint.GetTime(), logHist.Get()); + } else { + TimeSeries.push_back(std::move(LastPoint)); + } + } + + template <typename TConsumer> + void Consume(TConsumer&& consumer) { + if (TimeSeries.empty()) { + const auto& p = LastPoint; + consumer(p.GetTime(), p.GetValueType(), p.GetValue()); + } else { + for (const auto& p: TimeSeries) { + consumer(p.GetTime(), p.GetValueType(), p.GetValue()); + } + } + } +}; + +struct TCommonParts { + TInstant CommonTime; + TLabels CommonLabels; +}; + +class IHaltableMetricConsumer: public IMetricConsumer { +public: + virtual bool NeedToStop() const = 0; +}; + +// TODO(ivanzhukov@): check all states for cases when a json document is invalid +// e.g. "metrics" or "commonLabels" keys are specified multiple times +class TCommonPartsCollector: public IHaltableMetricConsumer { +public: + TCommonParts&& CommonParts() { + return std::move(CommonParts_); + } + +private: + bool NeedToStop() const override { + return TInstant::Zero() != CommonParts_.CommonTime && !CommonParts_.CommonLabels.Empty(); + } + + void OnStreamBegin() override { + } + + void OnStreamEnd() override { + } + + void OnCommonTime(TInstant time) override { + CommonParts_.CommonTime = time; + } + + void OnMetricBegin(EMetricType) override { + IsMetric_ = true; + } + + void OnMetricEnd() override { + IsMetric_ = false; + } + + void OnLabelsBegin() override { + } + + void OnLabelsEnd() override { + } + + void OnLabel(TStringBuf name, TStringBuf value) override { + if (!IsMetric_) { + CommonParts_.CommonLabels.Add(std::move(name), std::move(value)); + } + } + + void OnDouble(TInstant, double) override { + } + + void OnInt64(TInstant, i64) override { + } + + void OnUint64(TInstant, ui64) override { + } + + void OnHistogram(TInstant, IHistogramSnapshotPtr) override { + } + + void OnLogHistogram(TInstant, TLogHistogramSnapshotPtr) override { + } + + void OnSummaryDouble(TInstant, ISummaryDoubleSnapshotPtr) override { + } + +private: + TCommonParts CommonParts_; + bool IsMetric_{false}; +}; + +class TCommonPartsProxy: public IHaltableMetricConsumer { +public: + TCommonPartsProxy(TCommonParts&& commonParts, IMetricConsumer* c) + : CommonParts_{std::move(commonParts)} + , Consumer_{c} + {} + +private: + bool NeedToStop() const override { + return false; + } + + void OnStreamBegin() override { + Consumer_->OnStreamBegin(); + + if (!CommonParts_.CommonLabels.Empty()) { + Consumer_->OnLabelsBegin(); + + for (auto&& label : CommonParts_.CommonLabels) { + Consumer_->OnLabel(label.Name(), label.Value()); + } + + Consumer_->OnLabelsEnd(); + } + + if (TInstant::Zero() != CommonParts_.CommonTime) { + Consumer_->OnCommonTime(CommonParts_.CommonTime); + } + } + + void OnStreamEnd() override { + Consumer_->OnStreamEnd(); + } + + void OnCommonTime(TInstant) override { + } + + void OnMetricBegin(EMetricType type) override { + IsMetric_ = true; + + Consumer_->OnMetricBegin(type); + } + + void OnMetricEnd() override { + IsMetric_ = false; + + Consumer_->OnMetricEnd(); + } + + void OnLabelsBegin() override { + if (IsMetric_) { + Consumer_->OnLabelsBegin(); + } + } + + void OnLabelsEnd() override { + if (IsMetric_) { + Consumer_->OnLabelsEnd(); + } + } + + void OnLabel(TStringBuf name, TStringBuf value) override { + if (IsMetric_) { + Consumer_->OnLabel(std::move(name), std::move(value)); + } + } + + void OnDouble(TInstant time, double value) override { + Consumer_->OnDouble(time, value); + } + + void OnInt64(TInstant time, i64 value) override { + Consumer_->OnInt64(time, value); + } + + void OnUint64(TInstant time, ui64 value) override { + Consumer_->OnUint64(time, value); + } + + void OnHistogram(TInstant time, IHistogramSnapshotPtr snapshot) override { + Consumer_->OnHistogram(time, std::move(snapshot)); + } + + void OnLogHistogram(TInstant time, TLogHistogramSnapshotPtr snapshot) override { + Consumer_->OnLogHistogram(time, std::move(snapshot)); + } + + void OnSummaryDouble(TInstant time, ISummaryDoubleSnapshotPtr snapshot) override { + Consumer_->OnSummaryDouble(time, std::move(snapshot)); + } + +private: + const TCommonParts CommonParts_; + IMetricConsumer* Consumer_; + bool IsMetric_{false}; +}; + +/////////////////////////////////////////////////////////////////////// +// TDecoderJson +/////////////////////////////////////////////////////////////////////// +class TDecoderJson final: public NJson::TJsonCallbacks { + struct TState { + enum EState { + ROOT_OBJECT = 0x01, + + COMMON_LABELS, + COMMON_TS, + METRICS_ARRAY, + + METRIC_OBJECT, + METRIC_NAME, + METRIC_LABELS, + METRIC_TYPE, + METRIC_MODE, // TODO: must be deleted + METRIC_TIMESERIES, + METRIC_TS, + METRIC_VALUE, + METRIC_HIST, + METRIC_HIST_BOUNDS, + METRIC_HIST_BUCKETS, + METRIC_HIST_INF, + METRIC_DSUMMARY, + METRIC_DSUMMARY_SUM, + METRIC_DSUMMARY_MIN, + METRIC_DSUMMARY_MAX, + METRIC_DSUMMARY_LAST, + METRIC_DSUMMARY_COUNT, + METRIC_LOG_HIST, + METRIC_LOG_HIST_BASE, + METRIC_LOG_HIST_ZEROS, + METRIC_LOG_HIST_START_POWER, + METRIC_LOG_HIST_BUCKETS, + }; + + constexpr EState Current() const noexcept { + return static_cast<EState>(State_ & 0xFF); + } + + void ToNext(EState state) noexcept { + constexpr auto bitSize = 8 * sizeof(ui8); + State_ = (State_ << bitSize) | static_cast<ui8>(state); + } + + void ToPrev() noexcept { + constexpr auto bitSize = 8 * sizeof(ui8); + State_ = State_ >> bitSize; + } + + private: + ui64 State_ = static_cast<ui64>(ROOT_OBJECT); + }; + +public: + TDecoderJson(TStringBuf data, IHaltableMetricConsumer* metricConsumer, TStringBuf metricNameLabel) + : Data_(data) + , MetricConsumer_(metricConsumer) + , MetricNameLabel_(metricNameLabel) + { + } + +private: +#define PARSE_ENSURE(CONDITION, ...) \ +do { \ +if (Y_UNLIKELY(!(CONDITION))) { \ + ErrorMsg_ = TStringBuilder() << __VA_ARGS__; \ + return false; \ +} \ +} while (false) + + bool OnInteger(long long value) override { + switch (State_.Current()) { + case TState::COMMON_TS: + PARSE_ENSURE(value >= 0, "unexpected negative number in a common timestamp: " << value); + MetricConsumer_->OnCommonTime(TInstant::Seconds(value)); + State_.ToPrev(); + + if (MetricConsumer_->NeedToStop()) { + IsIntentionallyHalted_ = true; + return false; + } + + break; + + case TState::METRIC_TS: + PARSE_ENSURE(value >= 0, "unexpected negative number in a metric timestamp: " << value); + LastMetric_.SetLastTime(TInstant::Seconds(value)); + State_.ToPrev(); + break; + + case TState::METRIC_VALUE: + LastMetric_.SetLastValue(static_cast<i64>(value)); + State_.ToPrev(); + break; + + case TState::METRIC_HIST_BOUNDS: + LastMetric_.HistogramBuilder.AddBound(static_cast<double>(value)); + break; + + case TState::METRIC_HIST_BUCKETS: + PARSE_ENSURE(value >= 0 && static_cast<ui64>(value) <= Max<TBucketValues::value_type>(), "value is out of bounds " << value); + LastMetric_.HistogramBuilder.AddValue(value); + break; + + case TState::METRIC_HIST_INF: + PARSE_ENSURE(value >= 0, "unexpected negative number in histogram inf: " << value); + LastMetric_.HistogramBuilder.AddInf(value); + State_.ToPrev(); + break; + + case TState::METRIC_DSUMMARY_COUNT: + LastMetric_.SummaryBuilder.SetCount(value); + State_.ToPrev(); + break; + + case TState::METRIC_DSUMMARY_SUM: + LastMetric_.SummaryBuilder.SetSum(value); + State_.ToPrev(); + break; + case TState::METRIC_DSUMMARY_MIN: + LastMetric_.SummaryBuilder.SetMin(value); + State_.ToPrev(); + break; + case TState::METRIC_DSUMMARY_MAX: + LastMetric_.SummaryBuilder.SetMax(value); + State_.ToPrev(); + break; + case TState::METRIC_DSUMMARY_LAST: + LastMetric_.SummaryBuilder.SetLast(value); + State_.ToPrev(); + break; + + case TState::METRIC_LOG_HIST_BASE: + LastMetric_.LogHistBuilder.SetBase(value); + State_.ToPrev(); + break; + + case TState::METRIC_LOG_HIST_ZEROS: + LastMetric_.LogHistBuilder.SetZerosCount(value); + State_.ToPrev(); + break; + + case TState::METRIC_LOG_HIST_START_POWER: + LastMetric_.LogHistBuilder.SetStartPower(value); + State_.ToPrev(); + break; + + case TState::METRIC_LOG_HIST_BUCKETS: + LastMetric_.LogHistBuilder.AddBucketValue(value); + break; + + default: + return false; + } + return true; + } + + bool OnUInteger(unsigned long long value) override { + switch (State_.Current()) { + case TState::COMMON_TS: + MetricConsumer_->OnCommonTime(TInstant::Seconds(value)); + State_.ToPrev(); + + if (MetricConsumer_->NeedToStop()) { + IsIntentionallyHalted_ = true; + return false; + } + + break; + + case TState::METRIC_TS: + LastMetric_.SetLastTime(TInstant::Seconds(value)); + State_.ToPrev(); + break; + + case TState::METRIC_VALUE: + PARSE_ENSURE(value <= Max<ui64>(), "Metric value is out of bounds: " << value); + LastMetric_.SetLastValue(static_cast<ui64>(value)); + State_.ToPrev(); + break; + + case TState::METRIC_HIST_BOUNDS: + LastMetric_.HistogramBuilder.AddBound(static_cast<double>(value)); + break; + + case TState::METRIC_HIST_BUCKETS: + PARSE_ENSURE(value <= Max<TBucketValues::value_type>(), "Histogram bucket value is out of bounds: " << value); + LastMetric_.HistogramBuilder.AddValue(value); + break; + + case TState::METRIC_HIST_INF: + LastMetric_.HistogramBuilder.AddInf(value); + State_.ToPrev(); + break; + + case TState::METRIC_DSUMMARY_COUNT: + LastMetric_.SummaryBuilder.SetCount(value); + State_.ToPrev(); + break; + + case TState::METRIC_DSUMMARY_SUM: + LastMetric_.SummaryBuilder.SetSum(value); + State_.ToPrev(); + break; + case TState::METRIC_DSUMMARY_MIN: + LastMetric_.SummaryBuilder.SetMin(value); + State_.ToPrev(); + break; + case TState::METRIC_DSUMMARY_MAX: + LastMetric_.SummaryBuilder.SetMax(value); + State_.ToPrev(); + break; + case TState::METRIC_DSUMMARY_LAST: + LastMetric_.SummaryBuilder.SetLast(value); + State_.ToPrev(); + break; + + case TState::METRIC_LOG_HIST_BASE: + LastMetric_.LogHistBuilder.SetBase(value); + State_.ToPrev(); + break; + + case TState::METRIC_LOG_HIST_ZEROS: + LastMetric_.LogHistBuilder.SetZerosCount(value); + State_.ToPrev(); + break; + + case TState::METRIC_LOG_HIST_START_POWER: + LastMetric_.LogHistBuilder.SetStartPower(value); + State_.ToPrev(); + break; + + case TState::METRIC_LOG_HIST_BUCKETS: + LastMetric_.LogHistBuilder.AddBucketValue(value); + break; + + default: + return false; + } + return true; + } + + bool OnDouble(double value) override { + switch (State_.Current()) { + case TState::METRIC_VALUE: + LastMetric_.SetLastValue(value); + State_.ToPrev(); + break; + + case TState::METRIC_HIST_BOUNDS: + LastMetric_.HistogramBuilder.AddBound(value); + break; + + case TState::METRIC_DSUMMARY_SUM: + LastMetric_.SummaryBuilder.SetSum(value); + State_.ToPrev(); + break; + case TState::METRIC_DSUMMARY_MIN: + LastMetric_.SummaryBuilder.SetMin(value); + State_.ToPrev(); + break; + case TState::METRIC_DSUMMARY_MAX: + LastMetric_.SummaryBuilder.SetMax(value); + State_.ToPrev(); + break; + case TState::METRIC_DSUMMARY_LAST: + LastMetric_.SummaryBuilder.SetLast(value); + State_.ToPrev(); + break; + + case TState::METRIC_LOG_HIST_BASE: + LastMetric_.LogHistBuilder.SetBase(value); + State_.ToPrev(); + break; + + case TState::METRIC_LOG_HIST_BUCKETS: + LastMetric_.LogHistBuilder.AddBucketValue(value); + break; + + default: + return false; + } + return true; + } + + bool OnString(const TStringBuf& value) override { + switch (State_.Current()) { + case TState::COMMON_LABELS: + PARSE_ENSURE(!LastLabelName_.empty(), "empty label name in common labels"); + MetricConsumer_->OnLabel(LastLabelName_, TString{value}); + break; + + case TState::METRIC_LABELS: + PARSE_ENSURE(!LastLabelName_.empty(), "empty label name in metric labels"); + LastMetric_.Labels.Add(LastLabelName_, TString{value}); + break; + + case TState::METRIC_NAME: + PARSE_ENSURE(!value.empty(), "empty metric name"); + LastMetric_.Labels.Add(MetricNameLabel_, TString{value}); + State_.ToPrev(); + break; + + case TState::COMMON_TS: + MetricConsumer_->OnCommonTime(TInstant::ParseIso8601(value)); + State_.ToPrev(); + + if (MetricConsumer_->NeedToStop()) { + IsIntentionallyHalted_ = true; + return false; + } + + break; + + case TState::METRIC_TS: + LastMetric_.SetLastTime(TInstant::ParseIso8601(value)); + State_.ToPrev(); + break; + + case TState::METRIC_VALUE: + if (auto [doubleValue, ok] = ParseSpecDouble(value); ok) { + LastMetric_.SetLastValue(doubleValue); + } else { + return false; + } + State_.ToPrev(); + break; + + case TState::METRIC_TYPE: + LastMetric_.Type = MetricTypeFromStr(value); + State_.ToPrev(); + break; + + case TState::METRIC_MODE: + if (value == TStringBuf("deriv")) { + LastMetric_.Type = EMetricType::RATE; + } + State_.ToPrev(); + break; + + case TState::METRIC_DSUMMARY_SUM: + if (auto [doubleValue, ok] = ParseSpecDouble(value); ok) { + LastMetric_.SummaryBuilder.SetSum(doubleValue); + } else { + return false; + } + State_.ToPrev(); + break; + + case TState::METRIC_DSUMMARY_MIN: + if (auto [doubleValue, ok] = ParseSpecDouble(value); ok) { + LastMetric_.SummaryBuilder.SetMin(doubleValue); + } else { + return false; + } + State_.ToPrev(); + break; + + case TState::METRIC_DSUMMARY_MAX: + if (auto [doubleValue, ok] = ParseSpecDouble(value); ok) { + LastMetric_.SummaryBuilder.SetMax(doubleValue); + } else { + return false; + } + State_.ToPrev(); + break; + + case TState::METRIC_DSUMMARY_LAST: + if (auto [doubleValue, ok] = ParseSpecDouble(value); ok) { + LastMetric_.SummaryBuilder.SetLast(doubleValue); + } else { + return false; + } + State_.ToPrev(); + break; + + default: + return false; + } + + return true; + } + + bool OnMapKey(const TStringBuf& key) override { + switch (State_.Current()) { + case TState::ROOT_OBJECT: + if (key == TStringBuf("commonLabels") || key == TStringBuf("labels")) { + State_.ToNext(TState::COMMON_LABELS); + } else if (key == TStringBuf("ts")) { + State_.ToNext(TState::COMMON_TS); + } else if (key == TStringBuf("sensors") || key == TStringBuf("metrics")) { + State_.ToNext(TState::METRICS_ARRAY); + } + break; + + case TState::COMMON_LABELS: + case TState::METRIC_LABELS: + LastLabelName_ = key; + break; + + case TState::METRIC_OBJECT: + if (key == TStringBuf("labels")) { + State_.ToNext(TState::METRIC_LABELS); + } else if (key == TStringBuf("name")) { + State_.ToNext(TState::METRIC_NAME); + } else if (key == TStringBuf("ts")) { + PARSE_ENSURE(!LastMetric_.SeenTimeseries, + "mixed timeseries and ts attributes"); + LastMetric_.SeenTsOrValue = true; + State_.ToNext(TState::METRIC_TS); + } else if (key == TStringBuf("value")) { + PARSE_ENSURE(!LastMetric_.SeenTimeseries, + "mixed timeseries and value attributes"); + LastMetric_.SeenTsOrValue = true; + State_.ToNext(TState::METRIC_VALUE); + } else if (key == TStringBuf("timeseries")) { + PARSE_ENSURE(!LastMetric_.SeenTsOrValue, + "mixed timeseries and ts/value attributes"); + LastMetric_.SeenTimeseries = true; + State_.ToNext(TState::METRIC_TIMESERIES); + } else if (key == TStringBuf("mode")) { + State_.ToNext(TState::METRIC_MODE); + } else if (key == TStringBuf("kind") || key == TStringBuf("type")) { + State_.ToNext(TState::METRIC_TYPE); + } else if (key == TStringBuf("hist")) { + State_.ToNext(TState::METRIC_HIST); + } else if (key == TStringBuf("summary")) { + State_.ToNext(TState::METRIC_DSUMMARY); + } else if (key == TStringBuf("log_hist")) { + State_.ToNext(TState::METRIC_LOG_HIST); + } else if (key == TStringBuf("memOnly")) { + // deprecated. Skip it without errors for backward compatibility + } else { + ErrorMsg_ = TStringBuilder() << "unexpected key \"" << key << "\" in a metric schema"; + return false; + } + break; + + case TState::METRIC_TIMESERIES: + if (key == TStringBuf("ts")) { + State_.ToNext(TState::METRIC_TS); + } else if (key == TStringBuf("value")) { + State_.ToNext(TState::METRIC_VALUE); + } else if (key == TStringBuf("hist")) { + State_.ToNext(TState::METRIC_HIST); + } else if (key == TStringBuf("summary")) { + State_.ToNext(TState::METRIC_DSUMMARY); + } else if (key == TStringBuf("log_hist")) { + State_.ToNext(TState::METRIC_LOG_HIST); + } + break; + + case TState::METRIC_HIST: + if (key == TStringBuf("bounds")) { + State_.ToNext(TState::METRIC_HIST_BOUNDS); + } else if (key == TStringBuf("buckets")) { + State_.ToNext(TState::METRIC_HIST_BUCKETS); + } else if (key == TStringBuf("inf")) { + State_.ToNext(TState::METRIC_HIST_INF); + } + break; + + case TState::METRIC_LOG_HIST: + if (key == TStringBuf("base")) { + State_.ToNext(TState::METRIC_LOG_HIST_BASE); + } else if (key == TStringBuf("zeros_count")) { + State_.ToNext(TState::METRIC_LOG_HIST_ZEROS); + } else if (key == TStringBuf("start_power")) { + State_.ToNext(TState::METRIC_LOG_HIST_START_POWER); + } else if (key == TStringBuf("buckets")) { + State_.ToNext(TState::METRIC_LOG_HIST_BUCKETS); + } + break; + + case TState::METRIC_DSUMMARY: + if (key == TStringBuf("sum")) { + State_.ToNext(TState::METRIC_DSUMMARY_SUM); + } else if (key == TStringBuf("min")) { + State_.ToNext(TState::METRIC_DSUMMARY_MIN); + } else if (key == TStringBuf("max")) { + State_.ToNext(TState::METRIC_DSUMMARY_MAX); + } else if (key == TStringBuf("last")) { + State_.ToNext(TState::METRIC_DSUMMARY_LAST); + } else if (key == TStringBuf("count")) { + State_.ToNext(TState::METRIC_DSUMMARY_COUNT); + } + + break; + + + default: + return false; + } + + return true; + } + + bool OnOpenMap() override { + switch (State_.Current()) { + case TState::ROOT_OBJECT: + MetricConsumer_->OnStreamBegin(); + break; + + case TState::COMMON_LABELS: + MetricConsumer_->OnLabelsBegin(); + break; + + case TState::METRICS_ARRAY: + State_.ToNext(TState::METRIC_OBJECT); + LastMetric_.Clear(); + break; + + default: + break; + } + return true; + } + + bool OnCloseMap() override { + switch (State_.Current()) { + case TState::ROOT_OBJECT: + MetricConsumer_->OnStreamEnd(); + break; + + case TState::METRIC_LABELS: + State_.ToPrev(); + break; + + case TState::COMMON_LABELS: + MetricConsumer_->OnLabelsEnd(); + State_.ToPrev(); + + if (MetricConsumer_->NeedToStop()) { + IsIntentionallyHalted_ = true; + return false; + } + + break; + + case TState::METRIC_OBJECT: + ConsumeMetric(); + State_.ToPrev(); + break; + + case TState::METRIC_TIMESERIES: + LastMetric_.SaveLastPoint(); + break; + + case TState::METRIC_HIST: + case TState::METRIC_DSUMMARY: + case TState::METRIC_LOG_HIST: + State_.ToPrev(); + break; + + default: + break; + } + return true; + } + + bool OnOpenArray() override { + auto currentState = State_.Current(); + PARSE_ENSURE( + currentState == TState::METRICS_ARRAY || + currentState == TState::METRIC_TIMESERIES || + currentState == TState::METRIC_HIST_BOUNDS || + currentState == TState::METRIC_HIST_BUCKETS || + currentState == TState::METRIC_LOG_HIST_BUCKETS, + "unexpected array begin"); + return true; + } + + bool OnCloseArray() override { + switch (State_.Current()) { + case TState::METRICS_ARRAY: + case TState::METRIC_TIMESERIES: + case TState::METRIC_HIST_BOUNDS: + case TState::METRIC_HIST_BUCKETS: + case TState::METRIC_LOG_HIST_BUCKETS: + State_.ToPrev(); + break; + + default: + return false; + } + return true; + } + + void OnError(size_t off, TStringBuf reason) override { + if (IsIntentionallyHalted_) { + return; + } + + size_t snippetBeg = (off < 20) ? 0 : (off - 20); + TStringBuf snippet = Data_.SubStr(snippetBeg, 40); + + throw TJsonDecodeError() + << "cannot parse JSON, error at: " << off + << ", reason: " << (ErrorMsg_.empty() ? reason : TStringBuf{ErrorMsg_}) + << "\nsnippet: ..." << snippet << "..."; + } + + bool OnEnd() override { + return true; + } + + void ConsumeMetric() { + // for backwad compatibility all unknown metrics treated as gauges + if (LastMetric_.Type == EMetricType::UNKNOWN) { + if (LastMetric_.HistogramBuilder.Empty()) { + LastMetric_.Type = EMetricType::GAUGE; + } else { + LastMetric_.Type = EMetricType::HIST; + } + } + + // (1) begin metric + MetricConsumer_->OnMetricBegin(LastMetric_.Type); + + // (2) labels + if (!LastMetric_.Labels.empty()) { + MetricConsumer_->OnLabelsBegin(); + for (auto&& label : LastMetric_.Labels) { + MetricConsumer_->OnLabel(label.Name(), label.Value()); + } + MetricConsumer_->OnLabelsEnd(); + } + + // (3) values + switch (LastMetric_.Type) { + case EMetricType::GAUGE: + LastMetric_.Consume([this](TInstant time, EMetricValueType valueType, TMetricValue value) { + MetricConsumer_->OnDouble(time, value.AsDouble(valueType)); + }); + break; + + case EMetricType::IGAUGE: + LastMetric_.Consume([this](TInstant time, EMetricValueType valueType, TMetricValue value) { + MetricConsumer_->OnInt64(time, value.AsInt64(valueType)); + }); + break; + + case EMetricType::COUNTER: + case EMetricType::RATE: + LastMetric_.Consume([this](TInstant time, EMetricValueType valueType, TMetricValue value) { + MetricConsumer_->OnUint64(time, value.AsUint64(valueType)); + }); + break; + + case EMetricType::HIST: + case EMetricType::HIST_RATE: + if (LastMetric_.TimeSeries.empty()) { + auto time = LastMetric_.LastPoint.GetTime(); + auto histogram = LastMetric_.HistogramBuilder.Build(); + MetricConsumer_->OnHistogram(time, histogram); + } else { + for (const auto& p : LastMetric_.TimeSeries) { + DECODE_ENSURE(p.GetValueType() == EMetricValueType::HISTOGRAM, "Value is not a histogram"); + MetricConsumer_->OnHistogram(p.GetTime(), p.GetValue().AsHistogram()); + } + } + break; + + case EMetricType::DSUMMARY: + if (LastMetric_.TimeSeries.empty()) { + auto time = LastMetric_.LastPoint.GetTime(); + auto summary = LastMetric_.SummaryBuilder.Build(); + MetricConsumer_->OnSummaryDouble(time, summary); + } else { + for (const auto& p : LastMetric_.TimeSeries) { + DECODE_ENSURE(p.GetValueType() == EMetricValueType::SUMMARY, "Value is not a summary"); + MetricConsumer_->OnSummaryDouble(p.GetTime(), p.GetValue().AsSummaryDouble()); + } + } + break; + + case EMetricType::LOGHIST: + if (LastMetric_.TimeSeries.empty()) { + auto time = LastMetric_.LastPoint.GetTime(); + auto logHist = LastMetric_.LogHistBuilder.Build(); + MetricConsumer_->OnLogHistogram(time, logHist); + } else { + for (const auto& p : LastMetric_.TimeSeries) { + DECODE_ENSURE(p.GetValueType() == EMetricValueType::LOGHISTOGRAM, "Value is not a log_histogram"); + MetricConsumer_->OnLogHistogram(p.GetTime(), p.GetValue().AsLogHistogram()); + } + } + break; + + case EMetricType::UNKNOWN: + // TODO: output metric labels + ythrow yexception() << "unknown metric type"; + } + + // (4) end metric + MetricConsumer_->OnMetricEnd(); + } + +private: + TStringBuf Data_; + IHaltableMetricConsumer* MetricConsumer_; + TString MetricNameLabel_; + TState State_; + TString LastLabelName_; + TMetricCollector LastMetric_; + TString ErrorMsg_; + bool IsIntentionallyHalted_{false}; +}; + +} // namespace + +void DecodeJson(TStringBuf data, IMetricConsumer* c, TStringBuf metricNameLabel) { + TCommonPartsCollector commonPartsCollector; + { + TMemoryInput memIn(data); + TDecoderJson decoder(data, &commonPartsCollector, metricNameLabel); + // no need to check a return value. If there is an error, a TJsonDecodeError is thrown + NJson::ReadJson(&memIn, &decoder); + } + + TCommonPartsProxy commonPartsProxy(std::move(commonPartsCollector.CommonParts()), c); + { + TMemoryInput memIn(data); + TDecoderJson decoder(data, &commonPartsProxy, metricNameLabel); + // no need to check a return value. If there is an error, a TJsonDecodeError is thrown + NJson::ReadJson(&memIn, &decoder); + } +} + +#undef DECODE_ENSURE + +} diff --git a/library/cpp/monlib/encode/json/json_decoder_ut.cpp b/library/cpp/monlib/encode/json/json_decoder_ut.cpp new file mode 100644 index 0000000000..4464e1d26a --- /dev/null +++ b/library/cpp/monlib/encode/json/json_decoder_ut.cpp @@ -0,0 +1,179 @@ +#include "json_decoder.cpp" + +#include <library/cpp/monlib/consumers/collecting_consumer.h> +#include <library/cpp/testing/unittest/registar.h> + +#include <array> + + +using namespace NMonitoring; + +enum EJsonPart : ui8 { + METRICS = 0, + COMMON_TS = 1, + COMMON_LABELS = 2, +}; + +constexpr std::array<TStringBuf, 3> JSON_PARTS = { + TStringBuf(R"("metrics": [{ + "labels": { "key": "value" }, + "type": "GAUGE", + "value": 123 + }])"), + + TStringBuf(R"("ts": 1)"), + + TStringBuf(R"("commonLabels": { + "key1": "value1", + "key2": "value2" + })"), +}; + +TString BuildJson(std::initializer_list<EJsonPart> parts) { + TString data = "{"; + + for (auto it = parts.begin(); it != parts.end(); ++it) { + data += JSON_PARTS[*it]; + + if (it + 1 != parts.end()) { + data += ","; + } + } + + data += "}"; + return data; +} + +void ValidateCommonParts(TCommonParts&& commonParts, bool checkLabels, bool checkTs) { + if (checkTs) { + UNIT_ASSERT_VALUES_EQUAL(commonParts.CommonTime.MilliSeconds(), 1000); + } + + if (checkLabels) { + auto& labels = commonParts.CommonLabels; + UNIT_ASSERT_VALUES_EQUAL(labels.Size(), 2); + UNIT_ASSERT(labels.Has(TStringBuf("key1"))); + UNIT_ASSERT(labels.Has(TStringBuf("key2"))); + UNIT_ASSERT_VALUES_EQUAL(labels.Get(TStringBuf("key1")).value()->Value(), "value1"); + UNIT_ASSERT_VALUES_EQUAL(labels.Get(TStringBuf("key2")).value()->Value(), "value2"); + } +} + +void ValidateMetrics(const TVector<TMetricData>& metrics) { + UNIT_ASSERT_VALUES_EQUAL(metrics.size(), 1); + + auto& m = metrics[0]; + UNIT_ASSERT_VALUES_EQUAL(m.Kind, EMetricType::GAUGE); + auto& l = m.Labels; + UNIT_ASSERT_VALUES_EQUAL(l.Size(), 1); + UNIT_ASSERT_VALUES_EQUAL(l.Get(0)->Name(), "key"); + UNIT_ASSERT_VALUES_EQUAL(l.Get(0)->Value(), "value"); + + UNIT_ASSERT_VALUES_EQUAL(m.Values->Size(), 1); + UNIT_ASSERT_VALUES_EQUAL((*m.Values)[0].GetValue().AsDouble(), 123); +} + +void CheckCommonPartsCollector(TString data, bool shouldBeStopped, bool checkLabels = true, bool checkTs = true, TStringBuf metricNameLabel = "name") { + TCommonPartsCollector commonPartsCollector; + TMemoryInput memIn(data); + TDecoderJson decoder(data, &commonPartsCollector, metricNameLabel); + + bool isOk{false}; + UNIT_ASSERT_NO_EXCEPTION(isOk = NJson::ReadJson(&memIn, &decoder)); + UNIT_ASSERT_VALUES_EQUAL(isOk, !shouldBeStopped); + + ValidateCommonParts(commonPartsCollector.CommonParts(), checkLabels, checkTs); +} + +Y_UNIT_TEST_SUITE(TJsonDecoderTest) { + Y_UNIT_TEST(FullCommonParts) { + CheckCommonPartsCollector(BuildJson({COMMON_LABELS, COMMON_TS, METRICS}), true); + CheckCommonPartsCollector(BuildJson({COMMON_TS, COMMON_LABELS, METRICS}), true); + + CheckCommonPartsCollector(BuildJson({METRICS, COMMON_TS, COMMON_LABELS}), true); + CheckCommonPartsCollector(BuildJson({METRICS, COMMON_LABELS, COMMON_TS}), true); + + CheckCommonPartsCollector(BuildJson({COMMON_LABELS, METRICS, COMMON_TS}), true); + CheckCommonPartsCollector(BuildJson({COMMON_TS, METRICS, COMMON_LABELS}), true); + } + + Y_UNIT_TEST(PartialCommonParts) { + CheckCommonPartsCollector(BuildJson({COMMON_TS, METRICS}), false, false, true); + CheckCommonPartsCollector(BuildJson({COMMON_LABELS, METRICS}), false, true, false); + + CheckCommonPartsCollector(BuildJson({METRICS, COMMON_LABELS}), false, true, false); + CheckCommonPartsCollector(BuildJson({METRICS, COMMON_TS}), false, false, true); + + CheckCommonPartsCollector(BuildJson({METRICS}), false, false, false); + } + + Y_UNIT_TEST(CheckCommonPartsAndMetrics) { + auto data = BuildJson({COMMON_LABELS, COMMON_TS, METRICS}); + TCollectingConsumer collector; + + DecodeJson(data, &collector); + + TCommonParts commonParts; + commonParts.CommonTime = collector.CommonTime; + commonParts.CommonLabels = collector.CommonLabels; + + ValidateCommonParts(std::move(commonParts), true, true); + ValidateMetrics(collector.Metrics); + } + + Y_UNIT_TEST(CanParseHistogramsWithInf) { + const char* metricsData = R"({ +"metrics": + [ + { + "hist": { + "bounds": [ + 10 + ], + "buckets": [ + 11 + ], + "inf": 12 + }, + "name":"s1", + "type": "HIST_RATE" + }, + { + "hist": { + "bounds": [ + 20 + ], + "buckets": [ + 21 + ] + }, + "name":"s2", + "type":"HIST_RATE" + } + ] +})"; + TCollectingConsumer consumer(false); + DecodeJson(metricsData, &consumer); + + UNIT_ASSERT_VALUES_EQUAL(consumer.Metrics.size(), 2); + { + const auto& m = consumer.Metrics[0]; + UNIT_ASSERT_VALUES_EQUAL(m.Kind, EMetricType::HIST_RATE); + UNIT_ASSERT_VALUES_EQUAL(m.Values->Size(), 1); + const auto* histogram = (*m.Values)[0].GetValue().AsHistogram(); + UNIT_ASSERT_VALUES_EQUAL(histogram->Count(), 2); + UNIT_ASSERT_VALUES_EQUAL(histogram->UpperBound(1), Max<TBucketBound>()); + UNIT_ASSERT_VALUES_EQUAL(histogram->Value(0), 11); + UNIT_ASSERT_VALUES_EQUAL(histogram->Value(1), 12); + } + { + const auto& m = consumer.Metrics[1]; + UNIT_ASSERT_VALUES_EQUAL(m.Kind, EMetricType::HIST_RATE); + UNIT_ASSERT_VALUES_EQUAL(m.Values->Size(), 1); + const auto* histogram = (*m.Values)[0].GetValue().AsHistogram(); + UNIT_ASSERT_VALUES_EQUAL(histogram->Count(), 1); + UNIT_ASSERT_VALUES_EQUAL(histogram->UpperBound(0), 20); + UNIT_ASSERT_VALUES_EQUAL(histogram->Value(0), 21); + } + } +} diff --git a/library/cpp/monlib/encode/json/json_encoder.cpp b/library/cpp/monlib/encode/json/json_encoder.cpp new file mode 100644 index 0000000000..20d2bb6283 --- /dev/null +++ b/library/cpp/monlib/encode/json/json_encoder.cpp @@ -0,0 +1,556 @@ +#include "json.h" +#include "typed_point.h" + +#include <library/cpp/monlib/encode/buffered/buffered_encoder_base.h> +#include <library/cpp/monlib/encode/encoder_state.h> +#include <library/cpp/monlib/metrics/metric.h> +#include <library/cpp/monlib/metrics/metric_value.h> +#include <library/cpp/monlib/metrics/labels.h> + +#include <library/cpp/json/writer/json.h> + +#include <util/charset/utf8.h> +#include <util/generic/algorithm.h> + +namespace NMonitoring { + namespace { + enum class EJsonStyle { + Solomon, + Cloud + }; + + /////////////////////////////////////////////////////////////////////// + // TJsonWriter + /////////////////////////////////////////////////////////////////////// + class TJsonWriter { + public: + TJsonWriter(IOutputStream* out, int indentation, EJsonStyle style, TStringBuf metricNameLabel) + : Buf_(NJsonWriter::HEM_UNSAFE, out) + , Style_(style) + , MetricNameLabel_(metricNameLabel) + , CurrentMetricName_() + { + Buf_.SetIndentSpaces(indentation); + Buf_.SetWriteNanAsString(); + } + + void WriteTime(TInstant time) { + if (time != TInstant::Zero()) { + Buf_.WriteKey(TStringBuf("ts")); + if (Style_ == EJsonStyle::Solomon) { + Buf_.WriteULongLong(time.Seconds()); + } else { + Buf_.WriteString(time.ToString()); + } + } + } + + void WriteValue(double value) { + Buf_.WriteKey(TStringBuf("value")); + Buf_.WriteDouble(value); + } + + void WriteValue(i64 value) { + Buf_.WriteKey(TStringBuf("value")); + Buf_.WriteLongLong(value); + } + + void WriteValue(ui64 value) { + Buf_.WriteKey(TStringBuf("value")); + Buf_.WriteULongLong(value); + } + + void WriteValue(IHistogramSnapshot* s) { + Y_ENSURE(Style_ == EJsonStyle::Solomon); + + Buf_.WriteKey(TStringBuf("hist")); + Buf_.BeginObject(); + if (ui32 count = s->Count()) { + bool hasInf = (s->UpperBound(count - 1) == Max<double>()); + if (hasInf) { + count--; + } + + Buf_.WriteKey(TStringBuf("bounds")); + Buf_.BeginList(); + for (ui32 i = 0; i < count; i++) { + Buf_.WriteDouble(s->UpperBound(i)); + } + Buf_.EndList(); + + Buf_.WriteKey(TStringBuf("buckets")); + Buf_.BeginList(); + for (ui32 i = 0; i < count; i++) { + Buf_.WriteULongLong(s->Value(i)); + } + Buf_.EndList(); + + if (hasInf) { + Buf_.WriteKey(TStringBuf("inf")); + Buf_.WriteULongLong(s->Value(count)); + } + } + Buf_.EndObject(); + } + + void WriteValue(ISummaryDoubleSnapshot* s) { + Y_ENSURE(Style_ == EJsonStyle::Solomon); + + Buf_.WriteKey(TStringBuf("summary")); + Buf_.BeginObject(); + + Buf_.WriteKey(TStringBuf("sum")); + Buf_.WriteDouble(s->GetSum()); + + Buf_.WriteKey(TStringBuf("min")); + Buf_.WriteDouble(s->GetMin()); + + Buf_.WriteKey(TStringBuf("max")); + Buf_.WriteDouble(s->GetMax()); + + Buf_.WriteKey(TStringBuf("last")); + Buf_.WriteDouble(s->GetLast()); + + Buf_.WriteKey(TStringBuf("count")); + Buf_.WriteULongLong(s->GetCount()); + + Buf_.EndObject(); + } + + void WriteValue(TLogHistogramSnapshot* s) { + Y_ENSURE(Style_ == EJsonStyle::Solomon); + + Buf_.WriteKey(TStringBuf("log_hist")); + Buf_.BeginObject(); + + Buf_.WriteKey(TStringBuf("base")); + Buf_.WriteDouble(s->Base()); + + Buf_.WriteKey(TStringBuf("zeros_count")); + Buf_.WriteULongLong(s->ZerosCount()); + + Buf_.WriteKey(TStringBuf("start_power")); + Buf_.WriteInt(s->StartPower()); + + Buf_.WriteKey(TStringBuf("buckets")); + Buf_.BeginList(); + for (size_t i = 0; i < s->Count(); ++i) { + Buf_.WriteDouble(s->Bucket(i)); + } + Buf_.EndList(); + + Buf_.EndObject(); + } + + void WriteValue(EMetricValueType type, TMetricValue value) { + switch (type) { + case EMetricValueType::DOUBLE: + WriteValue(value.AsDouble()); + break; + + case EMetricValueType::INT64: + WriteValue(value.AsInt64()); + break; + + case EMetricValueType::UINT64: + WriteValue(value.AsUint64()); + break; + + case EMetricValueType::HISTOGRAM: + WriteValue(value.AsHistogram()); + break; + + case EMetricValueType::SUMMARY: + WriteValue(value.AsSummaryDouble()); + break; + + case EMetricValueType::LOGHISTOGRAM: + WriteValue(value.AsLogHistogram()); + break; + + case EMetricValueType::UNKNOWN: + ythrow yexception() << "unknown metric value type"; + } + } + + void WriteLabel(TStringBuf name, TStringBuf value) { + Y_ENSURE(IsUtf(name), "label name is not valid UTF-8 string"); + Y_ENSURE(IsUtf(value), "label value is not valid UTF-8 string"); + if (Style_ == EJsonStyle::Cloud && name == MetricNameLabel_) { + CurrentMetricName_ = value; + } else { + Buf_.WriteKey(name); + Buf_.WriteString(value); + } + } + + void WriteMetricType(EMetricType type) { + if (Style_ == EJsonStyle::Cloud) { + Buf_.WriteKey("type"); + Buf_.WriteString(MetricTypeToCloudStr(type)); + } else { + Buf_.WriteKey("kind"); + Buf_.WriteString(MetricTypeToStr(type)); + } + } + + void WriteName() { + if (Style_ != EJsonStyle::Cloud) { + return; + } + if (CurrentMetricName_.Empty()) { + ythrow yexception() << "label '" << MetricNameLabel_ << "' is not defined"; + } + Buf_.WriteKey("name"); + Buf_.WriteString(CurrentMetricName_); + CurrentMetricName_.clear(); + } + + private: + static TStringBuf MetricTypeToCloudStr(EMetricType type) { + switch (type) { + case EMetricType::GAUGE: + return TStringBuf("DGAUGE"); + case EMetricType::COUNTER: + return TStringBuf("COUNTER"); + case EMetricType::RATE: + return TStringBuf("RATE"); + case EMetricType::IGAUGE: + return TStringBuf("IGAUGE"); + default: + ythrow yexception() << "metric type '" << type << "' is not supported by cloud json format"; + } + } + + protected: + NJsonWriter::TBuf Buf_; + EJsonStyle Style_; + TString MetricNameLabel_; + TString CurrentMetricName_; + }; + + /////////////////////////////////////////////////////////////////////// + // TEncoderJson + /////////////////////////////////////////////////////////////////////// + class TEncoderJson final: public IMetricEncoder, public TJsonWriter { + public: + TEncoderJson(IOutputStream* out, int indentation, EJsonStyle style, TStringBuf metricNameLabel) + : TJsonWriter{out, indentation, style, metricNameLabel} + { + } + + ~TEncoderJson() override { + Close(); + } + + private: + void OnStreamBegin() override { + State_.Expect(TEncoderState::EState::ROOT); + Buf_.BeginObject(); + } + + void OnStreamEnd() override { + State_.Expect(TEncoderState::EState::ROOT); + if (!Buf_.KeyExpected()) { + // not closed metrics array + Buf_.EndList(); + } + Buf_.EndObject(); + } + + void OnCommonTime(TInstant time) override { + State_.Expect(TEncoderState::EState::ROOT); + WriteTime(time); + } + + void OnMetricBegin(EMetricType type) override { + State_.Switch(TEncoderState::EState::ROOT, TEncoderState::EState::METRIC); + if (Buf_.KeyExpected()) { + // first metric, so open metrics array + Buf_.WriteKey(TStringBuf(Style_ == EJsonStyle::Solomon ? "sensors" : "metrics")); + Buf_.BeginList(); + } + Buf_.BeginObject(); + WriteMetricType(type); + } + + void OnMetricEnd() override { + State_.Switch(TEncoderState::EState::METRIC, TEncoderState::EState::ROOT); + if (!Buf_.KeyExpected()) { + // not closed timeseries array + Buf_.EndList(); + } + + if (!TimeSeries_ && LastPoint_.HasValue()) { + // we have seen only one point between OnMetricBegin() and + // OnMetricEnd() calls + WriteTime(LastPoint_.GetTime()); + WriteValue(LastPoint_.GetValueType(), LastPoint_.GetValue()); + } + Buf_.EndObject(); + + LastPoint_ = {}; + TimeSeries_ = false; + } + + void OnLabelsBegin() override { + if (!Buf_.KeyExpected()) { + // not closed metrics or timeseries array if labels go after values + Buf_.EndList(); + } + if (State_ == TEncoderState::EState::ROOT) { + State_ = TEncoderState::EState::COMMON_LABELS; + Buf_.WriteKey(TStringBuf(Style_ == EJsonStyle::Solomon ? "commonLabels" : "labels")); + } else if (State_ == TEncoderState::EState::METRIC) { + State_ = TEncoderState::EState::METRIC_LABELS; + Buf_.WriteKey(TStringBuf("labels")); + } else { + State_.ThrowInvalid("expected METRIC or ROOT"); + } + Buf_.BeginObject(); + + EmptyLabels_ = true; + } + + void OnLabelsEnd() override { + if (State_ == TEncoderState::EState::METRIC_LABELS) { + State_ = TEncoderState::EState::METRIC; + } else if (State_ == TEncoderState::EState::COMMON_LABELS) { + State_ = TEncoderState::EState::ROOT; + } else { + State_.ThrowInvalid("expected LABELS or COMMON_LABELS"); + } + + Y_ENSURE(!EmptyLabels_, "Labels cannot be empty"); + Buf_.EndObject(); + if (State_ == TEncoderState::EState::METRIC) { + WriteName(); + } + } + + void OnLabel(TStringBuf name, TStringBuf value) override { + if (State_ == TEncoderState::EState::METRIC_LABELS || State_ == TEncoderState::EState::COMMON_LABELS) { + WriteLabel(name, value); + } else { + State_.ThrowInvalid("expected LABELS or COMMON_LABELS"); + } + + EmptyLabels_ = false; + } + + void OnDouble(TInstant time, double value) override { + State_.Expect(TEncoderState::EState::METRIC); + Write<double>(time, value); + } + + void OnInt64(TInstant time, i64 value) override { + State_.Expect(TEncoderState::EState::METRIC); + Write<i64>(time, value); + } + + void OnUint64(TInstant time, ui64 value) override { + State_.Expect(TEncoderState::EState::METRIC); + Write<ui64>(time, value); + } + + void OnHistogram(TInstant time, IHistogramSnapshotPtr snapshot) override { + State_.Expect(TEncoderState::EState::METRIC); + Write<IHistogramSnapshot*>(time, snapshot.Get()); + } + + void OnSummaryDouble(TInstant time, ISummaryDoubleSnapshotPtr snapshot) override { + State_.Expect(TEncoderState::EState::METRIC); + Write<ISummaryDoubleSnapshot*>(time, snapshot.Get()); + } + + void OnLogHistogram(TInstant time, TLogHistogramSnapshotPtr snapshot) override { + State_.Expect(TEncoderState::EState::METRIC); + Write<TLogHistogramSnapshot*>(time, snapshot.Get()); + } + + template <typename T> + void Write(TInstant time, T value) { + State_.Expect(TEncoderState::EState::METRIC); + + if (!LastPoint_.HasValue()) { + LastPoint_ = {time, value}; + } else { + // second point + // TODO: output types + Y_ENSURE(LastPoint_.GetValueType() == TValueType<T>::Type, + "mixed metric value types in one metric"); + + if (!TimeSeries_) { + Buf_.WriteKey(TStringBuf("timeseries")); + Buf_.BeginList(); + Buf_.BeginObject(); + Y_ENSURE(LastPoint_.GetTime() != TInstant::Zero(), + "time cannot be empty or zero in a timeseries point"); + WriteTime(LastPoint_.GetTime()); + WriteValue(LastPoint_.GetValueType(), LastPoint_.GetValue()); + Buf_.EndObject(); + TimeSeries_ = true; + } + + if (TimeSeries_) { + Buf_.BeginObject(); + Y_ENSURE(time != TInstant::Zero(), + "time cannot be empty or zero in a timeseries point"); + + WriteTime(time); + WriteValue(value); + Buf_.EndObject(); + } + } + } + + void Close() override { + LastPoint_ = {}; + } + + private: + TEncoderState State_; + TTypedPoint LastPoint_; + bool TimeSeries_ = false; + bool EmptyLabels_ = false; + }; + + /////////////////////////////////////////////////////////////////////// + // TBufferedJsonEncoder + /////////////////////////////////////////////////////////////////////// + class TBufferedJsonEncoder : public TBufferedEncoderBase, public TJsonWriter { + public: + TBufferedJsonEncoder(IOutputStream* out, int indentation, EJsonStyle style, TStringBuf metricNameLabel) + : TJsonWriter{out, indentation, style, metricNameLabel} + { + MetricsMergingMode_ = EMetricsMergingMode::MERGE_METRICS; + } + + ~TBufferedJsonEncoder() override { + Close(); + } + + void OnLabelsBegin() override { + TBufferedEncoderBase::OnLabelsBegin(); + EmptyLabels_ = true; + } + + void OnLabel(TStringBuf name, TStringBuf value) override { + TBufferedEncoderBase::OnLabel(name, value); + EmptyLabels_ = false; + } + + void OnLabel(ui32 name, ui32 value) override { + TBufferedEncoderBase::OnLabel(name, value); + EmptyLabels_ = false; + } + + void OnLabelsEnd() override { + TBufferedEncoderBase::OnLabelsEnd(); + Y_ENSURE(!EmptyLabels_, "Labels cannot be empty"); + } + + void Close() final { + if (Closed_) { + return; + } + + Closed_ = true; + + LabelValuesPool_.Build(); + LabelNamesPool_.Build(); + + Buf_.BeginObject(); + + WriteTime(CommonTime_); + if (CommonLabels_.size() > 0) { + Buf_.WriteKey(TStringBuf(Style_ == EJsonStyle::Solomon ? "commonLabels": "labels")); + WriteLabels(CommonLabels_, true); + } + + if (Metrics_.size() > 0) { + Buf_.WriteKey(TStringBuf(Style_ == EJsonStyle::Solomon ? "sensors" : "metrics")); + WriteMetrics(); + } + + Buf_.EndObject(); + } + + private: + void WriteMetrics() { + Buf_.BeginList(); + for (auto&& metric : Metrics_) { + WriteMetric(metric); + } + Buf_.EndList(); + } + + void WriteMetric(TMetric& metric) { + Buf_.BeginObject(); + + WriteMetricType(metric.MetricType); + + Buf_.WriteKey(TStringBuf("labels")); + WriteLabels(metric.Labels, false); + + metric.TimeSeries.SortByTs(); + if (metric.TimeSeries.Size() == 1) { + const auto& point = metric.TimeSeries[0]; + WriteTime(point.GetTime()); + WriteValue(metric.TimeSeries.GetValueType(), point.GetValue()); + } else if (metric.TimeSeries.Size() > 1) { + Buf_.WriteKey(TStringBuf("timeseries")); + Buf_.BeginList(); + metric.TimeSeries.ForEach([this](TInstant time, EMetricValueType type, TMetricValue value) { + Buf_.BeginObject(); + // make gcc 6.1 happy https://gcc.gnu.org/bugzilla/show_bug.cgi?id=61636 + this->WriteTime(time); + this->WriteValue(type, value); + Buf_.EndObject(); + }); + + Buf_.EndList(); + } + + Buf_.EndObject(); + } + + void WriteLabels(const TPooledLabels& labels, bool isCommon) { + Buf_.BeginObject(); + + for (auto i = 0u; i < labels.size(); ++i) { + TStringBuf name = LabelNamesPool_.Get(labels[i].Key->Index); + TStringBuf value = LabelValuesPool_.Get(labels[i].Value->Index); + + WriteLabel(name, value); + } + + Buf_.EndObject(); + + if (!isCommon) { + WriteName(); + } + } + + private: + bool Closed_{false}; + bool EmptyLabels_ = false; + }; + } + + IMetricEncoderPtr EncoderJson(IOutputStream* out, int indentation) { + return MakeHolder<TEncoderJson>(out, indentation, EJsonStyle::Solomon, ""); + } + + IMetricEncoderPtr BufferedEncoderJson(IOutputStream* out, int indentation) { + return MakeHolder<TBufferedJsonEncoder>(out, indentation, EJsonStyle::Solomon, ""); + } + + IMetricEncoderPtr EncoderCloudJson(IOutputStream* out, int indentation, TStringBuf metricNameLabel) { + return MakeHolder<TEncoderJson>(out, indentation, EJsonStyle::Cloud, metricNameLabel); + } + + IMetricEncoderPtr BufferedEncoderCloudJson(IOutputStream* out, int indentation, TStringBuf metricNameLabel) { + return MakeHolder<TBufferedJsonEncoder>(out, indentation, EJsonStyle::Cloud, metricNameLabel); + } +} diff --git a/library/cpp/monlib/encode/json/json_ut.cpp b/library/cpp/monlib/encode/json/json_ut.cpp new file mode 100644 index 0000000000..09e7909289 --- /dev/null +++ b/library/cpp/monlib/encode/json/json_ut.cpp @@ -0,0 +1,1290 @@ +#include "json.h" + +#include <library/cpp/monlib/encode/protobuf/protobuf.h> +#include <library/cpp/monlib/metrics/labels.h> + +#include <library/cpp/json/json_reader.h> +#include <library/cpp/resource/resource.h> +#include <library/cpp/testing/unittest/registar.h> + +#include <util/stream/str.h> +#include <util/string/builder.h> + +#include <limits> + +using namespace NMonitoring; + +namespace NMonitoring { + bool operator<(const TLabel& lhs, const TLabel& rhs) { + return lhs.Name() < rhs.Name() || + (lhs.Name() == rhs.Name() && lhs.Value() < rhs.Value()); + } +} +namespace { + void AssertLabels(const NProto::TMultiSample& actual, const TLabels& expected) { + UNIT_ASSERT_EQUAL(actual.LabelsSize(), expected.Size()); + + TSet<TLabel> actualSet; + TSet<TLabel> expectedSet; + Transform(expected.begin(), expected.end(), std::inserter(expectedSet, expectedSet.end()), [] (auto&& l) { + return TLabel{l.Name(), l.Value()}; + }); + + const auto& l = actual.GetLabels(); + Transform(std::begin(l), std::end(l), std::inserter(actualSet, std::begin(actualSet)), + [](auto&& elem) -> TLabel { + return {elem.GetName(), elem.GetValue()}; + }); + + TVector<TLabel> diff; + SetSymmetricDifference(std::begin(expectedSet), std::end(expectedSet), + std::begin(actualSet), std::end(actualSet), std::back_inserter(diff)); + + if (diff.size() > 0) { + for (auto&& l : diff) { + Cerr << l << Endl; + } + + UNIT_FAIL("Labels don't match"); + } + } + + void AssertLabelEqual(const NProto::TLabel& l, TStringBuf name, TStringBuf value) { + UNIT_ASSERT_STRINGS_EQUAL(l.GetName(), name); + UNIT_ASSERT_STRINGS_EQUAL(l.GetValue(), value); + } + + void AssertPointEqual(const NProto::TPoint& p, TInstant time, double value) { + UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds()); + UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kFloat64); + UNIT_ASSERT_DOUBLES_EQUAL(p.GetFloat64(), value, std::numeric_limits<double>::epsilon()); + } + + void AssertPointEqualNan(const NProto::TPoint& p, TInstant time) { + UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds()); + UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kFloat64); + UNIT_ASSERT(std::isnan(p.GetFloat64())); + } + + void AssertPointEqualInf(const NProto::TPoint& p, TInstant time, int sign) { + UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds()); + UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kFloat64); + UNIT_ASSERT(std::isinf(p.GetFloat64())); + if (sign < 0) { + UNIT_ASSERT(p.GetFloat64() < 0); + } + } + + void AssertPointEqual(const NProto::TPoint& p, TInstant time, ui64 value) { + UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds()); + UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kUint64); + UNIT_ASSERT_VALUES_EQUAL(p.GetUint64(), value); + } + + void AssertPointEqual(const NProto::TPoint& p, TInstant time, i64 value) { + UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds()); + UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kInt64); + UNIT_ASSERT_VALUES_EQUAL(p.GetInt64(), value); + } + + void AssertPointEqual(const NProto::TPoint& p, TInstant time, const IHistogramSnapshot& expected) { + UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds()); + UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kHistogram); + + const NProto::THistogram& h = p.GetHistogram(); + UNIT_ASSERT_VALUES_EQUAL(h.BoundsSize(), expected.Count()); + UNIT_ASSERT_VALUES_EQUAL(h.ValuesSize(), expected.Count()); + + for (size_t i = 0; i < h.BoundsSize(); i++) { + UNIT_ASSERT_DOUBLES_EQUAL(h.GetBounds(i), expected.UpperBound(i), Min<double>()); + UNIT_ASSERT_VALUES_EQUAL(h.GetValues(i), expected.Value(i)); + } + } + + void AssertPointEqual(const NProto::TPoint& p, TInstant time, const TLogHistogramSnapshot& expected) { + UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds()); + UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kLogHistogram); + + const double eps = 1e-10; + const NProto::TLogHistogram& h = p.GetLogHistogram(); + + UNIT_ASSERT_DOUBLES_EQUAL(h.GetBase(), expected.Base(), eps); + UNIT_ASSERT_VALUES_EQUAL(h.GetZerosCount(), expected.ZerosCount()); + UNIT_ASSERT_VALUES_EQUAL(h.GetStartPower(), expected.StartPower()); + UNIT_ASSERT_VALUES_EQUAL(h.BucketsSize(), expected.Count()); + for (size_t i = 0; i < expected.Count(); ++i) { + UNIT_ASSERT_DOUBLES_EQUAL(h.GetBuckets(i), expected.Bucket(i), eps); + } + } + + void AssertPointEqual(const NProto::TPoint& p, TInstant time, const ISummaryDoubleSnapshot& expected) { + UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds()); + UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kSummaryDouble); + auto actual = p.GetSummaryDouble(); + const double eps = 1e-10; + UNIT_ASSERT_DOUBLES_EQUAL(actual.GetSum(), expected.GetSum(), eps); + UNIT_ASSERT_DOUBLES_EQUAL(actual.GetMin(), expected.GetMin(), eps); + UNIT_ASSERT_DOUBLES_EQUAL(actual.GetMax(), expected.GetMax(), eps); + UNIT_ASSERT_DOUBLES_EQUAL(actual.GetLast(), expected.GetLast(), eps); + UNIT_ASSERT_VALUES_EQUAL(actual.GetCount(), expected.GetCount()); + } + +} // namespace + + +Y_UNIT_TEST_SUITE(TJsonTest) { + const TInstant now = TInstant::ParseIso8601Deprecated("2017-11-05T01:02:03Z"); + + Y_UNIT_TEST(Encode) { + auto check = [](bool cloud, bool buffered, TStringBuf expectedResourceKey) { + TString json; + TStringOutput out(json); + auto e = cloud + ? (buffered ? BufferedEncoderCloudJson(&out, 2, "metric") : EncoderCloudJson(&out, 2, "metric")) + : (buffered ? BufferedEncoderJson(&out, 2) : EncoderJson(&out, 2)); + e->OnStreamBegin(); + { // common time + e->OnCommonTime(TInstant::Seconds(1500000000)); + } + { // common labels + e->OnLabelsBegin(); + e->OnLabel("project", "solomon"); + e->OnLabelsEnd(); + } + { // metric #1 + e->OnMetricBegin(EMetricType::COUNTER); + { + e->OnLabelsBegin(); + e->OnLabel("metric", "single"); + e->OnLabel("labels", "l1"); + e->OnLabelsEnd(); + } + e->OnUint64(now, 17); + e->OnMetricEnd(); + } + { // metric #2 + e->OnMetricBegin(EMetricType::RATE); + { + e->OnLabelsBegin(); + e->OnLabel("metric", "single"); + e->OnLabel("labels", "l2"); + e->OnLabelsEnd(); + } + e->OnUint64(now, 17); + e->OnMetricEnd(); + } + { // metric #3 + e->OnMetricBegin(EMetricType::GAUGE); + e->OnDouble(now, 3.14); + { + e->OnLabelsBegin(); + e->OnLabel("metric", "single"); + e->OnLabel("labels", "l3"); + e->OnLabelsEnd(); + } + e->OnMetricEnd(); + } + { // metric #4 + e->OnMetricBegin(EMetricType::IGAUGE); + e->OnInt64(now, 42); + { + e->OnLabelsBegin(); + e->OnLabel("metric", "single_igauge"); + e->OnLabel("labels", "l4"); + e->OnLabelsEnd(); + } + e->OnMetricEnd(); + } + { // metric #5 + e->OnMetricBegin(EMetricType::GAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("metric", "multiple"); + e->OnLabel("labels", "l5"); + e->OnLabelsEnd(); + } + e->OnDouble(now, std::numeric_limits<double>::quiet_NaN()); + e->OnDouble(now + TDuration::Seconds(15), std::numeric_limits<double>::infinity()); + e->OnDouble(now + TDuration::Seconds(30), -std::numeric_limits<double>::infinity()); + e->OnMetricEnd(); + } + + { // metric #6 + e->OnMetricBegin(EMetricType::COUNTER); + e->OnUint64(now, 1337); + e->OnUint64(now + TDuration::Seconds(15), 1338); + { + e->OnLabelsBegin(); + e->OnLabel("metric", "multiple"); + e->OnLabel("labels", "l6"); + e->OnLabelsEnd(); + } + e->OnMetricEnd(); + } + e->OnStreamEnd(); + e->Close(); + json += "\n"; + + auto parseJson = [] (auto buf) { + NJson::TJsonValue value; + NJson::ReadJsonTree(buf, &value, true); + return value; + }; + + const auto expectedJson = NResource::Find(expectedResourceKey); + UNIT_ASSERT_EQUAL(parseJson(json), parseJson(expectedJson)); + }; + + check(false, false, "/expected.json"); + check(false, true, "/expected_buffered.json"); + check(true, false, "/expected_cloud.json"); + check(true, true, "/expected_cloud_buffered.json"); + } + + TLogHistogramSnapshotPtr TestLogHistogram(ui32 v = 1) { + TVector<double> buckets{0.5 * v, 0.25 * v, 0.25 * v, 0.5 * v}; + return MakeIntrusive<TLogHistogramSnapshot>(1.5, 1u, 0, std::move(buckets)); + } + + Y_UNIT_TEST(HistogramAndSummaryMetricTypesAreNotSupportedByCloudJson) { + const TInstant now = TInstant::ParseIso8601Deprecated("2017-11-05T01:02:03Z"); + + auto emit = [&](IMetricEncoder* encoder, EMetricType metricType) { + encoder->OnStreamBegin(); + { + encoder->OnMetricBegin(metricType); + { + encoder->OnLabelsBegin(); + encoder->OnLabel("name", "m"); + encoder->OnLabelsEnd(); + } + + switch (metricType) { + case EMetricType::HIST: { + auto histogram = ExponentialHistogram(6, 2); + encoder->OnHistogram(now, histogram->Snapshot()); + break; + } + case EMetricType::LOGHIST: { + auto histogram = TestLogHistogram(); + encoder->OnLogHistogram(now, histogram); + break; + } + case EMetricType::DSUMMARY: { + auto summary = MakeIntrusive<TSummaryDoubleSnapshot>(10., -0.5, 0.5, 0.3, 30u); + encoder->OnSummaryDouble(now, summary); + break; + } + default: + Y_FAIL("unexpected metric type [%s]", ToString(metricType).c_str()); + } + + encoder->OnMetricEnd(); + } + encoder->OnStreamEnd(); + encoder->Close(); + }; + + auto doTest = [&](bool buffered, EMetricType metricType) { + TString json; + TStringOutput out(json); + auto encoder = buffered ? BufferedEncoderCloudJson(&out, 2) : EncoderCloudJson(&out, 2); + const TString expectedMessage = TStringBuilder() + << "metric type '" << metricType << "' is not supported by cloud json format"; + UNIT_ASSERT_EXCEPTION_CONTAINS_C(emit(encoder.Get(), metricType), yexception, expectedMessage, + TString("buffered: ") + ToString(buffered)); + }; + + doTest(false, EMetricType::HIST); + doTest(false, EMetricType::LOGHIST); + doTest(false, EMetricType::DSUMMARY); + doTest(true, EMetricType::HIST); + doTest(true, EMetricType::LOGHIST); + doTest(true, EMetricType::DSUMMARY); + } + + Y_UNIT_TEST(MetricsWithDifferentLabelOrderGetMerged) { + TString json; + TStringOutput out(json); + auto e = BufferedEncoderJson(&out, 2); + + e->OnStreamBegin(); + { + e->OnMetricBegin(EMetricType::RATE); + { + e->OnLabelsBegin(); + e->OnLabel("metric", "hello"); + e->OnLabel("label", "world"); + e->OnLabelsEnd(); + } + e->OnUint64(TInstant::Zero(), 0); + e->OnMetricEnd(); + } + { + e->OnMetricBegin(EMetricType::RATE); + { + e->OnLabelsBegin(); + e->OnLabel("label", "world"); + e->OnLabel("metric", "hello"); + e->OnLabelsEnd(); + } + e->OnUint64(TInstant::Zero(), 1); + e->OnMetricEnd(); + } + e->OnStreamEnd(); + e->Close(); + json += "\n"; + + TString expectedJson = NResource::Find("/merged.json"); + // we cannot be sure regarding the label order in the result, + // so we'll have to parse the expected value and then compare it with actual + + NProto::TMultiSamplesList samples; + IMetricEncoderPtr d = EncoderProtobuf(&samples); + DecodeJson(expectedJson, d.Get()); + + UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 1); + { + const NProto::TMultiSample& s = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::RATE); + AssertLabels(s, TLabels{{"metric", "hello"}, {"label", "world"}}); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + AssertPointEqual(s.GetPoints(0), TInstant::Zero(), ui64(1)); + } + } + Y_UNIT_TEST(Decode1) { + NProto::TMultiSamplesList samples; + { + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + TString testJson = NResource::Find("/expected.json"); + DecodeJson(testJson, e.Get()); + } + + UNIT_ASSERT_VALUES_EQUAL( + TInstant::MilliSeconds(samples.GetCommonTime()), + TInstant::Seconds(1500000000)); + + UNIT_ASSERT_VALUES_EQUAL(samples.CommonLabelsSize(), 1); + AssertLabelEqual(samples.GetCommonLabels(0), "project", "solomon"); + + UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 6); + { + const NProto::TMultiSample& s = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::COUNTER); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2); + AssertLabelEqual(s.GetLabels(0), "metric", "single"); + AssertLabelEqual(s.GetLabels(1), "labels", "l1"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + AssertPointEqual(s.GetPoints(0), now, ui64(17)); + } + { + const NProto::TMultiSample& s = samples.GetSamples(1); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::RATE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2); + AssertLabelEqual(s.GetLabels(0), "metric", "single"); + AssertLabelEqual(s.GetLabels(1), "labels", "l2"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + AssertPointEqual(s.GetPoints(0), now, ui64(17)); + } + { + const NProto::TMultiSample& s = samples.GetSamples(2); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2); + AssertLabelEqual(s.GetLabels(0), "metric", "single"); + AssertLabelEqual(s.GetLabels(1), "labels", "l3"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + AssertPointEqual(s.GetPoints(0), now, 3.14); + } + { + const NProto::TMultiSample& s = samples.GetSamples(3); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::IGAUGE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2); + AssertLabelEqual(s.GetLabels(0), "metric", "single_igauge"); + AssertLabelEqual(s.GetLabels(1), "labels", "l4"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + AssertPointEqual(s.GetPoints(0), now, i64(42)); + } + { + const NProto::TMultiSample& s = samples.GetSamples(4); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2); + AssertLabelEqual(s.GetLabels(0), "metric", "multiple"); + AssertLabelEqual(s.GetLabels(1), "labels", "l5"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 3); + AssertPointEqualNan(s.GetPoints(0), now); + AssertPointEqualInf(s.GetPoints(1), now + TDuration::Seconds(15), 1); + AssertPointEqualInf(s.GetPoints(2), now + TDuration::Seconds(30), -11); + } + { + const NProto::TMultiSample& s = samples.GetSamples(5); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::COUNTER); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2); + AssertLabelEqual(s.GetLabels(0), "metric", "multiple"); + AssertLabelEqual(s.GetLabels(1), "labels", "l6"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 2); + AssertPointEqual(s.GetPoints(0), now, ui64(1337)); + AssertPointEqual(s.GetPoints(1), now + TDuration::Seconds(15), ui64(1338)); + } + } + + Y_UNIT_TEST(DecodeMetrics) { + NProto::TMultiSamplesList samples; + { + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + TString metricsJson = NResource::Find("/metrics.json"); + DecodeJson(metricsJson, e.Get()); + } + + UNIT_ASSERT_VALUES_EQUAL( + TInstant::MilliSeconds(samples.GetCommonTime()), + TInstant::ParseIso8601Deprecated("2017-08-27T12:34:56Z")); + + UNIT_ASSERT_VALUES_EQUAL(samples.CommonLabelsSize(), 3); + AssertLabelEqual(samples.GetCommonLabels(0), "project", "solomon"); + AssertLabelEqual(samples.GetCommonLabels(1), "cluster", "man"); + AssertLabelEqual(samples.GetCommonLabels(2), "service", "stockpile"); + + UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 4); + { + const NProto::TMultiSample& s = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "metric", "Memory"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + AssertPointEqual(s.GetPoints(0), TInstant::Zero(), 10.0); + } + { + const NProto::TMultiSample& s = samples.GetSamples(1); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::RATE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "metric", "UserTime"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + AssertPointEqual(s.GetPoints(0), TInstant::Zero(), ui64(1)); + } + { + const NProto::TMultiSample& s = samples.GetSamples(2); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2); + AssertLabelEqual(s.GetLabels(0), "export", "Oxygen"); + AssertLabelEqual(s.GetLabels(1), "metric", "QueueSize"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + auto ts = TInstant::ParseIso8601Deprecated("2017-11-05T12:34:56.000Z"); + AssertPointEqual(s.GetPoints(0), ts, 3.14159); + } + { + const NProto::TMultiSample& s = samples.GetSamples(3); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "metric", "Writes"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 2); + auto ts1 = TInstant::ParseIso8601Deprecated("2017-08-28T12:32:11Z"); + AssertPointEqual(s.GetPoints(0), ts1, -10.0); + auto ts2 = TInstant::Seconds(1503923187); + AssertPointEqual(s.GetPoints(1), ts2, 20.0); + } + } + + Y_UNIT_TEST(DecodeSensors) { + NProto::TMultiSamplesList samples; + { + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + TString sensorsJson = NResource::Find("/sensors.json"); + DecodeJson(sensorsJson, e.Get()); + } + + UNIT_ASSERT_VALUES_EQUAL( + TInstant::MilliSeconds(samples.GetCommonTime()), + TInstant::ParseIso8601Deprecated("2017-08-27T12:34:56Z")); + + UNIT_ASSERT_VALUES_EQUAL(samples.CommonLabelsSize(), 3); + AssertLabelEqual(samples.GetCommonLabels(0), "project", "solomon"); + AssertLabelEqual(samples.GetCommonLabels(1), "cluster", "man"); + AssertLabelEqual(samples.GetCommonLabels(2), "service", "stockpile"); + + UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 4); + { + const NProto::TMultiSample& s = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "metric", "Memory"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + AssertPointEqual(s.GetPoints(0), TInstant::Zero(), 10.0); + } + { + const NProto::TMultiSample& s = samples.GetSamples(1); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::RATE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "metric", "UserTime"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + AssertPointEqual(s.GetPoints(0), TInstant::Zero(), ui64(1)); + } + { + const NProto::TMultiSample& s = samples.GetSamples(2); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2); + AssertLabelEqual(s.GetLabels(0), "export", "Oxygen"); + AssertLabelEqual(s.GetLabels(1), "metric", "QueueSize"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + auto ts = TInstant::ParseIso8601Deprecated("2017-11-05T12:34:56.000Z"); + AssertPointEqual(s.GetPoints(0), ts, 3.14159); + } + { + const NProto::TMultiSample& s = samples.GetSamples(3); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "metric", "Writes"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 2); + auto ts1 = TInstant::ParseIso8601Deprecated("2017-08-28T12:32:11Z"); + AssertPointEqual(s.GetPoints(0), ts1, -10.0); + auto ts2 = TInstant::Seconds(1503923187); + AssertPointEqual(s.GetPoints(1), ts2, 20.0); + } + } + + Y_UNIT_TEST(DecodeToEncoder) { + auto testJson = NResource::Find("/test_decode_to_encode.json"); + + TStringStream Stream_; + auto encoder = BufferedEncoderJson(&Stream_, 4); + DecodeJson(testJson, encoder.Get()); + + encoder->Close(); + + auto val1 = NJson::ReadJsonFastTree(testJson, true); + auto val2 = NJson::ReadJsonFastTree(Stream_.Str(), true); + + UNIT_ASSERT_VALUES_EQUAL(val1, val2); + } + + void WriteEmptySeries(const IMetricEncoderPtr& e) { + e->OnStreamBegin(); + { + e->OnMetricBegin(EMetricType::COUNTER); + { + e->OnLabelsBegin(); + e->OnLabel("foo", "bar"); + e->OnLabelsEnd(); + } + e->OnMetricEnd(); + } + + e->OnStreamEnd(); + e->Close(); + } + + Y_UNIT_TEST(EncodeEmptySeries) { + TString json; + TStringOutput out(json); + + auto e = EncoderJson(&out, 2); + WriteEmptySeries(e); + json += "\n"; + + TString expectedJson = NResource::Find("/empty_series.json"); + UNIT_ASSERT_NO_DIFF(json, expectedJson); + } + + void WriteEmptyLabels(IMetricEncoderPtr& e) { + e->OnStreamBegin(); + e->OnMetricBegin(EMetricType::COUNTER); + + e->OnLabelsBegin(); + UNIT_ASSERT_EXCEPTION(e->OnLabelsEnd(), yexception); + } + + Y_UNIT_TEST(LabelsCannotBeEmpty) { + TString json; + TStringOutput out(json); + + auto e = EncoderJson(&out, 2); + WriteEmptyLabels(e); + } + + Y_UNIT_TEST(LabelsCannotBeEmptyBuffered) { + TString json; + TStringOutput out(json); + + auto e = BufferedEncoderJson(&out, 2); + WriteEmptyLabels(e); + } + + Y_UNIT_TEST(EncodeEmptySeriesBuffered) { + TString json; + TStringOutput out(json); + + auto e = BufferedEncoderJson(&out, 2); + WriteEmptySeries(e); + json += "\n"; + + TString expectedJson = NResource::Find("/empty_series.json"); + UNIT_ASSERT_NO_DIFF(json, expectedJson); + } + + Y_UNIT_TEST(BufferedEncoderMergesMetrics) { + TString json; + TStringOutput out(json); + + auto e = BufferedEncoderJson(&out, 2); + auto ts = 1; + + auto writeMetric = [&] (const TString& val) { + e->OnMetricBegin(EMetricType::COUNTER); + + e->OnLabelsBegin(); + e->OnLabel("foo", val); + e->OnLabelsEnd(); + e->OnUint64(TInstant::Seconds(ts++), 42); + + e->OnMetricEnd(); + }; + + e->OnStreamBegin(); + writeMetric("bar"); + writeMetric("bar"); + writeMetric("baz"); + writeMetric("bar"); + e->OnStreamEnd(); + e->Close(); + + json += "\n"; + + TString expectedJson = NResource::Find("/buffered_test.json"); + UNIT_ASSERT_NO_DIFF(json, expectedJson); + } + + Y_UNIT_TEST(JsonEncoderDisallowsValuesInTimeseriesWithoutTs) { + TStringStream out; + + auto e = EncoderJson(&out); + auto writePreamble = [&] { + e->OnStreamBegin(); + e->OnMetricBegin(EMetricType::COUNTER); + e->OnLabelsBegin(); + e->OnLabel("foo", "bar"); + e->OnLabelsEnd(); + }; + + // writing two values for a metric in a row will trigger + // timeseries object construction + writePreamble(); + e->OnUint64(TInstant::Zero(), 42); + UNIT_ASSERT_EXCEPTION(e->OnUint64(TInstant::Zero(), 42), yexception); + + e = EncoderJson(&out); + writePreamble(); + e->OnUint64(TInstant::Zero(), 42); + UNIT_ASSERT_EXCEPTION(e->OnUint64(TInstant::Now(), 42), yexception); + + e = EncoderJson(&out); + writePreamble(); + e->OnUint64(TInstant::Now(), 42); + UNIT_ASSERT_EXCEPTION(e->OnUint64(TInstant::Zero(), 42), yexception); + } + + Y_UNIT_TEST(BufferedJsonEncoderMergesTimeseriesWithoutTs) { + TStringStream out; + + { + auto e = BufferedEncoderJson(&out, 2); + e->OnStreamBegin(); + e->OnMetricBegin(EMetricType::COUNTER); + e->OnLabelsBegin(); + e->OnLabel("foo", "bar"); + e->OnLabelsEnd(); + // in buffered mode we are able to find values with same (in this case zero) + // timestamp and discard duplicates + e->OnUint64(TInstant::Zero(), 42); + e->OnUint64(TInstant::Zero(), 43); + e->OnUint64(TInstant::Zero(), 44); + e->OnUint64(TInstant::Zero(), 45); + e->OnMetricEnd(); + e->OnStreamEnd(); + } + + out << "\n"; + UNIT_ASSERT_NO_DIFF(out.Str(), NResource::Find("/buffered_ts_merge.json")); + } + + template <typename TFactory, typename TConsumer> + TString EncodeToString(TFactory factory, TConsumer consumer) { + TStringStream out; + { + IMetricEncoderPtr e = factory(&out, 2); + consumer(e.Get()); + } + out << '\n'; + return out.Str(); + } + + Y_UNIT_TEST(SummaryValueEncode) { + auto writeDocument = [](IMetricEncoder* e) { + e->OnStreamBegin(); + { + e->OnMetricBegin(EMetricType::DSUMMARY); + { + e->OnLabelsBegin(); + e->OnLabel("metric", "temperature"); + e->OnLabelsEnd(); + } + + e->OnSummaryDouble(now, MakeIntrusive<TSummaryDoubleSnapshot>(10., -0.5, 0.5, 0.3, 30u)); + e->OnMetricEnd(); + } + e->OnStreamEnd(); + }; + + TString result1 = EncodeToString(EncoderJson, writeDocument); + UNIT_ASSERT_NO_DIFF(result1, NResource::Find("/summary_value.json")); + + TString result2 = EncodeToString(BufferedEncoderJson, writeDocument); + UNIT_ASSERT_NO_DIFF(result2, NResource::Find("/summary_value.json")); + } + + ISummaryDoubleSnapshotPtr TestInfSummary() { + return MakeIntrusive<TSummaryDoubleSnapshot>( + std::numeric_limits<double>::quiet_NaN(), + -std::numeric_limits<double>::infinity(), + std::numeric_limits<double>::infinity(), + 0.3, + 30u); + } + + Y_UNIT_TEST(SummaryInfEncode) { + auto writeDocument = [](IMetricEncoder* e) { + e->OnStreamBegin(); + { + e->OnMetricBegin(EMetricType::DSUMMARY); + { + e->OnLabelsBegin(); + e->OnLabel("metric", "temperature"); + e->OnLabelsEnd(); + } + + e->OnSummaryDouble(now, TestInfSummary()); + e->OnMetricEnd(); + } + e->OnStreamEnd(); + }; + + TString result1 = EncodeToString(EncoderJson, writeDocument); + UNIT_ASSERT_NO_DIFF(result1, NResource::Find("/summary_inf.json")); + + TString result2 = EncodeToString(BufferedEncoderJson, writeDocument); + UNIT_ASSERT_NO_DIFF(result2, NResource::Find("/summary_inf.json")); + } + + Y_UNIT_TEST(SummaryInfDecode) { + NProto::TMultiSamplesList samples; + { + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + TString testJson = NResource::Find("/summary_inf.json"); + DecodeJson(testJson, e.Get()); + } + + UNIT_ASSERT_VALUES_EQUAL(1, samples.SamplesSize()); + const NProto::TMultiSample& s = samples.GetSamples(0); + + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::DSUMMARY); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "metric", "temperature"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + + auto actual = s.GetPoints(0).GetSummaryDouble(); + UNIT_ASSERT(std::isnan(actual.GetSum())); + UNIT_ASSERT(actual.GetMin() < 0); + UNIT_ASSERT(std::isinf(actual.GetMin())); + UNIT_ASSERT(actual.GetMax() > 0); + UNIT_ASSERT(std::isinf(actual.GetMax())); + } + + Y_UNIT_TEST(SummaryValueDecode) { + NProto::TMultiSamplesList samples; + { + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + TString testJson = NResource::Find("/summary_value.json"); + DecodeJson(testJson, e.Get()); + } + + UNIT_ASSERT_VALUES_EQUAL(1, samples.SamplesSize()); + const NProto::TMultiSample& s = samples.GetSamples(0); + + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::DSUMMARY); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "metric", "temperature"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + + auto snapshot = TSummaryDoubleSnapshot(10., -0.5, 0.5, 0.3, 30u); + AssertPointEqual(s.GetPoints(0), now, snapshot); + } + + Y_UNIT_TEST(SummaryTimeSeriesEncode) { + auto writeDocument = [](IMetricEncoder* e) { + e->OnStreamBegin(); + { + e->OnMetricBegin(EMetricType::DSUMMARY); + { + e->OnLabelsBegin(); + e->OnLabel("metric", "temperature"); + e->OnLabelsEnd(); + } + + TSummaryDoubleCollector summary; + summary.Collect(0.3); + summary.Collect(-0.5); + summary.Collect(1.); + + e->OnSummaryDouble(now, summary.Snapshot()); + + summary.Collect(-1.5); + summary.Collect(0.01); + + e->OnSummaryDouble(now + TDuration::Seconds(15), summary.Snapshot()); + + e->OnMetricEnd(); + } + e->OnStreamEnd(); + }; + + TString result1 = EncodeToString(EncoderJson, writeDocument); + UNIT_ASSERT_NO_DIFF(result1, NResource::Find("/summary_timeseries.json")); + + TString result2 = EncodeToString(BufferedEncoderJson, writeDocument); + UNIT_ASSERT_NO_DIFF(result2, NResource::Find("/summary_timeseries.json")); + } + + Y_UNIT_TEST(SummaryTimeSeriesDecode) { + NProto::TMultiSamplesList samples; + { + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + TString testJson = NResource::Find("/summary_timeseries.json"); + DecodeJson(testJson, e.Get()); + } + + UNIT_ASSERT_VALUES_EQUAL(1, samples.SamplesSize()); + const NProto::TMultiSample& s = samples.GetSamples(0); + + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::DSUMMARY); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "metric", "temperature"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 2); + + TSummaryDoubleCollector summary; + summary.Collect(0.3); + summary.Collect(-0.5); + summary.Collect(1.); + + AssertPointEqual(s.GetPoints(0), now, *summary.Snapshot()); + + summary.Collect(-1.5); + summary.Collect(0.01); + + AssertPointEqual(s.GetPoints(1), now + TDuration::Seconds(15), *summary.Snapshot()); + } + + Y_UNIT_TEST(LogHistogramValueEncode) { + auto writeDocument = [](IMetricEncoder* e) { + e->OnStreamBegin(); + { + e->OnMetricBegin(EMetricType::LOGHIST); + { + e->OnLabelsBegin(); + e->OnLabel("metric", "ms"); + e->OnLabelsEnd(); + } + + e->OnLogHistogram(now, TestLogHistogram()); + e->OnMetricEnd(); + } + e->OnStreamEnd(); + }; + + TString result1 = EncodeToString(EncoderJson, writeDocument); + UNIT_ASSERT_NO_DIFF(result1, NResource::Find("/log_histogram_value.json")); + + TString result2 = EncodeToString(BufferedEncoderJson, writeDocument); + UNIT_ASSERT_NO_DIFF(result2, NResource::Find("/log_histogram_value.json")); + } + + Y_UNIT_TEST(LogHistogramValueDecode) { + NProto::TMultiSamplesList samples; + { + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + TString testJson = NResource::Find("/log_histogram_value.json"); + DecodeJson(testJson, e.Get()); + } + + UNIT_ASSERT_VALUES_EQUAL(1, samples.SamplesSize()); + const NProto::TMultiSample& s = samples.GetSamples(0); + + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::LOGHISTOGRAM); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "metric", "ms"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + + auto snapshot = TestLogHistogram(); + AssertPointEqual(s.GetPoints(0), now, *snapshot); + } + + Y_UNIT_TEST(HistogramValueEncode) { + auto writeDocument = [](IMetricEncoder* e) { + e->OnStreamBegin(); + { + e->OnMetricBegin(EMetricType::HIST); + { + e->OnLabelsBegin(); + e->OnLabel("metric", "responseTimeMillis"); + e->OnLabelsEnd(); + } + + // {1: 1, 2: 1, 4: 2, 8: 4, 16: 8, inf: 83} + auto h = ExponentialHistogram(6, 2); + for (i64 i = 1; i < 100; i++) { + h->Collect(i); + } + + e->OnHistogram(now, h->Snapshot()); + e->OnMetricEnd(); + } + e->OnStreamEnd(); + }; + + TString result1 = EncodeToString(EncoderJson, writeDocument); + UNIT_ASSERT_NO_DIFF(result1, NResource::Find("/histogram_value.json")); + + TString result2 = EncodeToString(BufferedEncoderJson, writeDocument); + UNIT_ASSERT_NO_DIFF(result2, NResource::Find("/histogram_value.json")); + } + + Y_UNIT_TEST(LogHistogramTimeSeriesEncode) { + auto writeDocument = [](IMetricEncoder* e) { + e->OnStreamBegin(); + { + e->OnMetricBegin(EMetricType::LOGHIST); + { + e->OnLabelsBegin(); + e->OnLabel("metric", "ms"); + e->OnLabelsEnd(); + } + + e->OnLogHistogram(now, TestLogHistogram(1));; + + e->OnLogHistogram(now + TDuration::Seconds(15), TestLogHistogram(2)); + + e->OnMetricEnd(); + } + e->OnStreamEnd(); + }; + + TString result1 = EncodeToString(EncoderJson, writeDocument); + UNIT_ASSERT_NO_DIFF(result1, NResource::Find("/log_histogram_timeseries.json")); + + TString result2 = EncodeToString(BufferedEncoderJson, writeDocument); + UNIT_ASSERT_NO_DIFF(result2, NResource::Find("/log_histogram_timeseries.json")); + } + + Y_UNIT_TEST(LogHistogramTimeSeriesDecode) { + NProto::TMultiSamplesList samples; + { + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + TString testJson = NResource::Find("/log_histogram_timeseries.json"); + DecodeJson(testJson, e.Get()); + } + + UNIT_ASSERT_VALUES_EQUAL(1, samples.SamplesSize()); + const NProto::TMultiSample& s = samples.GetSamples(0); + + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::LOGHISTOGRAM); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "metric", "ms"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 2); + + auto logHist = TestLogHistogram(1); + AssertPointEqual(s.GetPoints(0), now, *logHist); + + logHist = TestLogHistogram(2); + AssertPointEqual(s.GetPoints(1), now + TDuration::Seconds(15), *logHist); + } + + void HistogramValueDecode(const TString& filePath) { + NProto::TMultiSamplesList samples; + { + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + TString testJson = NResource::Find(filePath); + DecodeJson(testJson, e.Get()); + } + + UNIT_ASSERT_VALUES_EQUAL(1, samples.SamplesSize()); + const NProto::TMultiSample& s = samples.GetSamples(0); + + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::HISTOGRAM); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "metric", "responseTimeMillis"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + + auto h = ExponentialHistogram(6, 2); + for (i64 i = 1; i < 100; i++) { + h->Collect(i); + } + + AssertPointEqual(s.GetPoints(0), now, *h->Snapshot()); + } + + Y_UNIT_TEST(HistogramValueDecode) { + HistogramValueDecode("/histogram_value.json"); + HistogramValueDecode("/histogram_value_inf_before_bounds.json"); + } + + Y_UNIT_TEST(HistogramTimeSeriesEncode) { + auto writeDocument = [](IMetricEncoder* e) { + e->OnStreamBegin(); + { + e->OnMetricBegin(EMetricType::HIST_RATE); + { + e->OnLabelsBegin(); + e->OnLabel("metric", "responseTimeMillis"); + e->OnLabelsEnd(); + } + + // {1: 1, 2: 1, 4: 2, 8: 4, 16: 8, inf: 83} + auto h = ExponentialHistogram(6, 2); + for (i64 i = 1; i < 100; i++) { + h->Collect(i); + } + e->OnHistogram(now, h->Snapshot()); + + // {1: 2, 2: 2, 4: 4, 8: 8, 16: 16, inf: 166} + for (i64 i = 1; i < 100; i++) { + h->Collect(i); + } + e->OnHistogram(now + TDuration::Seconds(15), h->Snapshot()); + + e->OnMetricEnd(); + } + e->OnStreamEnd(); + }; + + TString result1 = EncodeToString(EncoderJson, writeDocument); + UNIT_ASSERT_NO_DIFF(result1, NResource::Find("/histogram_timeseries.json")); + + TString result2 = EncodeToString(BufferedEncoderJson, writeDocument); + UNIT_ASSERT_NO_DIFF(result2, NResource::Find("/histogram_timeseries.json")); + } + + Y_UNIT_TEST(HistogramTimeSeriesDecode) { + NProto::TMultiSamplesList samples; + { + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + TString testJson = NResource::Find("/histogram_timeseries.json"); + DecodeJson(testJson, e.Get()); + } + + UNIT_ASSERT_VALUES_EQUAL(1, samples.SamplesSize()); + const NProto::TMultiSample& s = samples.GetSamples(0); + + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::HIST_RATE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "metric", "responseTimeMillis"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 2); + + auto h = ExponentialHistogram(6, 2); + for (i64 i = 1; i < 100; i++) { + h->Collect(i); + } + + AssertPointEqual(s.GetPoints(0), now, *h->Snapshot()); + + for (i64 i = 1; i < 100; i++) { + h->Collect(i); + } + + AssertPointEqual(s.GetPoints(1), now + TDuration::Seconds(15), *h->Snapshot()); + } + + Y_UNIT_TEST(IntGaugeEncode) { + auto writeDocument = [](IMetricEncoder* e) { + e->OnStreamBegin(); + { + e->OnMetricBegin(EMetricType::IGAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("metric", "a"); + e->OnLabelsEnd(); + } + e->OnInt64(now, Min<i64>()); + e->OnInt64(now + TDuration::Seconds(1), -1); + e->OnInt64(now + TDuration::Seconds(2), 0); + e->OnInt64(now + TDuration::Seconds(3), Max<i64>()); + e->OnMetricEnd(); + } + e->OnStreamEnd(); + }; + + TString result1 = EncodeToString(EncoderJson, writeDocument); + UNIT_ASSERT_NO_DIFF(result1, NResource::Find("/int_gauge.json")); + + TString result2 = EncodeToString(BufferedEncoderJson, writeDocument); + UNIT_ASSERT_NO_DIFF(result2, NResource::Find("/int_gauge.json")); + } + + Y_UNIT_TEST(InconsistentMetricTypes) { + auto emitMetrics = [](IMetricEncoder& encoder, const TString& expectedError) { + encoder.OnMetricBegin(EMetricType::GAUGE); + { + encoder.OnLabelsBegin(); + encoder.OnLabel("name", "m"); + encoder.OnLabel("l1", "v1"); + encoder.OnLabel("l2", "v2"); + encoder.OnLabelsEnd(); + } + encoder.OnDouble(now, 1.0); + encoder.OnMetricEnd(); + + encoder.OnMetricBegin(EMetricType::COUNTER); + { + encoder.OnLabelsBegin(); + encoder.OnLabel("name", "m"); + encoder.OnLabel("l1", "v1"); + encoder.OnLabel("l2", "v2"); + encoder.OnLabelsEnd(); + } + encoder.OnUint64(now, 1); + + UNIT_ASSERT_EXCEPTION_CONTAINS(encoder.OnMetricEnd(), + yexception, + expectedError); + }; + + { + TStringStream out; + auto encoder = BufferedEncoderJson(&out); + + encoder->OnStreamBegin(); + encoder->OnLabelsBegin(); + encoder->OnLabel("c", "cv"); + encoder->OnLabelsEnd(); + emitMetrics(*encoder, + "Time series point type mismatch: expected DOUBLE but found UINT64, " + "labels '{c=cv, l1=v1, l2=v2, name=m}'"); + } + + { + TStringStream out; + auto encoder = BufferedEncoderJson(&out); + + encoder->OnStreamBegin(); + encoder->OnLabelsBegin(); + encoder->OnLabel("l1", "v100"); + encoder->OnLabelsEnd(); + emitMetrics(*encoder, + "Time series point type mismatch: expected DOUBLE but found UINT64, " + "labels '{l1=v1, l2=v2, name=m}'"); + } + } + + Y_UNIT_TEST(IntGaugeDecode) { + NProto::TMultiSamplesList samples; + { + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + TString testJson = NResource::Find("/int_gauge.json"); + DecodeJson(testJson, e.Get()); + } + + UNIT_ASSERT_VALUES_EQUAL(1, samples.SamplesSize()); + const NProto::TMultiSample& s = samples.GetSamples(0); + + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::IGAUGE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "metric", "a"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 4); + AssertPointEqual(s.GetPoints(0), now, Min<i64>()); + AssertPointEqual(s.GetPoints(1), now + TDuration::Seconds(1), i64(-1)); + AssertPointEqual(s.GetPoints(2), now + TDuration::Seconds(2), i64(0)); + AssertPointEqual(s.GetPoints(3), now + TDuration::Seconds(3), Max<i64>()); + } + + Y_UNIT_TEST(FuzzerRegression) { + NProto::TMultiSamplesList samples; + { + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + for (auto f : { "/hist_crash.json", "/crash.json" }) { + TString testJson = NResource::Find(f); + UNIT_ASSERT_EXCEPTION(DecodeJson(testJson, e.Get()), yexception); + } + } + } + + Y_UNIT_TEST(LegacyNegativeRateThrows) { + const auto input = R"({ + "sensors": [ + { + "mode": "deriv", + "value": -1, + "labels": { "metric": "SystemTime" } + }, + } + ]}")"; + + NProto::TMultiSamplesList samples; + IMetricEncoderPtr e = EncoderProtobuf(&samples); + UNIT_ASSERT_EXCEPTION(DecodeJson(input, e.Get()), yexception); + } + + Y_UNIT_TEST(DecodeNamedMetrics) { + NProto::TMultiSamplesList samples; + { + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + TString metricsJson = NResource::Find("/named_metrics.json"); + DecodeJson(metricsJson, e.Get(), "sensor"); + } + + UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 2); + { + const NProto::TMultiSample& s = samples.GetSamples(0); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "sensor", "Memory"); + } + { + const NProto::TMultiSample& s = samples.GetSamples(1); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2); + AssertLabelEqual(s.GetLabels(0), "sensor", "QueueSize"); + AssertLabelEqual(s.GetLabels(1), "export", "Oxygen"); + } + } + +} diff --git a/library/cpp/monlib/encode/json/typed_point.h b/library/cpp/monlib/encode/json/typed_point.h new file mode 100644 index 0000000000..fbaa840c4b --- /dev/null +++ b/library/cpp/monlib/encode/json/typed_point.h @@ -0,0 +1,123 @@ +#pragma once + +#include <library/cpp/monlib/metrics/metric_value.h> + + +namespace NMonitoring { + + class TTypedPoint { + public: + TTypedPoint() + : Time_(TInstant::Zero()) + , ValueType_(EMetricValueType::UNKNOWN) + { + } + + template <typename T> + TTypedPoint(TInstant time, T value) + : Time_(time) + , ValueType_(TValueType<T>::Type) + , Value_(value) + { + Ref(); + } + + ~TTypedPoint() { + UnRef(); + } + + TTypedPoint(const TTypedPoint& rhs) + : Time_(rhs.Time_) + , ValueType_(rhs.ValueType_) + , Value_(rhs.Value_) + { + Ref(); + } + + TTypedPoint& operator=(const TTypedPoint& rhs) { + UnRef(); + + Time_ = rhs.Time_; + ValueType_ = rhs.ValueType_; + Value_ = rhs.Value_; + + Ref(); + return *this; + } + + TTypedPoint(TTypedPoint&& rhs) noexcept + : Time_(rhs.Time_) + , ValueType_(rhs.ValueType_) + , Value_(rhs.Value_) + { + rhs.ValueType_ = EMetricValueType::UNKNOWN; + rhs.Value_ = {}; + } + + TTypedPoint& operator=(TTypedPoint&& rhs) noexcept { + UnRef(); + + Time_ = rhs.Time_; + ValueType_ = rhs.ValueType_; + Value_ = rhs.Value_; + + rhs.ValueType_ = EMetricValueType::UNKNOWN; + rhs.Value_ = {}; + + return *this; + } + + TInstant GetTime() const noexcept { + return Time_; + } + + void SetTime(TInstant time) noexcept { + Time_ = time; + } + + TMetricValue GetValue() const noexcept { + return Value_; + } + + EMetricValueType GetValueType() const noexcept { + return ValueType_; + } + + template <typename T> + void SetValue(T value) noexcept { + ValueType_ = TValueType<T>::Type; + Value_ = TMetricValue{value}; + } + + bool HasValue() { + return ValueType_ != EMetricValueType::UNKNOWN; + } + + private: + void Ref() { + if (ValueType_ == EMetricValueType::HISTOGRAM) { + Value_.AsHistogram()->Ref(); + } else if (ValueType_ == EMetricValueType::SUMMARY) { + Value_.AsSummaryDouble()->Ref(); + } else if (ValueType_ == EMetricValueType::LOGHISTOGRAM) { + Value_.AsLogHistogram()->Ref(); + } + } + + void UnRef() { + if (ValueType_ == EMetricValueType::HISTOGRAM) { + Value_.AsHistogram()->UnRef(); + } else if (ValueType_ == EMetricValueType::SUMMARY) { + Value_.AsSummaryDouble()->UnRef(); + } else if (ValueType_ == EMetricValueType::LOGHISTOGRAM) { + Value_.AsLogHistogram()->UnRef(); + } + } + + private: + TInstant Time_; + EMetricValueType ValueType_; + TMetricValue Value_; + }; + +} diff --git a/library/cpp/monlib/encode/json/ut/buffered_test.json b/library/cpp/monlib/encode/json/ut/buffered_test.json new file mode 100644 index 0000000000..53212cf8e1 --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/buffered_test.json @@ -0,0 +1,36 @@ +{ + "sensors": + [ + { + "kind":"COUNTER", + "labels": + { + "foo":"bar" + }, + "timeseries": + [ + { + "ts":1, + "value":42 + }, + { + "ts":2, + "value":42 + }, + { + "ts":4, + "value":42 + } + ] + }, + { + "kind":"COUNTER", + "labels": + { + "foo":"baz" + }, + "ts":3, + "value":42 + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/buffered_ts_merge.json b/library/cpp/monlib/encode/json/ut/buffered_ts_merge.json new file mode 100644 index 0000000000..1d27efacb0 --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/buffered_ts_merge.json @@ -0,0 +1,13 @@ +{ + "sensors": + [ + { + "kind":"COUNTER", + "labels": + { + "foo":"bar" + }, + "value":45 + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/crash.json b/library/cpp/monlib/encode/json/ut/crash.json Binary files differnew file mode 100644 index 0000000000..8ff4369dc4 --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/crash.json diff --git a/library/cpp/monlib/encode/json/ut/empty_series.json b/library/cpp/monlib/encode/json/ut/empty_series.json new file mode 100644 index 0000000000..641e10cdea --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/empty_series.json @@ -0,0 +1,12 @@ +{ + "sensors": + [ + { + "kind":"COUNTER", + "labels": + { + "foo":"bar" + } + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/expected.json b/library/cpp/monlib/encode/json/ut/expected.json new file mode 100644 index 0000000000..ead853455b --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/expected.json @@ -0,0 +1,92 @@ +{ + "ts":1500000000, + "commonLabels": + { + "project":"solomon" + }, + "sensors": + [ + { + "kind":"COUNTER", + "labels": + { + "metric":"single", + "labels":"l1" + }, + "ts":1509843723, + "value":17 + }, + { + "kind":"RATE", + "labels": + { + "metric":"single", + "labels":"l2" + }, + "ts":1509843723, + "value":17 + }, + { + "kind":"GAUGE", + "labels": + { + "metric":"single", + "labels":"l3" + }, + "ts":1509843723, + "value":3.14 + }, + { + "kind":"IGAUGE", + "labels": + { + "metric":"single_igauge", + "labels":"l4" + }, + "ts":1509843723, + "value":42 + }, + { + "kind":"GAUGE", + "labels": + { + "metric":"multiple", + "labels":"l5" + }, + "timeseries": + [ + { + "ts":1509843723, + "value":"nan" + }, + { + "ts":1509843738, + "value":"inf" + }, + { + "ts":1509843753, + "value":"-inf" + } + ] + }, + { + "kind":"COUNTER", + "timeseries": + [ + { + "ts":1509843723, + "value":1337 + }, + { + "ts":1509843738, + "value":1338 + } + ], + "labels": + { + "metric":"multiple", + "labels":"l6" + } + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/expected_buffered.json b/library/cpp/monlib/encode/json/ut/expected_buffered.json new file mode 100644 index 0000000000..9a6a1d6201 --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/expected_buffered.json @@ -0,0 +1,92 @@ +{ + "ts":1500000000, + "commonLabels": + { + "project":"solomon" + }, + "sensors": + [ + { + "kind":"COUNTER", + "labels": + { + "labels":"l1", + "metric":"single" + }, + "ts":1509843723, + "value":17 + }, + { + "kind":"RATE", + "labels": + { + "labels":"l2", + "metric":"single" + }, + "ts":1509843723, + "value":17 + }, + { + "kind":"GAUGE", + "labels": + { + "labels":"l3", + "metric":"single" + }, + "ts":1509843723, + "value":3.14 + }, + { + "kind":"IGAUGE", + "labels": + { + "labels":"l4", + "metric":"single_igauge" + }, + "ts":1509843723, + "value":42 + }, + { + "kind":"GAUGE", + "labels": + { + "labels":"l5", + "metric":"multiple" + }, + "timeseries": + [ + { + "ts":1509843723, + "value":"nan" + }, + { + "ts":1509843738, + "value":"inf" + }, + { + "ts":1509843753, + "value":"-inf" + } + ] + }, + { + "kind":"COUNTER", + "labels": + { + "labels":"l6", + "metric":"multiple" + }, + "timeseries": + [ + { + "ts":1509843723, + "value":1337 + }, + { + "ts":1509843738, + "value":1338 + } + ] + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/expected_cloud.json b/library/cpp/monlib/encode/json/ut/expected_cloud.json new file mode 100644 index 0000000000..6184811579 --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/expected_cloud.json @@ -0,0 +1,92 @@ +{ + "ts":"2017-07-14T02:40:00.000000Z", + "labels": + { + "project":"solomon" + }, + "metrics": + [ + { + "type":"COUNTER", + "labels": + { + "labels":"l1" + }, + "name":"single", + "ts":"2017-11-05T01:02:03.000000Z", + "value":17 + }, + { + "type":"RATE", + "labels": + { + "labels":"l2" + }, + "name":"single", + "ts":"2017-11-05T01:02:03.000000Z", + "value":17 + }, + { + "type":"DGAUGE", + "labels": + { + "labels":"l3" + }, + "name":"single", + "ts":"2017-11-05T01:02:03.000000Z", + "value":3.14 + }, + { + "type":"IGAUGE", + "labels": + { + "labels":"l4" + }, + "name":"single_igauge", + "ts":"2017-11-05T01:02:03.000000Z", + "value":42 + }, + { + "type":"DGAUGE", + "labels": + { + "labels":"l5" + }, + "name":"multiple", + "timeseries": + [ + { + "ts":"2017-11-05T01:02:03.000000Z", + "value":"nan" + }, + { + "ts":"2017-11-05T01:02:18.000000Z", + "value":"inf" + }, + { + "ts":"2017-11-05T01:02:33.000000Z", + "value":"-inf" + } + ] + }, + { + "type":"COUNTER", + "timeseries": + [ + { + "ts":"2017-11-05T01:02:03.000000Z", + "value":1337 + }, + { + "ts":"2017-11-05T01:02:18.000000Z", + "value":1338 + } + ], + "labels": + { + "labels":"l6" + }, + "name":"multiple" + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/expected_cloud_buffered.json b/library/cpp/monlib/encode/json/ut/expected_cloud_buffered.json new file mode 100644 index 0000000000..be237d522b --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/expected_cloud_buffered.json @@ -0,0 +1,92 @@ +{ + "ts":"2017-07-14T02:40:00.000000Z", + "labels": + { + "project":"solomon" + }, + "metrics": + [ + { + "type":"COUNTER", + "labels": + { + "labels":"l1" + }, + "name":"single", + "ts":"2017-11-05T01:02:03.000000Z", + "value":17 + }, + { + "type":"RATE", + "labels": + { + "labels":"l2" + }, + "name":"single", + "ts":"2017-11-05T01:02:03.000000Z", + "value":17 + }, + { + "type":"DGAUGE", + "labels": + { + "labels":"l3" + }, + "name":"single", + "ts":"2017-11-05T01:02:03.000000Z", + "value":3.14 + }, + { + "type":"IGAUGE", + "labels": + { + "labels":"l4" + }, + "name":"single_igauge", + "ts":"2017-11-05T01:02:03.000000Z", + "value":42 + }, + { + "type":"DGAUGE", + "labels": + { + "labels":"l5" + }, + "name":"multiple", + "timeseries": + [ + { + "ts":"2017-11-05T01:02:03.000000Z", + "value":"nan" + }, + { + "ts":"2017-11-05T01:02:18.000000Z", + "value":"inf" + }, + { + "ts":"2017-11-05T01:02:33.000000Z", + "value":"-inf" + } + ] + }, + { + "type":"COUNTER", + "labels": + { + "labels":"l6" + }, + "name":"multiple", + "timeseries": + [ + { + "ts":"2017-11-05T01:02:03.000000Z", + "value":1337 + }, + { + "ts":"2017-11-05T01:02:18.000000Z", + "value":1338 + } + ] + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/hist_crash.json b/library/cpp/monlib/encode/json/ut/hist_crash.json Binary files differnew file mode 100644 index 0000000000..867d0fce7d --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/hist_crash.json diff --git a/library/cpp/monlib/encode/json/ut/histogram_timeseries.json b/library/cpp/monlib/encode/json/ut/histogram_timeseries.json new file mode 100644 index 0000000000..f6131ffded --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/histogram_timeseries.json @@ -0,0 +1,61 @@ +{ + "sensors": + [ + { + "kind":"HIST_RATE", + "labels": + { + "metric":"responseTimeMillis" + }, + "timeseries": + [ + { + "ts":1509843723, + "hist": + { + "bounds": + [ + 1, + 2, + 4, + 8, + 16 + ], + "buckets": + [ + 1, + 1, + 2, + 4, + 8 + ], + "inf":83 + } + }, + { + "ts":1509843738, + "hist": + { + "bounds": + [ + 1, + 2, + 4, + 8, + 16 + ], + "buckets": + [ + 2, + 2, + 4, + 8, + 16 + ], + "inf":166 + } + } + ] + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/histogram_value.json b/library/cpp/monlib/encode/json/ut/histogram_value.json new file mode 100644 index 0000000000..ec1ae5cdec --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/histogram_value.json @@ -0,0 +1,33 @@ +{ + "sensors": + [ + { + "kind":"HIST", + "labels": + { + "metric":"responseTimeMillis" + }, + "ts":1509843723, + "hist": + { + "bounds": + [ + 1, + 2, + 4, + 8, + 16 + ], + "buckets": + [ + 1, + 1, + 2, + 4, + 8 + ], + "inf":83 + } + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/histogram_value_inf_before_bounds.json b/library/cpp/monlib/encode/json/ut/histogram_value_inf_before_bounds.json new file mode 100644 index 0000000000..f8a17c8831 --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/histogram_value_inf_before_bounds.json @@ -0,0 +1,33 @@ +{ + "sensors": + [ + { + "kind":"HIST", + "labels": + { + "metric":"responseTimeMillis" + }, + "ts":1509843723, + "hist": + { + "inf":83, + "bounds": + [ + 1, + 2, + 4, + 8, + 16 + ], + "buckets": + [ + 1, + 1, + 2, + 4, + 8 + ] + } + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/int_gauge.json b/library/cpp/monlib/encode/json/ut/int_gauge.json new file mode 100644 index 0000000000..fbe57f873c --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/int_gauge.json @@ -0,0 +1,31 @@ +{ + "sensors": + [ + { + "kind":"IGAUGE", + "labels": + { + "metric":"a" + }, + "timeseries": + [ + { + "ts":1509843723, + "value":-9223372036854775808 + }, + { + "ts":1509843724, + "value":-1 + }, + { + "ts":1509843725, + "value":0 + }, + { + "ts":1509843726, + "value":9223372036854775807 + } + ] + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/log_histogram_timeseries.json b/library/cpp/monlib/encode/json/ut/log_histogram_timeseries.json new file mode 100644 index 0000000000..e811a2cc57 --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/log_histogram_timeseries.json @@ -0,0 +1,47 @@ +{ + "sensors": + [ + { + "kind":"LOGHIST", + "labels": + { + "metric":"ms" + }, + "timeseries": + [ + { + "ts":1509843723, + "log_hist": + { + "base":1.5, + "zeros_count":1, + "start_power":0, + "buckets": + [ + 0.5, + 0.25, + 0.25, + 0.5 + ] + } + }, + { + "ts":1509843738, + "log_hist": + { + "base":1.5, + "zeros_count":1, + "start_power":0, + "buckets": + [ + 1, + 0.5, + 0.5, + 1 + ] + } + } + ] + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/log_histogram_value.json b/library/cpp/monlib/encode/json/ut/log_histogram_value.json new file mode 100644 index 0000000000..002478293b --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/log_histogram_value.json @@ -0,0 +1,26 @@ +{ + "sensors": + [ + { + "kind":"LOGHIST", + "labels": + { + "metric":"ms" + }, + "ts":1509843723, + "log_hist": + { + "base":1.5, + "zeros_count":1, + "start_power":0, + "buckets": + [ + 0.5, + 0.25, + 0.25, + 0.5 + ] + } + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/merged.json b/library/cpp/monlib/encode/json/ut/merged.json new file mode 100644 index 0000000000..ea2c99a33c --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/merged.json @@ -0,0 +1,14 @@ +{ + "sensors": + [ + { + "kind":"RATE", + "labels": + { + "metric":"hello", + "label":"world" + }, + "value":1 + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/metrics.json b/library/cpp/monlib/encode/json/ut/metrics.json new file mode 100644 index 0000000000..2be4617d51 --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/metrics.json @@ -0,0 +1,43 @@ +{ + "labels": { + "project": "solomon", + "cluster": "man", + "service": "stockpile" + }, + "metrics": [ + { + "type": "DGAUGE", + "labels": { + "metric": "Memory" + }, + "value": 10 + }, + { + "type": "RATE", + "value": 1, + "labels": { "metric": "UserTime" } + }, + { + "type": "GAUGE", + "value": 3.14159, + "labels": { "export": "Oxygen", "metric": "QueueSize" }, + "ts": "2017-11-05T12:34:56.000Z", + "memOnly": true + }, + { + "type": "GAUGE", + "labels": { "metric": "Writes" }, + "timeseries": [ + { + "ts": "2017-08-28T12:32:11Z", + "value": -10 + }, + { + "value": 20, + "ts": 1503923187 + } + ] + } + ], + "ts": "2017-08-27T12:34:56Z" +} diff --git a/library/cpp/monlib/encode/json/ut/named_metrics.json b/library/cpp/monlib/encode/json/ut/named_metrics.json new file mode 100644 index 0000000000..98f93e8c39 --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/named_metrics.json @@ -0,0 +1,22 @@ +{ + "labels": { + "project": "solomon", + "cluster": "prod-sas", + "service": "stockpile" + }, + "metrics": [ + { + "type": "DGAUGE", + "name": "Memory", + "value": 1 + }, + { + "type": "DGAUGE", + "name": "QueueSize", + "labels": { + "export": "Oxygen" + }, + "value": 10 + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/sensors.json b/library/cpp/monlib/encode/json/ut/sensors.json new file mode 100644 index 0000000000..4d979a3c1e --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/sensors.json @@ -0,0 +1,40 @@ +{ + "commonLabels": { + "project": "solomon", + "cluster": "man", + "service": "stockpile" + }, + "sensors": [ + { + "labels": { + "metric": "Memory" + }, + "value": 10 + }, + { + "mode": "deriv", + "value": 1, + "labels": { "metric": "UserTime" } + }, + { + "value": 3.14159, + "labels": { "export": "Oxygen", "metric": "QueueSize" }, + "ts": "2017-11-05T12:34:56.000Z", + "memOnly": true + }, + { + "labels": { "metric": "Writes" }, + "timeseries": [ + { + "ts": "2017-08-28T12:32:11Z", + "value": -10 + }, + { + "value": 20, + "ts": 1503923187 + } + ] + } + ], + "ts": "2017-08-27T12:34:56Z" +} diff --git a/library/cpp/monlib/encode/json/ut/summary_inf.json b/library/cpp/monlib/encode/json/ut/summary_inf.json new file mode 100644 index 0000000000..625a6cd8ad --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/summary_inf.json @@ -0,0 +1,21 @@ +{ + "sensors": + [ + { + "kind":"DSUMMARY", + "labels": + { + "metric":"temperature" + }, + "ts":1509843723, + "summary": + { + "sum":"nan", + "min":"-inf", + "max":"inf", + "last":0.3, + "count":30 + } + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/summary_timeseries.json b/library/cpp/monlib/encode/json/ut/summary_timeseries.json new file mode 100644 index 0000000000..92007af3e6 --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/summary_timeseries.json @@ -0,0 +1,37 @@ +{ + "sensors": + [ + { + "kind":"DSUMMARY", + "labels": + { + "metric":"temperature" + }, + "timeseries": + [ + { + "ts":1509843723, + "summary": + { + "sum":0.8, + "min":-0.5, + "max":1, + "last":1, + "count":3 + } + }, + { + "ts":1509843738, + "summary": + { + "sum":-0.69, + "min":-1.5, + "max":1, + "last":0.01, + "count":5 + } + } + ] + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/summary_value.json b/library/cpp/monlib/encode/json/ut/summary_value.json new file mode 100644 index 0000000000..366394c5e1 --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/summary_value.json @@ -0,0 +1,21 @@ +{ + "sensors": + [ + { + "kind":"DSUMMARY", + "labels": + { + "metric":"temperature" + }, + "ts":1509843723, + "summary": + { + "sum":10, + "min":-0.5, + "max":0.5, + "last":0.3, + "count":30 + } + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/test_decode_to_encode.json b/library/cpp/monlib/encode/json/ut/test_decode_to_encode.json new file mode 100644 index 0000000000..65f0c5c6e2 --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/test_decode_to_encode.json @@ -0,0 +1,16 @@ +{ + "commonLabels": { + "project": "solomon", + "cluster": "man", + "service": "stockpile" + }, + "sensors": [ + { + "kind": "GAUGE", + "labels": { "export": "Oxygen", "metric": "QueueSize" }, + "ts": 1509885296, + "value": 3.14159 + } + ], + "ts": 1503837296 +} diff --git a/library/cpp/monlib/encode/json/ut/ya.make b/library/cpp/monlib/encode/json/ut/ya.make new file mode 100644 index 0000000000..e50c4f4903 --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/ya.make @@ -0,0 +1,46 @@ +UNITTEST_FOR(library/cpp/monlib/encode/json) + +OWNER( + g:solomon + jamel +) + +SRCS( + json_decoder_ut.cpp + json_ut.cpp +) + +RESOURCE( + buffered_test.json /buffered_test.json + buffered_ts_merge.json /buffered_ts_merge.json + empty_series.json /empty_series.json + expected.json /expected.json + expected_buffered.json /expected_buffered.json + expected_cloud.json /expected_cloud.json + expected_cloud_buffered.json /expected_cloud_buffered.json + merged.json /merged.json + histogram_timeseries.json /histogram_timeseries.json + histogram_value.json /histogram_value.json + histogram_value_inf_before_bounds.json /histogram_value_inf_before_bounds.json + int_gauge.json /int_gauge.json + sensors.json /sensors.json + metrics.json /metrics.json + named_metrics.json /named_metrics.json + test_decode_to_encode.json /test_decode_to_encode.json + crash.json /crash.json + hist_crash.json /hist_crash.json + summary_value.json /summary_value.json + summary_inf.json /summary_inf.json + summary_timeseries.json /summary_timeseries.json + log_histogram_value.json /log_histogram_value.json + log_histogram_timeseries.json /log_histogram_timeseries.json +) + +PEERDIR( + library/cpp/json + library/cpp/monlib/consumers + library/cpp/monlib/encode/protobuf + library/cpp/resource +) + +END() diff --git a/library/cpp/monlib/encode/json/ya.make b/library/cpp/monlib/encode/json/ya.make new file mode 100644 index 0000000000..a50fc412a9 --- /dev/null +++ b/library/cpp/monlib/encode/json/ya.make @@ -0,0 +1,21 @@ +LIBRARY() + +OWNER( + g:solomon + jamel +) + +SRCS( + json_decoder.cpp + json_encoder.cpp +) + +PEERDIR( + library/cpp/monlib/encode + library/cpp/monlib/encode/buffered + library/cpp/monlib/exception + library/cpp/json + library/cpp/json/writer +) + +END() diff --git a/library/cpp/monlib/encode/legacy_protobuf/legacy_proto_decoder.cpp b/library/cpp/monlib/encode/legacy_protobuf/legacy_proto_decoder.cpp new file mode 100644 index 0000000000..f87a2d7e8f --- /dev/null +++ b/library/cpp/monlib/encode/legacy_protobuf/legacy_proto_decoder.cpp @@ -0,0 +1,527 @@ +#include "legacy_protobuf.h" + +#include <library/cpp/monlib/encode/legacy_protobuf/protos/metric_meta.pb.h> +#include <library/cpp/monlib/metrics/metric_consumer.h> +#include <library/cpp/monlib/metrics/labels.h> + +#include <util/generic/yexception.h> +#include <util/generic/maybe.h> +#include <util/datetime/base.h> +#include <util/string/split.h> + +#include <google/protobuf/reflection.h> + +#include <algorithm> + +#ifdef LEGACY_PB_TRACE +#define TRACE(msg) \ + Cerr << msg << Endl +#else +#define TRACE(...) ; +#endif + +namespace NMonitoring { + namespace { + using TMaybeMeta = TMaybe<NMonProto::TMetricMeta>; + + TString ReadLabelValue(const NProtoBuf::Message& msg, const NProtoBuf::FieldDescriptor* d, const NProtoBuf::Reflection& r) { + using namespace NProtoBuf; + + switch (d->type()) { + case FieldDescriptor::TYPE_UINT32: + return ::ToString(r.GetUInt32(msg, d)); + case FieldDescriptor::TYPE_UINT64: + return ::ToString(r.GetUInt64(msg, d)); + case FieldDescriptor::TYPE_STRING: + return r.GetString(msg, d); + case FieldDescriptor::TYPE_ENUM: { + auto val = r.GetEnumValue(msg, d); + auto* valDesc = d->enum_type()->FindValueByNumber(val); + return valDesc->name(); + } + + default: + ythrow yexception() << "type " << d->type_name() << " cannot be used as a field value"; + } + + return {}; + } + + double ReadFieldAsDouble(const NProtoBuf::Message& msg, const NProtoBuf::FieldDescriptor* d, const NProtoBuf::Reflection& r) { + using namespace NProtoBuf; + + switch (d->type()) { + case FieldDescriptor::TYPE_DOUBLE: + return r.GetDouble(msg, d); + case FieldDescriptor::TYPE_BOOL: + return r.GetBool(msg, d) ? 1 : 0; + case FieldDescriptor::TYPE_INT32: + return r.GetInt32(msg, d); + case FieldDescriptor::TYPE_INT64: + return r.GetInt64(msg, d); + case FieldDescriptor::TYPE_UINT32: + return r.GetUInt32(msg, d); + case FieldDescriptor::TYPE_UINT64: + return r.GetUInt64(msg, d); + case FieldDescriptor::TYPE_SINT32: + return r.GetInt32(msg, d); + case FieldDescriptor::TYPE_SINT64: + return r.GetInt64(msg, d); + case FieldDescriptor::TYPE_FIXED32: + return r.GetUInt32(msg, d); + case FieldDescriptor::TYPE_FIXED64: + return r.GetUInt64(msg, d); + case FieldDescriptor::TYPE_SFIXED32: + return r.GetInt32(msg, d); + case FieldDescriptor::TYPE_SFIXED64: + return r.GetInt64(msg, d); + case FieldDescriptor::TYPE_FLOAT: + return r.GetFloat(msg, d); + case FieldDescriptor::TYPE_ENUM: + return r.GetEnumValue(msg, d); + default: + ythrow yexception() << "type " << d->type_name() << " cannot be used as a field value"; + } + + return std::numeric_limits<double>::quiet_NaN(); + } + + double ReadRepeatedAsDouble(const NProtoBuf::Message& msg, const NProtoBuf::FieldDescriptor* d, const NProtoBuf::Reflection& r, size_t i) { + using namespace NProtoBuf; + + switch (d->type()) { + case FieldDescriptor::TYPE_DOUBLE: + return r.GetRepeatedDouble(msg, d, i); + case FieldDescriptor::TYPE_BOOL: + return r.GetRepeatedBool(msg, d, i) ? 1 : 0; + case FieldDescriptor::TYPE_INT32: + return r.GetRepeatedInt32(msg, d, i); + case FieldDescriptor::TYPE_INT64: + return r.GetRepeatedInt64(msg, d, i); + case FieldDescriptor::TYPE_UINT32: + return r.GetRepeatedUInt32(msg, d, i); + case FieldDescriptor::TYPE_UINT64: + return r.GetRepeatedUInt64(msg, d, i); + case FieldDescriptor::TYPE_SINT32: + return r.GetRepeatedInt32(msg, d, i); + case FieldDescriptor::TYPE_SINT64: + return r.GetRepeatedInt64(msg, d, i); + case FieldDescriptor::TYPE_FIXED32: + return r.GetRepeatedUInt32(msg, d, i); + case FieldDescriptor::TYPE_FIXED64: + return r.GetRepeatedUInt64(msg, d, i); + case FieldDescriptor::TYPE_SFIXED32: + return r.GetRepeatedInt32(msg, d, i); + case FieldDescriptor::TYPE_SFIXED64: + return r.GetRepeatedInt64(msg, d, i); + case FieldDescriptor::TYPE_FLOAT: + return r.GetRepeatedFloat(msg, d, i); + case FieldDescriptor::TYPE_ENUM: + return r.GetRepeatedEnumValue(msg, d, i); + default: + ythrow yexception() << "type " << d->type_name() << " cannot be used as a field value"; + } + + return std::numeric_limits<double>::quiet_NaN(); + } + + TString LabelFromField(const NProtoBuf::Message& msg, const TString& name) { + const auto* fieldDesc = msg.GetDescriptor()->FindFieldByName(name); + const auto* reflection = msg.GetReflection(); + Y_ENSURE(fieldDesc && reflection, "Unable to get meta for field " << name); + + auto s = ReadLabelValue(msg, fieldDesc, *reflection); + std::replace(std::begin(s), s.vend(), ' ', '_'); + + return s; + } + + TMaybeMeta MaybeGetMeta(const NProtoBuf::FieldOptions& opts) { + if (opts.HasExtension(NMonProto::Metric)) { + return opts.GetExtension(NMonProto::Metric); + } + + return Nothing(); + } + + class ILabelGetter: public TThrRefBase { + public: + enum class EType { + Fixed = 1, + Lazy = 2, + }; + + virtual TLabel Get(const NProtoBuf::Message&) = 0; + virtual EType Type() const = 0; + }; + + class TFixedLabel: public ILabelGetter { + public: + explicit TFixedLabel(TLabel&& l) + : Label_{std::move(l)} + { + TRACE("found fixed label " << l); + } + + EType Type() const override { + return EType::Fixed; + } + TLabel Get(const NProtoBuf::Message&) override { + return Label_; + } + + private: + TLabel Label_; + }; + + using TFunction = std::function<TLabel(const NProtoBuf::Message&)>; + + class TLazyLabel: public ILabelGetter { + public: + TLazyLabel(TFunction&& fn) + : Fn_{std::move(fn)} + { + TRACE("found lazy label"); + } + + EType Type() const override { + return EType::Lazy; + } + TLabel Get(const NProtoBuf::Message& msg) override { + return Fn_(msg); + } + + private: + TFunction Fn_; + }; + + class TDecoderContext { + public: + void Init(const NProtoBuf::Message* msg) { + Message_ = msg; + Y_ENSURE(Message_); + Reflection_ = msg->GetReflection(); + Y_ENSURE(Reflection_); + + for (auto it = Labels_.begin(); it != Labels_.end(); ++it) { + if ((*it)->Type() == ILabelGetter::EType::Lazy) { + auto l = (*it)->Get(Message()); + *it = ::MakeIntrusive<TFixedLabel>(std::move(l)); + } else { + auto l = (*it)->Get(Message()); + } + } + } + + void Clear() noexcept { + Message_ = nullptr; + Reflection_ = nullptr; + } + + TDecoderContext CreateChildFromMeta(const NMonProto::TMetricMeta& metricMeta, const TString& name, i64 repeatedIdx = -1) { + TDecoderContext child{*this}; + child.Clear(); + + if (metricMeta.HasCustomPath()) { + if (const auto& nodePath = metricMeta.GetCustomPath()) { + child.AppendPath(nodePath); + } + } else if (metricMeta.GetPath()) { + child.AppendPath(name); + } + + if (metricMeta.HasKeys()) { + child.ParseKeys(metricMeta.GetKeys(), repeatedIdx); + } + + return child; + } + + TDecoderContext CreateChildFromRepeatedScalar(const NMonProto::TMetricMeta& metricMeta, i64 repeatedIdx = -1) { + TDecoderContext child{*this}; + child.Clear(); + + if (metricMeta.HasKeys()) { + child.ParseKeys(metricMeta.GetKeys(), repeatedIdx); + } + + return child; + } + + TDecoderContext CreateChildFromEls(const TString& name, const NMonProto::TExtraLabelMetrics& metrics, size_t idx, TMaybeMeta maybeMeta) { + TDecoderContext child{*this}; + child.Clear(); + + auto usePath = [&maybeMeta] { + return !maybeMeta->HasPath() || maybeMeta->GetPath(); + }; + + if (!name.empty() && (!maybeMeta || usePath())) { + child.AppendPath(name); + } + + child.Labels_.push_back(::MakeIntrusive<TLazyLabel>( + [ labelName = metrics.GetlabelName(), idx, &metrics ](const auto&) { + const auto& val = metrics.Getvalues(idx); + TString labelVal; + const auto uintLabel = val.GetlabelValueUint(); + + if (uintLabel) { + labelVal = ::ToString(uintLabel); + } else { + labelVal = val.GetlabelValue(); + } + + return TLabel{labelName, labelVal}; + })); + + return child; + } + + void ParseKeys(TStringBuf keys, i64 repeatedIdx = -1) { + auto parts = StringSplitter(keys) + .Split(' ') + .SkipEmpty(); + + for (auto part : parts) { + auto str = part.Token(); + + TStringBuf lhs, rhs; + + const bool isDynamic = str.TrySplit(':', lhs, rhs); + const bool isIndexing = isDynamic && rhs == TStringBuf("#"); + + if (isIndexing) { + TRACE("parsed index labels"); + + // <label_name>:# means that we should use index of the repeated + // field as label value + Y_ENSURE(repeatedIdx != -1); + Labels_.push_back(::MakeIntrusive<TLazyLabel>([=](const auto&) { + return TLabel{lhs, ::ToString(repeatedIdx)}; + })); + } else if (isDynamic) { + TRACE("parsed dynamic labels"); + + // <label_name>:<field_name> means that we need to take label value + // later from message's field + Labels_.push_back(::MakeIntrusive<TLazyLabel>([=](const auto& msg) { + return TLabel{lhs, LabelFromField(msg, TString{rhs})}; + })); + } else if (str.TrySplit('=', lhs, rhs)) { + TRACE("parsed static labels"); + + // <label_name>=<label_value> stands for constant label + Labels_.push_back(::MakeIntrusive<TFixedLabel>(TLabel{lhs, rhs})); + } else { + ythrow yexception() << "Incorrect Keys format"; + } + } + } + + void AppendPath(TStringBuf fieldName) { + Path_ += '/'; + Path_ += fieldName; + } + + const TString& Path() const { + return Path_; + } + + TLabels Labels() const { + TLabels result; + for (auto&& l : Labels_) { + result.Add(l->Get(Message())); + } + + return result; + } + + const NProtoBuf::Message& Message() const { + Y_VERIFY_DEBUG(Message_); + return *Message_; + } + + const NProtoBuf::Reflection& Reflection() const { + return *Reflection_; + } + + private: + const NProtoBuf::Message* Message_{nullptr}; + const NProtoBuf::Reflection* Reflection_{nullptr}; + + TString Path_; + TVector<TIntrusivePtr<ILabelGetter>> Labels_; + }; + + class TDecoder { + public: + TDecoder(IMetricConsumer* consumer, const NProtoBuf::Message& message, TInstant timestamp) + : Consumer_{consumer} + , Message_{message} + , Timestamp_{timestamp} + { + } + + void Decode() const { + Consumer_->OnStreamBegin(); + DecodeToStream(); + Consumer_->OnStreamEnd(); + } + + void DecodeToStream() const { + DecodeImpl(Message_, {}); + } + + private: + static const NMonProto::TExtraLabelMetrics& ExtractExtraMetrics(TDecoderContext& ctx, const NProtoBuf::FieldDescriptor& f) { + const auto& parent = ctx.Message(); + const auto& reflection = ctx.Reflection(); + auto& subMessage = reflection.GetMessage(parent, &f); + + return dynamic_cast<const NMonProto::TExtraLabelMetrics&>(subMessage); + } + + void DecodeImpl(const NProtoBuf::Message& msg, TDecoderContext ctx) const { + std::vector<const NProtoBuf::FieldDescriptor*> fields; + + ctx.Init(&msg); + + ctx.Reflection().ListFields(msg, &fields); + + for (const auto* f : fields) { + Y_ENSURE(f); + + const auto& opts = f->options(); + const auto isMessage = f->type() == NProtoBuf::FieldDescriptor::TYPE_MESSAGE; + const auto isExtraLabelMetrics = isMessage && f->message_type()->full_name() == "NMonProto.TExtraLabelMetrics"; + const auto maybeMeta = MaybeGetMeta(opts); + + if (!(maybeMeta || isExtraLabelMetrics)) { + continue; + } + + if (isExtraLabelMetrics) { + const auto& extra = ExtractExtraMetrics(ctx, *f); + RecurseExtraLabelMetrics(ctx, extra, f->name(), maybeMeta); + } else if (isMessage) { + RecurseMessage(ctx, *maybeMeta, *f); + } else if (f->is_repeated()) { + RecurseRepeatedScalar(ctx, *maybeMeta, *f); + } else if (maybeMeta->HasType()) { + const auto val = ReadFieldAsDouble(msg, f, ctx.Reflection()); + const bool isRate = maybeMeta->GetType() == NMonProto::EMetricType::RATE; + WriteMetric(val, ctx, f->name(), isRate); + } + } + } + + void RecurseRepeatedScalar(TDecoderContext ctx, const NMonProto::TMetricMeta& meta, const NProtoBuf::FieldDescriptor& f) const { + auto&& msg = ctx.Message(); + auto&& reflection = ctx.Reflection(); + const bool isRate = meta.GetType() == NMonProto::EMetricType::RATE; + + // this is a repeated scalar field, which makes metric only if it's indexing + for (auto i = 0; i < reflection.FieldSize(msg, &f); ++i) { + auto subCtx = ctx.CreateChildFromRepeatedScalar(meta, i); + subCtx.Init(&msg); + auto val = ReadRepeatedAsDouble(msg, &f, reflection, i); + WriteMetric(val, subCtx, f.name(), isRate); + } + } + + void RecurseExtraLabelMetrics(TDecoderContext ctx, const NMonProto::TExtraLabelMetrics& msg, const TString& name, const TMaybeMeta& meta) const { + auto i = 0; + for (const auto& val : msg.Getvalues()) { + auto subCtx = ctx.CreateChildFromEls(name, msg, i++, meta); + subCtx.Init(&val); + + const bool isRate = val.Hastype() + ? val.Gettype() == NMonProto::EMetricType::RATE + : meta->GetType() == NMonProto::EMetricType::RATE; + + double metricVal{0}; + if (isRate) { + metricVal = val.GetlongValue(); + } else { + metricVal = val.GetdoubleValue(); + } + + WriteMetric(metricVal, subCtx, "", isRate); + + for (const auto& child : val.Getchildren()) { + RecurseExtraLabelMetrics(subCtx, child, "", meta); + } + } + } + + void RecurseMessage(TDecoderContext ctx, const NMonProto::TMetricMeta& metricMeta, const NProtoBuf::FieldDescriptor& f) const { + const auto& msg = ctx.Message(); + const auto& reflection = ctx.Reflection(); + + if (f.is_repeated()) { + TRACE("recurse into repeated message " << f.name()); + for (auto i = 0; i < reflection.FieldSize(msg, &f); ++i) { + auto& subMessage = reflection.GetRepeatedMessage(msg, &f, i); + DecodeImpl(subMessage, ctx.CreateChildFromMeta(metricMeta, f.name(), i)); + } + } else { + TRACE("recurse into message " << f.name()); + auto& subMessage = reflection.GetMessage(msg, &f); + DecodeImpl(subMessage, ctx.CreateChildFromMeta(metricMeta, f.name())); + } + } + + inline void WriteValue(ui64 value) const { + Consumer_->OnUint64(Timestamp_, value); + } + + inline void WriteValue(double value) const { + Consumer_->OnDouble(Timestamp_, value); + } + + void WriteMetric(double value, const TDecoderContext& ctx, const TString& name, bool isRate) const { + if (isRate) { + Consumer_->OnMetricBegin(EMetricType::RATE); + WriteValue(static_cast<ui64>(value)); + } else { + Consumer_->OnMetricBegin(EMetricType::GAUGE); + WriteValue(static_cast<double>(value)); + } + + Consumer_->OnLabelsBegin(); + + for (const auto& label : ctx.Labels()) { + Consumer_->OnLabel(label.Name(), label.Value()); + } + + const auto fullPath = name.empty() + ? ctx.Path() + : ctx.Path() + '/' + name; + + if (fullPath) { + Consumer_->OnLabel("path", fullPath); + } + + Consumer_->OnLabelsEnd(); + Consumer_->OnMetricEnd(); + } + + private: + IMetricConsumer* Consumer_{nullptr}; + const NProtoBuf::Message& Message_; + TInstant Timestamp_; + }; + + } + + void DecodeLegacyProto(const NProtoBuf::Message& data, IMetricConsumer* consumer, TInstant ts) { + Y_ENSURE(consumer); + TDecoder(consumer, data, ts).Decode(); + } + + void DecodeLegacyProtoToStream(const NProtoBuf::Message& data, IMetricConsumer* consumer, TInstant ts) { + Y_ENSURE(consumer); + TDecoder(consumer, data, ts).DecodeToStream(); + } +} diff --git a/library/cpp/monlib/encode/legacy_protobuf/legacy_protobuf.h b/library/cpp/monlib/encode/legacy_protobuf/legacy_protobuf.h new file mode 100644 index 0000000000..7cf8985d65 --- /dev/null +++ b/library/cpp/monlib/encode/legacy_protobuf/legacy_protobuf.h @@ -0,0 +1,16 @@ +#pragma once + +#include <google/protobuf/message.h> +#include <util/datetime/base.h> + +namespace NMonitoring { + // Unsupported features of the original format: + // - histograms; + // - memOnly; + // - dropHost/ignorePath + + void DecodeLegacyProto(const NProtoBuf::Message& data, class IMetricConsumer* c, TInstant ts = TInstant::Zero()); + + /// Does not open/close consumer stream unlike the above function. + void DecodeLegacyProtoToStream(const NProtoBuf::Message& data, class IMetricConsumer* c, TInstant ts = TInstant::Zero()); +} diff --git a/library/cpp/monlib/encode/legacy_protobuf/legacy_protobuf_ut.cpp b/library/cpp/monlib/encode/legacy_protobuf/legacy_protobuf_ut.cpp new file mode 100644 index 0000000000..53683cb39c --- /dev/null +++ b/library/cpp/monlib/encode/legacy_protobuf/legacy_protobuf_ut.cpp @@ -0,0 +1,422 @@ +#include "legacy_protobuf.h" + +#include <library/cpp/testing/unittest/registar.h> + +#include <library/cpp/monlib/encode/legacy_protobuf/ut/test_cases.pb.h> +#include <library/cpp/monlib/encode/legacy_protobuf/protos/metric_meta.pb.h> + +#include <library/cpp/monlib/encode/protobuf/protobuf.h> +#include <library/cpp/monlib/encode/text/text.h> +#include <library/cpp/monlib/metrics/labels.h> + +#include <util/generic/algorithm.h> +#include <util/generic/hash_set.h> + +using namespace NMonitoring; + +TSimple MakeSimpleMessage() { + TSimple msg; + + msg.SetFoo(1); + msg.SetBar(2.); + msg.SetBaz(42.); + + return msg; +} + +IMetricEncoderPtr debugPrinter = EncoderText(&Cerr); + +namespace NMonitoring { + inline bool operator<(const TLabel& lhs, const TLabel& rhs) { + return lhs.Name() < rhs.Name() || + (lhs.Name() == rhs.Name() && lhs.Value() < rhs.Value()); + } + +} + +void SetLabelValue(NMonProto::TExtraLabelMetrics::TValue& val, TString s) { + val.SetlabelValue(s); +} + +void SetLabelValue(NMonProto::TExtraLabelMetrics::TValue& val, ui64 u) { + val.SetlabelValueUint(u); +} + +template <typename T, typename V> +NMonProto::TExtraLabelMetrics MakeExtra(TString labelName, V labelValue, T value, bool isDeriv) { + NMonProto::TExtraLabelMetrics metric; + auto* val = metric.Addvalues(); + + metric.SetlabelName(labelName); + SetLabelValue(*val, labelValue); + + if (isDeriv) { + val->SetlongValue(value); + } else { + val->SetdoubleValue(value); + } + + return metric; +} + +void AssertLabels(const TLabels& expected, const NProto::TMultiSample& actual) { + UNIT_ASSERT_EQUAL(actual.LabelsSize(), expected.Size()); + + TSet<TLabel> actualSet; + TSet<TLabel> expectedSet; + Transform(expected.begin(), expected.end(), std::inserter(expectedSet, expectedSet.end()), [] (auto&& l) { + return TLabel{l.Name(), l.Value()}; + }); + + const auto& l = actual.GetLabels(); + Transform(std::begin(l), std::end(l), std::inserter(actualSet, std::begin(actualSet)), + [](auto&& elem) -> TLabel { + return {elem.GetName(), elem.GetValue()}; + }); + + TVector<TLabel> diff; + SetSymmetricDifference(std::begin(expectedSet), std::end(expectedSet), + std::begin(actualSet), std::end(actualSet), std::back_inserter(diff)); + + if (diff.size() > 0) { + for (auto&& l : diff) { + Cerr << l << Endl; + } + + UNIT_FAIL("Labels don't match"); + } +} + +void AssertSimpleMessage(const NProto::TMultiSamplesList& samples, TString pathPrefix = "/") { + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 3); + + THashSet<TString> expectedValues{pathPrefix + "Foo", pathPrefix + "Bar", pathPrefix + "Baz"}; + + for (const auto& s : samples.GetSamples()) { + UNIT_ASSERT_EQUAL(s.LabelsSize(), 1); + UNIT_ASSERT_EQUAL(s.PointsSize(), 1); + + const auto labelVal = s.GetLabels(0).GetValue(); + UNIT_ASSERT(expectedValues.contains(labelVal)); + + if (labelVal == pathPrefix + "Foo") { + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_DOUBLES_EQUAL(s.GetPoints(0).GetFloat64(), 1, 1e-6); + } else if (labelVal == pathPrefix + "Bar") { + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_DOUBLES_EQUAL(s.GetPoints(0).GetFloat64(), 2, 1e-6); + } else if (labelVal == pathPrefix + "Baz") { + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::RATE); + UNIT_ASSERT_EQUAL(s.GetPoints(0).GetUint64(), 42); + } + } +} + +Y_UNIT_TEST_SUITE(TLegacyProtoDecoderTest) { + Y_UNIT_TEST(SimpleProto) { + NProto::TMultiSamplesList samples; + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + auto msg = MakeSimpleMessage(); + DecodeLegacyProto(msg, e.Get()); + + AssertSimpleMessage(samples); + }; + + Y_UNIT_TEST(RepeatedProto) { + NProto::TMultiSamplesList samples; + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + auto simple = MakeSimpleMessage(); + TRepeated msg; + msg.AddMessages()->CopyFrom(simple); + + DecodeLegacyProto(msg, e.Get()); + + AssertSimpleMessage(samples); + } + + Y_UNIT_TEST(RepeatedProtoWithPath) { + NProto::TMultiSamplesList samples; + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + auto simple = MakeSimpleMessage(); + TRepeatedWithPath msg; + msg.AddNamespace()->CopyFrom(simple); + + DecodeLegacyProto(msg, e.Get()); + + AssertSimpleMessage(samples, "/Namespace/"); + } + + Y_UNIT_TEST(DeepNesting) { + NProto::TMultiSamplesList samples; + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + auto simple = MakeSimpleMessage(); + TRepeatedWithPath internal; + internal.AddNamespace()->CopyFrom(simple); + + TDeepNesting msg; + msg.MutableNested()->CopyFrom(internal); + + DecodeLegacyProto(msg, e.Get()); + + AssertSimpleMessage(samples, "/Namespace/"); + } + + Y_UNIT_TEST(Keys) { + NProto::TMultiSamplesList samples; + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + auto simple = MakeSimpleMessage(); + simple.SetLabel("my_label_value"); + + TNestedWithKeys msg; + msg.AddNamespace()->CopyFrom(simple); + + DecodeLegacyProto(msg, e.Get()); + + auto i = 0; + for (const auto& s : samples.GetSamples()) { + UNIT_ASSERT_EQUAL(s.LabelsSize(), 4); + + bool foundLabel = false; + bool foundFixed = false; + bool foundNumbered = false; + + for (const auto& label : s.GetLabels()) { + if (label.GetName() == "my_label") { + foundLabel = true; + UNIT_ASSERT_STRINGS_EQUAL(label.GetValue(), "my_label_value"); + } else if (label.GetName() == "fixed_label") { + foundFixed = true; + UNIT_ASSERT_STRINGS_EQUAL(label.GetValue(), "fixed_value"); + } else if (label.GetName() == "numbered") { + foundNumbered = true; + UNIT_ASSERT_STRINGS_EQUAL(label.GetValue(), ::ToString(i)); + } + } + + UNIT_ASSERT(foundLabel); + UNIT_ASSERT(foundFixed); + UNIT_ASSERT(foundNumbered); + } + } + + Y_UNIT_TEST(NonStringKeys) { + NProto::TMultiSamplesList samples; + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + TNonStringKeys msg; + msg.SetFoo(42); + msg.SetEnum(ENUM); + msg.SetInt(43); + + TRepeatedNonStringKeys msgs; + msgs.AddNested()->CopyFrom(msg); + + DecodeLegacyProto(msgs, e.Get()); + + for (const auto& s : samples.GetSamples()) { + bool foundEnum = false; + bool foundInt = false; + + for (const auto& label : s.GetLabels()) { + if (label.GetName() == "enum") { + foundEnum = true; + UNIT_ASSERT_STRINGS_EQUAL(label.GetValue(), "ENUM"); + } else if (label.GetName() == "int") { + foundInt = true; + UNIT_ASSERT_STRINGS_EQUAL(label.GetValue(), "43"); + } + } + + UNIT_ASSERT(foundEnum); + UNIT_ASSERT(foundInt); + + UNIT_ASSERT_DOUBLES_EQUAL(s.GetPoints(0).GetFloat64(), 42, 1e-6); + } + } + + Y_UNIT_TEST(KeysFromNonLeafNodes) { + NProto::TMultiSamplesList samples; + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + auto simple = MakeSimpleMessage(); + simple.SetLabel("label_value"); + + TRepeatedWithName nested; + nested.SetName("my_name"); + nested.AddNested()->CopyFrom(simple); + + TKeysFromNonLeaf msg; + msg.AddNested()->CopyFrom(nested); + + DecodeLegacyProto(msg, e.Get()); + + AssertLabels({{"my_label", "label_value"}, {"path", "/Nested/Nested/Foo"}, {"name", "my_name"}}, samples.GetSamples(0)); + } + + Y_UNIT_TEST(SpacesAreGetReplaced) { + NProto::TMultiSamplesList samples; + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + auto simple = MakeSimpleMessage(); + simple.SetLabel("my label_value"); + + TNestedWithKeys msg; + msg.AddNamespace()->CopyFrom(simple); + + DecodeLegacyProto(msg, e.Get()); + + for (const auto& s : samples.GetSamples()) { + UNIT_ASSERT_EQUAL(s.LabelsSize(), 4); + + bool foundLabel = false; + + for (const auto& label : s.GetLabels()) { + if (label.GetName() == "my_label") { + foundLabel = true; + UNIT_ASSERT_STRINGS_EQUAL(label.GetValue(), "my_label_value"); + } + } + + UNIT_ASSERT(foundLabel); + } + } + + Y_UNIT_TEST(ExtraLabels) { + NProto::TMultiSamplesList samples; + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + TExtraLabels msg; + msg.MutableExtraAsIs()->CopyFrom(MakeExtra("label", "foo", 42, false)); + msg.MutableExtraDeriv()->CopyFrom(MakeExtra("deriv_label", "deriv_foo", 43, true)); + + DecodeLegacyProto(msg, e.Get()); + + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 2); + { + auto s = samples.GetSamples(0); + AssertLabels({{"label", "foo"}, {"path", "/ExtraAsIs"}}, s); + + UNIT_ASSERT_EQUAL(s.PointsSize(), 1); + auto point = s.GetPoints(0); + UNIT_ASSERT_DOUBLES_EQUAL(point.GetFloat64(), 42, 1e-6); + } + + { + auto s = samples.GetSamples(1); + AssertLabels({{"deriv_label", "deriv_foo"}, {"path", "/ExtraDeriv"}}, s); + + UNIT_ASSERT_EQUAL(s.PointsSize(), 1); + auto point = s.GetPoints(0); + UNIT_ASSERT_EQUAL(point.GetUint64(), 43); + } + } + + Y_UNIT_TEST(NestedExtraLabels) { + NProto::TMultiSamplesList samples; + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + TExtraLabels msg; + auto extra = MakeExtra("label", "foo", 42, false); + auto* val = extra.Mutablevalues(0); + { + auto child = MakeExtra("child1", "label1", 24, true); + child.Mutablevalues(0)->Settype(NMonProto::EMetricType::RATE); + val->Addchildren()->CopyFrom(child); + } + + { + auto child = MakeExtra("child2", 34, 23, false); + val->Addchildren()->CopyFrom(child); + } + msg.MutableExtraAsIs()->CopyFrom(extra); + + DecodeLegacyProto(msg, e.Get()); + + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 3); + { + auto s = samples.GetSamples(0); + AssertLabels({{"label", "foo"}, {"path", "/ExtraAsIs"}}, s); + + UNIT_ASSERT_EQUAL(s.PointsSize(), 1); + auto point = s.GetPoints(0); + UNIT_ASSERT_DOUBLES_EQUAL(point.GetFloat64(), 42, 1e-6); + } + + { + auto s = samples.GetSamples(1); + AssertLabels({{"label", "foo"}, {"child1", "label1"}, {"path", "/ExtraAsIs"}}, s); + + UNIT_ASSERT_EQUAL(s.PointsSize(), 1); + auto point = s.GetPoints(0); + UNIT_ASSERT_EQUAL(point.GetUint64(), 24); + } + + { + auto s = samples.GetSamples(2); + AssertLabels({{"label", "foo"}, {"child2", "34"}, {"path", "/ExtraAsIs"}}, s); + + UNIT_ASSERT_EQUAL(s.PointsSize(), 1); + auto point = s.GetPoints(0); + UNIT_ASSERT_EQUAL(point.GetFloat64(), 23); + } + } + + Y_UNIT_TEST(RobotLabels) { + NProto::TMultiSamplesList samples; + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + TNamedCounter responses; + responses.SetName("responses"); + responses.SetCount(42); + + TCrawlerCounters::TStatusCounters statusCounters; + statusCounters.AddZoraResponses()->CopyFrom(responses); + + TCrawlerCounters::TPolicyCounters policyCounters; + policyCounters.SetSubComponent("mySubComponent"); + policyCounters.SetName("myComponentName"); + policyCounters.SetZone("myComponentZone"); + policyCounters.MutableStatusCounters()->CopyFrom(statusCounters); + + TCrawlerCounters counters; + counters.SetComponent("myComponent"); + counters.AddPoliciesCounters()->CopyFrom(policyCounters); + + DecodeLegacyProto(counters, e.Get()); + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 1); + auto s = samples.GetSamples(0); + AssertLabels({ + {"SubComponent", "mySubComponent"}, {"Policy", "myComponentName"}, {"Zone", "myComponentZone"}, + {"ZoraResponse", "responses"}, {"path", "/PoliciesCounters/StatusCounters/ZoraResponses/Count"}}, s); + } + + Y_UNIT_TEST(ZoraLabels) { + NProto::TMultiSamplesList samples; + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + TTimeLogHist hist; + hist.AddBuckets(42); + hist.AddBuckets(0); + + TKiwiCounters counters; + counters.MutableTimes()->CopyFrom(hist); + + DecodeLegacyProto(counters, e.Get()); + + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 2); + + auto s = samples.GetSamples(0); + AssertLabels({{"slot", "0"}, {"path", "/Times/Buckets"}}, s); + UNIT_ASSERT_EQUAL(s.PointsSize(), 1); + UNIT_ASSERT_EQUAL(s.GetPoints(0).GetUint64(), 42); + + s = samples.GetSamples(1); + AssertLabels({{"slot", "1"}, {"path", "/Times/Buckets"}}, s); + UNIT_ASSERT_EQUAL(s.GetPoints(0).GetUint64(), 0); + } +} diff --git a/library/cpp/monlib/encode/legacy_protobuf/protos/metric_meta.proto b/library/cpp/monlib/encode/legacy_protobuf/protos/metric_meta.proto new file mode 100644 index 0000000000..fd23eb372b --- /dev/null +++ b/library/cpp/monlib/encode/legacy_protobuf/protos/metric_meta.proto @@ -0,0 +1,73 @@ +import "google/protobuf/descriptor.proto"; + +package NMonProto; + +option java_package = "ru.yandex.monlib.proto"; +option java_outer_classname = "MetricMetaProto"; + +enum EMetricType { + GAUGE = 1; + RATE = 2; +} + +enum EMemOnly { + DEFAULT = 0; + STORE = 1; + MEM_ONLY = 2; +} + +message TMetricMeta { + optional EMetricType Type = 1; + optional bool Path = 2; + optional string Keys = 3; + optional bool MemOnly = 4; + optional bool IgnorePath = 5; + optional string CustomPath = 6; +} + +enum THistogramBase { + MICROSECOND = 3; + MILLISECOND = 6; + SECOND = 9; + MINUTE = 12; + HOUR = 15; +} + +message THistogramEntry { + optional uint64 Multiplier = 1; + optional double Value = 2; +} + +message THistogram { + optional THistogramBase Base = 1; + optional string BaseStr = 2; + repeated THistogramEntry Entries = 5; +} + +// field of this type is recognized by Solomon +message TExtraLabelMetrics { + optional string labelName = 1; + + message TValue { + optional string labelValue = 1; + // used only if != 0 + optional uint64 labelValueUint = 21; + + optional uint64 longValue = 2; + optional double doubleValue = 3; + optional THistogram histogramValue = 4; + + optional EMetricType type = 7; + optional EMemOnly memOnly = 8; + optional bool dropHost = 9; + + repeated TExtraLabelMetrics children = 17; + } + + repeated TValue values = 2; +} + +extend google.protobuf.FieldOptions { + optional TMetricMeta Metric = 1719; +} + diff --git a/library/cpp/monlib/encode/legacy_protobuf/protos/python/ya.make b/library/cpp/monlib/encode/legacy_protobuf/protos/python/ya.make new file mode 100644 index 0000000000..095b307b01 --- /dev/null +++ b/library/cpp/monlib/encode/legacy_protobuf/protos/python/ya.make @@ -0,0 +1,3 @@ +OWNER(g:solomon) + +PY_PROTOS_FOR(library/cpp/monlib/encode/legacy_protobuf/protos) diff --git a/library/cpp/monlib/encode/legacy_protobuf/protos/ya.make b/library/cpp/monlib/encode/legacy_protobuf/protos/ya.make new file mode 100644 index 0000000000..489f361ab1 --- /dev/null +++ b/library/cpp/monlib/encode/legacy_protobuf/protos/ya.make @@ -0,0 +1,13 @@ +PROTO_LIBRARY() + +OWNER(g:solomon) + +SRCS( + metric_meta.proto +) + +IF (NOT PY_PROTOS_FOR) + EXCLUDE_TAGS(GO_PROTO) +ENDIF() + +END() diff --git a/library/cpp/monlib/encode/legacy_protobuf/ut/test_cases.proto b/library/cpp/monlib/encode/legacy_protobuf/ut/test_cases.proto new file mode 100644 index 0000000000..37e901de48 --- /dev/null +++ b/library/cpp/monlib/encode/legacy_protobuf/ut/test_cases.proto @@ -0,0 +1,90 @@ +import "library/cpp/monlib/encode/legacy_protobuf/protos/metric_meta.proto"; + +message TSimple { + optional uint64 Foo = 1 [ (NMonProto.Metric).Type = GAUGE ]; + optional double Bar = 2 [ (NMonProto.Metric).Type = GAUGE ]; + optional double Baz = 3 [ (NMonProto.Metric).Type = RATE ]; + optional string Label = 4; +} + +message TRepeated { + repeated TSimple Messages = 1 [ (NMonProto.Metric).Path = false ]; +}; + +message TRepeatedWithPath { + repeated TSimple Namespace = 1 [ (NMonProto.Metric).Path = true ]; +}; + +message TNestedWithKeys { + repeated TSimple Namespace = 1 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "my_label:Label fixed_label=fixed_value numbered:#" ]; +}; + +message TDeepNesting { + optional TRepeatedWithPath Nested = 1 [ (NMonProto.Metric).Path = false ]; +}; + +enum EEnum { + MY = 1; + ENUM = 2; +}; + +message TNonStringKeys { + optional uint32 Foo = 1 [ (NMonProto.Metric).Type = GAUGE ]; + optional EEnum Enum = 2; + optional uint32 Int = 3; +}; + +message TRepeatedNonStringKeys { + repeated TNonStringKeys Nested = 1 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "enum:Enum int:Int" ]; +}; + +message TExtraLabels { + optional NMonProto.TExtraLabelMetrics ExtraAsIs = 1 [ (NMonProto.Metric).Type = GAUGE ]; + optional NMonProto.TExtraLabelMetrics ExtraDeriv = 2 [ (NMonProto.Metric).Type = RATE ]; +}; + +message TRepeatedWithName { + optional string Name = 1; + repeated TSimple Nested = 2 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "my_label:Label" ]; +}; + +message TKeysFromNonLeaf { + repeated TRepeatedWithName Nested = 1 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "name:Name" ]; +}; + + +message TNamedCounter { + optional string Name = 1; + optional uint64 Count = 2 [ (NMonProto.Metric).Type = RATE ]; +} + +message TCrawlerCounters { + message TStatusCounters { + repeated TNamedCounter ZoraResponses = 3 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "ZoraResponse:Name" ]; + repeated TNamedCounter FetcherResponses = 4 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "SpiderResponse:Name" ]; + repeated TNamedCounter RotorResponses = 5 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "SpiderResponse:Name" ]; + repeated TNamedCounter PDFetchResponses = 6 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "PDFetchResponse:Name" ]; + repeated TNamedCounter CalcResponses = 7 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "CalcResponse:Name" ]; + repeated TNamedCounter Responses = 8 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "AggregatedResponse:Name" ]; + } + + message TPolicyCounters { + optional string SubComponent = 1; + optional string Name = 2; + optional string Zone = 3; + + optional TStatusCounters StatusCounters = 4 [ (NMonProto.Metric).Path = true ]; + } + + optional string Component = 1; + repeated TPolicyCounters PoliciesCounters = 3 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "SubComponent:SubComponent Policy:Name Zone:Zone" ]; +} + +message TTimeLogHist { + optional uint32 MinBucketMillisec = 1; + repeated uint64 Buckets = 2 [ (NMonProto.Metric).Type = RATE, (NMonProto.Metric).Keys = "slot:#" ]; +} + +message TKiwiCounters { + optional TTimeLogHist Times = 22 [ (NMonProto.Metric).Path = true ]; +} diff --git a/library/cpp/monlib/encode/legacy_protobuf/ut/ya.make b/library/cpp/monlib/encode/legacy_protobuf/ut/ya.make new file mode 100644 index 0000000000..479a0c46c9 --- /dev/null +++ b/library/cpp/monlib/encode/legacy_protobuf/ut/ya.make @@ -0,0 +1,18 @@ +UNITTEST_FOR(library/cpp/monlib/encode/legacy_protobuf) + +OWNER( + g:solomon + msherbakov +) + +SRCS( + legacy_protobuf_ut.cpp + test_cases.proto +) + +PEERDIR( + library/cpp/monlib/encode/protobuf + library/cpp/monlib/encode/text +) + +END() diff --git a/library/cpp/monlib/encode/legacy_protobuf/ya.make b/library/cpp/monlib/encode/legacy_protobuf/ya.make new file mode 100644 index 0000000000..74c82aac93 --- /dev/null +++ b/library/cpp/monlib/encode/legacy_protobuf/ya.make @@ -0,0 +1,16 @@ +LIBRARY() + +OWNER( + g:solomon + msherbakov +) + +SRCS( + legacy_proto_decoder.cpp +) + +PEERDIR( + library/cpp/monlib/encode/legacy_protobuf/protos +) + +END() diff --git a/library/cpp/monlib/encode/prometheus/fuzz/main.cpp b/library/cpp/monlib/encode/prometheus/fuzz/main.cpp new file mode 100644 index 0000000000..24bda2d32e --- /dev/null +++ b/library/cpp/monlib/encode/prometheus/fuzz/main.cpp @@ -0,0 +1,18 @@ +#include <library/cpp/monlib/encode/prometheus/prometheus.h> +#include <library/cpp/monlib/encode/fake/fake.h> + +#include <util/stream/mem.h> + + +extern "C" int LLVMFuzzerTestOneInput(const ui8* buf, size_t size) { + using namespace NMonitoring; + + try { + TStringBuf data(reinterpret_cast<const char*>(buf), size); + auto encoder = EncoderFake(); + DecodePrometheus(data, encoder.Get()); + } catch (...) { + } + + return 0; +} diff --git a/library/cpp/monlib/encode/prometheus/fuzz/ya.make b/library/cpp/monlib/encode/prometheus/fuzz/ya.make new file mode 100644 index 0000000000..4a6c796ed5 --- /dev/null +++ b/library/cpp/monlib/encode/prometheus/fuzz/ya.make @@ -0,0 +1,16 @@ +FUZZ() + +OWNER(g:solomon jamel) + +PEERDIR( + library/cpp/monlib/encode/prometheus + library/cpp/monlib/encode/fake +) + +SIZE(MEDIUM) + +SRCS( + main.cpp +) + +END() diff --git a/library/cpp/monlib/encode/prometheus/prometheus.h b/library/cpp/monlib/encode/prometheus/prometheus.h new file mode 100644 index 0000000000..2e7fa31c28 --- /dev/null +++ b/library/cpp/monlib/encode/prometheus/prometheus.h @@ -0,0 +1,18 @@ +#pragma once + +#include <library/cpp/monlib/encode/encoder.h> +#include <library/cpp/monlib/encode/format.h> + +#include <util/generic/yexception.h> + + +namespace NMonitoring { + + class TPrometheusDecodeException: public yexception { + }; + + IMetricEncoderPtr EncoderPrometheus(IOutputStream* out, TStringBuf metricNameLabel = "sensor"); + + void DecodePrometheus(TStringBuf data, IMetricConsumer* c, TStringBuf metricNameLabel = "sensor"); + +} diff --git a/library/cpp/monlib/encode/prometheus/prometheus_decoder.cpp b/library/cpp/monlib/encode/prometheus/prometheus_decoder.cpp new file mode 100644 index 0000000000..7e81357dbd --- /dev/null +++ b/library/cpp/monlib/encode/prometheus/prometheus_decoder.cpp @@ -0,0 +1,597 @@ +#include "prometheus.h" +#include "prometheus_model.h" + +#include <library/cpp/monlib/metrics/histogram_snapshot.h> +#include <library/cpp/monlib/metrics/metric.h> + +#include <util/datetime/base.h> +#include <util/generic/hash.h> +#include <util/string/cast.h> +#include <util/string/builder.h> +#include <util/generic/maybe.h> +#include <util/string/ascii.h> + +#include <cmath> + +#define Y_PARSER_FAIL(message) \ + ythrow ::NMonitoring::TPrometheusDecodeException() << message << " at line #" << CurrentLine_ + +#define Y_PARSER_ENSURE(cond, message) \ + Y_ENSURE_EX(cond, ::NMonitoring::TPrometheusDecodeException() << message << " at line #" << CurrentLine_) + + +namespace NMonitoring { + namespace { + constexpr ui32 MAX_LABEL_VALUE_LEN = 256; + + using TLabelsMap = THashMap<TString, TString>; + + TString LabelsToStr(const TLabelsMap& labels) { + TStringBuilder sb; + auto it = labels.begin(); + auto end = labels.end(); + + sb << '{'; + while (it != end) { + sb << it->first; + sb << '='; + sb << '"' << it->second << '"'; + + ++it; + if (it != end) { + sb << ", "; + } + } + sb << '}'; + return sb; + } + + template <typename T, typename U> + bool TryStaticCast(U val, T& out) { + static_assert(std::is_arithmetic_v<U>); + if constexpr (std::is_floating_point_v<T> || std::is_floating_point_v<U>) { + if (val > MaxFloor<T>() || val < -MaxFloor<T>()) { + return false; + } + + } else { + if (val > Max<T>() || val < Min<T>()) { + return false; + } + } + + out = static_cast<T>(val); + return true; + } + + /////////////////////////////////////////////////////////////////////// + // THistogramBuilder + /////////////////////////////////////////////////////////////////////// + class THistogramBuilder { + using TBucketData = std::pair<TBucketBound, TBucketValue>; + constexpr static TBucketData ZERO_BUCKET = { -std::numeric_limits<TBucketBound>::max(), 0 }; + public: + TStringBuf GetName() const noexcept { + return Name_; + } + + void SetName(TStringBuf name) noexcept { + Name_ = name; + } + + const TLabelsMap& GetLabels() const noexcept { + return *Labels_; + } + + void SetLabels(TLabelsMap&& labels) { + if (Labels_.Defined()) { + Y_ENSURE(Labels_ == labels, + "mixed labels in one histogram, prev: " << LabelsToStr(*Labels_) << + ", current: " << LabelsToStr(labels)); + } else { + Labels_.ConstructInPlace(std::move(labels)); + } + } + + TInstant GetTime() const noexcept { + return Time_; + } + + void SetTime(TInstant time) noexcept { + Time_ = time; + } + + bool Empty() const noexcept { + return Bounds_.empty(); + } + + bool Same(TStringBuf name, const TLabelsMap& labels) const noexcept { + return Name_ == name && Labels_ == labels; + } + + void AddBucket(TBucketBound bound, TBucketValue value) { + Y_ENSURE_EX(PrevBucket_.first < bound, TPrometheusDecodeException() << + "invalid order of histogram bounds " << PrevBucket_.first << + " >= " << bound); + + Y_ENSURE_EX(PrevBucket_.second <= value, TPrometheusDecodeException() << + "invalid order of histogram bucket values " << PrevBucket_.second << + " > " << value); + + // convert infinite bound value + if (bound == std::numeric_limits<TBucketBound>::infinity()) { + bound = HISTOGRAM_INF_BOUND; + } + + Bounds_.push_back(bound); + Values_.push_back(value - PrevBucket_.second); // keep only delta between buckets + + PrevBucket_ = { bound, value }; + } + + // will clear builder state + IHistogramSnapshotPtr ToSnapshot() { + Y_ENSURE_EX(!Empty(), TPrometheusDecodeException() << "histogram cannot be empty"); + Time_ = TInstant::Zero(); + PrevBucket_ = ZERO_BUCKET; + Labels_.Clear(); + auto snapshot = ExplicitHistogramSnapshot(Bounds_, Values_); + + Bounds_.clear(); + Values_.clear(); + + return snapshot; + } + + private: + TStringBuf Name_; + TMaybe<TLabelsMap> Labels_; + TInstant Time_; + TBucketBounds Bounds_; + TBucketValues Values_; + TBucketData PrevBucket_ = ZERO_BUCKET; + }; + + /////////////////////////////////////////////////////////////////////// + // EPrometheusMetricType + /////////////////////////////////////////////////////////////////////// + enum class EPrometheusMetricType { + GAUGE, + COUNTER, + SUMMARY, + UNTYPED, + HISTOGRAM, + }; + + /////////////////////////////////////////////////////////////////////// + // TPrometheusReader + /////////////////////////////////////////////////////////////////////// + class TPrometheusReader { + public: + TPrometheusReader(TStringBuf data, IMetricConsumer* c, TStringBuf metricNameLabel) + : Data_(data) + , Consumer_(c) + , MetricNameLabel_(metricNameLabel) + { + } + + void Read() { + Consumer_->OnStreamBegin(); + + if (HasRemaining()) { + ReadNextByte(); + SkipSpaces(); + + try { + while (HasRemaining()) { + switch (CurrentByte_) { + case '\n': + ReadNextByte(); // skip '\n' + CurrentLine_++; + SkipSpaces(); + break; + case '#': + ParseComment(); + break; + default: + ParseMetric(); + break; + } + } + + if (!HistogramBuilder_.Empty()) { + ConsumeHistogram(); + } + } catch (const TPrometheusDecodeException& e) { + throw e; + } catch (...) { + Y_PARSER_FAIL("unexpected error " << CurrentExceptionMessage()); + } + } + + Consumer_->OnStreamEnd(); + } + + private: + bool HasRemaining() const noexcept { + return CurrentPos_ < Data_.Size(); + } + + // # 'TYPE' metric_name {counter|gauge|histogram|summary|untyped} + // # 'HELP' metric_name some help info + // # general comment message + void ParseComment() { + SkipExpectedChar('#'); + SkipSpaces(); + + TStringBuf keyword = ReadToken(); + if (keyword == TStringBuf("TYPE")) { + SkipSpaces(); + + TStringBuf nextName = ReadTokenAsMetricName(); + Y_PARSER_ENSURE(!nextName.Empty(), "invalid metric name"); + + SkipSpaces(); + EPrometheusMetricType nextType = ReadType(); + + bool inserted = SeenTypes_.emplace(nextName, nextType).second; + Y_PARSER_ENSURE(inserted, "second TYPE line for metric " << nextName); + + if (nextType == EPrometheusMetricType::HISTOGRAM) { + if (!HistogramBuilder_.Empty()) { + ConsumeHistogram(); + } + HistogramBuilder_.SetName(nextName); + } + } else { + // skip HELP and general comments + SkipUntilEol(); + } + + Y_PARSER_ENSURE(CurrentByte_ == '\n', "expected '\\n', found '" << CurrentByte_ << '\''); + } + + // metric_name [labels] value [timestamp] + void ParseMetric() { + TStringBuf name = ReadTokenAsMetricName(); + SkipSpaces(); + + TLabelsMap labels = ReadLabels(); + SkipSpaces(); + + double value = ParseGoDouble(ReadToken()); + SkipSpaces(); + + TInstant time = TInstant::Zero(); + if (CurrentByte_ != '\n') { + time = TInstant::MilliSeconds(FromString<ui64>(ReadToken())); + } + + TStringBuf baseName = name; + EPrometheusMetricType type = EPrometheusMetricType::UNTYPED; + + if (auto* seenType = SeenTypes_.FindPtr(name)) { + type = *seenType; + } else { + baseName = NPrometheus::ToBaseName(name); + if (auto* baseType = SeenTypes_.FindPtr(baseName)) { + type = *baseType; + } + } + + switch (type) { + case EPrometheusMetricType::HISTOGRAM: + if (NPrometheus::IsBucket(name)) { + double bound = 0.0; + auto it = labels.find(NPrometheus::BUCKET_LABEL); + if (it != labels.end()) { + bound = ParseGoDouble(it->second); + labels.erase(it); + } else { + Y_PARSER_FAIL( + "metric " << name << "has no " << NPrometheus::BUCKET_LABEL << + " label at line #" << CurrentLine_); + } + + if (!HistogramBuilder_.Empty() && !HistogramBuilder_.Same(baseName, labels)) { + ConsumeHistogram(); + HistogramBuilder_.SetName(baseName); + } + + TBucketValue bucketVal; + Y_PARSER_ENSURE(TryStaticCast(value, bucketVal), "Cannot convert " << value << " to bucket value type"); + HistogramBuilder_.AddBucket(bound, bucketVal); + HistogramBuilder_.SetTime(time); + HistogramBuilder_.SetLabels(std::move(labels)); + } else if (NPrometheus::IsCount(name)) { + // translate x_count metric as COUNTER metric + ConsumeCounter(name, labels, time, value); + } else if (NPrometheus::IsSum(name)) { + // translate x_sum metric as GAUGE metric + ConsumeGauge(name, labels, time, value); + } else { + Y_PARSER_FAIL( + "metric " << name << + " should be part of HISTOGRAM " << baseName); + } + break; + + case EPrometheusMetricType::SUMMARY: + if (NPrometheus::IsCount(name)) { + // translate x_count metric as COUNTER metric + ConsumeCounter(name, labels, time, value); + } else if (NPrometheus::IsSum(name)) { + // translate x_sum metric as GAUGE metric + ConsumeGauge(name, labels, time, value); + } else { + ConsumeGauge(name, labels, time, value); + } + break; + + case EPrometheusMetricType::COUNTER: + ConsumeCounter(name, labels, time, value); + break; + + case EPrometheusMetricType::GAUGE: + ConsumeGauge(name, labels, time, value); + break; + + case EPrometheusMetricType::UNTYPED: + ConsumeGauge(name, labels, time, value); + break; + } + + Y_PARSER_ENSURE(CurrentByte_ == '\n', "expected '\\n', found '" << CurrentByte_ << '\''); + } + + // { name = "value", name2 = "value2", } + TLabelsMap ReadLabels() { + TLabelsMap labels; + if (CurrentByte_ != '{') { + return labels; + } + + SkipExpectedChar('{'); + SkipSpaces(); + + while (CurrentByte_ != '}') { + TStringBuf name = ReadTokenAsLabelName(); + SkipSpaces(); + + SkipExpectedChar('='); + SkipSpaces(); + + TString value = ReadTokenAsLabelValue(); + SkipSpaces(); + labels.emplace(name, value); + + if (CurrentByte_ == ',') { + SkipExpectedChar(','); + SkipSpaces(); + } + } + + SkipExpectedChar('}'); + return labels; + } + + EPrometheusMetricType ReadType() { + TStringBuf keyword = ReadToken(); + if (AsciiEqualsIgnoreCase(keyword, "GAUGE")) { + return EPrometheusMetricType::GAUGE; + } else if (AsciiEqualsIgnoreCase(keyword, "COUNTER")) { + return EPrometheusMetricType::COUNTER; + } else if (AsciiEqualsIgnoreCase(keyword, "SUMMARY")) { + return EPrometheusMetricType::SUMMARY; + } else if (AsciiEqualsIgnoreCase(keyword, "HISTOGRAM")) { + return EPrometheusMetricType::HISTOGRAM; + } else if (AsciiEqualsIgnoreCase(keyword, "UNTYPED")) { + return EPrometheusMetricType::UNTYPED; + } + + Y_PARSER_FAIL( + "unknown metric type: " << keyword << + " at line #" << CurrentLine_); + } + + Y_FORCE_INLINE void ReadNextByteUnsafe() { + CurrentByte_ = Data_[CurrentPos_++]; + } + + Y_FORCE_INLINE bool IsSpace(char ch) { + return ch == ' ' || ch == '\t'; + } + + void ReadNextByte() { + Y_PARSER_ENSURE(HasRemaining(), "unexpected end of file"); + ReadNextByteUnsafe(); + } + + void SkipExpectedChar(char ch) { + Y_PARSER_ENSURE(CurrentByte_ == ch, + "expected '" << CurrentByte_ << "', found '" << ch << '\''); + ReadNextByte(); + } + + void SkipSpaces() { + while (HasRemaining() && IsSpace(CurrentByte_)) { + ReadNextByteUnsafe(); + } + } + + void SkipUntilEol() { + while (HasRemaining() && CurrentByte_ != '\n') { + ReadNextByteUnsafe(); + } + } + + TStringBuf ReadToken() { + Y_VERIFY_DEBUG(CurrentPos_ > 0); + size_t begin = CurrentPos_ - 1; // read first byte again + while (HasRemaining() && !IsSpace(CurrentByte_) && CurrentByte_ != '\n') { + ReadNextByteUnsafe(); + } + return TokenFromPos(begin); + } + + TStringBuf ReadTokenAsMetricName() { + if (!NPrometheus::IsValidMetricNameStart(CurrentByte_)) { + return ""; + } + + Y_VERIFY_DEBUG(CurrentPos_ > 0); + size_t begin = CurrentPos_ - 1; // read first byte again + while (HasRemaining()) { + ReadNextByteUnsafe(); + if (!NPrometheus::IsValidMetricNameContinuation(CurrentByte_)) { + break; + } + } + return TokenFromPos(begin); + } + + TStringBuf ReadTokenAsLabelName() { + if (!NPrometheus::IsValidLabelNameStart(CurrentByte_)) { + return ""; + } + + Y_VERIFY_DEBUG(CurrentPos_ > 0); + size_t begin = CurrentPos_ - 1; // read first byte again + while (HasRemaining()) { + ReadNextByteUnsafe(); + if (!NPrometheus::IsValidLabelNameContinuation(CurrentByte_)) { + break; + } + } + return TokenFromPos(begin); + } + + TString ReadTokenAsLabelValue() { + TString labelValue; + + SkipExpectedChar('"'); + for (ui32 i = 0; i < MAX_LABEL_VALUE_LEN; i++) { + switch (CurrentByte_) { + case '"': + SkipExpectedChar('"'); + return labelValue; + + case '\n': + Y_PARSER_FAIL("label value contains unescaped new-line"); + + case '\\': + ReadNextByte(); + switch (CurrentByte_) { + case '"': + case '\\': + labelValue.append(CurrentByte_); + break; + case 'n': + labelValue.append('\n'); + break; + default: + Y_PARSER_FAIL("invalid escape sequence '" << CurrentByte_ << '\''); + } + break; + + default: + labelValue.append(CurrentByte_); + break; + } + + ReadNextByte(); + } + + Y_PARSER_FAIL("trying to parse too long label value, size >= " << MAX_LABEL_VALUE_LEN); + } + + TStringBuf TokenFromPos(size_t begin) { + Y_VERIFY_DEBUG(CurrentPos_ > begin); + size_t len = CurrentPos_ - begin - 1; + if (len == 0) { + return {}; + } + + return Data_.SubString(begin, len); + } + + void ConsumeLabels(TStringBuf name, const TLabelsMap& labels) { + Y_PARSER_ENSURE(labels.count(MetricNameLabel_) == 0, + "label name '" << MetricNameLabel_ << + "' is reserved, but is used with metric: " << name << LabelsToStr(labels)); + + Consumer_->OnLabelsBegin(); + Consumer_->OnLabel(MetricNameLabel_, TString(name)); // TODO: remove this string allocation + for (const auto& it: labels) { + Consumer_->OnLabel(it.first, it.second); + } + Consumer_->OnLabelsEnd(); + } + + void ConsumeCounter(TStringBuf name, const TLabelsMap& labels, TInstant time, double value) { + i64 intValue{0}; + // not nan + if (value == value) { + Y_PARSER_ENSURE(TryStaticCast(value, intValue), "value " << value << " is out of range"); + } + + // see https://st.yandex-team.ru/SOLOMON-4142 for more details + // why we convert Prometheus COUNTER into Solomon RATE + // TODO: need to fix after server-side aggregation become correct for COUNTERs + Consumer_->OnMetricBegin(EMetricType::RATE); + ConsumeLabels(name, labels); + Consumer_->OnUint64(time, intValue); + Consumer_->OnMetricEnd(); + } + + void ConsumeGauge(TStringBuf name, const TLabelsMap& labels, TInstant time, double value) { + Consumer_->OnMetricBegin(EMetricType::GAUGE); + ConsumeLabels(name, labels); + Consumer_->OnDouble(time, value); + Consumer_->OnMetricEnd(); + } + + void ConsumeHistogram() { + Consumer_->OnMetricBegin(EMetricType::HIST_RATE); + ConsumeLabels(HistogramBuilder_.GetName(), HistogramBuilder_.GetLabels()); + auto time = HistogramBuilder_.GetTime(); + auto hist = HistogramBuilder_.ToSnapshot(); + Consumer_->OnHistogram(time, std::move(hist)); + Consumer_->OnMetricEnd(); + } + + double ParseGoDouble(TStringBuf str) { + if (str == TStringBuf("+Inf")) { + return std::numeric_limits<double>::infinity(); + } else if (str == TStringBuf("-Inf")) { + return -std::numeric_limits<double>::infinity(); + } else if (str == TStringBuf("NaN")) { + return NAN; + } + + double r = 0.0; + if (TryFromString(str, r)) { + return r; + } + Y_PARSER_FAIL("cannot parse double value from '" << str << "\' at line #" << CurrentLine_); + } + + private: + TStringBuf Data_; + IMetricConsumer* Consumer_; + TStringBuf MetricNameLabel_; + THashMap<TString, EPrometheusMetricType> SeenTypes_; + THistogramBuilder HistogramBuilder_; + + ui32 CurrentLine_ = 1; + ui32 CurrentPos_ = 0; + char CurrentByte_ = 0; + }; + } // namespace + +void DecodePrometheus(TStringBuf data, IMetricConsumer* c, TStringBuf metricNameLabel) { + TPrometheusReader reader(data, c, metricNameLabel); + reader.Read(); +} + +} // namespace NMonitoring diff --git a/library/cpp/monlib/encode/prometheus/prometheus_decoder_ut.cpp b/library/cpp/monlib/encode/prometheus/prometheus_decoder_ut.cpp new file mode 100644 index 0000000000..49c2244fb4 --- /dev/null +++ b/library/cpp/monlib/encode/prometheus/prometheus_decoder_ut.cpp @@ -0,0 +1,478 @@ +#include "prometheus.h" + +#include <library/cpp/monlib/encode/protobuf/protobuf.h> + +#include <library/cpp/testing/unittest/registar.h> + +using namespace NMonitoring; + +#define ASSERT_LABEL_EQUAL(label, name, value) do { \ + UNIT_ASSERT_STRINGS_EQUAL((label).GetName(), name); \ + UNIT_ASSERT_STRINGS_EQUAL((label).GetValue(), value); \ + } while (false) + +#define ASSERT_DOUBLE_POINT(s, time, value) do { \ + UNIT_ASSERT_VALUES_EQUAL((s).GetTime(), (time).MilliSeconds()); \ + UNIT_ASSERT_EQUAL((s).GetValueCase(), NProto::TSingleSample::kFloat64); \ + UNIT_ASSERT_DOUBLES_EQUAL((s).GetFloat64(), value, std::numeric_limits<double>::epsilon()); \ + } while (false) + +#define ASSERT_UINT_POINT(s, time, value) do { \ + UNIT_ASSERT_VALUES_EQUAL((s).GetTime(), (time).MilliSeconds()); \ + UNIT_ASSERT_EQUAL((s).GetValueCase(), NProto::TSingleSample::kUint64); \ + UNIT_ASSERT_VALUES_EQUAL((s).GetUint64(), value); \ + } while (false) + +#define ASSERT_HIST_POINT(s, time, expected) do { \ + UNIT_ASSERT_VALUES_EQUAL((s).GetTime(), time.MilliSeconds()); \ + UNIT_ASSERT_EQUAL((s).GetValueCase(), NProto::TSingleSample::kHistogram);\ + UNIT_ASSERT_VALUES_EQUAL((s).GetHistogram().BoundsSize(), (expected).Count()); \ + UNIT_ASSERT_VALUES_EQUAL((s).GetHistogram().ValuesSize(), (expected).Count()); \ + for (size_t i = 0; i < (s).GetHistogram().BoundsSize(); i++) { \ + UNIT_ASSERT_DOUBLES_EQUAL((s).GetHistogram().GetBounds(i), (expected).UpperBound(i), Min<double>()); \ + UNIT_ASSERT_VALUES_EQUAL((s).GetHistogram().GetValues(i), (expected).Value(i)); \ + } \ + } while (false) + +Y_UNIT_TEST_SUITE(TPrometheusDecoderTest) { + + NProto::TSingleSamplesList Decode(TStringBuf data) { + NProto::TSingleSamplesList samples; + { + IMetricEncoderPtr e = EncoderProtobuf(&samples); + DecodePrometheus(data, e.Get()); + } + return samples; + } + + Y_UNIT_TEST(Empty) { + { + auto samples = Decode(""); + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 0); + } + { + auto samples = Decode("\n"); + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 0); + } + { + auto samples = Decode("\n \n \n"); + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 0); + } + { + auto samples = Decode("\t\n\t\n"); + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 0); + } + } + + Y_UNIT_TEST(Minimal) { + auto samples = Decode( + "minimal_metric 1.234\n" + "another_metric -3e3 103948\n" + "# Even that:\n" + "no_labels{} 3\n" + "# HELP line for non-existing metric will be ignored.\n"); + + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 3); + { + auto& s = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(1, s.LabelsSize()); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "minimal_metric"); + ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 1.234); + } + { + auto& s = samples.GetSamples(1); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 1); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "another_metric"); + ASSERT_DOUBLE_POINT(s, TInstant::MilliSeconds(103948), -3000.0); + } + { + auto& s = samples.GetSamples(2); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(1, s.LabelsSize()); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "no_labels"); + ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 3.0); + } + } + + Y_UNIT_TEST(Counter) { + auto samples = Decode( + "# A normal comment.\n" + "#\n" + "# TYPE name counter\n" + "name{labelname=\"val1\",basename=\"basevalue\"} NaN\n" + "name {labelname=\"val2\",basename=\"basevalue\"} 2.3 1234567890\n" + "# HELP name two-line\\n doc str\\\\ing\n"); + + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 2); + + { + auto& s = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 3); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "name"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "basename", "basevalue"); + ASSERT_LABEL_EQUAL(s.GetLabels(2), "labelname", "val1"); + ASSERT_UINT_POINT(s, TInstant::Zero(), ui64(0)); + } + { + auto& s = samples.GetSamples(1); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 3); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "name"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "basename", "basevalue"); + ASSERT_LABEL_EQUAL(s.GetLabels(2), "labelname", "val2"); + ASSERT_UINT_POINT(s, TInstant::MilliSeconds(1234567890), i64(2)); + } + } + + Y_UNIT_TEST(Gauge) { + auto samples = Decode( + "# A normal comment.\n" + "#\n" + " # HELP name2 \tdoc str\"ing 2\n" + " # TYPE name2 gauge\n" + "name2{labelname=\"val2\"\t,basename = \"basevalue2\"\t\t} +Inf 54321\n" + "name2{ labelname = \"val1\" , }-Inf\n"); + + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 2); + + { + auto& s = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 3); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "name2"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "basename", "basevalue2"); + ASSERT_LABEL_EQUAL(s.GetLabels(2), "labelname", "val2"); + ASSERT_DOUBLE_POINT(s, TInstant::MilliSeconds(54321), INFINITY); + } + { + auto& s = samples.GetSamples(1); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "name2"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "labelname", "val1"); + ASSERT_DOUBLE_POINT(s, TInstant::Zero(), -INFINITY); + } + } + + Y_UNIT_TEST(Summary) { + auto samples = Decode( + "# HELP \n" + "# TYPE my_summary summary\n" + "my_summary{n1=\"val1\",quantile=\"0.5\"} 110\n" + "my_summary{n1=\"val1\",quantile=\"0.9\"} 140 1\n" + "my_summary_count{n1=\"val1\"} 42\n" + "my_summary_sum{n1=\"val1\"} 08 15\n" + "# some\n" + "# funny comments\n" + "# HELP\n" + "# HELP my_summary\n" + "# HELP my_summary \n"); + + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 4); + + { + auto& s = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 3); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "my_summary"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "quantile", "0.5"); + ASSERT_LABEL_EQUAL(s.GetLabels(2), "n1", "val1"); + ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 110.0); + } + { + auto& s = samples.GetSamples(1); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 3); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "my_summary"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "quantile", "0.9"); + ASSERT_LABEL_EQUAL(s.GetLabels(2), "n1", "val1"); + ASSERT_DOUBLE_POINT(s, TInstant::MilliSeconds(1), 140.0); + } + { + auto& s = samples.GetSamples(2); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "my_summary_count"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "n1", "val1"); + ASSERT_UINT_POINT(s, TInstant::Zero(), 42); + } + { + auto& s = samples.GetSamples(3); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "my_summary_sum"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "n1", "val1"); + ASSERT_DOUBLE_POINT(s, TInstant::MilliSeconds(15), 8.0); + } + } + + Y_UNIT_TEST(Histogram) { + auto samples = Decode( + "# HELP request_duration_microseconds The response latency.\n" + "# TYPE request_duration_microseconds histogram\n" + "request_duration_microseconds_bucket{le=\"0\"} 0\n" + "request_duration_microseconds_bucket{le=\"100\"} 123\n" + "request_duration_microseconds_bucket{le=\"120\"} 412\n" + "request_duration_microseconds_bucket{le=\"144\"} 592\n" + "request_duration_microseconds_bucket{le=\"172.8\"} 1524\n" + "request_duration_microseconds_bucket{le=\"+Inf\"} 2693\n" + "request_duration_microseconds_sum 1.7560473e+06\n" + "request_duration_microseconds_count 2693\n"); + + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 3); + + { + auto& s = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 1); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "request_duration_microseconds_sum"); + ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 1756047.3); + } + { + auto& s = samples.GetSamples(1); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 1); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "request_duration_microseconds_count"); + ASSERT_UINT_POINT(s, TInstant::Zero(), 2693); + } + { + auto& s = samples.GetSamples(2); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::HIST_RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 1); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "request_duration_microseconds"); + auto hist = ExplicitHistogramSnapshot( + { 0, 100, 120, 144, 172.8, HISTOGRAM_INF_BOUND }, + { 0, 123, 289, 180, 932, 1169 }); + ASSERT_HIST_POINT(s, TInstant::Zero(), *hist); + } + } + + Y_UNIT_TEST(HistogramWithLabels) { + auto samples = Decode( + "# A histogram, which has a pretty complex representation in the text format:\n" + "# HELP http_request_duration_seconds A histogram of the request duration.\n" + "# TYPE http_request_duration_seconds histogram\n" + "http_request_duration_seconds_bucket{le=\"0.05\", method=\"POST\"} 24054\n" + "http_request_duration_seconds_bucket{method=\"POST\", le=\"0.1\"} 33444\n" + "http_request_duration_seconds_bucket{le=\"0.2\", method=\"POST\", } 100392\n" + "http_request_duration_seconds_bucket{le=\"0.5\",method=\"POST\",} 129389\n" + "http_request_duration_seconds_bucket{ method=\"POST\", le=\"1\", } 133988\n" + "http_request_duration_seconds_bucket{ le=\"+Inf\", method=\"POST\", } 144320\n" + "http_request_duration_seconds_sum{method=\"POST\"} 53423\n" + "http_request_duration_seconds_count{ method=\"POST\", } 144320\n"); + + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 3); + + { + auto& s = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "http_request_duration_seconds_sum"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "method", "POST"); + ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 53423.0); + } + { + auto& s = samples.GetSamples(1); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "http_request_duration_seconds_count"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "method", "POST"); + ASSERT_UINT_POINT(s, TInstant::Zero(), 144320); + } + { + auto& s = samples.GetSamples(2); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::HIST_RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "http_request_duration_seconds"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "method", "POST"); + auto hist = ExplicitHistogramSnapshot( + { 0.05, 0.1, 0.2, 0.5, 1, HISTOGRAM_INF_BOUND }, + { 24054, 9390, 66948, 28997, 4599, 10332 }); + ASSERT_HIST_POINT(s, TInstant::Zero(), *hist); + } + } + + Y_UNIT_TEST(MultipleHistograms) { + auto samples = Decode( + "# TYPE inboundBytesPerSec histogram\n" + "inboundBytesPerSec_bucket{client=\"mbus\", le=\"10.0\"} 1.0\n" + "inboundBytesPerSec_bucket{client=\"mbus\", le=\"20.0\"} 5.0\n" + "inboundBytesPerSec_bucket{client=\"mbus\", le=\"+Inf\"} 5.0\n" + "inboundBytesPerSec_count{client=\"mbus\"} 5.0\n" + "inboundBytesPerSec_bucket{client=\"grpc\", le=\"10.0\"} 1.0\n" + "inboundBytesPerSec_bucket{client=\"grpc\", le=\"20.0\"} 5.0\n" + "inboundBytesPerSec_bucket{client=\"grpc\", le=\"30.0\"} 5.0\n" + "inboundBytesPerSec_count{client=\"grpc\"} 5.0\n" + "# TYPE outboundBytesPerSec histogram\n" + "outboundBytesPerSec_bucket{client=\"grpc\", le=\"100.0\"} 1.0 1512216000000\n" + "outboundBytesPerSec_bucket{client=\"grpc\", le=\"200.0\"} 1.0 1512216000000\n" + "outboundBytesPerSec_bucket{client=\"grpc\", le=\"+Inf\"} 1.0 1512216000000\n" + "outboundBytesPerSec_count{client=\"grpc\"} 1.0 1512216000000\n"); + + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 6); + + { + auto& s = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "inboundBytesPerSec_count"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "client", "mbus"); + ASSERT_UINT_POINT(s, TInstant::Zero(), 5); + } + { + auto& s = samples.GetSamples(1); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::HIST_RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "inboundBytesPerSec"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "client", "mbus"); + auto hist = ExplicitHistogramSnapshot( + { 10, 20, HISTOGRAM_INF_BOUND }, + { 1, 4, 0 }); + ASSERT_HIST_POINT(s, TInstant::Zero(), *hist); + } + { + auto& s = samples.GetSamples(2); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "inboundBytesPerSec_count"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "client", "grpc"); + ASSERT_UINT_POINT(s, TInstant::Zero(), 5); + } + { + auto& s = samples.GetSamples(3); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::HIST_RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "inboundBytesPerSec"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "client", "grpc"); + auto hist = ExplicitHistogramSnapshot( + { 10, 20, 30 }, + { 1, 4, 0 }); + ASSERT_HIST_POINT(s, TInstant::Zero(), *hist); + } + { + auto& s = samples.GetSamples(4); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "outboundBytesPerSec_count"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "client", "grpc"); + ASSERT_UINT_POINT(s, TInstant::Seconds(1512216000), 1) ; + } + { + auto& s = samples.GetSamples(5); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::HIST_RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "outboundBytesPerSec"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "client", "grpc"); + auto hist = ExplicitHistogramSnapshot( + { 100, 200, HISTOGRAM_INF_BOUND }, + { 1, 0, 0 }); + ASSERT_HIST_POINT(s, TInstant::Seconds(1512216000), *hist); + } + } + + Y_UNIT_TEST(MixedTypes) { + auto samples = Decode( + "# HELP http_requests_total The total number of HTTP requests.\n" + "# TYPE http_requests_total counter\n" + "http_requests_total { } 1027 1395066363000\n" + "http_requests_total{method=\"post\",code=\"200\"} 1027 1395066363000\n" + "http_requests_total{method=\"post\",code=\"400\"} 3 1395066363000\n" + "\n" + "# Minimalistic line:\n" + "metric_without_timestamp_and_labels 12.47\n" + "\n" + "# HELP rpc_duration_seconds A summary of the RPC duration in seconds.\n" + "# TYPE rpc_duration_seconds summary\n" + "rpc_duration_seconds{quantile=\"0.01\"} 3102\n" + "rpc_duration_seconds{quantile=\"0.5\"} 4773\n" + "rpc_duration_seconds{quantile=\"0.9\"} 9001\n" + "rpc_duration_seconds_sum 1.7560473e+07\n" + "rpc_duration_seconds_count 2693\n" + "\n" + "# Another mMinimalistic line:\n" + "metric_with_timestamp 12.47 1234567890\n"); + + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 10); + + { + auto& s = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 1); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "http_requests_total"); + ASSERT_UINT_POINT(s, TInstant::Seconds(1395066363), 1027); + } + { + auto& s = samples.GetSamples(1); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 3); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "http_requests_total"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "method", "post"); + ASSERT_LABEL_EQUAL(s.GetLabels(2), "code", "200"); + ASSERT_UINT_POINT(s, TInstant::Seconds(1395066363), 1027); + } + { + auto& s = samples.GetSamples(2); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 3); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "http_requests_total"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "method", "post"); + ASSERT_LABEL_EQUAL(s.GetLabels(2), "code", "400"); + ASSERT_UINT_POINT(s, TInstant::Seconds(1395066363), 3); + } + { + auto& s = samples.GetSamples(3); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 1); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "metric_without_timestamp_and_labels"); + ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 12.47); + } + { + auto& s = samples.GetSamples(4); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "rpc_duration_seconds"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "quantile", "0.01"); + ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 3102); + } + { + auto& s = samples.GetSamples(5); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "rpc_duration_seconds"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "quantile", "0.5"); + ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 4773); + } + { + auto& s = samples.GetSamples(6); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "rpc_duration_seconds"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "quantile", "0.9"); + ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 9001); + } + { + auto& s = samples.GetSamples(7); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 1); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "rpc_duration_seconds_sum"); + ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 17560473); + } + { + auto& s = samples.GetSamples(8); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 1); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "rpc_duration_seconds_count"); + ASSERT_UINT_POINT(s, TInstant::Zero(), 2693); + } + { + auto& s = samples.GetSamples(9); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 1); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "metric_with_timestamp"); + ASSERT_DOUBLE_POINT(s, TInstant::MilliSeconds(1234567890), 12.47); + } + } +} diff --git a/library/cpp/monlib/encode/prometheus/prometheus_encoder.cpp b/library/cpp/monlib/encode/prometheus/prometheus_encoder.cpp new file mode 100644 index 0000000000..15efeb8c03 --- /dev/null +++ b/library/cpp/monlib/encode/prometheus/prometheus_encoder.cpp @@ -0,0 +1,413 @@ +#include "prometheus.h" +#include "prometheus_model.h" + +#include <library/cpp/monlib/encode/encoder_state.h> +#include <library/cpp/monlib/metrics/labels.h> +#include <library/cpp/monlib/metrics/metric_value.h> + +#include <util/string/cast.h> +#include <util/generic/hash_set.h> + + +namespace NMonitoring { + namespace { + /////////////////////////////////////////////////////////////////////// + // TPrometheusWriter + /////////////////////////////////////////////////////////////////////// + class TPrometheusWriter { + public: + explicit TPrometheusWriter(IOutputStream* out) + : Out_(out) + { + } + + void WriteType(EMetricType type, const TString& name) { + auto r = WrittenTypes_.insert(name); + if (!r.second) { + // type for this metric was already written + return; + } + + Out_->Write("# TYPE "); + WriteMetricName(name); + Out_->Write(' '); + + switch (type) { + case EMetricType::GAUGE: + case EMetricType::IGAUGE: + Out_->Write("gauge"); + break; + case EMetricType::RATE: + case EMetricType::COUNTER: + Out_->Write("counter"); + break; + case EMetricType::HIST: + case EMetricType::HIST_RATE: + Out_->Write("histogram"); + break; + case EMetricType::LOGHIST: + // TODO(@kbalakirev): implement this case + break; + case EMetricType::DSUMMARY: + ythrow yexception() << "writing summary type is forbiden"; + case EMetricType::UNKNOWN: + ythrow yexception() << "unknown metric type: " << MetricTypeToStr(type) + << ", name: " << name; + } + Out_->Write('\n'); + } + + void WriteDouble(TStringBuf name, const TLabels& labels, TInstant time, double value) { + WriteValue(name, "", labels, "", "", time, value); + } + + void WriteHistogram(TStringBuf name, const TLabels& labels, TInstant time, IHistogramSnapshot* h) { + Y_ENSURE(!labels.Has(NPrometheus::BUCKET_LABEL), + "histogram metric " << name << " has label '" << + NPrometheus::BUCKET_LABEL << "' which is reserved in Prometheus"); + + double totalCount = 0; + for (ui32 i = 0, count = h->Count(); i < count; i++) { + TBucketBound bound = h->UpperBound(i); + TStringBuf boundStr; + if (bound == HISTOGRAM_INF_BOUND) { + boundStr = TStringBuf("+Inf"); + } else { + size_t len = FloatToString(bound, TmpBuf_, Y_ARRAY_SIZE(TmpBuf_)); + boundStr = TStringBuf(TmpBuf_, len); + } + + TBucketValue value = h->Value(i); + totalCount += static_cast<double>(value); + + WriteValue( + name, NPrometheus::BUCKET_SUFFIX, + labels, NPrometheus::BUCKET_LABEL, boundStr, + time, + totalCount); + } + + WriteValue(name, NPrometheus::COUNT_SUFFIX, labels, "", "", time, totalCount); + } + + void WriteSummaryDouble(TStringBuf name, const TLabels& labels, TInstant time, ISummaryDoubleSnapshot* s) { + WriteValue(name, NPrometheus::SUM_SUFFIX, labels, "", "", time, s->GetSum()); + WriteValue(name, NPrometheus::MIN_SUFFIX, labels, "", "", time, s->GetMin()); + WriteValue(name, NPrometheus::MAX_SUFFIX, labels, "", "", time, s->GetMax()); + WriteValue(name, NPrometheus::LAST_SUFFIX, labels, "", "", time, s->GetLast()); + WriteValue(name, NPrometheus::COUNT_SUFFIX, labels, "", "", time, s->GetCount()); + } + + void WriteLn() { + Out_->Write('\n'); + } + + private: + // will replace invalid chars with '_' + void WriteMetricName(TStringBuf name) { + Y_ENSURE(!name.Empty(), "trying to write metric with empty name"); + + char ch = name[0]; + if (NPrometheus::IsValidMetricNameStart(ch)) { + Out_->Write(ch); + } else { + Out_->Write('_'); + } + + for (size_t i = 1, len = name.length(); i < len; i++) { + ch = name[i]; + if (NPrometheus::IsValidMetricNameContinuation(ch)) { + Out_->Write(ch); + } else { + Out_->Write('_'); + } + } + } + + void WriteLabels(const TLabels& labels, TStringBuf addLabelKey, TStringBuf addLabelValue) { + Out_->Write('{'); + for (auto&& l: labels) { + Out_->Write(l.Name()); + Out_->Write('='); + WriteLabelValue(l.Value()); + Out_->Write(", "); // trailign comma is supported in parsers + } + if (!addLabelKey.Empty() && !addLabelValue.Empty()) { + Out_->Write(addLabelKey); + Out_->Write('='); + WriteLabelValue(addLabelValue); + } + Out_->Write('}'); + } + + void WriteLabelValue(TStringBuf value) { + Out_->Write('"'); + for (char ch: value) { + if (ch == '"') { + Out_->Write("\\\""); + } else if (ch == '\\') { + Out_->Write("\\\\"); + } else if (ch == '\n') { + Out_->Write("\\n"); + } else { + Out_->Write(ch); + } + } + Out_->Write('"'); + } + + void WriteValue( + TStringBuf name, TStringBuf suffix, + const TLabels& labels, TStringBuf addLabelKey, TStringBuf addLabelValue, + TInstant time, double value) + { + // (1) name + WriteMetricName(name); + if (!suffix.Empty()) { + Out_->Write(suffix); + } + + // (2) labels + if (!labels.Empty() || !addLabelKey.Empty()) { + WriteLabels(labels, addLabelKey, addLabelValue); + } + Out_->Write(' '); + + // (3) value + { + size_t len = FloatToString(value, TmpBuf_, Y_ARRAY_SIZE(TmpBuf_)); + Out_->Write(TmpBuf_, len); + } + + // (4) time + if (ui64 timeMillis = time.MilliSeconds()) { + Out_->Write(' '); + size_t len = IntToString<10>(timeMillis, TmpBuf_, Y_ARRAY_SIZE(TmpBuf_)); + Out_->Write(TmpBuf_, len); + } + Out_->Write('\n'); + } + + private: + IOutputStream* Out_; + THashSet<TString> WrittenTypes_; + char TmpBuf_[512]; // used to convert doubles to strings + }; + + /////////////////////////////////////////////////////////////////////// + // TMetricState + /////////////////////////////////////////////////////////////////////// + struct TMetricState { + EMetricType Type = EMetricType::UNKNOWN; + TLabels Labels; + TInstant Time = TInstant::Zero(); + EMetricValueType ValueType = EMetricValueType::UNKNOWN; + TMetricValue Value; + + ~TMetricState() { + ClearValue(); + } + + void Clear() { + Type = EMetricType::UNKNOWN; + Labels.Clear(); + Time = TInstant::Zero(); + ClearValue(); + } + + void ClearValue() { + // TMetricValue does not keep ownership of histogram + if (ValueType == EMetricValueType::HISTOGRAM) { + Value.AsHistogram()->UnRef(); + } else if (ValueType == EMetricValueType::SUMMARY) { + Value.AsSummaryDouble()->UnRef(); + } + ValueType = EMetricValueType::UNKNOWN; + Value = {}; + } + + template <typename T> + void SetValue(T value) { + // TMetricValue does not keep ownership of histogram + if (ValueType == EMetricValueType::HISTOGRAM) { + Value.AsHistogram()->UnRef(); + } else if (ValueType == EMetricValueType::SUMMARY) { + Value.AsSummaryDouble()->UnRef(); + } + ValueType = TValueType<T>::Type; + Value = TMetricValue(value); + if (ValueType == EMetricValueType::HISTOGRAM) { + Value.AsHistogram()->Ref(); + } else if (ValueType == EMetricValueType::SUMMARY) { + Value.AsSummaryDouble()->Ref(); + } + } + }; + + /////////////////////////////////////////////////////////////////////// + // TPrometheusEncoder + /////////////////////////////////////////////////////////////////////// + class TPrometheusEncoder final: public IMetricEncoder { + public: + explicit TPrometheusEncoder(IOutputStream* out, TStringBuf metricNameLabel) + : Writer_(out) + , MetricNameLabel_(metricNameLabel) + { + } + + private: + void OnStreamBegin() override { + State_.Expect(TEncoderState::EState::ROOT); + } + + void OnStreamEnd() override { + State_.Expect(TEncoderState::EState::ROOT); + Writer_.WriteLn(); + } + + void OnCommonTime(TInstant time) override { + State_.Expect(TEncoderState::EState::ROOT); + CommonTime_ = time; + } + + void OnMetricBegin(EMetricType type) override { + State_.Switch(TEncoderState::EState::ROOT, TEncoderState::EState::METRIC); + MetricState_.Clear(); + MetricState_.Type = type; + } + + void OnMetricEnd() override { + State_.Switch(TEncoderState::EState::METRIC, TEncoderState::EState::ROOT); + WriteMetric(); + } + + void OnLabelsBegin() override { + if (State_ == TEncoderState::EState::METRIC) { + State_ = TEncoderState::EState::METRIC_LABELS; + } else if (State_ == TEncoderState::EState::ROOT) { + State_ = TEncoderState::EState::COMMON_LABELS; + } else { + State_.ThrowInvalid("expected METRIC or ROOT"); + } + } + + void OnLabelsEnd() override { + if (State_ == TEncoderState::EState::METRIC_LABELS) { + State_ = TEncoderState::EState::METRIC; + } else if (State_ == TEncoderState::EState::COMMON_LABELS) { + State_ = TEncoderState::EState::ROOT; + } else { + State_.ThrowInvalid("expected LABELS or COMMON_LABELS"); + } + } + + void OnLabel(TStringBuf name, TStringBuf value) override { + if (State_ == TEncoderState::EState::METRIC_LABELS) { + MetricState_.Labels.Add(name, value); + } else if (State_ == TEncoderState::EState::COMMON_LABELS) { + CommonLabels_.Add(name, value); + } else { + State_.ThrowInvalid("expected LABELS or COMMON_LABELS"); + } + } + + void OnDouble(TInstant time, double value) override { + State_.Expect(TEncoderState::EState::METRIC); + MetricState_.Time = time; + MetricState_.SetValue(value); + } + + void OnInt64(TInstant time, i64 value) override { + State_.Expect(TEncoderState::EState::METRIC); + MetricState_.Time = time; + MetricState_.SetValue(value); + } + + void OnUint64(TInstant time, ui64 value) override { + State_.Expect(TEncoderState::EState::METRIC); + MetricState_.Time = time; + MetricState_.SetValue(value); + } + + void OnHistogram(TInstant time, IHistogramSnapshotPtr snapshot) override { + State_.Expect(TEncoderState::EState::METRIC); + MetricState_.Time = time; + MetricState_.SetValue(snapshot.Get()); + } + + void OnSummaryDouble(TInstant time, ISummaryDoubleSnapshotPtr snapshot) override { + State_.Expect(TEncoderState::EState::METRIC); + MetricState_.Time = time; + MetricState_.SetValue(snapshot.Get()); + } + + void OnLogHistogram(TInstant, TLogHistogramSnapshotPtr) override { + // TODO(@kbalakirev): implement this function + } + + void Close() override { + } + + void WriteMetric() { + if (MetricState_.ValueType == EMetricValueType::UNKNOWN) { + return; + } + + // XXX: poor performace + for (auto&& l: CommonLabels_) { + MetricState_.Labels.Add(l.Name(), l.Value()); + } + + TMaybe<TLabel> nameLabel = MetricState_.Labels.Extract(MetricNameLabel_); + Y_ENSURE(nameLabel, + "labels " << MetricState_.Labels << + " does not contain label '" << MetricNameLabel_ << '\''); + + const TString& metricName = ToString(nameLabel->Value()); + if (MetricState_.Type != EMetricType::DSUMMARY) { + Writer_.WriteType(MetricState_.Type, metricName); + } + + if (MetricState_.Time == TInstant::Zero()) { + MetricState_.Time = CommonTime_; + } + + EMetricType type = MetricState_.Type; + if (type == EMetricType::HIST || type == EMetricType::HIST_RATE) { + Y_ENSURE(MetricState_.ValueType == EMetricValueType::HISTOGRAM, + "invalid value type for histogram: " << int(MetricState_.ValueType)); // TODO: to string conversion + Writer_.WriteHistogram( + metricName, + MetricState_.Labels, + MetricState_.Time, + MetricState_.Value.AsHistogram()); + } else if (type == EMetricType::DSUMMARY) { + Writer_.WriteSummaryDouble( + metricName, + MetricState_.Labels, + MetricState_.Time, + MetricState_.Value.AsSummaryDouble()); + } else { + Writer_.WriteDouble( + metricName, + MetricState_.Labels, + MetricState_.Time, + MetricState_.Value.AsDouble(MetricState_.ValueType)); + } + } + + private: + TEncoderState State_; + TPrometheusWriter Writer_; + TString MetricNameLabel_; + TInstant CommonTime_ = TInstant::Zero(); + TLabels CommonLabels_; + TMetricState MetricState_; + }; + } + + IMetricEncoderPtr EncoderPrometheus(IOutputStream* out, TStringBuf metricNameLabel) { + return MakeHolder<TPrometheusEncoder>(out, metricNameLabel); + } + +} // namespace NMonitoring diff --git a/library/cpp/monlib/encode/prometheus/prometheus_encoder_ut.cpp b/library/cpp/monlib/encode/prometheus/prometheus_encoder_ut.cpp new file mode 100644 index 0000000000..fd9debb060 --- /dev/null +++ b/library/cpp/monlib/encode/prometheus/prometheus_encoder_ut.cpp @@ -0,0 +1,414 @@ +#include "prometheus.h" + +#include <library/cpp/monlib/encode/protobuf/protobuf.h> +#include <library/cpp/monlib/metrics/metric_value.h> +#include <library/cpp/monlib/metrics/histogram_snapshot.h> + +#include <library/cpp/testing/unittest/registar.h> + +#include <util/stream/str.h> + +using namespace NMonitoring; + +Y_UNIT_TEST_SUITE(TPrometheusEncoderTest) { + + template <typename TFunc> + TString EncodeToString(TFunc fn) { + TStringStream ss; + IMetricEncoderPtr encoder = EncoderPrometheus(&ss); + fn(encoder.Get()); + return ss.Str(); + } + + ISummaryDoubleSnapshotPtr TestSummaryDouble() { + return MakeIntrusive<TSummaryDoubleSnapshot>(10.1, -0.45, 0.478, 0.3, 30u); + } + + Y_UNIT_TEST(Empty) { + auto result = EncodeToString([](IMetricEncoder* e) { + e->OnStreamBegin(); + e->OnStreamEnd(); + }); + UNIT_ASSERT_STRINGS_EQUAL(result, "\n"); + } + + Y_UNIT_TEST(DoubleGauge) { + auto result = EncodeToString([](IMetricEncoder* e) { + e->OnStreamBegin(); + { // no values + e->OnMetricBegin(EMetricType::GAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "cpuUsage"); + e->OnLabelsEnd(); + } + e->OnMetricEnd(); + } + { // one value no ts + e->OnMetricBegin(EMetricType::GAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "diskUsage"); + e->OnLabel("disk", "sda1"); + e->OnLabelsEnd(); + } + e->OnDouble(TInstant::Zero(), 1000); + e->OnMetricEnd(); + } + { // one value with ts + e->OnMetricBegin(EMetricType::GAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "memoryUsage"); + e->OnLabel("host", "solomon-man-00"); + e->OnLabel("dc", "man"); + e->OnLabelsEnd(); + } + e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 1000); + e->OnMetricEnd(); + } + { // many values + e->OnMetricBegin(EMetricType::GAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "bytesRx"); + e->OnLabel("host", "solomon-sas-01"); + e->OnLabel("dc", "sas"); + e->OnLabelsEnd(); + } + e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 2); + e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:05Z"), 4); + e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:10Z"), 8); + e->OnMetricEnd(); + } + { // already seen metric name + e->OnMetricBegin(EMetricType::GAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "diskUsage"); + e->OnLabel("disk", "sdb1"); + e->OnLabelsEnd(); + } + e->OnDouble(TInstant::Zero(), 1001); + e->OnMetricEnd(); + } + { // NaN + e->OnMetricBegin(EMetricType::GAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "nanValue"); + e->OnLabelsEnd(); + } + e->OnDouble(TInstant::Zero(), NAN); + e->OnMetricEnd(); + } + { // Inf + e->OnMetricBegin(EMetricType::GAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "infValue"); + e->OnLabelsEnd(); + } + e->OnDouble(TInstant::Zero(), INFINITY); + e->OnMetricEnd(); + } + { + e->OnMetricBegin(EMetricType::DSUMMARY); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "seconds"); + e->OnLabel("disk", "sdb1"); + e->OnLabelsEnd(); + } + e->OnSummaryDouble(TInstant::Zero(), TestSummaryDouble()); + e->OnMetricEnd(); + } + e->OnStreamEnd(); + }); + + UNIT_ASSERT_STRINGS_EQUAL(result, + "# TYPE diskUsage gauge\n" + "diskUsage{disk=\"sda1\", } 1000\n" + "# TYPE memoryUsage gauge\n" + "memoryUsage{host=\"solomon-man-00\", dc=\"man\", } 1000 1512216000000\n" + "# TYPE bytesRx gauge\n" + "bytesRx{host=\"solomon-sas-01\", dc=\"sas\", } 8 1512216010000\n" + "diskUsage{disk=\"sdb1\", } 1001\n" + "# TYPE nanValue gauge\n" + "nanValue nan\n" + "# TYPE infValue gauge\n" + "infValue inf\n" + "seconds_sum{disk=\"sdb1\", } 10.1\n" + "seconds_min{disk=\"sdb1\", } -0.45\n" + "seconds_max{disk=\"sdb1\", } 0.478\n" + "seconds_last{disk=\"sdb1\", } 0.3\n" + "seconds_count{disk=\"sdb1\", } 30\n" + "\n"); + } + + Y_UNIT_TEST(IntGauges) { + auto result = EncodeToString([](IMetricEncoder* e) { + e->OnStreamBegin(); + { // no values + e->OnMetricBegin(EMetricType::IGAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "cpuUsage"); + e->OnLabelsEnd(); + } + e->OnMetricEnd(); + } + { // one value no ts + e->OnMetricBegin(EMetricType::IGAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "diskUsage"); + e->OnLabel("disk", "sda1"); + e->OnLabelsEnd(); + } + e->OnInt64(TInstant::Zero(), 1000); + e->OnMetricEnd(); + } + { // one value with ts + e->OnMetricBegin(EMetricType::IGAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "memoryUsage"); + e->OnLabel("dc", "man"); + e->OnLabel("host", "solomon-man-00"); + e->OnLabelsEnd(); + } + e->OnInt64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 1000); + e->OnMetricEnd(); + } + { // many values + e->OnMetricBegin(EMetricType::IGAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "bytesRx"); + e->OnLabel("dc", "sas"); + e->OnLabel("host", "solomon-sas-01"); + e->OnLabelsEnd(); + } + e->OnInt64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 2); + e->OnInt64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:05Z"), 4); + e->OnInt64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:10Z"), 8); + e->OnMetricEnd(); + } + e->OnStreamEnd(); + }); + + UNIT_ASSERT_STRINGS_EQUAL(result, + "# TYPE diskUsage gauge\n" + "diskUsage{disk=\"sda1\", } 1000\n" + "# TYPE memoryUsage gauge\n" + "memoryUsage{dc=\"man\", host=\"solomon-man-00\", } 1000 1512216000000\n" + "# TYPE bytesRx gauge\n" + "bytesRx{dc=\"sas\", host=\"solomon-sas-01\", } 8 1512216010000\n" + "\n"); + } + + Y_UNIT_TEST(Counters) { + auto result = EncodeToString([](IMetricEncoder* e) { + e->OnStreamBegin(); + { // no values + e->OnMetricBegin(EMetricType::COUNTER); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "cpuUsage"); + e->OnLabelsEnd(); + } + e->OnMetricEnd(); + } + { // one value no ts + e->OnMetricBegin(EMetricType::COUNTER); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "diskUsage"); + e->OnLabel("disk", "sda1"); + e->OnLabelsEnd(); + } + e->OnInt64(TInstant::Zero(), 1000); + e->OnMetricEnd(); + } + { // one value with ts + e->OnMetricBegin(EMetricType::COUNTER); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "memoryUsage"); + e->OnLabel("host", "solomon-man-00"); + e->OnLabel("dc", "man"); + e->OnLabelsEnd(); + } + e->OnInt64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 1000); + e->OnMetricEnd(); + } + { // many values + e->OnMetricBegin(EMetricType::COUNTER); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "bytesRx"); + e->OnLabel("host", "solomon-sas-01"); + e->OnLabel("dc", "sas"); + e->OnLabelsEnd(); + } + e->OnInt64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 2); + e->OnInt64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:05Z"), 4); + e->OnInt64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:10Z"), 8); + e->OnMetricEnd(); + } + e->OnStreamEnd(); + }); + + UNIT_ASSERT_STRINGS_EQUAL(result, + "# TYPE diskUsage counter\n" + "diskUsage{disk=\"sda1\", } 1000\n" + "# TYPE memoryUsage counter\n" + "memoryUsage{host=\"solomon-man-00\", dc=\"man\", } 1000 1512216000000\n" + "# TYPE bytesRx counter\n" + "bytesRx{host=\"solomon-sas-01\", dc=\"sas\", } 8 1512216010000\n" + "\n"); + } + + Y_UNIT_TEST(Histograms) { + auto result = EncodeToString([](IMetricEncoder* e) { + e->OnStreamBegin(); + { // no values histogram + e->OnMetricBegin(EMetricType::HIST); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "cpuUsage"); + e->OnLabelsEnd(); + } + e->OnMetricEnd(); + } + { // one value no ts + e->OnMetricBegin(EMetricType::HIST); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "inboundBytesPerSec"); + e->OnLabel("client", "mbus"); + e->OnLabelsEnd(); + } + e->OnHistogram( + TInstant::Zero(), + ExplicitHistogramSnapshot({10, 20, HISTOGRAM_INF_BOUND}, {1, 4, 0})); + e->OnMetricEnd(); + } + { // one value no ts no +inf bucket + e->OnMetricBegin(EMetricType::HIST); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "inboundBytesPerSec"); + e->OnLabel("client", "grpc"); + e->OnLabelsEnd(); + } + e->OnHistogram( + TInstant::Zero(), + ExplicitHistogramSnapshot({10, 20, 30}, {1, 4, 0})); + e->OnMetricEnd(); + } + { // one value with ts + e->OnMetricBegin(EMetricType::HIST_RATE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "outboundBytesPerSec"); + e->OnLabel("client", "grps"); + e->OnLabelsEnd(); + } + e->OnHistogram( + TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), + ExplicitHistogramSnapshot({100, 200, HISTOGRAM_INF_BOUND}, {1, 0, 0})); + e->OnMetricEnd(); + } + { // many values + e->OnMetricBegin(EMetricType::HIST); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "bytesRx"); + e->OnLabel("host", "solomon-sas-01"); + e->OnLabel("dc", "sas"); + e->OnLabelsEnd(); + } + TBucketBounds bounds = {100, 200, HISTOGRAM_INF_BOUND}; + e->OnHistogram( + TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), + ExplicitHistogramSnapshot(bounds, {10, 0, 0})); + e->OnHistogram( + TInstant::ParseIso8601Deprecated("2017-12-02T12:00:05Z"), + ExplicitHistogramSnapshot(bounds, {10, 2, 0})); + e->OnHistogram( + TInstant::ParseIso8601Deprecated("2017-12-02T12:00:10Z"), + ExplicitHistogramSnapshot(bounds, {10, 2, 5})); + e->OnMetricEnd(); + } + e->OnStreamEnd(); + }); + + UNIT_ASSERT_STRINGS_EQUAL(result, + "# TYPE inboundBytesPerSec histogram\n" + "inboundBytesPerSec_bucket{client=\"mbus\", le=\"10\"} 1\n" + "inboundBytesPerSec_bucket{client=\"mbus\", le=\"20\"} 5\n" + "inboundBytesPerSec_bucket{client=\"mbus\", le=\"+Inf\"} 5\n" + "inboundBytesPerSec_count{client=\"mbus\", } 5\n" + "inboundBytesPerSec_bucket{client=\"grpc\", le=\"10\"} 1\n" + "inboundBytesPerSec_bucket{client=\"grpc\", le=\"20\"} 5\n" + "inboundBytesPerSec_bucket{client=\"grpc\", le=\"30\"} 5\n" + "inboundBytesPerSec_count{client=\"grpc\", } 5\n" + "# TYPE outboundBytesPerSec histogram\n" + "outboundBytesPerSec_bucket{client=\"grps\", le=\"100\"} 1 1512216000000\n" + "outboundBytesPerSec_bucket{client=\"grps\", le=\"200\"} 1 1512216000000\n" + "outboundBytesPerSec_bucket{client=\"grps\", le=\"+Inf\"} 1 1512216000000\n" + "outboundBytesPerSec_count{client=\"grps\", } 1 1512216000000\n" + "# TYPE bytesRx histogram\n" + "bytesRx_bucket{host=\"solomon-sas-01\", dc=\"sas\", le=\"100\"} 10 1512216010000\n" + "bytesRx_bucket{host=\"solomon-sas-01\", dc=\"sas\", le=\"200\"} 12 1512216010000\n" + "bytesRx_bucket{host=\"solomon-sas-01\", dc=\"sas\", le=\"+Inf\"} 17 1512216010000\n" + "bytesRx_count{host=\"solomon-sas-01\", dc=\"sas\", } 17 1512216010000\n" + "\n"); + } + + Y_UNIT_TEST(CommonLables) { + auto result = EncodeToString([](IMetricEncoder* e) { + e->OnStreamBegin(); + { // common time + e->OnCommonTime(TInstant::Seconds(1500000000)); + } + { // common labels + e->OnLabelsBegin(); + e->OnLabel("project", "solomon"); + e->OnLabelsEnd(); + } + { // metric #1 + e->OnMetricBegin(EMetricType::COUNTER); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "single"); + e->OnLabel("labels", "l1"); + e->OnLabelsEnd(); + } + e->OnUint64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:10Z"), 17); + e->OnMetricEnd(); + } + { // metric #2 + e->OnMetricBegin(EMetricType::COUNTER); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "two"); + e->OnLabel("labels", "l2"); + e->OnLabelsEnd(); + } + e->OnUint64(TInstant::Zero(), 42); + e->OnMetricEnd(); + } + e->OnStreamEnd(); + }); + UNIT_ASSERT_STRINGS_EQUAL(result, +R"(# TYPE single counter +single{labels="l1", project="solomon", } 17 1512216010000 +# TYPE two counter +two{labels="l2", project="solomon", } 42 1500000000000 + +)"); + } +} diff --git a/library/cpp/monlib/encode/prometheus/prometheus_model.h b/library/cpp/monlib/encode/prometheus/prometheus_model.h new file mode 100644 index 0000000000..cb7f2cb15b --- /dev/null +++ b/library/cpp/monlib/encode/prometheus/prometheus_model.h @@ -0,0 +1,70 @@ +#pragma once + +#include <util/generic/strbuf.h> + + +namespace NMonitoring { +namespace NPrometheus { + + // + // Prometheus specific names and validation rules. + // + // See https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md + // and https://github.com/prometheus/common/blob/master/expfmt/text_parse.go + // + + inline constexpr TStringBuf BUCKET_SUFFIX = "_bucket"; + inline constexpr TStringBuf COUNT_SUFFIX = "_count"; + inline constexpr TStringBuf SUM_SUFFIX = "_sum"; + inline constexpr TStringBuf MIN_SUFFIX = "_min"; + inline constexpr TStringBuf MAX_SUFFIX = "_max"; + inline constexpr TStringBuf LAST_SUFFIX = "_last"; + + // Used for the label that defines the upper bound of a bucket of a + // histogram ("le" -> "less or equal"). + inline constexpr TStringBuf BUCKET_LABEL = "le"; + + + inline bool IsValidLabelNameStart(char ch) { + return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || ch == '_'; + } + + inline bool IsValidLabelNameContinuation(char ch) { + return IsValidLabelNameStart(ch) || (ch >= '0' && ch <= '9'); + } + + inline bool IsValidMetricNameStart(char ch) { + return IsValidLabelNameStart(ch) || ch == ':'; + } + + inline bool IsValidMetricNameContinuation(char ch) { + return IsValidLabelNameContinuation(ch) || ch == ':'; + } + + inline bool IsSum(TStringBuf name) { + return name.EndsWith(SUM_SUFFIX); + } + + inline bool IsCount(TStringBuf name) { + return name.EndsWith(COUNT_SUFFIX); + } + + inline bool IsBucket(TStringBuf name) { + return name.EndsWith(BUCKET_SUFFIX); + } + + inline TStringBuf ToBaseName(TStringBuf name) { + if (IsBucket(name)) { + return name.SubString(0, name.length() - BUCKET_SUFFIX.length()); + } + if (IsCount(name)) { + return name.SubString(0, name.length() - COUNT_SUFFIX.length()); + } + if (IsSum(name)) { + return name.SubString(0, name.length() - SUM_SUFFIX.length()); + } + return name; + } + +} // namespace NPrometheus +} // namespace NMonitoring diff --git a/library/cpp/monlib/encode/prometheus/ut/ya.make b/library/cpp/monlib/encode/prometheus/ut/ya.make new file mode 100644 index 0000000000..fc468ffb68 --- /dev/null +++ b/library/cpp/monlib/encode/prometheus/ut/ya.make @@ -0,0 +1,17 @@ +UNITTEST_FOR(library/cpp/monlib/encode/prometheus) + +OWNER( + g:solomon + jamel +) + +SRCS( + prometheus_encoder_ut.cpp + prometheus_decoder_ut.cpp +) + +PEERDIR( + library/cpp/monlib/encode/protobuf +) + +END() diff --git a/library/cpp/monlib/encode/prometheus/ya.make b/library/cpp/monlib/encode/prometheus/ya.make new file mode 100644 index 0000000000..7f2483b166 --- /dev/null +++ b/library/cpp/monlib/encode/prometheus/ya.make @@ -0,0 +1,17 @@ +LIBRARY() + +OWNER( + jamel + g:solomon +) + +SRCS( + prometheus_decoder.cpp + prometheus_encoder.cpp +) + +PEERDIR( + library/cpp/monlib/encode +) + +END() diff --git a/library/cpp/monlib/encode/protobuf/protobuf.h b/library/cpp/monlib/encode/protobuf/protobuf.h new file mode 100644 index 0000000000..3f82cbdd84 --- /dev/null +++ b/library/cpp/monlib/encode/protobuf/protobuf.h @@ -0,0 +1,16 @@ +#pragma once + +#include <library/cpp/monlib/encode/encoder.h> + +#include <library/cpp/monlib/encode/protobuf/protos/samples.pb.h> + +namespace NMonitoring { + namespace NProto { + class TSingleSamplesList; + class TMultiSamplesList; + } + + IMetricEncoderPtr EncoderProtobuf(NProto::TSingleSamplesList* samples); + IMetricEncoderPtr EncoderProtobuf(NProto::TMultiSamplesList* samples); + +} diff --git a/library/cpp/monlib/encode/protobuf/protobuf_encoder.cpp b/library/cpp/monlib/encode/protobuf/protobuf_encoder.cpp new file mode 100644 index 0000000000..2d11b9d5ba --- /dev/null +++ b/library/cpp/monlib/encode/protobuf/protobuf_encoder.cpp @@ -0,0 +1,248 @@ +#include "protobuf.h" + +#include <util/datetime/base.h> + +namespace NMonitoring { + namespace { + NProto::EMetricType ConvertMetricType(EMetricType type) { + switch (type) { + case EMetricType::GAUGE: + return NProto::GAUGE; + case EMetricType::COUNTER: + return NProto::COUNTER; + case EMetricType::RATE: + return NProto::RATE; + case EMetricType::IGAUGE: + return NProto::IGAUGE; + case EMetricType::HIST: + return NProto::HISTOGRAM; + case EMetricType::HIST_RATE: + return NProto::HIST_RATE; + case EMetricType::DSUMMARY: + return NProto::DSUMMARY; + case EMetricType::LOGHIST: + return NProto::LOGHISTOGRAM; + case EMetricType::UNKNOWN: + return NProto::UNKNOWN; + } + } + + void FillHistogram( + const IHistogramSnapshot& snapshot, + NProto::THistogram* histogram) + { + for (ui32 i = 0; i < snapshot.Count(); i++) { + histogram->AddBounds(snapshot.UpperBound(i)); + histogram->AddValues(snapshot.Value(i)); + } + } + + void FillSummaryDouble(const ISummaryDoubleSnapshot& snapshot, NProto::TSummaryDouble* summary) { + summary->SetSum(snapshot.GetSum()); + summary->SetMin(snapshot.GetMin()); + summary->SetMax(snapshot.GetMax()); + summary->SetLast(snapshot.GetLast()); + summary->SetCount(snapshot.GetCount()); + } + + void FillLogHistogram(const TLogHistogramSnapshot& snapshot, NProto::TLogHistogram* logHist) { + logHist->SetBase(snapshot.Base()); + logHist->SetZerosCount(snapshot.ZerosCount()); + logHist->SetStartPower(snapshot.StartPower()); + for (ui32 i = 0; i < snapshot.Count(); ++i) { + logHist->AddBuckets(snapshot.Bucket(i)); + } + } + + /////////////////////////////////////////////////////////////////////////////// + // TSingleamplesEncoder + /////////////////////////////////////////////////////////////////////////////// + class TSingleSamplesEncoder final: public IMetricEncoder { + public: + TSingleSamplesEncoder(NProto::TSingleSamplesList* samples) + : Samples_(samples) + , Sample_(nullptr) + { + } + + private: + void OnStreamBegin() override { + } + void OnStreamEnd() override { + } + + void OnCommonTime(TInstant time) override { + Samples_->SetCommonTime(time.MilliSeconds()); + } + + void OnMetricBegin(EMetricType type) override { + Sample_ = Samples_->AddSamples(); + Sample_->SetMetricType(ConvertMetricType(type)); + } + + void OnMetricEnd() override { + Sample_ = nullptr; + } + + void OnLabelsBegin() override { + } + void OnLabelsEnd() override { + } + + void OnLabel(TStringBuf name, TStringBuf value) override { + NProto::TLabel* label = (Sample_ == nullptr) + ? Samples_->AddCommonLabels() + : Sample_->AddLabels(); + label->SetName(TString{name}); + label->SetValue(TString{value}); + } + + void OnDouble(TInstant time, double value) override { + Y_ENSURE(Sample_, "metric not started"); + Sample_->SetTime(time.MilliSeconds()); + Sample_->SetFloat64(value); + } + + void OnInt64(TInstant time, i64 value) override { + Y_ENSURE(Sample_, "metric not started"); + Sample_->SetTime(time.MilliSeconds()); + Sample_->SetInt64(value); + } + + void OnUint64(TInstant time, ui64 value) override { + Y_ENSURE(Sample_, "metric not started"); + Sample_->SetTime(time.MilliSeconds()); + Sample_->SetUint64(value); + } + + void OnHistogram(TInstant time, IHistogramSnapshotPtr snapshot) override { + Y_ENSURE(Sample_, "metric not started"); + Sample_->SetTime(time.MilliSeconds()); + FillHistogram(*snapshot, Sample_->MutableHistogram()); + } + + void OnSummaryDouble(TInstant time, ISummaryDoubleSnapshotPtr snapshot) override { + Y_ENSURE(Sample_, "metric not started"); + Sample_->SetTime(time.MilliSeconds()); + FillSummaryDouble(*snapshot, Sample_->MutableSummaryDouble()); + } + + void OnLogHistogram(TInstant time, TLogHistogramSnapshotPtr snapshot) override { + Y_ENSURE(Sample_, "metric not started"); + Sample_->SetTime(time.MilliSeconds()); + FillLogHistogram(*snapshot, Sample_->MutableLogHistogram()); + } + + void Close() override { + } + + private: + NProto::TSingleSamplesList* Samples_; + NProto::TSingleSample* Sample_; + }; + + /////////////////////////////////////////////////////////////////////////////// + // TMultiSamplesEncoder + /////////////////////////////////////////////////////////////////////////////// + class TMultiSamplesEncoder final: public IMetricEncoder { + public: + TMultiSamplesEncoder(NProto::TMultiSamplesList* samples) + : Samples_(samples) + , Sample_(nullptr) + { + } + + private: + void OnStreamBegin() override { + } + void OnStreamEnd() override { + } + + void OnCommonTime(TInstant time) override { + Samples_->SetCommonTime(time.MilliSeconds()); + } + + void OnMetricBegin(EMetricType type) override { + Sample_ = Samples_->AddSamples(); + Sample_->SetMetricType(ConvertMetricType(type)); + } + + void OnMetricEnd() override { + Sample_ = nullptr; + } + + void OnLabelsBegin() override { + } + void OnLabelsEnd() override { + } + + void OnLabel(TStringBuf name, TStringBuf value) override { + NProto::TLabel* label = (Sample_ == nullptr) + ? Samples_->AddCommonLabels() + : Sample_->AddLabels(); + + label->SetName(TString{name}); + label->SetValue(TString{value}); + } + + void OnDouble(TInstant time, double value) override { + Y_ENSURE(Sample_, "metric not started"); + NProto::TPoint* point = Sample_->AddPoints(); + point->SetTime(time.MilliSeconds()); + point->SetFloat64(value); + } + + void OnInt64(TInstant time, i64 value) override { + Y_ENSURE(Sample_, "metric not started"); + NProto::TPoint* point = Sample_->AddPoints(); + point->SetTime(time.MilliSeconds()); + point->SetInt64(value); + } + + void OnUint64(TInstant time, ui64 value) override { + Y_ENSURE(Sample_, "metric not started"); + NProto::TPoint* point = Sample_->AddPoints(); + point->SetTime(time.MilliSeconds()); + point->SetUint64(value); + } + + void OnHistogram(TInstant time, IHistogramSnapshotPtr snapshot) override { + Y_ENSURE(Sample_, "metric not started"); + NProto::TPoint* point = Sample_->AddPoints(); + point->SetTime(time.MilliSeconds()); + FillHistogram(*snapshot, point->MutableHistogram()); + } + + void OnSummaryDouble(TInstant time, ISummaryDoubleSnapshotPtr snapshot) override { + Y_ENSURE(Sample_, "metric not started"); + NProto::TPoint* point = Sample_->AddPoints(); + point->SetTime(time.MilliSeconds()); + FillSummaryDouble(*snapshot, point->MutableSummaryDouble()); + } + + void OnLogHistogram(TInstant time, TLogHistogramSnapshotPtr snapshot) override { + Y_ENSURE(Sample_, "metric not started"); + NProto::TPoint* point = Sample_->AddPoints(); + point->SetTime(time.MilliSeconds()); + FillLogHistogram(*snapshot, point->MutableLogHistogram()); + } + + void Close() override { + } + + private: + NProto::TMultiSamplesList* Samples_; + NProto::TMultiSample* Sample_; + }; + + } + + IMetricEncoderPtr EncoderProtobuf(NProto::TSingleSamplesList* samples) { + return MakeHolder<TSingleSamplesEncoder>(samples); + } + + IMetricEncoderPtr EncoderProtobuf(NProto::TMultiSamplesList* samples) { + return MakeHolder<TMultiSamplesEncoder>(samples); + } + +} diff --git a/library/cpp/monlib/encode/protobuf/protos/samples.proto b/library/cpp/monlib/encode/protobuf/protos/samples.proto new file mode 100644 index 0000000000..371f4181d2 --- /dev/null +++ b/library/cpp/monlib/encode/protobuf/protos/samples.proto @@ -0,0 +1,91 @@ +syntax = 'proto3'; + +package NMonitoring.NProto; + +option java_package = "ru.yandex.solomon.protos"; +option java_multiple_files = true; +option cc_enable_arenas = true; + +message TLabel { + string Name = 1; + string Value = 2; +} + +enum EMetricType { + UNKNOWN = 0; + GAUGE = 1; + IGAUGE = 2; + COUNTER = 3; + RATE = 4; + HISTOGRAM = 5; + HIST_RATE = 6; + DSUMMARY = 7; + LOGHISTOGRAM = 8; +} + +message THistogram { + repeated double Bounds = 1; // upper bounds of each bucket + repeated uint64 Values = 2; // values stored in each bucket +} + +message TLogHistogram { + double Base = 1; + uint64 ZerosCount = 2; + int32 StartPower = 3; + repeated double Buckets = 4; +} + +message TSummaryDouble { + double Sum = 1; + double Min = 2; + double Max = 3; + double Last = 4; + uint64 Count = 5; +} + +// see TSingleSample +message TPoint { + uint64 Time = 1; + oneof Value { + sfixed64 Int64 = 2; + fixed64 Uint64 = 3; + double Float64 = 4; + THistogram Histogram = 5; + TSummaryDouble SummaryDouble = 6; + TLogHistogram LogHistogram = 7; + } +} + +message TSingleSample { + repeated TLabel Labels = 1; + EMetricType MetricType = 2; + + // inlined TPoint + uint64 Time = 3; + oneof Value { + sfixed64 Int64 = 4; + fixed64 Uint64 = 5; + double Float64 = 6; + THistogram Histogram = 7; + TSummaryDouble SummaryDouble = 8; + TLogHistogram LogHistogram = 9; + } +} + +message TMultiSample { + repeated TLabel Labels = 1; + EMetricType MetricType = 2; + repeated TPoint Points = 3; +} + +message TSingleSamplesList { + uint64 CommonTime = 1; + repeated TLabel CommonLabels = 2; + repeated TSingleSample Samples = 3; +} + +message TMultiSamplesList { + uint64 CommonTime = 1; + repeated TLabel CommonLabels = 2; + repeated TMultiSample Samples = 3; +} diff --git a/library/cpp/monlib/encode/protobuf/protos/ya.make b/library/cpp/monlib/encode/protobuf/protos/ya.make new file mode 100644 index 0000000000..88ff3ddf88 --- /dev/null +++ b/library/cpp/monlib/encode/protobuf/protos/ya.make @@ -0,0 +1,14 @@ +PROTO_LIBRARY() + +OWNER( + jamel + g:solomon +) + +SRCS( + samples.proto +) + +EXCLUDE_TAGS(GO_PROTO) + +END() diff --git a/library/cpp/monlib/encode/protobuf/ya.make b/library/cpp/monlib/encode/protobuf/ya.make new file mode 100644 index 0000000000..9354958b6f --- /dev/null +++ b/library/cpp/monlib/encode/protobuf/ya.make @@ -0,0 +1,17 @@ +LIBRARY() + +OWNER( + jamel + g:solomon +) + +SRCS( + protobuf_encoder.cpp +) + +PEERDIR( + library/cpp/monlib/encode + library/cpp/monlib/encode/protobuf/protos +) + +END() diff --git a/library/cpp/monlib/encode/spack/compression.cpp b/library/cpp/monlib/encode/spack/compression.cpp new file mode 100644 index 0000000000..0d2152fc85 --- /dev/null +++ b/library/cpp/monlib/encode/spack/compression.cpp @@ -0,0 +1,383 @@ +#include "compression.h" + +#include <util/generic/buffer.h> +#include <util/generic/cast.h> +#include <util/generic/ptr.h> +#include <util/generic/scope.h> +#include <util/generic/size_literals.h> +#include <util/stream/format.h> +#include <util/stream/output.h> +#include <util/stream/walk.h> + +#include <contrib/libs/lz4/lz4.h> +#include <contrib/libs/xxhash/xxhash.h> +#include <contrib/libs/zlib/zlib.h> +#define ZSTD_STATIC_LINKING_ONLY +#include <contrib/libs/zstd/include/zstd.h> + +namespace NMonitoring { + namespace { + /////////////////////////////////////////////////////////////////////////////// + // Frame + /////////////////////////////////////////////////////////////////////////////// + using TCompressedSize = ui32; + using TUncompressedSize = ui32; + using TCheckSum = ui32; + + constexpr size_t COMPRESSED_FRAME_SIZE_LIMIT = 512_KB; + constexpr size_t UNCOMPRESSED_FRAME_SIZE_LIMIT = COMPRESSED_FRAME_SIZE_LIMIT; + constexpr size_t FRAME_SIZE_LIMIT = 2_MB; + constexpr size_t DEFAULT_FRAME_LEN = 64_KB; + + struct Y_PACKED TFrameHeader { + TCompressedSize CompressedSize; + TUncompressedSize UncompressedSize; + }; + + struct Y_PACKED TFrameFooter { + TCheckSum CheckSum; + }; + + /////////////////////////////////////////////////////////////////////////////// + // TBlock + /////////////////////////////////////////////////////////////////////////////// + struct TBlock: public TStringBuf { + template <typename T> + TBlock(T&& t) + : TStringBuf(t.data(), t.size()) + { + Y_ENSURE(t.data() != nullptr); + } + + char* data() noexcept { + return const_cast<char*>(TStringBuf::data()); + } + }; + + /////////////////////////////////////////////////////////////////////////////// + // XXHASH + /////////////////////////////////////////////////////////////////////////////// + struct TXxHash32 { + static TCheckSum Calc(TBlock in) { + static const ui32 SEED = 0x1337c0de; + return XXH32(in.data(), in.size(), SEED); + } + + static bool Check(TBlock in, TCheckSum checksum) { + return Calc(in) == checksum; + } + }; + + /////////////////////////////////////////////////////////////////////////////// + // Adler32 + /////////////////////////////////////////////////////////////////////////////// + struct TAdler32 { + static TCheckSum Calc(TBlock in) { + return adler32(1L, reinterpret_cast<const Bytef*>(in.data()), in.size()); + } + + static bool Check(TBlock in, TCheckSum checksum) { + return Calc(in) == checksum; + } + }; + + /////////////////////////////////////////////////////////////////////////////// + // LZ4 + /////////////////////////////////////////////////////////////////////////////// + struct TLz4Codec { + static size_t MaxCompressedLength(size_t in) { + int result = LZ4_compressBound(static_cast<int>(in)); + Y_ENSURE(result != 0, "lz4 input size is too large"); + return result; + } + + static size_t Compress(TBlock in, TBlock out) { + int rc = LZ4_compress_default( + in.data(), + out.data(), + SafeIntegerCast<int>(in.size()), + SafeIntegerCast<int>(out.size())); + Y_ENSURE(rc != 0, "lz4 compression failed"); + return rc; + } + + static void Decompress(TBlock in, TBlock out) { + int rc = LZ4_decompress_safe( + in.data(), + out.data(), + SafeIntegerCast<int>(in.size()), + SafeIntegerCast<int>(out.size())); + Y_ENSURE(rc >= 0, "the lz4 stream is detected malformed"); + } + }; + + /////////////////////////////////////////////////////////////////////////////// + // ZSTD + /////////////////////////////////////////////////////////////////////////////// + struct TZstdCodec { + static const int LEVEL = 11; + + static size_t MaxCompressedLength(size_t in) { + return ZSTD_compressBound(in); + } + + static size_t Compress(TBlock in, TBlock out) { + size_t rc = ZSTD_compress(out.data(), out.size(), in.data(), in.size(), LEVEL); + if (Y_UNLIKELY(ZSTD_isError(rc))) { + ythrow yexception() << TStringBuf("zstd compression failed: ") + << ZSTD_getErrorName(rc); + } + return rc; + } + + static void Decompress(TBlock in, TBlock out) { + size_t rc = ZSTD_decompress(out.data(), out.size(), in.data(), in.size()); + if (Y_UNLIKELY(ZSTD_isError(rc))) { + ythrow yexception() << TStringBuf("zstd decompression failed: ") + << ZSTD_getErrorName(rc); + } + Y_ENSURE(rc == out.size(), "zstd decompressed wrong size"); + } + }; + + /////////////////////////////////////////////////////////////////////////////// + // ZLIB + /////////////////////////////////////////////////////////////////////////////// + struct TZlibCodec { + static const int LEVEL = 6; + + static size_t MaxCompressedLength(size_t in) { + return compressBound(in); + } + + static size_t Compress(TBlock in, TBlock out) { + uLong ret = out.size(); + int rc = compress2( + reinterpret_cast<Bytef*>(out.data()), + &ret, + reinterpret_cast<const Bytef*>(in.data()), + in.size(), + LEVEL); + Y_ENSURE(rc == Z_OK, "zlib compression failed"); + return ret; + } + + static void Decompress(TBlock in, TBlock out) { + uLong ret = out.size(); + int rc = uncompress( + reinterpret_cast<Bytef*>(out.data()), + &ret, + reinterpret_cast<const Bytef*>(in.data()), + in.size()); + Y_ENSURE(rc == Z_OK, "zlib decompression failed"); + Y_ENSURE(ret == out.size(), "zlib decompressed wrong size"); + } + }; + + // + // Framed streams use next frame structure: + // + // +-----------------+-------------------+============+------------------+ + // | compressed size | uncompressed size | data | check sum | + // +-----------------+-------------------+============+------------------+ + // 4 bytes 4 bytes var len 4 bytes + // + + /////////////////////////////////////////////////////////////////////////////// + // TFramedInputStream + /////////////////////////////////////////////////////////////////////////////// + template <typename TCodecAlg, typename TCheckSumAlg> + class TFramedDecompressStream final: public IWalkInput { + public: + explicit TFramedDecompressStream(IInputStream* in) + : In_(in) + { + } + + private: + size_t DoUnboundedNext(const void** ptr) override { + if (!In_) { + return 0; + } + + TFrameHeader header; + In_->LoadOrFail(&header, sizeof(header)); + + if (header.CompressedSize == 0) { + In_ = nullptr; + return 0; + } + + Y_ENSURE(header.CompressedSize <= COMPRESSED_FRAME_SIZE_LIMIT, "Compressed frame size is limited to " + << HumanReadableSize(COMPRESSED_FRAME_SIZE_LIMIT, SF_BYTES) + << " but is " << HumanReadableSize(header.CompressedSize, SF_BYTES)); + + Y_ENSURE(header.UncompressedSize <= UNCOMPRESSED_FRAME_SIZE_LIMIT, "Uncompressed frame size is limited to " + << HumanReadableSize(UNCOMPRESSED_FRAME_SIZE_LIMIT, SF_BYTES) + << " but is " << HumanReadableSize(header.UncompressedSize, SF_BYTES)); + + Compressed_.Resize(header.CompressedSize); + In_->LoadOrFail(Compressed_.Data(), header.CompressedSize); + + TFrameFooter footer; + In_->LoadOrFail(&footer, sizeof(footer)); + Y_ENSURE(TCheckSumAlg::Check(Compressed_, footer.CheckSum), + "corrupted stream: check sum mismatch"); + + Uncompressed_.Resize(header.UncompressedSize); + TCodecAlg::Decompress(Compressed_, Uncompressed_); + + *ptr = Uncompressed_.Data(); + return Uncompressed_.Size(); + } + + private: + IInputStream* In_; + TBuffer Compressed_; + TBuffer Uncompressed_; + }; + + /////////////////////////////////////////////////////////////////////////////// + // TFramedOutputStream + /////////////////////////////////////////////////////////////////////////////// + template <typename TCodecAlg, typename TCheckSumAlg> + class TFramedCompressStream final: public IFramedCompressStream { + public: + explicit TFramedCompressStream(IOutputStream* out) + : Out_(out) + , Uncompressed_(DEFAULT_FRAME_LEN) + { + } + + ~TFramedCompressStream() override { + try { + Finish(); + } catch (...) { + } + } + + private: + void DoWrite(const void* buf, size_t len) override { + const char* in = static_cast<const char*>(buf); + + while (len != 0) { + const size_t avail = Uncompressed_.Avail(); + if (len < avail) { + Uncompressed_.Append(in, len); + return; + } + + Uncompressed_.Append(in, avail); + Y_ASSERT(Uncompressed_.Avail() == 0); + + in += avail; + len -= avail; + + WriteCompressedFrame(); + } + } + + void FlushWithoutEmptyFrame() override { + if (Out_ && !Uncompressed_.Empty()) { + WriteCompressedFrame(); + } + } + + void FinishAndWriteEmptyFrame() override { + if (Out_) { + Y_DEFER { + Out_ = nullptr; + }; + + if (!Uncompressed_.Empty()) { + WriteCompressedFrame(); + } + + WriteEmptyFrame(); + } + } + + void DoFlush() override { + FlushWithoutEmptyFrame(); + } + + void DoFinish() override { + FinishAndWriteEmptyFrame(); + } + + void WriteCompressedFrame() { + static const auto framePayload = sizeof(TFrameHeader) + sizeof(TFrameFooter); + const auto maxFrameSize = ui64(TCodecAlg::MaxCompressedLength(Uncompressed_.Size())) + framePayload; + Y_ENSURE(maxFrameSize <= FRAME_SIZE_LIMIT, "Frame size in encoder is limited to " + << HumanReadableSize(FRAME_SIZE_LIMIT, SF_BYTES) + << " but is " << HumanReadableSize(maxFrameSize, SF_BYTES)); + + Frame_.Resize(maxFrameSize); + + // compress + TBlock compressedBlock = Frame_; + compressedBlock.Skip(sizeof(TFrameHeader)); + compressedBlock.Trunc(TCodecAlg::Compress(Uncompressed_, compressedBlock)); + + // add header + auto header = reinterpret_cast<TFrameHeader*>(Frame_.Data()); + header->CompressedSize = SafeIntegerCast<TCompressedSize>(compressedBlock.size()); + header->UncompressedSize = SafeIntegerCast<TUncompressedSize>(Uncompressed_.Size()); + + // add footer + auto footer = reinterpret_cast<TFrameFooter*>( + Frame_.Data() + sizeof(TFrameHeader) + header->CompressedSize); + footer->CheckSum = TCheckSumAlg::Calc(compressedBlock); + + // write + Out_->Write(Frame_.Data(), header->CompressedSize + framePayload); + Uncompressed_.Clear(); + } + + void WriteEmptyFrame() { + static const auto framePayload = sizeof(TFrameHeader) + sizeof(TFrameFooter); + char buf[framePayload] = {0}; + Out_->Write(buf, sizeof(buf)); + } + + private: + IOutputStream* Out_; + TBuffer Uncompressed_; + TBuffer Frame_; + }; + + } + + THolder<IInputStream> CompressedInput(IInputStream* in, ECompression alg) { + switch (alg) { + case ECompression::IDENTITY: + return nullptr; + case ECompression::ZLIB: + return MakeHolder<TFramedDecompressStream<TZlibCodec, TAdler32>>(in); + case ECompression::ZSTD: + return MakeHolder<TFramedDecompressStream<TZstdCodec, TXxHash32>>(in); + case ECompression::LZ4: + return MakeHolder<TFramedDecompressStream<TLz4Codec, TXxHash32>>(in); + case ECompression::UNKNOWN: + return nullptr; + } + Y_FAIL("invalid compression algorithm"); + } + + THolder<IFramedCompressStream> CompressedOutput(IOutputStream* out, ECompression alg) { + switch (alg) { + case ECompression::IDENTITY: + return nullptr; + case ECompression::ZLIB: + return MakeHolder<TFramedCompressStream<TZlibCodec, TAdler32>>(out); + case ECompression::ZSTD: + return MakeHolder<TFramedCompressStream<TZstdCodec, TXxHash32>>(out); + case ECompression::LZ4: + return MakeHolder<TFramedCompressStream<TLz4Codec, TXxHash32>>(out); + case ECompression::UNKNOWN: + return nullptr; + } + Y_FAIL("invalid compression algorithm"); + } + +} diff --git a/library/cpp/monlib/encode/spack/compression.h b/library/cpp/monlib/encode/spack/compression.h new file mode 100644 index 0000000000..f74d8b424e --- /dev/null +++ b/library/cpp/monlib/encode/spack/compression.h @@ -0,0 +1,19 @@ +#pragma once + +#include "spack_v1.h" + +#include <util/stream/input.h> +#include <util/stream/output.h> + +namespace NMonitoring { + +class IFramedCompressStream: public IOutputStream { +public: + virtual void FlushWithoutEmptyFrame() = 0; + virtual void FinishAndWriteEmptyFrame() = 0; +}; + +THolder<IInputStream> CompressedInput(IInputStream* in, ECompression alg); +THolder<IFramedCompressStream> CompressedOutput(IOutputStream* out, ECompression alg); + +} // namespace NMonitoring diff --git a/library/cpp/monlib/encode/spack/fuzz/main.cpp b/library/cpp/monlib/encode/spack/fuzz/main.cpp new file mode 100644 index 0000000000..6a14afe71c --- /dev/null +++ b/library/cpp/monlib/encode/spack/fuzz/main.cpp @@ -0,0 +1,20 @@ +#include <library/cpp/monlib/encode/spack/spack_v1.h> +#include <library/cpp/monlib/encode/fake/fake.h> + +#include <util/stream/mem.h> + + +extern "C" int LLVMFuzzerTestOneInput(const ui8* data, size_t size) { + using namespace NMonitoring; + + TMemoryInput min{data, size}; + + auto encoder = EncoderFake(); + + try { + DecodeSpackV1(&min, encoder.Get()); + } catch (...) { + } + + return 0; +} diff --git a/library/cpp/monlib/encode/spack/fuzz/ya.make b/library/cpp/monlib/encode/spack/fuzz/ya.make new file mode 100644 index 0000000000..99b63eadd5 --- /dev/null +++ b/library/cpp/monlib/encode/spack/fuzz/ya.make @@ -0,0 +1,21 @@ +FUZZ() + +OWNER( + g:solomon + msherbakov +) + +FUZZ_OPTS(-rss_limit_mb=1024) + +SIZE(MEDIUM) + +PEERDIR( + library/cpp/monlib/encode/spack + library/cpp/monlib/encode/fake +) + +SRCS( + main.cpp +) + +END() diff --git a/library/cpp/monlib/encode/spack/spack_v1.h b/library/cpp/monlib/encode/spack/spack_v1.h new file mode 100644 index 0000000000..cf1c9417b9 --- /dev/null +++ b/library/cpp/monlib/encode/spack/spack_v1.h @@ -0,0 +1,115 @@ +#pragma once + +#include <library/cpp/monlib/encode/encoder.h> +#include <library/cpp/monlib/encode/format.h> +#include <library/cpp/monlib/metrics/metric.h> + +#include <util/generic/yexception.h> + +// +// format specification available here: +// https://wiki.yandex-team.ru/solomon/api/dataformat/spackv1/ +// + +class IInputStream; +class IOutputStream; + +namespace NMonitoring { + class TSpackDecodeError: public yexception { + }; + + constexpr auto EncodeMetricType(EMetricType mt) noexcept { + return static_cast<std::underlying_type_t<EMetricType>>(mt); + } + + EMetricType DecodeMetricType(ui8 byte); + + [[nodiscard]] + bool TryDecodeMetricType(ui8 byte, EMetricType* result); + + /////////////////////////////////////////////////////////////////////////////// + // EValueType + /////////////////////////////////////////////////////////////////////////////// + enum class EValueType : ui8 { + NONE = 0x00, + ONE_WITHOUT_TS = 0x01, + ONE_WITH_TS = 0x02, + MANY_WITH_TS = 0x03, + }; + + constexpr auto EncodeValueType(EValueType vt) noexcept { + return static_cast<std::underlying_type_t<EValueType>>(vt); + } + + EValueType DecodeValueType(ui8 byte); + + [[nodiscard]] + bool TryDecodeValueType(ui8 byte, EValueType* result); + + /////////////////////////////////////////////////////////////////////////////// + // ETimePrecision + /////////////////////////////////////////////////////////////////////////////// + enum class ETimePrecision : ui8 { + SECONDS = 0x00, + MILLIS = 0x01, + }; + + constexpr auto EncodeTimePrecision(ETimePrecision tp) noexcept { + return static_cast<std::underlying_type_t<ETimePrecision>>(tp); + } + + ETimePrecision DecodeTimePrecision(ui8 byte); + + [[nodiscard]] + bool TryDecodeTimePrecision(ui8 byte, ETimePrecision* result); + + /////////////////////////////////////////////////////////////////////////////// + // ECompression + /////////////////////////////////////////////////////////////////////////////// + ui8 EncodeCompression(ECompression c) noexcept; + + ECompression DecodeCompression(ui8 byte); + + [[nodiscard]] + bool TryDecodeCompression(ui8 byte, ECompression* result); + + /////////////////////////////////////////////////////////////////////////////// + // TSpackHeader + /////////////////////////////////////////////////////////////////////////////// + struct Y_PACKED TSpackHeader { + ui16 Magic = 0x5053; // "SP" + ui16 Version; // MSB - major version, LSB - minor version + ui16 HeaderSize = sizeof(TSpackHeader); + ui8 TimePrecision; + ui8 Compression; + ui32 LabelNamesSize; + ui32 LabelValuesSize; + ui32 MetricCount; + ui32 PointsCount; + // add new fields here + }; + + enum ESpackV1Version: ui16 { + SV1_00 = 0x0100, + SV1_01 = 0x0101, + SV1_02 = 0x0102 + }; + + IMetricEncoderPtr EncoderSpackV1( + IOutputStream* out, + ETimePrecision timePrecision, + ECompression compression, + EMetricsMergingMode mergingMode = EMetricsMergingMode::DEFAULT + ); + + IMetricEncoderPtr EncoderSpackV12( + IOutputStream* out, + ETimePrecision timePrecision, + ECompression compression, + EMetricsMergingMode mergingMode = EMetricsMergingMode::DEFAULT, + TStringBuf metricNameLabel = "name" + ); + + void DecodeSpackV1(IInputStream* in, IMetricConsumer* c, TStringBuf metricNameLabel = "name"); + +} diff --git a/library/cpp/monlib/encode/spack/spack_v1_decoder.cpp b/library/cpp/monlib/encode/spack/spack_v1_decoder.cpp new file mode 100644 index 0000000000..1f445fc80d --- /dev/null +++ b/library/cpp/monlib/encode/spack/spack_v1_decoder.cpp @@ -0,0 +1,458 @@ +#include "spack_v1.h" +#include "varint.h" +#include "compression.h" + +#include <library/cpp/monlib/encode/buffered/string_pool.h> +#include <library/cpp/monlib/exception/exception.h> +#include <library/cpp/monlib/metrics/histogram_collector.h> +#include <library/cpp/monlib/metrics/metric.h> + +#include <util/generic/yexception.h> +#include <util/generic/buffer.h> +#include <util/generic/size_literals.h> +#include <util/stream/format.h> + +#ifndef _little_endian_ +#error Unsupported platform +#endif + +namespace NMonitoring { + namespace { +#define DECODE_ENSURE(COND, ...) MONLIB_ENSURE_EX(COND, TSpackDecodeError() << __VA_ARGS__) + + constexpr ui64 LABEL_SIZE_LIMIT = 128_MB; + + /////////////////////////////////////////////////////////////////////// + // TDecoderSpackV1 + /////////////////////////////////////////////////////////////////////// + class TDecoderSpackV1 { + public: + TDecoderSpackV1(IInputStream* in, TStringBuf metricNameLabel) + : In_(in) + , MetricNameLabel_(metricNameLabel) + { + } + + void Decode(IMetricConsumer* c) { + c->OnStreamBegin(); + + // (1) read header + size_t readBytes = In_->Read(&Header_, sizeof(Header_)); + DECODE_ENSURE(readBytes == sizeof(Header_), "not enough data in input stream to read header"); + + ui8 version = ((Header_.Version >> 8) & 0xff); + DECODE_ENSURE(version == 1, "versions mismatch (expected: 1, got: " << +version << ')'); + + DECODE_ENSURE(Header_.HeaderSize >= sizeof(Header_), "invalid header size"); + if (size_t skipBytes = Header_.HeaderSize - sizeof(Header_)) { + DECODE_ENSURE(In_->Skip(skipBytes) == skipBytes, "input stream unexpectedly ended"); + } + + if (Header_.MetricCount == 0) { + // emulate empty stream + c->OnStreamEnd(); + return; + } + + // if compression enabled all below reads must go throught decompressor + auto compressedIn = CompressedInput(In_, DecodeCompression(Header_.Compression)); + if (compressedIn) { + In_ = compressedIn.Get(); + } + + TimePrecision_ = DecodeTimePrecision(Header_.TimePrecision); + + const ui64 labelSizeTotal = ui64(Header_.LabelNamesSize) + Header_.LabelValuesSize; + + DECODE_ENSURE(labelSizeTotal <= LABEL_SIZE_LIMIT, "Label names & values size of " << HumanReadableSize(labelSizeTotal, SF_BYTES) + << " exceeds the limit which is " << HumanReadableSize(LABEL_SIZE_LIMIT, SF_BYTES)); + + // (2) read string pools + TVector<char> namesBuf(Header_.LabelNamesSize); + readBytes = In_->Load(namesBuf.data(), namesBuf.size()); + DECODE_ENSURE(readBytes == Header_.LabelNamesSize, "not enough data to read label names pool"); + TStringPool labelNames(namesBuf.data(), namesBuf.size()); + + TVector<char> valuesBuf(Header_.LabelValuesSize); + readBytes = In_->Load(valuesBuf.data(), valuesBuf.size()); + DECODE_ENSURE(readBytes == Header_.LabelValuesSize, "not enough data to read label values pool"); + TStringPool labelValues(valuesBuf.data(), valuesBuf.size()); + + // (3) read common time + c->OnCommonTime(ReadTime()); + + // (4) read common labels + if (ui32 commonLabelsCount = ReadVarint()) { + c->OnLabelsBegin(); + ReadLabels(labelNames, labelValues, commonLabelsCount, c); + c->OnLabelsEnd(); + } + + // (5) read metrics + ReadMetrics(labelNames, labelValues, c); + c->OnStreamEnd(); + } + + private: + void ReadMetrics( + const TStringPool& labelNames, + const TStringPool& labelValues, + IMetricConsumer* c) + { + for (ui32 i = 0; i < Header_.MetricCount; i++) { + // (5.1) types byte + ui8 typesByte = ReadFixed<ui8>(); + EMetricType metricType = DecodeMetricType(typesByte >> 2); + EValueType valueType = DecodeValueType(typesByte & 0x03); + + c->OnMetricBegin(metricType); + + // TODO: use it + ReadFixed<ui8>(); // skip flags byte + + auto metricNameValueIndex = std::numeric_limits<ui32>::max(); + if (Header_.Version >= SV1_02) { + metricNameValueIndex = ReadVarint(); + } + + // (5.2) labels + ui32 labelsCount = ReadVarint(); + DECODE_ENSURE(Header_.Version >= SV1_02 || labelsCount > 0, "metric #" << i << " has no labels"); + c->OnLabelsBegin(); + if (Header_.Version >= SV1_02) { + c->OnLabel(MetricNameLabel_, labelValues.Get(metricNameValueIndex)); + } + ReadLabels(labelNames, labelValues, labelsCount, c); + c->OnLabelsEnd(); + + // (5.3) values + switch (valueType) { + case EValueType::NONE: + break; + case EValueType::ONE_WITHOUT_TS: + ReadValue(metricType, TInstant::Zero(), c); + break; + case EValueType::ONE_WITH_TS: { + TInstant time = ReadTime(); + ReadValue(metricType, time, c); + break; + } + case EValueType::MANY_WITH_TS: { + ui32 pointsCount = ReadVarint(); + for (ui32 i = 0; i < pointsCount; i++) { + TInstant time = ReadTime(); + ReadValue(metricType, time, c); + } + break; + } + } + + c->OnMetricEnd(); + } + } + + void ReadValue(EMetricType metricType, TInstant time, IMetricConsumer* c) { + switch (metricType) { + case EMetricType::GAUGE: + c->OnDouble(time, ReadFixed<double>()); + break; + + case EMetricType::IGAUGE: + c->OnInt64(time, ReadFixed<i64>()); + break; + + case EMetricType::COUNTER: + case EMetricType::RATE: + c->OnUint64(time, ReadFixed<ui64>()); + break; + + case EMetricType::DSUMMARY: + c->OnSummaryDouble(time, ReadSummaryDouble()); + break; + + case EMetricType::HIST: + case EMetricType::HIST_RATE: + c->OnHistogram(time, ReadHistogram()); + break; + + case EMetricType::LOGHIST: + c->OnLogHistogram(time, ReadLogHistogram()); + break; + + default: + throw TSpackDecodeError() << "Unsupported metric type: " << metricType; + } + } + + ISummaryDoubleSnapshotPtr ReadSummaryDouble() { + ui64 count = ReadFixed<ui64>(); + double sum = ReadFixed<double>(); + double min = ReadFixed<double>(); + double max = ReadFixed<double>(); + double last = ReadFixed<double>(); + return MakeIntrusive<TSummaryDoubleSnapshot>(sum, min, max, last, count); + } + + TLogHistogramSnapshotPtr ReadLogHistogram() { + double base = ReadFixed<double>(); + ui64 zerosCount = ReadFixed<ui64>(); + int startPower = static_cast<int>(ReadVarint()); + ui32 count = ReadVarint(); + // see https://a.yandex-team.ru/arc/trunk/arcadia/infra/yasm/stockpile_client/points.cpp?rev=r8593154#L31 + // and https://a.yandex-team.ru/arc/trunk/arcadia/infra/yasm/common/points/hgram/normal/normal.h?rev=r8268697#L9 + // TODO: share this constant value + Y_ENSURE(count <= 100u, "more than 100 buckets in log histogram: " << count); + TVector<double> buckets; + buckets.reserve(count); + for (ui32 i = 0; i < count; ++i) { + buckets.emplace_back(ReadFixed<double>()); + } + return MakeIntrusive<TLogHistogramSnapshot>(base, zerosCount, startPower, std::move(buckets)); + } + + IHistogramSnapshotPtr ReadHistogram() { + ui32 bucketsCount = ReadVarint(); + auto s = TExplicitHistogramSnapshot::New(bucketsCount); + + if (SV1_00 == Header_.Version) { // v1.0 + for (ui32 i = 0; i < bucketsCount; i++) { + i64 bound = ReadFixed<i64>(); + double doubleBound = (bound != Max<i64>()) + ? static_cast<double>(bound) + : Max<double>(); + + (*s)[i].first = doubleBound; + } + } else { + for (ui32 i = 0; i < bucketsCount; i++) { + double doubleBound = ReadFixed<double>(); + (*s)[i].first = doubleBound; + } + } + + + // values + for (ui32 i = 0; i < bucketsCount; i++) { + (*s)[i].second = ReadFixed<ui64>(); + } + return s; + } + + void ReadLabels( + const TStringPool& labelNames, + const TStringPool& labelValues, + ui32 count, + IMetricConsumer* c) + { + for (ui32 i = 0; i < count; i++) { + auto nameIdx = ReadVarint(); + auto valueIdx = ReadVarint(); + c->OnLabel(labelNames.Get(nameIdx), labelValues.Get(valueIdx)); + } + } + + TInstant ReadTime() { + switch (TimePrecision_) { + case ETimePrecision::SECONDS: + return TInstant::Seconds(ReadFixed<ui32>()); + case ETimePrecision::MILLIS: + return TInstant::MilliSeconds(ReadFixed<ui64>()); + } + Y_FAIL("invalid time precision"); + } + + template <typename T> + inline T ReadFixed() { + T value; + size_t readBytes = In_->Load(&value, sizeof(T)); + DECODE_ENSURE(readBytes == sizeof(T), "no enough data to read " << TypeName<T>()); + return value; + } + + inline ui32 ReadVarint() { + return ReadVarUInt32(In_); + } + + private: + IInputStream* In_; + TString MetricNameLabel_; + ETimePrecision TimePrecision_; + TSpackHeader Header_; + }; // class TDecoderSpackV1 + +#undef DECODE_ENSURE + } // namespace + + EValueType DecodeValueType(ui8 byte) { + EValueType result; + if (!TryDecodeValueType(byte, &result)) { + throw TSpackDecodeError() << "unknown value type: " << byte; + } + return result; + } + + bool TryDecodeValueType(ui8 byte, EValueType* result) { + if (byte == EncodeValueType(EValueType::NONE)) { + if (result) { + *result = EValueType::NONE; + } + return true; + } else if (byte == EncodeValueType(EValueType::ONE_WITHOUT_TS)) { + if (result) { + *result = EValueType::ONE_WITHOUT_TS; + } + return true; + } else if (byte == EncodeValueType(EValueType::ONE_WITH_TS)) { + if (result) { + *result = EValueType::ONE_WITH_TS; + } + return true; + } else if (byte == EncodeValueType(EValueType::MANY_WITH_TS)) { + if (result) { + *result = EValueType::MANY_WITH_TS; + } + return true; + } else { + return false; + } + } + + ETimePrecision DecodeTimePrecision(ui8 byte) { + ETimePrecision result; + if (!TryDecodeTimePrecision(byte, &result)) { + throw TSpackDecodeError() << "unknown time precision: " << byte; + } + return result; + } + + bool TryDecodeTimePrecision(ui8 byte, ETimePrecision* result) { + if (byte == EncodeTimePrecision(ETimePrecision::SECONDS)) { + if (result) { + *result = ETimePrecision::SECONDS; + } + return true; + } else if (byte == EncodeTimePrecision(ETimePrecision::MILLIS)) { + if (result) { + *result = ETimePrecision::MILLIS; + } + return true; + } else { + return false; + } + } + + EMetricType DecodeMetricType(ui8 byte) { + EMetricType result; + if (!TryDecodeMetricType(byte, &result)) { + throw TSpackDecodeError() << "unknown metric type: " << byte; + } + return result; + } + + bool TryDecodeMetricType(ui8 byte, EMetricType* result) { + if (byte == EncodeMetricType(EMetricType::GAUGE)) { + if (result) { + *result = EMetricType::GAUGE; + } + return true; + } else if (byte == EncodeMetricType(EMetricType::COUNTER)) { + if (result) { + *result = EMetricType::COUNTER; + } + return true; + } else if (byte == EncodeMetricType(EMetricType::RATE)) { + if (result) { + *result = EMetricType::RATE; + } + return true; + } else if (byte == EncodeMetricType(EMetricType::IGAUGE)) { + if (result) { + *result = EMetricType::IGAUGE; + } + return true; + } else if (byte == EncodeMetricType(EMetricType::HIST)) { + if (result) { + *result = EMetricType::HIST; + } + return true; + } else if (byte == EncodeMetricType(EMetricType::HIST_RATE)) { + if (result) { + *result = EMetricType::HIST_RATE; + } + return true; + } else if (byte == EncodeMetricType(EMetricType::DSUMMARY)) { + if (result) { + *result = EMetricType::DSUMMARY; + } + return true; + } else if (byte == EncodeMetricType(EMetricType::LOGHIST)) { + if (result) { + *result = EMetricType::LOGHIST; + } + return true; + } else if (byte == EncodeMetricType(EMetricType::UNKNOWN)) { + if (result) { + *result = EMetricType::UNKNOWN; + } + return true; + } else { + return false; + } + } + + ui8 EncodeCompression(ECompression c) noexcept { + switch (c) { + case ECompression::IDENTITY: + return 0x00; + case ECompression::ZLIB: + return 0x01; + case ECompression::ZSTD: + return 0x02; + case ECompression::LZ4: + return 0x03; + case ECompression::UNKNOWN: + return Max<ui8>(); + } + Y_FAIL(); // for GCC + } + + ECompression DecodeCompression(ui8 byte) { + ECompression result; + if (!TryDecodeCompression(byte, &result)) { + throw TSpackDecodeError() << "unknown compression alg: " << byte; + } + return result; + } + + bool TryDecodeCompression(ui8 byte, ECompression* result) { + if (byte == EncodeCompression(ECompression::IDENTITY)) { + if (result) { + *result = ECompression::IDENTITY; + } + return true; + } else if (byte == EncodeCompression(ECompression::ZLIB)) { + if (result) { + *result = ECompression::ZLIB; + } + return true; + } else if (byte == EncodeCompression(ECompression::ZSTD)) { + if (result) { + *result = ECompression::ZSTD; + } + return true; + } else if (byte == EncodeCompression(ECompression::LZ4)) { + if (result) { + *result = ECompression::LZ4; + } + return true; + } else { + return false; + } + } + + void DecodeSpackV1(IInputStream* in, IMetricConsumer* c, TStringBuf metricNameLabel) { + TDecoderSpackV1 decoder(in, metricNameLabel); + decoder.Decode(c); + } + +} diff --git a/library/cpp/monlib/encode/spack/spack_v1_encoder.cpp b/library/cpp/monlib/encode/spack/spack_v1_encoder.cpp new file mode 100644 index 0000000000..a2b0bb5f50 --- /dev/null +++ b/library/cpp/monlib/encode/spack/spack_v1_encoder.cpp @@ -0,0 +1,318 @@ +#include "spack_v1.h" +#include "compression.h" +#include "varint.h" + +#include <library/cpp/monlib/encode/buffered/buffered_encoder_base.h> + +#include <util/generic/cast.h> +#include <util/datetime/base.h> +#include <util/string/builder.h> + +#ifndef _little_endian_ +#error Unsupported platform +#endif + +namespace NMonitoring { + namespace { + /////////////////////////////////////////////////////////////////////// + // TEncoderSpackV1 + /////////////////////////////////////////////////////////////////////// + class TEncoderSpackV1 final: public TBufferedEncoderBase { + public: + TEncoderSpackV1( + IOutputStream* out, + ETimePrecision timePrecision, + ECompression compression, + EMetricsMergingMode mergingMode, + ESpackV1Version version, + TStringBuf metricNameLabel + ) + : Out_(out) + , TimePrecision_(timePrecision) + , Compression_(compression) + , Version_(version) + , MetricName_(Version_ >= SV1_02 ? LabelNamesPool_.PutIfAbsent(metricNameLabel) : nullptr) + { + MetricsMergingMode_ = mergingMode; + + LabelNamesPool_.SetSorted(true); + LabelValuesPool_.SetSorted(true); + } + + ~TEncoderSpackV1() override { + Close(); + } + + private: + void OnDouble(TInstant time, double value) override { + TBufferedEncoderBase::OnDouble(time, value); + } + + void OnInt64(TInstant time, i64 value) override { + TBufferedEncoderBase::OnInt64(time, value); + } + + void OnUint64(TInstant time, ui64 value) override { + TBufferedEncoderBase::OnUint64(time, value); + } + + void OnHistogram(TInstant time, IHistogramSnapshotPtr snapshot) override { + TBufferedEncoderBase::OnHistogram(time, snapshot); + } + + void OnSummaryDouble(TInstant time, ISummaryDoubleSnapshotPtr snapshot) override { + TBufferedEncoderBase::OnSummaryDouble(time, snapshot); + } + + void OnLogHistogram(TInstant time, TLogHistogramSnapshotPtr snapshot) override { + TBufferedEncoderBase::OnLogHistogram(time, snapshot); + } + + void Close() override { + if (Closed_) { + return; + } + Closed_ = true; + + LabelNamesPool_.Build(); + LabelValuesPool_.Build(); + + // Sort all points uniquely by ts -- the size can decrease + ui64 pointsCount = 0; + for (TMetric& metric : Metrics_) { + if (metric.TimeSeries.Size() > 1) { + metric.TimeSeries.SortByTs(); + } + + pointsCount += metric.TimeSeries.Size(); + } + + // (1) write header + TSpackHeader header; + header.Version = Version_; + header.TimePrecision = EncodeTimePrecision(TimePrecision_); + header.Compression = EncodeCompression(Compression_); + header.LabelNamesSize = static_cast<ui32>( + LabelNamesPool_.BytesSize() + LabelNamesPool_.Count()); + header.LabelValuesSize = static_cast<ui32>( + LabelValuesPool_.BytesSize() + LabelValuesPool_.Count()); + header.MetricCount = Metrics_.size(); + header.PointsCount = pointsCount; + Out_->Write(&header, sizeof(header)); + + // if compression enabled all below writes must go throught compressor + auto compressedOut = CompressedOutput(Out_, Compression_); + if (compressedOut) { + Out_ = compressedOut.Get(); + } + + // (2) write string pools + auto strPoolWrite = [this](TStringBuf str, ui32, ui32) { + Out_->Write(str); + Out_->Write('\0'); + }; + + LabelNamesPool_.ForEach(strPoolWrite); + LabelValuesPool_.ForEach(strPoolWrite); + + // (3) write common time + WriteTime(CommonTime_); + + // (4) write common labels' indexes + WriteLabels(CommonLabels_, nullptr); + + // (5) write metrics + // metrics count already written in header + for (TMetric& metric : Metrics_) { + // (5.1) types byte + ui8 typesByte = PackTypes(metric); + Out_->Write(&typesByte, sizeof(typesByte)); + + // TODO: implement + ui8 flagsByte = 0x00; + Out_->Write(&flagsByte, sizeof(flagsByte)); + + // v1.2 format addition — metric name + if (Version_ >= SV1_02) { + const auto it = FindIf(metric.Labels, [&](const auto& l) { + return l.Key == MetricName_; + }); + Y_ENSURE(it != metric.Labels.end(), + "metric name label '" << LabelNamesPool_.Get(MetricName_->Index) << "' not found, " + << "all metric labels '" << FormatLabels(metric.Labels) << "'"); + WriteVarUInt32(Out_, it->Value->Index); + } + + // (5.2) labels + WriteLabels(metric.Labels, MetricName_); + + // (5.3) values + switch (metric.TimeSeries.Size()) { + case 0: + break; + case 1: { + const auto& point = metric.TimeSeries[0]; + if (point.GetTime() != TInstant::Zero()) { + WriteTime(point.GetTime()); + } + EMetricValueType valueType = metric.TimeSeries.GetValueType(); + WriteValue(metric.MetricType, valueType, point.GetValue()); + break; + } + default: + WriteVarUInt32(Out_, static_cast<ui32>(metric.TimeSeries.Size())); + const TMetricTimeSeries& ts = metric.TimeSeries; + EMetricType metricType = metric.MetricType; + ts.ForEach([this, metricType](TInstant time, EMetricValueType valueType, TMetricValue value) { + // workaround for GCC bug + // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=61636 + this->WriteTime(time); + this->WriteValue(metricType, valueType, value); + }); + break; + } + } + } + + // store metric type and values type in one byte + ui8 PackTypes(const TMetric& metric) { + EValueType valueType; + if (metric.TimeSeries.Empty()) { + valueType = EValueType::NONE; + } else if (metric.TimeSeries.Size() == 1) { + TInstant time = metric.TimeSeries[0].GetTime(); + valueType = (time == TInstant::Zero()) + ? EValueType::ONE_WITHOUT_TS + : EValueType::ONE_WITH_TS; + } else { + valueType = EValueType::MANY_WITH_TS; + } + return (static_cast<ui8>(metric.MetricType) << 2) | static_cast<ui8>(valueType); + } + + void WriteLabels(const TPooledLabels& labels, const TPooledStr* skipKey) { + WriteVarUInt32(Out_, static_cast<ui32>(skipKey ? labels.size() - 1 : labels.size())); + for (auto&& label : labels) { + if (label.Key == skipKey) { + continue; + } + WriteVarUInt32(Out_, label.Key->Index); + WriteVarUInt32(Out_, label.Value->Index); + } + } + + void WriteValue(EMetricType metricType, EMetricValueType valueType, TMetricValue value) { + switch (metricType) { + case EMetricType::GAUGE: + WriteFixed(value.AsDouble(valueType)); + break; + + case EMetricType::IGAUGE: + WriteFixed(value.AsInt64(valueType)); + break; + + case EMetricType::COUNTER: + case EMetricType::RATE: + WriteFixed(value.AsUint64(valueType)); + break; + + case EMetricType::HIST: + case EMetricType::HIST_RATE: + WriteHistogram(*value.AsHistogram()); + break; + + case EMetricType::DSUMMARY: + WriteSummaryDouble(*value.AsSummaryDouble()); + break; + + case EMetricType::LOGHIST: + WriteLogHistogram(*value.AsLogHistogram()); + break; + + default: + ythrow yexception() << "unsupported metric type: " << metricType; + } + } + + void WriteTime(TInstant instant) { + switch (TimePrecision_) { + case ETimePrecision::SECONDS: { + ui32 time = static_cast<ui32>(instant.Seconds()); + Out_->Write(&time, sizeof(time)); + break; + } + case ETimePrecision::MILLIS: { + ui64 time = static_cast<ui64>(instant.MilliSeconds()); + Out_->Write(&time, sizeof(time)); + } + } + } + + template <typename T> + void WriteFixed(T value) { + Out_->Write(&value, sizeof(value)); + } + + void WriteHistogram(const IHistogramSnapshot& histogram) { + ui32 count = histogram.Count(); + WriteVarUInt32(Out_, count); + + for (ui32 i = 0; i < count; i++) { + double bound = histogram.UpperBound(i); + Out_->Write(&bound, sizeof(bound)); + } + for (ui32 i = 0; i < count; i++) { + ui64 value = histogram.Value(i); + Out_->Write(&value, sizeof(value)); + } + } + + void WriteLogHistogram(const TLogHistogramSnapshot& logHist) { + WriteFixed(logHist.Base()); + WriteFixed(logHist.ZerosCount()); + WriteVarUInt32(Out_, static_cast<ui32>(logHist.StartPower())); + WriteVarUInt32(Out_, logHist.Count()); + for (ui32 i = 0; i < logHist.Count(); ++i) { + WriteFixed(logHist.Bucket(i)); + } + } + + void WriteSummaryDouble(const ISummaryDoubleSnapshot& summary) { + WriteFixed(summary.GetCount()); + WriteFixed(summary.GetSum()); + WriteFixed(summary.GetMin()); + WriteFixed(summary.GetMax()); + WriteFixed(summary.GetLast()); + } + + private: + IOutputStream* Out_; + ETimePrecision TimePrecision_; + ECompression Compression_; + ESpackV1Version Version_; + const TPooledStr* MetricName_; + bool Closed_ = false; + }; + + } + + IMetricEncoderPtr EncoderSpackV1( + IOutputStream* out, + ETimePrecision timePrecision, + ECompression compression, + EMetricsMergingMode mergingMode + ) { + return MakeHolder<TEncoderSpackV1>(out, timePrecision, compression, mergingMode, SV1_01, ""); + } + + IMetricEncoderPtr EncoderSpackV12( + IOutputStream* out, + ETimePrecision timePrecision, + ECompression compression, + EMetricsMergingMode mergingMode, + TStringBuf metricNameLabel + ) { + Y_ENSURE(!metricNameLabel.Empty(), "metricNameLabel can't be empty"); + return MakeHolder<TEncoderSpackV1>(out, timePrecision, compression, mergingMode, SV1_02, metricNameLabel); + } +} diff --git a/library/cpp/monlib/encode/spack/spack_v1_ut.cpp b/library/cpp/monlib/encode/spack/spack_v1_ut.cpp new file mode 100644 index 0000000000..fe778eb7e0 --- /dev/null +++ b/library/cpp/monlib/encode/spack/spack_v1_ut.cpp @@ -0,0 +1,845 @@ +#include "spack_v1.h" + +#include <library/cpp/monlib/encode/protobuf/protobuf.h> +#include <library/cpp/monlib/metrics/labels.h> +#include <library/cpp/monlib/metrics/metric.h> + +#include <library/cpp/testing/unittest/registar.h> + +#include <util/generic/buffer.h> +#include <util/stream/buffer.h> +#include <util/string/hex.h> + +#include <utility> + +using namespace NMonitoring; + +#define UNIT_ASSERT_BINARY_EQUALS(a, b) \ + do { \ + auto size = Y_ARRAY_SIZE(b); \ + if (Y_UNLIKELY(::memcmp(a, b, size) != 0)) { \ + auto as = HexEncode(a, size); \ + auto bs = HexEncode(b, size); \ + UNIT_FAIL_IMPL("equal assertion failed " #a " == " #b, \ + "\n actual: " << as << "\nexpected: " << bs); \ + } \ + } while (0) + +void AssertLabelEqual(const NProto::TLabel& l, TStringBuf name, TStringBuf value) { + UNIT_ASSERT_STRINGS_EQUAL(l.GetName(), name); + UNIT_ASSERT_STRINGS_EQUAL(l.GetValue(), value); +} + +void AssertPointEqual(const NProto::TPoint& p, TInstant time, double value) { + UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds()); + UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kFloat64); + UNIT_ASSERT_DOUBLES_EQUAL(p.GetFloat64(), value, std::numeric_limits<double>::epsilon()); +} + +void AssertPointEqual(const NProto::TPoint& p, TInstant time, ui64 value) { + UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds()); + UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kUint64); + UNIT_ASSERT_VALUES_EQUAL(p.GetUint64(), value); +} + +void AssertPointEqual(const NProto::TPoint& p, TInstant time, i64 value) { + UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds()); + UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kInt64); + UNIT_ASSERT_VALUES_EQUAL(p.GetInt64(), value); +} + +Y_UNIT_TEST_SUITE(TSpackTest) { + ui8 expectedHeader_v1_0[] = { + 0x53, 0x50, // magic "SP" (fixed ui16) + // minor, major + 0x00, 0x01, // version (fixed ui16) + 0x18, 0x00, // header size (fixed ui16) + 0x00, // time precision (fixed ui8) + 0x00, // compression algorithm (fixed ui8) + 0x0d, 0x00, 0x00, 0x00, // label names size (fixed ui32) + 0x40, 0x00, 0x00, 0x00, // labels values size (fixed ui32) + 0x08, 0x00, 0x00, 0x00, // metric count (fixed ui32) + 0x08, 0x00, 0x00, 0x00, // points count (fixed ui32) + }; + + ui8 expectedHeader[] = { + 0x53, 0x50, // magic "SP" (fixed ui16) + // minor, major + 0x01, 0x01, // version (fixed ui16) + 0x18, 0x00, // header size (fixed ui16) + 0x00, // time precision (fixed ui8) + 0x00, // compression algorithm (fixed ui8) + 0x0d, 0x00, 0x00, 0x00, // label names size (fixed ui32) + 0x40, 0x00, 0x00, 0x00, // labels values size (fixed ui32) + 0x08, 0x00, 0x00, 0x00, // metric count (fixed ui32) + 0x08, 0x00, 0x00, 0x00, // points count (fixed ui32) + }; + + ui8 expectedStringPools[] = { + 0x6e, 0x61, 0x6d, 0x65, 0x00, // "name\0" + 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x00, // "project\0" + 0x73, 0x6f, 0x6c, 0x6f, 0x6d, 0x6f, 0x6e, 0x00, // "solomon\0" + 0x71, 0x31, 0x00, // "q1\0" + 0x71, 0x32, 0x00, // "q2\0" + 0x71, 0x33, 0x00, // "q3\0" + 0x61, 0x6e, 0x73, 0x77, 0x65, 0x72, 0x00, // "answer\0" + 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, // "responseTimeMillis\0" + 0x54, 0x69, 0x6d, 0x65, 0x4d, 0x69, 0x6c, 0x6c, + 0x69, 0x73, 0x00, + 0x62, 0x79, 0x74, 0x65, 0x73, 0x00, // "bytes\0" + 0x74, 0x65, 0x6D, 0x70, 0x65, 0x72, 0x61, 0x74, // "temperature\0" + 0x75, 0x72, 0x65, 0x00, + 0x6d, 0x73, 0x00, // "ms\0" + }; + + ui8 expectedCommonTime[] = { + 0x00, 0x2f, 0x68, 0x59, // common time in seconds (fixed ui32) + }; + + ui8 expectedCommonLabels[] = { + 0x01, // common labels count (varint) + 0x01, // label name index (varint) + 0x00, // label value index (varint) + }; + + ui8 expectedMetric1[] = { + 0x0C, // types (RATE | NONE) (fixed ui8) + 0x00, // flags (fixed ui8) + 0x01, // metric labels count (varint) + 0x00, // label name index (varint) + 0x01, // label value index (varint) + }; + + ui8 expectedMetric2[] = { + 0x09, // types (COUNTER | ONE_WITHOUT_TS) (fixed ui8) + 0x00, // flags (fixed ui8) + 0x01, // metric labels count (varint) + 0x00, // label name index (varint) + 0x02, // label value index (varint) + 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // value (fixed ui64) + }; + + ui8 expectedMetric3[] = { + 0x0a, // types (COUNTER | ONE_WITH_TS) (fixed ui8) + 0x00, // flags (fixed ui8) + 0x01, // metric labels count (varint) + 0x00, // label name index (varint) + 0x03, // label value index (varint) + 0x0b, 0x63, 0xfe, 0x59, // time in seconds (fixed ui32) + 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // value (fixed ui64) + }; + + ui8 expectedMetric4[] = { + 0x07, // types (GAUGE | MANY_WITH_TS) (fixed ui8) + 0x00, // flags (fixed ui8) + 0x01, // metric labels count (varint) + 0x00, // label name index (varint) + 0x04, // label value index (varint) + 0x02, // points count (varint) + 0x0b, 0x63, 0xfe, 0x59, // time in seconds (fixed ui32) + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x45, 0x40, // value (double IEEE754) + 0x1a, 0x63, 0xfe, 0x59, // time in seconds (fixed ui32) + 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x4d, 0x40 // value (double IEEE754) + }; + + ui8 expectedMetric5_v1_0[] = { + 0x16, // types (HIST | ONE_WITH_TS) (fixed ui8) + 0x00, // flags (fixed ui8) + 0x01, // metric labels count (varint) + 0x00, // label name index (varint) + 0x05, // label value index (varint) + 0x0b, 0x63, 0xfe, 0x59, // time in seconds (fixed ui32) + 0x06, // histogram buckets count (varint) + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // histogram bucket bounds (array of fixed ui64) + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // histogram bucket values + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x53, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + }; + + ui8 expectedMetric5[] = { + 0x16, // types (HIST | ONE_WITH_TS) (fixed ui8) + 0x00, // flags (fixed ui8) + 0x01, // metric labels count (varint) + 0x00, // label name index (varint) + 0x05, // label value index (varint) + 0x0b, 0x63, 0xfe, 0x59, // time in seconds (fixed ui32) + 0x06, // histogram buckets count (varint) + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x3f, // histogram bucket bounds (array of doubles) + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x40, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xef, 0x7f, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // histogram bucket values + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x53, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + }; + + ui8 expectedMetric6[] = { + 0x12, // types (IGAUGE | ONE_WITH_TS) (fixed ui8) + 0x00, // flags (fixed ui8) + 0x01, // metric labels count (varint) + 0x00, // label name index (varint) + 0x06, // label value index (varint) + 0x0b, 0x63, 0xfe, 0x59, // time in seconds (fixed ui32) + 0x39, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // value (fixed i64) + }; + + ui8 expectedMetric7[] = { + 0x1e, // types (DSUMMARY | ONE_WITH_TS) (fixed ui8) + 0x00, // flags (fixed ui8) + 0x01, // metric labels count (varint) + 0x00, // label name index (varint) + 0x07, // label value index (varint) + 0x0b, 0x63, 0xfe, 0x59, // time in seconds (fixed ui32) + 0x1e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // count (fixed ui64) + 0x33, 0x33, 0x33, 0x33, 0x33, 0x33, 0x24, 0x40, // sum (fixed double) + 0xcd, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xdc, 0xbf, // min (fixed double) + 0x64, 0x3b, 0xdf, 0x4f, 0x8d, 0x97, 0xde, 0x3f, // max (fixed double) + 0x33, 0x33, 0x33, 0x33, 0x33, 0x33, 0xd3, 0x3f, // last (fixed double) + }; + + ui8 expectedMetric8[] = { + 0x26, // types (LOGHIST | ONE_WITH_TS) (fixed ui8) + 0x00, // flags (fixed ui8) + 0x01, // metric labels count (variant) + 0x00, // label name index (variant) + 0x08, // label value index (variant) + 0x0b, 0x63, 0xfe, 0x59, // time in seconds (fixed ui32) + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF8, 0x3F, // base (fixed double) + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // zerosCount (fixed ui64) + 0x00, // startPower (variant) + 0x04, // buckets count (variant) + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE0, 0x3F, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD0, 0x3F, // bucket values + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD0, 0x3F, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE0, 0x3F, + }; + + const size_t expectedSize = + Y_ARRAY_SIZE(expectedHeader) + + Y_ARRAY_SIZE(expectedStringPools) + + Y_ARRAY_SIZE(expectedCommonTime) + + Y_ARRAY_SIZE(expectedCommonLabels) + + Y_ARRAY_SIZE(expectedMetric1) + + Y_ARRAY_SIZE(expectedMetric2) + + Y_ARRAY_SIZE(expectedMetric3) + + Y_ARRAY_SIZE(expectedMetric4) + + Y_ARRAY_SIZE(expectedMetric5) + + Y_ARRAY_SIZE(expectedMetric6) + + Y_ARRAY_SIZE(expectedMetric7) + + Y_ARRAY_SIZE(expectedMetric8); + + const TInstant now = TInstant::ParseIso8601Deprecated("2017-11-05T01:02:03Z"); + + // {1: 1, 2: 1, 4: 2, 8: 4, 16: 8, inf: 83} + IHistogramSnapshotPtr TestHistogram() { + auto h = ExponentialHistogram(6, 2); + for (i64 i = 1; i < 100; i++) { + h->Collect(i); + } + return h->Snapshot(); + } + + TLogHistogramSnapshotPtr TestLogHistogram() { + TVector buckets{0.5, 0.25, 0.25, 0.5}; + return MakeIntrusive<TLogHistogramSnapshot>(1.5, 1u, 0, std::move(buckets)); + } + + ISummaryDoubleSnapshotPtr TestSummaryDouble() { + return MakeIntrusive<TSummaryDoubleSnapshot>(10.1, -0.45, 0.478, 0.3, 30u); + } + + Y_UNIT_TEST(Encode) { + TBuffer buffer; + TBufferOutput out(buffer); + auto e = EncoderSpackV1( + &out, ETimePrecision::SECONDS, ECompression::IDENTITY); + + e->OnStreamBegin(); + { // common time + e->OnCommonTime(TInstant::Seconds(1500000000)); + } + { // common labels + e->OnLabelsBegin(); + e->OnLabel("project", "solomon"); + e->OnLabelsEnd(); + } + { // metric #1 + e->OnMetricBegin(EMetricType::RATE); + { + e->OnLabelsBegin(); + e->OnLabel("name", "q1"); + e->OnLabelsEnd(); + } + e->OnMetricEnd(); + } + { // metric #2 + e->OnMetricBegin(EMetricType::COUNTER); + { + e->OnLabelsBegin(); + e->OnLabel("name", "q2"); + e->OnLabelsEnd(); + } + // Only the last value will be encoded + e->OnUint64(TInstant::Zero(), 10); + e->OnUint64(TInstant::Zero(), 13); + e->OnUint64(TInstant::Zero(), 17); + e->OnMetricEnd(); + } + { // metric #3 + e->OnMetricBegin(EMetricType::COUNTER); + { + e->OnLabelsBegin(); + e->OnLabel("name", "q3"); + e->OnLabelsEnd(); + } + e->OnUint64(now, 10); + e->OnUint64(now, 13); + e->OnUint64(now, 17); + e->OnMetricEnd(); + } + { // metric #4 + e->OnMetricBegin(EMetricType::GAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("name", "answer"); + e->OnLabelsEnd(); + } + e->OnDouble(now, 42); + e->OnDouble(now + TDuration::Seconds(15), 59); + e->OnMetricEnd(); + } + { // metric #5 + e->OnMetricBegin(EMetricType::HIST); + { + e->OnLabelsBegin(); + e->OnLabel("name", "responseTimeMillis"); + e->OnLabelsEnd(); + } + + auto histogram = TestHistogram(); + e->OnHistogram(now, histogram); + e->OnMetricEnd(); + } + { // metric #6 + e->OnMetricBegin(EMetricType::IGAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("name", "bytes"); + e->OnLabelsEnd(); + } + e->OnInt64(now, 1337); + e->OnMetricEnd(); + } + { // metric 7 + e->OnMetricBegin(EMetricType::DSUMMARY); + { + e->OnLabelsBegin(); + e->OnLabel("name", "temperature"); + e->OnLabelsEnd(); + } + e->OnSummaryDouble(now, TestSummaryDouble()); + e->OnMetricEnd(); + } + { // metric 8 + e->OnMetricBegin(EMetricType::LOGHIST); + { + e->OnLabelsBegin(); + e->OnLabel("name", "ms"); + e->OnLabelsEnd(); + } + e->OnLogHistogram(now, TestLogHistogram()); + e->OnMetricEnd(); + } + e->OnStreamEnd(); + e->Close(); + + // Cout << "encoded: " << HexEncode(buffer.Data(), buffer.Size()) << Endl; + // Cout << "size: " << buffer.Size() << Endl; + + UNIT_ASSERT_VALUES_EQUAL(buffer.Size(), expectedSize); + + ui8* p = reinterpret_cast<ui8*>(buffer.Data()); + UNIT_ASSERT_BINARY_EQUALS(p, expectedHeader); + p += Y_ARRAY_SIZE(expectedHeader); + + UNIT_ASSERT_BINARY_EQUALS(p, expectedStringPools); + p += Y_ARRAY_SIZE(expectedStringPools); + + UNIT_ASSERT_BINARY_EQUALS(p, expectedCommonTime); + p += Y_ARRAY_SIZE(expectedCommonTime); + + UNIT_ASSERT_BINARY_EQUALS(p, expectedCommonLabels); + p += Y_ARRAY_SIZE(expectedCommonLabels); + + UNIT_ASSERT_BINARY_EQUALS(p, expectedMetric1); + p += Y_ARRAY_SIZE(expectedMetric1); + + UNIT_ASSERT_BINARY_EQUALS(p, expectedMetric2); + p += Y_ARRAY_SIZE(expectedMetric2); + + UNIT_ASSERT_BINARY_EQUALS(p, expectedMetric3); + p += Y_ARRAY_SIZE(expectedMetric3); + + UNIT_ASSERT_BINARY_EQUALS(p, expectedMetric4); + p += Y_ARRAY_SIZE(expectedMetric4); + + UNIT_ASSERT_BINARY_EQUALS(p, expectedMetric5); + p += Y_ARRAY_SIZE(expectedMetric5); + + UNIT_ASSERT_BINARY_EQUALS(p, expectedMetric6); + p += Y_ARRAY_SIZE(expectedMetric6); + + UNIT_ASSERT_BINARY_EQUALS(p, expectedMetric7); + p += Y_ARRAY_SIZE(expectedMetric7); + + UNIT_ASSERT_BINARY_EQUALS(p, expectedMetric8); + p += Y_ARRAY_SIZE(expectedMetric8); + } + + NProto::TMultiSamplesList GetMergingMetricSamples(EMetricsMergingMode mergingMode) { + TBuffer buffer; + TBufferOutput out(buffer); + + auto e = EncoderSpackV1( + &out, + ETimePrecision::SECONDS, + ECompression::IDENTITY, + mergingMode + ); + + e->OnStreamBegin(); + for (size_t i = 0; i != 3; ++i) { + e->OnMetricBegin(EMetricType::COUNTER); + { + e->OnLabelsBegin(); + e->OnLabel("name", "my_counter"); + e->OnLabelsEnd(); + } + e->OnUint64(TInstant::Zero() + TDuration::Seconds(i), i + 1); + e->OnMetricEnd(); + } + e->OnStreamEnd(); + e->Close(); + + NProto::TMultiSamplesList samples; + IMetricEncoderPtr eProto = EncoderProtobuf(&samples); + TBufferInput in(buffer); + DecodeSpackV1(&in, eProto.Get()); + + return samples; + } + + Y_UNIT_TEST(SpackEncoderMergesMetrics) { + { + NProto::TMultiSamplesList samples = GetMergingMetricSamples(EMetricsMergingMode::DEFAULT); + + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 3); + UNIT_ASSERT_EQUAL(samples.GetSamples(0).GetPoints(0).GetUint64(), 1); + UNIT_ASSERT_EQUAL(samples.GetSamples(1).GetPoints(0).GetUint64(), 2); + UNIT_ASSERT_EQUAL(samples.GetSamples(2).GetPoints(0).GetUint64(), 3); + } + + { + NProto::TMultiSamplesList samples = GetMergingMetricSamples(EMetricsMergingMode::MERGE_METRICS); + + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 1); + + auto sample0 = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(sample0.GetPoints(0).GetUint64(), 1); + UNIT_ASSERT_EQUAL(sample0.GetPoints(1).GetUint64(), 2); + UNIT_ASSERT_EQUAL(sample0.GetPoints(2).GetUint64(), 3); + } + } + + void DecodeDataToSamples(NProto::TMultiSamplesList & samples, ui16 version) { + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + TBuffer data(expectedSize); + if (SV1_00 == version) { // v1.0 + data.Append(reinterpret_cast<char*>(expectedHeader_v1_0), Y_ARRAY_SIZE(expectedHeader_v1_0)); + } else { + data.Append(reinterpret_cast<char*>(expectedHeader), Y_ARRAY_SIZE(expectedHeader)); + } + data.Append(reinterpret_cast<char*>(expectedStringPools), Y_ARRAY_SIZE(expectedStringPools)); + data.Append(reinterpret_cast<char*>(expectedCommonTime), Y_ARRAY_SIZE(expectedCommonTime)); + data.Append(reinterpret_cast<char*>(expectedCommonLabels), Y_ARRAY_SIZE(expectedCommonLabels)); + data.Append(reinterpret_cast<char*>(expectedMetric1), Y_ARRAY_SIZE(expectedMetric1)); + data.Append(reinterpret_cast<char*>(expectedMetric2), Y_ARRAY_SIZE(expectedMetric2)); + data.Append(reinterpret_cast<char*>(expectedMetric3), Y_ARRAY_SIZE(expectedMetric3)); + data.Append(reinterpret_cast<char*>(expectedMetric4), Y_ARRAY_SIZE(expectedMetric4)); + if (SV1_00 == version) { // v1.0 + data.Append(reinterpret_cast<char*>(expectedMetric5_v1_0), Y_ARRAY_SIZE(expectedMetric5_v1_0)); + } else { + data.Append(reinterpret_cast<char*>(expectedMetric5), Y_ARRAY_SIZE(expectedMetric5)); + } + data.Append(reinterpret_cast<char*>(expectedMetric6), Y_ARRAY_SIZE(expectedMetric6)); + data.Append(reinterpret_cast<char*>(expectedMetric7), Y_ARRAY_SIZE(expectedMetric7)); + data.Append(reinterpret_cast<char*>(expectedMetric8), Y_ARRAY_SIZE(expectedMetric8)); + TBufferInput in(data); + DecodeSpackV1(&in, e.Get()); + } + + void DecodeDataToSamples(NProto::TMultiSamplesList & samples) { + TSpackHeader header; + header.Version = SV1_01; + DecodeDataToSamples(samples, header.Version); + } + + Y_UNIT_TEST(Decode) { + NProto::TMultiSamplesList samples; + DecodeDataToSamples(samples); + + UNIT_ASSERT_VALUES_EQUAL( + TInstant::MilliSeconds(samples.GetCommonTime()), + TInstant::Seconds(1500000000)); + + UNIT_ASSERT_VALUES_EQUAL(samples.CommonLabelsSize(), 1); + AssertLabelEqual(samples.GetCommonLabels(0), "project", "solomon"); + + UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 8); + { + const NProto::TMultiSample& s = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::RATE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "name", "q1"); + } + { + const NProto::TMultiSample& s = samples.GetSamples(1); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::COUNTER); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "name", "q2"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + AssertPointEqual(s.GetPoints(0), TInstant::Zero(), ui64(17)); + } + { + const NProto::TMultiSample& s = samples.GetSamples(2); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::COUNTER); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "name", "q3"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + AssertPointEqual(s.GetPoints(0), now, ui64(17)); + } + { + const NProto::TMultiSample& s = samples.GetSamples(3); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "name", "answer"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 2); + AssertPointEqual(s.GetPoints(0), now, double(42)); + AssertPointEqual(s.GetPoints(1), now + TDuration::Seconds(15), double(59)); + } + { + const NProto::TMultiSample& s = samples.GetSamples(4); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::HISTOGRAM); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "name", "responseTimeMillis"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + + const NProto::TPoint& p = s.GetPoints(0); + UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), now.MilliSeconds()); + UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kHistogram); + + auto histogram = TestHistogram(); + + const NProto::THistogram& pointHistogram = p.GetHistogram(); + UNIT_ASSERT_VALUES_EQUAL(pointHistogram.BoundsSize(), histogram->Count()); + UNIT_ASSERT_VALUES_EQUAL(pointHistogram.ValuesSize(), histogram->Count()); + + for (size_t i = 0; i < pointHistogram.BoundsSize(); i++) { + UNIT_ASSERT_DOUBLES_EQUAL(pointHistogram.GetBounds(i), histogram->UpperBound(i), Min<double>()); + UNIT_ASSERT_VALUES_EQUAL(pointHistogram.GetValues(i), histogram->Value(i)); + } + } + { + const NProto::TMultiSample& s = samples.GetSamples(5); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::IGAUGE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "name", "bytes"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + AssertPointEqual(s.GetPoints(0), now, i64(1337)); + } + { + const NProto::TMultiSample& s = samples.GetSamples(6); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::DSUMMARY); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "name", "temperature"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + + const NProto::TPoint& p = s.GetPoints(0); + UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), now.MilliSeconds()); + UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kSummaryDouble); + + auto expected = TestSummaryDouble(); + + auto actual = p.GetSummaryDouble(); + + UNIT_ASSERT_VALUES_EQUAL(expected->GetSum(), actual.GetSum()); + UNIT_ASSERT_VALUES_EQUAL(expected->GetMin(), actual.GetMin()); + UNIT_ASSERT_VALUES_EQUAL(expected->GetMax(), actual.GetMax()); + UNIT_ASSERT_VALUES_EQUAL(expected->GetLast(), actual.GetLast()); + UNIT_ASSERT_VALUES_EQUAL(expected->GetCount(), actual.GetCount()); + } + { + const NProto::TMultiSample& s = samples.GetSamples(7); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::LOGHISTOGRAM); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "name", "ms"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + + const NProto::TPoint& p = s.GetPoints(0); + UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), now.MilliSeconds()); + UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kLogHistogram); + + auto expected = TestLogHistogram(); + auto actual = p.GetLogHistogram(); + + UNIT_ASSERT_VALUES_EQUAL(expected->ZerosCount(), actual.GetZerosCount()); + UNIT_ASSERT_VALUES_EQUAL(expected->Base(), actual.GetBase()); + UNIT_ASSERT_VALUES_EQUAL(expected->StartPower(), actual.GetStartPower()); + UNIT_ASSERT_VALUES_EQUAL(expected->Count(), actual.BucketsSize()); + for (size_t i = 0; i < expected->Count(); ++i) { + UNIT_ASSERT_VALUES_EQUAL(expected->Bucket(i), actual.GetBuckets(i)); + } + } + } + + void TestCompression(ECompression alg) { + TBuffer buffer; + { + TBufferOutput out(buffer); + auto e = EncoderSpackV1(&out, ETimePrecision::MILLIS, alg); + e->OnStreamBegin(); + { + e->OnMetricBegin(EMetricType::GAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("name", "answer"); + e->OnLabelsEnd(); + } + e->OnDouble(now, 42); + e->OnMetricEnd(); + } + e->OnStreamEnd(); + e->Close(); + } + + auto* header = reinterpret_cast<const TSpackHeader*>(buffer.Data()); + UNIT_ASSERT_EQUAL(DecodeCompression(header->Compression), alg); + + NProto::TMultiSamplesList samples; + { + IMetricEncoderPtr e = EncoderProtobuf(&samples); + TBufferInput in(buffer); + DecodeSpackV1(&in, e.Get()); + } + + UNIT_ASSERT_VALUES_EQUAL( + TInstant::MilliSeconds(samples.GetCommonTime()), + TInstant::Zero()); + + UNIT_ASSERT_VALUES_EQUAL(samples.CommonLabelsSize(), 0); + + UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 1); + { + const NProto::TMultiSample& s = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "name", "answer"); + AssertPointEqual(s.GetPoints(0), now, 42.0); + } + } + + Y_UNIT_TEST(CompressionIdentity) { + TestCompression(ECompression::IDENTITY); + } + + Y_UNIT_TEST(CompressionZlib) { + TestCompression(ECompression::ZLIB); + } + + Y_UNIT_TEST(CompressionZstd) { + TestCompression(ECompression::ZSTD); + } + + Y_UNIT_TEST(CompressionLz4) { + TestCompression(ECompression::LZ4); + } + + Y_UNIT_TEST(Decode_v1_0_histograms) { + // Check that histogram bounds decoded from different versions are the same + NProto::TMultiSamplesList samples, samples_v1_0; + DecodeDataToSamples(samples); + DecodeDataToSamples(samples_v1_0, /*version = */ SV1_00); + + const NProto::THistogram& pointHistogram = samples.GetSamples(4).GetPoints(0).GetHistogram(); + const NProto::THistogram& pointHistogram_v1_0 = samples_v1_0.GetSamples(4).GetPoints(0).GetHistogram(); + + for (size_t i = 0; i < pointHistogram.BoundsSize(); i++) { + UNIT_ASSERT_DOUBLES_EQUAL(pointHistogram.GetBounds(i), pointHistogram_v1_0.GetBounds(i), Min<double>()); + } + } + + Y_UNIT_TEST(SimpleV12) { + ui8 expectedSerialized[] = { + // header + 0x53, 0x50, // magic "SP" (fixed ui16) + // minor, major + 0x02, 0x01, // version (fixed ui16) + 0x18, 0x00, // header size (fixed ui16) + 0x00, // time precision (fixed ui8) + 0x00, // compression algorithm (fixed ui8) + 0x0A, 0x00, 0x00, 0x00, // label names size (fixed ui32) + 0x14, 0x00, 0x00, 0x00, // labels values size (fixed ui32) + 0x01, 0x00, 0x00, 0x00, // metric count (fixed ui32) + 0x01, 0x00, 0x00, 0x00, // points count (fixed ui32) + + // string pools + 0x73, 0x00, // "s\0" + 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x00, // "project\0" + 0x73, 0x6f, 0x6c, 0x6f, 0x6d, 0x6f, 0x6e, 0x00, // "solomon\0" + 0x74, 0x65, 0x6D, 0x70, 0x65, 0x72, 0x61, 0x74, // temperature + 0x75, 0x72, 0x65, 0x00, + + // common time + 0x00, 0x2f, 0x68, 0x59, // common time in seconds (fixed ui32) + + // common labels + 0x00, // common labels count (varint) + + // metric + 0x09, // types (COUNTER | ONE_WITHOUT_TS) (fixed ui8) + 0x00, // flags (fixed ui8) + 0x01, // name index (varint) + 0x01, // metric labels count (varint) + 0x01, // 'project' label name index (varint) + 0x00, // 'project' label value index (varint) + 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // value (fixed ui64) + }; + + // encode + { + TBuffer actualSerialized; + { + TBufferOutput out(actualSerialized); + auto e = EncoderSpackV12( + &out, + ETimePrecision::SECONDS, + ECompression::IDENTITY, + EMetricsMergingMode::DEFAULT, + "s"); + + e->OnStreamBegin(); + e->OnCommonTime(TInstant::Seconds(1500000000)); + + { + e->OnMetricBegin(EMetricType::COUNTER); + { + e->OnLabelsBegin(); + e->OnLabel("project", "solomon"); + e->OnLabel("s", "temperature"); + e->OnLabelsEnd(); + } + // Only the last value will be encoded + e->OnUint64(TInstant::Zero(), 10); + e->OnUint64(TInstant::Zero(), 13); + e->OnUint64(TInstant::Zero(), 17); + e->OnMetricEnd(); + } + + e->OnStreamEnd(); + e->Close(); + } + + UNIT_ASSERT_VALUES_EQUAL(actualSerialized.Size(), Y_ARRAY_SIZE(expectedSerialized)); + UNIT_ASSERT_BINARY_EQUALS(actualSerialized.Data(), expectedSerialized); + } + + // decode + { + NProto::TMultiSamplesList samples; + { + auto input = TMemoryInput(expectedSerialized, Y_ARRAY_SIZE(expectedSerialized)); + auto encoder = EncoderProtobuf(&samples); + DecodeSpackV1(&input, encoder.Get(), "s"); + } + + UNIT_ASSERT_VALUES_EQUAL(TInstant::MilliSeconds(samples.GetCommonTime()), + TInstant::Seconds(1500000000)); + + UNIT_ASSERT_VALUES_EQUAL(samples.CommonLabelsSize(), 0); + + UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 1); + { + const auto& s = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::COUNTER); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2); + AssertLabelEqual(s.GetLabels(0), "s", "temperature"); + AssertLabelEqual(s.GetLabels(1), "project", "solomon"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + AssertPointEqual(s.GetPoints(0), TInstant::Zero(), ui64(17)); + } + } + } + + Y_UNIT_TEST(V12MissingNameForOneMetric) { + TBuffer b; + TBufferOutput out(b); + auto e = EncoderSpackV12( + &out, + ETimePrecision::SECONDS, + ECompression::IDENTITY, + EMetricsMergingMode::DEFAULT, + "s"); + + UNIT_ASSERT_EXCEPTION_CONTAINS( + [&]() { + e->OnStreamBegin(); + { + e->OnMetricBegin(EMetricType::COUNTER); + { + e->OnLabelsBegin(); + e->OnLabel("s", "s1"); + e->OnLabelsEnd(); + } + e->OnUint64(TInstant::Zero(), 1); + e->OnMetricEnd(); + + e->OnMetricBegin(EMetricType::COUNTER); + { + e->OnLabelsBegin(); + e->OnLabel("project", "solomon"); + e->OnLabel("m", "v"); + e->OnLabelsEnd(); + } + e->OnUint64(TInstant::Zero(), 2); + e->OnMetricEnd(); + } + + e->OnStreamEnd(); + e->Close(); + }(), + yexception, + "metric name label 's' not found, all metric labels '{m=v, project=solomon}'"); + } +} diff --git a/library/cpp/monlib/encode/spack/ut/ya.make b/library/cpp/monlib/encode/spack/ut/ya.make new file mode 100644 index 0000000000..980bf54667 --- /dev/null +++ b/library/cpp/monlib/encode/spack/ut/ya.make @@ -0,0 +1,16 @@ +UNITTEST_FOR(library/cpp/monlib/encode/spack) + +OWNER( + g:solomon + jamel +) + +SRCS( + spack_v1_ut.cpp +) + +PEERDIR( + library/cpp/monlib/encode/protobuf +) + +END() diff --git a/library/cpp/monlib/encode/spack/varint.cpp b/library/cpp/monlib/encode/spack/varint.cpp new file mode 100644 index 0000000000..051cf17380 --- /dev/null +++ b/library/cpp/monlib/encode/spack/varint.cpp @@ -0,0 +1,79 @@ +#include "varint.h" + +#include <util/generic/yexception.h> +#include <util/stream/input.h> +#include <util/stream/output.h> + +namespace NMonitoring { + ui32 WriteVarUInt32(IOutputStream* output, ui32 value) { + bool stop = false; + ui32 written = 0; + while (!stop) { + ui8 byte = static_cast<ui8>(value | 0x80); + value >>= 7; + if (value == 0) { + stop = true; + byte &= 0x7F; + } + output->Write(byte); + written++; + } + return written; + } + + ui32 ReadVarUInt32(IInputStream* input) { + ui32 value = 0; + switch (TryReadVarUInt32(input, &value)) { + case EReadResult::OK: + return value; + case EReadResult::ERR_OVERFLOW: + ythrow yexception() << "the data is too long to read ui32"; + case EReadResult::ERR_UNEXPECTED_EOF: + ythrow yexception() << "the data unexpectedly ended"; + default: + ythrow yexception() << "unknown error while reading varint"; + } + } + + size_t ReadVarUInt32(const ui8* buf, size_t len, ui32* result) { + size_t count = 0; + ui32 value = 0; + + ui8 byte = 0; + do { + if (7 * count > 8 * sizeof(ui32)) { + ythrow yexception() << "the data is too long to read ui32"; + } + if (count == len) { + ythrow yexception() << "the data unexpectedly ended"; + } + byte = buf[count]; + value |= (static_cast<ui32>(byte & 0x7F)) << (7 * count); + ++count; + } while (byte & 0x80); + + *result = value; + return count; + } + +EReadResult TryReadVarUInt32(IInputStream* input, ui32* value) { + size_t count = 0; + ui32 result = 0; + + ui8 byte = 0; + do { + if (7 * count > 8 * sizeof(ui32)) { + return EReadResult::ERR_OVERFLOW; + } + if (input->Read(&byte, 1) != 1) { + return EReadResult::ERR_UNEXPECTED_EOF; + } + result |= (static_cast<ui32>(byte & 0x7F)) << (7 * count); + ++count; + } while (byte & 0x80); + + *value = result; + return EReadResult::OK; + } + +} diff --git a/library/cpp/monlib/encode/spack/varint.h b/library/cpp/monlib/encode/spack/varint.h new file mode 100644 index 0000000000..7ac522dd6c --- /dev/null +++ b/library/cpp/monlib/encode/spack/varint.h @@ -0,0 +1,23 @@ +#pragma once + +#include <util/system/types.h> + +class IInputStream; +class IOutputStream; + +namespace NMonitoring { + ui32 WriteVarUInt32(IOutputStream* output, ui32 value); + + ui32 ReadVarUInt32(IInputStream* input); + size_t ReadVarUInt32(const ui8* buf, size_t len, ui32* result); + + enum class EReadResult { + OK, + ERR_OVERFLOW, + ERR_UNEXPECTED_EOF, + }; + + [[nodiscard]] + EReadResult TryReadVarUInt32(IInputStream* input, ui32* value); + +} diff --git a/library/cpp/monlib/encode/spack/ya.make b/library/cpp/monlib/encode/spack/ya.make new file mode 100644 index 0000000000..78d3061291 --- /dev/null +++ b/library/cpp/monlib/encode/spack/ya.make @@ -0,0 +1,25 @@ +LIBRARY() + +OWNER( + g:solomon + jamel +) + +SRCS( + spack_v1_decoder.cpp + spack_v1_encoder.cpp + varint.cpp + compression.cpp +) + +PEERDIR( + library/cpp/monlib/encode/buffered + library/cpp/monlib/exception + + contrib/libs/lz4 + contrib/libs/xxhash + contrib/libs/zlib + contrib/libs/zstd +) + +END() diff --git a/library/cpp/monlib/encode/text/text.h b/library/cpp/monlib/encode/text/text.h new file mode 100644 index 0000000000..6b2be3937b --- /dev/null +++ b/library/cpp/monlib/encode/text/text.h @@ -0,0 +1,9 @@ +#pragma once + +#include <library/cpp/monlib/encode/encoder.h> + +class IOutputStream; + +namespace NMonitoring { + IMetricEncoderPtr EncoderText(IOutputStream* out, bool humanReadableTs = true); +} diff --git a/library/cpp/monlib/encode/text/text_encoder.cpp b/library/cpp/monlib/encode/text/text_encoder.cpp new file mode 100644 index 0000000000..10336261f0 --- /dev/null +++ b/library/cpp/monlib/encode/text/text_encoder.cpp @@ -0,0 +1,226 @@ +#include "text.h" + +#include <library/cpp/monlib/encode/encoder_state.h> +#include <library/cpp/monlib/metrics/labels.h> +#include <library/cpp/monlib/metrics/metric_value.h> + +#include <util/datetime/base.h> +#include <util/stream/format.h> + +namespace NMonitoring { + namespace { + class TEncoderText final: public IMetricEncoder { + public: + TEncoderText(IOutputStream* out, bool humanReadableTs) + : Out_(out) + , HumanReadableTs_(humanReadableTs) + { + } + + private: + void OnStreamBegin() override { + State_.Expect(TEncoderState::EState::ROOT); + } + + void OnStreamEnd() override { + State_.Expect(TEncoderState::EState::ROOT); + } + + void OnCommonTime(TInstant time) override { + State_.Expect(TEncoderState::EState::ROOT); + CommonTime_ = time; + if (time != TInstant::Zero()) { + Out_->Write(TStringBuf("common time: ")); + WriteTime(time); + Out_->Write('\n'); + } + } + + void OnMetricBegin(EMetricType type) override { + State_.Switch(TEncoderState::EState::ROOT, TEncoderState::EState::METRIC); + ClearLastMetricState(); + MetricType_ = type; + } + + void OnMetricEnd() override { + State_.Switch(TEncoderState::EState::METRIC, TEncoderState::EState::ROOT); + WriteMetric(); + } + + void OnLabelsBegin() override { + if (State_ == TEncoderState::EState::METRIC) { + State_ = TEncoderState::EState::METRIC_LABELS; + } else if (State_ == TEncoderState::EState::ROOT) { + State_ = TEncoderState::EState::COMMON_LABELS; + } else { + State_.ThrowInvalid("expected METRIC or ROOT"); + } + } + + void OnLabelsEnd() override { + if (State_ == TEncoderState::EState::METRIC_LABELS) { + State_ = TEncoderState::EState::METRIC; + } else if (State_ == TEncoderState::EState::COMMON_LABELS) { + State_ = TEncoderState::EState::ROOT; + Out_->Write(TStringBuf("common labels: ")); + WriteLabels(); + Out_->Write('\n'); + } else { + State_.ThrowInvalid("expected LABELS or COMMON_LABELS"); + } + } + + void OnLabel(TStringBuf name, TStringBuf value) override { + Labels_.Add(name, value); + } + + void OnDouble(TInstant time, double value) override { + State_.Expect(TEncoderState::EState::METRIC); + TimeSeries_.Add(time, value); + } + + void OnInt64(TInstant time, i64 value) override { + State_.Expect(TEncoderState::EState::METRIC); + TimeSeries_.Add(time, value); + } + + void OnUint64(TInstant time, ui64 value) override { + State_.Expect(TEncoderState::EState::METRIC); + TimeSeries_.Add(time, value); + } + + void OnHistogram(TInstant time, IHistogramSnapshotPtr snapshot) override { + State_.Expect(TEncoderState::EState::METRIC); + TimeSeries_.Add(time, snapshot.Get()); + } + + void OnSummaryDouble(TInstant time, ISummaryDoubleSnapshotPtr snapshot) override { + State_.Expect(TEncoderState::EState::METRIC); + TimeSeries_.Add(time, snapshot.Get()); + } + + void OnLogHistogram(TInstant ts, TLogHistogramSnapshotPtr snapshot) override { + State_.Expect(TEncoderState::EState::METRIC); + TimeSeries_.Add(ts, snapshot.Get()); + } + + void Close() override { + } + + void WriteTime(TInstant time) { + if (HumanReadableTs_) { + char buf[64]; + auto len = FormatDate8601(buf, sizeof(buf), time.TimeT()); + Out_->Write(buf, len); + } else { + (*Out_) << time.Seconds(); + } + } + + void WriteValue(EMetricValueType type, TMetricValue value) { + switch (type) { + case EMetricValueType::DOUBLE: + (*Out_) << value.AsDouble(); + break; + case EMetricValueType::INT64: + (*Out_) << value.AsInt64(); + break; + case EMetricValueType::UINT64: + (*Out_) << value.AsUint64(); + break; + case EMetricValueType::HISTOGRAM: + (*Out_) << *value.AsHistogram(); + break; + case EMetricValueType::SUMMARY: + (*Out_) << *value.AsSummaryDouble(); + break; + case EMetricValueType::LOGHISTOGRAM: + (*Out_) << *value.AsLogHistogram(); + break; + case EMetricValueType::UNKNOWN: + ythrow yexception() << "unknown metric value type"; + } + } + + void WriteLabels() { + auto& out = *Out_; + const auto size = Labels_.Size(); + size_t i = 0; + + out << '{'; + for (auto&& l : Labels_) { + out << l.Name() << TStringBuf("='") << l.Value() << '\''; + + ++i; + if (i < size) { + out << TStringBuf(", "); + } + }; + + out << '}'; + } + + void WriteMetric() { + // (1) type + TStringBuf typeStr = MetricTypeToStr(MetricType_); + (*Out_) << LeftPad(typeStr, MaxMetricTypeNameLength) << ' '; + + // (2) name and labels + auto name = Labels_.Extract(TStringBuf("sensor")); + if (name) { + if (name->Value().find(' ') != TString::npos) { + (*Out_) << '"' << name->Value() << '"'; + } else { + (*Out_) << name->Value(); + } + } + WriteLabels(); + + // (3) values + if (!TimeSeries_.Empty()) { + TimeSeries_.SortByTs(); + Out_->Write(TStringBuf(" [")); + for (size_t i = 0; i < TimeSeries_.Size(); i++) { + if (i > 0) { + Out_->Write(TStringBuf(", ")); + } + + const auto& point = TimeSeries_[i]; + if (point.GetTime() == CommonTime_ || point.GetTime() == TInstant::Zero()) { + WriteValue(TimeSeries_.GetValueType(), point.GetValue()); + } else { + Out_->Write('('); + WriteTime(point.GetTime()); + Out_->Write(TStringBuf(", ")); + WriteValue(TimeSeries_.GetValueType(), point.GetValue()); + Out_->Write(')'); + } + } + Out_->Write(']'); + } + Out_->Write('\n'); + } + + void ClearLastMetricState() { + MetricType_ = EMetricType::UNKNOWN; + Labels_.Clear(); + TimeSeries_.Clear(); + } + + private: + TEncoderState State_; + IOutputStream* Out_; + bool HumanReadableTs_; + TInstant CommonTime_ = TInstant::Zero(); + EMetricType MetricType_ = EMetricType::UNKNOWN; + TLabels Labels_; + TMetricTimeSeries TimeSeries_; + }; + + } + + IMetricEncoderPtr EncoderText(IOutputStream* out, bool humanReadableTs) { + return MakeHolder<TEncoderText>(out, humanReadableTs); + } + +} diff --git a/library/cpp/monlib/encode/text/text_encoder_ut.cpp b/library/cpp/monlib/encode/text/text_encoder_ut.cpp new file mode 100644 index 0000000000..554b6f5fa9 --- /dev/null +++ b/library/cpp/monlib/encode/text/text_encoder_ut.cpp @@ -0,0 +1,283 @@ +#include "text.h" + +#include <library/cpp/monlib/metrics/histogram_collector.h> + +#include <library/cpp/testing/unittest/registar.h> + +using namespace NMonitoring; + +Y_UNIT_TEST_SUITE(TTextText) { + template <typename TFunc> + TString EncodeToString(bool humanReadableTs, TFunc fn) { + TStringStream ss; + IMetricEncoderPtr encoder = EncoderText(&ss, humanReadableTs); + fn(encoder.Get()); + return ss.Str(); + } + + Y_UNIT_TEST(Empty) { + auto result = EncodeToString(true, [](IMetricEncoder* e) { + e->OnStreamBegin(); + e->OnStreamEnd(); + }); + UNIT_ASSERT_STRINGS_EQUAL(result, ""); + } + + Y_UNIT_TEST(CommonPart) { + auto result = EncodeToString(true, [](IMetricEncoder* e) { + e->OnStreamBegin(); + e->OnCommonTime(TInstant::ParseIso8601Deprecated("2017-01-02T03:04:05.006Z")); + { + e->OnLabelsBegin(); + e->OnLabel("project", "solomon"); + e->OnLabel("cluster", "man"); + e->OnLabel("service", "stockpile"); + e->OnLabelsEnd(); + } + e->OnStreamEnd(); + }); + UNIT_ASSERT_STRINGS_EQUAL(result, + "common time: 2017-01-02T03:04:05Z\n" + "common labels: {project='solomon', cluster='man', service='stockpile'}\n"); + } + + Y_UNIT_TEST(Gauges) { + auto result = EncodeToString(true, [](IMetricEncoder* e) { + e->OnStreamBegin(); + { // no values + e->OnMetricBegin(EMetricType::GAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "cpuUsage"); + e->OnLabelsEnd(); + } + e->OnMetricEnd(); + } + { // one value no ts + e->OnMetricBegin(EMetricType::GAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "diskUsage"); + e->OnLabel("disk", "sda1"); + e->OnLabelsEnd(); + } + e->OnDouble(TInstant::Zero(), 1000); + e->OnMetricEnd(); + } + { // one value with ts + e->OnMetricBegin(EMetricType::GAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "memoryUsage"); + e->OnLabel("host", "solomon-man-00"); + e->OnLabel("dc", "man"); + e->OnLabelsEnd(); + } + e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 1000); + e->OnMetricEnd(); + } + { // many values + e->OnMetricBegin(EMetricType::GAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "bytesRx"); + e->OnLabel("host", "solomon-sas-01"); + e->OnLabel("dc", "sas"); + e->OnLabelsEnd(); + } + e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 2); + e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:05Z"), 4); + e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:10Z"), 8); + e->OnMetricEnd(); + } + e->OnStreamEnd(); + }); + UNIT_ASSERT_STRINGS_EQUAL(result, + " GAUGE cpuUsage{}\n" + " GAUGE diskUsage{disk='sda1'} [1000]\n" + " GAUGE memoryUsage{host='solomon-man-00', dc='man'} [(2017-12-02T12:00:00Z, 1000)]\n" + " GAUGE bytesRx{host='solomon-sas-01', dc='sas'} [(2017-12-02T12:00:00Z, 2), (2017-12-02T12:00:05Z, 4), (2017-12-02T12:00:10Z, 8)]\n"); + } + + Y_UNIT_TEST(IntGauges) { + auto result = EncodeToString(true, [](IMetricEncoder* e) { + e->OnStreamBegin(); + { // no values + e->OnMetricBegin(EMetricType::IGAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "cpuUsage"); + e->OnLabelsEnd(); + } + e->OnMetricEnd(); + } + { // one value no ts + e->OnMetricBegin(EMetricType::IGAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "diskUsage"); + e->OnLabel("disk", "sda1"); + e->OnLabelsEnd(); + } + e->OnDouble(TInstant::Zero(), 1000); + e->OnMetricEnd(); + } + { // one value with ts + e->OnMetricBegin(EMetricType::IGAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "memoryUsage"); + e->OnLabel("host", "solomon-man-00"); + e->OnLabel("dc", "man"); + e->OnLabelsEnd(); + } + e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 1000); + e->OnMetricEnd(); + } + { // many values + e->OnMetricBegin(EMetricType::IGAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "bytesRx"); + e->OnLabel("host", "solomon-sas-01"); + e->OnLabel("dc", "sas"); + e->OnLabelsEnd(); + } + e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 2); + e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:05Z"), 4); + e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:10Z"), 8); + e->OnMetricEnd(); + } + e->OnStreamEnd(); + }); + UNIT_ASSERT_STRINGS_EQUAL(result, + " IGAUGE cpuUsage{}\n" + " IGAUGE diskUsage{disk='sda1'} [1000]\n" + " IGAUGE memoryUsage{host='solomon-man-00', dc='man'} [(2017-12-02T12:00:00Z, 1000)]\n" + " IGAUGE bytesRx{host='solomon-sas-01', dc='sas'} [(2017-12-02T12:00:00Z, 2), (2017-12-02T12:00:05Z, 4), (2017-12-02T12:00:10Z, 8)]\n"); + } + + Y_UNIT_TEST(Counters) { + auto doEncode = [](IMetricEncoder* e) { + e->OnStreamBegin(); + { // no values + e->OnMetricBegin(EMetricType::COUNTER); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "cpuUsage"); + e->OnLabelsEnd(); + } + e->OnMetricEnd(); + } + { // one value no ts + e->OnMetricBegin(EMetricType::COUNTER); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "diskUsage"); + e->OnLabel("disk", "sda1"); + e->OnLabelsEnd(); + } + e->OnUint64(TInstant::Zero(), 1000); + e->OnMetricEnd(); + } + { // one value with ts + e->OnMetricBegin(EMetricType::COUNTER); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "memoryUsage"); + e->OnLabel("host", "solomon-man-00"); + e->OnLabel("dc", "man"); + e->OnLabelsEnd(); + } + e->OnUint64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 1000); + e->OnMetricEnd(); + } + { // many values + e->OnMetricBegin(EMetricType::COUNTER); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "bytesRx"); + e->OnLabel("host", "solomon-sas-01"); + e->OnLabel("dc", "sas"); + e->OnLabelsEnd(); + } + e->OnUint64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 2); + e->OnUint64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:05Z"), 4); + e->OnUint64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:10Z"), 8); + e->OnMetricEnd(); + } + e->OnStreamEnd(); + }; + + auto result1 = EncodeToString(false, doEncode); + UNIT_ASSERT_STRINGS_EQUAL(result1, + " COUNTER cpuUsage{}\n" + " COUNTER diskUsage{disk='sda1'} [1000]\n" + " COUNTER memoryUsage{host='solomon-man-00', dc='man'} [(1512216000, 1000)]\n" + " COUNTER bytesRx{host='solomon-sas-01', dc='sas'} [(1512216000, 2), (1512216005, 4), (1512216010, 8)]\n"); + + auto result2 = EncodeToString(true, doEncode); + UNIT_ASSERT_STRINGS_EQUAL(result2, + " COUNTER cpuUsage{}\n" + " COUNTER diskUsage{disk='sda1'} [1000]\n" + " COUNTER memoryUsage{host='solomon-man-00', dc='man'} [(2017-12-02T12:00:00Z, 1000)]\n" + " COUNTER bytesRx{host='solomon-sas-01', dc='sas'} [(2017-12-02T12:00:00Z, 2), (2017-12-02T12:00:05Z, 4), (2017-12-02T12:00:10Z, 8)]\n"); + } + + Y_UNIT_TEST(Histograms) { + auto h = ExplicitHistogram({1, 2, 3, 4, 5}); + h->Collect(3); + h->Collect(5, 7); + h->Collect(13); + auto s = h->Snapshot(); + + TString result = EncodeToString(true, [s](IMetricEncoder* e) { + e->OnStreamBegin(); + { + e->OnMetricBegin(EMetricType::HIST); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "readTimeMillis"); + e->OnLabelsEnd(); + } + e->OnHistogram(TInstant::Zero(), s); + e->OnMetricEnd(); + } + { + e->OnMetricBegin(EMetricType::HIST_RATE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "writeTimeMillis"); + e->OnLabelsEnd(); + } + e->OnHistogram(TInstant::Zero(), s); + e->OnMetricEnd(); + } + e->OnStreamEnd(); + }); + + UNIT_ASSERT_STRINGS_EQUAL(result, + " HIST readTimeMillis{} [{1: 0, 2: 0, 3: 1, 4: 0, 5: 7, inf: 1}]\n" + "HIST_RATE writeTimeMillis{} [{1: 0, 2: 0, 3: 1, 4: 0, 5: 7, inf: 1}]\n"); + } + + Y_UNIT_TEST(Summary) { + auto s = MakeIntrusive<TSummaryDoubleSnapshot>(10.1, -0.45, 0.478, 0.3, 30u); + TString result = EncodeToString(true, [s](IMetricEncoder* e) { + e->OnStreamBegin(); + { + e->OnMetricBegin(EMetricType::DSUMMARY); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "temperature"); + e->OnLabelsEnd(); + } + e->OnSummaryDouble(TInstant::Zero(), s); + e->OnMetricEnd(); + } + e->OnStreamEnd(); + }); + UNIT_ASSERT_STRINGS_EQUAL(result, + " DSUMMARY temperature{} [{sum: 10.1, min: -0.45, max: 0.478, last: 0.3, count: 30}]\n"); + } +} diff --git a/library/cpp/monlib/encode/text/ut/ya.make b/library/cpp/monlib/encode/text/ut/ya.make new file mode 100644 index 0000000000..df23a252d1 --- /dev/null +++ b/library/cpp/monlib/encode/text/ut/ya.make @@ -0,0 +1,12 @@ +UNITTEST_FOR(library/cpp/monlib/encode/text) + +OWNER( + g:solomon + jamel +) + +SRCS( + text_encoder_ut.cpp +) + +END() diff --git a/library/cpp/monlib/encode/text/ya.make b/library/cpp/monlib/encode/text/ya.make new file mode 100644 index 0000000000..d296c78c1b --- /dev/null +++ b/library/cpp/monlib/encode/text/ya.make @@ -0,0 +1,16 @@ +LIBRARY() + +OWNER( + g:solomon + jamel +) + +SRCS( + text_encoder.cpp +) + +PEERDIR( + library/cpp/monlib/encode +) + +END() diff --git a/library/cpp/monlib/encode/unistat/unistat.h b/library/cpp/monlib/encode/unistat/unistat.h new file mode 100644 index 0000000000..1c43b7fa1b --- /dev/null +++ b/library/cpp/monlib/encode/unistat/unistat.h @@ -0,0 +1,13 @@ +#pragma once + +#include <util/generic/fwd.h> +#include <util/datetime/base.h> + +namespace NMonitoring { + /// Decodes unistat-style metrics + /// https://wiki.yandex-team.ru/golovan/stat-handle + void DecodeUnistat(TStringBuf data, class IMetricConsumer* c, TInstant ts = TInstant::Zero()); + + /// Assumes consumer's stream is open by the caller + void DecodeUnistatToStream(TStringBuf data, class IMetricConsumer* c, TInstant = TInstant::Zero()); +} diff --git a/library/cpp/monlib/encode/unistat/unistat_decoder.cpp b/library/cpp/monlib/encode/unistat/unistat_decoder.cpp new file mode 100644 index 0000000000..b2344b0905 --- /dev/null +++ b/library/cpp/monlib/encode/unistat/unistat_decoder.cpp @@ -0,0 +1,253 @@ +#include "unistat.h" + +#include <library/cpp/monlib/metrics/histogram_collector.h> +#include <library/cpp/monlib/metrics/labels.h> +#include <library/cpp/monlib/metrics/metric_type.h> +#include <library/cpp/monlib/metrics/metric_value.h> +#include <library/cpp/monlib/metrics/metric_consumer.h> + +#include <library/cpp/json/json_reader.h> + +#include <util/datetime/base.h> +#include <util/string/split.h> + +#include <contrib/libs/re2/re2/re2.h> + +using namespace NJson; + +const re2::RE2 NAME_RE{R"((?:[a-zA-Z0-9\.\-/@_]+_)+(?:[ad][vehmntx]{3}|summ|hgram|max))"}; + +namespace NMonitoring { + namespace { + bool IsNumber(const NJson::TJsonValue& j) { + switch (j.GetType()) { + case EJsonValueType::JSON_INTEGER: + case EJsonValueType::JSON_UINTEGER: + case EJsonValueType::JSON_DOUBLE: + return true; + + default: + return false; + } + } + + template <typename T> + T ExtractNumber(const TJsonValue& val) { + switch (val.GetType()) { + case EJsonValueType::JSON_INTEGER: + return static_cast<T>(val.GetInteger()); + case EJsonValueType::JSON_UINTEGER: + return static_cast<T>(val.GetUInteger()); + case EJsonValueType::JSON_DOUBLE: + return static_cast<T>(val.GetDouble()); + + default: + ythrow yexception() << "Expected number, but found " << val.GetType(); + } + } + + auto ExtractDouble = ExtractNumber<double>; + auto ExtractUi64 = ExtractNumber<ui64>; + + class THistogramBuilder { + public: + void Add(TBucketBound bound, TBucketValue value) { + /// XXX: yasm uses left-closed intervals, while in monlib we use right-closed ones, + /// so (-inf; 0) [0, 100) [100; +inf) + /// becomes (-inf; 0] (0, 100] (100; +inf) + /// but since we've already lost some information these no way to avoid this kind of error here + Bounds_.push_back(bound); + + /// this will always be 0 for the first bucket, + /// since there's no way to make (-inf; N) bucket in yasm + Values_.push_back(NextValue_); + + /// we will write this value into the next bucket so that [[0, 10], [100, 20], [200, 50]] + /// becomes (-inf; 0] -> 0; (0; 100] -> 10; (100; 200] -> 20; (200; +inf) -> 50 + NextValue_ = value; + } + + IHistogramSnapshotPtr Finalize() { + Bounds_.push_back(std::numeric_limits<TBucketBound>::max()); + Values_.push_back(NextValue_); + + Y_ENSURE(Bounds_.size() <= HISTOGRAM_MAX_BUCKETS_COUNT, + "Histogram is only allowed to have " << HISTOGRAM_MAX_BUCKETS_COUNT << " buckets, but has " << Bounds_.size()); + + return ExplicitHistogramSnapshot(Bounds_, Values_); + } + + public: + TBucketValue NextValue_ {0}; + TBucketBounds Bounds_; + TBucketValues Values_; + }; + + class TDecoderUnistat { + private: + public: + explicit TDecoderUnistat(IMetricConsumer* consumer, IInputStream* is, TInstant ts) + : Consumer_{consumer} + , Timestamp_{ts} { + ReadJsonTree(is, &Json_, /* throw */ true); + } + + void Decode() { + Y_ENSURE(Json_.IsArray(), "Expected array at the top level, but found " << Json_.GetType()); + + for (auto&& metric : Json_.GetArray()) { + Y_ENSURE(metric.IsArray(), "Metric must be an array"); + auto&& arr = metric.GetArray(); + Y_ENSURE(arr.size() == 2, "Metric must be an array of 2 elements"); + auto&& name = arr[0]; + auto&& value = arr[1]; + MetricContext_ = {}; + + ParseName(name.GetString()); + + if (value.IsArray()) { + OnHistogram(value); + } else if (IsNumber(value)) { + OnScalar(value); + } else { + ythrow yexception() << "Expected list or number, but found " << value.GetType(); + } + + WriteValue(); + } + } + + private: + void OnScalar(const TJsonValue& jsonValue) { + if (MetricContext_.IsDeriv) { + MetricContext_.Type = EMetricType::RATE; + MetricContext_.Value = TMetricValue{ExtractUi64(jsonValue)}; + } else { + MetricContext_.Type = EMetricType::GAUGE; + MetricContext_.Value = TMetricValue{ExtractDouble(jsonValue)}; + } + } + + void OnHistogram(const TJsonValue& jsonHist) { + if (MetricContext_.IsDeriv) { + MetricContext_.Type = EMetricType::HIST_RATE; + } else { + MetricContext_.Type = EMetricType::HIST; + } + + auto histogramBuilder = THistogramBuilder(); + + for (auto&& bucket : jsonHist.GetArray()) { + Y_ENSURE(bucket.IsArray(), "Expected an array, but found " << bucket.GetType()); + auto&& arr = bucket.GetArray(); + Y_ENSURE(arr.size() == 2, "Histogram bucket must be an array of 2 elements"); + const auto bound = ExtractDouble(arr[0]); + const auto weight = ExtractUi64(arr[1]); + histogramBuilder.Add(bound, weight); + } + + MetricContext_.Histogram = histogramBuilder.Finalize(); + MetricContext_.Value = TMetricValue{MetricContext_.Histogram.Get()}; + } + + bool IsDeriv(TStringBuf name) { + TStringBuf ignore, suffix; + name.RSplit('_', ignore, suffix); + + Y_ENSURE(suffix.size() >= 3 && suffix.size() <= 5, "Disallowed suffix value: " << suffix); + + if (suffix == TStringBuf("summ") || suffix == TStringBuf("hgram")) { + return true; + } else if (suffix == TStringBuf("max")) { + return false; + } + + return suffix[0] == 'd'; + } + + void ParseName(TStringBuf value) { + TVector<TStringBuf> parts; + StringSplitter(value).Split(';').SkipEmpty().Collect(&parts); + + Y_ENSURE(parts.size() >= 1 && parts.size() <= 16); + + TStringBuf name = parts.back(); + parts.pop_back(); + + Y_ENSURE(RE2::FullMatch(re2::StringPiece{name.data(), name.size()}, NAME_RE), + "Metric name " << name << " doesn't match regex " << NAME_RE.pattern()); + + MetricContext_.Name = name; + MetricContext_.IsDeriv = IsDeriv(MetricContext_.Name); + + for (auto tag : parts) { + TStringBuf n, v; + tag.Split('=', n, v); + Y_ENSURE(n && v, "Unexpected tag format in " << tag); + MetricContext_.Labels.Add(n, v); + } + } + + private: + void WriteValue() { + Consumer_->OnMetricBegin(MetricContext_.Type); + + Consumer_->OnLabelsBegin(); + Consumer_->OnLabel("sensor", TString{MetricContext_.Name}); + for (auto&& l : MetricContext_.Labels) { + Consumer_->OnLabel(l.Name(), l.Value()); + } + + Consumer_->OnLabelsEnd(); + + switch (MetricContext_.Type) { + case EMetricType::GAUGE: + Consumer_->OnDouble(Timestamp_, MetricContext_.Value.AsDouble()); + break; + case EMetricType::RATE: + Consumer_->OnUint64(Timestamp_, MetricContext_.Value.AsUint64()); + break; + case EMetricType::HIST: + case EMetricType::HIST_RATE: + Consumer_->OnHistogram(Timestamp_, MetricContext_.Value.AsHistogram()); + break; + case EMetricType::LOGHIST: + case EMetricType::DSUMMARY: + case EMetricType::IGAUGE: + case EMetricType::COUNTER: + case EMetricType::UNKNOWN: + ythrow yexception() << "Unexpected metric type: " << MetricContext_.Type; + } + + Consumer_->OnMetricEnd(); + } + + private: + IMetricConsumer* Consumer_; + NJson::TJsonValue Json_; + TInstant Timestamp_; + + struct { + TStringBuf Name; + EMetricType Type{EMetricType::UNKNOWN}; + TMetricValue Value; + bool IsDeriv{false}; + TLabels Labels; + IHistogramSnapshotPtr Histogram; + } MetricContext_; + }; + + } + + void DecodeUnistat(TStringBuf data, IMetricConsumer* c, TInstant ts) { + c->OnStreamBegin(); + DecodeUnistatToStream(data, c, ts); + c->OnStreamEnd(); + } + + void DecodeUnistatToStream(TStringBuf data, IMetricConsumer* c, TInstant ts) { + TMemoryInput in{data.data(), data.size()}; + TDecoderUnistat decoder(c, &in, ts); + decoder.Decode(); + } +} diff --git a/library/cpp/monlib/encode/unistat/unistat_ut.cpp b/library/cpp/monlib/encode/unistat/unistat_ut.cpp new file mode 100644 index 0000000000..dbbc238bf3 --- /dev/null +++ b/library/cpp/monlib/encode/unistat/unistat_ut.cpp @@ -0,0 +1,223 @@ +#include "unistat.h" + +#include <library/cpp/monlib/encode/protobuf/protobuf.h> +#include <library/cpp/monlib/metrics/labels.h> + +#include <library/cpp/testing/unittest/registar.h> + +using namespace NMonitoring; + +Y_UNIT_TEST_SUITE(TUnistatDecoderTest) { + Y_UNIT_TEST(ScalarMetric) { + constexpr auto input = TStringBuf(R"([["something_axxx", 42]])"); + + NProto::TMultiSamplesList samples; + auto encoder = EncoderProtobuf(&samples); + + DecodeUnistat(input, encoder.Get()); + + UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 1); + auto sample = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1); + UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1); + + auto label = sample.GetLabels(0); + auto point = sample.GetPoints(0); + UNIT_ASSERT_VALUES_EQUAL(point.GetFloat64(), 42.); + UNIT_ASSERT_VALUES_EQUAL(label.GetName(), "sensor"); + UNIT_ASSERT_VALUES_EQUAL(label.GetValue(), "something_axxx"); + } + + Y_UNIT_TEST(OverriddenTags) { + constexpr auto input = TStringBuf(R"([["ctype=foo;prj=bar;custom_tag=qwe;something_axxx", 42]])"); + + NProto::TMultiSamplesList samples; + auto encoder = EncoderProtobuf(&samples); + + DecodeUnistat(input, encoder.Get()); + + UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 1); + auto sample = samples.GetSamples(0); + UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1); + UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 4); + + const auto& labels = sample.GetLabels(); + TLabels actual; + for (auto&& l : labels) { + actual.Add(l.GetName(), l.GetValue()); + } + + TLabels expected{{"ctype", "foo"}, {"prj", "bar"}, {"custom_tag", "qwe"}, {"sensor", "something_axxx"}}; + + UNIT_ASSERT_VALUES_EQUAL(actual.size(), expected.size()); + for (auto&& l : actual) { + UNIT_ASSERT(expected.Extract(l.Name())->Value() == l.Value()); + } + } + + Y_UNIT_TEST(ThrowsOnTopLevelObject) { + constexpr auto input = TStringBuf(R"({["something_axxx", 42]})"); + + NProto::TMultiSamplesList samples; + auto encoder = EncoderProtobuf(&samples); + + UNIT_ASSERT_EXCEPTION(DecodeUnistat(input, encoder.Get()), yexception); + } + + Y_UNIT_TEST(ThrowsOnUnwrappedMetric) { + constexpr auto input = TStringBuf(R"(["something_axxx", 42])"); + + NProto::TMultiSamplesList samples; + auto encoder = EncoderProtobuf(&samples); + + UNIT_ASSERT_EXCEPTION(DecodeUnistat(input, encoder.Get()), yexception); + } + + Y_UNIT_TEST(HistogramMetric) { + constexpr auto input = TStringBuf(R"([["something_hgram", [[0, 1], [200, 2], [500, 3]] ]])"); + + NProto::TMultiSamplesList samples; + auto encoder = EncoderProtobuf(&samples); + + DecodeUnistat(input, encoder.Get()); + + auto sample = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::HIST_RATE); + UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1); + UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1); + + auto label = sample.GetLabels(0); + const auto point = sample.GetPoints(0); + const auto histogram = point.GetHistogram(); + const auto size = histogram.BoundsSize(); + UNIT_ASSERT_VALUES_EQUAL(size, 4); + + const TVector<double> expectedBounds {0, 200, 500, std::numeric_limits<double>::max()}; + const TVector<ui64> expectedValues {0, 1, 2, 3}; + + for (auto i = 0; i < 4; ++i) { + UNIT_ASSERT_VALUES_EQUAL(histogram.GetBounds(i), expectedBounds[i]); + UNIT_ASSERT_VALUES_EQUAL(histogram.GetValues(i), expectedValues[i]); + } + + UNIT_ASSERT_VALUES_EQUAL(label.GetName(), "sensor"); + UNIT_ASSERT_VALUES_EQUAL(label.GetValue(), "something_hgram"); + } + + Y_UNIT_TEST(AbsoluteHistogram) { + constexpr auto input = TStringBuf(R"([["something_ahhh", [[0, 1], [200, 2], [500, 3]] ]])"); + + NProto::TMultiSamplesList samples; + auto encoder = EncoderProtobuf(&samples); + + DecodeUnistat(input, encoder.Get()); + + auto sample = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::HISTOGRAM); + UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1); + UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1); + } + + Y_UNIT_TEST(AllowedMetricNames) { + NProto::TMultiSamplesList samples; + auto encoder = EncoderProtobuf(&samples); + + { + constexpr auto input = TStringBuf(R"([["a/A-b/c_D/__G_dmmm", [[0, 1], [200, 2], [500, 3]] ]])"); + UNIT_ASSERT_NO_EXCEPTION(DecodeUnistat(input, encoder.Get())); + } + } + + Y_UNIT_TEST(DisallowedMetricNames) { + NProto::TMultiSamplesList samples; + auto encoder = EncoderProtobuf(&samples); + + { + constexpr auto input = TStringBuf(R"([["someth!ng_ahhh", [[0, 1], [200, 2], [500, 3]] ]])"); + UNIT_ASSERT_EXCEPTION(DecodeUnistat(input, encoder.Get()), yexception); + } + + { + constexpr auto input = TStringBuf(R"([["foo_a", [[0, 1], [200, 2], [500, 3]] ]])"); + UNIT_ASSERT_EXCEPTION(DecodeUnistat(input, encoder.Get()), yexception); + } + + { + constexpr auto input = TStringBuf(R"([["foo_ahhh;tag=value", [[0, 1], [200, 2], [500, 3]] ]])"); + UNIT_ASSERT_EXCEPTION(DecodeUnistat(input, encoder.Get()), yexception); + } + } + + Y_UNIT_TEST(MultipleMetrics) { + constexpr auto input = TStringBuf(R"([["something_axxx", 42], ["some-other_dhhh", 53]])"); + + NProto::TMultiSamplesList samples; + auto encoder = EncoderProtobuf(&samples); + + DecodeUnistat(input, encoder.Get()); + + UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 2); + auto sample = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1); + UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1); + + auto label = sample.GetLabels(0); + auto point = sample.GetPoints(0); + UNIT_ASSERT_VALUES_EQUAL(point.GetFloat64(), 42.); + UNIT_ASSERT_VALUES_EQUAL(label.GetName(), "sensor"); + UNIT_ASSERT_VALUES_EQUAL(label.GetValue(), "something_axxx"); + + sample = samples.GetSamples(1); + UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::RATE); + UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1); + UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1); + + label = sample.GetLabels(0); + point = sample.GetPoints(0); + UNIT_ASSERT_VALUES_EQUAL(point.GetUint64(), 53); + UNIT_ASSERT_VALUES_EQUAL(label.GetName(), "sensor"); + UNIT_ASSERT_VALUES_EQUAL(label.GetValue(), "some-other_dhhh"); + } + + Y_UNIT_TEST(UnderscoreName) { + constexpr auto input = TStringBuf(R"([["something_anything_dmmm", 42]])"); + + NProto::TMultiSamplesList samples; + auto encoder = EncoderProtobuf(&samples); + DecodeUnistat(input, encoder.Get()); + + UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 1); + auto sample = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::RATE); + UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1); + UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1); + + auto label = sample.GetLabels(0); + auto point = sample.GetPoints(0); + UNIT_ASSERT_VALUES_EQUAL(point.GetUint64(), 42); + UNIT_ASSERT_VALUES_EQUAL(label.GetName(), "sensor"); + UNIT_ASSERT_VALUES_EQUAL(label.GetValue(), "something_anything_dmmm"); + } + + Y_UNIT_TEST(MaxAggr) { + constexpr auto input = TStringBuf(R"([["something_anything_max", 42]])"); + + NProto::TMultiSamplesList samples; + auto encoder = EncoderProtobuf(&samples); + DecodeUnistat(input, encoder.Get()); + + UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 1); + auto sample = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1); + UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1); + + auto label = sample.GetLabels(0); + auto point = sample.GetPoints(0); + UNIT_ASSERT_VALUES_EQUAL(point.GetFloat64(), 42.); + UNIT_ASSERT_VALUES_EQUAL(label.GetName(), "sensor"); + UNIT_ASSERT_VALUES_EQUAL(label.GetValue(), "something_anything_max"); + } +} diff --git a/library/cpp/monlib/encode/unistat/ut/ya.make b/library/cpp/monlib/encode/unistat/ut/ya.make new file mode 100644 index 0000000000..a652139f45 --- /dev/null +++ b/library/cpp/monlib/encode/unistat/ut/ya.make @@ -0,0 +1,16 @@ +UNITTEST_FOR(library/cpp/monlib/encode/unistat) + +OWNER( + msherbakov + g:solomon +) + +SRCS( + unistat_ut.cpp +) + +PEERDIR( + library/cpp/monlib/encode/protobuf +) + +END() diff --git a/library/cpp/monlib/encode/unistat/ya.make b/library/cpp/monlib/encode/unistat/ya.make new file mode 100644 index 0000000000..4ac2edadf4 --- /dev/null +++ b/library/cpp/monlib/encode/unistat/ya.make @@ -0,0 +1,18 @@ +LIBRARY() + +OWNER( + msherbakov + g:solomon +) + +PEERDIR( + contrib/libs/re2 + library/cpp/json + library/cpp/monlib/metrics +) + +SRCS( + unistat_decoder.cpp +) + +END() diff --git a/library/cpp/monlib/encode/ut/ya.make b/library/cpp/monlib/encode/ut/ya.make new file mode 100644 index 0000000000..1990386d76 --- /dev/null +++ b/library/cpp/monlib/encode/ut/ya.make @@ -0,0 +1,12 @@ +UNITTEST_FOR(library/cpp/monlib/encode) + +OWNER( + jamel + g:solomon +) + +SRCS( + format_ut.cpp +) + +END() diff --git a/library/cpp/monlib/encode/ya.make b/library/cpp/monlib/encode/ya.make new file mode 100644 index 0000000000..d1bb09f07b --- /dev/null +++ b/library/cpp/monlib/encode/ya.make @@ -0,0 +1,20 @@ +LIBRARY() + +OWNER( + g:solomon + jamel +) + +SRCS( + encoder.cpp + encoder_state.cpp + format.cpp +) + +PEERDIR( + library/cpp/monlib/metrics +) + +GENERATE_ENUM_SERIALIZATION_WITH_HEADER(encoder_state_enum.h) + +END() diff --git a/library/cpp/monlib/exception/exception.cpp b/library/cpp/monlib/exception/exception.cpp new file mode 100644 index 0000000000..5a8d53ceb0 --- /dev/null +++ b/library/cpp/monlib/exception/exception.cpp @@ -0,0 +1 @@ +#include "exception.h" diff --git a/library/cpp/monlib/exception/exception.h b/library/cpp/monlib/exception/exception.h new file mode 100644 index 0000000000..027c22b27d --- /dev/null +++ b/library/cpp/monlib/exception/exception.h @@ -0,0 +1,13 @@ +#pragma once + + +namespace NMonitoring { + +#define MONLIB_ENSURE_EX(CONDITION, THROW_EXPRESSION) \ + do { \ + if (Y_UNLIKELY(!(CONDITION))) { \ + throw THROW_EXPRESSION; \ + } \ + } while (false) + +} // namespace NSolomon diff --git a/library/cpp/monlib/exception/ya.make b/library/cpp/monlib/exception/ya.make new file mode 100644 index 0000000000..78660711d3 --- /dev/null +++ b/library/cpp/monlib/exception/ya.make @@ -0,0 +1,12 @@ +LIBRARY() + +OWNER(g:solomon) + +SRCS( + exception.cpp +) + +PEERDIR( +) + +END() diff --git a/library/cpp/monlib/messagebus/mon_messagebus.cpp b/library/cpp/monlib/messagebus/mon_messagebus.cpp new file mode 100644 index 0000000000..355b4386cd --- /dev/null +++ b/library/cpp/monlib/messagebus/mon_messagebus.cpp @@ -0,0 +1,11 @@ +#include <library/cpp/messagebus/www/www.h> + +#include "mon_messagebus.h" + +using namespace NMonitoring; + +void TBusNgMonPage::Output(NMonitoring::IMonHttpRequest& request) { + NBus::TBusWww::TOptionalParams params; + params.ParentLinks.push_back(NBus::TBusWww::TLink{"/", request.GetServiceTitle()}); + BusWww->ServeHttp(request.Output(), request.GetParams(), params); +} diff --git a/library/cpp/monlib/messagebus/mon_messagebus.h b/library/cpp/monlib/messagebus/mon_messagebus.h new file mode 100644 index 0000000000..e1fa73c69f --- /dev/null +++ b/library/cpp/monlib/messagebus/mon_messagebus.h @@ -0,0 +1,46 @@ +#pragma once + +#include <library/cpp/messagebus/ybus.h> +#include <library/cpp/messagebus/www/www.h> + +#include <library/cpp/monlib/service/pages/mon_page.h> + +namespace NMonitoring { + template <class TBusSmth> + class TBusSmthMonPage: public NMonitoring::IMonPage { + private: + TBusSmth* Smth; + + public: + explicit TBusSmthMonPage(const TString& name, const TString& title, TBusSmth* smth) + : IMonPage("msgbus/" + name, title) + , Smth(smth) + { + } + void Output(NMonitoring::IMonHttpRequest& request) override { + Y_UNUSED(request); + request.Output() << NMonitoring::HTTPOKHTML; + request.Output() << "<h2>" << Title << "</h2>"; + request.Output() << "<pre>"; + request.Output() << Smth->GetStatus(); + request.Output() << "</pre>"; + } + }; + + using TBusQueueMonPage = TBusSmthMonPage<NBus::TBusMessageQueue>; + using TBusModuleMonPage = TBusSmthMonPage<NBus::TBusModule>; + + class TBusNgMonPage: public NMonitoring::IMonPage { + public: + TIntrusivePtr<NBus::TBusWww> BusWww; + + public: + TBusNgMonPage() + : IMonPage("messagebus", "MessageBus") + , BusWww(new NBus::TBusWww) + { + } + void Output(NMonitoring::IMonHttpRequest& request) override; + }; + +} diff --git a/library/cpp/monlib/messagebus/mon_service_messagebus.cpp b/library/cpp/monlib/messagebus/mon_service_messagebus.cpp new file mode 100644 index 0000000000..4dd144ebe8 --- /dev/null +++ b/library/cpp/monlib/messagebus/mon_service_messagebus.cpp @@ -0,0 +1,8 @@ +#include "mon_service_messagebus.h" + +using namespace NMonitoring; + +TMonServiceMessageBus::TMonServiceMessageBus(ui16 port, const TString& title) + : TMonService2(port, title) +{ +} diff --git a/library/cpp/monlib/messagebus/mon_service_messagebus.h b/library/cpp/monlib/messagebus/mon_service_messagebus.h new file mode 100644 index 0000000000..fe791e8a9b --- /dev/null +++ b/library/cpp/monlib/messagebus/mon_service_messagebus.h @@ -0,0 +1,46 @@ +#pragma once + +#include "mon_messagebus.h" + +#include <library/cpp/monlib/service/monservice.h> + +#include <util/system/mutex.h> + +namespace NMonitoring { + class TMonServiceMessageBus: public TMonService2 { + private: + TMutex Mtx; + TIntrusivePtr<NMonitoring::TBusNgMonPage> BusNgMonPage; + + public: + TMonServiceMessageBus(ui16 port, const TString& title); + + private: + NBus::TBusWww* RegisterBusNgMonPage() { + TGuard<TMutex> g(Mtx); + if (!BusNgMonPage) { + BusNgMonPage = new NMonitoring::TBusNgMonPage(); + Register(BusNgMonPage.Get()); + } + return BusNgMonPage->BusWww.Get(); + } + + public: + void RegisterClientSession(NBus::TBusClientSessionPtr clientSession) { + RegisterBusNgMonPage()->RegisterClientSession(clientSession); + } + + void RegisterServerSession(NBus::TBusServerSessionPtr serverSession) { + RegisterBusNgMonPage()->RegisterServerSession(serverSession); + } + + void RegisterQueue(NBus::TBusMessageQueuePtr queue) { + RegisterBusNgMonPage()->RegisterQueue(queue); + } + + void RegisterModule(NBus::TBusModule* module) { + RegisterBusNgMonPage()->RegisterModule(module); + } + }; + +} diff --git a/library/cpp/monlib/messagebus/ya.make b/library/cpp/monlib/messagebus/ya.make new file mode 100644 index 0000000000..a0b5362296 --- /dev/null +++ b/library/cpp/monlib/messagebus/ya.make @@ -0,0 +1,16 @@ +LIBRARY() + +OWNER(g:solomon) + +SRCS( + mon_messagebus.cpp + mon_service_messagebus.cpp +) + +PEERDIR( + library/cpp/messagebus + library/cpp/messagebus/www + library/cpp/monlib/dynamic_counters +) + +END() diff --git a/library/cpp/monlib/metrics/atomics_array.h b/library/cpp/monlib/metrics/atomics_array.h new file mode 100644 index 0000000000..f19aebf291 --- /dev/null +++ b/library/cpp/monlib/metrics/atomics_array.h @@ -0,0 +1,52 @@ +#pragma once + +#include <util/generic/ptr.h> +#include <util/generic/vector.h> + +#include <atomic> + +namespace NMonitoring { + class TAtomicsArray { + public: + explicit TAtomicsArray(size_t size) + : Values_(new std::atomic<ui64>[size]) + , Size_(size) + { + for (size_t i = 0; i < Size_; i++) { + Values_[i].store(0, std::memory_order_relaxed); + } + } + + ui64 operator[](size_t index) const noexcept { + Y_VERIFY_DEBUG(index < Size_); + return Values_[index].load(std::memory_order_relaxed); + } + + size_t Size() const noexcept { + return Size_; + } + + void Add(size_t index, ui32 count) noexcept { + Y_VERIFY_DEBUG(index < Size_); + Values_[index].fetch_add(count, std::memory_order_relaxed); + } + + void Reset() noexcept { + for (size_t i = 0; i < Size_; i++) { + Values_[i].store(0, std::memory_order_relaxed); + } + } + + TVector<ui64> Copy() const { + TVector<ui64> copy(Reserve(Size_)); + for (size_t i = 0; i < Size_; i++) { + copy.push_back(Values_[i].load(std::memory_order_relaxed)); + } + return copy; + } + + private: + TArrayHolder<std::atomic<ui64>> Values_; + size_t Size_; + }; +} diff --git a/library/cpp/monlib/metrics/ewma.cpp b/library/cpp/monlib/metrics/ewma.cpp new file mode 100644 index 0000000000..8a296c3225 --- /dev/null +++ b/library/cpp/monlib/metrics/ewma.cpp @@ -0,0 +1,150 @@ +#include "ewma.h" +#include "metric.h" + +#include <atomic> +#include <cmath> + +namespace NMonitoring { +namespace { + constexpr auto DEFAULT_INTERVAL = TDuration::Seconds(5); + + const double ALPHA1 = 1. - std::exp(-double(DEFAULT_INTERVAL.Seconds())/60./1.); + const double ALPHA5 = 1. - std::exp(-double(DEFAULT_INTERVAL.Seconds())/60./5.); + const double ALPHA15 = 1. - std::exp(-double(DEFAULT_INTERVAL.Seconds())/60./15.); + + class TExpMovingAverage final: public IExpMovingAverage { + public: + explicit TExpMovingAverage(IGauge* metric, double alpha, TDuration interval) + : Metric_{metric} + , Alpha_{alpha} + , Interval_{interval.Seconds()} + { + Y_VERIFY(metric != nullptr, "Passing nullptr metric is not allowed"); + } + + ~TExpMovingAverage() override = default; + + // This method NOT thread safe + void Tick() override { + const auto current = Uncounted_.fetch_and(0); + const double instantRate = double(current) / Interval_; + + if (Y_UNLIKELY(!IsInitialized())) { + Metric_->Set(instantRate); + Init_ = true; + } else { + const double currentRate = Metric_->Get(); + Metric_->Set(Alpha_ * (instantRate - currentRate) + currentRate); + } + + } + + void Update(i64 value) override { + Uncounted_ += value; + } + + double Rate() const override { + return Metric_->Get(); + } + + void Reset() override { + Init_ = false; + Uncounted_ = 0; + } + + private: + bool IsInitialized() const { + return Init_; + } + + private: + std::atomic<i64> Uncounted_{0}; + std::atomic<bool> Init_{false}; + + IGauge* Metric_{nullptr}; + double Alpha_; + ui64 Interval_; + }; + + struct TFakeEwma: IExpMovingAverage { + void Tick() override {} + void Update(i64) override {} + double Rate() const override { return 0; } + void Reset() override {} + }; + +} // namespace + + TEwmaMeter::TEwmaMeter() + : Ewma_{MakeHolder<TFakeEwma>()} + { + } + + TEwmaMeter::TEwmaMeter(IExpMovingAveragePtr&& ewma) + : Ewma_{std::move(ewma)} + { + } + + TEwmaMeter::TEwmaMeter(TEwmaMeter&& other) { + if (&other == this) { + return; + } + + *this = std::move(other); + } + + TEwmaMeter& TEwmaMeter::operator=(TEwmaMeter&& other) { + Ewma_ = std::move(other.Ewma_); + LastTick_.store(other.LastTick_); + return *this; + } + + void TEwmaMeter::TickIfNeeded() { + constexpr ui64 INTERVAL_SECONDS = DEFAULT_INTERVAL.Seconds(); + + const auto now = TInstant::Now().Seconds(); + ui64 old = LastTick_.load(); + const auto secondsSinceLastTick = now - old; + + if (secondsSinceLastTick > INTERVAL_SECONDS) { + // round to the interval grid + const ui64 newLast = now - (secondsSinceLastTick % INTERVAL_SECONDS); + if (LastTick_.compare_exchange_strong(old, newLast)) { + for (size_t i = 0; i < secondsSinceLastTick / INTERVAL_SECONDS; ++i) { + Ewma_->Tick(); + } + } + } + } + + void TEwmaMeter::Mark() { + TickIfNeeded(); + Ewma_->Update(1); + } + + void TEwmaMeter::Mark(i64 value) { + TickIfNeeded(); + Ewma_->Update(value); + } + + double TEwmaMeter::Get() { + TickIfNeeded(); + return Ewma_->Rate(); + } + + IExpMovingAveragePtr OneMinuteEwma(IGauge* metric) { + return MakeHolder<TExpMovingAverage>(metric, ALPHA1, DEFAULT_INTERVAL); + } + + IExpMovingAveragePtr FiveMinuteEwma(IGauge* metric) { + return MakeHolder<TExpMovingAverage>(metric, ALPHA5, DEFAULT_INTERVAL); + } + + IExpMovingAveragePtr FiveteenMinuteEwma(IGauge* metric) { + return MakeHolder<TExpMovingAverage>(metric, ALPHA15, DEFAULT_INTERVAL); + } + + IExpMovingAveragePtr CreateEwma(IGauge* metric, double alpha, TDuration interval) { + return MakeHolder<TExpMovingAverage>(metric, alpha, interval); + } +} // namespace NMonitoring diff --git a/library/cpp/monlib/metrics/ewma.h b/library/cpp/monlib/metrics/ewma.h new file mode 100644 index 0000000000..9b2dad7cc5 --- /dev/null +++ b/library/cpp/monlib/metrics/ewma.h @@ -0,0 +1,47 @@ +#pragma once + +#include <util/datetime/base.h> +#include <util/generic/ptr.h> + +#include <atomic> + +namespace NMonitoring { + class IGauge; + + class IExpMovingAverage { + public: + virtual ~IExpMovingAverage() = default; + virtual void Tick() = 0; + virtual void Update(i64 value) = 0; + virtual double Rate() const = 0; + virtual void Reset() = 0; + }; + + using IExpMovingAveragePtr = THolder<IExpMovingAverage>; + + class TEwmaMeter { + public: + // Creates a fake EWMA that will always return 0. Mostly for usage convenience + TEwmaMeter(); + explicit TEwmaMeter(IExpMovingAveragePtr&& ewma); + + TEwmaMeter(TEwmaMeter&& other); + TEwmaMeter& operator=(TEwmaMeter&& other); + + void Mark(); + void Mark(i64 value); + + double Get(); + + private: + void TickIfNeeded(); + + private: + IExpMovingAveragePtr Ewma_; + std::atomic<ui64> LastTick_{TInstant::Now().Seconds()}; + }; + + IExpMovingAveragePtr OneMinuteEwma(IGauge* gauge); + IExpMovingAveragePtr FiveMinuteEwma(IGauge* gauge); + IExpMovingAveragePtr FiveteenMinuteEwma(IGauge* gauge); +} // namespace NMonitoring diff --git a/library/cpp/monlib/metrics/ewma_ut.cpp b/library/cpp/monlib/metrics/ewma_ut.cpp new file mode 100644 index 0000000000..01ef2478f7 --- /dev/null +++ b/library/cpp/monlib/metrics/ewma_ut.cpp @@ -0,0 +1,112 @@ +#include "ewma.h" +#include "metric.h" + +#include <library/cpp/testing/unittest/registar.h> + + +using namespace NMonitoring; + +const auto EPS = 1e-6; +void ElapseMinute(IExpMovingAverage& ewma) { + for (auto i = 0; i < 12; ++i) { + ewma.Tick(); + } +} + +Y_UNIT_TEST_SUITE(TEwmaTest) { + Y_UNIT_TEST(OneMinute) { + TGauge gauge; + + auto ewma = OneMinuteEwma(&gauge); + ewma->Update(3); + ewma->Tick(); + + TVector<double> expectedValues { + 0.6, + 0.22072766, + 0.08120117, + 0.02987224, + 0.01098938, + 0.00404277, + 0.00148725, + 0.00054713, + 0.00020128, + 0.00007405, + 0.00002724, + 0.00001002, + 0.00000369, + 0.00000136, + 0.00000050, + 0.00000018, + }; + + for (auto expectedValue : expectedValues) { + UNIT_ASSERT_DOUBLES_EQUAL(ewma->Rate(), expectedValue, EPS); + ElapseMinute(*ewma); + } + } + + Y_UNIT_TEST(FiveMinutes) { + TGauge gauge; + + auto ewma = FiveMinuteEwma(&gauge); + ewma->Update(3); + ewma->Tick(); + + TVector<double> expectedValues { + 0.6, + 0.49123845, + 0.40219203, + 0.32928698, + 0.26959738, + 0.22072766, + 0.18071653, + 0.14795818, + 0.12113791, + 0.09917933, + 0.08120117, + 0.06648190, + 0.05443077, + 0.04456415, + 0.03648604, + 0.02987224, + }; + + for (auto expectedValue : expectedValues) { + UNIT_ASSERT_DOUBLES_EQUAL(ewma->Rate(), expectedValue, EPS); + ElapseMinute(*ewma); + } + } + + Y_UNIT_TEST(FiveteenMinutes) { + TGauge gauge; + + auto ewma = FiveteenMinuteEwma(&gauge); + ewma->Update(3); + ewma->Tick(); + + TVector<double> expectedValues { + 0.6, + 0.56130419, + 0.52510399, + 0.49123845, + 0.45955700, + 0.42991879, + 0.40219203, + 0.37625345, + 0.35198773, + 0.32928698, + 0.30805027, + 0.28818318, + 0.26959738, + 0.25221023, + 0.23594443, + 0.22072766, + }; + + for (auto expectedValue : expectedValues) { + UNIT_ASSERT_DOUBLES_EQUAL(ewma->Rate(), expectedValue, EPS); + ElapseMinute(*ewma); + } + } +}; diff --git a/library/cpp/monlib/metrics/fake.cpp b/library/cpp/monlib/metrics/fake.cpp new file mode 100644 index 0000000000..b6f5e37af8 --- /dev/null +++ b/library/cpp/monlib/metrics/fake.cpp @@ -0,0 +1,100 @@ +#include "fake.h" + +namespace NMonitoring { + + IGauge* TFakeMetricRegistry::Gauge(ILabelsPtr labels) { + return Metric<TFakeGauge, EMetricType::GAUGE>(std::move(labels)); + } + + ILazyGauge* TFakeMetricRegistry::LazyGauge(ILabelsPtr labels, std::function<double()> supplier) { + Y_UNUSED(supplier); + return Metric<TFakeLazyGauge, EMetricType::GAUGE>(std::move(labels)); + } + + IIntGauge* TFakeMetricRegistry::IntGauge(ILabelsPtr labels) { + return Metric<TFakeIntGauge, EMetricType::IGAUGE>(std::move(labels)); + } + + ILazyIntGauge* TFakeMetricRegistry::LazyIntGauge(ILabelsPtr labels, std::function<i64()> supplier) { + Y_UNUSED(supplier); + return Metric<TFakeLazyIntGauge, EMetricType::IGAUGE>(std::move(labels)); + } + + ICounter* TFakeMetricRegistry::Counter(ILabelsPtr labels) { + return Metric<TFakeCounter, EMetricType::COUNTER>(std::move(labels)); + } + + ILazyCounter* TFakeMetricRegistry::LazyCounter(ILabelsPtr labels, std::function<ui64()> supplier) { + Y_UNUSED(supplier); + return Metric<TFakeLazyCounter, EMetricType::COUNTER>(std::move(labels)); + } + + IRate* TFakeMetricRegistry::Rate(ILabelsPtr labels) { + return Metric<TFakeRate, EMetricType::RATE>(std::move(labels)); + } + + ILazyRate* TFakeMetricRegistry::LazyRate(ILabelsPtr labels, std::function<ui64()> supplier) { + Y_UNUSED(supplier); + return Metric<TFakeLazyRate, EMetricType::RATE>(std::move(labels)); + } + + IHistogram* TFakeMetricRegistry::HistogramCounter(ILabelsPtr labels, IHistogramCollectorPtr collector) { + Y_UNUSED(collector); + return Metric<TFakeHistogram, EMetricType::HIST>(std::move(labels), false); + } + + void TFakeMetricRegistry::RemoveMetric(const ILabels& labels) noexcept { + TWriteGuard g{Lock_}; + Metrics_.erase(labels); + } + + void TFakeMetricRegistry::Accept(TInstant time, IMetricConsumer* consumer) const { + Y_UNUSED(time); + consumer->OnStreamBegin(); + consumer->OnStreamEnd(); + } + + IHistogram* TFakeMetricRegistry::HistogramRate(ILabelsPtr labels, IHistogramCollectorPtr collector) { + Y_UNUSED(collector); + return Metric<TFakeHistogram, EMetricType::HIST_RATE>(std::move(labels), true); + } + + void TFakeMetricRegistry::Append(TInstant time, IMetricConsumer* consumer) const { + Y_UNUSED(time, consumer); + } + + const TLabels& TFakeMetricRegistry::CommonLabels() const noexcept { + return CommonLabels_; + } + + template <typename TMetric, EMetricType type, typename TLabelsType, typename... Args> + TMetric* TFakeMetricRegistry::Metric(TLabelsType&& labels, Args&&... args) { + { + TReadGuard g{Lock_}; + + auto it = Metrics_.find(labels); + if (it != Metrics_.end()) { + Y_ENSURE(it->second->Type() == type, "cannot create metric " << labels + << " with type " << MetricTypeToStr(type) + << ", because registry already has same metric with type " << MetricTypeToStr(it->second->Type())); + return static_cast<TMetric*>(it->second.Get()); + } + } + + { + TWriteGuard g{Lock_}; + + IMetricPtr metric = MakeHolder<TMetric>(std::forward<Args>(args)...); + + // decltype(Metrics_)::iterator breaks build on windows + THashMap<ILabelsPtr, IMetricPtr>::iterator it; + if constexpr (!std::is_convertible_v<TLabelsType, ILabelsPtr>) { + it = Metrics_.emplace(new TLabels{std::forward<TLabelsType>(labels)}, std::move(metric)).first; + } else { + it = Metrics_.emplace(std::forward<TLabelsType>(labels), std::move(metric)).first; + } + + return static_cast<TMetric*>(it->second.Get()); + } + } +} // namespace NMonitoring diff --git a/library/cpp/monlib/metrics/fake.h b/library/cpp/monlib/metrics/fake.h new file mode 100644 index 0000000000..61ba4f2bd4 --- /dev/null +++ b/library/cpp/monlib/metrics/fake.h @@ -0,0 +1,165 @@ +#pragma once + +#include "metric.h" +#include "metric_registry.h" + +namespace NMonitoring { + class TFakeMetricRegistry: public IMetricRegistry { + public: + TFakeMetricRegistry() noexcept + : CommonLabels_{0} + { + } + + explicit TFakeMetricRegistry(TLabels commonLabels) noexcept + : CommonLabels_{std::move(commonLabels)} + { + } + + IGauge* Gauge(ILabelsPtr labels) override; + ILazyGauge* LazyGauge(ILabelsPtr labels, std::function<double()> supplier) override; + IIntGauge* IntGauge(ILabelsPtr labels) override; + ILazyIntGauge* LazyIntGauge(ILabelsPtr labels, std::function<i64()> supplier) override; + ICounter* Counter(ILabelsPtr labels) override; + ILazyCounter* LazyCounter(ILabelsPtr labels, std::function<ui64()> supplier) override; + IRate* Rate(ILabelsPtr labels) override; + ILazyRate* LazyRate(ILabelsPtr labels, std::function<ui64()> supplier) override; + + IHistogram* HistogramCounter( + ILabelsPtr labels, + IHistogramCollectorPtr collector) override; + + IHistogram* HistogramRate( + ILabelsPtr labels, + IHistogramCollectorPtr collector) override; + void Accept(TInstant time, IMetricConsumer* consumer) const override; + void Append(TInstant time, IMetricConsumer* consumer) const override; + + const TLabels& CommonLabels() const noexcept override; + void RemoveMetric(const ILabels& labels) noexcept override; + + private: + TRWMutex Lock_; + THashMap<ILabelsPtr, IMetricPtr> Metrics_; + + template <typename TMetric, EMetricType type, typename TLabelsType, typename... Args> + TMetric* Metric(TLabelsType&& labels, Args&&... args); + + const TLabels CommonLabels_; + }; + + template <typename TBase> + struct TFakeAcceptor: TBase { + void Accept(TInstant time, IMetricConsumer* consumer) const override { + Y_UNUSED(time, consumer); + } + }; + + struct TFakeIntGauge final: public TFakeAcceptor<IIntGauge> { + i64 Add(i64 n) noexcept override { + Y_UNUSED(n); + return 0; + } + + void Set(i64 n) noexcept override { + Y_UNUSED(n); + } + + i64 Get() const noexcept override { + return 0; + } + }; + + struct TFakeLazyIntGauge final: public TFakeAcceptor<ILazyIntGauge> { + i64 Get() const noexcept override { + return 0; + } + }; + + struct TFakeRate final: public TFakeAcceptor<IRate> { + ui64 Add(ui64 n) noexcept override { + Y_UNUSED(n); + return 0; + } + + ui64 Get() const noexcept override { + return 0; + } + + void Reset() noexcept override { + } + }; + + struct TFakeLazyRate final: public TFakeAcceptor<ILazyRate> { + ui64 Get() const noexcept override { + return 0; + } + }; + + struct TFakeGauge final: public TFakeAcceptor<IGauge> { + double Add(double n) noexcept override { + Y_UNUSED(n); + return 0; + } + + void Set(double n) noexcept override { + Y_UNUSED(n); + } + + double Get() const noexcept override { + return 0; + } + }; + + struct TFakeLazyGauge final: public TFakeAcceptor<ILazyGauge> { + double Get() const noexcept override { + return 0; + } + }; + + struct TFakeHistogram final: public IHistogram { + TFakeHistogram(bool isRate = false) + : IHistogram{isRate} + { + } + + void Record(double value) override { + Y_UNUSED(value); + } + + void Record(double value, ui32 count) override { + Y_UNUSED(value, count); + } + + IHistogramSnapshotPtr TakeSnapshot() const override { + return nullptr; + } + + void Accept(TInstant time, IMetricConsumer* consumer) const override { + Y_UNUSED(time, consumer); + } + + void Reset() override { + } + }; + + struct TFakeCounter final: public TFakeAcceptor<ICounter> { + ui64 Add(ui64 n) noexcept override { + Y_UNUSED(n); + return 0; + } + + ui64 Get() const noexcept override { + return 0; + } + + void Reset() noexcept override { + } + }; + + struct TFakeLazyCounter final: public TFakeAcceptor<ILazyCounter> { + ui64 Get() const noexcept override { + return 0; + } + }; +} // namespace NMonitoring diff --git a/library/cpp/monlib/metrics/fake_ut.cpp b/library/cpp/monlib/metrics/fake_ut.cpp new file mode 100644 index 0000000000..c3368ca302 --- /dev/null +++ b/library/cpp/monlib/metrics/fake_ut.cpp @@ -0,0 +1,34 @@ +#include "fake.h" + +#include <library/cpp/testing/unittest/registar.h> + +#include <util/generic/ptr.h> + +using namespace NMonitoring; + +Y_UNIT_TEST_SUITE(TFakeTest) { + + Y_UNIT_TEST(CreateOnStack) { + TFakeMetricRegistry registry; + } + + Y_UNIT_TEST(CreateOnHeap) { + auto registry = MakeAtomicShared<TFakeMetricRegistry>(); + UNIT_ASSERT(registry); + } + + Y_UNIT_TEST(Gauge) { + TFakeMetricRegistry registry(TLabels{{"common", "label"}}); + + IGauge* g = registry.Gauge(MakeLabels({{"my", "gauge"}})); + UNIT_ASSERT(g); + + UNIT_ASSERT_DOUBLES_EQUAL(g->Get(), 0.0, 1E-6); + g->Set(12.34); + UNIT_ASSERT_DOUBLES_EQUAL(g->Get(), 0.0, 1E-6); // no changes + + double val = g->Add(1.2); + UNIT_ASSERT_DOUBLES_EQUAL(g->Get(), 0.0, 1E-6); + UNIT_ASSERT_DOUBLES_EQUAL(val, 0.0, 1E-6); + } +} diff --git a/library/cpp/monlib/metrics/fwd.h b/library/cpp/monlib/metrics/fwd.h new file mode 100644 index 0000000000..b4327ee5d5 --- /dev/null +++ b/library/cpp/monlib/metrics/fwd.h @@ -0,0 +1,40 @@ +#pragma once + +namespace NMonitoring { + + struct ILabel; + struct ILabels; + + class ICounter; + class IGauge; + class IHistogram; + class IIntGauge; + class ILazyCounter; + class ILazyGauge; + class ILazyIntGauge; + class ILazyRate; + class IMetric; + class IRate; + class TCounter; + class TGauge; + class THistogram; + class TIntGauge; + class TLazyCounter; + class TLazyGauge; + class TLazyIntGauge; + class TLazyRate; + class TRate; + + class IMetricSupplier; + class IMetricFactory; + class IMetricConsumer; + + class IMetricRegistry; + class TMetricRegistry; + + class IHistogramCollector; + class IHistogramSnapshot; + + class IExpMovingAverage; + +} // namespace NMonitoring diff --git a/library/cpp/monlib/metrics/histogram_collector.h b/library/cpp/monlib/metrics/histogram_collector.h new file mode 100644 index 0000000000..9f6bbbdfb7 --- /dev/null +++ b/library/cpp/monlib/metrics/histogram_collector.h @@ -0,0 +1,119 @@ +#pragma once + +#include "histogram_snapshot.h" + +namespace NMonitoring { + + /////////////////////////////////////////////////////////////////////////// + // IHistogramCollector + /////////////////////////////////////////////////////////////////////////// + class IHistogramCollector { + public: + virtual ~IHistogramCollector() = default; + + /** + * Store {@code count} times given {@code value} in this collector. + */ + virtual void Collect(double value, ui32 count) = 0; + + /** + * Store given {@code value} in this collector. + */ + void Collect(double value) { + Collect(value, 1); + } + + /** + * Add counts from snapshot into this collector + */ + void Collect(const IHistogramSnapshot& snapshot) { + for (ui32 i = 0; i < snapshot.Count(); i++) { + Collect(snapshot.UpperBound(i), snapshot.Value(i)); + } + } + + /** + * Reset collector values + */ + virtual void Reset() = 0; + + /** + * @return snapshot of the state of this collector. + */ + virtual IHistogramSnapshotPtr Snapshot() const = 0; + }; + + using IHistogramCollectorPtr = THolder<IHistogramCollector>; + + /////////////////////////////////////////////////////////////////////////// + // free functions + /////////////////////////////////////////////////////////////////////////// + + /** + * <p>Creates histogram collector for a set of buckets with arbitrary + * bounds.</p> + * + * <p>Defines {@code bounds.size() + 1} buckets with these boundaries for + * bucket i:</p> + * <ul> + * <li>Upper bound (0 <= i < N-1): {@code bounds[i]}</li> + * <li>Lower bound (1 <= i < N): {@code bounds[i - 1]}</li> + * </ul> + * + * <p>For example, if the list of boundaries is:</p> + * <pre>0, 1, 2, 5, 10, 20</pre> + * + * <p>then there are five finite buckets with the following ranges:</p> + * <pre>(-INF, 0], (0, 1], (1, 2], (2, 5], (5, 10], (10, 20], (20, +INF)</pre> + * + * @param bounds array of upper bounds for buckets. Values must be sorted. + */ + IHistogramCollectorPtr ExplicitHistogram(TBucketBounds bounds); + + /** + * <p>Creates histogram collector for a sequence of buckets that have a + * width proportional to the value of the lower bound.</p> + * + * <p>Defines {@code bucketsCount} buckets with these boundaries for bucket i:</p> + * <ul> + * <li>Upper bound (0 <= i < N-1): {@code scale * (base ^ i)}</li> + * <li>Lower bound (1 <= i < N): {@code scale * (base ^ (i - 1))}</li> + * </ul> + * + * <p>For example, if {@code bucketsCount=6}, {@code base=2}, and {@code scale=3}, + * then the bucket ranges are as follows:</p> + * + * <pre>(-INF, 3], (3, 6], (6, 12], (12, 24], (24, 48], (48, +INF)</pre> + * + * @param bucketsCount the total number of buckets. The value must be >= 2. + * @param base the exponential growth factor for the buckets width. + * The value must be >= 1.0. + * @param scale the linear scale for the buckets. The value must be >= 1.0. + */ + IHistogramCollectorPtr ExponentialHistogram( + ui32 bucketsCount, double base, double scale = 1.0); + + /** + * <p>Creates histogram collector for a sequence of buckets that all have + * the same width (except overflow and underflow).</p> + * + * <p>Defines {@code bucketsCount} buckets with these boundaries for bucket i:</p> + * <ul> + * <li>Upper bound (0 <= i < N-1): {@code startValue + bucketWidth * i}</li> + * <li>Lower bound (1 <= i < N): {@code startValue + bucketWidth * (i - 1)}</li> + * </ul> + * + * <p>For example, if {@code bucketsCount=6}, {@code startValue=5}, and + * {@code bucketWidth=15}, then the bucket ranges are as follows:</p> + * + * <pre>(-INF, 5], (5, 20], (20, 35], (35, 50], (50, 65], (65, +INF)</pre> + * + * @param bucketsCount the total number of buckets. The value must be >= 2. + * @param startValue the upper boundary of the first bucket. + * @param bucketWidth the difference between the upper and lower bounds for + * each bucket. The value must be >= 1. + */ + IHistogramCollectorPtr LinearHistogram( + ui32 bucketsCount, TBucketBound startValue, TBucketBound bucketWidth); + +} // namespace NMonitoring diff --git a/library/cpp/monlib/metrics/histogram_collector_explicit.cpp b/library/cpp/monlib/metrics/histogram_collector_explicit.cpp new file mode 100644 index 0000000000..377fc233ef --- /dev/null +++ b/library/cpp/monlib/metrics/histogram_collector_explicit.cpp @@ -0,0 +1,55 @@ +#include "histogram_collector.h" +#include "atomics_array.h" + +#include <util/generic/algorithm.h> +#include <util/generic/vector.h> +#include <util/generic/yexception.h> +#include <util/generic/ylimits.h> + +namespace NMonitoring { + + /////////////////////////////////////////////////////////////////////////// + // TExplicitHistogramCollector + /////////////////////////////////////////////////////////////////////////// + class TExplicitHistogramCollector: public IHistogramCollector { + public: + TExplicitHistogramCollector(TBucketBounds bounds) + : Values_(bounds.size() + 1) + , Bounds_(std::move(bounds)) + { + // add one bucket as +INF + Bounds_.push_back(Max<TBucketBound>()); + } + + void Collect(double value, ui32 count) override { + auto it = LowerBound(Bounds_.begin(), Bounds_.end(), value); + auto index = std::distance(Bounds_.begin(), it); + Values_.Add(index, count); + } + + void Reset() override { + Values_.Reset(); + } + + IHistogramSnapshotPtr Snapshot() const override { + auto values = Values_.Copy(); + return ExplicitHistogramSnapshot(Bounds_, values); + } + + private: + TAtomicsArray Values_; + TBucketBounds Bounds_; + }; + + IHistogramCollectorPtr ExplicitHistogram(TBucketBounds bounds) { + Y_ENSURE(bounds.size() >= 1, + "explicit histogram must contain at least one bucket"); + Y_ENSURE(bounds.size() <= HISTOGRAM_MAX_BUCKETS_COUNT, + "buckets count must be <=" << HISTOGRAM_MAX_BUCKETS_COUNT + << ", but got: " << bounds.size()); + Y_ENSURE(IsSorted(bounds.begin(), bounds.end()), + "bounds for explicit histogram must be sorted"); + + return MakeHolder<TExplicitHistogramCollector>(bounds); + } +} diff --git a/library/cpp/monlib/metrics/histogram_collector_exponential.cpp b/library/cpp/monlib/metrics/histogram_collector_exponential.cpp new file mode 100644 index 0000000000..2f8a50a5f9 --- /dev/null +++ b/library/cpp/monlib/metrics/histogram_collector_exponential.cpp @@ -0,0 +1,68 @@ +#include "histogram_collector.h" +#include "atomics_array.h" + +#include <util/generic/algorithm.h> +#include <util/generic/vector.h> +#include <util/generic/yexception.h> +#include <util/generic/ylimits.h> + +namespace NMonitoring { + /////////////////////////////////////////////////////////////////////////// + // TExponentialHistogramCollector + /////////////////////////////////////////////////////////////////////////// + class TExponentialHistogramCollector: public IHistogramCollector { + public: + TExponentialHistogramCollector(ui32 bucketsCount, double base, double scale) + : Values_(bucketsCount) + , Base_(base) + , Scale_(scale) + , MinValue_(scale) + , MaxValue_(scale * std::pow(base, bucketsCount - 2)) + , LogOfBase_(std::log(base)) + { + } + + void Collect(double value, ui32 count) override { + ui32 index = Max<ui32>(); + if (value <= MinValue_) { + index = 0; + } else if (value > MaxValue_) { + index = Values_.Size() - 1; + } else { + double logBase = std::log(value / Scale_) / LogOfBase_; + index = static_cast<ui32>(std::ceil(logBase)); + } + Values_.Add(index, count); + } + + void Reset() override { + Values_.Reset(); + } + + IHistogramSnapshotPtr Snapshot() const override { + return new TExponentialHistogramSnapshot(Base_, Scale_, Values_.Copy()); + } + + private: + TAtomicsArray Values_; + double Base_; + double Scale_; + TBucketBound MinValue_; + TBucketBound MaxValue_; + double LogOfBase_; + }; + + IHistogramCollectorPtr ExponentialHistogram( + ui32 bucketsCount, double base, double scale) + { + Y_ENSURE(bucketsCount >= 2, + "exponential histogram must contain at least two buckets"); + Y_ENSURE(bucketsCount <= HISTOGRAM_MAX_BUCKETS_COUNT, + "buckets count must be <=" << HISTOGRAM_MAX_BUCKETS_COUNT + << ", but got: " << bucketsCount); + Y_ENSURE(base > 1.0, "base must be > 1.0, got: " << base); + Y_ENSURE(scale >= 1.0, "scale must be >= 1.0, got: " << scale); + + return MakeHolder<TExponentialHistogramCollector>(bucketsCount, base, scale); + } +} diff --git a/library/cpp/monlib/metrics/histogram_collector_linear.cpp b/library/cpp/monlib/metrics/histogram_collector_linear.cpp new file mode 100644 index 0000000000..f8ad86f3a4 --- /dev/null +++ b/library/cpp/monlib/metrics/histogram_collector_linear.cpp @@ -0,0 +1,67 @@ +#include "histogram_collector.h" +#include "atomics_array.h" + +#include <util/generic/algorithm.h> +#include <util/generic/vector.h> +#include <util/generic/yexception.h> +#include <util/generic/ylimits.h> + +#include <cmath> + +namespace NMonitoring { + /////////////////////////////////////////////////////////////////////////// + // TLinearHistogramCollector + /////////////////////////////////////////////////////////////////////////// + class TLinearHistogramCollector: public IHistogramCollector { + public: + TLinearHistogramCollector( + ui32 bucketsCount, TBucketBound startValue, TBucketBound bucketWidth) + : Values_(bucketsCount) + , StartValue_(startValue) + , BucketWidth_(bucketWidth) + , MaxValue_(startValue + bucketWidth * (bucketsCount - 2)) + { + } + + void Collect(double value, ui32 count) override { + ui32 index = Max<ui32>(); + if (value <= StartValue_) { + index = 0; + } else if (value > MaxValue_) { + index = Values_.Size() - 1; + } else { + double buckets = (value - StartValue_) / BucketWidth_; + index = static_cast<ui32>(std::ceil(buckets)); + } + Values_.Add(index, count); + } + + void Reset() override { + Values_.Reset(); + } + + IHistogramSnapshotPtr Snapshot() const override { + return new TLinearHistogramSnapshot( + StartValue_, BucketWidth_, Values_.Copy()); + } + + private: + TAtomicsArray Values_; + TBucketBound StartValue_; + double BucketWidth_; + TBucketBound MaxValue_; + }; + + IHistogramCollectorPtr LinearHistogram( + ui32 bucketsCount, TBucketBound startValue, TBucketBound bucketWidth) + { + Y_ENSURE(bucketsCount >= 2, + "linear histogram must contain at least two buckets"); + Y_ENSURE(bucketsCount <= HISTOGRAM_MAX_BUCKETS_COUNT, + "buckets count must be <=" << HISTOGRAM_MAX_BUCKETS_COUNT + << ", but got: " << bucketsCount); + Y_ENSURE(bucketWidth >= 1, "bucketWidth must be >= 1, got: " << bucketWidth); + + return MakeHolder<TLinearHistogramCollector>(bucketsCount, startValue, bucketWidth); + } +} diff --git a/library/cpp/monlib/metrics/histogram_collector_ut.cpp b/library/cpp/monlib/metrics/histogram_collector_ut.cpp new file mode 100644 index 0000000000..1cf66507fa --- /dev/null +++ b/library/cpp/monlib/metrics/histogram_collector_ut.cpp @@ -0,0 +1,114 @@ +#include "histogram_collector.h" + +#include <library/cpp/testing/unittest/registar.h> + +using namespace NMonitoring; + +Y_UNIT_TEST_SUITE(THistogramCollectorTest) { + void CheckSnapshot( + const IHistogramSnapshot& s, + const TBucketBounds& bounds, + const TBucketValues& values) { + UNIT_ASSERT_VALUES_EQUAL(bounds.size(), values.size()); + UNIT_ASSERT_VALUES_EQUAL(bounds.size(), s.Count()); + + double epsilon = std::numeric_limits<double>::epsilon(); + for (ui32 i = 0; i < s.Count(); i++) { + UNIT_ASSERT_DOUBLES_EQUAL(bounds[i], s.UpperBound(i), epsilon); + UNIT_ASSERT_VALUES_EQUAL(values[i], s.Value(i)); + } + } + + Y_UNIT_TEST(Explicit) { + auto histogram = ExplicitHistogram({0, 1, 2, 5, 10, 20}); + histogram->Collect(-2); + histogram->Collect(-1); + histogram->Collect(0); + histogram->Collect(1); + histogram->Collect(20); + histogram->Collect(21); + histogram->Collect(1000); + + TBucketBounds expectedBounds = {0, 1, 2, 5, 10, 20, Max<TBucketBound>()}; + TBucketValues expectedValues = {3, 1, 0, 0, 0, 1, 2}; + + CheckSnapshot(*histogram->Snapshot(), expectedBounds, expectedValues); + } + + Y_UNIT_TEST(ExplicitWithFloadBounds) { + auto histogram = ExplicitHistogram({0.1, 0.5, 1, 1.7, 10}); + histogram->Collect(0.3, 2); + histogram->Collect(0.01); + histogram->Collect(0.9); + histogram->Collect(0.6); + histogram->Collect(1.1); + histogram->Collect(0.7); + histogram->Collect(2.71); + + TBucketBounds expectedBounds = {0.1, 0.5, 1, 1.7, 10, Max<TBucketBound>()}; + TBucketValues expectedValues = {1, 2, 3, 1, 1, 0}; + + CheckSnapshot(*histogram->Snapshot(), expectedBounds, expectedValues); + } + + Y_UNIT_TEST(Exponential) { + auto histogram = ExponentialHistogram(6, 2.0, 3.0); + histogram->Collect(-1); + histogram->Collect(0); + histogram->Collect(1); + histogram->Collect(3); + histogram->Collect(4); + histogram->Collect(5); + histogram->Collect(22); + histogram->Collect(23); + histogram->Collect(24); + histogram->Collect(50); + histogram->Collect(100); + histogram->Collect(1000); + + TBucketBounds expectedBounds = {3, 6, 12, 24, 48, Max<TBucketBound>()}; + TBucketValues expectedValues = {4, 2, 0, 3, 0, 3}; + + CheckSnapshot(*histogram->Snapshot(), expectedBounds, expectedValues); + } + + Y_UNIT_TEST(Linear) { + auto histogram = LinearHistogram(6, 5, 15); + histogram->Collect(-1); + histogram->Collect(0); + histogram->Collect(1); + histogram->Collect(4); + histogram->Collect(5); + histogram->Collect(6); + histogram->Collect(64); + histogram->Collect(65); + histogram->Collect(66); + histogram->Collect(100); + histogram->Collect(1000); + + TBucketBounds expectedBounds = {5, 20, 35, 50, 65, Max<TBucketBound>()}; + TBucketValues expectedValues = {5, 1, 0, 0, 2, 3}; + + CheckSnapshot(*histogram->Snapshot(), expectedBounds, expectedValues); + } + + Y_UNIT_TEST(SnapshotOutput) { + auto histogram = ExplicitHistogram({0, 1, 2, 5, 10, 20}); + histogram->Collect(-2); + histogram->Collect(-1); + histogram->Collect(0); + histogram->Collect(1); + histogram->Collect(20); + histogram->Collect(21); + histogram->Collect(1000); + + auto snapshot = histogram->Snapshot(); + + TStringStream ss; + ss << *snapshot; + + UNIT_ASSERT_STRINGS_EQUAL( + "{0: 3, 1: 1, 2: 0, 5: 0, 10: 0, 20: 1, inf: 2}", + ss.Str()); + } +} diff --git a/library/cpp/monlib/metrics/histogram_snapshot.cpp b/library/cpp/monlib/metrics/histogram_snapshot.cpp new file mode 100644 index 0000000000..75b5811546 --- /dev/null +++ b/library/cpp/monlib/metrics/histogram_snapshot.cpp @@ -0,0 +1,63 @@ +#include "histogram_snapshot.h" + +#include <util/stream/output.h> + +#include <iostream> + + +namespace NMonitoring { + + IHistogramSnapshotPtr ExplicitHistogramSnapshot(TConstArrayRef<TBucketBound> bounds, TConstArrayRef<TBucketValue> values) { + Y_ENSURE(bounds.size() == values.size(), + "mismatched sizes: bounds(" << bounds.size() << + ") != buckets(" << values.size() << ')'); + + auto snapshot = TExplicitHistogramSnapshot::New(bounds.size()); + + for (size_t i = 0; i != bounds.size(); ++i) { + (*snapshot)[i].first = bounds[i]; + (*snapshot)[i].second = values[i]; + } + + return snapshot; + } + +} // namespace NMonitoring + +namespace { + +template <typename TStream> +auto& Output(TStream& os, const NMonitoring::IHistogramSnapshot& hist) { + os << TStringBuf("{"); + + ui32 i = 0; + ui32 count = hist.Count(); + + if (count > 0) { + for (; i < count - 1; ++i) { + os << hist.UpperBound(i) << TStringBuf(": ") << hist.Value(i); + os << TStringBuf(", "); + } + + if (hist.UpperBound(i) == Max<NMonitoring::TBucketBound>()) { + os << TStringBuf("inf: ") << hist.Value(i); + } else { + os << hist.UpperBound(i) << TStringBuf(": ") << hist.Value(i); + } + } + + os << TStringBuf("}"); + + return os; +} + +} // namespace + +std::ostream& operator<<(std::ostream& os, const NMonitoring::IHistogramSnapshot& hist) { + return Output(os, hist); +} + +template <> +void Out<NMonitoring::IHistogramSnapshot>(IOutputStream& os, const NMonitoring::IHistogramSnapshot& hist) { + Output(os, hist); +} diff --git a/library/cpp/monlib/metrics/histogram_snapshot.h b/library/cpp/monlib/metrics/histogram_snapshot.h new file mode 100644 index 0000000000..e8acf6ac2b --- /dev/null +++ b/library/cpp/monlib/metrics/histogram_snapshot.h @@ -0,0 +1,210 @@ +#pragma once + +#include <util/generic/array_ref.h> +#include <util/generic/ptr.h> +#include <util/generic/vector.h> +#include <util/generic/yexception.h> + +#include <cmath> +#include <limits> + + +namespace NMonitoring { + + using TBucketBound = double; + using TBucketValue = ui64; + + using TBucketBounds = TVector<TBucketBound>; + using TBucketValues = TVector<TBucketValue>; + + constexpr ui32 HISTOGRAM_MAX_BUCKETS_COUNT = 51; + constexpr TBucketBound HISTOGRAM_INF_BOUND = std::numeric_limits<TBucketBound>::max(); + + /////////////////////////////////////////////////////////////////////////// + // IHistogramSnapshot + /////////////////////////////////////////////////////////////////////////// + class IHistogramSnapshot: public TAtomicRefCount<IHistogramSnapshot> { + public: + virtual ~IHistogramSnapshot() = default; + + /** + * @return buckets count. + */ + virtual ui32 Count() const = 0; + + /** + * @return upper bound for the bucket with particular index. + */ + virtual TBucketBound UpperBound(ui32 index) const = 0; + + /** + * @return value stored in the bucket with particular index. + */ + virtual TBucketValue Value(ui32 index) const = 0; + }; + + using IHistogramSnapshotPtr = TIntrusivePtr<IHistogramSnapshot>; + + /////////////////////////////////////////////////////////////////////////////// + // TLinearHistogramSnapshot + /////////////////////////////////////////////////////////////////////////////// + class TLinearHistogramSnapshot: public IHistogramSnapshot { + public: + TLinearHistogramSnapshot( + TBucketBound startValue, TBucketBound bucketWidth, TBucketValues values) + : StartValue_(startValue) + , BucketWidth_(bucketWidth) + , Values_(std::move(values)) + { + } + + ui32 Count() const override { + return static_cast<ui32>(Values_.size()); + } + + TBucketBound UpperBound(ui32 index) const override { + Y_ASSERT(index < Values_.size()); + if (index == Count() - 1) { + return Max<TBucketBound>(); + } + return StartValue_ + BucketWidth_ * index; + } + + TBucketValue Value(ui32 index) const override { + Y_ASSERT(index < Values_.size()); + return Values_[index]; + } + + ui64 MemorySizeBytes() { + return sizeof(*this) + Values_.capacity() * sizeof(decltype(Values_)::value_type); + } + + private: + TBucketBound StartValue_; + TBucketBound BucketWidth_; + TBucketValues Values_; + }; + + /////////////////////////////////////////////////////////////////////////// + // TExponentialHistogramSnapshot + /////////////////////////////////////////////////////////////////////////// + class TExponentialHistogramSnapshot: public IHistogramSnapshot { + public: + TExponentialHistogramSnapshot( + double base, double scale, TBucketValues values) + : Base_(base) + , Scale_(scale) + , Values_(std::move(values)) + { + } + + ui32 Count() const override { + return static_cast<ui32>(Values_.size()); + } + + TBucketBound UpperBound(ui32 index) const override { + Y_ASSERT(index < Values_.size()); + if (index == Values_.size() - 1) { + return Max<TBucketBound>(); + } + return std::round(Scale_ * std::pow(Base_, index)); + } + + TBucketValue Value(ui32 index) const override { + Y_ASSERT(index < Values_.size()); + return Values_[index]; + } + + ui64 MemorySizeBytes() { + return sizeof(*this) + Values_.capacity() * sizeof(decltype(Values_)::value_type); + } + + private: + double Base_; + double Scale_; + TBucketValues Values_; + }; + + using TBucket = std::pair<TBucketBound, TBucketValue>; + + /////////////////////////////////////////////////////////////////////// + // TExplicitHistogramSnapshot + /////////////////////////////////////////////////////////////////////// + // + // Memory layout (single contiguous block): + // + // +------+-----------+--------------+--------+--------+- -+--------+--------+ + // | vptr | RefsCount | BucketsCount | Bound1 | Value1 | ... | BoundN | ValueN | + // +------+-----------+--------------+--------+--------+- -+--------+--------+ + // + class TExplicitHistogramSnapshot: public IHistogramSnapshot, private TNonCopyable { + public: + static TIntrusivePtr<TExplicitHistogramSnapshot> New(ui32 bucketsCount) { + size_t bucketsSize = bucketsCount * sizeof(TBucket); + Y_ENSURE(bucketsCount <= HISTOGRAM_MAX_BUCKETS_COUNT, "Cannot allocate a histogram with " << bucketsCount + << " buckets. Bucket count is limited to " << HISTOGRAM_MAX_BUCKETS_COUNT); + + return new(bucketsSize) TExplicitHistogramSnapshot(bucketsCount); + } + + TBucket& operator[](ui32 index) noexcept { + return Bucket(index); + } + + ui32 Count() const override { + return BucketsCount_; + } + + TBucketBound UpperBound(ui32 index) const override { + return Bucket(index).first; + } + + TBucketValue Value(ui32 index) const override { + return Bucket(index).second; + } + + ui64 MemorySizeBytes() const { + return sizeof(*this) + BucketsCount_ * sizeof(TBucket); + } + + private: + explicit TExplicitHistogramSnapshot(ui32 bucketsCount) noexcept + : BucketsCount_(bucketsCount) + { + } + + static void* operator new(size_t size, size_t bucketsSize) { + return ::operator new(size + bucketsSize); + } + + static void operator delete(void* mem) { + ::operator delete(mem); + } + + static void operator delete(void* mem, size_t, size_t) { + // this operator can be called as paired for custom new operator + ::operator delete(mem); + } + + TBucket& Bucket(ui32 index) noexcept { + Y_VERIFY_DEBUG(index < BucketsCount_); + return *(reinterpret_cast<TBucket*>(this + 1) + index); + } + + const TBucket& Bucket(ui32 index) const noexcept { + Y_VERIFY_DEBUG(index < BucketsCount_); + return *(reinterpret_cast<const TBucket*>(this + 1) + index); + } + + private: + ui32 BucketsCount_; + }; + + static_assert(alignof(TExplicitHistogramSnapshot) == alignof(TBucket), + "mismatched alingments of THistogramSnapshot and TBucket"); + + IHistogramSnapshotPtr ExplicitHistogramSnapshot(TConstArrayRef<TBucketBound> bounds, TConstArrayRef<TBucketValue> values); + +} // namespace NMonitoring + +std::ostream& operator<<(std::ostream& os, const NMonitoring::IHistogramSnapshot& hist); diff --git a/library/cpp/monlib/metrics/labels.cpp b/library/cpp/monlib/metrics/labels.cpp new file mode 100644 index 0000000000..1eaadb7cba --- /dev/null +++ b/library/cpp/monlib/metrics/labels.cpp @@ -0,0 +1,82 @@ +#include "labels.h" + +#include <util/stream/output.h> +#include <util/string/split.h> + +static void OutputLabels(IOutputStream& out, const NMonitoring::ILabels& labels) { + size_t i = 0; + out << '{'; + for (const auto& label: labels) { + if (i++ > 0) { + out << TStringBuf(", "); + } + out << label; + } + out << '}'; +} + +template <> +void Out<NMonitoring::ILabelsPtr>(IOutputStream& out, const NMonitoring::ILabelsPtr& labels) { + OutputLabels(out, *labels); +} + +template <> +void Out<NMonitoring::ILabels>(IOutputStream& out, const NMonitoring::ILabels& labels) { + OutputLabels(out, labels); +} + +template <> +void Out<NMonitoring::ILabel>(IOutputStream& out, const NMonitoring::ILabel& labels) { + out << labels.Name() << "=" << labels.Value(); +} + +Y_MONLIB_DEFINE_LABELS_OUT(NMonitoring::TLabels); +Y_MONLIB_DEFINE_LABEL_OUT(NMonitoring::TLabel); + +namespace NMonitoring { + bool TryLoadLabelsFromString(TStringBuf sb, ILabels& labels) { + if (sb.Empty()) { + return false; + } + + if (!sb.StartsWith('{') || !sb.EndsWith('}')) { + return false; + } + + sb.Skip(1); + sb.Chop(1); + + if (sb.Empty()) { + return true; + } + + bool ok = true; + TVector<std::pair<TStringBuf, TStringBuf>> rawLabels; + StringSplitter(sb).SplitBySet(" ,").SkipEmpty().Consume([&] (TStringBuf label) { + TStringBuf key, value; + ok &= label.TrySplit('=', key, value); + + if (!ok) { + return; + } + + rawLabels.emplace_back(key, value); + }); + + if (!ok) { + return false; + } + + for (auto&& [k, v] : rawLabels) { + labels.Add(k, v); + } + + return true; + } + + bool TryLoadLabelsFromString(IInputStream& is, ILabels& labels) { + TString str = is.ReadAll(); + return TryLoadLabelsFromString(str, labels); + } + +} // namespace NMonitoring diff --git a/library/cpp/monlib/metrics/labels.h b/library/cpp/monlib/metrics/labels.h new file mode 100644 index 0000000000..63dc997c28 --- /dev/null +++ b/library/cpp/monlib/metrics/labels.h @@ -0,0 +1,483 @@ +#pragma once + +#include <util/digest/multi.h> +#include <util/digest/sequence.h> +#include <util/generic/algorithm.h> +#include <util/generic/maybe.h> +#include <util/generic/string.h> +#include <util/generic/vector.h> +#include <util/stream/output.h> +#include <util/string/builder.h> +#include <util/string/strip.h> + +#include <optional> +#include <type_traits> + +namespace NMonitoring { + struct ILabel { + virtual ~ILabel() = default; + + virtual TStringBuf Name() const noexcept = 0; + virtual TStringBuf Value() const noexcept = 0; + }; + + /////////////////////////////////////////////////////////////////////////// + // TLabel + /////////////////////////////////////////////////////////////////////////// + template <typename TStringBackend> + class TLabelImpl: public ILabel { + public: + using TStringType = TStringBackend; + + TLabelImpl() = default; + + inline TLabelImpl(TStringBuf name, TStringBuf value) + : Name_{name} + , Value_{value} + { + } + + inline bool operator==(const TLabelImpl& rhs) const noexcept { + return Name_ == rhs.Name_ && Value_ == rhs.Value_; + } + + inline bool operator!=(const TLabelImpl& rhs) const noexcept { + return !(*this == rhs); + } + + inline TStringBuf Name() const noexcept { + return Name_; + } + + inline TStringBuf Value() const noexcept { + return Value_; + } + + inline const TStringBackend& NameStr() const { + return Name_; + } + + inline const TStringBackend& ValueStr() const { + return Value_; + } + + inline size_t Hash() const noexcept { + return MultiHash(Name_, Value_); + } + + TStringBackend ToString() const { + TStringBackend buf = Name_; + buf += '='; + buf += Value_; + + return buf; + } + + static TLabelImpl FromString(TStringBuf str) { + TStringBuf name, value; + Y_ENSURE(str.TrySplit('=', name, value), + "invalid label string format: '" << str << '\''); + + TStringBuf nameStripped = StripString(name); + Y_ENSURE(!nameStripped.empty(), "label name cannot be empty"); + + TStringBuf valueStripped = StripString(value); + Y_ENSURE(!valueStripped.empty(), "label value cannot be empty"); + + return {nameStripped, valueStripped}; + } + + static bool TryFromString(TStringBuf str, TLabelImpl& label) { + TStringBuf name, value; + if (!str.TrySplit('=', name, value)) { + return false; + } + + TStringBuf nameStripped = StripString(name); + if (nameStripped.empty()) { + return false; + } + + TStringBuf valueStripped = StripString(value); + if (valueStripped.empty()) { + return false; + } + + label = {nameStripped, valueStripped}; + return true; + } + + private: + TStringBackend Name_; + TStringBackend Value_; + }; + + using TLabel = TLabelImpl<TString>; + + struct ILabels { + struct TIterator { + TIterator() = default; + TIterator(const ILabels* labels, size_t idx = 0) + : Labels_{labels} + , Idx_{idx} + { + } + + TIterator& operator++() noexcept { + Idx_++; + return *this; + } + + void operator+=(size_t i) noexcept { + Idx_ += i; + } + + bool operator==(const TIterator& other) const noexcept { + return Idx_ == other.Idx_; + } + + bool operator!=(const TIterator& other) const noexcept { + return !(*this == other); + } + + const ILabel* operator->() const noexcept { + Y_VERIFY_DEBUG(Labels_); + return Labels_->Get(Idx_); + } + + const ILabel& operator*() const noexcept { + Y_VERIFY_DEBUG(Labels_); + return *Labels_->Get(Idx_); + } + + + private: + const ILabels* Labels_{nullptr}; + size_t Idx_{0}; + }; + + virtual ~ILabels() = default; + + virtual bool Add(TStringBuf name, TStringBuf value) noexcept = 0; + virtual bool Add(const ILabel& label) noexcept { + return Add(label.Name(), label.Value()); + } + + virtual bool Has(TStringBuf name) const noexcept = 0; + + virtual size_t Size() const noexcept = 0; + virtual bool Empty() const noexcept = 0; + virtual void Clear() noexcept = 0; + + virtual size_t Hash() const noexcept = 0; + + virtual std::optional<const ILabel*> Get(TStringBuf name) const = 0; + + // NB: there's no guarantee that indices are preserved after any object modification + virtual const ILabel* Get(size_t idx) const = 0; + + TIterator begin() const { + return TIterator{this}; + } + + TIterator end() const { + return TIterator{this, Size()}; + } + }; + + bool TryLoadLabelsFromString(TStringBuf sb, ILabels& labels); + bool TryLoadLabelsFromString(IInputStream& is, ILabels& labels); + + /////////////////////////////////////////////////////////////////////////// + // TLabels + /////////////////////////////////////////////////////////////////////////// + template <typename TStringBackend> + class TLabelsImpl: public ILabels { + public: + using value_type = TLabelImpl<TStringBackend>; + + TLabelsImpl() = default; + + explicit TLabelsImpl(::NDetail::TReserveTag rt) + : Labels_(std::move(rt)) + {} + + explicit TLabelsImpl(size_t count) + : Labels_(count) + {} + + explicit TLabelsImpl(size_t count, const value_type& label) + : Labels_(count, label) + {} + + TLabelsImpl(std::initializer_list<value_type> il) + : Labels_(std::move(il)) + {} + + TLabelsImpl(const TLabelsImpl&) = default; + TLabelsImpl& operator=(const TLabelsImpl&) = default; + + TLabelsImpl(TLabelsImpl&&) noexcept = default; + TLabelsImpl& operator=(TLabelsImpl&&) noexcept = default; + + inline bool operator==(const TLabelsImpl& rhs) const { + return Labels_ == rhs.Labels_; + } + + inline bool operator!=(const TLabelsImpl& rhs) const { + return Labels_ != rhs.Labels_; + } + + bool Add(TStringBuf name, TStringBuf value) noexcept override { + if (Has(name)) { + return false; + } + + Labels_.emplace_back(name, value); + return true; + } + + using ILabels::Add; + + bool Has(TStringBuf name) const noexcept override { + auto it = FindIf(Labels_, [name](const TLabelImpl<TStringBackend>& label) { + return name == TStringBuf{label.Name()}; + }); + return it != Labels_.end(); + } + + bool Has(const TString& name) const noexcept { + auto it = FindIf(Labels_, [name](const TLabelImpl<TStringBackend>& label) { + return name == TStringBuf{label.Name()}; + }); + return it != Labels_.end(); + } + + // XXX for backward compatibility + TMaybe<TLabelImpl<TStringBackend>> Find(TStringBuf name) const { + auto it = FindIf(Labels_, [name](const TLabelImpl<TStringBackend>& label) { + return name == TStringBuf{label.Name()}; + }); + if (it == Labels_.end()) { + return Nothing(); + } + return *it; + } + + std::optional<const ILabel*> Get(TStringBuf name) const override { + auto it = FindIf(Labels_, [name] (auto&& l) { + return name == l.Name(); + }); + + if (it == Labels_.end()) { + return {}; + } + + return &*it; + } + + const ILabel* Get(size_t idx) const noexcept override { + return &(*this)[idx]; + } + + TMaybe<TLabelImpl<TStringBackend>> Extract(TStringBuf name) { + auto it = FindIf(Labels_, [name](const TLabelImpl<TStringBackend>& label) { + return name == TStringBuf{label.Name()}; + }); + if (it == Labels_.end()) { + return Nothing(); + } + TLabel tmp = *it; + Labels_.erase(it); + return tmp; + } + + void SortByName() { + std::sort(Labels_.begin(), Labels_.end(), [](const auto& lhs, const auto& rhs) { + return lhs.Name() < rhs.Name(); + }); + } + + inline size_t Hash() const noexcept override { + return TSimpleRangeHash()(Labels_); + } + + inline TLabel* Data() const noexcept { + return const_cast<TLabel*>(Labels_.data()); + } + + inline size_t Size() const noexcept override { + return Labels_.size(); + } + + inline bool Empty() const noexcept override { + return Labels_.empty(); + } + + inline void Clear() noexcept override { + Labels_.clear(); + }; + + TLabelImpl<TStringBackend>& front() { + return Labels_.front(); + } + + const TLabelImpl<TStringBackend>& front() const { + return Labels_.front(); + } + + TLabelImpl<TStringBackend>& back() { + return Labels_.back(); + } + + const TLabelImpl<TStringBackend>& back() const { + return Labels_.back(); + } + + TLabelImpl<TStringBackend>& operator[](size_t index) { + return Labels_[index]; + } + + const TLabelImpl<TStringBackend>& operator[](size_t index) const { + return Labels_[index]; + } + + TLabelImpl<TStringBackend>& at(size_t index) { + return Labels_.at(index); + } + + const TLabelImpl<TStringBackend>& at(size_t index) const { + return Labels_.at(index); + } + + size_t capacity() const { + return Labels_.capacity(); + } + + TLabelImpl<TStringBackend>* data() { + return Labels_.data(); + } + + const TLabelImpl<TStringBackend>* data() const { + return Labels_.data(); + } + + size_t size() const { + return Labels_.size(); + } + + bool empty() const { + return Labels_.empty(); + } + + void clear() { + Labels_.clear(); + } + + using ILabels::begin; + using ILabels::end; + + using iterator = ILabels::TIterator; + using const_iterator = iterator; + + protected: + TVector<TLabelImpl<TStringBackend>>& AsVector() { + return Labels_; + } + + const TVector<TLabelImpl<TStringBackend>>& AsVector() const { + return Labels_; + } + + private: + TVector<TLabelImpl<TStringBackend>> Labels_; + }; + + using TLabels = TLabelsImpl<TString>; + using ILabelsPtr = THolder<ILabels>; + + template <typename T> + ILabelsPtr MakeLabels() { + return MakeHolder<TLabelsImpl<T>>(); + } + + template <typename T> + ILabelsPtr MakeLabels(std::initializer_list<TLabelImpl<T>> labels) { + return MakeHolder<TLabelsImpl<T>>(labels); + } + + inline ILabelsPtr MakeLabels(TLabels&& labels) { + return MakeHolder<TLabels>(std::move(labels)); + } +} + +template<> +struct THash<NMonitoring::ILabelsPtr> { + size_t operator()(const NMonitoring::ILabelsPtr& labels) const noexcept { + return labels->Hash(); + } + + size_t operator()(const NMonitoring::ILabels& labels) const noexcept { + return labels.Hash(); + } +}; + +template<typename TStringBackend> +struct THash<NMonitoring::TLabelsImpl<TStringBackend>> { + size_t operator()(const NMonitoring::TLabelsImpl<TStringBackend>& labels) const noexcept { + return labels.Hash(); + } +}; + +template <typename TStringBackend> +struct THash<NMonitoring::TLabelImpl<TStringBackend>> { + inline size_t operator()(const NMonitoring::TLabelImpl<TStringBackend>& label) const noexcept { + return label.Hash(); + } +}; + +inline bool operator==(const NMonitoring::ILabels& lhs, const NMonitoring::ILabels& rhs) { + if (lhs.Size() != rhs.Size()) { + return false; + } + + for (auto&& l : lhs) { + auto rl = rhs.Get(l.Name()); + if (!rl || (*rl)->Value() != l.Value()) { + return false; + } + } + + return true; +} + +bool operator==(const NMonitoring::ILabelsPtr& lhs, const NMonitoring::ILabelsPtr& rhs) = delete; +bool operator==(const NMonitoring::ILabels& lhs, const NMonitoring::ILabelsPtr& rhs) = delete; +bool operator==(const NMonitoring::ILabelsPtr& lhs, const NMonitoring::ILabels& rhs) = delete; + +template<> +struct TEqualTo<NMonitoring::ILabelsPtr> { + bool operator()(const NMonitoring::ILabelsPtr& lhs, const NMonitoring::ILabelsPtr& rhs) { + return *lhs == *rhs; + } + + bool operator()(const NMonitoring::ILabelsPtr& lhs, const NMonitoring::ILabels& rhs) { + return *lhs == rhs; + } + + bool operator()(const NMonitoring::ILabels& lhs, const NMonitoring::ILabelsPtr& rhs) { + return lhs == *rhs; + } +}; + +#define Y_MONLIB_DEFINE_LABELS_OUT(T) \ +template <> \ +void Out<T>(IOutputStream& out, const T& labels) { \ + Out<NMonitoring::ILabels>(out, labels); \ +} + +#define Y_MONLIB_DEFINE_LABEL_OUT(T) \ +template <> \ +void Out<T>(IOutputStream& out, const T& label) { \ + Out<NMonitoring::ILabel>(out, label); \ +} diff --git a/library/cpp/monlib/metrics/labels_ut.cpp b/library/cpp/monlib/metrics/labels_ut.cpp new file mode 100644 index 0000000000..f0e4f532ab --- /dev/null +++ b/library/cpp/monlib/metrics/labels_ut.cpp @@ -0,0 +1,194 @@ +#include "labels.h" + +#include <library/cpp/testing/unittest/registar.h> + +using namespace NMonitoring; + +Y_UNIT_TEST_SUITE(TLabelsTest) { + TLabel pSolomon("project", "solomon"); + TLabel pKikimr("project", "kikimr"); + + Y_UNIT_TEST(Equals) { + UNIT_ASSERT(pSolomon == TLabel("project", "solomon")); + + UNIT_ASSERT_STRINGS_EQUAL(pSolomon.Name(), "project"); + UNIT_ASSERT_STRINGS_EQUAL(pSolomon.Value(), "solomon"); + + UNIT_ASSERT(pSolomon != pKikimr); + } + + Y_UNIT_TEST(ToString) { + UNIT_ASSERT_STRINGS_EQUAL(pSolomon.ToString(), "project=solomon"); + UNIT_ASSERT_STRINGS_EQUAL(pKikimr.ToString(), "project=kikimr"); + } + + Y_UNIT_TEST(FromString) { + auto pYql = TLabel::FromString("project=yql"); + UNIT_ASSERT_EQUAL(pYql, TLabel("project", "yql")); + + UNIT_ASSERT_EQUAL(TLabel::FromString("k=v"), TLabel("k", "v")); + UNIT_ASSERT_EQUAL(TLabel::FromString("k=v "), TLabel("k", "v")); + UNIT_ASSERT_EQUAL(TLabel::FromString("k= v"), TLabel("k", "v")); + UNIT_ASSERT_EQUAL(TLabel::FromString("k =v"), TLabel("k", "v")); + UNIT_ASSERT_EQUAL(TLabel::FromString(" k=v"), TLabel("k", "v")); + UNIT_ASSERT_EQUAL(TLabel::FromString(" k = v "), TLabel("k", "v")); + + UNIT_ASSERT_EXCEPTION_CONTAINS( + TLabel::FromString(""), + yexception, + "invalid label string format"); + + UNIT_ASSERT_EXCEPTION_CONTAINS( + TLabel::FromString("k v"), + yexception, + "invalid label string format"); + + UNIT_ASSERT_EXCEPTION_CONTAINS( + TLabel::FromString(" =v"), + yexception, + "label name cannot be empty"); + + UNIT_ASSERT_EXCEPTION_CONTAINS( + TLabel::FromString("k= "), + yexception, + "label value cannot be empty"); + } + + Y_UNIT_TEST(TryFromString) { + TLabel pYql; + UNIT_ASSERT(TLabel::TryFromString("project=yql", pYql)); + UNIT_ASSERT_EQUAL(pYql, TLabel("project", "yql")); + + { + TLabel label; + UNIT_ASSERT(TLabel::TryFromString("k=v", label)); + UNIT_ASSERT_EQUAL(label, TLabel("k", "v")); + } + { + TLabel label; + UNIT_ASSERT(TLabel::TryFromString("k=v ", label)); + UNIT_ASSERT_EQUAL(label, TLabel("k", "v")); + } + { + TLabel label; + UNIT_ASSERT(TLabel::TryFromString("k= v", label)); + UNIT_ASSERT_EQUAL(label, TLabel("k", "v")); + } + { + TLabel label; + UNIT_ASSERT(TLabel::TryFromString("k =v", label)); + UNIT_ASSERT_EQUAL(label, TLabel("k", "v")); + } + { + TLabel label; + UNIT_ASSERT(TLabel::TryFromString(" k=v", label)); + UNIT_ASSERT_EQUAL(label, TLabel("k", "v")); + } + { + TLabel label; + UNIT_ASSERT(TLabel::TryFromString(" k = v ", label)); + UNIT_ASSERT_EQUAL(label, TLabel("k", "v")); + } + } + + Y_UNIT_TEST(Labels) { + TLabels labels; + UNIT_ASSERT(labels.Add(TStringBuf("name1"), TStringBuf("value1"))); + UNIT_ASSERT(labels.Size() == 1); + UNIT_ASSERT(labels.Has(TStringBuf("name1"))); + { + auto l = labels.Find("name1"); + UNIT_ASSERT(l.Defined()); + UNIT_ASSERT_STRINGS_EQUAL(l->Name(), "name1"); + UNIT_ASSERT_STRINGS_EQUAL(l->Value(), "value1"); + } + { + auto l = labels.Find("name2"); + UNIT_ASSERT(!l.Defined()); + } + + // duplicated name + UNIT_ASSERT(!labels.Add(TStringBuf("name1"), TStringBuf("value2"))); + UNIT_ASSERT(labels.Size() == 1); + + UNIT_ASSERT(labels.Add(TStringBuf("name2"), TStringBuf("value2"))); + UNIT_ASSERT(labels.Size() == 2); + UNIT_ASSERT(labels.Has(TStringBuf("name2"))); + { + auto l = labels.Find("name2"); + UNIT_ASSERT(l.Defined()); + UNIT_ASSERT_STRINGS_EQUAL(l->Name(), "name2"); + UNIT_ASSERT_STRINGS_EQUAL(l->Value(), "value2"); + } + + UNIT_ASSERT_EQUAL(labels[0], TLabel("name1", "value1")); + UNIT_ASSERT_EQUAL(labels[1], TLabel("name2", "value2")); + + TVector<TLabel> labelsCopy; + for (auto&& label : labels) { + labelsCopy.emplace_back(label.Name(), label.Value()); + } + + UNIT_ASSERT_EQUAL(labelsCopy, TVector<TLabel>({ + {"name1", "value1"}, + {"name2", "value2"}, + })); + } + + Y_UNIT_TEST(Hash) { + TLabel label("name", "value"); + UNIT_ASSERT_EQUAL(ULL(2378153472115172159), label.Hash()); + + { + TLabels labels = {{"name", "value"}}; + UNIT_ASSERT_EQUAL(ULL(5420514431458887014), labels.Hash()); + } + { + TLabels labels = {{"name1", "value1"}, {"name2", "value2"}}; + UNIT_ASSERT_EQUAL(ULL(2226975250396609813), labels.Hash()); + } + } + + Y_UNIT_TEST(MakeEmptyLabels) { + { + auto labels = MakeLabels<TString>(); + UNIT_ASSERT(labels); + UNIT_ASSERT(labels->Empty()); + UNIT_ASSERT_VALUES_EQUAL(labels->Size(), 0); + } + { + auto labels = MakeLabels<TStringBuf>(); + UNIT_ASSERT(labels); + UNIT_ASSERT(labels->Empty()); + UNIT_ASSERT_VALUES_EQUAL(labels->Size(), 0); + } + } + + Y_UNIT_TEST(MakeLabelsFromInitializerList) { + auto labels = MakeLabels<TString>({{"my", "label"}}); + UNIT_ASSERT(labels); + UNIT_ASSERT(!labels->Empty()); + UNIT_ASSERT_VALUES_EQUAL(labels->Size(), 1); + + UNIT_ASSERT(labels->Has("my")); + + auto label = labels->Get("my"); + UNIT_ASSERT(label.has_value()); + UNIT_ASSERT_STRINGS_EQUAL((*label)->Name(), "my"); + UNIT_ASSERT_STRINGS_EQUAL((*label)->Value(), "label"); + } + + Y_UNIT_TEST(MakeLabelsFromOtherLabel) { + auto labels = MakeLabels({{"my", "label"}}); + UNIT_ASSERT(labels); + UNIT_ASSERT(!labels->Empty()); + UNIT_ASSERT_VALUES_EQUAL(labels->Size(), 1); + + UNIT_ASSERT(labels->Has("my")); + + auto label = labels->Get("my"); + UNIT_ASSERT(label.has_value()); + UNIT_ASSERT_STRINGS_EQUAL((*label)->Name(), "my"); + UNIT_ASSERT_STRINGS_EQUAL((*label)->Value(), "label"); + } +} diff --git a/library/cpp/monlib/metrics/log_histogram_collector.h b/library/cpp/monlib/metrics/log_histogram_collector.h new file mode 100644 index 0000000000..b81f84ebf3 --- /dev/null +++ b/library/cpp/monlib/metrics/log_histogram_collector.h @@ -0,0 +1,158 @@ +#pragma once + +#include "log_histogram_snapshot.h" + +#include <util/generic/algorithm.h> +#include <util/generic/utility.h> +#include <util/generic/yexception.h> + +#include <mutex> +#include <cmath> + +namespace NMonitoring { + + class TLogHistogramCollector { + public: + static constexpr int DEFAULT_START_POWER = -1; + + explicit TLogHistogramCollector(int startPower = DEFAULT_START_POWER) + : StartPower_(startPower) + , CountZero_(0u) + {} + + void Collect(TLogHistogramSnapshot* logHist) { + std::lock_guard guard(Mutex_); + Merge(logHist); + } + + bool Collect(double value) { + std::lock_guard guard(Mutex_); + return CollectDouble(value); + } + + TLogHistogramSnapshotPtr Snapshot() const { + std::lock_guard guard(Mutex_); + return MakeIntrusive<TLogHistogramSnapshot>(BASE, CountZero_, StartPower_, Buckets_); + } + + void AddZeros(ui64 zerosCount) noexcept { + std::lock_guard guard(Mutex_); + CountZero_ += zerosCount; + } + + private: + int StartPower_; + ui64 CountZero_; + TVector<double> Buckets_; + mutable std::mutex Mutex_; + + static constexpr size_t MAX_BUCKETS = LOG_HIST_MAX_BUCKETS; + static constexpr double BASE = 1.5; + + private: + int EstimateBucketIndex(double value) const { + return (int) (std::floor(std::log(value) / std::log(BASE)) - StartPower_); + } + + void CollectPositiveDouble(double value) { + ssize_t idx = std::floor(std::log(value) / std::log(BASE)) - StartPower_; + if (idx >= Buckets_.ysize()) { + idx = ExtendUp(idx); + } else if (idx <= 0) { + idx = Max<ssize_t>(0, ExtendDown(idx, 1)); + } + ++Buckets_[idx]; + } + + bool CollectDouble(double value) { + if (Y_UNLIKELY(std::isnan(value) || std::isinf(value))) { + return false; + } + if (value <= 0.0) { + ++CountZero_; + } else { + CollectPositiveDouble(value); + } + return true; + } + + void Merge(TLogHistogramSnapshot* logHist) { + CountZero_ += logHist->ZerosCount(); + const i32 firstIdxBeforeExtend = logHist->StartPower() - StartPower_; + const i32 lastIdxBeforeExtend = firstIdxBeforeExtend + logHist->Count() - 1; + if (firstIdxBeforeExtend > Max<i16>() || firstIdxBeforeExtend < Min<i16>()) { + ythrow yexception() << "i16 overflow on first index"; + } + if (lastIdxBeforeExtend > Max<i16>() || lastIdxBeforeExtend < Min<i16>()) { + ythrow yexception() << "i16 overflow on last index"; + } + i64 firstIdx = ExtendBounds(firstIdxBeforeExtend, lastIdxBeforeExtend, 0).first; + size_t toMerge = std::min<ui32>(std::max<i64>(-firstIdx, (i64) 0), logHist->Count()); + if (toMerge) { + for (size_t i = 0; i < toMerge; ++i) { + Buckets_[0] += logHist->Bucket(i); + } + firstIdx = 0; + } + for (size_t i = toMerge; i != logHist->Count(); ++i) { + Buckets_[firstIdx] += logHist->Bucket(i); + ++firstIdx; + } + } + + int ExtendUp(int expectedIndex) { + Y_VERIFY_DEBUG(expectedIndex >= (int) Buckets_.size()); + const size_t toAdd = expectedIndex - Buckets_.size() + 1; + const size_t newSize = Buckets_.size() + toAdd; + if (newSize <= MAX_BUCKETS) { + Buckets_.resize(newSize, 0.0); + return expectedIndex; + } + + const size_t toRemove = newSize - MAX_BUCKETS; + const size_t actualToRemove = std::min<size_t>(toRemove, Buckets_.size()); + if (actualToRemove > 0) { + const double firstWeight = std::accumulate(Buckets_.cbegin(), Buckets_.cbegin() + actualToRemove, 0.0); + Buckets_.erase(Buckets_.cbegin(), Buckets_.cbegin() + actualToRemove); + if (Buckets_.empty()) { + Buckets_.push_back(firstWeight); + } else { + Buckets_[0] = firstWeight; + } + } + Buckets_.resize(MAX_BUCKETS, 0.0); + StartPower_ += toRemove; + return expectedIndex - toRemove; + } + + int ExtendDown(int expectedIndex, int margin) { + Y_VERIFY_DEBUG(expectedIndex <= 0); + int toAdd = std::min<int>(MAX_BUCKETS - Buckets_.size(), margin - expectedIndex); + if (toAdd > 0) { + Buckets_.insert(Buckets_.begin(), toAdd, 0.0); + StartPower_ -= toAdd; + } + return expectedIndex + toAdd; + } + + std::pair<ssize_t, ssize_t> ExtendBounds(ssize_t startIdx, ssize_t endIdx, ui8 margin) { + ssize_t realEndIdx; + ssize_t realStartIdx; + if (endIdx >= Buckets_.ysize()) { + Buckets_.reserve(std::max<size_t>(std::min<ui32>(endIdx - startIdx + 1ul, MAX_BUCKETS), 0ul)); + realEndIdx = ExtendUp(endIdx); + startIdx += realEndIdx - endIdx; + } else { + realEndIdx = endIdx; + } + if (startIdx < 1) { + realStartIdx = ExtendDown(startIdx, margin); + realEndIdx += realStartIdx - startIdx; + } else { + realStartIdx = startIdx; + } + return std::make_pair(realStartIdx, realEndIdx); + } + }; + +} // namespace NMonitoring diff --git a/library/cpp/monlib/metrics/log_histogram_collector_ut.cpp b/library/cpp/monlib/metrics/log_histogram_collector_ut.cpp new file mode 100644 index 0000000000..ac9a3522ce --- /dev/null +++ b/library/cpp/monlib/metrics/log_histogram_collector_ut.cpp @@ -0,0 +1,38 @@ +#include "log_histogram_collector.h" + +#include <library/cpp/testing/unittest/registar.h> + +Y_UNIT_TEST_SUITE(LogHistogramCollector) { + + Y_UNIT_TEST(ExtendUpEmpty) { + NMonitoring::TLogHistogramCollector collector(-1); + collector.Collect(4.1944122207138854e+17); + auto s = collector.Snapshot(); + UNIT_ASSERT_EQUAL(s->ZerosCount(), 0); + UNIT_ASSERT_EQUAL(s->StartPower(), 1); + UNIT_ASSERT_EQUAL(s->Count(), 100); + UNIT_ASSERT_EQUAL(s->Bucket(s->Count() - 1), 1); + } + + Y_UNIT_TEST(ExtendUpNonEmpty) { + NMonitoring::TLogHistogramCollector collector(-1); + collector.Collect(0.0); + collector.Collect(1/(1.5*1.5*1.5)); + collector.Collect(1/1.5); + auto s = collector.Snapshot(); + + UNIT_ASSERT_EQUAL(s->ZerosCount(), 1); + UNIT_ASSERT_EQUAL(s->StartPower(), -4); + UNIT_ASSERT_EQUAL(s->Count(), 3); + UNIT_ASSERT_EQUAL(s->Bucket(1), 1); + UNIT_ASSERT_EQUAL(s->Bucket(2), 1); + + collector.Collect(4.1944122207138854e+17); + s = collector.Snapshot(); + UNIT_ASSERT_EQUAL(s->ZerosCount(), 1); + UNIT_ASSERT_EQUAL(s->StartPower(), 1); + UNIT_ASSERT_EQUAL(s->Count(), 100); + UNIT_ASSERT_EQUAL(s->Bucket(0), 2); + UNIT_ASSERT_EQUAL(s->Bucket(99), 1); + } +} diff --git a/library/cpp/monlib/metrics/log_histogram_snapshot.cpp b/library/cpp/monlib/metrics/log_histogram_snapshot.cpp new file mode 100644 index 0000000000..21cf2ca2bb --- /dev/null +++ b/library/cpp/monlib/metrics/log_histogram_snapshot.cpp @@ -0,0 +1,35 @@ +#include "log_histogram_snapshot.h" + +#include <util/stream/output.h> + +#include <iostream> + + +namespace { + +template <typename TStream> +auto& Output(TStream& o, const NMonitoring::TLogHistogramSnapshot& hist) { + o << TStringBuf("{"); + + for (auto i = 0u; i < hist.Count(); ++i) { + o << hist.UpperBound(i) << TStringBuf(": ") << hist.Bucket(i); + o << TStringBuf(", "); + } + + o << TStringBuf("zeros: ") << hist.ZerosCount(); + + o << TStringBuf("}"); + + return o; +} + +} // namespace + +std::ostream& operator<<(std::ostream& os, const NMonitoring::TLogHistogramSnapshot& hist) { + return Output(os, hist); +} + +template <> +void Out<NMonitoring::TLogHistogramSnapshot>(IOutputStream& os, const NMonitoring::TLogHistogramSnapshot& hist) { + Output(os, hist); +} diff --git a/library/cpp/monlib/metrics/log_histogram_snapshot.h b/library/cpp/monlib/metrics/log_histogram_snapshot.h new file mode 100644 index 0000000000..7673b43751 --- /dev/null +++ b/library/cpp/monlib/metrics/log_histogram_snapshot.h @@ -0,0 +1,71 @@ +#pragma once + +#include <util/generic/ptr.h> +#include <util/generic/vector.h> + +#include <cmath> + +namespace NMonitoring { + + constexpr ui32 LOG_HIST_MAX_BUCKETS = 100; + + class TLogHistogramSnapshot: public TAtomicRefCount<TLogHistogramSnapshot> { + public: + TLogHistogramSnapshot(double base, ui64 zerosCount, int startPower, TVector<double> buckets) + : Base_(base) + , ZerosCount_(zerosCount) + , StartPower_(startPower) + , Buckets_(std::move(buckets)) { + } + + /** + * @return buckets count. + */ + ui32 Count() const noexcept { + return Buckets_.size(); + } + + /** + * @return upper bound for the bucket with particular index. + */ + double UpperBound(int index) const noexcept { + return std::pow(Base_, StartPower_ + index); + } + + /** + * @return value stored in the bucket with particular index. + */ + double Bucket(ui32 index) const noexcept { + return Buckets_[index]; + } + + /** + * @return nonpositive values count + */ + ui64 ZerosCount() const noexcept { + return ZerosCount_; + } + + double Base() const noexcept { + return Base_; + } + + int StartPower() const noexcept { + return StartPower_; + } + + ui64 MemorySizeBytes() const noexcept { + return sizeof(*this) + Buckets_.capacity() * sizeof(double); + } + + private: + double Base_; + ui64 ZerosCount_; + int StartPower_; + TVector<double> Buckets_; + }; + + using TLogHistogramSnapshotPtr = TIntrusivePtr<TLogHistogramSnapshot>; +} + +std::ostream& operator<<(std::ostream& os, const NMonitoring::TLogHistogramSnapshot& hist); diff --git a/library/cpp/monlib/metrics/metric.h b/library/cpp/monlib/metrics/metric.h new file mode 100644 index 0000000000..b8ce12d753 --- /dev/null +++ b/library/cpp/monlib/metrics/metric.h @@ -0,0 +1,388 @@ +#pragma once + +#include "metric_consumer.h" + +#include <util/datetime/base.h> +#include <util/generic/ptr.h> + +namespace NMonitoring { + /////////////////////////////////////////////////////////////////////////////// + // IMetric + /////////////////////////////////////////////////////////////////////////////// + class IMetric { + public: + virtual ~IMetric() = default; + + virtual EMetricType Type() const noexcept = 0; + virtual void Accept(TInstant time, IMetricConsumer* consumer) const = 0; + }; + + using IMetricPtr = THolder<IMetric>; + + class IGauge: public IMetric { + public: + EMetricType Type() const noexcept final { + return EMetricType::GAUGE; + } + + virtual double Add(double n) noexcept = 0; + virtual void Set(double n) noexcept = 0; + virtual double Get() const noexcept = 0; + virtual void Reset() noexcept { + Set(0); + } + }; + + class ILazyGauge: public IMetric { + public: + EMetricType Type() const noexcept final { + return EMetricType::GAUGE; + } + virtual double Get() const noexcept = 0; + }; + + class IIntGauge: public IMetric { + public: + EMetricType Type() const noexcept final { + return EMetricType::IGAUGE; + } + + virtual i64 Add(i64 n) noexcept = 0; + virtual i64 Inc() noexcept { + return Add(1); + } + + virtual i64 Dec() noexcept { + return Add(-1); + } + + virtual void Set(i64 value) noexcept = 0; + virtual i64 Get() const noexcept = 0; + virtual void Reset() noexcept { + Set(0); + } + }; + + class ILazyIntGauge: public IMetric { + public: + EMetricType Type() const noexcept final { + return EMetricType::IGAUGE; + } + + virtual i64 Get() const noexcept = 0; + }; + + class ICounter: public IMetric { + public: + EMetricType Type() const noexcept final { + return EMetricType::COUNTER; + } + + virtual ui64 Inc() noexcept { + return Add(1); + } + + virtual ui64 Add(ui64 n) noexcept = 0; + virtual ui64 Get() const noexcept = 0; + virtual void Reset() noexcept = 0; + }; + + class ILazyCounter: public IMetric { + public: + EMetricType Type() const noexcept final { + return EMetricType::COUNTER; + } + + virtual ui64 Get() const noexcept = 0; + }; + + class IRate: public IMetric { + public: + EMetricType Type() const noexcept final { + return EMetricType::RATE; + } + + virtual ui64 Inc() noexcept { + return Add(1); + } + + virtual ui64 Add(ui64 n) noexcept = 0; + virtual ui64 Get() const noexcept = 0; + virtual void Reset() noexcept = 0; + }; + + class ILazyRate: public IMetric { + public: + EMetricType Type() const noexcept final { + return EMetricType::RATE; + } + + virtual ui64 Get() const noexcept = 0; + }; + + class IHistogram: public IMetric { + public: + explicit IHistogram(bool isRate) + : IsRate_{isRate} + { + } + + EMetricType Type() const noexcept final { + return IsRate_ ? EMetricType::HIST_RATE : EMetricType::HIST; + } + + virtual void Record(double value) = 0; + virtual void Record(double value, ui32 count) = 0; + virtual IHistogramSnapshotPtr TakeSnapshot() const = 0; + virtual void Reset() = 0; + + protected: + const bool IsRate_; + }; + + /////////////////////////////////////////////////////////////////////////////// + // TGauge + /////////////////////////////////////////////////////////////////////////////// + class TGauge final: public IGauge { + public: + explicit TGauge(double value = 0.0) { + Set(value); + } + + double Add(double n) noexcept override { + double newValue; + double oldValue = Get(); + + do { + newValue = oldValue + n; + } while (!Value_.compare_exchange_weak(oldValue, newValue, std::memory_order_release, std::memory_order_consume)); + + return newValue; + } + + void Set(double n) noexcept override { + Value_.store(n, std::memory_order_relaxed); + } + + double Get() const noexcept override { + return Value_.load(std::memory_order_relaxed); + } + + void Accept(TInstant time, IMetricConsumer* consumer) const override { + consumer->OnDouble(time, Get()); + } + + private: + std::atomic<double> Value_; + }; + + /////////////////////////////////////////////////////////////////////////////// + // TLazyGauge + /////////////////////////////////////////////////////////////////////////////// + class TLazyGauge final: public ILazyGauge { + public: + explicit TLazyGauge(std::function<double()> supplier) + : Supplier_(std::move(supplier)) + { + } + + double Get() const noexcept override { + return Supplier_(); + } + + void Accept(TInstant time, IMetricConsumer* consumer) const override { + consumer->OnDouble(time, Get()); + } + + private: + std::function<double()> Supplier_; + }; + + /////////////////////////////////////////////////////////////////////////////// + // TIntGauge + /////////////////////////////////////////////////////////////////////////////// + class TIntGauge final: public IIntGauge { + public: + explicit TIntGauge(i64 value = 0) { + Set(value); + } + + i64 Add(i64 n) noexcept override { + return Value_.fetch_add(n, std::memory_order_relaxed) + n; + } + + void Set(i64 value) noexcept override { + Value_.store(value, std::memory_order_relaxed); + } + + i64 Get() const noexcept override { + return Value_.load(std::memory_order_relaxed); + } + + void Accept(TInstant time, IMetricConsumer* consumer) const override { + consumer->OnInt64(time, Get()); + } + + private: + std::atomic_int64_t Value_; + }; + + /////////////////////////////////////////////////////////////////////////////// + // TLazyIntGauge + /////////////////////////////////////////////////////////////////////////////// + class TLazyIntGauge final: public ILazyIntGauge { + public: + explicit TLazyIntGauge(std::function<i64()> supplier) + : Supplier_(std::move(supplier)) + { + } + + i64 Get() const noexcept override { + return Supplier_(); + } + + void Accept(TInstant time, IMetricConsumer* consumer) const override { + consumer->OnInt64(time, Get()); + } + + private: + std::function<i64()> Supplier_; + }; + + /////////////////////////////////////////////////////////////////////////////// + // TCounter + /////////////////////////////////////////////////////////////////////////////// + class TCounter final: public ICounter { + public: + explicit TCounter(ui64 value = 0) { + Value_.store(value, std::memory_order_relaxed); + } + + ui64 Add(ui64 n) noexcept override { + return Value_.fetch_add(n, std::memory_order_relaxed) + n; + } + + ui64 Get() const noexcept override { + return Value_.load(std::memory_order_relaxed); + } + + void Reset() noexcept override { + Value_.store(0, std::memory_order_relaxed); + } + + void Accept(TInstant time, IMetricConsumer* consumer) const override { + consumer->OnUint64(time, Get()); + } + + private: + std::atomic_uint64_t Value_; + }; + + /////////////////////////////////////////////////////////////////////////////// + // TLazyCounter + /////////////////////////////////////////////////////////////////////////////// + class TLazyCounter final: public ILazyCounter { + public: + explicit TLazyCounter(std::function<ui64()> supplier) + : Supplier_(std::move(supplier)) + { + } + + ui64 Get() const noexcept override { + return Supplier_(); + } + + void Accept(TInstant time, IMetricConsumer* consumer) const override { + consumer->OnUint64(time, Get()); + } + + private: + std::function<ui64()> Supplier_; + }; + + /////////////////////////////////////////////////////////////////////////////// + // TRate + /////////////////////////////////////////////////////////////////////////////// + class TRate final: public IRate { + public: + explicit TRate(ui64 value = 0) { + Value_.store(value, std::memory_order_relaxed); + } + + ui64 Add(ui64 n) noexcept override { + return Value_.fetch_add(n, std::memory_order_relaxed) + n; + } + + ui64 Get() const noexcept override { + return Value_.load(std::memory_order_relaxed); + } + + void Reset() noexcept override { + Value_.store(0, std::memory_order_relaxed); + } + + void Accept(TInstant time, IMetricConsumer* consumer) const override { + consumer->OnUint64(time, Get()); + } + + private: + std::atomic_uint64_t Value_; + }; + + /////////////////////////////////////////////////////////////////////////////// + // TLazyRate + /////////////////////////////////////////////////////////////////////////////// + class TLazyRate final: public ILazyRate { + public: + explicit TLazyRate(std::function<ui64()> supplier) + : Supplier_(std::move(supplier)) + { + } + + ui64 Get() const noexcept override { + return Supplier_(); + } + + void Accept(TInstant time, IMetricConsumer* consumer) const override { + consumer->OnUint64(time, Get()); + } + + private: + std::function<ui64()> Supplier_; + }; + + /////////////////////////////////////////////////////////////////////////////// + // THistogram + /////////////////////////////////////////////////////////////////////////////// + class THistogram final: public IHistogram { + public: + THistogram(IHistogramCollectorPtr collector, bool isRate) + : IHistogram(isRate) + , Collector_(std::move(collector)) + { + } + + void Record(double value) override { + Collector_->Collect(value); + } + + void Record(double value, ui32 count) override { + Collector_->Collect(value, count); + } + + void Accept(TInstant time, IMetricConsumer* consumer) const override { + consumer->OnHistogram(time, TakeSnapshot()); + } + + IHistogramSnapshotPtr TakeSnapshot() const override { + return Collector_->Snapshot(); + } + + void Reset() override { + Collector_->Reset(); + } + + private: + IHistogramCollectorPtr Collector_; + }; +} diff --git a/library/cpp/monlib/metrics/metric_consumer.cpp b/library/cpp/monlib/metrics/metric_consumer.cpp new file mode 100644 index 0000000000..121ee368f0 --- /dev/null +++ b/library/cpp/monlib/metrics/metric_consumer.cpp @@ -0,0 +1,15 @@ +#include "metric_consumer.h" + +#include <util/system/yassert.h> + +namespace NMonitoring { + void IMetricConsumer::OnLabel(ui32 name, ui32 value) { + Y_UNUSED(name, value); + Y_ENSURE(false, "Not implemented"); + } + + std::pair<ui32, ui32> IMetricConsumer::PrepareLabel(TStringBuf name, TStringBuf value) { + Y_UNUSED(name, value); + Y_ENSURE(false, "Not implemented"); + } +} diff --git a/library/cpp/monlib/metrics/metric_consumer.h b/library/cpp/monlib/metrics/metric_consumer.h new file mode 100644 index 0000000000..f7a727585a --- /dev/null +++ b/library/cpp/monlib/metrics/metric_consumer.h @@ -0,0 +1,40 @@ +#pragma once + +#include "metric_type.h" +#include "histogram_collector.h" +#include "summary_collector.h" +#include "log_histogram_snapshot.h" + +class TInstant; + +namespace NMonitoring { + class IMetricConsumer { + public: + virtual ~IMetricConsumer() = default; + + virtual void OnStreamBegin() = 0; + virtual void OnStreamEnd() = 0; + + virtual void OnCommonTime(TInstant time) = 0; + + virtual void OnMetricBegin(EMetricType type) = 0; + virtual void OnMetricEnd() = 0; + + virtual void OnLabelsBegin() = 0; + virtual void OnLabelsEnd() = 0; + virtual void OnLabel(TStringBuf name, TStringBuf value) = 0; + virtual void OnLabel(ui32 name, ui32 value); + virtual std::pair<ui32, ui32> PrepareLabel(TStringBuf name, TStringBuf value); + + virtual void OnDouble(TInstant time, double value) = 0; + virtual void OnInt64(TInstant time, i64 value) = 0; + virtual void OnUint64(TInstant time, ui64 value) = 0; + + virtual void OnHistogram(TInstant time, IHistogramSnapshotPtr snapshot) = 0; + virtual void OnLogHistogram(TInstant time, TLogHistogramSnapshotPtr snapshot) = 0; + virtual void OnSummaryDouble(TInstant time, ISummaryDoubleSnapshotPtr snapshot) = 0; + }; + + using IMetricConsumerPtr = THolder<IMetricConsumer>; + +} diff --git a/library/cpp/monlib/metrics/metric_registry.cpp b/library/cpp/monlib/metrics/metric_registry.cpp new file mode 100644 index 0000000000..e46141ccde --- /dev/null +++ b/library/cpp/monlib/metrics/metric_registry.cpp @@ -0,0 +1,225 @@ +#include "metric_registry.h" + +#include <memory> + +namespace NMonitoring { + namespace { + void ConsumeLabels(IMetricConsumer* consumer, const ILabels& labels) { + for (auto&& label: labels) { + consumer->OnLabel(label.Name(), label.Value()); + } + } + + template <typename TLabelsConsumer> + void ConsumeMetric(TInstant time, IMetricConsumer* consumer, IMetric* metric, TLabelsConsumer&& labelsConsumer) { + consumer->OnMetricBegin(metric->Type()); + + // (1) add labels + consumer->OnLabelsBegin(); + labelsConsumer(); + consumer->OnLabelsEnd(); + + // (2) add time and value + metric->Accept(time, consumer); + consumer->OnMetricEnd(); + } + } + + void WriteLabels(IMetricConsumer* consumer, const ILabels& labels) { + consumer->OnLabelsBegin(); + ConsumeLabels(consumer, labels); + consumer->OnLabelsEnd(); + } + + TMetricRegistry::TMetricRegistry() = default; + TMetricRegistry::~TMetricRegistry() = default; + + TMetricRegistry::TMetricRegistry(const TLabels& commonLabels) + : TMetricRegistry{} + { + CommonLabels_ = commonLabels; + } + + TMetricRegistry* TMetricRegistry::Instance() { + return Singleton<TMetricRegistry>(); + } + + TGauge* TMetricRegistry::Gauge(TLabels labels) { + return Metric<TGauge, EMetricType::GAUGE>(std::move(labels)); + } + + TGauge* TMetricRegistry::Gauge(ILabelsPtr labels) { + return Metric<TGauge, EMetricType::GAUGE>(std::move(labels)); + } + + TLazyGauge* TMetricRegistry::LazyGauge(TLabels labels, std::function<double()> supplier) { + return Metric<TLazyGauge, EMetricType::GAUGE>(std::move(labels), std::move(supplier)); + } + + TLazyGauge* TMetricRegistry::LazyGauge(ILabelsPtr labels, std::function<double()> supplier) { + return Metric<TLazyGauge, EMetricType::GAUGE>(std::move(labels), std::move(supplier)); + } + + TIntGauge* TMetricRegistry::IntGauge(TLabels labels) { + return Metric<TIntGauge, EMetricType::IGAUGE>(std::move(labels)); + } + + TIntGauge* TMetricRegistry::IntGauge(ILabelsPtr labels) { + return Metric<TIntGauge, EMetricType::IGAUGE>(std::move(labels)); + } + + TLazyIntGauge* TMetricRegistry::LazyIntGauge(TLabels labels, std::function<i64()> supplier) { + return Metric<TLazyIntGauge, EMetricType::GAUGE>(std::move(labels), std::move(supplier)); + } + + TLazyIntGauge* TMetricRegistry::LazyIntGauge(ILabelsPtr labels, std::function<i64()> supplier) { + return Metric<TLazyIntGauge, EMetricType::GAUGE>(std::move(labels), std::move(supplier)); + } + + TCounter* TMetricRegistry::Counter(TLabels labels) { + return Metric<TCounter, EMetricType::COUNTER>(std::move(labels)); + } + + TCounter* TMetricRegistry::Counter(ILabelsPtr labels) { + return Metric<TCounter, EMetricType::COUNTER>(std::move(labels)); + } + + TLazyCounter* TMetricRegistry::LazyCounter(TLabels labels, std::function<ui64()> supplier) { + return Metric<TLazyCounter, EMetricType::COUNTER>(std::move(labels), std::move(supplier)); + } + + TLazyCounter* TMetricRegistry::LazyCounter(ILabelsPtr labels, std::function<ui64()> supplier) { + return Metric<TLazyCounter, EMetricType::COUNTER>(std::move(labels), std::move(supplier)); + } + + TRate* TMetricRegistry::Rate(TLabels labels) { + return Metric<TRate, EMetricType::RATE>(std::move(labels)); + } + + TRate* TMetricRegistry::Rate(ILabelsPtr labels) { + return Metric<TRate, EMetricType::RATE>(std::move(labels)); + } + + TLazyRate* TMetricRegistry::LazyRate(TLabels labels, std::function<ui64()> supplier) { + return Metric<TLazyRate, EMetricType::RATE>(std::move(labels), std::move(supplier)); + } + + TLazyRate* TMetricRegistry::LazyRate(ILabelsPtr labels, std::function<ui64()> supplier) { + return Metric<TLazyRate, EMetricType::RATE>(std::move(labels), std::move(supplier)); + } + + THistogram* TMetricRegistry::HistogramCounter(TLabels labels, IHistogramCollectorPtr collector) { + return Metric<THistogram, EMetricType::HIST>(std::move(labels), std::move(collector), false); + } + + THistogram* TMetricRegistry::HistogramCounter(ILabelsPtr labels, IHistogramCollectorPtr collector) { + return Metric<THistogram, EMetricType::HIST>(std::move(labels), std::move(collector), false); + } + + THistogram* TMetricRegistry::HistogramRate(TLabels labels, IHistogramCollectorPtr collector) { + return Metric<THistogram, EMetricType::HIST_RATE>(std::move(labels), std::move(collector), true); + } + + THistogram* TMetricRegistry::HistogramRate(ILabelsPtr labels, IHistogramCollectorPtr collector) { + return Metric<THistogram, EMetricType::HIST_RATE>(std::move(labels), std::move(collector), true); + } + + void TMetricRegistry::Reset() { + TWriteGuard g{Lock_}; + for (auto& [label, metric] : Metrics_) { + switch (metric->Type()) { + case EMetricType::GAUGE: + static_cast<TGauge*>(metric.Get())->Set(.0); + break; + case EMetricType::IGAUGE: + static_cast<TIntGauge*>(metric.Get())->Set(0); + break; + case EMetricType::COUNTER: + static_cast<TCounter*>(metric.Get())->Reset(); + break; + case EMetricType::RATE: + static_cast<TRate*>(metric.Get())->Reset(); + break; + case EMetricType::HIST: + case EMetricType::HIST_RATE: + static_cast<THistogram*>(metric.Get())->Reset(); + break; + case EMetricType::UNKNOWN: + case EMetricType::DSUMMARY: + case EMetricType::LOGHIST: + break; + } + } + } + + template <typename TMetric, EMetricType type, typename TLabelsType, typename... Args> + TMetric* TMetricRegistry::Metric(TLabelsType&& labels, Args&&... args) { + { + TReadGuard g{Lock_}; + + auto it = Metrics_.find(labels); + if (it != Metrics_.end()) { + Y_ENSURE(it->second->Type() == type, "cannot create metric " << labels + << " with type " << MetricTypeToStr(type) + << ", because registry already has same metric with type " << MetricTypeToStr(it->second->Type())); + return static_cast<TMetric*>(it->second.Get()); + } + } + + { + IMetricPtr metric = MakeHolder<TMetric>(std::forward<Args>(args)...); + + TWriteGuard g{Lock_}; + // decltype(Metrics_)::iterator breaks build on windows + THashMap<ILabelsPtr, IMetricPtr>::iterator it; + if constexpr (!std::is_convertible_v<TLabelsType, ILabelsPtr>) { + it = Metrics_.emplace(new TLabels{std::forward<TLabelsType>(labels)}, std::move(metric)).first; + } else { + it = Metrics_.emplace(std::forward<TLabelsType>(labels), std::move(metric)).first; + } + + return static_cast<TMetric*>(it->second.Get()); + } + } + + void TMetricRegistry::RemoveMetric(const ILabels& labels) noexcept { + TWriteGuard g{Lock_}; + Metrics_.erase(labels); + } + + void TMetricRegistry::Accept(TInstant time, IMetricConsumer* consumer) const { + consumer->OnStreamBegin(); + + if (!CommonLabels_.Empty()) { + consumer->OnLabelsBegin(); + ConsumeLabels(consumer, CommonLabels_); + consumer->OnLabelsEnd(); + } + + { + TReadGuard g{Lock_}; + for (const auto& it: Metrics_) { + ILabels* labels = it.first.Get(); + IMetric* metric = it.second.Get(); + ConsumeMetric(time, consumer, metric, [&]() { + ConsumeLabels(consumer, *labels); + }); + } + } + + consumer->OnStreamEnd(); + } + + void TMetricRegistry::Append(TInstant time, IMetricConsumer* consumer) const { + TReadGuard g{Lock_}; + + for (const auto& it: Metrics_) { + ILabels* labels = it.first.Get(); + IMetric* metric = it.second.Get(); + ConsumeMetric(time, consumer, metric, [&]() { + ConsumeLabels(consumer, CommonLabels_); + ConsumeLabels(consumer, *labels); + }); + } + } +} diff --git a/library/cpp/monlib/metrics/metric_registry.h b/library/cpp/monlib/metrics/metric_registry.h new file mode 100644 index 0000000000..68b2d652cb --- /dev/null +++ b/library/cpp/monlib/metrics/metric_registry.h @@ -0,0 +1,122 @@ +#pragma once + +#include "labels.h" +#include "metric.h" + +#include <util/system/rwlock.h> + +#include <library/cpp/threading/light_rw_lock/lightrwlock.h> + + +namespace NMonitoring { + class IMetricFactory { + public: + virtual ~IMetricFactory() = default; + + virtual IGauge* Gauge(ILabelsPtr labels) = 0; + virtual ILazyGauge* LazyGauge(ILabelsPtr labels, std::function<double()> supplier) = 0; + virtual IIntGauge* IntGauge(ILabelsPtr labels) = 0; + virtual ILazyIntGauge* LazyIntGauge(ILabelsPtr labels, std::function<i64()> supplier) = 0; + virtual ICounter* Counter(ILabelsPtr labels) = 0; + virtual ILazyCounter* LazyCounter(ILabelsPtr labels, std::function<ui64()> supplier) = 0; + + virtual IRate* Rate(ILabelsPtr labels) = 0; + virtual ILazyRate* LazyRate(ILabelsPtr labels, std::function<ui64()> supplier) = 0; + + virtual IHistogram* HistogramCounter( + ILabelsPtr labels, + IHistogramCollectorPtr collector) = 0; + + virtual IHistogram* HistogramRate( + ILabelsPtr labels, + IHistogramCollectorPtr collector) = 0; + }; + + class IMetricSupplier { + public: + virtual ~IMetricSupplier() = default; + + virtual void Accept(TInstant time, IMetricConsumer* consumer) const = 0; + virtual void Append(TInstant time, IMetricConsumer* consumer) const = 0; + }; + + class IMetricRegistry: public IMetricSupplier, public IMetricFactory { + public: + virtual const TLabels& CommonLabels() const noexcept = 0; + virtual void RemoveMetric(const ILabels& labels) noexcept = 0; + }; + + + /////////////////////////////////////////////////////////////////////////////// + // TMetricRegistry + /////////////////////////////////////////////////////////////////////////////// + class TMetricRegistry: public IMetricRegistry { + public: + TMetricRegistry(); + ~TMetricRegistry(); + + explicit TMetricRegistry(const TLabels& commonLabels); + + /** + * Get a global metrics registry instance. + */ + static TMetricRegistry* Instance(); + + TGauge* Gauge(TLabels labels); + TLazyGauge* LazyGauge(TLabels labels, std::function<double()> supplier); + TIntGauge* IntGauge(TLabels labels); + TLazyIntGauge* LazyIntGauge(TLabels labels, std::function<i64()> supplier); + TCounter* Counter(TLabels labels); + TLazyCounter* LazyCounter(TLabels labels, std::function<ui64()> supplier); + TRate* Rate(TLabels labels); + TLazyRate* LazyRate(TLabels labels, std::function<ui64()> supplier); + + THistogram* HistogramCounter( + TLabels labels, + IHistogramCollectorPtr collector); + + THistogram* HistogramRate( + TLabels labels, + IHistogramCollectorPtr collector); + + void Reset(); + + void Accept(TInstant time, IMetricConsumer* consumer) const override; + void Append(TInstant time, IMetricConsumer* consumer) const override; + + const TLabels& CommonLabels() const noexcept override { + return CommonLabels_; + } + + void RemoveMetric(const ILabels& labels) noexcept override; + + private: + TGauge* Gauge(ILabelsPtr labels) override; + TLazyGauge* LazyGauge(ILabelsPtr labels, std::function<double()> supplier) override; + TIntGauge* IntGauge(ILabelsPtr labels) override; + TLazyIntGauge* LazyIntGauge(ILabelsPtr labels, std::function<i64()> supplier) override; + TCounter* Counter(ILabelsPtr labels) override; + TLazyCounter* LazyCounter(ILabelsPtr labels, std::function<ui64()> supplier) override; + TRate* Rate(ILabelsPtr labels) override; + TLazyRate* LazyRate(ILabelsPtr labels, std::function<ui64()> supplier) override; + + THistogram* HistogramCounter( + ILabelsPtr labels, + IHistogramCollectorPtr collector) override; + + THistogram* HistogramRate( + ILabelsPtr labels, + IHistogramCollectorPtr collector) override; + + private: + TRWMutex Lock_; + THashMap<ILabelsPtr, IMetricPtr> Metrics_; + + template <typename TMetric, EMetricType type, typename TLabelsType, typename... Args> + TMetric* Metric(TLabelsType&& labels, Args&&... args); + + TLabels CommonLabels_; + }; + + void WriteLabels(IMetricConsumer* consumer, const ILabels& labels); +} diff --git a/library/cpp/monlib/metrics/metric_registry_ut.cpp b/library/cpp/monlib/metrics/metric_registry_ut.cpp new file mode 100644 index 0000000000..afcbcd6801 --- /dev/null +++ b/library/cpp/monlib/metrics/metric_registry_ut.cpp @@ -0,0 +1,302 @@ +#include "metric_registry.h" + +#include <library/cpp/monlib/encode/protobuf/protobuf.h> +#include <library/cpp/monlib/encode/json/json.h> +#include <library/cpp/resource/resource.h> + +#include <library/cpp/testing/unittest/registar.h> + +#include <util/stream/str.h> + +using namespace NMonitoring; + +template<> +void Out<NMonitoring::NProto::TSingleSample::ValueCase>(IOutputStream& os, NMonitoring::NProto::TSingleSample::ValueCase val) { + switch (val) { + case NMonitoring::NProto::TSingleSample::ValueCase::kInt64: + os << "Int64"; + break; + case NMonitoring::NProto::TSingleSample::ValueCase::kUint64: + os << "Uint64"; + break; + case NMonitoring::NProto::TSingleSample::ValueCase::kHistogram: + os << "Histogram"; + break; + case NMonitoring::NProto::TSingleSample::ValueCase::kFloat64: + os << "Float64"; + break; + case NMonitoring::NProto::TSingleSample::ValueCase::kSummaryDouble: + os << "DSummary"; + break; + case NMonitoring::NProto::TSingleSample::ValueCase::kLogHistogram: + os << "LogHistogram"; + break; + case NMonitoring::NProto::TSingleSample::ValueCase::VALUE_NOT_SET: + os << "NOT SET"; + break; + } +} + +Y_UNIT_TEST_SUITE(TMetricRegistryTest) { + Y_UNIT_TEST(Gauge) { + TMetricRegistry registry(TLabels{{"common", "label"}}); + TGauge* g = registry.Gauge({{"my", "gauge"}}); + + UNIT_ASSERT_DOUBLES_EQUAL(g->Get(), 0.0, 1E-6); + g->Set(12.34); + UNIT_ASSERT_DOUBLES_EQUAL(g->Get(), 12.34, 1E-6); + + double val; + + val = g->Add(1.2); + UNIT_ASSERT_DOUBLES_EQUAL(g->Get(), 13.54, 1E-6); + UNIT_ASSERT_DOUBLES_EQUAL(g->Get(), val, 1E-6); + + val = g->Add(-3.47); + UNIT_ASSERT_DOUBLES_EQUAL(g->Get(), 10.07, 1E-6); + UNIT_ASSERT_DOUBLES_EQUAL(g->Get(), val, 1E-6); + } + + Y_UNIT_TEST(LazyGauge) { + TMetricRegistry registry(TLabels{{"common", "label"}}); + double val = 0.0; + TLazyGauge* g = registry.LazyGauge({{"my", "lazyGauge"}}, [&val](){return val;}); + + UNIT_ASSERT_DOUBLES_EQUAL(g->Get(), 0.0, 1E-6); + val = 12.34; + UNIT_ASSERT_DOUBLES_EQUAL(g->Get(), 12.34, 1E-6); + + val += 1.2; + UNIT_ASSERT_DOUBLES_EQUAL(g->Get(), 13.54, 1E-6); + UNIT_ASSERT_DOUBLES_EQUAL(g->Get(), val, 1E-6); + + val += -3.47; + UNIT_ASSERT_DOUBLES_EQUAL(g->Get(), 10.07, 1E-6); + UNIT_ASSERT_DOUBLES_EQUAL(g->Get(), val, 1E-6); + } + + Y_UNIT_TEST(IntGauge) { + TMetricRegistry registry(TLabels{{"common", "label"}}); + TIntGauge* g = registry.IntGauge({{"my", "gauge"}}); + + UNIT_ASSERT_VALUES_EQUAL(g->Get(), 0); + + i64 val; + + val = g->Inc(); + UNIT_ASSERT_VALUES_EQUAL(g->Get(), 1); + UNIT_ASSERT_VALUES_EQUAL(g->Get(), val); + + val = g->Dec(); + UNIT_ASSERT_VALUES_EQUAL(g->Get(), 0); + UNIT_ASSERT_VALUES_EQUAL(g->Get(), val); + + val = g->Add(1); + UNIT_ASSERT_VALUES_EQUAL(g->Get(), 1); + UNIT_ASSERT_VALUES_EQUAL(g->Get(), val); + + val = g->Add(2); + UNIT_ASSERT_VALUES_EQUAL(g->Get(), 3); + UNIT_ASSERT_VALUES_EQUAL(g->Get(), val); + + val = g->Add(-5); + UNIT_ASSERT_VALUES_EQUAL(g->Get(), -2); + UNIT_ASSERT_VALUES_EQUAL(g->Get(), val); + } + + Y_UNIT_TEST(LazyIntGauge) { + TMetricRegistry registry(TLabels{{"common", "label"}}); + i64 val = 0; + TLazyIntGauge* g = registry.LazyIntGauge({{"my", "gauge"}}, [&val](){return val;}); + + UNIT_ASSERT_VALUES_EQUAL(g->Get(), 0); + val += 1; + UNIT_ASSERT_VALUES_EQUAL(g->Get(), 1); + UNIT_ASSERT_VALUES_EQUAL(g->Get(), val); + + val -= 1; + UNIT_ASSERT_VALUES_EQUAL(g->Get(), 0); + UNIT_ASSERT_VALUES_EQUAL(g->Get(), val); + + val = 42; + UNIT_ASSERT_VALUES_EQUAL(g->Get(), val); + } + + Y_UNIT_TEST(Counter) { + TMetricRegistry registry(TLabels{{"common", "label"}}); + TCounter* c = registry.Counter({{"my", "counter"}}); + + UNIT_ASSERT_VALUES_EQUAL(c->Get(), 0); + UNIT_ASSERT_VALUES_EQUAL(c->Inc(), 1); + UNIT_ASSERT_VALUES_EQUAL(c->Get(), 1); + UNIT_ASSERT_VALUES_EQUAL(c->Add(10), 11); + UNIT_ASSERT_VALUES_EQUAL(c->Get(), 11); + } + + Y_UNIT_TEST(LazyCounter) { + TMetricRegistry registry(TLabels{{"common", "label"}}); + ui64 val = 0; + + TLazyCounter* c = registry.LazyCounter({{"my", "counter"}}, [&val](){return val;}); + + UNIT_ASSERT_VALUES_EQUAL(c->Get(), 0); + val = 42; + UNIT_ASSERT_VALUES_EQUAL(c->Get(), 42); + } + + Y_UNIT_TEST(LazyRate) { + TMetricRegistry registry(TLabels{{"common", "label"}}); + ui64 val = 0; + + TLazyRate* r = registry.LazyRate({{"my", "rate"}}, [&val](){return val;}); + + UNIT_ASSERT_VALUES_EQUAL(r->Get(), 0); + val = 42; + UNIT_ASSERT_VALUES_EQUAL(r->Get(), 42); + } + + Y_UNIT_TEST(DoubleCounter) { + TMetricRegistry registry(TLabels{{"common", "label"}}); + + TCounter* c = registry.Counter({{"my", "counter"}}); + UNIT_ASSERT_VALUES_EQUAL(c->Get(), 0); + c->Add(10); + + c = registry.Counter({{"my", "counter"}}); + UNIT_ASSERT_VALUES_EQUAL(c->Get(), 10); + } + + Y_UNIT_TEST(Sample) { + TMetricRegistry registry(TLabels{{"common", "label"}}); + + TGauge* g = registry.Gauge({{"my", "gauge"}}); + g->Set(12.34); + + TCounter* c = registry.Counter({{"my", "counter"}}); + c->Add(10); + + NProto::TSingleSamplesList samples; + auto encoder = EncoderProtobuf(&samples); + auto now = TInstant::Now(); + registry.Accept(now, encoder.Get()); + + UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 2); + UNIT_ASSERT_VALUES_EQUAL(samples.CommonLabelsSize(), 1); + { + const NProto::TLabel& label = samples.GetCommonLabels(0); + UNIT_ASSERT_STRINGS_EQUAL(label.GetName(), "common"); + UNIT_ASSERT_STRINGS_EQUAL(label.GetValue(), "label"); + } + + + for (const NProto::TSingleSample& sample : samples.GetSamples()) { + UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1); + UNIT_ASSERT_VALUES_EQUAL(sample.GetTime(), now.MilliSeconds()); + + if (sample.GetMetricType() == NProto::GAUGE) { + UNIT_ASSERT_VALUES_EQUAL(sample.GetValueCase(), NProto::TSingleSample::kFloat64); + UNIT_ASSERT_DOUBLES_EQUAL(sample.GetFloat64(), 12.34, 1E-6); + + const NProto::TLabel& label = sample.GetLabels(0); + UNIT_ASSERT_STRINGS_EQUAL(label.GetName(), "my"); + UNIT_ASSERT_STRINGS_EQUAL(label.GetValue(), "gauge"); + } else if (sample.GetMetricType() == NProto::COUNTER) { + UNIT_ASSERT_VALUES_EQUAL(sample.GetValueCase(), NProto::TSingleSample::kUint64); + UNIT_ASSERT_VALUES_EQUAL(sample.GetUint64(), 10); + + const NProto::TLabel& label = sample.GetLabels(0); + UNIT_ASSERT_STRINGS_EQUAL(label.GetName(), "my"); + UNIT_ASSERT_STRINGS_EQUAL(label.GetValue(), "counter"); + } else { + UNIT_FAIL("unexpected sample type"); + } + } + } + + Y_UNIT_TEST(Histograms) { + TMetricRegistry registry(TLabels{{"common", "label"}}); + + THistogram* h1 = registry.HistogramCounter( + {{"sensor", "readTimeMillis"}}, + ExponentialHistogram(5, 2)); + + THistogram* h2 = registry.HistogramRate( + {{"sensor", "writeTimeMillis"}}, + ExplicitHistogram({1, 5, 15, 20, 25})); + + for (i64 i = 0; i < 100; i++) { + h1->Record(i); + h2->Record(i); + } + + TStringStream ss; + { + auto encoder = EncoderJson(&ss, 2); + registry.Accept(TInstant::Zero(), encoder.Get()); + } + ss << '\n'; + + UNIT_ASSERT_NO_DIFF(ss.Str(), NResource::Find("/histograms.json")); + } + + Y_UNIT_TEST(StreamingEncoderTest) { + const TString expected { + "{\"commonLabels\":{\"common\":\"label\"}," + "\"sensors\":[{\"kind\":\"GAUGE\",\"labels\":{\"my\":\"gauge\"},\"value\":12.34}]}" + }; + + TMetricRegistry registry(TLabels{{"common", "label"}}); + + TGauge* g = registry.Gauge({{"my", "gauge"}}); + g->Set(12.34); + + TStringStream os; + auto encoder = EncoderJson(&os); + registry.Accept(TInstant::Zero(), encoder.Get()); + + UNIT_ASSERT_STRINGS_EQUAL(os.Str(), expected); + } + + Y_UNIT_TEST(CreatingSameMetricWithDifferentTypesShouldThrow) { + TMetricRegistry registry; + + registry.Gauge({{"foo", "bar"}}); + UNIT_ASSERT_EXCEPTION(registry.Counter({{"foo", "bar"}}), yexception); + + registry.HistogramCounter({{"bar", "baz"}}, nullptr); + UNIT_ASSERT_EXCEPTION(registry.HistogramRate({{"bar", "baz"}}, nullptr), yexception); + } + + Y_UNIT_TEST(EncodeRegistryWithCommonLabels) { + TMetricRegistry registry(TLabels{{"common", "label"}}); + + TGauge* g = registry.Gauge({{"my", "gauge"}}); + g->Set(12.34); + + // Append() adds common labels to each metric, allowing to combine + // several metric registries in one resulting blob + { + TStringStream os; + auto encoder = EncoderJson(&os); + encoder->OnStreamBegin(); + registry.Append(TInstant::Zero(), encoder.Get()); + encoder->OnStreamEnd(); + + UNIT_ASSERT_STRINGS_EQUAL( + os.Str(), + "{\"sensors\":[{\"kind\":\"GAUGE\",\"labels\":{\"common\":\"label\",\"my\":\"gauge\"},\"value\":12.34}]}"); + } + + // Accept() adds common labels to the beginning of the blob + { + TStringStream os; + auto encoder = EncoderJson(&os); + registry.Accept(TInstant::Zero(), encoder.Get()); + + UNIT_ASSERT_STRINGS_EQUAL( + os.Str(), + "{\"commonLabels\":{\"common\":\"label\"}," + "\"sensors\":[{\"kind\":\"GAUGE\",\"labels\":{\"my\":\"gauge\"},\"value\":12.34}]}"); + } + } +} diff --git a/library/cpp/monlib/metrics/metric_sub_registry.h b/library/cpp/monlib/metrics/metric_sub_registry.h new file mode 100644 index 0000000000..e83eeeafb2 --- /dev/null +++ b/library/cpp/monlib/metrics/metric_sub_registry.h @@ -0,0 +1,116 @@ +#pragma once + +#include "metric_registry.h" + +namespace NMonitoring { + +/** + * This registry is wrapping given delegate registry to add common labels + * to all created metrics through this sub registry. + */ +class TMetricSubRegistry final: public IMetricRegistry { +public: + /** + * Do not keep ownership of the given delegate. + */ + TMetricSubRegistry(TLabels commonLabels, IMetricRegistry* delegate) noexcept + : CommonLabels_{std::move(commonLabels)} + , DelegatePtr_{delegate} + { + } + + /** + * Keeps ownership of the given delegate. + */ + TMetricSubRegistry(TLabels commonLabels, std::shared_ptr<IMetricRegistry> delegate) noexcept + : CommonLabels_{std::move(commonLabels)} + , Delegate_{std::move(delegate)} + , DelegatePtr_{Delegate_.get()} + { + } + + IGauge* Gauge(ILabelsPtr labels) override { + AddCommonLabels(labels.Get()); + return DelegatePtr_->Gauge(std::move(labels)); + } + + ILazyGauge* LazyGauge(ILabelsPtr labels, std::function<double()> supplier) override { + AddCommonLabels(labels.Get()); + return DelegatePtr_->LazyGauge(std::move(labels), std::move(supplier)); + } + + IIntGauge* IntGauge(ILabelsPtr labels) override { + AddCommonLabels(labels.Get()); + return DelegatePtr_->IntGauge(std::move(labels)); + } + + ILazyIntGauge* LazyIntGauge(ILabelsPtr labels, std::function<i64()> supplier) override { + AddCommonLabels(labels.Get()); + return DelegatePtr_->LazyIntGauge(std::move(labels), std::move(supplier)); + } + + ICounter* Counter(ILabelsPtr labels) override { + AddCommonLabels(labels.Get()); + return DelegatePtr_->Counter(std::move(labels)); + } + + ILazyCounter* LazyCounter(ILabelsPtr labels, std::function<ui64()> supplier) override { + AddCommonLabels(labels.Get()); + return DelegatePtr_->LazyCounter(std::move(labels), std::move(supplier)); + } + + IRate* Rate(ILabelsPtr labels) override { + AddCommonLabels(labels.Get()); + return DelegatePtr_->Rate(std::move(labels)); + } + + ILazyRate* LazyRate(ILabelsPtr labels, std::function<ui64()> supplier) override { + AddCommonLabels(labels.Get()); + return DelegatePtr_->LazyRate(std::move(labels), std::move(supplier)); + } + + IHistogram* HistogramCounter(ILabelsPtr labels, IHistogramCollectorPtr collector) override { + AddCommonLabels(labels.Get()); + return DelegatePtr_->HistogramCounter(std::move(labels), std::move(collector)); + } + + IHistogram* HistogramRate(ILabelsPtr labels, IHistogramCollectorPtr collector) override { + AddCommonLabels(labels.Get()); + return DelegatePtr_->HistogramRate(std::move(labels), std::move(collector)); + } + + void Accept(TInstant time, IMetricConsumer* consumer) const override { + DelegatePtr_->Accept(time, consumer); + } + + void Append(TInstant time, IMetricConsumer* consumer) const override { + DelegatePtr_->Append(time, consumer); + } + + const TLabels& CommonLabels() const noexcept override { + return CommonLabels_; + } + + void RemoveMetric(const ILabels& labels) noexcept override { + TLabelsImpl<TStringBuf> toRemove; + for (auto& l: labels) { + toRemove.Add(l); + } + AddCommonLabels(&toRemove); + DelegatePtr_->RemoveMetric(toRemove); + } + +private: + void AddCommonLabels(ILabels* labels) const { + for (auto& label: CommonLabels_) { + labels->Add(label); + } + } + +private: + const TLabels CommonLabels_; + std::shared_ptr<IMetricRegistry> Delegate_; + IMetricRegistry* DelegatePtr_; +}; + +} // namespace NMonitoring diff --git a/library/cpp/monlib/metrics/metric_sub_registry_ut.cpp b/library/cpp/monlib/metrics/metric_sub_registry_ut.cpp new file mode 100644 index 0000000000..0c5d48b876 --- /dev/null +++ b/library/cpp/monlib/metrics/metric_sub_registry_ut.cpp @@ -0,0 +1,65 @@ +#include "metric_sub_registry.h" + +#include <library/cpp/testing/unittest/registar.h> + +using namespace NMonitoring; + +Y_UNIT_TEST_SUITE(TMetricSubRegistryTest) { + Y_UNIT_TEST(WrapRegistry) { + TMetricRegistry registry; + + { + TMetricSubRegistry subRegistry{{{"common", "label"}}, ®istry}; + IIntGauge* g = subRegistry.IntGauge(MakeLabels({{"my", "gauge"}})); + UNIT_ASSERT(g); + g->Set(42); + } + + TIntGauge* g = registry.IntGauge({{"my", "gauge"}, {"common", "label"}}); + UNIT_ASSERT(g); + UNIT_ASSERT_VALUES_EQUAL(g->Get(), 42); + } + + Y_UNIT_TEST(CommonLabelsDoNotOverrideGeneralLabel) { + TMetricRegistry registry; + + { + TMetricSubRegistry subRegistry{{{"common", "label"}, {"my", "notOverride"}}, ®istry}; + IIntGauge* g = subRegistry.IntGauge(MakeLabels({{"my", "gauge"}})); + UNIT_ASSERT(g); + g->Set(1234); + } + + TIntGauge* knownGauge = registry.IntGauge({{"my", "gauge"}, {"common", "label"}}); + UNIT_ASSERT(knownGauge); + UNIT_ASSERT_VALUES_EQUAL(knownGauge->Get(), 1234); + + TIntGauge* newGauge = registry.IntGauge({{"common", "label"}, {"my", "notOverride"}}); + UNIT_ASSERT(newGauge); + UNIT_ASSERT_VALUES_EQUAL(newGauge->Get(), 0); + } + + Y_UNIT_TEST(RemoveMetric) { + TMetricRegistry registry; + + { + TMetricSubRegistry subRegistry{{{"common", "label"}}, ®istry}; + IIntGauge* g = subRegistry.IntGauge(MakeLabels({{"my", "gauge"}})); + UNIT_ASSERT(g); + g->Set(1234); + } + + IIntGauge* g1 = registry.IntGauge({{"my", "gauge"}, {"common", "label"}}); + UNIT_ASSERT(g1); + UNIT_ASSERT_VALUES_EQUAL(g1->Get(), 1234); + + { + TMetricSubRegistry subRegistry{{{"common", "label"}}, ®istry}; + subRegistry.RemoveMetric(TLabels{{"my", "gauge"}}); + } + + IIntGauge* g2 = registry.IntGauge({{"my", "gauge"}, {"common", "label"}}); + UNIT_ASSERT(g2); + UNIT_ASSERT_VALUES_EQUAL(g2->Get(), 0); + } +} diff --git a/library/cpp/monlib/metrics/metric_type.cpp b/library/cpp/monlib/metrics/metric_type.cpp new file mode 100644 index 0000000000..a8a546e843 --- /dev/null +++ b/library/cpp/monlib/metrics/metric_type.cpp @@ -0,0 +1,57 @@ +#include "metric_type.h" + +#include <util/generic/strbuf.h> +#include <util/generic/yexception.h> +#include <util/stream/output.h> + +namespace NMonitoring { + TStringBuf MetricTypeToStr(EMetricType type) { + switch (type) { + case EMetricType::GAUGE: + return TStringBuf("GAUGE"); + case EMetricType::COUNTER: + return TStringBuf("COUNTER"); + case EMetricType::RATE: + return TStringBuf("RATE"); + case EMetricType::IGAUGE: + return TStringBuf("IGAUGE"); + case EMetricType::HIST: + return TStringBuf("HIST"); + case EMetricType::HIST_RATE: + return TStringBuf("HIST_RATE"); + case EMetricType::DSUMMARY: + return TStringBuf("DSUMMARY"); + case EMetricType::LOGHIST: + return TStringBuf("LOGHIST"); + default: + return TStringBuf("UNKNOWN"); + } + } + + EMetricType MetricTypeFromStr(TStringBuf str) { + if (str == TStringBuf("GAUGE") || str == TStringBuf("DGAUGE")) { + return EMetricType::GAUGE; + } else if (str == TStringBuf("COUNTER")) { + return EMetricType::COUNTER; + } else if (str == TStringBuf("RATE")) { + return EMetricType::RATE; + } else if (str == TStringBuf("IGAUGE")) { + return EMetricType::IGAUGE; + } else if (str == TStringBuf("HIST")) { + return EMetricType::HIST; + } else if (str == TStringBuf("HIST_RATE")) { + return EMetricType::HIST_RATE; + } else if (str == TStringBuf("DSUMMARY")) { + return EMetricType::DSUMMARY; + } else if (str == TStringBuf("LOGHIST")) { + return EMetricType::LOGHIST; + } else { + ythrow yexception() << "unknown metric type: " << str; + } + } +} + +template <> +void Out<NMonitoring::EMetricType>(IOutputStream& o, NMonitoring::EMetricType t) { + o << NMonitoring::MetricTypeToStr(t); +} diff --git a/library/cpp/monlib/metrics/metric_type.h b/library/cpp/monlib/metrics/metric_type.h new file mode 100644 index 0000000000..1984c42c1e --- /dev/null +++ b/library/cpp/monlib/metrics/metric_type.h @@ -0,0 +1,25 @@ +#pragma once + +#include <util/generic/fwd.h> + +namespace NMonitoring { + + constexpr ui32 MaxMetricTypeNameLength = 9; + + enum class EMetricType { + UNKNOWN = 0, + GAUGE = 1, + COUNTER = 2, + RATE = 3, + IGAUGE = 4, + HIST = 5, + HIST_RATE = 6, + DSUMMARY = 7, + // ISUMMARY = 8, reserved + LOGHIST = 9, + }; + + TStringBuf MetricTypeToStr(EMetricType type); + EMetricType MetricTypeFromStr(TStringBuf str); + +} diff --git a/library/cpp/monlib/metrics/metric_value.cpp b/library/cpp/monlib/metrics/metric_value.cpp new file mode 100644 index 0000000000..b95d7011c6 --- /dev/null +++ b/library/cpp/monlib/metrics/metric_value.cpp @@ -0,0 +1,27 @@ +#include "metric_value.h" + + +namespace NMonitoring { + void TMetricTimeSeries::SortByTs() { + SortPointsByTs(ValueType_, Points_); + } + + void TMetricTimeSeries::Clear() noexcept { + if (ValueType_ == EMetricValueType::HISTOGRAM) { + for (TPoint& p: Points_) { + SnapshotUnRef<EMetricValueType::HISTOGRAM>(p); + } + } else if (ValueType_ == EMetricValueType::SUMMARY) { + for (TPoint& p: Points_) { + SnapshotUnRef<EMetricValueType::SUMMARY>(p); + } + } else if (ValueType_ == EMetricValueType::LOGHISTOGRAM) { + for (TPoint& p: Points_) { + SnapshotUnRef<EMetricValueType::LOGHISTOGRAM>(p); + } + } + + Points_.clear(); + ValueType_ = EMetricValueType::UNKNOWN; + } +} diff --git a/library/cpp/monlib/metrics/metric_value.h b/library/cpp/monlib/metrics/metric_value.h new file mode 100644 index 0000000000..607fcc8602 --- /dev/null +++ b/library/cpp/monlib/metrics/metric_value.h @@ -0,0 +1,542 @@ +#pragma once + +#include "histogram_collector.h" +#include "metric_value_type.h" +#include "summary_collector.h" +#include "log_histogram_snapshot.h" + +#include <util/datetime/base.h> +#include <util/generic/algorithm.h> +#include <util/generic/vector.h> +#include <util/generic/cast.h> +#include <util/generic/ymath.h> + +namespace NMonitoring { + namespace NPrivate { + template <typename T> + T FromFloatSafe(double d) { + static_assert(std::is_integral<T>::value, "this function only converts floats to integers"); + Y_ENSURE(::IsValidFloat(d) && d >= Min<T>() && d <= MaxFloor<T>(), "Cannot convert " << d << " to an integer value"); + return static_cast<T>(d); + } + + inline auto POINT_KEY_FN = [](auto& p) { + return p.GetTime(); + }; + } // namespace NPrivate + + template <typename T, typename Enable = void> + struct TValueType; + + template <> + struct TValueType<double> { + static constexpr auto Type = EMetricValueType::DOUBLE; + }; + + template <> + struct TValueType<i64> { + static constexpr auto Type = EMetricValueType::INT64; + }; + + template <> + struct TValueType<ui64> { + static constexpr auto Type = EMetricValueType::UINT64; + }; + + template <> + struct TValueType<TLogHistogramSnapshot*> { + static constexpr auto Type = EMetricValueType::LOGHISTOGRAM; + }; + + template <typename T> + struct TValueType<T*, typename std::enable_if_t<std::is_base_of<IHistogramSnapshot, T>::value>> { + static constexpr auto Type = EMetricValueType::HISTOGRAM; + }; + + template <typename T> + struct TValueType<T*, typename std::enable_if_t<std::is_base_of<ISummaryDoubleSnapshot, T>::value>> { + static constexpr auto Type = EMetricValueType::SUMMARY; + }; + + /////////////////////////////////////////////////////////////////////////// + // TMetricValue + /////////////////////////////////////////////////////////////////////////// + // TMetricValue represents a generic value. It does not contain type + // information about a value. This is done to minimize object footprint. + // To read an actual value from the object the type must be checked + // first or provided to AsXxxx(type) member-functions. + // This class does not hold an ownership of an IHistogramSnapshot or + // SummarySnapshot, so this must be done somewhere outside. + class TMetricValue { + public: + TMetricValue() noexcept { + Value_.Uint64 = 0; + } + + explicit TMetricValue(double value) noexcept { + Value_.Double = value; + } + + explicit TMetricValue(i64 value) noexcept { + Value_.Int64 = value; + } + + explicit TMetricValue(ui64 value) noexcept { + Value_.Uint64 = value; + } + + explicit TMetricValue(IHistogramSnapshot* histogram) noexcept { + Value_.Histogram = histogram; + } + + explicit TMetricValue(ISummaryDoubleSnapshot* summary) noexcept { + Value_.Summary = summary; + } + + explicit TMetricValue(TLogHistogramSnapshot* logHist) noexcept { + Value_.LogHistogram = logHist; + } + + double AsDouble() const noexcept { + return Value_.Double; + } + + // will cast value into double, current value type is determined by + // the given type argument + double AsDouble(EMetricValueType type) const { + switch (type) { + case EMetricValueType::DOUBLE: + return Value_.Double; + case EMetricValueType::INT64: + return static_cast<double>(Value_.Int64); + case EMetricValueType::UINT64: + return static_cast<double>(Value_.Uint64); + case EMetricValueType::HISTOGRAM: + ythrow yexception() << "histogram cannot be casted to Double"; + case EMetricValueType::SUMMARY: + ythrow yexception() << "summary cannot be casted to Double"; + case EMetricValueType::LOGHISTOGRAM: + ythrow yexception() << "loghistogram cannot be casted to Double"; + case EMetricValueType::UNKNOWN: + ythrow yexception() << "unknown value type"; + } + Y_FAIL(); // for GCC + } + + ui64 AsUint64() const noexcept { + return Value_.Uint64; + } + + // will cast value into uint64, current value's type is determined by + // the given type argument + ui64 AsUint64(EMetricValueType type) const { + switch (type) { + case EMetricValueType::DOUBLE: + return NPrivate::FromFloatSafe<ui64>(Value_.Double); + case EMetricValueType::INT64: + return SafeIntegerCast<ui64>(Value_.Int64); + case EMetricValueType::UINT64: + return Value_.Uint64; + case EMetricValueType::HISTOGRAM: + ythrow yexception() << "histogram cannot be casted to Uint64"; + case EMetricValueType::SUMMARY: + ythrow yexception() << "summary cannot be casted to Uint64"; + case EMetricValueType::LOGHISTOGRAM: + ythrow yexception() << "loghistogram cannot be casted to Uint64"; + case EMetricValueType::UNKNOWN: + ythrow yexception() << "unknown value type"; + } + Y_FAIL(); // for GCC + } + + i64 AsInt64() const noexcept { + return Value_.Int64; + } + + // will cast value into int64, current value's type is determined by + // the given type argument + i64 AsInt64(EMetricValueType type) const { + switch (type) { + case EMetricValueType::DOUBLE: + return NPrivate::FromFloatSafe<i64>(Value_.Double); + case EMetricValueType::INT64: + return Value_.Int64; + case EMetricValueType::UINT64: + return SafeIntegerCast<i64>(Value_.Uint64); + case EMetricValueType::HISTOGRAM: + ythrow yexception() << "histogram cannot be casted to Int64"; + case EMetricValueType::SUMMARY: + ythrow yexception() << "summary cannot be casted to Int64"; + case EMetricValueType::LOGHISTOGRAM: + ythrow yexception() << "loghistogram cannot be casted to Int64"; + case EMetricValueType::UNKNOWN: + ythrow yexception() << "unknown value type"; + } + Y_FAIL(); // for GCC + } + + IHistogramSnapshot* AsHistogram() const noexcept { + return Value_.Histogram; + } + + IHistogramSnapshot* AsHistogram(EMetricValueType type) const { + if (type != EMetricValueType::HISTOGRAM) { + ythrow yexception() << type << " cannot be casted to Histogram"; + } + + return Value_.Histogram; + } + + ISummaryDoubleSnapshot* AsSummaryDouble() const noexcept { + return Value_.Summary; + } + + ISummaryDoubleSnapshot* AsSummaryDouble(EMetricValueType type) const { + if (type != EMetricValueType::SUMMARY) { + ythrow yexception() << type << " cannot be casted to SummaryDouble"; + } + + return Value_.Summary; + } + + TLogHistogramSnapshot* AsLogHistogram() const noexcept { + return Value_.LogHistogram; + } + + TLogHistogramSnapshot* AsLogHistogram(EMetricValueType type) const { + if (type != EMetricValueType::LOGHISTOGRAM) { + ythrow yexception() << type << " cannot be casted to LogHistogram"; + } + + return Value_.LogHistogram; + } + + protected: + union { + double Double; + i64 Int64; + ui64 Uint64; + IHistogramSnapshot* Histogram; + ISummaryDoubleSnapshot* Summary; + TLogHistogramSnapshot* LogHistogram; + } Value_; + }; + + /////////////////////////////////////////////////////////////////////////// + // TMetricValueWithType + /////////////////////////////////////////////////////////////////////////// + // Same as TMetricValue, but this type holds an ownership of + // snapshots and contains value type information. + class TMetricValueWithType: private TMetricValue, public TMoveOnly { + public: + using TBase = TMetricValue; + + template <typename T> + explicit TMetricValueWithType(T value) + : TBase(value) + , ValueType_{TValueType<T>::Type} + { + Ref(); + } + + TMetricValueWithType(TMetricValueWithType&& other) + : TBase(std::move(other)) + , ValueType_{other.ValueType_} + { + Ref(); + other.Clear(); + } + + TMetricValueWithType& operator=(TMetricValueWithType&& other) { + TBase::operator=(other); + ValueType_ = other.ValueType_; + + Ref(); + other.Clear(); + + return *this; + } + + ~TMetricValueWithType() { + UnRef(); + } + + void Clear() { + UnRef(); + ValueType_ = EMetricValueType::UNKNOWN; + } + + EMetricValueType GetType() const noexcept { + return ValueType_; + } + + double AsDouble() const { + return TBase::AsDouble(ValueType_); + } + + ui64 AsUint64() const { + return TBase::AsUint64(ValueType_); + } + + i64 AsInt64() const { + return TBase::AsInt64(ValueType_); + } + + IHistogramSnapshot* AsHistogram() const { + return TBase::AsHistogram(ValueType_); + } + + ISummaryDoubleSnapshot* AsSummaryDouble() const { + return TBase::AsSummaryDouble(ValueType_); + } + + TLogHistogramSnapshot* AsLogHistogram() const { + return TBase::AsLogHistogram(ValueType_); + } + + private: + void Ref() { + if (ValueType_ == EMetricValueType::SUMMARY) { + TBase::AsSummaryDouble()->Ref(); + } else if (ValueType_ == EMetricValueType::HISTOGRAM) { + TBase::AsHistogram()->Ref(); + } else if (ValueType_ == EMetricValueType::LOGHISTOGRAM) { + TBase::AsLogHistogram()->Ref(); + } + } + + void UnRef() { + if (ValueType_ == EMetricValueType::SUMMARY) { + TBase::AsSummaryDouble()->UnRef(); + } else if (ValueType_ == EMetricValueType::HISTOGRAM) { + TBase::AsHistogram()->UnRef(); + } else if (ValueType_ == EMetricValueType::LOGHISTOGRAM) { + TBase::AsLogHistogram()->UnRef(); + } + } + + private: + EMetricValueType ValueType_ = EMetricValueType::UNKNOWN; + }; + + static_assert(sizeof(TMetricValue) == sizeof(ui64), + "expected size of TMetricValue is one machine word"); + + /////////////////////////////////////////////////////////////////////////// + // TMetricTimeSeries + /////////////////////////////////////////////////////////////////////////// + class TMetricTimeSeries: private TMoveOnly { + public: + class TPoint { + public: + TPoint() + : Time_(TInstant::Zero()) + { + } + + template <typename T> + TPoint(TInstant time, T value) + : Time_(time) + , Value_(value) + { + } + + TInstant GetTime() const noexcept { + return Time_; + } + + TMetricValue GetValue() const noexcept { + return Value_; + } + + void ClearValue() { + Value_ = {}; + } + + private: + TInstant Time_; + TMetricValue Value_; + }; + + public: + TMetricTimeSeries() = default; + + TMetricTimeSeries(TMetricTimeSeries&& rhs) noexcept + : ValueType_(rhs.ValueType_) + , Points_(std::move(rhs.Points_)) + { + rhs.ValueType_ = EMetricValueType::UNKNOWN; + } + + TMetricTimeSeries& operator=(TMetricTimeSeries&& rhs) noexcept { + Clear(); + + ValueType_ = rhs.ValueType_; + rhs.ValueType_ = EMetricValueType::UNKNOWN; + + Points_ = std::move(rhs.Points_); + return *this; + } + + ~TMetricTimeSeries() { + Clear(); + } + + template <typename T> + void Add(TInstant time, T value) { + Add(TPoint(time, value), TValueType<T>::Type); + } + + void Add(TPoint point, EMetricValueType valueType) { + if (Empty()) { + ValueType_ = valueType; + } else { + CheckTypes(ValueType_, valueType); + } + Points_.push_back(point); + + if (ValueType_ == EMetricValueType::SUMMARY) { + TPoint& p = Points_.back(); + p.GetValue().AsSummaryDouble()->Ref(); + } else if (ValueType_ == EMetricValueType::HISTOGRAM) { + TPoint& p = Points_.back(); + p.GetValue().AsHistogram()->Ref(); + } else if (ValueType_ == EMetricValueType::LOGHISTOGRAM) { + TPoint& p = Points_.back(); + p.GetValue().AsLogHistogram()->Ref(); + } + } + + void CopyFrom(const TMetricTimeSeries& other) { + if (Empty()) { + ValueType_ = other.ValueType_; + } else { + CheckTypes(GetValueType(), other.GetValueType()); + } + + size_t prevSize = Points_.size(); + Copy(std::begin(other.Points_), std::end(other.Points_), + std::back_inserter(Points_)); + + if (ValueType_ == EMetricValueType::HISTOGRAM) { + for (size_t i = prevSize; i < Points_.size(); i++) { + TPoint& point = Points_[i]; + point.GetValue().AsHistogram()->Ref(); + } + } else if (ValueType_ == EMetricValueType::SUMMARY) { + for (size_t i = prevSize; i < Points_.size(); ++i) { + TPoint& point = Points_[i]; + point.GetValue().AsSummaryDouble()->Ref(); + } + } else if (ValueType_ == EMetricValueType::LOGHISTOGRAM) { + for (size_t i = prevSize; i < Points_.size(); ++i) { + TPoint& point = Points_[i]; + point.GetValue().AsLogHistogram()->Ref(); + } + } + } + + template <typename TConsumer> + void ForEach(TConsumer c) const { + for (const auto& point : Points_) { + c(point.GetTime(), ValueType_, point.GetValue()); + } + } + + bool Empty() const noexcept { + return Points_.empty(); + } + + size_t Size() const noexcept { + return Points_.size(); + } + + size_t Capacity() const noexcept { + return Points_.capacity(); + } + + const TPoint& operator[](size_t index) const noexcept { + return Points_[index]; + } + + void SortByTs(); + + void Clear() noexcept; + + EMetricValueType GetValueType() const noexcept { + return ValueType_; + } + + private: + static void CheckTypes(EMetricValueType t1, EMetricValueType t2) { + Y_ENSURE(t1 == t2, + "Series type mismatch: expected " << t1 << + ", but got " << t2); + } + + private: + EMetricValueType ValueType_ = EMetricValueType::UNKNOWN; + TVector<TPoint> Points_; + }; + + template <EMetricValueType valueType, typename TPoint> + static inline void SnapshotUnRef(TPoint& point) { + if constexpr (valueType == EMetricValueType::HISTOGRAM) { + if (auto* hist = point.GetValue().AsHistogram()) { + hist->UnRef(); + } + } else if constexpr (valueType == EMetricValueType::SUMMARY) { + if (auto* summary = point.GetValue().AsSummaryDouble()) { + summary->UnRef(); + } + } else if constexpr (valueType == EMetricValueType::LOGHISTOGRAM) { + if (auto* logHist = point.GetValue().AsLogHistogram()) { + logHist->UnRef(); + } + } + } + + template <EMetricValueType valueType, typename TPoint> + static void EraseDuplicates(TVector<TPoint>& points) { + // we have to manually clean reference to a snapshot from point + // while removing duplicates + auto result = points.rbegin(); + for (auto it = result + 1; it != points.rend(); ++it) { + if (result->GetTime() != it->GetTime() && ++result != it) { + SnapshotUnRef<valueType>(*result); + *result = *it; // (2) copy + it->ClearValue(); // (3) clean pointer in the source + } + } + + // erase tail points + for (auto it = result + 1; it != points.rend(); ++it) { + SnapshotUnRef<valueType>(*it); + } + points.erase(points.begin(), (result + 1).base()); + } + + template <typename TPoint> + void SortPointsByTs(EMetricValueType valueType, TVector<TPoint>& points) { + if (points.size() < 2) { + return; + } + + if (valueType != EMetricValueType::HISTOGRAM && valueType != EMetricValueType::SUMMARY + && valueType != EMetricValueType::LOGHISTOGRAM) { + // Stable sort + saving only the last point inside a group of duplicates + StableSortBy(points, NPrivate::POINT_KEY_FN); + auto it = UniqueBy(points.rbegin(), points.rend(), NPrivate::POINT_KEY_FN); + points.erase(points.begin(), it.base()); + } else { + StableSortBy(points, NPrivate::POINT_KEY_FN); + if (valueType == EMetricValueType::HISTOGRAM) { + EraseDuplicates<EMetricValueType::HISTOGRAM>(points); + } else if (valueType == EMetricValueType::LOGHISTOGRAM) { + EraseDuplicates<EMetricValueType::LOGHISTOGRAM>(points); + } else { + EraseDuplicates<EMetricValueType::SUMMARY>(points); + } + } + } +} diff --git a/library/cpp/monlib/metrics/metric_value_type.h b/library/cpp/monlib/metrics/metric_value_type.h new file mode 100644 index 0000000000..ab30a958c2 --- /dev/null +++ b/library/cpp/monlib/metrics/metric_value_type.h @@ -0,0 +1,16 @@ +#pragma once + + +namespace NMonitoring { + +enum class EMetricValueType { + UNKNOWN, + DOUBLE, + INT64, + UINT64, + HISTOGRAM, + SUMMARY, + LOGHISTOGRAM, +}; + +} // namespace NMonitoring diff --git a/library/cpp/monlib/metrics/metric_value_ut.cpp b/library/cpp/monlib/metrics/metric_value_ut.cpp new file mode 100644 index 0000000000..49b47c4057 --- /dev/null +++ b/library/cpp/monlib/metrics/metric_value_ut.cpp @@ -0,0 +1,507 @@ +#include "metric_value.h" + +#include <library/cpp/testing/unittest/registar.h> + +using namespace NMonitoring; + +Y_UNIT_TEST_SUITE(TMetricValueTest) { + + class TTestHistogram: public IHistogramSnapshot { + public: + TTestHistogram(ui32 count = 1) + : Count_{count} + {} + + private: + ui32 Count() const override { + return Count_; + } + + TBucketBound UpperBound(ui32 /*index*/) const override { + return 1234.56; + } + + TBucketValue Value(ui32 /*index*/) const override { + return 42; + } + + ui32 Count_{0}; + }; + + IHistogramSnapshotPtr MakeHistogramSnapshot() { + return MakeIntrusive<TTestHistogram>(); + } + + ISummaryDoubleSnapshotPtr MakeSummarySnapshot(ui64 count = 0u) { + return MakeIntrusive<TSummaryDoubleSnapshot>(0.0, 0.0, 0.0, 0.0, count); + } + + TLogHistogramSnapshotPtr MakeLogHistogram(ui64 count = 0) { + TVector<double> buckets; + for (ui64 i = 0; i < count; ++i) { + buckets.push_back(i); + } + return MakeIntrusive<TLogHistogramSnapshot>(1.5, 0u, 0, buckets); + } + + Y_UNIT_TEST(Sorted) { + auto ts1 = TInstant::Now(); + auto ts2 = ts1 + TDuration::Seconds(1); + + TMetricTimeSeries timeSeries; + timeSeries.Add(ts1, 3.14159); + timeSeries.Add(ts1, 6.28318); + timeSeries.Add(ts2, 2.71828); + + UNIT_ASSERT_EQUAL(timeSeries.Size(), 3); + + timeSeries.SortByTs(); + UNIT_ASSERT_EQUAL(timeSeries.Size(), 2); + + UNIT_ASSERT_EQUAL(ts1, timeSeries[0].GetTime()); + UNIT_ASSERT_DOUBLES_EQUAL(6.28318, timeSeries[0].GetValue().AsDouble(), Min<double>()); + + UNIT_ASSERT_EQUAL(ts2, timeSeries[1].GetTime()); + UNIT_ASSERT_DOUBLES_EQUAL(2.71828, timeSeries[1].GetValue().AsDouble(), Min<double>()); + } + + Y_UNIT_TEST(Histograms) { + auto ts = TInstant::Now(); + auto histogram = MakeIntrusive<TTestHistogram>(); + + UNIT_ASSERT_VALUES_EQUAL(1, histogram->RefCount()); + { + TMetricTimeSeries timeSeries; + timeSeries.Add(ts, histogram.Get()); + UNIT_ASSERT_VALUES_EQUAL(2, histogram->RefCount()); + } + UNIT_ASSERT_VALUES_EQUAL(1, histogram->RefCount()); + } + + Y_UNIT_TEST(Summary) { + auto ts = TInstant::Now(); + auto summary = MakeSummarySnapshot(); + UNIT_ASSERT_VALUES_EQUAL(1, summary->RefCount()); + { + TMetricTimeSeries timeSeries; + timeSeries.Add(ts, summary.Get()); + UNIT_ASSERT_VALUES_EQUAL(2, summary->RefCount()); + } + UNIT_ASSERT_VALUES_EQUAL(1, summary->RefCount()); + } + + Y_UNIT_TEST(LogHistogram) { + auto ts = TInstant::Now(); + auto logHist = MakeLogHistogram(); + UNIT_ASSERT_VALUES_EQUAL(1, logHist->RefCount()); + { + TMetricTimeSeries timeSeries; + timeSeries.Add(ts, logHist.Get()); + UNIT_ASSERT_VALUES_EQUAL(2, logHist->RefCount()); + } + UNIT_ASSERT_VALUES_EQUAL(1, logHist->RefCount()); + } + + Y_UNIT_TEST(TimeSeriesMovable) { + auto ts = TInstant::Now(); + auto histogram = MakeIntrusive<TTestHistogram>(); + + UNIT_ASSERT_VALUES_EQUAL(1, histogram->RefCount()); + { + TMetricTimeSeries timeSeriesA; + timeSeriesA.Add(ts, histogram.Get()); + UNIT_ASSERT_VALUES_EQUAL(2, histogram->RefCount()); + + TMetricTimeSeries timeSeriesB = std::move(timeSeriesA); + UNIT_ASSERT_VALUES_EQUAL(2, histogram->RefCount()); + + UNIT_ASSERT_VALUES_EQUAL(1, timeSeriesB.Size()); + UNIT_ASSERT_EQUAL(EMetricValueType::HISTOGRAM, timeSeriesB.GetValueType()); + + UNIT_ASSERT_VALUES_EQUAL(0, timeSeriesA.Size()); + UNIT_ASSERT_EQUAL(EMetricValueType::UNKNOWN, timeSeriesA.GetValueType()); + } + UNIT_ASSERT_VALUES_EQUAL(1, histogram->RefCount()); + } + + Y_UNIT_TEST(HistogramsUnique) { + auto ts1 = TInstant::Now(); + auto ts2 = ts1 + TDuration::Seconds(1); + auto ts3 = ts2 + TDuration::Seconds(1); + + auto h1 = MakeIntrusive<TTestHistogram>(); + auto h2 = MakeIntrusive<TTestHistogram>(); + auto h3 = MakeIntrusive<TTestHistogram>(); + + UNIT_ASSERT_VALUES_EQUAL(1, h1->RefCount()); + UNIT_ASSERT_VALUES_EQUAL(1, h2->RefCount()); + UNIT_ASSERT_VALUES_EQUAL(1, h3->RefCount()); + + { + TMetricTimeSeries timeSeries; + timeSeries.Add(ts1, h1.Get()); // drop at the head + timeSeries.Add(ts1, h1.Get()); + timeSeries.Add(ts1, h1.Get()); + + timeSeries.Add(ts2, h2.Get()); // drop in the middle + timeSeries.Add(ts2, h2.Get()); + timeSeries.Add(ts2, h2.Get()); + + timeSeries.Add(ts3, h3.Get()); // drop at the end + timeSeries.Add(ts3, h3.Get()); + timeSeries.Add(ts3, h3.Get()); + + UNIT_ASSERT_EQUAL(timeSeries.Size(), 9); + + UNIT_ASSERT_VALUES_EQUAL(4, h1->RefCount()); + UNIT_ASSERT_VALUES_EQUAL(4, h2->RefCount()); + UNIT_ASSERT_VALUES_EQUAL(4, h3->RefCount()); + + timeSeries.SortByTs(); + UNIT_ASSERT_EQUAL(timeSeries.Size(), 3); + + UNIT_ASSERT_VALUES_EQUAL(2, h1->RefCount()); + UNIT_ASSERT_VALUES_EQUAL(2, h2->RefCount()); + UNIT_ASSERT_VALUES_EQUAL(2, h3->RefCount()); + } + + UNIT_ASSERT_VALUES_EQUAL(1, h1->RefCount()); + UNIT_ASSERT_VALUES_EQUAL(1, h2->RefCount()); + UNIT_ASSERT_VALUES_EQUAL(1, h3->RefCount()); + } + + Y_UNIT_TEST(LogHistogramsUnique) { + auto ts1 = TInstant::Now(); + auto ts2 = ts1 + TDuration::Seconds(1); + auto ts3 = ts2 + TDuration::Seconds(1); + + auto h1 = MakeLogHistogram(); + auto h2 = MakeLogHistogram(); + auto h3 = MakeLogHistogram(); + + UNIT_ASSERT_VALUES_EQUAL(1, h1->RefCount()); + UNIT_ASSERT_VALUES_EQUAL(1, h2->RefCount()); + UNIT_ASSERT_VALUES_EQUAL(1, h3->RefCount()); + + { + TMetricTimeSeries timeSeries; + timeSeries.Add(ts1, h1.Get()); // drop at the head + timeSeries.Add(ts1, h1.Get()); + timeSeries.Add(ts1, h1.Get()); + + timeSeries.Add(ts2, h2.Get()); // drop in the middle + timeSeries.Add(ts2, h2.Get()); + timeSeries.Add(ts2, h2.Get()); + + timeSeries.Add(ts3, h3.Get()); // drop at the end + timeSeries.Add(ts3, h3.Get()); + timeSeries.Add(ts3, h3.Get()); + + UNIT_ASSERT_EQUAL(timeSeries.Size(), 9); + + UNIT_ASSERT_VALUES_EQUAL(4, h1->RefCount()); + UNIT_ASSERT_VALUES_EQUAL(4, h2->RefCount()); + UNIT_ASSERT_VALUES_EQUAL(4, h3->RefCount()); + + timeSeries.SortByTs(); + UNIT_ASSERT_EQUAL(timeSeries.Size(), 3); + + UNIT_ASSERT_VALUES_EQUAL(2, h1->RefCount()); + UNIT_ASSERT_VALUES_EQUAL(2, h2->RefCount()); + UNIT_ASSERT_VALUES_EQUAL(2, h3->RefCount()); + } + + UNIT_ASSERT_VALUES_EQUAL(1, h1->RefCount()); + UNIT_ASSERT_VALUES_EQUAL(1, h2->RefCount()); + UNIT_ASSERT_VALUES_EQUAL(1, h3->RefCount()); + } + + Y_UNIT_TEST(SummaryUnique) { + auto ts1 = TInstant::Now(); + auto ts2 = ts1 + TDuration::Seconds(1); + auto ts3 = ts2 + TDuration::Seconds(1); + + auto h1 = MakeSummarySnapshot(); + auto h2 = MakeSummarySnapshot(); + auto h3 = MakeSummarySnapshot(); + + UNIT_ASSERT_VALUES_EQUAL(1, h1->RefCount()); + UNIT_ASSERT_VALUES_EQUAL(1, h2->RefCount()); + UNIT_ASSERT_VALUES_EQUAL(1, h3->RefCount()); + + { + TMetricTimeSeries timeSeries; + timeSeries.Add(ts1, h1.Get()); // drop at the head + timeSeries.Add(ts1, h1.Get()); + timeSeries.Add(ts1, h1.Get()); + + timeSeries.Add(ts2, h2.Get()); // drop in the middle + timeSeries.Add(ts2, h2.Get()); + timeSeries.Add(ts2, h2.Get()); + + timeSeries.Add(ts3, h3.Get()); // drop at the end + timeSeries.Add(ts3, h3.Get()); + timeSeries.Add(ts3, h3.Get()); + + UNIT_ASSERT_EQUAL(timeSeries.Size(), 9); + + UNIT_ASSERT_VALUES_EQUAL(4, h1->RefCount()); + UNIT_ASSERT_VALUES_EQUAL(4, h2->RefCount()); + UNIT_ASSERT_VALUES_EQUAL(4, h3->RefCount()); + + timeSeries.SortByTs(); + UNIT_ASSERT_EQUAL(timeSeries.Size(), 3); + + UNIT_ASSERT_VALUES_EQUAL(2, h1->RefCount()); + UNIT_ASSERT_VALUES_EQUAL(2, h2->RefCount()); + UNIT_ASSERT_VALUES_EQUAL(2, h3->RefCount()); + } + + UNIT_ASSERT_VALUES_EQUAL(1, h1->RefCount()); + UNIT_ASSERT_VALUES_EQUAL(1, h2->RefCount()); + UNIT_ASSERT_VALUES_EQUAL(1, h3->RefCount()); + } + + Y_UNIT_TEST(HistogramsUnique2) { + auto ts1 = TInstant::Now(); + auto ts2 = ts1 + TDuration::Seconds(1); + auto ts3 = ts2 + TDuration::Seconds(1); + auto ts4 = ts3 + TDuration::Seconds(1); + auto ts5 = ts4 + TDuration::Seconds(1); + + auto h1 = MakeIntrusive<TTestHistogram>(1u); + auto h2 = MakeIntrusive<TTestHistogram>(2u); + auto h3 = MakeIntrusive<TTestHistogram>(3u); + auto h4 = MakeIntrusive<TTestHistogram>(4u); + auto h5 = MakeIntrusive<TTestHistogram>(5u); + auto h6 = MakeIntrusive<TTestHistogram>(6u); + auto h7 = MakeIntrusive<TTestHistogram>(7u); + + { + TMetricTimeSeries timeSeries; + timeSeries.Add(ts1, h1.Get()); + timeSeries.Add(ts1, h2.Get()); + + timeSeries.Add(ts2, h3.Get()); + + timeSeries.Add(ts3, h4.Get()); + timeSeries.Add(ts3, h5.Get()); + + timeSeries.Add(ts4, h6.Get()); + timeSeries.Add(ts5, h7.Get()); + + timeSeries.SortByTs(); + + UNIT_ASSERT_EQUAL(timeSeries.Size(), 5); + UNIT_ASSERT_EQUAL(timeSeries[0].GetValue().AsHistogram()->Count(), 2); + UNIT_ASSERT_EQUAL(timeSeries[1].GetValue().AsHistogram()->Count(), 3); + UNIT_ASSERT_EQUAL(timeSeries[2].GetValue().AsHistogram()->Count(), 5); + UNIT_ASSERT_EQUAL(timeSeries[3].GetValue().AsHistogram()->Count(), 6); + UNIT_ASSERT_EQUAL(timeSeries[4].GetValue().AsHistogram()->Count(), 7); + } + } + + Y_UNIT_TEST(LogHistogramsUnique2) { + auto ts1 = TInstant::Now(); + auto ts2 = ts1 + TDuration::Seconds(1); + auto ts3 = ts2 + TDuration::Seconds(1); + auto ts4 = ts3 + TDuration::Seconds(1); + auto ts5 = ts4 + TDuration::Seconds(1); + + auto h1 = MakeLogHistogram(1u); + auto h2 = MakeLogHistogram(2u); + auto h3 = MakeLogHistogram(3u); + auto h4 = MakeLogHistogram(4u); + auto h5 = MakeLogHistogram(5u); + auto h6 = MakeLogHistogram(6u); + auto h7 = MakeLogHistogram(7u); + + { + TMetricTimeSeries timeSeries; + timeSeries.Add(ts1, h1.Get()); + timeSeries.Add(ts1, h2.Get()); + + timeSeries.Add(ts2, h3.Get()); + + timeSeries.Add(ts3, h4.Get()); + timeSeries.Add(ts3, h5.Get()); + + timeSeries.Add(ts4, h6.Get()); + timeSeries.Add(ts5, h7.Get()); + + timeSeries.SortByTs(); + + UNIT_ASSERT_EQUAL(timeSeries.Size(), 5); + UNIT_ASSERT_EQUAL(timeSeries[0].GetValue().AsLogHistogram()->Count(), 2); + UNIT_ASSERT_EQUAL(timeSeries[1].GetValue().AsLogHistogram()->Count(), 3); + UNIT_ASSERT_EQUAL(timeSeries[2].GetValue().AsLogHistogram()->Count(), 5); + UNIT_ASSERT_EQUAL(timeSeries[3].GetValue().AsLogHistogram()->Count(), 6); + UNIT_ASSERT_EQUAL(timeSeries[4].GetValue().AsLogHistogram()->Count(), 7); + } + } + + Y_UNIT_TEST(SummaryUnique2) { + auto ts1 = TInstant::Now(); + auto ts2 = ts1 + TDuration::Seconds(1); + auto ts3 = ts2 + TDuration::Seconds(1); + auto ts4 = ts3 + TDuration::Seconds(1); + auto ts5 = ts4 + TDuration::Seconds(1); + + auto h1 = MakeSummarySnapshot(1u); + auto h2 = MakeSummarySnapshot(2u); + auto h3 = MakeSummarySnapshot(3u); + auto h4 = MakeSummarySnapshot(4u); + auto h5 = MakeSummarySnapshot(5u); + auto h6 = MakeSummarySnapshot(6u); + auto h7 = MakeSummarySnapshot(7u); + + { + TMetricTimeSeries timeSeries; + timeSeries.Add(ts1, h1.Get()); + timeSeries.Add(ts1, h2.Get()); + + timeSeries.Add(ts2, h3.Get()); + + timeSeries.Add(ts3, h4.Get()); + timeSeries.Add(ts3, h5.Get()); + + timeSeries.Add(ts4, h6.Get()); + timeSeries.Add(ts5, h7.Get()); + + timeSeries.SortByTs(); + + UNIT_ASSERT_EQUAL(timeSeries.Size(), 5); + UNIT_ASSERT_EQUAL(timeSeries[0].GetValue().AsSummaryDouble()->GetCount(), 2); + UNIT_ASSERT_EQUAL(timeSeries[1].GetValue().AsSummaryDouble()->GetCount(), 3); + UNIT_ASSERT_EQUAL(timeSeries[2].GetValue().AsSummaryDouble()->GetCount(), 5); + UNIT_ASSERT_EQUAL(timeSeries[3].GetValue().AsSummaryDouble()->GetCount(), 6); + UNIT_ASSERT_EQUAL(timeSeries[4].GetValue().AsSummaryDouble()->GetCount(), 7); + } + } + + Y_UNIT_TEST(TMetricValueWithType) { + // correct usage + { + double value = 1.23; + TMetricValueWithType v{value}; + + UNIT_ASSERT_VALUES_EQUAL(v.GetType(), EMetricValueType::DOUBLE); + UNIT_ASSERT_VALUES_EQUAL(v.AsDouble(), value); + } + { + ui64 value = 12; + TMetricValueWithType v{value}; + + UNIT_ASSERT_VALUES_EQUAL(v.GetType(), EMetricValueType::UINT64); + UNIT_ASSERT_VALUES_EQUAL(v.AsUint64(), value); + } + { + i64 value = i64(-12); + TMetricValueWithType v{value}; + + UNIT_ASSERT_VALUES_EQUAL(v.GetType(), EMetricValueType::INT64); + UNIT_ASSERT_VALUES_EQUAL(v.AsInt64(), value); + } + { + auto h = MakeHistogramSnapshot(); + UNIT_ASSERT_VALUES_EQUAL(h.RefCount(), 1); + + { + auto value = h.Get(); + TMetricValueWithType v{value}; + + UNIT_ASSERT_VALUES_EQUAL(h.RefCount(), 2); + + UNIT_ASSERT_VALUES_EQUAL(v.GetType(), EMetricValueType::HISTOGRAM); + UNIT_ASSERT_VALUES_EQUAL(v.AsHistogram(), value); + } + + UNIT_ASSERT_VALUES_EQUAL(h.RefCount(), 1); + } + { + auto s = MakeSummarySnapshot(); + auto value = s.Get(); + + UNIT_ASSERT_VALUES_EQUAL(s.RefCount(), 1); + + { + TMetricValueWithType v{value}; + + UNIT_ASSERT_VALUES_EQUAL(s.RefCount(), 2); + + UNIT_ASSERT_VALUES_EQUAL(v.GetType(), EMetricValueType::SUMMARY); + UNIT_ASSERT_VALUES_EQUAL(v.AsSummaryDouble(), value); + } + + UNIT_ASSERT_VALUES_EQUAL(s.RefCount(), 1); + } + { + auto s = MakeSummarySnapshot(); + auto value = s.Get(); + + UNIT_ASSERT_VALUES_EQUAL(s.RefCount(), 1); + + { + TMetricValueWithType v{value}; + + UNIT_ASSERT_VALUES_EQUAL(s.RefCount(), 2); + + v.Clear(); + UNIT_ASSERT_VALUES_EQUAL(s.RefCount(), 1); + } + + UNIT_ASSERT_VALUES_EQUAL(s.RefCount(), 1); + } + { + auto s = MakeSummarySnapshot(); + auto value = s.Get(); + + { + TMetricValueWithType v1{ui64{1}}; + + UNIT_ASSERT_VALUES_EQUAL(s.RefCount(), 1); + + { + TMetricValueWithType v2{value}; + UNIT_ASSERT_VALUES_EQUAL(s.RefCount(), 2); + + v1 = std::move(v2); + UNIT_ASSERT_VALUES_EQUAL(s.RefCount(), 2); + UNIT_ASSERT_VALUES_EQUAL(v1.AsSummaryDouble(), value); + UNIT_ASSERT_VALUES_EQUAL(v1.GetType(), EMetricValueType::SUMMARY); + UNIT_ASSERT_VALUES_EQUAL(v2.GetType(), EMetricValueType::UNKNOWN); + } + + UNIT_ASSERT_VALUES_EQUAL(s.RefCount(), 2); + } + + UNIT_ASSERT_VALUES_EQUAL(s.RefCount(), 1); + } + + // incorrect usage + { + TMetricValueWithType v{1.23}; + + UNIT_ASSERT_EXCEPTION(v.AsHistogram(), yexception); + UNIT_ASSERT_EXCEPTION(v.AsSummaryDouble(), yexception); + } + { + auto h = MakeHistogramSnapshot(); + TMetricValueWithType v{h.Get()}; + + UNIT_ASSERT_EXCEPTION(v.AsUint64(), yexception); + UNIT_ASSERT_EXCEPTION(v.AsInt64(), yexception); + UNIT_ASSERT_EXCEPTION(v.AsDouble(), yexception); + UNIT_ASSERT_EXCEPTION(v.AsSummaryDouble(), yexception); + } + { + auto s = MakeSummarySnapshot(); + TMetricValueWithType v{s.Get()}; + + UNIT_ASSERT_EXCEPTION(v.AsUint64(), yexception); + UNIT_ASSERT_EXCEPTION(v.AsInt64(), yexception); + UNIT_ASSERT_EXCEPTION(v.AsDouble(), yexception); + UNIT_ASSERT_EXCEPTION(v.AsHistogram(), yexception); + } + } +} diff --git a/library/cpp/monlib/metrics/summary_collector.cpp b/library/cpp/monlib/metrics/summary_collector.cpp new file mode 100644 index 0000000000..cae8560891 --- /dev/null +++ b/library/cpp/monlib/metrics/summary_collector.cpp @@ -0,0 +1 @@ +#include "summary_collector.h" diff --git a/library/cpp/monlib/metrics/summary_collector.h b/library/cpp/monlib/metrics/summary_collector.h new file mode 100644 index 0000000000..acba0fddf9 --- /dev/null +++ b/library/cpp/monlib/metrics/summary_collector.h @@ -0,0 +1,104 @@ +#pragma once + +#include "summary_snapshot.h" + +#include <atomic> +#include <limits> +#include <cmath> + +namespace NMonitoring { + + class ISummaryDoubleCollector { + public: + virtual ~ISummaryDoubleCollector() = default; + + virtual void Collect(double value) = 0; + + virtual ISummaryDoubleSnapshotPtr Snapshot() const = 0; + + virtual size_t SizeBytes() const = 0; + }; + + using ISummaryDoubleCollectorPtr = THolder<ISummaryDoubleCollector>; + + class TSummaryDoubleCollector final: public ISummaryDoubleCollector { + public: + TSummaryDoubleCollector() { + Sum_.store(0, std::memory_order_relaxed); + Min_.store(std::numeric_limits<double>::max(), std::memory_order_relaxed); + Max_.store(std::numeric_limits<double>::lowest(), std::memory_order_relaxed); + Count_.store(0, std::memory_order_relaxed); + } + + void Collect(double value) noexcept override { + if (std::isnan(value)) { + return; + } + UpdateSum(value); + UpdateMin(value); + UpdateMax(value); + Last_.store(value, std::memory_order_relaxed); + Count_.fetch_add(1ul, std::memory_order_relaxed); + } + + ISummaryDoubleSnapshotPtr Snapshot() const override { + return new TSummaryDoubleSnapshot( + Sum_.load(std::memory_order_relaxed), + Min_.load(std::memory_order_relaxed), + Max_.load(std::memory_order_relaxed), + Last_.load(std::memory_order_relaxed), + Count_.load(std::memory_order_relaxed)); + } + + size_t SizeBytes() const override { + return sizeof(*this); + } + + private: + std::atomic<double> Sum_; + std::atomic<double> Min_; + std::atomic<double> Max_; + std::atomic<double> Last_; + std::atomic_uint64_t Count_; + + void UpdateSum(double add) noexcept { + double newValue; + double oldValue = Sum_.load(std::memory_order_relaxed); + do { + newValue = oldValue + add; + } while (!Sum_.compare_exchange_weak( + oldValue, + newValue, + std::memory_order_release, + std::memory_order_consume)); + } + + void UpdateMin(double candidate) noexcept { + double oldValue = Min_.load(std::memory_order_relaxed); + do { + if (oldValue <= candidate) { + break; + } + } while (!Min_.compare_exchange_weak( + oldValue, + candidate, + std::memory_order_release, + std::memory_order_consume)); + } + + void UpdateMax(double candidate) noexcept { + double oldValue = Max_.load(std::memory_order_relaxed); + do { + if (oldValue >= candidate) { + break; + } + } while (!Max_.compare_exchange_weak( + oldValue, + candidate, + std::memory_order_release, + std::memory_order_consume)); + } + + }; + +} diff --git a/library/cpp/monlib/metrics/summary_collector_ut.cpp b/library/cpp/monlib/metrics/summary_collector_ut.cpp new file mode 100644 index 0000000000..191929550f --- /dev/null +++ b/library/cpp/monlib/metrics/summary_collector_ut.cpp @@ -0,0 +1,64 @@ +#include "summary_collector.h" + +#include <library/cpp/testing/unittest/registar.h> + +#include <util/random/random.h> + +#include <numeric> +#include <algorithm> + +namespace NMonitoring { + +Y_UNIT_TEST_SUITE(SummaryCollectorTest) { + + void CheckSnapshot(ISummaryDoubleSnapshotPtr snapshot, const TVector<double> values) { + const double eps = 1e-9; + + double sum = std::accumulate(values.begin(), values.end(), 0.0); + double min = *std::min_element(values.begin(), values.end()); + double max = *std::max_element(values.begin(), values.end()); + double last = values.back(); + ui64 count = values.size(); + + UNIT_ASSERT_DOUBLES_EQUAL(snapshot->GetSum(), sum, eps); + UNIT_ASSERT_DOUBLES_EQUAL(snapshot->GetMin(), min, eps); + UNIT_ASSERT_DOUBLES_EQUAL(snapshot->GetMax(), max, eps); + UNIT_ASSERT_DOUBLES_EQUAL(snapshot->GetLast(), last, eps); + UNIT_ASSERT_EQUAL(snapshot->GetCount(), count); + } + + Y_UNIT_TEST(Simple) { + { + TVector<double> test{05, -1.5, 0.0, 2.5, 0.25, -1.0}; + TSummaryDoubleCollector summary; + for (auto value : test) { + summary.Collect(value); + } + CheckSnapshot(summary.Snapshot(), test); + } + { + TVector<double> test{-1.0, 1.0, 9.0, -5000.0, 5000.0, 5.0, -5.0}; + TSummaryDoubleCollector summary; + for (auto value : test) { + summary.Collect(value); + } + CheckSnapshot(summary.Snapshot(), test); + } + } + + Y_UNIT_TEST(RandomStressTest) { + const ui32 attemts = 100; + for (ui32 i = 0; i < attemts; ++i) { + const ui32 size = 100; + TVector<double> values(size); + TSummaryDoubleCollector summary; + for (auto& value : values) { + value = RandomNumber<double>() - 0.5; + summary.Collect(value); + } + CheckSnapshot(summary.Snapshot(), values); + } + } +} + +} diff --git a/library/cpp/monlib/metrics/summary_snapshot.cpp b/library/cpp/monlib/metrics/summary_snapshot.cpp new file mode 100644 index 0000000000..0b13263337 --- /dev/null +++ b/library/cpp/monlib/metrics/summary_snapshot.cpp @@ -0,0 +1,34 @@ +#include "summary_snapshot.h" + +#include <util/stream/output.h> + +#include <iostream> + + +namespace { + +template <typename TStream> +auto& Output(TStream& o, const NMonitoring::ISummaryDoubleSnapshot& s) { + o << TStringBuf("{"); + + o << TStringBuf("sum: ") << s.GetSum() << TStringBuf(", "); + o << TStringBuf("min: ") << s.GetMin() << TStringBuf(", "); + o << TStringBuf("max: ") << s.GetMax() << TStringBuf(", "); + o << TStringBuf("last: ") << s.GetLast() << TStringBuf(", "); + o << TStringBuf("count: ") << s.GetCount(); + + o << TStringBuf("}"); + + return o; +} + +} // namespace + +std::ostream& operator<<(std::ostream& o, const NMonitoring::ISummaryDoubleSnapshot& s) { + return Output(o, s); +} + +template <> +void Out<NMonitoring::ISummaryDoubleSnapshot>(IOutputStream& o, const NMonitoring::ISummaryDoubleSnapshot& s) { + Output(o, s); +} diff --git a/library/cpp/monlib/metrics/summary_snapshot.h b/library/cpp/monlib/metrics/summary_snapshot.h new file mode 100644 index 0000000000..afcc895fd3 --- /dev/null +++ b/library/cpp/monlib/metrics/summary_snapshot.h @@ -0,0 +1,72 @@ +#pragma once + +#include <util/generic/ptr.h> + +namespace NMonitoring { + + class ISummaryDoubleSnapshot: public TAtomicRefCount<ISummaryDoubleSnapshot> { + public: + virtual ~ISummaryDoubleSnapshot() = default; + + // TODO: write documentation + + virtual ui64 GetCount() const = 0; + + virtual double GetSum() const = 0; + + virtual double GetMin() const = 0; + + virtual double GetMax() const = 0; + + virtual double GetLast() const = 0; + + virtual ui64 MemorySizeBytes() const = 0; + }; + + using ISummaryDoubleSnapshotPtr = TIntrusivePtr<ISummaryDoubleSnapshot>; + + class TSummaryDoubleSnapshot final: public ISummaryDoubleSnapshot { + public: + TSummaryDoubleSnapshot(double sum, double min, double max, double last, ui64 count) + : Sum_(sum) + , Min_(min) + , Max_(max) + , Last_(last) + , Count_(count) + {} + + ui64 GetCount() const noexcept override { + return Count_; + } + + double GetSum() const noexcept override { + return Sum_; + } + + double GetMin() const noexcept override { + return Min_; + } + + double GetMax() const noexcept override { + return Max_; + } + + virtual double GetLast() const noexcept override { + return Last_; + } + + ui64 MemorySizeBytes() const noexcept override { + return sizeof(*this); + } + + private: + double Sum_; + double Min_; + double Max_; + double Last_; + ui64 Count_; + }; + +} + +std::ostream& operator<<(std::ostream& os, const NMonitoring::ISummaryDoubleSnapshot& s); diff --git a/library/cpp/monlib/metrics/timer.h b/library/cpp/monlib/metrics/timer.h new file mode 100644 index 0000000000..5c4e26e37b --- /dev/null +++ b/library/cpp/monlib/metrics/timer.h @@ -0,0 +1,127 @@ +#pragma once + +#include "metric.h" + +#include <util/generic/typetraits.h> + +#include <chrono> + + +namespace NMonitoring { + + /** + * A timing scope to record elapsed time since creation. + */ + template <typename TMetric, + typename Resolution = std::chrono::milliseconds, + typename Clock = std::chrono::high_resolution_clock> + class TMetricTimerScope { + public: + explicit TMetricTimerScope(TMetric* metric) + : Metric_(metric) + , StartTime_(Clock::now()) + { + Y_ENSURE(Metric_); + } + + TMetricTimerScope(TMetricTimerScope&) = delete; + TMetricTimerScope& operator=(const TMetricTimerScope&) = delete; + + TMetricTimerScope(TMetricTimerScope&& other) { + *this = std::move(other); + } + + TMetricTimerScope& operator=(TMetricTimerScope&& other) { + Metric_ = other.Metric_; + other.Metric_ = nullptr; + StartTime_ = std::move(other.StartTime_); + + return *this; + } + + void Record() { + Y_VERIFY_DEBUG(Metric_); + if (Metric_ == nullptr) { + return; + } + + auto duration = std::chrono::duration_cast<Resolution>(Clock::now() - StartTime_).count(); + if constexpr (std::is_same<TMetric, TGauge>::value) { + Metric_->Set(duration); + } else if constexpr (std::is_same<TMetric, TIntGauge>::value) { + Metric_->Set(duration); + } else if constexpr (std::is_same<TMetric, TCounter>::value) { + Metric_->Add(duration); + } else if constexpr (std::is_same<TMetric, TRate>::value) { + Metric_->Add(duration); + } else if constexpr (std::is_same<TMetric, THistogram>::value) { + Metric_->Record(duration); + } else { + static_assert(TDependentFalse<TMetric>, "Not supported metric type"); + } + + Metric_ = nullptr; + } + + ~TMetricTimerScope() { + if (Metric_ == nullptr) { + return; + } + + Record(); + } + + private: + TMetric* Metric_{nullptr}; + typename Clock::time_point StartTime_; + }; + + /** + * @brief A class that is supposed to use to measure execution time of an asynchronuous operation. + * + * In order to be able to capture an object into a lambda which is then passed to TFuture::Subscribe/Apply, + * the object must be copy constructible (limitation of the std::function class). So, we cannot use the TMetricTimerScope + * with the abovementioned functions without storing it in a shared pointer or somewhere else. This class works around this + * issue with wrapping the timer with a auto_ptr-like hack Also, Record is const so that one doesn't need to make every lambda mutable + * just to record time measurement. + */ + template <typename TMetric, + typename Resolution = std::chrono::milliseconds, + typename Clock = std::chrono::high_resolution_clock> + class TFutureFriendlyTimer { + public: + explicit TFutureFriendlyTimer(TMetric* metric) + : Impl_{metric} + { + } + + TFutureFriendlyTimer(const TFutureFriendlyTimer& other) + : Impl_{std::move(other.Impl_)} + { + } + + TFutureFriendlyTimer& operator=(const TFutureFriendlyTimer& other) { + Impl_ = std::move(other.Impl_); + } + + TFutureFriendlyTimer(TFutureFriendlyTimer&&) = default; + TFutureFriendlyTimer& operator=(TFutureFriendlyTimer&& other) = default; + + void Record() const { + Impl_.Record(); + } + + private: + mutable TMetricTimerScope<TMetric, Resolution, Clock> Impl_; + }; + + template <typename TMetric> + TMetricTimerScope<TMetric> ScopeTimer(TMetric* metric) { + return TMetricTimerScope<TMetric>{metric}; + } + + template <typename TMetric> + TFutureFriendlyTimer<TMetric> FutureTimer(TMetric* metric) { + return TFutureFriendlyTimer<TMetric>{metric}; + } +} diff --git a/library/cpp/monlib/metrics/timer_ut.cpp b/library/cpp/monlib/metrics/timer_ut.cpp new file mode 100644 index 0000000000..c244a8c9e1 --- /dev/null +++ b/library/cpp/monlib/metrics/timer_ut.cpp @@ -0,0 +1,157 @@ +#include "timer.h" + +#include <library/cpp/testing/unittest/registar.h> +#include <library/cpp/threading/future/async.h> +#include <library/cpp/threading/future/future.h> + +using namespace NMonitoring; +using namespace NThreading; + +Y_UNIT_TEST_SUITE(TTimerTest) { + + using namespace std::chrono; + + struct TTestClock { + using time_point = time_point<high_resolution_clock>; + + static time_point TimePoint; + + static time_point now() { + return TimePoint; + } + }; + + TTestClock::time_point TTestClock::TimePoint; + + + Y_UNIT_TEST(Gauge) { + TTestClock::TimePoint = TTestClock::time_point::min(); + + TGauge gauge(0); + { + TMetricTimerScope<TGauge, milliseconds, TTestClock> t{&gauge}; + TTestClock::TimePoint += milliseconds(10); + } + UNIT_ASSERT_EQUAL(10, gauge.Get()); + + { + TMetricTimerScope<TGauge, milliseconds, TTestClock> t{&gauge}; + TTestClock::TimePoint += milliseconds(20); + } + UNIT_ASSERT_EQUAL(20, gauge.Get()); + } + + Y_UNIT_TEST(IntGauge) { + TTestClock::TimePoint = TTestClock::time_point::min(); + + TIntGauge gauge(0); + { + TMetricTimerScope<TIntGauge, milliseconds, TTestClock> t{&gauge}; + TTestClock::TimePoint += milliseconds(10); + } + UNIT_ASSERT_EQUAL(10, gauge.Get()); + + { + TMetricTimerScope<TIntGauge, milliseconds, TTestClock> t{&gauge}; + TTestClock::TimePoint += milliseconds(20); + } + UNIT_ASSERT_EQUAL(20, gauge.Get()); + } + + Y_UNIT_TEST(CounterNew) { + TTestClock::TimePoint = TTestClock::time_point::min(); + + TCounter counter(0); + { + TMetricTimerScope<TCounter, milliseconds, TTestClock> t{&counter}; + TTestClock::TimePoint += milliseconds(10); + } + UNIT_ASSERT_EQUAL(10, counter.Get()); + + { + TMetricTimerScope<TCounter, milliseconds, TTestClock> t{&counter}; + TTestClock::TimePoint += milliseconds(20); + } + UNIT_ASSERT_EQUAL(30, counter.Get()); + } + + Y_UNIT_TEST(Rate) { + TTestClock::TimePoint = TTestClock::time_point::min(); + + TRate rate(0); + { + TMetricTimerScope<TRate, milliseconds, TTestClock> t{&rate}; + TTestClock::TimePoint += milliseconds(10); + } + UNIT_ASSERT_EQUAL(10, rate.Get()); + + { + TMetricTimerScope<TRate, milliseconds, TTestClock> t{&rate}; + TTestClock::TimePoint += milliseconds(20); + } + UNIT_ASSERT_EQUAL(30, rate.Get()); + } + + Y_UNIT_TEST(Histogram) { + TTestClock::TimePoint = TTestClock::time_point::min(); + + auto assertHistogram = [](const TVector<ui64>& expected, IHistogramSnapshotPtr snapshot) { + UNIT_ASSERT_EQUAL(expected.size(), snapshot->Count()); + for (size_t i = 0; i < expected.size(); ++i) { + UNIT_ASSERT_EQUAL(expected[i], snapshot->Value(i)); + } + }; + + THistogram histogram(ExplicitHistogram({10, 20, 30}), true); + { + TMetricTimerScope<THistogram, milliseconds, TTestClock> t{&histogram}; + TTestClock::TimePoint += milliseconds(5); + } + assertHistogram({1, 0, 0, 0}, histogram.TakeSnapshot()); + + { + TMetricTimerScope<THistogram, milliseconds, TTestClock> t{&histogram}; + TTestClock::TimePoint += milliseconds(15); + } + assertHistogram({1, 1, 0, 0}, histogram.TakeSnapshot()); + } + + Y_UNIT_TEST(Moving) { + TTestClock::TimePoint = TTestClock::time_point::min(); + + TCounter counter(0); + { + TMetricTimerScope<TCounter, milliseconds, TTestClock> t{&counter}; + [tt = std::move(t)] { + TTestClock::TimePoint += milliseconds(5); + Y_UNUSED(tt); + }(); + + TTestClock::TimePoint += milliseconds(10); + } + + UNIT_ASSERT_EQUAL(counter.Get(), 5); + } + + Y_UNIT_TEST(MovingIntoApply) { + TTestClock::TimePoint = TTestClock::time_point::min(); + auto pool = CreateThreadPool(1); + + TCounter counter(0); + { + TFutureFriendlyTimer<TCounter, milliseconds, TTestClock> t{&counter}; + + auto f = Async([=] { + return; + }, *pool).Apply([tt = t] (auto) { + TTestClock::TimePoint += milliseconds(5); + tt.Record(); + }); + + f.Wait(); + TTestClock::TimePoint += milliseconds(10); + } + + UNIT_ASSERT_EQUAL(counter.Get(), 5); + } +} diff --git a/library/cpp/monlib/metrics/ut/histograms.json b/library/cpp/monlib/metrics/ut/histograms.json new file mode 100644 index 0000000000..a6e8b78fea --- /dev/null +++ b/library/cpp/monlib/metrics/ut/histograms.json @@ -0,0 +1,61 @@ +{ + "commonLabels": + { + "common":"label" + }, + "sensors": + [ + { + "kind":"HIST", + "labels": + { + "sensor":"readTimeMillis" + }, + "hist": + { + "bounds": + [ + 1, + 2, + 4, + 8 + ], + "buckets": + [ + 2, + 1, + 2, + 4 + ], + "inf":91 + } + }, + { + "kind":"HIST_RATE", + "labels": + { + "sensor":"writeTimeMillis" + }, + "hist": + { + "bounds": + [ + 1, + 5, + 15, + 20, + 25 + ], + "buckets": + [ + 2, + 4, + 10, + 5, + 5 + ], + "inf":74 + } + } + ] +} diff --git a/library/cpp/monlib/metrics/ut/ya.make b/library/cpp/monlib/metrics/ut/ya.make new file mode 100644 index 0000000000..aec9974fbd --- /dev/null +++ b/library/cpp/monlib/metrics/ut/ya.make @@ -0,0 +1,32 @@ +UNITTEST_FOR(library/cpp/monlib/metrics) + +OWNER( + jamel + g:solomon +) + +SRCS( + ewma_ut.cpp + fake_ut.cpp + histogram_collector_ut.cpp + labels_ut.cpp + log_histogram_collector_ut.cpp + metric_registry_ut.cpp + metric_sub_registry_ut.cpp + metric_value_ut.cpp + summary_collector_ut.cpp + timer_ut.cpp +) + +RESOURCE( + histograms.json /histograms.json +) + +PEERDIR( + library/cpp/resource + library/cpp/monlib/encode/protobuf + library/cpp/monlib/encode/json + library/cpp/threading/future +) + +END() diff --git a/library/cpp/monlib/metrics/ya.make b/library/cpp/monlib/metrics/ya.make new file mode 100644 index 0000000000..0e1fa143f9 --- /dev/null +++ b/library/cpp/monlib/metrics/ya.make @@ -0,0 +1,26 @@ +LIBRARY() + +OWNER( + g:solomon + jamel +) + +GENERATE_ENUM_SERIALIZATION_WITH_HEADER(metric_value_type.h) + +SRCS( + ewma.cpp + fake.cpp + histogram_collector_explicit.cpp + histogram_collector_exponential.cpp + histogram_collector_linear.cpp + histogram_snapshot.cpp + log_histogram_snapshot.cpp + labels.cpp + metric_registry.cpp + metric_consumer.cpp + metric_type.cpp + metric_value.cpp + summary_snapshot.cpp +) + +END() diff --git a/library/cpp/monlib/service/auth.cpp b/library/cpp/monlib/service/auth.cpp new file mode 100644 index 0000000000..ddabcfbbf7 --- /dev/null +++ b/library/cpp/monlib/service/auth.cpp @@ -0,0 +1,22 @@ +#include "auth.h" + +#include <util/generic/hash_set.h> + + +namespace NMonitoring { +namespace { + class TFakeAuthProvider final: public IAuthProvider { + public: + TAuthResult Check(const IHttpRequest&) override { + return TAuthResult::Ok(); + } + }; + +} // namespace + +THolder<IAuthProvider> CreateFakeAuth() { + return MakeHolder<TFakeAuthProvider>(); +} + + +} // namespace NMonitoring diff --git a/library/cpp/monlib/service/auth.h b/library/cpp/monlib/service/auth.h new file mode 100644 index 0000000000..ae53b8bd8e --- /dev/null +++ b/library/cpp/monlib/service/auth.h @@ -0,0 +1,48 @@ +#pragma once + +#include "mon_service_http_request.h" + +namespace NMonitoring { + enum class EAuthType { + None = 0, + Tvm = 1, + }; + + struct TAuthResult { + enum class EStatus { + NoCredentials = 0, + Denied, + Ok, + }; + + TAuthResult(EStatus status) + : Status{status} + { + } + + static TAuthResult Denied() { + return TAuthResult(EStatus::Denied); + } + + static TAuthResult NoCredentials() { + return TAuthResult(EStatus::NoCredentials); + } + + static TAuthResult Ok() { + return TAuthResult(EStatus::Ok); + } + + explicit operator bool() const { + return Status == EStatus::Ok; + } + + EStatus Status{EStatus::NoCredentials}; + }; + + struct IAuthProvider { + virtual ~IAuthProvider() = default; + virtual TAuthResult Check(const IHttpRequest& req) = 0; + }; + + THolder<IAuthProvider> CreateFakeAuth(); +} // namespace NMonitoring diff --git a/library/cpp/monlib/service/auth/tvm/auth.cpp b/library/cpp/monlib/service/auth/tvm/auth.cpp new file mode 100644 index 0000000000..e071c11ebc --- /dev/null +++ b/library/cpp/monlib/service/auth/tvm/auth.cpp @@ -0,0 +1,93 @@ +#include "auth.h" + +#include <util/generic/hash_set.h> + + +using namespace NTvmAuth; + + +namespace NMonitoring { +namespace { + template <class TTvmClientPtr = THolder<TTvmClient>> + class TTvmManager final: public ITvmManager { + public: + TTvmManager(NTvmApi::TClientSettings settings, TVector<TTvmId> clients, TLoggerPtr logger) + : AllowedClients_{clients.begin(), clients.end()} + , Tvm_(new TTvmClient{std::move(settings), std::move(logger)}) + { + } + + TTvmManager(NTvmTool::TClientSettings settings, TVector<TTvmId> clients, TLoggerPtr logger) + : AllowedClients_{clients.begin(), clients.end()} + , Tvm_(new TTvmClient{std::move(settings), std::move(logger)}) + { + } + + TTvmManager(TTvmClientPtr tvm, TVector<TTvmId> clients) + : AllowedClients_{clients.begin(), clients.end()} + , Tvm_(std::move(tvm)) + { + } + + bool IsAllowedClient(TTvmId clientId) override { + return AllowedClients_.contains(clientId); + } + + TCheckedServiceTicket CheckServiceTicket(TStringBuf ticket) override { + return Tvm_->CheckServiceTicket(ticket); + } + + private: + THashSet<TTvmId> AllowedClients_; + TTvmClientPtr Tvm_; + }; + + class TTvmAuthProvider final: public IAuthProvider { + public: + TTvmAuthProvider(THolder<ITvmManager> manager) + : TvmManager_{std::move(manager)} + { + } + + TAuthResult Check(const IHttpRequest& req) override { + auto ticketHeader = req.GetHeaders().FindHeader("X-Ya-Service-Ticket"); + if (!ticketHeader) { + return TAuthResult::NoCredentials(); + } + + const auto ticket = TvmManager_->CheckServiceTicket(ticketHeader->Value()); + if (!ticket) { + return TAuthResult::Denied(); + } + + return TvmManager_->IsAllowedClient(ticket.GetSrc()) + ? TAuthResult::Ok() + : TAuthResult::Denied(); + } + + private: + THolder<ITvmManager> TvmManager_; + }; +} // namespace + +THolder<ITvmManager> CreateDefaultTvmManager(NTvmApi::TClientSettings settings, TVector<TTvmId> allowedClients, TLoggerPtr logger) { + return MakeHolder<TTvmManager<>>(std::move(settings), std::move(allowedClients), std::move(logger)); +} + +THolder<ITvmManager> CreateDefaultTvmManager(NTvmTool::TClientSettings settings, TVector<TTvmId> allowedClients, TLoggerPtr logger) { + return MakeHolder<TTvmManager<>>(std::move(settings), std::move(allowedClients), std::move(logger)); +} + +THolder<ITvmManager> CreateDefaultTvmManager(TAtomicSharedPtr<NTvmAuth::TTvmClient> client, TVector<TTvmId> allowedClients) { + return MakeHolder<TTvmManager<TAtomicSharedPtr<NTvmAuth::TTvmClient>>>(std::move(client), std::move(allowedClients)); +} + +THolder<ITvmManager> CreateDefaultTvmManager(std::shared_ptr<NTvmAuth::TTvmClient> client, TVector<TTvmId> allowedClients) { + return MakeHolder<TTvmManager<std::shared_ptr<NTvmAuth::TTvmClient>>>(std::move(client), std::move(allowedClients)); +} + +THolder<IAuthProvider> CreateTvmAuth(THolder<ITvmManager> manager) { + return MakeHolder<TTvmAuthProvider>(std::move(manager)); +} + +} // namespace NMonitoring diff --git a/library/cpp/monlib/service/auth/tvm/auth.h b/library/cpp/monlib/service/auth/tvm/auth.h new file mode 100644 index 0000000000..432beff9d6 --- /dev/null +++ b/library/cpp/monlib/service/auth/tvm/auth.h @@ -0,0 +1,33 @@ +#pragma once + +#include <library/cpp/monlib/service/mon_service_http_request.h> +#include <library/cpp/monlib/service/auth.h> +#include <library/cpp/tvmauth/client/facade.h> + +namespace NMonitoring { + struct ITvmManager { + virtual ~ITvmManager() = default; + virtual bool IsAllowedClient(NTvmAuth::TTvmId clientId) = 0; + virtual NTvmAuth::TCheckedServiceTicket CheckServiceTicket(TStringBuf ticket) = 0; + }; + + THolder<ITvmManager> CreateDefaultTvmManager( + NTvmAuth::NTvmApi::TClientSettings settings, + TVector<NTvmAuth::TTvmId> allowedClients, + NTvmAuth::TLoggerPtr logger = NTvmAuth::TDevNullLogger::IAmBrave()); + + THolder<ITvmManager> CreateDefaultTvmManager( + NTvmAuth::NTvmTool::TClientSettings settings, + TVector<NTvmAuth::TTvmId> allowedClients, + NTvmAuth::TLoggerPtr logger = NTvmAuth::TDevNullLogger::IAmBrave()); + + THolder<ITvmManager> CreateDefaultTvmManager( + TAtomicSharedPtr<NTvmAuth::TTvmClient> client, + TVector<NTvmAuth::TTvmId> allowedClients); + + THolder<ITvmManager> CreateDefaultTvmManager( + std::shared_ptr<NTvmAuth::TTvmClient> client, + TVector<NTvmAuth::TTvmId> allowedClients); + + THolder<IAuthProvider> CreateTvmAuth(THolder<ITvmManager> tvmManager); +} // namespace NMonitoring diff --git a/library/cpp/monlib/service/auth/tvm/ya.make b/library/cpp/monlib/service/auth/tvm/ya.make new file mode 100644 index 0000000000..4437a65b62 --- /dev/null +++ b/library/cpp/monlib/service/auth/tvm/ya.make @@ -0,0 +1,14 @@ +LIBRARY() + +OWNER(g:solomon) + +SRCS( + auth.cpp +) + +PEERDIR( + library/cpp/tvmauth/client + library/cpp/monlib/service +) + +END() diff --git a/library/cpp/monlib/service/format.cpp b/library/cpp/monlib/service/format.cpp new file mode 100644 index 0000000000..b0d6a10246 --- /dev/null +++ b/library/cpp/monlib/service/format.cpp @@ -0,0 +1 @@ +#include "format.h" diff --git a/library/cpp/monlib/service/format.h b/library/cpp/monlib/service/format.h new file mode 100644 index 0000000000..0044b586b1 --- /dev/null +++ b/library/cpp/monlib/service/format.h @@ -0,0 +1,86 @@ +#pragma once + +#include <library/cpp/monlib/encode/format.h> + +#include <util/string/ascii.h> +#include <util/generic/yexception.h> +#include <util/generic/typetraits.h> + +namespace NMonitoring { + namespace NPrivate { + Y_HAS_MEMBER(Name, Name); + Y_HAS_MEMBER(second, Second); + } // namespace NPrivate + + template <typename TRequest> + ECompression ParseCompression(const TRequest& req) { + auto&& headers = req.GetHeaders(); + + constexpr auto isPlainPair = NPrivate::THasSecond<std::decay_t<decltype(*headers.begin())>>::value; + + auto it = FindIf(std::begin(headers), std::end(headers), + [=] (const auto& h) { + if constexpr (NPrivate::THasName<std::decay_t<decltype(h)>>::value) { + return AsciiCompareIgnoreCase(h.Name(), TStringBuf("accept-encoding")) == 0; + } else if (isPlainPair) { + return AsciiCompareIgnoreCase(h.first, TStringBuf("accept-encoding")) == 0; + } + }); + + if (it == std::end(headers)) { + return NMonitoring::ECompression::IDENTITY; + } + + NMonitoring::ECompression val{}; + if constexpr (isPlainPair) { + val = CompressionFromAcceptEncodingHeader(it->second); + } else { + val = CompressionFromAcceptEncodingHeader(it->Value()); + } + + return val != NMonitoring::ECompression::UNKNOWN + ? val + : NMonitoring::ECompression::IDENTITY; + } + + template <typename TRequest> + NMonitoring::EFormat ParseFormat(const TRequest& req) { + auto&& formatStr = req.GetParams() + .Get(TStringBuf("format")); + + if (!formatStr.empty()) { + if (formatStr == TStringBuf("SPACK")) { + return EFormat::SPACK; + } else if (formatStr == TStringBuf("TEXT")) { + return EFormat::TEXT; + } else if (formatStr == TStringBuf("JSON")) { + return EFormat::JSON; + } else { + ythrow yexception() << "unknown format: " << formatStr << ". Only spack is supported here"; + } + } + + auto&& headers = req.GetHeaders(); + constexpr auto isPlainPair = NPrivate::THasSecond<std::decay_t<decltype(*headers.begin())>>::value; + + auto it = FindIf(std::begin(headers), std::end(headers), + [=] (const auto& h) { + if constexpr (NPrivate::THasName<std::decay_t<decltype(h)>>::value) { + return AsciiCompareIgnoreCase(h.Name(), TStringBuf("accept")) == 0; + } else if (isPlainPair) { + return AsciiCompareIgnoreCase(h.first, TStringBuf("accept")) == 0; + } + }); + + if (it != std::end(headers)) { + if constexpr (isPlainPair) { + return FormatFromAcceptHeader(it->second); + } else { + return FormatFromAcceptHeader(it->Value()); + } + } + + return EFormat::UNKNOWN; + } + +} // namespace NMonitoring diff --git a/library/cpp/monlib/service/mon_service_http_request.cpp b/library/cpp/monlib/service/mon_service_http_request.cpp new file mode 100644 index 0000000000..5d805631d9 --- /dev/null +++ b/library/cpp/monlib/service/mon_service_http_request.cpp @@ -0,0 +1,85 @@ +#include "mon_service_http_request.h" +#include "monservice.h" + +using namespace NMonitoring; + +IMonHttpRequest::~IMonHttpRequest() { +} + +TMonService2HttpRequest::~TMonService2HttpRequest() { +} + +TString TMonService2HttpRequest::GetServiceTitle() const { + return MonService->GetTitle(); +} + +IOutputStream& TMonService2HttpRequest::Output() { + return *Out; +} + +HTTP_METHOD TMonService2HttpRequest::GetMethod() const { + return HttpRequest->GetMethod(); +} + +TStringBuf TMonService2HttpRequest::GetPathInfo() const { + return PathInfo; +} + +TStringBuf TMonService2HttpRequest::GetPath() const { + return HttpRequest->GetPath(); +} + +TStringBuf TMonService2HttpRequest::GetUri() const { + return HttpRequest->GetURI(); +} + +const TCgiParameters& TMonService2HttpRequest::GetParams() const { + return HttpRequest->GetParams(); +} + +const TCgiParameters& TMonService2HttpRequest::GetPostParams() const { + return HttpRequest->GetPostParams(); +} + +TStringBuf TMonService2HttpRequest::GetHeader(TStringBuf name) const { + const THttpHeaders& headers = HttpRequest->GetHeaders(); + const THttpInputHeader* header = headers.FindHeader(name); + if (header != nullptr) { + return header->Value(); + } + return TStringBuf(); +} + +const THttpHeaders& TMonService2HttpRequest::GetHeaders() const { + return HttpRequest->GetHeaders(); +} + +TString TMonService2HttpRequest::GetRemoteAddr() const { + return HttpRequest->GetRemoteAddr(); +} + +TStringBuf TMonService2HttpRequest::GetCookie(TStringBuf name) const { + TStringBuf cookie = GetHeader("Cookie"); + size_t size = cookie.size(); + size_t start = 0; + while (start < size) { + size_t semicolon = cookie.find(';', start); + auto pair = cookie.substr(start, semicolon - start); + if (!pair.empty()) { + size_t equal = pair.find('='); + if (equal != TStringBuf::npos) { + auto cookieName = pair.substr(0, equal); + if (cookieName == name) { + size_t valueStart = equal + 1; + auto cookieValue = pair.substr(valueStart, semicolon - valueStart); + return cookieValue; + } + } + start = semicolon; + while (start < size && (cookie[start] == ' ' || cookie[start] == ';')) { + ++start; + } + } + } + return TStringBuf(); +} diff --git a/library/cpp/monlib/service/mon_service_http_request.h b/library/cpp/monlib/service/mon_service_http_request.h new file mode 100644 index 0000000000..b4f2f8f0c5 --- /dev/null +++ b/library/cpp/monlib/service/mon_service_http_request.h @@ -0,0 +1,90 @@ +#pragma once + +#include "service.h" + +#include <util/stream/output.h> + +namespace NMonitoring { + class TMonService2; + class IMonPage; + + // XXX: IHttpRequest is already taken + struct IMonHttpRequest { + virtual ~IMonHttpRequest(); + + virtual IOutputStream& Output() = 0; + + virtual HTTP_METHOD GetMethod() const = 0; + virtual TStringBuf GetPath() const = 0; + virtual TStringBuf GetPathInfo() const = 0; + virtual TStringBuf GetUri() const = 0; + virtual const TCgiParameters& GetParams() const = 0; + virtual const TCgiParameters& GetPostParams() const = 0; + virtual TStringBuf GetPostContent() const = 0; + virtual const THttpHeaders& GetHeaders() const = 0; + virtual TStringBuf GetHeader(TStringBuf name) const = 0; + virtual TStringBuf GetCookie(TStringBuf name) const = 0; + virtual TString GetRemoteAddr() const = 0; + + virtual TString GetServiceTitle() const = 0; + + virtual IMonPage* GetPage() const = 0; + + virtual IMonHttpRequest* MakeChild(IMonPage* page, const TString& pathInfo) const = 0; + }; + + struct TMonService2HttpRequest: IMonHttpRequest { + IOutputStream* const Out; + const IHttpRequest* const HttpRequest; + TMonService2* const MonService; + IMonPage* const MonPage; + const TString PathInfo; + TMonService2HttpRequest* const Parent; + + TMonService2HttpRequest( + IOutputStream* out, const IHttpRequest* httpRequest, + TMonService2* monService, IMonPage* monPage, + const TString& pathInfo, + TMonService2HttpRequest* parent) + : Out(out) + , HttpRequest(httpRequest) + , MonService(monService) + , MonPage(monPage) + , PathInfo(pathInfo) + , Parent(parent) + { + } + + ~TMonService2HttpRequest() override; + + IOutputStream& Output() override; + HTTP_METHOD GetMethod() const override; + TStringBuf GetPath() const override; + TStringBuf GetPathInfo() const override; + TStringBuf GetUri() const override; + const TCgiParameters& GetParams() const override; + const TCgiParameters& GetPostParams() const override; + TStringBuf GetPostContent() const override { + return HttpRequest->GetPostContent(); + } + + TStringBuf GetHeader(TStringBuf name) const override; + TStringBuf GetCookie(TStringBuf name) const override; + const THttpHeaders& GetHeaders() const override; + TString GetRemoteAddr() const override; + + IMonPage* GetPage() const override { + return MonPage; + } + + TMonService2HttpRequest* MakeChild(IMonPage* page, const TString& pathInfo) const override { + return new TMonService2HttpRequest{ + Out, HttpRequest, MonService, page, + pathInfo, const_cast<TMonService2HttpRequest*>(this) + }; + } + + TString GetServiceTitle() const override; + }; + +} diff --git a/library/cpp/monlib/service/monservice.cpp b/library/cpp/monlib/service/monservice.cpp new file mode 100644 index 0000000000..d1b9cda1d2 --- /dev/null +++ b/library/cpp/monlib/service/monservice.cpp @@ -0,0 +1,129 @@ +#include "monservice.h" + +#include <library/cpp/malloc/api/malloc.h> +#include <library/cpp/string_utils/base64/base64.h> +#include <library/cpp/svnversion/svnversion.h> + +#include <util/generic/map.h> +#include <util/generic/ptr.h> +#include <util/system/hostname.h> + +#include <google/protobuf/text_format.h> + +using namespace NMonitoring; + +TMonService2::TMonService2(ui16 port, const TString& host, ui32 threads, const TString& title, THolder<IAuthProvider> auth) + : TMonService2(HttpServerOptions(port, host, threads), title, std::move(auth)) +{ +} + +TMonService2::TMonService2(const THttpServerOptions& options, const TString& title, THolder<IAuthProvider> auth) + : NMonitoring::TMtHttpServer(options, std::bind(&TMonService2::ServeRequest, this, std::placeholders::_1, std::placeholders::_2)) + , Title(title) + , IndexMonPage(new TIndexMonPage("", Title)) + , AuthProvider_{std::move(auth)} +{ + Y_VERIFY(!!title); + time_t t = time(nullptr); + ctime_r(&t, StartTime); +} + +TMonService2::TMonService2(const THttpServerOptions& options, TSimpleSharedPtr<IThreadPool> pool, const TString& title, THolder<IAuthProvider> auth) + : NMonitoring::TMtHttpServer(options, std::bind(&TMonService2::ServeRequest, this, std::placeholders::_1, std::placeholders::_2), std::move(pool)) + , Title(title) + , IndexMonPage(new TIndexMonPage("", Title)) + , AuthProvider_{std::move(auth)} +{ + Y_VERIFY(!!title); + time_t t = time(nullptr); + ctime_r(&t, StartTime); +} + +TMonService2::TMonService2(ui16 port, ui32 threads, const TString& title, THolder<IAuthProvider> auth) + : TMonService2(port, TString(), threads, title, std::move(auth)) +{ +} + +TMonService2::TMonService2(ui16 port, const TString& title, THolder<IAuthProvider> auth) + : TMonService2(port, TString(), 0, title, std::move(auth)) +{ +} + +void TMonService2::OutputIndex(IOutputStream& out) { + IndexMonPage->OutputIndex(out, true); +} + +void TMonService2::OutputIndexPage(IOutputStream& out) { + out << HTTPOKHTML; + out << "<html>\n"; + IndexMonPage->OutputHead(out); + OutputIndexBody(out); + out << "</html>\n"; +} + +void TMonService2::OutputIndexBody(IOutputStream& out) { + out << "<body>\n"; + + // part of common navbar + out << "<ol class='breadcrumb'>\n"; + out << "<li class='active'>" << Title << "</li>\n"; + out << "</ol>\n"; + + out << "<div class='container'>\n" + << "<h2>" << Title << "</h2>\n"; + OutputIndex(out); + out + << "<div>\n" + << "</body>\n"; +} + +void TMonService2::ServeRequest(IOutputStream& out, const NMonitoring::IHttpRequest& request) { + TString path = request.GetPath(); + Y_VERIFY(path.StartsWith('/')); + + if (AuthProvider_) { + const auto authResult = AuthProvider_->Check(request); + switch (authResult.Status) { + case TAuthResult::EStatus::NoCredentials: + out << HTTPUNAUTHORIZED; + return; + case TAuthResult::EStatus::Denied: + out << HTTPFORBIDDEN; + return; + case TAuthResult::EStatus::Ok: + break; + } + } + + if (path == "/") { + OutputIndexPage(out); + } else { + TMonService2HttpRequest monService2HttpRequest( + &out, &request, this, IndexMonPage.Get(), path, nullptr); + IndexMonPage->Output(monService2HttpRequest); + } +} + +void TMonService2::Register(IMonPage* page) { + IndexMonPage->Register(page); +} + +void TMonService2::Register(TMonPagePtr page) { + IndexMonPage->Register(std::move(page)); +} + +TIndexMonPage* TMonService2::RegisterIndexPage(const TString& path, const TString& title) { + return IndexMonPage->RegisterIndexPage(path, title); +} + +IMonPage* TMonService2::FindPage(const TString& relativePath) { + return IndexMonPage->FindPage(relativePath); +} + +TIndexMonPage* TMonService2::FindIndexPage(const TString& relativePath) { + return IndexMonPage->FindIndexPage(relativePath); +} + +void TMonService2::SortPages() { + IndexMonPage->SortPages(); +} diff --git a/library/cpp/monlib/service/monservice.h b/library/cpp/monlib/service/monservice.h new file mode 100644 index 0000000000..8f5e52fcdb --- /dev/null +++ b/library/cpp/monlib/service/monservice.h @@ -0,0 +1,73 @@ +#pragma once + +#include "service.h" +#include "auth.h" +#include "mon_service_http_request.h" + +#include <library/cpp/monlib/service/pages/index_mon_page.h> +#include <library/cpp/monlib/service/pages/mon_page.h> + +#include <util/system/progname.h> + +#include <functional> + +namespace NMonitoring { + class TMonService2: public TMtHttpServer { + protected: + const TString Title; + char StartTime[26]; + TIntrusivePtr<TIndexMonPage> IndexMonPage; + THolder<IAuthProvider> AuthProvider_; + + public: + static THttpServerOptions HttpServerOptions(ui16 port, const TString& host, ui32 threads) { + THttpServerOptions opts(port); + if (!host.empty()) { + opts.SetHost(host); + } + opts.SetClientTimeout(TDuration::Minutes(1)); + opts.EnableCompression(true); + opts.SetThreads(threads); + opts.SetMaxConnections(std::max<ui32>(100, threads)); + opts.EnableRejectExcessConnections(true); + return opts; + } + + static THttpServerOptions HttpServerOptions(ui16 port, ui32 threads) { + return HttpServerOptions(port, TString(), threads); + } + + public: + explicit TMonService2(ui16 port, const TString& title = GetProgramName(), THolder<IAuthProvider> auth = nullptr); + explicit TMonService2(ui16 port, ui32 threads, const TString& title = GetProgramName(), THolder<IAuthProvider> auth = nullptr); + explicit TMonService2(ui16 port, const TString& host, ui32 threads, const TString& title = GetProgramName(), THolder<IAuthProvider> auth = nullptr); + explicit TMonService2(const THttpServerOptions& options, const TString& title = GetProgramName(), THolder<IAuthProvider> auth = nullptr); + explicit TMonService2(const THttpServerOptions& options, TSimpleSharedPtr<IThreadPool> pool, const TString& title = GetProgramName(), THolder<IAuthProvider> auth = nullptr); + + ~TMonService2() override { + } + + const char* GetStartTime() const { + return StartTime; + } + + const TString& GetTitle() const { + return Title; + } + + virtual void ServeRequest(IOutputStream& out, const NMonitoring::IHttpRequest& request); + virtual void OutputIndex(IOutputStream& out); + virtual void OutputIndexPage(IOutputStream& out); + virtual void OutputIndexBody(IOutputStream& out); + + void Register(IMonPage* page); + void Register(TMonPagePtr page); + + TIndexMonPage* RegisterIndexPage(const TString& path, const TString& title); + + IMonPage* FindPage(const TString& relativePath); + TIndexMonPage* FindIndexPage(const TString& relativePath); + void SortPages(); + }; + +} diff --git a/library/cpp/monlib/service/pages/diag_mon_page.cpp b/library/cpp/monlib/service/pages/diag_mon_page.cpp new file mode 100644 index 0000000000..2493ff4fba --- /dev/null +++ b/library/cpp/monlib/service/pages/diag_mon_page.cpp @@ -0,0 +1,9 @@ +#include "diag_mon_page.h" + +using namespace NMonitoring; + +void TDiagMonPage::OutputText(IOutputStream& out, NMonitoring::IMonHttpRequest& request) { + out << "uri: " << request.GetUri() << "\n"; + out << "path: " << request.GetPath() << "\n"; + out << "path info: " << request.GetPathInfo() << "\n"; +} diff --git a/library/cpp/monlib/service/pages/diag_mon_page.h b/library/cpp/monlib/service/pages/diag_mon_page.h new file mode 100644 index 0000000000..761194d4ec --- /dev/null +++ b/library/cpp/monlib/service/pages/diag_mon_page.h @@ -0,0 +1,16 @@ +#pragma once + +#include "pre_mon_page.h" + +namespace NMonitoring { + // internal diagnostics page + struct TDiagMonPage: public TPreMonPage { + TDiagMonPage() + : TPreMonPage("diag", "Diagnostics Page") + { + } + + void OutputText(IOutputStream& out, NMonitoring::IMonHttpRequest&) override; + }; + +} diff --git a/library/cpp/monlib/service/pages/html_mon_page.cpp b/library/cpp/monlib/service/pages/html_mon_page.cpp new file mode 100644 index 0000000000..eb4eb3b66c --- /dev/null +++ b/library/cpp/monlib/service/pages/html_mon_page.cpp @@ -0,0 +1,60 @@ +#include "html_mon_page.h" + +#include <library/cpp/monlib/service/pages/templates.h> + +using namespace NMonitoring; + +void THtmlMonPage::Output(NMonitoring::IMonHttpRequest& request) { + IOutputStream& out = request.Output(); + + out << HTTPOKHTML; + HTML(out) { + out << "<!DOCTYPE html>\n"; + HTML_TAG() { + HEAD() { + if (!!Title) { + out << "<title>" << Title << "</title>\n"; + } + out << "<link rel='stylesheet' href='https://yastatic.net/bootstrap/3.3.1/css/bootstrap.min.css'>\n"; + out << "<script language='javascript' type='text/javascript' src='https://yastatic.net/jquery/2.1.3/jquery.min.js'></script>\n"; + out << "<script language='javascript' type='text/javascript' src='https://yastatic.net/bootstrap/3.3.1/js/bootstrap.min.js'></script>\n"; + + if (OutputTableSorterJsCss) { + out << "<link rel='stylesheet' href='/jquery.tablesorter.css'>\n"; + out << "<script language='javascript' type='text/javascript' src='/jquery.tablesorter.js'></script>\n"; + } + + out << "<style type=\"text/css\">\n"; + out << ".table-nonfluid { width: auto; }\n"; + out << ".narrow-line50 {line-height: 50%}\n"; + out << ".narrow-line60 {line-height: 60%}\n"; + out << ".narrow-line70 {line-height: 70%}\n"; + out << ".narrow-line80 {line-height: 80%}\n"; + out << ".narrow-line90 {line-height: 90%}\n"; + out << "</style>\n"; + } + BODY() { + OutputNavBar(out); + + DIV_CLASS("container") { + if (!!Title) { + out << "<h2>" << Title << "</h2>"; + } + OutputContent(request); + } + } + } + } +} + +void THtmlMonPage::NotFound(NMonitoring::IMonHttpRequest& request) const { + IOutputStream& out = request.Output(); + out << HTTPNOTFOUND; + out.Flush(); +} + +void THtmlMonPage::NoContent(NMonitoring::IMonHttpRequest& request) const { + IOutputStream& out = request.Output(); + out << HTTPNOCONTENT; + out.Flush(); +} diff --git a/library/cpp/monlib/service/pages/html_mon_page.h b/library/cpp/monlib/service/pages/html_mon_page.h new file mode 100644 index 0000000000..e87c53b62b --- /dev/null +++ b/library/cpp/monlib/service/pages/html_mon_page.h @@ -0,0 +1,25 @@ +#pragma once + +#include "mon_page.h" + +namespace NMonitoring { + struct THtmlMonPage: public IMonPage { + THtmlMonPage(const TString& path, + const TString& title = TString(), + bool outputTableSorterJsCss = false) + : IMonPage(path, title) + , OutputTableSorterJsCss(outputTableSorterJsCss) + { + } + + void Output(NMonitoring::IMonHttpRequest& request) override; + + void NotFound(NMonitoring::IMonHttpRequest& request) const; + void NoContent(NMonitoring::IMonHttpRequest& request) const; + + virtual void OutputContent(NMonitoring::IMonHttpRequest& request) = 0; + + bool OutputTableSorterJsCss; + }; + +} diff --git a/library/cpp/monlib/service/pages/index_mon_page.cpp b/library/cpp/monlib/service/pages/index_mon_page.cpp new file mode 100644 index 0000000000..83ff8b529a --- /dev/null +++ b/library/cpp/monlib/service/pages/index_mon_page.cpp @@ -0,0 +1,151 @@ +#include "index_mon_page.h" + +#include <util/generic/cast.h> +#include <util/string/ascii.h> + +using namespace NMonitoring; + +void TIndexMonPage::OutputIndexPage(IMonHttpRequest& request) { + request.Output() << HTTPOKHTML; + request.Output() << "<html>\n"; + OutputHead(request.Output()); + OutputBody(request); + request.Output() << "</html>\n"; +} + +void TIndexMonPage::Output(IMonHttpRequest& request) { + TStringBuf pathInfo = request.GetPathInfo(); + if (pathInfo.empty() || pathInfo == TStringBuf("/")) { + OutputIndexPage(request); + return; + } + + Y_VERIFY(pathInfo.StartsWith('/')); + + TMonPagePtr found; + // analogous to CGI PATH_INFO + { + TGuard<TMutex> g(Mtx); + TStringBuf pathTmp = request.GetPathInfo(); + for (;;) { + TPagesByPath::iterator i = PagesByPath.find(pathTmp); + if (i != PagesByPath.end()) { + found = i->second; + pathInfo = request.GetPathInfo().substr(pathTmp.size()); + Y_VERIFY(pathInfo.empty() || pathInfo.StartsWith('/')); + break; + } + size_t slash = pathTmp.find_last_of('/'); + Y_VERIFY(slash != TString::npos); + pathTmp = pathTmp.substr(0, slash); + if (!pathTmp) { + break; + } + } + } + if (found) { + THolder<IMonHttpRequest> child(request.MakeChild(found.Get(), TString{pathInfo})); + found->Output(*child); + } else { + request.Output() << HTTPNOTFOUND; + } +} + +void TIndexMonPage::OutputIndex(IOutputStream& out, bool pathEndsWithSlash) { + TGuard<TMutex> g(Mtx); + for (auto& Page : Pages) { + IMonPage* page = Page.Get(); + if (page->IsInIndex()) { + TString pathToDir = ""; + if (!pathEndsWithSlash) { + pathToDir = this->GetPath() + "/"; + } + out << "<a href='" << pathToDir << page->GetPath() << "'>" << page->GetTitle() << "</a><br/>\n"; + } + } +} + +void TIndexMonPage::Register(TMonPagePtr page) { + TGuard<TMutex> g(Mtx); + auto insres = PagesByPath.insert(std::make_pair("/" + page->GetPath(), page)); + if (insres.second) { + // new unique page just inserted, update Pages + Pages.push_back(page); + } else { + // a page with the given path is already present, replace it with the new page + + // find old page, sorry for O(n) + auto it = std::find(Pages.begin(), Pages.end(), insres.first->second); + *it = page; + // this already present, replace it + insres.first->second = page; + } + page->Parent = this; +} + +TIndexMonPage* TIndexMonPage::RegisterIndexPage(const TString& path, const TString& title) { + TGuard<TMutex> g(Mtx); + TIndexMonPage* page = VerifyDynamicCast<TIndexMonPage*>(FindPage(path)); + if (page) { + return page; + } + page = new TIndexMonPage(path, title); + Register(page); + return VerifyDynamicCast<TIndexMonPage*>(page); +} + +IMonPage* TIndexMonPage::FindPage(const TString& relativePath) { + TGuard<TMutex> g(Mtx); + + Y_VERIFY(!relativePath.StartsWith('/')); + TPagesByPath::iterator i = PagesByPath.find("/" + relativePath); + if (i == PagesByPath.end()) { + return nullptr; + } else { + return i->second.Get(); + } +} + +TIndexMonPage* TIndexMonPage::FindIndexPage(const TString& relativePath) { + return VerifyDynamicCast<TIndexMonPage*>(FindPage(relativePath)); +} + +void TIndexMonPage::OutputCommonJsCss(IOutputStream& out) { + out << "<link rel='stylesheet' href='https://yastatic.net/bootstrap/3.3.1/css/bootstrap.min.css'>\n"; + out << "<script language='javascript' type='text/javascript' src='https://yastatic.net/jquery/2.1.3/jquery.min.js'></script>\n"; + out << "<script language='javascript' type='text/javascript' src='https://yastatic.net/bootstrap/3.3.1/js/bootstrap.min.js'></script>\n"; +} + +void TIndexMonPage::OutputHead(IOutputStream& out) { + out << "<head>\n"; + OutputCommonJsCss(out); + out << "<title>" << Title << "</title>\n"; + out << "</head>\n"; +} + +void TIndexMonPage::OutputBody(IMonHttpRequest& req) { + auto& out = req.Output(); + out << "<body>\n"; + + // part of common navbar + OutputNavBar(out); + + out << "<div class='container'>\n" + << "<h2>" << Title << "</h2>\n"; + OutputIndex(out, req.GetPathInfo().EndsWith('/')); + out << "<div>\n" + << "</body>\n"; +} + +void TIndexMonPage::SortPages() { + TGuard<TMutex> g(Mtx); + std::sort(Pages.begin(), Pages.end(), [](const TMonPagePtr& a, const TMonPagePtr& b) { + return AsciiCompareIgnoreCase(a->GetTitle(), b->GetTitle()) < 0; + }); +} + +void TIndexMonPage::ClearPages() { + TGuard<TMutex> g(Mtx); + Pages.clear(); + PagesByPath.clear(); +} diff --git a/library/cpp/monlib/service/pages/index_mon_page.h b/library/cpp/monlib/service/pages/index_mon_page.h new file mode 100644 index 0000000000..bf514a3105 --- /dev/null +++ b/library/cpp/monlib/service/pages/index_mon_page.h @@ -0,0 +1,38 @@ +#pragma once + +#include "mon_page.h" + +namespace NMonitoring { + struct TIndexMonPage: public IMonPage { + TMutex Mtx; + typedef TVector<TMonPagePtr> TPages; + TPages Pages; + typedef THashMap<TString, TMonPagePtr> TPagesByPath; + TPagesByPath PagesByPath; + + TIndexMonPage(const TString& path, const TString& title) + : IMonPage(path, title) + { + } + + ~TIndexMonPage() override { + } + + void Output(IMonHttpRequest& request) override; + void OutputIndexPage(IMonHttpRequest& request); + virtual void OutputIndex(IOutputStream& out, bool pathEndsWithSlash); + virtual void OutputCommonJsCss(IOutputStream& out); + void OutputHead(IOutputStream& out); + void OutputBody(IMonHttpRequest& out); + + void Register(TMonPagePtr page); + TIndexMonPage* RegisterIndexPage(const TString& path, const TString& title); + + IMonPage* FindPage(const TString& relativePath); + TIndexMonPage* FindIndexPage(const TString& relativePath); + + void SortPages(); + void ClearPages(); + }; + +} diff --git a/library/cpp/monlib/service/pages/mon_page.cpp b/library/cpp/monlib/service/pages/mon_page.cpp new file mode 100644 index 0000000000..72033b1699 --- /dev/null +++ b/library/cpp/monlib/service/pages/mon_page.cpp @@ -0,0 +1,36 @@ +#include "mon_page.h" + +using namespace NMonitoring; + +IMonPage::IMonPage(const TString& path, const TString& title) + : Path(path) + , Title(title) +{ + Y_VERIFY(!Path.StartsWith('/')); + Y_VERIFY(!Path.EndsWith('/')); +} + +void IMonPage::OutputNavBar(IOutputStream& out) { + TVector<const IMonPage*> parents; + for (const IMonPage* p = this; p; p = p->Parent) { + parents.push_back(p); + } + std::reverse(parents.begin(), parents.end()); + + out << "<ol class='breadcrumb'>\n"; + + TString absolutePath; + for (size_t i = 0; i < parents.size(); ++i) { + const TString& title = parents[i]->GetTitle(); + if (i == parents.size() - 1) { + out << "<li>" << title << "</li>\n"; + } else { + if (!absolutePath.EndsWith('/')) { + absolutePath += '/'; + } + absolutePath += parents[i]->GetPath(); + out << "<li class='active'><a href='" << absolutePath << "'>" << title << "</a></li>\n"; + } + } + out << "</ol>\n"; +} diff --git a/library/cpp/monlib/service/pages/mon_page.h b/library/cpp/monlib/service/pages/mon_page.h new file mode 100644 index 0000000000..e396612bb0 --- /dev/null +++ b/library/cpp/monlib/service/pages/mon_page.h @@ -0,0 +1,66 @@ +#pragma once + +#include <library/cpp/monlib/service/service.h> +#include <library/cpp/monlib/service/mon_service_http_request.h> + +#include <util/generic/string.h> +#include <util/generic/ptr.h> + +namespace NMonitoring { + static const char HTTPOKTEXT[] = "HTTP/1.1 200 Ok\r\nContent-Type: text/plain\r\nConnection: Close\r\n\r\n"; + static const char HTTPOKBIN[] = "HTTP/1.1 200 Ok\r\nContent-Type: application/octet-stream\r\nConnection: Close\r\n\r\n"; + static const char HTTPOKHTML[] = "HTTP/1.1 200 Ok\r\nContent-Type: text/html\r\nConnection: Close\r\n\r\n"; + static const char HTTPOKJSON[] = "HTTP/1.1 200 Ok\r\nContent-Type: application/json\r\nConnection: Close\r\n\r\n"; + static const char HTTPOKSPACK[] = "HTTP/1.1 200 Ok\r\nContent-Type: application/x-solomon-spack\r\nConnection: Close\r\n\r\n"; + static const char HTTPOKPROMETHEUS[] = "HTTP/1.1 200 Ok\r\nContent-Type: text/plain\r\nConnection: Close\r\n\r\n"; + static const char HTTPOKJAVASCRIPT[] = "HTTP/1.1 200 Ok\r\nContent-Type: text/javascript\r\nConnection: Close\r\n\r\n"; + static const char HTTPOKCSS[] = "HTTP/1.1 200 Ok\r\nContent-Type: text/css\r\nConnection: Close\r\n\r\n"; + static const char HTTPNOCONTENT[] = "HTTP/1.1 204 No content\r\nConnection: Close\r\n\r\n"; + static const char HTTPNOTFOUND[] = "HTTP/1.1 404 Invalid URI\r\nConnection: Close\r\n\r\nInvalid URL\r\n"; + static const char HTTPUNAUTHORIZED[] = "HTTP/1.1 401 Unauthorized\r\nConnection: Close\r\n\r\nUnauthorized\r\n"; + static const char HTTPFORBIDDEN[] = "HTTP/1.1 403 Forbidden\r\nConnection: Close\r\n\r\nForbidden\r\n"; + + // Fonts + static const char HTTPOKFONTEOT[] = "HTTP/1.1 200 Ok\r\nContent-Type: application/vnd.ms-fontobject\r\nConnection: Close\r\n\r\n"; + static const char HTTPOKFONTTTF[] = "HTTP/1.1 200 Ok\r\nContent-Type: application/x-font-ttf\r\nConnection: Close\r\n\r\n"; + static const char HTTPOKFONTWOFF[] = "HTTP/1.1 200 Ok\r\nContent-Type: application/font-woff\r\nConnection: Close\r\n\r\n"; + static const char HTTPOKFONTWOFF2[] = "HTTP/1.1 200 Ok\r\nContent-Type: application/font-woff2\r\nConnection: Close\r\n\r\n"; + + // Images + static const char HTTPOKPNG[] = "HTTP/1.1 200 Ok\r\nContent-Type: image/png\r\nConnection: Close\r\n\r\n"; + static const char HTTPOKSVG[] = "HTTP/1.1 200 Ok\r\nContent-Type: image/svg+xml\r\nConnection: Close\r\n\r\n"; + + class IMonPage; + + using TMonPagePtr = TIntrusivePtr<IMonPage>; + + class IMonPage: public TAtomicRefCount<IMonPage> { + public: + const TString Path; + const TString Title; + const IMonPage* Parent = nullptr; + + public: + IMonPage(const TString& path, const TString& title = TString()); + + virtual ~IMonPage() { + } + + void OutputNavBar(IOutputStream& out); + + virtual const TString& GetPath() const { + return Path; + } + + virtual const TString& GetTitle() const { + return Title; + } + + bool IsInIndex() const { + return !Title.empty(); + } + + virtual void Output(IMonHttpRequest& request) = 0; + }; + +} diff --git a/library/cpp/monlib/service/pages/pre_mon_page.cpp b/library/cpp/monlib/service/pages/pre_mon_page.cpp new file mode 100644 index 0000000000..fc03a19b80 --- /dev/null +++ b/library/cpp/monlib/service/pages/pre_mon_page.cpp @@ -0,0 +1,18 @@ +#include "pre_mon_page.h" + +using namespace NMonitoring; + +void TPreMonPage::OutputContent(NMonitoring::IMonHttpRequest& request) { + auto& out = request.Output(); + if (PreTag) { + BeforePre(request); + out << "<pre>\n"; + OutputText(out, request); + out << "</pre>\n"; + } else { + OutputText(out, request); + } +} + +void TPreMonPage::BeforePre(NMonitoring::IMonHttpRequest&) { +} diff --git a/library/cpp/monlib/service/pages/pre_mon_page.h b/library/cpp/monlib/service/pages/pre_mon_page.h new file mode 100644 index 0000000000..c9a923d39a --- /dev/null +++ b/library/cpp/monlib/service/pages/pre_mon_page.h @@ -0,0 +1,27 @@ +#pragma once + +#include "html_mon_page.h" + +namespace NMonitoring { + struct TPreMonPage: public THtmlMonPage { + TPreMonPage(const TString& path, + const TString& title = TString(), + bool preTag = true, + bool outputTableSorterJsCss = false) + : THtmlMonPage(path, title, outputTableSorterJsCss) + , PreTag(preTag) + { + } + + void OutputContent(NMonitoring::IMonHttpRequest& request) override; + + // hook to customize output + virtual void BeforePre(NMonitoring::IMonHttpRequest& request); + + // put your text here + virtual void OutputText(IOutputStream& out, NMonitoring::IMonHttpRequest&) = 0; + + const bool PreTag; + }; + +} diff --git a/library/cpp/monlib/service/pages/registry_mon_page.cpp b/library/cpp/monlib/service/pages/registry_mon_page.cpp new file mode 100644 index 0000000000..c59e50f622 --- /dev/null +++ b/library/cpp/monlib/service/pages/registry_mon_page.cpp @@ -0,0 +1,46 @@ +#include "registry_mon_page.h" + +#include <library/cpp/monlib/encode/text/text.h> +#include <library/cpp/monlib/encode/json/json.h> +#include <library/cpp/monlib/encode/prometheus/prometheus.h> +#include <library/cpp/monlib/encode/spack/spack_v1.h> +#include <library/cpp/monlib/service/format.h> + +namespace NMonitoring { + void TMetricRegistryPage::Output(NMonitoring::IMonHttpRequest& request) { + const auto formatStr = TStringBuf{request.GetPathInfo()}.RNextTok('/'); + auto& out = request.Output(); + + if (!formatStr.empty()) { + IMetricEncoderPtr encoder; + TString resp; + + if (formatStr == TStringBuf("json")) { + resp = HTTPOKJSON; + encoder = NMonitoring::EncoderJson(&out); + } else if (formatStr == TStringBuf("spack")) { + resp = HTTPOKSPACK; + const auto compression = ParseCompression(request); + encoder = NMonitoring::EncoderSpackV1(&out, ETimePrecision::SECONDS, compression); + } else if (formatStr == TStringBuf("prometheus")) { + resp = HTTPOKPROMETHEUS; + encoder = NMonitoring::EncoderPrometheus(&out); + } else { + ythrow yexception() << "unsupported metric encoding format: " << formatStr; + } + + out.Write(resp); + RegistryRawPtr_->Accept(TInstant::Zero(), encoder.Get()); + + encoder->Close(); + } else { + THtmlMonPage::Output(request); + } + } + + void TMetricRegistryPage::OutputText(IOutputStream& out, NMonitoring::IMonHttpRequest&) { + IMetricEncoderPtr encoder = NMonitoring::EncoderText(&out); + RegistryRawPtr_->Accept(TInstant::Zero(), encoder.Get()); + } + +} diff --git a/library/cpp/monlib/service/pages/registry_mon_page.h b/library/cpp/monlib/service/pages/registry_mon_page.h new file mode 100644 index 0000000000..2d26d3319c --- /dev/null +++ b/library/cpp/monlib/service/pages/registry_mon_page.h @@ -0,0 +1,32 @@ +#pragma once + +#include "pre_mon_page.h" + +#include <library/cpp/monlib/metrics/metric_registry.h> + +namespace NMonitoring { + // For now this class can only enumerate all metrics without any grouping or serve JSON/Spack/Prometheus + class TMetricRegistryPage: public TPreMonPage { + public: + TMetricRegistryPage(const TString& path, const TString& title, TAtomicSharedPtr<IMetricSupplier> registry) + : TPreMonPage(path, title) + , Registry_(registry) + , RegistryRawPtr_(Registry_.Get()) + { + } + + TMetricRegistryPage(const TString& path, const TString& title, IMetricSupplier* registry) + : TPreMonPage(path, title) + , RegistryRawPtr_(registry) + { + } + + void Output(NMonitoring::IMonHttpRequest& request) override; + void OutputText(IOutputStream& out, NMonitoring::IMonHttpRequest&) override; + + private: + TAtomicSharedPtr<IMetricSupplier> Registry_; + IMetricSupplier* RegistryRawPtr_; + }; + +} diff --git a/library/cpp/monlib/service/pages/resource_mon_page.cpp b/library/cpp/monlib/service/pages/resource_mon_page.cpp new file mode 100644 index 0000000000..ec4ac6a1a7 --- /dev/null +++ b/library/cpp/monlib/service/pages/resource_mon_page.cpp @@ -0,0 +1,49 @@ +#include "resource_mon_page.h" + +using namespace NMonitoring; + +void TResourceMonPage::Output(NMonitoring::IMonHttpRequest& request) { + IOutputStream& out = request.Output(); + switch (ResourceType) { + case TEXT: + out << HTTPOKTEXT; + break; + case JSON: + out << HTTPOKJSON; + break; + case CSS: + out << HTTPOKCSS; + break; + case JAVASCRIPT: + out << HTTPOKJAVASCRIPT; + break; + case FONT_EOT: + out << HTTPOKFONTEOT; + break; + case FONT_TTF: + out << HTTPOKFONTTTF; + break; + case FONT_WOFF: + out << HTTPOKFONTWOFF; + break; + case FONT_WOFF2: + out << HTTPOKFONTWOFF2; + break; + case PNG: + out << HTTPOKPNG; + break; + case SVG: + out << HTTPOKSVG; + break; + default: + out << HTTPOKBIN; + break; + } + out << NResource::Find(ResourceName); +} + +void TResourceMonPage::NotFound(NMonitoring::IMonHttpRequest& request) const { + IOutputStream& out = request.Output(); + out << HTTPNOTFOUND; + out.Flush(); +} diff --git a/library/cpp/monlib/service/pages/resource_mon_page.h b/library/cpp/monlib/service/pages/resource_mon_page.h new file mode 100644 index 0000000000..f6ab67200e --- /dev/null +++ b/library/cpp/monlib/service/pages/resource_mon_page.h @@ -0,0 +1,43 @@ +#pragma once + +#include "mon_page.h" + +#include <library/cpp/resource/resource.h> + +namespace NMonitoring { + struct TResourceMonPage: public IMonPage { + public: + enum EResourceType { + BINARY, + TEXT, + JSON, + CSS, + JAVASCRIPT, + + FONT_EOT, + FONT_TTF, + FONT_WOFF, + FONT_WOFF2, + + PNG, + SVG + }; + + TResourceMonPage(const TString& path, const TString& resourceName, + const EResourceType& resourceType = BINARY) + : IMonPage(path, "") + , ResourceName(resourceName) + , ResourceType(resourceType) + { + } + + void Output(NMonitoring::IMonHttpRequest& request) override; + + void NotFound(NMonitoring::IMonHttpRequest& request) const; + + private: + TString ResourceName; + EResourceType ResourceType; + }; + +} diff --git a/library/cpp/monlib/service/pages/tablesorter/css_mon_page.h b/library/cpp/monlib/service/pages/tablesorter/css_mon_page.h new file mode 100644 index 0000000000..c2c8330089 --- /dev/null +++ b/library/cpp/monlib/service/pages/tablesorter/css_mon_page.h @@ -0,0 +1,13 @@ +#pragma once + +#include <library/cpp/monlib/service/pages/resource_mon_page.h> + +namespace NMonitoring { + struct TTablesorterCssMonPage: public TResourceMonPage { + TTablesorterCssMonPage() + : TResourceMonPage("jquery.tablesorter.css", "jquery.tablesorter.css", CSS) + { + } + }; + +} diff --git a/library/cpp/monlib/service/pages/tablesorter/js_mon_page.h b/library/cpp/monlib/service/pages/tablesorter/js_mon_page.h new file mode 100644 index 0000000000..f8a1d8254e --- /dev/null +++ b/library/cpp/monlib/service/pages/tablesorter/js_mon_page.h @@ -0,0 +1,13 @@ +#pragma once + +#include <library/cpp/monlib/service/pages/resource_mon_page.h> + +namespace NMonitoring { + struct TTablesorterJsMonPage: public TResourceMonPage { + TTablesorterJsMonPage() + : TResourceMonPage("jquery.tablesorter.js", "jquery.tablesorter.js", JAVASCRIPT) + { + } + }; + +} diff --git a/library/cpp/monlib/service/pages/tablesorter/resources/jquery.tablesorter.css b/library/cpp/monlib/service/pages/tablesorter/resources/jquery.tablesorter.css new file mode 100644 index 0000000000..1e57adfeb3 --- /dev/null +++ b/library/cpp/monlib/service/pages/tablesorter/resources/jquery.tablesorter.css @@ -0,0 +1,2 @@ +/* theme.bootstrap.css v2.22.3 */ .tablesorter-bootstrap{width:100%}.tablesorter-bootstrap tfoot td,.tablesorter-bootstrap tfoot th,.tablesorter-bootstrap thead td,.tablesorter-bootstrap thead th{font:14px/20px Arial,Sans-serif;font-weight:700;padding:4px;margin:0 0 18px;background-color:#eee}.tablesorter-bootstrap .tablesorter-header{cursor:pointer}.tablesorter-bootstrap .tablesorter-header-inner{position:relative;padding:4px 18px 4px 4px}.tablesorter-bootstrap .tablesorter-header i.tablesorter-icon{font-size:11px;position:absolute;right:2px;top:50%;margin-top:-7px;width:14px;height:14px;background-repeat:no-repeat;line-height:14px;display:inline-block}.tablesorter-bootstrap .bootstrap-icon-unsorted{background-image:url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAsAAAAOCAMAAADOvxanAAAAVFBMVEUAAABCQkJZWVkZGRnJyckgICAZGRkZGRn8/PweHh4dHR0aGhoaGhpUVFQbGxvQ0NDc3NxMTExSUlIbGxvr6+s4ODhKSkogICAtLS00NDQzMzMnJydSEPrQAAAAGHRSTlMA1ssZRLgdAQbDyisqsZo8QdXUq0r9xPepSRwiAAAAX0lEQVQI13XHSQKAIAwEwQAKxn13Ev7/T2Pu9qmarJKPXIicI4PH4hxaKNrhm2S8bJK5h4YzKHrzJNtK6yYT/TdXzpS5zuYg4MSQYF6i4IHExdw1UVRi05HPrrvT53a+qyMFC9t04gcAAAAASUVORK5CYII=)}.tablesorter-bootstrap .icon-white.bootstrap-icon-unsorted{background-image:url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAsAAAAOBAMAAAALT/umAAAAKlBMVEUAAAD///////////////////////////////////////////////////+Gu8ovAAAADXRSTlMA4EXKBtQqvR0+sxmalc142gAAAFdJREFUCNdjYGDoamAAAjZbMxCVfvd6AgMDd+3du9UMDKx3hWSvMjBwXZww8RYDGuC53NB8h4GB8a617UUGBs7Yu3cjGRhYVO9eVQFKOskKOQApFmUgBwBZ+xXRTttNdAAAAABJRU5ErkJggg==)}.tablesorter-bootstrap>tbody>tr.odd>td,.tablesorter-bootstrap>tbody>tr.tablesorter-hasChildRow.odd:hover~tr.tablesorter-hasChildRow.odd~.tablesorter-childRow.odd>td{background-color:#f9f9f9}.tablesorter-bootstrap>tbody>tr.even:hover>td,.tablesorter-bootstrap>tbody>tr.hover>td,.tablesorter-bootstrap>tbody>tr.odd:hover>td,.tablesorter-bootstrap>tbody>tr.tablesorter-hasChildRow.even:hover~.tablesorter-childRow.even>td,.tablesorter-bootstrap>tbody>tr.tablesorter-hasChildRow.odd:hover~.tablesorter-childRow.odd>td{background-color:#f5f5f5}.caption,.tablesorter-bootstrap>tbody>tr.even>td,.tablesorter-bootstrap>tbody>tr.tablesorter-hasChildRow.even:hover~tr.tablesorter-hasChildRow.even~.tablesorter-childRow.even>td{background-color:#fff}.tablesorter-bootstrap .tablesorter-processing{background-image:url(data:image/gif;base64,R0lGODlhFAAUAKEAAO7u7lpaWgAAAAAAACH/C05FVFNDQVBFMi4wAwEAAAAh+QQBCgACACwAAAAAFAAUAAACQZRvoIDtu1wLQUAlqKTVxqwhXIiBnDg6Y4eyx4lKW5XK7wrLeK3vbq8J2W4T4e1nMhpWrZCTt3xKZ8kgsggdJmUFACH5BAEKAAIALAcAAAALAAcAAAIUVB6ii7jajgCAuUmtovxtXnmdUAAAIfkEAQoAAgAsDQACAAcACwAAAhRUIpmHy/3gUVQAQO9NetuugCFWAAAh+QQBCgACACwNAAcABwALAAACE5QVcZjKbVo6ck2AF95m5/6BSwEAIfkEAQoAAgAsBwANAAsABwAAAhOUH3kr6QaAcSrGWe1VQl+mMUIBACH5BAEKAAIALAIADQALAAcAAAIUlICmh7ncTAgqijkruDiv7n2YUAAAIfkEAQoAAgAsAAAHAAcACwAAAhQUIGmHyedehIoqFXLKfPOAaZdWAAAh+QQFCgACACwAAAIABwALAAACFJQFcJiXb15zLYRl7cla8OtlGGgUADs=);background-position:center center!important;background-repeat:no-repeat!important}.tablesorter-bootstrap .tablesorter-filter-row input.tablesorter-filter,.tablesorter-bootstrap .tablesorter-filter-row select.tablesorter-filter{width:98%;margin:0;padding:4px 6px;color:#333;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box;-webkit-transition:height .1s ease;-moz-transition:height .1s ease;-o-transition:height .1s ease;transition:height .1s ease}.tablesorter-bootstrap .tablesorter-filter-row .tablesorter-filter.disabled{background-color:#eee;color:#555;cursor:not-allowed;border:1px solid #ccc;border-radius:4px;box-shadow:0 1px 1px rgba(0,0,0,.075) inset;box-sizing:border-box;transition:height .1s ease}.tablesorter-bootstrap .tablesorter-filter-row{background-color:#efefef}.tablesorter-bootstrap .tablesorter-filter-row td{background-color:#efefef;line-height:normal;text-align:center;padding:4px 6px;vertical-align:middle;-webkit-transition:line-height .1s ease;-moz-transition:line-height .1s ease;-o-transition:line-height .1s ease;transition:line-height .1s ease}.tablesorter-bootstrap .tablesorter-filter-row.hideme td{padding:2px;margin:0;line-height:0}.tablesorter-bootstrap .tablesorter-filter-row.hideme *{height:1px;min-height:0;border:0;padding:0;margin:0;opacity:0;filter:alpha(opacity=0)}.tablesorter .filtered{display:none}.tablesorter-bootstrap .tablesorter-pager select{padding:4px 6px}.tablesorter-bootstrap .tablesorter-pager .pagedisplay{border:0}.tablesorter-bootstrap tfoot i{font-size:11px}.tablesorter .tablesorter-errorRow td{text-align:center;cursor:pointer;background-color:#e6bf99} +/* colored table cells */ .table>tbody>tr.active>td,.table>tbody>tr.active>th,.table>tbody>tr>td.active,.table>tbody>tr>th.active,.table>tfoot>tr.active>td,.table>tfoot>tr.active>th,.table>tfoot>tr>td.active,.table>tfoot>tr>th.active,.table>thead>tr.active>td,.table>thead>tr.active>th,.table>thead>tr>td.active,.table>thead>tr>th.active{background-color:#f5f5f5!important}.table>tbody>tr.success>td,.table>tbody>tr.success>th,.table>tbody>tr>td.success,.table>tbody>tr>th.success,.table>tfoot>tr.success>td,.table>tfoot>tr.success>th,.table>tfoot>tr>td.success,.table>tfoot>tr>th.success,.table>thead>tr.success>td,.table>thead>tr.success>th,.table>thead>tr>td.success,.table>thead>tr>th.success{background-color:#dff0d8!important;color:#468847!important}.table-hover>tbody>tr.success:hover>td,.table-hover>tbody>tr.success:hover>th,.table-hover>tbody>tr>td.success:hover,.table-hover>tbody>tr>th.success:hover{background-color:#d0e9c6!important;color:#356635!important}.table>tbody>tr.danger>td,.table>tbody>tr.danger>th,.table>tbody>tr>td.danger,.table>tbody>tr>th.danger,.table>tfoot>tr.danger>td,.table>tfoot>tr.danger>th,.table>tfoot>tr>td.danger,.table>tfoot>tr>th.danger,.table>thead>tr.danger>td,.table>thead>tr.danger>th,.table>thead>tr>td.danger,.table>thead>tr>th.danger{background-color:#f2dede!important;color:#b94a48!important}.table-hover>tbody>tr.danger:hover>td,.table-hover>tbody>tr.danger:hover>th,.table-hover>tbody>tr>td.danger:hover,.table-hover>tbody>tr>th.danger:hover{background-color:#ebcccc!important;color:#953b39!important}.table>tbody>tr.warning>td,.table>tbody>tr.warning>th,.table>tbody>tr>td.warning,.table>tbody>tr>th.warning,.table>tfoot>tr.warning>td,.table>tfoot>tr.warning>th,.table>tfoot>tr>td.warning,.table>tfoot>tr>th.warning,.table>thead>tr.warning>td,.table>thead>tr.warning>th,.table>thead>tr>td.warning,.table>thead>tr>th.warning{background-color:#fcf8e3!important;color:#c09853!important}.table-hover>tbody>tr.warning:hover>td,.table-hover>tbody>tr.warning:hover>th,.table-hover>tbody>tr>td.warning:hover,.table-hover>tbody>tr>th.warning:hover{background-color:#faf2cc!important;color:#a47e3c!important} diff --git a/library/cpp/monlib/service/pages/tablesorter/resources/jquery.tablesorter.js b/library/cpp/monlib/service/pages/tablesorter/resources/jquery.tablesorter.js new file mode 100644 index 0000000000..3001b4b833 --- /dev/null +++ b/library/cpp/monlib/service/pages/tablesorter/resources/jquery.tablesorter.js @@ -0,0 +1,3 @@ +/* jquery.tablesorter.js v2.22.3 */ !function(e){"use strict";e.extend({tablesorter:new function(){function t(){var e=arguments[0],t=arguments.length>1?Array.prototype.slice.call(arguments):e;"undefined"!=typeof console&&"undefined"!=typeof console.log?console[/error/i.test(e)?"error":/warn/i.test(e)?"warn":"log"](t):alert(t)}function r(e,r){t(e+" ("+((new Date).getTime()-r.getTime())+"ms)")}function s(e){for(var t in e)return!1;return!0}function a(r,s,a,n){for(var o,i,d=v.parsers.length,c=!1,l="",p=!0;""===l&&p;)a++,s[a]?(c=s[a].cells[n],l=v.getElementText(r,c,n),i=e(c),r.debug&&t("Checking if value was empty on row "+a+", column: "+n+': "'+l+'"')):p=!1;for(;--d>=0;)if(o=v.parsers[d],o&&"text"!==o.id&&o.is&&o.is(l,r.table,c,i))return o;return v.getParserById("text")}function n(e,s){var n,o,i,d,c,l,p,u,g,f,h,m,b=e.table,y=0,w="";if(e.$tbodies=e.$table.children("tbody:not(."+e.cssInfoBlock+")"),h="undefined"==typeof s?e.$tbodies:s,m=h.length,0===m)return e.debug?t("Warning: *Empty table!* Not building a parser cache"):"";for(e.debug&&(f=new Date,t("Detecting parsers for each column")),o={extractors:[],parsers:[]};m>y;){if(n=h[y].rows,n.length)for(i=e.columns,d=0;i>d;d++)c=e.$headerIndexed[d],l=v.getColumnData(b,e.headers,d),g=v.getParserById(v.getData(c,l,"extractor")),u=v.getParserById(v.getData(c,l,"sorter")),p="false"===v.getData(c,l,"parser"),e.empties[d]=(v.getData(c,l,"empty")||e.emptyTo||(e.emptyToBottom?"bottom":"top")).toLowerCase(),e.strings[d]=(v.getData(c,l,"string")||e.stringTo||"max").toLowerCase(),p&&(u=v.getParserById("no-parser")),g||(g=!1),u||(u=a(e,n,-1,d)),e.debug&&(w+="column:"+d+"; extractor:"+g.id+"; parser:"+u.id+"; string:"+e.strings[d]+"; empty: "+e.empties[d]+"\n"),o.parsers[d]=u,o.extractors[d]=g;y+=o.parsers.length?m:1}e.debug&&(t(w?w:"No parsers detected"),r("Completed detecting parsers",f)),e.parsers=o.parsers,e.extractors=o.extractors}function o(s,a){var n,o,i,d,c,l,p,u,g,f,h,m,b,y,w=s.config,x=w.parsers;if(w.$tbodies=w.$table.children("tbody:not(."+w.cssInfoBlock+")"),p="undefined"==typeof a?w.$tbodies:a,w.cache={},w.totalRows=0,!x)return w.debug?t("Warning: *Empty table!* Not building a cache"):"";for(w.debug&&(f=new Date),w.showProcessing&&v.isProcessing(s,!0),l=0;l<p.length;l++){for(y=[],n=w.cache[l]={normalized:[]},h=p[l]&&p[l].rows.length||0,d=0;h>d;++d)if(m={child:[],raw:[]},u=e(p[l].rows[d]),g=[],u.hasClass(w.cssChildRow)&&0!==d)for(o=n.normalized.length-1,b=n.normalized[o][w.columns],b.$row=b.$row.add(u),u.prev().hasClass(w.cssChildRow)||u.prev().addClass(v.css.cssHasChild),i=u.children("th, td"),o=b.child.length,b.child[o]=[],c=0;c<w.columns;c++)b.child[o][c]=v.getParsedText(w,i[c],c);else{for(m.$row=u,m.order=d,c=0;c<w.columns;++c)"undefined"!=typeof x[c]?(o=v.getElementText(w,u[0].cells[c],c),m.raw.push(o),i=v.getParsedText(w,u[0].cells[c],c,o),g.push(i),"numeric"===(x[c].type||"").toLowerCase()&&(y[c]=Math.max(Math.abs(i)||0,y[c]||0))):w.debug&&t("No parser found for cell:",u[0].cells[c],"does it have a header?");g[w.columns]=m,n.normalized.push(g)}n.colMax=y,w.totalRows+=n.normalized.length}w.showProcessing&&v.isProcessing(s),w.debug&&r("Building cache for "+h+" rows",f)}function i(e,t){var a,n,o,i,d,c,l,p=e.config,u=p.widgetOptions,g=p.$tbodies,f=[],h=p.cache;if(s(h))return p.appender?p.appender(e,f):e.isUpdating?p.$table.trigger("updateComplete",e):"";for(p.debug&&(l=new Date),c=0;c<g.length;c++)if(o=g.eq(c),o.length){for(i=v.processTbody(e,o,!0),a=h[c].normalized,n=a.length,d=0;n>d;d++)f.push(a[d][p.columns].$row),p.appender&&(!p.pager||p.pager.removeRows&&u.pager_removeRows||p.pager.ajax)||i.append(a[d][p.columns].$row);v.processTbody(e,i,!1)}p.appender&&p.appender(e,f),p.debug&&r("Rebuilt table",l),t||p.appender||v.applyWidget(e),e.isUpdating&&p.$table.trigger("updateComplete",e)}function d(e){return/^d/i.test(e)||1===e}function c(s){var a,n,o,i,c,l,u,g,f=s.config;for(f.headerList=[],f.headerContent=[],f.debug&&(u=new Date),f.columns=v.computeColumnIndex(f.$table.children("thead, tfoot").children("tr")),i=f.cssIcon?'<i class="'+(f.cssIcon===v.css.icon?v.css.icon:f.cssIcon+" "+v.css.icon)+'"></i>':"",f.$headers=e(e.map(e(s).find(f.selectorHeaders),function(t,r){return n=e(t),n.parent().hasClass(f.cssIgnoreRow)?void 0:(a=v.getColumnData(s,f.headers,r,!0),f.headerContent[r]=n.html(),""===f.headerTemplate||n.find("."+v.css.headerIn).length||(c=f.headerTemplate.replace(/\{content\}/g,n.html()).replace(/\{icon\}/g,n.find("."+v.css.icon).length?"":i),f.onRenderTemplate&&(o=f.onRenderTemplate.apply(n,[r,c]),o&&"string"==typeof o&&(c=o)),n.html('<div class="'+v.css.headerIn+'">'+c+"</div>")),f.onRenderHeader&&f.onRenderHeader.apply(n,[r,f,f.$table]),t.column=parseInt(n.attr("data-column"),10),t.order=d(v.getData(n,a,"sortInitialOrder")||f.sortInitialOrder)?[1,0,2]:[0,1,2],t.count=-1,t.lockedOrder=!1,l=v.getData(n,a,"lockedOrder")||!1,"undefined"!=typeof l&&l!==!1&&(t.order=t.lockedOrder=d(l)?[1,1,1]:[0,0,0]),n.addClass(v.css.header+" "+f.cssHeader),f.headerList[r]=t,n.parent().addClass(v.css.headerRow+" "+f.cssHeaderRow).attr("role","row"),f.tabIndex&&n.attr("tabindex",0),t)})),f.$headerIndexed=[],g=0;g<f.columns;g++)n=f.$headers.filter('[data-column="'+g+'"]'),f.$headerIndexed[g]=n.not(".sorter-false").length?n.not(".sorter-false").filter(":last"):n.filter(":last");e(s).find(f.selectorHeaders).attr({scope:"col",role:"columnheader"}),p(s),f.debug&&(r("Built headers:",u),t(f.$headers))}function l(e,t,r){var s=e.config;s.$table.find(s.selectorRemove).remove(),n(s),o(e),y(s,t,r)}function p(e){var t,r,s,a,n=e.config,o=n.$headers.length;for(t=0;o>t;t++)s=n.$headers.eq(t),a=v.getColumnData(e,n.headers,t,!0),r="false"===v.getData(s,a,"sorter")||"false"===v.getData(s,a,"parser"),s[0].sortDisabled=r,s[r?"addClass":"removeClass"]("sorter-false").attr("aria-disabled",""+r),e.id&&(r?s.removeAttr("aria-controls"):s.attr("aria-controls",e.id))}function u(t){var r,s,a,n,o,i,d,c,l=t.config,p=l.sortList,u=p.length,g=v.css.sortNone+" "+l.cssNone,f=[v.css.sortAsc+" "+l.cssAsc,v.css.sortDesc+" "+l.cssDesc],h=[l.cssIconAsc,l.cssIconDesc,l.cssIconNone],m=["ascending","descending"],b=e(t).find("tfoot tr").children().add(e(l.namespace+"_extra_headers")).removeClass(f.join(" "));for(l.$headers.removeClass(f.join(" ")).addClass(g).attr("aria-sort","none").find("."+v.css.icon).removeClass(h.join(" ")).addClass(h[2]),a=0;u>a;a++)if(2!==p[a][1]&&(r=l.$headers.not(".sorter-false").filter('[data-column="'+p[a][0]+'"]'+(1===u?":last":"")),r.length)){for(n=0;n<r.length;n++)r[n].sortDisabled||r.eq(n).removeClass(g).addClass(f[p[a][1]]).attr("aria-sort",m[p[a][1]]).find("."+v.css.icon).removeClass(h[2]).addClass(h[p[a][1]]);b.length&&b.filter('[data-column="'+p[a][0]+'"]').removeClass(g).addClass(f[p[a][1]])}for(u=l.$headers.length,o=l.$headers.not(".sorter-false"),a=0;u>a;a++)i=o.eq(a),i.length&&(s=o[a],d=s.order[(s.count+1)%(l.sortReset?3:2)],c=e.trim(i.text())+": "+v.language[i.hasClass(v.css.sortAsc)?"sortAsc":i.hasClass(v.css.sortDesc)?"sortDesc":"sortNone"]+v.language[0===d?"nextAsc":1===d?"nextDesc":"nextNone"],i.attr("aria-label",c))}function g(t,r){var s,a,n,o,i,d,c,l,p=t.config,u=r||p.sortList,g=u.length;for(p.sortList=[],i=0;g>i;i++)if(l=u[i],s=parseInt(l[0],10),s<p.columns&&p.$headerIndexed[s]){switch(o=p.$headerIndexed[s][0],a=(""+l[1]).match(/^(1|d|s|o|n)/),a=a?a[0]:""){case"1":case"d":a=1;break;case"s":a=d||0;break;case"o":c=o.order[(d||0)%(p.sortReset?3:2)],a=0===c?1:1===c?0:2;break;case"n":o.count=o.count+1,a=o.order[o.count%(p.sortReset?3:2)];break;default:a=0}d=0===i?a:d,n=[s,parseInt(a,10)||0],p.sortList.push(n),a=e.inArray(n[1],o.order),o.count=a>=0?a:n[1]%(p.sortReset?3:2)}}function f(e,t){return e&&e[t]?e[t].type||"":""}function h(t,r,s){if(t.isUpdating)return setTimeout(function(){h(t,r,s)},50);var a,n,o,d,c,l,p,g=t.config,f=!s[g.sortMultiSortKey],b=g.$table,y=g.$headers.length;if(b.trigger("sortStart",t),r.count=s[g.sortResetKey]?2:(r.count+1)%(g.sortReset?3:2),g.sortRestart)for(n=r,o=0;y>o;o++)p=g.$headers.eq(o),p[0]===n||!f&&p.is("."+v.css.sortDesc+",."+v.css.sortAsc)||(p[0].count=-1);if(n=parseInt(e(r).attr("data-column"),10),f){if(g.sortList=[],null!==g.sortForce)for(a=g.sortForce,d=0;d<a.length;d++)a[d][0]!==n&&g.sortList.push(a[d]);if(c=r.order[r.count],2>c&&(g.sortList.push([n,c]),r.colSpan>1))for(d=1;d<r.colSpan;d++)g.sortList.push([n+d,c])}else{if(g.sortAppend&&g.sortList.length>1)for(d=0;d<g.sortAppend.length;d++)l=v.isValueInArray(g.sortAppend[d][0],g.sortList),l>=0&&g.sortList.splice(l,1);if(v.isValueInArray(n,g.sortList)>=0)for(d=0;d<g.sortList.length;d++)l=g.sortList[d],c=g.$headerIndexed[l[0]][0],l[0]===n&&(l[1]=c.order[r.count],2===l[1]&&(g.sortList.splice(d,1),c.count=-1));else if(c=r.order[r.count],2>c&&(g.sortList.push([n,c]),r.colSpan>1))for(d=1;d<r.colSpan;d++)g.sortList.push([n+d,c])}if(null!==g.sortAppend)for(a=g.sortAppend,d=0;d<a.length;d++)a[d][0]!==n&&g.sortList.push(a[d]);b.trigger("sortBegin",t),setTimeout(function(){u(t),m(t),i(t),b.trigger("sortEnd",t)},1)}function m(e){var t,a,n,o,i,d,c,l,p,u,g,h=0,m=e.config,b=m.textSorter||"",y=m.sortList,w=y.length,x=m.$tbodies.length;if(!m.serverSideSorting&&!s(m.cache)){for(m.debug&&(i=new Date),a=0;x>a;a++)d=m.cache[a].colMax,c=m.cache[a].normalized,c.sort(function(r,s){for(t=0;w>t;t++){if(o=y[t][0],l=y[t][1],h=0===l,m.sortStable&&r[o]===s[o]&&1===w)return r[m.columns].order-s[m.columns].order;if(n=/n/i.test(f(m.parsers,o)),n&&m.strings[o]?(n="boolean"==typeof m.string[m.strings[o]]?(h?1:-1)*(m.string[m.strings[o]]?-1:1):m.strings[o]?m.string[m.strings[o]]||0:0,p=m.numberSorter?m.numberSorter(r[o],s[o],h,d[o],e):v["sortNumeric"+(h?"Asc":"Desc")](r[o],s[o],n,d[o],o,e)):(u=h?r:s,g=h?s:r,p="function"==typeof b?b(u[o],g[o],h,o,e):"object"==typeof b&&b.hasOwnProperty(o)?b[o](u[o],g[o],h,o,e):v["sortNatural"+(h?"Asc":"Desc")](r[o],s[o],o,e,m)),p)return p}return r[m.columns].order-s[m.columns].order});m.debug&&r("Sorting on "+y.toString()+" and dir "+l+" time",i)}}function b(t,r){t.table.isUpdating&&t.$table.trigger("updateComplete",t.table),e.isFunction(r)&&r(t.table)}function y(t,r,s){var a=e.isArray(r)?r:t.sortList,n="undefined"==typeof r?t.resort:r;n===!1||t.serverSideSorting||t.table.isProcessing?(b(t,s),v.applyWidget(t.table,!1)):a.length?t.$table.trigger("sorton",[a,function(){b(t,s)},!0]):t.$table.trigger("sortReset",[function(){b(t,s),v.applyWidget(t.table,!1)}])}function w(t){var r=t.config,a=r.$table,d="sortReset update updateRows updateCell updateAll addRows updateComplete sorton appendCache updateCache applyWidgetId applyWidgets refreshWidgets destroy mouseup mouseleave ".split(" ").join(r.namespace+" ");a.unbind(d.replace(/\s+/g," ")).bind("sortReset"+r.namespace,function(s,a){s.stopPropagation(),r.sortList=[],u(t),m(t),i(t),e.isFunction(a)&&a(t)}).bind("updateAll"+r.namespace,function(e,s,a){e.stopPropagation(),t.isUpdating=!0,v.refreshWidgets(t,!0,!0),c(t),v.bindEvents(t,r.$headers,!0),w(t),l(t,s,a)}).bind("update"+r.namespace+" updateRows"+r.namespace,function(e,r,s){e.stopPropagation(),t.isUpdating=!0,p(t),l(t,r,s)}).bind("updateCell"+r.namespace,function(s,n,o,i){s.stopPropagation(),t.isUpdating=!0,a.find(r.selectorRemove).remove();var d,c,l,p,u=r.$tbodies,g=e(n),f=u.index(e.fn.closest?g.closest("tbody"):g.parents("tbody").filter(":first")),h=r.cache[f],m=e.fn.closest?g.closest("tr"):g.parents("tr").filter(":first");n=g[0],u.length&&f>=0&&(c=u.eq(f).find("tr").index(m),p=h.normalized[c],l=g.index(),d=v.getParsedText(r,n,l),p[l]=d,p[r.columns].$row=m,"numeric"===(r.parsers[l].type||"").toLowerCase()&&(h.colMax[l]=Math.max(Math.abs(d)||0,h.colMax[l]||0)),d="undefined"!==o?o:r.resort,d!==!1?y(r,d,i):(e.isFunction(i)&&i(t),r.$table.trigger("updateComplete",r.table)))}).bind("addRows"+r.namespace,function(a,o,i,d){if(a.stopPropagation(),t.isUpdating=!0,s(r.cache))p(t),l(t,i,d);else{o=e(o).attr("role","row");var c,u,g,f,h,m=o.filter("tr").length,b=r.$tbodies.index(o.parents("tbody").filter(":first"));for(r.parsers&&r.parsers.length||n(r),c=0;m>c;c++){for(g=o[c].cells.length,h=[],f={child:[],$row:o.eq(c),order:r.cache[b].normalized.length},u=0;g>u;u++)h[u]=v.getParsedText(r,o[c].cells[u],u),"numeric"===(r.parsers[u].type||"").toLowerCase()&&(r.cache[b].colMax[u]=Math.max(Math.abs(h[u])||0,r.cache[b].colMax[u]||0));h.push(f),r.cache[b].normalized.push(h)}y(r,i,d)}}).bind("updateComplete"+r.namespace,function(){t.isUpdating=!1}).bind("sorton"+r.namespace,function(r,n,d,c){var l=t.config;r.stopPropagation(),a.trigger("sortStart",this),g(t,n),u(t),l.delayInit&&s(l.cache)&&o(t),a.trigger("sortBegin",this),m(t),i(t,c),a.trigger("sortEnd",this),v.applyWidget(t),e.isFunction(d)&&d(t)}).bind("appendCache"+r.namespace,function(r,s,a){r.stopPropagation(),i(t,a),e.isFunction(s)&&s(t)}).bind("updateCache"+r.namespace,function(s,a,i){r.parsers&&r.parsers.length||n(r,i),o(t,i),e.isFunction(a)&&a(t)}).bind("applyWidgetId"+r.namespace,function(e,s){e.stopPropagation(),v.getWidgetById(s).format(t,r,r.widgetOptions)}).bind("applyWidgets"+r.namespace,function(e,r){e.stopPropagation(),v.applyWidget(t,r)}).bind("refreshWidgets"+r.namespace,function(e,r,s){e.stopPropagation(),v.refreshWidgets(t,r,s)}).bind("destroy"+r.namespace,function(e,r,s){e.stopPropagation(),v.destroy(t,r,s)}).bind("resetToLoadState"+r.namespace,function(){v.removeWidget(t,!0,!1),r=e.extend(!0,v.defaults,r.originalSettings),t.hasInitialized=!1,v.setup(t,r)})}var v=this;v.version="2.22.3",v.parsers=[],v.widgets=[],v.defaults={theme:"default",widthFixed:!1,showProcessing:!1,headerTemplate:"{content}",onRenderTemplate:null,onRenderHeader:null,cancelSelection:!0,tabIndex:!0,dateFormat:"mmddyyyy",sortMultiSortKey:"shiftKey",sortResetKey:"ctrlKey",usNumberFormat:!0,delayInit:!1,serverSideSorting:!1,resort:!0,headers:{},ignoreCase:!0,sortForce:null,sortList:[],sortAppend:null,sortStable:!1,sortInitialOrder:"asc",sortLocaleCompare:!1,sortReset:!1,sortRestart:!1,emptyTo:"bottom",stringTo:"max",textExtraction:"basic",textAttribute:"data-text",textSorter:null,numberSorter:null,widgets:[],widgetOptions:{zebra:["even","odd"]},initWidgets:!0,widgetClass:"widget-{name}",initialized:null,tableClass:"",cssAsc:"",cssDesc:"",cssNone:"",cssHeader:"",cssHeaderRow:"",cssProcessing:"",cssChildRow:"tablesorter-childRow",cssIcon:"tablesorter-icon",cssIconNone:"",cssIconAsc:"",cssIconDesc:"",cssInfoBlock:"tablesorter-infoOnly",cssNoSort:"tablesorter-noSort",cssIgnoreRow:"tablesorter-ignoreRow",pointerClick:"click",pointerDown:"mousedown",pointerUp:"mouseup",selectorHeaders:"> thead th, > thead td",selectorSort:"th, td",selectorRemove:".remove-me",debug:!1,headerList:[],empties:{},strings:{},parsers:[]},v.css={table:"tablesorter",cssHasChild:"tablesorter-hasChildRow",childRow:"tablesorter-childRow",colgroup:"tablesorter-colgroup",header:"tablesorter-header",headerRow:"tablesorter-headerRow",headerIn:"tablesorter-header-inner",icon:"tablesorter-icon",processing:"tablesorter-processing",sortAsc:"tablesorter-headerAsc",sortDesc:"tablesorter-headerDesc",sortNone:"tablesorter-headerUnSorted"},v.language={sortAsc:"Ascending sort applied, ",sortDesc:"Descending sort applied, ",sortNone:"No sort applied, ",nextAsc:"activate to apply an ascending sort",nextDesc:"activate to apply a descending sort",nextNone:"activate to remove the sort"},v.instanceMethods={},v.log=t,v.benchmark=r,v.getElementText=function(t,r,s){if(!r)return"";var a,n=t.textExtraction||"",o=r.jquery?r:e(r);return e.trim("string"==typeof n?"basic"===n&&"undefined"!=typeof(a=o.attr(t.textAttribute))?a:r.textContent||o.text():"function"==typeof n?n(o[0],t.table,s):"function"==typeof(a=v.getColumnData(t.table,n,s))?a(o[0],t.table,s):o[0].textContent||o.text())},v.getParsedText=function(e,t,r,s){"undefined"==typeof s&&(s=v.getElementText(e,t,r));var a=""+s,n=e.parsers[r],o=e.extractors[r];return n&&(o&&"function"==typeof o.format&&(s=o.format(s,e.table,t,r)),a="no-parser"===n.id?"":n.format(""+s,e.table,t,r),e.ignoreCase&&"string"==typeof a&&(a=a.toLowerCase())),a},v.construct=function(t){return this.each(function(){var r=this,s=e.extend(!0,{},v.defaults,t,v.instanceMethods);s.originalSettings=t,!r.hasInitialized&&v.buildTable&&"TABLE"!==this.nodeName?v.buildTable(r,s):v.setup(r,s)})},v.setup=function(r,s){if(!r||!r.tHead||0===r.tBodies.length||r.hasInitialized===!0)return s.debug?t("ERROR: stopping initialization! No table, thead, tbody or tablesorter has already been initialized"):"";var a="",i=e(r),d=e.metadata;r.hasInitialized=!1,r.isProcessing=!0,r.config=s,e.data(r,"tablesorter",s),s.debug&&e.data(r,"startoveralltimer",new Date),s.supportsDataObject=function(e){return e[0]=parseInt(e[0],10),e[0]>1||1===e[0]&&parseInt(e[1],10)>=4}(e.fn.jquery.split(".")),s.string={max:1,min:-1,emptymin:1,emptymax:-1,zero:0,none:0,"null":0,top:!0,bottom:!1},s.emptyTo=s.emptyTo.toLowerCase(),s.stringTo=s.stringTo.toLowerCase(),/tablesorter\-/.test(i.attr("class"))||(a=""!==s.theme?" tablesorter-"+s.theme:""),s.table=r,s.$table=i.addClass(v.css.table+" "+s.tableClass+a).attr("role","grid"),s.$headers=i.find(s.selectorHeaders),s.namespace=s.namespace?"."+s.namespace.replace(/\W/g,""):".tablesorter"+Math.random().toString(16).slice(2),s.$table.children().children("tr").attr("role","row"),s.$tbodies=i.children("tbody:not(."+s.cssInfoBlock+")").attr({"aria-live":"polite","aria-relevant":"all"}),s.$table.children("caption").length&&(a=s.$table.children("caption")[0],a.id||(a.id=s.namespace.slice(1)+"caption"),s.$table.attr("aria-labelledby",a.id)),s.widgetInit={},s.textExtraction=s.$table.attr("data-text-extraction")||s.textExtraction||"basic",c(r),v.fixColumnWidth(r),v.applyWidgetOptions(r,s),n(s),s.totalRows=0,s.delayInit||o(r),v.bindEvents(r,s.$headers,!0),w(r),s.supportsDataObject&&"undefined"!=typeof i.data().sortlist?s.sortList=i.data().sortlist:d&&i.metadata()&&i.metadata().sortlist&&(s.sortList=i.metadata().sortlist),v.applyWidget(r,!0),s.sortList.length>0?i.trigger("sorton",[s.sortList,{},!s.initWidgets,!0]):(u(r),s.initWidgets&&v.applyWidget(r,!1)),s.showProcessing&&i.unbind("sortBegin"+s.namespace+" sortEnd"+s.namespace).bind("sortBegin"+s.namespace+" sortEnd"+s.namespace,function(e){clearTimeout(s.processTimer),v.isProcessing(r),"sortBegin"===e.type&&(s.processTimer=setTimeout(function(){v.isProcessing(r,!0)},500))}),r.hasInitialized=!0,r.isProcessing=!1,s.debug&&v.benchmark("Overall initialization time",e.data(r,"startoveralltimer")),i.trigger("tablesorter-initialized",r),"function"==typeof s.initialized&&s.initialized(r)},v.fixColumnWidth=function(t){t=e(t)[0];var r,s,a,n,o,i=t.config,d=i.$table.children("colgroup");if(d.length&&d.hasClass(v.css.colgroup)&&d.remove(),i.widthFixed&&0===i.$table.children("colgroup").length){for(d=e('<colgroup class="'+v.css.colgroup+'">'),r=i.$table.width(),a=i.$tbodies.find("tr:first").children(":visible"),n=a.length,o=0;n>o;o++)s=parseInt(a.eq(o).width()/r*1e3,10)/10+"%",d.append(e("<col>").css("width",s));i.$table.prepend(d)}},v.getColumnData=function(t,r,s,a,n){if("undefined"!=typeof r&&null!==r){t=e(t)[0];var o,i,d=t.config,c=n||d.$headers,l=d.$headerIndexed&&d.$headerIndexed[s]||c.filter('[data-column="'+s+'"]:last');if(r[s])return a?r[s]:r[c.index(l)];for(i in r)if("string"==typeof i&&(o=l.filter(i).add(l.find(i)),o.length))return r[i]}},v.computeColumnIndex=function(t){var r,s,a,n,o,i,d,c,l,p,u,g,f=[],h=[],m={};for(r=0;r<t.length;r++)for(d=t[r].cells,s=0;s<d.length;s++){for(i=d[s],o=e(i),c=i.parentNode.rowIndex,l=c+"-"+o.index(),p=i.rowSpan||1,u=i.colSpan||1,"undefined"==typeof f[c]&&(f[c]=[]),a=0;a<f[c].length+1;a++)if("undefined"==typeof f[c][a]){g=a;break}for(m[l]=g,o.attr({"data-column":g}),a=c;c+p>a;a++)for("undefined"==typeof f[a]&&(f[a]=[]),h=f[a],n=g;g+u>n;n++)h[n]="x"}return h.length},v.isProcessing=function(t,r,s){t=e(t);var a=t[0].config,n=s||t.find("."+v.css.header);r?("undefined"!=typeof s&&a.sortList.length>0&&(n=n.filter(function(){return this.sortDisabled?!1:v.isValueInArray(parseFloat(e(this).attr("data-column")),a.sortList)>=0})),t.add(n).addClass(v.css.processing+" "+a.cssProcessing)):t.add(n).removeClass(v.css.processing+" "+a.cssProcessing)},v.processTbody=function(t,r,s){t=e(t)[0];var a;return s?(t.isProcessing=!0,r.before('<span class="tablesorter-savemyplace"/>'),a=e.fn.detach?r.detach():r.remove()):(a=e(t).find("span.tablesorter-savemyplace"),r.insertAfter(a),a.remove(),void(t.isProcessing=!1))},v.clearTableBody=function(t){e(t)[0].config.$tbodies.children().detach()},v.bindEvents=function(t,r,a){t=e(t)[0];var n,i=null,d=t.config;a!==!0&&(r.addClass(d.namespace.slice(1)+"_extra_headers"),n=e.fn.closest?r.closest("table")[0]:r.parents("table")[0],n&&"TABLE"===n.nodeName&&n!==t&&e(n).addClass(d.namespace.slice(1)+"_extra_table")),n=(d.pointerDown+" "+d.pointerUp+" "+d.pointerClick+" sort keyup ").replace(/\s+/g," ").split(" ").join(d.namespace+" "),r.find(d.selectorSort).add(r.filter(d.selectorSort)).unbind(n).bind(n,function(a,n){var c,l,p=e(a.target),u=" "+a.type+" ";if(!(1!==(a.which||a.button)&&!u.match(" "+d.pointerClick+" | sort | keyup ")||" keyup "===u&&13!==a.which||u.match(" "+d.pointerClick+" ")&&"undefined"!=typeof a.which||u.match(" "+d.pointerUp+" ")&&i!==a.target&&n!==!0)){if(u.match(" "+d.pointerDown+" "))return i=a.target,l=p.jquery.split("."),void("1"===l[0]&&l[1]<4&&a.preventDefault());if(i=null,/(input|select|button|textarea)/i.test(a.target.nodeName)||p.hasClass(d.cssNoSort)||p.parents("."+d.cssNoSort).length>0||p.parents("button").length>0)return!d.cancelSelection;d.delayInit&&s(d.cache)&&o(t),c=e.fn.closest?e(this).closest("th, td")[0]:/TH|TD/.test(this.nodeName)?this:e(this).parents("th, td")[0],c=d.$headers[r.index(c)],c.sortDisabled||h(t,c,a)}}),d.cancelSelection&&r.attr("unselectable","on").bind("selectstart",!1).css({"user-select":"none",MozUserSelect:"none"})},v.restoreHeaders=function(t){var r,s,a=e(t)[0].config,n=a.$table.find(a.selectorHeaders),o=n.length;for(r=0;o>r;r++)s=n.eq(r),s.find("."+v.css.headerIn).length&&s.html(a.headerContent[r])},v.destroy=function(t,r,s){if(t=e(t)[0],t.hasInitialized){v.removeWidget(t,!0,!1);var a,n=e(t),o=t.config,i=n.find("thead:first"),d=i.find("tr."+v.css.headerRow).removeClass(v.css.headerRow+" "+o.cssHeaderRow),c=n.find("tfoot:first > tr").children("th, td");r===!1&&e.inArray("uitheme",o.widgets)>=0&&(n.trigger("applyWidgetId",["uitheme"]),n.trigger("applyWidgetId",["zebra"])),i.find("tr").not(d).remove(),a="sortReset update updateAll updateRows updateCell addRows updateComplete sorton appendCache updateCache "+"applyWidgetId applyWidgets refreshWidgets destroy mouseup mouseleave keypress sortBegin sortEnd resetToLoadState ".split(" ").join(o.namespace+" "),n.removeData("tablesorter").unbind(a.replace(/\s+/g," ")),o.$headers.add(c).removeClass([v.css.header,o.cssHeader,o.cssAsc,o.cssDesc,v.css.sortAsc,v.css.sortDesc,v.css.sortNone].join(" ")).removeAttr("data-column").removeAttr("aria-label").attr("aria-disabled","true"),d.find(o.selectorSort).unbind("mousedown mouseup keypress ".split(" ").join(o.namespace+" ").replace(/\s+/g," ")),v.restoreHeaders(t),n.toggleClass(v.css.table+" "+o.tableClass+" tablesorter-"+o.theme,r===!1),t.hasInitialized=!1,delete t.config.cache,"function"==typeof s&&s(t)}},v.regex={chunk:/(^([+\-]?(?:0|[1-9]\d*)(?:\.\d*)?(?:[eE][+\-]?\d+)?)?$|^0x[0-9a-f]+$|\d+)/gi,chunks:/(^\\0|\\0$)/,hex:/^0x[0-9a-f]+$/i},v.sortNatural=function(e,t){if(e===t)return 0;var r,s,a,n,o,i,d,c,l=v.regex;if(l.hex.test(t)){if(s=parseInt(e.match(l.hex),16),n=parseInt(t.match(l.hex),16),n>s)return-1;if(s>n)return 1}for(r=e.replace(l.chunk,"\\0$1\\0").replace(l.chunks,"").split("\\0"),a=t.replace(l.chunk,"\\0$1\\0").replace(l.chunks,"").split("\\0"),c=Math.max(r.length,a.length),d=0;c>d;d++){if(o=isNaN(r[d])?r[d]||0:parseFloat(r[d])||0,i=isNaN(a[d])?a[d]||0:parseFloat(a[d])||0,isNaN(o)!==isNaN(i))return isNaN(o)?1:-1;if(typeof o!=typeof i&&(o+="",i+=""),i>o)return-1;if(o>i)return 1}return 0},v.sortNaturalAsc=function(e,t,r,s,a){if(e===t)return 0;var n=a.string[a.empties[r]||a.emptyTo];return""===e&&0!==n?"boolean"==typeof n?n?-1:1:-n||-1:""===t&&0!==n?"boolean"==typeof n?n?1:-1:n||1:v.sortNatural(e,t)},v.sortNaturalDesc=function(e,t,r,s,a){if(e===t)return 0;var n=a.string[a.empties[r]||a.emptyTo];return""===e&&0!==n?"boolean"==typeof n?n?-1:1:n||1:""===t&&0!==n?"boolean"==typeof n?n?1:-1:-n||-1:v.sortNatural(t,e)},v.sortText=function(e,t){return e>t?1:t>e?-1:0},v.getTextValue=function(e,t,r){if(r){var s,a=e?e.length:0,n=r+t;for(s=0;a>s;s++)n+=e.charCodeAt(s);return t*n}return 0},v.sortNumericAsc=function(e,t,r,s,a,n){if(e===t)return 0;var o=n.config,i=o.string[o.empties[a]||o.emptyTo];return""===e&&0!==i?"boolean"==typeof i?i?-1:1:-i||-1:""===t&&0!==i?"boolean"==typeof i?i?1:-1:i||1:(isNaN(e)&&(e=v.getTextValue(e,r,s)),isNaN(t)&&(t=v.getTextValue(t,r,s)),e-t)},v.sortNumericDesc=function(e,t,r,s,a,n){if(e===t)return 0;var o=n.config,i=o.string[o.empties[a]||o.emptyTo];return""===e&&0!==i?"boolean"==typeof i?i?-1:1:i||1:""===t&&0!==i?"boolean"==typeof i?i?1:-1:-i||-1:(isNaN(e)&&(e=v.getTextValue(e,r,s)),isNaN(t)&&(t=v.getTextValue(t,r,s)),t-e)},v.sortNumeric=function(e,t){return e-t},v.characterEquivalents={a:"áàâãäąå",A:"ÁÀÂÃÄĄÅ",c:"çćč",C:"ÇĆČ",e:"éèêëěę",E:"ÉÈÊËĚĘ",i:"íìİîïı",I:"ÍÌİÎÏ",o:"óòôõöō",O:"ÓÒÔÕÖŌ",ss:"ß",SS:"ẞ",u:"úùûüů",U:"ÚÙÛÜŮ"},v.replaceAccents=function(e){var t,r="[",s=v.characterEquivalents;if(!v.characterRegex){v.characterRegexArray={};for(t in s)"string"==typeof t&&(r+=s[t],v.characterRegexArray[t]=new RegExp("["+s[t]+"]","g"));v.characterRegex=new RegExp(r+"]")}if(v.characterRegex.test(e))for(t in s)"string"==typeof t&&(e=e.replace(v.characterRegexArray[t],t));return e},v.isValueInArray=function(e,t){var r,s=t.length;for(r=0;s>r;r++)if(t[r][0]===e)return r;return-1},v.addParser=function(e){var t,r=v.parsers.length,s=!0;for(t=0;r>t;t++)v.parsers[t].id.toLowerCase()===e.id.toLowerCase()&&(s=!1);s&&v.parsers.push(e)},v.addInstanceMethods=function(t){e.extend(v.instanceMethods,t)},v.getParserById=function(e){if("false"==e)return!1;var t,r=v.parsers.length;for(t=0;r>t;t++)if(v.parsers[t].id.toLowerCase()===e.toString().toLowerCase())return v.parsers[t];return!1},v.addWidget=function(e){v.widgets.push(e)},v.hasWidget=function(t,r){return t=e(t),t.length&&t[0].config&&t[0].config.widgetInit[r]||!1},v.getWidgetById=function(e){var t,r,s=v.widgets.length;for(t=0;s>t;t++)if(r=v.widgets[t],r&&r.hasOwnProperty("id")&&r.id.toLowerCase()===e.toLowerCase())return r},v.applyWidgetOptions=function(t,r){var s,a,n=r.widgets.length,o=r.widgetOptions;if(n)for(s=0;n>s;s++)a=v.getWidgetById(r.widgets[s]),a&&"options"in a&&(o=t.config.widgetOptions=e.extend(!0,{},a.options,o))},v.applyWidget=function(t,s,a){t=e(t)[0];var n,o,i,d,c,l,p,u=t.config,g=u.widgetOptions,f=" "+u.table.className+" ",h=[];if(s===!1||!t.hasInitialized||!t.isApplyingWidgets&&!t.isUpdating){if(u.debug&&(d=new Date),p=new RegExp("\\s"+u.widgetClass.replace(/\{name\}/i,"([\\w-]+)")+"\\s","g"),f.match(p)&&(l=f.match(p)))for(o=l.length,n=0;o>n;n++)u.widgets.push(l[n].replace(p,"$1"));if(u.widgets.length){for(t.isApplyingWidgets=!0,u.widgets=e.grep(u.widgets,function(t,r){return e.inArray(t,u.widgets)===r}),i=u.widgets||[],o=i.length,n=0;o>n;n++)p=v.getWidgetById(i[n]),p&&p.id&&(p.priority||(p.priority=10),h[n]=p);for(h.sort(function(e,t){return e.priority<t.priority?-1:e.priority===t.priority?0:1}),o=h.length,n=0;o>n;n++)h[n]&&((s||!u.widgetInit[h[n].id])&&(u.widgetInit[h[n].id]=!0,t.hasInitialized&&v.applyWidgetOptions(t,u),"init"in h[n]&&(u.debug&&(c=new Date),h[n].init(t,h[n],u,g),u.debug&&v.benchmark("Initializing "+h[n].id+" widget",c))),!s&&"format"in h[n]&&(u.debug&&(c=new Date),h[n].format(t,u,g,!1),u.debug&&v.benchmark((s?"Initializing ":"Applying ")+h[n].id+" widget",c)));s||"function"!=typeof a||a(t)}setTimeout(function(){t.isApplyingWidgets=!1,e.data(t,"lastWidgetApplication",new Date)},0),u.debug&&(l=u.widgets.length,r("Completed "+(s===!0?"initializing ":"applying ")+l+" widget"+(1!==l?"s":""),d))}},v.removeWidget=function(r,s,a){r=e(r)[0];var n,o,i,d,c=r.config;if(s===!0)for(s=[],d=v.widgets.length,i=0;d>i;i++)o=v.widgets[i],o&&o.id&&s.push(o.id);else s=(e.isArray(s)?s.join(","):s||"").toLowerCase().split(/[\s,]+/);for(d=s.length,n=0;d>n;n++)o=v.getWidgetById(s[n]),i=e.inArray(s[n],c.widgets),o&&"remove"in o&&(c.debug&&i>=0&&t('Removing "'+s[n]+'" widget'),o.remove(r,c,c.widgetOptions,a),c.widgetInit[s[n]]=!1),i>=0&&a!==!0&&c.widgets.splice(i,1)},v.refreshWidgets=function(t,r,s){t=e(t)[0];var a,n=t.config,o=n.widgets,i=v.widgets,d=i.length,c=[],l=function(t){e(t).trigger("refreshComplete")};for(a=0;d>a;a++)i[a]&&i[a].id&&(r||e.inArray(i[a].id,o)<0)&&c.push(i[a].id);v.removeWidget(t,c.join(","),!0),s!==!0?(v.applyWidget(t,r||!1,l),r&&v.applyWidget(t,!1,l)):l(t)},v.getColumnText=function(t,r,a){t=e(t)[0];var n,o,i,d,c,l,p,u,g,f,h="function"==typeof a,m="all"===r,b={raw:[],parsed:[],$cell:[]},y=t.config;if(!s(y)){for(c=y.$tbodies.length,n=0;c>n;n++)for(i=y.cache[n].normalized,l=i.length,o=0;l>o;o++)f=!0,d=i[o],u=m?d.slice(0,y.columns):d[r],d=d[y.columns],p=m?d.raw:d.raw[r],g=m?d.$row.children():d.$row.children().eq(r),h&&(f=a({tbodyIndex:n,rowIndex:o,parsed:u,raw:p,$row:d.$row,$cell:g})),f!==!1&&(b.parsed.push(u),b.raw.push(p),b.$cell.push(g));return b}},v.getData=function(t,r,s){var a,n,o="",i=e(t);return i.length?(a=e.metadata?i.metadata():!1,n=" "+(i.attr("class")||""),"undefined"!=typeof i.data(s)||"undefined"!=typeof i.data(s.toLowerCase())?o+=i.data(s)||i.data(s.toLowerCase()):a&&"undefined"!=typeof a[s]?o+=a[s]:r&&"undefined"!=typeof r[s]?o+=r[s]:" "!==n&&n.match(" "+s+"-")&&(o=n.match(new RegExp("\\s"+s+"-([\\w-]+)"))[1]||""),e.trim(o)):""},v.formatFloat=function(t,r){if("string"!=typeof t||""===t)return t;var s,a=r&&r.config?r.config.usNumberFormat!==!1:"undefined"!=typeof r?r:!0;return t=a?t.replace(/,/g,""):t.replace(/[\s|\.]/g,"").replace(/,/g,"."),/^\s*\([.\d]+\)/.test(t)&&(t=t.replace(/^\s*\(([.\d]+)\)/,"-$1")),s=parseFloat(t),isNaN(s)?e.trim(t):s},v.isDigit=function(e){return isNaN(e)?/^[\-+(]?\d+[)]?$/.test(e.toString().replace(/[,.'"\s]/g,"")):""!==e}}});var t=e.tablesorter;e.fn.extend({tablesorter:t.construct}),t.addParser({id:"no-parser",is:function(){return!1},format:function(){return""},type:"text"}),t.addParser({id:"text",is:function(){return!0},format:function(r,s){var a=s.config;return r&&(r=e.trim(a.ignoreCase?r.toLocaleLowerCase():r),r=a.sortLocaleCompare?t.replaceAccents(r):r),r},type:"text"}),t.addParser({id:"digit",is:function(e){return t.isDigit(e)},format:function(r,s){var a=t.formatFloat((r||"").replace(/[^\w,. \-()]/g,""),s);return r&&"number"==typeof a?a:r?e.trim(r&&s.config.ignoreCase?r.toLocaleLowerCase():r):r},type:"numeric"}),t.addParser({id:"currency",is:function(e){return/^\(?\d+[\u00a3$\u20ac\u00a4\u00a5\u00a2?.]|[\u00a3$\u20ac\u00a4\u00a5\u00a2?.]\d+\)?$/.test((e||"").replace(/[+\-,. ]/g,""))},format:function(r,s){var a=t.formatFloat((r||"").replace(/[^\w,. \-()]/g,""),s);return r&&"number"==typeof a?a:r?e.trim(r&&s.config.ignoreCase?r.toLocaleLowerCase():r):r},type:"numeric"}),t.addParser({id:"url",is:function(e){return/^(https?|ftp|file):\/\//.test(e)},format:function(t){return t?e.trim(t.replace(/(https?|ftp|file):\/\//,"")):t},parsed:!0,type:"text"}),t.addParser({id:"isoDate",is:function(e){return/^\d{4}[\/\-]\d{1,2}[\/\-]\d{1,2}/.test(e)},format:function(e){var t=e?new Date(e.replace(/-/g,"/")):e;return t instanceof Date&&isFinite(t)?t.getTime():e},type:"numeric"}),t.addParser({id:"percent",is:function(e){return/(\d\s*?%|%\s*?\d)/.test(e)&&e.length<15},format:function(e,r){return e?t.formatFloat(e.replace(/%/g,""),r):e},type:"numeric"}),t.addParser({id:"image",is:function(e,t,r,s){return s.find("img").length>0},format:function(t,r,s){return e(s).find("img").attr(r.config.imgAttr||"alt")||t},parsed:!0,type:"text"}),t.addParser({id:"usLongDate",is:function(e){return/^[A-Z]{3,10}\.?\s+\d{1,2},?\s+(\d{4})(\s+\d{1,2}:\d{2}(:\d{2})?(\s+[AP]M)?)?$/i.test(e)||/^\d{1,2}\s+[A-Z]{3,10}\s+\d{4}/i.test(e)},format:function(e){var t=e?new Date(e.replace(/(\S)([AP]M)$/i,"$1 $2")):e;return t instanceof Date&&isFinite(t)?t.getTime():e +},type:"numeric"}),t.addParser({id:"shortDate",is:function(e){return/(^\d{1,2}[\/\s]\d{1,2}[\/\s]\d{4})|(^\d{4}[\/\s]\d{1,2}[\/\s]\d{1,2})/.test((e||"").replace(/\s+/g," ").replace(/[\-.,]/g,"/"))},format:function(e,r,s,a){if(e){var n,o,i=r.config,d=i.$headerIndexed[a],c=d.length&&d[0].dateFormat||t.getData(d,t.getColumnData(r,i.headers,a),"dateFormat")||i.dateFormat;return o=e.replace(/\s+/g," ").replace(/[\-.,]/g,"/"),"mmddyyyy"===c?o=o.replace(/(\d{1,2})[\/\s](\d{1,2})[\/\s](\d{4})/,"$3/$1/$2"):"ddmmyyyy"===c?o=o.replace(/(\d{1,2})[\/\s](\d{1,2})[\/\s](\d{4})/,"$3/$2/$1"):"yyyymmdd"===c&&(o=o.replace(/(\d{4})[\/\s](\d{1,2})[\/\s](\d{1,2})/,"$1/$2/$3")),n=new Date(o),n instanceof Date&&isFinite(n)?n.getTime():e}return e},type:"numeric"}),t.addParser({id:"time",is:function(e){return/^(([0-2]?\d:[0-5]\d)|([0-1]?\d:[0-5]\d\s?([AP]M)))$/i.test(e)},format:function(e){var t=e?new Date("2000/01/01 "+e.replace(/(\S)([AP]M)$/i,"$1 $2")):e;return t instanceof Date&&isFinite(t)?t.getTime():e},type:"numeric"}),t.addParser({id:"metadata",is:function(){return!1},format:function(t,r,s){var a=r.config,n=a.parserMetadataName?a.parserMetadataName:"sortValue";return e(s).metadata()[n]},type:"numeric"}),t.addWidget({id:"zebra",priority:90,format:function(t,r,s){var a,n,o,i,d,c,l,p,u=new RegExp(r.cssChildRow,"i"),g=r.$tbodies.add(e(r.namespace+"_extra_table").children("tbody:not(."+r.cssInfoBlock+")"));for(r.debug&&(d=new Date),c=0;c<g.length;c++)for(o=0,a=g.eq(c).children("tr:visible").not(r.selectorRemove),p=a.length,l=0;p>l;l++)n=a.eq(l),u.test(n[0].className)||o++,i=o%2===0,n.removeClass(s.zebra[i?1:0]).addClass(s.zebra[i?0:1])},remove:function(e,r,s,a){if(!a){var n,o,i=r.$tbodies,d=(s.zebra||["even","odd"]).join(" ");for(n=0;n<i.length;n++)o=t.processTbody(e,i.eq(n),!0),o.children().removeClass(d),t.processTbody(e,o,!1)}}})}(jQuery);
/* jquery.tablesorter.widgets.js v2.22.3 */ !function(e){"function"==typeof define&&define.amd?define(["jquery"],e):"object"==typeof module&&"object"==typeof module.exports?module.exports=e(require("jquery")):e(jQuery)}(function(e){return function(e,t,r){"use strict";var i=e.tablesorter||{};i.storage=function(l,a,s,n){l=e(l)[0];var o,c,d,f=!1,u={},h=l.config,p=h&&h.widgetOptions,g=n&&n.useSessionStorage||p&&p.storage_useSessionStorage?"sessionStorage":"localStorage",m=e(l),b=n&&n.id||m.attr(n&&n.group||p&&p.storage_group||"data-table-group")||p&&p.storage_tableId||l.id||e(".tablesorter").index(m),y=n&&n.url||m.attr(n&&n.page||p&&p.storage_page||"data-table-page")||p&&p.storage_fixedUrl||h&&h.fixedUrl||t.location.pathname;if(g in t)try{t[g].setItem("_tmptest","temp"),f=!0,t[g].removeItem("_tmptest")}catch(_){h&&h.debug&&i.log(g+" is not supported in this browser")}return e.parseJSON&&(f?u=e.parseJSON(t[g][a]||"null")||{}:(c=r.cookie.split(/[;\s|=]/),o=e.inArray(a,c)+1,u=0!==o?e.parseJSON(c[o]||"null")||{}:{})),(s||""===s)&&t.JSON&&JSON.hasOwnProperty("stringify")?(u[y]||(u[y]={}),u[y][b]=s,f?t[g][a]=JSON.stringify(u):(d=new Date,d.setTime(d.getTime()+31536e6),r.cookie=a+"="+JSON.stringify(u).replace(/\"/g,'"')+"; expires="+d.toGMTString()+"; path=/"),void 0):u&&u[y]?u[y][b]:""}}(jQuery,window,document),function(e){"use strict";var t=e.tablesorter||{};t.themes={bootstrap:{table:"table table-bordered table-striped",caption:"caption",header:"bootstrap-header",sortNone:"",sortAsc:"",sortDesc:"",active:"",hover:"",icons:"",iconSortNone:"bootstrap-icon-unsorted",iconSortAsc:"icon-chevron-up glyphicon glyphicon-chevron-up",iconSortDesc:"icon-chevron-down glyphicon glyphicon-chevron-down",filterRow:"",footerRow:"",footerCells:"",even:"",odd:""},jui:{table:"ui-widget ui-widget-content ui-corner-all",caption:"ui-widget-content",header:"ui-widget-header ui-corner-all ui-state-default",sortNone:"",sortAsc:"",sortDesc:"",active:"ui-state-active",hover:"ui-state-hover",icons:"ui-icon",iconSortNone:"ui-icon-carat-2-n-s",iconSortAsc:"ui-icon-carat-1-n",iconSortDesc:"ui-icon-carat-1-s",filterRow:"",footerRow:"",footerCells:"",even:"ui-widget-content",odd:"ui-state-default"}},e.extend(t.css,{wrapper:"tablesorter-wrapper"}),t.addWidget({id:"uitheme",priority:10,format:function(r,i,l){var a,s,n,o,c,d,f,u,h,p,g,m,b=t.themes,y=i.$table.add(e(i.namespace+"_extra_table")),_=i.$headers.add(e(i.namespace+"_extra_headers")),v=i.theme||"jui",w=b[v]||{},x=e.trim([w.sortNone,w.sortDesc,w.sortAsc,w.active].join(" ")),C=e.trim([w.iconSortNone,w.iconSortDesc,w.iconSortAsc].join(" "));for(i.debug&&(o=new Date),y.hasClass("tablesorter-"+v)&&i.theme===i.appliedTheme&&l.uitheme_applied||(l.uitheme_applied=!0,h=b[i.appliedTheme]||{},m=!e.isEmptyObject(h),p=m?[h.sortNone,h.sortDesc,h.sortAsc,h.active].join(" "):"",g=m?[h.iconSortNone,h.iconSortDesc,h.iconSortAsc].join(" "):"",m&&(l.zebra[0]=e.trim(" "+l.zebra[0].replace(" "+h.even,"")),l.zebra[1]=e.trim(" "+l.zebra[1].replace(" "+h.odd,"")),i.$tbodies.children().removeClass([h.even,h.odd].join(" "))),w.even&&(l.zebra[0]+=" "+w.even),w.odd&&(l.zebra[1]+=" "+w.odd),y.children("caption").removeClass(h.caption||"").addClass(w.caption),f=y.removeClass((i.appliedTheme?"tablesorter-"+(i.appliedTheme||""):"")+" "+(h.table||"")).addClass("tablesorter-"+v+" "+(w.table||"")).children("tfoot"),i.appliedTheme=i.theme,f.length&&f.children("tr").removeClass(h.footerRow||"").addClass(w.footerRow).children("th, td").removeClass(h.footerCells||"").addClass(w.footerCells),_.removeClass((m?[h.header,h.hover,p].join(" "):"")||"").addClass(w.header).not(".sorter-false").unbind("mouseenter.tsuitheme mouseleave.tsuitheme").bind("mouseenter.tsuitheme mouseleave.tsuitheme",function(t){e(this)["mouseenter"===t.type?"addClass":"removeClass"](w.hover||"")}),_.each(function(){var r=e(this);r.find("."+t.css.wrapper).length||r.wrapInner('<div class="'+t.css.wrapper+'" style="position:relative;height:100%;width:100%"></div>')}),i.cssIcon&&_.find("."+t.css.icon).removeClass(m?[h.icons,g].join(" "):"").addClass(w.icons||""),y.hasClass("hasFilters")&&y.children("thead").children("."+t.css.filterRow).removeClass(m?h.filterRow||"":"").addClass(w.filterRow||"")),a=0;a<i.columns;a++)c=i.$headers.add(e(i.namespace+"_extra_headers")).not(".sorter-false").filter('[data-column="'+a+'"]'),d=t.css.icon?c.find("."+t.css.icon):e(),u=_.not(".sorter-false").filter('[data-column="'+a+'"]:last'),u.length&&(c.removeClass(x),d.removeClass(C),u[0].sortDisabled?d.removeClass(w.icons||""):(s=w.sortNone,n=w.iconSortNone,u.hasClass(t.css.sortAsc)?(s=[w.sortAsc,w.active].join(" "),n=w.iconSortAsc):u.hasClass(t.css.sortDesc)&&(s=[w.sortDesc,w.active].join(" "),n=w.iconSortDesc),c.addClass(s),d.addClass(n||"")));i.debug&&t.benchmark("Applying "+v+" theme",o)},remove:function(e,r,i,l){if(i.uitheme_applied){var a=r.$table,s=r.appliedTheme||"jui",n=t.themes[s]||t.themes.jui,o=a.children("thead").children(),c=n.sortNone+" "+n.sortDesc+" "+n.sortAsc,d=n.iconSortNone+" "+n.iconSortDesc+" "+n.iconSortAsc;a.removeClass("tablesorter-"+s+" "+n.table),i.uitheme_applied=!1,l||(a.find(t.css.header).removeClass(n.header),o.unbind("mouseenter.tsuitheme mouseleave.tsuitheme").removeClass(n.hover+" "+c+" "+n.active).filter("."+t.css.filterRow).removeClass(n.filterRow),o.find("."+t.css.icon).removeClass(n.icons+" "+d))}}})}(jQuery),function(e){"use strict";var t=e.tablesorter||{};t.addWidget({id:"columns",priority:30,options:{columns:["primary","secondary","tertiary"]},format:function(r,i,l){var a,s,n,o,c,d,f,u,h=i.$table,p=i.$tbodies,g=i.sortList,m=g.length,b=l&&l.columns||["primary","secondary","tertiary"],y=b.length-1;for(f=b.join(" "),s=0;s<p.length;s++)a=t.processTbody(r,p.eq(s),!0),n=a.children("tr"),n.each(function(){if(c=e(this),"none"!==this.style.display&&(d=c.children().removeClass(f),g&&g[0]&&(d.eq(g[0][0]).addClass(b[0]),m>1)))for(u=1;m>u;u++)d.eq(g[u][0]).addClass(b[u]||b[y])}),t.processTbody(r,a,!1);if(o=l.columns_thead!==!1?["thead tr"]:[],l.columns_tfoot!==!1&&o.push("tfoot tr"),o.length&&(n=h.find(o.join(",")).children().removeClass(f),m))for(u=0;m>u;u++)n.filter('[data-column="'+g[u][0]+'"]').addClass(b[u]||b[y])},remove:function(r,i,l){var a,s,n=i.$tbodies,o=(l.columns||["primary","secondary","tertiary"]).join(" ");for(i.$headers.removeClass(o),i.$table.children("tfoot").children("tr").children("th, td").removeClass(o),a=0;a<n.length;a++)s=t.processTbody(r,n.eq(a),!0),s.children("tr").each(function(){e(this).children().removeClass(o)}),t.processTbody(r,s,!1)}})}(jQuery),function(e){"use strict";var t=e.tablesorter||{},r=t.css;e.extend(r,{filterRow:"tablesorter-filter-row",filter:"tablesorter-filter",filterDisabled:"disabled",filterRowHide:"hideme"}),t.addWidget({id:"filter",priority:50,options:{filter_childRows:!1,filter_childByColumn:!1,filter_columnFilters:!0,filter_columnAnyMatch:!0,filter_cellFilter:"",filter_cssFilter:"",filter_defaultFilter:{},filter_excludeFilter:{},filter_external:"",filter_filteredRow:"filtered",filter_formatter:null,filter_functions:null,filter_hideEmpty:!0,filter_hideFilters:!1,filter_ignoreCase:!0,filter_liveSearch:!0,filter_onlyAvail:"filter-onlyAvail",filter_placeholder:{search:"",select:""},filter_reset:null,filter_saveFilters:!1,filter_searchDelay:300,filter_searchFiltered:!0,filter_selectSource:null,filter_startsWith:!1,filter_useParsedData:!1,filter_serversideFiltering:!1,filter_defaultAttrib:"data-value",filter_selectSourceSeparator:"|"},format:function(e,r,i){r.$table.hasClass("hasFilters")||t.filter.init(e,r,i)},remove:function(i,l,a,s){var n,o,c=l.$table,d=l.$tbodies,f="addRows updateCell update updateRows updateComplete appendCache filterReset filterEnd search ".split(" ").join(l.namespace+"filter ");if(c.removeClass("hasFilters").unbind(f.replace(/\s+/g," ")).find("."+r.filterRow).remove(),!s){for(n=0;n<d.length;n++)o=t.processTbody(i,d.eq(n),!0),o.children().removeClass(a.filter_filteredRow).show(),t.processTbody(i,o,!1);a.filter_reset&&e(document).undelegate(a.filter_reset,"click.tsfilter")}}}),t.filter={regex:{regex:/^\/((?:\\\/|[^\/])+)\/([mig]{0,3})?$/,child:/tablesorter-childRow/,filtered:/filtered/,type:/undefined|number/,exact:/(^[\"\'=]+)|([\"\'=]+$)/g,nondigit:/[^\w,. \-()]/g,operators:/[<>=]/g,query:"(q|query)"},types:{or:function(r,i,l){if(/\|/.test(i.iFilter)||t.filter.regex.orSplit.test(i.filter)){var a,s,n,o,c=e.extend({},i),d=i.index,f=i.parsed[d],u=i.filter.split(t.filter.regex.orSplit),h=i.iFilter.split(t.filter.regex.orSplit),p=u.length;for(a=0;p>a;a++)if(c.nestedFilters=!0,c.filter=""+(t.filter.parseFilter(r,u[a],d,f)||""),c.iFilter=""+(t.filter.parseFilter(r,h[a],d,f)||""),n="("+(t.filter.parseFilter(r,c.filter,d,f)||"")+")",o=new RegExp(i.isMatch?n:"^"+n+"$",r.widgetOptions.filter_ignoreCase?"i":""),s=o.test(c.exact)||t.filter.processTypes(r,c,l))return s;return s||!1}return null},and:function(r,i,l){if(t.filter.regex.andTest.test(i.filter)){var a,s,n,o,c,d=e.extend({},i),f=i.index,u=i.parsed[f],h=i.filter.split(t.filter.regex.andSplit),p=i.iFilter.split(t.filter.regex.andSplit),g=h.length;for(a=0;g>a;a++)d.nestedFilters=!0,d.filter=""+(t.filter.parseFilter(r,h[a],f,u)||""),d.iFilter=""+(t.filter.parseFilter(r,p[a],f,u)||""),o=("("+(t.filter.parseFilter(r,d.filter,f,u)||"")+")").replace(/\?/g,"\\S{1}").replace(/\*/g,"\\S*"),c=new RegExp(i.isMatch?o:"^"+o+"$",r.widgetOptions.filter_ignoreCase?"i":""),n=c.test(d.exact)||t.filter.processTypes(r,d,l),s=0===a?n:s&&n;return s||!1}return null},regex:function(e,r){if(t.filter.regex.regex.test(r.filter)){var i,l=r.filter_regexCache[r.index]||t.filter.regex.regex.exec(r.filter),a=l instanceof RegExp;try{a||(r.filter_regexCache[r.index]=l=new RegExp(l[1],l[2])),i=l.test(r.exact)}catch(s){i=!1}return i}return null},operators:function(r,i){if(/^[<>]=?/.test(i.iFilter)&&""!==i.iExact){var l,a,s,n=r.table,o=i.index,c=i.parsed[o],d=t.formatFloat(i.iFilter.replace(t.filter.regex.operators,""),n),f=r.parsers[o],u=d;return(c||"numeric"===f.type)&&(s=e.trim(""+i.iFilter.replace(t.filter.regex.operators,"")),a=t.filter.parseFilter(r,s,o,!0),d="number"!=typeof a||""===a||isNaN(a)?d:a),!c&&"numeric"!==f.type||isNaN(d)||"undefined"==typeof i.cache?(s=isNaN(i.iExact)?i.iExact.replace(t.filter.regex.nondigit,""):i.iExact,l=t.formatFloat(s,n)):l=i.cache,/>/.test(i.iFilter)?a=/>=/.test(i.iFilter)?l>=d:l>d:/</.test(i.iFilter)&&(a=/<=/.test(i.iFilter)?d>=l:d>l),a||""!==u||(a=!0),a}return null},notMatch:function(r,i){if(/^\!/.test(i.iFilter)){var l,a=i.iFilter.replace("!",""),s=t.filter.parseFilter(r,a,i.index,i.parsed[i.index])||"";return t.filter.regex.exact.test(s)?(s=s.replace(t.filter.regex.exact,""),""===s?!0:e.trim(s)!==i.iExact):(l=i.iExact.search(e.trim(s)),""===s?!0:!(r.widgetOptions.filter_startsWith?0===l:l>=0))}return null},exact:function(r,i){if(t.filter.regex.exact.test(i.iFilter)){var l=i.iFilter.replace(t.filter.regex.exact,""),a=t.filter.parseFilter(r,l,i.index,i.parsed[i.index])||"";return i.anyMatch?e.inArray(a,i.rowArray)>=0:a==i.iExact}return null},range:function(e,r){if(t.filter.regex.toTest.test(r.iFilter)){var i,l,a,s,n=e.table,o=r.index,c=r.parsed[o],d=r.iFilter.split(t.filter.regex.toSplit);return l=d[0].replace(t.filter.regex.nondigit,"")||"",a=t.formatFloat(t.filter.parseFilter(e,l,o,c),n),l=d[1].replace(t.filter.regex.nondigit,"")||"",s=t.formatFloat(t.filter.parseFilter(e,l,o,c),n),(c||"numeric"===e.parsers[o].type)&&(i=e.parsers[o].format(""+d[0],n,e.$headers.eq(o),o),a=""===i||isNaN(i)?a:i,i=e.parsers[o].format(""+d[1],n,e.$headers.eq(o),o),s=""===i||isNaN(i)?s:i),!c&&"numeric"!==e.parsers[o].type||isNaN(a)||isNaN(s)?(l=isNaN(r.iExact)?r.iExact.replace(t.filter.regex.nondigit,""):r.iExact,i=t.formatFloat(l,n)):i=r.cache,a>s&&(l=a,a=s,s=l),i>=a&&s>=i||""===a||""===s}return null},wild:function(e,r){if(/[\?\*\|]/.test(r.iFilter)){var i=r.index,l=r.parsed[i],a=""+(t.filter.parseFilter(e,r.iFilter,i,l)||"");return!/\?\*/.test(a)&&r.nestedFilters&&(a=r.isMatch?a:"^("+a+")$"),new RegExp(a.replace(/\?/g,"\\S{1}").replace(/\*/g,"\\S*"),e.widgetOptions.filter_ignoreCase?"i":"").test(r.exact)}return null},fuzzy:function(e,r){if(/^~/.test(r.iFilter)){var i,l=0,a=r.iExact.length,s=r.iFilter.slice(1),n=t.filter.parseFilter(e,s,r.index,r.parsed[r.index])||"";for(i=0;a>i;i++)r.iExact[i]===n[l]&&(l+=1);return l===n.length?!0:!1}return null}},init:function(i,l,a){t.language=e.extend(!0,{},{to:"to",or:"or",and:"and"},t.language);var s,n,o,c,d,f,u,h,p,g=t.filter.regex;if(l.$table.addClass("hasFilters"),a.searchTimer=null,a.filter_initTimer=null,a.filter_formatterCount=0,a.filter_formatterInit=[],a.filter_anyColumnSelector='[data-column="all"],[data-column="any"]',a.filter_multipleColumnSelector='[data-column*="-"],[data-column*=","]',u="\\{"+t.filter.regex.query+"\\}",e.extend(g,{child:new RegExp(l.cssChildRow),filtered:new RegExp(a.filter_filteredRow),alreadyFiltered:new RegExp("(\\s+("+t.language.or+"|-|"+t.language.to+")\\s+)","i"),toTest:new RegExp("\\s+(-|"+t.language.to+")\\s+","i"),toSplit:new RegExp("(?:\\s+(?:-|"+t.language.to+")\\s+)","gi"),andTest:new RegExp("\\s+("+t.language.and+"|&&)\\s+","i"),andSplit:new RegExp("(?:\\s+(?:"+t.language.and+"|&&)\\s+)","gi"),orSplit:new RegExp("(?:\\s+(?:"+t.language.or+")\\s+|\\|)","gi"),iQuery:new RegExp(u,"i"),igQuery:new RegExp(u,"ig")}),u=l.$headers.filter(".filter-false, .parser-false").length,a.filter_columnFilters!==!1&&u!==l.$headers.length&&t.filter.buildRow(i,l,a),o="addRows updateCell update updateRows updateComplete appendCache filterReset filterEnd search ".split(" ").join(l.namespace+"filter "),l.$table.bind(o,function(s,n){return u=a.filter_hideEmpty&&e.isEmptyObject(l.cache)&&!(l.delayInit&&"appendCache"===s.type),l.$table.find("."+r.filterRow).toggleClass(a.filter_filteredRow,u),/(search|filter)/.test(s.type)||(s.stopPropagation(),t.filter.buildDefault(i,!0)),"filterReset"===s.type?(l.$table.find("."+r.filter).add(a.filter_$externalFilters).val(""),t.filter.searching(i,[])):"filterEnd"===s.type?t.filter.buildDefault(i,!0):(n="search"===s.type?n:"updateComplete"===s.type?l.$table.data("lastSearch"):"",/(update|add)/.test(s.type)&&"updateComplete"!==s.type&&(l.lastCombinedFilter=null,l.lastSearch=[]),t.filter.searching(i,n,!0)),!1}),a.filter_reset&&(a.filter_reset instanceof e?a.filter_reset.click(function(){l.$table.trigger("filterReset")}):e(a.filter_reset).length&&e(document).undelegate(a.filter_reset,"click.tsfilter").delegate(a.filter_reset,"click.tsfilter",function(){l.$table.trigger("filterReset")})),a.filter_functions)for(d=0;d<l.columns;d++)if(h=t.getColumnData(i,a.filter_functions,d))if(c=l.$headerIndexed[d].removeClass("filter-select"),p=!(c.hasClass("filter-false")||c.hasClass("parser-false")),s="",h===!0&&p)t.filter.buildSelect(i,d);else if("object"==typeof h&&p){for(n in h)"string"==typeof n&&(s+=""===s?'<option value="">'+(c.data("placeholder")||c.attr("data-placeholder")||a.filter_placeholder.select||"")+"</option>":"",u=n,o=n,n.indexOf(a.filter_selectSourceSeparator)>=0&&(u=n.split(a.filter_selectSourceSeparator),o=u[1],u=u[0]),s+="<option "+(o===u?"":'data-function-name="'+n+'" ')+'value="'+u+'">'+o+"</option>");l.$table.find("thead").find("select."+r.filter+'[data-column="'+d+'"]').append(s),o=a.filter_selectSource,h=e.isFunction(o)?!0:t.getColumnData(i,o,d),h&&t.filter.buildSelect(l.table,d,"",!0,c.hasClass(a.filter_onlyAvail))}t.filter.buildDefault(i,!0),t.filter.bindSearch(i,l.$table.find("."+r.filter),!0),a.filter_external&&t.filter.bindSearch(i,a.filter_external),a.filter_hideFilters&&t.filter.hideFilters(i,l),l.showProcessing&&(o="filterStart filterEnd ".split(" ").join(l.namespace+"filter "),l.$table.unbind(o.replace(/\s+/g," ")).bind(o,function(a,s){c=s?l.$table.find("."+r.header).filter("[data-column]").filter(function(){return""!==s[e(this).data("column")]}):"",t.isProcessing(i,"filterStart"===a.type,s?c:"")})),l.filteredRows=l.totalRows,o="tablesorter-initialized pagerBeforeInitialized ".split(" ").join(l.namespace+"filter "),l.$table.unbind(o.replace(/\s+/g," ")).bind(o,function(){var e=this.config.widgetOptions;f=t.filter.setDefaults(i,l,e)||[],f.length&&(l.delayInit&&""===f.join("")||t.setFilters(i,f,!0)),l.$table.trigger("filterFomatterUpdate"),setTimeout(function(){e.filter_initialized||t.filter.filterInitComplete(l)},100)}),l.pager&&l.pager.initialized&&!a.filter_initialized&&(l.$table.trigger("filterFomatterUpdate"),setTimeout(function(){t.filter.filterInitComplete(l)},100))},formatterUpdated:function(e,t){var r=e.closest("table")[0].config.widgetOptions;r.filter_initialized||(r.filter_formatterInit[t]=1)},filterInitComplete:function(r){var i,l,a=r.widgetOptions,s=0,n=function(){a.filter_initialized=!0,r.$table.trigger("filterInit",r),t.filter.findRows(r.table,r.$table.data("lastSearch")||[])};if(e.isEmptyObject(a.filter_formatter))n();else{for(l=a.filter_formatterInit.length,i=0;l>i;i++)1===a.filter_formatterInit[i]&&s++;clearTimeout(a.filter_initTimer),a.filter_initialized||s!==a.filter_formatterCount?a.filter_initialized||(a.filter_initTimer=setTimeout(function(){n()},500)):n()}},setDefaults:function(r,i,l){var a,s,n,o,c,d=t.getFilters(r)||[];if(l.filter_saveFilters&&t.storage&&(s=t.storage(r,"tablesorter-filters")||[],a=e.isArray(s),a&&""===s.join("")||!a||(d=s)),""===d.join(""))for(c=i.$headers.add(l.filter_$externalFilters).filter("["+l.filter_defaultAttrib+"]"),n=0;n<=i.columns;n++)o=n===i.columns?"all":n,d[n]=c.filter('[data-column="'+o+'"]').attr(l.filter_defaultAttrib)||d[n]||"";return i.$table.data("lastSearch",d),d},parseFilter:function(e,t,r,i){return i?e.parsers[r].format(t,e.table,[],r):t},buildRow:function(i,l,a){var s,n,o,c,d,f,u,h,p=a.filter_cellFilter,g=l.columns,m=e.isArray(p),b='<tr role="row" class="'+r.filterRow+" "+l.cssIgnoreRow+'">';for(n=0;g>n;n++)b+="<td",b+=m?p[n]?' class="'+p[n]+'"':"":""!==p?' class="'+p+'"':"",b+="></td>";for(l.$filters=e(b+="</tr>").appendTo(l.$table.children("thead").eq(0)).find("td"),n=0;g>n;n++)d=!1,o=l.$headerIndexed[n],u=t.getColumnData(i,a.filter_functions,n),c=a.filter_functions&&u&&"function"!=typeof u||o.hasClass("filter-select"),s=t.getColumnData(i,l.headers,n),d="false"===t.getData(o[0],s,"filter")||"false"===t.getData(o[0],s,"parser"),c?b=e("<select>").appendTo(l.$filters.eq(n)):(u=t.getColumnData(i,a.filter_formatter,n),u?(a.filter_formatterCount++,b=u(l.$filters.eq(n),n),b&&0===b.length&&(b=l.$filters.eq(n).children("input")),b&&(0===b.parent().length||b.parent().length&&b.parent()[0]!==l.$filters[n])&&l.$filters.eq(n).append(b)):b=e('<input type="search">').appendTo(l.$filters.eq(n)),b&&(h=o.data("placeholder")||o.attr("data-placeholder")||a.filter_placeholder.search||"",b.attr("placeholder",h))),b&&(f=(e.isArray(a.filter_cssFilter)?"undefined"!=typeof a.filter_cssFilter[n]?a.filter_cssFilter[n]||"":"":a.filter_cssFilter)||"",b.addClass(r.filter+" "+f).attr("data-column",n),d&&(b.attr("placeholder","").addClass(r.filterDisabled)[0].disabled=!0))},bindSearch:function(r,i,l){if(r=e(r)[0],i=e(i),i.length){var a,s=r.config,n=s.widgetOptions,o=s.namespace+"filter",c=n.filter_$externalFilters;l!==!0&&(a=n.filter_anyColumnSelector+","+n.filter_multipleColumnSelector,n.filter_$anyMatch=i.filter(a),n.filter_$externalFilters=c&&c.length?n.filter_$externalFilters.add(i):i,t.setFilters(r,s.$table.data("lastSearch")||[],l===!1)),a="keypress keyup search change ".split(" ").join(o+" "),i.attr("data-lastSearchTime",(new Date).getTime()).unbind(a.replace(/\s+/g," ")).bind("keyup"+o,function(i){if(e(this).attr("data-lastSearchTime",(new Date).getTime()),27===i.which)this.value="";else{if(n.filter_liveSearch===!1)return;if(""!==this.value&&("number"==typeof n.filter_liveSearch&&this.value.length<n.filter_liveSearch||13!==i.which&&8!==i.which&&(i.which<32||i.which>=37&&i.which<=40)))return}t.filter.searching(r,!0,!0)}).bind("search change keypress ".split(" ").join(o+" "),function(i){var l=e(this).data("column");(13===i.which||"search"===i.type||"change"===i.type&&this.value!==s.lastSearch[l])&&(i.preventDefault(),e(this).attr("data-lastSearchTime",(new Date).getTime()),t.filter.searching(r,!1,!0))})}},searching:function(e,r,i){var l=e.config.widgetOptions;clearTimeout(l.searchTimer),"undefined"==typeof r||r===!0?l.searchTimer=setTimeout(function(){t.filter.checkFilters(e,r,i)},l.filter_liveSearch?l.filter_searchDelay:10):t.filter.checkFilters(e,r,i)},checkFilters:function(i,l,a){var s=i.config,n=s.widgetOptions,o=e.isArray(l),c=o?l:t.getFilters(i,!0),d=(c||[]).join("");return e.isEmptyObject(s.cache)?void(s.delayInit&&s.pager&&s.pager.initialized&&s.$table.trigger("updateCache",[function(){t.filter.checkFilters(i,!1,a)}])):(o&&(t.setFilters(i,c,!1,a!==!0),n.filter_initialized||(s.lastCombinedFilter="")),n.filter_hideFilters&&s.$table.find("."+r.filterRow).trigger(""===d?"mouseleave":"mouseenter"),s.lastCombinedFilter!==d||l===!1?(l===!1&&(s.lastCombinedFilter=null,s.lastSearch=[]),n.filter_initialized&&s.$table.trigger("filterStart",[c]),s.showProcessing?void setTimeout(function(){return t.filter.findRows(i,c,d),!1},30):(t.filter.findRows(i,c,d),!1)):void 0)},hideFilters:function(i,l){var a;l.$table.find("."+r.filterRow).bind("mouseenter mouseleave",function(t){var i=t,s=e(this);clearTimeout(a),a=setTimeout(function(){/enter|over/.test(i.type)?s.removeClass(r.filterRowHide):e(document.activeElement).closest("tr")[0]!==s[0]&&""===l.lastCombinedFilter&&s.addClass(r.filterRowHide)},200)}).find("input, select").bind("focus blur",function(i){var s=i,n=e(this).closest("tr");clearTimeout(a),a=setTimeout(function(){clearTimeout(a),""===t.getFilters(l.$table).join("")&&n.toggleClass(r.filterRowHide,"focus"!==s.type)},200)})},defaultFilter:function(r,i){if(""===r)return r;var l=t.filter.regex.iQuery,a=i.match(t.filter.regex.igQuery).length,s=a>1?e.trim(r).split(/\s/):[e.trim(r)],n=s.length-1,o=0,c=i;for(1>n&&a>1&&(s[1]=s[0]);l.test(c);)c=c.replace(l,s[o++]||""),l.test(c)&&n>o&&""!==(s[o]||"")&&(c=i.replace(l,c));return c},getLatestSearch:function(t){return t?t.sort(function(t,r){return e(r).attr("data-lastSearchTime")-e(t).attr("data-lastSearchTime")}):t||e()},multipleColumns:function(r,i){var l,a,s,n,o,c,d,f,u,h=r.widgetOptions,p=h.filter_initialized||!i.filter(h.filter_anyColumnSelector).length,g=[],m=e.trim(t.filter.getLatestSearch(i).attr("data-column")||"");if(p&&/-/.test(m))for(a=m.match(/(\d+)\s*-\s*(\d+)/g),u=a.length,f=0;u>f;f++){for(s=a[f].split(/\s*-\s*/),n=parseInt(s[0],10)||0,o=parseInt(s[1],10)||r.columns-1,n>o&&(l=n,n=o,o=l),o>=r.columns&&(o=r.columns-1);o>=n;n++)g.push(n);m=m.replace(a[f],"")}if(p&&/,/.test(m))for(c=m.split(/\s*,\s*/),u=c.length,d=0;u>d;d++)""!==c[d]&&(f=parseInt(c[d],10),f<r.columns&&g.push(f));if(!g.length)for(f=0;f<r.columns;f++)g.push(f);return g},processTypes:function(r,i,l){var a,s=null,n=null;for(a in t.filter.types)e.inArray(a,l.excludeMatch)<0&&null===n&&(n=t.filter.types[a](r,i,l),null!==n&&(s=n));return s},processRow:function(r,i,l){var a,s,n,o,c,d,f,u,h=t.filter.regex,p=r.widgetOptions,g=!0;if(i.$cells=i.$row.children(),i.anyMatchFlag){if(a=t.filter.multipleColumns(r,p.filter_$anyMatch),i.anyMatch=!0,i.isMatch=!0,i.rowArray=i.$cells.map(function(l){return e.inArray(l,a)>-1?(i.parsed[l]?u=i.cacheArray[l]:(u=i.rawArray[l],u=e.trim(p.filter_ignoreCase?u.toLowerCase():u),r.sortLocaleCompare&&(u=t.replaceAccents(u))),u):void 0}).get(),i.filter=i.anyMatchFilter,i.iFilter=i.iAnyMatchFilter,i.exact=i.rowArray.join(" "),i.iExact=p.filter_ignoreCase?i.exact.toLowerCase():i.exact,i.cache=i.cacheArray.slice(0,-1).join(" "),l.excludeMatch=l.noAnyMatch,c=t.filter.processTypes(r,i,l),null!==c)g=c;else if(p.filter_startsWith)for(g=!1,a=r.columns;!g&&a>0;)a--,g=g||0===i.rowArray[a].indexOf(i.iFilter);else g=(i.iExact+i.childRowText).indexOf(i.iFilter)>=0;if(i.anyMatch=!1,i.filters.join("")===i.filter)return g}for(a=0;a<r.columns;a++)i.filter=i.filters[a],i.index=a,l.excludeMatch=l.excludeFilter[a],i.filter&&(i.cache=i.cacheArray[a],p.filter_useParsedData||i.parsed[a]?i.exact=i.cache:(n=i.rawArray[a]||"",i.exact=r.sortLocaleCompare?t.replaceAccents(n):n),i.iExact=!h.type.test(typeof i.exact)&&p.filter_ignoreCase?i.exact.toLowerCase():i.exact,i.isMatch=r.$headerIndexed[i.index].hasClass("filter-match"),n=g,f=p.filter_columnFilters?r.$filters.add(r.$externalFilters).filter('[data-column="'+a+'"]').find("select option:selected").attr("data-function-name")||"":"",r.sortLocaleCompare&&(i.filter=t.replaceAccents(i.filter)),o=!0,p.filter_defaultFilter&&h.iQuery.test(l.defaultColFilter[a])&&(i.filter=t.filter.defaultFilter(i.filter,l.defaultColFilter[a]),o=!1),i.iFilter=p.filter_ignoreCase?(i.filter||"").toLowerCase():i.filter,d=l.functions[a],s=r.$headerIndexed[a].hasClass("filter-select"),c=null,(d||s&&o)&&(d===!0||s?c=i.isMatch?i.iExact.search(i.iFilter)>=0:i.filter===i.exact:"function"==typeof d?c=d(i.exact,i.cache,i.filter,a,i.$row,r,i):"function"==typeof d[f||i.filter]&&(u=f||i.filter,c=d[u](i.exact,i.cache,i.filter,a,i.$row,r,i))),null===c?(c=t.filter.processTypes(r,i,l),null!==c?n=c:(u=(i.iExact+i.childRowText).indexOf(t.filter.parseFilter(r,i.iFilter,a,i.parsed[a])),n=!p.filter_startsWith&&u>=0||p.filter_startsWith&&0===u)):n=c,g=n?g:!1);return g},findRows:function(r,i,l){if(r.config.lastCombinedFilter!==l&&r.config.widgetOptions.filter_initialized){var a,s,n,o,c,d,f,u,h,p,g,m,b,y,_,v,w,x,C,z,S,$,F=e.extend([],i),R=t.filter.regex,k=r.config,T=k.widgetOptions,A={anyMatch:!1,filters:i,filter_regexCache:[]},H={noAnyMatch:["range","notMatch","operators"],functions:[],excludeFilter:[],defaultColFilter:[],defaultAnyFilter:t.getColumnData(r,T.filter_defaultFilter,k.columns,!0)||""};for(A.parsed=k.$headers.map(function(i){return k.parsers&&k.parsers[i]&&k.parsers[i].parsed||t.getData&&"parsed"===t.getData(k.$headerIndexed[i],t.getColumnData(r,k.headers,i),"filter")||e(this).hasClass("filter-parsed")}).get(),u=0;u<k.columns;u++)H.functions[u]=t.getColumnData(r,T.filter_functions,u),H.defaultColFilter[u]=t.getColumnData(r,T.filter_defaultFilter,u)||"",H.excludeFilter[u]=(t.getColumnData(r,T.filter_excludeFilter,u,!0)||"").split(/\s+/);for(k.debug&&(t.log("Filter: Starting filter widget search",i),b=new Date),k.filteredRows=0,k.totalRows=0,l=(F||[]).join(""),d=0;d<k.$tbodies.length;d++){if(f=t.processTbody(r,k.$tbodies.eq(d),!0),u=k.columns,s=k.cache[d].normalized,o=e(e.map(s,function(e){return e[u].$row.get()})),""===l||T.filter_serversideFiltering)o.removeClass(T.filter_filteredRow).not("."+k.cssChildRow).css("display","");else{if(o=o.not("."+k.cssChildRow),a=o.length,(T.filter_$anyMatch&&T.filter_$anyMatch.length||"undefined"!=typeof i[k.columns])&&(A.anyMatchFlag=!0,A.anyMatchFilter=""+(i[k.columns]||T.filter_$anyMatch&&t.filter.getLatestSearch(T.filter_$anyMatch).val()||""),T.filter_columnAnyMatch)){for(x=A.anyMatchFilter.split(R.andSplit),C=!1,_=0;_<x.length;_++)z=x[_].split(":"),z.length>1&&(S=parseInt(z[0],10)-1,S>=0&&S<k.columns&&(i[S]=z[1],x.splice(_,1),_--,C=!0));C&&(A.anyMatchFilter=x.join(" && "))}if(w=T.filter_searchFiltered,g=k.lastSearch||k.$table.data("lastSearch")||[],w)for(_=0;u+1>_;_++)y=i[_]||"",w||(_=u),w=!(!w||!g.length||0!==y.indexOf(g[_]||"")||R.alreadyFiltered.test(y)||/[=\"\|!]/.test(y)||/(>=?\s*-\d)/.test(y)||/(<=?\s*\d)/.test(y)||""!==y&&k.$filters&&k.$filters.eq(_).find("select").length&&!k.$headerIndexed[_].hasClass("filter-match"));for(v=o.not("."+T.filter_filteredRow).length,w&&0===v&&(w=!1),k.debug&&t.log("Filter: Searching through "+(w&&a>v?v:"all")+" rows"),A.anyMatchFlag&&(k.sortLocaleCompare&&(A.anyMatchFilter=t.replaceAccents(A.anyMatchFilter)),T.filter_defaultFilter&&R.iQuery.test(H.defaultAnyFilter)&&(A.anyMatchFilter=t.filter.defaultFilter(A.anyMatchFilter,H.defaultAnyFilter),w=!1),A.iAnyMatchFilter=T.filter_ignoreCase&&k.ignoreCase?A.anyMatchFilter.toLowerCase():A.anyMatchFilter),c=0;a>c;c++)if($=o[c].className,h=c&&R.child.test($),!(h||w&&R.filtered.test($))){if(A.$row=o.eq(c),A.cacheArray=s[c],n=A.cacheArray[k.columns],A.rawArray=n.raw,A.childRowText="",!T.filter_childByColumn){for($="",p=n.child,_=0;_<p.length;_++)$+=" "+p[_].join("")||"";A.childRowText=T.filter_childRows?T.filter_ignoreCase?$.toLowerCase():$:""}if(m=t.filter.processRow(k,A,H),p=n.$row.filter(":gt( 0 )"),T.filter_childRows&&p.length){if(T.filter_childByColumn)for(_=0;_<p.length;_++)A.$row=p.eq(_),A.cacheArray=n.child[_],A.rawArray=A.cacheArray,m=m||t.filter.processRow(k,A,H);p.toggleClass(T.filter_filteredRow,!m)}n.$row.toggleClass(T.filter_filteredRow,!m)[0].display=m?"":"none"}}k.filteredRows+=o.not("."+T.filter_filteredRow).length,k.totalRows+=o.length,t.processTbody(r,f,!1)}k.lastCombinedFilter=l,k.lastSearch=F,k.$table.data("lastSearch",F),T.filter_saveFilters&&t.storage&&t.storage(r,"tablesorter-filters",F),k.debug&&t.benchmark("Completed filter widget search",b),T.filter_initialized&&k.$table.trigger("filterEnd",k),setTimeout(function(){k.$table.trigger("applyWidgets")},0)}},getOptionSource:function(r,i,l){r=e(r)[0];var a,s,n,o,c=r.config,d=c.widgetOptions,f=[],u=!1,h=d.filter_selectSource,p=c.$table.data("lastSearch")||[],g=e.isFunction(h)?!0:t.getColumnData(r,h,i);if(l&&""!==p[i]&&(l=!1),g===!0)u=h(r,i,l);else{if(g instanceof e||"string"===e.type(g)&&g.indexOf("</option>")>=0)return g;e.isArray(g)?u=g:"object"===e.type(h)&&g&&(u=g(r,i,l))}if(u===!1&&(u=t.filter.getOptions(r,i,l)),u=e.grep(u,function(t,r){return e.inArray(t,u)===r}),c.$headerIndexed[i].hasClass("filter-select-nosort"))return u;for(o=u.length,n=0;o>n;n++)s=u[n],f.push({t:s,p:c.parsers&&c.parsers.length&&c.parsers[i].format(s,r,[],i)||s});for(a=c.textSorter||"",f.sort(function(l,s){var n=l.p.toString(),o=s.p.toString();return e.isFunction(a)?a(n,o,!0,i,r):"object"==typeof a&&a.hasOwnProperty(i)?a[i](n,o,!0,i,r):t.sortNatural?t.sortNatural(n,o):!0}),u=[],o=f.length,n=0;o>n;n++)u.push(f[n].t);return u},getOptions:function(t,r,i){t=e(t)[0];var l,a,s,n,o,c=t.config,d=c.widgetOptions,f=[];for(a=0;a<c.$tbodies.length;a++)for(o=c.cache[a],s=c.cache[a].normalized.length,l=0;s>l;l++)n=o.row?o.row[l]:o.normalized[l][c.columns].$row[0],i&&n.className.match(d.filter_filteredRow)||f.push(d.filter_useParsedData||c.parsers[r].parsed||c.$headerIndexed[r].hasClass("filter-parsed")?""+o.normalized[l][r]:o.normalized[l][c.columns].raw[r]);return f},buildSelect:function(i,l,a,s,n){if(i=e(i)[0],l=parseInt(l,10),i.config.cache&&!e.isEmptyObject(i.config.cache)){var o,c,d,f,u,h,p=i.config,g=p.widgetOptions,m=p.$headerIndexed[l],b='<option value="">'+(m.data("placeholder")||m.attr("data-placeholder")||g.filter_placeholder.select||"")+"</option>",y=p.$table.find("thead").find("select."+r.filter+'[data-column="'+l+'"]').val();if(("undefined"==typeof a||""===a)&&(a=t.filter.getOptionSource(i,l,n)),e.isArray(a)){for(o=0;o<a.length;o++)d=a[o]=(""+a[o]).replace(/\"/g,"""),c=d,d.indexOf(g.filter_selectSourceSeparator)>=0&&(f=d.split(g.filter_selectSourceSeparator),c=f[0],d=f[1]),b+=""!==a[o]?"<option "+(c===d?"":'data-function-name="'+a[o]+'" ')+'value="'+c+'">'+d+"</option>":"";a=[]}u=(p.$filters?p.$filters:p.$table.children("thead")).find("."+r.filter),g.filter_$externalFilters&&(u=u&&u.length?u.add(g.filter_$externalFilters):g.filter_$externalFilters),h=u.filter('select[data-column="'+l+'"]'),h.length&&(h[s?"html":"append"](b),e.isArray(a)||h.append(a).val(y),h.val(y))}},buildDefault:function(e,r){var i,l,a,s=e.config,n=s.widgetOptions,o=s.columns;for(i=0;o>i;i++)l=s.$headerIndexed[i],a=!(l.hasClass("filter-false")||l.hasClass("parser-false")),(l.hasClass("filter-select")||t.getColumnData(e,n.filter_functions,i)===!0)&&a&&t.filter.buildSelect(e,i,"",r,l.hasClass(n.filter_onlyAvail))}},t.getFilters=function(i,l,a,s){var n,o,c,d,f=!1,u=i?e(i)[0].config:"",h=u?u.widgetOptions:"";if(l!==!0&&h&&!h.filter_columnFilters||e.isArray(a)&&a.join("")===u.lastCombinedFilter)return e(i).data("lastSearch");if(u&&(u.$filters&&(o=u.$filters.find("."+r.filter)),h.filter_$externalFilters&&(o=o&&o.length?o.add(h.filter_$externalFilters):h.filter_$externalFilters),o&&o.length))for(f=a||[],n=0;n<u.columns+1;n++)d=n===u.columns?h.filter_anyColumnSelector+","+h.filter_multipleColumnSelector:'[data-column="'+n+'"]',c=o.filter(d),c.length&&(c=t.filter.getLatestSearch(c),e.isArray(a)?(s&&c.length>1&&(c=c.slice(1)),n===u.columns&&(d=c.filter(h.filter_anyColumnSelector),c=d.length?d:c),c.val(a[n]).trigger("change.tsfilter")):(f[n]=c.val()||"",n===u.columns?c.slice(1).filter('[data-column*="'+c.attr("data-column")+'"]').val(f[n]):c.slice(1).val(f[n])),n===u.columns&&c.length&&(h.filter_$anyMatch=c)); + return 0===f.length&&(f=!1),f},t.setFilters=function(r,i,l,a){var s=r?e(r)[0].config:"",n=t.getFilters(r,!0,i,a);return s&&l&&(s.lastCombinedFilter=null,s.lastSearch=[],t.filter.searching(s.table,i,a),s.$table.trigger("filterFomatterUpdate")),!!n}}(jQuery),function(e,t){"use strict";var r=e.tablesorter||{};e.extend(r.css,{sticky:"tablesorter-stickyHeader",stickyVis:"tablesorter-sticky-visible",stickyHide:"tablesorter-sticky-hidden",stickyWrap:"tablesorter-sticky-wrapper"}),r.addHeaderResizeEvent=function(t,r,i){if(t=e(t)[0],t.config){var l={timer:250},a=e.extend({},l,i),s=t.config,n=s.widgetOptions,o=function(e){var t,r,i,l,a,o,c=s.$headers.length;for(n.resize_flag=!0,r=[],t=0;c>t;t++)i=s.$headers.eq(t),l=i.data("savedSizes")||[0,0],a=i[0].offsetWidth,o=i[0].offsetHeight,(a!==l[0]||o!==l[1])&&(i.data("savedSizes",[a,o]),r.push(i[0]));r.length&&e!==!1&&s.$table.trigger("resize",[r]),n.resize_flag=!1};return o(!1),clearInterval(n.resize_timer),r?(n.resize_flag=!1,!1):void(n.resize_timer=setInterval(function(){n.resize_flag||o()},a.timer))}},r.addWidget({id:"stickyHeaders",priority:60,options:{stickyHeaders:"",stickyHeaders_attachTo:null,stickyHeaders_xScroll:null,stickyHeaders_yScroll:null,stickyHeaders_offset:0,stickyHeaders_filteredToTop:!0,stickyHeaders_cloneId:"-sticky",stickyHeaders_addResizeEvent:!0,stickyHeaders_includeCaption:!0,stickyHeaders_zIndex:2},format:function(i,l,a){if(!(l.$table.hasClass("hasStickyHeaders")||e.inArray("filter",l.widgets)>=0&&!l.$table.hasClass("hasFilters"))){var s,n,o,c,d=l.$table,f=e(a.stickyHeaders_attachTo),u=l.namespace+"stickyheaders ",h=e(a.stickyHeaders_yScroll||a.stickyHeaders_attachTo||t),p=e(a.stickyHeaders_xScroll||a.stickyHeaders_attachTo||t),g=d.children("thead:first"),m=g.children("tr").not(".sticky-false").children(),b=d.children("tfoot"),y=isNaN(a.stickyHeaders_offset)?e(a.stickyHeaders_offset):"",_=y.length?y.height()||0:parseInt(a.stickyHeaders_offset,10)||0,v=d.parent().closest("."+r.css.table).hasClass("hasStickyHeaders")?d.parent().closest("table.tablesorter")[0].config.widgetOptions.$sticky.parent():[],w=v.length?v.height():0,x=a.$sticky=d.clone().addClass("containsStickyHeaders "+r.css.sticky+" "+a.stickyHeaders+" "+l.namespace.slice(1)+"_extra_table").wrap('<div class="'+r.css.stickyWrap+'">'),C=x.parent().addClass(r.css.stickyHide).css({position:f.length?"absolute":"fixed",padding:parseInt(x.parent().parent().css("padding-left"),10),top:_+w,left:0,visibility:"hidden",zIndex:a.stickyHeaders_zIndex||2}),z=x.children("thead:first"),S="",$=0,F=function(e,r){var i,l,a,s,n,o=e.filter(":visible"),c=o.length;for(i=0;c>i;i++)s=r.filter(":visible").eq(i),n=o.eq(i),"border-box"===n.css("box-sizing")?l=n.outerWidth():"collapse"===s.css("border-collapse")?t.getComputedStyle?l=parseFloat(t.getComputedStyle(n[0],null).width):(a=parseFloat(n.css("border-width")),l=n.outerWidth()-parseFloat(n.css("padding-left"))-parseFloat(n.css("padding-right"))-a):l=n.width(),s.css({width:l,"min-width":l,"max-width":l})},R=function(){_=y.length?y.height()||0:parseInt(a.stickyHeaders_offset,10)||0,$=0,C.css({left:f.length?parseInt(f.css("padding-left"),10)||0:d.offset().left-parseInt(d.css("margin-left"),10)-p.scrollLeft()-$,width:d.outerWidth()}),F(d,x),F(m,c)},k=function(t){if(d.is(":visible")){w=v.length?v.offset().top-h.scrollTop()+v.height():0;var i=d.offset(),l=e.isWindow(h[0]),a=e.isWindow(p[0]),s=(f.length?l?h.scrollTop():h.offset().top:h.scrollTop())+_+w,n=d.height()-(C.height()+(b.height()||0)),o=s>i.top&&s<i.top+n?"visible":"hidden",c={visibility:o};f.length&&(c.top=l?s-f.offset().top:f.scrollTop()),a&&(c.left=d.offset().left-parseInt(d.css("margin-left"),10)-p.scrollLeft()-$),v.length&&(c.top=(c.top||0)+_+w),C.removeClass(r.css.stickyVis+" "+r.css.stickyHide).addClass("visible"===o?r.css.stickyVis:r.css.stickyHide).css(c),(o!==S||t)&&(R(),S=o)}};if(f.length&&!f.css("position")&&f.css("position","relative"),x.attr("id")&&(x[0].id+=a.stickyHeaders_cloneId),x.find("thead:gt(0), tr.sticky-false").hide(),x.find("tbody, tfoot").remove(),x.find("caption").toggle(a.stickyHeaders_includeCaption),c=z.children().children(),x.css({height:0,width:0,margin:0}),c.find("."+r.css.resizer).remove(),d.addClass("hasStickyHeaders").bind("pagerComplete"+u,function(){R()}),r.bindEvents(i,z.children().children("."+r.css.header)),d.after(C),l.onRenderHeader)for(o=z.children("tr").children(),n=o.length,s=0;n>s;s++)l.onRenderHeader.apply(o.eq(s),[s,l,x]);p.add(h).unbind("scroll resize ".split(" ").join(u).replace(/\s+/g," ")).bind("scroll resize ".split(" ").join(u),function(e){k("resize"===e.type)}),l.$table.unbind("stickyHeadersUpdate"+u).bind("stickyHeadersUpdate"+u,function(){k(!0)}),a.stickyHeaders_addResizeEvent&&r.addHeaderResizeEvent(i),d.hasClass("hasFilters")&&a.filter_columnFilters&&(d.bind("filterEnd"+u,function(){var i=e(document.activeElement).closest("td"),s=i.parent().children().index(i);C.hasClass(r.css.stickyVis)&&a.stickyHeaders_filteredToTop&&(t.scrollTo(0,d.position().top),s>=0&&l.$filters&&l.$filters.eq(s).find("a, select, input").filter(":visible").focus())}),r.filter.bindSearch(d,c.find("."+r.css.filter)),a.filter_hideFilters&&r.filter.hideFilters(x,l)),d.trigger("stickyHeadersInit")}},remove:function(i,l,a){var s=l.namespace+"stickyheaders ";l.$table.removeClass("hasStickyHeaders").unbind("pagerComplete filterEnd stickyHeadersUpdate ".split(" ").join(s).replace(/\s+/g," ")).next("."+r.css.stickyWrap).remove(),a.$sticky&&a.$sticky.length&&a.$sticky.remove(),e(t).add(a.stickyHeaders_xScroll).add(a.stickyHeaders_yScroll).add(a.stickyHeaders_attachTo).unbind("scroll resize ".split(" ").join(s).replace(/\s+/g," ")),r.addHeaderResizeEvent(i,!1)}})}(jQuery,window),function(e,t){"use strict";var r=e.tablesorter||{};e.extend(r.css,{resizableContainer:"tablesorter-resizable-container",resizableHandle:"tablesorter-resizable-handle",resizableNoSelect:"tablesorter-disableSelection",resizableStorage:"tablesorter-resizable"}),e(function(){var t="<style>body."+r.css.resizableNoSelect+" { -ms-user-select: none; -moz-user-select: -moz-none;-khtml-user-select: none; -webkit-user-select: none; user-select: none; }."+r.css.resizableContainer+" { position: relative; height: 1px; }."+r.css.resizableHandle+" { position: absolute; display: inline-block; width: 8px;top: 1px; cursor: ew-resize; z-index: 3; user-select: none; -moz-user-select: none; }</style>";e(t).appendTo("body")}),r.resizable={init:function(t,i){if(!t.$table.hasClass("hasResizable")){t.$table.addClass("hasResizable");var l,a,s,n,o,c=t.$table,d=c.parent(),f=parseInt(c.css("margin-top"),10),u=i.resizable_={useStorage:r.storage&&i.resizable!==!1,$wrap:d,mouseXPosition:0,$target:null,$next:null,overflow:"auto"===d.css("overflow")||"scroll"===d.css("overflow")||"auto"===d.css("overflow-x")||"scroll"===d.css("overflow-x"),storedSizes:[]};for(r.resizableReset(t.table,!0),u.tableWidth=c.width(),u.fullWidth=Math.abs(d.width()-u.tableWidth)<20,u.useStorage&&u.overflow&&(r.storage(t.table,"tablesorter-table-original-css-width",u.tableWidth),o=r.storage(t.table,"tablesorter-table-resized-width")||"auto",r.resizable.setWidth(c,o,!0)),i.resizable_.storedSizes=n=(u.useStorage?r.storage(t.table,r.css.resizableStorage):[])||[],r.resizable.setWidths(t,i,n),r.resizable.updateStoredSizes(t,i),i.$resizable_container=e('<div class="'+r.css.resizableContainer+'">').css({top:f}).insertBefore(c),s=0;s<t.columns;s++)a=t.$headerIndexed[s],o=r.getColumnData(t.table,t.headers,s),l="false"===r.getData(a,o,"resizable"),l||e('<div class="'+r.css.resizableHandle+'">').appendTo(i.$resizable_container).attr({"data-column":s,unselectable:"on"}).data("header",a).bind("selectstart",!1);c.one("tablesorter-initialized",function(){r.resizable.setHandlePosition(t,i),r.resizable.bindings(this.config,this.config.widgetOptions)})}},updateStoredSizes:function(e,t){var r,i,l=e.columns,a=t.resizable_;for(a.storedSizes=[],r=0;l>r;r++)i=e.$headerIndexed[r],a.storedSizes[r]=i.is(":visible")?i.width():0},setWidth:function(e,t,r){e.css({width:t,"min-width":r?t:"","max-width":r?t:""})},setWidths:function(t,i,l){var a,s,n=i.resizable_,o=e(t.namespace+"_extra_headers"),c=t.$table.children("colgroup").children("col");if(l=l||n.storedSizes||[],l.length){for(a=0;a<t.columns;a++)r.resizable.setWidth(t.$headerIndexed[a],l[a],n.overflow),o.length&&(s=o.eq(a).add(c.eq(a)),r.resizable.setWidth(s,l[a],n.overflow));s=e(t.namespace+"_extra_table"),s.length&&!r.hasWidget(t.table,"scroller")&&r.resizable.setWidth(s,t.$table.outerWidth(),n.overflow)}},setHandlePosition:function(t,i){var l,a=r.hasWidget(t.table,"scroller"),s=t.$table.height(),n=i.$resizable_container.children(),o=Math.floor(n.width()/2);a&&(s=0,t.$table.closest("."+r.css.scrollerWrap).children().each(function(){var t=e(this);s+=t.filter('[style*="height"]').length?t.height():t.children("table").height()})),l=t.$table.position().left,n.each(function(){var r=e(this),a=parseInt(r.attr("data-column"),10),n=t.columns-1,c=r.data("header");c&&(c.is(":visible")?(n>a||a===n&&i.resizable_addLastColumn)&&r.css({display:"inline-block",height:s,left:c.position().left-l+c.outerWidth()-o}):r.hide())})},toggleTextSelection:function(t,i){var l=t.namespace+"tsresize";t.widgetOptions.resizable_.disabled=i,e("body").toggleClass(r.css.resizableNoSelect,i),i?e("body").attr("unselectable","on").bind("selectstart"+l,!1):e("body").removeAttr("unselectable").unbind("selectstart"+l)},bindings:function(i,l){var a=i.namespace+"tsresize";l.$resizable_container.children().bind("mousedown",function(t){var a,s=l.resizable_,n=e(i.namespace+"_extra_headers"),o=e(t.target).data("header");a=parseInt(o.attr("data-column"),10),s.$target=o=o.add(n.filter('[data-column="'+a+'"]')),s.target=a,s.$next=t.shiftKey||l.resizable_targetLast?o.parent().children().not(".resizable-false").filter(":last"):o.nextAll(":not(.resizable-false)").eq(0),a=parseInt(s.$next.attr("data-column"),10),s.$next=s.$next.add(n.filter('[data-column="'+a+'"]')),s.next=a,s.mouseXPosition=t.pageX,r.resizable.updateStoredSizes(i,l),r.resizable.toggleTextSelection(i,!0)}),e(document).bind("mousemove"+a,function(e){var t=l.resizable_;t.disabled&&0!==t.mouseXPosition&&t.$target&&(l.resizable_throttle?(clearTimeout(t.timer),t.timer=setTimeout(function(){r.resizable.mouseMove(i,l,e)},isNaN(l.resizable_throttle)?5:l.resizable_throttle)):r.resizable.mouseMove(i,l,e))}).bind("mouseup"+a,function(){l.resizable_.disabled&&(r.resizable.toggleTextSelection(i,!1),r.resizable.stopResize(i,l),r.resizable.setHandlePosition(i,l))}),e(t).bind("resize"+a+" resizeEnd"+a,function(){r.resizable.setHandlePosition(i,l)}),i.$table.bind("columnUpdate"+a,function(){r.resizable.setHandlePosition(i,l)}).find("thead:first").add(e(i.namespace+"_extra_table").find("thead:first")).bind("contextmenu"+a,function(){var e=0===l.resizable_.storedSizes.length;return r.resizableReset(i.table),r.resizable.setHandlePosition(i,l),l.resizable_.storedSizes=[],e})},mouseMove:function(t,i,l){if(0!==i.resizable_.mouseXPosition&&i.resizable_.$target){var a,s=0,n=i.resizable_,o=n.$next,c=n.storedSizes[n.target],d=l.pageX-n.mouseXPosition;if(n.overflow){if(c+d>0){for(n.storedSizes[n.target]+=d,r.resizable.setWidth(n.$target,n.storedSizes[n.target],!0),a=0;a<t.columns;a++)s+=n.storedSizes[a];r.resizable.setWidth(t.$table.add(e(t.namespace+"_extra_table")),s)}o.length||(n.$wrap[0].scrollLeft=t.$table.width())}else n.fullWidth?(n.storedSizes[n.target]+=d,n.storedSizes[n.next]-=d,r.resizable.setWidths(t,i)):(n.storedSizes[n.target]+=d,r.resizable.setWidths(t,i));n.mouseXPosition=l.pageX,t.$table.trigger("stickyHeadersUpdate")}},stopResize:function(e,t){var i=t.resizable_;r.resizable.updateStoredSizes(e,t),i.useStorage&&(r.storage(e.table,r.css.resizableStorage,i.storedSizes),r.storage(e.table,"tablesorter-table-resized-width",e.$table.width())),i.mouseXPosition=0,i.$target=i.$next=null,e.$table.trigger("stickyHeadersUpdate")}},r.addWidget({id:"resizable",priority:40,options:{resizable:!0,resizable_addLastColumn:!1,resizable_widths:[],resizable_throttle:!1,resizable_targetLast:!1,resizable_fullWidth:null},init:function(e,t,i,l){r.resizable.init(i,l)},remove:function(t,i,l,a){if(l.$resizable_container){var s=i.namespace+"tsresize";i.$table.add(e(i.namespace+"_extra_table")).removeClass("hasResizable").children("thead").unbind("contextmenu"+s),l.$resizable_container.remove(),r.resizable.toggleTextSelection(i,!1),r.resizableReset(t,a),e(document).unbind("mousemove"+s+" mouseup"+s)}}}),r.resizableReset=function(t,i){e(t).each(function(){var e,l,a=this.config,s=a&&a.widgetOptions,n=s.resizable_;if(t&&a&&a.$headerIndexed.length){for(n.overflow&&n.tableWidth&&(r.resizable.setWidth(a.$table,n.tableWidth,!0),n.useStorage&&r.storage(t,"tablesorter-table-resized-width","auto")),e=0;e<a.columns;e++)l=a.$headerIndexed[e],s.resizable_widths&&s.resizable_widths[e]?r.resizable.setWidth(l,s.resizable_widths[e],n.overflow):l.hasClass("resizable-false")||r.resizable.setWidth(l,"",n.overflow);a.$table.trigger("stickyHeadersUpdate"),r.storage&&!i&&r.storage(this,r.css.resizableStorage,{})}})}}(jQuery,window),function(e){"use strict";var t=e.tablesorter||{};t.addWidget({id:"saveSort",priority:20,options:{saveSort:!0},init:function(e,t,r,i){t.format(e,r,i,!0)},format:function(r,i,l,a){var s,n,o=i.$table,c=l.saveSort!==!1,d={sortList:i.sortList};i.debug&&(n=new Date),o.hasClass("hasSaveSort")?c&&r.hasInitialized&&t.storage&&(t.storage(r,"tablesorter-savesort",d),i.debug&&t.benchmark("saveSort widget: Saving last sort: "+i.sortList,n)):(o.addClass("hasSaveSort"),d="",t.storage&&(s=t.storage(r,"tablesorter-savesort"),d=s&&s.hasOwnProperty("sortList")&&e.isArray(s.sortList)?s.sortList:"",i.debug&&t.benchmark('saveSort: Last sort loaded: "'+d+'"',n),o.bind("saveSortReset",function(e){e.stopPropagation(),t.storage(r,"tablesorter-savesort","")})),a&&d&&d.length>0?i.sortList=d:r.hasInitialized&&d&&d.length>0&&o.trigger("sorton",[d]))},remove:function(e,r){r.$table.removeClass("hasSaveSort"),t.storage&&t.storage(e,"tablesorter-savesort","")}})}(jQuery),e.tablesorter});
/* custom */ $(document).ready(function(){$('.table-sortable').tablesorter({theme:'bootstrap',widgets:['uitheme','zebra','filter'],headerTemplate:'{content} {icon}',widthFixed:true,ignoreCase:true,widgetOptions:{filter_columnFilters:true,zebra:['even','odd'],filter_reset:'.reset'}});});
/* jquery.tablesorter.widgets.js */ !function(e){"function"==typeof define&&define.amd?define(["jquery"],e):"object"==typeof module&&"object"==typeof module.exports?module.exports=e(require("jquery")):e(jQuery)}(function(e){return function(e,t,r){"use strict";var i=e.tablesorter||{};i.storage=function(l,a,s,n){l=e(l)[0];var o,c,d,f=!1,u={},h=l.config,p=h&&h.widgetOptions,g=n&&n.useSessionStorage||p&&p.storage_useSessionStorage?"sessionStorage":"localStorage",m=e(l),b=n&&n.id||m.attr(n&&n.group||p&&p.storage_group||"data-table-group")||p&&p.storage_tableId||l.id||e(".tablesorter").index(m),y=n&&n.url||m.attr(n&&n.page||p&&p.storage_page||"data-table-page")||p&&p.storage_fixedUrl||h&&h.fixedUrl||t.location.pathname;if(g in t)try{t[g].setItem("_tmptest","temp"),f=!0,t[g].removeItem("_tmptest")}catch(_){h&&h.debug&&i.log(g+" is not supported in this browser")}return e.parseJSON&&(f?u=e.parseJSON(t[g][a]||"null")||{}:(c=r.cookie.split(/[;\s|=]/),o=e.inArray(a,c)+1,u=0!==o?e.parseJSON(c[o]||"null")||{}:{})),(s||""===s)&&t.JSON&&JSON.hasOwnProperty("stringify")?(u[y]||(u[y]={}),u[y][b]=s,f?t[g][a]=JSON.stringify(u):(d=new Date,d.setTime(d.getTime()+31536e6),r.cookie=a+"="+JSON.stringify(u).replace(/\"/g,'"')+"; expires="+d.toGMTString()+"; path=/"),void 0):u&&u[y]?u[y][b]:""}}(jQuery,window,document),function(e){"use strict";var t=e.tablesorter||{};t.themes={bootstrap:{table:"table table-bordered table-striped",caption:"caption",header:"bootstrap-header",sortNone:"",sortAsc:"",sortDesc:"",active:"",hover:"",icons:"",iconSortNone:"bootstrap-icon-unsorted",iconSortAsc:"icon-chevron-up glyphicon glyphicon-chevron-up",iconSortDesc:"icon-chevron-down glyphicon glyphicon-chevron-down",filterRow:"",footerRow:"",footerCells:"",even:"",odd:""},jui:{table:"ui-widget ui-widget-content ui-corner-all",caption:"ui-widget-content",header:"ui-widget-header ui-corner-all ui-state-default",sortNone:"",sortAsc:"",sortDesc:"",active:"ui-state-active",hover:"ui-state-hover",icons:"ui-icon",iconSortNone:"ui-icon-carat-2-n-s",iconSortAsc:"ui-icon-carat-1-n",iconSortDesc:"ui-icon-carat-1-s",filterRow:"",footerRow:"",footerCells:"",even:"ui-widget-content",odd:"ui-state-default"}},e.extend(t.css,{wrapper:"tablesorter-wrapper"}),t.addWidget({id:"uitheme",priority:10,format:function(r,i,l){var a,s,n,o,c,d,f,u,h,p,g,m,b=t.themes,y=i.$table.add(e(i.namespace+"_extra_table")),_=i.$headers.add(e(i.namespace+"_extra_headers")),v=i.theme||"jui",w=b[v]||{},x=e.trim([w.sortNone,w.sortDesc,w.sortAsc,w.active].join(" ")),C=e.trim([w.iconSortNone,w.iconSortDesc,w.iconSortAsc].join(" "));for(i.debug&&(o=new Date),y.hasClass("tablesorter-"+v)&&i.theme===i.appliedTheme&&l.uitheme_applied||(l.uitheme_applied=!0,h=b[i.appliedTheme]||{},m=!e.isEmptyObject(h),p=m?[h.sortNone,h.sortDesc,h.sortAsc,h.active].join(" "):"",g=m?[h.iconSortNone,h.iconSortDesc,h.iconSortAsc].join(" "):"",m&&(l.zebra[0]=e.trim(" "+l.zebra[0].replace(" "+h.even,"")),l.zebra[1]=e.trim(" "+l.zebra[1].replace(" "+h.odd,"")),i.$tbodies.children().removeClass([h.even,h.odd].join(" "))),w.even&&(l.zebra[0]+=" "+w.even),w.odd&&(l.zebra[1]+=" "+w.odd),y.children("caption").removeClass(h.caption||"").addClass(w.caption),f=y.removeClass((i.appliedTheme?"tablesorter-"+(i.appliedTheme||""):"")+" "+(h.table||"")).addClass("tablesorter-"+v+" "+(w.table||"")).children("tfoot"),i.appliedTheme=i.theme,f.length&&f.children("tr").removeClass(h.footerRow||"").addClass(w.footerRow).children("th, td").removeClass(h.footerCells||"").addClass(w.footerCells),_.removeClass((m?[h.header,h.hover,p].join(" "):"")||"").addClass(w.header).not(".sorter-false").unbind("mouseenter.tsuitheme mouseleave.tsuitheme").bind("mouseenter.tsuitheme mouseleave.tsuitheme",function(t){e(this)["mouseenter"===t.type?"addClass":"removeClass"](w.hover||"")}),_.each(function(){var r=e(this);r.find("."+t.css.wrapper).length||r.wrapInner('<div class="'+t.css.wrapper+'" style="position:relative;height:100%;width:100%"></div>')}),i.cssIcon&&_.find("."+t.css.icon).removeClass(m?[h.icons,g].join(" "):"").addClass(w.icons||""),y.hasClass("hasFilters")&&y.children("thead").children("."+t.css.filterRow).removeClass(m?h.filterRow||"":"").addClass(w.filterRow||"")),a=0;a<i.columns;a++)c=i.$headers.add(e(i.namespace+"_extra_headers")).not(".sorter-false").filter('[data-column="'+a+'"]'),d=t.css.icon?c.find("."+t.css.icon):e(),u=_.not(".sorter-false").filter('[data-column="'+a+'"]:last'),u.length&&(c.removeClass(x),d.removeClass(C),u[0].sortDisabled?d.removeClass(w.icons||""):(s=w.sortNone,n=w.iconSortNone,u.hasClass(t.css.sortAsc)?(s=[w.sortAsc,w.active].join(" "),n=w.iconSortAsc):u.hasClass(t.css.sortDesc)&&(s=[w.sortDesc,w.active].join(" "),n=w.iconSortDesc),c.addClass(s),d.addClass(n||"")));i.debug&&t.benchmark("Applying "+v+" theme",o)},remove:function(e,r,i,l){if(i.uitheme_applied){var a=r.$table,s=r.appliedTheme||"jui",n=t.themes[s]||t.themes.jui,o=a.children("thead").children(),c=n.sortNone+" "+n.sortDesc+" "+n.sortAsc,d=n.iconSortNone+" "+n.iconSortDesc+" "+n.iconSortAsc;a.removeClass("tablesorter-"+s+" "+n.table),i.uitheme_applied=!1,l||(a.find(t.css.header).removeClass(n.header),o.unbind("mouseenter.tsuitheme mouseleave.tsuitheme").removeClass(n.hover+" "+c+" "+n.active).filter("."+t.css.filterRow).removeClass(n.filterRow),o.find("."+t.css.icon).removeClass(n.icons+" "+d))}}})}(jQuery),function(e){"use strict";var t=e.tablesorter||{};t.addWidget({id:"columns",priority:30,options:{columns:["primary","secondary","tertiary"]},format:function(r,i,l){var a,s,n,o,c,d,f,u,h=i.$table,p=i.$tbodies,g=i.sortList,m=g.length,b=l&&l.columns||["primary","secondary","tertiary"],y=b.length-1;for(f=b.join(" "),s=0;s<p.length;s++)a=t.processTbody(r,p.eq(s),!0),n=a.children("tr"),n.each(function(){if(c=e(this),"none"!==this.style.display&&(d=c.children().removeClass(f),g&&g[0]&&(d.eq(g[0][0]).addClass(b[0]),m>1)))for(u=1;m>u;u++)d.eq(g[u][0]).addClass(b[u]||b[y])}),t.processTbody(r,a,!1);if(o=l.columns_thead!==!1?["thead tr"]:[],l.columns_tfoot!==!1&&o.push("tfoot tr"),o.length&&(n=h.find(o.join(",")).children().removeClass(f),m))for(u=0;m>u;u++)n.filter('[data-column="'+g[u][0]+'"]').addClass(b[u]||b[y])},remove:function(r,i,l){var a,s,n=i.$tbodies,o=(l.columns||["primary","secondary","tertiary"]).join(" ");for(i.$headers.removeClass(o),i.$table.children("tfoot").children("tr").children("th, td").removeClass(o),a=0;a<n.length;a++)s=t.processTbody(r,n.eq(a),!0),s.children("tr").each(function(){e(this).children().removeClass(o)}),t.processTbody(r,s,!1)}})}(jQuery),function(e){"use strict";var t=e.tablesorter||{},r=t.css;e.extend(r,{filterRow:"tablesorter-filter-row",filter:"tablesorter-filter",filterDisabled:"disabled",filterRowHide:"hideme"}),t.addWidget({id:"filter",priority:50,options:{filter_childRows:!1,filter_childByColumn:!1,filter_columnFilters:!0,filter_columnAnyMatch:!0,filter_cellFilter:"",filter_cssFilter:"",filter_defaultFilter:{},filter_excludeFilter:{},filter_external:"",filter_filteredRow:"filtered",filter_formatter:null,filter_functions:null,filter_hideEmpty:!0,filter_hideFilters:!1,filter_ignoreCase:!0,filter_liveSearch:!0,filter_onlyAvail:"filter-onlyAvail",filter_placeholder:{search:"",select:""},filter_reset:null,filter_saveFilters:!1,filter_searchDelay:300,filter_searchFiltered:!0,filter_selectSource:null,filter_startsWith:!1,filter_useParsedData:!1,filter_serversideFiltering:!1,filter_defaultAttrib:"data-value",filter_selectSourceSeparator:"|"},format:function(e,r,i){r.$table.hasClass("hasFilters")||t.filter.init(e,r,i)},remove:function(i,l,a,s){var n,o,c=l.$table,d=l.$tbodies,f="addRows updateCell update updateRows updateComplete appendCache filterReset filterEnd search ".split(" ").join(l.namespace+"filter ");if(c.removeClass("hasFilters").unbind(f.replace(/\s+/g," ")).find("."+r.filterRow).remove(),!s){for(n=0;n<d.length;n++)o=t.processTbody(i,d.eq(n),!0),o.children().removeClass(a.filter_filteredRow).show(),t.processTbody(i,o,!1);a.filter_reset&&e(document).undelegate(a.filter_reset,"click.tsfilter")}}}),t.filter={regex:{regex:/^\/((?:\\\/|[^\/])+)\/([mig]{0,3})?$/,child:/tablesorter-childRow/,filtered:/filtered/,type:/undefined|number/,exact:/(^[\"\'=]+)|([\"\'=]+$)/g,nondigit:/[^\w,. \-()]/g,operators:/[<>=]/g,query:"(q|query)"},types:{or:function(r,i,l){if(/\|/.test(i.iFilter)||t.filter.regex.orSplit.test(i.filter)){var a,s,n,o,c=e.extend({},i),d=i.index,f=i.parsed[d],u=i.filter.split(t.filter.regex.orSplit),h=i.iFilter.split(t.filter.regex.orSplit),p=u.length;for(a=0;p>a;a++)if(c.nestedFilters=!0,c.filter=""+(t.filter.parseFilter(r,u[a],d,f)||""),c.iFilter=""+(t.filter.parseFilter(r,h[a],d,f)||""),n="("+(t.filter.parseFilter(r,c.filter,d,f)||"")+")",o=new RegExp(i.isMatch?n:"^"+n+"$",r.widgetOptions.filter_ignoreCase?"i":""),s=o.test(c.exact)||t.filter.processTypes(r,c,l))return s;return s||!1}return null},and:function(r,i,l){if(t.filter.regex.andTest.test(i.filter)){var a,s,n,o,c,d=e.extend({},i),f=i.index,u=i.parsed[f],h=i.filter.split(t.filter.regex.andSplit),p=i.iFilter.split(t.filter.regex.andSplit),g=h.length;for(a=0;g>a;a++)d.nestedFilters=!0,d.filter=""+(t.filter.parseFilter(r,h[a],f,u)||""),d.iFilter=""+(t.filter.parseFilter(r,p[a],f,u)||""),o=("("+(t.filter.parseFilter(r,d.filter,f,u)||"")+")").replace(/\?/g,"\\S{1}").replace(/\*/g,"\\S*"),c=new RegExp(i.isMatch?o:"^"+o+"$",r.widgetOptions.filter_ignoreCase?"i":""),n=c.test(d.exact)||t.filter.processTypes(r,d,l),s=0===a?n:s&&n;return s||!1}return null},regex:function(e,r){if(t.filter.regex.regex.test(r.filter)){var i,l=r.filter_regexCache[r.index]||t.filter.regex.regex.exec(r.filter),a=l instanceof RegExp;try{a||(r.filter_regexCache[r.index]=l=new RegExp(l[1],l[2])),i=l.test(r.exact)}catch(s){i=!1}return i}return null},operators:function(r,i){if(/^[<>]=?/.test(i.iFilter)&&""!==i.iExact){var l,a,s,n=r.table,o=i.index,c=i.parsed[o],d=t.formatFloat(i.iFilter.replace(t.filter.regex.operators,""),n),f=r.parsers[o],u=d;return(c||"numeric"===f.type)&&(s=e.trim(""+i.iFilter.replace(t.filter.regex.operators,"")),a=t.filter.parseFilter(r,s,o,!0),d="number"!=typeof a||""===a||isNaN(a)?d:a),!c&&"numeric"!==f.type||isNaN(d)||"undefined"==typeof i.cache?(s=isNaN(i.iExact)?i.iExact.replace(t.filter.regex.nondigit,""):i.iExact,l=t.formatFloat(s,n)):l=i.cache,/>/.test(i.iFilter)?a=/>=/.test(i.iFilter)?l>=d:l>d:/</.test(i.iFilter)&&(a=/<=/.test(i.iFilter)?d>=l:d>l),a||""!==u||(a=!0),a}return null},notMatch:function(r,i){if(/^\!/.test(i.iFilter)){var l,a=i.iFilter.replace("!",""),s=t.filter.parseFilter(r,a,i.index,i.parsed[i.index])||"";return t.filter.regex.exact.test(s)?(s=s.replace(t.filter.regex.exact,""),""===s?!0:e.trim(s)!==i.iExact):(l=i.iExact.search(e.trim(s)),""===s?!0:!(r.widgetOptions.filter_startsWith?0===l:l>=0))}return null},exact:function(r,i){if(t.filter.regex.exact.test(i.iFilter)){var l=i.iFilter.replace(t.filter.regex.exact,""),a=t.filter.parseFilter(r,l,i.index,i.parsed[i.index])||"";return i.anyMatch?e.inArray(a,i.rowArray)>=0:a==i.iExact}return null},range:function(e,r){if(t.filter.regex.toTest.test(r.iFilter)){var i,l,a,s,n=e.table,o=r.index,c=r.parsed[o],d=r.iFilter.split(t.filter.regex.toSplit);return l=d[0].replace(t.filter.regex.nondigit,"")||"",a=t.formatFloat(t.filter.parseFilter(e,l,o,c),n),l=d[1].replace(t.filter.regex.nondigit,"")||"",s=t.formatFloat(t.filter.parseFilter(e,l,o,c),n),(c||"numeric"===e.parsers[o].type)&&(i=e.parsers[o].format(""+d[0],n,e.$headers.eq(o),o),a=""===i||isNaN(i)?a:i,i=e.parsers[o].format(""+d[1],n,e.$headers.eq(o),o),s=""===i||isNaN(i)?s:i),!c&&"numeric"!==e.parsers[o].type||isNaN(a)||isNaN(s)?(l=isNaN(r.iExact)?r.iExact.replace(t.filter.regex.nondigit,""):r.iExact,i=t.formatFloat(l,n)):i=r.cache,a>s&&(l=a,a=s,s=l),i>=a&&s>=i||""===a||""===s}return null},wild:function(e,r){if(/[\?\*\|]/.test(r.iFilter)){var i=r.index,l=r.parsed[i],a=""+(t.filter.parseFilter(e,r.iFilter,i,l)||"");return!/\?\*/.test(a)&&r.nestedFilters&&(a=r.isMatch?a:"^("+a+")$"),new RegExp(a.replace(/\?/g,"\\S{1}").replace(/\*/g,"\\S*"),e.widgetOptions.filter_ignoreCase?"i":"").test(r.exact)}return null},fuzzy:function(e,r){if(/^~/.test(r.iFilter)){var i,l=0,a=r.iExact.length,s=r.iFilter.slice(1),n=t.filter.parseFilter(e,s,r.index,r.parsed[r.index])||"";for(i=0;a>i;i++)r.iExact[i]===n[l]&&(l+=1);return l===n.length?!0:!1}return null}},init:function(i,l,a){t.language=e.extend(!0,{},{to:"to",or:"or",and:"and"},t.language);var s,n,o,c,d,f,u,h,p,g=t.filter.regex;if(l.$table.addClass("hasFilters"),a.searchTimer=null,a.filter_initTimer=null,a.filter_formatterCount=0,a.filter_formatterInit=[],a.filter_anyColumnSelector='[data-column="all"],[data-column="any"]',a.filter_multipleColumnSelector='[data-column*="-"],[data-column*=","]',u="\\{"+t.filter.regex.query+"\\}",e.extend(g,{child:new RegExp(l.cssChildRow),filtered:new RegExp(a.filter_filteredRow),alreadyFiltered:new RegExp("(\\s+("+t.language.or+"|-|"+t.language.to+")\\s+)","i"),toTest:new RegExp("\\s+(-|"+t.language.to+")\\s+","i"),toSplit:new RegExp("(?:\\s+(?:-|"+t.language.to+")\\s+)","gi"),andTest:new RegExp("\\s+("+t.language.and+"|&&)\\s+","i"),andSplit:new RegExp("(?:\\s+(?:"+t.language.and+"|&&)\\s+)","gi"),orSplit:new RegExp("(?:\\s+(?:"+t.language.or+")\\s+|\\|)","gi"),iQuery:new RegExp(u,"i"),igQuery:new RegExp(u,"ig")}),u=l.$headers.filter(".filter-false, .parser-false").length,a.filter_columnFilters!==!1&&u!==l.$headers.length&&t.filter.buildRow(i,l,a),o="addRows updateCell update updateRows updateComplete appendCache filterReset filterEnd search ".split(" ").join(l.namespace+"filter "),l.$table.bind(o,function(s,n){return u=a.filter_hideEmpty&&e.isEmptyObject(l.cache)&&!(l.delayInit&&"appendCache"===s.type),l.$table.find("."+r.filterRow).toggleClass(a.filter_filteredRow,u),/(search|filter)/.test(s.type)||(s.stopPropagation(),t.filter.buildDefault(i,!0)),"filterReset"===s.type?(l.$table.find("."+r.filter).add(a.filter_$externalFilters).val(""),t.filter.searching(i,[])):"filterEnd"===s.type?t.filter.buildDefault(i,!0):(n="search"===s.type?n:"updateComplete"===s.type?l.$table.data("lastSearch"):"",/(update|add)/.test(s.type)&&"updateComplete"!==s.type&&(l.lastCombinedFilter=null,l.lastSearch=[]),t.filter.searching(i,n,!0)),!1}),a.filter_reset&&(a.filter_reset instanceof e?a.filter_reset.click(function(){l.$table.trigger("filterReset")}):e(a.filter_reset).length&&e(document).undelegate(a.filter_reset,"click.tsfilter").delegate(a.filter_reset,"click.tsfilter",function(){l.$table.trigger("filterReset")})),a.filter_functions)for(d=0;d<l.columns;d++)if(h=t.getColumnData(i,a.filter_functions,d))if(c=l.$headerIndexed[d].removeClass("filter-select"),p=!(c.hasClass("filter-false")||c.hasClass("parser-false")),s="",h===!0&&p)t.filter.buildSelect(i,d);else if("object"==typeof h&&p){for(n in h)"string"==typeof n&&(s+=""===s?'<option value="">'+(c.data("placeholder")||c.attr("data-placeholder")||a.filter_placeholder.select||"")+"</option>":"",u=n,o=n,n.indexOf(a.filter_selectSourceSeparator)>=0&&(u=n.split(a.filter_selectSourceSeparator),o=u[1],u=u[0]),s+="<option "+(o===u?"":'data-function-name="'+n+'" ')+'value="'+u+'">'+o+"</option>");l.$table.find("thead").find("select."+r.filter+'[data-column="'+d+'"]').append(s),o=a.filter_selectSource,h=e.isFunction(o)?!0:t.getColumnData(i,o,d),h&&t.filter.buildSelect(l.table,d,"",!0,c.hasClass(a.filter_onlyAvail))}t.filter.buildDefault(i,!0),t.filter.bindSearch(i,l.$table.find("."+r.filter),!0),a.filter_external&&t.filter.bindSearch(i,a.filter_external),a.filter_hideFilters&&t.filter.hideFilters(i,l),l.showProcessing&&(o="filterStart filterEnd ".split(" ").join(l.namespace+"filter "),l.$table.unbind(o.replace(/\s+/g," ")).bind(o,function(a,s){c=s?l.$table.find("."+r.header).filter("[data-column]").filter(function(){return""!==s[e(this).data("column")]}):"",t.isProcessing(i,"filterStart"===a.type,s?c:"")})),l.filteredRows=l.totalRows,o="tablesorter-initialized pagerBeforeInitialized ".split(" ").join(l.namespace+"filter "),l.$table.unbind(o.replace(/\s+/g," ")).bind(o,function(){var e=this.config.widgetOptions;f=t.filter.setDefaults(i,l,e)||[],f.length&&(l.delayInit&&""===f.join("")||t.setFilters(i,f,!0)),l.$table.trigger("filterFomatterUpdate"),setTimeout(function(){e.filter_initialized||t.filter.filterInitComplete(l)},100)}),l.pager&&l.pager.initialized&&!a.filter_initialized&&(l.$table.trigger("filterFomatterUpdate"),setTimeout(function(){t.filter.filterInitComplete(l)},100))},formatterUpdated:function(e,t){var r=e.closest("table")[0].config.widgetOptions;r.filter_initialized||(r.filter_formatterInit[t]=1)},filterInitComplete:function(r){var i,l,a=r.widgetOptions,s=0,n=function(){a.filter_initialized=!0,r.$table.trigger("filterInit",r),t.filter.findRows(r.table,r.$table.data("lastSearch")||[])};if(e.isEmptyObject(a.filter_formatter))n();else{for(l=a.filter_formatterInit.length,i=0;l>i;i++)1===a.filter_formatterInit[i]&&s++;clearTimeout(a.filter_initTimer),a.filter_initialized||s!==a.filter_formatterCount?a.filter_initialized||(a.filter_initTimer=setTimeout(function(){n()},500)):n()}},setDefaults:function(r,i,l){var a,s,n,o,c,d=t.getFilters(r)||[];if(l.filter_saveFilters&&t.storage&&(s=t.storage(r,"tablesorter-filters")||[],a=e.isArray(s),a&&""===s.join("")||!a||(d=s)),""===d.join(""))for(c=i.$headers.add(l.filter_$externalFilters).filter("["+l.filter_defaultAttrib+"]"),n=0;n<=i.columns;n++)o=n===i.columns?"all":n,d[n]=c.filter('[data-column="'+o+'"]').attr(l.filter_defaultAttrib)||d[n]||"";return i.$table.data("lastSearch",d),d},parseFilter:function(e,t,r,i){return i?e.parsers[r].format(t,e.table,[],r):t},buildRow:function(i,l,a){var s,n,o,c,d,f,u,h,p=a.filter_cellFilter,g=l.columns,m=e.isArray(p),b='<tr role="row" class="'+r.filterRow+" "+l.cssIgnoreRow+'">';for(n=0;g>n;n++)b+="<td",b+=m?p[n]?' class="'+p[n]+'"':"":""!==p?' class="'+p+'"':"",b+="></td>";for(l.$filters=e(b+="</tr>").appendTo(l.$table.children("thead").eq(0)).find("td"),n=0;g>n;n++)d=!1,o=l.$headerIndexed[n],u=t.getColumnData(i,a.filter_functions,n),c=a.filter_functions&&u&&"function"!=typeof u||o.hasClass("filter-select"),s=t.getColumnData(i,l.headers,n),d="false"===t.getData(o[0],s,"filter")||"false"===t.getData(o[0],s,"parser"),c?b=e("<select>").appendTo(l.$filters.eq(n)):(u=t.getColumnData(i,a.filter_formatter,n),u?(a.filter_formatterCount++,b=u(l.$filters.eq(n),n),b&&0===b.length&&(b=l.$filters.eq(n).children("input")),b&&(0===b.parent().length||b.parent().length&&b.parent()[0]!==l.$filters[n])&&l.$filters.eq(n).append(b)):b=e('<input type="search">').appendTo(l.$filters.eq(n)),b&&(h=o.data("placeholder")||o.attr("data-placeholder")||a.filter_placeholder.search||"",b.attr("placeholder",h))),b&&(f=(e.isArray(a.filter_cssFilter)?"undefined"!=typeof a.filter_cssFilter[n]?a.filter_cssFilter[n]||"":"":a.filter_cssFilter)||"",b.addClass(r.filter+" "+f).attr("data-column",n),d&&(b.attr("placeholder","").addClass(r.filterDisabled)[0].disabled=!0))},bindSearch:function(r,i,l){if(r=e(r)[0],i=e(i),i.length){var a,s=r.config,n=s.widgetOptions,o=s.namespace+"filter",c=n.filter_$externalFilters;l!==!0&&(a=n.filter_anyColumnSelector+","+n.filter_multipleColumnSelector,n.filter_$anyMatch=i.filter(a),n.filter_$externalFilters=c&&c.length?n.filter_$externalFilters.add(i):i,t.setFilters(r,s.$table.data("lastSearch")||[],l===!1)),a="keypress keyup search change ".split(" ").join(o+" "),i.attr("data-lastSearchTime",(new Date).getTime()).unbind(a.replace(/\s+/g," ")).bind("keyup"+o,function(i){if(e(this).attr("data-lastSearchTime",(new Date).getTime()),27===i.which)this.value="";else{if(n.filter_liveSearch===!1)return;if(""!==this.value&&("number"==typeof n.filter_liveSearch&&this.value.length<n.filter_liveSearch||13!==i.which&&8!==i.which&&(i.which<32||i.which>=37&&i.which<=40)))return}t.filter.searching(r,!0,!0)}).bind("search change keypress ".split(" ").join(o+" "),function(i){var l=e(this).data("column");(13===i.which||"search"===i.type||"change"===i.type&&this.value!==s.lastSearch[l])&&(i.preventDefault(),e(this).attr("data-lastSearchTime",(new Date).getTime()),t.filter.searching(r,!1,!0))})}},searching:function(e,r,i){var l=e.config.widgetOptions;clearTimeout(l.searchTimer),"undefined"==typeof r||r===!0?l.searchTimer=setTimeout(function(){t.filter.checkFilters(e,r,i)},l.filter_liveSearch?l.filter_searchDelay:10):t.filter.checkFilters(e,r,i)},checkFilters:function(i,l,a){var s=i.config,n=s.widgetOptions,o=e.isArray(l),c=o?l:t.getFilters(i,!0),d=(c||[]).join("");return e.isEmptyObject(s.cache)?void(s.delayInit&&s.pager&&s.pager.initialized&&s.$table.trigger("updateCache",[function(){t.filter.checkFilters(i,!1,a)}])):(o&&(t.setFilters(i,c,!1,a!==!0),n.filter_initialized||(s.lastCombinedFilter="")),n.filter_hideFilters&&s.$table.find("."+r.filterRow).trigger(""===d?"mouseleave":"mouseenter"),s.lastCombinedFilter!==d||l===!1?(l===!1&&(s.lastCombinedFilter=null,s.lastSearch=[]),n.filter_initialized&&s.$table.trigger("filterStart",[c]),s.showProcessing?void setTimeout(function(){return t.filter.findRows(i,c,d),!1},30):(t.filter.findRows(i,c,d),!1)):void 0)},hideFilters:function(i,l){var a;l.$table.find("."+r.filterRow).bind("mouseenter mouseleave",function(t){var i=t,s=e(this);clearTimeout(a),a=setTimeout(function(){/enter|over/.test(i.type)?s.removeClass(r.filterRowHide):e(document.activeElement).closest("tr")[0]!==s[0]&&""===l.lastCombinedFilter&&s.addClass(r.filterRowHide)},200)}).find("input, select").bind("focus blur",function(i){var s=i,n=e(this).closest("tr");clearTimeout(a),a=setTimeout(function(){clearTimeout(a),""===t.getFilters(l.$table).join("")&&n.toggleClass(r.filterRowHide,"focus"!==s.type)},200)})},defaultFilter:function(r,i){if(""===r)return r;var l=t.filter.regex.iQuery,a=i.match(t.filter.regex.igQuery).length,s=a>1?e.trim(r).split(/\s/):[e.trim(r)],n=s.length-1,o=0,c=i;for(1>n&&a>1&&(s[1]=s[0]);l.test(c);)c=c.replace(l,s[o++]||""),l.test(c)&&n>o&&""!==(s[o]||"")&&(c=i.replace(l,c));return c},getLatestSearch:function(t){return t?t.sort(function(t,r){return e(r).attr("data-lastSearchTime")-e(t).attr("data-lastSearchTime")}):t||e()},multipleColumns:function(r,i){var l,a,s,n,o,c,d,f,u,h=r.widgetOptions,p=h.filter_initialized||!i.filter(h.filter_anyColumnSelector).length,g=[],m=e.trim(t.filter.getLatestSearch(i).attr("data-column")||"");if(p&&/-/.test(m))for(a=m.match(/(\d+)\s*-\s*(\d+)/g),u=a.length,f=0;u>f;f++){for(s=a[f].split(/\s*-\s*/),n=parseInt(s[0],10)||0,o=parseInt(s[1],10)||r.columns-1,n>o&&(l=n,n=o,o=l),o>=r.columns&&(o=r.columns-1);o>=n;n++)g.push(n);m=m.replace(a[f],"")}if(p&&/,/.test(m))for(c=m.split(/\s*,\s*/),u=c.length,d=0;u>d;d++)""!==c[d]&&(f=parseInt(c[d],10),f<r.columns&&g.push(f));if(!g.length)for(f=0;f<r.columns;f++)g.push(f);return g},processTypes:function(r,i,l){var a,s=null,n=null;for(a in t.filter.types)e.inArray(a,l.excludeMatch)<0&&null===n&&(n=t.filter.types[a](r,i,l),null!==n&&(s=n));return s},processRow:function(r,i,l){var a,s,n,o,c,d,f,u,h=t.filter.regex,p=r.widgetOptions,g=!0;if(i.$cells=i.$row.children(),i.anyMatchFlag){if(a=t.filter.multipleColumns(r,p.filter_$anyMatch),i.anyMatch=!0,i.isMatch=!0,i.rowArray=i.$cells.map(function(l){return e.inArray(l,a)>-1?(i.parsed[l]?u=i.cacheArray[l]:(u=i.rawArray[l],u=e.trim(p.filter_ignoreCase?u.toLowerCase():u),r.sortLocaleCompare&&(u=t.replaceAccents(u))),u):void 0}).get(),i.filter=i.anyMatchFilter,i.iFilter=i.iAnyMatchFilter,i.exact=i.rowArray.join(" "),i.iExact=p.filter_ignoreCase?i.exact.toLowerCase():i.exact,i.cache=i.cacheArray.slice(0,-1).join(" "),l.excludeMatch=l.noAnyMatch,c=t.filter.processTypes(r,i,l),null!==c)g=c;else if(p.filter_startsWith)for(g=!1,a=r.columns;!g&&a>0;)a--,g=g||0===i.rowArray[a].indexOf(i.iFilter);else g=(i.iExact+i.childRowText).indexOf(i.iFilter)>=0;if(i.anyMatch=!1,i.filters.join("")===i.filter)return g}for(a=0;a<r.columns;a++)i.filter=i.filters[a],i.index=a,l.excludeMatch=l.excludeFilter[a],i.filter&&(i.cache=i.cacheArray[a],p.filter_useParsedData||i.parsed[a]?i.exact=i.cache:(n=i.rawArray[a]||"",i.exact=r.sortLocaleCompare?t.replaceAccents(n):n),i.iExact=!h.type.test(typeof i.exact)&&p.filter_ignoreCase?i.exact.toLowerCase():i.exact,i.isMatch=r.$headerIndexed[i.index].hasClass("filter-match"),n=g,f=p.filter_columnFilters?r.$filters.add(r.$externalFilters).filter('[data-column="'+a+'"]').find("select option:selected").attr("data-function-name")||"":"",r.sortLocaleCompare&&(i.filter=t.replaceAccents(i.filter)),o=!0,p.filter_defaultFilter&&h.iQuery.test(l.defaultColFilter[a])&&(i.filter=t.filter.defaultFilter(i.filter,l.defaultColFilter[a]),o=!1),i.iFilter=p.filter_ignoreCase?(i.filter||"").toLowerCase():i.filter,d=l.functions[a],s=r.$headerIndexed[a].hasClass("filter-select"),c=null,(d||s&&o)&&(d===!0||s?c=i.isMatch?i.iExact.search(i.iFilter)>=0:i.filter===i.exact:"function"==typeof d?c=d(i.exact,i.cache,i.filter,a,i.$row,r,i):"function"==typeof d[f||i.filter]&&(u=f||i.filter,c=d[u](i.exact,i.cache,i.filter,a,i.$row,r,i))),null===c?(c=t.filter.processTypes(r,i,l),null!==c?n=c:(u=(i.iExact+i.childRowText).indexOf(t.filter.parseFilter(r,i.iFilter,a,i.parsed[a])),n=!p.filter_startsWith&&u>=0||p.filter_startsWith&&0===u)):n=c,g=n?g:!1);return g},findRows:function(r,i,l){if(r.config.lastCombinedFilter!==l&&r.config.widgetOptions.filter_initialized){var a,s,n,o,c,d,f,u,h,p,g,m,b,y,_,v,w,x,C,z,S,$,F=e.extend([],i),R=t.filter.regex,k=r.config,T=k.widgetOptions,A={anyMatch:!1,filters:i,filter_regexCache:[]},H={noAnyMatch:["range","notMatch","operators"],functions:[],excludeFilter:[],defaultColFilter:[],defaultAnyFilter:t.getColumnData(r,T.filter_defaultFilter,k.columns,!0)||""};for(A.parsed=k.$headers.map(function(i){return k.parsers&&k.parsers[i]&&k.parsers[i].parsed||t.getData&&"parsed"===t.getData(k.$headerIndexed[i],t.getColumnData(r,k.headers,i),"filter")||e(this).hasClass("filter-parsed")}).get(),u=0;u<k.columns;u++)H.functions[u]=t.getColumnData(r,T.filter_functions,u),H.defaultColFilter[u]=t.getColumnData(r,T.filter_defaultFilter,u)||"",H.excludeFilter[u]=(t.getColumnData(r,T.filter_excludeFilter,u,!0)||"").split(/\s+/);for(k.debug&&(t.log("Filter: Starting filter widget search",i),b=new Date),k.filteredRows=0,k.totalRows=0,l=(F||[]).join(""),d=0;d<k.$tbodies.length;d++){if(f=t.processTbody(r,k.$tbodies.eq(d),!0),u=k.columns,s=k.cache[d].normalized,o=e(e.map(s,function(e){return e[u].$row.get()})),""===l||T.filter_serversideFiltering)o.removeClass(T.filter_filteredRow).not("."+k.cssChildRow).css("display","");else{if(o=o.not("."+k.cssChildRow),a=o.length,(T.filter_$anyMatch&&T.filter_$anyMatch.length||"undefined"!=typeof i[k.columns])&&(A.anyMatchFlag=!0,A.anyMatchFilter=""+(i[k.columns]||T.filter_$anyMatch&&t.filter.getLatestSearch(T.filter_$anyMatch).val()||""),T.filter_columnAnyMatch)){for(x=A.anyMatchFilter.split(R.andSplit),C=!1,_=0;_<x.length;_++)z=x[_].split(":"),z.length>1&&(S=parseInt(z[0],10)-1,S>=0&&S<k.columns&&(i[S]=z[1],x.splice(_,1),_--,C=!0));C&&(A.anyMatchFilter=x.join(" && "))}if(w=T.filter_searchFiltered,g=k.lastSearch||k.$table.data("lastSearch")||[],w)for(_=0;u+1>_;_++)y=i[_]||"",w||(_=u),w=!(!w||!g.length||0!==y.indexOf(g[_]||"")||R.alreadyFiltered.test(y)||/[=\"\|!]/.test(y)||/(>=?\s*-\d)/.test(y)||/(<=?\s*\d)/.test(y)||""!==y&&k.$filters&&k.$filters.eq(_).find("select").length&&!k.$headerIndexed[_].hasClass("filter-match"));for(v=o.not("."+T.filter_filteredRow).length,w&&0===v&&(w=!1),k.debug&&t.log("Filter: Searching through "+(w&&a>v?v:"all")+" rows"),A.anyMatchFlag&&(k.sortLocaleCompare&&(A.anyMatchFilter=t.replaceAccents(A.anyMatchFilter)),T.filter_defaultFilter&&R.iQuery.test(H.defaultAnyFilter)&&(A.anyMatchFilter=t.filter.defaultFilter(A.anyMatchFilter,H.defaultAnyFilter),w=!1),A.iAnyMatchFilter=T.filter_ignoreCase&&k.ignoreCase?A.anyMatchFilter.toLowerCase():A.anyMatchFilter),c=0;a>c;c++)if($=o[c].className,h=c&&R.child.test($),!(h||w&&R.filtered.test($))){if(A.$row=o.eq(c),A.cacheArray=s[c],n=A.cacheArray[k.columns],A.rawArray=n.raw,A.childRowText="",!T.filter_childByColumn){for($="",p=n.child,_=0;_<p.length;_++)$+=" "+p[_].join("")||"";A.childRowText=T.filter_childRows?T.filter_ignoreCase?$.toLowerCase():$:""}if(m=t.filter.processRow(k,A,H),p=n.$row.filter(":gt( 0 )"),T.filter_childRows&&p.length){if(T.filter_childByColumn)for(_=0;_<p.length;_++)A.$row=p.eq(_),A.cacheArray=n.child[_],A.rawArray=A.cacheArray,m=m||t.filter.processRow(k,A,H);p.toggleClass(T.filter_filteredRow,!m)}n.$row.toggleClass(T.filter_filteredRow,!m)[0].display=m?"":"none"}}k.filteredRows+=o.not("."+T.filter_filteredRow).length,k.totalRows+=o.length,t.processTbody(r,f,!1)}k.lastCombinedFilter=l,k.lastSearch=F,k.$table.data("lastSearch",F),T.filter_saveFilters&&t.storage&&t.storage(r,"tablesorter-filters",F),k.debug&&t.benchmark("Completed filter widget search",b),T.filter_initialized&&k.$table.trigger("filterEnd",k),setTimeout(function(){k.$table.trigger("applyWidgets")},0)}},getOptionSource:function(r,i,l){r=e(r)[0];var a,s,n,o,c=r.config,d=c.widgetOptions,f=[],u=!1,h=d.filter_selectSource,p=c.$table.data("lastSearch")||[],g=e.isFunction(h)?!0:t.getColumnData(r,h,i);if(l&&""!==p[i]&&(l=!1),g===!0)u=h(r,i,l);else{if(g instanceof e||"string"===e.type(g)&&g.indexOf("</option>")>=0)return g;e.isArray(g)?u=g:"object"===e.type(h)&&g&&(u=g(r,i,l))}if(u===!1&&(u=t.filter.getOptions(r,i,l)),u=e.grep(u,function(t,r){return e.inArray(t,u)===r}),c.$headerIndexed[i].hasClass("filter-select-nosort"))return u;for(o=u.length,n=0;o>n;n++)s=u[n],f.push({t:s,p:c.parsers&&c.parsers.length&&c.parsers[i].format(s,r,[],i)||s});for(a=c.textSorter||"",f.sort(function(l,s){var n=l.p.toString(),o=s.p.toString();return e.isFunction(a)?a(n,o,!0,i,r):"object"==typeof a&&a.hasOwnProperty(i)?a[i](n,o,!0,i,r):t.sortNatural?t.sortNatural(n,o):!0}),u=[],o=f.length,n=0;o>n;n++)u.push(f[n].t);return u},getOptions:function(t,r,i){t=e(t)[0];var l,a,s,n,o,c=t.config,d=c.widgetOptions,f=[];for(a=0;a<c.$tbodies.length;a++)for(o=c.cache[a],s=c.cache[a].normalized.length,l=0;s>l;l++)n=o.row?o.row[l]:o.normalized[l][c.columns].$row[0],i&&n.className.match(d.filter_filteredRow)||f.push(d.filter_useParsedData||c.parsers[r].parsed||c.$headerIndexed[r].hasClass("filter-parsed")?""+o.normalized[l][r]:o.normalized[l][c.columns].raw[r]);return f},buildSelect:function(i,l,a,s,n){if(i=e(i)[0],l=parseInt(l,10),i.config.cache&&!e.isEmptyObject(i.config.cache)){var o,c,d,f,u,h,p=i.config,g=p.widgetOptions,m=p.$headerIndexed[l],b='<option value="">'+(m.data("placeholder")||m.attr("data-placeholder")||g.filter_placeholder.select||"")+"</option>",y=p.$table.find("thead").find("select."+r.filter+'[data-column="'+l+'"]').val();if(("undefined"==typeof a||""===a)&&(a=t.filter.getOptionSource(i,l,n)),e.isArray(a)){for(o=0;o<a.length;o++)d=a[o]=(""+a[o]).replace(/\"/g,"""),c=d,d.indexOf(g.filter_selectSourceSeparator)>=0&&(f=d.split(g.filter_selectSourceSeparator),c=f[0],d=f[1]),b+=""!==a[o]?"<option "+(c===d?"":'data-function-name="'+a[o]+'" ')+'value="'+c+'">'+d+"</option>":"";a=[]}u=(p.$filters?p.$filters:p.$table.children("thead")).find("."+r.filter),g.filter_$externalFilters&&(u=u&&u.length?u.add(g.filter_$externalFilters):g.filter_$externalFilters),h=u.filter('select[data-column="'+l+'"]'),h.length&&(h[s?"html":"append"](b),e.isArray(a)||h.append(a).val(y),h.val(y))}},buildDefault:function(e,r){var i,l,a,s=e.config,n=s.widgetOptions,o=s.columns;for(i=0;o>i;i++)l=s.$headerIndexed[i],a=!(l.hasClass("filter-false")||l.hasClass("parser-false")),(l.hasClass("filter-select")||t.getColumnData(e,n.filter_functions,i)===!0)&&a&&t.filter.buildSelect(e,i,"",r,l.hasClass(n.filter_onlyAvail))}},t.getFilters=function(i,l,a,s){var n,o,c,d,f=!1,u=i?e(i)[0].config:"",h=u?u.widgetOptions:"";if(l!==!0&&h&&!h.filter_columnFilters||e.isArray(a)&&a.join("")===u.lastCombinedFilter)return e(i).data("lastSearch");if(u&&(u.$filters&&(o=u.$filters.find("."+r.filter)),h.filter_$externalFilters&&(o=o&&o.length?o.add(h.filter_$externalFilters):h.filter_$externalFilters),o&&o.length))for(f=a||[],n=0;n<u.columns+1;n++)d=n===u.columns?h.filter_anyColumnSelector+","+h.filter_multipleColumnSelector:'[data-column="'+n+'"]',c=o.filter(d),c.length&&(c=t.filter.getLatestSearch(c),e.isArray(a)?(s&&c.length>1&&(c=c.slice(1)),n===u.columns&&(d=c.filter(h.filter_anyColumnSelector),c=d.length?d:c),c.val(a[n]).trigger("change.tsfilter")):(f[n]=c.val()||"",n===u.columns?c.slice(1).filter('[data-column*="'+c.attr("data-column")+'"]').val(f[n]):c.slice(1).val(f[n])),n===u.columns&&c.length&&(h.filter_$anyMatch=c)); return 0===f.length&&(f=!1),f},t.setFilters=function(r,i,l,a){var s=r?e(r)[0].config:"",n=t.getFilters(r,!0,i,a);return s&&l&&(s.lastCombinedFilter=null,s.lastSearch=[],t.filter.searching(s.table,i,a),s.$table.trigger("filterFomatterUpdate")),!!n}}(jQuery),function(e,t){"use strict";var r=e.tablesorter||{};e.extend(r.css,{sticky:"tablesorter-stickyHeader",stickyVis:"tablesorter-sticky-visible",stickyHide:"tablesorter-sticky-hidden",stickyWrap:"tablesorter-sticky-wrapper"}),r.addHeaderResizeEvent=function(t,r,i){if(t=e(t)[0],t.config){var l={timer:250},a=e.extend({},l,i),s=t.config,n=s.widgetOptions,o=function(e){var t,r,i,l,a,o,c=s.$headers.length;for(n.resize_flag=!0,r=[],t=0;c>t;t++)i=s.$headers.eq(t),l=i.data("savedSizes")||[0,0],a=i[0].offsetWidth,o=i[0].offsetHeight,(a!==l[0]||o!==l[1])&&(i.data("savedSizes",[a,o]),r.push(i[0]));r.length&&e!==!1&&s.$table.trigger("resize",[r]),n.resize_flag=!1};return o(!1),clearInterval(n.resize_timer),r?(n.resize_flag=!1,!1):void(n.resize_timer=setInterval(function(){n.resize_flag||o()},a.timer))}},r.addWidget({id:"stickyHeaders",priority:60,options:{stickyHeaders:"",stickyHeaders_attachTo:null,stickyHeaders_xScroll:null,stickyHeaders_yScroll:null,stickyHeaders_offset:0,stickyHeaders_filteredToTop:!0,stickyHeaders_cloneId:"-sticky",stickyHeaders_addResizeEvent:!0,stickyHeaders_includeCaption:!0,stickyHeaders_zIndex:2},format:function(i,l,a){if(!(l.$table.hasClass("hasStickyHeaders")||e.inArray("filter",l.widgets)>=0&&!l.$table.hasClass("hasFilters"))){var s,n,o,c,d=l.$table,f=e(a.stickyHeaders_attachTo),u=l.namespace+"stickyheaders ",h=e(a.stickyHeaders_yScroll||a.stickyHeaders_attachTo||t),p=e(a.stickyHeaders_xScroll||a.stickyHeaders_attachTo||t),g=d.children("thead:first"),m=g.children("tr").not(".sticky-false").children(),b=d.children("tfoot"),y=isNaN(a.stickyHeaders_offset)?e(a.stickyHeaders_offset):"",_=y.length?y.height()||0:parseInt(a.stickyHeaders_offset,10)||0,v=d.parent().closest("."+r.css.table).hasClass("hasStickyHeaders")?d.parent().closest("table.tablesorter")[0].config.widgetOptions.$sticky.parent():[],w=v.length?v.height():0,x=a.$sticky=d.clone().addClass("containsStickyHeaders "+r.css.sticky+" "+a.stickyHeaders+" "+l.namespace.slice(1)+"_extra_table").wrap('<div class="'+r.css.stickyWrap+'">'),C=x.parent().addClass(r.css.stickyHide).css({position:f.length?"absolute":"fixed",padding:parseInt(x.parent().parent().css("padding-left"),10),top:_+w,left:0,visibility:"hidden",zIndex:a.stickyHeaders_zIndex||2}),z=x.children("thead:first"),S="",$=0,F=function(e,r){var i,l,a,s,n,o=e.filter(":visible"),c=o.length;for(i=0;c>i;i++)s=r.filter(":visible").eq(i),n=o.eq(i),"border-box"===n.css("box-sizing")?l=n.outerWidth():"collapse"===s.css("border-collapse")?t.getComputedStyle?l=parseFloat(t.getComputedStyle(n[0],null).width):(a=parseFloat(n.css("border-width")),l=n.outerWidth()-parseFloat(n.css("padding-left"))-parseFloat(n.css("padding-right"))-a):l=n.width(),s.css({width:l,"min-width":l,"max-width":l})},R=function(){_=y.length?y.height()||0:parseInt(a.stickyHeaders_offset,10)||0,$=0,C.css({left:f.length?parseInt(f.css("padding-left"),10)||0:d.offset().left-parseInt(d.css("margin-left"),10)-p.scrollLeft()-$,width:d.outerWidth()}),F(d,x),F(m,c)},k=function(t){if(d.is(":visible")){w=v.length?v.offset().top-h.scrollTop()+v.height():0;var i=d.offset(),l=e.isWindow(h[0]),a=e.isWindow(p[0]),s=(f.length?l?h.scrollTop():h.offset().top:h.scrollTop())+_+w,n=d.height()-(C.height()+(b.height()||0)),o=s>i.top&&s<i.top+n?"visible":"hidden",c={visibility:o};f.length&&(c.top=l?s-f.offset().top:f.scrollTop()),a&&(c.left=d.offset().left-parseInt(d.css("margin-left"),10)-p.scrollLeft()-$),v.length&&(c.top=(c.top||0)+_+w),C.removeClass(r.css.stickyVis+" "+r.css.stickyHide).addClass("visible"===o?r.css.stickyVis:r.css.stickyHide).css(c),(o!==S||t)&&(R(),S=o)}};if(f.length&&!f.css("position")&&f.css("position","relative"),x.attr("id")&&(x[0].id+=a.stickyHeaders_cloneId),x.find("thead:gt(0), tr.sticky-false").hide(),x.find("tbody, tfoot").remove(),x.find("caption").toggle(a.stickyHeaders_includeCaption),c=z.children().children(),x.css({height:0,width:0,margin:0}),c.find("."+r.css.resizer).remove(),d.addClass("hasStickyHeaders").bind("pagerComplete"+u,function(){R()}),r.bindEvents(i,z.children().children("."+r.css.header)),d.after(C),l.onRenderHeader)for(o=z.children("tr").children(),n=o.length,s=0;n>s;s++)l.onRenderHeader.apply(o.eq(s),[s,l,x]);p.add(h).unbind("scroll resize ".split(" ").join(u).replace(/\s+/g," ")).bind("scroll resize ".split(" ").join(u),function(e){k("resize"===e.type)}),l.$table.unbind("stickyHeadersUpdate"+u).bind("stickyHeadersUpdate"+u,function(){k(!0)}),a.stickyHeaders_addResizeEvent&&r.addHeaderResizeEvent(i),d.hasClass("hasFilters")&&a.filter_columnFilters&&(d.bind("filterEnd"+u,function(){var i=e(document.activeElement).closest("td"),s=i.parent().children().index(i);C.hasClass(r.css.stickyVis)&&a.stickyHeaders_filteredToTop&&(t.scrollTo(0,d.position().top),s>=0&&l.$filters&&l.$filters.eq(s).find("a, select, input").filter(":visible").focus())}),r.filter.bindSearch(d,c.find("."+r.css.filter)),a.filter_hideFilters&&r.filter.hideFilters(x,l)),d.trigger("stickyHeadersInit")}},remove:function(i,l,a){var s=l.namespace+"stickyheaders ";l.$table.removeClass("hasStickyHeaders").unbind("pagerComplete filterEnd stickyHeadersUpdate ".split(" ").join(s).replace(/\s+/g," ")).next("."+r.css.stickyWrap).remove(),a.$sticky&&a.$sticky.length&&a.$sticky.remove(),e(t).add(a.stickyHeaders_xScroll).add(a.stickyHeaders_yScroll).add(a.stickyHeaders_attachTo).unbind("scroll resize ".split(" ").join(s).replace(/\s+/g," ")),r.addHeaderResizeEvent(i,!1)}})}(jQuery,window),function(e,t){"use strict";var r=e.tablesorter||{};e.extend(r.css,{resizableContainer:"tablesorter-resizable-container",resizableHandle:"tablesorter-resizable-handle",resizableNoSelect:"tablesorter-disableSelection",resizableStorage:"tablesorter-resizable"}),e(function(){var t="<style>body."+r.css.resizableNoSelect+" { -ms-user-select: none; -moz-user-select: -moz-none;-khtml-user-select: none; -webkit-user-select: none; user-select: none; }."+r.css.resizableContainer+" { position: relative; height: 1px; }."+r.css.resizableHandle+" { position: absolute; display: inline-block; width: 8px;top: 1px; cursor: ew-resize; z-index: 3; user-select: none; -moz-user-select: none; }</style>";e(t).appendTo("body")}),r.resizable={init:function(t,i){if(!t.$table.hasClass("hasResizable")){t.$table.addClass("hasResizable");var l,a,s,n,o,c=t.$table,d=c.parent(),f=parseInt(c.css("margin-top"),10),u=i.resizable_={useStorage:r.storage&&i.resizable!==!1,$wrap:d,mouseXPosition:0,$target:null,$next:null,overflow:"auto"===d.css("overflow")||"scroll"===d.css("overflow")||"auto"===d.css("overflow-x")||"scroll"===d.css("overflow-x"),storedSizes:[]};for(r.resizableReset(t.table,!0),u.tableWidth=c.width(),u.fullWidth=Math.abs(d.width()-u.tableWidth)<20,u.useStorage&&u.overflow&&(r.storage(t.table,"tablesorter-table-original-css-width",u.tableWidth),o=r.storage(t.table,"tablesorter-table-resized-width")||"auto",r.resizable.setWidth(c,o,!0)),i.resizable_.storedSizes=n=(u.useStorage?r.storage(t.table,r.css.resizableStorage):[])||[],r.resizable.setWidths(t,i,n),r.resizable.updateStoredSizes(t,i),i.$resizable_container=e('<div class="'+r.css.resizableContainer+'">').css({top:f}).insertBefore(c),s=0;s<t.columns;s++)a=t.$headerIndexed[s],o=r.getColumnData(t.table,t.headers,s),l="false"===r.getData(a,o,"resizable"),l||e('<div class="'+r.css.resizableHandle+'">').appendTo(i.$resizable_container).attr({"data-column":s,unselectable:"on"}).data("header",a).bind("selectstart",!1);c.one("tablesorter-initialized",function(){r.resizable.setHandlePosition(t,i),r.resizable.bindings(this.config,this.config.widgetOptions)})}},updateStoredSizes:function(e,t){var r,i,l=e.columns,a=t.resizable_;for(a.storedSizes=[],r=0;l>r;r++)i=e.$headerIndexed[r],a.storedSizes[r]=i.is(":visible")?i.width():0},setWidth:function(e,t,r){e.css({width:t,"min-width":r?t:"","max-width":r?t:""})},setWidths:function(t,i,l){var a,s,n=i.resizable_,o=e(t.namespace+"_extra_headers"),c=t.$table.children("colgroup").children("col");if(l=l||n.storedSizes||[],l.length){for(a=0;a<t.columns;a++)r.resizable.setWidth(t.$headerIndexed[a],l[a],n.overflow),o.length&&(s=o.eq(a).add(c.eq(a)),r.resizable.setWidth(s,l[a],n.overflow));s=e(t.namespace+"_extra_table"),s.length&&!r.hasWidget(t.table,"scroller")&&r.resizable.setWidth(s,t.$table.outerWidth(),n.overflow)}},setHandlePosition:function(t,i){var l,a=r.hasWidget(t.table,"scroller"),s=t.$table.height(),n=i.$resizable_container.children(),o=Math.floor(n.width()/2);a&&(s=0,t.$table.closest("."+r.css.scrollerWrap).children().each(function(){var t=e(this);s+=t.filter('[style*="height"]').length?t.height():t.children("table").height()})),l=t.$table.position().left,n.each(function(){var r=e(this),a=parseInt(r.attr("data-column"),10),n=t.columns-1,c=r.data("header");c&&(c.is(":visible")?(n>a||a===n&&i.resizable_addLastColumn)&&r.css({display:"inline-block",height:s,left:c.position().left-l+c.outerWidth()-o}):r.hide())})},toggleTextSelection:function(t,i){var l=t.namespace+"tsresize";t.widgetOptions.resizable_.disabled=i,e("body").toggleClass(r.css.resizableNoSelect,i),i?e("body").attr("unselectable","on").bind("selectstart"+l,!1):e("body").removeAttr("unselectable").unbind("selectstart"+l)},bindings:function(i,l){var a=i.namespace+"tsresize";l.$resizable_container.children().bind("mousedown",function(t){var a,s=l.resizable_,n=e(i.namespace+"_extra_headers"),o=e(t.target).data("header");a=parseInt(o.attr("data-column"),10),s.$target=o=o.add(n.filter('[data-column="'+a+'"]')),s.target=a,s.$next=t.shiftKey||l.resizable_targetLast?o.parent().children().not(".resizable-false").filter(":last"):o.nextAll(":not(.resizable-false)").eq(0),a=parseInt(s.$next.attr("data-column"),10),s.$next=s.$next.add(n.filter('[data-column="'+a+'"]')),s.next=a,s.mouseXPosition=t.pageX,r.resizable.updateStoredSizes(i,l),r.resizable.toggleTextSelection(i,!0)}),e(document).bind("mousemove"+a,function(e){var t=l.resizable_;t.disabled&&0!==t.mouseXPosition&&t.$target&&(l.resizable_throttle?(clearTimeout(t.timer),t.timer=setTimeout(function(){r.resizable.mouseMove(i,l,e)},isNaN(l.resizable_throttle)?5:l.resizable_throttle)):r.resizable.mouseMove(i,l,e))}).bind("mouseup"+a,function(){l.resizable_.disabled&&(r.resizable.toggleTextSelection(i,!1),r.resizable.stopResize(i,l),r.resizable.setHandlePosition(i,l))}),e(t).bind("resize"+a+" resizeEnd"+a,function(){r.resizable.setHandlePosition(i,l)}),i.$table.bind("columnUpdate"+a,function(){r.resizable.setHandlePosition(i,l)}).find("thead:first").add(e(i.namespace+"_extra_table").find("thead:first")).bind("contextmenu"+a,function(){var e=0===l.resizable_.storedSizes.length;return r.resizableReset(i.table),r.resizable.setHandlePosition(i,l),l.resizable_.storedSizes=[],e})},mouseMove:function(t,i,l){if(0!==i.resizable_.mouseXPosition&&i.resizable_.$target){var a,s=0,n=i.resizable_,o=n.$next,c=n.storedSizes[n.target],d=l.pageX-n.mouseXPosition;if(n.overflow){if(c+d>0){for(n.storedSizes[n.target]+=d,r.resizable.setWidth(n.$target,n.storedSizes[n.target],!0),a=0;a<t.columns;a++)s+=n.storedSizes[a];r.resizable.setWidth(t.$table.add(e(t.namespace+"_extra_table")),s)}o.length||(n.$wrap[0].scrollLeft=t.$table.width())}else n.fullWidth?(n.storedSizes[n.target]+=d,n.storedSizes[n.next]-=d,r.resizable.setWidths(t,i)):(n.storedSizes[n.target]+=d,r.resizable.setWidths(t,i));n.mouseXPosition=l.pageX,t.$table.trigger("stickyHeadersUpdate")}},stopResize:function(e,t){var i=t.resizable_;r.resizable.updateStoredSizes(e,t),i.useStorage&&(r.storage(e.table,r.css.resizableStorage,i.storedSizes),r.storage(e.table,"tablesorter-table-resized-width",e.$table.width())),i.mouseXPosition=0,i.$target=i.$next=null,e.$table.trigger("stickyHeadersUpdate")}},r.addWidget({id:"resizable",priority:40,options:{resizable:!0,resizable_addLastColumn:!1,resizable_widths:[],resizable_throttle:!1,resizable_targetLast:!1,resizable_fullWidth:null},init:function(e,t,i,l){r.resizable.init(i,l)},remove:function(t,i,l,a){if(l.$resizable_container){var s=i.namespace+"tsresize";i.$table.add(e(i.namespace+"_extra_table")).removeClass("hasResizable").children("thead").unbind("contextmenu"+s),l.$resizable_container.remove(),r.resizable.toggleTextSelection(i,!1),r.resizableReset(t,a),e(document).unbind("mousemove"+s+" mouseup"+s)}}}),r.resizableReset=function(t,i){e(t).each(function(){var e,l,a=this.config,s=a&&a.widgetOptions,n=s.resizable_;if(t&&a&&a.$headerIndexed.length){for(n.overflow&&n.tableWidth&&(r.resizable.setWidth(a.$table,n.tableWidth,!0),n.useStorage&&r.storage(t,"tablesorter-table-resized-width","auto")),e=0;e<a.columns;e++)l=a.$headerIndexed[e],s.resizable_widths&&s.resizable_widths[e]?r.resizable.setWidth(l,s.resizable_widths[e],n.overflow):l.hasClass("resizable-false")||r.resizable.setWidth(l,"",n.overflow);a.$table.trigger("stickyHeadersUpdate"),r.storage&&!i&&r.storage(this,r.css.resizableStorage,{})}})}}(jQuery,window),function(e){"use strict";var t=e.tablesorter||{};t.addWidget({id:"saveSort",priority:20,options:{saveSort:!0},init:function(e,t,r,i){t.format(e,r,i,!0)},format:function(r,i,l,a){var s,n,o=i.$table,c=l.saveSort!==!1,d={sortList:i.sortList};i.debug&&(n=new Date),o.hasClass("hasSaveSort")?c&&r.hasInitialized&&t.storage&&(t.storage(r,"tablesorter-savesort",d),i.debug&&t.benchmark("saveSort widget: Saving last sort: "+i.sortList,n)):(o.addClass("hasSaveSort"),d="",t.storage&&(s=t.storage(r,"tablesorter-savesort"),d=s&&s.hasOwnProperty("sortList")&&e.isArray(s.sortList)?s.sortList:"",i.debug&&t.benchmark('saveSort: Last sort loaded: "'+d+'"',n),o.bind("saveSortReset",function(e){e.stopPropagation(),t.storage(r,"tablesorter-savesort","")})),a&&d&&d.length>0?i.sortList=d:r.hasInitialized&&d&&d.length>0&&o.trigger("sorton",[d]))},remove:function(e,r){r.$table.removeClass("hasSaveSort"),t.storage&&t.storage(e,"tablesorter-savesort","")}})}(jQuery),e.tablesorter});
/* custom */ $(document).ready(function(){$('.table-sortable').tablesorter({theme:'bootstrap',widgets:['uitheme','zebra','filter','stickyHeaders','saveSort'],headerTemplate:'{content} {icon}',widthFixed:true,ignoreCase:true,widgetOptions:{filter_columnFilters:true,zebra:['even','odd'],filter_reset:'.reset'}});});
\ No newline at end of file diff --git a/library/cpp/monlib/service/pages/tablesorter/ya.make b/library/cpp/monlib/service/pages/tablesorter/ya.make new file mode 100644 index 0000000000..b5b6a64da8 --- /dev/null +++ b/library/cpp/monlib/service/pages/tablesorter/ya.make @@ -0,0 +1,14 @@ +LIBRARY() + +OWNER(blinkov) + +RESOURCE( + resources/jquery.tablesorter.css jquery.tablesorter.css + resources/jquery.tablesorter.js jquery.tablesorter.js +) + +PEERDIR( + library/cpp/monlib/dynamic_counters +) + +END() diff --git a/library/cpp/monlib/service/pages/templates.cpp b/library/cpp/monlib/service/pages/templates.cpp new file mode 100644 index 0000000000..ece12bea71 --- /dev/null +++ b/library/cpp/monlib/service/pages/templates.cpp @@ -0,0 +1,35 @@ +#include "templates.h" + +namespace NMonitoring { + extern const char HtmlTag[] = "html"; + extern const char HeadTag[] = "head"; + extern const char BodyTag[] = "body"; + extern const char DivTag[] = "div"; + extern const char TableTag[] = "table"; + extern const char TableHeadTag[] = "thead"; + extern const char TableBodyTag[] = "tbody"; + extern const char TableRTag[] = "tr"; + extern const char TableDTag[] = "td"; + extern const char TableHTag[] = "th"; + extern const char FormTag[] = "form"; + extern const char LabelTag[] = "label"; + extern const char SpanTag[] = "span"; + extern const char CaptionTag[] = "caption"; + extern const char PreTag[] = "pre"; + extern const char ParaTag[] = "p"; + extern const char H1Tag[] = "h1"; + extern const char H2Tag[] = "h2"; + extern const char H3Tag[] = "h3"; + extern const char H4Tag[] = "h4"; + extern const char H5Tag[] = "h5"; + extern const char H6Tag[] = "h6"; + extern const char SmallTag[] = "small"; + extern const char StrongTag[] = "strong"; + extern const char ListTag[] = "li"; + extern const char UListTag[] = "ul"; + extern const char OListTag[] = "ol"; + extern const char DListTag[] = "dl"; + extern const char DTermTag[] = "dt"; + extern const char DDescTag[] = "dd"; + +} diff --git a/library/cpp/monlib/service/pages/templates.h b/library/cpp/monlib/service/pages/templates.h new file mode 100644 index 0000000000..b4656f059f --- /dev/null +++ b/library/cpp/monlib/service/pages/templates.h @@ -0,0 +1,268 @@ +#pragma once + +#include <util/stream/output.h> +#include <util/system/defaults.h> + +#define WITH_SCOPED(var, value) WITH_SCOPED_I(var, value, Y_GENERATE_UNIQUE_ID(WITH_SCOPED_LABEL_)) + +#define WITH_SCOPED_I(var, value, label) \ + if (auto var = (value)) { \ + Y_UNUSED(var); \ + goto label; \ + } else \ + label \ + : + +#define TAG(name) WITH_SCOPED(tmp, NMonitoring::name(__stream)) +#define TAG_CLASS(name, cls) WITH_SCOPED(tmp, NMonitoring::name(__stream, cls)) +#define TAG_CLASS_STYLE(name, cls, style) WITH_SCOPED(tmp, NMonitoring::name(__stream, {{"class", cls}, {"style", style}})) +#define TAG_CLASS_ID(name, cls, id) WITH_SCOPED(tmp, NMonitoring::name(__stream, cls, "", id)) +#define TAG_CLASS_FOR(name, cls, for0) WITH_SCOPED(tmp, NMonitoring::name(__stream, cls, for0)) +#define TAG_ATTRS(name, ...) WITH_SCOPED(tmp, NMonitoring::name(__stream, ##__VA_ARGS__)) + +#define HTML(str) WITH_SCOPED(__stream, NMonitoring::TOutputStreamRef(str)) + +#define HEAD() TAG(THead) +#define BODY() TAG(TBody) +#define HTML_TAG() TAG(THtml) +#define DIV() TAG(TDiv) +#define DIV_CLASS(cls) TAG_CLASS(TDiv, cls) +#define DIV_CLASS_ID(cls, id) TAG_CLASS_ID(TDiv, cls, id) +#define PRE() TAG(TPre) +#define TABLE() TAG(TTable) +#define TABLE_CLASS(cls) TAG_CLASS(TTable, cls) +#define TABLE_SORTABLE() TABLE_CLASS("table-sortable") +#define TABLE_SORTABLE_CLASS(cls) TABLE_CLASS(cls " table-sortable") +#define TABLEHEAD() TAG(TTableHead) +#define TABLEHEAD_CLASS(cls) TAG_CLASS(TTableHead, cls) +#define TABLEBODY() TAG(TTableBody) +#define TABLEBODY_CLASS(cls) TAG_CLASS(TTableBody, cls) +#define TABLER() TAG(TTableR) +#define TABLER_CLASS(cls) TAG_CLASS(TTableR, cls) +#define TABLED() TAG(TTableD) +#define TABLED_CLASS(cls) TAG_CLASS(TTableD, cls) +#define TABLED_ATTRS(...) TAG_ATTRS(TTableD, ##__VA_ARGS__) +#define TABLEH() TAG(TTableH) +#define TABLEH_CLASS(cls) TAG_CLASS(TTableH, cls) +#define FORM() TAG(TFormC) +#define FORM_CLASS(cls) TAG_CLASS(TFormC, cls) +#define LABEL() TAG(TLabelC) +#define LABEL_CLASS(cls) TAG_CLASS(TLabelC, cls) +#define LABEL_CLASS_FOR(cls, for0) TAG_CLASS_FOR(TLabelC, cls, for0) +#define SPAN_CLASS(cls) TAG_CLASS(TSpanC, cls) +#define SPAN_CLASS_STYLE(cls, style) TAG_CLASS_STYLE(TSpanC, cls, style) + +#define PARA() TAG(TPara) +#define PARA_CLASS(cls) TAG_CLASS(TPara, cls) + +#define H1() TAG(TH1) +#define H1_CLASS(cls) TAG_CLASS(TH1, cls) +#define H2() TAG(TH2) +#define H2_CLASS(cls) TAG_CLASS(TH2, cls) +#define H3() TAG(TH3) +#define H3_CLASS(cls) TAG_CLASS(TH3, cls) +#define H4() TAG(TH4) +#define H4_CLASS(cls) TAG_CLASS(TH4, cls) +#define H5() TAG(TH5) +#define H5_CLASS(cls) TAG_CLASS(TH5, cls) +#define H6() TAG(TH6) +#define H6_CLASS(cls) TAG_CLASS(TH6, cls) + +#define SMALL() TAG(TSMALL) +#define STRONG() TAG(TSTRONG) + +#define LI() TAG(TLIST) +#define LI_CLASS(cls) TAG_CLASS(TLIST, cls) +#define UL() TAG(TULIST) +#define UL_CLASS(cls) TAG_CLASS(TULIST, cls) +#define OL() TAG(TOLIST) +#define OL_CLASS(cls) TAG_CLASS(TOLIST, cls) + +#define DL() TAG(DLIST) +#define DL_CLASS(cls) TAG_CLASS(DLIST, cls) +#define DT() TAG(DTERM) +#define DT_CLASS(cls) TAG_CLASS(DTERM, cls) +#define DD() TAG(DDESC) +#define DD_CLASS(cls) TAG_CLASS(DDESC, cls) + +#define CAPTION() TAG(TCaption) +#define CAPTION_CLASS(cls) CAPTION_CLASS(TCaption, cls) + +#define HTML_OUTPUT_PARAM(str, param) str << #param << ": " << param << "<br/>" +#define HTML_OUTPUT_TIME_PARAM(str, param) str << #param << ": " << ToStringLocalTimeUpToSeconds(param) << "<br/>" + +#define COLLAPSED_BUTTON_CONTENT(targetId, buttonText) \ + WITH_SCOPED(tmp, NMonitoring::TCollapsedButton(__stream, targetId, buttonText)) + +#define HREF(path) \ + WITH_SCOPED(tmp, NMonitoring::THref(__stream, path)) + +namespace NMonitoring { + struct THref { + THref(IOutputStream& str, TStringBuf path) + : Str(str) + { + Str << "<a href="<< path << '>'; + } + + ~THref() { + Str << "</a>"; + } + + explicit inline operator bool() const noexcept { + return true; // just to work with WITH_SCOPED + } + + IOutputStream& Str; + }; + + template <const char* tag> + struct TTag { + TTag(IOutputStream& str, TStringBuf cls = "", TStringBuf for0 = "", TStringBuf id = "") + : Str(str) + { + Str << "<" << tag; + + if (!cls.empty()) { + Str << " class=\"" << cls << "\""; + } + + if (!for0.empty()) { + Str << " for=\"" << for0 << "\""; + } + + if (!id.empty()) { + Str << "id=\"" << id << "\""; + } + Str << ">"; + } + + TTag(IOutputStream& str, std::initializer_list<std::pair<TStringBuf, TStringBuf>> attributes) + : Str(str) + { + Str << "<" << tag; + for (const std::pair<TStringBuf, TStringBuf>& attr : attributes) { + if (!attr.second.empty()) { + Str << ' ' << attr.first << "=\"" << attr.second << "\""; + } + } + Str << ">"; + } + + ~TTag() { + try { + Str << "</" << tag << ">"; + } catch (...) { + } + } + + explicit inline operator bool() const noexcept { + return true; // just to work with WITH_SCOPED + } + + IOutputStream& Str; + }; + + // a nice class for creating collapsable regions of html output + struct TCollapsedButton { + TCollapsedButton(IOutputStream& str, const TString& targetId, const TString& buttonText) + : Str(str) + { + Str << "<button type='button' class='btn' data-toggle='collapse' data-target='#" << targetId << "'>" + << buttonText << "</button>"; + Str << "<div id='" << targetId << "' class='collapse'>"; + } + + ~TCollapsedButton() { + try { + Str << "</div>"; + } catch (...) { + } + } + + explicit inline operator bool() const noexcept { + return true; // just to work with WITH_SCOPED + } + + IOutputStream& Str; + }; + + struct TOutputStreamRef { + TOutputStreamRef(IOutputStream& str) + : Str(str) + { + } + + inline operator IOutputStream&() noexcept { + return Str; + } + + explicit inline operator bool() const noexcept { + return true; // just to work with WITH_SCOPED + } + + IOutputStream& Str; + }; + + extern const char HtmlTag[5]; + extern const char HeadTag[5]; + extern const char BodyTag[5]; + extern const char DivTag[4]; + extern const char TableTag[6]; + extern const char TableHeadTag[6]; + extern const char TableBodyTag[6]; + extern const char TableRTag[3]; + extern const char TableDTag[3]; + extern const char TableHTag[3]; + extern const char FormTag[5]; + extern const char LabelTag[6]; + extern const char SpanTag[5]; + extern const char CaptionTag[8]; + extern const char PreTag[4]; + extern const char ParaTag[2]; + extern const char H1Tag[3]; + extern const char H2Tag[3]; + extern const char H3Tag[3]; + extern const char H4Tag[3]; + extern const char H5Tag[3]; + extern const char H6Tag[3]; + extern const char SmallTag[6]; + extern const char StrongTag[7]; + extern const char ListTag[3]; + extern const char UListTag[3]; + extern const char OListTag[3]; + extern const char DListTag[3]; + extern const char DTermTag[3]; + extern const char DDescTag[3]; + + typedef TTag<HtmlTag> THtml; + typedef TTag<HeadTag> THead; + typedef TTag<BodyTag> TBody; + typedef TTag<DivTag> TDiv; + typedef TTag<TableTag> TTable; + typedef TTag<TableHeadTag> TTableHead; + typedef TTag<TableBodyTag> TTableBody; + typedef TTag<TableRTag> TTableR; + typedef TTag<TableDTag> TTableD; + typedef TTag<TableHTag> TTableH; + typedef TTag<FormTag> TFormC; + typedef TTag<LabelTag> TLabelC; + typedef TTag<SpanTag> TSpanC; + typedef TTag<CaptionTag> TCaption; + typedef TTag<PreTag> TPre; + typedef TTag<ParaTag> TPara; + typedef TTag<H1Tag> TH1; + typedef TTag<H2Tag> TH2; + typedef TTag<H3Tag> TH3; + typedef TTag<H4Tag> TH4; + typedef TTag<H5Tag> TH5; + typedef TTag<H6Tag> TH6; + typedef TTag<SmallTag> TSMALL; + typedef TTag<StrongTag> TSTRONG; + typedef TTag<ListTag> TLIST; + typedef TTag<UListTag> TULIST; + typedef TTag<OListTag> TOLIST; + typedef TTag<DListTag> DLIST; + typedef TTag<DTermTag> DTERM; + typedef TTag<DDescTag> DDESC; +} diff --git a/library/cpp/monlib/service/pages/version_mon_page.cpp b/library/cpp/monlib/service/pages/version_mon_page.cpp new file mode 100644 index 0000000000..41e29417da --- /dev/null +++ b/library/cpp/monlib/service/pages/version_mon_page.cpp @@ -0,0 +1,16 @@ +#include <library/cpp/svnversion/svnversion.h> +#include <library/cpp/build_info/build_info.h> +#include <library/cpp/malloc/api/malloc.h> + +#include "version_mon_page.h" + +using namespace NMonitoring; + +void TVersionMonPage::OutputText(IOutputStream& out, NMonitoring::IMonHttpRequest&) { + const char* version = GetProgramSvnVersion(); + out << version; + if (!TString(version).EndsWith("\n")) + out << "\n"; + out << GetBuildInfo() << "\n\n"; + out << "linked with malloc: " << NMalloc::MallocInfo().Name << "\n"; +} diff --git a/library/cpp/monlib/service/pages/version_mon_page.h b/library/cpp/monlib/service/pages/version_mon_page.h new file mode 100644 index 0000000000..f7649947e4 --- /dev/null +++ b/library/cpp/monlib/service/pages/version_mon_page.h @@ -0,0 +1,15 @@ +#pragma once + +#include "pre_mon_page.h" + +namespace NMonitoring { + struct TVersionMonPage: public TPreMonPage { + TVersionMonPage(const TString& path = "ver", const TString& title = "Version") + : TPreMonPage(path, title) + { + } + + void OutputText(IOutputStream& out, NMonitoring::IMonHttpRequest&) override; + }; + +} diff --git a/library/cpp/monlib/service/pages/ya.make b/library/cpp/monlib/service/pages/ya.make new file mode 100644 index 0000000000..48d44a0838 --- /dev/null +++ b/library/cpp/monlib/service/pages/ya.make @@ -0,0 +1,31 @@ +LIBRARY() + +OWNER(g:solomon) + +NO_WSHADOW() + +SRCS( + diag_mon_page.cpp + html_mon_page.cpp + index_mon_page.cpp + mon_page.cpp + pre_mon_page.cpp + resource_mon_page.cpp + templates.cpp + version_mon_page.cpp + registry_mon_page.cpp +) + +PEERDIR( + library/cpp/build_info + library/cpp/malloc/api + library/cpp/svnversion + library/cpp/resource + library/cpp/monlib/service + library/cpp/monlib/encode/json + library/cpp/monlib/encode/text + library/cpp/monlib/encode/spack + library/cpp/monlib/encode/prometheus +) + +END() diff --git a/library/cpp/monlib/service/service.cpp b/library/cpp/monlib/service/service.cpp new file mode 100644 index 0000000000..929efbf816 --- /dev/null +++ b/library/cpp/monlib/service/service.cpp @@ -0,0 +1,268 @@ +#include "service.h" + +#include <library/cpp/coroutine/engine/sockpool.h> +#include <library/cpp/http/io/stream.h> +#include <library/cpp/http/fetch/httpheader.h> +#include <library/cpp/http/fetch/httpfsm.h> +#include <library/cpp/uri/http_url.h> + +#include <util/generic/buffer.h> +#include <util/stream/str.h> +#include <util/stream/buffer.h> +#include <util/stream/zerocopy.h> +#include <util/string/vector.h> + +namespace NMonitoring { + class THttpClient: public IHttpRequest { + public: + void ServeRequest(THttpInput& in, IOutputStream& out, const NAddr::IRemoteAddr* remoteAddr, const THandler& Handler) { + try { + try { + RemoteAddr = remoteAddr; + THttpHeaderParser parser; + parser.Init(&Header); + if (parser.Execute(in.FirstLine().data(), in.FirstLine().size()) < 0) { + out << "HTTP/1.1 400 Bad request\r\nConnection: Close\r\n\r\n"; + return; + } + if (Url.Parse(Header.GetUrl().data()) != THttpURL::ParsedOK) { + out << "HTTP/1.1 400 Invalid url\r\nConnection: Close\r\n\r\n"; + return; + } + TString path = GetPath(); + if (!path.StartsWith('/')) { + out << "HTTP/1.1 400 Bad request\r\nConnection: Close\r\n\r\n"; + return; + } + Headers = &in.Headers(); + CgiParams.Scan(Url.Get(THttpURL::FieldQuery)); + } catch (...) { + out << "HTTP/1.1 500 Internal server error\r\nConnection: Close\r\n\r\n"; + YSYSLOG(TLOG_ERR, "THttpClient: internal error while serving monitoring request: %s", CurrentExceptionMessage().data()); + } + + if (Header.http_method == HTTP_METHOD_POST) + TransferData(&in, &PostContent); + + Handler(out, *this); + out.Finish(); + } catch (...) { + auto msg = CurrentExceptionMessage(); + out << "HTTP/1.1 500 Internal server error\r\nConnection: Close\r\n\r\n" << msg; + out.Finish(); + YSYSLOG(TLOG_ERR, "THttpClient: error while serving monitoring request: %s", msg.data()); + } + } + + const char* GetURI() const override { + return Header.request_uri.c_str(); + } + const char* GetPath() const override { + return Url.Get(THttpURL::FieldPath); + } + const TCgiParameters& GetParams() const override { + return CgiParams; + } + const TCgiParameters& GetPostParams() const override { + if (PostParams.empty() && !PostContent.Buffer().Empty()) + const_cast<THttpClient*>(this)->ScanPostParams(); + return PostParams; + } + TStringBuf GetPostContent() const override { + return TStringBuf(PostContent.Buffer().Data(), PostContent.Buffer().Size()); + } + HTTP_METHOD GetMethod() const override { + return (HTTP_METHOD)Header.http_method; + } + void ScanPostParams() { + PostParams.Scan(TStringBuf(PostContent.Buffer().data(), PostContent.Buffer().size())); + } + + const THttpHeaders& GetHeaders() const override { + if (Headers != nullptr) { + return *Headers; + } + static THttpHeaders defaultHeaders; + return defaultHeaders; + } + + TString GetRemoteAddr() const override { + return RemoteAddr ? NAddr::PrintHostAndPort(*RemoteAddr) : TString(); + } + + private: + THttpRequestHeader Header; + const THttpHeaders* Headers = nullptr; + THttpURL Url; + TCgiParameters CgiParams; + TCgiParameters PostParams; + TBufferOutput PostContent; + const NAddr::IRemoteAddr* RemoteAddr = nullptr; + }; + + /* TCoHttpServer */ + + class TCoHttpServer::TConnection: public THttpClient { + public: + TConnection(const TCoHttpServer::TAcceptFull& acc, const TCoHttpServer& parent) + : Socket(acc.S->Release()) + , RemoteAddr(acc.Remote) + , Parent(parent) + { + } + + void operator()(TCont* c) { + try { + THolder<TConnection> me(this); + TContIO io(Socket, c); + THttpInput in(&io); + THttpOutput out(&io, &in); + // buffer reply so there will be ne context switching + TStringStream s; + ServeRequest(in, s, RemoteAddr, Parent.Handler); + out << s.Str(); + out.Finish(); + } catch (...) { + YSYSLOG(TLOG_WARNING, "TCoHttpServer::TConnection: error: %s\n", CurrentExceptionMessage().data()); + } + } + + private: + TSocketHolder Socket; + const NAddr::IRemoteAddr* RemoteAddr; + const TCoHttpServer& Parent; + }; + + TCoHttpServer::TCoHttpServer(TContExecutor& executor, const TString& bindAddr, TIpPort port, THandler handler) + : Executor(executor) + , Listener(this, &executor) + , Handler(std::move(handler)) + , BindAddr(bindAddr) + , Port(port) + { + try { + Listener.Bind(TIpAddress(bindAddr, port)); + } catch (yexception e) { + Y_FAIL("TCoHttpServer::TCoHttpServer: couldn't bind to %s:%d\n", bindAddr.data(), port); + } + } + + void TCoHttpServer::Start() { + Listener.Listen(); + } + + void TCoHttpServer::Stop() { + Listener.Stop(); + } + + void TCoHttpServer::OnAcceptFull(const TAcceptFull& acc) { + THolder<TConnection> conn(new TConnection(acc, *this)); + Executor.Create(*conn, "client"); + Y_UNUSED(conn.Release()); + } + + void TCoHttpServer::OnError() { + throw; // just rethrow + } + + void TCoHttpServer::ProcessRequest(IOutputStream& out, const IHttpRequest& request) { + try { + TNetworkAddress addr(BindAddr, Port); + TSocket sock(addr); + TSocketOutput sock_out(sock); + TSocketInput sock_in(sock); + sock_out << "GET " << request.GetURI() << " HTTP/1.0\r\n\r\n"; + THttpInput http_in(&sock_in); + try { + out << "HTTP/1.1 200 Ok\nConnection: Close\n\n"; + TransferData(&http_in, &out); + } catch (...) { + YSYSLOG(TLOG_DEBUG, "TCoHttpServer: while getting data from backend: %s", CurrentExceptionMessage().data()); + } + } catch (const yexception& /*e*/) { + out << "HTTP/1.1 500 Internal server error\nConnection: Close\n\n"; + YSYSLOG(TLOG_DEBUG, "TCoHttpServer: while getting data from backend: %s", CurrentExceptionMessage().data()); + } + } + + /* TMtHttpServer */ + + class TMtHttpServer::TConnection: public TClientRequest, public THttpClient { + public: + TConnection(const TMtHttpServer& parent) + : Parent(parent) + { + } + + bool Reply(void*) override { + ServeRequest(Input(), Output(), NAddr::GetPeerAddr(Socket()).Get(), Parent.Handler); + return true; + } + + private: + const TMtHttpServer& Parent; + }; + + TMtHttpServer::TMtHttpServer(const TOptions& options, THandler handler, IThreadFactory* pool) + : THttpServer(this, options, pool) + , Handler(std::move(handler)) + { + } + + TMtHttpServer::TMtHttpServer(const TOptions& options, THandler handler, TSimpleSharedPtr<IThreadPool> pool) + : THttpServer(this, /* mainWorkers = */pool, /* failWorkers = */pool, options) + , Handler(std::move(handler)) + { + } + + bool TMtHttpServer::Start() { + return THttpServer::Start(); + } + + void TMtHttpServer::StartOrThrow() { + if (!Start()) { + const auto& opts = THttpServer::Options(); + TNetworkAddress addr = opts.Host + ? TNetworkAddress(opts.Host, opts.Port) + : TNetworkAddress(opts.Port); + ythrow TSystemError(GetErrorCode()) << addr; + } + } + + void TMtHttpServer::Stop() { + THttpServer::Stop(); + } + + TClientRequest* TMtHttpServer::CreateClient() { + return new TConnection(*this); + } + + /* TService */ + + TMonService::TMonService(TContExecutor& executor, TIpPort internalPort, TIpPort externalPort, + THandler coHandler, THandler mtHandler) + : CoServer(executor, "127.0.0.1", internalPort, std::move(coHandler)) + , MtServer(THttpServerOptions(externalPort), std::bind(&TMonService::DispatchRequest, this, std::placeholders::_1, std::placeholders::_2)) + , MtHandler(std::move(mtHandler)) + { + } + + void TMonService::Start() { + MtServer.Start(); + CoServer.Start(); + } + + void TMonService::Stop() { + MtServer.Stop(); + CoServer.Stop(); + } + + void TMonService::DispatchRequest(IOutputStream& out, const IHttpRequest& request) { + if (strcmp(request.GetPath(), "/") == 0) { + out << "HTTP/1.1 200 Ok\nConnection: Close\n\n"; + MtHandler(out, request); + } else + CoServer.ProcessRequest(out, request); + } + +} diff --git a/library/cpp/monlib/service/service.h b/library/cpp/monlib/service/service.h new file mode 100644 index 0000000000..2f66dddaf8 --- /dev/null +++ b/library/cpp/monlib/service/service.h @@ -0,0 +1,112 @@ +#pragma once + +#include <library/cpp/coroutine/engine/impl.h> +#include <library/cpp/coroutine/listener/listen.h> +#include <library/cpp/http/fetch/httpheader.h> +#include <library/cpp/http/server/http.h> +#include <library/cpp/logger/all.h> + +#include <util/network/ip.h> +#include <library/cpp/cgiparam/cgiparam.h> + +#include <functional> + +struct TMonitor; + +namespace NMonitoring { + struct IHttpRequest { + virtual ~IHttpRequest() { + } + virtual const char* GetURI() const = 0; + virtual const char* GetPath() const = 0; + virtual const TCgiParameters& GetParams() const = 0; + virtual const TCgiParameters& GetPostParams() const = 0; + virtual TStringBuf GetPostContent() const = 0; + virtual HTTP_METHOD GetMethod() const = 0; + virtual const THttpHeaders& GetHeaders() const = 0; + virtual TString GetRemoteAddr() const = 0; + }; + // first param - output stream to write result to + // second param - URL of request + typedef std::function<void(IOutputStream&, const IHttpRequest&)> THandler; + + class TCoHttpServer: private TContListener::ICallBack { + public: + // initialize and schedule coroutines for execution + TCoHttpServer(TContExecutor& executor, const TString& bindAddr, TIpPort port, THandler handler); + void Start(); + void Stop(); + + // this function implements THandler interface + // by forwarding it to the httpserver + // @note this call may be blocking; don't use inside coroutines + // @throws may throw in case of connection error, etc + void ProcessRequest(IOutputStream&, const IHttpRequest&); + + private: + class TConnection; + + // ICallBack implementation + void OnAcceptFull(const TAcceptFull& a) override; + void OnError() override; + + private: + TContExecutor& Executor; + TContListener Listener; + THandler Handler; + TString BindAddr; + TIpPort Port; + }; + + class TMtHttpServer: public THttpServer, private THttpServer::ICallBack { + public: + TMtHttpServer(const TOptions& options, THandler handler, IThreadFactory* pool = nullptr); + TMtHttpServer(const TOptions& options, THandler handler, TSimpleSharedPtr<IThreadPool> pool); + + /** + * This will cause the server start to accept incoming connections. + * + * @return true if the port binding was successfull, + * false otherwise. + */ + bool Start(); + + /** + * Same as Start() member-function, but will throw TSystemError if + * there were some errors. + */ + void StartOrThrow(); + + /** + * Stops the server from accepting new connections. + */ + void Stop(); + + private: + class TConnection; + TClientRequest* CreateClient() override; + + THandler Handler; + }; + + // this class implements hybrid coroutine and threaded approach + // requests for main page which holds counters and simple tables are served in a thread + // requests for other pages which include access with inter-thread synchonization + // will be served in a coroutine context + class TMonService { + public: + TMonService(TContExecutor& executor, TIpPort internalPort, TIpPort externalPort, + THandler coHandler, THandler mtHandler); + void Start(); + void Stop(); + + protected: + void DispatchRequest(IOutputStream& out, const IHttpRequest&); + + private: + TCoHttpServer CoServer; + TMtHttpServer MtServer; + THandler MtHandler; + }; + +} diff --git a/library/cpp/monlib/service/ya.make b/library/cpp/monlib/service/ya.make new file mode 100644 index 0000000000..ad088fc2c6 --- /dev/null +++ b/library/cpp/monlib/service/ya.make @@ -0,0 +1,28 @@ +LIBRARY() + +OWNER(g:solomon) + +SRCS( + monservice.cpp + mon_service_http_request.cpp + service.cpp + format.cpp + auth.cpp +) + +PEERDIR( + library/cpp/string_utils/base64 + contrib/libs/protobuf + library/cpp/coroutine/engine + library/cpp/coroutine/listener + library/cpp/http/fetch + library/cpp/http/server + library/cpp/http/io + library/cpp/logger + library/cpp/malloc/api + library/cpp/svnversion + library/cpp/uri + library/cpp/cgiparam +) + +END() diff --git a/library/cpp/monlib/ya.make b/library/cpp/monlib/ya.make new file mode 100644 index 0000000000..9bd236d6fd --- /dev/null +++ b/library/cpp/monlib/ya.make @@ -0,0 +1,45 @@ +OWNER( + g:solomon + jamel +) + +RECURSE( + consumers + counters + counters/ut + deprecated + dynamic_counters + dynamic_counters/percentile + dynamic_counters/percentile/ut + dynamic_counters/ut + encode + encode/buffered + encode/buffered/ut + encode/fake + encode/fuzz + encode/json + encode/json/ut + encode/legacy_protobuf + encode/legacy_protobuf/ut + encode/prometheus + encode/prometheus/ut + encode/protobuf + encode/spack + encode/spack/ut + encode/text + encode/text/ut + encode/unistat + encode/unistat/ut + encode/ut + example + exception + libtimestats/ut + metrics + metrics/ut + messagebus + push_client + service + service/auth/tvm + service/pages + service/pages/tablesorter +) |