diff options
| author | Sergey Polovko <sergey@polovko.me> | 2022-02-10 16:47:03 +0300 |
|---|---|---|
| committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:47:03 +0300 |
| commit | 2e714b5ebd40a1f4cc31c27f1ad6e49ca6d895f5 (patch) | |
| tree | b83306b6e37edeea782e9eed673d89286c4fef35 /library/cpp/monlib/encode | |
| parent | 3e0b762a82514bac89c1dd6ea7211e381d8aa248 (diff) | |
| download | ydb-2e714b5ebd40a1f4cc31c27f1ad6e49ca6d895f5.tar.gz | |
Restoring authorship annotation for Sergey Polovko <sergey@polovko.me>. Commit 2 of 2.
Diffstat (limited to 'library/cpp/monlib/encode')
91 files changed, 6338 insertions, 6338 deletions
diff --git a/library/cpp/monlib/encode/buffered/buffered_encoder_base.cpp b/library/cpp/monlib/encode/buffered/buffered_encoder_base.cpp index b3d86b57ea..87c832d642 100644 --- a/library/cpp/monlib/encode/buffered/buffered_encoder_base.cpp +++ b/library/cpp/monlib/encode/buffered/buffered_encoder_base.cpp @@ -18,57 +18,57 @@ void TBufferedEncoderBase::OnCommonTime(TInstant time) { CommonTime_ = time; } -void TBufferedEncoderBase::OnMetricBegin(EMetricType type) { - State_.Switch(TEncoderState::EState::ROOT, TEncoderState::EState::METRIC); - Metrics_.emplace_back(); - Metrics_.back().MetricType = type; +void TBufferedEncoderBase::OnMetricBegin(EMetricType type) { + State_.Switch(TEncoderState::EState::ROOT, TEncoderState::EState::METRIC); + Metrics_.emplace_back(); + Metrics_.back().MetricType = type; } -void TBufferedEncoderBase::OnMetricEnd() { - State_.Switch(TEncoderState::EState::METRIC, TEncoderState::EState::ROOT); +void TBufferedEncoderBase::OnMetricEnd() { + State_.Switch(TEncoderState::EState::METRIC, TEncoderState::EState::ROOT); - switch (MetricsMergingMode_) { - case EMetricsMergingMode::MERGE_METRICS: { - auto& metric = Metrics_.back(); - Sort(metric.Labels, [] (const TPooledLabel& lhs, const TPooledLabel& rhs) { + switch (MetricsMergingMode_) { + case EMetricsMergingMode::MERGE_METRICS: { + auto& metric = Metrics_.back(); + Sort(metric.Labels, [] (const TPooledLabel& lhs, const TPooledLabel& rhs) { return std::tie(lhs.Key, lhs.Value) < std::tie(rhs.Key, rhs.Value); }); - auto it = MetricMap_.find(metric.Labels); - if (it == std::end(MetricMap_)) { - MetricMap_.emplace(metric.Labels, Metrics_.size() - 1); + auto it = MetricMap_.find(metric.Labels); + if (it == std::end(MetricMap_)) { + MetricMap_.emplace(metric.Labels, Metrics_.size() - 1); } else { - auto& existing = Metrics_[it->second].TimeSeries; + auto& existing = Metrics_[it->second].TimeSeries; - Y_ENSURE(existing.GetValueType() == metric.TimeSeries.GetValueType(), + Y_ENSURE(existing.GetValueType() == metric.TimeSeries.GetValueType(), "Time series point type mismatch: expected " << existing.GetValueType() << " but found " << metric.TimeSeries.GetValueType() << ", labels '" << FormatLabels(metric.Labels) << "'"); - existing.CopyFrom(metric.TimeSeries); - Metrics_.pop_back(); + existing.CopyFrom(metric.TimeSeries); + Metrics_.pop_back(); } break; } - case EMetricsMergingMode::DEFAULT: + case EMetricsMergingMode::DEFAULT: break; } } void TBufferedEncoderBase::OnLabelsBegin() { - if (State_ == TEncoderState::EState::METRIC) { - State_ = TEncoderState::EState::METRIC_LABELS; + if (State_ == TEncoderState::EState::METRIC) { + State_ = TEncoderState::EState::METRIC_LABELS; } else if (State_ == TEncoderState::EState::ROOT) { State_ = TEncoderState::EState::COMMON_LABELS; } else { - State_.ThrowInvalid("expected METRIC or ROOT"); + State_.ThrowInvalid("expected METRIC or ROOT"); } } void TBufferedEncoderBase::OnLabelsEnd() { - if (State_ == TEncoderState::EState::METRIC_LABELS) { - State_ = TEncoderState::EState::METRIC; + if (State_ == TEncoderState::EState::METRIC_LABELS) { + State_ = TEncoderState::EState::METRIC; } else if (State_ == TEncoderState::EState::COMMON_LABELS) { State_ = TEncoderState::EState::ROOT; } else { @@ -76,10 +76,10 @@ void TBufferedEncoderBase::OnLabelsEnd() { } } -void TBufferedEncoderBase::OnLabel(TStringBuf name, TStringBuf value) { +void TBufferedEncoderBase::OnLabel(TStringBuf name, TStringBuf value) { TPooledLabels* labels; - if (State_ == TEncoderState::EState::METRIC_LABELS) { - labels = &Metrics_.back().Labels; + if (State_ == TEncoderState::EState::METRIC_LABELS) { + labels = &Metrics_.back().Labels; } else if (State_ == TEncoderState::EState::COMMON_LABELS) { labels = &CommonLabels_; } else { @@ -102,46 +102,46 @@ void TBufferedEncoderBase::OnLabel(ui32 name, ui32 value) { labels->emplace_back(LabelNamesPool_.GetByIndex(name), LabelValuesPool_.GetByIndex(value)); } -std::pair<ui32, ui32> TBufferedEncoderBase::PrepareLabel(TStringBuf name, TStringBuf value) { +std::pair<ui32, ui32> TBufferedEncoderBase::PrepareLabel(TStringBuf name, TStringBuf value) { auto nameLabel = LabelNamesPool_.PutIfAbsent(name); auto valueLabel = LabelValuesPool_.PutIfAbsent(value); return std::make_pair(nameLabel->Index, valueLabel->Index); } void TBufferedEncoderBase::OnDouble(TInstant time, double value) { - State_.Expect(TEncoderState::EState::METRIC); - TMetric& metric = Metrics_.back(); - metric.TimeSeries.Add(time, value); + State_.Expect(TEncoderState::EState::METRIC); + TMetric& metric = Metrics_.back(); + metric.TimeSeries.Add(time, value); +} + +void TBufferedEncoderBase::OnInt64(TInstant time, i64 value) { + State_.Expect(TEncoderState::EState::METRIC); + TMetric& metric = Metrics_.back(); + metric.TimeSeries.Add(time, value); } -void TBufferedEncoderBase::OnInt64(TInstant time, i64 value) { - State_.Expect(TEncoderState::EState::METRIC); - TMetric& metric = Metrics_.back(); - metric.TimeSeries.Add(time, value); -} - void TBufferedEncoderBase::OnUint64(TInstant time, ui64 value) { - State_.Expect(TEncoderState::EState::METRIC); - TMetric& metric = Metrics_.back(); - metric.TimeSeries.Add(time, value); + State_.Expect(TEncoderState::EState::METRIC); + TMetric& metric = Metrics_.back(); + metric.TimeSeries.Add(time, value); +} + +void TBufferedEncoderBase::OnHistogram(TInstant time, IHistogramSnapshotPtr s) { + State_.Expect(TEncoderState::EState::METRIC); + TMetric& metric = Metrics_.back(); + metric.TimeSeries.Add(time, s.Get()); } -void TBufferedEncoderBase::OnHistogram(TInstant time, IHistogramSnapshotPtr s) { - State_.Expect(TEncoderState::EState::METRIC); - TMetric& metric = Metrics_.back(); - metric.TimeSeries.Add(time, s.Get()); -} - void TBufferedEncoderBase::OnSummaryDouble(TInstant time, ISummaryDoubleSnapshotPtr s) { - State_.Expect(TEncoderState::EState::METRIC); - TMetric& metric = Metrics_.back(); - metric.TimeSeries.Add(time, s.Get()); + State_.Expect(TEncoderState::EState::METRIC); + TMetric& metric = Metrics_.back(); + metric.TimeSeries.Add(time, s.Get()); } void TBufferedEncoderBase::OnLogHistogram(TInstant time, TLogHistogramSnapshotPtr s) { - State_.Expect(TEncoderState::EState::METRIC); - TMetric& metric = Metrics_.back(); - metric.TimeSeries.Add(time, s.Get()); + State_.Expect(TEncoderState::EState::METRIC); + TMetric& metric = Metrics_.back(); + metric.TimeSeries.Add(time, s.Get()); } TString TBufferedEncoderBase::FormatLabels(const TPooledLabels& labels) const { diff --git a/library/cpp/monlib/encode/buffered/buffered_encoder_base.h b/library/cpp/monlib/encode/buffered/buffered_encoder_base.h index d57ed5cd14..fe3714e58f 100644 --- a/library/cpp/monlib/encode/buffered/buffered_encoder_base.h +++ b/library/cpp/monlib/encode/buffered/buffered_encoder_base.h @@ -2,10 +2,10 @@ #include "string_pool.h" -#include <library/cpp/monlib/encode/encoder.h> -#include <library/cpp/monlib/encode/encoder_state.h> -#include <library/cpp/monlib/encode/format.h> -#include <library/cpp/monlib/metrics/metric_value.h> +#include <library/cpp/monlib/encode/encoder.h> +#include <library/cpp/monlib/encode/encoder_state.h> +#include <library/cpp/monlib/encode/format.h> +#include <library/cpp/monlib/metrics/metric_value.h> #include <util/datetime/base.h> #include <util/digest/numeric.h> @@ -13,30 +13,30 @@ namespace NMonitoring { -class TBufferedEncoderBase : public IMetricEncoder { +class TBufferedEncoderBase : public IMetricEncoder { public: void OnStreamBegin() override; void OnStreamEnd() override; void OnCommonTime(TInstant time) override; - void OnMetricBegin(EMetricType type) override; - void OnMetricEnd() override; + void OnMetricBegin(EMetricType type) override; + void OnMetricEnd() override; void OnLabelsBegin() override; void OnLabelsEnd() override; - void OnLabel(TStringBuf name, TStringBuf value) override; + void OnLabel(TStringBuf name, TStringBuf value) override; void OnLabel(ui32 name, ui32 value) override; - std::pair<ui32, ui32> PrepareLabel(TStringBuf name, TStringBuf value) override; + std::pair<ui32, ui32> PrepareLabel(TStringBuf name, TStringBuf value) override; void OnDouble(TInstant time, double value) override; - void OnInt64(TInstant time, i64 value) override; + void OnInt64(TInstant time, i64 value) override; void OnUint64(TInstant time, ui64 value) override; - void OnHistogram(TInstant time, IHistogramSnapshotPtr snapshot) override; + void OnHistogram(TInstant time, IHistogramSnapshotPtr snapshot) override; void OnSummaryDouble(TInstant time, ISummaryDoubleSnapshotPtr snapshot) override; void OnLogHistogram(TInstant, TLogHistogramSnapshotPtr) override; - + protected: using TPooledStr = TStringPoolBuilder::TValue; @@ -74,12 +74,12 @@ protected: } }; - using TMetricMap = THashMap<TPooledLabels, size_t, TPooledLabelsHash>; + using TMetricMap = THashMap<TPooledLabels, size_t, TPooledLabelsHash>; - struct TMetric { - EMetricType MetricType = EMetricType::UNKNOWN; + struct TMetric { + EMetricType MetricType = EMetricType::UNKNOWN; TPooledLabels Labels; - TMetricTimeSeries TimeSeries; + TMetricTimeSeries TimeSeries; }; protected: @@ -92,9 +92,9 @@ protected: TStringPoolBuilder LabelValuesPool_; TInstant CommonTime_ = TInstant::Zero(); TPooledLabels CommonLabels_; - TVector<TMetric> Metrics_; - TMetricMap MetricMap_; - EMetricsMergingMode MetricsMergingMode_ = EMetricsMergingMode::DEFAULT; + TVector<TMetric> Metrics_; + TMetricMap MetricMap_; + EMetricsMergingMode MetricsMergingMode_ = EMetricsMergingMode::DEFAULT; }; } diff --git a/library/cpp/monlib/encode/buffered/string_pool.cpp b/library/cpp/monlib/encode/buffered/string_pool.cpp index 2015ae9860..b4c7988ba3 100644 --- a/library/cpp/monlib/encode/buffered/string_pool.cpp +++ b/library/cpp/monlib/encode/buffered/string_pool.cpp @@ -1,24 +1,24 @@ -#include "string_pool.h" - -namespace NMonitoring { - //////////////////////////////////////////////////////////////////////////////// - // TStringPoolBuilder - //////////////////////////////////////////////////////////////////////////////// - const TStringPoolBuilder::TValue* TStringPoolBuilder::PutIfAbsent(TStringBuf str) { +#include "string_pool.h" + +namespace NMonitoring { + //////////////////////////////////////////////////////////////////////////////// + // TStringPoolBuilder + //////////////////////////////////////////////////////////////////////////////// + const TStringPoolBuilder::TValue* TStringPoolBuilder::PutIfAbsent(TStringBuf str) { Y_ENSURE(!IsBuilt_, "Cannot add more values after string has been built"); - + auto [it, isInserted] = StrMap_.try_emplace(str, Max<ui32>(), 0); if (isInserted) { BytesSize_ += str.size(); it->second.Index = StrVector_.size(); StrVector_.emplace_back(it->first, &it->second); - } + } TValue* value = &it->second; - ++value->Frequency; - return value; - } - + ++value->Frequency; + return value; + } + const TStringPoolBuilder::TValue* TStringPoolBuilder::GetByIndex(ui32 index) const { return StrVector_.at(index).second; } @@ -29,30 +29,30 @@ namespace NMonitoring { std::sort(StrVector_.begin(), StrVector_.end(), [](auto& a, auto& b) { return a.second->Frequency > b.second->Frequency; }); - + ui32 i = 0; for (auto& value : StrVector_) { value.second->Index = i++; } - } - + } + IsBuilt_ = true; return *this; - } - - //////////////////////////////////////////////////////////////////////////////// - // TStringPool - //////////////////////////////////////////////////////////////////////////////// - void TStringPool::InitIndex(const char* data, ui32 size) { - const char* begin = data; - const char* end = begin + size; - for (const char* p = begin; p != end; ++p) { - if (*p == '\0') { - Index_.push_back(TStringBuf(begin, p)); - begin = p + 1; - } - } - } - -} + } + + //////////////////////////////////////////////////////////////////////////////// + // TStringPool + //////////////////////////////////////////////////////////////////////////////// + void TStringPool::InitIndex(const char* data, ui32 size) { + const char* begin = data; + const char* end = begin + size; + for (const char* p = begin; p != end; ++p) { + if (*p == '\0') { + Index_.push_back(TStringBuf(begin, p)); + begin = p + 1; + } + } + } + +} diff --git a/library/cpp/monlib/encode/buffered/string_pool.h b/library/cpp/monlib/encode/buffered/string_pool.h index 2d67fd37a3..00e5644608 100644 --- a/library/cpp/monlib/encode/buffered/string_pool.h +++ b/library/cpp/monlib/encode/buffered/string_pool.h @@ -1,14 +1,14 @@ -#pragma once - -#include <util/generic/hash.h> -#include <util/generic/vector.h> - -namespace NMonitoring { - //////////////////////////////////////////////////////////////////////////////// - // TStringPoolBuilder - //////////////////////////////////////////////////////////////////////////////// - class TStringPoolBuilder { - public: +#pragma once + +#include <util/generic/hash.h> +#include <util/generic/vector.h> + +namespace NMonitoring { + //////////////////////////////////////////////////////////////////////////////// + // TStringPoolBuilder + //////////////////////////////////////////////////////////////////////////////// + class TStringPoolBuilder { + public: struct TValue: TNonCopyable { TValue(ui32 idx, ui32 freq) : Index{idx} @@ -16,77 +16,77 @@ namespace NMonitoring { { } - ui32 Index; - ui32 Frequency; - }; - - public: - const TValue* PutIfAbsent(TStringBuf str); + ui32 Index; + ui32 Frequency; + }; + + public: + const TValue* PutIfAbsent(TStringBuf str); const TValue* GetByIndex(ui32 index) const; - + /// Determines whether pool must be sorted by value frequencies TStringPoolBuilder& SetSorted(bool sorted) { RequiresSorting_ = sorted; return *this; } - + TStringPoolBuilder& Build(); - TStringBuf Get(ui32 index) const { + TStringBuf Get(ui32 index) const { Y_ENSURE(IsBuilt_, "Pool must be sorted first"); - return StrVector_.at(index).first; - } - + return StrVector_.at(index).first; + } + TStringBuf Get(const TValue* value) const { return StrVector_.at(value->Index).first; } - template <typename TConsumer> - void ForEach(TConsumer&& c) { + template <typename TConsumer> + void ForEach(TConsumer&& c) { Y_ENSURE(IsBuilt_, "Pool must be sorted first"); - for (const auto& value : StrVector_) { - c(value.first, value.second->Index, value.second->Frequency); - } - } - - size_t BytesSize() const noexcept { - return BytesSize_; - } - - size_t Count() const noexcept { - return StrMap_.size(); - } - - private: - THashMap<TString, TValue> StrMap_; - TVector<std::pair<TStringBuf, TValue*>> StrVector_; + for (const auto& value : StrVector_) { + c(value.first, value.second->Index, value.second->Frequency); + } + } + + size_t BytesSize() const noexcept { + return BytesSize_; + } + + size_t Count() const noexcept { + return StrMap_.size(); + } + + private: + THashMap<TString, TValue> StrMap_; + TVector<std::pair<TStringBuf, TValue*>> StrVector_; bool RequiresSorting_ = false; bool IsBuilt_ = false; - size_t BytesSize_ = 0; - }; - - //////////////////////////////////////////////////////////////////////////////// - // TStringPool - //////////////////////////////////////////////////////////////////////////////// - class TStringPool { - public: - TStringPool(const char* data, ui32 size) { - InitIndex(data, size); - } - - TStringBuf Get(ui32 i) const { - return Index_.at(i); - } - + size_t BytesSize_ = 0; + }; + + //////////////////////////////////////////////////////////////////////////////// + // TStringPool + //////////////////////////////////////////////////////////////////////////////// + class TStringPool { + public: + TStringPool(const char* data, ui32 size) { + InitIndex(data, size); + } + + TStringBuf Get(ui32 i) const { + return Index_.at(i); + } + size_t Size() const { return Index_.size(); } - private: - void InitIndex(const char* data, ui32 size); - - private: - TVector<TStringBuf> Index_; - }; - -} + private: + void InitIndex(const char* data, ui32 size); + + private: + TVector<TStringBuf> Index_; + }; + +} diff --git a/library/cpp/monlib/encode/buffered/string_pool_ut.cpp b/library/cpp/monlib/encode/buffered/string_pool_ut.cpp index 7f8b4c6724..9fc3421d0b 100644 --- a/library/cpp/monlib/encode/buffered/string_pool_ut.cpp +++ b/library/cpp/monlib/encode/buffered/string_pool_ut.cpp @@ -1,84 +1,84 @@ -#include "string_pool.h" - +#include "string_pool.h" + #include <library/cpp/testing/unittest/registar.h> - -using namespace NMonitoring; - + +using namespace NMonitoring; + Y_UNIT_TEST_SUITE(TStringPoolTest) { Y_UNIT_TEST(PutIfAbsent) { - TStringPoolBuilder strPool; + TStringPoolBuilder strPool; strPool.SetSorted(true); - auto* h1 = strPool.PutIfAbsent("one"); - auto* h2 = strPool.PutIfAbsent("two"); - auto* h3 = strPool.PutIfAbsent("two"); - UNIT_ASSERT(h1 != h2); - UNIT_ASSERT(h2 == h3); - - UNIT_ASSERT_VALUES_EQUAL(h1->Frequency, 1); + auto* h1 = strPool.PutIfAbsent("one"); + auto* h2 = strPool.PutIfAbsent("two"); + auto* h3 = strPool.PutIfAbsent("two"); + UNIT_ASSERT(h1 != h2); + UNIT_ASSERT(h2 == h3); + + UNIT_ASSERT_VALUES_EQUAL(h1->Frequency, 1); UNIT_ASSERT_VALUES_EQUAL(h1->Index, 0); - - UNIT_ASSERT_VALUES_EQUAL(h2->Frequency, 2); + + UNIT_ASSERT_VALUES_EQUAL(h2->Frequency, 2); UNIT_ASSERT_VALUES_EQUAL(h2->Index, 1); - - UNIT_ASSERT_VALUES_EQUAL(strPool.BytesSize(), 6); - UNIT_ASSERT_VALUES_EQUAL(strPool.Count(), 2); - } - + + UNIT_ASSERT_VALUES_EQUAL(strPool.BytesSize(), 6); + UNIT_ASSERT_VALUES_EQUAL(strPool.Count(), 2); + } + Y_UNIT_TEST(SortByFrequency) { - TStringPoolBuilder strPool; + TStringPoolBuilder strPool; strPool.SetSorted(true); - auto* h1 = strPool.PutIfAbsent("one"); - auto* h2 = strPool.PutIfAbsent("two"); - auto* h3 = strPool.PutIfAbsent("two"); - UNIT_ASSERT(h1 != h2); - UNIT_ASSERT(h2 == h3); - + auto* h1 = strPool.PutIfAbsent("one"); + auto* h2 = strPool.PutIfAbsent("two"); + auto* h3 = strPool.PutIfAbsent("two"); + UNIT_ASSERT(h1 != h2); + UNIT_ASSERT(h2 == h3); + strPool.Build(); - - UNIT_ASSERT_VALUES_EQUAL(h1->Frequency, 1); - UNIT_ASSERT_VALUES_EQUAL(h1->Index, 1); - - UNIT_ASSERT_VALUES_EQUAL(h2->Frequency, 2); - UNIT_ASSERT_VALUES_EQUAL(h2->Index, 0); - - UNIT_ASSERT_VALUES_EQUAL(strPool.BytesSize(), 6); - UNIT_ASSERT_VALUES_EQUAL(strPool.Count(), 2); - } - + + UNIT_ASSERT_VALUES_EQUAL(h1->Frequency, 1); + UNIT_ASSERT_VALUES_EQUAL(h1->Index, 1); + + UNIT_ASSERT_VALUES_EQUAL(h2->Frequency, 2); + UNIT_ASSERT_VALUES_EQUAL(h2->Index, 0); + + UNIT_ASSERT_VALUES_EQUAL(strPool.BytesSize(), 6); + UNIT_ASSERT_VALUES_EQUAL(strPool.Count(), 2); + } + Y_UNIT_TEST(ForEach) { - TStringPoolBuilder strPool; + TStringPoolBuilder strPool; strPool.SetSorted(true); - strPool.PutIfAbsent("one"); - strPool.PutIfAbsent("two"); - strPool.PutIfAbsent("two"); - strPool.PutIfAbsent("three"); - strPool.PutIfAbsent("three"); - strPool.PutIfAbsent("three"); - - UNIT_ASSERT_VALUES_EQUAL(strPool.BytesSize(), 11); - UNIT_ASSERT_VALUES_EQUAL(strPool.Count(), 3); - + strPool.PutIfAbsent("one"); + strPool.PutIfAbsent("two"); + strPool.PutIfAbsent("two"); + strPool.PutIfAbsent("three"); + strPool.PutIfAbsent("three"); + strPool.PutIfAbsent("three"); + + UNIT_ASSERT_VALUES_EQUAL(strPool.BytesSize(), 11); + UNIT_ASSERT_VALUES_EQUAL(strPool.Count(), 3); + strPool.Build(); - + TVector<TString> strings; TVector<ui32> indexes; TVector<ui32> frequences; - strPool.ForEach([&](TStringBuf str, ui32 index, ui32 freq) { - strings.emplace_back(str); - indexes.push_back(index); - frequences.push_back(freq); - }); - + strPool.ForEach([&](TStringBuf str, ui32 index, ui32 freq) { + strings.emplace_back(str); + indexes.push_back(index); + frequences.push_back(freq); + }); + TVector<TString> expectedStrings = {"three", "two", "one"}; - UNIT_ASSERT_EQUAL(strings, expectedStrings); - + UNIT_ASSERT_EQUAL(strings, expectedStrings); + TVector<ui32> expectedIndexes = {0, 1, 2}; - UNIT_ASSERT_EQUAL(indexes, expectedIndexes); - + UNIT_ASSERT_EQUAL(indexes, expectedIndexes); + TVector<ui32> expectedFrequences = {3, 2, 1}; - UNIT_ASSERT_EQUAL(frequences, expectedFrequences); - } -} + UNIT_ASSERT_EQUAL(frequences, expectedFrequences); + } +} diff --git a/library/cpp/monlib/encode/buffered/ut/ya.make b/library/cpp/monlib/encode/buffered/ut/ya.make index af2e09ac67..2157ac1490 100644 --- a/library/cpp/monlib/encode/buffered/ut/ya.make +++ b/library/cpp/monlib/encode/buffered/ut/ya.make @@ -1,4 +1,4 @@ -UNITTEST_FOR(library/cpp/monlib/encode/buffered) +UNITTEST_FOR(library/cpp/monlib/encode/buffered) OWNER( g:solomon diff --git a/library/cpp/monlib/encode/buffered/ya.make b/library/cpp/monlib/encode/buffered/ya.make index edaf29aa93..81b6a78b93 100644 --- a/library/cpp/monlib/encode/buffered/ya.make +++ b/library/cpp/monlib/encode/buffered/ya.make @@ -12,8 +12,8 @@ SRCS( ) PEERDIR( - library/cpp/monlib/encode - library/cpp/monlib/metrics + library/cpp/monlib/encode + library/cpp/monlib/metrics ) END() diff --git a/library/cpp/monlib/encode/encoder.cpp b/library/cpp/monlib/encode/encoder.cpp index 08b3fc4fd5..becf932689 100644 --- a/library/cpp/monlib/encode/encoder.cpp +++ b/library/cpp/monlib/encode/encoder.cpp @@ -1,6 +1,6 @@ -#include "encoder.h" - -namespace NMonitoring { - IMetricEncoder::~IMetricEncoder() { - } -} +#include "encoder.h" + +namespace NMonitoring { + IMetricEncoder::~IMetricEncoder() { + } +} diff --git a/library/cpp/monlib/encode/encoder.h b/library/cpp/monlib/encode/encoder.h index 5b307b4d45..a26a133d16 100644 --- a/library/cpp/monlib/encode/encoder.h +++ b/library/cpp/monlib/encode/encoder.h @@ -1,17 +1,17 @@ -#pragma once - -#include <util/generic/ptr.h> - -#include <library/cpp/monlib/metrics/metric_consumer.h> - -namespace NMonitoring { - class IMetricEncoder: public IMetricConsumer { - public: - virtual ~IMetricEncoder(); - - virtual void Close() = 0; - }; - - using IMetricEncoderPtr = THolder<IMetricEncoder>; - -} +#pragma once + +#include <util/generic/ptr.h> + +#include <library/cpp/monlib/metrics/metric_consumer.h> + +namespace NMonitoring { + class IMetricEncoder: public IMetricConsumer { + public: + virtual ~IMetricEncoder(); + + virtual void Close() = 0; + }; + + using IMetricEncoderPtr = THolder<IMetricEncoder>; + +} diff --git a/library/cpp/monlib/encode/encoder_state.cpp b/library/cpp/monlib/encode/encoder_state.cpp index 166369f876..0ece696b1a 100644 --- a/library/cpp/monlib/encode/encoder_state.cpp +++ b/library/cpp/monlib/encode/encoder_state.cpp @@ -1 +1 @@ -#include "encoder_state.h" +#include "encoder_state.h" diff --git a/library/cpp/monlib/encode/encoder_state.h b/library/cpp/monlib/encode/encoder_state.h index 51562a3d5f..e6a098f404 100644 --- a/library/cpp/monlib/encode/encoder_state.h +++ b/library/cpp/monlib/encode/encoder_state.h @@ -1,62 +1,62 @@ -#pragma once - +#pragma once + #include "encoder_state_enum.h" #include <util/generic/serialized_enum.h> -#include <util/generic/yexception.h> - +#include <util/generic/yexception.h> + -namespace NMonitoring { +namespace NMonitoring { template <typename EEncoderState> class TEncoderStateImpl { - public: + public: using EState = EEncoderState; - + explicit TEncoderStateImpl(EEncoderState state = EEncoderState::ROOT) - : State_(state) - { - } - + : State_(state) + { + } + TEncoderStateImpl& operator=(EEncoderState rhs) noexcept { - State_ = rhs; - return *this; - } - - inline bool operator==(EEncoderState rhs) const noexcept { - return State_ == rhs; - } - - inline bool operator!=(EEncoderState rhs) const noexcept { - return !operator==(rhs); - } - + State_ = rhs; + return *this; + } + + inline bool operator==(EEncoderState rhs) const noexcept { + return State_ == rhs; + } + + inline bool operator!=(EEncoderState rhs) const noexcept { + return !operator==(rhs); + } + [[noreturn]] inline void ThrowInvalid(TStringBuf message) const { - ythrow yexception() << "invalid encoder state: " - << ToStr() << ", " << message; - } - - inline void Expect(EEncoderState expected) const { - if (Y_UNLIKELY(State_ != expected)) { - ythrow yexception() - << "invalid encoder state: " << ToStr() + ythrow yexception() << "invalid encoder state: " + << ToStr() << ", " << message; + } + + inline void Expect(EEncoderState expected) const { + if (Y_UNLIKELY(State_ != expected)) { + ythrow yexception() + << "invalid encoder state: " << ToStr() << ", expected: " << TEncoderStateImpl(expected).ToStr(); - } - } - - inline void Switch(EEncoderState from, EEncoderState to) { - Expect(from); - State_ = to; - } - + } + } + + inline void Switch(EEncoderState from, EEncoderState to) { + Expect(from); + State_ = to; + } + TStringBuf ToStr() const noexcept { return NEnumSerializationRuntime::GetEnumNamesImpl<EEncoderState>().at(State_); } - - private: - EEncoderState State_; - }; - + + private: + EEncoderState State_; + }; + using TEncoderState = TEncoderStateImpl<EEncoderState>; } // namespace NMonitoring diff --git a/library/cpp/monlib/encode/encoder_state_enum.h b/library/cpp/monlib/encode/encoder_state_enum.h index 54f48a25e5..471604f91d 100644 --- a/library/cpp/monlib/encode/encoder_state_enum.h +++ b/library/cpp/monlib/encode/encoder_state_enum.h @@ -5,8 +5,8 @@ namespace NMonitoring { enum class EEncoderState { ROOT, COMMON_LABELS, - METRIC, - METRIC_LABELS, + METRIC, + METRIC_LABELS, }; } // namespace NMonitoring diff --git a/library/cpp/monlib/encode/fake/fake.cpp b/library/cpp/monlib/encode/fake/fake.cpp index 3bd3be8994..69d691361a 100644 --- a/library/cpp/monlib/encode/fake/fake.cpp +++ b/library/cpp/monlib/encode/fake/fake.cpp @@ -3,7 +3,7 @@ #include <util/datetime/base.h> namespace NMonitoring { - class TFakeEncoder: public IMetricEncoder { + class TFakeEncoder: public IMetricEncoder { public: void OnStreamBegin() override { } @@ -13,9 +13,9 @@ namespace NMonitoring { void OnCommonTime(TInstant) override { } - void OnMetricBegin(EMetricType) override { + void OnMetricBegin(EMetricType) override { } - void OnMetricEnd() override { + void OnMetricEnd() override { } void OnLabelsBegin() override { @@ -45,7 +45,7 @@ namespace NMonitoring { } }; - IMetricEncoderPtr EncoderFake() { + IMetricEncoderPtr EncoderFake() { return MakeHolder<TFakeEncoder>(); } } diff --git a/library/cpp/monlib/encode/fake/fake.h b/library/cpp/monlib/encode/fake/fake.h index 9b3e681c9b..8109326987 100644 --- a/library/cpp/monlib/encode/fake/fake.h +++ b/library/cpp/monlib/encode/fake/fake.h @@ -1,10 +1,10 @@ #pragma once -#include <library/cpp/monlib/encode/encoder.h> +#include <library/cpp/monlib/encode/encoder.h> class IOutputStream; namespace NMonitoring { - // Does nothing: just implements IMetricEncoder interface with stubs - IMetricEncoderPtr EncoderFake(); + // Does nothing: just implements IMetricEncoder interface with stubs + IMetricEncoderPtr EncoderFake(); } diff --git a/library/cpp/monlib/encode/format.cpp b/library/cpp/monlib/encode/format.cpp index d9bb4fcd04..400ce5a643 100644 --- a/library/cpp/monlib/encode/format.cpp +++ b/library/cpp/monlib/encode/format.cpp @@ -1,12 +1,12 @@ -#include "format.h" - -#include <util/string/ascii.h> +#include "format.h" + +#include <util/string/ascii.h> #include <util/string/split.h> -#include <util/string/strip.h> -#include <util/stream/output.h> -#include <util/string/cast.h> - -namespace NMonitoring { +#include <util/string/strip.h> +#include <util/stream/output.h> +#include <util/string/cast.h> + +namespace NMonitoring { static ECompression CompressionFromHeader(TStringBuf value) { if (value.empty()) { return ECompression::UNKNOWN; @@ -38,165 +38,165 @@ namespace NMonitoring { return EFormat::PROTOBUF; } else if (AsciiEqualsIgnoreCase(value, NFormatContenType::TEXT)) { return EFormat::TEXT; - } else if (AsciiEqualsIgnoreCase(value, NFormatContenType::PROMETHEUS)) { - return EFormat::PROMETHEUS; - } - + } else if (AsciiEqualsIgnoreCase(value, NFormatContenType::PROMETHEUS)) { + return EFormat::PROMETHEUS; + } + return EFormat::UNKNOWN; } EFormat FormatFromAcceptHeader(TStringBuf value) { EFormat result{EFormat::UNKNOWN}; - for (const auto& it : StringSplitter(value).Split(',').SkipEmpty()) { - TStringBuf token = StripString(it.Token()); - + for (const auto& it : StringSplitter(value).Split(',').SkipEmpty()) { + TStringBuf token = StripString(it.Token()); + result = FormatFromHttpMedia(token); if (result != EFormat::UNKNOWN) { break; - } - } - + } + } + return result; - } - + } + EFormat FormatFromContentType(TStringBuf value) { value = value.NextTok(';'); return FormatFromHttpMedia(value); } - TStringBuf ContentTypeByFormat(EFormat format) { - switch (format) { - case EFormat::SPACK: - return NFormatContenType::SPACK; - case EFormat::JSON: - return NFormatContenType::JSON; - case EFormat::PROTOBUF: - return NFormatContenType::PROTOBUF; - case EFormat::TEXT: - return NFormatContenType::TEXT; - case EFormat::PROMETHEUS: - return NFormatContenType::PROMETHEUS; - case EFormat::UNKNOWN: - return TStringBuf(); - } - - Y_FAIL(); // for GCC - } - - ECompression CompressionFromAcceptEncodingHeader(TStringBuf value) { + TStringBuf ContentTypeByFormat(EFormat format) { + switch (format) { + case EFormat::SPACK: + return NFormatContenType::SPACK; + case EFormat::JSON: + return NFormatContenType::JSON; + case EFormat::PROTOBUF: + return NFormatContenType::PROTOBUF; + case EFormat::TEXT: + return NFormatContenType::TEXT; + case EFormat::PROMETHEUS: + return NFormatContenType::PROMETHEUS; + case EFormat::UNKNOWN: + return TStringBuf(); + } + + Y_FAIL(); // for GCC + } + + ECompression CompressionFromAcceptEncodingHeader(TStringBuf value) { return CompressionFromHeader(value); } - + ECompression CompressionFromContentEncodingHeader(TStringBuf value) { return CompressionFromHeader(value); - } - - TStringBuf ContentEncodingByCompression(ECompression compression) { - switch (compression) { - case ECompression::IDENTITY: - return NFormatContentEncoding::IDENTITY; - case ECompression::ZLIB: - return NFormatContentEncoding::ZLIB; - case ECompression::LZ4: - return NFormatContentEncoding::LZ4; - case ECompression::ZSTD: - return NFormatContentEncoding::ZSTD; - case ECompression::UNKNOWN: - return TStringBuf(); - } - - Y_FAIL(); // for GCC - } - -} - -template <> -NMonitoring::EFormat FromStringImpl<NMonitoring::EFormat>(const char* str, size_t len) { - using NMonitoring::EFormat; - TStringBuf value(str, len); + } + + TStringBuf ContentEncodingByCompression(ECompression compression) { + switch (compression) { + case ECompression::IDENTITY: + return NFormatContentEncoding::IDENTITY; + case ECompression::ZLIB: + return NFormatContentEncoding::ZLIB; + case ECompression::LZ4: + return NFormatContentEncoding::LZ4; + case ECompression::ZSTD: + return NFormatContentEncoding::ZSTD; + case ECompression::UNKNOWN: + return TStringBuf(); + } + + Y_FAIL(); // for GCC + } + +} + +template <> +NMonitoring::EFormat FromStringImpl<NMonitoring::EFormat>(const char* str, size_t len) { + using NMonitoring::EFormat; + TStringBuf value(str, len); if (value == TStringBuf("SPACK")) { - return EFormat::SPACK; + return EFormat::SPACK; } else if (value == TStringBuf("JSON")) { - return EFormat::JSON; + return EFormat::JSON; } else if (value == TStringBuf("PROTOBUF")) { - return EFormat::PROTOBUF; + return EFormat::PROTOBUF; } else if (value == TStringBuf("TEXT")) { - return EFormat::TEXT; + return EFormat::TEXT; } else if (value == TStringBuf("PROMETHEUS")) { - return EFormat::PROMETHEUS; + return EFormat::PROMETHEUS; } else if (value == TStringBuf("UNKNOWN")) { - return EFormat::UNKNOWN; - } - ythrow yexception() << "unknown format: " << value; -} - -template <> -void Out<NMonitoring::EFormat>(IOutputStream& o, NMonitoring::EFormat f) { - using NMonitoring::EFormat; - switch (f) { - case EFormat::SPACK: + return EFormat::UNKNOWN; + } + ythrow yexception() << "unknown format: " << value; +} + +template <> +void Out<NMonitoring::EFormat>(IOutputStream& o, NMonitoring::EFormat f) { + using NMonitoring::EFormat; + switch (f) { + case EFormat::SPACK: o << TStringBuf("SPACK"); - return; - case EFormat::JSON: + return; + case EFormat::JSON: o << TStringBuf("JSON"); - return; - case EFormat::PROTOBUF: + return; + case EFormat::PROTOBUF: o << TStringBuf("PROTOBUF"); - return; - case EFormat::TEXT: + return; + case EFormat::TEXT: o << TStringBuf("TEXT"); - return; - case EFormat::PROMETHEUS: + return; + case EFormat::PROMETHEUS: o << TStringBuf("PROMETHEUS"); - return; - case EFormat::UNKNOWN: + return; + case EFormat::UNKNOWN: o << TStringBuf("UNKNOWN"); - return; - } - - Y_FAIL(); // for GCC -} - -template <> -NMonitoring::ECompression FromStringImpl<NMonitoring::ECompression>(const char* str, size_t len) { - using NMonitoring::ECompression; - TStringBuf value(str, len); + return; + } + + Y_FAIL(); // for GCC +} + +template <> +NMonitoring::ECompression FromStringImpl<NMonitoring::ECompression>(const char* str, size_t len) { + using NMonitoring::ECompression; + TStringBuf value(str, len); if (value == TStringBuf("IDENTITY")) { - return ECompression::IDENTITY; + return ECompression::IDENTITY; } else if (value == TStringBuf("ZLIB")) { - return ECompression::ZLIB; + return ECompression::ZLIB; } else if (value == TStringBuf("LZ4")) { - return ECompression::LZ4; + return ECompression::LZ4; } else if (value == TStringBuf("ZSTD")) { - return ECompression::ZSTD; + return ECompression::ZSTD; } else if (value == TStringBuf("UNKNOWN")) { - return ECompression::UNKNOWN; - } - ythrow yexception() << "unknown compression: " << value; -} - -template <> -void Out<NMonitoring::ECompression>(IOutputStream& o, NMonitoring::ECompression c) { - using NMonitoring::ECompression; - switch (c) { - case ECompression::IDENTITY: + return ECompression::UNKNOWN; + } + ythrow yexception() << "unknown compression: " << value; +} + +template <> +void Out<NMonitoring::ECompression>(IOutputStream& o, NMonitoring::ECompression c) { + using NMonitoring::ECompression; + switch (c) { + case ECompression::IDENTITY: o << TStringBuf("IDENTITY"); - return; - case ECompression::ZLIB: + return; + case ECompression::ZLIB: o << TStringBuf("ZLIB"); - return; - case ECompression::LZ4: + return; + case ECompression::LZ4: o << TStringBuf("LZ4"); - return; - case ECompression::ZSTD: + return; + case ECompression::ZSTD: o << TStringBuf("ZSTD"); - return; - case ECompression::UNKNOWN: + return; + case ECompression::UNKNOWN: o << TStringBuf("UNKNOWN"); - return; - } - - Y_FAIL(); // for GCC -} + return; + } + + Y_FAIL(); // for GCC +} diff --git a/library/cpp/monlib/encode/format.h b/library/cpp/monlib/encode/format.h index 8815a32a26..495d42d786 100644 --- a/library/cpp/monlib/encode/format.h +++ b/library/cpp/monlib/encode/format.h @@ -1,96 +1,96 @@ -#pragma once - -#include <util/generic/strbuf.h> - -namespace NMonitoring { - namespace NFormatContenType { +#pragma once + +#include <util/generic/strbuf.h> + +namespace NMonitoring { + namespace NFormatContenType { constexpr TStringBuf TEXT = "application/x-solomon-txt"; constexpr TStringBuf JSON = "application/json"; constexpr TStringBuf PROTOBUF = "application/x-solomon-pb"; constexpr TStringBuf SPACK = "application/x-solomon-spack"; constexpr TStringBuf PROMETHEUS = "text/plain"; - } - - namespace NFormatContentEncoding { + } + + namespace NFormatContentEncoding { constexpr TStringBuf IDENTITY = "identity"; constexpr TStringBuf ZLIB = "zlib"; constexpr TStringBuf LZ4 = "lz4"; constexpr TStringBuf ZSTD = "zstd"; - } - - /** - * Defines format types for metric encoders. - */ - enum class EFormat { - /** - * Special case when it was not possible to determine format. - */ - UNKNOWN, - - /** - * Read more https://wiki.yandex-team.ru/solomon/api/dataformat/spackv1 - */ - SPACK, - - /** - * Read more https://wiki.yandex-team.ru/solomon/api/dataformat/json - */ - JSON, - - /** - * Simple protobuf format, only for testing purposes. - */ - PROTOBUF, - - /** - * Simple text representation, only for debug purposes. - */ - TEXT, - - /** - * Prometheus text-based format - */ - PROMETHEUS, - }; - - /** - * Defines compression algorithms for metric encoders. - */ - enum class ECompression { - /** - * Special case when it was not possible to determine compression. - */ - UNKNOWN, - - /** - * Means no compression. - */ - IDENTITY, - - /** - * Using the zlib structure (defined in RFC 1950), with the - * deflate compression algorithm and Adler32 checkums. - */ - ZLIB, - - /** - * Using LZ4 compression algorithm (read http://lz4.org for more info) - * with XxHash32 checksums. - */ - LZ4, - - /** - * Using Zstandard compression algorithm (read http://zstd.net for more - * info) with XxHash32 checksums. - */ - ZSTD, - }; - - enum class EMetricsMergingMode { + } + + /** + * Defines format types for metric encoders. + */ + enum class EFormat { + /** + * Special case when it was not possible to determine format. + */ + UNKNOWN, + + /** + * Read more https://wiki.yandex-team.ru/solomon/api/dataformat/spackv1 + */ + SPACK, + + /** + * Read more https://wiki.yandex-team.ru/solomon/api/dataformat/json + */ + JSON, + + /** + * Simple protobuf format, only for testing purposes. + */ + PROTOBUF, + + /** + * Simple text representation, only for debug purposes. + */ + TEXT, + + /** + * Prometheus text-based format + */ + PROMETHEUS, + }; + + /** + * Defines compression algorithms for metric encoders. + */ + enum class ECompression { + /** + * Special case when it was not possible to determine compression. + */ + UNKNOWN, + + /** + * Means no compression. + */ + IDENTITY, + + /** + * Using the zlib structure (defined in RFC 1950), with the + * deflate compression algorithm and Adler32 checkums. + */ + ZLIB, + + /** + * Using LZ4 compression algorithm (read http://lz4.org for more info) + * with XxHash32 checksums. + */ + LZ4, + /** - * Do not merge metric batches. If several points of the same metric were + * Using Zstandard compression algorithm (read http://zstd.net for more + * info) with XxHash32 checksums. + */ + ZSTD, + }; + + enum class EMetricsMergingMode { + /** + * Do not merge metric batches. If several points of the same metric were * added multiple times accross different writes, paste them as - * separate metrics. + * separate metrics. * * Example: * COUNTER [(ts1, val1)] | COUNTER [(ts1, val1)] @@ -98,9 +98,9 @@ namespace NMonitoring { * COUNTER [(ts3, val3)] | COUNTER [(ts3, val3)] */ DEFAULT, - + /** - * If several points of the same metric were added multiple times across + * If several points of the same metric were added multiple times across * different writes, merge all values to one timeseries. * * Example: @@ -108,18 +108,18 @@ namespace NMonitoring { * COUNTER [(ts2, val2)] | --> COUNTER [(ts1, val1), (ts2, val2), (ts3, val3)] * COUNTER [(ts3, val3)] | */ - MERGE_METRICS, + MERGE_METRICS, }; - /** - * Matches serialization format by the given "Accept" header value. - * - * @param value value of the "Accept" header. - * @return most preffered serialization format type - */ + /** + * Matches serialization format by the given "Accept" header value. + * + * @param value value of the "Accept" header. + * @return most preffered serialization format type + */ EFormat FormatFromAcceptHeader(TStringBuf value); - - /** + + /** * Matches serialization format by the given "Content-Type" header value * * @param value value of the "Content-Type" header @@ -128,24 +128,24 @@ namespace NMonitoring { EFormat FormatFromContentType(TStringBuf value); /** - * Returns value for "Content-Type" header determined by the given - * format type. - * - * @param format serialization format type - * @return mime-type indentificator - * or empty string if format is UNKNOWN - */ - TStringBuf ContentTypeByFormat(EFormat format); - - /** - * Matches compression algorithm by the given "Accept-Encoding" header value. - * - * @param value value of the "Accept-Encoding" header. - * @return most preffered compression algorithm - */ - ECompression CompressionFromAcceptEncodingHeader(TStringBuf value); - - /** + * Returns value for "Content-Type" header determined by the given + * format type. + * + * @param format serialization format type + * @return mime-type indentificator + * or empty string if format is UNKNOWN + */ + TStringBuf ContentTypeByFormat(EFormat format); + + /** + * Matches compression algorithm by the given "Accept-Encoding" header value. + * + * @param value value of the "Accept-Encoding" header. + * @return most preffered compression algorithm + */ + ECompression CompressionFromAcceptEncodingHeader(TStringBuf value); + + /** * Matches compression algorithm by the given "Content-Encoding" header value. * * @param value value of the "Accept-Encoding" header. @@ -154,13 +154,13 @@ namespace NMonitoring { ECompression CompressionFromContentEncodingHeader(TStringBuf value); /** - * Returns value for "Content-Encoding" header determined by the given - * compression algorithm. - * - * @param compression encoding compression alg - * @return media-type compresion algorithm - * or empty string if compression is UNKNOWN - */ - TStringBuf ContentEncodingByCompression(ECompression compression); - -} + * Returns value for "Content-Encoding" header determined by the given + * compression algorithm. + * + * @param compression encoding compression alg + * @return media-type compresion algorithm + * or empty string if compression is UNKNOWN + */ + TStringBuf ContentEncodingByCompression(ECompression compression); + +} diff --git a/library/cpp/monlib/encode/format_ut.cpp b/library/cpp/monlib/encode/format_ut.cpp index 63475674d2..22a0e30c03 100644 --- a/library/cpp/monlib/encode/format_ut.cpp +++ b/library/cpp/monlib/encode/format_ut.cpp @@ -1,14 +1,14 @@ -#include "format.h" - +#include "format.h" + #include <library/cpp/testing/unittest/registar.h> - -#include <util/generic/string.h> -#include <util/string/builder.h> - -#include <array> - -using namespace NMonitoring; - + +#include <util/generic/string.h> +#include <util/string/builder.h> + +#include <array> + +using namespace NMonitoring; + Y_UNIT_TEST_SUITE(TFormatTest) { Y_UNIT_TEST(ContentTypeHeader) { UNIT_ASSERT_EQUAL(FormatFromContentType(""), EFormat::UNKNOWN); @@ -21,116 +21,116 @@ Y_UNIT_TEST_SUITE(TFormatTest) { Y_UNIT_TEST(AcceptHeader) { UNIT_ASSERT_EQUAL(FormatFromAcceptHeader(""), EFormat::UNKNOWN); UNIT_ASSERT_EQUAL(FormatFromAcceptHeader("*/*"), EFormat::UNKNOWN); - - UNIT_ASSERT_EQUAL( + + UNIT_ASSERT_EQUAL( FormatFromAcceptHeader("application/xml"), - EFormat::UNKNOWN); - - UNIT_ASSERT_EQUAL( + EFormat::UNKNOWN); + + UNIT_ASSERT_EQUAL( FormatFromAcceptHeader("application/json"), - EFormat::JSON); - - UNIT_ASSERT_EQUAL( + EFormat::JSON); + + UNIT_ASSERT_EQUAL( FormatFromAcceptHeader("application/x-solomon-spack"), - EFormat::SPACK); - - UNIT_ASSERT_EQUAL( + EFormat::SPACK); + + UNIT_ASSERT_EQUAL( FormatFromAcceptHeader("application/x-solomon-pb"), - EFormat::PROTOBUF); - - UNIT_ASSERT_EQUAL( - FormatFromAcceptHeader("application/x-solomon-txt"), - EFormat::TEXT); - - UNIT_ASSERT_EQUAL( + EFormat::PROTOBUF); + + UNIT_ASSERT_EQUAL( + FormatFromAcceptHeader("application/x-solomon-txt"), + EFormat::TEXT); + + UNIT_ASSERT_EQUAL( FormatFromAcceptHeader("application/json, text/plain"), - EFormat::JSON); - - UNIT_ASSERT_EQUAL( + EFormat::JSON); + + UNIT_ASSERT_EQUAL( FormatFromAcceptHeader("application/x-solomon-spack, application/json, text/plain"), - EFormat::SPACK); - - UNIT_ASSERT_EQUAL( + EFormat::SPACK); + + UNIT_ASSERT_EQUAL( FormatFromAcceptHeader(" , application/x-solomon-spack ,, application/json , text/plain"), - EFormat::SPACK); - - UNIT_ASSERT_EQUAL( + EFormat::SPACK); + + UNIT_ASSERT_EQUAL( FormatFromAcceptHeader("application/xml, application/x-solomon-spack, text/plain"), - EFormat::SPACK); - - UNIT_ASSERT_EQUAL( - FormatFromAcceptHeader("text/plain"), - EFormat::PROMETHEUS); - } - + EFormat::SPACK); + + UNIT_ASSERT_EQUAL( + FormatFromAcceptHeader("text/plain"), + EFormat::PROMETHEUS); + } + Y_UNIT_TEST(FormatToStrFromStr) { - const std::array<EFormat, 6> formats = {{ - EFormat::UNKNOWN, - EFormat::SPACK, - EFormat::JSON, - EFormat::PROTOBUF, - EFormat::TEXT, - EFormat::PROMETHEUS, - }}; - - for (EFormat f : formats) { - TString str = (TStringBuilder() << f); - EFormat g = FromString<EFormat>(str); - UNIT_ASSERT_EQUAL(f, g); - } - } - + const std::array<EFormat, 6> formats = {{ + EFormat::UNKNOWN, + EFormat::SPACK, + EFormat::JSON, + EFormat::PROTOBUF, + EFormat::TEXT, + EFormat::PROMETHEUS, + }}; + + for (EFormat f : formats) { + TString str = (TStringBuilder() << f); + EFormat g = FromString<EFormat>(str); + UNIT_ASSERT_EQUAL(f, g); + } + } + Y_UNIT_TEST(AcceptEncodingHeader) { - UNIT_ASSERT_EQUAL( - CompressionFromAcceptEncodingHeader(""), - ECompression::UNKNOWN); - - UNIT_ASSERT_EQUAL( - CompressionFromAcceptEncodingHeader("br"), - ECompression::UNKNOWN); - - UNIT_ASSERT_EQUAL( - CompressionFromAcceptEncodingHeader("identity"), - ECompression::IDENTITY); - - UNIT_ASSERT_EQUAL( - CompressionFromAcceptEncodingHeader("zlib"), - ECompression::ZLIB); - - UNIT_ASSERT_EQUAL( - CompressionFromAcceptEncodingHeader("lz4"), - ECompression::LZ4); - - UNIT_ASSERT_EQUAL( - CompressionFromAcceptEncodingHeader("zstd"), - ECompression::ZSTD); - - UNIT_ASSERT_EQUAL( - CompressionFromAcceptEncodingHeader("zstd, zlib"), - ECompression::ZSTD); - - UNIT_ASSERT_EQUAL( - CompressionFromAcceptEncodingHeader(" ,, , zstd , zlib"), - ECompression::ZSTD); - - UNIT_ASSERT_EQUAL( - CompressionFromAcceptEncodingHeader("br, deflate,lz4, zlib"), - ECompression::LZ4); - } - + UNIT_ASSERT_EQUAL( + CompressionFromAcceptEncodingHeader(""), + ECompression::UNKNOWN); + + UNIT_ASSERT_EQUAL( + CompressionFromAcceptEncodingHeader("br"), + ECompression::UNKNOWN); + + UNIT_ASSERT_EQUAL( + CompressionFromAcceptEncodingHeader("identity"), + ECompression::IDENTITY); + + UNIT_ASSERT_EQUAL( + CompressionFromAcceptEncodingHeader("zlib"), + ECompression::ZLIB); + + UNIT_ASSERT_EQUAL( + CompressionFromAcceptEncodingHeader("lz4"), + ECompression::LZ4); + + UNIT_ASSERT_EQUAL( + CompressionFromAcceptEncodingHeader("zstd"), + ECompression::ZSTD); + + UNIT_ASSERT_EQUAL( + CompressionFromAcceptEncodingHeader("zstd, zlib"), + ECompression::ZSTD); + + UNIT_ASSERT_EQUAL( + CompressionFromAcceptEncodingHeader(" ,, , zstd , zlib"), + ECompression::ZSTD); + + UNIT_ASSERT_EQUAL( + CompressionFromAcceptEncodingHeader("br, deflate,lz4, zlib"), + ECompression::LZ4); + } + Y_UNIT_TEST(CompressionToStrFromStr) { - const std::array<ECompression, 5> algs = {{ - ECompression::UNKNOWN, - ECompression::IDENTITY, - ECompression::ZLIB, - ECompression::LZ4, - ECompression::ZSTD, - }}; - - for (ECompression a : algs) { - TString str = (TStringBuilder() << a); - ECompression b = FromString<ECompression>(str); - UNIT_ASSERT_EQUAL(a, b); - } - } -} + const std::array<ECompression, 5> algs = {{ + ECompression::UNKNOWN, + ECompression::IDENTITY, + ECompression::ZLIB, + ECompression::LZ4, + ECompression::ZSTD, + }}; + + for (ECompression a : algs) { + TString str = (TStringBuilder() << a); + ECompression b = FromString<ECompression>(str); + UNIT_ASSERT_EQUAL(a, b); + } + } +} diff --git a/library/cpp/monlib/encode/fuzz/ya.make b/library/cpp/monlib/encode/fuzz/ya.make index 44b76eee49..d9ca172bae 100644 --- a/library/cpp/monlib/encode/fuzz/ya.make +++ b/library/cpp/monlib/encode/fuzz/ya.make @@ -1,5 +1,5 @@ RECURSE_ROOT_RELATIVE( - library/cpp/monlib/encode/json/fuzz - library/cpp/monlib/encode/prometheus/fuzz - library/cpp/monlib/encode/spack/fuzz + library/cpp/monlib/encode/json/fuzz + library/cpp/monlib/encode/prometheus/fuzz + library/cpp/monlib/encode/spack/fuzz ) diff --git a/library/cpp/monlib/encode/json/fuzz/main.cpp b/library/cpp/monlib/encode/json/fuzz/main.cpp index 65f98d8a8b..4f40310e06 100644 --- a/library/cpp/monlib/encode/json/fuzz/main.cpp +++ b/library/cpp/monlib/encode/json/fuzz/main.cpp @@ -1,5 +1,5 @@ -#include <library/cpp/monlib/encode/json/json.h> -#include <library/cpp/monlib/encode/fake/fake.h> +#include <library/cpp/monlib/encode/json/json.h> +#include <library/cpp/monlib/encode/fake/fake.h> #include <util/generic/strbuf.h> diff --git a/library/cpp/monlib/encode/json/fuzz/ya.make b/library/cpp/monlib/encode/json/fuzz/ya.make index 8ff5c6f9f9..75baa77716 100644 --- a/library/cpp/monlib/encode/json/fuzz/ya.make +++ b/library/cpp/monlib/encode/json/fuzz/ya.make @@ -6,8 +6,8 @@ OWNER( ) PEERDIR( - library/cpp/monlib/encode/json - library/cpp/monlib/encode/fake + library/cpp/monlib/encode/json + library/cpp/monlib/encode/fake ) SIZE(MEDIUM) diff --git a/library/cpp/monlib/encode/json/json.h b/library/cpp/monlib/encode/json/json.h index 2f0c8b0e1d..21530f20c3 100644 --- a/library/cpp/monlib/encode/json/json.h +++ b/library/cpp/monlib/encode/json/json.h @@ -1,20 +1,20 @@ -#pragma once - -#include <library/cpp/monlib/encode/encoder.h> -#include <library/cpp/monlib/encode/format.h> - +#pragma once -class IOutputStream; - -namespace NMonitoring { +#include <library/cpp/monlib/encode/encoder.h> +#include <library/cpp/monlib/encode/format.h> + + +class IOutputStream; + +namespace NMonitoring { class TJsonDecodeError: public yexception { }; - IMetricEncoderPtr EncoderJson(IOutputStream* out, int indentation = 0); - + IMetricEncoderPtr EncoderJson(IOutputStream* out, int indentation = 0); + /// Buffered encoder will merge series with same labels into one. - IMetricEncoderPtr BufferedEncoderJson(IOutputStream* out, int indentation = 0); + IMetricEncoderPtr BufferedEncoderJson(IOutputStream* out, int indentation = 0); IMetricEncoderPtr EncoderCloudJson(IOutputStream* out, int indentation = 0, @@ -25,5 +25,5 @@ namespace NMonitoring { TStringBuf metricNameLabel = "name"); void DecodeJson(TStringBuf data, IMetricConsumer* c, TStringBuf metricNameLabel = "name"); - -} + +} diff --git a/library/cpp/monlib/encode/json/json_decoder.cpp b/library/cpp/monlib/encode/json/json_decoder.cpp index c8824e1ccc..d44ff5fd28 100644 --- a/library/cpp/monlib/encode/json/json_decoder.cpp +++ b/library/cpp/monlib/encode/json/json_decoder.cpp @@ -1,24 +1,24 @@ -#include "json.h" -#include "typed_point.h" - +#include "json.h" +#include "typed_point.h" + #include <library/cpp/monlib/exception/exception.h> -#include <library/cpp/monlib/metrics/labels.h> -#include <library/cpp/monlib/metrics/metric_value.h> - +#include <library/cpp/monlib/metrics/labels.h> +#include <library/cpp/monlib/metrics/metric_value.h> + #include <library/cpp/json/json_reader.h> - -#include <util/datetime/base.h> -#include <util/string/cast.h> - -#include <limits> - -namespace NMonitoring { + +#include <util/datetime/base.h> +#include <util/string/cast.h> + +#include <limits> + +namespace NMonitoring { #define DECODE_ENSURE(COND, ...) MONLIB_ENSURE_EX(COND, TJsonDecodeError() << __VA_ARGS__) namespace { - + /////////////////////////////////////////////////////////////////////// // THistogramBuilder /////////////////////////////////////////////////////////////////////// @@ -32,11 +32,11 @@ public: } Bounds_.push_back(bound); } - + void AddValue(TBucketValue value) { Values_.push_back(value); } - + void AddInf(TBucketValue value) { InfPresented_ = true; InfValue_ = value; @@ -53,14 +53,14 @@ public: Bounds_.clear(); Values_.clear(); InfPresented_ = false; - + return snapshot; } - + bool Empty() const noexcept { return Bounds_.empty() && Values_.empty(); } - + void Clear() { Bounds_.clear(); Values_.clear(); @@ -69,7 +69,7 @@ public: private: TBucketBounds Bounds_; TBucketValues Values_; - + bool InfPresented_ = false; TBucketValue InfValue_; }; @@ -182,7 +182,7 @@ std::pair<double, bool> ParseSpecDouble(TStringBuf string) { return {0, false}; } } - + /////////////////////////////////////////////////////////////////////// // TMetricCollector /////////////////////////////////////////////////////////////////////// @@ -194,10 +194,10 @@ struct TMetricCollector { TLogHistogramBuilder LogHistBuilder; TTypedPoint LastPoint; TVector<TTypedPoint> TimeSeries; - + bool SeenTsOrValue = false; bool SeenTimeseries = false; - + void Clear() { Type = EMetricType::UNKNOWN; Labels.Clear(); @@ -209,20 +209,20 @@ struct TMetricCollector { SummaryBuilder.Clear(); LogHistBuilder.Clear(); } - + void AddLabel(const TLabel& label) { Labels.Add(label.Name(), label.Value()); } - + void SetLastTime(TInstant time) { LastPoint.SetTime(time); } - + template <typename T> void SetLastValue(T value) { LastPoint.SetValue(value); } - + void SaveLastPoint() { DECODE_ENSURE(LastPoint.GetTime() != TInstant::Zero(), "cannot add point without or zero timestamp"); @@ -248,21 +248,21 @@ struct TMetricCollector { } else { for (const auto& p: TimeSeries) { consumer(p.GetTime(), p.GetValueType(), p.GetValue()); - } + } } } }; - + struct TCommonParts { TInstant CommonTime; TLabels CommonLabels; }; - + class IHaltableMetricConsumer: public IMetricConsumer { public: virtual bool NeedToStop() const = 0; }; - + // TODO(ivanzhukov@): check all states for cases when a json document is invalid // e.g. "metrics" or "commonLabels" keys are specified multiple times class TCommonPartsCollector: public IHaltableMetricConsumer { @@ -433,11 +433,11 @@ class TDecoderJson final: public NJson::TJsonCallbacks { struct TState { enum EState { ROOT_OBJECT = 0x01, - + COMMON_LABELS, COMMON_TS, METRICS_ARRAY, - + METRIC_OBJECT, METRIC_NAME, METRIC_LABELS, @@ -462,21 +462,21 @@ class TDecoderJson final: public NJson::TJsonCallbacks { METRIC_LOG_HIST_START_POWER, METRIC_LOG_HIST_BUCKETS, }; - + constexpr EState Current() const noexcept { return static_cast<EState>(State_ & 0xFF); } - + void ToNext(EState state) noexcept { constexpr auto bitSize = 8 * sizeof(ui8); State_ = (State_ << bitSize) | static_cast<ui8>(state); } - + void ToPrev() noexcept { constexpr auto bitSize = 8 * sizeof(ui8); State_ = State_ >> bitSize; } - + private: ui64 State_ = static_cast<ui64>(ROOT_OBJECT); }; @@ -522,16 +522,16 @@ if (Y_UNLIKELY(!(CONDITION))) { \ LastMetric_.SetLastValue(static_cast<i64>(value)); State_.ToPrev(); break; - + case TState::METRIC_HIST_BOUNDS: LastMetric_.HistogramBuilder.AddBound(static_cast<double>(value)); break; - + case TState::METRIC_HIST_BUCKETS: PARSE_ENSURE(value >= 0 && static_cast<ui64>(value) <= Max<TBucketValues::value_type>(), "value is out of bounds " << value); LastMetric_.HistogramBuilder.AddValue(value); break; - + case TState::METRIC_HIST_INF: PARSE_ENSURE(value >= 0, "unexpected negative number in histogram inf: " << value); LastMetric_.HistogramBuilder.AddInf(value); @@ -584,7 +584,7 @@ if (Y_UNLIKELY(!(CONDITION))) { \ } return true; } - + bool OnUInteger(unsigned long long value) override { switch (State_.Current()) { case TState::COMMON_TS: @@ -597,32 +597,32 @@ if (Y_UNLIKELY(!(CONDITION))) { \ } break; - + case TState::METRIC_TS: LastMetric_.SetLastTime(TInstant::Seconds(value)); State_.ToPrev(); break; - + case TState::METRIC_VALUE: PARSE_ENSURE(value <= Max<ui64>(), "Metric value is out of bounds: " << value); LastMetric_.SetLastValue(static_cast<ui64>(value)); State_.ToPrev(); break; - + case TState::METRIC_HIST_BOUNDS: LastMetric_.HistogramBuilder.AddBound(static_cast<double>(value)); break; - + case TState::METRIC_HIST_BUCKETS: PARSE_ENSURE(value <= Max<TBucketValues::value_type>(), "Histogram bucket value is out of bounds: " << value); LastMetric_.HistogramBuilder.AddValue(value); break; - + case TState::METRIC_HIST_INF: LastMetric_.HistogramBuilder.AddInf(value); State_.ToPrev(); break; - + case TState::METRIC_DSUMMARY_COUNT: LastMetric_.SummaryBuilder.SetCount(value); State_.ToPrev(); @@ -669,18 +669,18 @@ if (Y_UNLIKELY(!(CONDITION))) { \ } return true; } - + bool OnDouble(double value) override { switch (State_.Current()) { case TState::METRIC_VALUE: LastMetric_.SetLastValue(value); State_.ToPrev(); break; - + case TState::METRIC_HIST_BOUNDS: LastMetric_.HistogramBuilder.AddBound(value); break; - + case TState::METRIC_DSUMMARY_SUM: LastMetric_.SummaryBuilder.SetSum(value); State_.ToPrev(); @@ -752,22 +752,22 @@ if (Y_UNLIKELY(!(CONDITION))) { \ LastMetric_.SetLastValue(doubleValue); } else { return false; - } + } State_.ToPrev(); break; - + case TState::METRIC_TYPE: LastMetric_.Type = MetricTypeFromStr(value); State_.ToPrev(); break; - + case TState::METRIC_MODE: if (value == TStringBuf("deriv")) { LastMetric_.Type = EMetricType::RATE; } State_.ToPrev(); break; - + case TState::METRIC_DSUMMARY_SUM: if (auto [doubleValue, ok] = ParseSpecDouble(value); ok) { LastMetric_.SummaryBuilder.SetSum(doubleValue); @@ -776,7 +776,7 @@ if (Y_UNLIKELY(!(CONDITION))) { \ } State_.ToPrev(); break; - + case TState::METRIC_DSUMMARY_MIN: if (auto [doubleValue, ok] = ParseSpecDouble(value); ok) { LastMetric_.SummaryBuilder.SetMin(doubleValue); @@ -785,7 +785,7 @@ if (Y_UNLIKELY(!(CONDITION))) { \ } State_.ToPrev(); break; - + case TState::METRIC_DSUMMARY_MAX: if (auto [doubleValue, ok] = ParseSpecDouble(value); ok) { LastMetric_.SummaryBuilder.SetMax(doubleValue); @@ -794,7 +794,7 @@ if (Y_UNLIKELY(!(CONDITION))) { \ } State_.ToPrev(); break; - + case TState::METRIC_DSUMMARY_LAST: if (auto [doubleValue, ok] = ParseSpecDouble(value); ok) { LastMetric_.SummaryBuilder.SetLast(doubleValue); @@ -803,11 +803,11 @@ if (Y_UNLIKELY(!(CONDITION))) { \ } State_.ToPrev(); break; - + default: return false; } - + return true; } @@ -877,7 +877,7 @@ if (Y_UNLIKELY(!(CONDITION))) { \ State_.ToNext(TState::METRIC_DSUMMARY); } else if (key == TStringBuf("log_hist")) { State_.ToNext(TState::METRIC_LOG_HIST); - } + } break; case TState::METRIC_HIST: @@ -889,7 +889,7 @@ if (Y_UNLIKELY(!(CONDITION))) { \ State_.ToNext(TState::METRIC_HIST_INF); } break; - + case TState::METRIC_LOG_HIST: if (key == TStringBuf("base")) { State_.ToNext(TState::METRIC_LOG_HIST_BASE); @@ -901,7 +901,7 @@ if (Y_UNLIKELY(!(CONDITION))) { \ State_.ToNext(TState::METRIC_LOG_HIST_BUCKETS); } break; - + case TState::METRIC_DSUMMARY: if (key == TStringBuf("sum")) { State_.ToNext(TState::METRIC_DSUMMARY_SUM); @@ -914,14 +914,14 @@ if (Y_UNLIKELY(!(CONDITION))) { \ } else if (key == TStringBuf("count")) { State_.ToNext(TState::METRIC_DSUMMARY_COUNT); } - + break; - - + + default: return false; } - + return true; } @@ -945,17 +945,17 @@ if (Y_UNLIKELY(!(CONDITION))) { \ } return true; } - + bool OnCloseMap() override { switch (State_.Current()) { case TState::ROOT_OBJECT: MetricConsumer_->OnStreamEnd(); break; - + case TState::METRIC_LABELS: State_.ToPrev(); break; - + case TState::COMMON_LABELS: MetricConsumer_->OnLabelsEnd(); State_.ToPrev(); @@ -966,22 +966,22 @@ if (Y_UNLIKELY(!(CONDITION))) { \ } break; - + case TState::METRIC_OBJECT: ConsumeMetric(); State_.ToPrev(); break; - + case TState::METRIC_TIMESERIES: LastMetric_.SaveLastPoint(); break; - + case TState::METRIC_HIST: case TState::METRIC_DSUMMARY: case TState::METRIC_LOG_HIST: State_.ToPrev(); break; - + default: break; } @@ -999,7 +999,7 @@ if (Y_UNLIKELY(!(CONDITION))) { \ "unexpected array begin"); return true; } - + bool OnCloseArray() override { switch (State_.Current()) { case TState::METRICS_ARRAY: @@ -1009,13 +1009,13 @@ if (Y_UNLIKELY(!(CONDITION))) { \ case TState::METRIC_LOG_HIST_BUCKETS: State_.ToPrev(); break; - + default: return false; } return true; } - + void OnError(size_t off, TStringBuf reason) override { if (IsIntentionallyHalted_) { return; @@ -1023,13 +1023,13 @@ if (Y_UNLIKELY(!(CONDITION))) { \ size_t snippetBeg = (off < 20) ? 0 : (off - 20); TStringBuf snippet = Data_.SubStr(snippetBeg, 40); - + throw TJsonDecodeError() << "cannot parse JSON, error at: " << off << ", reason: " << (ErrorMsg_.empty() ? reason : TStringBuf{ErrorMsg_}) << "\nsnippet: ..." << snippet << "..."; } - + bool OnEnd() override { return true; } @@ -1041,21 +1041,21 @@ if (Y_UNLIKELY(!(CONDITION))) { \ LastMetric_.Type = EMetricType::GAUGE; } else { LastMetric_.Type = EMetricType::HIST; - } + } } - + // (1) begin metric MetricConsumer_->OnMetricBegin(LastMetric_.Type); - + // (2) labels if (!LastMetric_.Labels.empty()) { MetricConsumer_->OnLabelsBegin(); for (auto&& label : LastMetric_.Labels) { MetricConsumer_->OnLabel(label.Name(), label.Value()); - } + } MetricConsumer_->OnLabelsEnd(); } - + // (3) values switch (LastMetric_.Type) { case EMetricType::GAUGE: @@ -1069,14 +1069,14 @@ if (Y_UNLIKELY(!(CONDITION))) { \ MetricConsumer_->OnInt64(time, value.AsInt64(valueType)); }); break; - + case EMetricType::COUNTER: case EMetricType::RATE: LastMetric_.Consume([this](TInstant time, EMetricValueType valueType, TMetricValue value) { MetricConsumer_->OnUint64(time, value.AsUint64(valueType)); }); break; - + case EMetricType::HIST: case EMetricType::HIST_RATE: if (LastMetric_.TimeSeries.empty()) { @@ -1087,10 +1087,10 @@ if (Y_UNLIKELY(!(CONDITION))) { \ for (const auto& p : LastMetric_.TimeSeries) { DECODE_ENSURE(p.GetValueType() == EMetricValueType::HISTOGRAM, "Value is not a histogram"); MetricConsumer_->OnHistogram(p.GetTime(), p.GetValue().AsHistogram()); - } - } + } + } break; - + case EMetricType::DSUMMARY: if (LastMetric_.TimeSeries.empty()) { auto time = LastMetric_.LastPoint.GetTime(); @@ -1103,7 +1103,7 @@ if (Y_UNLIKELY(!(CONDITION))) { \ } } break; - + case EMetricType::LOGHIST: if (LastMetric_.TimeSeries.empty()) { auto time = LastMetric_.LastPoint.GetTime(); @@ -1113,19 +1113,19 @@ if (Y_UNLIKELY(!(CONDITION))) { \ for (const auto& p : LastMetric_.TimeSeries) { DECODE_ENSURE(p.GetValueType() == EMetricValueType::LOGHISTOGRAM, "Value is not a log_histogram"); MetricConsumer_->OnLogHistogram(p.GetTime(), p.GetValue().AsLogHistogram()); - } - } + } + } break; - + case EMetricType::UNKNOWN: // TODO: output metric labels ythrow yexception() << "unknown metric type"; } - + // (4) end metric MetricConsumer_->OnMetricEnd(); } - + private: TStringBuf Data_; IHaltableMetricConsumer* MetricConsumer_; @@ -1136,9 +1136,9 @@ private: TString ErrorMsg_; bool IsIntentionallyHalted_{false}; }; - + } // namespace - + void DecodeJson(TStringBuf data, IMetricConsumer* c, TStringBuf metricNameLabel) { TCommonPartsCollector commonPartsCollector; { @@ -1159,4 +1159,4 @@ void DecodeJson(TStringBuf data, IMetricConsumer* c, TStringBuf metricNameLabel) #undef DECODE_ENSURE -} +} diff --git a/library/cpp/monlib/encode/json/json_encoder.cpp b/library/cpp/monlib/encode/json/json_encoder.cpp index 639177b5d1..20d2bb6283 100644 --- a/library/cpp/monlib/encode/json/json_encoder.cpp +++ b/library/cpp/monlib/encode/json/json_encoder.cpp @@ -1,39 +1,39 @@ -#include "json.h" -#include "typed_point.h" - -#include <library/cpp/monlib/encode/buffered/buffered_encoder_base.h> -#include <library/cpp/monlib/encode/encoder_state.h> -#include <library/cpp/monlib/metrics/metric.h> -#include <library/cpp/monlib/metrics/metric_value.h> -#include <library/cpp/monlib/metrics/labels.h> - +#include "json.h" +#include "typed_point.h" + +#include <library/cpp/monlib/encode/buffered/buffered_encoder_base.h> +#include <library/cpp/monlib/encode/encoder_state.h> +#include <library/cpp/monlib/metrics/metric.h> +#include <library/cpp/monlib/metrics/metric_value.h> +#include <library/cpp/monlib/metrics/labels.h> + #include <library/cpp/json/writer/json.h> - + #include <util/charset/utf8.h> #include <util/generic/algorithm.h> -namespace NMonitoring { - namespace { +namespace NMonitoring { + namespace { enum class EJsonStyle { Solomon, Cloud }; - /////////////////////////////////////////////////////////////////////// - // TJsonWriter - /////////////////////////////////////////////////////////////////////// + /////////////////////////////////////////////////////////////////////// + // TJsonWriter + /////////////////////////////////////////////////////////////////////// class TJsonWriter { - public: + public: TJsonWriter(IOutputStream* out, int indentation, EJsonStyle style, TStringBuf metricNameLabel) - : Buf_(NJsonWriter::HEM_UNSAFE, out) + : Buf_(NJsonWriter::HEM_UNSAFE, out) , Style_(style) , MetricNameLabel_(metricNameLabel) , CurrentMetricName_() - { - Buf_.SetIndentSpaces(indentation); - Buf_.SetWriteNanAsString(); - } - + { + Buf_.SetIndentSpaces(indentation); + Buf_.SetWriteNanAsString(); + } + void WriteTime(TInstant time) { if (time != TInstant::Zero()) { Buf_.WriteKey(TStringBuf("ts")); @@ -50,49 +50,49 @@ namespace NMonitoring { Buf_.WriteDouble(value); } - void WriteValue(i64 value) { + void WriteValue(i64 value) { Buf_.WriteKey(TStringBuf("value")); - Buf_.WriteLongLong(value); - } - + Buf_.WriteLongLong(value); + } + void WriteValue(ui64 value) { Buf_.WriteKey(TStringBuf("value")); Buf_.WriteULongLong(value); } - void WriteValue(IHistogramSnapshot* s) { + void WriteValue(IHistogramSnapshot* s) { Y_ENSURE(Style_ == EJsonStyle::Solomon); Buf_.WriteKey(TStringBuf("hist")); - Buf_.BeginObject(); - if (ui32 count = s->Count()) { - bool hasInf = (s->UpperBound(count - 1) == Max<double>()); - if (hasInf) { - count--; - } - + Buf_.BeginObject(); + if (ui32 count = s->Count()) { + bool hasInf = (s->UpperBound(count - 1) == Max<double>()); + if (hasInf) { + count--; + } + Buf_.WriteKey(TStringBuf("bounds")); - Buf_.BeginList(); - for (ui32 i = 0; i < count; i++) { - Buf_.WriteDouble(s->UpperBound(i)); - } - Buf_.EndList(); - + Buf_.BeginList(); + for (ui32 i = 0; i < count; i++) { + Buf_.WriteDouble(s->UpperBound(i)); + } + Buf_.EndList(); + Buf_.WriteKey(TStringBuf("buckets")); - Buf_.BeginList(); - for (ui32 i = 0; i < count; i++) { - Buf_.WriteULongLong(s->Value(i)); - } - Buf_.EndList(); - - if (hasInf) { + Buf_.BeginList(); + for (ui32 i = 0; i < count; i++) { + Buf_.WriteULongLong(s->Value(i)); + } + Buf_.EndList(); + + if (hasInf) { Buf_.WriteKey(TStringBuf("inf")); - Buf_.WriteULongLong(s->Value(count)); - } - } - Buf_.EndObject(); - } - + Buf_.WriteULongLong(s->Value(count)); + } + } + Buf_.EndObject(); + } + void WriteValue(ISummaryDoubleSnapshot* s) { Y_ENSURE(Style_ == EJsonStyle::Solomon); @@ -142,25 +142,25 @@ namespace NMonitoring { Buf_.EndObject(); } - void WriteValue(EMetricValueType type, TMetricValue value) { + void WriteValue(EMetricValueType type, TMetricValue value) { switch (type) { - case EMetricValueType::DOUBLE: - WriteValue(value.AsDouble()); + case EMetricValueType::DOUBLE: + WriteValue(value.AsDouble()); + break; + + case EMetricValueType::INT64: + WriteValue(value.AsInt64()); + break; + + case EMetricValueType::UINT64: + WriteValue(value.AsUint64()); break; - case EMetricValueType::INT64: - WriteValue(value.AsInt64()); - break; - - case EMetricValueType::UINT64: - WriteValue(value.AsUint64()); + case EMetricValueType::HISTOGRAM: + WriteValue(value.AsHistogram()); break; - case EMetricValueType::HISTOGRAM: - WriteValue(value.AsHistogram()); - break; - - case EMetricValueType::SUMMARY: + case EMetricValueType::SUMMARY: WriteValue(value.AsSummaryDouble()); break; @@ -168,7 +168,7 @@ namespace NMonitoring { WriteValue(value.AsLogHistogram()); break; - case EMetricValueType::UNKNOWN: + case EMetricValueType::UNKNOWN: ythrow yexception() << "unknown metric value type"; } } @@ -229,10 +229,10 @@ namespace NMonitoring { TString CurrentMetricName_; }; - /////////////////////////////////////////////////////////////////////// - // TEncoderJson - /////////////////////////////////////////////////////////////////////// - class TEncoderJson final: public IMetricEncoder, public TJsonWriter { + /////////////////////////////////////////////////////////////////////// + // TEncoderJson + /////////////////////////////////////////////////////////////////////// + class TEncoderJson final: public IMetricEncoder, public TJsonWriter { public: TEncoderJson(IOutputStream* out, int indentation, EJsonStyle style, TStringBuf metricNameLabel) : TJsonWriter{out, indentation, style, metricNameLabel} @@ -240,202 +240,202 @@ namespace NMonitoring { } ~TEncoderJson() override { - Close(); - } - - private: - void OnStreamBegin() override { + Close(); + } + + private: + void OnStreamBegin() override { State_.Expect(TEncoderState::EState::ROOT); - Buf_.BeginObject(); - } - - void OnStreamEnd() override { + Buf_.BeginObject(); + } + + void OnStreamEnd() override { State_.Expect(TEncoderState::EState::ROOT); - if (!Buf_.KeyExpected()) { - // not closed metrics array - Buf_.EndList(); - } - Buf_.EndObject(); - } - - void OnCommonTime(TInstant time) override { + if (!Buf_.KeyExpected()) { + // not closed metrics array + Buf_.EndList(); + } + Buf_.EndObject(); + } + + void OnCommonTime(TInstant time) override { State_.Expect(TEncoderState::EState::ROOT); - WriteTime(time); - } - - void OnMetricBegin(EMetricType type) override { - State_.Switch(TEncoderState::EState::ROOT, TEncoderState::EState::METRIC); - if (Buf_.KeyExpected()) { - // first metric, so open metrics array + WriteTime(time); + } + + void OnMetricBegin(EMetricType type) override { + State_.Switch(TEncoderState::EState::ROOT, TEncoderState::EState::METRIC); + if (Buf_.KeyExpected()) { + // first metric, so open metrics array Buf_.WriteKey(TStringBuf(Style_ == EJsonStyle::Solomon ? "sensors" : "metrics")); - Buf_.BeginList(); - } - Buf_.BeginObject(); + Buf_.BeginList(); + } + Buf_.BeginObject(); WriteMetricType(type); - } - - void OnMetricEnd() override { - State_.Switch(TEncoderState::EState::METRIC, TEncoderState::EState::ROOT); - if (!Buf_.KeyExpected()) { - // not closed timeseries array - Buf_.EndList(); - } - - if (!TimeSeries_ && LastPoint_.HasValue()) { - // we have seen only one point between OnMetricBegin() and - // OnMetricEnd() calls - WriteTime(LastPoint_.GetTime()); - WriteValue(LastPoint_.GetValueType(), LastPoint_.GetValue()); - } - Buf_.EndObject(); - - LastPoint_ = {}; - TimeSeries_ = false; - } - - void OnLabelsBegin() override { - if (!Buf_.KeyExpected()) { - // not closed metrics or timeseries array if labels go after values - Buf_.EndList(); - } + } + + void OnMetricEnd() override { + State_.Switch(TEncoderState::EState::METRIC, TEncoderState::EState::ROOT); + if (!Buf_.KeyExpected()) { + // not closed timeseries array + Buf_.EndList(); + } + + if (!TimeSeries_ && LastPoint_.HasValue()) { + // we have seen only one point between OnMetricBegin() and + // OnMetricEnd() calls + WriteTime(LastPoint_.GetTime()); + WriteValue(LastPoint_.GetValueType(), LastPoint_.GetValue()); + } + Buf_.EndObject(); + + LastPoint_ = {}; + TimeSeries_ = false; + } + + void OnLabelsBegin() override { + if (!Buf_.KeyExpected()) { + // not closed metrics or timeseries array if labels go after values + Buf_.EndList(); + } if (State_ == TEncoderState::EState::ROOT) { State_ = TEncoderState::EState::COMMON_LABELS; Buf_.WriteKey(TStringBuf(Style_ == EJsonStyle::Solomon ? "commonLabels" : "labels")); - } else if (State_ == TEncoderState::EState::METRIC) { - State_ = TEncoderState::EState::METRIC_LABELS; + } else if (State_ == TEncoderState::EState::METRIC) { + State_ = TEncoderState::EState::METRIC_LABELS; Buf_.WriteKey(TStringBuf("labels")); - } else { - State_.ThrowInvalid("expected METRIC or ROOT"); - } - Buf_.BeginObject(); + } else { + State_.ThrowInvalid("expected METRIC or ROOT"); + } + Buf_.BeginObject(); EmptyLabels_ = true; - } - - void OnLabelsEnd() override { - if (State_ == TEncoderState::EState::METRIC_LABELS) { - State_ = TEncoderState::EState::METRIC; + } + + void OnLabelsEnd() override { + if (State_ == TEncoderState::EState::METRIC_LABELS) { + State_ = TEncoderState::EState::METRIC; } else if (State_ == TEncoderState::EState::COMMON_LABELS) { State_ = TEncoderState::EState::ROOT; - } else { - State_.ThrowInvalid("expected LABELS or COMMON_LABELS"); - } + } else { + State_.ThrowInvalid("expected LABELS or COMMON_LABELS"); + } Y_ENSURE(!EmptyLabels_, "Labels cannot be empty"); - Buf_.EndObject(); + Buf_.EndObject(); if (State_ == TEncoderState::EState::METRIC) { WriteName(); } - } - - void OnLabel(TStringBuf name, TStringBuf value) override { - if (State_ == TEncoderState::EState::METRIC_LABELS || State_ == TEncoderState::EState::COMMON_LABELS) { + } + + void OnLabel(TStringBuf name, TStringBuf value) override { + if (State_ == TEncoderState::EState::METRIC_LABELS || State_ == TEncoderState::EState::COMMON_LABELS) { WriteLabel(name, value); - } else { - State_.ThrowInvalid("expected LABELS or COMMON_LABELS"); - } + } else { + State_.ThrowInvalid("expected LABELS or COMMON_LABELS"); + } EmptyLabels_ = false; - } - - void OnDouble(TInstant time, double value) override { - State_.Expect(TEncoderState::EState::METRIC); - Write<double>(time, value); - } - - void OnInt64(TInstant time, i64 value) override { - State_.Expect(TEncoderState::EState::METRIC); - Write<i64>(time, value); - } - - void OnUint64(TInstant time, ui64 value) override { - State_.Expect(TEncoderState::EState::METRIC); - Write<ui64>(time, value); - } - - void OnHistogram(TInstant time, IHistogramSnapshotPtr snapshot) override { - State_.Expect(TEncoderState::EState::METRIC); - Write<IHistogramSnapshot*>(time, snapshot.Get()); - } - + } + + void OnDouble(TInstant time, double value) override { + State_.Expect(TEncoderState::EState::METRIC); + Write<double>(time, value); + } + + void OnInt64(TInstant time, i64 value) override { + State_.Expect(TEncoderState::EState::METRIC); + Write<i64>(time, value); + } + + void OnUint64(TInstant time, ui64 value) override { + State_.Expect(TEncoderState::EState::METRIC); + Write<ui64>(time, value); + } + + void OnHistogram(TInstant time, IHistogramSnapshotPtr snapshot) override { + State_.Expect(TEncoderState::EState::METRIC); + Write<IHistogramSnapshot*>(time, snapshot.Get()); + } + void OnSummaryDouble(TInstant time, ISummaryDoubleSnapshotPtr snapshot) override { - State_.Expect(TEncoderState::EState::METRIC); + State_.Expect(TEncoderState::EState::METRIC); Write<ISummaryDoubleSnapshot*>(time, snapshot.Get()); } void OnLogHistogram(TInstant time, TLogHistogramSnapshotPtr snapshot) override { - State_.Expect(TEncoderState::EState::METRIC); + State_.Expect(TEncoderState::EState::METRIC); Write<TLogHistogramSnapshot*>(time, snapshot.Get()); } - template <typename T> - void Write(TInstant time, T value) { - State_.Expect(TEncoderState::EState::METRIC); - - if (!LastPoint_.HasValue()) { - LastPoint_ = {time, value}; - } else { - // second point - // TODO: output types - Y_ENSURE(LastPoint_.GetValueType() == TValueType<T>::Type, - "mixed metric value types in one metric"); - - if (!TimeSeries_) { + template <typename T> + void Write(TInstant time, T value) { + State_.Expect(TEncoderState::EState::METRIC); + + if (!LastPoint_.HasValue()) { + LastPoint_ = {time, value}; + } else { + // second point + // TODO: output types + Y_ENSURE(LastPoint_.GetValueType() == TValueType<T>::Type, + "mixed metric value types in one metric"); + + if (!TimeSeries_) { Buf_.WriteKey(TStringBuf("timeseries")); - Buf_.BeginList(); - Buf_.BeginObject(); - Y_ENSURE(LastPoint_.GetTime() != TInstant::Zero(), + Buf_.BeginList(); + Buf_.BeginObject(); + Y_ENSURE(LastPoint_.GetTime() != TInstant::Zero(), "time cannot be empty or zero in a timeseries point"); - WriteTime(LastPoint_.GetTime()); - WriteValue(LastPoint_.GetValueType(), LastPoint_.GetValue()); - Buf_.EndObject(); - TimeSeries_ = true; - } - - if (TimeSeries_) { - Buf_.BeginObject(); + WriteTime(LastPoint_.GetTime()); + WriteValue(LastPoint_.GetValueType(), LastPoint_.GetValue()); + Buf_.EndObject(); + TimeSeries_ = true; + } + + if (TimeSeries_) { + Buf_.BeginObject(); Y_ENSURE(time != TInstant::Zero(), "time cannot be empty or zero in a timeseries point"); - - WriteTime(time); - WriteValue(value); - Buf_.EndObject(); - } - } - } - - void Close() override { - LastPoint_ = {}; - } - + + WriteTime(time); + WriteValue(value); + Buf_.EndObject(); + } + } + } + + void Close() override { + LastPoint_ = {}; + } + private: TEncoderState State_; - TTypedPoint LastPoint_; + TTypedPoint LastPoint_; bool TimeSeries_ = false; bool EmptyLabels_ = false; }; - /////////////////////////////////////////////////////////////////////// - // TBufferedJsonEncoder - /////////////////////////////////////////////////////////////////////// + /////////////////////////////////////////////////////////////////////// + // TBufferedJsonEncoder + /////////////////////////////////////////////////////////////////////// class TBufferedJsonEncoder : public TBufferedEncoderBase, public TJsonWriter { public: TBufferedJsonEncoder(IOutputStream* out, int indentation, EJsonStyle style, TStringBuf metricNameLabel) : TJsonWriter{out, indentation, style, metricNameLabel} { - MetricsMergingMode_ = EMetricsMergingMode::MERGE_METRICS; - } - + MetricsMergingMode_ = EMetricsMergingMode::MERGE_METRICS; + } + ~TBufferedJsonEncoder() override { Close(); - } - + } + void OnLabelsBegin() override { TBufferedEncoderBase::OnLabelsBegin(); EmptyLabels_ = true; } - void OnLabel(TStringBuf name, TStringBuf value) override { + void OnLabel(TStringBuf name, TStringBuf value) override { TBufferedEncoderBase::OnLabel(name, value); EmptyLabels_ = false; } @@ -456,36 +456,36 @@ namespace NMonitoring { } Closed_ = true; - + LabelValuesPool_.Build(); LabelNamesPool_.Build(); - + Buf_.BeginObject(); - + WriteTime(CommonTime_); if (CommonLabels_.size() > 0) { Buf_.WriteKey(TStringBuf(Style_ == EJsonStyle::Solomon ? "commonLabels": "labels")); WriteLabels(CommonLabels_, true); - } + } - if (Metrics_.size() > 0) { + if (Metrics_.size() > 0) { Buf_.WriteKey(TStringBuf(Style_ == EJsonStyle::Solomon ? "sensors" : "metrics")); - WriteMetrics(); + WriteMetrics(); } Buf_.EndObject(); - } - - private: - void WriteMetrics() { + } + + private: + void WriteMetrics() { Buf_.BeginList(); - for (auto&& metric : Metrics_) { - WriteMetric(metric); + for (auto&& metric : Metrics_) { + WriteMetric(metric); } Buf_.EndList(); } - void WriteMetric(TMetric& metric) { + void WriteMetric(TMetric& metric) { Buf_.BeginObject(); WriteMetricType(metric.MetricType); @@ -493,19 +493,19 @@ namespace NMonitoring { Buf_.WriteKey(TStringBuf("labels")); WriteLabels(metric.Labels, false); - metric.TimeSeries.SortByTs(); - if (metric.TimeSeries.Size() == 1) { - const auto& point = metric.TimeSeries[0]; - WriteTime(point.GetTime()); - WriteValue(metric.TimeSeries.GetValueType(), point.GetValue()); - } else if (metric.TimeSeries.Size() > 1) { + metric.TimeSeries.SortByTs(); + if (metric.TimeSeries.Size() == 1) { + const auto& point = metric.TimeSeries[0]; + WriteTime(point.GetTime()); + WriteValue(metric.TimeSeries.GetValueType(), point.GetValue()); + } else if (metric.TimeSeries.Size() > 1) { Buf_.WriteKey(TStringBuf("timeseries")); Buf_.BeginList(); - metric.TimeSeries.ForEach([this](TInstant time, EMetricValueType type, TMetricValue value) { + metric.TimeSeries.ForEach([this](TInstant time, EMetricValueType type, TMetricValue value) { Buf_.BeginObject(); // make gcc 6.1 happy https://gcc.gnu.org/bugzilla/show_bug.cgi?id=61636 - this->WriteTime(time); - this->WriteValue(type, value); + this->WriteTime(time); + this->WriteValue(type, value); Buf_.EndObject(); }); @@ -535,14 +535,14 @@ namespace NMonitoring { private: bool Closed_{false}; bool EmptyLabels_ = false; - }; - } - - IMetricEncoderPtr EncoderJson(IOutputStream* out, int indentation) { + }; + } + + IMetricEncoderPtr EncoderJson(IOutputStream* out, int indentation) { return MakeHolder<TEncoderJson>(out, indentation, EJsonStyle::Solomon, ""); - } - - IMetricEncoderPtr BufferedEncoderJson(IOutputStream* out, int indentation) { + } + + IMetricEncoderPtr BufferedEncoderJson(IOutputStream* out, int indentation) { return MakeHolder<TBufferedJsonEncoder>(out, indentation, EJsonStyle::Solomon, ""); } @@ -553,4 +553,4 @@ namespace NMonitoring { IMetricEncoderPtr BufferedEncoderCloudJson(IOutputStream* out, int indentation, TStringBuf metricNameLabel) { return MakeHolder<TBufferedJsonEncoder>(out, indentation, EJsonStyle::Cloud, metricNameLabel); } -} +} diff --git a/library/cpp/monlib/encode/json/json_ut.cpp b/library/cpp/monlib/encode/json/json_ut.cpp index f80105795b..09e7909289 100644 --- a/library/cpp/monlib/encode/json/json_ut.cpp +++ b/library/cpp/monlib/encode/json/json_ut.cpp @@ -1,19 +1,19 @@ -#include "json.h" - -#include <library/cpp/monlib/encode/protobuf/protobuf.h> -#include <library/cpp/monlib/metrics/labels.h> - +#include "json.h" + +#include <library/cpp/monlib/encode/protobuf/protobuf.h> +#include <library/cpp/monlib/metrics/labels.h> + #include <library/cpp/json/json_reader.h> #include <library/cpp/resource/resource.h> #include <library/cpp/testing/unittest/registar.h> - -#include <util/stream/str.h> + +#include <util/stream/str.h> #include <util/string/builder.h> - -#include <limits> - -using namespace NMonitoring; - + +#include <limits> + +using namespace NMonitoring; + namespace NMonitoring { bool operator<(const TLabel& lhs, const TLabel& rhs) { return lhs.Name() < rhs.Name() || @@ -134,7 +134,7 @@ namespace { Y_UNIT_TEST_SUITE(TJsonTest) { const TInstant now = TInstant::ParseIso8601Deprecated("2017-11-05T01:02:03Z"); - + Y_UNIT_TEST(Encode) { auto check = [](bool cloud, bool buffered, TStringBuf expectedResourceKey) { TString json; @@ -145,12 +145,12 @@ Y_UNIT_TEST_SUITE(TJsonTest) { e->OnStreamBegin(); { // common time e->OnCommonTime(TInstant::Seconds(1500000000)); - } + } { // common labels - e->OnLabelsBegin(); + e->OnLabelsBegin(); e->OnLabel("project", "solomon"); - e->OnLabelsEnd(); - } + e->OnLabelsEnd(); + } { // metric #1 e->OnMetricBegin(EMetricType::COUNTER); { @@ -161,7 +161,7 @@ Y_UNIT_TEST_SUITE(TJsonTest) { } e->OnUint64(now, 17); e->OnMetricEnd(); - } + } { // metric #2 e->OnMetricBegin(EMetricType::RATE); { @@ -172,7 +172,7 @@ Y_UNIT_TEST_SUITE(TJsonTest) { } e->OnUint64(now, 17); e->OnMetricEnd(); - } + } { // metric #3 e->OnMetricBegin(EMetricType::GAUGE); e->OnDouble(now, 3.14); @@ -251,7 +251,7 @@ Y_UNIT_TEST_SUITE(TJsonTest) { auto emit = [&](IMetricEncoder* encoder, EMetricType metricType) { encoder->OnStreamBegin(); - { + { encoder->OnMetricBegin(metricType); { encoder->OnLabelsBegin(); @@ -280,11 +280,11 @@ Y_UNIT_TEST_SUITE(TJsonTest) { } encoder->OnMetricEnd(); - } + } encoder->OnStreamEnd(); encoder->Close(); }; - + auto doTest = [&](bool buffered, EMetricType metricType) { TString json; TStringOutput out(json); @@ -301,107 +301,107 @@ Y_UNIT_TEST_SUITE(TJsonTest) { doTest(true, EMetricType::HIST); doTest(true, EMetricType::LOGHIST); doTest(true, EMetricType::DSUMMARY); - } - - Y_UNIT_TEST(MetricsWithDifferentLabelOrderGetMerged) { + } + + Y_UNIT_TEST(MetricsWithDifferentLabelOrderGetMerged) { TString json; TStringOutput out(json); auto e = BufferedEncoderJson(&out, 2); - + e->OnStreamBegin(); { - e->OnMetricBegin(EMetricType::RATE); + e->OnMetricBegin(EMetricType::RATE); { e->OnLabelsBegin(); - e->OnLabel("metric", "hello"); + e->OnLabel("metric", "hello"); e->OnLabel("label", "world"); e->OnLabelsEnd(); } e->OnUint64(TInstant::Zero(), 0); - e->OnMetricEnd(); - } + e->OnMetricEnd(); + } { - e->OnMetricBegin(EMetricType::RATE); + e->OnMetricBegin(EMetricType::RATE); { e->OnLabelsBegin(); e->OnLabel("label", "world"); - e->OnLabel("metric", "hello"); + e->OnLabel("metric", "hello"); e->OnLabelsEnd(); } e->OnUint64(TInstant::Zero(), 1); - e->OnMetricEnd(); + e->OnMetricEnd(); } e->OnStreamEnd(); e->Close(); json += "\n"; - + TString expectedJson = NResource::Find("/merged.json"); // we cannot be sure regarding the label order in the result, // so we'll have to parse the expected value and then compare it with actual - + NProto::TMultiSamplesList samples; - IMetricEncoderPtr d = EncoderProtobuf(&samples); + IMetricEncoderPtr d = EncoderProtobuf(&samples); DecodeJson(expectedJson, d.Get()); - + UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 1); { const NProto::TMultiSample& s = samples.GetSamples(0); - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::RATE); - AssertLabels(s, TLabels{{"metric", "hello"}, {"label", "world"}}); - + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::RATE); + AssertLabels(s, TLabels{{"metric", "hello"}, {"label", "world"}}); + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); AssertPointEqual(s.GetPoints(0), TInstant::Zero(), ui64(1)); - } - } + } + } Y_UNIT_TEST(Decode1) { - NProto::TMultiSamplesList samples; - { - IMetricEncoderPtr e = EncoderProtobuf(&samples); - - TString testJson = NResource::Find("/expected.json"); - DecodeJson(testJson, e.Get()); - } - - UNIT_ASSERT_VALUES_EQUAL( - TInstant::MilliSeconds(samples.GetCommonTime()), - TInstant::Seconds(1500000000)); - - UNIT_ASSERT_VALUES_EQUAL(samples.CommonLabelsSize(), 1); - AssertLabelEqual(samples.GetCommonLabels(0), "project", "solomon"); - + NProto::TMultiSamplesList samples; + { + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + TString testJson = NResource::Find("/expected.json"); + DecodeJson(testJson, e.Get()); + } + + UNIT_ASSERT_VALUES_EQUAL( + TInstant::MilliSeconds(samples.GetCommonTime()), + TInstant::Seconds(1500000000)); + + UNIT_ASSERT_VALUES_EQUAL(samples.CommonLabelsSize(), 1); + AssertLabelEqual(samples.GetCommonLabels(0), "project", "solomon"); + UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 6); - { - const NProto::TMultiSample& s = samples.GetSamples(0); - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::COUNTER); - UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2); - AssertLabelEqual(s.GetLabels(0), "metric", "single"); + { + const NProto::TMultiSample& s = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::COUNTER); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2); + AssertLabelEqual(s.GetLabels(0), "metric", "single"); AssertLabelEqual(s.GetLabels(1), "labels", "l1"); - - UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); - AssertPointEqual(s.GetPoints(0), now, ui64(17)); - } - { - const NProto::TMultiSample& s = samples.GetSamples(1); - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::RATE); - UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2); - AssertLabelEqual(s.GetLabels(0), "metric", "single"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + AssertPointEqual(s.GetPoints(0), now, ui64(17)); + } + { + const NProto::TMultiSample& s = samples.GetSamples(1); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::RATE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2); + AssertLabelEqual(s.GetLabels(0), "metric", "single"); AssertLabelEqual(s.GetLabels(1), "labels", "l2"); - - UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); - AssertPointEqual(s.GetPoints(0), now, ui64(17)); - } - { - const NProto::TMultiSample& s = samples.GetSamples(2); - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); - UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2); - AssertLabelEqual(s.GetLabels(0), "metric", "single"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + AssertPointEqual(s.GetPoints(0), now, ui64(17)); + } + { + const NProto::TMultiSample& s = samples.GetSamples(2); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2); + AssertLabelEqual(s.GetLabels(0), "metric", "single"); AssertLabelEqual(s.GetLabels(1), "labels", "l3"); - - UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); - AssertPointEqual(s.GetPoints(0), now, 3.14); - } - { - const NProto::TMultiSample& s = samples.GetSamples(3); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + AssertPointEqual(s.GetPoints(0), now, 3.14); + } + { + const NProto::TMultiSample& s = samples.GetSamples(3); UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::IGAUGE); UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2); AssertLabelEqual(s.GetLabels(0), "metric", "single_igauge"); @@ -412,152 +412,152 @@ Y_UNIT_TEST_SUITE(TJsonTest) { } { const NProto::TMultiSample& s = samples.GetSamples(4); - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); - UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2); - AssertLabelEqual(s.GetLabels(0), "metric", "multiple"); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2); + AssertLabelEqual(s.GetLabels(0), "metric", "multiple"); AssertLabelEqual(s.GetLabels(1), "labels", "l5"); - - UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 3); - AssertPointEqualNan(s.GetPoints(0), now); - AssertPointEqualInf(s.GetPoints(1), now + TDuration::Seconds(15), 1); - AssertPointEqualInf(s.GetPoints(2), now + TDuration::Seconds(30), -11); - } - { + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 3); + AssertPointEqualNan(s.GetPoints(0), now); + AssertPointEqualInf(s.GetPoints(1), now + TDuration::Seconds(15), 1); + AssertPointEqualInf(s.GetPoints(2), now + TDuration::Seconds(30), -11); + } + { const NProto::TMultiSample& s = samples.GetSamples(5); - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::COUNTER); - UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2); - AssertLabelEqual(s.GetLabels(0), "metric", "multiple"); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::COUNTER); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2); + AssertLabelEqual(s.GetLabels(0), "metric", "multiple"); AssertLabelEqual(s.GetLabels(1), "labels", "l6"); - - UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 2); - AssertPointEqual(s.GetPoints(0), now, ui64(1337)); - AssertPointEqual(s.GetPoints(1), now + TDuration::Seconds(15), ui64(1338)); - } - } - - Y_UNIT_TEST(DecodeMetrics) { - NProto::TMultiSamplesList samples; - { - IMetricEncoderPtr e = EncoderProtobuf(&samples); - - TString metricsJson = NResource::Find("/metrics.json"); - DecodeJson(metricsJson, e.Get()); - } - - UNIT_ASSERT_VALUES_EQUAL( - TInstant::MilliSeconds(samples.GetCommonTime()), + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 2); + AssertPointEqual(s.GetPoints(0), now, ui64(1337)); + AssertPointEqual(s.GetPoints(1), now + TDuration::Seconds(15), ui64(1338)); + } + } + + Y_UNIT_TEST(DecodeMetrics) { + NProto::TMultiSamplesList samples; + { + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + TString metricsJson = NResource::Find("/metrics.json"); + DecodeJson(metricsJson, e.Get()); + } + + UNIT_ASSERT_VALUES_EQUAL( + TInstant::MilliSeconds(samples.GetCommonTime()), TInstant::ParseIso8601Deprecated("2017-08-27T12:34:56Z")); - - UNIT_ASSERT_VALUES_EQUAL(samples.CommonLabelsSize(), 3); - AssertLabelEqual(samples.GetCommonLabels(0), "project", "solomon"); - AssertLabelEqual(samples.GetCommonLabels(1), "cluster", "man"); - AssertLabelEqual(samples.GetCommonLabels(2), "service", "stockpile"); - + + UNIT_ASSERT_VALUES_EQUAL(samples.CommonLabelsSize(), 3); + AssertLabelEqual(samples.GetCommonLabels(0), "project", "solomon"); + AssertLabelEqual(samples.GetCommonLabels(1), "cluster", "man"); + AssertLabelEqual(samples.GetCommonLabels(2), "service", "stockpile"); + UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 4); - { - const NProto::TMultiSample& s = samples.GetSamples(0); - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); - UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); - AssertLabelEqual(s.GetLabels(0), "metric", "Memory"); - - UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); - AssertPointEqual(s.GetPoints(0), TInstant::Zero(), 10.0); - } - { - const NProto::TMultiSample& s = samples.GetSamples(1); - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::RATE); - UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); - AssertLabelEqual(s.GetLabels(0), "metric", "UserTime"); - - UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); - AssertPointEqual(s.GetPoints(0), TInstant::Zero(), ui64(1)); - } - { - const NProto::TMultiSample& s = samples.GetSamples(2); - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); - UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2); - AssertLabelEqual(s.GetLabels(0), "export", "Oxygen"); - AssertLabelEqual(s.GetLabels(1), "metric", "QueueSize"); - - UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + { + const NProto::TMultiSample& s = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "metric", "Memory"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + AssertPointEqual(s.GetPoints(0), TInstant::Zero(), 10.0); + } + { + const NProto::TMultiSample& s = samples.GetSamples(1); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::RATE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "metric", "UserTime"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + AssertPointEqual(s.GetPoints(0), TInstant::Zero(), ui64(1)); + } + { + const NProto::TMultiSample& s = samples.GetSamples(2); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2); + AssertLabelEqual(s.GetLabels(0), "export", "Oxygen"); + AssertLabelEqual(s.GetLabels(1), "metric", "QueueSize"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); auto ts = TInstant::ParseIso8601Deprecated("2017-11-05T12:34:56.000Z"); - AssertPointEqual(s.GetPoints(0), ts, 3.14159); - } - { + AssertPointEqual(s.GetPoints(0), ts, 3.14159); + } + { const NProto::TMultiSample& s = samples.GetSamples(3); - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); - UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); - AssertLabelEqual(s.GetLabels(0), "metric", "Writes"); - - UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 2); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "metric", "Writes"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 2); auto ts1 = TInstant::ParseIso8601Deprecated("2017-08-28T12:32:11Z"); - AssertPointEqual(s.GetPoints(0), ts1, -10.0); - auto ts2 = TInstant::Seconds(1503923187); - AssertPointEqual(s.GetPoints(1), ts2, 20.0); - } - } - - Y_UNIT_TEST(DecodeSensors) { - NProto::TMultiSamplesList samples; - { - IMetricEncoderPtr e = EncoderProtobuf(&samples); - - TString sensorsJson = NResource::Find("/sensors.json"); - DecodeJson(sensorsJson, e.Get()); - } - - UNIT_ASSERT_VALUES_EQUAL( - TInstant::MilliSeconds(samples.GetCommonTime()), - TInstant::ParseIso8601Deprecated("2017-08-27T12:34:56Z")); - - UNIT_ASSERT_VALUES_EQUAL(samples.CommonLabelsSize(), 3); - AssertLabelEqual(samples.GetCommonLabels(0), "project", "solomon"); - AssertLabelEqual(samples.GetCommonLabels(1), "cluster", "man"); - AssertLabelEqual(samples.GetCommonLabels(2), "service", "stockpile"); - - UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 4); - { - const NProto::TMultiSample& s = samples.GetSamples(0); - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); - UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); - AssertLabelEqual(s.GetLabels(0), "metric", "Memory"); - - UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); - AssertPointEqual(s.GetPoints(0), TInstant::Zero(), 10.0); - } - { - const NProto::TMultiSample& s = samples.GetSamples(1); - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::RATE); - UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); - AssertLabelEqual(s.GetLabels(0), "metric", "UserTime"); - - UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); - AssertPointEqual(s.GetPoints(0), TInstant::Zero(), ui64(1)); - } - { - const NProto::TMultiSample& s = samples.GetSamples(2); - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); - UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2); - AssertLabelEqual(s.GetLabels(0), "export", "Oxygen"); - AssertLabelEqual(s.GetLabels(1), "metric", "QueueSize"); - - UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); - auto ts = TInstant::ParseIso8601Deprecated("2017-11-05T12:34:56.000Z"); - AssertPointEqual(s.GetPoints(0), ts, 3.14159); - } - { - const NProto::TMultiSample& s = samples.GetSamples(3); - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); - UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); - AssertLabelEqual(s.GetLabels(0), "metric", "Writes"); - - UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 2); - auto ts1 = TInstant::ParseIso8601Deprecated("2017-08-28T12:32:11Z"); - AssertPointEqual(s.GetPoints(0), ts1, -10.0); - auto ts2 = TInstant::Seconds(1503923187); - AssertPointEqual(s.GetPoints(1), ts2, 20.0); - } - } + AssertPointEqual(s.GetPoints(0), ts1, -10.0); + auto ts2 = TInstant::Seconds(1503923187); + AssertPointEqual(s.GetPoints(1), ts2, 20.0); + } + } + + Y_UNIT_TEST(DecodeSensors) { + NProto::TMultiSamplesList samples; + { + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + TString sensorsJson = NResource::Find("/sensors.json"); + DecodeJson(sensorsJson, e.Get()); + } + + UNIT_ASSERT_VALUES_EQUAL( + TInstant::MilliSeconds(samples.GetCommonTime()), + TInstant::ParseIso8601Deprecated("2017-08-27T12:34:56Z")); + + UNIT_ASSERT_VALUES_EQUAL(samples.CommonLabelsSize(), 3); + AssertLabelEqual(samples.GetCommonLabels(0), "project", "solomon"); + AssertLabelEqual(samples.GetCommonLabels(1), "cluster", "man"); + AssertLabelEqual(samples.GetCommonLabels(2), "service", "stockpile"); + + UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 4); + { + const NProto::TMultiSample& s = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "metric", "Memory"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + AssertPointEqual(s.GetPoints(0), TInstant::Zero(), 10.0); + } + { + const NProto::TMultiSample& s = samples.GetSamples(1); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::RATE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "metric", "UserTime"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + AssertPointEqual(s.GetPoints(0), TInstant::Zero(), ui64(1)); + } + { + const NProto::TMultiSample& s = samples.GetSamples(2); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2); + AssertLabelEqual(s.GetLabels(0), "export", "Oxygen"); + AssertLabelEqual(s.GetLabels(1), "metric", "QueueSize"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + auto ts = TInstant::ParseIso8601Deprecated("2017-11-05T12:34:56.000Z"); + AssertPointEqual(s.GetPoints(0), ts, 3.14159); + } + { + const NProto::TMultiSample& s = samples.GetSamples(3); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "metric", "Writes"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 2); + auto ts1 = TInstant::ParseIso8601Deprecated("2017-08-28T12:32:11Z"); + AssertPointEqual(s.GetPoints(0), ts1, -10.0); + auto ts2 = TInstant::Seconds(1503923187); + AssertPointEqual(s.GetPoints(1), ts2, 20.0); + } + } Y_UNIT_TEST(DecodeToEncoder) { auto testJson = NResource::Find("/test_decode_to_encode.json"); @@ -574,16 +574,16 @@ Y_UNIT_TEST_SUITE(TJsonTest) { UNIT_ASSERT_VALUES_EQUAL(val1, val2); } - void WriteEmptySeries(const IMetricEncoderPtr& e) { + void WriteEmptySeries(const IMetricEncoderPtr& e) { e->OnStreamBegin(); { - e->OnMetricBegin(EMetricType::COUNTER); + e->OnMetricBegin(EMetricType::COUNTER); { e->OnLabelsBegin(); e->OnLabel("foo", "bar"); e->OnLabelsEnd(); } - e->OnMetricEnd(); + e->OnMetricEnd(); } e->OnStreamEnd(); @@ -602,9 +602,9 @@ Y_UNIT_TEST_SUITE(TJsonTest) { UNIT_ASSERT_NO_DIFF(json, expectedJson); } - void WriteEmptyLabels(IMetricEncoderPtr& e) { + void WriteEmptyLabels(IMetricEncoderPtr& e) { e->OnStreamBegin(); - e->OnMetricBegin(EMetricType::COUNTER); + e->OnMetricBegin(EMetricType::COUNTER); e->OnLabelsBegin(); UNIT_ASSERT_EXCEPTION(e->OnLabelsEnd(), yexception); @@ -638,29 +638,29 @@ Y_UNIT_TEST_SUITE(TJsonTest) { UNIT_ASSERT_NO_DIFF(json, expectedJson); } - Y_UNIT_TEST(BufferedEncoderMergesMetrics) { + Y_UNIT_TEST(BufferedEncoderMergesMetrics) { TString json; TStringOutput out(json); auto e = BufferedEncoderJson(&out, 2); auto ts = 1; - auto writeMetric = [&] (const TString& val) { - e->OnMetricBegin(EMetricType::COUNTER); + auto writeMetric = [&] (const TString& val) { + e->OnMetricBegin(EMetricType::COUNTER); e->OnLabelsBegin(); e->OnLabel("foo", val); e->OnLabelsEnd(); e->OnUint64(TInstant::Seconds(ts++), 42); - e->OnMetricEnd(); + e->OnMetricEnd(); }; e->OnStreamBegin(); - writeMetric("bar"); - writeMetric("bar"); - writeMetric("baz"); - writeMetric("bar"); + writeMetric("bar"); + writeMetric("bar"); + writeMetric("baz"); + writeMetric("bar"); e->OnStreamEnd(); e->Close(); @@ -676,13 +676,13 @@ Y_UNIT_TEST_SUITE(TJsonTest) { auto e = EncoderJson(&out); auto writePreamble = [&] { e->OnStreamBegin(); - e->OnMetricBegin(EMetricType::COUNTER); + e->OnMetricBegin(EMetricType::COUNTER); e->OnLabelsBegin(); e->OnLabel("foo", "bar"); e->OnLabelsEnd(); }; - // writing two values for a metric in a row will trigger + // writing two values for a metric in a row will trigger // timeseries object construction writePreamble(); e->OnUint64(TInstant::Zero(), 42); @@ -705,7 +705,7 @@ Y_UNIT_TEST_SUITE(TJsonTest) { { auto e = BufferedEncoderJson(&out, 2); e->OnStreamBegin(); - e->OnMetricBegin(EMetricType::COUNTER); + e->OnMetricBegin(EMetricType::COUNTER); e->OnLabelsBegin(); e->OnLabel("foo", "bar"); e->OnLabelsEnd(); @@ -715,38 +715,38 @@ Y_UNIT_TEST_SUITE(TJsonTest) { e->OnUint64(TInstant::Zero(), 43); e->OnUint64(TInstant::Zero(), 44); e->OnUint64(TInstant::Zero(), 45); - e->OnMetricEnd(); + e->OnMetricEnd(); e->OnStreamEnd(); } out << "\n"; UNIT_ASSERT_NO_DIFF(out.Str(), NResource::Find("/buffered_ts_merge.json")); - } - - template <typename TFactory, typename TConsumer> - TString EncodeToString(TFactory factory, TConsumer consumer) { - TStringStream out; - { - IMetricEncoderPtr e = factory(&out, 2); - consumer(e.Get()); - } - out << '\n'; - return out.Str(); } - + + template <typename TFactory, typename TConsumer> + TString EncodeToString(TFactory factory, TConsumer consumer) { + TStringStream out; + { + IMetricEncoderPtr e = factory(&out, 2); + consumer(e.Get()); + } + out << '\n'; + return out.Str(); + } + Y_UNIT_TEST(SummaryValueEncode) { - auto writeDocument = [](IMetricEncoder* e) { + auto writeDocument = [](IMetricEncoder* e) { e->OnStreamBegin(); { - e->OnMetricBegin(EMetricType::DSUMMARY); + e->OnMetricBegin(EMetricType::DSUMMARY); { e->OnLabelsBegin(); - e->OnLabel("metric", "temperature"); + e->OnLabel("metric", "temperature"); e->OnLabelsEnd(); } e->OnSummaryDouble(now, MakeIntrusive<TSummaryDoubleSnapshot>(10., -0.5, 0.5, 0.3, 30u)); - e->OnMetricEnd(); + e->OnMetricEnd(); } e->OnStreamEnd(); }; @@ -768,18 +768,18 @@ Y_UNIT_TEST_SUITE(TJsonTest) { } Y_UNIT_TEST(SummaryInfEncode) { - auto writeDocument = [](IMetricEncoder* e) { + auto writeDocument = [](IMetricEncoder* e) { e->OnStreamBegin(); { - e->OnMetricBegin(EMetricType::DSUMMARY); + e->OnMetricBegin(EMetricType::DSUMMARY); { e->OnLabelsBegin(); - e->OnLabel("metric", "temperature"); + e->OnLabel("metric", "temperature"); e->OnLabelsEnd(); } e->OnSummaryDouble(now, TestInfSummary()); - e->OnMetricEnd(); + e->OnMetricEnd(); } e->OnStreamEnd(); }; @@ -794,7 +794,7 @@ Y_UNIT_TEST_SUITE(TJsonTest) { Y_UNIT_TEST(SummaryInfDecode) { NProto::TMultiSamplesList samples; { - IMetricEncoderPtr e = EncoderProtobuf(&samples); + IMetricEncoderPtr e = EncoderProtobuf(&samples); TString testJson = NResource::Find("/summary_inf.json"); DecodeJson(testJson, e.Get()); @@ -803,9 +803,9 @@ Y_UNIT_TEST_SUITE(TJsonTest) { UNIT_ASSERT_VALUES_EQUAL(1, samples.SamplesSize()); const NProto::TMultiSample& s = samples.GetSamples(0); - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::DSUMMARY); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::DSUMMARY); UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); - AssertLabelEqual(s.GetLabels(0), "metric", "temperature"); + AssertLabelEqual(s.GetLabels(0), "metric", "temperature"); UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); @@ -820,7 +820,7 @@ Y_UNIT_TEST_SUITE(TJsonTest) { Y_UNIT_TEST(SummaryValueDecode) { NProto::TMultiSamplesList samples; { - IMetricEncoderPtr e = EncoderProtobuf(&samples); + IMetricEncoderPtr e = EncoderProtobuf(&samples); TString testJson = NResource::Find("/summary_value.json"); DecodeJson(testJson, e.Get()); @@ -829,9 +829,9 @@ Y_UNIT_TEST_SUITE(TJsonTest) { UNIT_ASSERT_VALUES_EQUAL(1, samples.SamplesSize()); const NProto::TMultiSample& s = samples.GetSamples(0); - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::DSUMMARY); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::DSUMMARY); UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); - AssertLabelEqual(s.GetLabels(0), "metric", "temperature"); + AssertLabelEqual(s.GetLabels(0), "metric", "temperature"); UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); @@ -840,13 +840,13 @@ Y_UNIT_TEST_SUITE(TJsonTest) { } Y_UNIT_TEST(SummaryTimeSeriesEncode) { - auto writeDocument = [](IMetricEncoder* e) { + auto writeDocument = [](IMetricEncoder* e) { e->OnStreamBegin(); { - e->OnMetricBegin(EMetricType::DSUMMARY); + e->OnMetricBegin(EMetricType::DSUMMARY); { e->OnLabelsBegin(); - e->OnLabel("metric", "temperature"); + e->OnLabel("metric", "temperature"); e->OnLabelsEnd(); } @@ -862,7 +862,7 @@ Y_UNIT_TEST_SUITE(TJsonTest) { e->OnSummaryDouble(now + TDuration::Seconds(15), summary.Snapshot()); - e->OnMetricEnd(); + e->OnMetricEnd(); } e->OnStreamEnd(); }; @@ -877,7 +877,7 @@ Y_UNIT_TEST_SUITE(TJsonTest) { Y_UNIT_TEST(SummaryTimeSeriesDecode) { NProto::TMultiSamplesList samples; { - IMetricEncoderPtr e = EncoderProtobuf(&samples); + IMetricEncoderPtr e = EncoderProtobuf(&samples); TString testJson = NResource::Find("/summary_timeseries.json"); DecodeJson(testJson, e.Get()); @@ -886,9 +886,9 @@ Y_UNIT_TEST_SUITE(TJsonTest) { UNIT_ASSERT_VALUES_EQUAL(1, samples.SamplesSize()); const NProto::TMultiSample& s = samples.GetSamples(0); - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::DSUMMARY); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::DSUMMARY); UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); - AssertLabelEqual(s.GetLabels(0), "metric", "temperature"); + AssertLabelEqual(s.GetLabels(0), "metric", "temperature"); UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 2); @@ -912,7 +912,7 @@ Y_UNIT_TEST_SUITE(TJsonTest) { e->OnMetricBegin(EMetricType::LOGHIST); { e->OnLabelsBegin(); - e->OnLabel("metric", "ms"); + e->OnLabel("metric", "ms"); e->OnLabelsEnd(); } @@ -943,7 +943,7 @@ Y_UNIT_TEST_SUITE(TJsonTest) { UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::LOGHISTOGRAM); UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); - AssertLabelEqual(s.GetLabels(0), "metric", "ms"); + AssertLabelEqual(s.GetLabels(0), "metric", "ms"); UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); @@ -951,36 +951,36 @@ Y_UNIT_TEST_SUITE(TJsonTest) { AssertPointEqual(s.GetPoints(0), now, *snapshot); } - Y_UNIT_TEST(HistogramValueEncode) { - auto writeDocument = [](IMetricEncoder* e) { - e->OnStreamBegin(); - { - e->OnMetricBegin(EMetricType::HIST); - { - e->OnLabelsBegin(); - e->OnLabel("metric", "responseTimeMillis"); - e->OnLabelsEnd(); - } - - // {1: 1, 2: 1, 4: 2, 8: 4, 16: 8, inf: 83} - auto h = ExponentialHistogram(6, 2); - for (i64 i = 1; i < 100; i++) { - h->Collect(i); - } - - e->OnHistogram(now, h->Snapshot()); - e->OnMetricEnd(); - } - e->OnStreamEnd(); - }; - - TString result1 = EncodeToString(EncoderJson, writeDocument); - UNIT_ASSERT_NO_DIFF(result1, NResource::Find("/histogram_value.json")); - - TString result2 = EncodeToString(BufferedEncoderJson, writeDocument); - UNIT_ASSERT_NO_DIFF(result2, NResource::Find("/histogram_value.json")); - } - + Y_UNIT_TEST(HistogramValueEncode) { + auto writeDocument = [](IMetricEncoder* e) { + e->OnStreamBegin(); + { + e->OnMetricBegin(EMetricType::HIST); + { + e->OnLabelsBegin(); + e->OnLabel("metric", "responseTimeMillis"); + e->OnLabelsEnd(); + } + + // {1: 1, 2: 1, 4: 2, 8: 4, 16: 8, inf: 83} + auto h = ExponentialHistogram(6, 2); + for (i64 i = 1; i < 100; i++) { + h->Collect(i); + } + + e->OnHistogram(now, h->Snapshot()); + e->OnMetricEnd(); + } + e->OnStreamEnd(); + }; + + TString result1 = EncodeToString(EncoderJson, writeDocument); + UNIT_ASSERT_NO_DIFF(result1, NResource::Find("/histogram_value.json")); + + TString result2 = EncodeToString(BufferedEncoderJson, writeDocument); + UNIT_ASSERT_NO_DIFF(result2, NResource::Find("/histogram_value.json")); + } + Y_UNIT_TEST(LogHistogramTimeSeriesEncode) { auto writeDocument = [](IMetricEncoder* e) { e->OnStreamBegin(); @@ -988,7 +988,7 @@ Y_UNIT_TEST_SUITE(TJsonTest) { e->OnMetricBegin(EMetricType::LOGHIST); { e->OnLabelsBegin(); - e->OnLabel("metric", "ms"); + e->OnLabel("metric", "ms"); e->OnLabelsEnd(); } @@ -1022,7 +1022,7 @@ Y_UNIT_TEST_SUITE(TJsonTest) { UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::LOGHISTOGRAM); UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); - AssertLabelEqual(s.GetLabels(0), "metric", "ms"); + AssertLabelEqual(s.GetLabels(0), "metric", "ms"); UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 2); @@ -1034,130 +1034,130 @@ Y_UNIT_TEST_SUITE(TJsonTest) { } void HistogramValueDecode(const TString& filePath) { - NProto::TMultiSamplesList samples; - { - IMetricEncoderPtr e = EncoderProtobuf(&samples); - + NProto::TMultiSamplesList samples; + { + IMetricEncoderPtr e = EncoderProtobuf(&samples); + TString testJson = NResource::Find(filePath); - DecodeJson(testJson, e.Get()); - } - - UNIT_ASSERT_VALUES_EQUAL(1, samples.SamplesSize()); - const NProto::TMultiSample& s = samples.GetSamples(0); - - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::HISTOGRAM); - UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); - AssertLabelEqual(s.GetLabels(0), "metric", "responseTimeMillis"); - - UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); - - auto h = ExponentialHistogram(6, 2); - for (i64 i = 1; i < 100; i++) { - h->Collect(i); - } - - AssertPointEqual(s.GetPoints(0), now, *h->Snapshot()); - } - + DecodeJson(testJson, e.Get()); + } + + UNIT_ASSERT_VALUES_EQUAL(1, samples.SamplesSize()); + const NProto::TMultiSample& s = samples.GetSamples(0); + + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::HISTOGRAM); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "metric", "responseTimeMillis"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + + auto h = ExponentialHistogram(6, 2); + for (i64 i = 1; i < 100; i++) { + h->Collect(i); + } + + AssertPointEqual(s.GetPoints(0), now, *h->Snapshot()); + } + Y_UNIT_TEST(HistogramValueDecode) { HistogramValueDecode("/histogram_value.json"); HistogramValueDecode("/histogram_value_inf_before_bounds.json"); } - Y_UNIT_TEST(HistogramTimeSeriesEncode) { - auto writeDocument = [](IMetricEncoder* e) { - e->OnStreamBegin(); - { - e->OnMetricBegin(EMetricType::HIST_RATE); - { - e->OnLabelsBegin(); - e->OnLabel("metric", "responseTimeMillis"); - e->OnLabelsEnd(); - } - - // {1: 1, 2: 1, 4: 2, 8: 4, 16: 8, inf: 83} - auto h = ExponentialHistogram(6, 2); - for (i64 i = 1; i < 100; i++) { - h->Collect(i); - } - e->OnHistogram(now, h->Snapshot()); - - // {1: 2, 2: 2, 4: 4, 8: 8, 16: 16, inf: 166} - for (i64 i = 1; i < 100; i++) { - h->Collect(i); - } - e->OnHistogram(now + TDuration::Seconds(15), h->Snapshot()); - - e->OnMetricEnd(); - } - e->OnStreamEnd(); - }; - - TString result1 = EncodeToString(EncoderJson, writeDocument); - UNIT_ASSERT_NO_DIFF(result1, NResource::Find("/histogram_timeseries.json")); - - TString result2 = EncodeToString(BufferedEncoderJson, writeDocument); - UNIT_ASSERT_NO_DIFF(result2, NResource::Find("/histogram_timeseries.json")); - } - - Y_UNIT_TEST(HistogramTimeSeriesDecode) { - NProto::TMultiSamplesList samples; - { - IMetricEncoderPtr e = EncoderProtobuf(&samples); - - TString testJson = NResource::Find("/histogram_timeseries.json"); - DecodeJson(testJson, e.Get()); - } - - UNIT_ASSERT_VALUES_EQUAL(1, samples.SamplesSize()); - const NProto::TMultiSample& s = samples.GetSamples(0); - - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::HIST_RATE); - UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); - AssertLabelEqual(s.GetLabels(0), "metric", "responseTimeMillis"); - - UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 2); - - auto h = ExponentialHistogram(6, 2); - for (i64 i = 1; i < 100; i++) { - h->Collect(i); - } - - AssertPointEqual(s.GetPoints(0), now, *h->Snapshot()); - - for (i64 i = 1; i < 100; i++) { - h->Collect(i); - } - - AssertPointEqual(s.GetPoints(1), now + TDuration::Seconds(15), *h->Snapshot()); - } - - Y_UNIT_TEST(IntGaugeEncode) { - auto writeDocument = [](IMetricEncoder* e) { - e->OnStreamBegin(); - { - e->OnMetricBegin(EMetricType::IGAUGE); - { - e->OnLabelsBegin(); - e->OnLabel("metric", "a"); - e->OnLabelsEnd(); - } - e->OnInt64(now, Min<i64>()); - e->OnInt64(now + TDuration::Seconds(1), -1); - e->OnInt64(now + TDuration::Seconds(2), 0); - e->OnInt64(now + TDuration::Seconds(3), Max<i64>()); - e->OnMetricEnd(); - } - e->OnStreamEnd(); - }; - - TString result1 = EncodeToString(EncoderJson, writeDocument); - UNIT_ASSERT_NO_DIFF(result1, NResource::Find("/int_gauge.json")); - - TString result2 = EncodeToString(BufferedEncoderJson, writeDocument); - UNIT_ASSERT_NO_DIFF(result2, NResource::Find("/int_gauge.json")); - } - + Y_UNIT_TEST(HistogramTimeSeriesEncode) { + auto writeDocument = [](IMetricEncoder* e) { + e->OnStreamBegin(); + { + e->OnMetricBegin(EMetricType::HIST_RATE); + { + e->OnLabelsBegin(); + e->OnLabel("metric", "responseTimeMillis"); + e->OnLabelsEnd(); + } + + // {1: 1, 2: 1, 4: 2, 8: 4, 16: 8, inf: 83} + auto h = ExponentialHistogram(6, 2); + for (i64 i = 1; i < 100; i++) { + h->Collect(i); + } + e->OnHistogram(now, h->Snapshot()); + + // {1: 2, 2: 2, 4: 4, 8: 8, 16: 16, inf: 166} + for (i64 i = 1; i < 100; i++) { + h->Collect(i); + } + e->OnHistogram(now + TDuration::Seconds(15), h->Snapshot()); + + e->OnMetricEnd(); + } + e->OnStreamEnd(); + }; + + TString result1 = EncodeToString(EncoderJson, writeDocument); + UNIT_ASSERT_NO_DIFF(result1, NResource::Find("/histogram_timeseries.json")); + + TString result2 = EncodeToString(BufferedEncoderJson, writeDocument); + UNIT_ASSERT_NO_DIFF(result2, NResource::Find("/histogram_timeseries.json")); + } + + Y_UNIT_TEST(HistogramTimeSeriesDecode) { + NProto::TMultiSamplesList samples; + { + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + TString testJson = NResource::Find("/histogram_timeseries.json"); + DecodeJson(testJson, e.Get()); + } + + UNIT_ASSERT_VALUES_EQUAL(1, samples.SamplesSize()); + const NProto::TMultiSample& s = samples.GetSamples(0); + + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::HIST_RATE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "metric", "responseTimeMillis"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 2); + + auto h = ExponentialHistogram(6, 2); + for (i64 i = 1; i < 100; i++) { + h->Collect(i); + } + + AssertPointEqual(s.GetPoints(0), now, *h->Snapshot()); + + for (i64 i = 1; i < 100; i++) { + h->Collect(i); + } + + AssertPointEqual(s.GetPoints(1), now + TDuration::Seconds(15), *h->Snapshot()); + } + + Y_UNIT_TEST(IntGaugeEncode) { + auto writeDocument = [](IMetricEncoder* e) { + e->OnStreamBegin(); + { + e->OnMetricBegin(EMetricType::IGAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("metric", "a"); + e->OnLabelsEnd(); + } + e->OnInt64(now, Min<i64>()); + e->OnInt64(now + TDuration::Seconds(1), -1); + e->OnInt64(now + TDuration::Seconds(2), 0); + e->OnInt64(now + TDuration::Seconds(3), Max<i64>()); + e->OnMetricEnd(); + } + e->OnStreamEnd(); + }; + + TString result1 = EncodeToString(EncoderJson, writeDocument); + UNIT_ASSERT_NO_DIFF(result1, NResource::Find("/int_gauge.json")); + + TString result2 = EncodeToString(BufferedEncoderJson, writeDocument); + UNIT_ASSERT_NO_DIFF(result2, NResource::Find("/int_gauge.json")); + } + Y_UNIT_TEST(InconsistentMetricTypes) { auto emitMetrics = [](IMetricEncoder& encoder, const TString& expectedError) { encoder.OnMetricBegin(EMetricType::GAUGE); @@ -1213,33 +1213,33 @@ Y_UNIT_TEST_SUITE(TJsonTest) { } } - Y_UNIT_TEST(IntGaugeDecode) { - NProto::TMultiSamplesList samples; - { - IMetricEncoderPtr e = EncoderProtobuf(&samples); - - TString testJson = NResource::Find("/int_gauge.json"); - DecodeJson(testJson, e.Get()); - } - - UNIT_ASSERT_VALUES_EQUAL(1, samples.SamplesSize()); - const NProto::TMultiSample& s = samples.GetSamples(0); - - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::IGAUGE); - UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); - AssertLabelEqual(s.GetLabels(0), "metric", "a"); - - UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 4); - AssertPointEqual(s.GetPoints(0), now, Min<i64>()); - AssertPointEqual(s.GetPoints(1), now + TDuration::Seconds(1), i64(-1)); - AssertPointEqual(s.GetPoints(2), now + TDuration::Seconds(2), i64(0)); - AssertPointEqual(s.GetPoints(3), now + TDuration::Seconds(3), Max<i64>()); - } + Y_UNIT_TEST(IntGaugeDecode) { + NProto::TMultiSamplesList samples; + { + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + TString testJson = NResource::Find("/int_gauge.json"); + DecodeJson(testJson, e.Get()); + } + + UNIT_ASSERT_VALUES_EQUAL(1, samples.SamplesSize()); + const NProto::TMultiSample& s = samples.GetSamples(0); + + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::IGAUGE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "metric", "a"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 4); + AssertPointEqual(s.GetPoints(0), now, Min<i64>()); + AssertPointEqual(s.GetPoints(1), now + TDuration::Seconds(1), i64(-1)); + AssertPointEqual(s.GetPoints(2), now + TDuration::Seconds(2), i64(0)); + AssertPointEqual(s.GetPoints(3), now + TDuration::Seconds(3), Max<i64>()); + } Y_UNIT_TEST(FuzzerRegression) { NProto::TMultiSamplesList samples; { - IMetricEncoderPtr e = EncoderProtobuf(&samples); + IMetricEncoderPtr e = EncoderProtobuf(&samples); for (auto f : { "/hist_crash.json", "/crash.json" }) { TString testJson = NResource::Find(f); @@ -1254,13 +1254,13 @@ Y_UNIT_TEST_SUITE(TJsonTest) { { "mode": "deriv", "value": -1, - "labels": { "metric": "SystemTime" } + "labels": { "metric": "SystemTime" } }, } ]}")"; NProto::TMultiSamplesList samples; - IMetricEncoderPtr e = EncoderProtobuf(&samples); + IMetricEncoderPtr e = EncoderProtobuf(&samples); UNIT_ASSERT_EXCEPTION(DecodeJson(input, e.Get()), yexception); } @@ -1287,4 +1287,4 @@ Y_UNIT_TEST_SUITE(TJsonTest) { } } -} +} diff --git a/library/cpp/monlib/encode/json/typed_point.h b/library/cpp/monlib/encode/json/typed_point.h index b606c82924..fbaa840c4b 100644 --- a/library/cpp/monlib/encode/json/typed_point.h +++ b/library/cpp/monlib/encode/json/typed_point.h @@ -1,123 +1,123 @@ -#pragma once - -#include <library/cpp/monlib/metrics/metric_value.h> - - -namespace NMonitoring { - - class TTypedPoint { - public: - TTypedPoint() - : Time_(TInstant::Zero()) - , ValueType_(EMetricValueType::UNKNOWN) - { - } - - template <typename T> - TTypedPoint(TInstant time, T value) - : Time_(time) - , ValueType_(TValueType<T>::Type) - , Value_(value) - { - Ref(); - } - - ~TTypedPoint() { - UnRef(); - } - - TTypedPoint(const TTypedPoint& rhs) - : Time_(rhs.Time_) - , ValueType_(rhs.ValueType_) - , Value_(rhs.Value_) - { - Ref(); - } - - TTypedPoint& operator=(const TTypedPoint& rhs) { - UnRef(); - - Time_ = rhs.Time_; - ValueType_ = rhs.ValueType_; - Value_ = rhs.Value_; - - Ref(); - return *this; - } - +#pragma once + +#include <library/cpp/monlib/metrics/metric_value.h> + + +namespace NMonitoring { + + class TTypedPoint { + public: + TTypedPoint() + : Time_(TInstant::Zero()) + , ValueType_(EMetricValueType::UNKNOWN) + { + } + + template <typename T> + TTypedPoint(TInstant time, T value) + : Time_(time) + , ValueType_(TValueType<T>::Type) + , Value_(value) + { + Ref(); + } + + ~TTypedPoint() { + UnRef(); + } + + TTypedPoint(const TTypedPoint& rhs) + : Time_(rhs.Time_) + , ValueType_(rhs.ValueType_) + , Value_(rhs.Value_) + { + Ref(); + } + + TTypedPoint& operator=(const TTypedPoint& rhs) { + UnRef(); + + Time_ = rhs.Time_; + ValueType_ = rhs.ValueType_; + Value_ = rhs.Value_; + + Ref(); + return *this; + } + TTypedPoint(TTypedPoint&& rhs) noexcept - : Time_(rhs.Time_) - , ValueType_(rhs.ValueType_) - , Value_(rhs.Value_) - { - rhs.ValueType_ = EMetricValueType::UNKNOWN; - rhs.Value_ = {}; - } - + : Time_(rhs.Time_) + , ValueType_(rhs.ValueType_) + , Value_(rhs.Value_) + { + rhs.ValueType_ = EMetricValueType::UNKNOWN; + rhs.Value_ = {}; + } + TTypedPoint& operator=(TTypedPoint&& rhs) noexcept { - UnRef(); - - Time_ = rhs.Time_; - ValueType_ = rhs.ValueType_; - Value_ = rhs.Value_; - - rhs.ValueType_ = EMetricValueType::UNKNOWN; - rhs.Value_ = {}; - - return *this; - } - - TInstant GetTime() const noexcept { - return Time_; - } - - void SetTime(TInstant time) noexcept { - Time_ = time; - } - - TMetricValue GetValue() const noexcept { - return Value_; - } - - EMetricValueType GetValueType() const noexcept { - return ValueType_; - } - - template <typename T> - void SetValue(T value) noexcept { - ValueType_ = TValueType<T>::Type; - Value_ = TMetricValue{value}; - } - - bool HasValue() { - return ValueType_ != EMetricValueType::UNKNOWN; - } - - private: - void Ref() { - if (ValueType_ == EMetricValueType::HISTOGRAM) { - Value_.AsHistogram()->Ref(); - } else if (ValueType_ == EMetricValueType::SUMMARY) { + UnRef(); + + Time_ = rhs.Time_; + ValueType_ = rhs.ValueType_; + Value_ = rhs.Value_; + + rhs.ValueType_ = EMetricValueType::UNKNOWN; + rhs.Value_ = {}; + + return *this; + } + + TInstant GetTime() const noexcept { + return Time_; + } + + void SetTime(TInstant time) noexcept { + Time_ = time; + } + + TMetricValue GetValue() const noexcept { + return Value_; + } + + EMetricValueType GetValueType() const noexcept { + return ValueType_; + } + + template <typename T> + void SetValue(T value) noexcept { + ValueType_ = TValueType<T>::Type; + Value_ = TMetricValue{value}; + } + + bool HasValue() { + return ValueType_ != EMetricValueType::UNKNOWN; + } + + private: + void Ref() { + if (ValueType_ == EMetricValueType::HISTOGRAM) { + Value_.AsHistogram()->Ref(); + } else if (ValueType_ == EMetricValueType::SUMMARY) { Value_.AsSummaryDouble()->Ref(); } else if (ValueType_ == EMetricValueType::LOGHISTOGRAM) { Value_.AsLogHistogram()->Ref(); } - } - - void UnRef() { - if (ValueType_ == EMetricValueType::HISTOGRAM) { - Value_.AsHistogram()->UnRef(); - } else if (ValueType_ == EMetricValueType::SUMMARY) { + } + + void UnRef() { + if (ValueType_ == EMetricValueType::HISTOGRAM) { + Value_.AsHistogram()->UnRef(); + } else if (ValueType_ == EMetricValueType::SUMMARY) { Value_.AsSummaryDouble()->UnRef(); } else if (ValueType_ == EMetricValueType::LOGHISTOGRAM) { Value_.AsLogHistogram()->UnRef(); } - } - - private: - TInstant Time_; - EMetricValueType ValueType_; - TMetricValue Value_; - }; - -} + } + + private: + TInstant Time_; + EMetricValueType ValueType_; + TMetricValue Value_; + }; + +} diff --git a/library/cpp/monlib/encode/json/ut/expected.json b/library/cpp/monlib/encode/json/ut/expected.json index 8541479be6..ead853455b 100644 --- a/library/cpp/monlib/encode/json/ut/expected.json +++ b/library/cpp/monlib/encode/json/ut/expected.json @@ -1,42 +1,42 @@ -{ - "ts":1500000000, - "commonLabels": - { - "project":"solomon" - }, - "sensors": - [ - { - "kind":"COUNTER", - "labels": - { - "metric":"single", +{ + "ts":1500000000, + "commonLabels": + { + "project":"solomon" + }, + "sensors": + [ + { + "kind":"COUNTER", + "labels": + { + "metric":"single", "labels":"l1" - }, - "ts":1509843723, - "value":17 - }, - { - "kind":"RATE", - "labels": - { - "metric":"single", + }, + "ts":1509843723, + "value":17 + }, + { + "kind":"RATE", + "labels": + { + "metric":"single", "labels":"l2" - }, - "ts":1509843723, - "value":17 - }, - { - "kind":"GAUGE", - "labels": - { - "metric":"single", + }, + "ts":1509843723, + "value":17 + }, + { + "kind":"GAUGE", + "labels": + { + "metric":"single", "labels":"l3" - }, - "ts":1509843723, - "value":3.14 - }, - { + }, + "ts":1509843723, + "value":3.14 + }, + { "kind":"IGAUGE", "labels": { @@ -47,46 +47,46 @@ "value":42 }, { - "kind":"GAUGE", - "labels": - { - "metric":"multiple", + "kind":"GAUGE", + "labels": + { + "metric":"multiple", "labels":"l5" - }, - "timeseries": - [ - { - "ts":1509843723, - "value":"nan" - }, - { - "ts":1509843738, - "value":"inf" - }, - { - "ts":1509843753, - "value":"-inf" - } - ] - }, - { - "kind":"COUNTER", - "timeseries": - [ - { - "ts":1509843723, - "value":1337 - }, - { - "ts":1509843738, - "value":1338 - } - ], - "labels": - { - "metric":"multiple", + }, + "timeseries": + [ + { + "ts":1509843723, + "value":"nan" + }, + { + "ts":1509843738, + "value":"inf" + }, + { + "ts":1509843753, + "value":"-inf" + } + ] + }, + { + "kind":"COUNTER", + "timeseries": + [ + { + "ts":1509843723, + "value":1337 + }, + { + "ts":1509843738, + "value":1338 + } + ], + "labels": + { + "metric":"multiple", "labels":"l6" - } - } - ] -} + } + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/expected_buffered.json b/library/cpp/monlib/encode/json/ut/expected_buffered.json index 17c0f06323..9a6a1d6201 100644 --- a/library/cpp/monlib/encode/json/ut/expected_buffered.json +++ b/library/cpp/monlib/encode/json/ut/expected_buffered.json @@ -1,42 +1,42 @@ -{ - "ts":1500000000, - "commonLabels": - { - "project":"solomon" - }, - "sensors": - [ - { - "kind":"COUNTER", - "labels": - { +{ + "ts":1500000000, + "commonLabels": + { + "project":"solomon" + }, + "sensors": + [ + { + "kind":"COUNTER", + "labels": + { "labels":"l1", "metric":"single" - }, - "ts":1509843723, - "value":17 - }, - { - "kind":"RATE", - "labels": - { + }, + "ts":1509843723, + "value":17 + }, + { + "kind":"RATE", + "labels": + { "labels":"l2", "metric":"single" - }, - "ts":1509843723, - "value":17 - }, - { - "kind":"GAUGE", - "labels": - { + }, + "ts":1509843723, + "value":17 + }, + { + "kind":"GAUGE", + "labels": + { "labels":"l3", "metric":"single" - }, - "ts":1509843723, - "value":3.14 - }, - { + }, + "ts":1509843723, + "value":3.14 + }, + { "kind":"IGAUGE", "labels": { @@ -47,46 +47,46 @@ "value":42 }, { - "kind":"GAUGE", - "labels": - { + "kind":"GAUGE", + "labels": + { "labels":"l5", "metric":"multiple" - }, - "timeseries": - [ - { - "ts":1509843723, - "value":"nan" - }, - { - "ts":1509843738, - "value":"inf" - }, - { - "ts":1509843753, - "value":"-inf" - } - ] - }, - { - "kind":"COUNTER", + }, + "timeseries": + [ + { + "ts":1509843723, + "value":"nan" + }, + { + "ts":1509843738, + "value":"inf" + }, + { + "ts":1509843753, + "value":"-inf" + } + ] + }, + { + "kind":"COUNTER", "labels": { "labels":"l6", "metric":"multiple" }, - "timeseries": - [ - { - "ts":1509843723, - "value":1337 - }, - { - "ts":1509843738, - "value":1338 - } + "timeseries": + [ + { + "ts":1509843723, + "value":1337 + }, + { + "ts":1509843738, + "value":1338 + } ] - } - ] -} + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/expected_cloud.json b/library/cpp/monlib/encode/json/ut/expected_cloud.json index 5c4dc65ffc..6184811579 100644 --- a/library/cpp/monlib/encode/json/ut/expected_cloud.json +++ b/library/cpp/monlib/encode/json/ut/expected_cloud.json @@ -1,47 +1,47 @@ -{ +{ "ts":"2017-07-14T02:40:00.000000Z", "labels": - { - "project":"solomon" - }, + { + "project":"solomon" + }, "metrics": - [ - { + [ + { "type":"COUNTER", - "labels": - { + "labels": + { "labels":"l1" - }, + }, "name":"single", "ts":"2017-11-05T01:02:03.000000Z", - "value":17 - }, - { + "value":17 + }, + { "type":"RATE", - "labels": - { + "labels": + { "labels":"l2" - }, + }, "name":"single", "ts":"2017-11-05T01:02:03.000000Z", - "value":17 - }, - { + "value":17 + }, + { "type":"DGAUGE", - "labels": - { + "labels": + { "labels":"l3" - }, + }, "name":"single", "ts":"2017-11-05T01:02:03.000000Z", - "value":3.14 - }, - { + "value":3.14 + }, + { "type":"IGAUGE", - "labels": - { + "labels": + { "labels":"l4" - }, + }, "name":"single_igauge", "ts":"2017-11-05T01:02:03.000000Z", "value":42 @@ -53,40 +53,40 @@ "labels":"l5" }, "name":"multiple", - "timeseries": - [ - { + "timeseries": + [ + { "ts":"2017-11-05T01:02:03.000000Z", - "value":"nan" - }, - { + "value":"nan" + }, + { "ts":"2017-11-05T01:02:18.000000Z", - "value":"inf" - }, - { + "value":"inf" + }, + { "ts":"2017-11-05T01:02:33.000000Z", - "value":"-inf" - } - ] - }, - { + "value":"-inf" + } + ] + }, + { "type":"COUNTER", - "timeseries": - [ - { + "timeseries": + [ + { "ts":"2017-11-05T01:02:03.000000Z", - "value":1337 - }, - { + "value":1337 + }, + { "ts":"2017-11-05T01:02:18.000000Z", - "value":1338 - } - ], - "labels": - { + "value":1338 + } + ], + "labels": + { "labels":"l6" }, "name":"multiple" - } - ] -} + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/histogram_timeseries.json b/library/cpp/monlib/encode/json/ut/histogram_timeseries.json index c3a6f8e8b0..f6131ffded 100644 --- a/library/cpp/monlib/encode/json/ut/histogram_timeseries.json +++ b/library/cpp/monlib/encode/json/ut/histogram_timeseries.json @@ -1,61 +1,61 @@ -{ - "sensors": - [ - { - "kind":"HIST_RATE", - "labels": - { - "metric":"responseTimeMillis" - }, - "timeseries": - [ - { - "ts":1509843723, - "hist": - { - "bounds": - [ - 1, - 2, - 4, - 8, - 16 - ], - "buckets": - [ - 1, - 1, - 2, - 4, - 8 - ], - "inf":83 - } - }, - { - "ts":1509843738, - "hist": - { - "bounds": - [ - 1, - 2, - 4, - 8, - 16 - ], - "buckets": - [ - 2, - 2, - 4, - 8, - 16 - ], - "inf":166 - } - } - ] - } - ] -} +{ + "sensors": + [ + { + "kind":"HIST_RATE", + "labels": + { + "metric":"responseTimeMillis" + }, + "timeseries": + [ + { + "ts":1509843723, + "hist": + { + "bounds": + [ + 1, + 2, + 4, + 8, + 16 + ], + "buckets": + [ + 1, + 1, + 2, + 4, + 8 + ], + "inf":83 + } + }, + { + "ts":1509843738, + "hist": + { + "bounds": + [ + 1, + 2, + 4, + 8, + 16 + ], + "buckets": + [ + 2, + 2, + 4, + 8, + 16 + ], + "inf":166 + } + } + ] + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/histogram_value.json b/library/cpp/monlib/encode/json/ut/histogram_value.json index 91b4ffa3dd..ec1ae5cdec 100644 --- a/library/cpp/monlib/encode/json/ut/histogram_value.json +++ b/library/cpp/monlib/encode/json/ut/histogram_value.json @@ -1,33 +1,33 @@ -{ - "sensors": - [ - { - "kind":"HIST", - "labels": - { - "metric":"responseTimeMillis" - }, - "ts":1509843723, - "hist": - { - "bounds": - [ - 1, - 2, - 4, - 8, - 16 - ], - "buckets": - [ - 1, - 1, - 2, - 4, - 8 - ], - "inf":83 - } - } - ] -} +{ + "sensors": + [ + { + "kind":"HIST", + "labels": + { + "metric":"responseTimeMillis" + }, + "ts":1509843723, + "hist": + { + "bounds": + [ + 1, + 2, + 4, + 8, + 16 + ], + "buckets": + [ + 1, + 1, + 2, + 4, + 8 + ], + "inf":83 + } + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/histogram_value_inf_before_bounds.json b/library/cpp/monlib/encode/json/ut/histogram_value_inf_before_bounds.json index 87dc527985..f8a17c8831 100644 --- a/library/cpp/monlib/encode/json/ut/histogram_value_inf_before_bounds.json +++ b/library/cpp/monlib/encode/json/ut/histogram_value_inf_before_bounds.json @@ -5,7 +5,7 @@ "kind":"HIST", "labels": { - "metric":"responseTimeMillis" + "metric":"responseTimeMillis" }, "ts":1509843723, "hist": diff --git a/library/cpp/monlib/encode/json/ut/int_gauge.json b/library/cpp/monlib/encode/json/ut/int_gauge.json index 95aa12aa52..fbe57f873c 100644 --- a/library/cpp/monlib/encode/json/ut/int_gauge.json +++ b/library/cpp/monlib/encode/json/ut/int_gauge.json @@ -1,31 +1,31 @@ -{ - "sensors": - [ - { - "kind":"IGAUGE", - "labels": - { - "metric":"a" - }, - "timeseries": - [ - { - "ts":1509843723, - "value":-9223372036854775808 - }, - { - "ts":1509843724, - "value":-1 - }, - { - "ts":1509843725, - "value":0 - }, - { - "ts":1509843726, - "value":9223372036854775807 - } - ] - } - ] -} +{ + "sensors": + [ + { + "kind":"IGAUGE", + "labels": + { + "metric":"a" + }, + "timeseries": + [ + { + "ts":1509843723, + "value":-9223372036854775808 + }, + { + "ts":1509843724, + "value":-1 + }, + { + "ts":1509843725, + "value":0 + }, + { + "ts":1509843726, + "value":9223372036854775807 + } + ] + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/log_histogram_timeseries.json b/library/cpp/monlib/encode/json/ut/log_histogram_timeseries.json index 1e2a4a113c..e811a2cc57 100644 --- a/library/cpp/monlib/encode/json/ut/log_histogram_timeseries.json +++ b/library/cpp/monlib/encode/json/ut/log_histogram_timeseries.json @@ -5,7 +5,7 @@ "kind":"LOGHIST", "labels": { - "metric":"ms" + "metric":"ms" }, "timeseries": [ diff --git a/library/cpp/monlib/encode/json/ut/log_histogram_value.json b/library/cpp/monlib/encode/json/ut/log_histogram_value.json index 9835221116..002478293b 100644 --- a/library/cpp/monlib/encode/json/ut/log_histogram_value.json +++ b/library/cpp/monlib/encode/json/ut/log_histogram_value.json @@ -5,7 +5,7 @@ "kind":"LOGHIST", "labels": { - "metric":"ms" + "metric":"ms" }, "ts":1509843723, "log_hist": diff --git a/library/cpp/monlib/encode/json/ut/merged.json b/library/cpp/monlib/encode/json/ut/merged.json index 15711471f0..ea2c99a33c 100644 --- a/library/cpp/monlib/encode/json/ut/merged.json +++ b/library/cpp/monlib/encode/json/ut/merged.json @@ -5,7 +5,7 @@ "kind":"RATE", "labels": { - "metric":"hello", + "metric":"hello", "label":"world" }, "value":1 diff --git a/library/cpp/monlib/encode/json/ut/metrics.json b/library/cpp/monlib/encode/json/ut/metrics.json index 0dc423d5e9..2be4617d51 100644 --- a/library/cpp/monlib/encode/json/ut/metrics.json +++ b/library/cpp/monlib/encode/json/ut/metrics.json @@ -1,43 +1,43 @@ -{ - "labels": { - "project": "solomon", - "cluster": "man", - "service": "stockpile" - }, - "metrics": [ - { - "type": "DGAUGE", - "labels": { - "metric": "Memory" - }, - "value": 10 - }, - { - "type": "RATE", - "value": 1, - "labels": { "metric": "UserTime" } - }, - { - "type": "GAUGE", - "value": 3.14159, - "labels": { "export": "Oxygen", "metric": "QueueSize" }, - "ts": "2017-11-05T12:34:56.000Z", - "memOnly": true - }, - { - "type": "GAUGE", - "labels": { "metric": "Writes" }, - "timeseries": [ - { - "ts": "2017-08-28T12:32:11Z", - "value": -10 - }, - { - "value": 20, - "ts": 1503923187 - } - ] - } - ], - "ts": "2017-08-27T12:34:56Z" -} +{ + "labels": { + "project": "solomon", + "cluster": "man", + "service": "stockpile" + }, + "metrics": [ + { + "type": "DGAUGE", + "labels": { + "metric": "Memory" + }, + "value": 10 + }, + { + "type": "RATE", + "value": 1, + "labels": { "metric": "UserTime" } + }, + { + "type": "GAUGE", + "value": 3.14159, + "labels": { "export": "Oxygen", "metric": "QueueSize" }, + "ts": "2017-11-05T12:34:56.000Z", + "memOnly": true + }, + { + "type": "GAUGE", + "labels": { "metric": "Writes" }, + "timeseries": [ + { + "ts": "2017-08-28T12:32:11Z", + "value": -10 + }, + { + "value": 20, + "ts": 1503923187 + } + ] + } + ], + "ts": "2017-08-27T12:34:56Z" +} diff --git a/library/cpp/monlib/encode/json/ut/sensors.json b/library/cpp/monlib/encode/json/ut/sensors.json index e128593492..4d979a3c1e 100644 --- a/library/cpp/monlib/encode/json/ut/sensors.json +++ b/library/cpp/monlib/encode/json/ut/sensors.json @@ -1,40 +1,40 @@ -{ - "commonLabels": { - "project": "solomon", - "cluster": "man", - "service": "stockpile" - }, - "sensors": [ - { - "labels": { - "metric": "Memory" - }, - "value": 10 - }, - { - "mode": "deriv", - "value": 1, - "labels": { "metric": "UserTime" } - }, - { - "value": 3.14159, - "labels": { "export": "Oxygen", "metric": "QueueSize" }, - "ts": "2017-11-05T12:34:56.000Z", - "memOnly": true - }, - { - "labels": { "metric": "Writes" }, - "timeseries": [ - { - "ts": "2017-08-28T12:32:11Z", - "value": -10 - }, - { - "value": 20, - "ts": 1503923187 - } - ] - } - ], - "ts": "2017-08-27T12:34:56Z" -} +{ + "commonLabels": { + "project": "solomon", + "cluster": "man", + "service": "stockpile" + }, + "sensors": [ + { + "labels": { + "metric": "Memory" + }, + "value": 10 + }, + { + "mode": "deriv", + "value": 1, + "labels": { "metric": "UserTime" } + }, + { + "value": 3.14159, + "labels": { "export": "Oxygen", "metric": "QueueSize" }, + "ts": "2017-11-05T12:34:56.000Z", + "memOnly": true + }, + { + "labels": { "metric": "Writes" }, + "timeseries": [ + { + "ts": "2017-08-28T12:32:11Z", + "value": -10 + }, + { + "value": 20, + "ts": 1503923187 + } + ] + } + ], + "ts": "2017-08-27T12:34:56Z" +} diff --git a/library/cpp/monlib/encode/json/ut/summary_inf.json b/library/cpp/monlib/encode/json/ut/summary_inf.json index 067102b7da..625a6cd8ad 100644 --- a/library/cpp/monlib/encode/json/ut/summary_inf.json +++ b/library/cpp/monlib/encode/json/ut/summary_inf.json @@ -5,7 +5,7 @@ "kind":"DSUMMARY", "labels": { - "metric":"temperature" + "metric":"temperature" }, "ts":1509843723, "summary": diff --git a/library/cpp/monlib/encode/json/ut/summary_timeseries.json b/library/cpp/monlib/encode/json/ut/summary_timeseries.json index dd06ec8544..92007af3e6 100644 --- a/library/cpp/monlib/encode/json/ut/summary_timeseries.json +++ b/library/cpp/monlib/encode/json/ut/summary_timeseries.json @@ -5,7 +5,7 @@ "kind":"DSUMMARY", "labels": { - "metric":"temperature" + "metric":"temperature" }, "timeseries": [ diff --git a/library/cpp/monlib/encode/json/ut/summary_value.json b/library/cpp/monlib/encode/json/ut/summary_value.json index 2ad9e9ad39..366394c5e1 100644 --- a/library/cpp/monlib/encode/json/ut/summary_value.json +++ b/library/cpp/monlib/encode/json/ut/summary_value.json @@ -5,7 +5,7 @@ "kind":"DSUMMARY", "labels": { - "metric":"temperature" + "metric":"temperature" }, "ts":1509843723, "summary": diff --git a/library/cpp/monlib/encode/json/ut/test_decode_to_encode.json b/library/cpp/monlib/encode/json/ut/test_decode_to_encode.json index db1e00c6e9..65f0c5c6e2 100644 --- a/library/cpp/monlib/encode/json/ut/test_decode_to_encode.json +++ b/library/cpp/monlib/encode/json/ut/test_decode_to_encode.json @@ -7,7 +7,7 @@ "sensors": [ { "kind": "GAUGE", - "labels": { "export": "Oxygen", "metric": "QueueSize" }, + "labels": { "export": "Oxygen", "metric": "QueueSize" }, "ts": 1509885296, "value": 3.14159 } diff --git a/library/cpp/monlib/encode/json/ut/ya.make b/library/cpp/monlib/encode/json/ut/ya.make index 4a24f76650..e50c4f4903 100644 --- a/library/cpp/monlib/encode/json/ut/ya.make +++ b/library/cpp/monlib/encode/json/ut/ya.make @@ -1,30 +1,30 @@ -UNITTEST_FOR(library/cpp/monlib/encode/json) - +UNITTEST_FOR(library/cpp/monlib/encode/json) + OWNER( g:solomon jamel ) - -SRCS( + +SRCS( json_decoder_ut.cpp - json_ut.cpp -) - -RESOURCE( + json_ut.cpp +) + +RESOURCE( buffered_test.json /buffered_test.json buffered_ts_merge.json /buffered_ts_merge.json - empty_series.json /empty_series.json - expected.json /expected.json + empty_series.json /empty_series.json + expected.json /expected.json expected_buffered.json /expected_buffered.json expected_cloud.json /expected_cloud.json expected_cloud_buffered.json /expected_cloud_buffered.json merged.json /merged.json - histogram_timeseries.json /histogram_timeseries.json - histogram_value.json /histogram_value.json + histogram_timeseries.json /histogram_timeseries.json + histogram_value.json /histogram_value.json histogram_value_inf_before_bounds.json /histogram_value_inf_before_bounds.json - int_gauge.json /int_gauge.json - sensors.json /sensors.json - metrics.json /metrics.json + int_gauge.json /int_gauge.json + sensors.json /sensors.json + metrics.json /metrics.json named_metrics.json /named_metrics.json test_decode_to_encode.json /test_decode_to_encode.json crash.json /crash.json @@ -34,13 +34,13 @@ RESOURCE( summary_timeseries.json /summary_timeseries.json log_histogram_value.json /log_histogram_value.json log_histogram_timeseries.json /log_histogram_timeseries.json -) - -PEERDIR( +) + +PEERDIR( library/cpp/json library/cpp/monlib/consumers - library/cpp/monlib/encode/protobuf + library/cpp/monlib/encode/protobuf library/cpp/resource -) - -END() +) + +END() diff --git a/library/cpp/monlib/encode/json/ya.make b/library/cpp/monlib/encode/json/ya.make index 221fbc2a6a..a50fc412a9 100644 --- a/library/cpp/monlib/encode/json/ya.make +++ b/library/cpp/monlib/encode/json/ya.make @@ -1,21 +1,21 @@ -LIBRARY() - +LIBRARY() + OWNER( g:solomon jamel ) - -SRCS( - json_decoder.cpp - json_encoder.cpp -) - -PEERDIR( + +SRCS( + json_decoder.cpp + json_encoder.cpp +) + +PEERDIR( library/cpp/monlib/encode library/cpp/monlib/encode/buffered library/cpp/monlib/exception library/cpp/json library/cpp/json/writer -) - -END() +) + +END() diff --git a/library/cpp/monlib/encode/legacy_protobuf/legacy_proto_decoder.cpp b/library/cpp/monlib/encode/legacy_protobuf/legacy_proto_decoder.cpp index 4a305152e3..f87a2d7e8f 100644 --- a/library/cpp/monlib/encode/legacy_protobuf/legacy_proto_decoder.cpp +++ b/library/cpp/monlib/encode/legacy_protobuf/legacy_proto_decoder.cpp @@ -1,8 +1,8 @@ #include "legacy_protobuf.h" -#include <library/cpp/monlib/encode/legacy_protobuf/protos/metric_meta.pb.h> -#include <library/cpp/monlib/metrics/metric_consumer.h> -#include <library/cpp/monlib/metrics/labels.h> +#include <library/cpp/monlib/encode/legacy_protobuf/protos/metric_meta.pb.h> +#include <library/cpp/monlib/metrics/metric_consumer.h> +#include <library/cpp/monlib/metrics/labels.h> #include <util/generic/yexception.h> #include <util/generic/maybe.h> @@ -22,7 +22,7 @@ namespace NMonitoring { namespace { - using TMaybeMeta = TMaybe<NMonProto::TMetricMeta>; + using TMaybeMeta = TMaybe<NMonProto::TMetricMeta>; TString ReadLabelValue(const NProtoBuf::Message& msg, const NProtoBuf::FieldDescriptor* d, const NProtoBuf::Reflection& r) { using namespace NProtoBuf; @@ -137,8 +137,8 @@ namespace NMonitoring { } TMaybeMeta MaybeGetMeta(const NProtoBuf::FieldOptions& opts) { - if (opts.HasExtension(NMonProto::Metric)) { - return opts.GetExtension(NMonProto::Metric); + if (opts.HasExtension(NMonProto::Metric)) { + return opts.GetExtension(NMonProto::Metric); } return Nothing(); @@ -218,37 +218,37 @@ namespace NMonitoring { Reflection_ = nullptr; } - TDecoderContext CreateChildFromMeta(const NMonProto::TMetricMeta& metricMeta, const TString& name, i64 repeatedIdx = -1) { + TDecoderContext CreateChildFromMeta(const NMonProto::TMetricMeta& metricMeta, const TString& name, i64 repeatedIdx = -1) { TDecoderContext child{*this}; child.Clear(); - if (metricMeta.HasCustomPath()) { - if (const auto& nodePath = metricMeta.GetCustomPath()) { + if (metricMeta.HasCustomPath()) { + if (const auto& nodePath = metricMeta.GetCustomPath()) { child.AppendPath(nodePath); } - } else if (metricMeta.GetPath()) { + } else if (metricMeta.GetPath()) { child.AppendPath(name); } - if (metricMeta.HasKeys()) { - child.ParseKeys(metricMeta.GetKeys(), repeatedIdx); + if (metricMeta.HasKeys()) { + child.ParseKeys(metricMeta.GetKeys(), repeatedIdx); } return child; } - TDecoderContext CreateChildFromRepeatedScalar(const NMonProto::TMetricMeta& metricMeta, i64 repeatedIdx = -1) { + TDecoderContext CreateChildFromRepeatedScalar(const NMonProto::TMetricMeta& metricMeta, i64 repeatedIdx = -1) { TDecoderContext child{*this}; child.Clear(); - if (metricMeta.HasKeys()) { - child.ParseKeys(metricMeta.GetKeys(), repeatedIdx); + if (metricMeta.HasKeys()) { + child.ParseKeys(metricMeta.GetKeys(), repeatedIdx); } return child; } - TDecoderContext CreateChildFromEls(const TString& name, const NMonProto::TExtraLabelMetrics& metrics, size_t idx, TMaybeMeta maybeMeta) { + TDecoderContext CreateChildFromEls(const TString& name, const NMonProto::TExtraLabelMetrics& metrics, size_t idx, TMaybeMeta maybeMeta) { TDecoderContext child{*this}; child.Clear(); @@ -261,8 +261,8 @@ namespace NMonitoring { } child.Labels_.push_back(::MakeIntrusive<TLazyLabel>( - [ labelName = metrics.GetlabelName(), idx, &metrics ](const auto&) { - const auto& val = metrics.Getvalues(idx); + [ labelName = metrics.GetlabelName(), idx, &metrics ](const auto&) { + const auto& val = metrics.Getvalues(idx); TString labelVal; const auto uintLabel = val.GetlabelValueUint(); @@ -356,7 +356,7 @@ namespace NMonitoring { class TDecoder { public: - TDecoder(IMetricConsumer* consumer, const NProtoBuf::Message& message, TInstant timestamp) + TDecoder(IMetricConsumer* consumer, const NProtoBuf::Message& message, TInstant timestamp) : Consumer_{consumer} , Message_{message} , Timestamp_{timestamp} @@ -374,12 +374,12 @@ namespace NMonitoring { } private: - static const NMonProto::TExtraLabelMetrics& ExtractExtraMetrics(TDecoderContext& ctx, const NProtoBuf::FieldDescriptor& f) { + static const NMonProto::TExtraLabelMetrics& ExtractExtraMetrics(TDecoderContext& ctx, const NProtoBuf::FieldDescriptor& f) { const auto& parent = ctx.Message(); const auto& reflection = ctx.Reflection(); auto& subMessage = reflection.GetMessage(parent, &f); - return dynamic_cast<const NMonProto::TExtraLabelMetrics&>(subMessage); + return dynamic_cast<const NMonProto::TExtraLabelMetrics&>(subMessage); } void DecodeImpl(const NProtoBuf::Message& msg, TDecoderContext ctx) const { @@ -394,68 +394,68 @@ namespace NMonitoring { const auto& opts = f->options(); const auto isMessage = f->type() == NProtoBuf::FieldDescriptor::TYPE_MESSAGE; - const auto isExtraLabelMetrics = isMessage && f->message_type()->full_name() == "NMonProto.TExtraLabelMetrics"; + const auto isExtraLabelMetrics = isMessage && f->message_type()->full_name() == "NMonProto.TExtraLabelMetrics"; const auto maybeMeta = MaybeGetMeta(opts); - if (!(maybeMeta || isExtraLabelMetrics)) { + if (!(maybeMeta || isExtraLabelMetrics)) { continue; } - if (isExtraLabelMetrics) { - const auto& extra = ExtractExtraMetrics(ctx, *f); - RecurseExtraLabelMetrics(ctx, extra, f->name(), maybeMeta); + if (isExtraLabelMetrics) { + const auto& extra = ExtractExtraMetrics(ctx, *f); + RecurseExtraLabelMetrics(ctx, extra, f->name(), maybeMeta); } else if (isMessage) { RecurseMessage(ctx, *maybeMeta, *f); } else if (f->is_repeated()) { RecurseRepeatedScalar(ctx, *maybeMeta, *f); - } else if (maybeMeta->HasType()) { + } else if (maybeMeta->HasType()) { const auto val = ReadFieldAsDouble(msg, f, ctx.Reflection()); - const bool isRate = maybeMeta->GetType() == NMonProto::EMetricType::RATE; - WriteMetric(val, ctx, f->name(), isRate); + const bool isRate = maybeMeta->GetType() == NMonProto::EMetricType::RATE; + WriteMetric(val, ctx, f->name(), isRate); } } } - void RecurseRepeatedScalar(TDecoderContext ctx, const NMonProto::TMetricMeta& meta, const NProtoBuf::FieldDescriptor& f) const { + void RecurseRepeatedScalar(TDecoderContext ctx, const NMonProto::TMetricMeta& meta, const NProtoBuf::FieldDescriptor& f) const { auto&& msg = ctx.Message(); auto&& reflection = ctx.Reflection(); - const bool isRate = meta.GetType() == NMonProto::EMetricType::RATE; + const bool isRate = meta.GetType() == NMonProto::EMetricType::RATE; - // this is a repeated scalar field, which makes metric only if it's indexing + // this is a repeated scalar field, which makes metric only if it's indexing for (auto i = 0; i < reflection.FieldSize(msg, &f); ++i) { auto subCtx = ctx.CreateChildFromRepeatedScalar(meta, i); subCtx.Init(&msg); auto val = ReadRepeatedAsDouble(msg, &f, reflection, i); - WriteMetric(val, subCtx, f.name(), isRate); + WriteMetric(val, subCtx, f.name(), isRate); } } - void RecurseExtraLabelMetrics(TDecoderContext ctx, const NMonProto::TExtraLabelMetrics& msg, const TString& name, const TMaybeMeta& meta) const { + void RecurseExtraLabelMetrics(TDecoderContext ctx, const NMonProto::TExtraLabelMetrics& msg, const TString& name, const TMaybeMeta& meta) const { auto i = 0; for (const auto& val : msg.Getvalues()) { auto subCtx = ctx.CreateChildFromEls(name, msg, i++, meta); subCtx.Init(&val); - const bool isRate = val.Hastype() - ? val.Gettype() == NMonProto::EMetricType::RATE - : meta->GetType() == NMonProto::EMetricType::RATE; + const bool isRate = val.Hastype() + ? val.Gettype() == NMonProto::EMetricType::RATE + : meta->GetType() == NMonProto::EMetricType::RATE; - double metricVal{0}; - if (isRate) { - metricVal = val.GetlongValue(); + double metricVal{0}; + if (isRate) { + metricVal = val.GetlongValue(); } else { - metricVal = val.GetdoubleValue(); + metricVal = val.GetdoubleValue(); } - WriteMetric(metricVal, subCtx, "", isRate); + WriteMetric(metricVal, subCtx, "", isRate); for (const auto& child : val.Getchildren()) { - RecurseExtraLabelMetrics(subCtx, child, "", meta); + RecurseExtraLabelMetrics(subCtx, child, "", meta); } } } - void RecurseMessage(TDecoderContext ctx, const NMonProto::TMetricMeta& metricMeta, const NProtoBuf::FieldDescriptor& f) const { + void RecurseMessage(TDecoderContext ctx, const NMonProto::TMetricMeta& metricMeta, const NProtoBuf::FieldDescriptor& f) const { const auto& msg = ctx.Message(); const auto& reflection = ctx.Reflection(); @@ -463,12 +463,12 @@ namespace NMonitoring { TRACE("recurse into repeated message " << f.name()); for (auto i = 0; i < reflection.FieldSize(msg, &f); ++i) { auto& subMessage = reflection.GetRepeatedMessage(msg, &f, i); - DecodeImpl(subMessage, ctx.CreateChildFromMeta(metricMeta, f.name(), i)); + DecodeImpl(subMessage, ctx.CreateChildFromMeta(metricMeta, f.name(), i)); } } else { TRACE("recurse into message " << f.name()); auto& subMessage = reflection.GetMessage(msg, &f); - DecodeImpl(subMessage, ctx.CreateChildFromMeta(metricMeta, f.name())); + DecodeImpl(subMessage, ctx.CreateChildFromMeta(metricMeta, f.name())); } } @@ -480,12 +480,12 @@ namespace NMonitoring { Consumer_->OnDouble(Timestamp_, value); } - void WriteMetric(double value, const TDecoderContext& ctx, const TString& name, bool isRate) const { - if (isRate) { - Consumer_->OnMetricBegin(EMetricType::RATE); + void WriteMetric(double value, const TDecoderContext& ctx, const TString& name, bool isRate) const { + if (isRate) { + Consumer_->OnMetricBegin(EMetricType::RATE); WriteValue(static_cast<ui64>(value)); } else { - Consumer_->OnMetricBegin(EMetricType::GAUGE); + Consumer_->OnMetricBegin(EMetricType::GAUGE); WriteValue(static_cast<double>(value)); } @@ -504,23 +504,23 @@ namespace NMonitoring { } Consumer_->OnLabelsEnd(); - Consumer_->OnMetricEnd(); + Consumer_->OnMetricEnd(); } private: - IMetricConsumer* Consumer_{nullptr}; + IMetricConsumer* Consumer_{nullptr}; const NProtoBuf::Message& Message_; TInstant Timestamp_; }; } - void DecodeLegacyProto(const NProtoBuf::Message& data, IMetricConsumer* consumer, TInstant ts) { + void DecodeLegacyProto(const NProtoBuf::Message& data, IMetricConsumer* consumer, TInstant ts) { Y_ENSURE(consumer); TDecoder(consumer, data, ts).Decode(); } - void DecodeLegacyProtoToStream(const NProtoBuf::Message& data, IMetricConsumer* consumer, TInstant ts) { + void DecodeLegacyProtoToStream(const NProtoBuf::Message& data, IMetricConsumer* consumer, TInstant ts) { Y_ENSURE(consumer); TDecoder(consumer, data, ts).DecodeToStream(); } diff --git a/library/cpp/monlib/encode/legacy_protobuf/legacy_protobuf.h b/library/cpp/monlib/encode/legacy_protobuf/legacy_protobuf.h index 50f65bf2c8..7cf8985d65 100644 --- a/library/cpp/monlib/encode/legacy_protobuf/legacy_protobuf.h +++ b/library/cpp/monlib/encode/legacy_protobuf/legacy_protobuf.h @@ -9,8 +9,8 @@ namespace NMonitoring { // - memOnly; // - dropHost/ignorePath - void DecodeLegacyProto(const NProtoBuf::Message& data, class IMetricConsumer* c, TInstant ts = TInstant::Zero()); + void DecodeLegacyProto(const NProtoBuf::Message& data, class IMetricConsumer* c, TInstant ts = TInstant::Zero()); /// Does not open/close consumer stream unlike the above function. - void DecodeLegacyProtoToStream(const NProtoBuf::Message& data, class IMetricConsumer* c, TInstant ts = TInstant::Zero()); + void DecodeLegacyProtoToStream(const NProtoBuf::Message& data, class IMetricConsumer* c, TInstant ts = TInstant::Zero()); } diff --git a/library/cpp/monlib/encode/legacy_protobuf/legacy_protobuf_ut.cpp b/library/cpp/monlib/encode/legacy_protobuf/legacy_protobuf_ut.cpp index a751f8fb14..53683cb39c 100644 --- a/library/cpp/monlib/encode/legacy_protobuf/legacy_protobuf_ut.cpp +++ b/library/cpp/monlib/encode/legacy_protobuf/legacy_protobuf_ut.cpp @@ -2,12 +2,12 @@ #include <library/cpp/testing/unittest/registar.h> -#include <library/cpp/monlib/encode/legacy_protobuf/ut/test_cases.pb.h> -#include <library/cpp/monlib/encode/legacy_protobuf/protos/metric_meta.pb.h> +#include <library/cpp/monlib/encode/legacy_protobuf/ut/test_cases.pb.h> +#include <library/cpp/monlib/encode/legacy_protobuf/protos/metric_meta.pb.h> -#include <library/cpp/monlib/encode/protobuf/protobuf.h> -#include <library/cpp/monlib/encode/text/text.h> -#include <library/cpp/monlib/metrics/labels.h> +#include <library/cpp/monlib/encode/protobuf/protobuf.h> +#include <library/cpp/monlib/encode/text/text.h> +#include <library/cpp/monlib/metrics/labels.h> #include <util/generic/algorithm.h> #include <util/generic/hash_set.h> @@ -24,7 +24,7 @@ TSimple MakeSimpleMessage() { return msg; } -IMetricEncoderPtr debugPrinter = EncoderText(&Cerr); +IMetricEncoderPtr debugPrinter = EncoderText(&Cerr); namespace NMonitoring { inline bool operator<(const TLabel& lhs, const TLabel& rhs) { @@ -34,20 +34,20 @@ namespace NMonitoring { } -void SetLabelValue(NMonProto::TExtraLabelMetrics::TValue& val, TString s) { +void SetLabelValue(NMonProto::TExtraLabelMetrics::TValue& val, TString s) { val.SetlabelValue(s); } -void SetLabelValue(NMonProto::TExtraLabelMetrics::TValue& val, ui64 u) { +void SetLabelValue(NMonProto::TExtraLabelMetrics::TValue& val, ui64 u) { val.SetlabelValueUint(u); } template <typename T, typename V> -NMonProto::TExtraLabelMetrics MakeExtra(TString labelName, V labelValue, T value, bool isDeriv) { - NMonProto::TExtraLabelMetrics metric; - auto* val = metric.Addvalues(); +NMonProto::TExtraLabelMetrics MakeExtra(TString labelName, V labelValue, T value, bool isDeriv) { + NMonProto::TExtraLabelMetrics metric; + auto* val = metric.Addvalues(); - metric.SetlabelName(labelName); + metric.SetlabelName(labelName); SetLabelValue(*val, labelValue); if (isDeriv) { @@ -56,7 +56,7 @@ NMonProto::TExtraLabelMetrics MakeExtra(TString labelName, V labelValue, T value val->SetdoubleValue(value); } - return metric; + return metric; } void AssertLabels(const TLabels& expected, const NProto::TMultiSample& actual) { @@ -100,13 +100,13 @@ void AssertSimpleMessage(const NProto::TMultiSamplesList& samples, TString pathP UNIT_ASSERT(expectedValues.contains(labelVal)); if (labelVal == pathPrefix + "Foo") { - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); UNIT_ASSERT_DOUBLES_EQUAL(s.GetPoints(0).GetFloat64(), 1, 1e-6); } else if (labelVal == pathPrefix + "Bar") { - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); UNIT_ASSERT_DOUBLES_EQUAL(s.GetPoints(0).GetFloat64(), 2, 1e-6); } else if (labelVal == pathPrefix + "Baz") { - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::RATE); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::RATE); UNIT_ASSERT_EQUAL(s.GetPoints(0).GetUint64(), 42); } } @@ -115,7 +115,7 @@ void AssertSimpleMessage(const NProto::TMultiSamplesList& samples, TString pathP Y_UNIT_TEST_SUITE(TLegacyProtoDecoderTest) { Y_UNIT_TEST(SimpleProto) { NProto::TMultiSamplesList samples; - IMetricEncoderPtr e = EncoderProtobuf(&samples); + IMetricEncoderPtr e = EncoderProtobuf(&samples); auto msg = MakeSimpleMessage(); DecodeLegacyProto(msg, e.Get()); @@ -125,7 +125,7 @@ Y_UNIT_TEST_SUITE(TLegacyProtoDecoderTest) { Y_UNIT_TEST(RepeatedProto) { NProto::TMultiSamplesList samples; - IMetricEncoderPtr e = EncoderProtobuf(&samples); + IMetricEncoderPtr e = EncoderProtobuf(&samples); auto simple = MakeSimpleMessage(); TRepeated msg; @@ -138,7 +138,7 @@ Y_UNIT_TEST_SUITE(TLegacyProtoDecoderTest) { Y_UNIT_TEST(RepeatedProtoWithPath) { NProto::TMultiSamplesList samples; - IMetricEncoderPtr e = EncoderProtobuf(&samples); + IMetricEncoderPtr e = EncoderProtobuf(&samples); auto simple = MakeSimpleMessage(); TRepeatedWithPath msg; @@ -151,7 +151,7 @@ Y_UNIT_TEST_SUITE(TLegacyProtoDecoderTest) { Y_UNIT_TEST(DeepNesting) { NProto::TMultiSamplesList samples; - IMetricEncoderPtr e = EncoderProtobuf(&samples); + IMetricEncoderPtr e = EncoderProtobuf(&samples); auto simple = MakeSimpleMessage(); TRepeatedWithPath internal; @@ -167,7 +167,7 @@ Y_UNIT_TEST_SUITE(TLegacyProtoDecoderTest) { Y_UNIT_TEST(Keys) { NProto::TMultiSamplesList samples; - IMetricEncoderPtr e = EncoderProtobuf(&samples); + IMetricEncoderPtr e = EncoderProtobuf(&samples); auto simple = MakeSimpleMessage(); simple.SetLabel("my_label_value"); @@ -206,7 +206,7 @@ Y_UNIT_TEST_SUITE(TLegacyProtoDecoderTest) { Y_UNIT_TEST(NonStringKeys) { NProto::TMultiSamplesList samples; - IMetricEncoderPtr e = EncoderProtobuf(&samples); + IMetricEncoderPtr e = EncoderProtobuf(&samples); TNonStringKeys msg; msg.SetFoo(42); @@ -241,7 +241,7 @@ Y_UNIT_TEST_SUITE(TLegacyProtoDecoderTest) { Y_UNIT_TEST(KeysFromNonLeafNodes) { NProto::TMultiSamplesList samples; - IMetricEncoderPtr e = EncoderProtobuf(&samples); + IMetricEncoderPtr e = EncoderProtobuf(&samples); auto simple = MakeSimpleMessage(); simple.SetLabel("label_value"); @@ -260,7 +260,7 @@ Y_UNIT_TEST_SUITE(TLegacyProtoDecoderTest) { Y_UNIT_TEST(SpacesAreGetReplaced) { NProto::TMultiSamplesList samples; - IMetricEncoderPtr e = EncoderProtobuf(&samples); + IMetricEncoderPtr e = EncoderProtobuf(&samples); auto simple = MakeSimpleMessage(); simple.SetLabel("my label_value"); @@ -288,7 +288,7 @@ Y_UNIT_TEST_SUITE(TLegacyProtoDecoderTest) { Y_UNIT_TEST(ExtraLabels) { NProto::TMultiSamplesList samples; - IMetricEncoderPtr e = EncoderProtobuf(&samples); + IMetricEncoderPtr e = EncoderProtobuf(&samples); TExtraLabels msg; msg.MutableExtraAsIs()->CopyFrom(MakeExtra("label", "foo", 42, false)); @@ -318,14 +318,14 @@ Y_UNIT_TEST_SUITE(TLegacyProtoDecoderTest) { Y_UNIT_TEST(NestedExtraLabels) { NProto::TMultiSamplesList samples; - IMetricEncoderPtr e = EncoderProtobuf(&samples); + IMetricEncoderPtr e = EncoderProtobuf(&samples); TExtraLabels msg; auto extra = MakeExtra("label", "foo", 42, false); auto* val = extra.Mutablevalues(0); { auto child = MakeExtra("child1", "label1", 24, true); - child.Mutablevalues(0)->Settype(NMonProto::EMetricType::RATE); + child.Mutablevalues(0)->Settype(NMonProto::EMetricType::RATE); val->Addchildren()->CopyFrom(child); } @@ -368,7 +368,7 @@ Y_UNIT_TEST_SUITE(TLegacyProtoDecoderTest) { Y_UNIT_TEST(RobotLabels) { NProto::TMultiSamplesList samples; - IMetricEncoderPtr e = EncoderProtobuf(&samples); + IMetricEncoderPtr e = EncoderProtobuf(&samples); TNamedCounter responses; responses.SetName("responses"); @@ -397,7 +397,7 @@ Y_UNIT_TEST_SUITE(TLegacyProtoDecoderTest) { Y_UNIT_TEST(ZoraLabels) { NProto::TMultiSamplesList samples; - IMetricEncoderPtr e = EncoderProtobuf(&samples); + IMetricEncoderPtr e = EncoderProtobuf(&samples); TTimeLogHist hist; hist.AddBuckets(42); diff --git a/library/cpp/monlib/encode/legacy_protobuf/protos/metric_meta.proto b/library/cpp/monlib/encode/legacy_protobuf/protos/metric_meta.proto index a8c5a2b93d..fd23eb372b 100644 --- a/library/cpp/monlib/encode/legacy_protobuf/protos/metric_meta.proto +++ b/library/cpp/monlib/encode/legacy_protobuf/protos/metric_meta.proto @@ -1,23 +1,23 @@ import "google/protobuf/descriptor.proto"; -package NMonProto; +package NMonProto; -option java_package = "ru.yandex.monlib.proto"; -option java_outer_classname = "MetricMetaProto"; +option java_package = "ru.yandex.monlib.proto"; +option java_outer_classname = "MetricMetaProto"; -enum EMetricType { - GAUGE = 1; - RATE = 2; +enum EMetricType { + GAUGE = 1; + RATE = 2; } -enum EMemOnly { +enum EMemOnly { DEFAULT = 0; STORE = 1; MEM_ONLY = 2; } -message TMetricMeta { - optional EMetricType Type = 1; +message TMetricMeta { + optional EMetricType Type = 1; optional bool Path = 2; optional string Keys = 3; optional bool MemOnly = 4; @@ -45,7 +45,7 @@ message THistogram { } // field of this type is recognized by Solomon -message TExtraLabelMetrics { +message TExtraLabelMetrics { optional string labelName = 1; message TValue { @@ -57,17 +57,17 @@ message TExtraLabelMetrics { optional double doubleValue = 3; optional THistogram histogramValue = 4; - optional EMetricType type = 7; - optional EMemOnly memOnly = 8; + optional EMetricType type = 7; + optional EMemOnly memOnly = 8; optional bool dropHost = 9; - repeated TExtraLabelMetrics children = 17; + repeated TExtraLabelMetrics children = 17; } repeated TValue values = 2; } extend google.protobuf.FieldOptions { - optional TMetricMeta Metric = 1719; + optional TMetricMeta Metric = 1719; } diff --git a/library/cpp/monlib/encode/legacy_protobuf/protos/python/ya.make b/library/cpp/monlib/encode/legacy_protobuf/protos/python/ya.make index ea96eaa950..095b307b01 100644 --- a/library/cpp/monlib/encode/legacy_protobuf/protos/python/ya.make +++ b/library/cpp/monlib/encode/legacy_protobuf/protos/python/ya.make @@ -1,3 +1,3 @@ OWNER(g:solomon) -PY_PROTOS_FOR(library/cpp/monlib/encode/legacy_protobuf/protos) +PY_PROTOS_FOR(library/cpp/monlib/encode/legacy_protobuf/protos) diff --git a/library/cpp/monlib/encode/legacy_protobuf/protos/ya.make b/library/cpp/monlib/encode/legacy_protobuf/protos/ya.make index b5f1098a48..489f361ab1 100644 --- a/library/cpp/monlib/encode/legacy_protobuf/protos/ya.make +++ b/library/cpp/monlib/encode/legacy_protobuf/protos/ya.make @@ -3,7 +3,7 @@ PROTO_LIBRARY() OWNER(g:solomon) SRCS( - metric_meta.proto + metric_meta.proto ) IF (NOT PY_PROTOS_FOR) diff --git a/library/cpp/monlib/encode/legacy_protobuf/ut/test_cases.proto b/library/cpp/monlib/encode/legacy_protobuf/ut/test_cases.proto index c51d870e07..37e901de48 100644 --- a/library/cpp/monlib/encode/legacy_protobuf/ut/test_cases.proto +++ b/library/cpp/monlib/encode/legacy_protobuf/ut/test_cases.proto @@ -1,26 +1,26 @@ -import "library/cpp/monlib/encode/legacy_protobuf/protos/metric_meta.proto"; +import "library/cpp/monlib/encode/legacy_protobuf/protos/metric_meta.proto"; message TSimple { - optional uint64 Foo = 1 [ (NMonProto.Metric).Type = GAUGE ]; - optional double Bar = 2 [ (NMonProto.Metric).Type = GAUGE ]; - optional double Baz = 3 [ (NMonProto.Metric).Type = RATE ]; + optional uint64 Foo = 1 [ (NMonProto.Metric).Type = GAUGE ]; + optional double Bar = 2 [ (NMonProto.Metric).Type = GAUGE ]; + optional double Baz = 3 [ (NMonProto.Metric).Type = RATE ]; optional string Label = 4; } message TRepeated { - repeated TSimple Messages = 1 [ (NMonProto.Metric).Path = false ]; + repeated TSimple Messages = 1 [ (NMonProto.Metric).Path = false ]; }; message TRepeatedWithPath { - repeated TSimple Namespace = 1 [ (NMonProto.Metric).Path = true ]; + repeated TSimple Namespace = 1 [ (NMonProto.Metric).Path = true ]; }; message TNestedWithKeys { - repeated TSimple Namespace = 1 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "my_label:Label fixed_label=fixed_value numbered:#" ]; + repeated TSimple Namespace = 1 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "my_label:Label fixed_label=fixed_value numbered:#" ]; }; message TDeepNesting { - optional TRepeatedWithPath Nested = 1 [ (NMonProto.Metric).Path = false ]; + optional TRepeatedWithPath Nested = 1 [ (NMonProto.Metric).Path = false ]; }; enum EEnum { @@ -29,43 +29,43 @@ enum EEnum { }; message TNonStringKeys { - optional uint32 Foo = 1 [ (NMonProto.Metric).Type = GAUGE ]; + optional uint32 Foo = 1 [ (NMonProto.Metric).Type = GAUGE ]; optional EEnum Enum = 2; optional uint32 Int = 3; }; message TRepeatedNonStringKeys { - repeated TNonStringKeys Nested = 1 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "enum:Enum int:Int" ]; + repeated TNonStringKeys Nested = 1 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "enum:Enum int:Int" ]; }; message TExtraLabels { - optional NMonProto.TExtraLabelMetrics ExtraAsIs = 1 [ (NMonProto.Metric).Type = GAUGE ]; - optional NMonProto.TExtraLabelMetrics ExtraDeriv = 2 [ (NMonProto.Metric).Type = RATE ]; + optional NMonProto.TExtraLabelMetrics ExtraAsIs = 1 [ (NMonProto.Metric).Type = GAUGE ]; + optional NMonProto.TExtraLabelMetrics ExtraDeriv = 2 [ (NMonProto.Metric).Type = RATE ]; }; message TRepeatedWithName { optional string Name = 1; - repeated TSimple Nested = 2 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "my_label:Label" ]; + repeated TSimple Nested = 2 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "my_label:Label" ]; }; message TKeysFromNonLeaf { - repeated TRepeatedWithName Nested = 1 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "name:Name" ]; + repeated TRepeatedWithName Nested = 1 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "name:Name" ]; }; message TNamedCounter { optional string Name = 1; - optional uint64 Count = 2 [ (NMonProto.Metric).Type = RATE ]; + optional uint64 Count = 2 [ (NMonProto.Metric).Type = RATE ]; } message TCrawlerCounters { message TStatusCounters { - repeated TNamedCounter ZoraResponses = 3 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "ZoraResponse:Name" ]; - repeated TNamedCounter FetcherResponses = 4 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "SpiderResponse:Name" ]; - repeated TNamedCounter RotorResponses = 5 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "SpiderResponse:Name" ]; - repeated TNamedCounter PDFetchResponses = 6 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "PDFetchResponse:Name" ]; - repeated TNamedCounter CalcResponses = 7 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "CalcResponse:Name" ]; - repeated TNamedCounter Responses = 8 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "AggregatedResponse:Name" ]; + repeated TNamedCounter ZoraResponses = 3 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "ZoraResponse:Name" ]; + repeated TNamedCounter FetcherResponses = 4 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "SpiderResponse:Name" ]; + repeated TNamedCounter RotorResponses = 5 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "SpiderResponse:Name" ]; + repeated TNamedCounter PDFetchResponses = 6 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "PDFetchResponse:Name" ]; + repeated TNamedCounter CalcResponses = 7 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "CalcResponse:Name" ]; + repeated TNamedCounter Responses = 8 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "AggregatedResponse:Name" ]; } message TPolicyCounters { @@ -73,18 +73,18 @@ message TCrawlerCounters { optional string Name = 2; optional string Zone = 3; - optional TStatusCounters StatusCounters = 4 [ (NMonProto.Metric).Path = true ]; + optional TStatusCounters StatusCounters = 4 [ (NMonProto.Metric).Path = true ]; } optional string Component = 1; - repeated TPolicyCounters PoliciesCounters = 3 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "SubComponent:SubComponent Policy:Name Zone:Zone" ]; + repeated TPolicyCounters PoliciesCounters = 3 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "SubComponent:SubComponent Policy:Name Zone:Zone" ]; } message TTimeLogHist { optional uint32 MinBucketMillisec = 1; - repeated uint64 Buckets = 2 [ (NMonProto.Metric).Type = RATE, (NMonProto.Metric).Keys = "slot:#" ]; + repeated uint64 Buckets = 2 [ (NMonProto.Metric).Type = RATE, (NMonProto.Metric).Keys = "slot:#" ]; } message TKiwiCounters { - optional TTimeLogHist Times = 22 [ (NMonProto.Metric).Path = true ]; + optional TTimeLogHist Times = 22 [ (NMonProto.Metric).Path = true ]; } diff --git a/library/cpp/monlib/encode/legacy_protobuf/ut/ya.make b/library/cpp/monlib/encode/legacy_protobuf/ut/ya.make index fa9b4ebb1a..479a0c46c9 100644 --- a/library/cpp/monlib/encode/legacy_protobuf/ut/ya.make +++ b/library/cpp/monlib/encode/legacy_protobuf/ut/ya.make @@ -1,4 +1,4 @@ -UNITTEST_FOR(library/cpp/monlib/encode/legacy_protobuf) +UNITTEST_FOR(library/cpp/monlib/encode/legacy_protobuf) OWNER( g:solomon @@ -11,8 +11,8 @@ SRCS( ) PEERDIR( - library/cpp/monlib/encode/protobuf - library/cpp/monlib/encode/text + library/cpp/monlib/encode/protobuf + library/cpp/monlib/encode/text ) END() diff --git a/library/cpp/monlib/encode/legacy_protobuf/ya.make b/library/cpp/monlib/encode/legacy_protobuf/ya.make index 5750032537..74c82aac93 100644 --- a/library/cpp/monlib/encode/legacy_protobuf/ya.make +++ b/library/cpp/monlib/encode/legacy_protobuf/ya.make @@ -10,7 +10,7 @@ SRCS( ) PEERDIR( - library/cpp/monlib/encode/legacy_protobuf/protos + library/cpp/monlib/encode/legacy_protobuf/protos ) END() diff --git a/library/cpp/monlib/encode/prometheus/fuzz/main.cpp b/library/cpp/monlib/encode/prometheus/fuzz/main.cpp index a9ed0afc45..24bda2d32e 100644 --- a/library/cpp/monlib/encode/prometheus/fuzz/main.cpp +++ b/library/cpp/monlib/encode/prometheus/fuzz/main.cpp @@ -1,18 +1,18 @@ -#include <library/cpp/monlib/encode/prometheus/prometheus.h> -#include <library/cpp/monlib/encode/fake/fake.h> - -#include <util/stream/mem.h> - - -extern "C" int LLVMFuzzerTestOneInput(const ui8* buf, size_t size) { - using namespace NMonitoring; - - try { - TStringBuf data(reinterpret_cast<const char*>(buf), size); - auto encoder = EncoderFake(); - DecodePrometheus(data, encoder.Get()); - } catch (...) { - } - - return 0; -} +#include <library/cpp/monlib/encode/prometheus/prometheus.h> +#include <library/cpp/monlib/encode/fake/fake.h> + +#include <util/stream/mem.h> + + +extern "C" int LLVMFuzzerTestOneInput(const ui8* buf, size_t size) { + using namespace NMonitoring; + + try { + TStringBuf data(reinterpret_cast<const char*>(buf), size); + auto encoder = EncoderFake(); + DecodePrometheus(data, encoder.Get()); + } catch (...) { + } + + return 0; +} diff --git a/library/cpp/monlib/encode/prometheus/fuzz/ya.make b/library/cpp/monlib/encode/prometheus/fuzz/ya.make index 0fc3845718..4a6c796ed5 100644 --- a/library/cpp/monlib/encode/prometheus/fuzz/ya.make +++ b/library/cpp/monlib/encode/prometheus/fuzz/ya.make @@ -1,16 +1,16 @@ -FUZZ() - -OWNER(g:solomon jamel) - -PEERDIR( - library/cpp/monlib/encode/prometheus - library/cpp/monlib/encode/fake -) - -SIZE(MEDIUM) - -SRCS( - main.cpp -) - -END() +FUZZ() + +OWNER(g:solomon jamel) + +PEERDIR( + library/cpp/monlib/encode/prometheus + library/cpp/monlib/encode/fake +) + +SIZE(MEDIUM) + +SRCS( + main.cpp +) + +END() diff --git a/library/cpp/monlib/encode/prometheus/prometheus.h b/library/cpp/monlib/encode/prometheus/prometheus.h index f031f5e933..2e7fa31c28 100644 --- a/library/cpp/monlib/encode/prometheus/prometheus.h +++ b/library/cpp/monlib/encode/prometheus/prometheus.h @@ -1,18 +1,18 @@ -#pragma once - -#include <library/cpp/monlib/encode/encoder.h> -#include <library/cpp/monlib/encode/format.h> - -#include <util/generic/yexception.h> - - -namespace NMonitoring { - - class TPrometheusDecodeException: public yexception { - }; - +#pragma once + +#include <library/cpp/monlib/encode/encoder.h> +#include <library/cpp/monlib/encode/format.h> + +#include <util/generic/yexception.h> + + +namespace NMonitoring { + + class TPrometheusDecodeException: public yexception { + }; + IMetricEncoderPtr EncoderPrometheus(IOutputStream* out, TStringBuf metricNameLabel = "sensor"); - + void DecodePrometheus(TStringBuf data, IMetricConsumer* c, TStringBuf metricNameLabel = "sensor"); - -} + +} diff --git a/library/cpp/monlib/encode/prometheus/prometheus_decoder.cpp b/library/cpp/monlib/encode/prometheus/prometheus_decoder.cpp index f11dd33e31..7e81357dbd 100644 --- a/library/cpp/monlib/encode/prometheus/prometheus_decoder.cpp +++ b/library/cpp/monlib/encode/prometheus/prometheus_decoder.cpp @@ -1,51 +1,51 @@ -#include "prometheus.h" -#include "prometheus_model.h" - -#include <library/cpp/monlib/metrics/histogram_snapshot.h> -#include <library/cpp/monlib/metrics/metric.h> - -#include <util/datetime/base.h> -#include <util/generic/hash.h> -#include <util/string/cast.h> -#include <util/string/builder.h> -#include <util/generic/maybe.h> -#include <util/string/ascii.h> - -#include <cmath> - -#define Y_PARSER_FAIL(message) \ - ythrow ::NMonitoring::TPrometheusDecodeException() << message << " at line #" << CurrentLine_ - -#define Y_PARSER_ENSURE(cond, message) \ - Y_ENSURE_EX(cond, ::NMonitoring::TPrometheusDecodeException() << message << " at line #" << CurrentLine_) - - -namespace NMonitoring { - namespace { +#include "prometheus.h" +#include "prometheus_model.h" + +#include <library/cpp/monlib/metrics/histogram_snapshot.h> +#include <library/cpp/monlib/metrics/metric.h> + +#include <util/datetime/base.h> +#include <util/generic/hash.h> +#include <util/string/cast.h> +#include <util/string/builder.h> +#include <util/generic/maybe.h> +#include <util/string/ascii.h> + +#include <cmath> + +#define Y_PARSER_FAIL(message) \ + ythrow ::NMonitoring::TPrometheusDecodeException() << message << " at line #" << CurrentLine_ + +#define Y_PARSER_ENSURE(cond, message) \ + Y_ENSURE_EX(cond, ::NMonitoring::TPrometheusDecodeException() << message << " at line #" << CurrentLine_) + + +namespace NMonitoring { + namespace { constexpr ui32 MAX_LABEL_VALUE_LEN = 256; - - using TLabelsMap = THashMap<TString, TString>; - - TString LabelsToStr(const TLabelsMap& labels) { - TStringBuilder sb; - auto it = labels.begin(); - auto end = labels.end(); - - sb << '{'; - while (it != end) { - sb << it->first; - sb << '='; - sb << '"' << it->second << '"'; - - ++it; - if (it != end) { - sb << ", "; - } - } - sb << '}'; - return sb; - } - + + using TLabelsMap = THashMap<TString, TString>; + + TString LabelsToStr(const TLabelsMap& labels) { + TStringBuilder sb; + auto it = labels.begin(); + auto end = labels.end(); + + sb << '{'; + while (it != end) { + sb << it->first; + sb << '='; + sb << '"' << it->second << '"'; + + ++it; + if (it != end) { + sb << ", "; + } + } + sb << '}'; + return sb; + } + template <typename T, typename U> bool TryStaticCast(U val, T& out) { static_assert(std::is_arithmetic_v<U>); @@ -64,534 +64,534 @@ namespace NMonitoring { return true; } - /////////////////////////////////////////////////////////////////////// - // THistogramBuilder - /////////////////////////////////////////////////////////////////////// - class THistogramBuilder { - using TBucketData = std::pair<TBucketBound, TBucketValue>; - constexpr static TBucketData ZERO_BUCKET = { -std::numeric_limits<TBucketBound>::max(), 0 }; - public: - TStringBuf GetName() const noexcept { - return Name_; - } - - void SetName(TStringBuf name) noexcept { - Name_ = name; - } - - const TLabelsMap& GetLabels() const noexcept { - return *Labels_; - } - - void SetLabels(TLabelsMap&& labels) { - if (Labels_.Defined()) { - Y_ENSURE(Labels_ == labels, - "mixed labels in one histogram, prev: " << LabelsToStr(*Labels_) << - ", current: " << LabelsToStr(labels)); - } else { - Labels_.ConstructInPlace(std::move(labels)); - } - } - - TInstant GetTime() const noexcept { - return Time_; - } - - void SetTime(TInstant time) noexcept { - Time_ = time; - } - - bool Empty() const noexcept { - return Bounds_.empty(); - } - - bool Same(TStringBuf name, const TLabelsMap& labels) const noexcept { - return Name_ == name && Labels_ == labels; - } - - void AddBucket(TBucketBound bound, TBucketValue value) { - Y_ENSURE_EX(PrevBucket_.first < bound, TPrometheusDecodeException() << - "invalid order of histogram bounds " << PrevBucket_.first << - " >= " << bound); - - Y_ENSURE_EX(PrevBucket_.second <= value, TPrometheusDecodeException() << - "invalid order of histogram bucket values " << PrevBucket_.second << - " > " << value); - - // convert infinite bound value - if (bound == std::numeric_limits<TBucketBound>::infinity()) { - bound = HISTOGRAM_INF_BOUND; - } - - Bounds_.push_back(bound); - Values_.push_back(value - PrevBucket_.second); // keep only delta between buckets - - PrevBucket_ = { bound, value }; - } - - // will clear builder state - IHistogramSnapshotPtr ToSnapshot() { - Y_ENSURE_EX(!Empty(), TPrometheusDecodeException() << "histogram cannot be empty"); - Time_ = TInstant::Zero(); - PrevBucket_ = ZERO_BUCKET; - Labels_.Clear(); + /////////////////////////////////////////////////////////////////////// + // THistogramBuilder + /////////////////////////////////////////////////////////////////////// + class THistogramBuilder { + using TBucketData = std::pair<TBucketBound, TBucketValue>; + constexpr static TBucketData ZERO_BUCKET = { -std::numeric_limits<TBucketBound>::max(), 0 }; + public: + TStringBuf GetName() const noexcept { + return Name_; + } + + void SetName(TStringBuf name) noexcept { + Name_ = name; + } + + const TLabelsMap& GetLabels() const noexcept { + return *Labels_; + } + + void SetLabels(TLabelsMap&& labels) { + if (Labels_.Defined()) { + Y_ENSURE(Labels_ == labels, + "mixed labels in one histogram, prev: " << LabelsToStr(*Labels_) << + ", current: " << LabelsToStr(labels)); + } else { + Labels_.ConstructInPlace(std::move(labels)); + } + } + + TInstant GetTime() const noexcept { + return Time_; + } + + void SetTime(TInstant time) noexcept { + Time_ = time; + } + + bool Empty() const noexcept { + return Bounds_.empty(); + } + + bool Same(TStringBuf name, const TLabelsMap& labels) const noexcept { + return Name_ == name && Labels_ == labels; + } + + void AddBucket(TBucketBound bound, TBucketValue value) { + Y_ENSURE_EX(PrevBucket_.first < bound, TPrometheusDecodeException() << + "invalid order of histogram bounds " << PrevBucket_.first << + " >= " << bound); + + Y_ENSURE_EX(PrevBucket_.second <= value, TPrometheusDecodeException() << + "invalid order of histogram bucket values " << PrevBucket_.second << + " > " << value); + + // convert infinite bound value + if (bound == std::numeric_limits<TBucketBound>::infinity()) { + bound = HISTOGRAM_INF_BOUND; + } + + Bounds_.push_back(bound); + Values_.push_back(value - PrevBucket_.second); // keep only delta between buckets + + PrevBucket_ = { bound, value }; + } + + // will clear builder state + IHistogramSnapshotPtr ToSnapshot() { + Y_ENSURE_EX(!Empty(), TPrometheusDecodeException() << "histogram cannot be empty"); + Time_ = TInstant::Zero(); + PrevBucket_ = ZERO_BUCKET; + Labels_.Clear(); auto snapshot = ExplicitHistogramSnapshot(Bounds_, Values_); Bounds_.clear(); Values_.clear(); return snapshot; - } - - private: - TStringBuf Name_; - TMaybe<TLabelsMap> Labels_; - TInstant Time_; - TBucketBounds Bounds_; - TBucketValues Values_; - TBucketData PrevBucket_ = ZERO_BUCKET; - }; - - /////////////////////////////////////////////////////////////////////// - // EPrometheusMetricType - /////////////////////////////////////////////////////////////////////// - enum class EPrometheusMetricType { - GAUGE, - COUNTER, - SUMMARY, - UNTYPED, - HISTOGRAM, - }; - - /////////////////////////////////////////////////////////////////////// - // TPrometheusReader - /////////////////////////////////////////////////////////////////////// - class TPrometheusReader { - public: + } + + private: + TStringBuf Name_; + TMaybe<TLabelsMap> Labels_; + TInstant Time_; + TBucketBounds Bounds_; + TBucketValues Values_; + TBucketData PrevBucket_ = ZERO_BUCKET; + }; + + /////////////////////////////////////////////////////////////////////// + // EPrometheusMetricType + /////////////////////////////////////////////////////////////////////// + enum class EPrometheusMetricType { + GAUGE, + COUNTER, + SUMMARY, + UNTYPED, + HISTOGRAM, + }; + + /////////////////////////////////////////////////////////////////////// + // TPrometheusReader + /////////////////////////////////////////////////////////////////////// + class TPrometheusReader { + public: TPrometheusReader(TStringBuf data, IMetricConsumer* c, TStringBuf metricNameLabel) - : Data_(data) - , Consumer_(c) + : Data_(data) + , Consumer_(c) , MetricNameLabel_(metricNameLabel) - { - } - - void Read() { - Consumer_->OnStreamBegin(); - - if (HasRemaining()) { - ReadNextByte(); - SkipSpaces(); - - try { - while (HasRemaining()) { - switch (CurrentByte_) { - case '\n': - ReadNextByte(); // skip '\n' - CurrentLine_++; - SkipSpaces(); - break; - case '#': - ParseComment(); - break; - default: - ParseMetric(); - break; - } - } - - if (!HistogramBuilder_.Empty()) { - ConsumeHistogram(); - } - } catch (const TPrometheusDecodeException& e) { - throw e; - } catch (...) { - Y_PARSER_FAIL("unexpected error " << CurrentExceptionMessage()); - } - } - - Consumer_->OnStreamEnd(); - } - - private: - bool HasRemaining() const noexcept { - return CurrentPos_ < Data_.Size(); - } - - // # 'TYPE' metric_name {counter|gauge|histogram|summary|untyped} - // # 'HELP' metric_name some help info - // # general comment message - void ParseComment() { - SkipExpectedChar('#'); - SkipSpaces(); - - TStringBuf keyword = ReadToken(); + { + } + + void Read() { + Consumer_->OnStreamBegin(); + + if (HasRemaining()) { + ReadNextByte(); + SkipSpaces(); + + try { + while (HasRemaining()) { + switch (CurrentByte_) { + case '\n': + ReadNextByte(); // skip '\n' + CurrentLine_++; + SkipSpaces(); + break; + case '#': + ParseComment(); + break; + default: + ParseMetric(); + break; + } + } + + if (!HistogramBuilder_.Empty()) { + ConsumeHistogram(); + } + } catch (const TPrometheusDecodeException& e) { + throw e; + } catch (...) { + Y_PARSER_FAIL("unexpected error " << CurrentExceptionMessage()); + } + } + + Consumer_->OnStreamEnd(); + } + + private: + bool HasRemaining() const noexcept { + return CurrentPos_ < Data_.Size(); + } + + // # 'TYPE' metric_name {counter|gauge|histogram|summary|untyped} + // # 'HELP' metric_name some help info + // # general comment message + void ParseComment() { + SkipExpectedChar('#'); + SkipSpaces(); + + TStringBuf keyword = ReadToken(); if (keyword == TStringBuf("TYPE")) { - SkipSpaces(); - - TStringBuf nextName = ReadTokenAsMetricName(); - Y_PARSER_ENSURE(!nextName.Empty(), "invalid metric name"); - - SkipSpaces(); - EPrometheusMetricType nextType = ReadType(); - - bool inserted = SeenTypes_.emplace(nextName, nextType).second; - Y_PARSER_ENSURE(inserted, "second TYPE line for metric " << nextName); - - if (nextType == EPrometheusMetricType::HISTOGRAM) { - if (!HistogramBuilder_.Empty()) { - ConsumeHistogram(); - } - HistogramBuilder_.SetName(nextName); - } - } else { - // skip HELP and general comments - SkipUntilEol(); - } - - Y_PARSER_ENSURE(CurrentByte_ == '\n', "expected '\\n', found '" << CurrentByte_ << '\''); - } - - // metric_name [labels] value [timestamp] - void ParseMetric() { - TStringBuf name = ReadTokenAsMetricName(); - SkipSpaces(); - - TLabelsMap labels = ReadLabels(); - SkipSpaces(); - - double value = ParseGoDouble(ReadToken()); - SkipSpaces(); - - TInstant time = TInstant::Zero(); - if (CurrentByte_ != '\n') { - time = TInstant::MilliSeconds(FromString<ui64>(ReadToken())); - } - - TStringBuf baseName = name; - EPrometheusMetricType type = EPrometheusMetricType::UNTYPED; - - if (auto* seenType = SeenTypes_.FindPtr(name)) { - type = *seenType; - } else { - baseName = NPrometheus::ToBaseName(name); - if (auto* baseType = SeenTypes_.FindPtr(baseName)) { - type = *baseType; - } - } - - switch (type) { - case EPrometheusMetricType::HISTOGRAM: - if (NPrometheus::IsBucket(name)) { - double bound = 0.0; - auto it = labels.find(NPrometheus::BUCKET_LABEL); - if (it != labels.end()) { - bound = ParseGoDouble(it->second); - labels.erase(it); - } else { - Y_PARSER_FAIL( - "metric " << name << "has no " << NPrometheus::BUCKET_LABEL << - " label at line #" << CurrentLine_); - } - - if (!HistogramBuilder_.Empty() && !HistogramBuilder_.Same(baseName, labels)) { - ConsumeHistogram(); - HistogramBuilder_.SetName(baseName); - } - + SkipSpaces(); + + TStringBuf nextName = ReadTokenAsMetricName(); + Y_PARSER_ENSURE(!nextName.Empty(), "invalid metric name"); + + SkipSpaces(); + EPrometheusMetricType nextType = ReadType(); + + bool inserted = SeenTypes_.emplace(nextName, nextType).second; + Y_PARSER_ENSURE(inserted, "second TYPE line for metric " << nextName); + + if (nextType == EPrometheusMetricType::HISTOGRAM) { + if (!HistogramBuilder_.Empty()) { + ConsumeHistogram(); + } + HistogramBuilder_.SetName(nextName); + } + } else { + // skip HELP and general comments + SkipUntilEol(); + } + + Y_PARSER_ENSURE(CurrentByte_ == '\n', "expected '\\n', found '" << CurrentByte_ << '\''); + } + + // metric_name [labels] value [timestamp] + void ParseMetric() { + TStringBuf name = ReadTokenAsMetricName(); + SkipSpaces(); + + TLabelsMap labels = ReadLabels(); + SkipSpaces(); + + double value = ParseGoDouble(ReadToken()); + SkipSpaces(); + + TInstant time = TInstant::Zero(); + if (CurrentByte_ != '\n') { + time = TInstant::MilliSeconds(FromString<ui64>(ReadToken())); + } + + TStringBuf baseName = name; + EPrometheusMetricType type = EPrometheusMetricType::UNTYPED; + + if (auto* seenType = SeenTypes_.FindPtr(name)) { + type = *seenType; + } else { + baseName = NPrometheus::ToBaseName(name); + if (auto* baseType = SeenTypes_.FindPtr(baseName)) { + type = *baseType; + } + } + + switch (type) { + case EPrometheusMetricType::HISTOGRAM: + if (NPrometheus::IsBucket(name)) { + double bound = 0.0; + auto it = labels.find(NPrometheus::BUCKET_LABEL); + if (it != labels.end()) { + bound = ParseGoDouble(it->second); + labels.erase(it); + } else { + Y_PARSER_FAIL( + "metric " << name << "has no " << NPrometheus::BUCKET_LABEL << + " label at line #" << CurrentLine_); + } + + if (!HistogramBuilder_.Empty() && !HistogramBuilder_.Same(baseName, labels)) { + ConsumeHistogram(); + HistogramBuilder_.SetName(baseName); + } + TBucketValue bucketVal; Y_PARSER_ENSURE(TryStaticCast(value, bucketVal), "Cannot convert " << value << " to bucket value type"); HistogramBuilder_.AddBucket(bound, bucketVal); - HistogramBuilder_.SetTime(time); - HistogramBuilder_.SetLabels(std::move(labels)); - } else if (NPrometheus::IsCount(name)) { - // translate x_count metric as COUNTER metric - ConsumeCounter(name, labels, time, value); - } else if (NPrometheus::IsSum(name)) { - // translate x_sum metric as GAUGE metric - ConsumeGauge(name, labels, time, value); - } else { - Y_PARSER_FAIL( - "metric " << name << - " should be part of HISTOGRAM " << baseName); - } - break; - - case EPrometheusMetricType::SUMMARY: - if (NPrometheus::IsCount(name)) { - // translate x_count metric as COUNTER metric - ConsumeCounter(name, labels, time, value); - } else if (NPrometheus::IsSum(name)) { - // translate x_sum metric as GAUGE metric - ConsumeGauge(name, labels, time, value); - } else { - ConsumeGauge(name, labels, time, value); - } - break; - - case EPrometheusMetricType::COUNTER: - ConsumeCounter(name, labels, time, value); - break; - - case EPrometheusMetricType::GAUGE: - ConsumeGauge(name, labels, time, value); - break; - - case EPrometheusMetricType::UNTYPED: - ConsumeGauge(name, labels, time, value); - break; - } - - Y_PARSER_ENSURE(CurrentByte_ == '\n', "expected '\\n', found '" << CurrentByte_ << '\''); - } - - // { name = "value", name2 = "value2", } - TLabelsMap ReadLabels() { - TLabelsMap labels; - if (CurrentByte_ != '{') { - return labels; - } - - SkipExpectedChar('{'); - SkipSpaces(); - - while (CurrentByte_ != '}') { - TStringBuf name = ReadTokenAsLabelName(); - SkipSpaces(); - - SkipExpectedChar('='); - SkipSpaces(); - - TString value = ReadTokenAsLabelValue(); - SkipSpaces(); - labels.emplace(name, value); - - if (CurrentByte_ == ',') { - SkipExpectedChar(','); - SkipSpaces(); - } - } - - SkipExpectedChar('}'); - return labels; - } - - EPrometheusMetricType ReadType() { - TStringBuf keyword = ReadToken(); - if (AsciiEqualsIgnoreCase(keyword, "GAUGE")) { - return EPrometheusMetricType::GAUGE; - } else if (AsciiEqualsIgnoreCase(keyword, "COUNTER")) { - return EPrometheusMetricType::COUNTER; - } else if (AsciiEqualsIgnoreCase(keyword, "SUMMARY")) { - return EPrometheusMetricType::SUMMARY; - } else if (AsciiEqualsIgnoreCase(keyword, "HISTOGRAM")) { - return EPrometheusMetricType::HISTOGRAM; - } else if (AsciiEqualsIgnoreCase(keyword, "UNTYPED")) { - return EPrometheusMetricType::UNTYPED; - } - - Y_PARSER_FAIL( - "unknown metric type: " << keyword << - " at line #" << CurrentLine_); - } - - Y_FORCE_INLINE void ReadNextByteUnsafe() { - CurrentByte_ = Data_[CurrentPos_++]; - } - + HistogramBuilder_.SetTime(time); + HistogramBuilder_.SetLabels(std::move(labels)); + } else if (NPrometheus::IsCount(name)) { + // translate x_count metric as COUNTER metric + ConsumeCounter(name, labels, time, value); + } else if (NPrometheus::IsSum(name)) { + // translate x_sum metric as GAUGE metric + ConsumeGauge(name, labels, time, value); + } else { + Y_PARSER_FAIL( + "metric " << name << + " should be part of HISTOGRAM " << baseName); + } + break; + + case EPrometheusMetricType::SUMMARY: + if (NPrometheus::IsCount(name)) { + // translate x_count metric as COUNTER metric + ConsumeCounter(name, labels, time, value); + } else if (NPrometheus::IsSum(name)) { + // translate x_sum metric as GAUGE metric + ConsumeGauge(name, labels, time, value); + } else { + ConsumeGauge(name, labels, time, value); + } + break; + + case EPrometheusMetricType::COUNTER: + ConsumeCounter(name, labels, time, value); + break; + + case EPrometheusMetricType::GAUGE: + ConsumeGauge(name, labels, time, value); + break; + + case EPrometheusMetricType::UNTYPED: + ConsumeGauge(name, labels, time, value); + break; + } + + Y_PARSER_ENSURE(CurrentByte_ == '\n', "expected '\\n', found '" << CurrentByte_ << '\''); + } + + // { name = "value", name2 = "value2", } + TLabelsMap ReadLabels() { + TLabelsMap labels; + if (CurrentByte_ != '{') { + return labels; + } + + SkipExpectedChar('{'); + SkipSpaces(); + + while (CurrentByte_ != '}') { + TStringBuf name = ReadTokenAsLabelName(); + SkipSpaces(); + + SkipExpectedChar('='); + SkipSpaces(); + + TString value = ReadTokenAsLabelValue(); + SkipSpaces(); + labels.emplace(name, value); + + if (CurrentByte_ == ',') { + SkipExpectedChar(','); + SkipSpaces(); + } + } + + SkipExpectedChar('}'); + return labels; + } + + EPrometheusMetricType ReadType() { + TStringBuf keyword = ReadToken(); + if (AsciiEqualsIgnoreCase(keyword, "GAUGE")) { + return EPrometheusMetricType::GAUGE; + } else if (AsciiEqualsIgnoreCase(keyword, "COUNTER")) { + return EPrometheusMetricType::COUNTER; + } else if (AsciiEqualsIgnoreCase(keyword, "SUMMARY")) { + return EPrometheusMetricType::SUMMARY; + } else if (AsciiEqualsIgnoreCase(keyword, "HISTOGRAM")) { + return EPrometheusMetricType::HISTOGRAM; + } else if (AsciiEqualsIgnoreCase(keyword, "UNTYPED")) { + return EPrometheusMetricType::UNTYPED; + } + + Y_PARSER_FAIL( + "unknown metric type: " << keyword << + " at line #" << CurrentLine_); + } + + Y_FORCE_INLINE void ReadNextByteUnsafe() { + CurrentByte_ = Data_[CurrentPos_++]; + } + Y_FORCE_INLINE bool IsSpace(char ch) { - return ch == ' ' || ch == '\t'; - } - - void ReadNextByte() { - Y_PARSER_ENSURE(HasRemaining(), "unexpected end of file"); - ReadNextByteUnsafe(); - } - - void SkipExpectedChar(char ch) { - Y_PARSER_ENSURE(CurrentByte_ == ch, - "expected '" << CurrentByte_ << "', found '" << ch << '\''); - ReadNextByte(); - } - - void SkipSpaces() { + return ch == ' ' || ch == '\t'; + } + + void ReadNextByte() { + Y_PARSER_ENSURE(HasRemaining(), "unexpected end of file"); + ReadNextByteUnsafe(); + } + + void SkipExpectedChar(char ch) { + Y_PARSER_ENSURE(CurrentByte_ == ch, + "expected '" << CurrentByte_ << "', found '" << ch << '\''); + ReadNextByte(); + } + + void SkipSpaces() { while (HasRemaining() && IsSpace(CurrentByte_)) { - ReadNextByteUnsafe(); - } - } - - void SkipUntilEol() { - while (HasRemaining() && CurrentByte_ != '\n') { - ReadNextByteUnsafe(); - } - } - - TStringBuf ReadToken() { - Y_VERIFY_DEBUG(CurrentPos_ > 0); - size_t begin = CurrentPos_ - 1; // read first byte again + ReadNextByteUnsafe(); + } + } + + void SkipUntilEol() { + while (HasRemaining() && CurrentByte_ != '\n') { + ReadNextByteUnsafe(); + } + } + + TStringBuf ReadToken() { + Y_VERIFY_DEBUG(CurrentPos_ > 0); + size_t begin = CurrentPos_ - 1; // read first byte again while (HasRemaining() && !IsSpace(CurrentByte_) && CurrentByte_ != '\n') { - ReadNextByteUnsafe(); - } - return TokenFromPos(begin); - } - - TStringBuf ReadTokenAsMetricName() { - if (!NPrometheus::IsValidMetricNameStart(CurrentByte_)) { - return ""; - } - - Y_VERIFY_DEBUG(CurrentPos_ > 0); - size_t begin = CurrentPos_ - 1; // read first byte again - while (HasRemaining()) { - ReadNextByteUnsafe(); - if (!NPrometheus::IsValidMetricNameContinuation(CurrentByte_)) { - break; - } - } - return TokenFromPos(begin); - } - - TStringBuf ReadTokenAsLabelName() { - if (!NPrometheus::IsValidLabelNameStart(CurrentByte_)) { - return ""; - } - - Y_VERIFY_DEBUG(CurrentPos_ > 0); - size_t begin = CurrentPos_ - 1; // read first byte again - while (HasRemaining()) { - ReadNextByteUnsafe(); - if (!NPrometheus::IsValidLabelNameContinuation(CurrentByte_)) { - break; - } - } - return TokenFromPos(begin); - } - - TString ReadTokenAsLabelValue() { - TString labelValue; - - SkipExpectedChar('"'); - for (ui32 i = 0; i < MAX_LABEL_VALUE_LEN; i++) { - switch (CurrentByte_) { - case '"': - SkipExpectedChar('"'); - return labelValue; - - case '\n': - Y_PARSER_FAIL("label value contains unescaped new-line"); - - case '\\': - ReadNextByte(); - switch (CurrentByte_) { - case '"': - case '\\': - labelValue.append(CurrentByte_); - break; - case 'n': - labelValue.append('\n'); - break; - default: - Y_PARSER_FAIL("invalid escape sequence '" << CurrentByte_ << '\''); - } - break; - - default: - labelValue.append(CurrentByte_); - break; - } - - ReadNextByte(); - } - - Y_PARSER_FAIL("trying to parse too long label value, size >= " << MAX_LABEL_VALUE_LEN); - } - - TStringBuf TokenFromPos(size_t begin) { - Y_VERIFY_DEBUG(CurrentPos_ > begin); - size_t len = CurrentPos_ - begin - 1; - if (len == 0) { - return {}; - } - - return Data_.SubString(begin, len); - } - - void ConsumeLabels(TStringBuf name, const TLabelsMap& labels) { + ReadNextByteUnsafe(); + } + return TokenFromPos(begin); + } + + TStringBuf ReadTokenAsMetricName() { + if (!NPrometheus::IsValidMetricNameStart(CurrentByte_)) { + return ""; + } + + Y_VERIFY_DEBUG(CurrentPos_ > 0); + size_t begin = CurrentPos_ - 1; // read first byte again + while (HasRemaining()) { + ReadNextByteUnsafe(); + if (!NPrometheus::IsValidMetricNameContinuation(CurrentByte_)) { + break; + } + } + return TokenFromPos(begin); + } + + TStringBuf ReadTokenAsLabelName() { + if (!NPrometheus::IsValidLabelNameStart(CurrentByte_)) { + return ""; + } + + Y_VERIFY_DEBUG(CurrentPos_ > 0); + size_t begin = CurrentPos_ - 1; // read first byte again + while (HasRemaining()) { + ReadNextByteUnsafe(); + if (!NPrometheus::IsValidLabelNameContinuation(CurrentByte_)) { + break; + } + } + return TokenFromPos(begin); + } + + TString ReadTokenAsLabelValue() { + TString labelValue; + + SkipExpectedChar('"'); + for (ui32 i = 0; i < MAX_LABEL_VALUE_LEN; i++) { + switch (CurrentByte_) { + case '"': + SkipExpectedChar('"'); + return labelValue; + + case '\n': + Y_PARSER_FAIL("label value contains unescaped new-line"); + + case '\\': + ReadNextByte(); + switch (CurrentByte_) { + case '"': + case '\\': + labelValue.append(CurrentByte_); + break; + case 'n': + labelValue.append('\n'); + break; + default: + Y_PARSER_FAIL("invalid escape sequence '" << CurrentByte_ << '\''); + } + break; + + default: + labelValue.append(CurrentByte_); + break; + } + + ReadNextByte(); + } + + Y_PARSER_FAIL("trying to parse too long label value, size >= " << MAX_LABEL_VALUE_LEN); + } + + TStringBuf TokenFromPos(size_t begin) { + Y_VERIFY_DEBUG(CurrentPos_ > begin); + size_t len = CurrentPos_ - begin - 1; + if (len == 0) { + return {}; + } + + return Data_.SubString(begin, len); + } + + void ConsumeLabels(TStringBuf name, const TLabelsMap& labels) { Y_PARSER_ENSURE(labels.count(MetricNameLabel_) == 0, "label name '" << MetricNameLabel_ << "' is reserved, but is used with metric: " << name << LabelsToStr(labels)); - - Consumer_->OnLabelsBegin(); + + Consumer_->OnLabelsBegin(); Consumer_->OnLabel(MetricNameLabel_, TString(name)); // TODO: remove this string allocation - for (const auto& it: labels) { - Consumer_->OnLabel(it.first, it.second); - } - Consumer_->OnLabelsEnd(); - } - - void ConsumeCounter(TStringBuf name, const TLabelsMap& labels, TInstant time, double value) { + for (const auto& it: labels) { + Consumer_->OnLabel(it.first, it.second); + } + Consumer_->OnLabelsEnd(); + } + + void ConsumeCounter(TStringBuf name, const TLabelsMap& labels, TInstant time, double value) { i64 intValue{0}; // not nan if (value == value) { Y_PARSER_ENSURE(TryStaticCast(value, intValue), "value " << value << " is out of range"); } - - // see https://st.yandex-team.ru/SOLOMON-4142 for more details - // why we convert Prometheus COUNTER into Solomon RATE - // TODO: need to fix after server-side aggregation become correct for COUNTERs - Consumer_->OnMetricBegin(EMetricType::RATE); - ConsumeLabels(name, labels); - Consumer_->OnUint64(time, intValue); - Consumer_->OnMetricEnd(); - } - - void ConsumeGauge(TStringBuf name, const TLabelsMap& labels, TInstant time, double value) { - Consumer_->OnMetricBegin(EMetricType::GAUGE); - ConsumeLabels(name, labels); - Consumer_->OnDouble(time, value); - Consumer_->OnMetricEnd(); - } - - void ConsumeHistogram() { - Consumer_->OnMetricBegin(EMetricType::HIST_RATE); - ConsumeLabels(HistogramBuilder_.GetName(), HistogramBuilder_.GetLabels()); - auto time = HistogramBuilder_.GetTime(); - auto hist = HistogramBuilder_.ToSnapshot(); - Consumer_->OnHistogram(time, std::move(hist)); - Consumer_->OnMetricEnd(); - } - - double ParseGoDouble(TStringBuf str) { + + // see https://st.yandex-team.ru/SOLOMON-4142 for more details + // why we convert Prometheus COUNTER into Solomon RATE + // TODO: need to fix after server-side aggregation become correct for COUNTERs + Consumer_->OnMetricBegin(EMetricType::RATE); + ConsumeLabels(name, labels); + Consumer_->OnUint64(time, intValue); + Consumer_->OnMetricEnd(); + } + + void ConsumeGauge(TStringBuf name, const TLabelsMap& labels, TInstant time, double value) { + Consumer_->OnMetricBegin(EMetricType::GAUGE); + ConsumeLabels(name, labels); + Consumer_->OnDouble(time, value); + Consumer_->OnMetricEnd(); + } + + void ConsumeHistogram() { + Consumer_->OnMetricBegin(EMetricType::HIST_RATE); + ConsumeLabels(HistogramBuilder_.GetName(), HistogramBuilder_.GetLabels()); + auto time = HistogramBuilder_.GetTime(); + auto hist = HistogramBuilder_.ToSnapshot(); + Consumer_->OnHistogram(time, std::move(hist)); + Consumer_->OnMetricEnd(); + } + + double ParseGoDouble(TStringBuf str) { if (str == TStringBuf("+Inf")) { - return std::numeric_limits<double>::infinity(); + return std::numeric_limits<double>::infinity(); } else if (str == TStringBuf("-Inf")) { - return -std::numeric_limits<double>::infinity(); + return -std::numeric_limits<double>::infinity(); } else if (str == TStringBuf("NaN")) { - return NAN; - } - - double r = 0.0; - if (TryFromString(str, r)) { - return r; - } - Y_PARSER_FAIL("cannot parse double value from '" << str << "\' at line #" << CurrentLine_); - } - - private: - TStringBuf Data_; - IMetricConsumer* Consumer_; + return NAN; + } + + double r = 0.0; + if (TryFromString(str, r)) { + return r; + } + Y_PARSER_FAIL("cannot parse double value from '" << str << "\' at line #" << CurrentLine_); + } + + private: + TStringBuf Data_; + IMetricConsumer* Consumer_; TStringBuf MetricNameLabel_; - THashMap<TString, EPrometheusMetricType> SeenTypes_; - THistogramBuilder HistogramBuilder_; - - ui32 CurrentLine_ = 1; - ui32 CurrentPos_ = 0; - char CurrentByte_ = 0; - }; - } // namespace - + THashMap<TString, EPrometheusMetricType> SeenTypes_; + THistogramBuilder HistogramBuilder_; + + ui32 CurrentLine_ = 1; + ui32 CurrentPos_ = 0; + char CurrentByte_ = 0; + }; + } // namespace + void DecodePrometheus(TStringBuf data, IMetricConsumer* c, TStringBuf metricNameLabel) { TPrometheusReader reader(data, c, metricNameLabel); - reader.Read(); -} - -} // namespace NMonitoring + reader.Read(); +} + +} // namespace NMonitoring diff --git a/library/cpp/monlib/encode/prometheus/prometheus_decoder_ut.cpp b/library/cpp/monlib/encode/prometheus/prometheus_decoder_ut.cpp index 580b9c6617..49c2244fb4 100644 --- a/library/cpp/monlib/encode/prometheus/prometheus_decoder_ut.cpp +++ b/library/cpp/monlib/encode/prometheus/prometheus_decoder_ut.cpp @@ -1,478 +1,478 @@ -#include "prometheus.h" - -#include <library/cpp/monlib/encode/protobuf/protobuf.h> - +#include "prometheus.h" + +#include <library/cpp/monlib/encode/protobuf/protobuf.h> + #include <library/cpp/testing/unittest/registar.h> - -using namespace NMonitoring; - -#define ASSERT_LABEL_EQUAL(label, name, value) do { \ - UNIT_ASSERT_STRINGS_EQUAL((label).GetName(), name); \ - UNIT_ASSERT_STRINGS_EQUAL((label).GetValue(), value); \ - } while (false) - -#define ASSERT_DOUBLE_POINT(s, time, value) do { \ - UNIT_ASSERT_VALUES_EQUAL((s).GetTime(), (time).MilliSeconds()); \ - UNIT_ASSERT_EQUAL((s).GetValueCase(), NProto::TSingleSample::kFloat64); \ - UNIT_ASSERT_DOUBLES_EQUAL((s).GetFloat64(), value, std::numeric_limits<double>::epsilon()); \ - } while (false) - -#define ASSERT_UINT_POINT(s, time, value) do { \ - UNIT_ASSERT_VALUES_EQUAL((s).GetTime(), (time).MilliSeconds()); \ - UNIT_ASSERT_EQUAL((s).GetValueCase(), NProto::TSingleSample::kUint64); \ - UNIT_ASSERT_VALUES_EQUAL((s).GetUint64(), value); \ - } while (false) - -#define ASSERT_HIST_POINT(s, time, expected) do { \ - UNIT_ASSERT_VALUES_EQUAL((s).GetTime(), time.MilliSeconds()); \ - UNIT_ASSERT_EQUAL((s).GetValueCase(), NProto::TSingleSample::kHistogram);\ - UNIT_ASSERT_VALUES_EQUAL((s).GetHistogram().BoundsSize(), (expected).Count()); \ - UNIT_ASSERT_VALUES_EQUAL((s).GetHistogram().ValuesSize(), (expected).Count()); \ - for (size_t i = 0; i < (s).GetHistogram().BoundsSize(); i++) { \ - UNIT_ASSERT_DOUBLES_EQUAL((s).GetHistogram().GetBounds(i), (expected).UpperBound(i), Min<double>()); \ - UNIT_ASSERT_VALUES_EQUAL((s).GetHistogram().GetValues(i), (expected).Value(i)); \ - } \ - } while (false) - -Y_UNIT_TEST_SUITE(TPrometheusDecoderTest) { - - NProto::TSingleSamplesList Decode(TStringBuf data) { - NProto::TSingleSamplesList samples; - { - IMetricEncoderPtr e = EncoderProtobuf(&samples); - DecodePrometheus(data, e.Get()); - } - return samples; - } - - Y_UNIT_TEST(Empty) { - { - auto samples = Decode(""); - UNIT_ASSERT_EQUAL(samples.SamplesSize(), 0); - } - { - auto samples = Decode("\n"); - UNIT_ASSERT_EQUAL(samples.SamplesSize(), 0); - } - { - auto samples = Decode("\n \n \n"); - UNIT_ASSERT_EQUAL(samples.SamplesSize(), 0); - } - { - auto samples = Decode("\t\n\t\n"); - UNIT_ASSERT_EQUAL(samples.SamplesSize(), 0); - } - } - - Y_UNIT_TEST(Minimal) { - auto samples = Decode( - "minimal_metric 1.234\n" - "another_metric -3e3 103948\n" - "# Even that:\n" - "no_labels{} 3\n" - "# HELP line for non-existing metric will be ignored.\n"); - - UNIT_ASSERT_EQUAL(samples.SamplesSize(), 3); - { - auto& s = samples.GetSamples(0); - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); - UNIT_ASSERT_EQUAL(1, s.LabelsSize()); - ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "minimal_metric"); - ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 1.234); - } - { - auto& s = samples.GetSamples(1); - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); - UNIT_ASSERT_EQUAL(s.LabelsSize(), 1); - ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "another_metric"); - ASSERT_DOUBLE_POINT(s, TInstant::MilliSeconds(103948), -3000.0); - } - { - auto& s = samples.GetSamples(2); - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); - UNIT_ASSERT_EQUAL(1, s.LabelsSize()); - ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "no_labels"); - ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 3.0); - } - } - - Y_UNIT_TEST(Counter) { - auto samples = Decode( - "# A normal comment.\n" - "#\n" - "# TYPE name counter\n" - "name{labelname=\"val1\",basename=\"basevalue\"} NaN\n" - "name {labelname=\"val2\",basename=\"basevalue\"} 2.3 1234567890\n" - "# HELP name two-line\\n doc str\\\\ing\n"); - - UNIT_ASSERT_EQUAL(samples.SamplesSize(), 2); - - { - auto& s = samples.GetSamples(0); - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE); - UNIT_ASSERT_EQUAL(s.LabelsSize(), 3); - ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "name"); - ASSERT_LABEL_EQUAL(s.GetLabels(1), "basename", "basevalue"); - ASSERT_LABEL_EQUAL(s.GetLabels(2), "labelname", "val1"); - ASSERT_UINT_POINT(s, TInstant::Zero(), ui64(0)); - } - { - auto& s = samples.GetSamples(1); - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE); - UNIT_ASSERT_EQUAL(s.LabelsSize(), 3); - ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "name"); - ASSERT_LABEL_EQUAL(s.GetLabels(1), "basename", "basevalue"); - ASSERT_LABEL_EQUAL(s.GetLabels(2), "labelname", "val2"); - ASSERT_UINT_POINT(s, TInstant::MilliSeconds(1234567890), i64(2)); - } - } - - Y_UNIT_TEST(Gauge) { - auto samples = Decode( - "# A normal comment.\n" - "#\n" - " # HELP name2 \tdoc str\"ing 2\n" - " # TYPE name2 gauge\n" - "name2{labelname=\"val2\"\t,basename = \"basevalue2\"\t\t} +Inf 54321\n" - "name2{ labelname = \"val1\" , }-Inf\n"); - - UNIT_ASSERT_EQUAL(samples.SamplesSize(), 2); - - { - auto& s = samples.GetSamples(0); - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); - UNIT_ASSERT_EQUAL(s.LabelsSize(), 3); - ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "name2"); - ASSERT_LABEL_EQUAL(s.GetLabels(1), "basename", "basevalue2"); - ASSERT_LABEL_EQUAL(s.GetLabels(2), "labelname", "val2"); - ASSERT_DOUBLE_POINT(s, TInstant::MilliSeconds(54321), INFINITY); - } - { - auto& s = samples.GetSamples(1); - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); - UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); - ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "name2"); - ASSERT_LABEL_EQUAL(s.GetLabels(1), "labelname", "val1"); - ASSERT_DOUBLE_POINT(s, TInstant::Zero(), -INFINITY); - } - } - - Y_UNIT_TEST(Summary) { - auto samples = Decode( - "# HELP \n" - "# TYPE my_summary summary\n" - "my_summary{n1=\"val1\",quantile=\"0.5\"} 110\n" - "my_summary{n1=\"val1\",quantile=\"0.9\"} 140 1\n" - "my_summary_count{n1=\"val1\"} 42\n" - "my_summary_sum{n1=\"val1\"} 08 15\n" - "# some\n" - "# funny comments\n" - "# HELP\n" - "# HELP my_summary\n" - "# HELP my_summary \n"); - - UNIT_ASSERT_EQUAL(samples.SamplesSize(), 4); - - { - auto& s = samples.GetSamples(0); - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); - UNIT_ASSERT_EQUAL(s.LabelsSize(), 3); - ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "my_summary"); - ASSERT_LABEL_EQUAL(s.GetLabels(1), "quantile", "0.5"); - ASSERT_LABEL_EQUAL(s.GetLabels(2), "n1", "val1"); - ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 110.0); - } - { - auto& s = samples.GetSamples(1); - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); - UNIT_ASSERT_EQUAL(s.LabelsSize(), 3); - ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "my_summary"); - ASSERT_LABEL_EQUAL(s.GetLabels(1), "quantile", "0.9"); - ASSERT_LABEL_EQUAL(s.GetLabels(2), "n1", "val1"); - ASSERT_DOUBLE_POINT(s, TInstant::MilliSeconds(1), 140.0); - } - { - auto& s = samples.GetSamples(2); - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE); - UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); - ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "my_summary_count"); - ASSERT_LABEL_EQUAL(s.GetLabels(1), "n1", "val1"); - ASSERT_UINT_POINT(s, TInstant::Zero(), 42); - } - { - auto& s = samples.GetSamples(3); - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); - UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); - ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "my_summary_sum"); - ASSERT_LABEL_EQUAL(s.GetLabels(1), "n1", "val1"); - ASSERT_DOUBLE_POINT(s, TInstant::MilliSeconds(15), 8.0); - } - } - - Y_UNIT_TEST(Histogram) { - auto samples = Decode( - "# HELP request_duration_microseconds The response latency.\n" - "# TYPE request_duration_microseconds histogram\n" - "request_duration_microseconds_bucket{le=\"0\"} 0\n" - "request_duration_microseconds_bucket{le=\"100\"} 123\n" - "request_duration_microseconds_bucket{le=\"120\"} 412\n" - "request_duration_microseconds_bucket{le=\"144\"} 592\n" - "request_duration_microseconds_bucket{le=\"172.8\"} 1524\n" - "request_duration_microseconds_bucket{le=\"+Inf\"} 2693\n" - "request_duration_microseconds_sum 1.7560473e+06\n" - "request_duration_microseconds_count 2693\n"); - - UNIT_ASSERT_EQUAL(samples.SamplesSize(), 3); - - { - auto& s = samples.GetSamples(0); - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); - UNIT_ASSERT_EQUAL(s.LabelsSize(), 1); - ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "request_duration_microseconds_sum"); - ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 1756047.3); - } - { - auto& s = samples.GetSamples(1); - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE); - UNIT_ASSERT_EQUAL(s.LabelsSize(), 1); - ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "request_duration_microseconds_count"); - ASSERT_UINT_POINT(s, TInstant::Zero(), 2693); - } - { - auto& s = samples.GetSamples(2); - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::HIST_RATE); - UNIT_ASSERT_EQUAL(s.LabelsSize(), 1); - ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "request_duration_microseconds"); - auto hist = ExplicitHistogramSnapshot( - { 0, 100, 120, 144, 172.8, HISTOGRAM_INF_BOUND }, - { 0, 123, 289, 180, 932, 1169 }); - ASSERT_HIST_POINT(s, TInstant::Zero(), *hist); - } - } - - Y_UNIT_TEST(HistogramWithLabels) { - auto samples = Decode( - "# A histogram, which has a pretty complex representation in the text format:\n" - "# HELP http_request_duration_seconds A histogram of the request duration.\n" - "# TYPE http_request_duration_seconds histogram\n" - "http_request_duration_seconds_bucket{le=\"0.05\", method=\"POST\"} 24054\n" - "http_request_duration_seconds_bucket{method=\"POST\", le=\"0.1\"} 33444\n" - "http_request_duration_seconds_bucket{le=\"0.2\", method=\"POST\", } 100392\n" - "http_request_duration_seconds_bucket{le=\"0.5\",method=\"POST\",} 129389\n" - "http_request_duration_seconds_bucket{ method=\"POST\", le=\"1\", } 133988\n" - "http_request_duration_seconds_bucket{ le=\"+Inf\", method=\"POST\", } 144320\n" - "http_request_duration_seconds_sum{method=\"POST\"} 53423\n" - "http_request_duration_seconds_count{ method=\"POST\", } 144320\n"); - - UNIT_ASSERT_EQUAL(samples.SamplesSize(), 3); - - { - auto& s = samples.GetSamples(0); - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); - UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); - ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "http_request_duration_seconds_sum"); - ASSERT_LABEL_EQUAL(s.GetLabels(1), "method", "POST"); - ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 53423.0); - } - { - auto& s = samples.GetSamples(1); - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE); - UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); - ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "http_request_duration_seconds_count"); - ASSERT_LABEL_EQUAL(s.GetLabels(1), "method", "POST"); - ASSERT_UINT_POINT(s, TInstant::Zero(), 144320); - } - { - auto& s = samples.GetSamples(2); - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::HIST_RATE); - UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); - ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "http_request_duration_seconds"); - ASSERT_LABEL_EQUAL(s.GetLabels(1), "method", "POST"); - auto hist = ExplicitHistogramSnapshot( - { 0.05, 0.1, 0.2, 0.5, 1, HISTOGRAM_INF_BOUND }, - { 24054, 9390, 66948, 28997, 4599, 10332 }); - ASSERT_HIST_POINT(s, TInstant::Zero(), *hist); - } - } - - Y_UNIT_TEST(MultipleHistograms) { - auto samples = Decode( - "# TYPE inboundBytesPerSec histogram\n" - "inboundBytesPerSec_bucket{client=\"mbus\", le=\"10.0\"} 1.0\n" - "inboundBytesPerSec_bucket{client=\"mbus\", le=\"20.0\"} 5.0\n" - "inboundBytesPerSec_bucket{client=\"mbus\", le=\"+Inf\"} 5.0\n" - "inboundBytesPerSec_count{client=\"mbus\"} 5.0\n" - "inboundBytesPerSec_bucket{client=\"grpc\", le=\"10.0\"} 1.0\n" - "inboundBytesPerSec_bucket{client=\"grpc\", le=\"20.0\"} 5.0\n" - "inboundBytesPerSec_bucket{client=\"grpc\", le=\"30.0\"} 5.0\n" - "inboundBytesPerSec_count{client=\"grpc\"} 5.0\n" - "# TYPE outboundBytesPerSec histogram\n" - "outboundBytesPerSec_bucket{client=\"grpc\", le=\"100.0\"} 1.0 1512216000000\n" - "outboundBytesPerSec_bucket{client=\"grpc\", le=\"200.0\"} 1.0 1512216000000\n" - "outboundBytesPerSec_bucket{client=\"grpc\", le=\"+Inf\"} 1.0 1512216000000\n" - "outboundBytesPerSec_count{client=\"grpc\"} 1.0 1512216000000\n"); - - UNIT_ASSERT_EQUAL(samples.SamplesSize(), 6); - - { - auto& s = samples.GetSamples(0); - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE); - UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); - ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "inboundBytesPerSec_count"); - ASSERT_LABEL_EQUAL(s.GetLabels(1), "client", "mbus"); - ASSERT_UINT_POINT(s, TInstant::Zero(), 5); - } - { - auto& s = samples.GetSamples(1); - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::HIST_RATE); - UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); - ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "inboundBytesPerSec"); - ASSERT_LABEL_EQUAL(s.GetLabels(1), "client", "mbus"); - auto hist = ExplicitHistogramSnapshot( - { 10, 20, HISTOGRAM_INF_BOUND }, - { 1, 4, 0 }); - ASSERT_HIST_POINT(s, TInstant::Zero(), *hist); - } - { - auto& s = samples.GetSamples(2); - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE); - UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); - ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "inboundBytesPerSec_count"); - ASSERT_LABEL_EQUAL(s.GetLabels(1), "client", "grpc"); - ASSERT_UINT_POINT(s, TInstant::Zero(), 5); - } - { - auto& s = samples.GetSamples(3); - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::HIST_RATE); - UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); - ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "inboundBytesPerSec"); - ASSERT_LABEL_EQUAL(s.GetLabels(1), "client", "grpc"); - auto hist = ExplicitHistogramSnapshot( - { 10, 20, 30 }, - { 1, 4, 0 }); - ASSERT_HIST_POINT(s, TInstant::Zero(), *hist); - } - { - auto& s = samples.GetSamples(4); - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE); - UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); - ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "outboundBytesPerSec_count"); - ASSERT_LABEL_EQUAL(s.GetLabels(1), "client", "grpc"); - ASSERT_UINT_POINT(s, TInstant::Seconds(1512216000), 1) ; - } - { - auto& s = samples.GetSamples(5); - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::HIST_RATE); - UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); - ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "outboundBytesPerSec"); - ASSERT_LABEL_EQUAL(s.GetLabels(1), "client", "grpc"); - auto hist = ExplicitHistogramSnapshot( - { 100, 200, HISTOGRAM_INF_BOUND }, - { 1, 0, 0 }); - ASSERT_HIST_POINT(s, TInstant::Seconds(1512216000), *hist); - } - } - - Y_UNIT_TEST(MixedTypes) { - auto samples = Decode( - "# HELP http_requests_total The total number of HTTP requests.\n" - "# TYPE http_requests_total counter\n" - "http_requests_total { } 1027 1395066363000\n" - "http_requests_total{method=\"post\",code=\"200\"} 1027 1395066363000\n" - "http_requests_total{method=\"post\",code=\"400\"} 3 1395066363000\n" - "\n" - "# Minimalistic line:\n" - "metric_without_timestamp_and_labels 12.47\n" - "\n" - "# HELP rpc_duration_seconds A summary of the RPC duration in seconds.\n" - "# TYPE rpc_duration_seconds summary\n" - "rpc_duration_seconds{quantile=\"0.01\"} 3102\n" - "rpc_duration_seconds{quantile=\"0.5\"} 4773\n" - "rpc_duration_seconds{quantile=\"0.9\"} 9001\n" - "rpc_duration_seconds_sum 1.7560473e+07\n" - "rpc_duration_seconds_count 2693\n" - "\n" - "# Another mMinimalistic line:\n" - "metric_with_timestamp 12.47 1234567890\n"); - - UNIT_ASSERT_EQUAL(samples.SamplesSize(), 10); - - { - auto& s = samples.GetSamples(0); - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE); - UNIT_ASSERT_EQUAL(s.LabelsSize(), 1); - ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "http_requests_total"); - ASSERT_UINT_POINT(s, TInstant::Seconds(1395066363), 1027); - } - { - auto& s = samples.GetSamples(1); - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE); - UNIT_ASSERT_EQUAL(s.LabelsSize(), 3); - ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "http_requests_total"); - ASSERT_LABEL_EQUAL(s.GetLabels(1), "method", "post"); - ASSERT_LABEL_EQUAL(s.GetLabels(2), "code", "200"); - ASSERT_UINT_POINT(s, TInstant::Seconds(1395066363), 1027); - } - { - auto& s = samples.GetSamples(2); - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE); - UNIT_ASSERT_EQUAL(s.LabelsSize(), 3); - ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "http_requests_total"); - ASSERT_LABEL_EQUAL(s.GetLabels(1), "method", "post"); - ASSERT_LABEL_EQUAL(s.GetLabels(2), "code", "400"); - ASSERT_UINT_POINT(s, TInstant::Seconds(1395066363), 3); - } - { - auto& s = samples.GetSamples(3); - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); - UNIT_ASSERT_EQUAL(s.LabelsSize(), 1); - ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "metric_without_timestamp_and_labels"); - ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 12.47); - } - { - auto& s = samples.GetSamples(4); - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); - UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); - ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "rpc_duration_seconds"); - ASSERT_LABEL_EQUAL(s.GetLabels(1), "quantile", "0.01"); - ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 3102); - } - { - auto& s = samples.GetSamples(5); - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); - UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); - ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "rpc_duration_seconds"); - ASSERT_LABEL_EQUAL(s.GetLabels(1), "quantile", "0.5"); - ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 4773); - } - { - auto& s = samples.GetSamples(6); - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); - UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); - ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "rpc_duration_seconds"); - ASSERT_LABEL_EQUAL(s.GetLabels(1), "quantile", "0.9"); - ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 9001); - } - { - auto& s = samples.GetSamples(7); - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); - UNIT_ASSERT_EQUAL(s.LabelsSize(), 1); - ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "rpc_duration_seconds_sum"); - ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 17560473); - } - { - auto& s = samples.GetSamples(8); - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE); - UNIT_ASSERT_EQUAL(s.LabelsSize(), 1); - ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "rpc_duration_seconds_count"); - ASSERT_UINT_POINT(s, TInstant::Zero(), 2693); - } - { - auto& s = samples.GetSamples(9); - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); - UNIT_ASSERT_EQUAL(s.LabelsSize(), 1); - ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "metric_with_timestamp"); - ASSERT_DOUBLE_POINT(s, TInstant::MilliSeconds(1234567890), 12.47); - } - } -} + +using namespace NMonitoring; + +#define ASSERT_LABEL_EQUAL(label, name, value) do { \ + UNIT_ASSERT_STRINGS_EQUAL((label).GetName(), name); \ + UNIT_ASSERT_STRINGS_EQUAL((label).GetValue(), value); \ + } while (false) + +#define ASSERT_DOUBLE_POINT(s, time, value) do { \ + UNIT_ASSERT_VALUES_EQUAL((s).GetTime(), (time).MilliSeconds()); \ + UNIT_ASSERT_EQUAL((s).GetValueCase(), NProto::TSingleSample::kFloat64); \ + UNIT_ASSERT_DOUBLES_EQUAL((s).GetFloat64(), value, std::numeric_limits<double>::epsilon()); \ + } while (false) + +#define ASSERT_UINT_POINT(s, time, value) do { \ + UNIT_ASSERT_VALUES_EQUAL((s).GetTime(), (time).MilliSeconds()); \ + UNIT_ASSERT_EQUAL((s).GetValueCase(), NProto::TSingleSample::kUint64); \ + UNIT_ASSERT_VALUES_EQUAL((s).GetUint64(), value); \ + } while (false) + +#define ASSERT_HIST_POINT(s, time, expected) do { \ + UNIT_ASSERT_VALUES_EQUAL((s).GetTime(), time.MilliSeconds()); \ + UNIT_ASSERT_EQUAL((s).GetValueCase(), NProto::TSingleSample::kHistogram);\ + UNIT_ASSERT_VALUES_EQUAL((s).GetHistogram().BoundsSize(), (expected).Count()); \ + UNIT_ASSERT_VALUES_EQUAL((s).GetHistogram().ValuesSize(), (expected).Count()); \ + for (size_t i = 0; i < (s).GetHistogram().BoundsSize(); i++) { \ + UNIT_ASSERT_DOUBLES_EQUAL((s).GetHistogram().GetBounds(i), (expected).UpperBound(i), Min<double>()); \ + UNIT_ASSERT_VALUES_EQUAL((s).GetHistogram().GetValues(i), (expected).Value(i)); \ + } \ + } while (false) + +Y_UNIT_TEST_SUITE(TPrometheusDecoderTest) { + + NProto::TSingleSamplesList Decode(TStringBuf data) { + NProto::TSingleSamplesList samples; + { + IMetricEncoderPtr e = EncoderProtobuf(&samples); + DecodePrometheus(data, e.Get()); + } + return samples; + } + + Y_UNIT_TEST(Empty) { + { + auto samples = Decode(""); + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 0); + } + { + auto samples = Decode("\n"); + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 0); + } + { + auto samples = Decode("\n \n \n"); + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 0); + } + { + auto samples = Decode("\t\n\t\n"); + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 0); + } + } + + Y_UNIT_TEST(Minimal) { + auto samples = Decode( + "minimal_metric 1.234\n" + "another_metric -3e3 103948\n" + "# Even that:\n" + "no_labels{} 3\n" + "# HELP line for non-existing metric will be ignored.\n"); + + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 3); + { + auto& s = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(1, s.LabelsSize()); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "minimal_metric"); + ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 1.234); + } + { + auto& s = samples.GetSamples(1); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 1); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "another_metric"); + ASSERT_DOUBLE_POINT(s, TInstant::MilliSeconds(103948), -3000.0); + } + { + auto& s = samples.GetSamples(2); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(1, s.LabelsSize()); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "no_labels"); + ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 3.0); + } + } + + Y_UNIT_TEST(Counter) { + auto samples = Decode( + "# A normal comment.\n" + "#\n" + "# TYPE name counter\n" + "name{labelname=\"val1\",basename=\"basevalue\"} NaN\n" + "name {labelname=\"val2\",basename=\"basevalue\"} 2.3 1234567890\n" + "# HELP name two-line\\n doc str\\\\ing\n"); + + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 2); + + { + auto& s = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 3); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "name"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "basename", "basevalue"); + ASSERT_LABEL_EQUAL(s.GetLabels(2), "labelname", "val1"); + ASSERT_UINT_POINT(s, TInstant::Zero(), ui64(0)); + } + { + auto& s = samples.GetSamples(1); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 3); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "name"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "basename", "basevalue"); + ASSERT_LABEL_EQUAL(s.GetLabels(2), "labelname", "val2"); + ASSERT_UINT_POINT(s, TInstant::MilliSeconds(1234567890), i64(2)); + } + } + + Y_UNIT_TEST(Gauge) { + auto samples = Decode( + "# A normal comment.\n" + "#\n" + " # HELP name2 \tdoc str\"ing 2\n" + " # TYPE name2 gauge\n" + "name2{labelname=\"val2\"\t,basename = \"basevalue2\"\t\t} +Inf 54321\n" + "name2{ labelname = \"val1\" , }-Inf\n"); + + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 2); + + { + auto& s = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 3); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "name2"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "basename", "basevalue2"); + ASSERT_LABEL_EQUAL(s.GetLabels(2), "labelname", "val2"); + ASSERT_DOUBLE_POINT(s, TInstant::MilliSeconds(54321), INFINITY); + } + { + auto& s = samples.GetSamples(1); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "name2"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "labelname", "val1"); + ASSERT_DOUBLE_POINT(s, TInstant::Zero(), -INFINITY); + } + } + + Y_UNIT_TEST(Summary) { + auto samples = Decode( + "# HELP \n" + "# TYPE my_summary summary\n" + "my_summary{n1=\"val1\",quantile=\"0.5\"} 110\n" + "my_summary{n1=\"val1\",quantile=\"0.9\"} 140 1\n" + "my_summary_count{n1=\"val1\"} 42\n" + "my_summary_sum{n1=\"val1\"} 08 15\n" + "# some\n" + "# funny comments\n" + "# HELP\n" + "# HELP my_summary\n" + "# HELP my_summary \n"); + + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 4); + + { + auto& s = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 3); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "my_summary"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "quantile", "0.5"); + ASSERT_LABEL_EQUAL(s.GetLabels(2), "n1", "val1"); + ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 110.0); + } + { + auto& s = samples.GetSamples(1); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 3); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "my_summary"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "quantile", "0.9"); + ASSERT_LABEL_EQUAL(s.GetLabels(2), "n1", "val1"); + ASSERT_DOUBLE_POINT(s, TInstant::MilliSeconds(1), 140.0); + } + { + auto& s = samples.GetSamples(2); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "my_summary_count"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "n1", "val1"); + ASSERT_UINT_POINT(s, TInstant::Zero(), 42); + } + { + auto& s = samples.GetSamples(3); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "my_summary_sum"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "n1", "val1"); + ASSERT_DOUBLE_POINT(s, TInstant::MilliSeconds(15), 8.0); + } + } + + Y_UNIT_TEST(Histogram) { + auto samples = Decode( + "# HELP request_duration_microseconds The response latency.\n" + "# TYPE request_duration_microseconds histogram\n" + "request_duration_microseconds_bucket{le=\"0\"} 0\n" + "request_duration_microseconds_bucket{le=\"100\"} 123\n" + "request_duration_microseconds_bucket{le=\"120\"} 412\n" + "request_duration_microseconds_bucket{le=\"144\"} 592\n" + "request_duration_microseconds_bucket{le=\"172.8\"} 1524\n" + "request_duration_microseconds_bucket{le=\"+Inf\"} 2693\n" + "request_duration_microseconds_sum 1.7560473e+06\n" + "request_duration_microseconds_count 2693\n"); + + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 3); + + { + auto& s = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 1); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "request_duration_microseconds_sum"); + ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 1756047.3); + } + { + auto& s = samples.GetSamples(1); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 1); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "request_duration_microseconds_count"); + ASSERT_UINT_POINT(s, TInstant::Zero(), 2693); + } + { + auto& s = samples.GetSamples(2); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::HIST_RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 1); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "request_duration_microseconds"); + auto hist = ExplicitHistogramSnapshot( + { 0, 100, 120, 144, 172.8, HISTOGRAM_INF_BOUND }, + { 0, 123, 289, 180, 932, 1169 }); + ASSERT_HIST_POINT(s, TInstant::Zero(), *hist); + } + } + + Y_UNIT_TEST(HistogramWithLabels) { + auto samples = Decode( + "# A histogram, which has a pretty complex representation in the text format:\n" + "# HELP http_request_duration_seconds A histogram of the request duration.\n" + "# TYPE http_request_duration_seconds histogram\n" + "http_request_duration_seconds_bucket{le=\"0.05\", method=\"POST\"} 24054\n" + "http_request_duration_seconds_bucket{method=\"POST\", le=\"0.1\"} 33444\n" + "http_request_duration_seconds_bucket{le=\"0.2\", method=\"POST\", } 100392\n" + "http_request_duration_seconds_bucket{le=\"0.5\",method=\"POST\",} 129389\n" + "http_request_duration_seconds_bucket{ method=\"POST\", le=\"1\", } 133988\n" + "http_request_duration_seconds_bucket{ le=\"+Inf\", method=\"POST\", } 144320\n" + "http_request_duration_seconds_sum{method=\"POST\"} 53423\n" + "http_request_duration_seconds_count{ method=\"POST\", } 144320\n"); + + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 3); + + { + auto& s = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "http_request_duration_seconds_sum"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "method", "POST"); + ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 53423.0); + } + { + auto& s = samples.GetSamples(1); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "http_request_duration_seconds_count"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "method", "POST"); + ASSERT_UINT_POINT(s, TInstant::Zero(), 144320); + } + { + auto& s = samples.GetSamples(2); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::HIST_RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "http_request_duration_seconds"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "method", "POST"); + auto hist = ExplicitHistogramSnapshot( + { 0.05, 0.1, 0.2, 0.5, 1, HISTOGRAM_INF_BOUND }, + { 24054, 9390, 66948, 28997, 4599, 10332 }); + ASSERT_HIST_POINT(s, TInstant::Zero(), *hist); + } + } + + Y_UNIT_TEST(MultipleHistograms) { + auto samples = Decode( + "# TYPE inboundBytesPerSec histogram\n" + "inboundBytesPerSec_bucket{client=\"mbus\", le=\"10.0\"} 1.0\n" + "inboundBytesPerSec_bucket{client=\"mbus\", le=\"20.0\"} 5.0\n" + "inboundBytesPerSec_bucket{client=\"mbus\", le=\"+Inf\"} 5.0\n" + "inboundBytesPerSec_count{client=\"mbus\"} 5.0\n" + "inboundBytesPerSec_bucket{client=\"grpc\", le=\"10.0\"} 1.0\n" + "inboundBytesPerSec_bucket{client=\"grpc\", le=\"20.0\"} 5.0\n" + "inboundBytesPerSec_bucket{client=\"grpc\", le=\"30.0\"} 5.0\n" + "inboundBytesPerSec_count{client=\"grpc\"} 5.0\n" + "# TYPE outboundBytesPerSec histogram\n" + "outboundBytesPerSec_bucket{client=\"grpc\", le=\"100.0\"} 1.0 1512216000000\n" + "outboundBytesPerSec_bucket{client=\"grpc\", le=\"200.0\"} 1.0 1512216000000\n" + "outboundBytesPerSec_bucket{client=\"grpc\", le=\"+Inf\"} 1.0 1512216000000\n" + "outboundBytesPerSec_count{client=\"grpc\"} 1.0 1512216000000\n"); + + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 6); + + { + auto& s = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "inboundBytesPerSec_count"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "client", "mbus"); + ASSERT_UINT_POINT(s, TInstant::Zero(), 5); + } + { + auto& s = samples.GetSamples(1); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::HIST_RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "inboundBytesPerSec"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "client", "mbus"); + auto hist = ExplicitHistogramSnapshot( + { 10, 20, HISTOGRAM_INF_BOUND }, + { 1, 4, 0 }); + ASSERT_HIST_POINT(s, TInstant::Zero(), *hist); + } + { + auto& s = samples.GetSamples(2); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "inboundBytesPerSec_count"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "client", "grpc"); + ASSERT_UINT_POINT(s, TInstant::Zero(), 5); + } + { + auto& s = samples.GetSamples(3); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::HIST_RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "inboundBytesPerSec"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "client", "grpc"); + auto hist = ExplicitHistogramSnapshot( + { 10, 20, 30 }, + { 1, 4, 0 }); + ASSERT_HIST_POINT(s, TInstant::Zero(), *hist); + } + { + auto& s = samples.GetSamples(4); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "outboundBytesPerSec_count"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "client", "grpc"); + ASSERT_UINT_POINT(s, TInstant::Seconds(1512216000), 1) ; + } + { + auto& s = samples.GetSamples(5); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::HIST_RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "outboundBytesPerSec"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "client", "grpc"); + auto hist = ExplicitHistogramSnapshot( + { 100, 200, HISTOGRAM_INF_BOUND }, + { 1, 0, 0 }); + ASSERT_HIST_POINT(s, TInstant::Seconds(1512216000), *hist); + } + } + + Y_UNIT_TEST(MixedTypes) { + auto samples = Decode( + "# HELP http_requests_total The total number of HTTP requests.\n" + "# TYPE http_requests_total counter\n" + "http_requests_total { } 1027 1395066363000\n" + "http_requests_total{method=\"post\",code=\"200\"} 1027 1395066363000\n" + "http_requests_total{method=\"post\",code=\"400\"} 3 1395066363000\n" + "\n" + "# Minimalistic line:\n" + "metric_without_timestamp_and_labels 12.47\n" + "\n" + "# HELP rpc_duration_seconds A summary of the RPC duration in seconds.\n" + "# TYPE rpc_duration_seconds summary\n" + "rpc_duration_seconds{quantile=\"0.01\"} 3102\n" + "rpc_duration_seconds{quantile=\"0.5\"} 4773\n" + "rpc_duration_seconds{quantile=\"0.9\"} 9001\n" + "rpc_duration_seconds_sum 1.7560473e+07\n" + "rpc_duration_seconds_count 2693\n" + "\n" + "# Another mMinimalistic line:\n" + "metric_with_timestamp 12.47 1234567890\n"); + + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 10); + + { + auto& s = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 1); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "http_requests_total"); + ASSERT_UINT_POINT(s, TInstant::Seconds(1395066363), 1027); + } + { + auto& s = samples.GetSamples(1); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 3); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "http_requests_total"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "method", "post"); + ASSERT_LABEL_EQUAL(s.GetLabels(2), "code", "200"); + ASSERT_UINT_POINT(s, TInstant::Seconds(1395066363), 1027); + } + { + auto& s = samples.GetSamples(2); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 3); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "http_requests_total"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "method", "post"); + ASSERT_LABEL_EQUAL(s.GetLabels(2), "code", "400"); + ASSERT_UINT_POINT(s, TInstant::Seconds(1395066363), 3); + } + { + auto& s = samples.GetSamples(3); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 1); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "metric_without_timestamp_and_labels"); + ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 12.47); + } + { + auto& s = samples.GetSamples(4); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "rpc_duration_seconds"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "quantile", "0.01"); + ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 3102); + } + { + auto& s = samples.GetSamples(5); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "rpc_duration_seconds"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "quantile", "0.5"); + ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 4773); + } + { + auto& s = samples.GetSamples(6); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "rpc_duration_seconds"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "quantile", "0.9"); + ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 9001); + } + { + auto& s = samples.GetSamples(7); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 1); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "rpc_duration_seconds_sum"); + ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 17560473); + } + { + auto& s = samples.GetSamples(8); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 1); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "rpc_duration_seconds_count"); + ASSERT_UINT_POINT(s, TInstant::Zero(), 2693); + } + { + auto& s = samples.GetSamples(9); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 1); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "metric_with_timestamp"); + ASSERT_DOUBLE_POINT(s, TInstant::MilliSeconds(1234567890), 12.47); + } + } +} diff --git a/library/cpp/monlib/encode/prometheus/prometheus_encoder.cpp b/library/cpp/monlib/encode/prometheus/prometheus_encoder.cpp index 3baf90fa2e..15efeb8c03 100644 --- a/library/cpp/monlib/encode/prometheus/prometheus_encoder.cpp +++ b/library/cpp/monlib/encode/prometheus/prometheus_encoder.cpp @@ -1,95 +1,95 @@ -#include "prometheus.h" -#include "prometheus_model.h" - -#include <library/cpp/monlib/encode/encoder_state.h> -#include <library/cpp/monlib/metrics/labels.h> -#include <library/cpp/monlib/metrics/metric_value.h> - -#include <util/string/cast.h> -#include <util/generic/hash_set.h> - - -namespace NMonitoring { - namespace { - /////////////////////////////////////////////////////////////////////// - // TPrometheusWriter - /////////////////////////////////////////////////////////////////////// - class TPrometheusWriter { - public: - explicit TPrometheusWriter(IOutputStream* out) - : Out_(out) - { - } - - void WriteType(EMetricType type, const TString& name) { - auto r = WrittenTypes_.insert(name); - if (!r.second) { - // type for this metric was already written - return; - } - - Out_->Write("# TYPE "); - WriteMetricName(name); - Out_->Write(' '); - - switch (type) { - case EMetricType::GAUGE: - case EMetricType::IGAUGE: - Out_->Write("gauge"); - break; - case EMetricType::RATE: - case EMetricType::COUNTER: - Out_->Write("counter"); - break; - case EMetricType::HIST: - case EMetricType::HIST_RATE: - Out_->Write("histogram"); - break; +#include "prometheus.h" +#include "prometheus_model.h" + +#include <library/cpp/monlib/encode/encoder_state.h> +#include <library/cpp/monlib/metrics/labels.h> +#include <library/cpp/monlib/metrics/metric_value.h> + +#include <util/string/cast.h> +#include <util/generic/hash_set.h> + + +namespace NMonitoring { + namespace { + /////////////////////////////////////////////////////////////////////// + // TPrometheusWriter + /////////////////////////////////////////////////////////////////////// + class TPrometheusWriter { + public: + explicit TPrometheusWriter(IOutputStream* out) + : Out_(out) + { + } + + void WriteType(EMetricType type, const TString& name) { + auto r = WrittenTypes_.insert(name); + if (!r.second) { + // type for this metric was already written + return; + } + + Out_->Write("# TYPE "); + WriteMetricName(name); + Out_->Write(' '); + + switch (type) { + case EMetricType::GAUGE: + case EMetricType::IGAUGE: + Out_->Write("gauge"); + break; + case EMetricType::RATE: + case EMetricType::COUNTER: + Out_->Write("counter"); + break; + case EMetricType::HIST: + case EMetricType::HIST_RATE: + Out_->Write("histogram"); + break; case EMetricType::LOGHIST: // TODO(@kbalakirev): implement this case break; - case EMetricType::DSUMMARY: + case EMetricType::DSUMMARY: ythrow yexception() << "writing summary type is forbiden"; - case EMetricType::UNKNOWN: - ythrow yexception() << "unknown metric type: " << MetricTypeToStr(type) - << ", name: " << name; - } - Out_->Write('\n'); - } - - void WriteDouble(TStringBuf name, const TLabels& labels, TInstant time, double value) { - WriteValue(name, "", labels, "", "", time, value); - } - - void WriteHistogram(TStringBuf name, const TLabels& labels, TInstant time, IHistogramSnapshot* h) { - Y_ENSURE(!labels.Has(NPrometheus::BUCKET_LABEL), - "histogram metric " << name << " has label '" << - NPrometheus::BUCKET_LABEL << "' which is reserved in Prometheus"); - - double totalCount = 0; - for (ui32 i = 0, count = h->Count(); i < count; i++) { - TBucketBound bound = h->UpperBound(i); - TStringBuf boundStr; - if (bound == HISTOGRAM_INF_BOUND) { + case EMetricType::UNKNOWN: + ythrow yexception() << "unknown metric type: " << MetricTypeToStr(type) + << ", name: " << name; + } + Out_->Write('\n'); + } + + void WriteDouble(TStringBuf name, const TLabels& labels, TInstant time, double value) { + WriteValue(name, "", labels, "", "", time, value); + } + + void WriteHistogram(TStringBuf name, const TLabels& labels, TInstant time, IHistogramSnapshot* h) { + Y_ENSURE(!labels.Has(NPrometheus::BUCKET_LABEL), + "histogram metric " << name << " has label '" << + NPrometheus::BUCKET_LABEL << "' which is reserved in Prometheus"); + + double totalCount = 0; + for (ui32 i = 0, count = h->Count(); i < count; i++) { + TBucketBound bound = h->UpperBound(i); + TStringBuf boundStr; + if (bound == HISTOGRAM_INF_BOUND) { boundStr = TStringBuf("+Inf"); - } else { - size_t len = FloatToString(bound, TmpBuf_, Y_ARRAY_SIZE(TmpBuf_)); - boundStr = TStringBuf(TmpBuf_, len); - } - - TBucketValue value = h->Value(i); - totalCount += static_cast<double>(value); - - WriteValue( - name, NPrometheus::BUCKET_SUFFIX, - labels, NPrometheus::BUCKET_LABEL, boundStr, - time, - totalCount); - } - - WriteValue(name, NPrometheus::COUNT_SUFFIX, labels, "", "", time, totalCount); - } - + } else { + size_t len = FloatToString(bound, TmpBuf_, Y_ARRAY_SIZE(TmpBuf_)); + boundStr = TStringBuf(TmpBuf_, len); + } + + TBucketValue value = h->Value(i); + totalCount += static_cast<double>(value); + + WriteValue( + name, NPrometheus::BUCKET_SUFFIX, + labels, NPrometheus::BUCKET_LABEL, boundStr, + time, + totalCount); + } + + WriteValue(name, NPrometheus::COUNT_SUFFIX, labels, "", "", time, totalCount); + } + void WriteSummaryDouble(TStringBuf name, const TLabels& labels, TInstant time, ISummaryDoubleSnapshot* s) { WriteValue(name, NPrometheus::SUM_SUFFIX, labels, "", "", time, s->GetSum()); WriteValue(name, NPrometheus::MIN_SUFFIX, labels, "", "", time, s->GetMin()); @@ -98,316 +98,316 @@ namespace NMonitoring { WriteValue(name, NPrometheus::COUNT_SUFFIX, labels, "", "", time, s->GetCount()); } - void WriteLn() { - Out_->Write('\n'); - } - - private: - // will replace invalid chars with '_' - void WriteMetricName(TStringBuf name) { - Y_ENSURE(!name.Empty(), "trying to write metric with empty name"); - - char ch = name[0]; - if (NPrometheus::IsValidMetricNameStart(ch)) { - Out_->Write(ch); - } else { - Out_->Write('_'); - } - - for (size_t i = 1, len = name.length(); i < len; i++) { - ch = name[i]; - if (NPrometheus::IsValidMetricNameContinuation(ch)) { - Out_->Write(ch); - } else { - Out_->Write('_'); - } - } - } - - void WriteLabels(const TLabels& labels, TStringBuf addLabelKey, TStringBuf addLabelValue) { - Out_->Write('{'); + void WriteLn() { + Out_->Write('\n'); + } + + private: + // will replace invalid chars with '_' + void WriteMetricName(TStringBuf name) { + Y_ENSURE(!name.Empty(), "trying to write metric with empty name"); + + char ch = name[0]; + if (NPrometheus::IsValidMetricNameStart(ch)) { + Out_->Write(ch); + } else { + Out_->Write('_'); + } + + for (size_t i = 1, len = name.length(); i < len; i++) { + ch = name[i]; + if (NPrometheus::IsValidMetricNameContinuation(ch)) { + Out_->Write(ch); + } else { + Out_->Write('_'); + } + } + } + + void WriteLabels(const TLabels& labels, TStringBuf addLabelKey, TStringBuf addLabelValue) { + Out_->Write('{'); for (auto&& l: labels) { - Out_->Write(l.Name()); - Out_->Write('='); - WriteLabelValue(l.Value()); - Out_->Write(", "); // trailign comma is supported in parsers - } - if (!addLabelKey.Empty() && !addLabelValue.Empty()) { - Out_->Write(addLabelKey); - Out_->Write('='); - WriteLabelValue(addLabelValue); - } - Out_->Write('}'); - } - - void WriteLabelValue(TStringBuf value) { - Out_->Write('"'); - for (char ch: value) { - if (ch == '"') { - Out_->Write("\\\""); - } else if (ch == '\\') { - Out_->Write("\\\\"); - } else if (ch == '\n') { - Out_->Write("\\n"); - } else { - Out_->Write(ch); - } - } - Out_->Write('"'); - } - - void WriteValue( - TStringBuf name, TStringBuf suffix, - const TLabels& labels, TStringBuf addLabelKey, TStringBuf addLabelValue, - TInstant time, double value) - { - // (1) name - WriteMetricName(name); - if (!suffix.Empty()) { - Out_->Write(suffix); - } - - // (2) labels - if (!labels.Empty() || !addLabelKey.Empty()) { - WriteLabels(labels, addLabelKey, addLabelValue); - } - Out_->Write(' '); - - // (3) value - { - size_t len = FloatToString(value, TmpBuf_, Y_ARRAY_SIZE(TmpBuf_)); - Out_->Write(TmpBuf_, len); - } - - // (4) time - if (ui64 timeMillis = time.MilliSeconds()) { - Out_->Write(' '); - size_t len = IntToString<10>(timeMillis, TmpBuf_, Y_ARRAY_SIZE(TmpBuf_)); - Out_->Write(TmpBuf_, len); - } - Out_->Write('\n'); - } - - private: - IOutputStream* Out_; - THashSet<TString> WrittenTypes_; - char TmpBuf_[512]; // used to convert doubles to strings - }; - - /////////////////////////////////////////////////////////////////////// - // TMetricState - /////////////////////////////////////////////////////////////////////// - struct TMetricState { - EMetricType Type = EMetricType::UNKNOWN; - TLabels Labels; - TInstant Time = TInstant::Zero(); - EMetricValueType ValueType = EMetricValueType::UNKNOWN; - TMetricValue Value; - - ~TMetricState() { - ClearValue(); - } - - void Clear() { - Type = EMetricType::UNKNOWN; - Labels.Clear(); - Time = TInstant::Zero(); - ClearValue(); - } - - void ClearValue() { - // TMetricValue does not keep ownership of histogram - if (ValueType == EMetricValueType::HISTOGRAM) { - Value.AsHistogram()->UnRef(); - } else if (ValueType == EMetricValueType::SUMMARY) { + Out_->Write(l.Name()); + Out_->Write('='); + WriteLabelValue(l.Value()); + Out_->Write(", "); // trailign comma is supported in parsers + } + if (!addLabelKey.Empty() && !addLabelValue.Empty()) { + Out_->Write(addLabelKey); + Out_->Write('='); + WriteLabelValue(addLabelValue); + } + Out_->Write('}'); + } + + void WriteLabelValue(TStringBuf value) { + Out_->Write('"'); + for (char ch: value) { + if (ch == '"') { + Out_->Write("\\\""); + } else if (ch == '\\') { + Out_->Write("\\\\"); + } else if (ch == '\n') { + Out_->Write("\\n"); + } else { + Out_->Write(ch); + } + } + Out_->Write('"'); + } + + void WriteValue( + TStringBuf name, TStringBuf suffix, + const TLabels& labels, TStringBuf addLabelKey, TStringBuf addLabelValue, + TInstant time, double value) + { + // (1) name + WriteMetricName(name); + if (!suffix.Empty()) { + Out_->Write(suffix); + } + + // (2) labels + if (!labels.Empty() || !addLabelKey.Empty()) { + WriteLabels(labels, addLabelKey, addLabelValue); + } + Out_->Write(' '); + + // (3) value + { + size_t len = FloatToString(value, TmpBuf_, Y_ARRAY_SIZE(TmpBuf_)); + Out_->Write(TmpBuf_, len); + } + + // (4) time + if (ui64 timeMillis = time.MilliSeconds()) { + Out_->Write(' '); + size_t len = IntToString<10>(timeMillis, TmpBuf_, Y_ARRAY_SIZE(TmpBuf_)); + Out_->Write(TmpBuf_, len); + } + Out_->Write('\n'); + } + + private: + IOutputStream* Out_; + THashSet<TString> WrittenTypes_; + char TmpBuf_[512]; // used to convert doubles to strings + }; + + /////////////////////////////////////////////////////////////////////// + // TMetricState + /////////////////////////////////////////////////////////////////////// + struct TMetricState { + EMetricType Type = EMetricType::UNKNOWN; + TLabels Labels; + TInstant Time = TInstant::Zero(); + EMetricValueType ValueType = EMetricValueType::UNKNOWN; + TMetricValue Value; + + ~TMetricState() { + ClearValue(); + } + + void Clear() { + Type = EMetricType::UNKNOWN; + Labels.Clear(); + Time = TInstant::Zero(); + ClearValue(); + } + + void ClearValue() { + // TMetricValue does not keep ownership of histogram + if (ValueType == EMetricValueType::HISTOGRAM) { + Value.AsHistogram()->UnRef(); + } else if (ValueType == EMetricValueType::SUMMARY) { Value.AsSummaryDouble()->UnRef(); - } - ValueType = EMetricValueType::UNKNOWN; - Value = {}; - } - - template <typename T> - void SetValue(T value) { - // TMetricValue does not keep ownership of histogram - if (ValueType == EMetricValueType::HISTOGRAM) { - Value.AsHistogram()->UnRef(); - } else if (ValueType == EMetricValueType::SUMMARY) { + } + ValueType = EMetricValueType::UNKNOWN; + Value = {}; + } + + template <typename T> + void SetValue(T value) { + // TMetricValue does not keep ownership of histogram + if (ValueType == EMetricValueType::HISTOGRAM) { + Value.AsHistogram()->UnRef(); + } else if (ValueType == EMetricValueType::SUMMARY) { Value.AsSummaryDouble()->UnRef(); - } - ValueType = TValueType<T>::Type; - Value = TMetricValue(value); - if (ValueType == EMetricValueType::HISTOGRAM) { - Value.AsHistogram()->Ref(); - } else if (ValueType == EMetricValueType::SUMMARY) { + } + ValueType = TValueType<T>::Type; + Value = TMetricValue(value); + if (ValueType == EMetricValueType::HISTOGRAM) { + Value.AsHistogram()->Ref(); + } else if (ValueType == EMetricValueType::SUMMARY) { Value.AsSummaryDouble()->Ref(); - } - } - }; - - /////////////////////////////////////////////////////////////////////// - // TPrometheusEncoder - /////////////////////////////////////////////////////////////////////// - class TPrometheusEncoder final: public IMetricEncoder { - public: + } + } + }; + + /////////////////////////////////////////////////////////////////////// + // TPrometheusEncoder + /////////////////////////////////////////////////////////////////////// + class TPrometheusEncoder final: public IMetricEncoder { + public: explicit TPrometheusEncoder(IOutputStream* out, TStringBuf metricNameLabel) - : Writer_(out) + : Writer_(out) , MetricNameLabel_(metricNameLabel) - { - } - - private: - void OnStreamBegin() override { + { + } + + private: + void OnStreamBegin() override { State_.Expect(TEncoderState::EState::ROOT); - } - - void OnStreamEnd() override { + } + + void OnStreamEnd() override { State_.Expect(TEncoderState::EState::ROOT); - Writer_.WriteLn(); - } - - void OnCommonTime(TInstant time) override { + Writer_.WriteLn(); + } + + void OnCommonTime(TInstant time) override { State_.Expect(TEncoderState::EState::ROOT); - CommonTime_ = time; - } - - void OnMetricBegin(EMetricType type) override { - State_.Switch(TEncoderState::EState::ROOT, TEncoderState::EState::METRIC); - MetricState_.Clear(); - MetricState_.Type = type; - } - - void OnMetricEnd() override { - State_.Switch(TEncoderState::EState::METRIC, TEncoderState::EState::ROOT); - WriteMetric(); - } - - void OnLabelsBegin() override { - if (State_ == TEncoderState::EState::METRIC) { - State_ = TEncoderState::EState::METRIC_LABELS; + CommonTime_ = time; + } + + void OnMetricBegin(EMetricType type) override { + State_.Switch(TEncoderState::EState::ROOT, TEncoderState::EState::METRIC); + MetricState_.Clear(); + MetricState_.Type = type; + } + + void OnMetricEnd() override { + State_.Switch(TEncoderState::EState::METRIC, TEncoderState::EState::ROOT); + WriteMetric(); + } + + void OnLabelsBegin() override { + if (State_ == TEncoderState::EState::METRIC) { + State_ = TEncoderState::EState::METRIC_LABELS; } else if (State_ == TEncoderState::EState::ROOT) { State_ = TEncoderState::EState::COMMON_LABELS; - } else { - State_.ThrowInvalid("expected METRIC or ROOT"); - } - } - - void OnLabelsEnd() override { - if (State_ == TEncoderState::EState::METRIC_LABELS) { - State_ = TEncoderState::EState::METRIC; + } else { + State_.ThrowInvalid("expected METRIC or ROOT"); + } + } + + void OnLabelsEnd() override { + if (State_ == TEncoderState::EState::METRIC_LABELS) { + State_ = TEncoderState::EState::METRIC; } else if (State_ == TEncoderState::EState::COMMON_LABELS) { State_ = TEncoderState::EState::ROOT; - } else { - State_.ThrowInvalid("expected LABELS or COMMON_LABELS"); - } - } - - void OnLabel(TStringBuf name, TStringBuf value) override { - if (State_ == TEncoderState::EState::METRIC_LABELS) { - MetricState_.Labels.Add(name, value); + } else { + State_.ThrowInvalid("expected LABELS or COMMON_LABELS"); + } + } + + void OnLabel(TStringBuf name, TStringBuf value) override { + if (State_ == TEncoderState::EState::METRIC_LABELS) { + MetricState_.Labels.Add(name, value); } else if (State_ == TEncoderState::EState::COMMON_LABELS) { - CommonLabels_.Add(name, value); - } else { - State_.ThrowInvalid("expected LABELS or COMMON_LABELS"); - } - } - - void OnDouble(TInstant time, double value) override { - State_.Expect(TEncoderState::EState::METRIC); - MetricState_.Time = time; - MetricState_.SetValue(value); - } - - void OnInt64(TInstant time, i64 value) override { - State_.Expect(TEncoderState::EState::METRIC); - MetricState_.Time = time; - MetricState_.SetValue(value); - } - - void OnUint64(TInstant time, ui64 value) override { - State_.Expect(TEncoderState::EState::METRIC); - MetricState_.Time = time; - MetricState_.SetValue(value); - } - - void OnHistogram(TInstant time, IHistogramSnapshotPtr snapshot) override { - State_.Expect(TEncoderState::EState::METRIC); - MetricState_.Time = time; - MetricState_.SetValue(snapshot.Get()); - } - + CommonLabels_.Add(name, value); + } else { + State_.ThrowInvalid("expected LABELS or COMMON_LABELS"); + } + } + + void OnDouble(TInstant time, double value) override { + State_.Expect(TEncoderState::EState::METRIC); + MetricState_.Time = time; + MetricState_.SetValue(value); + } + + void OnInt64(TInstant time, i64 value) override { + State_.Expect(TEncoderState::EState::METRIC); + MetricState_.Time = time; + MetricState_.SetValue(value); + } + + void OnUint64(TInstant time, ui64 value) override { + State_.Expect(TEncoderState::EState::METRIC); + MetricState_.Time = time; + MetricState_.SetValue(value); + } + + void OnHistogram(TInstant time, IHistogramSnapshotPtr snapshot) override { + State_.Expect(TEncoderState::EState::METRIC); + MetricState_.Time = time; + MetricState_.SetValue(snapshot.Get()); + } + void OnSummaryDouble(TInstant time, ISummaryDoubleSnapshotPtr snapshot) override { - State_.Expect(TEncoderState::EState::METRIC); - MetricState_.Time = time; - MetricState_.SetValue(snapshot.Get()); + State_.Expect(TEncoderState::EState::METRIC); + MetricState_.Time = time; + MetricState_.SetValue(snapshot.Get()); } void OnLogHistogram(TInstant, TLogHistogramSnapshotPtr) override { // TODO(@kbalakirev): implement this function } - void Close() override { - } - - void WriteMetric() { - if (MetricState_.ValueType == EMetricValueType::UNKNOWN) { - return; - } - - // XXX: poor performace + void Close() override { + } + + void WriteMetric() { + if (MetricState_.ValueType == EMetricValueType::UNKNOWN) { + return; + } + + // XXX: poor performace for (auto&& l: CommonLabels_) { - MetricState_.Labels.Add(l.Name(), l.Value()); - } - + MetricState_.Labels.Add(l.Name(), l.Value()); + } + TMaybe<TLabel> nameLabel = MetricState_.Labels.Extract(MetricNameLabel_); - Y_ENSURE(nameLabel, - "labels " << MetricState_.Labels << + Y_ENSURE(nameLabel, + "labels " << MetricState_.Labels << " does not contain label '" << MetricNameLabel_ << '\''); - + const TString& metricName = ToString(nameLabel->Value()); - if (MetricState_.Type != EMetricType::DSUMMARY) { - Writer_.WriteType(MetricState_.Type, metricName); + if (MetricState_.Type != EMetricType::DSUMMARY) { + Writer_.WriteType(MetricState_.Type, metricName); } - - if (MetricState_.Time == TInstant::Zero()) { - MetricState_.Time = CommonTime_; - } - - EMetricType type = MetricState_.Type; - if (type == EMetricType::HIST || type == EMetricType::HIST_RATE) { - Y_ENSURE(MetricState_.ValueType == EMetricValueType::HISTOGRAM, - "invalid value type for histogram: " << int(MetricState_.ValueType)); // TODO: to string conversion - Writer_.WriteHistogram( - metricName, - MetricState_.Labels, - MetricState_.Time, - MetricState_.Value.AsHistogram()); - } else if (type == EMetricType::DSUMMARY) { + + if (MetricState_.Time == TInstant::Zero()) { + MetricState_.Time = CommonTime_; + } + + EMetricType type = MetricState_.Type; + if (type == EMetricType::HIST || type == EMetricType::HIST_RATE) { + Y_ENSURE(MetricState_.ValueType == EMetricValueType::HISTOGRAM, + "invalid value type for histogram: " << int(MetricState_.ValueType)); // TODO: to string conversion + Writer_.WriteHistogram( + metricName, + MetricState_.Labels, + MetricState_.Time, + MetricState_.Value.AsHistogram()); + } else if (type == EMetricType::DSUMMARY) { Writer_.WriteSummaryDouble( metricName, - MetricState_.Labels, - MetricState_.Time, - MetricState_.Value.AsSummaryDouble()); - } else { - Writer_.WriteDouble( - metricName, - MetricState_.Labels, - MetricState_.Time, - MetricState_.Value.AsDouble(MetricState_.ValueType)); - } - } - - private: - TEncoderState State_; - TPrometheusWriter Writer_; + MetricState_.Labels, + MetricState_.Time, + MetricState_.Value.AsSummaryDouble()); + } else { + Writer_.WriteDouble( + metricName, + MetricState_.Labels, + MetricState_.Time, + MetricState_.Value.AsDouble(MetricState_.ValueType)); + } + } + + private: + TEncoderState State_; + TPrometheusWriter Writer_; TString MetricNameLabel_; - TInstant CommonTime_ = TInstant::Zero(); - TLabels CommonLabels_; - TMetricState MetricState_; - }; - } - + TInstant CommonTime_ = TInstant::Zero(); + TLabels CommonLabels_; + TMetricState MetricState_; + }; + } + IMetricEncoderPtr EncoderPrometheus(IOutputStream* out, TStringBuf metricNameLabel) { return MakeHolder<TPrometheusEncoder>(out, metricNameLabel); - } - -} // namespace NMonitoring + } + +} // namespace NMonitoring diff --git a/library/cpp/monlib/encode/prometheus/prometheus_encoder_ut.cpp b/library/cpp/monlib/encode/prometheus/prometheus_encoder_ut.cpp index 6820aa9ba6..fd9debb060 100644 --- a/library/cpp/monlib/encode/prometheus/prometheus_encoder_ut.cpp +++ b/library/cpp/monlib/encode/prometheus/prometheus_encoder_ut.cpp @@ -1,119 +1,119 @@ -#include "prometheus.h" - -#include <library/cpp/monlib/encode/protobuf/protobuf.h> -#include <library/cpp/monlib/metrics/metric_value.h> -#include <library/cpp/monlib/metrics/histogram_snapshot.h> - +#include "prometheus.h" + +#include <library/cpp/monlib/encode/protobuf/protobuf.h> +#include <library/cpp/monlib/metrics/metric_value.h> +#include <library/cpp/monlib/metrics/histogram_snapshot.h> + #include <library/cpp/testing/unittest/registar.h> - -#include <util/stream/str.h> - -using namespace NMonitoring; - -Y_UNIT_TEST_SUITE(TPrometheusEncoderTest) { - - template <typename TFunc> - TString EncodeToString(TFunc fn) { - TStringStream ss; - IMetricEncoderPtr encoder = EncoderPrometheus(&ss); - fn(encoder.Get()); - return ss.Str(); - } - + +#include <util/stream/str.h> + +using namespace NMonitoring; + +Y_UNIT_TEST_SUITE(TPrometheusEncoderTest) { + + template <typename TFunc> + TString EncodeToString(TFunc fn) { + TStringStream ss; + IMetricEncoderPtr encoder = EncoderPrometheus(&ss); + fn(encoder.Get()); + return ss.Str(); + } + ISummaryDoubleSnapshotPtr TestSummaryDouble() { return MakeIntrusive<TSummaryDoubleSnapshot>(10.1, -0.45, 0.478, 0.3, 30u); } - Y_UNIT_TEST(Empty) { - auto result = EncodeToString([](IMetricEncoder* e) { - e->OnStreamBegin(); - e->OnStreamEnd(); - }); - UNIT_ASSERT_STRINGS_EQUAL(result, "\n"); - } - - Y_UNIT_TEST(DoubleGauge) { - auto result = EncodeToString([](IMetricEncoder* e) { - e->OnStreamBegin(); - { // no values - e->OnMetricBegin(EMetricType::GAUGE); - { - e->OnLabelsBegin(); - e->OnLabel("sensor", "cpuUsage"); - e->OnLabelsEnd(); - } - e->OnMetricEnd(); - } - { // one value no ts - e->OnMetricBegin(EMetricType::GAUGE); - { - e->OnLabelsBegin(); - e->OnLabel("sensor", "diskUsage"); - e->OnLabel("disk", "sda1"); - e->OnLabelsEnd(); - } - e->OnDouble(TInstant::Zero(), 1000); - e->OnMetricEnd(); - } - { // one value with ts - e->OnMetricBegin(EMetricType::GAUGE); - { - e->OnLabelsBegin(); - e->OnLabel("sensor", "memoryUsage"); - e->OnLabel("host", "solomon-man-00"); - e->OnLabel("dc", "man"); - e->OnLabelsEnd(); - } - e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 1000); - e->OnMetricEnd(); - } - { // many values - e->OnMetricBegin(EMetricType::GAUGE); - { - e->OnLabelsBegin(); - e->OnLabel("sensor", "bytesRx"); - e->OnLabel("host", "solomon-sas-01"); - e->OnLabel("dc", "sas"); - e->OnLabelsEnd(); - } - e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 2); - e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:05Z"), 4); - e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:10Z"), 8); - e->OnMetricEnd(); - } - { // already seen metric name - e->OnMetricBegin(EMetricType::GAUGE); - { - e->OnLabelsBegin(); - e->OnLabel("sensor", "diskUsage"); - e->OnLabel("disk", "sdb1"); - e->OnLabelsEnd(); - } - e->OnDouble(TInstant::Zero(), 1001); - e->OnMetricEnd(); - } - { // NaN - e->OnMetricBegin(EMetricType::GAUGE); - { - e->OnLabelsBegin(); - e->OnLabel("sensor", "nanValue"); - e->OnLabelsEnd(); - } - e->OnDouble(TInstant::Zero(), NAN); - e->OnMetricEnd(); - } - { // Inf - e->OnMetricBegin(EMetricType::GAUGE); - { - e->OnLabelsBegin(); - e->OnLabel("sensor", "infValue"); - e->OnLabelsEnd(); - } - e->OnDouble(TInstant::Zero(), INFINITY); - e->OnMetricEnd(); - } + Y_UNIT_TEST(Empty) { + auto result = EncodeToString([](IMetricEncoder* e) { + e->OnStreamBegin(); + e->OnStreamEnd(); + }); + UNIT_ASSERT_STRINGS_EQUAL(result, "\n"); + } + + Y_UNIT_TEST(DoubleGauge) { + auto result = EncodeToString([](IMetricEncoder* e) { + e->OnStreamBegin(); + { // no values + e->OnMetricBegin(EMetricType::GAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "cpuUsage"); + e->OnLabelsEnd(); + } + e->OnMetricEnd(); + } + { // one value no ts + e->OnMetricBegin(EMetricType::GAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "diskUsage"); + e->OnLabel("disk", "sda1"); + e->OnLabelsEnd(); + } + e->OnDouble(TInstant::Zero(), 1000); + e->OnMetricEnd(); + } + { // one value with ts + e->OnMetricBegin(EMetricType::GAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "memoryUsage"); + e->OnLabel("host", "solomon-man-00"); + e->OnLabel("dc", "man"); + e->OnLabelsEnd(); + } + e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 1000); + e->OnMetricEnd(); + } + { // many values + e->OnMetricBegin(EMetricType::GAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "bytesRx"); + e->OnLabel("host", "solomon-sas-01"); + e->OnLabel("dc", "sas"); + e->OnLabelsEnd(); + } + e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 2); + e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:05Z"), 4); + e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:10Z"), 8); + e->OnMetricEnd(); + } + { // already seen metric name + e->OnMetricBegin(EMetricType::GAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "diskUsage"); + e->OnLabel("disk", "sdb1"); + e->OnLabelsEnd(); + } + e->OnDouble(TInstant::Zero(), 1001); + e->OnMetricEnd(); + } + { // NaN + e->OnMetricBegin(EMetricType::GAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "nanValue"); + e->OnLabelsEnd(); + } + e->OnDouble(TInstant::Zero(), NAN); + e->OnMetricEnd(); + } + { // Inf + e->OnMetricBegin(EMetricType::GAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "infValue"); + e->OnLabelsEnd(); + } + e->OnDouble(TInstant::Zero(), INFINITY); + e->OnMetricEnd(); + } { - e->OnMetricBegin(EMetricType::DSUMMARY); + e->OnMetricBegin(EMetricType::DSUMMARY); { e->OnLabelsBegin(); e->OnLabel("sensor", "seconds"); @@ -121,252 +121,252 @@ Y_UNIT_TEST_SUITE(TPrometheusEncoderTest) { e->OnLabelsEnd(); } e->OnSummaryDouble(TInstant::Zero(), TestSummaryDouble()); - e->OnMetricEnd(); + e->OnMetricEnd(); } - e->OnStreamEnd(); - }); - - UNIT_ASSERT_STRINGS_EQUAL(result, - "# TYPE diskUsage gauge\n" - "diskUsage{disk=\"sda1\", } 1000\n" - "# TYPE memoryUsage gauge\n" - "memoryUsage{host=\"solomon-man-00\", dc=\"man\", } 1000 1512216000000\n" - "# TYPE bytesRx gauge\n" - "bytesRx{host=\"solomon-sas-01\", dc=\"sas\", } 8 1512216010000\n" - "diskUsage{disk=\"sdb1\", } 1001\n" - "# TYPE nanValue gauge\n" - "nanValue nan\n" - "# TYPE infValue gauge\n" - "infValue inf\n" + e->OnStreamEnd(); + }); + + UNIT_ASSERT_STRINGS_EQUAL(result, + "# TYPE diskUsage gauge\n" + "diskUsage{disk=\"sda1\", } 1000\n" + "# TYPE memoryUsage gauge\n" + "memoryUsage{host=\"solomon-man-00\", dc=\"man\", } 1000 1512216000000\n" + "# TYPE bytesRx gauge\n" + "bytesRx{host=\"solomon-sas-01\", dc=\"sas\", } 8 1512216010000\n" + "diskUsage{disk=\"sdb1\", } 1001\n" + "# TYPE nanValue gauge\n" + "nanValue nan\n" + "# TYPE infValue gauge\n" + "infValue inf\n" "seconds_sum{disk=\"sdb1\", } 10.1\n" "seconds_min{disk=\"sdb1\", } -0.45\n" "seconds_max{disk=\"sdb1\", } 0.478\n" "seconds_last{disk=\"sdb1\", } 0.3\n" "seconds_count{disk=\"sdb1\", } 30\n" - "\n"); - } - - Y_UNIT_TEST(IntGauges) { - auto result = EncodeToString([](IMetricEncoder* e) { - e->OnStreamBegin(); - { // no values - e->OnMetricBegin(EMetricType::IGAUGE); - { - e->OnLabelsBegin(); - e->OnLabel("sensor", "cpuUsage"); - e->OnLabelsEnd(); - } - e->OnMetricEnd(); - } - { // one value no ts - e->OnMetricBegin(EMetricType::IGAUGE); - { - e->OnLabelsBegin(); - e->OnLabel("sensor", "diskUsage"); - e->OnLabel("disk", "sda1"); - e->OnLabelsEnd(); - } - e->OnInt64(TInstant::Zero(), 1000); - e->OnMetricEnd(); - } - { // one value with ts - e->OnMetricBegin(EMetricType::IGAUGE); - { - e->OnLabelsBegin(); - e->OnLabel("sensor", "memoryUsage"); - e->OnLabel("dc", "man"); - e->OnLabel("host", "solomon-man-00"); - e->OnLabelsEnd(); - } - e->OnInt64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 1000); - e->OnMetricEnd(); - } - { // many values - e->OnMetricBegin(EMetricType::IGAUGE); - { - e->OnLabelsBegin(); - e->OnLabel("sensor", "bytesRx"); - e->OnLabel("dc", "sas"); - e->OnLabel("host", "solomon-sas-01"); - e->OnLabelsEnd(); - } - e->OnInt64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 2); - e->OnInt64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:05Z"), 4); - e->OnInt64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:10Z"), 8); - e->OnMetricEnd(); - } - e->OnStreamEnd(); - }); - - UNIT_ASSERT_STRINGS_EQUAL(result, - "# TYPE diskUsage gauge\n" - "diskUsage{disk=\"sda1\", } 1000\n" - "# TYPE memoryUsage gauge\n" - "memoryUsage{dc=\"man\", host=\"solomon-man-00\", } 1000 1512216000000\n" - "# TYPE bytesRx gauge\n" - "bytesRx{dc=\"sas\", host=\"solomon-sas-01\", } 8 1512216010000\n" - "\n"); - } - - Y_UNIT_TEST(Counters) { - auto result = EncodeToString([](IMetricEncoder* e) { - e->OnStreamBegin(); - { // no values - e->OnMetricBegin(EMetricType::COUNTER); - { - e->OnLabelsBegin(); - e->OnLabel("sensor", "cpuUsage"); - e->OnLabelsEnd(); - } - e->OnMetricEnd(); - } - { // one value no ts - e->OnMetricBegin(EMetricType::COUNTER); - { - e->OnLabelsBegin(); - e->OnLabel("sensor", "diskUsage"); - e->OnLabel("disk", "sda1"); - e->OnLabelsEnd(); - } - e->OnInt64(TInstant::Zero(), 1000); - e->OnMetricEnd(); - } - { // one value with ts - e->OnMetricBegin(EMetricType::COUNTER); - { - e->OnLabelsBegin(); - e->OnLabel("sensor", "memoryUsage"); - e->OnLabel("host", "solomon-man-00"); - e->OnLabel("dc", "man"); - e->OnLabelsEnd(); - } - e->OnInt64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 1000); - e->OnMetricEnd(); - } - { // many values - e->OnMetricBegin(EMetricType::COUNTER); - { - e->OnLabelsBegin(); - e->OnLabel("sensor", "bytesRx"); - e->OnLabel("host", "solomon-sas-01"); - e->OnLabel("dc", "sas"); - e->OnLabelsEnd(); - } - e->OnInt64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 2); - e->OnInt64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:05Z"), 4); - e->OnInt64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:10Z"), 8); - e->OnMetricEnd(); - } - e->OnStreamEnd(); - }); - - UNIT_ASSERT_STRINGS_EQUAL(result, - "# TYPE diskUsage counter\n" - "diskUsage{disk=\"sda1\", } 1000\n" - "# TYPE memoryUsage counter\n" - "memoryUsage{host=\"solomon-man-00\", dc=\"man\", } 1000 1512216000000\n" - "# TYPE bytesRx counter\n" - "bytesRx{host=\"solomon-sas-01\", dc=\"sas\", } 8 1512216010000\n" - "\n"); - } - - Y_UNIT_TEST(Histograms) { - auto result = EncodeToString([](IMetricEncoder* e) { - e->OnStreamBegin(); - { // no values histogram - e->OnMetricBegin(EMetricType::HIST); - { - e->OnLabelsBegin(); - e->OnLabel("sensor", "cpuUsage"); - e->OnLabelsEnd(); - } - e->OnMetricEnd(); - } - { // one value no ts - e->OnMetricBegin(EMetricType::HIST); - { - e->OnLabelsBegin(); - e->OnLabel("sensor", "inboundBytesPerSec"); - e->OnLabel("client", "mbus"); - e->OnLabelsEnd(); - } - e->OnHistogram( - TInstant::Zero(), - ExplicitHistogramSnapshot({10, 20, HISTOGRAM_INF_BOUND}, {1, 4, 0})); - e->OnMetricEnd(); - } - { // one value no ts no +inf bucket - e->OnMetricBegin(EMetricType::HIST); - { - e->OnLabelsBegin(); - e->OnLabel("sensor", "inboundBytesPerSec"); - e->OnLabel("client", "grpc"); - e->OnLabelsEnd(); - } - e->OnHistogram( - TInstant::Zero(), - ExplicitHistogramSnapshot({10, 20, 30}, {1, 4, 0})); - e->OnMetricEnd(); - } - { // one value with ts - e->OnMetricBegin(EMetricType::HIST_RATE); - { - e->OnLabelsBegin(); - e->OnLabel("sensor", "outboundBytesPerSec"); - e->OnLabel("client", "grps"); - e->OnLabelsEnd(); - } - e->OnHistogram( - TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), - ExplicitHistogramSnapshot({100, 200, HISTOGRAM_INF_BOUND}, {1, 0, 0})); - e->OnMetricEnd(); - } - { // many values - e->OnMetricBegin(EMetricType::HIST); - { - e->OnLabelsBegin(); - e->OnLabel("sensor", "bytesRx"); - e->OnLabel("host", "solomon-sas-01"); - e->OnLabel("dc", "sas"); - e->OnLabelsEnd(); - } - TBucketBounds bounds = {100, 200, HISTOGRAM_INF_BOUND}; - e->OnHistogram( - TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), - ExplicitHistogramSnapshot(bounds, {10, 0, 0})); - e->OnHistogram( - TInstant::ParseIso8601Deprecated("2017-12-02T12:00:05Z"), - ExplicitHistogramSnapshot(bounds, {10, 2, 0})); - e->OnHistogram( - TInstant::ParseIso8601Deprecated("2017-12-02T12:00:10Z"), - ExplicitHistogramSnapshot(bounds, {10, 2, 5})); - e->OnMetricEnd(); - } - e->OnStreamEnd(); - }); - - UNIT_ASSERT_STRINGS_EQUAL(result, - "# TYPE inboundBytesPerSec histogram\n" - "inboundBytesPerSec_bucket{client=\"mbus\", le=\"10\"} 1\n" - "inboundBytesPerSec_bucket{client=\"mbus\", le=\"20\"} 5\n" - "inboundBytesPerSec_bucket{client=\"mbus\", le=\"+Inf\"} 5\n" - "inboundBytesPerSec_count{client=\"mbus\", } 5\n" - "inboundBytesPerSec_bucket{client=\"grpc\", le=\"10\"} 1\n" - "inboundBytesPerSec_bucket{client=\"grpc\", le=\"20\"} 5\n" - "inboundBytesPerSec_bucket{client=\"grpc\", le=\"30\"} 5\n" - "inboundBytesPerSec_count{client=\"grpc\", } 5\n" - "# TYPE outboundBytesPerSec histogram\n" - "outboundBytesPerSec_bucket{client=\"grps\", le=\"100\"} 1 1512216000000\n" - "outboundBytesPerSec_bucket{client=\"grps\", le=\"200\"} 1 1512216000000\n" - "outboundBytesPerSec_bucket{client=\"grps\", le=\"+Inf\"} 1 1512216000000\n" - "outboundBytesPerSec_count{client=\"grps\", } 1 1512216000000\n" - "# TYPE bytesRx histogram\n" - "bytesRx_bucket{host=\"solomon-sas-01\", dc=\"sas\", le=\"100\"} 10 1512216010000\n" - "bytesRx_bucket{host=\"solomon-sas-01\", dc=\"sas\", le=\"200\"} 12 1512216010000\n" - "bytesRx_bucket{host=\"solomon-sas-01\", dc=\"sas\", le=\"+Inf\"} 17 1512216010000\n" - "bytesRx_count{host=\"solomon-sas-01\", dc=\"sas\", } 17 1512216010000\n" - "\n"); - } + "\n"); + } + + Y_UNIT_TEST(IntGauges) { + auto result = EncodeToString([](IMetricEncoder* e) { + e->OnStreamBegin(); + { // no values + e->OnMetricBegin(EMetricType::IGAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "cpuUsage"); + e->OnLabelsEnd(); + } + e->OnMetricEnd(); + } + { // one value no ts + e->OnMetricBegin(EMetricType::IGAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "diskUsage"); + e->OnLabel("disk", "sda1"); + e->OnLabelsEnd(); + } + e->OnInt64(TInstant::Zero(), 1000); + e->OnMetricEnd(); + } + { // one value with ts + e->OnMetricBegin(EMetricType::IGAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "memoryUsage"); + e->OnLabel("dc", "man"); + e->OnLabel("host", "solomon-man-00"); + e->OnLabelsEnd(); + } + e->OnInt64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 1000); + e->OnMetricEnd(); + } + { // many values + e->OnMetricBegin(EMetricType::IGAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "bytesRx"); + e->OnLabel("dc", "sas"); + e->OnLabel("host", "solomon-sas-01"); + e->OnLabelsEnd(); + } + e->OnInt64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 2); + e->OnInt64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:05Z"), 4); + e->OnInt64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:10Z"), 8); + e->OnMetricEnd(); + } + e->OnStreamEnd(); + }); + + UNIT_ASSERT_STRINGS_EQUAL(result, + "# TYPE diskUsage gauge\n" + "diskUsage{disk=\"sda1\", } 1000\n" + "# TYPE memoryUsage gauge\n" + "memoryUsage{dc=\"man\", host=\"solomon-man-00\", } 1000 1512216000000\n" + "# TYPE bytesRx gauge\n" + "bytesRx{dc=\"sas\", host=\"solomon-sas-01\", } 8 1512216010000\n" + "\n"); + } + + Y_UNIT_TEST(Counters) { + auto result = EncodeToString([](IMetricEncoder* e) { + e->OnStreamBegin(); + { // no values + e->OnMetricBegin(EMetricType::COUNTER); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "cpuUsage"); + e->OnLabelsEnd(); + } + e->OnMetricEnd(); + } + { // one value no ts + e->OnMetricBegin(EMetricType::COUNTER); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "diskUsage"); + e->OnLabel("disk", "sda1"); + e->OnLabelsEnd(); + } + e->OnInt64(TInstant::Zero(), 1000); + e->OnMetricEnd(); + } + { // one value with ts + e->OnMetricBegin(EMetricType::COUNTER); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "memoryUsage"); + e->OnLabel("host", "solomon-man-00"); + e->OnLabel("dc", "man"); + e->OnLabelsEnd(); + } + e->OnInt64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 1000); + e->OnMetricEnd(); + } + { // many values + e->OnMetricBegin(EMetricType::COUNTER); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "bytesRx"); + e->OnLabel("host", "solomon-sas-01"); + e->OnLabel("dc", "sas"); + e->OnLabelsEnd(); + } + e->OnInt64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 2); + e->OnInt64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:05Z"), 4); + e->OnInt64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:10Z"), 8); + e->OnMetricEnd(); + } + e->OnStreamEnd(); + }); + + UNIT_ASSERT_STRINGS_EQUAL(result, + "# TYPE diskUsage counter\n" + "diskUsage{disk=\"sda1\", } 1000\n" + "# TYPE memoryUsage counter\n" + "memoryUsage{host=\"solomon-man-00\", dc=\"man\", } 1000 1512216000000\n" + "# TYPE bytesRx counter\n" + "bytesRx{host=\"solomon-sas-01\", dc=\"sas\", } 8 1512216010000\n" + "\n"); + } + + Y_UNIT_TEST(Histograms) { + auto result = EncodeToString([](IMetricEncoder* e) { + e->OnStreamBegin(); + { // no values histogram + e->OnMetricBegin(EMetricType::HIST); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "cpuUsage"); + e->OnLabelsEnd(); + } + e->OnMetricEnd(); + } + { // one value no ts + e->OnMetricBegin(EMetricType::HIST); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "inboundBytesPerSec"); + e->OnLabel("client", "mbus"); + e->OnLabelsEnd(); + } + e->OnHistogram( + TInstant::Zero(), + ExplicitHistogramSnapshot({10, 20, HISTOGRAM_INF_BOUND}, {1, 4, 0})); + e->OnMetricEnd(); + } + { // one value no ts no +inf bucket + e->OnMetricBegin(EMetricType::HIST); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "inboundBytesPerSec"); + e->OnLabel("client", "grpc"); + e->OnLabelsEnd(); + } + e->OnHistogram( + TInstant::Zero(), + ExplicitHistogramSnapshot({10, 20, 30}, {1, 4, 0})); + e->OnMetricEnd(); + } + { // one value with ts + e->OnMetricBegin(EMetricType::HIST_RATE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "outboundBytesPerSec"); + e->OnLabel("client", "grps"); + e->OnLabelsEnd(); + } + e->OnHistogram( + TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), + ExplicitHistogramSnapshot({100, 200, HISTOGRAM_INF_BOUND}, {1, 0, 0})); + e->OnMetricEnd(); + } + { // many values + e->OnMetricBegin(EMetricType::HIST); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "bytesRx"); + e->OnLabel("host", "solomon-sas-01"); + e->OnLabel("dc", "sas"); + e->OnLabelsEnd(); + } + TBucketBounds bounds = {100, 200, HISTOGRAM_INF_BOUND}; + e->OnHistogram( + TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), + ExplicitHistogramSnapshot(bounds, {10, 0, 0})); + e->OnHistogram( + TInstant::ParseIso8601Deprecated("2017-12-02T12:00:05Z"), + ExplicitHistogramSnapshot(bounds, {10, 2, 0})); + e->OnHistogram( + TInstant::ParseIso8601Deprecated("2017-12-02T12:00:10Z"), + ExplicitHistogramSnapshot(bounds, {10, 2, 5})); + e->OnMetricEnd(); + } + e->OnStreamEnd(); + }); + + UNIT_ASSERT_STRINGS_EQUAL(result, + "# TYPE inboundBytesPerSec histogram\n" + "inboundBytesPerSec_bucket{client=\"mbus\", le=\"10\"} 1\n" + "inboundBytesPerSec_bucket{client=\"mbus\", le=\"20\"} 5\n" + "inboundBytesPerSec_bucket{client=\"mbus\", le=\"+Inf\"} 5\n" + "inboundBytesPerSec_count{client=\"mbus\", } 5\n" + "inboundBytesPerSec_bucket{client=\"grpc\", le=\"10\"} 1\n" + "inboundBytesPerSec_bucket{client=\"grpc\", le=\"20\"} 5\n" + "inboundBytesPerSec_bucket{client=\"grpc\", le=\"30\"} 5\n" + "inboundBytesPerSec_count{client=\"grpc\", } 5\n" + "# TYPE outboundBytesPerSec histogram\n" + "outboundBytesPerSec_bucket{client=\"grps\", le=\"100\"} 1 1512216000000\n" + "outboundBytesPerSec_bucket{client=\"grps\", le=\"200\"} 1 1512216000000\n" + "outboundBytesPerSec_bucket{client=\"grps\", le=\"+Inf\"} 1 1512216000000\n" + "outboundBytesPerSec_count{client=\"grps\", } 1 1512216000000\n" + "# TYPE bytesRx histogram\n" + "bytesRx_bucket{host=\"solomon-sas-01\", dc=\"sas\", le=\"100\"} 10 1512216010000\n" + "bytesRx_bucket{host=\"solomon-sas-01\", dc=\"sas\", le=\"200\"} 12 1512216010000\n" + "bytesRx_bucket{host=\"solomon-sas-01\", dc=\"sas\", le=\"+Inf\"} 17 1512216010000\n" + "bytesRx_count{host=\"solomon-sas-01\", dc=\"sas\", } 17 1512216010000\n" + "\n"); + } Y_UNIT_TEST(CommonLables) { auto result = EncodeToString([](IMetricEncoder* e) { @@ -411,4 +411,4 @@ two{labels="l2", project="solomon", } 42 1500000000000 )"); } -} +} diff --git a/library/cpp/monlib/encode/prometheus/prometheus_model.h b/library/cpp/monlib/encode/prometheus/prometheus_model.h index f8e55868e7..cb7f2cb15b 100644 --- a/library/cpp/monlib/encode/prometheus/prometheus_model.h +++ b/library/cpp/monlib/encode/prometheus/prometheus_model.h @@ -1,70 +1,70 @@ -#pragma once - -#include <util/generic/strbuf.h> - - -namespace NMonitoring { -namespace NPrometheus { - - // - // Prometheus specific names and validation rules. - // - // See https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md - // and https://github.com/prometheus/common/blob/master/expfmt/text_parse.go - // - +#pragma once + +#include <util/generic/strbuf.h> + + +namespace NMonitoring { +namespace NPrometheus { + + // + // Prometheus specific names and validation rules. + // + // See https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md + // and https://github.com/prometheus/common/blob/master/expfmt/text_parse.go + // + inline constexpr TStringBuf BUCKET_SUFFIX = "_bucket"; inline constexpr TStringBuf COUNT_SUFFIX = "_count"; inline constexpr TStringBuf SUM_SUFFIX = "_sum"; inline constexpr TStringBuf MIN_SUFFIX = "_min"; inline constexpr TStringBuf MAX_SUFFIX = "_max"; inline constexpr TStringBuf LAST_SUFFIX = "_last"; - - // Used for the label that defines the upper bound of a bucket of a - // histogram ("le" -> "less or equal"). + + // Used for the label that defines the upper bound of a bucket of a + // histogram ("le" -> "less or equal"). inline constexpr TStringBuf BUCKET_LABEL = "le"; - - - inline bool IsValidLabelNameStart(char ch) { - return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || ch == '_'; - } - - inline bool IsValidLabelNameContinuation(char ch) { - return IsValidLabelNameStart(ch) || (ch >= '0' && ch <= '9'); - } - - inline bool IsValidMetricNameStart(char ch) { - return IsValidLabelNameStart(ch) || ch == ':'; - } - - inline bool IsValidMetricNameContinuation(char ch) { - return IsValidLabelNameContinuation(ch) || ch == ':'; - } - - inline bool IsSum(TStringBuf name) { - return name.EndsWith(SUM_SUFFIX); - } - - inline bool IsCount(TStringBuf name) { - return name.EndsWith(COUNT_SUFFIX); - } - - inline bool IsBucket(TStringBuf name) { - return name.EndsWith(BUCKET_SUFFIX); - } - - inline TStringBuf ToBaseName(TStringBuf name) { - if (IsBucket(name)) { - return name.SubString(0, name.length() - BUCKET_SUFFIX.length()); - } - if (IsCount(name)) { - return name.SubString(0, name.length() - COUNT_SUFFIX.length()); - } - if (IsSum(name)) { - return name.SubString(0, name.length() - SUM_SUFFIX.length()); - } - return name; - } - -} // namespace NPrometheus -} // namespace NMonitoring + + + inline bool IsValidLabelNameStart(char ch) { + return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || ch == '_'; + } + + inline bool IsValidLabelNameContinuation(char ch) { + return IsValidLabelNameStart(ch) || (ch >= '0' && ch <= '9'); + } + + inline bool IsValidMetricNameStart(char ch) { + return IsValidLabelNameStart(ch) || ch == ':'; + } + + inline bool IsValidMetricNameContinuation(char ch) { + return IsValidLabelNameContinuation(ch) || ch == ':'; + } + + inline bool IsSum(TStringBuf name) { + return name.EndsWith(SUM_SUFFIX); + } + + inline bool IsCount(TStringBuf name) { + return name.EndsWith(COUNT_SUFFIX); + } + + inline bool IsBucket(TStringBuf name) { + return name.EndsWith(BUCKET_SUFFIX); + } + + inline TStringBuf ToBaseName(TStringBuf name) { + if (IsBucket(name)) { + return name.SubString(0, name.length() - BUCKET_SUFFIX.length()); + } + if (IsCount(name)) { + return name.SubString(0, name.length() - COUNT_SUFFIX.length()); + } + if (IsSum(name)) { + return name.SubString(0, name.length() - SUM_SUFFIX.length()); + } + return name; + } + +} // namespace NPrometheus +} // namespace NMonitoring diff --git a/library/cpp/monlib/encode/prometheus/ut/ya.make b/library/cpp/monlib/encode/prometheus/ut/ya.make index 4d4070d194..fc468ffb68 100644 --- a/library/cpp/monlib/encode/prometheus/ut/ya.make +++ b/library/cpp/monlib/encode/prometheus/ut/ya.make @@ -1,17 +1,17 @@ -UNITTEST_FOR(library/cpp/monlib/encode/prometheus) - -OWNER( - g:solomon - jamel -) - -SRCS( - prometheus_encoder_ut.cpp - prometheus_decoder_ut.cpp -) - -PEERDIR( - library/cpp/monlib/encode/protobuf -) - -END() +UNITTEST_FOR(library/cpp/monlib/encode/prometheus) + +OWNER( + g:solomon + jamel +) + +SRCS( + prometheus_encoder_ut.cpp + prometheus_decoder_ut.cpp +) + +PEERDIR( + library/cpp/monlib/encode/protobuf +) + +END() diff --git a/library/cpp/monlib/encode/prometheus/ya.make b/library/cpp/monlib/encode/prometheus/ya.make index 03f18f78a4..7f2483b166 100644 --- a/library/cpp/monlib/encode/prometheus/ya.make +++ b/library/cpp/monlib/encode/prometheus/ya.make @@ -1,17 +1,17 @@ -LIBRARY() - -OWNER( - jamel - g:solomon -) - -SRCS( - prometheus_decoder.cpp - prometheus_encoder.cpp -) - -PEERDIR( - library/cpp/monlib/encode -) - -END() +LIBRARY() + +OWNER( + jamel + g:solomon +) + +SRCS( + prometheus_decoder.cpp + prometheus_encoder.cpp +) + +PEERDIR( + library/cpp/monlib/encode +) + +END() diff --git a/library/cpp/monlib/encode/protobuf/protobuf.h b/library/cpp/monlib/encode/protobuf/protobuf.h index 219b94bee4..3f82cbdd84 100644 --- a/library/cpp/monlib/encode/protobuf/protobuf.h +++ b/library/cpp/monlib/encode/protobuf/protobuf.h @@ -1,16 +1,16 @@ -#pragma once - -#include <library/cpp/monlib/encode/encoder.h> - -#include <library/cpp/monlib/encode/protobuf/protos/samples.pb.h> - -namespace NMonitoring { - namespace NProto { - class TSingleSamplesList; - class TMultiSamplesList; - } - - IMetricEncoderPtr EncoderProtobuf(NProto::TSingleSamplesList* samples); - IMetricEncoderPtr EncoderProtobuf(NProto::TMultiSamplesList* samples); - -} +#pragma once + +#include <library/cpp/monlib/encode/encoder.h> + +#include <library/cpp/monlib/encode/protobuf/protos/samples.pb.h> + +namespace NMonitoring { + namespace NProto { + class TSingleSamplesList; + class TMultiSamplesList; + } + + IMetricEncoderPtr EncoderProtobuf(NProto::TSingleSamplesList* samples); + IMetricEncoderPtr EncoderProtobuf(NProto::TMultiSamplesList* samples); + +} diff --git a/library/cpp/monlib/encode/protobuf/protobuf_encoder.cpp b/library/cpp/monlib/encode/protobuf/protobuf_encoder.cpp index 57fbbba605..2d11b9d5ba 100644 --- a/library/cpp/monlib/encode/protobuf/protobuf_encoder.cpp +++ b/library/cpp/monlib/encode/protobuf/protobuf_encoder.cpp @@ -1,42 +1,42 @@ -#include "protobuf.h" - -#include <util/datetime/base.h> - -namespace NMonitoring { - namespace { - NProto::EMetricType ConvertMetricType(EMetricType type) { - switch (type) { - case EMetricType::GAUGE: - return NProto::GAUGE; - case EMetricType::COUNTER: - return NProto::COUNTER; - case EMetricType::RATE: - return NProto::RATE; - case EMetricType::IGAUGE: - return NProto::IGAUGE; - case EMetricType::HIST: +#include "protobuf.h" + +#include <util/datetime/base.h> + +namespace NMonitoring { + namespace { + NProto::EMetricType ConvertMetricType(EMetricType type) { + switch (type) { + case EMetricType::GAUGE: + return NProto::GAUGE; + case EMetricType::COUNTER: + return NProto::COUNTER; + case EMetricType::RATE: + return NProto::RATE; + case EMetricType::IGAUGE: + return NProto::IGAUGE; + case EMetricType::HIST: return NProto::HISTOGRAM; - case EMetricType::HIST_RATE: + case EMetricType::HIST_RATE: return NProto::HIST_RATE; - case EMetricType::DSUMMARY: + case EMetricType::DSUMMARY: return NProto::DSUMMARY; case EMetricType::LOGHIST: return NProto::LOGHISTOGRAM; - case EMetricType::UNKNOWN: - return NProto::UNKNOWN; - } - } - - void FillHistogram( - const IHistogramSnapshot& snapshot, - NProto::THistogram* histogram) - { - for (ui32 i = 0; i < snapshot.Count(); i++) { - histogram->AddBounds(snapshot.UpperBound(i)); - histogram->AddValues(snapshot.Value(i)); - } - } - + case EMetricType::UNKNOWN: + return NProto::UNKNOWN; + } + } + + void FillHistogram( + const IHistogramSnapshot& snapshot, + NProto::THistogram* histogram) + { + for (ui32 i = 0; i < snapshot.Count(); i++) { + histogram->AddBounds(snapshot.UpperBound(i)); + histogram->AddValues(snapshot.Value(i)); + } + } + void FillSummaryDouble(const ISummaryDoubleSnapshot& snapshot, NProto::TSummaryDouble* summary) { summary->SetSum(snapshot.GetSum()); summary->SetMin(snapshot.GetMin()); @@ -54,195 +54,195 @@ namespace NMonitoring { } } - /////////////////////////////////////////////////////////////////////////////// - // TSingleamplesEncoder - /////////////////////////////////////////////////////////////////////////////// - class TSingleSamplesEncoder final: public IMetricEncoder { - public: - TSingleSamplesEncoder(NProto::TSingleSamplesList* samples) - : Samples_(samples) - , Sample_(nullptr) - { - } - - private: - void OnStreamBegin() override { - } - void OnStreamEnd() override { - } - - void OnCommonTime(TInstant time) override { - Samples_->SetCommonTime(time.MilliSeconds()); - } - - void OnMetricBegin(EMetricType type) override { - Sample_ = Samples_->AddSamples(); - Sample_->SetMetricType(ConvertMetricType(type)); - } - - void OnMetricEnd() override { - Sample_ = nullptr; - } - - void OnLabelsBegin() override { - } - void OnLabelsEnd() override { - } - - void OnLabel(TStringBuf name, TStringBuf value) override { - NProto::TLabel* label = (Sample_ == nullptr) - ? Samples_->AddCommonLabels() - : Sample_->AddLabels(); + /////////////////////////////////////////////////////////////////////////////// + // TSingleamplesEncoder + /////////////////////////////////////////////////////////////////////////////// + class TSingleSamplesEncoder final: public IMetricEncoder { + public: + TSingleSamplesEncoder(NProto::TSingleSamplesList* samples) + : Samples_(samples) + , Sample_(nullptr) + { + } + + private: + void OnStreamBegin() override { + } + void OnStreamEnd() override { + } + + void OnCommonTime(TInstant time) override { + Samples_->SetCommonTime(time.MilliSeconds()); + } + + void OnMetricBegin(EMetricType type) override { + Sample_ = Samples_->AddSamples(); + Sample_->SetMetricType(ConvertMetricType(type)); + } + + void OnMetricEnd() override { + Sample_ = nullptr; + } + + void OnLabelsBegin() override { + } + void OnLabelsEnd() override { + } + + void OnLabel(TStringBuf name, TStringBuf value) override { + NProto::TLabel* label = (Sample_ == nullptr) + ? Samples_->AddCommonLabels() + : Sample_->AddLabels(); label->SetName(TString{name}); label->SetValue(TString{value}); - } - - void OnDouble(TInstant time, double value) override { - Y_ENSURE(Sample_, "metric not started"); - Sample_->SetTime(time.MilliSeconds()); - Sample_->SetFloat64(value); - } - - void OnInt64(TInstant time, i64 value) override { - Y_ENSURE(Sample_, "metric not started"); - Sample_->SetTime(time.MilliSeconds()); - Sample_->SetInt64(value); - } - - void OnUint64(TInstant time, ui64 value) override { - Y_ENSURE(Sample_, "metric not started"); - Sample_->SetTime(time.MilliSeconds()); - Sample_->SetUint64(value); - } - - void OnHistogram(TInstant time, IHistogramSnapshotPtr snapshot) override { - Y_ENSURE(Sample_, "metric not started"); - Sample_->SetTime(time.MilliSeconds()); - FillHistogram(*snapshot, Sample_->MutableHistogram()); - } - + } + + void OnDouble(TInstant time, double value) override { + Y_ENSURE(Sample_, "metric not started"); + Sample_->SetTime(time.MilliSeconds()); + Sample_->SetFloat64(value); + } + + void OnInt64(TInstant time, i64 value) override { + Y_ENSURE(Sample_, "metric not started"); + Sample_->SetTime(time.MilliSeconds()); + Sample_->SetInt64(value); + } + + void OnUint64(TInstant time, ui64 value) override { + Y_ENSURE(Sample_, "metric not started"); + Sample_->SetTime(time.MilliSeconds()); + Sample_->SetUint64(value); + } + + void OnHistogram(TInstant time, IHistogramSnapshotPtr snapshot) override { + Y_ENSURE(Sample_, "metric not started"); + Sample_->SetTime(time.MilliSeconds()); + FillHistogram(*snapshot, Sample_->MutableHistogram()); + } + void OnSummaryDouble(TInstant time, ISummaryDoubleSnapshotPtr snapshot) override { - Y_ENSURE(Sample_, "metric not started"); + Y_ENSURE(Sample_, "metric not started"); Sample_->SetTime(time.MilliSeconds()); FillSummaryDouble(*snapshot, Sample_->MutableSummaryDouble()); } void OnLogHistogram(TInstant time, TLogHistogramSnapshotPtr snapshot) override { - Y_ENSURE(Sample_, "metric not started"); + Y_ENSURE(Sample_, "metric not started"); Sample_->SetTime(time.MilliSeconds()); FillLogHistogram(*snapshot, Sample_->MutableLogHistogram()); } - void Close() override { - } - - private: - NProto::TSingleSamplesList* Samples_; - NProto::TSingleSample* Sample_; - }; - - /////////////////////////////////////////////////////////////////////////////// - // TMultiSamplesEncoder - /////////////////////////////////////////////////////////////////////////////// - class TMultiSamplesEncoder final: public IMetricEncoder { - public: - TMultiSamplesEncoder(NProto::TMultiSamplesList* samples) - : Samples_(samples) - , Sample_(nullptr) - { - } - - private: - void OnStreamBegin() override { - } - void OnStreamEnd() override { - } - - void OnCommonTime(TInstant time) override { - Samples_->SetCommonTime(time.MilliSeconds()); - } - - void OnMetricBegin(EMetricType type) override { - Sample_ = Samples_->AddSamples(); - Sample_->SetMetricType(ConvertMetricType(type)); - } - - void OnMetricEnd() override { - Sample_ = nullptr; - } - - void OnLabelsBegin() override { - } - void OnLabelsEnd() override { - } - - void OnLabel(TStringBuf name, TStringBuf value) override { - NProto::TLabel* label = (Sample_ == nullptr) - ? Samples_->AddCommonLabels() - : Sample_->AddLabels(); - + void Close() override { + } + + private: + NProto::TSingleSamplesList* Samples_; + NProto::TSingleSample* Sample_; + }; + + /////////////////////////////////////////////////////////////////////////////// + // TMultiSamplesEncoder + /////////////////////////////////////////////////////////////////////////////// + class TMultiSamplesEncoder final: public IMetricEncoder { + public: + TMultiSamplesEncoder(NProto::TMultiSamplesList* samples) + : Samples_(samples) + , Sample_(nullptr) + { + } + + private: + void OnStreamBegin() override { + } + void OnStreamEnd() override { + } + + void OnCommonTime(TInstant time) override { + Samples_->SetCommonTime(time.MilliSeconds()); + } + + void OnMetricBegin(EMetricType type) override { + Sample_ = Samples_->AddSamples(); + Sample_->SetMetricType(ConvertMetricType(type)); + } + + void OnMetricEnd() override { + Sample_ = nullptr; + } + + void OnLabelsBegin() override { + } + void OnLabelsEnd() override { + } + + void OnLabel(TStringBuf name, TStringBuf value) override { + NProto::TLabel* label = (Sample_ == nullptr) + ? Samples_->AddCommonLabels() + : Sample_->AddLabels(); + label->SetName(TString{name}); label->SetValue(TString{value}); - } - - void OnDouble(TInstant time, double value) override { - Y_ENSURE(Sample_, "metric not started"); - NProto::TPoint* point = Sample_->AddPoints(); - point->SetTime(time.MilliSeconds()); - point->SetFloat64(value); - } - - void OnInt64(TInstant time, i64 value) override { - Y_ENSURE(Sample_, "metric not started"); - NProto::TPoint* point = Sample_->AddPoints(); - point->SetTime(time.MilliSeconds()); - point->SetInt64(value); - } - - void OnUint64(TInstant time, ui64 value) override { - Y_ENSURE(Sample_, "metric not started"); - NProto::TPoint* point = Sample_->AddPoints(); - point->SetTime(time.MilliSeconds()); - point->SetUint64(value); - } - - void OnHistogram(TInstant time, IHistogramSnapshotPtr snapshot) override { - Y_ENSURE(Sample_, "metric not started"); - NProto::TPoint* point = Sample_->AddPoints(); - point->SetTime(time.MilliSeconds()); - FillHistogram(*snapshot, point->MutableHistogram()); - } - + } + + void OnDouble(TInstant time, double value) override { + Y_ENSURE(Sample_, "metric not started"); + NProto::TPoint* point = Sample_->AddPoints(); + point->SetTime(time.MilliSeconds()); + point->SetFloat64(value); + } + + void OnInt64(TInstant time, i64 value) override { + Y_ENSURE(Sample_, "metric not started"); + NProto::TPoint* point = Sample_->AddPoints(); + point->SetTime(time.MilliSeconds()); + point->SetInt64(value); + } + + void OnUint64(TInstant time, ui64 value) override { + Y_ENSURE(Sample_, "metric not started"); + NProto::TPoint* point = Sample_->AddPoints(); + point->SetTime(time.MilliSeconds()); + point->SetUint64(value); + } + + void OnHistogram(TInstant time, IHistogramSnapshotPtr snapshot) override { + Y_ENSURE(Sample_, "metric not started"); + NProto::TPoint* point = Sample_->AddPoints(); + point->SetTime(time.MilliSeconds()); + FillHistogram(*snapshot, point->MutableHistogram()); + } + void OnSummaryDouble(TInstant time, ISummaryDoubleSnapshotPtr snapshot) override { - Y_ENSURE(Sample_, "metric not started"); + Y_ENSURE(Sample_, "metric not started"); NProto::TPoint* point = Sample_->AddPoints(); point->SetTime(time.MilliSeconds()); FillSummaryDouble(*snapshot, point->MutableSummaryDouble()); } void OnLogHistogram(TInstant time, TLogHistogramSnapshotPtr snapshot) override { - Y_ENSURE(Sample_, "metric not started"); + Y_ENSURE(Sample_, "metric not started"); NProto::TPoint* point = Sample_->AddPoints(); point->SetTime(time.MilliSeconds()); FillLogHistogram(*snapshot, point->MutableLogHistogram()); } - void Close() override { - } - - private: - NProto::TMultiSamplesList* Samples_; - NProto::TMultiSample* Sample_; - }; - - } - - IMetricEncoderPtr EncoderProtobuf(NProto::TSingleSamplesList* samples) { + void Close() override { + } + + private: + NProto::TMultiSamplesList* Samples_; + NProto::TMultiSample* Sample_; + }; + + } + + IMetricEncoderPtr EncoderProtobuf(NProto::TSingleSamplesList* samples) { return MakeHolder<TSingleSamplesEncoder>(samples); - } - - IMetricEncoderPtr EncoderProtobuf(NProto::TMultiSamplesList* samples) { + } + + IMetricEncoderPtr EncoderProtobuf(NProto::TMultiSamplesList* samples) { return MakeHolder<TMultiSamplesEncoder>(samples); - } - -} + } + +} diff --git a/library/cpp/monlib/encode/protobuf/protos/samples.proto b/library/cpp/monlib/encode/protobuf/protos/samples.proto index d5f0c6403e..371f4181d2 100644 --- a/library/cpp/monlib/encode/protobuf/protos/samples.proto +++ b/library/cpp/monlib/encode/protobuf/protos/samples.proto @@ -1,33 +1,33 @@ -syntax = 'proto3'; - -package NMonitoring.NProto; - -option java_package = "ru.yandex.solomon.protos"; -option java_multiple_files = true; -option cc_enable_arenas = true; - -message TLabel { - string Name = 1; - string Value = 2; -} - -enum EMetricType { - UNKNOWN = 0; - GAUGE = 1; - IGAUGE = 2; - COUNTER = 3; - RATE = 4; - HISTOGRAM = 5; +syntax = 'proto3'; + +package NMonitoring.NProto; + +option java_package = "ru.yandex.solomon.protos"; +option java_multiple_files = true; +option cc_enable_arenas = true; + +message TLabel { + string Name = 1; + string Value = 2; +} + +enum EMetricType { + UNKNOWN = 0; + GAUGE = 1; + IGAUGE = 2; + COUNTER = 3; + RATE = 4; + HISTOGRAM = 5; HIST_RATE = 6; DSUMMARY = 7; LOGHISTOGRAM = 8; -} - -message THistogram { - repeated double Bounds = 1; // upper bounds of each bucket - repeated uint64 Values = 2; // values stored in each bucket -} - +} + +message THistogram { + repeated double Bounds = 1; // upper bounds of each bucket + repeated uint64 Values = 2; // values stored in each bucket +} + message TLogHistogram { double Base = 1; uint64 ZerosCount = 2; @@ -43,49 +43,49 @@ message TSummaryDouble { uint64 Count = 5; } -// see TSingleSample -message TPoint { - uint64 Time = 1; - oneof Value { - sfixed64 Int64 = 2; - fixed64 Uint64 = 3; - double Float64 = 4; - THistogram Histogram = 5; +// see TSingleSample +message TPoint { + uint64 Time = 1; + oneof Value { + sfixed64 Int64 = 2; + fixed64 Uint64 = 3; + double Float64 = 4; + THistogram Histogram = 5; TSummaryDouble SummaryDouble = 6; TLogHistogram LogHistogram = 7; - } -} - -message TSingleSample { - repeated TLabel Labels = 1; - EMetricType MetricType = 2; - - // inlined TPoint - uint64 Time = 3; - oneof Value { - sfixed64 Int64 = 4; - fixed64 Uint64 = 5; - double Float64 = 6; - THistogram Histogram = 7; + } +} + +message TSingleSample { + repeated TLabel Labels = 1; + EMetricType MetricType = 2; + + // inlined TPoint + uint64 Time = 3; + oneof Value { + sfixed64 Int64 = 4; + fixed64 Uint64 = 5; + double Float64 = 6; + THistogram Histogram = 7; TSummaryDouble SummaryDouble = 8; TLogHistogram LogHistogram = 9; - } -} - -message TMultiSample { - repeated TLabel Labels = 1; - EMetricType MetricType = 2; - repeated TPoint Points = 3; -} - -message TSingleSamplesList { - uint64 CommonTime = 1; - repeated TLabel CommonLabels = 2; - repeated TSingleSample Samples = 3; -} - -message TMultiSamplesList { - uint64 CommonTime = 1; - repeated TLabel CommonLabels = 2; - repeated TMultiSample Samples = 3; -} + } +} + +message TMultiSample { + repeated TLabel Labels = 1; + EMetricType MetricType = 2; + repeated TPoint Points = 3; +} + +message TSingleSamplesList { + uint64 CommonTime = 1; + repeated TLabel CommonLabels = 2; + repeated TSingleSample Samples = 3; +} + +message TMultiSamplesList { + uint64 CommonTime = 1; + repeated TLabel CommonLabels = 2; + repeated TMultiSample Samples = 3; +} diff --git a/library/cpp/monlib/encode/protobuf/protos/ya.make b/library/cpp/monlib/encode/protobuf/protos/ya.make index 29a87a0b21..88ff3ddf88 100644 --- a/library/cpp/monlib/encode/protobuf/protos/ya.make +++ b/library/cpp/monlib/encode/protobuf/protos/ya.make @@ -1,14 +1,14 @@ -PROTO_LIBRARY() - +PROTO_LIBRARY() + OWNER( jamel g:solomon ) - -SRCS( - samples.proto -) - + +SRCS( + samples.proto +) + EXCLUDE_TAGS(GO_PROTO) -END() +END() diff --git a/library/cpp/monlib/encode/protobuf/ya.make b/library/cpp/monlib/encode/protobuf/ya.make index fd575ef731..9354958b6f 100644 --- a/library/cpp/monlib/encode/protobuf/ya.make +++ b/library/cpp/monlib/encode/protobuf/ya.make @@ -1,17 +1,17 @@ -LIBRARY() - +LIBRARY() + OWNER( jamel g:solomon ) - -SRCS( - protobuf_encoder.cpp -) - -PEERDIR( - library/cpp/monlib/encode - library/cpp/monlib/encode/protobuf/protos -) - -END() + +SRCS( + protobuf_encoder.cpp +) + +PEERDIR( + library/cpp/monlib/encode + library/cpp/monlib/encode/protobuf/protos +) + +END() diff --git a/library/cpp/monlib/encode/spack/compression.cpp b/library/cpp/monlib/encode/spack/compression.cpp index 0ad1eee866..0d2152fc85 100644 --- a/library/cpp/monlib/encode/spack/compression.cpp +++ b/library/cpp/monlib/encode/spack/compression.cpp @@ -1,213 +1,213 @@ -#include "compression.h" - +#include "compression.h" + #include <util/generic/buffer.h> -#include <util/generic/cast.h> -#include <util/generic/ptr.h> +#include <util/generic/cast.h> +#include <util/generic/ptr.h> #include <util/generic/scope.h> #include <util/generic/size_literals.h> #include <util/stream/format.h> #include <util/stream/output.h> -#include <util/stream/walk.h> - -#include <contrib/libs/lz4/lz4.h> +#include <util/stream/walk.h> + +#include <contrib/libs/lz4/lz4.h> #include <contrib/libs/xxhash/xxhash.h> -#include <contrib/libs/zlib/zlib.h> -#define ZSTD_STATIC_LINKING_ONLY +#include <contrib/libs/zlib/zlib.h> +#define ZSTD_STATIC_LINKING_ONLY #include <contrib/libs/zstd/include/zstd.h> - -namespace NMonitoring { - namespace { - /////////////////////////////////////////////////////////////////////////////// - // Frame - /////////////////////////////////////////////////////////////////////////////// - using TCompressedSize = ui32; - using TUncompressedSize = ui32; - using TCheckSum = ui32; - + +namespace NMonitoring { + namespace { + /////////////////////////////////////////////////////////////////////////////// + // Frame + /////////////////////////////////////////////////////////////////////////////// + using TCompressedSize = ui32; + using TUncompressedSize = ui32; + using TCheckSum = ui32; + constexpr size_t COMPRESSED_FRAME_SIZE_LIMIT = 512_KB; constexpr size_t UNCOMPRESSED_FRAME_SIZE_LIMIT = COMPRESSED_FRAME_SIZE_LIMIT; constexpr size_t FRAME_SIZE_LIMIT = 2_MB; constexpr size_t DEFAULT_FRAME_LEN = 64_KB; - + struct Y_PACKED TFrameHeader { - TCompressedSize CompressedSize; - TUncompressedSize UncompressedSize; - }; - + TCompressedSize CompressedSize; + TUncompressedSize UncompressedSize; + }; + struct Y_PACKED TFrameFooter { - TCheckSum CheckSum; - }; - - /////////////////////////////////////////////////////////////////////////////// - // TBlock - /////////////////////////////////////////////////////////////////////////////// - struct TBlock: public TStringBuf { - template <typename T> - TBlock(T&& t) + TCheckSum CheckSum; + }; + + /////////////////////////////////////////////////////////////////////////////// + // TBlock + /////////////////////////////////////////////////////////////////////////////// + struct TBlock: public TStringBuf { + template <typename T> + TBlock(T&& t) : TStringBuf(t.data(), t.size()) - { + { Y_ENSURE(t.data() != nullptr); - } - + } + char* data() noexcept { return const_cast<char*>(TStringBuf::data()); - } - }; - - /////////////////////////////////////////////////////////////////////////////// - // XXHASH - /////////////////////////////////////////////////////////////////////////////// - struct TXxHash32 { - static TCheckSum Calc(TBlock in) { - static const ui32 SEED = 0x1337c0de; + } + }; + + /////////////////////////////////////////////////////////////////////////////// + // XXHASH + /////////////////////////////////////////////////////////////////////////////// + struct TXxHash32 { + static TCheckSum Calc(TBlock in) { + static const ui32 SEED = 0x1337c0de; return XXH32(in.data(), in.size(), SEED); - } - - static bool Check(TBlock in, TCheckSum checksum) { - return Calc(in) == checksum; - } - }; - - /////////////////////////////////////////////////////////////////////////////// - // Adler32 - /////////////////////////////////////////////////////////////////////////////// - struct TAdler32 { - static TCheckSum Calc(TBlock in) { + } + + static bool Check(TBlock in, TCheckSum checksum) { + return Calc(in) == checksum; + } + }; + + /////////////////////////////////////////////////////////////////////////////// + // Adler32 + /////////////////////////////////////////////////////////////////////////////// + struct TAdler32 { + static TCheckSum Calc(TBlock in) { return adler32(1L, reinterpret_cast<const Bytef*>(in.data()), in.size()); - } - - static bool Check(TBlock in, TCheckSum checksum) { - return Calc(in) == checksum; - } - }; - - /////////////////////////////////////////////////////////////////////////////// - // LZ4 - /////////////////////////////////////////////////////////////////////////////// - struct TLz4Codec { - static size_t MaxCompressedLength(size_t in) { - int result = LZ4_compressBound(static_cast<int>(in)); - Y_ENSURE(result != 0, "lz4 input size is too large"); - return result; - } - - static size_t Compress(TBlock in, TBlock out) { - int rc = LZ4_compress_default( + } + + static bool Check(TBlock in, TCheckSum checksum) { + return Calc(in) == checksum; + } + }; + + /////////////////////////////////////////////////////////////////////////////// + // LZ4 + /////////////////////////////////////////////////////////////////////////////// + struct TLz4Codec { + static size_t MaxCompressedLength(size_t in) { + int result = LZ4_compressBound(static_cast<int>(in)); + Y_ENSURE(result != 0, "lz4 input size is too large"); + return result; + } + + static size_t Compress(TBlock in, TBlock out) { + int rc = LZ4_compress_default( in.data(), out.data(), SafeIntegerCast<int>(in.size()), SafeIntegerCast<int>(out.size())); - Y_ENSURE(rc != 0, "lz4 compression failed"); - return rc; - } - - static void Decompress(TBlock in, TBlock out) { - int rc = LZ4_decompress_safe( + Y_ENSURE(rc != 0, "lz4 compression failed"); + return rc; + } + + static void Decompress(TBlock in, TBlock out) { + int rc = LZ4_decompress_safe( in.data(), out.data(), SafeIntegerCast<int>(in.size()), SafeIntegerCast<int>(out.size())); - Y_ENSURE(rc >= 0, "the lz4 stream is detected malformed"); - } - }; - - /////////////////////////////////////////////////////////////////////////////// - // ZSTD - /////////////////////////////////////////////////////////////////////////////// - struct TZstdCodec { - static const int LEVEL = 11; - - static size_t MaxCompressedLength(size_t in) { - return ZSTD_compressBound(in); - } - - static size_t Compress(TBlock in, TBlock out) { + Y_ENSURE(rc >= 0, "the lz4 stream is detected malformed"); + } + }; + + /////////////////////////////////////////////////////////////////////////////// + // ZSTD + /////////////////////////////////////////////////////////////////////////////// + struct TZstdCodec { + static const int LEVEL = 11; + + static size_t MaxCompressedLength(size_t in) { + return ZSTD_compressBound(in); + } + + static size_t Compress(TBlock in, TBlock out) { size_t rc = ZSTD_compress(out.data(), out.size(), in.data(), in.size(), LEVEL); - if (Y_UNLIKELY(ZSTD_isError(rc))) { + if (Y_UNLIKELY(ZSTD_isError(rc))) { ythrow yexception() << TStringBuf("zstd compression failed: ") - << ZSTD_getErrorName(rc); - } - return rc; - } - - static void Decompress(TBlock in, TBlock out) { + << ZSTD_getErrorName(rc); + } + return rc; + } + + static void Decompress(TBlock in, TBlock out) { size_t rc = ZSTD_decompress(out.data(), out.size(), in.data(), in.size()); - if (Y_UNLIKELY(ZSTD_isError(rc))) { + if (Y_UNLIKELY(ZSTD_isError(rc))) { ythrow yexception() << TStringBuf("zstd decompression failed: ") - << ZSTD_getErrorName(rc); - } + << ZSTD_getErrorName(rc); + } Y_ENSURE(rc == out.size(), "zstd decompressed wrong size"); - } - }; - - /////////////////////////////////////////////////////////////////////////////// - // ZLIB - /////////////////////////////////////////////////////////////////////////////// - struct TZlibCodec { - static const int LEVEL = 6; - - static size_t MaxCompressedLength(size_t in) { - return compressBound(in); - } - - static size_t Compress(TBlock in, TBlock out) { + } + }; + + /////////////////////////////////////////////////////////////////////////////// + // ZLIB + /////////////////////////////////////////////////////////////////////////////// + struct TZlibCodec { + static const int LEVEL = 6; + + static size_t MaxCompressedLength(size_t in) { + return compressBound(in); + } + + static size_t Compress(TBlock in, TBlock out) { uLong ret = out.size(); - int rc = compress2( + int rc = compress2( reinterpret_cast<Bytef*>(out.data()), - &ret, + &ret, reinterpret_cast<const Bytef*>(in.data()), in.size(), - LEVEL); - Y_ENSURE(rc == Z_OK, "zlib compression failed"); - return ret; - } - - static void Decompress(TBlock in, TBlock out) { + LEVEL); + Y_ENSURE(rc == Z_OK, "zlib compression failed"); + return ret; + } + + static void Decompress(TBlock in, TBlock out) { uLong ret = out.size(); - int rc = uncompress( + int rc = uncompress( reinterpret_cast<Bytef*>(out.data()), - &ret, + &ret, reinterpret_cast<const Bytef*>(in.data()), in.size()); - Y_ENSURE(rc == Z_OK, "zlib decompression failed"); + Y_ENSURE(rc == Z_OK, "zlib decompression failed"); Y_ENSURE(ret == out.size(), "zlib decompressed wrong size"); - } - }; - - // - // Framed streams use next frame structure: - // - // +-----------------+-------------------+============+------------------+ - // | compressed size | uncompressed size | data | check sum | - // +-----------------+-------------------+============+------------------+ - // 4 bytes 4 bytes var len 4 bytes - // - - /////////////////////////////////////////////////////////////////////////////// - // TFramedInputStream - /////////////////////////////////////////////////////////////////////////////// - template <typename TCodecAlg, typename TCheckSumAlg> - class TFramedDecompressStream final: public IWalkInput { - public: - explicit TFramedDecompressStream(IInputStream* in) - : In_(in) - { - } - - private: - size_t DoUnboundedNext(const void** ptr) override { - if (!In_) { - return 0; - } - - TFrameHeader header; - In_->LoadOrFail(&header, sizeof(header)); - - if (header.CompressedSize == 0) { - In_ = nullptr; - return 0; - } - + } + }; + + // + // Framed streams use next frame structure: + // + // +-----------------+-------------------+============+------------------+ + // | compressed size | uncompressed size | data | check sum | + // +-----------------+-------------------+============+------------------+ + // 4 bytes 4 bytes var len 4 bytes + // + + /////////////////////////////////////////////////////////////////////////////// + // TFramedInputStream + /////////////////////////////////////////////////////////////////////////////// + template <typename TCodecAlg, typename TCheckSumAlg> + class TFramedDecompressStream final: public IWalkInput { + public: + explicit TFramedDecompressStream(IInputStream* in) + : In_(in) + { + } + + private: + size_t DoUnboundedNext(const void** ptr) override { + if (!In_) { + return 0; + } + + TFrameHeader header; + In_->LoadOrFail(&header, sizeof(header)); + + if (header.CompressedSize == 0) { + In_ = nullptr; + return 0; + } + Y_ENSURE(header.CompressedSize <= COMPRESSED_FRAME_SIZE_LIMIT, "Compressed frame size is limited to " << HumanReadableSize(COMPRESSED_FRAME_SIZE_LIMIT, SF_BYTES) << " but is " << HumanReadableSize(header.CompressedSize, SF_BYTES)); @@ -216,87 +216,87 @@ namespace NMonitoring { << HumanReadableSize(UNCOMPRESSED_FRAME_SIZE_LIMIT, SF_BYTES) << " but is " << HumanReadableSize(header.UncompressedSize, SF_BYTES)); - Compressed_.Resize(header.CompressedSize); - In_->LoadOrFail(Compressed_.Data(), header.CompressedSize); - - TFrameFooter footer; - In_->LoadOrFail(&footer, sizeof(footer)); - Y_ENSURE(TCheckSumAlg::Check(Compressed_, footer.CheckSum), - "corrupted stream: check sum mismatch"); - - Uncompressed_.Resize(header.UncompressedSize); - TCodecAlg::Decompress(Compressed_, Uncompressed_); - - *ptr = Uncompressed_.Data(); - return Uncompressed_.Size(); - } - - private: - IInputStream* In_; - TBuffer Compressed_; - TBuffer Uncompressed_; - }; - - /////////////////////////////////////////////////////////////////////////////// - // TFramedOutputStream - /////////////////////////////////////////////////////////////////////////////// - template <typename TCodecAlg, typename TCheckSumAlg> + Compressed_.Resize(header.CompressedSize); + In_->LoadOrFail(Compressed_.Data(), header.CompressedSize); + + TFrameFooter footer; + In_->LoadOrFail(&footer, sizeof(footer)); + Y_ENSURE(TCheckSumAlg::Check(Compressed_, footer.CheckSum), + "corrupted stream: check sum mismatch"); + + Uncompressed_.Resize(header.UncompressedSize); + TCodecAlg::Decompress(Compressed_, Uncompressed_); + + *ptr = Uncompressed_.Data(); + return Uncompressed_.Size(); + } + + private: + IInputStream* In_; + TBuffer Compressed_; + TBuffer Uncompressed_; + }; + + /////////////////////////////////////////////////////////////////////////////// + // TFramedOutputStream + /////////////////////////////////////////////////////////////////////////////// + template <typename TCodecAlg, typename TCheckSumAlg> class TFramedCompressStream final: public IFramedCompressStream { - public: - explicit TFramedCompressStream(IOutputStream* out) - : Out_(out) - , Uncompressed_(DEFAULT_FRAME_LEN) - { - } - + public: + explicit TFramedCompressStream(IOutputStream* out) + : Out_(out) + , Uncompressed_(DEFAULT_FRAME_LEN) + { + } + ~TFramedCompressStream() override { - try { - Finish(); - } catch (...) { - } - } - - private: - void DoWrite(const void* buf, size_t len) override { - const char* in = static_cast<const char*>(buf); - - while (len != 0) { - const size_t avail = Uncompressed_.Avail(); - if (len < avail) { - Uncompressed_.Append(in, len); - return; - } - - Uncompressed_.Append(in, avail); - Y_ASSERT(Uncompressed_.Avail() == 0); - - in += avail; - len -= avail; - - WriteCompressedFrame(); - } - } - + try { + Finish(); + } catch (...) { + } + } + + private: + void DoWrite(const void* buf, size_t len) override { + const char* in = static_cast<const char*>(buf); + + while (len != 0) { + const size_t avail = Uncompressed_.Avail(); + if (len < avail) { + Uncompressed_.Append(in, len); + return; + } + + Uncompressed_.Append(in, avail); + Y_ASSERT(Uncompressed_.Avail() == 0); + + in += avail; + len -= avail; + + WriteCompressedFrame(); + } + } + void FlushWithoutEmptyFrame() override { - if (Out_ && !Uncompressed_.Empty()) { - WriteCompressedFrame(); - } - } - + if (Out_ && !Uncompressed_.Empty()) { + WriteCompressedFrame(); + } + } + void FinishAndWriteEmptyFrame() override { - if (Out_) { + if (Out_) { Y_DEFER { - Out_ = nullptr; + Out_ = nullptr; }; if (!Uncompressed_.Empty()) { WriteCompressedFrame(); - } + } WriteEmptyFrame(); - } - } - + } + } + void DoFlush() override { FlushWithoutEmptyFrame(); } @@ -305,79 +305,79 @@ namespace NMonitoring { FinishAndWriteEmptyFrame(); } - void WriteCompressedFrame() { - static const auto framePayload = sizeof(TFrameHeader) + sizeof(TFrameFooter); + void WriteCompressedFrame() { + static const auto framePayload = sizeof(TFrameHeader) + sizeof(TFrameFooter); const auto maxFrameSize = ui64(TCodecAlg::MaxCompressedLength(Uncompressed_.Size())) + framePayload; Y_ENSURE(maxFrameSize <= FRAME_SIZE_LIMIT, "Frame size in encoder is limited to " << HumanReadableSize(FRAME_SIZE_LIMIT, SF_BYTES) << " but is " << HumanReadableSize(maxFrameSize, SF_BYTES)); - + Frame_.Resize(maxFrameSize); - // compress - TBlock compressedBlock = Frame_; - compressedBlock.Skip(sizeof(TFrameHeader)); - compressedBlock.Trunc(TCodecAlg::Compress(Uncompressed_, compressedBlock)); - - // add header - auto header = reinterpret_cast<TFrameHeader*>(Frame_.Data()); + // compress + TBlock compressedBlock = Frame_; + compressedBlock.Skip(sizeof(TFrameHeader)); + compressedBlock.Trunc(TCodecAlg::Compress(Uncompressed_, compressedBlock)); + + // add header + auto header = reinterpret_cast<TFrameHeader*>(Frame_.Data()); header->CompressedSize = SafeIntegerCast<TCompressedSize>(compressedBlock.size()); - header->UncompressedSize = SafeIntegerCast<TUncompressedSize>(Uncompressed_.Size()); - - // add footer - auto footer = reinterpret_cast<TFrameFooter*>( - Frame_.Data() + sizeof(TFrameHeader) + header->CompressedSize); - footer->CheckSum = TCheckSumAlg::Calc(compressedBlock); - - // write - Out_->Write(Frame_.Data(), header->CompressedSize + framePayload); - Uncompressed_.Clear(); - } - - void WriteEmptyFrame() { - static const auto framePayload = sizeof(TFrameHeader) + sizeof(TFrameFooter); - char buf[framePayload] = {0}; - Out_->Write(buf, sizeof(buf)); - } - - private: - IOutputStream* Out_; - TBuffer Uncompressed_; - TBuffer Frame_; - }; - - } - - THolder<IInputStream> CompressedInput(IInputStream* in, ECompression alg) { - switch (alg) { - case ECompression::IDENTITY: - return nullptr; - case ECompression::ZLIB: + header->UncompressedSize = SafeIntegerCast<TUncompressedSize>(Uncompressed_.Size()); + + // add footer + auto footer = reinterpret_cast<TFrameFooter*>( + Frame_.Data() + sizeof(TFrameHeader) + header->CompressedSize); + footer->CheckSum = TCheckSumAlg::Calc(compressedBlock); + + // write + Out_->Write(Frame_.Data(), header->CompressedSize + framePayload); + Uncompressed_.Clear(); + } + + void WriteEmptyFrame() { + static const auto framePayload = sizeof(TFrameHeader) + sizeof(TFrameFooter); + char buf[framePayload] = {0}; + Out_->Write(buf, sizeof(buf)); + } + + private: + IOutputStream* Out_; + TBuffer Uncompressed_; + TBuffer Frame_; + }; + + } + + THolder<IInputStream> CompressedInput(IInputStream* in, ECompression alg) { + switch (alg) { + case ECompression::IDENTITY: + return nullptr; + case ECompression::ZLIB: return MakeHolder<TFramedDecompressStream<TZlibCodec, TAdler32>>(in); - case ECompression::ZSTD: + case ECompression::ZSTD: return MakeHolder<TFramedDecompressStream<TZstdCodec, TXxHash32>>(in); - case ECompression::LZ4: + case ECompression::LZ4: return MakeHolder<TFramedDecompressStream<TLz4Codec, TXxHash32>>(in); - case ECompression::UNKNOWN: - return nullptr; - } - Y_FAIL("invalid compression algorithm"); - } - + case ECompression::UNKNOWN: + return nullptr; + } + Y_FAIL("invalid compression algorithm"); + } + THolder<IFramedCompressStream> CompressedOutput(IOutputStream* out, ECompression alg) { - switch (alg) { - case ECompression::IDENTITY: - return nullptr; - case ECompression::ZLIB: + switch (alg) { + case ECompression::IDENTITY: + return nullptr; + case ECompression::ZLIB: return MakeHolder<TFramedCompressStream<TZlibCodec, TAdler32>>(out); - case ECompression::ZSTD: + case ECompression::ZSTD: return MakeHolder<TFramedCompressStream<TZstdCodec, TXxHash32>>(out); - case ECompression::LZ4: + case ECompression::LZ4: return MakeHolder<TFramedCompressStream<TLz4Codec, TXxHash32>>(out); - case ECompression::UNKNOWN: - return nullptr; - } - Y_FAIL("invalid compression algorithm"); - } - -} + case ECompression::UNKNOWN: + return nullptr; + } + Y_FAIL("invalid compression algorithm"); + } + +} diff --git a/library/cpp/monlib/encode/spack/compression.h b/library/cpp/monlib/encode/spack/compression.h index 1f99b0f5a0..f74d8b424e 100644 --- a/library/cpp/monlib/encode/spack/compression.h +++ b/library/cpp/monlib/encode/spack/compression.h @@ -1,12 +1,12 @@ -#pragma once - -#include "spack_v1.h" - -#include <util/stream/input.h> -#include <util/stream/output.h> - -namespace NMonitoring { - +#pragma once + +#include "spack_v1.h" + +#include <util/stream/input.h> +#include <util/stream/output.h> + +namespace NMonitoring { + class IFramedCompressStream: public IOutputStream { public: virtual void FlushWithoutEmptyFrame() = 0; diff --git a/library/cpp/monlib/encode/spack/fuzz/main.cpp b/library/cpp/monlib/encode/spack/fuzz/main.cpp index ac64b8b232..6a14afe71c 100644 --- a/library/cpp/monlib/encode/spack/fuzz/main.cpp +++ b/library/cpp/monlib/encode/spack/fuzz/main.cpp @@ -1,5 +1,5 @@ -#include <library/cpp/monlib/encode/spack/spack_v1.h> -#include <library/cpp/monlib/encode/fake/fake.h> +#include <library/cpp/monlib/encode/spack/spack_v1.h> +#include <library/cpp/monlib/encode/fake/fake.h> #include <util/stream/mem.h> diff --git a/library/cpp/monlib/encode/spack/fuzz/ya.make b/library/cpp/monlib/encode/spack/fuzz/ya.make index c43ee84f36..99b63eadd5 100644 --- a/library/cpp/monlib/encode/spack/fuzz/ya.make +++ b/library/cpp/monlib/encode/spack/fuzz/ya.make @@ -10,8 +10,8 @@ FUZZ_OPTS(-rss_limit_mb=1024) SIZE(MEDIUM) PEERDIR( - library/cpp/monlib/encode/spack - library/cpp/monlib/encode/fake + library/cpp/monlib/encode/spack + library/cpp/monlib/encode/fake ) SRCS( diff --git a/library/cpp/monlib/encode/spack/spack_v1.h b/library/cpp/monlib/encode/spack/spack_v1.h index 628f07de5c..cf1c9417b9 100644 --- a/library/cpp/monlib/encode/spack/spack_v1.h +++ b/library/cpp/monlib/encode/spack/spack_v1.h @@ -1,107 +1,107 @@ -#pragma once - -#include <library/cpp/monlib/encode/encoder.h> -#include <library/cpp/monlib/encode/format.h> -#include <library/cpp/monlib/metrics/metric.h> - -#include <util/generic/yexception.h> - -// -// format specification available here: -// https://wiki.yandex-team.ru/solomon/api/dataformat/spackv1/ -// - -class IInputStream; -class IOutputStream; - -namespace NMonitoring { +#pragma once + +#include <library/cpp/monlib/encode/encoder.h> +#include <library/cpp/monlib/encode/format.h> +#include <library/cpp/monlib/metrics/metric.h> + +#include <util/generic/yexception.h> + +// +// format specification available here: +// https://wiki.yandex-team.ru/solomon/api/dataformat/spackv1/ +// + +class IInputStream; +class IOutputStream; + +namespace NMonitoring { class TSpackDecodeError: public yexception { }; - constexpr auto EncodeMetricType(EMetricType mt) noexcept { - return static_cast<std::underlying_type_t<EMetricType>>(mt); - } - - EMetricType DecodeMetricType(ui8 byte); - + constexpr auto EncodeMetricType(EMetricType mt) noexcept { + return static_cast<std::underlying_type_t<EMetricType>>(mt); + } + + EMetricType DecodeMetricType(ui8 byte); + [[nodiscard]] bool TryDecodeMetricType(ui8 byte, EMetricType* result); - /////////////////////////////////////////////////////////////////////////////// - // EValueType - /////////////////////////////////////////////////////////////////////////////// - enum class EValueType : ui8 { - NONE = 0x00, - ONE_WITHOUT_TS = 0x01, - ONE_WITH_TS = 0x02, - MANY_WITH_TS = 0x03, - }; - - constexpr auto EncodeValueType(EValueType vt) noexcept { - return static_cast<std::underlying_type_t<EValueType>>(vt); - } - - EValueType DecodeValueType(ui8 byte); - + /////////////////////////////////////////////////////////////////////////////// + // EValueType + /////////////////////////////////////////////////////////////////////////////// + enum class EValueType : ui8 { + NONE = 0x00, + ONE_WITHOUT_TS = 0x01, + ONE_WITH_TS = 0x02, + MANY_WITH_TS = 0x03, + }; + + constexpr auto EncodeValueType(EValueType vt) noexcept { + return static_cast<std::underlying_type_t<EValueType>>(vt); + } + + EValueType DecodeValueType(ui8 byte); + [[nodiscard]] bool TryDecodeValueType(ui8 byte, EValueType* result); - /////////////////////////////////////////////////////////////////////////////// - // ETimePrecision - /////////////////////////////////////////////////////////////////////////////// - enum class ETimePrecision : ui8 { - SECONDS = 0x00, - MILLIS = 0x01, - }; - - constexpr auto EncodeTimePrecision(ETimePrecision tp) noexcept { - return static_cast<std::underlying_type_t<ETimePrecision>>(tp); - } - - ETimePrecision DecodeTimePrecision(ui8 byte); - + /////////////////////////////////////////////////////////////////////////////// + // ETimePrecision + /////////////////////////////////////////////////////////////////////////////// + enum class ETimePrecision : ui8 { + SECONDS = 0x00, + MILLIS = 0x01, + }; + + constexpr auto EncodeTimePrecision(ETimePrecision tp) noexcept { + return static_cast<std::underlying_type_t<ETimePrecision>>(tp); + } + + ETimePrecision DecodeTimePrecision(ui8 byte); + [[nodiscard]] bool TryDecodeTimePrecision(ui8 byte, ETimePrecision* result); - /////////////////////////////////////////////////////////////////////////////// - // ECompression - /////////////////////////////////////////////////////////////////////////////// - ui8 EncodeCompression(ECompression c) noexcept; - - ECompression DecodeCompression(ui8 byte); - + /////////////////////////////////////////////////////////////////////////////// + // ECompression + /////////////////////////////////////////////////////////////////////////////// + ui8 EncodeCompression(ECompression c) noexcept; + + ECompression DecodeCompression(ui8 byte); + [[nodiscard]] bool TryDecodeCompression(ui8 byte, ECompression* result); - /////////////////////////////////////////////////////////////////////////////// - // TSpackHeader - /////////////////////////////////////////////////////////////////////////////// + /////////////////////////////////////////////////////////////////////////////// + // TSpackHeader + /////////////////////////////////////////////////////////////////////////////// struct Y_PACKED TSpackHeader { - ui16 Magic = 0x5053; // "SP" + ui16 Magic = 0x5053; // "SP" ui16 Version; // MSB - major version, LSB - minor version - ui16 HeaderSize = sizeof(TSpackHeader); - ui8 TimePrecision; - ui8 Compression; - ui32 LabelNamesSize; - ui32 LabelValuesSize; - ui32 MetricCount; - ui32 PointsCount; - // add new fields here - }; - + ui16 HeaderSize = sizeof(TSpackHeader); + ui8 TimePrecision; + ui8 Compression; + ui32 LabelNamesSize; + ui32 LabelValuesSize; + ui32 MetricCount; + ui32 PointsCount; + // add new fields here + }; + enum ESpackV1Version: ui16 { SV1_00 = 0x0100, SV1_01 = 0x0101, SV1_02 = 0x0102 }; - IMetricEncoderPtr EncoderSpackV1( - IOutputStream* out, - ETimePrecision timePrecision, + IMetricEncoderPtr EncoderSpackV1( + IOutputStream* out, + ETimePrecision timePrecision, ECompression compression, - EMetricsMergingMode mergingMode = EMetricsMergingMode::DEFAULT + EMetricsMergingMode mergingMode = EMetricsMergingMode::DEFAULT ); - + IMetricEncoderPtr EncoderSpackV12( IOutputStream* out, ETimePrecision timePrecision, @@ -109,7 +109,7 @@ namespace NMonitoring { EMetricsMergingMode mergingMode = EMetricsMergingMode::DEFAULT, TStringBuf metricNameLabel = "name" ); - + void DecodeSpackV1(IInputStream* in, IMetricConsumer* c, TStringBuf metricNameLabel = "name"); -} +} diff --git a/library/cpp/monlib/encode/spack/spack_v1_decoder.cpp b/library/cpp/monlib/encode/spack/spack_v1_decoder.cpp index a6dadc08a8..1f445fc80d 100644 --- a/library/cpp/monlib/encode/spack/spack_v1_decoder.cpp +++ b/library/cpp/monlib/encode/spack/spack_v1_decoder.cpp @@ -1,189 +1,189 @@ -#include "spack_v1.h" -#include "varint.h" -#include "compression.h" - -#include <library/cpp/monlib/encode/buffered/string_pool.h> +#include "spack_v1.h" +#include "varint.h" +#include "compression.h" + +#include <library/cpp/monlib/encode/buffered/string_pool.h> #include <library/cpp/monlib/exception/exception.h> -#include <library/cpp/monlib/metrics/histogram_collector.h> +#include <library/cpp/monlib/metrics/histogram_collector.h> #include <library/cpp/monlib/metrics/metric.h> - -#include <util/generic/yexception.h> -#include <util/generic/buffer.h> + +#include <util/generic/yexception.h> +#include <util/generic/buffer.h> #include <util/generic/size_literals.h> #include <util/stream/format.h> - -#ifndef _little_endian_ -#error Unsupported platform -#endif - -namespace NMonitoring { - namespace { + +#ifndef _little_endian_ +#error Unsupported platform +#endif + +namespace NMonitoring { + namespace { #define DECODE_ENSURE(COND, ...) MONLIB_ENSURE_EX(COND, TSpackDecodeError() << __VA_ARGS__) constexpr ui64 LABEL_SIZE_LIMIT = 128_MB; - /////////////////////////////////////////////////////////////////////// - // TDecoderSpackV1 - /////////////////////////////////////////////////////////////////////// - class TDecoderSpackV1 { - public: + /////////////////////////////////////////////////////////////////////// + // TDecoderSpackV1 + /////////////////////////////////////////////////////////////////////// + class TDecoderSpackV1 { + public: TDecoderSpackV1(IInputStream* in, TStringBuf metricNameLabel) - : In_(in) + : In_(in) , MetricNameLabel_(metricNameLabel) - { - } - - void Decode(IMetricConsumer* c) { - c->OnStreamBegin(); - - // (1) read header + { + } + + void Decode(IMetricConsumer* c) { + c->OnStreamBegin(); + + // (1) read header size_t readBytes = In_->Read(&Header_, sizeof(Header_)); DECODE_ENSURE(readBytes == sizeof(Header_), "not enough data in input stream to read header"); - + ui8 version = ((Header_.Version >> 8) & 0xff); DECODE_ENSURE(version == 1, "versions mismatch (expected: 1, got: " << +version << ')'); - + DECODE_ENSURE(Header_.HeaderSize >= sizeof(Header_), "invalid header size"); if (size_t skipBytes = Header_.HeaderSize - sizeof(Header_)) { DECODE_ENSURE(In_->Skip(skipBytes) == skipBytes, "input stream unexpectedly ended"); - } - - if (Header_.MetricCount == 0) { - // emulate empty stream - c->OnStreamEnd(); - return; - } - - // if compression enabled all below reads must go throught decompressor + } + + if (Header_.MetricCount == 0) { + // emulate empty stream + c->OnStreamEnd(); + return; + } + + // if compression enabled all below reads must go throught decompressor auto compressedIn = CompressedInput(In_, DecodeCompression(Header_.Compression)); - if (compressedIn) { - In_ = compressedIn.Get(); - } - + if (compressedIn) { + In_ = compressedIn.Get(); + } + TimePrecision_ = DecodeTimePrecision(Header_.TimePrecision); - + const ui64 labelSizeTotal = ui64(Header_.LabelNamesSize) + Header_.LabelValuesSize; DECODE_ENSURE(labelSizeTotal <= LABEL_SIZE_LIMIT, "Label names & values size of " << HumanReadableSize(labelSizeTotal, SF_BYTES) << " exceeds the limit which is " << HumanReadableSize(LABEL_SIZE_LIMIT, SF_BYTES)); - // (2) read string pools + // (2) read string pools TVector<char> namesBuf(Header_.LabelNamesSize); readBytes = In_->Load(namesBuf.data(), namesBuf.size()); DECODE_ENSURE(readBytes == Header_.LabelNamesSize, "not enough data to read label names pool"); TStringPool labelNames(namesBuf.data(), namesBuf.size()); - + TVector<char> valuesBuf(Header_.LabelValuesSize); readBytes = In_->Load(valuesBuf.data(), valuesBuf.size()); DECODE_ENSURE(readBytes == Header_.LabelValuesSize, "not enough data to read label values pool"); TStringPool labelValues(valuesBuf.data(), valuesBuf.size()); - - // (3) read common time - c->OnCommonTime(ReadTime()); - - // (4) read common labels - if (ui32 commonLabelsCount = ReadVarint()) { + + // (3) read common time + c->OnCommonTime(ReadTime()); + + // (4) read common labels + if (ui32 commonLabelsCount = ReadVarint()) { c->OnLabelsBegin(); - ReadLabels(labelNames, labelValues, commonLabelsCount, c); + ReadLabels(labelNames, labelValues, commonLabelsCount, c); c->OnLabelsEnd(); - } - - // (5) read metrics - ReadMetrics(labelNames, labelValues, c); - c->OnStreamEnd(); - } - - private: - void ReadMetrics( - const TStringPool& labelNames, - const TStringPool& labelValues, - IMetricConsumer* c) - { - for (ui32 i = 0; i < Header_.MetricCount; i++) { - // (5.1) types byte - ui8 typesByte = ReadFixed<ui8>(); - EMetricType metricType = DecodeMetricType(typesByte >> 2); - EValueType valueType = DecodeValueType(typesByte & 0x03); - - c->OnMetricBegin(metricType); - - // TODO: use it - ReadFixed<ui8>(); // skip flags byte - + } + + // (5) read metrics + ReadMetrics(labelNames, labelValues, c); + c->OnStreamEnd(); + } + + private: + void ReadMetrics( + const TStringPool& labelNames, + const TStringPool& labelValues, + IMetricConsumer* c) + { + for (ui32 i = 0; i < Header_.MetricCount; i++) { + // (5.1) types byte + ui8 typesByte = ReadFixed<ui8>(); + EMetricType metricType = DecodeMetricType(typesByte >> 2); + EValueType valueType = DecodeValueType(typesByte & 0x03); + + c->OnMetricBegin(metricType); + + // TODO: use it + ReadFixed<ui8>(); // skip flags byte + auto metricNameValueIndex = std::numeric_limits<ui32>::max(); if (Header_.Version >= SV1_02) { metricNameValueIndex = ReadVarint(); } - // (5.2) labels - ui32 labelsCount = ReadVarint(); + // (5.2) labels + ui32 labelsCount = ReadVarint(); DECODE_ENSURE(Header_.Version >= SV1_02 || labelsCount > 0, "metric #" << i << " has no labels"); c->OnLabelsBegin(); if (Header_.Version >= SV1_02) { c->OnLabel(MetricNameLabel_, labelValues.Get(metricNameValueIndex)); } - ReadLabels(labelNames, labelValues, labelsCount, c); + ReadLabels(labelNames, labelValues, labelsCount, c); c->OnLabelsEnd(); - - // (5.3) values - switch (valueType) { - case EValueType::NONE: - break; - case EValueType::ONE_WITHOUT_TS: - ReadValue(metricType, TInstant::Zero(), c); - break; - case EValueType::ONE_WITH_TS: { - TInstant time = ReadTime(); - ReadValue(metricType, time, c); - break; - } - case EValueType::MANY_WITH_TS: { - ui32 pointsCount = ReadVarint(); - for (ui32 i = 0; i < pointsCount; i++) { - TInstant time = ReadTime(); - ReadValue(metricType, time, c); - } - break; - } - } - - c->OnMetricEnd(); - } - } - - void ReadValue(EMetricType metricType, TInstant time, IMetricConsumer* c) { - switch (metricType) { - case EMetricType::GAUGE: - c->OnDouble(time, ReadFixed<double>()); - break; - - case EMetricType::IGAUGE: - c->OnInt64(time, ReadFixed<i64>()); - break; - - case EMetricType::COUNTER: - case EMetricType::RATE: - c->OnUint64(time, ReadFixed<ui64>()); - break; - - case EMetricType::DSUMMARY: + + // (5.3) values + switch (valueType) { + case EValueType::NONE: + break; + case EValueType::ONE_WITHOUT_TS: + ReadValue(metricType, TInstant::Zero(), c); + break; + case EValueType::ONE_WITH_TS: { + TInstant time = ReadTime(); + ReadValue(metricType, time, c); + break; + } + case EValueType::MANY_WITH_TS: { + ui32 pointsCount = ReadVarint(); + for (ui32 i = 0; i < pointsCount; i++) { + TInstant time = ReadTime(); + ReadValue(metricType, time, c); + } + break; + } + } + + c->OnMetricEnd(); + } + } + + void ReadValue(EMetricType metricType, TInstant time, IMetricConsumer* c) { + switch (metricType) { + case EMetricType::GAUGE: + c->OnDouble(time, ReadFixed<double>()); + break; + + case EMetricType::IGAUGE: + c->OnInt64(time, ReadFixed<i64>()); + break; + + case EMetricType::COUNTER: + case EMetricType::RATE: + c->OnUint64(time, ReadFixed<ui64>()); + break; + + case EMetricType::DSUMMARY: c->OnSummaryDouble(time, ReadSummaryDouble()); break; - case EMetricType::HIST: - case EMetricType::HIST_RATE: - c->OnHistogram(time, ReadHistogram()); - break; - + case EMetricType::HIST: + case EMetricType::HIST_RATE: + c->OnHistogram(time, ReadHistogram()); + break; + case EMetricType::LOGHIST: c->OnLogHistogram(time, ReadLogHistogram()); break; - default: + default: throw TSpackDecodeError() << "Unsupported metric type: " << metricType; - } - } - + } + } + ISummaryDoubleSnapshotPtr ReadSummaryDouble() { ui64 count = ReadFixed<ui64>(); double sum = ReadFixed<double>(); @@ -198,10 +198,10 @@ namespace NMonitoring { ui64 zerosCount = ReadFixed<ui64>(); int startPower = static_cast<int>(ReadVarint()); ui32 count = ReadVarint(); - // see https://a.yandex-team.ru/arc/trunk/arcadia/infra/yasm/stockpile_client/points.cpp?rev=r8593154#L31 - // and https://a.yandex-team.ru/arc/trunk/arcadia/infra/yasm/common/points/hgram/normal/normal.h?rev=r8268697#L9 - // TODO: share this constant value - Y_ENSURE(count <= 100u, "more than 100 buckets in log histogram: " << count); + // see https://a.yandex-team.ru/arc/trunk/arcadia/infra/yasm/stockpile_client/points.cpp?rev=r8593154#L31 + // and https://a.yandex-team.ru/arc/trunk/arcadia/infra/yasm/common/points/hgram/normal/normal.h?rev=r8268697#L9 + // TODO: share this constant value + Y_ENSURE(count <= 100u, "more than 100 buckets in log histogram: " << count); TVector<double> buckets; buckets.reserve(count); for (ui32 i = 0; i < count; ++i) { @@ -210,17 +210,17 @@ namespace NMonitoring { return MakeIntrusive<TLogHistogramSnapshot>(base, zerosCount, startPower, std::move(buckets)); } - IHistogramSnapshotPtr ReadHistogram() { - ui32 bucketsCount = ReadVarint(); + IHistogramSnapshotPtr ReadHistogram() { + ui32 bucketsCount = ReadVarint(); auto s = TExplicitHistogramSnapshot::New(bucketsCount); - + if (SV1_00 == Header_.Version) { // v1.0 for (ui32 i = 0; i < bucketsCount; i++) { i64 bound = ReadFixed<i64>(); double doubleBound = (bound != Max<i64>()) ? static_cast<double>(bound) : Max<double>(); - + (*s)[i].first = doubleBound; } } else { @@ -228,62 +228,62 @@ namespace NMonitoring { double doubleBound = ReadFixed<double>(); (*s)[i].first = doubleBound; } - } - - - // values - for (ui32 i = 0; i < bucketsCount; i++) { - (*s)[i].second = ReadFixed<ui64>(); - } - return s; - } - - void ReadLabels( - const TStringPool& labelNames, - const TStringPool& labelValues, - ui32 count, - IMetricConsumer* c) - { - for (ui32 i = 0; i < count; i++) { + } + + + // values + for (ui32 i = 0; i < bucketsCount; i++) { + (*s)[i].second = ReadFixed<ui64>(); + } + return s; + } + + void ReadLabels( + const TStringPool& labelNames, + const TStringPool& labelValues, + ui32 count, + IMetricConsumer* c) + { + for (ui32 i = 0; i < count; i++) { auto nameIdx = ReadVarint(); auto valueIdx = ReadVarint(); c->OnLabel(labelNames.Get(nameIdx), labelValues.Get(valueIdx)); - } - } - - TInstant ReadTime() { - switch (TimePrecision_) { - case ETimePrecision::SECONDS: - return TInstant::Seconds(ReadFixed<ui32>()); - case ETimePrecision::MILLIS: - return TInstant::MilliSeconds(ReadFixed<ui64>()); - } - Y_FAIL("invalid time precision"); - } - - template <typename T> - inline T ReadFixed() { - T value; - size_t readBytes = In_->Load(&value, sizeof(T)); + } + } + + TInstant ReadTime() { + switch (TimePrecision_) { + case ETimePrecision::SECONDS: + return TInstant::Seconds(ReadFixed<ui32>()); + case ETimePrecision::MILLIS: + return TInstant::MilliSeconds(ReadFixed<ui64>()); + } + Y_FAIL("invalid time precision"); + } + + template <typename T> + inline T ReadFixed() { + T value; + size_t readBytes = In_->Load(&value, sizeof(T)); DECODE_ENSURE(readBytes == sizeof(T), "no enough data to read " << TypeName<T>()); - return value; - } - - inline ui32 ReadVarint() { - return ReadVarUInt32(In_); - } - - private: - IInputStream* In_; + return value; + } + + inline ui32 ReadVarint() { + return ReadVarUInt32(In_); + } + + private: + IInputStream* In_; TString MetricNameLabel_; - ETimePrecision TimePrecision_; + ETimePrecision TimePrecision_; TSpackHeader Header_; }; // class TDecoderSpackV1 - + #undef DECODE_ENSURE } // namespace - - EValueType DecodeValueType(ui8 byte) { + + EValueType DecodeValueType(ui8 byte) { EValueType result; if (!TryDecodeValueType(byte, &result)) { throw TSpackDecodeError() << "unknown value type: " << byte; @@ -292,32 +292,32 @@ namespace NMonitoring { } bool TryDecodeValueType(ui8 byte, EValueType* result) { - if (byte == EncodeValueType(EValueType::NONE)) { + if (byte == EncodeValueType(EValueType::NONE)) { if (result) { *result = EValueType::NONE; } return true; - } else if (byte == EncodeValueType(EValueType::ONE_WITHOUT_TS)) { + } else if (byte == EncodeValueType(EValueType::ONE_WITHOUT_TS)) { if (result) { *result = EValueType::ONE_WITHOUT_TS; } return true; - } else if (byte == EncodeValueType(EValueType::ONE_WITH_TS)) { + } else if (byte == EncodeValueType(EValueType::ONE_WITH_TS)) { if (result) { *result = EValueType::ONE_WITH_TS; } return true; - } else if (byte == EncodeValueType(EValueType::MANY_WITH_TS)) { + } else if (byte == EncodeValueType(EValueType::MANY_WITH_TS)) { if (result) { *result = EValueType::MANY_WITH_TS; } return true; - } else { + } else { return false; - } - } - - ETimePrecision DecodeTimePrecision(ui8 byte) { + } + } + + ETimePrecision DecodeTimePrecision(ui8 byte) { ETimePrecision result; if (!TryDecodeTimePrecision(byte, &result)) { throw TSpackDecodeError() << "unknown time precision: " << byte; @@ -326,22 +326,22 @@ namespace NMonitoring { } bool TryDecodeTimePrecision(ui8 byte, ETimePrecision* result) { - if (byte == EncodeTimePrecision(ETimePrecision::SECONDS)) { + if (byte == EncodeTimePrecision(ETimePrecision::SECONDS)) { if (result) { *result = ETimePrecision::SECONDS; } return true; - } else if (byte == EncodeTimePrecision(ETimePrecision::MILLIS)) { + } else if (byte == EncodeTimePrecision(ETimePrecision::MILLIS)) { if (result) { *result = ETimePrecision::MILLIS; } return true; - } else { + } else { return false; - } - } - - EMetricType DecodeMetricType(ui8 byte) { + } + } + + EMetricType DecodeMetricType(ui8 byte) { EMetricType result; if (!TryDecodeMetricType(byte, &result)) { throw TSpackDecodeError() << "unknown metric type: " << byte; @@ -350,37 +350,37 @@ namespace NMonitoring { } bool TryDecodeMetricType(ui8 byte, EMetricType* result) { - if (byte == EncodeMetricType(EMetricType::GAUGE)) { + if (byte == EncodeMetricType(EMetricType::GAUGE)) { if (result) { *result = EMetricType::GAUGE; } return true; - } else if (byte == EncodeMetricType(EMetricType::COUNTER)) { + } else if (byte == EncodeMetricType(EMetricType::COUNTER)) { if (result) { *result = EMetricType::COUNTER; } return true; - } else if (byte == EncodeMetricType(EMetricType::RATE)) { + } else if (byte == EncodeMetricType(EMetricType::RATE)) { if (result) { *result = EMetricType::RATE; } return true; - } else if (byte == EncodeMetricType(EMetricType::IGAUGE)) { + } else if (byte == EncodeMetricType(EMetricType::IGAUGE)) { if (result) { *result = EMetricType::IGAUGE; } return true; - } else if (byte == EncodeMetricType(EMetricType::HIST)) { + } else if (byte == EncodeMetricType(EMetricType::HIST)) { if (result) { *result = EMetricType::HIST; } return true; - } else if (byte == EncodeMetricType(EMetricType::HIST_RATE)) { + } else if (byte == EncodeMetricType(EMetricType::HIST_RATE)) { if (result) { *result = EMetricType::HIST_RATE; } return true; - } else if (byte == EncodeMetricType(EMetricType::DSUMMARY)) { + } else if (byte == EncodeMetricType(EMetricType::DSUMMARY)) { if (result) { *result = EMetricType::DSUMMARY; } @@ -390,33 +390,33 @@ namespace NMonitoring { *result = EMetricType::LOGHIST; } return true; - } else if (byte == EncodeMetricType(EMetricType::UNKNOWN)) { + } else if (byte == EncodeMetricType(EMetricType::UNKNOWN)) { if (result) { *result = EMetricType::UNKNOWN; } return true; - } else { + } else { return false; - } - } - - ui8 EncodeCompression(ECompression c) noexcept { - switch (c) { - case ECompression::IDENTITY: - return 0x00; - case ECompression::ZLIB: - return 0x01; - case ECompression::ZSTD: - return 0x02; - case ECompression::LZ4: - return 0x03; - case ECompression::UNKNOWN: - return Max<ui8>(); - } - Y_FAIL(); // for GCC - } - - ECompression DecodeCompression(ui8 byte) { + } + } + + ui8 EncodeCompression(ECompression c) noexcept { + switch (c) { + case ECompression::IDENTITY: + return 0x00; + case ECompression::ZLIB: + return 0x01; + case ECompression::ZSTD: + return 0x02; + case ECompression::LZ4: + return 0x03; + case ECompression::UNKNOWN: + return Max<ui8>(); + } + Y_FAIL(); // for GCC + } + + ECompression DecodeCompression(ui8 byte) { ECompression result; if (!TryDecodeCompression(byte, &result)) { throw TSpackDecodeError() << "unknown compression alg: " << byte; @@ -425,34 +425,34 @@ namespace NMonitoring { } bool TryDecodeCompression(ui8 byte, ECompression* result) { - if (byte == EncodeCompression(ECompression::IDENTITY)) { + if (byte == EncodeCompression(ECompression::IDENTITY)) { if (result) { *result = ECompression::IDENTITY; } return true; - } else if (byte == EncodeCompression(ECompression::ZLIB)) { + } else if (byte == EncodeCompression(ECompression::ZLIB)) { if (result) { *result = ECompression::ZLIB; } return true; - } else if (byte == EncodeCompression(ECompression::ZSTD)) { + } else if (byte == EncodeCompression(ECompression::ZSTD)) { if (result) { *result = ECompression::ZSTD; } return true; - } else if (byte == EncodeCompression(ECompression::LZ4)) { + } else if (byte == EncodeCompression(ECompression::LZ4)) { if (result) { *result = ECompression::LZ4; } return true; - } else { + } else { return false; - } - } - + } + } + void DecodeSpackV1(IInputStream* in, IMetricConsumer* c, TStringBuf metricNameLabel) { TDecoderSpackV1 decoder(in, metricNameLabel); - decoder.Decode(c); - } - -} + decoder.Decode(c); + } + +} diff --git a/library/cpp/monlib/encode/spack/spack_v1_encoder.cpp b/library/cpp/monlib/encode/spack/spack_v1_encoder.cpp index f4f5b88073..a2b0bb5f50 100644 --- a/library/cpp/monlib/encode/spack/spack_v1_encoder.cpp +++ b/library/cpp/monlib/encode/spack/spack_v1_encoder.cpp @@ -1,65 +1,65 @@ -#include "spack_v1.h" -#include "compression.h" -#include "varint.h" - -#include <library/cpp/monlib/encode/buffered/buffered_encoder_base.h> - -#include <util/generic/cast.h> -#include <util/datetime/base.h> -#include <util/string/builder.h> - -#ifndef _little_endian_ -#error Unsupported platform -#endif - -namespace NMonitoring { - namespace { - /////////////////////////////////////////////////////////////////////// - // TEncoderSpackV1 - /////////////////////////////////////////////////////////////////////// +#include "spack_v1.h" +#include "compression.h" +#include "varint.h" + +#include <library/cpp/monlib/encode/buffered/buffered_encoder_base.h> + +#include <util/generic/cast.h> +#include <util/datetime/base.h> +#include <util/string/builder.h> + +#ifndef _little_endian_ +#error Unsupported platform +#endif + +namespace NMonitoring { + namespace { + /////////////////////////////////////////////////////////////////////// + // TEncoderSpackV1 + /////////////////////////////////////////////////////////////////////// class TEncoderSpackV1 final: public TBufferedEncoderBase { - public: - TEncoderSpackV1( - IOutputStream* out, - ETimePrecision timePrecision, + public: + TEncoderSpackV1( + IOutputStream* out, + ETimePrecision timePrecision, ECompression compression, EMetricsMergingMode mergingMode, ESpackV1Version version, TStringBuf metricNameLabel ) - : Out_(out) - , TimePrecision_(timePrecision) - , Compression_(compression) + : Out_(out) + , TimePrecision_(timePrecision) + , Compression_(compression) , Version_(version) , MetricName_(Version_ >= SV1_02 ? LabelNamesPool_.PutIfAbsent(metricNameLabel) : nullptr) - { - MetricsMergingMode_ = mergingMode; + { + MetricsMergingMode_ = mergingMode; LabelNamesPool_.SetSorted(true); LabelValuesPool_.SetSorted(true); - } - + } + ~TEncoderSpackV1() override { - Close(); - } - - private: - void OnDouble(TInstant time, double value) override { + Close(); + } + + private: + void OnDouble(TInstant time, double value) override { TBufferedEncoderBase::OnDouble(time, value); - } - - void OnInt64(TInstant time, i64 value) override { - TBufferedEncoderBase::OnInt64(time, value); - } - - void OnUint64(TInstant time, ui64 value) override { + } + + void OnInt64(TInstant time, i64 value) override { + TBufferedEncoderBase::OnInt64(time, value); + } + + void OnUint64(TInstant time, ui64 value) override { TBufferedEncoderBase::OnUint64(time, value); - } - - void OnHistogram(TInstant time, IHistogramSnapshotPtr snapshot) override { - TBufferedEncoderBase::OnHistogram(time, snapshot); - } - + } + + void OnHistogram(TInstant time, IHistogramSnapshotPtr snapshot) override { + TBufferedEncoderBase::OnHistogram(time, snapshot); + } + void OnSummaryDouble(TInstant time, ISummaryDoubleSnapshotPtr snapshot) override { TBufferedEncoderBase::OnSummaryDouble(time, snapshot); } @@ -68,70 +68,70 @@ namespace NMonitoring { TBufferedEncoderBase::OnLogHistogram(time, snapshot); } - void Close() override { - if (Closed_) { - return; - } - Closed_ = true; - + void Close() override { + if (Closed_) { + return; + } + Closed_ = true; + LabelNamesPool_.Build(); LabelValuesPool_.Build(); - + // Sort all points uniquely by ts -- the size can decrease ui64 pointsCount = 0; - for (TMetric& metric : Metrics_) { - if (metric.TimeSeries.Size() > 1) { - metric.TimeSeries.SortByTs(); + for (TMetric& metric : Metrics_) { + if (metric.TimeSeries.Size() > 1) { + metric.TimeSeries.SortByTs(); } - pointsCount += metric.TimeSeries.Size(); + pointsCount += metric.TimeSeries.Size(); } - // (1) write header - TSpackHeader header; + // (1) write header + TSpackHeader header; header.Version = Version_; - header.TimePrecision = EncodeTimePrecision(TimePrecision_); - header.Compression = EncodeCompression(Compression_); - header.LabelNamesSize = static_cast<ui32>( - LabelNamesPool_.BytesSize() + LabelNamesPool_.Count()); - header.LabelValuesSize = static_cast<ui32>( - LabelValuesPool_.BytesSize() + LabelValuesPool_.Count()); - header.MetricCount = Metrics_.size(); + header.TimePrecision = EncodeTimePrecision(TimePrecision_); + header.Compression = EncodeCompression(Compression_); + header.LabelNamesSize = static_cast<ui32>( + LabelNamesPool_.BytesSize() + LabelNamesPool_.Count()); + header.LabelValuesSize = static_cast<ui32>( + LabelValuesPool_.BytesSize() + LabelValuesPool_.Count()); + header.MetricCount = Metrics_.size(); header.PointsCount = pointsCount; - Out_->Write(&header, sizeof(header)); - - // if compression enabled all below writes must go throught compressor - auto compressedOut = CompressedOutput(Out_, Compression_); - if (compressedOut) { - Out_ = compressedOut.Get(); - } - - // (2) write string pools - auto strPoolWrite = [this](TStringBuf str, ui32, ui32) { - Out_->Write(str); - Out_->Write('\0'); - }; - - LabelNamesPool_.ForEach(strPoolWrite); - LabelValuesPool_.ForEach(strPoolWrite); - - // (3) write common time - WriteTime(CommonTime_); - - // (4) write common labels' indexes + Out_->Write(&header, sizeof(header)); + + // if compression enabled all below writes must go throught compressor + auto compressedOut = CompressedOutput(Out_, Compression_); + if (compressedOut) { + Out_ = compressedOut.Get(); + } + + // (2) write string pools + auto strPoolWrite = [this](TStringBuf str, ui32, ui32) { + Out_->Write(str); + Out_->Write('\0'); + }; + + LabelNamesPool_.ForEach(strPoolWrite); + LabelValuesPool_.ForEach(strPoolWrite); + + // (3) write common time + WriteTime(CommonTime_); + + // (4) write common labels' indexes WriteLabels(CommonLabels_, nullptr); - - // (5) write metrics - // metrics count already written in header - for (TMetric& metric : Metrics_) { - // (5.1) types byte - ui8 typesByte = PackTypes(metric); - Out_->Write(&typesByte, sizeof(typesByte)); - - // TODO: implement - ui8 flagsByte = 0x00; - Out_->Write(&flagsByte, sizeof(flagsByte)); - + + // (5) write metrics + // metrics count already written in header + for (TMetric& metric : Metrics_) { + // (5.1) types byte + ui8 typesByte = PackTypes(metric); + Out_->Write(&typesByte, sizeof(typesByte)); + + // TODO: implement + ui8 flagsByte = 0x00; + Out_->Write(&flagsByte, sizeof(flagsByte)); + // v1.2 format addition — metric name if (Version_ >= SV1_02) { const auto it = FindIf(metric.Labels, [&](const auto& l) { @@ -143,53 +143,53 @@ namespace NMonitoring { WriteVarUInt32(Out_, it->Value->Index); } - // (5.2) labels + // (5.2) labels WriteLabels(metric.Labels, MetricName_); - - // (5.3) values - switch (metric.TimeSeries.Size()) { - case 0: - break; - case 1: { - const auto& point = metric.TimeSeries[0]; - if (point.GetTime() != TInstant::Zero()) { - WriteTime(point.GetTime()); - } - EMetricValueType valueType = metric.TimeSeries.GetValueType(); - WriteValue(metric.MetricType, valueType, point.GetValue()); - break; - } - default: - WriteVarUInt32(Out_, static_cast<ui32>(metric.TimeSeries.Size())); - const TMetricTimeSeries& ts = metric.TimeSeries; - EMetricType metricType = metric.MetricType; - ts.ForEach([this, metricType](TInstant time, EMetricValueType valueType, TMetricValue value) { - // workaround for GCC bug - // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=61636 - this->WriteTime(time); - this->WriteValue(metricType, valueType, value); - }); - break; - } - } - } - - // store metric type and values type in one byte - ui8 PackTypes(const TMetric& metric) { - EValueType valueType; - if (metric.TimeSeries.Empty()) { - valueType = EValueType::NONE; - } else if (metric.TimeSeries.Size() == 1) { - TInstant time = metric.TimeSeries[0].GetTime(); - valueType = (time == TInstant::Zero()) - ? EValueType::ONE_WITHOUT_TS - : EValueType::ONE_WITH_TS; - } else { - valueType = EValueType::MANY_WITH_TS; - } - return (static_cast<ui8>(metric.MetricType) << 2) | static_cast<ui8>(valueType); - } - + + // (5.3) values + switch (metric.TimeSeries.Size()) { + case 0: + break; + case 1: { + const auto& point = metric.TimeSeries[0]; + if (point.GetTime() != TInstant::Zero()) { + WriteTime(point.GetTime()); + } + EMetricValueType valueType = metric.TimeSeries.GetValueType(); + WriteValue(metric.MetricType, valueType, point.GetValue()); + break; + } + default: + WriteVarUInt32(Out_, static_cast<ui32>(metric.TimeSeries.Size())); + const TMetricTimeSeries& ts = metric.TimeSeries; + EMetricType metricType = metric.MetricType; + ts.ForEach([this, metricType](TInstant time, EMetricValueType valueType, TMetricValue value) { + // workaround for GCC bug + // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=61636 + this->WriteTime(time); + this->WriteValue(metricType, valueType, value); + }); + break; + } + } + } + + // store metric type and values type in one byte + ui8 PackTypes(const TMetric& metric) { + EValueType valueType; + if (metric.TimeSeries.Empty()) { + valueType = EValueType::NONE; + } else if (metric.TimeSeries.Size() == 1) { + TInstant time = metric.TimeSeries[0].GetTime(); + valueType = (time == TInstant::Zero()) + ? EValueType::ONE_WITHOUT_TS + : EValueType::ONE_WITH_TS; + } else { + valueType = EValueType::MANY_WITH_TS; + } + return (static_cast<ui8>(metric.MetricType) << 2) | static_cast<ui8>(valueType); + } + void WriteLabels(const TPooledLabels& labels, const TPooledStr* skipKey) { WriteVarUInt32(Out_, static_cast<ui32>(skipKey ? labels.size() - 1 : labels.size())); for (auto&& label : labels) { @@ -198,30 +198,30 @@ namespace NMonitoring { } WriteVarUInt32(Out_, label.Key->Index); WriteVarUInt32(Out_, label.Value->Index); - } - } - - void WriteValue(EMetricType metricType, EMetricValueType valueType, TMetricValue value) { - switch (metricType) { - case EMetricType::GAUGE: - WriteFixed(value.AsDouble(valueType)); - break; - - case EMetricType::IGAUGE: - WriteFixed(value.AsInt64(valueType)); - break; - - case EMetricType::COUNTER: - case EMetricType::RATE: - WriteFixed(value.AsUint64(valueType)); - break; - - case EMetricType::HIST: - case EMetricType::HIST_RATE: - WriteHistogram(*value.AsHistogram()); - break; - - case EMetricType::DSUMMARY: + } + } + + void WriteValue(EMetricType metricType, EMetricValueType valueType, TMetricValue value) { + switch (metricType) { + case EMetricType::GAUGE: + WriteFixed(value.AsDouble(valueType)); + break; + + case EMetricType::IGAUGE: + WriteFixed(value.AsInt64(valueType)); + break; + + case EMetricType::COUNTER: + case EMetricType::RATE: + WriteFixed(value.AsUint64(valueType)); + break; + + case EMetricType::HIST: + case EMetricType::HIST_RATE: + WriteHistogram(*value.AsHistogram()); + break; + + case EMetricType::DSUMMARY: WriteSummaryDouble(*value.AsSummaryDouble()); break; @@ -229,44 +229,44 @@ namespace NMonitoring { WriteLogHistogram(*value.AsLogHistogram()); break; - default: - ythrow yexception() << "unsupported metric type: " << metricType; - } - } - - void WriteTime(TInstant instant) { - switch (TimePrecision_) { - case ETimePrecision::SECONDS: { - ui32 time = static_cast<ui32>(instant.Seconds()); - Out_->Write(&time, sizeof(time)); - break; - } - case ETimePrecision::MILLIS: { - ui64 time = static_cast<ui64>(instant.MilliSeconds()); - Out_->Write(&time, sizeof(time)); - } - } - } - - template <typename T> - void WriteFixed(T value) { - Out_->Write(&value, sizeof(value)); - } - - void WriteHistogram(const IHistogramSnapshot& histogram) { - ui32 count = histogram.Count(); - WriteVarUInt32(Out_, count); - - for (ui32 i = 0; i < count; i++) { + default: + ythrow yexception() << "unsupported metric type: " << metricType; + } + } + + void WriteTime(TInstant instant) { + switch (TimePrecision_) { + case ETimePrecision::SECONDS: { + ui32 time = static_cast<ui32>(instant.Seconds()); + Out_->Write(&time, sizeof(time)); + break; + } + case ETimePrecision::MILLIS: { + ui64 time = static_cast<ui64>(instant.MilliSeconds()); + Out_->Write(&time, sizeof(time)); + } + } + } + + template <typename T> + void WriteFixed(T value) { + Out_->Write(&value, sizeof(value)); + } + + void WriteHistogram(const IHistogramSnapshot& histogram) { + ui32 count = histogram.Count(); + WriteVarUInt32(Out_, count); + + for (ui32 i = 0; i < count; i++) { double bound = histogram.UpperBound(i); - Out_->Write(&bound, sizeof(bound)); - } - for (ui32 i = 0; i < count; i++) { - ui64 value = histogram.Value(i); - Out_->Write(&value, sizeof(value)); - } - } - + Out_->Write(&bound, sizeof(bound)); + } + for (ui32 i = 0; i < count; i++) { + ui64 value = histogram.Value(i); + Out_->Write(&value, sizeof(value)); + } + } + void WriteLogHistogram(const TLogHistogramSnapshot& logHist) { WriteFixed(logHist.Base()); WriteFixed(logHist.ZerosCount()); @@ -285,26 +285,26 @@ namespace NMonitoring { WriteFixed(summary.GetLast()); } - private: - IOutputStream* Out_; - ETimePrecision TimePrecision_; - ECompression Compression_; + private: + IOutputStream* Out_; + ETimePrecision TimePrecision_; + ECompression Compression_; ESpackV1Version Version_; const TPooledStr* MetricName_; - bool Closed_ = false; - }; - - } - - IMetricEncoderPtr EncoderSpackV1( - IOutputStream* out, - ETimePrecision timePrecision, + bool Closed_ = false; + }; + + } + + IMetricEncoderPtr EncoderSpackV1( + IOutputStream* out, + ETimePrecision timePrecision, ECompression compression, - EMetricsMergingMode mergingMode + EMetricsMergingMode mergingMode ) { return MakeHolder<TEncoderSpackV1>(out, timePrecision, compression, mergingMode, SV1_01, ""); - } - + } + IMetricEncoderPtr EncoderSpackV12( IOutputStream* out, ETimePrecision timePrecision, @@ -315,4 +315,4 @@ namespace NMonitoring { Y_ENSURE(!metricNameLabel.Empty(), "metricNameLabel can't be empty"); return MakeHolder<TEncoderSpackV1>(out, timePrecision, compression, mergingMode, SV1_02, metricNameLabel); } -} +} diff --git a/library/cpp/monlib/encode/spack/spack_v1_ut.cpp b/library/cpp/monlib/encode/spack/spack_v1_ut.cpp index 0981ac90e2..fe778eb7e0 100644 --- a/library/cpp/monlib/encode/spack/spack_v1_ut.cpp +++ b/library/cpp/monlib/encode/spack/spack_v1_ut.cpp @@ -1,67 +1,67 @@ -#include "spack_v1.h" - -#include <library/cpp/monlib/encode/protobuf/protobuf.h> -#include <library/cpp/monlib/metrics/labels.h> -#include <library/cpp/monlib/metrics/metric.h> - +#include "spack_v1.h" + +#include <library/cpp/monlib/encode/protobuf/protobuf.h> +#include <library/cpp/monlib/metrics/labels.h> +#include <library/cpp/monlib/metrics/metric.h> + #include <library/cpp/testing/unittest/registar.h> - -#include <util/generic/buffer.h> -#include <util/stream/buffer.h> -#include <util/string/hex.h> - -#include <utility> - -using namespace NMonitoring; - -#define UNIT_ASSERT_BINARY_EQUALS(a, b) \ - do { \ - auto size = Y_ARRAY_SIZE(b); \ - if (Y_UNLIKELY(::memcmp(a, b, size) != 0)) { \ - auto as = HexEncode(a, size); \ - auto bs = HexEncode(b, size); \ - UNIT_FAIL_IMPL("equal assertion failed " #a " == " #b, \ - "\n actual: " << as << "\nexpected: " << bs); \ - } \ - } while (0) - -void AssertLabelEqual(const NProto::TLabel& l, TStringBuf name, TStringBuf value) { - UNIT_ASSERT_STRINGS_EQUAL(l.GetName(), name); - UNIT_ASSERT_STRINGS_EQUAL(l.GetValue(), value); -} - -void AssertPointEqual(const NProto::TPoint& p, TInstant time, double value) { - UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds()); - UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kFloat64); - UNIT_ASSERT_DOUBLES_EQUAL(p.GetFloat64(), value, std::numeric_limits<double>::epsilon()); -} - -void AssertPointEqual(const NProto::TPoint& p, TInstant time, ui64 value) { - UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds()); - UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kUint64); - UNIT_ASSERT_VALUES_EQUAL(p.GetUint64(), value); -} - -void AssertPointEqual(const NProto::TPoint& p, TInstant time, i64 value) { - UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds()); - UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kInt64); - UNIT_ASSERT_VALUES_EQUAL(p.GetInt64(), value); -} - + +#include <util/generic/buffer.h> +#include <util/stream/buffer.h> +#include <util/string/hex.h> + +#include <utility> + +using namespace NMonitoring; + +#define UNIT_ASSERT_BINARY_EQUALS(a, b) \ + do { \ + auto size = Y_ARRAY_SIZE(b); \ + if (Y_UNLIKELY(::memcmp(a, b, size) != 0)) { \ + auto as = HexEncode(a, size); \ + auto bs = HexEncode(b, size); \ + UNIT_FAIL_IMPL("equal assertion failed " #a " == " #b, \ + "\n actual: " << as << "\nexpected: " << bs); \ + } \ + } while (0) + +void AssertLabelEqual(const NProto::TLabel& l, TStringBuf name, TStringBuf value) { + UNIT_ASSERT_STRINGS_EQUAL(l.GetName(), name); + UNIT_ASSERT_STRINGS_EQUAL(l.GetValue(), value); +} + +void AssertPointEqual(const NProto::TPoint& p, TInstant time, double value) { + UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds()); + UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kFloat64); + UNIT_ASSERT_DOUBLES_EQUAL(p.GetFloat64(), value, std::numeric_limits<double>::epsilon()); +} + +void AssertPointEqual(const NProto::TPoint& p, TInstant time, ui64 value) { + UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds()); + UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kUint64); + UNIT_ASSERT_VALUES_EQUAL(p.GetUint64(), value); +} + +void AssertPointEqual(const NProto::TPoint& p, TInstant time, i64 value) { + UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds()); + UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kInt64); + UNIT_ASSERT_VALUES_EQUAL(p.GetInt64(), value); +} + Y_UNIT_TEST_SUITE(TSpackTest) { ui8 expectedHeader_v1_0[] = { - 0x53, 0x50, // magic "SP" (fixed ui16) + 0x53, 0x50, // magic "SP" (fixed ui16) // minor, major - 0x00, 0x01, // version (fixed ui16) - 0x18, 0x00, // header size (fixed ui16) - 0x00, // time precision (fixed ui8) - 0x00, // compression algorithm (fixed ui8) - 0x0d, 0x00, 0x00, 0x00, // label names size (fixed ui32) + 0x00, 0x01, // version (fixed ui16) + 0x18, 0x00, // header size (fixed ui16) + 0x00, // time precision (fixed ui8) + 0x00, // compression algorithm (fixed ui8) + 0x0d, 0x00, 0x00, 0x00, // label names size (fixed ui32) 0x40, 0x00, 0x00, 0x00, // labels values size (fixed ui32) - 0x08, 0x00, 0x00, 0x00, // metric count (fixed ui32) + 0x08, 0x00, 0x00, 0x00, // metric count (fixed ui32) 0x08, 0x00, 0x00, 0x00, // points count (fixed ui32) - }; - + }; + ui8 expectedHeader[] = { 0x53, 0x50, // magic "SP" (fixed ui16) // minor, major @@ -69,105 +69,105 @@ Y_UNIT_TEST_SUITE(TSpackTest) { 0x18, 0x00, // header size (fixed ui16) 0x00, // time precision (fixed ui8) 0x00, // compression algorithm (fixed ui8) - 0x0d, 0x00, 0x00, 0x00, // label names size (fixed ui32) + 0x0d, 0x00, 0x00, 0x00, // label names size (fixed ui32) 0x40, 0x00, 0x00, 0x00, // labels values size (fixed ui32) - 0x08, 0x00, 0x00, 0x00, // metric count (fixed ui32) + 0x08, 0x00, 0x00, 0x00, // metric count (fixed ui32) 0x08, 0x00, 0x00, 0x00, // points count (fixed ui32) }; - ui8 expectedStringPools[] = { - 0x6e, 0x61, 0x6d, 0x65, 0x00, // "name\0" - 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x00, // "project\0" + ui8 expectedStringPools[] = { + 0x6e, 0x61, 0x6d, 0x65, 0x00, // "name\0" + 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x00, // "project\0" 0x73, 0x6f, 0x6c, 0x6f, 0x6d, 0x6f, 0x6e, 0x00, // "solomon\0" 0x71, 0x31, 0x00, // "q1\0" 0x71, 0x32, 0x00, // "q2\0" 0x71, 0x33, 0x00, // "q3\0" 0x61, 0x6e, 0x73, 0x77, 0x65, 0x72, 0x00, // "answer\0" - 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, // "responseTimeMillis\0" - 0x54, 0x69, 0x6d, 0x65, 0x4d, 0x69, 0x6c, 0x6c, - 0x69, 0x73, 0x00, + 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, // "responseTimeMillis\0" + 0x54, 0x69, 0x6d, 0x65, 0x4d, 0x69, 0x6c, 0x6c, + 0x69, 0x73, 0x00, 0x62, 0x79, 0x74, 0x65, 0x73, 0x00, // "bytes\0" 0x74, 0x65, 0x6D, 0x70, 0x65, 0x72, 0x61, 0x74, // "temperature\0" 0x75, 0x72, 0x65, 0x00, 0x6d, 0x73, 0x00, // "ms\0" - }; - - ui8 expectedCommonTime[] = { - 0x00, 0x2f, 0x68, 0x59, // common time in seconds (fixed ui32) - }; - - ui8 expectedCommonLabels[] = { - 0x01, // common labels count (varint) - 0x01, // label name index (varint) + }; + + ui8 expectedCommonTime[] = { + 0x00, 0x2f, 0x68, 0x59, // common time in seconds (fixed ui32) + }; + + ui8 expectedCommonLabels[] = { + 0x01, // common labels count (varint) + 0x01, // label name index (varint) 0x00, // label value index (varint) - }; - - ui8 expectedMetric1[] = { - 0x0C, // types (RATE | NONE) (fixed ui8) - 0x00, // flags (fixed ui8) - 0x01, // metric labels count (varint) - 0x00, // label name index (varint) + }; + + ui8 expectedMetric1[] = { + 0x0C, // types (RATE | NONE) (fixed ui8) + 0x00, // flags (fixed ui8) + 0x01, // metric labels count (varint) + 0x00, // label name index (varint) 0x01, // label value index (varint) - }; - - ui8 expectedMetric2[] = { - 0x09, // types (COUNTER | ONE_WITHOUT_TS) (fixed ui8) - 0x00, // flags (fixed ui8) - 0x01, // metric labels count (varint) - 0x00, // label name index (varint) + }; + + ui8 expectedMetric2[] = { + 0x09, // types (COUNTER | ONE_WITHOUT_TS) (fixed ui8) + 0x00, // flags (fixed ui8) + 0x01, // metric labels count (varint) + 0x00, // label name index (varint) 0x02, // label value index (varint) - 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // value (fixed ui64) - }; - - ui8 expectedMetric3[] = { - 0x0a, // types (COUNTER | ONE_WITH_TS) (fixed ui8) - 0x00, // flags (fixed ui8) - 0x01, // metric labels count (varint) - 0x00, // label name index (varint) - 0x03, // label value index (varint) - 0x0b, 0x63, 0xfe, 0x59, // time in seconds (fixed ui32) - 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // value (fixed ui64) - }; - - ui8 expectedMetric4[] = { - 0x07, // types (GAUGE | MANY_WITH_TS) (fixed ui8) - 0x00, // flags (fixed ui8) - 0x01, // metric labels count (varint) - 0x00, // label name index (varint) - 0x04, // label value index (varint) - 0x02, // points count (varint) - 0x0b, 0x63, 0xfe, 0x59, // time in seconds (fixed ui32) - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x45, 0x40, // value (double IEEE754) - 0x1a, 0x63, 0xfe, 0x59, // time in seconds (fixed ui32) - 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x4d, 0x40 // value (double IEEE754) - }; - - ui8 expectedMetric5_v1_0[] = { - 0x16, // types (HIST | ONE_WITH_TS) (fixed ui8) - 0x00, // flags (fixed ui8) - 0x01, // metric labels count (varint) - 0x00, // label name index (varint) + 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // value (fixed ui64) + }; + + ui8 expectedMetric3[] = { + 0x0a, // types (COUNTER | ONE_WITH_TS) (fixed ui8) + 0x00, // flags (fixed ui8) + 0x01, // metric labels count (varint) + 0x00, // label name index (varint) + 0x03, // label value index (varint) + 0x0b, 0x63, 0xfe, 0x59, // time in seconds (fixed ui32) + 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // value (fixed ui64) + }; + + ui8 expectedMetric4[] = { + 0x07, // types (GAUGE | MANY_WITH_TS) (fixed ui8) + 0x00, // flags (fixed ui8) + 0x01, // metric labels count (varint) + 0x00, // label name index (varint) + 0x04, // label value index (varint) + 0x02, // points count (varint) + 0x0b, 0x63, 0xfe, 0x59, // time in seconds (fixed ui32) + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x45, 0x40, // value (double IEEE754) + 0x1a, 0x63, 0xfe, 0x59, // time in seconds (fixed ui32) + 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x4d, 0x40 // value (double IEEE754) + }; + + ui8 expectedMetric5_v1_0[] = { + 0x16, // types (HIST | ONE_WITH_TS) (fixed ui8) + 0x00, // flags (fixed ui8) + 0x01, // metric labels count (varint) + 0x00, // label name index (varint) 0x05, // label value index (varint) - 0x0b, 0x63, 0xfe, 0x59, // time in seconds (fixed ui32) - 0x06, // histogram buckets count (varint) + 0x0b, 0x63, 0xfe, 0x59, // time in seconds (fixed ui32) + 0x06, // histogram buckets count (varint) 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // histogram bucket bounds (array of fixed ui64) - 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, - 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // histogram bucket values - 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x53, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - }; - - ui8 expectedMetric5[] = { + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // histogram bucket values + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x53, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + }; + + ui8 expectedMetric5[] = { 0x16, // types (HIST | ONE_WITH_TS) (fixed ui8) 0x00, // flags (fixed ui8) - 0x01, // metric labels count (varint) + 0x01, // metric labels count (varint) 0x00, // label name index (varint) 0x05, // label value index (varint) 0x0b, 0x63, 0xfe, 0x59, // time in seconds (fixed ui32) @@ -186,20 +186,20 @@ Y_UNIT_TEST_SUITE(TSpackTest) { 0x53, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, }; - ui8 expectedMetric6[] = { - 0x12, // types (IGAUGE | ONE_WITH_TS) (fixed ui8) - 0x00, // flags (fixed ui8) - 0x01, // metric labels count (varint) - 0x00, // label name index (varint) + ui8 expectedMetric6[] = { + 0x12, // types (IGAUGE | ONE_WITH_TS) (fixed ui8) + 0x00, // flags (fixed ui8) + 0x01, // metric labels count (varint) + 0x00, // label name index (varint) 0x06, // label value index (varint) - 0x0b, 0x63, 0xfe, 0x59, // time in seconds (fixed ui32) - 0x39, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // value (fixed i64) - }; - - ui8 expectedMetric7[] = { + 0x0b, 0x63, 0xfe, 0x59, // time in seconds (fixed ui32) + 0x39, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // value (fixed i64) + }; + + ui8 expectedMetric7[] = { 0x1e, // types (DSUMMARY | ONE_WITH_TS) (fixed ui8) 0x00, // flags (fixed ui8) - 0x01, // metric labels count (varint) + 0x01, // metric labels count (varint) 0x00, // label name index (varint) 0x07, // label value index (varint) 0x0b, 0x63, 0xfe, 0x59, // time in seconds (fixed ui32) @@ -210,10 +210,10 @@ Y_UNIT_TEST_SUITE(TSpackTest) { 0x33, 0x33, 0x33, 0x33, 0x33, 0x33, 0xd3, 0x3f, // last (fixed double) }; - ui8 expectedMetric8[] = { + ui8 expectedMetric8[] = { 0x26, // types (LOGHIST | ONE_WITH_TS) (fixed ui8) 0x00, // flags (fixed ui8) - 0x01, // metric labels count (variant) + 0x01, // metric labels count (variant) 0x00, // label name index (variant) 0x08, // label value index (variant) 0x0b, 0x63, 0xfe, 0x59, // time in seconds (fixed ui32) @@ -227,31 +227,31 @@ Y_UNIT_TEST_SUITE(TSpackTest) { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE0, 0x3F, }; - const size_t expectedSize = - Y_ARRAY_SIZE(expectedHeader) + - Y_ARRAY_SIZE(expectedStringPools) + - Y_ARRAY_SIZE(expectedCommonTime) + - Y_ARRAY_SIZE(expectedCommonLabels) + - Y_ARRAY_SIZE(expectedMetric1) + - Y_ARRAY_SIZE(expectedMetric2) + - Y_ARRAY_SIZE(expectedMetric3) + - Y_ARRAY_SIZE(expectedMetric4) + - Y_ARRAY_SIZE(expectedMetric5) + - Y_ARRAY_SIZE(expectedMetric6) + - Y_ARRAY_SIZE(expectedMetric7) + - Y_ARRAY_SIZE(expectedMetric8); - + const size_t expectedSize = + Y_ARRAY_SIZE(expectedHeader) + + Y_ARRAY_SIZE(expectedStringPools) + + Y_ARRAY_SIZE(expectedCommonTime) + + Y_ARRAY_SIZE(expectedCommonLabels) + + Y_ARRAY_SIZE(expectedMetric1) + + Y_ARRAY_SIZE(expectedMetric2) + + Y_ARRAY_SIZE(expectedMetric3) + + Y_ARRAY_SIZE(expectedMetric4) + + Y_ARRAY_SIZE(expectedMetric5) + + Y_ARRAY_SIZE(expectedMetric6) + + Y_ARRAY_SIZE(expectedMetric7) + + Y_ARRAY_SIZE(expectedMetric8); + const TInstant now = TInstant::ParseIso8601Deprecated("2017-11-05T01:02:03Z"); - - // {1: 1, 2: 1, 4: 2, 8: 4, 16: 8, inf: 83} - IHistogramSnapshotPtr TestHistogram() { - auto h = ExponentialHistogram(6, 2); - for (i64 i = 1; i < 100; i++) { - h->Collect(i); - } - return h->Snapshot(); - } - + + // {1: 1, 2: 1, 4: 2, 8: 4, 16: 8, inf: 83} + IHistogramSnapshotPtr TestHistogram() { + auto h = ExponentialHistogram(6, 2); + for (i64 i = 1; i < 100; i++) { + h->Collect(i); + } + return h->Snapshot(); + } + TLogHistogramSnapshotPtr TestLogHistogram() { TVector buckets{0.5, 0.25, 0.25, 0.5}; return MakeIntrusive<TLogHistogramSnapshot>(1.5, 1u, 0, std::move(buckets)); @@ -262,154 +262,154 @@ Y_UNIT_TEST_SUITE(TSpackTest) { } Y_UNIT_TEST(Encode) { - TBuffer buffer; - TBufferOutput out(buffer); - auto e = EncoderSpackV1( - &out, ETimePrecision::SECONDS, ECompression::IDENTITY); - - e->OnStreamBegin(); - { // common time - e->OnCommonTime(TInstant::Seconds(1500000000)); - } - { // common labels - e->OnLabelsBegin(); - e->OnLabel("project", "solomon"); - e->OnLabelsEnd(); - } - { // metric #1 - e->OnMetricBegin(EMetricType::RATE); - { - e->OnLabelsBegin(); - e->OnLabel("name", "q1"); - e->OnLabelsEnd(); - } - e->OnMetricEnd(); - } - { // metric #2 - e->OnMetricBegin(EMetricType::COUNTER); - { - e->OnLabelsBegin(); - e->OnLabel("name", "q2"); - e->OnLabelsEnd(); - } + TBuffer buffer; + TBufferOutput out(buffer); + auto e = EncoderSpackV1( + &out, ETimePrecision::SECONDS, ECompression::IDENTITY); + + e->OnStreamBegin(); + { // common time + e->OnCommonTime(TInstant::Seconds(1500000000)); + } + { // common labels + e->OnLabelsBegin(); + e->OnLabel("project", "solomon"); + e->OnLabelsEnd(); + } + { // metric #1 + e->OnMetricBegin(EMetricType::RATE); + { + e->OnLabelsBegin(); + e->OnLabel("name", "q1"); + e->OnLabelsEnd(); + } + e->OnMetricEnd(); + } + { // metric #2 + e->OnMetricBegin(EMetricType::COUNTER); + { + e->OnLabelsBegin(); + e->OnLabel("name", "q2"); + e->OnLabelsEnd(); + } // Only the last value will be encoded e->OnUint64(TInstant::Zero(), 10); e->OnUint64(TInstant::Zero(), 13); - e->OnUint64(TInstant::Zero(), 17); - e->OnMetricEnd(); - } - { // metric #3 - e->OnMetricBegin(EMetricType::COUNTER); - { - e->OnLabelsBegin(); - e->OnLabel("name", "q3"); - e->OnLabelsEnd(); - } + e->OnUint64(TInstant::Zero(), 17); + e->OnMetricEnd(); + } + { // metric #3 + e->OnMetricBegin(EMetricType::COUNTER); + { + e->OnLabelsBegin(); + e->OnLabel("name", "q3"); + e->OnLabelsEnd(); + } e->OnUint64(now, 10); e->OnUint64(now, 13); - e->OnUint64(now, 17); - e->OnMetricEnd(); - } - { // metric #4 - e->OnMetricBegin(EMetricType::GAUGE); - { - e->OnLabelsBegin(); - e->OnLabel("name", "answer"); - e->OnLabelsEnd(); - } - e->OnDouble(now, 42); - e->OnDouble(now + TDuration::Seconds(15), 59); - e->OnMetricEnd(); - } - { // metric #5 - e->OnMetricBegin(EMetricType::HIST); - { - e->OnLabelsBegin(); - e->OnLabel("name", "responseTimeMillis"); - e->OnLabelsEnd(); - } - - auto histogram = TestHistogram(); - e->OnHistogram(now, histogram); - e->OnMetricEnd(); - } - { // metric #6 - e->OnMetricBegin(EMetricType::IGAUGE); - { - e->OnLabelsBegin(); - e->OnLabel("name", "bytes"); - e->OnLabelsEnd(); - } - e->OnInt64(now, 1337); - e->OnMetricEnd(); - } - { // metric 7 - e->OnMetricBegin(EMetricType::DSUMMARY); + e->OnUint64(now, 17); + e->OnMetricEnd(); + } + { // metric #4 + e->OnMetricBegin(EMetricType::GAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("name", "answer"); + e->OnLabelsEnd(); + } + e->OnDouble(now, 42); + e->OnDouble(now + TDuration::Seconds(15), 59); + e->OnMetricEnd(); + } + { // metric #5 + e->OnMetricBegin(EMetricType::HIST); + { + e->OnLabelsBegin(); + e->OnLabel("name", "responseTimeMillis"); + e->OnLabelsEnd(); + } + + auto histogram = TestHistogram(); + e->OnHistogram(now, histogram); + e->OnMetricEnd(); + } + { // metric #6 + e->OnMetricBegin(EMetricType::IGAUGE); { e->OnLabelsBegin(); - e->OnLabel("name", "temperature"); + e->OnLabel("name", "bytes"); + e->OnLabelsEnd(); + } + e->OnInt64(now, 1337); + e->OnMetricEnd(); + } + { // metric 7 + e->OnMetricBegin(EMetricType::DSUMMARY); + { + e->OnLabelsBegin(); + e->OnLabel("name", "temperature"); e->OnLabelsEnd(); } e->OnSummaryDouble(now, TestSummaryDouble()); - e->OnMetricEnd(); + e->OnMetricEnd(); } - { // metric 8 + { // metric 8 e->OnMetricBegin(EMetricType::LOGHIST); { e->OnLabelsBegin(); - e->OnLabel("name", "ms"); + e->OnLabel("name", "ms"); e->OnLabelsEnd(); } e->OnLogHistogram(now, TestLogHistogram()); e->OnMetricEnd(); } - e->OnStreamEnd(); - e->Close(); - + e->OnStreamEnd(); + e->Close(); + // Cout << "encoded: " << HexEncode(buffer.Data(), buffer.Size()) << Endl; // Cout << "size: " << buffer.Size() << Endl; - - UNIT_ASSERT_VALUES_EQUAL(buffer.Size(), expectedSize); - - ui8* p = reinterpret_cast<ui8*>(buffer.Data()); - UNIT_ASSERT_BINARY_EQUALS(p, expectedHeader); - p += Y_ARRAY_SIZE(expectedHeader); - - UNIT_ASSERT_BINARY_EQUALS(p, expectedStringPools); - p += Y_ARRAY_SIZE(expectedStringPools); - - UNIT_ASSERT_BINARY_EQUALS(p, expectedCommonTime); - p += Y_ARRAY_SIZE(expectedCommonTime); - - UNIT_ASSERT_BINARY_EQUALS(p, expectedCommonLabels); - p += Y_ARRAY_SIZE(expectedCommonLabels); - - UNIT_ASSERT_BINARY_EQUALS(p, expectedMetric1); - p += Y_ARRAY_SIZE(expectedMetric1); - - UNIT_ASSERT_BINARY_EQUALS(p, expectedMetric2); - p += Y_ARRAY_SIZE(expectedMetric2); - - UNIT_ASSERT_BINARY_EQUALS(p, expectedMetric3); - p += Y_ARRAY_SIZE(expectedMetric3); - - UNIT_ASSERT_BINARY_EQUALS(p, expectedMetric4); - p += Y_ARRAY_SIZE(expectedMetric4); - - UNIT_ASSERT_BINARY_EQUALS(p, expectedMetric5); - p += Y_ARRAY_SIZE(expectedMetric5); - - UNIT_ASSERT_BINARY_EQUALS(p, expectedMetric6); - p += Y_ARRAY_SIZE(expectedMetric6); - - UNIT_ASSERT_BINARY_EQUALS(p, expectedMetric7); - p += Y_ARRAY_SIZE(expectedMetric7); - - UNIT_ASSERT_BINARY_EQUALS(p, expectedMetric8); - p += Y_ARRAY_SIZE(expectedMetric8); - } - - NProto::TMultiSamplesList GetMergingMetricSamples(EMetricsMergingMode mergingMode) { + + UNIT_ASSERT_VALUES_EQUAL(buffer.Size(), expectedSize); + + ui8* p = reinterpret_cast<ui8*>(buffer.Data()); + UNIT_ASSERT_BINARY_EQUALS(p, expectedHeader); + p += Y_ARRAY_SIZE(expectedHeader); + + UNIT_ASSERT_BINARY_EQUALS(p, expectedStringPools); + p += Y_ARRAY_SIZE(expectedStringPools); + + UNIT_ASSERT_BINARY_EQUALS(p, expectedCommonTime); + p += Y_ARRAY_SIZE(expectedCommonTime); + + UNIT_ASSERT_BINARY_EQUALS(p, expectedCommonLabels); + p += Y_ARRAY_SIZE(expectedCommonLabels); + + UNIT_ASSERT_BINARY_EQUALS(p, expectedMetric1); + p += Y_ARRAY_SIZE(expectedMetric1); + + UNIT_ASSERT_BINARY_EQUALS(p, expectedMetric2); + p += Y_ARRAY_SIZE(expectedMetric2); + + UNIT_ASSERT_BINARY_EQUALS(p, expectedMetric3); + p += Y_ARRAY_SIZE(expectedMetric3); + + UNIT_ASSERT_BINARY_EQUALS(p, expectedMetric4); + p += Y_ARRAY_SIZE(expectedMetric4); + + UNIT_ASSERT_BINARY_EQUALS(p, expectedMetric5); + p += Y_ARRAY_SIZE(expectedMetric5); + + UNIT_ASSERT_BINARY_EQUALS(p, expectedMetric6); + p += Y_ARRAY_SIZE(expectedMetric6); + + UNIT_ASSERT_BINARY_EQUALS(p, expectedMetric7); + p += Y_ARRAY_SIZE(expectedMetric7); + + UNIT_ASSERT_BINARY_EQUALS(p, expectedMetric8); + p += Y_ARRAY_SIZE(expectedMetric8); + } + + NProto::TMultiSamplesList GetMergingMetricSamples(EMetricsMergingMode mergingMode) { TBuffer buffer; TBufferOutput out(buffer); @@ -422,29 +422,29 @@ Y_UNIT_TEST_SUITE(TSpackTest) { e->OnStreamBegin(); for (size_t i = 0; i != 3; ++i) { - e->OnMetricBegin(EMetricType::COUNTER); + e->OnMetricBegin(EMetricType::COUNTER); { e->OnLabelsBegin(); - e->OnLabel("name", "my_counter"); + e->OnLabel("name", "my_counter"); e->OnLabelsEnd(); } e->OnUint64(TInstant::Zero() + TDuration::Seconds(i), i + 1); - e->OnMetricEnd(); + e->OnMetricEnd(); } e->OnStreamEnd(); e->Close(); NProto::TMultiSamplesList samples; - IMetricEncoderPtr eProto = EncoderProtobuf(&samples); + IMetricEncoderPtr eProto = EncoderProtobuf(&samples); TBufferInput in(buffer); DecodeSpackV1(&in, eProto.Get()); return samples; } - Y_UNIT_TEST(SpackEncoderMergesMetrics) { + Y_UNIT_TEST(SpackEncoderMergesMetrics) { { - NProto::TMultiSamplesList samples = GetMergingMetricSamples(EMetricsMergingMode::DEFAULT); + NProto::TMultiSamplesList samples = GetMergingMetricSamples(EMetricsMergingMode::DEFAULT); UNIT_ASSERT_EQUAL(samples.SamplesSize(), 3); UNIT_ASSERT_EQUAL(samples.GetSamples(0).GetPoints(0).GetUint64(), 1); @@ -453,7 +453,7 @@ Y_UNIT_TEST_SUITE(TSpackTest) { } { - NProto::TMultiSamplesList samples = GetMergingMetricSamples(EMetricsMergingMode::MERGE_METRICS); + NProto::TMultiSamplesList samples = GetMergingMetricSamples(EMetricsMergingMode::MERGE_METRICS); UNIT_ASSERT_EQUAL(samples.SamplesSize(), 1); @@ -465,29 +465,29 @@ Y_UNIT_TEST_SUITE(TSpackTest) { } void DecodeDataToSamples(NProto::TMultiSamplesList & samples, ui16 version) { - IMetricEncoderPtr e = EncoderProtobuf(&samples); - + IMetricEncoderPtr e = EncoderProtobuf(&samples); + TBuffer data(expectedSize); if (SV1_00 == version) { // v1.0 data.Append(reinterpret_cast<char*>(expectedHeader_v1_0), Y_ARRAY_SIZE(expectedHeader_v1_0)); } else { - data.Append(reinterpret_cast<char*>(expectedHeader), Y_ARRAY_SIZE(expectedHeader)); + data.Append(reinterpret_cast<char*>(expectedHeader), Y_ARRAY_SIZE(expectedHeader)); } data.Append(reinterpret_cast<char*>(expectedStringPools), Y_ARRAY_SIZE(expectedStringPools)); data.Append(reinterpret_cast<char*>(expectedCommonTime), Y_ARRAY_SIZE(expectedCommonTime)); data.Append(reinterpret_cast<char*>(expectedCommonLabels), Y_ARRAY_SIZE(expectedCommonLabels)); - data.Append(reinterpret_cast<char*>(expectedMetric1), Y_ARRAY_SIZE(expectedMetric1)); - data.Append(reinterpret_cast<char*>(expectedMetric2), Y_ARRAY_SIZE(expectedMetric2)); - data.Append(reinterpret_cast<char*>(expectedMetric3), Y_ARRAY_SIZE(expectedMetric3)); - data.Append(reinterpret_cast<char*>(expectedMetric4), Y_ARRAY_SIZE(expectedMetric4)); + data.Append(reinterpret_cast<char*>(expectedMetric1), Y_ARRAY_SIZE(expectedMetric1)); + data.Append(reinterpret_cast<char*>(expectedMetric2), Y_ARRAY_SIZE(expectedMetric2)); + data.Append(reinterpret_cast<char*>(expectedMetric3), Y_ARRAY_SIZE(expectedMetric3)); + data.Append(reinterpret_cast<char*>(expectedMetric4), Y_ARRAY_SIZE(expectedMetric4)); if (SV1_00 == version) { // v1.0 - data.Append(reinterpret_cast<char*>(expectedMetric5_v1_0), Y_ARRAY_SIZE(expectedMetric5_v1_0)); + data.Append(reinterpret_cast<char*>(expectedMetric5_v1_0), Y_ARRAY_SIZE(expectedMetric5_v1_0)); } else { - data.Append(reinterpret_cast<char*>(expectedMetric5), Y_ARRAY_SIZE(expectedMetric5)); - } - data.Append(reinterpret_cast<char*>(expectedMetric6), Y_ARRAY_SIZE(expectedMetric6)); - data.Append(reinterpret_cast<char*>(expectedMetric7), Y_ARRAY_SIZE(expectedMetric7)); - data.Append(reinterpret_cast<char*>(expectedMetric8), Y_ARRAY_SIZE(expectedMetric8)); + data.Append(reinterpret_cast<char*>(expectedMetric5), Y_ARRAY_SIZE(expectedMetric5)); + } + data.Append(reinterpret_cast<char*>(expectedMetric6), Y_ARRAY_SIZE(expectedMetric6)); + data.Append(reinterpret_cast<char*>(expectedMetric7), Y_ARRAY_SIZE(expectedMetric7)); + data.Append(reinterpret_cast<char*>(expectedMetric8), Y_ARRAY_SIZE(expectedMetric8)); TBufferInput in(data); DecodeSpackV1(&in, e.Get()); } @@ -502,85 +502,85 @@ Y_UNIT_TEST_SUITE(TSpackTest) { NProto::TMultiSamplesList samples; DecodeDataToSamples(samples); - UNIT_ASSERT_VALUES_EQUAL( - TInstant::MilliSeconds(samples.GetCommonTime()), - TInstant::Seconds(1500000000)); - - UNIT_ASSERT_VALUES_EQUAL(samples.CommonLabelsSize(), 1); - AssertLabelEqual(samples.GetCommonLabels(0), "project", "solomon"); - + UNIT_ASSERT_VALUES_EQUAL( + TInstant::MilliSeconds(samples.GetCommonTime()), + TInstant::Seconds(1500000000)); + + UNIT_ASSERT_VALUES_EQUAL(samples.CommonLabelsSize(), 1); + AssertLabelEqual(samples.GetCommonLabels(0), "project", "solomon"); + UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 8); - { - const NProto::TMultiSample& s = samples.GetSamples(0); - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::RATE); - UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); - AssertLabelEqual(s.GetLabels(0), "name", "q1"); - } - { - const NProto::TMultiSample& s = samples.GetSamples(1); - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::COUNTER); - UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); - AssertLabelEqual(s.GetLabels(0), "name", "q2"); - - UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); - AssertPointEqual(s.GetPoints(0), TInstant::Zero(), ui64(17)); - } - { - const NProto::TMultiSample& s = samples.GetSamples(2); - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::COUNTER); - UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); - AssertLabelEqual(s.GetLabels(0), "name", "q3"); - - UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); - AssertPointEqual(s.GetPoints(0), now, ui64(17)); - } - { - const NProto::TMultiSample& s = samples.GetSamples(3); - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); - UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); - AssertLabelEqual(s.GetLabels(0), "name", "answer"); - - UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 2); - AssertPointEqual(s.GetPoints(0), now, double(42)); - AssertPointEqual(s.GetPoints(1), now + TDuration::Seconds(15), double(59)); - } - { - const NProto::TMultiSample& s = samples.GetSamples(4); - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::HISTOGRAM); - UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); - AssertLabelEqual(s.GetLabels(0), "name", "responseTimeMillis"); - - UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); - - const NProto::TPoint& p = s.GetPoints(0); - UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), now.MilliSeconds()); - UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kHistogram); - - auto histogram = TestHistogram(); - - const NProto::THistogram& pointHistogram = p.GetHistogram(); - UNIT_ASSERT_VALUES_EQUAL(pointHistogram.BoundsSize(), histogram->Count()); - UNIT_ASSERT_VALUES_EQUAL(pointHistogram.ValuesSize(), histogram->Count()); - - for (size_t i = 0; i < pointHistogram.BoundsSize(); i++) { - UNIT_ASSERT_DOUBLES_EQUAL(pointHistogram.GetBounds(i), histogram->UpperBound(i), Min<double>()); - UNIT_ASSERT_VALUES_EQUAL(pointHistogram.GetValues(i), histogram->Value(i)); - } - } - { - const NProto::TMultiSample& s = samples.GetSamples(5); - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::IGAUGE); - UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); - AssertLabelEqual(s.GetLabels(0), "name", "bytes"); - - UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); - AssertPointEqual(s.GetPoints(0), now, i64(1337)); - } + { + const NProto::TMultiSample& s = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::RATE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "name", "q1"); + } + { + const NProto::TMultiSample& s = samples.GetSamples(1); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::COUNTER); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "name", "q2"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + AssertPointEqual(s.GetPoints(0), TInstant::Zero(), ui64(17)); + } + { + const NProto::TMultiSample& s = samples.GetSamples(2); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::COUNTER); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "name", "q3"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + AssertPointEqual(s.GetPoints(0), now, ui64(17)); + } + { + const NProto::TMultiSample& s = samples.GetSamples(3); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "name", "answer"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 2); + AssertPointEqual(s.GetPoints(0), now, double(42)); + AssertPointEqual(s.GetPoints(1), now + TDuration::Seconds(15), double(59)); + } + { + const NProto::TMultiSample& s = samples.GetSamples(4); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::HISTOGRAM); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "name", "responseTimeMillis"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + + const NProto::TPoint& p = s.GetPoints(0); + UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), now.MilliSeconds()); + UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kHistogram); + + auto histogram = TestHistogram(); + + const NProto::THistogram& pointHistogram = p.GetHistogram(); + UNIT_ASSERT_VALUES_EQUAL(pointHistogram.BoundsSize(), histogram->Count()); + UNIT_ASSERT_VALUES_EQUAL(pointHistogram.ValuesSize(), histogram->Count()); + + for (size_t i = 0; i < pointHistogram.BoundsSize(); i++) { + UNIT_ASSERT_DOUBLES_EQUAL(pointHistogram.GetBounds(i), histogram->UpperBound(i), Min<double>()); + UNIT_ASSERT_VALUES_EQUAL(pointHistogram.GetValues(i), histogram->Value(i)); + } + } + { + const NProto::TMultiSample& s = samples.GetSamples(5); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::IGAUGE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "name", "bytes"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + AssertPointEqual(s.GetPoints(0), now, i64(1337)); + } { const NProto::TMultiSample& s = samples.GetSamples(6); - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::DSUMMARY); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::DSUMMARY); UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); - AssertLabelEqual(s.GetLabels(0), "name", "temperature"); + AssertLabelEqual(s.GetLabels(0), "name", "temperature"); UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); @@ -602,7 +602,7 @@ Y_UNIT_TEST_SUITE(TSpackTest) { const NProto::TMultiSample& s = samples.GetSamples(7); UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::LOGHISTOGRAM); UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); - AssertLabelEqual(s.GetLabels(0), "name", "ms"); + AssertLabelEqual(s.GetLabels(0), "name", "ms"); UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); @@ -621,69 +621,69 @@ Y_UNIT_TEST_SUITE(TSpackTest) { UNIT_ASSERT_VALUES_EQUAL(expected->Bucket(i), actual.GetBuckets(i)); } } - } - - void TestCompression(ECompression alg) { - TBuffer buffer; - { - TBufferOutput out(buffer); - auto e = EncoderSpackV1(&out, ETimePrecision::MILLIS, alg); - e->OnStreamBegin(); - { - e->OnMetricBegin(EMetricType::GAUGE); - { - e->OnLabelsBegin(); - e->OnLabel("name", "answer"); - e->OnLabelsEnd(); - } - e->OnDouble(now, 42); - e->OnMetricEnd(); - } - e->OnStreamEnd(); - e->Close(); - } - - auto* header = reinterpret_cast<const TSpackHeader*>(buffer.Data()); - UNIT_ASSERT_EQUAL(DecodeCompression(header->Compression), alg); - - NProto::TMultiSamplesList samples; - { - IMetricEncoderPtr e = EncoderProtobuf(&samples); - TBufferInput in(buffer); - DecodeSpackV1(&in, e.Get()); - } - - UNIT_ASSERT_VALUES_EQUAL( - TInstant::MilliSeconds(samples.GetCommonTime()), - TInstant::Zero()); - - UNIT_ASSERT_VALUES_EQUAL(samples.CommonLabelsSize(), 0); - - UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 1); - { - const NProto::TMultiSample& s = samples.GetSamples(0); - UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); - UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); - AssertLabelEqual(s.GetLabels(0), "name", "answer"); - AssertPointEqual(s.GetPoints(0), now, 42.0); - } - } - + } + + void TestCompression(ECompression alg) { + TBuffer buffer; + { + TBufferOutput out(buffer); + auto e = EncoderSpackV1(&out, ETimePrecision::MILLIS, alg); + e->OnStreamBegin(); + { + e->OnMetricBegin(EMetricType::GAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("name", "answer"); + e->OnLabelsEnd(); + } + e->OnDouble(now, 42); + e->OnMetricEnd(); + } + e->OnStreamEnd(); + e->Close(); + } + + auto* header = reinterpret_cast<const TSpackHeader*>(buffer.Data()); + UNIT_ASSERT_EQUAL(DecodeCompression(header->Compression), alg); + + NProto::TMultiSamplesList samples; + { + IMetricEncoderPtr e = EncoderProtobuf(&samples); + TBufferInput in(buffer); + DecodeSpackV1(&in, e.Get()); + } + + UNIT_ASSERT_VALUES_EQUAL( + TInstant::MilliSeconds(samples.GetCommonTime()), + TInstant::Zero()); + + UNIT_ASSERT_VALUES_EQUAL(samples.CommonLabelsSize(), 0); + + UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 1); + { + const NProto::TMultiSample& s = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "name", "answer"); + AssertPointEqual(s.GetPoints(0), now, 42.0); + } + } + Y_UNIT_TEST(CompressionIdentity) { - TestCompression(ECompression::IDENTITY); - } - + TestCompression(ECompression::IDENTITY); + } + Y_UNIT_TEST(CompressionZlib) { - TestCompression(ECompression::ZLIB); - } - + TestCompression(ECompression::ZLIB); + } + Y_UNIT_TEST(CompressionZstd) { - TestCompression(ECompression::ZSTD); - } - + TestCompression(ECompression::ZSTD); + } + Y_UNIT_TEST(CompressionLz4) { - TestCompression(ECompression::LZ4); - } + TestCompression(ECompression::LZ4); + } Y_UNIT_TEST(Decode_v1_0_histograms) { // Check that histogram bounds decoded from different versions are the same @@ -842,4 +842,4 @@ Y_UNIT_TEST_SUITE(TSpackTest) { yexception, "metric name label 's' not found, all metric labels '{m=v, project=solomon}'"); } -} +} diff --git a/library/cpp/monlib/encode/spack/ut/ya.make b/library/cpp/monlib/encode/spack/ut/ya.make index 6f27ddf43c..980bf54667 100644 --- a/library/cpp/monlib/encode/spack/ut/ya.make +++ b/library/cpp/monlib/encode/spack/ut/ya.make @@ -1,16 +1,16 @@ -UNITTEST_FOR(library/cpp/monlib/encode/spack) - +UNITTEST_FOR(library/cpp/monlib/encode/spack) + OWNER( g:solomon jamel ) - -SRCS( - spack_v1_ut.cpp -) - -PEERDIR( - library/cpp/monlib/encode/protobuf -) - -END() + +SRCS( + spack_v1_ut.cpp +) + +PEERDIR( + library/cpp/monlib/encode/protobuf +) + +END() diff --git a/library/cpp/monlib/encode/spack/varint.cpp b/library/cpp/monlib/encode/spack/varint.cpp index dabfaa9afe..051cf17380 100644 --- a/library/cpp/monlib/encode/spack/varint.cpp +++ b/library/cpp/monlib/encode/spack/varint.cpp @@ -1,79 +1,79 @@ -#include "varint.h" - -#include <util/generic/yexception.h> -#include <util/stream/input.h> -#include <util/stream/output.h> - -namespace NMonitoring { - ui32 WriteVarUInt32(IOutputStream* output, ui32 value) { - bool stop = false; - ui32 written = 0; - while (!stop) { - ui8 byte = static_cast<ui8>(value | 0x80); - value >>= 7; - if (value == 0) { - stop = true; - byte &= 0x7F; - } - output->Write(byte); - written++; - } - return written; - } - - ui32 ReadVarUInt32(IInputStream* input) { +#include "varint.h" + +#include <util/generic/yexception.h> +#include <util/stream/input.h> +#include <util/stream/output.h> + +namespace NMonitoring { + ui32 WriteVarUInt32(IOutputStream* output, ui32 value) { + bool stop = false; + ui32 written = 0; + while (!stop) { + ui8 byte = static_cast<ui8>(value | 0x80); + value >>= 7; + if (value == 0) { + stop = true; + byte &= 0x7F; + } + output->Write(byte); + written++; + } + return written; + } + + ui32 ReadVarUInt32(IInputStream* input) { ui32 value = 0; switch (TryReadVarUInt32(input, &value)) { case EReadResult::OK: return value; case EReadResult::ERR_OVERFLOW: - ythrow yexception() << "the data is too long to read ui32"; + ythrow yexception() << "the data is too long to read ui32"; case EReadResult::ERR_UNEXPECTED_EOF: - ythrow yexception() << "the data unexpectedly ended"; + ythrow yexception() << "the data unexpectedly ended"; default: ythrow yexception() << "unknown error while reading varint"; } - } - - size_t ReadVarUInt32(const ui8* buf, size_t len, ui32* result) { - size_t count = 0; - ui32 value = 0; - - ui8 byte = 0; - do { - if (7 * count > 8 * sizeof(ui32)) { - ythrow yexception() << "the data is too long to read ui32"; - } - if (count == len) { - ythrow yexception() << "the data unexpectedly ended"; - } - byte = buf[count]; - value |= (static_cast<ui32>(byte & 0x7F)) << (7 * count); - ++count; - } while (byte & 0x80); - - *result = value; - return count; - } - + } + + size_t ReadVarUInt32(const ui8* buf, size_t len, ui32* result) { + size_t count = 0; + ui32 value = 0; + + ui8 byte = 0; + do { + if (7 * count > 8 * sizeof(ui32)) { + ythrow yexception() << "the data is too long to read ui32"; + } + if (count == len) { + ythrow yexception() << "the data unexpectedly ended"; + } + byte = buf[count]; + value |= (static_cast<ui32>(byte & 0x7F)) << (7 * count); + ++count; + } while (byte & 0x80); + + *result = value; + return count; + } + EReadResult TryReadVarUInt32(IInputStream* input, ui32* value) { - size_t count = 0; - ui32 result = 0; - - ui8 byte = 0; - do { - if (7 * count > 8 * sizeof(ui32)) { + size_t count = 0; + ui32 result = 0; + + ui8 byte = 0; + do { + if (7 * count > 8 * sizeof(ui32)) { return EReadResult::ERR_OVERFLOW; - } - if (input->Read(&byte, 1) != 1) { + } + if (input->Read(&byte, 1) != 1) { return EReadResult::ERR_UNEXPECTED_EOF; - } - result |= (static_cast<ui32>(byte & 0x7F)) << (7 * count); - ++count; - } while (byte & 0x80); - - *value = result; + } + result |= (static_cast<ui32>(byte & 0x7F)) << (7 * count); + ++count; + } while (byte & 0x80); + + *value = result; return EReadResult::OK; - } - -} + } + +} diff --git a/library/cpp/monlib/encode/spack/varint.h b/library/cpp/monlib/encode/spack/varint.h index 7bce244b20..7ac522dd6c 100644 --- a/library/cpp/monlib/encode/spack/varint.h +++ b/library/cpp/monlib/encode/spack/varint.h @@ -1,16 +1,16 @@ -#pragma once - -#include <util/system/types.h> - -class IInputStream; -class IOutputStream; - -namespace NMonitoring { - ui32 WriteVarUInt32(IOutputStream* output, ui32 value); - - ui32 ReadVarUInt32(IInputStream* input); - size_t ReadVarUInt32(const ui8* buf, size_t len, ui32* result); - +#pragma once + +#include <util/system/types.h> + +class IInputStream; +class IOutputStream; + +namespace NMonitoring { + ui32 WriteVarUInt32(IOutputStream* output, ui32 value); + + ui32 ReadVarUInt32(IInputStream* input); + size_t ReadVarUInt32(const ui8* buf, size_t len, ui32* result); + enum class EReadResult { OK, ERR_OVERFLOW, @@ -20,4 +20,4 @@ namespace NMonitoring { [[nodiscard]] EReadResult TryReadVarUInt32(IInputStream* input, ui32* value); -} +} diff --git a/library/cpp/monlib/encode/spack/ya.make b/library/cpp/monlib/encode/spack/ya.make index 1b1a27e7e2..78d3061291 100644 --- a/library/cpp/monlib/encode/spack/ya.make +++ b/library/cpp/monlib/encode/spack/ya.make @@ -1,25 +1,25 @@ -LIBRARY() - +LIBRARY() + OWNER( g:solomon jamel ) - -SRCS( - spack_v1_decoder.cpp - spack_v1_encoder.cpp - varint.cpp - compression.cpp -) - -PEERDIR( + +SRCS( + spack_v1_decoder.cpp + spack_v1_encoder.cpp + varint.cpp + compression.cpp +) + +PEERDIR( library/cpp/monlib/encode/buffered library/cpp/monlib/exception - contrib/libs/lz4 + contrib/libs/lz4 contrib/libs/xxhash - contrib/libs/zlib - contrib/libs/zstd -) - -END() + contrib/libs/zlib + contrib/libs/zstd +) + +END() diff --git a/library/cpp/monlib/encode/text/text.h b/library/cpp/monlib/encode/text/text.h index 8efc2b18a2..6b2be3937b 100644 --- a/library/cpp/monlib/encode/text/text.h +++ b/library/cpp/monlib/encode/text/text.h @@ -1,9 +1,9 @@ -#pragma once - -#include <library/cpp/monlib/encode/encoder.h> - -class IOutputStream; - -namespace NMonitoring { - IMetricEncoderPtr EncoderText(IOutputStream* out, bool humanReadableTs = true); -} +#pragma once + +#include <library/cpp/monlib/encode/encoder.h> + +class IOutputStream; + +namespace NMonitoring { + IMetricEncoderPtr EncoderText(IOutputStream* out, bool humanReadableTs = true); +} diff --git a/library/cpp/monlib/encode/text/text_encoder.cpp b/library/cpp/monlib/encode/text/text_encoder.cpp index cfddcb2405..10336261f0 100644 --- a/library/cpp/monlib/encode/text/text_encoder.cpp +++ b/library/cpp/monlib/encode/text/text_encoder.cpp @@ -1,148 +1,148 @@ -#include "text.h" - -#include <library/cpp/monlib/encode/encoder_state.h> -#include <library/cpp/monlib/metrics/labels.h> -#include <library/cpp/monlib/metrics/metric_value.h> - -#include <util/datetime/base.h> -#include <util/stream/format.h> - -namespace NMonitoring { - namespace { - class TEncoderText final: public IMetricEncoder { - public: - TEncoderText(IOutputStream* out, bool humanReadableTs) - : Out_(out) - , HumanReadableTs_(humanReadableTs) - { - } - - private: - void OnStreamBegin() override { +#include "text.h" + +#include <library/cpp/monlib/encode/encoder_state.h> +#include <library/cpp/monlib/metrics/labels.h> +#include <library/cpp/monlib/metrics/metric_value.h> + +#include <util/datetime/base.h> +#include <util/stream/format.h> + +namespace NMonitoring { + namespace { + class TEncoderText final: public IMetricEncoder { + public: + TEncoderText(IOutputStream* out, bool humanReadableTs) + : Out_(out) + , HumanReadableTs_(humanReadableTs) + { + } + + private: + void OnStreamBegin() override { State_.Expect(TEncoderState::EState::ROOT); - } - - void OnStreamEnd() override { + } + + void OnStreamEnd() override { State_.Expect(TEncoderState::EState::ROOT); - } - - void OnCommonTime(TInstant time) override { + } + + void OnCommonTime(TInstant time) override { State_.Expect(TEncoderState::EState::ROOT); - CommonTime_ = time; - if (time != TInstant::Zero()) { + CommonTime_ = time; + if (time != TInstant::Zero()) { Out_->Write(TStringBuf("common time: ")); - WriteTime(time); - Out_->Write('\n'); - } - } - - void OnMetricBegin(EMetricType type) override { - State_.Switch(TEncoderState::EState::ROOT, TEncoderState::EState::METRIC); - ClearLastMetricState(); - MetricType_ = type; - } - - void OnMetricEnd() override { - State_.Switch(TEncoderState::EState::METRIC, TEncoderState::EState::ROOT); - WriteMetric(); - } - - void OnLabelsBegin() override { - if (State_ == TEncoderState::EState::METRIC) { - State_ = TEncoderState::EState::METRIC_LABELS; + WriteTime(time); + Out_->Write('\n'); + } + } + + void OnMetricBegin(EMetricType type) override { + State_.Switch(TEncoderState::EState::ROOT, TEncoderState::EState::METRIC); + ClearLastMetricState(); + MetricType_ = type; + } + + void OnMetricEnd() override { + State_.Switch(TEncoderState::EState::METRIC, TEncoderState::EState::ROOT); + WriteMetric(); + } + + void OnLabelsBegin() override { + if (State_ == TEncoderState::EState::METRIC) { + State_ = TEncoderState::EState::METRIC_LABELS; } else if (State_ == TEncoderState::EState::ROOT) { State_ = TEncoderState::EState::COMMON_LABELS; - } else { - State_.ThrowInvalid("expected METRIC or ROOT"); - } - } - - void OnLabelsEnd() override { - if (State_ == TEncoderState::EState::METRIC_LABELS) { - State_ = TEncoderState::EState::METRIC; + } else { + State_.ThrowInvalid("expected METRIC or ROOT"); + } + } + + void OnLabelsEnd() override { + if (State_ == TEncoderState::EState::METRIC_LABELS) { + State_ = TEncoderState::EState::METRIC; } else if (State_ == TEncoderState::EState::COMMON_LABELS) { State_ = TEncoderState::EState::ROOT; Out_->Write(TStringBuf("common labels: ")); - WriteLabels(); - Out_->Write('\n'); - } else { - State_.ThrowInvalid("expected LABELS or COMMON_LABELS"); - } - } - - void OnLabel(TStringBuf name, TStringBuf value) override { - Labels_.Add(name, value); - } - - void OnDouble(TInstant time, double value) override { - State_.Expect(TEncoderState::EState::METRIC); - TimeSeries_.Add(time, value); - } - - void OnInt64(TInstant time, i64 value) override { - State_.Expect(TEncoderState::EState::METRIC); - TimeSeries_.Add(time, value); - } - - void OnUint64(TInstant time, ui64 value) override { - State_.Expect(TEncoderState::EState::METRIC); - TimeSeries_.Add(time, value); - } - - void OnHistogram(TInstant time, IHistogramSnapshotPtr snapshot) override { - State_.Expect(TEncoderState::EState::METRIC); - TimeSeries_.Add(time, snapshot.Get()); - } - + WriteLabels(); + Out_->Write('\n'); + } else { + State_.ThrowInvalid("expected LABELS or COMMON_LABELS"); + } + } + + void OnLabel(TStringBuf name, TStringBuf value) override { + Labels_.Add(name, value); + } + + void OnDouble(TInstant time, double value) override { + State_.Expect(TEncoderState::EState::METRIC); + TimeSeries_.Add(time, value); + } + + void OnInt64(TInstant time, i64 value) override { + State_.Expect(TEncoderState::EState::METRIC); + TimeSeries_.Add(time, value); + } + + void OnUint64(TInstant time, ui64 value) override { + State_.Expect(TEncoderState::EState::METRIC); + TimeSeries_.Add(time, value); + } + + void OnHistogram(TInstant time, IHistogramSnapshotPtr snapshot) override { + State_.Expect(TEncoderState::EState::METRIC); + TimeSeries_.Add(time, snapshot.Get()); + } + void OnSummaryDouble(TInstant time, ISummaryDoubleSnapshotPtr snapshot) override { - State_.Expect(TEncoderState::EState::METRIC); + State_.Expect(TEncoderState::EState::METRIC); TimeSeries_.Add(time, snapshot.Get()); } void OnLogHistogram(TInstant ts, TLogHistogramSnapshotPtr snapshot) override { - State_.Expect(TEncoderState::EState::METRIC); + State_.Expect(TEncoderState::EState::METRIC); TimeSeries_.Add(ts, snapshot.Get()); } - void Close() override { - } - - void WriteTime(TInstant time) { - if (HumanReadableTs_) { - char buf[64]; - auto len = FormatDate8601(buf, sizeof(buf), time.TimeT()); - Out_->Write(buf, len); - } else { - (*Out_) << time.Seconds(); - } - } - - void WriteValue(EMetricValueType type, TMetricValue value) { - switch (type) { - case EMetricValueType::DOUBLE: - (*Out_) << value.AsDouble(); - break; - case EMetricValueType::INT64: - (*Out_) << value.AsInt64(); - break; - case EMetricValueType::UINT64: - (*Out_) << value.AsUint64(); - break; - case EMetricValueType::HISTOGRAM: - (*Out_) << *value.AsHistogram(); - break; - case EMetricValueType::SUMMARY: + void Close() override { + } + + void WriteTime(TInstant time) { + if (HumanReadableTs_) { + char buf[64]; + auto len = FormatDate8601(buf, sizeof(buf), time.TimeT()); + Out_->Write(buf, len); + } else { + (*Out_) << time.Seconds(); + } + } + + void WriteValue(EMetricValueType type, TMetricValue value) { + switch (type) { + case EMetricValueType::DOUBLE: + (*Out_) << value.AsDouble(); + break; + case EMetricValueType::INT64: + (*Out_) << value.AsInt64(); + break; + case EMetricValueType::UINT64: + (*Out_) << value.AsUint64(); + break; + case EMetricValueType::HISTOGRAM: + (*Out_) << *value.AsHistogram(); + break; + case EMetricValueType::SUMMARY: (*Out_) << *value.AsSummaryDouble(); break; case EMetricValueType::LOGHISTOGRAM: (*Out_) << *value.AsLogHistogram(); break; - case EMetricValueType::UNKNOWN: - ythrow yexception() << "unknown metric value type"; - } - } - - void WriteLabels() { + case EMetricValueType::UNKNOWN: + ythrow yexception() << "unknown metric value type"; + } + } + + void WriteLabels() { auto& out = *Out_; const auto size = Labels_.Size(); size_t i = 0; @@ -154,73 +154,73 @@ namespace NMonitoring { ++i; if (i < size) { out << TStringBuf(", "); - } + } }; out << '}'; - } - - void WriteMetric() { - // (1) type - TStringBuf typeStr = MetricTypeToStr(MetricType_); - (*Out_) << LeftPad(typeStr, MaxMetricTypeNameLength) << ' '; - - // (2) name and labels + } + + void WriteMetric() { + // (1) type + TStringBuf typeStr = MetricTypeToStr(MetricType_); + (*Out_) << LeftPad(typeStr, MaxMetricTypeNameLength) << ' '; + + // (2) name and labels auto name = Labels_.Extract(TStringBuf("sensor")); - if (name) { - if (name->Value().find(' ') != TString::npos) { - (*Out_) << '"' << name->Value() << '"'; - } else { - (*Out_) << name->Value(); - } - } - WriteLabels(); - - // (3) values - if (!TimeSeries_.Empty()) { - TimeSeries_.SortByTs(); + if (name) { + if (name->Value().find(' ') != TString::npos) { + (*Out_) << '"' << name->Value() << '"'; + } else { + (*Out_) << name->Value(); + } + } + WriteLabels(); + + // (3) values + if (!TimeSeries_.Empty()) { + TimeSeries_.SortByTs(); Out_->Write(TStringBuf(" [")); - for (size_t i = 0; i < TimeSeries_.Size(); i++) { - if (i > 0) { + for (size_t i = 0; i < TimeSeries_.Size(); i++) { + if (i > 0) { Out_->Write(TStringBuf(", ")); - } - - const auto& point = TimeSeries_[i]; - if (point.GetTime() == CommonTime_ || point.GetTime() == TInstant::Zero()) { - WriteValue(TimeSeries_.GetValueType(), point.GetValue()); - } else { - Out_->Write('('); - WriteTime(point.GetTime()); + } + + const auto& point = TimeSeries_[i]; + if (point.GetTime() == CommonTime_ || point.GetTime() == TInstant::Zero()) { + WriteValue(TimeSeries_.GetValueType(), point.GetValue()); + } else { + Out_->Write('('); + WriteTime(point.GetTime()); Out_->Write(TStringBuf(", ")); - WriteValue(TimeSeries_.GetValueType(), point.GetValue()); - Out_->Write(')'); - } - } - Out_->Write(']'); - } - Out_->Write('\n'); - } - - void ClearLastMetricState() { - MetricType_ = EMetricType::UNKNOWN; - Labels_.Clear(); - TimeSeries_.Clear(); - } - - private: - TEncoderState State_; - IOutputStream* Out_; - bool HumanReadableTs_; - TInstant CommonTime_ = TInstant::Zero(); - EMetricType MetricType_ = EMetricType::UNKNOWN; - TLabels Labels_; - TMetricTimeSeries TimeSeries_; - }; - - } - - IMetricEncoderPtr EncoderText(IOutputStream* out, bool humanReadableTs) { + WriteValue(TimeSeries_.GetValueType(), point.GetValue()); + Out_->Write(')'); + } + } + Out_->Write(']'); + } + Out_->Write('\n'); + } + + void ClearLastMetricState() { + MetricType_ = EMetricType::UNKNOWN; + Labels_.Clear(); + TimeSeries_.Clear(); + } + + private: + TEncoderState State_; + IOutputStream* Out_; + bool HumanReadableTs_; + TInstant CommonTime_ = TInstant::Zero(); + EMetricType MetricType_ = EMetricType::UNKNOWN; + TLabels Labels_; + TMetricTimeSeries TimeSeries_; + }; + + } + + IMetricEncoderPtr EncoderText(IOutputStream* out, bool humanReadableTs) { return MakeHolder<TEncoderText>(out, humanReadableTs); - } - -} + } + +} diff --git a/library/cpp/monlib/encode/text/text_encoder_ut.cpp b/library/cpp/monlib/encode/text/text_encoder_ut.cpp index b817b38285..554b6f5fa9 100644 --- a/library/cpp/monlib/encode/text/text_encoder_ut.cpp +++ b/library/cpp/monlib/encode/text/text_encoder_ut.cpp @@ -1,283 +1,283 @@ -#include "text.h" - -#include <library/cpp/monlib/metrics/histogram_collector.h> - +#include "text.h" + +#include <library/cpp/monlib/metrics/histogram_collector.h> + #include <library/cpp/testing/unittest/registar.h> - -using namespace NMonitoring; - + +using namespace NMonitoring; + Y_UNIT_TEST_SUITE(TTextText) { - template <typename TFunc> - TString EncodeToString(bool humanReadableTs, TFunc fn) { - TStringStream ss; - IMetricEncoderPtr encoder = EncoderText(&ss, humanReadableTs); - fn(encoder.Get()); - return ss.Str(); - } - + template <typename TFunc> + TString EncodeToString(bool humanReadableTs, TFunc fn) { + TStringStream ss; + IMetricEncoderPtr encoder = EncoderText(&ss, humanReadableTs); + fn(encoder.Get()); + return ss.Str(); + } + Y_UNIT_TEST(Empty) { - auto result = EncodeToString(true, [](IMetricEncoder* e) { - e->OnStreamBegin(); - e->OnStreamEnd(); - }); - UNIT_ASSERT_STRINGS_EQUAL(result, ""); - } - + auto result = EncodeToString(true, [](IMetricEncoder* e) { + e->OnStreamBegin(); + e->OnStreamEnd(); + }); + UNIT_ASSERT_STRINGS_EQUAL(result, ""); + } + Y_UNIT_TEST(CommonPart) { - auto result = EncodeToString(true, [](IMetricEncoder* e) { - e->OnStreamBegin(); + auto result = EncodeToString(true, [](IMetricEncoder* e) { + e->OnStreamBegin(); e->OnCommonTime(TInstant::ParseIso8601Deprecated("2017-01-02T03:04:05.006Z")); - { - e->OnLabelsBegin(); - e->OnLabel("project", "solomon"); - e->OnLabel("cluster", "man"); - e->OnLabel("service", "stockpile"); - e->OnLabelsEnd(); - } - e->OnStreamEnd(); - }); - UNIT_ASSERT_STRINGS_EQUAL(result, - "common time: 2017-01-02T03:04:05Z\n" - "common labels: {project='solomon', cluster='man', service='stockpile'}\n"); - } - + { + e->OnLabelsBegin(); + e->OnLabel("project", "solomon"); + e->OnLabel("cluster", "man"); + e->OnLabel("service", "stockpile"); + e->OnLabelsEnd(); + } + e->OnStreamEnd(); + }); + UNIT_ASSERT_STRINGS_EQUAL(result, + "common time: 2017-01-02T03:04:05Z\n" + "common labels: {project='solomon', cluster='man', service='stockpile'}\n"); + } + Y_UNIT_TEST(Gauges) { - auto result = EncodeToString(true, [](IMetricEncoder* e) { - e->OnStreamBegin(); - { // no values - e->OnMetricBegin(EMetricType::GAUGE); - { - e->OnLabelsBegin(); - e->OnLabel("sensor", "cpuUsage"); - e->OnLabelsEnd(); - } - e->OnMetricEnd(); - } - { // one value no ts - e->OnMetricBegin(EMetricType::GAUGE); - { - e->OnLabelsBegin(); - e->OnLabel("sensor", "diskUsage"); - e->OnLabel("disk", "sda1"); - e->OnLabelsEnd(); - } - e->OnDouble(TInstant::Zero(), 1000); - e->OnMetricEnd(); - } - { // one value with ts - e->OnMetricBegin(EMetricType::GAUGE); - { - e->OnLabelsBegin(); - e->OnLabel("sensor", "memoryUsage"); - e->OnLabel("host", "solomon-man-00"); - e->OnLabel("dc", "man"); - e->OnLabelsEnd(); - } + auto result = EncodeToString(true, [](IMetricEncoder* e) { + e->OnStreamBegin(); + { // no values + e->OnMetricBegin(EMetricType::GAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "cpuUsage"); + e->OnLabelsEnd(); + } + e->OnMetricEnd(); + } + { // one value no ts + e->OnMetricBegin(EMetricType::GAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "diskUsage"); + e->OnLabel("disk", "sda1"); + e->OnLabelsEnd(); + } + e->OnDouble(TInstant::Zero(), 1000); + e->OnMetricEnd(); + } + { // one value with ts + e->OnMetricBegin(EMetricType::GAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "memoryUsage"); + e->OnLabel("host", "solomon-man-00"); + e->OnLabel("dc", "man"); + e->OnLabelsEnd(); + } + e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 1000); + e->OnMetricEnd(); + } + { // many values + e->OnMetricBegin(EMetricType::GAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "bytesRx"); + e->OnLabel("host", "solomon-sas-01"); + e->OnLabel("dc", "sas"); + e->OnLabelsEnd(); + } + e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 2); + e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:05Z"), 4); + e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:10Z"), 8); + e->OnMetricEnd(); + } + e->OnStreamEnd(); + }); + UNIT_ASSERT_STRINGS_EQUAL(result, + " GAUGE cpuUsage{}\n" + " GAUGE diskUsage{disk='sda1'} [1000]\n" + " GAUGE memoryUsage{host='solomon-man-00', dc='man'} [(2017-12-02T12:00:00Z, 1000)]\n" + " GAUGE bytesRx{host='solomon-sas-01', dc='sas'} [(2017-12-02T12:00:00Z, 2), (2017-12-02T12:00:05Z, 4), (2017-12-02T12:00:10Z, 8)]\n"); + } + + Y_UNIT_TEST(IntGauges) { + auto result = EncodeToString(true, [](IMetricEncoder* e) { + e->OnStreamBegin(); + { // no values + e->OnMetricBegin(EMetricType::IGAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "cpuUsage"); + e->OnLabelsEnd(); + } + e->OnMetricEnd(); + } + { // one value no ts + e->OnMetricBegin(EMetricType::IGAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "diskUsage"); + e->OnLabel("disk", "sda1"); + e->OnLabelsEnd(); + } + e->OnDouble(TInstant::Zero(), 1000); + e->OnMetricEnd(); + } + { // one value with ts + e->OnMetricBegin(EMetricType::IGAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "memoryUsage"); + e->OnLabel("host", "solomon-man-00"); + e->OnLabel("dc", "man"); + e->OnLabelsEnd(); + } e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 1000); - e->OnMetricEnd(); - } - { // many values - e->OnMetricBegin(EMetricType::GAUGE); - { - e->OnLabelsBegin(); - e->OnLabel("sensor", "bytesRx"); - e->OnLabel("host", "solomon-sas-01"); - e->OnLabel("dc", "sas"); - e->OnLabelsEnd(); - } + e->OnMetricEnd(); + } + { // many values + e->OnMetricBegin(EMetricType::IGAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "bytesRx"); + e->OnLabel("host", "solomon-sas-01"); + e->OnLabel("dc", "sas"); + e->OnLabelsEnd(); + } e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 2); e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:05Z"), 4); e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:10Z"), 8); - e->OnMetricEnd(); - } - e->OnStreamEnd(); - }); - UNIT_ASSERT_STRINGS_EQUAL(result, - " GAUGE cpuUsage{}\n" - " GAUGE diskUsage{disk='sda1'} [1000]\n" - " GAUGE memoryUsage{host='solomon-man-00', dc='man'} [(2017-12-02T12:00:00Z, 1000)]\n" - " GAUGE bytesRx{host='solomon-sas-01', dc='sas'} [(2017-12-02T12:00:00Z, 2), (2017-12-02T12:00:05Z, 4), (2017-12-02T12:00:10Z, 8)]\n"); - } - - Y_UNIT_TEST(IntGauges) { - auto result = EncodeToString(true, [](IMetricEncoder* e) { - e->OnStreamBegin(); - { // no values - e->OnMetricBegin(EMetricType::IGAUGE); - { - e->OnLabelsBegin(); - e->OnLabel("sensor", "cpuUsage"); - e->OnLabelsEnd(); - } - e->OnMetricEnd(); - } - { // one value no ts - e->OnMetricBegin(EMetricType::IGAUGE); - { - e->OnLabelsBegin(); - e->OnLabel("sensor", "diskUsage"); - e->OnLabel("disk", "sda1"); - e->OnLabelsEnd(); - } - e->OnDouble(TInstant::Zero(), 1000); - e->OnMetricEnd(); - } - { // one value with ts - e->OnMetricBegin(EMetricType::IGAUGE); - { - e->OnLabelsBegin(); - e->OnLabel("sensor", "memoryUsage"); - e->OnLabel("host", "solomon-man-00"); - e->OnLabel("dc", "man"); - e->OnLabelsEnd(); - } - e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 1000); - e->OnMetricEnd(); - } - { // many values - e->OnMetricBegin(EMetricType::IGAUGE); - { - e->OnLabelsBegin(); - e->OnLabel("sensor", "bytesRx"); - e->OnLabel("host", "solomon-sas-01"); - e->OnLabel("dc", "sas"); - e->OnLabelsEnd(); - } - e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 2); - e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:05Z"), 4); - e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:10Z"), 8); - e->OnMetricEnd(); - } - e->OnStreamEnd(); - }); - UNIT_ASSERT_STRINGS_EQUAL(result, - " IGAUGE cpuUsage{}\n" - " IGAUGE diskUsage{disk='sda1'} [1000]\n" - " IGAUGE memoryUsage{host='solomon-man-00', dc='man'} [(2017-12-02T12:00:00Z, 1000)]\n" - " IGAUGE bytesRx{host='solomon-sas-01', dc='sas'} [(2017-12-02T12:00:00Z, 2), (2017-12-02T12:00:05Z, 4), (2017-12-02T12:00:10Z, 8)]\n"); - } - + e->OnMetricEnd(); + } + e->OnStreamEnd(); + }); + UNIT_ASSERT_STRINGS_EQUAL(result, + " IGAUGE cpuUsage{}\n" + " IGAUGE diskUsage{disk='sda1'} [1000]\n" + " IGAUGE memoryUsage{host='solomon-man-00', dc='man'} [(2017-12-02T12:00:00Z, 1000)]\n" + " IGAUGE bytesRx{host='solomon-sas-01', dc='sas'} [(2017-12-02T12:00:00Z, 2), (2017-12-02T12:00:05Z, 4), (2017-12-02T12:00:10Z, 8)]\n"); + } + Y_UNIT_TEST(Counters) { - auto doEncode = [](IMetricEncoder* e) { - e->OnStreamBegin(); - { // no values - e->OnMetricBegin(EMetricType::COUNTER); - { - e->OnLabelsBegin(); - e->OnLabel("sensor", "cpuUsage"); - e->OnLabelsEnd(); - } - e->OnMetricEnd(); - } - { // one value no ts - e->OnMetricBegin(EMetricType::COUNTER); - { - e->OnLabelsBegin(); - e->OnLabel("sensor", "diskUsage"); - e->OnLabel("disk", "sda1"); - e->OnLabelsEnd(); - } - e->OnUint64(TInstant::Zero(), 1000); - e->OnMetricEnd(); - } - { // one value with ts - e->OnMetricBegin(EMetricType::COUNTER); - { - e->OnLabelsBegin(); - e->OnLabel("sensor", "memoryUsage"); - e->OnLabel("host", "solomon-man-00"); - e->OnLabel("dc", "man"); - e->OnLabelsEnd(); - } + auto doEncode = [](IMetricEncoder* e) { + e->OnStreamBegin(); + { // no values + e->OnMetricBegin(EMetricType::COUNTER); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "cpuUsage"); + e->OnLabelsEnd(); + } + e->OnMetricEnd(); + } + { // one value no ts + e->OnMetricBegin(EMetricType::COUNTER); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "diskUsage"); + e->OnLabel("disk", "sda1"); + e->OnLabelsEnd(); + } + e->OnUint64(TInstant::Zero(), 1000); + e->OnMetricEnd(); + } + { // one value with ts + e->OnMetricBegin(EMetricType::COUNTER); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "memoryUsage"); + e->OnLabel("host", "solomon-man-00"); + e->OnLabel("dc", "man"); + e->OnLabelsEnd(); + } e->OnUint64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 1000); - e->OnMetricEnd(); - } - { // many values - e->OnMetricBegin(EMetricType::COUNTER); - { - e->OnLabelsBegin(); - e->OnLabel("sensor", "bytesRx"); - e->OnLabel("host", "solomon-sas-01"); - e->OnLabel("dc", "sas"); - e->OnLabelsEnd(); - } + e->OnMetricEnd(); + } + { // many values + e->OnMetricBegin(EMetricType::COUNTER); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "bytesRx"); + e->OnLabel("host", "solomon-sas-01"); + e->OnLabel("dc", "sas"); + e->OnLabelsEnd(); + } e->OnUint64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 2); e->OnUint64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:05Z"), 4); e->OnUint64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:10Z"), 8); - e->OnMetricEnd(); - } - e->OnStreamEnd(); - }; - - auto result1 = EncodeToString(false, doEncode); - UNIT_ASSERT_STRINGS_EQUAL(result1, - " COUNTER cpuUsage{}\n" - " COUNTER diskUsage{disk='sda1'} [1000]\n" - " COUNTER memoryUsage{host='solomon-man-00', dc='man'} [(1512216000, 1000)]\n" - " COUNTER bytesRx{host='solomon-sas-01', dc='sas'} [(1512216000, 2), (1512216005, 4), (1512216010, 8)]\n"); - - auto result2 = EncodeToString(true, doEncode); - UNIT_ASSERT_STRINGS_EQUAL(result2, - " COUNTER cpuUsage{}\n" - " COUNTER diskUsage{disk='sda1'} [1000]\n" - " COUNTER memoryUsage{host='solomon-man-00', dc='man'} [(2017-12-02T12:00:00Z, 1000)]\n" - " COUNTER bytesRx{host='solomon-sas-01', dc='sas'} [(2017-12-02T12:00:00Z, 2), (2017-12-02T12:00:05Z, 4), (2017-12-02T12:00:10Z, 8)]\n"); - } - - Y_UNIT_TEST(Histograms) { - auto h = ExplicitHistogram({1, 2, 3, 4, 5}); - h->Collect(3); - h->Collect(5, 7); - h->Collect(13); - auto s = h->Snapshot(); - - TString result = EncodeToString(true, [s](IMetricEncoder* e) { - e->OnStreamBegin(); - { - e->OnMetricBegin(EMetricType::HIST); - { - e->OnLabelsBegin(); - e->OnLabel("sensor", "readTimeMillis"); - e->OnLabelsEnd(); - } - e->OnHistogram(TInstant::Zero(), s); - e->OnMetricEnd(); - } - { - e->OnMetricBegin(EMetricType::HIST_RATE); - { - e->OnLabelsBegin(); - e->OnLabel("sensor", "writeTimeMillis"); - e->OnLabelsEnd(); - } - e->OnHistogram(TInstant::Zero(), s); - e->OnMetricEnd(); - } - e->OnStreamEnd(); - }); - - UNIT_ASSERT_STRINGS_EQUAL(result, - " HIST readTimeMillis{} [{1: 0, 2: 0, 3: 1, 4: 0, 5: 7, inf: 1}]\n" - "HIST_RATE writeTimeMillis{} [{1: 0, 2: 0, 3: 1, 4: 0, 5: 7, inf: 1}]\n"); - } + e->OnMetricEnd(); + } + e->OnStreamEnd(); + }; + + auto result1 = EncodeToString(false, doEncode); + UNIT_ASSERT_STRINGS_EQUAL(result1, + " COUNTER cpuUsage{}\n" + " COUNTER diskUsage{disk='sda1'} [1000]\n" + " COUNTER memoryUsage{host='solomon-man-00', dc='man'} [(1512216000, 1000)]\n" + " COUNTER bytesRx{host='solomon-sas-01', dc='sas'} [(1512216000, 2), (1512216005, 4), (1512216010, 8)]\n"); + + auto result2 = EncodeToString(true, doEncode); + UNIT_ASSERT_STRINGS_EQUAL(result2, + " COUNTER cpuUsage{}\n" + " COUNTER diskUsage{disk='sda1'} [1000]\n" + " COUNTER memoryUsage{host='solomon-man-00', dc='man'} [(2017-12-02T12:00:00Z, 1000)]\n" + " COUNTER bytesRx{host='solomon-sas-01', dc='sas'} [(2017-12-02T12:00:00Z, 2), (2017-12-02T12:00:05Z, 4), (2017-12-02T12:00:10Z, 8)]\n"); + } + + Y_UNIT_TEST(Histograms) { + auto h = ExplicitHistogram({1, 2, 3, 4, 5}); + h->Collect(3); + h->Collect(5, 7); + h->Collect(13); + auto s = h->Snapshot(); + + TString result = EncodeToString(true, [s](IMetricEncoder* e) { + e->OnStreamBegin(); + { + e->OnMetricBegin(EMetricType::HIST); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "readTimeMillis"); + e->OnLabelsEnd(); + } + e->OnHistogram(TInstant::Zero(), s); + e->OnMetricEnd(); + } + { + e->OnMetricBegin(EMetricType::HIST_RATE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "writeTimeMillis"); + e->OnLabelsEnd(); + } + e->OnHistogram(TInstant::Zero(), s); + e->OnMetricEnd(); + } + e->OnStreamEnd(); + }); + + UNIT_ASSERT_STRINGS_EQUAL(result, + " HIST readTimeMillis{} [{1: 0, 2: 0, 3: 1, 4: 0, 5: 7, inf: 1}]\n" + "HIST_RATE writeTimeMillis{} [{1: 0, 2: 0, 3: 1, 4: 0, 5: 7, inf: 1}]\n"); + } Y_UNIT_TEST(Summary) { auto s = MakeIntrusive<TSummaryDoubleSnapshot>(10.1, -0.45, 0.478, 0.3, 30u); - TString result = EncodeToString(true, [s](IMetricEncoder* e) { + TString result = EncodeToString(true, [s](IMetricEncoder* e) { e->OnStreamBegin(); { - e->OnMetricBegin(EMetricType::DSUMMARY); + e->OnMetricBegin(EMetricType::DSUMMARY); { e->OnLabelsBegin(); e->OnLabel("sensor", "temperature"); e->OnLabelsEnd(); } e->OnSummaryDouble(TInstant::Zero(), s); - e->OnMetricEnd(); + e->OnMetricEnd(); } e->OnStreamEnd(); }); UNIT_ASSERT_STRINGS_EQUAL(result, " DSUMMARY temperature{} [{sum: 10.1, min: -0.45, max: 0.478, last: 0.3, count: 30}]\n"); } -} +} diff --git a/library/cpp/monlib/encode/text/ut/ya.make b/library/cpp/monlib/encode/text/ut/ya.make index 3d507800e0..df23a252d1 100644 --- a/library/cpp/monlib/encode/text/ut/ya.make +++ b/library/cpp/monlib/encode/text/ut/ya.make @@ -1,12 +1,12 @@ -UNITTEST_FOR(library/cpp/monlib/encode/text) - +UNITTEST_FOR(library/cpp/monlib/encode/text) + OWNER( g:solomon jamel ) - -SRCS( - text_encoder_ut.cpp -) - -END() + +SRCS( + text_encoder_ut.cpp +) + +END() diff --git a/library/cpp/monlib/encode/text/ya.make b/library/cpp/monlib/encode/text/ya.make index 85ae8043ec..d296c78c1b 100644 --- a/library/cpp/monlib/encode/text/ya.make +++ b/library/cpp/monlib/encode/text/ya.make @@ -1,16 +1,16 @@ -LIBRARY() - +LIBRARY() + OWNER( g:solomon jamel ) - -SRCS( - text_encoder.cpp -) - -PEERDIR( - library/cpp/monlib/encode -) - -END() + +SRCS( + text_encoder.cpp +) + +PEERDIR( + library/cpp/monlib/encode +) + +END() diff --git a/library/cpp/monlib/encode/unistat/unistat.h b/library/cpp/monlib/encode/unistat/unistat.h index c550dc3b6a..1c43b7fa1b 100644 --- a/library/cpp/monlib/encode/unistat/unistat.h +++ b/library/cpp/monlib/encode/unistat/unistat.h @@ -6,8 +6,8 @@ namespace NMonitoring { /// Decodes unistat-style metrics /// https://wiki.yandex-team.ru/golovan/stat-handle - void DecodeUnistat(TStringBuf data, class IMetricConsumer* c, TInstant ts = TInstant::Zero()); + void DecodeUnistat(TStringBuf data, class IMetricConsumer* c, TInstant ts = TInstant::Zero()); /// Assumes consumer's stream is open by the caller - void DecodeUnistatToStream(TStringBuf data, class IMetricConsumer* c, TInstant = TInstant::Zero()); + void DecodeUnistatToStream(TStringBuf data, class IMetricConsumer* c, TInstant = TInstant::Zero()); } diff --git a/library/cpp/monlib/encode/unistat/unistat_decoder.cpp b/library/cpp/monlib/encode/unistat/unistat_decoder.cpp index e97ca884da..b2344b0905 100644 --- a/library/cpp/monlib/encode/unistat/unistat_decoder.cpp +++ b/library/cpp/monlib/encode/unistat/unistat_decoder.cpp @@ -1,10 +1,10 @@ #include "unistat.h" -#include <library/cpp/monlib/metrics/histogram_collector.h> -#include <library/cpp/monlib/metrics/labels.h> -#include <library/cpp/monlib/metrics/metric_type.h> -#include <library/cpp/monlib/metrics/metric_value.h> -#include <library/cpp/monlib/metrics/metric_consumer.h> +#include <library/cpp/monlib/metrics/histogram_collector.h> +#include <library/cpp/monlib/metrics/labels.h> +#include <library/cpp/monlib/metrics/metric_type.h> +#include <library/cpp/monlib/metrics/metric_value.h> +#include <library/cpp/monlib/metrics/metric_consumer.h> #include <library/cpp/json/json_reader.h> @@ -86,7 +86,7 @@ namespace NMonitoring { class TDecoderUnistat { private: public: - explicit TDecoderUnistat(IMetricConsumer* consumer, IInputStream* is, TInstant ts) + explicit TDecoderUnistat(IMetricConsumer* consumer, IInputStream* is, TInstant ts) : Consumer_{consumer} , Timestamp_{ts} { ReadJsonTree(is, &Json_, /* throw */ true); @@ -120,19 +120,19 @@ namespace NMonitoring { private: void OnScalar(const TJsonValue& jsonValue) { if (MetricContext_.IsDeriv) { - MetricContext_.Type = EMetricType::RATE; - MetricContext_.Value = TMetricValue{ExtractUi64(jsonValue)}; + MetricContext_.Type = EMetricType::RATE; + MetricContext_.Value = TMetricValue{ExtractUi64(jsonValue)}; } else { - MetricContext_.Type = EMetricType::GAUGE; - MetricContext_.Value = TMetricValue{ExtractDouble(jsonValue)}; + MetricContext_.Type = EMetricType::GAUGE; + MetricContext_.Value = TMetricValue{ExtractDouble(jsonValue)}; } } void OnHistogram(const TJsonValue& jsonHist) { if (MetricContext_.IsDeriv) { - MetricContext_.Type = EMetricType::HIST_RATE; + MetricContext_.Type = EMetricType::HIST_RATE; } else { - MetricContext_.Type = EMetricType::HIST; + MetricContext_.Type = EMetricType::HIST; } auto histogramBuilder = THistogramBuilder(); @@ -147,7 +147,7 @@ namespace NMonitoring { } MetricContext_.Histogram = histogramBuilder.Finalize(); - MetricContext_.Value = TMetricValue{MetricContext_.Histogram.Get()}; + MetricContext_.Value = TMetricValue{MetricContext_.Histogram.Get()}; } bool IsDeriv(TStringBuf name) { @@ -190,7 +190,7 @@ namespace NMonitoring { private: void WriteValue() { - Consumer_->OnMetricBegin(MetricContext_.Type); + Consumer_->OnMetricBegin(MetricContext_.Type); Consumer_->OnLabelsBegin(); Consumer_->OnLabel("sensor", TString{MetricContext_.Name}); @@ -200,37 +200,37 @@ namespace NMonitoring { Consumer_->OnLabelsEnd(); - switch (MetricContext_.Type) { - case EMetricType::GAUGE: + switch (MetricContext_.Type) { + case EMetricType::GAUGE: Consumer_->OnDouble(Timestamp_, MetricContext_.Value.AsDouble()); break; - case EMetricType::RATE: + case EMetricType::RATE: Consumer_->OnUint64(Timestamp_, MetricContext_.Value.AsUint64()); break; - case EMetricType::HIST: - case EMetricType::HIST_RATE: + case EMetricType::HIST: + case EMetricType::HIST_RATE: Consumer_->OnHistogram(Timestamp_, MetricContext_.Value.AsHistogram()); break; case EMetricType::LOGHIST: - case EMetricType::DSUMMARY: - case EMetricType::IGAUGE: - case EMetricType::COUNTER: - case EMetricType::UNKNOWN: - ythrow yexception() << "Unexpected metric type: " << MetricContext_.Type; + case EMetricType::DSUMMARY: + case EMetricType::IGAUGE: + case EMetricType::COUNTER: + case EMetricType::UNKNOWN: + ythrow yexception() << "Unexpected metric type: " << MetricContext_.Type; } - Consumer_->OnMetricEnd(); + Consumer_->OnMetricEnd(); } private: - IMetricConsumer* Consumer_; + IMetricConsumer* Consumer_; NJson::TJsonValue Json_; TInstant Timestamp_; struct { TStringBuf Name; - EMetricType Type{EMetricType::UNKNOWN}; - TMetricValue Value; + EMetricType Type{EMetricType::UNKNOWN}; + TMetricValue Value; bool IsDeriv{false}; TLabels Labels; IHistogramSnapshotPtr Histogram; @@ -239,13 +239,13 @@ namespace NMonitoring { } - void DecodeUnistat(TStringBuf data, IMetricConsumer* c, TInstant ts) { + void DecodeUnistat(TStringBuf data, IMetricConsumer* c, TInstant ts) { c->OnStreamBegin(); DecodeUnistatToStream(data, c, ts); c->OnStreamEnd(); } - void DecodeUnistatToStream(TStringBuf data, IMetricConsumer* c, TInstant ts) { + void DecodeUnistatToStream(TStringBuf data, IMetricConsumer* c, TInstant ts) { TMemoryInput in{data.data(), data.size()}; TDecoderUnistat decoder(c, &in, ts); decoder.Decode(); diff --git a/library/cpp/monlib/encode/unistat/unistat_ut.cpp b/library/cpp/monlib/encode/unistat/unistat_ut.cpp index df71da3b47..dbbc238bf3 100644 --- a/library/cpp/monlib/encode/unistat/unistat_ut.cpp +++ b/library/cpp/monlib/encode/unistat/unistat_ut.cpp @@ -1,7 +1,7 @@ #include "unistat.h" -#include <library/cpp/monlib/encode/protobuf/protobuf.h> -#include <library/cpp/monlib/metrics/labels.h> +#include <library/cpp/monlib/encode/protobuf/protobuf.h> +#include <library/cpp/monlib/metrics/labels.h> #include <library/cpp/testing/unittest/registar.h> @@ -18,7 +18,7 @@ Y_UNIT_TEST_SUITE(TUnistatDecoderTest) { UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 1); auto sample = samples.GetSamples(0); - UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::GAUGE); UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1); UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1); @@ -83,7 +83,7 @@ Y_UNIT_TEST_SUITE(TUnistatDecoderTest) { DecodeUnistat(input, encoder.Get()); auto sample = samples.GetSamples(0); - UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::HIST_RATE); + UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::HIST_RATE); UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1); UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1); @@ -114,7 +114,7 @@ Y_UNIT_TEST_SUITE(TUnistatDecoderTest) { DecodeUnistat(input, encoder.Get()); auto sample = samples.GetSamples(0); - UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::HISTOGRAM); + UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::HISTOGRAM); UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1); UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1); } @@ -159,7 +159,7 @@ Y_UNIT_TEST_SUITE(TUnistatDecoderTest) { UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 2); auto sample = samples.GetSamples(0); - UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::GAUGE); UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1); UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1); @@ -170,7 +170,7 @@ Y_UNIT_TEST_SUITE(TUnistatDecoderTest) { UNIT_ASSERT_VALUES_EQUAL(label.GetValue(), "something_axxx"); sample = samples.GetSamples(1); - UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::RATE); + UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::RATE); UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1); UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1); @@ -190,7 +190,7 @@ Y_UNIT_TEST_SUITE(TUnistatDecoderTest) { UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 1); auto sample = samples.GetSamples(0); - UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::RATE); + UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::RATE); UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1); UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1); @@ -210,7 +210,7 @@ Y_UNIT_TEST_SUITE(TUnistatDecoderTest) { UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 1); auto sample = samples.GetSamples(0); - UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::GAUGE); UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1); UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1); diff --git a/library/cpp/monlib/encode/unistat/ut/ya.make b/library/cpp/monlib/encode/unistat/ut/ya.make index d9b2b51658..a652139f45 100644 --- a/library/cpp/monlib/encode/unistat/ut/ya.make +++ b/library/cpp/monlib/encode/unistat/ut/ya.make @@ -1,4 +1,4 @@ -UNITTEST_FOR(library/cpp/monlib/encode/unistat) +UNITTEST_FOR(library/cpp/monlib/encode/unistat) OWNER( msherbakov @@ -10,7 +10,7 @@ SRCS( ) PEERDIR( - library/cpp/monlib/encode/protobuf + library/cpp/monlib/encode/protobuf ) END() diff --git a/library/cpp/monlib/encode/unistat/ya.make b/library/cpp/monlib/encode/unistat/ya.make index ae3c1c40da..4ac2edadf4 100644 --- a/library/cpp/monlib/encode/unistat/ya.make +++ b/library/cpp/monlib/encode/unistat/ya.make @@ -8,7 +8,7 @@ OWNER( PEERDIR( contrib/libs/re2 library/cpp/json - library/cpp/monlib/metrics + library/cpp/monlib/metrics ) SRCS( diff --git a/library/cpp/monlib/encode/ut/ya.make b/library/cpp/monlib/encode/ut/ya.make index b83f62f2a2..1990386d76 100644 --- a/library/cpp/monlib/encode/ut/ya.make +++ b/library/cpp/monlib/encode/ut/ya.make @@ -1,12 +1,12 @@ -UNITTEST_FOR(library/cpp/monlib/encode) - +UNITTEST_FOR(library/cpp/monlib/encode) + OWNER( jamel g:solomon ) - -SRCS( - format_ut.cpp -) - -END() + +SRCS( + format_ut.cpp +) + +END() diff --git a/library/cpp/monlib/encode/ya.make b/library/cpp/monlib/encode/ya.make index a0dae3582f..d1bb09f07b 100644 --- a/library/cpp/monlib/encode/ya.make +++ b/library/cpp/monlib/encode/ya.make @@ -1,20 +1,20 @@ -LIBRARY() - +LIBRARY() + OWNER( g:solomon jamel ) - -SRCS( - encoder.cpp - encoder_state.cpp - format.cpp -) - -PEERDIR( - library/cpp/monlib/metrics -) - + +SRCS( + encoder.cpp + encoder_state.cpp + format.cpp +) + +PEERDIR( + library/cpp/monlib/metrics +) + GENERATE_ENUM_SERIALIZATION_WITH_HEADER(encoder_state_enum.h) -END() +END() |