diff options
author | Devtools Arcadia <arcadia-devtools@yandex-team.ru> | 2022-02-07 18:08:42 +0300 |
---|---|---|
committer | Devtools Arcadia <arcadia-devtools@mous.vla.yp-c.yandex.net> | 2022-02-07 18:08:42 +0300 |
commit | 1110808a9d39d4b808aef724c861a2e1a38d2a69 (patch) | |
tree | e26c9fed0de5d9873cce7e00bc214573dc2195b7 /library/cpp/monlib/encode | |
download | ydb-1110808a9d39d4b808aef724c861a2e1a38d2a69.tar.gz |
intermediate changes
ref:cde9a383711a11544ce7e107a78147fb96cc4029
Diffstat (limited to 'library/cpp/monlib/encode')
100 files changed, 12575 insertions, 0 deletions
diff --git a/library/cpp/monlib/encode/buffered/buffered_encoder_base.cpp b/library/cpp/monlib/encode/buffered/buffered_encoder_base.cpp new file mode 100644 index 0000000000..87c832d642 --- /dev/null +++ b/library/cpp/monlib/encode/buffered/buffered_encoder_base.cpp @@ -0,0 +1,170 @@ +#include "buffered_encoder_base.h" + +#include <util/string/join.h> +#include <util/string/builder.h> + +namespace NMonitoring { + +void TBufferedEncoderBase::OnStreamBegin() { + State_.Expect(TEncoderState::EState::ROOT); +} + +void TBufferedEncoderBase::OnStreamEnd() { + State_.Expect(TEncoderState::EState::ROOT); +} + +void TBufferedEncoderBase::OnCommonTime(TInstant time) { + State_.Expect(TEncoderState::EState::ROOT); + CommonTime_ = time; +} + +void TBufferedEncoderBase::OnMetricBegin(EMetricType type) { + State_.Switch(TEncoderState::EState::ROOT, TEncoderState::EState::METRIC); + Metrics_.emplace_back(); + Metrics_.back().MetricType = type; +} + +void TBufferedEncoderBase::OnMetricEnd() { + State_.Switch(TEncoderState::EState::METRIC, TEncoderState::EState::ROOT); + + switch (MetricsMergingMode_) { + case EMetricsMergingMode::MERGE_METRICS: { + auto& metric = Metrics_.back(); + Sort(metric.Labels, [] (const TPooledLabel& lhs, const TPooledLabel& rhs) { + return std::tie(lhs.Key, lhs.Value) < std::tie(rhs.Key, rhs.Value); + }); + + auto it = MetricMap_.find(metric.Labels); + if (it == std::end(MetricMap_)) { + MetricMap_.emplace(metric.Labels, Metrics_.size() - 1); + } else { + auto& existing = Metrics_[it->second].TimeSeries; + + Y_ENSURE(existing.GetValueType() == metric.TimeSeries.GetValueType(), + "Time series point type mismatch: expected " << existing.GetValueType() + << " but found " << metric.TimeSeries.GetValueType() + << ", labels '" << FormatLabels(metric.Labels) << "'"); + + existing.CopyFrom(metric.TimeSeries); + Metrics_.pop_back(); + } + + break; + } + case EMetricsMergingMode::DEFAULT: + break; + } +} + +void TBufferedEncoderBase::OnLabelsBegin() { + if (State_ == TEncoderState::EState::METRIC) { + State_ = TEncoderState::EState::METRIC_LABELS; + } else if (State_ == TEncoderState::EState::ROOT) { + State_ = TEncoderState::EState::COMMON_LABELS; + } else { + State_.ThrowInvalid("expected METRIC or ROOT"); + } +} + +void TBufferedEncoderBase::OnLabelsEnd() { + if (State_ == TEncoderState::EState::METRIC_LABELS) { + State_ = TEncoderState::EState::METRIC; + } else if (State_ == TEncoderState::EState::COMMON_LABELS) { + State_ = TEncoderState::EState::ROOT; + } else { + State_.ThrowInvalid("expected LABELS or COMMON_LABELS"); + } +} + +void TBufferedEncoderBase::OnLabel(TStringBuf name, TStringBuf value) { + TPooledLabels* labels; + if (State_ == TEncoderState::EState::METRIC_LABELS) { + labels = &Metrics_.back().Labels; + } else if (State_ == TEncoderState::EState::COMMON_LABELS) { + labels = &CommonLabels_; + } else { + State_.ThrowInvalid("expected LABELS or COMMON_LABELS"); + } + + labels->emplace_back(LabelNamesPool_.PutIfAbsent(name), LabelValuesPool_.PutIfAbsent(value)); +} + +void TBufferedEncoderBase::OnLabel(ui32 name, ui32 value) { + TPooledLabels* labels; + if (State_ == TEncoderState::EState::METRIC_LABELS) { + labels = &Metrics_.back().Labels; + } else if (State_ == TEncoderState::EState::COMMON_LABELS) { + labels = &CommonLabels_; + } else { + State_.ThrowInvalid("expected LABELS or COMMON_LABELS"); + } + + labels->emplace_back(LabelNamesPool_.GetByIndex(name), LabelValuesPool_.GetByIndex(value)); +} + +std::pair<ui32, ui32> TBufferedEncoderBase::PrepareLabel(TStringBuf name, TStringBuf value) { + auto nameLabel = LabelNamesPool_.PutIfAbsent(name); + auto valueLabel = LabelValuesPool_.PutIfAbsent(value); + return std::make_pair(nameLabel->Index, valueLabel->Index); +} + +void TBufferedEncoderBase::OnDouble(TInstant time, double value) { + State_.Expect(TEncoderState::EState::METRIC); + TMetric& metric = Metrics_.back(); + metric.TimeSeries.Add(time, value); +} + +void TBufferedEncoderBase::OnInt64(TInstant time, i64 value) { + State_.Expect(TEncoderState::EState::METRIC); + TMetric& metric = Metrics_.back(); + metric.TimeSeries.Add(time, value); +} + +void TBufferedEncoderBase::OnUint64(TInstant time, ui64 value) { + State_.Expect(TEncoderState::EState::METRIC); + TMetric& metric = Metrics_.back(); + metric.TimeSeries.Add(time, value); +} + +void TBufferedEncoderBase::OnHistogram(TInstant time, IHistogramSnapshotPtr s) { + State_.Expect(TEncoderState::EState::METRIC); + TMetric& metric = Metrics_.back(); + metric.TimeSeries.Add(time, s.Get()); +} + +void TBufferedEncoderBase::OnSummaryDouble(TInstant time, ISummaryDoubleSnapshotPtr s) { + State_.Expect(TEncoderState::EState::METRIC); + TMetric& metric = Metrics_.back(); + metric.TimeSeries.Add(time, s.Get()); +} + +void TBufferedEncoderBase::OnLogHistogram(TInstant time, TLogHistogramSnapshotPtr s) { + State_.Expect(TEncoderState::EState::METRIC); + TMetric& metric = Metrics_.back(); + metric.TimeSeries.Add(time, s.Get()); +} + +TString TBufferedEncoderBase::FormatLabels(const TPooledLabels& labels) const { + auto formattedLabels = TVector<TString>(Reserve(labels.size() + CommonLabels_.size())); + auto addLabel = [&](const TPooledLabel& l) { + auto formattedLabel = TStringBuilder() << LabelNamesPool_.Get(l.Key) << '=' << LabelValuesPool_.Get(l.Value); + formattedLabels.push_back(std::move(formattedLabel)); + }; + + for (const auto& l: labels) { + addLabel(l); + } + for (const auto& l: CommonLabels_) { + const auto it = FindIf(labels, [&](const TPooledLabel& label) { + return label.Key == l.Key; + }); + if (it == labels.end()) { + addLabel(l); + } + } + Sort(formattedLabels); + + return TStringBuilder() << "{" << JoinSeq(", ", formattedLabels) << "}"; +} + +} // namespace NMonitoring diff --git a/library/cpp/monlib/encode/buffered/buffered_encoder_base.h b/library/cpp/monlib/encode/buffered/buffered_encoder_base.h new file mode 100644 index 0000000000..fe3714e58f --- /dev/null +++ b/library/cpp/monlib/encode/buffered/buffered_encoder_base.h @@ -0,0 +1,100 @@ +#pragma once + +#include "string_pool.h" + +#include <library/cpp/monlib/encode/encoder.h> +#include <library/cpp/monlib/encode/encoder_state.h> +#include <library/cpp/monlib/encode/format.h> +#include <library/cpp/monlib/metrics/metric_value.h> + +#include <util/datetime/base.h> +#include <util/digest/numeric.h> + + +namespace NMonitoring { + +class TBufferedEncoderBase : public IMetricEncoder { +public: + void OnStreamBegin() override; + void OnStreamEnd() override; + + void OnCommonTime(TInstant time) override; + + void OnMetricBegin(EMetricType type) override; + void OnMetricEnd() override; + + void OnLabelsBegin() override; + void OnLabelsEnd() override; + void OnLabel(TStringBuf name, TStringBuf value) override; + void OnLabel(ui32 name, ui32 value) override; + std::pair<ui32, ui32> PrepareLabel(TStringBuf name, TStringBuf value) override; + + void OnDouble(TInstant time, double value) override; + void OnInt64(TInstant time, i64 value) override; + void OnUint64(TInstant time, ui64 value) override; + + void OnHistogram(TInstant time, IHistogramSnapshotPtr snapshot) override; + void OnSummaryDouble(TInstant time, ISummaryDoubleSnapshotPtr snapshot) override; + void OnLogHistogram(TInstant, TLogHistogramSnapshotPtr) override; + +protected: + using TPooledStr = TStringPoolBuilder::TValue; + + struct TPooledLabel { + TPooledLabel(const TPooledStr* key, const TPooledStr* value) + : Key{key} + , Value{value} + { + } + + bool operator==(const TPooledLabel& other) const { + return std::tie(Key, Value) == std::tie(other.Key, other.Value); + } + + bool operator!=(const TPooledLabel& other) const { + return !(*this == other); + } + + const TPooledStr* Key; + const TPooledStr* Value; + }; + + using TPooledLabels = TVector<TPooledLabel>; + + struct TPooledLabelsHash { + size_t operator()(const TPooledLabels& val) const { + size_t hash{0}; + + for (auto v : val) { + hash = CombineHashes<size_t>(hash, reinterpret_cast<size_t>(v.Key)); + hash = CombineHashes<size_t>(hash, reinterpret_cast<size_t>(v.Value)); + } + + return hash; + } + }; + + using TMetricMap = THashMap<TPooledLabels, size_t, TPooledLabelsHash>; + + struct TMetric { + EMetricType MetricType = EMetricType::UNKNOWN; + TPooledLabels Labels; + TMetricTimeSeries TimeSeries; + }; + +protected: + TString FormatLabels(const TPooledLabels& labels) const; + +protected: + TEncoderState State_; + + TStringPoolBuilder LabelNamesPool_; + TStringPoolBuilder LabelValuesPool_; + TInstant CommonTime_ = TInstant::Zero(); + TPooledLabels CommonLabels_; + TVector<TMetric> Metrics_; + TMetricMap MetricMap_; + EMetricsMergingMode MetricsMergingMode_ = EMetricsMergingMode::DEFAULT; +}; + +} diff --git a/library/cpp/monlib/encode/buffered/string_pool.cpp b/library/cpp/monlib/encode/buffered/string_pool.cpp new file mode 100644 index 0000000000..b4c7988ba3 --- /dev/null +++ b/library/cpp/monlib/encode/buffered/string_pool.cpp @@ -0,0 +1,58 @@ +#include "string_pool.h" + +namespace NMonitoring { + //////////////////////////////////////////////////////////////////////////////// + // TStringPoolBuilder + //////////////////////////////////////////////////////////////////////////////// + const TStringPoolBuilder::TValue* TStringPoolBuilder::PutIfAbsent(TStringBuf str) { + Y_ENSURE(!IsBuilt_, "Cannot add more values after string has been built"); + + auto [it, isInserted] = StrMap_.try_emplace(str, Max<ui32>(), 0); + if (isInserted) { + BytesSize_ += str.size(); + it->second.Index = StrVector_.size(); + StrVector_.emplace_back(it->first, &it->second); + } + + TValue* value = &it->second; + ++value->Frequency; + return value; + } + + const TStringPoolBuilder::TValue* TStringPoolBuilder::GetByIndex(ui32 index) const { + return StrVector_.at(index).second; + } + + TStringPoolBuilder& TStringPoolBuilder::Build() { + if (RequiresSorting_) { + // sort in reversed order + std::sort(StrVector_.begin(), StrVector_.end(), [](auto& a, auto& b) { + return a.second->Frequency > b.second->Frequency; + }); + + ui32 i = 0; + for (auto& value : StrVector_) { + value.second->Index = i++; + } + } + + IsBuilt_ = true; + + return *this; + } + + //////////////////////////////////////////////////////////////////////////////// + // TStringPool + //////////////////////////////////////////////////////////////////////////////// + void TStringPool::InitIndex(const char* data, ui32 size) { + const char* begin = data; + const char* end = begin + size; + for (const char* p = begin; p != end; ++p) { + if (*p == '\0') { + Index_.push_back(TStringBuf(begin, p)); + begin = p + 1; + } + } + } + +} diff --git a/library/cpp/monlib/encode/buffered/string_pool.h b/library/cpp/monlib/encode/buffered/string_pool.h new file mode 100644 index 0000000000..00e5644608 --- /dev/null +++ b/library/cpp/monlib/encode/buffered/string_pool.h @@ -0,0 +1,92 @@ +#pragma once + +#include <util/generic/hash.h> +#include <util/generic/vector.h> + +namespace NMonitoring { + //////////////////////////////////////////////////////////////////////////////// + // TStringPoolBuilder + //////////////////////////////////////////////////////////////////////////////// + class TStringPoolBuilder { + public: + struct TValue: TNonCopyable { + TValue(ui32 idx, ui32 freq) + : Index{idx} + , Frequency{freq} + { + } + + ui32 Index; + ui32 Frequency; + }; + + public: + const TValue* PutIfAbsent(TStringBuf str); + const TValue* GetByIndex(ui32 index) const; + + /// Determines whether pool must be sorted by value frequencies + TStringPoolBuilder& SetSorted(bool sorted) { + RequiresSorting_ = sorted; + return *this; + } + + TStringPoolBuilder& Build(); + + TStringBuf Get(ui32 index) const { + Y_ENSURE(IsBuilt_, "Pool must be sorted first"); + return StrVector_.at(index).first; + } + + TStringBuf Get(const TValue* value) const { + return StrVector_.at(value->Index).first; + } + + template <typename TConsumer> + void ForEach(TConsumer&& c) { + Y_ENSURE(IsBuilt_, "Pool must be sorted first"); + for (const auto& value : StrVector_) { + c(value.first, value.second->Index, value.second->Frequency); + } + } + + size_t BytesSize() const noexcept { + return BytesSize_; + } + + size_t Count() const noexcept { + return StrMap_.size(); + } + + private: + THashMap<TString, TValue> StrMap_; + TVector<std::pair<TStringBuf, TValue*>> StrVector_; + bool RequiresSorting_ = false; + bool IsBuilt_ = false; + size_t BytesSize_ = 0; + }; + + //////////////////////////////////////////////////////////////////////////////// + // TStringPool + //////////////////////////////////////////////////////////////////////////////// + class TStringPool { + public: + TStringPool(const char* data, ui32 size) { + InitIndex(data, size); + } + + TStringBuf Get(ui32 i) const { + return Index_.at(i); + } + + size_t Size() const { + return Index_.size(); + } + + private: + void InitIndex(const char* data, ui32 size); + + private: + TVector<TStringBuf> Index_; + }; + +} diff --git a/library/cpp/monlib/encode/buffered/string_pool_ut.cpp b/library/cpp/monlib/encode/buffered/string_pool_ut.cpp new file mode 100644 index 0000000000..9fc3421d0b --- /dev/null +++ b/library/cpp/monlib/encode/buffered/string_pool_ut.cpp @@ -0,0 +1,84 @@ +#include "string_pool.h" + +#include <library/cpp/testing/unittest/registar.h> + +using namespace NMonitoring; + +Y_UNIT_TEST_SUITE(TStringPoolTest) { + Y_UNIT_TEST(PutIfAbsent) { + TStringPoolBuilder strPool; + strPool.SetSorted(true); + + auto* h1 = strPool.PutIfAbsent("one"); + auto* h2 = strPool.PutIfAbsent("two"); + auto* h3 = strPool.PutIfAbsent("two"); + UNIT_ASSERT(h1 != h2); + UNIT_ASSERT(h2 == h3); + + UNIT_ASSERT_VALUES_EQUAL(h1->Frequency, 1); + UNIT_ASSERT_VALUES_EQUAL(h1->Index, 0); + + UNIT_ASSERT_VALUES_EQUAL(h2->Frequency, 2); + UNIT_ASSERT_VALUES_EQUAL(h2->Index, 1); + + UNIT_ASSERT_VALUES_EQUAL(strPool.BytesSize(), 6); + UNIT_ASSERT_VALUES_EQUAL(strPool.Count(), 2); + } + + Y_UNIT_TEST(SortByFrequency) { + TStringPoolBuilder strPool; + strPool.SetSorted(true); + + auto* h1 = strPool.PutIfAbsent("one"); + auto* h2 = strPool.PutIfAbsent("two"); + auto* h3 = strPool.PutIfAbsent("two"); + UNIT_ASSERT(h1 != h2); + UNIT_ASSERT(h2 == h3); + + strPool.Build(); + + UNIT_ASSERT_VALUES_EQUAL(h1->Frequency, 1); + UNIT_ASSERT_VALUES_EQUAL(h1->Index, 1); + + UNIT_ASSERT_VALUES_EQUAL(h2->Frequency, 2); + UNIT_ASSERT_VALUES_EQUAL(h2->Index, 0); + + UNIT_ASSERT_VALUES_EQUAL(strPool.BytesSize(), 6); + UNIT_ASSERT_VALUES_EQUAL(strPool.Count(), 2); + } + + Y_UNIT_TEST(ForEach) { + TStringPoolBuilder strPool; + strPool.SetSorted(true); + + strPool.PutIfAbsent("one"); + strPool.PutIfAbsent("two"); + strPool.PutIfAbsent("two"); + strPool.PutIfAbsent("three"); + strPool.PutIfAbsent("three"); + strPool.PutIfAbsent("three"); + + UNIT_ASSERT_VALUES_EQUAL(strPool.BytesSize(), 11); + UNIT_ASSERT_VALUES_EQUAL(strPool.Count(), 3); + + strPool.Build(); + + TVector<TString> strings; + TVector<ui32> indexes; + TVector<ui32> frequences; + strPool.ForEach([&](TStringBuf str, ui32 index, ui32 freq) { + strings.emplace_back(str); + indexes.push_back(index); + frequences.push_back(freq); + }); + + TVector<TString> expectedStrings = {"three", "two", "one"}; + UNIT_ASSERT_EQUAL(strings, expectedStrings); + + TVector<ui32> expectedIndexes = {0, 1, 2}; + UNIT_ASSERT_EQUAL(indexes, expectedIndexes); + + TVector<ui32> expectedFrequences = {3, 2, 1}; + UNIT_ASSERT_EQUAL(frequences, expectedFrequences); + } +} diff --git a/library/cpp/monlib/encode/buffered/ut/ya.make b/library/cpp/monlib/encode/buffered/ut/ya.make new file mode 100644 index 0000000000..2157ac1490 --- /dev/null +++ b/library/cpp/monlib/encode/buffered/ut/ya.make @@ -0,0 +1,12 @@ +UNITTEST_FOR(library/cpp/monlib/encode/buffered) + +OWNER( + g:solomon + jamel +) + +SRCS( + string_pool_ut.cpp +) + +END() diff --git a/library/cpp/monlib/encode/buffered/ya.make b/library/cpp/monlib/encode/buffered/ya.make new file mode 100644 index 0000000000..81b6a78b93 --- /dev/null +++ b/library/cpp/monlib/encode/buffered/ya.make @@ -0,0 +1,19 @@ +LIBRARY() + +OWNER( + g:solomon + jamel + msherbakov +) + +SRCS( + buffered_encoder_base.cpp + string_pool.cpp +) + +PEERDIR( + library/cpp/monlib/encode + library/cpp/monlib/metrics +) + +END() diff --git a/library/cpp/monlib/encode/encoder.cpp b/library/cpp/monlib/encode/encoder.cpp new file mode 100644 index 0000000000..becf932689 --- /dev/null +++ b/library/cpp/monlib/encode/encoder.cpp @@ -0,0 +1,6 @@ +#include "encoder.h" + +namespace NMonitoring { + IMetricEncoder::~IMetricEncoder() { + } +} diff --git a/library/cpp/monlib/encode/encoder.h b/library/cpp/monlib/encode/encoder.h new file mode 100644 index 0000000000..a26a133d16 --- /dev/null +++ b/library/cpp/monlib/encode/encoder.h @@ -0,0 +1,17 @@ +#pragma once + +#include <util/generic/ptr.h> + +#include <library/cpp/monlib/metrics/metric_consumer.h> + +namespace NMonitoring { + class IMetricEncoder: public IMetricConsumer { + public: + virtual ~IMetricEncoder(); + + virtual void Close() = 0; + }; + + using IMetricEncoderPtr = THolder<IMetricEncoder>; + +} diff --git a/library/cpp/monlib/encode/encoder_state.cpp b/library/cpp/monlib/encode/encoder_state.cpp new file mode 100644 index 0000000000..0ece696b1a --- /dev/null +++ b/library/cpp/monlib/encode/encoder_state.cpp @@ -0,0 +1 @@ +#include "encoder_state.h" diff --git a/library/cpp/monlib/encode/encoder_state.h b/library/cpp/monlib/encode/encoder_state.h new file mode 100644 index 0000000000..e6a098f404 --- /dev/null +++ b/library/cpp/monlib/encode/encoder_state.h @@ -0,0 +1,62 @@ +#pragma once + +#include "encoder_state_enum.h" + +#include <util/generic/serialized_enum.h> +#include <util/generic/yexception.h> + + +namespace NMonitoring { + + template <typename EEncoderState> + class TEncoderStateImpl { + public: + using EState = EEncoderState; + + explicit TEncoderStateImpl(EEncoderState state = EEncoderState::ROOT) + : State_(state) + { + } + + TEncoderStateImpl& operator=(EEncoderState rhs) noexcept { + State_ = rhs; + return *this; + } + + inline bool operator==(EEncoderState rhs) const noexcept { + return State_ == rhs; + } + + inline bool operator!=(EEncoderState rhs) const noexcept { + return !operator==(rhs); + } + + [[noreturn]] inline void ThrowInvalid(TStringBuf message) const { + ythrow yexception() << "invalid encoder state: " + << ToStr() << ", " << message; + } + + inline void Expect(EEncoderState expected) const { + if (Y_UNLIKELY(State_ != expected)) { + ythrow yexception() + << "invalid encoder state: " << ToStr() + << ", expected: " << TEncoderStateImpl(expected).ToStr(); + } + } + + inline void Switch(EEncoderState from, EEncoderState to) { + Expect(from); + State_ = to; + } + + TStringBuf ToStr() const noexcept { + return NEnumSerializationRuntime::GetEnumNamesImpl<EEncoderState>().at(State_); + } + + private: + EEncoderState State_; + }; + + using TEncoderState = TEncoderStateImpl<EEncoderState>; + +} // namespace NMonitoring diff --git a/library/cpp/monlib/encode/encoder_state_enum.h b/library/cpp/monlib/encode/encoder_state_enum.h new file mode 100644 index 0000000000..471604f91d --- /dev/null +++ b/library/cpp/monlib/encode/encoder_state_enum.h @@ -0,0 +1,12 @@ +#pragma once + +namespace NMonitoring { + + enum class EEncoderState { + ROOT, + COMMON_LABELS, + METRIC, + METRIC_LABELS, + }; + +} // namespace NMonitoring diff --git a/library/cpp/monlib/encode/fake/fake.cpp b/library/cpp/monlib/encode/fake/fake.cpp new file mode 100644 index 0000000000..69d691361a --- /dev/null +++ b/library/cpp/monlib/encode/fake/fake.cpp @@ -0,0 +1,51 @@ +#include "fake.h" + +#include <util/datetime/base.h> + +namespace NMonitoring { + class TFakeEncoder: public IMetricEncoder { + public: + void OnStreamBegin() override { + } + void OnStreamEnd() override { + } + + void OnCommonTime(TInstant) override { + } + + void OnMetricBegin(EMetricType) override { + } + void OnMetricEnd() override { + } + + void OnLabelsBegin() override { + } + void OnLabelsEnd() override { + } + void OnLabel(const TStringBuf, const TStringBuf) override { + } + + void OnDouble(TInstant, double) override { + } + void OnInt64(TInstant, i64) override { + } + void OnUint64(TInstant, ui64) override { + } + + void OnHistogram(TInstant, IHistogramSnapshotPtr) override { + } + + void OnSummaryDouble(TInstant, ISummaryDoubleSnapshotPtr) override { + } + + void OnLogHistogram(TInstant, TLogHistogramSnapshotPtr) override { + } + + void Close() override { + } + }; + + IMetricEncoderPtr EncoderFake() { + return MakeHolder<TFakeEncoder>(); + } +} diff --git a/library/cpp/monlib/encode/fake/fake.h b/library/cpp/monlib/encode/fake/fake.h new file mode 100644 index 0000000000..8109326987 --- /dev/null +++ b/library/cpp/monlib/encode/fake/fake.h @@ -0,0 +1,10 @@ +#pragma once + +#include <library/cpp/monlib/encode/encoder.h> + +class IOutputStream; + +namespace NMonitoring { + // Does nothing: just implements IMetricEncoder interface with stubs + IMetricEncoderPtr EncoderFake(); +} diff --git a/library/cpp/monlib/encode/fake/ya.make b/library/cpp/monlib/encode/fake/ya.make new file mode 100644 index 0000000000..ae96f45782 --- /dev/null +++ b/library/cpp/monlib/encode/fake/ya.make @@ -0,0 +1,12 @@ +LIBRARY() + +OWNER( + g:solomon + msherbakov +) + +SRCS( + fake.cpp +) + +END() diff --git a/library/cpp/monlib/encode/format.cpp b/library/cpp/monlib/encode/format.cpp new file mode 100644 index 0000000000..400ce5a643 --- /dev/null +++ b/library/cpp/monlib/encode/format.cpp @@ -0,0 +1,202 @@ +#include "format.h" + +#include <util/string/ascii.h> +#include <util/string/split.h> +#include <util/string/strip.h> +#include <util/stream/output.h> +#include <util/string/cast.h> + +namespace NMonitoring { + static ECompression CompressionFromHeader(TStringBuf value) { + if (value.empty()) { + return ECompression::UNKNOWN; + } + + for (const auto& it : StringSplitter(value).Split(',').SkipEmpty()) { + TStringBuf token = StripString(it.Token()); + + if (AsciiEqualsIgnoreCase(token, NFormatContentEncoding::IDENTITY)) { + return ECompression::IDENTITY; + } else if (AsciiEqualsIgnoreCase(token, NFormatContentEncoding::ZLIB)) { + return ECompression::ZLIB; + } else if (AsciiEqualsIgnoreCase(token, NFormatContentEncoding::LZ4)) { + return ECompression::LZ4; + } else if (AsciiEqualsIgnoreCase(token, NFormatContentEncoding::ZSTD)) { + return ECompression::ZSTD; + } + } + + return ECompression::UNKNOWN; + } + + static EFormat FormatFromHttpMedia(TStringBuf value) { + if (AsciiEqualsIgnoreCase(value, NFormatContenType::SPACK)) { + return EFormat::SPACK; + } else if (AsciiEqualsIgnoreCase(value, NFormatContenType::JSON)) { + return EFormat::JSON; + } else if (AsciiEqualsIgnoreCase(value, NFormatContenType::PROTOBUF)) { + return EFormat::PROTOBUF; + } else if (AsciiEqualsIgnoreCase(value, NFormatContenType::TEXT)) { + return EFormat::TEXT; + } else if (AsciiEqualsIgnoreCase(value, NFormatContenType::PROMETHEUS)) { + return EFormat::PROMETHEUS; + } + + return EFormat::UNKNOWN; + } + + EFormat FormatFromAcceptHeader(TStringBuf value) { + EFormat result{EFormat::UNKNOWN}; + + for (const auto& it : StringSplitter(value).Split(',').SkipEmpty()) { + TStringBuf token = StripString(it.Token()); + + result = FormatFromHttpMedia(token); + if (result != EFormat::UNKNOWN) { + break; + } + } + + return result; + } + + EFormat FormatFromContentType(TStringBuf value) { + value = value.NextTok(';'); + + return FormatFromHttpMedia(value); + } + + TStringBuf ContentTypeByFormat(EFormat format) { + switch (format) { + case EFormat::SPACK: + return NFormatContenType::SPACK; + case EFormat::JSON: + return NFormatContenType::JSON; + case EFormat::PROTOBUF: + return NFormatContenType::PROTOBUF; + case EFormat::TEXT: + return NFormatContenType::TEXT; + case EFormat::PROMETHEUS: + return NFormatContenType::PROMETHEUS; + case EFormat::UNKNOWN: + return TStringBuf(); + } + + Y_FAIL(); // for GCC + } + + ECompression CompressionFromAcceptEncodingHeader(TStringBuf value) { + return CompressionFromHeader(value); + } + + ECompression CompressionFromContentEncodingHeader(TStringBuf value) { + return CompressionFromHeader(value); + } + + TStringBuf ContentEncodingByCompression(ECompression compression) { + switch (compression) { + case ECompression::IDENTITY: + return NFormatContentEncoding::IDENTITY; + case ECompression::ZLIB: + return NFormatContentEncoding::ZLIB; + case ECompression::LZ4: + return NFormatContentEncoding::LZ4; + case ECompression::ZSTD: + return NFormatContentEncoding::ZSTD; + case ECompression::UNKNOWN: + return TStringBuf(); + } + + Y_FAIL(); // for GCC + } + +} + +template <> +NMonitoring::EFormat FromStringImpl<NMonitoring::EFormat>(const char* str, size_t len) { + using NMonitoring::EFormat; + TStringBuf value(str, len); + if (value == TStringBuf("SPACK")) { + return EFormat::SPACK; + } else if (value == TStringBuf("JSON")) { + return EFormat::JSON; + } else if (value == TStringBuf("PROTOBUF")) { + return EFormat::PROTOBUF; + } else if (value == TStringBuf("TEXT")) { + return EFormat::TEXT; + } else if (value == TStringBuf("PROMETHEUS")) { + return EFormat::PROMETHEUS; + } else if (value == TStringBuf("UNKNOWN")) { + return EFormat::UNKNOWN; + } + ythrow yexception() << "unknown format: " << value; +} + +template <> +void Out<NMonitoring::EFormat>(IOutputStream& o, NMonitoring::EFormat f) { + using NMonitoring::EFormat; + switch (f) { + case EFormat::SPACK: + o << TStringBuf("SPACK"); + return; + case EFormat::JSON: + o << TStringBuf("JSON"); + return; + case EFormat::PROTOBUF: + o << TStringBuf("PROTOBUF"); + return; + case EFormat::TEXT: + o << TStringBuf("TEXT"); + return; + case EFormat::PROMETHEUS: + o << TStringBuf("PROMETHEUS"); + return; + case EFormat::UNKNOWN: + o << TStringBuf("UNKNOWN"); + return; + } + + Y_FAIL(); // for GCC +} + +template <> +NMonitoring::ECompression FromStringImpl<NMonitoring::ECompression>(const char* str, size_t len) { + using NMonitoring::ECompression; + TStringBuf value(str, len); + if (value == TStringBuf("IDENTITY")) { + return ECompression::IDENTITY; + } else if (value == TStringBuf("ZLIB")) { + return ECompression::ZLIB; + } else if (value == TStringBuf("LZ4")) { + return ECompression::LZ4; + } else if (value == TStringBuf("ZSTD")) { + return ECompression::ZSTD; + } else if (value == TStringBuf("UNKNOWN")) { + return ECompression::UNKNOWN; + } + ythrow yexception() << "unknown compression: " << value; +} + +template <> +void Out<NMonitoring::ECompression>(IOutputStream& o, NMonitoring::ECompression c) { + using NMonitoring::ECompression; + switch (c) { + case ECompression::IDENTITY: + o << TStringBuf("IDENTITY"); + return; + case ECompression::ZLIB: + o << TStringBuf("ZLIB"); + return; + case ECompression::LZ4: + o << TStringBuf("LZ4"); + return; + case ECompression::ZSTD: + o << TStringBuf("ZSTD"); + return; + case ECompression::UNKNOWN: + o << TStringBuf("UNKNOWN"); + return; + } + + Y_FAIL(); // for GCC +} diff --git a/library/cpp/monlib/encode/format.h b/library/cpp/monlib/encode/format.h new file mode 100644 index 0000000000..495d42d786 --- /dev/null +++ b/library/cpp/monlib/encode/format.h @@ -0,0 +1,166 @@ +#pragma once + +#include <util/generic/strbuf.h> + +namespace NMonitoring { + namespace NFormatContenType { + constexpr TStringBuf TEXT = "application/x-solomon-txt"; + constexpr TStringBuf JSON = "application/json"; + constexpr TStringBuf PROTOBUF = "application/x-solomon-pb"; + constexpr TStringBuf SPACK = "application/x-solomon-spack"; + constexpr TStringBuf PROMETHEUS = "text/plain"; + } + + namespace NFormatContentEncoding { + constexpr TStringBuf IDENTITY = "identity"; + constexpr TStringBuf ZLIB = "zlib"; + constexpr TStringBuf LZ4 = "lz4"; + constexpr TStringBuf ZSTD = "zstd"; + } + + /** + * Defines format types for metric encoders. + */ + enum class EFormat { + /** + * Special case when it was not possible to determine format. + */ + UNKNOWN, + + /** + * Read more https://wiki.yandex-team.ru/solomon/api/dataformat/spackv1 + */ + SPACK, + + /** + * Read more https://wiki.yandex-team.ru/solomon/api/dataformat/json + */ + JSON, + + /** + * Simple protobuf format, only for testing purposes. + */ + PROTOBUF, + + /** + * Simple text representation, only for debug purposes. + */ + TEXT, + + /** + * Prometheus text-based format + */ + PROMETHEUS, + }; + + /** + * Defines compression algorithms for metric encoders. + */ + enum class ECompression { + /** + * Special case when it was not possible to determine compression. + */ + UNKNOWN, + + /** + * Means no compression. + */ + IDENTITY, + + /** + * Using the zlib structure (defined in RFC 1950), with the + * deflate compression algorithm and Adler32 checkums. + */ + ZLIB, + + /** + * Using LZ4 compression algorithm (read http://lz4.org for more info) + * with XxHash32 checksums. + */ + LZ4, + + /** + * Using Zstandard compression algorithm (read http://zstd.net for more + * info) with XxHash32 checksums. + */ + ZSTD, + }; + + enum class EMetricsMergingMode { + /** + * Do not merge metric batches. If several points of the same metric were + * added multiple times accross different writes, paste them as + * separate metrics. + * + * Example: + * COUNTER [(ts1, val1)] | COUNTER [(ts1, val1)] + * COUNTER [(ts2, val2)] | --> COUNTER [(ts2, val2)] + * COUNTER [(ts3, val3)] | COUNTER [(ts3, val3)] + */ + DEFAULT, + + /** + * If several points of the same metric were added multiple times across + * different writes, merge all values to one timeseries. + * + * Example: + * COUNTER [(ts1, val1)] | + * COUNTER [(ts2, val2)] | --> COUNTER [(ts1, val1), (ts2, val2), (ts3, val3)] + * COUNTER [(ts3, val3)] | + */ + MERGE_METRICS, + }; + + /** + * Matches serialization format by the given "Accept" header value. + * + * @param value value of the "Accept" header. + * @return most preffered serialization format type + */ + EFormat FormatFromAcceptHeader(TStringBuf value); + + /** + * Matches serialization format by the given "Content-Type" header value + * + * @param value value of the "Content-Type" header + * @return message format + */ + EFormat FormatFromContentType(TStringBuf value); + + /** + * Returns value for "Content-Type" header determined by the given + * format type. + * + * @param format serialization format type + * @return mime-type indentificator + * or empty string if format is UNKNOWN + */ + TStringBuf ContentTypeByFormat(EFormat format); + + /** + * Matches compression algorithm by the given "Accept-Encoding" header value. + * + * @param value value of the "Accept-Encoding" header. + * @return most preffered compression algorithm + */ + ECompression CompressionFromAcceptEncodingHeader(TStringBuf value); + + /** + * Matches compression algorithm by the given "Content-Encoding" header value. + * + * @param value value of the "Accept-Encoding" header. + * @return most preffered compression algorithm + */ + ECompression CompressionFromContentEncodingHeader(TStringBuf value); + + /** + * Returns value for "Content-Encoding" header determined by the given + * compression algorithm. + * + * @param compression encoding compression alg + * @return media-type compresion algorithm + * or empty string if compression is UNKNOWN + */ + TStringBuf ContentEncodingByCompression(ECompression compression); + +} diff --git a/library/cpp/monlib/encode/format_ut.cpp b/library/cpp/monlib/encode/format_ut.cpp new file mode 100644 index 0000000000..22a0e30c03 --- /dev/null +++ b/library/cpp/monlib/encode/format_ut.cpp @@ -0,0 +1,136 @@ +#include "format.h" + +#include <library/cpp/testing/unittest/registar.h> + +#include <util/generic/string.h> +#include <util/string/builder.h> + +#include <array> + +using namespace NMonitoring; + +Y_UNIT_TEST_SUITE(TFormatTest) { + Y_UNIT_TEST(ContentTypeHeader) { + UNIT_ASSERT_EQUAL(FormatFromContentType(""), EFormat::UNKNOWN); + UNIT_ASSERT_EQUAL(FormatFromContentType("application/json;some=stuff"), EFormat::JSON); + UNIT_ASSERT_EQUAL(FormatFromContentType("application/x-solomon-spack"), EFormat::SPACK); + UNIT_ASSERT_EQUAL(FormatFromContentType("application/xml"), EFormat::UNKNOWN); + UNIT_ASSERT_EQUAL(FormatFromContentType(";application/xml"), EFormat::UNKNOWN); + } + + Y_UNIT_TEST(AcceptHeader) { + UNIT_ASSERT_EQUAL(FormatFromAcceptHeader(""), EFormat::UNKNOWN); + UNIT_ASSERT_EQUAL(FormatFromAcceptHeader("*/*"), EFormat::UNKNOWN); + + UNIT_ASSERT_EQUAL( + FormatFromAcceptHeader("application/xml"), + EFormat::UNKNOWN); + + UNIT_ASSERT_EQUAL( + FormatFromAcceptHeader("application/json"), + EFormat::JSON); + + UNIT_ASSERT_EQUAL( + FormatFromAcceptHeader("application/x-solomon-spack"), + EFormat::SPACK); + + UNIT_ASSERT_EQUAL( + FormatFromAcceptHeader("application/x-solomon-pb"), + EFormat::PROTOBUF); + + UNIT_ASSERT_EQUAL( + FormatFromAcceptHeader("application/x-solomon-txt"), + EFormat::TEXT); + + UNIT_ASSERT_EQUAL( + FormatFromAcceptHeader("application/json, text/plain"), + EFormat::JSON); + + UNIT_ASSERT_EQUAL( + FormatFromAcceptHeader("application/x-solomon-spack, application/json, text/plain"), + EFormat::SPACK); + + UNIT_ASSERT_EQUAL( + FormatFromAcceptHeader(" , application/x-solomon-spack ,, application/json , text/plain"), + EFormat::SPACK); + + UNIT_ASSERT_EQUAL( + FormatFromAcceptHeader("application/xml, application/x-solomon-spack, text/plain"), + EFormat::SPACK); + + UNIT_ASSERT_EQUAL( + FormatFromAcceptHeader("text/plain"), + EFormat::PROMETHEUS); + } + + Y_UNIT_TEST(FormatToStrFromStr) { + const std::array<EFormat, 6> formats = {{ + EFormat::UNKNOWN, + EFormat::SPACK, + EFormat::JSON, + EFormat::PROTOBUF, + EFormat::TEXT, + EFormat::PROMETHEUS, + }}; + + for (EFormat f : formats) { + TString str = (TStringBuilder() << f); + EFormat g = FromString<EFormat>(str); + UNIT_ASSERT_EQUAL(f, g); + } + } + + Y_UNIT_TEST(AcceptEncodingHeader) { + UNIT_ASSERT_EQUAL( + CompressionFromAcceptEncodingHeader(""), + ECompression::UNKNOWN); + + UNIT_ASSERT_EQUAL( + CompressionFromAcceptEncodingHeader("br"), + ECompression::UNKNOWN); + + UNIT_ASSERT_EQUAL( + CompressionFromAcceptEncodingHeader("identity"), + ECompression::IDENTITY); + + UNIT_ASSERT_EQUAL( + CompressionFromAcceptEncodingHeader("zlib"), + ECompression::ZLIB); + + UNIT_ASSERT_EQUAL( + CompressionFromAcceptEncodingHeader("lz4"), + ECompression::LZ4); + + UNIT_ASSERT_EQUAL( + CompressionFromAcceptEncodingHeader("zstd"), + ECompression::ZSTD); + + UNIT_ASSERT_EQUAL( + CompressionFromAcceptEncodingHeader("zstd, zlib"), + ECompression::ZSTD); + + UNIT_ASSERT_EQUAL( + CompressionFromAcceptEncodingHeader(" ,, , zstd , zlib"), + ECompression::ZSTD); + + UNIT_ASSERT_EQUAL( + CompressionFromAcceptEncodingHeader("br, deflate,lz4, zlib"), + ECompression::LZ4); + } + + Y_UNIT_TEST(CompressionToStrFromStr) { + const std::array<ECompression, 5> algs = {{ + ECompression::UNKNOWN, + ECompression::IDENTITY, + ECompression::ZLIB, + ECompression::LZ4, + ECompression::ZSTD, + }}; + + for (ECompression a : algs) { + TString str = (TStringBuilder() << a); + ECompression b = FromString<ECompression>(str); + UNIT_ASSERT_EQUAL(a, b); + } + } +} diff --git a/library/cpp/monlib/encode/fuzz/ya.make b/library/cpp/monlib/encode/fuzz/ya.make new file mode 100644 index 0000000000..d9ca172bae --- /dev/null +++ b/library/cpp/monlib/encode/fuzz/ya.make @@ -0,0 +1,5 @@ +RECURSE_ROOT_RELATIVE( + library/cpp/monlib/encode/json/fuzz + library/cpp/monlib/encode/prometheus/fuzz + library/cpp/monlib/encode/spack/fuzz +) diff --git a/library/cpp/monlib/encode/json/fuzz/main.cpp b/library/cpp/monlib/encode/json/fuzz/main.cpp new file mode 100644 index 0000000000..4f40310e06 --- /dev/null +++ b/library/cpp/monlib/encode/json/fuzz/main.cpp @@ -0,0 +1,16 @@ +#include <library/cpp/monlib/encode/json/json.h> +#include <library/cpp/monlib/encode/fake/fake.h> + +#include <util/generic/strbuf.h> + + +extern "C" int LLVMFuzzerTestOneInput(const ui8* data, size_t size) { + auto encoder = NMonitoring::EncoderFake(); + + try { + NMonitoring::DecodeJson({reinterpret_cast<const char*>(data), size}, encoder.Get()); + } catch (...) { + } + + return 0; +} diff --git a/library/cpp/monlib/encode/json/fuzz/ya.make b/library/cpp/monlib/encode/json/fuzz/ya.make new file mode 100644 index 0000000000..75baa77716 --- /dev/null +++ b/library/cpp/monlib/encode/json/fuzz/ya.make @@ -0,0 +1,19 @@ +FUZZ() + +OWNER( + g:solomon + msherbakov +) + +PEERDIR( + library/cpp/monlib/encode/json + library/cpp/monlib/encode/fake +) + +SIZE(MEDIUM) + +SRCS( + main.cpp +) + +END() diff --git a/library/cpp/monlib/encode/json/json.h b/library/cpp/monlib/encode/json/json.h new file mode 100644 index 0000000000..21530f20c3 --- /dev/null +++ b/library/cpp/monlib/encode/json/json.h @@ -0,0 +1,29 @@ +#pragma once + +#include <library/cpp/monlib/encode/encoder.h> +#include <library/cpp/monlib/encode/format.h> + + +class IOutputStream; + +namespace NMonitoring { + + class TJsonDecodeError: public yexception { + }; + + IMetricEncoderPtr EncoderJson(IOutputStream* out, int indentation = 0); + + /// Buffered encoder will merge series with same labels into one. + IMetricEncoderPtr BufferedEncoderJson(IOutputStream* out, int indentation = 0); + + IMetricEncoderPtr EncoderCloudJson(IOutputStream* out, + int indentation = 0, + TStringBuf metricNameLabel = "name"); + + IMetricEncoderPtr BufferedEncoderCloudJson(IOutputStream* out, + int indentation = 0, + TStringBuf metricNameLabel = "name"); + + void DecodeJson(TStringBuf data, IMetricConsumer* c, TStringBuf metricNameLabel = "name"); + +} diff --git a/library/cpp/monlib/encode/json/json_decoder.cpp b/library/cpp/monlib/encode/json/json_decoder.cpp new file mode 100644 index 0000000000..d44ff5fd28 --- /dev/null +++ b/library/cpp/monlib/encode/json/json_decoder.cpp @@ -0,0 +1,1162 @@ +#include "json.h" +#include "typed_point.h" + + +#include <library/cpp/monlib/exception/exception.h> +#include <library/cpp/monlib/metrics/labels.h> +#include <library/cpp/monlib/metrics/metric_value.h> + +#include <library/cpp/json/json_reader.h> + +#include <util/datetime/base.h> +#include <util/string/cast.h> + +#include <limits> + +namespace NMonitoring { + +#define DECODE_ENSURE(COND, ...) MONLIB_ENSURE_EX(COND, TJsonDecodeError() << __VA_ARGS__) + +namespace { + +/////////////////////////////////////////////////////////////////////// +// THistogramBuilder +/////////////////////////////////////////////////////////////////////// +class THistogramBuilder { +public: + void AddBound(TBucketBound bound) { + if (!Bounds_.empty()) { + DECODE_ENSURE(Bounds_.back() < bound, + "non sorted bounds, " << Bounds_.back() << + " >= " << bound); + } + Bounds_.push_back(bound); + } + + void AddValue(TBucketValue value) { + Values_.push_back(value); + } + + void AddInf(TBucketValue value) { + InfPresented_ = true; + InfValue_ = value; + } + + IHistogramSnapshotPtr Build() { + if (InfPresented_) { + Bounds_.push_back(Max<TBucketBound>()); + Values_.push_back(InfValue_); + } + + auto snapshot = ExplicitHistogramSnapshot(Bounds_, Values_); + + Bounds_.clear(); + Values_.clear(); + InfPresented_ = false; + + return snapshot; + } + + bool Empty() const noexcept { + return Bounds_.empty() && Values_.empty(); + } + + void Clear() { + Bounds_.clear(); + Values_.clear(); + } + +private: + TBucketBounds Bounds_; + TBucketValues Values_; + + bool InfPresented_ = false; + TBucketValue InfValue_; +}; + +class TSummaryDoubleBuilder { +public: + ISummaryDoubleSnapshotPtr Build() const { + return MakeIntrusive<TSummaryDoubleSnapshot>(Sum_, Min_, Max_, Last_, Count_); + } + + void SetSum(double sum) { + Empty_ = false; + Sum_ = sum; + } + + void SetMin(double min) { + Empty_ = false; + Min_ = min; + } + + void SetMax(double max) { + Empty_ = false; + Max_ = max; + } + + void SetLast(double last) { + Empty_ = false; + Last_ = last; + } + + void SetCount(ui64 count) { + Empty_ = false; + Count_ = count; + } + + void Clear() { + Empty_ = true; + Sum_ = 0; + Min_ = 0; + Max_ = 0; + Last_ = 0; + Count_ = 0; + } + + bool Empty() const { + return Empty_; + } + +private: + double Sum_ = 0; + double Min_ = 0; + double Max_ = 0; + double Last_ = 0; + ui64 Count_ = 0; + bool Empty_ = true; +}; + +class TLogHistogramBuilder { +public: + void SetBase(double base) { + DECODE_ENSURE(base > 0, "base must be positive"); + Base_ = base; + } + + void SetZerosCount(ui64 zerosCount) { + DECODE_ENSURE(zerosCount >= 0, "zeros count must be positive"); + ZerosCount_ = zerosCount; + } + + void SetStartPower(int startPower) { + StartPower_ = startPower; + } + + void AddBucketValue(double value) { + DECODE_ENSURE(value > 0.0, "bucket values must be positive"); + DECODE_ENSURE(value < std::numeric_limits<double>::max(), "bucket values must be finite"); + Buckets_.push_back(value); + } + + void Clear() { + Buckets_.clear(); + Base_ = 1.5; + ZerosCount_ = 0; + StartPower_ = 0; + } + + bool Empty() const { + return Buckets_.empty() && ZerosCount_ == 0; + } + + TLogHistogramSnapshotPtr Build() { + return MakeIntrusive<TLogHistogramSnapshot>(Base_, ZerosCount_, StartPower_, std::move(Buckets_)); + } + +private: + double Base_ = 1.5; + ui64 ZerosCount_ = 0; + int StartPower_ = 0; + TVector<double> Buckets_; +}; + +std::pair<double, bool> ParseSpecDouble(TStringBuf string) { + if (string == TStringBuf("nan") || string == TStringBuf("NaN")) { + return {std::numeric_limits<double>::quiet_NaN(), true}; + } else if (string == TStringBuf("inf") || string == TStringBuf("Infinity")) { + return {std::numeric_limits<double>::infinity(), true}; + } else if (string == TStringBuf("-inf") || string == TStringBuf("-Infinity")) { + return {-std::numeric_limits<double>::infinity(), true}; + } else { + return {0, false}; + } +} + +/////////////////////////////////////////////////////////////////////// +// TMetricCollector +/////////////////////////////////////////////////////////////////////// +struct TMetricCollector { + EMetricType Type = EMetricType::UNKNOWN; + TLabels Labels; + THistogramBuilder HistogramBuilder; + TSummaryDoubleBuilder SummaryBuilder; + TLogHistogramBuilder LogHistBuilder; + TTypedPoint LastPoint; + TVector<TTypedPoint> TimeSeries; + + bool SeenTsOrValue = false; + bool SeenTimeseries = false; + + void Clear() { + Type = EMetricType::UNKNOWN; + Labels.Clear(); + SeenTsOrValue = false; + SeenTimeseries = false; + TimeSeries.clear(); + LastPoint = {}; + HistogramBuilder.Clear(); + SummaryBuilder.Clear(); + LogHistBuilder.Clear(); + } + + void AddLabel(const TLabel& label) { + Labels.Add(label.Name(), label.Value()); + } + + void SetLastTime(TInstant time) { + LastPoint.SetTime(time); + } + + template <typename T> + void SetLastValue(T value) { + LastPoint.SetValue(value); + } + + void SaveLastPoint() { + DECODE_ENSURE(LastPoint.GetTime() != TInstant::Zero(), + "cannot add point without or zero timestamp"); + if (!HistogramBuilder.Empty()) { + auto histogram = HistogramBuilder.Build(); + TimeSeries.emplace_back(LastPoint.GetTime(), histogram.Get()); + } else if (!SummaryBuilder.Empty()) { + auto summary = SummaryBuilder.Build(); + TimeSeries.emplace_back(LastPoint.GetTime(), summary.Get()); + } else if (!LogHistBuilder.Empty()) { + auto logHist = LogHistBuilder.Build(); + TimeSeries.emplace_back(LastPoint.GetTime(), logHist.Get()); + } else { + TimeSeries.push_back(std::move(LastPoint)); + } + } + + template <typename TConsumer> + void Consume(TConsumer&& consumer) { + if (TimeSeries.empty()) { + const auto& p = LastPoint; + consumer(p.GetTime(), p.GetValueType(), p.GetValue()); + } else { + for (const auto& p: TimeSeries) { + consumer(p.GetTime(), p.GetValueType(), p.GetValue()); + } + } + } +}; + +struct TCommonParts { + TInstant CommonTime; + TLabels CommonLabels; +}; + +class IHaltableMetricConsumer: public IMetricConsumer { +public: + virtual bool NeedToStop() const = 0; +}; + +// TODO(ivanzhukov@): check all states for cases when a json document is invalid +// e.g. "metrics" or "commonLabels" keys are specified multiple times +class TCommonPartsCollector: public IHaltableMetricConsumer { +public: + TCommonParts&& CommonParts() { + return std::move(CommonParts_); + } + +private: + bool NeedToStop() const override { + return TInstant::Zero() != CommonParts_.CommonTime && !CommonParts_.CommonLabels.Empty(); + } + + void OnStreamBegin() override { + } + + void OnStreamEnd() override { + } + + void OnCommonTime(TInstant time) override { + CommonParts_.CommonTime = time; + } + + void OnMetricBegin(EMetricType) override { + IsMetric_ = true; + } + + void OnMetricEnd() override { + IsMetric_ = false; + } + + void OnLabelsBegin() override { + } + + void OnLabelsEnd() override { + } + + void OnLabel(TStringBuf name, TStringBuf value) override { + if (!IsMetric_) { + CommonParts_.CommonLabels.Add(std::move(name), std::move(value)); + } + } + + void OnDouble(TInstant, double) override { + } + + void OnInt64(TInstant, i64) override { + } + + void OnUint64(TInstant, ui64) override { + } + + void OnHistogram(TInstant, IHistogramSnapshotPtr) override { + } + + void OnLogHistogram(TInstant, TLogHistogramSnapshotPtr) override { + } + + void OnSummaryDouble(TInstant, ISummaryDoubleSnapshotPtr) override { + } + +private: + TCommonParts CommonParts_; + bool IsMetric_{false}; +}; + +class TCommonPartsProxy: public IHaltableMetricConsumer { +public: + TCommonPartsProxy(TCommonParts&& commonParts, IMetricConsumer* c) + : CommonParts_{std::move(commonParts)} + , Consumer_{c} + {} + +private: + bool NeedToStop() const override { + return false; + } + + void OnStreamBegin() override { + Consumer_->OnStreamBegin(); + + if (!CommonParts_.CommonLabels.Empty()) { + Consumer_->OnLabelsBegin(); + + for (auto&& label : CommonParts_.CommonLabels) { + Consumer_->OnLabel(label.Name(), label.Value()); + } + + Consumer_->OnLabelsEnd(); + } + + if (TInstant::Zero() != CommonParts_.CommonTime) { + Consumer_->OnCommonTime(CommonParts_.CommonTime); + } + } + + void OnStreamEnd() override { + Consumer_->OnStreamEnd(); + } + + void OnCommonTime(TInstant) override { + } + + void OnMetricBegin(EMetricType type) override { + IsMetric_ = true; + + Consumer_->OnMetricBegin(type); + } + + void OnMetricEnd() override { + IsMetric_ = false; + + Consumer_->OnMetricEnd(); + } + + void OnLabelsBegin() override { + if (IsMetric_) { + Consumer_->OnLabelsBegin(); + } + } + + void OnLabelsEnd() override { + if (IsMetric_) { + Consumer_->OnLabelsEnd(); + } + } + + void OnLabel(TStringBuf name, TStringBuf value) override { + if (IsMetric_) { + Consumer_->OnLabel(std::move(name), std::move(value)); + } + } + + void OnDouble(TInstant time, double value) override { + Consumer_->OnDouble(time, value); + } + + void OnInt64(TInstant time, i64 value) override { + Consumer_->OnInt64(time, value); + } + + void OnUint64(TInstant time, ui64 value) override { + Consumer_->OnUint64(time, value); + } + + void OnHistogram(TInstant time, IHistogramSnapshotPtr snapshot) override { + Consumer_->OnHistogram(time, std::move(snapshot)); + } + + void OnLogHistogram(TInstant time, TLogHistogramSnapshotPtr snapshot) override { + Consumer_->OnLogHistogram(time, std::move(snapshot)); + } + + void OnSummaryDouble(TInstant time, ISummaryDoubleSnapshotPtr snapshot) override { + Consumer_->OnSummaryDouble(time, std::move(snapshot)); + } + +private: + const TCommonParts CommonParts_; + IMetricConsumer* Consumer_; + bool IsMetric_{false}; +}; + +/////////////////////////////////////////////////////////////////////// +// TDecoderJson +/////////////////////////////////////////////////////////////////////// +class TDecoderJson final: public NJson::TJsonCallbacks { + struct TState { + enum EState { + ROOT_OBJECT = 0x01, + + COMMON_LABELS, + COMMON_TS, + METRICS_ARRAY, + + METRIC_OBJECT, + METRIC_NAME, + METRIC_LABELS, + METRIC_TYPE, + METRIC_MODE, // TODO: must be deleted + METRIC_TIMESERIES, + METRIC_TS, + METRIC_VALUE, + METRIC_HIST, + METRIC_HIST_BOUNDS, + METRIC_HIST_BUCKETS, + METRIC_HIST_INF, + METRIC_DSUMMARY, + METRIC_DSUMMARY_SUM, + METRIC_DSUMMARY_MIN, + METRIC_DSUMMARY_MAX, + METRIC_DSUMMARY_LAST, + METRIC_DSUMMARY_COUNT, + METRIC_LOG_HIST, + METRIC_LOG_HIST_BASE, + METRIC_LOG_HIST_ZEROS, + METRIC_LOG_HIST_START_POWER, + METRIC_LOG_HIST_BUCKETS, + }; + + constexpr EState Current() const noexcept { + return static_cast<EState>(State_ & 0xFF); + } + + void ToNext(EState state) noexcept { + constexpr auto bitSize = 8 * sizeof(ui8); + State_ = (State_ << bitSize) | static_cast<ui8>(state); + } + + void ToPrev() noexcept { + constexpr auto bitSize = 8 * sizeof(ui8); + State_ = State_ >> bitSize; + } + + private: + ui64 State_ = static_cast<ui64>(ROOT_OBJECT); + }; + +public: + TDecoderJson(TStringBuf data, IHaltableMetricConsumer* metricConsumer, TStringBuf metricNameLabel) + : Data_(data) + , MetricConsumer_(metricConsumer) + , MetricNameLabel_(metricNameLabel) + { + } + +private: +#define PARSE_ENSURE(CONDITION, ...) \ +do { \ +if (Y_UNLIKELY(!(CONDITION))) { \ + ErrorMsg_ = TStringBuilder() << __VA_ARGS__; \ + return false; \ +} \ +} while (false) + + bool OnInteger(long long value) override { + switch (State_.Current()) { + case TState::COMMON_TS: + PARSE_ENSURE(value >= 0, "unexpected negative number in a common timestamp: " << value); + MetricConsumer_->OnCommonTime(TInstant::Seconds(value)); + State_.ToPrev(); + + if (MetricConsumer_->NeedToStop()) { + IsIntentionallyHalted_ = true; + return false; + } + + break; + + case TState::METRIC_TS: + PARSE_ENSURE(value >= 0, "unexpected negative number in a metric timestamp: " << value); + LastMetric_.SetLastTime(TInstant::Seconds(value)); + State_.ToPrev(); + break; + + case TState::METRIC_VALUE: + LastMetric_.SetLastValue(static_cast<i64>(value)); + State_.ToPrev(); + break; + + case TState::METRIC_HIST_BOUNDS: + LastMetric_.HistogramBuilder.AddBound(static_cast<double>(value)); + break; + + case TState::METRIC_HIST_BUCKETS: + PARSE_ENSURE(value >= 0 && static_cast<ui64>(value) <= Max<TBucketValues::value_type>(), "value is out of bounds " << value); + LastMetric_.HistogramBuilder.AddValue(value); + break; + + case TState::METRIC_HIST_INF: + PARSE_ENSURE(value >= 0, "unexpected negative number in histogram inf: " << value); + LastMetric_.HistogramBuilder.AddInf(value); + State_.ToPrev(); + break; + + case TState::METRIC_DSUMMARY_COUNT: + LastMetric_.SummaryBuilder.SetCount(value); + State_.ToPrev(); + break; + + case TState::METRIC_DSUMMARY_SUM: + LastMetric_.SummaryBuilder.SetSum(value); + State_.ToPrev(); + break; + case TState::METRIC_DSUMMARY_MIN: + LastMetric_.SummaryBuilder.SetMin(value); + State_.ToPrev(); + break; + case TState::METRIC_DSUMMARY_MAX: + LastMetric_.SummaryBuilder.SetMax(value); + State_.ToPrev(); + break; + case TState::METRIC_DSUMMARY_LAST: + LastMetric_.SummaryBuilder.SetLast(value); + State_.ToPrev(); + break; + + case TState::METRIC_LOG_HIST_BASE: + LastMetric_.LogHistBuilder.SetBase(value); + State_.ToPrev(); + break; + + case TState::METRIC_LOG_HIST_ZEROS: + LastMetric_.LogHistBuilder.SetZerosCount(value); + State_.ToPrev(); + break; + + case TState::METRIC_LOG_HIST_START_POWER: + LastMetric_.LogHistBuilder.SetStartPower(value); + State_.ToPrev(); + break; + + case TState::METRIC_LOG_HIST_BUCKETS: + LastMetric_.LogHistBuilder.AddBucketValue(value); + break; + + default: + return false; + } + return true; + } + + bool OnUInteger(unsigned long long value) override { + switch (State_.Current()) { + case TState::COMMON_TS: + MetricConsumer_->OnCommonTime(TInstant::Seconds(value)); + State_.ToPrev(); + + if (MetricConsumer_->NeedToStop()) { + IsIntentionallyHalted_ = true; + return false; + } + + break; + + case TState::METRIC_TS: + LastMetric_.SetLastTime(TInstant::Seconds(value)); + State_.ToPrev(); + break; + + case TState::METRIC_VALUE: + PARSE_ENSURE(value <= Max<ui64>(), "Metric value is out of bounds: " << value); + LastMetric_.SetLastValue(static_cast<ui64>(value)); + State_.ToPrev(); + break; + + case TState::METRIC_HIST_BOUNDS: + LastMetric_.HistogramBuilder.AddBound(static_cast<double>(value)); + break; + + case TState::METRIC_HIST_BUCKETS: + PARSE_ENSURE(value <= Max<TBucketValues::value_type>(), "Histogram bucket value is out of bounds: " << value); + LastMetric_.HistogramBuilder.AddValue(value); + break; + + case TState::METRIC_HIST_INF: + LastMetric_.HistogramBuilder.AddInf(value); + State_.ToPrev(); + break; + + case TState::METRIC_DSUMMARY_COUNT: + LastMetric_.SummaryBuilder.SetCount(value); + State_.ToPrev(); + break; + + case TState::METRIC_DSUMMARY_SUM: + LastMetric_.SummaryBuilder.SetSum(value); + State_.ToPrev(); + break; + case TState::METRIC_DSUMMARY_MIN: + LastMetric_.SummaryBuilder.SetMin(value); + State_.ToPrev(); + break; + case TState::METRIC_DSUMMARY_MAX: + LastMetric_.SummaryBuilder.SetMax(value); + State_.ToPrev(); + break; + case TState::METRIC_DSUMMARY_LAST: + LastMetric_.SummaryBuilder.SetLast(value); + State_.ToPrev(); + break; + + case TState::METRIC_LOG_HIST_BASE: + LastMetric_.LogHistBuilder.SetBase(value); + State_.ToPrev(); + break; + + case TState::METRIC_LOG_HIST_ZEROS: + LastMetric_.LogHistBuilder.SetZerosCount(value); + State_.ToPrev(); + break; + + case TState::METRIC_LOG_HIST_START_POWER: + LastMetric_.LogHistBuilder.SetStartPower(value); + State_.ToPrev(); + break; + + case TState::METRIC_LOG_HIST_BUCKETS: + LastMetric_.LogHistBuilder.AddBucketValue(value); + break; + + default: + return false; + } + return true; + } + + bool OnDouble(double value) override { + switch (State_.Current()) { + case TState::METRIC_VALUE: + LastMetric_.SetLastValue(value); + State_.ToPrev(); + break; + + case TState::METRIC_HIST_BOUNDS: + LastMetric_.HistogramBuilder.AddBound(value); + break; + + case TState::METRIC_DSUMMARY_SUM: + LastMetric_.SummaryBuilder.SetSum(value); + State_.ToPrev(); + break; + case TState::METRIC_DSUMMARY_MIN: + LastMetric_.SummaryBuilder.SetMin(value); + State_.ToPrev(); + break; + case TState::METRIC_DSUMMARY_MAX: + LastMetric_.SummaryBuilder.SetMax(value); + State_.ToPrev(); + break; + case TState::METRIC_DSUMMARY_LAST: + LastMetric_.SummaryBuilder.SetLast(value); + State_.ToPrev(); + break; + + case TState::METRIC_LOG_HIST_BASE: + LastMetric_.LogHistBuilder.SetBase(value); + State_.ToPrev(); + break; + + case TState::METRIC_LOG_HIST_BUCKETS: + LastMetric_.LogHistBuilder.AddBucketValue(value); + break; + + default: + return false; + } + return true; + } + + bool OnString(const TStringBuf& value) override { + switch (State_.Current()) { + case TState::COMMON_LABELS: + PARSE_ENSURE(!LastLabelName_.empty(), "empty label name in common labels"); + MetricConsumer_->OnLabel(LastLabelName_, TString{value}); + break; + + case TState::METRIC_LABELS: + PARSE_ENSURE(!LastLabelName_.empty(), "empty label name in metric labels"); + LastMetric_.Labels.Add(LastLabelName_, TString{value}); + break; + + case TState::METRIC_NAME: + PARSE_ENSURE(!value.empty(), "empty metric name"); + LastMetric_.Labels.Add(MetricNameLabel_, TString{value}); + State_.ToPrev(); + break; + + case TState::COMMON_TS: + MetricConsumer_->OnCommonTime(TInstant::ParseIso8601(value)); + State_.ToPrev(); + + if (MetricConsumer_->NeedToStop()) { + IsIntentionallyHalted_ = true; + return false; + } + + break; + + case TState::METRIC_TS: + LastMetric_.SetLastTime(TInstant::ParseIso8601(value)); + State_.ToPrev(); + break; + + case TState::METRIC_VALUE: + if (auto [doubleValue, ok] = ParseSpecDouble(value); ok) { + LastMetric_.SetLastValue(doubleValue); + } else { + return false; + } + State_.ToPrev(); + break; + + case TState::METRIC_TYPE: + LastMetric_.Type = MetricTypeFromStr(value); + State_.ToPrev(); + break; + + case TState::METRIC_MODE: + if (value == TStringBuf("deriv")) { + LastMetric_.Type = EMetricType::RATE; + } + State_.ToPrev(); + break; + + case TState::METRIC_DSUMMARY_SUM: + if (auto [doubleValue, ok] = ParseSpecDouble(value); ok) { + LastMetric_.SummaryBuilder.SetSum(doubleValue); + } else { + return false; + } + State_.ToPrev(); + break; + + case TState::METRIC_DSUMMARY_MIN: + if (auto [doubleValue, ok] = ParseSpecDouble(value); ok) { + LastMetric_.SummaryBuilder.SetMin(doubleValue); + } else { + return false; + } + State_.ToPrev(); + break; + + case TState::METRIC_DSUMMARY_MAX: + if (auto [doubleValue, ok] = ParseSpecDouble(value); ok) { + LastMetric_.SummaryBuilder.SetMax(doubleValue); + } else { + return false; + } + State_.ToPrev(); + break; + + case TState::METRIC_DSUMMARY_LAST: + if (auto [doubleValue, ok] = ParseSpecDouble(value); ok) { + LastMetric_.SummaryBuilder.SetLast(doubleValue); + } else { + return false; + } + State_.ToPrev(); + break; + + default: + return false; + } + + return true; + } + + bool OnMapKey(const TStringBuf& key) override { + switch (State_.Current()) { + case TState::ROOT_OBJECT: + if (key == TStringBuf("commonLabels") || key == TStringBuf("labels")) { + State_.ToNext(TState::COMMON_LABELS); + } else if (key == TStringBuf("ts")) { + State_.ToNext(TState::COMMON_TS); + } else if (key == TStringBuf("sensors") || key == TStringBuf("metrics")) { + State_.ToNext(TState::METRICS_ARRAY); + } + break; + + case TState::COMMON_LABELS: + case TState::METRIC_LABELS: + LastLabelName_ = key; + break; + + case TState::METRIC_OBJECT: + if (key == TStringBuf("labels")) { + State_.ToNext(TState::METRIC_LABELS); + } else if (key == TStringBuf("name")) { + State_.ToNext(TState::METRIC_NAME); + } else if (key == TStringBuf("ts")) { + PARSE_ENSURE(!LastMetric_.SeenTimeseries, + "mixed timeseries and ts attributes"); + LastMetric_.SeenTsOrValue = true; + State_.ToNext(TState::METRIC_TS); + } else if (key == TStringBuf("value")) { + PARSE_ENSURE(!LastMetric_.SeenTimeseries, + "mixed timeseries and value attributes"); + LastMetric_.SeenTsOrValue = true; + State_.ToNext(TState::METRIC_VALUE); + } else if (key == TStringBuf("timeseries")) { + PARSE_ENSURE(!LastMetric_.SeenTsOrValue, + "mixed timeseries and ts/value attributes"); + LastMetric_.SeenTimeseries = true; + State_.ToNext(TState::METRIC_TIMESERIES); + } else if (key == TStringBuf("mode")) { + State_.ToNext(TState::METRIC_MODE); + } else if (key == TStringBuf("kind") || key == TStringBuf("type")) { + State_.ToNext(TState::METRIC_TYPE); + } else if (key == TStringBuf("hist")) { + State_.ToNext(TState::METRIC_HIST); + } else if (key == TStringBuf("summary")) { + State_.ToNext(TState::METRIC_DSUMMARY); + } else if (key == TStringBuf("log_hist")) { + State_.ToNext(TState::METRIC_LOG_HIST); + } else if (key == TStringBuf("memOnly")) { + // deprecated. Skip it without errors for backward compatibility + } else { + ErrorMsg_ = TStringBuilder() << "unexpected key \"" << key << "\" in a metric schema"; + return false; + } + break; + + case TState::METRIC_TIMESERIES: + if (key == TStringBuf("ts")) { + State_.ToNext(TState::METRIC_TS); + } else if (key == TStringBuf("value")) { + State_.ToNext(TState::METRIC_VALUE); + } else if (key == TStringBuf("hist")) { + State_.ToNext(TState::METRIC_HIST); + } else if (key == TStringBuf("summary")) { + State_.ToNext(TState::METRIC_DSUMMARY); + } else if (key == TStringBuf("log_hist")) { + State_.ToNext(TState::METRIC_LOG_HIST); + } + break; + + case TState::METRIC_HIST: + if (key == TStringBuf("bounds")) { + State_.ToNext(TState::METRIC_HIST_BOUNDS); + } else if (key == TStringBuf("buckets")) { + State_.ToNext(TState::METRIC_HIST_BUCKETS); + } else if (key == TStringBuf("inf")) { + State_.ToNext(TState::METRIC_HIST_INF); + } + break; + + case TState::METRIC_LOG_HIST: + if (key == TStringBuf("base")) { + State_.ToNext(TState::METRIC_LOG_HIST_BASE); + } else if (key == TStringBuf("zeros_count")) { + State_.ToNext(TState::METRIC_LOG_HIST_ZEROS); + } else if (key == TStringBuf("start_power")) { + State_.ToNext(TState::METRIC_LOG_HIST_START_POWER); + } else if (key == TStringBuf("buckets")) { + State_.ToNext(TState::METRIC_LOG_HIST_BUCKETS); + } + break; + + case TState::METRIC_DSUMMARY: + if (key == TStringBuf("sum")) { + State_.ToNext(TState::METRIC_DSUMMARY_SUM); + } else if (key == TStringBuf("min")) { + State_.ToNext(TState::METRIC_DSUMMARY_MIN); + } else if (key == TStringBuf("max")) { + State_.ToNext(TState::METRIC_DSUMMARY_MAX); + } else if (key == TStringBuf("last")) { + State_.ToNext(TState::METRIC_DSUMMARY_LAST); + } else if (key == TStringBuf("count")) { + State_.ToNext(TState::METRIC_DSUMMARY_COUNT); + } + + break; + + + default: + return false; + } + + return true; + } + + bool OnOpenMap() override { + switch (State_.Current()) { + case TState::ROOT_OBJECT: + MetricConsumer_->OnStreamBegin(); + break; + + case TState::COMMON_LABELS: + MetricConsumer_->OnLabelsBegin(); + break; + + case TState::METRICS_ARRAY: + State_.ToNext(TState::METRIC_OBJECT); + LastMetric_.Clear(); + break; + + default: + break; + } + return true; + } + + bool OnCloseMap() override { + switch (State_.Current()) { + case TState::ROOT_OBJECT: + MetricConsumer_->OnStreamEnd(); + break; + + case TState::METRIC_LABELS: + State_.ToPrev(); + break; + + case TState::COMMON_LABELS: + MetricConsumer_->OnLabelsEnd(); + State_.ToPrev(); + + if (MetricConsumer_->NeedToStop()) { + IsIntentionallyHalted_ = true; + return false; + } + + break; + + case TState::METRIC_OBJECT: + ConsumeMetric(); + State_.ToPrev(); + break; + + case TState::METRIC_TIMESERIES: + LastMetric_.SaveLastPoint(); + break; + + case TState::METRIC_HIST: + case TState::METRIC_DSUMMARY: + case TState::METRIC_LOG_HIST: + State_.ToPrev(); + break; + + default: + break; + } + return true; + } + + bool OnOpenArray() override { + auto currentState = State_.Current(); + PARSE_ENSURE( + currentState == TState::METRICS_ARRAY || + currentState == TState::METRIC_TIMESERIES || + currentState == TState::METRIC_HIST_BOUNDS || + currentState == TState::METRIC_HIST_BUCKETS || + currentState == TState::METRIC_LOG_HIST_BUCKETS, + "unexpected array begin"); + return true; + } + + bool OnCloseArray() override { + switch (State_.Current()) { + case TState::METRICS_ARRAY: + case TState::METRIC_TIMESERIES: + case TState::METRIC_HIST_BOUNDS: + case TState::METRIC_HIST_BUCKETS: + case TState::METRIC_LOG_HIST_BUCKETS: + State_.ToPrev(); + break; + + default: + return false; + } + return true; + } + + void OnError(size_t off, TStringBuf reason) override { + if (IsIntentionallyHalted_) { + return; + } + + size_t snippetBeg = (off < 20) ? 0 : (off - 20); + TStringBuf snippet = Data_.SubStr(snippetBeg, 40); + + throw TJsonDecodeError() + << "cannot parse JSON, error at: " << off + << ", reason: " << (ErrorMsg_.empty() ? reason : TStringBuf{ErrorMsg_}) + << "\nsnippet: ..." << snippet << "..."; + } + + bool OnEnd() override { + return true; + } + + void ConsumeMetric() { + // for backwad compatibility all unknown metrics treated as gauges + if (LastMetric_.Type == EMetricType::UNKNOWN) { + if (LastMetric_.HistogramBuilder.Empty()) { + LastMetric_.Type = EMetricType::GAUGE; + } else { + LastMetric_.Type = EMetricType::HIST; + } + } + + // (1) begin metric + MetricConsumer_->OnMetricBegin(LastMetric_.Type); + + // (2) labels + if (!LastMetric_.Labels.empty()) { + MetricConsumer_->OnLabelsBegin(); + for (auto&& label : LastMetric_.Labels) { + MetricConsumer_->OnLabel(label.Name(), label.Value()); + } + MetricConsumer_->OnLabelsEnd(); + } + + // (3) values + switch (LastMetric_.Type) { + case EMetricType::GAUGE: + LastMetric_.Consume([this](TInstant time, EMetricValueType valueType, TMetricValue value) { + MetricConsumer_->OnDouble(time, value.AsDouble(valueType)); + }); + break; + + case EMetricType::IGAUGE: + LastMetric_.Consume([this](TInstant time, EMetricValueType valueType, TMetricValue value) { + MetricConsumer_->OnInt64(time, value.AsInt64(valueType)); + }); + break; + + case EMetricType::COUNTER: + case EMetricType::RATE: + LastMetric_.Consume([this](TInstant time, EMetricValueType valueType, TMetricValue value) { + MetricConsumer_->OnUint64(time, value.AsUint64(valueType)); + }); + break; + + case EMetricType::HIST: + case EMetricType::HIST_RATE: + if (LastMetric_.TimeSeries.empty()) { + auto time = LastMetric_.LastPoint.GetTime(); + auto histogram = LastMetric_.HistogramBuilder.Build(); + MetricConsumer_->OnHistogram(time, histogram); + } else { + for (const auto& p : LastMetric_.TimeSeries) { + DECODE_ENSURE(p.GetValueType() == EMetricValueType::HISTOGRAM, "Value is not a histogram"); + MetricConsumer_->OnHistogram(p.GetTime(), p.GetValue().AsHistogram()); + } + } + break; + + case EMetricType::DSUMMARY: + if (LastMetric_.TimeSeries.empty()) { + auto time = LastMetric_.LastPoint.GetTime(); + auto summary = LastMetric_.SummaryBuilder.Build(); + MetricConsumer_->OnSummaryDouble(time, summary); + } else { + for (const auto& p : LastMetric_.TimeSeries) { + DECODE_ENSURE(p.GetValueType() == EMetricValueType::SUMMARY, "Value is not a summary"); + MetricConsumer_->OnSummaryDouble(p.GetTime(), p.GetValue().AsSummaryDouble()); + } + } + break; + + case EMetricType::LOGHIST: + if (LastMetric_.TimeSeries.empty()) { + auto time = LastMetric_.LastPoint.GetTime(); + auto logHist = LastMetric_.LogHistBuilder.Build(); + MetricConsumer_->OnLogHistogram(time, logHist); + } else { + for (const auto& p : LastMetric_.TimeSeries) { + DECODE_ENSURE(p.GetValueType() == EMetricValueType::LOGHISTOGRAM, "Value is not a log_histogram"); + MetricConsumer_->OnLogHistogram(p.GetTime(), p.GetValue().AsLogHistogram()); + } + } + break; + + case EMetricType::UNKNOWN: + // TODO: output metric labels + ythrow yexception() << "unknown metric type"; + } + + // (4) end metric + MetricConsumer_->OnMetricEnd(); + } + +private: + TStringBuf Data_; + IHaltableMetricConsumer* MetricConsumer_; + TString MetricNameLabel_; + TState State_; + TString LastLabelName_; + TMetricCollector LastMetric_; + TString ErrorMsg_; + bool IsIntentionallyHalted_{false}; +}; + +} // namespace + +void DecodeJson(TStringBuf data, IMetricConsumer* c, TStringBuf metricNameLabel) { + TCommonPartsCollector commonPartsCollector; + { + TMemoryInput memIn(data); + TDecoderJson decoder(data, &commonPartsCollector, metricNameLabel); + // no need to check a return value. If there is an error, a TJsonDecodeError is thrown + NJson::ReadJson(&memIn, &decoder); + } + + TCommonPartsProxy commonPartsProxy(std::move(commonPartsCollector.CommonParts()), c); + { + TMemoryInput memIn(data); + TDecoderJson decoder(data, &commonPartsProxy, metricNameLabel); + // no need to check a return value. If there is an error, a TJsonDecodeError is thrown + NJson::ReadJson(&memIn, &decoder); + } +} + +#undef DECODE_ENSURE + +} diff --git a/library/cpp/monlib/encode/json/json_decoder_ut.cpp b/library/cpp/monlib/encode/json/json_decoder_ut.cpp new file mode 100644 index 0000000000..4464e1d26a --- /dev/null +++ b/library/cpp/monlib/encode/json/json_decoder_ut.cpp @@ -0,0 +1,179 @@ +#include "json_decoder.cpp" + +#include <library/cpp/monlib/consumers/collecting_consumer.h> +#include <library/cpp/testing/unittest/registar.h> + +#include <array> + + +using namespace NMonitoring; + +enum EJsonPart : ui8 { + METRICS = 0, + COMMON_TS = 1, + COMMON_LABELS = 2, +}; + +constexpr std::array<TStringBuf, 3> JSON_PARTS = { + TStringBuf(R"("metrics": [{ + "labels": { "key": "value" }, + "type": "GAUGE", + "value": 123 + }])"), + + TStringBuf(R"("ts": 1)"), + + TStringBuf(R"("commonLabels": { + "key1": "value1", + "key2": "value2" + })"), +}; + +TString BuildJson(std::initializer_list<EJsonPart> parts) { + TString data = "{"; + + for (auto it = parts.begin(); it != parts.end(); ++it) { + data += JSON_PARTS[*it]; + + if (it + 1 != parts.end()) { + data += ","; + } + } + + data += "}"; + return data; +} + +void ValidateCommonParts(TCommonParts&& commonParts, bool checkLabels, bool checkTs) { + if (checkTs) { + UNIT_ASSERT_VALUES_EQUAL(commonParts.CommonTime.MilliSeconds(), 1000); + } + + if (checkLabels) { + auto& labels = commonParts.CommonLabels; + UNIT_ASSERT_VALUES_EQUAL(labels.Size(), 2); + UNIT_ASSERT(labels.Has(TStringBuf("key1"))); + UNIT_ASSERT(labels.Has(TStringBuf("key2"))); + UNIT_ASSERT_VALUES_EQUAL(labels.Get(TStringBuf("key1")).value()->Value(), "value1"); + UNIT_ASSERT_VALUES_EQUAL(labels.Get(TStringBuf("key2")).value()->Value(), "value2"); + } +} + +void ValidateMetrics(const TVector<TMetricData>& metrics) { + UNIT_ASSERT_VALUES_EQUAL(metrics.size(), 1); + + auto& m = metrics[0]; + UNIT_ASSERT_VALUES_EQUAL(m.Kind, EMetricType::GAUGE); + auto& l = m.Labels; + UNIT_ASSERT_VALUES_EQUAL(l.Size(), 1); + UNIT_ASSERT_VALUES_EQUAL(l.Get(0)->Name(), "key"); + UNIT_ASSERT_VALUES_EQUAL(l.Get(0)->Value(), "value"); + + UNIT_ASSERT_VALUES_EQUAL(m.Values->Size(), 1); + UNIT_ASSERT_VALUES_EQUAL((*m.Values)[0].GetValue().AsDouble(), 123); +} + +void CheckCommonPartsCollector(TString data, bool shouldBeStopped, bool checkLabels = true, bool checkTs = true, TStringBuf metricNameLabel = "name") { + TCommonPartsCollector commonPartsCollector; + TMemoryInput memIn(data); + TDecoderJson decoder(data, &commonPartsCollector, metricNameLabel); + + bool isOk{false}; + UNIT_ASSERT_NO_EXCEPTION(isOk = NJson::ReadJson(&memIn, &decoder)); + UNIT_ASSERT_VALUES_EQUAL(isOk, !shouldBeStopped); + + ValidateCommonParts(commonPartsCollector.CommonParts(), checkLabels, checkTs); +} + +Y_UNIT_TEST_SUITE(TJsonDecoderTest) { + Y_UNIT_TEST(FullCommonParts) { + CheckCommonPartsCollector(BuildJson({COMMON_LABELS, COMMON_TS, METRICS}), true); + CheckCommonPartsCollector(BuildJson({COMMON_TS, COMMON_LABELS, METRICS}), true); + + CheckCommonPartsCollector(BuildJson({METRICS, COMMON_TS, COMMON_LABELS}), true); + CheckCommonPartsCollector(BuildJson({METRICS, COMMON_LABELS, COMMON_TS}), true); + + CheckCommonPartsCollector(BuildJson({COMMON_LABELS, METRICS, COMMON_TS}), true); + CheckCommonPartsCollector(BuildJson({COMMON_TS, METRICS, COMMON_LABELS}), true); + } + + Y_UNIT_TEST(PartialCommonParts) { + CheckCommonPartsCollector(BuildJson({COMMON_TS, METRICS}), false, false, true); + CheckCommonPartsCollector(BuildJson({COMMON_LABELS, METRICS}), false, true, false); + + CheckCommonPartsCollector(BuildJson({METRICS, COMMON_LABELS}), false, true, false); + CheckCommonPartsCollector(BuildJson({METRICS, COMMON_TS}), false, false, true); + + CheckCommonPartsCollector(BuildJson({METRICS}), false, false, false); + } + + Y_UNIT_TEST(CheckCommonPartsAndMetrics) { + auto data = BuildJson({COMMON_LABELS, COMMON_TS, METRICS}); + TCollectingConsumer collector; + + DecodeJson(data, &collector); + + TCommonParts commonParts; + commonParts.CommonTime = collector.CommonTime; + commonParts.CommonLabels = collector.CommonLabels; + + ValidateCommonParts(std::move(commonParts), true, true); + ValidateMetrics(collector.Metrics); + } + + Y_UNIT_TEST(CanParseHistogramsWithInf) { + const char* metricsData = R"({ +"metrics": + [ + { + "hist": { + "bounds": [ + 10 + ], + "buckets": [ + 11 + ], + "inf": 12 + }, + "name":"s1", + "type": "HIST_RATE" + }, + { + "hist": { + "bounds": [ + 20 + ], + "buckets": [ + 21 + ] + }, + "name":"s2", + "type":"HIST_RATE" + } + ] +})"; + TCollectingConsumer consumer(false); + DecodeJson(metricsData, &consumer); + + UNIT_ASSERT_VALUES_EQUAL(consumer.Metrics.size(), 2); + { + const auto& m = consumer.Metrics[0]; + UNIT_ASSERT_VALUES_EQUAL(m.Kind, EMetricType::HIST_RATE); + UNIT_ASSERT_VALUES_EQUAL(m.Values->Size(), 1); + const auto* histogram = (*m.Values)[0].GetValue().AsHistogram(); + UNIT_ASSERT_VALUES_EQUAL(histogram->Count(), 2); + UNIT_ASSERT_VALUES_EQUAL(histogram->UpperBound(1), Max<TBucketBound>()); + UNIT_ASSERT_VALUES_EQUAL(histogram->Value(0), 11); + UNIT_ASSERT_VALUES_EQUAL(histogram->Value(1), 12); + } + { + const auto& m = consumer.Metrics[1]; + UNIT_ASSERT_VALUES_EQUAL(m.Kind, EMetricType::HIST_RATE); + UNIT_ASSERT_VALUES_EQUAL(m.Values->Size(), 1); + const auto* histogram = (*m.Values)[0].GetValue().AsHistogram(); + UNIT_ASSERT_VALUES_EQUAL(histogram->Count(), 1); + UNIT_ASSERT_VALUES_EQUAL(histogram->UpperBound(0), 20); + UNIT_ASSERT_VALUES_EQUAL(histogram->Value(0), 21); + } + } +} diff --git a/library/cpp/monlib/encode/json/json_encoder.cpp b/library/cpp/monlib/encode/json/json_encoder.cpp new file mode 100644 index 0000000000..20d2bb6283 --- /dev/null +++ b/library/cpp/monlib/encode/json/json_encoder.cpp @@ -0,0 +1,556 @@ +#include "json.h" +#include "typed_point.h" + +#include <library/cpp/monlib/encode/buffered/buffered_encoder_base.h> +#include <library/cpp/monlib/encode/encoder_state.h> +#include <library/cpp/monlib/metrics/metric.h> +#include <library/cpp/monlib/metrics/metric_value.h> +#include <library/cpp/monlib/metrics/labels.h> + +#include <library/cpp/json/writer/json.h> + +#include <util/charset/utf8.h> +#include <util/generic/algorithm.h> + +namespace NMonitoring { + namespace { + enum class EJsonStyle { + Solomon, + Cloud + }; + + /////////////////////////////////////////////////////////////////////// + // TJsonWriter + /////////////////////////////////////////////////////////////////////// + class TJsonWriter { + public: + TJsonWriter(IOutputStream* out, int indentation, EJsonStyle style, TStringBuf metricNameLabel) + : Buf_(NJsonWriter::HEM_UNSAFE, out) + , Style_(style) + , MetricNameLabel_(metricNameLabel) + , CurrentMetricName_() + { + Buf_.SetIndentSpaces(indentation); + Buf_.SetWriteNanAsString(); + } + + void WriteTime(TInstant time) { + if (time != TInstant::Zero()) { + Buf_.WriteKey(TStringBuf("ts")); + if (Style_ == EJsonStyle::Solomon) { + Buf_.WriteULongLong(time.Seconds()); + } else { + Buf_.WriteString(time.ToString()); + } + } + } + + void WriteValue(double value) { + Buf_.WriteKey(TStringBuf("value")); + Buf_.WriteDouble(value); + } + + void WriteValue(i64 value) { + Buf_.WriteKey(TStringBuf("value")); + Buf_.WriteLongLong(value); + } + + void WriteValue(ui64 value) { + Buf_.WriteKey(TStringBuf("value")); + Buf_.WriteULongLong(value); + } + + void WriteValue(IHistogramSnapshot* s) { + Y_ENSURE(Style_ == EJsonStyle::Solomon); + + Buf_.WriteKey(TStringBuf("hist")); + Buf_.BeginObject(); + if (ui32 count = s->Count()) { + bool hasInf = (s->UpperBound(count - 1) == Max<double>()); + if (hasInf) { + count--; + } + + Buf_.WriteKey(TStringBuf("bounds")); + Buf_.BeginList(); + for (ui32 i = 0; i < count; i++) { + Buf_.WriteDouble(s->UpperBound(i)); + } + Buf_.EndList(); + + Buf_.WriteKey(TStringBuf("buckets")); + Buf_.BeginList(); + for (ui32 i = 0; i < count; i++) { + Buf_.WriteULongLong(s->Value(i)); + } + Buf_.EndList(); + + if (hasInf) { + Buf_.WriteKey(TStringBuf("inf")); + Buf_.WriteULongLong(s->Value(count)); + } + } + Buf_.EndObject(); + } + + void WriteValue(ISummaryDoubleSnapshot* s) { + Y_ENSURE(Style_ == EJsonStyle::Solomon); + + Buf_.WriteKey(TStringBuf("summary")); + Buf_.BeginObject(); + + Buf_.WriteKey(TStringBuf("sum")); + Buf_.WriteDouble(s->GetSum()); + + Buf_.WriteKey(TStringBuf("min")); + Buf_.WriteDouble(s->GetMin()); + + Buf_.WriteKey(TStringBuf("max")); + Buf_.WriteDouble(s->GetMax()); + + Buf_.WriteKey(TStringBuf("last")); + Buf_.WriteDouble(s->GetLast()); + + Buf_.WriteKey(TStringBuf("count")); + Buf_.WriteULongLong(s->GetCount()); + + Buf_.EndObject(); + } + + void WriteValue(TLogHistogramSnapshot* s) { + Y_ENSURE(Style_ == EJsonStyle::Solomon); + + Buf_.WriteKey(TStringBuf("log_hist")); + Buf_.BeginObject(); + + Buf_.WriteKey(TStringBuf("base")); + Buf_.WriteDouble(s->Base()); + + Buf_.WriteKey(TStringBuf("zeros_count")); + Buf_.WriteULongLong(s->ZerosCount()); + + Buf_.WriteKey(TStringBuf("start_power")); + Buf_.WriteInt(s->StartPower()); + + Buf_.WriteKey(TStringBuf("buckets")); + Buf_.BeginList(); + for (size_t i = 0; i < s->Count(); ++i) { + Buf_.WriteDouble(s->Bucket(i)); + } + Buf_.EndList(); + + Buf_.EndObject(); + } + + void WriteValue(EMetricValueType type, TMetricValue value) { + switch (type) { + case EMetricValueType::DOUBLE: + WriteValue(value.AsDouble()); + break; + + case EMetricValueType::INT64: + WriteValue(value.AsInt64()); + break; + + case EMetricValueType::UINT64: + WriteValue(value.AsUint64()); + break; + + case EMetricValueType::HISTOGRAM: + WriteValue(value.AsHistogram()); + break; + + case EMetricValueType::SUMMARY: + WriteValue(value.AsSummaryDouble()); + break; + + case EMetricValueType::LOGHISTOGRAM: + WriteValue(value.AsLogHistogram()); + break; + + case EMetricValueType::UNKNOWN: + ythrow yexception() << "unknown metric value type"; + } + } + + void WriteLabel(TStringBuf name, TStringBuf value) { + Y_ENSURE(IsUtf(name), "label name is not valid UTF-8 string"); + Y_ENSURE(IsUtf(value), "label value is not valid UTF-8 string"); + if (Style_ == EJsonStyle::Cloud && name == MetricNameLabel_) { + CurrentMetricName_ = value; + } else { + Buf_.WriteKey(name); + Buf_.WriteString(value); + } + } + + void WriteMetricType(EMetricType type) { + if (Style_ == EJsonStyle::Cloud) { + Buf_.WriteKey("type"); + Buf_.WriteString(MetricTypeToCloudStr(type)); + } else { + Buf_.WriteKey("kind"); + Buf_.WriteString(MetricTypeToStr(type)); + } + } + + void WriteName() { + if (Style_ != EJsonStyle::Cloud) { + return; + } + if (CurrentMetricName_.Empty()) { + ythrow yexception() << "label '" << MetricNameLabel_ << "' is not defined"; + } + Buf_.WriteKey("name"); + Buf_.WriteString(CurrentMetricName_); + CurrentMetricName_.clear(); + } + + private: + static TStringBuf MetricTypeToCloudStr(EMetricType type) { + switch (type) { + case EMetricType::GAUGE: + return TStringBuf("DGAUGE"); + case EMetricType::COUNTER: + return TStringBuf("COUNTER"); + case EMetricType::RATE: + return TStringBuf("RATE"); + case EMetricType::IGAUGE: + return TStringBuf("IGAUGE"); + default: + ythrow yexception() << "metric type '" << type << "' is not supported by cloud json format"; + } + } + + protected: + NJsonWriter::TBuf Buf_; + EJsonStyle Style_; + TString MetricNameLabel_; + TString CurrentMetricName_; + }; + + /////////////////////////////////////////////////////////////////////// + // TEncoderJson + /////////////////////////////////////////////////////////////////////// + class TEncoderJson final: public IMetricEncoder, public TJsonWriter { + public: + TEncoderJson(IOutputStream* out, int indentation, EJsonStyle style, TStringBuf metricNameLabel) + : TJsonWriter{out, indentation, style, metricNameLabel} + { + } + + ~TEncoderJson() override { + Close(); + } + + private: + void OnStreamBegin() override { + State_.Expect(TEncoderState::EState::ROOT); + Buf_.BeginObject(); + } + + void OnStreamEnd() override { + State_.Expect(TEncoderState::EState::ROOT); + if (!Buf_.KeyExpected()) { + // not closed metrics array + Buf_.EndList(); + } + Buf_.EndObject(); + } + + void OnCommonTime(TInstant time) override { + State_.Expect(TEncoderState::EState::ROOT); + WriteTime(time); + } + + void OnMetricBegin(EMetricType type) override { + State_.Switch(TEncoderState::EState::ROOT, TEncoderState::EState::METRIC); + if (Buf_.KeyExpected()) { + // first metric, so open metrics array + Buf_.WriteKey(TStringBuf(Style_ == EJsonStyle::Solomon ? "sensors" : "metrics")); + Buf_.BeginList(); + } + Buf_.BeginObject(); + WriteMetricType(type); + } + + void OnMetricEnd() override { + State_.Switch(TEncoderState::EState::METRIC, TEncoderState::EState::ROOT); + if (!Buf_.KeyExpected()) { + // not closed timeseries array + Buf_.EndList(); + } + + if (!TimeSeries_ && LastPoint_.HasValue()) { + // we have seen only one point between OnMetricBegin() and + // OnMetricEnd() calls + WriteTime(LastPoint_.GetTime()); + WriteValue(LastPoint_.GetValueType(), LastPoint_.GetValue()); + } + Buf_.EndObject(); + + LastPoint_ = {}; + TimeSeries_ = false; + } + + void OnLabelsBegin() override { + if (!Buf_.KeyExpected()) { + // not closed metrics or timeseries array if labels go after values + Buf_.EndList(); + } + if (State_ == TEncoderState::EState::ROOT) { + State_ = TEncoderState::EState::COMMON_LABELS; + Buf_.WriteKey(TStringBuf(Style_ == EJsonStyle::Solomon ? "commonLabels" : "labels")); + } else if (State_ == TEncoderState::EState::METRIC) { + State_ = TEncoderState::EState::METRIC_LABELS; + Buf_.WriteKey(TStringBuf("labels")); + } else { + State_.ThrowInvalid("expected METRIC or ROOT"); + } + Buf_.BeginObject(); + + EmptyLabels_ = true; + } + + void OnLabelsEnd() override { + if (State_ == TEncoderState::EState::METRIC_LABELS) { + State_ = TEncoderState::EState::METRIC; + } else if (State_ == TEncoderState::EState::COMMON_LABELS) { + State_ = TEncoderState::EState::ROOT; + } else { + State_.ThrowInvalid("expected LABELS or COMMON_LABELS"); + } + + Y_ENSURE(!EmptyLabels_, "Labels cannot be empty"); + Buf_.EndObject(); + if (State_ == TEncoderState::EState::METRIC) { + WriteName(); + } + } + + void OnLabel(TStringBuf name, TStringBuf value) override { + if (State_ == TEncoderState::EState::METRIC_LABELS || State_ == TEncoderState::EState::COMMON_LABELS) { + WriteLabel(name, value); + } else { + State_.ThrowInvalid("expected LABELS or COMMON_LABELS"); + } + + EmptyLabels_ = false; + } + + void OnDouble(TInstant time, double value) override { + State_.Expect(TEncoderState::EState::METRIC); + Write<double>(time, value); + } + + void OnInt64(TInstant time, i64 value) override { + State_.Expect(TEncoderState::EState::METRIC); + Write<i64>(time, value); + } + + void OnUint64(TInstant time, ui64 value) override { + State_.Expect(TEncoderState::EState::METRIC); + Write<ui64>(time, value); + } + + void OnHistogram(TInstant time, IHistogramSnapshotPtr snapshot) override { + State_.Expect(TEncoderState::EState::METRIC); + Write<IHistogramSnapshot*>(time, snapshot.Get()); + } + + void OnSummaryDouble(TInstant time, ISummaryDoubleSnapshotPtr snapshot) override { + State_.Expect(TEncoderState::EState::METRIC); + Write<ISummaryDoubleSnapshot*>(time, snapshot.Get()); + } + + void OnLogHistogram(TInstant time, TLogHistogramSnapshotPtr snapshot) override { + State_.Expect(TEncoderState::EState::METRIC); + Write<TLogHistogramSnapshot*>(time, snapshot.Get()); + } + + template <typename T> + void Write(TInstant time, T value) { + State_.Expect(TEncoderState::EState::METRIC); + + if (!LastPoint_.HasValue()) { + LastPoint_ = {time, value}; + } else { + // second point + // TODO: output types + Y_ENSURE(LastPoint_.GetValueType() == TValueType<T>::Type, + "mixed metric value types in one metric"); + + if (!TimeSeries_) { + Buf_.WriteKey(TStringBuf("timeseries")); + Buf_.BeginList(); + Buf_.BeginObject(); + Y_ENSURE(LastPoint_.GetTime() != TInstant::Zero(), + "time cannot be empty or zero in a timeseries point"); + WriteTime(LastPoint_.GetTime()); + WriteValue(LastPoint_.GetValueType(), LastPoint_.GetValue()); + Buf_.EndObject(); + TimeSeries_ = true; + } + + if (TimeSeries_) { + Buf_.BeginObject(); + Y_ENSURE(time != TInstant::Zero(), + "time cannot be empty or zero in a timeseries point"); + + WriteTime(time); + WriteValue(value); + Buf_.EndObject(); + } + } + } + + void Close() override { + LastPoint_ = {}; + } + + private: + TEncoderState State_; + TTypedPoint LastPoint_; + bool TimeSeries_ = false; + bool EmptyLabels_ = false; + }; + + /////////////////////////////////////////////////////////////////////// + // TBufferedJsonEncoder + /////////////////////////////////////////////////////////////////////// + class TBufferedJsonEncoder : public TBufferedEncoderBase, public TJsonWriter { + public: + TBufferedJsonEncoder(IOutputStream* out, int indentation, EJsonStyle style, TStringBuf metricNameLabel) + : TJsonWriter{out, indentation, style, metricNameLabel} + { + MetricsMergingMode_ = EMetricsMergingMode::MERGE_METRICS; + } + + ~TBufferedJsonEncoder() override { + Close(); + } + + void OnLabelsBegin() override { + TBufferedEncoderBase::OnLabelsBegin(); + EmptyLabels_ = true; + } + + void OnLabel(TStringBuf name, TStringBuf value) override { + TBufferedEncoderBase::OnLabel(name, value); + EmptyLabels_ = false; + } + + void OnLabel(ui32 name, ui32 value) override { + TBufferedEncoderBase::OnLabel(name, value); + EmptyLabels_ = false; + } + + void OnLabelsEnd() override { + TBufferedEncoderBase::OnLabelsEnd(); + Y_ENSURE(!EmptyLabels_, "Labels cannot be empty"); + } + + void Close() final { + if (Closed_) { + return; + } + + Closed_ = true; + + LabelValuesPool_.Build(); + LabelNamesPool_.Build(); + + Buf_.BeginObject(); + + WriteTime(CommonTime_); + if (CommonLabels_.size() > 0) { + Buf_.WriteKey(TStringBuf(Style_ == EJsonStyle::Solomon ? "commonLabels": "labels")); + WriteLabels(CommonLabels_, true); + } + + if (Metrics_.size() > 0) { + Buf_.WriteKey(TStringBuf(Style_ == EJsonStyle::Solomon ? "sensors" : "metrics")); + WriteMetrics(); + } + + Buf_.EndObject(); + } + + private: + void WriteMetrics() { + Buf_.BeginList(); + for (auto&& metric : Metrics_) { + WriteMetric(metric); + } + Buf_.EndList(); + } + + void WriteMetric(TMetric& metric) { + Buf_.BeginObject(); + + WriteMetricType(metric.MetricType); + + Buf_.WriteKey(TStringBuf("labels")); + WriteLabels(metric.Labels, false); + + metric.TimeSeries.SortByTs(); + if (metric.TimeSeries.Size() == 1) { + const auto& point = metric.TimeSeries[0]; + WriteTime(point.GetTime()); + WriteValue(metric.TimeSeries.GetValueType(), point.GetValue()); + } else if (metric.TimeSeries.Size() > 1) { + Buf_.WriteKey(TStringBuf("timeseries")); + Buf_.BeginList(); + metric.TimeSeries.ForEach([this](TInstant time, EMetricValueType type, TMetricValue value) { + Buf_.BeginObject(); + // make gcc 6.1 happy https://gcc.gnu.org/bugzilla/show_bug.cgi?id=61636 + this->WriteTime(time); + this->WriteValue(type, value); + Buf_.EndObject(); + }); + + Buf_.EndList(); + } + + Buf_.EndObject(); + } + + void WriteLabels(const TPooledLabels& labels, bool isCommon) { + Buf_.BeginObject(); + + for (auto i = 0u; i < labels.size(); ++i) { + TStringBuf name = LabelNamesPool_.Get(labels[i].Key->Index); + TStringBuf value = LabelValuesPool_.Get(labels[i].Value->Index); + + WriteLabel(name, value); + } + + Buf_.EndObject(); + + if (!isCommon) { + WriteName(); + } + } + + private: + bool Closed_{false}; + bool EmptyLabels_ = false; + }; + } + + IMetricEncoderPtr EncoderJson(IOutputStream* out, int indentation) { + return MakeHolder<TEncoderJson>(out, indentation, EJsonStyle::Solomon, ""); + } + + IMetricEncoderPtr BufferedEncoderJson(IOutputStream* out, int indentation) { + return MakeHolder<TBufferedJsonEncoder>(out, indentation, EJsonStyle::Solomon, ""); + } + + IMetricEncoderPtr EncoderCloudJson(IOutputStream* out, int indentation, TStringBuf metricNameLabel) { + return MakeHolder<TEncoderJson>(out, indentation, EJsonStyle::Cloud, metricNameLabel); + } + + IMetricEncoderPtr BufferedEncoderCloudJson(IOutputStream* out, int indentation, TStringBuf metricNameLabel) { + return MakeHolder<TBufferedJsonEncoder>(out, indentation, EJsonStyle::Cloud, metricNameLabel); + } +} diff --git a/library/cpp/monlib/encode/json/json_ut.cpp b/library/cpp/monlib/encode/json/json_ut.cpp new file mode 100644 index 0000000000..09e7909289 --- /dev/null +++ b/library/cpp/monlib/encode/json/json_ut.cpp @@ -0,0 +1,1290 @@ +#include "json.h" + +#include <library/cpp/monlib/encode/protobuf/protobuf.h> +#include <library/cpp/monlib/metrics/labels.h> + +#include <library/cpp/json/json_reader.h> +#include <library/cpp/resource/resource.h> +#include <library/cpp/testing/unittest/registar.h> + +#include <util/stream/str.h> +#include <util/string/builder.h> + +#include <limits> + +using namespace NMonitoring; + +namespace NMonitoring { + bool operator<(const TLabel& lhs, const TLabel& rhs) { + return lhs.Name() < rhs.Name() || + (lhs.Name() == rhs.Name() && lhs.Value() < rhs.Value()); + } +} +namespace { + void AssertLabels(const NProto::TMultiSample& actual, const TLabels& expected) { + UNIT_ASSERT_EQUAL(actual.LabelsSize(), expected.Size()); + + TSet<TLabel> actualSet; + TSet<TLabel> expectedSet; + Transform(expected.begin(), expected.end(), std::inserter(expectedSet, expectedSet.end()), [] (auto&& l) { + return TLabel{l.Name(), l.Value()}; + }); + + const auto& l = actual.GetLabels(); + Transform(std::begin(l), std::end(l), std::inserter(actualSet, std::begin(actualSet)), + [](auto&& elem) -> TLabel { + return {elem.GetName(), elem.GetValue()}; + }); + + TVector<TLabel> diff; + SetSymmetricDifference(std::begin(expectedSet), std::end(expectedSet), + std::begin(actualSet), std::end(actualSet), std::back_inserter(diff)); + + if (diff.size() > 0) { + for (auto&& l : diff) { + Cerr << l << Endl; + } + + UNIT_FAIL("Labels don't match"); + } + } + + void AssertLabelEqual(const NProto::TLabel& l, TStringBuf name, TStringBuf value) { + UNIT_ASSERT_STRINGS_EQUAL(l.GetName(), name); + UNIT_ASSERT_STRINGS_EQUAL(l.GetValue(), value); + } + + void AssertPointEqual(const NProto::TPoint& p, TInstant time, double value) { + UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds()); + UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kFloat64); + UNIT_ASSERT_DOUBLES_EQUAL(p.GetFloat64(), value, std::numeric_limits<double>::epsilon()); + } + + void AssertPointEqualNan(const NProto::TPoint& p, TInstant time) { + UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds()); + UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kFloat64); + UNIT_ASSERT(std::isnan(p.GetFloat64())); + } + + void AssertPointEqualInf(const NProto::TPoint& p, TInstant time, int sign) { + UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds()); + UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kFloat64); + UNIT_ASSERT(std::isinf(p.GetFloat64())); + if (sign < 0) { + UNIT_ASSERT(p.GetFloat64() < 0); + } + } + + void AssertPointEqual(const NProto::TPoint& p, TInstant time, ui64 value) { + UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds()); + UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kUint64); + UNIT_ASSERT_VALUES_EQUAL(p.GetUint64(), value); + } + + void AssertPointEqual(const NProto::TPoint& p, TInstant time, i64 value) { + UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds()); + UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kInt64); + UNIT_ASSERT_VALUES_EQUAL(p.GetInt64(), value); + } + + void AssertPointEqual(const NProto::TPoint& p, TInstant time, const IHistogramSnapshot& expected) { + UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds()); + UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kHistogram); + + const NProto::THistogram& h = p.GetHistogram(); + UNIT_ASSERT_VALUES_EQUAL(h.BoundsSize(), expected.Count()); + UNIT_ASSERT_VALUES_EQUAL(h.ValuesSize(), expected.Count()); + + for (size_t i = 0; i < h.BoundsSize(); i++) { + UNIT_ASSERT_DOUBLES_EQUAL(h.GetBounds(i), expected.UpperBound(i), Min<double>()); + UNIT_ASSERT_VALUES_EQUAL(h.GetValues(i), expected.Value(i)); + } + } + + void AssertPointEqual(const NProto::TPoint& p, TInstant time, const TLogHistogramSnapshot& expected) { + UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds()); + UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kLogHistogram); + + const double eps = 1e-10; + const NProto::TLogHistogram& h = p.GetLogHistogram(); + + UNIT_ASSERT_DOUBLES_EQUAL(h.GetBase(), expected.Base(), eps); + UNIT_ASSERT_VALUES_EQUAL(h.GetZerosCount(), expected.ZerosCount()); + UNIT_ASSERT_VALUES_EQUAL(h.GetStartPower(), expected.StartPower()); + UNIT_ASSERT_VALUES_EQUAL(h.BucketsSize(), expected.Count()); + for (size_t i = 0; i < expected.Count(); ++i) { + UNIT_ASSERT_DOUBLES_EQUAL(h.GetBuckets(i), expected.Bucket(i), eps); + } + } + + void AssertPointEqual(const NProto::TPoint& p, TInstant time, const ISummaryDoubleSnapshot& expected) { + UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds()); + UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kSummaryDouble); + auto actual = p.GetSummaryDouble(); + const double eps = 1e-10; + UNIT_ASSERT_DOUBLES_EQUAL(actual.GetSum(), expected.GetSum(), eps); + UNIT_ASSERT_DOUBLES_EQUAL(actual.GetMin(), expected.GetMin(), eps); + UNIT_ASSERT_DOUBLES_EQUAL(actual.GetMax(), expected.GetMax(), eps); + UNIT_ASSERT_DOUBLES_EQUAL(actual.GetLast(), expected.GetLast(), eps); + UNIT_ASSERT_VALUES_EQUAL(actual.GetCount(), expected.GetCount()); + } + +} // namespace + + +Y_UNIT_TEST_SUITE(TJsonTest) { + const TInstant now = TInstant::ParseIso8601Deprecated("2017-11-05T01:02:03Z"); + + Y_UNIT_TEST(Encode) { + auto check = [](bool cloud, bool buffered, TStringBuf expectedResourceKey) { + TString json; + TStringOutput out(json); + auto e = cloud + ? (buffered ? BufferedEncoderCloudJson(&out, 2, "metric") : EncoderCloudJson(&out, 2, "metric")) + : (buffered ? BufferedEncoderJson(&out, 2) : EncoderJson(&out, 2)); + e->OnStreamBegin(); + { // common time + e->OnCommonTime(TInstant::Seconds(1500000000)); + } + { // common labels + e->OnLabelsBegin(); + e->OnLabel("project", "solomon"); + e->OnLabelsEnd(); + } + { // metric #1 + e->OnMetricBegin(EMetricType::COUNTER); + { + e->OnLabelsBegin(); + e->OnLabel("metric", "single"); + e->OnLabel("labels", "l1"); + e->OnLabelsEnd(); + } + e->OnUint64(now, 17); + e->OnMetricEnd(); + } + { // metric #2 + e->OnMetricBegin(EMetricType::RATE); + { + e->OnLabelsBegin(); + e->OnLabel("metric", "single"); + e->OnLabel("labels", "l2"); + e->OnLabelsEnd(); + } + e->OnUint64(now, 17); + e->OnMetricEnd(); + } + { // metric #3 + e->OnMetricBegin(EMetricType::GAUGE); + e->OnDouble(now, 3.14); + { + e->OnLabelsBegin(); + e->OnLabel("metric", "single"); + e->OnLabel("labels", "l3"); + e->OnLabelsEnd(); + } + e->OnMetricEnd(); + } + { // metric #4 + e->OnMetricBegin(EMetricType::IGAUGE); + e->OnInt64(now, 42); + { + e->OnLabelsBegin(); + e->OnLabel("metric", "single_igauge"); + e->OnLabel("labels", "l4"); + e->OnLabelsEnd(); + } + e->OnMetricEnd(); + } + { // metric #5 + e->OnMetricBegin(EMetricType::GAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("metric", "multiple"); + e->OnLabel("labels", "l5"); + e->OnLabelsEnd(); + } + e->OnDouble(now, std::numeric_limits<double>::quiet_NaN()); + e->OnDouble(now + TDuration::Seconds(15), std::numeric_limits<double>::infinity()); + e->OnDouble(now + TDuration::Seconds(30), -std::numeric_limits<double>::infinity()); + e->OnMetricEnd(); + } + + { // metric #6 + e->OnMetricBegin(EMetricType::COUNTER); + e->OnUint64(now, 1337); + e->OnUint64(now + TDuration::Seconds(15), 1338); + { + e->OnLabelsBegin(); + e->OnLabel("metric", "multiple"); + e->OnLabel("labels", "l6"); + e->OnLabelsEnd(); + } + e->OnMetricEnd(); + } + e->OnStreamEnd(); + e->Close(); + json += "\n"; + + auto parseJson = [] (auto buf) { + NJson::TJsonValue value; + NJson::ReadJsonTree(buf, &value, true); + return value; + }; + + const auto expectedJson = NResource::Find(expectedResourceKey); + UNIT_ASSERT_EQUAL(parseJson(json), parseJson(expectedJson)); + }; + + check(false, false, "/expected.json"); + check(false, true, "/expected_buffered.json"); + check(true, false, "/expected_cloud.json"); + check(true, true, "/expected_cloud_buffered.json"); + } + + TLogHistogramSnapshotPtr TestLogHistogram(ui32 v = 1) { + TVector<double> buckets{0.5 * v, 0.25 * v, 0.25 * v, 0.5 * v}; + return MakeIntrusive<TLogHistogramSnapshot>(1.5, 1u, 0, std::move(buckets)); + } + + Y_UNIT_TEST(HistogramAndSummaryMetricTypesAreNotSupportedByCloudJson) { + const TInstant now = TInstant::ParseIso8601Deprecated("2017-11-05T01:02:03Z"); + + auto emit = [&](IMetricEncoder* encoder, EMetricType metricType) { + encoder->OnStreamBegin(); + { + encoder->OnMetricBegin(metricType); + { + encoder->OnLabelsBegin(); + encoder->OnLabel("name", "m"); + encoder->OnLabelsEnd(); + } + + switch (metricType) { + case EMetricType::HIST: { + auto histogram = ExponentialHistogram(6, 2); + encoder->OnHistogram(now, histogram->Snapshot()); + break; + } + case EMetricType::LOGHIST: { + auto histogram = TestLogHistogram(); + encoder->OnLogHistogram(now, histogram); + break; + } + case EMetricType::DSUMMARY: { + auto summary = MakeIntrusive<TSummaryDoubleSnapshot>(10., -0.5, 0.5, 0.3, 30u); + encoder->OnSummaryDouble(now, summary); + break; + } + default: + Y_FAIL("unexpected metric type [%s]", ToString(metricType).c_str()); + } + + encoder->OnMetricEnd(); + } + encoder->OnStreamEnd(); + encoder->Close(); + }; + + auto doTest = [&](bool buffered, EMetricType metricType) { + TString json; + TStringOutput out(json); + auto encoder = buffered ? BufferedEncoderCloudJson(&out, 2) : EncoderCloudJson(&out, 2); + const TString expectedMessage = TStringBuilder() + << "metric type '" << metricType << "' is not supported by cloud json format"; + UNIT_ASSERT_EXCEPTION_CONTAINS_C(emit(encoder.Get(), metricType), yexception, expectedMessage, + TString("buffered: ") + ToString(buffered)); + }; + + doTest(false, EMetricType::HIST); + doTest(false, EMetricType::LOGHIST); + doTest(false, EMetricType::DSUMMARY); + doTest(true, EMetricType::HIST); + doTest(true, EMetricType::LOGHIST); + doTest(true, EMetricType::DSUMMARY); + } + + Y_UNIT_TEST(MetricsWithDifferentLabelOrderGetMerged) { + TString json; + TStringOutput out(json); + auto e = BufferedEncoderJson(&out, 2); + + e->OnStreamBegin(); + { + e->OnMetricBegin(EMetricType::RATE); + { + e->OnLabelsBegin(); + e->OnLabel("metric", "hello"); + e->OnLabel("label", "world"); + e->OnLabelsEnd(); + } + e->OnUint64(TInstant::Zero(), 0); + e->OnMetricEnd(); + } + { + e->OnMetricBegin(EMetricType::RATE); + { + e->OnLabelsBegin(); + e->OnLabel("label", "world"); + e->OnLabel("metric", "hello"); + e->OnLabelsEnd(); + } + e->OnUint64(TInstant::Zero(), 1); + e->OnMetricEnd(); + } + e->OnStreamEnd(); + e->Close(); + json += "\n"; + + TString expectedJson = NResource::Find("/merged.json"); + // we cannot be sure regarding the label order in the result, + // so we'll have to parse the expected value and then compare it with actual + + NProto::TMultiSamplesList samples; + IMetricEncoderPtr d = EncoderProtobuf(&samples); + DecodeJson(expectedJson, d.Get()); + + UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 1); + { + const NProto::TMultiSample& s = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::RATE); + AssertLabels(s, TLabels{{"metric", "hello"}, {"label", "world"}}); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + AssertPointEqual(s.GetPoints(0), TInstant::Zero(), ui64(1)); + } + } + Y_UNIT_TEST(Decode1) { + NProto::TMultiSamplesList samples; + { + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + TString testJson = NResource::Find("/expected.json"); + DecodeJson(testJson, e.Get()); + } + + UNIT_ASSERT_VALUES_EQUAL( + TInstant::MilliSeconds(samples.GetCommonTime()), + TInstant::Seconds(1500000000)); + + UNIT_ASSERT_VALUES_EQUAL(samples.CommonLabelsSize(), 1); + AssertLabelEqual(samples.GetCommonLabels(0), "project", "solomon"); + + UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 6); + { + const NProto::TMultiSample& s = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::COUNTER); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2); + AssertLabelEqual(s.GetLabels(0), "metric", "single"); + AssertLabelEqual(s.GetLabels(1), "labels", "l1"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + AssertPointEqual(s.GetPoints(0), now, ui64(17)); + } + { + const NProto::TMultiSample& s = samples.GetSamples(1); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::RATE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2); + AssertLabelEqual(s.GetLabels(0), "metric", "single"); + AssertLabelEqual(s.GetLabels(1), "labels", "l2"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + AssertPointEqual(s.GetPoints(0), now, ui64(17)); + } + { + const NProto::TMultiSample& s = samples.GetSamples(2); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2); + AssertLabelEqual(s.GetLabels(0), "metric", "single"); + AssertLabelEqual(s.GetLabels(1), "labels", "l3"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + AssertPointEqual(s.GetPoints(0), now, 3.14); + } + { + const NProto::TMultiSample& s = samples.GetSamples(3); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::IGAUGE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2); + AssertLabelEqual(s.GetLabels(0), "metric", "single_igauge"); + AssertLabelEqual(s.GetLabels(1), "labels", "l4"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + AssertPointEqual(s.GetPoints(0), now, i64(42)); + } + { + const NProto::TMultiSample& s = samples.GetSamples(4); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2); + AssertLabelEqual(s.GetLabels(0), "metric", "multiple"); + AssertLabelEqual(s.GetLabels(1), "labels", "l5"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 3); + AssertPointEqualNan(s.GetPoints(0), now); + AssertPointEqualInf(s.GetPoints(1), now + TDuration::Seconds(15), 1); + AssertPointEqualInf(s.GetPoints(2), now + TDuration::Seconds(30), -11); + } + { + const NProto::TMultiSample& s = samples.GetSamples(5); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::COUNTER); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2); + AssertLabelEqual(s.GetLabels(0), "metric", "multiple"); + AssertLabelEqual(s.GetLabels(1), "labels", "l6"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 2); + AssertPointEqual(s.GetPoints(0), now, ui64(1337)); + AssertPointEqual(s.GetPoints(1), now + TDuration::Seconds(15), ui64(1338)); + } + } + + Y_UNIT_TEST(DecodeMetrics) { + NProto::TMultiSamplesList samples; + { + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + TString metricsJson = NResource::Find("/metrics.json"); + DecodeJson(metricsJson, e.Get()); + } + + UNIT_ASSERT_VALUES_EQUAL( + TInstant::MilliSeconds(samples.GetCommonTime()), + TInstant::ParseIso8601Deprecated("2017-08-27T12:34:56Z")); + + UNIT_ASSERT_VALUES_EQUAL(samples.CommonLabelsSize(), 3); + AssertLabelEqual(samples.GetCommonLabels(0), "project", "solomon"); + AssertLabelEqual(samples.GetCommonLabels(1), "cluster", "man"); + AssertLabelEqual(samples.GetCommonLabels(2), "service", "stockpile"); + + UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 4); + { + const NProto::TMultiSample& s = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "metric", "Memory"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + AssertPointEqual(s.GetPoints(0), TInstant::Zero(), 10.0); + } + { + const NProto::TMultiSample& s = samples.GetSamples(1); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::RATE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "metric", "UserTime"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + AssertPointEqual(s.GetPoints(0), TInstant::Zero(), ui64(1)); + } + { + const NProto::TMultiSample& s = samples.GetSamples(2); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2); + AssertLabelEqual(s.GetLabels(0), "export", "Oxygen"); + AssertLabelEqual(s.GetLabels(1), "metric", "QueueSize"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + auto ts = TInstant::ParseIso8601Deprecated("2017-11-05T12:34:56.000Z"); + AssertPointEqual(s.GetPoints(0), ts, 3.14159); + } + { + const NProto::TMultiSample& s = samples.GetSamples(3); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "metric", "Writes"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 2); + auto ts1 = TInstant::ParseIso8601Deprecated("2017-08-28T12:32:11Z"); + AssertPointEqual(s.GetPoints(0), ts1, -10.0); + auto ts2 = TInstant::Seconds(1503923187); + AssertPointEqual(s.GetPoints(1), ts2, 20.0); + } + } + + Y_UNIT_TEST(DecodeSensors) { + NProto::TMultiSamplesList samples; + { + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + TString sensorsJson = NResource::Find("/sensors.json"); + DecodeJson(sensorsJson, e.Get()); + } + + UNIT_ASSERT_VALUES_EQUAL( + TInstant::MilliSeconds(samples.GetCommonTime()), + TInstant::ParseIso8601Deprecated("2017-08-27T12:34:56Z")); + + UNIT_ASSERT_VALUES_EQUAL(samples.CommonLabelsSize(), 3); + AssertLabelEqual(samples.GetCommonLabels(0), "project", "solomon"); + AssertLabelEqual(samples.GetCommonLabels(1), "cluster", "man"); + AssertLabelEqual(samples.GetCommonLabels(2), "service", "stockpile"); + + UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 4); + { + const NProto::TMultiSample& s = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "metric", "Memory"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + AssertPointEqual(s.GetPoints(0), TInstant::Zero(), 10.0); + } + { + const NProto::TMultiSample& s = samples.GetSamples(1); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::RATE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "metric", "UserTime"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + AssertPointEqual(s.GetPoints(0), TInstant::Zero(), ui64(1)); + } + { + const NProto::TMultiSample& s = samples.GetSamples(2); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2); + AssertLabelEqual(s.GetLabels(0), "export", "Oxygen"); + AssertLabelEqual(s.GetLabels(1), "metric", "QueueSize"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + auto ts = TInstant::ParseIso8601Deprecated("2017-11-05T12:34:56.000Z"); + AssertPointEqual(s.GetPoints(0), ts, 3.14159); + } + { + const NProto::TMultiSample& s = samples.GetSamples(3); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "metric", "Writes"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 2); + auto ts1 = TInstant::ParseIso8601Deprecated("2017-08-28T12:32:11Z"); + AssertPointEqual(s.GetPoints(0), ts1, -10.0); + auto ts2 = TInstant::Seconds(1503923187); + AssertPointEqual(s.GetPoints(1), ts2, 20.0); + } + } + + Y_UNIT_TEST(DecodeToEncoder) { + auto testJson = NResource::Find("/test_decode_to_encode.json"); + + TStringStream Stream_; + auto encoder = BufferedEncoderJson(&Stream_, 4); + DecodeJson(testJson, encoder.Get()); + + encoder->Close(); + + auto val1 = NJson::ReadJsonFastTree(testJson, true); + auto val2 = NJson::ReadJsonFastTree(Stream_.Str(), true); + + UNIT_ASSERT_VALUES_EQUAL(val1, val2); + } + + void WriteEmptySeries(const IMetricEncoderPtr& e) { + e->OnStreamBegin(); + { + e->OnMetricBegin(EMetricType::COUNTER); + { + e->OnLabelsBegin(); + e->OnLabel("foo", "bar"); + e->OnLabelsEnd(); + } + e->OnMetricEnd(); + } + + e->OnStreamEnd(); + e->Close(); + } + + Y_UNIT_TEST(EncodeEmptySeries) { + TString json; + TStringOutput out(json); + + auto e = EncoderJson(&out, 2); + WriteEmptySeries(e); + json += "\n"; + + TString expectedJson = NResource::Find("/empty_series.json"); + UNIT_ASSERT_NO_DIFF(json, expectedJson); + } + + void WriteEmptyLabels(IMetricEncoderPtr& e) { + e->OnStreamBegin(); + e->OnMetricBegin(EMetricType::COUNTER); + + e->OnLabelsBegin(); + UNIT_ASSERT_EXCEPTION(e->OnLabelsEnd(), yexception); + } + + Y_UNIT_TEST(LabelsCannotBeEmpty) { + TString json; + TStringOutput out(json); + + auto e = EncoderJson(&out, 2); + WriteEmptyLabels(e); + } + + Y_UNIT_TEST(LabelsCannotBeEmptyBuffered) { + TString json; + TStringOutput out(json); + + auto e = BufferedEncoderJson(&out, 2); + WriteEmptyLabels(e); + } + + Y_UNIT_TEST(EncodeEmptySeriesBuffered) { + TString json; + TStringOutput out(json); + + auto e = BufferedEncoderJson(&out, 2); + WriteEmptySeries(e); + json += "\n"; + + TString expectedJson = NResource::Find("/empty_series.json"); + UNIT_ASSERT_NO_DIFF(json, expectedJson); + } + + Y_UNIT_TEST(BufferedEncoderMergesMetrics) { + TString json; + TStringOutput out(json); + + auto e = BufferedEncoderJson(&out, 2); + auto ts = 1; + + auto writeMetric = [&] (const TString& val) { + e->OnMetricBegin(EMetricType::COUNTER); + + e->OnLabelsBegin(); + e->OnLabel("foo", val); + e->OnLabelsEnd(); + e->OnUint64(TInstant::Seconds(ts++), 42); + + e->OnMetricEnd(); + }; + + e->OnStreamBegin(); + writeMetric("bar"); + writeMetric("bar"); + writeMetric("baz"); + writeMetric("bar"); + e->OnStreamEnd(); + e->Close(); + + json += "\n"; + + TString expectedJson = NResource::Find("/buffered_test.json"); + UNIT_ASSERT_NO_DIFF(json, expectedJson); + } + + Y_UNIT_TEST(JsonEncoderDisallowsValuesInTimeseriesWithoutTs) { + TStringStream out; + + auto e = EncoderJson(&out); + auto writePreamble = [&] { + e->OnStreamBegin(); + e->OnMetricBegin(EMetricType::COUNTER); + e->OnLabelsBegin(); + e->OnLabel("foo", "bar"); + e->OnLabelsEnd(); + }; + + // writing two values for a metric in a row will trigger + // timeseries object construction + writePreamble(); + e->OnUint64(TInstant::Zero(), 42); + UNIT_ASSERT_EXCEPTION(e->OnUint64(TInstant::Zero(), 42), yexception); + + e = EncoderJson(&out); + writePreamble(); + e->OnUint64(TInstant::Zero(), 42); + UNIT_ASSERT_EXCEPTION(e->OnUint64(TInstant::Now(), 42), yexception); + + e = EncoderJson(&out); + writePreamble(); + e->OnUint64(TInstant::Now(), 42); + UNIT_ASSERT_EXCEPTION(e->OnUint64(TInstant::Zero(), 42), yexception); + } + + Y_UNIT_TEST(BufferedJsonEncoderMergesTimeseriesWithoutTs) { + TStringStream out; + + { + auto e = BufferedEncoderJson(&out, 2); + e->OnStreamBegin(); + e->OnMetricBegin(EMetricType::COUNTER); + e->OnLabelsBegin(); + e->OnLabel("foo", "bar"); + e->OnLabelsEnd(); + // in buffered mode we are able to find values with same (in this case zero) + // timestamp and discard duplicates + e->OnUint64(TInstant::Zero(), 42); + e->OnUint64(TInstant::Zero(), 43); + e->OnUint64(TInstant::Zero(), 44); + e->OnUint64(TInstant::Zero(), 45); + e->OnMetricEnd(); + e->OnStreamEnd(); + } + + out << "\n"; + UNIT_ASSERT_NO_DIFF(out.Str(), NResource::Find("/buffered_ts_merge.json")); + } + + template <typename TFactory, typename TConsumer> + TString EncodeToString(TFactory factory, TConsumer consumer) { + TStringStream out; + { + IMetricEncoderPtr e = factory(&out, 2); + consumer(e.Get()); + } + out << '\n'; + return out.Str(); + } + + Y_UNIT_TEST(SummaryValueEncode) { + auto writeDocument = [](IMetricEncoder* e) { + e->OnStreamBegin(); + { + e->OnMetricBegin(EMetricType::DSUMMARY); + { + e->OnLabelsBegin(); + e->OnLabel("metric", "temperature"); + e->OnLabelsEnd(); + } + + e->OnSummaryDouble(now, MakeIntrusive<TSummaryDoubleSnapshot>(10., -0.5, 0.5, 0.3, 30u)); + e->OnMetricEnd(); + } + e->OnStreamEnd(); + }; + + TString result1 = EncodeToString(EncoderJson, writeDocument); + UNIT_ASSERT_NO_DIFF(result1, NResource::Find("/summary_value.json")); + + TString result2 = EncodeToString(BufferedEncoderJson, writeDocument); + UNIT_ASSERT_NO_DIFF(result2, NResource::Find("/summary_value.json")); + } + + ISummaryDoubleSnapshotPtr TestInfSummary() { + return MakeIntrusive<TSummaryDoubleSnapshot>( + std::numeric_limits<double>::quiet_NaN(), + -std::numeric_limits<double>::infinity(), + std::numeric_limits<double>::infinity(), + 0.3, + 30u); + } + + Y_UNIT_TEST(SummaryInfEncode) { + auto writeDocument = [](IMetricEncoder* e) { + e->OnStreamBegin(); + { + e->OnMetricBegin(EMetricType::DSUMMARY); + { + e->OnLabelsBegin(); + e->OnLabel("metric", "temperature"); + e->OnLabelsEnd(); + } + + e->OnSummaryDouble(now, TestInfSummary()); + e->OnMetricEnd(); + } + e->OnStreamEnd(); + }; + + TString result1 = EncodeToString(EncoderJson, writeDocument); + UNIT_ASSERT_NO_DIFF(result1, NResource::Find("/summary_inf.json")); + + TString result2 = EncodeToString(BufferedEncoderJson, writeDocument); + UNIT_ASSERT_NO_DIFF(result2, NResource::Find("/summary_inf.json")); + } + + Y_UNIT_TEST(SummaryInfDecode) { + NProto::TMultiSamplesList samples; + { + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + TString testJson = NResource::Find("/summary_inf.json"); + DecodeJson(testJson, e.Get()); + } + + UNIT_ASSERT_VALUES_EQUAL(1, samples.SamplesSize()); + const NProto::TMultiSample& s = samples.GetSamples(0); + + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::DSUMMARY); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "metric", "temperature"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + + auto actual = s.GetPoints(0).GetSummaryDouble(); + UNIT_ASSERT(std::isnan(actual.GetSum())); + UNIT_ASSERT(actual.GetMin() < 0); + UNIT_ASSERT(std::isinf(actual.GetMin())); + UNIT_ASSERT(actual.GetMax() > 0); + UNIT_ASSERT(std::isinf(actual.GetMax())); + } + + Y_UNIT_TEST(SummaryValueDecode) { + NProto::TMultiSamplesList samples; + { + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + TString testJson = NResource::Find("/summary_value.json"); + DecodeJson(testJson, e.Get()); + } + + UNIT_ASSERT_VALUES_EQUAL(1, samples.SamplesSize()); + const NProto::TMultiSample& s = samples.GetSamples(0); + + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::DSUMMARY); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "metric", "temperature"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + + auto snapshot = TSummaryDoubleSnapshot(10., -0.5, 0.5, 0.3, 30u); + AssertPointEqual(s.GetPoints(0), now, snapshot); + } + + Y_UNIT_TEST(SummaryTimeSeriesEncode) { + auto writeDocument = [](IMetricEncoder* e) { + e->OnStreamBegin(); + { + e->OnMetricBegin(EMetricType::DSUMMARY); + { + e->OnLabelsBegin(); + e->OnLabel("metric", "temperature"); + e->OnLabelsEnd(); + } + + TSummaryDoubleCollector summary; + summary.Collect(0.3); + summary.Collect(-0.5); + summary.Collect(1.); + + e->OnSummaryDouble(now, summary.Snapshot()); + + summary.Collect(-1.5); + summary.Collect(0.01); + + e->OnSummaryDouble(now + TDuration::Seconds(15), summary.Snapshot()); + + e->OnMetricEnd(); + } + e->OnStreamEnd(); + }; + + TString result1 = EncodeToString(EncoderJson, writeDocument); + UNIT_ASSERT_NO_DIFF(result1, NResource::Find("/summary_timeseries.json")); + + TString result2 = EncodeToString(BufferedEncoderJson, writeDocument); + UNIT_ASSERT_NO_DIFF(result2, NResource::Find("/summary_timeseries.json")); + } + + Y_UNIT_TEST(SummaryTimeSeriesDecode) { + NProto::TMultiSamplesList samples; + { + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + TString testJson = NResource::Find("/summary_timeseries.json"); + DecodeJson(testJson, e.Get()); + } + + UNIT_ASSERT_VALUES_EQUAL(1, samples.SamplesSize()); + const NProto::TMultiSample& s = samples.GetSamples(0); + + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::DSUMMARY); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "metric", "temperature"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 2); + + TSummaryDoubleCollector summary; + summary.Collect(0.3); + summary.Collect(-0.5); + summary.Collect(1.); + + AssertPointEqual(s.GetPoints(0), now, *summary.Snapshot()); + + summary.Collect(-1.5); + summary.Collect(0.01); + + AssertPointEqual(s.GetPoints(1), now + TDuration::Seconds(15), *summary.Snapshot()); + } + + Y_UNIT_TEST(LogHistogramValueEncode) { + auto writeDocument = [](IMetricEncoder* e) { + e->OnStreamBegin(); + { + e->OnMetricBegin(EMetricType::LOGHIST); + { + e->OnLabelsBegin(); + e->OnLabel("metric", "ms"); + e->OnLabelsEnd(); + } + + e->OnLogHistogram(now, TestLogHistogram()); + e->OnMetricEnd(); + } + e->OnStreamEnd(); + }; + + TString result1 = EncodeToString(EncoderJson, writeDocument); + UNIT_ASSERT_NO_DIFF(result1, NResource::Find("/log_histogram_value.json")); + + TString result2 = EncodeToString(BufferedEncoderJson, writeDocument); + UNIT_ASSERT_NO_DIFF(result2, NResource::Find("/log_histogram_value.json")); + } + + Y_UNIT_TEST(LogHistogramValueDecode) { + NProto::TMultiSamplesList samples; + { + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + TString testJson = NResource::Find("/log_histogram_value.json"); + DecodeJson(testJson, e.Get()); + } + + UNIT_ASSERT_VALUES_EQUAL(1, samples.SamplesSize()); + const NProto::TMultiSample& s = samples.GetSamples(0); + + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::LOGHISTOGRAM); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "metric", "ms"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + + auto snapshot = TestLogHistogram(); + AssertPointEqual(s.GetPoints(0), now, *snapshot); + } + + Y_UNIT_TEST(HistogramValueEncode) { + auto writeDocument = [](IMetricEncoder* e) { + e->OnStreamBegin(); + { + e->OnMetricBegin(EMetricType::HIST); + { + e->OnLabelsBegin(); + e->OnLabel("metric", "responseTimeMillis"); + e->OnLabelsEnd(); + } + + // {1: 1, 2: 1, 4: 2, 8: 4, 16: 8, inf: 83} + auto h = ExponentialHistogram(6, 2); + for (i64 i = 1; i < 100; i++) { + h->Collect(i); + } + + e->OnHistogram(now, h->Snapshot()); + e->OnMetricEnd(); + } + e->OnStreamEnd(); + }; + + TString result1 = EncodeToString(EncoderJson, writeDocument); + UNIT_ASSERT_NO_DIFF(result1, NResource::Find("/histogram_value.json")); + + TString result2 = EncodeToString(BufferedEncoderJson, writeDocument); + UNIT_ASSERT_NO_DIFF(result2, NResource::Find("/histogram_value.json")); + } + + Y_UNIT_TEST(LogHistogramTimeSeriesEncode) { + auto writeDocument = [](IMetricEncoder* e) { + e->OnStreamBegin(); + { + e->OnMetricBegin(EMetricType::LOGHIST); + { + e->OnLabelsBegin(); + e->OnLabel("metric", "ms"); + e->OnLabelsEnd(); + } + + e->OnLogHistogram(now, TestLogHistogram(1));; + + e->OnLogHistogram(now + TDuration::Seconds(15), TestLogHistogram(2)); + + e->OnMetricEnd(); + } + e->OnStreamEnd(); + }; + + TString result1 = EncodeToString(EncoderJson, writeDocument); + UNIT_ASSERT_NO_DIFF(result1, NResource::Find("/log_histogram_timeseries.json")); + + TString result2 = EncodeToString(BufferedEncoderJson, writeDocument); + UNIT_ASSERT_NO_DIFF(result2, NResource::Find("/log_histogram_timeseries.json")); + } + + Y_UNIT_TEST(LogHistogramTimeSeriesDecode) { + NProto::TMultiSamplesList samples; + { + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + TString testJson = NResource::Find("/log_histogram_timeseries.json"); + DecodeJson(testJson, e.Get()); + } + + UNIT_ASSERT_VALUES_EQUAL(1, samples.SamplesSize()); + const NProto::TMultiSample& s = samples.GetSamples(0); + + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::LOGHISTOGRAM); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "metric", "ms"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 2); + + auto logHist = TestLogHistogram(1); + AssertPointEqual(s.GetPoints(0), now, *logHist); + + logHist = TestLogHistogram(2); + AssertPointEqual(s.GetPoints(1), now + TDuration::Seconds(15), *logHist); + } + + void HistogramValueDecode(const TString& filePath) { + NProto::TMultiSamplesList samples; + { + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + TString testJson = NResource::Find(filePath); + DecodeJson(testJson, e.Get()); + } + + UNIT_ASSERT_VALUES_EQUAL(1, samples.SamplesSize()); + const NProto::TMultiSample& s = samples.GetSamples(0); + + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::HISTOGRAM); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "metric", "responseTimeMillis"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + + auto h = ExponentialHistogram(6, 2); + for (i64 i = 1; i < 100; i++) { + h->Collect(i); + } + + AssertPointEqual(s.GetPoints(0), now, *h->Snapshot()); + } + + Y_UNIT_TEST(HistogramValueDecode) { + HistogramValueDecode("/histogram_value.json"); + HistogramValueDecode("/histogram_value_inf_before_bounds.json"); + } + + Y_UNIT_TEST(HistogramTimeSeriesEncode) { + auto writeDocument = [](IMetricEncoder* e) { + e->OnStreamBegin(); + { + e->OnMetricBegin(EMetricType::HIST_RATE); + { + e->OnLabelsBegin(); + e->OnLabel("metric", "responseTimeMillis"); + e->OnLabelsEnd(); + } + + // {1: 1, 2: 1, 4: 2, 8: 4, 16: 8, inf: 83} + auto h = ExponentialHistogram(6, 2); + for (i64 i = 1; i < 100; i++) { + h->Collect(i); + } + e->OnHistogram(now, h->Snapshot()); + + // {1: 2, 2: 2, 4: 4, 8: 8, 16: 16, inf: 166} + for (i64 i = 1; i < 100; i++) { + h->Collect(i); + } + e->OnHistogram(now + TDuration::Seconds(15), h->Snapshot()); + + e->OnMetricEnd(); + } + e->OnStreamEnd(); + }; + + TString result1 = EncodeToString(EncoderJson, writeDocument); + UNIT_ASSERT_NO_DIFF(result1, NResource::Find("/histogram_timeseries.json")); + + TString result2 = EncodeToString(BufferedEncoderJson, writeDocument); + UNIT_ASSERT_NO_DIFF(result2, NResource::Find("/histogram_timeseries.json")); + } + + Y_UNIT_TEST(HistogramTimeSeriesDecode) { + NProto::TMultiSamplesList samples; + { + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + TString testJson = NResource::Find("/histogram_timeseries.json"); + DecodeJson(testJson, e.Get()); + } + + UNIT_ASSERT_VALUES_EQUAL(1, samples.SamplesSize()); + const NProto::TMultiSample& s = samples.GetSamples(0); + + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::HIST_RATE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "metric", "responseTimeMillis"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 2); + + auto h = ExponentialHistogram(6, 2); + for (i64 i = 1; i < 100; i++) { + h->Collect(i); + } + + AssertPointEqual(s.GetPoints(0), now, *h->Snapshot()); + + for (i64 i = 1; i < 100; i++) { + h->Collect(i); + } + + AssertPointEqual(s.GetPoints(1), now + TDuration::Seconds(15), *h->Snapshot()); + } + + Y_UNIT_TEST(IntGaugeEncode) { + auto writeDocument = [](IMetricEncoder* e) { + e->OnStreamBegin(); + { + e->OnMetricBegin(EMetricType::IGAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("metric", "a"); + e->OnLabelsEnd(); + } + e->OnInt64(now, Min<i64>()); + e->OnInt64(now + TDuration::Seconds(1), -1); + e->OnInt64(now + TDuration::Seconds(2), 0); + e->OnInt64(now + TDuration::Seconds(3), Max<i64>()); + e->OnMetricEnd(); + } + e->OnStreamEnd(); + }; + + TString result1 = EncodeToString(EncoderJson, writeDocument); + UNIT_ASSERT_NO_DIFF(result1, NResource::Find("/int_gauge.json")); + + TString result2 = EncodeToString(BufferedEncoderJson, writeDocument); + UNIT_ASSERT_NO_DIFF(result2, NResource::Find("/int_gauge.json")); + } + + Y_UNIT_TEST(InconsistentMetricTypes) { + auto emitMetrics = [](IMetricEncoder& encoder, const TString& expectedError) { + encoder.OnMetricBegin(EMetricType::GAUGE); + { + encoder.OnLabelsBegin(); + encoder.OnLabel("name", "m"); + encoder.OnLabel("l1", "v1"); + encoder.OnLabel("l2", "v2"); + encoder.OnLabelsEnd(); + } + encoder.OnDouble(now, 1.0); + encoder.OnMetricEnd(); + + encoder.OnMetricBegin(EMetricType::COUNTER); + { + encoder.OnLabelsBegin(); + encoder.OnLabel("name", "m"); + encoder.OnLabel("l1", "v1"); + encoder.OnLabel("l2", "v2"); + encoder.OnLabelsEnd(); + } + encoder.OnUint64(now, 1); + + UNIT_ASSERT_EXCEPTION_CONTAINS(encoder.OnMetricEnd(), + yexception, + expectedError); + }; + + { + TStringStream out; + auto encoder = BufferedEncoderJson(&out); + + encoder->OnStreamBegin(); + encoder->OnLabelsBegin(); + encoder->OnLabel("c", "cv"); + encoder->OnLabelsEnd(); + emitMetrics(*encoder, + "Time series point type mismatch: expected DOUBLE but found UINT64, " + "labels '{c=cv, l1=v1, l2=v2, name=m}'"); + } + + { + TStringStream out; + auto encoder = BufferedEncoderJson(&out); + + encoder->OnStreamBegin(); + encoder->OnLabelsBegin(); + encoder->OnLabel("l1", "v100"); + encoder->OnLabelsEnd(); + emitMetrics(*encoder, + "Time series point type mismatch: expected DOUBLE but found UINT64, " + "labels '{l1=v1, l2=v2, name=m}'"); + } + } + + Y_UNIT_TEST(IntGaugeDecode) { + NProto::TMultiSamplesList samples; + { + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + TString testJson = NResource::Find("/int_gauge.json"); + DecodeJson(testJson, e.Get()); + } + + UNIT_ASSERT_VALUES_EQUAL(1, samples.SamplesSize()); + const NProto::TMultiSample& s = samples.GetSamples(0); + + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::IGAUGE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "metric", "a"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 4); + AssertPointEqual(s.GetPoints(0), now, Min<i64>()); + AssertPointEqual(s.GetPoints(1), now + TDuration::Seconds(1), i64(-1)); + AssertPointEqual(s.GetPoints(2), now + TDuration::Seconds(2), i64(0)); + AssertPointEqual(s.GetPoints(3), now + TDuration::Seconds(3), Max<i64>()); + } + + Y_UNIT_TEST(FuzzerRegression) { + NProto::TMultiSamplesList samples; + { + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + for (auto f : { "/hist_crash.json", "/crash.json" }) { + TString testJson = NResource::Find(f); + UNIT_ASSERT_EXCEPTION(DecodeJson(testJson, e.Get()), yexception); + } + } + } + + Y_UNIT_TEST(LegacyNegativeRateThrows) { + const auto input = R"({ + "sensors": [ + { + "mode": "deriv", + "value": -1, + "labels": { "metric": "SystemTime" } + }, + } + ]}")"; + + NProto::TMultiSamplesList samples; + IMetricEncoderPtr e = EncoderProtobuf(&samples); + UNIT_ASSERT_EXCEPTION(DecodeJson(input, e.Get()), yexception); + } + + Y_UNIT_TEST(DecodeNamedMetrics) { + NProto::TMultiSamplesList samples; + { + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + TString metricsJson = NResource::Find("/named_metrics.json"); + DecodeJson(metricsJson, e.Get(), "sensor"); + } + + UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 2); + { + const NProto::TMultiSample& s = samples.GetSamples(0); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "sensor", "Memory"); + } + { + const NProto::TMultiSample& s = samples.GetSamples(1); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2); + AssertLabelEqual(s.GetLabels(0), "sensor", "QueueSize"); + AssertLabelEqual(s.GetLabels(1), "export", "Oxygen"); + } + } + +} diff --git a/library/cpp/monlib/encode/json/typed_point.h b/library/cpp/monlib/encode/json/typed_point.h new file mode 100644 index 0000000000..fbaa840c4b --- /dev/null +++ b/library/cpp/monlib/encode/json/typed_point.h @@ -0,0 +1,123 @@ +#pragma once + +#include <library/cpp/monlib/metrics/metric_value.h> + + +namespace NMonitoring { + + class TTypedPoint { + public: + TTypedPoint() + : Time_(TInstant::Zero()) + , ValueType_(EMetricValueType::UNKNOWN) + { + } + + template <typename T> + TTypedPoint(TInstant time, T value) + : Time_(time) + , ValueType_(TValueType<T>::Type) + , Value_(value) + { + Ref(); + } + + ~TTypedPoint() { + UnRef(); + } + + TTypedPoint(const TTypedPoint& rhs) + : Time_(rhs.Time_) + , ValueType_(rhs.ValueType_) + , Value_(rhs.Value_) + { + Ref(); + } + + TTypedPoint& operator=(const TTypedPoint& rhs) { + UnRef(); + + Time_ = rhs.Time_; + ValueType_ = rhs.ValueType_; + Value_ = rhs.Value_; + + Ref(); + return *this; + } + + TTypedPoint(TTypedPoint&& rhs) noexcept + : Time_(rhs.Time_) + , ValueType_(rhs.ValueType_) + , Value_(rhs.Value_) + { + rhs.ValueType_ = EMetricValueType::UNKNOWN; + rhs.Value_ = {}; + } + + TTypedPoint& operator=(TTypedPoint&& rhs) noexcept { + UnRef(); + + Time_ = rhs.Time_; + ValueType_ = rhs.ValueType_; + Value_ = rhs.Value_; + + rhs.ValueType_ = EMetricValueType::UNKNOWN; + rhs.Value_ = {}; + + return *this; + } + + TInstant GetTime() const noexcept { + return Time_; + } + + void SetTime(TInstant time) noexcept { + Time_ = time; + } + + TMetricValue GetValue() const noexcept { + return Value_; + } + + EMetricValueType GetValueType() const noexcept { + return ValueType_; + } + + template <typename T> + void SetValue(T value) noexcept { + ValueType_ = TValueType<T>::Type; + Value_ = TMetricValue{value}; + } + + bool HasValue() { + return ValueType_ != EMetricValueType::UNKNOWN; + } + + private: + void Ref() { + if (ValueType_ == EMetricValueType::HISTOGRAM) { + Value_.AsHistogram()->Ref(); + } else if (ValueType_ == EMetricValueType::SUMMARY) { + Value_.AsSummaryDouble()->Ref(); + } else if (ValueType_ == EMetricValueType::LOGHISTOGRAM) { + Value_.AsLogHistogram()->Ref(); + } + } + + void UnRef() { + if (ValueType_ == EMetricValueType::HISTOGRAM) { + Value_.AsHistogram()->UnRef(); + } else if (ValueType_ == EMetricValueType::SUMMARY) { + Value_.AsSummaryDouble()->UnRef(); + } else if (ValueType_ == EMetricValueType::LOGHISTOGRAM) { + Value_.AsLogHistogram()->UnRef(); + } + } + + private: + TInstant Time_; + EMetricValueType ValueType_; + TMetricValue Value_; + }; + +} diff --git a/library/cpp/monlib/encode/json/ut/buffered_test.json b/library/cpp/monlib/encode/json/ut/buffered_test.json new file mode 100644 index 0000000000..53212cf8e1 --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/buffered_test.json @@ -0,0 +1,36 @@ +{ + "sensors": + [ + { + "kind":"COUNTER", + "labels": + { + "foo":"bar" + }, + "timeseries": + [ + { + "ts":1, + "value":42 + }, + { + "ts":2, + "value":42 + }, + { + "ts":4, + "value":42 + } + ] + }, + { + "kind":"COUNTER", + "labels": + { + "foo":"baz" + }, + "ts":3, + "value":42 + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/buffered_ts_merge.json b/library/cpp/monlib/encode/json/ut/buffered_ts_merge.json new file mode 100644 index 0000000000..1d27efacb0 --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/buffered_ts_merge.json @@ -0,0 +1,13 @@ +{ + "sensors": + [ + { + "kind":"COUNTER", + "labels": + { + "foo":"bar" + }, + "value":45 + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/crash.json b/library/cpp/monlib/encode/json/ut/crash.json Binary files differnew file mode 100644 index 0000000000..8ff4369dc4 --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/crash.json diff --git a/library/cpp/monlib/encode/json/ut/empty_series.json b/library/cpp/monlib/encode/json/ut/empty_series.json new file mode 100644 index 0000000000..641e10cdea --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/empty_series.json @@ -0,0 +1,12 @@ +{ + "sensors": + [ + { + "kind":"COUNTER", + "labels": + { + "foo":"bar" + } + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/expected.json b/library/cpp/monlib/encode/json/ut/expected.json new file mode 100644 index 0000000000..ead853455b --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/expected.json @@ -0,0 +1,92 @@ +{ + "ts":1500000000, + "commonLabels": + { + "project":"solomon" + }, + "sensors": + [ + { + "kind":"COUNTER", + "labels": + { + "metric":"single", + "labels":"l1" + }, + "ts":1509843723, + "value":17 + }, + { + "kind":"RATE", + "labels": + { + "metric":"single", + "labels":"l2" + }, + "ts":1509843723, + "value":17 + }, + { + "kind":"GAUGE", + "labels": + { + "metric":"single", + "labels":"l3" + }, + "ts":1509843723, + "value":3.14 + }, + { + "kind":"IGAUGE", + "labels": + { + "metric":"single_igauge", + "labels":"l4" + }, + "ts":1509843723, + "value":42 + }, + { + "kind":"GAUGE", + "labels": + { + "metric":"multiple", + "labels":"l5" + }, + "timeseries": + [ + { + "ts":1509843723, + "value":"nan" + }, + { + "ts":1509843738, + "value":"inf" + }, + { + "ts":1509843753, + "value":"-inf" + } + ] + }, + { + "kind":"COUNTER", + "timeseries": + [ + { + "ts":1509843723, + "value":1337 + }, + { + "ts":1509843738, + "value":1338 + } + ], + "labels": + { + "metric":"multiple", + "labels":"l6" + } + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/expected_buffered.json b/library/cpp/monlib/encode/json/ut/expected_buffered.json new file mode 100644 index 0000000000..9a6a1d6201 --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/expected_buffered.json @@ -0,0 +1,92 @@ +{ + "ts":1500000000, + "commonLabels": + { + "project":"solomon" + }, + "sensors": + [ + { + "kind":"COUNTER", + "labels": + { + "labels":"l1", + "metric":"single" + }, + "ts":1509843723, + "value":17 + }, + { + "kind":"RATE", + "labels": + { + "labels":"l2", + "metric":"single" + }, + "ts":1509843723, + "value":17 + }, + { + "kind":"GAUGE", + "labels": + { + "labels":"l3", + "metric":"single" + }, + "ts":1509843723, + "value":3.14 + }, + { + "kind":"IGAUGE", + "labels": + { + "labels":"l4", + "metric":"single_igauge" + }, + "ts":1509843723, + "value":42 + }, + { + "kind":"GAUGE", + "labels": + { + "labels":"l5", + "metric":"multiple" + }, + "timeseries": + [ + { + "ts":1509843723, + "value":"nan" + }, + { + "ts":1509843738, + "value":"inf" + }, + { + "ts":1509843753, + "value":"-inf" + } + ] + }, + { + "kind":"COUNTER", + "labels": + { + "labels":"l6", + "metric":"multiple" + }, + "timeseries": + [ + { + "ts":1509843723, + "value":1337 + }, + { + "ts":1509843738, + "value":1338 + } + ] + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/expected_cloud.json b/library/cpp/monlib/encode/json/ut/expected_cloud.json new file mode 100644 index 0000000000..6184811579 --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/expected_cloud.json @@ -0,0 +1,92 @@ +{ + "ts":"2017-07-14T02:40:00.000000Z", + "labels": + { + "project":"solomon" + }, + "metrics": + [ + { + "type":"COUNTER", + "labels": + { + "labels":"l1" + }, + "name":"single", + "ts":"2017-11-05T01:02:03.000000Z", + "value":17 + }, + { + "type":"RATE", + "labels": + { + "labels":"l2" + }, + "name":"single", + "ts":"2017-11-05T01:02:03.000000Z", + "value":17 + }, + { + "type":"DGAUGE", + "labels": + { + "labels":"l3" + }, + "name":"single", + "ts":"2017-11-05T01:02:03.000000Z", + "value":3.14 + }, + { + "type":"IGAUGE", + "labels": + { + "labels":"l4" + }, + "name":"single_igauge", + "ts":"2017-11-05T01:02:03.000000Z", + "value":42 + }, + { + "type":"DGAUGE", + "labels": + { + "labels":"l5" + }, + "name":"multiple", + "timeseries": + [ + { + "ts":"2017-11-05T01:02:03.000000Z", + "value":"nan" + }, + { + "ts":"2017-11-05T01:02:18.000000Z", + "value":"inf" + }, + { + "ts":"2017-11-05T01:02:33.000000Z", + "value":"-inf" + } + ] + }, + { + "type":"COUNTER", + "timeseries": + [ + { + "ts":"2017-11-05T01:02:03.000000Z", + "value":1337 + }, + { + "ts":"2017-11-05T01:02:18.000000Z", + "value":1338 + } + ], + "labels": + { + "labels":"l6" + }, + "name":"multiple" + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/expected_cloud_buffered.json b/library/cpp/monlib/encode/json/ut/expected_cloud_buffered.json new file mode 100644 index 0000000000..be237d522b --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/expected_cloud_buffered.json @@ -0,0 +1,92 @@ +{ + "ts":"2017-07-14T02:40:00.000000Z", + "labels": + { + "project":"solomon" + }, + "metrics": + [ + { + "type":"COUNTER", + "labels": + { + "labels":"l1" + }, + "name":"single", + "ts":"2017-11-05T01:02:03.000000Z", + "value":17 + }, + { + "type":"RATE", + "labels": + { + "labels":"l2" + }, + "name":"single", + "ts":"2017-11-05T01:02:03.000000Z", + "value":17 + }, + { + "type":"DGAUGE", + "labels": + { + "labels":"l3" + }, + "name":"single", + "ts":"2017-11-05T01:02:03.000000Z", + "value":3.14 + }, + { + "type":"IGAUGE", + "labels": + { + "labels":"l4" + }, + "name":"single_igauge", + "ts":"2017-11-05T01:02:03.000000Z", + "value":42 + }, + { + "type":"DGAUGE", + "labels": + { + "labels":"l5" + }, + "name":"multiple", + "timeseries": + [ + { + "ts":"2017-11-05T01:02:03.000000Z", + "value":"nan" + }, + { + "ts":"2017-11-05T01:02:18.000000Z", + "value":"inf" + }, + { + "ts":"2017-11-05T01:02:33.000000Z", + "value":"-inf" + } + ] + }, + { + "type":"COUNTER", + "labels": + { + "labels":"l6" + }, + "name":"multiple", + "timeseries": + [ + { + "ts":"2017-11-05T01:02:03.000000Z", + "value":1337 + }, + { + "ts":"2017-11-05T01:02:18.000000Z", + "value":1338 + } + ] + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/hist_crash.json b/library/cpp/monlib/encode/json/ut/hist_crash.json Binary files differnew file mode 100644 index 0000000000..867d0fce7d --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/hist_crash.json diff --git a/library/cpp/monlib/encode/json/ut/histogram_timeseries.json b/library/cpp/monlib/encode/json/ut/histogram_timeseries.json new file mode 100644 index 0000000000..f6131ffded --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/histogram_timeseries.json @@ -0,0 +1,61 @@ +{ + "sensors": + [ + { + "kind":"HIST_RATE", + "labels": + { + "metric":"responseTimeMillis" + }, + "timeseries": + [ + { + "ts":1509843723, + "hist": + { + "bounds": + [ + 1, + 2, + 4, + 8, + 16 + ], + "buckets": + [ + 1, + 1, + 2, + 4, + 8 + ], + "inf":83 + } + }, + { + "ts":1509843738, + "hist": + { + "bounds": + [ + 1, + 2, + 4, + 8, + 16 + ], + "buckets": + [ + 2, + 2, + 4, + 8, + 16 + ], + "inf":166 + } + } + ] + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/histogram_value.json b/library/cpp/monlib/encode/json/ut/histogram_value.json new file mode 100644 index 0000000000..ec1ae5cdec --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/histogram_value.json @@ -0,0 +1,33 @@ +{ + "sensors": + [ + { + "kind":"HIST", + "labels": + { + "metric":"responseTimeMillis" + }, + "ts":1509843723, + "hist": + { + "bounds": + [ + 1, + 2, + 4, + 8, + 16 + ], + "buckets": + [ + 1, + 1, + 2, + 4, + 8 + ], + "inf":83 + } + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/histogram_value_inf_before_bounds.json b/library/cpp/monlib/encode/json/ut/histogram_value_inf_before_bounds.json new file mode 100644 index 0000000000..f8a17c8831 --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/histogram_value_inf_before_bounds.json @@ -0,0 +1,33 @@ +{ + "sensors": + [ + { + "kind":"HIST", + "labels": + { + "metric":"responseTimeMillis" + }, + "ts":1509843723, + "hist": + { + "inf":83, + "bounds": + [ + 1, + 2, + 4, + 8, + 16 + ], + "buckets": + [ + 1, + 1, + 2, + 4, + 8 + ] + } + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/int_gauge.json b/library/cpp/monlib/encode/json/ut/int_gauge.json new file mode 100644 index 0000000000..fbe57f873c --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/int_gauge.json @@ -0,0 +1,31 @@ +{ + "sensors": + [ + { + "kind":"IGAUGE", + "labels": + { + "metric":"a" + }, + "timeseries": + [ + { + "ts":1509843723, + "value":-9223372036854775808 + }, + { + "ts":1509843724, + "value":-1 + }, + { + "ts":1509843725, + "value":0 + }, + { + "ts":1509843726, + "value":9223372036854775807 + } + ] + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/log_histogram_timeseries.json b/library/cpp/monlib/encode/json/ut/log_histogram_timeseries.json new file mode 100644 index 0000000000..e811a2cc57 --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/log_histogram_timeseries.json @@ -0,0 +1,47 @@ +{ + "sensors": + [ + { + "kind":"LOGHIST", + "labels": + { + "metric":"ms" + }, + "timeseries": + [ + { + "ts":1509843723, + "log_hist": + { + "base":1.5, + "zeros_count":1, + "start_power":0, + "buckets": + [ + 0.5, + 0.25, + 0.25, + 0.5 + ] + } + }, + { + "ts":1509843738, + "log_hist": + { + "base":1.5, + "zeros_count":1, + "start_power":0, + "buckets": + [ + 1, + 0.5, + 0.5, + 1 + ] + } + } + ] + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/log_histogram_value.json b/library/cpp/monlib/encode/json/ut/log_histogram_value.json new file mode 100644 index 0000000000..002478293b --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/log_histogram_value.json @@ -0,0 +1,26 @@ +{ + "sensors": + [ + { + "kind":"LOGHIST", + "labels": + { + "metric":"ms" + }, + "ts":1509843723, + "log_hist": + { + "base":1.5, + "zeros_count":1, + "start_power":0, + "buckets": + [ + 0.5, + 0.25, + 0.25, + 0.5 + ] + } + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/merged.json b/library/cpp/monlib/encode/json/ut/merged.json new file mode 100644 index 0000000000..ea2c99a33c --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/merged.json @@ -0,0 +1,14 @@ +{ + "sensors": + [ + { + "kind":"RATE", + "labels": + { + "metric":"hello", + "label":"world" + }, + "value":1 + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/metrics.json b/library/cpp/monlib/encode/json/ut/metrics.json new file mode 100644 index 0000000000..2be4617d51 --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/metrics.json @@ -0,0 +1,43 @@ +{ + "labels": { + "project": "solomon", + "cluster": "man", + "service": "stockpile" + }, + "metrics": [ + { + "type": "DGAUGE", + "labels": { + "metric": "Memory" + }, + "value": 10 + }, + { + "type": "RATE", + "value": 1, + "labels": { "metric": "UserTime" } + }, + { + "type": "GAUGE", + "value": 3.14159, + "labels": { "export": "Oxygen", "metric": "QueueSize" }, + "ts": "2017-11-05T12:34:56.000Z", + "memOnly": true + }, + { + "type": "GAUGE", + "labels": { "metric": "Writes" }, + "timeseries": [ + { + "ts": "2017-08-28T12:32:11Z", + "value": -10 + }, + { + "value": 20, + "ts": 1503923187 + } + ] + } + ], + "ts": "2017-08-27T12:34:56Z" +} diff --git a/library/cpp/monlib/encode/json/ut/named_metrics.json b/library/cpp/monlib/encode/json/ut/named_metrics.json new file mode 100644 index 0000000000..98f93e8c39 --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/named_metrics.json @@ -0,0 +1,22 @@ +{ + "labels": { + "project": "solomon", + "cluster": "prod-sas", + "service": "stockpile" + }, + "metrics": [ + { + "type": "DGAUGE", + "name": "Memory", + "value": 1 + }, + { + "type": "DGAUGE", + "name": "QueueSize", + "labels": { + "export": "Oxygen" + }, + "value": 10 + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/sensors.json b/library/cpp/monlib/encode/json/ut/sensors.json new file mode 100644 index 0000000000..4d979a3c1e --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/sensors.json @@ -0,0 +1,40 @@ +{ + "commonLabels": { + "project": "solomon", + "cluster": "man", + "service": "stockpile" + }, + "sensors": [ + { + "labels": { + "metric": "Memory" + }, + "value": 10 + }, + { + "mode": "deriv", + "value": 1, + "labels": { "metric": "UserTime" } + }, + { + "value": 3.14159, + "labels": { "export": "Oxygen", "metric": "QueueSize" }, + "ts": "2017-11-05T12:34:56.000Z", + "memOnly": true + }, + { + "labels": { "metric": "Writes" }, + "timeseries": [ + { + "ts": "2017-08-28T12:32:11Z", + "value": -10 + }, + { + "value": 20, + "ts": 1503923187 + } + ] + } + ], + "ts": "2017-08-27T12:34:56Z" +} diff --git a/library/cpp/monlib/encode/json/ut/summary_inf.json b/library/cpp/monlib/encode/json/ut/summary_inf.json new file mode 100644 index 0000000000..625a6cd8ad --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/summary_inf.json @@ -0,0 +1,21 @@ +{ + "sensors": + [ + { + "kind":"DSUMMARY", + "labels": + { + "metric":"temperature" + }, + "ts":1509843723, + "summary": + { + "sum":"nan", + "min":"-inf", + "max":"inf", + "last":0.3, + "count":30 + } + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/summary_timeseries.json b/library/cpp/monlib/encode/json/ut/summary_timeseries.json new file mode 100644 index 0000000000..92007af3e6 --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/summary_timeseries.json @@ -0,0 +1,37 @@ +{ + "sensors": + [ + { + "kind":"DSUMMARY", + "labels": + { + "metric":"temperature" + }, + "timeseries": + [ + { + "ts":1509843723, + "summary": + { + "sum":0.8, + "min":-0.5, + "max":1, + "last":1, + "count":3 + } + }, + { + "ts":1509843738, + "summary": + { + "sum":-0.69, + "min":-1.5, + "max":1, + "last":0.01, + "count":5 + } + } + ] + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/summary_value.json b/library/cpp/monlib/encode/json/ut/summary_value.json new file mode 100644 index 0000000000..366394c5e1 --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/summary_value.json @@ -0,0 +1,21 @@ +{ + "sensors": + [ + { + "kind":"DSUMMARY", + "labels": + { + "metric":"temperature" + }, + "ts":1509843723, + "summary": + { + "sum":10, + "min":-0.5, + "max":0.5, + "last":0.3, + "count":30 + } + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/test_decode_to_encode.json b/library/cpp/monlib/encode/json/ut/test_decode_to_encode.json new file mode 100644 index 0000000000..65f0c5c6e2 --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/test_decode_to_encode.json @@ -0,0 +1,16 @@ +{ + "commonLabels": { + "project": "solomon", + "cluster": "man", + "service": "stockpile" + }, + "sensors": [ + { + "kind": "GAUGE", + "labels": { "export": "Oxygen", "metric": "QueueSize" }, + "ts": 1509885296, + "value": 3.14159 + } + ], + "ts": 1503837296 +} diff --git a/library/cpp/monlib/encode/json/ut/ya.make b/library/cpp/monlib/encode/json/ut/ya.make new file mode 100644 index 0000000000..e50c4f4903 --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/ya.make @@ -0,0 +1,46 @@ +UNITTEST_FOR(library/cpp/monlib/encode/json) + +OWNER( + g:solomon + jamel +) + +SRCS( + json_decoder_ut.cpp + json_ut.cpp +) + +RESOURCE( + buffered_test.json /buffered_test.json + buffered_ts_merge.json /buffered_ts_merge.json + empty_series.json /empty_series.json + expected.json /expected.json + expected_buffered.json /expected_buffered.json + expected_cloud.json /expected_cloud.json + expected_cloud_buffered.json /expected_cloud_buffered.json + merged.json /merged.json + histogram_timeseries.json /histogram_timeseries.json + histogram_value.json /histogram_value.json + histogram_value_inf_before_bounds.json /histogram_value_inf_before_bounds.json + int_gauge.json /int_gauge.json + sensors.json /sensors.json + metrics.json /metrics.json + named_metrics.json /named_metrics.json + test_decode_to_encode.json /test_decode_to_encode.json + crash.json /crash.json + hist_crash.json /hist_crash.json + summary_value.json /summary_value.json + summary_inf.json /summary_inf.json + summary_timeseries.json /summary_timeseries.json + log_histogram_value.json /log_histogram_value.json + log_histogram_timeseries.json /log_histogram_timeseries.json +) + +PEERDIR( + library/cpp/json + library/cpp/monlib/consumers + library/cpp/monlib/encode/protobuf + library/cpp/resource +) + +END() diff --git a/library/cpp/monlib/encode/json/ya.make b/library/cpp/monlib/encode/json/ya.make new file mode 100644 index 0000000000..a50fc412a9 --- /dev/null +++ b/library/cpp/monlib/encode/json/ya.make @@ -0,0 +1,21 @@ +LIBRARY() + +OWNER( + g:solomon + jamel +) + +SRCS( + json_decoder.cpp + json_encoder.cpp +) + +PEERDIR( + library/cpp/monlib/encode + library/cpp/monlib/encode/buffered + library/cpp/monlib/exception + library/cpp/json + library/cpp/json/writer +) + +END() diff --git a/library/cpp/monlib/encode/legacy_protobuf/legacy_proto_decoder.cpp b/library/cpp/monlib/encode/legacy_protobuf/legacy_proto_decoder.cpp new file mode 100644 index 0000000000..f87a2d7e8f --- /dev/null +++ b/library/cpp/monlib/encode/legacy_protobuf/legacy_proto_decoder.cpp @@ -0,0 +1,527 @@ +#include "legacy_protobuf.h" + +#include <library/cpp/monlib/encode/legacy_protobuf/protos/metric_meta.pb.h> +#include <library/cpp/monlib/metrics/metric_consumer.h> +#include <library/cpp/monlib/metrics/labels.h> + +#include <util/generic/yexception.h> +#include <util/generic/maybe.h> +#include <util/datetime/base.h> +#include <util/string/split.h> + +#include <google/protobuf/reflection.h> + +#include <algorithm> + +#ifdef LEGACY_PB_TRACE +#define TRACE(msg) \ + Cerr << msg << Endl +#else +#define TRACE(...) ; +#endif + +namespace NMonitoring { + namespace { + using TMaybeMeta = TMaybe<NMonProto::TMetricMeta>; + + TString ReadLabelValue(const NProtoBuf::Message& msg, const NProtoBuf::FieldDescriptor* d, const NProtoBuf::Reflection& r) { + using namespace NProtoBuf; + + switch (d->type()) { + case FieldDescriptor::TYPE_UINT32: + return ::ToString(r.GetUInt32(msg, d)); + case FieldDescriptor::TYPE_UINT64: + return ::ToString(r.GetUInt64(msg, d)); + case FieldDescriptor::TYPE_STRING: + return r.GetString(msg, d); + case FieldDescriptor::TYPE_ENUM: { + auto val = r.GetEnumValue(msg, d); + auto* valDesc = d->enum_type()->FindValueByNumber(val); + return valDesc->name(); + } + + default: + ythrow yexception() << "type " << d->type_name() << " cannot be used as a field value"; + } + + return {}; + } + + double ReadFieldAsDouble(const NProtoBuf::Message& msg, const NProtoBuf::FieldDescriptor* d, const NProtoBuf::Reflection& r) { + using namespace NProtoBuf; + + switch (d->type()) { + case FieldDescriptor::TYPE_DOUBLE: + return r.GetDouble(msg, d); + case FieldDescriptor::TYPE_BOOL: + return r.GetBool(msg, d) ? 1 : 0; + case FieldDescriptor::TYPE_INT32: + return r.GetInt32(msg, d); + case FieldDescriptor::TYPE_INT64: + return r.GetInt64(msg, d); + case FieldDescriptor::TYPE_UINT32: + return r.GetUInt32(msg, d); + case FieldDescriptor::TYPE_UINT64: + return r.GetUInt64(msg, d); + case FieldDescriptor::TYPE_SINT32: + return r.GetInt32(msg, d); + case FieldDescriptor::TYPE_SINT64: + return r.GetInt64(msg, d); + case FieldDescriptor::TYPE_FIXED32: + return r.GetUInt32(msg, d); + case FieldDescriptor::TYPE_FIXED64: + return r.GetUInt64(msg, d); + case FieldDescriptor::TYPE_SFIXED32: + return r.GetInt32(msg, d); + case FieldDescriptor::TYPE_SFIXED64: + return r.GetInt64(msg, d); + case FieldDescriptor::TYPE_FLOAT: + return r.GetFloat(msg, d); + case FieldDescriptor::TYPE_ENUM: + return r.GetEnumValue(msg, d); + default: + ythrow yexception() << "type " << d->type_name() << " cannot be used as a field value"; + } + + return std::numeric_limits<double>::quiet_NaN(); + } + + double ReadRepeatedAsDouble(const NProtoBuf::Message& msg, const NProtoBuf::FieldDescriptor* d, const NProtoBuf::Reflection& r, size_t i) { + using namespace NProtoBuf; + + switch (d->type()) { + case FieldDescriptor::TYPE_DOUBLE: + return r.GetRepeatedDouble(msg, d, i); + case FieldDescriptor::TYPE_BOOL: + return r.GetRepeatedBool(msg, d, i) ? 1 : 0; + case FieldDescriptor::TYPE_INT32: + return r.GetRepeatedInt32(msg, d, i); + case FieldDescriptor::TYPE_INT64: + return r.GetRepeatedInt64(msg, d, i); + case FieldDescriptor::TYPE_UINT32: + return r.GetRepeatedUInt32(msg, d, i); + case FieldDescriptor::TYPE_UINT64: + return r.GetRepeatedUInt64(msg, d, i); + case FieldDescriptor::TYPE_SINT32: + return r.GetRepeatedInt32(msg, d, i); + case FieldDescriptor::TYPE_SINT64: + return r.GetRepeatedInt64(msg, d, i); + case FieldDescriptor::TYPE_FIXED32: + return r.GetRepeatedUInt32(msg, d, i); + case FieldDescriptor::TYPE_FIXED64: + return r.GetRepeatedUInt64(msg, d, i); + case FieldDescriptor::TYPE_SFIXED32: + return r.GetRepeatedInt32(msg, d, i); + case FieldDescriptor::TYPE_SFIXED64: + return r.GetRepeatedInt64(msg, d, i); + case FieldDescriptor::TYPE_FLOAT: + return r.GetRepeatedFloat(msg, d, i); + case FieldDescriptor::TYPE_ENUM: + return r.GetRepeatedEnumValue(msg, d, i); + default: + ythrow yexception() << "type " << d->type_name() << " cannot be used as a field value"; + } + + return std::numeric_limits<double>::quiet_NaN(); + } + + TString LabelFromField(const NProtoBuf::Message& msg, const TString& name) { + const auto* fieldDesc = msg.GetDescriptor()->FindFieldByName(name); + const auto* reflection = msg.GetReflection(); + Y_ENSURE(fieldDesc && reflection, "Unable to get meta for field " << name); + + auto s = ReadLabelValue(msg, fieldDesc, *reflection); + std::replace(std::begin(s), s.vend(), ' ', '_'); + + return s; + } + + TMaybeMeta MaybeGetMeta(const NProtoBuf::FieldOptions& opts) { + if (opts.HasExtension(NMonProto::Metric)) { + return opts.GetExtension(NMonProto::Metric); + } + + return Nothing(); + } + + class ILabelGetter: public TThrRefBase { + public: + enum class EType { + Fixed = 1, + Lazy = 2, + }; + + virtual TLabel Get(const NProtoBuf::Message&) = 0; + virtual EType Type() const = 0; + }; + + class TFixedLabel: public ILabelGetter { + public: + explicit TFixedLabel(TLabel&& l) + : Label_{std::move(l)} + { + TRACE("found fixed label " << l); + } + + EType Type() const override { + return EType::Fixed; + } + TLabel Get(const NProtoBuf::Message&) override { + return Label_; + } + + private: + TLabel Label_; + }; + + using TFunction = std::function<TLabel(const NProtoBuf::Message&)>; + + class TLazyLabel: public ILabelGetter { + public: + TLazyLabel(TFunction&& fn) + : Fn_{std::move(fn)} + { + TRACE("found lazy label"); + } + + EType Type() const override { + return EType::Lazy; + } + TLabel Get(const NProtoBuf::Message& msg) override { + return Fn_(msg); + } + + private: + TFunction Fn_; + }; + + class TDecoderContext { + public: + void Init(const NProtoBuf::Message* msg) { + Message_ = msg; + Y_ENSURE(Message_); + Reflection_ = msg->GetReflection(); + Y_ENSURE(Reflection_); + + for (auto it = Labels_.begin(); it != Labels_.end(); ++it) { + if ((*it)->Type() == ILabelGetter::EType::Lazy) { + auto l = (*it)->Get(Message()); + *it = ::MakeIntrusive<TFixedLabel>(std::move(l)); + } else { + auto l = (*it)->Get(Message()); + } + } + } + + void Clear() noexcept { + Message_ = nullptr; + Reflection_ = nullptr; + } + + TDecoderContext CreateChildFromMeta(const NMonProto::TMetricMeta& metricMeta, const TString& name, i64 repeatedIdx = -1) { + TDecoderContext child{*this}; + child.Clear(); + + if (metricMeta.HasCustomPath()) { + if (const auto& nodePath = metricMeta.GetCustomPath()) { + child.AppendPath(nodePath); + } + } else if (metricMeta.GetPath()) { + child.AppendPath(name); + } + + if (metricMeta.HasKeys()) { + child.ParseKeys(metricMeta.GetKeys(), repeatedIdx); + } + + return child; + } + + TDecoderContext CreateChildFromRepeatedScalar(const NMonProto::TMetricMeta& metricMeta, i64 repeatedIdx = -1) { + TDecoderContext child{*this}; + child.Clear(); + + if (metricMeta.HasKeys()) { + child.ParseKeys(metricMeta.GetKeys(), repeatedIdx); + } + + return child; + } + + TDecoderContext CreateChildFromEls(const TString& name, const NMonProto::TExtraLabelMetrics& metrics, size_t idx, TMaybeMeta maybeMeta) { + TDecoderContext child{*this}; + child.Clear(); + + auto usePath = [&maybeMeta] { + return !maybeMeta->HasPath() || maybeMeta->GetPath(); + }; + + if (!name.empty() && (!maybeMeta || usePath())) { + child.AppendPath(name); + } + + child.Labels_.push_back(::MakeIntrusive<TLazyLabel>( + [ labelName = metrics.GetlabelName(), idx, &metrics ](const auto&) { + const auto& val = metrics.Getvalues(idx); + TString labelVal; + const auto uintLabel = val.GetlabelValueUint(); + + if (uintLabel) { + labelVal = ::ToString(uintLabel); + } else { + labelVal = val.GetlabelValue(); + } + + return TLabel{labelName, labelVal}; + })); + + return child; + } + + void ParseKeys(TStringBuf keys, i64 repeatedIdx = -1) { + auto parts = StringSplitter(keys) + .Split(' ') + .SkipEmpty(); + + for (auto part : parts) { + auto str = part.Token(); + + TStringBuf lhs, rhs; + + const bool isDynamic = str.TrySplit(':', lhs, rhs); + const bool isIndexing = isDynamic && rhs == TStringBuf("#"); + + if (isIndexing) { + TRACE("parsed index labels"); + + // <label_name>:# means that we should use index of the repeated + // field as label value + Y_ENSURE(repeatedIdx != -1); + Labels_.push_back(::MakeIntrusive<TLazyLabel>([=](const auto&) { + return TLabel{lhs, ::ToString(repeatedIdx)}; + })); + } else if (isDynamic) { + TRACE("parsed dynamic labels"); + + // <label_name>:<field_name> means that we need to take label value + // later from message's field + Labels_.push_back(::MakeIntrusive<TLazyLabel>([=](const auto& msg) { + return TLabel{lhs, LabelFromField(msg, TString{rhs})}; + })); + } else if (str.TrySplit('=', lhs, rhs)) { + TRACE("parsed static labels"); + + // <label_name>=<label_value> stands for constant label + Labels_.push_back(::MakeIntrusive<TFixedLabel>(TLabel{lhs, rhs})); + } else { + ythrow yexception() << "Incorrect Keys format"; + } + } + } + + void AppendPath(TStringBuf fieldName) { + Path_ += '/'; + Path_ += fieldName; + } + + const TString& Path() const { + return Path_; + } + + TLabels Labels() const { + TLabels result; + for (auto&& l : Labels_) { + result.Add(l->Get(Message())); + } + + return result; + } + + const NProtoBuf::Message& Message() const { + Y_VERIFY_DEBUG(Message_); + return *Message_; + } + + const NProtoBuf::Reflection& Reflection() const { + return *Reflection_; + } + + private: + const NProtoBuf::Message* Message_{nullptr}; + const NProtoBuf::Reflection* Reflection_{nullptr}; + + TString Path_; + TVector<TIntrusivePtr<ILabelGetter>> Labels_; + }; + + class TDecoder { + public: + TDecoder(IMetricConsumer* consumer, const NProtoBuf::Message& message, TInstant timestamp) + : Consumer_{consumer} + , Message_{message} + , Timestamp_{timestamp} + { + } + + void Decode() const { + Consumer_->OnStreamBegin(); + DecodeToStream(); + Consumer_->OnStreamEnd(); + } + + void DecodeToStream() const { + DecodeImpl(Message_, {}); + } + + private: + static const NMonProto::TExtraLabelMetrics& ExtractExtraMetrics(TDecoderContext& ctx, const NProtoBuf::FieldDescriptor& f) { + const auto& parent = ctx.Message(); + const auto& reflection = ctx.Reflection(); + auto& subMessage = reflection.GetMessage(parent, &f); + + return dynamic_cast<const NMonProto::TExtraLabelMetrics&>(subMessage); + } + + void DecodeImpl(const NProtoBuf::Message& msg, TDecoderContext ctx) const { + std::vector<const NProtoBuf::FieldDescriptor*> fields; + + ctx.Init(&msg); + + ctx.Reflection().ListFields(msg, &fields); + + for (const auto* f : fields) { + Y_ENSURE(f); + + const auto& opts = f->options(); + const auto isMessage = f->type() == NProtoBuf::FieldDescriptor::TYPE_MESSAGE; + const auto isExtraLabelMetrics = isMessage && f->message_type()->full_name() == "NMonProto.TExtraLabelMetrics"; + const auto maybeMeta = MaybeGetMeta(opts); + + if (!(maybeMeta || isExtraLabelMetrics)) { + continue; + } + + if (isExtraLabelMetrics) { + const auto& extra = ExtractExtraMetrics(ctx, *f); + RecurseExtraLabelMetrics(ctx, extra, f->name(), maybeMeta); + } else if (isMessage) { + RecurseMessage(ctx, *maybeMeta, *f); + } else if (f->is_repeated()) { + RecurseRepeatedScalar(ctx, *maybeMeta, *f); + } else if (maybeMeta->HasType()) { + const auto val = ReadFieldAsDouble(msg, f, ctx.Reflection()); + const bool isRate = maybeMeta->GetType() == NMonProto::EMetricType::RATE; + WriteMetric(val, ctx, f->name(), isRate); + } + } + } + + void RecurseRepeatedScalar(TDecoderContext ctx, const NMonProto::TMetricMeta& meta, const NProtoBuf::FieldDescriptor& f) const { + auto&& msg = ctx.Message(); + auto&& reflection = ctx.Reflection(); + const bool isRate = meta.GetType() == NMonProto::EMetricType::RATE; + + // this is a repeated scalar field, which makes metric only if it's indexing + for (auto i = 0; i < reflection.FieldSize(msg, &f); ++i) { + auto subCtx = ctx.CreateChildFromRepeatedScalar(meta, i); + subCtx.Init(&msg); + auto val = ReadRepeatedAsDouble(msg, &f, reflection, i); + WriteMetric(val, subCtx, f.name(), isRate); + } + } + + void RecurseExtraLabelMetrics(TDecoderContext ctx, const NMonProto::TExtraLabelMetrics& msg, const TString& name, const TMaybeMeta& meta) const { + auto i = 0; + for (const auto& val : msg.Getvalues()) { + auto subCtx = ctx.CreateChildFromEls(name, msg, i++, meta); + subCtx.Init(&val); + + const bool isRate = val.Hastype() + ? val.Gettype() == NMonProto::EMetricType::RATE + : meta->GetType() == NMonProto::EMetricType::RATE; + + double metricVal{0}; + if (isRate) { + metricVal = val.GetlongValue(); + } else { + metricVal = val.GetdoubleValue(); + } + + WriteMetric(metricVal, subCtx, "", isRate); + + for (const auto& child : val.Getchildren()) { + RecurseExtraLabelMetrics(subCtx, child, "", meta); + } + } + } + + void RecurseMessage(TDecoderContext ctx, const NMonProto::TMetricMeta& metricMeta, const NProtoBuf::FieldDescriptor& f) const { + const auto& msg = ctx.Message(); + const auto& reflection = ctx.Reflection(); + + if (f.is_repeated()) { + TRACE("recurse into repeated message " << f.name()); + for (auto i = 0; i < reflection.FieldSize(msg, &f); ++i) { + auto& subMessage = reflection.GetRepeatedMessage(msg, &f, i); + DecodeImpl(subMessage, ctx.CreateChildFromMeta(metricMeta, f.name(), i)); + } + } else { + TRACE("recurse into message " << f.name()); + auto& subMessage = reflection.GetMessage(msg, &f); + DecodeImpl(subMessage, ctx.CreateChildFromMeta(metricMeta, f.name())); + } + } + + inline void WriteValue(ui64 value) const { + Consumer_->OnUint64(Timestamp_, value); + } + + inline void WriteValue(double value) const { + Consumer_->OnDouble(Timestamp_, value); + } + + void WriteMetric(double value, const TDecoderContext& ctx, const TString& name, bool isRate) const { + if (isRate) { + Consumer_->OnMetricBegin(EMetricType::RATE); + WriteValue(static_cast<ui64>(value)); + } else { + Consumer_->OnMetricBegin(EMetricType::GAUGE); + WriteValue(static_cast<double>(value)); + } + + Consumer_->OnLabelsBegin(); + + for (const auto& label : ctx.Labels()) { + Consumer_->OnLabel(label.Name(), label.Value()); + } + + const auto fullPath = name.empty() + ? ctx.Path() + : ctx.Path() + '/' + name; + + if (fullPath) { + Consumer_->OnLabel("path", fullPath); + } + + Consumer_->OnLabelsEnd(); + Consumer_->OnMetricEnd(); + } + + private: + IMetricConsumer* Consumer_{nullptr}; + const NProtoBuf::Message& Message_; + TInstant Timestamp_; + }; + + } + + void DecodeLegacyProto(const NProtoBuf::Message& data, IMetricConsumer* consumer, TInstant ts) { + Y_ENSURE(consumer); + TDecoder(consumer, data, ts).Decode(); + } + + void DecodeLegacyProtoToStream(const NProtoBuf::Message& data, IMetricConsumer* consumer, TInstant ts) { + Y_ENSURE(consumer); + TDecoder(consumer, data, ts).DecodeToStream(); + } +} diff --git a/library/cpp/monlib/encode/legacy_protobuf/legacy_protobuf.h b/library/cpp/monlib/encode/legacy_protobuf/legacy_protobuf.h new file mode 100644 index 0000000000..7cf8985d65 --- /dev/null +++ b/library/cpp/monlib/encode/legacy_protobuf/legacy_protobuf.h @@ -0,0 +1,16 @@ +#pragma once + +#include <google/protobuf/message.h> +#include <util/datetime/base.h> + +namespace NMonitoring { + // Unsupported features of the original format: + // - histograms; + // - memOnly; + // - dropHost/ignorePath + + void DecodeLegacyProto(const NProtoBuf::Message& data, class IMetricConsumer* c, TInstant ts = TInstant::Zero()); + + /// Does not open/close consumer stream unlike the above function. + void DecodeLegacyProtoToStream(const NProtoBuf::Message& data, class IMetricConsumer* c, TInstant ts = TInstant::Zero()); +} diff --git a/library/cpp/monlib/encode/legacy_protobuf/legacy_protobuf_ut.cpp b/library/cpp/monlib/encode/legacy_protobuf/legacy_protobuf_ut.cpp new file mode 100644 index 0000000000..53683cb39c --- /dev/null +++ b/library/cpp/monlib/encode/legacy_protobuf/legacy_protobuf_ut.cpp @@ -0,0 +1,422 @@ +#include "legacy_protobuf.h" + +#include <library/cpp/testing/unittest/registar.h> + +#include <library/cpp/monlib/encode/legacy_protobuf/ut/test_cases.pb.h> +#include <library/cpp/monlib/encode/legacy_protobuf/protos/metric_meta.pb.h> + +#include <library/cpp/monlib/encode/protobuf/protobuf.h> +#include <library/cpp/monlib/encode/text/text.h> +#include <library/cpp/monlib/metrics/labels.h> + +#include <util/generic/algorithm.h> +#include <util/generic/hash_set.h> + +using namespace NMonitoring; + +TSimple MakeSimpleMessage() { + TSimple msg; + + msg.SetFoo(1); + msg.SetBar(2.); + msg.SetBaz(42.); + + return msg; +} + +IMetricEncoderPtr debugPrinter = EncoderText(&Cerr); + +namespace NMonitoring { + inline bool operator<(const TLabel& lhs, const TLabel& rhs) { + return lhs.Name() < rhs.Name() || + (lhs.Name() == rhs.Name() && lhs.Value() < rhs.Value()); + } + +} + +void SetLabelValue(NMonProto::TExtraLabelMetrics::TValue& val, TString s) { + val.SetlabelValue(s); +} + +void SetLabelValue(NMonProto::TExtraLabelMetrics::TValue& val, ui64 u) { + val.SetlabelValueUint(u); +} + +template <typename T, typename V> +NMonProto::TExtraLabelMetrics MakeExtra(TString labelName, V labelValue, T value, bool isDeriv) { + NMonProto::TExtraLabelMetrics metric; + auto* val = metric.Addvalues(); + + metric.SetlabelName(labelName); + SetLabelValue(*val, labelValue); + + if (isDeriv) { + val->SetlongValue(value); + } else { + val->SetdoubleValue(value); + } + + return metric; +} + +void AssertLabels(const TLabels& expected, const NProto::TMultiSample& actual) { + UNIT_ASSERT_EQUAL(actual.LabelsSize(), expected.Size()); + + TSet<TLabel> actualSet; + TSet<TLabel> expectedSet; + Transform(expected.begin(), expected.end(), std::inserter(expectedSet, expectedSet.end()), [] (auto&& l) { + return TLabel{l.Name(), l.Value()}; + }); + + const auto& l = actual.GetLabels(); + Transform(std::begin(l), std::end(l), std::inserter(actualSet, std::begin(actualSet)), + [](auto&& elem) -> TLabel { + return {elem.GetName(), elem.GetValue()}; + }); + + TVector<TLabel> diff; + SetSymmetricDifference(std::begin(expectedSet), std::end(expectedSet), + std::begin(actualSet), std::end(actualSet), std::back_inserter(diff)); + + if (diff.size() > 0) { + for (auto&& l : diff) { + Cerr << l << Endl; + } + + UNIT_FAIL("Labels don't match"); + } +} + +void AssertSimpleMessage(const NProto::TMultiSamplesList& samples, TString pathPrefix = "/") { + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 3); + + THashSet<TString> expectedValues{pathPrefix + "Foo", pathPrefix + "Bar", pathPrefix + "Baz"}; + + for (const auto& s : samples.GetSamples()) { + UNIT_ASSERT_EQUAL(s.LabelsSize(), 1); + UNIT_ASSERT_EQUAL(s.PointsSize(), 1); + + const auto labelVal = s.GetLabels(0).GetValue(); + UNIT_ASSERT(expectedValues.contains(labelVal)); + + if (labelVal == pathPrefix + "Foo") { + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_DOUBLES_EQUAL(s.GetPoints(0).GetFloat64(), 1, 1e-6); + } else if (labelVal == pathPrefix + "Bar") { + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_DOUBLES_EQUAL(s.GetPoints(0).GetFloat64(), 2, 1e-6); + } else if (labelVal == pathPrefix + "Baz") { + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::RATE); + UNIT_ASSERT_EQUAL(s.GetPoints(0).GetUint64(), 42); + } + } +} + +Y_UNIT_TEST_SUITE(TLegacyProtoDecoderTest) { + Y_UNIT_TEST(SimpleProto) { + NProto::TMultiSamplesList samples; + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + auto msg = MakeSimpleMessage(); + DecodeLegacyProto(msg, e.Get()); + + AssertSimpleMessage(samples); + }; + + Y_UNIT_TEST(RepeatedProto) { + NProto::TMultiSamplesList samples; + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + auto simple = MakeSimpleMessage(); + TRepeated msg; + msg.AddMessages()->CopyFrom(simple); + + DecodeLegacyProto(msg, e.Get()); + + AssertSimpleMessage(samples); + } + + Y_UNIT_TEST(RepeatedProtoWithPath) { + NProto::TMultiSamplesList samples; + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + auto simple = MakeSimpleMessage(); + TRepeatedWithPath msg; + msg.AddNamespace()->CopyFrom(simple); + + DecodeLegacyProto(msg, e.Get()); + + AssertSimpleMessage(samples, "/Namespace/"); + } + + Y_UNIT_TEST(DeepNesting) { + NProto::TMultiSamplesList samples; + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + auto simple = MakeSimpleMessage(); + TRepeatedWithPath internal; + internal.AddNamespace()->CopyFrom(simple); + + TDeepNesting msg; + msg.MutableNested()->CopyFrom(internal); + + DecodeLegacyProto(msg, e.Get()); + + AssertSimpleMessage(samples, "/Namespace/"); + } + + Y_UNIT_TEST(Keys) { + NProto::TMultiSamplesList samples; + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + auto simple = MakeSimpleMessage(); + simple.SetLabel("my_label_value"); + + TNestedWithKeys msg; + msg.AddNamespace()->CopyFrom(simple); + + DecodeLegacyProto(msg, e.Get()); + + auto i = 0; + for (const auto& s : samples.GetSamples()) { + UNIT_ASSERT_EQUAL(s.LabelsSize(), 4); + + bool foundLabel = false; + bool foundFixed = false; + bool foundNumbered = false; + + for (const auto& label : s.GetLabels()) { + if (label.GetName() == "my_label") { + foundLabel = true; + UNIT_ASSERT_STRINGS_EQUAL(label.GetValue(), "my_label_value"); + } else if (label.GetName() == "fixed_label") { + foundFixed = true; + UNIT_ASSERT_STRINGS_EQUAL(label.GetValue(), "fixed_value"); + } else if (label.GetName() == "numbered") { + foundNumbered = true; + UNIT_ASSERT_STRINGS_EQUAL(label.GetValue(), ::ToString(i)); + } + } + + UNIT_ASSERT(foundLabel); + UNIT_ASSERT(foundFixed); + UNIT_ASSERT(foundNumbered); + } + } + + Y_UNIT_TEST(NonStringKeys) { + NProto::TMultiSamplesList samples; + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + TNonStringKeys msg; + msg.SetFoo(42); + msg.SetEnum(ENUM); + msg.SetInt(43); + + TRepeatedNonStringKeys msgs; + msgs.AddNested()->CopyFrom(msg); + + DecodeLegacyProto(msgs, e.Get()); + + for (const auto& s : samples.GetSamples()) { + bool foundEnum = false; + bool foundInt = false; + + for (const auto& label : s.GetLabels()) { + if (label.GetName() == "enum") { + foundEnum = true; + UNIT_ASSERT_STRINGS_EQUAL(label.GetValue(), "ENUM"); + } else if (label.GetName() == "int") { + foundInt = true; + UNIT_ASSERT_STRINGS_EQUAL(label.GetValue(), "43"); + } + } + + UNIT_ASSERT(foundEnum); + UNIT_ASSERT(foundInt); + + UNIT_ASSERT_DOUBLES_EQUAL(s.GetPoints(0).GetFloat64(), 42, 1e-6); + } + } + + Y_UNIT_TEST(KeysFromNonLeafNodes) { + NProto::TMultiSamplesList samples; + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + auto simple = MakeSimpleMessage(); + simple.SetLabel("label_value"); + + TRepeatedWithName nested; + nested.SetName("my_name"); + nested.AddNested()->CopyFrom(simple); + + TKeysFromNonLeaf msg; + msg.AddNested()->CopyFrom(nested); + + DecodeLegacyProto(msg, e.Get()); + + AssertLabels({{"my_label", "label_value"}, {"path", "/Nested/Nested/Foo"}, {"name", "my_name"}}, samples.GetSamples(0)); + } + + Y_UNIT_TEST(SpacesAreGetReplaced) { + NProto::TMultiSamplesList samples; + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + auto simple = MakeSimpleMessage(); + simple.SetLabel("my label_value"); + + TNestedWithKeys msg; + msg.AddNamespace()->CopyFrom(simple); + + DecodeLegacyProto(msg, e.Get()); + + for (const auto& s : samples.GetSamples()) { + UNIT_ASSERT_EQUAL(s.LabelsSize(), 4); + + bool foundLabel = false; + + for (const auto& label : s.GetLabels()) { + if (label.GetName() == "my_label") { + foundLabel = true; + UNIT_ASSERT_STRINGS_EQUAL(label.GetValue(), "my_label_value"); + } + } + + UNIT_ASSERT(foundLabel); + } + } + + Y_UNIT_TEST(ExtraLabels) { + NProto::TMultiSamplesList samples; + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + TExtraLabels msg; + msg.MutableExtraAsIs()->CopyFrom(MakeExtra("label", "foo", 42, false)); + msg.MutableExtraDeriv()->CopyFrom(MakeExtra("deriv_label", "deriv_foo", 43, true)); + + DecodeLegacyProto(msg, e.Get()); + + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 2); + { + auto s = samples.GetSamples(0); + AssertLabels({{"label", "foo"}, {"path", "/ExtraAsIs"}}, s); + + UNIT_ASSERT_EQUAL(s.PointsSize(), 1); + auto point = s.GetPoints(0); + UNIT_ASSERT_DOUBLES_EQUAL(point.GetFloat64(), 42, 1e-6); + } + + { + auto s = samples.GetSamples(1); + AssertLabels({{"deriv_label", "deriv_foo"}, {"path", "/ExtraDeriv"}}, s); + + UNIT_ASSERT_EQUAL(s.PointsSize(), 1); + auto point = s.GetPoints(0); + UNIT_ASSERT_EQUAL(point.GetUint64(), 43); + } + } + + Y_UNIT_TEST(NestedExtraLabels) { + NProto::TMultiSamplesList samples; + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + TExtraLabels msg; + auto extra = MakeExtra("label", "foo", 42, false); + auto* val = extra.Mutablevalues(0); + { + auto child = MakeExtra("child1", "label1", 24, true); + child.Mutablevalues(0)->Settype(NMonProto::EMetricType::RATE); + val->Addchildren()->CopyFrom(child); + } + + { + auto child = MakeExtra("child2", 34, 23, false); + val->Addchildren()->CopyFrom(child); + } + msg.MutableExtraAsIs()->CopyFrom(extra); + + DecodeLegacyProto(msg, e.Get()); + + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 3); + { + auto s = samples.GetSamples(0); + AssertLabels({{"label", "foo"}, {"path", "/ExtraAsIs"}}, s); + + UNIT_ASSERT_EQUAL(s.PointsSize(), 1); + auto point = s.GetPoints(0); + UNIT_ASSERT_DOUBLES_EQUAL(point.GetFloat64(), 42, 1e-6); + } + + { + auto s = samples.GetSamples(1); + AssertLabels({{"label", "foo"}, {"child1", "label1"}, {"path", "/ExtraAsIs"}}, s); + + UNIT_ASSERT_EQUAL(s.PointsSize(), 1); + auto point = s.GetPoints(0); + UNIT_ASSERT_EQUAL(point.GetUint64(), 24); + } + + { + auto s = samples.GetSamples(2); + AssertLabels({{"label", "foo"}, {"child2", "34"}, {"path", "/ExtraAsIs"}}, s); + + UNIT_ASSERT_EQUAL(s.PointsSize(), 1); + auto point = s.GetPoints(0); + UNIT_ASSERT_EQUAL(point.GetFloat64(), 23); + } + } + + Y_UNIT_TEST(RobotLabels) { + NProto::TMultiSamplesList samples; + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + TNamedCounter responses; + responses.SetName("responses"); + responses.SetCount(42); + + TCrawlerCounters::TStatusCounters statusCounters; + statusCounters.AddZoraResponses()->CopyFrom(responses); + + TCrawlerCounters::TPolicyCounters policyCounters; + policyCounters.SetSubComponent("mySubComponent"); + policyCounters.SetName("myComponentName"); + policyCounters.SetZone("myComponentZone"); + policyCounters.MutableStatusCounters()->CopyFrom(statusCounters); + + TCrawlerCounters counters; + counters.SetComponent("myComponent"); + counters.AddPoliciesCounters()->CopyFrom(policyCounters); + + DecodeLegacyProto(counters, e.Get()); + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 1); + auto s = samples.GetSamples(0); + AssertLabels({ + {"SubComponent", "mySubComponent"}, {"Policy", "myComponentName"}, {"Zone", "myComponentZone"}, + {"ZoraResponse", "responses"}, {"path", "/PoliciesCounters/StatusCounters/ZoraResponses/Count"}}, s); + } + + Y_UNIT_TEST(ZoraLabels) { + NProto::TMultiSamplesList samples; + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + TTimeLogHist hist; + hist.AddBuckets(42); + hist.AddBuckets(0); + + TKiwiCounters counters; + counters.MutableTimes()->CopyFrom(hist); + + DecodeLegacyProto(counters, e.Get()); + + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 2); + + auto s = samples.GetSamples(0); + AssertLabels({{"slot", "0"}, {"path", "/Times/Buckets"}}, s); + UNIT_ASSERT_EQUAL(s.PointsSize(), 1); + UNIT_ASSERT_EQUAL(s.GetPoints(0).GetUint64(), 42); + + s = samples.GetSamples(1); + AssertLabels({{"slot", "1"}, {"path", "/Times/Buckets"}}, s); + UNIT_ASSERT_EQUAL(s.GetPoints(0).GetUint64(), 0); + } +} diff --git a/library/cpp/monlib/encode/legacy_protobuf/protos/metric_meta.proto b/library/cpp/monlib/encode/legacy_protobuf/protos/metric_meta.proto new file mode 100644 index 0000000000..fd23eb372b --- /dev/null +++ b/library/cpp/monlib/encode/legacy_protobuf/protos/metric_meta.proto @@ -0,0 +1,73 @@ +import "google/protobuf/descriptor.proto"; + +package NMonProto; + +option java_package = "ru.yandex.monlib.proto"; +option java_outer_classname = "MetricMetaProto"; + +enum EMetricType { + GAUGE = 1; + RATE = 2; +} + +enum EMemOnly { + DEFAULT = 0; + STORE = 1; + MEM_ONLY = 2; +} + +message TMetricMeta { + optional EMetricType Type = 1; + optional bool Path = 2; + optional string Keys = 3; + optional bool MemOnly = 4; + optional bool IgnorePath = 5; + optional string CustomPath = 6; +} + +enum THistogramBase { + MICROSECOND = 3; + MILLISECOND = 6; + SECOND = 9; + MINUTE = 12; + HOUR = 15; +} + +message THistogramEntry { + optional uint64 Multiplier = 1; + optional double Value = 2; +} + +message THistogram { + optional THistogramBase Base = 1; + optional string BaseStr = 2; + repeated THistogramEntry Entries = 5; +} + +// field of this type is recognized by Solomon +message TExtraLabelMetrics { + optional string labelName = 1; + + message TValue { + optional string labelValue = 1; + // used only if != 0 + optional uint64 labelValueUint = 21; + + optional uint64 longValue = 2; + optional double doubleValue = 3; + optional THistogram histogramValue = 4; + + optional EMetricType type = 7; + optional EMemOnly memOnly = 8; + optional bool dropHost = 9; + + repeated TExtraLabelMetrics children = 17; + } + + repeated TValue values = 2; +} + +extend google.protobuf.FieldOptions { + optional TMetricMeta Metric = 1719; +} + diff --git a/library/cpp/monlib/encode/legacy_protobuf/protos/python/ya.make b/library/cpp/monlib/encode/legacy_protobuf/protos/python/ya.make new file mode 100644 index 0000000000..095b307b01 --- /dev/null +++ b/library/cpp/monlib/encode/legacy_protobuf/protos/python/ya.make @@ -0,0 +1,3 @@ +OWNER(g:solomon) + +PY_PROTOS_FOR(library/cpp/monlib/encode/legacy_protobuf/protos) diff --git a/library/cpp/monlib/encode/legacy_protobuf/protos/ya.make b/library/cpp/monlib/encode/legacy_protobuf/protos/ya.make new file mode 100644 index 0000000000..489f361ab1 --- /dev/null +++ b/library/cpp/monlib/encode/legacy_protobuf/protos/ya.make @@ -0,0 +1,13 @@ +PROTO_LIBRARY() + +OWNER(g:solomon) + +SRCS( + metric_meta.proto +) + +IF (NOT PY_PROTOS_FOR) + EXCLUDE_TAGS(GO_PROTO) +ENDIF() + +END() diff --git a/library/cpp/monlib/encode/legacy_protobuf/ut/test_cases.proto b/library/cpp/monlib/encode/legacy_protobuf/ut/test_cases.proto new file mode 100644 index 0000000000..37e901de48 --- /dev/null +++ b/library/cpp/monlib/encode/legacy_protobuf/ut/test_cases.proto @@ -0,0 +1,90 @@ +import "library/cpp/monlib/encode/legacy_protobuf/protos/metric_meta.proto"; + +message TSimple { + optional uint64 Foo = 1 [ (NMonProto.Metric).Type = GAUGE ]; + optional double Bar = 2 [ (NMonProto.Metric).Type = GAUGE ]; + optional double Baz = 3 [ (NMonProto.Metric).Type = RATE ]; + optional string Label = 4; +} + +message TRepeated { + repeated TSimple Messages = 1 [ (NMonProto.Metric).Path = false ]; +}; + +message TRepeatedWithPath { + repeated TSimple Namespace = 1 [ (NMonProto.Metric).Path = true ]; +}; + +message TNestedWithKeys { + repeated TSimple Namespace = 1 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "my_label:Label fixed_label=fixed_value numbered:#" ]; +}; + +message TDeepNesting { + optional TRepeatedWithPath Nested = 1 [ (NMonProto.Metric).Path = false ]; +}; + +enum EEnum { + MY = 1; + ENUM = 2; +}; + +message TNonStringKeys { + optional uint32 Foo = 1 [ (NMonProto.Metric).Type = GAUGE ]; + optional EEnum Enum = 2; + optional uint32 Int = 3; +}; + +message TRepeatedNonStringKeys { + repeated TNonStringKeys Nested = 1 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "enum:Enum int:Int" ]; +}; + +message TExtraLabels { + optional NMonProto.TExtraLabelMetrics ExtraAsIs = 1 [ (NMonProto.Metric).Type = GAUGE ]; + optional NMonProto.TExtraLabelMetrics ExtraDeriv = 2 [ (NMonProto.Metric).Type = RATE ]; +}; + +message TRepeatedWithName { + optional string Name = 1; + repeated TSimple Nested = 2 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "my_label:Label" ]; +}; + +message TKeysFromNonLeaf { + repeated TRepeatedWithName Nested = 1 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "name:Name" ]; +}; + + +message TNamedCounter { + optional string Name = 1; + optional uint64 Count = 2 [ (NMonProto.Metric).Type = RATE ]; +} + +message TCrawlerCounters { + message TStatusCounters { + repeated TNamedCounter ZoraResponses = 3 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "ZoraResponse:Name" ]; + repeated TNamedCounter FetcherResponses = 4 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "SpiderResponse:Name" ]; + repeated TNamedCounter RotorResponses = 5 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "SpiderResponse:Name" ]; + repeated TNamedCounter PDFetchResponses = 6 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "PDFetchResponse:Name" ]; + repeated TNamedCounter CalcResponses = 7 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "CalcResponse:Name" ]; + repeated TNamedCounter Responses = 8 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "AggregatedResponse:Name" ]; + } + + message TPolicyCounters { + optional string SubComponent = 1; + optional string Name = 2; + optional string Zone = 3; + + optional TStatusCounters StatusCounters = 4 [ (NMonProto.Metric).Path = true ]; + } + + optional string Component = 1; + repeated TPolicyCounters PoliciesCounters = 3 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "SubComponent:SubComponent Policy:Name Zone:Zone" ]; +} + +message TTimeLogHist { + optional uint32 MinBucketMillisec = 1; + repeated uint64 Buckets = 2 [ (NMonProto.Metric).Type = RATE, (NMonProto.Metric).Keys = "slot:#" ]; +} + +message TKiwiCounters { + optional TTimeLogHist Times = 22 [ (NMonProto.Metric).Path = true ]; +} diff --git a/library/cpp/monlib/encode/legacy_protobuf/ut/ya.make b/library/cpp/monlib/encode/legacy_protobuf/ut/ya.make new file mode 100644 index 0000000000..479a0c46c9 --- /dev/null +++ b/library/cpp/monlib/encode/legacy_protobuf/ut/ya.make @@ -0,0 +1,18 @@ +UNITTEST_FOR(library/cpp/monlib/encode/legacy_protobuf) + +OWNER( + g:solomon + msherbakov +) + +SRCS( + legacy_protobuf_ut.cpp + test_cases.proto +) + +PEERDIR( + library/cpp/monlib/encode/protobuf + library/cpp/monlib/encode/text +) + +END() diff --git a/library/cpp/monlib/encode/legacy_protobuf/ya.make b/library/cpp/monlib/encode/legacy_protobuf/ya.make new file mode 100644 index 0000000000..74c82aac93 --- /dev/null +++ b/library/cpp/monlib/encode/legacy_protobuf/ya.make @@ -0,0 +1,16 @@ +LIBRARY() + +OWNER( + g:solomon + msherbakov +) + +SRCS( + legacy_proto_decoder.cpp +) + +PEERDIR( + library/cpp/monlib/encode/legacy_protobuf/protos +) + +END() diff --git a/library/cpp/monlib/encode/prometheus/fuzz/main.cpp b/library/cpp/monlib/encode/prometheus/fuzz/main.cpp new file mode 100644 index 0000000000..24bda2d32e --- /dev/null +++ b/library/cpp/monlib/encode/prometheus/fuzz/main.cpp @@ -0,0 +1,18 @@ +#include <library/cpp/monlib/encode/prometheus/prometheus.h> +#include <library/cpp/monlib/encode/fake/fake.h> + +#include <util/stream/mem.h> + + +extern "C" int LLVMFuzzerTestOneInput(const ui8* buf, size_t size) { + using namespace NMonitoring; + + try { + TStringBuf data(reinterpret_cast<const char*>(buf), size); + auto encoder = EncoderFake(); + DecodePrometheus(data, encoder.Get()); + } catch (...) { + } + + return 0; +} diff --git a/library/cpp/monlib/encode/prometheus/fuzz/ya.make b/library/cpp/monlib/encode/prometheus/fuzz/ya.make new file mode 100644 index 0000000000..4a6c796ed5 --- /dev/null +++ b/library/cpp/monlib/encode/prometheus/fuzz/ya.make @@ -0,0 +1,16 @@ +FUZZ() + +OWNER(g:solomon jamel) + +PEERDIR( + library/cpp/monlib/encode/prometheus + library/cpp/monlib/encode/fake +) + +SIZE(MEDIUM) + +SRCS( + main.cpp +) + +END() diff --git a/library/cpp/monlib/encode/prometheus/prometheus.h b/library/cpp/monlib/encode/prometheus/prometheus.h new file mode 100644 index 0000000000..2e7fa31c28 --- /dev/null +++ b/library/cpp/monlib/encode/prometheus/prometheus.h @@ -0,0 +1,18 @@ +#pragma once + +#include <library/cpp/monlib/encode/encoder.h> +#include <library/cpp/monlib/encode/format.h> + +#include <util/generic/yexception.h> + + +namespace NMonitoring { + + class TPrometheusDecodeException: public yexception { + }; + + IMetricEncoderPtr EncoderPrometheus(IOutputStream* out, TStringBuf metricNameLabel = "sensor"); + + void DecodePrometheus(TStringBuf data, IMetricConsumer* c, TStringBuf metricNameLabel = "sensor"); + +} diff --git a/library/cpp/monlib/encode/prometheus/prometheus_decoder.cpp b/library/cpp/monlib/encode/prometheus/prometheus_decoder.cpp new file mode 100644 index 0000000000..7e81357dbd --- /dev/null +++ b/library/cpp/monlib/encode/prometheus/prometheus_decoder.cpp @@ -0,0 +1,597 @@ +#include "prometheus.h" +#include "prometheus_model.h" + +#include <library/cpp/monlib/metrics/histogram_snapshot.h> +#include <library/cpp/monlib/metrics/metric.h> + +#include <util/datetime/base.h> +#include <util/generic/hash.h> +#include <util/string/cast.h> +#include <util/string/builder.h> +#include <util/generic/maybe.h> +#include <util/string/ascii.h> + +#include <cmath> + +#define Y_PARSER_FAIL(message) \ + ythrow ::NMonitoring::TPrometheusDecodeException() << message << " at line #" << CurrentLine_ + +#define Y_PARSER_ENSURE(cond, message) \ + Y_ENSURE_EX(cond, ::NMonitoring::TPrometheusDecodeException() << message << " at line #" << CurrentLine_) + + +namespace NMonitoring { + namespace { + constexpr ui32 MAX_LABEL_VALUE_LEN = 256; + + using TLabelsMap = THashMap<TString, TString>; + + TString LabelsToStr(const TLabelsMap& labels) { + TStringBuilder sb; + auto it = labels.begin(); + auto end = labels.end(); + + sb << '{'; + while (it != end) { + sb << it->first; + sb << '='; + sb << '"' << it->second << '"'; + + ++it; + if (it != end) { + sb << ", "; + } + } + sb << '}'; + return sb; + } + + template <typename T, typename U> + bool TryStaticCast(U val, T& out) { + static_assert(std::is_arithmetic_v<U>); + if constexpr (std::is_floating_point_v<T> || std::is_floating_point_v<U>) { + if (val > MaxFloor<T>() || val < -MaxFloor<T>()) { + return false; + } + + } else { + if (val > Max<T>() || val < Min<T>()) { + return false; + } + } + + out = static_cast<T>(val); + return true; + } + + /////////////////////////////////////////////////////////////////////// + // THistogramBuilder + /////////////////////////////////////////////////////////////////////// + class THistogramBuilder { + using TBucketData = std::pair<TBucketBound, TBucketValue>; + constexpr static TBucketData ZERO_BUCKET = { -std::numeric_limits<TBucketBound>::max(), 0 }; + public: + TStringBuf GetName() const noexcept { + return Name_; + } + + void SetName(TStringBuf name) noexcept { + Name_ = name; + } + + const TLabelsMap& GetLabels() const noexcept { + return *Labels_; + } + + void SetLabels(TLabelsMap&& labels) { + if (Labels_.Defined()) { + Y_ENSURE(Labels_ == labels, + "mixed labels in one histogram, prev: " << LabelsToStr(*Labels_) << + ", current: " << LabelsToStr(labels)); + } else { + Labels_.ConstructInPlace(std::move(labels)); + } + } + + TInstant GetTime() const noexcept { + return Time_; + } + + void SetTime(TInstant time) noexcept { + Time_ = time; + } + + bool Empty() const noexcept { + return Bounds_.empty(); + } + + bool Same(TStringBuf name, const TLabelsMap& labels) const noexcept { + return Name_ == name && Labels_ == labels; + } + + void AddBucket(TBucketBound bound, TBucketValue value) { + Y_ENSURE_EX(PrevBucket_.first < bound, TPrometheusDecodeException() << + "invalid order of histogram bounds " << PrevBucket_.first << + " >= " << bound); + + Y_ENSURE_EX(PrevBucket_.second <= value, TPrometheusDecodeException() << + "invalid order of histogram bucket values " << PrevBucket_.second << + " > " << value); + + // convert infinite bound value + if (bound == std::numeric_limits<TBucketBound>::infinity()) { + bound = HISTOGRAM_INF_BOUND; + } + + Bounds_.push_back(bound); + Values_.push_back(value - PrevBucket_.second); // keep only delta between buckets + + PrevBucket_ = { bound, value }; + } + + // will clear builder state + IHistogramSnapshotPtr ToSnapshot() { + Y_ENSURE_EX(!Empty(), TPrometheusDecodeException() << "histogram cannot be empty"); + Time_ = TInstant::Zero(); + PrevBucket_ = ZERO_BUCKET; + Labels_.Clear(); + auto snapshot = ExplicitHistogramSnapshot(Bounds_, Values_); + + Bounds_.clear(); + Values_.clear(); + + return snapshot; + } + + private: + TStringBuf Name_; + TMaybe<TLabelsMap> Labels_; + TInstant Time_; + TBucketBounds Bounds_; + TBucketValues Values_; + TBucketData PrevBucket_ = ZERO_BUCKET; + }; + + /////////////////////////////////////////////////////////////////////// + // EPrometheusMetricType + /////////////////////////////////////////////////////////////////////// + enum class EPrometheusMetricType { + GAUGE, + COUNTER, + SUMMARY, + UNTYPED, + HISTOGRAM, + }; + + /////////////////////////////////////////////////////////////////////// + // TPrometheusReader + /////////////////////////////////////////////////////////////////////// + class TPrometheusReader { + public: + TPrometheusReader(TStringBuf data, IMetricConsumer* c, TStringBuf metricNameLabel) + : Data_(data) + , Consumer_(c) + , MetricNameLabel_(metricNameLabel) + { + } + + void Read() { + Consumer_->OnStreamBegin(); + + if (HasRemaining()) { + ReadNextByte(); + SkipSpaces(); + + try { + while (HasRemaining()) { + switch (CurrentByte_) { + case '\n': + ReadNextByte(); // skip '\n' + CurrentLine_++; + SkipSpaces(); + break; + case '#': + ParseComment(); + break; + default: + ParseMetric(); + break; + } + } + + if (!HistogramBuilder_.Empty()) { + ConsumeHistogram(); + } + } catch (const TPrometheusDecodeException& e) { + throw e; + } catch (...) { + Y_PARSER_FAIL("unexpected error " << CurrentExceptionMessage()); + } + } + + Consumer_->OnStreamEnd(); + } + + private: + bool HasRemaining() const noexcept { + return CurrentPos_ < Data_.Size(); + } + + // # 'TYPE' metric_name {counter|gauge|histogram|summary|untyped} + // # 'HELP' metric_name some help info + // # general comment message + void ParseComment() { + SkipExpectedChar('#'); + SkipSpaces(); + + TStringBuf keyword = ReadToken(); + if (keyword == TStringBuf("TYPE")) { + SkipSpaces(); + + TStringBuf nextName = ReadTokenAsMetricName(); + Y_PARSER_ENSURE(!nextName.Empty(), "invalid metric name"); + + SkipSpaces(); + EPrometheusMetricType nextType = ReadType(); + + bool inserted = SeenTypes_.emplace(nextName, nextType).second; + Y_PARSER_ENSURE(inserted, "second TYPE line for metric " << nextName); + + if (nextType == EPrometheusMetricType::HISTOGRAM) { + if (!HistogramBuilder_.Empty()) { + ConsumeHistogram(); + } + HistogramBuilder_.SetName(nextName); + } + } else { + // skip HELP and general comments + SkipUntilEol(); + } + + Y_PARSER_ENSURE(CurrentByte_ == '\n', "expected '\\n', found '" << CurrentByte_ << '\''); + } + + // metric_name [labels] value [timestamp] + void ParseMetric() { + TStringBuf name = ReadTokenAsMetricName(); + SkipSpaces(); + + TLabelsMap labels = ReadLabels(); + SkipSpaces(); + + double value = ParseGoDouble(ReadToken()); + SkipSpaces(); + + TInstant time = TInstant::Zero(); + if (CurrentByte_ != '\n') { + time = TInstant::MilliSeconds(FromString<ui64>(ReadToken())); + } + + TStringBuf baseName = name; + EPrometheusMetricType type = EPrometheusMetricType::UNTYPED; + + if (auto* seenType = SeenTypes_.FindPtr(name)) { + type = *seenType; + } else { + baseName = NPrometheus::ToBaseName(name); + if (auto* baseType = SeenTypes_.FindPtr(baseName)) { + type = *baseType; + } + } + + switch (type) { + case EPrometheusMetricType::HISTOGRAM: + if (NPrometheus::IsBucket(name)) { + double bound = 0.0; + auto it = labels.find(NPrometheus::BUCKET_LABEL); + if (it != labels.end()) { + bound = ParseGoDouble(it->second); + labels.erase(it); + } else { + Y_PARSER_FAIL( + "metric " << name << "has no " << NPrometheus::BUCKET_LABEL << + " label at line #" << CurrentLine_); + } + + if (!HistogramBuilder_.Empty() && !HistogramBuilder_.Same(baseName, labels)) { + ConsumeHistogram(); + HistogramBuilder_.SetName(baseName); + } + + TBucketValue bucketVal; + Y_PARSER_ENSURE(TryStaticCast(value, bucketVal), "Cannot convert " << value << " to bucket value type"); + HistogramBuilder_.AddBucket(bound, bucketVal); + HistogramBuilder_.SetTime(time); + HistogramBuilder_.SetLabels(std::move(labels)); + } else if (NPrometheus::IsCount(name)) { + // translate x_count metric as COUNTER metric + ConsumeCounter(name, labels, time, value); + } else if (NPrometheus::IsSum(name)) { + // translate x_sum metric as GAUGE metric + ConsumeGauge(name, labels, time, value); + } else { + Y_PARSER_FAIL( + "metric " << name << + " should be part of HISTOGRAM " << baseName); + } + break; + + case EPrometheusMetricType::SUMMARY: + if (NPrometheus::IsCount(name)) { + // translate x_count metric as COUNTER metric + ConsumeCounter(name, labels, time, value); + } else if (NPrometheus::IsSum(name)) { + // translate x_sum metric as GAUGE metric + ConsumeGauge(name, labels, time, value); + } else { + ConsumeGauge(name, labels, time, value); + } + break; + + case EPrometheusMetricType::COUNTER: + ConsumeCounter(name, labels, time, value); + break; + + case EPrometheusMetricType::GAUGE: + ConsumeGauge(name, labels, time, value); + break; + + case EPrometheusMetricType::UNTYPED: + ConsumeGauge(name, labels, time, value); + break; + } + + Y_PARSER_ENSURE(CurrentByte_ == '\n', "expected '\\n', found '" << CurrentByte_ << '\''); + } + + // { name = "value", name2 = "value2", } + TLabelsMap ReadLabels() { + TLabelsMap labels; + if (CurrentByte_ != '{') { + return labels; + } + + SkipExpectedChar('{'); + SkipSpaces(); + + while (CurrentByte_ != '}') { + TStringBuf name = ReadTokenAsLabelName(); + SkipSpaces(); + + SkipExpectedChar('='); + SkipSpaces(); + + TString value = ReadTokenAsLabelValue(); + SkipSpaces(); + labels.emplace(name, value); + + if (CurrentByte_ == ',') { + SkipExpectedChar(','); + SkipSpaces(); + } + } + + SkipExpectedChar('}'); + return labels; + } + + EPrometheusMetricType ReadType() { + TStringBuf keyword = ReadToken(); + if (AsciiEqualsIgnoreCase(keyword, "GAUGE")) { + return EPrometheusMetricType::GAUGE; + } else if (AsciiEqualsIgnoreCase(keyword, "COUNTER")) { + return EPrometheusMetricType::COUNTER; + } else if (AsciiEqualsIgnoreCase(keyword, "SUMMARY")) { + return EPrometheusMetricType::SUMMARY; + } else if (AsciiEqualsIgnoreCase(keyword, "HISTOGRAM")) { + return EPrometheusMetricType::HISTOGRAM; + } else if (AsciiEqualsIgnoreCase(keyword, "UNTYPED")) { + return EPrometheusMetricType::UNTYPED; + } + + Y_PARSER_FAIL( + "unknown metric type: " << keyword << + " at line #" << CurrentLine_); + } + + Y_FORCE_INLINE void ReadNextByteUnsafe() { + CurrentByte_ = Data_[CurrentPos_++]; + } + + Y_FORCE_INLINE bool IsSpace(char ch) { + return ch == ' ' || ch == '\t'; + } + + void ReadNextByte() { + Y_PARSER_ENSURE(HasRemaining(), "unexpected end of file"); + ReadNextByteUnsafe(); + } + + void SkipExpectedChar(char ch) { + Y_PARSER_ENSURE(CurrentByte_ == ch, + "expected '" << CurrentByte_ << "', found '" << ch << '\''); + ReadNextByte(); + } + + void SkipSpaces() { + while (HasRemaining() && IsSpace(CurrentByte_)) { + ReadNextByteUnsafe(); + } + } + + void SkipUntilEol() { + while (HasRemaining() && CurrentByte_ != '\n') { + ReadNextByteUnsafe(); + } + } + + TStringBuf ReadToken() { + Y_VERIFY_DEBUG(CurrentPos_ > 0); + size_t begin = CurrentPos_ - 1; // read first byte again + while (HasRemaining() && !IsSpace(CurrentByte_) && CurrentByte_ != '\n') { + ReadNextByteUnsafe(); + } + return TokenFromPos(begin); + } + + TStringBuf ReadTokenAsMetricName() { + if (!NPrometheus::IsValidMetricNameStart(CurrentByte_)) { + return ""; + } + + Y_VERIFY_DEBUG(CurrentPos_ > 0); + size_t begin = CurrentPos_ - 1; // read first byte again + while (HasRemaining()) { + ReadNextByteUnsafe(); + if (!NPrometheus::IsValidMetricNameContinuation(CurrentByte_)) { + break; + } + } + return TokenFromPos(begin); + } + + TStringBuf ReadTokenAsLabelName() { + if (!NPrometheus::IsValidLabelNameStart(CurrentByte_)) { + return ""; + } + + Y_VERIFY_DEBUG(CurrentPos_ > 0); + size_t begin = CurrentPos_ - 1; // read first byte again + while (HasRemaining()) { + ReadNextByteUnsafe(); + if (!NPrometheus::IsValidLabelNameContinuation(CurrentByte_)) { + break; + } + } + return TokenFromPos(begin); + } + + TString ReadTokenAsLabelValue() { + TString labelValue; + + SkipExpectedChar('"'); + for (ui32 i = 0; i < MAX_LABEL_VALUE_LEN; i++) { + switch (CurrentByte_) { + case '"': + SkipExpectedChar('"'); + return labelValue; + + case '\n': + Y_PARSER_FAIL("label value contains unescaped new-line"); + + case '\\': + ReadNextByte(); + switch (CurrentByte_) { + case '"': + case '\\': + labelValue.append(CurrentByte_); + break; + case 'n': + labelValue.append('\n'); + break; + default: + Y_PARSER_FAIL("invalid escape sequence '" << CurrentByte_ << '\''); + } + break; + + default: + labelValue.append(CurrentByte_); + break; + } + + ReadNextByte(); + } + + Y_PARSER_FAIL("trying to parse too long label value, size >= " << MAX_LABEL_VALUE_LEN); + } + + TStringBuf TokenFromPos(size_t begin) { + Y_VERIFY_DEBUG(CurrentPos_ > begin); + size_t len = CurrentPos_ - begin - 1; + if (len == 0) { + return {}; + } + + return Data_.SubString(begin, len); + } + + void ConsumeLabels(TStringBuf name, const TLabelsMap& labels) { + Y_PARSER_ENSURE(labels.count(MetricNameLabel_) == 0, + "label name '" << MetricNameLabel_ << + "' is reserved, but is used with metric: " << name << LabelsToStr(labels)); + + Consumer_->OnLabelsBegin(); + Consumer_->OnLabel(MetricNameLabel_, TString(name)); // TODO: remove this string allocation + for (const auto& it: labels) { + Consumer_->OnLabel(it.first, it.second); + } + Consumer_->OnLabelsEnd(); + } + + void ConsumeCounter(TStringBuf name, const TLabelsMap& labels, TInstant time, double value) { + i64 intValue{0}; + // not nan + if (value == value) { + Y_PARSER_ENSURE(TryStaticCast(value, intValue), "value " << value << " is out of range"); + } + + // see https://st.yandex-team.ru/SOLOMON-4142 for more details + // why we convert Prometheus COUNTER into Solomon RATE + // TODO: need to fix after server-side aggregation become correct for COUNTERs + Consumer_->OnMetricBegin(EMetricType::RATE); + ConsumeLabels(name, labels); + Consumer_->OnUint64(time, intValue); + Consumer_->OnMetricEnd(); + } + + void ConsumeGauge(TStringBuf name, const TLabelsMap& labels, TInstant time, double value) { + Consumer_->OnMetricBegin(EMetricType::GAUGE); + ConsumeLabels(name, labels); + Consumer_->OnDouble(time, value); + Consumer_->OnMetricEnd(); + } + + void ConsumeHistogram() { + Consumer_->OnMetricBegin(EMetricType::HIST_RATE); + ConsumeLabels(HistogramBuilder_.GetName(), HistogramBuilder_.GetLabels()); + auto time = HistogramBuilder_.GetTime(); + auto hist = HistogramBuilder_.ToSnapshot(); + Consumer_->OnHistogram(time, std::move(hist)); + Consumer_->OnMetricEnd(); + } + + double ParseGoDouble(TStringBuf str) { + if (str == TStringBuf("+Inf")) { + return std::numeric_limits<double>::infinity(); + } else if (str == TStringBuf("-Inf")) { + return -std::numeric_limits<double>::infinity(); + } else if (str == TStringBuf("NaN")) { + return NAN; + } + + double r = 0.0; + if (TryFromString(str, r)) { + return r; + } + Y_PARSER_FAIL("cannot parse double value from '" << str << "\' at line #" << CurrentLine_); + } + + private: + TStringBuf Data_; + IMetricConsumer* Consumer_; + TStringBuf MetricNameLabel_; + THashMap<TString, EPrometheusMetricType> SeenTypes_; + THistogramBuilder HistogramBuilder_; + + ui32 CurrentLine_ = 1; + ui32 CurrentPos_ = 0; + char CurrentByte_ = 0; + }; + } // namespace + +void DecodePrometheus(TStringBuf data, IMetricConsumer* c, TStringBuf metricNameLabel) { + TPrometheusReader reader(data, c, metricNameLabel); + reader.Read(); +} + +} // namespace NMonitoring diff --git a/library/cpp/monlib/encode/prometheus/prometheus_decoder_ut.cpp b/library/cpp/monlib/encode/prometheus/prometheus_decoder_ut.cpp new file mode 100644 index 0000000000..49c2244fb4 --- /dev/null +++ b/library/cpp/monlib/encode/prometheus/prometheus_decoder_ut.cpp @@ -0,0 +1,478 @@ +#include "prometheus.h" + +#include <library/cpp/monlib/encode/protobuf/protobuf.h> + +#include <library/cpp/testing/unittest/registar.h> + +using namespace NMonitoring; + +#define ASSERT_LABEL_EQUAL(label, name, value) do { \ + UNIT_ASSERT_STRINGS_EQUAL((label).GetName(), name); \ + UNIT_ASSERT_STRINGS_EQUAL((label).GetValue(), value); \ + } while (false) + +#define ASSERT_DOUBLE_POINT(s, time, value) do { \ + UNIT_ASSERT_VALUES_EQUAL((s).GetTime(), (time).MilliSeconds()); \ + UNIT_ASSERT_EQUAL((s).GetValueCase(), NProto::TSingleSample::kFloat64); \ + UNIT_ASSERT_DOUBLES_EQUAL((s).GetFloat64(), value, std::numeric_limits<double>::epsilon()); \ + } while (false) + +#define ASSERT_UINT_POINT(s, time, value) do { \ + UNIT_ASSERT_VALUES_EQUAL((s).GetTime(), (time).MilliSeconds()); \ + UNIT_ASSERT_EQUAL((s).GetValueCase(), NProto::TSingleSample::kUint64); \ + UNIT_ASSERT_VALUES_EQUAL((s).GetUint64(), value); \ + } while (false) + +#define ASSERT_HIST_POINT(s, time, expected) do { \ + UNIT_ASSERT_VALUES_EQUAL((s).GetTime(), time.MilliSeconds()); \ + UNIT_ASSERT_EQUAL((s).GetValueCase(), NProto::TSingleSample::kHistogram);\ + UNIT_ASSERT_VALUES_EQUAL((s).GetHistogram().BoundsSize(), (expected).Count()); \ + UNIT_ASSERT_VALUES_EQUAL((s).GetHistogram().ValuesSize(), (expected).Count()); \ + for (size_t i = 0; i < (s).GetHistogram().BoundsSize(); i++) { \ + UNIT_ASSERT_DOUBLES_EQUAL((s).GetHistogram().GetBounds(i), (expected).UpperBound(i), Min<double>()); \ + UNIT_ASSERT_VALUES_EQUAL((s).GetHistogram().GetValues(i), (expected).Value(i)); \ + } \ + } while (false) + +Y_UNIT_TEST_SUITE(TPrometheusDecoderTest) { + + NProto::TSingleSamplesList Decode(TStringBuf data) { + NProto::TSingleSamplesList samples; + { + IMetricEncoderPtr e = EncoderProtobuf(&samples); + DecodePrometheus(data, e.Get()); + } + return samples; + } + + Y_UNIT_TEST(Empty) { + { + auto samples = Decode(""); + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 0); + } + { + auto samples = Decode("\n"); + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 0); + } + { + auto samples = Decode("\n \n \n"); + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 0); + } + { + auto samples = Decode("\t\n\t\n"); + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 0); + } + } + + Y_UNIT_TEST(Minimal) { + auto samples = Decode( + "minimal_metric 1.234\n" + "another_metric -3e3 103948\n" + "# Even that:\n" + "no_labels{} 3\n" + "# HELP line for non-existing metric will be ignored.\n"); + + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 3); + { + auto& s = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(1, s.LabelsSize()); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "minimal_metric"); + ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 1.234); + } + { + auto& s = samples.GetSamples(1); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 1); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "another_metric"); + ASSERT_DOUBLE_POINT(s, TInstant::MilliSeconds(103948), -3000.0); + } + { + auto& s = samples.GetSamples(2); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(1, s.LabelsSize()); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "no_labels"); + ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 3.0); + } + } + + Y_UNIT_TEST(Counter) { + auto samples = Decode( + "# A normal comment.\n" + "#\n" + "# TYPE name counter\n" + "name{labelname=\"val1\",basename=\"basevalue\"} NaN\n" + "name {labelname=\"val2\",basename=\"basevalue\"} 2.3 1234567890\n" + "# HELP name two-line\\n doc str\\\\ing\n"); + + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 2); + + { + auto& s = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 3); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "name"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "basename", "basevalue"); + ASSERT_LABEL_EQUAL(s.GetLabels(2), "labelname", "val1"); + ASSERT_UINT_POINT(s, TInstant::Zero(), ui64(0)); + } + { + auto& s = samples.GetSamples(1); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 3); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "name"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "basename", "basevalue"); + ASSERT_LABEL_EQUAL(s.GetLabels(2), "labelname", "val2"); + ASSERT_UINT_POINT(s, TInstant::MilliSeconds(1234567890), i64(2)); + } + } + + Y_UNIT_TEST(Gauge) { + auto samples = Decode( + "# A normal comment.\n" + "#\n" + " # HELP name2 \tdoc str\"ing 2\n" + " # TYPE name2 gauge\n" + "name2{labelname=\"val2\"\t,basename = \"basevalue2\"\t\t} +Inf 54321\n" + "name2{ labelname = \"val1\" , }-Inf\n"); + + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 2); + + { + auto& s = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 3); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "name2"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "basename", "basevalue2"); + ASSERT_LABEL_EQUAL(s.GetLabels(2), "labelname", "val2"); + ASSERT_DOUBLE_POINT(s, TInstant::MilliSeconds(54321), INFINITY); + } + { + auto& s = samples.GetSamples(1); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "name2"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "labelname", "val1"); + ASSERT_DOUBLE_POINT(s, TInstant::Zero(), -INFINITY); + } + } + + Y_UNIT_TEST(Summary) { + auto samples = Decode( + "# HELP \n" + "# TYPE my_summary summary\n" + "my_summary{n1=\"val1\",quantile=\"0.5\"} 110\n" + "my_summary{n1=\"val1\",quantile=\"0.9\"} 140 1\n" + "my_summary_count{n1=\"val1\"} 42\n" + "my_summary_sum{n1=\"val1\"} 08 15\n" + "# some\n" + "# funny comments\n" + "# HELP\n" + "# HELP my_summary\n" + "# HELP my_summary \n"); + + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 4); + + { + auto& s = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 3); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "my_summary"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "quantile", "0.5"); + ASSERT_LABEL_EQUAL(s.GetLabels(2), "n1", "val1"); + ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 110.0); + } + { + auto& s = samples.GetSamples(1); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 3); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "my_summary"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "quantile", "0.9"); + ASSERT_LABEL_EQUAL(s.GetLabels(2), "n1", "val1"); + ASSERT_DOUBLE_POINT(s, TInstant::MilliSeconds(1), 140.0); + } + { + auto& s = samples.GetSamples(2); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "my_summary_count"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "n1", "val1"); + ASSERT_UINT_POINT(s, TInstant::Zero(), 42); + } + { + auto& s = samples.GetSamples(3); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "my_summary_sum"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "n1", "val1"); + ASSERT_DOUBLE_POINT(s, TInstant::MilliSeconds(15), 8.0); + } + } + + Y_UNIT_TEST(Histogram) { + auto samples = Decode( + "# HELP request_duration_microseconds The response latency.\n" + "# TYPE request_duration_microseconds histogram\n" + "request_duration_microseconds_bucket{le=\"0\"} 0\n" + "request_duration_microseconds_bucket{le=\"100\"} 123\n" + "request_duration_microseconds_bucket{le=\"120\"} 412\n" + "request_duration_microseconds_bucket{le=\"144\"} 592\n" + "request_duration_microseconds_bucket{le=\"172.8\"} 1524\n" + "request_duration_microseconds_bucket{le=\"+Inf\"} 2693\n" + "request_duration_microseconds_sum 1.7560473e+06\n" + "request_duration_microseconds_count 2693\n"); + + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 3); + + { + auto& s = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 1); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "request_duration_microseconds_sum"); + ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 1756047.3); + } + { + auto& s = samples.GetSamples(1); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 1); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "request_duration_microseconds_count"); + ASSERT_UINT_POINT(s, TInstant::Zero(), 2693); + } + { + auto& s = samples.GetSamples(2); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::HIST_RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 1); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "request_duration_microseconds"); + auto hist = ExplicitHistogramSnapshot( + { 0, 100, 120, 144, 172.8, HISTOGRAM_INF_BOUND }, + { 0, 123, 289, 180, 932, 1169 }); + ASSERT_HIST_POINT(s, TInstant::Zero(), *hist); + } + } + + Y_UNIT_TEST(HistogramWithLabels) { + auto samples = Decode( + "# A histogram, which has a pretty complex representation in the text format:\n" + "# HELP http_request_duration_seconds A histogram of the request duration.\n" + "# TYPE http_request_duration_seconds histogram\n" + "http_request_duration_seconds_bucket{le=\"0.05\", method=\"POST\"} 24054\n" + "http_request_duration_seconds_bucket{method=\"POST\", le=\"0.1\"} 33444\n" + "http_request_duration_seconds_bucket{le=\"0.2\", method=\"POST\", } 100392\n" + "http_request_duration_seconds_bucket{le=\"0.5\",method=\"POST\",} 129389\n" + "http_request_duration_seconds_bucket{ method=\"POST\", le=\"1\", } 133988\n" + "http_request_duration_seconds_bucket{ le=\"+Inf\", method=\"POST\", } 144320\n" + "http_request_duration_seconds_sum{method=\"POST\"} 53423\n" + "http_request_duration_seconds_count{ method=\"POST\", } 144320\n"); + + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 3); + + { + auto& s = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "http_request_duration_seconds_sum"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "method", "POST"); + ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 53423.0); + } + { + auto& s = samples.GetSamples(1); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "http_request_duration_seconds_count"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "method", "POST"); + ASSERT_UINT_POINT(s, TInstant::Zero(), 144320); + } + { + auto& s = samples.GetSamples(2); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::HIST_RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "http_request_duration_seconds"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "method", "POST"); + auto hist = ExplicitHistogramSnapshot( + { 0.05, 0.1, 0.2, 0.5, 1, HISTOGRAM_INF_BOUND }, + { 24054, 9390, 66948, 28997, 4599, 10332 }); + ASSERT_HIST_POINT(s, TInstant::Zero(), *hist); + } + } + + Y_UNIT_TEST(MultipleHistograms) { + auto samples = Decode( + "# TYPE inboundBytesPerSec histogram\n" + "inboundBytesPerSec_bucket{client=\"mbus\", le=\"10.0\"} 1.0\n" + "inboundBytesPerSec_bucket{client=\"mbus\", le=\"20.0\"} 5.0\n" + "inboundBytesPerSec_bucket{client=\"mbus\", le=\"+Inf\"} 5.0\n" + "inboundBytesPerSec_count{client=\"mbus\"} 5.0\n" + "inboundBytesPerSec_bucket{client=\"grpc\", le=\"10.0\"} 1.0\n" + "inboundBytesPerSec_bucket{client=\"grpc\", le=\"20.0\"} 5.0\n" + "inboundBytesPerSec_bucket{client=\"grpc\", le=\"30.0\"} 5.0\n" + "inboundBytesPerSec_count{client=\"grpc\"} 5.0\n" + "# TYPE outboundBytesPerSec histogram\n" + "outboundBytesPerSec_bucket{client=\"grpc\", le=\"100.0\"} 1.0 1512216000000\n" + "outboundBytesPerSec_bucket{client=\"grpc\", le=\"200.0\"} 1.0 1512216000000\n" + "outboundBytesPerSec_bucket{client=\"grpc\", le=\"+Inf\"} 1.0 1512216000000\n" + "outboundBytesPerSec_count{client=\"grpc\"} 1.0 1512216000000\n"); + + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 6); + + { + auto& s = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "inboundBytesPerSec_count"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "client", "mbus"); + ASSERT_UINT_POINT(s, TInstant::Zero(), 5); + } + { + auto& s = samples.GetSamples(1); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::HIST_RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "inboundBytesPerSec"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "client", "mbus"); + auto hist = ExplicitHistogramSnapshot( + { 10, 20, HISTOGRAM_INF_BOUND }, + { 1, 4, 0 }); + ASSERT_HIST_POINT(s, TInstant::Zero(), *hist); + } + { + auto& s = samples.GetSamples(2); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "inboundBytesPerSec_count"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "client", "grpc"); + ASSERT_UINT_POINT(s, TInstant::Zero(), 5); + } + { + auto& s = samples.GetSamples(3); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::HIST_RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "inboundBytesPerSec"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "client", "grpc"); + auto hist = ExplicitHistogramSnapshot( + { 10, 20, 30 }, + { 1, 4, 0 }); + ASSERT_HIST_POINT(s, TInstant::Zero(), *hist); + } + { + auto& s = samples.GetSamples(4); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "outboundBytesPerSec_count"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "client", "grpc"); + ASSERT_UINT_POINT(s, TInstant::Seconds(1512216000), 1) ; + } + { + auto& s = samples.GetSamples(5); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::HIST_RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "outboundBytesPerSec"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "client", "grpc"); + auto hist = ExplicitHistogramSnapshot( + { 100, 200, HISTOGRAM_INF_BOUND }, + { 1, 0, 0 }); + ASSERT_HIST_POINT(s, TInstant::Seconds(1512216000), *hist); + } + } + + Y_UNIT_TEST(MixedTypes) { + auto samples = Decode( + "# HELP http_requests_total The total number of HTTP requests.\n" + "# TYPE http_requests_total counter\n" + "http_requests_total { } 1027 1395066363000\n" + "http_requests_total{method=\"post\",code=\"200\"} 1027 1395066363000\n" + "http_requests_total{method=\"post\",code=\"400\"} 3 1395066363000\n" + "\n" + "# Minimalistic line:\n" + "metric_without_timestamp_and_labels 12.47\n" + "\n" + "# HELP rpc_duration_seconds A summary of the RPC duration in seconds.\n" + "# TYPE rpc_duration_seconds summary\n" + "rpc_duration_seconds{quantile=\"0.01\"} 3102\n" + "rpc_duration_seconds{quantile=\"0.5\"} 4773\n" + "rpc_duration_seconds{quantile=\"0.9\"} 9001\n" + "rpc_duration_seconds_sum 1.7560473e+07\n" + "rpc_duration_seconds_count 2693\n" + "\n" + "# Another mMinimalistic line:\n" + "metric_with_timestamp 12.47 1234567890\n"); + + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 10); + + { + auto& s = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 1); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "http_requests_total"); + ASSERT_UINT_POINT(s, TInstant::Seconds(1395066363), 1027); + } + { + auto& s = samples.GetSamples(1); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 3); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "http_requests_total"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "method", "post"); + ASSERT_LABEL_EQUAL(s.GetLabels(2), "code", "200"); + ASSERT_UINT_POINT(s, TInstant::Seconds(1395066363), 1027); + } + { + auto& s = samples.GetSamples(2); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 3); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "http_requests_total"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "method", "post"); + ASSERT_LABEL_EQUAL(s.GetLabels(2), "code", "400"); + ASSERT_UINT_POINT(s, TInstant::Seconds(1395066363), 3); + } + { + auto& s = samples.GetSamples(3); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 1); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "metric_without_timestamp_and_labels"); + ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 12.47); + } + { + auto& s = samples.GetSamples(4); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "rpc_duration_seconds"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "quantile", "0.01"); + ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 3102); + } + { + auto& s = samples.GetSamples(5); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "rpc_duration_seconds"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "quantile", "0.5"); + ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 4773); + } + { + auto& s = samples.GetSamples(6); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "rpc_duration_seconds"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "quantile", "0.9"); + ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 9001); + } + { + auto& s = samples.GetSamples(7); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 1); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "rpc_duration_seconds_sum"); + ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 17560473); + } + { + auto& s = samples.GetSamples(8); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 1); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "rpc_duration_seconds_count"); + ASSERT_UINT_POINT(s, TInstant::Zero(), 2693); + } + { + auto& s = samples.GetSamples(9); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 1); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "metric_with_timestamp"); + ASSERT_DOUBLE_POINT(s, TInstant::MilliSeconds(1234567890), 12.47); + } + } +} diff --git a/library/cpp/monlib/encode/prometheus/prometheus_encoder.cpp b/library/cpp/monlib/encode/prometheus/prometheus_encoder.cpp new file mode 100644 index 0000000000..15efeb8c03 --- /dev/null +++ b/library/cpp/monlib/encode/prometheus/prometheus_encoder.cpp @@ -0,0 +1,413 @@ +#include "prometheus.h" +#include "prometheus_model.h" + +#include <library/cpp/monlib/encode/encoder_state.h> +#include <library/cpp/monlib/metrics/labels.h> +#include <library/cpp/monlib/metrics/metric_value.h> + +#include <util/string/cast.h> +#include <util/generic/hash_set.h> + + +namespace NMonitoring { + namespace { + /////////////////////////////////////////////////////////////////////// + // TPrometheusWriter + /////////////////////////////////////////////////////////////////////// + class TPrometheusWriter { + public: + explicit TPrometheusWriter(IOutputStream* out) + : Out_(out) + { + } + + void WriteType(EMetricType type, const TString& name) { + auto r = WrittenTypes_.insert(name); + if (!r.second) { + // type for this metric was already written + return; + } + + Out_->Write("# TYPE "); + WriteMetricName(name); + Out_->Write(' '); + + switch (type) { + case EMetricType::GAUGE: + case EMetricType::IGAUGE: + Out_->Write("gauge"); + break; + case EMetricType::RATE: + case EMetricType::COUNTER: + Out_->Write("counter"); + break; + case EMetricType::HIST: + case EMetricType::HIST_RATE: + Out_->Write("histogram"); + break; + case EMetricType::LOGHIST: + // TODO(@kbalakirev): implement this case + break; + case EMetricType::DSUMMARY: + ythrow yexception() << "writing summary type is forbiden"; + case EMetricType::UNKNOWN: + ythrow yexception() << "unknown metric type: " << MetricTypeToStr(type) + << ", name: " << name; + } + Out_->Write('\n'); + } + + void WriteDouble(TStringBuf name, const TLabels& labels, TInstant time, double value) { + WriteValue(name, "", labels, "", "", time, value); + } + + void WriteHistogram(TStringBuf name, const TLabels& labels, TInstant time, IHistogramSnapshot* h) { + Y_ENSURE(!labels.Has(NPrometheus::BUCKET_LABEL), + "histogram metric " << name << " has label '" << + NPrometheus::BUCKET_LABEL << "' which is reserved in Prometheus"); + + double totalCount = 0; + for (ui32 i = 0, count = h->Count(); i < count; i++) { + TBucketBound bound = h->UpperBound(i); + TStringBuf boundStr; + if (bound == HISTOGRAM_INF_BOUND) { + boundStr = TStringBuf("+Inf"); + } else { + size_t len = FloatToString(bound, TmpBuf_, Y_ARRAY_SIZE(TmpBuf_)); + boundStr = TStringBuf(TmpBuf_, len); + } + + TBucketValue value = h->Value(i); + totalCount += static_cast<double>(value); + + WriteValue( + name, NPrometheus::BUCKET_SUFFIX, + labels, NPrometheus::BUCKET_LABEL, boundStr, + time, + totalCount); + } + + WriteValue(name, NPrometheus::COUNT_SUFFIX, labels, "", "", time, totalCount); + } + + void WriteSummaryDouble(TStringBuf name, const TLabels& labels, TInstant time, ISummaryDoubleSnapshot* s) { + WriteValue(name, NPrometheus::SUM_SUFFIX, labels, "", "", time, s->GetSum()); + WriteValue(name, NPrometheus::MIN_SUFFIX, labels, "", "", time, s->GetMin()); + WriteValue(name, NPrometheus::MAX_SUFFIX, labels, "", "", time, s->GetMax()); + WriteValue(name, NPrometheus::LAST_SUFFIX, labels, "", "", time, s->GetLast()); + WriteValue(name, NPrometheus::COUNT_SUFFIX, labels, "", "", time, s->GetCount()); + } + + void WriteLn() { + Out_->Write('\n'); + } + + private: + // will replace invalid chars with '_' + void WriteMetricName(TStringBuf name) { + Y_ENSURE(!name.Empty(), "trying to write metric with empty name"); + + char ch = name[0]; + if (NPrometheus::IsValidMetricNameStart(ch)) { + Out_->Write(ch); + } else { + Out_->Write('_'); + } + + for (size_t i = 1, len = name.length(); i < len; i++) { + ch = name[i]; + if (NPrometheus::IsValidMetricNameContinuation(ch)) { + Out_->Write(ch); + } else { + Out_->Write('_'); + } + } + } + + void WriteLabels(const TLabels& labels, TStringBuf addLabelKey, TStringBuf addLabelValue) { + Out_->Write('{'); + for (auto&& l: labels) { + Out_->Write(l.Name()); + Out_->Write('='); + WriteLabelValue(l.Value()); + Out_->Write(", "); // trailign comma is supported in parsers + } + if (!addLabelKey.Empty() && !addLabelValue.Empty()) { + Out_->Write(addLabelKey); + Out_->Write('='); + WriteLabelValue(addLabelValue); + } + Out_->Write('}'); + } + + void WriteLabelValue(TStringBuf value) { + Out_->Write('"'); + for (char ch: value) { + if (ch == '"') { + Out_->Write("\\\""); + } else if (ch == '\\') { + Out_->Write("\\\\"); + } else if (ch == '\n') { + Out_->Write("\\n"); + } else { + Out_->Write(ch); + } + } + Out_->Write('"'); + } + + void WriteValue( + TStringBuf name, TStringBuf suffix, + const TLabels& labels, TStringBuf addLabelKey, TStringBuf addLabelValue, + TInstant time, double value) + { + // (1) name + WriteMetricName(name); + if (!suffix.Empty()) { + Out_->Write(suffix); + } + + // (2) labels + if (!labels.Empty() || !addLabelKey.Empty()) { + WriteLabels(labels, addLabelKey, addLabelValue); + } + Out_->Write(' '); + + // (3) value + { + size_t len = FloatToString(value, TmpBuf_, Y_ARRAY_SIZE(TmpBuf_)); + Out_->Write(TmpBuf_, len); + } + + // (4) time + if (ui64 timeMillis = time.MilliSeconds()) { + Out_->Write(' '); + size_t len = IntToString<10>(timeMillis, TmpBuf_, Y_ARRAY_SIZE(TmpBuf_)); + Out_->Write(TmpBuf_, len); + } + Out_->Write('\n'); + } + + private: + IOutputStream* Out_; + THashSet<TString> WrittenTypes_; + char TmpBuf_[512]; // used to convert doubles to strings + }; + + /////////////////////////////////////////////////////////////////////// + // TMetricState + /////////////////////////////////////////////////////////////////////// + struct TMetricState { + EMetricType Type = EMetricType::UNKNOWN; + TLabels Labels; + TInstant Time = TInstant::Zero(); + EMetricValueType ValueType = EMetricValueType::UNKNOWN; + TMetricValue Value; + + ~TMetricState() { + ClearValue(); + } + + void Clear() { + Type = EMetricType::UNKNOWN; + Labels.Clear(); + Time = TInstant::Zero(); + ClearValue(); + } + + void ClearValue() { + // TMetricValue does not keep ownership of histogram + if (ValueType == EMetricValueType::HISTOGRAM) { + Value.AsHistogram()->UnRef(); + } else if (ValueType == EMetricValueType::SUMMARY) { + Value.AsSummaryDouble()->UnRef(); + } + ValueType = EMetricValueType::UNKNOWN; + Value = {}; + } + + template <typename T> + void SetValue(T value) { + // TMetricValue does not keep ownership of histogram + if (ValueType == EMetricValueType::HISTOGRAM) { + Value.AsHistogram()->UnRef(); + } else if (ValueType == EMetricValueType::SUMMARY) { + Value.AsSummaryDouble()->UnRef(); + } + ValueType = TValueType<T>::Type; + Value = TMetricValue(value); + if (ValueType == EMetricValueType::HISTOGRAM) { + Value.AsHistogram()->Ref(); + } else if (ValueType == EMetricValueType::SUMMARY) { + Value.AsSummaryDouble()->Ref(); + } + } + }; + + /////////////////////////////////////////////////////////////////////// + // TPrometheusEncoder + /////////////////////////////////////////////////////////////////////// + class TPrometheusEncoder final: public IMetricEncoder { + public: + explicit TPrometheusEncoder(IOutputStream* out, TStringBuf metricNameLabel) + : Writer_(out) + , MetricNameLabel_(metricNameLabel) + { + } + + private: + void OnStreamBegin() override { + State_.Expect(TEncoderState::EState::ROOT); + } + + void OnStreamEnd() override { + State_.Expect(TEncoderState::EState::ROOT); + Writer_.WriteLn(); + } + + void OnCommonTime(TInstant time) override { + State_.Expect(TEncoderState::EState::ROOT); + CommonTime_ = time; + } + + void OnMetricBegin(EMetricType type) override { + State_.Switch(TEncoderState::EState::ROOT, TEncoderState::EState::METRIC); + MetricState_.Clear(); + MetricState_.Type = type; + } + + void OnMetricEnd() override { + State_.Switch(TEncoderState::EState::METRIC, TEncoderState::EState::ROOT); + WriteMetric(); + } + + void OnLabelsBegin() override { + if (State_ == TEncoderState::EState::METRIC) { + State_ = TEncoderState::EState::METRIC_LABELS; + } else if (State_ == TEncoderState::EState::ROOT) { + State_ = TEncoderState::EState::COMMON_LABELS; + } else { + State_.ThrowInvalid("expected METRIC or ROOT"); + } + } + + void OnLabelsEnd() override { + if (State_ == TEncoderState::EState::METRIC_LABELS) { + State_ = TEncoderState::EState::METRIC; + } else if (State_ == TEncoderState::EState::COMMON_LABELS) { + State_ = TEncoderState::EState::ROOT; + } else { + State_.ThrowInvalid("expected LABELS or COMMON_LABELS"); + } + } + + void OnLabel(TStringBuf name, TStringBuf value) override { + if (State_ == TEncoderState::EState::METRIC_LABELS) { + MetricState_.Labels.Add(name, value); + } else if (State_ == TEncoderState::EState::COMMON_LABELS) { + CommonLabels_.Add(name, value); + } else { + State_.ThrowInvalid("expected LABELS or COMMON_LABELS"); + } + } + + void OnDouble(TInstant time, double value) override { + State_.Expect(TEncoderState::EState::METRIC); + MetricState_.Time = time; + MetricState_.SetValue(value); + } + + void OnInt64(TInstant time, i64 value) override { + State_.Expect(TEncoderState::EState::METRIC); + MetricState_.Time = time; + MetricState_.SetValue(value); + } + + void OnUint64(TInstant time, ui64 value) override { + State_.Expect(TEncoderState::EState::METRIC); + MetricState_.Time = time; + MetricState_.SetValue(value); + } + + void OnHistogram(TInstant time, IHistogramSnapshotPtr snapshot) override { + State_.Expect(TEncoderState::EState::METRIC); + MetricState_.Time = time; + MetricState_.SetValue(snapshot.Get()); + } + + void OnSummaryDouble(TInstant time, ISummaryDoubleSnapshotPtr snapshot) override { + State_.Expect(TEncoderState::EState::METRIC); + MetricState_.Time = time; + MetricState_.SetValue(snapshot.Get()); + } + + void OnLogHistogram(TInstant, TLogHistogramSnapshotPtr) override { + // TODO(@kbalakirev): implement this function + } + + void Close() override { + } + + void WriteMetric() { + if (MetricState_.ValueType == EMetricValueType::UNKNOWN) { + return; + } + + // XXX: poor performace + for (auto&& l: CommonLabels_) { + MetricState_.Labels.Add(l.Name(), l.Value()); + } + + TMaybe<TLabel> nameLabel = MetricState_.Labels.Extract(MetricNameLabel_); + Y_ENSURE(nameLabel, + "labels " << MetricState_.Labels << + " does not contain label '" << MetricNameLabel_ << '\''); + + const TString& metricName = ToString(nameLabel->Value()); + if (MetricState_.Type != EMetricType::DSUMMARY) { + Writer_.WriteType(MetricState_.Type, metricName); + } + + if (MetricState_.Time == TInstant::Zero()) { + MetricState_.Time = CommonTime_; + } + + EMetricType type = MetricState_.Type; + if (type == EMetricType::HIST || type == EMetricType::HIST_RATE) { + Y_ENSURE(MetricState_.ValueType == EMetricValueType::HISTOGRAM, + "invalid value type for histogram: " << int(MetricState_.ValueType)); // TODO: to string conversion + Writer_.WriteHistogram( + metricName, + MetricState_.Labels, + MetricState_.Time, + MetricState_.Value.AsHistogram()); + } else if (type == EMetricType::DSUMMARY) { + Writer_.WriteSummaryDouble( + metricName, + MetricState_.Labels, + MetricState_.Time, + MetricState_.Value.AsSummaryDouble()); + } else { + Writer_.WriteDouble( + metricName, + MetricState_.Labels, + MetricState_.Time, + MetricState_.Value.AsDouble(MetricState_.ValueType)); + } + } + + private: + TEncoderState State_; + TPrometheusWriter Writer_; + TString MetricNameLabel_; + TInstant CommonTime_ = TInstant::Zero(); + TLabels CommonLabels_; + TMetricState MetricState_; + }; + } + + IMetricEncoderPtr EncoderPrometheus(IOutputStream* out, TStringBuf metricNameLabel) { + return MakeHolder<TPrometheusEncoder>(out, metricNameLabel); + } + +} // namespace NMonitoring diff --git a/library/cpp/monlib/encode/prometheus/prometheus_encoder_ut.cpp b/library/cpp/monlib/encode/prometheus/prometheus_encoder_ut.cpp new file mode 100644 index 0000000000..fd9debb060 --- /dev/null +++ b/library/cpp/monlib/encode/prometheus/prometheus_encoder_ut.cpp @@ -0,0 +1,414 @@ +#include "prometheus.h" + +#include <library/cpp/monlib/encode/protobuf/protobuf.h> +#include <library/cpp/monlib/metrics/metric_value.h> +#include <library/cpp/monlib/metrics/histogram_snapshot.h> + +#include <library/cpp/testing/unittest/registar.h> + +#include <util/stream/str.h> + +using namespace NMonitoring; + +Y_UNIT_TEST_SUITE(TPrometheusEncoderTest) { + + template <typename TFunc> + TString EncodeToString(TFunc fn) { + TStringStream ss; + IMetricEncoderPtr encoder = EncoderPrometheus(&ss); + fn(encoder.Get()); + return ss.Str(); + } + + ISummaryDoubleSnapshotPtr TestSummaryDouble() { + return MakeIntrusive<TSummaryDoubleSnapshot>(10.1, -0.45, 0.478, 0.3, 30u); + } + + Y_UNIT_TEST(Empty) { + auto result = EncodeToString([](IMetricEncoder* e) { + e->OnStreamBegin(); + e->OnStreamEnd(); + }); + UNIT_ASSERT_STRINGS_EQUAL(result, "\n"); + } + + Y_UNIT_TEST(DoubleGauge) { + auto result = EncodeToString([](IMetricEncoder* e) { + e->OnStreamBegin(); + { // no values + e->OnMetricBegin(EMetricType::GAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "cpuUsage"); + e->OnLabelsEnd(); + } + e->OnMetricEnd(); + } + { // one value no ts + e->OnMetricBegin(EMetricType::GAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "diskUsage"); + e->OnLabel("disk", "sda1"); + e->OnLabelsEnd(); + } + e->OnDouble(TInstant::Zero(), 1000); + e->OnMetricEnd(); + } + { // one value with ts + e->OnMetricBegin(EMetricType::GAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "memoryUsage"); + e->OnLabel("host", "solomon-man-00"); + e->OnLabel("dc", "man"); + e->OnLabelsEnd(); + } + e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 1000); + e->OnMetricEnd(); + } + { // many values + e->OnMetricBegin(EMetricType::GAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "bytesRx"); + e->OnLabel("host", "solomon-sas-01"); + e->OnLabel("dc", "sas"); + e->OnLabelsEnd(); + } + e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 2); + e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:05Z"), 4); + e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:10Z"), 8); + e->OnMetricEnd(); + } + { // already seen metric name + e->OnMetricBegin(EMetricType::GAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "diskUsage"); + e->OnLabel("disk", "sdb1"); + e->OnLabelsEnd(); + } + e->OnDouble(TInstant::Zero(), 1001); + e->OnMetricEnd(); + } + { // NaN + e->OnMetricBegin(EMetricType::GAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "nanValue"); + e->OnLabelsEnd(); + } + e->OnDouble(TInstant::Zero(), NAN); + e->OnMetricEnd(); + } + { // Inf + e->OnMetricBegin(EMetricType::GAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "infValue"); + e->OnLabelsEnd(); + } + e->OnDouble(TInstant::Zero(), INFINITY); + e->OnMetricEnd(); + } + { + e->OnMetricBegin(EMetricType::DSUMMARY); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "seconds"); + e->OnLabel("disk", "sdb1"); + e->OnLabelsEnd(); + } + e->OnSummaryDouble(TInstant::Zero(), TestSummaryDouble()); + e->OnMetricEnd(); + } + e->OnStreamEnd(); + }); + + UNIT_ASSERT_STRINGS_EQUAL(result, + "# TYPE diskUsage gauge\n" + "diskUsage{disk=\"sda1\", } 1000\n" + "# TYPE memoryUsage gauge\n" + "memoryUsage{host=\"solomon-man-00\", dc=\"man\", } 1000 1512216000000\n" + "# TYPE bytesRx gauge\n" + "bytesRx{host=\"solomon-sas-01\", dc=\"sas\", } 8 1512216010000\n" + "diskUsage{disk=\"sdb1\", } 1001\n" + "# TYPE nanValue gauge\n" + "nanValue nan\n" + "# TYPE infValue gauge\n" + "infValue inf\n" + "seconds_sum{disk=\"sdb1\", } 10.1\n" + "seconds_min{disk=\"sdb1\", } -0.45\n" + "seconds_max{disk=\"sdb1\", } 0.478\n" + "seconds_last{disk=\"sdb1\", } 0.3\n" + "seconds_count{disk=\"sdb1\", } 30\n" + "\n"); + } + + Y_UNIT_TEST(IntGauges) { + auto result = EncodeToString([](IMetricEncoder* e) { + e->OnStreamBegin(); + { // no values + e->OnMetricBegin(EMetricType::IGAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "cpuUsage"); + e->OnLabelsEnd(); + } + e->OnMetricEnd(); + } + { // one value no ts + e->OnMetricBegin(EMetricType::IGAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "diskUsage"); + e->OnLabel("disk", "sda1"); + e->OnLabelsEnd(); + } + e->OnInt64(TInstant::Zero(), 1000); + e->OnMetricEnd(); + } + { // one value with ts + e->OnMetricBegin(EMetricType::IGAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "memoryUsage"); + e->OnLabel("dc", "man"); + e->OnLabel("host", "solomon-man-00"); + e->OnLabelsEnd(); + } + e->OnInt64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 1000); + e->OnMetricEnd(); + } + { // many values + e->OnMetricBegin(EMetricType::IGAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "bytesRx"); + e->OnLabel("dc", "sas"); + e->OnLabel("host", "solomon-sas-01"); + e->OnLabelsEnd(); + } + e->OnInt64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 2); + e->OnInt64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:05Z"), 4); + e->OnInt64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:10Z"), 8); + e->OnMetricEnd(); + } + e->OnStreamEnd(); + }); + + UNIT_ASSERT_STRINGS_EQUAL(result, + "# TYPE diskUsage gauge\n" + "diskUsage{disk=\"sda1\", } 1000\n" + "# TYPE memoryUsage gauge\n" + "memoryUsage{dc=\"man\", host=\"solomon-man-00\", } 1000 1512216000000\n" + "# TYPE bytesRx gauge\n" + "bytesRx{dc=\"sas\", host=\"solomon-sas-01\", } 8 1512216010000\n" + "\n"); + } + + Y_UNIT_TEST(Counters) { + auto result = EncodeToString([](IMetricEncoder* e) { + e->OnStreamBegin(); + { // no values + e->OnMetricBegin(EMetricType::COUNTER); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "cpuUsage"); + e->OnLabelsEnd(); + } + e->OnMetricEnd(); + } + { // one value no ts + e->OnMetricBegin(EMetricType::COUNTER); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "diskUsage"); + e->OnLabel("disk", "sda1"); + e->OnLabelsEnd(); + } + e->OnInt64(TInstant::Zero(), 1000); + e->OnMetricEnd(); + } + { // one value with ts + e->OnMetricBegin(EMetricType::COUNTER); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "memoryUsage"); + e->OnLabel("host", "solomon-man-00"); + e->OnLabel("dc", "man"); + e->OnLabelsEnd(); + } + e->OnInt64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 1000); + e->OnMetricEnd(); + } + { // many values + e->OnMetricBegin(EMetricType::COUNTER); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "bytesRx"); + e->OnLabel("host", "solomon-sas-01"); + e->OnLabel("dc", "sas"); + e->OnLabelsEnd(); + } + e->OnInt64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 2); + e->OnInt64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:05Z"), 4); + e->OnInt64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:10Z"), 8); + e->OnMetricEnd(); + } + e->OnStreamEnd(); + }); + + UNIT_ASSERT_STRINGS_EQUAL(result, + "# TYPE diskUsage counter\n" + "diskUsage{disk=\"sda1\", } 1000\n" + "# TYPE memoryUsage counter\n" + "memoryUsage{host=\"solomon-man-00\", dc=\"man\", } 1000 1512216000000\n" + "# TYPE bytesRx counter\n" + "bytesRx{host=\"solomon-sas-01\", dc=\"sas\", } 8 1512216010000\n" + "\n"); + } + + Y_UNIT_TEST(Histograms) { + auto result = EncodeToString([](IMetricEncoder* e) { + e->OnStreamBegin(); + { // no values histogram + e->OnMetricBegin(EMetricType::HIST); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "cpuUsage"); + e->OnLabelsEnd(); + } + e->OnMetricEnd(); + } + { // one value no ts + e->OnMetricBegin(EMetricType::HIST); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "inboundBytesPerSec"); + e->OnLabel("client", "mbus"); + e->OnLabelsEnd(); + } + e->OnHistogram( + TInstant::Zero(), + ExplicitHistogramSnapshot({10, 20, HISTOGRAM_INF_BOUND}, {1, 4, 0})); + e->OnMetricEnd(); + } + { // one value no ts no +inf bucket + e->OnMetricBegin(EMetricType::HIST); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "inboundBytesPerSec"); + e->OnLabel("client", "grpc"); + e->OnLabelsEnd(); + } + e->OnHistogram( + TInstant::Zero(), + ExplicitHistogramSnapshot({10, 20, 30}, {1, 4, 0})); + e->OnMetricEnd(); + } + { // one value with ts + e->OnMetricBegin(EMetricType::HIST_RATE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "outboundBytesPerSec"); + e->OnLabel("client", "grps"); + e->OnLabelsEnd(); + } + e->OnHistogram( + TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), + ExplicitHistogramSnapshot({100, 200, HISTOGRAM_INF_BOUND}, {1, 0, 0})); + e->OnMetricEnd(); + } + { // many values + e->OnMetricBegin(EMetricType::HIST); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "bytesRx"); + e->OnLabel("host", "solomon-sas-01"); + e->OnLabel("dc", "sas"); + e->OnLabelsEnd(); + } + TBucketBounds bounds = {100, 200, HISTOGRAM_INF_BOUND}; + e->OnHistogram( + TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), + ExplicitHistogramSnapshot(bounds, {10, 0, 0})); + e->OnHistogram( + TInstant::ParseIso8601Deprecated("2017-12-02T12:00:05Z"), + ExplicitHistogramSnapshot(bounds, {10, 2, 0})); + e->OnHistogram( + TInstant::ParseIso8601Deprecated("2017-12-02T12:00:10Z"), + ExplicitHistogramSnapshot(bounds, {10, 2, 5})); + e->OnMetricEnd(); + } + e->OnStreamEnd(); + }); + + UNIT_ASSERT_STRINGS_EQUAL(result, + "# TYPE inboundBytesPerSec histogram\n" + "inboundBytesPerSec_bucket{client=\"mbus\", le=\"10\"} 1\n" + "inboundBytesPerSec_bucket{client=\"mbus\", le=\"20\"} 5\n" + "inboundBytesPerSec_bucket{client=\"mbus\", le=\"+Inf\"} 5\n" + "inboundBytesPerSec_count{client=\"mbus\", } 5\n" + "inboundBytesPerSec_bucket{client=\"grpc\", le=\"10\"} 1\n" + "inboundBytesPerSec_bucket{client=\"grpc\", le=\"20\"} 5\n" + "inboundBytesPerSec_bucket{client=\"grpc\", le=\"30\"} 5\n" + "inboundBytesPerSec_count{client=\"grpc\", } 5\n" + "# TYPE outboundBytesPerSec histogram\n" + "outboundBytesPerSec_bucket{client=\"grps\", le=\"100\"} 1 1512216000000\n" + "outboundBytesPerSec_bucket{client=\"grps\", le=\"200\"} 1 1512216000000\n" + "outboundBytesPerSec_bucket{client=\"grps\", le=\"+Inf\"} 1 1512216000000\n" + "outboundBytesPerSec_count{client=\"grps\", } 1 1512216000000\n" + "# TYPE bytesRx histogram\n" + "bytesRx_bucket{host=\"solomon-sas-01\", dc=\"sas\", le=\"100\"} 10 1512216010000\n" + "bytesRx_bucket{host=\"solomon-sas-01\", dc=\"sas\", le=\"200\"} 12 1512216010000\n" + "bytesRx_bucket{host=\"solomon-sas-01\", dc=\"sas\", le=\"+Inf\"} 17 1512216010000\n" + "bytesRx_count{host=\"solomon-sas-01\", dc=\"sas\", } 17 1512216010000\n" + "\n"); + } + + Y_UNIT_TEST(CommonLables) { + auto result = EncodeToString([](IMetricEncoder* e) { + e->OnStreamBegin(); + { // common time + e->OnCommonTime(TInstant::Seconds(1500000000)); + } + { // common labels + e->OnLabelsBegin(); + e->OnLabel("project", "solomon"); + e->OnLabelsEnd(); + } + { // metric #1 + e->OnMetricBegin(EMetricType::COUNTER); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "single"); + e->OnLabel("labels", "l1"); + e->OnLabelsEnd(); + } + e->OnUint64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:10Z"), 17); + e->OnMetricEnd(); + } + { // metric #2 + e->OnMetricBegin(EMetricType::COUNTER); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "two"); + e->OnLabel("labels", "l2"); + e->OnLabelsEnd(); + } + e->OnUint64(TInstant::Zero(), 42); + e->OnMetricEnd(); + } + e->OnStreamEnd(); + }); + UNIT_ASSERT_STRINGS_EQUAL(result, +R"(# TYPE single counter +single{labels="l1", project="solomon", } 17 1512216010000 +# TYPE two counter +two{labels="l2", project="solomon", } 42 1500000000000 + +)"); + } +} diff --git a/library/cpp/monlib/encode/prometheus/prometheus_model.h b/library/cpp/monlib/encode/prometheus/prometheus_model.h new file mode 100644 index 0000000000..cb7f2cb15b --- /dev/null +++ b/library/cpp/monlib/encode/prometheus/prometheus_model.h @@ -0,0 +1,70 @@ +#pragma once + +#include <util/generic/strbuf.h> + + +namespace NMonitoring { +namespace NPrometheus { + + // + // Prometheus specific names and validation rules. + // + // See https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md + // and https://github.com/prometheus/common/blob/master/expfmt/text_parse.go + // + + inline constexpr TStringBuf BUCKET_SUFFIX = "_bucket"; + inline constexpr TStringBuf COUNT_SUFFIX = "_count"; + inline constexpr TStringBuf SUM_SUFFIX = "_sum"; + inline constexpr TStringBuf MIN_SUFFIX = "_min"; + inline constexpr TStringBuf MAX_SUFFIX = "_max"; + inline constexpr TStringBuf LAST_SUFFIX = "_last"; + + // Used for the label that defines the upper bound of a bucket of a + // histogram ("le" -> "less or equal"). + inline constexpr TStringBuf BUCKET_LABEL = "le"; + + + inline bool IsValidLabelNameStart(char ch) { + return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || ch == '_'; + } + + inline bool IsValidLabelNameContinuation(char ch) { + return IsValidLabelNameStart(ch) || (ch >= '0' && ch <= '9'); + } + + inline bool IsValidMetricNameStart(char ch) { + return IsValidLabelNameStart(ch) || ch == ':'; + } + + inline bool IsValidMetricNameContinuation(char ch) { + return IsValidLabelNameContinuation(ch) || ch == ':'; + } + + inline bool IsSum(TStringBuf name) { + return name.EndsWith(SUM_SUFFIX); + } + + inline bool IsCount(TStringBuf name) { + return name.EndsWith(COUNT_SUFFIX); + } + + inline bool IsBucket(TStringBuf name) { + return name.EndsWith(BUCKET_SUFFIX); + } + + inline TStringBuf ToBaseName(TStringBuf name) { + if (IsBucket(name)) { + return name.SubString(0, name.length() - BUCKET_SUFFIX.length()); + } + if (IsCount(name)) { + return name.SubString(0, name.length() - COUNT_SUFFIX.length()); + } + if (IsSum(name)) { + return name.SubString(0, name.length() - SUM_SUFFIX.length()); + } + return name; + } + +} // namespace NPrometheus +} // namespace NMonitoring diff --git a/library/cpp/monlib/encode/prometheus/ut/ya.make b/library/cpp/monlib/encode/prometheus/ut/ya.make new file mode 100644 index 0000000000..fc468ffb68 --- /dev/null +++ b/library/cpp/monlib/encode/prometheus/ut/ya.make @@ -0,0 +1,17 @@ +UNITTEST_FOR(library/cpp/monlib/encode/prometheus) + +OWNER( + g:solomon + jamel +) + +SRCS( + prometheus_encoder_ut.cpp + prometheus_decoder_ut.cpp +) + +PEERDIR( + library/cpp/monlib/encode/protobuf +) + +END() diff --git a/library/cpp/monlib/encode/prometheus/ya.make b/library/cpp/monlib/encode/prometheus/ya.make new file mode 100644 index 0000000000..7f2483b166 --- /dev/null +++ b/library/cpp/monlib/encode/prometheus/ya.make @@ -0,0 +1,17 @@ +LIBRARY() + +OWNER( + jamel + g:solomon +) + +SRCS( + prometheus_decoder.cpp + prometheus_encoder.cpp +) + +PEERDIR( + library/cpp/monlib/encode +) + +END() diff --git a/library/cpp/monlib/encode/protobuf/protobuf.h b/library/cpp/monlib/encode/protobuf/protobuf.h new file mode 100644 index 0000000000..3f82cbdd84 --- /dev/null +++ b/library/cpp/monlib/encode/protobuf/protobuf.h @@ -0,0 +1,16 @@ +#pragma once + +#include <library/cpp/monlib/encode/encoder.h> + +#include <library/cpp/monlib/encode/protobuf/protos/samples.pb.h> + +namespace NMonitoring { + namespace NProto { + class TSingleSamplesList; + class TMultiSamplesList; + } + + IMetricEncoderPtr EncoderProtobuf(NProto::TSingleSamplesList* samples); + IMetricEncoderPtr EncoderProtobuf(NProto::TMultiSamplesList* samples); + +} diff --git a/library/cpp/monlib/encode/protobuf/protobuf_encoder.cpp b/library/cpp/monlib/encode/protobuf/protobuf_encoder.cpp new file mode 100644 index 0000000000..2d11b9d5ba --- /dev/null +++ b/library/cpp/monlib/encode/protobuf/protobuf_encoder.cpp @@ -0,0 +1,248 @@ +#include "protobuf.h" + +#include <util/datetime/base.h> + +namespace NMonitoring { + namespace { + NProto::EMetricType ConvertMetricType(EMetricType type) { + switch (type) { + case EMetricType::GAUGE: + return NProto::GAUGE; + case EMetricType::COUNTER: + return NProto::COUNTER; + case EMetricType::RATE: + return NProto::RATE; + case EMetricType::IGAUGE: + return NProto::IGAUGE; + case EMetricType::HIST: + return NProto::HISTOGRAM; + case EMetricType::HIST_RATE: + return NProto::HIST_RATE; + case EMetricType::DSUMMARY: + return NProto::DSUMMARY; + case EMetricType::LOGHIST: + return NProto::LOGHISTOGRAM; + case EMetricType::UNKNOWN: + return NProto::UNKNOWN; + } + } + + void FillHistogram( + const IHistogramSnapshot& snapshot, + NProto::THistogram* histogram) + { + for (ui32 i = 0; i < snapshot.Count(); i++) { + histogram->AddBounds(snapshot.UpperBound(i)); + histogram->AddValues(snapshot.Value(i)); + } + } + + void FillSummaryDouble(const ISummaryDoubleSnapshot& snapshot, NProto::TSummaryDouble* summary) { + summary->SetSum(snapshot.GetSum()); + summary->SetMin(snapshot.GetMin()); + summary->SetMax(snapshot.GetMax()); + summary->SetLast(snapshot.GetLast()); + summary->SetCount(snapshot.GetCount()); + } + + void FillLogHistogram(const TLogHistogramSnapshot& snapshot, NProto::TLogHistogram* logHist) { + logHist->SetBase(snapshot.Base()); + logHist->SetZerosCount(snapshot.ZerosCount()); + logHist->SetStartPower(snapshot.StartPower()); + for (ui32 i = 0; i < snapshot.Count(); ++i) { + logHist->AddBuckets(snapshot.Bucket(i)); + } + } + + /////////////////////////////////////////////////////////////////////////////// + // TSingleamplesEncoder + /////////////////////////////////////////////////////////////////////////////// + class TSingleSamplesEncoder final: public IMetricEncoder { + public: + TSingleSamplesEncoder(NProto::TSingleSamplesList* samples) + : Samples_(samples) + , Sample_(nullptr) + { + } + + private: + void OnStreamBegin() override { + } + void OnStreamEnd() override { + } + + void OnCommonTime(TInstant time) override { + Samples_->SetCommonTime(time.MilliSeconds()); + } + + void OnMetricBegin(EMetricType type) override { + Sample_ = Samples_->AddSamples(); + Sample_->SetMetricType(ConvertMetricType(type)); + } + + void OnMetricEnd() override { + Sample_ = nullptr; + } + + void OnLabelsBegin() override { + } + void OnLabelsEnd() override { + } + + void OnLabel(TStringBuf name, TStringBuf value) override { + NProto::TLabel* label = (Sample_ == nullptr) + ? Samples_->AddCommonLabels() + : Sample_->AddLabels(); + label->SetName(TString{name}); + label->SetValue(TString{value}); + } + + void OnDouble(TInstant time, double value) override { + Y_ENSURE(Sample_, "metric not started"); + Sample_->SetTime(time.MilliSeconds()); + Sample_->SetFloat64(value); + } + + void OnInt64(TInstant time, i64 value) override { + Y_ENSURE(Sample_, "metric not started"); + Sample_->SetTime(time.MilliSeconds()); + Sample_->SetInt64(value); + } + + void OnUint64(TInstant time, ui64 value) override { + Y_ENSURE(Sample_, "metric not started"); + Sample_->SetTime(time.MilliSeconds()); + Sample_->SetUint64(value); + } + + void OnHistogram(TInstant time, IHistogramSnapshotPtr snapshot) override { + Y_ENSURE(Sample_, "metric not started"); + Sample_->SetTime(time.MilliSeconds()); + FillHistogram(*snapshot, Sample_->MutableHistogram()); + } + + void OnSummaryDouble(TInstant time, ISummaryDoubleSnapshotPtr snapshot) override { + Y_ENSURE(Sample_, "metric not started"); + Sample_->SetTime(time.MilliSeconds()); + FillSummaryDouble(*snapshot, Sample_->MutableSummaryDouble()); + } + + void OnLogHistogram(TInstant time, TLogHistogramSnapshotPtr snapshot) override { + Y_ENSURE(Sample_, "metric not started"); + Sample_->SetTime(time.MilliSeconds()); + FillLogHistogram(*snapshot, Sample_->MutableLogHistogram()); + } + + void Close() override { + } + + private: + NProto::TSingleSamplesList* Samples_; + NProto::TSingleSample* Sample_; + }; + + /////////////////////////////////////////////////////////////////////////////// + // TMultiSamplesEncoder + /////////////////////////////////////////////////////////////////////////////// + class TMultiSamplesEncoder final: public IMetricEncoder { + public: + TMultiSamplesEncoder(NProto::TMultiSamplesList* samples) + : Samples_(samples) + , Sample_(nullptr) + { + } + + private: + void OnStreamBegin() override { + } + void OnStreamEnd() override { + } + + void OnCommonTime(TInstant time) override { + Samples_->SetCommonTime(time.MilliSeconds()); + } + + void OnMetricBegin(EMetricType type) override { + Sample_ = Samples_->AddSamples(); + Sample_->SetMetricType(ConvertMetricType(type)); + } + + void OnMetricEnd() override { + Sample_ = nullptr; + } + + void OnLabelsBegin() override { + } + void OnLabelsEnd() override { + } + + void OnLabel(TStringBuf name, TStringBuf value) override { + NProto::TLabel* label = (Sample_ == nullptr) + ? Samples_->AddCommonLabels() + : Sample_->AddLabels(); + + label->SetName(TString{name}); + label->SetValue(TString{value}); + } + + void OnDouble(TInstant time, double value) override { + Y_ENSURE(Sample_, "metric not started"); + NProto::TPoint* point = Sample_->AddPoints(); + point->SetTime(time.MilliSeconds()); + point->SetFloat64(value); + } + + void OnInt64(TInstant time, i64 value) override { + Y_ENSURE(Sample_, "metric not started"); + NProto::TPoint* point = Sample_->AddPoints(); + point->SetTime(time.MilliSeconds()); + point->SetInt64(value); + } + + void OnUint64(TInstant time, ui64 value) override { + Y_ENSURE(Sample_, "metric not started"); + NProto::TPoint* point = Sample_->AddPoints(); + point->SetTime(time.MilliSeconds()); + point->SetUint64(value); + } + + void OnHistogram(TInstant time, IHistogramSnapshotPtr snapshot) override { + Y_ENSURE(Sample_, "metric not started"); + NProto::TPoint* point = Sample_->AddPoints(); + point->SetTime(time.MilliSeconds()); + FillHistogram(*snapshot, point->MutableHistogram()); + } + + void OnSummaryDouble(TInstant time, ISummaryDoubleSnapshotPtr snapshot) override { + Y_ENSURE(Sample_, "metric not started"); + NProto::TPoint* point = Sample_->AddPoints(); + point->SetTime(time.MilliSeconds()); + FillSummaryDouble(*snapshot, point->MutableSummaryDouble()); + } + + void OnLogHistogram(TInstant time, TLogHistogramSnapshotPtr snapshot) override { + Y_ENSURE(Sample_, "metric not started"); + NProto::TPoint* point = Sample_->AddPoints(); + point->SetTime(time.MilliSeconds()); + FillLogHistogram(*snapshot, point->MutableLogHistogram()); + } + + void Close() override { + } + + private: + NProto::TMultiSamplesList* Samples_; + NProto::TMultiSample* Sample_; + }; + + } + + IMetricEncoderPtr EncoderProtobuf(NProto::TSingleSamplesList* samples) { + return MakeHolder<TSingleSamplesEncoder>(samples); + } + + IMetricEncoderPtr EncoderProtobuf(NProto::TMultiSamplesList* samples) { + return MakeHolder<TMultiSamplesEncoder>(samples); + } + +} diff --git a/library/cpp/monlib/encode/protobuf/protos/samples.proto b/library/cpp/monlib/encode/protobuf/protos/samples.proto new file mode 100644 index 0000000000..371f4181d2 --- /dev/null +++ b/library/cpp/monlib/encode/protobuf/protos/samples.proto @@ -0,0 +1,91 @@ +syntax = 'proto3'; + +package NMonitoring.NProto; + +option java_package = "ru.yandex.solomon.protos"; +option java_multiple_files = true; +option cc_enable_arenas = true; + +message TLabel { + string Name = 1; + string Value = 2; +} + +enum EMetricType { + UNKNOWN = 0; + GAUGE = 1; + IGAUGE = 2; + COUNTER = 3; + RATE = 4; + HISTOGRAM = 5; + HIST_RATE = 6; + DSUMMARY = 7; + LOGHISTOGRAM = 8; +} + +message THistogram { + repeated double Bounds = 1; // upper bounds of each bucket + repeated uint64 Values = 2; // values stored in each bucket +} + +message TLogHistogram { + double Base = 1; + uint64 ZerosCount = 2; + int32 StartPower = 3; + repeated double Buckets = 4; +} + +message TSummaryDouble { + double Sum = 1; + double Min = 2; + double Max = 3; + double Last = 4; + uint64 Count = 5; +} + +// see TSingleSample +message TPoint { + uint64 Time = 1; + oneof Value { + sfixed64 Int64 = 2; + fixed64 Uint64 = 3; + double Float64 = 4; + THistogram Histogram = 5; + TSummaryDouble SummaryDouble = 6; + TLogHistogram LogHistogram = 7; + } +} + +message TSingleSample { + repeated TLabel Labels = 1; + EMetricType MetricType = 2; + + // inlined TPoint + uint64 Time = 3; + oneof Value { + sfixed64 Int64 = 4; + fixed64 Uint64 = 5; + double Float64 = 6; + THistogram Histogram = 7; + TSummaryDouble SummaryDouble = 8; + TLogHistogram LogHistogram = 9; + } +} + +message TMultiSample { + repeated TLabel Labels = 1; + EMetricType MetricType = 2; + repeated TPoint Points = 3; +} + +message TSingleSamplesList { + uint64 CommonTime = 1; + repeated TLabel CommonLabels = 2; + repeated TSingleSample Samples = 3; +} + +message TMultiSamplesList { + uint64 CommonTime = 1; + repeated TLabel CommonLabels = 2; + repeated TMultiSample Samples = 3; +} diff --git a/library/cpp/monlib/encode/protobuf/protos/ya.make b/library/cpp/monlib/encode/protobuf/protos/ya.make new file mode 100644 index 0000000000..88ff3ddf88 --- /dev/null +++ b/library/cpp/monlib/encode/protobuf/protos/ya.make @@ -0,0 +1,14 @@ +PROTO_LIBRARY() + +OWNER( + jamel + g:solomon +) + +SRCS( + samples.proto +) + +EXCLUDE_TAGS(GO_PROTO) + +END() diff --git a/library/cpp/monlib/encode/protobuf/ya.make b/library/cpp/monlib/encode/protobuf/ya.make new file mode 100644 index 0000000000..9354958b6f --- /dev/null +++ b/library/cpp/monlib/encode/protobuf/ya.make @@ -0,0 +1,17 @@ +LIBRARY() + +OWNER( + jamel + g:solomon +) + +SRCS( + protobuf_encoder.cpp +) + +PEERDIR( + library/cpp/monlib/encode + library/cpp/monlib/encode/protobuf/protos +) + +END() diff --git a/library/cpp/monlib/encode/spack/compression.cpp b/library/cpp/monlib/encode/spack/compression.cpp new file mode 100644 index 0000000000..0d2152fc85 --- /dev/null +++ b/library/cpp/monlib/encode/spack/compression.cpp @@ -0,0 +1,383 @@ +#include "compression.h" + +#include <util/generic/buffer.h> +#include <util/generic/cast.h> +#include <util/generic/ptr.h> +#include <util/generic/scope.h> +#include <util/generic/size_literals.h> +#include <util/stream/format.h> +#include <util/stream/output.h> +#include <util/stream/walk.h> + +#include <contrib/libs/lz4/lz4.h> +#include <contrib/libs/xxhash/xxhash.h> +#include <contrib/libs/zlib/zlib.h> +#define ZSTD_STATIC_LINKING_ONLY +#include <contrib/libs/zstd/include/zstd.h> + +namespace NMonitoring { + namespace { + /////////////////////////////////////////////////////////////////////////////// + // Frame + /////////////////////////////////////////////////////////////////////////////// + using TCompressedSize = ui32; + using TUncompressedSize = ui32; + using TCheckSum = ui32; + + constexpr size_t COMPRESSED_FRAME_SIZE_LIMIT = 512_KB; + constexpr size_t UNCOMPRESSED_FRAME_SIZE_LIMIT = COMPRESSED_FRAME_SIZE_LIMIT; + constexpr size_t FRAME_SIZE_LIMIT = 2_MB; + constexpr size_t DEFAULT_FRAME_LEN = 64_KB; + + struct Y_PACKED TFrameHeader { + TCompressedSize CompressedSize; + TUncompressedSize UncompressedSize; + }; + + struct Y_PACKED TFrameFooter { + TCheckSum CheckSum; + }; + + /////////////////////////////////////////////////////////////////////////////// + // TBlock + /////////////////////////////////////////////////////////////////////////////// + struct TBlock: public TStringBuf { + template <typename T> + TBlock(T&& t) + : TStringBuf(t.data(), t.size()) + { + Y_ENSURE(t.data() != nullptr); + } + + char* data() noexcept { + return const_cast<char*>(TStringBuf::data()); + } + }; + + /////////////////////////////////////////////////////////////////////////////// + // XXHASH + /////////////////////////////////////////////////////////////////////////////// + struct TXxHash32 { + static TCheckSum Calc(TBlock in) { + static const ui32 SEED = 0x1337c0de; + return XXH32(in.data(), in.size(), SEED); + } + + static bool Check(TBlock in, TCheckSum checksum) { + return Calc(in) == checksum; + } + }; + + /////////////////////////////////////////////////////////////////////////////// + // Adler32 + /////////////////////////////////////////////////////////////////////////////// + struct TAdler32 { + static TCheckSum Calc(TBlock in) { + return adler32(1L, reinterpret_cast<const Bytef*>(in.data()), in.size()); + } + + static bool Check(TBlock in, TCheckSum checksum) { + return Calc(in) == checksum; + } + }; + + /////////////////////////////////////////////////////////////////////////////// + // LZ4 + /////////////////////////////////////////////////////////////////////////////// + struct TLz4Codec { + static size_t MaxCompressedLength(size_t in) { + int result = LZ4_compressBound(static_cast<int>(in)); + Y_ENSURE(result != 0, "lz4 input size is too large"); + return result; + } + + static size_t Compress(TBlock in, TBlock out) { + int rc = LZ4_compress_default( + in.data(), + out.data(), + SafeIntegerCast<int>(in.size()), + SafeIntegerCast<int>(out.size())); + Y_ENSURE(rc != 0, "lz4 compression failed"); + return rc; + } + + static void Decompress(TBlock in, TBlock out) { + int rc = LZ4_decompress_safe( + in.data(), + out.data(), + SafeIntegerCast<int>(in.size()), + SafeIntegerCast<int>(out.size())); + Y_ENSURE(rc >= 0, "the lz4 stream is detected malformed"); + } + }; + + /////////////////////////////////////////////////////////////////////////////// + // ZSTD + /////////////////////////////////////////////////////////////////////////////// + struct TZstdCodec { + static const int LEVEL = 11; + + static size_t MaxCompressedLength(size_t in) { + return ZSTD_compressBound(in); + } + + static size_t Compress(TBlock in, TBlock out) { + size_t rc = ZSTD_compress(out.data(), out.size(), in.data(), in.size(), LEVEL); + if (Y_UNLIKELY(ZSTD_isError(rc))) { + ythrow yexception() << TStringBuf("zstd compression failed: ") + << ZSTD_getErrorName(rc); + } + return rc; + } + + static void Decompress(TBlock in, TBlock out) { + size_t rc = ZSTD_decompress(out.data(), out.size(), in.data(), in.size()); + if (Y_UNLIKELY(ZSTD_isError(rc))) { + ythrow yexception() << TStringBuf("zstd decompression failed: ") + << ZSTD_getErrorName(rc); + } + Y_ENSURE(rc == out.size(), "zstd decompressed wrong size"); + } + }; + + /////////////////////////////////////////////////////////////////////////////// + // ZLIB + /////////////////////////////////////////////////////////////////////////////// + struct TZlibCodec { + static const int LEVEL = 6; + + static size_t MaxCompressedLength(size_t in) { + return compressBound(in); + } + + static size_t Compress(TBlock in, TBlock out) { + uLong ret = out.size(); + int rc = compress2( + reinterpret_cast<Bytef*>(out.data()), + &ret, + reinterpret_cast<const Bytef*>(in.data()), + in.size(), + LEVEL); + Y_ENSURE(rc == Z_OK, "zlib compression failed"); + return ret; + } + + static void Decompress(TBlock in, TBlock out) { + uLong ret = out.size(); + int rc = uncompress( + reinterpret_cast<Bytef*>(out.data()), + &ret, + reinterpret_cast<const Bytef*>(in.data()), + in.size()); + Y_ENSURE(rc == Z_OK, "zlib decompression failed"); + Y_ENSURE(ret == out.size(), "zlib decompressed wrong size"); + } + }; + + // + // Framed streams use next frame structure: + // + // +-----------------+-------------------+============+------------------+ + // | compressed size | uncompressed size | data | check sum | + // +-----------------+-------------------+============+------------------+ + // 4 bytes 4 bytes var len 4 bytes + // + + /////////////////////////////////////////////////////////////////////////////// + // TFramedInputStream + /////////////////////////////////////////////////////////////////////////////// + template <typename TCodecAlg, typename TCheckSumAlg> + class TFramedDecompressStream final: public IWalkInput { + public: + explicit TFramedDecompressStream(IInputStream* in) + : In_(in) + { + } + + private: + size_t DoUnboundedNext(const void** ptr) override { + if (!In_) { + return 0; + } + + TFrameHeader header; + In_->LoadOrFail(&header, sizeof(header)); + + if (header.CompressedSize == 0) { + In_ = nullptr; + return 0; + } + + Y_ENSURE(header.CompressedSize <= COMPRESSED_FRAME_SIZE_LIMIT, "Compressed frame size is limited to " + << HumanReadableSize(COMPRESSED_FRAME_SIZE_LIMIT, SF_BYTES) + << " but is " << HumanReadableSize(header.CompressedSize, SF_BYTES)); + + Y_ENSURE(header.UncompressedSize <= UNCOMPRESSED_FRAME_SIZE_LIMIT, "Uncompressed frame size is limited to " + << HumanReadableSize(UNCOMPRESSED_FRAME_SIZE_LIMIT, SF_BYTES) + << " but is " << HumanReadableSize(header.UncompressedSize, SF_BYTES)); + + Compressed_.Resize(header.CompressedSize); + In_->LoadOrFail(Compressed_.Data(), header.CompressedSize); + + TFrameFooter footer; + In_->LoadOrFail(&footer, sizeof(footer)); + Y_ENSURE(TCheckSumAlg::Check(Compressed_, footer.CheckSum), + "corrupted stream: check sum mismatch"); + + Uncompressed_.Resize(header.UncompressedSize); + TCodecAlg::Decompress(Compressed_, Uncompressed_); + + *ptr = Uncompressed_.Data(); + return Uncompressed_.Size(); + } + + private: + IInputStream* In_; + TBuffer Compressed_; + TBuffer Uncompressed_; + }; + + /////////////////////////////////////////////////////////////////////////////// + // TFramedOutputStream + /////////////////////////////////////////////////////////////////////////////// + template <typename TCodecAlg, typename TCheckSumAlg> + class TFramedCompressStream final: public IFramedCompressStream { + public: + explicit TFramedCompressStream(IOutputStream* out) + : Out_(out) + , Uncompressed_(DEFAULT_FRAME_LEN) + { + } + + ~TFramedCompressStream() override { + try { + Finish(); + } catch (...) { + } + } + + private: + void DoWrite(const void* buf, size_t len) override { + const char* in = static_cast<const char*>(buf); + + while (len != 0) { + const size_t avail = Uncompressed_.Avail(); + if (len < avail) { + Uncompressed_.Append(in, len); + return; + } + + Uncompressed_.Append(in, avail); + Y_ASSERT(Uncompressed_.Avail() == 0); + + in += avail; + len -= avail; + + WriteCompressedFrame(); + } + } + + void FlushWithoutEmptyFrame() override { + if (Out_ && !Uncompressed_.Empty()) { + WriteCompressedFrame(); + } + } + + void FinishAndWriteEmptyFrame() override { + if (Out_) { + Y_DEFER { + Out_ = nullptr; + }; + + if (!Uncompressed_.Empty()) { + WriteCompressedFrame(); + } + + WriteEmptyFrame(); + } + } + + void DoFlush() override { + FlushWithoutEmptyFrame(); + } + + void DoFinish() override { + FinishAndWriteEmptyFrame(); + } + + void WriteCompressedFrame() { + static const auto framePayload = sizeof(TFrameHeader) + sizeof(TFrameFooter); + const auto maxFrameSize = ui64(TCodecAlg::MaxCompressedLength(Uncompressed_.Size())) + framePayload; + Y_ENSURE(maxFrameSize <= FRAME_SIZE_LIMIT, "Frame size in encoder is limited to " + << HumanReadableSize(FRAME_SIZE_LIMIT, SF_BYTES) + << " but is " << HumanReadableSize(maxFrameSize, SF_BYTES)); + + Frame_.Resize(maxFrameSize); + + // compress + TBlock compressedBlock = Frame_; + compressedBlock.Skip(sizeof(TFrameHeader)); + compressedBlock.Trunc(TCodecAlg::Compress(Uncompressed_, compressedBlock)); + + // add header + auto header = reinterpret_cast<TFrameHeader*>(Frame_.Data()); + header->CompressedSize = SafeIntegerCast<TCompressedSize>(compressedBlock.size()); + header->UncompressedSize = SafeIntegerCast<TUncompressedSize>(Uncompressed_.Size()); + + // add footer + auto footer = reinterpret_cast<TFrameFooter*>( + Frame_.Data() + sizeof(TFrameHeader) + header->CompressedSize); + footer->CheckSum = TCheckSumAlg::Calc(compressedBlock); + + // write + Out_->Write(Frame_.Data(), header->CompressedSize + framePayload); + Uncompressed_.Clear(); + } + + void WriteEmptyFrame() { + static const auto framePayload = sizeof(TFrameHeader) + sizeof(TFrameFooter); + char buf[framePayload] = {0}; + Out_->Write(buf, sizeof(buf)); + } + + private: + IOutputStream* Out_; + TBuffer Uncompressed_; + TBuffer Frame_; + }; + + } + + THolder<IInputStream> CompressedInput(IInputStream* in, ECompression alg) { + switch (alg) { + case ECompression::IDENTITY: + return nullptr; + case ECompression::ZLIB: + return MakeHolder<TFramedDecompressStream<TZlibCodec, TAdler32>>(in); + case ECompression::ZSTD: + return MakeHolder<TFramedDecompressStream<TZstdCodec, TXxHash32>>(in); + case ECompression::LZ4: + return MakeHolder<TFramedDecompressStream<TLz4Codec, TXxHash32>>(in); + case ECompression::UNKNOWN: + return nullptr; + } + Y_FAIL("invalid compression algorithm"); + } + + THolder<IFramedCompressStream> CompressedOutput(IOutputStream* out, ECompression alg) { + switch (alg) { + case ECompression::IDENTITY: + return nullptr; + case ECompression::ZLIB: + return MakeHolder<TFramedCompressStream<TZlibCodec, TAdler32>>(out); + case ECompression::ZSTD: + return MakeHolder<TFramedCompressStream<TZstdCodec, TXxHash32>>(out); + case ECompression::LZ4: + return MakeHolder<TFramedCompressStream<TLz4Codec, TXxHash32>>(out); + case ECompression::UNKNOWN: + return nullptr; + } + Y_FAIL("invalid compression algorithm"); + } + +} diff --git a/library/cpp/monlib/encode/spack/compression.h b/library/cpp/monlib/encode/spack/compression.h new file mode 100644 index 0000000000..f74d8b424e --- /dev/null +++ b/library/cpp/monlib/encode/spack/compression.h @@ -0,0 +1,19 @@ +#pragma once + +#include "spack_v1.h" + +#include <util/stream/input.h> +#include <util/stream/output.h> + +namespace NMonitoring { + +class IFramedCompressStream: public IOutputStream { +public: + virtual void FlushWithoutEmptyFrame() = 0; + virtual void FinishAndWriteEmptyFrame() = 0; +}; + +THolder<IInputStream> CompressedInput(IInputStream* in, ECompression alg); +THolder<IFramedCompressStream> CompressedOutput(IOutputStream* out, ECompression alg); + +} // namespace NMonitoring diff --git a/library/cpp/monlib/encode/spack/fuzz/main.cpp b/library/cpp/monlib/encode/spack/fuzz/main.cpp new file mode 100644 index 0000000000..6a14afe71c --- /dev/null +++ b/library/cpp/monlib/encode/spack/fuzz/main.cpp @@ -0,0 +1,20 @@ +#include <library/cpp/monlib/encode/spack/spack_v1.h> +#include <library/cpp/monlib/encode/fake/fake.h> + +#include <util/stream/mem.h> + + +extern "C" int LLVMFuzzerTestOneInput(const ui8* data, size_t size) { + using namespace NMonitoring; + + TMemoryInput min{data, size}; + + auto encoder = EncoderFake(); + + try { + DecodeSpackV1(&min, encoder.Get()); + } catch (...) { + } + + return 0; +} diff --git a/library/cpp/monlib/encode/spack/fuzz/ya.make b/library/cpp/monlib/encode/spack/fuzz/ya.make new file mode 100644 index 0000000000..99b63eadd5 --- /dev/null +++ b/library/cpp/monlib/encode/spack/fuzz/ya.make @@ -0,0 +1,21 @@ +FUZZ() + +OWNER( + g:solomon + msherbakov +) + +FUZZ_OPTS(-rss_limit_mb=1024) + +SIZE(MEDIUM) + +PEERDIR( + library/cpp/monlib/encode/spack + library/cpp/monlib/encode/fake +) + +SRCS( + main.cpp +) + +END() diff --git a/library/cpp/monlib/encode/spack/spack_v1.h b/library/cpp/monlib/encode/spack/spack_v1.h new file mode 100644 index 0000000000..cf1c9417b9 --- /dev/null +++ b/library/cpp/monlib/encode/spack/spack_v1.h @@ -0,0 +1,115 @@ +#pragma once + +#include <library/cpp/monlib/encode/encoder.h> +#include <library/cpp/monlib/encode/format.h> +#include <library/cpp/monlib/metrics/metric.h> + +#include <util/generic/yexception.h> + +// +// format specification available here: +// https://wiki.yandex-team.ru/solomon/api/dataformat/spackv1/ +// + +class IInputStream; +class IOutputStream; + +namespace NMonitoring { + class TSpackDecodeError: public yexception { + }; + + constexpr auto EncodeMetricType(EMetricType mt) noexcept { + return static_cast<std::underlying_type_t<EMetricType>>(mt); + } + + EMetricType DecodeMetricType(ui8 byte); + + [[nodiscard]] + bool TryDecodeMetricType(ui8 byte, EMetricType* result); + + /////////////////////////////////////////////////////////////////////////////// + // EValueType + /////////////////////////////////////////////////////////////////////////////// + enum class EValueType : ui8 { + NONE = 0x00, + ONE_WITHOUT_TS = 0x01, + ONE_WITH_TS = 0x02, + MANY_WITH_TS = 0x03, + }; + + constexpr auto EncodeValueType(EValueType vt) noexcept { + return static_cast<std::underlying_type_t<EValueType>>(vt); + } + + EValueType DecodeValueType(ui8 byte); + + [[nodiscard]] + bool TryDecodeValueType(ui8 byte, EValueType* result); + + /////////////////////////////////////////////////////////////////////////////// + // ETimePrecision + /////////////////////////////////////////////////////////////////////////////// + enum class ETimePrecision : ui8 { + SECONDS = 0x00, + MILLIS = 0x01, + }; + + constexpr auto EncodeTimePrecision(ETimePrecision tp) noexcept { + return static_cast<std::underlying_type_t<ETimePrecision>>(tp); + } + + ETimePrecision DecodeTimePrecision(ui8 byte); + + [[nodiscard]] + bool TryDecodeTimePrecision(ui8 byte, ETimePrecision* result); + + /////////////////////////////////////////////////////////////////////////////// + // ECompression + /////////////////////////////////////////////////////////////////////////////// + ui8 EncodeCompression(ECompression c) noexcept; + + ECompression DecodeCompression(ui8 byte); + + [[nodiscard]] + bool TryDecodeCompression(ui8 byte, ECompression* result); + + /////////////////////////////////////////////////////////////////////////////// + // TSpackHeader + /////////////////////////////////////////////////////////////////////////////// + struct Y_PACKED TSpackHeader { + ui16 Magic = 0x5053; // "SP" + ui16 Version; // MSB - major version, LSB - minor version + ui16 HeaderSize = sizeof(TSpackHeader); + ui8 TimePrecision; + ui8 Compression; + ui32 LabelNamesSize; + ui32 LabelValuesSize; + ui32 MetricCount; + ui32 PointsCount; + // add new fields here + }; + + enum ESpackV1Version: ui16 { + SV1_00 = 0x0100, + SV1_01 = 0x0101, + SV1_02 = 0x0102 + }; + + IMetricEncoderPtr EncoderSpackV1( + IOutputStream* out, + ETimePrecision timePrecision, + ECompression compression, + EMetricsMergingMode mergingMode = EMetricsMergingMode::DEFAULT + ); + + IMetricEncoderPtr EncoderSpackV12( + IOutputStream* out, + ETimePrecision timePrecision, + ECompression compression, + EMetricsMergingMode mergingMode = EMetricsMergingMode::DEFAULT, + TStringBuf metricNameLabel = "name" + ); + + void DecodeSpackV1(IInputStream* in, IMetricConsumer* c, TStringBuf metricNameLabel = "name"); + +} diff --git a/library/cpp/monlib/encode/spack/spack_v1_decoder.cpp b/library/cpp/monlib/encode/spack/spack_v1_decoder.cpp new file mode 100644 index 0000000000..1f445fc80d --- /dev/null +++ b/library/cpp/monlib/encode/spack/spack_v1_decoder.cpp @@ -0,0 +1,458 @@ +#include "spack_v1.h" +#include "varint.h" +#include "compression.h" + +#include <library/cpp/monlib/encode/buffered/string_pool.h> +#include <library/cpp/monlib/exception/exception.h> +#include <library/cpp/monlib/metrics/histogram_collector.h> +#include <library/cpp/monlib/metrics/metric.h> + +#include <util/generic/yexception.h> +#include <util/generic/buffer.h> +#include <util/generic/size_literals.h> +#include <util/stream/format.h> + +#ifndef _little_endian_ +#error Unsupported platform +#endif + +namespace NMonitoring { + namespace { +#define DECODE_ENSURE(COND, ...) MONLIB_ENSURE_EX(COND, TSpackDecodeError() << __VA_ARGS__) + + constexpr ui64 LABEL_SIZE_LIMIT = 128_MB; + + /////////////////////////////////////////////////////////////////////// + // TDecoderSpackV1 + /////////////////////////////////////////////////////////////////////// + class TDecoderSpackV1 { + public: + TDecoderSpackV1(IInputStream* in, TStringBuf metricNameLabel) + : In_(in) + , MetricNameLabel_(metricNameLabel) + { + } + + void Decode(IMetricConsumer* c) { + c->OnStreamBegin(); + + // (1) read header + size_t readBytes = In_->Read(&Header_, sizeof(Header_)); + DECODE_ENSURE(readBytes == sizeof(Header_), "not enough data in input stream to read header"); + + ui8 version = ((Header_.Version >> 8) & 0xff); + DECODE_ENSURE(version == 1, "versions mismatch (expected: 1, got: " << +version << ')'); + + DECODE_ENSURE(Header_.HeaderSize >= sizeof(Header_), "invalid header size"); + if (size_t skipBytes = Header_.HeaderSize - sizeof(Header_)) { + DECODE_ENSURE(In_->Skip(skipBytes) == skipBytes, "input stream unexpectedly ended"); + } + + if (Header_.MetricCount == 0) { + // emulate empty stream + c->OnStreamEnd(); + return; + } + + // if compression enabled all below reads must go throught decompressor + auto compressedIn = CompressedInput(In_, DecodeCompression(Header_.Compression)); + if (compressedIn) { + In_ = compressedIn.Get(); + } + + TimePrecision_ = DecodeTimePrecision(Header_.TimePrecision); + + const ui64 labelSizeTotal = ui64(Header_.LabelNamesSize) + Header_.LabelValuesSize; + + DECODE_ENSURE(labelSizeTotal <= LABEL_SIZE_LIMIT, "Label names & values size of " << HumanReadableSize(labelSizeTotal, SF_BYTES) + << " exceeds the limit which is " << HumanReadableSize(LABEL_SIZE_LIMIT, SF_BYTES)); + + // (2) read string pools + TVector<char> namesBuf(Header_.LabelNamesSize); + readBytes = In_->Load(namesBuf.data(), namesBuf.size()); + DECODE_ENSURE(readBytes == Header_.LabelNamesSize, "not enough data to read label names pool"); + TStringPool labelNames(namesBuf.data(), namesBuf.size()); + + TVector<char> valuesBuf(Header_.LabelValuesSize); + readBytes = In_->Load(valuesBuf.data(), valuesBuf.size()); + DECODE_ENSURE(readBytes == Header_.LabelValuesSize, "not enough data to read label values pool"); + TStringPool labelValues(valuesBuf.data(), valuesBuf.size()); + + // (3) read common time + c->OnCommonTime(ReadTime()); + + // (4) read common labels + if (ui32 commonLabelsCount = ReadVarint()) { + c->OnLabelsBegin(); + ReadLabels(labelNames, labelValues, commonLabelsCount, c); + c->OnLabelsEnd(); + } + + // (5) read metrics + ReadMetrics(labelNames, labelValues, c); + c->OnStreamEnd(); + } + + private: + void ReadMetrics( + const TStringPool& labelNames, + const TStringPool& labelValues, + IMetricConsumer* c) + { + for (ui32 i = 0; i < Header_.MetricCount; i++) { + // (5.1) types byte + ui8 typesByte = ReadFixed<ui8>(); + EMetricType metricType = DecodeMetricType(typesByte >> 2); + EValueType valueType = DecodeValueType(typesByte & 0x03); + + c->OnMetricBegin(metricType); + + // TODO: use it + ReadFixed<ui8>(); // skip flags byte + + auto metricNameValueIndex = std::numeric_limits<ui32>::max(); + if (Header_.Version >= SV1_02) { + metricNameValueIndex = ReadVarint(); + } + + // (5.2) labels + ui32 labelsCount = ReadVarint(); + DECODE_ENSURE(Header_.Version >= SV1_02 || labelsCount > 0, "metric #" << i << " has no labels"); + c->OnLabelsBegin(); + if (Header_.Version >= SV1_02) { + c->OnLabel(MetricNameLabel_, labelValues.Get(metricNameValueIndex)); + } + ReadLabels(labelNames, labelValues, labelsCount, c); + c->OnLabelsEnd(); + + // (5.3) values + switch (valueType) { + case EValueType::NONE: + break; + case EValueType::ONE_WITHOUT_TS: + ReadValue(metricType, TInstant::Zero(), c); + break; + case EValueType::ONE_WITH_TS: { + TInstant time = ReadTime(); + ReadValue(metricType, time, c); + break; + } + case EValueType::MANY_WITH_TS: { + ui32 pointsCount = ReadVarint(); + for (ui32 i = 0; i < pointsCount; i++) { + TInstant time = ReadTime(); + ReadValue(metricType, time, c); + } + break; + } + } + + c->OnMetricEnd(); + } + } + + void ReadValue(EMetricType metricType, TInstant time, IMetricConsumer* c) { + switch (metricType) { + case EMetricType::GAUGE: + c->OnDouble(time, ReadFixed<double>()); + break; + + case EMetricType::IGAUGE: + c->OnInt64(time, ReadFixed<i64>()); + break; + + case EMetricType::COUNTER: + case EMetricType::RATE: + c->OnUint64(time, ReadFixed<ui64>()); + break; + + case EMetricType::DSUMMARY: + c->OnSummaryDouble(time, ReadSummaryDouble()); + break; + + case EMetricType::HIST: + case EMetricType::HIST_RATE: + c->OnHistogram(time, ReadHistogram()); + break; + + case EMetricType::LOGHIST: + c->OnLogHistogram(time, ReadLogHistogram()); + break; + + default: + throw TSpackDecodeError() << "Unsupported metric type: " << metricType; + } + } + + ISummaryDoubleSnapshotPtr ReadSummaryDouble() { + ui64 count = ReadFixed<ui64>(); + double sum = ReadFixed<double>(); + double min = ReadFixed<double>(); + double max = ReadFixed<double>(); + double last = ReadFixed<double>(); + return MakeIntrusive<TSummaryDoubleSnapshot>(sum, min, max, last, count); + } + + TLogHistogramSnapshotPtr ReadLogHistogram() { + double base = ReadFixed<double>(); + ui64 zerosCount = ReadFixed<ui64>(); + int startPower = static_cast<int>(ReadVarint()); + ui32 count = ReadVarint(); + // see https://a.yandex-team.ru/arc/trunk/arcadia/infra/yasm/stockpile_client/points.cpp?rev=r8593154#L31 + // and https://a.yandex-team.ru/arc/trunk/arcadia/infra/yasm/common/points/hgram/normal/normal.h?rev=r8268697#L9 + // TODO: share this constant value + Y_ENSURE(count <= 100u, "more than 100 buckets in log histogram: " << count); + TVector<double> buckets; + buckets.reserve(count); + for (ui32 i = 0; i < count; ++i) { + buckets.emplace_back(ReadFixed<double>()); + } + return MakeIntrusive<TLogHistogramSnapshot>(base, zerosCount, startPower, std::move(buckets)); + } + + IHistogramSnapshotPtr ReadHistogram() { + ui32 bucketsCount = ReadVarint(); + auto s = TExplicitHistogramSnapshot::New(bucketsCount); + + if (SV1_00 == Header_.Version) { // v1.0 + for (ui32 i = 0; i < bucketsCount; i++) { + i64 bound = ReadFixed<i64>(); + double doubleBound = (bound != Max<i64>()) + ? static_cast<double>(bound) + : Max<double>(); + + (*s)[i].first = doubleBound; + } + } else { + for (ui32 i = 0; i < bucketsCount; i++) { + double doubleBound = ReadFixed<double>(); + (*s)[i].first = doubleBound; + } + } + + + // values + for (ui32 i = 0; i < bucketsCount; i++) { + (*s)[i].second = ReadFixed<ui64>(); + } + return s; + } + + void ReadLabels( + const TStringPool& labelNames, + const TStringPool& labelValues, + ui32 count, + IMetricConsumer* c) + { + for (ui32 i = 0; i < count; i++) { + auto nameIdx = ReadVarint(); + auto valueIdx = ReadVarint(); + c->OnLabel(labelNames.Get(nameIdx), labelValues.Get(valueIdx)); + } + } + + TInstant ReadTime() { + switch (TimePrecision_) { + case ETimePrecision::SECONDS: + return TInstant::Seconds(ReadFixed<ui32>()); + case ETimePrecision::MILLIS: + return TInstant::MilliSeconds(ReadFixed<ui64>()); + } + Y_FAIL("invalid time precision"); + } + + template <typename T> + inline T ReadFixed() { + T value; + size_t readBytes = In_->Load(&value, sizeof(T)); + DECODE_ENSURE(readBytes == sizeof(T), "no enough data to read " << TypeName<T>()); + return value; + } + + inline ui32 ReadVarint() { + return ReadVarUInt32(In_); + } + + private: + IInputStream* In_; + TString MetricNameLabel_; + ETimePrecision TimePrecision_; + TSpackHeader Header_; + }; // class TDecoderSpackV1 + +#undef DECODE_ENSURE + } // namespace + + EValueType DecodeValueType(ui8 byte) { + EValueType result; + if (!TryDecodeValueType(byte, &result)) { + throw TSpackDecodeError() << "unknown value type: " << byte; + } + return result; + } + + bool TryDecodeValueType(ui8 byte, EValueType* result) { + if (byte == EncodeValueType(EValueType::NONE)) { + if (result) { + *result = EValueType::NONE; + } + return true; + } else if (byte == EncodeValueType(EValueType::ONE_WITHOUT_TS)) { + if (result) { + *result = EValueType::ONE_WITHOUT_TS; + } + return true; + } else if (byte == EncodeValueType(EValueType::ONE_WITH_TS)) { + if (result) { + *result = EValueType::ONE_WITH_TS; + } + return true; + } else if (byte == EncodeValueType(EValueType::MANY_WITH_TS)) { + if (result) { + *result = EValueType::MANY_WITH_TS; + } + return true; + } else { + return false; + } + } + + ETimePrecision DecodeTimePrecision(ui8 byte) { + ETimePrecision result; + if (!TryDecodeTimePrecision(byte, &result)) { + throw TSpackDecodeError() << "unknown time precision: " << byte; + } + return result; + } + + bool TryDecodeTimePrecision(ui8 byte, ETimePrecision* result) { + if (byte == EncodeTimePrecision(ETimePrecision::SECONDS)) { + if (result) { + *result = ETimePrecision::SECONDS; + } + return true; + } else if (byte == EncodeTimePrecision(ETimePrecision::MILLIS)) { + if (result) { + *result = ETimePrecision::MILLIS; + } + return true; + } else { + return false; + } + } + + EMetricType DecodeMetricType(ui8 byte) { + EMetricType result; + if (!TryDecodeMetricType(byte, &result)) { + throw TSpackDecodeError() << "unknown metric type: " << byte; + } + return result; + } + + bool TryDecodeMetricType(ui8 byte, EMetricType* result) { + if (byte == EncodeMetricType(EMetricType::GAUGE)) { + if (result) { + *result = EMetricType::GAUGE; + } + return true; + } else if (byte == EncodeMetricType(EMetricType::COUNTER)) { + if (result) { + *result = EMetricType::COUNTER; + } + return true; + } else if (byte == EncodeMetricType(EMetricType::RATE)) { + if (result) { + *result = EMetricType::RATE; + } + return true; + } else if (byte == EncodeMetricType(EMetricType::IGAUGE)) { + if (result) { + *result = EMetricType::IGAUGE; + } + return true; + } else if (byte == EncodeMetricType(EMetricType::HIST)) { + if (result) { + *result = EMetricType::HIST; + } + return true; + } else if (byte == EncodeMetricType(EMetricType::HIST_RATE)) { + if (result) { + *result = EMetricType::HIST_RATE; + } + return true; + } else if (byte == EncodeMetricType(EMetricType::DSUMMARY)) { + if (result) { + *result = EMetricType::DSUMMARY; + } + return true; + } else if (byte == EncodeMetricType(EMetricType::LOGHIST)) { + if (result) { + *result = EMetricType::LOGHIST; + } + return true; + } else if (byte == EncodeMetricType(EMetricType::UNKNOWN)) { + if (result) { + *result = EMetricType::UNKNOWN; + } + return true; + } else { + return false; + } + } + + ui8 EncodeCompression(ECompression c) noexcept { + switch (c) { + case ECompression::IDENTITY: + return 0x00; + case ECompression::ZLIB: + return 0x01; + case ECompression::ZSTD: + return 0x02; + case ECompression::LZ4: + return 0x03; + case ECompression::UNKNOWN: + return Max<ui8>(); + } + Y_FAIL(); // for GCC + } + + ECompression DecodeCompression(ui8 byte) { + ECompression result; + if (!TryDecodeCompression(byte, &result)) { + throw TSpackDecodeError() << "unknown compression alg: " << byte; + } + return result; + } + + bool TryDecodeCompression(ui8 byte, ECompression* result) { + if (byte == EncodeCompression(ECompression::IDENTITY)) { + if (result) { + *result = ECompression::IDENTITY; + } + return true; + } else if (byte == EncodeCompression(ECompression::ZLIB)) { + if (result) { + *result = ECompression::ZLIB; + } + return true; + } else if (byte == EncodeCompression(ECompression::ZSTD)) { + if (result) { + *result = ECompression::ZSTD; + } + return true; + } else if (byte == EncodeCompression(ECompression::LZ4)) { + if (result) { + *result = ECompression::LZ4; + } + return true; + } else { + return false; + } + } + + void DecodeSpackV1(IInputStream* in, IMetricConsumer* c, TStringBuf metricNameLabel) { + TDecoderSpackV1 decoder(in, metricNameLabel); + decoder.Decode(c); + } + +} diff --git a/library/cpp/monlib/encode/spack/spack_v1_encoder.cpp b/library/cpp/monlib/encode/spack/spack_v1_encoder.cpp new file mode 100644 index 0000000000..a2b0bb5f50 --- /dev/null +++ b/library/cpp/monlib/encode/spack/spack_v1_encoder.cpp @@ -0,0 +1,318 @@ +#include "spack_v1.h" +#include "compression.h" +#include "varint.h" + +#include <library/cpp/monlib/encode/buffered/buffered_encoder_base.h> + +#include <util/generic/cast.h> +#include <util/datetime/base.h> +#include <util/string/builder.h> + +#ifndef _little_endian_ +#error Unsupported platform +#endif + +namespace NMonitoring { + namespace { + /////////////////////////////////////////////////////////////////////// + // TEncoderSpackV1 + /////////////////////////////////////////////////////////////////////// + class TEncoderSpackV1 final: public TBufferedEncoderBase { + public: + TEncoderSpackV1( + IOutputStream* out, + ETimePrecision timePrecision, + ECompression compression, + EMetricsMergingMode mergingMode, + ESpackV1Version version, + TStringBuf metricNameLabel + ) + : Out_(out) + , TimePrecision_(timePrecision) + , Compression_(compression) + , Version_(version) + , MetricName_(Version_ >= SV1_02 ? LabelNamesPool_.PutIfAbsent(metricNameLabel) : nullptr) + { + MetricsMergingMode_ = mergingMode; + + LabelNamesPool_.SetSorted(true); + LabelValuesPool_.SetSorted(true); + } + + ~TEncoderSpackV1() override { + Close(); + } + + private: + void OnDouble(TInstant time, double value) override { + TBufferedEncoderBase::OnDouble(time, value); + } + + void OnInt64(TInstant time, i64 value) override { + TBufferedEncoderBase::OnInt64(time, value); + } + + void OnUint64(TInstant time, ui64 value) override { + TBufferedEncoderBase::OnUint64(time, value); + } + + void OnHistogram(TInstant time, IHistogramSnapshotPtr snapshot) override { + TBufferedEncoderBase::OnHistogram(time, snapshot); + } + + void OnSummaryDouble(TInstant time, ISummaryDoubleSnapshotPtr snapshot) override { + TBufferedEncoderBase::OnSummaryDouble(time, snapshot); + } + + void OnLogHistogram(TInstant time, TLogHistogramSnapshotPtr snapshot) override { + TBufferedEncoderBase::OnLogHistogram(time, snapshot); + } + + void Close() override { + if (Closed_) { + return; + } + Closed_ = true; + + LabelNamesPool_.Build(); + LabelValuesPool_.Build(); + + // Sort all points uniquely by ts -- the size can decrease + ui64 pointsCount = 0; + for (TMetric& metric : Metrics_) { + if (metric.TimeSeries.Size() > 1) { + metric.TimeSeries.SortByTs(); + } + + pointsCount += metric.TimeSeries.Size(); + } + + // (1) write header + TSpackHeader header; + header.Version = Version_; + header.TimePrecision = EncodeTimePrecision(TimePrecision_); + header.Compression = EncodeCompression(Compression_); + header.LabelNamesSize = static_cast<ui32>( + LabelNamesPool_.BytesSize() + LabelNamesPool_.Count()); + header.LabelValuesSize = static_cast<ui32>( + LabelValuesPool_.BytesSize() + LabelValuesPool_.Count()); + header.MetricCount = Metrics_.size(); + header.PointsCount = pointsCount; + Out_->Write(&header, sizeof(header)); + + // if compression enabled all below writes must go throught compressor + auto compressedOut = CompressedOutput(Out_, Compression_); + if (compressedOut) { + Out_ = compressedOut.Get(); + } + + // (2) write string pools + auto strPoolWrite = [this](TStringBuf str, ui32, ui32) { + Out_->Write(str); + Out_->Write('\0'); + }; + + LabelNamesPool_.ForEach(strPoolWrite); + LabelValuesPool_.ForEach(strPoolWrite); + + // (3) write common time + WriteTime(CommonTime_); + + // (4) write common labels' indexes + WriteLabels(CommonLabels_, nullptr); + + // (5) write metrics + // metrics count already written in header + for (TMetric& metric : Metrics_) { + // (5.1) types byte + ui8 typesByte = PackTypes(metric); + Out_->Write(&typesByte, sizeof(typesByte)); + + // TODO: implement + ui8 flagsByte = 0x00; + Out_->Write(&flagsByte, sizeof(flagsByte)); + + // v1.2 format addition — metric name + if (Version_ >= SV1_02) { + const auto it = FindIf(metric.Labels, [&](const auto& l) { + return l.Key == MetricName_; + }); + Y_ENSURE(it != metric.Labels.end(), + "metric name label '" << LabelNamesPool_.Get(MetricName_->Index) << "' not found, " + << "all metric labels '" << FormatLabels(metric.Labels) << "'"); + WriteVarUInt32(Out_, it->Value->Index); + } + + // (5.2) labels + WriteLabels(metric.Labels, MetricName_); + + // (5.3) values + switch (metric.TimeSeries.Size()) { + case 0: + break; + case 1: { + const auto& point = metric.TimeSeries[0]; + if (point.GetTime() != TInstant::Zero()) { + WriteTime(point.GetTime()); + } + EMetricValueType valueType = metric.TimeSeries.GetValueType(); + WriteValue(metric.MetricType, valueType, point.GetValue()); + break; + } + default: + WriteVarUInt32(Out_, static_cast<ui32>(metric.TimeSeries.Size())); + const TMetricTimeSeries& ts = metric.TimeSeries; + EMetricType metricType = metric.MetricType; + ts.ForEach([this, metricType](TInstant time, EMetricValueType valueType, TMetricValue value) { + // workaround for GCC bug + // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=61636 + this->WriteTime(time); + this->WriteValue(metricType, valueType, value); + }); + break; + } + } + } + + // store metric type and values type in one byte + ui8 PackTypes(const TMetric& metric) { + EValueType valueType; + if (metric.TimeSeries.Empty()) { + valueType = EValueType::NONE; + } else if (metric.TimeSeries.Size() == 1) { + TInstant time = metric.TimeSeries[0].GetTime(); + valueType = (time == TInstant::Zero()) + ? EValueType::ONE_WITHOUT_TS + : EValueType::ONE_WITH_TS; + } else { + valueType = EValueType::MANY_WITH_TS; + } + return (static_cast<ui8>(metric.MetricType) << 2) | static_cast<ui8>(valueType); + } + + void WriteLabels(const TPooledLabels& labels, const TPooledStr* skipKey) { + WriteVarUInt32(Out_, static_cast<ui32>(skipKey ? labels.size() - 1 : labels.size())); + for (auto&& label : labels) { + if (label.Key == skipKey) { + continue; + } + WriteVarUInt32(Out_, label.Key->Index); + WriteVarUInt32(Out_, label.Value->Index); + } + } + + void WriteValue(EMetricType metricType, EMetricValueType valueType, TMetricValue value) { + switch (metricType) { + case EMetricType::GAUGE: + WriteFixed(value.AsDouble(valueType)); + break; + + case EMetricType::IGAUGE: + WriteFixed(value.AsInt64(valueType)); + break; + + case EMetricType::COUNTER: + case EMetricType::RATE: + WriteFixed(value.AsUint64(valueType)); + break; + + case EMetricType::HIST: + case EMetricType::HIST_RATE: + WriteHistogram(*value.AsHistogram()); + break; + + case EMetricType::DSUMMARY: + WriteSummaryDouble(*value.AsSummaryDouble()); + break; + + case EMetricType::LOGHIST: + WriteLogHistogram(*value.AsLogHistogram()); + break; + + default: + ythrow yexception() << "unsupported metric type: " << metricType; + } + } + + void WriteTime(TInstant instant) { + switch (TimePrecision_) { + case ETimePrecision::SECONDS: { + ui32 time = static_cast<ui32>(instant.Seconds()); + Out_->Write(&time, sizeof(time)); + break; + } + case ETimePrecision::MILLIS: { + ui64 time = static_cast<ui64>(instant.MilliSeconds()); + Out_->Write(&time, sizeof(time)); + } + } + } + + template <typename T> + void WriteFixed(T value) { + Out_->Write(&value, sizeof(value)); + } + + void WriteHistogram(const IHistogramSnapshot& histogram) { + ui32 count = histogram.Count(); + WriteVarUInt32(Out_, count); + + for (ui32 i = 0; i < count; i++) { + double bound = histogram.UpperBound(i); + Out_->Write(&bound, sizeof(bound)); + } + for (ui32 i = 0; i < count; i++) { + ui64 value = histogram.Value(i); + Out_->Write(&value, sizeof(value)); + } + } + + void WriteLogHistogram(const TLogHistogramSnapshot& logHist) { + WriteFixed(logHist.Base()); + WriteFixed(logHist.ZerosCount()); + WriteVarUInt32(Out_, static_cast<ui32>(logHist.StartPower())); + WriteVarUInt32(Out_, logHist.Count()); + for (ui32 i = 0; i < logHist.Count(); ++i) { + WriteFixed(logHist.Bucket(i)); + } + } + + void WriteSummaryDouble(const ISummaryDoubleSnapshot& summary) { + WriteFixed(summary.GetCount()); + WriteFixed(summary.GetSum()); + WriteFixed(summary.GetMin()); + WriteFixed(summary.GetMax()); + WriteFixed(summary.GetLast()); + } + + private: + IOutputStream* Out_; + ETimePrecision TimePrecision_; + ECompression Compression_; + ESpackV1Version Version_; + const TPooledStr* MetricName_; + bool Closed_ = false; + }; + + } + + IMetricEncoderPtr EncoderSpackV1( + IOutputStream* out, + ETimePrecision timePrecision, + ECompression compression, + EMetricsMergingMode mergingMode + ) { + return MakeHolder<TEncoderSpackV1>(out, timePrecision, compression, mergingMode, SV1_01, ""); + } + + IMetricEncoderPtr EncoderSpackV12( + IOutputStream* out, + ETimePrecision timePrecision, + ECompression compression, + EMetricsMergingMode mergingMode, + TStringBuf metricNameLabel + ) { + Y_ENSURE(!metricNameLabel.Empty(), "metricNameLabel can't be empty"); + return MakeHolder<TEncoderSpackV1>(out, timePrecision, compression, mergingMode, SV1_02, metricNameLabel); + } +} diff --git a/library/cpp/monlib/encode/spack/spack_v1_ut.cpp b/library/cpp/monlib/encode/spack/spack_v1_ut.cpp new file mode 100644 index 0000000000..fe778eb7e0 --- /dev/null +++ b/library/cpp/monlib/encode/spack/spack_v1_ut.cpp @@ -0,0 +1,845 @@ +#include "spack_v1.h" + +#include <library/cpp/monlib/encode/protobuf/protobuf.h> +#include <library/cpp/monlib/metrics/labels.h> +#include <library/cpp/monlib/metrics/metric.h> + +#include <library/cpp/testing/unittest/registar.h> + +#include <util/generic/buffer.h> +#include <util/stream/buffer.h> +#include <util/string/hex.h> + +#include <utility> + +using namespace NMonitoring; + +#define UNIT_ASSERT_BINARY_EQUALS(a, b) \ + do { \ + auto size = Y_ARRAY_SIZE(b); \ + if (Y_UNLIKELY(::memcmp(a, b, size) != 0)) { \ + auto as = HexEncode(a, size); \ + auto bs = HexEncode(b, size); \ + UNIT_FAIL_IMPL("equal assertion failed " #a " == " #b, \ + "\n actual: " << as << "\nexpected: " << bs); \ + } \ + } while (0) + +void AssertLabelEqual(const NProto::TLabel& l, TStringBuf name, TStringBuf value) { + UNIT_ASSERT_STRINGS_EQUAL(l.GetName(), name); + UNIT_ASSERT_STRINGS_EQUAL(l.GetValue(), value); +} + +void AssertPointEqual(const NProto::TPoint& p, TInstant time, double value) { + UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds()); + UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kFloat64); + UNIT_ASSERT_DOUBLES_EQUAL(p.GetFloat64(), value, std::numeric_limits<double>::epsilon()); +} + +void AssertPointEqual(const NProto::TPoint& p, TInstant time, ui64 value) { + UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds()); + UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kUint64); + UNIT_ASSERT_VALUES_EQUAL(p.GetUint64(), value); +} + +void AssertPointEqual(const NProto::TPoint& p, TInstant time, i64 value) { + UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds()); + UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kInt64); + UNIT_ASSERT_VALUES_EQUAL(p.GetInt64(), value); +} + +Y_UNIT_TEST_SUITE(TSpackTest) { + ui8 expectedHeader_v1_0[] = { + 0x53, 0x50, // magic "SP" (fixed ui16) + // minor, major + 0x00, 0x01, // version (fixed ui16) + 0x18, 0x00, // header size (fixed ui16) + 0x00, // time precision (fixed ui8) + 0x00, // compression algorithm (fixed ui8) + 0x0d, 0x00, 0x00, 0x00, // label names size (fixed ui32) + 0x40, 0x00, 0x00, 0x00, // labels values size (fixed ui32) + 0x08, 0x00, 0x00, 0x00, // metric count (fixed ui32) + 0x08, 0x00, 0x00, 0x00, // points count (fixed ui32) + }; + + ui8 expectedHeader[] = { + 0x53, 0x50, // magic "SP" (fixed ui16) + // minor, major + 0x01, 0x01, // version (fixed ui16) + 0x18, 0x00, // header size (fixed ui16) + 0x00, // time precision (fixed ui8) + 0x00, // compression algorithm (fixed ui8) + 0x0d, 0x00, 0x00, 0x00, // label names size (fixed ui32) + 0x40, 0x00, 0x00, 0x00, // labels values size (fixed ui32) + 0x08, 0x00, 0x00, 0x00, // metric count (fixed ui32) + 0x08, 0x00, 0x00, 0x00, // points count (fixed ui32) + }; + + ui8 expectedStringPools[] = { + 0x6e, 0x61, 0x6d, 0x65, 0x00, // "name\0" + 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x00, // "project\0" + 0x73, 0x6f, 0x6c, 0x6f, 0x6d, 0x6f, 0x6e, 0x00, // "solomon\0" + 0x71, 0x31, 0x00, // "q1\0" + 0x71, 0x32, 0x00, // "q2\0" + 0x71, 0x33, 0x00, // "q3\0" + 0x61, 0x6e, 0x73, 0x77, 0x65, 0x72, 0x00, // "answer\0" + 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, // "responseTimeMillis\0" + 0x54, 0x69, 0x6d, 0x65, 0x4d, 0x69, 0x6c, 0x6c, + 0x69, 0x73, 0x00, + 0x62, 0x79, 0x74, 0x65, 0x73, 0x00, // "bytes\0" + 0x74, 0x65, 0x6D, 0x70, 0x65, 0x72, 0x61, 0x74, // "temperature\0" + 0x75, 0x72, 0x65, 0x00, + 0x6d, 0x73, 0x00, // "ms\0" + }; + + ui8 expectedCommonTime[] = { + 0x00, 0x2f, 0x68, 0x59, // common time in seconds (fixed ui32) + }; + + ui8 expectedCommonLabels[] = { + 0x01, // common labels count (varint) + 0x01, // label name index (varint) + 0x00, // label value index (varint) + }; + + ui8 expectedMetric1[] = { + 0x0C, // types (RATE | NONE) (fixed ui8) + 0x00, // flags (fixed ui8) + 0x01, // metric labels count (varint) + 0x00, // label name index (varint) + 0x01, // label value index (varint) + }; + + ui8 expectedMetric2[] = { + 0x09, // types (COUNTER | ONE_WITHOUT_TS) (fixed ui8) + 0x00, // flags (fixed ui8) + 0x01, // metric labels count (varint) + 0x00, // label name index (varint) + 0x02, // label value index (varint) + 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // value (fixed ui64) + }; + + ui8 expectedMetric3[] = { + 0x0a, // types (COUNTER | ONE_WITH_TS) (fixed ui8) + 0x00, // flags (fixed ui8) + 0x01, // metric labels count (varint) + 0x00, // label name index (varint) + 0x03, // label value index (varint) + 0x0b, 0x63, 0xfe, 0x59, // time in seconds (fixed ui32) + 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // value (fixed ui64) + }; + + ui8 expectedMetric4[] = { + 0x07, // types (GAUGE | MANY_WITH_TS) (fixed ui8) + 0x00, // flags (fixed ui8) + 0x01, // metric labels count (varint) + 0x00, // label name index (varint) + 0x04, // label value index (varint) + 0x02, // points count (varint) + 0x0b, 0x63, 0xfe, 0x59, // time in seconds (fixed ui32) + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x45, 0x40, // value (double IEEE754) + 0x1a, 0x63, 0xfe, 0x59, // time in seconds (fixed ui32) + 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x4d, 0x40 // value (double IEEE754) + }; + + ui8 expectedMetric5_v1_0[] = { + 0x16, // types (HIST | ONE_WITH_TS) (fixed ui8) + 0x00, // flags (fixed ui8) + 0x01, // metric labels count (varint) + 0x00, // label name index (varint) + 0x05, // label value index (varint) + 0x0b, 0x63, 0xfe, 0x59, // time in seconds (fixed ui32) + 0x06, // histogram buckets count (varint) + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // histogram bucket bounds (array of fixed ui64) + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // histogram bucket values + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x53, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + }; + + ui8 expectedMetric5[] = { + 0x16, // types (HIST | ONE_WITH_TS) (fixed ui8) + 0x00, // flags (fixed ui8) + 0x01, // metric labels count (varint) + 0x00, // label name index (varint) + 0x05, // label value index (varint) + 0x0b, 0x63, 0xfe, 0x59, // time in seconds (fixed ui32) + 0x06, // histogram buckets count (varint) + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x3f, // histogram bucket bounds (array of doubles) + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x40, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xef, 0x7f, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // histogram bucket values + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x53, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + }; + + ui8 expectedMetric6[] = { + 0x12, // types (IGAUGE | ONE_WITH_TS) (fixed ui8) + 0x00, // flags (fixed ui8) + 0x01, // metric labels count (varint) + 0x00, // label name index (varint) + 0x06, // label value index (varint) + 0x0b, 0x63, 0xfe, 0x59, // time in seconds (fixed ui32) + 0x39, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // value (fixed i64) + }; + + ui8 expectedMetric7[] = { + 0x1e, // types (DSUMMARY | ONE_WITH_TS) (fixed ui8) + 0x00, // flags (fixed ui8) + 0x01, // metric labels count (varint) + 0x00, // label name index (varint) + 0x07, // label value index (varint) + 0x0b, 0x63, 0xfe, 0x59, // time in seconds (fixed ui32) + 0x1e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // count (fixed ui64) + 0x33, 0x33, 0x33, 0x33, 0x33, 0x33, 0x24, 0x40, // sum (fixed double) + 0xcd, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xdc, 0xbf, // min (fixed double) + 0x64, 0x3b, 0xdf, 0x4f, 0x8d, 0x97, 0xde, 0x3f, // max (fixed double) + 0x33, 0x33, 0x33, 0x33, 0x33, 0x33, 0xd3, 0x3f, // last (fixed double) + }; + + ui8 expectedMetric8[] = { + 0x26, // types (LOGHIST | ONE_WITH_TS) (fixed ui8) + 0x00, // flags (fixed ui8) + 0x01, // metric labels count (variant) + 0x00, // label name index (variant) + 0x08, // label value index (variant) + 0x0b, 0x63, 0xfe, 0x59, // time in seconds (fixed ui32) + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF8, 0x3F, // base (fixed double) + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // zerosCount (fixed ui64) + 0x00, // startPower (variant) + 0x04, // buckets count (variant) + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE0, 0x3F, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD0, 0x3F, // bucket values + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD0, 0x3F, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE0, 0x3F, + }; + + const size_t expectedSize = + Y_ARRAY_SIZE(expectedHeader) + + Y_ARRAY_SIZE(expectedStringPools) + + Y_ARRAY_SIZE(expectedCommonTime) + + Y_ARRAY_SIZE(expectedCommonLabels) + + Y_ARRAY_SIZE(expectedMetric1) + + Y_ARRAY_SIZE(expectedMetric2) + + Y_ARRAY_SIZE(expectedMetric3) + + Y_ARRAY_SIZE(expectedMetric4) + + Y_ARRAY_SIZE(expectedMetric5) + + Y_ARRAY_SIZE(expectedMetric6) + + Y_ARRAY_SIZE(expectedMetric7) + + Y_ARRAY_SIZE(expectedMetric8); + + const TInstant now = TInstant::ParseIso8601Deprecated("2017-11-05T01:02:03Z"); + + // {1: 1, 2: 1, 4: 2, 8: 4, 16: 8, inf: 83} + IHistogramSnapshotPtr TestHistogram() { + auto h = ExponentialHistogram(6, 2); + for (i64 i = 1; i < 100; i++) { + h->Collect(i); + } + return h->Snapshot(); + } + + TLogHistogramSnapshotPtr TestLogHistogram() { + TVector buckets{0.5, 0.25, 0.25, 0.5}; + return MakeIntrusive<TLogHistogramSnapshot>(1.5, 1u, 0, std::move(buckets)); + } + + ISummaryDoubleSnapshotPtr TestSummaryDouble() { + return MakeIntrusive<TSummaryDoubleSnapshot>(10.1, -0.45, 0.478, 0.3, 30u); + } + + Y_UNIT_TEST(Encode) { + TBuffer buffer; + TBufferOutput out(buffer); + auto e = EncoderSpackV1( + &out, ETimePrecision::SECONDS, ECompression::IDENTITY); + + e->OnStreamBegin(); + { // common time + e->OnCommonTime(TInstant::Seconds(1500000000)); + } + { // common labels + e->OnLabelsBegin(); + e->OnLabel("project", "solomon"); + e->OnLabelsEnd(); + } + { // metric #1 + e->OnMetricBegin(EMetricType::RATE); + { + e->OnLabelsBegin(); + e->OnLabel("name", "q1"); + e->OnLabelsEnd(); + } + e->OnMetricEnd(); + } + { // metric #2 + e->OnMetricBegin(EMetricType::COUNTER); + { + e->OnLabelsBegin(); + e->OnLabel("name", "q2"); + e->OnLabelsEnd(); + } + // Only the last value will be encoded + e->OnUint64(TInstant::Zero(), 10); + e->OnUint64(TInstant::Zero(), 13); + e->OnUint64(TInstant::Zero(), 17); + e->OnMetricEnd(); + } + { // metric #3 + e->OnMetricBegin(EMetricType::COUNTER); + { + e->OnLabelsBegin(); + e->OnLabel("name", "q3"); + e->OnLabelsEnd(); + } + e->OnUint64(now, 10); + e->OnUint64(now, 13); + e->OnUint64(now, 17); + e->OnMetricEnd(); + } + { // metric #4 + e->OnMetricBegin(EMetricType::GAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("name", "answer"); + e->OnLabelsEnd(); + } + e->OnDouble(now, 42); + e->OnDouble(now + TDuration::Seconds(15), 59); + e->OnMetricEnd(); + } + { // metric #5 + e->OnMetricBegin(EMetricType::HIST); + { + e->OnLabelsBegin(); + e->OnLabel("name", "responseTimeMillis"); + e->OnLabelsEnd(); + } + + auto histogram = TestHistogram(); + e->OnHistogram(now, histogram); + e->OnMetricEnd(); + } + { // metric #6 + e->OnMetricBegin(EMetricType::IGAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("name", "bytes"); + e->OnLabelsEnd(); + } + e->OnInt64(now, 1337); + e->OnMetricEnd(); + } + { // metric 7 + e->OnMetricBegin(EMetricType::DSUMMARY); + { + e->OnLabelsBegin(); + e->OnLabel("name", "temperature"); + e->OnLabelsEnd(); + } + e->OnSummaryDouble(now, TestSummaryDouble()); + e->OnMetricEnd(); + } + { // metric 8 + e->OnMetricBegin(EMetricType::LOGHIST); + { + e->OnLabelsBegin(); + e->OnLabel("name", "ms"); + e->OnLabelsEnd(); + } + e->OnLogHistogram(now, TestLogHistogram()); + e->OnMetricEnd(); + } + e->OnStreamEnd(); + e->Close(); + + // Cout << "encoded: " << HexEncode(buffer.Data(), buffer.Size()) << Endl; + // Cout << "size: " << buffer.Size() << Endl; + + UNIT_ASSERT_VALUES_EQUAL(buffer.Size(), expectedSize); + + ui8* p = reinterpret_cast<ui8*>(buffer.Data()); + UNIT_ASSERT_BINARY_EQUALS(p, expectedHeader); + p += Y_ARRAY_SIZE(expectedHeader); + + UNIT_ASSERT_BINARY_EQUALS(p, expectedStringPools); + p += Y_ARRAY_SIZE(expectedStringPools); + + UNIT_ASSERT_BINARY_EQUALS(p, expectedCommonTime); + p += Y_ARRAY_SIZE(expectedCommonTime); + + UNIT_ASSERT_BINARY_EQUALS(p, expectedCommonLabels); + p += Y_ARRAY_SIZE(expectedCommonLabels); + + UNIT_ASSERT_BINARY_EQUALS(p, expectedMetric1); + p += Y_ARRAY_SIZE(expectedMetric1); + + UNIT_ASSERT_BINARY_EQUALS(p, expectedMetric2); + p += Y_ARRAY_SIZE(expectedMetric2); + + UNIT_ASSERT_BINARY_EQUALS(p, expectedMetric3); + p += Y_ARRAY_SIZE(expectedMetric3); + + UNIT_ASSERT_BINARY_EQUALS(p, expectedMetric4); + p += Y_ARRAY_SIZE(expectedMetric4); + + UNIT_ASSERT_BINARY_EQUALS(p, expectedMetric5); + p += Y_ARRAY_SIZE(expectedMetric5); + + UNIT_ASSERT_BINARY_EQUALS(p, expectedMetric6); + p += Y_ARRAY_SIZE(expectedMetric6); + + UNIT_ASSERT_BINARY_EQUALS(p, expectedMetric7); + p += Y_ARRAY_SIZE(expectedMetric7); + + UNIT_ASSERT_BINARY_EQUALS(p, expectedMetric8); + p += Y_ARRAY_SIZE(expectedMetric8); + } + + NProto::TMultiSamplesList GetMergingMetricSamples(EMetricsMergingMode mergingMode) { + TBuffer buffer; + TBufferOutput out(buffer); + + auto e = EncoderSpackV1( + &out, + ETimePrecision::SECONDS, + ECompression::IDENTITY, + mergingMode + ); + + e->OnStreamBegin(); + for (size_t i = 0; i != 3; ++i) { + e->OnMetricBegin(EMetricType::COUNTER); + { + e->OnLabelsBegin(); + e->OnLabel("name", "my_counter"); + e->OnLabelsEnd(); + } + e->OnUint64(TInstant::Zero() + TDuration::Seconds(i), i + 1); + e->OnMetricEnd(); + } + e->OnStreamEnd(); + e->Close(); + + NProto::TMultiSamplesList samples; + IMetricEncoderPtr eProto = EncoderProtobuf(&samples); + TBufferInput in(buffer); + DecodeSpackV1(&in, eProto.Get()); + + return samples; + } + + Y_UNIT_TEST(SpackEncoderMergesMetrics) { + { + NProto::TMultiSamplesList samples = GetMergingMetricSamples(EMetricsMergingMode::DEFAULT); + + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 3); + UNIT_ASSERT_EQUAL(samples.GetSamples(0).GetPoints(0).GetUint64(), 1); + UNIT_ASSERT_EQUAL(samples.GetSamples(1).GetPoints(0).GetUint64(), 2); + UNIT_ASSERT_EQUAL(samples.GetSamples(2).GetPoints(0).GetUint64(), 3); + } + + { + NProto::TMultiSamplesList samples = GetMergingMetricSamples(EMetricsMergingMode::MERGE_METRICS); + + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 1); + + auto sample0 = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(sample0.GetPoints(0).GetUint64(), 1); + UNIT_ASSERT_EQUAL(sample0.GetPoints(1).GetUint64(), 2); + UNIT_ASSERT_EQUAL(sample0.GetPoints(2).GetUint64(), 3); + } + } + + void DecodeDataToSamples(NProto::TMultiSamplesList & samples, ui16 version) { + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + TBuffer data(expectedSize); + if (SV1_00 == version) { // v1.0 + data.Append(reinterpret_cast<char*>(expectedHeader_v1_0), Y_ARRAY_SIZE(expectedHeader_v1_0)); + } else { + data.Append(reinterpret_cast<char*>(expectedHeader), Y_ARRAY_SIZE(expectedHeader)); + } + data.Append(reinterpret_cast<char*>(expectedStringPools), Y_ARRAY_SIZE(expectedStringPools)); + data.Append(reinterpret_cast<char*>(expectedCommonTime), Y_ARRAY_SIZE(expectedCommonTime)); + data.Append(reinterpret_cast<char*>(expectedCommonLabels), Y_ARRAY_SIZE(expectedCommonLabels)); + data.Append(reinterpret_cast<char*>(expectedMetric1), Y_ARRAY_SIZE(expectedMetric1)); + data.Append(reinterpret_cast<char*>(expectedMetric2), Y_ARRAY_SIZE(expectedMetric2)); + data.Append(reinterpret_cast<char*>(expectedMetric3), Y_ARRAY_SIZE(expectedMetric3)); + data.Append(reinterpret_cast<char*>(expectedMetric4), Y_ARRAY_SIZE(expectedMetric4)); + if (SV1_00 == version) { // v1.0 + data.Append(reinterpret_cast<char*>(expectedMetric5_v1_0), Y_ARRAY_SIZE(expectedMetric5_v1_0)); + } else { + data.Append(reinterpret_cast<char*>(expectedMetric5), Y_ARRAY_SIZE(expectedMetric5)); + } + data.Append(reinterpret_cast<char*>(expectedMetric6), Y_ARRAY_SIZE(expectedMetric6)); + data.Append(reinterpret_cast<char*>(expectedMetric7), Y_ARRAY_SIZE(expectedMetric7)); + data.Append(reinterpret_cast<char*>(expectedMetric8), Y_ARRAY_SIZE(expectedMetric8)); + TBufferInput in(data); + DecodeSpackV1(&in, e.Get()); + } + + void DecodeDataToSamples(NProto::TMultiSamplesList & samples) { + TSpackHeader header; + header.Version = SV1_01; + DecodeDataToSamples(samples, header.Version); + } + + Y_UNIT_TEST(Decode) { + NProto::TMultiSamplesList samples; + DecodeDataToSamples(samples); + + UNIT_ASSERT_VALUES_EQUAL( + TInstant::MilliSeconds(samples.GetCommonTime()), + TInstant::Seconds(1500000000)); + + UNIT_ASSERT_VALUES_EQUAL(samples.CommonLabelsSize(), 1); + AssertLabelEqual(samples.GetCommonLabels(0), "project", "solomon"); + + UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 8); + { + const NProto::TMultiSample& s = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::RATE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "name", "q1"); + } + { + const NProto::TMultiSample& s = samples.GetSamples(1); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::COUNTER); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "name", "q2"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + AssertPointEqual(s.GetPoints(0), TInstant::Zero(), ui64(17)); + } + { + const NProto::TMultiSample& s = samples.GetSamples(2); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::COUNTER); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "name", "q3"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + AssertPointEqual(s.GetPoints(0), now, ui64(17)); + } + { + const NProto::TMultiSample& s = samples.GetSamples(3); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "name", "answer"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 2); + AssertPointEqual(s.GetPoints(0), now, double(42)); + AssertPointEqual(s.GetPoints(1), now + TDuration::Seconds(15), double(59)); + } + { + const NProto::TMultiSample& s = samples.GetSamples(4); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::HISTOGRAM); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "name", "responseTimeMillis"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + + const NProto::TPoint& p = s.GetPoints(0); + UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), now.MilliSeconds()); + UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kHistogram); + + auto histogram = TestHistogram(); + + const NProto::THistogram& pointHistogram = p.GetHistogram(); + UNIT_ASSERT_VALUES_EQUAL(pointHistogram.BoundsSize(), histogram->Count()); + UNIT_ASSERT_VALUES_EQUAL(pointHistogram.ValuesSize(), histogram->Count()); + + for (size_t i = 0; i < pointHistogram.BoundsSize(); i++) { + UNIT_ASSERT_DOUBLES_EQUAL(pointHistogram.GetBounds(i), histogram->UpperBound(i), Min<double>()); + UNIT_ASSERT_VALUES_EQUAL(pointHistogram.GetValues(i), histogram->Value(i)); + } + } + { + const NProto::TMultiSample& s = samples.GetSamples(5); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::IGAUGE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "name", "bytes"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + AssertPointEqual(s.GetPoints(0), now, i64(1337)); + } + { + const NProto::TMultiSample& s = samples.GetSamples(6); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::DSUMMARY); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "name", "temperature"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + + const NProto::TPoint& p = s.GetPoints(0); + UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), now.MilliSeconds()); + UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kSummaryDouble); + + auto expected = TestSummaryDouble(); + + auto actual = p.GetSummaryDouble(); + + UNIT_ASSERT_VALUES_EQUAL(expected->GetSum(), actual.GetSum()); + UNIT_ASSERT_VALUES_EQUAL(expected->GetMin(), actual.GetMin()); + UNIT_ASSERT_VALUES_EQUAL(expected->GetMax(), actual.GetMax()); + UNIT_ASSERT_VALUES_EQUAL(expected->GetLast(), actual.GetLast()); + UNIT_ASSERT_VALUES_EQUAL(expected->GetCount(), actual.GetCount()); + } + { + const NProto::TMultiSample& s = samples.GetSamples(7); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::LOGHISTOGRAM); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "name", "ms"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + + const NProto::TPoint& p = s.GetPoints(0); + UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), now.MilliSeconds()); + UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kLogHistogram); + + auto expected = TestLogHistogram(); + auto actual = p.GetLogHistogram(); + + UNIT_ASSERT_VALUES_EQUAL(expected->ZerosCount(), actual.GetZerosCount()); + UNIT_ASSERT_VALUES_EQUAL(expected->Base(), actual.GetBase()); + UNIT_ASSERT_VALUES_EQUAL(expected->StartPower(), actual.GetStartPower()); + UNIT_ASSERT_VALUES_EQUAL(expected->Count(), actual.BucketsSize()); + for (size_t i = 0; i < expected->Count(); ++i) { + UNIT_ASSERT_VALUES_EQUAL(expected->Bucket(i), actual.GetBuckets(i)); + } + } + } + + void TestCompression(ECompression alg) { + TBuffer buffer; + { + TBufferOutput out(buffer); + auto e = EncoderSpackV1(&out, ETimePrecision::MILLIS, alg); + e->OnStreamBegin(); + { + e->OnMetricBegin(EMetricType::GAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("name", "answer"); + e->OnLabelsEnd(); + } + e->OnDouble(now, 42); + e->OnMetricEnd(); + } + e->OnStreamEnd(); + e->Close(); + } + + auto* header = reinterpret_cast<const TSpackHeader*>(buffer.Data()); + UNIT_ASSERT_EQUAL(DecodeCompression(header->Compression), alg); + + NProto::TMultiSamplesList samples; + { + IMetricEncoderPtr e = EncoderProtobuf(&samples); + TBufferInput in(buffer); + DecodeSpackV1(&in, e.Get()); + } + + UNIT_ASSERT_VALUES_EQUAL( + TInstant::MilliSeconds(samples.GetCommonTime()), + TInstant::Zero()); + + UNIT_ASSERT_VALUES_EQUAL(samples.CommonLabelsSize(), 0); + + UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 1); + { + const NProto::TMultiSample& s = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "name", "answer"); + AssertPointEqual(s.GetPoints(0), now, 42.0); + } + } + + Y_UNIT_TEST(CompressionIdentity) { + TestCompression(ECompression::IDENTITY); + } + + Y_UNIT_TEST(CompressionZlib) { + TestCompression(ECompression::ZLIB); + } + + Y_UNIT_TEST(CompressionZstd) { + TestCompression(ECompression::ZSTD); + } + + Y_UNIT_TEST(CompressionLz4) { + TestCompression(ECompression::LZ4); + } + + Y_UNIT_TEST(Decode_v1_0_histograms) { + // Check that histogram bounds decoded from different versions are the same + NProto::TMultiSamplesList samples, samples_v1_0; + DecodeDataToSamples(samples); + DecodeDataToSamples(samples_v1_0, /*version = */ SV1_00); + + const NProto::THistogram& pointHistogram = samples.GetSamples(4).GetPoints(0).GetHistogram(); + const NProto::THistogram& pointHistogram_v1_0 = samples_v1_0.GetSamples(4).GetPoints(0).GetHistogram(); + + for (size_t i = 0; i < pointHistogram.BoundsSize(); i++) { + UNIT_ASSERT_DOUBLES_EQUAL(pointHistogram.GetBounds(i), pointHistogram_v1_0.GetBounds(i), Min<double>()); + } + } + + Y_UNIT_TEST(SimpleV12) { + ui8 expectedSerialized[] = { + // header + 0x53, 0x50, // magic "SP" (fixed ui16) + // minor, major + 0x02, 0x01, // version (fixed ui16) + 0x18, 0x00, // header size (fixed ui16) + 0x00, // time precision (fixed ui8) + 0x00, // compression algorithm (fixed ui8) + 0x0A, 0x00, 0x00, 0x00, // label names size (fixed ui32) + 0x14, 0x00, 0x00, 0x00, // labels values size (fixed ui32) + 0x01, 0x00, 0x00, 0x00, // metric count (fixed ui32) + 0x01, 0x00, 0x00, 0x00, // points count (fixed ui32) + + // string pools + 0x73, 0x00, // "s\0" + 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x00, // "project\0" + 0x73, 0x6f, 0x6c, 0x6f, 0x6d, 0x6f, 0x6e, 0x00, // "solomon\0" + 0x74, 0x65, 0x6D, 0x70, 0x65, 0x72, 0x61, 0x74, // temperature + 0x75, 0x72, 0x65, 0x00, + + // common time + 0x00, 0x2f, 0x68, 0x59, // common time in seconds (fixed ui32) + + // common labels + 0x00, // common labels count (varint) + + // metric + 0x09, // types (COUNTER | ONE_WITHOUT_TS) (fixed ui8) + 0x00, // flags (fixed ui8) + 0x01, // name index (varint) + 0x01, // metric labels count (varint) + 0x01, // 'project' label name index (varint) + 0x00, // 'project' label value index (varint) + 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // value (fixed ui64) + }; + + // encode + { + TBuffer actualSerialized; + { + TBufferOutput out(actualSerialized); + auto e = EncoderSpackV12( + &out, + ETimePrecision::SECONDS, + ECompression::IDENTITY, + EMetricsMergingMode::DEFAULT, + "s"); + + e->OnStreamBegin(); + e->OnCommonTime(TInstant::Seconds(1500000000)); + + { + e->OnMetricBegin(EMetricType::COUNTER); + { + e->OnLabelsBegin(); + e->OnLabel("project", "solomon"); + e->OnLabel("s", "temperature"); + e->OnLabelsEnd(); + } + // Only the last value will be encoded + e->OnUint64(TInstant::Zero(), 10); + e->OnUint64(TInstant::Zero(), 13); + e->OnUint64(TInstant::Zero(), 17); + e->OnMetricEnd(); + } + + e->OnStreamEnd(); + e->Close(); + } + + UNIT_ASSERT_VALUES_EQUAL(actualSerialized.Size(), Y_ARRAY_SIZE(expectedSerialized)); + UNIT_ASSERT_BINARY_EQUALS(actualSerialized.Data(), expectedSerialized); + } + + // decode + { + NProto::TMultiSamplesList samples; + { + auto input = TMemoryInput(expectedSerialized, Y_ARRAY_SIZE(expectedSerialized)); + auto encoder = EncoderProtobuf(&samples); + DecodeSpackV1(&input, encoder.Get(), "s"); + } + + UNIT_ASSERT_VALUES_EQUAL(TInstant::MilliSeconds(samples.GetCommonTime()), + TInstant::Seconds(1500000000)); + + UNIT_ASSERT_VALUES_EQUAL(samples.CommonLabelsSize(), 0); + + UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 1); + { + const auto& s = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::COUNTER); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2); + AssertLabelEqual(s.GetLabels(0), "s", "temperature"); + AssertLabelEqual(s.GetLabels(1), "project", "solomon"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + AssertPointEqual(s.GetPoints(0), TInstant::Zero(), ui64(17)); + } + } + } + + Y_UNIT_TEST(V12MissingNameForOneMetric) { + TBuffer b; + TBufferOutput out(b); + auto e = EncoderSpackV12( + &out, + ETimePrecision::SECONDS, + ECompression::IDENTITY, + EMetricsMergingMode::DEFAULT, + "s"); + + UNIT_ASSERT_EXCEPTION_CONTAINS( + [&]() { + e->OnStreamBegin(); + { + e->OnMetricBegin(EMetricType::COUNTER); + { + e->OnLabelsBegin(); + e->OnLabel("s", "s1"); + e->OnLabelsEnd(); + } + e->OnUint64(TInstant::Zero(), 1); + e->OnMetricEnd(); + + e->OnMetricBegin(EMetricType::COUNTER); + { + e->OnLabelsBegin(); + e->OnLabel("project", "solomon"); + e->OnLabel("m", "v"); + e->OnLabelsEnd(); + } + e->OnUint64(TInstant::Zero(), 2); + e->OnMetricEnd(); + } + + e->OnStreamEnd(); + e->Close(); + }(), + yexception, + "metric name label 's' not found, all metric labels '{m=v, project=solomon}'"); + } +} diff --git a/library/cpp/monlib/encode/spack/ut/ya.make b/library/cpp/monlib/encode/spack/ut/ya.make new file mode 100644 index 0000000000..980bf54667 --- /dev/null +++ b/library/cpp/monlib/encode/spack/ut/ya.make @@ -0,0 +1,16 @@ +UNITTEST_FOR(library/cpp/monlib/encode/spack) + +OWNER( + g:solomon + jamel +) + +SRCS( + spack_v1_ut.cpp +) + +PEERDIR( + library/cpp/monlib/encode/protobuf +) + +END() diff --git a/library/cpp/monlib/encode/spack/varint.cpp b/library/cpp/monlib/encode/spack/varint.cpp new file mode 100644 index 0000000000..051cf17380 --- /dev/null +++ b/library/cpp/monlib/encode/spack/varint.cpp @@ -0,0 +1,79 @@ +#include "varint.h" + +#include <util/generic/yexception.h> +#include <util/stream/input.h> +#include <util/stream/output.h> + +namespace NMonitoring { + ui32 WriteVarUInt32(IOutputStream* output, ui32 value) { + bool stop = false; + ui32 written = 0; + while (!stop) { + ui8 byte = static_cast<ui8>(value | 0x80); + value >>= 7; + if (value == 0) { + stop = true; + byte &= 0x7F; + } + output->Write(byte); + written++; + } + return written; + } + + ui32 ReadVarUInt32(IInputStream* input) { + ui32 value = 0; + switch (TryReadVarUInt32(input, &value)) { + case EReadResult::OK: + return value; + case EReadResult::ERR_OVERFLOW: + ythrow yexception() << "the data is too long to read ui32"; + case EReadResult::ERR_UNEXPECTED_EOF: + ythrow yexception() << "the data unexpectedly ended"; + default: + ythrow yexception() << "unknown error while reading varint"; + } + } + + size_t ReadVarUInt32(const ui8* buf, size_t len, ui32* result) { + size_t count = 0; + ui32 value = 0; + + ui8 byte = 0; + do { + if (7 * count > 8 * sizeof(ui32)) { + ythrow yexception() << "the data is too long to read ui32"; + } + if (count == len) { + ythrow yexception() << "the data unexpectedly ended"; + } + byte = buf[count]; + value |= (static_cast<ui32>(byte & 0x7F)) << (7 * count); + ++count; + } while (byte & 0x80); + + *result = value; + return count; + } + +EReadResult TryReadVarUInt32(IInputStream* input, ui32* value) { + size_t count = 0; + ui32 result = 0; + + ui8 byte = 0; + do { + if (7 * count > 8 * sizeof(ui32)) { + return EReadResult::ERR_OVERFLOW; + } + if (input->Read(&byte, 1) != 1) { + return EReadResult::ERR_UNEXPECTED_EOF; + } + result |= (static_cast<ui32>(byte & 0x7F)) << (7 * count); + ++count; + } while (byte & 0x80); + + *value = result; + return EReadResult::OK; + } + +} diff --git a/library/cpp/monlib/encode/spack/varint.h b/library/cpp/monlib/encode/spack/varint.h new file mode 100644 index 0000000000..7ac522dd6c --- /dev/null +++ b/library/cpp/monlib/encode/spack/varint.h @@ -0,0 +1,23 @@ +#pragma once + +#include <util/system/types.h> + +class IInputStream; +class IOutputStream; + +namespace NMonitoring { + ui32 WriteVarUInt32(IOutputStream* output, ui32 value); + + ui32 ReadVarUInt32(IInputStream* input); + size_t ReadVarUInt32(const ui8* buf, size_t len, ui32* result); + + enum class EReadResult { + OK, + ERR_OVERFLOW, + ERR_UNEXPECTED_EOF, + }; + + [[nodiscard]] + EReadResult TryReadVarUInt32(IInputStream* input, ui32* value); + +} diff --git a/library/cpp/monlib/encode/spack/ya.make b/library/cpp/monlib/encode/spack/ya.make new file mode 100644 index 0000000000..78d3061291 --- /dev/null +++ b/library/cpp/monlib/encode/spack/ya.make @@ -0,0 +1,25 @@ +LIBRARY() + +OWNER( + g:solomon + jamel +) + +SRCS( + spack_v1_decoder.cpp + spack_v1_encoder.cpp + varint.cpp + compression.cpp +) + +PEERDIR( + library/cpp/monlib/encode/buffered + library/cpp/monlib/exception + + contrib/libs/lz4 + contrib/libs/xxhash + contrib/libs/zlib + contrib/libs/zstd +) + +END() diff --git a/library/cpp/monlib/encode/text/text.h b/library/cpp/monlib/encode/text/text.h new file mode 100644 index 0000000000..6b2be3937b --- /dev/null +++ b/library/cpp/monlib/encode/text/text.h @@ -0,0 +1,9 @@ +#pragma once + +#include <library/cpp/monlib/encode/encoder.h> + +class IOutputStream; + +namespace NMonitoring { + IMetricEncoderPtr EncoderText(IOutputStream* out, bool humanReadableTs = true); +} diff --git a/library/cpp/monlib/encode/text/text_encoder.cpp b/library/cpp/monlib/encode/text/text_encoder.cpp new file mode 100644 index 0000000000..10336261f0 --- /dev/null +++ b/library/cpp/monlib/encode/text/text_encoder.cpp @@ -0,0 +1,226 @@ +#include "text.h" + +#include <library/cpp/monlib/encode/encoder_state.h> +#include <library/cpp/monlib/metrics/labels.h> +#include <library/cpp/monlib/metrics/metric_value.h> + +#include <util/datetime/base.h> +#include <util/stream/format.h> + +namespace NMonitoring { + namespace { + class TEncoderText final: public IMetricEncoder { + public: + TEncoderText(IOutputStream* out, bool humanReadableTs) + : Out_(out) + , HumanReadableTs_(humanReadableTs) + { + } + + private: + void OnStreamBegin() override { + State_.Expect(TEncoderState::EState::ROOT); + } + + void OnStreamEnd() override { + State_.Expect(TEncoderState::EState::ROOT); + } + + void OnCommonTime(TInstant time) override { + State_.Expect(TEncoderState::EState::ROOT); + CommonTime_ = time; + if (time != TInstant::Zero()) { + Out_->Write(TStringBuf("common time: ")); + WriteTime(time); + Out_->Write('\n'); + } + } + + void OnMetricBegin(EMetricType type) override { + State_.Switch(TEncoderState::EState::ROOT, TEncoderState::EState::METRIC); + ClearLastMetricState(); + MetricType_ = type; + } + + void OnMetricEnd() override { + State_.Switch(TEncoderState::EState::METRIC, TEncoderState::EState::ROOT); + WriteMetric(); + } + + void OnLabelsBegin() override { + if (State_ == TEncoderState::EState::METRIC) { + State_ = TEncoderState::EState::METRIC_LABELS; + } else if (State_ == TEncoderState::EState::ROOT) { + State_ = TEncoderState::EState::COMMON_LABELS; + } else { + State_.ThrowInvalid("expected METRIC or ROOT"); + } + } + + void OnLabelsEnd() override { + if (State_ == TEncoderState::EState::METRIC_LABELS) { + State_ = TEncoderState::EState::METRIC; + } else if (State_ == TEncoderState::EState::COMMON_LABELS) { + State_ = TEncoderState::EState::ROOT; + Out_->Write(TStringBuf("common labels: ")); + WriteLabels(); + Out_->Write('\n'); + } else { + State_.ThrowInvalid("expected LABELS or COMMON_LABELS"); + } + } + + void OnLabel(TStringBuf name, TStringBuf value) override { + Labels_.Add(name, value); + } + + void OnDouble(TInstant time, double value) override { + State_.Expect(TEncoderState::EState::METRIC); + TimeSeries_.Add(time, value); + } + + void OnInt64(TInstant time, i64 value) override { + State_.Expect(TEncoderState::EState::METRIC); + TimeSeries_.Add(time, value); + } + + void OnUint64(TInstant time, ui64 value) override { + State_.Expect(TEncoderState::EState::METRIC); + TimeSeries_.Add(time, value); + } + + void OnHistogram(TInstant time, IHistogramSnapshotPtr snapshot) override { + State_.Expect(TEncoderState::EState::METRIC); + TimeSeries_.Add(time, snapshot.Get()); + } + + void OnSummaryDouble(TInstant time, ISummaryDoubleSnapshotPtr snapshot) override { + State_.Expect(TEncoderState::EState::METRIC); + TimeSeries_.Add(time, snapshot.Get()); + } + + void OnLogHistogram(TInstant ts, TLogHistogramSnapshotPtr snapshot) override { + State_.Expect(TEncoderState::EState::METRIC); + TimeSeries_.Add(ts, snapshot.Get()); + } + + void Close() override { + } + + void WriteTime(TInstant time) { + if (HumanReadableTs_) { + char buf[64]; + auto len = FormatDate8601(buf, sizeof(buf), time.TimeT()); + Out_->Write(buf, len); + } else { + (*Out_) << time.Seconds(); + } + } + + void WriteValue(EMetricValueType type, TMetricValue value) { + switch (type) { + case EMetricValueType::DOUBLE: + (*Out_) << value.AsDouble(); + break; + case EMetricValueType::INT64: + (*Out_) << value.AsInt64(); + break; + case EMetricValueType::UINT64: + (*Out_) << value.AsUint64(); + break; + case EMetricValueType::HISTOGRAM: + (*Out_) << *value.AsHistogram(); + break; + case EMetricValueType::SUMMARY: + (*Out_) << *value.AsSummaryDouble(); + break; + case EMetricValueType::LOGHISTOGRAM: + (*Out_) << *value.AsLogHistogram(); + break; + case EMetricValueType::UNKNOWN: + ythrow yexception() << "unknown metric value type"; + } + } + + void WriteLabels() { + auto& out = *Out_; + const auto size = Labels_.Size(); + size_t i = 0; + + out << '{'; + for (auto&& l : Labels_) { + out << l.Name() << TStringBuf("='") << l.Value() << '\''; + + ++i; + if (i < size) { + out << TStringBuf(", "); + } + }; + + out << '}'; + } + + void WriteMetric() { + // (1) type + TStringBuf typeStr = MetricTypeToStr(MetricType_); + (*Out_) << LeftPad(typeStr, MaxMetricTypeNameLength) << ' '; + + // (2) name and labels + auto name = Labels_.Extract(TStringBuf("sensor")); + if (name) { + if (name->Value().find(' ') != TString::npos) { + (*Out_) << '"' << name->Value() << '"'; + } else { + (*Out_) << name->Value(); + } + } + WriteLabels(); + + // (3) values + if (!TimeSeries_.Empty()) { + TimeSeries_.SortByTs(); + Out_->Write(TStringBuf(" [")); + for (size_t i = 0; i < TimeSeries_.Size(); i++) { + if (i > 0) { + Out_->Write(TStringBuf(", ")); + } + + const auto& point = TimeSeries_[i]; + if (point.GetTime() == CommonTime_ || point.GetTime() == TInstant::Zero()) { + WriteValue(TimeSeries_.GetValueType(), point.GetValue()); + } else { + Out_->Write('('); + WriteTime(point.GetTime()); + Out_->Write(TStringBuf(", ")); + WriteValue(TimeSeries_.GetValueType(), point.GetValue()); + Out_->Write(')'); + } + } + Out_->Write(']'); + } + Out_->Write('\n'); + } + + void ClearLastMetricState() { + MetricType_ = EMetricType::UNKNOWN; + Labels_.Clear(); + TimeSeries_.Clear(); + } + + private: + TEncoderState State_; + IOutputStream* Out_; + bool HumanReadableTs_; + TInstant CommonTime_ = TInstant::Zero(); + EMetricType MetricType_ = EMetricType::UNKNOWN; + TLabels Labels_; + TMetricTimeSeries TimeSeries_; + }; + + } + + IMetricEncoderPtr EncoderText(IOutputStream* out, bool humanReadableTs) { + return MakeHolder<TEncoderText>(out, humanReadableTs); + } + +} diff --git a/library/cpp/monlib/encode/text/text_encoder_ut.cpp b/library/cpp/monlib/encode/text/text_encoder_ut.cpp new file mode 100644 index 0000000000..554b6f5fa9 --- /dev/null +++ b/library/cpp/monlib/encode/text/text_encoder_ut.cpp @@ -0,0 +1,283 @@ +#include "text.h" + +#include <library/cpp/monlib/metrics/histogram_collector.h> + +#include <library/cpp/testing/unittest/registar.h> + +using namespace NMonitoring; + +Y_UNIT_TEST_SUITE(TTextText) { + template <typename TFunc> + TString EncodeToString(bool humanReadableTs, TFunc fn) { + TStringStream ss; + IMetricEncoderPtr encoder = EncoderText(&ss, humanReadableTs); + fn(encoder.Get()); + return ss.Str(); + } + + Y_UNIT_TEST(Empty) { + auto result = EncodeToString(true, [](IMetricEncoder* e) { + e->OnStreamBegin(); + e->OnStreamEnd(); + }); + UNIT_ASSERT_STRINGS_EQUAL(result, ""); + } + + Y_UNIT_TEST(CommonPart) { + auto result = EncodeToString(true, [](IMetricEncoder* e) { + e->OnStreamBegin(); + e->OnCommonTime(TInstant::ParseIso8601Deprecated("2017-01-02T03:04:05.006Z")); + { + e->OnLabelsBegin(); + e->OnLabel("project", "solomon"); + e->OnLabel("cluster", "man"); + e->OnLabel("service", "stockpile"); + e->OnLabelsEnd(); + } + e->OnStreamEnd(); + }); + UNIT_ASSERT_STRINGS_EQUAL(result, + "common time: 2017-01-02T03:04:05Z\n" + "common labels: {project='solomon', cluster='man', service='stockpile'}\n"); + } + + Y_UNIT_TEST(Gauges) { + auto result = EncodeToString(true, [](IMetricEncoder* e) { + e->OnStreamBegin(); + { // no values + e->OnMetricBegin(EMetricType::GAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "cpuUsage"); + e->OnLabelsEnd(); + } + e->OnMetricEnd(); + } + { // one value no ts + e->OnMetricBegin(EMetricType::GAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "diskUsage"); + e->OnLabel("disk", "sda1"); + e->OnLabelsEnd(); + } + e->OnDouble(TInstant::Zero(), 1000); + e->OnMetricEnd(); + } + { // one value with ts + e->OnMetricBegin(EMetricType::GAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "memoryUsage"); + e->OnLabel("host", "solomon-man-00"); + e->OnLabel("dc", "man"); + e->OnLabelsEnd(); + } + e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 1000); + e->OnMetricEnd(); + } + { // many values + e->OnMetricBegin(EMetricType::GAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "bytesRx"); + e->OnLabel("host", "solomon-sas-01"); + e->OnLabel("dc", "sas"); + e->OnLabelsEnd(); + } + e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 2); + e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:05Z"), 4); + e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:10Z"), 8); + e->OnMetricEnd(); + } + e->OnStreamEnd(); + }); + UNIT_ASSERT_STRINGS_EQUAL(result, + " GAUGE cpuUsage{}\n" + " GAUGE diskUsage{disk='sda1'} [1000]\n" + " GAUGE memoryUsage{host='solomon-man-00', dc='man'} [(2017-12-02T12:00:00Z, 1000)]\n" + " GAUGE bytesRx{host='solomon-sas-01', dc='sas'} [(2017-12-02T12:00:00Z, 2), (2017-12-02T12:00:05Z, 4), (2017-12-02T12:00:10Z, 8)]\n"); + } + + Y_UNIT_TEST(IntGauges) { + auto result = EncodeToString(true, [](IMetricEncoder* e) { + e->OnStreamBegin(); + { // no values + e->OnMetricBegin(EMetricType::IGAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "cpuUsage"); + e->OnLabelsEnd(); + } + e->OnMetricEnd(); + } + { // one value no ts + e->OnMetricBegin(EMetricType::IGAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "diskUsage"); + e->OnLabel("disk", "sda1"); + e->OnLabelsEnd(); + } + e->OnDouble(TInstant::Zero(), 1000); + e->OnMetricEnd(); + } + { // one value with ts + e->OnMetricBegin(EMetricType::IGAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "memoryUsage"); + e->OnLabel("host", "solomon-man-00"); + e->OnLabel("dc", "man"); + e->OnLabelsEnd(); + } + e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 1000); + e->OnMetricEnd(); + } + { // many values + e->OnMetricBegin(EMetricType::IGAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "bytesRx"); + e->OnLabel("host", "solomon-sas-01"); + e->OnLabel("dc", "sas"); + e->OnLabelsEnd(); + } + e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 2); + e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:05Z"), 4); + e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:10Z"), 8); + e->OnMetricEnd(); + } + e->OnStreamEnd(); + }); + UNIT_ASSERT_STRINGS_EQUAL(result, + " IGAUGE cpuUsage{}\n" + " IGAUGE diskUsage{disk='sda1'} [1000]\n" + " IGAUGE memoryUsage{host='solomon-man-00', dc='man'} [(2017-12-02T12:00:00Z, 1000)]\n" + " IGAUGE bytesRx{host='solomon-sas-01', dc='sas'} [(2017-12-02T12:00:00Z, 2), (2017-12-02T12:00:05Z, 4), (2017-12-02T12:00:10Z, 8)]\n"); + } + + Y_UNIT_TEST(Counters) { + auto doEncode = [](IMetricEncoder* e) { + e->OnStreamBegin(); + { // no values + e->OnMetricBegin(EMetricType::COUNTER); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "cpuUsage"); + e->OnLabelsEnd(); + } + e->OnMetricEnd(); + } + { // one value no ts + e->OnMetricBegin(EMetricType::COUNTER); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "diskUsage"); + e->OnLabel("disk", "sda1"); + e->OnLabelsEnd(); + } + e->OnUint64(TInstant::Zero(), 1000); + e->OnMetricEnd(); + } + { // one value with ts + e->OnMetricBegin(EMetricType::COUNTER); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "memoryUsage"); + e->OnLabel("host", "solomon-man-00"); + e->OnLabel("dc", "man"); + e->OnLabelsEnd(); + } + e->OnUint64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 1000); + e->OnMetricEnd(); + } + { // many values + e->OnMetricBegin(EMetricType::COUNTER); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "bytesRx"); + e->OnLabel("host", "solomon-sas-01"); + e->OnLabel("dc", "sas"); + e->OnLabelsEnd(); + } + e->OnUint64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 2); + e->OnUint64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:05Z"), 4); + e->OnUint64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:10Z"), 8); + e->OnMetricEnd(); + } + e->OnStreamEnd(); + }; + + auto result1 = EncodeToString(false, doEncode); + UNIT_ASSERT_STRINGS_EQUAL(result1, + " COUNTER cpuUsage{}\n" + " COUNTER diskUsage{disk='sda1'} [1000]\n" + " COUNTER memoryUsage{host='solomon-man-00', dc='man'} [(1512216000, 1000)]\n" + " COUNTER bytesRx{host='solomon-sas-01', dc='sas'} [(1512216000, 2), (1512216005, 4), (1512216010, 8)]\n"); + + auto result2 = EncodeToString(true, doEncode); + UNIT_ASSERT_STRINGS_EQUAL(result2, + " COUNTER cpuUsage{}\n" + " COUNTER diskUsage{disk='sda1'} [1000]\n" + " COUNTER memoryUsage{host='solomon-man-00', dc='man'} [(2017-12-02T12:00:00Z, 1000)]\n" + " COUNTER bytesRx{host='solomon-sas-01', dc='sas'} [(2017-12-02T12:00:00Z, 2), (2017-12-02T12:00:05Z, 4), (2017-12-02T12:00:10Z, 8)]\n"); + } + + Y_UNIT_TEST(Histograms) { + auto h = ExplicitHistogram({1, 2, 3, 4, 5}); + h->Collect(3); + h->Collect(5, 7); + h->Collect(13); + auto s = h->Snapshot(); + + TString result = EncodeToString(true, [s](IMetricEncoder* e) { + e->OnStreamBegin(); + { + e->OnMetricBegin(EMetricType::HIST); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "readTimeMillis"); + e->OnLabelsEnd(); + } + e->OnHistogram(TInstant::Zero(), s); + e->OnMetricEnd(); + } + { + e->OnMetricBegin(EMetricType::HIST_RATE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "writeTimeMillis"); + e->OnLabelsEnd(); + } + e->OnHistogram(TInstant::Zero(), s); + e->OnMetricEnd(); + } + e->OnStreamEnd(); + }); + + UNIT_ASSERT_STRINGS_EQUAL(result, + " HIST readTimeMillis{} [{1: 0, 2: 0, 3: 1, 4: 0, 5: 7, inf: 1}]\n" + "HIST_RATE writeTimeMillis{} [{1: 0, 2: 0, 3: 1, 4: 0, 5: 7, inf: 1}]\n"); + } + + Y_UNIT_TEST(Summary) { + auto s = MakeIntrusive<TSummaryDoubleSnapshot>(10.1, -0.45, 0.478, 0.3, 30u); + TString result = EncodeToString(true, [s](IMetricEncoder* e) { + e->OnStreamBegin(); + { + e->OnMetricBegin(EMetricType::DSUMMARY); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "temperature"); + e->OnLabelsEnd(); + } + e->OnSummaryDouble(TInstant::Zero(), s); + e->OnMetricEnd(); + } + e->OnStreamEnd(); + }); + UNIT_ASSERT_STRINGS_EQUAL(result, + " DSUMMARY temperature{} [{sum: 10.1, min: -0.45, max: 0.478, last: 0.3, count: 30}]\n"); + } +} diff --git a/library/cpp/monlib/encode/text/ut/ya.make b/library/cpp/monlib/encode/text/ut/ya.make new file mode 100644 index 0000000000..df23a252d1 --- /dev/null +++ b/library/cpp/monlib/encode/text/ut/ya.make @@ -0,0 +1,12 @@ +UNITTEST_FOR(library/cpp/monlib/encode/text) + +OWNER( + g:solomon + jamel +) + +SRCS( + text_encoder_ut.cpp +) + +END() diff --git a/library/cpp/monlib/encode/text/ya.make b/library/cpp/monlib/encode/text/ya.make new file mode 100644 index 0000000000..d296c78c1b --- /dev/null +++ b/library/cpp/monlib/encode/text/ya.make @@ -0,0 +1,16 @@ +LIBRARY() + +OWNER( + g:solomon + jamel +) + +SRCS( + text_encoder.cpp +) + +PEERDIR( + library/cpp/monlib/encode +) + +END() diff --git a/library/cpp/monlib/encode/unistat/unistat.h b/library/cpp/monlib/encode/unistat/unistat.h new file mode 100644 index 0000000000..1c43b7fa1b --- /dev/null +++ b/library/cpp/monlib/encode/unistat/unistat.h @@ -0,0 +1,13 @@ +#pragma once + +#include <util/generic/fwd.h> +#include <util/datetime/base.h> + +namespace NMonitoring { + /// Decodes unistat-style metrics + /// https://wiki.yandex-team.ru/golovan/stat-handle + void DecodeUnistat(TStringBuf data, class IMetricConsumer* c, TInstant ts = TInstant::Zero()); + + /// Assumes consumer's stream is open by the caller + void DecodeUnistatToStream(TStringBuf data, class IMetricConsumer* c, TInstant = TInstant::Zero()); +} diff --git a/library/cpp/monlib/encode/unistat/unistat_decoder.cpp b/library/cpp/monlib/encode/unistat/unistat_decoder.cpp new file mode 100644 index 0000000000..b2344b0905 --- /dev/null +++ b/library/cpp/monlib/encode/unistat/unistat_decoder.cpp @@ -0,0 +1,253 @@ +#include "unistat.h" + +#include <library/cpp/monlib/metrics/histogram_collector.h> +#include <library/cpp/monlib/metrics/labels.h> +#include <library/cpp/monlib/metrics/metric_type.h> +#include <library/cpp/monlib/metrics/metric_value.h> +#include <library/cpp/monlib/metrics/metric_consumer.h> + +#include <library/cpp/json/json_reader.h> + +#include <util/datetime/base.h> +#include <util/string/split.h> + +#include <contrib/libs/re2/re2/re2.h> + +using namespace NJson; + +const re2::RE2 NAME_RE{R"((?:[a-zA-Z0-9\.\-/@_]+_)+(?:[ad][vehmntx]{3}|summ|hgram|max))"}; + +namespace NMonitoring { + namespace { + bool IsNumber(const NJson::TJsonValue& j) { + switch (j.GetType()) { + case EJsonValueType::JSON_INTEGER: + case EJsonValueType::JSON_UINTEGER: + case EJsonValueType::JSON_DOUBLE: + return true; + + default: + return false; + } + } + + template <typename T> + T ExtractNumber(const TJsonValue& val) { + switch (val.GetType()) { + case EJsonValueType::JSON_INTEGER: + return static_cast<T>(val.GetInteger()); + case EJsonValueType::JSON_UINTEGER: + return static_cast<T>(val.GetUInteger()); + case EJsonValueType::JSON_DOUBLE: + return static_cast<T>(val.GetDouble()); + + default: + ythrow yexception() << "Expected number, but found " << val.GetType(); + } + } + + auto ExtractDouble = ExtractNumber<double>; + auto ExtractUi64 = ExtractNumber<ui64>; + + class THistogramBuilder { + public: + void Add(TBucketBound bound, TBucketValue value) { + /// XXX: yasm uses left-closed intervals, while in monlib we use right-closed ones, + /// so (-inf; 0) [0, 100) [100; +inf) + /// becomes (-inf; 0] (0, 100] (100; +inf) + /// but since we've already lost some information these no way to avoid this kind of error here + Bounds_.push_back(bound); + + /// this will always be 0 for the first bucket, + /// since there's no way to make (-inf; N) bucket in yasm + Values_.push_back(NextValue_); + + /// we will write this value into the next bucket so that [[0, 10], [100, 20], [200, 50]] + /// becomes (-inf; 0] -> 0; (0; 100] -> 10; (100; 200] -> 20; (200; +inf) -> 50 + NextValue_ = value; + } + + IHistogramSnapshotPtr Finalize() { + Bounds_.push_back(std::numeric_limits<TBucketBound>::max()); + Values_.push_back(NextValue_); + + Y_ENSURE(Bounds_.size() <= HISTOGRAM_MAX_BUCKETS_COUNT, + "Histogram is only allowed to have " << HISTOGRAM_MAX_BUCKETS_COUNT << " buckets, but has " << Bounds_.size()); + + return ExplicitHistogramSnapshot(Bounds_, Values_); + } + + public: + TBucketValue NextValue_ {0}; + TBucketBounds Bounds_; + TBucketValues Values_; + }; + + class TDecoderUnistat { + private: + public: + explicit TDecoderUnistat(IMetricConsumer* consumer, IInputStream* is, TInstant ts) + : Consumer_{consumer} + , Timestamp_{ts} { + ReadJsonTree(is, &Json_, /* throw */ true); + } + + void Decode() { + Y_ENSURE(Json_.IsArray(), "Expected array at the top level, but found " << Json_.GetType()); + + for (auto&& metric : Json_.GetArray()) { + Y_ENSURE(metric.IsArray(), "Metric must be an array"); + auto&& arr = metric.GetArray(); + Y_ENSURE(arr.size() == 2, "Metric must be an array of 2 elements"); + auto&& name = arr[0]; + auto&& value = arr[1]; + MetricContext_ = {}; + + ParseName(name.GetString()); + + if (value.IsArray()) { + OnHistogram(value); + } else if (IsNumber(value)) { + OnScalar(value); + } else { + ythrow yexception() << "Expected list or number, but found " << value.GetType(); + } + + WriteValue(); + } + } + + private: + void OnScalar(const TJsonValue& jsonValue) { + if (MetricContext_.IsDeriv) { + MetricContext_.Type = EMetricType::RATE; + MetricContext_.Value = TMetricValue{ExtractUi64(jsonValue)}; + } else { + MetricContext_.Type = EMetricType::GAUGE; + MetricContext_.Value = TMetricValue{ExtractDouble(jsonValue)}; + } + } + + void OnHistogram(const TJsonValue& jsonHist) { + if (MetricContext_.IsDeriv) { + MetricContext_.Type = EMetricType::HIST_RATE; + } else { + MetricContext_.Type = EMetricType::HIST; + } + + auto histogramBuilder = THistogramBuilder(); + + for (auto&& bucket : jsonHist.GetArray()) { + Y_ENSURE(bucket.IsArray(), "Expected an array, but found " << bucket.GetType()); + auto&& arr = bucket.GetArray(); + Y_ENSURE(arr.size() == 2, "Histogram bucket must be an array of 2 elements"); + const auto bound = ExtractDouble(arr[0]); + const auto weight = ExtractUi64(arr[1]); + histogramBuilder.Add(bound, weight); + } + + MetricContext_.Histogram = histogramBuilder.Finalize(); + MetricContext_.Value = TMetricValue{MetricContext_.Histogram.Get()}; + } + + bool IsDeriv(TStringBuf name) { + TStringBuf ignore, suffix; + name.RSplit('_', ignore, suffix); + + Y_ENSURE(suffix.size() >= 3 && suffix.size() <= 5, "Disallowed suffix value: " << suffix); + + if (suffix == TStringBuf("summ") || suffix == TStringBuf("hgram")) { + return true; + } else if (suffix == TStringBuf("max")) { + return false; + } + + return suffix[0] == 'd'; + } + + void ParseName(TStringBuf value) { + TVector<TStringBuf> parts; + StringSplitter(value).Split(';').SkipEmpty().Collect(&parts); + + Y_ENSURE(parts.size() >= 1 && parts.size() <= 16); + + TStringBuf name = parts.back(); + parts.pop_back(); + + Y_ENSURE(RE2::FullMatch(re2::StringPiece{name.data(), name.size()}, NAME_RE), + "Metric name " << name << " doesn't match regex " << NAME_RE.pattern()); + + MetricContext_.Name = name; + MetricContext_.IsDeriv = IsDeriv(MetricContext_.Name); + + for (auto tag : parts) { + TStringBuf n, v; + tag.Split('=', n, v); + Y_ENSURE(n && v, "Unexpected tag format in " << tag); + MetricContext_.Labels.Add(n, v); + } + } + + private: + void WriteValue() { + Consumer_->OnMetricBegin(MetricContext_.Type); + + Consumer_->OnLabelsBegin(); + Consumer_->OnLabel("sensor", TString{MetricContext_.Name}); + for (auto&& l : MetricContext_.Labels) { + Consumer_->OnLabel(l.Name(), l.Value()); + } + + Consumer_->OnLabelsEnd(); + + switch (MetricContext_.Type) { + case EMetricType::GAUGE: + Consumer_->OnDouble(Timestamp_, MetricContext_.Value.AsDouble()); + break; + case EMetricType::RATE: + Consumer_->OnUint64(Timestamp_, MetricContext_.Value.AsUint64()); + break; + case EMetricType::HIST: + case EMetricType::HIST_RATE: + Consumer_->OnHistogram(Timestamp_, MetricContext_.Value.AsHistogram()); + break; + case EMetricType::LOGHIST: + case EMetricType::DSUMMARY: + case EMetricType::IGAUGE: + case EMetricType::COUNTER: + case EMetricType::UNKNOWN: + ythrow yexception() << "Unexpected metric type: " << MetricContext_.Type; + } + + Consumer_->OnMetricEnd(); + } + + private: + IMetricConsumer* Consumer_; + NJson::TJsonValue Json_; + TInstant Timestamp_; + + struct { + TStringBuf Name; + EMetricType Type{EMetricType::UNKNOWN}; + TMetricValue Value; + bool IsDeriv{false}; + TLabels Labels; + IHistogramSnapshotPtr Histogram; + } MetricContext_; + }; + + } + + void DecodeUnistat(TStringBuf data, IMetricConsumer* c, TInstant ts) { + c->OnStreamBegin(); + DecodeUnistatToStream(data, c, ts); + c->OnStreamEnd(); + } + + void DecodeUnistatToStream(TStringBuf data, IMetricConsumer* c, TInstant ts) { + TMemoryInput in{data.data(), data.size()}; + TDecoderUnistat decoder(c, &in, ts); + decoder.Decode(); + } +} diff --git a/library/cpp/monlib/encode/unistat/unistat_ut.cpp b/library/cpp/monlib/encode/unistat/unistat_ut.cpp new file mode 100644 index 0000000000..dbbc238bf3 --- /dev/null +++ b/library/cpp/monlib/encode/unistat/unistat_ut.cpp @@ -0,0 +1,223 @@ +#include "unistat.h" + +#include <library/cpp/monlib/encode/protobuf/protobuf.h> +#include <library/cpp/monlib/metrics/labels.h> + +#include <library/cpp/testing/unittest/registar.h> + +using namespace NMonitoring; + +Y_UNIT_TEST_SUITE(TUnistatDecoderTest) { + Y_UNIT_TEST(ScalarMetric) { + constexpr auto input = TStringBuf(R"([["something_axxx", 42]])"); + + NProto::TMultiSamplesList samples; + auto encoder = EncoderProtobuf(&samples); + + DecodeUnistat(input, encoder.Get()); + + UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 1); + auto sample = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1); + UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1); + + auto label = sample.GetLabels(0); + auto point = sample.GetPoints(0); + UNIT_ASSERT_VALUES_EQUAL(point.GetFloat64(), 42.); + UNIT_ASSERT_VALUES_EQUAL(label.GetName(), "sensor"); + UNIT_ASSERT_VALUES_EQUAL(label.GetValue(), "something_axxx"); + } + + Y_UNIT_TEST(OverriddenTags) { + constexpr auto input = TStringBuf(R"([["ctype=foo;prj=bar;custom_tag=qwe;something_axxx", 42]])"); + + NProto::TMultiSamplesList samples; + auto encoder = EncoderProtobuf(&samples); + + DecodeUnistat(input, encoder.Get()); + + UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 1); + auto sample = samples.GetSamples(0); + UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1); + UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 4); + + const auto& labels = sample.GetLabels(); + TLabels actual; + for (auto&& l : labels) { + actual.Add(l.GetName(), l.GetValue()); + } + + TLabels expected{{"ctype", "foo"}, {"prj", "bar"}, {"custom_tag", "qwe"}, {"sensor", "something_axxx"}}; + + UNIT_ASSERT_VALUES_EQUAL(actual.size(), expected.size()); + for (auto&& l : actual) { + UNIT_ASSERT(expected.Extract(l.Name())->Value() == l.Value()); + } + } + + Y_UNIT_TEST(ThrowsOnTopLevelObject) { + constexpr auto input = TStringBuf(R"({["something_axxx", 42]})"); + + NProto::TMultiSamplesList samples; + auto encoder = EncoderProtobuf(&samples); + + UNIT_ASSERT_EXCEPTION(DecodeUnistat(input, encoder.Get()), yexception); + } + + Y_UNIT_TEST(ThrowsOnUnwrappedMetric) { + constexpr auto input = TStringBuf(R"(["something_axxx", 42])"); + + NProto::TMultiSamplesList samples; + auto encoder = EncoderProtobuf(&samples); + + UNIT_ASSERT_EXCEPTION(DecodeUnistat(input, encoder.Get()), yexception); + } + + Y_UNIT_TEST(HistogramMetric) { + constexpr auto input = TStringBuf(R"([["something_hgram", [[0, 1], [200, 2], [500, 3]] ]])"); + + NProto::TMultiSamplesList samples; + auto encoder = EncoderProtobuf(&samples); + + DecodeUnistat(input, encoder.Get()); + + auto sample = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::HIST_RATE); + UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1); + UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1); + + auto label = sample.GetLabels(0); + const auto point = sample.GetPoints(0); + const auto histogram = point.GetHistogram(); + const auto size = histogram.BoundsSize(); + UNIT_ASSERT_VALUES_EQUAL(size, 4); + + const TVector<double> expectedBounds {0, 200, 500, std::numeric_limits<double>::max()}; + const TVector<ui64> expectedValues {0, 1, 2, 3}; + + for (auto i = 0; i < 4; ++i) { + UNIT_ASSERT_VALUES_EQUAL(histogram.GetBounds(i), expectedBounds[i]); + UNIT_ASSERT_VALUES_EQUAL(histogram.GetValues(i), expectedValues[i]); + } + + UNIT_ASSERT_VALUES_EQUAL(label.GetName(), "sensor"); + UNIT_ASSERT_VALUES_EQUAL(label.GetValue(), "something_hgram"); + } + + Y_UNIT_TEST(AbsoluteHistogram) { + constexpr auto input = TStringBuf(R"([["something_ahhh", [[0, 1], [200, 2], [500, 3]] ]])"); + + NProto::TMultiSamplesList samples; + auto encoder = EncoderProtobuf(&samples); + + DecodeUnistat(input, encoder.Get()); + + auto sample = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::HISTOGRAM); + UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1); + UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1); + } + + Y_UNIT_TEST(AllowedMetricNames) { + NProto::TMultiSamplesList samples; + auto encoder = EncoderProtobuf(&samples); + + { + constexpr auto input = TStringBuf(R"([["a/A-b/c_D/__G_dmmm", [[0, 1], [200, 2], [500, 3]] ]])"); + UNIT_ASSERT_NO_EXCEPTION(DecodeUnistat(input, encoder.Get())); + } + } + + Y_UNIT_TEST(DisallowedMetricNames) { + NProto::TMultiSamplesList samples; + auto encoder = EncoderProtobuf(&samples); + + { + constexpr auto input = TStringBuf(R"([["someth!ng_ahhh", [[0, 1], [200, 2], [500, 3]] ]])"); + UNIT_ASSERT_EXCEPTION(DecodeUnistat(input, encoder.Get()), yexception); + } + + { + constexpr auto input = TStringBuf(R"([["foo_a", [[0, 1], [200, 2], [500, 3]] ]])"); + UNIT_ASSERT_EXCEPTION(DecodeUnistat(input, encoder.Get()), yexception); + } + + { + constexpr auto input = TStringBuf(R"([["foo_ahhh;tag=value", [[0, 1], [200, 2], [500, 3]] ]])"); + UNIT_ASSERT_EXCEPTION(DecodeUnistat(input, encoder.Get()), yexception); + } + } + + Y_UNIT_TEST(MultipleMetrics) { + constexpr auto input = TStringBuf(R"([["something_axxx", 42], ["some-other_dhhh", 53]])"); + + NProto::TMultiSamplesList samples; + auto encoder = EncoderProtobuf(&samples); + + DecodeUnistat(input, encoder.Get()); + + UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 2); + auto sample = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1); + UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1); + + auto label = sample.GetLabels(0); + auto point = sample.GetPoints(0); + UNIT_ASSERT_VALUES_EQUAL(point.GetFloat64(), 42.); + UNIT_ASSERT_VALUES_EQUAL(label.GetName(), "sensor"); + UNIT_ASSERT_VALUES_EQUAL(label.GetValue(), "something_axxx"); + + sample = samples.GetSamples(1); + UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::RATE); + UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1); + UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1); + + label = sample.GetLabels(0); + point = sample.GetPoints(0); + UNIT_ASSERT_VALUES_EQUAL(point.GetUint64(), 53); + UNIT_ASSERT_VALUES_EQUAL(label.GetName(), "sensor"); + UNIT_ASSERT_VALUES_EQUAL(label.GetValue(), "some-other_dhhh"); + } + + Y_UNIT_TEST(UnderscoreName) { + constexpr auto input = TStringBuf(R"([["something_anything_dmmm", 42]])"); + + NProto::TMultiSamplesList samples; + auto encoder = EncoderProtobuf(&samples); + DecodeUnistat(input, encoder.Get()); + + UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 1); + auto sample = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::RATE); + UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1); + UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1); + + auto label = sample.GetLabels(0); + auto point = sample.GetPoints(0); + UNIT_ASSERT_VALUES_EQUAL(point.GetUint64(), 42); + UNIT_ASSERT_VALUES_EQUAL(label.GetName(), "sensor"); + UNIT_ASSERT_VALUES_EQUAL(label.GetValue(), "something_anything_dmmm"); + } + + Y_UNIT_TEST(MaxAggr) { + constexpr auto input = TStringBuf(R"([["something_anything_max", 42]])"); + + NProto::TMultiSamplesList samples; + auto encoder = EncoderProtobuf(&samples); + DecodeUnistat(input, encoder.Get()); + + UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 1); + auto sample = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1); + UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1); + + auto label = sample.GetLabels(0); + auto point = sample.GetPoints(0); + UNIT_ASSERT_VALUES_EQUAL(point.GetFloat64(), 42.); + UNIT_ASSERT_VALUES_EQUAL(label.GetName(), "sensor"); + UNIT_ASSERT_VALUES_EQUAL(label.GetValue(), "something_anything_max"); + } +} diff --git a/library/cpp/monlib/encode/unistat/ut/ya.make b/library/cpp/monlib/encode/unistat/ut/ya.make new file mode 100644 index 0000000000..a652139f45 --- /dev/null +++ b/library/cpp/monlib/encode/unistat/ut/ya.make @@ -0,0 +1,16 @@ +UNITTEST_FOR(library/cpp/monlib/encode/unistat) + +OWNER( + msherbakov + g:solomon +) + +SRCS( + unistat_ut.cpp +) + +PEERDIR( + library/cpp/monlib/encode/protobuf +) + +END() diff --git a/library/cpp/monlib/encode/unistat/ya.make b/library/cpp/monlib/encode/unistat/ya.make new file mode 100644 index 0000000000..4ac2edadf4 --- /dev/null +++ b/library/cpp/monlib/encode/unistat/ya.make @@ -0,0 +1,18 @@ +LIBRARY() + +OWNER( + msherbakov + g:solomon +) + +PEERDIR( + contrib/libs/re2 + library/cpp/json + library/cpp/monlib/metrics +) + +SRCS( + unistat_decoder.cpp +) + +END() diff --git a/library/cpp/monlib/encode/ut/ya.make b/library/cpp/monlib/encode/ut/ya.make new file mode 100644 index 0000000000..1990386d76 --- /dev/null +++ b/library/cpp/monlib/encode/ut/ya.make @@ -0,0 +1,12 @@ +UNITTEST_FOR(library/cpp/monlib/encode) + +OWNER( + jamel + g:solomon +) + +SRCS( + format_ut.cpp +) + +END() diff --git a/library/cpp/monlib/encode/ya.make b/library/cpp/monlib/encode/ya.make new file mode 100644 index 0000000000..d1bb09f07b --- /dev/null +++ b/library/cpp/monlib/encode/ya.make @@ -0,0 +1,20 @@ +LIBRARY() + +OWNER( + g:solomon + jamel +) + +SRCS( + encoder.cpp + encoder_state.cpp + format.cpp +) + +PEERDIR( + library/cpp/monlib/metrics +) + +GENERATE_ENUM_SERIALIZATION_WITH_HEADER(encoder_state_enum.h) + +END() |