aboutsummaryrefslogtreecommitdiffstats
path: root/library/cpp/monlib/encode
diff options
context:
space:
mode:
authorDevtools Arcadia <arcadia-devtools@yandex-team.ru>2022-02-07 18:08:42 +0300
committerDevtools Arcadia <arcadia-devtools@mous.vla.yp-c.yandex.net>2022-02-07 18:08:42 +0300
commit1110808a9d39d4b808aef724c861a2e1a38d2a69 (patch)
treee26c9fed0de5d9873cce7e00bc214573dc2195b7 /library/cpp/monlib/encode
downloadydb-1110808a9d39d4b808aef724c861a2e1a38d2a69.tar.gz
intermediate changes
ref:cde9a383711a11544ce7e107a78147fb96cc4029
Diffstat (limited to 'library/cpp/monlib/encode')
-rw-r--r--library/cpp/monlib/encode/buffered/buffered_encoder_base.cpp170
-rw-r--r--library/cpp/monlib/encode/buffered/buffered_encoder_base.h100
-rw-r--r--library/cpp/monlib/encode/buffered/string_pool.cpp58
-rw-r--r--library/cpp/monlib/encode/buffered/string_pool.h92
-rw-r--r--library/cpp/monlib/encode/buffered/string_pool_ut.cpp84
-rw-r--r--library/cpp/monlib/encode/buffered/ut/ya.make12
-rw-r--r--library/cpp/monlib/encode/buffered/ya.make19
-rw-r--r--library/cpp/monlib/encode/encoder.cpp6
-rw-r--r--library/cpp/monlib/encode/encoder.h17
-rw-r--r--library/cpp/monlib/encode/encoder_state.cpp1
-rw-r--r--library/cpp/monlib/encode/encoder_state.h62
-rw-r--r--library/cpp/monlib/encode/encoder_state_enum.h12
-rw-r--r--library/cpp/monlib/encode/fake/fake.cpp51
-rw-r--r--library/cpp/monlib/encode/fake/fake.h10
-rw-r--r--library/cpp/monlib/encode/fake/ya.make12
-rw-r--r--library/cpp/monlib/encode/format.cpp202
-rw-r--r--library/cpp/monlib/encode/format.h166
-rw-r--r--library/cpp/monlib/encode/format_ut.cpp136
-rw-r--r--library/cpp/monlib/encode/fuzz/ya.make5
-rw-r--r--library/cpp/monlib/encode/json/fuzz/main.cpp16
-rw-r--r--library/cpp/monlib/encode/json/fuzz/ya.make19
-rw-r--r--library/cpp/monlib/encode/json/json.h29
-rw-r--r--library/cpp/monlib/encode/json/json_decoder.cpp1162
-rw-r--r--library/cpp/monlib/encode/json/json_decoder_ut.cpp179
-rw-r--r--library/cpp/monlib/encode/json/json_encoder.cpp556
-rw-r--r--library/cpp/monlib/encode/json/json_ut.cpp1290
-rw-r--r--library/cpp/monlib/encode/json/typed_point.h123
-rw-r--r--library/cpp/monlib/encode/json/ut/buffered_test.json36
-rw-r--r--library/cpp/monlib/encode/json/ut/buffered_ts_merge.json13
-rw-r--r--library/cpp/monlib/encode/json/ut/crash.jsonbin0 -> 655 bytes
-rw-r--r--library/cpp/monlib/encode/json/ut/empty_series.json12
-rw-r--r--library/cpp/monlib/encode/json/ut/expected.json92
-rw-r--r--library/cpp/monlib/encode/json/ut/expected_buffered.json92
-rw-r--r--library/cpp/monlib/encode/json/ut/expected_cloud.json92
-rw-r--r--library/cpp/monlib/encode/json/ut/expected_cloud_buffered.json92
-rw-r--r--library/cpp/monlib/encode/json/ut/hist_crash.jsonbin0 -> 213 bytes
-rw-r--r--library/cpp/monlib/encode/json/ut/histogram_timeseries.json61
-rw-r--r--library/cpp/monlib/encode/json/ut/histogram_value.json33
-rw-r--r--library/cpp/monlib/encode/json/ut/histogram_value_inf_before_bounds.json33
-rw-r--r--library/cpp/monlib/encode/json/ut/int_gauge.json31
-rw-r--r--library/cpp/monlib/encode/json/ut/log_histogram_timeseries.json47
-rw-r--r--library/cpp/monlib/encode/json/ut/log_histogram_value.json26
-rw-r--r--library/cpp/monlib/encode/json/ut/merged.json14
-rw-r--r--library/cpp/monlib/encode/json/ut/metrics.json43
-rw-r--r--library/cpp/monlib/encode/json/ut/named_metrics.json22
-rw-r--r--library/cpp/monlib/encode/json/ut/sensors.json40
-rw-r--r--library/cpp/monlib/encode/json/ut/summary_inf.json21
-rw-r--r--library/cpp/monlib/encode/json/ut/summary_timeseries.json37
-rw-r--r--library/cpp/monlib/encode/json/ut/summary_value.json21
-rw-r--r--library/cpp/monlib/encode/json/ut/test_decode_to_encode.json16
-rw-r--r--library/cpp/monlib/encode/json/ut/ya.make46
-rw-r--r--library/cpp/monlib/encode/json/ya.make21
-rw-r--r--library/cpp/monlib/encode/legacy_protobuf/legacy_proto_decoder.cpp527
-rw-r--r--library/cpp/monlib/encode/legacy_protobuf/legacy_protobuf.h16
-rw-r--r--library/cpp/monlib/encode/legacy_protobuf/legacy_protobuf_ut.cpp422
-rw-r--r--library/cpp/monlib/encode/legacy_protobuf/protos/metric_meta.proto73
-rw-r--r--library/cpp/monlib/encode/legacy_protobuf/protos/python/ya.make3
-rw-r--r--library/cpp/monlib/encode/legacy_protobuf/protos/ya.make13
-rw-r--r--library/cpp/monlib/encode/legacy_protobuf/ut/test_cases.proto90
-rw-r--r--library/cpp/monlib/encode/legacy_protobuf/ut/ya.make18
-rw-r--r--library/cpp/monlib/encode/legacy_protobuf/ya.make16
-rw-r--r--library/cpp/monlib/encode/prometheus/fuzz/main.cpp18
-rw-r--r--library/cpp/monlib/encode/prometheus/fuzz/ya.make16
-rw-r--r--library/cpp/monlib/encode/prometheus/prometheus.h18
-rw-r--r--library/cpp/monlib/encode/prometheus/prometheus_decoder.cpp597
-rw-r--r--library/cpp/monlib/encode/prometheus/prometheus_decoder_ut.cpp478
-rw-r--r--library/cpp/monlib/encode/prometheus/prometheus_encoder.cpp413
-rw-r--r--library/cpp/monlib/encode/prometheus/prometheus_encoder_ut.cpp414
-rw-r--r--library/cpp/monlib/encode/prometheus/prometheus_model.h70
-rw-r--r--library/cpp/monlib/encode/prometheus/ut/ya.make17
-rw-r--r--library/cpp/monlib/encode/prometheus/ya.make17
-rw-r--r--library/cpp/monlib/encode/protobuf/protobuf.h16
-rw-r--r--library/cpp/monlib/encode/protobuf/protobuf_encoder.cpp248
-rw-r--r--library/cpp/monlib/encode/protobuf/protos/samples.proto91
-rw-r--r--library/cpp/monlib/encode/protobuf/protos/ya.make14
-rw-r--r--library/cpp/monlib/encode/protobuf/ya.make17
-rw-r--r--library/cpp/monlib/encode/spack/compression.cpp383
-rw-r--r--library/cpp/monlib/encode/spack/compression.h19
-rw-r--r--library/cpp/monlib/encode/spack/fuzz/main.cpp20
-rw-r--r--library/cpp/monlib/encode/spack/fuzz/ya.make21
-rw-r--r--library/cpp/monlib/encode/spack/spack_v1.h115
-rw-r--r--library/cpp/monlib/encode/spack/spack_v1_decoder.cpp458
-rw-r--r--library/cpp/monlib/encode/spack/spack_v1_encoder.cpp318
-rw-r--r--library/cpp/monlib/encode/spack/spack_v1_ut.cpp845
-rw-r--r--library/cpp/monlib/encode/spack/ut/ya.make16
-rw-r--r--library/cpp/monlib/encode/spack/varint.cpp79
-rw-r--r--library/cpp/monlib/encode/spack/varint.h23
-rw-r--r--library/cpp/monlib/encode/spack/ya.make25
-rw-r--r--library/cpp/monlib/encode/text/text.h9
-rw-r--r--library/cpp/monlib/encode/text/text_encoder.cpp226
-rw-r--r--library/cpp/monlib/encode/text/text_encoder_ut.cpp283
-rw-r--r--library/cpp/monlib/encode/text/ut/ya.make12
-rw-r--r--library/cpp/monlib/encode/text/ya.make16
-rw-r--r--library/cpp/monlib/encode/unistat/unistat.h13
-rw-r--r--library/cpp/monlib/encode/unistat/unistat_decoder.cpp253
-rw-r--r--library/cpp/monlib/encode/unistat/unistat_ut.cpp223
-rw-r--r--library/cpp/monlib/encode/unistat/ut/ya.make16
-rw-r--r--library/cpp/monlib/encode/unistat/ya.make18
-rw-r--r--library/cpp/monlib/encode/ut/ya.make12
-rw-r--r--library/cpp/monlib/encode/ya.make20
100 files changed, 12575 insertions, 0 deletions
diff --git a/library/cpp/monlib/encode/buffered/buffered_encoder_base.cpp b/library/cpp/monlib/encode/buffered/buffered_encoder_base.cpp
new file mode 100644
index 0000000000..87c832d642
--- /dev/null
+++ b/library/cpp/monlib/encode/buffered/buffered_encoder_base.cpp
@@ -0,0 +1,170 @@
+#include "buffered_encoder_base.h"
+
+#include <util/string/join.h>
+#include <util/string/builder.h>
+
+namespace NMonitoring {
+
+void TBufferedEncoderBase::OnStreamBegin() {
+ State_.Expect(TEncoderState::EState::ROOT);
+}
+
+void TBufferedEncoderBase::OnStreamEnd() {
+ State_.Expect(TEncoderState::EState::ROOT);
+}
+
+void TBufferedEncoderBase::OnCommonTime(TInstant time) {
+ State_.Expect(TEncoderState::EState::ROOT);
+ CommonTime_ = time;
+}
+
+void TBufferedEncoderBase::OnMetricBegin(EMetricType type) {
+ State_.Switch(TEncoderState::EState::ROOT, TEncoderState::EState::METRIC);
+ Metrics_.emplace_back();
+ Metrics_.back().MetricType = type;
+}
+
+void TBufferedEncoderBase::OnMetricEnd() {
+ State_.Switch(TEncoderState::EState::METRIC, TEncoderState::EState::ROOT);
+
+ switch (MetricsMergingMode_) {
+ case EMetricsMergingMode::MERGE_METRICS: {
+ auto& metric = Metrics_.back();
+ Sort(metric.Labels, [] (const TPooledLabel& lhs, const TPooledLabel& rhs) {
+ return std::tie(lhs.Key, lhs.Value) < std::tie(rhs.Key, rhs.Value);
+ });
+
+ auto it = MetricMap_.find(metric.Labels);
+ if (it == std::end(MetricMap_)) {
+ MetricMap_.emplace(metric.Labels, Metrics_.size() - 1);
+ } else {
+ auto& existing = Metrics_[it->second].TimeSeries;
+
+ Y_ENSURE(existing.GetValueType() == metric.TimeSeries.GetValueType(),
+ "Time series point type mismatch: expected " << existing.GetValueType()
+ << " but found " << metric.TimeSeries.GetValueType()
+ << ", labels '" << FormatLabels(metric.Labels) << "'");
+
+ existing.CopyFrom(metric.TimeSeries);
+ Metrics_.pop_back();
+ }
+
+ break;
+ }
+ case EMetricsMergingMode::DEFAULT:
+ break;
+ }
+}
+
+void TBufferedEncoderBase::OnLabelsBegin() {
+ if (State_ == TEncoderState::EState::METRIC) {
+ State_ = TEncoderState::EState::METRIC_LABELS;
+ } else if (State_ == TEncoderState::EState::ROOT) {
+ State_ = TEncoderState::EState::COMMON_LABELS;
+ } else {
+ State_.ThrowInvalid("expected METRIC or ROOT");
+ }
+}
+
+void TBufferedEncoderBase::OnLabelsEnd() {
+ if (State_ == TEncoderState::EState::METRIC_LABELS) {
+ State_ = TEncoderState::EState::METRIC;
+ } else if (State_ == TEncoderState::EState::COMMON_LABELS) {
+ State_ = TEncoderState::EState::ROOT;
+ } else {
+ State_.ThrowInvalid("expected LABELS or COMMON_LABELS");
+ }
+}
+
+void TBufferedEncoderBase::OnLabel(TStringBuf name, TStringBuf value) {
+ TPooledLabels* labels;
+ if (State_ == TEncoderState::EState::METRIC_LABELS) {
+ labels = &Metrics_.back().Labels;
+ } else if (State_ == TEncoderState::EState::COMMON_LABELS) {
+ labels = &CommonLabels_;
+ } else {
+ State_.ThrowInvalid("expected LABELS or COMMON_LABELS");
+ }
+
+ labels->emplace_back(LabelNamesPool_.PutIfAbsent(name), LabelValuesPool_.PutIfAbsent(value));
+}
+
+void TBufferedEncoderBase::OnLabel(ui32 name, ui32 value) {
+ TPooledLabels* labels;
+ if (State_ == TEncoderState::EState::METRIC_LABELS) {
+ labels = &Metrics_.back().Labels;
+ } else if (State_ == TEncoderState::EState::COMMON_LABELS) {
+ labels = &CommonLabels_;
+ } else {
+ State_.ThrowInvalid("expected LABELS or COMMON_LABELS");
+ }
+
+ labels->emplace_back(LabelNamesPool_.GetByIndex(name), LabelValuesPool_.GetByIndex(value));
+}
+
+std::pair<ui32, ui32> TBufferedEncoderBase::PrepareLabel(TStringBuf name, TStringBuf value) {
+ auto nameLabel = LabelNamesPool_.PutIfAbsent(name);
+ auto valueLabel = LabelValuesPool_.PutIfAbsent(value);
+ return std::make_pair(nameLabel->Index, valueLabel->Index);
+}
+
+void TBufferedEncoderBase::OnDouble(TInstant time, double value) {
+ State_.Expect(TEncoderState::EState::METRIC);
+ TMetric& metric = Metrics_.back();
+ metric.TimeSeries.Add(time, value);
+}
+
+void TBufferedEncoderBase::OnInt64(TInstant time, i64 value) {
+ State_.Expect(TEncoderState::EState::METRIC);
+ TMetric& metric = Metrics_.back();
+ metric.TimeSeries.Add(time, value);
+}
+
+void TBufferedEncoderBase::OnUint64(TInstant time, ui64 value) {
+ State_.Expect(TEncoderState::EState::METRIC);
+ TMetric& metric = Metrics_.back();
+ metric.TimeSeries.Add(time, value);
+}
+
+void TBufferedEncoderBase::OnHistogram(TInstant time, IHistogramSnapshotPtr s) {
+ State_.Expect(TEncoderState::EState::METRIC);
+ TMetric& metric = Metrics_.back();
+ metric.TimeSeries.Add(time, s.Get());
+}
+
+void TBufferedEncoderBase::OnSummaryDouble(TInstant time, ISummaryDoubleSnapshotPtr s) {
+ State_.Expect(TEncoderState::EState::METRIC);
+ TMetric& metric = Metrics_.back();
+ metric.TimeSeries.Add(time, s.Get());
+}
+
+void TBufferedEncoderBase::OnLogHistogram(TInstant time, TLogHistogramSnapshotPtr s) {
+ State_.Expect(TEncoderState::EState::METRIC);
+ TMetric& metric = Metrics_.back();
+ metric.TimeSeries.Add(time, s.Get());
+}
+
+TString TBufferedEncoderBase::FormatLabels(const TPooledLabels& labels) const {
+ auto formattedLabels = TVector<TString>(Reserve(labels.size() + CommonLabels_.size()));
+ auto addLabel = [&](const TPooledLabel& l) {
+ auto formattedLabel = TStringBuilder() << LabelNamesPool_.Get(l.Key) << '=' << LabelValuesPool_.Get(l.Value);
+ formattedLabels.push_back(std::move(formattedLabel));
+ };
+
+ for (const auto& l: labels) {
+ addLabel(l);
+ }
+ for (const auto& l: CommonLabels_) {
+ const auto it = FindIf(labels, [&](const TPooledLabel& label) {
+ return label.Key == l.Key;
+ });
+ if (it == labels.end()) {
+ addLabel(l);
+ }
+ }
+ Sort(formattedLabels);
+
+ return TStringBuilder() << "{" << JoinSeq(", ", formattedLabels) << "}";
+}
+
+} // namespace NMonitoring
diff --git a/library/cpp/monlib/encode/buffered/buffered_encoder_base.h b/library/cpp/monlib/encode/buffered/buffered_encoder_base.h
new file mode 100644
index 0000000000..fe3714e58f
--- /dev/null
+++ b/library/cpp/monlib/encode/buffered/buffered_encoder_base.h
@@ -0,0 +1,100 @@
+#pragma once
+
+#include "string_pool.h"
+
+#include <library/cpp/monlib/encode/encoder.h>
+#include <library/cpp/monlib/encode/encoder_state.h>
+#include <library/cpp/monlib/encode/format.h>
+#include <library/cpp/monlib/metrics/metric_value.h>
+
+#include <util/datetime/base.h>
+#include <util/digest/numeric.h>
+
+
+namespace NMonitoring {
+
+class TBufferedEncoderBase : public IMetricEncoder {
+public:
+ void OnStreamBegin() override;
+ void OnStreamEnd() override;
+
+ void OnCommonTime(TInstant time) override;
+
+ void OnMetricBegin(EMetricType type) override;
+ void OnMetricEnd() override;
+
+ void OnLabelsBegin() override;
+ void OnLabelsEnd() override;
+ void OnLabel(TStringBuf name, TStringBuf value) override;
+ void OnLabel(ui32 name, ui32 value) override;
+ std::pair<ui32, ui32> PrepareLabel(TStringBuf name, TStringBuf value) override;
+
+ void OnDouble(TInstant time, double value) override;
+ void OnInt64(TInstant time, i64 value) override;
+ void OnUint64(TInstant time, ui64 value) override;
+
+ void OnHistogram(TInstant time, IHistogramSnapshotPtr snapshot) override;
+ void OnSummaryDouble(TInstant time, ISummaryDoubleSnapshotPtr snapshot) override;
+ void OnLogHistogram(TInstant, TLogHistogramSnapshotPtr) override;
+
+protected:
+ using TPooledStr = TStringPoolBuilder::TValue;
+
+ struct TPooledLabel {
+ TPooledLabel(const TPooledStr* key, const TPooledStr* value)
+ : Key{key}
+ , Value{value}
+ {
+ }
+
+ bool operator==(const TPooledLabel& other) const {
+ return std::tie(Key, Value) == std::tie(other.Key, other.Value);
+ }
+
+ bool operator!=(const TPooledLabel& other) const {
+ return !(*this == other);
+ }
+
+ const TPooledStr* Key;
+ const TPooledStr* Value;
+ };
+
+ using TPooledLabels = TVector<TPooledLabel>;
+
+ struct TPooledLabelsHash {
+ size_t operator()(const TPooledLabels& val) const {
+ size_t hash{0};
+
+ for (auto v : val) {
+ hash = CombineHashes<size_t>(hash, reinterpret_cast<size_t>(v.Key));
+ hash = CombineHashes<size_t>(hash, reinterpret_cast<size_t>(v.Value));
+ }
+
+ return hash;
+ }
+ };
+
+ using TMetricMap = THashMap<TPooledLabels, size_t, TPooledLabelsHash>;
+
+ struct TMetric {
+ EMetricType MetricType = EMetricType::UNKNOWN;
+ TPooledLabels Labels;
+ TMetricTimeSeries TimeSeries;
+ };
+
+protected:
+ TString FormatLabels(const TPooledLabels& labels) const;
+
+protected:
+ TEncoderState State_;
+
+ TStringPoolBuilder LabelNamesPool_;
+ TStringPoolBuilder LabelValuesPool_;
+ TInstant CommonTime_ = TInstant::Zero();
+ TPooledLabels CommonLabels_;
+ TVector<TMetric> Metrics_;
+ TMetricMap MetricMap_;
+ EMetricsMergingMode MetricsMergingMode_ = EMetricsMergingMode::DEFAULT;
+};
+
+}
diff --git a/library/cpp/monlib/encode/buffered/string_pool.cpp b/library/cpp/monlib/encode/buffered/string_pool.cpp
new file mode 100644
index 0000000000..b4c7988ba3
--- /dev/null
+++ b/library/cpp/monlib/encode/buffered/string_pool.cpp
@@ -0,0 +1,58 @@
+#include "string_pool.h"
+
+namespace NMonitoring {
+ ////////////////////////////////////////////////////////////////////////////////
+ // TStringPoolBuilder
+ ////////////////////////////////////////////////////////////////////////////////
+ const TStringPoolBuilder::TValue* TStringPoolBuilder::PutIfAbsent(TStringBuf str) {
+ Y_ENSURE(!IsBuilt_, "Cannot add more values after string has been built");
+
+ auto [it, isInserted] = StrMap_.try_emplace(str, Max<ui32>(), 0);
+ if (isInserted) {
+ BytesSize_ += str.size();
+ it->second.Index = StrVector_.size();
+ StrVector_.emplace_back(it->first, &it->second);
+ }
+
+ TValue* value = &it->second;
+ ++value->Frequency;
+ return value;
+ }
+
+ const TStringPoolBuilder::TValue* TStringPoolBuilder::GetByIndex(ui32 index) const {
+ return StrVector_.at(index).second;
+ }
+
+ TStringPoolBuilder& TStringPoolBuilder::Build() {
+ if (RequiresSorting_) {
+ // sort in reversed order
+ std::sort(StrVector_.begin(), StrVector_.end(), [](auto& a, auto& b) {
+ return a.second->Frequency > b.second->Frequency;
+ });
+
+ ui32 i = 0;
+ for (auto& value : StrVector_) {
+ value.second->Index = i++;
+ }
+ }
+
+ IsBuilt_ = true;
+
+ return *this;
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ // TStringPool
+ ////////////////////////////////////////////////////////////////////////////////
+ void TStringPool::InitIndex(const char* data, ui32 size) {
+ const char* begin = data;
+ const char* end = begin + size;
+ for (const char* p = begin; p != end; ++p) {
+ if (*p == '\0') {
+ Index_.push_back(TStringBuf(begin, p));
+ begin = p + 1;
+ }
+ }
+ }
+
+}
diff --git a/library/cpp/monlib/encode/buffered/string_pool.h b/library/cpp/monlib/encode/buffered/string_pool.h
new file mode 100644
index 0000000000..00e5644608
--- /dev/null
+++ b/library/cpp/monlib/encode/buffered/string_pool.h
@@ -0,0 +1,92 @@
+#pragma once
+
+#include <util/generic/hash.h>
+#include <util/generic/vector.h>
+
+namespace NMonitoring {
+ ////////////////////////////////////////////////////////////////////////////////
+ // TStringPoolBuilder
+ ////////////////////////////////////////////////////////////////////////////////
+ class TStringPoolBuilder {
+ public:
+ struct TValue: TNonCopyable {
+ TValue(ui32 idx, ui32 freq)
+ : Index{idx}
+ , Frequency{freq}
+ {
+ }
+
+ ui32 Index;
+ ui32 Frequency;
+ };
+
+ public:
+ const TValue* PutIfAbsent(TStringBuf str);
+ const TValue* GetByIndex(ui32 index) const;
+
+ /// Determines whether pool must be sorted by value frequencies
+ TStringPoolBuilder& SetSorted(bool sorted) {
+ RequiresSorting_ = sorted;
+ return *this;
+ }
+
+ TStringPoolBuilder& Build();
+
+ TStringBuf Get(ui32 index) const {
+ Y_ENSURE(IsBuilt_, "Pool must be sorted first");
+ return StrVector_.at(index).first;
+ }
+
+ TStringBuf Get(const TValue* value) const {
+ return StrVector_.at(value->Index).first;
+ }
+
+ template <typename TConsumer>
+ void ForEach(TConsumer&& c) {
+ Y_ENSURE(IsBuilt_, "Pool must be sorted first");
+ for (const auto& value : StrVector_) {
+ c(value.first, value.second->Index, value.second->Frequency);
+ }
+ }
+
+ size_t BytesSize() const noexcept {
+ return BytesSize_;
+ }
+
+ size_t Count() const noexcept {
+ return StrMap_.size();
+ }
+
+ private:
+ THashMap<TString, TValue> StrMap_;
+ TVector<std::pair<TStringBuf, TValue*>> StrVector_;
+ bool RequiresSorting_ = false;
+ bool IsBuilt_ = false;
+ size_t BytesSize_ = 0;
+ };
+
+ ////////////////////////////////////////////////////////////////////////////////
+ // TStringPool
+ ////////////////////////////////////////////////////////////////////////////////
+ class TStringPool {
+ public:
+ TStringPool(const char* data, ui32 size) {
+ InitIndex(data, size);
+ }
+
+ TStringBuf Get(ui32 i) const {
+ return Index_.at(i);
+ }
+
+ size_t Size() const {
+ return Index_.size();
+ }
+
+ private:
+ void InitIndex(const char* data, ui32 size);
+
+ private:
+ TVector<TStringBuf> Index_;
+ };
+
+}
diff --git a/library/cpp/monlib/encode/buffered/string_pool_ut.cpp b/library/cpp/monlib/encode/buffered/string_pool_ut.cpp
new file mode 100644
index 0000000000..9fc3421d0b
--- /dev/null
+++ b/library/cpp/monlib/encode/buffered/string_pool_ut.cpp
@@ -0,0 +1,84 @@
+#include "string_pool.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+
+using namespace NMonitoring;
+
+Y_UNIT_TEST_SUITE(TStringPoolTest) {
+ Y_UNIT_TEST(PutIfAbsent) {
+ TStringPoolBuilder strPool;
+ strPool.SetSorted(true);
+
+ auto* h1 = strPool.PutIfAbsent("one");
+ auto* h2 = strPool.PutIfAbsent("two");
+ auto* h3 = strPool.PutIfAbsent("two");
+ UNIT_ASSERT(h1 != h2);
+ UNIT_ASSERT(h2 == h3);
+
+ UNIT_ASSERT_VALUES_EQUAL(h1->Frequency, 1);
+ UNIT_ASSERT_VALUES_EQUAL(h1->Index, 0);
+
+ UNIT_ASSERT_VALUES_EQUAL(h2->Frequency, 2);
+ UNIT_ASSERT_VALUES_EQUAL(h2->Index, 1);
+
+ UNIT_ASSERT_VALUES_EQUAL(strPool.BytesSize(), 6);
+ UNIT_ASSERT_VALUES_EQUAL(strPool.Count(), 2);
+ }
+
+ Y_UNIT_TEST(SortByFrequency) {
+ TStringPoolBuilder strPool;
+ strPool.SetSorted(true);
+
+ auto* h1 = strPool.PutIfAbsent("one");
+ auto* h2 = strPool.PutIfAbsent("two");
+ auto* h3 = strPool.PutIfAbsent("two");
+ UNIT_ASSERT(h1 != h2);
+ UNIT_ASSERT(h2 == h3);
+
+ strPool.Build();
+
+ UNIT_ASSERT_VALUES_EQUAL(h1->Frequency, 1);
+ UNIT_ASSERT_VALUES_EQUAL(h1->Index, 1);
+
+ UNIT_ASSERT_VALUES_EQUAL(h2->Frequency, 2);
+ UNIT_ASSERT_VALUES_EQUAL(h2->Index, 0);
+
+ UNIT_ASSERT_VALUES_EQUAL(strPool.BytesSize(), 6);
+ UNIT_ASSERT_VALUES_EQUAL(strPool.Count(), 2);
+ }
+
+ Y_UNIT_TEST(ForEach) {
+ TStringPoolBuilder strPool;
+ strPool.SetSorted(true);
+
+ strPool.PutIfAbsent("one");
+ strPool.PutIfAbsent("two");
+ strPool.PutIfAbsent("two");
+ strPool.PutIfAbsent("three");
+ strPool.PutIfAbsent("three");
+ strPool.PutIfAbsent("three");
+
+ UNIT_ASSERT_VALUES_EQUAL(strPool.BytesSize(), 11);
+ UNIT_ASSERT_VALUES_EQUAL(strPool.Count(), 3);
+
+ strPool.Build();
+
+ TVector<TString> strings;
+ TVector<ui32> indexes;
+ TVector<ui32> frequences;
+ strPool.ForEach([&](TStringBuf str, ui32 index, ui32 freq) {
+ strings.emplace_back(str);
+ indexes.push_back(index);
+ frequences.push_back(freq);
+ });
+
+ TVector<TString> expectedStrings = {"three", "two", "one"};
+ UNIT_ASSERT_EQUAL(strings, expectedStrings);
+
+ TVector<ui32> expectedIndexes = {0, 1, 2};
+ UNIT_ASSERT_EQUAL(indexes, expectedIndexes);
+
+ TVector<ui32> expectedFrequences = {3, 2, 1};
+ UNIT_ASSERT_EQUAL(frequences, expectedFrequences);
+ }
+}
diff --git a/library/cpp/monlib/encode/buffered/ut/ya.make b/library/cpp/monlib/encode/buffered/ut/ya.make
new file mode 100644
index 0000000000..2157ac1490
--- /dev/null
+++ b/library/cpp/monlib/encode/buffered/ut/ya.make
@@ -0,0 +1,12 @@
+UNITTEST_FOR(library/cpp/monlib/encode/buffered)
+
+OWNER(
+ g:solomon
+ jamel
+)
+
+SRCS(
+ string_pool_ut.cpp
+)
+
+END()
diff --git a/library/cpp/monlib/encode/buffered/ya.make b/library/cpp/monlib/encode/buffered/ya.make
new file mode 100644
index 0000000000..81b6a78b93
--- /dev/null
+++ b/library/cpp/monlib/encode/buffered/ya.make
@@ -0,0 +1,19 @@
+LIBRARY()
+
+OWNER(
+ g:solomon
+ jamel
+ msherbakov
+)
+
+SRCS(
+ buffered_encoder_base.cpp
+ string_pool.cpp
+)
+
+PEERDIR(
+ library/cpp/monlib/encode
+ library/cpp/monlib/metrics
+)
+
+END()
diff --git a/library/cpp/monlib/encode/encoder.cpp b/library/cpp/monlib/encode/encoder.cpp
new file mode 100644
index 0000000000..becf932689
--- /dev/null
+++ b/library/cpp/monlib/encode/encoder.cpp
@@ -0,0 +1,6 @@
+#include "encoder.h"
+
+namespace NMonitoring {
+ IMetricEncoder::~IMetricEncoder() {
+ }
+}
diff --git a/library/cpp/monlib/encode/encoder.h b/library/cpp/monlib/encode/encoder.h
new file mode 100644
index 0000000000..a26a133d16
--- /dev/null
+++ b/library/cpp/monlib/encode/encoder.h
@@ -0,0 +1,17 @@
+#pragma once
+
+#include <util/generic/ptr.h>
+
+#include <library/cpp/monlib/metrics/metric_consumer.h>
+
+namespace NMonitoring {
+ class IMetricEncoder: public IMetricConsumer {
+ public:
+ virtual ~IMetricEncoder();
+
+ virtual void Close() = 0;
+ };
+
+ using IMetricEncoderPtr = THolder<IMetricEncoder>;
+
+}
diff --git a/library/cpp/monlib/encode/encoder_state.cpp b/library/cpp/monlib/encode/encoder_state.cpp
new file mode 100644
index 0000000000..0ece696b1a
--- /dev/null
+++ b/library/cpp/monlib/encode/encoder_state.cpp
@@ -0,0 +1 @@
+#include "encoder_state.h"
diff --git a/library/cpp/monlib/encode/encoder_state.h b/library/cpp/monlib/encode/encoder_state.h
new file mode 100644
index 0000000000..e6a098f404
--- /dev/null
+++ b/library/cpp/monlib/encode/encoder_state.h
@@ -0,0 +1,62 @@
+#pragma once
+
+#include "encoder_state_enum.h"
+
+#include <util/generic/serialized_enum.h>
+#include <util/generic/yexception.h>
+
+
+namespace NMonitoring {
+
+ template <typename EEncoderState>
+ class TEncoderStateImpl {
+ public:
+ using EState = EEncoderState;
+
+ explicit TEncoderStateImpl(EEncoderState state = EEncoderState::ROOT)
+ : State_(state)
+ {
+ }
+
+ TEncoderStateImpl& operator=(EEncoderState rhs) noexcept {
+ State_ = rhs;
+ return *this;
+ }
+
+ inline bool operator==(EEncoderState rhs) const noexcept {
+ return State_ == rhs;
+ }
+
+ inline bool operator!=(EEncoderState rhs) const noexcept {
+ return !operator==(rhs);
+ }
+
+ [[noreturn]] inline void ThrowInvalid(TStringBuf message) const {
+ ythrow yexception() << "invalid encoder state: "
+ << ToStr() << ", " << message;
+ }
+
+ inline void Expect(EEncoderState expected) const {
+ if (Y_UNLIKELY(State_ != expected)) {
+ ythrow yexception()
+ << "invalid encoder state: " << ToStr()
+ << ", expected: " << TEncoderStateImpl(expected).ToStr();
+ }
+ }
+
+ inline void Switch(EEncoderState from, EEncoderState to) {
+ Expect(from);
+ State_ = to;
+ }
+
+ TStringBuf ToStr() const noexcept {
+ return NEnumSerializationRuntime::GetEnumNamesImpl<EEncoderState>().at(State_);
+ }
+
+ private:
+ EEncoderState State_;
+ };
+
+ using TEncoderState = TEncoderStateImpl<EEncoderState>;
+
+} // namespace NMonitoring
diff --git a/library/cpp/monlib/encode/encoder_state_enum.h b/library/cpp/monlib/encode/encoder_state_enum.h
new file mode 100644
index 0000000000..471604f91d
--- /dev/null
+++ b/library/cpp/monlib/encode/encoder_state_enum.h
@@ -0,0 +1,12 @@
+#pragma once
+
+namespace NMonitoring {
+
+ enum class EEncoderState {
+ ROOT,
+ COMMON_LABELS,
+ METRIC,
+ METRIC_LABELS,
+ };
+
+} // namespace NMonitoring
diff --git a/library/cpp/monlib/encode/fake/fake.cpp b/library/cpp/monlib/encode/fake/fake.cpp
new file mode 100644
index 0000000000..69d691361a
--- /dev/null
+++ b/library/cpp/monlib/encode/fake/fake.cpp
@@ -0,0 +1,51 @@
+#include "fake.h"
+
+#include <util/datetime/base.h>
+
+namespace NMonitoring {
+ class TFakeEncoder: public IMetricEncoder {
+ public:
+ void OnStreamBegin() override {
+ }
+ void OnStreamEnd() override {
+ }
+
+ void OnCommonTime(TInstant) override {
+ }
+
+ void OnMetricBegin(EMetricType) override {
+ }
+ void OnMetricEnd() override {
+ }
+
+ void OnLabelsBegin() override {
+ }
+ void OnLabelsEnd() override {
+ }
+ void OnLabel(const TStringBuf, const TStringBuf) override {
+ }
+
+ void OnDouble(TInstant, double) override {
+ }
+ void OnInt64(TInstant, i64) override {
+ }
+ void OnUint64(TInstant, ui64) override {
+ }
+
+ void OnHistogram(TInstant, IHistogramSnapshotPtr) override {
+ }
+
+ void OnSummaryDouble(TInstant, ISummaryDoubleSnapshotPtr) override {
+ }
+
+ void OnLogHistogram(TInstant, TLogHistogramSnapshotPtr) override {
+ }
+
+ void Close() override {
+ }
+ };
+
+ IMetricEncoderPtr EncoderFake() {
+ return MakeHolder<TFakeEncoder>();
+ }
+}
diff --git a/library/cpp/monlib/encode/fake/fake.h b/library/cpp/monlib/encode/fake/fake.h
new file mode 100644
index 0000000000..8109326987
--- /dev/null
+++ b/library/cpp/monlib/encode/fake/fake.h
@@ -0,0 +1,10 @@
+#pragma once
+
+#include <library/cpp/monlib/encode/encoder.h>
+
+class IOutputStream;
+
+namespace NMonitoring {
+ // Does nothing: just implements IMetricEncoder interface with stubs
+ IMetricEncoderPtr EncoderFake();
+}
diff --git a/library/cpp/monlib/encode/fake/ya.make b/library/cpp/monlib/encode/fake/ya.make
new file mode 100644
index 0000000000..ae96f45782
--- /dev/null
+++ b/library/cpp/monlib/encode/fake/ya.make
@@ -0,0 +1,12 @@
+LIBRARY()
+
+OWNER(
+ g:solomon
+ msherbakov
+)
+
+SRCS(
+ fake.cpp
+)
+
+END()
diff --git a/library/cpp/monlib/encode/format.cpp b/library/cpp/monlib/encode/format.cpp
new file mode 100644
index 0000000000..400ce5a643
--- /dev/null
+++ b/library/cpp/monlib/encode/format.cpp
@@ -0,0 +1,202 @@
+#include "format.h"
+
+#include <util/string/ascii.h>
+#include <util/string/split.h>
+#include <util/string/strip.h>
+#include <util/stream/output.h>
+#include <util/string/cast.h>
+
+namespace NMonitoring {
+ static ECompression CompressionFromHeader(TStringBuf value) {
+ if (value.empty()) {
+ return ECompression::UNKNOWN;
+ }
+
+ for (const auto& it : StringSplitter(value).Split(',').SkipEmpty()) {
+ TStringBuf token = StripString(it.Token());
+
+ if (AsciiEqualsIgnoreCase(token, NFormatContentEncoding::IDENTITY)) {
+ return ECompression::IDENTITY;
+ } else if (AsciiEqualsIgnoreCase(token, NFormatContentEncoding::ZLIB)) {
+ return ECompression::ZLIB;
+ } else if (AsciiEqualsIgnoreCase(token, NFormatContentEncoding::LZ4)) {
+ return ECompression::LZ4;
+ } else if (AsciiEqualsIgnoreCase(token, NFormatContentEncoding::ZSTD)) {
+ return ECompression::ZSTD;
+ }
+ }
+
+ return ECompression::UNKNOWN;
+ }
+
+ static EFormat FormatFromHttpMedia(TStringBuf value) {
+ if (AsciiEqualsIgnoreCase(value, NFormatContenType::SPACK)) {
+ return EFormat::SPACK;
+ } else if (AsciiEqualsIgnoreCase(value, NFormatContenType::JSON)) {
+ return EFormat::JSON;
+ } else if (AsciiEqualsIgnoreCase(value, NFormatContenType::PROTOBUF)) {
+ return EFormat::PROTOBUF;
+ } else if (AsciiEqualsIgnoreCase(value, NFormatContenType::TEXT)) {
+ return EFormat::TEXT;
+ } else if (AsciiEqualsIgnoreCase(value, NFormatContenType::PROMETHEUS)) {
+ return EFormat::PROMETHEUS;
+ }
+
+ return EFormat::UNKNOWN;
+ }
+
+ EFormat FormatFromAcceptHeader(TStringBuf value) {
+ EFormat result{EFormat::UNKNOWN};
+
+ for (const auto& it : StringSplitter(value).Split(',').SkipEmpty()) {
+ TStringBuf token = StripString(it.Token());
+
+ result = FormatFromHttpMedia(token);
+ if (result != EFormat::UNKNOWN) {
+ break;
+ }
+ }
+
+ return result;
+ }
+
+ EFormat FormatFromContentType(TStringBuf value) {
+ value = value.NextTok(';');
+
+ return FormatFromHttpMedia(value);
+ }
+
+ TStringBuf ContentTypeByFormat(EFormat format) {
+ switch (format) {
+ case EFormat::SPACK:
+ return NFormatContenType::SPACK;
+ case EFormat::JSON:
+ return NFormatContenType::JSON;
+ case EFormat::PROTOBUF:
+ return NFormatContenType::PROTOBUF;
+ case EFormat::TEXT:
+ return NFormatContenType::TEXT;
+ case EFormat::PROMETHEUS:
+ return NFormatContenType::PROMETHEUS;
+ case EFormat::UNKNOWN:
+ return TStringBuf();
+ }
+
+ Y_FAIL(); // for GCC
+ }
+
+ ECompression CompressionFromAcceptEncodingHeader(TStringBuf value) {
+ return CompressionFromHeader(value);
+ }
+
+ ECompression CompressionFromContentEncodingHeader(TStringBuf value) {
+ return CompressionFromHeader(value);
+ }
+
+ TStringBuf ContentEncodingByCompression(ECompression compression) {
+ switch (compression) {
+ case ECompression::IDENTITY:
+ return NFormatContentEncoding::IDENTITY;
+ case ECompression::ZLIB:
+ return NFormatContentEncoding::ZLIB;
+ case ECompression::LZ4:
+ return NFormatContentEncoding::LZ4;
+ case ECompression::ZSTD:
+ return NFormatContentEncoding::ZSTD;
+ case ECompression::UNKNOWN:
+ return TStringBuf();
+ }
+
+ Y_FAIL(); // for GCC
+ }
+
+}
+
+template <>
+NMonitoring::EFormat FromStringImpl<NMonitoring::EFormat>(const char* str, size_t len) {
+ using NMonitoring::EFormat;
+ TStringBuf value(str, len);
+ if (value == TStringBuf("SPACK")) {
+ return EFormat::SPACK;
+ } else if (value == TStringBuf("JSON")) {
+ return EFormat::JSON;
+ } else if (value == TStringBuf("PROTOBUF")) {
+ return EFormat::PROTOBUF;
+ } else if (value == TStringBuf("TEXT")) {
+ return EFormat::TEXT;
+ } else if (value == TStringBuf("PROMETHEUS")) {
+ return EFormat::PROMETHEUS;
+ } else if (value == TStringBuf("UNKNOWN")) {
+ return EFormat::UNKNOWN;
+ }
+ ythrow yexception() << "unknown format: " << value;
+}
+
+template <>
+void Out<NMonitoring::EFormat>(IOutputStream& o, NMonitoring::EFormat f) {
+ using NMonitoring::EFormat;
+ switch (f) {
+ case EFormat::SPACK:
+ o << TStringBuf("SPACK");
+ return;
+ case EFormat::JSON:
+ o << TStringBuf("JSON");
+ return;
+ case EFormat::PROTOBUF:
+ o << TStringBuf("PROTOBUF");
+ return;
+ case EFormat::TEXT:
+ o << TStringBuf("TEXT");
+ return;
+ case EFormat::PROMETHEUS:
+ o << TStringBuf("PROMETHEUS");
+ return;
+ case EFormat::UNKNOWN:
+ o << TStringBuf("UNKNOWN");
+ return;
+ }
+
+ Y_FAIL(); // for GCC
+}
+
+template <>
+NMonitoring::ECompression FromStringImpl<NMonitoring::ECompression>(const char* str, size_t len) {
+ using NMonitoring::ECompression;
+ TStringBuf value(str, len);
+ if (value == TStringBuf("IDENTITY")) {
+ return ECompression::IDENTITY;
+ } else if (value == TStringBuf("ZLIB")) {
+ return ECompression::ZLIB;
+ } else if (value == TStringBuf("LZ4")) {
+ return ECompression::LZ4;
+ } else if (value == TStringBuf("ZSTD")) {
+ return ECompression::ZSTD;
+ } else if (value == TStringBuf("UNKNOWN")) {
+ return ECompression::UNKNOWN;
+ }
+ ythrow yexception() << "unknown compression: " << value;
+}
+
+template <>
+void Out<NMonitoring::ECompression>(IOutputStream& o, NMonitoring::ECompression c) {
+ using NMonitoring::ECompression;
+ switch (c) {
+ case ECompression::IDENTITY:
+ o << TStringBuf("IDENTITY");
+ return;
+ case ECompression::ZLIB:
+ o << TStringBuf("ZLIB");
+ return;
+ case ECompression::LZ4:
+ o << TStringBuf("LZ4");
+ return;
+ case ECompression::ZSTD:
+ o << TStringBuf("ZSTD");
+ return;
+ case ECompression::UNKNOWN:
+ o << TStringBuf("UNKNOWN");
+ return;
+ }
+
+ Y_FAIL(); // for GCC
+}
diff --git a/library/cpp/monlib/encode/format.h b/library/cpp/monlib/encode/format.h
new file mode 100644
index 0000000000..495d42d786
--- /dev/null
+++ b/library/cpp/monlib/encode/format.h
@@ -0,0 +1,166 @@
+#pragma once
+
+#include <util/generic/strbuf.h>
+
+namespace NMonitoring {
+ namespace NFormatContenType {
+ constexpr TStringBuf TEXT = "application/x-solomon-txt";
+ constexpr TStringBuf JSON = "application/json";
+ constexpr TStringBuf PROTOBUF = "application/x-solomon-pb";
+ constexpr TStringBuf SPACK = "application/x-solomon-spack";
+ constexpr TStringBuf PROMETHEUS = "text/plain";
+ }
+
+ namespace NFormatContentEncoding {
+ constexpr TStringBuf IDENTITY = "identity";
+ constexpr TStringBuf ZLIB = "zlib";
+ constexpr TStringBuf LZ4 = "lz4";
+ constexpr TStringBuf ZSTD = "zstd";
+ }
+
+ /**
+ * Defines format types for metric encoders.
+ */
+ enum class EFormat {
+ /**
+ * Special case when it was not possible to determine format.
+ */
+ UNKNOWN,
+
+ /**
+ * Read more https://wiki.yandex-team.ru/solomon/api/dataformat/spackv1
+ */
+ SPACK,
+
+ /**
+ * Read more https://wiki.yandex-team.ru/solomon/api/dataformat/json
+ */
+ JSON,
+
+ /**
+ * Simple protobuf format, only for testing purposes.
+ */
+ PROTOBUF,
+
+ /**
+ * Simple text representation, only for debug purposes.
+ */
+ TEXT,
+
+ /**
+ * Prometheus text-based format
+ */
+ PROMETHEUS,
+ };
+
+ /**
+ * Defines compression algorithms for metric encoders.
+ */
+ enum class ECompression {
+ /**
+ * Special case when it was not possible to determine compression.
+ */
+ UNKNOWN,
+
+ /**
+ * Means no compression.
+ */
+ IDENTITY,
+
+ /**
+ * Using the zlib structure (defined in RFC 1950), with the
+ * deflate compression algorithm and Adler32 checkums.
+ */
+ ZLIB,
+
+ /**
+ * Using LZ4 compression algorithm (read http://lz4.org for more info)
+ * with XxHash32 checksums.
+ */
+ LZ4,
+
+ /**
+ * Using Zstandard compression algorithm (read http://zstd.net for more
+ * info) with XxHash32 checksums.
+ */
+ ZSTD,
+ };
+
+ enum class EMetricsMergingMode {
+ /**
+ * Do not merge metric batches. If several points of the same metric were
+ * added multiple times accross different writes, paste them as
+ * separate metrics.
+ *
+ * Example:
+ * COUNTER [(ts1, val1)] | COUNTER [(ts1, val1)]
+ * COUNTER [(ts2, val2)] | --> COUNTER [(ts2, val2)]
+ * COUNTER [(ts3, val3)] | COUNTER [(ts3, val3)]
+ */
+ DEFAULT,
+
+ /**
+ * If several points of the same metric were added multiple times across
+ * different writes, merge all values to one timeseries.
+ *
+ * Example:
+ * COUNTER [(ts1, val1)] |
+ * COUNTER [(ts2, val2)] | --> COUNTER [(ts1, val1), (ts2, val2), (ts3, val3)]
+ * COUNTER [(ts3, val3)] |
+ */
+ MERGE_METRICS,
+ };
+
+ /**
+ * Matches serialization format by the given "Accept" header value.
+ *
+ * @param value value of the "Accept" header.
+ * @return most preffered serialization format type
+ */
+ EFormat FormatFromAcceptHeader(TStringBuf value);
+
+ /**
+ * Matches serialization format by the given "Content-Type" header value
+ *
+ * @param value value of the "Content-Type" header
+ * @return message format
+ */
+ EFormat FormatFromContentType(TStringBuf value);
+
+ /**
+ * Returns value for "Content-Type" header determined by the given
+ * format type.
+ *
+ * @param format serialization format type
+ * @return mime-type indentificator
+ * or empty string if format is UNKNOWN
+ */
+ TStringBuf ContentTypeByFormat(EFormat format);
+
+ /**
+ * Matches compression algorithm by the given "Accept-Encoding" header value.
+ *
+ * @param value value of the "Accept-Encoding" header.
+ * @return most preffered compression algorithm
+ */
+ ECompression CompressionFromAcceptEncodingHeader(TStringBuf value);
+
+ /**
+ * Matches compression algorithm by the given "Content-Encoding" header value.
+ *
+ * @param value value of the "Accept-Encoding" header.
+ * @return most preffered compression algorithm
+ */
+ ECompression CompressionFromContentEncodingHeader(TStringBuf value);
+
+ /**
+ * Returns value for "Content-Encoding" header determined by the given
+ * compression algorithm.
+ *
+ * @param compression encoding compression alg
+ * @return media-type compresion algorithm
+ * or empty string if compression is UNKNOWN
+ */
+ TStringBuf ContentEncodingByCompression(ECompression compression);
+
+}
diff --git a/library/cpp/monlib/encode/format_ut.cpp b/library/cpp/monlib/encode/format_ut.cpp
new file mode 100644
index 0000000000..22a0e30c03
--- /dev/null
+++ b/library/cpp/monlib/encode/format_ut.cpp
@@ -0,0 +1,136 @@
+#include "format.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <util/generic/string.h>
+#include <util/string/builder.h>
+
+#include <array>
+
+using namespace NMonitoring;
+
+Y_UNIT_TEST_SUITE(TFormatTest) {
+ Y_UNIT_TEST(ContentTypeHeader) {
+ UNIT_ASSERT_EQUAL(FormatFromContentType(""), EFormat::UNKNOWN);
+ UNIT_ASSERT_EQUAL(FormatFromContentType("application/json;some=stuff"), EFormat::JSON);
+ UNIT_ASSERT_EQUAL(FormatFromContentType("application/x-solomon-spack"), EFormat::SPACK);
+ UNIT_ASSERT_EQUAL(FormatFromContentType("application/xml"), EFormat::UNKNOWN);
+ UNIT_ASSERT_EQUAL(FormatFromContentType(";application/xml"), EFormat::UNKNOWN);
+ }
+
+ Y_UNIT_TEST(AcceptHeader) {
+ UNIT_ASSERT_EQUAL(FormatFromAcceptHeader(""), EFormat::UNKNOWN);
+ UNIT_ASSERT_EQUAL(FormatFromAcceptHeader("*/*"), EFormat::UNKNOWN);
+
+ UNIT_ASSERT_EQUAL(
+ FormatFromAcceptHeader("application/xml"),
+ EFormat::UNKNOWN);
+
+ UNIT_ASSERT_EQUAL(
+ FormatFromAcceptHeader("application/json"),
+ EFormat::JSON);
+
+ UNIT_ASSERT_EQUAL(
+ FormatFromAcceptHeader("application/x-solomon-spack"),
+ EFormat::SPACK);
+
+ UNIT_ASSERT_EQUAL(
+ FormatFromAcceptHeader("application/x-solomon-pb"),
+ EFormat::PROTOBUF);
+
+ UNIT_ASSERT_EQUAL(
+ FormatFromAcceptHeader("application/x-solomon-txt"),
+ EFormat::TEXT);
+
+ UNIT_ASSERT_EQUAL(
+ FormatFromAcceptHeader("application/json, text/plain"),
+ EFormat::JSON);
+
+ UNIT_ASSERT_EQUAL(
+ FormatFromAcceptHeader("application/x-solomon-spack, application/json, text/plain"),
+ EFormat::SPACK);
+
+ UNIT_ASSERT_EQUAL(
+ FormatFromAcceptHeader(" , application/x-solomon-spack ,, application/json , text/plain"),
+ EFormat::SPACK);
+
+ UNIT_ASSERT_EQUAL(
+ FormatFromAcceptHeader("application/xml, application/x-solomon-spack, text/plain"),
+ EFormat::SPACK);
+
+ UNIT_ASSERT_EQUAL(
+ FormatFromAcceptHeader("text/plain"),
+ EFormat::PROMETHEUS);
+ }
+
+ Y_UNIT_TEST(FormatToStrFromStr) {
+ const std::array<EFormat, 6> formats = {{
+ EFormat::UNKNOWN,
+ EFormat::SPACK,
+ EFormat::JSON,
+ EFormat::PROTOBUF,
+ EFormat::TEXT,
+ EFormat::PROMETHEUS,
+ }};
+
+ for (EFormat f : formats) {
+ TString str = (TStringBuilder() << f);
+ EFormat g = FromString<EFormat>(str);
+ UNIT_ASSERT_EQUAL(f, g);
+ }
+ }
+
+ Y_UNIT_TEST(AcceptEncodingHeader) {
+ UNIT_ASSERT_EQUAL(
+ CompressionFromAcceptEncodingHeader(""),
+ ECompression::UNKNOWN);
+
+ UNIT_ASSERT_EQUAL(
+ CompressionFromAcceptEncodingHeader("br"),
+ ECompression::UNKNOWN);
+
+ UNIT_ASSERT_EQUAL(
+ CompressionFromAcceptEncodingHeader("identity"),
+ ECompression::IDENTITY);
+
+ UNIT_ASSERT_EQUAL(
+ CompressionFromAcceptEncodingHeader("zlib"),
+ ECompression::ZLIB);
+
+ UNIT_ASSERT_EQUAL(
+ CompressionFromAcceptEncodingHeader("lz4"),
+ ECompression::LZ4);
+
+ UNIT_ASSERT_EQUAL(
+ CompressionFromAcceptEncodingHeader("zstd"),
+ ECompression::ZSTD);
+
+ UNIT_ASSERT_EQUAL(
+ CompressionFromAcceptEncodingHeader("zstd, zlib"),
+ ECompression::ZSTD);
+
+ UNIT_ASSERT_EQUAL(
+ CompressionFromAcceptEncodingHeader(" ,, , zstd , zlib"),
+ ECompression::ZSTD);
+
+ UNIT_ASSERT_EQUAL(
+ CompressionFromAcceptEncodingHeader("br, deflate,lz4, zlib"),
+ ECompression::LZ4);
+ }
+
+ Y_UNIT_TEST(CompressionToStrFromStr) {
+ const std::array<ECompression, 5> algs = {{
+ ECompression::UNKNOWN,
+ ECompression::IDENTITY,
+ ECompression::ZLIB,
+ ECompression::LZ4,
+ ECompression::ZSTD,
+ }};
+
+ for (ECompression a : algs) {
+ TString str = (TStringBuilder() << a);
+ ECompression b = FromString<ECompression>(str);
+ UNIT_ASSERT_EQUAL(a, b);
+ }
+ }
+}
diff --git a/library/cpp/monlib/encode/fuzz/ya.make b/library/cpp/monlib/encode/fuzz/ya.make
new file mode 100644
index 0000000000..d9ca172bae
--- /dev/null
+++ b/library/cpp/monlib/encode/fuzz/ya.make
@@ -0,0 +1,5 @@
+RECURSE_ROOT_RELATIVE(
+ library/cpp/monlib/encode/json/fuzz
+ library/cpp/monlib/encode/prometheus/fuzz
+ library/cpp/monlib/encode/spack/fuzz
+)
diff --git a/library/cpp/monlib/encode/json/fuzz/main.cpp b/library/cpp/monlib/encode/json/fuzz/main.cpp
new file mode 100644
index 0000000000..4f40310e06
--- /dev/null
+++ b/library/cpp/monlib/encode/json/fuzz/main.cpp
@@ -0,0 +1,16 @@
+#include <library/cpp/monlib/encode/json/json.h>
+#include <library/cpp/monlib/encode/fake/fake.h>
+
+#include <util/generic/strbuf.h>
+
+
+extern "C" int LLVMFuzzerTestOneInput(const ui8* data, size_t size) {
+ auto encoder = NMonitoring::EncoderFake();
+
+ try {
+ NMonitoring::DecodeJson({reinterpret_cast<const char*>(data), size}, encoder.Get());
+ } catch (...) {
+ }
+
+ return 0;
+}
diff --git a/library/cpp/monlib/encode/json/fuzz/ya.make b/library/cpp/monlib/encode/json/fuzz/ya.make
new file mode 100644
index 0000000000..75baa77716
--- /dev/null
+++ b/library/cpp/monlib/encode/json/fuzz/ya.make
@@ -0,0 +1,19 @@
+FUZZ()
+
+OWNER(
+ g:solomon
+ msherbakov
+)
+
+PEERDIR(
+ library/cpp/monlib/encode/json
+ library/cpp/monlib/encode/fake
+)
+
+SIZE(MEDIUM)
+
+SRCS(
+ main.cpp
+)
+
+END()
diff --git a/library/cpp/monlib/encode/json/json.h b/library/cpp/monlib/encode/json/json.h
new file mode 100644
index 0000000000..21530f20c3
--- /dev/null
+++ b/library/cpp/monlib/encode/json/json.h
@@ -0,0 +1,29 @@
+#pragma once
+
+#include <library/cpp/monlib/encode/encoder.h>
+#include <library/cpp/monlib/encode/format.h>
+
+
+class IOutputStream;
+
+namespace NMonitoring {
+
+ class TJsonDecodeError: public yexception {
+ };
+
+ IMetricEncoderPtr EncoderJson(IOutputStream* out, int indentation = 0);
+
+ /// Buffered encoder will merge series with same labels into one.
+ IMetricEncoderPtr BufferedEncoderJson(IOutputStream* out, int indentation = 0);
+
+ IMetricEncoderPtr EncoderCloudJson(IOutputStream* out,
+ int indentation = 0,
+ TStringBuf metricNameLabel = "name");
+
+ IMetricEncoderPtr BufferedEncoderCloudJson(IOutputStream* out,
+ int indentation = 0,
+ TStringBuf metricNameLabel = "name");
+
+ void DecodeJson(TStringBuf data, IMetricConsumer* c, TStringBuf metricNameLabel = "name");
+
+}
diff --git a/library/cpp/monlib/encode/json/json_decoder.cpp b/library/cpp/monlib/encode/json/json_decoder.cpp
new file mode 100644
index 0000000000..d44ff5fd28
--- /dev/null
+++ b/library/cpp/monlib/encode/json/json_decoder.cpp
@@ -0,0 +1,1162 @@
+#include "json.h"
+#include "typed_point.h"
+
+
+#include <library/cpp/monlib/exception/exception.h>
+#include <library/cpp/monlib/metrics/labels.h>
+#include <library/cpp/monlib/metrics/metric_value.h>
+
+#include <library/cpp/json/json_reader.h>
+
+#include <util/datetime/base.h>
+#include <util/string/cast.h>
+
+#include <limits>
+
+namespace NMonitoring {
+
+#define DECODE_ENSURE(COND, ...) MONLIB_ENSURE_EX(COND, TJsonDecodeError() << __VA_ARGS__)
+
+namespace {
+
+///////////////////////////////////////////////////////////////////////
+// THistogramBuilder
+///////////////////////////////////////////////////////////////////////
+class THistogramBuilder {
+public:
+ void AddBound(TBucketBound bound) {
+ if (!Bounds_.empty()) {
+ DECODE_ENSURE(Bounds_.back() < bound,
+ "non sorted bounds, " << Bounds_.back() <<
+ " >= " << bound);
+ }
+ Bounds_.push_back(bound);
+ }
+
+ void AddValue(TBucketValue value) {
+ Values_.push_back(value);
+ }
+
+ void AddInf(TBucketValue value) {
+ InfPresented_ = true;
+ InfValue_ = value;
+ }
+
+ IHistogramSnapshotPtr Build() {
+ if (InfPresented_) {
+ Bounds_.push_back(Max<TBucketBound>());
+ Values_.push_back(InfValue_);
+ }
+
+ auto snapshot = ExplicitHistogramSnapshot(Bounds_, Values_);
+
+ Bounds_.clear();
+ Values_.clear();
+ InfPresented_ = false;
+
+ return snapshot;
+ }
+
+ bool Empty() const noexcept {
+ return Bounds_.empty() && Values_.empty();
+ }
+
+ void Clear() {
+ Bounds_.clear();
+ Values_.clear();
+ }
+
+private:
+ TBucketBounds Bounds_;
+ TBucketValues Values_;
+
+ bool InfPresented_ = false;
+ TBucketValue InfValue_;
+};
+
+class TSummaryDoubleBuilder {
+public:
+ ISummaryDoubleSnapshotPtr Build() const {
+ return MakeIntrusive<TSummaryDoubleSnapshot>(Sum_, Min_, Max_, Last_, Count_);
+ }
+
+ void SetSum(double sum) {
+ Empty_ = false;
+ Sum_ = sum;
+ }
+
+ void SetMin(double min) {
+ Empty_ = false;
+ Min_ = min;
+ }
+
+ void SetMax(double max) {
+ Empty_ = false;
+ Max_ = max;
+ }
+
+ void SetLast(double last) {
+ Empty_ = false;
+ Last_ = last;
+ }
+
+ void SetCount(ui64 count) {
+ Empty_ = false;
+ Count_ = count;
+ }
+
+ void Clear() {
+ Empty_ = true;
+ Sum_ = 0;
+ Min_ = 0;
+ Max_ = 0;
+ Last_ = 0;
+ Count_ = 0;
+ }
+
+ bool Empty() const {
+ return Empty_;
+ }
+
+private:
+ double Sum_ = 0;
+ double Min_ = 0;
+ double Max_ = 0;
+ double Last_ = 0;
+ ui64 Count_ = 0;
+ bool Empty_ = true;
+};
+
+class TLogHistogramBuilder {
+public:
+ void SetBase(double base) {
+ DECODE_ENSURE(base > 0, "base must be positive");
+ Base_ = base;
+ }
+
+ void SetZerosCount(ui64 zerosCount) {
+ DECODE_ENSURE(zerosCount >= 0, "zeros count must be positive");
+ ZerosCount_ = zerosCount;
+ }
+
+ void SetStartPower(int startPower) {
+ StartPower_ = startPower;
+ }
+
+ void AddBucketValue(double value) {
+ DECODE_ENSURE(value > 0.0, "bucket values must be positive");
+ DECODE_ENSURE(value < std::numeric_limits<double>::max(), "bucket values must be finite");
+ Buckets_.push_back(value);
+ }
+
+ void Clear() {
+ Buckets_.clear();
+ Base_ = 1.5;
+ ZerosCount_ = 0;
+ StartPower_ = 0;
+ }
+
+ bool Empty() const {
+ return Buckets_.empty() && ZerosCount_ == 0;
+ }
+
+ TLogHistogramSnapshotPtr Build() {
+ return MakeIntrusive<TLogHistogramSnapshot>(Base_, ZerosCount_, StartPower_, std::move(Buckets_));
+ }
+
+private:
+ double Base_ = 1.5;
+ ui64 ZerosCount_ = 0;
+ int StartPower_ = 0;
+ TVector<double> Buckets_;
+};
+
+std::pair<double, bool> ParseSpecDouble(TStringBuf string) {
+ if (string == TStringBuf("nan") || string == TStringBuf("NaN")) {
+ return {std::numeric_limits<double>::quiet_NaN(), true};
+ } else if (string == TStringBuf("inf") || string == TStringBuf("Infinity")) {
+ return {std::numeric_limits<double>::infinity(), true};
+ } else if (string == TStringBuf("-inf") || string == TStringBuf("-Infinity")) {
+ return {-std::numeric_limits<double>::infinity(), true};
+ } else {
+ return {0, false};
+ }
+}
+
+///////////////////////////////////////////////////////////////////////
+// TMetricCollector
+///////////////////////////////////////////////////////////////////////
+struct TMetricCollector {
+ EMetricType Type = EMetricType::UNKNOWN;
+ TLabels Labels;
+ THistogramBuilder HistogramBuilder;
+ TSummaryDoubleBuilder SummaryBuilder;
+ TLogHistogramBuilder LogHistBuilder;
+ TTypedPoint LastPoint;
+ TVector<TTypedPoint> TimeSeries;
+
+ bool SeenTsOrValue = false;
+ bool SeenTimeseries = false;
+
+ void Clear() {
+ Type = EMetricType::UNKNOWN;
+ Labels.Clear();
+ SeenTsOrValue = false;
+ SeenTimeseries = false;
+ TimeSeries.clear();
+ LastPoint = {};
+ HistogramBuilder.Clear();
+ SummaryBuilder.Clear();
+ LogHistBuilder.Clear();
+ }
+
+ void AddLabel(const TLabel& label) {
+ Labels.Add(label.Name(), label.Value());
+ }
+
+ void SetLastTime(TInstant time) {
+ LastPoint.SetTime(time);
+ }
+
+ template <typename T>
+ void SetLastValue(T value) {
+ LastPoint.SetValue(value);
+ }
+
+ void SaveLastPoint() {
+ DECODE_ENSURE(LastPoint.GetTime() != TInstant::Zero(),
+ "cannot add point without or zero timestamp");
+ if (!HistogramBuilder.Empty()) {
+ auto histogram = HistogramBuilder.Build();
+ TimeSeries.emplace_back(LastPoint.GetTime(), histogram.Get());
+ } else if (!SummaryBuilder.Empty()) {
+ auto summary = SummaryBuilder.Build();
+ TimeSeries.emplace_back(LastPoint.GetTime(), summary.Get());
+ } else if (!LogHistBuilder.Empty()) {
+ auto logHist = LogHistBuilder.Build();
+ TimeSeries.emplace_back(LastPoint.GetTime(), logHist.Get());
+ } else {
+ TimeSeries.push_back(std::move(LastPoint));
+ }
+ }
+
+ template <typename TConsumer>
+ void Consume(TConsumer&& consumer) {
+ if (TimeSeries.empty()) {
+ const auto& p = LastPoint;
+ consumer(p.GetTime(), p.GetValueType(), p.GetValue());
+ } else {
+ for (const auto& p: TimeSeries) {
+ consumer(p.GetTime(), p.GetValueType(), p.GetValue());
+ }
+ }
+ }
+};
+
+struct TCommonParts {
+ TInstant CommonTime;
+ TLabels CommonLabels;
+};
+
+class IHaltableMetricConsumer: public IMetricConsumer {
+public:
+ virtual bool NeedToStop() const = 0;
+};
+
+// TODO(ivanzhukov@): check all states for cases when a json document is invalid
+// e.g. "metrics" or "commonLabels" keys are specified multiple times
+class TCommonPartsCollector: public IHaltableMetricConsumer {
+public:
+ TCommonParts&& CommonParts() {
+ return std::move(CommonParts_);
+ }
+
+private:
+ bool NeedToStop() const override {
+ return TInstant::Zero() != CommonParts_.CommonTime && !CommonParts_.CommonLabels.Empty();
+ }
+
+ void OnStreamBegin() override {
+ }
+
+ void OnStreamEnd() override {
+ }
+
+ void OnCommonTime(TInstant time) override {
+ CommonParts_.CommonTime = time;
+ }
+
+ void OnMetricBegin(EMetricType) override {
+ IsMetric_ = true;
+ }
+
+ void OnMetricEnd() override {
+ IsMetric_ = false;
+ }
+
+ void OnLabelsBegin() override {
+ }
+
+ void OnLabelsEnd() override {
+ }
+
+ void OnLabel(TStringBuf name, TStringBuf value) override {
+ if (!IsMetric_) {
+ CommonParts_.CommonLabels.Add(std::move(name), std::move(value));
+ }
+ }
+
+ void OnDouble(TInstant, double) override {
+ }
+
+ void OnInt64(TInstant, i64) override {
+ }
+
+ void OnUint64(TInstant, ui64) override {
+ }
+
+ void OnHistogram(TInstant, IHistogramSnapshotPtr) override {
+ }
+
+ void OnLogHistogram(TInstant, TLogHistogramSnapshotPtr) override {
+ }
+
+ void OnSummaryDouble(TInstant, ISummaryDoubleSnapshotPtr) override {
+ }
+
+private:
+ TCommonParts CommonParts_;
+ bool IsMetric_{false};
+};
+
+class TCommonPartsProxy: public IHaltableMetricConsumer {
+public:
+ TCommonPartsProxy(TCommonParts&& commonParts, IMetricConsumer* c)
+ : CommonParts_{std::move(commonParts)}
+ , Consumer_{c}
+ {}
+
+private:
+ bool NeedToStop() const override {
+ return false;
+ }
+
+ void OnStreamBegin() override {
+ Consumer_->OnStreamBegin();
+
+ if (!CommonParts_.CommonLabels.Empty()) {
+ Consumer_->OnLabelsBegin();
+
+ for (auto&& label : CommonParts_.CommonLabels) {
+ Consumer_->OnLabel(label.Name(), label.Value());
+ }
+
+ Consumer_->OnLabelsEnd();
+ }
+
+ if (TInstant::Zero() != CommonParts_.CommonTime) {
+ Consumer_->OnCommonTime(CommonParts_.CommonTime);
+ }
+ }
+
+ void OnStreamEnd() override {
+ Consumer_->OnStreamEnd();
+ }
+
+ void OnCommonTime(TInstant) override {
+ }
+
+ void OnMetricBegin(EMetricType type) override {
+ IsMetric_ = true;
+
+ Consumer_->OnMetricBegin(type);
+ }
+
+ void OnMetricEnd() override {
+ IsMetric_ = false;
+
+ Consumer_->OnMetricEnd();
+ }
+
+ void OnLabelsBegin() override {
+ if (IsMetric_) {
+ Consumer_->OnLabelsBegin();
+ }
+ }
+
+ void OnLabelsEnd() override {
+ if (IsMetric_) {
+ Consumer_->OnLabelsEnd();
+ }
+ }
+
+ void OnLabel(TStringBuf name, TStringBuf value) override {
+ if (IsMetric_) {
+ Consumer_->OnLabel(std::move(name), std::move(value));
+ }
+ }
+
+ void OnDouble(TInstant time, double value) override {
+ Consumer_->OnDouble(time, value);
+ }
+
+ void OnInt64(TInstant time, i64 value) override {
+ Consumer_->OnInt64(time, value);
+ }
+
+ void OnUint64(TInstant time, ui64 value) override {
+ Consumer_->OnUint64(time, value);
+ }
+
+ void OnHistogram(TInstant time, IHistogramSnapshotPtr snapshot) override {
+ Consumer_->OnHistogram(time, std::move(snapshot));
+ }
+
+ void OnLogHistogram(TInstant time, TLogHistogramSnapshotPtr snapshot) override {
+ Consumer_->OnLogHistogram(time, std::move(snapshot));
+ }
+
+ void OnSummaryDouble(TInstant time, ISummaryDoubleSnapshotPtr snapshot) override {
+ Consumer_->OnSummaryDouble(time, std::move(snapshot));
+ }
+
+private:
+ const TCommonParts CommonParts_;
+ IMetricConsumer* Consumer_;
+ bool IsMetric_{false};
+};
+
+///////////////////////////////////////////////////////////////////////
+// TDecoderJson
+///////////////////////////////////////////////////////////////////////
+class TDecoderJson final: public NJson::TJsonCallbacks {
+ struct TState {
+ enum EState {
+ ROOT_OBJECT = 0x01,
+
+ COMMON_LABELS,
+ COMMON_TS,
+ METRICS_ARRAY,
+
+ METRIC_OBJECT,
+ METRIC_NAME,
+ METRIC_LABELS,
+ METRIC_TYPE,
+ METRIC_MODE, // TODO: must be deleted
+ METRIC_TIMESERIES,
+ METRIC_TS,
+ METRIC_VALUE,
+ METRIC_HIST,
+ METRIC_HIST_BOUNDS,
+ METRIC_HIST_BUCKETS,
+ METRIC_HIST_INF,
+ METRIC_DSUMMARY,
+ METRIC_DSUMMARY_SUM,
+ METRIC_DSUMMARY_MIN,
+ METRIC_DSUMMARY_MAX,
+ METRIC_DSUMMARY_LAST,
+ METRIC_DSUMMARY_COUNT,
+ METRIC_LOG_HIST,
+ METRIC_LOG_HIST_BASE,
+ METRIC_LOG_HIST_ZEROS,
+ METRIC_LOG_HIST_START_POWER,
+ METRIC_LOG_HIST_BUCKETS,
+ };
+
+ constexpr EState Current() const noexcept {
+ return static_cast<EState>(State_ & 0xFF);
+ }
+
+ void ToNext(EState state) noexcept {
+ constexpr auto bitSize = 8 * sizeof(ui8);
+ State_ = (State_ << bitSize) | static_cast<ui8>(state);
+ }
+
+ void ToPrev() noexcept {
+ constexpr auto bitSize = 8 * sizeof(ui8);
+ State_ = State_ >> bitSize;
+ }
+
+ private:
+ ui64 State_ = static_cast<ui64>(ROOT_OBJECT);
+ };
+
+public:
+ TDecoderJson(TStringBuf data, IHaltableMetricConsumer* metricConsumer, TStringBuf metricNameLabel)
+ : Data_(data)
+ , MetricConsumer_(metricConsumer)
+ , MetricNameLabel_(metricNameLabel)
+ {
+ }
+
+private:
+#define PARSE_ENSURE(CONDITION, ...) \
+do { \
+if (Y_UNLIKELY(!(CONDITION))) { \
+ ErrorMsg_ = TStringBuilder() << __VA_ARGS__; \
+ return false; \
+} \
+} while (false)
+
+ bool OnInteger(long long value) override {
+ switch (State_.Current()) {
+ case TState::COMMON_TS:
+ PARSE_ENSURE(value >= 0, "unexpected negative number in a common timestamp: " << value);
+ MetricConsumer_->OnCommonTime(TInstant::Seconds(value));
+ State_.ToPrev();
+
+ if (MetricConsumer_->NeedToStop()) {
+ IsIntentionallyHalted_ = true;
+ return false;
+ }
+
+ break;
+
+ case TState::METRIC_TS:
+ PARSE_ENSURE(value >= 0, "unexpected negative number in a metric timestamp: " << value);
+ LastMetric_.SetLastTime(TInstant::Seconds(value));
+ State_.ToPrev();
+ break;
+
+ case TState::METRIC_VALUE:
+ LastMetric_.SetLastValue(static_cast<i64>(value));
+ State_.ToPrev();
+ break;
+
+ case TState::METRIC_HIST_BOUNDS:
+ LastMetric_.HistogramBuilder.AddBound(static_cast<double>(value));
+ break;
+
+ case TState::METRIC_HIST_BUCKETS:
+ PARSE_ENSURE(value >= 0 && static_cast<ui64>(value) <= Max<TBucketValues::value_type>(), "value is out of bounds " << value);
+ LastMetric_.HistogramBuilder.AddValue(value);
+ break;
+
+ case TState::METRIC_HIST_INF:
+ PARSE_ENSURE(value >= 0, "unexpected negative number in histogram inf: " << value);
+ LastMetric_.HistogramBuilder.AddInf(value);
+ State_.ToPrev();
+ break;
+
+ case TState::METRIC_DSUMMARY_COUNT:
+ LastMetric_.SummaryBuilder.SetCount(value);
+ State_.ToPrev();
+ break;
+
+ case TState::METRIC_DSUMMARY_SUM:
+ LastMetric_.SummaryBuilder.SetSum(value);
+ State_.ToPrev();
+ break;
+ case TState::METRIC_DSUMMARY_MIN:
+ LastMetric_.SummaryBuilder.SetMin(value);
+ State_.ToPrev();
+ break;
+ case TState::METRIC_DSUMMARY_MAX:
+ LastMetric_.SummaryBuilder.SetMax(value);
+ State_.ToPrev();
+ break;
+ case TState::METRIC_DSUMMARY_LAST:
+ LastMetric_.SummaryBuilder.SetLast(value);
+ State_.ToPrev();
+ break;
+
+ case TState::METRIC_LOG_HIST_BASE:
+ LastMetric_.LogHistBuilder.SetBase(value);
+ State_.ToPrev();
+ break;
+
+ case TState::METRIC_LOG_HIST_ZEROS:
+ LastMetric_.LogHistBuilder.SetZerosCount(value);
+ State_.ToPrev();
+ break;
+
+ case TState::METRIC_LOG_HIST_START_POWER:
+ LastMetric_.LogHistBuilder.SetStartPower(value);
+ State_.ToPrev();
+ break;
+
+ case TState::METRIC_LOG_HIST_BUCKETS:
+ LastMetric_.LogHistBuilder.AddBucketValue(value);
+ break;
+
+ default:
+ return false;
+ }
+ return true;
+ }
+
+ bool OnUInteger(unsigned long long value) override {
+ switch (State_.Current()) {
+ case TState::COMMON_TS:
+ MetricConsumer_->OnCommonTime(TInstant::Seconds(value));
+ State_.ToPrev();
+
+ if (MetricConsumer_->NeedToStop()) {
+ IsIntentionallyHalted_ = true;
+ return false;
+ }
+
+ break;
+
+ case TState::METRIC_TS:
+ LastMetric_.SetLastTime(TInstant::Seconds(value));
+ State_.ToPrev();
+ break;
+
+ case TState::METRIC_VALUE:
+ PARSE_ENSURE(value <= Max<ui64>(), "Metric value is out of bounds: " << value);
+ LastMetric_.SetLastValue(static_cast<ui64>(value));
+ State_.ToPrev();
+ break;
+
+ case TState::METRIC_HIST_BOUNDS:
+ LastMetric_.HistogramBuilder.AddBound(static_cast<double>(value));
+ break;
+
+ case TState::METRIC_HIST_BUCKETS:
+ PARSE_ENSURE(value <= Max<TBucketValues::value_type>(), "Histogram bucket value is out of bounds: " << value);
+ LastMetric_.HistogramBuilder.AddValue(value);
+ break;
+
+ case TState::METRIC_HIST_INF:
+ LastMetric_.HistogramBuilder.AddInf(value);
+ State_.ToPrev();
+ break;
+
+ case TState::METRIC_DSUMMARY_COUNT:
+ LastMetric_.SummaryBuilder.SetCount(value);
+ State_.ToPrev();
+ break;
+
+ case TState::METRIC_DSUMMARY_SUM:
+ LastMetric_.SummaryBuilder.SetSum(value);
+ State_.ToPrev();
+ break;
+ case TState::METRIC_DSUMMARY_MIN:
+ LastMetric_.SummaryBuilder.SetMin(value);
+ State_.ToPrev();
+ break;
+ case TState::METRIC_DSUMMARY_MAX:
+ LastMetric_.SummaryBuilder.SetMax(value);
+ State_.ToPrev();
+ break;
+ case TState::METRIC_DSUMMARY_LAST:
+ LastMetric_.SummaryBuilder.SetLast(value);
+ State_.ToPrev();
+ break;
+
+ case TState::METRIC_LOG_HIST_BASE:
+ LastMetric_.LogHistBuilder.SetBase(value);
+ State_.ToPrev();
+ break;
+
+ case TState::METRIC_LOG_HIST_ZEROS:
+ LastMetric_.LogHistBuilder.SetZerosCount(value);
+ State_.ToPrev();
+ break;
+
+ case TState::METRIC_LOG_HIST_START_POWER:
+ LastMetric_.LogHistBuilder.SetStartPower(value);
+ State_.ToPrev();
+ break;
+
+ case TState::METRIC_LOG_HIST_BUCKETS:
+ LastMetric_.LogHistBuilder.AddBucketValue(value);
+ break;
+
+ default:
+ return false;
+ }
+ return true;
+ }
+
+ bool OnDouble(double value) override {
+ switch (State_.Current()) {
+ case TState::METRIC_VALUE:
+ LastMetric_.SetLastValue(value);
+ State_.ToPrev();
+ break;
+
+ case TState::METRIC_HIST_BOUNDS:
+ LastMetric_.HistogramBuilder.AddBound(value);
+ break;
+
+ case TState::METRIC_DSUMMARY_SUM:
+ LastMetric_.SummaryBuilder.SetSum(value);
+ State_.ToPrev();
+ break;
+ case TState::METRIC_DSUMMARY_MIN:
+ LastMetric_.SummaryBuilder.SetMin(value);
+ State_.ToPrev();
+ break;
+ case TState::METRIC_DSUMMARY_MAX:
+ LastMetric_.SummaryBuilder.SetMax(value);
+ State_.ToPrev();
+ break;
+ case TState::METRIC_DSUMMARY_LAST:
+ LastMetric_.SummaryBuilder.SetLast(value);
+ State_.ToPrev();
+ break;
+
+ case TState::METRIC_LOG_HIST_BASE:
+ LastMetric_.LogHistBuilder.SetBase(value);
+ State_.ToPrev();
+ break;
+
+ case TState::METRIC_LOG_HIST_BUCKETS:
+ LastMetric_.LogHistBuilder.AddBucketValue(value);
+ break;
+
+ default:
+ return false;
+ }
+ return true;
+ }
+
+ bool OnString(const TStringBuf& value) override {
+ switch (State_.Current()) {
+ case TState::COMMON_LABELS:
+ PARSE_ENSURE(!LastLabelName_.empty(), "empty label name in common labels");
+ MetricConsumer_->OnLabel(LastLabelName_, TString{value});
+ break;
+
+ case TState::METRIC_LABELS:
+ PARSE_ENSURE(!LastLabelName_.empty(), "empty label name in metric labels");
+ LastMetric_.Labels.Add(LastLabelName_, TString{value});
+ break;
+
+ case TState::METRIC_NAME:
+ PARSE_ENSURE(!value.empty(), "empty metric name");
+ LastMetric_.Labels.Add(MetricNameLabel_, TString{value});
+ State_.ToPrev();
+ break;
+
+ case TState::COMMON_TS:
+ MetricConsumer_->OnCommonTime(TInstant::ParseIso8601(value));
+ State_.ToPrev();
+
+ if (MetricConsumer_->NeedToStop()) {
+ IsIntentionallyHalted_ = true;
+ return false;
+ }
+
+ break;
+
+ case TState::METRIC_TS:
+ LastMetric_.SetLastTime(TInstant::ParseIso8601(value));
+ State_.ToPrev();
+ break;
+
+ case TState::METRIC_VALUE:
+ if (auto [doubleValue, ok] = ParseSpecDouble(value); ok) {
+ LastMetric_.SetLastValue(doubleValue);
+ } else {
+ return false;
+ }
+ State_.ToPrev();
+ break;
+
+ case TState::METRIC_TYPE:
+ LastMetric_.Type = MetricTypeFromStr(value);
+ State_.ToPrev();
+ break;
+
+ case TState::METRIC_MODE:
+ if (value == TStringBuf("deriv")) {
+ LastMetric_.Type = EMetricType::RATE;
+ }
+ State_.ToPrev();
+ break;
+
+ case TState::METRIC_DSUMMARY_SUM:
+ if (auto [doubleValue, ok] = ParseSpecDouble(value); ok) {
+ LastMetric_.SummaryBuilder.SetSum(doubleValue);
+ } else {
+ return false;
+ }
+ State_.ToPrev();
+ break;
+
+ case TState::METRIC_DSUMMARY_MIN:
+ if (auto [doubleValue, ok] = ParseSpecDouble(value); ok) {
+ LastMetric_.SummaryBuilder.SetMin(doubleValue);
+ } else {
+ return false;
+ }
+ State_.ToPrev();
+ break;
+
+ case TState::METRIC_DSUMMARY_MAX:
+ if (auto [doubleValue, ok] = ParseSpecDouble(value); ok) {
+ LastMetric_.SummaryBuilder.SetMax(doubleValue);
+ } else {
+ return false;
+ }
+ State_.ToPrev();
+ break;
+
+ case TState::METRIC_DSUMMARY_LAST:
+ if (auto [doubleValue, ok] = ParseSpecDouble(value); ok) {
+ LastMetric_.SummaryBuilder.SetLast(doubleValue);
+ } else {
+ return false;
+ }
+ State_.ToPrev();
+ break;
+
+ default:
+ return false;
+ }
+
+ return true;
+ }
+
+ bool OnMapKey(const TStringBuf& key) override {
+ switch (State_.Current()) {
+ case TState::ROOT_OBJECT:
+ if (key == TStringBuf("commonLabels") || key == TStringBuf("labels")) {
+ State_.ToNext(TState::COMMON_LABELS);
+ } else if (key == TStringBuf("ts")) {
+ State_.ToNext(TState::COMMON_TS);
+ } else if (key == TStringBuf("sensors") || key == TStringBuf("metrics")) {
+ State_.ToNext(TState::METRICS_ARRAY);
+ }
+ break;
+
+ case TState::COMMON_LABELS:
+ case TState::METRIC_LABELS:
+ LastLabelName_ = key;
+ break;
+
+ case TState::METRIC_OBJECT:
+ if (key == TStringBuf("labels")) {
+ State_.ToNext(TState::METRIC_LABELS);
+ } else if (key == TStringBuf("name")) {
+ State_.ToNext(TState::METRIC_NAME);
+ } else if (key == TStringBuf("ts")) {
+ PARSE_ENSURE(!LastMetric_.SeenTimeseries,
+ "mixed timeseries and ts attributes");
+ LastMetric_.SeenTsOrValue = true;
+ State_.ToNext(TState::METRIC_TS);
+ } else if (key == TStringBuf("value")) {
+ PARSE_ENSURE(!LastMetric_.SeenTimeseries,
+ "mixed timeseries and value attributes");
+ LastMetric_.SeenTsOrValue = true;
+ State_.ToNext(TState::METRIC_VALUE);
+ } else if (key == TStringBuf("timeseries")) {
+ PARSE_ENSURE(!LastMetric_.SeenTsOrValue,
+ "mixed timeseries and ts/value attributes");
+ LastMetric_.SeenTimeseries = true;
+ State_.ToNext(TState::METRIC_TIMESERIES);
+ } else if (key == TStringBuf("mode")) {
+ State_.ToNext(TState::METRIC_MODE);
+ } else if (key == TStringBuf("kind") || key == TStringBuf("type")) {
+ State_.ToNext(TState::METRIC_TYPE);
+ } else if (key == TStringBuf("hist")) {
+ State_.ToNext(TState::METRIC_HIST);
+ } else if (key == TStringBuf("summary")) {
+ State_.ToNext(TState::METRIC_DSUMMARY);
+ } else if (key == TStringBuf("log_hist")) {
+ State_.ToNext(TState::METRIC_LOG_HIST);
+ } else if (key == TStringBuf("memOnly")) {
+ // deprecated. Skip it without errors for backward compatibility
+ } else {
+ ErrorMsg_ = TStringBuilder() << "unexpected key \"" << key << "\" in a metric schema";
+ return false;
+ }
+ break;
+
+ case TState::METRIC_TIMESERIES:
+ if (key == TStringBuf("ts")) {
+ State_.ToNext(TState::METRIC_TS);
+ } else if (key == TStringBuf("value")) {
+ State_.ToNext(TState::METRIC_VALUE);
+ } else if (key == TStringBuf("hist")) {
+ State_.ToNext(TState::METRIC_HIST);
+ } else if (key == TStringBuf("summary")) {
+ State_.ToNext(TState::METRIC_DSUMMARY);
+ } else if (key == TStringBuf("log_hist")) {
+ State_.ToNext(TState::METRIC_LOG_HIST);
+ }
+ break;
+
+ case TState::METRIC_HIST:
+ if (key == TStringBuf("bounds")) {
+ State_.ToNext(TState::METRIC_HIST_BOUNDS);
+ } else if (key == TStringBuf("buckets")) {
+ State_.ToNext(TState::METRIC_HIST_BUCKETS);
+ } else if (key == TStringBuf("inf")) {
+ State_.ToNext(TState::METRIC_HIST_INF);
+ }
+ break;
+
+ case TState::METRIC_LOG_HIST:
+ if (key == TStringBuf("base")) {
+ State_.ToNext(TState::METRIC_LOG_HIST_BASE);
+ } else if (key == TStringBuf("zeros_count")) {
+ State_.ToNext(TState::METRIC_LOG_HIST_ZEROS);
+ } else if (key == TStringBuf("start_power")) {
+ State_.ToNext(TState::METRIC_LOG_HIST_START_POWER);
+ } else if (key == TStringBuf("buckets")) {
+ State_.ToNext(TState::METRIC_LOG_HIST_BUCKETS);
+ }
+ break;
+
+ case TState::METRIC_DSUMMARY:
+ if (key == TStringBuf("sum")) {
+ State_.ToNext(TState::METRIC_DSUMMARY_SUM);
+ } else if (key == TStringBuf("min")) {
+ State_.ToNext(TState::METRIC_DSUMMARY_MIN);
+ } else if (key == TStringBuf("max")) {
+ State_.ToNext(TState::METRIC_DSUMMARY_MAX);
+ } else if (key == TStringBuf("last")) {
+ State_.ToNext(TState::METRIC_DSUMMARY_LAST);
+ } else if (key == TStringBuf("count")) {
+ State_.ToNext(TState::METRIC_DSUMMARY_COUNT);
+ }
+
+ break;
+
+
+ default:
+ return false;
+ }
+
+ return true;
+ }
+
+ bool OnOpenMap() override {
+ switch (State_.Current()) {
+ case TState::ROOT_OBJECT:
+ MetricConsumer_->OnStreamBegin();
+ break;
+
+ case TState::COMMON_LABELS:
+ MetricConsumer_->OnLabelsBegin();
+ break;
+
+ case TState::METRICS_ARRAY:
+ State_.ToNext(TState::METRIC_OBJECT);
+ LastMetric_.Clear();
+ break;
+
+ default:
+ break;
+ }
+ return true;
+ }
+
+ bool OnCloseMap() override {
+ switch (State_.Current()) {
+ case TState::ROOT_OBJECT:
+ MetricConsumer_->OnStreamEnd();
+ break;
+
+ case TState::METRIC_LABELS:
+ State_.ToPrev();
+ break;
+
+ case TState::COMMON_LABELS:
+ MetricConsumer_->OnLabelsEnd();
+ State_.ToPrev();
+
+ if (MetricConsumer_->NeedToStop()) {
+ IsIntentionallyHalted_ = true;
+ return false;
+ }
+
+ break;
+
+ case TState::METRIC_OBJECT:
+ ConsumeMetric();
+ State_.ToPrev();
+ break;
+
+ case TState::METRIC_TIMESERIES:
+ LastMetric_.SaveLastPoint();
+ break;
+
+ case TState::METRIC_HIST:
+ case TState::METRIC_DSUMMARY:
+ case TState::METRIC_LOG_HIST:
+ State_.ToPrev();
+ break;
+
+ default:
+ break;
+ }
+ return true;
+ }
+
+ bool OnOpenArray() override {
+ auto currentState = State_.Current();
+ PARSE_ENSURE(
+ currentState == TState::METRICS_ARRAY ||
+ currentState == TState::METRIC_TIMESERIES ||
+ currentState == TState::METRIC_HIST_BOUNDS ||
+ currentState == TState::METRIC_HIST_BUCKETS ||
+ currentState == TState::METRIC_LOG_HIST_BUCKETS,
+ "unexpected array begin");
+ return true;
+ }
+
+ bool OnCloseArray() override {
+ switch (State_.Current()) {
+ case TState::METRICS_ARRAY:
+ case TState::METRIC_TIMESERIES:
+ case TState::METRIC_HIST_BOUNDS:
+ case TState::METRIC_HIST_BUCKETS:
+ case TState::METRIC_LOG_HIST_BUCKETS:
+ State_.ToPrev();
+ break;
+
+ default:
+ return false;
+ }
+ return true;
+ }
+
+ void OnError(size_t off, TStringBuf reason) override {
+ if (IsIntentionallyHalted_) {
+ return;
+ }
+
+ size_t snippetBeg = (off < 20) ? 0 : (off - 20);
+ TStringBuf snippet = Data_.SubStr(snippetBeg, 40);
+
+ throw TJsonDecodeError()
+ << "cannot parse JSON, error at: " << off
+ << ", reason: " << (ErrorMsg_.empty() ? reason : TStringBuf{ErrorMsg_})
+ << "\nsnippet: ..." << snippet << "...";
+ }
+
+ bool OnEnd() override {
+ return true;
+ }
+
+ void ConsumeMetric() {
+ // for backwad compatibility all unknown metrics treated as gauges
+ if (LastMetric_.Type == EMetricType::UNKNOWN) {
+ if (LastMetric_.HistogramBuilder.Empty()) {
+ LastMetric_.Type = EMetricType::GAUGE;
+ } else {
+ LastMetric_.Type = EMetricType::HIST;
+ }
+ }
+
+ // (1) begin metric
+ MetricConsumer_->OnMetricBegin(LastMetric_.Type);
+
+ // (2) labels
+ if (!LastMetric_.Labels.empty()) {
+ MetricConsumer_->OnLabelsBegin();
+ for (auto&& label : LastMetric_.Labels) {
+ MetricConsumer_->OnLabel(label.Name(), label.Value());
+ }
+ MetricConsumer_->OnLabelsEnd();
+ }
+
+ // (3) values
+ switch (LastMetric_.Type) {
+ case EMetricType::GAUGE:
+ LastMetric_.Consume([this](TInstant time, EMetricValueType valueType, TMetricValue value) {
+ MetricConsumer_->OnDouble(time, value.AsDouble(valueType));
+ });
+ break;
+
+ case EMetricType::IGAUGE:
+ LastMetric_.Consume([this](TInstant time, EMetricValueType valueType, TMetricValue value) {
+ MetricConsumer_->OnInt64(time, value.AsInt64(valueType));
+ });
+ break;
+
+ case EMetricType::COUNTER:
+ case EMetricType::RATE:
+ LastMetric_.Consume([this](TInstant time, EMetricValueType valueType, TMetricValue value) {
+ MetricConsumer_->OnUint64(time, value.AsUint64(valueType));
+ });
+ break;
+
+ case EMetricType::HIST:
+ case EMetricType::HIST_RATE:
+ if (LastMetric_.TimeSeries.empty()) {
+ auto time = LastMetric_.LastPoint.GetTime();
+ auto histogram = LastMetric_.HistogramBuilder.Build();
+ MetricConsumer_->OnHistogram(time, histogram);
+ } else {
+ for (const auto& p : LastMetric_.TimeSeries) {
+ DECODE_ENSURE(p.GetValueType() == EMetricValueType::HISTOGRAM, "Value is not a histogram");
+ MetricConsumer_->OnHistogram(p.GetTime(), p.GetValue().AsHistogram());
+ }
+ }
+ break;
+
+ case EMetricType::DSUMMARY:
+ if (LastMetric_.TimeSeries.empty()) {
+ auto time = LastMetric_.LastPoint.GetTime();
+ auto summary = LastMetric_.SummaryBuilder.Build();
+ MetricConsumer_->OnSummaryDouble(time, summary);
+ } else {
+ for (const auto& p : LastMetric_.TimeSeries) {
+ DECODE_ENSURE(p.GetValueType() == EMetricValueType::SUMMARY, "Value is not a summary");
+ MetricConsumer_->OnSummaryDouble(p.GetTime(), p.GetValue().AsSummaryDouble());
+ }
+ }
+ break;
+
+ case EMetricType::LOGHIST:
+ if (LastMetric_.TimeSeries.empty()) {
+ auto time = LastMetric_.LastPoint.GetTime();
+ auto logHist = LastMetric_.LogHistBuilder.Build();
+ MetricConsumer_->OnLogHistogram(time, logHist);
+ } else {
+ for (const auto& p : LastMetric_.TimeSeries) {
+ DECODE_ENSURE(p.GetValueType() == EMetricValueType::LOGHISTOGRAM, "Value is not a log_histogram");
+ MetricConsumer_->OnLogHistogram(p.GetTime(), p.GetValue().AsLogHistogram());
+ }
+ }
+ break;
+
+ case EMetricType::UNKNOWN:
+ // TODO: output metric labels
+ ythrow yexception() << "unknown metric type";
+ }
+
+ // (4) end metric
+ MetricConsumer_->OnMetricEnd();
+ }
+
+private:
+ TStringBuf Data_;
+ IHaltableMetricConsumer* MetricConsumer_;
+ TString MetricNameLabel_;
+ TState State_;
+ TString LastLabelName_;
+ TMetricCollector LastMetric_;
+ TString ErrorMsg_;
+ bool IsIntentionallyHalted_{false};
+};
+
+} // namespace
+
+void DecodeJson(TStringBuf data, IMetricConsumer* c, TStringBuf metricNameLabel) {
+ TCommonPartsCollector commonPartsCollector;
+ {
+ TMemoryInput memIn(data);
+ TDecoderJson decoder(data, &commonPartsCollector, metricNameLabel);
+ // no need to check a return value. If there is an error, a TJsonDecodeError is thrown
+ NJson::ReadJson(&memIn, &decoder);
+ }
+
+ TCommonPartsProxy commonPartsProxy(std::move(commonPartsCollector.CommonParts()), c);
+ {
+ TMemoryInput memIn(data);
+ TDecoderJson decoder(data, &commonPartsProxy, metricNameLabel);
+ // no need to check a return value. If there is an error, a TJsonDecodeError is thrown
+ NJson::ReadJson(&memIn, &decoder);
+ }
+}
+
+#undef DECODE_ENSURE
+
+}
diff --git a/library/cpp/monlib/encode/json/json_decoder_ut.cpp b/library/cpp/monlib/encode/json/json_decoder_ut.cpp
new file mode 100644
index 0000000000..4464e1d26a
--- /dev/null
+++ b/library/cpp/monlib/encode/json/json_decoder_ut.cpp
@@ -0,0 +1,179 @@
+#include "json_decoder.cpp"
+
+#include <library/cpp/monlib/consumers/collecting_consumer.h>
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <array>
+
+
+using namespace NMonitoring;
+
+enum EJsonPart : ui8 {
+ METRICS = 0,
+ COMMON_TS = 1,
+ COMMON_LABELS = 2,
+};
+
+constexpr std::array<TStringBuf, 3> JSON_PARTS = {
+ TStringBuf(R"("metrics": [{
+ "labels": { "key": "value" },
+ "type": "GAUGE",
+ "value": 123
+ }])"),
+
+ TStringBuf(R"("ts": 1)"),
+
+ TStringBuf(R"("commonLabels": {
+ "key1": "value1",
+ "key2": "value2"
+ })"),
+};
+
+TString BuildJson(std::initializer_list<EJsonPart> parts) {
+ TString data = "{";
+
+ for (auto it = parts.begin(); it != parts.end(); ++it) {
+ data += JSON_PARTS[*it];
+
+ if (it + 1 != parts.end()) {
+ data += ",";
+ }
+ }
+
+ data += "}";
+ return data;
+}
+
+void ValidateCommonParts(TCommonParts&& commonParts, bool checkLabels, bool checkTs) {
+ if (checkTs) {
+ UNIT_ASSERT_VALUES_EQUAL(commonParts.CommonTime.MilliSeconds(), 1000);
+ }
+
+ if (checkLabels) {
+ auto& labels = commonParts.CommonLabels;
+ UNIT_ASSERT_VALUES_EQUAL(labels.Size(), 2);
+ UNIT_ASSERT(labels.Has(TStringBuf("key1")));
+ UNIT_ASSERT(labels.Has(TStringBuf("key2")));
+ UNIT_ASSERT_VALUES_EQUAL(labels.Get(TStringBuf("key1")).value()->Value(), "value1");
+ UNIT_ASSERT_VALUES_EQUAL(labels.Get(TStringBuf("key2")).value()->Value(), "value2");
+ }
+}
+
+void ValidateMetrics(const TVector<TMetricData>& metrics) {
+ UNIT_ASSERT_VALUES_EQUAL(metrics.size(), 1);
+
+ auto& m = metrics[0];
+ UNIT_ASSERT_VALUES_EQUAL(m.Kind, EMetricType::GAUGE);
+ auto& l = m.Labels;
+ UNIT_ASSERT_VALUES_EQUAL(l.Size(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(l.Get(0)->Name(), "key");
+ UNIT_ASSERT_VALUES_EQUAL(l.Get(0)->Value(), "value");
+
+ UNIT_ASSERT_VALUES_EQUAL(m.Values->Size(), 1);
+ UNIT_ASSERT_VALUES_EQUAL((*m.Values)[0].GetValue().AsDouble(), 123);
+}
+
+void CheckCommonPartsCollector(TString data, bool shouldBeStopped, bool checkLabels = true, bool checkTs = true, TStringBuf metricNameLabel = "name") {
+ TCommonPartsCollector commonPartsCollector;
+ TMemoryInput memIn(data);
+ TDecoderJson decoder(data, &commonPartsCollector, metricNameLabel);
+
+ bool isOk{false};
+ UNIT_ASSERT_NO_EXCEPTION(isOk = NJson::ReadJson(&memIn, &decoder));
+ UNIT_ASSERT_VALUES_EQUAL(isOk, !shouldBeStopped);
+
+ ValidateCommonParts(commonPartsCollector.CommonParts(), checkLabels, checkTs);
+}
+
+Y_UNIT_TEST_SUITE(TJsonDecoderTest) {
+ Y_UNIT_TEST(FullCommonParts) {
+ CheckCommonPartsCollector(BuildJson({COMMON_LABELS, COMMON_TS, METRICS}), true);
+ CheckCommonPartsCollector(BuildJson({COMMON_TS, COMMON_LABELS, METRICS}), true);
+
+ CheckCommonPartsCollector(BuildJson({METRICS, COMMON_TS, COMMON_LABELS}), true);
+ CheckCommonPartsCollector(BuildJson({METRICS, COMMON_LABELS, COMMON_TS}), true);
+
+ CheckCommonPartsCollector(BuildJson({COMMON_LABELS, METRICS, COMMON_TS}), true);
+ CheckCommonPartsCollector(BuildJson({COMMON_TS, METRICS, COMMON_LABELS}), true);
+ }
+
+ Y_UNIT_TEST(PartialCommonParts) {
+ CheckCommonPartsCollector(BuildJson({COMMON_TS, METRICS}), false, false, true);
+ CheckCommonPartsCollector(BuildJson({COMMON_LABELS, METRICS}), false, true, false);
+
+ CheckCommonPartsCollector(BuildJson({METRICS, COMMON_LABELS}), false, true, false);
+ CheckCommonPartsCollector(BuildJson({METRICS, COMMON_TS}), false, false, true);
+
+ CheckCommonPartsCollector(BuildJson({METRICS}), false, false, false);
+ }
+
+ Y_UNIT_TEST(CheckCommonPartsAndMetrics) {
+ auto data = BuildJson({COMMON_LABELS, COMMON_TS, METRICS});
+ TCollectingConsumer collector;
+
+ DecodeJson(data, &collector);
+
+ TCommonParts commonParts;
+ commonParts.CommonTime = collector.CommonTime;
+ commonParts.CommonLabels = collector.CommonLabels;
+
+ ValidateCommonParts(std::move(commonParts), true, true);
+ ValidateMetrics(collector.Metrics);
+ }
+
+ Y_UNIT_TEST(CanParseHistogramsWithInf) {
+ const char* metricsData = R"({
+"metrics":
+ [
+ {
+ "hist": {
+ "bounds": [
+ 10
+ ],
+ "buckets": [
+ 11
+ ],
+ "inf": 12
+ },
+ "name":"s1",
+ "type": "HIST_RATE"
+ },
+ {
+ "hist": {
+ "bounds": [
+ 20
+ ],
+ "buckets": [
+ 21
+ ]
+ },
+ "name":"s2",
+ "type":"HIST_RATE"
+ }
+ ]
+})";
+ TCollectingConsumer consumer(false);
+ DecodeJson(metricsData, &consumer);
+
+ UNIT_ASSERT_VALUES_EQUAL(consumer.Metrics.size(), 2);
+ {
+ const auto& m = consumer.Metrics[0];
+ UNIT_ASSERT_VALUES_EQUAL(m.Kind, EMetricType::HIST_RATE);
+ UNIT_ASSERT_VALUES_EQUAL(m.Values->Size(), 1);
+ const auto* histogram = (*m.Values)[0].GetValue().AsHistogram();
+ UNIT_ASSERT_VALUES_EQUAL(histogram->Count(), 2);
+ UNIT_ASSERT_VALUES_EQUAL(histogram->UpperBound(1), Max<TBucketBound>());
+ UNIT_ASSERT_VALUES_EQUAL(histogram->Value(0), 11);
+ UNIT_ASSERT_VALUES_EQUAL(histogram->Value(1), 12);
+ }
+ {
+ const auto& m = consumer.Metrics[1];
+ UNIT_ASSERT_VALUES_EQUAL(m.Kind, EMetricType::HIST_RATE);
+ UNIT_ASSERT_VALUES_EQUAL(m.Values->Size(), 1);
+ const auto* histogram = (*m.Values)[0].GetValue().AsHistogram();
+ UNIT_ASSERT_VALUES_EQUAL(histogram->Count(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(histogram->UpperBound(0), 20);
+ UNIT_ASSERT_VALUES_EQUAL(histogram->Value(0), 21);
+ }
+ }
+}
diff --git a/library/cpp/monlib/encode/json/json_encoder.cpp b/library/cpp/monlib/encode/json/json_encoder.cpp
new file mode 100644
index 0000000000..20d2bb6283
--- /dev/null
+++ b/library/cpp/monlib/encode/json/json_encoder.cpp
@@ -0,0 +1,556 @@
+#include "json.h"
+#include "typed_point.h"
+
+#include <library/cpp/monlib/encode/buffered/buffered_encoder_base.h>
+#include <library/cpp/monlib/encode/encoder_state.h>
+#include <library/cpp/monlib/metrics/metric.h>
+#include <library/cpp/monlib/metrics/metric_value.h>
+#include <library/cpp/monlib/metrics/labels.h>
+
+#include <library/cpp/json/writer/json.h>
+
+#include <util/charset/utf8.h>
+#include <util/generic/algorithm.h>
+
+namespace NMonitoring {
+ namespace {
+ enum class EJsonStyle {
+ Solomon,
+ Cloud
+ };
+
+ ///////////////////////////////////////////////////////////////////////
+ // TJsonWriter
+ ///////////////////////////////////////////////////////////////////////
+ class TJsonWriter {
+ public:
+ TJsonWriter(IOutputStream* out, int indentation, EJsonStyle style, TStringBuf metricNameLabel)
+ : Buf_(NJsonWriter::HEM_UNSAFE, out)
+ , Style_(style)
+ , MetricNameLabel_(metricNameLabel)
+ , CurrentMetricName_()
+ {
+ Buf_.SetIndentSpaces(indentation);
+ Buf_.SetWriteNanAsString();
+ }
+
+ void WriteTime(TInstant time) {
+ if (time != TInstant::Zero()) {
+ Buf_.WriteKey(TStringBuf("ts"));
+ if (Style_ == EJsonStyle::Solomon) {
+ Buf_.WriteULongLong(time.Seconds());
+ } else {
+ Buf_.WriteString(time.ToString());
+ }
+ }
+ }
+
+ void WriteValue(double value) {
+ Buf_.WriteKey(TStringBuf("value"));
+ Buf_.WriteDouble(value);
+ }
+
+ void WriteValue(i64 value) {
+ Buf_.WriteKey(TStringBuf("value"));
+ Buf_.WriteLongLong(value);
+ }
+
+ void WriteValue(ui64 value) {
+ Buf_.WriteKey(TStringBuf("value"));
+ Buf_.WriteULongLong(value);
+ }
+
+ void WriteValue(IHistogramSnapshot* s) {
+ Y_ENSURE(Style_ == EJsonStyle::Solomon);
+
+ Buf_.WriteKey(TStringBuf("hist"));
+ Buf_.BeginObject();
+ if (ui32 count = s->Count()) {
+ bool hasInf = (s->UpperBound(count - 1) == Max<double>());
+ if (hasInf) {
+ count--;
+ }
+
+ Buf_.WriteKey(TStringBuf("bounds"));
+ Buf_.BeginList();
+ for (ui32 i = 0; i < count; i++) {
+ Buf_.WriteDouble(s->UpperBound(i));
+ }
+ Buf_.EndList();
+
+ Buf_.WriteKey(TStringBuf("buckets"));
+ Buf_.BeginList();
+ for (ui32 i = 0; i < count; i++) {
+ Buf_.WriteULongLong(s->Value(i));
+ }
+ Buf_.EndList();
+
+ if (hasInf) {
+ Buf_.WriteKey(TStringBuf("inf"));
+ Buf_.WriteULongLong(s->Value(count));
+ }
+ }
+ Buf_.EndObject();
+ }
+
+ void WriteValue(ISummaryDoubleSnapshot* s) {
+ Y_ENSURE(Style_ == EJsonStyle::Solomon);
+
+ Buf_.WriteKey(TStringBuf("summary"));
+ Buf_.BeginObject();
+
+ Buf_.WriteKey(TStringBuf("sum"));
+ Buf_.WriteDouble(s->GetSum());
+
+ Buf_.WriteKey(TStringBuf("min"));
+ Buf_.WriteDouble(s->GetMin());
+
+ Buf_.WriteKey(TStringBuf("max"));
+ Buf_.WriteDouble(s->GetMax());
+
+ Buf_.WriteKey(TStringBuf("last"));
+ Buf_.WriteDouble(s->GetLast());
+
+ Buf_.WriteKey(TStringBuf("count"));
+ Buf_.WriteULongLong(s->GetCount());
+
+ Buf_.EndObject();
+ }
+
+ void WriteValue(TLogHistogramSnapshot* s) {
+ Y_ENSURE(Style_ == EJsonStyle::Solomon);
+
+ Buf_.WriteKey(TStringBuf("log_hist"));
+ Buf_.BeginObject();
+
+ Buf_.WriteKey(TStringBuf("base"));
+ Buf_.WriteDouble(s->Base());
+
+ Buf_.WriteKey(TStringBuf("zeros_count"));
+ Buf_.WriteULongLong(s->ZerosCount());
+
+ Buf_.WriteKey(TStringBuf("start_power"));
+ Buf_.WriteInt(s->StartPower());
+
+ Buf_.WriteKey(TStringBuf("buckets"));
+ Buf_.BeginList();
+ for (size_t i = 0; i < s->Count(); ++i) {
+ Buf_.WriteDouble(s->Bucket(i));
+ }
+ Buf_.EndList();
+
+ Buf_.EndObject();
+ }
+
+ void WriteValue(EMetricValueType type, TMetricValue value) {
+ switch (type) {
+ case EMetricValueType::DOUBLE:
+ WriteValue(value.AsDouble());
+ break;
+
+ case EMetricValueType::INT64:
+ WriteValue(value.AsInt64());
+ break;
+
+ case EMetricValueType::UINT64:
+ WriteValue(value.AsUint64());
+ break;
+
+ case EMetricValueType::HISTOGRAM:
+ WriteValue(value.AsHistogram());
+ break;
+
+ case EMetricValueType::SUMMARY:
+ WriteValue(value.AsSummaryDouble());
+ break;
+
+ case EMetricValueType::LOGHISTOGRAM:
+ WriteValue(value.AsLogHistogram());
+ break;
+
+ case EMetricValueType::UNKNOWN:
+ ythrow yexception() << "unknown metric value type";
+ }
+ }
+
+ void WriteLabel(TStringBuf name, TStringBuf value) {
+ Y_ENSURE(IsUtf(name), "label name is not valid UTF-8 string");
+ Y_ENSURE(IsUtf(value), "label value is not valid UTF-8 string");
+ if (Style_ == EJsonStyle::Cloud && name == MetricNameLabel_) {
+ CurrentMetricName_ = value;
+ } else {
+ Buf_.WriteKey(name);
+ Buf_.WriteString(value);
+ }
+ }
+
+ void WriteMetricType(EMetricType type) {
+ if (Style_ == EJsonStyle::Cloud) {
+ Buf_.WriteKey("type");
+ Buf_.WriteString(MetricTypeToCloudStr(type));
+ } else {
+ Buf_.WriteKey("kind");
+ Buf_.WriteString(MetricTypeToStr(type));
+ }
+ }
+
+ void WriteName() {
+ if (Style_ != EJsonStyle::Cloud) {
+ return;
+ }
+ if (CurrentMetricName_.Empty()) {
+ ythrow yexception() << "label '" << MetricNameLabel_ << "' is not defined";
+ }
+ Buf_.WriteKey("name");
+ Buf_.WriteString(CurrentMetricName_);
+ CurrentMetricName_.clear();
+ }
+
+ private:
+ static TStringBuf MetricTypeToCloudStr(EMetricType type) {
+ switch (type) {
+ case EMetricType::GAUGE:
+ return TStringBuf("DGAUGE");
+ case EMetricType::COUNTER:
+ return TStringBuf("COUNTER");
+ case EMetricType::RATE:
+ return TStringBuf("RATE");
+ case EMetricType::IGAUGE:
+ return TStringBuf("IGAUGE");
+ default:
+ ythrow yexception() << "metric type '" << type << "' is not supported by cloud json format";
+ }
+ }
+
+ protected:
+ NJsonWriter::TBuf Buf_;
+ EJsonStyle Style_;
+ TString MetricNameLabel_;
+ TString CurrentMetricName_;
+ };
+
+ ///////////////////////////////////////////////////////////////////////
+ // TEncoderJson
+ ///////////////////////////////////////////////////////////////////////
+ class TEncoderJson final: public IMetricEncoder, public TJsonWriter {
+ public:
+ TEncoderJson(IOutputStream* out, int indentation, EJsonStyle style, TStringBuf metricNameLabel)
+ : TJsonWriter{out, indentation, style, metricNameLabel}
+ {
+ }
+
+ ~TEncoderJson() override {
+ Close();
+ }
+
+ private:
+ void OnStreamBegin() override {
+ State_.Expect(TEncoderState::EState::ROOT);
+ Buf_.BeginObject();
+ }
+
+ void OnStreamEnd() override {
+ State_.Expect(TEncoderState::EState::ROOT);
+ if (!Buf_.KeyExpected()) {
+ // not closed metrics array
+ Buf_.EndList();
+ }
+ Buf_.EndObject();
+ }
+
+ void OnCommonTime(TInstant time) override {
+ State_.Expect(TEncoderState::EState::ROOT);
+ WriteTime(time);
+ }
+
+ void OnMetricBegin(EMetricType type) override {
+ State_.Switch(TEncoderState::EState::ROOT, TEncoderState::EState::METRIC);
+ if (Buf_.KeyExpected()) {
+ // first metric, so open metrics array
+ Buf_.WriteKey(TStringBuf(Style_ == EJsonStyle::Solomon ? "sensors" : "metrics"));
+ Buf_.BeginList();
+ }
+ Buf_.BeginObject();
+ WriteMetricType(type);
+ }
+
+ void OnMetricEnd() override {
+ State_.Switch(TEncoderState::EState::METRIC, TEncoderState::EState::ROOT);
+ if (!Buf_.KeyExpected()) {
+ // not closed timeseries array
+ Buf_.EndList();
+ }
+
+ if (!TimeSeries_ && LastPoint_.HasValue()) {
+ // we have seen only one point between OnMetricBegin() and
+ // OnMetricEnd() calls
+ WriteTime(LastPoint_.GetTime());
+ WriteValue(LastPoint_.GetValueType(), LastPoint_.GetValue());
+ }
+ Buf_.EndObject();
+
+ LastPoint_ = {};
+ TimeSeries_ = false;
+ }
+
+ void OnLabelsBegin() override {
+ if (!Buf_.KeyExpected()) {
+ // not closed metrics or timeseries array if labels go after values
+ Buf_.EndList();
+ }
+ if (State_ == TEncoderState::EState::ROOT) {
+ State_ = TEncoderState::EState::COMMON_LABELS;
+ Buf_.WriteKey(TStringBuf(Style_ == EJsonStyle::Solomon ? "commonLabels" : "labels"));
+ } else if (State_ == TEncoderState::EState::METRIC) {
+ State_ = TEncoderState::EState::METRIC_LABELS;
+ Buf_.WriteKey(TStringBuf("labels"));
+ } else {
+ State_.ThrowInvalid("expected METRIC or ROOT");
+ }
+ Buf_.BeginObject();
+
+ EmptyLabels_ = true;
+ }
+
+ void OnLabelsEnd() override {
+ if (State_ == TEncoderState::EState::METRIC_LABELS) {
+ State_ = TEncoderState::EState::METRIC;
+ } else if (State_ == TEncoderState::EState::COMMON_LABELS) {
+ State_ = TEncoderState::EState::ROOT;
+ } else {
+ State_.ThrowInvalid("expected LABELS or COMMON_LABELS");
+ }
+
+ Y_ENSURE(!EmptyLabels_, "Labels cannot be empty");
+ Buf_.EndObject();
+ if (State_ == TEncoderState::EState::METRIC) {
+ WriteName();
+ }
+ }
+
+ void OnLabel(TStringBuf name, TStringBuf value) override {
+ if (State_ == TEncoderState::EState::METRIC_LABELS || State_ == TEncoderState::EState::COMMON_LABELS) {
+ WriteLabel(name, value);
+ } else {
+ State_.ThrowInvalid("expected LABELS or COMMON_LABELS");
+ }
+
+ EmptyLabels_ = false;
+ }
+
+ void OnDouble(TInstant time, double value) override {
+ State_.Expect(TEncoderState::EState::METRIC);
+ Write<double>(time, value);
+ }
+
+ void OnInt64(TInstant time, i64 value) override {
+ State_.Expect(TEncoderState::EState::METRIC);
+ Write<i64>(time, value);
+ }
+
+ void OnUint64(TInstant time, ui64 value) override {
+ State_.Expect(TEncoderState::EState::METRIC);
+ Write<ui64>(time, value);
+ }
+
+ void OnHistogram(TInstant time, IHistogramSnapshotPtr snapshot) override {
+ State_.Expect(TEncoderState::EState::METRIC);
+ Write<IHistogramSnapshot*>(time, snapshot.Get());
+ }
+
+ void OnSummaryDouble(TInstant time, ISummaryDoubleSnapshotPtr snapshot) override {
+ State_.Expect(TEncoderState::EState::METRIC);
+ Write<ISummaryDoubleSnapshot*>(time, snapshot.Get());
+ }
+
+ void OnLogHistogram(TInstant time, TLogHistogramSnapshotPtr snapshot) override {
+ State_.Expect(TEncoderState::EState::METRIC);
+ Write<TLogHistogramSnapshot*>(time, snapshot.Get());
+ }
+
+ template <typename T>
+ void Write(TInstant time, T value) {
+ State_.Expect(TEncoderState::EState::METRIC);
+
+ if (!LastPoint_.HasValue()) {
+ LastPoint_ = {time, value};
+ } else {
+ // second point
+ // TODO: output types
+ Y_ENSURE(LastPoint_.GetValueType() == TValueType<T>::Type,
+ "mixed metric value types in one metric");
+
+ if (!TimeSeries_) {
+ Buf_.WriteKey(TStringBuf("timeseries"));
+ Buf_.BeginList();
+ Buf_.BeginObject();
+ Y_ENSURE(LastPoint_.GetTime() != TInstant::Zero(),
+ "time cannot be empty or zero in a timeseries point");
+ WriteTime(LastPoint_.GetTime());
+ WriteValue(LastPoint_.GetValueType(), LastPoint_.GetValue());
+ Buf_.EndObject();
+ TimeSeries_ = true;
+ }
+
+ if (TimeSeries_) {
+ Buf_.BeginObject();
+ Y_ENSURE(time != TInstant::Zero(),
+ "time cannot be empty or zero in a timeseries point");
+
+ WriteTime(time);
+ WriteValue(value);
+ Buf_.EndObject();
+ }
+ }
+ }
+
+ void Close() override {
+ LastPoint_ = {};
+ }
+
+ private:
+ TEncoderState State_;
+ TTypedPoint LastPoint_;
+ bool TimeSeries_ = false;
+ bool EmptyLabels_ = false;
+ };
+
+ ///////////////////////////////////////////////////////////////////////
+ // TBufferedJsonEncoder
+ ///////////////////////////////////////////////////////////////////////
+ class TBufferedJsonEncoder : public TBufferedEncoderBase, public TJsonWriter {
+ public:
+ TBufferedJsonEncoder(IOutputStream* out, int indentation, EJsonStyle style, TStringBuf metricNameLabel)
+ : TJsonWriter{out, indentation, style, metricNameLabel}
+ {
+ MetricsMergingMode_ = EMetricsMergingMode::MERGE_METRICS;
+ }
+
+ ~TBufferedJsonEncoder() override {
+ Close();
+ }
+
+ void OnLabelsBegin() override {
+ TBufferedEncoderBase::OnLabelsBegin();
+ EmptyLabels_ = true;
+ }
+
+ void OnLabel(TStringBuf name, TStringBuf value) override {
+ TBufferedEncoderBase::OnLabel(name, value);
+ EmptyLabels_ = false;
+ }
+
+ void OnLabel(ui32 name, ui32 value) override {
+ TBufferedEncoderBase::OnLabel(name, value);
+ EmptyLabels_ = false;
+ }
+
+ void OnLabelsEnd() override {
+ TBufferedEncoderBase::OnLabelsEnd();
+ Y_ENSURE(!EmptyLabels_, "Labels cannot be empty");
+ }
+
+ void Close() final {
+ if (Closed_) {
+ return;
+ }
+
+ Closed_ = true;
+
+ LabelValuesPool_.Build();
+ LabelNamesPool_.Build();
+
+ Buf_.BeginObject();
+
+ WriteTime(CommonTime_);
+ if (CommonLabels_.size() > 0) {
+ Buf_.WriteKey(TStringBuf(Style_ == EJsonStyle::Solomon ? "commonLabels": "labels"));
+ WriteLabels(CommonLabels_, true);
+ }
+
+ if (Metrics_.size() > 0) {
+ Buf_.WriteKey(TStringBuf(Style_ == EJsonStyle::Solomon ? "sensors" : "metrics"));
+ WriteMetrics();
+ }
+
+ Buf_.EndObject();
+ }
+
+ private:
+ void WriteMetrics() {
+ Buf_.BeginList();
+ for (auto&& metric : Metrics_) {
+ WriteMetric(metric);
+ }
+ Buf_.EndList();
+ }
+
+ void WriteMetric(TMetric& metric) {
+ Buf_.BeginObject();
+
+ WriteMetricType(metric.MetricType);
+
+ Buf_.WriteKey(TStringBuf("labels"));
+ WriteLabels(metric.Labels, false);
+
+ metric.TimeSeries.SortByTs();
+ if (metric.TimeSeries.Size() == 1) {
+ const auto& point = metric.TimeSeries[0];
+ WriteTime(point.GetTime());
+ WriteValue(metric.TimeSeries.GetValueType(), point.GetValue());
+ } else if (metric.TimeSeries.Size() > 1) {
+ Buf_.WriteKey(TStringBuf("timeseries"));
+ Buf_.BeginList();
+ metric.TimeSeries.ForEach([this](TInstant time, EMetricValueType type, TMetricValue value) {
+ Buf_.BeginObject();
+ // make gcc 6.1 happy https://gcc.gnu.org/bugzilla/show_bug.cgi?id=61636
+ this->WriteTime(time);
+ this->WriteValue(type, value);
+ Buf_.EndObject();
+ });
+
+ Buf_.EndList();
+ }
+
+ Buf_.EndObject();
+ }
+
+ void WriteLabels(const TPooledLabels& labels, bool isCommon) {
+ Buf_.BeginObject();
+
+ for (auto i = 0u; i < labels.size(); ++i) {
+ TStringBuf name = LabelNamesPool_.Get(labels[i].Key->Index);
+ TStringBuf value = LabelValuesPool_.Get(labels[i].Value->Index);
+
+ WriteLabel(name, value);
+ }
+
+ Buf_.EndObject();
+
+ if (!isCommon) {
+ WriteName();
+ }
+ }
+
+ private:
+ bool Closed_{false};
+ bool EmptyLabels_ = false;
+ };
+ }
+
+ IMetricEncoderPtr EncoderJson(IOutputStream* out, int indentation) {
+ return MakeHolder<TEncoderJson>(out, indentation, EJsonStyle::Solomon, "");
+ }
+
+ IMetricEncoderPtr BufferedEncoderJson(IOutputStream* out, int indentation) {
+ return MakeHolder<TBufferedJsonEncoder>(out, indentation, EJsonStyle::Solomon, "");
+ }
+
+ IMetricEncoderPtr EncoderCloudJson(IOutputStream* out, int indentation, TStringBuf metricNameLabel) {
+ return MakeHolder<TEncoderJson>(out, indentation, EJsonStyle::Cloud, metricNameLabel);
+ }
+
+ IMetricEncoderPtr BufferedEncoderCloudJson(IOutputStream* out, int indentation, TStringBuf metricNameLabel) {
+ return MakeHolder<TBufferedJsonEncoder>(out, indentation, EJsonStyle::Cloud, metricNameLabel);
+ }
+}
diff --git a/library/cpp/monlib/encode/json/json_ut.cpp b/library/cpp/monlib/encode/json/json_ut.cpp
new file mode 100644
index 0000000000..09e7909289
--- /dev/null
+++ b/library/cpp/monlib/encode/json/json_ut.cpp
@@ -0,0 +1,1290 @@
+#include "json.h"
+
+#include <library/cpp/monlib/encode/protobuf/protobuf.h>
+#include <library/cpp/monlib/metrics/labels.h>
+
+#include <library/cpp/json/json_reader.h>
+#include <library/cpp/resource/resource.h>
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <util/stream/str.h>
+#include <util/string/builder.h>
+
+#include <limits>
+
+using namespace NMonitoring;
+
+namespace NMonitoring {
+ bool operator<(const TLabel& lhs, const TLabel& rhs) {
+ return lhs.Name() < rhs.Name() ||
+ (lhs.Name() == rhs.Name() && lhs.Value() < rhs.Value());
+ }
+}
+namespace {
+ void AssertLabels(const NProto::TMultiSample& actual, const TLabels& expected) {
+ UNIT_ASSERT_EQUAL(actual.LabelsSize(), expected.Size());
+
+ TSet<TLabel> actualSet;
+ TSet<TLabel> expectedSet;
+ Transform(expected.begin(), expected.end(), std::inserter(expectedSet, expectedSet.end()), [] (auto&& l) {
+ return TLabel{l.Name(), l.Value()};
+ });
+
+ const auto& l = actual.GetLabels();
+ Transform(std::begin(l), std::end(l), std::inserter(actualSet, std::begin(actualSet)),
+ [](auto&& elem) -> TLabel {
+ return {elem.GetName(), elem.GetValue()};
+ });
+
+ TVector<TLabel> diff;
+ SetSymmetricDifference(std::begin(expectedSet), std::end(expectedSet),
+ std::begin(actualSet), std::end(actualSet), std::back_inserter(diff));
+
+ if (diff.size() > 0) {
+ for (auto&& l : diff) {
+ Cerr << l << Endl;
+ }
+
+ UNIT_FAIL("Labels don't match");
+ }
+ }
+
+ void AssertLabelEqual(const NProto::TLabel& l, TStringBuf name, TStringBuf value) {
+ UNIT_ASSERT_STRINGS_EQUAL(l.GetName(), name);
+ UNIT_ASSERT_STRINGS_EQUAL(l.GetValue(), value);
+ }
+
+ void AssertPointEqual(const NProto::TPoint& p, TInstant time, double value) {
+ UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds());
+ UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kFloat64);
+ UNIT_ASSERT_DOUBLES_EQUAL(p.GetFloat64(), value, std::numeric_limits<double>::epsilon());
+ }
+
+ void AssertPointEqualNan(const NProto::TPoint& p, TInstant time) {
+ UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds());
+ UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kFloat64);
+ UNIT_ASSERT(std::isnan(p.GetFloat64()));
+ }
+
+ void AssertPointEqualInf(const NProto::TPoint& p, TInstant time, int sign) {
+ UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds());
+ UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kFloat64);
+ UNIT_ASSERT(std::isinf(p.GetFloat64()));
+ if (sign < 0) {
+ UNIT_ASSERT(p.GetFloat64() < 0);
+ }
+ }
+
+ void AssertPointEqual(const NProto::TPoint& p, TInstant time, ui64 value) {
+ UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds());
+ UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kUint64);
+ UNIT_ASSERT_VALUES_EQUAL(p.GetUint64(), value);
+ }
+
+ void AssertPointEqual(const NProto::TPoint& p, TInstant time, i64 value) {
+ UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds());
+ UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kInt64);
+ UNIT_ASSERT_VALUES_EQUAL(p.GetInt64(), value);
+ }
+
+ void AssertPointEqual(const NProto::TPoint& p, TInstant time, const IHistogramSnapshot& expected) {
+ UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds());
+ UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kHistogram);
+
+ const NProto::THistogram& h = p.GetHistogram();
+ UNIT_ASSERT_VALUES_EQUAL(h.BoundsSize(), expected.Count());
+ UNIT_ASSERT_VALUES_EQUAL(h.ValuesSize(), expected.Count());
+
+ for (size_t i = 0; i < h.BoundsSize(); i++) {
+ UNIT_ASSERT_DOUBLES_EQUAL(h.GetBounds(i), expected.UpperBound(i), Min<double>());
+ UNIT_ASSERT_VALUES_EQUAL(h.GetValues(i), expected.Value(i));
+ }
+ }
+
+ void AssertPointEqual(const NProto::TPoint& p, TInstant time, const TLogHistogramSnapshot& expected) {
+ UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds());
+ UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kLogHistogram);
+
+ const double eps = 1e-10;
+ const NProto::TLogHistogram& h = p.GetLogHistogram();
+
+ UNIT_ASSERT_DOUBLES_EQUAL(h.GetBase(), expected.Base(), eps);
+ UNIT_ASSERT_VALUES_EQUAL(h.GetZerosCount(), expected.ZerosCount());
+ UNIT_ASSERT_VALUES_EQUAL(h.GetStartPower(), expected.StartPower());
+ UNIT_ASSERT_VALUES_EQUAL(h.BucketsSize(), expected.Count());
+ for (size_t i = 0; i < expected.Count(); ++i) {
+ UNIT_ASSERT_DOUBLES_EQUAL(h.GetBuckets(i), expected.Bucket(i), eps);
+ }
+ }
+
+ void AssertPointEqual(const NProto::TPoint& p, TInstant time, const ISummaryDoubleSnapshot& expected) {
+ UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds());
+ UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kSummaryDouble);
+ auto actual = p.GetSummaryDouble();
+ const double eps = 1e-10;
+ UNIT_ASSERT_DOUBLES_EQUAL(actual.GetSum(), expected.GetSum(), eps);
+ UNIT_ASSERT_DOUBLES_EQUAL(actual.GetMin(), expected.GetMin(), eps);
+ UNIT_ASSERT_DOUBLES_EQUAL(actual.GetMax(), expected.GetMax(), eps);
+ UNIT_ASSERT_DOUBLES_EQUAL(actual.GetLast(), expected.GetLast(), eps);
+ UNIT_ASSERT_VALUES_EQUAL(actual.GetCount(), expected.GetCount());
+ }
+
+} // namespace
+
+
+Y_UNIT_TEST_SUITE(TJsonTest) {
+ const TInstant now = TInstant::ParseIso8601Deprecated("2017-11-05T01:02:03Z");
+
+ Y_UNIT_TEST(Encode) {
+ auto check = [](bool cloud, bool buffered, TStringBuf expectedResourceKey) {
+ TString json;
+ TStringOutput out(json);
+ auto e = cloud
+ ? (buffered ? BufferedEncoderCloudJson(&out, 2, "metric") : EncoderCloudJson(&out, 2, "metric"))
+ : (buffered ? BufferedEncoderJson(&out, 2) : EncoderJson(&out, 2));
+ e->OnStreamBegin();
+ { // common time
+ e->OnCommonTime(TInstant::Seconds(1500000000));
+ }
+ { // common labels
+ e->OnLabelsBegin();
+ e->OnLabel("project", "solomon");
+ e->OnLabelsEnd();
+ }
+ { // metric #1
+ e->OnMetricBegin(EMetricType::COUNTER);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("metric", "single");
+ e->OnLabel("labels", "l1");
+ e->OnLabelsEnd();
+ }
+ e->OnUint64(now, 17);
+ e->OnMetricEnd();
+ }
+ { // metric #2
+ e->OnMetricBegin(EMetricType::RATE);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("metric", "single");
+ e->OnLabel("labels", "l2");
+ e->OnLabelsEnd();
+ }
+ e->OnUint64(now, 17);
+ e->OnMetricEnd();
+ }
+ { // metric #3
+ e->OnMetricBegin(EMetricType::GAUGE);
+ e->OnDouble(now, 3.14);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("metric", "single");
+ e->OnLabel("labels", "l3");
+ e->OnLabelsEnd();
+ }
+ e->OnMetricEnd();
+ }
+ { // metric #4
+ e->OnMetricBegin(EMetricType::IGAUGE);
+ e->OnInt64(now, 42);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("metric", "single_igauge");
+ e->OnLabel("labels", "l4");
+ e->OnLabelsEnd();
+ }
+ e->OnMetricEnd();
+ }
+ { // metric #5
+ e->OnMetricBegin(EMetricType::GAUGE);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("metric", "multiple");
+ e->OnLabel("labels", "l5");
+ e->OnLabelsEnd();
+ }
+ e->OnDouble(now, std::numeric_limits<double>::quiet_NaN());
+ e->OnDouble(now + TDuration::Seconds(15), std::numeric_limits<double>::infinity());
+ e->OnDouble(now + TDuration::Seconds(30), -std::numeric_limits<double>::infinity());
+ e->OnMetricEnd();
+ }
+
+ { // metric #6
+ e->OnMetricBegin(EMetricType::COUNTER);
+ e->OnUint64(now, 1337);
+ e->OnUint64(now + TDuration::Seconds(15), 1338);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("metric", "multiple");
+ e->OnLabel("labels", "l6");
+ e->OnLabelsEnd();
+ }
+ e->OnMetricEnd();
+ }
+ e->OnStreamEnd();
+ e->Close();
+ json += "\n";
+
+ auto parseJson = [] (auto buf) {
+ NJson::TJsonValue value;
+ NJson::ReadJsonTree(buf, &value, true);
+ return value;
+ };
+
+ const auto expectedJson = NResource::Find(expectedResourceKey);
+ UNIT_ASSERT_EQUAL(parseJson(json), parseJson(expectedJson));
+ };
+
+ check(false, false, "/expected.json");
+ check(false, true, "/expected_buffered.json");
+ check(true, false, "/expected_cloud.json");
+ check(true, true, "/expected_cloud_buffered.json");
+ }
+
+ TLogHistogramSnapshotPtr TestLogHistogram(ui32 v = 1) {
+ TVector<double> buckets{0.5 * v, 0.25 * v, 0.25 * v, 0.5 * v};
+ return MakeIntrusive<TLogHistogramSnapshot>(1.5, 1u, 0, std::move(buckets));
+ }
+
+ Y_UNIT_TEST(HistogramAndSummaryMetricTypesAreNotSupportedByCloudJson) {
+ const TInstant now = TInstant::ParseIso8601Deprecated("2017-11-05T01:02:03Z");
+
+ auto emit = [&](IMetricEncoder* encoder, EMetricType metricType) {
+ encoder->OnStreamBegin();
+ {
+ encoder->OnMetricBegin(metricType);
+ {
+ encoder->OnLabelsBegin();
+ encoder->OnLabel("name", "m");
+ encoder->OnLabelsEnd();
+ }
+
+ switch (metricType) {
+ case EMetricType::HIST: {
+ auto histogram = ExponentialHistogram(6, 2);
+ encoder->OnHistogram(now, histogram->Snapshot());
+ break;
+ }
+ case EMetricType::LOGHIST: {
+ auto histogram = TestLogHistogram();
+ encoder->OnLogHistogram(now, histogram);
+ break;
+ }
+ case EMetricType::DSUMMARY: {
+ auto summary = MakeIntrusive<TSummaryDoubleSnapshot>(10., -0.5, 0.5, 0.3, 30u);
+ encoder->OnSummaryDouble(now, summary);
+ break;
+ }
+ default:
+ Y_FAIL("unexpected metric type [%s]", ToString(metricType).c_str());
+ }
+
+ encoder->OnMetricEnd();
+ }
+ encoder->OnStreamEnd();
+ encoder->Close();
+ };
+
+ auto doTest = [&](bool buffered, EMetricType metricType) {
+ TString json;
+ TStringOutput out(json);
+ auto encoder = buffered ? BufferedEncoderCloudJson(&out, 2) : EncoderCloudJson(&out, 2);
+ const TString expectedMessage = TStringBuilder()
+ << "metric type '" << metricType << "' is not supported by cloud json format";
+ UNIT_ASSERT_EXCEPTION_CONTAINS_C(emit(encoder.Get(), metricType), yexception, expectedMessage,
+ TString("buffered: ") + ToString(buffered));
+ };
+
+ doTest(false, EMetricType::HIST);
+ doTest(false, EMetricType::LOGHIST);
+ doTest(false, EMetricType::DSUMMARY);
+ doTest(true, EMetricType::HIST);
+ doTest(true, EMetricType::LOGHIST);
+ doTest(true, EMetricType::DSUMMARY);
+ }
+
+ Y_UNIT_TEST(MetricsWithDifferentLabelOrderGetMerged) {
+ TString json;
+ TStringOutput out(json);
+ auto e = BufferedEncoderJson(&out, 2);
+
+ e->OnStreamBegin();
+ {
+ e->OnMetricBegin(EMetricType::RATE);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("metric", "hello");
+ e->OnLabel("label", "world");
+ e->OnLabelsEnd();
+ }
+ e->OnUint64(TInstant::Zero(), 0);
+ e->OnMetricEnd();
+ }
+ {
+ e->OnMetricBegin(EMetricType::RATE);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("label", "world");
+ e->OnLabel("metric", "hello");
+ e->OnLabelsEnd();
+ }
+ e->OnUint64(TInstant::Zero(), 1);
+ e->OnMetricEnd();
+ }
+ e->OnStreamEnd();
+ e->Close();
+ json += "\n";
+
+ TString expectedJson = NResource::Find("/merged.json");
+ // we cannot be sure regarding the label order in the result,
+ // so we'll have to parse the expected value and then compare it with actual
+
+ NProto::TMultiSamplesList samples;
+ IMetricEncoderPtr d = EncoderProtobuf(&samples);
+ DecodeJson(expectedJson, d.Get());
+
+ UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 1);
+ {
+ const NProto::TMultiSample& s = samples.GetSamples(0);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::RATE);
+ AssertLabels(s, TLabels{{"metric", "hello"}, {"label", "world"}});
+
+ UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1);
+ AssertPointEqual(s.GetPoints(0), TInstant::Zero(), ui64(1));
+ }
+ }
+ Y_UNIT_TEST(Decode1) {
+ NProto::TMultiSamplesList samples;
+ {
+ IMetricEncoderPtr e = EncoderProtobuf(&samples);
+
+ TString testJson = NResource::Find("/expected.json");
+ DecodeJson(testJson, e.Get());
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(
+ TInstant::MilliSeconds(samples.GetCommonTime()),
+ TInstant::Seconds(1500000000));
+
+ UNIT_ASSERT_VALUES_EQUAL(samples.CommonLabelsSize(), 1);
+ AssertLabelEqual(samples.GetCommonLabels(0), "project", "solomon");
+
+ UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 6);
+ {
+ const NProto::TMultiSample& s = samples.GetSamples(0);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::COUNTER);
+ UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2);
+ AssertLabelEqual(s.GetLabels(0), "metric", "single");
+ AssertLabelEqual(s.GetLabels(1), "labels", "l1");
+
+ UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1);
+ AssertPointEqual(s.GetPoints(0), now, ui64(17));
+ }
+ {
+ const NProto::TMultiSample& s = samples.GetSamples(1);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::RATE);
+ UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2);
+ AssertLabelEqual(s.GetLabels(0), "metric", "single");
+ AssertLabelEqual(s.GetLabels(1), "labels", "l2");
+
+ UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1);
+ AssertPointEqual(s.GetPoints(0), now, ui64(17));
+ }
+ {
+ const NProto::TMultiSample& s = samples.GetSamples(2);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE);
+ UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2);
+ AssertLabelEqual(s.GetLabels(0), "metric", "single");
+ AssertLabelEqual(s.GetLabels(1), "labels", "l3");
+
+ UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1);
+ AssertPointEqual(s.GetPoints(0), now, 3.14);
+ }
+ {
+ const NProto::TMultiSample& s = samples.GetSamples(3);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::IGAUGE);
+ UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2);
+ AssertLabelEqual(s.GetLabels(0), "metric", "single_igauge");
+ AssertLabelEqual(s.GetLabels(1), "labels", "l4");
+
+ UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1);
+ AssertPointEqual(s.GetPoints(0), now, i64(42));
+ }
+ {
+ const NProto::TMultiSample& s = samples.GetSamples(4);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE);
+ UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2);
+ AssertLabelEqual(s.GetLabels(0), "metric", "multiple");
+ AssertLabelEqual(s.GetLabels(1), "labels", "l5");
+
+ UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 3);
+ AssertPointEqualNan(s.GetPoints(0), now);
+ AssertPointEqualInf(s.GetPoints(1), now + TDuration::Seconds(15), 1);
+ AssertPointEqualInf(s.GetPoints(2), now + TDuration::Seconds(30), -11);
+ }
+ {
+ const NProto::TMultiSample& s = samples.GetSamples(5);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::COUNTER);
+ UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2);
+ AssertLabelEqual(s.GetLabels(0), "metric", "multiple");
+ AssertLabelEqual(s.GetLabels(1), "labels", "l6");
+
+ UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 2);
+ AssertPointEqual(s.GetPoints(0), now, ui64(1337));
+ AssertPointEqual(s.GetPoints(1), now + TDuration::Seconds(15), ui64(1338));
+ }
+ }
+
+ Y_UNIT_TEST(DecodeMetrics) {
+ NProto::TMultiSamplesList samples;
+ {
+ IMetricEncoderPtr e = EncoderProtobuf(&samples);
+
+ TString metricsJson = NResource::Find("/metrics.json");
+ DecodeJson(metricsJson, e.Get());
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(
+ TInstant::MilliSeconds(samples.GetCommonTime()),
+ TInstant::ParseIso8601Deprecated("2017-08-27T12:34:56Z"));
+
+ UNIT_ASSERT_VALUES_EQUAL(samples.CommonLabelsSize(), 3);
+ AssertLabelEqual(samples.GetCommonLabels(0), "project", "solomon");
+ AssertLabelEqual(samples.GetCommonLabels(1), "cluster", "man");
+ AssertLabelEqual(samples.GetCommonLabels(2), "service", "stockpile");
+
+ UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 4);
+ {
+ const NProto::TMultiSample& s = samples.GetSamples(0);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE);
+ UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1);
+ AssertLabelEqual(s.GetLabels(0), "metric", "Memory");
+
+ UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1);
+ AssertPointEqual(s.GetPoints(0), TInstant::Zero(), 10.0);
+ }
+ {
+ const NProto::TMultiSample& s = samples.GetSamples(1);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::RATE);
+ UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1);
+ AssertLabelEqual(s.GetLabels(0), "metric", "UserTime");
+
+ UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1);
+ AssertPointEqual(s.GetPoints(0), TInstant::Zero(), ui64(1));
+ }
+ {
+ const NProto::TMultiSample& s = samples.GetSamples(2);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE);
+ UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2);
+ AssertLabelEqual(s.GetLabels(0), "export", "Oxygen");
+ AssertLabelEqual(s.GetLabels(1), "metric", "QueueSize");
+
+ UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1);
+ auto ts = TInstant::ParseIso8601Deprecated("2017-11-05T12:34:56.000Z");
+ AssertPointEqual(s.GetPoints(0), ts, 3.14159);
+ }
+ {
+ const NProto::TMultiSample& s = samples.GetSamples(3);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE);
+ UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1);
+ AssertLabelEqual(s.GetLabels(0), "metric", "Writes");
+
+ UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 2);
+ auto ts1 = TInstant::ParseIso8601Deprecated("2017-08-28T12:32:11Z");
+ AssertPointEqual(s.GetPoints(0), ts1, -10.0);
+ auto ts2 = TInstant::Seconds(1503923187);
+ AssertPointEqual(s.GetPoints(1), ts2, 20.0);
+ }
+ }
+
+ Y_UNIT_TEST(DecodeSensors) {
+ NProto::TMultiSamplesList samples;
+ {
+ IMetricEncoderPtr e = EncoderProtobuf(&samples);
+
+ TString sensorsJson = NResource::Find("/sensors.json");
+ DecodeJson(sensorsJson, e.Get());
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(
+ TInstant::MilliSeconds(samples.GetCommonTime()),
+ TInstant::ParseIso8601Deprecated("2017-08-27T12:34:56Z"));
+
+ UNIT_ASSERT_VALUES_EQUAL(samples.CommonLabelsSize(), 3);
+ AssertLabelEqual(samples.GetCommonLabels(0), "project", "solomon");
+ AssertLabelEqual(samples.GetCommonLabels(1), "cluster", "man");
+ AssertLabelEqual(samples.GetCommonLabels(2), "service", "stockpile");
+
+ UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 4);
+ {
+ const NProto::TMultiSample& s = samples.GetSamples(0);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE);
+ UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1);
+ AssertLabelEqual(s.GetLabels(0), "metric", "Memory");
+
+ UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1);
+ AssertPointEqual(s.GetPoints(0), TInstant::Zero(), 10.0);
+ }
+ {
+ const NProto::TMultiSample& s = samples.GetSamples(1);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::RATE);
+ UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1);
+ AssertLabelEqual(s.GetLabels(0), "metric", "UserTime");
+
+ UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1);
+ AssertPointEqual(s.GetPoints(0), TInstant::Zero(), ui64(1));
+ }
+ {
+ const NProto::TMultiSample& s = samples.GetSamples(2);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE);
+ UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2);
+ AssertLabelEqual(s.GetLabels(0), "export", "Oxygen");
+ AssertLabelEqual(s.GetLabels(1), "metric", "QueueSize");
+
+ UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1);
+ auto ts = TInstant::ParseIso8601Deprecated("2017-11-05T12:34:56.000Z");
+ AssertPointEqual(s.GetPoints(0), ts, 3.14159);
+ }
+ {
+ const NProto::TMultiSample& s = samples.GetSamples(3);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE);
+ UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1);
+ AssertLabelEqual(s.GetLabels(0), "metric", "Writes");
+
+ UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 2);
+ auto ts1 = TInstant::ParseIso8601Deprecated("2017-08-28T12:32:11Z");
+ AssertPointEqual(s.GetPoints(0), ts1, -10.0);
+ auto ts2 = TInstant::Seconds(1503923187);
+ AssertPointEqual(s.GetPoints(1), ts2, 20.0);
+ }
+ }
+
+ Y_UNIT_TEST(DecodeToEncoder) {
+ auto testJson = NResource::Find("/test_decode_to_encode.json");
+
+ TStringStream Stream_;
+ auto encoder = BufferedEncoderJson(&Stream_, 4);
+ DecodeJson(testJson, encoder.Get());
+
+ encoder->Close();
+
+ auto val1 = NJson::ReadJsonFastTree(testJson, true);
+ auto val2 = NJson::ReadJsonFastTree(Stream_.Str(), true);
+
+ UNIT_ASSERT_VALUES_EQUAL(val1, val2);
+ }
+
+ void WriteEmptySeries(const IMetricEncoderPtr& e) {
+ e->OnStreamBegin();
+ {
+ e->OnMetricBegin(EMetricType::COUNTER);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("foo", "bar");
+ e->OnLabelsEnd();
+ }
+ e->OnMetricEnd();
+ }
+
+ e->OnStreamEnd();
+ e->Close();
+ }
+
+ Y_UNIT_TEST(EncodeEmptySeries) {
+ TString json;
+ TStringOutput out(json);
+
+ auto e = EncoderJson(&out, 2);
+ WriteEmptySeries(e);
+ json += "\n";
+
+ TString expectedJson = NResource::Find("/empty_series.json");
+ UNIT_ASSERT_NO_DIFF(json, expectedJson);
+ }
+
+ void WriteEmptyLabels(IMetricEncoderPtr& e) {
+ e->OnStreamBegin();
+ e->OnMetricBegin(EMetricType::COUNTER);
+
+ e->OnLabelsBegin();
+ UNIT_ASSERT_EXCEPTION(e->OnLabelsEnd(), yexception);
+ }
+
+ Y_UNIT_TEST(LabelsCannotBeEmpty) {
+ TString json;
+ TStringOutput out(json);
+
+ auto e = EncoderJson(&out, 2);
+ WriteEmptyLabels(e);
+ }
+
+ Y_UNIT_TEST(LabelsCannotBeEmptyBuffered) {
+ TString json;
+ TStringOutput out(json);
+
+ auto e = BufferedEncoderJson(&out, 2);
+ WriteEmptyLabels(e);
+ }
+
+ Y_UNIT_TEST(EncodeEmptySeriesBuffered) {
+ TString json;
+ TStringOutput out(json);
+
+ auto e = BufferedEncoderJson(&out, 2);
+ WriteEmptySeries(e);
+ json += "\n";
+
+ TString expectedJson = NResource::Find("/empty_series.json");
+ UNIT_ASSERT_NO_DIFF(json, expectedJson);
+ }
+
+ Y_UNIT_TEST(BufferedEncoderMergesMetrics) {
+ TString json;
+ TStringOutput out(json);
+
+ auto e = BufferedEncoderJson(&out, 2);
+ auto ts = 1;
+
+ auto writeMetric = [&] (const TString& val) {
+ e->OnMetricBegin(EMetricType::COUNTER);
+
+ e->OnLabelsBegin();
+ e->OnLabel("foo", val);
+ e->OnLabelsEnd();
+ e->OnUint64(TInstant::Seconds(ts++), 42);
+
+ e->OnMetricEnd();
+ };
+
+ e->OnStreamBegin();
+ writeMetric("bar");
+ writeMetric("bar");
+ writeMetric("baz");
+ writeMetric("bar");
+ e->OnStreamEnd();
+ e->Close();
+
+ json += "\n";
+
+ TString expectedJson = NResource::Find("/buffered_test.json");
+ UNIT_ASSERT_NO_DIFF(json, expectedJson);
+ }
+
+ Y_UNIT_TEST(JsonEncoderDisallowsValuesInTimeseriesWithoutTs) {
+ TStringStream out;
+
+ auto e = EncoderJson(&out);
+ auto writePreamble = [&] {
+ e->OnStreamBegin();
+ e->OnMetricBegin(EMetricType::COUNTER);
+ e->OnLabelsBegin();
+ e->OnLabel("foo", "bar");
+ e->OnLabelsEnd();
+ };
+
+ // writing two values for a metric in a row will trigger
+ // timeseries object construction
+ writePreamble();
+ e->OnUint64(TInstant::Zero(), 42);
+ UNIT_ASSERT_EXCEPTION(e->OnUint64(TInstant::Zero(), 42), yexception);
+
+ e = EncoderJson(&out);
+ writePreamble();
+ e->OnUint64(TInstant::Zero(), 42);
+ UNIT_ASSERT_EXCEPTION(e->OnUint64(TInstant::Now(), 42), yexception);
+
+ e = EncoderJson(&out);
+ writePreamble();
+ e->OnUint64(TInstant::Now(), 42);
+ UNIT_ASSERT_EXCEPTION(e->OnUint64(TInstant::Zero(), 42), yexception);
+ }
+
+ Y_UNIT_TEST(BufferedJsonEncoderMergesTimeseriesWithoutTs) {
+ TStringStream out;
+
+ {
+ auto e = BufferedEncoderJson(&out, 2);
+ e->OnStreamBegin();
+ e->OnMetricBegin(EMetricType::COUNTER);
+ e->OnLabelsBegin();
+ e->OnLabel("foo", "bar");
+ e->OnLabelsEnd();
+ // in buffered mode we are able to find values with same (in this case zero)
+ // timestamp and discard duplicates
+ e->OnUint64(TInstant::Zero(), 42);
+ e->OnUint64(TInstant::Zero(), 43);
+ e->OnUint64(TInstant::Zero(), 44);
+ e->OnUint64(TInstant::Zero(), 45);
+ e->OnMetricEnd();
+ e->OnStreamEnd();
+ }
+
+ out << "\n";
+ UNIT_ASSERT_NO_DIFF(out.Str(), NResource::Find("/buffered_ts_merge.json"));
+ }
+
+ template <typename TFactory, typename TConsumer>
+ TString EncodeToString(TFactory factory, TConsumer consumer) {
+ TStringStream out;
+ {
+ IMetricEncoderPtr e = factory(&out, 2);
+ consumer(e.Get());
+ }
+ out << '\n';
+ return out.Str();
+ }
+
+ Y_UNIT_TEST(SummaryValueEncode) {
+ auto writeDocument = [](IMetricEncoder* e) {
+ e->OnStreamBegin();
+ {
+ e->OnMetricBegin(EMetricType::DSUMMARY);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("metric", "temperature");
+ e->OnLabelsEnd();
+ }
+
+ e->OnSummaryDouble(now, MakeIntrusive<TSummaryDoubleSnapshot>(10., -0.5, 0.5, 0.3, 30u));
+ e->OnMetricEnd();
+ }
+ e->OnStreamEnd();
+ };
+
+ TString result1 = EncodeToString(EncoderJson, writeDocument);
+ UNIT_ASSERT_NO_DIFF(result1, NResource::Find("/summary_value.json"));
+
+ TString result2 = EncodeToString(BufferedEncoderJson, writeDocument);
+ UNIT_ASSERT_NO_DIFF(result2, NResource::Find("/summary_value.json"));
+ }
+
+ ISummaryDoubleSnapshotPtr TestInfSummary() {
+ return MakeIntrusive<TSummaryDoubleSnapshot>(
+ std::numeric_limits<double>::quiet_NaN(),
+ -std::numeric_limits<double>::infinity(),
+ std::numeric_limits<double>::infinity(),
+ 0.3,
+ 30u);
+ }
+
+ Y_UNIT_TEST(SummaryInfEncode) {
+ auto writeDocument = [](IMetricEncoder* e) {
+ e->OnStreamBegin();
+ {
+ e->OnMetricBegin(EMetricType::DSUMMARY);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("metric", "temperature");
+ e->OnLabelsEnd();
+ }
+
+ e->OnSummaryDouble(now, TestInfSummary());
+ e->OnMetricEnd();
+ }
+ e->OnStreamEnd();
+ };
+
+ TString result1 = EncodeToString(EncoderJson, writeDocument);
+ UNIT_ASSERT_NO_DIFF(result1, NResource::Find("/summary_inf.json"));
+
+ TString result2 = EncodeToString(BufferedEncoderJson, writeDocument);
+ UNIT_ASSERT_NO_DIFF(result2, NResource::Find("/summary_inf.json"));
+ }
+
+ Y_UNIT_TEST(SummaryInfDecode) {
+ NProto::TMultiSamplesList samples;
+ {
+ IMetricEncoderPtr e = EncoderProtobuf(&samples);
+
+ TString testJson = NResource::Find("/summary_inf.json");
+ DecodeJson(testJson, e.Get());
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(1, samples.SamplesSize());
+ const NProto::TMultiSample& s = samples.GetSamples(0);
+
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::DSUMMARY);
+ UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1);
+ AssertLabelEqual(s.GetLabels(0), "metric", "temperature");
+
+ UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1);
+
+ auto actual = s.GetPoints(0).GetSummaryDouble();
+ UNIT_ASSERT(std::isnan(actual.GetSum()));
+ UNIT_ASSERT(actual.GetMin() < 0);
+ UNIT_ASSERT(std::isinf(actual.GetMin()));
+ UNIT_ASSERT(actual.GetMax() > 0);
+ UNIT_ASSERT(std::isinf(actual.GetMax()));
+ }
+
+ Y_UNIT_TEST(SummaryValueDecode) {
+ NProto::TMultiSamplesList samples;
+ {
+ IMetricEncoderPtr e = EncoderProtobuf(&samples);
+
+ TString testJson = NResource::Find("/summary_value.json");
+ DecodeJson(testJson, e.Get());
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(1, samples.SamplesSize());
+ const NProto::TMultiSample& s = samples.GetSamples(0);
+
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::DSUMMARY);
+ UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1);
+ AssertLabelEqual(s.GetLabels(0), "metric", "temperature");
+
+ UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1);
+
+ auto snapshot = TSummaryDoubleSnapshot(10., -0.5, 0.5, 0.3, 30u);
+ AssertPointEqual(s.GetPoints(0), now, snapshot);
+ }
+
+ Y_UNIT_TEST(SummaryTimeSeriesEncode) {
+ auto writeDocument = [](IMetricEncoder* e) {
+ e->OnStreamBegin();
+ {
+ e->OnMetricBegin(EMetricType::DSUMMARY);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("metric", "temperature");
+ e->OnLabelsEnd();
+ }
+
+ TSummaryDoubleCollector summary;
+ summary.Collect(0.3);
+ summary.Collect(-0.5);
+ summary.Collect(1.);
+
+ e->OnSummaryDouble(now, summary.Snapshot());
+
+ summary.Collect(-1.5);
+ summary.Collect(0.01);
+
+ e->OnSummaryDouble(now + TDuration::Seconds(15), summary.Snapshot());
+
+ e->OnMetricEnd();
+ }
+ e->OnStreamEnd();
+ };
+
+ TString result1 = EncodeToString(EncoderJson, writeDocument);
+ UNIT_ASSERT_NO_DIFF(result1, NResource::Find("/summary_timeseries.json"));
+
+ TString result2 = EncodeToString(BufferedEncoderJson, writeDocument);
+ UNIT_ASSERT_NO_DIFF(result2, NResource::Find("/summary_timeseries.json"));
+ }
+
+ Y_UNIT_TEST(SummaryTimeSeriesDecode) {
+ NProto::TMultiSamplesList samples;
+ {
+ IMetricEncoderPtr e = EncoderProtobuf(&samples);
+
+ TString testJson = NResource::Find("/summary_timeseries.json");
+ DecodeJson(testJson, e.Get());
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(1, samples.SamplesSize());
+ const NProto::TMultiSample& s = samples.GetSamples(0);
+
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::DSUMMARY);
+ UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1);
+ AssertLabelEqual(s.GetLabels(0), "metric", "temperature");
+
+ UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 2);
+
+ TSummaryDoubleCollector summary;
+ summary.Collect(0.3);
+ summary.Collect(-0.5);
+ summary.Collect(1.);
+
+ AssertPointEqual(s.GetPoints(0), now, *summary.Snapshot());
+
+ summary.Collect(-1.5);
+ summary.Collect(0.01);
+
+ AssertPointEqual(s.GetPoints(1), now + TDuration::Seconds(15), *summary.Snapshot());
+ }
+
+ Y_UNIT_TEST(LogHistogramValueEncode) {
+ auto writeDocument = [](IMetricEncoder* e) {
+ e->OnStreamBegin();
+ {
+ e->OnMetricBegin(EMetricType::LOGHIST);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("metric", "ms");
+ e->OnLabelsEnd();
+ }
+
+ e->OnLogHistogram(now, TestLogHistogram());
+ e->OnMetricEnd();
+ }
+ e->OnStreamEnd();
+ };
+
+ TString result1 = EncodeToString(EncoderJson, writeDocument);
+ UNIT_ASSERT_NO_DIFF(result1, NResource::Find("/log_histogram_value.json"));
+
+ TString result2 = EncodeToString(BufferedEncoderJson, writeDocument);
+ UNIT_ASSERT_NO_DIFF(result2, NResource::Find("/log_histogram_value.json"));
+ }
+
+ Y_UNIT_TEST(LogHistogramValueDecode) {
+ NProto::TMultiSamplesList samples;
+ {
+ IMetricEncoderPtr e = EncoderProtobuf(&samples);
+
+ TString testJson = NResource::Find("/log_histogram_value.json");
+ DecodeJson(testJson, e.Get());
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(1, samples.SamplesSize());
+ const NProto::TMultiSample& s = samples.GetSamples(0);
+
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::LOGHISTOGRAM);
+ UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1);
+ AssertLabelEqual(s.GetLabels(0), "metric", "ms");
+
+ UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1);
+
+ auto snapshot = TestLogHistogram();
+ AssertPointEqual(s.GetPoints(0), now, *snapshot);
+ }
+
+ Y_UNIT_TEST(HistogramValueEncode) {
+ auto writeDocument = [](IMetricEncoder* e) {
+ e->OnStreamBegin();
+ {
+ e->OnMetricBegin(EMetricType::HIST);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("metric", "responseTimeMillis");
+ e->OnLabelsEnd();
+ }
+
+ // {1: 1, 2: 1, 4: 2, 8: 4, 16: 8, inf: 83}
+ auto h = ExponentialHistogram(6, 2);
+ for (i64 i = 1; i < 100; i++) {
+ h->Collect(i);
+ }
+
+ e->OnHistogram(now, h->Snapshot());
+ e->OnMetricEnd();
+ }
+ e->OnStreamEnd();
+ };
+
+ TString result1 = EncodeToString(EncoderJson, writeDocument);
+ UNIT_ASSERT_NO_DIFF(result1, NResource::Find("/histogram_value.json"));
+
+ TString result2 = EncodeToString(BufferedEncoderJson, writeDocument);
+ UNIT_ASSERT_NO_DIFF(result2, NResource::Find("/histogram_value.json"));
+ }
+
+ Y_UNIT_TEST(LogHistogramTimeSeriesEncode) {
+ auto writeDocument = [](IMetricEncoder* e) {
+ e->OnStreamBegin();
+ {
+ e->OnMetricBegin(EMetricType::LOGHIST);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("metric", "ms");
+ e->OnLabelsEnd();
+ }
+
+ e->OnLogHistogram(now, TestLogHistogram(1));;
+
+ e->OnLogHistogram(now + TDuration::Seconds(15), TestLogHistogram(2));
+
+ e->OnMetricEnd();
+ }
+ e->OnStreamEnd();
+ };
+
+ TString result1 = EncodeToString(EncoderJson, writeDocument);
+ UNIT_ASSERT_NO_DIFF(result1, NResource::Find("/log_histogram_timeseries.json"));
+
+ TString result2 = EncodeToString(BufferedEncoderJson, writeDocument);
+ UNIT_ASSERT_NO_DIFF(result2, NResource::Find("/log_histogram_timeseries.json"));
+ }
+
+ Y_UNIT_TEST(LogHistogramTimeSeriesDecode) {
+ NProto::TMultiSamplesList samples;
+ {
+ IMetricEncoderPtr e = EncoderProtobuf(&samples);
+
+ TString testJson = NResource::Find("/log_histogram_timeseries.json");
+ DecodeJson(testJson, e.Get());
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(1, samples.SamplesSize());
+ const NProto::TMultiSample& s = samples.GetSamples(0);
+
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::LOGHISTOGRAM);
+ UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1);
+ AssertLabelEqual(s.GetLabels(0), "metric", "ms");
+
+ UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 2);
+
+ auto logHist = TestLogHistogram(1);
+ AssertPointEqual(s.GetPoints(0), now, *logHist);
+
+ logHist = TestLogHistogram(2);
+ AssertPointEqual(s.GetPoints(1), now + TDuration::Seconds(15), *logHist);
+ }
+
+ void HistogramValueDecode(const TString& filePath) {
+ NProto::TMultiSamplesList samples;
+ {
+ IMetricEncoderPtr e = EncoderProtobuf(&samples);
+
+ TString testJson = NResource::Find(filePath);
+ DecodeJson(testJson, e.Get());
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(1, samples.SamplesSize());
+ const NProto::TMultiSample& s = samples.GetSamples(0);
+
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::HISTOGRAM);
+ UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1);
+ AssertLabelEqual(s.GetLabels(0), "metric", "responseTimeMillis");
+
+ UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1);
+
+ auto h = ExponentialHistogram(6, 2);
+ for (i64 i = 1; i < 100; i++) {
+ h->Collect(i);
+ }
+
+ AssertPointEqual(s.GetPoints(0), now, *h->Snapshot());
+ }
+
+ Y_UNIT_TEST(HistogramValueDecode) {
+ HistogramValueDecode("/histogram_value.json");
+ HistogramValueDecode("/histogram_value_inf_before_bounds.json");
+ }
+
+ Y_UNIT_TEST(HistogramTimeSeriesEncode) {
+ auto writeDocument = [](IMetricEncoder* e) {
+ e->OnStreamBegin();
+ {
+ e->OnMetricBegin(EMetricType::HIST_RATE);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("metric", "responseTimeMillis");
+ e->OnLabelsEnd();
+ }
+
+ // {1: 1, 2: 1, 4: 2, 8: 4, 16: 8, inf: 83}
+ auto h = ExponentialHistogram(6, 2);
+ for (i64 i = 1; i < 100; i++) {
+ h->Collect(i);
+ }
+ e->OnHistogram(now, h->Snapshot());
+
+ // {1: 2, 2: 2, 4: 4, 8: 8, 16: 16, inf: 166}
+ for (i64 i = 1; i < 100; i++) {
+ h->Collect(i);
+ }
+ e->OnHistogram(now + TDuration::Seconds(15), h->Snapshot());
+
+ e->OnMetricEnd();
+ }
+ e->OnStreamEnd();
+ };
+
+ TString result1 = EncodeToString(EncoderJson, writeDocument);
+ UNIT_ASSERT_NO_DIFF(result1, NResource::Find("/histogram_timeseries.json"));
+
+ TString result2 = EncodeToString(BufferedEncoderJson, writeDocument);
+ UNIT_ASSERT_NO_DIFF(result2, NResource::Find("/histogram_timeseries.json"));
+ }
+
+ Y_UNIT_TEST(HistogramTimeSeriesDecode) {
+ NProto::TMultiSamplesList samples;
+ {
+ IMetricEncoderPtr e = EncoderProtobuf(&samples);
+
+ TString testJson = NResource::Find("/histogram_timeseries.json");
+ DecodeJson(testJson, e.Get());
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(1, samples.SamplesSize());
+ const NProto::TMultiSample& s = samples.GetSamples(0);
+
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::HIST_RATE);
+ UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1);
+ AssertLabelEqual(s.GetLabels(0), "metric", "responseTimeMillis");
+
+ UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 2);
+
+ auto h = ExponentialHistogram(6, 2);
+ for (i64 i = 1; i < 100; i++) {
+ h->Collect(i);
+ }
+
+ AssertPointEqual(s.GetPoints(0), now, *h->Snapshot());
+
+ for (i64 i = 1; i < 100; i++) {
+ h->Collect(i);
+ }
+
+ AssertPointEqual(s.GetPoints(1), now + TDuration::Seconds(15), *h->Snapshot());
+ }
+
+ Y_UNIT_TEST(IntGaugeEncode) {
+ auto writeDocument = [](IMetricEncoder* e) {
+ e->OnStreamBegin();
+ {
+ e->OnMetricBegin(EMetricType::IGAUGE);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("metric", "a");
+ e->OnLabelsEnd();
+ }
+ e->OnInt64(now, Min<i64>());
+ e->OnInt64(now + TDuration::Seconds(1), -1);
+ e->OnInt64(now + TDuration::Seconds(2), 0);
+ e->OnInt64(now + TDuration::Seconds(3), Max<i64>());
+ e->OnMetricEnd();
+ }
+ e->OnStreamEnd();
+ };
+
+ TString result1 = EncodeToString(EncoderJson, writeDocument);
+ UNIT_ASSERT_NO_DIFF(result1, NResource::Find("/int_gauge.json"));
+
+ TString result2 = EncodeToString(BufferedEncoderJson, writeDocument);
+ UNIT_ASSERT_NO_DIFF(result2, NResource::Find("/int_gauge.json"));
+ }
+
+ Y_UNIT_TEST(InconsistentMetricTypes) {
+ auto emitMetrics = [](IMetricEncoder& encoder, const TString& expectedError) {
+ encoder.OnMetricBegin(EMetricType::GAUGE);
+ {
+ encoder.OnLabelsBegin();
+ encoder.OnLabel("name", "m");
+ encoder.OnLabel("l1", "v1");
+ encoder.OnLabel("l2", "v2");
+ encoder.OnLabelsEnd();
+ }
+ encoder.OnDouble(now, 1.0);
+ encoder.OnMetricEnd();
+
+ encoder.OnMetricBegin(EMetricType::COUNTER);
+ {
+ encoder.OnLabelsBegin();
+ encoder.OnLabel("name", "m");
+ encoder.OnLabel("l1", "v1");
+ encoder.OnLabel("l2", "v2");
+ encoder.OnLabelsEnd();
+ }
+ encoder.OnUint64(now, 1);
+
+ UNIT_ASSERT_EXCEPTION_CONTAINS(encoder.OnMetricEnd(),
+ yexception,
+ expectedError);
+ };
+
+ {
+ TStringStream out;
+ auto encoder = BufferedEncoderJson(&out);
+
+ encoder->OnStreamBegin();
+ encoder->OnLabelsBegin();
+ encoder->OnLabel("c", "cv");
+ encoder->OnLabelsEnd();
+ emitMetrics(*encoder,
+ "Time series point type mismatch: expected DOUBLE but found UINT64, "
+ "labels '{c=cv, l1=v1, l2=v2, name=m}'");
+ }
+
+ {
+ TStringStream out;
+ auto encoder = BufferedEncoderJson(&out);
+
+ encoder->OnStreamBegin();
+ encoder->OnLabelsBegin();
+ encoder->OnLabel("l1", "v100");
+ encoder->OnLabelsEnd();
+ emitMetrics(*encoder,
+ "Time series point type mismatch: expected DOUBLE but found UINT64, "
+ "labels '{l1=v1, l2=v2, name=m}'");
+ }
+ }
+
+ Y_UNIT_TEST(IntGaugeDecode) {
+ NProto::TMultiSamplesList samples;
+ {
+ IMetricEncoderPtr e = EncoderProtobuf(&samples);
+
+ TString testJson = NResource::Find("/int_gauge.json");
+ DecodeJson(testJson, e.Get());
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(1, samples.SamplesSize());
+ const NProto::TMultiSample& s = samples.GetSamples(0);
+
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::IGAUGE);
+ UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1);
+ AssertLabelEqual(s.GetLabels(0), "metric", "a");
+
+ UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 4);
+ AssertPointEqual(s.GetPoints(0), now, Min<i64>());
+ AssertPointEqual(s.GetPoints(1), now + TDuration::Seconds(1), i64(-1));
+ AssertPointEqual(s.GetPoints(2), now + TDuration::Seconds(2), i64(0));
+ AssertPointEqual(s.GetPoints(3), now + TDuration::Seconds(3), Max<i64>());
+ }
+
+ Y_UNIT_TEST(FuzzerRegression) {
+ NProto::TMultiSamplesList samples;
+ {
+ IMetricEncoderPtr e = EncoderProtobuf(&samples);
+
+ for (auto f : { "/hist_crash.json", "/crash.json" }) {
+ TString testJson = NResource::Find(f);
+ UNIT_ASSERT_EXCEPTION(DecodeJson(testJson, e.Get()), yexception);
+ }
+ }
+ }
+
+ Y_UNIT_TEST(LegacyNegativeRateThrows) {
+ const auto input = R"({
+ "sensors": [
+ {
+ "mode": "deriv",
+ "value": -1,
+ "labels": { "metric": "SystemTime" }
+ },
+ }
+ ]}")";
+
+ NProto::TMultiSamplesList samples;
+ IMetricEncoderPtr e = EncoderProtobuf(&samples);
+ UNIT_ASSERT_EXCEPTION(DecodeJson(input, e.Get()), yexception);
+ }
+
+ Y_UNIT_TEST(DecodeNamedMetrics) {
+ NProto::TMultiSamplesList samples;
+ {
+ IMetricEncoderPtr e = EncoderProtobuf(&samples);
+
+ TString metricsJson = NResource::Find("/named_metrics.json");
+ DecodeJson(metricsJson, e.Get(), "sensor");
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 2);
+ {
+ const NProto::TMultiSample& s = samples.GetSamples(0);
+ UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1);
+ AssertLabelEqual(s.GetLabels(0), "sensor", "Memory");
+ }
+ {
+ const NProto::TMultiSample& s = samples.GetSamples(1);
+ UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2);
+ AssertLabelEqual(s.GetLabels(0), "sensor", "QueueSize");
+ AssertLabelEqual(s.GetLabels(1), "export", "Oxygen");
+ }
+ }
+
+}
diff --git a/library/cpp/monlib/encode/json/typed_point.h b/library/cpp/monlib/encode/json/typed_point.h
new file mode 100644
index 0000000000..fbaa840c4b
--- /dev/null
+++ b/library/cpp/monlib/encode/json/typed_point.h
@@ -0,0 +1,123 @@
+#pragma once
+
+#include <library/cpp/monlib/metrics/metric_value.h>
+
+
+namespace NMonitoring {
+
+ class TTypedPoint {
+ public:
+ TTypedPoint()
+ : Time_(TInstant::Zero())
+ , ValueType_(EMetricValueType::UNKNOWN)
+ {
+ }
+
+ template <typename T>
+ TTypedPoint(TInstant time, T value)
+ : Time_(time)
+ , ValueType_(TValueType<T>::Type)
+ , Value_(value)
+ {
+ Ref();
+ }
+
+ ~TTypedPoint() {
+ UnRef();
+ }
+
+ TTypedPoint(const TTypedPoint& rhs)
+ : Time_(rhs.Time_)
+ , ValueType_(rhs.ValueType_)
+ , Value_(rhs.Value_)
+ {
+ Ref();
+ }
+
+ TTypedPoint& operator=(const TTypedPoint& rhs) {
+ UnRef();
+
+ Time_ = rhs.Time_;
+ ValueType_ = rhs.ValueType_;
+ Value_ = rhs.Value_;
+
+ Ref();
+ return *this;
+ }
+
+ TTypedPoint(TTypedPoint&& rhs) noexcept
+ : Time_(rhs.Time_)
+ , ValueType_(rhs.ValueType_)
+ , Value_(rhs.Value_)
+ {
+ rhs.ValueType_ = EMetricValueType::UNKNOWN;
+ rhs.Value_ = {};
+ }
+
+ TTypedPoint& operator=(TTypedPoint&& rhs) noexcept {
+ UnRef();
+
+ Time_ = rhs.Time_;
+ ValueType_ = rhs.ValueType_;
+ Value_ = rhs.Value_;
+
+ rhs.ValueType_ = EMetricValueType::UNKNOWN;
+ rhs.Value_ = {};
+
+ return *this;
+ }
+
+ TInstant GetTime() const noexcept {
+ return Time_;
+ }
+
+ void SetTime(TInstant time) noexcept {
+ Time_ = time;
+ }
+
+ TMetricValue GetValue() const noexcept {
+ return Value_;
+ }
+
+ EMetricValueType GetValueType() const noexcept {
+ return ValueType_;
+ }
+
+ template <typename T>
+ void SetValue(T value) noexcept {
+ ValueType_ = TValueType<T>::Type;
+ Value_ = TMetricValue{value};
+ }
+
+ bool HasValue() {
+ return ValueType_ != EMetricValueType::UNKNOWN;
+ }
+
+ private:
+ void Ref() {
+ if (ValueType_ == EMetricValueType::HISTOGRAM) {
+ Value_.AsHistogram()->Ref();
+ } else if (ValueType_ == EMetricValueType::SUMMARY) {
+ Value_.AsSummaryDouble()->Ref();
+ } else if (ValueType_ == EMetricValueType::LOGHISTOGRAM) {
+ Value_.AsLogHistogram()->Ref();
+ }
+ }
+
+ void UnRef() {
+ if (ValueType_ == EMetricValueType::HISTOGRAM) {
+ Value_.AsHistogram()->UnRef();
+ } else if (ValueType_ == EMetricValueType::SUMMARY) {
+ Value_.AsSummaryDouble()->UnRef();
+ } else if (ValueType_ == EMetricValueType::LOGHISTOGRAM) {
+ Value_.AsLogHistogram()->UnRef();
+ }
+ }
+
+ private:
+ TInstant Time_;
+ EMetricValueType ValueType_;
+ TMetricValue Value_;
+ };
+
+}
diff --git a/library/cpp/monlib/encode/json/ut/buffered_test.json b/library/cpp/monlib/encode/json/ut/buffered_test.json
new file mode 100644
index 0000000000..53212cf8e1
--- /dev/null
+++ b/library/cpp/monlib/encode/json/ut/buffered_test.json
@@ -0,0 +1,36 @@
+{
+ "sensors":
+ [
+ {
+ "kind":"COUNTER",
+ "labels":
+ {
+ "foo":"bar"
+ },
+ "timeseries":
+ [
+ {
+ "ts":1,
+ "value":42
+ },
+ {
+ "ts":2,
+ "value":42
+ },
+ {
+ "ts":4,
+ "value":42
+ }
+ ]
+ },
+ {
+ "kind":"COUNTER",
+ "labels":
+ {
+ "foo":"baz"
+ },
+ "ts":3,
+ "value":42
+ }
+ ]
+}
diff --git a/library/cpp/monlib/encode/json/ut/buffered_ts_merge.json b/library/cpp/monlib/encode/json/ut/buffered_ts_merge.json
new file mode 100644
index 0000000000..1d27efacb0
--- /dev/null
+++ b/library/cpp/monlib/encode/json/ut/buffered_ts_merge.json
@@ -0,0 +1,13 @@
+{
+ "sensors":
+ [
+ {
+ "kind":"COUNTER",
+ "labels":
+ {
+ "foo":"bar"
+ },
+ "value":45
+ }
+ ]
+}
diff --git a/library/cpp/monlib/encode/json/ut/crash.json b/library/cpp/monlib/encode/json/ut/crash.json
new file mode 100644
index 0000000000..8ff4369dc4
--- /dev/null
+++ b/library/cpp/monlib/encode/json/ut/crash.json
Binary files differ
diff --git a/library/cpp/monlib/encode/json/ut/empty_series.json b/library/cpp/monlib/encode/json/ut/empty_series.json
new file mode 100644
index 0000000000..641e10cdea
--- /dev/null
+++ b/library/cpp/monlib/encode/json/ut/empty_series.json
@@ -0,0 +1,12 @@
+{
+ "sensors":
+ [
+ {
+ "kind":"COUNTER",
+ "labels":
+ {
+ "foo":"bar"
+ }
+ }
+ ]
+}
diff --git a/library/cpp/monlib/encode/json/ut/expected.json b/library/cpp/monlib/encode/json/ut/expected.json
new file mode 100644
index 0000000000..ead853455b
--- /dev/null
+++ b/library/cpp/monlib/encode/json/ut/expected.json
@@ -0,0 +1,92 @@
+{
+ "ts":1500000000,
+ "commonLabels":
+ {
+ "project":"solomon"
+ },
+ "sensors":
+ [
+ {
+ "kind":"COUNTER",
+ "labels":
+ {
+ "metric":"single",
+ "labels":"l1"
+ },
+ "ts":1509843723,
+ "value":17
+ },
+ {
+ "kind":"RATE",
+ "labels":
+ {
+ "metric":"single",
+ "labels":"l2"
+ },
+ "ts":1509843723,
+ "value":17
+ },
+ {
+ "kind":"GAUGE",
+ "labels":
+ {
+ "metric":"single",
+ "labels":"l3"
+ },
+ "ts":1509843723,
+ "value":3.14
+ },
+ {
+ "kind":"IGAUGE",
+ "labels":
+ {
+ "metric":"single_igauge",
+ "labels":"l4"
+ },
+ "ts":1509843723,
+ "value":42
+ },
+ {
+ "kind":"GAUGE",
+ "labels":
+ {
+ "metric":"multiple",
+ "labels":"l5"
+ },
+ "timeseries":
+ [
+ {
+ "ts":1509843723,
+ "value":"nan"
+ },
+ {
+ "ts":1509843738,
+ "value":"inf"
+ },
+ {
+ "ts":1509843753,
+ "value":"-inf"
+ }
+ ]
+ },
+ {
+ "kind":"COUNTER",
+ "timeseries":
+ [
+ {
+ "ts":1509843723,
+ "value":1337
+ },
+ {
+ "ts":1509843738,
+ "value":1338
+ }
+ ],
+ "labels":
+ {
+ "metric":"multiple",
+ "labels":"l6"
+ }
+ }
+ ]
+}
diff --git a/library/cpp/monlib/encode/json/ut/expected_buffered.json b/library/cpp/monlib/encode/json/ut/expected_buffered.json
new file mode 100644
index 0000000000..9a6a1d6201
--- /dev/null
+++ b/library/cpp/monlib/encode/json/ut/expected_buffered.json
@@ -0,0 +1,92 @@
+{
+ "ts":1500000000,
+ "commonLabels":
+ {
+ "project":"solomon"
+ },
+ "sensors":
+ [
+ {
+ "kind":"COUNTER",
+ "labels":
+ {
+ "labels":"l1",
+ "metric":"single"
+ },
+ "ts":1509843723,
+ "value":17
+ },
+ {
+ "kind":"RATE",
+ "labels":
+ {
+ "labels":"l2",
+ "metric":"single"
+ },
+ "ts":1509843723,
+ "value":17
+ },
+ {
+ "kind":"GAUGE",
+ "labels":
+ {
+ "labels":"l3",
+ "metric":"single"
+ },
+ "ts":1509843723,
+ "value":3.14
+ },
+ {
+ "kind":"IGAUGE",
+ "labels":
+ {
+ "labels":"l4",
+ "metric":"single_igauge"
+ },
+ "ts":1509843723,
+ "value":42
+ },
+ {
+ "kind":"GAUGE",
+ "labels":
+ {
+ "labels":"l5",
+ "metric":"multiple"
+ },
+ "timeseries":
+ [
+ {
+ "ts":1509843723,
+ "value":"nan"
+ },
+ {
+ "ts":1509843738,
+ "value":"inf"
+ },
+ {
+ "ts":1509843753,
+ "value":"-inf"
+ }
+ ]
+ },
+ {
+ "kind":"COUNTER",
+ "labels":
+ {
+ "labels":"l6",
+ "metric":"multiple"
+ },
+ "timeseries":
+ [
+ {
+ "ts":1509843723,
+ "value":1337
+ },
+ {
+ "ts":1509843738,
+ "value":1338
+ }
+ ]
+ }
+ ]
+}
diff --git a/library/cpp/monlib/encode/json/ut/expected_cloud.json b/library/cpp/monlib/encode/json/ut/expected_cloud.json
new file mode 100644
index 0000000000..6184811579
--- /dev/null
+++ b/library/cpp/monlib/encode/json/ut/expected_cloud.json
@@ -0,0 +1,92 @@
+{
+ "ts":"2017-07-14T02:40:00.000000Z",
+ "labels":
+ {
+ "project":"solomon"
+ },
+ "metrics":
+ [
+ {
+ "type":"COUNTER",
+ "labels":
+ {
+ "labels":"l1"
+ },
+ "name":"single",
+ "ts":"2017-11-05T01:02:03.000000Z",
+ "value":17
+ },
+ {
+ "type":"RATE",
+ "labels":
+ {
+ "labels":"l2"
+ },
+ "name":"single",
+ "ts":"2017-11-05T01:02:03.000000Z",
+ "value":17
+ },
+ {
+ "type":"DGAUGE",
+ "labels":
+ {
+ "labels":"l3"
+ },
+ "name":"single",
+ "ts":"2017-11-05T01:02:03.000000Z",
+ "value":3.14
+ },
+ {
+ "type":"IGAUGE",
+ "labels":
+ {
+ "labels":"l4"
+ },
+ "name":"single_igauge",
+ "ts":"2017-11-05T01:02:03.000000Z",
+ "value":42
+ },
+ {
+ "type":"DGAUGE",
+ "labels":
+ {
+ "labels":"l5"
+ },
+ "name":"multiple",
+ "timeseries":
+ [
+ {
+ "ts":"2017-11-05T01:02:03.000000Z",
+ "value":"nan"
+ },
+ {
+ "ts":"2017-11-05T01:02:18.000000Z",
+ "value":"inf"
+ },
+ {
+ "ts":"2017-11-05T01:02:33.000000Z",
+ "value":"-inf"
+ }
+ ]
+ },
+ {
+ "type":"COUNTER",
+ "timeseries":
+ [
+ {
+ "ts":"2017-11-05T01:02:03.000000Z",
+ "value":1337
+ },
+ {
+ "ts":"2017-11-05T01:02:18.000000Z",
+ "value":1338
+ }
+ ],
+ "labels":
+ {
+ "labels":"l6"
+ },
+ "name":"multiple"
+ }
+ ]
+}
diff --git a/library/cpp/monlib/encode/json/ut/expected_cloud_buffered.json b/library/cpp/monlib/encode/json/ut/expected_cloud_buffered.json
new file mode 100644
index 0000000000..be237d522b
--- /dev/null
+++ b/library/cpp/monlib/encode/json/ut/expected_cloud_buffered.json
@@ -0,0 +1,92 @@
+{
+ "ts":"2017-07-14T02:40:00.000000Z",
+ "labels":
+ {
+ "project":"solomon"
+ },
+ "metrics":
+ [
+ {
+ "type":"COUNTER",
+ "labels":
+ {
+ "labels":"l1"
+ },
+ "name":"single",
+ "ts":"2017-11-05T01:02:03.000000Z",
+ "value":17
+ },
+ {
+ "type":"RATE",
+ "labels":
+ {
+ "labels":"l2"
+ },
+ "name":"single",
+ "ts":"2017-11-05T01:02:03.000000Z",
+ "value":17
+ },
+ {
+ "type":"DGAUGE",
+ "labels":
+ {
+ "labels":"l3"
+ },
+ "name":"single",
+ "ts":"2017-11-05T01:02:03.000000Z",
+ "value":3.14
+ },
+ {
+ "type":"IGAUGE",
+ "labels":
+ {
+ "labels":"l4"
+ },
+ "name":"single_igauge",
+ "ts":"2017-11-05T01:02:03.000000Z",
+ "value":42
+ },
+ {
+ "type":"DGAUGE",
+ "labels":
+ {
+ "labels":"l5"
+ },
+ "name":"multiple",
+ "timeseries":
+ [
+ {
+ "ts":"2017-11-05T01:02:03.000000Z",
+ "value":"nan"
+ },
+ {
+ "ts":"2017-11-05T01:02:18.000000Z",
+ "value":"inf"
+ },
+ {
+ "ts":"2017-11-05T01:02:33.000000Z",
+ "value":"-inf"
+ }
+ ]
+ },
+ {
+ "type":"COUNTER",
+ "labels":
+ {
+ "labels":"l6"
+ },
+ "name":"multiple",
+ "timeseries":
+ [
+ {
+ "ts":"2017-11-05T01:02:03.000000Z",
+ "value":1337
+ },
+ {
+ "ts":"2017-11-05T01:02:18.000000Z",
+ "value":1338
+ }
+ ]
+ }
+ ]
+}
diff --git a/library/cpp/monlib/encode/json/ut/hist_crash.json b/library/cpp/monlib/encode/json/ut/hist_crash.json
new file mode 100644
index 0000000000..867d0fce7d
--- /dev/null
+++ b/library/cpp/monlib/encode/json/ut/hist_crash.json
Binary files differ
diff --git a/library/cpp/monlib/encode/json/ut/histogram_timeseries.json b/library/cpp/monlib/encode/json/ut/histogram_timeseries.json
new file mode 100644
index 0000000000..f6131ffded
--- /dev/null
+++ b/library/cpp/monlib/encode/json/ut/histogram_timeseries.json
@@ -0,0 +1,61 @@
+{
+ "sensors":
+ [
+ {
+ "kind":"HIST_RATE",
+ "labels":
+ {
+ "metric":"responseTimeMillis"
+ },
+ "timeseries":
+ [
+ {
+ "ts":1509843723,
+ "hist":
+ {
+ "bounds":
+ [
+ 1,
+ 2,
+ 4,
+ 8,
+ 16
+ ],
+ "buckets":
+ [
+ 1,
+ 1,
+ 2,
+ 4,
+ 8
+ ],
+ "inf":83
+ }
+ },
+ {
+ "ts":1509843738,
+ "hist":
+ {
+ "bounds":
+ [
+ 1,
+ 2,
+ 4,
+ 8,
+ 16
+ ],
+ "buckets":
+ [
+ 2,
+ 2,
+ 4,
+ 8,
+ 16
+ ],
+ "inf":166
+ }
+ }
+ ]
+ }
+ ]
+}
diff --git a/library/cpp/monlib/encode/json/ut/histogram_value.json b/library/cpp/monlib/encode/json/ut/histogram_value.json
new file mode 100644
index 0000000000..ec1ae5cdec
--- /dev/null
+++ b/library/cpp/monlib/encode/json/ut/histogram_value.json
@@ -0,0 +1,33 @@
+{
+ "sensors":
+ [
+ {
+ "kind":"HIST",
+ "labels":
+ {
+ "metric":"responseTimeMillis"
+ },
+ "ts":1509843723,
+ "hist":
+ {
+ "bounds":
+ [
+ 1,
+ 2,
+ 4,
+ 8,
+ 16
+ ],
+ "buckets":
+ [
+ 1,
+ 1,
+ 2,
+ 4,
+ 8
+ ],
+ "inf":83
+ }
+ }
+ ]
+}
diff --git a/library/cpp/monlib/encode/json/ut/histogram_value_inf_before_bounds.json b/library/cpp/monlib/encode/json/ut/histogram_value_inf_before_bounds.json
new file mode 100644
index 0000000000..f8a17c8831
--- /dev/null
+++ b/library/cpp/monlib/encode/json/ut/histogram_value_inf_before_bounds.json
@@ -0,0 +1,33 @@
+{
+ "sensors":
+ [
+ {
+ "kind":"HIST",
+ "labels":
+ {
+ "metric":"responseTimeMillis"
+ },
+ "ts":1509843723,
+ "hist":
+ {
+ "inf":83,
+ "bounds":
+ [
+ 1,
+ 2,
+ 4,
+ 8,
+ 16
+ ],
+ "buckets":
+ [
+ 1,
+ 1,
+ 2,
+ 4,
+ 8
+ ]
+ }
+ }
+ ]
+}
diff --git a/library/cpp/monlib/encode/json/ut/int_gauge.json b/library/cpp/monlib/encode/json/ut/int_gauge.json
new file mode 100644
index 0000000000..fbe57f873c
--- /dev/null
+++ b/library/cpp/monlib/encode/json/ut/int_gauge.json
@@ -0,0 +1,31 @@
+{
+ "sensors":
+ [
+ {
+ "kind":"IGAUGE",
+ "labels":
+ {
+ "metric":"a"
+ },
+ "timeseries":
+ [
+ {
+ "ts":1509843723,
+ "value":-9223372036854775808
+ },
+ {
+ "ts":1509843724,
+ "value":-1
+ },
+ {
+ "ts":1509843725,
+ "value":0
+ },
+ {
+ "ts":1509843726,
+ "value":9223372036854775807
+ }
+ ]
+ }
+ ]
+}
diff --git a/library/cpp/monlib/encode/json/ut/log_histogram_timeseries.json b/library/cpp/monlib/encode/json/ut/log_histogram_timeseries.json
new file mode 100644
index 0000000000..e811a2cc57
--- /dev/null
+++ b/library/cpp/monlib/encode/json/ut/log_histogram_timeseries.json
@@ -0,0 +1,47 @@
+{
+ "sensors":
+ [
+ {
+ "kind":"LOGHIST",
+ "labels":
+ {
+ "metric":"ms"
+ },
+ "timeseries":
+ [
+ {
+ "ts":1509843723,
+ "log_hist":
+ {
+ "base":1.5,
+ "zeros_count":1,
+ "start_power":0,
+ "buckets":
+ [
+ 0.5,
+ 0.25,
+ 0.25,
+ 0.5
+ ]
+ }
+ },
+ {
+ "ts":1509843738,
+ "log_hist":
+ {
+ "base":1.5,
+ "zeros_count":1,
+ "start_power":0,
+ "buckets":
+ [
+ 1,
+ 0.5,
+ 0.5,
+ 1
+ ]
+ }
+ }
+ ]
+ }
+ ]
+}
diff --git a/library/cpp/monlib/encode/json/ut/log_histogram_value.json b/library/cpp/monlib/encode/json/ut/log_histogram_value.json
new file mode 100644
index 0000000000..002478293b
--- /dev/null
+++ b/library/cpp/monlib/encode/json/ut/log_histogram_value.json
@@ -0,0 +1,26 @@
+{
+ "sensors":
+ [
+ {
+ "kind":"LOGHIST",
+ "labels":
+ {
+ "metric":"ms"
+ },
+ "ts":1509843723,
+ "log_hist":
+ {
+ "base":1.5,
+ "zeros_count":1,
+ "start_power":0,
+ "buckets":
+ [
+ 0.5,
+ 0.25,
+ 0.25,
+ 0.5
+ ]
+ }
+ }
+ ]
+}
diff --git a/library/cpp/monlib/encode/json/ut/merged.json b/library/cpp/monlib/encode/json/ut/merged.json
new file mode 100644
index 0000000000..ea2c99a33c
--- /dev/null
+++ b/library/cpp/monlib/encode/json/ut/merged.json
@@ -0,0 +1,14 @@
+{
+ "sensors":
+ [
+ {
+ "kind":"RATE",
+ "labels":
+ {
+ "metric":"hello",
+ "label":"world"
+ },
+ "value":1
+ }
+ ]
+}
diff --git a/library/cpp/monlib/encode/json/ut/metrics.json b/library/cpp/monlib/encode/json/ut/metrics.json
new file mode 100644
index 0000000000..2be4617d51
--- /dev/null
+++ b/library/cpp/monlib/encode/json/ut/metrics.json
@@ -0,0 +1,43 @@
+{
+ "labels": {
+ "project": "solomon",
+ "cluster": "man",
+ "service": "stockpile"
+ },
+ "metrics": [
+ {
+ "type": "DGAUGE",
+ "labels": {
+ "metric": "Memory"
+ },
+ "value": 10
+ },
+ {
+ "type": "RATE",
+ "value": 1,
+ "labels": { "metric": "UserTime" }
+ },
+ {
+ "type": "GAUGE",
+ "value": 3.14159,
+ "labels": { "export": "Oxygen", "metric": "QueueSize" },
+ "ts": "2017-11-05T12:34:56.000Z",
+ "memOnly": true
+ },
+ {
+ "type": "GAUGE",
+ "labels": { "metric": "Writes" },
+ "timeseries": [
+ {
+ "ts": "2017-08-28T12:32:11Z",
+ "value": -10
+ },
+ {
+ "value": 20,
+ "ts": 1503923187
+ }
+ ]
+ }
+ ],
+ "ts": "2017-08-27T12:34:56Z"
+}
diff --git a/library/cpp/monlib/encode/json/ut/named_metrics.json b/library/cpp/monlib/encode/json/ut/named_metrics.json
new file mode 100644
index 0000000000..98f93e8c39
--- /dev/null
+++ b/library/cpp/monlib/encode/json/ut/named_metrics.json
@@ -0,0 +1,22 @@
+{
+ "labels": {
+ "project": "solomon",
+ "cluster": "prod-sas",
+ "service": "stockpile"
+ },
+ "metrics": [
+ {
+ "type": "DGAUGE",
+ "name": "Memory",
+ "value": 1
+ },
+ {
+ "type": "DGAUGE",
+ "name": "QueueSize",
+ "labels": {
+ "export": "Oxygen"
+ },
+ "value": 10
+ }
+ ]
+}
diff --git a/library/cpp/monlib/encode/json/ut/sensors.json b/library/cpp/monlib/encode/json/ut/sensors.json
new file mode 100644
index 0000000000..4d979a3c1e
--- /dev/null
+++ b/library/cpp/monlib/encode/json/ut/sensors.json
@@ -0,0 +1,40 @@
+{
+ "commonLabels": {
+ "project": "solomon",
+ "cluster": "man",
+ "service": "stockpile"
+ },
+ "sensors": [
+ {
+ "labels": {
+ "metric": "Memory"
+ },
+ "value": 10
+ },
+ {
+ "mode": "deriv",
+ "value": 1,
+ "labels": { "metric": "UserTime" }
+ },
+ {
+ "value": 3.14159,
+ "labels": { "export": "Oxygen", "metric": "QueueSize" },
+ "ts": "2017-11-05T12:34:56.000Z",
+ "memOnly": true
+ },
+ {
+ "labels": { "metric": "Writes" },
+ "timeseries": [
+ {
+ "ts": "2017-08-28T12:32:11Z",
+ "value": -10
+ },
+ {
+ "value": 20,
+ "ts": 1503923187
+ }
+ ]
+ }
+ ],
+ "ts": "2017-08-27T12:34:56Z"
+}
diff --git a/library/cpp/monlib/encode/json/ut/summary_inf.json b/library/cpp/monlib/encode/json/ut/summary_inf.json
new file mode 100644
index 0000000000..625a6cd8ad
--- /dev/null
+++ b/library/cpp/monlib/encode/json/ut/summary_inf.json
@@ -0,0 +1,21 @@
+{
+ "sensors":
+ [
+ {
+ "kind":"DSUMMARY",
+ "labels":
+ {
+ "metric":"temperature"
+ },
+ "ts":1509843723,
+ "summary":
+ {
+ "sum":"nan",
+ "min":"-inf",
+ "max":"inf",
+ "last":0.3,
+ "count":30
+ }
+ }
+ ]
+}
diff --git a/library/cpp/monlib/encode/json/ut/summary_timeseries.json b/library/cpp/monlib/encode/json/ut/summary_timeseries.json
new file mode 100644
index 0000000000..92007af3e6
--- /dev/null
+++ b/library/cpp/monlib/encode/json/ut/summary_timeseries.json
@@ -0,0 +1,37 @@
+{
+ "sensors":
+ [
+ {
+ "kind":"DSUMMARY",
+ "labels":
+ {
+ "metric":"temperature"
+ },
+ "timeseries":
+ [
+ {
+ "ts":1509843723,
+ "summary":
+ {
+ "sum":0.8,
+ "min":-0.5,
+ "max":1,
+ "last":1,
+ "count":3
+ }
+ },
+ {
+ "ts":1509843738,
+ "summary":
+ {
+ "sum":-0.69,
+ "min":-1.5,
+ "max":1,
+ "last":0.01,
+ "count":5
+ }
+ }
+ ]
+ }
+ ]
+}
diff --git a/library/cpp/monlib/encode/json/ut/summary_value.json b/library/cpp/monlib/encode/json/ut/summary_value.json
new file mode 100644
index 0000000000..366394c5e1
--- /dev/null
+++ b/library/cpp/monlib/encode/json/ut/summary_value.json
@@ -0,0 +1,21 @@
+{
+ "sensors":
+ [
+ {
+ "kind":"DSUMMARY",
+ "labels":
+ {
+ "metric":"temperature"
+ },
+ "ts":1509843723,
+ "summary":
+ {
+ "sum":10,
+ "min":-0.5,
+ "max":0.5,
+ "last":0.3,
+ "count":30
+ }
+ }
+ ]
+}
diff --git a/library/cpp/monlib/encode/json/ut/test_decode_to_encode.json b/library/cpp/monlib/encode/json/ut/test_decode_to_encode.json
new file mode 100644
index 0000000000..65f0c5c6e2
--- /dev/null
+++ b/library/cpp/monlib/encode/json/ut/test_decode_to_encode.json
@@ -0,0 +1,16 @@
+{
+ "commonLabels": {
+ "project": "solomon",
+ "cluster": "man",
+ "service": "stockpile"
+ },
+ "sensors": [
+ {
+ "kind": "GAUGE",
+ "labels": { "export": "Oxygen", "metric": "QueueSize" },
+ "ts": 1509885296,
+ "value": 3.14159
+ }
+ ],
+ "ts": 1503837296
+}
diff --git a/library/cpp/monlib/encode/json/ut/ya.make b/library/cpp/monlib/encode/json/ut/ya.make
new file mode 100644
index 0000000000..e50c4f4903
--- /dev/null
+++ b/library/cpp/monlib/encode/json/ut/ya.make
@@ -0,0 +1,46 @@
+UNITTEST_FOR(library/cpp/monlib/encode/json)
+
+OWNER(
+ g:solomon
+ jamel
+)
+
+SRCS(
+ json_decoder_ut.cpp
+ json_ut.cpp
+)
+
+RESOURCE(
+ buffered_test.json /buffered_test.json
+ buffered_ts_merge.json /buffered_ts_merge.json
+ empty_series.json /empty_series.json
+ expected.json /expected.json
+ expected_buffered.json /expected_buffered.json
+ expected_cloud.json /expected_cloud.json
+ expected_cloud_buffered.json /expected_cloud_buffered.json
+ merged.json /merged.json
+ histogram_timeseries.json /histogram_timeseries.json
+ histogram_value.json /histogram_value.json
+ histogram_value_inf_before_bounds.json /histogram_value_inf_before_bounds.json
+ int_gauge.json /int_gauge.json
+ sensors.json /sensors.json
+ metrics.json /metrics.json
+ named_metrics.json /named_metrics.json
+ test_decode_to_encode.json /test_decode_to_encode.json
+ crash.json /crash.json
+ hist_crash.json /hist_crash.json
+ summary_value.json /summary_value.json
+ summary_inf.json /summary_inf.json
+ summary_timeseries.json /summary_timeseries.json
+ log_histogram_value.json /log_histogram_value.json
+ log_histogram_timeseries.json /log_histogram_timeseries.json
+)
+
+PEERDIR(
+ library/cpp/json
+ library/cpp/monlib/consumers
+ library/cpp/monlib/encode/protobuf
+ library/cpp/resource
+)
+
+END()
diff --git a/library/cpp/monlib/encode/json/ya.make b/library/cpp/monlib/encode/json/ya.make
new file mode 100644
index 0000000000..a50fc412a9
--- /dev/null
+++ b/library/cpp/monlib/encode/json/ya.make
@@ -0,0 +1,21 @@
+LIBRARY()
+
+OWNER(
+ g:solomon
+ jamel
+)
+
+SRCS(
+ json_decoder.cpp
+ json_encoder.cpp
+)
+
+PEERDIR(
+ library/cpp/monlib/encode
+ library/cpp/monlib/encode/buffered
+ library/cpp/monlib/exception
+ library/cpp/json
+ library/cpp/json/writer
+)
+
+END()
diff --git a/library/cpp/monlib/encode/legacy_protobuf/legacy_proto_decoder.cpp b/library/cpp/monlib/encode/legacy_protobuf/legacy_proto_decoder.cpp
new file mode 100644
index 0000000000..f87a2d7e8f
--- /dev/null
+++ b/library/cpp/monlib/encode/legacy_protobuf/legacy_proto_decoder.cpp
@@ -0,0 +1,527 @@
+#include "legacy_protobuf.h"
+
+#include <library/cpp/monlib/encode/legacy_protobuf/protos/metric_meta.pb.h>
+#include <library/cpp/monlib/metrics/metric_consumer.h>
+#include <library/cpp/monlib/metrics/labels.h>
+
+#include <util/generic/yexception.h>
+#include <util/generic/maybe.h>
+#include <util/datetime/base.h>
+#include <util/string/split.h>
+
+#include <google/protobuf/reflection.h>
+
+#include <algorithm>
+
+#ifdef LEGACY_PB_TRACE
+#define TRACE(msg) \
+ Cerr << msg << Endl
+#else
+#define TRACE(...) ;
+#endif
+
+namespace NMonitoring {
+ namespace {
+ using TMaybeMeta = TMaybe<NMonProto::TMetricMeta>;
+
+ TString ReadLabelValue(const NProtoBuf::Message& msg, const NProtoBuf::FieldDescriptor* d, const NProtoBuf::Reflection& r) {
+ using namespace NProtoBuf;
+
+ switch (d->type()) {
+ case FieldDescriptor::TYPE_UINT32:
+ return ::ToString(r.GetUInt32(msg, d));
+ case FieldDescriptor::TYPE_UINT64:
+ return ::ToString(r.GetUInt64(msg, d));
+ case FieldDescriptor::TYPE_STRING:
+ return r.GetString(msg, d);
+ case FieldDescriptor::TYPE_ENUM: {
+ auto val = r.GetEnumValue(msg, d);
+ auto* valDesc = d->enum_type()->FindValueByNumber(val);
+ return valDesc->name();
+ }
+
+ default:
+ ythrow yexception() << "type " << d->type_name() << " cannot be used as a field value";
+ }
+
+ return {};
+ }
+
+ double ReadFieldAsDouble(const NProtoBuf::Message& msg, const NProtoBuf::FieldDescriptor* d, const NProtoBuf::Reflection& r) {
+ using namespace NProtoBuf;
+
+ switch (d->type()) {
+ case FieldDescriptor::TYPE_DOUBLE:
+ return r.GetDouble(msg, d);
+ case FieldDescriptor::TYPE_BOOL:
+ return r.GetBool(msg, d) ? 1 : 0;
+ case FieldDescriptor::TYPE_INT32:
+ return r.GetInt32(msg, d);
+ case FieldDescriptor::TYPE_INT64:
+ return r.GetInt64(msg, d);
+ case FieldDescriptor::TYPE_UINT32:
+ return r.GetUInt32(msg, d);
+ case FieldDescriptor::TYPE_UINT64:
+ return r.GetUInt64(msg, d);
+ case FieldDescriptor::TYPE_SINT32:
+ return r.GetInt32(msg, d);
+ case FieldDescriptor::TYPE_SINT64:
+ return r.GetInt64(msg, d);
+ case FieldDescriptor::TYPE_FIXED32:
+ return r.GetUInt32(msg, d);
+ case FieldDescriptor::TYPE_FIXED64:
+ return r.GetUInt64(msg, d);
+ case FieldDescriptor::TYPE_SFIXED32:
+ return r.GetInt32(msg, d);
+ case FieldDescriptor::TYPE_SFIXED64:
+ return r.GetInt64(msg, d);
+ case FieldDescriptor::TYPE_FLOAT:
+ return r.GetFloat(msg, d);
+ case FieldDescriptor::TYPE_ENUM:
+ return r.GetEnumValue(msg, d);
+ default:
+ ythrow yexception() << "type " << d->type_name() << " cannot be used as a field value";
+ }
+
+ return std::numeric_limits<double>::quiet_NaN();
+ }
+
+ double ReadRepeatedAsDouble(const NProtoBuf::Message& msg, const NProtoBuf::FieldDescriptor* d, const NProtoBuf::Reflection& r, size_t i) {
+ using namespace NProtoBuf;
+
+ switch (d->type()) {
+ case FieldDescriptor::TYPE_DOUBLE:
+ return r.GetRepeatedDouble(msg, d, i);
+ case FieldDescriptor::TYPE_BOOL:
+ return r.GetRepeatedBool(msg, d, i) ? 1 : 0;
+ case FieldDescriptor::TYPE_INT32:
+ return r.GetRepeatedInt32(msg, d, i);
+ case FieldDescriptor::TYPE_INT64:
+ return r.GetRepeatedInt64(msg, d, i);
+ case FieldDescriptor::TYPE_UINT32:
+ return r.GetRepeatedUInt32(msg, d, i);
+ case FieldDescriptor::TYPE_UINT64:
+ return r.GetRepeatedUInt64(msg, d, i);
+ case FieldDescriptor::TYPE_SINT32:
+ return r.GetRepeatedInt32(msg, d, i);
+ case FieldDescriptor::TYPE_SINT64:
+ return r.GetRepeatedInt64(msg, d, i);
+ case FieldDescriptor::TYPE_FIXED32:
+ return r.GetRepeatedUInt32(msg, d, i);
+ case FieldDescriptor::TYPE_FIXED64:
+ return r.GetRepeatedUInt64(msg, d, i);
+ case FieldDescriptor::TYPE_SFIXED32:
+ return r.GetRepeatedInt32(msg, d, i);
+ case FieldDescriptor::TYPE_SFIXED64:
+ return r.GetRepeatedInt64(msg, d, i);
+ case FieldDescriptor::TYPE_FLOAT:
+ return r.GetRepeatedFloat(msg, d, i);
+ case FieldDescriptor::TYPE_ENUM:
+ return r.GetRepeatedEnumValue(msg, d, i);
+ default:
+ ythrow yexception() << "type " << d->type_name() << " cannot be used as a field value";
+ }
+
+ return std::numeric_limits<double>::quiet_NaN();
+ }
+
+ TString LabelFromField(const NProtoBuf::Message& msg, const TString& name) {
+ const auto* fieldDesc = msg.GetDescriptor()->FindFieldByName(name);
+ const auto* reflection = msg.GetReflection();
+ Y_ENSURE(fieldDesc && reflection, "Unable to get meta for field " << name);
+
+ auto s = ReadLabelValue(msg, fieldDesc, *reflection);
+ std::replace(std::begin(s), s.vend(), ' ', '_');
+
+ return s;
+ }
+
+ TMaybeMeta MaybeGetMeta(const NProtoBuf::FieldOptions& opts) {
+ if (opts.HasExtension(NMonProto::Metric)) {
+ return opts.GetExtension(NMonProto::Metric);
+ }
+
+ return Nothing();
+ }
+
+ class ILabelGetter: public TThrRefBase {
+ public:
+ enum class EType {
+ Fixed = 1,
+ Lazy = 2,
+ };
+
+ virtual TLabel Get(const NProtoBuf::Message&) = 0;
+ virtual EType Type() const = 0;
+ };
+
+ class TFixedLabel: public ILabelGetter {
+ public:
+ explicit TFixedLabel(TLabel&& l)
+ : Label_{std::move(l)}
+ {
+ TRACE("found fixed label " << l);
+ }
+
+ EType Type() const override {
+ return EType::Fixed;
+ }
+ TLabel Get(const NProtoBuf::Message&) override {
+ return Label_;
+ }
+
+ private:
+ TLabel Label_;
+ };
+
+ using TFunction = std::function<TLabel(const NProtoBuf::Message&)>;
+
+ class TLazyLabel: public ILabelGetter {
+ public:
+ TLazyLabel(TFunction&& fn)
+ : Fn_{std::move(fn)}
+ {
+ TRACE("found lazy label");
+ }
+
+ EType Type() const override {
+ return EType::Lazy;
+ }
+ TLabel Get(const NProtoBuf::Message& msg) override {
+ return Fn_(msg);
+ }
+
+ private:
+ TFunction Fn_;
+ };
+
+ class TDecoderContext {
+ public:
+ void Init(const NProtoBuf::Message* msg) {
+ Message_ = msg;
+ Y_ENSURE(Message_);
+ Reflection_ = msg->GetReflection();
+ Y_ENSURE(Reflection_);
+
+ for (auto it = Labels_.begin(); it != Labels_.end(); ++it) {
+ if ((*it)->Type() == ILabelGetter::EType::Lazy) {
+ auto l = (*it)->Get(Message());
+ *it = ::MakeIntrusive<TFixedLabel>(std::move(l));
+ } else {
+ auto l = (*it)->Get(Message());
+ }
+ }
+ }
+
+ void Clear() noexcept {
+ Message_ = nullptr;
+ Reflection_ = nullptr;
+ }
+
+ TDecoderContext CreateChildFromMeta(const NMonProto::TMetricMeta& metricMeta, const TString& name, i64 repeatedIdx = -1) {
+ TDecoderContext child{*this};
+ child.Clear();
+
+ if (metricMeta.HasCustomPath()) {
+ if (const auto& nodePath = metricMeta.GetCustomPath()) {
+ child.AppendPath(nodePath);
+ }
+ } else if (metricMeta.GetPath()) {
+ child.AppendPath(name);
+ }
+
+ if (metricMeta.HasKeys()) {
+ child.ParseKeys(metricMeta.GetKeys(), repeatedIdx);
+ }
+
+ return child;
+ }
+
+ TDecoderContext CreateChildFromRepeatedScalar(const NMonProto::TMetricMeta& metricMeta, i64 repeatedIdx = -1) {
+ TDecoderContext child{*this};
+ child.Clear();
+
+ if (metricMeta.HasKeys()) {
+ child.ParseKeys(metricMeta.GetKeys(), repeatedIdx);
+ }
+
+ return child;
+ }
+
+ TDecoderContext CreateChildFromEls(const TString& name, const NMonProto::TExtraLabelMetrics& metrics, size_t idx, TMaybeMeta maybeMeta) {
+ TDecoderContext child{*this};
+ child.Clear();
+
+ auto usePath = [&maybeMeta] {
+ return !maybeMeta->HasPath() || maybeMeta->GetPath();
+ };
+
+ if (!name.empty() && (!maybeMeta || usePath())) {
+ child.AppendPath(name);
+ }
+
+ child.Labels_.push_back(::MakeIntrusive<TLazyLabel>(
+ [ labelName = metrics.GetlabelName(), idx, &metrics ](const auto&) {
+ const auto& val = metrics.Getvalues(idx);
+ TString labelVal;
+ const auto uintLabel = val.GetlabelValueUint();
+
+ if (uintLabel) {
+ labelVal = ::ToString(uintLabel);
+ } else {
+ labelVal = val.GetlabelValue();
+ }
+
+ return TLabel{labelName, labelVal};
+ }));
+
+ return child;
+ }
+
+ void ParseKeys(TStringBuf keys, i64 repeatedIdx = -1) {
+ auto parts = StringSplitter(keys)
+ .Split(' ')
+ .SkipEmpty();
+
+ for (auto part : parts) {
+ auto str = part.Token();
+
+ TStringBuf lhs, rhs;
+
+ const bool isDynamic = str.TrySplit(':', lhs, rhs);
+ const bool isIndexing = isDynamic && rhs == TStringBuf("#");
+
+ if (isIndexing) {
+ TRACE("parsed index labels");
+
+ // <label_name>:# means that we should use index of the repeated
+ // field as label value
+ Y_ENSURE(repeatedIdx != -1);
+ Labels_.push_back(::MakeIntrusive<TLazyLabel>([=](const auto&) {
+ return TLabel{lhs, ::ToString(repeatedIdx)};
+ }));
+ } else if (isDynamic) {
+ TRACE("parsed dynamic labels");
+
+ // <label_name>:<field_name> means that we need to take label value
+ // later from message's field
+ Labels_.push_back(::MakeIntrusive<TLazyLabel>([=](const auto& msg) {
+ return TLabel{lhs, LabelFromField(msg, TString{rhs})};
+ }));
+ } else if (str.TrySplit('=', lhs, rhs)) {
+ TRACE("parsed static labels");
+
+ // <label_name>=<label_value> stands for constant label
+ Labels_.push_back(::MakeIntrusive<TFixedLabel>(TLabel{lhs, rhs}));
+ } else {
+ ythrow yexception() << "Incorrect Keys format";
+ }
+ }
+ }
+
+ void AppendPath(TStringBuf fieldName) {
+ Path_ += '/';
+ Path_ += fieldName;
+ }
+
+ const TString& Path() const {
+ return Path_;
+ }
+
+ TLabels Labels() const {
+ TLabels result;
+ for (auto&& l : Labels_) {
+ result.Add(l->Get(Message()));
+ }
+
+ return result;
+ }
+
+ const NProtoBuf::Message& Message() const {
+ Y_VERIFY_DEBUG(Message_);
+ return *Message_;
+ }
+
+ const NProtoBuf::Reflection& Reflection() const {
+ return *Reflection_;
+ }
+
+ private:
+ const NProtoBuf::Message* Message_{nullptr};
+ const NProtoBuf::Reflection* Reflection_{nullptr};
+
+ TString Path_;
+ TVector<TIntrusivePtr<ILabelGetter>> Labels_;
+ };
+
+ class TDecoder {
+ public:
+ TDecoder(IMetricConsumer* consumer, const NProtoBuf::Message& message, TInstant timestamp)
+ : Consumer_{consumer}
+ , Message_{message}
+ , Timestamp_{timestamp}
+ {
+ }
+
+ void Decode() const {
+ Consumer_->OnStreamBegin();
+ DecodeToStream();
+ Consumer_->OnStreamEnd();
+ }
+
+ void DecodeToStream() const {
+ DecodeImpl(Message_, {});
+ }
+
+ private:
+ static const NMonProto::TExtraLabelMetrics& ExtractExtraMetrics(TDecoderContext& ctx, const NProtoBuf::FieldDescriptor& f) {
+ const auto& parent = ctx.Message();
+ const auto& reflection = ctx.Reflection();
+ auto& subMessage = reflection.GetMessage(parent, &f);
+
+ return dynamic_cast<const NMonProto::TExtraLabelMetrics&>(subMessage);
+ }
+
+ void DecodeImpl(const NProtoBuf::Message& msg, TDecoderContext ctx) const {
+ std::vector<const NProtoBuf::FieldDescriptor*> fields;
+
+ ctx.Init(&msg);
+
+ ctx.Reflection().ListFields(msg, &fields);
+
+ for (const auto* f : fields) {
+ Y_ENSURE(f);
+
+ const auto& opts = f->options();
+ const auto isMessage = f->type() == NProtoBuf::FieldDescriptor::TYPE_MESSAGE;
+ const auto isExtraLabelMetrics = isMessage && f->message_type()->full_name() == "NMonProto.TExtraLabelMetrics";
+ const auto maybeMeta = MaybeGetMeta(opts);
+
+ if (!(maybeMeta || isExtraLabelMetrics)) {
+ continue;
+ }
+
+ if (isExtraLabelMetrics) {
+ const auto& extra = ExtractExtraMetrics(ctx, *f);
+ RecurseExtraLabelMetrics(ctx, extra, f->name(), maybeMeta);
+ } else if (isMessage) {
+ RecurseMessage(ctx, *maybeMeta, *f);
+ } else if (f->is_repeated()) {
+ RecurseRepeatedScalar(ctx, *maybeMeta, *f);
+ } else if (maybeMeta->HasType()) {
+ const auto val = ReadFieldAsDouble(msg, f, ctx.Reflection());
+ const bool isRate = maybeMeta->GetType() == NMonProto::EMetricType::RATE;
+ WriteMetric(val, ctx, f->name(), isRate);
+ }
+ }
+ }
+
+ void RecurseRepeatedScalar(TDecoderContext ctx, const NMonProto::TMetricMeta& meta, const NProtoBuf::FieldDescriptor& f) const {
+ auto&& msg = ctx.Message();
+ auto&& reflection = ctx.Reflection();
+ const bool isRate = meta.GetType() == NMonProto::EMetricType::RATE;
+
+ // this is a repeated scalar field, which makes metric only if it's indexing
+ for (auto i = 0; i < reflection.FieldSize(msg, &f); ++i) {
+ auto subCtx = ctx.CreateChildFromRepeatedScalar(meta, i);
+ subCtx.Init(&msg);
+ auto val = ReadRepeatedAsDouble(msg, &f, reflection, i);
+ WriteMetric(val, subCtx, f.name(), isRate);
+ }
+ }
+
+ void RecurseExtraLabelMetrics(TDecoderContext ctx, const NMonProto::TExtraLabelMetrics& msg, const TString& name, const TMaybeMeta& meta) const {
+ auto i = 0;
+ for (const auto& val : msg.Getvalues()) {
+ auto subCtx = ctx.CreateChildFromEls(name, msg, i++, meta);
+ subCtx.Init(&val);
+
+ const bool isRate = val.Hastype()
+ ? val.Gettype() == NMonProto::EMetricType::RATE
+ : meta->GetType() == NMonProto::EMetricType::RATE;
+
+ double metricVal{0};
+ if (isRate) {
+ metricVal = val.GetlongValue();
+ } else {
+ metricVal = val.GetdoubleValue();
+ }
+
+ WriteMetric(metricVal, subCtx, "", isRate);
+
+ for (const auto& child : val.Getchildren()) {
+ RecurseExtraLabelMetrics(subCtx, child, "", meta);
+ }
+ }
+ }
+
+ void RecurseMessage(TDecoderContext ctx, const NMonProto::TMetricMeta& metricMeta, const NProtoBuf::FieldDescriptor& f) const {
+ const auto& msg = ctx.Message();
+ const auto& reflection = ctx.Reflection();
+
+ if (f.is_repeated()) {
+ TRACE("recurse into repeated message " << f.name());
+ for (auto i = 0; i < reflection.FieldSize(msg, &f); ++i) {
+ auto& subMessage = reflection.GetRepeatedMessage(msg, &f, i);
+ DecodeImpl(subMessage, ctx.CreateChildFromMeta(metricMeta, f.name(), i));
+ }
+ } else {
+ TRACE("recurse into message " << f.name());
+ auto& subMessage = reflection.GetMessage(msg, &f);
+ DecodeImpl(subMessage, ctx.CreateChildFromMeta(metricMeta, f.name()));
+ }
+ }
+
+ inline void WriteValue(ui64 value) const {
+ Consumer_->OnUint64(Timestamp_, value);
+ }
+
+ inline void WriteValue(double value) const {
+ Consumer_->OnDouble(Timestamp_, value);
+ }
+
+ void WriteMetric(double value, const TDecoderContext& ctx, const TString& name, bool isRate) const {
+ if (isRate) {
+ Consumer_->OnMetricBegin(EMetricType::RATE);
+ WriteValue(static_cast<ui64>(value));
+ } else {
+ Consumer_->OnMetricBegin(EMetricType::GAUGE);
+ WriteValue(static_cast<double>(value));
+ }
+
+ Consumer_->OnLabelsBegin();
+
+ for (const auto& label : ctx.Labels()) {
+ Consumer_->OnLabel(label.Name(), label.Value());
+ }
+
+ const auto fullPath = name.empty()
+ ? ctx.Path()
+ : ctx.Path() + '/' + name;
+
+ if (fullPath) {
+ Consumer_->OnLabel("path", fullPath);
+ }
+
+ Consumer_->OnLabelsEnd();
+ Consumer_->OnMetricEnd();
+ }
+
+ private:
+ IMetricConsumer* Consumer_{nullptr};
+ const NProtoBuf::Message& Message_;
+ TInstant Timestamp_;
+ };
+
+ }
+
+ void DecodeLegacyProto(const NProtoBuf::Message& data, IMetricConsumer* consumer, TInstant ts) {
+ Y_ENSURE(consumer);
+ TDecoder(consumer, data, ts).Decode();
+ }
+
+ void DecodeLegacyProtoToStream(const NProtoBuf::Message& data, IMetricConsumer* consumer, TInstant ts) {
+ Y_ENSURE(consumer);
+ TDecoder(consumer, data, ts).DecodeToStream();
+ }
+}
diff --git a/library/cpp/monlib/encode/legacy_protobuf/legacy_protobuf.h b/library/cpp/monlib/encode/legacy_protobuf/legacy_protobuf.h
new file mode 100644
index 0000000000..7cf8985d65
--- /dev/null
+++ b/library/cpp/monlib/encode/legacy_protobuf/legacy_protobuf.h
@@ -0,0 +1,16 @@
+#pragma once
+
+#include <google/protobuf/message.h>
+#include <util/datetime/base.h>
+
+namespace NMonitoring {
+ // Unsupported features of the original format:
+ // - histograms;
+ // - memOnly;
+ // - dropHost/ignorePath
+
+ void DecodeLegacyProto(const NProtoBuf::Message& data, class IMetricConsumer* c, TInstant ts = TInstant::Zero());
+
+ /// Does not open/close consumer stream unlike the above function.
+ void DecodeLegacyProtoToStream(const NProtoBuf::Message& data, class IMetricConsumer* c, TInstant ts = TInstant::Zero());
+}
diff --git a/library/cpp/monlib/encode/legacy_protobuf/legacy_protobuf_ut.cpp b/library/cpp/monlib/encode/legacy_protobuf/legacy_protobuf_ut.cpp
new file mode 100644
index 0000000000..53683cb39c
--- /dev/null
+++ b/library/cpp/monlib/encode/legacy_protobuf/legacy_protobuf_ut.cpp
@@ -0,0 +1,422 @@
+#include "legacy_protobuf.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <library/cpp/monlib/encode/legacy_protobuf/ut/test_cases.pb.h>
+#include <library/cpp/monlib/encode/legacy_protobuf/protos/metric_meta.pb.h>
+
+#include <library/cpp/monlib/encode/protobuf/protobuf.h>
+#include <library/cpp/monlib/encode/text/text.h>
+#include <library/cpp/monlib/metrics/labels.h>
+
+#include <util/generic/algorithm.h>
+#include <util/generic/hash_set.h>
+
+using namespace NMonitoring;
+
+TSimple MakeSimpleMessage() {
+ TSimple msg;
+
+ msg.SetFoo(1);
+ msg.SetBar(2.);
+ msg.SetBaz(42.);
+
+ return msg;
+}
+
+IMetricEncoderPtr debugPrinter = EncoderText(&Cerr);
+
+namespace NMonitoring {
+ inline bool operator<(const TLabel& lhs, const TLabel& rhs) {
+ return lhs.Name() < rhs.Name() ||
+ (lhs.Name() == rhs.Name() && lhs.Value() < rhs.Value());
+ }
+
+}
+
+void SetLabelValue(NMonProto::TExtraLabelMetrics::TValue& val, TString s) {
+ val.SetlabelValue(s);
+}
+
+void SetLabelValue(NMonProto::TExtraLabelMetrics::TValue& val, ui64 u) {
+ val.SetlabelValueUint(u);
+}
+
+template <typename T, typename V>
+NMonProto::TExtraLabelMetrics MakeExtra(TString labelName, V labelValue, T value, bool isDeriv) {
+ NMonProto::TExtraLabelMetrics metric;
+ auto* val = metric.Addvalues();
+
+ metric.SetlabelName(labelName);
+ SetLabelValue(*val, labelValue);
+
+ if (isDeriv) {
+ val->SetlongValue(value);
+ } else {
+ val->SetdoubleValue(value);
+ }
+
+ return metric;
+}
+
+void AssertLabels(const TLabels& expected, const NProto::TMultiSample& actual) {
+ UNIT_ASSERT_EQUAL(actual.LabelsSize(), expected.Size());
+
+ TSet<TLabel> actualSet;
+ TSet<TLabel> expectedSet;
+ Transform(expected.begin(), expected.end(), std::inserter(expectedSet, expectedSet.end()), [] (auto&& l) {
+ return TLabel{l.Name(), l.Value()};
+ });
+
+ const auto& l = actual.GetLabels();
+ Transform(std::begin(l), std::end(l), std::inserter(actualSet, std::begin(actualSet)),
+ [](auto&& elem) -> TLabel {
+ return {elem.GetName(), elem.GetValue()};
+ });
+
+ TVector<TLabel> diff;
+ SetSymmetricDifference(std::begin(expectedSet), std::end(expectedSet),
+ std::begin(actualSet), std::end(actualSet), std::back_inserter(diff));
+
+ if (diff.size() > 0) {
+ for (auto&& l : diff) {
+ Cerr << l << Endl;
+ }
+
+ UNIT_FAIL("Labels don't match");
+ }
+}
+
+void AssertSimpleMessage(const NProto::TMultiSamplesList& samples, TString pathPrefix = "/") {
+ UNIT_ASSERT_EQUAL(samples.SamplesSize(), 3);
+
+ THashSet<TString> expectedValues{pathPrefix + "Foo", pathPrefix + "Bar", pathPrefix + "Baz"};
+
+ for (const auto& s : samples.GetSamples()) {
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 1);
+ UNIT_ASSERT_EQUAL(s.PointsSize(), 1);
+
+ const auto labelVal = s.GetLabels(0).GetValue();
+ UNIT_ASSERT(expectedValues.contains(labelVal));
+
+ if (labelVal == pathPrefix + "Foo") {
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE);
+ UNIT_ASSERT_DOUBLES_EQUAL(s.GetPoints(0).GetFloat64(), 1, 1e-6);
+ } else if (labelVal == pathPrefix + "Bar") {
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE);
+ UNIT_ASSERT_DOUBLES_EQUAL(s.GetPoints(0).GetFloat64(), 2, 1e-6);
+ } else if (labelVal == pathPrefix + "Baz") {
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::RATE);
+ UNIT_ASSERT_EQUAL(s.GetPoints(0).GetUint64(), 42);
+ }
+ }
+}
+
+Y_UNIT_TEST_SUITE(TLegacyProtoDecoderTest) {
+ Y_UNIT_TEST(SimpleProto) {
+ NProto::TMultiSamplesList samples;
+ IMetricEncoderPtr e = EncoderProtobuf(&samples);
+
+ auto msg = MakeSimpleMessage();
+ DecodeLegacyProto(msg, e.Get());
+
+ AssertSimpleMessage(samples);
+ };
+
+ Y_UNIT_TEST(RepeatedProto) {
+ NProto::TMultiSamplesList samples;
+ IMetricEncoderPtr e = EncoderProtobuf(&samples);
+
+ auto simple = MakeSimpleMessage();
+ TRepeated msg;
+ msg.AddMessages()->CopyFrom(simple);
+
+ DecodeLegacyProto(msg, e.Get());
+
+ AssertSimpleMessage(samples);
+ }
+
+ Y_UNIT_TEST(RepeatedProtoWithPath) {
+ NProto::TMultiSamplesList samples;
+ IMetricEncoderPtr e = EncoderProtobuf(&samples);
+
+ auto simple = MakeSimpleMessage();
+ TRepeatedWithPath msg;
+ msg.AddNamespace()->CopyFrom(simple);
+
+ DecodeLegacyProto(msg, e.Get());
+
+ AssertSimpleMessage(samples, "/Namespace/");
+ }
+
+ Y_UNIT_TEST(DeepNesting) {
+ NProto::TMultiSamplesList samples;
+ IMetricEncoderPtr e = EncoderProtobuf(&samples);
+
+ auto simple = MakeSimpleMessage();
+ TRepeatedWithPath internal;
+ internal.AddNamespace()->CopyFrom(simple);
+
+ TDeepNesting msg;
+ msg.MutableNested()->CopyFrom(internal);
+
+ DecodeLegacyProto(msg, e.Get());
+
+ AssertSimpleMessage(samples, "/Namespace/");
+ }
+
+ Y_UNIT_TEST(Keys) {
+ NProto::TMultiSamplesList samples;
+ IMetricEncoderPtr e = EncoderProtobuf(&samples);
+
+ auto simple = MakeSimpleMessage();
+ simple.SetLabel("my_label_value");
+
+ TNestedWithKeys msg;
+ msg.AddNamespace()->CopyFrom(simple);
+
+ DecodeLegacyProto(msg, e.Get());
+
+ auto i = 0;
+ for (const auto& s : samples.GetSamples()) {
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 4);
+
+ bool foundLabel = false;
+ bool foundFixed = false;
+ bool foundNumbered = false;
+
+ for (const auto& label : s.GetLabels()) {
+ if (label.GetName() == "my_label") {
+ foundLabel = true;
+ UNIT_ASSERT_STRINGS_EQUAL(label.GetValue(), "my_label_value");
+ } else if (label.GetName() == "fixed_label") {
+ foundFixed = true;
+ UNIT_ASSERT_STRINGS_EQUAL(label.GetValue(), "fixed_value");
+ } else if (label.GetName() == "numbered") {
+ foundNumbered = true;
+ UNIT_ASSERT_STRINGS_EQUAL(label.GetValue(), ::ToString(i));
+ }
+ }
+
+ UNIT_ASSERT(foundLabel);
+ UNIT_ASSERT(foundFixed);
+ UNIT_ASSERT(foundNumbered);
+ }
+ }
+
+ Y_UNIT_TEST(NonStringKeys) {
+ NProto::TMultiSamplesList samples;
+ IMetricEncoderPtr e = EncoderProtobuf(&samples);
+
+ TNonStringKeys msg;
+ msg.SetFoo(42);
+ msg.SetEnum(ENUM);
+ msg.SetInt(43);
+
+ TRepeatedNonStringKeys msgs;
+ msgs.AddNested()->CopyFrom(msg);
+
+ DecodeLegacyProto(msgs, e.Get());
+
+ for (const auto& s : samples.GetSamples()) {
+ bool foundEnum = false;
+ bool foundInt = false;
+
+ for (const auto& label : s.GetLabels()) {
+ if (label.GetName() == "enum") {
+ foundEnum = true;
+ UNIT_ASSERT_STRINGS_EQUAL(label.GetValue(), "ENUM");
+ } else if (label.GetName() == "int") {
+ foundInt = true;
+ UNIT_ASSERT_STRINGS_EQUAL(label.GetValue(), "43");
+ }
+ }
+
+ UNIT_ASSERT(foundEnum);
+ UNIT_ASSERT(foundInt);
+
+ UNIT_ASSERT_DOUBLES_EQUAL(s.GetPoints(0).GetFloat64(), 42, 1e-6);
+ }
+ }
+
+ Y_UNIT_TEST(KeysFromNonLeafNodes) {
+ NProto::TMultiSamplesList samples;
+ IMetricEncoderPtr e = EncoderProtobuf(&samples);
+
+ auto simple = MakeSimpleMessage();
+ simple.SetLabel("label_value");
+
+ TRepeatedWithName nested;
+ nested.SetName("my_name");
+ nested.AddNested()->CopyFrom(simple);
+
+ TKeysFromNonLeaf msg;
+ msg.AddNested()->CopyFrom(nested);
+
+ DecodeLegacyProto(msg, e.Get());
+
+ AssertLabels({{"my_label", "label_value"}, {"path", "/Nested/Nested/Foo"}, {"name", "my_name"}}, samples.GetSamples(0));
+ }
+
+ Y_UNIT_TEST(SpacesAreGetReplaced) {
+ NProto::TMultiSamplesList samples;
+ IMetricEncoderPtr e = EncoderProtobuf(&samples);
+
+ auto simple = MakeSimpleMessage();
+ simple.SetLabel("my label_value");
+
+ TNestedWithKeys msg;
+ msg.AddNamespace()->CopyFrom(simple);
+
+ DecodeLegacyProto(msg, e.Get());
+
+ for (const auto& s : samples.GetSamples()) {
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 4);
+
+ bool foundLabel = false;
+
+ for (const auto& label : s.GetLabels()) {
+ if (label.GetName() == "my_label") {
+ foundLabel = true;
+ UNIT_ASSERT_STRINGS_EQUAL(label.GetValue(), "my_label_value");
+ }
+ }
+
+ UNIT_ASSERT(foundLabel);
+ }
+ }
+
+ Y_UNIT_TEST(ExtraLabels) {
+ NProto::TMultiSamplesList samples;
+ IMetricEncoderPtr e = EncoderProtobuf(&samples);
+
+ TExtraLabels msg;
+ msg.MutableExtraAsIs()->CopyFrom(MakeExtra("label", "foo", 42, false));
+ msg.MutableExtraDeriv()->CopyFrom(MakeExtra("deriv_label", "deriv_foo", 43, true));
+
+ DecodeLegacyProto(msg, e.Get());
+
+ UNIT_ASSERT_EQUAL(samples.SamplesSize(), 2);
+ {
+ auto s = samples.GetSamples(0);
+ AssertLabels({{"label", "foo"}, {"path", "/ExtraAsIs"}}, s);
+
+ UNIT_ASSERT_EQUAL(s.PointsSize(), 1);
+ auto point = s.GetPoints(0);
+ UNIT_ASSERT_DOUBLES_EQUAL(point.GetFloat64(), 42, 1e-6);
+ }
+
+ {
+ auto s = samples.GetSamples(1);
+ AssertLabels({{"deriv_label", "deriv_foo"}, {"path", "/ExtraDeriv"}}, s);
+
+ UNIT_ASSERT_EQUAL(s.PointsSize(), 1);
+ auto point = s.GetPoints(0);
+ UNIT_ASSERT_EQUAL(point.GetUint64(), 43);
+ }
+ }
+
+ Y_UNIT_TEST(NestedExtraLabels) {
+ NProto::TMultiSamplesList samples;
+ IMetricEncoderPtr e = EncoderProtobuf(&samples);
+
+ TExtraLabels msg;
+ auto extra = MakeExtra("label", "foo", 42, false);
+ auto* val = extra.Mutablevalues(0);
+ {
+ auto child = MakeExtra("child1", "label1", 24, true);
+ child.Mutablevalues(0)->Settype(NMonProto::EMetricType::RATE);
+ val->Addchildren()->CopyFrom(child);
+ }
+
+ {
+ auto child = MakeExtra("child2", 34, 23, false);
+ val->Addchildren()->CopyFrom(child);
+ }
+ msg.MutableExtraAsIs()->CopyFrom(extra);
+
+ DecodeLegacyProto(msg, e.Get());
+
+ UNIT_ASSERT_EQUAL(samples.SamplesSize(), 3);
+ {
+ auto s = samples.GetSamples(0);
+ AssertLabels({{"label", "foo"}, {"path", "/ExtraAsIs"}}, s);
+
+ UNIT_ASSERT_EQUAL(s.PointsSize(), 1);
+ auto point = s.GetPoints(0);
+ UNIT_ASSERT_DOUBLES_EQUAL(point.GetFloat64(), 42, 1e-6);
+ }
+
+ {
+ auto s = samples.GetSamples(1);
+ AssertLabels({{"label", "foo"}, {"child1", "label1"}, {"path", "/ExtraAsIs"}}, s);
+
+ UNIT_ASSERT_EQUAL(s.PointsSize(), 1);
+ auto point = s.GetPoints(0);
+ UNIT_ASSERT_EQUAL(point.GetUint64(), 24);
+ }
+
+ {
+ auto s = samples.GetSamples(2);
+ AssertLabels({{"label", "foo"}, {"child2", "34"}, {"path", "/ExtraAsIs"}}, s);
+
+ UNIT_ASSERT_EQUAL(s.PointsSize(), 1);
+ auto point = s.GetPoints(0);
+ UNIT_ASSERT_EQUAL(point.GetFloat64(), 23);
+ }
+ }
+
+ Y_UNIT_TEST(RobotLabels) {
+ NProto::TMultiSamplesList samples;
+ IMetricEncoderPtr e = EncoderProtobuf(&samples);
+
+ TNamedCounter responses;
+ responses.SetName("responses");
+ responses.SetCount(42);
+
+ TCrawlerCounters::TStatusCounters statusCounters;
+ statusCounters.AddZoraResponses()->CopyFrom(responses);
+
+ TCrawlerCounters::TPolicyCounters policyCounters;
+ policyCounters.SetSubComponent("mySubComponent");
+ policyCounters.SetName("myComponentName");
+ policyCounters.SetZone("myComponentZone");
+ policyCounters.MutableStatusCounters()->CopyFrom(statusCounters);
+
+ TCrawlerCounters counters;
+ counters.SetComponent("myComponent");
+ counters.AddPoliciesCounters()->CopyFrom(policyCounters);
+
+ DecodeLegacyProto(counters, e.Get());
+ UNIT_ASSERT_EQUAL(samples.SamplesSize(), 1);
+ auto s = samples.GetSamples(0);
+ AssertLabels({
+ {"SubComponent", "mySubComponent"}, {"Policy", "myComponentName"}, {"Zone", "myComponentZone"},
+ {"ZoraResponse", "responses"}, {"path", "/PoliciesCounters/StatusCounters/ZoraResponses/Count"}}, s);
+ }
+
+ Y_UNIT_TEST(ZoraLabels) {
+ NProto::TMultiSamplesList samples;
+ IMetricEncoderPtr e = EncoderProtobuf(&samples);
+
+ TTimeLogHist hist;
+ hist.AddBuckets(42);
+ hist.AddBuckets(0);
+
+ TKiwiCounters counters;
+ counters.MutableTimes()->CopyFrom(hist);
+
+ DecodeLegacyProto(counters, e.Get());
+
+ UNIT_ASSERT_EQUAL(samples.SamplesSize(), 2);
+
+ auto s = samples.GetSamples(0);
+ AssertLabels({{"slot", "0"}, {"path", "/Times/Buckets"}}, s);
+ UNIT_ASSERT_EQUAL(s.PointsSize(), 1);
+ UNIT_ASSERT_EQUAL(s.GetPoints(0).GetUint64(), 42);
+
+ s = samples.GetSamples(1);
+ AssertLabels({{"slot", "1"}, {"path", "/Times/Buckets"}}, s);
+ UNIT_ASSERT_EQUAL(s.GetPoints(0).GetUint64(), 0);
+ }
+}
diff --git a/library/cpp/monlib/encode/legacy_protobuf/protos/metric_meta.proto b/library/cpp/monlib/encode/legacy_protobuf/protos/metric_meta.proto
new file mode 100644
index 0000000000..fd23eb372b
--- /dev/null
+++ b/library/cpp/monlib/encode/legacy_protobuf/protos/metric_meta.proto
@@ -0,0 +1,73 @@
+import "google/protobuf/descriptor.proto";
+
+package NMonProto;
+
+option java_package = "ru.yandex.monlib.proto";
+option java_outer_classname = "MetricMetaProto";
+
+enum EMetricType {
+ GAUGE = 1;
+ RATE = 2;
+}
+
+enum EMemOnly {
+ DEFAULT = 0;
+ STORE = 1;
+ MEM_ONLY = 2;
+}
+
+message TMetricMeta {
+ optional EMetricType Type = 1;
+ optional bool Path = 2;
+ optional string Keys = 3;
+ optional bool MemOnly = 4;
+ optional bool IgnorePath = 5;
+ optional string CustomPath = 6;
+}
+
+enum THistogramBase {
+ MICROSECOND = 3;
+ MILLISECOND = 6;
+ SECOND = 9;
+ MINUTE = 12;
+ HOUR = 15;
+}
+
+message THistogramEntry {
+ optional uint64 Multiplier = 1;
+ optional double Value = 2;
+}
+
+message THistogram {
+ optional THistogramBase Base = 1;
+ optional string BaseStr = 2;
+ repeated THistogramEntry Entries = 5;
+}
+
+// field of this type is recognized by Solomon
+message TExtraLabelMetrics {
+ optional string labelName = 1;
+
+ message TValue {
+ optional string labelValue = 1;
+ // used only if != 0
+ optional uint64 labelValueUint = 21;
+
+ optional uint64 longValue = 2;
+ optional double doubleValue = 3;
+ optional THistogram histogramValue = 4;
+
+ optional EMetricType type = 7;
+ optional EMemOnly memOnly = 8;
+ optional bool dropHost = 9;
+
+ repeated TExtraLabelMetrics children = 17;
+ }
+
+ repeated TValue values = 2;
+}
+
+extend google.protobuf.FieldOptions {
+ optional TMetricMeta Metric = 1719;
+}
+
diff --git a/library/cpp/monlib/encode/legacy_protobuf/protos/python/ya.make b/library/cpp/monlib/encode/legacy_protobuf/protos/python/ya.make
new file mode 100644
index 0000000000..095b307b01
--- /dev/null
+++ b/library/cpp/monlib/encode/legacy_protobuf/protos/python/ya.make
@@ -0,0 +1,3 @@
+OWNER(g:solomon)
+
+PY_PROTOS_FOR(library/cpp/monlib/encode/legacy_protobuf/protos)
diff --git a/library/cpp/monlib/encode/legacy_protobuf/protos/ya.make b/library/cpp/monlib/encode/legacy_protobuf/protos/ya.make
new file mode 100644
index 0000000000..489f361ab1
--- /dev/null
+++ b/library/cpp/monlib/encode/legacy_protobuf/protos/ya.make
@@ -0,0 +1,13 @@
+PROTO_LIBRARY()
+
+OWNER(g:solomon)
+
+SRCS(
+ metric_meta.proto
+)
+
+IF (NOT PY_PROTOS_FOR)
+ EXCLUDE_TAGS(GO_PROTO)
+ENDIF()
+
+END()
diff --git a/library/cpp/monlib/encode/legacy_protobuf/ut/test_cases.proto b/library/cpp/monlib/encode/legacy_protobuf/ut/test_cases.proto
new file mode 100644
index 0000000000..37e901de48
--- /dev/null
+++ b/library/cpp/monlib/encode/legacy_protobuf/ut/test_cases.proto
@@ -0,0 +1,90 @@
+import "library/cpp/monlib/encode/legacy_protobuf/protos/metric_meta.proto";
+
+message TSimple {
+ optional uint64 Foo = 1 [ (NMonProto.Metric).Type = GAUGE ];
+ optional double Bar = 2 [ (NMonProto.Metric).Type = GAUGE ];
+ optional double Baz = 3 [ (NMonProto.Metric).Type = RATE ];
+ optional string Label = 4;
+}
+
+message TRepeated {
+ repeated TSimple Messages = 1 [ (NMonProto.Metric).Path = false ];
+};
+
+message TRepeatedWithPath {
+ repeated TSimple Namespace = 1 [ (NMonProto.Metric).Path = true ];
+};
+
+message TNestedWithKeys {
+ repeated TSimple Namespace = 1 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "my_label:Label fixed_label=fixed_value numbered:#" ];
+};
+
+message TDeepNesting {
+ optional TRepeatedWithPath Nested = 1 [ (NMonProto.Metric).Path = false ];
+};
+
+enum EEnum {
+ MY = 1;
+ ENUM = 2;
+};
+
+message TNonStringKeys {
+ optional uint32 Foo = 1 [ (NMonProto.Metric).Type = GAUGE ];
+ optional EEnum Enum = 2;
+ optional uint32 Int = 3;
+};
+
+message TRepeatedNonStringKeys {
+ repeated TNonStringKeys Nested = 1 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "enum:Enum int:Int" ];
+};
+
+message TExtraLabels {
+ optional NMonProto.TExtraLabelMetrics ExtraAsIs = 1 [ (NMonProto.Metric).Type = GAUGE ];
+ optional NMonProto.TExtraLabelMetrics ExtraDeriv = 2 [ (NMonProto.Metric).Type = RATE ];
+};
+
+message TRepeatedWithName {
+ optional string Name = 1;
+ repeated TSimple Nested = 2 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "my_label:Label" ];
+};
+
+message TKeysFromNonLeaf {
+ repeated TRepeatedWithName Nested = 1 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "name:Name" ];
+};
+
+
+message TNamedCounter {
+ optional string Name = 1;
+ optional uint64 Count = 2 [ (NMonProto.Metric).Type = RATE ];
+}
+
+message TCrawlerCounters {
+ message TStatusCounters {
+ repeated TNamedCounter ZoraResponses = 3 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "ZoraResponse:Name" ];
+ repeated TNamedCounter FetcherResponses = 4 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "SpiderResponse:Name" ];
+ repeated TNamedCounter RotorResponses = 5 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "SpiderResponse:Name" ];
+ repeated TNamedCounter PDFetchResponses = 6 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "PDFetchResponse:Name" ];
+ repeated TNamedCounter CalcResponses = 7 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "CalcResponse:Name" ];
+ repeated TNamedCounter Responses = 8 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "AggregatedResponse:Name" ];
+ }
+
+ message TPolicyCounters {
+ optional string SubComponent = 1;
+ optional string Name = 2;
+ optional string Zone = 3;
+
+ optional TStatusCounters StatusCounters = 4 [ (NMonProto.Metric).Path = true ];
+ }
+
+ optional string Component = 1;
+ repeated TPolicyCounters PoliciesCounters = 3 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "SubComponent:SubComponent Policy:Name Zone:Zone" ];
+}
+
+message TTimeLogHist {
+ optional uint32 MinBucketMillisec = 1;
+ repeated uint64 Buckets = 2 [ (NMonProto.Metric).Type = RATE, (NMonProto.Metric).Keys = "slot:#" ];
+}
+
+message TKiwiCounters {
+ optional TTimeLogHist Times = 22 [ (NMonProto.Metric).Path = true ];
+}
diff --git a/library/cpp/monlib/encode/legacy_protobuf/ut/ya.make b/library/cpp/monlib/encode/legacy_protobuf/ut/ya.make
new file mode 100644
index 0000000000..479a0c46c9
--- /dev/null
+++ b/library/cpp/monlib/encode/legacy_protobuf/ut/ya.make
@@ -0,0 +1,18 @@
+UNITTEST_FOR(library/cpp/monlib/encode/legacy_protobuf)
+
+OWNER(
+ g:solomon
+ msherbakov
+)
+
+SRCS(
+ legacy_protobuf_ut.cpp
+ test_cases.proto
+)
+
+PEERDIR(
+ library/cpp/monlib/encode/protobuf
+ library/cpp/monlib/encode/text
+)
+
+END()
diff --git a/library/cpp/monlib/encode/legacy_protobuf/ya.make b/library/cpp/monlib/encode/legacy_protobuf/ya.make
new file mode 100644
index 0000000000..74c82aac93
--- /dev/null
+++ b/library/cpp/monlib/encode/legacy_protobuf/ya.make
@@ -0,0 +1,16 @@
+LIBRARY()
+
+OWNER(
+ g:solomon
+ msherbakov
+)
+
+SRCS(
+ legacy_proto_decoder.cpp
+)
+
+PEERDIR(
+ library/cpp/monlib/encode/legacy_protobuf/protos
+)
+
+END()
diff --git a/library/cpp/monlib/encode/prometheus/fuzz/main.cpp b/library/cpp/monlib/encode/prometheus/fuzz/main.cpp
new file mode 100644
index 0000000000..24bda2d32e
--- /dev/null
+++ b/library/cpp/monlib/encode/prometheus/fuzz/main.cpp
@@ -0,0 +1,18 @@
+#include <library/cpp/monlib/encode/prometheus/prometheus.h>
+#include <library/cpp/monlib/encode/fake/fake.h>
+
+#include <util/stream/mem.h>
+
+
+extern "C" int LLVMFuzzerTestOneInput(const ui8* buf, size_t size) {
+ using namespace NMonitoring;
+
+ try {
+ TStringBuf data(reinterpret_cast<const char*>(buf), size);
+ auto encoder = EncoderFake();
+ DecodePrometheus(data, encoder.Get());
+ } catch (...) {
+ }
+
+ return 0;
+}
diff --git a/library/cpp/monlib/encode/prometheus/fuzz/ya.make b/library/cpp/monlib/encode/prometheus/fuzz/ya.make
new file mode 100644
index 0000000000..4a6c796ed5
--- /dev/null
+++ b/library/cpp/monlib/encode/prometheus/fuzz/ya.make
@@ -0,0 +1,16 @@
+FUZZ()
+
+OWNER(g:solomon jamel)
+
+PEERDIR(
+ library/cpp/monlib/encode/prometheus
+ library/cpp/monlib/encode/fake
+)
+
+SIZE(MEDIUM)
+
+SRCS(
+ main.cpp
+)
+
+END()
diff --git a/library/cpp/monlib/encode/prometheus/prometheus.h b/library/cpp/monlib/encode/prometheus/prometheus.h
new file mode 100644
index 0000000000..2e7fa31c28
--- /dev/null
+++ b/library/cpp/monlib/encode/prometheus/prometheus.h
@@ -0,0 +1,18 @@
+#pragma once
+
+#include <library/cpp/monlib/encode/encoder.h>
+#include <library/cpp/monlib/encode/format.h>
+
+#include <util/generic/yexception.h>
+
+
+namespace NMonitoring {
+
+ class TPrometheusDecodeException: public yexception {
+ };
+
+ IMetricEncoderPtr EncoderPrometheus(IOutputStream* out, TStringBuf metricNameLabel = "sensor");
+
+ void DecodePrometheus(TStringBuf data, IMetricConsumer* c, TStringBuf metricNameLabel = "sensor");
+
+}
diff --git a/library/cpp/monlib/encode/prometheus/prometheus_decoder.cpp b/library/cpp/monlib/encode/prometheus/prometheus_decoder.cpp
new file mode 100644
index 0000000000..7e81357dbd
--- /dev/null
+++ b/library/cpp/monlib/encode/prometheus/prometheus_decoder.cpp
@@ -0,0 +1,597 @@
+#include "prometheus.h"
+#include "prometheus_model.h"
+
+#include <library/cpp/monlib/metrics/histogram_snapshot.h>
+#include <library/cpp/monlib/metrics/metric.h>
+
+#include <util/datetime/base.h>
+#include <util/generic/hash.h>
+#include <util/string/cast.h>
+#include <util/string/builder.h>
+#include <util/generic/maybe.h>
+#include <util/string/ascii.h>
+
+#include <cmath>
+
+#define Y_PARSER_FAIL(message) \
+ ythrow ::NMonitoring::TPrometheusDecodeException() << message << " at line #" << CurrentLine_
+
+#define Y_PARSER_ENSURE(cond, message) \
+ Y_ENSURE_EX(cond, ::NMonitoring::TPrometheusDecodeException() << message << " at line #" << CurrentLine_)
+
+
+namespace NMonitoring {
+ namespace {
+ constexpr ui32 MAX_LABEL_VALUE_LEN = 256;
+
+ using TLabelsMap = THashMap<TString, TString>;
+
+ TString LabelsToStr(const TLabelsMap& labels) {
+ TStringBuilder sb;
+ auto it = labels.begin();
+ auto end = labels.end();
+
+ sb << '{';
+ while (it != end) {
+ sb << it->first;
+ sb << '=';
+ sb << '"' << it->second << '"';
+
+ ++it;
+ if (it != end) {
+ sb << ", ";
+ }
+ }
+ sb << '}';
+ return sb;
+ }
+
+ template <typename T, typename U>
+ bool TryStaticCast(U val, T& out) {
+ static_assert(std::is_arithmetic_v<U>);
+ if constexpr (std::is_floating_point_v<T> || std::is_floating_point_v<U>) {
+ if (val > MaxFloor<T>() || val < -MaxFloor<T>()) {
+ return false;
+ }
+
+ } else {
+ if (val > Max<T>() || val < Min<T>()) {
+ return false;
+ }
+ }
+
+ out = static_cast<T>(val);
+ return true;
+ }
+
+ ///////////////////////////////////////////////////////////////////////
+ // THistogramBuilder
+ ///////////////////////////////////////////////////////////////////////
+ class THistogramBuilder {
+ using TBucketData = std::pair<TBucketBound, TBucketValue>;
+ constexpr static TBucketData ZERO_BUCKET = { -std::numeric_limits<TBucketBound>::max(), 0 };
+ public:
+ TStringBuf GetName() const noexcept {
+ return Name_;
+ }
+
+ void SetName(TStringBuf name) noexcept {
+ Name_ = name;
+ }
+
+ const TLabelsMap& GetLabels() const noexcept {
+ return *Labels_;
+ }
+
+ void SetLabels(TLabelsMap&& labels) {
+ if (Labels_.Defined()) {
+ Y_ENSURE(Labels_ == labels,
+ "mixed labels in one histogram, prev: " << LabelsToStr(*Labels_) <<
+ ", current: " << LabelsToStr(labels));
+ } else {
+ Labels_.ConstructInPlace(std::move(labels));
+ }
+ }
+
+ TInstant GetTime() const noexcept {
+ return Time_;
+ }
+
+ void SetTime(TInstant time) noexcept {
+ Time_ = time;
+ }
+
+ bool Empty() const noexcept {
+ return Bounds_.empty();
+ }
+
+ bool Same(TStringBuf name, const TLabelsMap& labels) const noexcept {
+ return Name_ == name && Labels_ == labels;
+ }
+
+ void AddBucket(TBucketBound bound, TBucketValue value) {
+ Y_ENSURE_EX(PrevBucket_.first < bound, TPrometheusDecodeException() <<
+ "invalid order of histogram bounds " << PrevBucket_.first <<
+ " >= " << bound);
+
+ Y_ENSURE_EX(PrevBucket_.second <= value, TPrometheusDecodeException() <<
+ "invalid order of histogram bucket values " << PrevBucket_.second <<
+ " > " << value);
+
+ // convert infinite bound value
+ if (bound == std::numeric_limits<TBucketBound>::infinity()) {
+ bound = HISTOGRAM_INF_BOUND;
+ }
+
+ Bounds_.push_back(bound);
+ Values_.push_back(value - PrevBucket_.second); // keep only delta between buckets
+
+ PrevBucket_ = { bound, value };
+ }
+
+ // will clear builder state
+ IHistogramSnapshotPtr ToSnapshot() {
+ Y_ENSURE_EX(!Empty(), TPrometheusDecodeException() << "histogram cannot be empty");
+ Time_ = TInstant::Zero();
+ PrevBucket_ = ZERO_BUCKET;
+ Labels_.Clear();
+ auto snapshot = ExplicitHistogramSnapshot(Bounds_, Values_);
+
+ Bounds_.clear();
+ Values_.clear();
+
+ return snapshot;
+ }
+
+ private:
+ TStringBuf Name_;
+ TMaybe<TLabelsMap> Labels_;
+ TInstant Time_;
+ TBucketBounds Bounds_;
+ TBucketValues Values_;
+ TBucketData PrevBucket_ = ZERO_BUCKET;
+ };
+
+ ///////////////////////////////////////////////////////////////////////
+ // EPrometheusMetricType
+ ///////////////////////////////////////////////////////////////////////
+ enum class EPrometheusMetricType {
+ GAUGE,
+ COUNTER,
+ SUMMARY,
+ UNTYPED,
+ HISTOGRAM,
+ };
+
+ ///////////////////////////////////////////////////////////////////////
+ // TPrometheusReader
+ ///////////////////////////////////////////////////////////////////////
+ class TPrometheusReader {
+ public:
+ TPrometheusReader(TStringBuf data, IMetricConsumer* c, TStringBuf metricNameLabel)
+ : Data_(data)
+ , Consumer_(c)
+ , MetricNameLabel_(metricNameLabel)
+ {
+ }
+
+ void Read() {
+ Consumer_->OnStreamBegin();
+
+ if (HasRemaining()) {
+ ReadNextByte();
+ SkipSpaces();
+
+ try {
+ while (HasRemaining()) {
+ switch (CurrentByte_) {
+ case '\n':
+ ReadNextByte(); // skip '\n'
+ CurrentLine_++;
+ SkipSpaces();
+ break;
+ case '#':
+ ParseComment();
+ break;
+ default:
+ ParseMetric();
+ break;
+ }
+ }
+
+ if (!HistogramBuilder_.Empty()) {
+ ConsumeHistogram();
+ }
+ } catch (const TPrometheusDecodeException& e) {
+ throw e;
+ } catch (...) {
+ Y_PARSER_FAIL("unexpected error " << CurrentExceptionMessage());
+ }
+ }
+
+ Consumer_->OnStreamEnd();
+ }
+
+ private:
+ bool HasRemaining() const noexcept {
+ return CurrentPos_ < Data_.Size();
+ }
+
+ // # 'TYPE' metric_name {counter|gauge|histogram|summary|untyped}
+ // # 'HELP' metric_name some help info
+ // # general comment message
+ void ParseComment() {
+ SkipExpectedChar('#');
+ SkipSpaces();
+
+ TStringBuf keyword = ReadToken();
+ if (keyword == TStringBuf("TYPE")) {
+ SkipSpaces();
+
+ TStringBuf nextName = ReadTokenAsMetricName();
+ Y_PARSER_ENSURE(!nextName.Empty(), "invalid metric name");
+
+ SkipSpaces();
+ EPrometheusMetricType nextType = ReadType();
+
+ bool inserted = SeenTypes_.emplace(nextName, nextType).second;
+ Y_PARSER_ENSURE(inserted, "second TYPE line for metric " << nextName);
+
+ if (nextType == EPrometheusMetricType::HISTOGRAM) {
+ if (!HistogramBuilder_.Empty()) {
+ ConsumeHistogram();
+ }
+ HistogramBuilder_.SetName(nextName);
+ }
+ } else {
+ // skip HELP and general comments
+ SkipUntilEol();
+ }
+
+ Y_PARSER_ENSURE(CurrentByte_ == '\n', "expected '\\n', found '" << CurrentByte_ << '\'');
+ }
+
+ // metric_name [labels] value [timestamp]
+ void ParseMetric() {
+ TStringBuf name = ReadTokenAsMetricName();
+ SkipSpaces();
+
+ TLabelsMap labels = ReadLabels();
+ SkipSpaces();
+
+ double value = ParseGoDouble(ReadToken());
+ SkipSpaces();
+
+ TInstant time = TInstant::Zero();
+ if (CurrentByte_ != '\n') {
+ time = TInstant::MilliSeconds(FromString<ui64>(ReadToken()));
+ }
+
+ TStringBuf baseName = name;
+ EPrometheusMetricType type = EPrometheusMetricType::UNTYPED;
+
+ if (auto* seenType = SeenTypes_.FindPtr(name)) {
+ type = *seenType;
+ } else {
+ baseName = NPrometheus::ToBaseName(name);
+ if (auto* baseType = SeenTypes_.FindPtr(baseName)) {
+ type = *baseType;
+ }
+ }
+
+ switch (type) {
+ case EPrometheusMetricType::HISTOGRAM:
+ if (NPrometheus::IsBucket(name)) {
+ double bound = 0.0;
+ auto it = labels.find(NPrometheus::BUCKET_LABEL);
+ if (it != labels.end()) {
+ bound = ParseGoDouble(it->second);
+ labels.erase(it);
+ } else {
+ Y_PARSER_FAIL(
+ "metric " << name << "has no " << NPrometheus::BUCKET_LABEL <<
+ " label at line #" << CurrentLine_);
+ }
+
+ if (!HistogramBuilder_.Empty() && !HistogramBuilder_.Same(baseName, labels)) {
+ ConsumeHistogram();
+ HistogramBuilder_.SetName(baseName);
+ }
+
+ TBucketValue bucketVal;
+ Y_PARSER_ENSURE(TryStaticCast(value, bucketVal), "Cannot convert " << value << " to bucket value type");
+ HistogramBuilder_.AddBucket(bound, bucketVal);
+ HistogramBuilder_.SetTime(time);
+ HistogramBuilder_.SetLabels(std::move(labels));
+ } else if (NPrometheus::IsCount(name)) {
+ // translate x_count metric as COUNTER metric
+ ConsumeCounter(name, labels, time, value);
+ } else if (NPrometheus::IsSum(name)) {
+ // translate x_sum metric as GAUGE metric
+ ConsumeGauge(name, labels, time, value);
+ } else {
+ Y_PARSER_FAIL(
+ "metric " << name <<
+ " should be part of HISTOGRAM " << baseName);
+ }
+ break;
+
+ case EPrometheusMetricType::SUMMARY:
+ if (NPrometheus::IsCount(name)) {
+ // translate x_count metric as COUNTER metric
+ ConsumeCounter(name, labels, time, value);
+ } else if (NPrometheus::IsSum(name)) {
+ // translate x_sum metric as GAUGE metric
+ ConsumeGauge(name, labels, time, value);
+ } else {
+ ConsumeGauge(name, labels, time, value);
+ }
+ break;
+
+ case EPrometheusMetricType::COUNTER:
+ ConsumeCounter(name, labels, time, value);
+ break;
+
+ case EPrometheusMetricType::GAUGE:
+ ConsumeGauge(name, labels, time, value);
+ break;
+
+ case EPrometheusMetricType::UNTYPED:
+ ConsumeGauge(name, labels, time, value);
+ break;
+ }
+
+ Y_PARSER_ENSURE(CurrentByte_ == '\n', "expected '\\n', found '" << CurrentByte_ << '\'');
+ }
+
+ // { name = "value", name2 = "value2", }
+ TLabelsMap ReadLabels() {
+ TLabelsMap labels;
+ if (CurrentByte_ != '{') {
+ return labels;
+ }
+
+ SkipExpectedChar('{');
+ SkipSpaces();
+
+ while (CurrentByte_ != '}') {
+ TStringBuf name = ReadTokenAsLabelName();
+ SkipSpaces();
+
+ SkipExpectedChar('=');
+ SkipSpaces();
+
+ TString value = ReadTokenAsLabelValue();
+ SkipSpaces();
+ labels.emplace(name, value);
+
+ if (CurrentByte_ == ',') {
+ SkipExpectedChar(',');
+ SkipSpaces();
+ }
+ }
+
+ SkipExpectedChar('}');
+ return labels;
+ }
+
+ EPrometheusMetricType ReadType() {
+ TStringBuf keyword = ReadToken();
+ if (AsciiEqualsIgnoreCase(keyword, "GAUGE")) {
+ return EPrometheusMetricType::GAUGE;
+ } else if (AsciiEqualsIgnoreCase(keyword, "COUNTER")) {
+ return EPrometheusMetricType::COUNTER;
+ } else if (AsciiEqualsIgnoreCase(keyword, "SUMMARY")) {
+ return EPrometheusMetricType::SUMMARY;
+ } else if (AsciiEqualsIgnoreCase(keyword, "HISTOGRAM")) {
+ return EPrometheusMetricType::HISTOGRAM;
+ } else if (AsciiEqualsIgnoreCase(keyword, "UNTYPED")) {
+ return EPrometheusMetricType::UNTYPED;
+ }
+
+ Y_PARSER_FAIL(
+ "unknown metric type: " << keyword <<
+ " at line #" << CurrentLine_);
+ }
+
+ Y_FORCE_INLINE void ReadNextByteUnsafe() {
+ CurrentByte_ = Data_[CurrentPos_++];
+ }
+
+ Y_FORCE_INLINE bool IsSpace(char ch) {
+ return ch == ' ' || ch == '\t';
+ }
+
+ void ReadNextByte() {
+ Y_PARSER_ENSURE(HasRemaining(), "unexpected end of file");
+ ReadNextByteUnsafe();
+ }
+
+ void SkipExpectedChar(char ch) {
+ Y_PARSER_ENSURE(CurrentByte_ == ch,
+ "expected '" << CurrentByte_ << "', found '" << ch << '\'');
+ ReadNextByte();
+ }
+
+ void SkipSpaces() {
+ while (HasRemaining() && IsSpace(CurrentByte_)) {
+ ReadNextByteUnsafe();
+ }
+ }
+
+ void SkipUntilEol() {
+ while (HasRemaining() && CurrentByte_ != '\n') {
+ ReadNextByteUnsafe();
+ }
+ }
+
+ TStringBuf ReadToken() {
+ Y_VERIFY_DEBUG(CurrentPos_ > 0);
+ size_t begin = CurrentPos_ - 1; // read first byte again
+ while (HasRemaining() && !IsSpace(CurrentByte_) && CurrentByte_ != '\n') {
+ ReadNextByteUnsafe();
+ }
+ return TokenFromPos(begin);
+ }
+
+ TStringBuf ReadTokenAsMetricName() {
+ if (!NPrometheus::IsValidMetricNameStart(CurrentByte_)) {
+ return "";
+ }
+
+ Y_VERIFY_DEBUG(CurrentPos_ > 0);
+ size_t begin = CurrentPos_ - 1; // read first byte again
+ while (HasRemaining()) {
+ ReadNextByteUnsafe();
+ if (!NPrometheus::IsValidMetricNameContinuation(CurrentByte_)) {
+ break;
+ }
+ }
+ return TokenFromPos(begin);
+ }
+
+ TStringBuf ReadTokenAsLabelName() {
+ if (!NPrometheus::IsValidLabelNameStart(CurrentByte_)) {
+ return "";
+ }
+
+ Y_VERIFY_DEBUG(CurrentPos_ > 0);
+ size_t begin = CurrentPos_ - 1; // read first byte again
+ while (HasRemaining()) {
+ ReadNextByteUnsafe();
+ if (!NPrometheus::IsValidLabelNameContinuation(CurrentByte_)) {
+ break;
+ }
+ }
+ return TokenFromPos(begin);
+ }
+
+ TString ReadTokenAsLabelValue() {
+ TString labelValue;
+
+ SkipExpectedChar('"');
+ for (ui32 i = 0; i < MAX_LABEL_VALUE_LEN; i++) {
+ switch (CurrentByte_) {
+ case '"':
+ SkipExpectedChar('"');
+ return labelValue;
+
+ case '\n':
+ Y_PARSER_FAIL("label value contains unescaped new-line");
+
+ case '\\':
+ ReadNextByte();
+ switch (CurrentByte_) {
+ case '"':
+ case '\\':
+ labelValue.append(CurrentByte_);
+ break;
+ case 'n':
+ labelValue.append('\n');
+ break;
+ default:
+ Y_PARSER_FAIL("invalid escape sequence '" << CurrentByte_ << '\'');
+ }
+ break;
+
+ default:
+ labelValue.append(CurrentByte_);
+ break;
+ }
+
+ ReadNextByte();
+ }
+
+ Y_PARSER_FAIL("trying to parse too long label value, size >= " << MAX_LABEL_VALUE_LEN);
+ }
+
+ TStringBuf TokenFromPos(size_t begin) {
+ Y_VERIFY_DEBUG(CurrentPos_ > begin);
+ size_t len = CurrentPos_ - begin - 1;
+ if (len == 0) {
+ return {};
+ }
+
+ return Data_.SubString(begin, len);
+ }
+
+ void ConsumeLabels(TStringBuf name, const TLabelsMap& labels) {
+ Y_PARSER_ENSURE(labels.count(MetricNameLabel_) == 0,
+ "label name '" << MetricNameLabel_ <<
+ "' is reserved, but is used with metric: " << name << LabelsToStr(labels));
+
+ Consumer_->OnLabelsBegin();
+ Consumer_->OnLabel(MetricNameLabel_, TString(name)); // TODO: remove this string allocation
+ for (const auto& it: labels) {
+ Consumer_->OnLabel(it.first, it.second);
+ }
+ Consumer_->OnLabelsEnd();
+ }
+
+ void ConsumeCounter(TStringBuf name, const TLabelsMap& labels, TInstant time, double value) {
+ i64 intValue{0};
+ // not nan
+ if (value == value) {
+ Y_PARSER_ENSURE(TryStaticCast(value, intValue), "value " << value << " is out of range");
+ }
+
+ // see https://st.yandex-team.ru/SOLOMON-4142 for more details
+ // why we convert Prometheus COUNTER into Solomon RATE
+ // TODO: need to fix after server-side aggregation become correct for COUNTERs
+ Consumer_->OnMetricBegin(EMetricType::RATE);
+ ConsumeLabels(name, labels);
+ Consumer_->OnUint64(time, intValue);
+ Consumer_->OnMetricEnd();
+ }
+
+ void ConsumeGauge(TStringBuf name, const TLabelsMap& labels, TInstant time, double value) {
+ Consumer_->OnMetricBegin(EMetricType::GAUGE);
+ ConsumeLabels(name, labels);
+ Consumer_->OnDouble(time, value);
+ Consumer_->OnMetricEnd();
+ }
+
+ void ConsumeHistogram() {
+ Consumer_->OnMetricBegin(EMetricType::HIST_RATE);
+ ConsumeLabels(HistogramBuilder_.GetName(), HistogramBuilder_.GetLabels());
+ auto time = HistogramBuilder_.GetTime();
+ auto hist = HistogramBuilder_.ToSnapshot();
+ Consumer_->OnHistogram(time, std::move(hist));
+ Consumer_->OnMetricEnd();
+ }
+
+ double ParseGoDouble(TStringBuf str) {
+ if (str == TStringBuf("+Inf")) {
+ return std::numeric_limits<double>::infinity();
+ } else if (str == TStringBuf("-Inf")) {
+ return -std::numeric_limits<double>::infinity();
+ } else if (str == TStringBuf("NaN")) {
+ return NAN;
+ }
+
+ double r = 0.0;
+ if (TryFromString(str, r)) {
+ return r;
+ }
+ Y_PARSER_FAIL("cannot parse double value from '" << str << "\' at line #" << CurrentLine_);
+ }
+
+ private:
+ TStringBuf Data_;
+ IMetricConsumer* Consumer_;
+ TStringBuf MetricNameLabel_;
+ THashMap<TString, EPrometheusMetricType> SeenTypes_;
+ THistogramBuilder HistogramBuilder_;
+
+ ui32 CurrentLine_ = 1;
+ ui32 CurrentPos_ = 0;
+ char CurrentByte_ = 0;
+ };
+ } // namespace
+
+void DecodePrometheus(TStringBuf data, IMetricConsumer* c, TStringBuf metricNameLabel) {
+ TPrometheusReader reader(data, c, metricNameLabel);
+ reader.Read();
+}
+
+} // namespace NMonitoring
diff --git a/library/cpp/monlib/encode/prometheus/prometheus_decoder_ut.cpp b/library/cpp/monlib/encode/prometheus/prometheus_decoder_ut.cpp
new file mode 100644
index 0000000000..49c2244fb4
--- /dev/null
+++ b/library/cpp/monlib/encode/prometheus/prometheus_decoder_ut.cpp
@@ -0,0 +1,478 @@
+#include "prometheus.h"
+
+#include <library/cpp/monlib/encode/protobuf/protobuf.h>
+
+#include <library/cpp/testing/unittest/registar.h>
+
+using namespace NMonitoring;
+
+#define ASSERT_LABEL_EQUAL(label, name, value) do { \
+ UNIT_ASSERT_STRINGS_EQUAL((label).GetName(), name); \
+ UNIT_ASSERT_STRINGS_EQUAL((label).GetValue(), value); \
+ } while (false)
+
+#define ASSERT_DOUBLE_POINT(s, time, value) do { \
+ UNIT_ASSERT_VALUES_EQUAL((s).GetTime(), (time).MilliSeconds()); \
+ UNIT_ASSERT_EQUAL((s).GetValueCase(), NProto::TSingleSample::kFloat64); \
+ UNIT_ASSERT_DOUBLES_EQUAL((s).GetFloat64(), value, std::numeric_limits<double>::epsilon()); \
+ } while (false)
+
+#define ASSERT_UINT_POINT(s, time, value) do { \
+ UNIT_ASSERT_VALUES_EQUAL((s).GetTime(), (time).MilliSeconds()); \
+ UNIT_ASSERT_EQUAL((s).GetValueCase(), NProto::TSingleSample::kUint64); \
+ UNIT_ASSERT_VALUES_EQUAL((s).GetUint64(), value); \
+ } while (false)
+
+#define ASSERT_HIST_POINT(s, time, expected) do { \
+ UNIT_ASSERT_VALUES_EQUAL((s).GetTime(), time.MilliSeconds()); \
+ UNIT_ASSERT_EQUAL((s).GetValueCase(), NProto::TSingleSample::kHistogram);\
+ UNIT_ASSERT_VALUES_EQUAL((s).GetHistogram().BoundsSize(), (expected).Count()); \
+ UNIT_ASSERT_VALUES_EQUAL((s).GetHistogram().ValuesSize(), (expected).Count()); \
+ for (size_t i = 0; i < (s).GetHistogram().BoundsSize(); i++) { \
+ UNIT_ASSERT_DOUBLES_EQUAL((s).GetHistogram().GetBounds(i), (expected).UpperBound(i), Min<double>()); \
+ UNIT_ASSERT_VALUES_EQUAL((s).GetHistogram().GetValues(i), (expected).Value(i)); \
+ } \
+ } while (false)
+
+Y_UNIT_TEST_SUITE(TPrometheusDecoderTest) {
+
+ NProto::TSingleSamplesList Decode(TStringBuf data) {
+ NProto::TSingleSamplesList samples;
+ {
+ IMetricEncoderPtr e = EncoderProtobuf(&samples);
+ DecodePrometheus(data, e.Get());
+ }
+ return samples;
+ }
+
+ Y_UNIT_TEST(Empty) {
+ {
+ auto samples = Decode("");
+ UNIT_ASSERT_EQUAL(samples.SamplesSize(), 0);
+ }
+ {
+ auto samples = Decode("\n");
+ UNIT_ASSERT_EQUAL(samples.SamplesSize(), 0);
+ }
+ {
+ auto samples = Decode("\n \n \n");
+ UNIT_ASSERT_EQUAL(samples.SamplesSize(), 0);
+ }
+ {
+ auto samples = Decode("\t\n\t\n");
+ UNIT_ASSERT_EQUAL(samples.SamplesSize(), 0);
+ }
+ }
+
+ Y_UNIT_TEST(Minimal) {
+ auto samples = Decode(
+ "minimal_metric 1.234\n"
+ "another_metric -3e3 103948\n"
+ "# Even that:\n"
+ "no_labels{} 3\n"
+ "# HELP line for non-existing metric will be ignored.\n");
+
+ UNIT_ASSERT_EQUAL(samples.SamplesSize(), 3);
+ {
+ auto& s = samples.GetSamples(0);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE);
+ UNIT_ASSERT_EQUAL(1, s.LabelsSize());
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "minimal_metric");
+ ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 1.234);
+ }
+ {
+ auto& s = samples.GetSamples(1);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 1);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "another_metric");
+ ASSERT_DOUBLE_POINT(s, TInstant::MilliSeconds(103948), -3000.0);
+ }
+ {
+ auto& s = samples.GetSamples(2);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE);
+ UNIT_ASSERT_EQUAL(1, s.LabelsSize());
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "no_labels");
+ ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 3.0);
+ }
+ }
+
+ Y_UNIT_TEST(Counter) {
+ auto samples = Decode(
+ "# A normal comment.\n"
+ "#\n"
+ "# TYPE name counter\n"
+ "name{labelname=\"val1\",basename=\"basevalue\"} NaN\n"
+ "name {labelname=\"val2\",basename=\"basevalue\"} 2.3 1234567890\n"
+ "# HELP name two-line\\n doc str\\\\ing\n");
+
+ UNIT_ASSERT_EQUAL(samples.SamplesSize(), 2);
+
+ {
+ auto& s = samples.GetSamples(0);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 3);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "name");
+ ASSERT_LABEL_EQUAL(s.GetLabels(1), "basename", "basevalue");
+ ASSERT_LABEL_EQUAL(s.GetLabels(2), "labelname", "val1");
+ ASSERT_UINT_POINT(s, TInstant::Zero(), ui64(0));
+ }
+ {
+ auto& s = samples.GetSamples(1);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 3);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "name");
+ ASSERT_LABEL_EQUAL(s.GetLabels(1), "basename", "basevalue");
+ ASSERT_LABEL_EQUAL(s.GetLabels(2), "labelname", "val2");
+ ASSERT_UINT_POINT(s, TInstant::MilliSeconds(1234567890), i64(2));
+ }
+ }
+
+ Y_UNIT_TEST(Gauge) {
+ auto samples = Decode(
+ "# A normal comment.\n"
+ "#\n"
+ " # HELP name2 \tdoc str\"ing 2\n"
+ " # TYPE name2 gauge\n"
+ "name2{labelname=\"val2\"\t,basename = \"basevalue2\"\t\t} +Inf 54321\n"
+ "name2{ labelname = \"val1\" , }-Inf\n");
+
+ UNIT_ASSERT_EQUAL(samples.SamplesSize(), 2);
+
+ {
+ auto& s = samples.GetSamples(0);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 3);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "name2");
+ ASSERT_LABEL_EQUAL(s.GetLabels(1), "basename", "basevalue2");
+ ASSERT_LABEL_EQUAL(s.GetLabels(2), "labelname", "val2");
+ ASSERT_DOUBLE_POINT(s, TInstant::MilliSeconds(54321), INFINITY);
+ }
+ {
+ auto& s = samples.GetSamples(1);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 2);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "name2");
+ ASSERT_LABEL_EQUAL(s.GetLabels(1), "labelname", "val1");
+ ASSERT_DOUBLE_POINT(s, TInstant::Zero(), -INFINITY);
+ }
+ }
+
+ Y_UNIT_TEST(Summary) {
+ auto samples = Decode(
+ "# HELP \n"
+ "# TYPE my_summary summary\n"
+ "my_summary{n1=\"val1\",quantile=\"0.5\"} 110\n"
+ "my_summary{n1=\"val1\",quantile=\"0.9\"} 140 1\n"
+ "my_summary_count{n1=\"val1\"} 42\n"
+ "my_summary_sum{n1=\"val1\"} 08 15\n"
+ "# some\n"
+ "# funny comments\n"
+ "# HELP\n"
+ "# HELP my_summary\n"
+ "# HELP my_summary \n");
+
+ UNIT_ASSERT_EQUAL(samples.SamplesSize(), 4);
+
+ {
+ auto& s = samples.GetSamples(0);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 3);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "my_summary");
+ ASSERT_LABEL_EQUAL(s.GetLabels(1), "quantile", "0.5");
+ ASSERT_LABEL_EQUAL(s.GetLabels(2), "n1", "val1");
+ ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 110.0);
+ }
+ {
+ auto& s = samples.GetSamples(1);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 3);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "my_summary");
+ ASSERT_LABEL_EQUAL(s.GetLabels(1), "quantile", "0.9");
+ ASSERT_LABEL_EQUAL(s.GetLabels(2), "n1", "val1");
+ ASSERT_DOUBLE_POINT(s, TInstant::MilliSeconds(1), 140.0);
+ }
+ {
+ auto& s = samples.GetSamples(2);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 2);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "my_summary_count");
+ ASSERT_LABEL_EQUAL(s.GetLabels(1), "n1", "val1");
+ ASSERT_UINT_POINT(s, TInstant::Zero(), 42);
+ }
+ {
+ auto& s = samples.GetSamples(3);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 2);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "my_summary_sum");
+ ASSERT_LABEL_EQUAL(s.GetLabels(1), "n1", "val1");
+ ASSERT_DOUBLE_POINT(s, TInstant::MilliSeconds(15), 8.0);
+ }
+ }
+
+ Y_UNIT_TEST(Histogram) {
+ auto samples = Decode(
+ "# HELP request_duration_microseconds The response latency.\n"
+ "# TYPE request_duration_microseconds histogram\n"
+ "request_duration_microseconds_bucket{le=\"0\"} 0\n"
+ "request_duration_microseconds_bucket{le=\"100\"} 123\n"
+ "request_duration_microseconds_bucket{le=\"120\"} 412\n"
+ "request_duration_microseconds_bucket{le=\"144\"} 592\n"
+ "request_duration_microseconds_bucket{le=\"172.8\"} 1524\n"
+ "request_duration_microseconds_bucket{le=\"+Inf\"} 2693\n"
+ "request_duration_microseconds_sum 1.7560473e+06\n"
+ "request_duration_microseconds_count 2693\n");
+
+ UNIT_ASSERT_EQUAL(samples.SamplesSize(), 3);
+
+ {
+ auto& s = samples.GetSamples(0);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 1);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "request_duration_microseconds_sum");
+ ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 1756047.3);
+ }
+ {
+ auto& s = samples.GetSamples(1);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 1);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "request_duration_microseconds_count");
+ ASSERT_UINT_POINT(s, TInstant::Zero(), 2693);
+ }
+ {
+ auto& s = samples.GetSamples(2);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::HIST_RATE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 1);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "request_duration_microseconds");
+ auto hist = ExplicitHistogramSnapshot(
+ { 0, 100, 120, 144, 172.8, HISTOGRAM_INF_BOUND },
+ { 0, 123, 289, 180, 932, 1169 });
+ ASSERT_HIST_POINT(s, TInstant::Zero(), *hist);
+ }
+ }
+
+ Y_UNIT_TEST(HistogramWithLabels) {
+ auto samples = Decode(
+ "# A histogram, which has a pretty complex representation in the text format:\n"
+ "# HELP http_request_duration_seconds A histogram of the request duration.\n"
+ "# TYPE http_request_duration_seconds histogram\n"
+ "http_request_duration_seconds_bucket{le=\"0.05\", method=\"POST\"} 24054\n"
+ "http_request_duration_seconds_bucket{method=\"POST\", le=\"0.1\"} 33444\n"
+ "http_request_duration_seconds_bucket{le=\"0.2\", method=\"POST\", } 100392\n"
+ "http_request_duration_seconds_bucket{le=\"0.5\",method=\"POST\",} 129389\n"
+ "http_request_duration_seconds_bucket{ method=\"POST\", le=\"1\", } 133988\n"
+ "http_request_duration_seconds_bucket{ le=\"+Inf\", method=\"POST\", } 144320\n"
+ "http_request_duration_seconds_sum{method=\"POST\"} 53423\n"
+ "http_request_duration_seconds_count{ method=\"POST\", } 144320\n");
+
+ UNIT_ASSERT_EQUAL(samples.SamplesSize(), 3);
+
+ {
+ auto& s = samples.GetSamples(0);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 2);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "http_request_duration_seconds_sum");
+ ASSERT_LABEL_EQUAL(s.GetLabels(1), "method", "POST");
+ ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 53423.0);
+ }
+ {
+ auto& s = samples.GetSamples(1);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 2);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "http_request_duration_seconds_count");
+ ASSERT_LABEL_EQUAL(s.GetLabels(1), "method", "POST");
+ ASSERT_UINT_POINT(s, TInstant::Zero(), 144320);
+ }
+ {
+ auto& s = samples.GetSamples(2);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::HIST_RATE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 2);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "http_request_duration_seconds");
+ ASSERT_LABEL_EQUAL(s.GetLabels(1), "method", "POST");
+ auto hist = ExplicitHistogramSnapshot(
+ { 0.05, 0.1, 0.2, 0.5, 1, HISTOGRAM_INF_BOUND },
+ { 24054, 9390, 66948, 28997, 4599, 10332 });
+ ASSERT_HIST_POINT(s, TInstant::Zero(), *hist);
+ }
+ }
+
+ Y_UNIT_TEST(MultipleHistograms) {
+ auto samples = Decode(
+ "# TYPE inboundBytesPerSec histogram\n"
+ "inboundBytesPerSec_bucket{client=\"mbus\", le=\"10.0\"} 1.0\n"
+ "inboundBytesPerSec_bucket{client=\"mbus\", le=\"20.0\"} 5.0\n"
+ "inboundBytesPerSec_bucket{client=\"mbus\", le=\"+Inf\"} 5.0\n"
+ "inboundBytesPerSec_count{client=\"mbus\"} 5.0\n"
+ "inboundBytesPerSec_bucket{client=\"grpc\", le=\"10.0\"} 1.0\n"
+ "inboundBytesPerSec_bucket{client=\"grpc\", le=\"20.0\"} 5.0\n"
+ "inboundBytesPerSec_bucket{client=\"grpc\", le=\"30.0\"} 5.0\n"
+ "inboundBytesPerSec_count{client=\"grpc\"} 5.0\n"
+ "# TYPE outboundBytesPerSec histogram\n"
+ "outboundBytesPerSec_bucket{client=\"grpc\", le=\"100.0\"} 1.0 1512216000000\n"
+ "outboundBytesPerSec_bucket{client=\"grpc\", le=\"200.0\"} 1.0 1512216000000\n"
+ "outboundBytesPerSec_bucket{client=\"grpc\", le=\"+Inf\"} 1.0 1512216000000\n"
+ "outboundBytesPerSec_count{client=\"grpc\"} 1.0 1512216000000\n");
+
+ UNIT_ASSERT_EQUAL(samples.SamplesSize(), 6);
+
+ {
+ auto& s = samples.GetSamples(0);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 2);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "inboundBytesPerSec_count");
+ ASSERT_LABEL_EQUAL(s.GetLabels(1), "client", "mbus");
+ ASSERT_UINT_POINT(s, TInstant::Zero(), 5);
+ }
+ {
+ auto& s = samples.GetSamples(1);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::HIST_RATE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 2);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "inboundBytesPerSec");
+ ASSERT_LABEL_EQUAL(s.GetLabels(1), "client", "mbus");
+ auto hist = ExplicitHistogramSnapshot(
+ { 10, 20, HISTOGRAM_INF_BOUND },
+ { 1, 4, 0 });
+ ASSERT_HIST_POINT(s, TInstant::Zero(), *hist);
+ }
+ {
+ auto& s = samples.GetSamples(2);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 2);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "inboundBytesPerSec_count");
+ ASSERT_LABEL_EQUAL(s.GetLabels(1), "client", "grpc");
+ ASSERT_UINT_POINT(s, TInstant::Zero(), 5);
+ }
+ {
+ auto& s = samples.GetSamples(3);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::HIST_RATE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 2);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "inboundBytesPerSec");
+ ASSERT_LABEL_EQUAL(s.GetLabels(1), "client", "grpc");
+ auto hist = ExplicitHistogramSnapshot(
+ { 10, 20, 30 },
+ { 1, 4, 0 });
+ ASSERT_HIST_POINT(s, TInstant::Zero(), *hist);
+ }
+ {
+ auto& s = samples.GetSamples(4);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 2);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "outboundBytesPerSec_count");
+ ASSERT_LABEL_EQUAL(s.GetLabels(1), "client", "grpc");
+ ASSERT_UINT_POINT(s, TInstant::Seconds(1512216000), 1) ;
+ }
+ {
+ auto& s = samples.GetSamples(5);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::HIST_RATE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 2);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "outboundBytesPerSec");
+ ASSERT_LABEL_EQUAL(s.GetLabels(1), "client", "grpc");
+ auto hist = ExplicitHistogramSnapshot(
+ { 100, 200, HISTOGRAM_INF_BOUND },
+ { 1, 0, 0 });
+ ASSERT_HIST_POINT(s, TInstant::Seconds(1512216000), *hist);
+ }
+ }
+
+ Y_UNIT_TEST(MixedTypes) {
+ auto samples = Decode(
+ "# HELP http_requests_total The total number of HTTP requests.\n"
+ "# TYPE http_requests_total counter\n"
+ "http_requests_total { } 1027 1395066363000\n"
+ "http_requests_total{method=\"post\",code=\"200\"} 1027 1395066363000\n"
+ "http_requests_total{method=\"post\",code=\"400\"} 3 1395066363000\n"
+ "\n"
+ "# Minimalistic line:\n"
+ "metric_without_timestamp_and_labels 12.47\n"
+ "\n"
+ "# HELP rpc_duration_seconds A summary of the RPC duration in seconds.\n"
+ "# TYPE rpc_duration_seconds summary\n"
+ "rpc_duration_seconds{quantile=\"0.01\"} 3102\n"
+ "rpc_duration_seconds{quantile=\"0.5\"} 4773\n"
+ "rpc_duration_seconds{quantile=\"0.9\"} 9001\n"
+ "rpc_duration_seconds_sum 1.7560473e+07\n"
+ "rpc_duration_seconds_count 2693\n"
+ "\n"
+ "# Another mMinimalistic line:\n"
+ "metric_with_timestamp 12.47 1234567890\n");
+
+ UNIT_ASSERT_EQUAL(samples.SamplesSize(), 10);
+
+ {
+ auto& s = samples.GetSamples(0);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 1);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "http_requests_total");
+ ASSERT_UINT_POINT(s, TInstant::Seconds(1395066363), 1027);
+ }
+ {
+ auto& s = samples.GetSamples(1);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 3);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "http_requests_total");
+ ASSERT_LABEL_EQUAL(s.GetLabels(1), "method", "post");
+ ASSERT_LABEL_EQUAL(s.GetLabels(2), "code", "200");
+ ASSERT_UINT_POINT(s, TInstant::Seconds(1395066363), 1027);
+ }
+ {
+ auto& s = samples.GetSamples(2);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 3);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "http_requests_total");
+ ASSERT_LABEL_EQUAL(s.GetLabels(1), "method", "post");
+ ASSERT_LABEL_EQUAL(s.GetLabels(2), "code", "400");
+ ASSERT_UINT_POINT(s, TInstant::Seconds(1395066363), 3);
+ }
+ {
+ auto& s = samples.GetSamples(3);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 1);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "metric_without_timestamp_and_labels");
+ ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 12.47);
+ }
+ {
+ auto& s = samples.GetSamples(4);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 2);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "rpc_duration_seconds");
+ ASSERT_LABEL_EQUAL(s.GetLabels(1), "quantile", "0.01");
+ ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 3102);
+ }
+ {
+ auto& s = samples.GetSamples(5);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 2);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "rpc_duration_seconds");
+ ASSERT_LABEL_EQUAL(s.GetLabels(1), "quantile", "0.5");
+ ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 4773);
+ }
+ {
+ auto& s = samples.GetSamples(6);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 2);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "rpc_duration_seconds");
+ ASSERT_LABEL_EQUAL(s.GetLabels(1), "quantile", "0.9");
+ ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 9001);
+ }
+ {
+ auto& s = samples.GetSamples(7);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 1);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "rpc_duration_seconds_sum");
+ ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 17560473);
+ }
+ {
+ auto& s = samples.GetSamples(8);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 1);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "rpc_duration_seconds_count");
+ ASSERT_UINT_POINT(s, TInstant::Zero(), 2693);
+ }
+ {
+ auto& s = samples.GetSamples(9);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 1);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "metric_with_timestamp");
+ ASSERT_DOUBLE_POINT(s, TInstant::MilliSeconds(1234567890), 12.47);
+ }
+ }
+}
diff --git a/library/cpp/monlib/encode/prometheus/prometheus_encoder.cpp b/library/cpp/monlib/encode/prometheus/prometheus_encoder.cpp
new file mode 100644
index 0000000000..15efeb8c03
--- /dev/null
+++ b/library/cpp/monlib/encode/prometheus/prometheus_encoder.cpp
@@ -0,0 +1,413 @@
+#include "prometheus.h"
+#include "prometheus_model.h"
+
+#include <library/cpp/monlib/encode/encoder_state.h>
+#include <library/cpp/monlib/metrics/labels.h>
+#include <library/cpp/monlib/metrics/metric_value.h>
+
+#include <util/string/cast.h>
+#include <util/generic/hash_set.h>
+
+
+namespace NMonitoring {
+ namespace {
+ ///////////////////////////////////////////////////////////////////////
+ // TPrometheusWriter
+ ///////////////////////////////////////////////////////////////////////
+ class TPrometheusWriter {
+ public:
+ explicit TPrometheusWriter(IOutputStream* out)
+ : Out_(out)
+ {
+ }
+
+ void WriteType(EMetricType type, const TString& name) {
+ auto r = WrittenTypes_.insert(name);
+ if (!r.second) {
+ // type for this metric was already written
+ return;
+ }
+
+ Out_->Write("# TYPE ");
+ WriteMetricName(name);
+ Out_->Write(' ');
+
+ switch (type) {
+ case EMetricType::GAUGE:
+ case EMetricType::IGAUGE:
+ Out_->Write("gauge");
+ break;
+ case EMetricType::RATE:
+ case EMetricType::COUNTER:
+ Out_->Write("counter");
+ break;
+ case EMetricType::HIST:
+ case EMetricType::HIST_RATE:
+ Out_->Write("histogram");
+ break;
+ case EMetricType::LOGHIST:
+ // TODO(@kbalakirev): implement this case
+ break;
+ case EMetricType::DSUMMARY:
+ ythrow yexception() << "writing summary type is forbiden";
+ case EMetricType::UNKNOWN:
+ ythrow yexception() << "unknown metric type: " << MetricTypeToStr(type)
+ << ", name: " << name;
+ }
+ Out_->Write('\n');
+ }
+
+ void WriteDouble(TStringBuf name, const TLabels& labels, TInstant time, double value) {
+ WriteValue(name, "", labels, "", "", time, value);
+ }
+
+ void WriteHistogram(TStringBuf name, const TLabels& labels, TInstant time, IHistogramSnapshot* h) {
+ Y_ENSURE(!labels.Has(NPrometheus::BUCKET_LABEL),
+ "histogram metric " << name << " has label '" <<
+ NPrometheus::BUCKET_LABEL << "' which is reserved in Prometheus");
+
+ double totalCount = 0;
+ for (ui32 i = 0, count = h->Count(); i < count; i++) {
+ TBucketBound bound = h->UpperBound(i);
+ TStringBuf boundStr;
+ if (bound == HISTOGRAM_INF_BOUND) {
+ boundStr = TStringBuf("+Inf");
+ } else {
+ size_t len = FloatToString(bound, TmpBuf_, Y_ARRAY_SIZE(TmpBuf_));
+ boundStr = TStringBuf(TmpBuf_, len);
+ }
+
+ TBucketValue value = h->Value(i);
+ totalCount += static_cast<double>(value);
+
+ WriteValue(
+ name, NPrometheus::BUCKET_SUFFIX,
+ labels, NPrometheus::BUCKET_LABEL, boundStr,
+ time,
+ totalCount);
+ }
+
+ WriteValue(name, NPrometheus::COUNT_SUFFIX, labels, "", "", time, totalCount);
+ }
+
+ void WriteSummaryDouble(TStringBuf name, const TLabels& labels, TInstant time, ISummaryDoubleSnapshot* s) {
+ WriteValue(name, NPrometheus::SUM_SUFFIX, labels, "", "", time, s->GetSum());
+ WriteValue(name, NPrometheus::MIN_SUFFIX, labels, "", "", time, s->GetMin());
+ WriteValue(name, NPrometheus::MAX_SUFFIX, labels, "", "", time, s->GetMax());
+ WriteValue(name, NPrometheus::LAST_SUFFIX, labels, "", "", time, s->GetLast());
+ WriteValue(name, NPrometheus::COUNT_SUFFIX, labels, "", "", time, s->GetCount());
+ }
+
+ void WriteLn() {
+ Out_->Write('\n');
+ }
+
+ private:
+ // will replace invalid chars with '_'
+ void WriteMetricName(TStringBuf name) {
+ Y_ENSURE(!name.Empty(), "trying to write metric with empty name");
+
+ char ch = name[0];
+ if (NPrometheus::IsValidMetricNameStart(ch)) {
+ Out_->Write(ch);
+ } else {
+ Out_->Write('_');
+ }
+
+ for (size_t i = 1, len = name.length(); i < len; i++) {
+ ch = name[i];
+ if (NPrometheus::IsValidMetricNameContinuation(ch)) {
+ Out_->Write(ch);
+ } else {
+ Out_->Write('_');
+ }
+ }
+ }
+
+ void WriteLabels(const TLabels& labels, TStringBuf addLabelKey, TStringBuf addLabelValue) {
+ Out_->Write('{');
+ for (auto&& l: labels) {
+ Out_->Write(l.Name());
+ Out_->Write('=');
+ WriteLabelValue(l.Value());
+ Out_->Write(", "); // trailign comma is supported in parsers
+ }
+ if (!addLabelKey.Empty() && !addLabelValue.Empty()) {
+ Out_->Write(addLabelKey);
+ Out_->Write('=');
+ WriteLabelValue(addLabelValue);
+ }
+ Out_->Write('}');
+ }
+
+ void WriteLabelValue(TStringBuf value) {
+ Out_->Write('"');
+ for (char ch: value) {
+ if (ch == '"') {
+ Out_->Write("\\\"");
+ } else if (ch == '\\') {
+ Out_->Write("\\\\");
+ } else if (ch == '\n') {
+ Out_->Write("\\n");
+ } else {
+ Out_->Write(ch);
+ }
+ }
+ Out_->Write('"');
+ }
+
+ void WriteValue(
+ TStringBuf name, TStringBuf suffix,
+ const TLabels& labels, TStringBuf addLabelKey, TStringBuf addLabelValue,
+ TInstant time, double value)
+ {
+ // (1) name
+ WriteMetricName(name);
+ if (!suffix.Empty()) {
+ Out_->Write(suffix);
+ }
+
+ // (2) labels
+ if (!labels.Empty() || !addLabelKey.Empty()) {
+ WriteLabels(labels, addLabelKey, addLabelValue);
+ }
+ Out_->Write(' ');
+
+ // (3) value
+ {
+ size_t len = FloatToString(value, TmpBuf_, Y_ARRAY_SIZE(TmpBuf_));
+ Out_->Write(TmpBuf_, len);
+ }
+
+ // (4) time
+ if (ui64 timeMillis = time.MilliSeconds()) {
+ Out_->Write(' ');
+ size_t len = IntToString<10>(timeMillis, TmpBuf_, Y_ARRAY_SIZE(TmpBuf_));
+ Out_->Write(TmpBuf_, len);
+ }
+ Out_->Write('\n');
+ }
+
+ private:
+ IOutputStream* Out_;
+ THashSet<TString> WrittenTypes_;
+ char TmpBuf_[512]; // used to convert doubles to strings
+ };
+
+ ///////////////////////////////////////////////////////////////////////
+ // TMetricState
+ ///////////////////////////////////////////////////////////////////////
+ struct TMetricState {
+ EMetricType Type = EMetricType::UNKNOWN;
+ TLabels Labels;
+ TInstant Time = TInstant::Zero();
+ EMetricValueType ValueType = EMetricValueType::UNKNOWN;
+ TMetricValue Value;
+
+ ~TMetricState() {
+ ClearValue();
+ }
+
+ void Clear() {
+ Type = EMetricType::UNKNOWN;
+ Labels.Clear();
+ Time = TInstant::Zero();
+ ClearValue();
+ }
+
+ void ClearValue() {
+ // TMetricValue does not keep ownership of histogram
+ if (ValueType == EMetricValueType::HISTOGRAM) {
+ Value.AsHistogram()->UnRef();
+ } else if (ValueType == EMetricValueType::SUMMARY) {
+ Value.AsSummaryDouble()->UnRef();
+ }
+ ValueType = EMetricValueType::UNKNOWN;
+ Value = {};
+ }
+
+ template <typename T>
+ void SetValue(T value) {
+ // TMetricValue does not keep ownership of histogram
+ if (ValueType == EMetricValueType::HISTOGRAM) {
+ Value.AsHistogram()->UnRef();
+ } else if (ValueType == EMetricValueType::SUMMARY) {
+ Value.AsSummaryDouble()->UnRef();
+ }
+ ValueType = TValueType<T>::Type;
+ Value = TMetricValue(value);
+ if (ValueType == EMetricValueType::HISTOGRAM) {
+ Value.AsHistogram()->Ref();
+ } else if (ValueType == EMetricValueType::SUMMARY) {
+ Value.AsSummaryDouble()->Ref();
+ }
+ }
+ };
+
+ ///////////////////////////////////////////////////////////////////////
+ // TPrometheusEncoder
+ ///////////////////////////////////////////////////////////////////////
+ class TPrometheusEncoder final: public IMetricEncoder {
+ public:
+ explicit TPrometheusEncoder(IOutputStream* out, TStringBuf metricNameLabel)
+ : Writer_(out)
+ , MetricNameLabel_(metricNameLabel)
+ {
+ }
+
+ private:
+ void OnStreamBegin() override {
+ State_.Expect(TEncoderState::EState::ROOT);
+ }
+
+ void OnStreamEnd() override {
+ State_.Expect(TEncoderState::EState::ROOT);
+ Writer_.WriteLn();
+ }
+
+ void OnCommonTime(TInstant time) override {
+ State_.Expect(TEncoderState::EState::ROOT);
+ CommonTime_ = time;
+ }
+
+ void OnMetricBegin(EMetricType type) override {
+ State_.Switch(TEncoderState::EState::ROOT, TEncoderState::EState::METRIC);
+ MetricState_.Clear();
+ MetricState_.Type = type;
+ }
+
+ void OnMetricEnd() override {
+ State_.Switch(TEncoderState::EState::METRIC, TEncoderState::EState::ROOT);
+ WriteMetric();
+ }
+
+ void OnLabelsBegin() override {
+ if (State_ == TEncoderState::EState::METRIC) {
+ State_ = TEncoderState::EState::METRIC_LABELS;
+ } else if (State_ == TEncoderState::EState::ROOT) {
+ State_ = TEncoderState::EState::COMMON_LABELS;
+ } else {
+ State_.ThrowInvalid("expected METRIC or ROOT");
+ }
+ }
+
+ void OnLabelsEnd() override {
+ if (State_ == TEncoderState::EState::METRIC_LABELS) {
+ State_ = TEncoderState::EState::METRIC;
+ } else if (State_ == TEncoderState::EState::COMMON_LABELS) {
+ State_ = TEncoderState::EState::ROOT;
+ } else {
+ State_.ThrowInvalid("expected LABELS or COMMON_LABELS");
+ }
+ }
+
+ void OnLabel(TStringBuf name, TStringBuf value) override {
+ if (State_ == TEncoderState::EState::METRIC_LABELS) {
+ MetricState_.Labels.Add(name, value);
+ } else if (State_ == TEncoderState::EState::COMMON_LABELS) {
+ CommonLabels_.Add(name, value);
+ } else {
+ State_.ThrowInvalid("expected LABELS or COMMON_LABELS");
+ }
+ }
+
+ void OnDouble(TInstant time, double value) override {
+ State_.Expect(TEncoderState::EState::METRIC);
+ MetricState_.Time = time;
+ MetricState_.SetValue(value);
+ }
+
+ void OnInt64(TInstant time, i64 value) override {
+ State_.Expect(TEncoderState::EState::METRIC);
+ MetricState_.Time = time;
+ MetricState_.SetValue(value);
+ }
+
+ void OnUint64(TInstant time, ui64 value) override {
+ State_.Expect(TEncoderState::EState::METRIC);
+ MetricState_.Time = time;
+ MetricState_.SetValue(value);
+ }
+
+ void OnHistogram(TInstant time, IHistogramSnapshotPtr snapshot) override {
+ State_.Expect(TEncoderState::EState::METRIC);
+ MetricState_.Time = time;
+ MetricState_.SetValue(snapshot.Get());
+ }
+
+ void OnSummaryDouble(TInstant time, ISummaryDoubleSnapshotPtr snapshot) override {
+ State_.Expect(TEncoderState::EState::METRIC);
+ MetricState_.Time = time;
+ MetricState_.SetValue(snapshot.Get());
+ }
+
+ void OnLogHistogram(TInstant, TLogHistogramSnapshotPtr) override {
+ // TODO(@kbalakirev): implement this function
+ }
+
+ void Close() override {
+ }
+
+ void WriteMetric() {
+ if (MetricState_.ValueType == EMetricValueType::UNKNOWN) {
+ return;
+ }
+
+ // XXX: poor performace
+ for (auto&& l: CommonLabels_) {
+ MetricState_.Labels.Add(l.Name(), l.Value());
+ }
+
+ TMaybe<TLabel> nameLabel = MetricState_.Labels.Extract(MetricNameLabel_);
+ Y_ENSURE(nameLabel,
+ "labels " << MetricState_.Labels <<
+ " does not contain label '" << MetricNameLabel_ << '\'');
+
+ const TString& metricName = ToString(nameLabel->Value());
+ if (MetricState_.Type != EMetricType::DSUMMARY) {
+ Writer_.WriteType(MetricState_.Type, metricName);
+ }
+
+ if (MetricState_.Time == TInstant::Zero()) {
+ MetricState_.Time = CommonTime_;
+ }
+
+ EMetricType type = MetricState_.Type;
+ if (type == EMetricType::HIST || type == EMetricType::HIST_RATE) {
+ Y_ENSURE(MetricState_.ValueType == EMetricValueType::HISTOGRAM,
+ "invalid value type for histogram: " << int(MetricState_.ValueType)); // TODO: to string conversion
+ Writer_.WriteHistogram(
+ metricName,
+ MetricState_.Labels,
+ MetricState_.Time,
+ MetricState_.Value.AsHistogram());
+ } else if (type == EMetricType::DSUMMARY) {
+ Writer_.WriteSummaryDouble(
+ metricName,
+ MetricState_.Labels,
+ MetricState_.Time,
+ MetricState_.Value.AsSummaryDouble());
+ } else {
+ Writer_.WriteDouble(
+ metricName,
+ MetricState_.Labels,
+ MetricState_.Time,
+ MetricState_.Value.AsDouble(MetricState_.ValueType));
+ }
+ }
+
+ private:
+ TEncoderState State_;
+ TPrometheusWriter Writer_;
+ TString MetricNameLabel_;
+ TInstant CommonTime_ = TInstant::Zero();
+ TLabels CommonLabels_;
+ TMetricState MetricState_;
+ };
+ }
+
+ IMetricEncoderPtr EncoderPrometheus(IOutputStream* out, TStringBuf metricNameLabel) {
+ return MakeHolder<TPrometheusEncoder>(out, metricNameLabel);
+ }
+
+} // namespace NMonitoring
diff --git a/library/cpp/monlib/encode/prometheus/prometheus_encoder_ut.cpp b/library/cpp/monlib/encode/prometheus/prometheus_encoder_ut.cpp
new file mode 100644
index 0000000000..fd9debb060
--- /dev/null
+++ b/library/cpp/monlib/encode/prometheus/prometheus_encoder_ut.cpp
@@ -0,0 +1,414 @@
+#include "prometheus.h"
+
+#include <library/cpp/monlib/encode/protobuf/protobuf.h>
+#include <library/cpp/monlib/metrics/metric_value.h>
+#include <library/cpp/monlib/metrics/histogram_snapshot.h>
+
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <util/stream/str.h>
+
+using namespace NMonitoring;
+
+Y_UNIT_TEST_SUITE(TPrometheusEncoderTest) {
+
+ template <typename TFunc>
+ TString EncodeToString(TFunc fn) {
+ TStringStream ss;
+ IMetricEncoderPtr encoder = EncoderPrometheus(&ss);
+ fn(encoder.Get());
+ return ss.Str();
+ }
+
+ ISummaryDoubleSnapshotPtr TestSummaryDouble() {
+ return MakeIntrusive<TSummaryDoubleSnapshot>(10.1, -0.45, 0.478, 0.3, 30u);
+ }
+
+ Y_UNIT_TEST(Empty) {
+ auto result = EncodeToString([](IMetricEncoder* e) {
+ e->OnStreamBegin();
+ e->OnStreamEnd();
+ });
+ UNIT_ASSERT_STRINGS_EQUAL(result, "\n");
+ }
+
+ Y_UNIT_TEST(DoubleGauge) {
+ auto result = EncodeToString([](IMetricEncoder* e) {
+ e->OnStreamBegin();
+ { // no values
+ e->OnMetricBegin(EMetricType::GAUGE);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "cpuUsage");
+ e->OnLabelsEnd();
+ }
+ e->OnMetricEnd();
+ }
+ { // one value no ts
+ e->OnMetricBegin(EMetricType::GAUGE);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "diskUsage");
+ e->OnLabel("disk", "sda1");
+ e->OnLabelsEnd();
+ }
+ e->OnDouble(TInstant::Zero(), 1000);
+ e->OnMetricEnd();
+ }
+ { // one value with ts
+ e->OnMetricBegin(EMetricType::GAUGE);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "memoryUsage");
+ e->OnLabel("host", "solomon-man-00");
+ e->OnLabel("dc", "man");
+ e->OnLabelsEnd();
+ }
+ e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 1000);
+ e->OnMetricEnd();
+ }
+ { // many values
+ e->OnMetricBegin(EMetricType::GAUGE);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "bytesRx");
+ e->OnLabel("host", "solomon-sas-01");
+ e->OnLabel("dc", "sas");
+ e->OnLabelsEnd();
+ }
+ e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 2);
+ e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:05Z"), 4);
+ e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:10Z"), 8);
+ e->OnMetricEnd();
+ }
+ { // already seen metric name
+ e->OnMetricBegin(EMetricType::GAUGE);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "diskUsage");
+ e->OnLabel("disk", "sdb1");
+ e->OnLabelsEnd();
+ }
+ e->OnDouble(TInstant::Zero(), 1001);
+ e->OnMetricEnd();
+ }
+ { // NaN
+ e->OnMetricBegin(EMetricType::GAUGE);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "nanValue");
+ e->OnLabelsEnd();
+ }
+ e->OnDouble(TInstant::Zero(), NAN);
+ e->OnMetricEnd();
+ }
+ { // Inf
+ e->OnMetricBegin(EMetricType::GAUGE);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "infValue");
+ e->OnLabelsEnd();
+ }
+ e->OnDouble(TInstant::Zero(), INFINITY);
+ e->OnMetricEnd();
+ }
+ {
+ e->OnMetricBegin(EMetricType::DSUMMARY);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "seconds");
+ e->OnLabel("disk", "sdb1");
+ e->OnLabelsEnd();
+ }
+ e->OnSummaryDouble(TInstant::Zero(), TestSummaryDouble());
+ e->OnMetricEnd();
+ }
+ e->OnStreamEnd();
+ });
+
+ UNIT_ASSERT_STRINGS_EQUAL(result,
+ "# TYPE diskUsage gauge\n"
+ "diskUsage{disk=\"sda1\", } 1000\n"
+ "# TYPE memoryUsage gauge\n"
+ "memoryUsage{host=\"solomon-man-00\", dc=\"man\", } 1000 1512216000000\n"
+ "# TYPE bytesRx gauge\n"
+ "bytesRx{host=\"solomon-sas-01\", dc=\"sas\", } 8 1512216010000\n"
+ "diskUsage{disk=\"sdb1\", } 1001\n"
+ "# TYPE nanValue gauge\n"
+ "nanValue nan\n"
+ "# TYPE infValue gauge\n"
+ "infValue inf\n"
+ "seconds_sum{disk=\"sdb1\", } 10.1\n"
+ "seconds_min{disk=\"sdb1\", } -0.45\n"
+ "seconds_max{disk=\"sdb1\", } 0.478\n"
+ "seconds_last{disk=\"sdb1\", } 0.3\n"
+ "seconds_count{disk=\"sdb1\", } 30\n"
+ "\n");
+ }
+
+ Y_UNIT_TEST(IntGauges) {
+ auto result = EncodeToString([](IMetricEncoder* e) {
+ e->OnStreamBegin();
+ { // no values
+ e->OnMetricBegin(EMetricType::IGAUGE);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "cpuUsage");
+ e->OnLabelsEnd();
+ }
+ e->OnMetricEnd();
+ }
+ { // one value no ts
+ e->OnMetricBegin(EMetricType::IGAUGE);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "diskUsage");
+ e->OnLabel("disk", "sda1");
+ e->OnLabelsEnd();
+ }
+ e->OnInt64(TInstant::Zero(), 1000);
+ e->OnMetricEnd();
+ }
+ { // one value with ts
+ e->OnMetricBegin(EMetricType::IGAUGE);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "memoryUsage");
+ e->OnLabel("dc", "man");
+ e->OnLabel("host", "solomon-man-00");
+ e->OnLabelsEnd();
+ }
+ e->OnInt64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 1000);
+ e->OnMetricEnd();
+ }
+ { // many values
+ e->OnMetricBegin(EMetricType::IGAUGE);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "bytesRx");
+ e->OnLabel("dc", "sas");
+ e->OnLabel("host", "solomon-sas-01");
+ e->OnLabelsEnd();
+ }
+ e->OnInt64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 2);
+ e->OnInt64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:05Z"), 4);
+ e->OnInt64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:10Z"), 8);
+ e->OnMetricEnd();
+ }
+ e->OnStreamEnd();
+ });
+
+ UNIT_ASSERT_STRINGS_EQUAL(result,
+ "# TYPE diskUsage gauge\n"
+ "diskUsage{disk=\"sda1\", } 1000\n"
+ "# TYPE memoryUsage gauge\n"
+ "memoryUsage{dc=\"man\", host=\"solomon-man-00\", } 1000 1512216000000\n"
+ "# TYPE bytesRx gauge\n"
+ "bytesRx{dc=\"sas\", host=\"solomon-sas-01\", } 8 1512216010000\n"
+ "\n");
+ }
+
+ Y_UNIT_TEST(Counters) {
+ auto result = EncodeToString([](IMetricEncoder* e) {
+ e->OnStreamBegin();
+ { // no values
+ e->OnMetricBegin(EMetricType::COUNTER);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "cpuUsage");
+ e->OnLabelsEnd();
+ }
+ e->OnMetricEnd();
+ }
+ { // one value no ts
+ e->OnMetricBegin(EMetricType::COUNTER);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "diskUsage");
+ e->OnLabel("disk", "sda1");
+ e->OnLabelsEnd();
+ }
+ e->OnInt64(TInstant::Zero(), 1000);
+ e->OnMetricEnd();
+ }
+ { // one value with ts
+ e->OnMetricBegin(EMetricType::COUNTER);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "memoryUsage");
+ e->OnLabel("host", "solomon-man-00");
+ e->OnLabel("dc", "man");
+ e->OnLabelsEnd();
+ }
+ e->OnInt64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 1000);
+ e->OnMetricEnd();
+ }
+ { // many values
+ e->OnMetricBegin(EMetricType::COUNTER);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "bytesRx");
+ e->OnLabel("host", "solomon-sas-01");
+ e->OnLabel("dc", "sas");
+ e->OnLabelsEnd();
+ }
+ e->OnInt64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 2);
+ e->OnInt64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:05Z"), 4);
+ e->OnInt64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:10Z"), 8);
+ e->OnMetricEnd();
+ }
+ e->OnStreamEnd();
+ });
+
+ UNIT_ASSERT_STRINGS_EQUAL(result,
+ "# TYPE diskUsage counter\n"
+ "diskUsage{disk=\"sda1\", } 1000\n"
+ "# TYPE memoryUsage counter\n"
+ "memoryUsage{host=\"solomon-man-00\", dc=\"man\", } 1000 1512216000000\n"
+ "# TYPE bytesRx counter\n"
+ "bytesRx{host=\"solomon-sas-01\", dc=\"sas\", } 8 1512216010000\n"
+ "\n");
+ }
+
+ Y_UNIT_TEST(Histograms) {
+ auto result = EncodeToString([](IMetricEncoder* e) {
+ e->OnStreamBegin();
+ { // no values histogram
+ e->OnMetricBegin(EMetricType::HIST);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "cpuUsage");
+ e->OnLabelsEnd();
+ }
+ e->OnMetricEnd();
+ }
+ { // one value no ts
+ e->OnMetricBegin(EMetricType::HIST);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "inboundBytesPerSec");
+ e->OnLabel("client", "mbus");
+ e->OnLabelsEnd();
+ }
+ e->OnHistogram(
+ TInstant::Zero(),
+ ExplicitHistogramSnapshot({10, 20, HISTOGRAM_INF_BOUND}, {1, 4, 0}));
+ e->OnMetricEnd();
+ }
+ { // one value no ts no +inf bucket
+ e->OnMetricBegin(EMetricType::HIST);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "inboundBytesPerSec");
+ e->OnLabel("client", "grpc");
+ e->OnLabelsEnd();
+ }
+ e->OnHistogram(
+ TInstant::Zero(),
+ ExplicitHistogramSnapshot({10, 20, 30}, {1, 4, 0}));
+ e->OnMetricEnd();
+ }
+ { // one value with ts
+ e->OnMetricBegin(EMetricType::HIST_RATE);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "outboundBytesPerSec");
+ e->OnLabel("client", "grps");
+ e->OnLabelsEnd();
+ }
+ e->OnHistogram(
+ TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"),
+ ExplicitHistogramSnapshot({100, 200, HISTOGRAM_INF_BOUND}, {1, 0, 0}));
+ e->OnMetricEnd();
+ }
+ { // many values
+ e->OnMetricBegin(EMetricType::HIST);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "bytesRx");
+ e->OnLabel("host", "solomon-sas-01");
+ e->OnLabel("dc", "sas");
+ e->OnLabelsEnd();
+ }
+ TBucketBounds bounds = {100, 200, HISTOGRAM_INF_BOUND};
+ e->OnHistogram(
+ TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"),
+ ExplicitHistogramSnapshot(bounds, {10, 0, 0}));
+ e->OnHistogram(
+ TInstant::ParseIso8601Deprecated("2017-12-02T12:00:05Z"),
+ ExplicitHistogramSnapshot(bounds, {10, 2, 0}));
+ e->OnHistogram(
+ TInstant::ParseIso8601Deprecated("2017-12-02T12:00:10Z"),
+ ExplicitHistogramSnapshot(bounds, {10, 2, 5}));
+ e->OnMetricEnd();
+ }
+ e->OnStreamEnd();
+ });
+
+ UNIT_ASSERT_STRINGS_EQUAL(result,
+ "# TYPE inboundBytesPerSec histogram\n"
+ "inboundBytesPerSec_bucket{client=\"mbus\", le=\"10\"} 1\n"
+ "inboundBytesPerSec_bucket{client=\"mbus\", le=\"20\"} 5\n"
+ "inboundBytesPerSec_bucket{client=\"mbus\", le=\"+Inf\"} 5\n"
+ "inboundBytesPerSec_count{client=\"mbus\", } 5\n"
+ "inboundBytesPerSec_bucket{client=\"grpc\", le=\"10\"} 1\n"
+ "inboundBytesPerSec_bucket{client=\"grpc\", le=\"20\"} 5\n"
+ "inboundBytesPerSec_bucket{client=\"grpc\", le=\"30\"} 5\n"
+ "inboundBytesPerSec_count{client=\"grpc\", } 5\n"
+ "# TYPE outboundBytesPerSec histogram\n"
+ "outboundBytesPerSec_bucket{client=\"grps\", le=\"100\"} 1 1512216000000\n"
+ "outboundBytesPerSec_bucket{client=\"grps\", le=\"200\"} 1 1512216000000\n"
+ "outboundBytesPerSec_bucket{client=\"grps\", le=\"+Inf\"} 1 1512216000000\n"
+ "outboundBytesPerSec_count{client=\"grps\", } 1 1512216000000\n"
+ "# TYPE bytesRx histogram\n"
+ "bytesRx_bucket{host=\"solomon-sas-01\", dc=\"sas\", le=\"100\"} 10 1512216010000\n"
+ "bytesRx_bucket{host=\"solomon-sas-01\", dc=\"sas\", le=\"200\"} 12 1512216010000\n"
+ "bytesRx_bucket{host=\"solomon-sas-01\", dc=\"sas\", le=\"+Inf\"} 17 1512216010000\n"
+ "bytesRx_count{host=\"solomon-sas-01\", dc=\"sas\", } 17 1512216010000\n"
+ "\n");
+ }
+
+ Y_UNIT_TEST(CommonLables) {
+ auto result = EncodeToString([](IMetricEncoder* e) {
+ e->OnStreamBegin();
+ { // common time
+ e->OnCommonTime(TInstant::Seconds(1500000000));
+ }
+ { // common labels
+ e->OnLabelsBegin();
+ e->OnLabel("project", "solomon");
+ e->OnLabelsEnd();
+ }
+ { // metric #1
+ e->OnMetricBegin(EMetricType::COUNTER);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "single");
+ e->OnLabel("labels", "l1");
+ e->OnLabelsEnd();
+ }
+ e->OnUint64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:10Z"), 17);
+ e->OnMetricEnd();
+ }
+ { // metric #2
+ e->OnMetricBegin(EMetricType::COUNTER);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "two");
+ e->OnLabel("labels", "l2");
+ e->OnLabelsEnd();
+ }
+ e->OnUint64(TInstant::Zero(), 42);
+ e->OnMetricEnd();
+ }
+ e->OnStreamEnd();
+ });
+ UNIT_ASSERT_STRINGS_EQUAL(result,
+R"(# TYPE single counter
+single{labels="l1", project="solomon", } 17 1512216010000
+# TYPE two counter
+two{labels="l2", project="solomon", } 42 1500000000000
+
+)");
+ }
+}
diff --git a/library/cpp/monlib/encode/prometheus/prometheus_model.h b/library/cpp/monlib/encode/prometheus/prometheus_model.h
new file mode 100644
index 0000000000..cb7f2cb15b
--- /dev/null
+++ b/library/cpp/monlib/encode/prometheus/prometheus_model.h
@@ -0,0 +1,70 @@
+#pragma once
+
+#include <util/generic/strbuf.h>
+
+
+namespace NMonitoring {
+namespace NPrometheus {
+
+ //
+ // Prometheus specific names and validation rules.
+ //
+ // See https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md
+ // and https://github.com/prometheus/common/blob/master/expfmt/text_parse.go
+ //
+
+ inline constexpr TStringBuf BUCKET_SUFFIX = "_bucket";
+ inline constexpr TStringBuf COUNT_SUFFIX = "_count";
+ inline constexpr TStringBuf SUM_SUFFIX = "_sum";
+ inline constexpr TStringBuf MIN_SUFFIX = "_min";
+ inline constexpr TStringBuf MAX_SUFFIX = "_max";
+ inline constexpr TStringBuf LAST_SUFFIX = "_last";
+
+ // Used for the label that defines the upper bound of a bucket of a
+ // histogram ("le" -> "less or equal").
+ inline constexpr TStringBuf BUCKET_LABEL = "le";
+
+
+ inline bool IsValidLabelNameStart(char ch) {
+ return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || ch == '_';
+ }
+
+ inline bool IsValidLabelNameContinuation(char ch) {
+ return IsValidLabelNameStart(ch) || (ch >= '0' && ch <= '9');
+ }
+
+ inline bool IsValidMetricNameStart(char ch) {
+ return IsValidLabelNameStart(ch) || ch == ':';
+ }
+
+ inline bool IsValidMetricNameContinuation(char ch) {
+ return IsValidLabelNameContinuation(ch) || ch == ':';
+ }
+
+ inline bool IsSum(TStringBuf name) {
+ return name.EndsWith(SUM_SUFFIX);
+ }
+
+ inline bool IsCount(TStringBuf name) {
+ return name.EndsWith(COUNT_SUFFIX);
+ }
+
+ inline bool IsBucket(TStringBuf name) {
+ return name.EndsWith(BUCKET_SUFFIX);
+ }
+
+ inline TStringBuf ToBaseName(TStringBuf name) {
+ if (IsBucket(name)) {
+ return name.SubString(0, name.length() - BUCKET_SUFFIX.length());
+ }
+ if (IsCount(name)) {
+ return name.SubString(0, name.length() - COUNT_SUFFIX.length());
+ }
+ if (IsSum(name)) {
+ return name.SubString(0, name.length() - SUM_SUFFIX.length());
+ }
+ return name;
+ }
+
+} // namespace NPrometheus
+} // namespace NMonitoring
diff --git a/library/cpp/monlib/encode/prometheus/ut/ya.make b/library/cpp/monlib/encode/prometheus/ut/ya.make
new file mode 100644
index 0000000000..fc468ffb68
--- /dev/null
+++ b/library/cpp/monlib/encode/prometheus/ut/ya.make
@@ -0,0 +1,17 @@
+UNITTEST_FOR(library/cpp/monlib/encode/prometheus)
+
+OWNER(
+ g:solomon
+ jamel
+)
+
+SRCS(
+ prometheus_encoder_ut.cpp
+ prometheus_decoder_ut.cpp
+)
+
+PEERDIR(
+ library/cpp/monlib/encode/protobuf
+)
+
+END()
diff --git a/library/cpp/monlib/encode/prometheus/ya.make b/library/cpp/monlib/encode/prometheus/ya.make
new file mode 100644
index 0000000000..7f2483b166
--- /dev/null
+++ b/library/cpp/monlib/encode/prometheus/ya.make
@@ -0,0 +1,17 @@
+LIBRARY()
+
+OWNER(
+ jamel
+ g:solomon
+)
+
+SRCS(
+ prometheus_decoder.cpp
+ prometheus_encoder.cpp
+)
+
+PEERDIR(
+ library/cpp/monlib/encode
+)
+
+END()
diff --git a/library/cpp/monlib/encode/protobuf/protobuf.h b/library/cpp/monlib/encode/protobuf/protobuf.h
new file mode 100644
index 0000000000..3f82cbdd84
--- /dev/null
+++ b/library/cpp/monlib/encode/protobuf/protobuf.h
@@ -0,0 +1,16 @@
+#pragma once
+
+#include <library/cpp/monlib/encode/encoder.h>
+
+#include <library/cpp/monlib/encode/protobuf/protos/samples.pb.h>
+
+namespace NMonitoring {
+ namespace NProto {
+ class TSingleSamplesList;
+ class TMultiSamplesList;
+ }
+
+ IMetricEncoderPtr EncoderProtobuf(NProto::TSingleSamplesList* samples);
+ IMetricEncoderPtr EncoderProtobuf(NProto::TMultiSamplesList* samples);
+
+}
diff --git a/library/cpp/monlib/encode/protobuf/protobuf_encoder.cpp b/library/cpp/monlib/encode/protobuf/protobuf_encoder.cpp
new file mode 100644
index 0000000000..2d11b9d5ba
--- /dev/null
+++ b/library/cpp/monlib/encode/protobuf/protobuf_encoder.cpp
@@ -0,0 +1,248 @@
+#include "protobuf.h"
+
+#include <util/datetime/base.h>
+
+namespace NMonitoring {
+ namespace {
+ NProto::EMetricType ConvertMetricType(EMetricType type) {
+ switch (type) {
+ case EMetricType::GAUGE:
+ return NProto::GAUGE;
+ case EMetricType::COUNTER:
+ return NProto::COUNTER;
+ case EMetricType::RATE:
+ return NProto::RATE;
+ case EMetricType::IGAUGE:
+ return NProto::IGAUGE;
+ case EMetricType::HIST:
+ return NProto::HISTOGRAM;
+ case EMetricType::HIST_RATE:
+ return NProto::HIST_RATE;
+ case EMetricType::DSUMMARY:
+ return NProto::DSUMMARY;
+ case EMetricType::LOGHIST:
+ return NProto::LOGHISTOGRAM;
+ case EMetricType::UNKNOWN:
+ return NProto::UNKNOWN;
+ }
+ }
+
+ void FillHistogram(
+ const IHistogramSnapshot& snapshot,
+ NProto::THistogram* histogram)
+ {
+ for (ui32 i = 0; i < snapshot.Count(); i++) {
+ histogram->AddBounds(snapshot.UpperBound(i));
+ histogram->AddValues(snapshot.Value(i));
+ }
+ }
+
+ void FillSummaryDouble(const ISummaryDoubleSnapshot& snapshot, NProto::TSummaryDouble* summary) {
+ summary->SetSum(snapshot.GetSum());
+ summary->SetMin(snapshot.GetMin());
+ summary->SetMax(snapshot.GetMax());
+ summary->SetLast(snapshot.GetLast());
+ summary->SetCount(snapshot.GetCount());
+ }
+
+ void FillLogHistogram(const TLogHistogramSnapshot& snapshot, NProto::TLogHistogram* logHist) {
+ logHist->SetBase(snapshot.Base());
+ logHist->SetZerosCount(snapshot.ZerosCount());
+ logHist->SetStartPower(snapshot.StartPower());
+ for (ui32 i = 0; i < snapshot.Count(); ++i) {
+ logHist->AddBuckets(snapshot.Bucket(i));
+ }
+ }
+
+ ///////////////////////////////////////////////////////////////////////////////
+ // TSingleamplesEncoder
+ ///////////////////////////////////////////////////////////////////////////////
+ class TSingleSamplesEncoder final: public IMetricEncoder {
+ public:
+ TSingleSamplesEncoder(NProto::TSingleSamplesList* samples)
+ : Samples_(samples)
+ , Sample_(nullptr)
+ {
+ }
+
+ private:
+ void OnStreamBegin() override {
+ }
+ void OnStreamEnd() override {
+ }
+
+ void OnCommonTime(TInstant time) override {
+ Samples_->SetCommonTime(time.MilliSeconds());
+ }
+
+ void OnMetricBegin(EMetricType type) override {
+ Sample_ = Samples_->AddSamples();
+ Sample_->SetMetricType(ConvertMetricType(type));
+ }
+
+ void OnMetricEnd() override {
+ Sample_ = nullptr;
+ }
+
+ void OnLabelsBegin() override {
+ }
+ void OnLabelsEnd() override {
+ }
+
+ void OnLabel(TStringBuf name, TStringBuf value) override {
+ NProto::TLabel* label = (Sample_ == nullptr)
+ ? Samples_->AddCommonLabels()
+ : Sample_->AddLabels();
+ label->SetName(TString{name});
+ label->SetValue(TString{value});
+ }
+
+ void OnDouble(TInstant time, double value) override {
+ Y_ENSURE(Sample_, "metric not started");
+ Sample_->SetTime(time.MilliSeconds());
+ Sample_->SetFloat64(value);
+ }
+
+ void OnInt64(TInstant time, i64 value) override {
+ Y_ENSURE(Sample_, "metric not started");
+ Sample_->SetTime(time.MilliSeconds());
+ Sample_->SetInt64(value);
+ }
+
+ void OnUint64(TInstant time, ui64 value) override {
+ Y_ENSURE(Sample_, "metric not started");
+ Sample_->SetTime(time.MilliSeconds());
+ Sample_->SetUint64(value);
+ }
+
+ void OnHistogram(TInstant time, IHistogramSnapshotPtr snapshot) override {
+ Y_ENSURE(Sample_, "metric not started");
+ Sample_->SetTime(time.MilliSeconds());
+ FillHistogram(*snapshot, Sample_->MutableHistogram());
+ }
+
+ void OnSummaryDouble(TInstant time, ISummaryDoubleSnapshotPtr snapshot) override {
+ Y_ENSURE(Sample_, "metric not started");
+ Sample_->SetTime(time.MilliSeconds());
+ FillSummaryDouble(*snapshot, Sample_->MutableSummaryDouble());
+ }
+
+ void OnLogHistogram(TInstant time, TLogHistogramSnapshotPtr snapshot) override {
+ Y_ENSURE(Sample_, "metric not started");
+ Sample_->SetTime(time.MilliSeconds());
+ FillLogHistogram(*snapshot, Sample_->MutableLogHistogram());
+ }
+
+ void Close() override {
+ }
+
+ private:
+ NProto::TSingleSamplesList* Samples_;
+ NProto::TSingleSample* Sample_;
+ };
+
+ ///////////////////////////////////////////////////////////////////////////////
+ // TMultiSamplesEncoder
+ ///////////////////////////////////////////////////////////////////////////////
+ class TMultiSamplesEncoder final: public IMetricEncoder {
+ public:
+ TMultiSamplesEncoder(NProto::TMultiSamplesList* samples)
+ : Samples_(samples)
+ , Sample_(nullptr)
+ {
+ }
+
+ private:
+ void OnStreamBegin() override {
+ }
+ void OnStreamEnd() override {
+ }
+
+ void OnCommonTime(TInstant time) override {
+ Samples_->SetCommonTime(time.MilliSeconds());
+ }
+
+ void OnMetricBegin(EMetricType type) override {
+ Sample_ = Samples_->AddSamples();
+ Sample_->SetMetricType(ConvertMetricType(type));
+ }
+
+ void OnMetricEnd() override {
+ Sample_ = nullptr;
+ }
+
+ void OnLabelsBegin() override {
+ }
+ void OnLabelsEnd() override {
+ }
+
+ void OnLabel(TStringBuf name, TStringBuf value) override {
+ NProto::TLabel* label = (Sample_ == nullptr)
+ ? Samples_->AddCommonLabels()
+ : Sample_->AddLabels();
+
+ label->SetName(TString{name});
+ label->SetValue(TString{value});
+ }
+
+ void OnDouble(TInstant time, double value) override {
+ Y_ENSURE(Sample_, "metric not started");
+ NProto::TPoint* point = Sample_->AddPoints();
+ point->SetTime(time.MilliSeconds());
+ point->SetFloat64(value);
+ }
+
+ void OnInt64(TInstant time, i64 value) override {
+ Y_ENSURE(Sample_, "metric not started");
+ NProto::TPoint* point = Sample_->AddPoints();
+ point->SetTime(time.MilliSeconds());
+ point->SetInt64(value);
+ }
+
+ void OnUint64(TInstant time, ui64 value) override {
+ Y_ENSURE(Sample_, "metric not started");
+ NProto::TPoint* point = Sample_->AddPoints();
+ point->SetTime(time.MilliSeconds());
+ point->SetUint64(value);
+ }
+
+ void OnHistogram(TInstant time, IHistogramSnapshotPtr snapshot) override {
+ Y_ENSURE(Sample_, "metric not started");
+ NProto::TPoint* point = Sample_->AddPoints();
+ point->SetTime(time.MilliSeconds());
+ FillHistogram(*snapshot, point->MutableHistogram());
+ }
+
+ void OnSummaryDouble(TInstant time, ISummaryDoubleSnapshotPtr snapshot) override {
+ Y_ENSURE(Sample_, "metric not started");
+ NProto::TPoint* point = Sample_->AddPoints();
+ point->SetTime(time.MilliSeconds());
+ FillSummaryDouble(*snapshot, point->MutableSummaryDouble());
+ }
+
+ void OnLogHistogram(TInstant time, TLogHistogramSnapshotPtr snapshot) override {
+ Y_ENSURE(Sample_, "metric not started");
+ NProto::TPoint* point = Sample_->AddPoints();
+ point->SetTime(time.MilliSeconds());
+ FillLogHistogram(*snapshot, point->MutableLogHistogram());
+ }
+
+ void Close() override {
+ }
+
+ private:
+ NProto::TMultiSamplesList* Samples_;
+ NProto::TMultiSample* Sample_;
+ };
+
+ }
+
+ IMetricEncoderPtr EncoderProtobuf(NProto::TSingleSamplesList* samples) {
+ return MakeHolder<TSingleSamplesEncoder>(samples);
+ }
+
+ IMetricEncoderPtr EncoderProtobuf(NProto::TMultiSamplesList* samples) {
+ return MakeHolder<TMultiSamplesEncoder>(samples);
+ }
+
+}
diff --git a/library/cpp/monlib/encode/protobuf/protos/samples.proto b/library/cpp/monlib/encode/protobuf/protos/samples.proto
new file mode 100644
index 0000000000..371f4181d2
--- /dev/null
+++ b/library/cpp/monlib/encode/protobuf/protos/samples.proto
@@ -0,0 +1,91 @@
+syntax = 'proto3';
+
+package NMonitoring.NProto;
+
+option java_package = "ru.yandex.solomon.protos";
+option java_multiple_files = true;
+option cc_enable_arenas = true;
+
+message TLabel {
+ string Name = 1;
+ string Value = 2;
+}
+
+enum EMetricType {
+ UNKNOWN = 0;
+ GAUGE = 1;
+ IGAUGE = 2;
+ COUNTER = 3;
+ RATE = 4;
+ HISTOGRAM = 5;
+ HIST_RATE = 6;
+ DSUMMARY = 7;
+ LOGHISTOGRAM = 8;
+}
+
+message THistogram {
+ repeated double Bounds = 1; // upper bounds of each bucket
+ repeated uint64 Values = 2; // values stored in each bucket
+}
+
+message TLogHistogram {
+ double Base = 1;
+ uint64 ZerosCount = 2;
+ int32 StartPower = 3;
+ repeated double Buckets = 4;
+}
+
+message TSummaryDouble {
+ double Sum = 1;
+ double Min = 2;
+ double Max = 3;
+ double Last = 4;
+ uint64 Count = 5;
+}
+
+// see TSingleSample
+message TPoint {
+ uint64 Time = 1;
+ oneof Value {
+ sfixed64 Int64 = 2;
+ fixed64 Uint64 = 3;
+ double Float64 = 4;
+ THistogram Histogram = 5;
+ TSummaryDouble SummaryDouble = 6;
+ TLogHistogram LogHistogram = 7;
+ }
+}
+
+message TSingleSample {
+ repeated TLabel Labels = 1;
+ EMetricType MetricType = 2;
+
+ // inlined TPoint
+ uint64 Time = 3;
+ oneof Value {
+ sfixed64 Int64 = 4;
+ fixed64 Uint64 = 5;
+ double Float64 = 6;
+ THistogram Histogram = 7;
+ TSummaryDouble SummaryDouble = 8;
+ TLogHistogram LogHistogram = 9;
+ }
+}
+
+message TMultiSample {
+ repeated TLabel Labels = 1;
+ EMetricType MetricType = 2;
+ repeated TPoint Points = 3;
+}
+
+message TSingleSamplesList {
+ uint64 CommonTime = 1;
+ repeated TLabel CommonLabels = 2;
+ repeated TSingleSample Samples = 3;
+}
+
+message TMultiSamplesList {
+ uint64 CommonTime = 1;
+ repeated TLabel CommonLabels = 2;
+ repeated TMultiSample Samples = 3;
+}
diff --git a/library/cpp/monlib/encode/protobuf/protos/ya.make b/library/cpp/monlib/encode/protobuf/protos/ya.make
new file mode 100644
index 0000000000..88ff3ddf88
--- /dev/null
+++ b/library/cpp/monlib/encode/protobuf/protos/ya.make
@@ -0,0 +1,14 @@
+PROTO_LIBRARY()
+
+OWNER(
+ jamel
+ g:solomon
+)
+
+SRCS(
+ samples.proto
+)
+
+EXCLUDE_TAGS(GO_PROTO)
+
+END()
diff --git a/library/cpp/monlib/encode/protobuf/ya.make b/library/cpp/monlib/encode/protobuf/ya.make
new file mode 100644
index 0000000000..9354958b6f
--- /dev/null
+++ b/library/cpp/monlib/encode/protobuf/ya.make
@@ -0,0 +1,17 @@
+LIBRARY()
+
+OWNER(
+ jamel
+ g:solomon
+)
+
+SRCS(
+ protobuf_encoder.cpp
+)
+
+PEERDIR(
+ library/cpp/monlib/encode
+ library/cpp/monlib/encode/protobuf/protos
+)
+
+END()
diff --git a/library/cpp/monlib/encode/spack/compression.cpp b/library/cpp/monlib/encode/spack/compression.cpp
new file mode 100644
index 0000000000..0d2152fc85
--- /dev/null
+++ b/library/cpp/monlib/encode/spack/compression.cpp
@@ -0,0 +1,383 @@
+#include "compression.h"
+
+#include <util/generic/buffer.h>
+#include <util/generic/cast.h>
+#include <util/generic/ptr.h>
+#include <util/generic/scope.h>
+#include <util/generic/size_literals.h>
+#include <util/stream/format.h>
+#include <util/stream/output.h>
+#include <util/stream/walk.h>
+
+#include <contrib/libs/lz4/lz4.h>
+#include <contrib/libs/xxhash/xxhash.h>
+#include <contrib/libs/zlib/zlib.h>
+#define ZSTD_STATIC_LINKING_ONLY
+#include <contrib/libs/zstd/include/zstd.h>
+
+namespace NMonitoring {
+ namespace {
+ ///////////////////////////////////////////////////////////////////////////////
+ // Frame
+ ///////////////////////////////////////////////////////////////////////////////
+ using TCompressedSize = ui32;
+ using TUncompressedSize = ui32;
+ using TCheckSum = ui32;
+
+ constexpr size_t COMPRESSED_FRAME_SIZE_LIMIT = 512_KB;
+ constexpr size_t UNCOMPRESSED_FRAME_SIZE_LIMIT = COMPRESSED_FRAME_SIZE_LIMIT;
+ constexpr size_t FRAME_SIZE_LIMIT = 2_MB;
+ constexpr size_t DEFAULT_FRAME_LEN = 64_KB;
+
+ struct Y_PACKED TFrameHeader {
+ TCompressedSize CompressedSize;
+ TUncompressedSize UncompressedSize;
+ };
+
+ struct Y_PACKED TFrameFooter {
+ TCheckSum CheckSum;
+ };
+
+ ///////////////////////////////////////////////////////////////////////////////
+ // TBlock
+ ///////////////////////////////////////////////////////////////////////////////
+ struct TBlock: public TStringBuf {
+ template <typename T>
+ TBlock(T&& t)
+ : TStringBuf(t.data(), t.size())
+ {
+ Y_ENSURE(t.data() != nullptr);
+ }
+
+ char* data() noexcept {
+ return const_cast<char*>(TStringBuf::data());
+ }
+ };
+
+ ///////////////////////////////////////////////////////////////////////////////
+ // XXHASH
+ ///////////////////////////////////////////////////////////////////////////////
+ struct TXxHash32 {
+ static TCheckSum Calc(TBlock in) {
+ static const ui32 SEED = 0x1337c0de;
+ return XXH32(in.data(), in.size(), SEED);
+ }
+
+ static bool Check(TBlock in, TCheckSum checksum) {
+ return Calc(in) == checksum;
+ }
+ };
+
+ ///////////////////////////////////////////////////////////////////////////////
+ // Adler32
+ ///////////////////////////////////////////////////////////////////////////////
+ struct TAdler32 {
+ static TCheckSum Calc(TBlock in) {
+ return adler32(1L, reinterpret_cast<const Bytef*>(in.data()), in.size());
+ }
+
+ static bool Check(TBlock in, TCheckSum checksum) {
+ return Calc(in) == checksum;
+ }
+ };
+
+ ///////////////////////////////////////////////////////////////////////////////
+ // LZ4
+ ///////////////////////////////////////////////////////////////////////////////
+ struct TLz4Codec {
+ static size_t MaxCompressedLength(size_t in) {
+ int result = LZ4_compressBound(static_cast<int>(in));
+ Y_ENSURE(result != 0, "lz4 input size is too large");
+ return result;
+ }
+
+ static size_t Compress(TBlock in, TBlock out) {
+ int rc = LZ4_compress_default(
+ in.data(),
+ out.data(),
+ SafeIntegerCast<int>(in.size()),
+ SafeIntegerCast<int>(out.size()));
+ Y_ENSURE(rc != 0, "lz4 compression failed");
+ return rc;
+ }
+
+ static void Decompress(TBlock in, TBlock out) {
+ int rc = LZ4_decompress_safe(
+ in.data(),
+ out.data(),
+ SafeIntegerCast<int>(in.size()),
+ SafeIntegerCast<int>(out.size()));
+ Y_ENSURE(rc >= 0, "the lz4 stream is detected malformed");
+ }
+ };
+
+ ///////////////////////////////////////////////////////////////////////////////
+ // ZSTD
+ ///////////////////////////////////////////////////////////////////////////////
+ struct TZstdCodec {
+ static const int LEVEL = 11;
+
+ static size_t MaxCompressedLength(size_t in) {
+ return ZSTD_compressBound(in);
+ }
+
+ static size_t Compress(TBlock in, TBlock out) {
+ size_t rc = ZSTD_compress(out.data(), out.size(), in.data(), in.size(), LEVEL);
+ if (Y_UNLIKELY(ZSTD_isError(rc))) {
+ ythrow yexception() << TStringBuf("zstd compression failed: ")
+ << ZSTD_getErrorName(rc);
+ }
+ return rc;
+ }
+
+ static void Decompress(TBlock in, TBlock out) {
+ size_t rc = ZSTD_decompress(out.data(), out.size(), in.data(), in.size());
+ if (Y_UNLIKELY(ZSTD_isError(rc))) {
+ ythrow yexception() << TStringBuf("zstd decompression failed: ")
+ << ZSTD_getErrorName(rc);
+ }
+ Y_ENSURE(rc == out.size(), "zstd decompressed wrong size");
+ }
+ };
+
+ ///////////////////////////////////////////////////////////////////////////////
+ // ZLIB
+ ///////////////////////////////////////////////////////////////////////////////
+ struct TZlibCodec {
+ static const int LEVEL = 6;
+
+ static size_t MaxCompressedLength(size_t in) {
+ return compressBound(in);
+ }
+
+ static size_t Compress(TBlock in, TBlock out) {
+ uLong ret = out.size();
+ int rc = compress2(
+ reinterpret_cast<Bytef*>(out.data()),
+ &ret,
+ reinterpret_cast<const Bytef*>(in.data()),
+ in.size(),
+ LEVEL);
+ Y_ENSURE(rc == Z_OK, "zlib compression failed");
+ return ret;
+ }
+
+ static void Decompress(TBlock in, TBlock out) {
+ uLong ret = out.size();
+ int rc = uncompress(
+ reinterpret_cast<Bytef*>(out.data()),
+ &ret,
+ reinterpret_cast<const Bytef*>(in.data()),
+ in.size());
+ Y_ENSURE(rc == Z_OK, "zlib decompression failed");
+ Y_ENSURE(ret == out.size(), "zlib decompressed wrong size");
+ }
+ };
+
+ //
+ // Framed streams use next frame structure:
+ //
+ // +-----------------+-------------------+============+------------------+
+ // | compressed size | uncompressed size | data | check sum |
+ // +-----------------+-------------------+============+------------------+
+ // 4 bytes 4 bytes var len 4 bytes
+ //
+
+ ///////////////////////////////////////////////////////////////////////////////
+ // TFramedInputStream
+ ///////////////////////////////////////////////////////////////////////////////
+ template <typename TCodecAlg, typename TCheckSumAlg>
+ class TFramedDecompressStream final: public IWalkInput {
+ public:
+ explicit TFramedDecompressStream(IInputStream* in)
+ : In_(in)
+ {
+ }
+
+ private:
+ size_t DoUnboundedNext(const void** ptr) override {
+ if (!In_) {
+ return 0;
+ }
+
+ TFrameHeader header;
+ In_->LoadOrFail(&header, sizeof(header));
+
+ if (header.CompressedSize == 0) {
+ In_ = nullptr;
+ return 0;
+ }
+
+ Y_ENSURE(header.CompressedSize <= COMPRESSED_FRAME_SIZE_LIMIT, "Compressed frame size is limited to "
+ << HumanReadableSize(COMPRESSED_FRAME_SIZE_LIMIT, SF_BYTES)
+ << " but is " << HumanReadableSize(header.CompressedSize, SF_BYTES));
+
+ Y_ENSURE(header.UncompressedSize <= UNCOMPRESSED_FRAME_SIZE_LIMIT, "Uncompressed frame size is limited to "
+ << HumanReadableSize(UNCOMPRESSED_FRAME_SIZE_LIMIT, SF_BYTES)
+ << " but is " << HumanReadableSize(header.UncompressedSize, SF_BYTES));
+
+ Compressed_.Resize(header.CompressedSize);
+ In_->LoadOrFail(Compressed_.Data(), header.CompressedSize);
+
+ TFrameFooter footer;
+ In_->LoadOrFail(&footer, sizeof(footer));
+ Y_ENSURE(TCheckSumAlg::Check(Compressed_, footer.CheckSum),
+ "corrupted stream: check sum mismatch");
+
+ Uncompressed_.Resize(header.UncompressedSize);
+ TCodecAlg::Decompress(Compressed_, Uncompressed_);
+
+ *ptr = Uncompressed_.Data();
+ return Uncompressed_.Size();
+ }
+
+ private:
+ IInputStream* In_;
+ TBuffer Compressed_;
+ TBuffer Uncompressed_;
+ };
+
+ ///////////////////////////////////////////////////////////////////////////////
+ // TFramedOutputStream
+ ///////////////////////////////////////////////////////////////////////////////
+ template <typename TCodecAlg, typename TCheckSumAlg>
+ class TFramedCompressStream final: public IFramedCompressStream {
+ public:
+ explicit TFramedCompressStream(IOutputStream* out)
+ : Out_(out)
+ , Uncompressed_(DEFAULT_FRAME_LEN)
+ {
+ }
+
+ ~TFramedCompressStream() override {
+ try {
+ Finish();
+ } catch (...) {
+ }
+ }
+
+ private:
+ void DoWrite(const void* buf, size_t len) override {
+ const char* in = static_cast<const char*>(buf);
+
+ while (len != 0) {
+ const size_t avail = Uncompressed_.Avail();
+ if (len < avail) {
+ Uncompressed_.Append(in, len);
+ return;
+ }
+
+ Uncompressed_.Append(in, avail);
+ Y_ASSERT(Uncompressed_.Avail() == 0);
+
+ in += avail;
+ len -= avail;
+
+ WriteCompressedFrame();
+ }
+ }
+
+ void FlushWithoutEmptyFrame() override {
+ if (Out_ && !Uncompressed_.Empty()) {
+ WriteCompressedFrame();
+ }
+ }
+
+ void FinishAndWriteEmptyFrame() override {
+ if (Out_) {
+ Y_DEFER {
+ Out_ = nullptr;
+ };
+
+ if (!Uncompressed_.Empty()) {
+ WriteCompressedFrame();
+ }
+
+ WriteEmptyFrame();
+ }
+ }
+
+ void DoFlush() override {
+ FlushWithoutEmptyFrame();
+ }
+
+ void DoFinish() override {
+ FinishAndWriteEmptyFrame();
+ }
+
+ void WriteCompressedFrame() {
+ static const auto framePayload = sizeof(TFrameHeader) + sizeof(TFrameFooter);
+ const auto maxFrameSize = ui64(TCodecAlg::MaxCompressedLength(Uncompressed_.Size())) + framePayload;
+ Y_ENSURE(maxFrameSize <= FRAME_SIZE_LIMIT, "Frame size in encoder is limited to "
+ << HumanReadableSize(FRAME_SIZE_LIMIT, SF_BYTES)
+ << " but is " << HumanReadableSize(maxFrameSize, SF_BYTES));
+
+ Frame_.Resize(maxFrameSize);
+
+ // compress
+ TBlock compressedBlock = Frame_;
+ compressedBlock.Skip(sizeof(TFrameHeader));
+ compressedBlock.Trunc(TCodecAlg::Compress(Uncompressed_, compressedBlock));
+
+ // add header
+ auto header = reinterpret_cast<TFrameHeader*>(Frame_.Data());
+ header->CompressedSize = SafeIntegerCast<TCompressedSize>(compressedBlock.size());
+ header->UncompressedSize = SafeIntegerCast<TUncompressedSize>(Uncompressed_.Size());
+
+ // add footer
+ auto footer = reinterpret_cast<TFrameFooter*>(
+ Frame_.Data() + sizeof(TFrameHeader) + header->CompressedSize);
+ footer->CheckSum = TCheckSumAlg::Calc(compressedBlock);
+
+ // write
+ Out_->Write(Frame_.Data(), header->CompressedSize + framePayload);
+ Uncompressed_.Clear();
+ }
+
+ void WriteEmptyFrame() {
+ static const auto framePayload = sizeof(TFrameHeader) + sizeof(TFrameFooter);
+ char buf[framePayload] = {0};
+ Out_->Write(buf, sizeof(buf));
+ }
+
+ private:
+ IOutputStream* Out_;
+ TBuffer Uncompressed_;
+ TBuffer Frame_;
+ };
+
+ }
+
+ THolder<IInputStream> CompressedInput(IInputStream* in, ECompression alg) {
+ switch (alg) {
+ case ECompression::IDENTITY:
+ return nullptr;
+ case ECompression::ZLIB:
+ return MakeHolder<TFramedDecompressStream<TZlibCodec, TAdler32>>(in);
+ case ECompression::ZSTD:
+ return MakeHolder<TFramedDecompressStream<TZstdCodec, TXxHash32>>(in);
+ case ECompression::LZ4:
+ return MakeHolder<TFramedDecompressStream<TLz4Codec, TXxHash32>>(in);
+ case ECompression::UNKNOWN:
+ return nullptr;
+ }
+ Y_FAIL("invalid compression algorithm");
+ }
+
+ THolder<IFramedCompressStream> CompressedOutput(IOutputStream* out, ECompression alg) {
+ switch (alg) {
+ case ECompression::IDENTITY:
+ return nullptr;
+ case ECompression::ZLIB:
+ return MakeHolder<TFramedCompressStream<TZlibCodec, TAdler32>>(out);
+ case ECompression::ZSTD:
+ return MakeHolder<TFramedCompressStream<TZstdCodec, TXxHash32>>(out);
+ case ECompression::LZ4:
+ return MakeHolder<TFramedCompressStream<TLz4Codec, TXxHash32>>(out);
+ case ECompression::UNKNOWN:
+ return nullptr;
+ }
+ Y_FAIL("invalid compression algorithm");
+ }
+
+}
diff --git a/library/cpp/monlib/encode/spack/compression.h b/library/cpp/monlib/encode/spack/compression.h
new file mode 100644
index 0000000000..f74d8b424e
--- /dev/null
+++ b/library/cpp/monlib/encode/spack/compression.h
@@ -0,0 +1,19 @@
+#pragma once
+
+#include "spack_v1.h"
+
+#include <util/stream/input.h>
+#include <util/stream/output.h>
+
+namespace NMonitoring {
+
+class IFramedCompressStream: public IOutputStream {
+public:
+ virtual void FlushWithoutEmptyFrame() = 0;
+ virtual void FinishAndWriteEmptyFrame() = 0;
+};
+
+THolder<IInputStream> CompressedInput(IInputStream* in, ECompression alg);
+THolder<IFramedCompressStream> CompressedOutput(IOutputStream* out, ECompression alg);
+
+} // namespace NMonitoring
diff --git a/library/cpp/monlib/encode/spack/fuzz/main.cpp b/library/cpp/monlib/encode/spack/fuzz/main.cpp
new file mode 100644
index 0000000000..6a14afe71c
--- /dev/null
+++ b/library/cpp/monlib/encode/spack/fuzz/main.cpp
@@ -0,0 +1,20 @@
+#include <library/cpp/monlib/encode/spack/spack_v1.h>
+#include <library/cpp/monlib/encode/fake/fake.h>
+
+#include <util/stream/mem.h>
+
+
+extern "C" int LLVMFuzzerTestOneInput(const ui8* data, size_t size) {
+ using namespace NMonitoring;
+
+ TMemoryInput min{data, size};
+
+ auto encoder = EncoderFake();
+
+ try {
+ DecodeSpackV1(&min, encoder.Get());
+ } catch (...) {
+ }
+
+ return 0;
+}
diff --git a/library/cpp/monlib/encode/spack/fuzz/ya.make b/library/cpp/monlib/encode/spack/fuzz/ya.make
new file mode 100644
index 0000000000..99b63eadd5
--- /dev/null
+++ b/library/cpp/monlib/encode/spack/fuzz/ya.make
@@ -0,0 +1,21 @@
+FUZZ()
+
+OWNER(
+ g:solomon
+ msherbakov
+)
+
+FUZZ_OPTS(-rss_limit_mb=1024)
+
+SIZE(MEDIUM)
+
+PEERDIR(
+ library/cpp/monlib/encode/spack
+ library/cpp/monlib/encode/fake
+)
+
+SRCS(
+ main.cpp
+)
+
+END()
diff --git a/library/cpp/monlib/encode/spack/spack_v1.h b/library/cpp/monlib/encode/spack/spack_v1.h
new file mode 100644
index 0000000000..cf1c9417b9
--- /dev/null
+++ b/library/cpp/monlib/encode/spack/spack_v1.h
@@ -0,0 +1,115 @@
+#pragma once
+
+#include <library/cpp/monlib/encode/encoder.h>
+#include <library/cpp/monlib/encode/format.h>
+#include <library/cpp/monlib/metrics/metric.h>
+
+#include <util/generic/yexception.h>
+
+//
+// format specification available here:
+// https://wiki.yandex-team.ru/solomon/api/dataformat/spackv1/
+//
+
+class IInputStream;
+class IOutputStream;
+
+namespace NMonitoring {
+ class TSpackDecodeError: public yexception {
+ };
+
+ constexpr auto EncodeMetricType(EMetricType mt) noexcept {
+ return static_cast<std::underlying_type_t<EMetricType>>(mt);
+ }
+
+ EMetricType DecodeMetricType(ui8 byte);
+
+ [[nodiscard]]
+ bool TryDecodeMetricType(ui8 byte, EMetricType* result);
+
+ ///////////////////////////////////////////////////////////////////////////////
+ // EValueType
+ ///////////////////////////////////////////////////////////////////////////////
+ enum class EValueType : ui8 {
+ NONE = 0x00,
+ ONE_WITHOUT_TS = 0x01,
+ ONE_WITH_TS = 0x02,
+ MANY_WITH_TS = 0x03,
+ };
+
+ constexpr auto EncodeValueType(EValueType vt) noexcept {
+ return static_cast<std::underlying_type_t<EValueType>>(vt);
+ }
+
+ EValueType DecodeValueType(ui8 byte);
+
+ [[nodiscard]]
+ bool TryDecodeValueType(ui8 byte, EValueType* result);
+
+ ///////////////////////////////////////////////////////////////////////////////
+ // ETimePrecision
+ ///////////////////////////////////////////////////////////////////////////////
+ enum class ETimePrecision : ui8 {
+ SECONDS = 0x00,
+ MILLIS = 0x01,
+ };
+
+ constexpr auto EncodeTimePrecision(ETimePrecision tp) noexcept {
+ return static_cast<std::underlying_type_t<ETimePrecision>>(tp);
+ }
+
+ ETimePrecision DecodeTimePrecision(ui8 byte);
+
+ [[nodiscard]]
+ bool TryDecodeTimePrecision(ui8 byte, ETimePrecision* result);
+
+ ///////////////////////////////////////////////////////////////////////////////
+ // ECompression
+ ///////////////////////////////////////////////////////////////////////////////
+ ui8 EncodeCompression(ECompression c) noexcept;
+
+ ECompression DecodeCompression(ui8 byte);
+
+ [[nodiscard]]
+ bool TryDecodeCompression(ui8 byte, ECompression* result);
+
+ ///////////////////////////////////////////////////////////////////////////////
+ // TSpackHeader
+ ///////////////////////////////////////////////////////////////////////////////
+ struct Y_PACKED TSpackHeader {
+ ui16 Magic = 0x5053; // "SP"
+ ui16 Version; // MSB - major version, LSB - minor version
+ ui16 HeaderSize = sizeof(TSpackHeader);
+ ui8 TimePrecision;
+ ui8 Compression;
+ ui32 LabelNamesSize;
+ ui32 LabelValuesSize;
+ ui32 MetricCount;
+ ui32 PointsCount;
+ // add new fields here
+ };
+
+ enum ESpackV1Version: ui16 {
+ SV1_00 = 0x0100,
+ SV1_01 = 0x0101,
+ SV1_02 = 0x0102
+ };
+
+ IMetricEncoderPtr EncoderSpackV1(
+ IOutputStream* out,
+ ETimePrecision timePrecision,
+ ECompression compression,
+ EMetricsMergingMode mergingMode = EMetricsMergingMode::DEFAULT
+ );
+
+ IMetricEncoderPtr EncoderSpackV12(
+ IOutputStream* out,
+ ETimePrecision timePrecision,
+ ECompression compression,
+ EMetricsMergingMode mergingMode = EMetricsMergingMode::DEFAULT,
+ TStringBuf metricNameLabel = "name"
+ );
+
+ void DecodeSpackV1(IInputStream* in, IMetricConsumer* c, TStringBuf metricNameLabel = "name");
+
+}
diff --git a/library/cpp/monlib/encode/spack/spack_v1_decoder.cpp b/library/cpp/monlib/encode/spack/spack_v1_decoder.cpp
new file mode 100644
index 0000000000..1f445fc80d
--- /dev/null
+++ b/library/cpp/monlib/encode/spack/spack_v1_decoder.cpp
@@ -0,0 +1,458 @@
+#include "spack_v1.h"
+#include "varint.h"
+#include "compression.h"
+
+#include <library/cpp/monlib/encode/buffered/string_pool.h>
+#include <library/cpp/monlib/exception/exception.h>
+#include <library/cpp/monlib/metrics/histogram_collector.h>
+#include <library/cpp/monlib/metrics/metric.h>
+
+#include <util/generic/yexception.h>
+#include <util/generic/buffer.h>
+#include <util/generic/size_literals.h>
+#include <util/stream/format.h>
+
+#ifndef _little_endian_
+#error Unsupported platform
+#endif
+
+namespace NMonitoring {
+ namespace {
+#define DECODE_ENSURE(COND, ...) MONLIB_ENSURE_EX(COND, TSpackDecodeError() << __VA_ARGS__)
+
+ constexpr ui64 LABEL_SIZE_LIMIT = 128_MB;
+
+ ///////////////////////////////////////////////////////////////////////
+ // TDecoderSpackV1
+ ///////////////////////////////////////////////////////////////////////
+ class TDecoderSpackV1 {
+ public:
+ TDecoderSpackV1(IInputStream* in, TStringBuf metricNameLabel)
+ : In_(in)
+ , MetricNameLabel_(metricNameLabel)
+ {
+ }
+
+ void Decode(IMetricConsumer* c) {
+ c->OnStreamBegin();
+
+ // (1) read header
+ size_t readBytes = In_->Read(&Header_, sizeof(Header_));
+ DECODE_ENSURE(readBytes == sizeof(Header_), "not enough data in input stream to read header");
+
+ ui8 version = ((Header_.Version >> 8) & 0xff);
+ DECODE_ENSURE(version == 1, "versions mismatch (expected: 1, got: " << +version << ')');
+
+ DECODE_ENSURE(Header_.HeaderSize >= sizeof(Header_), "invalid header size");
+ if (size_t skipBytes = Header_.HeaderSize - sizeof(Header_)) {
+ DECODE_ENSURE(In_->Skip(skipBytes) == skipBytes, "input stream unexpectedly ended");
+ }
+
+ if (Header_.MetricCount == 0) {
+ // emulate empty stream
+ c->OnStreamEnd();
+ return;
+ }
+
+ // if compression enabled all below reads must go throught decompressor
+ auto compressedIn = CompressedInput(In_, DecodeCompression(Header_.Compression));
+ if (compressedIn) {
+ In_ = compressedIn.Get();
+ }
+
+ TimePrecision_ = DecodeTimePrecision(Header_.TimePrecision);
+
+ const ui64 labelSizeTotal = ui64(Header_.LabelNamesSize) + Header_.LabelValuesSize;
+
+ DECODE_ENSURE(labelSizeTotal <= LABEL_SIZE_LIMIT, "Label names & values size of " << HumanReadableSize(labelSizeTotal, SF_BYTES)
+ << " exceeds the limit which is " << HumanReadableSize(LABEL_SIZE_LIMIT, SF_BYTES));
+
+ // (2) read string pools
+ TVector<char> namesBuf(Header_.LabelNamesSize);
+ readBytes = In_->Load(namesBuf.data(), namesBuf.size());
+ DECODE_ENSURE(readBytes == Header_.LabelNamesSize, "not enough data to read label names pool");
+ TStringPool labelNames(namesBuf.data(), namesBuf.size());
+
+ TVector<char> valuesBuf(Header_.LabelValuesSize);
+ readBytes = In_->Load(valuesBuf.data(), valuesBuf.size());
+ DECODE_ENSURE(readBytes == Header_.LabelValuesSize, "not enough data to read label values pool");
+ TStringPool labelValues(valuesBuf.data(), valuesBuf.size());
+
+ // (3) read common time
+ c->OnCommonTime(ReadTime());
+
+ // (4) read common labels
+ if (ui32 commonLabelsCount = ReadVarint()) {
+ c->OnLabelsBegin();
+ ReadLabels(labelNames, labelValues, commonLabelsCount, c);
+ c->OnLabelsEnd();
+ }
+
+ // (5) read metrics
+ ReadMetrics(labelNames, labelValues, c);
+ c->OnStreamEnd();
+ }
+
+ private:
+ void ReadMetrics(
+ const TStringPool& labelNames,
+ const TStringPool& labelValues,
+ IMetricConsumer* c)
+ {
+ for (ui32 i = 0; i < Header_.MetricCount; i++) {
+ // (5.1) types byte
+ ui8 typesByte = ReadFixed<ui8>();
+ EMetricType metricType = DecodeMetricType(typesByte >> 2);
+ EValueType valueType = DecodeValueType(typesByte & 0x03);
+
+ c->OnMetricBegin(metricType);
+
+ // TODO: use it
+ ReadFixed<ui8>(); // skip flags byte
+
+ auto metricNameValueIndex = std::numeric_limits<ui32>::max();
+ if (Header_.Version >= SV1_02) {
+ metricNameValueIndex = ReadVarint();
+ }
+
+ // (5.2) labels
+ ui32 labelsCount = ReadVarint();
+ DECODE_ENSURE(Header_.Version >= SV1_02 || labelsCount > 0, "metric #" << i << " has no labels");
+ c->OnLabelsBegin();
+ if (Header_.Version >= SV1_02) {
+ c->OnLabel(MetricNameLabel_, labelValues.Get(metricNameValueIndex));
+ }
+ ReadLabels(labelNames, labelValues, labelsCount, c);
+ c->OnLabelsEnd();
+
+ // (5.3) values
+ switch (valueType) {
+ case EValueType::NONE:
+ break;
+ case EValueType::ONE_WITHOUT_TS:
+ ReadValue(metricType, TInstant::Zero(), c);
+ break;
+ case EValueType::ONE_WITH_TS: {
+ TInstant time = ReadTime();
+ ReadValue(metricType, time, c);
+ break;
+ }
+ case EValueType::MANY_WITH_TS: {
+ ui32 pointsCount = ReadVarint();
+ for (ui32 i = 0; i < pointsCount; i++) {
+ TInstant time = ReadTime();
+ ReadValue(metricType, time, c);
+ }
+ break;
+ }
+ }
+
+ c->OnMetricEnd();
+ }
+ }
+
+ void ReadValue(EMetricType metricType, TInstant time, IMetricConsumer* c) {
+ switch (metricType) {
+ case EMetricType::GAUGE:
+ c->OnDouble(time, ReadFixed<double>());
+ break;
+
+ case EMetricType::IGAUGE:
+ c->OnInt64(time, ReadFixed<i64>());
+ break;
+
+ case EMetricType::COUNTER:
+ case EMetricType::RATE:
+ c->OnUint64(time, ReadFixed<ui64>());
+ break;
+
+ case EMetricType::DSUMMARY:
+ c->OnSummaryDouble(time, ReadSummaryDouble());
+ break;
+
+ case EMetricType::HIST:
+ case EMetricType::HIST_RATE:
+ c->OnHistogram(time, ReadHistogram());
+ break;
+
+ case EMetricType::LOGHIST:
+ c->OnLogHistogram(time, ReadLogHistogram());
+ break;
+
+ default:
+ throw TSpackDecodeError() << "Unsupported metric type: " << metricType;
+ }
+ }
+
+ ISummaryDoubleSnapshotPtr ReadSummaryDouble() {
+ ui64 count = ReadFixed<ui64>();
+ double sum = ReadFixed<double>();
+ double min = ReadFixed<double>();
+ double max = ReadFixed<double>();
+ double last = ReadFixed<double>();
+ return MakeIntrusive<TSummaryDoubleSnapshot>(sum, min, max, last, count);
+ }
+
+ TLogHistogramSnapshotPtr ReadLogHistogram() {
+ double base = ReadFixed<double>();
+ ui64 zerosCount = ReadFixed<ui64>();
+ int startPower = static_cast<int>(ReadVarint());
+ ui32 count = ReadVarint();
+ // see https://a.yandex-team.ru/arc/trunk/arcadia/infra/yasm/stockpile_client/points.cpp?rev=r8593154#L31
+ // and https://a.yandex-team.ru/arc/trunk/arcadia/infra/yasm/common/points/hgram/normal/normal.h?rev=r8268697#L9
+ // TODO: share this constant value
+ Y_ENSURE(count <= 100u, "more than 100 buckets in log histogram: " << count);
+ TVector<double> buckets;
+ buckets.reserve(count);
+ for (ui32 i = 0; i < count; ++i) {
+ buckets.emplace_back(ReadFixed<double>());
+ }
+ return MakeIntrusive<TLogHistogramSnapshot>(base, zerosCount, startPower, std::move(buckets));
+ }
+
+ IHistogramSnapshotPtr ReadHistogram() {
+ ui32 bucketsCount = ReadVarint();
+ auto s = TExplicitHistogramSnapshot::New(bucketsCount);
+
+ if (SV1_00 == Header_.Version) { // v1.0
+ for (ui32 i = 0; i < bucketsCount; i++) {
+ i64 bound = ReadFixed<i64>();
+ double doubleBound = (bound != Max<i64>())
+ ? static_cast<double>(bound)
+ : Max<double>();
+
+ (*s)[i].first = doubleBound;
+ }
+ } else {
+ for (ui32 i = 0; i < bucketsCount; i++) {
+ double doubleBound = ReadFixed<double>();
+ (*s)[i].first = doubleBound;
+ }
+ }
+
+
+ // values
+ for (ui32 i = 0; i < bucketsCount; i++) {
+ (*s)[i].second = ReadFixed<ui64>();
+ }
+ return s;
+ }
+
+ void ReadLabels(
+ const TStringPool& labelNames,
+ const TStringPool& labelValues,
+ ui32 count,
+ IMetricConsumer* c)
+ {
+ for (ui32 i = 0; i < count; i++) {
+ auto nameIdx = ReadVarint();
+ auto valueIdx = ReadVarint();
+ c->OnLabel(labelNames.Get(nameIdx), labelValues.Get(valueIdx));
+ }
+ }
+
+ TInstant ReadTime() {
+ switch (TimePrecision_) {
+ case ETimePrecision::SECONDS:
+ return TInstant::Seconds(ReadFixed<ui32>());
+ case ETimePrecision::MILLIS:
+ return TInstant::MilliSeconds(ReadFixed<ui64>());
+ }
+ Y_FAIL("invalid time precision");
+ }
+
+ template <typename T>
+ inline T ReadFixed() {
+ T value;
+ size_t readBytes = In_->Load(&value, sizeof(T));
+ DECODE_ENSURE(readBytes == sizeof(T), "no enough data to read " << TypeName<T>());
+ return value;
+ }
+
+ inline ui32 ReadVarint() {
+ return ReadVarUInt32(In_);
+ }
+
+ private:
+ IInputStream* In_;
+ TString MetricNameLabel_;
+ ETimePrecision TimePrecision_;
+ TSpackHeader Header_;
+ }; // class TDecoderSpackV1
+
+#undef DECODE_ENSURE
+ } // namespace
+
+ EValueType DecodeValueType(ui8 byte) {
+ EValueType result;
+ if (!TryDecodeValueType(byte, &result)) {
+ throw TSpackDecodeError() << "unknown value type: " << byte;
+ }
+ return result;
+ }
+
+ bool TryDecodeValueType(ui8 byte, EValueType* result) {
+ if (byte == EncodeValueType(EValueType::NONE)) {
+ if (result) {
+ *result = EValueType::NONE;
+ }
+ return true;
+ } else if (byte == EncodeValueType(EValueType::ONE_WITHOUT_TS)) {
+ if (result) {
+ *result = EValueType::ONE_WITHOUT_TS;
+ }
+ return true;
+ } else if (byte == EncodeValueType(EValueType::ONE_WITH_TS)) {
+ if (result) {
+ *result = EValueType::ONE_WITH_TS;
+ }
+ return true;
+ } else if (byte == EncodeValueType(EValueType::MANY_WITH_TS)) {
+ if (result) {
+ *result = EValueType::MANY_WITH_TS;
+ }
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ ETimePrecision DecodeTimePrecision(ui8 byte) {
+ ETimePrecision result;
+ if (!TryDecodeTimePrecision(byte, &result)) {
+ throw TSpackDecodeError() << "unknown time precision: " << byte;
+ }
+ return result;
+ }
+
+ bool TryDecodeTimePrecision(ui8 byte, ETimePrecision* result) {
+ if (byte == EncodeTimePrecision(ETimePrecision::SECONDS)) {
+ if (result) {
+ *result = ETimePrecision::SECONDS;
+ }
+ return true;
+ } else if (byte == EncodeTimePrecision(ETimePrecision::MILLIS)) {
+ if (result) {
+ *result = ETimePrecision::MILLIS;
+ }
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ EMetricType DecodeMetricType(ui8 byte) {
+ EMetricType result;
+ if (!TryDecodeMetricType(byte, &result)) {
+ throw TSpackDecodeError() << "unknown metric type: " << byte;
+ }
+ return result;
+ }
+
+ bool TryDecodeMetricType(ui8 byte, EMetricType* result) {
+ if (byte == EncodeMetricType(EMetricType::GAUGE)) {
+ if (result) {
+ *result = EMetricType::GAUGE;
+ }
+ return true;
+ } else if (byte == EncodeMetricType(EMetricType::COUNTER)) {
+ if (result) {
+ *result = EMetricType::COUNTER;
+ }
+ return true;
+ } else if (byte == EncodeMetricType(EMetricType::RATE)) {
+ if (result) {
+ *result = EMetricType::RATE;
+ }
+ return true;
+ } else if (byte == EncodeMetricType(EMetricType::IGAUGE)) {
+ if (result) {
+ *result = EMetricType::IGAUGE;
+ }
+ return true;
+ } else if (byte == EncodeMetricType(EMetricType::HIST)) {
+ if (result) {
+ *result = EMetricType::HIST;
+ }
+ return true;
+ } else if (byte == EncodeMetricType(EMetricType::HIST_RATE)) {
+ if (result) {
+ *result = EMetricType::HIST_RATE;
+ }
+ return true;
+ } else if (byte == EncodeMetricType(EMetricType::DSUMMARY)) {
+ if (result) {
+ *result = EMetricType::DSUMMARY;
+ }
+ return true;
+ } else if (byte == EncodeMetricType(EMetricType::LOGHIST)) {
+ if (result) {
+ *result = EMetricType::LOGHIST;
+ }
+ return true;
+ } else if (byte == EncodeMetricType(EMetricType::UNKNOWN)) {
+ if (result) {
+ *result = EMetricType::UNKNOWN;
+ }
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ ui8 EncodeCompression(ECompression c) noexcept {
+ switch (c) {
+ case ECompression::IDENTITY:
+ return 0x00;
+ case ECompression::ZLIB:
+ return 0x01;
+ case ECompression::ZSTD:
+ return 0x02;
+ case ECompression::LZ4:
+ return 0x03;
+ case ECompression::UNKNOWN:
+ return Max<ui8>();
+ }
+ Y_FAIL(); // for GCC
+ }
+
+ ECompression DecodeCompression(ui8 byte) {
+ ECompression result;
+ if (!TryDecodeCompression(byte, &result)) {
+ throw TSpackDecodeError() << "unknown compression alg: " << byte;
+ }
+ return result;
+ }
+
+ bool TryDecodeCompression(ui8 byte, ECompression* result) {
+ if (byte == EncodeCompression(ECompression::IDENTITY)) {
+ if (result) {
+ *result = ECompression::IDENTITY;
+ }
+ return true;
+ } else if (byte == EncodeCompression(ECompression::ZLIB)) {
+ if (result) {
+ *result = ECompression::ZLIB;
+ }
+ return true;
+ } else if (byte == EncodeCompression(ECompression::ZSTD)) {
+ if (result) {
+ *result = ECompression::ZSTD;
+ }
+ return true;
+ } else if (byte == EncodeCompression(ECompression::LZ4)) {
+ if (result) {
+ *result = ECompression::LZ4;
+ }
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ void DecodeSpackV1(IInputStream* in, IMetricConsumer* c, TStringBuf metricNameLabel) {
+ TDecoderSpackV1 decoder(in, metricNameLabel);
+ decoder.Decode(c);
+ }
+
+}
diff --git a/library/cpp/monlib/encode/spack/spack_v1_encoder.cpp b/library/cpp/monlib/encode/spack/spack_v1_encoder.cpp
new file mode 100644
index 0000000000..a2b0bb5f50
--- /dev/null
+++ b/library/cpp/monlib/encode/spack/spack_v1_encoder.cpp
@@ -0,0 +1,318 @@
+#include "spack_v1.h"
+#include "compression.h"
+#include "varint.h"
+
+#include <library/cpp/monlib/encode/buffered/buffered_encoder_base.h>
+
+#include <util/generic/cast.h>
+#include <util/datetime/base.h>
+#include <util/string/builder.h>
+
+#ifndef _little_endian_
+#error Unsupported platform
+#endif
+
+namespace NMonitoring {
+ namespace {
+ ///////////////////////////////////////////////////////////////////////
+ // TEncoderSpackV1
+ ///////////////////////////////////////////////////////////////////////
+ class TEncoderSpackV1 final: public TBufferedEncoderBase {
+ public:
+ TEncoderSpackV1(
+ IOutputStream* out,
+ ETimePrecision timePrecision,
+ ECompression compression,
+ EMetricsMergingMode mergingMode,
+ ESpackV1Version version,
+ TStringBuf metricNameLabel
+ )
+ : Out_(out)
+ , TimePrecision_(timePrecision)
+ , Compression_(compression)
+ , Version_(version)
+ , MetricName_(Version_ >= SV1_02 ? LabelNamesPool_.PutIfAbsent(metricNameLabel) : nullptr)
+ {
+ MetricsMergingMode_ = mergingMode;
+
+ LabelNamesPool_.SetSorted(true);
+ LabelValuesPool_.SetSorted(true);
+ }
+
+ ~TEncoderSpackV1() override {
+ Close();
+ }
+
+ private:
+ void OnDouble(TInstant time, double value) override {
+ TBufferedEncoderBase::OnDouble(time, value);
+ }
+
+ void OnInt64(TInstant time, i64 value) override {
+ TBufferedEncoderBase::OnInt64(time, value);
+ }
+
+ void OnUint64(TInstant time, ui64 value) override {
+ TBufferedEncoderBase::OnUint64(time, value);
+ }
+
+ void OnHistogram(TInstant time, IHistogramSnapshotPtr snapshot) override {
+ TBufferedEncoderBase::OnHistogram(time, snapshot);
+ }
+
+ void OnSummaryDouble(TInstant time, ISummaryDoubleSnapshotPtr snapshot) override {
+ TBufferedEncoderBase::OnSummaryDouble(time, snapshot);
+ }
+
+ void OnLogHistogram(TInstant time, TLogHistogramSnapshotPtr snapshot) override {
+ TBufferedEncoderBase::OnLogHistogram(time, snapshot);
+ }
+
+ void Close() override {
+ if (Closed_) {
+ return;
+ }
+ Closed_ = true;
+
+ LabelNamesPool_.Build();
+ LabelValuesPool_.Build();
+
+ // Sort all points uniquely by ts -- the size can decrease
+ ui64 pointsCount = 0;
+ for (TMetric& metric : Metrics_) {
+ if (metric.TimeSeries.Size() > 1) {
+ metric.TimeSeries.SortByTs();
+ }
+
+ pointsCount += metric.TimeSeries.Size();
+ }
+
+ // (1) write header
+ TSpackHeader header;
+ header.Version = Version_;
+ header.TimePrecision = EncodeTimePrecision(TimePrecision_);
+ header.Compression = EncodeCompression(Compression_);
+ header.LabelNamesSize = static_cast<ui32>(
+ LabelNamesPool_.BytesSize() + LabelNamesPool_.Count());
+ header.LabelValuesSize = static_cast<ui32>(
+ LabelValuesPool_.BytesSize() + LabelValuesPool_.Count());
+ header.MetricCount = Metrics_.size();
+ header.PointsCount = pointsCount;
+ Out_->Write(&header, sizeof(header));
+
+ // if compression enabled all below writes must go throught compressor
+ auto compressedOut = CompressedOutput(Out_, Compression_);
+ if (compressedOut) {
+ Out_ = compressedOut.Get();
+ }
+
+ // (2) write string pools
+ auto strPoolWrite = [this](TStringBuf str, ui32, ui32) {
+ Out_->Write(str);
+ Out_->Write('\0');
+ };
+
+ LabelNamesPool_.ForEach(strPoolWrite);
+ LabelValuesPool_.ForEach(strPoolWrite);
+
+ // (3) write common time
+ WriteTime(CommonTime_);
+
+ // (4) write common labels' indexes
+ WriteLabels(CommonLabels_, nullptr);
+
+ // (5) write metrics
+ // metrics count already written in header
+ for (TMetric& metric : Metrics_) {
+ // (5.1) types byte
+ ui8 typesByte = PackTypes(metric);
+ Out_->Write(&typesByte, sizeof(typesByte));
+
+ // TODO: implement
+ ui8 flagsByte = 0x00;
+ Out_->Write(&flagsByte, sizeof(flagsByte));
+
+ // v1.2 format addition — metric name
+ if (Version_ >= SV1_02) {
+ const auto it = FindIf(metric.Labels, [&](const auto& l) {
+ return l.Key == MetricName_;
+ });
+ Y_ENSURE(it != metric.Labels.end(),
+ "metric name label '" << LabelNamesPool_.Get(MetricName_->Index) << "' not found, "
+ << "all metric labels '" << FormatLabels(metric.Labels) << "'");
+ WriteVarUInt32(Out_, it->Value->Index);
+ }
+
+ // (5.2) labels
+ WriteLabels(metric.Labels, MetricName_);
+
+ // (5.3) values
+ switch (metric.TimeSeries.Size()) {
+ case 0:
+ break;
+ case 1: {
+ const auto& point = metric.TimeSeries[0];
+ if (point.GetTime() != TInstant::Zero()) {
+ WriteTime(point.GetTime());
+ }
+ EMetricValueType valueType = metric.TimeSeries.GetValueType();
+ WriteValue(metric.MetricType, valueType, point.GetValue());
+ break;
+ }
+ default:
+ WriteVarUInt32(Out_, static_cast<ui32>(metric.TimeSeries.Size()));
+ const TMetricTimeSeries& ts = metric.TimeSeries;
+ EMetricType metricType = metric.MetricType;
+ ts.ForEach([this, metricType](TInstant time, EMetricValueType valueType, TMetricValue value) {
+ // workaround for GCC bug
+ // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=61636
+ this->WriteTime(time);
+ this->WriteValue(metricType, valueType, value);
+ });
+ break;
+ }
+ }
+ }
+
+ // store metric type and values type in one byte
+ ui8 PackTypes(const TMetric& metric) {
+ EValueType valueType;
+ if (metric.TimeSeries.Empty()) {
+ valueType = EValueType::NONE;
+ } else if (metric.TimeSeries.Size() == 1) {
+ TInstant time = metric.TimeSeries[0].GetTime();
+ valueType = (time == TInstant::Zero())
+ ? EValueType::ONE_WITHOUT_TS
+ : EValueType::ONE_WITH_TS;
+ } else {
+ valueType = EValueType::MANY_WITH_TS;
+ }
+ return (static_cast<ui8>(metric.MetricType) << 2) | static_cast<ui8>(valueType);
+ }
+
+ void WriteLabels(const TPooledLabels& labels, const TPooledStr* skipKey) {
+ WriteVarUInt32(Out_, static_cast<ui32>(skipKey ? labels.size() - 1 : labels.size()));
+ for (auto&& label : labels) {
+ if (label.Key == skipKey) {
+ continue;
+ }
+ WriteVarUInt32(Out_, label.Key->Index);
+ WriteVarUInt32(Out_, label.Value->Index);
+ }
+ }
+
+ void WriteValue(EMetricType metricType, EMetricValueType valueType, TMetricValue value) {
+ switch (metricType) {
+ case EMetricType::GAUGE:
+ WriteFixed(value.AsDouble(valueType));
+ break;
+
+ case EMetricType::IGAUGE:
+ WriteFixed(value.AsInt64(valueType));
+ break;
+
+ case EMetricType::COUNTER:
+ case EMetricType::RATE:
+ WriteFixed(value.AsUint64(valueType));
+ break;
+
+ case EMetricType::HIST:
+ case EMetricType::HIST_RATE:
+ WriteHistogram(*value.AsHistogram());
+ break;
+
+ case EMetricType::DSUMMARY:
+ WriteSummaryDouble(*value.AsSummaryDouble());
+ break;
+
+ case EMetricType::LOGHIST:
+ WriteLogHistogram(*value.AsLogHistogram());
+ break;
+
+ default:
+ ythrow yexception() << "unsupported metric type: " << metricType;
+ }
+ }
+
+ void WriteTime(TInstant instant) {
+ switch (TimePrecision_) {
+ case ETimePrecision::SECONDS: {
+ ui32 time = static_cast<ui32>(instant.Seconds());
+ Out_->Write(&time, sizeof(time));
+ break;
+ }
+ case ETimePrecision::MILLIS: {
+ ui64 time = static_cast<ui64>(instant.MilliSeconds());
+ Out_->Write(&time, sizeof(time));
+ }
+ }
+ }
+
+ template <typename T>
+ void WriteFixed(T value) {
+ Out_->Write(&value, sizeof(value));
+ }
+
+ void WriteHistogram(const IHistogramSnapshot& histogram) {
+ ui32 count = histogram.Count();
+ WriteVarUInt32(Out_, count);
+
+ for (ui32 i = 0; i < count; i++) {
+ double bound = histogram.UpperBound(i);
+ Out_->Write(&bound, sizeof(bound));
+ }
+ for (ui32 i = 0; i < count; i++) {
+ ui64 value = histogram.Value(i);
+ Out_->Write(&value, sizeof(value));
+ }
+ }
+
+ void WriteLogHistogram(const TLogHistogramSnapshot& logHist) {
+ WriteFixed(logHist.Base());
+ WriteFixed(logHist.ZerosCount());
+ WriteVarUInt32(Out_, static_cast<ui32>(logHist.StartPower()));
+ WriteVarUInt32(Out_, logHist.Count());
+ for (ui32 i = 0; i < logHist.Count(); ++i) {
+ WriteFixed(logHist.Bucket(i));
+ }
+ }
+
+ void WriteSummaryDouble(const ISummaryDoubleSnapshot& summary) {
+ WriteFixed(summary.GetCount());
+ WriteFixed(summary.GetSum());
+ WriteFixed(summary.GetMin());
+ WriteFixed(summary.GetMax());
+ WriteFixed(summary.GetLast());
+ }
+
+ private:
+ IOutputStream* Out_;
+ ETimePrecision TimePrecision_;
+ ECompression Compression_;
+ ESpackV1Version Version_;
+ const TPooledStr* MetricName_;
+ bool Closed_ = false;
+ };
+
+ }
+
+ IMetricEncoderPtr EncoderSpackV1(
+ IOutputStream* out,
+ ETimePrecision timePrecision,
+ ECompression compression,
+ EMetricsMergingMode mergingMode
+ ) {
+ return MakeHolder<TEncoderSpackV1>(out, timePrecision, compression, mergingMode, SV1_01, "");
+ }
+
+ IMetricEncoderPtr EncoderSpackV12(
+ IOutputStream* out,
+ ETimePrecision timePrecision,
+ ECompression compression,
+ EMetricsMergingMode mergingMode,
+ TStringBuf metricNameLabel
+ ) {
+ Y_ENSURE(!metricNameLabel.Empty(), "metricNameLabel can't be empty");
+ return MakeHolder<TEncoderSpackV1>(out, timePrecision, compression, mergingMode, SV1_02, metricNameLabel);
+ }
+}
diff --git a/library/cpp/monlib/encode/spack/spack_v1_ut.cpp b/library/cpp/monlib/encode/spack/spack_v1_ut.cpp
new file mode 100644
index 0000000000..fe778eb7e0
--- /dev/null
+++ b/library/cpp/monlib/encode/spack/spack_v1_ut.cpp
@@ -0,0 +1,845 @@
+#include "spack_v1.h"
+
+#include <library/cpp/monlib/encode/protobuf/protobuf.h>
+#include <library/cpp/monlib/metrics/labels.h>
+#include <library/cpp/monlib/metrics/metric.h>
+
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <util/generic/buffer.h>
+#include <util/stream/buffer.h>
+#include <util/string/hex.h>
+
+#include <utility>
+
+using namespace NMonitoring;
+
+#define UNIT_ASSERT_BINARY_EQUALS(a, b) \
+ do { \
+ auto size = Y_ARRAY_SIZE(b); \
+ if (Y_UNLIKELY(::memcmp(a, b, size) != 0)) { \
+ auto as = HexEncode(a, size); \
+ auto bs = HexEncode(b, size); \
+ UNIT_FAIL_IMPL("equal assertion failed " #a " == " #b, \
+ "\n actual: " << as << "\nexpected: " << bs); \
+ } \
+ } while (0)
+
+void AssertLabelEqual(const NProto::TLabel& l, TStringBuf name, TStringBuf value) {
+ UNIT_ASSERT_STRINGS_EQUAL(l.GetName(), name);
+ UNIT_ASSERT_STRINGS_EQUAL(l.GetValue(), value);
+}
+
+void AssertPointEqual(const NProto::TPoint& p, TInstant time, double value) {
+ UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds());
+ UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kFloat64);
+ UNIT_ASSERT_DOUBLES_EQUAL(p.GetFloat64(), value, std::numeric_limits<double>::epsilon());
+}
+
+void AssertPointEqual(const NProto::TPoint& p, TInstant time, ui64 value) {
+ UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds());
+ UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kUint64);
+ UNIT_ASSERT_VALUES_EQUAL(p.GetUint64(), value);
+}
+
+void AssertPointEqual(const NProto::TPoint& p, TInstant time, i64 value) {
+ UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds());
+ UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kInt64);
+ UNIT_ASSERT_VALUES_EQUAL(p.GetInt64(), value);
+}
+
+Y_UNIT_TEST_SUITE(TSpackTest) {
+ ui8 expectedHeader_v1_0[] = {
+ 0x53, 0x50, // magic "SP" (fixed ui16)
+ // minor, major
+ 0x00, 0x01, // version (fixed ui16)
+ 0x18, 0x00, // header size (fixed ui16)
+ 0x00, // time precision (fixed ui8)
+ 0x00, // compression algorithm (fixed ui8)
+ 0x0d, 0x00, 0x00, 0x00, // label names size (fixed ui32)
+ 0x40, 0x00, 0x00, 0x00, // labels values size (fixed ui32)
+ 0x08, 0x00, 0x00, 0x00, // metric count (fixed ui32)
+ 0x08, 0x00, 0x00, 0x00, // points count (fixed ui32)
+ };
+
+ ui8 expectedHeader[] = {
+ 0x53, 0x50, // magic "SP" (fixed ui16)
+ // minor, major
+ 0x01, 0x01, // version (fixed ui16)
+ 0x18, 0x00, // header size (fixed ui16)
+ 0x00, // time precision (fixed ui8)
+ 0x00, // compression algorithm (fixed ui8)
+ 0x0d, 0x00, 0x00, 0x00, // label names size (fixed ui32)
+ 0x40, 0x00, 0x00, 0x00, // labels values size (fixed ui32)
+ 0x08, 0x00, 0x00, 0x00, // metric count (fixed ui32)
+ 0x08, 0x00, 0x00, 0x00, // points count (fixed ui32)
+ };
+
+ ui8 expectedStringPools[] = {
+ 0x6e, 0x61, 0x6d, 0x65, 0x00, // "name\0"
+ 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x00, // "project\0"
+ 0x73, 0x6f, 0x6c, 0x6f, 0x6d, 0x6f, 0x6e, 0x00, // "solomon\0"
+ 0x71, 0x31, 0x00, // "q1\0"
+ 0x71, 0x32, 0x00, // "q2\0"
+ 0x71, 0x33, 0x00, // "q3\0"
+ 0x61, 0x6e, 0x73, 0x77, 0x65, 0x72, 0x00, // "answer\0"
+ 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, // "responseTimeMillis\0"
+ 0x54, 0x69, 0x6d, 0x65, 0x4d, 0x69, 0x6c, 0x6c,
+ 0x69, 0x73, 0x00,
+ 0x62, 0x79, 0x74, 0x65, 0x73, 0x00, // "bytes\0"
+ 0x74, 0x65, 0x6D, 0x70, 0x65, 0x72, 0x61, 0x74, // "temperature\0"
+ 0x75, 0x72, 0x65, 0x00,
+ 0x6d, 0x73, 0x00, // "ms\0"
+ };
+
+ ui8 expectedCommonTime[] = {
+ 0x00, 0x2f, 0x68, 0x59, // common time in seconds (fixed ui32)
+ };
+
+ ui8 expectedCommonLabels[] = {
+ 0x01, // common labels count (varint)
+ 0x01, // label name index (varint)
+ 0x00, // label value index (varint)
+ };
+
+ ui8 expectedMetric1[] = {
+ 0x0C, // types (RATE | NONE) (fixed ui8)
+ 0x00, // flags (fixed ui8)
+ 0x01, // metric labels count (varint)
+ 0x00, // label name index (varint)
+ 0x01, // label value index (varint)
+ };
+
+ ui8 expectedMetric2[] = {
+ 0x09, // types (COUNTER | ONE_WITHOUT_TS) (fixed ui8)
+ 0x00, // flags (fixed ui8)
+ 0x01, // metric labels count (varint)
+ 0x00, // label name index (varint)
+ 0x02, // label value index (varint)
+ 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // value (fixed ui64)
+ };
+
+ ui8 expectedMetric3[] = {
+ 0x0a, // types (COUNTER | ONE_WITH_TS) (fixed ui8)
+ 0x00, // flags (fixed ui8)
+ 0x01, // metric labels count (varint)
+ 0x00, // label name index (varint)
+ 0x03, // label value index (varint)
+ 0x0b, 0x63, 0xfe, 0x59, // time in seconds (fixed ui32)
+ 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // value (fixed ui64)
+ };
+
+ ui8 expectedMetric4[] = {
+ 0x07, // types (GAUGE | MANY_WITH_TS) (fixed ui8)
+ 0x00, // flags (fixed ui8)
+ 0x01, // metric labels count (varint)
+ 0x00, // label name index (varint)
+ 0x04, // label value index (varint)
+ 0x02, // points count (varint)
+ 0x0b, 0x63, 0xfe, 0x59, // time in seconds (fixed ui32)
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x45, 0x40, // value (double IEEE754)
+ 0x1a, 0x63, 0xfe, 0x59, // time in seconds (fixed ui32)
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x4d, 0x40 // value (double IEEE754)
+ };
+
+ ui8 expectedMetric5_v1_0[] = {
+ 0x16, // types (HIST | ONE_WITH_TS) (fixed ui8)
+ 0x00, // flags (fixed ui8)
+ 0x01, // metric labels count (varint)
+ 0x00, // label name index (varint)
+ 0x05, // label value index (varint)
+ 0x0b, 0x63, 0xfe, 0x59, // time in seconds (fixed ui32)
+ 0x06, // histogram buckets count (varint)
+ 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // histogram bucket bounds (array of fixed ui64)
+ 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f,
+ 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // histogram bucket values
+ 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x53, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ };
+
+ ui8 expectedMetric5[] = {
+ 0x16, // types (HIST | ONE_WITH_TS) (fixed ui8)
+ 0x00, // flags (fixed ui8)
+ 0x01, // metric labels count (varint)
+ 0x00, // label name index (varint)
+ 0x05, // label value index (varint)
+ 0x0b, 0x63, 0xfe, 0x59, // time in seconds (fixed ui32)
+ 0x06, // histogram buckets count (varint)
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x3f, // histogram bucket bounds (array of doubles)
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x40,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x40,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x40,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xef, 0x7f,
+ 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // histogram bucket values
+ 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x53, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ };
+
+ ui8 expectedMetric6[] = {
+ 0x12, // types (IGAUGE | ONE_WITH_TS) (fixed ui8)
+ 0x00, // flags (fixed ui8)
+ 0x01, // metric labels count (varint)
+ 0x00, // label name index (varint)
+ 0x06, // label value index (varint)
+ 0x0b, 0x63, 0xfe, 0x59, // time in seconds (fixed ui32)
+ 0x39, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // value (fixed i64)
+ };
+
+ ui8 expectedMetric7[] = {
+ 0x1e, // types (DSUMMARY | ONE_WITH_TS) (fixed ui8)
+ 0x00, // flags (fixed ui8)
+ 0x01, // metric labels count (varint)
+ 0x00, // label name index (varint)
+ 0x07, // label value index (varint)
+ 0x0b, 0x63, 0xfe, 0x59, // time in seconds (fixed ui32)
+ 0x1e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // count (fixed ui64)
+ 0x33, 0x33, 0x33, 0x33, 0x33, 0x33, 0x24, 0x40, // sum (fixed double)
+ 0xcd, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xdc, 0xbf, // min (fixed double)
+ 0x64, 0x3b, 0xdf, 0x4f, 0x8d, 0x97, 0xde, 0x3f, // max (fixed double)
+ 0x33, 0x33, 0x33, 0x33, 0x33, 0x33, 0xd3, 0x3f, // last (fixed double)
+ };
+
+ ui8 expectedMetric8[] = {
+ 0x26, // types (LOGHIST | ONE_WITH_TS) (fixed ui8)
+ 0x00, // flags (fixed ui8)
+ 0x01, // metric labels count (variant)
+ 0x00, // label name index (variant)
+ 0x08, // label value index (variant)
+ 0x0b, 0x63, 0xfe, 0x59, // time in seconds (fixed ui32)
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF8, 0x3F, // base (fixed double)
+ 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // zerosCount (fixed ui64)
+ 0x00, // startPower (variant)
+ 0x04, // buckets count (variant)
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE0, 0x3F,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD0, 0x3F, // bucket values
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD0, 0x3F,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE0, 0x3F,
+ };
+
+ const size_t expectedSize =
+ Y_ARRAY_SIZE(expectedHeader) +
+ Y_ARRAY_SIZE(expectedStringPools) +
+ Y_ARRAY_SIZE(expectedCommonTime) +
+ Y_ARRAY_SIZE(expectedCommonLabels) +
+ Y_ARRAY_SIZE(expectedMetric1) +
+ Y_ARRAY_SIZE(expectedMetric2) +
+ Y_ARRAY_SIZE(expectedMetric3) +
+ Y_ARRAY_SIZE(expectedMetric4) +
+ Y_ARRAY_SIZE(expectedMetric5) +
+ Y_ARRAY_SIZE(expectedMetric6) +
+ Y_ARRAY_SIZE(expectedMetric7) +
+ Y_ARRAY_SIZE(expectedMetric8);
+
+ const TInstant now = TInstant::ParseIso8601Deprecated("2017-11-05T01:02:03Z");
+
+ // {1: 1, 2: 1, 4: 2, 8: 4, 16: 8, inf: 83}
+ IHistogramSnapshotPtr TestHistogram() {
+ auto h = ExponentialHistogram(6, 2);
+ for (i64 i = 1; i < 100; i++) {
+ h->Collect(i);
+ }
+ return h->Snapshot();
+ }
+
+ TLogHistogramSnapshotPtr TestLogHistogram() {
+ TVector buckets{0.5, 0.25, 0.25, 0.5};
+ return MakeIntrusive<TLogHistogramSnapshot>(1.5, 1u, 0, std::move(buckets));
+ }
+
+ ISummaryDoubleSnapshotPtr TestSummaryDouble() {
+ return MakeIntrusive<TSummaryDoubleSnapshot>(10.1, -0.45, 0.478, 0.3, 30u);
+ }
+
+ Y_UNIT_TEST(Encode) {
+ TBuffer buffer;
+ TBufferOutput out(buffer);
+ auto e = EncoderSpackV1(
+ &out, ETimePrecision::SECONDS, ECompression::IDENTITY);
+
+ e->OnStreamBegin();
+ { // common time
+ e->OnCommonTime(TInstant::Seconds(1500000000));
+ }
+ { // common labels
+ e->OnLabelsBegin();
+ e->OnLabel("project", "solomon");
+ e->OnLabelsEnd();
+ }
+ { // metric #1
+ e->OnMetricBegin(EMetricType::RATE);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("name", "q1");
+ e->OnLabelsEnd();
+ }
+ e->OnMetricEnd();
+ }
+ { // metric #2
+ e->OnMetricBegin(EMetricType::COUNTER);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("name", "q2");
+ e->OnLabelsEnd();
+ }
+ // Only the last value will be encoded
+ e->OnUint64(TInstant::Zero(), 10);
+ e->OnUint64(TInstant::Zero(), 13);
+ e->OnUint64(TInstant::Zero(), 17);
+ e->OnMetricEnd();
+ }
+ { // metric #3
+ e->OnMetricBegin(EMetricType::COUNTER);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("name", "q3");
+ e->OnLabelsEnd();
+ }
+ e->OnUint64(now, 10);
+ e->OnUint64(now, 13);
+ e->OnUint64(now, 17);
+ e->OnMetricEnd();
+ }
+ { // metric #4
+ e->OnMetricBegin(EMetricType::GAUGE);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("name", "answer");
+ e->OnLabelsEnd();
+ }
+ e->OnDouble(now, 42);
+ e->OnDouble(now + TDuration::Seconds(15), 59);
+ e->OnMetricEnd();
+ }
+ { // metric #5
+ e->OnMetricBegin(EMetricType::HIST);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("name", "responseTimeMillis");
+ e->OnLabelsEnd();
+ }
+
+ auto histogram = TestHistogram();
+ e->OnHistogram(now, histogram);
+ e->OnMetricEnd();
+ }
+ { // metric #6
+ e->OnMetricBegin(EMetricType::IGAUGE);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("name", "bytes");
+ e->OnLabelsEnd();
+ }
+ e->OnInt64(now, 1337);
+ e->OnMetricEnd();
+ }
+ { // metric 7
+ e->OnMetricBegin(EMetricType::DSUMMARY);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("name", "temperature");
+ e->OnLabelsEnd();
+ }
+ e->OnSummaryDouble(now, TestSummaryDouble());
+ e->OnMetricEnd();
+ }
+ { // metric 8
+ e->OnMetricBegin(EMetricType::LOGHIST);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("name", "ms");
+ e->OnLabelsEnd();
+ }
+ e->OnLogHistogram(now, TestLogHistogram());
+ e->OnMetricEnd();
+ }
+ e->OnStreamEnd();
+ e->Close();
+
+ // Cout << "encoded: " << HexEncode(buffer.Data(), buffer.Size()) << Endl;
+ // Cout << "size: " << buffer.Size() << Endl;
+
+ UNIT_ASSERT_VALUES_EQUAL(buffer.Size(), expectedSize);
+
+ ui8* p = reinterpret_cast<ui8*>(buffer.Data());
+ UNIT_ASSERT_BINARY_EQUALS(p, expectedHeader);
+ p += Y_ARRAY_SIZE(expectedHeader);
+
+ UNIT_ASSERT_BINARY_EQUALS(p, expectedStringPools);
+ p += Y_ARRAY_SIZE(expectedStringPools);
+
+ UNIT_ASSERT_BINARY_EQUALS(p, expectedCommonTime);
+ p += Y_ARRAY_SIZE(expectedCommonTime);
+
+ UNIT_ASSERT_BINARY_EQUALS(p, expectedCommonLabels);
+ p += Y_ARRAY_SIZE(expectedCommonLabels);
+
+ UNIT_ASSERT_BINARY_EQUALS(p, expectedMetric1);
+ p += Y_ARRAY_SIZE(expectedMetric1);
+
+ UNIT_ASSERT_BINARY_EQUALS(p, expectedMetric2);
+ p += Y_ARRAY_SIZE(expectedMetric2);
+
+ UNIT_ASSERT_BINARY_EQUALS(p, expectedMetric3);
+ p += Y_ARRAY_SIZE(expectedMetric3);
+
+ UNIT_ASSERT_BINARY_EQUALS(p, expectedMetric4);
+ p += Y_ARRAY_SIZE(expectedMetric4);
+
+ UNIT_ASSERT_BINARY_EQUALS(p, expectedMetric5);
+ p += Y_ARRAY_SIZE(expectedMetric5);
+
+ UNIT_ASSERT_BINARY_EQUALS(p, expectedMetric6);
+ p += Y_ARRAY_SIZE(expectedMetric6);
+
+ UNIT_ASSERT_BINARY_EQUALS(p, expectedMetric7);
+ p += Y_ARRAY_SIZE(expectedMetric7);
+
+ UNIT_ASSERT_BINARY_EQUALS(p, expectedMetric8);
+ p += Y_ARRAY_SIZE(expectedMetric8);
+ }
+
+ NProto::TMultiSamplesList GetMergingMetricSamples(EMetricsMergingMode mergingMode) {
+ TBuffer buffer;
+ TBufferOutput out(buffer);
+
+ auto e = EncoderSpackV1(
+ &out,
+ ETimePrecision::SECONDS,
+ ECompression::IDENTITY,
+ mergingMode
+ );
+
+ e->OnStreamBegin();
+ for (size_t i = 0; i != 3; ++i) {
+ e->OnMetricBegin(EMetricType::COUNTER);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("name", "my_counter");
+ e->OnLabelsEnd();
+ }
+ e->OnUint64(TInstant::Zero() + TDuration::Seconds(i), i + 1);
+ e->OnMetricEnd();
+ }
+ e->OnStreamEnd();
+ e->Close();
+
+ NProto::TMultiSamplesList samples;
+ IMetricEncoderPtr eProto = EncoderProtobuf(&samples);
+ TBufferInput in(buffer);
+ DecodeSpackV1(&in, eProto.Get());
+
+ return samples;
+ }
+
+ Y_UNIT_TEST(SpackEncoderMergesMetrics) {
+ {
+ NProto::TMultiSamplesList samples = GetMergingMetricSamples(EMetricsMergingMode::DEFAULT);
+
+ UNIT_ASSERT_EQUAL(samples.SamplesSize(), 3);
+ UNIT_ASSERT_EQUAL(samples.GetSamples(0).GetPoints(0).GetUint64(), 1);
+ UNIT_ASSERT_EQUAL(samples.GetSamples(1).GetPoints(0).GetUint64(), 2);
+ UNIT_ASSERT_EQUAL(samples.GetSamples(2).GetPoints(0).GetUint64(), 3);
+ }
+
+ {
+ NProto::TMultiSamplesList samples = GetMergingMetricSamples(EMetricsMergingMode::MERGE_METRICS);
+
+ UNIT_ASSERT_EQUAL(samples.SamplesSize(), 1);
+
+ auto sample0 = samples.GetSamples(0);
+ UNIT_ASSERT_EQUAL(sample0.GetPoints(0).GetUint64(), 1);
+ UNIT_ASSERT_EQUAL(sample0.GetPoints(1).GetUint64(), 2);
+ UNIT_ASSERT_EQUAL(sample0.GetPoints(2).GetUint64(), 3);
+ }
+ }
+
+ void DecodeDataToSamples(NProto::TMultiSamplesList & samples, ui16 version) {
+ IMetricEncoderPtr e = EncoderProtobuf(&samples);
+
+ TBuffer data(expectedSize);
+ if (SV1_00 == version) { // v1.0
+ data.Append(reinterpret_cast<char*>(expectedHeader_v1_0), Y_ARRAY_SIZE(expectedHeader_v1_0));
+ } else {
+ data.Append(reinterpret_cast<char*>(expectedHeader), Y_ARRAY_SIZE(expectedHeader));
+ }
+ data.Append(reinterpret_cast<char*>(expectedStringPools), Y_ARRAY_SIZE(expectedStringPools));
+ data.Append(reinterpret_cast<char*>(expectedCommonTime), Y_ARRAY_SIZE(expectedCommonTime));
+ data.Append(reinterpret_cast<char*>(expectedCommonLabels), Y_ARRAY_SIZE(expectedCommonLabels));
+ data.Append(reinterpret_cast<char*>(expectedMetric1), Y_ARRAY_SIZE(expectedMetric1));
+ data.Append(reinterpret_cast<char*>(expectedMetric2), Y_ARRAY_SIZE(expectedMetric2));
+ data.Append(reinterpret_cast<char*>(expectedMetric3), Y_ARRAY_SIZE(expectedMetric3));
+ data.Append(reinterpret_cast<char*>(expectedMetric4), Y_ARRAY_SIZE(expectedMetric4));
+ if (SV1_00 == version) { // v1.0
+ data.Append(reinterpret_cast<char*>(expectedMetric5_v1_0), Y_ARRAY_SIZE(expectedMetric5_v1_0));
+ } else {
+ data.Append(reinterpret_cast<char*>(expectedMetric5), Y_ARRAY_SIZE(expectedMetric5));
+ }
+ data.Append(reinterpret_cast<char*>(expectedMetric6), Y_ARRAY_SIZE(expectedMetric6));
+ data.Append(reinterpret_cast<char*>(expectedMetric7), Y_ARRAY_SIZE(expectedMetric7));
+ data.Append(reinterpret_cast<char*>(expectedMetric8), Y_ARRAY_SIZE(expectedMetric8));
+ TBufferInput in(data);
+ DecodeSpackV1(&in, e.Get());
+ }
+
+ void DecodeDataToSamples(NProto::TMultiSamplesList & samples) {
+ TSpackHeader header;
+ header.Version = SV1_01;
+ DecodeDataToSamples(samples, header.Version);
+ }
+
+ Y_UNIT_TEST(Decode) {
+ NProto::TMultiSamplesList samples;
+ DecodeDataToSamples(samples);
+
+ UNIT_ASSERT_VALUES_EQUAL(
+ TInstant::MilliSeconds(samples.GetCommonTime()),
+ TInstant::Seconds(1500000000));
+
+ UNIT_ASSERT_VALUES_EQUAL(samples.CommonLabelsSize(), 1);
+ AssertLabelEqual(samples.GetCommonLabels(0), "project", "solomon");
+
+ UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 8);
+ {
+ const NProto::TMultiSample& s = samples.GetSamples(0);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::RATE);
+ UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1);
+ AssertLabelEqual(s.GetLabels(0), "name", "q1");
+ }
+ {
+ const NProto::TMultiSample& s = samples.GetSamples(1);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::COUNTER);
+ UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1);
+ AssertLabelEqual(s.GetLabels(0), "name", "q2");
+
+ UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1);
+ AssertPointEqual(s.GetPoints(0), TInstant::Zero(), ui64(17));
+ }
+ {
+ const NProto::TMultiSample& s = samples.GetSamples(2);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::COUNTER);
+ UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1);
+ AssertLabelEqual(s.GetLabels(0), "name", "q3");
+
+ UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1);
+ AssertPointEqual(s.GetPoints(0), now, ui64(17));
+ }
+ {
+ const NProto::TMultiSample& s = samples.GetSamples(3);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE);
+ UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1);
+ AssertLabelEqual(s.GetLabels(0), "name", "answer");
+
+ UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 2);
+ AssertPointEqual(s.GetPoints(0), now, double(42));
+ AssertPointEqual(s.GetPoints(1), now + TDuration::Seconds(15), double(59));
+ }
+ {
+ const NProto::TMultiSample& s = samples.GetSamples(4);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::HISTOGRAM);
+ UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1);
+ AssertLabelEqual(s.GetLabels(0), "name", "responseTimeMillis");
+
+ UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1);
+
+ const NProto::TPoint& p = s.GetPoints(0);
+ UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), now.MilliSeconds());
+ UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kHistogram);
+
+ auto histogram = TestHistogram();
+
+ const NProto::THistogram& pointHistogram = p.GetHistogram();
+ UNIT_ASSERT_VALUES_EQUAL(pointHistogram.BoundsSize(), histogram->Count());
+ UNIT_ASSERT_VALUES_EQUAL(pointHistogram.ValuesSize(), histogram->Count());
+
+ for (size_t i = 0; i < pointHistogram.BoundsSize(); i++) {
+ UNIT_ASSERT_DOUBLES_EQUAL(pointHistogram.GetBounds(i), histogram->UpperBound(i), Min<double>());
+ UNIT_ASSERT_VALUES_EQUAL(pointHistogram.GetValues(i), histogram->Value(i));
+ }
+ }
+ {
+ const NProto::TMultiSample& s = samples.GetSamples(5);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::IGAUGE);
+ UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1);
+ AssertLabelEqual(s.GetLabels(0), "name", "bytes");
+
+ UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1);
+ AssertPointEqual(s.GetPoints(0), now, i64(1337));
+ }
+ {
+ const NProto::TMultiSample& s = samples.GetSamples(6);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::DSUMMARY);
+ UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1);
+ AssertLabelEqual(s.GetLabels(0), "name", "temperature");
+
+ UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1);
+
+ const NProto::TPoint& p = s.GetPoints(0);
+ UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), now.MilliSeconds());
+ UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kSummaryDouble);
+
+ auto expected = TestSummaryDouble();
+
+ auto actual = p.GetSummaryDouble();
+
+ UNIT_ASSERT_VALUES_EQUAL(expected->GetSum(), actual.GetSum());
+ UNIT_ASSERT_VALUES_EQUAL(expected->GetMin(), actual.GetMin());
+ UNIT_ASSERT_VALUES_EQUAL(expected->GetMax(), actual.GetMax());
+ UNIT_ASSERT_VALUES_EQUAL(expected->GetLast(), actual.GetLast());
+ UNIT_ASSERT_VALUES_EQUAL(expected->GetCount(), actual.GetCount());
+ }
+ {
+ const NProto::TMultiSample& s = samples.GetSamples(7);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::LOGHISTOGRAM);
+ UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1);
+ AssertLabelEqual(s.GetLabels(0), "name", "ms");
+
+ UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1);
+
+ const NProto::TPoint& p = s.GetPoints(0);
+ UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), now.MilliSeconds());
+ UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kLogHistogram);
+
+ auto expected = TestLogHistogram();
+ auto actual = p.GetLogHistogram();
+
+ UNIT_ASSERT_VALUES_EQUAL(expected->ZerosCount(), actual.GetZerosCount());
+ UNIT_ASSERT_VALUES_EQUAL(expected->Base(), actual.GetBase());
+ UNIT_ASSERT_VALUES_EQUAL(expected->StartPower(), actual.GetStartPower());
+ UNIT_ASSERT_VALUES_EQUAL(expected->Count(), actual.BucketsSize());
+ for (size_t i = 0; i < expected->Count(); ++i) {
+ UNIT_ASSERT_VALUES_EQUAL(expected->Bucket(i), actual.GetBuckets(i));
+ }
+ }
+ }
+
+ void TestCompression(ECompression alg) {
+ TBuffer buffer;
+ {
+ TBufferOutput out(buffer);
+ auto e = EncoderSpackV1(&out, ETimePrecision::MILLIS, alg);
+ e->OnStreamBegin();
+ {
+ e->OnMetricBegin(EMetricType::GAUGE);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("name", "answer");
+ e->OnLabelsEnd();
+ }
+ e->OnDouble(now, 42);
+ e->OnMetricEnd();
+ }
+ e->OnStreamEnd();
+ e->Close();
+ }
+
+ auto* header = reinterpret_cast<const TSpackHeader*>(buffer.Data());
+ UNIT_ASSERT_EQUAL(DecodeCompression(header->Compression), alg);
+
+ NProto::TMultiSamplesList samples;
+ {
+ IMetricEncoderPtr e = EncoderProtobuf(&samples);
+ TBufferInput in(buffer);
+ DecodeSpackV1(&in, e.Get());
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(
+ TInstant::MilliSeconds(samples.GetCommonTime()),
+ TInstant::Zero());
+
+ UNIT_ASSERT_VALUES_EQUAL(samples.CommonLabelsSize(), 0);
+
+ UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 1);
+ {
+ const NProto::TMultiSample& s = samples.GetSamples(0);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE);
+ UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1);
+ AssertLabelEqual(s.GetLabels(0), "name", "answer");
+ AssertPointEqual(s.GetPoints(0), now, 42.0);
+ }
+ }
+
+ Y_UNIT_TEST(CompressionIdentity) {
+ TestCompression(ECompression::IDENTITY);
+ }
+
+ Y_UNIT_TEST(CompressionZlib) {
+ TestCompression(ECompression::ZLIB);
+ }
+
+ Y_UNIT_TEST(CompressionZstd) {
+ TestCompression(ECompression::ZSTD);
+ }
+
+ Y_UNIT_TEST(CompressionLz4) {
+ TestCompression(ECompression::LZ4);
+ }
+
+ Y_UNIT_TEST(Decode_v1_0_histograms) {
+ // Check that histogram bounds decoded from different versions are the same
+ NProto::TMultiSamplesList samples, samples_v1_0;
+ DecodeDataToSamples(samples);
+ DecodeDataToSamples(samples_v1_0, /*version = */ SV1_00);
+
+ const NProto::THistogram& pointHistogram = samples.GetSamples(4).GetPoints(0).GetHistogram();
+ const NProto::THistogram& pointHistogram_v1_0 = samples_v1_0.GetSamples(4).GetPoints(0).GetHistogram();
+
+ for (size_t i = 0; i < pointHistogram.BoundsSize(); i++) {
+ UNIT_ASSERT_DOUBLES_EQUAL(pointHistogram.GetBounds(i), pointHistogram_v1_0.GetBounds(i), Min<double>());
+ }
+ }
+
+ Y_UNIT_TEST(SimpleV12) {
+ ui8 expectedSerialized[] = {
+ // header
+ 0x53, 0x50, // magic "SP" (fixed ui16)
+ // minor, major
+ 0x02, 0x01, // version (fixed ui16)
+ 0x18, 0x00, // header size (fixed ui16)
+ 0x00, // time precision (fixed ui8)
+ 0x00, // compression algorithm (fixed ui8)
+ 0x0A, 0x00, 0x00, 0x00, // label names size (fixed ui32)
+ 0x14, 0x00, 0x00, 0x00, // labels values size (fixed ui32)
+ 0x01, 0x00, 0x00, 0x00, // metric count (fixed ui32)
+ 0x01, 0x00, 0x00, 0x00, // points count (fixed ui32)
+
+ // string pools
+ 0x73, 0x00, // "s\0"
+ 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x00, // "project\0"
+ 0x73, 0x6f, 0x6c, 0x6f, 0x6d, 0x6f, 0x6e, 0x00, // "solomon\0"
+ 0x74, 0x65, 0x6D, 0x70, 0x65, 0x72, 0x61, 0x74, // temperature
+ 0x75, 0x72, 0x65, 0x00,
+
+ // common time
+ 0x00, 0x2f, 0x68, 0x59, // common time in seconds (fixed ui32)
+
+ // common labels
+ 0x00, // common labels count (varint)
+
+ // metric
+ 0x09, // types (COUNTER | ONE_WITHOUT_TS) (fixed ui8)
+ 0x00, // flags (fixed ui8)
+ 0x01, // name index (varint)
+ 0x01, // metric labels count (varint)
+ 0x01, // 'project' label name index (varint)
+ 0x00, // 'project' label value index (varint)
+ 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // value (fixed ui64)
+ };
+
+ // encode
+ {
+ TBuffer actualSerialized;
+ {
+ TBufferOutput out(actualSerialized);
+ auto e = EncoderSpackV12(
+ &out,
+ ETimePrecision::SECONDS,
+ ECompression::IDENTITY,
+ EMetricsMergingMode::DEFAULT,
+ "s");
+
+ e->OnStreamBegin();
+ e->OnCommonTime(TInstant::Seconds(1500000000));
+
+ {
+ e->OnMetricBegin(EMetricType::COUNTER);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("project", "solomon");
+ e->OnLabel("s", "temperature");
+ e->OnLabelsEnd();
+ }
+ // Only the last value will be encoded
+ e->OnUint64(TInstant::Zero(), 10);
+ e->OnUint64(TInstant::Zero(), 13);
+ e->OnUint64(TInstant::Zero(), 17);
+ e->OnMetricEnd();
+ }
+
+ e->OnStreamEnd();
+ e->Close();
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(actualSerialized.Size(), Y_ARRAY_SIZE(expectedSerialized));
+ UNIT_ASSERT_BINARY_EQUALS(actualSerialized.Data(), expectedSerialized);
+ }
+
+ // decode
+ {
+ NProto::TMultiSamplesList samples;
+ {
+ auto input = TMemoryInput(expectedSerialized, Y_ARRAY_SIZE(expectedSerialized));
+ auto encoder = EncoderProtobuf(&samples);
+ DecodeSpackV1(&input, encoder.Get(), "s");
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(TInstant::MilliSeconds(samples.GetCommonTime()),
+ TInstant::Seconds(1500000000));
+
+ UNIT_ASSERT_VALUES_EQUAL(samples.CommonLabelsSize(), 0);
+
+ UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 1);
+ {
+ const auto& s = samples.GetSamples(0);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::COUNTER);
+ UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2);
+ AssertLabelEqual(s.GetLabels(0), "s", "temperature");
+ AssertLabelEqual(s.GetLabels(1), "project", "solomon");
+
+ UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1);
+ AssertPointEqual(s.GetPoints(0), TInstant::Zero(), ui64(17));
+ }
+ }
+ }
+
+ Y_UNIT_TEST(V12MissingNameForOneMetric) {
+ TBuffer b;
+ TBufferOutput out(b);
+ auto e = EncoderSpackV12(
+ &out,
+ ETimePrecision::SECONDS,
+ ECompression::IDENTITY,
+ EMetricsMergingMode::DEFAULT,
+ "s");
+
+ UNIT_ASSERT_EXCEPTION_CONTAINS(
+ [&]() {
+ e->OnStreamBegin();
+ {
+ e->OnMetricBegin(EMetricType::COUNTER);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("s", "s1");
+ e->OnLabelsEnd();
+ }
+ e->OnUint64(TInstant::Zero(), 1);
+ e->OnMetricEnd();
+
+ e->OnMetricBegin(EMetricType::COUNTER);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("project", "solomon");
+ e->OnLabel("m", "v");
+ e->OnLabelsEnd();
+ }
+ e->OnUint64(TInstant::Zero(), 2);
+ e->OnMetricEnd();
+ }
+
+ e->OnStreamEnd();
+ e->Close();
+ }(),
+ yexception,
+ "metric name label 's' not found, all metric labels '{m=v, project=solomon}'");
+ }
+}
diff --git a/library/cpp/monlib/encode/spack/ut/ya.make b/library/cpp/monlib/encode/spack/ut/ya.make
new file mode 100644
index 0000000000..980bf54667
--- /dev/null
+++ b/library/cpp/monlib/encode/spack/ut/ya.make
@@ -0,0 +1,16 @@
+UNITTEST_FOR(library/cpp/monlib/encode/spack)
+
+OWNER(
+ g:solomon
+ jamel
+)
+
+SRCS(
+ spack_v1_ut.cpp
+)
+
+PEERDIR(
+ library/cpp/monlib/encode/protobuf
+)
+
+END()
diff --git a/library/cpp/monlib/encode/spack/varint.cpp b/library/cpp/monlib/encode/spack/varint.cpp
new file mode 100644
index 0000000000..051cf17380
--- /dev/null
+++ b/library/cpp/monlib/encode/spack/varint.cpp
@@ -0,0 +1,79 @@
+#include "varint.h"
+
+#include <util/generic/yexception.h>
+#include <util/stream/input.h>
+#include <util/stream/output.h>
+
+namespace NMonitoring {
+ ui32 WriteVarUInt32(IOutputStream* output, ui32 value) {
+ bool stop = false;
+ ui32 written = 0;
+ while (!stop) {
+ ui8 byte = static_cast<ui8>(value | 0x80);
+ value >>= 7;
+ if (value == 0) {
+ stop = true;
+ byte &= 0x7F;
+ }
+ output->Write(byte);
+ written++;
+ }
+ return written;
+ }
+
+ ui32 ReadVarUInt32(IInputStream* input) {
+ ui32 value = 0;
+ switch (TryReadVarUInt32(input, &value)) {
+ case EReadResult::OK:
+ return value;
+ case EReadResult::ERR_OVERFLOW:
+ ythrow yexception() << "the data is too long to read ui32";
+ case EReadResult::ERR_UNEXPECTED_EOF:
+ ythrow yexception() << "the data unexpectedly ended";
+ default:
+ ythrow yexception() << "unknown error while reading varint";
+ }
+ }
+
+ size_t ReadVarUInt32(const ui8* buf, size_t len, ui32* result) {
+ size_t count = 0;
+ ui32 value = 0;
+
+ ui8 byte = 0;
+ do {
+ if (7 * count > 8 * sizeof(ui32)) {
+ ythrow yexception() << "the data is too long to read ui32";
+ }
+ if (count == len) {
+ ythrow yexception() << "the data unexpectedly ended";
+ }
+ byte = buf[count];
+ value |= (static_cast<ui32>(byte & 0x7F)) << (7 * count);
+ ++count;
+ } while (byte & 0x80);
+
+ *result = value;
+ return count;
+ }
+
+EReadResult TryReadVarUInt32(IInputStream* input, ui32* value) {
+ size_t count = 0;
+ ui32 result = 0;
+
+ ui8 byte = 0;
+ do {
+ if (7 * count > 8 * sizeof(ui32)) {
+ return EReadResult::ERR_OVERFLOW;
+ }
+ if (input->Read(&byte, 1) != 1) {
+ return EReadResult::ERR_UNEXPECTED_EOF;
+ }
+ result |= (static_cast<ui32>(byte & 0x7F)) << (7 * count);
+ ++count;
+ } while (byte & 0x80);
+
+ *value = result;
+ return EReadResult::OK;
+ }
+
+}
diff --git a/library/cpp/monlib/encode/spack/varint.h b/library/cpp/monlib/encode/spack/varint.h
new file mode 100644
index 0000000000..7ac522dd6c
--- /dev/null
+++ b/library/cpp/monlib/encode/spack/varint.h
@@ -0,0 +1,23 @@
+#pragma once
+
+#include <util/system/types.h>
+
+class IInputStream;
+class IOutputStream;
+
+namespace NMonitoring {
+ ui32 WriteVarUInt32(IOutputStream* output, ui32 value);
+
+ ui32 ReadVarUInt32(IInputStream* input);
+ size_t ReadVarUInt32(const ui8* buf, size_t len, ui32* result);
+
+ enum class EReadResult {
+ OK,
+ ERR_OVERFLOW,
+ ERR_UNEXPECTED_EOF,
+ };
+
+ [[nodiscard]]
+ EReadResult TryReadVarUInt32(IInputStream* input, ui32* value);
+
+}
diff --git a/library/cpp/monlib/encode/spack/ya.make b/library/cpp/monlib/encode/spack/ya.make
new file mode 100644
index 0000000000..78d3061291
--- /dev/null
+++ b/library/cpp/monlib/encode/spack/ya.make
@@ -0,0 +1,25 @@
+LIBRARY()
+
+OWNER(
+ g:solomon
+ jamel
+)
+
+SRCS(
+ spack_v1_decoder.cpp
+ spack_v1_encoder.cpp
+ varint.cpp
+ compression.cpp
+)
+
+PEERDIR(
+ library/cpp/monlib/encode/buffered
+ library/cpp/monlib/exception
+
+ contrib/libs/lz4
+ contrib/libs/xxhash
+ contrib/libs/zlib
+ contrib/libs/zstd
+)
+
+END()
diff --git a/library/cpp/monlib/encode/text/text.h b/library/cpp/monlib/encode/text/text.h
new file mode 100644
index 0000000000..6b2be3937b
--- /dev/null
+++ b/library/cpp/monlib/encode/text/text.h
@@ -0,0 +1,9 @@
+#pragma once
+
+#include <library/cpp/monlib/encode/encoder.h>
+
+class IOutputStream;
+
+namespace NMonitoring {
+ IMetricEncoderPtr EncoderText(IOutputStream* out, bool humanReadableTs = true);
+}
diff --git a/library/cpp/monlib/encode/text/text_encoder.cpp b/library/cpp/monlib/encode/text/text_encoder.cpp
new file mode 100644
index 0000000000..10336261f0
--- /dev/null
+++ b/library/cpp/monlib/encode/text/text_encoder.cpp
@@ -0,0 +1,226 @@
+#include "text.h"
+
+#include <library/cpp/monlib/encode/encoder_state.h>
+#include <library/cpp/monlib/metrics/labels.h>
+#include <library/cpp/monlib/metrics/metric_value.h>
+
+#include <util/datetime/base.h>
+#include <util/stream/format.h>
+
+namespace NMonitoring {
+ namespace {
+ class TEncoderText final: public IMetricEncoder {
+ public:
+ TEncoderText(IOutputStream* out, bool humanReadableTs)
+ : Out_(out)
+ , HumanReadableTs_(humanReadableTs)
+ {
+ }
+
+ private:
+ void OnStreamBegin() override {
+ State_.Expect(TEncoderState::EState::ROOT);
+ }
+
+ void OnStreamEnd() override {
+ State_.Expect(TEncoderState::EState::ROOT);
+ }
+
+ void OnCommonTime(TInstant time) override {
+ State_.Expect(TEncoderState::EState::ROOT);
+ CommonTime_ = time;
+ if (time != TInstant::Zero()) {
+ Out_->Write(TStringBuf("common time: "));
+ WriteTime(time);
+ Out_->Write('\n');
+ }
+ }
+
+ void OnMetricBegin(EMetricType type) override {
+ State_.Switch(TEncoderState::EState::ROOT, TEncoderState::EState::METRIC);
+ ClearLastMetricState();
+ MetricType_ = type;
+ }
+
+ void OnMetricEnd() override {
+ State_.Switch(TEncoderState::EState::METRIC, TEncoderState::EState::ROOT);
+ WriteMetric();
+ }
+
+ void OnLabelsBegin() override {
+ if (State_ == TEncoderState::EState::METRIC) {
+ State_ = TEncoderState::EState::METRIC_LABELS;
+ } else if (State_ == TEncoderState::EState::ROOT) {
+ State_ = TEncoderState::EState::COMMON_LABELS;
+ } else {
+ State_.ThrowInvalid("expected METRIC or ROOT");
+ }
+ }
+
+ void OnLabelsEnd() override {
+ if (State_ == TEncoderState::EState::METRIC_LABELS) {
+ State_ = TEncoderState::EState::METRIC;
+ } else if (State_ == TEncoderState::EState::COMMON_LABELS) {
+ State_ = TEncoderState::EState::ROOT;
+ Out_->Write(TStringBuf("common labels: "));
+ WriteLabels();
+ Out_->Write('\n');
+ } else {
+ State_.ThrowInvalid("expected LABELS or COMMON_LABELS");
+ }
+ }
+
+ void OnLabel(TStringBuf name, TStringBuf value) override {
+ Labels_.Add(name, value);
+ }
+
+ void OnDouble(TInstant time, double value) override {
+ State_.Expect(TEncoderState::EState::METRIC);
+ TimeSeries_.Add(time, value);
+ }
+
+ void OnInt64(TInstant time, i64 value) override {
+ State_.Expect(TEncoderState::EState::METRIC);
+ TimeSeries_.Add(time, value);
+ }
+
+ void OnUint64(TInstant time, ui64 value) override {
+ State_.Expect(TEncoderState::EState::METRIC);
+ TimeSeries_.Add(time, value);
+ }
+
+ void OnHistogram(TInstant time, IHistogramSnapshotPtr snapshot) override {
+ State_.Expect(TEncoderState::EState::METRIC);
+ TimeSeries_.Add(time, snapshot.Get());
+ }
+
+ void OnSummaryDouble(TInstant time, ISummaryDoubleSnapshotPtr snapshot) override {
+ State_.Expect(TEncoderState::EState::METRIC);
+ TimeSeries_.Add(time, snapshot.Get());
+ }
+
+ void OnLogHistogram(TInstant ts, TLogHistogramSnapshotPtr snapshot) override {
+ State_.Expect(TEncoderState::EState::METRIC);
+ TimeSeries_.Add(ts, snapshot.Get());
+ }
+
+ void Close() override {
+ }
+
+ void WriteTime(TInstant time) {
+ if (HumanReadableTs_) {
+ char buf[64];
+ auto len = FormatDate8601(buf, sizeof(buf), time.TimeT());
+ Out_->Write(buf, len);
+ } else {
+ (*Out_) << time.Seconds();
+ }
+ }
+
+ void WriteValue(EMetricValueType type, TMetricValue value) {
+ switch (type) {
+ case EMetricValueType::DOUBLE:
+ (*Out_) << value.AsDouble();
+ break;
+ case EMetricValueType::INT64:
+ (*Out_) << value.AsInt64();
+ break;
+ case EMetricValueType::UINT64:
+ (*Out_) << value.AsUint64();
+ break;
+ case EMetricValueType::HISTOGRAM:
+ (*Out_) << *value.AsHistogram();
+ break;
+ case EMetricValueType::SUMMARY:
+ (*Out_) << *value.AsSummaryDouble();
+ break;
+ case EMetricValueType::LOGHISTOGRAM:
+ (*Out_) << *value.AsLogHistogram();
+ break;
+ case EMetricValueType::UNKNOWN:
+ ythrow yexception() << "unknown metric value type";
+ }
+ }
+
+ void WriteLabels() {
+ auto& out = *Out_;
+ const auto size = Labels_.Size();
+ size_t i = 0;
+
+ out << '{';
+ for (auto&& l : Labels_) {
+ out << l.Name() << TStringBuf("='") << l.Value() << '\'';
+
+ ++i;
+ if (i < size) {
+ out << TStringBuf(", ");
+ }
+ };
+
+ out << '}';
+ }
+
+ void WriteMetric() {
+ // (1) type
+ TStringBuf typeStr = MetricTypeToStr(MetricType_);
+ (*Out_) << LeftPad(typeStr, MaxMetricTypeNameLength) << ' ';
+
+ // (2) name and labels
+ auto name = Labels_.Extract(TStringBuf("sensor"));
+ if (name) {
+ if (name->Value().find(' ') != TString::npos) {
+ (*Out_) << '"' << name->Value() << '"';
+ } else {
+ (*Out_) << name->Value();
+ }
+ }
+ WriteLabels();
+
+ // (3) values
+ if (!TimeSeries_.Empty()) {
+ TimeSeries_.SortByTs();
+ Out_->Write(TStringBuf(" ["));
+ for (size_t i = 0; i < TimeSeries_.Size(); i++) {
+ if (i > 0) {
+ Out_->Write(TStringBuf(", "));
+ }
+
+ const auto& point = TimeSeries_[i];
+ if (point.GetTime() == CommonTime_ || point.GetTime() == TInstant::Zero()) {
+ WriteValue(TimeSeries_.GetValueType(), point.GetValue());
+ } else {
+ Out_->Write('(');
+ WriteTime(point.GetTime());
+ Out_->Write(TStringBuf(", "));
+ WriteValue(TimeSeries_.GetValueType(), point.GetValue());
+ Out_->Write(')');
+ }
+ }
+ Out_->Write(']');
+ }
+ Out_->Write('\n');
+ }
+
+ void ClearLastMetricState() {
+ MetricType_ = EMetricType::UNKNOWN;
+ Labels_.Clear();
+ TimeSeries_.Clear();
+ }
+
+ private:
+ TEncoderState State_;
+ IOutputStream* Out_;
+ bool HumanReadableTs_;
+ TInstant CommonTime_ = TInstant::Zero();
+ EMetricType MetricType_ = EMetricType::UNKNOWN;
+ TLabels Labels_;
+ TMetricTimeSeries TimeSeries_;
+ };
+
+ }
+
+ IMetricEncoderPtr EncoderText(IOutputStream* out, bool humanReadableTs) {
+ return MakeHolder<TEncoderText>(out, humanReadableTs);
+ }
+
+}
diff --git a/library/cpp/monlib/encode/text/text_encoder_ut.cpp b/library/cpp/monlib/encode/text/text_encoder_ut.cpp
new file mode 100644
index 0000000000..554b6f5fa9
--- /dev/null
+++ b/library/cpp/monlib/encode/text/text_encoder_ut.cpp
@@ -0,0 +1,283 @@
+#include "text.h"
+
+#include <library/cpp/monlib/metrics/histogram_collector.h>
+
+#include <library/cpp/testing/unittest/registar.h>
+
+using namespace NMonitoring;
+
+Y_UNIT_TEST_SUITE(TTextText) {
+ template <typename TFunc>
+ TString EncodeToString(bool humanReadableTs, TFunc fn) {
+ TStringStream ss;
+ IMetricEncoderPtr encoder = EncoderText(&ss, humanReadableTs);
+ fn(encoder.Get());
+ return ss.Str();
+ }
+
+ Y_UNIT_TEST(Empty) {
+ auto result = EncodeToString(true, [](IMetricEncoder* e) {
+ e->OnStreamBegin();
+ e->OnStreamEnd();
+ });
+ UNIT_ASSERT_STRINGS_EQUAL(result, "");
+ }
+
+ Y_UNIT_TEST(CommonPart) {
+ auto result = EncodeToString(true, [](IMetricEncoder* e) {
+ e->OnStreamBegin();
+ e->OnCommonTime(TInstant::ParseIso8601Deprecated("2017-01-02T03:04:05.006Z"));
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("project", "solomon");
+ e->OnLabel("cluster", "man");
+ e->OnLabel("service", "stockpile");
+ e->OnLabelsEnd();
+ }
+ e->OnStreamEnd();
+ });
+ UNIT_ASSERT_STRINGS_EQUAL(result,
+ "common time: 2017-01-02T03:04:05Z\n"
+ "common labels: {project='solomon', cluster='man', service='stockpile'}\n");
+ }
+
+ Y_UNIT_TEST(Gauges) {
+ auto result = EncodeToString(true, [](IMetricEncoder* e) {
+ e->OnStreamBegin();
+ { // no values
+ e->OnMetricBegin(EMetricType::GAUGE);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "cpuUsage");
+ e->OnLabelsEnd();
+ }
+ e->OnMetricEnd();
+ }
+ { // one value no ts
+ e->OnMetricBegin(EMetricType::GAUGE);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "diskUsage");
+ e->OnLabel("disk", "sda1");
+ e->OnLabelsEnd();
+ }
+ e->OnDouble(TInstant::Zero(), 1000);
+ e->OnMetricEnd();
+ }
+ { // one value with ts
+ e->OnMetricBegin(EMetricType::GAUGE);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "memoryUsage");
+ e->OnLabel("host", "solomon-man-00");
+ e->OnLabel("dc", "man");
+ e->OnLabelsEnd();
+ }
+ e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 1000);
+ e->OnMetricEnd();
+ }
+ { // many values
+ e->OnMetricBegin(EMetricType::GAUGE);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "bytesRx");
+ e->OnLabel("host", "solomon-sas-01");
+ e->OnLabel("dc", "sas");
+ e->OnLabelsEnd();
+ }
+ e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 2);
+ e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:05Z"), 4);
+ e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:10Z"), 8);
+ e->OnMetricEnd();
+ }
+ e->OnStreamEnd();
+ });
+ UNIT_ASSERT_STRINGS_EQUAL(result,
+ " GAUGE cpuUsage{}\n"
+ " GAUGE diskUsage{disk='sda1'} [1000]\n"
+ " GAUGE memoryUsage{host='solomon-man-00', dc='man'} [(2017-12-02T12:00:00Z, 1000)]\n"
+ " GAUGE bytesRx{host='solomon-sas-01', dc='sas'} [(2017-12-02T12:00:00Z, 2), (2017-12-02T12:00:05Z, 4), (2017-12-02T12:00:10Z, 8)]\n");
+ }
+
+ Y_UNIT_TEST(IntGauges) {
+ auto result = EncodeToString(true, [](IMetricEncoder* e) {
+ e->OnStreamBegin();
+ { // no values
+ e->OnMetricBegin(EMetricType::IGAUGE);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "cpuUsage");
+ e->OnLabelsEnd();
+ }
+ e->OnMetricEnd();
+ }
+ { // one value no ts
+ e->OnMetricBegin(EMetricType::IGAUGE);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "diskUsage");
+ e->OnLabel("disk", "sda1");
+ e->OnLabelsEnd();
+ }
+ e->OnDouble(TInstant::Zero(), 1000);
+ e->OnMetricEnd();
+ }
+ { // one value with ts
+ e->OnMetricBegin(EMetricType::IGAUGE);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "memoryUsage");
+ e->OnLabel("host", "solomon-man-00");
+ e->OnLabel("dc", "man");
+ e->OnLabelsEnd();
+ }
+ e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 1000);
+ e->OnMetricEnd();
+ }
+ { // many values
+ e->OnMetricBegin(EMetricType::IGAUGE);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "bytesRx");
+ e->OnLabel("host", "solomon-sas-01");
+ e->OnLabel("dc", "sas");
+ e->OnLabelsEnd();
+ }
+ e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 2);
+ e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:05Z"), 4);
+ e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:10Z"), 8);
+ e->OnMetricEnd();
+ }
+ e->OnStreamEnd();
+ });
+ UNIT_ASSERT_STRINGS_EQUAL(result,
+ " IGAUGE cpuUsage{}\n"
+ " IGAUGE diskUsage{disk='sda1'} [1000]\n"
+ " IGAUGE memoryUsage{host='solomon-man-00', dc='man'} [(2017-12-02T12:00:00Z, 1000)]\n"
+ " IGAUGE bytesRx{host='solomon-sas-01', dc='sas'} [(2017-12-02T12:00:00Z, 2), (2017-12-02T12:00:05Z, 4), (2017-12-02T12:00:10Z, 8)]\n");
+ }
+
+ Y_UNIT_TEST(Counters) {
+ auto doEncode = [](IMetricEncoder* e) {
+ e->OnStreamBegin();
+ { // no values
+ e->OnMetricBegin(EMetricType::COUNTER);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "cpuUsage");
+ e->OnLabelsEnd();
+ }
+ e->OnMetricEnd();
+ }
+ { // one value no ts
+ e->OnMetricBegin(EMetricType::COUNTER);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "diskUsage");
+ e->OnLabel("disk", "sda1");
+ e->OnLabelsEnd();
+ }
+ e->OnUint64(TInstant::Zero(), 1000);
+ e->OnMetricEnd();
+ }
+ { // one value with ts
+ e->OnMetricBegin(EMetricType::COUNTER);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "memoryUsage");
+ e->OnLabel("host", "solomon-man-00");
+ e->OnLabel("dc", "man");
+ e->OnLabelsEnd();
+ }
+ e->OnUint64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 1000);
+ e->OnMetricEnd();
+ }
+ { // many values
+ e->OnMetricBegin(EMetricType::COUNTER);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "bytesRx");
+ e->OnLabel("host", "solomon-sas-01");
+ e->OnLabel("dc", "sas");
+ e->OnLabelsEnd();
+ }
+ e->OnUint64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 2);
+ e->OnUint64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:05Z"), 4);
+ e->OnUint64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:10Z"), 8);
+ e->OnMetricEnd();
+ }
+ e->OnStreamEnd();
+ };
+
+ auto result1 = EncodeToString(false, doEncode);
+ UNIT_ASSERT_STRINGS_EQUAL(result1,
+ " COUNTER cpuUsage{}\n"
+ " COUNTER diskUsage{disk='sda1'} [1000]\n"
+ " COUNTER memoryUsage{host='solomon-man-00', dc='man'} [(1512216000, 1000)]\n"
+ " COUNTER bytesRx{host='solomon-sas-01', dc='sas'} [(1512216000, 2), (1512216005, 4), (1512216010, 8)]\n");
+
+ auto result2 = EncodeToString(true, doEncode);
+ UNIT_ASSERT_STRINGS_EQUAL(result2,
+ " COUNTER cpuUsage{}\n"
+ " COUNTER diskUsage{disk='sda1'} [1000]\n"
+ " COUNTER memoryUsage{host='solomon-man-00', dc='man'} [(2017-12-02T12:00:00Z, 1000)]\n"
+ " COUNTER bytesRx{host='solomon-sas-01', dc='sas'} [(2017-12-02T12:00:00Z, 2), (2017-12-02T12:00:05Z, 4), (2017-12-02T12:00:10Z, 8)]\n");
+ }
+
+ Y_UNIT_TEST(Histograms) {
+ auto h = ExplicitHistogram({1, 2, 3, 4, 5});
+ h->Collect(3);
+ h->Collect(5, 7);
+ h->Collect(13);
+ auto s = h->Snapshot();
+
+ TString result = EncodeToString(true, [s](IMetricEncoder* e) {
+ e->OnStreamBegin();
+ {
+ e->OnMetricBegin(EMetricType::HIST);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "readTimeMillis");
+ e->OnLabelsEnd();
+ }
+ e->OnHistogram(TInstant::Zero(), s);
+ e->OnMetricEnd();
+ }
+ {
+ e->OnMetricBegin(EMetricType::HIST_RATE);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "writeTimeMillis");
+ e->OnLabelsEnd();
+ }
+ e->OnHistogram(TInstant::Zero(), s);
+ e->OnMetricEnd();
+ }
+ e->OnStreamEnd();
+ });
+
+ UNIT_ASSERT_STRINGS_EQUAL(result,
+ " HIST readTimeMillis{} [{1: 0, 2: 0, 3: 1, 4: 0, 5: 7, inf: 1}]\n"
+ "HIST_RATE writeTimeMillis{} [{1: 0, 2: 0, 3: 1, 4: 0, 5: 7, inf: 1}]\n");
+ }
+
+ Y_UNIT_TEST(Summary) {
+ auto s = MakeIntrusive<TSummaryDoubleSnapshot>(10.1, -0.45, 0.478, 0.3, 30u);
+ TString result = EncodeToString(true, [s](IMetricEncoder* e) {
+ e->OnStreamBegin();
+ {
+ e->OnMetricBegin(EMetricType::DSUMMARY);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "temperature");
+ e->OnLabelsEnd();
+ }
+ e->OnSummaryDouble(TInstant::Zero(), s);
+ e->OnMetricEnd();
+ }
+ e->OnStreamEnd();
+ });
+ UNIT_ASSERT_STRINGS_EQUAL(result,
+ " DSUMMARY temperature{} [{sum: 10.1, min: -0.45, max: 0.478, last: 0.3, count: 30}]\n");
+ }
+}
diff --git a/library/cpp/monlib/encode/text/ut/ya.make b/library/cpp/monlib/encode/text/ut/ya.make
new file mode 100644
index 0000000000..df23a252d1
--- /dev/null
+++ b/library/cpp/monlib/encode/text/ut/ya.make
@@ -0,0 +1,12 @@
+UNITTEST_FOR(library/cpp/monlib/encode/text)
+
+OWNER(
+ g:solomon
+ jamel
+)
+
+SRCS(
+ text_encoder_ut.cpp
+)
+
+END()
diff --git a/library/cpp/monlib/encode/text/ya.make b/library/cpp/monlib/encode/text/ya.make
new file mode 100644
index 0000000000..d296c78c1b
--- /dev/null
+++ b/library/cpp/monlib/encode/text/ya.make
@@ -0,0 +1,16 @@
+LIBRARY()
+
+OWNER(
+ g:solomon
+ jamel
+)
+
+SRCS(
+ text_encoder.cpp
+)
+
+PEERDIR(
+ library/cpp/monlib/encode
+)
+
+END()
diff --git a/library/cpp/monlib/encode/unistat/unistat.h b/library/cpp/monlib/encode/unistat/unistat.h
new file mode 100644
index 0000000000..1c43b7fa1b
--- /dev/null
+++ b/library/cpp/monlib/encode/unistat/unistat.h
@@ -0,0 +1,13 @@
+#pragma once
+
+#include <util/generic/fwd.h>
+#include <util/datetime/base.h>
+
+namespace NMonitoring {
+ /// Decodes unistat-style metrics
+ /// https://wiki.yandex-team.ru/golovan/stat-handle
+ void DecodeUnistat(TStringBuf data, class IMetricConsumer* c, TInstant ts = TInstant::Zero());
+
+ /// Assumes consumer's stream is open by the caller
+ void DecodeUnistatToStream(TStringBuf data, class IMetricConsumer* c, TInstant = TInstant::Zero());
+}
diff --git a/library/cpp/monlib/encode/unistat/unistat_decoder.cpp b/library/cpp/monlib/encode/unistat/unistat_decoder.cpp
new file mode 100644
index 0000000000..b2344b0905
--- /dev/null
+++ b/library/cpp/monlib/encode/unistat/unistat_decoder.cpp
@@ -0,0 +1,253 @@
+#include "unistat.h"
+
+#include <library/cpp/monlib/metrics/histogram_collector.h>
+#include <library/cpp/monlib/metrics/labels.h>
+#include <library/cpp/monlib/metrics/metric_type.h>
+#include <library/cpp/monlib/metrics/metric_value.h>
+#include <library/cpp/monlib/metrics/metric_consumer.h>
+
+#include <library/cpp/json/json_reader.h>
+
+#include <util/datetime/base.h>
+#include <util/string/split.h>
+
+#include <contrib/libs/re2/re2/re2.h>
+
+using namespace NJson;
+
+const re2::RE2 NAME_RE{R"((?:[a-zA-Z0-9\.\-/@_]+_)+(?:[ad][vehmntx]{3}|summ|hgram|max))"};
+
+namespace NMonitoring {
+ namespace {
+ bool IsNumber(const NJson::TJsonValue& j) {
+ switch (j.GetType()) {
+ case EJsonValueType::JSON_INTEGER:
+ case EJsonValueType::JSON_UINTEGER:
+ case EJsonValueType::JSON_DOUBLE:
+ return true;
+
+ default:
+ return false;
+ }
+ }
+
+ template <typename T>
+ T ExtractNumber(const TJsonValue& val) {
+ switch (val.GetType()) {
+ case EJsonValueType::JSON_INTEGER:
+ return static_cast<T>(val.GetInteger());
+ case EJsonValueType::JSON_UINTEGER:
+ return static_cast<T>(val.GetUInteger());
+ case EJsonValueType::JSON_DOUBLE:
+ return static_cast<T>(val.GetDouble());
+
+ default:
+ ythrow yexception() << "Expected number, but found " << val.GetType();
+ }
+ }
+
+ auto ExtractDouble = ExtractNumber<double>;
+ auto ExtractUi64 = ExtractNumber<ui64>;
+
+ class THistogramBuilder {
+ public:
+ void Add(TBucketBound bound, TBucketValue value) {
+ /// XXX: yasm uses left-closed intervals, while in monlib we use right-closed ones,
+ /// so (-inf; 0) [0, 100) [100; +inf)
+ /// becomes (-inf; 0] (0, 100] (100; +inf)
+ /// but since we've already lost some information these no way to avoid this kind of error here
+ Bounds_.push_back(bound);
+
+ /// this will always be 0 for the first bucket,
+ /// since there's no way to make (-inf; N) bucket in yasm
+ Values_.push_back(NextValue_);
+
+ /// we will write this value into the next bucket so that [[0, 10], [100, 20], [200, 50]]
+ /// becomes (-inf; 0] -> 0; (0; 100] -> 10; (100; 200] -> 20; (200; +inf) -> 50
+ NextValue_ = value;
+ }
+
+ IHistogramSnapshotPtr Finalize() {
+ Bounds_.push_back(std::numeric_limits<TBucketBound>::max());
+ Values_.push_back(NextValue_);
+
+ Y_ENSURE(Bounds_.size() <= HISTOGRAM_MAX_BUCKETS_COUNT,
+ "Histogram is only allowed to have " << HISTOGRAM_MAX_BUCKETS_COUNT << " buckets, but has " << Bounds_.size());
+
+ return ExplicitHistogramSnapshot(Bounds_, Values_);
+ }
+
+ public:
+ TBucketValue NextValue_ {0};
+ TBucketBounds Bounds_;
+ TBucketValues Values_;
+ };
+
+ class TDecoderUnistat {
+ private:
+ public:
+ explicit TDecoderUnistat(IMetricConsumer* consumer, IInputStream* is, TInstant ts)
+ : Consumer_{consumer}
+ , Timestamp_{ts} {
+ ReadJsonTree(is, &Json_, /* throw */ true);
+ }
+
+ void Decode() {
+ Y_ENSURE(Json_.IsArray(), "Expected array at the top level, but found " << Json_.GetType());
+
+ for (auto&& metric : Json_.GetArray()) {
+ Y_ENSURE(metric.IsArray(), "Metric must be an array");
+ auto&& arr = metric.GetArray();
+ Y_ENSURE(arr.size() == 2, "Metric must be an array of 2 elements");
+ auto&& name = arr[0];
+ auto&& value = arr[1];
+ MetricContext_ = {};
+
+ ParseName(name.GetString());
+
+ if (value.IsArray()) {
+ OnHistogram(value);
+ } else if (IsNumber(value)) {
+ OnScalar(value);
+ } else {
+ ythrow yexception() << "Expected list or number, but found " << value.GetType();
+ }
+
+ WriteValue();
+ }
+ }
+
+ private:
+ void OnScalar(const TJsonValue& jsonValue) {
+ if (MetricContext_.IsDeriv) {
+ MetricContext_.Type = EMetricType::RATE;
+ MetricContext_.Value = TMetricValue{ExtractUi64(jsonValue)};
+ } else {
+ MetricContext_.Type = EMetricType::GAUGE;
+ MetricContext_.Value = TMetricValue{ExtractDouble(jsonValue)};
+ }
+ }
+
+ void OnHistogram(const TJsonValue& jsonHist) {
+ if (MetricContext_.IsDeriv) {
+ MetricContext_.Type = EMetricType::HIST_RATE;
+ } else {
+ MetricContext_.Type = EMetricType::HIST;
+ }
+
+ auto histogramBuilder = THistogramBuilder();
+
+ for (auto&& bucket : jsonHist.GetArray()) {
+ Y_ENSURE(bucket.IsArray(), "Expected an array, but found " << bucket.GetType());
+ auto&& arr = bucket.GetArray();
+ Y_ENSURE(arr.size() == 2, "Histogram bucket must be an array of 2 elements");
+ const auto bound = ExtractDouble(arr[0]);
+ const auto weight = ExtractUi64(arr[1]);
+ histogramBuilder.Add(bound, weight);
+ }
+
+ MetricContext_.Histogram = histogramBuilder.Finalize();
+ MetricContext_.Value = TMetricValue{MetricContext_.Histogram.Get()};
+ }
+
+ bool IsDeriv(TStringBuf name) {
+ TStringBuf ignore, suffix;
+ name.RSplit('_', ignore, suffix);
+
+ Y_ENSURE(suffix.size() >= 3 && suffix.size() <= 5, "Disallowed suffix value: " << suffix);
+
+ if (suffix == TStringBuf("summ") || suffix == TStringBuf("hgram")) {
+ return true;
+ } else if (suffix == TStringBuf("max")) {
+ return false;
+ }
+
+ return suffix[0] == 'd';
+ }
+
+ void ParseName(TStringBuf value) {
+ TVector<TStringBuf> parts;
+ StringSplitter(value).Split(';').SkipEmpty().Collect(&parts);
+
+ Y_ENSURE(parts.size() >= 1 && parts.size() <= 16);
+
+ TStringBuf name = parts.back();
+ parts.pop_back();
+
+ Y_ENSURE(RE2::FullMatch(re2::StringPiece{name.data(), name.size()}, NAME_RE),
+ "Metric name " << name << " doesn't match regex " << NAME_RE.pattern());
+
+ MetricContext_.Name = name;
+ MetricContext_.IsDeriv = IsDeriv(MetricContext_.Name);
+
+ for (auto tag : parts) {
+ TStringBuf n, v;
+ tag.Split('=', n, v);
+ Y_ENSURE(n && v, "Unexpected tag format in " << tag);
+ MetricContext_.Labels.Add(n, v);
+ }
+ }
+
+ private:
+ void WriteValue() {
+ Consumer_->OnMetricBegin(MetricContext_.Type);
+
+ Consumer_->OnLabelsBegin();
+ Consumer_->OnLabel("sensor", TString{MetricContext_.Name});
+ for (auto&& l : MetricContext_.Labels) {
+ Consumer_->OnLabel(l.Name(), l.Value());
+ }
+
+ Consumer_->OnLabelsEnd();
+
+ switch (MetricContext_.Type) {
+ case EMetricType::GAUGE:
+ Consumer_->OnDouble(Timestamp_, MetricContext_.Value.AsDouble());
+ break;
+ case EMetricType::RATE:
+ Consumer_->OnUint64(Timestamp_, MetricContext_.Value.AsUint64());
+ break;
+ case EMetricType::HIST:
+ case EMetricType::HIST_RATE:
+ Consumer_->OnHistogram(Timestamp_, MetricContext_.Value.AsHistogram());
+ break;
+ case EMetricType::LOGHIST:
+ case EMetricType::DSUMMARY:
+ case EMetricType::IGAUGE:
+ case EMetricType::COUNTER:
+ case EMetricType::UNKNOWN:
+ ythrow yexception() << "Unexpected metric type: " << MetricContext_.Type;
+ }
+
+ Consumer_->OnMetricEnd();
+ }
+
+ private:
+ IMetricConsumer* Consumer_;
+ NJson::TJsonValue Json_;
+ TInstant Timestamp_;
+
+ struct {
+ TStringBuf Name;
+ EMetricType Type{EMetricType::UNKNOWN};
+ TMetricValue Value;
+ bool IsDeriv{false};
+ TLabels Labels;
+ IHistogramSnapshotPtr Histogram;
+ } MetricContext_;
+ };
+
+ }
+
+ void DecodeUnistat(TStringBuf data, IMetricConsumer* c, TInstant ts) {
+ c->OnStreamBegin();
+ DecodeUnistatToStream(data, c, ts);
+ c->OnStreamEnd();
+ }
+
+ void DecodeUnistatToStream(TStringBuf data, IMetricConsumer* c, TInstant ts) {
+ TMemoryInput in{data.data(), data.size()};
+ TDecoderUnistat decoder(c, &in, ts);
+ decoder.Decode();
+ }
+}
diff --git a/library/cpp/monlib/encode/unistat/unistat_ut.cpp b/library/cpp/monlib/encode/unistat/unistat_ut.cpp
new file mode 100644
index 0000000000..dbbc238bf3
--- /dev/null
+++ b/library/cpp/monlib/encode/unistat/unistat_ut.cpp
@@ -0,0 +1,223 @@
+#include "unistat.h"
+
+#include <library/cpp/monlib/encode/protobuf/protobuf.h>
+#include <library/cpp/monlib/metrics/labels.h>
+
+#include <library/cpp/testing/unittest/registar.h>
+
+using namespace NMonitoring;
+
+Y_UNIT_TEST_SUITE(TUnistatDecoderTest) {
+ Y_UNIT_TEST(ScalarMetric) {
+ constexpr auto input = TStringBuf(R"([["something_axxx", 42]])");
+
+ NProto::TMultiSamplesList samples;
+ auto encoder = EncoderProtobuf(&samples);
+
+ DecodeUnistat(input, encoder.Get());
+
+ UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 1);
+ auto sample = samples.GetSamples(0);
+ UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::GAUGE);
+ UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1);
+
+ auto label = sample.GetLabels(0);
+ auto point = sample.GetPoints(0);
+ UNIT_ASSERT_VALUES_EQUAL(point.GetFloat64(), 42.);
+ UNIT_ASSERT_VALUES_EQUAL(label.GetName(), "sensor");
+ UNIT_ASSERT_VALUES_EQUAL(label.GetValue(), "something_axxx");
+ }
+
+ Y_UNIT_TEST(OverriddenTags) {
+ constexpr auto input = TStringBuf(R"([["ctype=foo;prj=bar;custom_tag=qwe;something_axxx", 42]])");
+
+ NProto::TMultiSamplesList samples;
+ auto encoder = EncoderProtobuf(&samples);
+
+ DecodeUnistat(input, encoder.Get());
+
+ UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 1);
+ auto sample = samples.GetSamples(0);
+ UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 4);
+
+ const auto& labels = sample.GetLabels();
+ TLabels actual;
+ for (auto&& l : labels) {
+ actual.Add(l.GetName(), l.GetValue());
+ }
+
+ TLabels expected{{"ctype", "foo"}, {"prj", "bar"}, {"custom_tag", "qwe"}, {"sensor", "something_axxx"}};
+
+ UNIT_ASSERT_VALUES_EQUAL(actual.size(), expected.size());
+ for (auto&& l : actual) {
+ UNIT_ASSERT(expected.Extract(l.Name())->Value() == l.Value());
+ }
+ }
+
+ Y_UNIT_TEST(ThrowsOnTopLevelObject) {
+ constexpr auto input = TStringBuf(R"({["something_axxx", 42]})");
+
+ NProto::TMultiSamplesList samples;
+ auto encoder = EncoderProtobuf(&samples);
+
+ UNIT_ASSERT_EXCEPTION(DecodeUnistat(input, encoder.Get()), yexception);
+ }
+
+ Y_UNIT_TEST(ThrowsOnUnwrappedMetric) {
+ constexpr auto input = TStringBuf(R"(["something_axxx", 42])");
+
+ NProto::TMultiSamplesList samples;
+ auto encoder = EncoderProtobuf(&samples);
+
+ UNIT_ASSERT_EXCEPTION(DecodeUnistat(input, encoder.Get()), yexception);
+ }
+
+ Y_UNIT_TEST(HistogramMetric) {
+ constexpr auto input = TStringBuf(R"([["something_hgram", [[0, 1], [200, 2], [500, 3]] ]])");
+
+ NProto::TMultiSamplesList samples;
+ auto encoder = EncoderProtobuf(&samples);
+
+ DecodeUnistat(input, encoder.Get());
+
+ auto sample = samples.GetSamples(0);
+ UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::HIST_RATE);
+ UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1);
+
+ auto label = sample.GetLabels(0);
+ const auto point = sample.GetPoints(0);
+ const auto histogram = point.GetHistogram();
+ const auto size = histogram.BoundsSize();
+ UNIT_ASSERT_VALUES_EQUAL(size, 4);
+
+ const TVector<double> expectedBounds {0, 200, 500, std::numeric_limits<double>::max()};
+ const TVector<ui64> expectedValues {0, 1, 2, 3};
+
+ for (auto i = 0; i < 4; ++i) {
+ UNIT_ASSERT_VALUES_EQUAL(histogram.GetBounds(i), expectedBounds[i]);
+ UNIT_ASSERT_VALUES_EQUAL(histogram.GetValues(i), expectedValues[i]);
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(label.GetName(), "sensor");
+ UNIT_ASSERT_VALUES_EQUAL(label.GetValue(), "something_hgram");
+ }
+
+ Y_UNIT_TEST(AbsoluteHistogram) {
+ constexpr auto input = TStringBuf(R"([["something_ahhh", [[0, 1], [200, 2], [500, 3]] ]])");
+
+ NProto::TMultiSamplesList samples;
+ auto encoder = EncoderProtobuf(&samples);
+
+ DecodeUnistat(input, encoder.Get());
+
+ auto sample = samples.GetSamples(0);
+ UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::HISTOGRAM);
+ UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1);
+ }
+
+ Y_UNIT_TEST(AllowedMetricNames) {
+ NProto::TMultiSamplesList samples;
+ auto encoder = EncoderProtobuf(&samples);
+
+ {
+ constexpr auto input = TStringBuf(R"([["a/A-b/c_D/__G_dmmm", [[0, 1], [200, 2], [500, 3]] ]])");
+ UNIT_ASSERT_NO_EXCEPTION(DecodeUnistat(input, encoder.Get()));
+ }
+ }
+
+ Y_UNIT_TEST(DisallowedMetricNames) {
+ NProto::TMultiSamplesList samples;
+ auto encoder = EncoderProtobuf(&samples);
+
+ {
+ constexpr auto input = TStringBuf(R"([["someth!ng_ahhh", [[0, 1], [200, 2], [500, 3]] ]])");
+ UNIT_ASSERT_EXCEPTION(DecodeUnistat(input, encoder.Get()), yexception);
+ }
+
+ {
+ constexpr auto input = TStringBuf(R"([["foo_a", [[0, 1], [200, 2], [500, 3]] ]])");
+ UNIT_ASSERT_EXCEPTION(DecodeUnistat(input, encoder.Get()), yexception);
+ }
+
+ {
+ constexpr auto input = TStringBuf(R"([["foo_ahhh;tag=value", [[0, 1], [200, 2], [500, 3]] ]])");
+ UNIT_ASSERT_EXCEPTION(DecodeUnistat(input, encoder.Get()), yexception);
+ }
+ }
+
+ Y_UNIT_TEST(MultipleMetrics) {
+ constexpr auto input = TStringBuf(R"([["something_axxx", 42], ["some-other_dhhh", 53]])");
+
+ NProto::TMultiSamplesList samples;
+ auto encoder = EncoderProtobuf(&samples);
+
+ DecodeUnistat(input, encoder.Get());
+
+ UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 2);
+ auto sample = samples.GetSamples(0);
+ UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::GAUGE);
+ UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1);
+
+ auto label = sample.GetLabels(0);
+ auto point = sample.GetPoints(0);
+ UNIT_ASSERT_VALUES_EQUAL(point.GetFloat64(), 42.);
+ UNIT_ASSERT_VALUES_EQUAL(label.GetName(), "sensor");
+ UNIT_ASSERT_VALUES_EQUAL(label.GetValue(), "something_axxx");
+
+ sample = samples.GetSamples(1);
+ UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::RATE);
+ UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1);
+
+ label = sample.GetLabels(0);
+ point = sample.GetPoints(0);
+ UNIT_ASSERT_VALUES_EQUAL(point.GetUint64(), 53);
+ UNIT_ASSERT_VALUES_EQUAL(label.GetName(), "sensor");
+ UNIT_ASSERT_VALUES_EQUAL(label.GetValue(), "some-other_dhhh");
+ }
+
+ Y_UNIT_TEST(UnderscoreName) {
+ constexpr auto input = TStringBuf(R"([["something_anything_dmmm", 42]])");
+
+ NProto::TMultiSamplesList samples;
+ auto encoder = EncoderProtobuf(&samples);
+ DecodeUnistat(input, encoder.Get());
+
+ UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 1);
+ auto sample = samples.GetSamples(0);
+ UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::RATE);
+ UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1);
+
+ auto label = sample.GetLabels(0);
+ auto point = sample.GetPoints(0);
+ UNIT_ASSERT_VALUES_EQUAL(point.GetUint64(), 42);
+ UNIT_ASSERT_VALUES_EQUAL(label.GetName(), "sensor");
+ UNIT_ASSERT_VALUES_EQUAL(label.GetValue(), "something_anything_dmmm");
+ }
+
+ Y_UNIT_TEST(MaxAggr) {
+ constexpr auto input = TStringBuf(R"([["something_anything_max", 42]])");
+
+ NProto::TMultiSamplesList samples;
+ auto encoder = EncoderProtobuf(&samples);
+ DecodeUnistat(input, encoder.Get());
+
+ UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 1);
+ auto sample = samples.GetSamples(0);
+ UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::GAUGE);
+ UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1);
+
+ auto label = sample.GetLabels(0);
+ auto point = sample.GetPoints(0);
+ UNIT_ASSERT_VALUES_EQUAL(point.GetFloat64(), 42.);
+ UNIT_ASSERT_VALUES_EQUAL(label.GetName(), "sensor");
+ UNIT_ASSERT_VALUES_EQUAL(label.GetValue(), "something_anything_max");
+ }
+}
diff --git a/library/cpp/monlib/encode/unistat/ut/ya.make b/library/cpp/monlib/encode/unistat/ut/ya.make
new file mode 100644
index 0000000000..a652139f45
--- /dev/null
+++ b/library/cpp/monlib/encode/unistat/ut/ya.make
@@ -0,0 +1,16 @@
+UNITTEST_FOR(library/cpp/monlib/encode/unistat)
+
+OWNER(
+ msherbakov
+ g:solomon
+)
+
+SRCS(
+ unistat_ut.cpp
+)
+
+PEERDIR(
+ library/cpp/monlib/encode/protobuf
+)
+
+END()
diff --git a/library/cpp/monlib/encode/unistat/ya.make b/library/cpp/monlib/encode/unistat/ya.make
new file mode 100644
index 0000000000..4ac2edadf4
--- /dev/null
+++ b/library/cpp/monlib/encode/unistat/ya.make
@@ -0,0 +1,18 @@
+LIBRARY()
+
+OWNER(
+ msherbakov
+ g:solomon
+)
+
+PEERDIR(
+ contrib/libs/re2
+ library/cpp/json
+ library/cpp/monlib/metrics
+)
+
+SRCS(
+ unistat_decoder.cpp
+)
+
+END()
diff --git a/library/cpp/monlib/encode/ut/ya.make b/library/cpp/monlib/encode/ut/ya.make
new file mode 100644
index 0000000000..1990386d76
--- /dev/null
+++ b/library/cpp/monlib/encode/ut/ya.make
@@ -0,0 +1,12 @@
+UNITTEST_FOR(library/cpp/monlib/encode)
+
+OWNER(
+ jamel
+ g:solomon
+)
+
+SRCS(
+ format_ut.cpp
+)
+
+END()
diff --git a/library/cpp/monlib/encode/ya.make b/library/cpp/monlib/encode/ya.make
new file mode 100644
index 0000000000..d1bb09f07b
--- /dev/null
+++ b/library/cpp/monlib/encode/ya.make
@@ -0,0 +1,20 @@
+LIBRARY()
+
+OWNER(
+ g:solomon
+ jamel
+)
+
+SRCS(
+ encoder.cpp
+ encoder_state.cpp
+ format.cpp
+)
+
+PEERDIR(
+ library/cpp/monlib/metrics
+)
+
+GENERATE_ENUM_SERIALIZATION_WITH_HEADER(encoder_state_enum.h)
+
+END()