aboutsummaryrefslogtreecommitdiffstats
path: root/library/cpp/monlib/encode/prometheus
diff options
context:
space:
mode:
authorDevtools Arcadia <arcadia-devtools@yandex-team.ru>2022-02-07 18:08:42 +0300
committerDevtools Arcadia <arcadia-devtools@mous.vla.yp-c.yandex.net>2022-02-07 18:08:42 +0300
commit1110808a9d39d4b808aef724c861a2e1a38d2a69 (patch)
treee26c9fed0de5d9873cce7e00bc214573dc2195b7 /library/cpp/monlib/encode/prometheus
downloadydb-1110808a9d39d4b808aef724c861a2e1a38d2a69.tar.gz
intermediate changes
ref:cde9a383711a11544ce7e107a78147fb96cc4029
Diffstat (limited to 'library/cpp/monlib/encode/prometheus')
-rw-r--r--library/cpp/monlib/encode/prometheus/fuzz/main.cpp18
-rw-r--r--library/cpp/monlib/encode/prometheus/fuzz/ya.make16
-rw-r--r--library/cpp/monlib/encode/prometheus/prometheus.h18
-rw-r--r--library/cpp/monlib/encode/prometheus/prometheus_decoder.cpp597
-rw-r--r--library/cpp/monlib/encode/prometheus/prometheus_decoder_ut.cpp478
-rw-r--r--library/cpp/monlib/encode/prometheus/prometheus_encoder.cpp413
-rw-r--r--library/cpp/monlib/encode/prometheus/prometheus_encoder_ut.cpp414
-rw-r--r--library/cpp/monlib/encode/prometheus/prometheus_model.h70
-rw-r--r--library/cpp/monlib/encode/prometheus/ut/ya.make17
-rw-r--r--library/cpp/monlib/encode/prometheus/ya.make17
10 files changed, 2058 insertions, 0 deletions
diff --git a/library/cpp/monlib/encode/prometheus/fuzz/main.cpp b/library/cpp/monlib/encode/prometheus/fuzz/main.cpp
new file mode 100644
index 0000000000..24bda2d32e
--- /dev/null
+++ b/library/cpp/monlib/encode/prometheus/fuzz/main.cpp
@@ -0,0 +1,18 @@
+#include <library/cpp/monlib/encode/prometheus/prometheus.h>
+#include <library/cpp/monlib/encode/fake/fake.h>
+
+#include <util/stream/mem.h>
+
+
+extern "C" int LLVMFuzzerTestOneInput(const ui8* buf, size_t size) {
+ using namespace NMonitoring;
+
+ try {
+ TStringBuf data(reinterpret_cast<const char*>(buf), size);
+ auto encoder = EncoderFake();
+ DecodePrometheus(data, encoder.Get());
+ } catch (...) {
+ }
+
+ return 0;
+}
diff --git a/library/cpp/monlib/encode/prometheus/fuzz/ya.make b/library/cpp/monlib/encode/prometheus/fuzz/ya.make
new file mode 100644
index 0000000000..4a6c796ed5
--- /dev/null
+++ b/library/cpp/monlib/encode/prometheus/fuzz/ya.make
@@ -0,0 +1,16 @@
+FUZZ()
+
+OWNER(g:solomon jamel)
+
+PEERDIR(
+ library/cpp/monlib/encode/prometheus
+ library/cpp/monlib/encode/fake
+)
+
+SIZE(MEDIUM)
+
+SRCS(
+ main.cpp
+)
+
+END()
diff --git a/library/cpp/monlib/encode/prometheus/prometheus.h b/library/cpp/monlib/encode/prometheus/prometheus.h
new file mode 100644
index 0000000000..2e7fa31c28
--- /dev/null
+++ b/library/cpp/monlib/encode/prometheus/prometheus.h
@@ -0,0 +1,18 @@
+#pragma once
+
+#include <library/cpp/monlib/encode/encoder.h>
+#include <library/cpp/monlib/encode/format.h>
+
+#include <util/generic/yexception.h>
+
+
+namespace NMonitoring {
+
+ class TPrometheusDecodeException: public yexception {
+ };
+
+ IMetricEncoderPtr EncoderPrometheus(IOutputStream* out, TStringBuf metricNameLabel = "sensor");
+
+ void DecodePrometheus(TStringBuf data, IMetricConsumer* c, TStringBuf metricNameLabel = "sensor");
+
+}
diff --git a/library/cpp/monlib/encode/prometheus/prometheus_decoder.cpp b/library/cpp/monlib/encode/prometheus/prometheus_decoder.cpp
new file mode 100644
index 0000000000..7e81357dbd
--- /dev/null
+++ b/library/cpp/monlib/encode/prometheus/prometheus_decoder.cpp
@@ -0,0 +1,597 @@
+#include "prometheus.h"
+#include "prometheus_model.h"
+
+#include <library/cpp/monlib/metrics/histogram_snapshot.h>
+#include <library/cpp/monlib/metrics/metric.h>
+
+#include <util/datetime/base.h>
+#include <util/generic/hash.h>
+#include <util/string/cast.h>
+#include <util/string/builder.h>
+#include <util/generic/maybe.h>
+#include <util/string/ascii.h>
+
+#include <cmath>
+
+#define Y_PARSER_FAIL(message) \
+ ythrow ::NMonitoring::TPrometheusDecodeException() << message << " at line #" << CurrentLine_
+
+#define Y_PARSER_ENSURE(cond, message) \
+ Y_ENSURE_EX(cond, ::NMonitoring::TPrometheusDecodeException() << message << " at line #" << CurrentLine_)
+
+
+namespace NMonitoring {
+ namespace {
+ constexpr ui32 MAX_LABEL_VALUE_LEN = 256;
+
+ using TLabelsMap = THashMap<TString, TString>;
+
+ TString LabelsToStr(const TLabelsMap& labels) {
+ TStringBuilder sb;
+ auto it = labels.begin();
+ auto end = labels.end();
+
+ sb << '{';
+ while (it != end) {
+ sb << it->first;
+ sb << '=';
+ sb << '"' << it->second << '"';
+
+ ++it;
+ if (it != end) {
+ sb << ", ";
+ }
+ }
+ sb << '}';
+ return sb;
+ }
+
+ template <typename T, typename U>
+ bool TryStaticCast(U val, T& out) {
+ static_assert(std::is_arithmetic_v<U>);
+ if constexpr (std::is_floating_point_v<T> || std::is_floating_point_v<U>) {
+ if (val > MaxFloor<T>() || val < -MaxFloor<T>()) {
+ return false;
+ }
+
+ } else {
+ if (val > Max<T>() || val < Min<T>()) {
+ return false;
+ }
+ }
+
+ out = static_cast<T>(val);
+ return true;
+ }
+
+ ///////////////////////////////////////////////////////////////////////
+ // THistogramBuilder
+ ///////////////////////////////////////////////////////////////////////
+ class THistogramBuilder {
+ using TBucketData = std::pair<TBucketBound, TBucketValue>;
+ constexpr static TBucketData ZERO_BUCKET = { -std::numeric_limits<TBucketBound>::max(), 0 };
+ public:
+ TStringBuf GetName() const noexcept {
+ return Name_;
+ }
+
+ void SetName(TStringBuf name) noexcept {
+ Name_ = name;
+ }
+
+ const TLabelsMap& GetLabels() const noexcept {
+ return *Labels_;
+ }
+
+ void SetLabels(TLabelsMap&& labels) {
+ if (Labels_.Defined()) {
+ Y_ENSURE(Labels_ == labels,
+ "mixed labels in one histogram, prev: " << LabelsToStr(*Labels_) <<
+ ", current: " << LabelsToStr(labels));
+ } else {
+ Labels_.ConstructInPlace(std::move(labels));
+ }
+ }
+
+ TInstant GetTime() const noexcept {
+ return Time_;
+ }
+
+ void SetTime(TInstant time) noexcept {
+ Time_ = time;
+ }
+
+ bool Empty() const noexcept {
+ return Bounds_.empty();
+ }
+
+ bool Same(TStringBuf name, const TLabelsMap& labels) const noexcept {
+ return Name_ == name && Labels_ == labels;
+ }
+
+ void AddBucket(TBucketBound bound, TBucketValue value) {
+ Y_ENSURE_EX(PrevBucket_.first < bound, TPrometheusDecodeException() <<
+ "invalid order of histogram bounds " << PrevBucket_.first <<
+ " >= " << bound);
+
+ Y_ENSURE_EX(PrevBucket_.second <= value, TPrometheusDecodeException() <<
+ "invalid order of histogram bucket values " << PrevBucket_.second <<
+ " > " << value);
+
+ // convert infinite bound value
+ if (bound == std::numeric_limits<TBucketBound>::infinity()) {
+ bound = HISTOGRAM_INF_BOUND;
+ }
+
+ Bounds_.push_back(bound);
+ Values_.push_back(value - PrevBucket_.second); // keep only delta between buckets
+
+ PrevBucket_ = { bound, value };
+ }
+
+ // will clear builder state
+ IHistogramSnapshotPtr ToSnapshot() {
+ Y_ENSURE_EX(!Empty(), TPrometheusDecodeException() << "histogram cannot be empty");
+ Time_ = TInstant::Zero();
+ PrevBucket_ = ZERO_BUCKET;
+ Labels_.Clear();
+ auto snapshot = ExplicitHistogramSnapshot(Bounds_, Values_);
+
+ Bounds_.clear();
+ Values_.clear();
+
+ return snapshot;
+ }
+
+ private:
+ TStringBuf Name_;
+ TMaybe<TLabelsMap> Labels_;
+ TInstant Time_;
+ TBucketBounds Bounds_;
+ TBucketValues Values_;
+ TBucketData PrevBucket_ = ZERO_BUCKET;
+ };
+
+ ///////////////////////////////////////////////////////////////////////
+ // EPrometheusMetricType
+ ///////////////////////////////////////////////////////////////////////
+ enum class EPrometheusMetricType {
+ GAUGE,
+ COUNTER,
+ SUMMARY,
+ UNTYPED,
+ HISTOGRAM,
+ };
+
+ ///////////////////////////////////////////////////////////////////////
+ // TPrometheusReader
+ ///////////////////////////////////////////////////////////////////////
+ class TPrometheusReader {
+ public:
+ TPrometheusReader(TStringBuf data, IMetricConsumer* c, TStringBuf metricNameLabel)
+ : Data_(data)
+ , Consumer_(c)
+ , MetricNameLabel_(metricNameLabel)
+ {
+ }
+
+ void Read() {
+ Consumer_->OnStreamBegin();
+
+ if (HasRemaining()) {
+ ReadNextByte();
+ SkipSpaces();
+
+ try {
+ while (HasRemaining()) {
+ switch (CurrentByte_) {
+ case '\n':
+ ReadNextByte(); // skip '\n'
+ CurrentLine_++;
+ SkipSpaces();
+ break;
+ case '#':
+ ParseComment();
+ break;
+ default:
+ ParseMetric();
+ break;
+ }
+ }
+
+ if (!HistogramBuilder_.Empty()) {
+ ConsumeHistogram();
+ }
+ } catch (const TPrometheusDecodeException& e) {
+ throw e;
+ } catch (...) {
+ Y_PARSER_FAIL("unexpected error " << CurrentExceptionMessage());
+ }
+ }
+
+ Consumer_->OnStreamEnd();
+ }
+
+ private:
+ bool HasRemaining() const noexcept {
+ return CurrentPos_ < Data_.Size();
+ }
+
+ // # 'TYPE' metric_name {counter|gauge|histogram|summary|untyped}
+ // # 'HELP' metric_name some help info
+ // # general comment message
+ void ParseComment() {
+ SkipExpectedChar('#');
+ SkipSpaces();
+
+ TStringBuf keyword = ReadToken();
+ if (keyword == TStringBuf("TYPE")) {
+ SkipSpaces();
+
+ TStringBuf nextName = ReadTokenAsMetricName();
+ Y_PARSER_ENSURE(!nextName.Empty(), "invalid metric name");
+
+ SkipSpaces();
+ EPrometheusMetricType nextType = ReadType();
+
+ bool inserted = SeenTypes_.emplace(nextName, nextType).second;
+ Y_PARSER_ENSURE(inserted, "second TYPE line for metric " << nextName);
+
+ if (nextType == EPrometheusMetricType::HISTOGRAM) {
+ if (!HistogramBuilder_.Empty()) {
+ ConsumeHistogram();
+ }
+ HistogramBuilder_.SetName(nextName);
+ }
+ } else {
+ // skip HELP and general comments
+ SkipUntilEol();
+ }
+
+ Y_PARSER_ENSURE(CurrentByte_ == '\n', "expected '\\n', found '" << CurrentByte_ << '\'');
+ }
+
+ // metric_name [labels] value [timestamp]
+ void ParseMetric() {
+ TStringBuf name = ReadTokenAsMetricName();
+ SkipSpaces();
+
+ TLabelsMap labels = ReadLabels();
+ SkipSpaces();
+
+ double value = ParseGoDouble(ReadToken());
+ SkipSpaces();
+
+ TInstant time = TInstant::Zero();
+ if (CurrentByte_ != '\n') {
+ time = TInstant::MilliSeconds(FromString<ui64>(ReadToken()));
+ }
+
+ TStringBuf baseName = name;
+ EPrometheusMetricType type = EPrometheusMetricType::UNTYPED;
+
+ if (auto* seenType = SeenTypes_.FindPtr(name)) {
+ type = *seenType;
+ } else {
+ baseName = NPrometheus::ToBaseName(name);
+ if (auto* baseType = SeenTypes_.FindPtr(baseName)) {
+ type = *baseType;
+ }
+ }
+
+ switch (type) {
+ case EPrometheusMetricType::HISTOGRAM:
+ if (NPrometheus::IsBucket(name)) {
+ double bound = 0.0;
+ auto it = labels.find(NPrometheus::BUCKET_LABEL);
+ if (it != labels.end()) {
+ bound = ParseGoDouble(it->second);
+ labels.erase(it);
+ } else {
+ Y_PARSER_FAIL(
+ "metric " << name << "has no " << NPrometheus::BUCKET_LABEL <<
+ " label at line #" << CurrentLine_);
+ }
+
+ if (!HistogramBuilder_.Empty() && !HistogramBuilder_.Same(baseName, labels)) {
+ ConsumeHistogram();
+ HistogramBuilder_.SetName(baseName);
+ }
+
+ TBucketValue bucketVal;
+ Y_PARSER_ENSURE(TryStaticCast(value, bucketVal), "Cannot convert " << value << " to bucket value type");
+ HistogramBuilder_.AddBucket(bound, bucketVal);
+ HistogramBuilder_.SetTime(time);
+ HistogramBuilder_.SetLabels(std::move(labels));
+ } else if (NPrometheus::IsCount(name)) {
+ // translate x_count metric as COUNTER metric
+ ConsumeCounter(name, labels, time, value);
+ } else if (NPrometheus::IsSum(name)) {
+ // translate x_sum metric as GAUGE metric
+ ConsumeGauge(name, labels, time, value);
+ } else {
+ Y_PARSER_FAIL(
+ "metric " << name <<
+ " should be part of HISTOGRAM " << baseName);
+ }
+ break;
+
+ case EPrometheusMetricType::SUMMARY:
+ if (NPrometheus::IsCount(name)) {
+ // translate x_count metric as COUNTER metric
+ ConsumeCounter(name, labels, time, value);
+ } else if (NPrometheus::IsSum(name)) {
+ // translate x_sum metric as GAUGE metric
+ ConsumeGauge(name, labels, time, value);
+ } else {
+ ConsumeGauge(name, labels, time, value);
+ }
+ break;
+
+ case EPrometheusMetricType::COUNTER:
+ ConsumeCounter(name, labels, time, value);
+ break;
+
+ case EPrometheusMetricType::GAUGE:
+ ConsumeGauge(name, labels, time, value);
+ break;
+
+ case EPrometheusMetricType::UNTYPED:
+ ConsumeGauge(name, labels, time, value);
+ break;
+ }
+
+ Y_PARSER_ENSURE(CurrentByte_ == '\n', "expected '\\n', found '" << CurrentByte_ << '\'');
+ }
+
+ // { name = "value", name2 = "value2", }
+ TLabelsMap ReadLabels() {
+ TLabelsMap labels;
+ if (CurrentByte_ != '{') {
+ return labels;
+ }
+
+ SkipExpectedChar('{');
+ SkipSpaces();
+
+ while (CurrentByte_ != '}') {
+ TStringBuf name = ReadTokenAsLabelName();
+ SkipSpaces();
+
+ SkipExpectedChar('=');
+ SkipSpaces();
+
+ TString value = ReadTokenAsLabelValue();
+ SkipSpaces();
+ labels.emplace(name, value);
+
+ if (CurrentByte_ == ',') {
+ SkipExpectedChar(',');
+ SkipSpaces();
+ }
+ }
+
+ SkipExpectedChar('}');
+ return labels;
+ }
+
+ EPrometheusMetricType ReadType() {
+ TStringBuf keyword = ReadToken();
+ if (AsciiEqualsIgnoreCase(keyword, "GAUGE")) {
+ return EPrometheusMetricType::GAUGE;
+ } else if (AsciiEqualsIgnoreCase(keyword, "COUNTER")) {
+ return EPrometheusMetricType::COUNTER;
+ } else if (AsciiEqualsIgnoreCase(keyword, "SUMMARY")) {
+ return EPrometheusMetricType::SUMMARY;
+ } else if (AsciiEqualsIgnoreCase(keyword, "HISTOGRAM")) {
+ return EPrometheusMetricType::HISTOGRAM;
+ } else if (AsciiEqualsIgnoreCase(keyword, "UNTYPED")) {
+ return EPrometheusMetricType::UNTYPED;
+ }
+
+ Y_PARSER_FAIL(
+ "unknown metric type: " << keyword <<
+ " at line #" << CurrentLine_);
+ }
+
+ Y_FORCE_INLINE void ReadNextByteUnsafe() {
+ CurrentByte_ = Data_[CurrentPos_++];
+ }
+
+ Y_FORCE_INLINE bool IsSpace(char ch) {
+ return ch == ' ' || ch == '\t';
+ }
+
+ void ReadNextByte() {
+ Y_PARSER_ENSURE(HasRemaining(), "unexpected end of file");
+ ReadNextByteUnsafe();
+ }
+
+ void SkipExpectedChar(char ch) {
+ Y_PARSER_ENSURE(CurrentByte_ == ch,
+ "expected '" << CurrentByte_ << "', found '" << ch << '\'');
+ ReadNextByte();
+ }
+
+ void SkipSpaces() {
+ while (HasRemaining() && IsSpace(CurrentByte_)) {
+ ReadNextByteUnsafe();
+ }
+ }
+
+ void SkipUntilEol() {
+ while (HasRemaining() && CurrentByte_ != '\n') {
+ ReadNextByteUnsafe();
+ }
+ }
+
+ TStringBuf ReadToken() {
+ Y_VERIFY_DEBUG(CurrentPos_ > 0);
+ size_t begin = CurrentPos_ - 1; // read first byte again
+ while (HasRemaining() && !IsSpace(CurrentByte_) && CurrentByte_ != '\n') {
+ ReadNextByteUnsafe();
+ }
+ return TokenFromPos(begin);
+ }
+
+ TStringBuf ReadTokenAsMetricName() {
+ if (!NPrometheus::IsValidMetricNameStart(CurrentByte_)) {
+ return "";
+ }
+
+ Y_VERIFY_DEBUG(CurrentPos_ > 0);
+ size_t begin = CurrentPos_ - 1; // read first byte again
+ while (HasRemaining()) {
+ ReadNextByteUnsafe();
+ if (!NPrometheus::IsValidMetricNameContinuation(CurrentByte_)) {
+ break;
+ }
+ }
+ return TokenFromPos(begin);
+ }
+
+ TStringBuf ReadTokenAsLabelName() {
+ if (!NPrometheus::IsValidLabelNameStart(CurrentByte_)) {
+ return "";
+ }
+
+ Y_VERIFY_DEBUG(CurrentPos_ > 0);
+ size_t begin = CurrentPos_ - 1; // read first byte again
+ while (HasRemaining()) {
+ ReadNextByteUnsafe();
+ if (!NPrometheus::IsValidLabelNameContinuation(CurrentByte_)) {
+ break;
+ }
+ }
+ return TokenFromPos(begin);
+ }
+
+ TString ReadTokenAsLabelValue() {
+ TString labelValue;
+
+ SkipExpectedChar('"');
+ for (ui32 i = 0; i < MAX_LABEL_VALUE_LEN; i++) {
+ switch (CurrentByte_) {
+ case '"':
+ SkipExpectedChar('"');
+ return labelValue;
+
+ case '\n':
+ Y_PARSER_FAIL("label value contains unescaped new-line");
+
+ case '\\':
+ ReadNextByte();
+ switch (CurrentByte_) {
+ case '"':
+ case '\\':
+ labelValue.append(CurrentByte_);
+ break;
+ case 'n':
+ labelValue.append('\n');
+ break;
+ default:
+ Y_PARSER_FAIL("invalid escape sequence '" << CurrentByte_ << '\'');
+ }
+ break;
+
+ default:
+ labelValue.append(CurrentByte_);
+ break;
+ }
+
+ ReadNextByte();
+ }
+
+ Y_PARSER_FAIL("trying to parse too long label value, size >= " << MAX_LABEL_VALUE_LEN);
+ }
+
+ TStringBuf TokenFromPos(size_t begin) {
+ Y_VERIFY_DEBUG(CurrentPos_ > begin);
+ size_t len = CurrentPos_ - begin - 1;
+ if (len == 0) {
+ return {};
+ }
+
+ return Data_.SubString(begin, len);
+ }
+
+ void ConsumeLabels(TStringBuf name, const TLabelsMap& labels) {
+ Y_PARSER_ENSURE(labels.count(MetricNameLabel_) == 0,
+ "label name '" << MetricNameLabel_ <<
+ "' is reserved, but is used with metric: " << name << LabelsToStr(labels));
+
+ Consumer_->OnLabelsBegin();
+ Consumer_->OnLabel(MetricNameLabel_, TString(name)); // TODO: remove this string allocation
+ for (const auto& it: labels) {
+ Consumer_->OnLabel(it.first, it.second);
+ }
+ Consumer_->OnLabelsEnd();
+ }
+
+ void ConsumeCounter(TStringBuf name, const TLabelsMap& labels, TInstant time, double value) {
+ i64 intValue{0};
+ // not nan
+ if (value == value) {
+ Y_PARSER_ENSURE(TryStaticCast(value, intValue), "value " << value << " is out of range");
+ }
+
+ // see https://st.yandex-team.ru/SOLOMON-4142 for more details
+ // why we convert Prometheus COUNTER into Solomon RATE
+ // TODO: need to fix after server-side aggregation become correct for COUNTERs
+ Consumer_->OnMetricBegin(EMetricType::RATE);
+ ConsumeLabels(name, labels);
+ Consumer_->OnUint64(time, intValue);
+ Consumer_->OnMetricEnd();
+ }
+
+ void ConsumeGauge(TStringBuf name, const TLabelsMap& labels, TInstant time, double value) {
+ Consumer_->OnMetricBegin(EMetricType::GAUGE);
+ ConsumeLabels(name, labels);
+ Consumer_->OnDouble(time, value);
+ Consumer_->OnMetricEnd();
+ }
+
+ void ConsumeHistogram() {
+ Consumer_->OnMetricBegin(EMetricType::HIST_RATE);
+ ConsumeLabels(HistogramBuilder_.GetName(), HistogramBuilder_.GetLabels());
+ auto time = HistogramBuilder_.GetTime();
+ auto hist = HistogramBuilder_.ToSnapshot();
+ Consumer_->OnHistogram(time, std::move(hist));
+ Consumer_->OnMetricEnd();
+ }
+
+ double ParseGoDouble(TStringBuf str) {
+ if (str == TStringBuf("+Inf")) {
+ return std::numeric_limits<double>::infinity();
+ } else if (str == TStringBuf("-Inf")) {
+ return -std::numeric_limits<double>::infinity();
+ } else if (str == TStringBuf("NaN")) {
+ return NAN;
+ }
+
+ double r = 0.0;
+ if (TryFromString(str, r)) {
+ return r;
+ }
+ Y_PARSER_FAIL("cannot parse double value from '" << str << "\' at line #" << CurrentLine_);
+ }
+
+ private:
+ TStringBuf Data_;
+ IMetricConsumer* Consumer_;
+ TStringBuf MetricNameLabel_;
+ THashMap<TString, EPrometheusMetricType> SeenTypes_;
+ THistogramBuilder HistogramBuilder_;
+
+ ui32 CurrentLine_ = 1;
+ ui32 CurrentPos_ = 0;
+ char CurrentByte_ = 0;
+ };
+ } // namespace
+
+void DecodePrometheus(TStringBuf data, IMetricConsumer* c, TStringBuf metricNameLabel) {
+ TPrometheusReader reader(data, c, metricNameLabel);
+ reader.Read();
+}
+
+} // namespace NMonitoring
diff --git a/library/cpp/monlib/encode/prometheus/prometheus_decoder_ut.cpp b/library/cpp/monlib/encode/prometheus/prometheus_decoder_ut.cpp
new file mode 100644
index 0000000000..49c2244fb4
--- /dev/null
+++ b/library/cpp/monlib/encode/prometheus/prometheus_decoder_ut.cpp
@@ -0,0 +1,478 @@
+#include "prometheus.h"
+
+#include <library/cpp/monlib/encode/protobuf/protobuf.h>
+
+#include <library/cpp/testing/unittest/registar.h>
+
+using namespace NMonitoring;
+
+#define ASSERT_LABEL_EQUAL(label, name, value) do { \
+ UNIT_ASSERT_STRINGS_EQUAL((label).GetName(), name); \
+ UNIT_ASSERT_STRINGS_EQUAL((label).GetValue(), value); \
+ } while (false)
+
+#define ASSERT_DOUBLE_POINT(s, time, value) do { \
+ UNIT_ASSERT_VALUES_EQUAL((s).GetTime(), (time).MilliSeconds()); \
+ UNIT_ASSERT_EQUAL((s).GetValueCase(), NProto::TSingleSample::kFloat64); \
+ UNIT_ASSERT_DOUBLES_EQUAL((s).GetFloat64(), value, std::numeric_limits<double>::epsilon()); \
+ } while (false)
+
+#define ASSERT_UINT_POINT(s, time, value) do { \
+ UNIT_ASSERT_VALUES_EQUAL((s).GetTime(), (time).MilliSeconds()); \
+ UNIT_ASSERT_EQUAL((s).GetValueCase(), NProto::TSingleSample::kUint64); \
+ UNIT_ASSERT_VALUES_EQUAL((s).GetUint64(), value); \
+ } while (false)
+
+#define ASSERT_HIST_POINT(s, time, expected) do { \
+ UNIT_ASSERT_VALUES_EQUAL((s).GetTime(), time.MilliSeconds()); \
+ UNIT_ASSERT_EQUAL((s).GetValueCase(), NProto::TSingleSample::kHistogram);\
+ UNIT_ASSERT_VALUES_EQUAL((s).GetHistogram().BoundsSize(), (expected).Count()); \
+ UNIT_ASSERT_VALUES_EQUAL((s).GetHistogram().ValuesSize(), (expected).Count()); \
+ for (size_t i = 0; i < (s).GetHistogram().BoundsSize(); i++) { \
+ UNIT_ASSERT_DOUBLES_EQUAL((s).GetHistogram().GetBounds(i), (expected).UpperBound(i), Min<double>()); \
+ UNIT_ASSERT_VALUES_EQUAL((s).GetHistogram().GetValues(i), (expected).Value(i)); \
+ } \
+ } while (false)
+
+Y_UNIT_TEST_SUITE(TPrometheusDecoderTest) {
+
+ NProto::TSingleSamplesList Decode(TStringBuf data) {
+ NProto::TSingleSamplesList samples;
+ {
+ IMetricEncoderPtr e = EncoderProtobuf(&samples);
+ DecodePrometheus(data, e.Get());
+ }
+ return samples;
+ }
+
+ Y_UNIT_TEST(Empty) {
+ {
+ auto samples = Decode("");
+ UNIT_ASSERT_EQUAL(samples.SamplesSize(), 0);
+ }
+ {
+ auto samples = Decode("\n");
+ UNIT_ASSERT_EQUAL(samples.SamplesSize(), 0);
+ }
+ {
+ auto samples = Decode("\n \n \n");
+ UNIT_ASSERT_EQUAL(samples.SamplesSize(), 0);
+ }
+ {
+ auto samples = Decode("\t\n\t\n");
+ UNIT_ASSERT_EQUAL(samples.SamplesSize(), 0);
+ }
+ }
+
+ Y_UNIT_TEST(Minimal) {
+ auto samples = Decode(
+ "minimal_metric 1.234\n"
+ "another_metric -3e3 103948\n"
+ "# Even that:\n"
+ "no_labels{} 3\n"
+ "# HELP line for non-existing metric will be ignored.\n");
+
+ UNIT_ASSERT_EQUAL(samples.SamplesSize(), 3);
+ {
+ auto& s = samples.GetSamples(0);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE);
+ UNIT_ASSERT_EQUAL(1, s.LabelsSize());
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "minimal_metric");
+ ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 1.234);
+ }
+ {
+ auto& s = samples.GetSamples(1);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 1);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "another_metric");
+ ASSERT_DOUBLE_POINT(s, TInstant::MilliSeconds(103948), -3000.0);
+ }
+ {
+ auto& s = samples.GetSamples(2);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE);
+ UNIT_ASSERT_EQUAL(1, s.LabelsSize());
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "no_labels");
+ ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 3.0);
+ }
+ }
+
+ Y_UNIT_TEST(Counter) {
+ auto samples = Decode(
+ "# A normal comment.\n"
+ "#\n"
+ "# TYPE name counter\n"
+ "name{labelname=\"val1\",basename=\"basevalue\"} NaN\n"
+ "name {labelname=\"val2\",basename=\"basevalue\"} 2.3 1234567890\n"
+ "# HELP name two-line\\n doc str\\\\ing\n");
+
+ UNIT_ASSERT_EQUAL(samples.SamplesSize(), 2);
+
+ {
+ auto& s = samples.GetSamples(0);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 3);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "name");
+ ASSERT_LABEL_EQUAL(s.GetLabels(1), "basename", "basevalue");
+ ASSERT_LABEL_EQUAL(s.GetLabels(2), "labelname", "val1");
+ ASSERT_UINT_POINT(s, TInstant::Zero(), ui64(0));
+ }
+ {
+ auto& s = samples.GetSamples(1);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 3);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "name");
+ ASSERT_LABEL_EQUAL(s.GetLabels(1), "basename", "basevalue");
+ ASSERT_LABEL_EQUAL(s.GetLabels(2), "labelname", "val2");
+ ASSERT_UINT_POINT(s, TInstant::MilliSeconds(1234567890), i64(2));
+ }
+ }
+
+ Y_UNIT_TEST(Gauge) {
+ auto samples = Decode(
+ "# A normal comment.\n"
+ "#\n"
+ " # HELP name2 \tdoc str\"ing 2\n"
+ " # TYPE name2 gauge\n"
+ "name2{labelname=\"val2\"\t,basename = \"basevalue2\"\t\t} +Inf 54321\n"
+ "name2{ labelname = \"val1\" , }-Inf\n");
+
+ UNIT_ASSERT_EQUAL(samples.SamplesSize(), 2);
+
+ {
+ auto& s = samples.GetSamples(0);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 3);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "name2");
+ ASSERT_LABEL_EQUAL(s.GetLabels(1), "basename", "basevalue2");
+ ASSERT_LABEL_EQUAL(s.GetLabels(2), "labelname", "val2");
+ ASSERT_DOUBLE_POINT(s, TInstant::MilliSeconds(54321), INFINITY);
+ }
+ {
+ auto& s = samples.GetSamples(1);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 2);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "name2");
+ ASSERT_LABEL_EQUAL(s.GetLabels(1), "labelname", "val1");
+ ASSERT_DOUBLE_POINT(s, TInstant::Zero(), -INFINITY);
+ }
+ }
+
+ Y_UNIT_TEST(Summary) {
+ auto samples = Decode(
+ "# HELP \n"
+ "# TYPE my_summary summary\n"
+ "my_summary{n1=\"val1\",quantile=\"0.5\"} 110\n"
+ "my_summary{n1=\"val1\",quantile=\"0.9\"} 140 1\n"
+ "my_summary_count{n1=\"val1\"} 42\n"
+ "my_summary_sum{n1=\"val1\"} 08 15\n"
+ "# some\n"
+ "# funny comments\n"
+ "# HELP\n"
+ "# HELP my_summary\n"
+ "# HELP my_summary \n");
+
+ UNIT_ASSERT_EQUAL(samples.SamplesSize(), 4);
+
+ {
+ auto& s = samples.GetSamples(0);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 3);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "my_summary");
+ ASSERT_LABEL_EQUAL(s.GetLabels(1), "quantile", "0.5");
+ ASSERT_LABEL_EQUAL(s.GetLabels(2), "n1", "val1");
+ ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 110.0);
+ }
+ {
+ auto& s = samples.GetSamples(1);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 3);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "my_summary");
+ ASSERT_LABEL_EQUAL(s.GetLabels(1), "quantile", "0.9");
+ ASSERT_LABEL_EQUAL(s.GetLabels(2), "n1", "val1");
+ ASSERT_DOUBLE_POINT(s, TInstant::MilliSeconds(1), 140.0);
+ }
+ {
+ auto& s = samples.GetSamples(2);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 2);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "my_summary_count");
+ ASSERT_LABEL_EQUAL(s.GetLabels(1), "n1", "val1");
+ ASSERT_UINT_POINT(s, TInstant::Zero(), 42);
+ }
+ {
+ auto& s = samples.GetSamples(3);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 2);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "my_summary_sum");
+ ASSERT_LABEL_EQUAL(s.GetLabels(1), "n1", "val1");
+ ASSERT_DOUBLE_POINT(s, TInstant::MilliSeconds(15), 8.0);
+ }
+ }
+
+ Y_UNIT_TEST(Histogram) {
+ auto samples = Decode(
+ "# HELP request_duration_microseconds The response latency.\n"
+ "# TYPE request_duration_microseconds histogram\n"
+ "request_duration_microseconds_bucket{le=\"0\"} 0\n"
+ "request_duration_microseconds_bucket{le=\"100\"} 123\n"
+ "request_duration_microseconds_bucket{le=\"120\"} 412\n"
+ "request_duration_microseconds_bucket{le=\"144\"} 592\n"
+ "request_duration_microseconds_bucket{le=\"172.8\"} 1524\n"
+ "request_duration_microseconds_bucket{le=\"+Inf\"} 2693\n"
+ "request_duration_microseconds_sum 1.7560473e+06\n"
+ "request_duration_microseconds_count 2693\n");
+
+ UNIT_ASSERT_EQUAL(samples.SamplesSize(), 3);
+
+ {
+ auto& s = samples.GetSamples(0);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 1);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "request_duration_microseconds_sum");
+ ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 1756047.3);
+ }
+ {
+ auto& s = samples.GetSamples(1);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 1);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "request_duration_microseconds_count");
+ ASSERT_UINT_POINT(s, TInstant::Zero(), 2693);
+ }
+ {
+ auto& s = samples.GetSamples(2);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::HIST_RATE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 1);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "request_duration_microseconds");
+ auto hist = ExplicitHistogramSnapshot(
+ { 0, 100, 120, 144, 172.8, HISTOGRAM_INF_BOUND },
+ { 0, 123, 289, 180, 932, 1169 });
+ ASSERT_HIST_POINT(s, TInstant::Zero(), *hist);
+ }
+ }
+
+ Y_UNIT_TEST(HistogramWithLabels) {
+ auto samples = Decode(
+ "# A histogram, which has a pretty complex representation in the text format:\n"
+ "# HELP http_request_duration_seconds A histogram of the request duration.\n"
+ "# TYPE http_request_duration_seconds histogram\n"
+ "http_request_duration_seconds_bucket{le=\"0.05\", method=\"POST\"} 24054\n"
+ "http_request_duration_seconds_bucket{method=\"POST\", le=\"0.1\"} 33444\n"
+ "http_request_duration_seconds_bucket{le=\"0.2\", method=\"POST\", } 100392\n"
+ "http_request_duration_seconds_bucket{le=\"0.5\",method=\"POST\",} 129389\n"
+ "http_request_duration_seconds_bucket{ method=\"POST\", le=\"1\", } 133988\n"
+ "http_request_duration_seconds_bucket{ le=\"+Inf\", method=\"POST\", } 144320\n"
+ "http_request_duration_seconds_sum{method=\"POST\"} 53423\n"
+ "http_request_duration_seconds_count{ method=\"POST\", } 144320\n");
+
+ UNIT_ASSERT_EQUAL(samples.SamplesSize(), 3);
+
+ {
+ auto& s = samples.GetSamples(0);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 2);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "http_request_duration_seconds_sum");
+ ASSERT_LABEL_EQUAL(s.GetLabels(1), "method", "POST");
+ ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 53423.0);
+ }
+ {
+ auto& s = samples.GetSamples(1);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 2);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "http_request_duration_seconds_count");
+ ASSERT_LABEL_EQUAL(s.GetLabels(1), "method", "POST");
+ ASSERT_UINT_POINT(s, TInstant::Zero(), 144320);
+ }
+ {
+ auto& s = samples.GetSamples(2);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::HIST_RATE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 2);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "http_request_duration_seconds");
+ ASSERT_LABEL_EQUAL(s.GetLabels(1), "method", "POST");
+ auto hist = ExplicitHistogramSnapshot(
+ { 0.05, 0.1, 0.2, 0.5, 1, HISTOGRAM_INF_BOUND },
+ { 24054, 9390, 66948, 28997, 4599, 10332 });
+ ASSERT_HIST_POINT(s, TInstant::Zero(), *hist);
+ }
+ }
+
+ Y_UNIT_TEST(MultipleHistograms) {
+ auto samples = Decode(
+ "# TYPE inboundBytesPerSec histogram\n"
+ "inboundBytesPerSec_bucket{client=\"mbus\", le=\"10.0\"} 1.0\n"
+ "inboundBytesPerSec_bucket{client=\"mbus\", le=\"20.0\"} 5.0\n"
+ "inboundBytesPerSec_bucket{client=\"mbus\", le=\"+Inf\"} 5.0\n"
+ "inboundBytesPerSec_count{client=\"mbus\"} 5.0\n"
+ "inboundBytesPerSec_bucket{client=\"grpc\", le=\"10.0\"} 1.0\n"
+ "inboundBytesPerSec_bucket{client=\"grpc\", le=\"20.0\"} 5.0\n"
+ "inboundBytesPerSec_bucket{client=\"grpc\", le=\"30.0\"} 5.0\n"
+ "inboundBytesPerSec_count{client=\"grpc\"} 5.0\n"
+ "# TYPE outboundBytesPerSec histogram\n"
+ "outboundBytesPerSec_bucket{client=\"grpc\", le=\"100.0\"} 1.0 1512216000000\n"
+ "outboundBytesPerSec_bucket{client=\"grpc\", le=\"200.0\"} 1.0 1512216000000\n"
+ "outboundBytesPerSec_bucket{client=\"grpc\", le=\"+Inf\"} 1.0 1512216000000\n"
+ "outboundBytesPerSec_count{client=\"grpc\"} 1.0 1512216000000\n");
+
+ UNIT_ASSERT_EQUAL(samples.SamplesSize(), 6);
+
+ {
+ auto& s = samples.GetSamples(0);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 2);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "inboundBytesPerSec_count");
+ ASSERT_LABEL_EQUAL(s.GetLabels(1), "client", "mbus");
+ ASSERT_UINT_POINT(s, TInstant::Zero(), 5);
+ }
+ {
+ auto& s = samples.GetSamples(1);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::HIST_RATE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 2);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "inboundBytesPerSec");
+ ASSERT_LABEL_EQUAL(s.GetLabels(1), "client", "mbus");
+ auto hist = ExplicitHistogramSnapshot(
+ { 10, 20, HISTOGRAM_INF_BOUND },
+ { 1, 4, 0 });
+ ASSERT_HIST_POINT(s, TInstant::Zero(), *hist);
+ }
+ {
+ auto& s = samples.GetSamples(2);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 2);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "inboundBytesPerSec_count");
+ ASSERT_LABEL_EQUAL(s.GetLabels(1), "client", "grpc");
+ ASSERT_UINT_POINT(s, TInstant::Zero(), 5);
+ }
+ {
+ auto& s = samples.GetSamples(3);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::HIST_RATE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 2);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "inboundBytesPerSec");
+ ASSERT_LABEL_EQUAL(s.GetLabels(1), "client", "grpc");
+ auto hist = ExplicitHistogramSnapshot(
+ { 10, 20, 30 },
+ { 1, 4, 0 });
+ ASSERT_HIST_POINT(s, TInstant::Zero(), *hist);
+ }
+ {
+ auto& s = samples.GetSamples(4);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 2);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "outboundBytesPerSec_count");
+ ASSERT_LABEL_EQUAL(s.GetLabels(1), "client", "grpc");
+ ASSERT_UINT_POINT(s, TInstant::Seconds(1512216000), 1) ;
+ }
+ {
+ auto& s = samples.GetSamples(5);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::HIST_RATE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 2);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "outboundBytesPerSec");
+ ASSERT_LABEL_EQUAL(s.GetLabels(1), "client", "grpc");
+ auto hist = ExplicitHistogramSnapshot(
+ { 100, 200, HISTOGRAM_INF_BOUND },
+ { 1, 0, 0 });
+ ASSERT_HIST_POINT(s, TInstant::Seconds(1512216000), *hist);
+ }
+ }
+
+ Y_UNIT_TEST(MixedTypes) {
+ auto samples = Decode(
+ "# HELP http_requests_total The total number of HTTP requests.\n"
+ "# TYPE http_requests_total counter\n"
+ "http_requests_total { } 1027 1395066363000\n"
+ "http_requests_total{method=\"post\",code=\"200\"} 1027 1395066363000\n"
+ "http_requests_total{method=\"post\",code=\"400\"} 3 1395066363000\n"
+ "\n"
+ "# Minimalistic line:\n"
+ "metric_without_timestamp_and_labels 12.47\n"
+ "\n"
+ "# HELP rpc_duration_seconds A summary of the RPC duration in seconds.\n"
+ "# TYPE rpc_duration_seconds summary\n"
+ "rpc_duration_seconds{quantile=\"0.01\"} 3102\n"
+ "rpc_duration_seconds{quantile=\"0.5\"} 4773\n"
+ "rpc_duration_seconds{quantile=\"0.9\"} 9001\n"
+ "rpc_duration_seconds_sum 1.7560473e+07\n"
+ "rpc_duration_seconds_count 2693\n"
+ "\n"
+ "# Another mMinimalistic line:\n"
+ "metric_with_timestamp 12.47 1234567890\n");
+
+ UNIT_ASSERT_EQUAL(samples.SamplesSize(), 10);
+
+ {
+ auto& s = samples.GetSamples(0);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 1);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "http_requests_total");
+ ASSERT_UINT_POINT(s, TInstant::Seconds(1395066363), 1027);
+ }
+ {
+ auto& s = samples.GetSamples(1);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 3);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "http_requests_total");
+ ASSERT_LABEL_EQUAL(s.GetLabels(1), "method", "post");
+ ASSERT_LABEL_EQUAL(s.GetLabels(2), "code", "200");
+ ASSERT_UINT_POINT(s, TInstant::Seconds(1395066363), 1027);
+ }
+ {
+ auto& s = samples.GetSamples(2);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 3);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "http_requests_total");
+ ASSERT_LABEL_EQUAL(s.GetLabels(1), "method", "post");
+ ASSERT_LABEL_EQUAL(s.GetLabels(2), "code", "400");
+ ASSERT_UINT_POINT(s, TInstant::Seconds(1395066363), 3);
+ }
+ {
+ auto& s = samples.GetSamples(3);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 1);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "metric_without_timestamp_and_labels");
+ ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 12.47);
+ }
+ {
+ auto& s = samples.GetSamples(4);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 2);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "rpc_duration_seconds");
+ ASSERT_LABEL_EQUAL(s.GetLabels(1), "quantile", "0.01");
+ ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 3102);
+ }
+ {
+ auto& s = samples.GetSamples(5);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 2);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "rpc_duration_seconds");
+ ASSERT_LABEL_EQUAL(s.GetLabels(1), "quantile", "0.5");
+ ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 4773);
+ }
+ {
+ auto& s = samples.GetSamples(6);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 2);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "rpc_duration_seconds");
+ ASSERT_LABEL_EQUAL(s.GetLabels(1), "quantile", "0.9");
+ ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 9001);
+ }
+ {
+ auto& s = samples.GetSamples(7);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 1);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "rpc_duration_seconds_sum");
+ ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 17560473);
+ }
+ {
+ auto& s = samples.GetSamples(8);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 1);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "rpc_duration_seconds_count");
+ ASSERT_UINT_POINT(s, TInstant::Zero(), 2693);
+ }
+ {
+ auto& s = samples.GetSamples(9);
+ UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE);
+ UNIT_ASSERT_EQUAL(s.LabelsSize(), 1);
+ ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "metric_with_timestamp");
+ ASSERT_DOUBLE_POINT(s, TInstant::MilliSeconds(1234567890), 12.47);
+ }
+ }
+}
diff --git a/library/cpp/monlib/encode/prometheus/prometheus_encoder.cpp b/library/cpp/monlib/encode/prometheus/prometheus_encoder.cpp
new file mode 100644
index 0000000000..15efeb8c03
--- /dev/null
+++ b/library/cpp/monlib/encode/prometheus/prometheus_encoder.cpp
@@ -0,0 +1,413 @@
+#include "prometheus.h"
+#include "prometheus_model.h"
+
+#include <library/cpp/monlib/encode/encoder_state.h>
+#include <library/cpp/monlib/metrics/labels.h>
+#include <library/cpp/monlib/metrics/metric_value.h>
+
+#include <util/string/cast.h>
+#include <util/generic/hash_set.h>
+
+
+namespace NMonitoring {
+ namespace {
+ ///////////////////////////////////////////////////////////////////////
+ // TPrometheusWriter
+ ///////////////////////////////////////////////////////////////////////
+ class TPrometheusWriter {
+ public:
+ explicit TPrometheusWriter(IOutputStream* out)
+ : Out_(out)
+ {
+ }
+
+ void WriteType(EMetricType type, const TString& name) {
+ auto r = WrittenTypes_.insert(name);
+ if (!r.second) {
+ // type for this metric was already written
+ return;
+ }
+
+ Out_->Write("# TYPE ");
+ WriteMetricName(name);
+ Out_->Write(' ');
+
+ switch (type) {
+ case EMetricType::GAUGE:
+ case EMetricType::IGAUGE:
+ Out_->Write("gauge");
+ break;
+ case EMetricType::RATE:
+ case EMetricType::COUNTER:
+ Out_->Write("counter");
+ break;
+ case EMetricType::HIST:
+ case EMetricType::HIST_RATE:
+ Out_->Write("histogram");
+ break;
+ case EMetricType::LOGHIST:
+ // TODO(@kbalakirev): implement this case
+ break;
+ case EMetricType::DSUMMARY:
+ ythrow yexception() << "writing summary type is forbiden";
+ case EMetricType::UNKNOWN:
+ ythrow yexception() << "unknown metric type: " << MetricTypeToStr(type)
+ << ", name: " << name;
+ }
+ Out_->Write('\n');
+ }
+
+ void WriteDouble(TStringBuf name, const TLabels& labels, TInstant time, double value) {
+ WriteValue(name, "", labels, "", "", time, value);
+ }
+
+ void WriteHistogram(TStringBuf name, const TLabels& labels, TInstant time, IHistogramSnapshot* h) {
+ Y_ENSURE(!labels.Has(NPrometheus::BUCKET_LABEL),
+ "histogram metric " << name << " has label '" <<
+ NPrometheus::BUCKET_LABEL << "' which is reserved in Prometheus");
+
+ double totalCount = 0;
+ for (ui32 i = 0, count = h->Count(); i < count; i++) {
+ TBucketBound bound = h->UpperBound(i);
+ TStringBuf boundStr;
+ if (bound == HISTOGRAM_INF_BOUND) {
+ boundStr = TStringBuf("+Inf");
+ } else {
+ size_t len = FloatToString(bound, TmpBuf_, Y_ARRAY_SIZE(TmpBuf_));
+ boundStr = TStringBuf(TmpBuf_, len);
+ }
+
+ TBucketValue value = h->Value(i);
+ totalCount += static_cast<double>(value);
+
+ WriteValue(
+ name, NPrometheus::BUCKET_SUFFIX,
+ labels, NPrometheus::BUCKET_LABEL, boundStr,
+ time,
+ totalCount);
+ }
+
+ WriteValue(name, NPrometheus::COUNT_SUFFIX, labels, "", "", time, totalCount);
+ }
+
+ void WriteSummaryDouble(TStringBuf name, const TLabels& labels, TInstant time, ISummaryDoubleSnapshot* s) {
+ WriteValue(name, NPrometheus::SUM_SUFFIX, labels, "", "", time, s->GetSum());
+ WriteValue(name, NPrometheus::MIN_SUFFIX, labels, "", "", time, s->GetMin());
+ WriteValue(name, NPrometheus::MAX_SUFFIX, labels, "", "", time, s->GetMax());
+ WriteValue(name, NPrometheus::LAST_SUFFIX, labels, "", "", time, s->GetLast());
+ WriteValue(name, NPrometheus::COUNT_SUFFIX, labels, "", "", time, s->GetCount());
+ }
+
+ void WriteLn() {
+ Out_->Write('\n');
+ }
+
+ private:
+ // will replace invalid chars with '_'
+ void WriteMetricName(TStringBuf name) {
+ Y_ENSURE(!name.Empty(), "trying to write metric with empty name");
+
+ char ch = name[0];
+ if (NPrometheus::IsValidMetricNameStart(ch)) {
+ Out_->Write(ch);
+ } else {
+ Out_->Write('_');
+ }
+
+ for (size_t i = 1, len = name.length(); i < len; i++) {
+ ch = name[i];
+ if (NPrometheus::IsValidMetricNameContinuation(ch)) {
+ Out_->Write(ch);
+ } else {
+ Out_->Write('_');
+ }
+ }
+ }
+
+ void WriteLabels(const TLabels& labels, TStringBuf addLabelKey, TStringBuf addLabelValue) {
+ Out_->Write('{');
+ for (auto&& l: labels) {
+ Out_->Write(l.Name());
+ Out_->Write('=');
+ WriteLabelValue(l.Value());
+ Out_->Write(", "); // trailign comma is supported in parsers
+ }
+ if (!addLabelKey.Empty() && !addLabelValue.Empty()) {
+ Out_->Write(addLabelKey);
+ Out_->Write('=');
+ WriteLabelValue(addLabelValue);
+ }
+ Out_->Write('}');
+ }
+
+ void WriteLabelValue(TStringBuf value) {
+ Out_->Write('"');
+ for (char ch: value) {
+ if (ch == '"') {
+ Out_->Write("\\\"");
+ } else if (ch == '\\') {
+ Out_->Write("\\\\");
+ } else if (ch == '\n') {
+ Out_->Write("\\n");
+ } else {
+ Out_->Write(ch);
+ }
+ }
+ Out_->Write('"');
+ }
+
+ void WriteValue(
+ TStringBuf name, TStringBuf suffix,
+ const TLabels& labels, TStringBuf addLabelKey, TStringBuf addLabelValue,
+ TInstant time, double value)
+ {
+ // (1) name
+ WriteMetricName(name);
+ if (!suffix.Empty()) {
+ Out_->Write(suffix);
+ }
+
+ // (2) labels
+ if (!labels.Empty() || !addLabelKey.Empty()) {
+ WriteLabels(labels, addLabelKey, addLabelValue);
+ }
+ Out_->Write(' ');
+
+ // (3) value
+ {
+ size_t len = FloatToString(value, TmpBuf_, Y_ARRAY_SIZE(TmpBuf_));
+ Out_->Write(TmpBuf_, len);
+ }
+
+ // (4) time
+ if (ui64 timeMillis = time.MilliSeconds()) {
+ Out_->Write(' ');
+ size_t len = IntToString<10>(timeMillis, TmpBuf_, Y_ARRAY_SIZE(TmpBuf_));
+ Out_->Write(TmpBuf_, len);
+ }
+ Out_->Write('\n');
+ }
+
+ private:
+ IOutputStream* Out_;
+ THashSet<TString> WrittenTypes_;
+ char TmpBuf_[512]; // used to convert doubles to strings
+ };
+
+ ///////////////////////////////////////////////////////////////////////
+ // TMetricState
+ ///////////////////////////////////////////////////////////////////////
+ struct TMetricState {
+ EMetricType Type = EMetricType::UNKNOWN;
+ TLabels Labels;
+ TInstant Time = TInstant::Zero();
+ EMetricValueType ValueType = EMetricValueType::UNKNOWN;
+ TMetricValue Value;
+
+ ~TMetricState() {
+ ClearValue();
+ }
+
+ void Clear() {
+ Type = EMetricType::UNKNOWN;
+ Labels.Clear();
+ Time = TInstant::Zero();
+ ClearValue();
+ }
+
+ void ClearValue() {
+ // TMetricValue does not keep ownership of histogram
+ if (ValueType == EMetricValueType::HISTOGRAM) {
+ Value.AsHistogram()->UnRef();
+ } else if (ValueType == EMetricValueType::SUMMARY) {
+ Value.AsSummaryDouble()->UnRef();
+ }
+ ValueType = EMetricValueType::UNKNOWN;
+ Value = {};
+ }
+
+ template <typename T>
+ void SetValue(T value) {
+ // TMetricValue does not keep ownership of histogram
+ if (ValueType == EMetricValueType::HISTOGRAM) {
+ Value.AsHistogram()->UnRef();
+ } else if (ValueType == EMetricValueType::SUMMARY) {
+ Value.AsSummaryDouble()->UnRef();
+ }
+ ValueType = TValueType<T>::Type;
+ Value = TMetricValue(value);
+ if (ValueType == EMetricValueType::HISTOGRAM) {
+ Value.AsHistogram()->Ref();
+ } else if (ValueType == EMetricValueType::SUMMARY) {
+ Value.AsSummaryDouble()->Ref();
+ }
+ }
+ };
+
+ ///////////////////////////////////////////////////////////////////////
+ // TPrometheusEncoder
+ ///////////////////////////////////////////////////////////////////////
+ class TPrometheusEncoder final: public IMetricEncoder {
+ public:
+ explicit TPrometheusEncoder(IOutputStream* out, TStringBuf metricNameLabel)
+ : Writer_(out)
+ , MetricNameLabel_(metricNameLabel)
+ {
+ }
+
+ private:
+ void OnStreamBegin() override {
+ State_.Expect(TEncoderState::EState::ROOT);
+ }
+
+ void OnStreamEnd() override {
+ State_.Expect(TEncoderState::EState::ROOT);
+ Writer_.WriteLn();
+ }
+
+ void OnCommonTime(TInstant time) override {
+ State_.Expect(TEncoderState::EState::ROOT);
+ CommonTime_ = time;
+ }
+
+ void OnMetricBegin(EMetricType type) override {
+ State_.Switch(TEncoderState::EState::ROOT, TEncoderState::EState::METRIC);
+ MetricState_.Clear();
+ MetricState_.Type = type;
+ }
+
+ void OnMetricEnd() override {
+ State_.Switch(TEncoderState::EState::METRIC, TEncoderState::EState::ROOT);
+ WriteMetric();
+ }
+
+ void OnLabelsBegin() override {
+ if (State_ == TEncoderState::EState::METRIC) {
+ State_ = TEncoderState::EState::METRIC_LABELS;
+ } else if (State_ == TEncoderState::EState::ROOT) {
+ State_ = TEncoderState::EState::COMMON_LABELS;
+ } else {
+ State_.ThrowInvalid("expected METRIC or ROOT");
+ }
+ }
+
+ void OnLabelsEnd() override {
+ if (State_ == TEncoderState::EState::METRIC_LABELS) {
+ State_ = TEncoderState::EState::METRIC;
+ } else if (State_ == TEncoderState::EState::COMMON_LABELS) {
+ State_ = TEncoderState::EState::ROOT;
+ } else {
+ State_.ThrowInvalid("expected LABELS or COMMON_LABELS");
+ }
+ }
+
+ void OnLabel(TStringBuf name, TStringBuf value) override {
+ if (State_ == TEncoderState::EState::METRIC_LABELS) {
+ MetricState_.Labels.Add(name, value);
+ } else if (State_ == TEncoderState::EState::COMMON_LABELS) {
+ CommonLabels_.Add(name, value);
+ } else {
+ State_.ThrowInvalid("expected LABELS or COMMON_LABELS");
+ }
+ }
+
+ void OnDouble(TInstant time, double value) override {
+ State_.Expect(TEncoderState::EState::METRIC);
+ MetricState_.Time = time;
+ MetricState_.SetValue(value);
+ }
+
+ void OnInt64(TInstant time, i64 value) override {
+ State_.Expect(TEncoderState::EState::METRIC);
+ MetricState_.Time = time;
+ MetricState_.SetValue(value);
+ }
+
+ void OnUint64(TInstant time, ui64 value) override {
+ State_.Expect(TEncoderState::EState::METRIC);
+ MetricState_.Time = time;
+ MetricState_.SetValue(value);
+ }
+
+ void OnHistogram(TInstant time, IHistogramSnapshotPtr snapshot) override {
+ State_.Expect(TEncoderState::EState::METRIC);
+ MetricState_.Time = time;
+ MetricState_.SetValue(snapshot.Get());
+ }
+
+ void OnSummaryDouble(TInstant time, ISummaryDoubleSnapshotPtr snapshot) override {
+ State_.Expect(TEncoderState::EState::METRIC);
+ MetricState_.Time = time;
+ MetricState_.SetValue(snapshot.Get());
+ }
+
+ void OnLogHistogram(TInstant, TLogHistogramSnapshotPtr) override {
+ // TODO(@kbalakirev): implement this function
+ }
+
+ void Close() override {
+ }
+
+ void WriteMetric() {
+ if (MetricState_.ValueType == EMetricValueType::UNKNOWN) {
+ return;
+ }
+
+ // XXX: poor performace
+ for (auto&& l: CommonLabels_) {
+ MetricState_.Labels.Add(l.Name(), l.Value());
+ }
+
+ TMaybe<TLabel> nameLabel = MetricState_.Labels.Extract(MetricNameLabel_);
+ Y_ENSURE(nameLabel,
+ "labels " << MetricState_.Labels <<
+ " does not contain label '" << MetricNameLabel_ << '\'');
+
+ const TString& metricName = ToString(nameLabel->Value());
+ if (MetricState_.Type != EMetricType::DSUMMARY) {
+ Writer_.WriteType(MetricState_.Type, metricName);
+ }
+
+ if (MetricState_.Time == TInstant::Zero()) {
+ MetricState_.Time = CommonTime_;
+ }
+
+ EMetricType type = MetricState_.Type;
+ if (type == EMetricType::HIST || type == EMetricType::HIST_RATE) {
+ Y_ENSURE(MetricState_.ValueType == EMetricValueType::HISTOGRAM,
+ "invalid value type for histogram: " << int(MetricState_.ValueType)); // TODO: to string conversion
+ Writer_.WriteHistogram(
+ metricName,
+ MetricState_.Labels,
+ MetricState_.Time,
+ MetricState_.Value.AsHistogram());
+ } else if (type == EMetricType::DSUMMARY) {
+ Writer_.WriteSummaryDouble(
+ metricName,
+ MetricState_.Labels,
+ MetricState_.Time,
+ MetricState_.Value.AsSummaryDouble());
+ } else {
+ Writer_.WriteDouble(
+ metricName,
+ MetricState_.Labels,
+ MetricState_.Time,
+ MetricState_.Value.AsDouble(MetricState_.ValueType));
+ }
+ }
+
+ private:
+ TEncoderState State_;
+ TPrometheusWriter Writer_;
+ TString MetricNameLabel_;
+ TInstant CommonTime_ = TInstant::Zero();
+ TLabels CommonLabels_;
+ TMetricState MetricState_;
+ };
+ }
+
+ IMetricEncoderPtr EncoderPrometheus(IOutputStream* out, TStringBuf metricNameLabel) {
+ return MakeHolder<TPrometheusEncoder>(out, metricNameLabel);
+ }
+
+} // namespace NMonitoring
diff --git a/library/cpp/monlib/encode/prometheus/prometheus_encoder_ut.cpp b/library/cpp/monlib/encode/prometheus/prometheus_encoder_ut.cpp
new file mode 100644
index 0000000000..fd9debb060
--- /dev/null
+++ b/library/cpp/monlib/encode/prometheus/prometheus_encoder_ut.cpp
@@ -0,0 +1,414 @@
+#include "prometheus.h"
+
+#include <library/cpp/monlib/encode/protobuf/protobuf.h>
+#include <library/cpp/monlib/metrics/metric_value.h>
+#include <library/cpp/monlib/metrics/histogram_snapshot.h>
+
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <util/stream/str.h>
+
+using namespace NMonitoring;
+
+Y_UNIT_TEST_SUITE(TPrometheusEncoderTest) {
+
+ template <typename TFunc>
+ TString EncodeToString(TFunc fn) {
+ TStringStream ss;
+ IMetricEncoderPtr encoder = EncoderPrometheus(&ss);
+ fn(encoder.Get());
+ return ss.Str();
+ }
+
+ ISummaryDoubleSnapshotPtr TestSummaryDouble() {
+ return MakeIntrusive<TSummaryDoubleSnapshot>(10.1, -0.45, 0.478, 0.3, 30u);
+ }
+
+ Y_UNIT_TEST(Empty) {
+ auto result = EncodeToString([](IMetricEncoder* e) {
+ e->OnStreamBegin();
+ e->OnStreamEnd();
+ });
+ UNIT_ASSERT_STRINGS_EQUAL(result, "\n");
+ }
+
+ Y_UNIT_TEST(DoubleGauge) {
+ auto result = EncodeToString([](IMetricEncoder* e) {
+ e->OnStreamBegin();
+ { // no values
+ e->OnMetricBegin(EMetricType::GAUGE);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "cpuUsage");
+ e->OnLabelsEnd();
+ }
+ e->OnMetricEnd();
+ }
+ { // one value no ts
+ e->OnMetricBegin(EMetricType::GAUGE);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "diskUsage");
+ e->OnLabel("disk", "sda1");
+ e->OnLabelsEnd();
+ }
+ e->OnDouble(TInstant::Zero(), 1000);
+ e->OnMetricEnd();
+ }
+ { // one value with ts
+ e->OnMetricBegin(EMetricType::GAUGE);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "memoryUsage");
+ e->OnLabel("host", "solomon-man-00");
+ e->OnLabel("dc", "man");
+ e->OnLabelsEnd();
+ }
+ e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 1000);
+ e->OnMetricEnd();
+ }
+ { // many values
+ e->OnMetricBegin(EMetricType::GAUGE);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "bytesRx");
+ e->OnLabel("host", "solomon-sas-01");
+ e->OnLabel("dc", "sas");
+ e->OnLabelsEnd();
+ }
+ e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 2);
+ e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:05Z"), 4);
+ e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:10Z"), 8);
+ e->OnMetricEnd();
+ }
+ { // already seen metric name
+ e->OnMetricBegin(EMetricType::GAUGE);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "diskUsage");
+ e->OnLabel("disk", "sdb1");
+ e->OnLabelsEnd();
+ }
+ e->OnDouble(TInstant::Zero(), 1001);
+ e->OnMetricEnd();
+ }
+ { // NaN
+ e->OnMetricBegin(EMetricType::GAUGE);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "nanValue");
+ e->OnLabelsEnd();
+ }
+ e->OnDouble(TInstant::Zero(), NAN);
+ e->OnMetricEnd();
+ }
+ { // Inf
+ e->OnMetricBegin(EMetricType::GAUGE);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "infValue");
+ e->OnLabelsEnd();
+ }
+ e->OnDouble(TInstant::Zero(), INFINITY);
+ e->OnMetricEnd();
+ }
+ {
+ e->OnMetricBegin(EMetricType::DSUMMARY);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "seconds");
+ e->OnLabel("disk", "sdb1");
+ e->OnLabelsEnd();
+ }
+ e->OnSummaryDouble(TInstant::Zero(), TestSummaryDouble());
+ e->OnMetricEnd();
+ }
+ e->OnStreamEnd();
+ });
+
+ UNIT_ASSERT_STRINGS_EQUAL(result,
+ "# TYPE diskUsage gauge\n"
+ "diskUsage{disk=\"sda1\", } 1000\n"
+ "# TYPE memoryUsage gauge\n"
+ "memoryUsage{host=\"solomon-man-00\", dc=\"man\", } 1000 1512216000000\n"
+ "# TYPE bytesRx gauge\n"
+ "bytesRx{host=\"solomon-sas-01\", dc=\"sas\", } 8 1512216010000\n"
+ "diskUsage{disk=\"sdb1\", } 1001\n"
+ "# TYPE nanValue gauge\n"
+ "nanValue nan\n"
+ "# TYPE infValue gauge\n"
+ "infValue inf\n"
+ "seconds_sum{disk=\"sdb1\", } 10.1\n"
+ "seconds_min{disk=\"sdb1\", } -0.45\n"
+ "seconds_max{disk=\"sdb1\", } 0.478\n"
+ "seconds_last{disk=\"sdb1\", } 0.3\n"
+ "seconds_count{disk=\"sdb1\", } 30\n"
+ "\n");
+ }
+
+ Y_UNIT_TEST(IntGauges) {
+ auto result = EncodeToString([](IMetricEncoder* e) {
+ e->OnStreamBegin();
+ { // no values
+ e->OnMetricBegin(EMetricType::IGAUGE);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "cpuUsage");
+ e->OnLabelsEnd();
+ }
+ e->OnMetricEnd();
+ }
+ { // one value no ts
+ e->OnMetricBegin(EMetricType::IGAUGE);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "diskUsage");
+ e->OnLabel("disk", "sda1");
+ e->OnLabelsEnd();
+ }
+ e->OnInt64(TInstant::Zero(), 1000);
+ e->OnMetricEnd();
+ }
+ { // one value with ts
+ e->OnMetricBegin(EMetricType::IGAUGE);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "memoryUsage");
+ e->OnLabel("dc", "man");
+ e->OnLabel("host", "solomon-man-00");
+ e->OnLabelsEnd();
+ }
+ e->OnInt64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 1000);
+ e->OnMetricEnd();
+ }
+ { // many values
+ e->OnMetricBegin(EMetricType::IGAUGE);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "bytesRx");
+ e->OnLabel("dc", "sas");
+ e->OnLabel("host", "solomon-sas-01");
+ e->OnLabelsEnd();
+ }
+ e->OnInt64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 2);
+ e->OnInt64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:05Z"), 4);
+ e->OnInt64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:10Z"), 8);
+ e->OnMetricEnd();
+ }
+ e->OnStreamEnd();
+ });
+
+ UNIT_ASSERT_STRINGS_EQUAL(result,
+ "# TYPE diskUsage gauge\n"
+ "diskUsage{disk=\"sda1\", } 1000\n"
+ "# TYPE memoryUsage gauge\n"
+ "memoryUsage{dc=\"man\", host=\"solomon-man-00\", } 1000 1512216000000\n"
+ "# TYPE bytesRx gauge\n"
+ "bytesRx{dc=\"sas\", host=\"solomon-sas-01\", } 8 1512216010000\n"
+ "\n");
+ }
+
+ Y_UNIT_TEST(Counters) {
+ auto result = EncodeToString([](IMetricEncoder* e) {
+ e->OnStreamBegin();
+ { // no values
+ e->OnMetricBegin(EMetricType::COUNTER);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "cpuUsage");
+ e->OnLabelsEnd();
+ }
+ e->OnMetricEnd();
+ }
+ { // one value no ts
+ e->OnMetricBegin(EMetricType::COUNTER);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "diskUsage");
+ e->OnLabel("disk", "sda1");
+ e->OnLabelsEnd();
+ }
+ e->OnInt64(TInstant::Zero(), 1000);
+ e->OnMetricEnd();
+ }
+ { // one value with ts
+ e->OnMetricBegin(EMetricType::COUNTER);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "memoryUsage");
+ e->OnLabel("host", "solomon-man-00");
+ e->OnLabel("dc", "man");
+ e->OnLabelsEnd();
+ }
+ e->OnInt64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 1000);
+ e->OnMetricEnd();
+ }
+ { // many values
+ e->OnMetricBegin(EMetricType::COUNTER);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "bytesRx");
+ e->OnLabel("host", "solomon-sas-01");
+ e->OnLabel("dc", "sas");
+ e->OnLabelsEnd();
+ }
+ e->OnInt64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 2);
+ e->OnInt64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:05Z"), 4);
+ e->OnInt64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:10Z"), 8);
+ e->OnMetricEnd();
+ }
+ e->OnStreamEnd();
+ });
+
+ UNIT_ASSERT_STRINGS_EQUAL(result,
+ "# TYPE diskUsage counter\n"
+ "diskUsage{disk=\"sda1\", } 1000\n"
+ "# TYPE memoryUsage counter\n"
+ "memoryUsage{host=\"solomon-man-00\", dc=\"man\", } 1000 1512216000000\n"
+ "# TYPE bytesRx counter\n"
+ "bytesRx{host=\"solomon-sas-01\", dc=\"sas\", } 8 1512216010000\n"
+ "\n");
+ }
+
+ Y_UNIT_TEST(Histograms) {
+ auto result = EncodeToString([](IMetricEncoder* e) {
+ e->OnStreamBegin();
+ { // no values histogram
+ e->OnMetricBegin(EMetricType::HIST);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "cpuUsage");
+ e->OnLabelsEnd();
+ }
+ e->OnMetricEnd();
+ }
+ { // one value no ts
+ e->OnMetricBegin(EMetricType::HIST);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "inboundBytesPerSec");
+ e->OnLabel("client", "mbus");
+ e->OnLabelsEnd();
+ }
+ e->OnHistogram(
+ TInstant::Zero(),
+ ExplicitHistogramSnapshot({10, 20, HISTOGRAM_INF_BOUND}, {1, 4, 0}));
+ e->OnMetricEnd();
+ }
+ { // one value no ts no +inf bucket
+ e->OnMetricBegin(EMetricType::HIST);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "inboundBytesPerSec");
+ e->OnLabel("client", "grpc");
+ e->OnLabelsEnd();
+ }
+ e->OnHistogram(
+ TInstant::Zero(),
+ ExplicitHistogramSnapshot({10, 20, 30}, {1, 4, 0}));
+ e->OnMetricEnd();
+ }
+ { // one value with ts
+ e->OnMetricBegin(EMetricType::HIST_RATE);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "outboundBytesPerSec");
+ e->OnLabel("client", "grps");
+ e->OnLabelsEnd();
+ }
+ e->OnHistogram(
+ TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"),
+ ExplicitHistogramSnapshot({100, 200, HISTOGRAM_INF_BOUND}, {1, 0, 0}));
+ e->OnMetricEnd();
+ }
+ { // many values
+ e->OnMetricBegin(EMetricType::HIST);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "bytesRx");
+ e->OnLabel("host", "solomon-sas-01");
+ e->OnLabel("dc", "sas");
+ e->OnLabelsEnd();
+ }
+ TBucketBounds bounds = {100, 200, HISTOGRAM_INF_BOUND};
+ e->OnHistogram(
+ TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"),
+ ExplicitHistogramSnapshot(bounds, {10, 0, 0}));
+ e->OnHistogram(
+ TInstant::ParseIso8601Deprecated("2017-12-02T12:00:05Z"),
+ ExplicitHistogramSnapshot(bounds, {10, 2, 0}));
+ e->OnHistogram(
+ TInstant::ParseIso8601Deprecated("2017-12-02T12:00:10Z"),
+ ExplicitHistogramSnapshot(bounds, {10, 2, 5}));
+ e->OnMetricEnd();
+ }
+ e->OnStreamEnd();
+ });
+
+ UNIT_ASSERT_STRINGS_EQUAL(result,
+ "# TYPE inboundBytesPerSec histogram\n"
+ "inboundBytesPerSec_bucket{client=\"mbus\", le=\"10\"} 1\n"
+ "inboundBytesPerSec_bucket{client=\"mbus\", le=\"20\"} 5\n"
+ "inboundBytesPerSec_bucket{client=\"mbus\", le=\"+Inf\"} 5\n"
+ "inboundBytesPerSec_count{client=\"mbus\", } 5\n"
+ "inboundBytesPerSec_bucket{client=\"grpc\", le=\"10\"} 1\n"
+ "inboundBytesPerSec_bucket{client=\"grpc\", le=\"20\"} 5\n"
+ "inboundBytesPerSec_bucket{client=\"grpc\", le=\"30\"} 5\n"
+ "inboundBytesPerSec_count{client=\"grpc\", } 5\n"
+ "# TYPE outboundBytesPerSec histogram\n"
+ "outboundBytesPerSec_bucket{client=\"grps\", le=\"100\"} 1 1512216000000\n"
+ "outboundBytesPerSec_bucket{client=\"grps\", le=\"200\"} 1 1512216000000\n"
+ "outboundBytesPerSec_bucket{client=\"grps\", le=\"+Inf\"} 1 1512216000000\n"
+ "outboundBytesPerSec_count{client=\"grps\", } 1 1512216000000\n"
+ "# TYPE bytesRx histogram\n"
+ "bytesRx_bucket{host=\"solomon-sas-01\", dc=\"sas\", le=\"100\"} 10 1512216010000\n"
+ "bytesRx_bucket{host=\"solomon-sas-01\", dc=\"sas\", le=\"200\"} 12 1512216010000\n"
+ "bytesRx_bucket{host=\"solomon-sas-01\", dc=\"sas\", le=\"+Inf\"} 17 1512216010000\n"
+ "bytesRx_count{host=\"solomon-sas-01\", dc=\"sas\", } 17 1512216010000\n"
+ "\n");
+ }
+
+ Y_UNIT_TEST(CommonLables) {
+ auto result = EncodeToString([](IMetricEncoder* e) {
+ e->OnStreamBegin();
+ { // common time
+ e->OnCommonTime(TInstant::Seconds(1500000000));
+ }
+ { // common labels
+ e->OnLabelsBegin();
+ e->OnLabel("project", "solomon");
+ e->OnLabelsEnd();
+ }
+ { // metric #1
+ e->OnMetricBegin(EMetricType::COUNTER);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "single");
+ e->OnLabel("labels", "l1");
+ e->OnLabelsEnd();
+ }
+ e->OnUint64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:10Z"), 17);
+ e->OnMetricEnd();
+ }
+ { // metric #2
+ e->OnMetricBegin(EMetricType::COUNTER);
+ {
+ e->OnLabelsBegin();
+ e->OnLabel("sensor", "two");
+ e->OnLabel("labels", "l2");
+ e->OnLabelsEnd();
+ }
+ e->OnUint64(TInstant::Zero(), 42);
+ e->OnMetricEnd();
+ }
+ e->OnStreamEnd();
+ });
+ UNIT_ASSERT_STRINGS_EQUAL(result,
+R"(# TYPE single counter
+single{labels="l1", project="solomon", } 17 1512216010000
+# TYPE two counter
+two{labels="l2", project="solomon", } 42 1500000000000
+
+)");
+ }
+}
diff --git a/library/cpp/monlib/encode/prometheus/prometheus_model.h b/library/cpp/monlib/encode/prometheus/prometheus_model.h
new file mode 100644
index 0000000000..cb7f2cb15b
--- /dev/null
+++ b/library/cpp/monlib/encode/prometheus/prometheus_model.h
@@ -0,0 +1,70 @@
+#pragma once
+
+#include <util/generic/strbuf.h>
+
+
+namespace NMonitoring {
+namespace NPrometheus {
+
+ //
+ // Prometheus specific names and validation rules.
+ //
+ // See https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md
+ // and https://github.com/prometheus/common/blob/master/expfmt/text_parse.go
+ //
+
+ inline constexpr TStringBuf BUCKET_SUFFIX = "_bucket";
+ inline constexpr TStringBuf COUNT_SUFFIX = "_count";
+ inline constexpr TStringBuf SUM_SUFFIX = "_sum";
+ inline constexpr TStringBuf MIN_SUFFIX = "_min";
+ inline constexpr TStringBuf MAX_SUFFIX = "_max";
+ inline constexpr TStringBuf LAST_SUFFIX = "_last";
+
+ // Used for the label that defines the upper bound of a bucket of a
+ // histogram ("le" -> "less or equal").
+ inline constexpr TStringBuf BUCKET_LABEL = "le";
+
+
+ inline bool IsValidLabelNameStart(char ch) {
+ return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || ch == '_';
+ }
+
+ inline bool IsValidLabelNameContinuation(char ch) {
+ return IsValidLabelNameStart(ch) || (ch >= '0' && ch <= '9');
+ }
+
+ inline bool IsValidMetricNameStart(char ch) {
+ return IsValidLabelNameStart(ch) || ch == ':';
+ }
+
+ inline bool IsValidMetricNameContinuation(char ch) {
+ return IsValidLabelNameContinuation(ch) || ch == ':';
+ }
+
+ inline bool IsSum(TStringBuf name) {
+ return name.EndsWith(SUM_SUFFIX);
+ }
+
+ inline bool IsCount(TStringBuf name) {
+ return name.EndsWith(COUNT_SUFFIX);
+ }
+
+ inline bool IsBucket(TStringBuf name) {
+ return name.EndsWith(BUCKET_SUFFIX);
+ }
+
+ inline TStringBuf ToBaseName(TStringBuf name) {
+ if (IsBucket(name)) {
+ return name.SubString(0, name.length() - BUCKET_SUFFIX.length());
+ }
+ if (IsCount(name)) {
+ return name.SubString(0, name.length() - COUNT_SUFFIX.length());
+ }
+ if (IsSum(name)) {
+ return name.SubString(0, name.length() - SUM_SUFFIX.length());
+ }
+ return name;
+ }
+
+} // namespace NPrometheus
+} // namespace NMonitoring
diff --git a/library/cpp/monlib/encode/prometheus/ut/ya.make b/library/cpp/monlib/encode/prometheus/ut/ya.make
new file mode 100644
index 0000000000..fc468ffb68
--- /dev/null
+++ b/library/cpp/monlib/encode/prometheus/ut/ya.make
@@ -0,0 +1,17 @@
+UNITTEST_FOR(library/cpp/monlib/encode/prometheus)
+
+OWNER(
+ g:solomon
+ jamel
+)
+
+SRCS(
+ prometheus_encoder_ut.cpp
+ prometheus_decoder_ut.cpp
+)
+
+PEERDIR(
+ library/cpp/monlib/encode/protobuf
+)
+
+END()
diff --git a/library/cpp/monlib/encode/prometheus/ya.make b/library/cpp/monlib/encode/prometheus/ya.make
new file mode 100644
index 0000000000..7f2483b166
--- /dev/null
+++ b/library/cpp/monlib/encode/prometheus/ya.make
@@ -0,0 +1,17 @@
+LIBRARY()
+
+OWNER(
+ jamel
+ g:solomon
+)
+
+SRCS(
+ prometheus_decoder.cpp
+ prometheus_encoder.cpp
+)
+
+PEERDIR(
+ library/cpp/monlib/encode
+)
+
+END()