diff options
author | Devtools Arcadia <arcadia-devtools@yandex-team.ru> | 2022-02-07 18:08:42 +0300 |
---|---|---|
committer | Devtools Arcadia <arcadia-devtools@mous.vla.yp-c.yandex.net> | 2022-02-07 18:08:42 +0300 |
commit | 1110808a9d39d4b808aef724c861a2e1a38d2a69 (patch) | |
tree | e26c9fed0de5d9873cce7e00bc214573dc2195b7 /library/cpp/monlib/encode/prometheus | |
download | ydb-1110808a9d39d4b808aef724c861a2e1a38d2a69.tar.gz |
intermediate changes
ref:cde9a383711a11544ce7e107a78147fb96cc4029
Diffstat (limited to 'library/cpp/monlib/encode/prometheus')
10 files changed, 2058 insertions, 0 deletions
diff --git a/library/cpp/monlib/encode/prometheus/fuzz/main.cpp b/library/cpp/monlib/encode/prometheus/fuzz/main.cpp new file mode 100644 index 0000000000..24bda2d32e --- /dev/null +++ b/library/cpp/monlib/encode/prometheus/fuzz/main.cpp @@ -0,0 +1,18 @@ +#include <library/cpp/monlib/encode/prometheus/prometheus.h> +#include <library/cpp/monlib/encode/fake/fake.h> + +#include <util/stream/mem.h> + + +extern "C" int LLVMFuzzerTestOneInput(const ui8* buf, size_t size) { + using namespace NMonitoring; + + try { + TStringBuf data(reinterpret_cast<const char*>(buf), size); + auto encoder = EncoderFake(); + DecodePrometheus(data, encoder.Get()); + } catch (...) { + } + + return 0; +} diff --git a/library/cpp/monlib/encode/prometheus/fuzz/ya.make b/library/cpp/monlib/encode/prometheus/fuzz/ya.make new file mode 100644 index 0000000000..4a6c796ed5 --- /dev/null +++ b/library/cpp/monlib/encode/prometheus/fuzz/ya.make @@ -0,0 +1,16 @@ +FUZZ() + +OWNER(g:solomon jamel) + +PEERDIR( + library/cpp/monlib/encode/prometheus + library/cpp/monlib/encode/fake +) + +SIZE(MEDIUM) + +SRCS( + main.cpp +) + +END() diff --git a/library/cpp/monlib/encode/prometheus/prometheus.h b/library/cpp/monlib/encode/prometheus/prometheus.h new file mode 100644 index 0000000000..2e7fa31c28 --- /dev/null +++ b/library/cpp/monlib/encode/prometheus/prometheus.h @@ -0,0 +1,18 @@ +#pragma once + +#include <library/cpp/monlib/encode/encoder.h> +#include <library/cpp/monlib/encode/format.h> + +#include <util/generic/yexception.h> + + +namespace NMonitoring { + + class TPrometheusDecodeException: public yexception { + }; + + IMetricEncoderPtr EncoderPrometheus(IOutputStream* out, TStringBuf metricNameLabel = "sensor"); + + void DecodePrometheus(TStringBuf data, IMetricConsumer* c, TStringBuf metricNameLabel = "sensor"); + +} diff --git a/library/cpp/monlib/encode/prometheus/prometheus_decoder.cpp b/library/cpp/monlib/encode/prometheus/prometheus_decoder.cpp new file mode 100644 index 0000000000..7e81357dbd --- /dev/null +++ b/library/cpp/monlib/encode/prometheus/prometheus_decoder.cpp @@ -0,0 +1,597 @@ +#include "prometheus.h" +#include "prometheus_model.h" + +#include <library/cpp/monlib/metrics/histogram_snapshot.h> +#include <library/cpp/monlib/metrics/metric.h> + +#include <util/datetime/base.h> +#include <util/generic/hash.h> +#include <util/string/cast.h> +#include <util/string/builder.h> +#include <util/generic/maybe.h> +#include <util/string/ascii.h> + +#include <cmath> + +#define Y_PARSER_FAIL(message) \ + ythrow ::NMonitoring::TPrometheusDecodeException() << message << " at line #" << CurrentLine_ + +#define Y_PARSER_ENSURE(cond, message) \ + Y_ENSURE_EX(cond, ::NMonitoring::TPrometheusDecodeException() << message << " at line #" << CurrentLine_) + + +namespace NMonitoring { + namespace { + constexpr ui32 MAX_LABEL_VALUE_LEN = 256; + + using TLabelsMap = THashMap<TString, TString>; + + TString LabelsToStr(const TLabelsMap& labels) { + TStringBuilder sb; + auto it = labels.begin(); + auto end = labels.end(); + + sb << '{'; + while (it != end) { + sb << it->first; + sb << '='; + sb << '"' << it->second << '"'; + + ++it; + if (it != end) { + sb << ", "; + } + } + sb << '}'; + return sb; + } + + template <typename T, typename U> + bool TryStaticCast(U val, T& out) { + static_assert(std::is_arithmetic_v<U>); + if constexpr (std::is_floating_point_v<T> || std::is_floating_point_v<U>) { + if (val > MaxFloor<T>() || val < -MaxFloor<T>()) { + return false; + } + + } else { + if (val > Max<T>() || val < Min<T>()) { + return false; + } + } + + out = static_cast<T>(val); + return true; + } + + /////////////////////////////////////////////////////////////////////// + // THistogramBuilder + /////////////////////////////////////////////////////////////////////// + class THistogramBuilder { + using TBucketData = std::pair<TBucketBound, TBucketValue>; + constexpr static TBucketData ZERO_BUCKET = { -std::numeric_limits<TBucketBound>::max(), 0 }; + public: + TStringBuf GetName() const noexcept { + return Name_; + } + + void SetName(TStringBuf name) noexcept { + Name_ = name; + } + + const TLabelsMap& GetLabels() const noexcept { + return *Labels_; + } + + void SetLabels(TLabelsMap&& labels) { + if (Labels_.Defined()) { + Y_ENSURE(Labels_ == labels, + "mixed labels in one histogram, prev: " << LabelsToStr(*Labels_) << + ", current: " << LabelsToStr(labels)); + } else { + Labels_.ConstructInPlace(std::move(labels)); + } + } + + TInstant GetTime() const noexcept { + return Time_; + } + + void SetTime(TInstant time) noexcept { + Time_ = time; + } + + bool Empty() const noexcept { + return Bounds_.empty(); + } + + bool Same(TStringBuf name, const TLabelsMap& labels) const noexcept { + return Name_ == name && Labels_ == labels; + } + + void AddBucket(TBucketBound bound, TBucketValue value) { + Y_ENSURE_EX(PrevBucket_.first < bound, TPrometheusDecodeException() << + "invalid order of histogram bounds " << PrevBucket_.first << + " >= " << bound); + + Y_ENSURE_EX(PrevBucket_.second <= value, TPrometheusDecodeException() << + "invalid order of histogram bucket values " << PrevBucket_.second << + " > " << value); + + // convert infinite bound value + if (bound == std::numeric_limits<TBucketBound>::infinity()) { + bound = HISTOGRAM_INF_BOUND; + } + + Bounds_.push_back(bound); + Values_.push_back(value - PrevBucket_.second); // keep only delta between buckets + + PrevBucket_ = { bound, value }; + } + + // will clear builder state + IHistogramSnapshotPtr ToSnapshot() { + Y_ENSURE_EX(!Empty(), TPrometheusDecodeException() << "histogram cannot be empty"); + Time_ = TInstant::Zero(); + PrevBucket_ = ZERO_BUCKET; + Labels_.Clear(); + auto snapshot = ExplicitHistogramSnapshot(Bounds_, Values_); + + Bounds_.clear(); + Values_.clear(); + + return snapshot; + } + + private: + TStringBuf Name_; + TMaybe<TLabelsMap> Labels_; + TInstant Time_; + TBucketBounds Bounds_; + TBucketValues Values_; + TBucketData PrevBucket_ = ZERO_BUCKET; + }; + + /////////////////////////////////////////////////////////////////////// + // EPrometheusMetricType + /////////////////////////////////////////////////////////////////////// + enum class EPrometheusMetricType { + GAUGE, + COUNTER, + SUMMARY, + UNTYPED, + HISTOGRAM, + }; + + /////////////////////////////////////////////////////////////////////// + // TPrometheusReader + /////////////////////////////////////////////////////////////////////// + class TPrometheusReader { + public: + TPrometheusReader(TStringBuf data, IMetricConsumer* c, TStringBuf metricNameLabel) + : Data_(data) + , Consumer_(c) + , MetricNameLabel_(metricNameLabel) + { + } + + void Read() { + Consumer_->OnStreamBegin(); + + if (HasRemaining()) { + ReadNextByte(); + SkipSpaces(); + + try { + while (HasRemaining()) { + switch (CurrentByte_) { + case '\n': + ReadNextByte(); // skip '\n' + CurrentLine_++; + SkipSpaces(); + break; + case '#': + ParseComment(); + break; + default: + ParseMetric(); + break; + } + } + + if (!HistogramBuilder_.Empty()) { + ConsumeHistogram(); + } + } catch (const TPrometheusDecodeException& e) { + throw e; + } catch (...) { + Y_PARSER_FAIL("unexpected error " << CurrentExceptionMessage()); + } + } + + Consumer_->OnStreamEnd(); + } + + private: + bool HasRemaining() const noexcept { + return CurrentPos_ < Data_.Size(); + } + + // # 'TYPE' metric_name {counter|gauge|histogram|summary|untyped} + // # 'HELP' metric_name some help info + // # general comment message + void ParseComment() { + SkipExpectedChar('#'); + SkipSpaces(); + + TStringBuf keyword = ReadToken(); + if (keyword == TStringBuf("TYPE")) { + SkipSpaces(); + + TStringBuf nextName = ReadTokenAsMetricName(); + Y_PARSER_ENSURE(!nextName.Empty(), "invalid metric name"); + + SkipSpaces(); + EPrometheusMetricType nextType = ReadType(); + + bool inserted = SeenTypes_.emplace(nextName, nextType).second; + Y_PARSER_ENSURE(inserted, "second TYPE line for metric " << nextName); + + if (nextType == EPrometheusMetricType::HISTOGRAM) { + if (!HistogramBuilder_.Empty()) { + ConsumeHistogram(); + } + HistogramBuilder_.SetName(nextName); + } + } else { + // skip HELP and general comments + SkipUntilEol(); + } + + Y_PARSER_ENSURE(CurrentByte_ == '\n', "expected '\\n', found '" << CurrentByte_ << '\''); + } + + // metric_name [labels] value [timestamp] + void ParseMetric() { + TStringBuf name = ReadTokenAsMetricName(); + SkipSpaces(); + + TLabelsMap labels = ReadLabels(); + SkipSpaces(); + + double value = ParseGoDouble(ReadToken()); + SkipSpaces(); + + TInstant time = TInstant::Zero(); + if (CurrentByte_ != '\n') { + time = TInstant::MilliSeconds(FromString<ui64>(ReadToken())); + } + + TStringBuf baseName = name; + EPrometheusMetricType type = EPrometheusMetricType::UNTYPED; + + if (auto* seenType = SeenTypes_.FindPtr(name)) { + type = *seenType; + } else { + baseName = NPrometheus::ToBaseName(name); + if (auto* baseType = SeenTypes_.FindPtr(baseName)) { + type = *baseType; + } + } + + switch (type) { + case EPrometheusMetricType::HISTOGRAM: + if (NPrometheus::IsBucket(name)) { + double bound = 0.0; + auto it = labels.find(NPrometheus::BUCKET_LABEL); + if (it != labels.end()) { + bound = ParseGoDouble(it->second); + labels.erase(it); + } else { + Y_PARSER_FAIL( + "metric " << name << "has no " << NPrometheus::BUCKET_LABEL << + " label at line #" << CurrentLine_); + } + + if (!HistogramBuilder_.Empty() && !HistogramBuilder_.Same(baseName, labels)) { + ConsumeHistogram(); + HistogramBuilder_.SetName(baseName); + } + + TBucketValue bucketVal; + Y_PARSER_ENSURE(TryStaticCast(value, bucketVal), "Cannot convert " << value << " to bucket value type"); + HistogramBuilder_.AddBucket(bound, bucketVal); + HistogramBuilder_.SetTime(time); + HistogramBuilder_.SetLabels(std::move(labels)); + } else if (NPrometheus::IsCount(name)) { + // translate x_count metric as COUNTER metric + ConsumeCounter(name, labels, time, value); + } else if (NPrometheus::IsSum(name)) { + // translate x_sum metric as GAUGE metric + ConsumeGauge(name, labels, time, value); + } else { + Y_PARSER_FAIL( + "metric " << name << + " should be part of HISTOGRAM " << baseName); + } + break; + + case EPrometheusMetricType::SUMMARY: + if (NPrometheus::IsCount(name)) { + // translate x_count metric as COUNTER metric + ConsumeCounter(name, labels, time, value); + } else if (NPrometheus::IsSum(name)) { + // translate x_sum metric as GAUGE metric + ConsumeGauge(name, labels, time, value); + } else { + ConsumeGauge(name, labels, time, value); + } + break; + + case EPrometheusMetricType::COUNTER: + ConsumeCounter(name, labels, time, value); + break; + + case EPrometheusMetricType::GAUGE: + ConsumeGauge(name, labels, time, value); + break; + + case EPrometheusMetricType::UNTYPED: + ConsumeGauge(name, labels, time, value); + break; + } + + Y_PARSER_ENSURE(CurrentByte_ == '\n', "expected '\\n', found '" << CurrentByte_ << '\''); + } + + // { name = "value", name2 = "value2", } + TLabelsMap ReadLabels() { + TLabelsMap labels; + if (CurrentByte_ != '{') { + return labels; + } + + SkipExpectedChar('{'); + SkipSpaces(); + + while (CurrentByte_ != '}') { + TStringBuf name = ReadTokenAsLabelName(); + SkipSpaces(); + + SkipExpectedChar('='); + SkipSpaces(); + + TString value = ReadTokenAsLabelValue(); + SkipSpaces(); + labels.emplace(name, value); + + if (CurrentByte_ == ',') { + SkipExpectedChar(','); + SkipSpaces(); + } + } + + SkipExpectedChar('}'); + return labels; + } + + EPrometheusMetricType ReadType() { + TStringBuf keyword = ReadToken(); + if (AsciiEqualsIgnoreCase(keyword, "GAUGE")) { + return EPrometheusMetricType::GAUGE; + } else if (AsciiEqualsIgnoreCase(keyword, "COUNTER")) { + return EPrometheusMetricType::COUNTER; + } else if (AsciiEqualsIgnoreCase(keyword, "SUMMARY")) { + return EPrometheusMetricType::SUMMARY; + } else if (AsciiEqualsIgnoreCase(keyword, "HISTOGRAM")) { + return EPrometheusMetricType::HISTOGRAM; + } else if (AsciiEqualsIgnoreCase(keyword, "UNTYPED")) { + return EPrometheusMetricType::UNTYPED; + } + + Y_PARSER_FAIL( + "unknown metric type: " << keyword << + " at line #" << CurrentLine_); + } + + Y_FORCE_INLINE void ReadNextByteUnsafe() { + CurrentByte_ = Data_[CurrentPos_++]; + } + + Y_FORCE_INLINE bool IsSpace(char ch) { + return ch == ' ' || ch == '\t'; + } + + void ReadNextByte() { + Y_PARSER_ENSURE(HasRemaining(), "unexpected end of file"); + ReadNextByteUnsafe(); + } + + void SkipExpectedChar(char ch) { + Y_PARSER_ENSURE(CurrentByte_ == ch, + "expected '" << CurrentByte_ << "', found '" << ch << '\''); + ReadNextByte(); + } + + void SkipSpaces() { + while (HasRemaining() && IsSpace(CurrentByte_)) { + ReadNextByteUnsafe(); + } + } + + void SkipUntilEol() { + while (HasRemaining() && CurrentByte_ != '\n') { + ReadNextByteUnsafe(); + } + } + + TStringBuf ReadToken() { + Y_VERIFY_DEBUG(CurrentPos_ > 0); + size_t begin = CurrentPos_ - 1; // read first byte again + while (HasRemaining() && !IsSpace(CurrentByte_) && CurrentByte_ != '\n') { + ReadNextByteUnsafe(); + } + return TokenFromPos(begin); + } + + TStringBuf ReadTokenAsMetricName() { + if (!NPrometheus::IsValidMetricNameStart(CurrentByte_)) { + return ""; + } + + Y_VERIFY_DEBUG(CurrentPos_ > 0); + size_t begin = CurrentPos_ - 1; // read first byte again + while (HasRemaining()) { + ReadNextByteUnsafe(); + if (!NPrometheus::IsValidMetricNameContinuation(CurrentByte_)) { + break; + } + } + return TokenFromPos(begin); + } + + TStringBuf ReadTokenAsLabelName() { + if (!NPrometheus::IsValidLabelNameStart(CurrentByte_)) { + return ""; + } + + Y_VERIFY_DEBUG(CurrentPos_ > 0); + size_t begin = CurrentPos_ - 1; // read first byte again + while (HasRemaining()) { + ReadNextByteUnsafe(); + if (!NPrometheus::IsValidLabelNameContinuation(CurrentByte_)) { + break; + } + } + return TokenFromPos(begin); + } + + TString ReadTokenAsLabelValue() { + TString labelValue; + + SkipExpectedChar('"'); + for (ui32 i = 0; i < MAX_LABEL_VALUE_LEN; i++) { + switch (CurrentByte_) { + case '"': + SkipExpectedChar('"'); + return labelValue; + + case '\n': + Y_PARSER_FAIL("label value contains unescaped new-line"); + + case '\\': + ReadNextByte(); + switch (CurrentByte_) { + case '"': + case '\\': + labelValue.append(CurrentByte_); + break; + case 'n': + labelValue.append('\n'); + break; + default: + Y_PARSER_FAIL("invalid escape sequence '" << CurrentByte_ << '\''); + } + break; + + default: + labelValue.append(CurrentByte_); + break; + } + + ReadNextByte(); + } + + Y_PARSER_FAIL("trying to parse too long label value, size >= " << MAX_LABEL_VALUE_LEN); + } + + TStringBuf TokenFromPos(size_t begin) { + Y_VERIFY_DEBUG(CurrentPos_ > begin); + size_t len = CurrentPos_ - begin - 1; + if (len == 0) { + return {}; + } + + return Data_.SubString(begin, len); + } + + void ConsumeLabels(TStringBuf name, const TLabelsMap& labels) { + Y_PARSER_ENSURE(labels.count(MetricNameLabel_) == 0, + "label name '" << MetricNameLabel_ << + "' is reserved, but is used with metric: " << name << LabelsToStr(labels)); + + Consumer_->OnLabelsBegin(); + Consumer_->OnLabel(MetricNameLabel_, TString(name)); // TODO: remove this string allocation + for (const auto& it: labels) { + Consumer_->OnLabel(it.first, it.second); + } + Consumer_->OnLabelsEnd(); + } + + void ConsumeCounter(TStringBuf name, const TLabelsMap& labels, TInstant time, double value) { + i64 intValue{0}; + // not nan + if (value == value) { + Y_PARSER_ENSURE(TryStaticCast(value, intValue), "value " << value << " is out of range"); + } + + // see https://st.yandex-team.ru/SOLOMON-4142 for more details + // why we convert Prometheus COUNTER into Solomon RATE + // TODO: need to fix after server-side aggregation become correct for COUNTERs + Consumer_->OnMetricBegin(EMetricType::RATE); + ConsumeLabels(name, labels); + Consumer_->OnUint64(time, intValue); + Consumer_->OnMetricEnd(); + } + + void ConsumeGauge(TStringBuf name, const TLabelsMap& labels, TInstant time, double value) { + Consumer_->OnMetricBegin(EMetricType::GAUGE); + ConsumeLabels(name, labels); + Consumer_->OnDouble(time, value); + Consumer_->OnMetricEnd(); + } + + void ConsumeHistogram() { + Consumer_->OnMetricBegin(EMetricType::HIST_RATE); + ConsumeLabels(HistogramBuilder_.GetName(), HistogramBuilder_.GetLabels()); + auto time = HistogramBuilder_.GetTime(); + auto hist = HistogramBuilder_.ToSnapshot(); + Consumer_->OnHistogram(time, std::move(hist)); + Consumer_->OnMetricEnd(); + } + + double ParseGoDouble(TStringBuf str) { + if (str == TStringBuf("+Inf")) { + return std::numeric_limits<double>::infinity(); + } else if (str == TStringBuf("-Inf")) { + return -std::numeric_limits<double>::infinity(); + } else if (str == TStringBuf("NaN")) { + return NAN; + } + + double r = 0.0; + if (TryFromString(str, r)) { + return r; + } + Y_PARSER_FAIL("cannot parse double value from '" << str << "\' at line #" << CurrentLine_); + } + + private: + TStringBuf Data_; + IMetricConsumer* Consumer_; + TStringBuf MetricNameLabel_; + THashMap<TString, EPrometheusMetricType> SeenTypes_; + THistogramBuilder HistogramBuilder_; + + ui32 CurrentLine_ = 1; + ui32 CurrentPos_ = 0; + char CurrentByte_ = 0; + }; + } // namespace + +void DecodePrometheus(TStringBuf data, IMetricConsumer* c, TStringBuf metricNameLabel) { + TPrometheusReader reader(data, c, metricNameLabel); + reader.Read(); +} + +} // namespace NMonitoring diff --git a/library/cpp/monlib/encode/prometheus/prometheus_decoder_ut.cpp b/library/cpp/monlib/encode/prometheus/prometheus_decoder_ut.cpp new file mode 100644 index 0000000000..49c2244fb4 --- /dev/null +++ b/library/cpp/monlib/encode/prometheus/prometheus_decoder_ut.cpp @@ -0,0 +1,478 @@ +#include "prometheus.h" + +#include <library/cpp/monlib/encode/protobuf/protobuf.h> + +#include <library/cpp/testing/unittest/registar.h> + +using namespace NMonitoring; + +#define ASSERT_LABEL_EQUAL(label, name, value) do { \ + UNIT_ASSERT_STRINGS_EQUAL((label).GetName(), name); \ + UNIT_ASSERT_STRINGS_EQUAL((label).GetValue(), value); \ + } while (false) + +#define ASSERT_DOUBLE_POINT(s, time, value) do { \ + UNIT_ASSERT_VALUES_EQUAL((s).GetTime(), (time).MilliSeconds()); \ + UNIT_ASSERT_EQUAL((s).GetValueCase(), NProto::TSingleSample::kFloat64); \ + UNIT_ASSERT_DOUBLES_EQUAL((s).GetFloat64(), value, std::numeric_limits<double>::epsilon()); \ + } while (false) + +#define ASSERT_UINT_POINT(s, time, value) do { \ + UNIT_ASSERT_VALUES_EQUAL((s).GetTime(), (time).MilliSeconds()); \ + UNIT_ASSERT_EQUAL((s).GetValueCase(), NProto::TSingleSample::kUint64); \ + UNIT_ASSERT_VALUES_EQUAL((s).GetUint64(), value); \ + } while (false) + +#define ASSERT_HIST_POINT(s, time, expected) do { \ + UNIT_ASSERT_VALUES_EQUAL((s).GetTime(), time.MilliSeconds()); \ + UNIT_ASSERT_EQUAL((s).GetValueCase(), NProto::TSingleSample::kHistogram);\ + UNIT_ASSERT_VALUES_EQUAL((s).GetHistogram().BoundsSize(), (expected).Count()); \ + UNIT_ASSERT_VALUES_EQUAL((s).GetHistogram().ValuesSize(), (expected).Count()); \ + for (size_t i = 0; i < (s).GetHistogram().BoundsSize(); i++) { \ + UNIT_ASSERT_DOUBLES_EQUAL((s).GetHistogram().GetBounds(i), (expected).UpperBound(i), Min<double>()); \ + UNIT_ASSERT_VALUES_EQUAL((s).GetHistogram().GetValues(i), (expected).Value(i)); \ + } \ + } while (false) + +Y_UNIT_TEST_SUITE(TPrometheusDecoderTest) { + + NProto::TSingleSamplesList Decode(TStringBuf data) { + NProto::TSingleSamplesList samples; + { + IMetricEncoderPtr e = EncoderProtobuf(&samples); + DecodePrometheus(data, e.Get()); + } + return samples; + } + + Y_UNIT_TEST(Empty) { + { + auto samples = Decode(""); + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 0); + } + { + auto samples = Decode("\n"); + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 0); + } + { + auto samples = Decode("\n \n \n"); + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 0); + } + { + auto samples = Decode("\t\n\t\n"); + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 0); + } + } + + Y_UNIT_TEST(Minimal) { + auto samples = Decode( + "minimal_metric 1.234\n" + "another_metric -3e3 103948\n" + "# Even that:\n" + "no_labels{} 3\n" + "# HELP line for non-existing metric will be ignored.\n"); + + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 3); + { + auto& s = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(1, s.LabelsSize()); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "minimal_metric"); + ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 1.234); + } + { + auto& s = samples.GetSamples(1); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 1); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "another_metric"); + ASSERT_DOUBLE_POINT(s, TInstant::MilliSeconds(103948), -3000.0); + } + { + auto& s = samples.GetSamples(2); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(1, s.LabelsSize()); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "no_labels"); + ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 3.0); + } + } + + Y_UNIT_TEST(Counter) { + auto samples = Decode( + "# A normal comment.\n" + "#\n" + "# TYPE name counter\n" + "name{labelname=\"val1\",basename=\"basevalue\"} NaN\n" + "name {labelname=\"val2\",basename=\"basevalue\"} 2.3 1234567890\n" + "# HELP name two-line\\n doc str\\\\ing\n"); + + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 2); + + { + auto& s = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 3); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "name"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "basename", "basevalue"); + ASSERT_LABEL_EQUAL(s.GetLabels(2), "labelname", "val1"); + ASSERT_UINT_POINT(s, TInstant::Zero(), ui64(0)); + } + { + auto& s = samples.GetSamples(1); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 3); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "name"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "basename", "basevalue"); + ASSERT_LABEL_EQUAL(s.GetLabels(2), "labelname", "val2"); + ASSERT_UINT_POINT(s, TInstant::MilliSeconds(1234567890), i64(2)); + } + } + + Y_UNIT_TEST(Gauge) { + auto samples = Decode( + "# A normal comment.\n" + "#\n" + " # HELP name2 \tdoc str\"ing 2\n" + " # TYPE name2 gauge\n" + "name2{labelname=\"val2\"\t,basename = \"basevalue2\"\t\t} +Inf 54321\n" + "name2{ labelname = \"val1\" , }-Inf\n"); + + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 2); + + { + auto& s = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 3); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "name2"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "basename", "basevalue2"); + ASSERT_LABEL_EQUAL(s.GetLabels(2), "labelname", "val2"); + ASSERT_DOUBLE_POINT(s, TInstant::MilliSeconds(54321), INFINITY); + } + { + auto& s = samples.GetSamples(1); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "name2"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "labelname", "val1"); + ASSERT_DOUBLE_POINT(s, TInstant::Zero(), -INFINITY); + } + } + + Y_UNIT_TEST(Summary) { + auto samples = Decode( + "# HELP \n" + "# TYPE my_summary summary\n" + "my_summary{n1=\"val1\",quantile=\"0.5\"} 110\n" + "my_summary{n1=\"val1\",quantile=\"0.9\"} 140 1\n" + "my_summary_count{n1=\"val1\"} 42\n" + "my_summary_sum{n1=\"val1\"} 08 15\n" + "# some\n" + "# funny comments\n" + "# HELP\n" + "# HELP my_summary\n" + "# HELP my_summary \n"); + + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 4); + + { + auto& s = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 3); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "my_summary"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "quantile", "0.5"); + ASSERT_LABEL_EQUAL(s.GetLabels(2), "n1", "val1"); + ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 110.0); + } + { + auto& s = samples.GetSamples(1); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 3); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "my_summary"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "quantile", "0.9"); + ASSERT_LABEL_EQUAL(s.GetLabels(2), "n1", "val1"); + ASSERT_DOUBLE_POINT(s, TInstant::MilliSeconds(1), 140.0); + } + { + auto& s = samples.GetSamples(2); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "my_summary_count"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "n1", "val1"); + ASSERT_UINT_POINT(s, TInstant::Zero(), 42); + } + { + auto& s = samples.GetSamples(3); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "my_summary_sum"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "n1", "val1"); + ASSERT_DOUBLE_POINT(s, TInstant::MilliSeconds(15), 8.0); + } + } + + Y_UNIT_TEST(Histogram) { + auto samples = Decode( + "# HELP request_duration_microseconds The response latency.\n" + "# TYPE request_duration_microseconds histogram\n" + "request_duration_microseconds_bucket{le=\"0\"} 0\n" + "request_duration_microseconds_bucket{le=\"100\"} 123\n" + "request_duration_microseconds_bucket{le=\"120\"} 412\n" + "request_duration_microseconds_bucket{le=\"144\"} 592\n" + "request_duration_microseconds_bucket{le=\"172.8\"} 1524\n" + "request_duration_microseconds_bucket{le=\"+Inf\"} 2693\n" + "request_duration_microseconds_sum 1.7560473e+06\n" + "request_duration_microseconds_count 2693\n"); + + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 3); + + { + auto& s = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 1); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "request_duration_microseconds_sum"); + ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 1756047.3); + } + { + auto& s = samples.GetSamples(1); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 1); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "request_duration_microseconds_count"); + ASSERT_UINT_POINT(s, TInstant::Zero(), 2693); + } + { + auto& s = samples.GetSamples(2); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::HIST_RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 1); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "request_duration_microseconds"); + auto hist = ExplicitHistogramSnapshot( + { 0, 100, 120, 144, 172.8, HISTOGRAM_INF_BOUND }, + { 0, 123, 289, 180, 932, 1169 }); + ASSERT_HIST_POINT(s, TInstant::Zero(), *hist); + } + } + + Y_UNIT_TEST(HistogramWithLabels) { + auto samples = Decode( + "# A histogram, which has a pretty complex representation in the text format:\n" + "# HELP http_request_duration_seconds A histogram of the request duration.\n" + "# TYPE http_request_duration_seconds histogram\n" + "http_request_duration_seconds_bucket{le=\"0.05\", method=\"POST\"} 24054\n" + "http_request_duration_seconds_bucket{method=\"POST\", le=\"0.1\"} 33444\n" + "http_request_duration_seconds_bucket{le=\"0.2\", method=\"POST\", } 100392\n" + "http_request_duration_seconds_bucket{le=\"0.5\",method=\"POST\",} 129389\n" + "http_request_duration_seconds_bucket{ method=\"POST\", le=\"1\", } 133988\n" + "http_request_duration_seconds_bucket{ le=\"+Inf\", method=\"POST\", } 144320\n" + "http_request_duration_seconds_sum{method=\"POST\"} 53423\n" + "http_request_duration_seconds_count{ method=\"POST\", } 144320\n"); + + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 3); + + { + auto& s = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "http_request_duration_seconds_sum"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "method", "POST"); + ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 53423.0); + } + { + auto& s = samples.GetSamples(1); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "http_request_duration_seconds_count"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "method", "POST"); + ASSERT_UINT_POINT(s, TInstant::Zero(), 144320); + } + { + auto& s = samples.GetSamples(2); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::HIST_RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "http_request_duration_seconds"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "method", "POST"); + auto hist = ExplicitHistogramSnapshot( + { 0.05, 0.1, 0.2, 0.5, 1, HISTOGRAM_INF_BOUND }, + { 24054, 9390, 66948, 28997, 4599, 10332 }); + ASSERT_HIST_POINT(s, TInstant::Zero(), *hist); + } + } + + Y_UNIT_TEST(MultipleHistograms) { + auto samples = Decode( + "# TYPE inboundBytesPerSec histogram\n" + "inboundBytesPerSec_bucket{client=\"mbus\", le=\"10.0\"} 1.0\n" + "inboundBytesPerSec_bucket{client=\"mbus\", le=\"20.0\"} 5.0\n" + "inboundBytesPerSec_bucket{client=\"mbus\", le=\"+Inf\"} 5.0\n" + "inboundBytesPerSec_count{client=\"mbus\"} 5.0\n" + "inboundBytesPerSec_bucket{client=\"grpc\", le=\"10.0\"} 1.0\n" + "inboundBytesPerSec_bucket{client=\"grpc\", le=\"20.0\"} 5.0\n" + "inboundBytesPerSec_bucket{client=\"grpc\", le=\"30.0\"} 5.0\n" + "inboundBytesPerSec_count{client=\"grpc\"} 5.0\n" + "# TYPE outboundBytesPerSec histogram\n" + "outboundBytesPerSec_bucket{client=\"grpc\", le=\"100.0\"} 1.0 1512216000000\n" + "outboundBytesPerSec_bucket{client=\"grpc\", le=\"200.0\"} 1.0 1512216000000\n" + "outboundBytesPerSec_bucket{client=\"grpc\", le=\"+Inf\"} 1.0 1512216000000\n" + "outboundBytesPerSec_count{client=\"grpc\"} 1.0 1512216000000\n"); + + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 6); + + { + auto& s = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "inboundBytesPerSec_count"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "client", "mbus"); + ASSERT_UINT_POINT(s, TInstant::Zero(), 5); + } + { + auto& s = samples.GetSamples(1); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::HIST_RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "inboundBytesPerSec"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "client", "mbus"); + auto hist = ExplicitHistogramSnapshot( + { 10, 20, HISTOGRAM_INF_BOUND }, + { 1, 4, 0 }); + ASSERT_HIST_POINT(s, TInstant::Zero(), *hist); + } + { + auto& s = samples.GetSamples(2); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "inboundBytesPerSec_count"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "client", "grpc"); + ASSERT_UINT_POINT(s, TInstant::Zero(), 5); + } + { + auto& s = samples.GetSamples(3); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::HIST_RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "inboundBytesPerSec"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "client", "grpc"); + auto hist = ExplicitHistogramSnapshot( + { 10, 20, 30 }, + { 1, 4, 0 }); + ASSERT_HIST_POINT(s, TInstant::Zero(), *hist); + } + { + auto& s = samples.GetSamples(4); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "outboundBytesPerSec_count"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "client", "grpc"); + ASSERT_UINT_POINT(s, TInstant::Seconds(1512216000), 1) ; + } + { + auto& s = samples.GetSamples(5); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::HIST_RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "outboundBytesPerSec"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "client", "grpc"); + auto hist = ExplicitHistogramSnapshot( + { 100, 200, HISTOGRAM_INF_BOUND }, + { 1, 0, 0 }); + ASSERT_HIST_POINT(s, TInstant::Seconds(1512216000), *hist); + } + } + + Y_UNIT_TEST(MixedTypes) { + auto samples = Decode( + "# HELP http_requests_total The total number of HTTP requests.\n" + "# TYPE http_requests_total counter\n" + "http_requests_total { } 1027 1395066363000\n" + "http_requests_total{method=\"post\",code=\"200\"} 1027 1395066363000\n" + "http_requests_total{method=\"post\",code=\"400\"} 3 1395066363000\n" + "\n" + "# Minimalistic line:\n" + "metric_without_timestamp_and_labels 12.47\n" + "\n" + "# HELP rpc_duration_seconds A summary of the RPC duration in seconds.\n" + "# TYPE rpc_duration_seconds summary\n" + "rpc_duration_seconds{quantile=\"0.01\"} 3102\n" + "rpc_duration_seconds{quantile=\"0.5\"} 4773\n" + "rpc_duration_seconds{quantile=\"0.9\"} 9001\n" + "rpc_duration_seconds_sum 1.7560473e+07\n" + "rpc_duration_seconds_count 2693\n" + "\n" + "# Another mMinimalistic line:\n" + "metric_with_timestamp 12.47 1234567890\n"); + + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 10); + + { + auto& s = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 1); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "http_requests_total"); + ASSERT_UINT_POINT(s, TInstant::Seconds(1395066363), 1027); + } + { + auto& s = samples.GetSamples(1); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 3); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "http_requests_total"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "method", "post"); + ASSERT_LABEL_EQUAL(s.GetLabels(2), "code", "200"); + ASSERT_UINT_POINT(s, TInstant::Seconds(1395066363), 1027); + } + { + auto& s = samples.GetSamples(2); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 3); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "http_requests_total"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "method", "post"); + ASSERT_LABEL_EQUAL(s.GetLabels(2), "code", "400"); + ASSERT_UINT_POINT(s, TInstant::Seconds(1395066363), 3); + } + { + auto& s = samples.GetSamples(3); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 1); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "metric_without_timestamp_and_labels"); + ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 12.47); + } + { + auto& s = samples.GetSamples(4); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "rpc_duration_seconds"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "quantile", "0.01"); + ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 3102); + } + { + auto& s = samples.GetSamples(5); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "rpc_duration_seconds"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "quantile", "0.5"); + ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 4773); + } + { + auto& s = samples.GetSamples(6); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 2); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "rpc_duration_seconds"); + ASSERT_LABEL_EQUAL(s.GetLabels(1), "quantile", "0.9"); + ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 9001); + } + { + auto& s = samples.GetSamples(7); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 1); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "rpc_duration_seconds_sum"); + ASSERT_DOUBLE_POINT(s, TInstant::Zero(), 17560473); + } + { + auto& s = samples.GetSamples(8); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::RATE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 1); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "rpc_duration_seconds_count"); + ASSERT_UINT_POINT(s, TInstant::Zero(), 2693); + } + { + auto& s = samples.GetSamples(9); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::EMetricType::GAUGE); + UNIT_ASSERT_EQUAL(s.LabelsSize(), 1); + ASSERT_LABEL_EQUAL(s.GetLabels(0), "sensor", "metric_with_timestamp"); + ASSERT_DOUBLE_POINT(s, TInstant::MilliSeconds(1234567890), 12.47); + } + } +} diff --git a/library/cpp/monlib/encode/prometheus/prometheus_encoder.cpp b/library/cpp/monlib/encode/prometheus/prometheus_encoder.cpp new file mode 100644 index 0000000000..15efeb8c03 --- /dev/null +++ b/library/cpp/monlib/encode/prometheus/prometheus_encoder.cpp @@ -0,0 +1,413 @@ +#include "prometheus.h" +#include "prometheus_model.h" + +#include <library/cpp/monlib/encode/encoder_state.h> +#include <library/cpp/monlib/metrics/labels.h> +#include <library/cpp/monlib/metrics/metric_value.h> + +#include <util/string/cast.h> +#include <util/generic/hash_set.h> + + +namespace NMonitoring { + namespace { + /////////////////////////////////////////////////////////////////////// + // TPrometheusWriter + /////////////////////////////////////////////////////////////////////// + class TPrometheusWriter { + public: + explicit TPrometheusWriter(IOutputStream* out) + : Out_(out) + { + } + + void WriteType(EMetricType type, const TString& name) { + auto r = WrittenTypes_.insert(name); + if (!r.second) { + // type for this metric was already written + return; + } + + Out_->Write("# TYPE "); + WriteMetricName(name); + Out_->Write(' '); + + switch (type) { + case EMetricType::GAUGE: + case EMetricType::IGAUGE: + Out_->Write("gauge"); + break; + case EMetricType::RATE: + case EMetricType::COUNTER: + Out_->Write("counter"); + break; + case EMetricType::HIST: + case EMetricType::HIST_RATE: + Out_->Write("histogram"); + break; + case EMetricType::LOGHIST: + // TODO(@kbalakirev): implement this case + break; + case EMetricType::DSUMMARY: + ythrow yexception() << "writing summary type is forbiden"; + case EMetricType::UNKNOWN: + ythrow yexception() << "unknown metric type: " << MetricTypeToStr(type) + << ", name: " << name; + } + Out_->Write('\n'); + } + + void WriteDouble(TStringBuf name, const TLabels& labels, TInstant time, double value) { + WriteValue(name, "", labels, "", "", time, value); + } + + void WriteHistogram(TStringBuf name, const TLabels& labels, TInstant time, IHistogramSnapshot* h) { + Y_ENSURE(!labels.Has(NPrometheus::BUCKET_LABEL), + "histogram metric " << name << " has label '" << + NPrometheus::BUCKET_LABEL << "' which is reserved in Prometheus"); + + double totalCount = 0; + for (ui32 i = 0, count = h->Count(); i < count; i++) { + TBucketBound bound = h->UpperBound(i); + TStringBuf boundStr; + if (bound == HISTOGRAM_INF_BOUND) { + boundStr = TStringBuf("+Inf"); + } else { + size_t len = FloatToString(bound, TmpBuf_, Y_ARRAY_SIZE(TmpBuf_)); + boundStr = TStringBuf(TmpBuf_, len); + } + + TBucketValue value = h->Value(i); + totalCount += static_cast<double>(value); + + WriteValue( + name, NPrometheus::BUCKET_SUFFIX, + labels, NPrometheus::BUCKET_LABEL, boundStr, + time, + totalCount); + } + + WriteValue(name, NPrometheus::COUNT_SUFFIX, labels, "", "", time, totalCount); + } + + void WriteSummaryDouble(TStringBuf name, const TLabels& labels, TInstant time, ISummaryDoubleSnapshot* s) { + WriteValue(name, NPrometheus::SUM_SUFFIX, labels, "", "", time, s->GetSum()); + WriteValue(name, NPrometheus::MIN_SUFFIX, labels, "", "", time, s->GetMin()); + WriteValue(name, NPrometheus::MAX_SUFFIX, labels, "", "", time, s->GetMax()); + WriteValue(name, NPrometheus::LAST_SUFFIX, labels, "", "", time, s->GetLast()); + WriteValue(name, NPrometheus::COUNT_SUFFIX, labels, "", "", time, s->GetCount()); + } + + void WriteLn() { + Out_->Write('\n'); + } + + private: + // will replace invalid chars with '_' + void WriteMetricName(TStringBuf name) { + Y_ENSURE(!name.Empty(), "trying to write metric with empty name"); + + char ch = name[0]; + if (NPrometheus::IsValidMetricNameStart(ch)) { + Out_->Write(ch); + } else { + Out_->Write('_'); + } + + for (size_t i = 1, len = name.length(); i < len; i++) { + ch = name[i]; + if (NPrometheus::IsValidMetricNameContinuation(ch)) { + Out_->Write(ch); + } else { + Out_->Write('_'); + } + } + } + + void WriteLabels(const TLabels& labels, TStringBuf addLabelKey, TStringBuf addLabelValue) { + Out_->Write('{'); + for (auto&& l: labels) { + Out_->Write(l.Name()); + Out_->Write('='); + WriteLabelValue(l.Value()); + Out_->Write(", "); // trailign comma is supported in parsers + } + if (!addLabelKey.Empty() && !addLabelValue.Empty()) { + Out_->Write(addLabelKey); + Out_->Write('='); + WriteLabelValue(addLabelValue); + } + Out_->Write('}'); + } + + void WriteLabelValue(TStringBuf value) { + Out_->Write('"'); + for (char ch: value) { + if (ch == '"') { + Out_->Write("\\\""); + } else if (ch == '\\') { + Out_->Write("\\\\"); + } else if (ch == '\n') { + Out_->Write("\\n"); + } else { + Out_->Write(ch); + } + } + Out_->Write('"'); + } + + void WriteValue( + TStringBuf name, TStringBuf suffix, + const TLabels& labels, TStringBuf addLabelKey, TStringBuf addLabelValue, + TInstant time, double value) + { + // (1) name + WriteMetricName(name); + if (!suffix.Empty()) { + Out_->Write(suffix); + } + + // (2) labels + if (!labels.Empty() || !addLabelKey.Empty()) { + WriteLabels(labels, addLabelKey, addLabelValue); + } + Out_->Write(' '); + + // (3) value + { + size_t len = FloatToString(value, TmpBuf_, Y_ARRAY_SIZE(TmpBuf_)); + Out_->Write(TmpBuf_, len); + } + + // (4) time + if (ui64 timeMillis = time.MilliSeconds()) { + Out_->Write(' '); + size_t len = IntToString<10>(timeMillis, TmpBuf_, Y_ARRAY_SIZE(TmpBuf_)); + Out_->Write(TmpBuf_, len); + } + Out_->Write('\n'); + } + + private: + IOutputStream* Out_; + THashSet<TString> WrittenTypes_; + char TmpBuf_[512]; // used to convert doubles to strings + }; + + /////////////////////////////////////////////////////////////////////// + // TMetricState + /////////////////////////////////////////////////////////////////////// + struct TMetricState { + EMetricType Type = EMetricType::UNKNOWN; + TLabels Labels; + TInstant Time = TInstant::Zero(); + EMetricValueType ValueType = EMetricValueType::UNKNOWN; + TMetricValue Value; + + ~TMetricState() { + ClearValue(); + } + + void Clear() { + Type = EMetricType::UNKNOWN; + Labels.Clear(); + Time = TInstant::Zero(); + ClearValue(); + } + + void ClearValue() { + // TMetricValue does not keep ownership of histogram + if (ValueType == EMetricValueType::HISTOGRAM) { + Value.AsHistogram()->UnRef(); + } else if (ValueType == EMetricValueType::SUMMARY) { + Value.AsSummaryDouble()->UnRef(); + } + ValueType = EMetricValueType::UNKNOWN; + Value = {}; + } + + template <typename T> + void SetValue(T value) { + // TMetricValue does not keep ownership of histogram + if (ValueType == EMetricValueType::HISTOGRAM) { + Value.AsHistogram()->UnRef(); + } else if (ValueType == EMetricValueType::SUMMARY) { + Value.AsSummaryDouble()->UnRef(); + } + ValueType = TValueType<T>::Type; + Value = TMetricValue(value); + if (ValueType == EMetricValueType::HISTOGRAM) { + Value.AsHistogram()->Ref(); + } else if (ValueType == EMetricValueType::SUMMARY) { + Value.AsSummaryDouble()->Ref(); + } + } + }; + + /////////////////////////////////////////////////////////////////////// + // TPrometheusEncoder + /////////////////////////////////////////////////////////////////////// + class TPrometheusEncoder final: public IMetricEncoder { + public: + explicit TPrometheusEncoder(IOutputStream* out, TStringBuf metricNameLabel) + : Writer_(out) + , MetricNameLabel_(metricNameLabel) + { + } + + private: + void OnStreamBegin() override { + State_.Expect(TEncoderState::EState::ROOT); + } + + void OnStreamEnd() override { + State_.Expect(TEncoderState::EState::ROOT); + Writer_.WriteLn(); + } + + void OnCommonTime(TInstant time) override { + State_.Expect(TEncoderState::EState::ROOT); + CommonTime_ = time; + } + + void OnMetricBegin(EMetricType type) override { + State_.Switch(TEncoderState::EState::ROOT, TEncoderState::EState::METRIC); + MetricState_.Clear(); + MetricState_.Type = type; + } + + void OnMetricEnd() override { + State_.Switch(TEncoderState::EState::METRIC, TEncoderState::EState::ROOT); + WriteMetric(); + } + + void OnLabelsBegin() override { + if (State_ == TEncoderState::EState::METRIC) { + State_ = TEncoderState::EState::METRIC_LABELS; + } else if (State_ == TEncoderState::EState::ROOT) { + State_ = TEncoderState::EState::COMMON_LABELS; + } else { + State_.ThrowInvalid("expected METRIC or ROOT"); + } + } + + void OnLabelsEnd() override { + if (State_ == TEncoderState::EState::METRIC_LABELS) { + State_ = TEncoderState::EState::METRIC; + } else if (State_ == TEncoderState::EState::COMMON_LABELS) { + State_ = TEncoderState::EState::ROOT; + } else { + State_.ThrowInvalid("expected LABELS or COMMON_LABELS"); + } + } + + void OnLabel(TStringBuf name, TStringBuf value) override { + if (State_ == TEncoderState::EState::METRIC_LABELS) { + MetricState_.Labels.Add(name, value); + } else if (State_ == TEncoderState::EState::COMMON_LABELS) { + CommonLabels_.Add(name, value); + } else { + State_.ThrowInvalid("expected LABELS or COMMON_LABELS"); + } + } + + void OnDouble(TInstant time, double value) override { + State_.Expect(TEncoderState::EState::METRIC); + MetricState_.Time = time; + MetricState_.SetValue(value); + } + + void OnInt64(TInstant time, i64 value) override { + State_.Expect(TEncoderState::EState::METRIC); + MetricState_.Time = time; + MetricState_.SetValue(value); + } + + void OnUint64(TInstant time, ui64 value) override { + State_.Expect(TEncoderState::EState::METRIC); + MetricState_.Time = time; + MetricState_.SetValue(value); + } + + void OnHistogram(TInstant time, IHistogramSnapshotPtr snapshot) override { + State_.Expect(TEncoderState::EState::METRIC); + MetricState_.Time = time; + MetricState_.SetValue(snapshot.Get()); + } + + void OnSummaryDouble(TInstant time, ISummaryDoubleSnapshotPtr snapshot) override { + State_.Expect(TEncoderState::EState::METRIC); + MetricState_.Time = time; + MetricState_.SetValue(snapshot.Get()); + } + + void OnLogHistogram(TInstant, TLogHistogramSnapshotPtr) override { + // TODO(@kbalakirev): implement this function + } + + void Close() override { + } + + void WriteMetric() { + if (MetricState_.ValueType == EMetricValueType::UNKNOWN) { + return; + } + + // XXX: poor performace + for (auto&& l: CommonLabels_) { + MetricState_.Labels.Add(l.Name(), l.Value()); + } + + TMaybe<TLabel> nameLabel = MetricState_.Labels.Extract(MetricNameLabel_); + Y_ENSURE(nameLabel, + "labels " << MetricState_.Labels << + " does not contain label '" << MetricNameLabel_ << '\''); + + const TString& metricName = ToString(nameLabel->Value()); + if (MetricState_.Type != EMetricType::DSUMMARY) { + Writer_.WriteType(MetricState_.Type, metricName); + } + + if (MetricState_.Time == TInstant::Zero()) { + MetricState_.Time = CommonTime_; + } + + EMetricType type = MetricState_.Type; + if (type == EMetricType::HIST || type == EMetricType::HIST_RATE) { + Y_ENSURE(MetricState_.ValueType == EMetricValueType::HISTOGRAM, + "invalid value type for histogram: " << int(MetricState_.ValueType)); // TODO: to string conversion + Writer_.WriteHistogram( + metricName, + MetricState_.Labels, + MetricState_.Time, + MetricState_.Value.AsHistogram()); + } else if (type == EMetricType::DSUMMARY) { + Writer_.WriteSummaryDouble( + metricName, + MetricState_.Labels, + MetricState_.Time, + MetricState_.Value.AsSummaryDouble()); + } else { + Writer_.WriteDouble( + metricName, + MetricState_.Labels, + MetricState_.Time, + MetricState_.Value.AsDouble(MetricState_.ValueType)); + } + } + + private: + TEncoderState State_; + TPrometheusWriter Writer_; + TString MetricNameLabel_; + TInstant CommonTime_ = TInstant::Zero(); + TLabels CommonLabels_; + TMetricState MetricState_; + }; + } + + IMetricEncoderPtr EncoderPrometheus(IOutputStream* out, TStringBuf metricNameLabel) { + return MakeHolder<TPrometheusEncoder>(out, metricNameLabel); + } + +} // namespace NMonitoring diff --git a/library/cpp/monlib/encode/prometheus/prometheus_encoder_ut.cpp b/library/cpp/monlib/encode/prometheus/prometheus_encoder_ut.cpp new file mode 100644 index 0000000000..fd9debb060 --- /dev/null +++ b/library/cpp/monlib/encode/prometheus/prometheus_encoder_ut.cpp @@ -0,0 +1,414 @@ +#include "prometheus.h" + +#include <library/cpp/monlib/encode/protobuf/protobuf.h> +#include <library/cpp/monlib/metrics/metric_value.h> +#include <library/cpp/monlib/metrics/histogram_snapshot.h> + +#include <library/cpp/testing/unittest/registar.h> + +#include <util/stream/str.h> + +using namespace NMonitoring; + +Y_UNIT_TEST_SUITE(TPrometheusEncoderTest) { + + template <typename TFunc> + TString EncodeToString(TFunc fn) { + TStringStream ss; + IMetricEncoderPtr encoder = EncoderPrometheus(&ss); + fn(encoder.Get()); + return ss.Str(); + } + + ISummaryDoubleSnapshotPtr TestSummaryDouble() { + return MakeIntrusive<TSummaryDoubleSnapshot>(10.1, -0.45, 0.478, 0.3, 30u); + } + + Y_UNIT_TEST(Empty) { + auto result = EncodeToString([](IMetricEncoder* e) { + e->OnStreamBegin(); + e->OnStreamEnd(); + }); + UNIT_ASSERT_STRINGS_EQUAL(result, "\n"); + } + + Y_UNIT_TEST(DoubleGauge) { + auto result = EncodeToString([](IMetricEncoder* e) { + e->OnStreamBegin(); + { // no values + e->OnMetricBegin(EMetricType::GAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "cpuUsage"); + e->OnLabelsEnd(); + } + e->OnMetricEnd(); + } + { // one value no ts + e->OnMetricBegin(EMetricType::GAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "diskUsage"); + e->OnLabel("disk", "sda1"); + e->OnLabelsEnd(); + } + e->OnDouble(TInstant::Zero(), 1000); + e->OnMetricEnd(); + } + { // one value with ts + e->OnMetricBegin(EMetricType::GAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "memoryUsage"); + e->OnLabel("host", "solomon-man-00"); + e->OnLabel("dc", "man"); + e->OnLabelsEnd(); + } + e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 1000); + e->OnMetricEnd(); + } + { // many values + e->OnMetricBegin(EMetricType::GAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "bytesRx"); + e->OnLabel("host", "solomon-sas-01"); + e->OnLabel("dc", "sas"); + e->OnLabelsEnd(); + } + e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 2); + e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:05Z"), 4); + e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:10Z"), 8); + e->OnMetricEnd(); + } + { // already seen metric name + e->OnMetricBegin(EMetricType::GAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "diskUsage"); + e->OnLabel("disk", "sdb1"); + e->OnLabelsEnd(); + } + e->OnDouble(TInstant::Zero(), 1001); + e->OnMetricEnd(); + } + { // NaN + e->OnMetricBegin(EMetricType::GAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "nanValue"); + e->OnLabelsEnd(); + } + e->OnDouble(TInstant::Zero(), NAN); + e->OnMetricEnd(); + } + { // Inf + e->OnMetricBegin(EMetricType::GAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "infValue"); + e->OnLabelsEnd(); + } + e->OnDouble(TInstant::Zero(), INFINITY); + e->OnMetricEnd(); + } + { + e->OnMetricBegin(EMetricType::DSUMMARY); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "seconds"); + e->OnLabel("disk", "sdb1"); + e->OnLabelsEnd(); + } + e->OnSummaryDouble(TInstant::Zero(), TestSummaryDouble()); + e->OnMetricEnd(); + } + e->OnStreamEnd(); + }); + + UNIT_ASSERT_STRINGS_EQUAL(result, + "# TYPE diskUsage gauge\n" + "diskUsage{disk=\"sda1\", } 1000\n" + "# TYPE memoryUsage gauge\n" + "memoryUsage{host=\"solomon-man-00\", dc=\"man\", } 1000 1512216000000\n" + "# TYPE bytesRx gauge\n" + "bytesRx{host=\"solomon-sas-01\", dc=\"sas\", } 8 1512216010000\n" + "diskUsage{disk=\"sdb1\", } 1001\n" + "# TYPE nanValue gauge\n" + "nanValue nan\n" + "# TYPE infValue gauge\n" + "infValue inf\n" + "seconds_sum{disk=\"sdb1\", } 10.1\n" + "seconds_min{disk=\"sdb1\", } -0.45\n" + "seconds_max{disk=\"sdb1\", } 0.478\n" + "seconds_last{disk=\"sdb1\", } 0.3\n" + "seconds_count{disk=\"sdb1\", } 30\n" + "\n"); + } + + Y_UNIT_TEST(IntGauges) { + auto result = EncodeToString([](IMetricEncoder* e) { + e->OnStreamBegin(); + { // no values + e->OnMetricBegin(EMetricType::IGAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "cpuUsage"); + e->OnLabelsEnd(); + } + e->OnMetricEnd(); + } + { // one value no ts + e->OnMetricBegin(EMetricType::IGAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "diskUsage"); + e->OnLabel("disk", "sda1"); + e->OnLabelsEnd(); + } + e->OnInt64(TInstant::Zero(), 1000); + e->OnMetricEnd(); + } + { // one value with ts + e->OnMetricBegin(EMetricType::IGAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "memoryUsage"); + e->OnLabel("dc", "man"); + e->OnLabel("host", "solomon-man-00"); + e->OnLabelsEnd(); + } + e->OnInt64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 1000); + e->OnMetricEnd(); + } + { // many values + e->OnMetricBegin(EMetricType::IGAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "bytesRx"); + e->OnLabel("dc", "sas"); + e->OnLabel("host", "solomon-sas-01"); + e->OnLabelsEnd(); + } + e->OnInt64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 2); + e->OnInt64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:05Z"), 4); + e->OnInt64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:10Z"), 8); + e->OnMetricEnd(); + } + e->OnStreamEnd(); + }); + + UNIT_ASSERT_STRINGS_EQUAL(result, + "# TYPE diskUsage gauge\n" + "diskUsage{disk=\"sda1\", } 1000\n" + "# TYPE memoryUsage gauge\n" + "memoryUsage{dc=\"man\", host=\"solomon-man-00\", } 1000 1512216000000\n" + "# TYPE bytesRx gauge\n" + "bytesRx{dc=\"sas\", host=\"solomon-sas-01\", } 8 1512216010000\n" + "\n"); + } + + Y_UNIT_TEST(Counters) { + auto result = EncodeToString([](IMetricEncoder* e) { + e->OnStreamBegin(); + { // no values + e->OnMetricBegin(EMetricType::COUNTER); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "cpuUsage"); + e->OnLabelsEnd(); + } + e->OnMetricEnd(); + } + { // one value no ts + e->OnMetricBegin(EMetricType::COUNTER); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "diskUsage"); + e->OnLabel("disk", "sda1"); + e->OnLabelsEnd(); + } + e->OnInt64(TInstant::Zero(), 1000); + e->OnMetricEnd(); + } + { // one value with ts + e->OnMetricBegin(EMetricType::COUNTER); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "memoryUsage"); + e->OnLabel("host", "solomon-man-00"); + e->OnLabel("dc", "man"); + e->OnLabelsEnd(); + } + e->OnInt64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 1000); + e->OnMetricEnd(); + } + { // many values + e->OnMetricBegin(EMetricType::COUNTER); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "bytesRx"); + e->OnLabel("host", "solomon-sas-01"); + e->OnLabel("dc", "sas"); + e->OnLabelsEnd(); + } + e->OnInt64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 2); + e->OnInt64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:05Z"), 4); + e->OnInt64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:10Z"), 8); + e->OnMetricEnd(); + } + e->OnStreamEnd(); + }); + + UNIT_ASSERT_STRINGS_EQUAL(result, + "# TYPE diskUsage counter\n" + "diskUsage{disk=\"sda1\", } 1000\n" + "# TYPE memoryUsage counter\n" + "memoryUsage{host=\"solomon-man-00\", dc=\"man\", } 1000 1512216000000\n" + "# TYPE bytesRx counter\n" + "bytesRx{host=\"solomon-sas-01\", dc=\"sas\", } 8 1512216010000\n" + "\n"); + } + + Y_UNIT_TEST(Histograms) { + auto result = EncodeToString([](IMetricEncoder* e) { + e->OnStreamBegin(); + { // no values histogram + e->OnMetricBegin(EMetricType::HIST); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "cpuUsage"); + e->OnLabelsEnd(); + } + e->OnMetricEnd(); + } + { // one value no ts + e->OnMetricBegin(EMetricType::HIST); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "inboundBytesPerSec"); + e->OnLabel("client", "mbus"); + e->OnLabelsEnd(); + } + e->OnHistogram( + TInstant::Zero(), + ExplicitHistogramSnapshot({10, 20, HISTOGRAM_INF_BOUND}, {1, 4, 0})); + e->OnMetricEnd(); + } + { // one value no ts no +inf bucket + e->OnMetricBegin(EMetricType::HIST); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "inboundBytesPerSec"); + e->OnLabel("client", "grpc"); + e->OnLabelsEnd(); + } + e->OnHistogram( + TInstant::Zero(), + ExplicitHistogramSnapshot({10, 20, 30}, {1, 4, 0})); + e->OnMetricEnd(); + } + { // one value with ts + e->OnMetricBegin(EMetricType::HIST_RATE); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "outboundBytesPerSec"); + e->OnLabel("client", "grps"); + e->OnLabelsEnd(); + } + e->OnHistogram( + TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), + ExplicitHistogramSnapshot({100, 200, HISTOGRAM_INF_BOUND}, {1, 0, 0})); + e->OnMetricEnd(); + } + { // many values + e->OnMetricBegin(EMetricType::HIST); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "bytesRx"); + e->OnLabel("host", "solomon-sas-01"); + e->OnLabel("dc", "sas"); + e->OnLabelsEnd(); + } + TBucketBounds bounds = {100, 200, HISTOGRAM_INF_BOUND}; + e->OnHistogram( + TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), + ExplicitHistogramSnapshot(bounds, {10, 0, 0})); + e->OnHistogram( + TInstant::ParseIso8601Deprecated("2017-12-02T12:00:05Z"), + ExplicitHistogramSnapshot(bounds, {10, 2, 0})); + e->OnHistogram( + TInstant::ParseIso8601Deprecated("2017-12-02T12:00:10Z"), + ExplicitHistogramSnapshot(bounds, {10, 2, 5})); + e->OnMetricEnd(); + } + e->OnStreamEnd(); + }); + + UNIT_ASSERT_STRINGS_EQUAL(result, + "# TYPE inboundBytesPerSec histogram\n" + "inboundBytesPerSec_bucket{client=\"mbus\", le=\"10\"} 1\n" + "inboundBytesPerSec_bucket{client=\"mbus\", le=\"20\"} 5\n" + "inboundBytesPerSec_bucket{client=\"mbus\", le=\"+Inf\"} 5\n" + "inboundBytesPerSec_count{client=\"mbus\", } 5\n" + "inboundBytesPerSec_bucket{client=\"grpc\", le=\"10\"} 1\n" + "inboundBytesPerSec_bucket{client=\"grpc\", le=\"20\"} 5\n" + "inboundBytesPerSec_bucket{client=\"grpc\", le=\"30\"} 5\n" + "inboundBytesPerSec_count{client=\"grpc\", } 5\n" + "# TYPE outboundBytesPerSec histogram\n" + "outboundBytesPerSec_bucket{client=\"grps\", le=\"100\"} 1 1512216000000\n" + "outboundBytesPerSec_bucket{client=\"grps\", le=\"200\"} 1 1512216000000\n" + "outboundBytesPerSec_bucket{client=\"grps\", le=\"+Inf\"} 1 1512216000000\n" + "outboundBytesPerSec_count{client=\"grps\", } 1 1512216000000\n" + "# TYPE bytesRx histogram\n" + "bytesRx_bucket{host=\"solomon-sas-01\", dc=\"sas\", le=\"100\"} 10 1512216010000\n" + "bytesRx_bucket{host=\"solomon-sas-01\", dc=\"sas\", le=\"200\"} 12 1512216010000\n" + "bytesRx_bucket{host=\"solomon-sas-01\", dc=\"sas\", le=\"+Inf\"} 17 1512216010000\n" + "bytesRx_count{host=\"solomon-sas-01\", dc=\"sas\", } 17 1512216010000\n" + "\n"); + } + + Y_UNIT_TEST(CommonLables) { + auto result = EncodeToString([](IMetricEncoder* e) { + e->OnStreamBegin(); + { // common time + e->OnCommonTime(TInstant::Seconds(1500000000)); + } + { // common labels + e->OnLabelsBegin(); + e->OnLabel("project", "solomon"); + e->OnLabelsEnd(); + } + { // metric #1 + e->OnMetricBegin(EMetricType::COUNTER); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "single"); + e->OnLabel("labels", "l1"); + e->OnLabelsEnd(); + } + e->OnUint64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:10Z"), 17); + e->OnMetricEnd(); + } + { // metric #2 + e->OnMetricBegin(EMetricType::COUNTER); + { + e->OnLabelsBegin(); + e->OnLabel("sensor", "two"); + e->OnLabel("labels", "l2"); + e->OnLabelsEnd(); + } + e->OnUint64(TInstant::Zero(), 42); + e->OnMetricEnd(); + } + e->OnStreamEnd(); + }); + UNIT_ASSERT_STRINGS_EQUAL(result, +R"(# TYPE single counter +single{labels="l1", project="solomon", } 17 1512216010000 +# TYPE two counter +two{labels="l2", project="solomon", } 42 1500000000000 + +)"); + } +} diff --git a/library/cpp/monlib/encode/prometheus/prometheus_model.h b/library/cpp/monlib/encode/prometheus/prometheus_model.h new file mode 100644 index 0000000000..cb7f2cb15b --- /dev/null +++ b/library/cpp/monlib/encode/prometheus/prometheus_model.h @@ -0,0 +1,70 @@ +#pragma once + +#include <util/generic/strbuf.h> + + +namespace NMonitoring { +namespace NPrometheus { + + // + // Prometheus specific names and validation rules. + // + // See https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md + // and https://github.com/prometheus/common/blob/master/expfmt/text_parse.go + // + + inline constexpr TStringBuf BUCKET_SUFFIX = "_bucket"; + inline constexpr TStringBuf COUNT_SUFFIX = "_count"; + inline constexpr TStringBuf SUM_SUFFIX = "_sum"; + inline constexpr TStringBuf MIN_SUFFIX = "_min"; + inline constexpr TStringBuf MAX_SUFFIX = "_max"; + inline constexpr TStringBuf LAST_SUFFIX = "_last"; + + // Used for the label that defines the upper bound of a bucket of a + // histogram ("le" -> "less or equal"). + inline constexpr TStringBuf BUCKET_LABEL = "le"; + + + inline bool IsValidLabelNameStart(char ch) { + return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || ch == '_'; + } + + inline bool IsValidLabelNameContinuation(char ch) { + return IsValidLabelNameStart(ch) || (ch >= '0' && ch <= '9'); + } + + inline bool IsValidMetricNameStart(char ch) { + return IsValidLabelNameStart(ch) || ch == ':'; + } + + inline bool IsValidMetricNameContinuation(char ch) { + return IsValidLabelNameContinuation(ch) || ch == ':'; + } + + inline bool IsSum(TStringBuf name) { + return name.EndsWith(SUM_SUFFIX); + } + + inline bool IsCount(TStringBuf name) { + return name.EndsWith(COUNT_SUFFIX); + } + + inline bool IsBucket(TStringBuf name) { + return name.EndsWith(BUCKET_SUFFIX); + } + + inline TStringBuf ToBaseName(TStringBuf name) { + if (IsBucket(name)) { + return name.SubString(0, name.length() - BUCKET_SUFFIX.length()); + } + if (IsCount(name)) { + return name.SubString(0, name.length() - COUNT_SUFFIX.length()); + } + if (IsSum(name)) { + return name.SubString(0, name.length() - SUM_SUFFIX.length()); + } + return name; + } + +} // namespace NPrometheus +} // namespace NMonitoring diff --git a/library/cpp/monlib/encode/prometheus/ut/ya.make b/library/cpp/monlib/encode/prometheus/ut/ya.make new file mode 100644 index 0000000000..fc468ffb68 --- /dev/null +++ b/library/cpp/monlib/encode/prometheus/ut/ya.make @@ -0,0 +1,17 @@ +UNITTEST_FOR(library/cpp/monlib/encode/prometheus) + +OWNER( + g:solomon + jamel +) + +SRCS( + prometheus_encoder_ut.cpp + prometheus_decoder_ut.cpp +) + +PEERDIR( + library/cpp/monlib/encode/protobuf +) + +END() diff --git a/library/cpp/monlib/encode/prometheus/ya.make b/library/cpp/monlib/encode/prometheus/ya.make new file mode 100644 index 0000000000..7f2483b166 --- /dev/null +++ b/library/cpp/monlib/encode/prometheus/ya.make @@ -0,0 +1,17 @@ +LIBRARY() + +OWNER( + jamel + g:solomon +) + +SRCS( + prometheus_decoder.cpp + prometheus_encoder.cpp +) + +PEERDIR( + library/cpp/monlib/encode +) + +END() |