aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorrobot-ydb-importer <robot-ydb-importer@yandex-team.com>2024-02-14 19:47:36 +0300
committerInnokentii Mokin <innokentii@ydb.tech>2024-02-16 18:35:13 +0000
commitd6ee6054676c603f8afb27b5bd8ce7fe0a5bfbc0 (patch)
tree4aa69116e7818a4aae0bfedbfa29639b0f0b90e8
parent59ded8ecfcd805c109471346a0d4d1f269bdaa59 (diff)
downloadydb-d6ee6054676c603f8afb27b5bd8ce7fe0a5bfbc0.tar.gz
YDB Import 566
96265cd0cc64e1b9bb31fe97b915ed2a09caf1cb
-rw-r--r--contrib/python/ydb/ya.make23
-rw-r--r--library/cpp/monlib/encode/unistat/unistat.h13
-rw-r--r--library/cpp/monlib/encode/unistat/unistat_decoder.cpp312
-rw-r--r--library/cpp/monlib/encode/unistat/unistat_ut.cpp341
-rw-r--r--library/cpp/monlib/encode/unistat/ut/ya.make11
-rw-r--r--library/cpp/monlib/encode/unistat/ya.make17
-rw-r--r--library/python/monlib/encoder.pxd79
-rw-r--r--library/python/monlib/encoder.pyx260
-rw-r--r--library/python/monlib/labels.pxd47
-rw-r--r--library/python/monlib/metric.pxd103
-rw-r--r--library/python/monlib/metric.pyx162
-rw-r--r--library/python/monlib/metric_consumer.pxd8
-rw-r--r--library/python/monlib/metric_registry.pxd29
-rw-r--r--library/python/monlib/metric_registry.pyx277
-rw-r--r--library/python/monlib/ut/metric_ut.pyx113
-rw-r--r--library/python/monlib/ut/py2/test.py313
-rw-r--r--library/python/monlib/ut/py2/test_metric.py3
-rw-r--r--library/python/monlib/ut/py2/ya.make17
-rw-r--r--library/python/monlib/ut/py3/test.py311
-rw-r--r--library/python/monlib/ut/py3/test_metric.py3
-rw-r--r--library/python/monlib/ut/py3/ya.make17
-rw-r--r--library/python/monlib/ut/ya.make7
-rw-r--r--library/python/monlib/ya.make21
-rw-r--r--util/datetime/base.pxd126
24 files changed, 2613 insertions, 0 deletions
diff --git a/contrib/python/ydb/ya.make b/contrib/python/ydb/ya.make
new file mode 100644
index 0000000000..1886a29d3d
--- /dev/null
+++ b/contrib/python/ydb/ya.make
@@ -0,0 +1,23 @@
+PY23_LIBRARY()
+
+LICENSE(Service-Py23-Proxy)
+
+IF (PYTHON2)
+ PEERDIR(contrib/python/ydb/py2)
+ELSE()
+ PEERDIR(contrib/python/ydb/py3)
+ENDIF()
+
+PEERDIR(
+ contrib/ydb/public/api/grpc
+ contrib/ydb/public/api/grpc/draft
+)
+
+NO_LINT()
+
+END()
+
+RECURSE(
+ py2
+ py3
+) \ No newline at end of file
diff --git a/library/cpp/monlib/encode/unistat/unistat.h b/library/cpp/monlib/encode/unistat/unistat.h
new file mode 100644
index 0000000000..300fb6270f
--- /dev/null
+++ b/library/cpp/monlib/encode/unistat/unistat.h
@@ -0,0 +1,13 @@
+#pragma once
+
+#include <util/generic/fwd.h>
+#include <util/datetime/base.h>
+
+namespace NMonitoring {
+ /// Decodes unistat-style metrics
+ /// https://wiki.yandex-team.ru/golovan/stat-handle
+ void DecodeUnistat(TStringBuf data, class IMetricConsumer* c, TStringBuf metricNameLabel = "sensor", TInstant ts = TInstant::Zero());
+
+ /// Assumes consumer's stream is open by the caller
+ void DecodeUnistatToStream(TStringBuf data, class IMetricConsumer* c, TStringBuf metricNameLabel = "sensor", TInstant ts = TInstant::Zero());
+}
diff --git a/library/cpp/monlib/encode/unistat/unistat_decoder.cpp b/library/cpp/monlib/encode/unistat/unistat_decoder.cpp
new file mode 100644
index 0000000000..a2b787365c
--- /dev/null
+++ b/library/cpp/monlib/encode/unistat/unistat_decoder.cpp
@@ -0,0 +1,312 @@
+#include "unistat.h"
+
+#include <library/cpp/monlib/metrics/histogram_collector.h>
+#include <library/cpp/monlib/metrics/labels.h>
+#include <library/cpp/monlib/metrics/metric_type.h>
+#include <library/cpp/monlib/metrics/metric_value.h>
+#include <library/cpp/monlib/metrics/metric_consumer.h>
+
+#include <library/cpp/json/json_reader.h>
+
+#include <util/datetime/base.h>
+#include <util/string/split.h>
+
+#include <contrib/libs/re2/re2/re2.h>
+
+using namespace NJson;
+
+const re2::RE2 NAME_RE{R"((?:[a-zA-Z0-9\.\-/@_]+_)+(?:[ad][vehmntx]{3}|summ|hgram|max))"};
+
+namespace NMonitoring {
+ namespace {
+ bool IsNumber(const NJson::TJsonValue& j) {
+ switch (j.GetType()) {
+ case EJsonValueType::JSON_INTEGER:
+ case EJsonValueType::JSON_UINTEGER:
+ case EJsonValueType::JSON_DOUBLE:
+ return true;
+
+ default:
+ return false;
+ }
+ }
+
+ template <typename T>
+ T ExtractNumber(const TJsonValue& val) {
+ switch (val.GetType()) {
+ case EJsonValueType::JSON_INTEGER:
+ return static_cast<T>(val.GetInteger());
+ case EJsonValueType::JSON_UINTEGER:
+ return static_cast<T>(val.GetUInteger());
+ case EJsonValueType::JSON_DOUBLE:
+ return static_cast<T>(val.GetDouble());
+
+ default:
+ ythrow yexception() << "Expected number, but found " << val.GetType();
+ }
+ }
+
+ auto ExtractDouble = ExtractNumber<double>;
+ auto ExtractUi64 = ExtractNumber<ui64>;
+
+ class THistogramBuilder {
+ public:
+ void Add(TBucketBound bound, TBucketValue value) {
+ /// XXX: yasm uses left-closed intervals, while in monlib we use right-closed ones,
+ /// so (-inf; 0) [0, 100) [100; +inf)
+ /// becomes (-inf; 0] (0, 100] (100; +inf)
+ /// but since we've already lost some information these no way to avoid this kind of error here
+ Bounds_.push_back(bound);
+
+ /// this will always be 0 for the first bucket,
+ /// since there's no way to make (-inf; N) bucket in yasm
+ Values_.push_back(NextValue_);
+
+ /// we will write this value into the next bucket so that [[0, 10], [100, 20], [200, 50]]
+ /// becomes (-inf; 0] -> 0; (0; 100] -> 10; (100; 200] -> 20; (200; +inf) -> 50
+ NextValue_ = value;
+ }
+
+ IHistogramSnapshotPtr Finalize() {
+ Bounds_.push_back(std::numeric_limits<TBucketBound>::max());
+ Values_.push_back(NextValue_);
+
+ return ExplicitHistogramSnapshot(Bounds_, Values_, true);
+ }
+
+ public:
+ TBucketValue NextValue_ {0};
+ TBucketBounds Bounds_;
+ TBucketValues Values_;
+ };
+
+ class LogHistogramBuilder {
+ public:
+ void Add(double value) {
+ while (Values_.size() < HISTOGRAM_MAX_BUCKETS_COUNT && value > MaxValueWithBickets(Values_.size())) {
+ Values_.push_back(0);
+ }
+
+ ui32 index = 0;
+ if (value > MaxValueWithBickets(HISTOGRAM_MAX_BUCKETS_COUNT)) {
+ index = Values_.size() - 1;
+ } else if (value > MIN_VALUE) {
+ double logBase = std::log(value) / std::log(BASE);
+ index = static_cast<ui32>(std::ceil(logBase));
+ }
+ ++Values_[index];
+ }
+
+ IHistogramSnapshotPtr Finalize() && {
+ return new TExponentialHistogramSnapshot(BASE, MIN_VALUE, std::move(Values_));
+ }
+
+ private:
+ static constexpr double BASE = 2;
+ static constexpr double MIN_VALUE = 1;
+
+ static double MaxValueWithBickets(ui64 buckets) {
+ return std::pow(BASE, buckets - 2);
+ }
+
+ private:
+ TBucketValues Values_ = TBucketValues(2, 0);
+ };
+
+ class TDecoderUnistat {
+ private:
+ public:
+ explicit TDecoderUnistat(IMetricConsumer* consumer, IInputStream* is, TStringBuf metricNameLabel, TInstant ts)
+ : Consumer_{consumer},
+ MetricNameLabel(metricNameLabel),
+ Timestamp_{ts} {
+ ReadJsonTree(is, &Json_, /* throw */ true);
+ }
+
+ void Decode() {
+ Y_ENSURE(Json_.IsArray(), "Expected array at the top level, but found " << Json_.GetType());
+
+ for (auto&& metric : Json_.GetArray()) {
+ Y_ENSURE(metric.IsArray(), "Metric must be an array");
+ auto&& arr = metric.GetArray();
+ Y_ENSURE(arr.size() == 2, "Metric must be an array of 2 elements");
+ auto&& name = arr[0];
+ auto&& value = arr[1];
+ MetricContext_ = {};
+
+ ParseName(name.GetString());
+
+ if (value.IsArray()) {
+ const auto& array = value.GetArray();
+ if (!array.empty() && IsNumber(array[0])) {
+ OnLogHistogram(value);
+ } else {
+ OnHistogram(value);
+ }
+ } else if (IsNumber(value)) {
+ if (MetricContext_.Name.EndsWith("_ahhh")) {
+ OnLogHistogram(value);
+ } else {
+ OnScalar(value);
+ }
+ } else {
+ ythrow yexception() << "Expected list or number, but found " << value.GetType();
+ }
+
+ WriteValue();
+ }
+ }
+
+ private:
+ void OnScalar(const TJsonValue& jsonValue) {
+ if (MetricContext_.IsDeriv) {
+ MetricContext_.Type = EMetricType::RATE;
+ MetricContext_.Value = TMetricValue{ExtractUi64(jsonValue)};
+ } else {
+ MetricContext_.Type = EMetricType::GAUGE;
+ MetricContext_.Value = TMetricValue{ExtractDouble(jsonValue)};
+ }
+ }
+
+ void OnLogHistogram(const TJsonValue& value) {
+ Y_ENSURE(MetricContext_.Name.EndsWith("_ahhh"), "Values list is supported only for _ahhh metrics");
+ MetricContext_.Type = EMetricType::HIST;
+
+ LogHistogramBuilder histogramBuilder;
+ if (IsNumber(value)) {
+ histogramBuilder.Add(value.GetDouble());
+ } else {
+ for (auto&& item: value.GetArray()) {
+ Y_ENSURE(IsNumber(item), "Expected a number, but found " << item.GetType());
+ histogramBuilder.Add(item.GetDouble());
+ }
+ }
+
+ MetricContext_.Histogram = std::move(histogramBuilder).Finalize();
+ MetricContext_.Value = TMetricValue{MetricContext_.Histogram.Get()};
+ }
+
+ void OnHistogram(const TJsonValue& jsonHist) {
+ if (MetricContext_.IsDeriv) {
+ MetricContext_.Type = EMetricType::HIST_RATE;
+ } else {
+ MetricContext_.Type = EMetricType::HIST;
+ }
+
+ auto histogramBuilder = THistogramBuilder();
+
+ for (auto&& bucket : jsonHist.GetArray()) {
+ Y_ENSURE(bucket.IsArray(), "Expected an array, but found " << bucket.GetType());
+ auto&& arr = bucket.GetArray();
+ Y_ENSURE(arr.size() == 2, "Histogram bucket must be an array of 2 elements");
+ const auto bound = ExtractDouble(arr[0]);
+ const auto weight = ExtractUi64(arr[1]);
+ histogramBuilder.Add(bound, weight);
+ }
+
+ MetricContext_.Histogram = histogramBuilder.Finalize();
+ MetricContext_.Value = TMetricValue{MetricContext_.Histogram.Get()};
+ }
+
+ bool IsDeriv(TStringBuf name) {
+ TStringBuf ignore, suffix;
+ name.RSplit('_', ignore, suffix);
+
+ Y_ENSURE(suffix.size() >= 3 && suffix.size() <= 5, "Disallowed suffix value: " << suffix);
+
+ if (suffix == TStringBuf("summ") || suffix == TStringBuf("hgram")) {
+ return true;
+ } else if (suffix == TStringBuf("max")) {
+ return false;
+ }
+
+ return suffix[0] == 'd';
+ }
+
+ void ParseName(TStringBuf value) {
+ TVector<TStringBuf> parts;
+ StringSplitter(value).Split(';').SkipEmpty().Collect(&parts);
+
+ Y_ENSURE(parts.size() >= 1 && parts.size() <= 16);
+
+ TStringBuf name = parts.back();
+ parts.pop_back();
+
+ Y_ENSURE(RE2::FullMatch(re2::StringPiece{name.data(), name.size()}, NAME_RE),
+ "Metric name " << name << " doesn't match regex " << NAME_RE.pattern());
+
+ MetricContext_.Name = name;
+ MetricContext_.IsDeriv = IsDeriv(MetricContext_.Name);
+
+ for (auto tag : parts) {
+ TStringBuf n, v;
+ tag.Split('=', n, v);
+ Y_ENSURE(n && v, "Unexpected tag format in " << tag);
+ MetricContext_.Labels.Add(n, v);
+ }
+ }
+
+ private:
+ void WriteValue() {
+ Consumer_->OnMetricBegin(MetricContext_.Type);
+
+ Consumer_->OnLabelsBegin();
+ Consumer_->OnLabel(MetricNameLabel, TString{MetricContext_.Name});
+ for (auto&& l : MetricContext_.Labels) {
+ Consumer_->OnLabel(l.Name(), l.Value());
+ }
+
+ Consumer_->OnLabelsEnd();
+
+ switch (MetricContext_.Type) {
+ case EMetricType::GAUGE:
+ Consumer_->OnDouble(Timestamp_, MetricContext_.Value.AsDouble());
+ break;
+ case EMetricType::RATE:
+ Consumer_->OnUint64(Timestamp_, MetricContext_.Value.AsUint64());
+ break;
+ case EMetricType::HIST:
+ case EMetricType::HIST_RATE:
+ Consumer_->OnHistogram(Timestamp_, MetricContext_.Value.AsHistogram());
+ break;
+ case EMetricType::LOGHIST:
+ case EMetricType::DSUMMARY:
+ case EMetricType::IGAUGE:
+ case EMetricType::COUNTER:
+ case EMetricType::UNKNOWN:
+ ythrow yexception() << "Unexpected metric type: " << MetricContext_.Type;
+ }
+
+ Consumer_->OnMetricEnd();
+ }
+
+ private:
+ IMetricConsumer* Consumer_;
+ NJson::TJsonValue Json_;
+ TStringBuf MetricNameLabel;
+ TInstant Timestamp_;
+
+ struct {
+ TStringBuf Name;
+ EMetricType Type{EMetricType::UNKNOWN};
+ TMetricValue Value;
+ bool IsDeriv{false};
+ TLabels Labels;
+ IHistogramSnapshotPtr Histogram;
+ } MetricContext_;
+ };
+
+ }
+
+ void DecodeUnistat(TStringBuf data, IMetricConsumer* c, TStringBuf metricNameLabel, TInstant ts) {
+ c->OnStreamBegin();
+ DecodeUnistatToStream(data, c, metricNameLabel, ts);
+ c->OnStreamEnd();
+ }
+
+ void DecodeUnistatToStream(TStringBuf data, IMetricConsumer* c, TStringBuf metricNameLabel, TInstant ts) {
+ TMemoryInput in{data.data(), data.size()};
+ TDecoderUnistat decoder(c, &in, metricNameLabel, ts);
+ decoder.Decode();
+ }
+}
diff --git a/library/cpp/monlib/encode/unistat/unistat_ut.cpp b/library/cpp/monlib/encode/unistat/unistat_ut.cpp
new file mode 100644
index 0000000000..f15fc7c5b7
--- /dev/null
+++ b/library/cpp/monlib/encode/unistat/unistat_ut.cpp
@@ -0,0 +1,341 @@
+#include "unistat.h"
+
+#include <library/cpp/monlib/encode/protobuf/protobuf.h>
+#include <library/cpp/monlib/metrics/labels.h>
+
+#include <library/cpp/testing/unittest/registar.h>
+
+using namespace NMonitoring;
+
+Y_UNIT_TEST_SUITE(TUnistatDecoderTest) {
+ Y_UNIT_TEST(MetricNameLabel) {
+ constexpr auto input = TStringBuf(R"([["something_axxx", 42]])");
+
+ NProto::TMultiSamplesList samples;
+ auto encoder = EncoderProtobuf(&samples);
+
+ DecodeUnistat(input, encoder.Get(), "metric_name_label");
+
+ UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 1);
+ auto sample = samples.GetSamples(0);
+
+ auto label = sample.GetLabels(0);
+ UNIT_ASSERT_VALUES_EQUAL(label.GetName(), "metric_name_label");
+ }
+
+ Y_UNIT_TEST(ScalarMetric) {
+ constexpr auto input = TStringBuf(R"([["something_axxx", 42]])");
+
+ NProto::TMultiSamplesList samples;
+ auto encoder = EncoderProtobuf(&samples);
+
+ DecodeUnistat(input, encoder.Get());
+
+ UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 1);
+ auto sample = samples.GetSamples(0);
+ UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::GAUGE);
+ UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1);
+
+ auto label = sample.GetLabels(0);
+ auto point = sample.GetPoints(0);
+ UNIT_ASSERT_VALUES_EQUAL(point.GetFloat64(), 42.);
+ UNIT_ASSERT_VALUES_EQUAL(label.GetName(), "sensor");
+ UNIT_ASSERT_VALUES_EQUAL(label.GetValue(), "something_axxx");
+ }
+
+ Y_UNIT_TEST(OverriddenTags) {
+ constexpr auto input = TStringBuf(R"([["ctype=foo;prj=bar;custom_tag=qwe;something_axxx", 42]])");
+
+ NProto::TMultiSamplesList samples;
+ auto encoder = EncoderProtobuf(&samples);
+
+ DecodeUnistat(input, encoder.Get());
+
+ UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 1);
+ auto sample = samples.GetSamples(0);
+ UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 4);
+
+ const auto& labels = sample.GetLabels();
+ TLabels actual;
+ for (auto&& l : labels) {
+ actual.Add(l.GetName(), l.GetValue());
+ }
+
+ TLabels expected{{"ctype", "foo"}, {"prj", "bar"}, {"custom_tag", "qwe"}, {"sensor", "something_axxx"}};
+
+ UNIT_ASSERT_VALUES_EQUAL(actual.size(), expected.size());
+ for (auto&& l : actual) {
+ UNIT_ASSERT(expected.Extract(l.Name())->Value() == l.Value());
+ }
+ }
+
+ Y_UNIT_TEST(ThrowsOnTopLevelObject) {
+ constexpr auto input = TStringBuf(R"({["something_axxx", 42]})");
+
+ NProto::TMultiSamplesList samples;
+ auto encoder = EncoderProtobuf(&samples);
+
+ UNIT_ASSERT_EXCEPTION(DecodeUnistat(input, encoder.Get()), yexception);
+ }
+
+ Y_UNIT_TEST(ThrowsOnUnwrappedMetric) {
+ constexpr auto input = TStringBuf(R"(["something_axxx", 42])");
+
+ NProto::TMultiSamplesList samples;
+ auto encoder = EncoderProtobuf(&samples);
+
+ UNIT_ASSERT_EXCEPTION(DecodeUnistat(input, encoder.Get()), yexception);
+ }
+
+ Y_UNIT_TEST(HistogramMetric) {
+ constexpr auto input = TStringBuf(R"([["something_hgram", [[0, 1], [200, 2], [500, 3]] ]])");
+
+ NProto::TMultiSamplesList samples;
+ auto encoder = EncoderProtobuf(&samples);
+
+ DecodeUnistat(input, encoder.Get());
+
+ auto sample = samples.GetSamples(0);
+ UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::HIST_RATE);
+ UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1);
+
+ auto label = sample.GetLabels(0);
+ const auto point = sample.GetPoints(0);
+ const auto histogram = point.GetHistogram();
+ const auto size = histogram.BoundsSize();
+ UNIT_ASSERT_VALUES_EQUAL(size, 4);
+
+ const TVector<double> expectedBounds {0, 200, 500, std::numeric_limits<double>::max()};
+ const TVector<ui64> expectedValues {0, 1, 2, 3};
+
+ for (auto i = 0; i < 4; ++i) {
+ UNIT_ASSERT_VALUES_EQUAL(histogram.GetBounds(i), expectedBounds[i]);
+ UNIT_ASSERT_VALUES_EQUAL(histogram.GetValues(i), expectedValues[i]);
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(label.GetName(), "sensor");
+ UNIT_ASSERT_VALUES_EQUAL(label.GetValue(), "something_hgram");
+ }
+
+ Y_UNIT_TEST(AbsoluteHistogram) {
+ constexpr auto input = TStringBuf(R"([["something_ahhh", [[0, 1], [200, 2], [500, 3]] ]])");
+
+ NProto::TMultiSamplesList samples;
+ auto encoder = EncoderProtobuf(&samples);
+
+ DecodeUnistat(input, encoder.Get());
+
+ auto sample = samples.GetSamples(0);
+ UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::HISTOGRAM);
+ UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1);
+ }
+
+ Y_UNIT_TEST(LogHistogram) {
+ constexpr auto input = TStringBuf(R"([["something_ahhh", [1, 2, 3] ]])");
+
+ NProto::TMultiSamplesList samples;
+ auto encoder = EncoderProtobuf(&samples);
+
+ DecodeUnistat(input, encoder.Get());
+
+ auto sample = samples.GetSamples(0);
+ UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::HISTOGRAM);
+ UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1);
+
+ auto histogram = sample.GetPoints(0).GetHistogram();
+ UNIT_ASSERT_VALUES_EQUAL(histogram.BoundsSize(), 4);
+ TVector<double> expectedBounds = {1, 2, 4, std::numeric_limits<double>::max()};
+ TVector<ui64> expectedValues = {1, 1, 1, 0};
+
+ for (auto i = 0; i < 4; ++i) {
+ UNIT_ASSERT_VALUES_EQUAL(histogram.GetBounds(i), expectedBounds[i]);
+ UNIT_ASSERT_VALUES_EQUAL(histogram.GetValues(i), expectedValues[i]);
+ }
+ }
+
+ Y_UNIT_TEST(LogHistogramOverflow) {
+ constexpr auto input = TStringBuf(R"([["something_ahhh", [1125899906842624, 2251799813685248] ]])");
+
+ NProto::TMultiSamplesList samples;
+ auto encoder = EncoderProtobuf(&samples);
+
+ DecodeUnistat(input, encoder.Get());
+
+ auto sample = samples.GetSamples(0);
+ UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::HISTOGRAM);
+ UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1);
+
+ auto histogram = sample.GetPoints(0).GetHistogram();
+ UNIT_ASSERT_VALUES_EQUAL(histogram.BoundsSize(), HISTOGRAM_MAX_BUCKETS_COUNT);
+
+ TVector<double> expectedBounds;
+ for (ui64 i = 0; i < 50; ++i) {
+ expectedBounds.push_back(std::pow(2, i));
+ }
+ expectedBounds.push_back(std::numeric_limits<double>::max());
+
+ TVector<ui64> expectedValues;
+ for (ui64 i = 0; i < 50; ++i) {
+ expectedValues.push_back(0);
+ }
+ expectedValues.push_back(2);
+
+ for (auto i = 0; i < 4; ++i) {
+ UNIT_ASSERT_VALUES_EQUAL(histogram.GetBounds(i), expectedBounds[i]);
+ UNIT_ASSERT_VALUES_EQUAL(histogram.GetValues(i), expectedValues[i]);
+ }
+ }
+
+ Y_UNIT_TEST(LogHistogramSingle) {
+ constexpr auto input = TStringBuf(R"([["something_ahhh", 4 ]])");
+
+ NProto::TMultiSamplesList samples;
+ auto encoder = EncoderProtobuf(&samples);
+
+ DecodeUnistat(input, encoder.Get());
+
+ auto sample = samples.GetSamples(0);
+ UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::HISTOGRAM);
+ UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1);
+
+ auto histogram = sample.GetPoints(0).GetHistogram();
+ UNIT_ASSERT_VALUES_EQUAL(histogram.BoundsSize(), 4);
+ TVector<double> expectedBounds = {1, 2, 4, std::numeric_limits<double>::max()};
+ TVector<ui64> expectedValues = {0, 0, 1, 0};
+
+ for (auto i = 0; i < 4; ++i) {
+ UNIT_ASSERT_VALUES_EQUAL(histogram.GetBounds(i), expectedBounds[i]);
+ UNIT_ASSERT_VALUES_EQUAL(histogram.GetValues(i), expectedValues[i]);
+ }
+ }
+
+ Y_UNIT_TEST(LogHistogramInvalid) {
+ NProto::TMultiSamplesList samples;
+ auto encoder = EncoderProtobuf(&samples);
+
+ {
+ constexpr auto input = TStringBuf(R"([["something_ahhh", [1, 2, [3, 4]] ]])");
+ UNIT_ASSERT_EXCEPTION(DecodeUnistat(input, encoder.Get()), yexception);
+ }
+
+ {
+ constexpr auto input = TStringBuf(R"([["something_ahhh", [[3, 4], 1, 2] ]])");
+ UNIT_ASSERT_EXCEPTION(DecodeUnistat(input, encoder.Get()), yexception);
+ }
+
+ {
+ constexpr auto input = TStringBuf(R"([["something_hgram", 1, 2, 3, 4 ]])");
+ UNIT_ASSERT_EXCEPTION(DecodeUnistat(input, encoder.Get()), yexception);
+ }
+ }
+
+
+ Y_UNIT_TEST(AllowedMetricNames) {
+ NProto::TMultiSamplesList samples;
+ auto encoder = EncoderProtobuf(&samples);
+
+ {
+ constexpr auto input = TStringBuf(R"([["a/A-b/c_D/__G_dmmm", [[0, 1], [200, 2], [500, 3]] ]])");
+ UNIT_ASSERT_NO_EXCEPTION(DecodeUnistat(input, encoder.Get()));
+ }
+ }
+
+ Y_UNIT_TEST(DisallowedMetricNames) {
+ NProto::TMultiSamplesList samples;
+ auto encoder = EncoderProtobuf(&samples);
+
+ {
+ constexpr auto input = TStringBuf(R"([["someth!ng_ahhh", [[0, 1], [200, 2], [500, 3]] ]])");
+ UNIT_ASSERT_EXCEPTION(DecodeUnistat(input, encoder.Get()), yexception);
+ }
+
+ {
+ constexpr auto input = TStringBuf(R"([["foo_a", [[0, 1], [200, 2], [500, 3]] ]])");
+ UNIT_ASSERT_EXCEPTION(DecodeUnistat(input, encoder.Get()), yexception);
+ }
+
+ {
+ constexpr auto input = TStringBuf(R"([["foo_ahhh;tag=value", [[0, 1], [200, 2], [500, 3]] ]])");
+ UNIT_ASSERT_EXCEPTION(DecodeUnistat(input, encoder.Get()), yexception);
+ }
+ }
+
+ Y_UNIT_TEST(MultipleMetrics) {
+ constexpr auto input = TStringBuf(R"([["something_axxx", 42], ["some-other_dhhh", 53]])");
+
+ NProto::TMultiSamplesList samples;
+ auto encoder = EncoderProtobuf(&samples);
+
+ DecodeUnistat(input, encoder.Get());
+
+ UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 2);
+ auto sample = samples.GetSamples(0);
+ UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::GAUGE);
+ UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1);
+
+ auto label = sample.GetLabels(0);
+ auto point = sample.GetPoints(0);
+ UNIT_ASSERT_VALUES_EQUAL(point.GetFloat64(), 42.);
+ UNIT_ASSERT_VALUES_EQUAL(label.GetName(), "sensor");
+ UNIT_ASSERT_VALUES_EQUAL(label.GetValue(), "something_axxx");
+
+ sample = samples.GetSamples(1);
+ UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::RATE);
+ UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1);
+
+ label = sample.GetLabels(0);
+ point = sample.GetPoints(0);
+ UNIT_ASSERT_VALUES_EQUAL(point.GetUint64(), 53);
+ UNIT_ASSERT_VALUES_EQUAL(label.GetName(), "sensor");
+ UNIT_ASSERT_VALUES_EQUAL(label.GetValue(), "some-other_dhhh");
+ }
+
+ Y_UNIT_TEST(UnderscoreName) {
+ constexpr auto input = TStringBuf(R"([["something_anything_dmmm", 42]])");
+
+ NProto::TMultiSamplesList samples;
+ auto encoder = EncoderProtobuf(&samples);
+ DecodeUnistat(input, encoder.Get());
+
+ UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 1);
+ auto sample = samples.GetSamples(0);
+ UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::RATE);
+ UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1);
+
+ auto label = sample.GetLabels(0);
+ auto point = sample.GetPoints(0);
+ UNIT_ASSERT_VALUES_EQUAL(point.GetUint64(), 42);
+ UNIT_ASSERT_VALUES_EQUAL(label.GetName(), "sensor");
+ UNIT_ASSERT_VALUES_EQUAL(label.GetValue(), "something_anything_dmmm");
+ }
+
+ Y_UNIT_TEST(MaxAggr) {
+ constexpr auto input = TStringBuf(R"([["something_anything_max", 42]])");
+
+ NProto::TMultiSamplesList samples;
+ auto encoder = EncoderProtobuf(&samples);
+ DecodeUnistat(input, encoder.Get());
+
+ UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 1);
+ auto sample = samples.GetSamples(0);
+ UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::GAUGE);
+ UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1);
+
+ auto label = sample.GetLabels(0);
+ auto point = sample.GetPoints(0);
+ UNIT_ASSERT_VALUES_EQUAL(point.GetFloat64(), 42.);
+ UNIT_ASSERT_VALUES_EQUAL(label.GetName(), "sensor");
+ UNIT_ASSERT_VALUES_EQUAL(label.GetValue(), "something_anything_max");
+ }
+}
diff --git a/library/cpp/monlib/encode/unistat/ut/ya.make b/library/cpp/monlib/encode/unistat/ut/ya.make
new file mode 100644
index 0000000000..1f4590b3fa
--- /dev/null
+++ b/library/cpp/monlib/encode/unistat/ut/ya.make
@@ -0,0 +1,11 @@
+UNITTEST_FOR(library/cpp/monlib/encode/unistat)
+
+SRCS(
+ unistat_ut.cpp
+)
+
+PEERDIR(
+ library/cpp/monlib/encode/protobuf
+)
+
+END()
diff --git a/library/cpp/monlib/encode/unistat/ya.make b/library/cpp/monlib/encode/unistat/ya.make
new file mode 100644
index 0000000000..e054cd707e
--- /dev/null
+++ b/library/cpp/monlib/encode/unistat/ya.make
@@ -0,0 +1,17 @@
+LIBRARY()
+
+PEERDIR(
+ contrib/libs/re2
+ library/cpp/json
+ library/cpp/monlib/metrics
+)
+
+SRCS(
+ unistat_decoder.cpp
+)
+
+END()
+
+RECURSE_FOR_TESTS(
+ ut
+)
diff --git a/library/python/monlib/encoder.pxd b/library/python/monlib/encoder.pxd
new file mode 100644
index 0000000000..f879b66b78
--- /dev/null
+++ b/library/python/monlib/encoder.pxd
@@ -0,0 +1,79 @@
+from util.generic.string cimport TStringBuf, TString
+from util.generic.ptr cimport THolder
+from util.stream.output cimport IOutputStream
+
+from library.python.monlib.metric_consumer cimport IMetricConsumer
+
+
+cdef extern from "util/stream/input.h" nogil:
+ cdef cppclass IInputStream:
+ pass
+
+
+cdef extern from "util/system/file.h" nogil:
+ cdef cppclass TFile:
+ TFile()
+ TFile(TFile)
+ pass
+
+ cdef TFile Duplicate(int)
+
+
+cdef extern from "library/cpp/monlib/encode/encoder.h" namespace "NMonitoring" nogil:
+ cdef cppclass IMetricEncoder:
+ void Close()
+
+ cdef cppclass ECompression:
+ pass
+
+ ctypedef THolder[IMetricEncoder] IMetricEncoderPtr
+
+
+cdef extern from "library/cpp/monlib/encode/unistat/unistat.h" namespace "NMonitoring" nogil:
+ cdef void DecodeUnistat(TStringBuf data, IMetricConsumer* c)
+
+
+cdef extern from "library/cpp/monlib/encode/json/json.h" namespace "NMonitoring" nogil:
+ cdef IMetricEncoderPtr EncoderJson(IOutputStream* out, int indentation)
+ cdef IMetricEncoderPtr BufferedEncoderJson(IOutputStream* out, int indentation)
+
+ cdef void DecodeJson(TStringBuf data, IMetricConsumer* c)
+
+
+cdef extern from "library/cpp/monlib/encode/spack/spack_v1.h" namespace "NMonitoring" nogil:
+ cdef IMetricEncoderPtr EncoderSpackV1(IOutputStream* out, ETimePrecision, ECompression)
+
+ cdef void DecodeSpackV1(IInputStream* input, IMetricConsumer* c) except +
+ cdef cppclass ETimePrecision:
+ pass
+
+ cdef cppclass EValueType:
+ pass
+
+
+cdef extern from "library/cpp/monlib/encode/spack/spack_v1.h" namespace "NMonitoring::ETimePrecision" nogil:
+ cdef ETimePrecision SECONDS "NMonitoring::ETimePrecision::SECONDS"
+ cdef ETimePrecision MILLIS "NMonitoring::ETimePrecision::MILLIS"
+
+
+cdef extern from "library/cpp/monlib/encode/encoder.h" namespace "NMonitoring::ECompression" nogil:
+ cdef ECompression UNKNOWN "NMonitoring::ECompression::UNKNOWN"
+ cdef ECompression IDENTITY "NMonitoring::ECompression::IDENTITY"
+ cdef ECompression ZLIB "NMonitoring::ECompression::ZLIB"
+ cdef ECompression LZ4 "NMonitoring::ECompression::LZ4"
+ cdef ECompression ZSTD "NMonitoring::ECompression::ZSTD"
+
+
+cdef class Encoder:
+ cdef IMetricEncoderPtr __wrapped
+ cdef THolder[TFile] __file
+ cdef THolder[IOutputStream] __stream
+
+ cdef IMetricEncoder* native(self)
+
+ cdef _make_stream(self, py_stream)
+
+ @staticmethod
+ cdef Encoder create_spack(object stream, ETimePrecision timePrecision, ECompression compression)
+ @staticmethod
+ cdef Encoder create_json(object stream, int indent)
diff --git a/library/python/monlib/encoder.pyx b/library/python/monlib/encoder.pyx
new file mode 100644
index 0000000000..05cf4fec9a
--- /dev/null
+++ b/library/python/monlib/encoder.pyx
@@ -0,0 +1,260 @@
+from util.generic.string cimport TString, TStringBuf
+from util.generic.ptr cimport THolder
+
+from cython.operator cimport dereference as deref
+
+import sys
+
+from datetime import datetime
+from os import dup
+
+
+cdef extern from "util/stream/fwd.h" nogil:
+ cdef cppclass TAdaptivelyBuffered[T]:
+ TAdaptivelyBuffered(TFile) except +
+
+ ctypedef TAdaptivelyBuffered[TUnbufferedFileOutput] TFileOutput
+
+cdef extern from "util/stream/mem.h" nogil:
+ cdef cppclass TMemoryInput:
+ TMemoryInput(const TStringBuf buf)
+
+
+cdef extern from "util/stream/file.h" nogil:
+ cdef cppclass TUnbufferedFileOutput:
+ TUnbufferedFileOutput(TFile)
+
+ cdef cppclass TFileInput:
+ TFileInput(TFile) except +
+
+
+cdef extern from "util/stream/str.h" nogil:
+ cdef cppclass TStringStream:
+ const TString& Str() const
+
+
+cdef class Encoder:
+ cdef IMetricEncoder* native(self):
+ return self.__wrapped.Get()
+
+ def close(self):
+ deref(self.__wrapped.Get()).Close()
+
+ def dumps(self):
+ return (<TStringStream&?>deref(self.__stream.Get())).Str()
+
+ cdef _make_stream(self, py_stream):
+ if py_stream is not None:
+ fd = Duplicate(py_stream.fileno())
+
+ self.__file.Reset(new TFile(fd))
+ f = self.__file.Get()
+ self.__stream.Reset(<IOutputStream*>(new TFileOutput(deref(f))))
+ else:
+ self.__stream.Reset(<IOutputStream*>(new TStringStream()))
+
+ @staticmethod
+ cdef Encoder create_spack(object stream, ETimePrecision precision, ECompression compression):
+ cdef Encoder wrapper = Encoder.__new__(Encoder)
+ wrapper._make_stream(stream)
+
+ wrapper.__wrapped = EncoderSpackV1(wrapper.__stream.Get(),
+ precision,
+ compression)
+
+ return wrapper
+
+ @staticmethod
+ cdef Encoder create_json(object stream, int indent):
+ cdef Encoder wrapper = Encoder.__new__(Encoder)
+ wrapper._make_stream(stream)
+
+ wrapper.__wrapped = EncoderJson(wrapper.__stream.Get(), indent)
+
+ return wrapper
+
+
+cpdef Encoder create_json_encoder(object stream, int indent):
+ return Encoder.create_json(stream, indent)
+
+
+cdef class TimePrecision:
+ Millis = <int>MILLIS
+ Seconds = <int>SECONDS
+
+ @staticmethod
+ cdef ETimePrecision to_native(int p) except *:
+ if p == TimePrecision.Millis:
+ return MILLIS
+ elif p == TimePrecision.Seconds:
+ return SECONDS
+
+ raise ValueError('Unsupported TimePrecision value')
+
+cdef class Compression:
+ Identity = <int>IDENTITY
+ Lz4 = <int>LZ4
+ Zlib = <int>ZLIB
+ Zstd = <int>ZSTD
+
+ @staticmethod
+ cdef ECompression to_native(int p) except *:
+ if p == Compression.Identity:
+ return IDENTITY
+ elif p == Compression.Lz4:
+ return LZ4
+ elif p == Compression.Zlib:
+ return ZLIB
+ elif p == Compression.Zstd:
+ return ZSTD
+
+ raise ValueError('Unsupported Compression value')
+
+
+# XXX: timestamps
+def dump(registry, fp, format='spack', **kwargs):
+ """
+ Dumps metrics held by the metric registry to a file. Output can be additionally
+ adjusted using kwargs, which may differ depending on the selected format.
+
+ :param registry: Metric registry object
+ :param fp: File descriptor to serialize to
+ :param format: Format to serialize to (allowed values: spack). Default: json
+
+ Keyword arguments:
+ :param time_precision: Time precision (spack)
+ :param compression: Compression codec (spack)
+ :param indent: Pretty-print indentation for object members and arrays (json)
+ :param timestamp: Metric timestamp datetime
+ :returns: Nothing
+ """
+ if not hasattr(fp, 'fileno'):
+ raise TypeError('Expected a file-like object, but got ' + str(type(fp)))
+
+ if format == 'spack':
+ time_precision = TimePrecision.to_native(kwargs.get('time_precision', TimePrecision.Seconds))
+ compression = Compression.to_native(kwargs.get('compression', Compression.Identity))
+ encoder = Encoder.create_spack(fp, time_precision, compression)
+ elif format == 'json':
+ indent = int(kwargs.get('indent', 0))
+ encoder = Encoder.create_json(fp, indent)
+ timestamp = kwargs.get('timestamp', datetime.utcfromtimestamp(0))
+
+ registry.accept(timestamp, encoder)
+ encoder.close()
+
+
+def dumps(registry, format='spack', **kwargs):
+ """
+ Dumps metrics held by the metric registry to a string. Output can be additionally
+ adjusted using kwargs, which may differ depending on the selected format.
+
+ :param registry: Metric registry object
+ :param format: Format to serialize to (allowed values: spack). Default: json
+
+ Keyword arguments:
+ :param time_precision: Time precision (spack)
+ :param compression: Compression codec (spack)
+ :param indent: Pretty-print indentation for object members and arrays (json)
+ :param timestamp: Metric timestamp datetime
+ :returns: A string of the specified format
+ """
+ if format == 'spack':
+ time_precision = TimePrecision.to_native(kwargs.get('time_precision', TimePrecision.Seconds))
+ compression = Compression.to_native(kwargs.get('compression', Compression.Identity))
+ encoder = Encoder.create_spack(None, time_precision, compression)
+ elif format == 'json':
+ indent = int(kwargs.get('indent', 0))
+ encoder = Encoder.create_json(None, indent)
+ timestamp = kwargs.get('timestamp', datetime.utcfromtimestamp(0))
+
+ registry.accept(timestamp, encoder)
+ encoder.close()
+
+ s = encoder.dumps()
+
+ return s
+
+
+def load(fp, from_format='spack', to_format='json'):
+ """
+ Converts metrics from one format to another.
+
+ :param fp: File to load data from
+ :param from_format: Source string format (allowed values: json, spack, unistat). Default: spack
+ :param to_format: Target format (allowed values: json, spack). Default: json
+ :returns: a string containing metrics in the specified format
+ """
+ if from_format == to_format:
+ return fp.read()
+
+ cdef THolder[TFile] file
+ file.Reset(new TFile(Duplicate(fp.fileno())))
+
+ cdef THolder[TFileInput] input
+ input.Reset(new TFileInput(deref(file.Get())))
+
+ if to_format == 'json':
+ encoder = Encoder.create_json(None, 0)
+ elif to_format == 'spack':
+ encoder = Encoder.create_spack(None, SECONDS, IDENTITY)
+ else:
+ raise ValueError('Unsupported format ' + to_format)
+
+ if from_format == 'spack':
+ DecodeSpackV1(<IInputStream*>(input.Get()), <IMetricConsumer*?>encoder.native())
+ elif from_format == 'json':
+ s = open(fp, 'r').read()
+ DecodeJson(TStringBuf(s), <IMetricConsumer*?>encoder.native())
+ elif from_format == 'unistat':
+ s = open(fp, 'r').read()
+ DecodeJson(TStringBuf(s), <IMetricConsumer*?>encoder.native())
+
+ else:
+ raise ValueError('Unsupported format ' + from_format)
+
+ encoder.close()
+ s = encoder.dumps()
+
+ return s
+
+
+def loads(s, from_format='spack', to_format='json', compression=Compression.Identity):
+ """
+ Converts metrics from one format to another.
+
+ :param s: String to load from
+ :param from_format: Source string format (allowed values: json, spack, unistat). Default: spack
+ :param to_format: Target format (allowed values: json, spack). Default: json
+ :returns: a string containing metrics in the specified format
+ """
+ if from_format == to_format:
+ return s
+
+ if sys.version_info[0] >= 3 and not isinstance(s, bytes):
+ s = s.encode('iso-8859-15')
+
+ cdef THolder[TMemoryInput] input
+
+ if to_format == 'json':
+ encoder = Encoder.create_json(None, 0)
+ elif to_format == 'spack':
+ comp = Compression.to_native(compression)
+ encoder = Encoder.create_spack(None, SECONDS, comp)
+ else:
+ raise ValueError('Unsupported format ' + to_format)
+
+ if from_format == 'spack':
+ input.Reset(new TMemoryInput(s))
+ DecodeSpackV1(<IInputStream*>(input.Get()), <IMetricConsumer*?>encoder.native())
+ elif from_format == 'json':
+ DecodeJson(TStringBuf(s), <IMetricConsumer*?>encoder.native())
+ elif from_format == 'unistat':
+ DecodeUnistat(TStringBuf(s), <IMetricConsumer*?>encoder.native())
+ else:
+ raise ValueError('Unsupported format ' + from_format)
+
+ encoder.close()
+ s = encoder.dumps()
+
+ return s
diff --git a/library/python/monlib/labels.pxd b/library/python/monlib/labels.pxd
new file mode 100644
index 0000000000..cc782433c4
--- /dev/null
+++ b/library/python/monlib/labels.pxd
@@ -0,0 +1,47 @@
+from libcpp cimport bool
+
+from util.generic.maybe cimport TMaybe
+from util.generic.string cimport TStringBuf, TString
+
+
+cdef extern from "library/cpp/monlib/metrics/labels.h" namespace "NMonitoring" nogil:
+ cdef cppclass ILabel:
+ const TStringBuf Name() const
+ const TStringBuf Value() const
+
+ cdef cppclass ILabels:
+ bool Add(TStringBuf name, TStringBuf value)
+ bool Add(const TString& name, const TString& value)
+
+ size_t Size() const
+
+ cdef cppclass TLabel:
+ TLabel() except +
+ TLabel(TStringBuf name, TStringBuf value) except +
+ const TString& Name() const
+ const TString& Value() const
+
+ TString ToString() const
+ bool operator!=(const TLabel&) const
+ bool operator==(const TLabel&) const
+
+ cdef cppclass TLabels:
+ cppclass const_iterator:
+ const TLabel& operator*() const
+ bool operator!=(const_iterator) const
+ bool operator==(const_iterator) const
+
+ TLabels() except +
+
+ bool Add(const TLabel&) except +
+ bool Add(TStringBuf name, TStringBuf value) except +
+ bool Add(const TString& name, const TString& value) except +
+ bool operator==(const TLabels&) const
+
+ TMaybe[TLabel] Find(TStringBuf name) const
+ TMaybe[TLabel] Extract(TStringBuf name) except +
+
+ size_t Size() const
+
+ const_iterator begin() const
+ const_iterator end() const
diff --git a/library/python/monlib/metric.pxd b/library/python/monlib/metric.pxd
new file mode 100644
index 0000000000..afa28ea015
--- /dev/null
+++ b/library/python/monlib/metric.pxd
@@ -0,0 +1,103 @@
+from libcpp cimport bool
+
+from util.system.types cimport ui64, ui32, i64
+from util.generic.ptr cimport THolder, TIntrusivePtr
+from util.generic.vector cimport TVector
+
+
+cdef extern from "library/cpp/monlib/metrics/histogram_collector.h" namespace "NMonitoring" nogil:
+ ctypedef double TBucketBound
+ ctypedef ui64 TBucketValue
+
+ cdef cppclass IHistogramSnapshot:
+ ui32 Count() const
+ TBucketBound UpperBound(ui32 index) const
+ TBucketValue Value(ui32 index) const
+
+ ctypedef TIntrusivePtr[IHistogramSnapshot] IHistogramSnapshotPtr
+
+ cdef cppclass IHistogramCollector:
+ void Collect(i64 value)
+ void Collect(i64 value, ui32 count)
+ IHistogramSnapshotPtr Snapshot() const
+
+ ctypedef THolder[IHistogramCollector] IHistogramCollectorPtr
+
+ IHistogramCollectorPtr ExponentialHistogram(ui32 bucketsCount, double base, double scale) except +
+ IHistogramCollectorPtr ExplicitHistogram(const TVector[double]& buckets) except +
+ IHistogramCollectorPtr LinearHistogram(ui32 bucketsCount, i64 startValue, i64 bucketWidth) except +
+
+
+cdef extern from "library/cpp/monlib/metrics/metric.h" namespace "NMonitoring" nogil:
+ cdef cppclass TGauge:
+ TGauge(double value) except +
+
+ void Set(double)
+ double Get() const
+ double Add(double)
+
+ cdef cppclass TIntGauge:
+ TIntGauge(ui64 value) except +
+
+ void Set(ui64)
+ ui64 Get() const
+ ui64 Add(double)
+ ui64 Inc()
+ ui64 Dec()
+
+ cdef cppclass TCounter:
+ TCounter(ui64 value) except +
+
+ void Set(ui64)
+ ui64 Get() const
+ void Inc()
+ void Reset()
+
+ cdef cppclass TRate:
+ TRate(ui64 value) except +
+
+ void Add(ui64)
+ ui64 Get() const
+ void Inc()
+
+ cdef cppclass THistogram:
+ THistogram(IHistogramCollectorPtr collector, bool isRate) except +
+
+ void Record(double value)
+ void Record(double value, ui32 count)
+
+
+cdef class Gauge:
+ cdef TGauge* __wrapped
+
+ @staticmethod
+ cdef Gauge from_ptr(TGauge* native)
+
+
+cdef class Counter:
+ cdef TCounter* __wrapped
+
+ @staticmethod
+ cdef Counter from_ptr(TCounter* native)
+
+
+cdef class Rate:
+ cdef TRate* __wrapped
+
+ @staticmethod
+ cdef Rate from_ptr(TRate* native)
+
+
+cdef class IntGauge:
+ cdef TIntGauge* __wrapped
+
+ @staticmethod
+ cdef IntGauge from_ptr(TIntGauge* native)
+
+
+cdef class Histogram:
+ cdef THistogram* __wrapped
+ cdef bool __is_owner
+
+ @staticmethod
+ cdef Histogram from_ptr(THistogram* native)
diff --git a/library/python/monlib/metric.pyx b/library/python/monlib/metric.pyx
new file mode 100644
index 0000000000..7b51752335
--- /dev/null
+++ b/library/python/monlib/metric.pyx
@@ -0,0 +1,162 @@
+from libcpp cimport bool
+
+from util.system.types cimport ui32, ui64, i64
+from library.python.monlib.metric cimport (
+ TGauge, TCounter, TRate, TIntGauge, THistogram,
+ IHistogramCollectorPtr)
+
+cdef class Gauge:
+ """
+ Represents a floating point absolute value
+ """
+ @staticmethod
+ cdef Gauge from_ptr(TGauge* native):
+ cdef Gauge wrapper = Gauge.__new__(Gauge)
+ wrapper.__wrapped = native
+
+ return wrapper
+
+ def set(self, double value):
+ """
+ Set metric to the specified value
+ :param value: metric value
+ """
+ self.__wrapped.Set(value)
+
+ def get(self):
+ """
+ Get metric value.
+ :param value: metric value
+ """
+ return self.__wrapped.Get()
+
+ def add(self, double value):
+ """
+ Add value to metric.
+ :param value: metric value
+ """
+ return self.__wrapped.Add(value)
+
+
+cdef class IntGauge:
+ """
+ Represents an integer absolute value
+ """
+ @staticmethod
+ cdef IntGauge from_ptr(TIntGauge* native):
+ cdef IntGauge wrapper = IntGauge.__new__(IntGauge)
+ wrapper.__wrapped = native
+
+ return wrapper
+
+ def set(self, i64 value):
+ """
+ Set metric to the specified value
+ :param value: metric value
+ """
+ self.__wrapped.Set(value)
+
+ def get(self):
+ """
+ Get metric value
+ :param value: metric value
+ """
+ return self.__wrapped.Get()
+
+ def add(self, i64 value):
+ """
+ Add value to metric.
+ :param value: metric value
+ """
+ return self.__wrapped.Add(value)
+
+ def inc(self):
+ """
+ Add 1 to metric.
+ """
+ return self.__wrapped.Inc()
+
+ def dec(self):
+ """
+ Add -1 to metric.
+ """
+ return self.__wrapped.Dec()
+
+
+cdef class Counter:
+ """
+ Represents a counter value
+ """
+ @staticmethod
+ cdef Counter from_ptr(TCounter* native):
+ cdef Counter wrapper = Counter.__new__(Counter)
+ wrapper.__wrapped = native
+
+ return wrapper
+
+ def get(self):
+ return self.__wrapped.Get()
+
+ def inc(self):
+ """
+ Increment metric value
+ """
+ return self.__wrapped.Inc()
+
+ def reset(self):
+ """
+ Reset metric value to zero
+ """
+ return self.__wrapped.Reset()
+
+
+cdef class Rate:
+ """
+ Represents a time derivative
+ """
+ @staticmethod
+ cdef Rate from_ptr(TRate* native):
+ cdef Rate wrapper = Rate.__new__(Rate)
+ wrapper.__wrapped = native
+
+ return wrapper
+
+ def get(self):
+ return self.__wrapped.Get()
+
+ def inc(self):
+ """
+ Increment metric value
+ """
+ return self.__wrapped.Inc()
+
+ def add(self, ui64 value):
+ """
+ Add the value to metric
+ :param value: value to add to metric
+ """
+ return self.__wrapped.Add(value)
+
+cdef class Histogram:
+ """
+ Represents some value distribution
+ """
+ @staticmethod
+ cdef Histogram from_ptr(THistogram* native):
+ cdef Histogram wrapper = Histogram.__new__(Histogram, 0)
+ wrapper.__is_owner = False
+ wrapper.__wrapped = native
+
+ return wrapper
+
+ def __dealloc__(self):
+ if self.__is_owner:
+ del self.__wrapped
+
+ def collect(self, double value, ui32 count=1):
+ """
+ Add a few points with same value to the distribution
+ :param value: points' value
+ :param value: point count
+ """
+ return self.__wrapped.Record(value, count)
diff --git a/library/python/monlib/metric_consumer.pxd b/library/python/monlib/metric_consumer.pxd
new file mode 100644
index 0000000000..7d259f7e3b
--- /dev/null
+++ b/library/python/monlib/metric_consumer.pxd
@@ -0,0 +1,8 @@
+from util.generic.ptr cimport TIntrusivePtr
+
+
+cdef extern from "library/cpp/monlib/metrics/metric_consumer.h" namespace "NMonitoring" nogil:
+ cdef cppclass IMetricConsumer:
+ pass
+
+ ctypedef TIntrusivePtr[IMetricConsumer] IMetricConsumerPtr
diff --git a/library/python/monlib/metric_registry.pxd b/library/python/monlib/metric_registry.pxd
new file mode 100644
index 0000000000..e57c963929
--- /dev/null
+++ b/library/python/monlib/metric_registry.pxd
@@ -0,0 +1,29 @@
+from util.datetime.base cimport TInstant
+
+from library.python.monlib.labels cimport ILabels, TLabels
+from library.python.monlib.metric_consumer cimport IMetricConsumer
+from library.python.monlib.metric cimport (
+ TGauge, TIntGauge, TRate, TCounter, THistogram,
+ IHistogramCollectorPtr)
+
+
+cdef extern from "library/cpp/monlib/metrics/metric_registry.h" namespace "NMonitoring" nogil:
+ cdef cppclass TMetricRegistry:
+ TMetricRegistry() except +
+ TMetricRegistry(const TLabels&) except +
+
+ TGauge* Gauge(const TLabels&) except +
+ TIntGauge* IntGauge(const TLabels&) except +
+ TCounter* Counter(const TLabels&) except +
+ TRate* Rate(const TLabels&) except +
+ THistogram* HistogramCounter(const TLabels&, IHistogramCollectorPtr collector) except +
+ THistogram* HistogramRate(const TLabels&, IHistogramCollectorPtr collector) except +
+
+ void Reset() except +
+
+ void Accept(TInstant time, IMetricConsumer* consumer) except +
+ void Append(TInstant time, IMetricConsumer* consumer) except +
+
+ const TLabels& CommonLabels() const
+
+ void RemoveMetric(const TLabels&)
diff --git a/library/python/monlib/metric_registry.pyx b/library/python/monlib/metric_registry.pyx
new file mode 100644
index 0000000000..800a1abd1b
--- /dev/null
+++ b/library/python/monlib/metric_registry.pyx
@@ -0,0 +1,277 @@
+from library.python.monlib.encoder cimport Encoder
+from library.python.monlib.labels cimport TLabels
+from library.python.monlib.metric cimport (
+ Gauge, IntGauge, Counter, Rate, Histogram, IHistogramCollectorPtr,
+ ExponentialHistogram, ExplicitHistogram, LinearHistogram)
+from library.python.monlib.metric_consumer cimport IMetricConsumer
+from library.python.monlib.metric_registry cimport TMetricRegistry
+
+from util.generic.ptr cimport THolder
+from util.generic.string cimport TString
+from util.datetime.base cimport TInstant
+from util.system.types cimport ui32
+from util.generic.vector cimport TVector
+
+from libcpp.string cimport string
+
+from cython.operator cimport address, dereference as deref
+
+import datetime as dt
+import sys
+
+
+cdef extern from "<utility>" namespace "std" nogil:
+ cdef IHistogramCollectorPtr&& move(IHistogramCollectorPtr t)
+
+
+def get_or_raise(kwargs, key):
+ value = kwargs.get(key)
+ if value is None:
+ raise ValueError(key + ' argument is required but not specified')
+
+ return value
+
+
+class HistogramType(object):
+ Exponential = 0
+ Explicit = 1
+ Linear = 2
+
+
+cdef class MetricRegistry:
+ """
+ Represents an entity holding a set of counters of different types identified by labels
+
+ Example usage:
+ .. ::
+ registry = MetricRegistry()
+
+ response_times = registry.histogram_rate(
+ {'path': 'ping', 'sensor': 'responseTimeMillis'},
+ HistogramType.Explicit, buckets=[10, 20, 50, 200, 500])
+
+ requests = registry.rate({'path': 'ping', 'sensor': 'requestRate'})
+ uptime = registry.gauge({'sensor': 'serverUptimeSeconds'})
+
+ # ...
+ requests.inc()
+ uptime.set(time.time() - start_time)
+
+ # ...
+ dumps(registry)
+ """
+ cdef THolder[TMetricRegistry] __wrapped
+
+ def __cinit__(self, labels=None):
+ cdef TLabels native_labels = MetricRegistry._py_to_native_labels(labels)
+ self.__wrapped.Reset(new TMetricRegistry(native_labels))
+
+ @staticmethod
+ cdef TLabels _py_to_native_labels(dict labels):
+ cdef TLabels native_labels = TLabels()
+
+ if labels is not None:
+ for name, value in labels.items():
+ native_labels.Add(TString(<string>name.encode('utf-8')), TString(<string>value.encode('utf-8')))
+
+ return native_labels
+
+ @staticmethod
+ cdef _native_to_py_labels(const TLabels& native_labels):
+ result = dict()
+
+ cdef TLabels.const_iterator it = native_labels.begin()
+ while it != native_labels.end():
+ name = TString(deref(it).Name())
+ value = TString(deref(it).Value())
+ if (isinstance(name, bytes)):
+ name = name.decode('utf-8')
+
+ if (isinstance(value, bytes)):
+ value = value.decode('utf-8')
+
+ result[name] = value
+ it += 1
+
+ return result
+
+ def _histogram(self, labels, is_rate, hist_type, **kwargs):
+ cdef TLabels native_labels = MetricRegistry._py_to_native_labels(labels)
+ cdef IHistogramCollectorPtr collector
+ cdef TVector[double] native_buckets
+
+ if hist_type == HistogramType.Exponential:
+ buckets = int(get_or_raise(kwargs, 'bucket_count'))
+ base = float(get_or_raise(kwargs, 'base'))
+ scale = float(kwargs.get('scale', 1.))
+ collector = move(ExponentialHistogram(buckets, base, scale))
+ elif hist_type == HistogramType.Explicit:
+ buckets = get_or_raise(kwargs, 'buckets')
+ native_buckets = buckets
+ collector = move(ExplicitHistogram(native_buckets))
+ elif hist_type == HistogramType.Linear:
+ buckets = get_or_raise(kwargs, 'bucket_count')
+ start_value = get_or_raise(kwargs, 'start_value')
+ bucket_width = get_or_raise(kwargs, 'bucket_width')
+ collector = move(LinearHistogram(buckets, start_value, bucket_width))
+ else:
+ # XXX: string representation
+ raise ValueError('histogram type {} is not supported'.format(str(hist_type)))
+
+ cdef THistogram* native_hist
+ if is_rate:
+ native_hist = self.__wrapped.Get().HistogramRate(native_labels, move(collector))
+ else:
+ native_hist = self.__wrapped.Get().HistogramCounter(native_labels, move(collector))
+
+ return Histogram.from_ptr(native_hist)
+
+ @property
+ def common_labels(self):
+ """
+ Gets labels that are common among all the counters in this registry
+
+ :returns: Common labels as a dict
+ """
+ cdef const TLabels* native = address(self.__wrapped.Get().CommonLabels())
+ labels = MetricRegistry._native_to_py_labels(deref(native))
+
+ return labels
+
+ def gauge(self, labels):
+ """
+ Gets a gauge counter or creates a new one in case counter with the specified labels
+ does not exist
+
+ :param labels: A dict of labels which identifies counter
+ :returns: Gauge counter
+ """
+ cdef TLabels native_labels = MetricRegistry._py_to_native_labels(labels)
+ native_gauge = self.__wrapped.Get().Gauge(native_labels)
+ return Gauge.from_ptr(native_gauge)
+
+ def int_gauge(self, labels):
+ """
+ Gets a gauge counter or creates a new one in case counter with the specified labels
+ does not exist
+
+ :param labels: A dict of labels which identifies counter
+ :returns: IntGauge counter
+ """
+ cdef TLabels native_labels = MetricRegistry._py_to_native_labels(labels)
+ native_gauge = self.__wrapped.Get().IntGauge(native_labels)
+ return IntGauge.from_ptr(native_gauge)
+
+ def counter(self, labels):
+ """
+ Gets a counter or creates a new one in case counter with the specified labels
+ does not exist
+
+ :param labels: A dict of labels which identifies counter
+ :returns: Counter counter
+ """
+ cdef TLabels native_labels = MetricRegistry._py_to_native_labels(labels)
+ native_counter = self.__wrapped.Get().Counter(native_labels)
+ return Counter.from_ptr(native_counter)
+
+ def rate(self, labels):
+ """
+ Gets a rate counter or creates a new one in case counter with the specified labels
+ does not exist
+
+ :param labels: A dict of labels which identifies counter
+ :returns: Rate counter
+ """
+ cdef TLabels native_labels = MetricRegistry._py_to_native_labels(labels)
+ native_rate = self.__wrapped.Get().Rate(native_labels)
+ return Rate.from_ptr(native_rate)
+
+ def histogram_counter(self, labels, hist_type, **kwargs):
+ """
+ Gets a histogram counter or creates a new one in case counter with the specified labels
+ does not exist
+
+ :param labels: A dict of labels which identifies counter
+ :param hist_type: Specifies the way histogram buckets are defined (allowed values: explicit, exponential, linear)
+
+ Keyword arguments:
+ :param buckets: A list of bucket upper bounds (explicit)
+ :param bucket_count: Number of buckets (linear, exponential)
+ :param base: the exponential growth factor for buckets' width (exponential)
+ :param scale: linear scale for the buckets. Must be >= 1.0 (exponential)
+ :param start_value: the upper bound of the first bucket (linear)
+
+ :returns: Histogram counter
+
+ Example usage:
+ .. ::
+ my_histogram = registry.histogram_counter(
+ {'path': 'ping', 'sensor': 'responseTimeMillis'},
+ HistogramType.Explicit, buckets=[10, 20, 50, 200, 500])
+ # (-inf; 10] (10; 20] (20; 50] (200; 500] (500; +inf)
+
+ # or:
+ my_histogram = registry.histogram_counter(
+ {'path': 'ping', 'sensor': 'responseTimeMillis'},
+ HistogramType.Linear, bucket_count=4, bucket_width=10, start_value=0)
+ # (-inf; 0] (0; 10] (10; 20] (20; +inf)
+
+ # or:
+ my_histogram = registry.histogram_counter(
+ {'path': 'ping', 'sensor': 'responseTimeMillis'},
+ HistogramType.Exponential, bucket_count=6, base=2, scale=3)
+ # (-inf; 3] (3; 6] (6; 12] (12; 24] (24; 48] (48; +inf)
+ ::
+ """
+ return self._histogram(labels, False, hist_type, **kwargs)
+
+ def histogram_rate(self, labels, hist_type, **kwargs):
+ """
+ Gets a histogram rate counter or creates a new one in case counter with the specified labels
+ does not exist
+
+ :param labels: A dict of labels which identifies counter
+ :param hist_type: Specifies the way histogram buckets are defined (allowed values: explicit, exponential, linear)
+
+ Keyword arguments:
+ :param buckets: A list of bucket upper bounds (explicit)
+ :param bucket_count: Number of buckets (linear, exponential)
+ :param base: the exponential growth factor for buckets' width (exponential)
+ :param scale: linear scale for the buckets. Must be >= 1.0 (exponential)
+ :param start_value: the upper bound of the first bucket (linear)
+
+ :returns: Histogram counter
+
+ Example usage:
+ .. ::
+ my_histogram = registry.histogram_counter(
+ {'path': 'ping', 'sensor': 'responseTimeMillis'},
+ HistogramType.Explicit, buckets=[10, 20, 50, 200, 500])
+ # (-inf; 10] (10; 20] (20; 50] (200; 500] (500; +inf)
+
+ # or:
+ my_histogram = registry.histogram_counter(
+ {'path': 'ping', 'sensor': 'responseTimeMillis'},
+ HistogramType.Linear, bucket_count=4, bucket_width=10, start_value=0)
+ # (-inf; 0] (0; 10] (10; 20] (20; +inf)
+
+ # or:
+ my_histogram = registry.histogram_counter(
+ {'path': 'ping', 'sensor': 'responseTimeMillis'},
+ HistogramType.Exponential, bucket_count=6, base=2, scale=3)
+ # (-inf; 3] (3; 6] (6; 12] (12; 24] (24; 48] (48; +inf)
+ ::
+ """
+ return self._histogram(labels, True, hist_type, **kwargs)
+
+ def reset(self):
+ self.__wrapped.Get().Reset()
+
+ def accept(self, time, Encoder encoder):
+ cdef IMetricConsumer* ptr = <IMetricConsumer*>encoder.native()
+ timestamp = int((time - dt.datetime(1970, 1, 1)).total_seconds())
+ self.__wrapped.Get().Accept(TInstant.Seconds(timestamp), ptr)
+
+ def remove_metric(self, labels):
+ cdef TLabels native_labels = MetricRegistry._py_to_native_labels(labels)
+ self.__wrapped.Get().RemoveMetric(native_labels)
diff --git a/library/python/monlib/ut/metric_ut.pyx b/library/python/monlib/ut/metric_ut.pyx
new file mode 100644
index 0000000000..3513eaf9d1
--- /dev/null
+++ b/library/python/monlib/ut/metric_ut.pyx
@@ -0,0 +1,113 @@
+from library.python.monlib.labels cimport TLabels, TLabel
+from library.python.monlib.metric cimport (
+ TGauge, TCounter,
+ TRate, THistogram,
+ IHistogramCollectorPtr, ExponentialHistogram,
+ IHistogramSnapshotPtr
+)
+
+from library.python.monlib.metric_registry cimport TMetricRegistry
+
+from util.generic.string cimport TStringBuf, TString
+from util.generic.maybe cimport TMaybe
+from util.generic.ptr cimport THolder
+
+from cython.operator cimport dereference as deref
+
+import pytest
+import unittest
+
+
+cdef extern from "<utility>" namespace "std" nogil:
+ cdef IHistogramCollectorPtr&& move(IHistogramCollectorPtr t)
+
+
+class TestMetric(unittest.TestCase):
+ def test_labels(self):
+ cdef TLabels labels = TLabels()
+ cdef TString name = "foo"
+ cdef TString value = "bar"
+
+ labels.Add(name, value)
+
+ cdef TMaybe[TLabel] label = labels.Find(name)
+
+ assert label.Defined()
+ assert label.GetRef().Name() == "foo"
+ assert label.GetRef().Value() == "bar"
+
+ def test_metric_registry(self):
+ cdef TLabels labels = TLabels()
+
+ labels.Add(TString("common"), TString("label"))
+
+ cdef THolder[TMetricRegistry] registry
+ registry.Reset(new TMetricRegistry(labels))
+
+ assert deref(registry.Get()).CommonLabels() == labels
+
+ cdef TLabels metric_labels = TLabels()
+ metric_labels.Add(TString("name"), TString("gauge"))
+ g = deref(registry.Get()).Gauge(metric_labels)
+ assert g.Get() == 0.
+
+ metric_labels = TLabels()
+ metric_labels.Add(TString("name"), TString("counter"))
+ c = deref(registry.Get()).Counter(metric_labels)
+ assert c.Get() == 0.
+
+ metric_labels = TLabels()
+ metric_labels.Add(TString("name"), TString("rate"))
+ r = deref(registry.Get()).Rate(metric_labels)
+ assert r.Get() == 0.
+
+ metric_labels = TLabels()
+ metric_labels.Add(TString("name"), TString("int_gauge"))
+ ig = deref(registry.Get()).IntGauge(metric_labels)
+ assert ig.Get() == 0
+
+ def test_metric_registry_throws_on_duplicate(self):
+ cdef THolder[TMetricRegistry] registry
+ registry.Reset(new TMetricRegistry())
+
+ cdef TLabels metric_labels = TLabels()
+ metric_labels.Add(TString("my"), TString("metric"))
+ g = deref(registry.Get()).Gauge(metric_labels)
+ with pytest.raises(RuntimeError):
+ deref(registry.Get()).Counter(metric_labels)
+
+ def test_counter_histogram(self):
+ cdef THolder[TMetricRegistry] registry
+ registry.Reset(new TMetricRegistry())
+
+ cdef TLabels metric_labels = TLabels()
+ metric_labels.Add(TString("name"), TString("histogram"))
+
+ cdef IHistogramCollectorPtr collector = move(ExponentialHistogram(6, 2, 3))
+ collector_ptr = collector.Get()
+ hist = registry.Get().HistogramCounter(metric_labels, move(collector))
+ hist.Record(1)
+ hist.Record(1000, 4)
+
+ cdef IHistogramSnapshotPtr snapshot = collector_ptr.Snapshot()
+ assert deref(snapshot.Get()).Count() == 6
+ assert snapshot.Get().Value(0) == 1
+
+ def test_rate_histogram(self):
+ cdef THolder[TMetricRegistry] registry
+ registry.Reset(new TMetricRegistry())
+
+ cdef TLabels metric_labels = TLabels()
+ metric_labels.Add(TString("name"), TString("histogram"))
+
+ cdef IHistogramCollectorPtr collector = move(ExponentialHistogram(6, 2, 3))
+ collector_ptr = collector.Get()
+ hist = registry.Get().HistogramRate(metric_labels, move(collector))
+ hist.Record(1)
+ hist.Record(1000, 4)
+
+ cdef IHistogramSnapshotPtr snapshot = collector_ptr.Snapshot()
+ assert deref(snapshot.Get()).Count() == 6
+ assert snapshot.Get().Value(0) == 1
+ assert snapshot.Get().Value(5) == 4
+ assert snapshot.Get().Value(5) == 4
diff --git a/library/python/monlib/ut/py2/test.py b/library/python/monlib/ut/py2/test.py
new file mode 100644
index 0000000000..2880120a12
--- /dev/null
+++ b/library/python/monlib/ut/py2/test.py
@@ -0,0 +1,313 @@
+# -- coding: utf-8 --
+
+from __future__ import print_function
+
+import sys # noqa
+import json
+
+from tempfile import TemporaryFile
+
+import pytest # noqa
+
+from library.python.monlib.metric_registry import MetricRegistry, HistogramType
+from library.python.monlib.encoder import dump, dumps, TimePrecision, load, loads # noqa
+
+
+def test_common_labels(request):
+ labels = {'my': 'label'}
+ registry = MetricRegistry(labels)
+ assert registry.common_labels == labels
+
+ with pytest.raises(TypeError):
+ MetricRegistry('foo')
+
+ with pytest.raises(TypeError):
+ MetricRegistry([])
+
+
+def test_json_serialization(request):
+ registry = MetricRegistry()
+ labels = {'foo': 'gauge'}
+
+ g = registry.gauge(labels)
+
+ g.set(10.0)
+ g.set(20)
+
+ c = registry.counter({'foo': 'counter'})
+ c.inc()
+
+ r = registry.rate({'foo': 'rate'})
+ r.add(10)
+
+ out = dumps(registry, format='json', precision=TimePrecision.Seconds)
+ expected = json.loads("""{
+ "sensors":
+ [
+ {
+ "kind":"RATE",
+ "labels":
+ {
+ "foo":"rate"
+ },
+ "value":10
+ },
+ {
+ "kind":"COUNTER",
+ "labels":
+ {
+ "foo":"counter"
+ },
+ "value":1
+ },
+ {
+ "kind":"GAUGE",
+ "labels":
+ {
+ "foo":"gauge"
+ },
+ "value":20
+ }
+ ]
+ }
+ """)
+
+ j = json.loads(out)
+ assert j == expected
+
+
+EXPECTED_EXPLICIT = json.loads("""
+ {
+ "sensors":
+ [
+ {
+ "kind":"HIST",
+ "labels":
+ {
+ "foo":"hist"
+ },
+ "hist":
+ {
+ "bounds":
+ [
+ 2,
+ 10,
+ 500
+ ],
+ "buckets":
+ [
+ 1,
+ 0,
+ 0
+ ],
+ "inf":1
+ }
+ }
+ ]
+ }
+""")
+
+EXPECTED_EXPONENTIAL = json.loads("""{
+ "sensors":
+ [
+ {
+ "kind":"HIST",
+ "labels":
+ {
+ "foo":"hist"
+ },
+ "hist":
+ {
+ "bounds":
+ [
+ 3,
+ 6,
+ 12,
+ 24,
+ 48
+ ],
+ "buckets":
+ [
+ 1,
+ 0,
+ 0,
+ 0,
+ 0
+ ],
+ "inf":1
+ }
+ }
+ ]
+}
+""")
+
+EXPECTED_LINEAR = json.loads("""
+ { "sensors":
+ [
+ {
+ "kind":"HIST",
+ "labels":
+ {
+ "foo":"hist"
+ },
+ "hist":
+ {
+ "bounds":
+ [
+ 1
+ ],
+ "buckets":
+ [
+ 1
+ ],
+ "inf":1
+ }
+ }
+ ]
+}""")
+
+
+@pytest.mark.parametrize('type,args,expected', [
+ (HistogramType.Linear, dict(bucket_count=2, start_value=1, bucket_width=1), EXPECTED_LINEAR),
+ (HistogramType.Explicit, dict(buckets=[2, 10, 500]), EXPECTED_EXPLICIT),
+ (HistogramType.Exponential, dict(bucket_count=6, base=2, scale=3), EXPECTED_EXPONENTIAL),
+])
+@pytest.mark.parametrize('rate', [True, False])
+def test_histograms(request, type, args, expected, rate):
+ registry = MetricRegistry()
+ labels = {'foo': 'hist'}
+
+ h = registry.histogram_counter(labels, type, **args) if not rate else registry.histogram_rate(labels, type, **args)
+ h.collect(1)
+ h.collect(1000, 1)
+
+ s = dumps(registry, format='json')
+
+ if rate:
+ expected['sensors'][0]['kind'] = u'HIST_RATE'
+ else:
+ expected['sensors'][0]['kind'] = u'HIST'
+
+ assert json.loads(s) == expected
+
+
+@pytest.mark.parametrize('fmt', ['json', 'spack'])
+def test_stream_load(request, fmt):
+ expected = json.loads("""{"sensors":[{"kind":"GAUGE","labels":{"foo":"gauge"},"value":42}]}""")
+ registry = MetricRegistry()
+ labels = {'foo': 'gauge'}
+
+ g = registry.gauge(labels)
+ g.set(42)
+
+ with TemporaryFile() as f:
+ dump(registry, f, format=fmt)
+ f.flush()
+ f.seek(0, 0)
+ s = load(f, from_format=fmt, to_format='json')
+ assert json.loads(s) == expected
+
+
+@pytest.mark.parametrize('fmt', ['json', 'spack'])
+def test_stream_loads(request, fmt):
+ expected = json.loads("""{"sensors":[{"kind":"GAUGE","labels":{"foo":"gauge"},"value":42}]}""")
+ registry = MetricRegistry()
+ labels = {'foo': 'gauge'}
+
+ g = registry.gauge(labels)
+ g.set(42)
+
+ s = dumps(registry, format=fmt)
+ j = loads(s, from_format=fmt, to_format='json')
+ assert json.loads(j) == expected
+
+
+@pytest.mark.parametrize('fmt', ['json', 'spack'])
+def test_utf(request, fmt):
+ expected = json.loads(u"""{"sensors":[{"kind":"GAUGE","labels":{"foo":"gaugeह", "bàr":"Münich"},"value":42}]}""".encode('utf-8'))
+ registry = MetricRegistry()
+ labels = {'foo': u'gaugeह', u'bàr': u'Münich'}
+
+ g = registry.gauge(labels)
+ g.set(42)
+
+ s = dumps(registry, format=fmt)
+ j = loads(s, from_format=fmt, to_format='json')
+ assert json.loads(j) == expected
+
+
+def test_gauge_sensors():
+ registry = MetricRegistry()
+ g = registry.gauge({'a': 'b'})
+ ig = registry.int_gauge({'c': 'd'})
+
+ g.set(2)
+ assert g.add(3.5) == 5.5
+ assert g.get() == 5.5
+
+ ig.set(2)
+ assert ig.inc() == 3
+ assert ig.dec() == 2
+ assert ig.add(3) == 5
+ assert ig.get() == 5
+
+
+UNISTAT_DATA = """[
+ ["signal1_max", 10],
+ ["signal2_hgram", [[0, 100], [50, 200], [200, 300]]],
+ ["prj=some-project;signal3_summ", 3],
+ ["signal4_summ", 5]
+]"""
+
+
+EXPECTED = json.loads("""
+{
+ "sensors": [
+ {
+ "kind": "GAUGE",
+ "labels": {
+ "sensor": "signal1_max"
+ },
+ "value": 10
+ },
+ {
+ "hist": {
+ "buckets": [
+ 0,
+ 100,
+ 200
+ ],
+ "bounds": [
+ 0,
+ 50,
+ 200
+ ],
+ "inf": 300
+ },
+ "kind": "HIST_RATE",
+ "labels": {
+ "sensor": "signal2_hgram"
+ }
+ },
+ {
+ "kind": "RATE",
+ "labels": {
+ "sensor": "signal3_summ",
+ "prj": "some-project"
+ },
+ "value": 3
+ },
+ {
+ "kind": "RATE",
+ "labels": {
+ "sensor": "signal4_summ"
+ },
+ "value": 5
+ }
+ ]
+}""")
+
+
+def test_unistat_conversion(request):
+ j = loads(UNISTAT_DATA, from_format='unistat', to_format='json')
+ assert json.loads(j) == EXPECTED
diff --git a/library/python/monlib/ut/py2/test_metric.py b/library/python/monlib/ut/py2/test_metric.py
new file mode 100644
index 0000000000..fe391ce35d
--- /dev/null
+++ b/library/python/monlib/ut/py2/test_metric.py
@@ -0,0 +1,3 @@
+from library.python.monlib.ut.metric_ut import TestMetric
+
+__all__ = ['TestMetric']
diff --git a/library/python/monlib/ut/py2/ya.make b/library/python/monlib/ut/py2/ya.make
new file mode 100644
index 0000000000..b158bb357b
--- /dev/null
+++ b/library/python/monlib/ut/py2/ya.make
@@ -0,0 +1,17 @@
+PY2TEST()
+
+TEST_SRCS(
+ test_metric.py
+ test.py
+)
+
+SRCDIR(
+ library/python/monlib/ut
+)
+
+PEERDIR(
+ library/python/monlib
+ library/python/monlib/ut
+)
+
+END()
diff --git a/library/python/monlib/ut/py3/test.py b/library/python/monlib/ut/py3/test.py
new file mode 100644
index 0000000000..431241c657
--- /dev/null
+++ b/library/python/monlib/ut/py3/test.py
@@ -0,0 +1,311 @@
+from __future__ import print_function
+
+import sys # noqa
+import json
+
+from tempfile import TemporaryFile
+
+import pytest # noqa
+
+from library.python.monlib.metric_registry import MetricRegistry, HistogramType
+from library.python.monlib.encoder import dump, dumps, TimePrecision, load, loads # noqa
+
+
+def test_common_labels(request):
+ labels = {'my': 'label'}
+ registry = MetricRegistry(labels)
+ assert registry.common_labels == labels
+
+ with pytest.raises(TypeError):
+ MetricRegistry('foo')
+
+ with pytest.raises(TypeError):
+ MetricRegistry([])
+
+
+def test_json_serialization(request):
+ registry = MetricRegistry()
+ labels = {'foo': 'gauge'}
+
+ g = registry.gauge(labels)
+
+ g.set(10.0)
+ g.set(20)
+
+ c = registry.counter({'foo': 'counter'})
+ c.inc()
+
+ r = registry.rate({'foo': 'rate'})
+ r.add(10)
+
+ out = dumps(registry, format='json', precision=TimePrecision.Seconds)
+ expected = json.loads("""{
+ "sensors":
+ [
+ {
+ "kind":"RATE",
+ "labels":
+ {
+ "foo":"rate"
+ },
+ "value":10
+ },
+ {
+ "kind":"COUNTER",
+ "labels":
+ {
+ "foo":"counter"
+ },
+ "value":1
+ },
+ {
+ "kind":"GAUGE",
+ "labels":
+ {
+ "foo":"gauge"
+ },
+ "value":20
+ }
+ ]
+ }
+ """)
+
+ j = json.loads(out)
+ assert j == expected
+
+
+EXPECTED_EXPLICIT = json.loads("""
+ {
+ "sensors":
+ [
+ {
+ "kind":"HIST",
+ "labels":
+ {
+ "foo":"hist"
+ },
+ "hist":
+ {
+ "bounds":
+ [
+ 2,
+ 10,
+ 500
+ ],
+ "buckets":
+ [
+ 1,
+ 0,
+ 0
+ ],
+ "inf":1
+ }
+ }
+ ]
+ }
+""")
+
+EXPECTED_EXPONENTIAL = json.loads("""{
+ "sensors":
+ [
+ {
+ "kind":"HIST",
+ "labels":
+ {
+ "foo":"hist"
+ },
+ "hist":
+ {
+ "bounds":
+ [
+ 3,
+ 6,
+ 12,
+ 24,
+ 48
+ ],
+ "buckets":
+ [
+ 1,
+ 0,
+ 0,
+ 0,
+ 0
+ ],
+ "inf":1
+ }
+ }
+ ]
+}
+""")
+
+EXPECTED_LINEAR = json.loads("""
+ { "sensors":
+ [
+ {
+ "kind":"HIST",
+ "labels":
+ {
+ "foo":"hist"
+ },
+ "hist":
+ {
+ "bounds":
+ [
+ 1
+ ],
+ "buckets":
+ [
+ 1
+ ],
+ "inf":1
+ }
+ }
+ ]
+}""")
+
+
+@pytest.mark.parametrize('type,args,expected', [
+ (HistogramType.Linear, dict(bucket_count=2, start_value=1, bucket_width=1), EXPECTED_LINEAR),
+ (HistogramType.Explicit, dict(buckets=[2, 10, 500]), EXPECTED_EXPLICIT),
+ (HistogramType.Exponential, dict(bucket_count=6, base=2, scale=3), EXPECTED_EXPONENTIAL),
+])
+@pytest.mark.parametrize('rate', [True, False])
+def test_histograms(request, type, args, expected, rate):
+ registry = MetricRegistry()
+ labels = {'foo': 'hist'}
+
+ h = registry.histogram_counter(labels, type, **args) if not rate else registry.histogram_rate(labels, type, **args)
+ h.collect(1)
+ h.collect(1000)
+
+ s = dumps(registry, format='json')
+
+ if rate:
+ expected['sensors'][0]['kind'] = u'HIST_RATE'
+ else:
+ expected['sensors'][0]['kind'] = u'HIST'
+
+ assert json.loads(s) == expected
+
+
+@pytest.mark.parametrize('fmt', ['json', 'spack'])
+def test_stream_load(request, fmt):
+ expected = json.loads("""{"sensors":[{"kind":"GAUGE","labels":{"foo":"gauge"},"value":42}]}""")
+ registry = MetricRegistry()
+ labels = {'foo': 'gauge'}
+
+ g = registry.gauge(labels)
+ g.set(42)
+
+ with TemporaryFile() as f:
+ dump(registry, f, format=fmt)
+ f.flush()
+ f.seek(0, 0)
+ s = load(f, from_format=fmt, to_format='json')
+ assert json.loads(s) == expected
+
+
+@pytest.mark.parametrize('fmt', ['json', 'spack'])
+def test_stream_loads(request, fmt):
+ expected = json.loads("""{"sensors":[{"kind":"GAUGE","labels":{"foo":"gauge"},"value":42}]}""")
+ registry = MetricRegistry()
+ labels = {'foo': 'gauge'}
+
+ g = registry.gauge(labels)
+ g.set(42)
+
+ s = dumps(registry, format=fmt)
+ j = loads(s, from_format=fmt, to_format='json')
+ assert json.loads(j) == expected
+
+
+@pytest.mark.parametrize('fmt', ['json', 'spack'])
+def test_utf(request, fmt):
+ expected = json.loads(u"""{"sensors":[{"kind":"GAUGE","labels":{"foo":"gaugeह", "bàr":"Münich"},"value":42}]}""")
+ registry = MetricRegistry()
+ labels = {'foo': u'gaugeह', u'bàr': u'Münich'}
+
+ g = registry.gauge(labels)
+ g.set(42)
+
+ s = dumps(registry, format=fmt)
+ j = loads(s, from_format=fmt, to_format='json')
+ assert json.loads(j) == expected
+
+
+def test_gauge_sensors():
+ registry = MetricRegistry()
+ g = registry.gauge({'a': 'b'})
+ ig = registry.int_gauge({'c': 'd'})
+
+ g.set(2)
+ assert g.add(3.5) == 5.5
+ assert g.get() == 5.5
+
+ ig.set(2)
+ assert ig.inc() == 3
+ assert ig.dec() == 2
+ assert ig.add(3) == 5
+ assert ig.get() == 5
+
+
+UNISTAT_DATA = """[
+ ["signal1_max", 10],
+ ["signal2_hgram", [[0, 100], [50, 200], [200, 300]]],
+ ["prj=some-project;signal3_summ", 3],
+ ["signal4_summ", 5]
+]"""
+
+
+EXPECTED = json.loads("""
+{
+ "sensors": [
+ {
+ "kind": "GAUGE",
+ "labels": {
+ "sensor": "signal1_max"
+ },
+ "value": 10
+ },
+ {
+ "hist": {
+ "buckets": [
+ 0,
+ 100,
+ 200
+ ],
+ "bounds": [
+ 0,
+ 50,
+ 200
+ ],
+ "inf": 300
+ },
+ "kind": "HIST_RATE",
+ "labels": {
+ "sensor": "signal2_hgram"
+ }
+ },
+ {
+ "kind": "RATE",
+ "labels": {
+ "sensor": "signal3_summ",
+ "prj": "some-project"
+ },
+ "value": 3
+ },
+ {
+ "kind": "RATE",
+ "labels": {
+ "sensor": "signal4_summ"
+ },
+ "value": 5
+ }
+ ]
+}""")
+
+
+def test_unistat_conversion(request):
+ j = loads(UNISTAT_DATA, from_format='unistat', to_format='json')
+ assert json.loads(j) == EXPECTED
diff --git a/library/python/monlib/ut/py3/test_metric.py b/library/python/monlib/ut/py3/test_metric.py
new file mode 100644
index 0000000000..fe391ce35d
--- /dev/null
+++ b/library/python/monlib/ut/py3/test_metric.py
@@ -0,0 +1,3 @@
+from library.python.monlib.ut.metric_ut import TestMetric
+
+__all__ = ['TestMetric']
diff --git a/library/python/monlib/ut/py3/ya.make b/library/python/monlib/ut/py3/ya.make
new file mode 100644
index 0000000000..d711132df8
--- /dev/null
+++ b/library/python/monlib/ut/py3/ya.make
@@ -0,0 +1,17 @@
+PY3TEST()
+
+TEST_SRCS(
+ test_metric.py
+ test.py
+)
+
+SRCDIR(
+ library/python/monlib/ut
+)
+
+PEERDIR(
+ library/python/monlib
+ library/python/monlib/ut
+)
+
+END()
diff --git a/library/python/monlib/ut/ya.make b/library/python/monlib/ut/ya.make
new file mode 100644
index 0000000000..9082c0e323
--- /dev/null
+++ b/library/python/monlib/ut/ya.make
@@ -0,0 +1,7 @@
+PY23_LIBRARY()
+
+PY_SRCS(
+ metric_ut.pyx
+)
+
+END()
diff --git a/library/python/monlib/ya.make b/library/python/monlib/ya.make
new file mode 100644
index 0000000000..5c7de8de58
--- /dev/null
+++ b/library/python/monlib/ya.make
@@ -0,0 +1,21 @@
+PY23_LIBRARY()
+
+PY_SRCS(
+ encoder.pyx
+ metric.pyx
+ metric_registry.pyx
+)
+
+PEERDIR(
+ library/cpp/monlib/metrics
+ library/cpp/monlib/encode/json
+ library/cpp/monlib/encode/spack
+ library/cpp/monlib/encode/unistat
+)
+
+END()
+
+RECURSE_FOR_TESTS(
+ ut/py2
+ ut/py3
+)
diff --git a/util/datetime/base.pxd b/util/datetime/base.pxd
new file mode 100644
index 0000000000..1d863b1bd8
--- /dev/null
+++ b/util/datetime/base.pxd
@@ -0,0 +1,126 @@
+from libc.stdint cimport uint32_t
+from libc.stdint cimport uint64_t
+from libcpp cimport bool as bool_t
+from posix.types cimport time_t
+
+from util.generic.string cimport TString, TStringBuf
+
+
+cdef extern from "<util/datetime/base.h>" nogil:
+
+ cdef cppclass TTimeBase:
+ TTimeBase()
+ TTimeBase(uint64_t)
+
+ uint64_t GetValue()
+ double SecondsFloat()
+ uint64_t MicroSeconds()
+ uint64_t MilliSeconds()
+ uint64_t Seconds()
+ uint64_t Minutes()
+ uint64_t Hours()
+ uint64_t Days()
+ uint64_t NanoSeconds()
+ uint32_t MicroSecondsOfSecond()
+ uint32_t MilliSecondsOfSecond()
+ uint32_t NanoSecondsOfSecond()
+
+
+ cdef cppclass TInstant(TTimeBase):
+ TInstant()
+ TInstant(uint64_t)
+
+ @staticmethod
+ TInstant Now() except +
+ @staticmethod
+ TInstant Max()
+ @staticmethod
+ TInstant Zero()
+ @staticmethod
+ TInstant MicroSeconds(uint64_t)
+ @staticmethod
+ TInstant MilliSeconds(uint64_t)
+ @staticmethod
+ TInstant Seconds(uint64_t)
+ @staticmethod
+ TInstant Minutes(uint64_t)
+ @staticmethod
+ TInstant Hours(uint64_t)
+ @staticmethod
+ TInstant Days(uint64_t)
+
+ time_t TimeT()
+
+ TString ToString() except +
+ TString ToStringUpToSeconds() except +
+ TString ToStringLocal() except +
+ TString ToStringLocalUpToSeconds() except +
+ TString FormatLocalTime(const char*)
+ TString FormatGmTime(const char* format)
+
+ @staticmethod
+ TInstant ParseIso8601(const TStringBuf) except +
+ @staticmethod
+ TInstant ParseRfc822(const TStringBuf) except +
+ @staticmethod
+ TInstant ParseHttp(const TStringBuf) except +
+ @staticmethod
+ TInstant ParseX509Validity(const TStringBuf) except +
+
+ @staticmethod
+ bool_t TryParseIso8601(const TStringBuf, TInstant&) except +
+ @staticmethod
+ bool_t TryParseRfc822(const TStringBuf, TInstant&) except +
+ @staticmethod
+ bool_t TryParseHttp(const TStringBuf, TInstant&) except +
+ @staticmethod
+ bool_t TryParseX509(const TStringBuf, TInstant&) except +
+
+ @staticmethod
+ TInstant ParseIso8601Deprecated(const TStringBuf) except +
+ @staticmethod
+ TInstant ParseRfc822Deprecated(const TStringBuf) except +
+ @staticmethod
+ TInstant ParseHttpDeprecated(const TStringBuf) except +
+ @staticmethod
+ TInstant ParseX509ValidityDeprecated(const TStringBuf) except +
+
+ @staticmethod
+ bool_t TryParseIso8601Deprecated(const TStringBuf, TInstant&) except +
+ @staticmethod
+ bool_t TryParseRfc822Deprecated(const TStringBuf, TInstant&) except +
+ @staticmethod
+ bool_t TryParseHttpDeprecated(const TStringBuf, TInstant&) except +
+ @staticmethod
+ bool_t TryParseX509Deprecated(const TStringBuf, TInstant&) except +
+
+
+ cdef cppclass TDuration(TTimeBase):
+ TDuration()
+ TDuration(uint64_t)
+
+ @staticmethod
+ TDuration MicroSeconds(uint64_t)
+
+ TInstant ToDeadLine() except +
+ TInstant ToDeadLine(TInstant) except +
+
+ @staticmethod
+ TDuration Max()
+ @staticmethod
+ TDuration Zero()
+ @staticmethod
+ TDuration Seconds(uint64_t)
+ @staticmethod
+ TDuration Minutes(uint64_t)
+ @staticmethod
+ TDuration Hours(uint64_t)
+ @staticmethod
+ TDuration Days(uint64_t)
+
+ @staticmethod
+ TDuration Parse(const TStringBuf)
+ @staticmethod
+ bool_t TryParse(const TStringBuf, TDuration&)
+
+ TString ToString() except +