diff options
author | robot-ydb-importer <robot-ydb-importer@yandex-team.com> | 2024-02-14 19:47:36 +0300 |
---|---|---|
committer | Innokentii Mokin <innokentii@ydb.tech> | 2024-02-16 18:35:13 +0000 |
commit | d6ee6054676c603f8afb27b5bd8ce7fe0a5bfbc0 (patch) | |
tree | 4aa69116e7818a4aae0bfedbfa29639b0f0b90e8 | |
parent | 59ded8ecfcd805c109471346a0d4d1f269bdaa59 (diff) | |
download | ydb-d6ee6054676c603f8afb27b5bd8ce7fe0a5bfbc0.tar.gz |
YDB Import 566
96265cd0cc64e1b9bb31fe97b915ed2a09caf1cb
24 files changed, 2613 insertions, 0 deletions
diff --git a/contrib/python/ydb/ya.make b/contrib/python/ydb/ya.make new file mode 100644 index 0000000000..1886a29d3d --- /dev/null +++ b/contrib/python/ydb/ya.make @@ -0,0 +1,23 @@ +PY23_LIBRARY() + +LICENSE(Service-Py23-Proxy) + +IF (PYTHON2) + PEERDIR(contrib/python/ydb/py2) +ELSE() + PEERDIR(contrib/python/ydb/py3) +ENDIF() + +PEERDIR( + contrib/ydb/public/api/grpc + contrib/ydb/public/api/grpc/draft +) + +NO_LINT() + +END() + +RECURSE( + py2 + py3 +)
\ No newline at end of file diff --git a/library/cpp/monlib/encode/unistat/unistat.h b/library/cpp/monlib/encode/unistat/unistat.h new file mode 100644 index 0000000000..300fb6270f --- /dev/null +++ b/library/cpp/monlib/encode/unistat/unistat.h @@ -0,0 +1,13 @@ +#pragma once + +#include <util/generic/fwd.h> +#include <util/datetime/base.h> + +namespace NMonitoring { + /// Decodes unistat-style metrics + /// https://wiki.yandex-team.ru/golovan/stat-handle + void DecodeUnistat(TStringBuf data, class IMetricConsumer* c, TStringBuf metricNameLabel = "sensor", TInstant ts = TInstant::Zero()); + + /// Assumes consumer's stream is open by the caller + void DecodeUnistatToStream(TStringBuf data, class IMetricConsumer* c, TStringBuf metricNameLabel = "sensor", TInstant ts = TInstant::Zero()); +} diff --git a/library/cpp/monlib/encode/unistat/unistat_decoder.cpp b/library/cpp/monlib/encode/unistat/unistat_decoder.cpp new file mode 100644 index 0000000000..a2b787365c --- /dev/null +++ b/library/cpp/monlib/encode/unistat/unistat_decoder.cpp @@ -0,0 +1,312 @@ +#include "unistat.h" + +#include <library/cpp/monlib/metrics/histogram_collector.h> +#include <library/cpp/monlib/metrics/labels.h> +#include <library/cpp/monlib/metrics/metric_type.h> +#include <library/cpp/monlib/metrics/metric_value.h> +#include <library/cpp/monlib/metrics/metric_consumer.h> + +#include <library/cpp/json/json_reader.h> + +#include <util/datetime/base.h> +#include <util/string/split.h> + +#include <contrib/libs/re2/re2/re2.h> + +using namespace NJson; + +const re2::RE2 NAME_RE{R"((?:[a-zA-Z0-9\.\-/@_]+_)+(?:[ad][vehmntx]{3}|summ|hgram|max))"}; + +namespace NMonitoring { + namespace { + bool IsNumber(const NJson::TJsonValue& j) { + switch (j.GetType()) { + case EJsonValueType::JSON_INTEGER: + case EJsonValueType::JSON_UINTEGER: + case EJsonValueType::JSON_DOUBLE: + return true; + + default: + return false; + } + } + + template <typename T> + T ExtractNumber(const TJsonValue& val) { + switch (val.GetType()) { + case EJsonValueType::JSON_INTEGER: + return static_cast<T>(val.GetInteger()); + case EJsonValueType::JSON_UINTEGER: + return static_cast<T>(val.GetUInteger()); + case EJsonValueType::JSON_DOUBLE: + return static_cast<T>(val.GetDouble()); + + default: + ythrow yexception() << "Expected number, but found " << val.GetType(); + } + } + + auto ExtractDouble = ExtractNumber<double>; + auto ExtractUi64 = ExtractNumber<ui64>; + + class THistogramBuilder { + public: + void Add(TBucketBound bound, TBucketValue value) { + /// XXX: yasm uses left-closed intervals, while in monlib we use right-closed ones, + /// so (-inf; 0) [0, 100) [100; +inf) + /// becomes (-inf; 0] (0, 100] (100; +inf) + /// but since we've already lost some information these no way to avoid this kind of error here + Bounds_.push_back(bound); + + /// this will always be 0 for the first bucket, + /// since there's no way to make (-inf; N) bucket in yasm + Values_.push_back(NextValue_); + + /// we will write this value into the next bucket so that [[0, 10], [100, 20], [200, 50]] + /// becomes (-inf; 0] -> 0; (0; 100] -> 10; (100; 200] -> 20; (200; +inf) -> 50 + NextValue_ = value; + } + + IHistogramSnapshotPtr Finalize() { + Bounds_.push_back(std::numeric_limits<TBucketBound>::max()); + Values_.push_back(NextValue_); + + return ExplicitHistogramSnapshot(Bounds_, Values_, true); + } + + public: + TBucketValue NextValue_ {0}; + TBucketBounds Bounds_; + TBucketValues Values_; + }; + + class LogHistogramBuilder { + public: + void Add(double value) { + while (Values_.size() < HISTOGRAM_MAX_BUCKETS_COUNT && value > MaxValueWithBickets(Values_.size())) { + Values_.push_back(0); + } + + ui32 index = 0; + if (value > MaxValueWithBickets(HISTOGRAM_MAX_BUCKETS_COUNT)) { + index = Values_.size() - 1; + } else if (value > MIN_VALUE) { + double logBase = std::log(value) / std::log(BASE); + index = static_cast<ui32>(std::ceil(logBase)); + } + ++Values_[index]; + } + + IHistogramSnapshotPtr Finalize() && { + return new TExponentialHistogramSnapshot(BASE, MIN_VALUE, std::move(Values_)); + } + + private: + static constexpr double BASE = 2; + static constexpr double MIN_VALUE = 1; + + static double MaxValueWithBickets(ui64 buckets) { + return std::pow(BASE, buckets - 2); + } + + private: + TBucketValues Values_ = TBucketValues(2, 0); + }; + + class TDecoderUnistat { + private: + public: + explicit TDecoderUnistat(IMetricConsumer* consumer, IInputStream* is, TStringBuf metricNameLabel, TInstant ts) + : Consumer_{consumer}, + MetricNameLabel(metricNameLabel), + Timestamp_{ts} { + ReadJsonTree(is, &Json_, /* throw */ true); + } + + void Decode() { + Y_ENSURE(Json_.IsArray(), "Expected array at the top level, but found " << Json_.GetType()); + + for (auto&& metric : Json_.GetArray()) { + Y_ENSURE(metric.IsArray(), "Metric must be an array"); + auto&& arr = metric.GetArray(); + Y_ENSURE(arr.size() == 2, "Metric must be an array of 2 elements"); + auto&& name = arr[0]; + auto&& value = arr[1]; + MetricContext_ = {}; + + ParseName(name.GetString()); + + if (value.IsArray()) { + const auto& array = value.GetArray(); + if (!array.empty() && IsNumber(array[0])) { + OnLogHistogram(value); + } else { + OnHistogram(value); + } + } else if (IsNumber(value)) { + if (MetricContext_.Name.EndsWith("_ahhh")) { + OnLogHistogram(value); + } else { + OnScalar(value); + } + } else { + ythrow yexception() << "Expected list or number, but found " << value.GetType(); + } + + WriteValue(); + } + } + + private: + void OnScalar(const TJsonValue& jsonValue) { + if (MetricContext_.IsDeriv) { + MetricContext_.Type = EMetricType::RATE; + MetricContext_.Value = TMetricValue{ExtractUi64(jsonValue)}; + } else { + MetricContext_.Type = EMetricType::GAUGE; + MetricContext_.Value = TMetricValue{ExtractDouble(jsonValue)}; + } + } + + void OnLogHistogram(const TJsonValue& value) { + Y_ENSURE(MetricContext_.Name.EndsWith("_ahhh"), "Values list is supported only for _ahhh metrics"); + MetricContext_.Type = EMetricType::HIST; + + LogHistogramBuilder histogramBuilder; + if (IsNumber(value)) { + histogramBuilder.Add(value.GetDouble()); + } else { + for (auto&& item: value.GetArray()) { + Y_ENSURE(IsNumber(item), "Expected a number, but found " << item.GetType()); + histogramBuilder.Add(item.GetDouble()); + } + } + + MetricContext_.Histogram = std::move(histogramBuilder).Finalize(); + MetricContext_.Value = TMetricValue{MetricContext_.Histogram.Get()}; + } + + void OnHistogram(const TJsonValue& jsonHist) { + if (MetricContext_.IsDeriv) { + MetricContext_.Type = EMetricType::HIST_RATE; + } else { + MetricContext_.Type = EMetricType::HIST; + } + + auto histogramBuilder = THistogramBuilder(); + + for (auto&& bucket : jsonHist.GetArray()) { + Y_ENSURE(bucket.IsArray(), "Expected an array, but found " << bucket.GetType()); + auto&& arr = bucket.GetArray(); + Y_ENSURE(arr.size() == 2, "Histogram bucket must be an array of 2 elements"); + const auto bound = ExtractDouble(arr[0]); + const auto weight = ExtractUi64(arr[1]); + histogramBuilder.Add(bound, weight); + } + + MetricContext_.Histogram = histogramBuilder.Finalize(); + MetricContext_.Value = TMetricValue{MetricContext_.Histogram.Get()}; + } + + bool IsDeriv(TStringBuf name) { + TStringBuf ignore, suffix; + name.RSplit('_', ignore, suffix); + + Y_ENSURE(suffix.size() >= 3 && suffix.size() <= 5, "Disallowed suffix value: " << suffix); + + if (suffix == TStringBuf("summ") || suffix == TStringBuf("hgram")) { + return true; + } else if (suffix == TStringBuf("max")) { + return false; + } + + return suffix[0] == 'd'; + } + + void ParseName(TStringBuf value) { + TVector<TStringBuf> parts; + StringSplitter(value).Split(';').SkipEmpty().Collect(&parts); + + Y_ENSURE(parts.size() >= 1 && parts.size() <= 16); + + TStringBuf name = parts.back(); + parts.pop_back(); + + Y_ENSURE(RE2::FullMatch(re2::StringPiece{name.data(), name.size()}, NAME_RE), + "Metric name " << name << " doesn't match regex " << NAME_RE.pattern()); + + MetricContext_.Name = name; + MetricContext_.IsDeriv = IsDeriv(MetricContext_.Name); + + for (auto tag : parts) { + TStringBuf n, v; + tag.Split('=', n, v); + Y_ENSURE(n && v, "Unexpected tag format in " << tag); + MetricContext_.Labels.Add(n, v); + } + } + + private: + void WriteValue() { + Consumer_->OnMetricBegin(MetricContext_.Type); + + Consumer_->OnLabelsBegin(); + Consumer_->OnLabel(MetricNameLabel, TString{MetricContext_.Name}); + for (auto&& l : MetricContext_.Labels) { + Consumer_->OnLabel(l.Name(), l.Value()); + } + + Consumer_->OnLabelsEnd(); + + switch (MetricContext_.Type) { + case EMetricType::GAUGE: + Consumer_->OnDouble(Timestamp_, MetricContext_.Value.AsDouble()); + break; + case EMetricType::RATE: + Consumer_->OnUint64(Timestamp_, MetricContext_.Value.AsUint64()); + break; + case EMetricType::HIST: + case EMetricType::HIST_RATE: + Consumer_->OnHistogram(Timestamp_, MetricContext_.Value.AsHistogram()); + break; + case EMetricType::LOGHIST: + case EMetricType::DSUMMARY: + case EMetricType::IGAUGE: + case EMetricType::COUNTER: + case EMetricType::UNKNOWN: + ythrow yexception() << "Unexpected metric type: " << MetricContext_.Type; + } + + Consumer_->OnMetricEnd(); + } + + private: + IMetricConsumer* Consumer_; + NJson::TJsonValue Json_; + TStringBuf MetricNameLabel; + TInstant Timestamp_; + + struct { + TStringBuf Name; + EMetricType Type{EMetricType::UNKNOWN}; + TMetricValue Value; + bool IsDeriv{false}; + TLabels Labels; + IHistogramSnapshotPtr Histogram; + } MetricContext_; + }; + + } + + void DecodeUnistat(TStringBuf data, IMetricConsumer* c, TStringBuf metricNameLabel, TInstant ts) { + c->OnStreamBegin(); + DecodeUnistatToStream(data, c, metricNameLabel, ts); + c->OnStreamEnd(); + } + + void DecodeUnistatToStream(TStringBuf data, IMetricConsumer* c, TStringBuf metricNameLabel, TInstant ts) { + TMemoryInput in{data.data(), data.size()}; + TDecoderUnistat decoder(c, &in, metricNameLabel, ts); + decoder.Decode(); + } +} diff --git a/library/cpp/monlib/encode/unistat/unistat_ut.cpp b/library/cpp/monlib/encode/unistat/unistat_ut.cpp new file mode 100644 index 0000000000..f15fc7c5b7 --- /dev/null +++ b/library/cpp/monlib/encode/unistat/unistat_ut.cpp @@ -0,0 +1,341 @@ +#include "unistat.h" + +#include <library/cpp/monlib/encode/protobuf/protobuf.h> +#include <library/cpp/monlib/metrics/labels.h> + +#include <library/cpp/testing/unittest/registar.h> + +using namespace NMonitoring; + +Y_UNIT_TEST_SUITE(TUnistatDecoderTest) { + Y_UNIT_TEST(MetricNameLabel) { + constexpr auto input = TStringBuf(R"([["something_axxx", 42]])"); + + NProto::TMultiSamplesList samples; + auto encoder = EncoderProtobuf(&samples); + + DecodeUnistat(input, encoder.Get(), "metric_name_label"); + + UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 1); + auto sample = samples.GetSamples(0); + + auto label = sample.GetLabels(0); + UNIT_ASSERT_VALUES_EQUAL(label.GetName(), "metric_name_label"); + } + + Y_UNIT_TEST(ScalarMetric) { + constexpr auto input = TStringBuf(R"([["something_axxx", 42]])"); + + NProto::TMultiSamplesList samples; + auto encoder = EncoderProtobuf(&samples); + + DecodeUnistat(input, encoder.Get()); + + UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 1); + auto sample = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1); + UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1); + + auto label = sample.GetLabels(0); + auto point = sample.GetPoints(0); + UNIT_ASSERT_VALUES_EQUAL(point.GetFloat64(), 42.); + UNIT_ASSERT_VALUES_EQUAL(label.GetName(), "sensor"); + UNIT_ASSERT_VALUES_EQUAL(label.GetValue(), "something_axxx"); + } + + Y_UNIT_TEST(OverriddenTags) { + constexpr auto input = TStringBuf(R"([["ctype=foo;prj=bar;custom_tag=qwe;something_axxx", 42]])"); + + NProto::TMultiSamplesList samples; + auto encoder = EncoderProtobuf(&samples); + + DecodeUnistat(input, encoder.Get()); + + UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 1); + auto sample = samples.GetSamples(0); + UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1); + UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 4); + + const auto& labels = sample.GetLabels(); + TLabels actual; + for (auto&& l : labels) { + actual.Add(l.GetName(), l.GetValue()); + } + + TLabels expected{{"ctype", "foo"}, {"prj", "bar"}, {"custom_tag", "qwe"}, {"sensor", "something_axxx"}}; + + UNIT_ASSERT_VALUES_EQUAL(actual.size(), expected.size()); + for (auto&& l : actual) { + UNIT_ASSERT(expected.Extract(l.Name())->Value() == l.Value()); + } + } + + Y_UNIT_TEST(ThrowsOnTopLevelObject) { + constexpr auto input = TStringBuf(R"({["something_axxx", 42]})"); + + NProto::TMultiSamplesList samples; + auto encoder = EncoderProtobuf(&samples); + + UNIT_ASSERT_EXCEPTION(DecodeUnistat(input, encoder.Get()), yexception); + } + + Y_UNIT_TEST(ThrowsOnUnwrappedMetric) { + constexpr auto input = TStringBuf(R"(["something_axxx", 42])"); + + NProto::TMultiSamplesList samples; + auto encoder = EncoderProtobuf(&samples); + + UNIT_ASSERT_EXCEPTION(DecodeUnistat(input, encoder.Get()), yexception); + } + + Y_UNIT_TEST(HistogramMetric) { + constexpr auto input = TStringBuf(R"([["something_hgram", [[0, 1], [200, 2], [500, 3]] ]])"); + + NProto::TMultiSamplesList samples; + auto encoder = EncoderProtobuf(&samples); + + DecodeUnistat(input, encoder.Get()); + + auto sample = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::HIST_RATE); + UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1); + UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1); + + auto label = sample.GetLabels(0); + const auto point = sample.GetPoints(0); + const auto histogram = point.GetHistogram(); + const auto size = histogram.BoundsSize(); + UNIT_ASSERT_VALUES_EQUAL(size, 4); + + const TVector<double> expectedBounds {0, 200, 500, std::numeric_limits<double>::max()}; + const TVector<ui64> expectedValues {0, 1, 2, 3}; + + for (auto i = 0; i < 4; ++i) { + UNIT_ASSERT_VALUES_EQUAL(histogram.GetBounds(i), expectedBounds[i]); + UNIT_ASSERT_VALUES_EQUAL(histogram.GetValues(i), expectedValues[i]); + } + + UNIT_ASSERT_VALUES_EQUAL(label.GetName(), "sensor"); + UNIT_ASSERT_VALUES_EQUAL(label.GetValue(), "something_hgram"); + } + + Y_UNIT_TEST(AbsoluteHistogram) { + constexpr auto input = TStringBuf(R"([["something_ahhh", [[0, 1], [200, 2], [500, 3]] ]])"); + + NProto::TMultiSamplesList samples; + auto encoder = EncoderProtobuf(&samples); + + DecodeUnistat(input, encoder.Get()); + + auto sample = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::HISTOGRAM); + UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1); + UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1); + } + + Y_UNIT_TEST(LogHistogram) { + constexpr auto input = TStringBuf(R"([["something_ahhh", [1, 2, 3] ]])"); + + NProto::TMultiSamplesList samples; + auto encoder = EncoderProtobuf(&samples); + + DecodeUnistat(input, encoder.Get()); + + auto sample = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::HISTOGRAM); + UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1); + UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1); + + auto histogram = sample.GetPoints(0).GetHistogram(); + UNIT_ASSERT_VALUES_EQUAL(histogram.BoundsSize(), 4); + TVector<double> expectedBounds = {1, 2, 4, std::numeric_limits<double>::max()}; + TVector<ui64> expectedValues = {1, 1, 1, 0}; + + for (auto i = 0; i < 4; ++i) { + UNIT_ASSERT_VALUES_EQUAL(histogram.GetBounds(i), expectedBounds[i]); + UNIT_ASSERT_VALUES_EQUAL(histogram.GetValues(i), expectedValues[i]); + } + } + + Y_UNIT_TEST(LogHistogramOverflow) { + constexpr auto input = TStringBuf(R"([["something_ahhh", [1125899906842624, 2251799813685248] ]])"); + + NProto::TMultiSamplesList samples; + auto encoder = EncoderProtobuf(&samples); + + DecodeUnistat(input, encoder.Get()); + + auto sample = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::HISTOGRAM); + UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1); + UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1); + + auto histogram = sample.GetPoints(0).GetHistogram(); + UNIT_ASSERT_VALUES_EQUAL(histogram.BoundsSize(), HISTOGRAM_MAX_BUCKETS_COUNT); + + TVector<double> expectedBounds; + for (ui64 i = 0; i < 50; ++i) { + expectedBounds.push_back(std::pow(2, i)); + } + expectedBounds.push_back(std::numeric_limits<double>::max()); + + TVector<ui64> expectedValues; + for (ui64 i = 0; i < 50; ++i) { + expectedValues.push_back(0); + } + expectedValues.push_back(2); + + for (auto i = 0; i < 4; ++i) { + UNIT_ASSERT_VALUES_EQUAL(histogram.GetBounds(i), expectedBounds[i]); + UNIT_ASSERT_VALUES_EQUAL(histogram.GetValues(i), expectedValues[i]); + } + } + + Y_UNIT_TEST(LogHistogramSingle) { + constexpr auto input = TStringBuf(R"([["something_ahhh", 4 ]])"); + + NProto::TMultiSamplesList samples; + auto encoder = EncoderProtobuf(&samples); + + DecodeUnistat(input, encoder.Get()); + + auto sample = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::HISTOGRAM); + UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1); + UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1); + + auto histogram = sample.GetPoints(0).GetHistogram(); + UNIT_ASSERT_VALUES_EQUAL(histogram.BoundsSize(), 4); + TVector<double> expectedBounds = {1, 2, 4, std::numeric_limits<double>::max()}; + TVector<ui64> expectedValues = {0, 0, 1, 0}; + + for (auto i = 0; i < 4; ++i) { + UNIT_ASSERT_VALUES_EQUAL(histogram.GetBounds(i), expectedBounds[i]); + UNIT_ASSERT_VALUES_EQUAL(histogram.GetValues(i), expectedValues[i]); + } + } + + Y_UNIT_TEST(LogHistogramInvalid) { + NProto::TMultiSamplesList samples; + auto encoder = EncoderProtobuf(&samples); + + { + constexpr auto input = TStringBuf(R"([["something_ahhh", [1, 2, [3, 4]] ]])"); + UNIT_ASSERT_EXCEPTION(DecodeUnistat(input, encoder.Get()), yexception); + } + + { + constexpr auto input = TStringBuf(R"([["something_ahhh", [[3, 4], 1, 2] ]])"); + UNIT_ASSERT_EXCEPTION(DecodeUnistat(input, encoder.Get()), yexception); + } + + { + constexpr auto input = TStringBuf(R"([["something_hgram", 1, 2, 3, 4 ]])"); + UNIT_ASSERT_EXCEPTION(DecodeUnistat(input, encoder.Get()), yexception); + } + } + + + Y_UNIT_TEST(AllowedMetricNames) { + NProto::TMultiSamplesList samples; + auto encoder = EncoderProtobuf(&samples); + + { + constexpr auto input = TStringBuf(R"([["a/A-b/c_D/__G_dmmm", [[0, 1], [200, 2], [500, 3]] ]])"); + UNIT_ASSERT_NO_EXCEPTION(DecodeUnistat(input, encoder.Get())); + } + } + + Y_UNIT_TEST(DisallowedMetricNames) { + NProto::TMultiSamplesList samples; + auto encoder = EncoderProtobuf(&samples); + + { + constexpr auto input = TStringBuf(R"([["someth!ng_ahhh", [[0, 1], [200, 2], [500, 3]] ]])"); + UNIT_ASSERT_EXCEPTION(DecodeUnistat(input, encoder.Get()), yexception); + } + + { + constexpr auto input = TStringBuf(R"([["foo_a", [[0, 1], [200, 2], [500, 3]] ]])"); + UNIT_ASSERT_EXCEPTION(DecodeUnistat(input, encoder.Get()), yexception); + } + + { + constexpr auto input = TStringBuf(R"([["foo_ahhh;tag=value", [[0, 1], [200, 2], [500, 3]] ]])"); + UNIT_ASSERT_EXCEPTION(DecodeUnistat(input, encoder.Get()), yexception); + } + } + + Y_UNIT_TEST(MultipleMetrics) { + constexpr auto input = TStringBuf(R"([["something_axxx", 42], ["some-other_dhhh", 53]])"); + + NProto::TMultiSamplesList samples; + auto encoder = EncoderProtobuf(&samples); + + DecodeUnistat(input, encoder.Get()); + + UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 2); + auto sample = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1); + UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1); + + auto label = sample.GetLabels(0); + auto point = sample.GetPoints(0); + UNIT_ASSERT_VALUES_EQUAL(point.GetFloat64(), 42.); + UNIT_ASSERT_VALUES_EQUAL(label.GetName(), "sensor"); + UNIT_ASSERT_VALUES_EQUAL(label.GetValue(), "something_axxx"); + + sample = samples.GetSamples(1); + UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::RATE); + UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1); + UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1); + + label = sample.GetLabels(0); + point = sample.GetPoints(0); + UNIT_ASSERT_VALUES_EQUAL(point.GetUint64(), 53); + UNIT_ASSERT_VALUES_EQUAL(label.GetName(), "sensor"); + UNIT_ASSERT_VALUES_EQUAL(label.GetValue(), "some-other_dhhh"); + } + + Y_UNIT_TEST(UnderscoreName) { + constexpr auto input = TStringBuf(R"([["something_anything_dmmm", 42]])"); + + NProto::TMultiSamplesList samples; + auto encoder = EncoderProtobuf(&samples); + DecodeUnistat(input, encoder.Get()); + + UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 1); + auto sample = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::RATE); + UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1); + UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1); + + auto label = sample.GetLabels(0); + auto point = sample.GetPoints(0); + UNIT_ASSERT_VALUES_EQUAL(point.GetUint64(), 42); + UNIT_ASSERT_VALUES_EQUAL(label.GetName(), "sensor"); + UNIT_ASSERT_VALUES_EQUAL(label.GetValue(), "something_anything_dmmm"); + } + + Y_UNIT_TEST(MaxAggr) { + constexpr auto input = TStringBuf(R"([["something_anything_max", 42]])"); + + NProto::TMultiSamplesList samples; + auto encoder = EncoderProtobuf(&samples); + DecodeUnistat(input, encoder.Get()); + + UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 1); + auto sample = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1); + UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1); + + auto label = sample.GetLabels(0); + auto point = sample.GetPoints(0); + UNIT_ASSERT_VALUES_EQUAL(point.GetFloat64(), 42.); + UNIT_ASSERT_VALUES_EQUAL(label.GetName(), "sensor"); + UNIT_ASSERT_VALUES_EQUAL(label.GetValue(), "something_anything_max"); + } +} diff --git a/library/cpp/monlib/encode/unistat/ut/ya.make b/library/cpp/monlib/encode/unistat/ut/ya.make new file mode 100644 index 0000000000..1f4590b3fa --- /dev/null +++ b/library/cpp/monlib/encode/unistat/ut/ya.make @@ -0,0 +1,11 @@ +UNITTEST_FOR(library/cpp/monlib/encode/unistat) + +SRCS( + unistat_ut.cpp +) + +PEERDIR( + library/cpp/monlib/encode/protobuf +) + +END() diff --git a/library/cpp/monlib/encode/unistat/ya.make b/library/cpp/monlib/encode/unistat/ya.make new file mode 100644 index 0000000000..e054cd707e --- /dev/null +++ b/library/cpp/monlib/encode/unistat/ya.make @@ -0,0 +1,17 @@ +LIBRARY() + +PEERDIR( + contrib/libs/re2 + library/cpp/json + library/cpp/monlib/metrics +) + +SRCS( + unistat_decoder.cpp +) + +END() + +RECURSE_FOR_TESTS( + ut +) diff --git a/library/python/monlib/encoder.pxd b/library/python/monlib/encoder.pxd new file mode 100644 index 0000000000..f879b66b78 --- /dev/null +++ b/library/python/monlib/encoder.pxd @@ -0,0 +1,79 @@ +from util.generic.string cimport TStringBuf, TString +from util.generic.ptr cimport THolder +from util.stream.output cimport IOutputStream + +from library.python.monlib.metric_consumer cimport IMetricConsumer + + +cdef extern from "util/stream/input.h" nogil: + cdef cppclass IInputStream: + pass + + +cdef extern from "util/system/file.h" nogil: + cdef cppclass TFile: + TFile() + TFile(TFile) + pass + + cdef TFile Duplicate(int) + + +cdef extern from "library/cpp/monlib/encode/encoder.h" namespace "NMonitoring" nogil: + cdef cppclass IMetricEncoder: + void Close() + + cdef cppclass ECompression: + pass + + ctypedef THolder[IMetricEncoder] IMetricEncoderPtr + + +cdef extern from "library/cpp/monlib/encode/unistat/unistat.h" namespace "NMonitoring" nogil: + cdef void DecodeUnistat(TStringBuf data, IMetricConsumer* c) + + +cdef extern from "library/cpp/monlib/encode/json/json.h" namespace "NMonitoring" nogil: + cdef IMetricEncoderPtr EncoderJson(IOutputStream* out, int indentation) + cdef IMetricEncoderPtr BufferedEncoderJson(IOutputStream* out, int indentation) + + cdef void DecodeJson(TStringBuf data, IMetricConsumer* c) + + +cdef extern from "library/cpp/monlib/encode/spack/spack_v1.h" namespace "NMonitoring" nogil: + cdef IMetricEncoderPtr EncoderSpackV1(IOutputStream* out, ETimePrecision, ECompression) + + cdef void DecodeSpackV1(IInputStream* input, IMetricConsumer* c) except + + cdef cppclass ETimePrecision: + pass + + cdef cppclass EValueType: + pass + + +cdef extern from "library/cpp/monlib/encode/spack/spack_v1.h" namespace "NMonitoring::ETimePrecision" nogil: + cdef ETimePrecision SECONDS "NMonitoring::ETimePrecision::SECONDS" + cdef ETimePrecision MILLIS "NMonitoring::ETimePrecision::MILLIS" + + +cdef extern from "library/cpp/monlib/encode/encoder.h" namespace "NMonitoring::ECompression" nogil: + cdef ECompression UNKNOWN "NMonitoring::ECompression::UNKNOWN" + cdef ECompression IDENTITY "NMonitoring::ECompression::IDENTITY" + cdef ECompression ZLIB "NMonitoring::ECompression::ZLIB" + cdef ECompression LZ4 "NMonitoring::ECompression::LZ4" + cdef ECompression ZSTD "NMonitoring::ECompression::ZSTD" + + +cdef class Encoder: + cdef IMetricEncoderPtr __wrapped + cdef THolder[TFile] __file + cdef THolder[IOutputStream] __stream + + cdef IMetricEncoder* native(self) + + cdef _make_stream(self, py_stream) + + @staticmethod + cdef Encoder create_spack(object stream, ETimePrecision timePrecision, ECompression compression) + @staticmethod + cdef Encoder create_json(object stream, int indent) diff --git a/library/python/monlib/encoder.pyx b/library/python/monlib/encoder.pyx new file mode 100644 index 0000000000..05cf4fec9a --- /dev/null +++ b/library/python/monlib/encoder.pyx @@ -0,0 +1,260 @@ +from util.generic.string cimport TString, TStringBuf +from util.generic.ptr cimport THolder + +from cython.operator cimport dereference as deref + +import sys + +from datetime import datetime +from os import dup + + +cdef extern from "util/stream/fwd.h" nogil: + cdef cppclass TAdaptivelyBuffered[T]: + TAdaptivelyBuffered(TFile) except + + + ctypedef TAdaptivelyBuffered[TUnbufferedFileOutput] TFileOutput + +cdef extern from "util/stream/mem.h" nogil: + cdef cppclass TMemoryInput: + TMemoryInput(const TStringBuf buf) + + +cdef extern from "util/stream/file.h" nogil: + cdef cppclass TUnbufferedFileOutput: + TUnbufferedFileOutput(TFile) + + cdef cppclass TFileInput: + TFileInput(TFile) except + + + +cdef extern from "util/stream/str.h" nogil: + cdef cppclass TStringStream: + const TString& Str() const + + +cdef class Encoder: + cdef IMetricEncoder* native(self): + return self.__wrapped.Get() + + def close(self): + deref(self.__wrapped.Get()).Close() + + def dumps(self): + return (<TStringStream&?>deref(self.__stream.Get())).Str() + + cdef _make_stream(self, py_stream): + if py_stream is not None: + fd = Duplicate(py_stream.fileno()) + + self.__file.Reset(new TFile(fd)) + f = self.__file.Get() + self.__stream.Reset(<IOutputStream*>(new TFileOutput(deref(f)))) + else: + self.__stream.Reset(<IOutputStream*>(new TStringStream())) + + @staticmethod + cdef Encoder create_spack(object stream, ETimePrecision precision, ECompression compression): + cdef Encoder wrapper = Encoder.__new__(Encoder) + wrapper._make_stream(stream) + + wrapper.__wrapped = EncoderSpackV1(wrapper.__stream.Get(), + precision, + compression) + + return wrapper + + @staticmethod + cdef Encoder create_json(object stream, int indent): + cdef Encoder wrapper = Encoder.__new__(Encoder) + wrapper._make_stream(stream) + + wrapper.__wrapped = EncoderJson(wrapper.__stream.Get(), indent) + + return wrapper + + +cpdef Encoder create_json_encoder(object stream, int indent): + return Encoder.create_json(stream, indent) + + +cdef class TimePrecision: + Millis = <int>MILLIS + Seconds = <int>SECONDS + + @staticmethod + cdef ETimePrecision to_native(int p) except *: + if p == TimePrecision.Millis: + return MILLIS + elif p == TimePrecision.Seconds: + return SECONDS + + raise ValueError('Unsupported TimePrecision value') + +cdef class Compression: + Identity = <int>IDENTITY + Lz4 = <int>LZ4 + Zlib = <int>ZLIB + Zstd = <int>ZSTD + + @staticmethod + cdef ECompression to_native(int p) except *: + if p == Compression.Identity: + return IDENTITY + elif p == Compression.Lz4: + return LZ4 + elif p == Compression.Zlib: + return ZLIB + elif p == Compression.Zstd: + return ZSTD + + raise ValueError('Unsupported Compression value') + + +# XXX: timestamps +def dump(registry, fp, format='spack', **kwargs): + """ + Dumps metrics held by the metric registry to a file. Output can be additionally + adjusted using kwargs, which may differ depending on the selected format. + + :param registry: Metric registry object + :param fp: File descriptor to serialize to + :param format: Format to serialize to (allowed values: spack). Default: json + + Keyword arguments: + :param time_precision: Time precision (spack) + :param compression: Compression codec (spack) + :param indent: Pretty-print indentation for object members and arrays (json) + :param timestamp: Metric timestamp datetime + :returns: Nothing + """ + if not hasattr(fp, 'fileno'): + raise TypeError('Expected a file-like object, but got ' + str(type(fp))) + + if format == 'spack': + time_precision = TimePrecision.to_native(kwargs.get('time_precision', TimePrecision.Seconds)) + compression = Compression.to_native(kwargs.get('compression', Compression.Identity)) + encoder = Encoder.create_spack(fp, time_precision, compression) + elif format == 'json': + indent = int(kwargs.get('indent', 0)) + encoder = Encoder.create_json(fp, indent) + timestamp = kwargs.get('timestamp', datetime.utcfromtimestamp(0)) + + registry.accept(timestamp, encoder) + encoder.close() + + +def dumps(registry, format='spack', **kwargs): + """ + Dumps metrics held by the metric registry to a string. Output can be additionally + adjusted using kwargs, which may differ depending on the selected format. + + :param registry: Metric registry object + :param format: Format to serialize to (allowed values: spack). Default: json + + Keyword arguments: + :param time_precision: Time precision (spack) + :param compression: Compression codec (spack) + :param indent: Pretty-print indentation for object members and arrays (json) + :param timestamp: Metric timestamp datetime + :returns: A string of the specified format + """ + if format == 'spack': + time_precision = TimePrecision.to_native(kwargs.get('time_precision', TimePrecision.Seconds)) + compression = Compression.to_native(kwargs.get('compression', Compression.Identity)) + encoder = Encoder.create_spack(None, time_precision, compression) + elif format == 'json': + indent = int(kwargs.get('indent', 0)) + encoder = Encoder.create_json(None, indent) + timestamp = kwargs.get('timestamp', datetime.utcfromtimestamp(0)) + + registry.accept(timestamp, encoder) + encoder.close() + + s = encoder.dumps() + + return s + + +def load(fp, from_format='spack', to_format='json'): + """ + Converts metrics from one format to another. + + :param fp: File to load data from + :param from_format: Source string format (allowed values: json, spack, unistat). Default: spack + :param to_format: Target format (allowed values: json, spack). Default: json + :returns: a string containing metrics in the specified format + """ + if from_format == to_format: + return fp.read() + + cdef THolder[TFile] file + file.Reset(new TFile(Duplicate(fp.fileno()))) + + cdef THolder[TFileInput] input + input.Reset(new TFileInput(deref(file.Get()))) + + if to_format == 'json': + encoder = Encoder.create_json(None, 0) + elif to_format == 'spack': + encoder = Encoder.create_spack(None, SECONDS, IDENTITY) + else: + raise ValueError('Unsupported format ' + to_format) + + if from_format == 'spack': + DecodeSpackV1(<IInputStream*>(input.Get()), <IMetricConsumer*?>encoder.native()) + elif from_format == 'json': + s = open(fp, 'r').read() + DecodeJson(TStringBuf(s), <IMetricConsumer*?>encoder.native()) + elif from_format == 'unistat': + s = open(fp, 'r').read() + DecodeJson(TStringBuf(s), <IMetricConsumer*?>encoder.native()) + + else: + raise ValueError('Unsupported format ' + from_format) + + encoder.close() + s = encoder.dumps() + + return s + + +def loads(s, from_format='spack', to_format='json', compression=Compression.Identity): + """ + Converts metrics from one format to another. + + :param s: String to load from + :param from_format: Source string format (allowed values: json, spack, unistat). Default: spack + :param to_format: Target format (allowed values: json, spack). Default: json + :returns: a string containing metrics in the specified format + """ + if from_format == to_format: + return s + + if sys.version_info[0] >= 3 and not isinstance(s, bytes): + s = s.encode('iso-8859-15') + + cdef THolder[TMemoryInput] input + + if to_format == 'json': + encoder = Encoder.create_json(None, 0) + elif to_format == 'spack': + comp = Compression.to_native(compression) + encoder = Encoder.create_spack(None, SECONDS, comp) + else: + raise ValueError('Unsupported format ' + to_format) + + if from_format == 'spack': + input.Reset(new TMemoryInput(s)) + DecodeSpackV1(<IInputStream*>(input.Get()), <IMetricConsumer*?>encoder.native()) + elif from_format == 'json': + DecodeJson(TStringBuf(s), <IMetricConsumer*?>encoder.native()) + elif from_format == 'unistat': + DecodeUnistat(TStringBuf(s), <IMetricConsumer*?>encoder.native()) + else: + raise ValueError('Unsupported format ' + from_format) + + encoder.close() + s = encoder.dumps() + + return s diff --git a/library/python/monlib/labels.pxd b/library/python/monlib/labels.pxd new file mode 100644 index 0000000000..cc782433c4 --- /dev/null +++ b/library/python/monlib/labels.pxd @@ -0,0 +1,47 @@ +from libcpp cimport bool + +from util.generic.maybe cimport TMaybe +from util.generic.string cimport TStringBuf, TString + + +cdef extern from "library/cpp/monlib/metrics/labels.h" namespace "NMonitoring" nogil: + cdef cppclass ILabel: + const TStringBuf Name() const + const TStringBuf Value() const + + cdef cppclass ILabels: + bool Add(TStringBuf name, TStringBuf value) + bool Add(const TString& name, const TString& value) + + size_t Size() const + + cdef cppclass TLabel: + TLabel() except + + TLabel(TStringBuf name, TStringBuf value) except + + const TString& Name() const + const TString& Value() const + + TString ToString() const + bool operator!=(const TLabel&) const + bool operator==(const TLabel&) const + + cdef cppclass TLabels: + cppclass const_iterator: + const TLabel& operator*() const + bool operator!=(const_iterator) const + bool operator==(const_iterator) const + + TLabels() except + + + bool Add(const TLabel&) except + + bool Add(TStringBuf name, TStringBuf value) except + + bool Add(const TString& name, const TString& value) except + + bool operator==(const TLabels&) const + + TMaybe[TLabel] Find(TStringBuf name) const + TMaybe[TLabel] Extract(TStringBuf name) except + + + size_t Size() const + + const_iterator begin() const + const_iterator end() const diff --git a/library/python/monlib/metric.pxd b/library/python/monlib/metric.pxd new file mode 100644 index 0000000000..afa28ea015 --- /dev/null +++ b/library/python/monlib/metric.pxd @@ -0,0 +1,103 @@ +from libcpp cimport bool + +from util.system.types cimport ui64, ui32, i64 +from util.generic.ptr cimport THolder, TIntrusivePtr +from util.generic.vector cimport TVector + + +cdef extern from "library/cpp/monlib/metrics/histogram_collector.h" namespace "NMonitoring" nogil: + ctypedef double TBucketBound + ctypedef ui64 TBucketValue + + cdef cppclass IHistogramSnapshot: + ui32 Count() const + TBucketBound UpperBound(ui32 index) const + TBucketValue Value(ui32 index) const + + ctypedef TIntrusivePtr[IHistogramSnapshot] IHistogramSnapshotPtr + + cdef cppclass IHistogramCollector: + void Collect(i64 value) + void Collect(i64 value, ui32 count) + IHistogramSnapshotPtr Snapshot() const + + ctypedef THolder[IHistogramCollector] IHistogramCollectorPtr + + IHistogramCollectorPtr ExponentialHistogram(ui32 bucketsCount, double base, double scale) except + + IHistogramCollectorPtr ExplicitHistogram(const TVector[double]& buckets) except + + IHistogramCollectorPtr LinearHistogram(ui32 bucketsCount, i64 startValue, i64 bucketWidth) except + + + +cdef extern from "library/cpp/monlib/metrics/metric.h" namespace "NMonitoring" nogil: + cdef cppclass TGauge: + TGauge(double value) except + + + void Set(double) + double Get() const + double Add(double) + + cdef cppclass TIntGauge: + TIntGauge(ui64 value) except + + + void Set(ui64) + ui64 Get() const + ui64 Add(double) + ui64 Inc() + ui64 Dec() + + cdef cppclass TCounter: + TCounter(ui64 value) except + + + void Set(ui64) + ui64 Get() const + void Inc() + void Reset() + + cdef cppclass TRate: + TRate(ui64 value) except + + + void Add(ui64) + ui64 Get() const + void Inc() + + cdef cppclass THistogram: + THistogram(IHistogramCollectorPtr collector, bool isRate) except + + + void Record(double value) + void Record(double value, ui32 count) + + +cdef class Gauge: + cdef TGauge* __wrapped + + @staticmethod + cdef Gauge from_ptr(TGauge* native) + + +cdef class Counter: + cdef TCounter* __wrapped + + @staticmethod + cdef Counter from_ptr(TCounter* native) + + +cdef class Rate: + cdef TRate* __wrapped + + @staticmethod + cdef Rate from_ptr(TRate* native) + + +cdef class IntGauge: + cdef TIntGauge* __wrapped + + @staticmethod + cdef IntGauge from_ptr(TIntGauge* native) + + +cdef class Histogram: + cdef THistogram* __wrapped + cdef bool __is_owner + + @staticmethod + cdef Histogram from_ptr(THistogram* native) diff --git a/library/python/monlib/metric.pyx b/library/python/monlib/metric.pyx new file mode 100644 index 0000000000..7b51752335 --- /dev/null +++ b/library/python/monlib/metric.pyx @@ -0,0 +1,162 @@ +from libcpp cimport bool + +from util.system.types cimport ui32, ui64, i64 +from library.python.monlib.metric cimport ( + TGauge, TCounter, TRate, TIntGauge, THistogram, + IHistogramCollectorPtr) + +cdef class Gauge: + """ + Represents a floating point absolute value + """ + @staticmethod + cdef Gauge from_ptr(TGauge* native): + cdef Gauge wrapper = Gauge.__new__(Gauge) + wrapper.__wrapped = native + + return wrapper + + def set(self, double value): + """ + Set metric to the specified value + :param value: metric value + """ + self.__wrapped.Set(value) + + def get(self): + """ + Get metric value. + :param value: metric value + """ + return self.__wrapped.Get() + + def add(self, double value): + """ + Add value to metric. + :param value: metric value + """ + return self.__wrapped.Add(value) + + +cdef class IntGauge: + """ + Represents an integer absolute value + """ + @staticmethod + cdef IntGauge from_ptr(TIntGauge* native): + cdef IntGauge wrapper = IntGauge.__new__(IntGauge) + wrapper.__wrapped = native + + return wrapper + + def set(self, i64 value): + """ + Set metric to the specified value + :param value: metric value + """ + self.__wrapped.Set(value) + + def get(self): + """ + Get metric value + :param value: metric value + """ + return self.__wrapped.Get() + + def add(self, i64 value): + """ + Add value to metric. + :param value: metric value + """ + return self.__wrapped.Add(value) + + def inc(self): + """ + Add 1 to metric. + """ + return self.__wrapped.Inc() + + def dec(self): + """ + Add -1 to metric. + """ + return self.__wrapped.Dec() + + +cdef class Counter: + """ + Represents a counter value + """ + @staticmethod + cdef Counter from_ptr(TCounter* native): + cdef Counter wrapper = Counter.__new__(Counter) + wrapper.__wrapped = native + + return wrapper + + def get(self): + return self.__wrapped.Get() + + def inc(self): + """ + Increment metric value + """ + return self.__wrapped.Inc() + + def reset(self): + """ + Reset metric value to zero + """ + return self.__wrapped.Reset() + + +cdef class Rate: + """ + Represents a time derivative + """ + @staticmethod + cdef Rate from_ptr(TRate* native): + cdef Rate wrapper = Rate.__new__(Rate) + wrapper.__wrapped = native + + return wrapper + + def get(self): + return self.__wrapped.Get() + + def inc(self): + """ + Increment metric value + """ + return self.__wrapped.Inc() + + def add(self, ui64 value): + """ + Add the value to metric + :param value: value to add to metric + """ + return self.__wrapped.Add(value) + +cdef class Histogram: + """ + Represents some value distribution + """ + @staticmethod + cdef Histogram from_ptr(THistogram* native): + cdef Histogram wrapper = Histogram.__new__(Histogram, 0) + wrapper.__is_owner = False + wrapper.__wrapped = native + + return wrapper + + def __dealloc__(self): + if self.__is_owner: + del self.__wrapped + + def collect(self, double value, ui32 count=1): + """ + Add a few points with same value to the distribution + :param value: points' value + :param value: point count + """ + return self.__wrapped.Record(value, count) diff --git a/library/python/monlib/metric_consumer.pxd b/library/python/monlib/metric_consumer.pxd new file mode 100644 index 0000000000..7d259f7e3b --- /dev/null +++ b/library/python/monlib/metric_consumer.pxd @@ -0,0 +1,8 @@ +from util.generic.ptr cimport TIntrusivePtr + + +cdef extern from "library/cpp/monlib/metrics/metric_consumer.h" namespace "NMonitoring" nogil: + cdef cppclass IMetricConsumer: + pass + + ctypedef TIntrusivePtr[IMetricConsumer] IMetricConsumerPtr diff --git a/library/python/monlib/metric_registry.pxd b/library/python/monlib/metric_registry.pxd new file mode 100644 index 0000000000..e57c963929 --- /dev/null +++ b/library/python/monlib/metric_registry.pxd @@ -0,0 +1,29 @@ +from util.datetime.base cimport TInstant + +from library.python.monlib.labels cimport ILabels, TLabels +from library.python.monlib.metric_consumer cimport IMetricConsumer +from library.python.monlib.metric cimport ( + TGauge, TIntGauge, TRate, TCounter, THistogram, + IHistogramCollectorPtr) + + +cdef extern from "library/cpp/monlib/metrics/metric_registry.h" namespace "NMonitoring" nogil: + cdef cppclass TMetricRegistry: + TMetricRegistry() except + + TMetricRegistry(const TLabels&) except + + + TGauge* Gauge(const TLabels&) except + + TIntGauge* IntGauge(const TLabels&) except + + TCounter* Counter(const TLabels&) except + + TRate* Rate(const TLabels&) except + + THistogram* HistogramCounter(const TLabels&, IHistogramCollectorPtr collector) except + + THistogram* HistogramRate(const TLabels&, IHistogramCollectorPtr collector) except + + + void Reset() except + + + void Accept(TInstant time, IMetricConsumer* consumer) except + + void Append(TInstant time, IMetricConsumer* consumer) except + + + const TLabels& CommonLabels() const + + void RemoveMetric(const TLabels&) diff --git a/library/python/monlib/metric_registry.pyx b/library/python/monlib/metric_registry.pyx new file mode 100644 index 0000000000..800a1abd1b --- /dev/null +++ b/library/python/monlib/metric_registry.pyx @@ -0,0 +1,277 @@ +from library.python.monlib.encoder cimport Encoder +from library.python.monlib.labels cimport TLabels +from library.python.monlib.metric cimport ( + Gauge, IntGauge, Counter, Rate, Histogram, IHistogramCollectorPtr, + ExponentialHistogram, ExplicitHistogram, LinearHistogram) +from library.python.monlib.metric_consumer cimport IMetricConsumer +from library.python.monlib.metric_registry cimport TMetricRegistry + +from util.generic.ptr cimport THolder +from util.generic.string cimport TString +from util.datetime.base cimport TInstant +from util.system.types cimport ui32 +from util.generic.vector cimport TVector + +from libcpp.string cimport string + +from cython.operator cimport address, dereference as deref + +import datetime as dt +import sys + + +cdef extern from "<utility>" namespace "std" nogil: + cdef IHistogramCollectorPtr&& move(IHistogramCollectorPtr t) + + +def get_or_raise(kwargs, key): + value = kwargs.get(key) + if value is None: + raise ValueError(key + ' argument is required but not specified') + + return value + + +class HistogramType(object): + Exponential = 0 + Explicit = 1 + Linear = 2 + + +cdef class MetricRegistry: + """ + Represents an entity holding a set of counters of different types identified by labels + + Example usage: + .. :: + registry = MetricRegistry() + + response_times = registry.histogram_rate( + {'path': 'ping', 'sensor': 'responseTimeMillis'}, + HistogramType.Explicit, buckets=[10, 20, 50, 200, 500]) + + requests = registry.rate({'path': 'ping', 'sensor': 'requestRate'}) + uptime = registry.gauge({'sensor': 'serverUptimeSeconds'}) + + # ... + requests.inc() + uptime.set(time.time() - start_time) + + # ... + dumps(registry) + """ + cdef THolder[TMetricRegistry] __wrapped + + def __cinit__(self, labels=None): + cdef TLabels native_labels = MetricRegistry._py_to_native_labels(labels) + self.__wrapped.Reset(new TMetricRegistry(native_labels)) + + @staticmethod + cdef TLabels _py_to_native_labels(dict labels): + cdef TLabels native_labels = TLabels() + + if labels is not None: + for name, value in labels.items(): + native_labels.Add(TString(<string>name.encode('utf-8')), TString(<string>value.encode('utf-8'))) + + return native_labels + + @staticmethod + cdef _native_to_py_labels(const TLabels& native_labels): + result = dict() + + cdef TLabels.const_iterator it = native_labels.begin() + while it != native_labels.end(): + name = TString(deref(it).Name()) + value = TString(deref(it).Value()) + if (isinstance(name, bytes)): + name = name.decode('utf-8') + + if (isinstance(value, bytes)): + value = value.decode('utf-8') + + result[name] = value + it += 1 + + return result + + def _histogram(self, labels, is_rate, hist_type, **kwargs): + cdef TLabels native_labels = MetricRegistry._py_to_native_labels(labels) + cdef IHistogramCollectorPtr collector + cdef TVector[double] native_buckets + + if hist_type == HistogramType.Exponential: + buckets = int(get_or_raise(kwargs, 'bucket_count')) + base = float(get_or_raise(kwargs, 'base')) + scale = float(kwargs.get('scale', 1.)) + collector = move(ExponentialHistogram(buckets, base, scale)) + elif hist_type == HistogramType.Explicit: + buckets = get_or_raise(kwargs, 'buckets') + native_buckets = buckets + collector = move(ExplicitHistogram(native_buckets)) + elif hist_type == HistogramType.Linear: + buckets = get_or_raise(kwargs, 'bucket_count') + start_value = get_or_raise(kwargs, 'start_value') + bucket_width = get_or_raise(kwargs, 'bucket_width') + collector = move(LinearHistogram(buckets, start_value, bucket_width)) + else: + # XXX: string representation + raise ValueError('histogram type {} is not supported'.format(str(hist_type))) + + cdef THistogram* native_hist + if is_rate: + native_hist = self.__wrapped.Get().HistogramRate(native_labels, move(collector)) + else: + native_hist = self.__wrapped.Get().HistogramCounter(native_labels, move(collector)) + + return Histogram.from_ptr(native_hist) + + @property + def common_labels(self): + """ + Gets labels that are common among all the counters in this registry + + :returns: Common labels as a dict + """ + cdef const TLabels* native = address(self.__wrapped.Get().CommonLabels()) + labels = MetricRegistry._native_to_py_labels(deref(native)) + + return labels + + def gauge(self, labels): + """ + Gets a gauge counter or creates a new one in case counter with the specified labels + does not exist + + :param labels: A dict of labels which identifies counter + :returns: Gauge counter + """ + cdef TLabels native_labels = MetricRegistry._py_to_native_labels(labels) + native_gauge = self.__wrapped.Get().Gauge(native_labels) + return Gauge.from_ptr(native_gauge) + + def int_gauge(self, labels): + """ + Gets a gauge counter or creates a new one in case counter with the specified labels + does not exist + + :param labels: A dict of labels which identifies counter + :returns: IntGauge counter + """ + cdef TLabels native_labels = MetricRegistry._py_to_native_labels(labels) + native_gauge = self.__wrapped.Get().IntGauge(native_labels) + return IntGauge.from_ptr(native_gauge) + + def counter(self, labels): + """ + Gets a counter or creates a new one in case counter with the specified labels + does not exist + + :param labels: A dict of labels which identifies counter + :returns: Counter counter + """ + cdef TLabels native_labels = MetricRegistry._py_to_native_labels(labels) + native_counter = self.__wrapped.Get().Counter(native_labels) + return Counter.from_ptr(native_counter) + + def rate(self, labels): + """ + Gets a rate counter or creates a new one in case counter with the specified labels + does not exist + + :param labels: A dict of labels which identifies counter + :returns: Rate counter + """ + cdef TLabels native_labels = MetricRegistry._py_to_native_labels(labels) + native_rate = self.__wrapped.Get().Rate(native_labels) + return Rate.from_ptr(native_rate) + + def histogram_counter(self, labels, hist_type, **kwargs): + """ + Gets a histogram counter or creates a new one in case counter with the specified labels + does not exist + + :param labels: A dict of labels which identifies counter + :param hist_type: Specifies the way histogram buckets are defined (allowed values: explicit, exponential, linear) + + Keyword arguments: + :param buckets: A list of bucket upper bounds (explicit) + :param bucket_count: Number of buckets (linear, exponential) + :param base: the exponential growth factor for buckets' width (exponential) + :param scale: linear scale for the buckets. Must be >= 1.0 (exponential) + :param start_value: the upper bound of the first bucket (linear) + + :returns: Histogram counter + + Example usage: + .. :: + my_histogram = registry.histogram_counter( + {'path': 'ping', 'sensor': 'responseTimeMillis'}, + HistogramType.Explicit, buckets=[10, 20, 50, 200, 500]) + # (-inf; 10] (10; 20] (20; 50] (200; 500] (500; +inf) + + # or: + my_histogram = registry.histogram_counter( + {'path': 'ping', 'sensor': 'responseTimeMillis'}, + HistogramType.Linear, bucket_count=4, bucket_width=10, start_value=0) + # (-inf; 0] (0; 10] (10; 20] (20; +inf) + + # or: + my_histogram = registry.histogram_counter( + {'path': 'ping', 'sensor': 'responseTimeMillis'}, + HistogramType.Exponential, bucket_count=6, base=2, scale=3) + # (-inf; 3] (3; 6] (6; 12] (12; 24] (24; 48] (48; +inf) + :: + """ + return self._histogram(labels, False, hist_type, **kwargs) + + def histogram_rate(self, labels, hist_type, **kwargs): + """ + Gets a histogram rate counter or creates a new one in case counter with the specified labels + does not exist + + :param labels: A dict of labels which identifies counter + :param hist_type: Specifies the way histogram buckets are defined (allowed values: explicit, exponential, linear) + + Keyword arguments: + :param buckets: A list of bucket upper bounds (explicit) + :param bucket_count: Number of buckets (linear, exponential) + :param base: the exponential growth factor for buckets' width (exponential) + :param scale: linear scale for the buckets. Must be >= 1.0 (exponential) + :param start_value: the upper bound of the first bucket (linear) + + :returns: Histogram counter + + Example usage: + .. :: + my_histogram = registry.histogram_counter( + {'path': 'ping', 'sensor': 'responseTimeMillis'}, + HistogramType.Explicit, buckets=[10, 20, 50, 200, 500]) + # (-inf; 10] (10; 20] (20; 50] (200; 500] (500; +inf) + + # or: + my_histogram = registry.histogram_counter( + {'path': 'ping', 'sensor': 'responseTimeMillis'}, + HistogramType.Linear, bucket_count=4, bucket_width=10, start_value=0) + # (-inf; 0] (0; 10] (10; 20] (20; +inf) + + # or: + my_histogram = registry.histogram_counter( + {'path': 'ping', 'sensor': 'responseTimeMillis'}, + HistogramType.Exponential, bucket_count=6, base=2, scale=3) + # (-inf; 3] (3; 6] (6; 12] (12; 24] (24; 48] (48; +inf) + :: + """ + return self._histogram(labels, True, hist_type, **kwargs) + + def reset(self): + self.__wrapped.Get().Reset() + + def accept(self, time, Encoder encoder): + cdef IMetricConsumer* ptr = <IMetricConsumer*>encoder.native() + timestamp = int((time - dt.datetime(1970, 1, 1)).total_seconds()) + self.__wrapped.Get().Accept(TInstant.Seconds(timestamp), ptr) + + def remove_metric(self, labels): + cdef TLabels native_labels = MetricRegistry._py_to_native_labels(labels) + self.__wrapped.Get().RemoveMetric(native_labels) diff --git a/library/python/monlib/ut/metric_ut.pyx b/library/python/monlib/ut/metric_ut.pyx new file mode 100644 index 0000000000..3513eaf9d1 --- /dev/null +++ b/library/python/monlib/ut/metric_ut.pyx @@ -0,0 +1,113 @@ +from library.python.monlib.labels cimport TLabels, TLabel +from library.python.monlib.metric cimport ( + TGauge, TCounter, + TRate, THistogram, + IHistogramCollectorPtr, ExponentialHistogram, + IHistogramSnapshotPtr +) + +from library.python.monlib.metric_registry cimport TMetricRegistry + +from util.generic.string cimport TStringBuf, TString +from util.generic.maybe cimport TMaybe +from util.generic.ptr cimport THolder + +from cython.operator cimport dereference as deref + +import pytest +import unittest + + +cdef extern from "<utility>" namespace "std" nogil: + cdef IHistogramCollectorPtr&& move(IHistogramCollectorPtr t) + + +class TestMetric(unittest.TestCase): + def test_labels(self): + cdef TLabels labels = TLabels() + cdef TString name = "foo" + cdef TString value = "bar" + + labels.Add(name, value) + + cdef TMaybe[TLabel] label = labels.Find(name) + + assert label.Defined() + assert label.GetRef().Name() == "foo" + assert label.GetRef().Value() == "bar" + + def test_metric_registry(self): + cdef TLabels labels = TLabels() + + labels.Add(TString("common"), TString("label")) + + cdef THolder[TMetricRegistry] registry + registry.Reset(new TMetricRegistry(labels)) + + assert deref(registry.Get()).CommonLabels() == labels + + cdef TLabels metric_labels = TLabels() + metric_labels.Add(TString("name"), TString("gauge")) + g = deref(registry.Get()).Gauge(metric_labels) + assert g.Get() == 0. + + metric_labels = TLabels() + metric_labels.Add(TString("name"), TString("counter")) + c = deref(registry.Get()).Counter(metric_labels) + assert c.Get() == 0. + + metric_labels = TLabels() + metric_labels.Add(TString("name"), TString("rate")) + r = deref(registry.Get()).Rate(metric_labels) + assert r.Get() == 0. + + metric_labels = TLabels() + metric_labels.Add(TString("name"), TString("int_gauge")) + ig = deref(registry.Get()).IntGauge(metric_labels) + assert ig.Get() == 0 + + def test_metric_registry_throws_on_duplicate(self): + cdef THolder[TMetricRegistry] registry + registry.Reset(new TMetricRegistry()) + + cdef TLabels metric_labels = TLabels() + metric_labels.Add(TString("my"), TString("metric")) + g = deref(registry.Get()).Gauge(metric_labels) + with pytest.raises(RuntimeError): + deref(registry.Get()).Counter(metric_labels) + + def test_counter_histogram(self): + cdef THolder[TMetricRegistry] registry + registry.Reset(new TMetricRegistry()) + + cdef TLabels metric_labels = TLabels() + metric_labels.Add(TString("name"), TString("histogram")) + + cdef IHistogramCollectorPtr collector = move(ExponentialHistogram(6, 2, 3)) + collector_ptr = collector.Get() + hist = registry.Get().HistogramCounter(metric_labels, move(collector)) + hist.Record(1) + hist.Record(1000, 4) + + cdef IHistogramSnapshotPtr snapshot = collector_ptr.Snapshot() + assert deref(snapshot.Get()).Count() == 6 + assert snapshot.Get().Value(0) == 1 + + def test_rate_histogram(self): + cdef THolder[TMetricRegistry] registry + registry.Reset(new TMetricRegistry()) + + cdef TLabels metric_labels = TLabels() + metric_labels.Add(TString("name"), TString("histogram")) + + cdef IHistogramCollectorPtr collector = move(ExponentialHistogram(6, 2, 3)) + collector_ptr = collector.Get() + hist = registry.Get().HistogramRate(metric_labels, move(collector)) + hist.Record(1) + hist.Record(1000, 4) + + cdef IHistogramSnapshotPtr snapshot = collector_ptr.Snapshot() + assert deref(snapshot.Get()).Count() == 6 + assert snapshot.Get().Value(0) == 1 + assert snapshot.Get().Value(5) == 4 + assert snapshot.Get().Value(5) == 4 diff --git a/library/python/monlib/ut/py2/test.py b/library/python/monlib/ut/py2/test.py new file mode 100644 index 0000000000..2880120a12 --- /dev/null +++ b/library/python/monlib/ut/py2/test.py @@ -0,0 +1,313 @@ +# -- coding: utf-8 -- + +from __future__ import print_function + +import sys # noqa +import json + +from tempfile import TemporaryFile + +import pytest # noqa + +from library.python.monlib.metric_registry import MetricRegistry, HistogramType +from library.python.monlib.encoder import dump, dumps, TimePrecision, load, loads # noqa + + +def test_common_labels(request): + labels = {'my': 'label'} + registry = MetricRegistry(labels) + assert registry.common_labels == labels + + with pytest.raises(TypeError): + MetricRegistry('foo') + + with pytest.raises(TypeError): + MetricRegistry([]) + + +def test_json_serialization(request): + registry = MetricRegistry() + labels = {'foo': 'gauge'} + + g = registry.gauge(labels) + + g.set(10.0) + g.set(20) + + c = registry.counter({'foo': 'counter'}) + c.inc() + + r = registry.rate({'foo': 'rate'}) + r.add(10) + + out = dumps(registry, format='json', precision=TimePrecision.Seconds) + expected = json.loads("""{ + "sensors": + [ + { + "kind":"RATE", + "labels": + { + "foo":"rate" + }, + "value":10 + }, + { + "kind":"COUNTER", + "labels": + { + "foo":"counter" + }, + "value":1 + }, + { + "kind":"GAUGE", + "labels": + { + "foo":"gauge" + }, + "value":20 + } + ] + } + """) + + j = json.loads(out) + assert j == expected + + +EXPECTED_EXPLICIT = json.loads(""" + { + "sensors": + [ + { + "kind":"HIST", + "labels": + { + "foo":"hist" + }, + "hist": + { + "bounds": + [ + 2, + 10, + 500 + ], + "buckets": + [ + 1, + 0, + 0 + ], + "inf":1 + } + } + ] + } +""") + +EXPECTED_EXPONENTIAL = json.loads("""{ + "sensors": + [ + { + "kind":"HIST", + "labels": + { + "foo":"hist" + }, + "hist": + { + "bounds": + [ + 3, + 6, + 12, + 24, + 48 + ], + "buckets": + [ + 1, + 0, + 0, + 0, + 0 + ], + "inf":1 + } + } + ] +} +""") + +EXPECTED_LINEAR = json.loads(""" + { "sensors": + [ + { + "kind":"HIST", + "labels": + { + "foo":"hist" + }, + "hist": + { + "bounds": + [ + 1 + ], + "buckets": + [ + 1 + ], + "inf":1 + } + } + ] +}""") + + +@pytest.mark.parametrize('type,args,expected', [ + (HistogramType.Linear, dict(bucket_count=2, start_value=1, bucket_width=1), EXPECTED_LINEAR), + (HistogramType.Explicit, dict(buckets=[2, 10, 500]), EXPECTED_EXPLICIT), + (HistogramType.Exponential, dict(bucket_count=6, base=2, scale=3), EXPECTED_EXPONENTIAL), +]) +@pytest.mark.parametrize('rate', [True, False]) +def test_histograms(request, type, args, expected, rate): + registry = MetricRegistry() + labels = {'foo': 'hist'} + + h = registry.histogram_counter(labels, type, **args) if not rate else registry.histogram_rate(labels, type, **args) + h.collect(1) + h.collect(1000, 1) + + s = dumps(registry, format='json') + + if rate: + expected['sensors'][0]['kind'] = u'HIST_RATE' + else: + expected['sensors'][0]['kind'] = u'HIST' + + assert json.loads(s) == expected + + +@pytest.mark.parametrize('fmt', ['json', 'spack']) +def test_stream_load(request, fmt): + expected = json.loads("""{"sensors":[{"kind":"GAUGE","labels":{"foo":"gauge"},"value":42}]}""") + registry = MetricRegistry() + labels = {'foo': 'gauge'} + + g = registry.gauge(labels) + g.set(42) + + with TemporaryFile() as f: + dump(registry, f, format=fmt) + f.flush() + f.seek(0, 0) + s = load(f, from_format=fmt, to_format='json') + assert json.loads(s) == expected + + +@pytest.mark.parametrize('fmt', ['json', 'spack']) +def test_stream_loads(request, fmt): + expected = json.loads("""{"sensors":[{"kind":"GAUGE","labels":{"foo":"gauge"},"value":42}]}""") + registry = MetricRegistry() + labels = {'foo': 'gauge'} + + g = registry.gauge(labels) + g.set(42) + + s = dumps(registry, format=fmt) + j = loads(s, from_format=fmt, to_format='json') + assert json.loads(j) == expected + + +@pytest.mark.parametrize('fmt', ['json', 'spack']) +def test_utf(request, fmt): + expected = json.loads(u"""{"sensors":[{"kind":"GAUGE","labels":{"foo":"gaugeह", "bàr":"Münich"},"value":42}]}""".encode('utf-8')) + registry = MetricRegistry() + labels = {'foo': u'gaugeह', u'bàr': u'Münich'} + + g = registry.gauge(labels) + g.set(42) + + s = dumps(registry, format=fmt) + j = loads(s, from_format=fmt, to_format='json') + assert json.loads(j) == expected + + +def test_gauge_sensors(): + registry = MetricRegistry() + g = registry.gauge({'a': 'b'}) + ig = registry.int_gauge({'c': 'd'}) + + g.set(2) + assert g.add(3.5) == 5.5 + assert g.get() == 5.5 + + ig.set(2) + assert ig.inc() == 3 + assert ig.dec() == 2 + assert ig.add(3) == 5 + assert ig.get() == 5 + + +UNISTAT_DATA = """[ + ["signal1_max", 10], + ["signal2_hgram", [[0, 100], [50, 200], [200, 300]]], + ["prj=some-project;signal3_summ", 3], + ["signal4_summ", 5] +]""" + + +EXPECTED = json.loads(""" +{ + "sensors": [ + { + "kind": "GAUGE", + "labels": { + "sensor": "signal1_max" + }, + "value": 10 + }, + { + "hist": { + "buckets": [ + 0, + 100, + 200 + ], + "bounds": [ + 0, + 50, + 200 + ], + "inf": 300 + }, + "kind": "HIST_RATE", + "labels": { + "sensor": "signal2_hgram" + } + }, + { + "kind": "RATE", + "labels": { + "sensor": "signal3_summ", + "prj": "some-project" + }, + "value": 3 + }, + { + "kind": "RATE", + "labels": { + "sensor": "signal4_summ" + }, + "value": 5 + } + ] +}""") + + +def test_unistat_conversion(request): + j = loads(UNISTAT_DATA, from_format='unistat', to_format='json') + assert json.loads(j) == EXPECTED diff --git a/library/python/monlib/ut/py2/test_metric.py b/library/python/monlib/ut/py2/test_metric.py new file mode 100644 index 0000000000..fe391ce35d --- /dev/null +++ b/library/python/monlib/ut/py2/test_metric.py @@ -0,0 +1,3 @@ +from library.python.monlib.ut.metric_ut import TestMetric + +__all__ = ['TestMetric'] diff --git a/library/python/monlib/ut/py2/ya.make b/library/python/monlib/ut/py2/ya.make new file mode 100644 index 0000000000..b158bb357b --- /dev/null +++ b/library/python/monlib/ut/py2/ya.make @@ -0,0 +1,17 @@ +PY2TEST() + +TEST_SRCS( + test_metric.py + test.py +) + +SRCDIR( + library/python/monlib/ut +) + +PEERDIR( + library/python/monlib + library/python/monlib/ut +) + +END() diff --git a/library/python/monlib/ut/py3/test.py b/library/python/monlib/ut/py3/test.py new file mode 100644 index 0000000000..431241c657 --- /dev/null +++ b/library/python/monlib/ut/py3/test.py @@ -0,0 +1,311 @@ +from __future__ import print_function + +import sys # noqa +import json + +from tempfile import TemporaryFile + +import pytest # noqa + +from library.python.monlib.metric_registry import MetricRegistry, HistogramType +from library.python.monlib.encoder import dump, dumps, TimePrecision, load, loads # noqa + + +def test_common_labels(request): + labels = {'my': 'label'} + registry = MetricRegistry(labels) + assert registry.common_labels == labels + + with pytest.raises(TypeError): + MetricRegistry('foo') + + with pytest.raises(TypeError): + MetricRegistry([]) + + +def test_json_serialization(request): + registry = MetricRegistry() + labels = {'foo': 'gauge'} + + g = registry.gauge(labels) + + g.set(10.0) + g.set(20) + + c = registry.counter({'foo': 'counter'}) + c.inc() + + r = registry.rate({'foo': 'rate'}) + r.add(10) + + out = dumps(registry, format='json', precision=TimePrecision.Seconds) + expected = json.loads("""{ + "sensors": + [ + { + "kind":"RATE", + "labels": + { + "foo":"rate" + }, + "value":10 + }, + { + "kind":"COUNTER", + "labels": + { + "foo":"counter" + }, + "value":1 + }, + { + "kind":"GAUGE", + "labels": + { + "foo":"gauge" + }, + "value":20 + } + ] + } + """) + + j = json.loads(out) + assert j == expected + + +EXPECTED_EXPLICIT = json.loads(""" + { + "sensors": + [ + { + "kind":"HIST", + "labels": + { + "foo":"hist" + }, + "hist": + { + "bounds": + [ + 2, + 10, + 500 + ], + "buckets": + [ + 1, + 0, + 0 + ], + "inf":1 + } + } + ] + } +""") + +EXPECTED_EXPONENTIAL = json.loads("""{ + "sensors": + [ + { + "kind":"HIST", + "labels": + { + "foo":"hist" + }, + "hist": + { + "bounds": + [ + 3, + 6, + 12, + 24, + 48 + ], + "buckets": + [ + 1, + 0, + 0, + 0, + 0 + ], + "inf":1 + } + } + ] +} +""") + +EXPECTED_LINEAR = json.loads(""" + { "sensors": + [ + { + "kind":"HIST", + "labels": + { + "foo":"hist" + }, + "hist": + { + "bounds": + [ + 1 + ], + "buckets": + [ + 1 + ], + "inf":1 + } + } + ] +}""") + + +@pytest.mark.parametrize('type,args,expected', [ + (HistogramType.Linear, dict(bucket_count=2, start_value=1, bucket_width=1), EXPECTED_LINEAR), + (HistogramType.Explicit, dict(buckets=[2, 10, 500]), EXPECTED_EXPLICIT), + (HistogramType.Exponential, dict(bucket_count=6, base=2, scale=3), EXPECTED_EXPONENTIAL), +]) +@pytest.mark.parametrize('rate', [True, False]) +def test_histograms(request, type, args, expected, rate): + registry = MetricRegistry() + labels = {'foo': 'hist'} + + h = registry.histogram_counter(labels, type, **args) if not rate else registry.histogram_rate(labels, type, **args) + h.collect(1) + h.collect(1000) + + s = dumps(registry, format='json') + + if rate: + expected['sensors'][0]['kind'] = u'HIST_RATE' + else: + expected['sensors'][0]['kind'] = u'HIST' + + assert json.loads(s) == expected + + +@pytest.mark.parametrize('fmt', ['json', 'spack']) +def test_stream_load(request, fmt): + expected = json.loads("""{"sensors":[{"kind":"GAUGE","labels":{"foo":"gauge"},"value":42}]}""") + registry = MetricRegistry() + labels = {'foo': 'gauge'} + + g = registry.gauge(labels) + g.set(42) + + with TemporaryFile() as f: + dump(registry, f, format=fmt) + f.flush() + f.seek(0, 0) + s = load(f, from_format=fmt, to_format='json') + assert json.loads(s) == expected + + +@pytest.mark.parametrize('fmt', ['json', 'spack']) +def test_stream_loads(request, fmt): + expected = json.loads("""{"sensors":[{"kind":"GAUGE","labels":{"foo":"gauge"},"value":42}]}""") + registry = MetricRegistry() + labels = {'foo': 'gauge'} + + g = registry.gauge(labels) + g.set(42) + + s = dumps(registry, format=fmt) + j = loads(s, from_format=fmt, to_format='json') + assert json.loads(j) == expected + + +@pytest.mark.parametrize('fmt', ['json', 'spack']) +def test_utf(request, fmt): + expected = json.loads(u"""{"sensors":[{"kind":"GAUGE","labels":{"foo":"gaugeह", "bàr":"Münich"},"value":42}]}""") + registry = MetricRegistry() + labels = {'foo': u'gaugeह', u'bàr': u'Münich'} + + g = registry.gauge(labels) + g.set(42) + + s = dumps(registry, format=fmt) + j = loads(s, from_format=fmt, to_format='json') + assert json.loads(j) == expected + + +def test_gauge_sensors(): + registry = MetricRegistry() + g = registry.gauge({'a': 'b'}) + ig = registry.int_gauge({'c': 'd'}) + + g.set(2) + assert g.add(3.5) == 5.5 + assert g.get() == 5.5 + + ig.set(2) + assert ig.inc() == 3 + assert ig.dec() == 2 + assert ig.add(3) == 5 + assert ig.get() == 5 + + +UNISTAT_DATA = """[ + ["signal1_max", 10], + ["signal2_hgram", [[0, 100], [50, 200], [200, 300]]], + ["prj=some-project;signal3_summ", 3], + ["signal4_summ", 5] +]""" + + +EXPECTED = json.loads(""" +{ + "sensors": [ + { + "kind": "GAUGE", + "labels": { + "sensor": "signal1_max" + }, + "value": 10 + }, + { + "hist": { + "buckets": [ + 0, + 100, + 200 + ], + "bounds": [ + 0, + 50, + 200 + ], + "inf": 300 + }, + "kind": "HIST_RATE", + "labels": { + "sensor": "signal2_hgram" + } + }, + { + "kind": "RATE", + "labels": { + "sensor": "signal3_summ", + "prj": "some-project" + }, + "value": 3 + }, + { + "kind": "RATE", + "labels": { + "sensor": "signal4_summ" + }, + "value": 5 + } + ] +}""") + + +def test_unistat_conversion(request): + j = loads(UNISTAT_DATA, from_format='unistat', to_format='json') + assert json.loads(j) == EXPECTED diff --git a/library/python/monlib/ut/py3/test_metric.py b/library/python/monlib/ut/py3/test_metric.py new file mode 100644 index 0000000000..fe391ce35d --- /dev/null +++ b/library/python/monlib/ut/py3/test_metric.py @@ -0,0 +1,3 @@ +from library.python.monlib.ut.metric_ut import TestMetric + +__all__ = ['TestMetric'] diff --git a/library/python/monlib/ut/py3/ya.make b/library/python/monlib/ut/py3/ya.make new file mode 100644 index 0000000000..d711132df8 --- /dev/null +++ b/library/python/monlib/ut/py3/ya.make @@ -0,0 +1,17 @@ +PY3TEST() + +TEST_SRCS( + test_metric.py + test.py +) + +SRCDIR( + library/python/monlib/ut +) + +PEERDIR( + library/python/monlib + library/python/monlib/ut +) + +END() diff --git a/library/python/monlib/ut/ya.make b/library/python/monlib/ut/ya.make new file mode 100644 index 0000000000..9082c0e323 --- /dev/null +++ b/library/python/monlib/ut/ya.make @@ -0,0 +1,7 @@ +PY23_LIBRARY() + +PY_SRCS( + metric_ut.pyx +) + +END() diff --git a/library/python/monlib/ya.make b/library/python/monlib/ya.make new file mode 100644 index 0000000000..5c7de8de58 --- /dev/null +++ b/library/python/monlib/ya.make @@ -0,0 +1,21 @@ +PY23_LIBRARY() + +PY_SRCS( + encoder.pyx + metric.pyx + metric_registry.pyx +) + +PEERDIR( + library/cpp/monlib/metrics + library/cpp/monlib/encode/json + library/cpp/monlib/encode/spack + library/cpp/monlib/encode/unistat +) + +END() + +RECURSE_FOR_TESTS( + ut/py2 + ut/py3 +) diff --git a/util/datetime/base.pxd b/util/datetime/base.pxd new file mode 100644 index 0000000000..1d863b1bd8 --- /dev/null +++ b/util/datetime/base.pxd @@ -0,0 +1,126 @@ +from libc.stdint cimport uint32_t +from libc.stdint cimport uint64_t +from libcpp cimport bool as bool_t +from posix.types cimport time_t + +from util.generic.string cimport TString, TStringBuf + + +cdef extern from "<util/datetime/base.h>" nogil: + + cdef cppclass TTimeBase: + TTimeBase() + TTimeBase(uint64_t) + + uint64_t GetValue() + double SecondsFloat() + uint64_t MicroSeconds() + uint64_t MilliSeconds() + uint64_t Seconds() + uint64_t Minutes() + uint64_t Hours() + uint64_t Days() + uint64_t NanoSeconds() + uint32_t MicroSecondsOfSecond() + uint32_t MilliSecondsOfSecond() + uint32_t NanoSecondsOfSecond() + + + cdef cppclass TInstant(TTimeBase): + TInstant() + TInstant(uint64_t) + + @staticmethod + TInstant Now() except + + @staticmethod + TInstant Max() + @staticmethod + TInstant Zero() + @staticmethod + TInstant MicroSeconds(uint64_t) + @staticmethod + TInstant MilliSeconds(uint64_t) + @staticmethod + TInstant Seconds(uint64_t) + @staticmethod + TInstant Minutes(uint64_t) + @staticmethod + TInstant Hours(uint64_t) + @staticmethod + TInstant Days(uint64_t) + + time_t TimeT() + + TString ToString() except + + TString ToStringUpToSeconds() except + + TString ToStringLocal() except + + TString ToStringLocalUpToSeconds() except + + TString FormatLocalTime(const char*) + TString FormatGmTime(const char* format) + + @staticmethod + TInstant ParseIso8601(const TStringBuf) except + + @staticmethod + TInstant ParseRfc822(const TStringBuf) except + + @staticmethod + TInstant ParseHttp(const TStringBuf) except + + @staticmethod + TInstant ParseX509Validity(const TStringBuf) except + + + @staticmethod + bool_t TryParseIso8601(const TStringBuf, TInstant&) except + + @staticmethod + bool_t TryParseRfc822(const TStringBuf, TInstant&) except + + @staticmethod + bool_t TryParseHttp(const TStringBuf, TInstant&) except + + @staticmethod + bool_t TryParseX509(const TStringBuf, TInstant&) except + + + @staticmethod + TInstant ParseIso8601Deprecated(const TStringBuf) except + + @staticmethod + TInstant ParseRfc822Deprecated(const TStringBuf) except + + @staticmethod + TInstant ParseHttpDeprecated(const TStringBuf) except + + @staticmethod + TInstant ParseX509ValidityDeprecated(const TStringBuf) except + + + @staticmethod + bool_t TryParseIso8601Deprecated(const TStringBuf, TInstant&) except + + @staticmethod + bool_t TryParseRfc822Deprecated(const TStringBuf, TInstant&) except + + @staticmethod + bool_t TryParseHttpDeprecated(const TStringBuf, TInstant&) except + + @staticmethod + bool_t TryParseX509Deprecated(const TStringBuf, TInstant&) except + + + + cdef cppclass TDuration(TTimeBase): + TDuration() + TDuration(uint64_t) + + @staticmethod + TDuration MicroSeconds(uint64_t) + + TInstant ToDeadLine() except + + TInstant ToDeadLine(TInstant) except + + + @staticmethod + TDuration Max() + @staticmethod + TDuration Zero() + @staticmethod + TDuration Seconds(uint64_t) + @staticmethod + TDuration Minutes(uint64_t) + @staticmethod + TDuration Hours(uint64_t) + @staticmethod + TDuration Days(uint64_t) + + @staticmethod + TDuration Parse(const TStringBuf) + @staticmethod + bool_t TryParse(const TStringBuf, TDuration&) + + TString ToString() except + |