diff options
author | Devtools Arcadia <arcadia-devtools@yandex-team.ru> | 2022-02-07 18:08:42 +0300 |
---|---|---|
committer | Devtools Arcadia <arcadia-devtools@mous.vla.yp-c.yandex.net> | 2022-02-07 18:08:42 +0300 |
commit | 1110808a9d39d4b808aef724c861a2e1a38d2a69 (patch) | |
tree | e26c9fed0de5d9873cce7e00bc214573dc2195b7 /library/cpp/monlib/encode/json | |
download | ydb-1110808a9d39d4b808aef724c861a2e1a38d2a69.tar.gz |
intermediate changes
ref:cde9a383711a11544ce7e107a78147fb96cc4029
Diffstat (limited to 'library/cpp/monlib/encode/json')
33 files changed, 4315 insertions, 0 deletions
diff --git a/library/cpp/monlib/encode/json/fuzz/main.cpp b/library/cpp/monlib/encode/json/fuzz/main.cpp new file mode 100644 index 0000000000..4f40310e06 --- /dev/null +++ b/library/cpp/monlib/encode/json/fuzz/main.cpp @@ -0,0 +1,16 @@ +#include <library/cpp/monlib/encode/json/json.h> +#include <library/cpp/monlib/encode/fake/fake.h> + +#include <util/generic/strbuf.h> + + +extern "C" int LLVMFuzzerTestOneInput(const ui8* data, size_t size) { + auto encoder = NMonitoring::EncoderFake(); + + try { + NMonitoring::DecodeJson({reinterpret_cast<const char*>(data), size}, encoder.Get()); + } catch (...) { + } + + return 0; +} diff --git a/library/cpp/monlib/encode/json/fuzz/ya.make b/library/cpp/monlib/encode/json/fuzz/ya.make new file mode 100644 index 0000000000..75baa77716 --- /dev/null +++ b/library/cpp/monlib/encode/json/fuzz/ya.make @@ -0,0 +1,19 @@ +FUZZ() + +OWNER( + g:solomon + msherbakov +) + +PEERDIR( + library/cpp/monlib/encode/json + library/cpp/monlib/encode/fake +) + +SIZE(MEDIUM) + +SRCS( + main.cpp +) + +END() diff --git a/library/cpp/monlib/encode/json/json.h b/library/cpp/monlib/encode/json/json.h new file mode 100644 index 0000000000..21530f20c3 --- /dev/null +++ b/library/cpp/monlib/encode/json/json.h @@ -0,0 +1,29 @@ +#pragma once + +#include <library/cpp/monlib/encode/encoder.h> +#include <library/cpp/monlib/encode/format.h> + + +class IOutputStream; + +namespace NMonitoring { + + class TJsonDecodeError: public yexception { + }; + + IMetricEncoderPtr EncoderJson(IOutputStream* out, int indentation = 0); + + /// Buffered encoder will merge series with same labels into one. + IMetricEncoderPtr BufferedEncoderJson(IOutputStream* out, int indentation = 0); + + IMetricEncoderPtr EncoderCloudJson(IOutputStream* out, + int indentation = 0, + TStringBuf metricNameLabel = "name"); + + IMetricEncoderPtr BufferedEncoderCloudJson(IOutputStream* out, + int indentation = 0, + TStringBuf metricNameLabel = "name"); + + void DecodeJson(TStringBuf data, IMetricConsumer* c, TStringBuf metricNameLabel = "name"); + +} diff --git a/library/cpp/monlib/encode/json/json_decoder.cpp b/library/cpp/monlib/encode/json/json_decoder.cpp new file mode 100644 index 0000000000..d44ff5fd28 --- /dev/null +++ b/library/cpp/monlib/encode/json/json_decoder.cpp @@ -0,0 +1,1162 @@ +#include "json.h" +#include "typed_point.h" + + +#include <library/cpp/monlib/exception/exception.h> +#include <library/cpp/monlib/metrics/labels.h> +#include <library/cpp/monlib/metrics/metric_value.h> + +#include <library/cpp/json/json_reader.h> + +#include <util/datetime/base.h> +#include <util/string/cast.h> + +#include <limits> + +namespace NMonitoring { + +#define DECODE_ENSURE(COND, ...) MONLIB_ENSURE_EX(COND, TJsonDecodeError() << __VA_ARGS__) + +namespace { + +/////////////////////////////////////////////////////////////////////// +// THistogramBuilder +/////////////////////////////////////////////////////////////////////// +class THistogramBuilder { +public: + void AddBound(TBucketBound bound) { + if (!Bounds_.empty()) { + DECODE_ENSURE(Bounds_.back() < bound, + "non sorted bounds, " << Bounds_.back() << + " >= " << bound); + } + Bounds_.push_back(bound); + } + + void AddValue(TBucketValue value) { + Values_.push_back(value); + } + + void AddInf(TBucketValue value) { + InfPresented_ = true; + InfValue_ = value; + } + + IHistogramSnapshotPtr Build() { + if (InfPresented_) { + Bounds_.push_back(Max<TBucketBound>()); + Values_.push_back(InfValue_); + } + + auto snapshot = ExplicitHistogramSnapshot(Bounds_, Values_); + + Bounds_.clear(); + Values_.clear(); + InfPresented_ = false; + + return snapshot; + } + + bool Empty() const noexcept { + return Bounds_.empty() && Values_.empty(); + } + + void Clear() { + Bounds_.clear(); + Values_.clear(); + } + +private: + TBucketBounds Bounds_; + TBucketValues Values_; + + bool InfPresented_ = false; + TBucketValue InfValue_; +}; + +class TSummaryDoubleBuilder { +public: + ISummaryDoubleSnapshotPtr Build() const { + return MakeIntrusive<TSummaryDoubleSnapshot>(Sum_, Min_, Max_, Last_, Count_); + } + + void SetSum(double sum) { + Empty_ = false; + Sum_ = sum; + } + + void SetMin(double min) { + Empty_ = false; + Min_ = min; + } + + void SetMax(double max) { + Empty_ = false; + Max_ = max; + } + + void SetLast(double last) { + Empty_ = false; + Last_ = last; + } + + void SetCount(ui64 count) { + Empty_ = false; + Count_ = count; + } + + void Clear() { + Empty_ = true; + Sum_ = 0; + Min_ = 0; + Max_ = 0; + Last_ = 0; + Count_ = 0; + } + + bool Empty() const { + return Empty_; + } + +private: + double Sum_ = 0; + double Min_ = 0; + double Max_ = 0; + double Last_ = 0; + ui64 Count_ = 0; + bool Empty_ = true; +}; + +class TLogHistogramBuilder { +public: + void SetBase(double base) { + DECODE_ENSURE(base > 0, "base must be positive"); + Base_ = base; + } + + void SetZerosCount(ui64 zerosCount) { + DECODE_ENSURE(zerosCount >= 0, "zeros count must be positive"); + ZerosCount_ = zerosCount; + } + + void SetStartPower(int startPower) { + StartPower_ = startPower; + } + + void AddBucketValue(double value) { + DECODE_ENSURE(value > 0.0, "bucket values must be positive"); + DECODE_ENSURE(value < std::numeric_limits<double>::max(), "bucket values must be finite"); + Buckets_.push_back(value); + } + + void Clear() { + Buckets_.clear(); + Base_ = 1.5; + ZerosCount_ = 0; + StartPower_ = 0; + } + + bool Empty() const { + return Buckets_.empty() && ZerosCount_ == 0; + } + + TLogHistogramSnapshotPtr Build() { + return MakeIntrusive<TLogHistogramSnapshot>(Base_, ZerosCount_, StartPower_, std::move(Buckets_)); + } + +private: + double Base_ = 1.5; + ui64 ZerosCount_ = 0; + int StartPower_ = 0; + TVector<double> Buckets_; +}; + +std::pair<double, bool> ParseSpecDouble(TStringBuf string) { + if (string == TStringBuf("nan") || string == TStringBuf("NaN")) { + return {std::numeric_limits<double>::quiet_NaN(), true}; + } else if (string == TStringBuf("inf") || string == TStringBuf("Infinity")) { + return {std::numeric_limits<double>::infinity(), true}; + } else if (string == TStringBuf("-inf") || string == TStringBuf("-Infinity")) { + return {-std::numeric_limits<double>::infinity(), true}; + } else { + return {0, false}; + } +} + +/////////////////////////////////////////////////////////////////////// +// TMetricCollector +/////////////////////////////////////////////////////////////////////// +struct TMetricCollector { + EMetricType Type = EMetricType::UNKNOWN; + TLabels Labels; + THistogramBuilder HistogramBuilder; + TSummaryDoubleBuilder SummaryBuilder; + TLogHistogramBuilder LogHistBuilder; + TTypedPoint LastPoint; + TVector<TTypedPoint> TimeSeries; + + bool SeenTsOrValue = false; + bool SeenTimeseries = false; + + void Clear() { + Type = EMetricType::UNKNOWN; + Labels.Clear(); + SeenTsOrValue = false; + SeenTimeseries = false; + TimeSeries.clear(); + LastPoint = {}; + HistogramBuilder.Clear(); + SummaryBuilder.Clear(); + LogHistBuilder.Clear(); + } + + void AddLabel(const TLabel& label) { + Labels.Add(label.Name(), label.Value()); + } + + void SetLastTime(TInstant time) { + LastPoint.SetTime(time); + } + + template <typename T> + void SetLastValue(T value) { + LastPoint.SetValue(value); + } + + void SaveLastPoint() { + DECODE_ENSURE(LastPoint.GetTime() != TInstant::Zero(), + "cannot add point without or zero timestamp"); + if (!HistogramBuilder.Empty()) { + auto histogram = HistogramBuilder.Build(); + TimeSeries.emplace_back(LastPoint.GetTime(), histogram.Get()); + } else if (!SummaryBuilder.Empty()) { + auto summary = SummaryBuilder.Build(); + TimeSeries.emplace_back(LastPoint.GetTime(), summary.Get()); + } else if (!LogHistBuilder.Empty()) { + auto logHist = LogHistBuilder.Build(); + TimeSeries.emplace_back(LastPoint.GetTime(), logHist.Get()); + } else { + TimeSeries.push_back(std::move(LastPoint)); + } + } + + template <typename TConsumer> + void Consume(TConsumer&& consumer) { + if (TimeSeries.empty()) { + const auto& p = LastPoint; + consumer(p.GetTime(), p.GetValueType(), p.GetValue()); + } else { + for (const auto& p: TimeSeries) { + consumer(p.GetTime(), p.GetValueType(), p.GetValue()); + } + } + } +}; + +struct TCommonParts { + TInstant CommonTime; + TLabels CommonLabels; +}; + +class IHaltableMetricConsumer: public IMetricConsumer { +public: + virtual bool NeedToStop() const = 0; +}; + +// TODO(ivanzhukov@): check all states for cases when a json document is invalid +// e.g. "metrics" or "commonLabels" keys are specified multiple times +class TCommonPartsCollector: public IHaltableMetricConsumer { +public: + TCommonParts&& CommonParts() { + return std::move(CommonParts_); + } + +private: + bool NeedToStop() const override { + return TInstant::Zero() != CommonParts_.CommonTime && !CommonParts_.CommonLabels.Empty(); + } + + void OnStreamBegin() override { + } + + void OnStreamEnd() override { + } + + void OnCommonTime(TInstant time) override { + CommonParts_.CommonTime = time; + } + + void OnMetricBegin(EMetricType) override { + IsMetric_ = true; + } + + void OnMetricEnd() override { + IsMetric_ = false; + } + + void OnLabelsBegin() override { + } + + void OnLabelsEnd() override { + } + + void OnLabel(TStringBuf name, TStringBuf value) override { + if (!IsMetric_) { + CommonParts_.CommonLabels.Add(std::move(name), std::move(value)); + } + } + + void OnDouble(TInstant, double) override { + } + + void OnInt64(TInstant, i64) override { + } + + void OnUint64(TInstant, ui64) override { + } + + void OnHistogram(TInstant, IHistogramSnapshotPtr) override { + } + + void OnLogHistogram(TInstant, TLogHistogramSnapshotPtr) override { + } + + void OnSummaryDouble(TInstant, ISummaryDoubleSnapshotPtr) override { + } + +private: + TCommonParts CommonParts_; + bool IsMetric_{false}; +}; + +class TCommonPartsProxy: public IHaltableMetricConsumer { +public: + TCommonPartsProxy(TCommonParts&& commonParts, IMetricConsumer* c) + : CommonParts_{std::move(commonParts)} + , Consumer_{c} + {} + +private: + bool NeedToStop() const override { + return false; + } + + void OnStreamBegin() override { + Consumer_->OnStreamBegin(); + + if (!CommonParts_.CommonLabels.Empty()) { + Consumer_->OnLabelsBegin(); + + for (auto&& label : CommonParts_.CommonLabels) { + Consumer_->OnLabel(label.Name(), label.Value()); + } + + Consumer_->OnLabelsEnd(); + } + + if (TInstant::Zero() != CommonParts_.CommonTime) { + Consumer_->OnCommonTime(CommonParts_.CommonTime); + } + } + + void OnStreamEnd() override { + Consumer_->OnStreamEnd(); + } + + void OnCommonTime(TInstant) override { + } + + void OnMetricBegin(EMetricType type) override { + IsMetric_ = true; + + Consumer_->OnMetricBegin(type); + } + + void OnMetricEnd() override { + IsMetric_ = false; + + Consumer_->OnMetricEnd(); + } + + void OnLabelsBegin() override { + if (IsMetric_) { + Consumer_->OnLabelsBegin(); + } + } + + void OnLabelsEnd() override { + if (IsMetric_) { + Consumer_->OnLabelsEnd(); + } + } + + void OnLabel(TStringBuf name, TStringBuf value) override { + if (IsMetric_) { + Consumer_->OnLabel(std::move(name), std::move(value)); + } + } + + void OnDouble(TInstant time, double value) override { + Consumer_->OnDouble(time, value); + } + + void OnInt64(TInstant time, i64 value) override { + Consumer_->OnInt64(time, value); + } + + void OnUint64(TInstant time, ui64 value) override { + Consumer_->OnUint64(time, value); + } + + void OnHistogram(TInstant time, IHistogramSnapshotPtr snapshot) override { + Consumer_->OnHistogram(time, std::move(snapshot)); + } + + void OnLogHistogram(TInstant time, TLogHistogramSnapshotPtr snapshot) override { + Consumer_->OnLogHistogram(time, std::move(snapshot)); + } + + void OnSummaryDouble(TInstant time, ISummaryDoubleSnapshotPtr snapshot) override { + Consumer_->OnSummaryDouble(time, std::move(snapshot)); + } + +private: + const TCommonParts CommonParts_; + IMetricConsumer* Consumer_; + bool IsMetric_{false}; +}; + +/////////////////////////////////////////////////////////////////////// +// TDecoderJson +/////////////////////////////////////////////////////////////////////// +class TDecoderJson final: public NJson::TJsonCallbacks { + struct TState { + enum EState { + ROOT_OBJECT = 0x01, + + COMMON_LABELS, + COMMON_TS, + METRICS_ARRAY, + + METRIC_OBJECT, + METRIC_NAME, + METRIC_LABELS, + METRIC_TYPE, + METRIC_MODE, // TODO: must be deleted + METRIC_TIMESERIES, + METRIC_TS, + METRIC_VALUE, + METRIC_HIST, + METRIC_HIST_BOUNDS, + METRIC_HIST_BUCKETS, + METRIC_HIST_INF, + METRIC_DSUMMARY, + METRIC_DSUMMARY_SUM, + METRIC_DSUMMARY_MIN, + METRIC_DSUMMARY_MAX, + METRIC_DSUMMARY_LAST, + METRIC_DSUMMARY_COUNT, + METRIC_LOG_HIST, + METRIC_LOG_HIST_BASE, + METRIC_LOG_HIST_ZEROS, + METRIC_LOG_HIST_START_POWER, + METRIC_LOG_HIST_BUCKETS, + }; + + constexpr EState Current() const noexcept { + return static_cast<EState>(State_ & 0xFF); + } + + void ToNext(EState state) noexcept { + constexpr auto bitSize = 8 * sizeof(ui8); + State_ = (State_ << bitSize) | static_cast<ui8>(state); + } + + void ToPrev() noexcept { + constexpr auto bitSize = 8 * sizeof(ui8); + State_ = State_ >> bitSize; + } + + private: + ui64 State_ = static_cast<ui64>(ROOT_OBJECT); + }; + +public: + TDecoderJson(TStringBuf data, IHaltableMetricConsumer* metricConsumer, TStringBuf metricNameLabel) + : Data_(data) + , MetricConsumer_(metricConsumer) + , MetricNameLabel_(metricNameLabel) + { + } + +private: +#define PARSE_ENSURE(CONDITION, ...) \ +do { \ +if (Y_UNLIKELY(!(CONDITION))) { \ + ErrorMsg_ = TStringBuilder() << __VA_ARGS__; \ + return false; \ +} \ +} while (false) + + bool OnInteger(long long value) override { + switch (State_.Current()) { + case TState::COMMON_TS: + PARSE_ENSURE(value >= 0, "unexpected negative number in a common timestamp: " << value); + MetricConsumer_->OnCommonTime(TInstant::Seconds(value)); + State_.ToPrev(); + + if (MetricConsumer_->NeedToStop()) { + IsIntentionallyHalted_ = true; + return false; + } + + break; + + case TState::METRIC_TS: + PARSE_ENSURE(value >= 0, "unexpected negative number in a metric timestamp: " << value); + LastMetric_.SetLastTime(TInstant::Seconds(value)); + State_.ToPrev(); + break; + + case TState::METRIC_VALUE: + LastMetric_.SetLastValue(static_cast<i64>(value)); + State_.ToPrev(); + break; + + case TState::METRIC_HIST_BOUNDS: + LastMetric_.HistogramBuilder.AddBound(static_cast<double>(value)); + break; + + case TState::METRIC_HIST_BUCKETS: + PARSE_ENSURE(value >= 0 && static_cast<ui64>(value) <= Max<TBucketValues::value_type>(), "value is out of bounds " << value); + LastMetric_.HistogramBuilder.AddValue(value); + break; + + case TState::METRIC_HIST_INF: + PARSE_ENSURE(value >= 0, "unexpected negative number in histogram inf: " << value); + LastMetric_.HistogramBuilder.AddInf(value); + State_.ToPrev(); + break; + + case TState::METRIC_DSUMMARY_COUNT: + LastMetric_.SummaryBuilder.SetCount(value); + State_.ToPrev(); + break; + + case TState::METRIC_DSUMMARY_SUM: + LastMetric_.SummaryBuilder.SetSum(value); + State_.ToPrev(); + break; + case TState::METRIC_DSUMMARY_MIN: + LastMetric_.SummaryBuilder.SetMin(value); + State_.ToPrev(); + break; + case TState::METRIC_DSUMMARY_MAX: + LastMetric_.SummaryBuilder.SetMax(value); + State_.ToPrev(); + break; + case TState::METRIC_DSUMMARY_LAST: + LastMetric_.SummaryBuilder.SetLast(value); + State_.ToPrev(); + break; + + case TState::METRIC_LOG_HIST_BASE: + LastMetric_.LogHistBuilder.SetBase(value); + State_.ToPrev(); + break; + + case TState::METRIC_LOG_HIST_ZEROS: + LastMetric_.LogHistBuilder.SetZerosCount(value); + State_.ToPrev(); + break; + + case TState::METRIC_LOG_HIST_START_POWER: + LastMetric_.LogHistBuilder.SetStartPower(value); + State_.ToPrev(); + break; + + case TState::METRIC_LOG_HIST_BUCKETS: + LastMetric_.LogHistBuilder.AddBucketValue(value); + break; + + default: + return false; + } + return true; + } + + bool OnUInteger(unsigned long long value) override { + switch (State_.Current()) { + case TState::COMMON_TS: + MetricConsumer_->OnCommonTime(TInstant::Seconds(value)); + State_.ToPrev(); + + if (MetricConsumer_->NeedToStop()) { + IsIntentionallyHalted_ = true; + return false; + } + + break; + + case TState::METRIC_TS: + LastMetric_.SetLastTime(TInstant::Seconds(value)); + State_.ToPrev(); + break; + + case TState::METRIC_VALUE: + PARSE_ENSURE(value <= Max<ui64>(), "Metric value is out of bounds: " << value); + LastMetric_.SetLastValue(static_cast<ui64>(value)); + State_.ToPrev(); + break; + + case TState::METRIC_HIST_BOUNDS: + LastMetric_.HistogramBuilder.AddBound(static_cast<double>(value)); + break; + + case TState::METRIC_HIST_BUCKETS: + PARSE_ENSURE(value <= Max<TBucketValues::value_type>(), "Histogram bucket value is out of bounds: " << value); + LastMetric_.HistogramBuilder.AddValue(value); + break; + + case TState::METRIC_HIST_INF: + LastMetric_.HistogramBuilder.AddInf(value); + State_.ToPrev(); + break; + + case TState::METRIC_DSUMMARY_COUNT: + LastMetric_.SummaryBuilder.SetCount(value); + State_.ToPrev(); + break; + + case TState::METRIC_DSUMMARY_SUM: + LastMetric_.SummaryBuilder.SetSum(value); + State_.ToPrev(); + break; + case TState::METRIC_DSUMMARY_MIN: + LastMetric_.SummaryBuilder.SetMin(value); + State_.ToPrev(); + break; + case TState::METRIC_DSUMMARY_MAX: + LastMetric_.SummaryBuilder.SetMax(value); + State_.ToPrev(); + break; + case TState::METRIC_DSUMMARY_LAST: + LastMetric_.SummaryBuilder.SetLast(value); + State_.ToPrev(); + break; + + case TState::METRIC_LOG_HIST_BASE: + LastMetric_.LogHistBuilder.SetBase(value); + State_.ToPrev(); + break; + + case TState::METRIC_LOG_HIST_ZEROS: + LastMetric_.LogHistBuilder.SetZerosCount(value); + State_.ToPrev(); + break; + + case TState::METRIC_LOG_HIST_START_POWER: + LastMetric_.LogHistBuilder.SetStartPower(value); + State_.ToPrev(); + break; + + case TState::METRIC_LOG_HIST_BUCKETS: + LastMetric_.LogHistBuilder.AddBucketValue(value); + break; + + default: + return false; + } + return true; + } + + bool OnDouble(double value) override { + switch (State_.Current()) { + case TState::METRIC_VALUE: + LastMetric_.SetLastValue(value); + State_.ToPrev(); + break; + + case TState::METRIC_HIST_BOUNDS: + LastMetric_.HistogramBuilder.AddBound(value); + break; + + case TState::METRIC_DSUMMARY_SUM: + LastMetric_.SummaryBuilder.SetSum(value); + State_.ToPrev(); + break; + case TState::METRIC_DSUMMARY_MIN: + LastMetric_.SummaryBuilder.SetMin(value); + State_.ToPrev(); + break; + case TState::METRIC_DSUMMARY_MAX: + LastMetric_.SummaryBuilder.SetMax(value); + State_.ToPrev(); + break; + case TState::METRIC_DSUMMARY_LAST: + LastMetric_.SummaryBuilder.SetLast(value); + State_.ToPrev(); + break; + + case TState::METRIC_LOG_HIST_BASE: + LastMetric_.LogHistBuilder.SetBase(value); + State_.ToPrev(); + break; + + case TState::METRIC_LOG_HIST_BUCKETS: + LastMetric_.LogHistBuilder.AddBucketValue(value); + break; + + default: + return false; + } + return true; + } + + bool OnString(const TStringBuf& value) override { + switch (State_.Current()) { + case TState::COMMON_LABELS: + PARSE_ENSURE(!LastLabelName_.empty(), "empty label name in common labels"); + MetricConsumer_->OnLabel(LastLabelName_, TString{value}); + break; + + case TState::METRIC_LABELS: + PARSE_ENSURE(!LastLabelName_.empty(), "empty label name in metric labels"); + LastMetric_.Labels.Add(LastLabelName_, TString{value}); + break; + + case TState::METRIC_NAME: + PARSE_ENSURE(!value.empty(), "empty metric name"); + LastMetric_.Labels.Add(MetricNameLabel_, TString{value}); + State_.ToPrev(); + break; + + case TState::COMMON_TS: + MetricConsumer_->OnCommonTime(TInstant::ParseIso8601(value)); + State_.ToPrev(); + + if (MetricConsumer_->NeedToStop()) { + IsIntentionallyHalted_ = true; + return false; + } + + break; + + case TState::METRIC_TS: + LastMetric_.SetLastTime(TInstant::ParseIso8601(value)); + State_.ToPrev(); + break; + + case TState::METRIC_VALUE: + if (auto [doubleValue, ok] = ParseSpecDouble(value); ok) { + LastMetric_.SetLastValue(doubleValue); + } else { + return false; + } + State_.ToPrev(); + break; + + case TState::METRIC_TYPE: + LastMetric_.Type = MetricTypeFromStr(value); + State_.ToPrev(); + break; + + case TState::METRIC_MODE: + if (value == TStringBuf("deriv")) { + LastMetric_.Type = EMetricType::RATE; + } + State_.ToPrev(); + break; + + case TState::METRIC_DSUMMARY_SUM: + if (auto [doubleValue, ok] = ParseSpecDouble(value); ok) { + LastMetric_.SummaryBuilder.SetSum(doubleValue); + } else { + return false; + } + State_.ToPrev(); + break; + + case TState::METRIC_DSUMMARY_MIN: + if (auto [doubleValue, ok] = ParseSpecDouble(value); ok) { + LastMetric_.SummaryBuilder.SetMin(doubleValue); + } else { + return false; + } + State_.ToPrev(); + break; + + case TState::METRIC_DSUMMARY_MAX: + if (auto [doubleValue, ok] = ParseSpecDouble(value); ok) { + LastMetric_.SummaryBuilder.SetMax(doubleValue); + } else { + return false; + } + State_.ToPrev(); + break; + + case TState::METRIC_DSUMMARY_LAST: + if (auto [doubleValue, ok] = ParseSpecDouble(value); ok) { + LastMetric_.SummaryBuilder.SetLast(doubleValue); + } else { + return false; + } + State_.ToPrev(); + break; + + default: + return false; + } + + return true; + } + + bool OnMapKey(const TStringBuf& key) override { + switch (State_.Current()) { + case TState::ROOT_OBJECT: + if (key == TStringBuf("commonLabels") || key == TStringBuf("labels")) { + State_.ToNext(TState::COMMON_LABELS); + } else if (key == TStringBuf("ts")) { + State_.ToNext(TState::COMMON_TS); + } else if (key == TStringBuf("sensors") || key == TStringBuf("metrics")) { + State_.ToNext(TState::METRICS_ARRAY); + } + break; + + case TState::COMMON_LABELS: + case TState::METRIC_LABELS: + LastLabelName_ = key; + break; + + case TState::METRIC_OBJECT: + if (key == TStringBuf("labels")) { + State_.ToNext(TState::METRIC_LABELS); + } else if (key == TStringBuf("name")) { + State_.ToNext(TState::METRIC_NAME); + } else if (key == TStringBuf("ts")) { + PARSE_ENSURE(!LastMetric_.SeenTimeseries, + "mixed timeseries and ts attributes"); + LastMetric_.SeenTsOrValue = true; + State_.ToNext(TState::METRIC_TS); + } else if (key == TStringBuf("value")) { + PARSE_ENSURE(!LastMetric_.SeenTimeseries, + "mixed timeseries and value attributes"); + LastMetric_.SeenTsOrValue = true; + State_.ToNext(TState::METRIC_VALUE); + } else if (key == TStringBuf("timeseries")) { + PARSE_ENSURE(!LastMetric_.SeenTsOrValue, + "mixed timeseries and ts/value attributes"); + LastMetric_.SeenTimeseries = true; + State_.ToNext(TState::METRIC_TIMESERIES); + } else if (key == TStringBuf("mode")) { + State_.ToNext(TState::METRIC_MODE); + } else if (key == TStringBuf("kind") || key == TStringBuf("type")) { + State_.ToNext(TState::METRIC_TYPE); + } else if (key == TStringBuf("hist")) { + State_.ToNext(TState::METRIC_HIST); + } else if (key == TStringBuf("summary")) { + State_.ToNext(TState::METRIC_DSUMMARY); + } else if (key == TStringBuf("log_hist")) { + State_.ToNext(TState::METRIC_LOG_HIST); + } else if (key == TStringBuf("memOnly")) { + // deprecated. Skip it without errors for backward compatibility + } else { + ErrorMsg_ = TStringBuilder() << "unexpected key \"" << key << "\" in a metric schema"; + return false; + } + break; + + case TState::METRIC_TIMESERIES: + if (key == TStringBuf("ts")) { + State_.ToNext(TState::METRIC_TS); + } else if (key == TStringBuf("value")) { + State_.ToNext(TState::METRIC_VALUE); + } else if (key == TStringBuf("hist")) { + State_.ToNext(TState::METRIC_HIST); + } else if (key == TStringBuf("summary")) { + State_.ToNext(TState::METRIC_DSUMMARY); + } else if (key == TStringBuf("log_hist")) { + State_.ToNext(TState::METRIC_LOG_HIST); + } + break; + + case TState::METRIC_HIST: + if (key == TStringBuf("bounds")) { + State_.ToNext(TState::METRIC_HIST_BOUNDS); + } else if (key == TStringBuf("buckets")) { + State_.ToNext(TState::METRIC_HIST_BUCKETS); + } else if (key == TStringBuf("inf")) { + State_.ToNext(TState::METRIC_HIST_INF); + } + break; + + case TState::METRIC_LOG_HIST: + if (key == TStringBuf("base")) { + State_.ToNext(TState::METRIC_LOG_HIST_BASE); + } else if (key == TStringBuf("zeros_count")) { + State_.ToNext(TState::METRIC_LOG_HIST_ZEROS); + } else if (key == TStringBuf("start_power")) { + State_.ToNext(TState::METRIC_LOG_HIST_START_POWER); + } else if (key == TStringBuf("buckets")) { + State_.ToNext(TState::METRIC_LOG_HIST_BUCKETS); + } + break; + + case TState::METRIC_DSUMMARY: + if (key == TStringBuf("sum")) { + State_.ToNext(TState::METRIC_DSUMMARY_SUM); + } else if (key == TStringBuf("min")) { + State_.ToNext(TState::METRIC_DSUMMARY_MIN); + } else if (key == TStringBuf("max")) { + State_.ToNext(TState::METRIC_DSUMMARY_MAX); + } else if (key == TStringBuf("last")) { + State_.ToNext(TState::METRIC_DSUMMARY_LAST); + } else if (key == TStringBuf("count")) { + State_.ToNext(TState::METRIC_DSUMMARY_COUNT); + } + + break; + + + default: + return false; + } + + return true; + } + + bool OnOpenMap() override { + switch (State_.Current()) { + case TState::ROOT_OBJECT: + MetricConsumer_->OnStreamBegin(); + break; + + case TState::COMMON_LABELS: + MetricConsumer_->OnLabelsBegin(); + break; + + case TState::METRICS_ARRAY: + State_.ToNext(TState::METRIC_OBJECT); + LastMetric_.Clear(); + break; + + default: + break; + } + return true; + } + + bool OnCloseMap() override { + switch (State_.Current()) { + case TState::ROOT_OBJECT: + MetricConsumer_->OnStreamEnd(); + break; + + case TState::METRIC_LABELS: + State_.ToPrev(); + break; + + case TState::COMMON_LABELS: + MetricConsumer_->OnLabelsEnd(); + State_.ToPrev(); + + if (MetricConsumer_->NeedToStop()) { + IsIntentionallyHalted_ = true; + return false; + } + + break; + + case TState::METRIC_OBJECT: + ConsumeMetric(); + State_.ToPrev(); + break; + + case TState::METRIC_TIMESERIES: + LastMetric_.SaveLastPoint(); + break; + + case TState::METRIC_HIST: + case TState::METRIC_DSUMMARY: + case TState::METRIC_LOG_HIST: + State_.ToPrev(); + break; + + default: + break; + } + return true; + } + + bool OnOpenArray() override { + auto currentState = State_.Current(); + PARSE_ENSURE( + currentState == TState::METRICS_ARRAY || + currentState == TState::METRIC_TIMESERIES || + currentState == TState::METRIC_HIST_BOUNDS || + currentState == TState::METRIC_HIST_BUCKETS || + currentState == TState::METRIC_LOG_HIST_BUCKETS, + "unexpected array begin"); + return true; + } + + bool OnCloseArray() override { + switch (State_.Current()) { + case TState::METRICS_ARRAY: + case TState::METRIC_TIMESERIES: + case TState::METRIC_HIST_BOUNDS: + case TState::METRIC_HIST_BUCKETS: + case TState::METRIC_LOG_HIST_BUCKETS: + State_.ToPrev(); + break; + + default: + return false; + } + return true; + } + + void OnError(size_t off, TStringBuf reason) override { + if (IsIntentionallyHalted_) { + return; + } + + size_t snippetBeg = (off < 20) ? 0 : (off - 20); + TStringBuf snippet = Data_.SubStr(snippetBeg, 40); + + throw TJsonDecodeError() + << "cannot parse JSON, error at: " << off + << ", reason: " << (ErrorMsg_.empty() ? reason : TStringBuf{ErrorMsg_}) + << "\nsnippet: ..." << snippet << "..."; + } + + bool OnEnd() override { + return true; + } + + void ConsumeMetric() { + // for backwad compatibility all unknown metrics treated as gauges + if (LastMetric_.Type == EMetricType::UNKNOWN) { + if (LastMetric_.HistogramBuilder.Empty()) { + LastMetric_.Type = EMetricType::GAUGE; + } else { + LastMetric_.Type = EMetricType::HIST; + } + } + + // (1) begin metric + MetricConsumer_->OnMetricBegin(LastMetric_.Type); + + // (2) labels + if (!LastMetric_.Labels.empty()) { + MetricConsumer_->OnLabelsBegin(); + for (auto&& label : LastMetric_.Labels) { + MetricConsumer_->OnLabel(label.Name(), label.Value()); + } + MetricConsumer_->OnLabelsEnd(); + } + + // (3) values + switch (LastMetric_.Type) { + case EMetricType::GAUGE: + LastMetric_.Consume([this](TInstant time, EMetricValueType valueType, TMetricValue value) { + MetricConsumer_->OnDouble(time, value.AsDouble(valueType)); + }); + break; + + case EMetricType::IGAUGE: + LastMetric_.Consume([this](TInstant time, EMetricValueType valueType, TMetricValue value) { + MetricConsumer_->OnInt64(time, value.AsInt64(valueType)); + }); + break; + + case EMetricType::COUNTER: + case EMetricType::RATE: + LastMetric_.Consume([this](TInstant time, EMetricValueType valueType, TMetricValue value) { + MetricConsumer_->OnUint64(time, value.AsUint64(valueType)); + }); + break; + + case EMetricType::HIST: + case EMetricType::HIST_RATE: + if (LastMetric_.TimeSeries.empty()) { + auto time = LastMetric_.LastPoint.GetTime(); + auto histogram = LastMetric_.HistogramBuilder.Build(); + MetricConsumer_->OnHistogram(time, histogram); + } else { + for (const auto& p : LastMetric_.TimeSeries) { + DECODE_ENSURE(p.GetValueType() == EMetricValueType::HISTOGRAM, "Value is not a histogram"); + MetricConsumer_->OnHistogram(p.GetTime(), p.GetValue().AsHistogram()); + } + } + break; + + case EMetricType::DSUMMARY: + if (LastMetric_.TimeSeries.empty()) { + auto time = LastMetric_.LastPoint.GetTime(); + auto summary = LastMetric_.SummaryBuilder.Build(); + MetricConsumer_->OnSummaryDouble(time, summary); + } else { + for (const auto& p : LastMetric_.TimeSeries) { + DECODE_ENSURE(p.GetValueType() == EMetricValueType::SUMMARY, "Value is not a summary"); + MetricConsumer_->OnSummaryDouble(p.GetTime(), p.GetValue().AsSummaryDouble()); + } + } + break; + + case EMetricType::LOGHIST: + if (LastMetric_.TimeSeries.empty()) { + auto time = LastMetric_.LastPoint.GetTime(); + auto logHist = LastMetric_.LogHistBuilder.Build(); + MetricConsumer_->OnLogHistogram(time, logHist); + } else { + for (const auto& p : LastMetric_.TimeSeries) { + DECODE_ENSURE(p.GetValueType() == EMetricValueType::LOGHISTOGRAM, "Value is not a log_histogram"); + MetricConsumer_->OnLogHistogram(p.GetTime(), p.GetValue().AsLogHistogram()); + } + } + break; + + case EMetricType::UNKNOWN: + // TODO: output metric labels + ythrow yexception() << "unknown metric type"; + } + + // (4) end metric + MetricConsumer_->OnMetricEnd(); + } + +private: + TStringBuf Data_; + IHaltableMetricConsumer* MetricConsumer_; + TString MetricNameLabel_; + TState State_; + TString LastLabelName_; + TMetricCollector LastMetric_; + TString ErrorMsg_; + bool IsIntentionallyHalted_{false}; +}; + +} // namespace + +void DecodeJson(TStringBuf data, IMetricConsumer* c, TStringBuf metricNameLabel) { + TCommonPartsCollector commonPartsCollector; + { + TMemoryInput memIn(data); + TDecoderJson decoder(data, &commonPartsCollector, metricNameLabel); + // no need to check a return value. If there is an error, a TJsonDecodeError is thrown + NJson::ReadJson(&memIn, &decoder); + } + + TCommonPartsProxy commonPartsProxy(std::move(commonPartsCollector.CommonParts()), c); + { + TMemoryInput memIn(data); + TDecoderJson decoder(data, &commonPartsProxy, metricNameLabel); + // no need to check a return value. If there is an error, a TJsonDecodeError is thrown + NJson::ReadJson(&memIn, &decoder); + } +} + +#undef DECODE_ENSURE + +} diff --git a/library/cpp/monlib/encode/json/json_decoder_ut.cpp b/library/cpp/monlib/encode/json/json_decoder_ut.cpp new file mode 100644 index 0000000000..4464e1d26a --- /dev/null +++ b/library/cpp/monlib/encode/json/json_decoder_ut.cpp @@ -0,0 +1,179 @@ +#include "json_decoder.cpp" + +#include <library/cpp/monlib/consumers/collecting_consumer.h> +#include <library/cpp/testing/unittest/registar.h> + +#include <array> + + +using namespace NMonitoring; + +enum EJsonPart : ui8 { + METRICS = 0, + COMMON_TS = 1, + COMMON_LABELS = 2, +}; + +constexpr std::array<TStringBuf, 3> JSON_PARTS = { + TStringBuf(R"("metrics": [{ + "labels": { "key": "value" }, + "type": "GAUGE", + "value": 123 + }])"), + + TStringBuf(R"("ts": 1)"), + + TStringBuf(R"("commonLabels": { + "key1": "value1", + "key2": "value2" + })"), +}; + +TString BuildJson(std::initializer_list<EJsonPart> parts) { + TString data = "{"; + + for (auto it = parts.begin(); it != parts.end(); ++it) { + data += JSON_PARTS[*it]; + + if (it + 1 != parts.end()) { + data += ","; + } + } + + data += "}"; + return data; +} + +void ValidateCommonParts(TCommonParts&& commonParts, bool checkLabels, bool checkTs) { + if (checkTs) { + UNIT_ASSERT_VALUES_EQUAL(commonParts.CommonTime.MilliSeconds(), 1000); + } + + if (checkLabels) { + auto& labels = commonParts.CommonLabels; + UNIT_ASSERT_VALUES_EQUAL(labels.Size(), 2); + UNIT_ASSERT(labels.Has(TStringBuf("key1"))); + UNIT_ASSERT(labels.Has(TStringBuf("key2"))); + UNIT_ASSERT_VALUES_EQUAL(labels.Get(TStringBuf("key1")).value()->Value(), "value1"); + UNIT_ASSERT_VALUES_EQUAL(labels.Get(TStringBuf("key2")).value()->Value(), "value2"); + } +} + +void ValidateMetrics(const TVector<TMetricData>& metrics) { + UNIT_ASSERT_VALUES_EQUAL(metrics.size(), 1); + + auto& m = metrics[0]; + UNIT_ASSERT_VALUES_EQUAL(m.Kind, EMetricType::GAUGE); + auto& l = m.Labels; + UNIT_ASSERT_VALUES_EQUAL(l.Size(), 1); + UNIT_ASSERT_VALUES_EQUAL(l.Get(0)->Name(), "key"); + UNIT_ASSERT_VALUES_EQUAL(l.Get(0)->Value(), "value"); + + UNIT_ASSERT_VALUES_EQUAL(m.Values->Size(), 1); + UNIT_ASSERT_VALUES_EQUAL((*m.Values)[0].GetValue().AsDouble(), 123); +} + +void CheckCommonPartsCollector(TString data, bool shouldBeStopped, bool checkLabels = true, bool checkTs = true, TStringBuf metricNameLabel = "name") { + TCommonPartsCollector commonPartsCollector; + TMemoryInput memIn(data); + TDecoderJson decoder(data, &commonPartsCollector, metricNameLabel); + + bool isOk{false}; + UNIT_ASSERT_NO_EXCEPTION(isOk = NJson::ReadJson(&memIn, &decoder)); + UNIT_ASSERT_VALUES_EQUAL(isOk, !shouldBeStopped); + + ValidateCommonParts(commonPartsCollector.CommonParts(), checkLabels, checkTs); +} + +Y_UNIT_TEST_SUITE(TJsonDecoderTest) { + Y_UNIT_TEST(FullCommonParts) { + CheckCommonPartsCollector(BuildJson({COMMON_LABELS, COMMON_TS, METRICS}), true); + CheckCommonPartsCollector(BuildJson({COMMON_TS, COMMON_LABELS, METRICS}), true); + + CheckCommonPartsCollector(BuildJson({METRICS, COMMON_TS, COMMON_LABELS}), true); + CheckCommonPartsCollector(BuildJson({METRICS, COMMON_LABELS, COMMON_TS}), true); + + CheckCommonPartsCollector(BuildJson({COMMON_LABELS, METRICS, COMMON_TS}), true); + CheckCommonPartsCollector(BuildJson({COMMON_TS, METRICS, COMMON_LABELS}), true); + } + + Y_UNIT_TEST(PartialCommonParts) { + CheckCommonPartsCollector(BuildJson({COMMON_TS, METRICS}), false, false, true); + CheckCommonPartsCollector(BuildJson({COMMON_LABELS, METRICS}), false, true, false); + + CheckCommonPartsCollector(BuildJson({METRICS, COMMON_LABELS}), false, true, false); + CheckCommonPartsCollector(BuildJson({METRICS, COMMON_TS}), false, false, true); + + CheckCommonPartsCollector(BuildJson({METRICS}), false, false, false); + } + + Y_UNIT_TEST(CheckCommonPartsAndMetrics) { + auto data = BuildJson({COMMON_LABELS, COMMON_TS, METRICS}); + TCollectingConsumer collector; + + DecodeJson(data, &collector); + + TCommonParts commonParts; + commonParts.CommonTime = collector.CommonTime; + commonParts.CommonLabels = collector.CommonLabels; + + ValidateCommonParts(std::move(commonParts), true, true); + ValidateMetrics(collector.Metrics); + } + + Y_UNIT_TEST(CanParseHistogramsWithInf) { + const char* metricsData = R"({ +"metrics": + [ + { + "hist": { + "bounds": [ + 10 + ], + "buckets": [ + 11 + ], + "inf": 12 + }, + "name":"s1", + "type": "HIST_RATE" + }, + { + "hist": { + "bounds": [ + 20 + ], + "buckets": [ + 21 + ] + }, + "name":"s2", + "type":"HIST_RATE" + } + ] +})"; + TCollectingConsumer consumer(false); + DecodeJson(metricsData, &consumer); + + UNIT_ASSERT_VALUES_EQUAL(consumer.Metrics.size(), 2); + { + const auto& m = consumer.Metrics[0]; + UNIT_ASSERT_VALUES_EQUAL(m.Kind, EMetricType::HIST_RATE); + UNIT_ASSERT_VALUES_EQUAL(m.Values->Size(), 1); + const auto* histogram = (*m.Values)[0].GetValue().AsHistogram(); + UNIT_ASSERT_VALUES_EQUAL(histogram->Count(), 2); + UNIT_ASSERT_VALUES_EQUAL(histogram->UpperBound(1), Max<TBucketBound>()); + UNIT_ASSERT_VALUES_EQUAL(histogram->Value(0), 11); + UNIT_ASSERT_VALUES_EQUAL(histogram->Value(1), 12); + } + { + const auto& m = consumer.Metrics[1]; + UNIT_ASSERT_VALUES_EQUAL(m.Kind, EMetricType::HIST_RATE); + UNIT_ASSERT_VALUES_EQUAL(m.Values->Size(), 1); + const auto* histogram = (*m.Values)[0].GetValue().AsHistogram(); + UNIT_ASSERT_VALUES_EQUAL(histogram->Count(), 1); + UNIT_ASSERT_VALUES_EQUAL(histogram->UpperBound(0), 20); + UNIT_ASSERT_VALUES_EQUAL(histogram->Value(0), 21); + } + } +} diff --git a/library/cpp/monlib/encode/json/json_encoder.cpp b/library/cpp/monlib/encode/json/json_encoder.cpp new file mode 100644 index 0000000000..20d2bb6283 --- /dev/null +++ b/library/cpp/monlib/encode/json/json_encoder.cpp @@ -0,0 +1,556 @@ +#include "json.h" +#include "typed_point.h" + +#include <library/cpp/monlib/encode/buffered/buffered_encoder_base.h> +#include <library/cpp/monlib/encode/encoder_state.h> +#include <library/cpp/monlib/metrics/metric.h> +#include <library/cpp/monlib/metrics/metric_value.h> +#include <library/cpp/monlib/metrics/labels.h> + +#include <library/cpp/json/writer/json.h> + +#include <util/charset/utf8.h> +#include <util/generic/algorithm.h> + +namespace NMonitoring { + namespace { + enum class EJsonStyle { + Solomon, + Cloud + }; + + /////////////////////////////////////////////////////////////////////// + // TJsonWriter + /////////////////////////////////////////////////////////////////////// + class TJsonWriter { + public: + TJsonWriter(IOutputStream* out, int indentation, EJsonStyle style, TStringBuf metricNameLabel) + : Buf_(NJsonWriter::HEM_UNSAFE, out) + , Style_(style) + , MetricNameLabel_(metricNameLabel) + , CurrentMetricName_() + { + Buf_.SetIndentSpaces(indentation); + Buf_.SetWriteNanAsString(); + } + + void WriteTime(TInstant time) { + if (time != TInstant::Zero()) { + Buf_.WriteKey(TStringBuf("ts")); + if (Style_ == EJsonStyle::Solomon) { + Buf_.WriteULongLong(time.Seconds()); + } else { + Buf_.WriteString(time.ToString()); + } + } + } + + void WriteValue(double value) { + Buf_.WriteKey(TStringBuf("value")); + Buf_.WriteDouble(value); + } + + void WriteValue(i64 value) { + Buf_.WriteKey(TStringBuf("value")); + Buf_.WriteLongLong(value); + } + + void WriteValue(ui64 value) { + Buf_.WriteKey(TStringBuf("value")); + Buf_.WriteULongLong(value); + } + + void WriteValue(IHistogramSnapshot* s) { + Y_ENSURE(Style_ == EJsonStyle::Solomon); + + Buf_.WriteKey(TStringBuf("hist")); + Buf_.BeginObject(); + if (ui32 count = s->Count()) { + bool hasInf = (s->UpperBound(count - 1) == Max<double>()); + if (hasInf) { + count--; + } + + Buf_.WriteKey(TStringBuf("bounds")); + Buf_.BeginList(); + for (ui32 i = 0; i < count; i++) { + Buf_.WriteDouble(s->UpperBound(i)); + } + Buf_.EndList(); + + Buf_.WriteKey(TStringBuf("buckets")); + Buf_.BeginList(); + for (ui32 i = 0; i < count; i++) { + Buf_.WriteULongLong(s->Value(i)); + } + Buf_.EndList(); + + if (hasInf) { + Buf_.WriteKey(TStringBuf("inf")); + Buf_.WriteULongLong(s->Value(count)); + } + } + Buf_.EndObject(); + } + + void WriteValue(ISummaryDoubleSnapshot* s) { + Y_ENSURE(Style_ == EJsonStyle::Solomon); + + Buf_.WriteKey(TStringBuf("summary")); + Buf_.BeginObject(); + + Buf_.WriteKey(TStringBuf("sum")); + Buf_.WriteDouble(s->GetSum()); + + Buf_.WriteKey(TStringBuf("min")); + Buf_.WriteDouble(s->GetMin()); + + Buf_.WriteKey(TStringBuf("max")); + Buf_.WriteDouble(s->GetMax()); + + Buf_.WriteKey(TStringBuf("last")); + Buf_.WriteDouble(s->GetLast()); + + Buf_.WriteKey(TStringBuf("count")); + Buf_.WriteULongLong(s->GetCount()); + + Buf_.EndObject(); + } + + void WriteValue(TLogHistogramSnapshot* s) { + Y_ENSURE(Style_ == EJsonStyle::Solomon); + + Buf_.WriteKey(TStringBuf("log_hist")); + Buf_.BeginObject(); + + Buf_.WriteKey(TStringBuf("base")); + Buf_.WriteDouble(s->Base()); + + Buf_.WriteKey(TStringBuf("zeros_count")); + Buf_.WriteULongLong(s->ZerosCount()); + + Buf_.WriteKey(TStringBuf("start_power")); + Buf_.WriteInt(s->StartPower()); + + Buf_.WriteKey(TStringBuf("buckets")); + Buf_.BeginList(); + for (size_t i = 0; i < s->Count(); ++i) { + Buf_.WriteDouble(s->Bucket(i)); + } + Buf_.EndList(); + + Buf_.EndObject(); + } + + void WriteValue(EMetricValueType type, TMetricValue value) { + switch (type) { + case EMetricValueType::DOUBLE: + WriteValue(value.AsDouble()); + break; + + case EMetricValueType::INT64: + WriteValue(value.AsInt64()); + break; + + case EMetricValueType::UINT64: + WriteValue(value.AsUint64()); + break; + + case EMetricValueType::HISTOGRAM: + WriteValue(value.AsHistogram()); + break; + + case EMetricValueType::SUMMARY: + WriteValue(value.AsSummaryDouble()); + break; + + case EMetricValueType::LOGHISTOGRAM: + WriteValue(value.AsLogHistogram()); + break; + + case EMetricValueType::UNKNOWN: + ythrow yexception() << "unknown metric value type"; + } + } + + void WriteLabel(TStringBuf name, TStringBuf value) { + Y_ENSURE(IsUtf(name), "label name is not valid UTF-8 string"); + Y_ENSURE(IsUtf(value), "label value is not valid UTF-8 string"); + if (Style_ == EJsonStyle::Cloud && name == MetricNameLabel_) { + CurrentMetricName_ = value; + } else { + Buf_.WriteKey(name); + Buf_.WriteString(value); + } + } + + void WriteMetricType(EMetricType type) { + if (Style_ == EJsonStyle::Cloud) { + Buf_.WriteKey("type"); + Buf_.WriteString(MetricTypeToCloudStr(type)); + } else { + Buf_.WriteKey("kind"); + Buf_.WriteString(MetricTypeToStr(type)); + } + } + + void WriteName() { + if (Style_ != EJsonStyle::Cloud) { + return; + } + if (CurrentMetricName_.Empty()) { + ythrow yexception() << "label '" << MetricNameLabel_ << "' is not defined"; + } + Buf_.WriteKey("name"); + Buf_.WriteString(CurrentMetricName_); + CurrentMetricName_.clear(); + } + + private: + static TStringBuf MetricTypeToCloudStr(EMetricType type) { + switch (type) { + case EMetricType::GAUGE: + return TStringBuf("DGAUGE"); + case EMetricType::COUNTER: + return TStringBuf("COUNTER"); + case EMetricType::RATE: + return TStringBuf("RATE"); + case EMetricType::IGAUGE: + return TStringBuf("IGAUGE"); + default: + ythrow yexception() << "metric type '" << type << "' is not supported by cloud json format"; + } + } + + protected: + NJsonWriter::TBuf Buf_; + EJsonStyle Style_; + TString MetricNameLabel_; + TString CurrentMetricName_; + }; + + /////////////////////////////////////////////////////////////////////// + // TEncoderJson + /////////////////////////////////////////////////////////////////////// + class TEncoderJson final: public IMetricEncoder, public TJsonWriter { + public: + TEncoderJson(IOutputStream* out, int indentation, EJsonStyle style, TStringBuf metricNameLabel) + : TJsonWriter{out, indentation, style, metricNameLabel} + { + } + + ~TEncoderJson() override { + Close(); + } + + private: + void OnStreamBegin() override { + State_.Expect(TEncoderState::EState::ROOT); + Buf_.BeginObject(); + } + + void OnStreamEnd() override { + State_.Expect(TEncoderState::EState::ROOT); + if (!Buf_.KeyExpected()) { + // not closed metrics array + Buf_.EndList(); + } + Buf_.EndObject(); + } + + void OnCommonTime(TInstant time) override { + State_.Expect(TEncoderState::EState::ROOT); + WriteTime(time); + } + + void OnMetricBegin(EMetricType type) override { + State_.Switch(TEncoderState::EState::ROOT, TEncoderState::EState::METRIC); + if (Buf_.KeyExpected()) { + // first metric, so open metrics array + Buf_.WriteKey(TStringBuf(Style_ == EJsonStyle::Solomon ? "sensors" : "metrics")); + Buf_.BeginList(); + } + Buf_.BeginObject(); + WriteMetricType(type); + } + + void OnMetricEnd() override { + State_.Switch(TEncoderState::EState::METRIC, TEncoderState::EState::ROOT); + if (!Buf_.KeyExpected()) { + // not closed timeseries array + Buf_.EndList(); + } + + if (!TimeSeries_ && LastPoint_.HasValue()) { + // we have seen only one point between OnMetricBegin() and + // OnMetricEnd() calls + WriteTime(LastPoint_.GetTime()); + WriteValue(LastPoint_.GetValueType(), LastPoint_.GetValue()); + } + Buf_.EndObject(); + + LastPoint_ = {}; + TimeSeries_ = false; + } + + void OnLabelsBegin() override { + if (!Buf_.KeyExpected()) { + // not closed metrics or timeseries array if labels go after values + Buf_.EndList(); + } + if (State_ == TEncoderState::EState::ROOT) { + State_ = TEncoderState::EState::COMMON_LABELS; + Buf_.WriteKey(TStringBuf(Style_ == EJsonStyle::Solomon ? "commonLabels" : "labels")); + } else if (State_ == TEncoderState::EState::METRIC) { + State_ = TEncoderState::EState::METRIC_LABELS; + Buf_.WriteKey(TStringBuf("labels")); + } else { + State_.ThrowInvalid("expected METRIC or ROOT"); + } + Buf_.BeginObject(); + + EmptyLabels_ = true; + } + + void OnLabelsEnd() override { + if (State_ == TEncoderState::EState::METRIC_LABELS) { + State_ = TEncoderState::EState::METRIC; + } else if (State_ == TEncoderState::EState::COMMON_LABELS) { + State_ = TEncoderState::EState::ROOT; + } else { + State_.ThrowInvalid("expected LABELS or COMMON_LABELS"); + } + + Y_ENSURE(!EmptyLabels_, "Labels cannot be empty"); + Buf_.EndObject(); + if (State_ == TEncoderState::EState::METRIC) { + WriteName(); + } + } + + void OnLabel(TStringBuf name, TStringBuf value) override { + if (State_ == TEncoderState::EState::METRIC_LABELS || State_ == TEncoderState::EState::COMMON_LABELS) { + WriteLabel(name, value); + } else { + State_.ThrowInvalid("expected LABELS or COMMON_LABELS"); + } + + EmptyLabels_ = false; + } + + void OnDouble(TInstant time, double value) override { + State_.Expect(TEncoderState::EState::METRIC); + Write<double>(time, value); + } + + void OnInt64(TInstant time, i64 value) override { + State_.Expect(TEncoderState::EState::METRIC); + Write<i64>(time, value); + } + + void OnUint64(TInstant time, ui64 value) override { + State_.Expect(TEncoderState::EState::METRIC); + Write<ui64>(time, value); + } + + void OnHistogram(TInstant time, IHistogramSnapshotPtr snapshot) override { + State_.Expect(TEncoderState::EState::METRIC); + Write<IHistogramSnapshot*>(time, snapshot.Get()); + } + + void OnSummaryDouble(TInstant time, ISummaryDoubleSnapshotPtr snapshot) override { + State_.Expect(TEncoderState::EState::METRIC); + Write<ISummaryDoubleSnapshot*>(time, snapshot.Get()); + } + + void OnLogHistogram(TInstant time, TLogHistogramSnapshotPtr snapshot) override { + State_.Expect(TEncoderState::EState::METRIC); + Write<TLogHistogramSnapshot*>(time, snapshot.Get()); + } + + template <typename T> + void Write(TInstant time, T value) { + State_.Expect(TEncoderState::EState::METRIC); + + if (!LastPoint_.HasValue()) { + LastPoint_ = {time, value}; + } else { + // second point + // TODO: output types + Y_ENSURE(LastPoint_.GetValueType() == TValueType<T>::Type, + "mixed metric value types in one metric"); + + if (!TimeSeries_) { + Buf_.WriteKey(TStringBuf("timeseries")); + Buf_.BeginList(); + Buf_.BeginObject(); + Y_ENSURE(LastPoint_.GetTime() != TInstant::Zero(), + "time cannot be empty or zero in a timeseries point"); + WriteTime(LastPoint_.GetTime()); + WriteValue(LastPoint_.GetValueType(), LastPoint_.GetValue()); + Buf_.EndObject(); + TimeSeries_ = true; + } + + if (TimeSeries_) { + Buf_.BeginObject(); + Y_ENSURE(time != TInstant::Zero(), + "time cannot be empty or zero in a timeseries point"); + + WriteTime(time); + WriteValue(value); + Buf_.EndObject(); + } + } + } + + void Close() override { + LastPoint_ = {}; + } + + private: + TEncoderState State_; + TTypedPoint LastPoint_; + bool TimeSeries_ = false; + bool EmptyLabels_ = false; + }; + + /////////////////////////////////////////////////////////////////////// + // TBufferedJsonEncoder + /////////////////////////////////////////////////////////////////////// + class TBufferedJsonEncoder : public TBufferedEncoderBase, public TJsonWriter { + public: + TBufferedJsonEncoder(IOutputStream* out, int indentation, EJsonStyle style, TStringBuf metricNameLabel) + : TJsonWriter{out, indentation, style, metricNameLabel} + { + MetricsMergingMode_ = EMetricsMergingMode::MERGE_METRICS; + } + + ~TBufferedJsonEncoder() override { + Close(); + } + + void OnLabelsBegin() override { + TBufferedEncoderBase::OnLabelsBegin(); + EmptyLabels_ = true; + } + + void OnLabel(TStringBuf name, TStringBuf value) override { + TBufferedEncoderBase::OnLabel(name, value); + EmptyLabels_ = false; + } + + void OnLabel(ui32 name, ui32 value) override { + TBufferedEncoderBase::OnLabel(name, value); + EmptyLabels_ = false; + } + + void OnLabelsEnd() override { + TBufferedEncoderBase::OnLabelsEnd(); + Y_ENSURE(!EmptyLabels_, "Labels cannot be empty"); + } + + void Close() final { + if (Closed_) { + return; + } + + Closed_ = true; + + LabelValuesPool_.Build(); + LabelNamesPool_.Build(); + + Buf_.BeginObject(); + + WriteTime(CommonTime_); + if (CommonLabels_.size() > 0) { + Buf_.WriteKey(TStringBuf(Style_ == EJsonStyle::Solomon ? "commonLabels": "labels")); + WriteLabels(CommonLabels_, true); + } + + if (Metrics_.size() > 0) { + Buf_.WriteKey(TStringBuf(Style_ == EJsonStyle::Solomon ? "sensors" : "metrics")); + WriteMetrics(); + } + + Buf_.EndObject(); + } + + private: + void WriteMetrics() { + Buf_.BeginList(); + for (auto&& metric : Metrics_) { + WriteMetric(metric); + } + Buf_.EndList(); + } + + void WriteMetric(TMetric& metric) { + Buf_.BeginObject(); + + WriteMetricType(metric.MetricType); + + Buf_.WriteKey(TStringBuf("labels")); + WriteLabels(metric.Labels, false); + + metric.TimeSeries.SortByTs(); + if (metric.TimeSeries.Size() == 1) { + const auto& point = metric.TimeSeries[0]; + WriteTime(point.GetTime()); + WriteValue(metric.TimeSeries.GetValueType(), point.GetValue()); + } else if (metric.TimeSeries.Size() > 1) { + Buf_.WriteKey(TStringBuf("timeseries")); + Buf_.BeginList(); + metric.TimeSeries.ForEach([this](TInstant time, EMetricValueType type, TMetricValue value) { + Buf_.BeginObject(); + // make gcc 6.1 happy https://gcc.gnu.org/bugzilla/show_bug.cgi?id=61636 + this->WriteTime(time); + this->WriteValue(type, value); + Buf_.EndObject(); + }); + + Buf_.EndList(); + } + + Buf_.EndObject(); + } + + void WriteLabels(const TPooledLabels& labels, bool isCommon) { + Buf_.BeginObject(); + + for (auto i = 0u; i < labels.size(); ++i) { + TStringBuf name = LabelNamesPool_.Get(labels[i].Key->Index); + TStringBuf value = LabelValuesPool_.Get(labels[i].Value->Index); + + WriteLabel(name, value); + } + + Buf_.EndObject(); + + if (!isCommon) { + WriteName(); + } + } + + private: + bool Closed_{false}; + bool EmptyLabels_ = false; + }; + } + + IMetricEncoderPtr EncoderJson(IOutputStream* out, int indentation) { + return MakeHolder<TEncoderJson>(out, indentation, EJsonStyle::Solomon, ""); + } + + IMetricEncoderPtr BufferedEncoderJson(IOutputStream* out, int indentation) { + return MakeHolder<TBufferedJsonEncoder>(out, indentation, EJsonStyle::Solomon, ""); + } + + IMetricEncoderPtr EncoderCloudJson(IOutputStream* out, int indentation, TStringBuf metricNameLabel) { + return MakeHolder<TEncoderJson>(out, indentation, EJsonStyle::Cloud, metricNameLabel); + } + + IMetricEncoderPtr BufferedEncoderCloudJson(IOutputStream* out, int indentation, TStringBuf metricNameLabel) { + return MakeHolder<TBufferedJsonEncoder>(out, indentation, EJsonStyle::Cloud, metricNameLabel); + } +} diff --git a/library/cpp/monlib/encode/json/json_ut.cpp b/library/cpp/monlib/encode/json/json_ut.cpp new file mode 100644 index 0000000000..09e7909289 --- /dev/null +++ b/library/cpp/monlib/encode/json/json_ut.cpp @@ -0,0 +1,1290 @@ +#include "json.h" + +#include <library/cpp/monlib/encode/protobuf/protobuf.h> +#include <library/cpp/monlib/metrics/labels.h> + +#include <library/cpp/json/json_reader.h> +#include <library/cpp/resource/resource.h> +#include <library/cpp/testing/unittest/registar.h> + +#include <util/stream/str.h> +#include <util/string/builder.h> + +#include <limits> + +using namespace NMonitoring; + +namespace NMonitoring { + bool operator<(const TLabel& lhs, const TLabel& rhs) { + return lhs.Name() < rhs.Name() || + (lhs.Name() == rhs.Name() && lhs.Value() < rhs.Value()); + } +} +namespace { + void AssertLabels(const NProto::TMultiSample& actual, const TLabels& expected) { + UNIT_ASSERT_EQUAL(actual.LabelsSize(), expected.Size()); + + TSet<TLabel> actualSet; + TSet<TLabel> expectedSet; + Transform(expected.begin(), expected.end(), std::inserter(expectedSet, expectedSet.end()), [] (auto&& l) { + return TLabel{l.Name(), l.Value()}; + }); + + const auto& l = actual.GetLabels(); + Transform(std::begin(l), std::end(l), std::inserter(actualSet, std::begin(actualSet)), + [](auto&& elem) -> TLabel { + return {elem.GetName(), elem.GetValue()}; + }); + + TVector<TLabel> diff; + SetSymmetricDifference(std::begin(expectedSet), std::end(expectedSet), + std::begin(actualSet), std::end(actualSet), std::back_inserter(diff)); + + if (diff.size() > 0) { + for (auto&& l : diff) { + Cerr << l << Endl; + } + + UNIT_FAIL("Labels don't match"); + } + } + + void AssertLabelEqual(const NProto::TLabel& l, TStringBuf name, TStringBuf value) { + UNIT_ASSERT_STRINGS_EQUAL(l.GetName(), name); + UNIT_ASSERT_STRINGS_EQUAL(l.GetValue(), value); + } + + void AssertPointEqual(const NProto::TPoint& p, TInstant time, double value) { + UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds()); + UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kFloat64); + UNIT_ASSERT_DOUBLES_EQUAL(p.GetFloat64(), value, std::numeric_limits<double>::epsilon()); + } + + void AssertPointEqualNan(const NProto::TPoint& p, TInstant time) { + UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds()); + UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kFloat64); + UNIT_ASSERT(std::isnan(p.GetFloat64())); + } + + void AssertPointEqualInf(const NProto::TPoint& p, TInstant time, int sign) { + UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds()); + UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kFloat64); + UNIT_ASSERT(std::isinf(p.GetFloat64())); + if (sign < 0) { + UNIT_ASSERT(p.GetFloat64() < 0); + } + } + + void AssertPointEqual(const NProto::TPoint& p, TInstant time, ui64 value) { + UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds()); + UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kUint64); + UNIT_ASSERT_VALUES_EQUAL(p.GetUint64(), value); + } + + void AssertPointEqual(const NProto::TPoint& p, TInstant time, i64 value) { + UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds()); + UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kInt64); + UNIT_ASSERT_VALUES_EQUAL(p.GetInt64(), value); + } + + void AssertPointEqual(const NProto::TPoint& p, TInstant time, const IHistogramSnapshot& expected) { + UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds()); + UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kHistogram); + + const NProto::THistogram& h = p.GetHistogram(); + UNIT_ASSERT_VALUES_EQUAL(h.BoundsSize(), expected.Count()); + UNIT_ASSERT_VALUES_EQUAL(h.ValuesSize(), expected.Count()); + + for (size_t i = 0; i < h.BoundsSize(); i++) { + UNIT_ASSERT_DOUBLES_EQUAL(h.GetBounds(i), expected.UpperBound(i), Min<double>()); + UNIT_ASSERT_VALUES_EQUAL(h.GetValues(i), expected.Value(i)); + } + } + + void AssertPointEqual(const NProto::TPoint& p, TInstant time, const TLogHistogramSnapshot& expected) { + UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds()); + UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kLogHistogram); + + const double eps = 1e-10; + const NProto::TLogHistogram& h = p.GetLogHistogram(); + + UNIT_ASSERT_DOUBLES_EQUAL(h.GetBase(), expected.Base(), eps); + UNIT_ASSERT_VALUES_EQUAL(h.GetZerosCount(), expected.ZerosCount()); + UNIT_ASSERT_VALUES_EQUAL(h.GetStartPower(), expected.StartPower()); + UNIT_ASSERT_VALUES_EQUAL(h.BucketsSize(), expected.Count()); + for (size_t i = 0; i < expected.Count(); ++i) { + UNIT_ASSERT_DOUBLES_EQUAL(h.GetBuckets(i), expected.Bucket(i), eps); + } + } + + void AssertPointEqual(const NProto::TPoint& p, TInstant time, const ISummaryDoubleSnapshot& expected) { + UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds()); + UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kSummaryDouble); + auto actual = p.GetSummaryDouble(); + const double eps = 1e-10; + UNIT_ASSERT_DOUBLES_EQUAL(actual.GetSum(), expected.GetSum(), eps); + UNIT_ASSERT_DOUBLES_EQUAL(actual.GetMin(), expected.GetMin(), eps); + UNIT_ASSERT_DOUBLES_EQUAL(actual.GetMax(), expected.GetMax(), eps); + UNIT_ASSERT_DOUBLES_EQUAL(actual.GetLast(), expected.GetLast(), eps); + UNIT_ASSERT_VALUES_EQUAL(actual.GetCount(), expected.GetCount()); + } + +} // namespace + + +Y_UNIT_TEST_SUITE(TJsonTest) { + const TInstant now = TInstant::ParseIso8601Deprecated("2017-11-05T01:02:03Z"); + + Y_UNIT_TEST(Encode) { + auto check = [](bool cloud, bool buffered, TStringBuf expectedResourceKey) { + TString json; + TStringOutput out(json); + auto e = cloud + ? (buffered ? BufferedEncoderCloudJson(&out, 2, "metric") : EncoderCloudJson(&out, 2, "metric")) + : (buffered ? BufferedEncoderJson(&out, 2) : EncoderJson(&out, 2)); + e->OnStreamBegin(); + { // common time + e->OnCommonTime(TInstant::Seconds(1500000000)); + } + { // common labels + e->OnLabelsBegin(); + e->OnLabel("project", "solomon"); + e->OnLabelsEnd(); + } + { // metric #1 + e->OnMetricBegin(EMetricType::COUNTER); + { + e->OnLabelsBegin(); + e->OnLabel("metric", "single"); + e->OnLabel("labels", "l1"); + e->OnLabelsEnd(); + } + e->OnUint64(now, 17); + e->OnMetricEnd(); + } + { // metric #2 + e->OnMetricBegin(EMetricType::RATE); + { + e->OnLabelsBegin(); + e->OnLabel("metric", "single"); + e->OnLabel("labels", "l2"); + e->OnLabelsEnd(); + } + e->OnUint64(now, 17); + e->OnMetricEnd(); + } + { // metric #3 + e->OnMetricBegin(EMetricType::GAUGE); + e->OnDouble(now, 3.14); + { + e->OnLabelsBegin(); + e->OnLabel("metric", "single"); + e->OnLabel("labels", "l3"); + e->OnLabelsEnd(); + } + e->OnMetricEnd(); + } + { // metric #4 + e->OnMetricBegin(EMetricType::IGAUGE); + e->OnInt64(now, 42); + { + e->OnLabelsBegin(); + e->OnLabel("metric", "single_igauge"); + e->OnLabel("labels", "l4"); + e->OnLabelsEnd(); + } + e->OnMetricEnd(); + } + { // metric #5 + e->OnMetricBegin(EMetricType::GAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("metric", "multiple"); + e->OnLabel("labels", "l5"); + e->OnLabelsEnd(); + } + e->OnDouble(now, std::numeric_limits<double>::quiet_NaN()); + e->OnDouble(now + TDuration::Seconds(15), std::numeric_limits<double>::infinity()); + e->OnDouble(now + TDuration::Seconds(30), -std::numeric_limits<double>::infinity()); + e->OnMetricEnd(); + } + + { // metric #6 + e->OnMetricBegin(EMetricType::COUNTER); + e->OnUint64(now, 1337); + e->OnUint64(now + TDuration::Seconds(15), 1338); + { + e->OnLabelsBegin(); + e->OnLabel("metric", "multiple"); + e->OnLabel("labels", "l6"); + e->OnLabelsEnd(); + } + e->OnMetricEnd(); + } + e->OnStreamEnd(); + e->Close(); + json += "\n"; + + auto parseJson = [] (auto buf) { + NJson::TJsonValue value; + NJson::ReadJsonTree(buf, &value, true); + return value; + }; + + const auto expectedJson = NResource::Find(expectedResourceKey); + UNIT_ASSERT_EQUAL(parseJson(json), parseJson(expectedJson)); + }; + + check(false, false, "/expected.json"); + check(false, true, "/expected_buffered.json"); + check(true, false, "/expected_cloud.json"); + check(true, true, "/expected_cloud_buffered.json"); + } + + TLogHistogramSnapshotPtr TestLogHistogram(ui32 v = 1) { + TVector<double> buckets{0.5 * v, 0.25 * v, 0.25 * v, 0.5 * v}; + return MakeIntrusive<TLogHistogramSnapshot>(1.5, 1u, 0, std::move(buckets)); + } + + Y_UNIT_TEST(HistogramAndSummaryMetricTypesAreNotSupportedByCloudJson) { + const TInstant now = TInstant::ParseIso8601Deprecated("2017-11-05T01:02:03Z"); + + auto emit = [&](IMetricEncoder* encoder, EMetricType metricType) { + encoder->OnStreamBegin(); + { + encoder->OnMetricBegin(metricType); + { + encoder->OnLabelsBegin(); + encoder->OnLabel("name", "m"); + encoder->OnLabelsEnd(); + } + + switch (metricType) { + case EMetricType::HIST: { + auto histogram = ExponentialHistogram(6, 2); + encoder->OnHistogram(now, histogram->Snapshot()); + break; + } + case EMetricType::LOGHIST: { + auto histogram = TestLogHistogram(); + encoder->OnLogHistogram(now, histogram); + break; + } + case EMetricType::DSUMMARY: { + auto summary = MakeIntrusive<TSummaryDoubleSnapshot>(10., -0.5, 0.5, 0.3, 30u); + encoder->OnSummaryDouble(now, summary); + break; + } + default: + Y_FAIL("unexpected metric type [%s]", ToString(metricType).c_str()); + } + + encoder->OnMetricEnd(); + } + encoder->OnStreamEnd(); + encoder->Close(); + }; + + auto doTest = [&](bool buffered, EMetricType metricType) { + TString json; + TStringOutput out(json); + auto encoder = buffered ? BufferedEncoderCloudJson(&out, 2) : EncoderCloudJson(&out, 2); + const TString expectedMessage = TStringBuilder() + << "metric type '" << metricType << "' is not supported by cloud json format"; + UNIT_ASSERT_EXCEPTION_CONTAINS_C(emit(encoder.Get(), metricType), yexception, expectedMessage, + TString("buffered: ") + ToString(buffered)); + }; + + doTest(false, EMetricType::HIST); + doTest(false, EMetricType::LOGHIST); + doTest(false, EMetricType::DSUMMARY); + doTest(true, EMetricType::HIST); + doTest(true, EMetricType::LOGHIST); + doTest(true, EMetricType::DSUMMARY); + } + + Y_UNIT_TEST(MetricsWithDifferentLabelOrderGetMerged) { + TString json; + TStringOutput out(json); + auto e = BufferedEncoderJson(&out, 2); + + e->OnStreamBegin(); + { + e->OnMetricBegin(EMetricType::RATE); + { + e->OnLabelsBegin(); + e->OnLabel("metric", "hello"); + e->OnLabel("label", "world"); + e->OnLabelsEnd(); + } + e->OnUint64(TInstant::Zero(), 0); + e->OnMetricEnd(); + } + { + e->OnMetricBegin(EMetricType::RATE); + { + e->OnLabelsBegin(); + e->OnLabel("label", "world"); + e->OnLabel("metric", "hello"); + e->OnLabelsEnd(); + } + e->OnUint64(TInstant::Zero(), 1); + e->OnMetricEnd(); + } + e->OnStreamEnd(); + e->Close(); + json += "\n"; + + TString expectedJson = NResource::Find("/merged.json"); + // we cannot be sure regarding the label order in the result, + // so we'll have to parse the expected value and then compare it with actual + + NProto::TMultiSamplesList samples; + IMetricEncoderPtr d = EncoderProtobuf(&samples); + DecodeJson(expectedJson, d.Get()); + + UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 1); + { + const NProto::TMultiSample& s = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::RATE); + AssertLabels(s, TLabels{{"metric", "hello"}, {"label", "world"}}); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + AssertPointEqual(s.GetPoints(0), TInstant::Zero(), ui64(1)); + } + } + Y_UNIT_TEST(Decode1) { + NProto::TMultiSamplesList samples; + { + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + TString testJson = NResource::Find("/expected.json"); + DecodeJson(testJson, e.Get()); + } + + UNIT_ASSERT_VALUES_EQUAL( + TInstant::MilliSeconds(samples.GetCommonTime()), + TInstant::Seconds(1500000000)); + + UNIT_ASSERT_VALUES_EQUAL(samples.CommonLabelsSize(), 1); + AssertLabelEqual(samples.GetCommonLabels(0), "project", "solomon"); + + UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 6); + { + const NProto::TMultiSample& s = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::COUNTER); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2); + AssertLabelEqual(s.GetLabels(0), "metric", "single"); + AssertLabelEqual(s.GetLabels(1), "labels", "l1"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + AssertPointEqual(s.GetPoints(0), now, ui64(17)); + } + { + const NProto::TMultiSample& s = samples.GetSamples(1); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::RATE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2); + AssertLabelEqual(s.GetLabels(0), "metric", "single"); + AssertLabelEqual(s.GetLabels(1), "labels", "l2"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + AssertPointEqual(s.GetPoints(0), now, ui64(17)); + } + { + const NProto::TMultiSample& s = samples.GetSamples(2); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2); + AssertLabelEqual(s.GetLabels(0), "metric", "single"); + AssertLabelEqual(s.GetLabels(1), "labels", "l3"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + AssertPointEqual(s.GetPoints(0), now, 3.14); + } + { + const NProto::TMultiSample& s = samples.GetSamples(3); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::IGAUGE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2); + AssertLabelEqual(s.GetLabels(0), "metric", "single_igauge"); + AssertLabelEqual(s.GetLabels(1), "labels", "l4"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + AssertPointEqual(s.GetPoints(0), now, i64(42)); + } + { + const NProto::TMultiSample& s = samples.GetSamples(4); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2); + AssertLabelEqual(s.GetLabels(0), "metric", "multiple"); + AssertLabelEqual(s.GetLabels(1), "labels", "l5"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 3); + AssertPointEqualNan(s.GetPoints(0), now); + AssertPointEqualInf(s.GetPoints(1), now + TDuration::Seconds(15), 1); + AssertPointEqualInf(s.GetPoints(2), now + TDuration::Seconds(30), -11); + } + { + const NProto::TMultiSample& s = samples.GetSamples(5); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::COUNTER); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2); + AssertLabelEqual(s.GetLabels(0), "metric", "multiple"); + AssertLabelEqual(s.GetLabels(1), "labels", "l6"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 2); + AssertPointEqual(s.GetPoints(0), now, ui64(1337)); + AssertPointEqual(s.GetPoints(1), now + TDuration::Seconds(15), ui64(1338)); + } + } + + Y_UNIT_TEST(DecodeMetrics) { + NProto::TMultiSamplesList samples; + { + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + TString metricsJson = NResource::Find("/metrics.json"); + DecodeJson(metricsJson, e.Get()); + } + + UNIT_ASSERT_VALUES_EQUAL( + TInstant::MilliSeconds(samples.GetCommonTime()), + TInstant::ParseIso8601Deprecated("2017-08-27T12:34:56Z")); + + UNIT_ASSERT_VALUES_EQUAL(samples.CommonLabelsSize(), 3); + AssertLabelEqual(samples.GetCommonLabels(0), "project", "solomon"); + AssertLabelEqual(samples.GetCommonLabels(1), "cluster", "man"); + AssertLabelEqual(samples.GetCommonLabels(2), "service", "stockpile"); + + UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 4); + { + const NProto::TMultiSample& s = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "metric", "Memory"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + AssertPointEqual(s.GetPoints(0), TInstant::Zero(), 10.0); + } + { + const NProto::TMultiSample& s = samples.GetSamples(1); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::RATE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "metric", "UserTime"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + AssertPointEqual(s.GetPoints(0), TInstant::Zero(), ui64(1)); + } + { + const NProto::TMultiSample& s = samples.GetSamples(2); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2); + AssertLabelEqual(s.GetLabels(0), "export", "Oxygen"); + AssertLabelEqual(s.GetLabels(1), "metric", "QueueSize"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + auto ts = TInstant::ParseIso8601Deprecated("2017-11-05T12:34:56.000Z"); + AssertPointEqual(s.GetPoints(0), ts, 3.14159); + } + { + const NProto::TMultiSample& s = samples.GetSamples(3); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "metric", "Writes"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 2); + auto ts1 = TInstant::ParseIso8601Deprecated("2017-08-28T12:32:11Z"); + AssertPointEqual(s.GetPoints(0), ts1, -10.0); + auto ts2 = TInstant::Seconds(1503923187); + AssertPointEqual(s.GetPoints(1), ts2, 20.0); + } + } + + Y_UNIT_TEST(DecodeSensors) { + NProto::TMultiSamplesList samples; + { + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + TString sensorsJson = NResource::Find("/sensors.json"); + DecodeJson(sensorsJson, e.Get()); + } + + UNIT_ASSERT_VALUES_EQUAL( + TInstant::MilliSeconds(samples.GetCommonTime()), + TInstant::ParseIso8601Deprecated("2017-08-27T12:34:56Z")); + + UNIT_ASSERT_VALUES_EQUAL(samples.CommonLabelsSize(), 3); + AssertLabelEqual(samples.GetCommonLabels(0), "project", "solomon"); + AssertLabelEqual(samples.GetCommonLabels(1), "cluster", "man"); + AssertLabelEqual(samples.GetCommonLabels(2), "service", "stockpile"); + + UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 4); + { + const NProto::TMultiSample& s = samples.GetSamples(0); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "metric", "Memory"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + AssertPointEqual(s.GetPoints(0), TInstant::Zero(), 10.0); + } + { + const NProto::TMultiSample& s = samples.GetSamples(1); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::RATE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "metric", "UserTime"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + AssertPointEqual(s.GetPoints(0), TInstant::Zero(), ui64(1)); + } + { + const NProto::TMultiSample& s = samples.GetSamples(2); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2); + AssertLabelEqual(s.GetLabels(0), "export", "Oxygen"); + AssertLabelEqual(s.GetLabels(1), "metric", "QueueSize"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + auto ts = TInstant::ParseIso8601Deprecated("2017-11-05T12:34:56.000Z"); + AssertPointEqual(s.GetPoints(0), ts, 3.14159); + } + { + const NProto::TMultiSample& s = samples.GetSamples(3); + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "metric", "Writes"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 2); + auto ts1 = TInstant::ParseIso8601Deprecated("2017-08-28T12:32:11Z"); + AssertPointEqual(s.GetPoints(0), ts1, -10.0); + auto ts2 = TInstant::Seconds(1503923187); + AssertPointEqual(s.GetPoints(1), ts2, 20.0); + } + } + + Y_UNIT_TEST(DecodeToEncoder) { + auto testJson = NResource::Find("/test_decode_to_encode.json"); + + TStringStream Stream_; + auto encoder = BufferedEncoderJson(&Stream_, 4); + DecodeJson(testJson, encoder.Get()); + + encoder->Close(); + + auto val1 = NJson::ReadJsonFastTree(testJson, true); + auto val2 = NJson::ReadJsonFastTree(Stream_.Str(), true); + + UNIT_ASSERT_VALUES_EQUAL(val1, val2); + } + + void WriteEmptySeries(const IMetricEncoderPtr& e) { + e->OnStreamBegin(); + { + e->OnMetricBegin(EMetricType::COUNTER); + { + e->OnLabelsBegin(); + e->OnLabel("foo", "bar"); + e->OnLabelsEnd(); + } + e->OnMetricEnd(); + } + + e->OnStreamEnd(); + e->Close(); + } + + Y_UNIT_TEST(EncodeEmptySeries) { + TString json; + TStringOutput out(json); + + auto e = EncoderJson(&out, 2); + WriteEmptySeries(e); + json += "\n"; + + TString expectedJson = NResource::Find("/empty_series.json"); + UNIT_ASSERT_NO_DIFF(json, expectedJson); + } + + void WriteEmptyLabels(IMetricEncoderPtr& e) { + e->OnStreamBegin(); + e->OnMetricBegin(EMetricType::COUNTER); + + e->OnLabelsBegin(); + UNIT_ASSERT_EXCEPTION(e->OnLabelsEnd(), yexception); + } + + Y_UNIT_TEST(LabelsCannotBeEmpty) { + TString json; + TStringOutput out(json); + + auto e = EncoderJson(&out, 2); + WriteEmptyLabels(e); + } + + Y_UNIT_TEST(LabelsCannotBeEmptyBuffered) { + TString json; + TStringOutput out(json); + + auto e = BufferedEncoderJson(&out, 2); + WriteEmptyLabels(e); + } + + Y_UNIT_TEST(EncodeEmptySeriesBuffered) { + TString json; + TStringOutput out(json); + + auto e = BufferedEncoderJson(&out, 2); + WriteEmptySeries(e); + json += "\n"; + + TString expectedJson = NResource::Find("/empty_series.json"); + UNIT_ASSERT_NO_DIFF(json, expectedJson); + } + + Y_UNIT_TEST(BufferedEncoderMergesMetrics) { + TString json; + TStringOutput out(json); + + auto e = BufferedEncoderJson(&out, 2); + auto ts = 1; + + auto writeMetric = [&] (const TString& val) { + e->OnMetricBegin(EMetricType::COUNTER); + + e->OnLabelsBegin(); + e->OnLabel("foo", val); + e->OnLabelsEnd(); + e->OnUint64(TInstant::Seconds(ts++), 42); + + e->OnMetricEnd(); + }; + + e->OnStreamBegin(); + writeMetric("bar"); + writeMetric("bar"); + writeMetric("baz"); + writeMetric("bar"); + e->OnStreamEnd(); + e->Close(); + + json += "\n"; + + TString expectedJson = NResource::Find("/buffered_test.json"); + UNIT_ASSERT_NO_DIFF(json, expectedJson); + } + + Y_UNIT_TEST(JsonEncoderDisallowsValuesInTimeseriesWithoutTs) { + TStringStream out; + + auto e = EncoderJson(&out); + auto writePreamble = [&] { + e->OnStreamBegin(); + e->OnMetricBegin(EMetricType::COUNTER); + e->OnLabelsBegin(); + e->OnLabel("foo", "bar"); + e->OnLabelsEnd(); + }; + + // writing two values for a metric in a row will trigger + // timeseries object construction + writePreamble(); + e->OnUint64(TInstant::Zero(), 42); + UNIT_ASSERT_EXCEPTION(e->OnUint64(TInstant::Zero(), 42), yexception); + + e = EncoderJson(&out); + writePreamble(); + e->OnUint64(TInstant::Zero(), 42); + UNIT_ASSERT_EXCEPTION(e->OnUint64(TInstant::Now(), 42), yexception); + + e = EncoderJson(&out); + writePreamble(); + e->OnUint64(TInstant::Now(), 42); + UNIT_ASSERT_EXCEPTION(e->OnUint64(TInstant::Zero(), 42), yexception); + } + + Y_UNIT_TEST(BufferedJsonEncoderMergesTimeseriesWithoutTs) { + TStringStream out; + + { + auto e = BufferedEncoderJson(&out, 2); + e->OnStreamBegin(); + e->OnMetricBegin(EMetricType::COUNTER); + e->OnLabelsBegin(); + e->OnLabel("foo", "bar"); + e->OnLabelsEnd(); + // in buffered mode we are able to find values with same (in this case zero) + // timestamp and discard duplicates + e->OnUint64(TInstant::Zero(), 42); + e->OnUint64(TInstant::Zero(), 43); + e->OnUint64(TInstant::Zero(), 44); + e->OnUint64(TInstant::Zero(), 45); + e->OnMetricEnd(); + e->OnStreamEnd(); + } + + out << "\n"; + UNIT_ASSERT_NO_DIFF(out.Str(), NResource::Find("/buffered_ts_merge.json")); + } + + template <typename TFactory, typename TConsumer> + TString EncodeToString(TFactory factory, TConsumer consumer) { + TStringStream out; + { + IMetricEncoderPtr e = factory(&out, 2); + consumer(e.Get()); + } + out << '\n'; + return out.Str(); + } + + Y_UNIT_TEST(SummaryValueEncode) { + auto writeDocument = [](IMetricEncoder* e) { + e->OnStreamBegin(); + { + e->OnMetricBegin(EMetricType::DSUMMARY); + { + e->OnLabelsBegin(); + e->OnLabel("metric", "temperature"); + e->OnLabelsEnd(); + } + + e->OnSummaryDouble(now, MakeIntrusive<TSummaryDoubleSnapshot>(10., -0.5, 0.5, 0.3, 30u)); + e->OnMetricEnd(); + } + e->OnStreamEnd(); + }; + + TString result1 = EncodeToString(EncoderJson, writeDocument); + UNIT_ASSERT_NO_DIFF(result1, NResource::Find("/summary_value.json")); + + TString result2 = EncodeToString(BufferedEncoderJson, writeDocument); + UNIT_ASSERT_NO_DIFF(result2, NResource::Find("/summary_value.json")); + } + + ISummaryDoubleSnapshotPtr TestInfSummary() { + return MakeIntrusive<TSummaryDoubleSnapshot>( + std::numeric_limits<double>::quiet_NaN(), + -std::numeric_limits<double>::infinity(), + std::numeric_limits<double>::infinity(), + 0.3, + 30u); + } + + Y_UNIT_TEST(SummaryInfEncode) { + auto writeDocument = [](IMetricEncoder* e) { + e->OnStreamBegin(); + { + e->OnMetricBegin(EMetricType::DSUMMARY); + { + e->OnLabelsBegin(); + e->OnLabel("metric", "temperature"); + e->OnLabelsEnd(); + } + + e->OnSummaryDouble(now, TestInfSummary()); + e->OnMetricEnd(); + } + e->OnStreamEnd(); + }; + + TString result1 = EncodeToString(EncoderJson, writeDocument); + UNIT_ASSERT_NO_DIFF(result1, NResource::Find("/summary_inf.json")); + + TString result2 = EncodeToString(BufferedEncoderJson, writeDocument); + UNIT_ASSERT_NO_DIFF(result2, NResource::Find("/summary_inf.json")); + } + + Y_UNIT_TEST(SummaryInfDecode) { + NProto::TMultiSamplesList samples; + { + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + TString testJson = NResource::Find("/summary_inf.json"); + DecodeJson(testJson, e.Get()); + } + + UNIT_ASSERT_VALUES_EQUAL(1, samples.SamplesSize()); + const NProto::TMultiSample& s = samples.GetSamples(0); + + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::DSUMMARY); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "metric", "temperature"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + + auto actual = s.GetPoints(0).GetSummaryDouble(); + UNIT_ASSERT(std::isnan(actual.GetSum())); + UNIT_ASSERT(actual.GetMin() < 0); + UNIT_ASSERT(std::isinf(actual.GetMin())); + UNIT_ASSERT(actual.GetMax() > 0); + UNIT_ASSERT(std::isinf(actual.GetMax())); + } + + Y_UNIT_TEST(SummaryValueDecode) { + NProto::TMultiSamplesList samples; + { + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + TString testJson = NResource::Find("/summary_value.json"); + DecodeJson(testJson, e.Get()); + } + + UNIT_ASSERT_VALUES_EQUAL(1, samples.SamplesSize()); + const NProto::TMultiSample& s = samples.GetSamples(0); + + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::DSUMMARY); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "metric", "temperature"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + + auto snapshot = TSummaryDoubleSnapshot(10., -0.5, 0.5, 0.3, 30u); + AssertPointEqual(s.GetPoints(0), now, snapshot); + } + + Y_UNIT_TEST(SummaryTimeSeriesEncode) { + auto writeDocument = [](IMetricEncoder* e) { + e->OnStreamBegin(); + { + e->OnMetricBegin(EMetricType::DSUMMARY); + { + e->OnLabelsBegin(); + e->OnLabel("metric", "temperature"); + e->OnLabelsEnd(); + } + + TSummaryDoubleCollector summary; + summary.Collect(0.3); + summary.Collect(-0.5); + summary.Collect(1.); + + e->OnSummaryDouble(now, summary.Snapshot()); + + summary.Collect(-1.5); + summary.Collect(0.01); + + e->OnSummaryDouble(now + TDuration::Seconds(15), summary.Snapshot()); + + e->OnMetricEnd(); + } + e->OnStreamEnd(); + }; + + TString result1 = EncodeToString(EncoderJson, writeDocument); + UNIT_ASSERT_NO_DIFF(result1, NResource::Find("/summary_timeseries.json")); + + TString result2 = EncodeToString(BufferedEncoderJson, writeDocument); + UNIT_ASSERT_NO_DIFF(result2, NResource::Find("/summary_timeseries.json")); + } + + Y_UNIT_TEST(SummaryTimeSeriesDecode) { + NProto::TMultiSamplesList samples; + { + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + TString testJson = NResource::Find("/summary_timeseries.json"); + DecodeJson(testJson, e.Get()); + } + + UNIT_ASSERT_VALUES_EQUAL(1, samples.SamplesSize()); + const NProto::TMultiSample& s = samples.GetSamples(0); + + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::DSUMMARY); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "metric", "temperature"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 2); + + TSummaryDoubleCollector summary; + summary.Collect(0.3); + summary.Collect(-0.5); + summary.Collect(1.); + + AssertPointEqual(s.GetPoints(0), now, *summary.Snapshot()); + + summary.Collect(-1.5); + summary.Collect(0.01); + + AssertPointEqual(s.GetPoints(1), now + TDuration::Seconds(15), *summary.Snapshot()); + } + + Y_UNIT_TEST(LogHistogramValueEncode) { + auto writeDocument = [](IMetricEncoder* e) { + e->OnStreamBegin(); + { + e->OnMetricBegin(EMetricType::LOGHIST); + { + e->OnLabelsBegin(); + e->OnLabel("metric", "ms"); + e->OnLabelsEnd(); + } + + e->OnLogHistogram(now, TestLogHistogram()); + e->OnMetricEnd(); + } + e->OnStreamEnd(); + }; + + TString result1 = EncodeToString(EncoderJson, writeDocument); + UNIT_ASSERT_NO_DIFF(result1, NResource::Find("/log_histogram_value.json")); + + TString result2 = EncodeToString(BufferedEncoderJson, writeDocument); + UNIT_ASSERT_NO_DIFF(result2, NResource::Find("/log_histogram_value.json")); + } + + Y_UNIT_TEST(LogHistogramValueDecode) { + NProto::TMultiSamplesList samples; + { + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + TString testJson = NResource::Find("/log_histogram_value.json"); + DecodeJson(testJson, e.Get()); + } + + UNIT_ASSERT_VALUES_EQUAL(1, samples.SamplesSize()); + const NProto::TMultiSample& s = samples.GetSamples(0); + + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::LOGHISTOGRAM); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "metric", "ms"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + + auto snapshot = TestLogHistogram(); + AssertPointEqual(s.GetPoints(0), now, *snapshot); + } + + Y_UNIT_TEST(HistogramValueEncode) { + auto writeDocument = [](IMetricEncoder* e) { + e->OnStreamBegin(); + { + e->OnMetricBegin(EMetricType::HIST); + { + e->OnLabelsBegin(); + e->OnLabel("metric", "responseTimeMillis"); + e->OnLabelsEnd(); + } + + // {1: 1, 2: 1, 4: 2, 8: 4, 16: 8, inf: 83} + auto h = ExponentialHistogram(6, 2); + for (i64 i = 1; i < 100; i++) { + h->Collect(i); + } + + e->OnHistogram(now, h->Snapshot()); + e->OnMetricEnd(); + } + e->OnStreamEnd(); + }; + + TString result1 = EncodeToString(EncoderJson, writeDocument); + UNIT_ASSERT_NO_DIFF(result1, NResource::Find("/histogram_value.json")); + + TString result2 = EncodeToString(BufferedEncoderJson, writeDocument); + UNIT_ASSERT_NO_DIFF(result2, NResource::Find("/histogram_value.json")); + } + + Y_UNIT_TEST(LogHistogramTimeSeriesEncode) { + auto writeDocument = [](IMetricEncoder* e) { + e->OnStreamBegin(); + { + e->OnMetricBegin(EMetricType::LOGHIST); + { + e->OnLabelsBegin(); + e->OnLabel("metric", "ms"); + e->OnLabelsEnd(); + } + + e->OnLogHistogram(now, TestLogHistogram(1));; + + e->OnLogHistogram(now + TDuration::Seconds(15), TestLogHistogram(2)); + + e->OnMetricEnd(); + } + e->OnStreamEnd(); + }; + + TString result1 = EncodeToString(EncoderJson, writeDocument); + UNIT_ASSERT_NO_DIFF(result1, NResource::Find("/log_histogram_timeseries.json")); + + TString result2 = EncodeToString(BufferedEncoderJson, writeDocument); + UNIT_ASSERT_NO_DIFF(result2, NResource::Find("/log_histogram_timeseries.json")); + } + + Y_UNIT_TEST(LogHistogramTimeSeriesDecode) { + NProto::TMultiSamplesList samples; + { + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + TString testJson = NResource::Find("/log_histogram_timeseries.json"); + DecodeJson(testJson, e.Get()); + } + + UNIT_ASSERT_VALUES_EQUAL(1, samples.SamplesSize()); + const NProto::TMultiSample& s = samples.GetSamples(0); + + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::LOGHISTOGRAM); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "metric", "ms"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 2); + + auto logHist = TestLogHistogram(1); + AssertPointEqual(s.GetPoints(0), now, *logHist); + + logHist = TestLogHistogram(2); + AssertPointEqual(s.GetPoints(1), now + TDuration::Seconds(15), *logHist); + } + + void HistogramValueDecode(const TString& filePath) { + NProto::TMultiSamplesList samples; + { + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + TString testJson = NResource::Find(filePath); + DecodeJson(testJson, e.Get()); + } + + UNIT_ASSERT_VALUES_EQUAL(1, samples.SamplesSize()); + const NProto::TMultiSample& s = samples.GetSamples(0); + + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::HISTOGRAM); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "metric", "responseTimeMillis"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + + auto h = ExponentialHistogram(6, 2); + for (i64 i = 1; i < 100; i++) { + h->Collect(i); + } + + AssertPointEqual(s.GetPoints(0), now, *h->Snapshot()); + } + + Y_UNIT_TEST(HistogramValueDecode) { + HistogramValueDecode("/histogram_value.json"); + HistogramValueDecode("/histogram_value_inf_before_bounds.json"); + } + + Y_UNIT_TEST(HistogramTimeSeriesEncode) { + auto writeDocument = [](IMetricEncoder* e) { + e->OnStreamBegin(); + { + e->OnMetricBegin(EMetricType::HIST_RATE); + { + e->OnLabelsBegin(); + e->OnLabel("metric", "responseTimeMillis"); + e->OnLabelsEnd(); + } + + // {1: 1, 2: 1, 4: 2, 8: 4, 16: 8, inf: 83} + auto h = ExponentialHistogram(6, 2); + for (i64 i = 1; i < 100; i++) { + h->Collect(i); + } + e->OnHistogram(now, h->Snapshot()); + + // {1: 2, 2: 2, 4: 4, 8: 8, 16: 16, inf: 166} + for (i64 i = 1; i < 100; i++) { + h->Collect(i); + } + e->OnHistogram(now + TDuration::Seconds(15), h->Snapshot()); + + e->OnMetricEnd(); + } + e->OnStreamEnd(); + }; + + TString result1 = EncodeToString(EncoderJson, writeDocument); + UNIT_ASSERT_NO_DIFF(result1, NResource::Find("/histogram_timeseries.json")); + + TString result2 = EncodeToString(BufferedEncoderJson, writeDocument); + UNIT_ASSERT_NO_DIFF(result2, NResource::Find("/histogram_timeseries.json")); + } + + Y_UNIT_TEST(HistogramTimeSeriesDecode) { + NProto::TMultiSamplesList samples; + { + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + TString testJson = NResource::Find("/histogram_timeseries.json"); + DecodeJson(testJson, e.Get()); + } + + UNIT_ASSERT_VALUES_EQUAL(1, samples.SamplesSize()); + const NProto::TMultiSample& s = samples.GetSamples(0); + + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::HIST_RATE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "metric", "responseTimeMillis"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 2); + + auto h = ExponentialHistogram(6, 2); + for (i64 i = 1; i < 100; i++) { + h->Collect(i); + } + + AssertPointEqual(s.GetPoints(0), now, *h->Snapshot()); + + for (i64 i = 1; i < 100; i++) { + h->Collect(i); + } + + AssertPointEqual(s.GetPoints(1), now + TDuration::Seconds(15), *h->Snapshot()); + } + + Y_UNIT_TEST(IntGaugeEncode) { + auto writeDocument = [](IMetricEncoder* e) { + e->OnStreamBegin(); + { + e->OnMetricBegin(EMetricType::IGAUGE); + { + e->OnLabelsBegin(); + e->OnLabel("metric", "a"); + e->OnLabelsEnd(); + } + e->OnInt64(now, Min<i64>()); + e->OnInt64(now + TDuration::Seconds(1), -1); + e->OnInt64(now + TDuration::Seconds(2), 0); + e->OnInt64(now + TDuration::Seconds(3), Max<i64>()); + e->OnMetricEnd(); + } + e->OnStreamEnd(); + }; + + TString result1 = EncodeToString(EncoderJson, writeDocument); + UNIT_ASSERT_NO_DIFF(result1, NResource::Find("/int_gauge.json")); + + TString result2 = EncodeToString(BufferedEncoderJson, writeDocument); + UNIT_ASSERT_NO_DIFF(result2, NResource::Find("/int_gauge.json")); + } + + Y_UNIT_TEST(InconsistentMetricTypes) { + auto emitMetrics = [](IMetricEncoder& encoder, const TString& expectedError) { + encoder.OnMetricBegin(EMetricType::GAUGE); + { + encoder.OnLabelsBegin(); + encoder.OnLabel("name", "m"); + encoder.OnLabel("l1", "v1"); + encoder.OnLabel("l2", "v2"); + encoder.OnLabelsEnd(); + } + encoder.OnDouble(now, 1.0); + encoder.OnMetricEnd(); + + encoder.OnMetricBegin(EMetricType::COUNTER); + { + encoder.OnLabelsBegin(); + encoder.OnLabel("name", "m"); + encoder.OnLabel("l1", "v1"); + encoder.OnLabel("l2", "v2"); + encoder.OnLabelsEnd(); + } + encoder.OnUint64(now, 1); + + UNIT_ASSERT_EXCEPTION_CONTAINS(encoder.OnMetricEnd(), + yexception, + expectedError); + }; + + { + TStringStream out; + auto encoder = BufferedEncoderJson(&out); + + encoder->OnStreamBegin(); + encoder->OnLabelsBegin(); + encoder->OnLabel("c", "cv"); + encoder->OnLabelsEnd(); + emitMetrics(*encoder, + "Time series point type mismatch: expected DOUBLE but found UINT64, " + "labels '{c=cv, l1=v1, l2=v2, name=m}'"); + } + + { + TStringStream out; + auto encoder = BufferedEncoderJson(&out); + + encoder->OnStreamBegin(); + encoder->OnLabelsBegin(); + encoder->OnLabel("l1", "v100"); + encoder->OnLabelsEnd(); + emitMetrics(*encoder, + "Time series point type mismatch: expected DOUBLE but found UINT64, " + "labels '{l1=v1, l2=v2, name=m}'"); + } + } + + Y_UNIT_TEST(IntGaugeDecode) { + NProto::TMultiSamplesList samples; + { + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + TString testJson = NResource::Find("/int_gauge.json"); + DecodeJson(testJson, e.Get()); + } + + UNIT_ASSERT_VALUES_EQUAL(1, samples.SamplesSize()); + const NProto::TMultiSample& s = samples.GetSamples(0); + + UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::IGAUGE); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "metric", "a"); + + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 4); + AssertPointEqual(s.GetPoints(0), now, Min<i64>()); + AssertPointEqual(s.GetPoints(1), now + TDuration::Seconds(1), i64(-1)); + AssertPointEqual(s.GetPoints(2), now + TDuration::Seconds(2), i64(0)); + AssertPointEqual(s.GetPoints(3), now + TDuration::Seconds(3), Max<i64>()); + } + + Y_UNIT_TEST(FuzzerRegression) { + NProto::TMultiSamplesList samples; + { + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + for (auto f : { "/hist_crash.json", "/crash.json" }) { + TString testJson = NResource::Find(f); + UNIT_ASSERT_EXCEPTION(DecodeJson(testJson, e.Get()), yexception); + } + } + } + + Y_UNIT_TEST(LegacyNegativeRateThrows) { + const auto input = R"({ + "sensors": [ + { + "mode": "deriv", + "value": -1, + "labels": { "metric": "SystemTime" } + }, + } + ]}")"; + + NProto::TMultiSamplesList samples; + IMetricEncoderPtr e = EncoderProtobuf(&samples); + UNIT_ASSERT_EXCEPTION(DecodeJson(input, e.Get()), yexception); + } + + Y_UNIT_TEST(DecodeNamedMetrics) { + NProto::TMultiSamplesList samples; + { + IMetricEncoderPtr e = EncoderProtobuf(&samples); + + TString metricsJson = NResource::Find("/named_metrics.json"); + DecodeJson(metricsJson, e.Get(), "sensor"); + } + + UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 2); + { + const NProto::TMultiSample& s = samples.GetSamples(0); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); + AssertLabelEqual(s.GetLabels(0), "sensor", "Memory"); + } + { + const NProto::TMultiSample& s = samples.GetSamples(1); + UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 2); + AssertLabelEqual(s.GetLabels(0), "sensor", "QueueSize"); + AssertLabelEqual(s.GetLabels(1), "export", "Oxygen"); + } + } + +} diff --git a/library/cpp/monlib/encode/json/typed_point.h b/library/cpp/monlib/encode/json/typed_point.h new file mode 100644 index 0000000000..fbaa840c4b --- /dev/null +++ b/library/cpp/monlib/encode/json/typed_point.h @@ -0,0 +1,123 @@ +#pragma once + +#include <library/cpp/monlib/metrics/metric_value.h> + + +namespace NMonitoring { + + class TTypedPoint { + public: + TTypedPoint() + : Time_(TInstant::Zero()) + , ValueType_(EMetricValueType::UNKNOWN) + { + } + + template <typename T> + TTypedPoint(TInstant time, T value) + : Time_(time) + , ValueType_(TValueType<T>::Type) + , Value_(value) + { + Ref(); + } + + ~TTypedPoint() { + UnRef(); + } + + TTypedPoint(const TTypedPoint& rhs) + : Time_(rhs.Time_) + , ValueType_(rhs.ValueType_) + , Value_(rhs.Value_) + { + Ref(); + } + + TTypedPoint& operator=(const TTypedPoint& rhs) { + UnRef(); + + Time_ = rhs.Time_; + ValueType_ = rhs.ValueType_; + Value_ = rhs.Value_; + + Ref(); + return *this; + } + + TTypedPoint(TTypedPoint&& rhs) noexcept + : Time_(rhs.Time_) + , ValueType_(rhs.ValueType_) + , Value_(rhs.Value_) + { + rhs.ValueType_ = EMetricValueType::UNKNOWN; + rhs.Value_ = {}; + } + + TTypedPoint& operator=(TTypedPoint&& rhs) noexcept { + UnRef(); + + Time_ = rhs.Time_; + ValueType_ = rhs.ValueType_; + Value_ = rhs.Value_; + + rhs.ValueType_ = EMetricValueType::UNKNOWN; + rhs.Value_ = {}; + + return *this; + } + + TInstant GetTime() const noexcept { + return Time_; + } + + void SetTime(TInstant time) noexcept { + Time_ = time; + } + + TMetricValue GetValue() const noexcept { + return Value_; + } + + EMetricValueType GetValueType() const noexcept { + return ValueType_; + } + + template <typename T> + void SetValue(T value) noexcept { + ValueType_ = TValueType<T>::Type; + Value_ = TMetricValue{value}; + } + + bool HasValue() { + return ValueType_ != EMetricValueType::UNKNOWN; + } + + private: + void Ref() { + if (ValueType_ == EMetricValueType::HISTOGRAM) { + Value_.AsHistogram()->Ref(); + } else if (ValueType_ == EMetricValueType::SUMMARY) { + Value_.AsSummaryDouble()->Ref(); + } else if (ValueType_ == EMetricValueType::LOGHISTOGRAM) { + Value_.AsLogHistogram()->Ref(); + } + } + + void UnRef() { + if (ValueType_ == EMetricValueType::HISTOGRAM) { + Value_.AsHistogram()->UnRef(); + } else if (ValueType_ == EMetricValueType::SUMMARY) { + Value_.AsSummaryDouble()->UnRef(); + } else if (ValueType_ == EMetricValueType::LOGHISTOGRAM) { + Value_.AsLogHistogram()->UnRef(); + } + } + + private: + TInstant Time_; + EMetricValueType ValueType_; + TMetricValue Value_; + }; + +} diff --git a/library/cpp/monlib/encode/json/ut/buffered_test.json b/library/cpp/monlib/encode/json/ut/buffered_test.json new file mode 100644 index 0000000000..53212cf8e1 --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/buffered_test.json @@ -0,0 +1,36 @@ +{ + "sensors": + [ + { + "kind":"COUNTER", + "labels": + { + "foo":"bar" + }, + "timeseries": + [ + { + "ts":1, + "value":42 + }, + { + "ts":2, + "value":42 + }, + { + "ts":4, + "value":42 + } + ] + }, + { + "kind":"COUNTER", + "labels": + { + "foo":"baz" + }, + "ts":3, + "value":42 + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/buffered_ts_merge.json b/library/cpp/monlib/encode/json/ut/buffered_ts_merge.json new file mode 100644 index 0000000000..1d27efacb0 --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/buffered_ts_merge.json @@ -0,0 +1,13 @@ +{ + "sensors": + [ + { + "kind":"COUNTER", + "labels": + { + "foo":"bar" + }, + "value":45 + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/crash.json b/library/cpp/monlib/encode/json/ut/crash.json Binary files differnew file mode 100644 index 0000000000..8ff4369dc4 --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/crash.json diff --git a/library/cpp/monlib/encode/json/ut/empty_series.json b/library/cpp/monlib/encode/json/ut/empty_series.json new file mode 100644 index 0000000000..641e10cdea --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/empty_series.json @@ -0,0 +1,12 @@ +{ + "sensors": + [ + { + "kind":"COUNTER", + "labels": + { + "foo":"bar" + } + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/expected.json b/library/cpp/monlib/encode/json/ut/expected.json new file mode 100644 index 0000000000..ead853455b --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/expected.json @@ -0,0 +1,92 @@ +{ + "ts":1500000000, + "commonLabels": + { + "project":"solomon" + }, + "sensors": + [ + { + "kind":"COUNTER", + "labels": + { + "metric":"single", + "labels":"l1" + }, + "ts":1509843723, + "value":17 + }, + { + "kind":"RATE", + "labels": + { + "metric":"single", + "labels":"l2" + }, + "ts":1509843723, + "value":17 + }, + { + "kind":"GAUGE", + "labels": + { + "metric":"single", + "labels":"l3" + }, + "ts":1509843723, + "value":3.14 + }, + { + "kind":"IGAUGE", + "labels": + { + "metric":"single_igauge", + "labels":"l4" + }, + "ts":1509843723, + "value":42 + }, + { + "kind":"GAUGE", + "labels": + { + "metric":"multiple", + "labels":"l5" + }, + "timeseries": + [ + { + "ts":1509843723, + "value":"nan" + }, + { + "ts":1509843738, + "value":"inf" + }, + { + "ts":1509843753, + "value":"-inf" + } + ] + }, + { + "kind":"COUNTER", + "timeseries": + [ + { + "ts":1509843723, + "value":1337 + }, + { + "ts":1509843738, + "value":1338 + } + ], + "labels": + { + "metric":"multiple", + "labels":"l6" + } + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/expected_buffered.json b/library/cpp/monlib/encode/json/ut/expected_buffered.json new file mode 100644 index 0000000000..9a6a1d6201 --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/expected_buffered.json @@ -0,0 +1,92 @@ +{ + "ts":1500000000, + "commonLabels": + { + "project":"solomon" + }, + "sensors": + [ + { + "kind":"COUNTER", + "labels": + { + "labels":"l1", + "metric":"single" + }, + "ts":1509843723, + "value":17 + }, + { + "kind":"RATE", + "labels": + { + "labels":"l2", + "metric":"single" + }, + "ts":1509843723, + "value":17 + }, + { + "kind":"GAUGE", + "labels": + { + "labels":"l3", + "metric":"single" + }, + "ts":1509843723, + "value":3.14 + }, + { + "kind":"IGAUGE", + "labels": + { + "labels":"l4", + "metric":"single_igauge" + }, + "ts":1509843723, + "value":42 + }, + { + "kind":"GAUGE", + "labels": + { + "labels":"l5", + "metric":"multiple" + }, + "timeseries": + [ + { + "ts":1509843723, + "value":"nan" + }, + { + "ts":1509843738, + "value":"inf" + }, + { + "ts":1509843753, + "value":"-inf" + } + ] + }, + { + "kind":"COUNTER", + "labels": + { + "labels":"l6", + "metric":"multiple" + }, + "timeseries": + [ + { + "ts":1509843723, + "value":1337 + }, + { + "ts":1509843738, + "value":1338 + } + ] + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/expected_cloud.json b/library/cpp/monlib/encode/json/ut/expected_cloud.json new file mode 100644 index 0000000000..6184811579 --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/expected_cloud.json @@ -0,0 +1,92 @@ +{ + "ts":"2017-07-14T02:40:00.000000Z", + "labels": + { + "project":"solomon" + }, + "metrics": + [ + { + "type":"COUNTER", + "labels": + { + "labels":"l1" + }, + "name":"single", + "ts":"2017-11-05T01:02:03.000000Z", + "value":17 + }, + { + "type":"RATE", + "labels": + { + "labels":"l2" + }, + "name":"single", + "ts":"2017-11-05T01:02:03.000000Z", + "value":17 + }, + { + "type":"DGAUGE", + "labels": + { + "labels":"l3" + }, + "name":"single", + "ts":"2017-11-05T01:02:03.000000Z", + "value":3.14 + }, + { + "type":"IGAUGE", + "labels": + { + "labels":"l4" + }, + "name":"single_igauge", + "ts":"2017-11-05T01:02:03.000000Z", + "value":42 + }, + { + "type":"DGAUGE", + "labels": + { + "labels":"l5" + }, + "name":"multiple", + "timeseries": + [ + { + "ts":"2017-11-05T01:02:03.000000Z", + "value":"nan" + }, + { + "ts":"2017-11-05T01:02:18.000000Z", + "value":"inf" + }, + { + "ts":"2017-11-05T01:02:33.000000Z", + "value":"-inf" + } + ] + }, + { + "type":"COUNTER", + "timeseries": + [ + { + "ts":"2017-11-05T01:02:03.000000Z", + "value":1337 + }, + { + "ts":"2017-11-05T01:02:18.000000Z", + "value":1338 + } + ], + "labels": + { + "labels":"l6" + }, + "name":"multiple" + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/expected_cloud_buffered.json b/library/cpp/monlib/encode/json/ut/expected_cloud_buffered.json new file mode 100644 index 0000000000..be237d522b --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/expected_cloud_buffered.json @@ -0,0 +1,92 @@ +{ + "ts":"2017-07-14T02:40:00.000000Z", + "labels": + { + "project":"solomon" + }, + "metrics": + [ + { + "type":"COUNTER", + "labels": + { + "labels":"l1" + }, + "name":"single", + "ts":"2017-11-05T01:02:03.000000Z", + "value":17 + }, + { + "type":"RATE", + "labels": + { + "labels":"l2" + }, + "name":"single", + "ts":"2017-11-05T01:02:03.000000Z", + "value":17 + }, + { + "type":"DGAUGE", + "labels": + { + "labels":"l3" + }, + "name":"single", + "ts":"2017-11-05T01:02:03.000000Z", + "value":3.14 + }, + { + "type":"IGAUGE", + "labels": + { + "labels":"l4" + }, + "name":"single_igauge", + "ts":"2017-11-05T01:02:03.000000Z", + "value":42 + }, + { + "type":"DGAUGE", + "labels": + { + "labels":"l5" + }, + "name":"multiple", + "timeseries": + [ + { + "ts":"2017-11-05T01:02:03.000000Z", + "value":"nan" + }, + { + "ts":"2017-11-05T01:02:18.000000Z", + "value":"inf" + }, + { + "ts":"2017-11-05T01:02:33.000000Z", + "value":"-inf" + } + ] + }, + { + "type":"COUNTER", + "labels": + { + "labels":"l6" + }, + "name":"multiple", + "timeseries": + [ + { + "ts":"2017-11-05T01:02:03.000000Z", + "value":1337 + }, + { + "ts":"2017-11-05T01:02:18.000000Z", + "value":1338 + } + ] + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/hist_crash.json b/library/cpp/monlib/encode/json/ut/hist_crash.json Binary files differnew file mode 100644 index 0000000000..867d0fce7d --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/hist_crash.json diff --git a/library/cpp/monlib/encode/json/ut/histogram_timeseries.json b/library/cpp/monlib/encode/json/ut/histogram_timeseries.json new file mode 100644 index 0000000000..f6131ffded --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/histogram_timeseries.json @@ -0,0 +1,61 @@ +{ + "sensors": + [ + { + "kind":"HIST_RATE", + "labels": + { + "metric":"responseTimeMillis" + }, + "timeseries": + [ + { + "ts":1509843723, + "hist": + { + "bounds": + [ + 1, + 2, + 4, + 8, + 16 + ], + "buckets": + [ + 1, + 1, + 2, + 4, + 8 + ], + "inf":83 + } + }, + { + "ts":1509843738, + "hist": + { + "bounds": + [ + 1, + 2, + 4, + 8, + 16 + ], + "buckets": + [ + 2, + 2, + 4, + 8, + 16 + ], + "inf":166 + } + } + ] + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/histogram_value.json b/library/cpp/monlib/encode/json/ut/histogram_value.json new file mode 100644 index 0000000000..ec1ae5cdec --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/histogram_value.json @@ -0,0 +1,33 @@ +{ + "sensors": + [ + { + "kind":"HIST", + "labels": + { + "metric":"responseTimeMillis" + }, + "ts":1509843723, + "hist": + { + "bounds": + [ + 1, + 2, + 4, + 8, + 16 + ], + "buckets": + [ + 1, + 1, + 2, + 4, + 8 + ], + "inf":83 + } + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/histogram_value_inf_before_bounds.json b/library/cpp/monlib/encode/json/ut/histogram_value_inf_before_bounds.json new file mode 100644 index 0000000000..f8a17c8831 --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/histogram_value_inf_before_bounds.json @@ -0,0 +1,33 @@ +{ + "sensors": + [ + { + "kind":"HIST", + "labels": + { + "metric":"responseTimeMillis" + }, + "ts":1509843723, + "hist": + { + "inf":83, + "bounds": + [ + 1, + 2, + 4, + 8, + 16 + ], + "buckets": + [ + 1, + 1, + 2, + 4, + 8 + ] + } + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/int_gauge.json b/library/cpp/monlib/encode/json/ut/int_gauge.json new file mode 100644 index 0000000000..fbe57f873c --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/int_gauge.json @@ -0,0 +1,31 @@ +{ + "sensors": + [ + { + "kind":"IGAUGE", + "labels": + { + "metric":"a" + }, + "timeseries": + [ + { + "ts":1509843723, + "value":-9223372036854775808 + }, + { + "ts":1509843724, + "value":-1 + }, + { + "ts":1509843725, + "value":0 + }, + { + "ts":1509843726, + "value":9223372036854775807 + } + ] + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/log_histogram_timeseries.json b/library/cpp/monlib/encode/json/ut/log_histogram_timeseries.json new file mode 100644 index 0000000000..e811a2cc57 --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/log_histogram_timeseries.json @@ -0,0 +1,47 @@ +{ + "sensors": + [ + { + "kind":"LOGHIST", + "labels": + { + "metric":"ms" + }, + "timeseries": + [ + { + "ts":1509843723, + "log_hist": + { + "base":1.5, + "zeros_count":1, + "start_power":0, + "buckets": + [ + 0.5, + 0.25, + 0.25, + 0.5 + ] + } + }, + { + "ts":1509843738, + "log_hist": + { + "base":1.5, + "zeros_count":1, + "start_power":0, + "buckets": + [ + 1, + 0.5, + 0.5, + 1 + ] + } + } + ] + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/log_histogram_value.json b/library/cpp/monlib/encode/json/ut/log_histogram_value.json new file mode 100644 index 0000000000..002478293b --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/log_histogram_value.json @@ -0,0 +1,26 @@ +{ + "sensors": + [ + { + "kind":"LOGHIST", + "labels": + { + "metric":"ms" + }, + "ts":1509843723, + "log_hist": + { + "base":1.5, + "zeros_count":1, + "start_power":0, + "buckets": + [ + 0.5, + 0.25, + 0.25, + 0.5 + ] + } + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/merged.json b/library/cpp/monlib/encode/json/ut/merged.json new file mode 100644 index 0000000000..ea2c99a33c --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/merged.json @@ -0,0 +1,14 @@ +{ + "sensors": + [ + { + "kind":"RATE", + "labels": + { + "metric":"hello", + "label":"world" + }, + "value":1 + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/metrics.json b/library/cpp/monlib/encode/json/ut/metrics.json new file mode 100644 index 0000000000..2be4617d51 --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/metrics.json @@ -0,0 +1,43 @@ +{ + "labels": { + "project": "solomon", + "cluster": "man", + "service": "stockpile" + }, + "metrics": [ + { + "type": "DGAUGE", + "labels": { + "metric": "Memory" + }, + "value": 10 + }, + { + "type": "RATE", + "value": 1, + "labels": { "metric": "UserTime" } + }, + { + "type": "GAUGE", + "value": 3.14159, + "labels": { "export": "Oxygen", "metric": "QueueSize" }, + "ts": "2017-11-05T12:34:56.000Z", + "memOnly": true + }, + { + "type": "GAUGE", + "labels": { "metric": "Writes" }, + "timeseries": [ + { + "ts": "2017-08-28T12:32:11Z", + "value": -10 + }, + { + "value": 20, + "ts": 1503923187 + } + ] + } + ], + "ts": "2017-08-27T12:34:56Z" +} diff --git a/library/cpp/monlib/encode/json/ut/named_metrics.json b/library/cpp/monlib/encode/json/ut/named_metrics.json new file mode 100644 index 0000000000..98f93e8c39 --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/named_metrics.json @@ -0,0 +1,22 @@ +{ + "labels": { + "project": "solomon", + "cluster": "prod-sas", + "service": "stockpile" + }, + "metrics": [ + { + "type": "DGAUGE", + "name": "Memory", + "value": 1 + }, + { + "type": "DGAUGE", + "name": "QueueSize", + "labels": { + "export": "Oxygen" + }, + "value": 10 + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/sensors.json b/library/cpp/monlib/encode/json/ut/sensors.json new file mode 100644 index 0000000000..4d979a3c1e --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/sensors.json @@ -0,0 +1,40 @@ +{ + "commonLabels": { + "project": "solomon", + "cluster": "man", + "service": "stockpile" + }, + "sensors": [ + { + "labels": { + "metric": "Memory" + }, + "value": 10 + }, + { + "mode": "deriv", + "value": 1, + "labels": { "metric": "UserTime" } + }, + { + "value": 3.14159, + "labels": { "export": "Oxygen", "metric": "QueueSize" }, + "ts": "2017-11-05T12:34:56.000Z", + "memOnly": true + }, + { + "labels": { "metric": "Writes" }, + "timeseries": [ + { + "ts": "2017-08-28T12:32:11Z", + "value": -10 + }, + { + "value": 20, + "ts": 1503923187 + } + ] + } + ], + "ts": "2017-08-27T12:34:56Z" +} diff --git a/library/cpp/monlib/encode/json/ut/summary_inf.json b/library/cpp/monlib/encode/json/ut/summary_inf.json new file mode 100644 index 0000000000..625a6cd8ad --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/summary_inf.json @@ -0,0 +1,21 @@ +{ + "sensors": + [ + { + "kind":"DSUMMARY", + "labels": + { + "metric":"temperature" + }, + "ts":1509843723, + "summary": + { + "sum":"nan", + "min":"-inf", + "max":"inf", + "last":0.3, + "count":30 + } + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/summary_timeseries.json b/library/cpp/monlib/encode/json/ut/summary_timeseries.json new file mode 100644 index 0000000000..92007af3e6 --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/summary_timeseries.json @@ -0,0 +1,37 @@ +{ + "sensors": + [ + { + "kind":"DSUMMARY", + "labels": + { + "metric":"temperature" + }, + "timeseries": + [ + { + "ts":1509843723, + "summary": + { + "sum":0.8, + "min":-0.5, + "max":1, + "last":1, + "count":3 + } + }, + { + "ts":1509843738, + "summary": + { + "sum":-0.69, + "min":-1.5, + "max":1, + "last":0.01, + "count":5 + } + } + ] + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/summary_value.json b/library/cpp/monlib/encode/json/ut/summary_value.json new file mode 100644 index 0000000000..366394c5e1 --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/summary_value.json @@ -0,0 +1,21 @@ +{ + "sensors": + [ + { + "kind":"DSUMMARY", + "labels": + { + "metric":"temperature" + }, + "ts":1509843723, + "summary": + { + "sum":10, + "min":-0.5, + "max":0.5, + "last":0.3, + "count":30 + } + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/test_decode_to_encode.json b/library/cpp/monlib/encode/json/ut/test_decode_to_encode.json new file mode 100644 index 0000000000..65f0c5c6e2 --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/test_decode_to_encode.json @@ -0,0 +1,16 @@ +{ + "commonLabels": { + "project": "solomon", + "cluster": "man", + "service": "stockpile" + }, + "sensors": [ + { + "kind": "GAUGE", + "labels": { "export": "Oxygen", "metric": "QueueSize" }, + "ts": 1509885296, + "value": 3.14159 + } + ], + "ts": 1503837296 +} diff --git a/library/cpp/monlib/encode/json/ut/ya.make b/library/cpp/monlib/encode/json/ut/ya.make new file mode 100644 index 0000000000..e50c4f4903 --- /dev/null +++ b/library/cpp/monlib/encode/json/ut/ya.make @@ -0,0 +1,46 @@ +UNITTEST_FOR(library/cpp/monlib/encode/json) + +OWNER( + g:solomon + jamel +) + +SRCS( + json_decoder_ut.cpp + json_ut.cpp +) + +RESOURCE( + buffered_test.json /buffered_test.json + buffered_ts_merge.json /buffered_ts_merge.json + empty_series.json /empty_series.json + expected.json /expected.json + expected_buffered.json /expected_buffered.json + expected_cloud.json /expected_cloud.json + expected_cloud_buffered.json /expected_cloud_buffered.json + merged.json /merged.json + histogram_timeseries.json /histogram_timeseries.json + histogram_value.json /histogram_value.json + histogram_value_inf_before_bounds.json /histogram_value_inf_before_bounds.json + int_gauge.json /int_gauge.json + sensors.json /sensors.json + metrics.json /metrics.json + named_metrics.json /named_metrics.json + test_decode_to_encode.json /test_decode_to_encode.json + crash.json /crash.json + hist_crash.json /hist_crash.json + summary_value.json /summary_value.json + summary_inf.json /summary_inf.json + summary_timeseries.json /summary_timeseries.json + log_histogram_value.json /log_histogram_value.json + log_histogram_timeseries.json /log_histogram_timeseries.json +) + +PEERDIR( + library/cpp/json + library/cpp/monlib/consumers + library/cpp/monlib/encode/protobuf + library/cpp/resource +) + +END() diff --git a/library/cpp/monlib/encode/json/ya.make b/library/cpp/monlib/encode/json/ya.make new file mode 100644 index 0000000000..a50fc412a9 --- /dev/null +++ b/library/cpp/monlib/encode/json/ya.make @@ -0,0 +1,21 @@ +LIBRARY() + +OWNER( + g:solomon + jamel +) + +SRCS( + json_decoder.cpp + json_encoder.cpp +) + +PEERDIR( + library/cpp/monlib/encode + library/cpp/monlib/encode/buffered + library/cpp/monlib/exception + library/cpp/json + library/cpp/json/writer +) + +END() |