intermediate changes

ref:cde9a383711a11544ce7e107a78147fb96cc4029

5 files changed, 546 insertions, 0 deletions

diff --git a/library/cpp/monlib/encode/text/text.h b/library/cpp/monlib/encode/text/text.h
new file mode 100644
index 0000000000..6b2be3937b
--- /dev/null
+++ b/library/cpp/monlib/encode/text/text.h
@@ -0,0 +1,9 @@
+#pragma once
+
+#include <library/cpp/monlib/encode/encoder.h>
+
+class IOutputStream;
+
+namespace NMonitoring {
+    IMetricEncoderPtr EncoderText(IOutputStream* out, bool humanReadableTs = true);
+}
diff --git a/library/cpp/monlib/encode/text/text_encoder.cpp b/library/cpp/monlib/encode/text/text_encoder.cpp
new file mode 100644
index 0000000000..10336261f0
--- /dev/null
+++ b/library/cpp/monlib/encode/text/text_encoder.cpp
@@ -0,0 +1,226 @@
+#include "text.h"
+
+#include <library/cpp/monlib/encode/encoder_state.h>
+#include <library/cpp/monlib/metrics/labels.h>
+#include <library/cpp/monlib/metrics/metric_value.h>
+
+#include <util/datetime/base.h>
+#include <util/stream/format.h>
+
+namespace NMonitoring {
+    namespace {
+        class TEncoderText final: public IMetricEncoder {
+        public:
+            TEncoderText(IOutputStream* out, bool humanReadableTs)
+                : Out_(out)
+                , HumanReadableTs_(humanReadableTs)
+            {
+            }
+
+        private:
+            void OnStreamBegin() override {
+                State_.Expect(TEncoderState::EState::ROOT);
+            }
+
+            void OnStreamEnd() override {
+                State_.Expect(TEncoderState::EState::ROOT);
+            }
+
+            void OnCommonTime(TInstant time) override {
+                State_.Expect(TEncoderState::EState::ROOT);
+                CommonTime_ = time;
+                if (time != TInstant::Zero()) {
+                    Out_->Write(TStringBuf("common time: "));
+                    WriteTime(time);
+                    Out_->Write('\n');
+                }
+            }
+
+            void OnMetricBegin(EMetricType type) override {
+                State_.Switch(TEncoderState::EState::ROOT, TEncoderState::EState::METRIC);
+                ClearLastMetricState();
+                MetricType_ = type;
+            }
+
+            void OnMetricEnd() override {
+                State_.Switch(TEncoderState::EState::METRIC, TEncoderState::EState::ROOT);
+                WriteMetric();
+            }
+
+            void OnLabelsBegin() override {
+                if (State_ == TEncoderState::EState::METRIC) {
+                    State_ = TEncoderState::EState::METRIC_LABELS;
+                } else if (State_ == TEncoderState::EState::ROOT) {
+                    State_ = TEncoderState::EState::COMMON_LABELS;
+                } else {
+                    State_.ThrowInvalid("expected METRIC or ROOT");
+                }
+            }
+
+            void OnLabelsEnd() override {
+                if (State_ == TEncoderState::EState::METRIC_LABELS) {
+                    State_ = TEncoderState::EState::METRIC;
+                } else if (State_ == TEncoderState::EState::COMMON_LABELS) {
+                    State_ = TEncoderState::EState::ROOT;
+                    Out_->Write(TStringBuf("common labels: "));
+                    WriteLabels();
+                    Out_->Write('\n');
+                } else {
+                    State_.ThrowInvalid("expected LABELS or COMMON_LABELS");
+                }
+            }
+
+            void OnLabel(TStringBuf name, TStringBuf value) override {
+                Labels_.Add(name, value);
+            }
+
+            void OnDouble(TInstant time, double value) override {
+                State_.Expect(TEncoderState::EState::METRIC);
+                TimeSeries_.Add(time, value);
+            }
+
+            void OnInt64(TInstant time, i64 value) override {
+                State_.Expect(TEncoderState::EState::METRIC);
+                TimeSeries_.Add(time, value);
+            }
+
+            void OnUint64(TInstant time, ui64 value) override {
+                State_.Expect(TEncoderState::EState::METRIC);
+                TimeSeries_.Add(time, value);
+            }
+
+            void OnHistogram(TInstant time, IHistogramSnapshotPtr snapshot) override {
+                State_.Expect(TEncoderState::EState::METRIC);
+                TimeSeries_.Add(time, snapshot.Get());
+            }
+
+            void OnSummaryDouble(TInstant time, ISummaryDoubleSnapshotPtr snapshot) override {
+                State_.Expect(TEncoderState::EState::METRIC);
+                TimeSeries_.Add(time, snapshot.Get());
+            }
+
+            void OnLogHistogram(TInstant ts, TLogHistogramSnapshotPtr snapshot) override {
+                State_.Expect(TEncoderState::EState::METRIC);
+                TimeSeries_.Add(ts, snapshot.Get());
+            }
+
+            void Close() override {
+            }
+
+            void WriteTime(TInstant time) {
+                if (HumanReadableTs_) {
+                    char buf[64];
+                    auto len = FormatDate8601(buf, sizeof(buf), time.TimeT());
+                    Out_->Write(buf, len);
+                } else {
+                    (*Out_) << time.Seconds();
+                }
+            }
+
+            void WriteValue(EMetricValueType type, TMetricValue value) {
+                switch (type) {
+                case EMetricValueType::DOUBLE:
+                    (*Out_) << value.AsDouble();
+                    break;
+                case EMetricValueType::INT64:
+                    (*Out_) << value.AsInt64();
+                    break;
+                case EMetricValueType::UINT64:
+                    (*Out_) << value.AsUint64();
+                    break;
+                case EMetricValueType::HISTOGRAM:
+                    (*Out_) << *value.AsHistogram();
+                    break;
+                case EMetricValueType::SUMMARY:
+                    (*Out_) << *value.AsSummaryDouble();
+                    break;
+                case EMetricValueType::LOGHISTOGRAM:
+                    (*Out_) << *value.AsLogHistogram();
+                    break;
+                case EMetricValueType::UNKNOWN:
+                    ythrow yexception() << "unknown metric value type";
+                }
+            }
+
+            void WriteLabels() {
+                auto& out = *Out_;
+                const auto size = Labels_.Size();
+                size_t i = 0;
+
+                out << '{';
+                for (auto&& l : Labels_) {
+                    out << l.Name() << TStringBuf("='") << l.Value() << '\'';
+
+                    ++i;
+                    if (i < size) {
+                        out << TStringBuf(", ");
+                    }
+                };
+
+                out << '}';
+            }
+
+            void WriteMetric() {
+                // (1) type
+                TStringBuf typeStr = MetricTypeToStr(MetricType_);
+                (*Out_) << LeftPad(typeStr, MaxMetricTypeNameLength) << ' ';
+
+                // (2) name and labels
+                auto name = Labels_.Extract(TStringBuf("sensor"));
+                if (name) {
+                    if (name->Value().find(' ') != TString::npos) {
+                        (*Out_) << '"' << name->Value() << '"';
+                    } else {
+                        (*Out_) << name->Value();
+                    }
+                }
+                WriteLabels();
+
+                // (3) values
+                if (!TimeSeries_.Empty()) {
+                    TimeSeries_.SortByTs();
+                    Out_->Write(TStringBuf(" ["));
+                    for (size_t i = 0; i < TimeSeries_.Size(); i++) {
+                        if (i > 0) {
+                            Out_->Write(TStringBuf(", "));
+                        }
+
+                        const auto& point = TimeSeries_[i];
+                        if (point.GetTime() == CommonTime_ || point.GetTime() == TInstant::Zero()) {
+                            WriteValue(TimeSeries_.GetValueType(), point.GetValue());
+                        } else {
+                            Out_->Write('(');
+                            WriteTime(point.GetTime());
+                            Out_->Write(TStringBuf(", "));
+                            WriteValue(TimeSeries_.GetValueType(), point.GetValue());
+                            Out_->Write(')');
+                        }
+                    }
+                    Out_->Write(']');
+                }
+                Out_->Write('\n');
+            }
+
+            void ClearLastMetricState() {
+                MetricType_ = EMetricType::UNKNOWN;
+                Labels_.Clear();
+                TimeSeries_.Clear();
+            }
+
+        private:
+            TEncoderState State_;
+            IOutputStream* Out_;
+            bool HumanReadableTs_;
+            TInstant CommonTime_ = TInstant::Zero();
+            EMetricType MetricType_ = EMetricType::UNKNOWN;
+            TLabels Labels_;
+            TMetricTimeSeries TimeSeries_;
+        };
+
+    }
+
+    IMetricEncoderPtr EncoderText(IOutputStream* out, bool humanReadableTs) {
+        return MakeHolder<TEncoderText>(out, humanReadableTs);
+    }
+
+}
diff --git a/library/cpp/monlib/encode/text/text_encoder_ut.cpp b/library/cpp/monlib/encode/text/text_encoder_ut.cpp
new file mode 100644
index 0000000000..554b6f5fa9
--- /dev/null
+++ b/library/cpp/monlib/encode/text/text_encoder_ut.cpp
@@ -0,0 +1,283 @@
+#include "text.h"
+
+#include <library/cpp/monlib/metrics/histogram_collector.h>
+
+#include <library/cpp/testing/unittest/registar.h>
+
+using namespace NMonitoring;
+
+Y_UNIT_TEST_SUITE(TTextText) {
+    template <typename TFunc>
+    TString EncodeToString(bool humanReadableTs, TFunc fn) {
+        TStringStream ss;
+        IMetricEncoderPtr encoder = EncoderText(&ss, humanReadableTs);
+        fn(encoder.Get());
+        return ss.Str();
+    }
+
+    Y_UNIT_TEST(Empty) {
+        auto result = EncodeToString(true, [](IMetricEncoder* e) {
+            e->OnStreamBegin();
+            e->OnStreamEnd();
+        });
+        UNIT_ASSERT_STRINGS_EQUAL(result, "");
+    }
+
+    Y_UNIT_TEST(CommonPart) {
+        auto result = EncodeToString(true, [](IMetricEncoder* e) {
+            e->OnStreamBegin();
+            e->OnCommonTime(TInstant::ParseIso8601Deprecated("2017-01-02T03:04:05.006Z"));
+            {
+                e->OnLabelsBegin();
+                e->OnLabel("project", "solomon");
+                e->OnLabel("cluster", "man");
+                e->OnLabel("service", "stockpile");
+                e->OnLabelsEnd();
+            }
+            e->OnStreamEnd();
+        });
+        UNIT_ASSERT_STRINGS_EQUAL(result,
+                                  "common time: 2017-01-02T03:04:05Z\n"
+                                  "common labels: {project='solomon', cluster='man', service='stockpile'}\n");
+    }
+
+    Y_UNIT_TEST(Gauges) {
+        auto result = EncodeToString(true, [](IMetricEncoder* e) {
+            e->OnStreamBegin();
+            { // no values
+                e->OnMetricBegin(EMetricType::GAUGE);
+                {
+                    e->OnLabelsBegin();
+                    e->OnLabel("sensor", "cpuUsage");
+                    e->OnLabelsEnd();
+                }
+                e->OnMetricEnd();
+            }
+            { // one value no ts
+                e->OnMetricBegin(EMetricType::GAUGE);
+                {
+                    e->OnLabelsBegin();
+                    e->OnLabel("sensor", "diskUsage");
+                    e->OnLabel("disk", "sda1");
+                    e->OnLabelsEnd();
+                }
+                e->OnDouble(TInstant::Zero(), 1000);
+                e->OnMetricEnd();
+            }
+            { // one value with ts
+                e->OnMetricBegin(EMetricType::GAUGE);
+                {
+                    e->OnLabelsBegin();
+                    e->OnLabel("sensor", "memoryUsage");
+                    e->OnLabel("host", "solomon-man-00");
+                    e->OnLabel("dc", "man");
+                    e->OnLabelsEnd();
+                }
+                e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 1000);
+                e->OnMetricEnd();
+            }
+            { // many values
+                e->OnMetricBegin(EMetricType::GAUGE);
+                {
+                    e->OnLabelsBegin();
+                    e->OnLabel("sensor", "bytesRx");
+                    e->OnLabel("host", "solomon-sas-01");
+                    e->OnLabel("dc", "sas");
+                    e->OnLabelsEnd();
+                }
+                e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 2);
+                e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:05Z"), 4);
+                e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:10Z"), 8);
+                e->OnMetricEnd();
+            }
+            e->OnStreamEnd();
+        });
+        UNIT_ASSERT_STRINGS_EQUAL(result,
+                                  "    GAUGE cpuUsage{}\n"
+                                  "    GAUGE diskUsage{disk='sda1'} [1000]\n"
+                                  "    GAUGE memoryUsage{host='solomon-man-00', dc='man'} [(2017-12-02T12:00:00Z, 1000)]\n"
+                                  "    GAUGE bytesRx{host='solomon-sas-01', dc='sas'} [(2017-12-02T12:00:00Z, 2), (2017-12-02T12:00:05Z, 4), (2017-12-02T12:00:10Z, 8)]\n");
+    }
+
+    Y_UNIT_TEST(IntGauges) {
+        auto result = EncodeToString(true, [](IMetricEncoder* e) {
+            e->OnStreamBegin();
+            { // no values
+                e->OnMetricBegin(EMetricType::IGAUGE);
+                {
+                    e->OnLabelsBegin();
+                    e->OnLabel("sensor", "cpuUsage");
+                    e->OnLabelsEnd();
+                }
+                e->OnMetricEnd();
+            }
+            { // one value no ts
+                e->OnMetricBegin(EMetricType::IGAUGE);
+                {
+                    e->OnLabelsBegin();
+                    e->OnLabel("sensor", "diskUsage");
+                    e->OnLabel("disk", "sda1");
+                    e->OnLabelsEnd();
+                }
+                e->OnDouble(TInstant::Zero(), 1000);
+                e->OnMetricEnd();
+            }
+            { // one value with ts
+                e->OnMetricBegin(EMetricType::IGAUGE);
+                {
+                    e->OnLabelsBegin();
+                    e->OnLabel("sensor", "memoryUsage");
+                    e->OnLabel("host", "solomon-man-00");
+                    e->OnLabel("dc", "man");
+                    e->OnLabelsEnd();
+                }
+                e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 1000);
+                e->OnMetricEnd();
+            }
+            { // many values
+                e->OnMetricBegin(EMetricType::IGAUGE);
+                {
+                    e->OnLabelsBegin();
+                    e->OnLabel("sensor", "bytesRx");
+                    e->OnLabel("host", "solomon-sas-01");
+                    e->OnLabel("dc", "sas");
+                    e->OnLabelsEnd();
+                }
+                e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 2);
+                e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:05Z"), 4);
+                e->OnDouble(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:10Z"), 8);
+                e->OnMetricEnd();
+            }
+            e->OnStreamEnd();
+        });
+        UNIT_ASSERT_STRINGS_EQUAL(result,
+                                  "   IGAUGE cpuUsage{}\n"
+                                  "   IGAUGE diskUsage{disk='sda1'} [1000]\n"
+                                  "   IGAUGE memoryUsage{host='solomon-man-00', dc='man'} [(2017-12-02T12:00:00Z, 1000)]\n"
+                                  "   IGAUGE bytesRx{host='solomon-sas-01', dc='sas'} [(2017-12-02T12:00:00Z, 2), (2017-12-02T12:00:05Z, 4), (2017-12-02T12:00:10Z, 8)]\n");
+    }
+
+    Y_UNIT_TEST(Counters) {
+        auto doEncode = [](IMetricEncoder* e) {
+            e->OnStreamBegin();
+            { // no values
+                e->OnMetricBegin(EMetricType::COUNTER);
+                {
+                    e->OnLabelsBegin();
+                    e->OnLabel("sensor", "cpuUsage");
+                    e->OnLabelsEnd();
+                }
+                e->OnMetricEnd();
+            }
+            { // one value no ts
+                e->OnMetricBegin(EMetricType::COUNTER);
+                {
+                    e->OnLabelsBegin();
+                    e->OnLabel("sensor", "diskUsage");
+                    e->OnLabel("disk", "sda1");
+                    e->OnLabelsEnd();
+                }
+                e->OnUint64(TInstant::Zero(), 1000);
+                e->OnMetricEnd();
+            }
+            { // one value with ts
+                e->OnMetricBegin(EMetricType::COUNTER);
+                {
+                    e->OnLabelsBegin();
+                    e->OnLabel("sensor", "memoryUsage");
+                    e->OnLabel("host", "solomon-man-00");
+                    e->OnLabel("dc", "man");
+                    e->OnLabelsEnd();
+                }
+                e->OnUint64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 1000);
+                e->OnMetricEnd();
+            }
+            { // many values
+                e->OnMetricBegin(EMetricType::COUNTER);
+                {
+                    e->OnLabelsBegin();
+                    e->OnLabel("sensor", "bytesRx");
+                    e->OnLabel("host", "solomon-sas-01");
+                    e->OnLabel("dc", "sas");
+                    e->OnLabelsEnd();
+                }
+                e->OnUint64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:00Z"), 2);
+                e->OnUint64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:05Z"), 4);
+                e->OnUint64(TInstant::ParseIso8601Deprecated("2017-12-02T12:00:10Z"), 8);
+                e->OnMetricEnd();
+            }
+            e->OnStreamEnd();
+        };
+
+        auto result1 = EncodeToString(false, doEncode);
+        UNIT_ASSERT_STRINGS_EQUAL(result1,
+                                  "  COUNTER cpuUsage{}\n"
+                                  "  COUNTER diskUsage{disk='sda1'} [1000]\n"
+                                  "  COUNTER memoryUsage{host='solomon-man-00', dc='man'} [(1512216000, 1000)]\n"
+                                  "  COUNTER bytesRx{host='solomon-sas-01', dc='sas'} [(1512216000, 2), (1512216005, 4), (1512216010, 8)]\n");
+
+        auto result2 = EncodeToString(true, doEncode);
+        UNIT_ASSERT_STRINGS_EQUAL(result2,
+                                  "  COUNTER cpuUsage{}\n"
+                                  "  COUNTER diskUsage{disk='sda1'} [1000]\n"
+                                  "  COUNTER memoryUsage{host='solomon-man-00', dc='man'} [(2017-12-02T12:00:00Z, 1000)]\n"
+                                  "  COUNTER bytesRx{host='solomon-sas-01', dc='sas'} [(2017-12-02T12:00:00Z, 2), (2017-12-02T12:00:05Z, 4), (2017-12-02T12:00:10Z, 8)]\n");
+    }
+
+    Y_UNIT_TEST(Histograms) {
+        auto h = ExplicitHistogram({1, 2, 3, 4, 5});
+        h->Collect(3);
+        h->Collect(5, 7);
+        h->Collect(13);
+        auto s = h->Snapshot();
+
+        TString result = EncodeToString(true, [s](IMetricEncoder* e) {
+            e->OnStreamBegin();
+            {
+                e->OnMetricBegin(EMetricType::HIST);
+                {
+                    e->OnLabelsBegin();
+                    e->OnLabel("sensor", "readTimeMillis");
+                    e->OnLabelsEnd();
+                }
+                e->OnHistogram(TInstant::Zero(), s);
+                e->OnMetricEnd();
+            }
+            {
+                e->OnMetricBegin(EMetricType::HIST_RATE);
+                {
+                    e->OnLabelsBegin();
+                    e->OnLabel("sensor", "writeTimeMillis");
+                    e->OnLabelsEnd();
+                }
+                e->OnHistogram(TInstant::Zero(), s);
+                e->OnMetricEnd();
+            }
+            e->OnStreamEnd();
+        });
+
+        UNIT_ASSERT_STRINGS_EQUAL(result,
+                                  "     HIST readTimeMillis{} [{1: 0, 2: 0, 3: 1, 4: 0, 5: 7, inf: 1}]\n"
+                                  "HIST_RATE writeTimeMillis{} [{1: 0, 2: 0, 3: 1, 4: 0, 5: 7, inf: 1}]\n");
+    }
+
+    Y_UNIT_TEST(Summary) {
+        auto s = MakeIntrusive<TSummaryDoubleSnapshot>(10.1, -0.45, 0.478, 0.3, 30u);
+        TString result = EncodeToString(true, [s](IMetricEncoder* e) {
+            e->OnStreamBegin();
+            {
+                e->OnMetricBegin(EMetricType::DSUMMARY);
+                {
+                    e->OnLabelsBegin();
+                    e->OnLabel("sensor", "temperature");
+                    e->OnLabelsEnd();
+                }
+                e->OnSummaryDouble(TInstant::Zero(), s);
+                e->OnMetricEnd();
+            }
+            e->OnStreamEnd();
+        });
+        UNIT_ASSERT_STRINGS_EQUAL(result,
+                " DSUMMARY temperature{} [{sum: 10.1, min: -0.45, max: 0.478, last: 0.3, count: 30}]\n");
+    }
+}
diff --git a/library/cpp/monlib/encode/text/ut/ya.make b/library/cpp/monlib/encode/text/ut/ya.make
new file mode 100644
index 0000000000..df23a252d1
--- /dev/null
+++ b/library/cpp/monlib/encode/text/ut/ya.make
@@ -0,0 +1,12 @@
+UNITTEST_FOR(library/cpp/monlib/encode/text)
+
+OWNER(
+    g:solomon
+    jamel
+)
+
+SRCS(
+    text_encoder_ut.cpp
+)
+
+END()
diff --git a/library/cpp/monlib/encode/text/ya.make b/library/cpp/monlib/encode/text/ya.make
new file mode 100644
index 0000000000..d296c78c1b
--- /dev/null
+++ b/library/cpp/monlib/encode/text/ya.make
@@ -0,0 +1,16 @@
+LIBRARY()
+
+OWNER(
+    g:solomon
+    jamel
+)
+
+SRCS(
+    text_encoder.cpp
+)
+
+PEERDIR(
+    library/cpp/monlib/encode
+)
+
+END()