diff options
author | msherbakov <msherbakov@yandex-team.ru> | 2022-02-10 16:49:16 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:49:16 +0300 |
commit | c224a621661ddd69699f9476922eb316607ef57e (patch) | |
tree | 33f4d878aa0a9faa964005e06bfab0272313aa71 /library/cpp/monlib/encode/legacy_protobuf | |
parent | 29d0b2eeae154d04156e0698067c0c21a97ea61d (diff) | |
download | ydb-c224a621661ddd69699f9476922eb316607ef57e.tar.gz |
Restoring authorship annotation for <msherbakov@yandex-team.ru>. Commit 1 of 2.
Diffstat (limited to 'library/cpp/monlib/encode/legacy_protobuf')
7 files changed, 959 insertions, 959 deletions
diff --git a/library/cpp/monlib/encode/legacy_protobuf/legacy_proto_decoder.cpp b/library/cpp/monlib/encode/legacy_protobuf/legacy_proto_decoder.cpp index f87a2d7e8f..4eba48619c 100644 --- a/library/cpp/monlib/encode/legacy_protobuf/legacy_proto_decoder.cpp +++ b/library/cpp/monlib/encode/legacy_protobuf/legacy_proto_decoder.cpp @@ -1,527 +1,527 @@ -#include "legacy_protobuf.h" - +#include "legacy_protobuf.h" + #include <library/cpp/monlib/encode/legacy_protobuf/protos/metric_meta.pb.h> #include <library/cpp/monlib/metrics/metric_consumer.h> #include <library/cpp/monlib/metrics/labels.h> - -#include <util/generic/yexception.h> -#include <util/generic/maybe.h> -#include <util/datetime/base.h> + +#include <util/generic/yexception.h> +#include <util/generic/maybe.h> +#include <util/datetime/base.h> #include <util/string/split.h> - + #include <google/protobuf/reflection.h> - -#include <algorithm> - -#ifdef LEGACY_PB_TRACE -#define TRACE(msg) \ - Cerr << msg << Endl -#else -#define TRACE(...) ; -#endif - -namespace NMonitoring { - namespace { + +#include <algorithm> + +#ifdef LEGACY_PB_TRACE +#define TRACE(msg) \ + Cerr << msg << Endl +#else +#define TRACE(...) ; +#endif + +namespace NMonitoring { + namespace { using TMaybeMeta = TMaybe<NMonProto::TMetricMeta>; - - TString ReadLabelValue(const NProtoBuf::Message& msg, const NProtoBuf::FieldDescriptor* d, const NProtoBuf::Reflection& r) { - using namespace NProtoBuf; - - switch (d->type()) { - case FieldDescriptor::TYPE_UINT32: - return ::ToString(r.GetUInt32(msg, d)); - case FieldDescriptor::TYPE_UINT64: - return ::ToString(r.GetUInt64(msg, d)); - case FieldDescriptor::TYPE_STRING: - return r.GetString(msg, d); - case FieldDescriptor::TYPE_ENUM: { - auto val = r.GetEnumValue(msg, d); - auto* valDesc = d->enum_type()->FindValueByNumber(val); - return valDesc->name(); - } - - default: - ythrow yexception() << "type " << d->type_name() << " cannot be used as a field value"; - } - - return {}; - } - - double ReadFieldAsDouble(const NProtoBuf::Message& msg, const NProtoBuf::FieldDescriptor* d, const NProtoBuf::Reflection& r) { - using namespace NProtoBuf; - - switch (d->type()) { - case FieldDescriptor::TYPE_DOUBLE: - return r.GetDouble(msg, d); - case FieldDescriptor::TYPE_BOOL: - return r.GetBool(msg, d) ? 1 : 0; - case FieldDescriptor::TYPE_INT32: - return r.GetInt32(msg, d); - case FieldDescriptor::TYPE_INT64: - return r.GetInt64(msg, d); - case FieldDescriptor::TYPE_UINT32: - return r.GetUInt32(msg, d); - case FieldDescriptor::TYPE_UINT64: - return r.GetUInt64(msg, d); - case FieldDescriptor::TYPE_SINT32: - return r.GetInt32(msg, d); - case FieldDescriptor::TYPE_SINT64: - return r.GetInt64(msg, d); - case FieldDescriptor::TYPE_FIXED32: - return r.GetUInt32(msg, d); - case FieldDescriptor::TYPE_FIXED64: - return r.GetUInt64(msg, d); - case FieldDescriptor::TYPE_SFIXED32: - return r.GetInt32(msg, d); - case FieldDescriptor::TYPE_SFIXED64: - return r.GetInt64(msg, d); - case FieldDescriptor::TYPE_FLOAT: - return r.GetFloat(msg, d); - case FieldDescriptor::TYPE_ENUM: - return r.GetEnumValue(msg, d); - default: - ythrow yexception() << "type " << d->type_name() << " cannot be used as a field value"; - } - - return std::numeric_limits<double>::quiet_NaN(); - } - - double ReadRepeatedAsDouble(const NProtoBuf::Message& msg, const NProtoBuf::FieldDescriptor* d, const NProtoBuf::Reflection& r, size_t i) { - using namespace NProtoBuf; - - switch (d->type()) { - case FieldDescriptor::TYPE_DOUBLE: - return r.GetRepeatedDouble(msg, d, i); - case FieldDescriptor::TYPE_BOOL: - return r.GetRepeatedBool(msg, d, i) ? 1 : 0; - case FieldDescriptor::TYPE_INT32: - return r.GetRepeatedInt32(msg, d, i); - case FieldDescriptor::TYPE_INT64: - return r.GetRepeatedInt64(msg, d, i); - case FieldDescriptor::TYPE_UINT32: - return r.GetRepeatedUInt32(msg, d, i); - case FieldDescriptor::TYPE_UINT64: - return r.GetRepeatedUInt64(msg, d, i); - case FieldDescriptor::TYPE_SINT32: - return r.GetRepeatedInt32(msg, d, i); - case FieldDescriptor::TYPE_SINT64: - return r.GetRepeatedInt64(msg, d, i); - case FieldDescriptor::TYPE_FIXED32: - return r.GetRepeatedUInt32(msg, d, i); - case FieldDescriptor::TYPE_FIXED64: - return r.GetRepeatedUInt64(msg, d, i); - case FieldDescriptor::TYPE_SFIXED32: - return r.GetRepeatedInt32(msg, d, i); - case FieldDescriptor::TYPE_SFIXED64: - return r.GetRepeatedInt64(msg, d, i); - case FieldDescriptor::TYPE_FLOAT: - return r.GetRepeatedFloat(msg, d, i); - case FieldDescriptor::TYPE_ENUM: - return r.GetRepeatedEnumValue(msg, d, i); - default: - ythrow yexception() << "type " << d->type_name() << " cannot be used as a field value"; - } - - return std::numeric_limits<double>::quiet_NaN(); - } - - TString LabelFromField(const NProtoBuf::Message& msg, const TString& name) { - const auto* fieldDesc = msg.GetDescriptor()->FindFieldByName(name); - const auto* reflection = msg.GetReflection(); - Y_ENSURE(fieldDesc && reflection, "Unable to get meta for field " << name); - - auto s = ReadLabelValue(msg, fieldDesc, *reflection); - std::replace(std::begin(s), s.vend(), ' ', '_'); - - return s; - } - - TMaybeMeta MaybeGetMeta(const NProtoBuf::FieldOptions& opts) { + + TString ReadLabelValue(const NProtoBuf::Message& msg, const NProtoBuf::FieldDescriptor* d, const NProtoBuf::Reflection& r) { + using namespace NProtoBuf; + + switch (d->type()) { + case FieldDescriptor::TYPE_UINT32: + return ::ToString(r.GetUInt32(msg, d)); + case FieldDescriptor::TYPE_UINT64: + return ::ToString(r.GetUInt64(msg, d)); + case FieldDescriptor::TYPE_STRING: + return r.GetString(msg, d); + case FieldDescriptor::TYPE_ENUM: { + auto val = r.GetEnumValue(msg, d); + auto* valDesc = d->enum_type()->FindValueByNumber(val); + return valDesc->name(); + } + + default: + ythrow yexception() << "type " << d->type_name() << " cannot be used as a field value"; + } + + return {}; + } + + double ReadFieldAsDouble(const NProtoBuf::Message& msg, const NProtoBuf::FieldDescriptor* d, const NProtoBuf::Reflection& r) { + using namespace NProtoBuf; + + switch (d->type()) { + case FieldDescriptor::TYPE_DOUBLE: + return r.GetDouble(msg, d); + case FieldDescriptor::TYPE_BOOL: + return r.GetBool(msg, d) ? 1 : 0; + case FieldDescriptor::TYPE_INT32: + return r.GetInt32(msg, d); + case FieldDescriptor::TYPE_INT64: + return r.GetInt64(msg, d); + case FieldDescriptor::TYPE_UINT32: + return r.GetUInt32(msg, d); + case FieldDescriptor::TYPE_UINT64: + return r.GetUInt64(msg, d); + case FieldDescriptor::TYPE_SINT32: + return r.GetInt32(msg, d); + case FieldDescriptor::TYPE_SINT64: + return r.GetInt64(msg, d); + case FieldDescriptor::TYPE_FIXED32: + return r.GetUInt32(msg, d); + case FieldDescriptor::TYPE_FIXED64: + return r.GetUInt64(msg, d); + case FieldDescriptor::TYPE_SFIXED32: + return r.GetInt32(msg, d); + case FieldDescriptor::TYPE_SFIXED64: + return r.GetInt64(msg, d); + case FieldDescriptor::TYPE_FLOAT: + return r.GetFloat(msg, d); + case FieldDescriptor::TYPE_ENUM: + return r.GetEnumValue(msg, d); + default: + ythrow yexception() << "type " << d->type_name() << " cannot be used as a field value"; + } + + return std::numeric_limits<double>::quiet_NaN(); + } + + double ReadRepeatedAsDouble(const NProtoBuf::Message& msg, const NProtoBuf::FieldDescriptor* d, const NProtoBuf::Reflection& r, size_t i) { + using namespace NProtoBuf; + + switch (d->type()) { + case FieldDescriptor::TYPE_DOUBLE: + return r.GetRepeatedDouble(msg, d, i); + case FieldDescriptor::TYPE_BOOL: + return r.GetRepeatedBool(msg, d, i) ? 1 : 0; + case FieldDescriptor::TYPE_INT32: + return r.GetRepeatedInt32(msg, d, i); + case FieldDescriptor::TYPE_INT64: + return r.GetRepeatedInt64(msg, d, i); + case FieldDescriptor::TYPE_UINT32: + return r.GetRepeatedUInt32(msg, d, i); + case FieldDescriptor::TYPE_UINT64: + return r.GetRepeatedUInt64(msg, d, i); + case FieldDescriptor::TYPE_SINT32: + return r.GetRepeatedInt32(msg, d, i); + case FieldDescriptor::TYPE_SINT64: + return r.GetRepeatedInt64(msg, d, i); + case FieldDescriptor::TYPE_FIXED32: + return r.GetRepeatedUInt32(msg, d, i); + case FieldDescriptor::TYPE_FIXED64: + return r.GetRepeatedUInt64(msg, d, i); + case FieldDescriptor::TYPE_SFIXED32: + return r.GetRepeatedInt32(msg, d, i); + case FieldDescriptor::TYPE_SFIXED64: + return r.GetRepeatedInt64(msg, d, i); + case FieldDescriptor::TYPE_FLOAT: + return r.GetRepeatedFloat(msg, d, i); + case FieldDescriptor::TYPE_ENUM: + return r.GetRepeatedEnumValue(msg, d, i); + default: + ythrow yexception() << "type " << d->type_name() << " cannot be used as a field value"; + } + + return std::numeric_limits<double>::quiet_NaN(); + } + + TString LabelFromField(const NProtoBuf::Message& msg, const TString& name) { + const auto* fieldDesc = msg.GetDescriptor()->FindFieldByName(name); + const auto* reflection = msg.GetReflection(); + Y_ENSURE(fieldDesc && reflection, "Unable to get meta for field " << name); + + auto s = ReadLabelValue(msg, fieldDesc, *reflection); + std::replace(std::begin(s), s.vend(), ' ', '_'); + + return s; + } + + TMaybeMeta MaybeGetMeta(const NProtoBuf::FieldOptions& opts) { if (opts.HasExtension(NMonProto::Metric)) { return opts.GetExtension(NMonProto::Metric); - } - - return Nothing(); - } - - class ILabelGetter: public TThrRefBase { - public: - enum class EType { - Fixed = 1, - Lazy = 2, - }; - - virtual TLabel Get(const NProtoBuf::Message&) = 0; - virtual EType Type() const = 0; - }; - - class TFixedLabel: public ILabelGetter { - public: - explicit TFixedLabel(TLabel&& l) - : Label_{std::move(l)} - { - TRACE("found fixed label " << l); - } - + } + + return Nothing(); + } + + class ILabelGetter: public TThrRefBase { + public: + enum class EType { + Fixed = 1, + Lazy = 2, + }; + + virtual TLabel Get(const NProtoBuf::Message&) = 0; + virtual EType Type() const = 0; + }; + + class TFixedLabel: public ILabelGetter { + public: + explicit TFixedLabel(TLabel&& l) + : Label_{std::move(l)} + { + TRACE("found fixed label " << l); + } + EType Type() const override { - return EType::Fixed; - } + return EType::Fixed; + } TLabel Get(const NProtoBuf::Message&) override { - return Label_; - } - - private: - TLabel Label_; - }; - - using TFunction = std::function<TLabel(const NProtoBuf::Message&)>; - - class TLazyLabel: public ILabelGetter { - public: - TLazyLabel(TFunction&& fn) - : Fn_{std::move(fn)} - { - TRACE("found lazy label"); - } - + return Label_; + } + + private: + TLabel Label_; + }; + + using TFunction = std::function<TLabel(const NProtoBuf::Message&)>; + + class TLazyLabel: public ILabelGetter { + public: + TLazyLabel(TFunction&& fn) + : Fn_{std::move(fn)} + { + TRACE("found lazy label"); + } + EType Type() const override { - return EType::Lazy; - } + return EType::Lazy; + } TLabel Get(const NProtoBuf::Message& msg) override { - return Fn_(msg); - } - - private: - TFunction Fn_; - }; - - class TDecoderContext { - public: - void Init(const NProtoBuf::Message* msg) { - Message_ = msg; - Y_ENSURE(Message_); - Reflection_ = msg->GetReflection(); - Y_ENSURE(Reflection_); - - for (auto it = Labels_.begin(); it != Labels_.end(); ++it) { - if ((*it)->Type() == ILabelGetter::EType::Lazy) { - auto l = (*it)->Get(Message()); - *it = ::MakeIntrusive<TFixedLabel>(std::move(l)); - } else { - auto l = (*it)->Get(Message()); - } - } - } - - void Clear() noexcept { - Message_ = nullptr; - Reflection_ = nullptr; - } - + return Fn_(msg); + } + + private: + TFunction Fn_; + }; + + class TDecoderContext { + public: + void Init(const NProtoBuf::Message* msg) { + Message_ = msg; + Y_ENSURE(Message_); + Reflection_ = msg->GetReflection(); + Y_ENSURE(Reflection_); + + for (auto it = Labels_.begin(); it != Labels_.end(); ++it) { + if ((*it)->Type() == ILabelGetter::EType::Lazy) { + auto l = (*it)->Get(Message()); + *it = ::MakeIntrusive<TFixedLabel>(std::move(l)); + } else { + auto l = (*it)->Get(Message()); + } + } + } + + void Clear() noexcept { + Message_ = nullptr; + Reflection_ = nullptr; + } + TDecoderContext CreateChildFromMeta(const NMonProto::TMetricMeta& metricMeta, const TString& name, i64 repeatedIdx = -1) { - TDecoderContext child{*this}; - child.Clear(); - + TDecoderContext child{*this}; + child.Clear(); + if (metricMeta.HasCustomPath()) { if (const auto& nodePath = metricMeta.GetCustomPath()) { child.AppendPath(nodePath); } } else if (metricMeta.GetPath()) { - child.AppendPath(name); - } - + child.AppendPath(name); + } + if (metricMeta.HasKeys()) { child.ParseKeys(metricMeta.GetKeys(), repeatedIdx); - } - - return child; - } - + } + + return child; + } + TDecoderContext CreateChildFromRepeatedScalar(const NMonProto::TMetricMeta& metricMeta, i64 repeatedIdx = -1) { - TDecoderContext child{*this}; - child.Clear(); - + TDecoderContext child{*this}; + child.Clear(); + if (metricMeta.HasKeys()) { child.ParseKeys(metricMeta.GetKeys(), repeatedIdx); - } - - return child; - } - + } + + return child; + } + TDecoderContext CreateChildFromEls(const TString& name, const NMonProto::TExtraLabelMetrics& metrics, size_t idx, TMaybeMeta maybeMeta) { - TDecoderContext child{*this}; - child.Clear(); - - auto usePath = [&maybeMeta] { - return !maybeMeta->HasPath() || maybeMeta->GetPath(); - }; - + TDecoderContext child{*this}; + child.Clear(); + + auto usePath = [&maybeMeta] { + return !maybeMeta->HasPath() || maybeMeta->GetPath(); + }; + if (!name.empty() && (!maybeMeta || usePath())) { - child.AppendPath(name); - } - - child.Labels_.push_back(::MakeIntrusive<TLazyLabel>( + child.AppendPath(name); + } + + child.Labels_.push_back(::MakeIntrusive<TLazyLabel>( [ labelName = metrics.GetlabelName(), idx, &metrics ](const auto&) { const auto& val = metrics.Getvalues(idx); - TString labelVal; - const auto uintLabel = val.GetlabelValueUint(); - - if (uintLabel) { - labelVal = ::ToString(uintLabel); - } else { - labelVal = val.GetlabelValue(); - } - - return TLabel{labelName, labelVal}; - })); - - return child; - } - - void ParseKeys(TStringBuf keys, i64 repeatedIdx = -1) { - auto parts = StringSplitter(keys) - .Split(' ') - .SkipEmpty(); - - for (auto part : parts) { - auto str = part.Token(); - - TStringBuf lhs, rhs; - - const bool isDynamic = str.TrySplit(':', lhs, rhs); + TString labelVal; + const auto uintLabel = val.GetlabelValueUint(); + + if (uintLabel) { + labelVal = ::ToString(uintLabel); + } else { + labelVal = val.GetlabelValue(); + } + + return TLabel{labelName, labelVal}; + })); + + return child; + } + + void ParseKeys(TStringBuf keys, i64 repeatedIdx = -1) { + auto parts = StringSplitter(keys) + .Split(' ') + .SkipEmpty(); + + for (auto part : parts) { + auto str = part.Token(); + + TStringBuf lhs, rhs; + + const bool isDynamic = str.TrySplit(':', lhs, rhs); const bool isIndexing = isDynamic && rhs == TStringBuf("#"); - - if (isIndexing) { - TRACE("parsed index labels"); - - // <label_name>:# means that we should use index of the repeated - // field as label value - Y_ENSURE(repeatedIdx != -1); - Labels_.push_back(::MakeIntrusive<TLazyLabel>([=](const auto&) { - return TLabel{lhs, ::ToString(repeatedIdx)}; - })); - } else if (isDynamic) { - TRACE("parsed dynamic labels"); - - // <label_name>:<field_name> means that we need to take label value - // later from message's field - Labels_.push_back(::MakeIntrusive<TLazyLabel>([=](const auto& msg) { - return TLabel{lhs, LabelFromField(msg, TString{rhs})}; - })); - } else if (str.TrySplit('=', lhs, rhs)) { - TRACE("parsed static labels"); - - // <label_name>=<label_value> stands for constant label - Labels_.push_back(::MakeIntrusive<TFixedLabel>(TLabel{lhs, rhs})); - } else { - ythrow yexception() << "Incorrect Keys format"; - } - } - } - - void AppendPath(TStringBuf fieldName) { - Path_ += '/'; - Path_ += fieldName; - } - - const TString& Path() const { - return Path_; - } - - TLabels Labels() const { - TLabels result; - for (auto&& l : Labels_) { - result.Add(l->Get(Message())); - } - - return result; - } - - const NProtoBuf::Message& Message() const { - Y_VERIFY_DEBUG(Message_); - return *Message_; - } - - const NProtoBuf::Reflection& Reflection() const { - return *Reflection_; - } - - private: - const NProtoBuf::Message* Message_{nullptr}; - const NProtoBuf::Reflection* Reflection_{nullptr}; - - TString Path_; - TVector<TIntrusivePtr<ILabelGetter>> Labels_; - }; - - class TDecoder { - public: + + if (isIndexing) { + TRACE("parsed index labels"); + + // <label_name>:# means that we should use index of the repeated + // field as label value + Y_ENSURE(repeatedIdx != -1); + Labels_.push_back(::MakeIntrusive<TLazyLabel>([=](const auto&) { + return TLabel{lhs, ::ToString(repeatedIdx)}; + })); + } else if (isDynamic) { + TRACE("parsed dynamic labels"); + + // <label_name>:<field_name> means that we need to take label value + // later from message's field + Labels_.push_back(::MakeIntrusive<TLazyLabel>([=](const auto& msg) { + return TLabel{lhs, LabelFromField(msg, TString{rhs})}; + })); + } else if (str.TrySplit('=', lhs, rhs)) { + TRACE("parsed static labels"); + + // <label_name>=<label_value> stands for constant label + Labels_.push_back(::MakeIntrusive<TFixedLabel>(TLabel{lhs, rhs})); + } else { + ythrow yexception() << "Incorrect Keys format"; + } + } + } + + void AppendPath(TStringBuf fieldName) { + Path_ += '/'; + Path_ += fieldName; + } + + const TString& Path() const { + return Path_; + } + + TLabels Labels() const { + TLabels result; + for (auto&& l : Labels_) { + result.Add(l->Get(Message())); + } + + return result; + } + + const NProtoBuf::Message& Message() const { + Y_VERIFY_DEBUG(Message_); + return *Message_; + } + + const NProtoBuf::Reflection& Reflection() const { + return *Reflection_; + } + + private: + const NProtoBuf::Message* Message_{nullptr}; + const NProtoBuf::Reflection* Reflection_{nullptr}; + + TString Path_; + TVector<TIntrusivePtr<ILabelGetter>> Labels_; + }; + + class TDecoder { + public: TDecoder(IMetricConsumer* consumer, const NProtoBuf::Message& message, TInstant timestamp) - : Consumer_{consumer} - , Message_{message} - , Timestamp_{timestamp} - { - } - - void Decode() const { - Consumer_->OnStreamBegin(); - DecodeToStream(); - Consumer_->OnStreamEnd(); - } - - void DecodeToStream() const { - DecodeImpl(Message_, {}); - } - - private: + : Consumer_{consumer} + , Message_{message} + , Timestamp_{timestamp} + { + } + + void Decode() const { + Consumer_->OnStreamBegin(); + DecodeToStream(); + Consumer_->OnStreamEnd(); + } + + void DecodeToStream() const { + DecodeImpl(Message_, {}); + } + + private: static const NMonProto::TExtraLabelMetrics& ExtractExtraMetrics(TDecoderContext& ctx, const NProtoBuf::FieldDescriptor& f) { - const auto& parent = ctx.Message(); - const auto& reflection = ctx.Reflection(); - auto& subMessage = reflection.GetMessage(parent, &f); - + const auto& parent = ctx.Message(); + const auto& reflection = ctx.Reflection(); + auto& subMessage = reflection.GetMessage(parent, &f); + return dynamic_cast<const NMonProto::TExtraLabelMetrics&>(subMessage); - } - - void DecodeImpl(const NProtoBuf::Message& msg, TDecoderContext ctx) const { - std::vector<const NProtoBuf::FieldDescriptor*> fields; - - ctx.Init(&msg); - - ctx.Reflection().ListFields(msg, &fields); - - for (const auto* f : fields) { - Y_ENSURE(f); - - const auto& opts = f->options(); - const auto isMessage = f->type() == NProtoBuf::FieldDescriptor::TYPE_MESSAGE; + } + + void DecodeImpl(const NProtoBuf::Message& msg, TDecoderContext ctx) const { + std::vector<const NProtoBuf::FieldDescriptor*> fields; + + ctx.Init(&msg); + + ctx.Reflection().ListFields(msg, &fields); + + for (const auto* f : fields) { + Y_ENSURE(f); + + const auto& opts = f->options(); + const auto isMessage = f->type() == NProtoBuf::FieldDescriptor::TYPE_MESSAGE; const auto isExtraLabelMetrics = isMessage && f->message_type()->full_name() == "NMonProto.TExtraLabelMetrics"; - const auto maybeMeta = MaybeGetMeta(opts); - + const auto maybeMeta = MaybeGetMeta(opts); + if (!(maybeMeta || isExtraLabelMetrics)) { - continue; - } - + continue; + } + if (isExtraLabelMetrics) { const auto& extra = ExtractExtraMetrics(ctx, *f); RecurseExtraLabelMetrics(ctx, extra, f->name(), maybeMeta); - } else if (isMessage) { - RecurseMessage(ctx, *maybeMeta, *f); - } else if (f->is_repeated()) { - RecurseRepeatedScalar(ctx, *maybeMeta, *f); + } else if (isMessage) { + RecurseMessage(ctx, *maybeMeta, *f); + } else if (f->is_repeated()) { + RecurseRepeatedScalar(ctx, *maybeMeta, *f); } else if (maybeMeta->HasType()) { - const auto val = ReadFieldAsDouble(msg, f, ctx.Reflection()); + const auto val = ReadFieldAsDouble(msg, f, ctx.Reflection()); const bool isRate = maybeMeta->GetType() == NMonProto::EMetricType::RATE; WriteMetric(val, ctx, f->name(), isRate); - } - } - } - + } + } + } + void RecurseRepeatedScalar(TDecoderContext ctx, const NMonProto::TMetricMeta& meta, const NProtoBuf::FieldDescriptor& f) const { - auto&& msg = ctx.Message(); - auto&& reflection = ctx.Reflection(); + auto&& msg = ctx.Message(); + auto&& reflection = ctx.Reflection(); const bool isRate = meta.GetType() == NMonProto::EMetricType::RATE; - + // this is a repeated scalar field, which makes metric only if it's indexing - for (auto i = 0; i < reflection.FieldSize(msg, &f); ++i) { - auto subCtx = ctx.CreateChildFromRepeatedScalar(meta, i); - subCtx.Init(&msg); - auto val = ReadRepeatedAsDouble(msg, &f, reflection, i); + for (auto i = 0; i < reflection.FieldSize(msg, &f); ++i) { + auto subCtx = ctx.CreateChildFromRepeatedScalar(meta, i); + subCtx.Init(&msg); + auto val = ReadRepeatedAsDouble(msg, &f, reflection, i); WriteMetric(val, subCtx, f.name(), isRate); - } - } - + } + } + void RecurseExtraLabelMetrics(TDecoderContext ctx, const NMonProto::TExtraLabelMetrics& msg, const TString& name, const TMaybeMeta& meta) const { - auto i = 0; - for (const auto& val : msg.Getvalues()) { - auto subCtx = ctx.CreateChildFromEls(name, msg, i++, meta); - subCtx.Init(&val); - + auto i = 0; + for (const auto& val : msg.Getvalues()) { + auto subCtx = ctx.CreateChildFromEls(name, msg, i++, meta); + subCtx.Init(&val); + const bool isRate = val.Hastype() ? val.Gettype() == NMonProto::EMetricType::RATE : meta->GetType() == NMonProto::EMetricType::RATE; - + double metricVal{0}; if (isRate) { metricVal = val.GetlongValue(); - } else { + } else { metricVal = val.GetdoubleValue(); - } - + } + WriteMetric(metricVal, subCtx, "", isRate); - - for (const auto& child : val.Getchildren()) { + + for (const auto& child : val.Getchildren()) { RecurseExtraLabelMetrics(subCtx, child, "", meta); - } - } - } - + } + } + } + void RecurseMessage(TDecoderContext ctx, const NMonProto::TMetricMeta& metricMeta, const NProtoBuf::FieldDescriptor& f) const { - const auto& msg = ctx.Message(); - const auto& reflection = ctx.Reflection(); - - if (f.is_repeated()) { - TRACE("recurse into repeated message " << f.name()); - for (auto i = 0; i < reflection.FieldSize(msg, &f); ++i) { - auto& subMessage = reflection.GetRepeatedMessage(msg, &f, i); + const auto& msg = ctx.Message(); + const auto& reflection = ctx.Reflection(); + + if (f.is_repeated()) { + TRACE("recurse into repeated message " << f.name()); + for (auto i = 0; i < reflection.FieldSize(msg, &f); ++i) { + auto& subMessage = reflection.GetRepeatedMessage(msg, &f, i); DecodeImpl(subMessage, ctx.CreateChildFromMeta(metricMeta, f.name(), i)); - } - } else { - TRACE("recurse into message " << f.name()); - auto& subMessage = reflection.GetMessage(msg, &f); + } + } else { + TRACE("recurse into message " << f.name()); + auto& subMessage = reflection.GetMessage(msg, &f); DecodeImpl(subMessage, ctx.CreateChildFromMeta(metricMeta, f.name())); - } - } - - inline void WriteValue(ui64 value) const { - Consumer_->OnUint64(Timestamp_, value); - } - - inline void WriteValue(double value) const { - Consumer_->OnDouble(Timestamp_, value); - } - + } + } + + inline void WriteValue(ui64 value) const { + Consumer_->OnUint64(Timestamp_, value); + } + + inline void WriteValue(double value) const { + Consumer_->OnDouble(Timestamp_, value); + } + void WriteMetric(double value, const TDecoderContext& ctx, const TString& name, bool isRate) const { if (isRate) { Consumer_->OnMetricBegin(EMetricType::RATE); - WriteValue(static_cast<ui64>(value)); - } else { + WriteValue(static_cast<ui64>(value)); + } else { Consumer_->OnMetricBegin(EMetricType::GAUGE); - WriteValue(static_cast<double>(value)); - } - - Consumer_->OnLabelsBegin(); - - for (const auto& label : ctx.Labels()) { - Consumer_->OnLabel(label.Name(), label.Value()); - } - + WriteValue(static_cast<double>(value)); + } + + Consumer_->OnLabelsBegin(); + + for (const auto& label : ctx.Labels()) { + Consumer_->OnLabel(label.Name(), label.Value()); + } + const auto fullPath = name.empty() - ? ctx.Path() - : ctx.Path() + '/' + name; - - if (fullPath) { - Consumer_->OnLabel("path", fullPath); - } - - Consumer_->OnLabelsEnd(); + ? ctx.Path() + : ctx.Path() + '/' + name; + + if (fullPath) { + Consumer_->OnLabel("path", fullPath); + } + + Consumer_->OnLabelsEnd(); Consumer_->OnMetricEnd(); - } - - private: + } + + private: IMetricConsumer* Consumer_{nullptr}; - const NProtoBuf::Message& Message_; - TInstant Timestamp_; - }; - - } - + const NProtoBuf::Message& Message_; + TInstant Timestamp_; + }; + + } + void DecodeLegacyProto(const NProtoBuf::Message& data, IMetricConsumer* consumer, TInstant ts) { - Y_ENSURE(consumer); - TDecoder(consumer, data, ts).Decode(); - } - + Y_ENSURE(consumer); + TDecoder(consumer, data, ts).Decode(); + } + void DecodeLegacyProtoToStream(const NProtoBuf::Message& data, IMetricConsumer* consumer, TInstant ts) { - Y_ENSURE(consumer); - TDecoder(consumer, data, ts).DecodeToStream(); - } -} + Y_ENSURE(consumer); + TDecoder(consumer, data, ts).DecodeToStream(); + } +} diff --git a/library/cpp/monlib/encode/legacy_protobuf/legacy_protobuf.h b/library/cpp/monlib/encode/legacy_protobuf/legacy_protobuf.h index 7cf8985d65..c1bd7cd436 100644 --- a/library/cpp/monlib/encode/legacy_protobuf/legacy_protobuf.h +++ b/library/cpp/monlib/encode/legacy_protobuf/legacy_protobuf.h @@ -1,16 +1,16 @@ -#pragma once - +#pragma once + #include <google/protobuf/message.h> -#include <util/datetime/base.h> - -namespace NMonitoring { - // Unsupported features of the original format: - // - histograms; - // - memOnly; - // - dropHost/ignorePath - +#include <util/datetime/base.h> + +namespace NMonitoring { + // Unsupported features of the original format: + // - histograms; + // - memOnly; + // - dropHost/ignorePath + void DecodeLegacyProto(const NProtoBuf::Message& data, class IMetricConsumer* c, TInstant ts = TInstant::Zero()); - - /// Does not open/close consumer stream unlike the above function. + + /// Does not open/close consumer stream unlike the above function. void DecodeLegacyProtoToStream(const NProtoBuf::Message& data, class IMetricConsumer* c, TInstant ts = TInstant::Zero()); -} +} diff --git a/library/cpp/monlib/encode/legacy_protobuf/legacy_protobuf_ut.cpp b/library/cpp/monlib/encode/legacy_protobuf/legacy_protobuf_ut.cpp index 53683cb39c..140ef1ef58 100644 --- a/library/cpp/monlib/encode/legacy_protobuf/legacy_protobuf_ut.cpp +++ b/library/cpp/monlib/encode/legacy_protobuf/legacy_protobuf_ut.cpp @@ -1,422 +1,422 @@ -#include "legacy_protobuf.h" - +#include "legacy_protobuf.h" + #include <library/cpp/testing/unittest/registar.h> - + #include <library/cpp/monlib/encode/legacy_protobuf/ut/test_cases.pb.h> #include <library/cpp/monlib/encode/legacy_protobuf/protos/metric_meta.pb.h> - + #include <library/cpp/monlib/encode/protobuf/protobuf.h> #include <library/cpp/monlib/encode/text/text.h> #include <library/cpp/monlib/metrics/labels.h> - -#include <util/generic/algorithm.h> -#include <util/generic/hash_set.h> - -using namespace NMonitoring; - -TSimple MakeSimpleMessage() { - TSimple msg; - - msg.SetFoo(1); - msg.SetBar(2.); - msg.SetBaz(42.); - - return msg; -} - + +#include <util/generic/algorithm.h> +#include <util/generic/hash_set.h> + +using namespace NMonitoring; + +TSimple MakeSimpleMessage() { + TSimple msg; + + msg.SetFoo(1); + msg.SetBar(2.); + msg.SetBaz(42.); + + return msg; +} + IMetricEncoderPtr debugPrinter = EncoderText(&Cerr); - -namespace NMonitoring { - inline bool operator<(const TLabel& lhs, const TLabel& rhs) { - return lhs.Name() < rhs.Name() || - (lhs.Name() == rhs.Name() && lhs.Value() < rhs.Value()); - } - -} - + +namespace NMonitoring { + inline bool operator<(const TLabel& lhs, const TLabel& rhs) { + return lhs.Name() < rhs.Name() || + (lhs.Name() == rhs.Name() && lhs.Value() < rhs.Value()); + } + +} + void SetLabelValue(NMonProto::TExtraLabelMetrics::TValue& val, TString s) { - val.SetlabelValue(s); -} - + val.SetlabelValue(s); +} + void SetLabelValue(NMonProto::TExtraLabelMetrics::TValue& val, ui64 u) { - val.SetlabelValueUint(u); -} - -template <typename T, typename V> + val.SetlabelValueUint(u); +} + +template <typename T, typename V> NMonProto::TExtraLabelMetrics MakeExtra(TString labelName, V labelValue, T value, bool isDeriv) { NMonProto::TExtraLabelMetrics metric; auto* val = metric.Addvalues(); - + metric.SetlabelName(labelName); - SetLabelValue(*val, labelValue); - - if (isDeriv) { - val->SetlongValue(value); - } else { - val->SetdoubleValue(value); - } - + SetLabelValue(*val, labelValue); + + if (isDeriv) { + val->SetlongValue(value); + } else { + val->SetdoubleValue(value); + } + return metric; -} - -void AssertLabels(const TLabels& expected, const NProto::TMultiSample& actual) { - UNIT_ASSERT_EQUAL(actual.LabelsSize(), expected.Size()); - - TSet<TLabel> actualSet; - TSet<TLabel> expectedSet; - Transform(expected.begin(), expected.end(), std::inserter(expectedSet, expectedSet.end()), [] (auto&& l) { - return TLabel{l.Name(), l.Value()}; - }); - - const auto& l = actual.GetLabels(); - Transform(std::begin(l), std::end(l), std::inserter(actualSet, std::begin(actualSet)), - [](auto&& elem) -> TLabel { - return {elem.GetName(), elem.GetValue()}; - }); - - TVector<TLabel> diff; - SetSymmetricDifference(std::begin(expectedSet), std::end(expectedSet), - std::begin(actualSet), std::end(actualSet), std::back_inserter(diff)); - - if (diff.size() > 0) { - for (auto&& l : diff) { - Cerr << l << Endl; - } - - UNIT_FAIL("Labels don't match"); - } -} - -void AssertSimpleMessage(const NProto::TMultiSamplesList& samples, TString pathPrefix = "/") { - UNIT_ASSERT_EQUAL(samples.SamplesSize(), 3); - - THashSet<TString> expectedValues{pathPrefix + "Foo", pathPrefix + "Bar", pathPrefix + "Baz"}; - - for (const auto& s : samples.GetSamples()) { - UNIT_ASSERT_EQUAL(s.LabelsSize(), 1); - UNIT_ASSERT_EQUAL(s.PointsSize(), 1); - - const auto labelVal = s.GetLabels(0).GetValue(); +} + +void AssertLabels(const TLabels& expected, const NProto::TMultiSample& actual) { + UNIT_ASSERT_EQUAL(actual.LabelsSize(), expected.Size()); + + TSet<TLabel> actualSet; + TSet<TLabel> expectedSet; + Transform(expected.begin(), expected.end(), std::inserter(expectedSet, expectedSet.end()), [] (auto&& l) { + return TLabel{l.Name(), l.Value()}; + }); + + const auto& l = actual.GetLabels(); + Transform(std::begin(l), std::end(l), std::inserter(actualSet, std::begin(actualSet)), + [](auto&& elem) -> TLabel { + return {elem.GetName(), elem.GetValue()}; + }); + + TVector<TLabel> diff; + SetSymmetricDifference(std::begin(expectedSet), std::end(expectedSet), + std::begin(actualSet), std::end(actualSet), std::back_inserter(diff)); + + if (diff.size() > 0) { + for (auto&& l : diff) { + Cerr << l << Endl; + } + + UNIT_FAIL("Labels don't match"); + } +} + +void AssertSimpleMessage(const NProto::TMultiSamplesList& samples, TString pathPrefix = "/") { + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 3); + + THashSet<TString> expectedValues{pathPrefix + "Foo", pathPrefix + "Bar", pathPrefix + "Baz"}; + + for (const auto& s : samples.GetSamples()) { + UNIT_ASSERT_EQUAL(s.LabelsSize(), 1); + UNIT_ASSERT_EQUAL(s.PointsSize(), 1); + + const auto labelVal = s.GetLabels(0).GetValue(); UNIT_ASSERT(expectedValues.contains(labelVal)); - - if (labelVal == pathPrefix + "Foo") { + + if (labelVal == pathPrefix + "Foo") { UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); - UNIT_ASSERT_DOUBLES_EQUAL(s.GetPoints(0).GetFloat64(), 1, 1e-6); - } else if (labelVal == pathPrefix + "Bar") { + UNIT_ASSERT_DOUBLES_EQUAL(s.GetPoints(0).GetFloat64(), 1, 1e-6); + } else if (labelVal == pathPrefix + "Bar") { UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); - UNIT_ASSERT_DOUBLES_EQUAL(s.GetPoints(0).GetFloat64(), 2, 1e-6); - } else if (labelVal == pathPrefix + "Baz") { + UNIT_ASSERT_DOUBLES_EQUAL(s.GetPoints(0).GetFloat64(), 2, 1e-6); + } else if (labelVal == pathPrefix + "Baz") { UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::RATE); - UNIT_ASSERT_EQUAL(s.GetPoints(0).GetUint64(), 42); - } - } -} - -Y_UNIT_TEST_SUITE(TLegacyProtoDecoderTest) { - Y_UNIT_TEST(SimpleProto) { - NProto::TMultiSamplesList samples; + UNIT_ASSERT_EQUAL(s.GetPoints(0).GetUint64(), 42); + } + } +} + +Y_UNIT_TEST_SUITE(TLegacyProtoDecoderTest) { + Y_UNIT_TEST(SimpleProto) { + NProto::TMultiSamplesList samples; IMetricEncoderPtr e = EncoderProtobuf(&samples); - - auto msg = MakeSimpleMessage(); - DecodeLegacyProto(msg, e.Get()); - - AssertSimpleMessage(samples); - }; - - Y_UNIT_TEST(RepeatedProto) { - NProto::TMultiSamplesList samples; + + auto msg = MakeSimpleMessage(); + DecodeLegacyProto(msg, e.Get()); + + AssertSimpleMessage(samples); + }; + + Y_UNIT_TEST(RepeatedProto) { + NProto::TMultiSamplesList samples; IMetricEncoderPtr e = EncoderProtobuf(&samples); - - auto simple = MakeSimpleMessage(); - TRepeated msg; - msg.AddMessages()->CopyFrom(simple); - - DecodeLegacyProto(msg, e.Get()); - - AssertSimpleMessage(samples); - } - - Y_UNIT_TEST(RepeatedProtoWithPath) { - NProto::TMultiSamplesList samples; + + auto simple = MakeSimpleMessage(); + TRepeated msg; + msg.AddMessages()->CopyFrom(simple); + + DecodeLegacyProto(msg, e.Get()); + + AssertSimpleMessage(samples); + } + + Y_UNIT_TEST(RepeatedProtoWithPath) { + NProto::TMultiSamplesList samples; IMetricEncoderPtr e = EncoderProtobuf(&samples); - - auto simple = MakeSimpleMessage(); - TRepeatedWithPath msg; - msg.AddNamespace()->CopyFrom(simple); - - DecodeLegacyProto(msg, e.Get()); - - AssertSimpleMessage(samples, "/Namespace/"); - } - - Y_UNIT_TEST(DeepNesting) { - NProto::TMultiSamplesList samples; + + auto simple = MakeSimpleMessage(); + TRepeatedWithPath msg; + msg.AddNamespace()->CopyFrom(simple); + + DecodeLegacyProto(msg, e.Get()); + + AssertSimpleMessage(samples, "/Namespace/"); + } + + Y_UNIT_TEST(DeepNesting) { + NProto::TMultiSamplesList samples; IMetricEncoderPtr e = EncoderProtobuf(&samples); - - auto simple = MakeSimpleMessage(); - TRepeatedWithPath internal; - internal.AddNamespace()->CopyFrom(simple); - - TDeepNesting msg; - msg.MutableNested()->CopyFrom(internal); - - DecodeLegacyProto(msg, e.Get()); - - AssertSimpleMessage(samples, "/Namespace/"); - } - - Y_UNIT_TEST(Keys) { - NProto::TMultiSamplesList samples; + + auto simple = MakeSimpleMessage(); + TRepeatedWithPath internal; + internal.AddNamespace()->CopyFrom(simple); + + TDeepNesting msg; + msg.MutableNested()->CopyFrom(internal); + + DecodeLegacyProto(msg, e.Get()); + + AssertSimpleMessage(samples, "/Namespace/"); + } + + Y_UNIT_TEST(Keys) { + NProto::TMultiSamplesList samples; IMetricEncoderPtr e = EncoderProtobuf(&samples); - - auto simple = MakeSimpleMessage(); - simple.SetLabel("my_label_value"); - - TNestedWithKeys msg; - msg.AddNamespace()->CopyFrom(simple); - - DecodeLegacyProto(msg, e.Get()); - - auto i = 0; - for (const auto& s : samples.GetSamples()) { - UNIT_ASSERT_EQUAL(s.LabelsSize(), 4); - - bool foundLabel = false; - bool foundFixed = false; - bool foundNumbered = false; - - for (const auto& label : s.GetLabels()) { - if (label.GetName() == "my_label") { - foundLabel = true; - UNIT_ASSERT_STRINGS_EQUAL(label.GetValue(), "my_label_value"); - } else if (label.GetName() == "fixed_label") { - foundFixed = true; - UNIT_ASSERT_STRINGS_EQUAL(label.GetValue(), "fixed_value"); - } else if (label.GetName() == "numbered") { - foundNumbered = true; - UNIT_ASSERT_STRINGS_EQUAL(label.GetValue(), ::ToString(i)); - } - } - - UNIT_ASSERT(foundLabel); - UNIT_ASSERT(foundFixed); - UNIT_ASSERT(foundNumbered); - } - } - - Y_UNIT_TEST(NonStringKeys) { - NProto::TMultiSamplesList samples; + + auto simple = MakeSimpleMessage(); + simple.SetLabel("my_label_value"); + + TNestedWithKeys msg; + msg.AddNamespace()->CopyFrom(simple); + + DecodeLegacyProto(msg, e.Get()); + + auto i = 0; + for (const auto& s : samples.GetSamples()) { + UNIT_ASSERT_EQUAL(s.LabelsSize(), 4); + + bool foundLabel = false; + bool foundFixed = false; + bool foundNumbered = false; + + for (const auto& label : s.GetLabels()) { + if (label.GetName() == "my_label") { + foundLabel = true; + UNIT_ASSERT_STRINGS_EQUAL(label.GetValue(), "my_label_value"); + } else if (label.GetName() == "fixed_label") { + foundFixed = true; + UNIT_ASSERT_STRINGS_EQUAL(label.GetValue(), "fixed_value"); + } else if (label.GetName() == "numbered") { + foundNumbered = true; + UNIT_ASSERT_STRINGS_EQUAL(label.GetValue(), ::ToString(i)); + } + } + + UNIT_ASSERT(foundLabel); + UNIT_ASSERT(foundFixed); + UNIT_ASSERT(foundNumbered); + } + } + + Y_UNIT_TEST(NonStringKeys) { + NProto::TMultiSamplesList samples; IMetricEncoderPtr e = EncoderProtobuf(&samples); - - TNonStringKeys msg; - msg.SetFoo(42); - msg.SetEnum(ENUM); - msg.SetInt(43); - - TRepeatedNonStringKeys msgs; - msgs.AddNested()->CopyFrom(msg); - - DecodeLegacyProto(msgs, e.Get()); - - for (const auto& s : samples.GetSamples()) { - bool foundEnum = false; - bool foundInt = false; - - for (const auto& label : s.GetLabels()) { - if (label.GetName() == "enum") { - foundEnum = true; - UNIT_ASSERT_STRINGS_EQUAL(label.GetValue(), "ENUM"); - } else if (label.GetName() == "int") { - foundInt = true; - UNIT_ASSERT_STRINGS_EQUAL(label.GetValue(), "43"); - } - } - - UNIT_ASSERT(foundEnum); - UNIT_ASSERT(foundInt); - - UNIT_ASSERT_DOUBLES_EQUAL(s.GetPoints(0).GetFloat64(), 42, 1e-6); - } - } - - Y_UNIT_TEST(KeysFromNonLeafNodes) { - NProto::TMultiSamplesList samples; + + TNonStringKeys msg; + msg.SetFoo(42); + msg.SetEnum(ENUM); + msg.SetInt(43); + + TRepeatedNonStringKeys msgs; + msgs.AddNested()->CopyFrom(msg); + + DecodeLegacyProto(msgs, e.Get()); + + for (const auto& s : samples.GetSamples()) { + bool foundEnum = false; + bool foundInt = false; + + for (const auto& label : s.GetLabels()) { + if (label.GetName() == "enum") { + foundEnum = true; + UNIT_ASSERT_STRINGS_EQUAL(label.GetValue(), "ENUM"); + } else if (label.GetName() == "int") { + foundInt = true; + UNIT_ASSERT_STRINGS_EQUAL(label.GetValue(), "43"); + } + } + + UNIT_ASSERT(foundEnum); + UNIT_ASSERT(foundInt); + + UNIT_ASSERT_DOUBLES_EQUAL(s.GetPoints(0).GetFloat64(), 42, 1e-6); + } + } + + Y_UNIT_TEST(KeysFromNonLeafNodes) { + NProto::TMultiSamplesList samples; IMetricEncoderPtr e = EncoderProtobuf(&samples); - - auto simple = MakeSimpleMessage(); - simple.SetLabel("label_value"); - - TRepeatedWithName nested; - nested.SetName("my_name"); - nested.AddNested()->CopyFrom(simple); - - TKeysFromNonLeaf msg; - msg.AddNested()->CopyFrom(nested); - - DecodeLegacyProto(msg, e.Get()); - - AssertLabels({{"my_label", "label_value"}, {"path", "/Nested/Nested/Foo"}, {"name", "my_name"}}, samples.GetSamples(0)); - } - - Y_UNIT_TEST(SpacesAreGetReplaced) { - NProto::TMultiSamplesList samples; + + auto simple = MakeSimpleMessage(); + simple.SetLabel("label_value"); + + TRepeatedWithName nested; + nested.SetName("my_name"); + nested.AddNested()->CopyFrom(simple); + + TKeysFromNonLeaf msg; + msg.AddNested()->CopyFrom(nested); + + DecodeLegacyProto(msg, e.Get()); + + AssertLabels({{"my_label", "label_value"}, {"path", "/Nested/Nested/Foo"}, {"name", "my_name"}}, samples.GetSamples(0)); + } + + Y_UNIT_TEST(SpacesAreGetReplaced) { + NProto::TMultiSamplesList samples; IMetricEncoderPtr e = EncoderProtobuf(&samples); - - auto simple = MakeSimpleMessage(); - simple.SetLabel("my label_value"); - - TNestedWithKeys msg; - msg.AddNamespace()->CopyFrom(simple); - - DecodeLegacyProto(msg, e.Get()); - - for (const auto& s : samples.GetSamples()) { - UNIT_ASSERT_EQUAL(s.LabelsSize(), 4); - - bool foundLabel = false; - - for (const auto& label : s.GetLabels()) { - if (label.GetName() == "my_label") { - foundLabel = true; - UNIT_ASSERT_STRINGS_EQUAL(label.GetValue(), "my_label_value"); - } - } - - UNIT_ASSERT(foundLabel); - } - } - - Y_UNIT_TEST(ExtraLabels) { - NProto::TMultiSamplesList samples; + + auto simple = MakeSimpleMessage(); + simple.SetLabel("my label_value"); + + TNestedWithKeys msg; + msg.AddNamespace()->CopyFrom(simple); + + DecodeLegacyProto(msg, e.Get()); + + for (const auto& s : samples.GetSamples()) { + UNIT_ASSERT_EQUAL(s.LabelsSize(), 4); + + bool foundLabel = false; + + for (const auto& label : s.GetLabels()) { + if (label.GetName() == "my_label") { + foundLabel = true; + UNIT_ASSERT_STRINGS_EQUAL(label.GetValue(), "my_label_value"); + } + } + + UNIT_ASSERT(foundLabel); + } + } + + Y_UNIT_TEST(ExtraLabels) { + NProto::TMultiSamplesList samples; IMetricEncoderPtr e = EncoderProtobuf(&samples); - - TExtraLabels msg; - msg.MutableExtraAsIs()->CopyFrom(MakeExtra("label", "foo", 42, false)); - msg.MutableExtraDeriv()->CopyFrom(MakeExtra("deriv_label", "deriv_foo", 43, true)); - - DecodeLegacyProto(msg, e.Get()); - - UNIT_ASSERT_EQUAL(samples.SamplesSize(), 2); - { - auto s = samples.GetSamples(0); - AssertLabels({{"label", "foo"}, {"path", "/ExtraAsIs"}}, s); - - UNIT_ASSERT_EQUAL(s.PointsSize(), 1); - auto point = s.GetPoints(0); - UNIT_ASSERT_DOUBLES_EQUAL(point.GetFloat64(), 42, 1e-6); - } - - { - auto s = samples.GetSamples(1); - AssertLabels({{"deriv_label", "deriv_foo"}, {"path", "/ExtraDeriv"}}, s); - - UNIT_ASSERT_EQUAL(s.PointsSize(), 1); - auto point = s.GetPoints(0); - UNIT_ASSERT_EQUAL(point.GetUint64(), 43); - } - } - - Y_UNIT_TEST(NestedExtraLabels) { - NProto::TMultiSamplesList samples; + + TExtraLabels msg; + msg.MutableExtraAsIs()->CopyFrom(MakeExtra("label", "foo", 42, false)); + msg.MutableExtraDeriv()->CopyFrom(MakeExtra("deriv_label", "deriv_foo", 43, true)); + + DecodeLegacyProto(msg, e.Get()); + + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 2); + { + auto s = samples.GetSamples(0); + AssertLabels({{"label", "foo"}, {"path", "/ExtraAsIs"}}, s); + + UNIT_ASSERT_EQUAL(s.PointsSize(), 1); + auto point = s.GetPoints(0); + UNIT_ASSERT_DOUBLES_EQUAL(point.GetFloat64(), 42, 1e-6); + } + + { + auto s = samples.GetSamples(1); + AssertLabels({{"deriv_label", "deriv_foo"}, {"path", "/ExtraDeriv"}}, s); + + UNIT_ASSERT_EQUAL(s.PointsSize(), 1); + auto point = s.GetPoints(0); + UNIT_ASSERT_EQUAL(point.GetUint64(), 43); + } + } + + Y_UNIT_TEST(NestedExtraLabels) { + NProto::TMultiSamplesList samples; IMetricEncoderPtr e = EncoderProtobuf(&samples); - - TExtraLabels msg; - auto extra = MakeExtra("label", "foo", 42, false); - auto* val = extra.Mutablevalues(0); - { - auto child = MakeExtra("child1", "label1", 24, true); + + TExtraLabels msg; + auto extra = MakeExtra("label", "foo", 42, false); + auto* val = extra.Mutablevalues(0); + { + auto child = MakeExtra("child1", "label1", 24, true); child.Mutablevalues(0)->Settype(NMonProto::EMetricType::RATE); - val->Addchildren()->CopyFrom(child); - } - - { - auto child = MakeExtra("child2", 34, 23, false); - val->Addchildren()->CopyFrom(child); - } - msg.MutableExtraAsIs()->CopyFrom(extra); - - DecodeLegacyProto(msg, e.Get()); - - UNIT_ASSERT_EQUAL(samples.SamplesSize(), 3); - { - auto s = samples.GetSamples(0); - AssertLabels({{"label", "foo"}, {"path", "/ExtraAsIs"}}, s); - - UNIT_ASSERT_EQUAL(s.PointsSize(), 1); - auto point = s.GetPoints(0); - UNIT_ASSERT_DOUBLES_EQUAL(point.GetFloat64(), 42, 1e-6); - } - - { - auto s = samples.GetSamples(1); - AssertLabels({{"label", "foo"}, {"child1", "label1"}, {"path", "/ExtraAsIs"}}, s); - - UNIT_ASSERT_EQUAL(s.PointsSize(), 1); - auto point = s.GetPoints(0); - UNIT_ASSERT_EQUAL(point.GetUint64(), 24); - } - - { - auto s = samples.GetSamples(2); - AssertLabels({{"label", "foo"}, {"child2", "34"}, {"path", "/ExtraAsIs"}}, s); - - UNIT_ASSERT_EQUAL(s.PointsSize(), 1); - auto point = s.GetPoints(0); - UNIT_ASSERT_EQUAL(point.GetFloat64(), 23); - } - } - - Y_UNIT_TEST(RobotLabels) { - NProto::TMultiSamplesList samples; + val->Addchildren()->CopyFrom(child); + } + + { + auto child = MakeExtra("child2", 34, 23, false); + val->Addchildren()->CopyFrom(child); + } + msg.MutableExtraAsIs()->CopyFrom(extra); + + DecodeLegacyProto(msg, e.Get()); + + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 3); + { + auto s = samples.GetSamples(0); + AssertLabels({{"label", "foo"}, {"path", "/ExtraAsIs"}}, s); + + UNIT_ASSERT_EQUAL(s.PointsSize(), 1); + auto point = s.GetPoints(0); + UNIT_ASSERT_DOUBLES_EQUAL(point.GetFloat64(), 42, 1e-6); + } + + { + auto s = samples.GetSamples(1); + AssertLabels({{"label", "foo"}, {"child1", "label1"}, {"path", "/ExtraAsIs"}}, s); + + UNIT_ASSERT_EQUAL(s.PointsSize(), 1); + auto point = s.GetPoints(0); + UNIT_ASSERT_EQUAL(point.GetUint64(), 24); + } + + { + auto s = samples.GetSamples(2); + AssertLabels({{"label", "foo"}, {"child2", "34"}, {"path", "/ExtraAsIs"}}, s); + + UNIT_ASSERT_EQUAL(s.PointsSize(), 1); + auto point = s.GetPoints(0); + UNIT_ASSERT_EQUAL(point.GetFloat64(), 23); + } + } + + Y_UNIT_TEST(RobotLabels) { + NProto::TMultiSamplesList samples; IMetricEncoderPtr e = EncoderProtobuf(&samples); - - TNamedCounter responses; - responses.SetName("responses"); - responses.SetCount(42); - - TCrawlerCounters::TStatusCounters statusCounters; - statusCounters.AddZoraResponses()->CopyFrom(responses); - - TCrawlerCounters::TPolicyCounters policyCounters; - policyCounters.SetSubComponent("mySubComponent"); - policyCounters.SetName("myComponentName"); - policyCounters.SetZone("myComponentZone"); - policyCounters.MutableStatusCounters()->CopyFrom(statusCounters); - - TCrawlerCounters counters; - counters.SetComponent("myComponent"); - counters.AddPoliciesCounters()->CopyFrom(policyCounters); - - DecodeLegacyProto(counters, e.Get()); - UNIT_ASSERT_EQUAL(samples.SamplesSize(), 1); - auto s = samples.GetSamples(0); - AssertLabels({ - {"SubComponent", "mySubComponent"}, {"Policy", "myComponentName"}, {"Zone", "myComponentZone"}, - {"ZoraResponse", "responses"}, {"path", "/PoliciesCounters/StatusCounters/ZoraResponses/Count"}}, s); - } - - Y_UNIT_TEST(ZoraLabels) { - NProto::TMultiSamplesList samples; + + TNamedCounter responses; + responses.SetName("responses"); + responses.SetCount(42); + + TCrawlerCounters::TStatusCounters statusCounters; + statusCounters.AddZoraResponses()->CopyFrom(responses); + + TCrawlerCounters::TPolicyCounters policyCounters; + policyCounters.SetSubComponent("mySubComponent"); + policyCounters.SetName("myComponentName"); + policyCounters.SetZone("myComponentZone"); + policyCounters.MutableStatusCounters()->CopyFrom(statusCounters); + + TCrawlerCounters counters; + counters.SetComponent("myComponent"); + counters.AddPoliciesCounters()->CopyFrom(policyCounters); + + DecodeLegacyProto(counters, e.Get()); + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 1); + auto s = samples.GetSamples(0); + AssertLabels({ + {"SubComponent", "mySubComponent"}, {"Policy", "myComponentName"}, {"Zone", "myComponentZone"}, + {"ZoraResponse", "responses"}, {"path", "/PoliciesCounters/StatusCounters/ZoraResponses/Count"}}, s); + } + + Y_UNIT_TEST(ZoraLabels) { + NProto::TMultiSamplesList samples; IMetricEncoderPtr e = EncoderProtobuf(&samples); - - TTimeLogHist hist; - hist.AddBuckets(42); - hist.AddBuckets(0); - - TKiwiCounters counters; - counters.MutableTimes()->CopyFrom(hist); - - DecodeLegacyProto(counters, e.Get()); - - UNIT_ASSERT_EQUAL(samples.SamplesSize(), 2); - - auto s = samples.GetSamples(0); - AssertLabels({{"slot", "0"}, {"path", "/Times/Buckets"}}, s); - UNIT_ASSERT_EQUAL(s.PointsSize(), 1); - UNIT_ASSERT_EQUAL(s.GetPoints(0).GetUint64(), 42); - - s = samples.GetSamples(1); - AssertLabels({{"slot", "1"}, {"path", "/Times/Buckets"}}, s); - UNIT_ASSERT_EQUAL(s.GetPoints(0).GetUint64(), 0); - } -} + + TTimeLogHist hist; + hist.AddBuckets(42); + hist.AddBuckets(0); + + TKiwiCounters counters; + counters.MutableTimes()->CopyFrom(hist); + + DecodeLegacyProto(counters, e.Get()); + + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 2); + + auto s = samples.GetSamples(0); + AssertLabels({{"slot", "0"}, {"path", "/Times/Buckets"}}, s); + UNIT_ASSERT_EQUAL(s.PointsSize(), 1); + UNIT_ASSERT_EQUAL(s.GetPoints(0).GetUint64(), 42); + + s = samples.GetSamples(1); + AssertLabels({{"slot", "1"}, {"path", "/Times/Buckets"}}, s); + UNIT_ASSERT_EQUAL(s.GetPoints(0).GetUint64(), 0); + } +} diff --git a/library/cpp/monlib/encode/legacy_protobuf/protos/ya.make b/library/cpp/monlib/encode/legacy_protobuf/protos/ya.make index 489f361ab1..01505f61ca 100644 --- a/library/cpp/monlib/encode/legacy_protobuf/protos/ya.make +++ b/library/cpp/monlib/encode/legacy_protobuf/protos/ya.make @@ -1,13 +1,13 @@ -PROTO_LIBRARY() - -OWNER(g:solomon) - -SRCS( +PROTO_LIBRARY() + +OWNER(g:solomon) + +SRCS( metric_meta.proto -) - +) + IF (NOT PY_PROTOS_FOR) EXCLUDE_TAGS(GO_PROTO) ENDIF() -END() +END() diff --git a/library/cpp/monlib/encode/legacy_protobuf/ut/test_cases.proto b/library/cpp/monlib/encode/legacy_protobuf/ut/test_cases.proto index 37e901de48..de870cb9e2 100644 --- a/library/cpp/monlib/encode/legacy_protobuf/ut/test_cases.proto +++ b/library/cpp/monlib/encode/legacy_protobuf/ut/test_cases.proto @@ -1,90 +1,90 @@ import "library/cpp/monlib/encode/legacy_protobuf/protos/metric_meta.proto"; - -message TSimple { + +message TSimple { optional uint64 Foo = 1 [ (NMonProto.Metric).Type = GAUGE ]; optional double Bar = 2 [ (NMonProto.Metric).Type = GAUGE ]; optional double Baz = 3 [ (NMonProto.Metric).Type = RATE ]; - optional string Label = 4; -} - -message TRepeated { + optional string Label = 4; +} + +message TRepeated { repeated TSimple Messages = 1 [ (NMonProto.Metric).Path = false ]; -}; - -message TRepeatedWithPath { +}; + +message TRepeatedWithPath { repeated TSimple Namespace = 1 [ (NMonProto.Metric).Path = true ]; -}; - -message TNestedWithKeys { +}; + +message TNestedWithKeys { repeated TSimple Namespace = 1 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "my_label:Label fixed_label=fixed_value numbered:#" ]; -}; - -message TDeepNesting { +}; + +message TDeepNesting { optional TRepeatedWithPath Nested = 1 [ (NMonProto.Metric).Path = false ]; -}; - -enum EEnum { - MY = 1; - ENUM = 2; -}; - -message TNonStringKeys { +}; + +enum EEnum { + MY = 1; + ENUM = 2; +}; + +message TNonStringKeys { optional uint32 Foo = 1 [ (NMonProto.Metric).Type = GAUGE ]; - optional EEnum Enum = 2; - optional uint32 Int = 3; -}; - -message TRepeatedNonStringKeys { + optional EEnum Enum = 2; + optional uint32 Int = 3; +}; + +message TRepeatedNonStringKeys { repeated TNonStringKeys Nested = 1 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "enum:Enum int:Int" ]; -}; - -message TExtraLabels { +}; + +message TExtraLabels { optional NMonProto.TExtraLabelMetrics ExtraAsIs = 1 [ (NMonProto.Metric).Type = GAUGE ]; optional NMonProto.TExtraLabelMetrics ExtraDeriv = 2 [ (NMonProto.Metric).Type = RATE ]; -}; - -message TRepeatedWithName { - optional string Name = 1; +}; + +message TRepeatedWithName { + optional string Name = 1; repeated TSimple Nested = 2 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "my_label:Label" ]; -}; - -message TKeysFromNonLeaf { +}; + +message TKeysFromNonLeaf { repeated TRepeatedWithName Nested = 1 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "name:Name" ]; -}; - - -message TNamedCounter { - optional string Name = 1; +}; + + +message TNamedCounter { + optional string Name = 1; optional uint64 Count = 2 [ (NMonProto.Metric).Type = RATE ]; -} - -message TCrawlerCounters { - message TStatusCounters { +} + +message TCrawlerCounters { + message TStatusCounters { repeated TNamedCounter ZoraResponses = 3 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "ZoraResponse:Name" ]; repeated TNamedCounter FetcherResponses = 4 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "SpiderResponse:Name" ]; repeated TNamedCounter RotorResponses = 5 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "SpiderResponse:Name" ]; repeated TNamedCounter PDFetchResponses = 6 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "PDFetchResponse:Name" ]; repeated TNamedCounter CalcResponses = 7 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "CalcResponse:Name" ]; repeated TNamedCounter Responses = 8 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "AggregatedResponse:Name" ]; - } - - message TPolicyCounters { - optional string SubComponent = 1; - optional string Name = 2; - optional string Zone = 3; - + } + + message TPolicyCounters { + optional string SubComponent = 1; + optional string Name = 2; + optional string Zone = 3; + optional TStatusCounters StatusCounters = 4 [ (NMonProto.Metric).Path = true ]; - } - - optional string Component = 1; + } + + optional string Component = 1; repeated TPolicyCounters PoliciesCounters = 3 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "SubComponent:SubComponent Policy:Name Zone:Zone" ]; -} - -message TTimeLogHist { - optional uint32 MinBucketMillisec = 1; +} + +message TTimeLogHist { + optional uint32 MinBucketMillisec = 1; repeated uint64 Buckets = 2 [ (NMonProto.Metric).Type = RATE, (NMonProto.Metric).Keys = "slot:#" ]; -} - -message TKiwiCounters { +} + +message TKiwiCounters { optional TTimeLogHist Times = 22 [ (NMonProto.Metric).Path = true ]; -} +} diff --git a/library/cpp/monlib/encode/legacy_protobuf/ut/ya.make b/library/cpp/monlib/encode/legacy_protobuf/ut/ya.make index 479a0c46c9..f066b6c327 100644 --- a/library/cpp/monlib/encode/legacy_protobuf/ut/ya.make +++ b/library/cpp/monlib/encode/legacy_protobuf/ut/ya.make @@ -1,18 +1,18 @@ UNITTEST_FOR(library/cpp/monlib/encode/legacy_protobuf) - + OWNER( g:solomon msherbakov ) - -SRCS( - legacy_protobuf_ut.cpp - test_cases.proto -) - + +SRCS( + legacy_protobuf_ut.cpp + test_cases.proto +) + PEERDIR( library/cpp/monlib/encode/protobuf library/cpp/monlib/encode/text ) - -END() + +END() diff --git a/library/cpp/monlib/encode/legacy_protobuf/ya.make b/library/cpp/monlib/encode/legacy_protobuf/ya.make index 74c82aac93..16a8ebb1f3 100644 --- a/library/cpp/monlib/encode/legacy_protobuf/ya.make +++ b/library/cpp/monlib/encode/legacy_protobuf/ya.make @@ -1,16 +1,16 @@ -LIBRARY() - +LIBRARY() + OWNER( g:solomon msherbakov ) - -SRCS( - legacy_proto_decoder.cpp -) - -PEERDIR( + +SRCS( + legacy_proto_decoder.cpp +) + +PEERDIR( library/cpp/monlib/encode/legacy_protobuf/protos -) - -END() +) + +END() |