diff options
author | dave11ar <dave11ar@yandex-team.com> | 2023-10-14 13:56:16 +0300 |
---|---|---|
committer | dave11ar <dave11ar@yandex-team.com> | 2023-10-14 14:17:10 +0300 |
commit | ca170f490f0f8181344b8a63afc3c262632b8636 (patch) | |
tree | f53d3a409db26e16231303c59f49eb433eb2573b | |
parent | 4371a757495f375ca7c5b1730f4e9f741a71ec40 (diff) | |
download | ydb-ca170f490f0f8181344b8a63afc3c262632b8636.tar.gz |
YT-20007: Add per instance summary configuration, consider disabled profiling mode
-rw-r--r-- | yt/yt/library/profiling/sensor.cpp | 53 | ||||
-rw-r--r-- | yt/yt/library/profiling/sensor.h | 34 | ||||
-rw-r--r-- | yt/yt/library/profiling/solomon/cube.cpp | 120 | ||||
-rw-r--r-- | yt/yt/library/profiling/solomon/cube.h | 4 | ||||
-rw-r--r-- | yt/yt/library/profiling/solomon/exporter.cpp | 44 | ||||
-rw-r--r-- | yt/yt/library/profiling/solomon/exporter.h | 4 | ||||
-rw-r--r-- | yt/yt/library/profiling/solomon/registry.cpp | 28 | ||||
-rw-r--r-- | yt/yt/library/profiling/solomon/remote.cpp | 1 | ||||
-rw-r--r-- | yt/yt/library/profiling/solomon/sensor_dump.proto | 2 | ||||
-rw-r--r-- | yt/yt/library/profiling/solomon/sensor_service.cpp | 2 | ||||
-rw-r--r-- | yt/yt/library/profiling/solomon/sensor_set.cpp | 41 | ||||
-rw-r--r-- | yt/yt/library/profiling/solomon/sensor_set.h | 6 | ||||
-rw-r--r-- | yt/yt/library/profiling/unittests/exporter_ut.cpp | 36 | ||||
-rw-r--r-- | yt/yt/library/profiling/unittests/solomon_ut.cpp | 73 |
14 files changed, 361 insertions, 87 deletions
diff --git a/yt/yt/library/profiling/sensor.cpp b/yt/yt/library/profiling/sensor.cpp index df47f260b5..8b8ed3d712 100644 --- a/yt/yt/library/profiling/sensor.cpp +++ b/yt/yt/library/profiling/sensor.cpp @@ -13,6 +13,29 @@ namespace NYT::NProfiling { //////////////////////////////////////////////////////////////////////////////// +TSummaryPolicyConflicts GetSummaryPolicyConflicts(ESummaryPolicy policy) +{ + bool isAllPolicy = Any(policy & ESummaryPolicy::All); + int specifiedAggregateCount = Any(policy & ESummaryPolicy::Sum) + + Any(policy & ESummaryPolicy::Min) + + Any(policy & ESummaryPolicy::Max) + + Any(policy & ESummaryPolicy::Avg); + + return { + .AllPolicyWithSpecifiedAggregates = isAllPolicy && specifiedAggregateCount > 0, + .OmitNameLabelSuffixWithSeveralAggregates = (isAllPolicy || specifiedAggregateCount != 1) && + Any(policy & ESummaryPolicy::OmitNameLabelSuffix), + }; +} + +bool CheckSummaryPolicy(ESummaryPolicy policy) +{ + auto conflicts = GetSummaryPolicyConflicts(policy); + return !conflicts.AllPolicyWithSpecifiedAggregates && !conflicts.OmitNameLabelSuffixWithSeveralAggregates; +} + +//////////////////////////////////////////////////////////////////////////////// + void TCounter::Increment(i64 delta) const { if (!Counter_) { @@ -248,15 +271,15 @@ TRateHistogram::operator bool() const TString ToString(const TSensorOptions& options) { return Format( - "{sparse=%v;global=%v;hot=%v;histogram_min=%v;histogram_max=%v;time_histogram_bounds=%v;histogram_bounds=%v}", + "{sparse=%v;global=%v;hot=%v;histogram_min=%v;histogram_max=%v;time_histogram_bounds=%v;histogram_bounds=%v;summary_policy=%v}", options.Sparse, options.Global, options.Hot, options.HistogramMin, options.HistogramMax, options.TimeHistogramBounds, - options.HistogramBounds - ); + options.HistogramBounds, + options.SummaryPolicy); } bool TSensorOptions::IsCompatibleWith(const TSensorOptions& other) const @@ -265,7 +288,8 @@ bool TSensorOptions::IsCompatibleWith(const TSensorOptions& other) const Global == other.Global && DisableSensorsRename == other.DisableSensorsRename && DisableDefault == other.DisableDefault && - DisableProjections == other.DisableProjections; + DisableProjections == other.DisableProjections && + SummaryPolicy == other.SummaryPolicy; } //////////////////////////////////////////////////////////////////////////////// @@ -516,36 +540,45 @@ TTimeGauge TProfiler::TimeGauge(const TString& name) const return gauge; } -TSummary TProfiler::Summary(const TString& name) const +TSummary TProfiler::Summary(const TString& name, ESummaryPolicy summaryPolicy) const { if (!Impl_) { return {}; } + auto options = Options_; + options.SummaryPolicy = summaryPolicy; + TSummary summary; - summary.Summary_ = Impl_->RegisterSummary(Namespace_ + Prefix_ + name, Tags_, Options_); + summary.Summary_ = Impl_->RegisterSummary(Namespace_ + Prefix_ + name, Tags_, std::move(options)); return summary; } -TGauge TProfiler::GaugeSummary(const TString& name) const +TGauge TProfiler::GaugeSummary(const TString& name, ESummaryPolicy summaryPolicy) const { if (!Impl_) { return {}; } + auto options = Options_; + options.SummaryPolicy = summaryPolicy; + TGauge gauge; - gauge.Gauge_ = Impl_->RegisterGaugeSummary(Namespace_ + Prefix_ + name, Tags_, Options_); + gauge.Gauge_ = Impl_->RegisterGaugeSummary(Namespace_ + Prefix_ + name, Tags_, std::move(options)); return gauge; } -TTimeGauge TProfiler::TimeGaugeSummary(const TString& name) const +TTimeGauge TProfiler::TimeGaugeSummary(const TString& name, ESummaryPolicy summaryPolicy) const { if (!Impl_) { return {}; } + auto options = Options_; + options.SummaryPolicy = summaryPolicy; + TTimeGauge gauge; - gauge.Gauge_ = Impl_->RegisterTimeGaugeSummary(Namespace_ + Prefix_ + name, Tags_, Options_); + gauge.Gauge_ = Impl_->RegisterTimeGaugeSummary(Namespace_ + Prefix_ + name, Tags_, std::move(options)); return gauge; } diff --git a/yt/yt/library/profiling/sensor.h b/yt/yt/library/profiling/sensor.h index 196f76404a..2aa7299f32 100644 --- a/yt/yt/library/profiling/sensor.h +++ b/yt/yt/library/profiling/sensor.h @@ -13,6 +13,8 @@ #include <library/cpp/yt/cpu_clock/clock.h> +#include <library/cpp/yt/misc/enum.h> + #include <vector> namespace NYT::NProfiling { @@ -175,6 +177,30 @@ private: //////////////////////////////////////////////////////////////////////////////// +DEFINE_BIT_ENUM_WITH_UNDERLYING_TYPE(ESummaryPolicy, ui8, + ((Default) (0x0000)) + // Aggregation policy. + ((All) (0x0001)) + ((Sum) (0x0002)) + ((Min) (0x0004)) + ((Max) (0x0008)) + ((Avg) (0x0010)) + // Export policy. + ((OmitNameLabelSuffix) (0x0020)) +); + +struct TSummaryPolicyConflicts +{ + bool AllPolicyWithSpecifiedAggregates; + bool OmitNameLabelSuffixWithSeveralAggregates; +}; + +TSummaryPolicyConflicts GetSummaryPolicyConflicts(ESummaryPolicy policy); + +bool CheckSummaryPolicy(ESummaryPolicy policy); + +//////////////////////////////////////////////////////////////////////////////// + struct TSensorOptions { bool Global = false; @@ -192,6 +218,8 @@ struct TSensorOptions std::vector<double> HistogramBounds; + ESummaryPolicy SummaryPolicy = ESummaryPolicy::Default; + bool IsCompatibleWith(const TSensorOptions& other) const; }; @@ -314,19 +342,19 @@ public: TTimeGauge TimeGauge(const TString& name) const; //! Summary is used to measure distribution of values. - TSummary Summary(const TString& name) const; + TSummary Summary(const TString& name, ESummaryPolicy summaryPolicy = ESummaryPolicy::Default) const; //! GaugeSummary is used to aggregate multiple values locally. /*! * Each TGauge tracks single value. Values are aggregated using Summary rules. */ - TGauge GaugeSummary(const TString& name) const; + TGauge GaugeSummary(const TString& name, ESummaryPolicy summaryPolicy = ESummaryPolicy::Default) const; //! TimeGaugeSummary is used to aggregate multiple values locally. /*! * Each TGauge tracks single value. Values are aggregated using Summary rules. */ - TTimeGauge TimeGaugeSummary(const TString& name) const; + TTimeGauge TimeGaugeSummary(const TString& name, ESummaryPolicy summaryPolicy = ESummaryPolicy::Default) const; //! Timer is used to measure distribution of event durations. /*! diff --git a/yt/yt/library/profiling/solomon/cube.cpp b/yt/yt/library/profiling/solomon/cube.cpp index 213fc560aa..5b8f72d6b7 100644 --- a/yt/yt/library/profiling/solomon/cube.cpp +++ b/yt/yt/library/profiling/solomon/cube.cpp @@ -179,6 +179,8 @@ int TCube<T>::ReadSensors( TTagWriter* tagWriter, ::NMonitoring::IMetricConsumer* consumer) const { + YT_VERIFY(CheckSummaryPolicy(options.SummaryPolicy)); + int sensorsEmitted = 0; auto prepareNameLabel = [&] (std::optional<TStringBuf> suffix) { @@ -217,6 +219,8 @@ int TCube<T>::ReadSensors( }; auto nameLabel = prepareNameLabel({}); + auto sumNameLabel = prepareNameLabel(".sum"); + auto minNameLabel = prepareNameLabel(".min"); auto maxNameLabel = prepareNameLabel(".max"); auto avgNameLabel = prepareNameLabel(".avg"); auto rateNameLabel = prepareNameLabel(".rate"); @@ -224,6 +228,8 @@ int TCube<T>::ReadSensors( auto hostLabel = consumer->PrepareLabel("host", options.Host.value_or("")); auto ytAggrLabel = consumer->PrepareLabel("yt_aggr", "1"); + // Set allowAggregate to true to aggregate by host. + // Currently Monitoring supports only sum aggregation. auto writeLabels = [&] (const auto& tagIds, std::pair<ui32, ui32> nameLabel, bool allowAggregate) { consumer->OnLabelsBegin(); @@ -338,34 +344,74 @@ int TCube<T>::ReadSensors( }; auto writeSummary = [&, tagIds=tagIds] (auto makeSummary) { - if (options.ExportSummary) { - consumer->OnMetricBegin(NMonitoring::EMetricType::DSUMMARY); - writeLabels(tagIds, nameLabel, true); + bool omitSuffix = Any(options.SummaryPolicy & ESummaryPolicy::OmitNameLabelSuffix); - rangeValues([&] (auto value, auto time, const auto& /* indices */) { - sensorCount += 5; - consumer->OnSummaryDouble(time, makeSummary(value)); - }); + auto writeMetric = [&] ( + ESummaryPolicy policyBit, + std::pair<ui32, ui32> specificNameLabel, + bool aggregate, + NMonitoring::EMetricType type, + auto cb) + { + if (Any(options.SummaryPolicy & policyBit)) { + consumer->OnMetricBegin(type); + writeLabels(tagIds, omitSuffix ? nameLabel : specificNameLabel, aggregate); - consumer->OnMetricEnd(); - } + rangeValues(cb); - if (options.ExportSummaryAsMax) { - consumer->OnMetricBegin(NMonitoring::EMetricType::GAUGE); - writeLabels(tagIds, maxNameLabel, false); - - rangeValues([&] (auto value, auto time, const auto& /* indices */) { - sensorCount += 1; - consumer->OnDouble(time, makeSummary(value)->GetMax()); + consumer->OnMetricEnd(); + } + }; + + auto writeGaugeSummaryMetric = [&] ( + ESummaryPolicy policyBit, + std::pair<ui32, ui32> specificNameLabel, + bool aggregate, + double (NMonitoring::TSummaryDoubleSnapshot::*valueGetter)() const) + { + writeMetric( + policyBit, + specificNameLabel, + aggregate, + NMonitoring::EMetricType::GAUGE, + [&] (auto value, auto time, const auto& /*indices*/) { + sensorCount += 1; + consumer->OnDouble(time, (makeSummary(value).Get()->*valueGetter)()); + }); + }; + + writeMetric( + ESummaryPolicy::All, + nameLabel, + /*aggregate*/ true, + NMonitoring::EMetricType::DSUMMARY, + [&] (auto value, auto time, const auto& /*indices*/) { + sensorCount += 5; + consumer->OnSummaryDouble(time, makeSummary(value)); }); - consumer->OnMetricEnd(); - } - - if (options.ExportSummaryAsAvg) { + writeGaugeSummaryMetric( + ESummaryPolicy::Sum, + sumNameLabel, + /*aggregate*/ true, + &NMonitoring::TSummaryDoubleSnapshot::GetSum); + + writeGaugeSummaryMetric( + ESummaryPolicy::Min, + minNameLabel, + /*aggregate*/ false, + &NMonitoring::TSummaryDoubleSnapshot::GetMin); + + writeGaugeSummaryMetric( + ESummaryPolicy::Max, + maxNameLabel, + /*aggregate*/ false, + &NMonitoring::TSummaryDoubleSnapshot::GetMax); + + if (Any(options.SummaryPolicy & ESummaryPolicy::Avg)) { bool empty = true; - rangeValues([&] (auto value, auto time, const auto& /* indices */) { + rangeValues([&] (auto value, auto time, const auto& /*indices*/) { auto snapshot = makeSummary(value); if (snapshot->GetCount() == 0) { return; @@ -374,7 +420,7 @@ int TCube<T>::ReadSensors( if (empty) { empty = false; consumer->OnMetricBegin(NMonitoring::EMetricType::GAUGE); - writeLabels(tagIds, avgNameLabel, false); + writeLabels(tagIds, omitSuffix ? nameLabel : avgNameLabel, false); } sensorCount += 1; @@ -575,15 +621,7 @@ int TCube<T>::ReadSensorValues( fluent.Value(value); ++valuesRead; } else if constexpr (std::is_same_v<T, TSummarySnapshot<double>>) { - if (options.ExportSummaryAsMax && options.SummaryAsMaxForAllTime) { - fluent - .BeginMap() - .Item("max").Value(value.Max()) - .Item("all_time_max").Value(Rollup(projection, index).Max()) - .EndMap(); - } else if (options.ExportSummaryAsMax) { - fluent.Value(value.Max()); - } else { + if (Any(options.SummaryPolicy & ESummaryPolicy::All)) { fluent .BeginMap() .Item("sum").Value(value.Sum()) @@ -592,16 +630,32 @@ int TCube<T>::ReadSensorValues( .Item("last").Value(value.Last()) .Item("count").Value(static_cast<ui64>(value.Count())) .EndMap(); + } else if (Any(options.SummaryPolicy & ESummaryPolicy::Sum)) { + fluent.Value(value.Sum()); + } else if (Any(options.SummaryPolicy & ESummaryPolicy::Min)) { + fluent.Value(value.Min()); + } else if (Any(options.SummaryPolicy & ESummaryPolicy::Max)) { + if (options.SummaryAsMaxForAllTime) { + fluent + .BeginMap() + .Item("max").Value(value.Max()) + .Item("all_time_max").Value(Rollup(projection, index).Max()) + .EndMap(); + } else { + fluent.Value(value.Max()); + } + } else if (Any(options.SummaryPolicy & ESummaryPolicy::Avg)) { + fluent.Value(value.Count() > 0 ? value.Sum() / value.Count() : NAN); } ++valuesRead; } else if constexpr (std::is_same_v<T, TSummarySnapshot<TDuration>>) { - if (options.ExportSummaryAsMax && options.SummaryAsMaxForAllTime) { + if (Any(options.SummaryPolicy & ESummaryPolicy::Max) && options.SummaryAsMaxForAllTime) { fluent .BeginMap() .Item("max").Value(value.Max().SecondsFloat()) .Item("all_time_max").Value(Rollup(projection, index).Max().SecondsFloat()) .EndMap(); - } else if (options.ExportSummaryAsMax) { + } else if (Any(options.SummaryPolicy & ESummaryPolicy::Max)) { fluent.Value(value.Max().SecondsFloat()); } else { fluent diff --git a/yt/yt/library/profiling/solomon/cube.h b/yt/yt/library/profiling/solomon/cube.h index 57aaed994b..7e51f6f961 100644 --- a/yt/yt/library/profiling/solomon/cube.h +++ b/yt/yt/library/profiling/solomon/cube.h @@ -31,9 +31,7 @@ struct TReadOptions bool EnableSolomonAggregationWorkaround = false; // Direct summary export is not supported by solomon, yet. - bool ExportSummary = false; - bool ExportSummaryAsMax = false; - bool ExportSummaryAsAvg = false; + ESummaryPolicy SummaryPolicy = ESummaryPolicy::Default; bool MarkAggregates = false; diff --git a/yt/yt/library/profiling/solomon/exporter.cpp b/yt/yt/library/profiling/solomon/exporter.cpp index d03f553620..4e6ec19b2d 100644 --- a/yt/yt/library/profiling/solomon/exporter.cpp +++ b/yt/yt/library/profiling/solomon/exporter.cpp @@ -166,6 +166,22 @@ TShardConfigPtr TSolomonExporterConfig::MatchShard(const TString& sensorName) return matchedShard; } +ESummaryPolicy TSolomonExporterConfig::GetSummaryPolicy() const +{ + auto policy = ESummaryPolicy::Default; + if (ExportSummary) { + policy |= ESummaryPolicy::All; + } + if (ExportSummaryAsMax) { + policy |= ESummaryPolicy::Max; + } + if (ExportSummaryAsAvg) { + policy |= ESummaryPolicy::Avg; + } + + return policy; +} + //////////////////////////////////////////////////////////////////////////////// TSolomonExporter::TSolomonExporter( @@ -540,9 +556,10 @@ bool TSolomonExporter::ReadSensors( readOptions.Times.emplace_back(std::vector<int>{Registry_->IndexOf(Window_.back().first)}, TInstant::Zero()); readOptions.ConvertCountersToRateGauge = false; readOptions.EnableHistogramCompat = true; - readOptions.ExportSummary |= Config_->ExportSummary; - readOptions.ExportSummaryAsMax |= Config_->ExportSummaryAsMax; - readOptions.ExportSummaryAsAvg |= Config_->ExportSummaryAsAvg; + + readOptions.SummaryPolicy |= Config_->GetSummaryPolicy(); + ValidateSummaryPolicy(readOptions.SummaryPolicy); + readOptions.MarkAggregates |= Config_->MarkAggregates; if (!readOptions.Host && Config_->Host) { readOptions.Host = Config_->Host; @@ -786,9 +803,7 @@ void TSolomonExporter::DoHandleShard( options.EnableSolomonAggregationWorkaround = isSolomon; options.Times = readWindow; - options.ExportSummary = Config_->ExportSummary; - options.ExportSummaryAsMax = Config_->ExportSummaryAsMax; - options.ExportSummaryAsAvg = Config_->ExportSummaryAsAvg; + options.SummaryPolicy = Config_->GetSummaryPolicy(); options.MarkAggregates = Config_->MarkAggregates; options.StripSensorsNamePrefix = Config_->StripSensorsNamePrefix; options.LingerWindowSize = Config_->LingerTimeout / gridStep; @@ -973,6 +988,23 @@ void TSolomonExporter::CleanResponseCache() } } +void TSolomonExporter::ValidateSummaryPolicy(ESummaryPolicy policy) +{ + static const TError SummaryPolicyError("Invalid summary policy in read options"); + + auto summaryPolicyConflicts = GetSummaryPolicyConflicts(policy); + if (summaryPolicyConflicts.AllPolicyWithSpecifiedAggregates) { + THROW_ERROR SummaryPolicyError + << TError("%Qv policy can be used only without specified policies", ESummaryPolicy::All) + << TErrorAttribute("policy", policy); + } + if (summaryPolicyConflicts.OmitNameLabelSuffixWithSeveralAggregates) { + THROW_ERROR SummaryPolicyError + << TError("%Qv option can be used only with single specified policy", ESummaryPolicy::OmitNameLabelSuffix) + << TErrorAttribute("policy", policy); + } +} + bool TSolomonExporter::FilterDefaultGrid(const TString& sensorName) { auto shard = Config_->MatchShard(sensorName); diff --git a/yt/yt/library/profiling/solomon/exporter.h b/yt/yt/library/profiling/solomon/exporter.h index 4cfa47e9b9..6ec0ce7dc1 100644 --- a/yt/yt/library/profiling/solomon/exporter.h +++ b/yt/yt/library/profiling/solomon/exporter.h @@ -86,6 +86,8 @@ struct TSolomonExporterConfig TShardConfigPtr MatchShard(const TString& sensorName); + ESummaryPolicy GetSummaryPolicy() const; + REGISTER_YSON_STRUCT(TSolomonExporterConfig); static void Register(TRegistrar registrar); @@ -210,6 +212,8 @@ private: bool FilterDefaultGrid(const TString& sensorName); + static void ValidateSummaryPolicy(ESummaryPolicy policy); + // For locking. friend class TSensorService; }; diff --git a/yt/yt/library/profiling/solomon/registry.cpp b/yt/yt/library/profiling/solomon/registry.cpp index bd24c87b04..b231a64782 100644 --- a/yt/yt/library/profiling/solomon/registry.cpp +++ b/yt/yt/library/profiling/solomon/registry.cpp @@ -40,7 +40,7 @@ ICounterImplPtr TSolomonRegistry::RegisterCounter( TSensorOptions options) { return SelectImpl<ICounterImpl, TSimpleCounter, TPerCpuCounter>(options.Hot, [&, this] (const auto& counter) { - DoRegister([this, name, tags, options, counter] () { + DoRegister([this, name, tags, options = std::move(options), counter] () { auto reader = [ptr = counter.Get()] { return ptr->GetValue(); }; @@ -59,7 +59,7 @@ ITimeCounterImplPtr TSolomonRegistry::RegisterTimeCounter( return SelectImpl<ITimeCounterImpl, TSimpleTimeCounter, TPerCpuTimeCounter>( options.Hot, [&, this] (const auto& counter) { - DoRegister([this, name, tags, options, counter] () { + DoRegister([this, name, tags, options = std::move(options), counter] () { auto set = FindSet(name, options); set->AddTimeCounter(New<TTimeCounterState>(counter, Tags_.Encode(tags), tags)); }); @@ -76,7 +76,7 @@ IGaugeImplPtr TSolomonRegistry::RegisterGauge( gauge->Update(std::numeric_limits<double>::quiet_NaN()); } - DoRegister([this, name, tags, options, gauge] () { + DoRegister([this, name, tags, options = std::move(options), gauge] () { auto reader = [ptr = gauge.Get()] { return ptr->GetValue(); }; @@ -94,7 +94,7 @@ ITimeGaugeImplPtr TSolomonRegistry::RegisterTimeGauge( { auto gauge = New<TSimpleTimeGauge>(); - DoRegister([this, name, tags, options, gauge] () { + DoRegister([this, name, tags, options = std::move(options), gauge] () { auto reader = [ptr = gauge.Get()] { return ptr->GetValue().SecondsFloat(); }; @@ -112,7 +112,7 @@ ISummaryImplPtr TSolomonRegistry::RegisterSummary( TSensorOptions options) { return SelectImpl<ISummaryImpl, TSimpleSummary<double>, TPerCpuSummary<double>>(options.Hot, [&, this] (const auto& summary) { - DoRegister([this, name, tags, options, summary] () { + DoRegister([this, name, tags, options = std::move(options), summary] () { auto set = FindSet(name, options); set->AddSummary(New<TSummaryState>(summary, Tags_.Encode(tags), tags)); }); @@ -125,7 +125,7 @@ IGaugeImplPtr TSolomonRegistry::RegisterGaugeSummary( TSensorOptions options) { auto gauge = New<TSimpleGauge>(); - DoRegister([this, name, tags, options, gauge] () { + DoRegister([this, name, tags, options = std::move(options), gauge] () { auto set = FindSet(name, options); set->AddSummary(New<TSummaryState>(gauge, Tags_.Encode(tags), tags)); }); @@ -139,7 +139,7 @@ ITimeGaugeImplPtr TSolomonRegistry::RegisterTimeGaugeSummary( TSensorOptions options) { auto gauge = New<TSimpleTimeGauge>(); - DoRegister([this, name, tags, options, gauge] () { + DoRegister([this, name, tags, options = std::move(options), gauge] () { auto set = FindSet(name, options); set->AddTimerSummary(New<TTimerSummaryState>(gauge, Tags_.Encode(tags), tags)); }); @@ -155,7 +155,7 @@ ITimerImplPtr TSolomonRegistry::RegisterTimerSummary( return SelectImpl<ITimerImpl, TSimpleSummary<TDuration>, TPerCpuSummary<TDuration>>( options.Hot, [&, this] (const auto& timer) { - DoRegister([this, name, tags, options, timer] () { + DoRegister([this, name, tags, options = std::move(options), timer] () { auto set = FindSet(name, options); set->AddTimerSummary(New<TTimerSummaryState>(timer, Tags_.Encode(tags), tags)); }); @@ -168,7 +168,7 @@ ITimerImplPtr TSolomonRegistry::RegisterTimeHistogram( TSensorOptions options) { auto hist = New<THistogram>(options); - DoRegister([this, name, tags, options, hist] () { + DoRegister([this, name, tags, options = std::move(options), hist] () { auto set = FindSet(name, options); set->AddTimeHistogram(New<THistogramState>(hist, Tags_.Encode(tags), tags)); }); @@ -181,7 +181,7 @@ IHistogramImplPtr TSolomonRegistry::RegisterGaugeHistogram( TSensorOptions options) { auto hist = New<THistogram>(options); - DoRegister([this, name, tags, options, hist] () { + DoRegister([this, name, tags, options = std::move(options), hist] () { auto set = FindSet(name, options); set->AddGaugeHistogram(New<THistogramState>(hist, Tags_.Encode(tags), tags)); }); @@ -194,7 +194,7 @@ IHistogramImplPtr TSolomonRegistry::RegisterRateHistogram( TSensorOptions options) { auto hist = New<THistogram>(options); - DoRegister([this, name, tags, options, hist] () { + DoRegister([this, name, tags, options = std::move(options), hist] () { auto set = FindSet(name, options); set->AddRateHistogram(New<THistogramState>(hist, Tags_.Encode(tags), tags)); }); @@ -208,7 +208,7 @@ void TSolomonRegistry::RegisterFuncCounter( const TRefCountedPtr& owner, std::function<i64()> reader) { - DoRegister([this, name, tags, options, owner, reader] () { + DoRegister([this, name, tags, options = std::move(options), owner, reader] () { auto set = FindSet(name, options); set->AddCounter(New<TCounterState>(owner, reader, Tags_.Encode(tags), tags)); }); @@ -221,7 +221,7 @@ void TSolomonRegistry::RegisterFuncGauge( const TRefCountedPtr& owner, std::function<double()> reader) { - DoRegister([this, name, tags, options, owner, reader] () { + DoRegister([this, name, tags, options = std::move(options), owner, reader] () { auto set = FindSet(name, options); set->AddGauge(New<TGaugeState>(owner, reader, Tags_.Encode(tags), tags)); }); @@ -233,7 +233,7 @@ void TSolomonRegistry::RegisterProducer( TSensorOptions options, const ISensorProducerPtr& producer) { - DoRegister([this, prefix, tags, options, producer] () { + DoRegister([this, prefix, tags, options = std::move(options), producer] () { Producers_.AddProducer(New<TProducerState>(prefix, tags, options, producer)); }); } diff --git a/yt/yt/library/profiling/solomon/remote.cpp b/yt/yt/library/profiling/solomon/remote.cpp index 5e177de31e..b335782a12 100644 --- a/yt/yt/library/profiling/solomon/remote.cpp +++ b/yt/yt/library/profiling/solomon/remote.cpp @@ -105,6 +105,7 @@ void TRemoteRegistry::Transfer(const NProto::TSensorDump& dump) options.Global = cube.global(); options.DisableSensorsRename = cube.disable_sensors_rename(); options.DisableDefault = cube.disable_default(); + options.SummaryPolicy = NYT::FromProto<ESummaryPolicy>(cube.summary_policy()); auto sensorName = cube.name(); auto sensorSet = Registry_->FindSet(cube.name(), options); diff --git a/yt/yt/library/profiling/solomon/sensor_dump.proto b/yt/yt/library/profiling/solomon/sensor_dump.proto index 40d775c6bb..7180e0af39 100644 --- a/yt/yt/library/profiling/solomon/sensor_dump.proto +++ b/yt/yt/library/profiling/solomon/sensor_dump.proto @@ -18,6 +18,8 @@ message TCube { optional bool global = 4 [default=false]; optional bool disable_sensors_rename = 5 [default=false]; optional bool disable_default = 6 [default=false]; + + optional uint64 summary_policy = 7; // ESummaryPolicy }; message TProjection { diff --git a/yt/yt/library/profiling/solomon/sensor_service.cpp b/yt/yt/library/profiling/solomon/sensor_service.cpp index 04de53b703..94520feb80 100644 --- a/yt/yt/library/profiling/solomon/sensor_service.cpp +++ b/yt/yt/library/profiling/solomon/sensor_service.cpp @@ -100,7 +100,7 @@ private: TTagList tags(options.TagMap.begin(), options.TagMap.end()); TReadOptions readOptions{ - .ExportSummaryAsMax = options.ExportSummaryAsMax, + .SummaryPolicy = options.ExportSummaryAsMax ? ESummaryPolicy::Max : ESummaryPolicy::Default, .ReadAllProjections = options.ReadAllProjections, .SummaryAsMaxForAllTime = options.SummaryAsMaxForAllTime, }; diff --git a/yt/yt/library/profiling/solomon/sensor_set.cpp b/yt/yt/library/profiling/solomon/sensor_set.cpp index 1e9fa0ba5e..489923fbdb 100644 --- a/yt/yt/library/profiling/solomon/sensor_set.cpp +++ b/yt/yt/library/profiling/solomon/sensor_set.cpp @@ -1,6 +1,8 @@ #include "sensor_set.h" #include "private.h" +#include <yt/yt/core/misc/protobuf_helpers.h> + #include <library/cpp/yt/assert/assert.h> #include <library/cpp/monlib/metrics/summary_snapshot.h> @@ -15,8 +17,12 @@ const static auto& Logger = SolomonLogger; //////////////////////////////////////////////////////////////////////////////// -TSensorSet::TSensorSet(TSensorOptions options, i64 iteration, int windowSize, int gridFactor) - : Options_(options) +TSensorSet::TSensorSet( + TSensorOptions options, + i64 iteration, + int windowSize, + int gridFactor) + : Options_(std::move(options)) , GridFactor_(gridFactor) , CountersCube_{windowSize, iteration} , TimeCountersCube_{windowSize, iteration} @@ -45,7 +51,7 @@ void TSensorSet::Profile(const TProfiler &profiler) SensorsEmitted_ = profiler.Gauge("/sensors_emitted"); } -void TSensorSet::ValidateOptions(TSensorOptions options) +void TSensorSet::ValidateOptions(const TSensorOptions& options) { if (!Options_.IsCompatibleWith(options)) { OnError(TError("Conflicting sensor settings") @@ -278,7 +284,7 @@ int TSensorSet::Collect() void TSensorSet::ReadSensors( const TString& name, - const TReadOptions& options, + TReadOptions readOptions, TTagWriter* tagWriter, ::NMonitoring::IMetricConsumer* consumer) const { @@ -286,11 +292,13 @@ void TSensorSet::ReadSensors( return; } - auto readOptions = options; readOptions.Sparse = Options_.Sparse; readOptions.Global = Options_.Global; readOptions.DisableSensorsRename = Options_.DisableSensorsRename; readOptions.DisableDefault = Options_.DisableDefault; + if (Options_.SummaryPolicy != ESummaryPolicy::Default) { + readOptions.SummaryPolicy = Options_.SummaryPolicy; + } int sensorsEmitted = 0; @@ -309,7 +317,7 @@ void TSensorSet::ReadSensors( int TSensorSet::ReadSensorValues( const TTagIdList& tagIds, int index, - const TReadOptions& options, + TReadOptions readOptions, const TTagRegistry& tagRegistry, TFluentAny fluent) const { @@ -318,15 +326,19 @@ int TSensorSet::ReadSensorValues( << Error_; } + if (Options_.SummaryPolicy != ESummaryPolicy::Default) { + readOptions.SummaryPolicy = Options_.SummaryPolicy; + } + int valuesRead = 0; - valuesRead += CountersCube_.ReadSensorValues(tagIds, index, options, tagRegistry, fluent); - valuesRead += TimeCountersCube_.ReadSensorValues(tagIds, index, options, tagRegistry, fluent); - valuesRead += GaugesCube_.ReadSensorValues(tagIds, index, options, tagRegistry, fluent); - valuesRead += SummariesCube_.ReadSensorValues(tagIds, index, options, tagRegistry, fluent); - valuesRead += TimersCube_.ReadSensorValues(tagIds, index, options, tagRegistry, fluent); - valuesRead += TimeHistogramsCube_.ReadSensorValues(tagIds, index, options, tagRegistry, fluent); - valuesRead += GaugeHistogramsCube_.ReadSensorValues(tagIds, index, options, tagRegistry, fluent); - valuesRead += RateHistogramsCube_.ReadSensorValues(tagIds, index, options, tagRegistry, fluent); + valuesRead += CountersCube_.ReadSensorValues(tagIds, index, readOptions, tagRegistry, fluent); + valuesRead += TimeCountersCube_.ReadSensorValues(tagIds, index, readOptions, tagRegistry, fluent); + valuesRead += GaugesCube_.ReadSensorValues(tagIds, index, readOptions, tagRegistry, fluent); + valuesRead += SummariesCube_.ReadSensorValues(tagIds, index, readOptions, tagRegistry, fluent); + valuesRead += TimersCube_.ReadSensorValues(tagIds, index, readOptions, tagRegistry, fluent); + valuesRead += TimeHistogramsCube_.ReadSensorValues(tagIds, index, readOptions, tagRegistry, fluent); + valuesRead += GaugeHistogramsCube_.ReadSensorValues(tagIds, index, readOptions, tagRegistry, fluent); + valuesRead += RateHistogramsCube_.ReadSensorValues(tagIds, index, readOptions, tagRegistry, fluent); return valuesRead; } @@ -400,6 +412,7 @@ void TSensorSet::DumpCube(NProto::TCube *cube) const cube->set_global(Options_.Global); cube->set_disable_default(Options_.DisableDefault); cube->set_disable_sensors_rename(Options_.DisableSensorsRename); + cube->set_summary_policy(ToProto<ui64>(Options_.SummaryPolicy)); CountersCube_.DumpCube(cube); TimeCountersCube_.DumpCube(cube); diff --git a/yt/yt/library/profiling/solomon/sensor_set.h b/yt/yt/library/profiling/solomon/sensor_set.h index 010db9aea8..fb71767bdd 100644 --- a/yt/yt/library/profiling/solomon/sensor_set.h +++ b/yt/yt/library/profiling/solomon/sensor_set.h @@ -190,7 +190,7 @@ public: bool IsEmpty() const; void Profile(const TProfiler& profiler); - void ValidateOptions(TSensorOptions options); + void ValidateOptions(const TSensorOptions& options); void AddCounter(TCounterStatePtr counter); void AddGauge(TGaugeStatePtr gauge); @@ -207,14 +207,14 @@ public: void ReadSensors( const TString& name, - const TReadOptions& options, + TReadOptions readOptions, TTagWriter* tagWriter, ::NMonitoring::IMetricConsumer* consumer) const; int ReadSensorValues( const TTagIdList& tagIds, int index, - const TReadOptions& options, + TReadOptions readOptions, const TTagRegistry& tagRegistry, NYTree::TFluentAny fluent) const; diff --git a/yt/yt/library/profiling/unittests/exporter_ut.cpp b/yt/yt/library/profiling/unittests/exporter_ut.cpp index 11c50734eb..2b62a504f4 100644 --- a/yt/yt/library/profiling/unittests/exporter_ut.cpp +++ b/yt/yt/library/profiling/unittests/exporter_ut.cpp @@ -218,5 +218,41 @@ TEST(TSolomonExporter, ReadSensorsStripSensorsOption) //////////////////////////////////////////////////////////////////////////////// +class TSummaryPolicyTest + : public ::testing::TestWithParam<ESummaryPolicy> +{ }; + +INSTANTIATE_TEST_SUITE_P( + TSolomonExporter, + TSummaryPolicyTest, + testing::Values( + ESummaryPolicy::All | ESummaryPolicy::Sum, + ESummaryPolicy::All | ESummaryPolicy::Max, + ESummaryPolicy::All | ESummaryPolicy::Min, + ESummaryPolicy::All | ESummaryPolicy::Avg, + ESummaryPolicy::All | ESummaryPolicy::OmitNameLabelSuffix, + ESummaryPolicy::Max | ESummaryPolicy::Avg | ESummaryPolicy::OmitNameLabelSuffix)); + +TEST_P(TSummaryPolicyTest, InvalidReadOptions) +{ + auto registry = New<TSolomonRegistry>(); + TProfiler profiler(registry, "yt"); + + auto config = New<TSolomonExporterConfig>(); + config->ExportSummaryAsMax = false; + config->EnableSelfProfiling = false; + config->Shards.emplace("yt", New<TShardConfig>()); + + auto exporter = New<TSolomonExporter>(config, registry); + exporter->Start(); + + Sleep(TDuration::Seconds(5)); + + ASSERT_THROW(exporter->ReadJson({.SummaryPolicy = GetParam()}, "yt"), TErrorException); + exporter->Stop(); +} + +//////////////////////////////////////////////////////////////////////////////// + } // namespace } // namespace NYT::NProfiling diff --git a/yt/yt/library/profiling/unittests/solomon_ut.cpp b/yt/yt/library/profiling/unittests/solomon_ut.cpp index 708a8af93b..2efaf3e3cc 100644 --- a/yt/yt/library/profiling/unittests/solomon_ut.cpp +++ b/yt/yt/library/profiling/unittests/solomon_ut.cpp @@ -103,6 +103,7 @@ struct TTestMetricConsumer << ", count: " << snapshot->GetCount() << ", last: " << snapshot->GetLast() << "}" << Endl; + Summaries[FormatName()] = snapshot; } TString Name; @@ -110,6 +111,7 @@ struct TTestMetricConsumer THashMap<TString, i64> Counters; THashMap<TString, double> Gauges; + THashMap<TString, NMonitoring::ISummaryDoubleSnapshotPtr> Summaries; THashMap<TString, NMonitoring::IHistogramSnapshotPtr> Histograms; std::vector<TString> LabelsCache; @@ -874,5 +876,76 @@ TEST(TSolomonRegistry, ProducerRemoveSupport) //////////////////////////////////////////////////////////////////////////////// +class TGaugeSummaryTriple +{ +public: + TGaugeSummaryTriple(TProfiler* profiler, const TString& name, ESummaryPolicy policy) + : First_(profiler->GaugeSummary(name, policy)) + , Second_(profiler->GaugeSummary(name, policy)) + , Third_(profiler->GaugeSummary(name, policy)) + { } + + void Update(double a, double b, double c) + { + First_.Update(a); + Second_.Update(b); + Third_.Update(c); + } + +private: + TGauge First_; + TGauge Second_; + TGauge Third_; +}; + +class TOmitNameLabelSuffixTest + : public ::testing::TestWithParam<bool> +{ }; + +INSTANTIATE_TEST_SUITE_P( + TSolomonRegistry, + TOmitNameLabelSuffixTest, + testing::Values(false, true)); + +TEST_P(TOmitNameLabelSuffixTest, GaugeSummary) +{ + bool omitNameLabelSuffix = GetParam(); + + auto impl = New<TSolomonRegistry>(); + impl->SetWindowSize(12); + TProfiler profiler(impl, "/d"); + + ESummaryPolicy additionalPolicy = omitNameLabelSuffix ? ESummaryPolicy::OmitNameLabelSuffix : ESummaryPolicy::Default; + + TGaugeSummaryTriple all(&profiler, "all", ESummaryPolicy::All); + TGaugeSummaryTriple sum(&profiler, "sum", ESummaryPolicy::Sum | additionalPolicy); + TGaugeSummaryTriple min(&profiler, "min", ESummaryPolicy::Min | additionalPolicy); + TGaugeSummaryTriple max(&profiler, "max", ESummaryPolicy::Max | additionalPolicy); + TGaugeSummaryTriple avg(&profiler, "avg", ESummaryPolicy::Avg | additionalPolicy); + + all.Update(40, 20, 50); + + sum.Update(21, 31, 41); + min.Update(1337, 32, 322); + max.Update(22, 44, 11); + avg.Update(55, 44, 22); + + auto result = CollectSensors(impl); + auto& gauges = result.Gauges; + auto& summaries = result.Summaries; + + ASSERT_NEAR(summaries["yt.dall{}"]->GetSum(), 110, 1e-6); + ASSERT_NEAR(summaries["yt.dall{}"]->GetMin(), 20, 1e-6); + ASSERT_NEAR(summaries["yt.dall{}"]->GetMax(), 50, 1e-6); + ASSERT_NEAR(summaries["yt.dall{}"]->GetCount(), 3, 1e-6); + + ASSERT_NEAR(gauges[Format("yt.dsum%v{}", omitNameLabelSuffix ? "" : ".sum")], 93, 1e-6); + ASSERT_NEAR(gauges[Format("yt.dmin%v{}", omitNameLabelSuffix ? "" : ".min")], 32, 1e-6); + ASSERT_NEAR(gauges[Format("yt.dmax%v{}", omitNameLabelSuffix ? "" : ".max")], 44, 1e-6); + ASSERT_NEAR(gauges[Format("yt.davg%v{}", omitNameLabelSuffix ? "" : ".avg")], 40 + 1 / 3.0, 1e-6); +} + +//////////////////////////////////////////////////////////////////////////////// + } // namespace } // namespace NYT::NProfiling |