aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authordave11ar <dave11ar@yandex-team.com>2023-10-14 13:56:16 +0300
committerdave11ar <dave11ar@yandex-team.com>2023-10-14 14:17:10 +0300
commitca170f490f0f8181344b8a63afc3c262632b8636 (patch)
treef53d3a409db26e16231303c59f49eb433eb2573b
parent4371a757495f375ca7c5b1730f4e9f741a71ec40 (diff)
downloadydb-ca170f490f0f8181344b8a63afc3c262632b8636.tar.gz
YT-20007: Add per instance summary configuration, consider disabled profiling mode
-rw-r--r--yt/yt/library/profiling/sensor.cpp53
-rw-r--r--yt/yt/library/profiling/sensor.h34
-rw-r--r--yt/yt/library/profiling/solomon/cube.cpp120
-rw-r--r--yt/yt/library/profiling/solomon/cube.h4
-rw-r--r--yt/yt/library/profiling/solomon/exporter.cpp44
-rw-r--r--yt/yt/library/profiling/solomon/exporter.h4
-rw-r--r--yt/yt/library/profiling/solomon/registry.cpp28
-rw-r--r--yt/yt/library/profiling/solomon/remote.cpp1
-rw-r--r--yt/yt/library/profiling/solomon/sensor_dump.proto2
-rw-r--r--yt/yt/library/profiling/solomon/sensor_service.cpp2
-rw-r--r--yt/yt/library/profiling/solomon/sensor_set.cpp41
-rw-r--r--yt/yt/library/profiling/solomon/sensor_set.h6
-rw-r--r--yt/yt/library/profiling/unittests/exporter_ut.cpp36
-rw-r--r--yt/yt/library/profiling/unittests/solomon_ut.cpp73
14 files changed, 361 insertions, 87 deletions
diff --git a/yt/yt/library/profiling/sensor.cpp b/yt/yt/library/profiling/sensor.cpp
index df47f260b5..8b8ed3d712 100644
--- a/yt/yt/library/profiling/sensor.cpp
+++ b/yt/yt/library/profiling/sensor.cpp
@@ -13,6 +13,29 @@ namespace NYT::NProfiling {
////////////////////////////////////////////////////////////////////////////////
+TSummaryPolicyConflicts GetSummaryPolicyConflicts(ESummaryPolicy policy)
+{
+ bool isAllPolicy = Any(policy & ESummaryPolicy::All);
+ int specifiedAggregateCount = Any(policy & ESummaryPolicy::Sum) +
+ Any(policy & ESummaryPolicy::Min) +
+ Any(policy & ESummaryPolicy::Max) +
+ Any(policy & ESummaryPolicy::Avg);
+
+ return {
+ .AllPolicyWithSpecifiedAggregates = isAllPolicy && specifiedAggregateCount > 0,
+ .OmitNameLabelSuffixWithSeveralAggregates = (isAllPolicy || specifiedAggregateCount != 1) &&
+ Any(policy & ESummaryPolicy::OmitNameLabelSuffix),
+ };
+}
+
+bool CheckSummaryPolicy(ESummaryPolicy policy)
+{
+ auto conflicts = GetSummaryPolicyConflicts(policy);
+ return !conflicts.AllPolicyWithSpecifiedAggregates && !conflicts.OmitNameLabelSuffixWithSeveralAggregates;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
void TCounter::Increment(i64 delta) const
{
if (!Counter_) {
@@ -248,15 +271,15 @@ TRateHistogram::operator bool() const
TString ToString(const TSensorOptions& options)
{
return Format(
- "{sparse=%v;global=%v;hot=%v;histogram_min=%v;histogram_max=%v;time_histogram_bounds=%v;histogram_bounds=%v}",
+ "{sparse=%v;global=%v;hot=%v;histogram_min=%v;histogram_max=%v;time_histogram_bounds=%v;histogram_bounds=%v;summary_policy=%v}",
options.Sparse,
options.Global,
options.Hot,
options.HistogramMin,
options.HistogramMax,
options.TimeHistogramBounds,
- options.HistogramBounds
- );
+ options.HistogramBounds,
+ options.SummaryPolicy);
}
bool TSensorOptions::IsCompatibleWith(const TSensorOptions& other) const
@@ -265,7 +288,8 @@ bool TSensorOptions::IsCompatibleWith(const TSensorOptions& other) const
Global == other.Global &&
DisableSensorsRename == other.DisableSensorsRename &&
DisableDefault == other.DisableDefault &&
- DisableProjections == other.DisableProjections;
+ DisableProjections == other.DisableProjections &&
+ SummaryPolicy == other.SummaryPolicy;
}
////////////////////////////////////////////////////////////////////////////////
@@ -516,36 +540,45 @@ TTimeGauge TProfiler::TimeGauge(const TString& name) const
return gauge;
}
-TSummary TProfiler::Summary(const TString& name) const
+TSummary TProfiler::Summary(const TString& name, ESummaryPolicy summaryPolicy) const
{
if (!Impl_) {
return {};
}
+ auto options = Options_;
+ options.SummaryPolicy = summaryPolicy;
+
TSummary summary;
- summary.Summary_ = Impl_->RegisterSummary(Namespace_ + Prefix_ + name, Tags_, Options_);
+ summary.Summary_ = Impl_->RegisterSummary(Namespace_ + Prefix_ + name, Tags_, std::move(options));
return summary;
}
-TGauge TProfiler::GaugeSummary(const TString& name) const
+TGauge TProfiler::GaugeSummary(const TString& name, ESummaryPolicy summaryPolicy) const
{
if (!Impl_) {
return {};
}
+ auto options = Options_;
+ options.SummaryPolicy = summaryPolicy;
+
TGauge gauge;
- gauge.Gauge_ = Impl_->RegisterGaugeSummary(Namespace_ + Prefix_ + name, Tags_, Options_);
+ gauge.Gauge_ = Impl_->RegisterGaugeSummary(Namespace_ + Prefix_ + name, Tags_, std::move(options));
return gauge;
}
-TTimeGauge TProfiler::TimeGaugeSummary(const TString& name) const
+TTimeGauge TProfiler::TimeGaugeSummary(const TString& name, ESummaryPolicy summaryPolicy) const
{
if (!Impl_) {
return {};
}
+ auto options = Options_;
+ options.SummaryPolicy = summaryPolicy;
+
TTimeGauge gauge;
- gauge.Gauge_ = Impl_->RegisterTimeGaugeSummary(Namespace_ + Prefix_ + name, Tags_, Options_);
+ gauge.Gauge_ = Impl_->RegisterTimeGaugeSummary(Namespace_ + Prefix_ + name, Tags_, std::move(options));
return gauge;
}
diff --git a/yt/yt/library/profiling/sensor.h b/yt/yt/library/profiling/sensor.h
index 196f76404a..2aa7299f32 100644
--- a/yt/yt/library/profiling/sensor.h
+++ b/yt/yt/library/profiling/sensor.h
@@ -13,6 +13,8 @@
#include <library/cpp/yt/cpu_clock/clock.h>
+#include <library/cpp/yt/misc/enum.h>
+
#include <vector>
namespace NYT::NProfiling {
@@ -175,6 +177,30 @@ private:
////////////////////////////////////////////////////////////////////////////////
+DEFINE_BIT_ENUM_WITH_UNDERLYING_TYPE(ESummaryPolicy, ui8,
+ ((Default) (0x0000))
+ // Aggregation policy.
+ ((All) (0x0001))
+ ((Sum) (0x0002))
+ ((Min) (0x0004))
+ ((Max) (0x0008))
+ ((Avg) (0x0010))
+ // Export policy.
+ ((OmitNameLabelSuffix) (0x0020))
+);
+
+struct TSummaryPolicyConflicts
+{
+ bool AllPolicyWithSpecifiedAggregates;
+ bool OmitNameLabelSuffixWithSeveralAggregates;
+};
+
+TSummaryPolicyConflicts GetSummaryPolicyConflicts(ESummaryPolicy policy);
+
+bool CheckSummaryPolicy(ESummaryPolicy policy);
+
+////////////////////////////////////////////////////////////////////////////////
+
struct TSensorOptions
{
bool Global = false;
@@ -192,6 +218,8 @@ struct TSensorOptions
std::vector<double> HistogramBounds;
+ ESummaryPolicy SummaryPolicy = ESummaryPolicy::Default;
+
bool IsCompatibleWith(const TSensorOptions& other) const;
};
@@ -314,19 +342,19 @@ public:
TTimeGauge TimeGauge(const TString& name) const;
//! Summary is used to measure distribution of values.
- TSummary Summary(const TString& name) const;
+ TSummary Summary(const TString& name, ESummaryPolicy summaryPolicy = ESummaryPolicy::Default) const;
//! GaugeSummary is used to aggregate multiple values locally.
/*!
* Each TGauge tracks single value. Values are aggregated using Summary rules.
*/
- TGauge GaugeSummary(const TString& name) const;
+ TGauge GaugeSummary(const TString& name, ESummaryPolicy summaryPolicy = ESummaryPolicy::Default) const;
//! TimeGaugeSummary is used to aggregate multiple values locally.
/*!
* Each TGauge tracks single value. Values are aggregated using Summary rules.
*/
- TTimeGauge TimeGaugeSummary(const TString& name) const;
+ TTimeGauge TimeGaugeSummary(const TString& name, ESummaryPolicy summaryPolicy = ESummaryPolicy::Default) const;
//! Timer is used to measure distribution of event durations.
/*!
diff --git a/yt/yt/library/profiling/solomon/cube.cpp b/yt/yt/library/profiling/solomon/cube.cpp
index 213fc560aa..5b8f72d6b7 100644
--- a/yt/yt/library/profiling/solomon/cube.cpp
+++ b/yt/yt/library/profiling/solomon/cube.cpp
@@ -179,6 +179,8 @@ int TCube<T>::ReadSensors(
TTagWriter* tagWriter,
::NMonitoring::IMetricConsumer* consumer) const
{
+ YT_VERIFY(CheckSummaryPolicy(options.SummaryPolicy));
+
int sensorsEmitted = 0;
auto prepareNameLabel = [&] (std::optional<TStringBuf> suffix) {
@@ -217,6 +219,8 @@ int TCube<T>::ReadSensors(
};
auto nameLabel = prepareNameLabel({});
+ auto sumNameLabel = prepareNameLabel(".sum");
+ auto minNameLabel = prepareNameLabel(".min");
auto maxNameLabel = prepareNameLabel(".max");
auto avgNameLabel = prepareNameLabel(".avg");
auto rateNameLabel = prepareNameLabel(".rate");
@@ -224,6 +228,8 @@ int TCube<T>::ReadSensors(
auto hostLabel = consumer->PrepareLabel("host", options.Host.value_or(""));
auto ytAggrLabel = consumer->PrepareLabel("yt_aggr", "1");
+ // Set allowAggregate to true to aggregate by host.
+ // Currently Monitoring supports only sum aggregation.
auto writeLabels = [&] (const auto& tagIds, std::pair<ui32, ui32> nameLabel, bool allowAggregate) {
consumer->OnLabelsBegin();
@@ -338,34 +344,74 @@ int TCube<T>::ReadSensors(
};
auto writeSummary = [&, tagIds=tagIds] (auto makeSummary) {
- if (options.ExportSummary) {
- consumer->OnMetricBegin(NMonitoring::EMetricType::DSUMMARY);
- writeLabels(tagIds, nameLabel, true);
+ bool omitSuffix = Any(options.SummaryPolicy & ESummaryPolicy::OmitNameLabelSuffix);
- rangeValues([&] (auto value, auto time, const auto& /* indices */) {
- sensorCount += 5;
- consumer->OnSummaryDouble(time, makeSummary(value));
- });
+ auto writeMetric = [&] (
+ ESummaryPolicy policyBit,
+ std::pair<ui32, ui32> specificNameLabel,
+ bool aggregate,
+ NMonitoring::EMetricType type,
+ auto cb)
+ {
+ if (Any(options.SummaryPolicy & policyBit)) {
+ consumer->OnMetricBegin(type);
+ writeLabels(tagIds, omitSuffix ? nameLabel : specificNameLabel, aggregate);
- consumer->OnMetricEnd();
- }
+ rangeValues(cb);
- if (options.ExportSummaryAsMax) {
- consumer->OnMetricBegin(NMonitoring::EMetricType::GAUGE);
- writeLabels(tagIds, maxNameLabel, false);
-
- rangeValues([&] (auto value, auto time, const auto& /* indices */) {
- sensorCount += 1;
- consumer->OnDouble(time, makeSummary(value)->GetMax());
+ consumer->OnMetricEnd();
+ }
+ };
+
+ auto writeGaugeSummaryMetric = [&] (
+ ESummaryPolicy policyBit,
+ std::pair<ui32, ui32> specificNameLabel,
+ bool aggregate,
+ double (NMonitoring::TSummaryDoubleSnapshot::*valueGetter)() const)
+ {
+ writeMetric(
+ policyBit,
+ specificNameLabel,
+ aggregate,
+ NMonitoring::EMetricType::GAUGE,
+ [&] (auto value, auto time, const auto& /*indices*/) {
+ sensorCount += 1;
+ consumer->OnDouble(time, (makeSummary(value).Get()->*valueGetter)());
+ });
+ };
+
+ writeMetric(
+ ESummaryPolicy::All,
+ nameLabel,
+ /*aggregate*/ true,
+ NMonitoring::EMetricType::DSUMMARY,
+ [&] (auto value, auto time, const auto& /*indices*/) {
+ sensorCount += 5;
+ consumer->OnSummaryDouble(time, makeSummary(value));
});
- consumer->OnMetricEnd();
- }
-
- if (options.ExportSummaryAsAvg) {
+ writeGaugeSummaryMetric(
+ ESummaryPolicy::Sum,
+ sumNameLabel,
+ /*aggregate*/ true,
+ &NMonitoring::TSummaryDoubleSnapshot::GetSum);
+
+ writeGaugeSummaryMetric(
+ ESummaryPolicy::Min,
+ minNameLabel,
+ /*aggregate*/ false,
+ &NMonitoring::TSummaryDoubleSnapshot::GetMin);
+
+ writeGaugeSummaryMetric(
+ ESummaryPolicy::Max,
+ maxNameLabel,
+ /*aggregate*/ false,
+ &NMonitoring::TSummaryDoubleSnapshot::GetMax);
+
+ if (Any(options.SummaryPolicy & ESummaryPolicy::Avg)) {
bool empty = true;
- rangeValues([&] (auto value, auto time, const auto& /* indices */) {
+ rangeValues([&] (auto value, auto time, const auto& /*indices*/) {
auto snapshot = makeSummary(value);
if (snapshot->GetCount() == 0) {
return;
@@ -374,7 +420,7 @@ int TCube<T>::ReadSensors(
if (empty) {
empty = false;
consumer->OnMetricBegin(NMonitoring::EMetricType::GAUGE);
- writeLabels(tagIds, avgNameLabel, false);
+ writeLabels(tagIds, omitSuffix ? nameLabel : avgNameLabel, false);
}
sensorCount += 1;
@@ -575,15 +621,7 @@ int TCube<T>::ReadSensorValues(
fluent.Value(value);
++valuesRead;
} else if constexpr (std::is_same_v<T, TSummarySnapshot<double>>) {
- if (options.ExportSummaryAsMax && options.SummaryAsMaxForAllTime) {
- fluent
- .BeginMap()
- .Item("max").Value(value.Max())
- .Item("all_time_max").Value(Rollup(projection, index).Max())
- .EndMap();
- } else if (options.ExportSummaryAsMax) {
- fluent.Value(value.Max());
- } else {
+ if (Any(options.SummaryPolicy & ESummaryPolicy::All)) {
fluent
.BeginMap()
.Item("sum").Value(value.Sum())
@@ -592,16 +630,32 @@ int TCube<T>::ReadSensorValues(
.Item("last").Value(value.Last())
.Item("count").Value(static_cast<ui64>(value.Count()))
.EndMap();
+ } else if (Any(options.SummaryPolicy & ESummaryPolicy::Sum)) {
+ fluent.Value(value.Sum());
+ } else if (Any(options.SummaryPolicy & ESummaryPolicy::Min)) {
+ fluent.Value(value.Min());
+ } else if (Any(options.SummaryPolicy & ESummaryPolicy::Max)) {
+ if (options.SummaryAsMaxForAllTime) {
+ fluent
+ .BeginMap()
+ .Item("max").Value(value.Max())
+ .Item("all_time_max").Value(Rollup(projection, index).Max())
+ .EndMap();
+ } else {
+ fluent.Value(value.Max());
+ }
+ } else if (Any(options.SummaryPolicy & ESummaryPolicy::Avg)) {
+ fluent.Value(value.Count() > 0 ? value.Sum() / value.Count() : NAN);
}
++valuesRead;
} else if constexpr (std::is_same_v<T, TSummarySnapshot<TDuration>>) {
- if (options.ExportSummaryAsMax && options.SummaryAsMaxForAllTime) {
+ if (Any(options.SummaryPolicy & ESummaryPolicy::Max) && options.SummaryAsMaxForAllTime) {
fluent
.BeginMap()
.Item("max").Value(value.Max().SecondsFloat())
.Item("all_time_max").Value(Rollup(projection, index).Max().SecondsFloat())
.EndMap();
- } else if (options.ExportSummaryAsMax) {
+ } else if (Any(options.SummaryPolicy & ESummaryPolicy::Max)) {
fluent.Value(value.Max().SecondsFloat());
} else {
fluent
diff --git a/yt/yt/library/profiling/solomon/cube.h b/yt/yt/library/profiling/solomon/cube.h
index 57aaed994b..7e51f6f961 100644
--- a/yt/yt/library/profiling/solomon/cube.h
+++ b/yt/yt/library/profiling/solomon/cube.h
@@ -31,9 +31,7 @@ struct TReadOptions
bool EnableSolomonAggregationWorkaround = false;
// Direct summary export is not supported by solomon, yet.
- bool ExportSummary = false;
- bool ExportSummaryAsMax = false;
- bool ExportSummaryAsAvg = false;
+ ESummaryPolicy SummaryPolicy = ESummaryPolicy::Default;
bool MarkAggregates = false;
diff --git a/yt/yt/library/profiling/solomon/exporter.cpp b/yt/yt/library/profiling/solomon/exporter.cpp
index d03f553620..4e6ec19b2d 100644
--- a/yt/yt/library/profiling/solomon/exporter.cpp
+++ b/yt/yt/library/profiling/solomon/exporter.cpp
@@ -166,6 +166,22 @@ TShardConfigPtr TSolomonExporterConfig::MatchShard(const TString& sensorName)
return matchedShard;
}
+ESummaryPolicy TSolomonExporterConfig::GetSummaryPolicy() const
+{
+ auto policy = ESummaryPolicy::Default;
+ if (ExportSummary) {
+ policy |= ESummaryPolicy::All;
+ }
+ if (ExportSummaryAsMax) {
+ policy |= ESummaryPolicy::Max;
+ }
+ if (ExportSummaryAsAvg) {
+ policy |= ESummaryPolicy::Avg;
+ }
+
+ return policy;
+}
+
////////////////////////////////////////////////////////////////////////////////
TSolomonExporter::TSolomonExporter(
@@ -540,9 +556,10 @@ bool TSolomonExporter::ReadSensors(
readOptions.Times.emplace_back(std::vector<int>{Registry_->IndexOf(Window_.back().first)}, TInstant::Zero());
readOptions.ConvertCountersToRateGauge = false;
readOptions.EnableHistogramCompat = true;
- readOptions.ExportSummary |= Config_->ExportSummary;
- readOptions.ExportSummaryAsMax |= Config_->ExportSummaryAsMax;
- readOptions.ExportSummaryAsAvg |= Config_->ExportSummaryAsAvg;
+
+ readOptions.SummaryPolicy |= Config_->GetSummaryPolicy();
+ ValidateSummaryPolicy(readOptions.SummaryPolicy);
+
readOptions.MarkAggregates |= Config_->MarkAggregates;
if (!readOptions.Host && Config_->Host) {
readOptions.Host = Config_->Host;
@@ -786,9 +803,7 @@ void TSolomonExporter::DoHandleShard(
options.EnableSolomonAggregationWorkaround = isSolomon;
options.Times = readWindow;
- options.ExportSummary = Config_->ExportSummary;
- options.ExportSummaryAsMax = Config_->ExportSummaryAsMax;
- options.ExportSummaryAsAvg = Config_->ExportSummaryAsAvg;
+ options.SummaryPolicy = Config_->GetSummaryPolicy();
options.MarkAggregates = Config_->MarkAggregates;
options.StripSensorsNamePrefix = Config_->StripSensorsNamePrefix;
options.LingerWindowSize = Config_->LingerTimeout / gridStep;
@@ -973,6 +988,23 @@ void TSolomonExporter::CleanResponseCache()
}
}
+void TSolomonExporter::ValidateSummaryPolicy(ESummaryPolicy policy)
+{
+ static const TError SummaryPolicyError("Invalid summary policy in read options");
+
+ auto summaryPolicyConflicts = GetSummaryPolicyConflicts(policy);
+ if (summaryPolicyConflicts.AllPolicyWithSpecifiedAggregates) {
+ THROW_ERROR SummaryPolicyError
+ << TError("%Qv policy can be used only without specified policies", ESummaryPolicy::All)
+ << TErrorAttribute("policy", policy);
+ }
+ if (summaryPolicyConflicts.OmitNameLabelSuffixWithSeveralAggregates) {
+ THROW_ERROR SummaryPolicyError
+ << TError("%Qv option can be used only with single specified policy", ESummaryPolicy::OmitNameLabelSuffix)
+ << TErrorAttribute("policy", policy);
+ }
+}
+
bool TSolomonExporter::FilterDefaultGrid(const TString& sensorName)
{
auto shard = Config_->MatchShard(sensorName);
diff --git a/yt/yt/library/profiling/solomon/exporter.h b/yt/yt/library/profiling/solomon/exporter.h
index 4cfa47e9b9..6ec0ce7dc1 100644
--- a/yt/yt/library/profiling/solomon/exporter.h
+++ b/yt/yt/library/profiling/solomon/exporter.h
@@ -86,6 +86,8 @@ struct TSolomonExporterConfig
TShardConfigPtr MatchShard(const TString& sensorName);
+ ESummaryPolicy GetSummaryPolicy() const;
+
REGISTER_YSON_STRUCT(TSolomonExporterConfig);
static void Register(TRegistrar registrar);
@@ -210,6 +212,8 @@ private:
bool FilterDefaultGrid(const TString& sensorName);
+ static void ValidateSummaryPolicy(ESummaryPolicy policy);
+
// For locking.
friend class TSensorService;
};
diff --git a/yt/yt/library/profiling/solomon/registry.cpp b/yt/yt/library/profiling/solomon/registry.cpp
index bd24c87b04..b231a64782 100644
--- a/yt/yt/library/profiling/solomon/registry.cpp
+++ b/yt/yt/library/profiling/solomon/registry.cpp
@@ -40,7 +40,7 @@ ICounterImplPtr TSolomonRegistry::RegisterCounter(
TSensorOptions options)
{
return SelectImpl<ICounterImpl, TSimpleCounter, TPerCpuCounter>(options.Hot, [&, this] (const auto& counter) {
- DoRegister([this, name, tags, options, counter] () {
+ DoRegister([this, name, tags, options = std::move(options), counter] () {
auto reader = [ptr = counter.Get()] {
return ptr->GetValue();
};
@@ -59,7 +59,7 @@ ITimeCounterImplPtr TSolomonRegistry::RegisterTimeCounter(
return SelectImpl<ITimeCounterImpl, TSimpleTimeCounter, TPerCpuTimeCounter>(
options.Hot,
[&, this] (const auto& counter) {
- DoRegister([this, name, tags, options, counter] () {
+ DoRegister([this, name, tags, options = std::move(options), counter] () {
auto set = FindSet(name, options);
set->AddTimeCounter(New<TTimeCounterState>(counter, Tags_.Encode(tags), tags));
});
@@ -76,7 +76,7 @@ IGaugeImplPtr TSolomonRegistry::RegisterGauge(
gauge->Update(std::numeric_limits<double>::quiet_NaN());
}
- DoRegister([this, name, tags, options, gauge] () {
+ DoRegister([this, name, tags, options = std::move(options), gauge] () {
auto reader = [ptr = gauge.Get()] {
return ptr->GetValue();
};
@@ -94,7 +94,7 @@ ITimeGaugeImplPtr TSolomonRegistry::RegisterTimeGauge(
{
auto gauge = New<TSimpleTimeGauge>();
- DoRegister([this, name, tags, options, gauge] () {
+ DoRegister([this, name, tags, options = std::move(options), gauge] () {
auto reader = [ptr = gauge.Get()] {
return ptr->GetValue().SecondsFloat();
};
@@ -112,7 +112,7 @@ ISummaryImplPtr TSolomonRegistry::RegisterSummary(
TSensorOptions options)
{
return SelectImpl<ISummaryImpl, TSimpleSummary<double>, TPerCpuSummary<double>>(options.Hot, [&, this] (const auto& summary) {
- DoRegister([this, name, tags, options, summary] () {
+ DoRegister([this, name, tags, options = std::move(options), summary] () {
auto set = FindSet(name, options);
set->AddSummary(New<TSummaryState>(summary, Tags_.Encode(tags), tags));
});
@@ -125,7 +125,7 @@ IGaugeImplPtr TSolomonRegistry::RegisterGaugeSummary(
TSensorOptions options)
{
auto gauge = New<TSimpleGauge>();
- DoRegister([this, name, tags, options, gauge] () {
+ DoRegister([this, name, tags, options = std::move(options), gauge] () {
auto set = FindSet(name, options);
set->AddSummary(New<TSummaryState>(gauge, Tags_.Encode(tags), tags));
});
@@ -139,7 +139,7 @@ ITimeGaugeImplPtr TSolomonRegistry::RegisterTimeGaugeSummary(
TSensorOptions options)
{
auto gauge = New<TSimpleTimeGauge>();
- DoRegister([this, name, tags, options, gauge] () {
+ DoRegister([this, name, tags, options = std::move(options), gauge] () {
auto set = FindSet(name, options);
set->AddTimerSummary(New<TTimerSummaryState>(gauge, Tags_.Encode(tags), tags));
});
@@ -155,7 +155,7 @@ ITimerImplPtr TSolomonRegistry::RegisterTimerSummary(
return SelectImpl<ITimerImpl, TSimpleSummary<TDuration>, TPerCpuSummary<TDuration>>(
options.Hot,
[&, this] (const auto& timer) {
- DoRegister([this, name, tags, options, timer] () {
+ DoRegister([this, name, tags, options = std::move(options), timer] () {
auto set = FindSet(name, options);
set->AddTimerSummary(New<TTimerSummaryState>(timer, Tags_.Encode(tags), tags));
});
@@ -168,7 +168,7 @@ ITimerImplPtr TSolomonRegistry::RegisterTimeHistogram(
TSensorOptions options)
{
auto hist = New<THistogram>(options);
- DoRegister([this, name, tags, options, hist] () {
+ DoRegister([this, name, tags, options = std::move(options), hist] () {
auto set = FindSet(name, options);
set->AddTimeHistogram(New<THistogramState>(hist, Tags_.Encode(tags), tags));
});
@@ -181,7 +181,7 @@ IHistogramImplPtr TSolomonRegistry::RegisterGaugeHistogram(
TSensorOptions options)
{
auto hist = New<THistogram>(options);
- DoRegister([this, name, tags, options, hist] () {
+ DoRegister([this, name, tags, options = std::move(options), hist] () {
auto set = FindSet(name, options);
set->AddGaugeHistogram(New<THistogramState>(hist, Tags_.Encode(tags), tags));
});
@@ -194,7 +194,7 @@ IHistogramImplPtr TSolomonRegistry::RegisterRateHistogram(
TSensorOptions options)
{
auto hist = New<THistogram>(options);
- DoRegister([this, name, tags, options, hist] () {
+ DoRegister([this, name, tags, options = std::move(options), hist] () {
auto set = FindSet(name, options);
set->AddRateHistogram(New<THistogramState>(hist, Tags_.Encode(tags), tags));
});
@@ -208,7 +208,7 @@ void TSolomonRegistry::RegisterFuncCounter(
const TRefCountedPtr& owner,
std::function<i64()> reader)
{
- DoRegister([this, name, tags, options, owner, reader] () {
+ DoRegister([this, name, tags, options = std::move(options), owner, reader] () {
auto set = FindSet(name, options);
set->AddCounter(New<TCounterState>(owner, reader, Tags_.Encode(tags), tags));
});
@@ -221,7 +221,7 @@ void TSolomonRegistry::RegisterFuncGauge(
const TRefCountedPtr& owner,
std::function<double()> reader)
{
- DoRegister([this, name, tags, options, owner, reader] () {
+ DoRegister([this, name, tags, options = std::move(options), owner, reader] () {
auto set = FindSet(name, options);
set->AddGauge(New<TGaugeState>(owner, reader, Tags_.Encode(tags), tags));
});
@@ -233,7 +233,7 @@ void TSolomonRegistry::RegisterProducer(
TSensorOptions options,
const ISensorProducerPtr& producer)
{
- DoRegister([this, prefix, tags, options, producer] () {
+ DoRegister([this, prefix, tags, options = std::move(options), producer] () {
Producers_.AddProducer(New<TProducerState>(prefix, tags, options, producer));
});
}
diff --git a/yt/yt/library/profiling/solomon/remote.cpp b/yt/yt/library/profiling/solomon/remote.cpp
index 5e177de31e..b335782a12 100644
--- a/yt/yt/library/profiling/solomon/remote.cpp
+++ b/yt/yt/library/profiling/solomon/remote.cpp
@@ -105,6 +105,7 @@ void TRemoteRegistry::Transfer(const NProto::TSensorDump& dump)
options.Global = cube.global();
options.DisableSensorsRename = cube.disable_sensors_rename();
options.DisableDefault = cube.disable_default();
+ options.SummaryPolicy = NYT::FromProto<ESummaryPolicy>(cube.summary_policy());
auto sensorName = cube.name();
auto sensorSet = Registry_->FindSet(cube.name(), options);
diff --git a/yt/yt/library/profiling/solomon/sensor_dump.proto b/yt/yt/library/profiling/solomon/sensor_dump.proto
index 40d775c6bb..7180e0af39 100644
--- a/yt/yt/library/profiling/solomon/sensor_dump.proto
+++ b/yt/yt/library/profiling/solomon/sensor_dump.proto
@@ -18,6 +18,8 @@ message TCube {
optional bool global = 4 [default=false];
optional bool disable_sensors_rename = 5 [default=false];
optional bool disable_default = 6 [default=false];
+
+ optional uint64 summary_policy = 7; // ESummaryPolicy
};
message TProjection {
diff --git a/yt/yt/library/profiling/solomon/sensor_service.cpp b/yt/yt/library/profiling/solomon/sensor_service.cpp
index 04de53b703..94520feb80 100644
--- a/yt/yt/library/profiling/solomon/sensor_service.cpp
+++ b/yt/yt/library/profiling/solomon/sensor_service.cpp
@@ -100,7 +100,7 @@ private:
TTagList tags(options.TagMap.begin(), options.TagMap.end());
TReadOptions readOptions{
- .ExportSummaryAsMax = options.ExportSummaryAsMax,
+ .SummaryPolicy = options.ExportSummaryAsMax ? ESummaryPolicy::Max : ESummaryPolicy::Default,
.ReadAllProjections = options.ReadAllProjections,
.SummaryAsMaxForAllTime = options.SummaryAsMaxForAllTime,
};
diff --git a/yt/yt/library/profiling/solomon/sensor_set.cpp b/yt/yt/library/profiling/solomon/sensor_set.cpp
index 1e9fa0ba5e..489923fbdb 100644
--- a/yt/yt/library/profiling/solomon/sensor_set.cpp
+++ b/yt/yt/library/profiling/solomon/sensor_set.cpp
@@ -1,6 +1,8 @@
#include "sensor_set.h"
#include "private.h"
+#include <yt/yt/core/misc/protobuf_helpers.h>
+
#include <library/cpp/yt/assert/assert.h>
#include <library/cpp/monlib/metrics/summary_snapshot.h>
@@ -15,8 +17,12 @@ const static auto& Logger = SolomonLogger;
////////////////////////////////////////////////////////////////////////////////
-TSensorSet::TSensorSet(TSensorOptions options, i64 iteration, int windowSize, int gridFactor)
- : Options_(options)
+TSensorSet::TSensorSet(
+ TSensorOptions options,
+ i64 iteration,
+ int windowSize,
+ int gridFactor)
+ : Options_(std::move(options))
, GridFactor_(gridFactor)
, CountersCube_{windowSize, iteration}
, TimeCountersCube_{windowSize, iteration}
@@ -45,7 +51,7 @@ void TSensorSet::Profile(const TProfiler &profiler)
SensorsEmitted_ = profiler.Gauge("/sensors_emitted");
}
-void TSensorSet::ValidateOptions(TSensorOptions options)
+void TSensorSet::ValidateOptions(const TSensorOptions& options)
{
if (!Options_.IsCompatibleWith(options)) {
OnError(TError("Conflicting sensor settings")
@@ -278,7 +284,7 @@ int TSensorSet::Collect()
void TSensorSet::ReadSensors(
const TString& name,
- const TReadOptions& options,
+ TReadOptions readOptions,
TTagWriter* tagWriter,
::NMonitoring::IMetricConsumer* consumer) const
{
@@ -286,11 +292,13 @@ void TSensorSet::ReadSensors(
return;
}
- auto readOptions = options;
readOptions.Sparse = Options_.Sparse;
readOptions.Global = Options_.Global;
readOptions.DisableSensorsRename = Options_.DisableSensorsRename;
readOptions.DisableDefault = Options_.DisableDefault;
+ if (Options_.SummaryPolicy != ESummaryPolicy::Default) {
+ readOptions.SummaryPolicy = Options_.SummaryPolicy;
+ }
int sensorsEmitted = 0;
@@ -309,7 +317,7 @@ void TSensorSet::ReadSensors(
int TSensorSet::ReadSensorValues(
const TTagIdList& tagIds,
int index,
- const TReadOptions& options,
+ TReadOptions readOptions,
const TTagRegistry& tagRegistry,
TFluentAny fluent) const
{
@@ -318,15 +326,19 @@ int TSensorSet::ReadSensorValues(
<< Error_;
}
+ if (Options_.SummaryPolicy != ESummaryPolicy::Default) {
+ readOptions.SummaryPolicy = Options_.SummaryPolicy;
+ }
+
int valuesRead = 0;
- valuesRead += CountersCube_.ReadSensorValues(tagIds, index, options, tagRegistry, fluent);
- valuesRead += TimeCountersCube_.ReadSensorValues(tagIds, index, options, tagRegistry, fluent);
- valuesRead += GaugesCube_.ReadSensorValues(tagIds, index, options, tagRegistry, fluent);
- valuesRead += SummariesCube_.ReadSensorValues(tagIds, index, options, tagRegistry, fluent);
- valuesRead += TimersCube_.ReadSensorValues(tagIds, index, options, tagRegistry, fluent);
- valuesRead += TimeHistogramsCube_.ReadSensorValues(tagIds, index, options, tagRegistry, fluent);
- valuesRead += GaugeHistogramsCube_.ReadSensorValues(tagIds, index, options, tagRegistry, fluent);
- valuesRead += RateHistogramsCube_.ReadSensorValues(tagIds, index, options, tagRegistry, fluent);
+ valuesRead += CountersCube_.ReadSensorValues(tagIds, index, readOptions, tagRegistry, fluent);
+ valuesRead += TimeCountersCube_.ReadSensorValues(tagIds, index, readOptions, tagRegistry, fluent);
+ valuesRead += GaugesCube_.ReadSensorValues(tagIds, index, readOptions, tagRegistry, fluent);
+ valuesRead += SummariesCube_.ReadSensorValues(tagIds, index, readOptions, tagRegistry, fluent);
+ valuesRead += TimersCube_.ReadSensorValues(tagIds, index, readOptions, tagRegistry, fluent);
+ valuesRead += TimeHistogramsCube_.ReadSensorValues(tagIds, index, readOptions, tagRegistry, fluent);
+ valuesRead += GaugeHistogramsCube_.ReadSensorValues(tagIds, index, readOptions, tagRegistry, fluent);
+ valuesRead += RateHistogramsCube_.ReadSensorValues(tagIds, index, readOptions, tagRegistry, fluent);
return valuesRead;
}
@@ -400,6 +412,7 @@ void TSensorSet::DumpCube(NProto::TCube *cube) const
cube->set_global(Options_.Global);
cube->set_disable_default(Options_.DisableDefault);
cube->set_disable_sensors_rename(Options_.DisableSensorsRename);
+ cube->set_summary_policy(ToProto<ui64>(Options_.SummaryPolicy));
CountersCube_.DumpCube(cube);
TimeCountersCube_.DumpCube(cube);
diff --git a/yt/yt/library/profiling/solomon/sensor_set.h b/yt/yt/library/profiling/solomon/sensor_set.h
index 010db9aea8..fb71767bdd 100644
--- a/yt/yt/library/profiling/solomon/sensor_set.h
+++ b/yt/yt/library/profiling/solomon/sensor_set.h
@@ -190,7 +190,7 @@ public:
bool IsEmpty() const;
void Profile(const TProfiler& profiler);
- void ValidateOptions(TSensorOptions options);
+ void ValidateOptions(const TSensorOptions& options);
void AddCounter(TCounterStatePtr counter);
void AddGauge(TGaugeStatePtr gauge);
@@ -207,14 +207,14 @@ public:
void ReadSensors(
const TString& name,
- const TReadOptions& options,
+ TReadOptions readOptions,
TTagWriter* tagWriter,
::NMonitoring::IMetricConsumer* consumer) const;
int ReadSensorValues(
const TTagIdList& tagIds,
int index,
- const TReadOptions& options,
+ TReadOptions readOptions,
const TTagRegistry& tagRegistry,
NYTree::TFluentAny fluent) const;
diff --git a/yt/yt/library/profiling/unittests/exporter_ut.cpp b/yt/yt/library/profiling/unittests/exporter_ut.cpp
index 11c50734eb..2b62a504f4 100644
--- a/yt/yt/library/profiling/unittests/exporter_ut.cpp
+++ b/yt/yt/library/profiling/unittests/exporter_ut.cpp
@@ -218,5 +218,41 @@ TEST(TSolomonExporter, ReadSensorsStripSensorsOption)
////////////////////////////////////////////////////////////////////////////////
+class TSummaryPolicyTest
+ : public ::testing::TestWithParam<ESummaryPolicy>
+{ };
+
+INSTANTIATE_TEST_SUITE_P(
+ TSolomonExporter,
+ TSummaryPolicyTest,
+ testing::Values(
+ ESummaryPolicy::All | ESummaryPolicy::Sum,
+ ESummaryPolicy::All | ESummaryPolicy::Max,
+ ESummaryPolicy::All | ESummaryPolicy::Min,
+ ESummaryPolicy::All | ESummaryPolicy::Avg,
+ ESummaryPolicy::All | ESummaryPolicy::OmitNameLabelSuffix,
+ ESummaryPolicy::Max | ESummaryPolicy::Avg | ESummaryPolicy::OmitNameLabelSuffix));
+
+TEST_P(TSummaryPolicyTest, InvalidReadOptions)
+{
+ auto registry = New<TSolomonRegistry>();
+ TProfiler profiler(registry, "yt");
+
+ auto config = New<TSolomonExporterConfig>();
+ config->ExportSummaryAsMax = false;
+ config->EnableSelfProfiling = false;
+ config->Shards.emplace("yt", New<TShardConfig>());
+
+ auto exporter = New<TSolomonExporter>(config, registry);
+ exporter->Start();
+
+ Sleep(TDuration::Seconds(5));
+
+ ASSERT_THROW(exporter->ReadJson({.SummaryPolicy = GetParam()}, "yt"), TErrorException);
+ exporter->Stop();
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
} // namespace
} // namespace NYT::NProfiling
diff --git a/yt/yt/library/profiling/unittests/solomon_ut.cpp b/yt/yt/library/profiling/unittests/solomon_ut.cpp
index 708a8af93b..2efaf3e3cc 100644
--- a/yt/yt/library/profiling/unittests/solomon_ut.cpp
+++ b/yt/yt/library/profiling/unittests/solomon_ut.cpp
@@ -103,6 +103,7 @@ struct TTestMetricConsumer
<< ", count: " << snapshot->GetCount()
<< ", last: " << snapshot->GetLast()
<< "}" << Endl;
+ Summaries[FormatName()] = snapshot;
}
TString Name;
@@ -110,6 +111,7 @@ struct TTestMetricConsumer
THashMap<TString, i64> Counters;
THashMap<TString, double> Gauges;
+ THashMap<TString, NMonitoring::ISummaryDoubleSnapshotPtr> Summaries;
THashMap<TString, NMonitoring::IHistogramSnapshotPtr> Histograms;
std::vector<TString> LabelsCache;
@@ -874,5 +876,76 @@ TEST(TSolomonRegistry, ProducerRemoveSupport)
////////////////////////////////////////////////////////////////////////////////
+class TGaugeSummaryTriple
+{
+public:
+ TGaugeSummaryTriple(TProfiler* profiler, const TString& name, ESummaryPolicy policy)
+ : First_(profiler->GaugeSummary(name, policy))
+ , Second_(profiler->GaugeSummary(name, policy))
+ , Third_(profiler->GaugeSummary(name, policy))
+ { }
+
+ void Update(double a, double b, double c)
+ {
+ First_.Update(a);
+ Second_.Update(b);
+ Third_.Update(c);
+ }
+
+private:
+ TGauge First_;
+ TGauge Second_;
+ TGauge Third_;
+};
+
+class TOmitNameLabelSuffixTest
+ : public ::testing::TestWithParam<bool>
+{ };
+
+INSTANTIATE_TEST_SUITE_P(
+ TSolomonRegistry,
+ TOmitNameLabelSuffixTest,
+ testing::Values(false, true));
+
+TEST_P(TOmitNameLabelSuffixTest, GaugeSummary)
+{
+ bool omitNameLabelSuffix = GetParam();
+
+ auto impl = New<TSolomonRegistry>();
+ impl->SetWindowSize(12);
+ TProfiler profiler(impl, "/d");
+
+ ESummaryPolicy additionalPolicy = omitNameLabelSuffix ? ESummaryPolicy::OmitNameLabelSuffix : ESummaryPolicy::Default;
+
+ TGaugeSummaryTriple all(&profiler, "all", ESummaryPolicy::All);
+ TGaugeSummaryTriple sum(&profiler, "sum", ESummaryPolicy::Sum | additionalPolicy);
+ TGaugeSummaryTriple min(&profiler, "min", ESummaryPolicy::Min | additionalPolicy);
+ TGaugeSummaryTriple max(&profiler, "max", ESummaryPolicy::Max | additionalPolicy);
+ TGaugeSummaryTriple avg(&profiler, "avg", ESummaryPolicy::Avg | additionalPolicy);
+
+ all.Update(40, 20, 50);
+
+ sum.Update(21, 31, 41);
+ min.Update(1337, 32, 322);
+ max.Update(22, 44, 11);
+ avg.Update(55, 44, 22);
+
+ auto result = CollectSensors(impl);
+ auto& gauges = result.Gauges;
+ auto& summaries = result.Summaries;
+
+ ASSERT_NEAR(summaries["yt.dall{}"]->GetSum(), 110, 1e-6);
+ ASSERT_NEAR(summaries["yt.dall{}"]->GetMin(), 20, 1e-6);
+ ASSERT_NEAR(summaries["yt.dall{}"]->GetMax(), 50, 1e-6);
+ ASSERT_NEAR(summaries["yt.dall{}"]->GetCount(), 3, 1e-6);
+
+ ASSERT_NEAR(gauges[Format("yt.dsum%v{}", omitNameLabelSuffix ? "" : ".sum")], 93, 1e-6);
+ ASSERT_NEAR(gauges[Format("yt.dmin%v{}", omitNameLabelSuffix ? "" : ".min")], 32, 1e-6);
+ ASSERT_NEAR(gauges[Format("yt.dmax%v{}", omitNameLabelSuffix ? "" : ".max")], 44, 1e-6);
+ ASSERT_NEAR(gauges[Format("yt.davg%v{}", omitNameLabelSuffix ? "" : ".avg")], 40 + 1 / 3.0, 1e-6);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
} // namespace
} // namespace NYT::NProfiling