aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorazevaykin <azevaykin@yandex-team.com>2023-06-08 17:49:01 +0300
committerazevaykin <azevaykin@yandex-team.com>2023-06-08 17:49:01 +0300
commitef7da4cd6164363ba6bf9c4ab31d2736fc25d71e (patch)
treecf51520203e280133bf5d4f843f12018d520fe32
parent6a06e462fa9b0d494944d1b5ea5630160f6a7391 (diff)
downloadydb-ef7da4cd6164363ba6bf9c4ab31d2736fc25d71e.tar.gz
Workload topic percentile
-rw-r--r--ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_run_full.cpp12
-rw-r--r--ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_run_full.h2
-rw-r--r--ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_run_read.cpp12
-rw-r--r--ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_run_read.h3
-rw-r--r--ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_run_write.cpp12
-rw-r--r--ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_run_write.h2
-rw-r--r--ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_stats_collector.cpp35
-rw-r--r--ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_stats_collector.h3
8 files changed, 66 insertions, 15 deletions
diff --git a/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_run_full.cpp b/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_run_full.cpp
index 6c8507ab8a..7bc53a8960 100644
--- a/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_run_full.cpp
+++ b/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_run_full.cpp
@@ -69,14 +69,22 @@ void TCommandWorkloadTopicRunFull::Config(TConfig& config) {
.Optional()
.DefaultValue((TStringBuilder() << NTopic::ECodec::RAW))
.StoreMappedResultT<TString>(&Codec, &TCommandWorkloadTopicParams::StrToCodec);
+ config.Opts->AddLongOption("percentile", "Percentile for output statistics.")
+ .DefaultValue(50)
+ .StoreResult(&Percentile);
config.Opts->MutuallyExclusive("message-rate", "byte-rate");
config.IsNetworkIntensive = true;
}
-void TCommandWorkloadTopicRunFull::Parse(TConfig& config) {
+void TCommandWorkloadTopicRunFull::Parse(TConfig& config)
+{
TClientCommand::Parse(config);
+
+ if (Percentile >= 100) {
+ throw TMisuseException() << "--percentile should be less than 100.";
+ }
}
int TCommandWorkloadTopicRunFull::Run(TConfig& config) {
@@ -84,7 +92,7 @@ int TCommandWorkloadTopicRunFull::Run(TConfig& config) {
Log->SetFormatter(GetPrefixLogFormatter(""));
Driver = std::make_unique<NYdb::TDriver>(CreateDriver(config, CreateLogBackend("cerr", TClientCommand::TConfig::VerbosityLevelToELogPriority(config.VerbosityLevel))));
- StatsCollector = std::make_shared<TTopicWorkloadStatsCollector>(ProducerThreadCount, ConsumerCount * ConsumerThreadCount, Quiet, PrintTimestamp, WindowDurationSec, Seconds, ErrorFlag);
+ StatsCollector = std::make_shared<TTopicWorkloadStatsCollector>(ProducerThreadCount, ConsumerCount * ConsumerThreadCount, Quiet, PrintTimestamp, WindowDurationSec, Seconds, Percentile, ErrorFlag);
StatsCollector->PrintHeader();
auto describeTopicResult = TCommandWorkloadTopicDescribe::DescribeTopic(config.Database, *Driver);
diff --git a/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_run_full.h b/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_run_full.h
index 49c2be1e81..38ccfec8b0 100644
--- a/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_run_full.h
+++ b/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_run_full.h
@@ -17,6 +17,8 @@ namespace NYdb {
private:
size_t Seconds;
+ ui8 Percentile;
+
size_t MessageRate;
size_t ByteRate;
size_t MessageSize;
diff --git a/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_run_read.cpp b/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_run_read.cpp
index ba93c8ee2e..14c7fa5a54 100644
--- a/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_run_read.cpp
+++ b/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_run_read.cpp
@@ -50,12 +50,20 @@ void TCommandWorkloadTopicRunRead::Config(TConfig& config) {
config.Opts->AddLongOption('t', "threads", "Number of consumer threads.")
.DefaultValue(1)
.StoreResult(&ConsumerThreadCount);
+ config.Opts->AddLongOption("percentile", "Percentile for output statistics.")
+ .DefaultValue(50)
+ .StoreResult(&Percentile);
config.IsNetworkIntensive = true;
}
-void TCommandWorkloadTopicRunRead::Parse(TConfig& config) {
+void TCommandWorkloadTopicRunRead::Parse(TConfig& config)
+{
TClientCommand::Parse(config);
+
+ if (Percentile >= 100) {
+ throw TMisuseException() << "--percentile should be less than 100.";
+ }
}
int TCommandWorkloadTopicRunRead::Run(TConfig& config) {
@@ -63,7 +71,7 @@ int TCommandWorkloadTopicRunRead::Run(TConfig& config) {
Log->SetFormatter(GetPrefixLogFormatter(""));
Driver = std::make_unique<NYdb::TDriver>(CreateDriver(config, CreateLogBackend("cerr", TClientCommand::TConfig::VerbosityLevelToELogPriority(config.VerbosityLevel))));
- StatsCollector = std::make_shared<TTopicWorkloadStatsCollector>(0, ConsumerCount * ConsumerThreadCount, Quiet, PrintTimestamp, WindowDurationSec, Seconds, ErrorFlag);
+ StatsCollector = std::make_shared<TTopicWorkloadStatsCollector>(0, ConsumerCount * ConsumerThreadCount, Quiet, PrintTimestamp, WindowDurationSec, Seconds, Percentile, ErrorFlag);
StatsCollector->PrintHeader();
std::vector<std::future<void>> threads;
diff --git a/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_run_read.h b/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_run_read.h
index bfe4ed1b1b..8c68c5154b 100644
--- a/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_run_read.h
+++ b/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_run_read.h
@@ -17,7 +17,8 @@ namespace NYdb {
private:
size_t Seconds;
-
+ ui8 Percentile;
+
ui32 ConsumerThreadCount;
ui32 ConsumerCount;
diff --git a/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_run_write.cpp b/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_run_write.cpp
index d883830f73..d3d142bbb9 100644
--- a/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_run_write.cpp
+++ b/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_run_write.cpp
@@ -62,21 +62,29 @@ void TCommandWorkloadTopicRunWrite::Config(TConfig& config) {
.Optional()
.DefaultValue((TStringBuilder() << NTopic::ECodec::RAW))
.StoreMappedResultT<TString>(&Codec, &TCommandWorkloadTopicParams::StrToCodec);
+ config.Opts->AddLongOption("percentile", "Percentile for output statistics.")
+ .DefaultValue(50)
+ .StoreResult(&Percentile);
config.Opts->MutuallyExclusive("message-rate", "byte-rate");
config.IsNetworkIntensive = true;
}
-void TCommandWorkloadTopicRunWrite::Parse(TConfig& config) {
+void TCommandWorkloadTopicRunWrite::Parse(TConfig& config)
+{
TClientCommand::Parse(config);
+
+ if (Percentile >= 100) {
+ throw TMisuseException() << "--percentile should be less than 100.";
+ }
}
int TCommandWorkloadTopicRunWrite::Run(TConfig& config) {
Log = std::make_shared<TLog>(CreateLogBackend("cerr", TClientCommand::TConfig::VerbosityLevelToELogPriority(config.VerbosityLevel)));
Log->SetFormatter(GetPrefixLogFormatter(""));
Driver = std::make_unique<NYdb::TDriver>(CreateDriver(config, CreateLogBackend("cerr", TClientCommand::TConfig::VerbosityLevelToELogPriority(config.VerbosityLevel))));
- StatsCollector = std::make_shared<TTopicWorkloadStatsCollector>(ProducerThreadCount, 0, Quiet, PrintTimestamp, WindowDurationSec, Seconds, ErrorFlag);
+ StatsCollector = std::make_shared<TTopicWorkloadStatsCollector>(ProducerThreadCount, 0, Quiet, PrintTimestamp, WindowDurationSec, Seconds, Percentile, ErrorFlag);
StatsCollector->PrintHeader();
auto describeTopicResult = TCommandWorkloadTopicDescribe::DescribeTopic(config.Database, *Driver);
diff --git a/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_run_write.h b/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_run_write.h
index 1889eff9c8..baa0c0586f 100644
--- a/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_run_write.h
+++ b/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_run_write.h
@@ -16,6 +16,8 @@ namespace NYdb {
virtual int Run(TConfig& config) override;
private:
size_t Seconds;
+ ui8 Percentile;
+
size_t MessageRate;
size_t ByteRate;
size_t MessageSize;
diff --git a/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_stats_collector.cpp b/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_stats_collector.cpp
index 775b8c161b..1b04d53bb1 100644
--- a/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_stats_collector.cpp
+++ b/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_stats_collector.cpp
@@ -8,6 +8,7 @@ TTopicWorkloadStatsCollector::TTopicWorkloadStatsCollector(
size_t writerCount, size_t readerCount,
bool quiet, bool printTimestamp,
ui32 windowDurationSec, ui32 totalDurationSec,
+ ui8 percentile,
std::shared_ptr<std::atomic_bool> errorFlag)
: WriterCount(writerCount)
, ReaderCount(readerCount)
@@ -15,6 +16,7 @@ TTopicWorkloadStatsCollector::TTopicWorkloadStatsCollector(
, PrintTimestamp(printTimestamp)
, WindowDurationSec(windowDurationSec)
, TotalDurationSec(totalDurationSec)
+ , Percentile(percentile)
, ErrorFlag(errorFlag)
, WindowStats(MakeHolder<TTopicWorkloadStats>())
{
@@ -35,9 +37,26 @@ void TTopicWorkloadStatsCollector::PrintHeader(bool total) const {
if (Quiet && !total)
return;
- Cout << "Window\t" << (WriterCount > 0 ? "Write speed\tWrite time\tInflight\t" : "") << (ReaderCount > 0 ? "Lag\t\tLag time\tRead speed\tFull time\t" : "") << (PrintTimestamp ? "Timestamp" : "") << Endl;
- Cout << "#\t" << (WriterCount > 0 ? "msg/s\tMB/s\tP99(ms)\t\tP99(msg)\t" : "") << (ReaderCount > 0 ? "P99(msg)\tP99(ms)\t\tmsg/s\tMB/s\tP99(ms)" : "");
- Cout << Endl;
+ TStringBuilder header;
+
+ header << "Window\t";
+ if (WriterCount > 0)
+ header << "Write speed\tWrite time\tInflight\t";
+ if(ReaderCount > 0)
+ header << "Lag\t\tLag time\tRead speed\tFull time\t";
+ if(PrintTimestamp)
+ header << "Timestamp";
+ header << "\n";
+
+ header << "#\t";
+ auto percentile = TStringBuilder() << "P" << (ui32)Percentile;
+ if(WriterCount > 0)
+ header << "msg/s\tMB/s\t" << percentile << "(ms)\t\t" << percentile << "(msg)\t";
+ if(ReaderCount > 0)
+ header << percentile << "(msg)\t" << percentile << "(ms)\t\tmsg/s\tMB/s\t" << percentile << "(ms)";
+ header << "\n";
+
+ Cout << header << Flush;
}
void TTopicWorkloadStatsCollector::PrintWindowStatsLoop() {
@@ -77,15 +96,15 @@ void TTopicWorkloadStatsCollector::PrintStats(TMaybe<ui32> windowIt) const {
if (WriterCount > 0) {
Cout << "\t" << (int)(stats.WriteMessages / seconds)
<< "\t" << (int)(stats.WriteBytes / seconds / 1024 / 1024)
- << "\t" << stats.WriteTimeHist.GetValueAtPercentile(99) << "\t"
- << "\t" << stats.InflightMessagesHist.GetValueAtPercentile(99) << "\t";
+ << "\t" << stats.WriteTimeHist.GetValueAtPercentile(Percentile) << "\t"
+ << "\t" << stats.InflightMessagesHist.GetValueAtPercentile(Percentile) << "\t";
}
if (ReaderCount > 0) {
- Cout << "\t" << stats.LagMessagesHist.GetValueAtPercentile(99) << "\t"
- << "\t" << stats.LagTimeHist.GetValueAtPercentile(99) << "\t"
+ Cout << "\t" << stats.LagMessagesHist.GetValueAtPercentile(Percentile) << "\t"
+ << "\t" << stats.LagTimeHist.GetValueAtPercentile(Percentile) << "\t"
<< "\t" << (int)(stats.ReadMessages / seconds)
<< "\t" << (int)(stats.ReadBytes / seconds / 1024 / 1024)
- << "\t" << stats.FullTimeHist.GetValueAtPercentile(99) << "\t";
+ << "\t" << stats.FullTimeHist.GetValueAtPercentile(Percentile) << "\t";
}
if (PrintTimestamp) {
Cout << "\t" << Now().ToStringUpToSeconds();
diff --git a/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_stats_collector.h b/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_stats_collector.h
index a895f8122a..28ef596d21 100644
--- a/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_stats_collector.h
+++ b/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_stats_collector.h
@@ -14,6 +14,7 @@ namespace NYdb {
size_t producerCount, size_t consumerCount,
bool quiet, bool printTimestamp,
ui32 windowDurationSec, ui32 totalDurationSec,
+ ui8 Percentile,
std::shared_ptr<std::atomic_bool> errorFlag);
void PrintWindowStatsLoop();
@@ -47,6 +48,8 @@ namespace NYdb {
double WindowDurationSec;
double TotalDurationSec;
+ ui8 Percentile;
+
std::shared_ptr<std::atomic_bool> ErrorFlag;
THolder<TTopicWorkloadStats> WindowStats;