diff options
author | azevaykin <azevaykin@yandex-team.com> | 2023-06-08 17:49:01 +0300 |
---|---|---|
committer | azevaykin <azevaykin@yandex-team.com> | 2023-06-08 17:49:01 +0300 |
commit | ef7da4cd6164363ba6bf9c4ab31d2736fc25d71e (patch) | |
tree | cf51520203e280133bf5d4f843f12018d520fe32 | |
parent | 6a06e462fa9b0d494944d1b5ea5630160f6a7391 (diff) | |
download | ydb-ef7da4cd6164363ba6bf9c4ab31d2736fc25d71e.tar.gz |
Workload topic percentile
8 files changed, 66 insertions, 15 deletions
diff --git a/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_run_full.cpp b/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_run_full.cpp index 6c8507ab8a..7bc53a8960 100644 --- a/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_run_full.cpp +++ b/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_run_full.cpp @@ -69,14 +69,22 @@ void TCommandWorkloadTopicRunFull::Config(TConfig& config) { .Optional() .DefaultValue((TStringBuilder() << NTopic::ECodec::RAW)) .StoreMappedResultT<TString>(&Codec, &TCommandWorkloadTopicParams::StrToCodec); + config.Opts->AddLongOption("percentile", "Percentile for output statistics.") + .DefaultValue(50) + .StoreResult(&Percentile); config.Opts->MutuallyExclusive("message-rate", "byte-rate"); config.IsNetworkIntensive = true; } -void TCommandWorkloadTopicRunFull::Parse(TConfig& config) { +void TCommandWorkloadTopicRunFull::Parse(TConfig& config) +{ TClientCommand::Parse(config); + + if (Percentile >= 100) { + throw TMisuseException() << "--percentile should be less than 100."; + } } int TCommandWorkloadTopicRunFull::Run(TConfig& config) { @@ -84,7 +92,7 @@ int TCommandWorkloadTopicRunFull::Run(TConfig& config) { Log->SetFormatter(GetPrefixLogFormatter("")); Driver = std::make_unique<NYdb::TDriver>(CreateDriver(config, CreateLogBackend("cerr", TClientCommand::TConfig::VerbosityLevelToELogPriority(config.VerbosityLevel)))); - StatsCollector = std::make_shared<TTopicWorkloadStatsCollector>(ProducerThreadCount, ConsumerCount * ConsumerThreadCount, Quiet, PrintTimestamp, WindowDurationSec, Seconds, ErrorFlag); + StatsCollector = std::make_shared<TTopicWorkloadStatsCollector>(ProducerThreadCount, ConsumerCount * ConsumerThreadCount, Quiet, PrintTimestamp, WindowDurationSec, Seconds, Percentile, ErrorFlag); StatsCollector->PrintHeader(); auto describeTopicResult = TCommandWorkloadTopicDescribe::DescribeTopic(config.Database, *Driver); diff --git a/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_run_full.h b/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_run_full.h index 49c2be1e81..38ccfec8b0 100644 --- a/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_run_full.h +++ b/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_run_full.h @@ -17,6 +17,8 @@ namespace NYdb { private: size_t Seconds; + ui8 Percentile; + size_t MessageRate; size_t ByteRate; size_t MessageSize; diff --git a/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_run_read.cpp b/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_run_read.cpp index ba93c8ee2e..14c7fa5a54 100644 --- a/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_run_read.cpp +++ b/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_run_read.cpp @@ -50,12 +50,20 @@ void TCommandWorkloadTopicRunRead::Config(TConfig& config) { config.Opts->AddLongOption('t', "threads", "Number of consumer threads.") .DefaultValue(1) .StoreResult(&ConsumerThreadCount); + config.Opts->AddLongOption("percentile", "Percentile for output statistics.") + .DefaultValue(50) + .StoreResult(&Percentile); config.IsNetworkIntensive = true; } -void TCommandWorkloadTopicRunRead::Parse(TConfig& config) { +void TCommandWorkloadTopicRunRead::Parse(TConfig& config) +{ TClientCommand::Parse(config); + + if (Percentile >= 100) { + throw TMisuseException() << "--percentile should be less than 100."; + } } int TCommandWorkloadTopicRunRead::Run(TConfig& config) { @@ -63,7 +71,7 @@ int TCommandWorkloadTopicRunRead::Run(TConfig& config) { Log->SetFormatter(GetPrefixLogFormatter("")); Driver = std::make_unique<NYdb::TDriver>(CreateDriver(config, CreateLogBackend("cerr", TClientCommand::TConfig::VerbosityLevelToELogPriority(config.VerbosityLevel)))); - StatsCollector = std::make_shared<TTopicWorkloadStatsCollector>(0, ConsumerCount * ConsumerThreadCount, Quiet, PrintTimestamp, WindowDurationSec, Seconds, ErrorFlag); + StatsCollector = std::make_shared<TTopicWorkloadStatsCollector>(0, ConsumerCount * ConsumerThreadCount, Quiet, PrintTimestamp, WindowDurationSec, Seconds, Percentile, ErrorFlag); StatsCollector->PrintHeader(); std::vector<std::future<void>> threads; diff --git a/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_run_read.h b/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_run_read.h index bfe4ed1b1b..8c68c5154b 100644 --- a/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_run_read.h +++ b/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_run_read.h @@ -17,7 +17,8 @@ namespace NYdb { private: size_t Seconds; - + ui8 Percentile; + ui32 ConsumerThreadCount; ui32 ConsumerCount; diff --git a/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_run_write.cpp b/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_run_write.cpp index d883830f73..d3d142bbb9 100644 --- a/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_run_write.cpp +++ b/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_run_write.cpp @@ -62,21 +62,29 @@ void TCommandWorkloadTopicRunWrite::Config(TConfig& config) { .Optional() .DefaultValue((TStringBuilder() << NTopic::ECodec::RAW)) .StoreMappedResultT<TString>(&Codec, &TCommandWorkloadTopicParams::StrToCodec); + config.Opts->AddLongOption("percentile", "Percentile for output statistics.") + .DefaultValue(50) + .StoreResult(&Percentile); config.Opts->MutuallyExclusive("message-rate", "byte-rate"); config.IsNetworkIntensive = true; } -void TCommandWorkloadTopicRunWrite::Parse(TConfig& config) { +void TCommandWorkloadTopicRunWrite::Parse(TConfig& config) +{ TClientCommand::Parse(config); + + if (Percentile >= 100) { + throw TMisuseException() << "--percentile should be less than 100."; + } } int TCommandWorkloadTopicRunWrite::Run(TConfig& config) { Log = std::make_shared<TLog>(CreateLogBackend("cerr", TClientCommand::TConfig::VerbosityLevelToELogPriority(config.VerbosityLevel))); Log->SetFormatter(GetPrefixLogFormatter("")); Driver = std::make_unique<NYdb::TDriver>(CreateDriver(config, CreateLogBackend("cerr", TClientCommand::TConfig::VerbosityLevelToELogPriority(config.VerbosityLevel)))); - StatsCollector = std::make_shared<TTopicWorkloadStatsCollector>(ProducerThreadCount, 0, Quiet, PrintTimestamp, WindowDurationSec, Seconds, ErrorFlag); + StatsCollector = std::make_shared<TTopicWorkloadStatsCollector>(ProducerThreadCount, 0, Quiet, PrintTimestamp, WindowDurationSec, Seconds, Percentile, ErrorFlag); StatsCollector->PrintHeader(); auto describeTopicResult = TCommandWorkloadTopicDescribe::DescribeTopic(config.Database, *Driver); diff --git a/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_run_write.h b/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_run_write.h index 1889eff9c8..baa0c0586f 100644 --- a/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_run_write.h +++ b/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_run_write.h @@ -16,6 +16,8 @@ namespace NYdb { virtual int Run(TConfig& config) override; private: size_t Seconds; + ui8 Percentile; + size_t MessageRate; size_t ByteRate; size_t MessageSize; diff --git a/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_stats_collector.cpp b/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_stats_collector.cpp index 775b8c161b..1b04d53bb1 100644 --- a/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_stats_collector.cpp +++ b/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_stats_collector.cpp @@ -8,6 +8,7 @@ TTopicWorkloadStatsCollector::TTopicWorkloadStatsCollector( size_t writerCount, size_t readerCount, bool quiet, bool printTimestamp, ui32 windowDurationSec, ui32 totalDurationSec, + ui8 percentile, std::shared_ptr<std::atomic_bool> errorFlag) : WriterCount(writerCount) , ReaderCount(readerCount) @@ -15,6 +16,7 @@ TTopicWorkloadStatsCollector::TTopicWorkloadStatsCollector( , PrintTimestamp(printTimestamp) , WindowDurationSec(windowDurationSec) , TotalDurationSec(totalDurationSec) + , Percentile(percentile) , ErrorFlag(errorFlag) , WindowStats(MakeHolder<TTopicWorkloadStats>()) { @@ -35,9 +37,26 @@ void TTopicWorkloadStatsCollector::PrintHeader(bool total) const { if (Quiet && !total) return; - Cout << "Window\t" << (WriterCount > 0 ? "Write speed\tWrite time\tInflight\t" : "") << (ReaderCount > 0 ? "Lag\t\tLag time\tRead speed\tFull time\t" : "") << (PrintTimestamp ? "Timestamp" : "") << Endl; - Cout << "#\t" << (WriterCount > 0 ? "msg/s\tMB/s\tP99(ms)\t\tP99(msg)\t" : "") << (ReaderCount > 0 ? "P99(msg)\tP99(ms)\t\tmsg/s\tMB/s\tP99(ms)" : ""); - Cout << Endl; + TStringBuilder header; + + header << "Window\t"; + if (WriterCount > 0) + header << "Write speed\tWrite time\tInflight\t"; + if(ReaderCount > 0) + header << "Lag\t\tLag time\tRead speed\tFull time\t"; + if(PrintTimestamp) + header << "Timestamp"; + header << "\n"; + + header << "#\t"; + auto percentile = TStringBuilder() << "P" << (ui32)Percentile; + if(WriterCount > 0) + header << "msg/s\tMB/s\t" << percentile << "(ms)\t\t" << percentile << "(msg)\t"; + if(ReaderCount > 0) + header << percentile << "(msg)\t" << percentile << "(ms)\t\tmsg/s\tMB/s\t" << percentile << "(ms)"; + header << "\n"; + + Cout << header << Flush; } void TTopicWorkloadStatsCollector::PrintWindowStatsLoop() { @@ -77,15 +96,15 @@ void TTopicWorkloadStatsCollector::PrintStats(TMaybe<ui32> windowIt) const { if (WriterCount > 0) { Cout << "\t" << (int)(stats.WriteMessages / seconds) << "\t" << (int)(stats.WriteBytes / seconds / 1024 / 1024) - << "\t" << stats.WriteTimeHist.GetValueAtPercentile(99) << "\t" - << "\t" << stats.InflightMessagesHist.GetValueAtPercentile(99) << "\t"; + << "\t" << stats.WriteTimeHist.GetValueAtPercentile(Percentile) << "\t" + << "\t" << stats.InflightMessagesHist.GetValueAtPercentile(Percentile) << "\t"; } if (ReaderCount > 0) { - Cout << "\t" << stats.LagMessagesHist.GetValueAtPercentile(99) << "\t" - << "\t" << stats.LagTimeHist.GetValueAtPercentile(99) << "\t" + Cout << "\t" << stats.LagMessagesHist.GetValueAtPercentile(Percentile) << "\t" + << "\t" << stats.LagTimeHist.GetValueAtPercentile(Percentile) << "\t" << "\t" << (int)(stats.ReadMessages / seconds) << "\t" << (int)(stats.ReadBytes / seconds / 1024 / 1024) - << "\t" << stats.FullTimeHist.GetValueAtPercentile(99) << "\t"; + << "\t" << stats.FullTimeHist.GetValueAtPercentile(Percentile) << "\t"; } if (PrintTimestamp) { Cout << "\t" << Now().ToStringUpToSeconds(); diff --git a/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_stats_collector.h b/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_stats_collector.h index a895f8122a..28ef596d21 100644 --- a/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_stats_collector.h +++ b/ydb/public/lib/ydb_cli/commands/topic_workload/topic_workload_stats_collector.h @@ -14,6 +14,7 @@ namespace NYdb { size_t producerCount, size_t consumerCount, bool quiet, bool printTimestamp, ui32 windowDurationSec, ui32 totalDurationSec, + ui8 Percentile, std::shared_ptr<std::atomic_bool> errorFlag); void PrintWindowStatsLoop(); @@ -47,6 +48,8 @@ namespace NYdb { double WindowDurationSec; double TotalDurationSec; + ui8 Percentile; + std::shared_ptr<std::atomic_bool> ErrorFlag; THolder<TTopicWorkloadStats> WindowStats; |