diff options
author | msherbakov <msherbakov@yandex-team.ru> | 2022-02-10 16:49:17 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:49:17 +0300 |
commit | a0ffafe83b7d6229709a32fa942c71d672ac989c (patch) | |
tree | 5d5cb817648f650d76cf1076100726fd9b8448e8 | |
parent | c224a621661ddd69699f9476922eb316607ef57e (diff) | |
download | ydb-a0ffafe83b7d6229709a32fa942c71d672ac989c.tar.gz |
Restoring authorship annotation for <msherbakov@yandex-team.ru>. Commit 2 of 2.
236 files changed, 6979 insertions, 6979 deletions
diff --git a/library/cpp/actors/core/actor.h b/library/cpp/actors/core/actor.h index c30ac17946..ed29bd14b9 100644 --- a/library/cpp/actors/core/actor.h +++ b/library/cpp/actors/core/actor.h @@ -234,7 +234,7 @@ namespace NActors { INTERCONNECT_SESSION_TCP = 13, INTERCONNECT_COMMON = 171, SELF_PING_ACTOR = 207, - TEST_ACTOR_RUNTIME = 283, + TEST_ACTOR_RUNTIME = 283, INTERCONNECT_HANDSHAKE = 284, INTERCONNECT_POLLER = 285, INTERCONNECT_SESSION_KILLER = 286, diff --git a/library/cpp/actors/core/log.cpp b/library/cpp/actors/core/log.cpp index 46af66104d..5f63b5af58 100644 --- a/library/cpp/actors/core/log.cpp +++ b/library/cpp/actors/core/log.cpp @@ -217,8 +217,8 @@ namespace NActors { TString formatted; vsprintf(formatted, c, params); - auto ok = OutputRecord(time, NLog::EPrio(priority), component, formatted); - Y_UNUSED(ok); + auto ok = OutputRecord(time, NLog::EPrio(priority), component, formatted); + Y_UNUSED(ok); va_end(params); } } @@ -230,9 +230,9 @@ namespace NActors { void TLoggerActor::LogIgnoredCount(TInstant now) { TString message = Sprintf("Ignored IgnoredCount# %" PRIu64 " log records due to logger overflow!", IgnoredCount); - if (!OutputRecord(now, NActors::NLog::EPrio::Error, Settings->LoggerComponent, message)) { - BecomeDefunct(); - } + if (!OutputRecord(now, NActors::NLog::EPrio::Error, Settings->LoggerComponent, message)) { + BecomeDefunct(); + } } void TLoggerActor::HandleIgnoredEvent(TLogIgnored::TPtr& ev, const NActors::TActorContext& ctx) { @@ -242,14 +242,14 @@ namespace NActors { PassedCount = 0; } - void TLoggerActor::HandleIgnoredEventDrop() { - // logger backend is unavailable, just ignore - } - - void TLoggerActor::WriteMessageStat(const NLog::TEvLog& ev) { + void TLoggerActor::HandleIgnoredEventDrop() { + // logger backend is unavailable, just ignore + } + + void TLoggerActor::WriteMessageStat(const NLog::TEvLog& ev) { Metrics->IncActorMsgs(); - const auto prio = ev.Level.ToPrio(); + const auto prio = ev.Level.ToPrio(); switch (prio) { case ::NActors::NLog::EPrio::Alert: @@ -262,11 +262,11 @@ namespace NActors { break; } - } - - void TLoggerActor::HandleLogEvent(NLog::TEvLog::TPtr& ev, const NActors::TActorContext& ctx) { - i64 delayMillisec = (ctx.Now() - ev->Get()->Stamp).MilliSeconds(); - WriteMessageStat(*ev->Get()); + } + + void TLoggerActor::HandleLogEvent(NLog::TEvLog::TPtr& ev, const NActors::TActorContext& ctx) { + i64 delayMillisec = (ctx.Now() - ev->Get()->Stamp).MilliSeconds(); + WriteMessageStat(*ev->Get()); if (Settings->AllowDrop) { // Disable throttling if it was enabled previously if (AtomicGet(IsOverflow)) @@ -291,17 +291,17 @@ namespace NActors { AtomicSet(IsOverflow, 0); } - const auto prio = ev->Get()->Level.ToPrio(); - if (!OutputRecord(ev->Get()->Stamp, prio, ev->Get()->Component, ev->Get()->Line)) { - BecomeDefunct(); - } + const auto prio = ev->Get()->Level.ToPrio(); + if (!OutputRecord(ev->Get()->Stamp, prio, ev->Get()->Component, ev->Get()->Line)) { + BecomeDefunct(); + } + } + + void TLoggerActor::BecomeDefunct() { + Become(&TThis::StateDefunct); + Schedule(WakeupInterval, new TEvents::TEvWakeup); } - void TLoggerActor::BecomeDefunct() { - Become(&TThis::StateDefunct); - Schedule(WakeupInterval, new TEvents::TEvWakeup); - } - void TLoggerActor::HandleLogComponentLevelRequest(TLogComponentLevelRequest::TPtr& ev, const NActors::TActorContext& ctx) { Metrics->IncLevelRequests(); TString explanation; @@ -367,7 +367,7 @@ namespace NActors { * 4. Log level changes (last N changes) */ void TLoggerActor::HandleMonInfo(NMon::TEvHttpInfo::TPtr& ev, const TActorContext& ctx) { - const auto& params = ev->Get()->Request.GetParams(); + const auto& params = ev->Get()->Request.GetParams(); NLog::EComponent component = NLog::InvalidComponent; NLog::EPriority priority = NLog::PRI_DEBUG; NLog::EPriority samplingPriority = NLog::PRI_DEBUG; @@ -574,8 +574,8 @@ namespace NActors { constexpr size_t TimeBufSize = 512; - bool TLoggerActor::OutputRecord(TInstant time, NLog::EPrio priority, NLog::EComponent component, - const TString& formatted) noexcept try { + bool TLoggerActor::OutputRecord(TInstant time, NLog::EPrio priority, NLog::EComponent component, + const TString& formatted) noexcept try { const auto logPrio = ::ELogPriority(ui16(priority)); char buf[TimeBufSize]; @@ -634,21 +634,21 @@ namespace NActors { TLogRecord(logPrio, logRecord.data(), logRecord.size())); } break; } - - return true; - } catch (...) { - return false; + + return true; + } catch (...) { + return false; } - void TLoggerActor::HandleLogEventDrop(const NLog::TEvLog::TPtr& ev) { - WriteMessageStat(*ev->Get()); + void TLoggerActor::HandleLogEventDrop(const NLog::TEvLog::TPtr& ev) { + WriteMessageStat(*ev->Get()); Metrics->IncDroppedMsgs(); - } - - void TLoggerActor::HandleWakeup() { - Become(&TThis::StateFunc); - } - + } + + void TLoggerActor::HandleWakeup() { + Become(&TThis::StateFunc); + } + const char* TLoggerActor::FormatLocalTimestamp(TInstant time, char* buf) { struct tm localTime; time.LocalTime(&localTime); diff --git a/library/cpp/actors/core/log.h b/library/cpp/actors/core/log.h index 0bd6e41729..c11a7cf3c1 100644 --- a/library/cpp/actors/core/log.h +++ b/library/cpp/actors/core/log.h @@ -219,16 +219,16 @@ namespace NActors { } } - STFUNC(StateDefunct) { - switch (ev->GetTypeRewrite()) { - cFunc(TLogIgnored::EventType, HandleIgnoredEventDrop); - hFunc(NLog::TEvLog, HandleLogEventDrop); - HFunc(TLogComponentLevelRequest, HandleLogComponentLevelRequest); - HFunc(NMon::TEvHttpInfo, HandleMonInfo); - cFunc(TEvents::TEvWakeup::EventType, HandleWakeup); - } - } - + STFUNC(StateDefunct) { + switch (ev->GetTypeRewrite()) { + cFunc(TLogIgnored::EventType, HandleIgnoredEventDrop); + hFunc(NLog::TEvLog, HandleLogEventDrop); + HFunc(TLogComponentLevelRequest, HandleLogComponentLevelRequest); + HFunc(NMon::TEvHttpInfo, HandleMonInfo); + cFunc(TEvents::TEvWakeup::EventType, HandleWakeup); + } + } + // Directly call logger instead of sending a message void Log(TInstant time, NLog::EPriority priority, NLog::EComponent component, const char* c, ...); @@ -240,21 +240,21 @@ namespace NActors { ui64 IgnoredCount = 0; ui64 PassedCount = 0; static TAtomic IsOverflow; - TDuration WakeupInterval{TDuration::Seconds(5)}; + TDuration WakeupInterval{TDuration::Seconds(5)}; std::unique_ptr<ILoggerMetrics> Metrics; - void BecomeDefunct(); + void BecomeDefunct(); void HandleIgnoredEvent(TLogIgnored::TPtr& ev, const NActors::TActorContext& ctx); - void HandleIgnoredEventDrop(); + void HandleIgnoredEventDrop(); void HandleLogEvent(NLog::TEvLog::TPtr& ev, const TActorContext& ctx); - void HandleLogEventDrop(const NLog::TEvLog::TPtr& ev); + void HandleLogEventDrop(const NLog::TEvLog::TPtr& ev); void HandleLogComponentLevelRequest(TLogComponentLevelRequest::TPtr& ev, const TActorContext& ctx); void HandleMonInfo(NMon::TEvHttpInfo::TPtr& ev, const TActorContext& ctx); - void HandleWakeup(); - [[nodiscard]] bool OutputRecord(TInstant time, NLog::EPrio priority, NLog::EComponent component, const TString& formatted) noexcept; + void HandleWakeup(); + [[nodiscard]] bool OutputRecord(TInstant time, NLog::EPrio priority, NLog::EComponent component, const TString& formatted) noexcept; void RenderComponentPriorities(IOutputStream& str); void LogIgnoredCount(TInstant now); - void WriteMessageStat(const NLog::TEvLog& ev); + void WriteMessageStat(const NLog::TEvLog& ev); static const char* FormatLocalTimestamp(TInstant time, char* buf); }; @@ -305,19 +305,19 @@ namespace NActors { // Logging adaptors for memory log and logging into filesystem ///////////////////////////////////////////////////////////////////// - namespace NDetail { - inline void Y_PRINTF_FORMAT(2, 3) PrintfV(TString& dst, const char* format, ...) { - va_list params; - va_start(params, format); - vsprintf(dst, format, params); - va_end(params); - } - - inline void PrintfV(TString& dst, const char* format, va_list params) { - vsprintf(dst, format, params); - } - } // namespace NDetail - + namespace NDetail { + inline void Y_PRINTF_FORMAT(2, 3) PrintfV(TString& dst, const char* format, ...) { + va_list params; + va_start(params, format); + vsprintf(dst, format, params); + va_end(params); + } + + inline void PrintfV(TString& dst, const char* format, va_list params) { + vsprintf(dst, format, params); + } + } // namespace NDetail + template <typename TCtx> inline void DeliverLogMessage(TCtx& ctx, NLog::EPriority mPriority, NLog::EComponent mComponent, TString &&str) { @@ -326,21 +326,21 @@ namespace NActors { ctx.Send(new IEventHandle(mSettings->LoggerActorId, TActorId(), new NLog::TEvLog(mPriority, mComponent, std::move(str)))); } - template <typename TCtx, typename... TArgs> + template <typename TCtx, typename... TArgs> inline void MemLogAdapter( TCtx& actorCtxOrSystem, NLog::EPriority mPriority, NLog::EComponent mComponent, - const char* format, TArgs&&... params) { + const char* format, TArgs&&... params) { TString Formatted; - if constexpr (sizeof... (params) > 0) { - NDetail::PrintfV(Formatted, format, std::forward<TArgs>(params)...); - } else { - NDetail::PrintfV(Formatted, "%s", format); - } - + if constexpr (sizeof... (params) > 0) { + NDetail::PrintfV(Formatted, format, std::forward<TArgs>(params)...); + } else { + NDetail::PrintfV(Formatted, "%s", format); + } + MemLogWrite(Formatted.data(), Formatted.size(), true); DeliverLogMessage(actorCtxOrSystem, mPriority, mComponent, std::move(Formatted)); } diff --git a/library/cpp/actors/core/log_ut.cpp b/library/cpp/actors/core/log_ut.cpp index f5396cc8b1..09b5f88ea2 100644 --- a/library/cpp/actors/core/log_ut.cpp +++ b/library/cpp/actors/core/log_ut.cpp @@ -1,27 +1,27 @@ -#include "log.h" - +#include "log.h" + #include <library/cpp/testing/unittest/registar.h> - + #include <library/cpp/actors/testlib/test_runtime.h> - -using namespace NMonitoring; -using namespace NActors; -using namespace NActors::NLog; - -namespace { - const TString& ServiceToString(int) { - static const TString FAKE{"FAKE"}; - return FAKE; - } - - TIntrusivePtr<TSettings> DefaultSettings() { + +using namespace NMonitoring; +using namespace NActors; +using namespace NActors::NLog; + +namespace { + const TString& ServiceToString(int) { + static const TString FAKE{"FAKE"}; + return FAKE; + } + + TIntrusivePtr<TSettings> DefaultSettings() { auto loggerId = TActorId{0, "Logger"}; - auto s = MakeIntrusive<TSettings>(loggerId, 0, EPriority::PRI_TRACE); - s->SetAllowDrop(false); - s->Append(0, 1, ServiceToString); - return s; - } - + auto s = MakeIntrusive<TSettings>(loggerId, 0, EPriority::PRI_TRACE); + s->SetAllowDrop(false); + s->Append(0, 1, ServiceToString); + return s; + } + TIntrusivePtr<TSettings> DroppingSettings(ui64 timeThresholdMs) { auto loggerId = TActorId{0, "Logger"}; auto s = MakeIntrusive<TSettings>( @@ -35,123 +35,123 @@ namespace { return s; } - class TMockBackend: public TLogBackend { - public: - using TWriteImpl = std::function<void(const TLogRecord&)>; - using TReopenImpl = std::function<void()>; - - static void REOPEN_NOP() { } - - TMockBackend(TWriteImpl writeImpl, TReopenImpl reopenImpl = REOPEN_NOP) - : WriteImpl_{writeImpl} - , ReopenImpl_{reopenImpl} - { - } - - void WriteData(const TLogRecord& r) override { - WriteImpl_(r); - } - - void ReopenLog() override { } - - void SetWriteImpl(TWriteImpl writeImpl) { - WriteImpl_ = writeImpl; - } - - private: - TWriteImpl WriteImpl_; - TReopenImpl ReopenImpl_; - }; - - void ThrowAlways(const TLogRecord&) { - ythrow yexception(); - }; - - struct TFixture { + class TMockBackend: public TLogBackend { + public: + using TWriteImpl = std::function<void(const TLogRecord&)>; + using TReopenImpl = std::function<void()>; + + static void REOPEN_NOP() { } + + TMockBackend(TWriteImpl writeImpl, TReopenImpl reopenImpl = REOPEN_NOP) + : WriteImpl_{writeImpl} + , ReopenImpl_{reopenImpl} + { + } + + void WriteData(const TLogRecord& r) override { + WriteImpl_(r); + } + + void ReopenLog() override { } + + void SetWriteImpl(TWriteImpl writeImpl) { + WriteImpl_ = writeImpl; + } + + private: + TWriteImpl WriteImpl_; + TReopenImpl ReopenImpl_; + }; + + void ThrowAlways(const TLogRecord&) { + ythrow yexception(); + }; + + struct TFixture { TFixture( TIntrusivePtr<TSettings> settings, TMockBackend::TWriteImpl writeImpl = ThrowAlways) { - Runtime.Initialize(); - LogBackend.reset(new TMockBackend{writeImpl}); + Runtime.Initialize(); + LogBackend.reset(new TMockBackend{writeImpl}); LoggerActor = Runtime.Register(new TLoggerActor{settings, LogBackend, Counters}); - Runtime.SetScheduledEventFilter([] (auto&&, auto&&, auto&&, auto) { - return false; - }); - } - + Runtime.SetScheduledEventFilter([] (auto&&, auto&&, auto&&, auto) { + return false; + }); + } + TFixture(TMockBackend::TWriteImpl writeImpl = ThrowAlways) : TFixture(DefaultSettings(), writeImpl) {} - void WriteLog() { - Runtime.Send(new IEventHandle{LoggerActor, {}, new TEvLog(TInstant::Zero(), TLevel{EPrio::Emerg}, 0, "foo")}); - } - + void WriteLog() { + Runtime.Send(new IEventHandle{LoggerActor, {}, new TEvLog(TInstant::Zero(), TLevel{EPrio::Emerg}, 0, "foo")}); + } + void WriteLog(TInstant ts) { Runtime.Send(new IEventHandle{LoggerActor, {}, new TEvLog(ts, TLevel{EPrio::Emerg}, 0, "foo")}); } - void Wakeup() { - Runtime.Send(new IEventHandle{LoggerActor, {}, new TEvents::TEvWakeup}); - } - - TIntrusivePtr<TDynamicCounters> Counters{MakeIntrusive<TDynamicCounters>()}; - std::shared_ptr<TMockBackend> LogBackend; + void Wakeup() { + Runtime.Send(new IEventHandle{LoggerActor, {}, new TEvents::TEvWakeup}); + } + + TIntrusivePtr<TDynamicCounters> Counters{MakeIntrusive<TDynamicCounters>()}; + std::shared_ptr<TMockBackend> LogBackend; TActorId LoggerActor; - TTestActorRuntimeBase Runtime; - }; -} - - -Y_UNIT_TEST_SUITE(TLoggerActorTest) { - Y_UNIT_TEST(NoCrashOnWriteFailure) { - TFixture test; - test.WriteLog(); - // everything is okay as long as we get here - } - - Y_UNIT_TEST(SubsequentWritesAreIgnored) { - size_t count{0}; - auto countWrites = [&count] (auto&& r) { - count++; - ThrowAlways(r); - }; - - TFixture test{countWrites}; - test.WriteLog(); - UNIT_ASSERT_VALUES_EQUAL(count, 1); - - // at this point we should have started dropping messages - for (auto i = 0; i < 5; ++i) { - test.WriteLog(); - } - - UNIT_ASSERT_VALUES_EQUAL(count, 1); - } - - Y_UNIT_TEST(LoggerCanRecover) { - TFixture test; - test.WriteLog(); - - TVector<TString> messages; - auto acceptWrites = [&] (const TLogRecord& r) { - messages.emplace_back(r.Data, r.Len); - }; - - auto scheduled = test.Runtime.CaptureScheduledEvents(); - UNIT_ASSERT_VALUES_EQUAL(scheduled.size(), 1); - - test.LogBackend->SetWriteImpl(acceptWrites); - test.Wakeup(); - - const auto COUNT = 10; - for (auto i = 0; i < COUNT; ++i) { - test.WriteLog(); - } - - UNIT_ASSERT_VALUES_EQUAL(messages.size(), COUNT); - } + TTestActorRuntimeBase Runtime; + }; +} + + +Y_UNIT_TEST_SUITE(TLoggerActorTest) { + Y_UNIT_TEST(NoCrashOnWriteFailure) { + TFixture test; + test.WriteLog(); + // everything is okay as long as we get here + } + + Y_UNIT_TEST(SubsequentWritesAreIgnored) { + size_t count{0}; + auto countWrites = [&count] (auto&& r) { + count++; + ThrowAlways(r); + }; + + TFixture test{countWrites}; + test.WriteLog(); + UNIT_ASSERT_VALUES_EQUAL(count, 1); + + // at this point we should have started dropping messages + for (auto i = 0; i < 5; ++i) { + test.WriteLog(); + } + + UNIT_ASSERT_VALUES_EQUAL(count, 1); + } + + Y_UNIT_TEST(LoggerCanRecover) { + TFixture test; + test.WriteLog(); + + TVector<TString> messages; + auto acceptWrites = [&] (const TLogRecord& r) { + messages.emplace_back(r.Data, r.Len); + }; + + auto scheduled = test.Runtime.CaptureScheduledEvents(); + UNIT_ASSERT_VALUES_EQUAL(scheduled.size(), 1); + + test.LogBackend->SetWriteImpl(acceptWrites); + test.Wakeup(); + + const auto COUNT = 10; + for (auto i = 0; i < COUNT; ++i) { + test.WriteLog(); + } + + UNIT_ASSERT_VALUES_EQUAL(messages.size(), COUNT); + } Y_UNIT_TEST(ShouldObeyTimeThresholdMsWhenOverloaded) { TFixture test{DroppingSettings(5000)}; @@ -182,4 +182,4 @@ Y_UNIT_TEST_SUITE(TLoggerActorTest) { UNIT_ASSERT_VALUES_EQUAL(messages.size(), COUNT + 1); } -} +} diff --git a/library/cpp/actors/core/mon.h b/library/cpp/actors/core/mon.h index 666303dca7..c450f2338e 100644 --- a/library/cpp/actors/core/mon.h +++ b/library/cpp/actors/core/mon.h @@ -21,20 +21,20 @@ namespace NActors { // request info from an actor in HTML format struct TEvHttpInfo: public NActors::TEventLocal<TEvHttpInfo, HttpInfo> { - TEvHttpInfo(const NMonitoring::IMonHttpRequest& request, int subReqId = 0) + TEvHttpInfo(const NMonitoring::IMonHttpRequest& request, int subReqId = 0) : Request(request) , SubRequestId(subReqId) { } - TEvHttpInfo(const NMonitoring::IMonHttpRequest& request, const TString& userToken) + TEvHttpInfo(const NMonitoring::IMonHttpRequest& request, const TString& userToken) : Request(request) , UserToken(userToken) , SubRequestId(0) { } - const NMonitoring::IMonHttpRequest& Request; + const NMonitoring::IMonHttpRequest& Request; TString UserToken; // built and serialized // SubRequestId != 0 means that we assemble reply from multiple parts and SubRequestId contains this part id int SubRequestId; diff --git a/library/cpp/actors/core/process_stats.cpp b/library/cpp/actors/core/process_stats.cpp index cfc95034c9..0e1dbd0031 100644 --- a/library/cpp/actors/core/process_stats.cpp +++ b/library/cpp/actors/core/process_stats.cpp @@ -136,23 +136,23 @@ namespace NActors { #endif -namespace { +namespace { // Periodically collects process stats and exposes them as mon counters - template <typename TDerived> - class TProcStatCollectingActor: public TActorBootstrapped<TProcStatCollectingActor<TDerived>> { + template <typename TDerived> + class TProcStatCollectingActor: public TActorBootstrapped<TProcStatCollectingActor<TDerived>> { public: static constexpr IActor::EActivityType ActorActivityType() { return IActor::ACTORLIB_STATS; } - TProcStatCollectingActor(TDuration interval) - : Interval(interval) + TProcStatCollectingActor(TDuration interval) + : Interval(interval) { } void Bootstrap(const TActorContext& ctx) { - ctx.Schedule(Interval, new TEvents::TEvWakeup()); - Self()->Become(&TDerived::StateWork); + ctx.Schedule(Interval, new TEvents::TEvWakeup()); + Self()->Become(&TDerived::StateWork); } STFUNC(StateWork) { @@ -163,85 +163,85 @@ namespace { private: void Wakeup(const TActorContext& ctx) { - Self()->UpdateCounters(ProcStat); - ctx.Schedule(Interval, new TEvents::TEvWakeup()); + Self()->UpdateCounters(ProcStat); + ctx.Schedule(Interval, new TEvents::TEvWakeup()); } - TDerived* Self() { + TDerived* Self() { ProcStat.Fill(getpid()); - return static_cast<TDerived*>(this); + return static_cast<TDerived*>(this); } - - private: - const TDuration Interval; - TProcStat ProcStat; + + private: + const TDuration Interval; + TProcStat ProcStat; }; - // Periodically collects process stats and exposes them as mon counters - class TDynamicCounterCollector: public TProcStatCollectingActor<TDynamicCounterCollector> { - using TBase = TProcStatCollectingActor<TDynamicCounterCollector>; - public: + // Periodically collects process stats and exposes them as mon counters + class TDynamicCounterCollector: public TProcStatCollectingActor<TDynamicCounterCollector> { + using TBase = TProcStatCollectingActor<TDynamicCounterCollector>; + public: TDynamicCounterCollector( ui32 intervalSeconds, NMonitoring::TDynamicCounterPtr counters) - : TBase{TDuration::Seconds(intervalSeconds)} - { - ProcStatGroup = counters->GetSubgroup("counters", "utils"); - - VmSize = ProcStatGroup->GetCounter("Process/VmSize", false); - AnonRssSize = ProcStatGroup->GetCounter("Process/AnonRssSize", false); - FileRssSize = ProcStatGroup->GetCounter("Process/FileRssSize", false); + : TBase{TDuration::Seconds(intervalSeconds)} + { + ProcStatGroup = counters->GetSubgroup("counters", "utils"); + + VmSize = ProcStatGroup->GetCounter("Process/VmSize", false); + AnonRssSize = ProcStatGroup->GetCounter("Process/AnonRssSize", false); + FileRssSize = ProcStatGroup->GetCounter("Process/FileRssSize", false); CGroupMemLimit = ProcStatGroup->GetCounter("Process/CGroupMemLimit", false); - UserTime = ProcStatGroup->GetCounter("Process/UserTime", true); - SysTime = ProcStatGroup->GetCounter("Process/SystemTime", true); - MinorPageFaults = ProcStatGroup->GetCounter("Process/MinorPageFaults", true); - MajorPageFaults = ProcStatGroup->GetCounter("Process/MajorPageFaults", true); + UserTime = ProcStatGroup->GetCounter("Process/UserTime", true); + SysTime = ProcStatGroup->GetCounter("Process/SystemTime", true); + MinorPageFaults = ProcStatGroup->GetCounter("Process/MinorPageFaults", true); + MajorPageFaults = ProcStatGroup->GetCounter("Process/MajorPageFaults", true); UptimeSeconds = ProcStatGroup->GetCounter("Process/UptimeSeconds", false); NumThreads = ProcStatGroup->GetCounter("Process/NumThreads", false); SystemUptimeSeconds = ProcStatGroup->GetCounter("System/UptimeSeconds", false); - } - - void UpdateCounters(const TProcStat& procStat) { - *VmSize = procStat.Vsize; - *AnonRssSize = procStat.AnonRss; - *FileRssSize = procStat.FileRss; + } + + void UpdateCounters(const TProcStat& procStat) { + *VmSize = procStat.Vsize; + *AnonRssSize = procStat.AnonRss; + *FileRssSize = procStat.FileRss; if (procStat.CGroupMemLim) { *CGroupMemLimit = procStat.CGroupMemLim; } - *UserTime = procStat.Utime; - *SysTime = procStat.Stime; - *MinorPageFaults = procStat.MinFlt; - *MajorPageFaults = procStat.MajFlt; + *UserTime = procStat.Utime; + *SysTime = procStat.Stime; + *MinorPageFaults = procStat.MinFlt; + *MajorPageFaults = procStat.MajFlt; *UptimeSeconds = procStat.Uptime.Seconds(); *NumThreads = procStat.NumThreads; *SystemUptimeSeconds = procStat.Uptime.Seconds(); - } - - private: + } + + private: NMonitoring::TDynamicCounterPtr ProcStatGroup; - NMonitoring::TDynamicCounters::TCounterPtr VmSize; - NMonitoring::TDynamicCounters::TCounterPtr AnonRssSize; - NMonitoring::TDynamicCounters::TCounterPtr FileRssSize; + NMonitoring::TDynamicCounters::TCounterPtr VmSize; + NMonitoring::TDynamicCounters::TCounterPtr AnonRssSize; + NMonitoring::TDynamicCounters::TCounterPtr FileRssSize; NMonitoring::TDynamicCounters::TCounterPtr CGroupMemLimit; - NMonitoring::TDynamicCounters::TCounterPtr UserTime; - NMonitoring::TDynamicCounters::TCounterPtr SysTime; - NMonitoring::TDynamicCounters::TCounterPtr MinorPageFaults; - NMonitoring::TDynamicCounters::TCounterPtr MajorPageFaults; + NMonitoring::TDynamicCounters::TCounterPtr UserTime; + NMonitoring::TDynamicCounters::TCounterPtr SysTime; + NMonitoring::TDynamicCounters::TCounterPtr MinorPageFaults; + NMonitoring::TDynamicCounters::TCounterPtr MajorPageFaults; NMonitoring::TDynamicCounters::TCounterPtr UptimeSeconds; NMonitoring::TDynamicCounters::TCounterPtr NumThreads; NMonitoring::TDynamicCounters::TCounterPtr SystemUptimeSeconds; - }; - - - class TRegistryCollector: public TProcStatCollectingActor<TRegistryCollector> { - using TBase = TProcStatCollectingActor<TRegistryCollector>; - public: + }; + + + class TRegistryCollector: public TProcStatCollectingActor<TRegistryCollector> { + using TBase = TProcStatCollectingActor<TRegistryCollector>; + public: TRegistryCollector(TDuration interval, NMonitoring::TMetricRegistry& registry) - : TBase{interval} - { - VmSize = registry.IntGauge({{"sensor", "process.VmSize"}}); - AnonRssSize = registry.IntGauge({{"sensor", "process.AnonRssSize"}}); - FileRssSize = registry.IntGauge({{"sensor", "process.FileRssSize"}}); + : TBase{interval} + { + VmSize = registry.IntGauge({{"sensor", "process.VmSize"}}); + AnonRssSize = registry.IntGauge({{"sensor", "process.AnonRssSize"}}); + FileRssSize = registry.IntGauge({{"sensor", "process.FileRssSize"}}); CGroupMemLimit = registry.IntGauge({{"sensor", "process.CGroupMemLimit"}}); UptimeSeconds = registry.IntGauge({{"sensor", "process.UptimeSeconds"}}); NumThreads = registry.IntGauge({{"sensor", "process.NumThreads"}}); @@ -251,12 +251,12 @@ namespace { SysTime = registry.Rate({{"sensor", "process.SystemTime"}}); MinorPageFaults = registry.Rate({{"sensor", "process.MinorPageFaults"}}); MajorPageFaults = registry.Rate({{"sensor", "process.MajorPageFaults"}}); - } - - void UpdateCounters(const TProcStat& procStat) { - VmSize->Set(procStat.Vsize); - AnonRssSize->Set(procStat.AnonRss); - FileRssSize->Set(procStat.FileRss); + } + + void UpdateCounters(const TProcStat& procStat) { + VmSize->Set(procStat.Vsize); + AnonRssSize->Set(procStat.AnonRss); + FileRssSize->Set(procStat.FileRss); CGroupMemLimit->Set(procStat.CGroupMemLim); UptimeSeconds->Set(procStat.Uptime.Seconds()); NumThreads->Set(procStat.NumThreads); @@ -276,12 +276,12 @@ namespace { MajorPageFaults->Reset(); MajorPageFaults->Add(procStat.MajFlt); - } - - private: - NMonitoring::TIntGauge* VmSize; - NMonitoring::TIntGauge* AnonRssSize; - NMonitoring::TIntGauge* FileRssSize; + } + + private: + NMonitoring::TIntGauge* VmSize; + NMonitoring::TIntGauge* AnonRssSize; + NMonitoring::TIntGauge* FileRssSize; NMonitoring::TIntGauge* CGroupMemLimit; NMonitoring::TRate* UserTime; NMonitoring::TRate* SysTime; @@ -290,14 +290,14 @@ namespace { NMonitoring::TIntGauge* UptimeSeconds; NMonitoring::TIntGauge* NumThreads; NMonitoring::TIntGauge* SystemUptimeSeconds; - }; -} // namespace - + }; +} // namespace + IActor* CreateProcStatCollector(ui32 intervalSec, NMonitoring::TDynamicCounterPtr counters) { return new TDynamicCounterCollector(intervalSec, counters); } IActor* CreateProcStatCollector(TDuration interval, NMonitoring::TMetricRegistry& registry) { - return new TRegistryCollector(interval, registry); - } + return new TRegistryCollector(interval, registry); + } } diff --git a/library/cpp/actors/core/process_stats.h b/library/cpp/actors/core/process_stats.h index e2f3a8276d..66346d0b5a 100644 --- a/library/cpp/actors/core/process_stats.h +++ b/library/cpp/actors/core/process_stats.h @@ -5,10 +5,10 @@ #include <library/cpp/monlib/dynamic_counters/counters.h> -namespace NMonitoring { +namespace NMonitoring { class TMetricRegistry; -} - +} + namespace NActors { struct TProcStat { ui64 Rss; diff --git a/library/cpp/actors/core/ut/ya.make b/library/cpp/actors/core/ut/ya.make index 918ed3f656..3ee28d5850 100644 --- a/library/cpp/actors/core/ut/ya.make +++ b/library/cpp/actors/core/ut/ya.make @@ -38,7 +38,7 @@ SRCS( event_pb_ut.cpp executor_pool_basic_ut.cpp executor_pool_united_ut.cpp - log_ut.cpp + log_ut.cpp memory_tracker_ut.cpp scheduler_actor_ut.cpp ) diff --git a/library/cpp/actors/http/http_cache.cpp b/library/cpp/actors/http/http_cache.cpp index 99fdb7ebb6..27c4eeb6f3 100644 --- a/library/cpp/actors/http/http_cache.cpp +++ b/library/cpp/actors/http/http_cache.cpp @@ -1,6 +1,6 @@ #include "http.h" -#include "http_proxy.h" -#include "http_cache.h" +#include "http_proxy.h" +#include "http_cache.h" #include <library/cpp/actors/core/actor_bootstrapped.h> #include <library/cpp/actors/core/executor_pool_basic.h> #include <library/cpp/actors/core/log.h> diff --git a/library/cpp/actors/http/http_static.cpp b/library/cpp/actors/http/http_static.cpp index 8e1e649dae..c075c5f693 100644 --- a/library/cpp/actors/http/http_static.cpp +++ b/library/cpp/actors/http/http_static.cpp @@ -1,5 +1,5 @@ -#include "http_proxy.h" -#include "http_static.h" +#include "http_proxy.h" +#include "http_static.h" #include <library/cpp/actors/core/executor_pool_basic.h> #include <library/cpp/actors/core/log.h> #include <library/cpp/actors/core/scheduler_basic.h> diff --git a/library/cpp/actors/http/http_ut.cpp b/library/cpp/actors/http/http_ut.cpp index 735fe66380..4c922f8d0f 100644 --- a/library/cpp/actors/http/http_ut.cpp +++ b/library/cpp/actors/http/http_ut.cpp @@ -175,11 +175,11 @@ Y_UNIT_TEST_SUITE(HttpProxy) { } Y_UNIT_TEST(BasicRunning) { - NActors::TTestActorRuntimeBase actorSystem; + NActors::TTestActorRuntimeBase actorSystem; TPortManager portManager; TIpPort port = portManager.GetTcpPort(); TAutoPtr<NActors::IEventHandle> handle; - actorSystem.Initialize(); + actorSystem.Initialize(); NMonitoring::TMetricRegistry sensors; NActors::IActor* proxy = NHttp::CreateHttpProxy(sensors); diff --git a/library/cpp/actors/http/ut/ya.make b/library/cpp/actors/http/ut/ya.make index 8d83a1ea91..8b4c04c4d3 100644 --- a/library/cpp/actors/http/ut/ya.make +++ b/library/cpp/actors/http/ut/ya.make @@ -8,11 +8,11 @@ PEERDIR( library/cpp/actors/testlib ) -IF (NOT OS_WINDOWS) +IF (NOT OS_WINDOWS) SRCS( http_ut.cpp ) -ELSE() -ENDIF() +ELSE() +ENDIF() END() diff --git a/library/cpp/actors/testlib/test_runtime.cpp b/library/cpp/actors/testlib/test_runtime.cpp index 567df9e141..6fa25b9965 100644 --- a/library/cpp/actors/testlib/test_runtime.cpp +++ b/library/cpp/actors/testlib/test_runtime.cpp @@ -1,5 +1,5 @@ #include "test_runtime.h" - + #include <library/cpp/actors/core/actor_bootstrapped.h> #include <library/cpp/actors/core/callstack.h> #include <library/cpp/actors/core/executor_pool_basic.h> @@ -35,7 +35,7 @@ namespace { namespace NActors { ui64 TScheduledEventQueueItem::NextUniqueId = 0; - void PrintEvent(TAutoPtr<IEventHandle>& ev, const TTestActorRuntimeBase* runtime) { + void PrintEvent(TAutoPtr<IEventHandle>& ev, const TTestActorRuntimeBase* runtime) { Cerr << "mailbox: " << ev->GetRecipientRewrite().Hint() << ", type: " << Sprintf("%08x", ev->GetTypeRewrite()) << ", from " << ev->Sender.LocalId(); TString name = runtime->GetActorName(ev->Sender); @@ -56,7 +56,7 @@ namespace NActors { Cerr << "\n"; } - TTestActorRuntimeBase::TNodeDataBase::TNodeDataBase() { + TTestActorRuntimeBase::TNodeDataBase::TNodeDataBase() { ActorSystemTimestamp = nullptr; ActorSystemMonotonic = nullptr; } @@ -82,13 +82,13 @@ namespace NActors { } - class TTestActorRuntimeBase::TEdgeActor : public TActor<TEdgeActor> { + class TTestActorRuntimeBase::TEdgeActor : public TActor<TEdgeActor> { public: - static constexpr EActivityType ActorActivityType() { - return TEST_ACTOR_RUNTIME; + static constexpr EActivityType ActorActivityType() { + return TEST_ACTOR_RUNTIME; } - TEdgeActor(TTestActorRuntimeBase* runtime) + TEdgeActor(TTestActorRuntimeBase* runtime) : TActor(&TEdgeActor::StateFunc) , Runtime(runtime) { @@ -123,7 +123,7 @@ namespace NActors { } private: - TTestActorRuntimeBase* Runtime; + TTestActorRuntimeBase* Runtime; }; void TEventMailBox::Send(TAutoPtr<IEventHandle> ev) { @@ -218,9 +218,9 @@ namespace NActors { return Sent.size(); } - class TTestActorRuntimeBase::TTimeProvider : public ITimeProvider { + class TTestActorRuntimeBase::TTimeProvider : public ITimeProvider { public: - TTimeProvider(TTestActorRuntimeBase& runtime) + TTimeProvider(TTestActorRuntimeBase& runtime) : Runtime(runtime) { } @@ -230,12 +230,12 @@ namespace NActors { } private: - TTestActorRuntimeBase& Runtime; + TTestActorRuntimeBase& Runtime; }; - class TTestActorRuntimeBase::TSchedulerThreadStub : public ISchedulerThread { + class TTestActorRuntimeBase::TSchedulerThreadStub : public ISchedulerThread { public: - TSchedulerThreadStub(TTestActorRuntimeBase* runtime, TTestActorRuntimeBase::TNodeDataBase* node) + TSchedulerThreadStub(TTestActorRuntimeBase* runtime, TTestActorRuntimeBase::TNodeDataBase* node) : Runtime(runtime) , Node(node) { @@ -263,13 +263,13 @@ namespace NActors { } private: - TTestActorRuntimeBase* Runtime; - TTestActorRuntimeBase::TNodeDataBase* Node; + TTestActorRuntimeBase* Runtime; + TTestActorRuntimeBase::TNodeDataBase* Node; }; - class TTestActorRuntimeBase::TExecutorPoolStub : public IExecutorPool { + class TTestActorRuntimeBase::TExecutorPoolStub : public IExecutorPool { public: - TExecutorPoolStub(TTestActorRuntimeBase* runtime, ui32 nodeIndex, TTestActorRuntimeBase::TNodeDataBase* node, ui32 poolId) + TExecutorPoolStub(TTestActorRuntimeBase* runtime, ui32 nodeIndex, TTestActorRuntimeBase::TNodeDataBase* node, ui32 poolId) : IExecutorPool(poolId) , Runtime(runtime) , NodeIndex(nodeIndex) @@ -277,7 +277,7 @@ namespace NActors { { } - TTestActorRuntimeBase* GetRuntime() { + TTestActorRuntimeBase* GetRuntime() { return Runtime; } @@ -437,26 +437,26 @@ namespace NActors { } private: - TTestActorRuntimeBase* const Runtime; + TTestActorRuntimeBase* const Runtime; const ui32 NodeIndex; - TTestActorRuntimeBase::TNodeDataBase* const Node; + TTestActorRuntimeBase::TNodeDataBase* const Node; }; - IExecutorPool* TTestActorRuntimeBase::CreateExecutorPoolStub(TTestActorRuntimeBase* runtime, ui32 nodeIndex, TTestActorRuntimeBase::TNodeDataBase* node, ui32 poolId) { - return new TExecutorPoolStub{runtime, nodeIndex, node, poolId}; - } + IExecutorPool* TTestActorRuntimeBase::CreateExecutorPoolStub(TTestActorRuntimeBase* runtime, ui32 nodeIndex, TTestActorRuntimeBase::TNodeDataBase* node, ui32 poolId) { + return new TExecutorPoolStub{runtime, nodeIndex, node, poolId}; + } + - - ui32 TTestActorRuntimeBase::NextNodeId = 1; - - TTestActorRuntimeBase::TTestActorRuntimeBase(THeSingleSystemEnv) - : TTestActorRuntimeBase(1, 1, false) + ui32 TTestActorRuntimeBase::NextNodeId = 1; + + TTestActorRuntimeBase::TTestActorRuntimeBase(THeSingleSystemEnv) + : TTestActorRuntimeBase(1, 1, false) { SingleSysEnv = true; } - TTestActorRuntimeBase::TTestActorRuntimeBase(ui32 nodeCount, ui32 dataCenterCount, bool useRealThreads) - : ScheduledCount(0) + TTestActorRuntimeBase::TTestActorRuntimeBase(ui32 nodeCount, ui32 dataCenterCount, bool useRealThreads) + : ScheduledCount(0) , ScheduledLimit(100000) , MainThreadId(TThread::CurrentThreadId()) , ClusterUUID(MakeClusterId()) @@ -468,53 +468,53 @@ namespace NActors { , DispatchCyclesCount(0) , DispatchedEventsCount(0) , NeedMonitoring(false) - , RandomProvider(CreateDeterministicRandomProvider(DefaultRandomSeed)) - , TimeProvider(new TTimeProvider(*this)) - , ShouldContinue() + , RandomProvider(CreateDeterministicRandomProvider(DefaultRandomSeed)) + , TimeProvider(new TTimeProvider(*this)) + , ShouldContinue() , CurrentTimestamp(0) , DispatchTimeout(DEFAULT_DISPATCH_TIMEOUT) , ReschedulingDelay(TDuration::MicroSeconds(0)) - , ObserverFunc(&TTestActorRuntimeBase::DefaultObserverFunc) + , ObserverFunc(&TTestActorRuntimeBase::DefaultObserverFunc) , ScheduledEventsSelectorFunc(&CollapsedTimeScheduledEventsSelector) - , EventFilterFunc(&TTestActorRuntimeBase::DefaultFilterFunc) - , ScheduledEventFilterFunc(&TTestActorRuntimeBase::NopFilterFunc) - , RegistrationObserver(&TTestActorRuntimeBase::DefaultRegistrationObserver) + , EventFilterFunc(&TTestActorRuntimeBase::DefaultFilterFunc) + , ScheduledEventFilterFunc(&TTestActorRuntimeBase::NopFilterFunc) + , RegistrationObserver(&TTestActorRuntimeBase::DefaultRegistrationObserver) , CurrentDispatchContext(nullptr) { SetDispatcherRandomSeed(TInstant::Now(), 0); - EnableActorCallstack(); - } + EnableActorCallstack(); + } - void TTestActorRuntimeBase::InitNode(TNodeDataBase* node, size_t nodeIndex) { + void TTestActorRuntimeBase::InitNode(TNodeDataBase* node, size_t nodeIndex) { const NActors::TActorId loggerActorId = NActors::TActorId(FirstNodeId + nodeIndex, "logger"); node->LogSettings = new NActors::NLog::TSettings(loggerActorId, 410 /* NKikimrServices::LOGGER */, - NActors::NLog::PRI_WARN, NActors::NLog::PRI_WARN, 0); + NActors::NLog::PRI_WARN, NActors::NLog::PRI_WARN, 0); node->LogSettings->SetAllowDrop(false); node->LogSettings->SetThrottleDelay(TDuration::Zero()); - node->DynamicCounters = new NMonitoring::TDynamicCounters; + node->DynamicCounters = new NMonitoring::TDynamicCounters; - InitNodeImpl(node, nodeIndex); + InitNodeImpl(node, nodeIndex); } - void TTestActorRuntimeBase::InitNodeImpl(TNodeDataBase* node, size_t nodeIndex) { + void TTestActorRuntimeBase::InitNodeImpl(TNodeDataBase* node, size_t nodeIndex) { node->LogSettings->Append( NActorsServices::EServiceCommon_MIN, NActorsServices::EServiceCommon_MAX, NActorsServices::EServiceCommon_Name ); - - if (!UseRealThreads) { - node->SchedulerPool.Reset(CreateExecutorPoolStub(this, nodeIndex, node, 0)); - node->MailboxTable.Reset(new TMailboxTable()); - node->ActorSystem = MakeActorSystem(nodeIndex, node); + + if (!UseRealThreads) { + node->SchedulerPool.Reset(CreateExecutorPoolStub(this, nodeIndex, node, 0)); + node->MailboxTable.Reset(new TMailboxTable()); + node->ActorSystem = MakeActorSystem(nodeIndex, node); node->ExecutorThread.Reset(new TExecutorThread(0, 0, node->ActorSystem.Get(), node->SchedulerPool.Get(), node->MailboxTable.Get(), "TestExecutor")); - } else { - node->ActorSystem = MakeActorSystem(nodeIndex, node); - } - - node->ActorSystem->Start(); - } - + } else { + node->ActorSystem = MakeActorSystem(nodeIndex, node); + } + + node->ActorSystem->Start(); + } + bool TTestActorRuntimeBase::AllowSendFrom(TNodeDataBase* node, TAutoPtr<IEventHandle>& ev) { ui64 senderLocalId = ev->Sender.LocalId(); ui64 senderMailboxHint = ev->Sender.Hint(); @@ -527,16 +527,16 @@ namespace NActors { return true; } - TTestActorRuntimeBase::TTestActorRuntimeBase(ui32 nodeCount, ui32 dataCenterCount) - : TTestActorRuntimeBase(nodeCount, dataCenterCount, false) { + TTestActorRuntimeBase::TTestActorRuntimeBase(ui32 nodeCount, ui32 dataCenterCount) + : TTestActorRuntimeBase(nodeCount, dataCenterCount, false) { } - TTestActorRuntimeBase::TTestActorRuntimeBase(ui32 nodeCount, bool useRealThreads) - : TTestActorRuntimeBase(nodeCount, nodeCount, useRealThreads) { + TTestActorRuntimeBase::TTestActorRuntimeBase(ui32 nodeCount, bool useRealThreads) + : TTestActorRuntimeBase(nodeCount, nodeCount, useRealThreads) { } - TTestActorRuntimeBase::~TTestActorRuntimeBase() { - CleanupNodes(); + TTestActorRuntimeBase::~TTestActorRuntimeBase() { + CleanupNodes(); Cerr.Flush(); Cerr.Flush(); Clog.Flush(); @@ -544,41 +544,41 @@ namespace NActors { DisableActorCallstack(); } - void TTestActorRuntimeBase::CleanupNodes() { - Nodes.clear(); - } - - bool TTestActorRuntimeBase::IsRealThreads() const { + void TTestActorRuntimeBase::CleanupNodes() { + Nodes.clear(); + } + + bool TTestActorRuntimeBase::IsRealThreads() const { return UseRealThreads; } - TTestActorRuntimeBase::EEventAction TTestActorRuntimeBase::DefaultObserverFunc(TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event) { + TTestActorRuntimeBase::EEventAction TTestActorRuntimeBase::DefaultObserverFunc(TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event) { Y_UNUSED(runtime); Y_UNUSED(event); return EEventAction::PROCESS; } - void TTestActorRuntimeBase::DroppingScheduledEventsSelector(TTestActorRuntimeBase& runtime, TScheduledEventsList& scheduledEvents, TEventsList& queue) { + void TTestActorRuntimeBase::DroppingScheduledEventsSelector(TTestActorRuntimeBase& runtime, TScheduledEventsList& scheduledEvents, TEventsList& queue) { Y_UNUSED(runtime); Y_UNUSED(queue); scheduledEvents.clear(); } - bool TTestActorRuntimeBase::DefaultFilterFunc(TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event) { + bool TTestActorRuntimeBase::DefaultFilterFunc(TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event) { Y_UNUSED(runtime); Y_UNUSED(event); return false; } - bool TTestActorRuntimeBase::NopFilterFunc(TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event, TDuration delay, TInstant& deadline) { - Y_UNUSED(runtime); + bool TTestActorRuntimeBase::NopFilterFunc(TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event, TDuration delay, TInstant& deadline) { + Y_UNUSED(runtime); Y_UNUSED(delay); - Y_UNUSED(event); + Y_UNUSED(event); Y_UNUSED(deadline); return true; } - + void TTestActorRuntimeBase::DefaultRegistrationObserver(TTestActorRuntimeBase& runtime, const TActorId& parentId, const TActorId& actorId) { if (runtime.ScheduleWhiteList.find(parentId) != runtime.ScheduleWhiteList.end()) { runtime.ScheduleWhiteList.insert(actorId); @@ -634,7 +634,7 @@ namespace NActors { } }; - void TTestActorRuntimeBase::CollapsedTimeScheduledEventsSelector(TTestActorRuntimeBase& runtime, TScheduledEventsList& scheduledEvents, TEventsList& queue) { + void TTestActorRuntimeBase::CollapsedTimeScheduledEventsSelector(TTestActorRuntimeBase& runtime, TScheduledEventsList& scheduledEvents, TEventsList& queue) { if (scheduledEvents.empty()) return; @@ -682,46 +682,46 @@ namespace NActors { runtime.UpdateCurrentTime(time); } - TTestActorRuntimeBase::TEventObserver TTestActorRuntimeBase::SetObserverFunc(TEventObserver observerFunc) { + TTestActorRuntimeBase::TEventObserver TTestActorRuntimeBase::SetObserverFunc(TEventObserver observerFunc) { TGuard<TMutex> guard(Mutex); auto result = ObserverFunc; ObserverFunc = observerFunc; return result; } - TTestActorRuntimeBase::TScheduledEventsSelector TTestActorRuntimeBase::SetScheduledEventsSelectorFunc(TScheduledEventsSelector scheduledEventsSelectorFunc) { + TTestActorRuntimeBase::TScheduledEventsSelector TTestActorRuntimeBase::SetScheduledEventsSelectorFunc(TScheduledEventsSelector scheduledEventsSelectorFunc) { TGuard<TMutex> guard(Mutex); auto result = ScheduledEventsSelectorFunc; ScheduledEventsSelectorFunc = scheduledEventsSelectorFunc; return result; } - TTestActorRuntimeBase::TEventFilter TTestActorRuntimeBase::SetEventFilter(TEventFilter filterFunc) { + TTestActorRuntimeBase::TEventFilter TTestActorRuntimeBase::SetEventFilter(TEventFilter filterFunc) { TGuard<TMutex> guard(Mutex); auto result = EventFilterFunc; EventFilterFunc = filterFunc; return result; } - TTestActorRuntimeBase::TScheduledEventFilter TTestActorRuntimeBase::SetScheduledEventFilter(TScheduledEventFilter filterFunc) { + TTestActorRuntimeBase::TScheduledEventFilter TTestActorRuntimeBase::SetScheduledEventFilter(TScheduledEventFilter filterFunc) { TGuard<TMutex> guard(Mutex); auto result = ScheduledEventFilterFunc; ScheduledEventFilterFunc = filterFunc; return result; } - TTestActorRuntimeBase::TRegistrationObserver TTestActorRuntimeBase::SetRegistrationObserverFunc(TRegistrationObserver observerFunc) { + TTestActorRuntimeBase::TRegistrationObserver TTestActorRuntimeBase::SetRegistrationObserverFunc(TRegistrationObserver observerFunc) { TGuard<TMutex> guard(Mutex); auto result = RegistrationObserver; RegistrationObserver = observerFunc; return result; } - bool TTestActorRuntimeBase::IsVerbose() { + bool TTestActorRuntimeBase::IsVerbose() { return VERBOSE; } - void TTestActorRuntimeBase::SetVerbose(bool verbose) { + void TTestActorRuntimeBase::SetVerbose(bool verbose) { VERBOSE = verbose; } @@ -730,7 +730,7 @@ namespace NActors { Y_VERIFY(nodeIndex < NodeCount); auto node = Nodes[nodeIndex + FirstNodeId]; if (!node) { - node = GetNodeFactory().CreateNode(); + node = GetNodeFactory().CreateNode(); Nodes[nodeIndex + FirstNodeId] = node; } @@ -738,51 +738,51 @@ namespace NActors { node->LocalServices.push_back(std::make_pair(actorId, cmd)); } - void TTestActorRuntimeBase::InitNodes() { - NextNodeId += NodeCount; - Y_VERIFY(NodeCount > 0); + void TTestActorRuntimeBase::InitNodes() { + NextNodeId += NodeCount; + Y_VERIFY(NodeCount > 0); - for (ui32 nodeIndex = 0; nodeIndex < NodeCount; ++nodeIndex) { - auto nodeIt = Nodes.emplace(FirstNodeId + nodeIndex, GetNodeFactory().CreateNode()).first; - TNodeDataBase* node = nodeIt->second.Get(); - InitNode(node, nodeIndex); + for (ui32 nodeIndex = 0; nodeIndex < NodeCount; ++nodeIndex) { + auto nodeIt = Nodes.emplace(FirstNodeId + nodeIndex, GetNodeFactory().CreateNode()).first; + TNodeDataBase* node = nodeIt->second.Get(); + InitNode(node, nodeIndex); } - } + } - void TTestActorRuntimeBase::Initialize() { - InitNodes(); - IsInitialized = true; + void TTestActorRuntimeBase::Initialize() { + InitNodes(); + IsInitialized = true; } void SetupCrossDC() { } - TDuration TTestActorRuntimeBase::SetDispatchTimeout(TDuration timeout) { + TDuration TTestActorRuntimeBase::SetDispatchTimeout(TDuration timeout) { TGuard<TMutex> guard(Mutex); TDuration oldTimeout = DispatchTimeout; DispatchTimeout = timeout; return oldTimeout; } - TDuration TTestActorRuntimeBase::SetReschedulingDelay(TDuration delay) { + TDuration TTestActorRuntimeBase::SetReschedulingDelay(TDuration delay) { TGuard<TMutex> guard(Mutex); TDuration oldDelay = ReschedulingDelay; ReschedulingDelay = delay; return oldDelay; } - void TTestActorRuntimeBase::SetLogBackend(const TAutoPtr<TLogBackend> logBackend) { + void TTestActorRuntimeBase::SetLogBackend(const TAutoPtr<TLogBackend> logBackend) { Y_VERIFY(!IsInitialized); TGuard<TMutex> guard(Mutex); LogBackend = logBackend; } - void TTestActorRuntimeBase::SetLogPriority(NActors::NLog::EComponent component, NActors::NLog::EPriority priority) { + void TTestActorRuntimeBase::SetLogPriority(NActors::NLog::EComponent component, NActors::NLog::EPriority priority) { TGuard<TMutex> guard(Mutex); for (ui32 nodeIndex = 0; nodeIndex < NodeCount; ++nodeIndex) { - TNodeDataBase* node = Nodes[FirstNodeId + nodeIndex].Get(); + TNodeDataBase* node = Nodes[FirstNodeId + nodeIndex].Get(); TString explanation; auto status = node->LogSettings->SetLevel(priority, component, explanation); if (status) { @@ -791,13 +791,13 @@ namespace NActors { } } - TInstant TTestActorRuntimeBase::GetCurrentTime() const { + TInstant TTestActorRuntimeBase::GetCurrentTime() const { TGuard<TMutex> guard(Mutex); Y_VERIFY(!UseRealThreads); return TInstant::MicroSeconds(CurrentTimestamp); } - void TTestActorRuntimeBase::UpdateCurrentTime(TInstant newTime) { + void TTestActorRuntimeBase::UpdateCurrentTime(TInstant newTime) { static int counter = 0; ++counter; if (VERBOSE) { @@ -814,25 +814,25 @@ namespace NActors { } } - void TTestActorRuntimeBase::AdvanceCurrentTime(TDuration duration) { + void TTestActorRuntimeBase::AdvanceCurrentTime(TDuration duration) { UpdateCurrentTime(GetCurrentTime() + duration); } - TIntrusivePtr<ITimeProvider> TTestActorRuntimeBase::GetTimeProvider() { + TIntrusivePtr<ITimeProvider> TTestActorRuntimeBase::GetTimeProvider() { Y_VERIFY(!UseRealThreads); return TimeProvider; } - ui32 TTestActorRuntimeBase::GetNodeId(ui32 index) const { + ui32 TTestActorRuntimeBase::GetNodeId(ui32 index) const { Y_VERIFY(index < NodeCount); return FirstNodeId + index; } - ui32 TTestActorRuntimeBase::GetNodeCount() const { + ui32 TTestActorRuntimeBase::GetNodeCount() const { return NodeCount; } - ui64 TTestActorRuntimeBase::AllocateLocalId() { + ui64 TTestActorRuntimeBase::AllocateLocalId() { TGuard<TMutex> guard(Mutex); ui64 nextId = ++LocalId; if (VERBOSE) { @@ -842,7 +842,7 @@ namespace NActors { return nextId; } - ui32 TTestActorRuntimeBase::InterconnectPoolId() const { + ui32 TTestActorRuntimeBase::InterconnectPoolId() const { if (UseRealThreads && NSan::TSanIsOn()) { // Interconnect coroutines may move across threads // Use a special single-threaded pool to avoid that @@ -851,7 +851,7 @@ namespace NActors { return 0; } - TString TTestActorRuntimeBase::GetTempDir() { + TString TTestActorRuntimeBase::GetTempDir() { if (!TmpDir) TmpDir.Reset(new TTempDir()); return (*TmpDir)(); @@ -861,7 +861,7 @@ namespace NActors { ui64 revolvingCounter, const TActorId& parentId) { Y_VERIFY(nodeIndex < NodeCount); TGuard<TMutex> guard(Mutex); - TNodeDataBase* node = Nodes[FirstNodeId + nodeIndex].Get(); + TNodeDataBase* node = Nodes[FirstNodeId + nodeIndex].Get(); if (UseRealThreads) { Y_VERIFY(poolId < node->ExecutorPools.size()); return node->ExecutorPools[poolId]->Register(actor, mailboxType, revolvingCounter, parentId); @@ -929,7 +929,7 @@ namespace NActors { const TActorId& parentId) { Y_VERIFY(nodeIndex < NodeCount); TGuard<TMutex> guard(Mutex); - TNodeDataBase* node = Nodes[FirstNodeId + nodeIndex].Get(); + TNodeDataBase* node = Nodes[FirstNodeId + nodeIndex].Get(); if (UseRealThreads) { Y_VERIFY(poolId < node->ExecutorPools.size()); return node->ExecutorPools[poolId]->Register(actor, mailbox, hint, parentId); @@ -952,7 +952,7 @@ namespace NActors { TActorId TTestActorRuntimeBase::RegisterService(const TActorId& serviceId, const TActorId& actorId, ui32 nodeIndex) { TGuard<TMutex> guard(Mutex); Y_VERIFY(nodeIndex < NodeCount); - TNodeDataBase* node = Nodes[FirstNodeId + nodeIndex].Get(); + TNodeDataBase* node = Nodes[FirstNodeId + nodeIndex].Get(); if (!UseRealThreads) { IActor* actor = FindActor(actorId, node); node->LocalServicesActors[serviceId] = actor; @@ -971,7 +971,7 @@ namespace NActors { return edgeActor; } - TEventsList TTestActorRuntimeBase::CaptureEvents() { + TEventsList TTestActorRuntimeBase::CaptureEvents() { TGuard<TMutex> guard(Mutex); TEventsList result; for (auto& mbox : Mailboxes) { @@ -981,7 +981,7 @@ namespace NActors { return result; } - TEventsList TTestActorRuntimeBase::CaptureMailboxEvents(ui32 hint, ui32 nodeId) { + TEventsList TTestActorRuntimeBase::CaptureMailboxEvents(ui32 hint, ui32 nodeId) { TGuard<TMutex> guard(Mutex); Y_VERIFY(nodeId >= FirstNodeId && nodeId < FirstNodeId + NodeCount); TEventsList result; @@ -989,14 +989,14 @@ namespace NActors { return result; } - void TTestActorRuntimeBase::PushFront(TAutoPtr<IEventHandle>& ev) { + void TTestActorRuntimeBase::PushFront(TAutoPtr<IEventHandle>& ev) { TGuard<TMutex> guard(Mutex); ui32 nodeId = ev->GetRecipientRewrite().NodeId(); Y_VERIFY(nodeId != 0); GetMailbox(nodeId, ev->GetRecipientRewrite().Hint()).PushFront(ev); } - void TTestActorRuntimeBase::PushEventsFront(TEventsList& events) { + void TTestActorRuntimeBase::PushEventsFront(TEventsList& events) { TGuard<TMutex> guard(Mutex); for (auto rit = events.rbegin(); rit != events.rend(); ++rit) { if (*rit) { @@ -1010,7 +1010,7 @@ namespace NActors { events.clear(); } - void TTestActorRuntimeBase::PushMailboxEventsFront(ui32 hint, ui32 nodeId, TEventsList& events) { + void TTestActorRuntimeBase::PushMailboxEventsFront(ui32 hint, ui32 nodeId, TEventsList& events) { TGuard<TMutex> guard(Mutex); Y_VERIFY(nodeId >= FirstNodeId && nodeId < FirstNodeId + NodeCount); TEventsList result; @@ -1018,7 +1018,7 @@ namespace NActors { events.clear(); } - TScheduledEventsList TTestActorRuntimeBase::CaptureScheduledEvents() { + TScheduledEventsList TTestActorRuntimeBase::CaptureScheduledEvents() { TGuard<TMutex> guard(Mutex); TScheduledEventsList result; for (auto& mbox : Mailboxes) { @@ -1028,28 +1028,28 @@ namespace NActors { return result; } - bool TTestActorRuntimeBase::DispatchEvents(const TDispatchOptions& options) { + bool TTestActorRuntimeBase::DispatchEvents(const TDispatchOptions& options) { return DispatchEvents(options, TInstant::Max()); } - bool TTestActorRuntimeBase::DispatchEvents(const TDispatchOptions& options, TDuration simTimeout) { + bool TTestActorRuntimeBase::DispatchEvents(const TDispatchOptions& options, TDuration simTimeout) { return DispatchEvents(options, TInstant::MicroSeconds(CurrentTimestamp) + simTimeout); } - bool TTestActorRuntimeBase::DispatchEvents(const TDispatchOptions& options, TInstant simDeadline) { + bool TTestActorRuntimeBase::DispatchEvents(const TDispatchOptions& options, TInstant simDeadline) { TGuard<TMutex> guard(Mutex); return DispatchEventsInternal(options, simDeadline); } // Mutex must be locked by caller! - bool TTestActorRuntimeBase::DispatchEventsInternal(const TDispatchOptions& options, TInstant simDeadline) { + bool TTestActorRuntimeBase::DispatchEventsInternal(const TDispatchOptions& options, TInstant simDeadline) { TDispatchContext localContext; localContext.Options = &options; localContext.PrevContext = nullptr; bool verbose = !options.Quiet && VERBOSE; struct TDispatchContextSetter { - TDispatchContextSetter(TTestActorRuntimeBase& runtime, TDispatchContext& lastContext) + TDispatchContextSetter(TTestActorRuntimeBase& runtime, TDispatchContext& lastContext) : Runtime(runtime) { lastContext.PrevContext = Runtime.CurrentDispatchContext; @@ -1060,7 +1060,7 @@ namespace NActors { Runtime.CurrentDispatchContext = Runtime.CurrentDispatchContext->PrevContext; } - TTestActorRuntimeBase& Runtime; + TTestActorRuntimeBase& Runtime; } DispatchContextSetter(*this, localContext); TInstant dispatchTime = TInstant::MicroSeconds(0); @@ -1072,7 +1072,7 @@ namespace NActors { } struct TTempEdgeEventsCaptor { - TTempEdgeEventsCaptor(TTestActorRuntimeBase& runtime) + TTempEdgeEventsCaptor(TTestActorRuntimeBase& runtime) : Runtime(runtime) , HasEvents(false) { @@ -1103,7 +1103,7 @@ namespace NActors { } } - TTestActorRuntimeBase& Runtime; + TTestActorRuntimeBase& Runtime; TEventMailBoxList Store; bool HasEvents; }; @@ -1354,7 +1354,7 @@ namespace NActors { return false; } - void TTestActorRuntimeBase::HandleNonEmptyMailboxesForEachContext(TEventMailboxId mboxId) { + void TTestActorRuntimeBase::HandleNonEmptyMailboxesForEachContext(TEventMailboxId mboxId) { TDispatchContext* context = CurrentDispatchContext; while (context) { const auto& nonEmptyMailboxes = context->Options->NonEmptyMailboxes; @@ -1366,7 +1366,7 @@ namespace NActors { } } - void TTestActorRuntimeBase::UpdateFinalEventsStatsForEachContext(IEventHandle& ev) { + void TTestActorRuntimeBase::UpdateFinalEventsStatsForEachContext(IEventHandle& ev) { TDispatchContext* context = CurrentDispatchContext; while (context) { for (const auto& finalEvent : context->Options->FinalEvents) { @@ -1382,14 +1382,14 @@ namespace NActors { } } - void TTestActorRuntimeBase::Send(IEventHandle* ev, ui32 senderNodeIndex, bool viaActorSystem) { + void TTestActorRuntimeBase::Send(IEventHandle* ev, ui32 senderNodeIndex, bool viaActorSystem) { TGuard<TMutex> guard(Mutex); Y_VERIFY(senderNodeIndex < NodeCount, "senderNodeIndex# %" PRIu32 " < NodeCount# %" PRIu32, senderNodeIndex, NodeCount); SendInternal(ev, senderNodeIndex, viaActorSystem); } - void TTestActorRuntimeBase::Schedule(IEventHandle* ev, const TDuration& duration, ui32 nodeIndex) { + void TTestActorRuntimeBase::Schedule(IEventHandle* ev, const TDuration& duration, ui32 nodeIndex) { TGuard<TMutex> guard(Mutex); Y_VERIFY(nodeIndex < NodeCount); ui32 nodeId = FirstNodeId + nodeIndex; @@ -1400,12 +1400,12 @@ namespace NActors { Cerr << "Event was added to scheduled queue\n"; } - void TTestActorRuntimeBase::ClearCounters() { + void TTestActorRuntimeBase::ClearCounters() { TGuard<TMutex> guard(Mutex); EvCounters.clear(); } - ui64 TTestActorRuntimeBase::GetCounter(ui32 evType) const { + ui64 TTestActorRuntimeBase::GetCounter(ui32 evType) const { TGuard<TMutex> guard(Mutex); auto it = EvCounters.find(evType); if (it == EvCounters.end()) @@ -1417,7 +1417,7 @@ namespace NActors { TActorId TTestActorRuntimeBase::GetLocalServiceId(const TActorId& serviceId, ui32 nodeIndex) { TGuard<TMutex> guard(Mutex); Y_VERIFY(nodeIndex < NodeCount); - TNodeDataBase* node = Nodes[FirstNodeId + nodeIndex].Get(); + TNodeDataBase* node = Nodes[FirstNodeId + nodeIndex].Get(); return node->ActorSystem->LookupLocalService(serviceId); } @@ -1465,7 +1465,7 @@ namespace NActors { Y_VERIFY(nodeIndexFrom < NodeCount); Y_VERIFY(nodeIndexTo < NodeCount); Y_VERIFY(nodeIndexFrom != nodeIndexTo); - TNodeDataBase* node = Nodes[FirstNodeId + nodeIndexFrom].Get(); + TNodeDataBase* node = Nodes[FirstNodeId + nodeIndexFrom].Get(); return node->ActorSystem->InterconnectProxy(FirstNodeId + nodeIndexTo); } @@ -1474,7 +1474,7 @@ namespace NActors { BlockedOutput.insert(actorId); } - void TTestActorRuntimeBase::SetDispatcherRandomSeed(TInstant time, ui64 iteration) { + void TTestActorRuntimeBase::SetDispatcherRandomSeed(TInstant time, ui64 iteration) { ui64 days = (time.Hours() / 24); DispatcherRandomSeed = (days << 32) ^ iteration; DispatcherRandomProvider = CreateDeterministicRandomProvider(DispatcherRandomSeed); @@ -1490,7 +1490,7 @@ namespace NActors { Y_VERIFY(nodeIndex < NodeCount); auto nodeIt = Nodes.find(FirstNodeId + nodeIndex); Y_VERIFY(nodeIt != Nodes.end()); - TNodeDataBase* node = nodeIt->second.Get(); + TNodeDataBase* node = nodeIt->second.Get(); return FindActor(actorId, node); } @@ -1514,11 +1514,11 @@ namespace NActors { return ScheduleWhiteList.find(actorId) != ScheduleWhiteList.end(); } - TIntrusivePtr<NMonitoring::TDynamicCounters> TTestActorRuntimeBase::GetDynamicCounters(ui32 nodeIndex) { + TIntrusivePtr<NMonitoring::TDynamicCounters> TTestActorRuntimeBase::GetDynamicCounters(ui32 nodeIndex) { TGuard<TMutex> guard(Mutex); Y_VERIFY(nodeIndex < NodeCount); ui32 nodeId = FirstNodeId + nodeIndex; - TNodeDataBase* node = Nodes[nodeId].Get(); + TNodeDataBase* node = Nodes[nodeId].Get(); return node->DynamicCounters; } @@ -1526,10 +1526,10 @@ namespace NActors { NeedMonitoring = true; } - void TTestActorRuntimeBase::SendInternal(IEventHandle* ev, ui32 nodeIndex, bool viaActorSystem) { + void TTestActorRuntimeBase::SendInternal(IEventHandle* ev, ui32 nodeIndex, bool viaActorSystem) { Y_VERIFY(nodeIndex < NodeCount); ui32 nodeId = FirstNodeId + nodeIndex; - TNodeDataBase* node = Nodes[nodeId].Get(); + TNodeDataBase* node = Nodes[nodeId].Get(); ui32 targetNode = ev->GetRecipientRewrite().NodeId(); ui32 targetNodeIndex; if (targetNode == 0) { @@ -1607,10 +1607,10 @@ namespace NActors { return actor; } - THolder<TActorSystemSetup> TTestActorRuntimeBase::MakeActorSystemSetup(ui32 nodeIndex, TNodeDataBase* node) { + THolder<TActorSystemSetup> TTestActorRuntimeBase::MakeActorSystemSetup(ui32 nodeIndex, TNodeDataBase* node) { THolder<TActorSystemSetup> setup(new TActorSystemSetup); setup->NodeId = FirstNodeId + nodeIndex; - + if (UseRealThreads) { setup->ExecutorsCount = 5; setup->Executors.Reset(new TAutoPtr<IExecutorPool>[5]); @@ -1627,20 +1627,20 @@ namespace NActors { setup->Executors[0].Reset(new TExecutorPoolStub(this, nodeIndex, node, 0)); } - InitActorSystemSetup(*setup); - - return setup; - } - - THolder<TActorSystem> TTestActorRuntimeBase::MakeActorSystem(ui32 nodeIndex, TNodeDataBase* node) { - auto setup = MakeActorSystemSetup(nodeIndex, node); - + InitActorSystemSetup(*setup); + + return setup; + } + + THolder<TActorSystem> TTestActorRuntimeBase::MakeActorSystem(ui32 nodeIndex, TNodeDataBase* node) { + auto setup = MakeActorSystemSetup(nodeIndex, node); + node->ExecutorPools.resize(setup->ExecutorsCount); for (ui32 i = 0; i < setup->ExecutorsCount; ++i) { node->ExecutorPools[i] = setup->Executors[i].Get(); } - const auto& interconnectCounters = GetCountersForComponent(node->DynamicCounters, "interconnect"); + const auto& interconnectCounters = GetCountersForComponent(node->DynamicCounters, "interconnect"); setup->LocalServices = node->LocalServices; setup->Interconnect.ProxyActors.resize(FirstNodeId + NodeCount); @@ -1686,22 +1686,22 @@ namespace NActors { if (!SingleSysEnv) { // Single system env should do this self TAutoPtr<TLogBackend> logBackend = LogBackend ? LogBackend : NActors::CreateStderrBackend(); NActors::TLoggerActor *loggerActor = new NActors::TLoggerActor(node->LogSettings, - logBackend, GetCountersForComponent(node->DynamicCounters, "utils")); - NActors::TActorSetupCmd loggerActorCmd(loggerActor, NActors::TMailboxType::Simple, node->GetLoggerPoolId()); + logBackend, GetCountersForComponent(node->DynamicCounters, "utils")); + NActors::TActorSetupCmd loggerActorCmd(loggerActor, NActors::TMailboxType::Simple, node->GetLoggerPoolId()); std::pair<NActors::TActorId, NActors::TActorSetupCmd> loggerActorPair(node->LogSettings->LoggerActorId, loggerActorCmd); setup->LocalServices.push_back(loggerActorPair); } - return THolder<TActorSystem>(new TActorSystem(setup, node->GetAppData(), node->LogSettings)); + return THolder<TActorSystem>(new TActorSystem(setup, node->GetAppData(), node->LogSettings)); } - TActorSystem* TTestActorRuntimeBase::SingleSys() const { + TActorSystem* TTestActorRuntimeBase::SingleSys() const { Y_VERIFY(Nodes.size() == 1, "Works only for single system env"); return Nodes.begin()->second->ActorSystem.Get(); } - TActorSystem* TTestActorRuntimeBase::GetAnyNodeActorSystem() { + TActorSystem* TTestActorRuntimeBase::GetAnyNodeActorSystem() { for (auto& x : Nodes) { return x.second->ActorSystem.Get(); } @@ -1715,7 +1715,7 @@ namespace NActors { } - TEventMailBox& TTestActorRuntimeBase::GetMailbox(ui32 nodeId, ui32 hint) { + TEventMailBox& TTestActorRuntimeBase::GetMailbox(ui32 nodeId, ui32 hint) { TGuard<TMutex> guard(Mutex); auto mboxId = TEventMailboxId(nodeId, hint); auto it = Mailboxes.find(mboxId); @@ -1726,7 +1726,7 @@ namespace NActors { return *it->second; } - void TTestActorRuntimeBase::ClearMailbox(ui32 nodeId, ui32 hint) { + void TTestActorRuntimeBase::ClearMailbox(ui32 nodeId, ui32 hint) { TGuard<TMutex> guard(Mutex); auto mboxId = TEventMailboxId(nodeId, hint); Mailboxes.erase(mboxId); @@ -1753,8 +1753,8 @@ namespace NActors { public: class TReplyActor : public TActor<TReplyActor> { public: - static constexpr EActivityType ActorActivityType() { - return TEST_ACTOR_RUNTIME; + static constexpr EActivityType ActorActivityType() { + return TEST_ACTOR_RUNTIME; } TReplyActor(TStrandingActorDecorator* owner) @@ -1769,8 +1769,8 @@ namespace NActors { TStrandingActorDecorator* const Owner; }; - static constexpr EActivityType ActorActivityType() { - return TEST_ACTOR_RUNTIME; + static constexpr EActivityType ActorActivityType() { + return TEST_ACTOR_RUNTIME; } TStrandingActorDecorator(const TActorId& delegatee, bool isSync, const TVector<TActorId>& additionalActors, @@ -1864,7 +1864,7 @@ namespace NActors { TActorId ReplyId; bool HasReply; TDispatchOptions DelegateeOptions; - TTestActorRuntimeBase* Runtime; + TTestActorRuntimeBase* Runtime; THolder<IReplyChecker> ReplyChecker; }; @@ -1889,7 +1889,7 @@ namespace NActors { private: TSimpleSharedPtr<TStrandingActorDecoratorContext> Context; - TTestActorRuntimeBase* Runtime; + TTestActorRuntimeBase* Runtime; TReplyCheckerCreator CreateReplyChecker; }; diff --git a/library/cpp/actors/testlib/test_runtime.h b/library/cpp/actors/testlib/test_runtime.h index 6baeebcb37..26e3b45c98 100644 --- a/library/cpp/actors/testlib/test_runtime.h +++ b/library/cpp/actors/testlib/test_runtime.h @@ -1,5 +1,5 @@ #pragma once - + #include <library/cpp/actors/core/actor.h> #include <library/cpp/actors/core/actorsystem.h> #include <library/cpp/actors/core/log.h> @@ -182,7 +182,7 @@ namespace NActors { } }; - class TTestActorRuntimeBase: public TNonCopyable { + class TTestActorRuntimeBase: public TNonCopyable { public: class TEdgeActor; class TSchedulerThreadStub; @@ -195,24 +195,24 @@ namespace NActors { RESCHEDULE }; - typedef std::function<EEventAction(TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event)> TEventObserver; - typedef std::function<void(TTestActorRuntimeBase& runtime, TScheduledEventsList& scheduledEvents, TEventsList& queue)> TScheduledEventsSelector; - typedef std::function<bool(TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event)> TEventFilter; - typedef std::function<bool(TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event, TDuration delay, TInstant& deadline)> TScheduledEventFilter; + typedef std::function<EEventAction(TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event)> TEventObserver; + typedef std::function<void(TTestActorRuntimeBase& runtime, TScheduledEventsList& scheduledEvents, TEventsList& queue)> TScheduledEventsSelector; + typedef std::function<bool(TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event)> TEventFilter; + typedef std::function<bool(TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event, TDuration delay, TInstant& deadline)> TScheduledEventFilter; typedef std::function<void(TTestActorRuntimeBase& runtime, const TActorId& parentId, const TActorId& actorId)> TRegistrationObserver; - TTestActorRuntimeBase(THeSingleSystemEnv); - TTestActorRuntimeBase(ui32 nodeCount, ui32 dataCenterCount, bool UseRealThreads); - TTestActorRuntimeBase(ui32 nodeCount, ui32 dataCenterCount); - TTestActorRuntimeBase(ui32 nodeCount = 1, bool useRealThreads = false); - virtual ~TTestActorRuntimeBase(); + TTestActorRuntimeBase(THeSingleSystemEnv); + TTestActorRuntimeBase(ui32 nodeCount, ui32 dataCenterCount, bool UseRealThreads); + TTestActorRuntimeBase(ui32 nodeCount, ui32 dataCenterCount); + TTestActorRuntimeBase(ui32 nodeCount = 1, bool useRealThreads = false); + virtual ~TTestActorRuntimeBase(); bool IsRealThreads() const; - static EEventAction DefaultObserverFunc(TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event); - static void DroppingScheduledEventsSelector(TTestActorRuntimeBase& runtime, TScheduledEventsList& scheduledEvents, TEventsList& queue); - static void CollapsedTimeScheduledEventsSelector(TTestActorRuntimeBase& runtime, TScheduledEventsList& scheduledEvents, TEventsList& queue); - static bool DefaultFilterFunc(TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event); - static bool NopFilterFunc(TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event, TDuration delay, TInstant& deadline); + static EEventAction DefaultObserverFunc(TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event); + static void DroppingScheduledEventsSelector(TTestActorRuntimeBase& runtime, TScheduledEventsList& scheduledEvents, TEventsList& queue); + static void CollapsedTimeScheduledEventsSelector(TTestActorRuntimeBase& runtime, TScheduledEventsList& scheduledEvents, TEventsList& queue); + static bool DefaultFilterFunc(TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event); + static bool NopFilterFunc(TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event, TDuration delay, TInstant& deadline); static void DefaultRegistrationObserver(TTestActorRuntimeBase& runtime, const TActorId& parentId, const TActorId& actorId); TEventObserver SetObserverFunc(TEventObserver observerFunc); TScheduledEventsSelector SetScheduledEventsSelectorFunc(TScheduledEventsSelector scheduledEventsSelectorFunc); @@ -233,7 +233,7 @@ namespace NActors { void UpdateCurrentTime(TInstant newTime); void AdvanceCurrentTime(TDuration duration); void AddLocalService(const TActorId& actorId, const TActorSetupCmd& cmd, ui32 nodeIndex = 0); - virtual void Initialize(); + virtual void Initialize(); ui32 GetNodeId(ui32 index = 0) const; ui32 GetNodeCount() const; ui64 AllocateLocalId(); @@ -291,7 +291,7 @@ namespace NActors { TEvent* GrabEdgeEventIf(TAutoPtr<IEventHandle>& handle, std::function<bool(const TEvent&)> predicate, TDuration simTimeout = TDuration::Max()) { handle.Destroy(); const ui32 eventType = TEvent::EventType; - WaitForEdgeEvents([&](TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event) { + WaitForEdgeEvents([&](TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event) { Y_UNUSED(runtime); if (event->GetTypeRewrite() != eventType) return false; @@ -323,7 +323,7 @@ namespace NActors { { typename TEvent::TPtr handle; const ui32 eventType = TEvent::EventType; - WaitForEdgeEvents([&](TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event) { + WaitForEdgeEvents([&](TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event) { Y_UNUSED(runtime); if (event->GetTypeRewrite() != eventType) return false; @@ -383,7 +383,7 @@ namespace NActors { std::tuple<TEvents*...> GrabEdgeEvents(TAutoPtr<IEventHandle>& handle, TDuration simTimeout = TDuration::Max()) { handle.Destroy(); auto eventTypes = { TEvents::EventType... }; - WaitForEdgeEvents([&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>& event) { + WaitForEdgeEvents([&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>& event) { if (std::find(std::begin(eventTypes), std::end(eventTypes), event->GetTypeRewrite()) == std::end(eventTypes)) return false; handle = event; @@ -472,26 +472,26 @@ namespace NActors { } protected: - struct TNodeDataBase; - TNodeDataBase* GetRawNode(ui32 node) const { + struct TNodeDataBase; + TNodeDataBase* GetRawNode(ui32 node) const { return Nodes.at(FirstNodeId + node).Get(); } - static IExecutorPool* CreateExecutorPoolStub(TTestActorRuntimeBase* runtime, ui32 nodeIndex, TNodeDataBase* node, ui32 poolId); - virtual TIntrusivePtr<NMonitoring::TDynamicCounters> GetCountersForComponent(TIntrusivePtr<NMonitoring::TDynamicCounters> counters, const char* component) { - Y_UNUSED(counters); - Y_UNUSED(component); - - // do nothing, just return the existing counters - return counters; - } - - THolder<TActorSystemSetup> MakeActorSystemSetup(ui32 nodeIndex, TNodeDataBase* node); - THolder<TActorSystem> MakeActorSystem(ui32 nodeIndex, TNodeDataBase* node); - virtual void InitActorSystemSetup(TActorSystemSetup& setup) { - Y_UNUSED(setup); - } - + static IExecutorPool* CreateExecutorPoolStub(TTestActorRuntimeBase* runtime, ui32 nodeIndex, TNodeDataBase* node, ui32 poolId); + virtual TIntrusivePtr<NMonitoring::TDynamicCounters> GetCountersForComponent(TIntrusivePtr<NMonitoring::TDynamicCounters> counters, const char* component) { + Y_UNUSED(counters); + Y_UNUSED(component); + + // do nothing, just return the existing counters + return counters; + } + + THolder<TActorSystemSetup> MakeActorSystemSetup(ui32 nodeIndex, TNodeDataBase* node); + THolder<TActorSystem> MakeActorSystem(ui32 nodeIndex, TNodeDataBase* node); + virtual void InitActorSystemSetup(TActorSystemSetup& setup) { + Y_UNUSED(setup); + } + private: IActor* FindActor(const TActorId& actorId, TNodeDataBase* node) const; void SendInternal(IEventHandle* ev, ui32 nodeIndex, bool viaActorSystem); @@ -506,18 +506,18 @@ namespace NActors { ui64 ScheduledLimit; THolder<TTempDir> TmpDir; const TThread::TId MainThreadId; - + protected: bool UseRealInterconnect = false; TInterconnectMock InterconnectMock; - bool IsInitialized = false; - bool SingleSysEnv = false; + bool IsInitialized = false; + bool SingleSysEnv = false; const TString ClusterUUID; const ui32 FirstNodeId; const ui32 NodeCount; const ui32 DataCenterCount; const bool UseRealThreads; - + ui64 LocalId; TMutex Mutex; TCondVar MailboxesHasEvents; @@ -532,28 +532,28 @@ namespace NActors { TAutoPtr<TLogBackend> LogBackend; bool NeedMonitoring; - TIntrusivePtr<IRandomProvider> RandomProvider; - TIntrusivePtr<ITimeProvider> TimeProvider; - + TIntrusivePtr<IRandomProvider> RandomProvider; + TIntrusivePtr<ITimeProvider> TimeProvider; + protected: - struct TNodeDataBase: public TThrRefBase { - TNodeDataBase(); + struct TNodeDataBase: public TThrRefBase { + TNodeDataBase(); void Stop(); - virtual ~TNodeDataBase(); - virtual ui64 GetLoggerPoolId() const { - return 0; - } - - template <typename T = void> - T* GetAppData() { - return static_cast<T*>(AppData0.get()); - } - - template <typename T = void> - const T* GetAppData() const { - return static_cast<T*>(AppData0.get()); - } - + virtual ~TNodeDataBase(); + virtual ui64 GetLoggerPoolId() const { + return 0; + } + + template <typename T = void> + T* GetAppData() { + return static_cast<T*>(AppData0.get()); + } + + template <typename T = void> + const T* GetAppData() const { + return static_cast<T*>(AppData0.get()); + } + TIntrusivePtr<NMonitoring::TDynamicCounters> DynamicCounters; TIntrusivePtr<NActors::NLog::TSettings> LogSettings; TIntrusivePtr<NInterconnect::TPollerThreads> Poller; @@ -563,44 +563,44 @@ namespace NActors { TMap<TActorId, IActor*> LocalServicesActors; TMap<IActor*, TActorId> ActorToActorId; THolder<TMailboxTable> MailboxTable; - std::shared_ptr<void> AppData0; + std::shared_ptr<void> AppData0; THolder<TActorSystem> ActorSystem; - THolder<IExecutorPool> SchedulerPool; + THolder<IExecutorPool> SchedulerPool; TVector<IExecutorPool*> ExecutorPools; THolder<TExecutorThread> ExecutorThread; - }; + }; - struct INodeFactory { - virtual ~INodeFactory() = default; - virtual TIntrusivePtr<TNodeDataBase> CreateNode() = 0; + struct INodeFactory { + virtual ~INodeFactory() = default; + virtual TIntrusivePtr<TNodeDataBase> CreateNode() = 0; }; - struct TDefaultNodeFactory final: INodeFactory { - virtual TIntrusivePtr<TNodeDataBase> CreateNode() override { - return new TNodeDataBase(); - } - }; - - INodeFactory& GetNodeFactory() { - return *NodeFactory; - } - - virtual TNodeDataBase* GetNodeById(size_t idx) { - return Nodes[idx].Get(); - } - - void InitNodes(); - void CleanupNodes(); - virtual void InitNodeImpl(TNodeDataBase*, size_t); - + struct TDefaultNodeFactory final: INodeFactory { + virtual TIntrusivePtr<TNodeDataBase> CreateNode() override { + return new TNodeDataBase(); + } + }; + + INodeFactory& GetNodeFactory() { + return *NodeFactory; + } + + virtual TNodeDataBase* GetNodeById(size_t idx) { + return Nodes[idx].Get(); + } + + void InitNodes(); + void CleanupNodes(); + virtual void InitNodeImpl(TNodeDataBase*, size_t); + static bool AllowSendFrom(TNodeDataBase* node, TAutoPtr<IEventHandle>& ev); - protected: - THolder<INodeFactory> NodeFactory{new TDefaultNodeFactory}; - + protected: + THolder<INodeFactory> NodeFactory{new TDefaultNodeFactory}; + private: - void InitNode(TNodeDataBase* node, size_t idx); - + void InitNode(TNodeDataBase* node, size_t idx); + struct TDispatchContext { const TDispatchOptions* Options; TDispatchContext* PrevContext; @@ -610,8 +610,8 @@ namespace NActors { bool FinalEventFound = false; }; - TProgramShouldContinue ShouldContinue; - TMap<ui32, TIntrusivePtr<TNodeDataBase>> Nodes; + TProgramShouldContinue ShouldContinue; + TMap<ui32, TIntrusivePtr<TNodeDataBase>> Nodes; ui64 CurrentTimestamp; TSet<TActorId> EdgeActors; THashMap<TEventMailboxId, TActorId, TEventMailboxId::THash> EdgeActorByMailbox; diff --git a/library/cpp/actors/ya.make b/library/cpp/actors/ya.make index b50699b855..737c7fbc18 100644 --- a/library/cpp/actors/ya.make +++ b/library/cpp/actors/ya.make @@ -11,6 +11,6 @@ RECURSE( protos util wilson - testlib - http + testlib + http ) diff --git a/library/cpp/grpc/client/grpc_client_low.cpp b/library/cpp/grpc/client/grpc_client_low.cpp index b2d0ea6eb2..73cc908ef8 100644 --- a/library/cpp/grpc/client/grpc_client_low.cpp +++ b/library/cpp/grpc/client/grpc_client_low.cpp @@ -15,7 +15,7 @@ #include <netinet/tcp.h> #endif -namespace NGrpc { +namespace NGrpc { void EnableGRpcTracing() { grpc_tracer_set_enabled("tcp", true); @@ -583,4 +583,4 @@ void TGRpcClientLow::ForgetContext(TContextImpl* context) { } } -} // namespace NGRpc +} // namespace NGRpc diff --git a/library/cpp/grpc/client/grpc_client_low.h b/library/cpp/grpc/client/grpc_client_low.h index e2a80e624e..ab0a0627be 100644 --- a/library/cpp/grpc/client/grpc_client_low.h +++ b/library/cpp/grpc/client/grpc_client_low.h @@ -21,7 +21,7 @@ * This file contains low level logic for grpc * This file should not be used in high level code without special reason */ -namespace NGrpc { +namespace NGrpc { const size_t DEFAULT_NUM_THREADS = 2; @@ -1396,4 +1396,4 @@ private: std::mutex JoinMutex_; }; -} // namespace NGRpc +} // namespace NGRpc diff --git a/library/cpp/grpc/client/grpc_common.h b/library/cpp/grpc/client/grpc_common.h index 12a3f7c28e..ffcdafe045 100644 --- a/library/cpp/grpc/client/grpc_common.h +++ b/library/cpp/grpc/client/grpc_common.h @@ -1,7 +1,7 @@ #pragma once #include <grpc++/grpc++.h> -#include <grpc++/resource_quota.h> +#include <grpc++/resource_quota.h> #include <util/datetime/base.h> #include <unordered_map> @@ -9,19 +9,19 @@ constexpr ui64 DEFAULT_GRPC_MESSAGE_SIZE_LIMIT = 64000000; -namespace NGrpc { +namespace NGrpc { struct TGRpcClientConfig { TString Locator; // format host:port TDuration Timeout = TDuration::Max(); // request timeout ui64 MaxMessageSize = DEFAULT_GRPC_MESSAGE_SIZE_LIMIT; // Max request and response size - ui64 MaxInboundMessageSize = 0; // overrides MaxMessageSize for incoming requests - ui64 MaxOutboundMessageSize = 0; // overrides MaxMessageSize for outgoing requests + ui64 MaxInboundMessageSize = 0; // overrides MaxMessageSize for incoming requests + ui64 MaxOutboundMessageSize = 0; // overrides MaxMessageSize for outgoing requests ui32 MaxInFlight = 0; bool EnableSsl = false; TString SslCaCert; //Implicitly enables Ssl if not empty grpc_compression_algorithm CompressionAlgoritm = GRPC_COMPRESS_NONE; - ui64 MemQuota = 0; + ui64 MemQuota = 0; std::unordered_map<TString, TString> StringChannelParams; std::unordered_map<TString, int> IntChannelParams; TString LoadBalancingPolicy = { }; @@ -48,10 +48,10 @@ struct TGRpcClientConfig { inline std::shared_ptr<grpc::ChannelInterface> CreateChannelInterface(const TGRpcClientConfig& config, grpc_socket_mutator* mutator = nullptr){ grpc::ChannelArguments args; - args.SetMaxReceiveMessageSize(config.MaxInboundMessageSize ? config.MaxInboundMessageSize : config.MaxMessageSize); - args.SetMaxSendMessageSize(config.MaxOutboundMessageSize ? config.MaxOutboundMessageSize : config.MaxMessageSize); + args.SetMaxReceiveMessageSize(config.MaxInboundMessageSize ? config.MaxInboundMessageSize : config.MaxMessageSize); + args.SetMaxSendMessageSize(config.MaxOutboundMessageSize ? config.MaxOutboundMessageSize : config.MaxMessageSize); args.SetCompressionAlgorithm(config.CompressionAlgoritm); - + for (const auto& kvp: config.StringChannelParams) { args.SetString(kvp.first, kvp.second); } @@ -60,11 +60,11 @@ inline std::shared_ptr<grpc::ChannelInterface> CreateChannelInterface(const TGRp args.SetInt(kvp.first, kvp.second); } - if (config.MemQuota) { - grpc::ResourceQuota quota; - quota.Resize(config.MemQuota); - args.SetResourceQuota(quota); - } + if (config.MemQuota) { + grpc::ResourceQuota quota; + quota.Resize(config.MemQuota); + args.SetResourceQuota(quota); + } if (mutator) { args.SetSocketMutator(mutator); } @@ -81,4 +81,4 @@ inline std::shared_ptr<grpc::ChannelInterface> CreateChannelInterface(const TGRp } } -} // namespace NGRpc +} // namespace NGRpc diff --git a/library/cpp/grpc/server/grpc_counters.h b/library/cpp/grpc/server/grpc_counters.h index 387163cfea..0b6c36c84c 100644 --- a/library/cpp/grpc/server/grpc_counters.h +++ b/library/cpp/grpc/server/grpc_counters.h @@ -6,14 +6,14 @@ namespace NGrpc { -struct ICounterBlock : public TThrRefBase { - virtual void CountNotOkRequest() = 0; - virtual void CountNotOkResponse() = 0; - virtual void CountNotAuthenticated() = 0; - virtual void CountResourceExhausted() = 0; - virtual void CountRequestBytes(ui32 requestSize) = 0; - virtual void CountResponseBytes(ui32 responseSize) = 0; - virtual void StartProcessing(ui32 requestSize) = 0; +struct ICounterBlock : public TThrRefBase { + virtual void CountNotOkRequest() = 0; + virtual void CountNotOkResponse() = 0; + virtual void CountNotAuthenticated() = 0; + virtual void CountResourceExhausted() = 0; + virtual void CountRequestBytes(ui32 requestSize) = 0; + virtual void CountResponseBytes(ui32 responseSize) = 0; + virtual void StartProcessing(ui32 requestSize) = 0; virtual void FinishProcessing(ui32 requestSize, ui32 responseSize, bool ok, ui32 status, TDuration requestDuration) = 0; virtual void CountRequestsWithoutDatabase() {} virtual void CountRequestsWithoutToken() {} @@ -21,11 +21,11 @@ struct ICounterBlock : public TThrRefBase { virtual TIntrusivePtr<ICounterBlock> Clone() { return this; } virtual void UseDatabase(const TString& database) { Y_UNUSED(database); } -}; - +}; + using ICounterBlockPtr = TIntrusivePtr<ICounterBlock>; -class TCounterBlock final : public ICounterBlock { +class TCounterBlock final : public ICounterBlock { NMonitoring::TDynamicCounters::TCounterPtr TotalCounter; NMonitoring::TDynamicCounters::TCounterPtr InflyCounter; NMonitoring::TDynamicCounters::TCounterPtr NotOkRequestCounter; @@ -36,7 +36,7 @@ class TCounterBlock final : public ICounterBlock { NMonitoring::TDynamicCounters::TCounterPtr NotAuthenticated; NMonitoring::TDynamicCounters::TCounterPtr ResourceExhausted; bool Percentile = false; - NMonitoring::TPercentileTracker<4, 512, 15> RequestHistMs; + NMonitoring::TPercentileTracker<4, 512, 15> RequestHistMs; std::array<NMonitoring::TDynamicCounters::TCounterPtr, 2> GRpcStatusCounters; public: @@ -66,31 +66,31 @@ public: } } - void CountNotOkRequest() override { + void CountNotOkRequest() override { NotOkRequestCounter->Inc(); } - void CountNotOkResponse() override { + void CountNotOkResponse() override { NotOkResponseCounter->Inc(); } - void CountNotAuthenticated() override { + void CountNotAuthenticated() override { NotAuthenticated->Inc(); } - void CountResourceExhausted() override { + void CountResourceExhausted() override { ResourceExhausted->Inc(); } - void CountRequestBytes(ui32 requestSize) override { + void CountRequestBytes(ui32 requestSize) override { *RequestBytes += requestSize; } - void CountResponseBytes(ui32 responseSize) override { + void CountResponseBytes(ui32 responseSize) override { *ResponseBytes += responseSize; } - void StartProcessing(ui32 requestSize) override { + void StartProcessing(ui32 requestSize) override { TotalCounter->Inc(); InflyCounter->Inc(); *RequestBytes += requestSize; diff --git a/library/cpp/grpc/server/grpc_request.h b/library/cpp/grpc/server/grpc_request.h index c88d941815..5bd8d3902b 100644 --- a/library/cpp/grpc/server/grpc_request.h +++ b/library/cpp/grpc/server/grpc_request.h @@ -38,13 +38,13 @@ IStreamAdaptor::TPtr CreateStreamAdaptor(); /////////////////////////////////////////////////////////////////////////////// template<typename TIn, typename TOut, typename TService, typename TInProtoPrinter, typename TOutProtoPrinter> -class TGRpcRequestImpl +class TGRpcRequestImpl : public TBaseAsyncContext<TService> , public IQueueEvent , public IRequestContextBase { using TThis = TGRpcRequestImpl<TIn, TOut, TService, TInProtoPrinter, TOutProtoPrinter>; - + public: using TOnRequest = std::function<void (IRequestContextBase* ctx)>; using TRequestCallback = void (TService::TCurrentGRpcService::AsyncService::*)(grpc::ServerContext*, TIn*, @@ -52,13 +52,13 @@ public: using TStreamRequestCallback = void (TService::TCurrentGRpcService::AsyncService::*)(grpc::ServerContext*, TIn*, grpc::ServerAsyncWriter<TOut>*, grpc::CompletionQueue*, grpc::ServerCompletionQueue*, void*); - TGRpcRequestImpl(TService* server, + TGRpcRequestImpl(TService* server, typename TService::TCurrentGRpcService::AsyncService* service, grpc::ServerCompletionQueue* cq, TOnRequest cb, TRequestCallback requestCallback, const char* name, - TLoggerPtr logger, + TLoggerPtr logger, ICounterBlockPtr counters, IGRpcRequestLimiterPtr limiter) : TBaseAsyncContext<TService>(service, cq) @@ -71,22 +71,22 @@ public: , Counters_(std::move(counters)) , RequestLimiter_(std::move(limiter)) , Writer_(new grpc::ServerAsyncResponseWriter<TUniversalResponseRef<TOut>>(&this->Context)) - , StateFunc_(&TThis::SetRequestDone) + , StateFunc_(&TThis::SetRequestDone) { AuthState_ = Server_->NeedAuth() ? TAuthState(true) : TAuthState(false); Request_ = google::protobuf::Arena::CreateMessage<TIn>(&Arena_); Y_VERIFY(Request_); - GRPC_LOG_DEBUG(Logger_, "[%p] created request Name# %s", this, Name_); + GRPC_LOG_DEBUG(Logger_, "[%p] created request Name# %s", this, Name_); FinishPromise_ = NThreading::NewPromise<EFinishStatus>(); } - TGRpcRequestImpl(TService* server, + TGRpcRequestImpl(TService* server, typename TService::TCurrentGRpcService::AsyncService* service, grpc::ServerCompletionQueue* cq, TOnRequest cb, TStreamRequestCallback requestCallback, const char* name, - TLoggerPtr logger, + TLoggerPtr logger, ICounterBlockPtr counters, IGRpcRequestLimiterPtr limiter) : TBaseAsyncContext<TService>(service, cq) @@ -99,12 +99,12 @@ public: , Counters_(std::move(counters)) , RequestLimiter_(std::move(limiter)) , StreamWriter_(new grpc::ServerAsyncWriter<TUniversalResponse<TOut>>(&this->Context)) - , StateFunc_(&TThis::SetRequestDone) + , StateFunc_(&TThis::SetRequestDone) { AuthState_ = Server_->NeedAuth() ? TAuthState(true) : TAuthState(false); Request_ = google::protobuf::Arena::CreateMessage<TIn>(&Arena_); Y_VERIFY(Request_); - GRPC_LOG_DEBUG(Logger_, "[%p] created streaming request Name# %s", this, Name_); + GRPC_LOG_DEBUG(Logger_, "[%p] created streaming request Name# %s", this, Name_); FinishPromise_ = NThreading::NewPromise<EFinishStatus>(); StreamAdaptor_ = CreateStreamAdaptor(); } @@ -138,7 +138,7 @@ public: } } - ~TGRpcRequestImpl() { + ~TGRpcRequestImpl() { // No direct dtor call allowed Y_ASSERT(RefCount() == 0); } @@ -190,10 +190,10 @@ public: WriteByteDataOk(resp); } - void ReplyError(grpc::StatusCode code, const TString& msg) override { + void ReplyError(grpc::StatusCode code, const TString& msg) override { FinishGrpcStatus(code, msg, false); - } - + } + void ReplyUnauthenticated(const TString& in) override { const TString message = in.empty() ? TString("unauthenticated") : TString("unauthenticated, ") + in; FinishGrpcStatus(grpc::StatusCode::UNAUTHENTICATED, message, false); @@ -232,10 +232,10 @@ private: void Clone() { if (!Server_->IsShuttingDown()) { if (RequestCallback_) { - MakeIntrusive<TThis>( + MakeIntrusive<TThis>( Server_, this->Service, this->CQ, Cb_, RequestCallback_, Name_, Logger_, Counters_->Clone(), RequestLimiter_)->Run(); } else { - MakeIntrusive<TThis>( + MakeIntrusive<TThis>( Server_, this->Service, this->CQ, Cb_, StreamRequestCallback_, Name_, Logger_, Counters_->Clone(), RequestLimiter_)->Run(); } } @@ -254,7 +254,7 @@ private: if (Writer_) { GRPC_LOG_DEBUG(Logger_, "[%p] issuing response Name# %s data# %s peer# %s", this, Name_, makeResponseString().data(), this->Context.peer().c_str()); - StateFunc_ = &TThis::SetFinishDone; + StateFunc_ = &TThis::SetFinishDone; ResponseSize = sz; Y_VERIFY(this->Context.c_call()); Writer_->Finish(TUniversalResponseRef<TOut>(resp), grpc::Status::OK, GetGRpcTag()); @@ -281,7 +281,7 @@ private: if (Writer_) { GRPC_LOG_DEBUG(Logger_, "[%p] issuing response Name# %s data# byteString peer# %s", this, Name_, this->Context.peer().c_str()); - StateFunc_ = &TThis::SetFinishDone; + StateFunc_ = &TThis::SetFinishDone; ResponseSize = sz; Writer_->Finish(TUniversalResponseRef<TOut>(resp), grpc::Status::OK, GetGRpcTag()); } else { @@ -342,7 +342,7 @@ private: } return resp; }; - GRPC_LOG_DEBUG(Logger_, "[%p] received request Name# %s ok# %s data# %s peer# %s", this, Name_, + GRPC_LOG_DEBUG(Logger_, "[%p] received request Name# %s ok# %s data# %s peer# %s", this, Name_, ok ? "true" : "false", makeRequestString().data(), this->Context.peer().c_str()); if (this->Context.c_call() == nullptr) { @@ -352,7 +352,7 @@ private: } else if (!(RequestRegistered_ = Server_->RegisterRequestCtx(this))) { // Request cannot be registered due to shutdown // It's unsafe to continue, so drop this request without processing - GRPC_LOG_DEBUG(Logger_, "[%p] dropping request Name# %s due to shutdown", this, Name_); + GRPC_LOG_DEBUG(Logger_, "[%p] dropping request Name# %s due to shutdown", this, Name_); this->Context.TryCancel(); return false; } @@ -423,7 +423,7 @@ private: } bool SetFinishDone(bool ok) { - GRPC_LOG_DEBUG(Logger_, "[%p] finished request Name# %s ok# %s peer# %s", this, Name_, + GRPC_LOG_DEBUG(Logger_, "[%p] finished request Name# %s ok# %s peer# %s", this, Name_, ok ? "true" : "false", this->Context.peer().c_str()); //PrintBackTrace(); DecRequest(); @@ -433,7 +433,7 @@ private: } bool SetFinishError(bool ok) { - GRPC_LOG_DEBUG(Logger_, "[%p] finished request with error Name# %s ok# %s peer# %s", this, Name_, + GRPC_LOG_DEBUG(Logger_, "[%p] finished request with error Name# %s ok# %s peer# %s", this, Name_, ok ? "true" : "false", this->Context.peer().c_str()); if (!SkipUpdateCountersOnError) { DecRequest(); @@ -479,14 +479,14 @@ private: Server_->DecRequest(); } - using TStateFunc = bool (TThis::*)(bool); + using TStateFunc = bool (TThis::*)(bool); TService* Server_; TOnRequest Cb_; TRequestCallback RequestCallback_; TStreamRequestCallback StreamRequestCallback_; const char* const Name_; - TLoggerPtr Logger_; - ICounterBlockPtr Counters_; + TLoggerPtr Logger_; + ICounterBlockPtr Counters_; IGRpcRequestLimiterPtr RequestLimiter_; THolder<grpc::ServerAsyncResponseWriter<TUniversalResponseRef<TOut>>> Writer_; @@ -503,8 +503,8 @@ private: TAuthState AuthState_ = 0; bool RequestRegistered_ = false; - using TFixedEvent = TQueueFixedEvent<TGRpcRequestImpl>; - TFixedEvent OnFinishTag = { this, &TGRpcRequestImpl::OnFinish }; + using TFixedEvent = TQueueFixedEvent<TGRpcRequestImpl>; + TFixedEvent OnFinishTag = { this, &TGRpcRequestImpl::OnFinish }; NThreading::TPromise<EFinishStatus> FinishPromise_; bool SkipUpdateCountersOnError = false; IStreamAdaptor::TPtr StreamAdaptor_; @@ -513,31 +513,31 @@ private: template<typename TIn, typename TOut, typename TService, typename TInProtoPrinter=google::protobuf::TextFormat::Printer, typename TOutProtoPrinter=google::protobuf::TextFormat::Printer> class TGRpcRequest: public TGRpcRequestImpl<TIn, TOut, TService, TInProtoPrinter, TOutProtoPrinter> { using TBase = TGRpcRequestImpl<TIn, TOut, TService, TInProtoPrinter, TOutProtoPrinter>; -public: - TGRpcRequest(TService* server, - typename TService::TCurrentGRpcService::AsyncService* service, - grpc::ServerCompletionQueue* cq, - typename TBase::TOnRequest cb, - typename TBase::TRequestCallback requestCallback, - const char* name, +public: + TGRpcRequest(TService* server, + typename TService::TCurrentGRpcService::AsyncService* service, + grpc::ServerCompletionQueue* cq, + typename TBase::TOnRequest cb, + typename TBase::TRequestCallback requestCallback, + const char* name, TLoggerPtr logger, ICounterBlockPtr counters, IGRpcRequestLimiterPtr limiter = nullptr) : TBase{server, service, cq, std::move(cb), std::move(requestCallback), name, std::move(logger), std::move(counters), std::move(limiter)} - { - } - - TGRpcRequest(TService* server, - typename TService::TCurrentGRpcService::AsyncService* service, - grpc::ServerCompletionQueue* cq, - typename TBase::TOnRequest cb, - typename TBase::TStreamRequestCallback requestCallback, - const char* name, + { + } + + TGRpcRequest(TService* server, + typename TService::TCurrentGRpcService::AsyncService* service, + grpc::ServerCompletionQueue* cq, + typename TBase::TOnRequest cb, + typename TBase::TStreamRequestCallback requestCallback, + const char* name, TLoggerPtr logger, - ICounterBlockPtr counters) + ICounterBlockPtr counters) : TBase{server, service, cq, std::move(cb), std::move(requestCallback), name, std::move(logger), std::move(counters), nullptr} - { - } -}; - + { + } +}; + } // namespace NGrpc diff --git a/library/cpp/grpc/server/grpc_request_base.h b/library/cpp/grpc/server/grpc_request_base.h index acf3871569..fcfce1c181 100644 --- a/library/cpp/grpc/server/grpc_request_base.h +++ b/library/cpp/grpc/server/grpc_request_base.h @@ -60,16 +60,16 @@ public: //! Implementation can swap protobuf message virtual void Reply(NProtoBuf::Message* resp, ui32 status = 0) = 0; - //! Send serialised response (The request shoult be created for bytes response type) + //! Send serialised response (The request shoult be created for bytes response type) //! Implementation can swap ByteBuffer virtual void Reply(grpc::ByteBuffer* resp, ui32 status = 0) = 0; //! Send grpc UNAUTHENTICATED status virtual void ReplyUnauthenticated(const TString& in) = 0; - //! Send grpc error - virtual void ReplyError(grpc::StatusCode code, const TString& msg) = 0; - + //! Send grpc error + virtual void ReplyError(grpc::StatusCode code, const TString& msg) = 0; + //! Returns deadline (server epoch related) if peer set it on its side, or Instanse::Max() otherwise virtual TInstant Deadline() const = 0; diff --git a/library/cpp/grpc/ya.make b/library/cpp/grpc/ya.make index 9239947ad1..3635124115 100644 --- a/library/cpp/grpc/ya.make +++ b/library/cpp/grpc/ya.make @@ -1,5 +1,5 @@ -RECURSE( - client +RECURSE( + client common server -) +) diff --git a/library/cpp/ipmath/ipmath.cpp b/library/cpp/ipmath/ipmath.cpp index ccda3bdc7c..b8cca00c80 100644 --- a/library/cpp/ipmath/ipmath.cpp +++ b/library/cpp/ipmath/ipmath.cpp @@ -1,357 +1,357 @@ -#include "ipmath.h" - -namespace { - constexpr auto IPV4_BITS = 32; - constexpr auto IPV6_BITS = 128; - - const ui128 MAX_IPV4_ADDR = Max<ui32>(); - const ui128 MAX_IPV6_ADDR = Max<ui128>(); - - TStringBuf TypeToString(TIpv6Address::TIpType type) { - switch (type) { - case TIpv6Address::Ipv4: +#include "ipmath.h" + +namespace { + constexpr auto IPV4_BITS = 32; + constexpr auto IPV6_BITS = 128; + + const ui128 MAX_IPV4_ADDR = Max<ui32>(); + const ui128 MAX_IPV6_ADDR = Max<ui128>(); + + TStringBuf TypeToString(TIpv6Address::TIpType type) { + switch (type) { + case TIpv6Address::Ipv4: return TStringBuf("IPv4"); - case TIpv6Address::Ipv6: + case TIpv6Address::Ipv6: return TStringBuf("IPv6"); - default: + default: return TStringBuf("UNKNOWN"); - } - } - - size_t MaxPrefixLenForType(TIpv6Address::TIpType type) { - switch (type) { - case TIpv6Address::Ipv4: - return IPV4_BITS; - case TIpv6Address::Ipv6: - return IPV6_BITS; - case TIpv6Address::LAST: - ythrow yexception() << "invalid type"; - } - } - - template <ui8 ADDR_LEN> - ui128 LowerBoundForPrefix(ui128 value, ui8 prefixLen) { - const int shift = ADDR_LEN - prefixLen; - const ui128 shifted = (shift < 128) ? (ui128{1} << shift) : 0; - ui128 mask = ~(shifted - 1); - return value & mask; - } - - template <ui8 ADDR_LEN> - ui128 UpperBoundForPrefix(ui128 value, ui8 prefixLen) { - const int shift = ADDR_LEN - prefixLen; - const ui128 shifted = (shift < 128) ? (ui128{1} << shift) : 0; - ui128 mask = shifted - 1; - return value | mask; - } - - auto LowerBoundForPrefix4 = LowerBoundForPrefix<IPV4_BITS>; - auto LowerBoundForPrefix6 = LowerBoundForPrefix<IPV6_BITS>; - auto UpperBoundForPrefix4 = UpperBoundForPrefix<IPV4_BITS>; - auto UpperBoundForPrefix6 = UpperBoundForPrefix<IPV6_BITS>; - - TIpv6Address IpFromStringSafe(const TString& s) { - bool ok{}; - auto addr = TIpv6Address::FromString(s, ok); - Y_ENSURE(ok, "Failed to parse an IP address from " << s); - return addr; - } - - /// it's different from TIpv6Address::IsValid for 0.0.0.0 - bool IsValid(TIpv6Address addr) { - switch (addr.Type()) { - case TIpv6Address::Ipv4: - case TIpv6Address::Ipv6: - return true; - - case TIpv6Address::LAST: - return false; - } - } - - bool HasNext(TIpv6Address addr) { - switch (addr.Type()) { - case TIpv6Address::Ipv4: - return ui128(addr) != MAX_IPV4_ADDR; - case TIpv6Address::Ipv6: - return ui128(addr) != MAX_IPV6_ADDR; - case TIpv6Address::LAST: - return false; - } - } - - TIpv6Address Next(TIpv6Address addr) { - return {ui128(addr) + 1, addr.Type()}; - } -} // namespace - -TIpv6Address LowerBoundForPrefix(TIpv6Address value, ui8 prefixLen) { - auto type = value.Type(); - switch (type) { - case TIpv6Address::Ipv4: - return {LowerBoundForPrefix4(value, prefixLen), type}; - case TIpv6Address::Ipv6: - return {LowerBoundForPrefix6(value, prefixLen), type}; - default: - ythrow yexception() << "invalid type"; - } -} - -TIpv6Address UpperBoundForPrefix(TIpv6Address value, ui8 prefixLen) { - auto type = value.Type(); - switch (type) { - case TIpv6Address::Ipv4: - return {UpperBoundForPrefix4(value, prefixLen), type}; - case TIpv6Address::Ipv6: - return {UpperBoundForPrefix6(value, prefixLen), type}; - default: - ythrow yexception() << "invalid type"; - } -} - -TIpAddressRange::TIpAddressRangeBuilder::operator TIpAddressRange() { - return Build(); -} - -TIpAddressRange TIpAddressRange::TIpAddressRangeBuilder::Build() { - return TIpAddressRange{Start_, End_}; -} - -TIpAddressRange::TIpAddressRangeBuilder::TIpAddressRangeBuilder(const TString& from) - : TIpAddressRangeBuilder{IpFromStringSafe(from)} -{ -} - -TIpAddressRange::TIpAddressRangeBuilder::TIpAddressRangeBuilder(TIpv6Address from) { - Y_ENSURE_EX(IsValid(from), TInvalidIpRangeException() << "Address " << from.ToString() << " is invalid"); - Start_ = from; - End_ = Start_; -} - -TIpAddressRange::TIpAddressRangeBuilder& TIpAddressRange::TIpAddressRangeBuilder::To(const TString& to) { - End_ = IpFromStringSafe(to); - return *this; -} - -TIpAddressRange::TIpAddressRangeBuilder& TIpAddressRange::TIpAddressRangeBuilder::To(TIpv6Address to) { - Y_ENSURE_EX(IsValid(to), TInvalidIpRangeException() << "Address " << to.ToString() << " is invalid"); - Start_ = to; - return *this; -} - -TIpAddressRange::TIpAddressRangeBuilder& TIpAddressRange::TIpAddressRangeBuilder::WithPrefix(ui8 len) { - Y_ENSURE_EX(IsValid(Start_), TInvalidIpRangeException() << "Start value must be set before prefix"); - const auto type = Start_.Type(); - const auto maxLen = MaxPrefixLenForType(type); - Y_ENSURE_EX(len <= maxLen, TInvalidIpRangeException() << "Maximum prefix length for this address type is " - << maxLen << ", but requested " << (ui32)len); - - const auto lowerBound = LowerBoundForPrefix(Start_, len); - Y_ENSURE_EX(Start_ == lowerBound, TInvalidIpRangeException() << "Cannot create IP range from start address " - << Start_ << " with prefix length " << (ui32)len); - - End_ = UpperBoundForPrefix(Start_, len); - - return *this; -} - -void TIpAddressRange::Init(TIpv6Address from, TIpv6Address to) { - Start_ = from; - End_ = to; - - Y_ENSURE_EX(Start_ <= End_, TInvalidIpRangeException() << "Invalid IP address range: from " << Start_ << " to " << End_); - Y_ENSURE_EX(Start_.Type() == End_.Type(), TInvalidIpRangeException() - << "Address type mismtach: start address type is " << TypeToString(Start_.Type()) - << " end type is " << TypeToString(End_.Type())); -} - -TIpAddressRange::TIpAddressRange(TIpv6Address start, TIpv6Address end) { - Y_ENSURE_EX(IsValid(start), TInvalidIpRangeException() << "start address " << start.ToString() << " is invalid"); - Y_ENSURE_EX(IsValid(end), TInvalidIpRangeException() << "end address " << end.ToString() << " is invalid"); - Init(start, end); -} - -TIpAddressRange::TIpAddressRange(const TString& start, const TString& end) { - auto startAddr = IpFromStringSafe(start); - auto endAddr = IpFromStringSafe(end); - Init(startAddr, endAddr); -} - -TIpAddressRange::~TIpAddressRange() { -} - -TIpAddressRange::TIpType TIpAddressRange::Type() const { - return Start_.Type(); -} - -ui128 TIpAddressRange::Size() const { - return ui128(End_) - ui128(Start_) + 1; -} - -bool TIpAddressRange::IsSingle() const { - return Start_ == End_; -} - -bool TIpAddressRange::Contains(const TIpAddressRange& other) const { - return Start_ <= other.Start_ && End_ >= other.End_; -} - -bool TIpAddressRange::Contains(const TIpv6Address& addr) const { - return Start_ <= addr && End_ >= addr; -} - -bool TIpAddressRange::Overlaps(const TIpAddressRange& other) const { - return Start_ <= other.End_ && other.Start_ <= End_; -} - -bool TIpAddressRange::IsConsecutive(const TIpAddressRange& other) const { - return (HasNext(End_) && Next(End_) == other.Start_) - || (HasNext(other.End_) && Next(other.End_) == Start_); -} - -TIpAddressRange TIpAddressRange::Union(const TIpAddressRange& other) const { - Y_ENSURE(IsConsecutive(other) || Overlaps(other), "Can merge only consecutive or overlapping ranges"); - Y_ENSURE(other.Start_.Type() == Start_.Type(), "Cannot merge ranges of addresses of different types"); - - auto s = Start_; - auto e = End_; - - s = {Min<ui128>(Start_, other.Start_), Start_.Type()}; - e = {Max<ui128>(End_, other.End_), End_.Type()}; - - return {s, e}; -} - -TIpAddressRange TIpAddressRange::FromCidrString(const TString& str) { - if (auto result = TryFromCidrString(str)) { - return *result; - } - - ythrow TInvalidIpRangeException() << "Cannot parse " << str << " as a CIDR string"; -} - -TMaybe<TIpAddressRange> TIpAddressRange::TryFromCidrString(const TString& str) { - auto idx = str.rfind('/'); - if (idx == TString::npos) { - return Nothing(); - } - - TStringBuf sb{str}; - TStringBuf address, prefix; - sb.SplitAt(idx, address, prefix); - prefix.Skip(1); - - ui8 prefixLen{}; - if (!::TryFromString(prefix, prefixLen)) { - return Nothing(); - } - - return TIpAddressRange::From(TString{address}) - .WithPrefix(prefixLen); -} - -TIpAddressRange TIpAddressRange::FromRangeString(const TString& str) { - if (auto result = TryFromRangeString(str)) { - return *result; - } - - ythrow TInvalidIpRangeException() << "Cannot parse " << str << " as a range string"; -} - -TMaybe<TIpAddressRange> TIpAddressRange::TryFromRangeString(const TString& str) { - auto idx = str.find('-'); - if (idx == TString::npos) { - return Nothing(); - } - - TStringBuf sb{str}; - TStringBuf start, end; - sb.SplitAt(idx, start, end); - end.Skip(1); - - return TIpAddressRange::From(TString{start}).To(TString{end}); -} - -TIpAddressRange TIpAddressRange::FromString(const TString& str) { - if (auto result = TryFromString(str)) { - return *result; - } - - ythrow TInvalidIpRangeException() << "Cannot parse an IP address from " << str; -} - -TMaybe<TIpAddressRange> TIpAddressRange::TryFromString(const TString& str) { - if (auto idx = str.find('/'); idx != TString::npos) { - return TryFromCidrString(str); - } else if (idx = str.find('-'); idx != TString::npos) { - return TryFromRangeString(str); - } else { - bool ok{}; - auto addr = TIpv6Address::FromString(str, ok); - if (!ok) { - return Nothing(); - } - - return TIpAddressRange::From(addr); - } -} - -TString TIpAddressRange::ToRangeString() const { - bool ok{}; - return TStringBuilder() << Start_.ToString(ok) << "-" << End_.ToString(ok); -} - -TIpAddressRange::TIterator TIpAddressRange::begin() const { - return Begin(); -} - -TIpAddressRange::TIterator TIpAddressRange::Begin() const { - return TIpAddressRange::TIterator{Start_}; -} - -TIpAddressRange::TIterator TIpAddressRange::end() const { - return End(); -} - -TIpAddressRange::TIterator TIpAddressRange::End() const { - return TIpAddressRange::TIterator{{ui128(End_) + 1, End_.Type()}}; -} - -TIpAddressRange::TIpAddressRangeBuilder TIpAddressRange::From(TIpv6Address from) { - return TIpAddressRangeBuilder{from}; -}; - -TIpAddressRange::TIpAddressRangeBuilder TIpAddressRange::From(const TString& from) { - return TIpAddressRangeBuilder{from}; -}; - -bool operator==(const TIpAddressRange& lhs, const TIpAddressRange& rhs) { - return lhs.Start_ == rhs.Start_ && lhs.End_ == rhs.End_; -} - -bool operator!=(const TIpAddressRange& lhs, const TIpAddressRange& rhs) { - return !(lhs == rhs); -} - -TIpAddressRange::TIterator::TIterator(TIpv6Address val) noexcept - : Current_{val} -{ -} - -bool TIpAddressRange::TIterator::operator==(const TIpAddressRange::TIterator& other) noexcept { - return Current_ == other.Current_; -} - -bool TIpAddressRange::TIterator::operator!=(const TIpAddressRange::TIterator& other) noexcept { - return !(*this == other); -} - -TIpAddressRange::TIterator& TIpAddressRange::TIterator::operator++() noexcept { - ui128 numeric = Current_; - Current_ = {numeric + 1, Current_.Type()}; - return *this; -} - -const TIpv6Address& TIpAddressRange::TIterator::operator*() noexcept { - return Current_; -} + } + } + + size_t MaxPrefixLenForType(TIpv6Address::TIpType type) { + switch (type) { + case TIpv6Address::Ipv4: + return IPV4_BITS; + case TIpv6Address::Ipv6: + return IPV6_BITS; + case TIpv6Address::LAST: + ythrow yexception() << "invalid type"; + } + } + + template <ui8 ADDR_LEN> + ui128 LowerBoundForPrefix(ui128 value, ui8 prefixLen) { + const int shift = ADDR_LEN - prefixLen; + const ui128 shifted = (shift < 128) ? (ui128{1} << shift) : 0; + ui128 mask = ~(shifted - 1); + return value & mask; + } + + template <ui8 ADDR_LEN> + ui128 UpperBoundForPrefix(ui128 value, ui8 prefixLen) { + const int shift = ADDR_LEN - prefixLen; + const ui128 shifted = (shift < 128) ? (ui128{1} << shift) : 0; + ui128 mask = shifted - 1; + return value | mask; + } + + auto LowerBoundForPrefix4 = LowerBoundForPrefix<IPV4_BITS>; + auto LowerBoundForPrefix6 = LowerBoundForPrefix<IPV6_BITS>; + auto UpperBoundForPrefix4 = UpperBoundForPrefix<IPV4_BITS>; + auto UpperBoundForPrefix6 = UpperBoundForPrefix<IPV6_BITS>; + + TIpv6Address IpFromStringSafe(const TString& s) { + bool ok{}; + auto addr = TIpv6Address::FromString(s, ok); + Y_ENSURE(ok, "Failed to parse an IP address from " << s); + return addr; + } + + /// it's different from TIpv6Address::IsValid for 0.0.0.0 + bool IsValid(TIpv6Address addr) { + switch (addr.Type()) { + case TIpv6Address::Ipv4: + case TIpv6Address::Ipv6: + return true; + + case TIpv6Address::LAST: + return false; + } + } + + bool HasNext(TIpv6Address addr) { + switch (addr.Type()) { + case TIpv6Address::Ipv4: + return ui128(addr) != MAX_IPV4_ADDR; + case TIpv6Address::Ipv6: + return ui128(addr) != MAX_IPV6_ADDR; + case TIpv6Address::LAST: + return false; + } + } + + TIpv6Address Next(TIpv6Address addr) { + return {ui128(addr) + 1, addr.Type()}; + } +} // namespace + +TIpv6Address LowerBoundForPrefix(TIpv6Address value, ui8 prefixLen) { + auto type = value.Type(); + switch (type) { + case TIpv6Address::Ipv4: + return {LowerBoundForPrefix4(value, prefixLen), type}; + case TIpv6Address::Ipv6: + return {LowerBoundForPrefix6(value, prefixLen), type}; + default: + ythrow yexception() << "invalid type"; + } +} + +TIpv6Address UpperBoundForPrefix(TIpv6Address value, ui8 prefixLen) { + auto type = value.Type(); + switch (type) { + case TIpv6Address::Ipv4: + return {UpperBoundForPrefix4(value, prefixLen), type}; + case TIpv6Address::Ipv6: + return {UpperBoundForPrefix6(value, prefixLen), type}; + default: + ythrow yexception() << "invalid type"; + } +} + +TIpAddressRange::TIpAddressRangeBuilder::operator TIpAddressRange() { + return Build(); +} + +TIpAddressRange TIpAddressRange::TIpAddressRangeBuilder::Build() { + return TIpAddressRange{Start_, End_}; +} + +TIpAddressRange::TIpAddressRangeBuilder::TIpAddressRangeBuilder(const TString& from) + : TIpAddressRangeBuilder{IpFromStringSafe(from)} +{ +} + +TIpAddressRange::TIpAddressRangeBuilder::TIpAddressRangeBuilder(TIpv6Address from) { + Y_ENSURE_EX(IsValid(from), TInvalidIpRangeException() << "Address " << from.ToString() << " is invalid"); + Start_ = from; + End_ = Start_; +} + +TIpAddressRange::TIpAddressRangeBuilder& TIpAddressRange::TIpAddressRangeBuilder::To(const TString& to) { + End_ = IpFromStringSafe(to); + return *this; +} + +TIpAddressRange::TIpAddressRangeBuilder& TIpAddressRange::TIpAddressRangeBuilder::To(TIpv6Address to) { + Y_ENSURE_EX(IsValid(to), TInvalidIpRangeException() << "Address " << to.ToString() << " is invalid"); + Start_ = to; + return *this; +} + +TIpAddressRange::TIpAddressRangeBuilder& TIpAddressRange::TIpAddressRangeBuilder::WithPrefix(ui8 len) { + Y_ENSURE_EX(IsValid(Start_), TInvalidIpRangeException() << "Start value must be set before prefix"); + const auto type = Start_.Type(); + const auto maxLen = MaxPrefixLenForType(type); + Y_ENSURE_EX(len <= maxLen, TInvalidIpRangeException() << "Maximum prefix length for this address type is " + << maxLen << ", but requested " << (ui32)len); + + const auto lowerBound = LowerBoundForPrefix(Start_, len); + Y_ENSURE_EX(Start_ == lowerBound, TInvalidIpRangeException() << "Cannot create IP range from start address " + << Start_ << " with prefix length " << (ui32)len); + + End_ = UpperBoundForPrefix(Start_, len); + + return *this; +} + +void TIpAddressRange::Init(TIpv6Address from, TIpv6Address to) { + Start_ = from; + End_ = to; + + Y_ENSURE_EX(Start_ <= End_, TInvalidIpRangeException() << "Invalid IP address range: from " << Start_ << " to " << End_); + Y_ENSURE_EX(Start_.Type() == End_.Type(), TInvalidIpRangeException() + << "Address type mismtach: start address type is " << TypeToString(Start_.Type()) + << " end type is " << TypeToString(End_.Type())); +} + +TIpAddressRange::TIpAddressRange(TIpv6Address start, TIpv6Address end) { + Y_ENSURE_EX(IsValid(start), TInvalidIpRangeException() << "start address " << start.ToString() << " is invalid"); + Y_ENSURE_EX(IsValid(end), TInvalidIpRangeException() << "end address " << end.ToString() << " is invalid"); + Init(start, end); +} + +TIpAddressRange::TIpAddressRange(const TString& start, const TString& end) { + auto startAddr = IpFromStringSafe(start); + auto endAddr = IpFromStringSafe(end); + Init(startAddr, endAddr); +} + +TIpAddressRange::~TIpAddressRange() { +} + +TIpAddressRange::TIpType TIpAddressRange::Type() const { + return Start_.Type(); +} + +ui128 TIpAddressRange::Size() const { + return ui128(End_) - ui128(Start_) + 1; +} + +bool TIpAddressRange::IsSingle() const { + return Start_ == End_; +} + +bool TIpAddressRange::Contains(const TIpAddressRange& other) const { + return Start_ <= other.Start_ && End_ >= other.End_; +} + +bool TIpAddressRange::Contains(const TIpv6Address& addr) const { + return Start_ <= addr && End_ >= addr; +} + +bool TIpAddressRange::Overlaps(const TIpAddressRange& other) const { + return Start_ <= other.End_ && other.Start_ <= End_; +} + +bool TIpAddressRange::IsConsecutive(const TIpAddressRange& other) const { + return (HasNext(End_) && Next(End_) == other.Start_) + || (HasNext(other.End_) && Next(other.End_) == Start_); +} + +TIpAddressRange TIpAddressRange::Union(const TIpAddressRange& other) const { + Y_ENSURE(IsConsecutive(other) || Overlaps(other), "Can merge only consecutive or overlapping ranges"); + Y_ENSURE(other.Start_.Type() == Start_.Type(), "Cannot merge ranges of addresses of different types"); + + auto s = Start_; + auto e = End_; + + s = {Min<ui128>(Start_, other.Start_), Start_.Type()}; + e = {Max<ui128>(End_, other.End_), End_.Type()}; + + return {s, e}; +} + +TIpAddressRange TIpAddressRange::FromCidrString(const TString& str) { + if (auto result = TryFromCidrString(str)) { + return *result; + } + + ythrow TInvalidIpRangeException() << "Cannot parse " << str << " as a CIDR string"; +} + +TMaybe<TIpAddressRange> TIpAddressRange::TryFromCidrString(const TString& str) { + auto idx = str.rfind('/'); + if (idx == TString::npos) { + return Nothing(); + } + + TStringBuf sb{str}; + TStringBuf address, prefix; + sb.SplitAt(idx, address, prefix); + prefix.Skip(1); + + ui8 prefixLen{}; + if (!::TryFromString(prefix, prefixLen)) { + return Nothing(); + } + + return TIpAddressRange::From(TString{address}) + .WithPrefix(prefixLen); +} + +TIpAddressRange TIpAddressRange::FromRangeString(const TString& str) { + if (auto result = TryFromRangeString(str)) { + return *result; + } + + ythrow TInvalidIpRangeException() << "Cannot parse " << str << " as a range string"; +} + +TMaybe<TIpAddressRange> TIpAddressRange::TryFromRangeString(const TString& str) { + auto idx = str.find('-'); + if (idx == TString::npos) { + return Nothing(); + } + + TStringBuf sb{str}; + TStringBuf start, end; + sb.SplitAt(idx, start, end); + end.Skip(1); + + return TIpAddressRange::From(TString{start}).To(TString{end}); +} + +TIpAddressRange TIpAddressRange::FromString(const TString& str) { + if (auto result = TryFromString(str)) { + return *result; + } + + ythrow TInvalidIpRangeException() << "Cannot parse an IP address from " << str; +} + +TMaybe<TIpAddressRange> TIpAddressRange::TryFromString(const TString& str) { + if (auto idx = str.find('/'); idx != TString::npos) { + return TryFromCidrString(str); + } else if (idx = str.find('-'); idx != TString::npos) { + return TryFromRangeString(str); + } else { + bool ok{}; + auto addr = TIpv6Address::FromString(str, ok); + if (!ok) { + return Nothing(); + } + + return TIpAddressRange::From(addr); + } +} + +TString TIpAddressRange::ToRangeString() const { + bool ok{}; + return TStringBuilder() << Start_.ToString(ok) << "-" << End_.ToString(ok); +} + +TIpAddressRange::TIterator TIpAddressRange::begin() const { + return Begin(); +} + +TIpAddressRange::TIterator TIpAddressRange::Begin() const { + return TIpAddressRange::TIterator{Start_}; +} + +TIpAddressRange::TIterator TIpAddressRange::end() const { + return End(); +} + +TIpAddressRange::TIterator TIpAddressRange::End() const { + return TIpAddressRange::TIterator{{ui128(End_) + 1, End_.Type()}}; +} + +TIpAddressRange::TIpAddressRangeBuilder TIpAddressRange::From(TIpv6Address from) { + return TIpAddressRangeBuilder{from}; +}; + +TIpAddressRange::TIpAddressRangeBuilder TIpAddressRange::From(const TString& from) { + return TIpAddressRangeBuilder{from}; +}; + +bool operator==(const TIpAddressRange& lhs, const TIpAddressRange& rhs) { + return lhs.Start_ == rhs.Start_ && lhs.End_ == rhs.End_; +} + +bool operator!=(const TIpAddressRange& lhs, const TIpAddressRange& rhs) { + return !(lhs == rhs); +} + +TIpAddressRange::TIterator::TIterator(TIpv6Address val) noexcept + : Current_{val} +{ +} + +bool TIpAddressRange::TIterator::operator==(const TIpAddressRange::TIterator& other) noexcept { + return Current_ == other.Current_; +} + +bool TIpAddressRange::TIterator::operator!=(const TIpAddressRange::TIterator& other) noexcept { + return !(*this == other); +} + +TIpAddressRange::TIterator& TIpAddressRange::TIterator::operator++() noexcept { + ui128 numeric = Current_; + Current_ = {numeric + 1, Current_.Type()}; + return *this; +} + +const TIpv6Address& TIpAddressRange::TIterator::operator*() noexcept { + return Current_; +} diff --git a/library/cpp/ipmath/ipmath.h b/library/cpp/ipmath/ipmath.h index 1985994aa7..b6df5416f8 100644 --- a/library/cpp/ipmath/ipmath.h +++ b/library/cpp/ipmath/ipmath.h @@ -1,160 +1,160 @@ -#pragma once - -#include <library/cpp/ipv6_address/ipv6_address.h> - -#include <util/generic/maybe.h> -#include <util/ysaveload.h> - -struct TInvalidIpRangeException: public virtual yexception { -}; - -class TIpAddressRange { - friend bool operator==(const TIpAddressRange& lhs, const TIpAddressRange& rhs); - friend bool operator!=(const TIpAddressRange& lhs, const TIpAddressRange& rhs); - - class TIpAddressRangeBuilder; -public: - class TIterator; - using TIpType = TIpv6Address::TIpType; - - TIpAddressRange() = default; - TIpAddressRange(TIpv6Address start, TIpv6Address end); - TIpAddressRange(const TString& start, const TString& end); - ~TIpAddressRange(); - - static TIpAddressRangeBuilder From(TIpv6Address from); - static TIpAddressRangeBuilder From(const TString& from); - - /** - * Parses a string tormatted in Classless Iter-Domain Routing (CIDR) notation. - * @param str a CIDR-formatted string, e.g. "192.168.0.0/16" - * @return a new TIpAddressRange - * @throws TInvalidIpRangeException if the string cannot be parsed. - */ - static TIpAddressRange FromCidrString(const TString& str); - static TMaybe<TIpAddressRange> TryFromCidrString(const TString& str); - - /** - * Parses a string formatted as two dash-separated addresses. - * @param str a CIDR-formatted string, e.g. "192.168.0.0-192.168.0.2" - * @return a new TIpAddressRange - * @throws TInvalidIpRangeException if the string cannot be parsed. - */ - static TIpAddressRange FromRangeString(const TString& str); - static TMaybe<TIpAddressRange> TryFromRangeString(const TString& str); - - TString ToRangeString() const; - - /** - * Tries to guess the format and parse it. Format must be one of: CIDR ("10.0.0.0/24"), range ("10.0.0.0-10.0.0.10") or a single address. - * @return a new TIpAddressRange - * @throws TInvlidIpRangeException if the string doesn't match any known format or if parsing failed. - */ - static TIpAddressRange FromString(const TString& str); - static TMaybe<TIpAddressRange> TryFromString(const TString& str); - - TIpType Type() const; - - // XXX: uint128 cannot hold size of the complete range of IPv6 addresses. Use IsComplete to determine whether this is the case. - ui128 Size() const; - - /** - * Determines whether this range contains only one address. - * @return true if contains only one address, otherwise false. - */ - bool IsSingle() const; - bool IsComplete() const; - - bool Contains(const TIpAddressRange& other) const; - bool Contains(const TIpv6Address& addr) const; - - bool Overlaps(const TIpAddressRange& other) const; - - /** - * Determines whether two ranges follow one after another without any gaps. - * @return true if either this range follows the given one or vice versa, otherwise false. - */ - bool IsConsecutive(const TIpAddressRange& other) const; - - /** - * Concatenates another range into this one. - * Note, that ranges must be either consecutive or overlapping. - * @throws yexception if ranges are neither consecutive nor overlapping. - */ - TIpAddressRange Union(const TIpAddressRange& other) const; - - template <typename TFunction> - void ForEach(TFunction func); - - // for-each compliance interface - TIterator begin() const; - TIterator end() const; - - // Arcadia style-guide friendly - TIterator Begin() const; - TIterator End() const; - - Y_SAVELOAD_DEFINE(Start_, End_); - -private: - void Init(TIpv6Address, TIpv6Address); - - TIpv6Address Start_; - TIpv6Address End_; -}; - -bool operator==(const TIpAddressRange& lhs, const TIpAddressRange& rhs); -bool operator!=(const TIpAddressRange& lhs, const TIpAddressRange& rhs); - -TIpv6Address LowerBoundForPrefix(TIpv6Address value, ui8 prefixLen); -TIpv6Address UpperBoundForPrefix(TIpv6Address value, ui8 prefixLen); - - -class TIpAddressRange::TIpAddressRangeBuilder { - friend class TIpAddressRange; - TIpAddressRangeBuilder() = default; - TIpAddressRangeBuilder(TIpv6Address from); - TIpAddressRangeBuilder(const TString& from); - TIpAddressRangeBuilder(const TIpAddressRangeBuilder&) = default; - TIpAddressRangeBuilder& operator=(const TIpAddressRangeBuilder&) = default; - - TIpAddressRangeBuilder(TIpAddressRangeBuilder&&) = default; - TIpAddressRangeBuilder& operator=(TIpAddressRangeBuilder&&) = default; - -public: - operator TIpAddressRange(); - TIpAddressRange Build(); - - TIpAddressRangeBuilder& To(const TString&); - TIpAddressRangeBuilder& To(TIpv6Address); - - TIpAddressRangeBuilder& WithPrefix(ui8 len); - -private: - TIpv6Address Start_; - TIpv6Address End_; -}; - - -class TIpAddressRange::TIterator { -public: - TIterator(TIpv6Address val) noexcept; - - bool operator==(const TIpAddressRange::TIterator& other) noexcept; - bool operator!=(const TIpAddressRange::TIterator& other) noexcept; - - TIterator& operator++() noexcept; - const TIpv6Address& operator*() noexcept; - -private: - TIpv6Address Current_; -}; - - -template <typename TFunction> -void TIpAddressRange::ForEach(TFunction func) { - static_assert(std::is_invocable<TFunction, TIpv6Address>::value, "function must take single address argument"); - for (auto addr : *this) { - func(addr); - } -} +#pragma once + +#include <library/cpp/ipv6_address/ipv6_address.h> + +#include <util/generic/maybe.h> +#include <util/ysaveload.h> + +struct TInvalidIpRangeException: public virtual yexception { +}; + +class TIpAddressRange { + friend bool operator==(const TIpAddressRange& lhs, const TIpAddressRange& rhs); + friend bool operator!=(const TIpAddressRange& lhs, const TIpAddressRange& rhs); + + class TIpAddressRangeBuilder; +public: + class TIterator; + using TIpType = TIpv6Address::TIpType; + + TIpAddressRange() = default; + TIpAddressRange(TIpv6Address start, TIpv6Address end); + TIpAddressRange(const TString& start, const TString& end); + ~TIpAddressRange(); + + static TIpAddressRangeBuilder From(TIpv6Address from); + static TIpAddressRangeBuilder From(const TString& from); + + /** + * Parses a string tormatted in Classless Iter-Domain Routing (CIDR) notation. + * @param str a CIDR-formatted string, e.g. "192.168.0.0/16" + * @return a new TIpAddressRange + * @throws TInvalidIpRangeException if the string cannot be parsed. + */ + static TIpAddressRange FromCidrString(const TString& str); + static TMaybe<TIpAddressRange> TryFromCidrString(const TString& str); + + /** + * Parses a string formatted as two dash-separated addresses. + * @param str a CIDR-formatted string, e.g. "192.168.0.0-192.168.0.2" + * @return a new TIpAddressRange + * @throws TInvalidIpRangeException if the string cannot be parsed. + */ + static TIpAddressRange FromRangeString(const TString& str); + static TMaybe<TIpAddressRange> TryFromRangeString(const TString& str); + + TString ToRangeString() const; + + /** + * Tries to guess the format and parse it. Format must be one of: CIDR ("10.0.0.0/24"), range ("10.0.0.0-10.0.0.10") or a single address. + * @return a new TIpAddressRange + * @throws TInvlidIpRangeException if the string doesn't match any known format or if parsing failed. + */ + static TIpAddressRange FromString(const TString& str); + static TMaybe<TIpAddressRange> TryFromString(const TString& str); + + TIpType Type() const; + + // XXX: uint128 cannot hold size of the complete range of IPv6 addresses. Use IsComplete to determine whether this is the case. + ui128 Size() const; + + /** + * Determines whether this range contains only one address. + * @return true if contains only one address, otherwise false. + */ + bool IsSingle() const; + bool IsComplete() const; + + bool Contains(const TIpAddressRange& other) const; + bool Contains(const TIpv6Address& addr) const; + + bool Overlaps(const TIpAddressRange& other) const; + + /** + * Determines whether two ranges follow one after another without any gaps. + * @return true if either this range follows the given one or vice versa, otherwise false. + */ + bool IsConsecutive(const TIpAddressRange& other) const; + + /** + * Concatenates another range into this one. + * Note, that ranges must be either consecutive or overlapping. + * @throws yexception if ranges are neither consecutive nor overlapping. + */ + TIpAddressRange Union(const TIpAddressRange& other) const; + + template <typename TFunction> + void ForEach(TFunction func); + + // for-each compliance interface + TIterator begin() const; + TIterator end() const; + + // Arcadia style-guide friendly + TIterator Begin() const; + TIterator End() const; + + Y_SAVELOAD_DEFINE(Start_, End_); + +private: + void Init(TIpv6Address, TIpv6Address); + + TIpv6Address Start_; + TIpv6Address End_; +}; + +bool operator==(const TIpAddressRange& lhs, const TIpAddressRange& rhs); +bool operator!=(const TIpAddressRange& lhs, const TIpAddressRange& rhs); + +TIpv6Address LowerBoundForPrefix(TIpv6Address value, ui8 prefixLen); +TIpv6Address UpperBoundForPrefix(TIpv6Address value, ui8 prefixLen); + + +class TIpAddressRange::TIpAddressRangeBuilder { + friend class TIpAddressRange; + TIpAddressRangeBuilder() = default; + TIpAddressRangeBuilder(TIpv6Address from); + TIpAddressRangeBuilder(const TString& from); + TIpAddressRangeBuilder(const TIpAddressRangeBuilder&) = default; + TIpAddressRangeBuilder& operator=(const TIpAddressRangeBuilder&) = default; + + TIpAddressRangeBuilder(TIpAddressRangeBuilder&&) = default; + TIpAddressRangeBuilder& operator=(TIpAddressRangeBuilder&&) = default; + +public: + operator TIpAddressRange(); + TIpAddressRange Build(); + + TIpAddressRangeBuilder& To(const TString&); + TIpAddressRangeBuilder& To(TIpv6Address); + + TIpAddressRangeBuilder& WithPrefix(ui8 len); + +private: + TIpv6Address Start_; + TIpv6Address End_; +}; + + +class TIpAddressRange::TIterator { +public: + TIterator(TIpv6Address val) noexcept; + + bool operator==(const TIpAddressRange::TIterator& other) noexcept; + bool operator!=(const TIpAddressRange::TIterator& other) noexcept; + + TIterator& operator++() noexcept; + const TIpv6Address& operator*() noexcept; + +private: + TIpv6Address Current_; +}; + + +template <typename TFunction> +void TIpAddressRange::ForEach(TFunction func) { + static_assert(std::is_invocable<TFunction, TIpv6Address>::value, "function must take single address argument"); + for (auto addr : *this) { + func(addr); + } +} diff --git a/library/cpp/ipmath/ipmath_ut.cpp b/library/cpp/ipmath/ipmath_ut.cpp index 4a509afd16..5fe459ecc8 100644 --- a/library/cpp/ipmath/ipmath_ut.cpp +++ b/library/cpp/ipmath/ipmath_ut.cpp @@ -1,507 +1,507 @@ -#include "ipmath.h" -#include "range_set.h" - +#include "ipmath.h" +#include "range_set.h" + #include <library/cpp/testing/unittest/registar.h> #include <library/cpp/testing/gmock_in_unittest/gmock.h> - -#include <library/cpp/ipv6_address/ipv6_address.h> - -using namespace testing; - -static constexpr auto MIN_IPV6_ADDR = "::"; -static constexpr auto MAX_IPV6_ADDR = "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff"; - -std::ostream& operator<<(std::ostream& os, const TIpAddressRange& r) { - auto s = r.ToRangeString(); - os.write(s.data(), s.size()); - return os; -} - -std::ostream& operator<<(std::ostream& os, const TIpRangeSet& rangeSet) { - os << "{\n"; - - for (auto&& r : rangeSet) { - os << r << '\n'; - } - - os << "}\n"; - return os; -} - -class TIpRangeTests: public TTestBase { -public: - UNIT_TEST_SUITE(TIpRangeTests); - UNIT_TEST(IpRangeFromIpv4); - UNIT_TEST(IpRangeFromIpv6); - UNIT_TEST(FullIpRange); - UNIT_TEST(IpRangeFromCidr); - UNIT_TEST(IpRangeFromIpv4Builder); - UNIT_TEST(IpRangeFromInvalidIpv4); - UNIT_TEST(IpRangeFromInvalidIpv6); - UNIT_TEST(RangeFromSingleAddress); - UNIT_TEST(RangeFromRangeString); - UNIT_TEST(ManualIteration); - UNIT_TEST(RangeRelations); - UNIT_TEST(RangeUnion); - UNIT_TEST_SUITE_END(); - - void RangeFromSingleAddress() { - for (auto addrStr : {"192.168.0.1", "::2"}) { - auto range = TIpAddressRange::From(addrStr).Build(); - ASSERT_THAT(range.Size(), Eq(1)); - ASSERT_TRUE(range.IsSingle()); - - auto range2 = TIpAddressRange{addrStr, addrStr}; - ASSERT_THAT(range2, Eq(range)); - - TVector<ui128> result; - range.ForEach([&result] (TIpv6Address addr) { - result.push_back(addr); - }); - - bool ok{}; - ASSERT_THAT(result, ElementsAre(TIpv6Address::FromString(addrStr, ok))); - } - - } - - void IpRangeFromIpv4() { - bool ok{}; - - auto s = TIpv6Address::FromString("192.168.0.0", ok); - ASSERT_TRUE(ok); - auto e = TIpv6Address::FromString("192.168.0.255", ok); - ASSERT_TRUE(ok); - - TIpAddressRange range{s, e}; - - ASSERT_THAT(range.Size(), Eq(256)); - ASSERT_THAT(range.Type(), Eq(TIpAddressRange::TIpType::Ipv4)); - - TIpAddressRange range2{"192.168.0.0", "192.168.0.255"}; - ASSERT_THAT(range2.Size(), Eq(256)); - ASSERT_THAT(range2, Eq(range)); - } - - void IpRangeFromIpv6() { - bool ok{}; - - auto s = TIpv6Address::FromString("ffce:abcd::", ok); - ASSERT_TRUE(ok); - auto e = TIpv6Address::FromString("ffce:abcd::00ff", ok); - ASSERT_TRUE(ok); - - TIpAddressRange range{s, e}; - - ASSERT_THAT(range.Size(), Eq(256)); - - TIpAddressRange range2{"ffce:abcd::", "ffce:abcd::00ff"}; - ASSERT_THAT(range2.Size(), Eq(256)); - ASSERT_THAT(range.Type(), Eq(TIpAddressRange::TIpType::Ipv6)); - ASSERT_THAT(range2, Eq(range)); - } - - - void FullIpRange() { - auto check = [] (auto start, auto end, ui128 expectedSize) { - auto range = TIpAddressRange::From(start).To(end).Build(); - ASSERT_THAT(range.Size(), Eq(expectedSize)); - }; - - check("0.0.0.0", "255.255.255.255", ui128(Max<ui32>()) + 1); - // XXX - // check(MIN_IPV6_ADDR, MAX_IPV6_ADDR, ui128(Max<ui128>() + 1)); - } - - void IpRangeFromCidr() { - auto range = TIpAddressRange::FromCidrString("10.0.0.0/30"); - - ASSERT_THAT(range.Size(), Eq(4)); - TVector<TIpv6Address> result; - Copy(range.begin(), range.end(), std::back_inserter(result)); - - bool ok; - TVector<TIpv6Address> expected { - TIpv6Address::FromString("10.0.0.0", ok), - TIpv6Address::FromString("10.0.0.1", ok), - TIpv6Address::FromString("10.0.0.2", ok), - TIpv6Address::FromString("10.0.0.3", ok), - }; - - ASSERT_THAT(result, ElementsAreArray(expected)); - - // single host - ASSERT_THAT(TIpAddressRange::FromCidrString("ffce:abcd::/128"), Eq(TIpAddressRange::From("ffce:abcd::").Build())); - ASSERT_THAT(TIpAddressRange::FromCidrString("192.168.0.1/32"), Eq(TIpAddressRange::From("192.168.0.1").Build())); - - // full range - ASSERT_THAT(TIpAddressRange::FromCidrString("::/0"), Eq(TIpAddressRange::From(MIN_IPV6_ADDR).To(MAX_IPV6_ADDR).Build())); - ASSERT_THAT(TIpAddressRange::FromCidrString("0.0.0.0/0"), Eq(TIpAddressRange::From("0.0.0.0").To("255.255.255.255").Build())); - - // illformed - ASSERT_THROW(TIpAddressRange::FromCidrString("::/"), TInvalidIpRangeException); - ASSERT_THROW(TIpAddressRange::FromCidrString("::"), TInvalidIpRangeException); - ASSERT_THROW(TIpAddressRange::FromCidrString("/::"), TInvalidIpRangeException); - ASSERT_THROW(TIpAddressRange::FromCidrString("::/150"), TInvalidIpRangeException); - } - - void RangeFromRangeString() { - { - auto range = TIpAddressRange::FromRangeString("10.0.0.0-10.0.0.3"); - - TVector<TIpv6Address> result; - Copy(range.begin(), range.end(), std::back_inserter(result)); - - bool ok; - TVector<TIpv6Address> expected { - TIpv6Address::FromString("10.0.0.0", ok), - TIpv6Address::FromString("10.0.0.1", ok), - TIpv6Address::FromString("10.0.0.2", ok), - TIpv6Address::FromString("10.0.0.3", ok), - }; - - ASSERT_THAT(result, ElementsAreArray(expected)); - } - { - auto range = TIpAddressRange::FromRangeString("10.0.0.0-10.0.0.3"); - - TVector<TIpv6Address> result; - Copy(range.begin(), range.end(), std::back_inserter(result)); - - bool ok; - TVector<TIpv6Address> expected { - TIpv6Address::FromString("10.0.0.0", ok), - TIpv6Address::FromString("10.0.0.1", ok), - TIpv6Address::FromString("10.0.0.2", ok), - TIpv6Address::FromString("10.0.0.3", ok), - }; - - ASSERT_THAT(result, ElementsAreArray(expected)); - } - } - - void IpRangeFromIpv4Builder() { - auto range = TIpAddressRange::From("192.168.0.0") - .To("192.168.0.255") - .Build(); - - ASSERT_THAT(range.Size(), Eq(256)); - } - - void IpRangeFromInvalidIpv4() { - auto build = [] (auto from, auto to) { - return TIpAddressRange::From(from).To(to).Build(); - }; - - ASSERT_THROW(build("192.168.0.255", "192.168.0.0"), yexception); - ASSERT_THROW(build("192.168.0.0", "192.168.0.300"), yexception); - ASSERT_THROW(build("192.168.0.300", "192.168.0.330"), yexception); - ASSERT_THROW(build("192.168.0.0", "::1"), yexception); - ASSERT_THROW(build(TIpv6Address{}, TIpv6Address{}), yexception); - } - - void IpRangeFromInvalidIpv6() { - auto build = [] (auto from, auto to) { - return TIpAddressRange::From(from).To(to).Build(); - }; - - ASSERT_THROW(build("ffce:abcd::00ff", "ffce:abcd::"), yexception); - ASSERT_THROW(build("ffce:abcd::", "ffce:abcd::fffff"), yexception); - ASSERT_THROW(build("ffce:abcd::10000", "ffce:abcd::ffff"), yexception); - ASSERT_THROW(build("ffce:abcd::", TIpv6Address{}), yexception); - - auto ctor = [] (auto s, auto e) { - return TIpAddressRange{s, e}; - }; - - ASSERT_THROW(ctor(TIpv6Address{}, TIpv6Address{}), yexception); - ASSERT_THROW(ctor("", ""), yexception); - } - - void ManualIteration() { - { - TIpAddressRange range{"::", "::"}; - auto it = range.Begin(); - ++it; - bool ok; - ASSERT_THAT(*it, Eq(TIpv6Address::FromString("::1", ok))); - - for (auto i = 0; i < 254; ++i, ++it) { - } - - ASSERT_THAT(*it, Eq(TIpv6Address::FromString("::ff", ok))); - } - - { - TIpAddressRange range{"0.0.0.0", "0.0.0.0"}; - auto it = range.Begin(); - ++it; - bool ok; - ASSERT_THAT(*it, Eq(TIpv6Address::FromString("0.0.0.1", ok))); - - for (auto i = 0; i < 254; ++i, ++it) { - } - - ASSERT_THAT(*it, Eq(TIpv6Address::FromString("0.0.0.255", ok))); - } - } - - void RangeRelations() { - { - auto range = TIpAddressRange::From(MIN_IPV6_ADDR) - .To(MAX_IPV6_ADDR) - .Build(); - - ASSERT_TRUE(range.Overlaps(range)); - ASSERT_TRUE(range.Contains(range)); - // XXX - //ASSERT_FALSE(range.IsConsecutive(range)); - } - { - auto range = TIpAddressRange::From("0.0.0.1").To("0.0.0.4").Build(); - auto range0 = TIpAddressRange::From("0.0.0.0").Build(); - auto range1 = TIpAddressRange::From("0.0.0.1").Build(); - auto range2 = TIpAddressRange::From("0.0.0.5").Build(); - auto range4 = TIpAddressRange::From("0.0.0.4").Build(); - - ASSERT_FALSE(range.Overlaps(range0)); - ASSERT_TRUE(range.IsConsecutive(range0)); - ASSERT_FALSE(range.Contains(range0)); - - ASSERT_TRUE(range.Overlaps(range1)); - ASSERT_FALSE(range.IsConsecutive(range1)); - ASSERT_TRUE(range.Contains(range1)); - - ASSERT_TRUE(range.Overlaps(range4)); - ASSERT_FALSE(range.IsConsecutive(range4)); - ASSERT_TRUE(range.Contains(range4)); - } - { - auto range = TIpAddressRange::From("0.0.0.1").To("0.0.0.4").Build(); - auto range2 = TIpAddressRange::From("0.0.0.0").To("0.0.0.2").Build(); - - ASSERT_TRUE(range.Overlaps(range2)); - ASSERT_FALSE(range.IsConsecutive(range2)); - ASSERT_FALSE(range.Contains(range2)); - - bool ok; - ASSERT_TRUE(range.Contains(TIpv6Address::FromString("0.0.0.1", ok))); - ASSERT_TRUE(range.Contains(TIpv6Address::FromString("0.0.0.2", ok))); - ASSERT_FALSE(range.Contains(TIpv6Address::FromString("0.0.0.5", ok))); - } - } - - void RangeUnion() { - { - auto range = TIpAddressRange::From(MIN_IPV6_ADDR) - .To(MAX_IPV6_ADDR) - .Build(); - - ASSERT_THAT(range.Union(range), Eq(range)); - ASSERT_THAT(range.Union(TIpAddressRange::From("::")), range); - ASSERT_THAT(range.Union(TIpAddressRange::From("::1")), range); - - ASSERT_THROW(range.Union(TIpAddressRange::From("0.0.0.0")), yexception); - } - - { - auto expected = TIpAddressRange::From("0.0.0.1").To("0.0.0.10").Build(); - - auto range = TIpAddressRange{"0.0.0.1", "0.0.0.3"}.Union({"0.0.0.4", "0.0.0.10"}); - ASSERT_THAT(range, Eq(expected)); - - auto range2 = TIpAddressRange{"0.0.0.1", "0.0.0.3"}.Union({"0.0.0.2", "0.0.0.10"}); - ASSERT_THAT(range2, Eq(expected)); - - auto range3 = TIpAddressRange{"0.0.0.2", "0.0.0.3"}.Union({"0.0.0.1", "0.0.0.10"}); - ASSERT_THAT(range2, Eq(expected)); - - auto range4 = TIpAddressRange{"0.0.0.1", "0.0.0.10"}.Union({"0.0.0.2", "0.0.0.3"}); - ASSERT_THAT(range2, Eq(expected)); - - ASSERT_THROW(range.Union(TIpAddressRange::From("10.0.0.0")), yexception); - } - } -}; - -class TRangeSetTests: public TTestBase { -public: - UNIT_TEST_SUITE(TRangeSetTests); - UNIT_TEST(AddDisjoint); - UNIT_TEST(AddOverlapping); - UNIT_TEST(AddConsecutive); - UNIT_TEST(DisallowsMixingTypes); - UNIT_TEST(MembershipTest); - UNIT_TEST_SUITE_END(); - - void AddDisjoint() { - TIpRangeSet set; - TVector<TIpAddressRange> expected { - TIpAddressRange::From("0.0.0.0").To("0.0.0.2").Build(), - TIpAddressRange::From("0.0.0.4").To("255.255.255.255").Build(), - }; - - for (auto&& r : expected) { - set.Add(r); - } - - ASSERT_THAT(set, ElementsAreArray(expected)); - } - - void TestAdding(const TVector<TIpAddressRange>& toInsert, const TVector<TIpAddressRange>& expected) { - TIpRangeSet set; - { - set.Add(toInsert); - - ASSERT_THAT(set, ElementsAreArray(expected)); - } - - { - for (auto it = toInsert.rbegin(); it != toInsert.rend(); ++it) { - set.Add(*it); - } - - ASSERT_THAT(set, ElementsAreArray(expected)); - } - } - - void AddOverlapping() { - { - TVector<TIpAddressRange> toInsert { - TIpAddressRange::From("0.0.0.0").To("0.0.0.2").Build(), - TIpAddressRange::From("0.0.0.2").To("0.0.0.4").Build(), - TIpAddressRange::From("0.0.0.4").To("255.255.255.255").Build(), - }; - - TVector<TIpAddressRange> expected { - TIpAddressRange::From("0.0.0.0").To("255.255.255.255").Build(), - }; - - TestAdding(toInsert, expected); - } - { - TVector<TIpAddressRange> toInsert { - TIpAddressRange::From("0.0.0.0").To("0.0.0.5").Build(), - TIpAddressRange::From("0.0.0.8").To("0.0.0.10").Build(), - TIpAddressRange::From("0.0.0.7").To("0.0.0.12").Build(), - }; - - TVector<TIpAddressRange> expected { - TIpAddressRange::From("0.0.0.0").To("0.0.0.5").Build(), - TIpAddressRange::From("0.0.0.7").To("0.0.0.12").Build(), - }; - - TestAdding(toInsert, expected); - } - { - TVector<TIpAddressRange> toInsert { - TIpAddressRange::From("0.0.0.0").To("0.0.0.5").Build(), - TIpAddressRange::From("0.0.0.7").To("0.0.0.12").Build(), - TIpAddressRange::From("0.0.0.8").To("0.0.0.10").Build(), - }; - - TVector<TIpAddressRange> expected { - TIpAddressRange::From("0.0.0.0").To("0.0.0.5").Build(), - TIpAddressRange::From("0.0.0.7").To("0.0.0.12").Build(), - }; - - TestAdding(toInsert, expected); - } - { - TVector<TIpAddressRange> toInsert { - TIpAddressRange::From("0.0.0.0").To("0.0.0.5").Build(), - TIpAddressRange::From("0.0.0.3").To("0.0.0.10").Build(), - }; - - TVector<TIpAddressRange> expected { - TIpAddressRange::From("0.0.0.0").To("0.0.0.10").Build(), - }; - - TestAdding(toInsert, expected); - } - } - - void DisallowsMixingTypes() { - TVector<TIpAddressRange> toInsert { - TIpAddressRange::From("0.0.0.0").To("0.0.0.5").Build(), - TIpAddressRange::From("::1").Build(), - }; - - TIpRangeSet rangeSet; - - ASSERT_THROW([&] { rangeSet.Add(toInsert); }(), yexception); - ASSERT_THROW([&] { rangeSet.Add(toInsert[1]); rangeSet.Add(toInsert[0]); }(), yexception); - } - - void AddConsecutive() { - { - TVector<TIpAddressRange> toInsert { - TIpAddressRange::From("0.0.0.0").To("0.0.0.5").Build(), - TIpAddressRange::From("0.0.0.6").To("0.0.0.7").Build(), - TIpAddressRange::From("0.0.0.8").To("0.0.0.10").Build(), - }; - - TVector<TIpAddressRange> expected { - TIpAddressRange::From("0.0.0.0").To("0.0.0.10").Build(), - }; - - TestAdding(toInsert, expected); - } - { - TVector<TIpAddressRange> toInsert { - TIpAddressRange::From("0.0.0.0").To("255.255.255.255").Build(), - TIpAddressRange::From("0.0.0.0").To("255.255.255.255").Build(), - }; - - TVector<TIpAddressRange> expected { - TIpAddressRange::From("0.0.0.0").To("255.255.255.255").Build(), - }; - - TestAdding(toInsert, expected); - } - } - - void MembershipTest() { - TVector<TIpAddressRange> toInsert { - TIpAddressRange::From("0.0.0.0").To("0.0.0.5").Build(), - TIpAddressRange::From("0.0.0.7").To("0.0.0.12").Build(), - TIpAddressRange::From("0.0.0.8").To("0.0.0.10").Build(), - }; - - TIpRangeSet rangeSet; - rangeSet.Add(toInsert); - - TVector<TIpAddressRange> in { - TIpAddressRange::From("0.0.0.0").To("0.0.0.5").Build(), - TIpAddressRange::From("0.0.0.7").To("0.0.0.12").Build(), - }; - - TVector<TIpAddressRange> notIn { - TIpAddressRange::From("0.0.0.6").Build(), - TIpAddressRange::From("0.0.0.13").To("0.0.0.255").Build(), - // enumerating full range is slow and makes little sense - TIpAddressRange::From("255.255.255.0").To("255.255.255.255").Build(), - }; - - for (auto&& range : in) { - for (auto&& addr : range) { - ASSERT_TRUE(rangeSet.Contains(addr)); - ASSERT_THAT(*rangeSet.Find(addr), Eq(range)); - } - } - - for (auto&& range : notIn) { - for (auto&& addr : range) { - ASSERT_FALSE(rangeSet.Contains(addr)); - ASSERT_THAT(rangeSet.Find(addr), rangeSet.End()); - } - } - - bool ok{}; - ASSERT_THAT(rangeSet.Find(TIpv6Address::FromString("::1", ok)), Eq(rangeSet.End())); - ASSERT_FALSE(rangeSet.Contains(TIpv6Address::FromString("::1", ok))); - } -}; - -UNIT_TEST_SUITE_REGISTRATION(TIpRangeTests); -UNIT_TEST_SUITE_REGISTRATION(TRangeSetTests); + +#include <library/cpp/ipv6_address/ipv6_address.h> + +using namespace testing; + +static constexpr auto MIN_IPV6_ADDR = "::"; +static constexpr auto MAX_IPV6_ADDR = "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff"; + +std::ostream& operator<<(std::ostream& os, const TIpAddressRange& r) { + auto s = r.ToRangeString(); + os.write(s.data(), s.size()); + return os; +} + +std::ostream& operator<<(std::ostream& os, const TIpRangeSet& rangeSet) { + os << "{\n"; + + for (auto&& r : rangeSet) { + os << r << '\n'; + } + + os << "}\n"; + return os; +} + +class TIpRangeTests: public TTestBase { +public: + UNIT_TEST_SUITE(TIpRangeTests); + UNIT_TEST(IpRangeFromIpv4); + UNIT_TEST(IpRangeFromIpv6); + UNIT_TEST(FullIpRange); + UNIT_TEST(IpRangeFromCidr); + UNIT_TEST(IpRangeFromIpv4Builder); + UNIT_TEST(IpRangeFromInvalidIpv4); + UNIT_TEST(IpRangeFromInvalidIpv6); + UNIT_TEST(RangeFromSingleAddress); + UNIT_TEST(RangeFromRangeString); + UNIT_TEST(ManualIteration); + UNIT_TEST(RangeRelations); + UNIT_TEST(RangeUnion); + UNIT_TEST_SUITE_END(); + + void RangeFromSingleAddress() { + for (auto addrStr : {"192.168.0.1", "::2"}) { + auto range = TIpAddressRange::From(addrStr).Build(); + ASSERT_THAT(range.Size(), Eq(1)); + ASSERT_TRUE(range.IsSingle()); + + auto range2 = TIpAddressRange{addrStr, addrStr}; + ASSERT_THAT(range2, Eq(range)); + + TVector<ui128> result; + range.ForEach([&result] (TIpv6Address addr) { + result.push_back(addr); + }); + + bool ok{}; + ASSERT_THAT(result, ElementsAre(TIpv6Address::FromString(addrStr, ok))); + } + + } + + void IpRangeFromIpv4() { + bool ok{}; + + auto s = TIpv6Address::FromString("192.168.0.0", ok); + ASSERT_TRUE(ok); + auto e = TIpv6Address::FromString("192.168.0.255", ok); + ASSERT_TRUE(ok); + + TIpAddressRange range{s, e}; + + ASSERT_THAT(range.Size(), Eq(256)); + ASSERT_THAT(range.Type(), Eq(TIpAddressRange::TIpType::Ipv4)); + + TIpAddressRange range2{"192.168.0.0", "192.168.0.255"}; + ASSERT_THAT(range2.Size(), Eq(256)); + ASSERT_THAT(range2, Eq(range)); + } + + void IpRangeFromIpv6() { + bool ok{}; + + auto s = TIpv6Address::FromString("ffce:abcd::", ok); + ASSERT_TRUE(ok); + auto e = TIpv6Address::FromString("ffce:abcd::00ff", ok); + ASSERT_TRUE(ok); + + TIpAddressRange range{s, e}; + + ASSERT_THAT(range.Size(), Eq(256)); + + TIpAddressRange range2{"ffce:abcd::", "ffce:abcd::00ff"}; + ASSERT_THAT(range2.Size(), Eq(256)); + ASSERT_THAT(range.Type(), Eq(TIpAddressRange::TIpType::Ipv6)); + ASSERT_THAT(range2, Eq(range)); + } + + + void FullIpRange() { + auto check = [] (auto start, auto end, ui128 expectedSize) { + auto range = TIpAddressRange::From(start).To(end).Build(); + ASSERT_THAT(range.Size(), Eq(expectedSize)); + }; + + check("0.0.0.0", "255.255.255.255", ui128(Max<ui32>()) + 1); + // XXX + // check(MIN_IPV6_ADDR, MAX_IPV6_ADDR, ui128(Max<ui128>() + 1)); + } + + void IpRangeFromCidr() { + auto range = TIpAddressRange::FromCidrString("10.0.0.0/30"); + + ASSERT_THAT(range.Size(), Eq(4)); + TVector<TIpv6Address> result; + Copy(range.begin(), range.end(), std::back_inserter(result)); + + bool ok; + TVector<TIpv6Address> expected { + TIpv6Address::FromString("10.0.0.0", ok), + TIpv6Address::FromString("10.0.0.1", ok), + TIpv6Address::FromString("10.0.0.2", ok), + TIpv6Address::FromString("10.0.0.3", ok), + }; + + ASSERT_THAT(result, ElementsAreArray(expected)); + + // single host + ASSERT_THAT(TIpAddressRange::FromCidrString("ffce:abcd::/128"), Eq(TIpAddressRange::From("ffce:abcd::").Build())); + ASSERT_THAT(TIpAddressRange::FromCidrString("192.168.0.1/32"), Eq(TIpAddressRange::From("192.168.0.1").Build())); + + // full range + ASSERT_THAT(TIpAddressRange::FromCidrString("::/0"), Eq(TIpAddressRange::From(MIN_IPV6_ADDR).To(MAX_IPV6_ADDR).Build())); + ASSERT_THAT(TIpAddressRange::FromCidrString("0.0.0.0/0"), Eq(TIpAddressRange::From("0.0.0.0").To("255.255.255.255").Build())); + + // illformed + ASSERT_THROW(TIpAddressRange::FromCidrString("::/"), TInvalidIpRangeException); + ASSERT_THROW(TIpAddressRange::FromCidrString("::"), TInvalidIpRangeException); + ASSERT_THROW(TIpAddressRange::FromCidrString("/::"), TInvalidIpRangeException); + ASSERT_THROW(TIpAddressRange::FromCidrString("::/150"), TInvalidIpRangeException); + } + + void RangeFromRangeString() { + { + auto range = TIpAddressRange::FromRangeString("10.0.0.0-10.0.0.3"); + + TVector<TIpv6Address> result; + Copy(range.begin(), range.end(), std::back_inserter(result)); + + bool ok; + TVector<TIpv6Address> expected { + TIpv6Address::FromString("10.0.0.0", ok), + TIpv6Address::FromString("10.0.0.1", ok), + TIpv6Address::FromString("10.0.0.2", ok), + TIpv6Address::FromString("10.0.0.3", ok), + }; + + ASSERT_THAT(result, ElementsAreArray(expected)); + } + { + auto range = TIpAddressRange::FromRangeString("10.0.0.0-10.0.0.3"); + + TVector<TIpv6Address> result; + Copy(range.begin(), range.end(), std::back_inserter(result)); + + bool ok; + TVector<TIpv6Address> expected { + TIpv6Address::FromString("10.0.0.0", ok), + TIpv6Address::FromString("10.0.0.1", ok), + TIpv6Address::FromString("10.0.0.2", ok), + TIpv6Address::FromString("10.0.0.3", ok), + }; + + ASSERT_THAT(result, ElementsAreArray(expected)); + } + } + + void IpRangeFromIpv4Builder() { + auto range = TIpAddressRange::From("192.168.0.0") + .To("192.168.0.255") + .Build(); + + ASSERT_THAT(range.Size(), Eq(256)); + } + + void IpRangeFromInvalidIpv4() { + auto build = [] (auto from, auto to) { + return TIpAddressRange::From(from).To(to).Build(); + }; + + ASSERT_THROW(build("192.168.0.255", "192.168.0.0"), yexception); + ASSERT_THROW(build("192.168.0.0", "192.168.0.300"), yexception); + ASSERT_THROW(build("192.168.0.300", "192.168.0.330"), yexception); + ASSERT_THROW(build("192.168.0.0", "::1"), yexception); + ASSERT_THROW(build(TIpv6Address{}, TIpv6Address{}), yexception); + } + + void IpRangeFromInvalidIpv6() { + auto build = [] (auto from, auto to) { + return TIpAddressRange::From(from).To(to).Build(); + }; + + ASSERT_THROW(build("ffce:abcd::00ff", "ffce:abcd::"), yexception); + ASSERT_THROW(build("ffce:abcd::", "ffce:abcd::fffff"), yexception); + ASSERT_THROW(build("ffce:abcd::10000", "ffce:abcd::ffff"), yexception); + ASSERT_THROW(build("ffce:abcd::", TIpv6Address{}), yexception); + + auto ctor = [] (auto s, auto e) { + return TIpAddressRange{s, e}; + }; + + ASSERT_THROW(ctor(TIpv6Address{}, TIpv6Address{}), yexception); + ASSERT_THROW(ctor("", ""), yexception); + } + + void ManualIteration() { + { + TIpAddressRange range{"::", "::"}; + auto it = range.Begin(); + ++it; + bool ok; + ASSERT_THAT(*it, Eq(TIpv6Address::FromString("::1", ok))); + + for (auto i = 0; i < 254; ++i, ++it) { + } + + ASSERT_THAT(*it, Eq(TIpv6Address::FromString("::ff", ok))); + } + + { + TIpAddressRange range{"0.0.0.0", "0.0.0.0"}; + auto it = range.Begin(); + ++it; + bool ok; + ASSERT_THAT(*it, Eq(TIpv6Address::FromString("0.0.0.1", ok))); + + for (auto i = 0; i < 254; ++i, ++it) { + } + + ASSERT_THAT(*it, Eq(TIpv6Address::FromString("0.0.0.255", ok))); + } + } + + void RangeRelations() { + { + auto range = TIpAddressRange::From(MIN_IPV6_ADDR) + .To(MAX_IPV6_ADDR) + .Build(); + + ASSERT_TRUE(range.Overlaps(range)); + ASSERT_TRUE(range.Contains(range)); + // XXX + //ASSERT_FALSE(range.IsConsecutive(range)); + } + { + auto range = TIpAddressRange::From("0.0.0.1").To("0.0.0.4").Build(); + auto range0 = TIpAddressRange::From("0.0.0.0").Build(); + auto range1 = TIpAddressRange::From("0.0.0.1").Build(); + auto range2 = TIpAddressRange::From("0.0.0.5").Build(); + auto range4 = TIpAddressRange::From("0.0.0.4").Build(); + + ASSERT_FALSE(range.Overlaps(range0)); + ASSERT_TRUE(range.IsConsecutive(range0)); + ASSERT_FALSE(range.Contains(range0)); + + ASSERT_TRUE(range.Overlaps(range1)); + ASSERT_FALSE(range.IsConsecutive(range1)); + ASSERT_TRUE(range.Contains(range1)); + + ASSERT_TRUE(range.Overlaps(range4)); + ASSERT_FALSE(range.IsConsecutive(range4)); + ASSERT_TRUE(range.Contains(range4)); + } + { + auto range = TIpAddressRange::From("0.0.0.1").To("0.0.0.4").Build(); + auto range2 = TIpAddressRange::From("0.0.0.0").To("0.0.0.2").Build(); + + ASSERT_TRUE(range.Overlaps(range2)); + ASSERT_FALSE(range.IsConsecutive(range2)); + ASSERT_FALSE(range.Contains(range2)); + + bool ok; + ASSERT_TRUE(range.Contains(TIpv6Address::FromString("0.0.0.1", ok))); + ASSERT_TRUE(range.Contains(TIpv6Address::FromString("0.0.0.2", ok))); + ASSERT_FALSE(range.Contains(TIpv6Address::FromString("0.0.0.5", ok))); + } + } + + void RangeUnion() { + { + auto range = TIpAddressRange::From(MIN_IPV6_ADDR) + .To(MAX_IPV6_ADDR) + .Build(); + + ASSERT_THAT(range.Union(range), Eq(range)); + ASSERT_THAT(range.Union(TIpAddressRange::From("::")), range); + ASSERT_THAT(range.Union(TIpAddressRange::From("::1")), range); + + ASSERT_THROW(range.Union(TIpAddressRange::From("0.0.0.0")), yexception); + } + + { + auto expected = TIpAddressRange::From("0.0.0.1").To("0.0.0.10").Build(); + + auto range = TIpAddressRange{"0.0.0.1", "0.0.0.3"}.Union({"0.0.0.4", "0.0.0.10"}); + ASSERT_THAT(range, Eq(expected)); + + auto range2 = TIpAddressRange{"0.0.0.1", "0.0.0.3"}.Union({"0.0.0.2", "0.0.0.10"}); + ASSERT_THAT(range2, Eq(expected)); + + auto range3 = TIpAddressRange{"0.0.0.2", "0.0.0.3"}.Union({"0.0.0.1", "0.0.0.10"}); + ASSERT_THAT(range2, Eq(expected)); + + auto range4 = TIpAddressRange{"0.0.0.1", "0.0.0.10"}.Union({"0.0.0.2", "0.0.0.3"}); + ASSERT_THAT(range2, Eq(expected)); + + ASSERT_THROW(range.Union(TIpAddressRange::From("10.0.0.0")), yexception); + } + } +}; + +class TRangeSetTests: public TTestBase { +public: + UNIT_TEST_SUITE(TRangeSetTests); + UNIT_TEST(AddDisjoint); + UNIT_TEST(AddOverlapping); + UNIT_TEST(AddConsecutive); + UNIT_TEST(DisallowsMixingTypes); + UNIT_TEST(MembershipTest); + UNIT_TEST_SUITE_END(); + + void AddDisjoint() { + TIpRangeSet set; + TVector<TIpAddressRange> expected { + TIpAddressRange::From("0.0.0.0").To("0.0.0.2").Build(), + TIpAddressRange::From("0.0.0.4").To("255.255.255.255").Build(), + }; + + for (auto&& r : expected) { + set.Add(r); + } + + ASSERT_THAT(set, ElementsAreArray(expected)); + } + + void TestAdding(const TVector<TIpAddressRange>& toInsert, const TVector<TIpAddressRange>& expected) { + TIpRangeSet set; + { + set.Add(toInsert); + + ASSERT_THAT(set, ElementsAreArray(expected)); + } + + { + for (auto it = toInsert.rbegin(); it != toInsert.rend(); ++it) { + set.Add(*it); + } + + ASSERT_THAT(set, ElementsAreArray(expected)); + } + } + + void AddOverlapping() { + { + TVector<TIpAddressRange> toInsert { + TIpAddressRange::From("0.0.0.0").To("0.0.0.2").Build(), + TIpAddressRange::From("0.0.0.2").To("0.0.0.4").Build(), + TIpAddressRange::From("0.0.0.4").To("255.255.255.255").Build(), + }; + + TVector<TIpAddressRange> expected { + TIpAddressRange::From("0.0.0.0").To("255.255.255.255").Build(), + }; + + TestAdding(toInsert, expected); + } + { + TVector<TIpAddressRange> toInsert { + TIpAddressRange::From("0.0.0.0").To("0.0.0.5").Build(), + TIpAddressRange::From("0.0.0.8").To("0.0.0.10").Build(), + TIpAddressRange::From("0.0.0.7").To("0.0.0.12").Build(), + }; + + TVector<TIpAddressRange> expected { + TIpAddressRange::From("0.0.0.0").To("0.0.0.5").Build(), + TIpAddressRange::From("0.0.0.7").To("0.0.0.12").Build(), + }; + + TestAdding(toInsert, expected); + } + { + TVector<TIpAddressRange> toInsert { + TIpAddressRange::From("0.0.0.0").To("0.0.0.5").Build(), + TIpAddressRange::From("0.0.0.7").To("0.0.0.12").Build(), + TIpAddressRange::From("0.0.0.8").To("0.0.0.10").Build(), + }; + + TVector<TIpAddressRange> expected { + TIpAddressRange::From("0.0.0.0").To("0.0.0.5").Build(), + TIpAddressRange::From("0.0.0.7").To("0.0.0.12").Build(), + }; + + TestAdding(toInsert, expected); + } + { + TVector<TIpAddressRange> toInsert { + TIpAddressRange::From("0.0.0.0").To("0.0.0.5").Build(), + TIpAddressRange::From("0.0.0.3").To("0.0.0.10").Build(), + }; + + TVector<TIpAddressRange> expected { + TIpAddressRange::From("0.0.0.0").To("0.0.0.10").Build(), + }; + + TestAdding(toInsert, expected); + } + } + + void DisallowsMixingTypes() { + TVector<TIpAddressRange> toInsert { + TIpAddressRange::From("0.0.0.0").To("0.0.0.5").Build(), + TIpAddressRange::From("::1").Build(), + }; + + TIpRangeSet rangeSet; + + ASSERT_THROW([&] { rangeSet.Add(toInsert); }(), yexception); + ASSERT_THROW([&] { rangeSet.Add(toInsert[1]); rangeSet.Add(toInsert[0]); }(), yexception); + } + + void AddConsecutive() { + { + TVector<TIpAddressRange> toInsert { + TIpAddressRange::From("0.0.0.0").To("0.0.0.5").Build(), + TIpAddressRange::From("0.0.0.6").To("0.0.0.7").Build(), + TIpAddressRange::From("0.0.0.8").To("0.0.0.10").Build(), + }; + + TVector<TIpAddressRange> expected { + TIpAddressRange::From("0.0.0.0").To("0.0.0.10").Build(), + }; + + TestAdding(toInsert, expected); + } + { + TVector<TIpAddressRange> toInsert { + TIpAddressRange::From("0.0.0.0").To("255.255.255.255").Build(), + TIpAddressRange::From("0.0.0.0").To("255.255.255.255").Build(), + }; + + TVector<TIpAddressRange> expected { + TIpAddressRange::From("0.0.0.0").To("255.255.255.255").Build(), + }; + + TestAdding(toInsert, expected); + } + } + + void MembershipTest() { + TVector<TIpAddressRange> toInsert { + TIpAddressRange::From("0.0.0.0").To("0.0.0.5").Build(), + TIpAddressRange::From("0.0.0.7").To("0.0.0.12").Build(), + TIpAddressRange::From("0.0.0.8").To("0.0.0.10").Build(), + }; + + TIpRangeSet rangeSet; + rangeSet.Add(toInsert); + + TVector<TIpAddressRange> in { + TIpAddressRange::From("0.0.0.0").To("0.0.0.5").Build(), + TIpAddressRange::From("0.0.0.7").To("0.0.0.12").Build(), + }; + + TVector<TIpAddressRange> notIn { + TIpAddressRange::From("0.0.0.6").Build(), + TIpAddressRange::From("0.0.0.13").To("0.0.0.255").Build(), + // enumerating full range is slow and makes little sense + TIpAddressRange::From("255.255.255.0").To("255.255.255.255").Build(), + }; + + for (auto&& range : in) { + for (auto&& addr : range) { + ASSERT_TRUE(rangeSet.Contains(addr)); + ASSERT_THAT(*rangeSet.Find(addr), Eq(range)); + } + } + + for (auto&& range : notIn) { + for (auto&& addr : range) { + ASSERT_FALSE(rangeSet.Contains(addr)); + ASSERT_THAT(rangeSet.Find(addr), rangeSet.End()); + } + } + + bool ok{}; + ASSERT_THAT(rangeSet.Find(TIpv6Address::FromString("::1", ok)), Eq(rangeSet.End())); + ASSERT_FALSE(rangeSet.Contains(TIpv6Address::FromString("::1", ok))); + } +}; + +UNIT_TEST_SUITE_REGISTRATION(TIpRangeTests); +UNIT_TEST_SUITE_REGISTRATION(TRangeSetTests); diff --git a/library/cpp/ipmath/range_set.cpp b/library/cpp/ipmath/range_set.cpp index 503ccaf03f..55f42e451d 100644 --- a/library/cpp/ipmath/range_set.cpp +++ b/library/cpp/ipmath/range_set.cpp @@ -1,99 +1,99 @@ -#include "range_set.h" - -#include <util/generic/algorithm.h> - -namespace { - bool ShouldJoin(const TIpAddressRange& lhs, const TIpAddressRange& rhs) { - return lhs.Overlaps(rhs) || lhs.IsConsecutive(rhs); - } -} - -bool TIpRangeSet::TRangeLess::operator()(const TIpAddressRange& lhs, const TIpAddressRange& rhs) const { - return *lhs.Begin() < *rhs.Begin(); -} - -TIpRangeSet::TIpRangeSet() = default; -TIpRangeSet::~TIpRangeSet() = default; - -void TIpRangeSet::Add(TIpAddressRange r) { - Y_ENSURE(IsEmpty() || r.Type() == Type(), "Mixing IPv4 and IPv6 ranges is disallowed"); - +#include "range_set.h" + +#include <util/generic/algorithm.h> + +namespace { + bool ShouldJoin(const TIpAddressRange& lhs, const TIpAddressRange& rhs) { + return lhs.Overlaps(rhs) || lhs.IsConsecutive(rhs); + } +} + +bool TIpRangeSet::TRangeLess::operator()(const TIpAddressRange& lhs, const TIpAddressRange& rhs) const { + return *lhs.Begin() < *rhs.Begin(); +} + +TIpRangeSet::TIpRangeSet() = default; +TIpRangeSet::~TIpRangeSet() = default; + +void TIpRangeSet::Add(TIpAddressRange r) { + Y_ENSURE(IsEmpty() || r.Type() == Type(), "Mixing IPv4 and IPv6 ranges is disallowed"); + auto lowerIt = Ranges_.lower_bound(r); - - // still may overlap the last interval in our tree - if (IsEmpty()) { - Ranges_.insert(r); - return; - } else if (lowerIt == Ranges_.end()) { - if (auto it = Ranges_.rbegin(); ShouldJoin(*it, r)) { - auto unitedRange = it->Union(r); - Ranges_.erase(--it.base()); - Ranges_.insert(unitedRange); - } else { - Ranges_.insert(r); - } - - return; - } - - - TIpAddressRange unitedRange{r}; - - auto joined = lowerIt; - if (lowerIt != Ranges_.begin()) { - if (ShouldJoin(unitedRange, *(--joined))) { - unitedRange = unitedRange.Union(*joined); - } else { - ++joined; - } - } - - auto it = lowerIt; - for (; it != Ranges_.end() && ShouldJoin(*it, unitedRange); ++it) { - unitedRange = unitedRange.Union(*it); - } - - Ranges_.erase(joined, it); - Ranges_.insert(unitedRange); -} - -TIpAddressRange::TIpType TIpRangeSet::Type() const { - return IsEmpty() - ? TIpAddressRange::TIpType::LAST - : Ranges_.begin()->Type(); -} - -bool TIpRangeSet::IsEmpty() const { - return Ranges_.empty(); -} - -TIpRangeSet::TIterator TIpRangeSet::Find(TIpv6Address addr) const { - if (IsEmpty() || addr.Type() != Type()) { - return End(); - } - + + // still may overlap the last interval in our tree + if (IsEmpty()) { + Ranges_.insert(r); + return; + } else if (lowerIt == Ranges_.end()) { + if (auto it = Ranges_.rbegin(); ShouldJoin(*it, r)) { + auto unitedRange = it->Union(r); + Ranges_.erase(--it.base()); + Ranges_.insert(unitedRange); + } else { + Ranges_.insert(r); + } + + return; + } + + + TIpAddressRange unitedRange{r}; + + auto joined = lowerIt; + if (lowerIt != Ranges_.begin()) { + if (ShouldJoin(unitedRange, *(--joined))) { + unitedRange = unitedRange.Union(*joined); + } else { + ++joined; + } + } + + auto it = lowerIt; + for (; it != Ranges_.end() && ShouldJoin(*it, unitedRange); ++it) { + unitedRange = unitedRange.Union(*it); + } + + Ranges_.erase(joined, it); + Ranges_.insert(unitedRange); +} + +TIpAddressRange::TIpType TIpRangeSet::Type() const { + return IsEmpty() + ? TIpAddressRange::TIpType::LAST + : Ranges_.begin()->Type(); +} + +bool TIpRangeSet::IsEmpty() const { + return Ranges_.empty(); +} + +TIpRangeSet::TIterator TIpRangeSet::Find(TIpv6Address addr) const { + if (IsEmpty() || addr.Type() != Type()) { + return End(); + } + auto lowerIt = Ranges_.lower_bound(TIpAddressRange(addr, addr)); - - if (lowerIt == Ranges_.begin()) { - return lowerIt->Contains(addr) - ? lowerIt - : End(); - } else if (lowerIt == Ranges_.end()) { - auto rbegin = Ranges_.crbegin(); - return rbegin->Contains(addr) - ? (++rbegin).base() - : End(); - } else if (lowerIt->Contains(addr)) { - return lowerIt; - } - - --lowerIt; - - return lowerIt->Contains(addr) - ? lowerIt - : End(); -} - -bool TIpRangeSet::Contains(TIpv6Address addr) const { - return Find(addr) != End(); -} + + if (lowerIt == Ranges_.begin()) { + return lowerIt->Contains(addr) + ? lowerIt + : End(); + } else if (lowerIt == Ranges_.end()) { + auto rbegin = Ranges_.crbegin(); + return rbegin->Contains(addr) + ? (++rbegin).base() + : End(); + } else if (lowerIt->Contains(addr)) { + return lowerIt; + } + + --lowerIt; + + return lowerIt->Contains(addr) + ? lowerIt + : End(); +} + +bool TIpRangeSet::Contains(TIpv6Address addr) const { + return Find(addr) != End(); +} diff --git a/library/cpp/ipmath/range_set.h b/library/cpp/ipmath/range_set.h index 23a648fb86..d9e2451822 100644 --- a/library/cpp/ipmath/range_set.h +++ b/library/cpp/ipmath/range_set.h @@ -1,66 +1,66 @@ -#pragma once - -#include "ipmath.h" - -#include <util/generic/set.h> -#include <util/ysaveload.h> - - -/// @brief Maintains a disjoint set of added ranges. Allows for efficient membership queries -/// for an address in a set of IP ranges. -class TIpRangeSet { - struct TRangeLess { - bool operator()(const TIpAddressRange& lhs, const TIpAddressRange& rhs) const; - }; - - using TTree = TSet<TIpAddressRange, TRangeLess>; - -public: - using iterator = TTree::iterator; - using const_iterator = TTree::const_iterator; - using value_type = TTree::value_type; - using TIterator = TTree::iterator; - using TConstIterator = TTree::const_iterator; - - TIpRangeSet(); - ~TIpRangeSet(); - - void Add(TIpAddressRange range); - - template <typename TContainer> - void Add(TContainer&& addrs) { - using T = typename std::decay<TContainer>::type::value_type; - static_assert(std::is_convertible<T, TIpAddressRange>::value); - - for (auto&& addr : addrs) { - Add(addr); - } - } - - TIpAddressRange::TIpType Type() const; - - bool IsEmpty() const; - bool Contains(TIpv6Address addr) const; - TConstIterator Find(TIpv6Address addr) const; - - TConstIterator Begin() const { - return Ranges_.begin(); - } - - TConstIterator End() const { - return Ranges_.end(); - } - - TConstIterator begin() const { - return Begin(); - } - - TConstIterator end() const { - return End(); - } - - Y_SAVELOAD_DEFINE(Ranges_); - -private: - TTree Ranges_; -}; +#pragma once + +#include "ipmath.h" + +#include <util/generic/set.h> +#include <util/ysaveload.h> + + +/// @brief Maintains a disjoint set of added ranges. Allows for efficient membership queries +/// for an address in a set of IP ranges. +class TIpRangeSet { + struct TRangeLess { + bool operator()(const TIpAddressRange& lhs, const TIpAddressRange& rhs) const; + }; + + using TTree = TSet<TIpAddressRange, TRangeLess>; + +public: + using iterator = TTree::iterator; + using const_iterator = TTree::const_iterator; + using value_type = TTree::value_type; + using TIterator = TTree::iterator; + using TConstIterator = TTree::const_iterator; + + TIpRangeSet(); + ~TIpRangeSet(); + + void Add(TIpAddressRange range); + + template <typename TContainer> + void Add(TContainer&& addrs) { + using T = typename std::decay<TContainer>::type::value_type; + static_assert(std::is_convertible<T, TIpAddressRange>::value); + + for (auto&& addr : addrs) { + Add(addr); + } + } + + TIpAddressRange::TIpType Type() const; + + bool IsEmpty() const; + bool Contains(TIpv6Address addr) const; + TConstIterator Find(TIpv6Address addr) const; + + TConstIterator Begin() const { + return Ranges_.begin(); + } + + TConstIterator End() const { + return Ranges_.end(); + } + + TConstIterator begin() const { + return Begin(); + } + + TConstIterator end() const { + return End(); + } + + Y_SAVELOAD_DEFINE(Ranges_); + +private: + TTree Ranges_; +}; diff --git a/library/cpp/ipmath/ut/ya.make b/library/cpp/ipmath/ut/ya.make index 1593646ade..b860cefd03 100644 --- a/library/cpp/ipmath/ut/ya.make +++ b/library/cpp/ipmath/ut/ya.make @@ -1,16 +1,16 @@ -UNITTEST_FOR(library/cpp/ipmath) - -OWNER( - msherbakov - g:solomon -) - -SRCS( - ipmath_ut.cpp -) - -PEERDIR( +UNITTEST_FOR(library/cpp/ipmath) + +OWNER( + msherbakov + g:solomon +) + +SRCS( + ipmath_ut.cpp +) + +PEERDIR( library/cpp/testing/gmock_in_unittest -) - -END() +) + +END() diff --git a/library/cpp/ipmath/ya.make b/library/cpp/ipmath/ya.make index a9f75d6cb7..244838962e 100644 --- a/library/cpp/ipmath/ya.make +++ b/library/cpp/ipmath/ya.make @@ -1,17 +1,17 @@ -LIBRARY() - -OWNER( - msherbakov - g:solomon -) - -SRCS( - ipmath.cpp - range_set.cpp -) - -PEERDIR(library/cpp/ipv6_address) - -END() - -RECURSE_FOR_TESTS(ut) +LIBRARY() + +OWNER( + msherbakov + g:solomon +) + +SRCS( + ipmath.cpp + range_set.cpp +) + +PEERDIR(library/cpp/ipv6_address) + +END() + +RECURSE_FOR_TESTS(ut) diff --git a/library/cpp/json/writer/json.cpp b/library/cpp/json/writer/json.cpp index fcc2980f9b..02370c2d79 100644 --- a/library/cpp/json/writer/json.cpp +++ b/library/cpp/json/writer/json.cpp @@ -221,7 +221,7 @@ namespace NJsonWriter { TValueContext TBuf::WriteBool(bool b) { constexpr TStringBuf trueVal = "true"; constexpr TStringBuf falseVal = "false"; - UnsafeWriteValue(b ? trueVal : falseVal); + UnsafeWriteValue(b ? trueVal : falseVal); return TValueContext(*this); } diff --git a/library/cpp/json/writer/json_ut.cpp b/library/cpp/json/writer/json_ut.cpp index d5ee4ff1c3..9980555683 100644 --- a/library/cpp/json/writer/json_ut.cpp +++ b/library/cpp/json/writer/json_ut.cpp @@ -1,5 +1,5 @@ #include <library/cpp/testing/unittest/registar.h> -#include <util/system/sanitizers.h> +#include <util/system/sanitizers.h> #include "json.h" #include <library/cpp/json/json_value.h> @@ -265,43 +265,43 @@ Y_UNIT_TEST_SUITE(JsonWriter) { UNIT_ASSERT_STRINGS_EQUAL(buf.Str(), R"({"\u003C\u003E&":"Ololo","<>&":"Ololo2"})"); } } - - Y_UNIT_TEST(WriteUninitializedBoolDoesntCrashProgram) { - // makes sense only in release build w/ address sanitizer - // - // passing uninitialized boolean into WriteBool can make cleverly optimized code which is emitted by compiler crash program - // https://stackoverflow.com/questions/54120862/does-the-c-standard-allow-for-an-uninitialized-bool-to-crash-a-program - - // looks like compiler can detect UB at compile time in simple cases, but not in this one - class TSensorConf { - public: - class TAggrRuleItem { - public: - TVector<TString> Cond; - TVector<TString> Target; - }; - - TString ToString() const { - NJson::TJsonValue jsonValue; - NJsonWriter::TBuf jsonOutput; - jsonOutput.BeginObject() - .WriteKey("rawDataMemOnly").WriteBool(RawDataMemOnly) - .WriteKey("aggrRules").BeginList(); - - jsonOutput.EndList() - .EndObject(); - - return jsonOutput.Str(); - } - - TVector<TAggrRuleItem> AggrRules; - bool RawDataMemOnly; - }; - - TSensorConf s; - NSan::Unpoison(&s.RawDataMemOnly, sizeof(s.RawDataMemOnly)); - auto p = s.ToString(); - // doesn't really matter - UNIT_ASSERT(!p.empty()); - } + + Y_UNIT_TEST(WriteUninitializedBoolDoesntCrashProgram) { + // makes sense only in release build w/ address sanitizer + // + // passing uninitialized boolean into WriteBool can make cleverly optimized code which is emitted by compiler crash program + // https://stackoverflow.com/questions/54120862/does-the-c-standard-allow-for-an-uninitialized-bool-to-crash-a-program + + // looks like compiler can detect UB at compile time in simple cases, but not in this one + class TSensorConf { + public: + class TAggrRuleItem { + public: + TVector<TString> Cond; + TVector<TString> Target; + }; + + TString ToString() const { + NJson::TJsonValue jsonValue; + NJsonWriter::TBuf jsonOutput; + jsonOutput.BeginObject() + .WriteKey("rawDataMemOnly").WriteBool(RawDataMemOnly) + .WriteKey("aggrRules").BeginList(); + + jsonOutput.EndList() + .EndObject(); + + return jsonOutput.Str(); + } + + TVector<TAggrRuleItem> AggrRules; + bool RawDataMemOnly; + }; + + TSensorConf s; + NSan::Unpoison(&s.RawDataMemOnly, sizeof(s.RawDataMemOnly)); + auto p = s.ToString(); + // doesn't really matter + UNIT_ASSERT(!p.empty()); + } } diff --git a/library/cpp/lwtrace/mon/mon_lwtrace.cpp b/library/cpp/lwtrace/mon/mon_lwtrace.cpp index ffa7421016..a61ee9ce22 100644 --- a/library/cpp/lwtrace/mon/mon_lwtrace.cpp +++ b/library/cpp/lwtrace/mon/mon_lwtrace.cpp @@ -3789,7 +3789,7 @@ public: StartTime = CTimeR(&stime); } - virtual void Output(NMonitoring::IMonHttpRequest& request) { + virtual void Output(NMonitoring::IMonHttpRequest& request) { TStringStream out; try { if (request.GetParams().Get("mode") == "") { @@ -3836,11 +3836,11 @@ public: } } } - request.Output() << out.Str(); + request.Output() << out.Str(); } private: - void OutputNavbar(const NMonitoring::IMonHttpRequest& request, IOutputStream& out) + void OutputNavbar(const NMonitoring::IMonHttpRequest& request, IOutputStream& out) { TString active = " class=\"active\""; out << @@ -3867,7 +3867,7 @@ private: } } - void OutputTracesAndSnapshots(const NMonitoring::IMonHttpRequest& request, IOutputStream& out) + void OutputTracesAndSnapshots(const NMonitoring::IMonHttpRequest& request, IOutputStream& out) { TLogSources logSources(Cleaner); TraceMngr->ReadTraces(logSources); @@ -3890,7 +3890,7 @@ private: } } - void OutputProbes(const NMonitoring::IMonHttpRequest& request, IOutputStream& out) + void OutputProbes(const NMonitoring::IMonHttpRequest& request, IOutputStream& out) { TStringStream ss; TProbesHtmlPrinter printer; @@ -3983,7 +3983,7 @@ private: } } - void OutputLog(const NMonitoring::IMonHttpRequest& request, IOutputStream& out) + void OutputLog(const NMonitoring::IMonHttpRequest& request, IOutputStream& out) { if (request.GetParams().NumOfValues("id") == 0) { ythrow yexception() << "Cgi-parameter 'id' is not specified"; @@ -4052,7 +4052,7 @@ private: } } - void OutputQuery(const NMonitoring::IMonHttpRequest& request, IOutputStream& out) + void OutputQuery(const NMonitoring::IMonHttpRequest& request, IOutputStream& out) { if (request.GetParams().NumOfValues("id") == 0) { ythrow yexception() << "Cgi-parameter 'id' is not specified"; @@ -4066,7 +4066,7 @@ private: } } - void OutputBuilder(const NMonitoring::IMonHttpRequest& request, IOutputStream& out) + void OutputBuilder(const NMonitoring::IMonHttpRequest& request, IOutputStream& out) { Y_UNUSED(request); WWW_HTML(out) { @@ -4093,7 +4093,7 @@ private: } } - void OutputAnalytics(const NMonitoring::IMonHttpRequest& request, TStringStream& out) + void OutputAnalytics(const NMonitoring::IMonHttpRequest& request, TStringStream& out) { using namespace NAnalytics; const TCgiParameters& e = request.GetParams(); @@ -4515,14 +4515,14 @@ private: } } - TDuration GetGetTimeout(const NMonitoring::IMonHttpRequest& request) + TDuration GetGetTimeout(const NMonitoring::IMonHttpRequest& request) { return (request.GetParams().Has("timeout")? TDuration::Seconds(FromString<double>(request.GetParams().Get("timeout"))): TDuration::Max()); } - void PostNew(const NMonitoring::IMonHttpRequest& request, IOutputStream& out) + void PostNew(const NMonitoring::IMonHttpRequest& request, IOutputStream& out) { WWW_CHECK(request.GetPostParams().Has("id"), "POST parameter 'id' is not specified"); const TString& id = request.GetPostParams().Get("id"); @@ -4562,7 +4562,7 @@ private: } } - void PostDelete(const NMonitoring::IMonHttpRequest& request, IOutputStream& out) + void PostDelete(const NMonitoring::IMonHttpRequest& request, IOutputStream& out) { WWW_CHECK(request.GetPostParams().Has("id"), "POST parameter 'id' is not specified"); const TString& id = request.GetPostParams().Get("id"); @@ -4589,7 +4589,7 @@ private: } } - void PostSnapshot(const NMonitoring::IMonHttpRequest& request, IOutputStream& out) + void PostSnapshot(const NMonitoring::IMonHttpRequest& request, IOutputStream& out) { WWW_CHECK(request.GetPostParams().Has("id"), "POST parameter 'id' is not specified"); const TString& id = request.GetPostParams().Get("id"); @@ -4624,7 +4624,7 @@ private: } } - void PostSetTimeout(const NMonitoring::IMonHttpRequest& request, IOutputStream& out) + void PostSetTimeout(const NMonitoring::IMonHttpRequest& request, IOutputStream& out) { WWW_CHECK(request.GetPostParams().Has("id"), "POST parameter 'id' is not specified"); const TString& id = request.GetPostParams().Get("id"); diff --git a/library/cpp/monlib/dynamic_counters/counters.cpp b/library/cpp/monlib/dynamic_counters/counters.cpp index 68bf4d6edf..3635d87d0d 100644 --- a/library/cpp/monlib/dynamic_counters/counters.cpp +++ b/library/cpp/monlib/dynamic_counters/counters.cpp @@ -6,68 +6,68 @@ using namespace NMonitoring; -namespace { - TDynamicCounters* AsDynamicCounters(const TIntrusivePtr<TCountableBase>& ptr) { - return dynamic_cast<TDynamicCounters*>(ptr.Get()); - } - - TCounterForPtr* AsCounter(const TIntrusivePtr<TCountableBase>& ptr) { - return dynamic_cast<TCounterForPtr*>(ptr.Get()); - } - - TExpiringCounter* AsExpiringCounter(const TIntrusivePtr<TCountableBase>& ptr) { - return dynamic_cast<TExpiringCounter*>(ptr.Get()); - } - +namespace { + TDynamicCounters* AsDynamicCounters(const TIntrusivePtr<TCountableBase>& ptr) { + return dynamic_cast<TDynamicCounters*>(ptr.Get()); + } + + TCounterForPtr* AsCounter(const TIntrusivePtr<TCountableBase>& ptr) { + return dynamic_cast<TCounterForPtr*>(ptr.Get()); + } + + TExpiringCounter* AsExpiringCounter(const TIntrusivePtr<TCountableBase>& ptr) { + return dynamic_cast<TExpiringCounter*>(ptr.Get()); + } + TExpiringHistogramCounter* AsExpiringHistogramCounter(const TIntrusivePtr<TCountableBase>& ptr) { return dynamic_cast<TExpiringHistogramCounter*>(ptr.Get()); } - THistogramCounter* AsHistogram(const TIntrusivePtr<TCountableBase>& ptr) { - return dynamic_cast<THistogramCounter*>(ptr.Get()); - } - - TIntrusivePtr<TCounterForPtr> AsCounterRef(const TIntrusivePtr<TCountableBase>& ptr) { - return VerifyDynamicCast<TCounterForPtr*>(ptr.Get()); - } - - TIntrusivePtr<TDynamicCounters> AsGroupRef(const TIntrusivePtr<TCountableBase>& ptr) { - return VerifyDynamicCast<TDynamicCounters*>(ptr.Get()); - } - - THistogramPtr AsHistogramRef(const TIntrusivePtr<TCountableBase>& ptr) { - return VerifyDynamicCast<THistogramCounter*>(ptr.Get()); - } + THistogramCounter* AsHistogram(const TIntrusivePtr<TCountableBase>& ptr) { + return dynamic_cast<THistogramCounter*>(ptr.Get()); + } + + TIntrusivePtr<TCounterForPtr> AsCounterRef(const TIntrusivePtr<TCountableBase>& ptr) { + return VerifyDynamicCast<TCounterForPtr*>(ptr.Get()); + } + + TIntrusivePtr<TDynamicCounters> AsGroupRef(const TIntrusivePtr<TCountableBase>& ptr) { + return VerifyDynamicCast<TDynamicCounters*>(ptr.Get()); + } + + THistogramPtr AsHistogramRef(const TIntrusivePtr<TCountableBase>& ptr) { + return VerifyDynamicCast<THistogramCounter*>(ptr.Get()); + } bool IsExpiringCounter(const TIntrusivePtr<TCountableBase>& ptr) { return AsExpiringCounter(ptr) != nullptr || AsExpiringHistogramCounter(ptr) != nullptr; } -} - +} + static constexpr TStringBuf INDENT = " "; -TDynamicCounters::TDynamicCounters(EVisibility vis) -{ - Visibility_ = vis; +TDynamicCounters::TDynamicCounters(EVisibility vis) +{ + Visibility_ = vis; } - + TDynamicCounters::~TDynamicCounters() { } -TDynamicCounters::TCounterPtr TDynamicCounters::GetExpiringCounter(const TString& value, bool derivative, EVisibility vis) { - return GetExpiringNamedCounter("sensor", value, derivative, vis); -} - -TDynamicCounters::TCounterPtr TDynamicCounters::GetExpiringNamedCounter(const TString& name, const TString& value, bool derivative, EVisibility vis) { - return AsCounterRef(GetNamedCounterImpl<true, TExpiringCounter>(name, value, derivative, vis)); -} - -TDynamicCounters::TCounterPtr TDynamicCounters::GetCounter(const TString& value, bool derivative, EVisibility vis) { - return GetNamedCounter("sensor", value, derivative, vis); +TDynamicCounters::TCounterPtr TDynamicCounters::GetExpiringCounter(const TString& value, bool derivative, EVisibility vis) { + return GetExpiringNamedCounter("sensor", value, derivative, vis); +} + +TDynamicCounters::TCounterPtr TDynamicCounters::GetExpiringNamedCounter(const TString& name, const TString& value, bool derivative, EVisibility vis) { + return AsCounterRef(GetNamedCounterImpl<true, TExpiringCounter>(name, value, derivative, vis)); +} + +TDynamicCounters::TCounterPtr TDynamicCounters::GetCounter(const TString& value, bool derivative, EVisibility vis) { + return GetNamedCounter("sensor", value, derivative, vis); } -TDynamicCounters::TCounterPtr TDynamicCounters::GetNamedCounter(const TString& name, const TString& value, bool derivative, EVisibility vis) { - return AsCounterRef(GetNamedCounterImpl<false, TCounterForPtr>(name, value, derivative, vis)); +TDynamicCounters::TCounterPtr TDynamicCounters::GetNamedCounter(const TString& name, const TString& value, bool derivative, EVisibility vis) { + return AsCounterRef(GetNamedCounterImpl<false, TCounterForPtr>(name, value, derivative, vis)); } THistogramPtr TDynamicCounters::GetHistogram(const TString& value, IHistogramCollectorPtr collector, bool derivative, EVisibility vis) { @@ -110,7 +110,7 @@ void TDynamicCounters::RemoveNamedCounter(const TString& name, const TString &va auto g = LockForUpdate("RemoveNamedCounter", name, value); if (const auto it = Counters.find({name, value}); it != Counters.end() && AsCounter(it->second)) { Counters.erase(it); - } + } } TIntrusivePtr<TDynamicCounters> TDynamicCounters::GetSubgroup(const TString& name, const TString& value) { @@ -172,15 +172,15 @@ void TDynamicCounters::ResetCounters(bool derivOnly) { } } -void TDynamicCounters::RegisterCountable(const TString& name, const TString& value, TCountablePtr countable) { - Y_VERIFY(countable); +void TDynamicCounters::RegisterCountable(const TString& name, const TString& value, TCountablePtr countable) { + Y_VERIFY(countable); auto g = LockForUpdate("RegisterCountable", name, value); const bool inserted = Counters.emplace(TChildId(name, value), std::move(countable)).second; Y_VERIFY(inserted); } void TDynamicCounters::RegisterSubgroup(const TString& name, const TString& value, TIntrusivePtr<TDynamicCounters> subgroup) { - RegisterCountable(name, value, subgroup); + RegisterCountable(name, value, subgroup); } void TDynamicCounters::OutputHtml(IOutputStream& os) const { @@ -245,36 +245,36 @@ void TDynamicCounters::OutputPlainText(IOutputStream& os, const TString& indent) } void TDynamicCounters::Accept(const TString& labelName, const TString& labelValue, ICountableConsumer& consumer) const { - if (!IsVisible(Visibility(), consumer.Visibility())) { - return; - } - + if (!IsVisible(Visibility(), consumer.Visibility())) { + return; + } + consumer.OnGroupBegin(labelName, labelValue, this); for (auto& [key, value] : ReadSnapshot()) { value->Accept(key.LabelName, key.LabelValue, consumer); } consumer.OnGroupEnd(labelName, labelValue, this); } - -void TDynamicCounters::RemoveExpired() const { + +void TDynamicCounters::RemoveExpired() const { if (AtomicGet(ExpiringCount) == 0) { - return; - } - + return; + } + TWriteGuard g(Lock); TAtomicBase count = 0; - for (auto it = Counters.begin(); it != Counters.end();) { + for (auto it = Counters.begin(); it != Counters.end();) { if (IsExpiringCounter(it->second) && it->second->RefCount() == 1) { - it = Counters.erase(it); + it = Counters.erase(it); ++count; - } else { - ++it; - } - } + } else { + ++it; + } + } AtomicSub(ExpiringCount, count); -} +} template <bool expiring, class TCounterType, class... TArgs> TDynamicCounters::TCountablePtr TDynamicCounters::GetNamedCounterImpl(const TString& name, const TString& value, TArgs&&... args) { diff --git a/library/cpp/monlib/dynamic_counters/counters.h b/library/cpp/monlib/dynamic_counters/counters.h index d644575d1b..dc178cfbe0 100644 --- a/library/cpp/monlib/dynamic_counters/counters.h +++ b/library/cpp/monlib/dynamic_counters/counters.h @@ -6,7 +6,7 @@ #include <library/cpp/threading/light_rw_lock/lightrwlock.h> #include <library/cpp/containers/stack_vector/stack_vec.h> -#include <util/generic/cast.h> +#include <util/generic/cast.h> #include <util/generic/map.h> #include <util/generic/ptr.h> #include <util/string/cast.h> @@ -19,43 +19,43 @@ namespace NMonitoring { struct TDynamicCounters; struct ICountableConsumer; - + struct TCountableBase: public TAtomicRefCount<TCountableBase> { - // Private means that the object must not be serialized unless the consumer - // has explicitly specified this by setting its Visibility to Private. - // - // Works only for the methods that accept ICountableConsumer - enum class EVisibility: ui8 { - Unspecified, - Public, - Private, - }; - + // Private means that the object must not be serialized unless the consumer + // has explicitly specified this by setting its Visibility to Private. + // + // Works only for the methods that accept ICountableConsumer + enum class EVisibility: ui8 { + Unspecified, + Public, + Private, + }; + virtual ~TCountableBase() { } virtual void Accept( const TString& labelName, const TString& labelValue, ICountableConsumer& consumer) const = 0; - - virtual EVisibility Visibility() const { - return Visibility_; - } - - protected: - EVisibility Visibility_{EVisibility::Unspecified}; + + virtual EVisibility Visibility() const { + return Visibility_; + } + + protected: + EVisibility Visibility_{EVisibility::Unspecified}; }; - inline bool IsVisible(TCountableBase::EVisibility myLevel, TCountableBase::EVisibility consumerLevel) { - if (myLevel == TCountableBase::EVisibility::Private - && consumerLevel != TCountableBase::EVisibility::Private) { - - return false; - } - - return true; - } - + inline bool IsVisible(TCountableBase::EVisibility myLevel, TCountableBase::EVisibility consumerLevel) { + if (myLevel == TCountableBase::EVisibility::Private + && consumerLevel != TCountableBase::EVisibility::Private) { + + return false; + } + + return true; + } + struct ICountableConsumer { virtual ~ICountableConsumer() { } @@ -75,10 +75,10 @@ namespace NMonitoring { virtual void OnGroupEnd( const TString& labelName, const TString& labelValue, const TDynamicCounters* group) = 0; - - virtual TCountableBase::EVisibility Visibility() const { - return TCountableBase::EVisibility::Unspecified; - } + + virtual TCountableBase::EVisibility Visibility() const { + return TCountableBase::EVisibility::Unspecified; + } }; #ifdef _MSC_VER @@ -87,10 +87,10 @@ namespace NMonitoring { #endif // _MSC_VER struct TCounterForPtr: public TDeprecatedCounter, public TCountableBase { - TCounterForPtr(bool derivative = false, EVisibility vis = EVisibility::Public) + TCounterForPtr(bool derivative = false, EVisibility vis = EVisibility::Public) : TDeprecatedCounter(0ULL, derivative) { - Visibility_ = vis; + Visibility_ = vis; } TCounterForPtr(const TCounterForPtr&) = delete; @@ -99,15 +99,15 @@ namespace NMonitoring { void Accept( const TString& labelName, const TString& labelValue, ICountableConsumer& consumer) const override { - if (IsVisible(Visibility(), consumer.Visibility())) { - consumer.OnCounter(labelName, labelValue, this); - } + if (IsVisible(Visibility(), consumer.Visibility())) { + consumer.OnCounter(labelName, labelValue, this); + } + } + + TCountableBase::EVisibility Visibility() const override { + return Visibility_; } - TCountableBase::EVisibility Visibility() const override { - return Visibility_; - } - using TDeprecatedCounter::operator++; using TDeprecatedCounter::operator--; using TDeprecatedCounter::operator+=; @@ -116,25 +116,25 @@ namespace NMonitoring { using TDeprecatedCounter::operator!; }; - struct TExpiringCounter: public TCounterForPtr { - explicit TExpiringCounter(bool derivative = false, EVisibility vis = EVisibility::Public) - : TCounterForPtr{derivative} - { - Visibility_ = vis; - } - - void Reset() { + struct TExpiringCounter: public TCounterForPtr { + explicit TExpiringCounter(bool derivative = false, EVisibility vis = EVisibility::Public) + : TCounterForPtr{derivative} + { + Visibility_ = vis; + } + + void Reset() { TDeprecatedCounter::operator=(0); - } - }; - + } + }; + struct THistogramCounter: public TCountableBase { explicit THistogramCounter( IHistogramCollectorPtr collector, bool derivative = true, EVisibility vis = EVisibility::Public) : Collector_(std::move(collector)) , Derivative_(derivative) { - Visibility_ = vis; + Visibility_ = vis; } void Collect(i64 value) { @@ -157,9 +157,9 @@ namespace NMonitoring { const TString& labelName, const TString& labelValue, ICountableConsumer& consumer) const override { - if (IsVisible(Visibility(), consumer.Visibility())) { + if (IsVisible(Visibility(), consumer.Visibility())) { consumer.OnHistogram(labelName, labelValue, Collector_->Snapshot(), Derivative_); - } + } } void Reset() { @@ -190,7 +190,7 @@ namespace NMonitoring { typedef TIntrusivePtr<TDynamicCounters> TDynamicCounterPtr; struct TDynamicCounters: public TCountableBase { public: - using TCounterPtr = TIntrusivePtr<TCounterForPtr>; + using TCounterPtr = TIntrusivePtr<TCounterForPtr>; using TOnLookupPtr = void (*)(const char *methodName, const TString &name, const TString &value); private: @@ -225,14 +225,14 @@ namespace NMonitoring { }; using TCounters = TMap<TChildId, TCountablePtr>; - using TLabels = TVector<TChildId>; - - /// XXX: hack for deferred removal of expired counters. Remove once Output* functions are not used for serialization - mutable TCounters Counters; + using TLabels = TVector<TChildId>; + + /// XXX: hack for deferred removal of expired counters. Remove once Output* functions are not used for serialization + mutable TCounters Counters; mutable TAtomic ExpiringCount = 0; public: - TDynamicCounters(TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public); + TDynamicCounters(TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public); TDynamicCounters(const TDynamicCounters *origin) : LookupCounter(origin->LookupCounter) @@ -270,56 +270,56 @@ namespace NMonitoring { return items; } - TCounterPtr GetCounter( - const TString& value, - bool derivative = false, - TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public); - - TCounterPtr GetNamedCounter( - const TString& name, - const TString& value, - bool derivative = false, - TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public); - - THistogramPtr GetHistogram( - const TString& value, - IHistogramCollectorPtr collector, + TCounterPtr GetCounter( + const TString& value, + bool derivative = false, + TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public); + + TCounterPtr GetNamedCounter( + const TString& name, + const TString& value, + bool derivative = false, + TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public); + + THistogramPtr GetHistogram( + const TString& value, + IHistogramCollectorPtr collector, bool derivative = true, - TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public); - - THistogramPtr GetNamedHistogram( - const TString& name, - const TString& value, - IHistogramCollectorPtr collector, + TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public); + + THistogramPtr GetNamedHistogram( + const TString& name, + const TString& value, + IHistogramCollectorPtr collector, bool derivative = true, - TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public); - - // These counters will be automatically removed from the registry - // when last reference to the counter expires. - TCounterPtr GetExpiringCounter( - const TString& value, - bool derivative = false, - TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public); - - TCounterPtr GetExpiringNamedCounter( - const TString& name, - const TString& value, - bool derivative = false, - TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public); - - THistogramPtr GetExpiringHistogram( - const TString& value, - IHistogramCollectorPtr collector, + TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public); + + // These counters will be automatically removed from the registry + // when last reference to the counter expires. + TCounterPtr GetExpiringCounter( + const TString& value, + bool derivative = false, + TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public); + + TCounterPtr GetExpiringNamedCounter( + const TString& name, + const TString& value, + bool derivative = false, + TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public); + + THistogramPtr GetExpiringHistogram( + const TString& value, + IHistogramCollectorPtr collector, bool derivative = true, - TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public); - - THistogramPtr GetExpiringNamedHistogram( - const TString& name, - const TString& value, - IHistogramCollectorPtr collector, + TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public); + + THistogramPtr GetExpiringNamedHistogram( + const TString& name, + const TString& value, + IHistogramCollectorPtr collector, bool derivative = true, - TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public); - + TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public); + TCounterPtr FindCounter(const TString& value) const; TCounterPtr FindNamedCounter(const TString& name, const TString& value) const; @@ -339,14 +339,14 @@ namespace NMonitoring { // Recursively reset all/deriv counters to 0. void ResetCounters(bool derivOnly = false); - void RegisterSubgroup(const TString& name, - const TString& value, - TIntrusivePtr<TDynamicCounters> subgroup); - + void RegisterSubgroup(const TString& name, + const TString& value, + TIntrusivePtr<TDynamicCounters> subgroup); + void OutputHtml(IOutputStream& os) const; void EnumerateSubgroups(const std::function<void(const TString& name, const TString& value)>& output) const; - - // mostly for debugging purposes -- use accept with encoder instead + + // mostly for debugging purposes -- use accept with encoder instead void OutputPlainText(IOutputStream& os, const TString& indent = "") const; void Accept( @@ -361,8 +361,8 @@ namespace NMonitoring { return counters; } - void RegisterCountable(const TString& name, const TString& value, TCountablePtr countable); - void RemoveExpired() const; + void RegisterCountable(const TString& name, const TString& value, TCountablePtr countable); + void RemoveExpired() const; template <bool expiring, class TCounterType, class... TArgs> TCountablePtr GetNamedCounterImpl(const TString& name, const TString& value, TArgs&&... args); diff --git a/library/cpp/monlib/dynamic_counters/counters_ut.cpp b/library/cpp/monlib/dynamic_counters/counters_ut.cpp index eb66b15fd7..3591037e0a 100644 --- a/library/cpp/monlib/dynamic_counters/counters_ut.cpp +++ b/library/cpp/monlib/dynamic_counters/counters_ut.cpp @@ -224,48 +224,48 @@ Y_UNIT_TEST_SUITE(TDynamicCountersTest) { " sensor:2 = 0\n" "}\n"); } - - Y_UNIT_TEST(ExpiringCounters) { - TDynamicCounterPtr rootGroup{new TDynamicCounters()}; - - { - auto c = rootGroup->GetExpiringCounter("foo"); + + Y_UNIT_TEST(ExpiringCounters) { + TDynamicCounterPtr rootGroup{new TDynamicCounters()}; + + { + auto c = rootGroup->GetExpiringCounter("foo"); auto h = rootGroup->GetExpiringHistogram("bar", ExplicitHistogram({1, 42})); h->Collect(15); - - TStringStream ss; - TCountersPrinter printer(&ss); - rootGroup->Accept("root", "counters", printer); - UNIT_ASSERT_STRINGS_EQUAL(ss.Str(), - "root:counters {\n" + + TStringStream ss; + TCountersPrinter printer(&ss); + rootGroup->Accept("root", "counters", printer); + UNIT_ASSERT_STRINGS_EQUAL(ss.Str(), + "root:counters {\n" " sensor:bar = {1: 0, 42: 1, inf: 0}\n" - " sensor:foo = 0\n" - "}\n"); - } - - TStringStream ss; - TCountersPrinter printer(&ss); - rootGroup->Accept("root", "counters", printer); - UNIT_ASSERT_STRINGS_EQUAL(ss.Str(), - "root:counters {\n" - "}\n"); + " sensor:foo = 0\n" + "}\n"); + } + + TStringStream ss; + TCountersPrinter printer(&ss); + rootGroup->Accept("root", "counters", printer); + UNIT_ASSERT_STRINGS_EQUAL(ss.Str(), + "root:counters {\n" + "}\n"); } - - Y_UNIT_TEST(ExpiringCountersDiesAfterRegistry) { - TDynamicCounters::TCounterPtr ptr; - - { - TDynamicCounterPtr rootGroup{new TDynamicCounters()}; - ptr = rootGroup->GetExpiringCounter("foo"); - - TStringStream ss; - TCountersPrinter printer(&ss); - rootGroup->Accept("root", "counters", printer); - UNIT_ASSERT_STRINGS_EQUAL(ss.Str(), - "root:counters {\n" - " sensor:foo = 0\n" - "}\n"); - } + + Y_UNIT_TEST(ExpiringCountersDiesAfterRegistry) { + TDynamicCounters::TCounterPtr ptr; + + { + TDynamicCounterPtr rootGroup{new TDynamicCounters()}; + ptr = rootGroup->GetExpiringCounter("foo"); + + TStringStream ss; + TCountersPrinter printer(&ss); + rootGroup->Accept("root", "counters", printer); + UNIT_ASSERT_STRINGS_EQUAL(ss.Str(), + "root:counters {\n" + " sensor:foo = 0\n" + "}\n"); + } } Y_UNIT_TEST(HistogramCounter) { diff --git a/library/cpp/monlib/dynamic_counters/encode.cpp b/library/cpp/monlib/dynamic_counters/encode.cpp index 47927c0e65..ffa48d276e 100644 --- a/library/cpp/monlib/dynamic_counters/encode.cpp +++ b/library/cpp/monlib/dynamic_counters/encode.cpp @@ -17,7 +17,7 @@ namespace NMonitoring { public: explicit TConsumer(NMonitoring::IMetricEncoderPtr encoderImpl, TCountableBase::EVisibility vis) : EncoderImpl_(std::move(encoderImpl)) - , Visibility_{vis} + , Visibility_{vis} { } @@ -73,10 +73,10 @@ namespace NMonitoring { } } - TCountableBase::EVisibility Visibility() const override { - return Visibility_; - } - + TCountableBase::EVisibility Visibility() const override { + return Visibility_; + } + private: void EncodeLabels(const TString& labelName, const TString& labelValue) { EncoderImpl_->OnLabelsBegin(); @@ -90,12 +90,12 @@ namespace NMonitoring { private: NMonitoring::IMetricEncoderPtr EncoderImpl_; TVector<TLabel> ParentLabels_; - TCountableBase::EVisibility Visibility_; + TCountableBase::EVisibility Visibility_; }; } - THolder<ICountableConsumer> CreateEncoder(IOutputStream* out, EFormat format, TCountableBase::EVisibility vis) { + THolder<ICountableConsumer> CreateEncoder(IOutputStream* out, EFormat format, TCountableBase::EVisibility vis) { switch (format) { case EFormat::JSON: return MakeHolder<TConsumer>(NMonitoring::EncoderJson(out), vis); @@ -103,7 +103,7 @@ namespace NMonitoring { return MakeHolder<TConsumer>(NMonitoring::EncoderSpackV1( out, NMonitoring::ETimePrecision::SECONDS, - NMonitoring::ECompression::ZSTD), vis); + NMonitoring::ECompression::ZSTD), vis); case EFormat::PROMETHEUS: return MakeHolder<TConsumer>(NMonitoring::EncoderPrometheus( out), vis); diff --git a/library/cpp/monlib/dynamic_counters/encode.h b/library/cpp/monlib/dynamic_counters/encode.h index c6e190c729..c79964d7cb 100644 --- a/library/cpp/monlib/dynamic_counters/encode.h +++ b/library/cpp/monlib/dynamic_counters/encode.h @@ -7,11 +7,11 @@ namespace NMonitoring { - THolder<ICountableConsumer> CreateEncoder( - IOutputStream* out, - EFormat format, - TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public - ); + THolder<ICountableConsumer> CreateEncoder( + IOutputStream* out, + EFormat format, + TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public + ); THolder<ICountableConsumer> AsCountableConsumer( NMonitoring::IMetricEncoderPtr encoder, diff --git a/library/cpp/monlib/dynamic_counters/encode_ut.cpp b/library/cpp/monlib/dynamic_counters/encode_ut.cpp index 0e48009ee2..52d77b6b41 100644 --- a/library/cpp/monlib/dynamic_counters/encode_ut.cpp +++ b/library/cpp/monlib/dynamic_counters/encode_ut.cpp @@ -167,48 +167,48 @@ namespace NMonitoring { AssertResult(samples); } - - Y_UNIT_TEST(PrivateSubgroupIsNotSerialized) { - TBuffer result; - auto subGroup = MakeIntrusive<TDynamicCounters>(TCountableBase::EVisibility::Private); - subGroup->GetCounter("hello"); - Data.RegisterSubgroup("foo", "bar", subGroup); - - { - TBufferOutput out(result); - auto encoder = CreateEncoder(&out, EFormat::SPACK); - Data.Accept(TString(), TString(), *encoder); - } - - NProto::TSingleSamplesList samples; - { - auto e = EncoderProtobuf(&samples); - TBufferInput in(result); - DecodeSpackV1(&in, e.Get()); - } - - AssertResult(samples); - } - - Y_UNIT_TEST(PrivateCounterIsNotSerialized) { - TBuffer result; - Data.GetCounter("foo", false, TCountableBase::EVisibility::Private); - - { - TBufferOutput out(result); - auto encoder = CreateEncoder(&out, EFormat::SPACK); - Data.Accept(TString(), TString(), *encoder); - } - - NProto::TSingleSamplesList samples; - { - auto e = EncoderProtobuf(&samples); - TBufferInput in(result); - DecodeSpackV1(&in, e.Get()); - } - - AssertResult(samples); - } + + Y_UNIT_TEST(PrivateSubgroupIsNotSerialized) { + TBuffer result; + auto subGroup = MakeIntrusive<TDynamicCounters>(TCountableBase::EVisibility::Private); + subGroup->GetCounter("hello"); + Data.RegisterSubgroup("foo", "bar", subGroup); + + { + TBufferOutput out(result); + auto encoder = CreateEncoder(&out, EFormat::SPACK); + Data.Accept(TString(), TString(), *encoder); + } + + NProto::TSingleSamplesList samples; + { + auto e = EncoderProtobuf(&samples); + TBufferInput in(result); + DecodeSpackV1(&in, e.Get()); + } + + AssertResult(samples); + } + + Y_UNIT_TEST(PrivateCounterIsNotSerialized) { + TBuffer result; + Data.GetCounter("foo", false, TCountableBase::EVisibility::Private); + + { + TBufferOutput out(result); + auto encoder = CreateEncoder(&out, EFormat::SPACK); + Data.Accept(TString(), TString(), *encoder); + } + + NProto::TSingleSamplesList samples; + { + auto e = EncoderProtobuf(&samples); + TBufferInput in(result); + DecodeSpackV1(&in, e.Get()); + } + + AssertResult(samples); + } Y_UNIT_TEST(ToJson) { TString result = ToJson(Data); diff --git a/library/cpp/monlib/dynamic_counters/golovan_page.cpp b/library/cpp/monlib/dynamic_counters/golovan_page.cpp index 3fc3152365..49cf2d39bb 100644 --- a/library/cpp/monlib/dynamic_counters/golovan_page.cpp +++ b/library/cpp/monlib/dynamic_counters/golovan_page.cpp @@ -72,8 +72,8 @@ TGolovanCountersPage::TGolovanCountersPage(const TString& path, TIntrusivePtr<NM { } -void TGolovanCountersPage::Output(IMonHttpRequest& request) { - TGolovanCountableConsumer consumer(request.Output(), OutputCallback); +void TGolovanCountersPage::Output(IMonHttpRequest& request) { + TGolovanCountableConsumer consumer(request.Output(), OutputCallback); Counters->Accept("", "", consumer); consumer.Flush(); } diff --git a/library/cpp/monlib/dynamic_counters/golovan_page.h b/library/cpp/monlib/dynamic_counters/golovan_page.h index b323094c1b..e1772c7734 100644 --- a/library/cpp/monlib/dynamic_counters/golovan_page.h +++ b/library/cpp/monlib/dynamic_counters/golovan_page.h @@ -18,7 +18,7 @@ public: TGolovanCountersPage(const TString& path, TIntrusivePtr<NMonitoring::TDynamicCounters> counters, TOutputCallback outputCallback = nullptr); - void Output(NMonitoring::IMonHttpRequest& request) override; + void Output(NMonitoring::IMonHttpRequest& request) override; private: TOutputCallback OutputCallback; diff --git a/library/cpp/monlib/dynamic_counters/page.cpp b/library/cpp/monlib/dynamic_counters/page.cpp index 85214ac883..5124a47bb3 100644 --- a/library/cpp/monlib/dynamic_counters/page.cpp +++ b/library/cpp/monlib/dynamic_counters/page.cpp @@ -14,7 +14,7 @@ namespace { currentCounters(nullptr); } -TMaybe<EFormat> ParseFormat(TStringBuf str) { +TMaybe<EFormat> ParseFormat(TStringBuf str) { if (str == TStringBuf("json")) { return EFormat::JSON; } else if (str == TStringBuf("spack")) { @@ -26,20 +26,20 @@ TMaybe<EFormat> ParseFormat(TStringBuf str) { } } -void TDynamicCountersPage::Output(NMonitoring::IMonHttpRequest& request) { +void TDynamicCountersPage::Output(NMonitoring::IMonHttpRequest& request) { if (OutputCallback) { OutputCallback(); } - TCountableBase::EVisibility visibility{ - TCountableBase::EVisibility::Public - }; - - TVector<TStringBuf> parts; - StringSplitter(request.GetPathInfo()) - .Split('/') - .SkipEmpty() - .Collect(&parts); + TCountableBase::EVisibility visibility{ + TCountableBase::EVisibility::Public + }; + + TVector<TStringBuf> parts; + StringSplitter(request.GetPathInfo()) + .Split('/') + .SkipEmpty() + .Collect(&parts); TMaybe<EFormat> format = !parts.empty() ? ParseFormat(parts.back()) : Nothing(); if (format) { @@ -47,29 +47,29 @@ void TDynamicCountersPage::Output(NMonitoring::IMonHttpRequest& request) { } if (!parts.empty() && parts.back() == TStringBuf("private")) { - visibility = TCountableBase::EVisibility::Private; - parts.pop_back(); - } - + visibility = TCountableBase::EVisibility::Private; + parts.pop_back(); + } + auto counters = Counters; - for (const auto& escaped : parts) { - const auto part = CGIUnescapeRet(escaped); - + for (const auto& escaped : parts) { + const auto part = CGIUnescapeRet(escaped); + TVector<TString> labels; - StringSplitter(part).Split('=').SkipEmpty().Collect(&labels); + StringSplitter(part).Split('=').SkipEmpty().Collect(&labels); if (labels.size() != 2U) return NotFound(request); - if (const auto child = counters->FindSubgroup( - labels.front(), - labels.back())) { - + if (const auto child = counters->FindSubgroup( + labels.front(), + labels.back())) { + counters = child; - } else { - return HandleAbsentSubgroup(request); - } + } else { + return HandleAbsentSubgroup(request); + } } if (!format) { @@ -95,21 +95,21 @@ void TDynamicCountersPage::Output(NMonitoring::IMonHttpRequest& request) { out.Flush(); } -void TDynamicCountersPage::HandleAbsentSubgroup(IMonHttpRequest& request) { - if (UnknownGroupPolicy == EUnknownGroupPolicy::Error) { - NotFound(request); - } else if (UnknownGroupPolicy == EUnknownGroupPolicy::Ignore) { - NoContent(request); - } else { - Y_FAIL("Unsupported policy set"); - } -} - -void TDynamicCountersPage::BeforePre(IMonHttpRequest& request) { - IOutputStream& out = request.Output(); +void TDynamicCountersPage::HandleAbsentSubgroup(IMonHttpRequest& request) { + if (UnknownGroupPolicy == EUnknownGroupPolicy::Error) { + NotFound(request); + } else if (UnknownGroupPolicy == EUnknownGroupPolicy::Ignore) { + NoContent(request); + } else { + Y_FAIL("Unsupported policy set"); + } +} + +void TDynamicCountersPage::BeforePre(IMonHttpRequest& request) { + IOutputStream& out = request.Output(); HTML(out) { DIV() { - out << "<a href='" << request.GetPath() << "/json'>Counters as JSON</a>"; + out << "<a href='" << request.GetPath() << "/json'>Counters as JSON</a>"; out << " for <a href='https://wiki.yandex-team.ru/solomon/'>Solomon</a>"; } @@ -117,13 +117,13 @@ void TDynamicCountersPage::BeforePre(IMonHttpRequest& request) { out << "Counters subgroups"; } UL() { - currentCounters->EnumerateSubgroups([&](const TString& name, const TString& value) { - LI() { - TString pathPart = name + "=" + value; - Quote(pathPart, ""); - out << "\n<a href='" << request.GetPath() << "/" << pathPart << "'>" << name << " " << value << "</a>"; - } - }); + currentCounters->EnumerateSubgroups([&](const TString& name, const TString& value) { + LI() { + TString pathPart = name + "=" + value; + Quote(pathPart, ""); + out << "\n<a href='" << request.GetPath() << "/" << pathPart << "'>" << name << " " << value << "</a>"; + } + }); } H4() { @@ -132,10 +132,10 @@ void TDynamicCountersPage::BeforePre(IMonHttpRequest& request) { } } -void TDynamicCountersPage::OutputText(IOutputStream& out, IMonHttpRequest&) { +void TDynamicCountersPage::OutputText(IOutputStream& out, IMonHttpRequest&) { currentCounters->OutputPlainText(out); } - -void TDynamicCountersPage::SetUnknownGroupPolicy(EUnknownGroupPolicy value) { - UnknownGroupPolicy = value; -} + +void TDynamicCountersPage::SetUnknownGroupPolicy(EUnknownGroupPolicy value) { + UnknownGroupPolicy = value; +} diff --git a/library/cpp/monlib/dynamic_counters/page.h b/library/cpp/monlib/dynamic_counters/page.h index 470d427b40..1f0ef6a5ea 100644 --- a/library/cpp/monlib/dynamic_counters/page.h +++ b/library/cpp/monlib/dynamic_counters/page.h @@ -9,11 +9,11 @@ #include <functional> namespace NMonitoring { - enum class EUnknownGroupPolicy { - Error, // send 404 - Ignore, // send 204 - }; - + enum class EUnknownGroupPolicy { + Error, // send 404 + Ignore, // send 204 + }; + struct TDynamicCountersPage: public TPreMonPage { public: using TOutputCallback = std::function<void()>; @@ -21,11 +21,11 @@ namespace NMonitoring { private: const TIntrusivePtr<TDynamicCounters> Counters; TOutputCallback OutputCallback; - EUnknownGroupPolicy UnknownGroupPolicy {EUnknownGroupPolicy::Error}; + EUnknownGroupPolicy UnknownGroupPolicy {EUnknownGroupPolicy::Error}; + + private: + void HandleAbsentSubgroup(IMonHttpRequest& request); - private: - void HandleAbsentSubgroup(IMonHttpRequest& request); - public: TDynamicCountersPage(const TString& path, const TString& title, @@ -37,14 +37,14 @@ namespace NMonitoring { { } - void Output(NMonitoring::IMonHttpRequest& request) override; + void Output(NMonitoring::IMonHttpRequest& request) override; + + void BeforePre(NMonitoring::IMonHttpRequest& request) override; - void BeforePre(NMonitoring::IMonHttpRequest& request) override; + void OutputText(IOutputStream& out, NMonitoring::IMonHttpRequest&) override; - void OutputText(IOutputStream& out, NMonitoring::IMonHttpRequest&) override; - - /// If set to Error, responds with 404 if the requested subgroup is not found. This is the default. - /// If set to Ignore, responds with 204 if the requested subgroup is not found - void SetUnknownGroupPolicy(EUnknownGroupPolicy value); + /// If set to Error, responds with 404 if the requested subgroup is not found. This is the default. + /// If set to Ignore, responds with 204 if the requested subgroup is not found + void SetUnknownGroupPolicy(EUnknownGroupPolicy value); }; } diff --git a/library/cpp/monlib/dynamic_counters/percentile/percentile.h b/library/cpp/monlib/dynamic_counters/percentile/percentile.h index 08b7ec5f0f..73c482bce9 100644 --- a/library/cpp/monlib/dynamic_counters/percentile/percentile.h +++ b/library/cpp/monlib/dynamic_counters/percentile/percentile.h @@ -1,59 +1,59 @@ -#pragma once - +#pragma once + #include "percentile_base.h" - -namespace NMonitoring { - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -// Percentile tracker for monitoring -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -template <size_t BUCKET_SIZE, size_t BUCKET_COUNT, size_t FRAME_COUNT> + +namespace NMonitoring { + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Percentile tracker for monitoring +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +template <size_t BUCKET_SIZE, size_t BUCKET_COUNT, size_t FRAME_COUNT> struct TPercentileTracker : public TPercentileBase { - TAtomic Items[BUCKET_COUNT]; - TAtomicBase Frame[FRAME_COUNT][BUCKET_COUNT]; - size_t CurrentFrame; - - TPercentileTracker() - : CurrentFrame(0) - { - for (size_t i = 0; i < BUCKET_COUNT; ++i) { - AtomicSet(Items[i], 0); - } - for (size_t frame = 0; frame < FRAME_COUNT; ++frame) { - for (size_t bucket = 0; bucket < BUCKET_COUNT; ++bucket) { - Frame[frame][bucket] = 0; - } - } - } - - void Increment(size_t value) { - AtomicIncrement(Items[Min((value + BUCKET_SIZE - 1) / BUCKET_SIZE, BUCKET_COUNT - 1)]); - } - - // shift frame (call periodically) - void Update() { - TVector<TAtomicBase> totals(BUCKET_COUNT); - totals.resize(BUCKET_COUNT); - TAtomicBase total = 0; - for (size_t i = 0; i < BUCKET_COUNT; ++i) { - TAtomicBase item = AtomicGet(Items[i]); - TAtomicBase prevItem = Frame[CurrentFrame][i]; - Frame[CurrentFrame][i] = item; - total += item - prevItem; - totals[i] = total; - } - - for (size_t i = 0; i < Percentiles.size(); ++i) { - TPercentile &percentile = Percentiles[i]; - auto threshold = (TAtomicBase)(percentile.first * (float)total); - threshold = Min(threshold, total); - auto it = LowerBound(totals.begin(), totals.end(), threshold); - size_t index = it - totals.begin(); - (*percentile.second) = index * BUCKET_SIZE; - } - CurrentFrame = (CurrentFrame + 1) % FRAME_COUNT; - } -}; - -} // NMonitoring + TAtomic Items[BUCKET_COUNT]; + TAtomicBase Frame[FRAME_COUNT][BUCKET_COUNT]; + size_t CurrentFrame; + + TPercentileTracker() + : CurrentFrame(0) + { + for (size_t i = 0; i < BUCKET_COUNT; ++i) { + AtomicSet(Items[i], 0); + } + for (size_t frame = 0; frame < FRAME_COUNT; ++frame) { + for (size_t bucket = 0; bucket < BUCKET_COUNT; ++bucket) { + Frame[frame][bucket] = 0; + } + } + } + + void Increment(size_t value) { + AtomicIncrement(Items[Min((value + BUCKET_SIZE - 1) / BUCKET_SIZE, BUCKET_COUNT - 1)]); + } + + // shift frame (call periodically) + void Update() { + TVector<TAtomicBase> totals(BUCKET_COUNT); + totals.resize(BUCKET_COUNT); + TAtomicBase total = 0; + for (size_t i = 0; i < BUCKET_COUNT; ++i) { + TAtomicBase item = AtomicGet(Items[i]); + TAtomicBase prevItem = Frame[CurrentFrame][i]; + Frame[CurrentFrame][i] = item; + total += item - prevItem; + totals[i] = total; + } + + for (size_t i = 0; i < Percentiles.size(); ++i) { + TPercentile &percentile = Percentiles[i]; + auto threshold = (TAtomicBase)(percentile.first * (float)total); + threshold = Min(threshold, total); + auto it = LowerBound(totals.begin(), totals.end(), threshold); + size_t index = it - totals.begin(); + (*percentile.second) = index * BUCKET_SIZE; + } + CurrentFrame = (CurrentFrame + 1) % FRAME_COUNT; + } +}; + +} // NMonitoring diff --git a/library/cpp/monlib/dynamic_counters/percentile/percentile_lg.h b/library/cpp/monlib/dynamic_counters/percentile/percentile_lg.h index b39c53e3df..0042cd9a6a 100644 --- a/library/cpp/monlib/dynamic_counters/percentile/percentile_lg.h +++ b/library/cpp/monlib/dynamic_counters/percentile/percentile_lg.h @@ -1,182 +1,182 @@ -#pragma once - +#pragma once + #include <library/cpp/containers/stack_vector/stack_vec.h> - -#include <util/generic/bitops.h> - -#include <cmath> - + +#include <util/generic/bitops.h> + +#include <cmath> + #include "percentile_base.h" -namespace NMonitoring { - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -// Percentile tracker for monitoring -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -template <size_t BASE_BITS, size_t EXP_BITS, size_t FRAME_COUNT> +namespace NMonitoring { + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Percentile tracker for monitoring +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +template <size_t BASE_BITS, size_t EXP_BITS, size_t FRAME_COUNT> struct TPercentileTrackerLg : public TPercentileBase { static constexpr size_t BUCKET_COUNT = size_t(1) << EXP_BITS; static constexpr size_t BUCKET_SIZE = size_t(1) << BASE_BITS; - static constexpr size_t ITEMS_COUNT = BUCKET_COUNT * BUCKET_SIZE; + static constexpr size_t ITEMS_COUNT = BUCKET_COUNT * BUCKET_SIZE; static constexpr size_t TRACKER_LIMIT = BUCKET_SIZE * ((size_t(1) << BUCKET_COUNT) - 1) - (size_t(1) << (BUCKET_COUNT - 1)); static constexpr size_t MAX_GRANULARITY = size_t(1) << (BUCKET_COUNT - 1); - size_t Borders[BUCKET_COUNT]; - TAtomic Items[ITEMS_COUNT]; - TAtomicBase Frame[FRAME_COUNT][ITEMS_COUNT]; - size_t CurrentFrame; - - TPercentileTrackerLg() - : CurrentFrame(0) - { - Borders[0] = 0; - for (size_t i = 1; i < BUCKET_COUNT; ++i) { - Borders[i] = Borders[i-1] + (BUCKET_SIZE << (i - 1)); - } - for (size_t i = 0; i < ITEMS_COUNT; ++i) { - AtomicSet(Items[i], 0); - } - for (size_t frame = 0; frame < FRAME_COUNT; ++frame) { - for (size_t bucket = 0; bucket < ITEMS_COUNT; ++bucket) { - Frame[frame][bucket] = 0; - } - } - } - - size_t inline BucketIdxIf(size_t value) { - static_assert(BASE_BITS == 5, "if-based bucket calculation cannot be used if BASE_BITS != 5"); - size_t bucket_idx; - if (value < 8160) { - if (value < 480) { - if (value < 96) { - if (value < 32) { - bucket_idx = 0; - } else { - bucket_idx = 1; - } - } else { - if (value < 224) { - bucket_idx = 2; - } else { - bucket_idx = 3; - } - } - } else { - if (value < 2016) { - if (value < 992) { - bucket_idx = 4; - } else { - bucket_idx = 5; - } - } else { - if (value < 4064) { - bucket_idx = 6; - } else { - bucket_idx = 7; - } - } - } - } else { - if (value < 131040) { - if (value < 32736) { - if (value < 16352) { - bucket_idx = 8; - } else { - bucket_idx = 9; - } - } else { - if (value < 65504) { - bucket_idx = 10; - } else { - bucket_idx = 11; - } - } - } else { - if (value < 524256) { - if (value < 262112) { - bucket_idx = 12; - } else { - bucket_idx = 13; - } - } else { - if (value < 1048544) { - bucket_idx = 14; - } else { - bucket_idx = 15; - } - } - } - } - return Min(bucket_idx, BUCKET_COUNT - 1); - } - - size_t inline BucketIdxBinarySearch(size_t value) { - size_t l = 0; - size_t r = BUCKET_COUNT; - while (l < r - 1) { - size_t mid = (l + r) / 2; - if (value < Borders[mid]) { - r = mid; - } else { - l = mid; - } - } - return l; - } - - size_t inline BucketIdxMostSignificantBit(size_t value) { - size_t bucket_idx = MostSignificantBit(value + BUCKET_SIZE) - BASE_BITS; - return Min(bucket_idx, BUCKET_COUNT - 1); - } - - void Increment(size_t value) { + size_t Borders[BUCKET_COUNT]; + TAtomic Items[ITEMS_COUNT]; + TAtomicBase Frame[FRAME_COUNT][ITEMS_COUNT]; + size_t CurrentFrame; + + TPercentileTrackerLg() + : CurrentFrame(0) + { + Borders[0] = 0; + for (size_t i = 1; i < BUCKET_COUNT; ++i) { + Borders[i] = Borders[i-1] + (BUCKET_SIZE << (i - 1)); + } + for (size_t i = 0; i < ITEMS_COUNT; ++i) { + AtomicSet(Items[i], 0); + } + for (size_t frame = 0; frame < FRAME_COUNT; ++frame) { + for (size_t bucket = 0; bucket < ITEMS_COUNT; ++bucket) { + Frame[frame][bucket] = 0; + } + } + } + + size_t inline BucketIdxIf(size_t value) { + static_assert(BASE_BITS == 5, "if-based bucket calculation cannot be used if BASE_BITS != 5"); + size_t bucket_idx; + if (value < 8160) { + if (value < 480) { + if (value < 96) { + if (value < 32) { + bucket_idx = 0; + } else { + bucket_idx = 1; + } + } else { + if (value < 224) { + bucket_idx = 2; + } else { + bucket_idx = 3; + } + } + } else { + if (value < 2016) { + if (value < 992) { + bucket_idx = 4; + } else { + bucket_idx = 5; + } + } else { + if (value < 4064) { + bucket_idx = 6; + } else { + bucket_idx = 7; + } + } + } + } else { + if (value < 131040) { + if (value < 32736) { + if (value < 16352) { + bucket_idx = 8; + } else { + bucket_idx = 9; + } + } else { + if (value < 65504) { + bucket_idx = 10; + } else { + bucket_idx = 11; + } + } + } else { + if (value < 524256) { + if (value < 262112) { + bucket_idx = 12; + } else { + bucket_idx = 13; + } + } else { + if (value < 1048544) { + bucket_idx = 14; + } else { + bucket_idx = 15; + } + } + } + } + return Min(bucket_idx, BUCKET_COUNT - 1); + } + + size_t inline BucketIdxBinarySearch(size_t value) { + size_t l = 0; + size_t r = BUCKET_COUNT; + while (l < r - 1) { + size_t mid = (l + r) / 2; + if (value < Borders[mid]) { + r = mid; + } else { + l = mid; + } + } + return l; + } + + size_t inline BucketIdxMostSignificantBit(size_t value) { + size_t bucket_idx = MostSignificantBit(value + BUCKET_SIZE) - BASE_BITS; + return Min(bucket_idx, BUCKET_COUNT - 1); + } + + void Increment(size_t value) { size_t bucket_idx = BucketIdxMostSignificantBit(value); - size_t inside_bucket_idx = (value - Borders[bucket_idx] + (1 << bucket_idx) - 1) >> bucket_idx; - size_t idx = bucket_idx * BUCKET_SIZE + inside_bucket_idx; - AtomicIncrement(Items[Min(idx, ITEMS_COUNT - 1)]); - } - - // Needed only for tests - size_t GetPercentile(float threshold) { - TStackVec<TAtomicBase, ITEMS_COUNT> totals(ITEMS_COUNT); - TAtomicBase total = 0; - for (size_t i = 0; i < ITEMS_COUNT; ++i) { - total += AtomicGet(Items[i]); - totals[i] = total; - } - TAtomicBase item_threshold = std::llround(threshold * (float)total); - item_threshold = Min(item_threshold, total); - auto it = LowerBound(totals.begin(), totals.end(), item_threshold); - size_t index = it - totals.begin(); - size_t bucket_idx = index / BUCKET_SIZE; - return Borders[bucket_idx] + ((index % BUCKET_SIZE) << bucket_idx); - } - - // shift frame (call periodically) - void Update() { - TStackVec<TAtomicBase, ITEMS_COUNT> totals(ITEMS_COUNT); - TAtomicBase total = 0; - for (size_t i = 0; i < ITEMS_COUNT; ++i) { - TAtomicBase item = AtomicGet(Items[i]); - TAtomicBase prevItem = Frame[CurrentFrame][i]; - Frame[CurrentFrame][i] = item; - total += item - prevItem; - totals[i] = total; - } - - for (size_t i = 0; i < Percentiles.size(); ++i) { - TPercentile &percentile = Percentiles[i]; - TAtomicBase threshold = std::llround(percentile.first * (float)total); - threshold = Min(threshold, total); - auto it = LowerBound(totals.begin(), totals.end(), threshold); - size_t index = it - totals.begin(); - size_t bucket_idx = index / BUCKET_SIZE; - (*percentile.second) = Borders[bucket_idx] + ((index % BUCKET_SIZE) << bucket_idx); - } - CurrentFrame = (CurrentFrame + 1) % FRAME_COUNT; - } -}; - -} // NMonitoring + size_t inside_bucket_idx = (value - Borders[bucket_idx] + (1 << bucket_idx) - 1) >> bucket_idx; + size_t idx = bucket_idx * BUCKET_SIZE + inside_bucket_idx; + AtomicIncrement(Items[Min(idx, ITEMS_COUNT - 1)]); + } + + // Needed only for tests + size_t GetPercentile(float threshold) { + TStackVec<TAtomicBase, ITEMS_COUNT> totals(ITEMS_COUNT); + TAtomicBase total = 0; + for (size_t i = 0; i < ITEMS_COUNT; ++i) { + total += AtomicGet(Items[i]); + totals[i] = total; + } + TAtomicBase item_threshold = std::llround(threshold * (float)total); + item_threshold = Min(item_threshold, total); + auto it = LowerBound(totals.begin(), totals.end(), item_threshold); + size_t index = it - totals.begin(); + size_t bucket_idx = index / BUCKET_SIZE; + return Borders[bucket_idx] + ((index % BUCKET_SIZE) << bucket_idx); + } + + // shift frame (call periodically) + void Update() { + TStackVec<TAtomicBase, ITEMS_COUNT> totals(ITEMS_COUNT); + TAtomicBase total = 0; + for (size_t i = 0; i < ITEMS_COUNT; ++i) { + TAtomicBase item = AtomicGet(Items[i]); + TAtomicBase prevItem = Frame[CurrentFrame][i]; + Frame[CurrentFrame][i] = item; + total += item - prevItem; + totals[i] = total; + } + + for (size_t i = 0; i < Percentiles.size(); ++i) { + TPercentile &percentile = Percentiles[i]; + TAtomicBase threshold = std::llround(percentile.first * (float)total); + threshold = Min(threshold, total); + auto it = LowerBound(totals.begin(), totals.end(), threshold); + size_t index = it - totals.begin(); + size_t bucket_idx = index / BUCKET_SIZE; + (*percentile.second) = Borders[bucket_idx] + ((index % BUCKET_SIZE) << bucket_idx); + } + CurrentFrame = (CurrentFrame + 1) % FRAME_COUNT; + } +}; + +} // NMonitoring diff --git a/library/cpp/monlib/dynamic_counters/percentile/percentile_ut.cpp b/library/cpp/monlib/dynamic_counters/percentile/percentile_ut.cpp index b09004bd7f..6c8bb54ec9 100644 --- a/library/cpp/monlib/dynamic_counters/percentile/percentile_ut.cpp +++ b/library/cpp/monlib/dynamic_counters/percentile/percentile_ut.cpp @@ -1,10 +1,10 @@ -#include "percentile_lg.h" +#include "percentile_lg.h" #include <library/cpp/testing/unittest/registar.h> - -using namespace NMonitoring; - -Y_UNIT_TEST_SUITE(PercentileTest) { - + +using namespace NMonitoring; + +Y_UNIT_TEST_SUITE(PercentileTest) { + template<size_t A, size_t B, size_t B_BEGIN> void printSizeAndLimit() { using TPerc = TPercentileTrackerLg<A, B, 15>; @@ -57,73 +57,73 @@ void printSizeAndLimit() { } } - Y_UNIT_TEST(BucketIdxIfvsBucketIdxBinarySearch) { - for (size_t var = 0; var < 5; var++) { - if (var == 0) { - TPercentileTrackerLg<3, 2, 15> tracker; - for (size_t i = 0; i < 3000000; i += 1) { - size_t num1 = tracker.BucketIdxMostSignificantBit(i); - size_t num2 = tracker.BucketIdxBinarySearch(i); - UNIT_ASSERT_EQUAL(num1, num2); - } - } else if (var == 1) { - TPercentileTrackerLg<4, 4, 15> tracker; - for (size_t i = 0; i < 3000000; i += 1) { - size_t num1 = tracker.BucketIdxMostSignificantBit(i); - size_t num2 = tracker.BucketIdxBinarySearch(i); - UNIT_ASSERT_EQUAL(num1, num2); - } - } else if (var == 2) { - TPercentileTrackerLg<5, 3, 15> tracker; - for (size_t i = 0; i < 3000000; i += 1) { - size_t num1 = tracker.BucketIdxMostSignificantBit(i); - size_t num2 = tracker.BucketIdxBinarySearch(i); - size_t num3 = tracker.BucketIdxIf(i); - UNIT_ASSERT_EQUAL(num1, num2); - UNIT_ASSERT_EQUAL(num2, num3); - } - } else if (var == 3) { - TPercentileTrackerLg<5, 4, 15> tracker; - for (size_t i = 0; i < 3000000; i += 1) { - size_t num1 = tracker.BucketIdxMostSignificantBit(i); - size_t num2 = tracker.BucketIdxBinarySearch(i); - size_t num3 = tracker.BucketIdxIf(i); - UNIT_ASSERT_EQUAL(num1, num2); - UNIT_ASSERT_EQUAL(num2, num3); - } - } else if (var == 4) { - TPercentileTrackerLg<6, 5, 15> tracker; - for (size_t i = 0; i < 3000000; i += 1) { - size_t num1 = tracker.BucketIdxMostSignificantBit(i); - size_t num2 = tracker.BucketIdxBinarySearch(i); - UNIT_ASSERT_EQUAL(num1, num2); - } - for (size_t i = 0; i < 400000000000ul; i += 1303) { - size_t num1 = tracker.BucketIdxMostSignificantBit(i); - size_t num2 = tracker.BucketIdxBinarySearch(i); - UNIT_ASSERT_EQUAL(num1, num2); - } - } - } - } - - Y_UNIT_TEST(DifferentPercentiles) { - TPercentileTrackerLg<5, 4, 15> tracker; - TVector<size_t> values({0, 115, 1216, 15, 3234567, 1234567, 216546, 263421, 751654, 96, 224, 223, 225}); - TVector<size_t> percentiles50({0, 0, 116, 15, 116, 116, 1216, 1216, 217056, 1216, 1216, 224, 232}); - TVector<size_t> percentiles75({0, 116, 116, 116, 1216, 1245152, 217056, 270304, 753632, 753632, - 270304, 270304, 270304}); - TVector<size_t> percentiles90({ 0, 116, 1216, 1216, 2064352, 1245152, 1245152, 1245152, 1245152, - 1245152, 1245152, 1245152, 1245152}); - TVector<size_t> percentiles100({ 0, 116, 1216, 1216, 2064352, 2064352, 2064352, 2064352, 2064352, - 2064352, 2064352, 2064352, 2064352 }); - - for (size_t i = 0; i < values.size(); ++i) { - tracker.Increment(values[i]); - UNIT_ASSERT_EQUAL(tracker.GetPercentile(0.5), percentiles50[i]); - UNIT_ASSERT_EQUAL(tracker.GetPercentile(0.75), percentiles75[i]); - UNIT_ASSERT_EQUAL(tracker.GetPercentile(0.90), percentiles90[i]); - UNIT_ASSERT_EQUAL(tracker.GetPercentile(1.0), percentiles100[i]); - } - } -} + Y_UNIT_TEST(BucketIdxIfvsBucketIdxBinarySearch) { + for (size_t var = 0; var < 5; var++) { + if (var == 0) { + TPercentileTrackerLg<3, 2, 15> tracker; + for (size_t i = 0; i < 3000000; i += 1) { + size_t num1 = tracker.BucketIdxMostSignificantBit(i); + size_t num2 = tracker.BucketIdxBinarySearch(i); + UNIT_ASSERT_EQUAL(num1, num2); + } + } else if (var == 1) { + TPercentileTrackerLg<4, 4, 15> tracker; + for (size_t i = 0; i < 3000000; i += 1) { + size_t num1 = tracker.BucketIdxMostSignificantBit(i); + size_t num2 = tracker.BucketIdxBinarySearch(i); + UNIT_ASSERT_EQUAL(num1, num2); + } + } else if (var == 2) { + TPercentileTrackerLg<5, 3, 15> tracker; + for (size_t i = 0; i < 3000000; i += 1) { + size_t num1 = tracker.BucketIdxMostSignificantBit(i); + size_t num2 = tracker.BucketIdxBinarySearch(i); + size_t num3 = tracker.BucketIdxIf(i); + UNIT_ASSERT_EQUAL(num1, num2); + UNIT_ASSERT_EQUAL(num2, num3); + } + } else if (var == 3) { + TPercentileTrackerLg<5, 4, 15> tracker; + for (size_t i = 0; i < 3000000; i += 1) { + size_t num1 = tracker.BucketIdxMostSignificantBit(i); + size_t num2 = tracker.BucketIdxBinarySearch(i); + size_t num3 = tracker.BucketIdxIf(i); + UNIT_ASSERT_EQUAL(num1, num2); + UNIT_ASSERT_EQUAL(num2, num3); + } + } else if (var == 4) { + TPercentileTrackerLg<6, 5, 15> tracker; + for (size_t i = 0; i < 3000000; i += 1) { + size_t num1 = tracker.BucketIdxMostSignificantBit(i); + size_t num2 = tracker.BucketIdxBinarySearch(i); + UNIT_ASSERT_EQUAL(num1, num2); + } + for (size_t i = 0; i < 400000000000ul; i += 1303) { + size_t num1 = tracker.BucketIdxMostSignificantBit(i); + size_t num2 = tracker.BucketIdxBinarySearch(i); + UNIT_ASSERT_EQUAL(num1, num2); + } + } + } + } + + Y_UNIT_TEST(DifferentPercentiles) { + TPercentileTrackerLg<5, 4, 15> tracker; + TVector<size_t> values({0, 115, 1216, 15, 3234567, 1234567, 216546, 263421, 751654, 96, 224, 223, 225}); + TVector<size_t> percentiles50({0, 0, 116, 15, 116, 116, 1216, 1216, 217056, 1216, 1216, 224, 232}); + TVector<size_t> percentiles75({0, 116, 116, 116, 1216, 1245152, 217056, 270304, 753632, 753632, + 270304, 270304, 270304}); + TVector<size_t> percentiles90({ 0, 116, 1216, 1216, 2064352, 1245152, 1245152, 1245152, 1245152, + 1245152, 1245152, 1245152, 1245152}); + TVector<size_t> percentiles100({ 0, 116, 1216, 1216, 2064352, 2064352, 2064352, 2064352, 2064352, + 2064352, 2064352, 2064352, 2064352 }); + + for (size_t i = 0; i < values.size(); ++i) { + tracker.Increment(values[i]); + UNIT_ASSERT_EQUAL(tracker.GetPercentile(0.5), percentiles50[i]); + UNIT_ASSERT_EQUAL(tracker.GetPercentile(0.75), percentiles75[i]); + UNIT_ASSERT_EQUAL(tracker.GetPercentile(0.90), percentiles90[i]); + UNIT_ASSERT_EQUAL(tracker.GetPercentile(1.0), percentiles100[i]); + } + } +} diff --git a/library/cpp/monlib/dynamic_counters/percentile/ut/ya.make b/library/cpp/monlib/dynamic_counters/percentile/ut/ya.make index 94faeee43d..f9f3564101 100644 --- a/library/cpp/monlib/dynamic_counters/percentile/ut/ya.make +++ b/library/cpp/monlib/dynamic_counters/percentile/ut/ya.make @@ -1,9 +1,9 @@ UNITTEST_FOR(library/cpp/monlib/dynamic_counters/percentile) - - OWNER(alexvru g:kikimr g:solomon) - - SRCS( - percentile_ut.cpp - ) - -END() + + OWNER(alexvru g:kikimr g:solomon) + + SRCS( + percentile_ut.cpp + ) + +END() diff --git a/library/cpp/monlib/dynamic_counters/percentile/ya.make b/library/cpp/monlib/dynamic_counters/percentile/ya.make index b255fc21fc..cb52cdd9ad 100644 --- a/library/cpp/monlib/dynamic_counters/percentile/ya.make +++ b/library/cpp/monlib/dynamic_counters/percentile/ya.make @@ -1,15 +1,15 @@ -LIBRARY() - - OWNER(alexvru g:kikimr g:solomon) - - SRCS( - percentile.h - percentile_lg.h - ) - - PEERDIR( +LIBRARY() + + OWNER(alexvru g:kikimr g:solomon) + + SRCS( + percentile.h + percentile_lg.h + ) + + PEERDIR( library/cpp/containers/stack_vector library/cpp/monlib/dynamic_counters - ) - -END() + ) + +END() diff --git a/library/cpp/monlib/encode/buffered/buffered_encoder_base.cpp b/library/cpp/monlib/encode/buffered/buffered_encoder_base.cpp index 022e025fa7..87c832d642 100644 --- a/library/cpp/monlib/encode/buffered/buffered_encoder_base.cpp +++ b/library/cpp/monlib/encode/buffered/buffered_encoder_base.cpp @@ -1,29 +1,29 @@ -#include "buffered_encoder_base.h" - +#include "buffered_encoder_base.h" + #include <util/string/join.h> #include <util/string/builder.h> -namespace NMonitoring { - -void TBufferedEncoderBase::OnStreamBegin() { +namespace NMonitoring { + +void TBufferedEncoderBase::OnStreamBegin() { State_.Expect(TEncoderState::EState::ROOT); -} - -void TBufferedEncoderBase::OnStreamEnd() { +} + +void TBufferedEncoderBase::OnStreamEnd() { State_.Expect(TEncoderState::EState::ROOT); -} - -void TBufferedEncoderBase::OnCommonTime(TInstant time) { +} + +void TBufferedEncoderBase::OnCommonTime(TInstant time) { State_.Expect(TEncoderState::EState::ROOT); - CommonTime_ = time; -} - + CommonTime_ = time; +} + void TBufferedEncoderBase::OnMetricBegin(EMetricType type) { State_.Switch(TEncoderState::EState::ROOT, TEncoderState::EState::METRIC); Metrics_.emplace_back(); Metrics_.back().MetricType = type; -} - +} + void TBufferedEncoderBase::OnMetricEnd() { State_.Switch(TEncoderState::EState::METRIC, TEncoderState::EState::ROOT); @@ -41,7 +41,7 @@ void TBufferedEncoderBase::OnMetricEnd() { auto& existing = Metrics_[it->second].TimeSeries; Y_ENSURE(existing.GetValueType() == metric.TimeSeries.GetValueType(), - "Time series point type mismatch: expected " << existing.GetValueType() + "Time series point type mismatch: expected " << existing.GetValueType() << " but found " << metric.TimeSeries.GetValueType() << ", labels '" << FormatLabels(metric.Labels) << "'"); @@ -54,41 +54,41 @@ void TBufferedEncoderBase::OnMetricEnd() { case EMetricsMergingMode::DEFAULT: break; } -} - -void TBufferedEncoderBase::OnLabelsBegin() { +} + +void TBufferedEncoderBase::OnLabelsBegin() { if (State_ == TEncoderState::EState::METRIC) { State_ = TEncoderState::EState::METRIC_LABELS; } else if (State_ == TEncoderState::EState::ROOT) { State_ = TEncoderState::EState::COMMON_LABELS; - } else { + } else { State_.ThrowInvalid("expected METRIC or ROOT"); - } -} - -void TBufferedEncoderBase::OnLabelsEnd() { + } +} + +void TBufferedEncoderBase::OnLabelsEnd() { if (State_ == TEncoderState::EState::METRIC_LABELS) { State_ = TEncoderState::EState::METRIC; } else if (State_ == TEncoderState::EState::COMMON_LABELS) { State_ = TEncoderState::EState::ROOT; - } else { - State_.ThrowInvalid("expected LABELS or COMMON_LABELS"); - } -} - + } else { + State_.ThrowInvalid("expected LABELS or COMMON_LABELS"); + } +} + void TBufferedEncoderBase::OnLabel(TStringBuf name, TStringBuf value) { - TPooledLabels* labels; + TPooledLabels* labels; if (State_ == TEncoderState::EState::METRIC_LABELS) { labels = &Metrics_.back().Labels; } else if (State_ == TEncoderState::EState::COMMON_LABELS) { - labels = &CommonLabels_; - } else { - State_.ThrowInvalid("expected LABELS or COMMON_LABELS"); - } - - labels->emplace_back(LabelNamesPool_.PutIfAbsent(name), LabelValuesPool_.PutIfAbsent(value)); -} - + labels = &CommonLabels_; + } else { + State_.ThrowInvalid("expected LABELS or COMMON_LABELS"); + } + + labels->emplace_back(LabelNamesPool_.PutIfAbsent(name), LabelValuesPool_.PutIfAbsent(value)); +} + void TBufferedEncoderBase::OnLabel(ui32 name, ui32 value) { TPooledLabels* labels; if (State_ == TEncoderState::EState::METRIC_LABELS) { @@ -108,24 +108,24 @@ std::pair<ui32, ui32> TBufferedEncoderBase::PrepareLabel(TStringBuf name, TStrin return std::make_pair(nameLabel->Index, valueLabel->Index); } -void TBufferedEncoderBase::OnDouble(TInstant time, double value) { +void TBufferedEncoderBase::OnDouble(TInstant time, double value) { State_.Expect(TEncoderState::EState::METRIC); TMetric& metric = Metrics_.back(); metric.TimeSeries.Add(time, value); -} - +} + void TBufferedEncoderBase::OnInt64(TInstant time, i64 value) { State_.Expect(TEncoderState::EState::METRIC); TMetric& metric = Metrics_.back(); metric.TimeSeries.Add(time, value); } -void TBufferedEncoderBase::OnUint64(TInstant time, ui64 value) { +void TBufferedEncoderBase::OnUint64(TInstant time, ui64 value) { State_.Expect(TEncoderState::EState::METRIC); TMetric& metric = Metrics_.back(); metric.TimeSeries.Add(time, value); -} - +} + void TBufferedEncoderBase::OnHistogram(TInstant time, IHistogramSnapshotPtr s) { State_.Expect(TEncoderState::EState::METRIC); TMetric& metric = Metrics_.back(); @@ -167,4 +167,4 @@ TString TBufferedEncoderBase::FormatLabels(const TPooledLabels& labels) const { return TStringBuilder() << "{" << JoinSeq(", ", formattedLabels) << "}"; } -} // namespace NMonitoring +} // namespace NMonitoring diff --git a/library/cpp/monlib/encode/buffered/buffered_encoder_base.h b/library/cpp/monlib/encode/buffered/buffered_encoder_base.h index 38f63eac1d..fe3714e58f 100644 --- a/library/cpp/monlib/encode/buffered/buffered_encoder_base.h +++ b/library/cpp/monlib/encode/buffered/buffered_encoder_base.h @@ -1,100 +1,100 @@ -#pragma once - -#include "string_pool.h" - +#pragma once + +#include "string_pool.h" + #include <library/cpp/monlib/encode/encoder.h> #include <library/cpp/monlib/encode/encoder_state.h> #include <library/cpp/monlib/encode/format.h> #include <library/cpp/monlib/metrics/metric_value.h> - -#include <util/datetime/base.h> -#include <util/digest/numeric.h> - - -namespace NMonitoring { - + +#include <util/datetime/base.h> +#include <util/digest/numeric.h> + + +namespace NMonitoring { + class TBufferedEncoderBase : public IMetricEncoder { -public: - void OnStreamBegin() override; - void OnStreamEnd() override; - - void OnCommonTime(TInstant time) override; - +public: + void OnStreamBegin() override; + void OnStreamEnd() override; + + void OnCommonTime(TInstant time) override; + void OnMetricBegin(EMetricType type) override; void OnMetricEnd() override; - - void OnLabelsBegin() override; - void OnLabelsEnd() override; + + void OnLabelsBegin() override; + void OnLabelsEnd() override; void OnLabel(TStringBuf name, TStringBuf value) override; void OnLabel(ui32 name, ui32 value) override; std::pair<ui32, ui32> PrepareLabel(TStringBuf name, TStringBuf value) override; - - void OnDouble(TInstant time, double value) override; + + void OnDouble(TInstant time, double value) override; void OnInt64(TInstant time, i64 value) override; - void OnUint64(TInstant time, ui64 value) override; - + void OnUint64(TInstant time, ui64 value) override; + void OnHistogram(TInstant time, IHistogramSnapshotPtr snapshot) override; void OnSummaryDouble(TInstant time, ISummaryDoubleSnapshotPtr snapshot) override; void OnLogHistogram(TInstant, TLogHistogramSnapshotPtr) override; -protected: - using TPooledStr = TStringPoolBuilder::TValue; - - struct TPooledLabel { - TPooledLabel(const TPooledStr* key, const TPooledStr* value) - : Key{key} - , Value{value} - { - } - - bool operator==(const TPooledLabel& other) const { - return std::tie(Key, Value) == std::tie(other.Key, other.Value); - } - - bool operator!=(const TPooledLabel& other) const { - return !(*this == other); - } - - const TPooledStr* Key; - const TPooledStr* Value; - }; - - using TPooledLabels = TVector<TPooledLabel>; - - struct TPooledLabelsHash { - size_t operator()(const TPooledLabels& val) const { - size_t hash{0}; - - for (auto v : val) { - hash = CombineHashes<size_t>(hash, reinterpret_cast<size_t>(v.Key)); - hash = CombineHashes<size_t>(hash, reinterpret_cast<size_t>(v.Value)); - } - - return hash; - } - }; - +protected: + using TPooledStr = TStringPoolBuilder::TValue; + + struct TPooledLabel { + TPooledLabel(const TPooledStr* key, const TPooledStr* value) + : Key{key} + , Value{value} + { + } + + bool operator==(const TPooledLabel& other) const { + return std::tie(Key, Value) == std::tie(other.Key, other.Value); + } + + bool operator!=(const TPooledLabel& other) const { + return !(*this == other); + } + + const TPooledStr* Key; + const TPooledStr* Value; + }; + + using TPooledLabels = TVector<TPooledLabel>; + + struct TPooledLabelsHash { + size_t operator()(const TPooledLabels& val) const { + size_t hash{0}; + + for (auto v : val) { + hash = CombineHashes<size_t>(hash, reinterpret_cast<size_t>(v.Key)); + hash = CombineHashes<size_t>(hash, reinterpret_cast<size_t>(v.Value)); + } + + return hash; + } + }; + using TMetricMap = THashMap<TPooledLabels, size_t, TPooledLabelsHash>; - + struct TMetric { EMetricType MetricType = EMetricType::UNKNOWN; - TPooledLabels Labels; + TPooledLabels Labels; TMetricTimeSeries TimeSeries; - }; - + }; + protected: TString FormatLabels(const TPooledLabels& labels) const; protected: - TEncoderState State_; - - TStringPoolBuilder LabelNamesPool_; - TStringPoolBuilder LabelValuesPool_; - TInstant CommonTime_ = TInstant::Zero(); - TPooledLabels CommonLabels_; + TEncoderState State_; + + TStringPoolBuilder LabelNamesPool_; + TStringPoolBuilder LabelValuesPool_; + TInstant CommonTime_ = TInstant::Zero(); + TPooledLabels CommonLabels_; TVector<TMetric> Metrics_; TMetricMap MetricMap_; EMetricsMergingMode MetricsMergingMode_ = EMetricsMergingMode::DEFAULT; -}; - -} +}; + +} diff --git a/library/cpp/monlib/encode/buffered/string_pool.cpp b/library/cpp/monlib/encode/buffered/string_pool.cpp index 3c274f27f7..b4c7988ba3 100644 --- a/library/cpp/monlib/encode/buffered/string_pool.cpp +++ b/library/cpp/monlib/encode/buffered/string_pool.cpp @@ -5,16 +5,16 @@ namespace NMonitoring { // TStringPoolBuilder //////////////////////////////////////////////////////////////////////////////// const TStringPoolBuilder::TValue* TStringPoolBuilder::PutIfAbsent(TStringBuf str) { - Y_ENSURE(!IsBuilt_, "Cannot add more values after string has been built"); + Y_ENSURE(!IsBuilt_, "Cannot add more values after string has been built"); - auto [it, isInserted] = StrMap_.try_emplace(str, Max<ui32>(), 0); - if (isInserted) { + auto [it, isInserted] = StrMap_.try_emplace(str, Max<ui32>(), 0); + if (isInserted) { BytesSize_ += str.size(); it->second.Index = StrVector_.size(); StrVector_.emplace_back(it->first, &it->second); } - - TValue* value = &it->second; + + TValue* value = &it->second; ++value->Frequency; return value; } @@ -23,12 +23,12 @@ namespace NMonitoring { return StrVector_.at(index).second; } - TStringPoolBuilder& TStringPoolBuilder::Build() { - if (RequiresSorting_) { - // sort in reversed order - std::sort(StrVector_.begin(), StrVector_.end(), [](auto& a, auto& b) { - return a.second->Frequency > b.second->Frequency; - }); + TStringPoolBuilder& TStringPoolBuilder::Build() { + if (RequiresSorting_) { + // sort in reversed order + std::sort(StrVector_.begin(), StrVector_.end(), [](auto& a, auto& b) { + return a.second->Frequency > b.second->Frequency; + }); ui32 i = 0; for (auto& value : StrVector_) { @@ -36,9 +36,9 @@ namespace NMonitoring { } } - IsBuilt_ = true; - - return *this; + IsBuilt_ = true; + + return *this; } //////////////////////////////////////////////////////////////////////////////// diff --git a/library/cpp/monlib/encode/buffered/string_pool.h b/library/cpp/monlib/encode/buffered/string_pool.h index 39fab1673f..00e5644608 100644 --- a/library/cpp/monlib/encode/buffered/string_pool.h +++ b/library/cpp/monlib/encode/buffered/string_pool.h @@ -9,13 +9,13 @@ namespace NMonitoring { //////////////////////////////////////////////////////////////////////////////// class TStringPoolBuilder { public: - struct TValue: TNonCopyable { - TValue(ui32 idx, ui32 freq) - : Index{idx} - , Frequency{freq} - { - } - + struct TValue: TNonCopyable { + TValue(ui32 idx, ui32 freq) + : Index{idx} + , Frequency{freq} + { + } + ui32 Index; ui32 Frequency; }; @@ -24,16 +24,16 @@ namespace NMonitoring { const TValue* PutIfAbsent(TStringBuf str); const TValue* GetByIndex(ui32 index) const; - /// Determines whether pool must be sorted by value frequencies - TStringPoolBuilder& SetSorted(bool sorted) { - RequiresSorting_ = sorted; - return *this; - } + /// Determines whether pool must be sorted by value frequencies + TStringPoolBuilder& SetSorted(bool sorted) { + RequiresSorting_ = sorted; + return *this; + } + + TStringPoolBuilder& Build(); - TStringPoolBuilder& Build(); - TStringBuf Get(ui32 index) const { - Y_ENSURE(IsBuilt_, "Pool must be sorted first"); + Y_ENSURE(IsBuilt_, "Pool must be sorted first"); return StrVector_.at(index).first; } @@ -43,7 +43,7 @@ namespace NMonitoring { template <typename TConsumer> void ForEach(TConsumer&& c) { - Y_ENSURE(IsBuilt_, "Pool must be sorted first"); + Y_ENSURE(IsBuilt_, "Pool must be sorted first"); for (const auto& value : StrVector_) { c(value.first, value.second->Index, value.second->Frequency); } @@ -60,8 +60,8 @@ namespace NMonitoring { private: THashMap<TString, TValue> StrMap_; TVector<std::pair<TStringBuf, TValue*>> StrVector_; - bool RequiresSorting_ = false; - bool IsBuilt_ = false; + bool RequiresSorting_ = false; + bool IsBuilt_ = false; size_t BytesSize_ = 0; }; diff --git a/library/cpp/monlib/encode/buffered/string_pool_ut.cpp b/library/cpp/monlib/encode/buffered/string_pool_ut.cpp index e597b0c23d..9fc3421d0b 100644 --- a/library/cpp/monlib/encode/buffered/string_pool_ut.cpp +++ b/library/cpp/monlib/encode/buffered/string_pool_ut.cpp @@ -7,8 +7,8 @@ using namespace NMonitoring; Y_UNIT_TEST_SUITE(TStringPoolTest) { Y_UNIT_TEST(PutIfAbsent) { TStringPoolBuilder strPool; - strPool.SetSorted(true); - + strPool.SetSorted(true); + auto* h1 = strPool.PutIfAbsent("one"); auto* h2 = strPool.PutIfAbsent("two"); auto* h3 = strPool.PutIfAbsent("two"); @@ -27,15 +27,15 @@ Y_UNIT_TEST_SUITE(TStringPoolTest) { Y_UNIT_TEST(SortByFrequency) { TStringPoolBuilder strPool; - strPool.SetSorted(true); - + strPool.SetSorted(true); + auto* h1 = strPool.PutIfAbsent("one"); auto* h2 = strPool.PutIfAbsent("two"); auto* h3 = strPool.PutIfAbsent("two"); UNIT_ASSERT(h1 != h2); UNIT_ASSERT(h2 == h3); - strPool.Build(); + strPool.Build(); UNIT_ASSERT_VALUES_EQUAL(h1->Frequency, 1); UNIT_ASSERT_VALUES_EQUAL(h1->Index, 1); @@ -61,7 +61,7 @@ Y_UNIT_TEST_SUITE(TStringPoolTest) { UNIT_ASSERT_VALUES_EQUAL(strPool.BytesSize(), 11); UNIT_ASSERT_VALUES_EQUAL(strPool.Count(), 3); - strPool.Build(); + strPool.Build(); TVector<TString> strings; TVector<ui32> indexes; diff --git a/library/cpp/monlib/encode/buffered/ut/ya.make b/library/cpp/monlib/encode/buffered/ut/ya.make index 3020c035d5..2157ac1490 100644 --- a/library/cpp/monlib/encode/buffered/ut/ya.make +++ b/library/cpp/monlib/encode/buffered/ut/ya.make @@ -1,12 +1,12 @@ UNITTEST_FOR(library/cpp/monlib/encode/buffered) - + OWNER( g:solomon jamel ) - -SRCS( - string_pool_ut.cpp -) - -END() + +SRCS( + string_pool_ut.cpp +) + +END() diff --git a/library/cpp/monlib/encode/buffered/ya.make b/library/cpp/monlib/encode/buffered/ya.make index ae749f5a16..81b6a78b93 100644 --- a/library/cpp/monlib/encode/buffered/ya.make +++ b/library/cpp/monlib/encode/buffered/ya.make @@ -1,19 +1,19 @@ -LIBRARY() - +LIBRARY() + OWNER( g:solomon jamel msherbakov ) - -SRCS( - buffered_encoder_base.cpp - string_pool.cpp -) - -PEERDIR( + +SRCS( + buffered_encoder_base.cpp + string_pool.cpp +) + +PEERDIR( library/cpp/monlib/encode library/cpp/monlib/metrics -) - -END() +) + +END() diff --git a/library/cpp/monlib/encode/fake/fake.cpp b/library/cpp/monlib/encode/fake/fake.cpp index e4d9f44835..69d691361a 100644 --- a/library/cpp/monlib/encode/fake/fake.cpp +++ b/library/cpp/monlib/encode/fake/fake.cpp @@ -1,51 +1,51 @@ -#include "fake.h" - -#include <util/datetime/base.h> - -namespace NMonitoring { +#include "fake.h" + +#include <util/datetime/base.h> + +namespace NMonitoring { class TFakeEncoder: public IMetricEncoder { - public: - void OnStreamBegin() override { - } - void OnStreamEnd() override { - } - - void OnCommonTime(TInstant) override { - } - + public: + void OnStreamBegin() override { + } + void OnStreamEnd() override { + } + + void OnCommonTime(TInstant) override { + } + void OnMetricBegin(EMetricType) override { - } + } void OnMetricEnd() override { - } - - void OnLabelsBegin() override { - } - void OnLabelsEnd() override { - } - void OnLabel(const TStringBuf, const TStringBuf) override { - } - - void OnDouble(TInstant, double) override { - } - void OnInt64(TInstant, i64) override { - } - void OnUint64(TInstant, ui64) override { - } - - void OnHistogram(TInstant, IHistogramSnapshotPtr) override { - } - + } + + void OnLabelsBegin() override { + } + void OnLabelsEnd() override { + } + void OnLabel(const TStringBuf, const TStringBuf) override { + } + + void OnDouble(TInstant, double) override { + } + void OnInt64(TInstant, i64) override { + } + void OnUint64(TInstant, ui64) override { + } + + void OnHistogram(TInstant, IHistogramSnapshotPtr) override { + } + void OnSummaryDouble(TInstant, ISummaryDoubleSnapshotPtr) override { } void OnLogHistogram(TInstant, TLogHistogramSnapshotPtr) override { } - void Close() override { - } - }; - + void Close() override { + } + }; + IMetricEncoderPtr EncoderFake() { return MakeHolder<TFakeEncoder>(); - } -} + } +} diff --git a/library/cpp/monlib/encode/fake/fake.h b/library/cpp/monlib/encode/fake/fake.h index 7874f9bb3e..8109326987 100644 --- a/library/cpp/monlib/encode/fake/fake.h +++ b/library/cpp/monlib/encode/fake/fake.h @@ -1,10 +1,10 @@ -#pragma once - +#pragma once + #include <library/cpp/monlib/encode/encoder.h> - -class IOutputStream; - -namespace NMonitoring { + +class IOutputStream; + +namespace NMonitoring { // Does nothing: just implements IMetricEncoder interface with stubs IMetricEncoderPtr EncoderFake(); -} +} diff --git a/library/cpp/monlib/encode/fake/ya.make b/library/cpp/monlib/encode/fake/ya.make index 0be0f851eb..ae96f45782 100644 --- a/library/cpp/monlib/encode/fake/ya.make +++ b/library/cpp/monlib/encode/fake/ya.make @@ -1,12 +1,12 @@ -LIBRARY() - +LIBRARY() + OWNER( g:solomon msherbakov ) - -SRCS( - fake.cpp -) - -END() + +SRCS( + fake.cpp +) + +END() diff --git a/library/cpp/monlib/encode/format.cpp b/library/cpp/monlib/encode/format.cpp index 8861e3dcc2..400ce5a643 100644 --- a/library/cpp/monlib/encode/format.cpp +++ b/library/cpp/monlib/encode/format.cpp @@ -7,65 +7,65 @@ #include <util/string/cast.h> namespace NMonitoring { - static ECompression CompressionFromHeader(TStringBuf value) { - if (value.empty()) { - return ECompression::UNKNOWN; - } - - for (const auto& it : StringSplitter(value).Split(',').SkipEmpty()) { - TStringBuf token = StripString(it.Token()); - - if (AsciiEqualsIgnoreCase(token, NFormatContentEncoding::IDENTITY)) { - return ECompression::IDENTITY; - } else if (AsciiEqualsIgnoreCase(token, NFormatContentEncoding::ZLIB)) { - return ECompression::ZLIB; - } else if (AsciiEqualsIgnoreCase(token, NFormatContentEncoding::LZ4)) { - return ECompression::LZ4; - } else if (AsciiEqualsIgnoreCase(token, NFormatContentEncoding::ZSTD)) { - return ECompression::ZSTD; - } - } - - return ECompression::UNKNOWN; - } - - static EFormat FormatFromHttpMedia(TStringBuf value) { - if (AsciiEqualsIgnoreCase(value, NFormatContenType::SPACK)) { - return EFormat::SPACK; - } else if (AsciiEqualsIgnoreCase(value, NFormatContenType::JSON)) { - return EFormat::JSON; - } else if (AsciiEqualsIgnoreCase(value, NFormatContenType::PROTOBUF)) { - return EFormat::PROTOBUF; - } else if (AsciiEqualsIgnoreCase(value, NFormatContenType::TEXT)) { - return EFormat::TEXT; + static ECompression CompressionFromHeader(TStringBuf value) { + if (value.empty()) { + return ECompression::UNKNOWN; + } + + for (const auto& it : StringSplitter(value).Split(',').SkipEmpty()) { + TStringBuf token = StripString(it.Token()); + + if (AsciiEqualsIgnoreCase(token, NFormatContentEncoding::IDENTITY)) { + return ECompression::IDENTITY; + } else if (AsciiEqualsIgnoreCase(token, NFormatContentEncoding::ZLIB)) { + return ECompression::ZLIB; + } else if (AsciiEqualsIgnoreCase(token, NFormatContentEncoding::LZ4)) { + return ECompression::LZ4; + } else if (AsciiEqualsIgnoreCase(token, NFormatContentEncoding::ZSTD)) { + return ECompression::ZSTD; + } + } + + return ECompression::UNKNOWN; + } + + static EFormat FormatFromHttpMedia(TStringBuf value) { + if (AsciiEqualsIgnoreCase(value, NFormatContenType::SPACK)) { + return EFormat::SPACK; + } else if (AsciiEqualsIgnoreCase(value, NFormatContenType::JSON)) { + return EFormat::JSON; + } else if (AsciiEqualsIgnoreCase(value, NFormatContenType::PROTOBUF)) { + return EFormat::PROTOBUF; + } else if (AsciiEqualsIgnoreCase(value, NFormatContenType::TEXT)) { + return EFormat::TEXT; } else if (AsciiEqualsIgnoreCase(value, NFormatContenType::PROMETHEUS)) { return EFormat::PROMETHEUS; } - return EFormat::UNKNOWN; - } - - EFormat FormatFromAcceptHeader(TStringBuf value) { - EFormat result{EFormat::UNKNOWN}; - + return EFormat::UNKNOWN; + } + + EFormat FormatFromAcceptHeader(TStringBuf value) { + EFormat result{EFormat::UNKNOWN}; + for (const auto& it : StringSplitter(value).Split(',').SkipEmpty()) { TStringBuf token = StripString(it.Token()); - result = FormatFromHttpMedia(token); - if (result != EFormat::UNKNOWN) { - break; + result = FormatFromHttpMedia(token); + if (result != EFormat::UNKNOWN) { + break; } } - return result; + return result; + } + + EFormat FormatFromContentType(TStringBuf value) { + value = value.NextTok(';'); + + return FormatFromHttpMedia(value); } - EFormat FormatFromContentType(TStringBuf value) { - value = value.NextTok(';'); - - return FormatFromHttpMedia(value); - } - TStringBuf ContentTypeByFormat(EFormat format) { switch (format) { case EFormat::SPACK: @@ -86,11 +86,11 @@ namespace NMonitoring { } ECompression CompressionFromAcceptEncodingHeader(TStringBuf value) { - return CompressionFromHeader(value); - } + return CompressionFromHeader(value); + } - ECompression CompressionFromContentEncodingHeader(TStringBuf value) { - return CompressionFromHeader(value); + ECompression CompressionFromContentEncodingHeader(TStringBuf value) { + return CompressionFromHeader(value); } TStringBuf ContentEncodingByCompression(ECompression compression) { diff --git a/library/cpp/monlib/encode/format.h b/library/cpp/monlib/encode/format.h index e7e5313af6..495d42d786 100644 --- a/library/cpp/monlib/encode/format.h +++ b/library/cpp/monlib/encode/format.h @@ -117,17 +117,17 @@ namespace NMonitoring { * @param value value of the "Accept" header. * @return most preffered serialization format type */ - EFormat FormatFromAcceptHeader(TStringBuf value); + EFormat FormatFromAcceptHeader(TStringBuf value); + + /** + * Matches serialization format by the given "Content-Type" header value + * + * @param value value of the "Content-Type" header + * @return message format + */ + EFormat FormatFromContentType(TStringBuf value); /** - * Matches serialization format by the given "Content-Type" header value - * - * @param value value of the "Content-Type" header - * @return message format - */ - EFormat FormatFromContentType(TStringBuf value); - - /** * Returns value for "Content-Type" header determined by the given * format type. * @@ -146,14 +146,14 @@ namespace NMonitoring { ECompression CompressionFromAcceptEncodingHeader(TStringBuf value); /** - * Matches compression algorithm by the given "Content-Encoding" header value. - * - * @param value value of the "Accept-Encoding" header. - * @return most preffered compression algorithm - */ - ECompression CompressionFromContentEncodingHeader(TStringBuf value); - - /** + * Matches compression algorithm by the given "Content-Encoding" header value. + * + * @param value value of the "Accept-Encoding" header. + * @return most preffered compression algorithm + */ + ECompression CompressionFromContentEncodingHeader(TStringBuf value); + + /** * Returns value for "Content-Encoding" header determined by the given * compression algorithm. * diff --git a/library/cpp/monlib/encode/format_ut.cpp b/library/cpp/monlib/encode/format_ut.cpp index f92f886c31..22a0e30c03 100644 --- a/library/cpp/monlib/encode/format_ut.cpp +++ b/library/cpp/monlib/encode/format_ut.cpp @@ -10,32 +10,32 @@ using namespace NMonitoring; Y_UNIT_TEST_SUITE(TFormatTest) { - Y_UNIT_TEST(ContentTypeHeader) { - UNIT_ASSERT_EQUAL(FormatFromContentType(""), EFormat::UNKNOWN); - UNIT_ASSERT_EQUAL(FormatFromContentType("application/json;some=stuff"), EFormat::JSON); - UNIT_ASSERT_EQUAL(FormatFromContentType("application/x-solomon-spack"), EFormat::SPACK); - UNIT_ASSERT_EQUAL(FormatFromContentType("application/xml"), EFormat::UNKNOWN); - UNIT_ASSERT_EQUAL(FormatFromContentType(";application/xml"), EFormat::UNKNOWN); - } - + Y_UNIT_TEST(ContentTypeHeader) { + UNIT_ASSERT_EQUAL(FormatFromContentType(""), EFormat::UNKNOWN); + UNIT_ASSERT_EQUAL(FormatFromContentType("application/json;some=stuff"), EFormat::JSON); + UNIT_ASSERT_EQUAL(FormatFromContentType("application/x-solomon-spack"), EFormat::SPACK); + UNIT_ASSERT_EQUAL(FormatFromContentType("application/xml"), EFormat::UNKNOWN); + UNIT_ASSERT_EQUAL(FormatFromContentType(";application/xml"), EFormat::UNKNOWN); + } + Y_UNIT_TEST(AcceptHeader) { - UNIT_ASSERT_EQUAL(FormatFromAcceptHeader(""), EFormat::UNKNOWN); - UNIT_ASSERT_EQUAL(FormatFromAcceptHeader("*/*"), EFormat::UNKNOWN); + UNIT_ASSERT_EQUAL(FormatFromAcceptHeader(""), EFormat::UNKNOWN); + UNIT_ASSERT_EQUAL(FormatFromAcceptHeader("*/*"), EFormat::UNKNOWN); UNIT_ASSERT_EQUAL( - FormatFromAcceptHeader("application/xml"), + FormatFromAcceptHeader("application/xml"), EFormat::UNKNOWN); UNIT_ASSERT_EQUAL( - FormatFromAcceptHeader("application/json"), + FormatFromAcceptHeader("application/json"), EFormat::JSON); UNIT_ASSERT_EQUAL( - FormatFromAcceptHeader("application/x-solomon-spack"), + FormatFromAcceptHeader("application/x-solomon-spack"), EFormat::SPACK); UNIT_ASSERT_EQUAL( - FormatFromAcceptHeader("application/x-solomon-pb"), + FormatFromAcceptHeader("application/x-solomon-pb"), EFormat::PROTOBUF); UNIT_ASSERT_EQUAL( @@ -43,19 +43,19 @@ Y_UNIT_TEST_SUITE(TFormatTest) { EFormat::TEXT); UNIT_ASSERT_EQUAL( - FormatFromAcceptHeader("application/json, text/plain"), + FormatFromAcceptHeader("application/json, text/plain"), EFormat::JSON); UNIT_ASSERT_EQUAL( - FormatFromAcceptHeader("application/x-solomon-spack, application/json, text/plain"), + FormatFromAcceptHeader("application/x-solomon-spack, application/json, text/plain"), EFormat::SPACK); UNIT_ASSERT_EQUAL( - FormatFromAcceptHeader(" , application/x-solomon-spack ,, application/json , text/plain"), + FormatFromAcceptHeader(" , application/x-solomon-spack ,, application/json , text/plain"), EFormat::SPACK); UNIT_ASSERT_EQUAL( - FormatFromAcceptHeader("application/xml, application/x-solomon-spack, text/plain"), + FormatFromAcceptHeader("application/xml, application/x-solomon-spack, text/plain"), EFormat::SPACK); UNIT_ASSERT_EQUAL( diff --git a/library/cpp/monlib/encode/fuzz/ya.make b/library/cpp/monlib/encode/fuzz/ya.make index 19f98efb7b..d9ca172bae 100644 --- a/library/cpp/monlib/encode/fuzz/ya.make +++ b/library/cpp/monlib/encode/fuzz/ya.make @@ -1,5 +1,5 @@ -RECURSE_ROOT_RELATIVE( +RECURSE_ROOT_RELATIVE( library/cpp/monlib/encode/json/fuzz library/cpp/monlib/encode/prometheus/fuzz library/cpp/monlib/encode/spack/fuzz -) +) diff --git a/library/cpp/monlib/encode/json/fuzz/main.cpp b/library/cpp/monlib/encode/json/fuzz/main.cpp index b85a973a1b..4f40310e06 100644 --- a/library/cpp/monlib/encode/json/fuzz/main.cpp +++ b/library/cpp/monlib/encode/json/fuzz/main.cpp @@ -1,16 +1,16 @@ #include <library/cpp/monlib/encode/json/json.h> #include <library/cpp/monlib/encode/fake/fake.h> - -#include <util/generic/strbuf.h> - - -extern "C" int LLVMFuzzerTestOneInput(const ui8* data, size_t size) { - auto encoder = NMonitoring::EncoderFake(); - - try { - NMonitoring::DecodeJson({reinterpret_cast<const char*>(data), size}, encoder.Get()); - } catch (...) { - } - - return 0; -} + +#include <util/generic/strbuf.h> + + +extern "C" int LLVMFuzzerTestOneInput(const ui8* data, size_t size) { + auto encoder = NMonitoring::EncoderFake(); + + try { + NMonitoring::DecodeJson({reinterpret_cast<const char*>(data), size}, encoder.Get()); + } catch (...) { + } + + return 0; +} diff --git a/library/cpp/monlib/encode/json/fuzz/ya.make b/library/cpp/monlib/encode/json/fuzz/ya.make index 85a12a37f0..75baa77716 100644 --- a/library/cpp/monlib/encode/json/fuzz/ya.make +++ b/library/cpp/monlib/encode/json/fuzz/ya.make @@ -1,19 +1,19 @@ -FUZZ() - +FUZZ() + OWNER( g:solomon msherbakov ) - -PEERDIR( + +PEERDIR( library/cpp/monlib/encode/json library/cpp/monlib/encode/fake -) - -SIZE(MEDIUM) - +) + +SIZE(MEDIUM) + SRCS( main.cpp ) - -END() + +END() diff --git a/library/cpp/monlib/encode/json/json.h b/library/cpp/monlib/encode/json/json.h index 9f4db4a743..21530f20c3 100644 --- a/library/cpp/monlib/encode/json/json.h +++ b/library/cpp/monlib/encode/json/json.h @@ -13,9 +13,9 @@ namespace NMonitoring { IMetricEncoderPtr EncoderJson(IOutputStream* out, int indentation = 0); - /// Buffered encoder will merge series with same labels into one. + /// Buffered encoder will merge series with same labels into one. IMetricEncoderPtr BufferedEncoderJson(IOutputStream* out, int indentation = 0); - + IMetricEncoderPtr EncoderCloudJson(IOutputStream* out, int indentation = 0, TStringBuf metricNameLabel = "name"); diff --git a/library/cpp/monlib/encode/json/json_decoder.cpp b/library/cpp/monlib/encode/json/json_decoder.cpp index 9468ea6447..d44ff5fd28 100644 --- a/library/cpp/monlib/encode/json/json_decoder.cpp +++ b/library/cpp/monlib/encode/json/json_decoder.cpp @@ -879,7 +879,7 @@ if (Y_UNLIKELY(!(CONDITION))) { \ State_.ToNext(TState::METRIC_LOG_HIST); } break; - + case TState::METRIC_HIST: if (key == TStringBuf("bounds")) { State_.ToNext(TState::METRIC_HIST_BOUNDS); diff --git a/library/cpp/monlib/encode/json/json_encoder.cpp b/library/cpp/monlib/encode/json/json_encoder.cpp index 95c7b2949f..20d2bb6283 100644 --- a/library/cpp/monlib/encode/json/json_encoder.cpp +++ b/library/cpp/monlib/encode/json/json_encoder.cpp @@ -10,8 +10,8 @@ #include <library/cpp/json/writer/json.h> #include <util/charset/utf8.h> -#include <util/generic/algorithm.h> - +#include <util/generic/algorithm.h> + namespace NMonitoring { namespace { enum class EJsonStyle { @@ -22,7 +22,7 @@ namespace NMonitoring { /////////////////////////////////////////////////////////////////////// // TJsonWriter /////////////////////////////////////////////////////////////////////// - class TJsonWriter { + class TJsonWriter { public: TJsonWriter(IOutputStream* out, int indentation, EJsonStyle style, TStringBuf metricNameLabel) : Buf_(NJsonWriter::HEM_UNSAFE, out) @@ -34,32 +34,32 @@ namespace NMonitoring { Buf_.SetWriteNanAsString(); } - void WriteTime(TInstant time) { - if (time != TInstant::Zero()) { + void WriteTime(TInstant time) { + if (time != TInstant::Zero()) { Buf_.WriteKey(TStringBuf("ts")); if (Style_ == EJsonStyle::Solomon) { Buf_.WriteULongLong(time.Seconds()); } else { Buf_.WriteString(time.ToString()); } - } - } - - void WriteValue(double value) { + } + } + + void WriteValue(double value) { Buf_.WriteKey(TStringBuf("value")); - Buf_.WriteDouble(value); - } - + Buf_.WriteDouble(value); + } + void WriteValue(i64 value) { Buf_.WriteKey(TStringBuf("value")); Buf_.WriteLongLong(value); } - void WriteValue(ui64 value) { + void WriteValue(ui64 value) { Buf_.WriteKey(TStringBuf("value")); - Buf_.WriteULongLong(value); - } - + Buf_.WriteULongLong(value); + } + void WriteValue(IHistogramSnapshot* s) { Y_ENSURE(Style_ == EJsonStyle::Solomon); @@ -143,19 +143,19 @@ namespace NMonitoring { } void WriteValue(EMetricValueType type, TMetricValue value) { - switch (type) { + switch (type) { case EMetricValueType::DOUBLE: WriteValue(value.AsDouble()); - break; - + break; + case EMetricValueType::INT64: WriteValue(value.AsInt64()); break; case EMetricValueType::UINT64: WriteValue(value.AsUint64()); - break; - + break; + case EMetricValueType::HISTOGRAM: WriteValue(value.AsHistogram()); break; @@ -170,20 +170,20 @@ namespace NMonitoring { case EMetricValueType::UNKNOWN: ythrow yexception() << "unknown metric value type"; - } - } - - void WriteLabel(TStringBuf name, TStringBuf value) { - Y_ENSURE(IsUtf(name), "label name is not valid UTF-8 string"); - Y_ENSURE(IsUtf(value), "label value is not valid UTF-8 string"); + } + } + + void WriteLabel(TStringBuf name, TStringBuf value) { + Y_ENSURE(IsUtf(name), "label name is not valid UTF-8 string"); + Y_ENSURE(IsUtf(value), "label value is not valid UTF-8 string"); if (Style_ == EJsonStyle::Cloud && name == MetricNameLabel_) { CurrentMetricName_ = value; } else { Buf_.WriteKey(name); Buf_.WriteString(value); } - } - + } + void WriteMetricType(EMetricType type) { if (Style_ == EJsonStyle::Cloud) { Buf_.WriteKey("type"); @@ -222,23 +222,23 @@ namespace NMonitoring { } } - protected: - NJsonWriter::TBuf Buf_; + protected: + NJsonWriter::TBuf Buf_; EJsonStyle Style_; TString MetricNameLabel_; TString CurrentMetricName_; - }; - + }; + /////////////////////////////////////////////////////////////////////// // TEncoderJson /////////////////////////////////////////////////////////////////////// class TEncoderJson final: public IMetricEncoder, public TJsonWriter { - public: + public: TEncoderJson(IOutputStream* out, int indentation, EJsonStyle style, TStringBuf metricNameLabel) : TJsonWriter{out, indentation, style, metricNameLabel} - { - } - + { + } + ~TEncoderJson() override { Close(); } @@ -308,8 +308,8 @@ namespace NMonitoring { State_.ThrowInvalid("expected METRIC or ROOT"); } Buf_.BeginObject(); - - EmptyLabels_ = true; + + EmptyLabels_ = true; } void OnLabelsEnd() override { @@ -320,8 +320,8 @@ namespace NMonitoring { } else { State_.ThrowInvalid("expected LABELS or COMMON_LABELS"); } - - Y_ENSURE(!EmptyLabels_, "Labels cannot be empty"); + + Y_ENSURE(!EmptyLabels_, "Labels cannot be empty"); Buf_.EndObject(); if (State_ == TEncoderState::EState::METRIC) { WriteName(); @@ -330,12 +330,12 @@ namespace NMonitoring { void OnLabel(TStringBuf name, TStringBuf value) override { if (State_ == TEncoderState::EState::METRIC_LABELS || State_ == TEncoderState::EState::COMMON_LABELS) { - WriteLabel(name, value); + WriteLabel(name, value); } else { State_.ThrowInvalid("expected LABELS or COMMON_LABELS"); } - - EmptyLabels_ = false; + + EmptyLabels_ = false; } void OnDouble(TInstant time, double value) override { @@ -385,7 +385,7 @@ namespace NMonitoring { Buf_.BeginList(); Buf_.BeginObject(); Y_ENSURE(LastPoint_.GetTime() != TInstant::Zero(), - "time cannot be empty or zero in a timeseries point"); + "time cannot be empty or zero in a timeseries point"); WriteTime(LastPoint_.GetTime()); WriteValue(LastPoint_.GetValueType(), LastPoint_.GetValue()); Buf_.EndObject(); @@ -394,8 +394,8 @@ namespace NMonitoring { if (TimeSeries_) { Buf_.BeginObject(); - Y_ENSURE(time != TInstant::Zero(), - "time cannot be empty or zero in a timeseries point"); + Y_ENSURE(time != TInstant::Zero(), + "time cannot be empty or zero in a timeseries point"); WriteTime(time); WriteValue(value); @@ -408,26 +408,26 @@ namespace NMonitoring { LastPoint_ = {}; } - private: - TEncoderState State_; + private: + TEncoderState State_; TTypedPoint LastPoint_; - bool TimeSeries_ = false; - bool EmptyLabels_ = false; - }; - + bool TimeSeries_ = false; + bool EmptyLabels_ = false; + }; + /////////////////////////////////////////////////////////////////////// // TBufferedJsonEncoder /////////////////////////////////////////////////////////////////////// - class TBufferedJsonEncoder : public TBufferedEncoderBase, public TJsonWriter { - public: + class TBufferedJsonEncoder : public TBufferedEncoderBase, public TJsonWriter { + public: TBufferedJsonEncoder(IOutputStream* out, int indentation, EJsonStyle style, TStringBuf metricNameLabel) : TJsonWriter{out, indentation, style, metricNameLabel} - { + { MetricsMergingMode_ = EMetricsMergingMode::MERGE_METRICS; } ~TBufferedJsonEncoder() override { - Close(); + Close(); } void OnLabelsBegin() override { @@ -445,54 +445,54 @@ namespace NMonitoring { EmptyLabels_ = false; } - void OnLabelsEnd() override { - TBufferedEncoderBase::OnLabelsEnd(); + void OnLabelsEnd() override { + TBufferedEncoderBase::OnLabelsEnd(); Y_ENSURE(!EmptyLabels_, "Labels cannot be empty"); - } - + } + void Close() final { - if (Closed_) { - return; - } - - Closed_ = true; + if (Closed_) { + return; + } - LabelValuesPool_.Build(); - LabelNamesPool_.Build(); + Closed_ = true; - Buf_.BeginObject(); + LabelValuesPool_.Build(); + LabelNamesPool_.Build(); + + Buf_.BeginObject(); - WriteTime(CommonTime_); - if (CommonLabels_.size() > 0) { + WriteTime(CommonTime_); + if (CommonLabels_.size() > 0) { Buf_.WriteKey(TStringBuf(Style_ == EJsonStyle::Solomon ? "commonLabels": "labels")); WriteLabels(CommonLabels_, true); } - + if (Metrics_.size() > 0) { Buf_.WriteKey(TStringBuf(Style_ == EJsonStyle::Solomon ? "sensors" : "metrics")); WriteMetrics(); - } - - Buf_.EndObject(); + } + + Buf_.EndObject(); } private: void WriteMetrics() { - Buf_.BeginList(); + Buf_.BeginList(); for (auto&& metric : Metrics_) { WriteMetric(metric); - } - Buf_.EndList(); - } - + } + Buf_.EndList(); + } + void WriteMetric(TMetric& metric) { - Buf_.BeginObject(); - + Buf_.BeginObject(); + WriteMetricType(metric.MetricType); - + Buf_.WriteKey(TStringBuf("labels")); WriteLabels(metric.Labels, false); - + metric.TimeSeries.SortByTs(); if (metric.TimeSeries.Size() == 1) { const auto& point = metric.TimeSeries[0]; @@ -500,40 +500,40 @@ namespace NMonitoring { WriteValue(metric.TimeSeries.GetValueType(), point.GetValue()); } else if (metric.TimeSeries.Size() > 1) { Buf_.WriteKey(TStringBuf("timeseries")); - Buf_.BeginList(); + Buf_.BeginList(); metric.TimeSeries.ForEach([this](TInstant time, EMetricValueType type, TMetricValue value) { - Buf_.BeginObject(); - // make gcc 6.1 happy https://gcc.gnu.org/bugzilla/show_bug.cgi?id=61636 + Buf_.BeginObject(); + // make gcc 6.1 happy https://gcc.gnu.org/bugzilla/show_bug.cgi?id=61636 this->WriteTime(time); this->WriteValue(type, value); - Buf_.EndObject(); - }); - - Buf_.EndList(); - } - - Buf_.EndObject(); - } - + Buf_.EndObject(); + }); + + Buf_.EndList(); + } + + Buf_.EndObject(); + } + void WriteLabels(const TPooledLabels& labels, bool isCommon) { - Buf_.BeginObject(); - - for (auto i = 0u; i < labels.size(); ++i) { - TStringBuf name = LabelNamesPool_.Get(labels[i].Key->Index); - TStringBuf value = LabelValuesPool_.Get(labels[i].Value->Index); - - WriteLabel(name, value); - } - - Buf_.EndObject(); + Buf_.BeginObject(); + + for (auto i = 0u; i < labels.size(); ++i) { + TStringBuf name = LabelNamesPool_.Get(labels[i].Key->Index); + TStringBuf value = LabelValuesPool_.Get(labels[i].Value->Index); + + WriteLabel(name, value); + } + + Buf_.EndObject(); if (!isCommon) { WriteName(); } - } - - private: - bool Closed_{false}; + } + + private: + bool Closed_{false}; bool EmptyLabels_ = false; }; } @@ -544,7 +544,7 @@ namespace NMonitoring { IMetricEncoderPtr BufferedEncoderJson(IOutputStream* out, int indentation) { return MakeHolder<TBufferedJsonEncoder>(out, indentation, EJsonStyle::Solomon, ""); - } + } IMetricEncoderPtr EncoderCloudJson(IOutputStream* out, int indentation, TStringBuf metricNameLabel) { return MakeHolder<TEncoderJson>(out, indentation, EJsonStyle::Cloud, metricNameLabel); diff --git a/library/cpp/monlib/encode/json/json_ut.cpp b/library/cpp/monlib/encode/json/json_ut.cpp index b3c3b879d4..09e7909289 100644 --- a/library/cpp/monlib/encode/json/json_ut.cpp +++ b/library/cpp/monlib/encode/json/json_ut.cpp @@ -14,93 +14,93 @@ using namespace NMonitoring; -namespace NMonitoring { - bool operator<(const TLabel& lhs, const TLabel& rhs) { - return lhs.Name() < rhs.Name() || - (lhs.Name() == rhs.Name() && lhs.Value() < rhs.Value()); - } -} -namespace { - void AssertLabels(const NProto::TMultiSample& actual, const TLabels& expected) { - UNIT_ASSERT_EQUAL(actual.LabelsSize(), expected.Size()); - - TSet<TLabel> actualSet; - TSet<TLabel> expectedSet; - Transform(expected.begin(), expected.end(), std::inserter(expectedSet, expectedSet.end()), [] (auto&& l) { - return TLabel{l.Name(), l.Value()}; - }); - - const auto& l = actual.GetLabels(); - Transform(std::begin(l), std::end(l), std::inserter(actualSet, std::begin(actualSet)), - [](auto&& elem) -> TLabel { - return {elem.GetName(), elem.GetValue()}; - }); - - TVector<TLabel> diff; - SetSymmetricDifference(std::begin(expectedSet), std::end(expectedSet), - std::begin(actualSet), std::end(actualSet), std::back_inserter(diff)); - - if (diff.size() > 0) { - for (auto&& l : diff) { - Cerr << l << Endl; - } - - UNIT_FAIL("Labels don't match"); - } - } - - void AssertLabelEqual(const NProto::TLabel& l, TStringBuf name, TStringBuf value) { - UNIT_ASSERT_STRINGS_EQUAL(l.GetName(), name); - UNIT_ASSERT_STRINGS_EQUAL(l.GetValue(), value); - } - - void AssertPointEqual(const NProto::TPoint& p, TInstant time, double value) { - UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds()); - UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kFloat64); - UNIT_ASSERT_DOUBLES_EQUAL(p.GetFloat64(), value, std::numeric_limits<double>::epsilon()); - } - - void AssertPointEqualNan(const NProto::TPoint& p, TInstant time) { - UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds()); - UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kFloat64); - UNIT_ASSERT(std::isnan(p.GetFloat64())); - } - - void AssertPointEqualInf(const NProto::TPoint& p, TInstant time, int sign) { - UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds()); - UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kFloat64); - UNIT_ASSERT(std::isinf(p.GetFloat64())); - if (sign < 0) { - UNIT_ASSERT(p.GetFloat64() < 0); - } - } - - void AssertPointEqual(const NProto::TPoint& p, TInstant time, ui64 value) { - UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds()); - UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kUint64); - UNIT_ASSERT_VALUES_EQUAL(p.GetUint64(), value); - } - - void AssertPointEqual(const NProto::TPoint& p, TInstant time, i64 value) { - UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds()); - UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kInt64); - UNIT_ASSERT_VALUES_EQUAL(p.GetInt64(), value); - } - - void AssertPointEqual(const NProto::TPoint& p, TInstant time, const IHistogramSnapshot& expected) { - UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds()); - UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kHistogram); - - const NProto::THistogram& h = p.GetHistogram(); - UNIT_ASSERT_VALUES_EQUAL(h.BoundsSize(), expected.Count()); - UNIT_ASSERT_VALUES_EQUAL(h.ValuesSize(), expected.Count()); - - for (size_t i = 0; i < h.BoundsSize(); i++) { - UNIT_ASSERT_DOUBLES_EQUAL(h.GetBounds(i), expected.UpperBound(i), Min<double>()); - UNIT_ASSERT_VALUES_EQUAL(h.GetValues(i), expected.Value(i)); - } - } - +namespace NMonitoring { + bool operator<(const TLabel& lhs, const TLabel& rhs) { + return lhs.Name() < rhs.Name() || + (lhs.Name() == rhs.Name() && lhs.Value() < rhs.Value()); + } +} +namespace { + void AssertLabels(const NProto::TMultiSample& actual, const TLabels& expected) { + UNIT_ASSERT_EQUAL(actual.LabelsSize(), expected.Size()); + + TSet<TLabel> actualSet; + TSet<TLabel> expectedSet; + Transform(expected.begin(), expected.end(), std::inserter(expectedSet, expectedSet.end()), [] (auto&& l) { + return TLabel{l.Name(), l.Value()}; + }); + + const auto& l = actual.GetLabels(); + Transform(std::begin(l), std::end(l), std::inserter(actualSet, std::begin(actualSet)), + [](auto&& elem) -> TLabel { + return {elem.GetName(), elem.GetValue()}; + }); + + TVector<TLabel> diff; + SetSymmetricDifference(std::begin(expectedSet), std::end(expectedSet), + std::begin(actualSet), std::end(actualSet), std::back_inserter(diff)); + + if (diff.size() > 0) { + for (auto&& l : diff) { + Cerr << l << Endl; + } + + UNIT_FAIL("Labels don't match"); + } + } + + void AssertLabelEqual(const NProto::TLabel& l, TStringBuf name, TStringBuf value) { + UNIT_ASSERT_STRINGS_EQUAL(l.GetName(), name); + UNIT_ASSERT_STRINGS_EQUAL(l.GetValue(), value); + } + + void AssertPointEqual(const NProto::TPoint& p, TInstant time, double value) { + UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds()); + UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kFloat64); + UNIT_ASSERT_DOUBLES_EQUAL(p.GetFloat64(), value, std::numeric_limits<double>::epsilon()); + } + + void AssertPointEqualNan(const NProto::TPoint& p, TInstant time) { + UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds()); + UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kFloat64); + UNIT_ASSERT(std::isnan(p.GetFloat64())); + } + + void AssertPointEqualInf(const NProto::TPoint& p, TInstant time, int sign) { + UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds()); + UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kFloat64); + UNIT_ASSERT(std::isinf(p.GetFloat64())); + if (sign < 0) { + UNIT_ASSERT(p.GetFloat64() < 0); + } + } + + void AssertPointEqual(const NProto::TPoint& p, TInstant time, ui64 value) { + UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds()); + UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kUint64); + UNIT_ASSERT_VALUES_EQUAL(p.GetUint64(), value); + } + + void AssertPointEqual(const NProto::TPoint& p, TInstant time, i64 value) { + UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds()); + UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kInt64); + UNIT_ASSERT_VALUES_EQUAL(p.GetInt64(), value); + } + + void AssertPointEqual(const NProto::TPoint& p, TInstant time, const IHistogramSnapshot& expected) { + UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds()); + UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kHistogram); + + const NProto::THistogram& h = p.GetHistogram(); + UNIT_ASSERT_VALUES_EQUAL(h.BoundsSize(), expected.Count()); + UNIT_ASSERT_VALUES_EQUAL(h.ValuesSize(), expected.Count()); + + for (size_t i = 0; i < h.BoundsSize(); i++) { + UNIT_ASSERT_DOUBLES_EQUAL(h.GetBounds(i), expected.UpperBound(i), Min<double>()); + UNIT_ASSERT_VALUES_EQUAL(h.GetValues(i), expected.Value(i)); + } + } + void AssertPointEqual(const NProto::TPoint& p, TInstant time, const TLogHistogramSnapshot& expected) { UNIT_ASSERT_VALUES_EQUAL(p.GetTime(), time.MilliSeconds()); UNIT_ASSERT_EQUAL(p.GetValueCase(), NProto::TPoint::kLogHistogram); @@ -129,9 +129,9 @@ namespace { UNIT_ASSERT_VALUES_EQUAL(actual.GetCount(), expected.GetCount()); } -} // namespace - - +} // namespace + + Y_UNIT_TEST_SUITE(TJsonTest) { const TInstant now = TInstant::ParseIso8601Deprecated("2017-11-05T01:02:03Z"); @@ -304,53 +304,53 @@ Y_UNIT_TEST_SUITE(TJsonTest) { } Y_UNIT_TEST(MetricsWithDifferentLabelOrderGetMerged) { - TString json; - TStringOutput out(json); - auto e = BufferedEncoderJson(&out, 2); + TString json; + TStringOutput out(json); + auto e = BufferedEncoderJson(&out, 2); - e->OnStreamBegin(); - { + e->OnStreamBegin(); + { e->OnMetricBegin(EMetricType::RATE); - { - e->OnLabelsBegin(); + { + e->OnLabelsBegin(); e->OnLabel("metric", "hello"); - e->OnLabel("label", "world"); - e->OnLabelsEnd(); - } - e->OnUint64(TInstant::Zero(), 0); + e->OnLabel("label", "world"); + e->OnLabelsEnd(); + } + e->OnUint64(TInstant::Zero(), 0); e->OnMetricEnd(); } - { + { e->OnMetricBegin(EMetricType::RATE); - { - e->OnLabelsBegin(); - e->OnLabel("label", "world"); + { + e->OnLabelsBegin(); + e->OnLabel("label", "world"); e->OnLabel("metric", "hello"); - e->OnLabelsEnd(); - } - e->OnUint64(TInstant::Zero(), 1); + e->OnLabelsEnd(); + } + e->OnUint64(TInstant::Zero(), 1); e->OnMetricEnd(); - } - e->OnStreamEnd(); - e->Close(); - json += "\n"; + } + e->OnStreamEnd(); + e->Close(); + json += "\n"; - TString expectedJson = NResource::Find("/merged.json"); - // we cannot be sure regarding the label order in the result, - // so we'll have to parse the expected value and then compare it with actual + TString expectedJson = NResource::Find("/merged.json"); + // we cannot be sure regarding the label order in the result, + // so we'll have to parse the expected value and then compare it with actual - NProto::TMultiSamplesList samples; + NProto::TMultiSamplesList samples; IMetricEncoderPtr d = EncoderProtobuf(&samples); - DecodeJson(expectedJson, d.Get()); + DecodeJson(expectedJson, d.Get()); - UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 1); - { - const NProto::TMultiSample& s = samples.GetSamples(0); + UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 1); + { + const NProto::TMultiSample& s = samples.GetSamples(0); UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::RATE); AssertLabels(s, TLabels{{"metric", "hello"}, {"label", "world"}}); - UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); - AssertPointEqual(s.GetPoints(0), TInstant::Zero(), ui64(1)); + UNIT_ASSERT_VALUES_EQUAL(s.PointsSize(), 1); + AssertPointEqual(s.GetPoints(0), TInstant::Zero(), ui64(1)); } } Y_UNIT_TEST(Decode1) { @@ -453,7 +453,7 @@ Y_UNIT_TEST_SUITE(TJsonTest) { AssertLabelEqual(samples.GetCommonLabels(1), "cluster", "man"); AssertLabelEqual(samples.GetCommonLabels(2), "service", "stockpile"); - UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 4); + UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 4); { const NProto::TMultiSample& s = samples.GetSamples(0); UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); @@ -484,7 +484,7 @@ Y_UNIT_TEST_SUITE(TJsonTest) { AssertPointEqual(s.GetPoints(0), ts, 3.14159); } { - const NProto::TMultiSample& s = samples.GetSamples(3); + const NProto::TMultiSample& s = samples.GetSamples(3); UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); UNIT_ASSERT_VALUES_EQUAL(s.LabelsSize(), 1); AssertLabelEqual(s.GetLabels(0), "metric", "Writes"); @@ -558,7 +558,7 @@ Y_UNIT_TEST_SUITE(TJsonTest) { AssertPointEqual(s.GetPoints(1), ts2, 20.0); } } - + Y_UNIT_TEST(DecodeToEncoder) { auto testJson = NResource::Find("/test_decode_to_encode.json"); @@ -575,154 +575,154 @@ Y_UNIT_TEST_SUITE(TJsonTest) { } void WriteEmptySeries(const IMetricEncoderPtr& e) { - e->OnStreamBegin(); - { + e->OnStreamBegin(); + { e->OnMetricBegin(EMetricType::COUNTER); - { - e->OnLabelsBegin(); - e->OnLabel("foo", "bar"); - e->OnLabelsEnd(); - } + { + e->OnLabelsBegin(); + e->OnLabel("foo", "bar"); + e->OnLabelsEnd(); + } e->OnMetricEnd(); - } - - e->OnStreamEnd(); - e->Close(); - } - - Y_UNIT_TEST(EncodeEmptySeries) { - TString json; - TStringOutput out(json); - - auto e = EncoderJson(&out, 2); - WriteEmptySeries(e); - json += "\n"; - - TString expectedJson = NResource::Find("/empty_series.json"); - UNIT_ASSERT_NO_DIFF(json, expectedJson); - } - + } + + e->OnStreamEnd(); + e->Close(); + } + + Y_UNIT_TEST(EncodeEmptySeries) { + TString json; + TStringOutput out(json); + + auto e = EncoderJson(&out, 2); + WriteEmptySeries(e); + json += "\n"; + + TString expectedJson = NResource::Find("/empty_series.json"); + UNIT_ASSERT_NO_DIFF(json, expectedJson); + } + void WriteEmptyLabels(IMetricEncoderPtr& e) { - e->OnStreamBegin(); + e->OnStreamBegin(); e->OnMetricBegin(EMetricType::COUNTER); - - e->OnLabelsBegin(); - UNIT_ASSERT_EXCEPTION(e->OnLabelsEnd(), yexception); - } - - Y_UNIT_TEST(LabelsCannotBeEmpty) { - TString json; - TStringOutput out(json); - - auto e = EncoderJson(&out, 2); - WriteEmptyLabels(e); - } - - Y_UNIT_TEST(LabelsCannotBeEmptyBuffered) { - TString json; - TStringOutput out(json); - - auto e = BufferedEncoderJson(&out, 2); - WriteEmptyLabels(e); - } - - Y_UNIT_TEST(EncodeEmptySeriesBuffered) { - TString json; - TStringOutput out(json); - - auto e = BufferedEncoderJson(&out, 2); - WriteEmptySeries(e); - json += "\n"; - - TString expectedJson = NResource::Find("/empty_series.json"); - UNIT_ASSERT_NO_DIFF(json, expectedJson); - } - + + e->OnLabelsBegin(); + UNIT_ASSERT_EXCEPTION(e->OnLabelsEnd(), yexception); + } + + Y_UNIT_TEST(LabelsCannotBeEmpty) { + TString json; + TStringOutput out(json); + + auto e = EncoderJson(&out, 2); + WriteEmptyLabels(e); + } + + Y_UNIT_TEST(LabelsCannotBeEmptyBuffered) { + TString json; + TStringOutput out(json); + + auto e = BufferedEncoderJson(&out, 2); + WriteEmptyLabels(e); + } + + Y_UNIT_TEST(EncodeEmptySeriesBuffered) { + TString json; + TStringOutput out(json); + + auto e = BufferedEncoderJson(&out, 2); + WriteEmptySeries(e); + json += "\n"; + + TString expectedJson = NResource::Find("/empty_series.json"); + UNIT_ASSERT_NO_DIFF(json, expectedJson); + } + Y_UNIT_TEST(BufferedEncoderMergesMetrics) { - TString json; - TStringOutput out(json); - - auto e = BufferedEncoderJson(&out, 2); - auto ts = 1; - + TString json; + TStringOutput out(json); + + auto e = BufferedEncoderJson(&out, 2); + auto ts = 1; + auto writeMetric = [&] (const TString& val) { e->OnMetricBegin(EMetricType::COUNTER); - - e->OnLabelsBegin(); - e->OnLabel("foo", val); - e->OnLabelsEnd(); - e->OnUint64(TInstant::Seconds(ts++), 42); - + + e->OnLabelsBegin(); + e->OnLabel("foo", val); + e->OnLabelsEnd(); + e->OnUint64(TInstant::Seconds(ts++), 42); + e->OnMetricEnd(); - }; - - e->OnStreamBegin(); + }; + + e->OnStreamBegin(); writeMetric("bar"); writeMetric("bar"); writeMetric("baz"); writeMetric("bar"); - e->OnStreamEnd(); - e->Close(); - - json += "\n"; - - TString expectedJson = NResource::Find("/buffered_test.json"); - UNIT_ASSERT_NO_DIFF(json, expectedJson); - } - - Y_UNIT_TEST(JsonEncoderDisallowsValuesInTimeseriesWithoutTs) { - TStringStream out; - - auto e = EncoderJson(&out); - auto writePreamble = [&] { - e->OnStreamBegin(); + e->OnStreamEnd(); + e->Close(); + + json += "\n"; + + TString expectedJson = NResource::Find("/buffered_test.json"); + UNIT_ASSERT_NO_DIFF(json, expectedJson); + } + + Y_UNIT_TEST(JsonEncoderDisallowsValuesInTimeseriesWithoutTs) { + TStringStream out; + + auto e = EncoderJson(&out); + auto writePreamble = [&] { + e->OnStreamBegin(); e->OnMetricBegin(EMetricType::COUNTER); - e->OnLabelsBegin(); - e->OnLabel("foo", "bar"); - e->OnLabelsEnd(); - }; - + e->OnLabelsBegin(); + e->OnLabel("foo", "bar"); + e->OnLabelsEnd(); + }; + // writing two values for a metric in a row will trigger - // timeseries object construction - writePreamble(); - e->OnUint64(TInstant::Zero(), 42); - UNIT_ASSERT_EXCEPTION(e->OnUint64(TInstant::Zero(), 42), yexception); - - e = EncoderJson(&out); - writePreamble(); - e->OnUint64(TInstant::Zero(), 42); - UNIT_ASSERT_EXCEPTION(e->OnUint64(TInstant::Now(), 42), yexception); - - e = EncoderJson(&out); - writePreamble(); - e->OnUint64(TInstant::Now(), 42); - UNIT_ASSERT_EXCEPTION(e->OnUint64(TInstant::Zero(), 42), yexception); - } - - Y_UNIT_TEST(BufferedJsonEncoderMergesTimeseriesWithoutTs) { - TStringStream out; - - { - auto e = BufferedEncoderJson(&out, 2); - e->OnStreamBegin(); + // timeseries object construction + writePreamble(); + e->OnUint64(TInstant::Zero(), 42); + UNIT_ASSERT_EXCEPTION(e->OnUint64(TInstant::Zero(), 42), yexception); + + e = EncoderJson(&out); + writePreamble(); + e->OnUint64(TInstant::Zero(), 42); + UNIT_ASSERT_EXCEPTION(e->OnUint64(TInstant::Now(), 42), yexception); + + e = EncoderJson(&out); + writePreamble(); + e->OnUint64(TInstant::Now(), 42); + UNIT_ASSERT_EXCEPTION(e->OnUint64(TInstant::Zero(), 42), yexception); + } + + Y_UNIT_TEST(BufferedJsonEncoderMergesTimeseriesWithoutTs) { + TStringStream out; + + { + auto e = BufferedEncoderJson(&out, 2); + e->OnStreamBegin(); e->OnMetricBegin(EMetricType::COUNTER); - e->OnLabelsBegin(); - e->OnLabel("foo", "bar"); - e->OnLabelsEnd(); - // in buffered mode we are able to find values with same (in this case zero) - // timestamp and discard duplicates - e->OnUint64(TInstant::Zero(), 42); - e->OnUint64(TInstant::Zero(), 43); - e->OnUint64(TInstant::Zero(), 44); - e->OnUint64(TInstant::Zero(), 45); + e->OnLabelsBegin(); + e->OnLabel("foo", "bar"); + e->OnLabelsEnd(); + // in buffered mode we are able to find values with same (in this case zero) + // timestamp and discard duplicates + e->OnUint64(TInstant::Zero(), 42); + e->OnUint64(TInstant::Zero(), 43); + e->OnUint64(TInstant::Zero(), 44); + e->OnUint64(TInstant::Zero(), 45); e->OnMetricEnd(); - e->OnStreamEnd(); - } - - out << "\n"; - UNIT_ASSERT_NO_DIFF(out.Str(), NResource::Find("/buffered_ts_merge.json")); + e->OnStreamEnd(); + } + + out << "\n"; + UNIT_ASSERT_NO_DIFF(out.Str(), NResource::Find("/buffered_ts_merge.json")); } - + template <typename TFactory, typename TConsumer> TString EncodeToString(TFactory factory, TConsumer consumer) { TStringStream out; @@ -732,7 +732,7 @@ Y_UNIT_TEST_SUITE(TJsonTest) { } out << '\n'; return out.Str(); - } + } Y_UNIT_TEST(SummaryValueEncode) { auto writeDocument = [](IMetricEncoder* e) { @@ -1235,35 +1235,35 @@ Y_UNIT_TEST_SUITE(TJsonTest) { AssertPointEqual(s.GetPoints(2), now + TDuration::Seconds(2), i64(0)); AssertPointEqual(s.GetPoints(3), now + TDuration::Seconds(3), Max<i64>()); } - - Y_UNIT_TEST(FuzzerRegression) { - NProto::TMultiSamplesList samples; - { + + Y_UNIT_TEST(FuzzerRegression) { + NProto::TMultiSamplesList samples; + { IMetricEncoderPtr e = EncoderProtobuf(&samples); - - for (auto f : { "/hist_crash.json", "/crash.json" }) { - TString testJson = NResource::Find(f); - UNIT_ASSERT_EXCEPTION(DecodeJson(testJson, e.Get()), yexception); - } - } - } - - Y_UNIT_TEST(LegacyNegativeRateThrows) { - const auto input = R"({ - "sensors": [ - { - "mode": "deriv", - "value": -1, + + for (auto f : { "/hist_crash.json", "/crash.json" }) { + TString testJson = NResource::Find(f); + UNIT_ASSERT_EXCEPTION(DecodeJson(testJson, e.Get()), yexception); + } + } + } + + Y_UNIT_TEST(LegacyNegativeRateThrows) { + const auto input = R"({ + "sensors": [ + { + "mode": "deriv", + "value": -1, "labels": { "metric": "SystemTime" } - }, - } - ]}")"; - - NProto::TMultiSamplesList samples; + }, + } + ]}")"; + + NProto::TMultiSamplesList samples; IMetricEncoderPtr e = EncoderProtobuf(&samples); - UNIT_ASSERT_EXCEPTION(DecodeJson(input, e.Get()), yexception); - } - + UNIT_ASSERT_EXCEPTION(DecodeJson(input, e.Get()), yexception); + } + Y_UNIT_TEST(DecodeNamedMetrics) { NProto::TMultiSamplesList samples; { diff --git a/library/cpp/monlib/encode/json/ut/buffered_test.json b/library/cpp/monlib/encode/json/ut/buffered_test.json index 580b929de0..53212cf8e1 100644 --- a/library/cpp/monlib/encode/json/ut/buffered_test.json +++ b/library/cpp/monlib/encode/json/ut/buffered_test.json @@ -1,36 +1,36 @@ -{ - "sensors": - [ - { - "kind":"COUNTER", - "labels": - { - "foo":"bar" - }, - "timeseries": - [ - { - "ts":1, - "value":42 - }, - { - "ts":2, - "value":42 - }, - { - "ts":4, - "value":42 - } - ] - }, - { - "kind":"COUNTER", - "labels": - { - "foo":"baz" - }, - "ts":3, - "value":42 - } - ] -} +{ + "sensors": + [ + { + "kind":"COUNTER", + "labels": + { + "foo":"bar" + }, + "timeseries": + [ + { + "ts":1, + "value":42 + }, + { + "ts":2, + "value":42 + }, + { + "ts":4, + "value":42 + } + ] + }, + { + "kind":"COUNTER", + "labels": + { + "foo":"baz" + }, + "ts":3, + "value":42 + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/buffered_ts_merge.json b/library/cpp/monlib/encode/json/ut/buffered_ts_merge.json index e71c9dd206..1d27efacb0 100644 --- a/library/cpp/monlib/encode/json/ut/buffered_ts_merge.json +++ b/library/cpp/monlib/encode/json/ut/buffered_ts_merge.json @@ -1,13 +1,13 @@ -{ - "sensors": - [ - { - "kind":"COUNTER", - "labels": - { - "foo":"bar" - }, +{ + "sensors": + [ + { + "kind":"COUNTER", + "labels": + { + "foo":"bar" + }, "value":45 - } - ] -} + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/empty_series.json b/library/cpp/monlib/encode/json/ut/empty_series.json index 877c957cab..641e10cdea 100644 --- a/library/cpp/monlib/encode/json/ut/empty_series.json +++ b/library/cpp/monlib/encode/json/ut/empty_series.json @@ -1,12 +1,12 @@ -{ - "sensors": - [ - { - "kind":"COUNTER", - "labels": - { - "foo":"bar" - } - } - ] -} +{ + "sensors": + [ + { + "kind":"COUNTER", + "labels": + { + "foo":"bar" + } + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/merged.json b/library/cpp/monlib/encode/json/ut/merged.json index fe39861f95..ea2c99a33c 100644 --- a/library/cpp/monlib/encode/json/ut/merged.json +++ b/library/cpp/monlib/encode/json/ut/merged.json @@ -1,14 +1,14 @@ -{ - "sensors": - [ - { - "kind":"RATE", - "labels": - { +{ + "sensors": + [ + { + "kind":"RATE", + "labels": + { "metric":"hello", - "label":"world" - }, - "value":1 - } - ] -} + "label":"world" + }, + "value":1 + } + ] +} diff --git a/library/cpp/monlib/encode/json/ut/ya.make b/library/cpp/monlib/encode/json/ut/ya.make index 96b4cef0c5..e50c4f4903 100644 --- a/library/cpp/monlib/encode/json/ut/ya.make +++ b/library/cpp/monlib/encode/json/ut/ya.make @@ -11,14 +11,14 @@ SRCS( ) RESOURCE( - buffered_test.json /buffered_test.json - buffered_ts_merge.json /buffered_ts_merge.json + buffered_test.json /buffered_test.json + buffered_ts_merge.json /buffered_ts_merge.json empty_series.json /empty_series.json expected.json /expected.json expected_buffered.json /expected_buffered.json expected_cloud.json /expected_cloud.json expected_cloud_buffered.json /expected_cloud_buffered.json - merged.json /merged.json + merged.json /merged.json histogram_timeseries.json /histogram_timeseries.json histogram_value.json /histogram_value.json histogram_value_inf_before_bounds.json /histogram_value_inf_before_bounds.json @@ -27,8 +27,8 @@ RESOURCE( metrics.json /metrics.json named_metrics.json /named_metrics.json test_decode_to_encode.json /test_decode_to_encode.json - crash.json /crash.json - hist_crash.json /hist_crash.json + crash.json /crash.json + hist_crash.json /hist_crash.json summary_value.json /summary_value.json summary_inf.json /summary_inf.json summary_timeseries.json /summary_timeseries.json diff --git a/library/cpp/monlib/encode/legacy_protobuf/legacy_proto_decoder.cpp b/library/cpp/monlib/encode/legacy_protobuf/legacy_proto_decoder.cpp index 4eba48619c..f87a2d7e8f 100644 --- a/library/cpp/monlib/encode/legacy_protobuf/legacy_proto_decoder.cpp +++ b/library/cpp/monlib/encode/legacy_protobuf/legacy_proto_decoder.cpp @@ -1,527 +1,527 @@ -#include "legacy_protobuf.h" - +#include "legacy_protobuf.h" + #include <library/cpp/monlib/encode/legacy_protobuf/protos/metric_meta.pb.h> #include <library/cpp/monlib/metrics/metric_consumer.h> #include <library/cpp/monlib/metrics/labels.h> - -#include <util/generic/yexception.h> -#include <util/generic/maybe.h> -#include <util/datetime/base.h> + +#include <util/generic/yexception.h> +#include <util/generic/maybe.h> +#include <util/datetime/base.h> #include <util/string/split.h> - + #include <google/protobuf/reflection.h> - -#include <algorithm> - -#ifdef LEGACY_PB_TRACE -#define TRACE(msg) \ - Cerr << msg << Endl -#else -#define TRACE(...) ; -#endif - -namespace NMonitoring { - namespace { + +#include <algorithm> + +#ifdef LEGACY_PB_TRACE +#define TRACE(msg) \ + Cerr << msg << Endl +#else +#define TRACE(...) ; +#endif + +namespace NMonitoring { + namespace { using TMaybeMeta = TMaybe<NMonProto::TMetricMeta>; - - TString ReadLabelValue(const NProtoBuf::Message& msg, const NProtoBuf::FieldDescriptor* d, const NProtoBuf::Reflection& r) { - using namespace NProtoBuf; - - switch (d->type()) { - case FieldDescriptor::TYPE_UINT32: - return ::ToString(r.GetUInt32(msg, d)); - case FieldDescriptor::TYPE_UINT64: - return ::ToString(r.GetUInt64(msg, d)); - case FieldDescriptor::TYPE_STRING: - return r.GetString(msg, d); - case FieldDescriptor::TYPE_ENUM: { - auto val = r.GetEnumValue(msg, d); - auto* valDesc = d->enum_type()->FindValueByNumber(val); - return valDesc->name(); - } - - default: - ythrow yexception() << "type " << d->type_name() << " cannot be used as a field value"; - } - - return {}; - } - - double ReadFieldAsDouble(const NProtoBuf::Message& msg, const NProtoBuf::FieldDescriptor* d, const NProtoBuf::Reflection& r) { - using namespace NProtoBuf; - - switch (d->type()) { - case FieldDescriptor::TYPE_DOUBLE: - return r.GetDouble(msg, d); - case FieldDescriptor::TYPE_BOOL: - return r.GetBool(msg, d) ? 1 : 0; - case FieldDescriptor::TYPE_INT32: - return r.GetInt32(msg, d); - case FieldDescriptor::TYPE_INT64: - return r.GetInt64(msg, d); - case FieldDescriptor::TYPE_UINT32: - return r.GetUInt32(msg, d); - case FieldDescriptor::TYPE_UINT64: - return r.GetUInt64(msg, d); - case FieldDescriptor::TYPE_SINT32: - return r.GetInt32(msg, d); - case FieldDescriptor::TYPE_SINT64: - return r.GetInt64(msg, d); - case FieldDescriptor::TYPE_FIXED32: - return r.GetUInt32(msg, d); - case FieldDescriptor::TYPE_FIXED64: - return r.GetUInt64(msg, d); - case FieldDescriptor::TYPE_SFIXED32: - return r.GetInt32(msg, d); - case FieldDescriptor::TYPE_SFIXED64: - return r.GetInt64(msg, d); - case FieldDescriptor::TYPE_FLOAT: - return r.GetFloat(msg, d); - case FieldDescriptor::TYPE_ENUM: - return r.GetEnumValue(msg, d); - default: - ythrow yexception() << "type " << d->type_name() << " cannot be used as a field value"; - } - - return std::numeric_limits<double>::quiet_NaN(); - } - - double ReadRepeatedAsDouble(const NProtoBuf::Message& msg, const NProtoBuf::FieldDescriptor* d, const NProtoBuf::Reflection& r, size_t i) { - using namespace NProtoBuf; - - switch (d->type()) { - case FieldDescriptor::TYPE_DOUBLE: - return r.GetRepeatedDouble(msg, d, i); - case FieldDescriptor::TYPE_BOOL: - return r.GetRepeatedBool(msg, d, i) ? 1 : 0; - case FieldDescriptor::TYPE_INT32: - return r.GetRepeatedInt32(msg, d, i); - case FieldDescriptor::TYPE_INT64: - return r.GetRepeatedInt64(msg, d, i); - case FieldDescriptor::TYPE_UINT32: - return r.GetRepeatedUInt32(msg, d, i); - case FieldDescriptor::TYPE_UINT64: - return r.GetRepeatedUInt64(msg, d, i); - case FieldDescriptor::TYPE_SINT32: - return r.GetRepeatedInt32(msg, d, i); - case FieldDescriptor::TYPE_SINT64: - return r.GetRepeatedInt64(msg, d, i); - case FieldDescriptor::TYPE_FIXED32: - return r.GetRepeatedUInt32(msg, d, i); - case FieldDescriptor::TYPE_FIXED64: - return r.GetRepeatedUInt64(msg, d, i); - case FieldDescriptor::TYPE_SFIXED32: - return r.GetRepeatedInt32(msg, d, i); - case FieldDescriptor::TYPE_SFIXED64: - return r.GetRepeatedInt64(msg, d, i); - case FieldDescriptor::TYPE_FLOAT: - return r.GetRepeatedFloat(msg, d, i); - case FieldDescriptor::TYPE_ENUM: - return r.GetRepeatedEnumValue(msg, d, i); - default: - ythrow yexception() << "type " << d->type_name() << " cannot be used as a field value"; - } - - return std::numeric_limits<double>::quiet_NaN(); - } - - TString LabelFromField(const NProtoBuf::Message& msg, const TString& name) { - const auto* fieldDesc = msg.GetDescriptor()->FindFieldByName(name); - const auto* reflection = msg.GetReflection(); - Y_ENSURE(fieldDesc && reflection, "Unable to get meta for field " << name); - - auto s = ReadLabelValue(msg, fieldDesc, *reflection); - std::replace(std::begin(s), s.vend(), ' ', '_'); - - return s; - } - - TMaybeMeta MaybeGetMeta(const NProtoBuf::FieldOptions& opts) { + + TString ReadLabelValue(const NProtoBuf::Message& msg, const NProtoBuf::FieldDescriptor* d, const NProtoBuf::Reflection& r) { + using namespace NProtoBuf; + + switch (d->type()) { + case FieldDescriptor::TYPE_UINT32: + return ::ToString(r.GetUInt32(msg, d)); + case FieldDescriptor::TYPE_UINT64: + return ::ToString(r.GetUInt64(msg, d)); + case FieldDescriptor::TYPE_STRING: + return r.GetString(msg, d); + case FieldDescriptor::TYPE_ENUM: { + auto val = r.GetEnumValue(msg, d); + auto* valDesc = d->enum_type()->FindValueByNumber(val); + return valDesc->name(); + } + + default: + ythrow yexception() << "type " << d->type_name() << " cannot be used as a field value"; + } + + return {}; + } + + double ReadFieldAsDouble(const NProtoBuf::Message& msg, const NProtoBuf::FieldDescriptor* d, const NProtoBuf::Reflection& r) { + using namespace NProtoBuf; + + switch (d->type()) { + case FieldDescriptor::TYPE_DOUBLE: + return r.GetDouble(msg, d); + case FieldDescriptor::TYPE_BOOL: + return r.GetBool(msg, d) ? 1 : 0; + case FieldDescriptor::TYPE_INT32: + return r.GetInt32(msg, d); + case FieldDescriptor::TYPE_INT64: + return r.GetInt64(msg, d); + case FieldDescriptor::TYPE_UINT32: + return r.GetUInt32(msg, d); + case FieldDescriptor::TYPE_UINT64: + return r.GetUInt64(msg, d); + case FieldDescriptor::TYPE_SINT32: + return r.GetInt32(msg, d); + case FieldDescriptor::TYPE_SINT64: + return r.GetInt64(msg, d); + case FieldDescriptor::TYPE_FIXED32: + return r.GetUInt32(msg, d); + case FieldDescriptor::TYPE_FIXED64: + return r.GetUInt64(msg, d); + case FieldDescriptor::TYPE_SFIXED32: + return r.GetInt32(msg, d); + case FieldDescriptor::TYPE_SFIXED64: + return r.GetInt64(msg, d); + case FieldDescriptor::TYPE_FLOAT: + return r.GetFloat(msg, d); + case FieldDescriptor::TYPE_ENUM: + return r.GetEnumValue(msg, d); + default: + ythrow yexception() << "type " << d->type_name() << " cannot be used as a field value"; + } + + return std::numeric_limits<double>::quiet_NaN(); + } + + double ReadRepeatedAsDouble(const NProtoBuf::Message& msg, const NProtoBuf::FieldDescriptor* d, const NProtoBuf::Reflection& r, size_t i) { + using namespace NProtoBuf; + + switch (d->type()) { + case FieldDescriptor::TYPE_DOUBLE: + return r.GetRepeatedDouble(msg, d, i); + case FieldDescriptor::TYPE_BOOL: + return r.GetRepeatedBool(msg, d, i) ? 1 : 0; + case FieldDescriptor::TYPE_INT32: + return r.GetRepeatedInt32(msg, d, i); + case FieldDescriptor::TYPE_INT64: + return r.GetRepeatedInt64(msg, d, i); + case FieldDescriptor::TYPE_UINT32: + return r.GetRepeatedUInt32(msg, d, i); + case FieldDescriptor::TYPE_UINT64: + return r.GetRepeatedUInt64(msg, d, i); + case FieldDescriptor::TYPE_SINT32: + return r.GetRepeatedInt32(msg, d, i); + case FieldDescriptor::TYPE_SINT64: + return r.GetRepeatedInt64(msg, d, i); + case FieldDescriptor::TYPE_FIXED32: + return r.GetRepeatedUInt32(msg, d, i); + case FieldDescriptor::TYPE_FIXED64: + return r.GetRepeatedUInt64(msg, d, i); + case FieldDescriptor::TYPE_SFIXED32: + return r.GetRepeatedInt32(msg, d, i); + case FieldDescriptor::TYPE_SFIXED64: + return r.GetRepeatedInt64(msg, d, i); + case FieldDescriptor::TYPE_FLOAT: + return r.GetRepeatedFloat(msg, d, i); + case FieldDescriptor::TYPE_ENUM: + return r.GetRepeatedEnumValue(msg, d, i); + default: + ythrow yexception() << "type " << d->type_name() << " cannot be used as a field value"; + } + + return std::numeric_limits<double>::quiet_NaN(); + } + + TString LabelFromField(const NProtoBuf::Message& msg, const TString& name) { + const auto* fieldDesc = msg.GetDescriptor()->FindFieldByName(name); + const auto* reflection = msg.GetReflection(); + Y_ENSURE(fieldDesc && reflection, "Unable to get meta for field " << name); + + auto s = ReadLabelValue(msg, fieldDesc, *reflection); + std::replace(std::begin(s), s.vend(), ' ', '_'); + + return s; + } + + TMaybeMeta MaybeGetMeta(const NProtoBuf::FieldOptions& opts) { if (opts.HasExtension(NMonProto::Metric)) { return opts.GetExtension(NMonProto::Metric); - } - - return Nothing(); - } - - class ILabelGetter: public TThrRefBase { - public: - enum class EType { - Fixed = 1, - Lazy = 2, - }; - - virtual TLabel Get(const NProtoBuf::Message&) = 0; - virtual EType Type() const = 0; - }; - - class TFixedLabel: public ILabelGetter { - public: - explicit TFixedLabel(TLabel&& l) - : Label_{std::move(l)} - { - TRACE("found fixed label " << l); - } - + } + + return Nothing(); + } + + class ILabelGetter: public TThrRefBase { + public: + enum class EType { + Fixed = 1, + Lazy = 2, + }; + + virtual TLabel Get(const NProtoBuf::Message&) = 0; + virtual EType Type() const = 0; + }; + + class TFixedLabel: public ILabelGetter { + public: + explicit TFixedLabel(TLabel&& l) + : Label_{std::move(l)} + { + TRACE("found fixed label " << l); + } + EType Type() const override { - return EType::Fixed; - } + return EType::Fixed; + } TLabel Get(const NProtoBuf::Message&) override { - return Label_; - } - - private: - TLabel Label_; - }; - - using TFunction = std::function<TLabel(const NProtoBuf::Message&)>; - - class TLazyLabel: public ILabelGetter { - public: - TLazyLabel(TFunction&& fn) - : Fn_{std::move(fn)} - { - TRACE("found lazy label"); - } - + return Label_; + } + + private: + TLabel Label_; + }; + + using TFunction = std::function<TLabel(const NProtoBuf::Message&)>; + + class TLazyLabel: public ILabelGetter { + public: + TLazyLabel(TFunction&& fn) + : Fn_{std::move(fn)} + { + TRACE("found lazy label"); + } + EType Type() const override { - return EType::Lazy; - } + return EType::Lazy; + } TLabel Get(const NProtoBuf::Message& msg) override { - return Fn_(msg); - } - - private: - TFunction Fn_; - }; - - class TDecoderContext { - public: - void Init(const NProtoBuf::Message* msg) { - Message_ = msg; - Y_ENSURE(Message_); - Reflection_ = msg->GetReflection(); - Y_ENSURE(Reflection_); - - for (auto it = Labels_.begin(); it != Labels_.end(); ++it) { - if ((*it)->Type() == ILabelGetter::EType::Lazy) { - auto l = (*it)->Get(Message()); - *it = ::MakeIntrusive<TFixedLabel>(std::move(l)); - } else { - auto l = (*it)->Get(Message()); - } - } - } - - void Clear() noexcept { - Message_ = nullptr; - Reflection_ = nullptr; - } - + return Fn_(msg); + } + + private: + TFunction Fn_; + }; + + class TDecoderContext { + public: + void Init(const NProtoBuf::Message* msg) { + Message_ = msg; + Y_ENSURE(Message_); + Reflection_ = msg->GetReflection(); + Y_ENSURE(Reflection_); + + for (auto it = Labels_.begin(); it != Labels_.end(); ++it) { + if ((*it)->Type() == ILabelGetter::EType::Lazy) { + auto l = (*it)->Get(Message()); + *it = ::MakeIntrusive<TFixedLabel>(std::move(l)); + } else { + auto l = (*it)->Get(Message()); + } + } + } + + void Clear() noexcept { + Message_ = nullptr; + Reflection_ = nullptr; + } + TDecoderContext CreateChildFromMeta(const NMonProto::TMetricMeta& metricMeta, const TString& name, i64 repeatedIdx = -1) { - TDecoderContext child{*this}; - child.Clear(); - + TDecoderContext child{*this}; + child.Clear(); + if (metricMeta.HasCustomPath()) { if (const auto& nodePath = metricMeta.GetCustomPath()) { child.AppendPath(nodePath); } } else if (metricMeta.GetPath()) { - child.AppendPath(name); - } - + child.AppendPath(name); + } + if (metricMeta.HasKeys()) { child.ParseKeys(metricMeta.GetKeys(), repeatedIdx); - } - - return child; - } - + } + + return child; + } + TDecoderContext CreateChildFromRepeatedScalar(const NMonProto::TMetricMeta& metricMeta, i64 repeatedIdx = -1) { - TDecoderContext child{*this}; - child.Clear(); - + TDecoderContext child{*this}; + child.Clear(); + if (metricMeta.HasKeys()) { child.ParseKeys(metricMeta.GetKeys(), repeatedIdx); - } - - return child; - } - + } + + return child; + } + TDecoderContext CreateChildFromEls(const TString& name, const NMonProto::TExtraLabelMetrics& metrics, size_t idx, TMaybeMeta maybeMeta) { - TDecoderContext child{*this}; - child.Clear(); - - auto usePath = [&maybeMeta] { - return !maybeMeta->HasPath() || maybeMeta->GetPath(); - }; - + TDecoderContext child{*this}; + child.Clear(); + + auto usePath = [&maybeMeta] { + return !maybeMeta->HasPath() || maybeMeta->GetPath(); + }; + if (!name.empty() && (!maybeMeta || usePath())) { - child.AppendPath(name); - } - - child.Labels_.push_back(::MakeIntrusive<TLazyLabel>( + child.AppendPath(name); + } + + child.Labels_.push_back(::MakeIntrusive<TLazyLabel>( [ labelName = metrics.GetlabelName(), idx, &metrics ](const auto&) { const auto& val = metrics.Getvalues(idx); - TString labelVal; - const auto uintLabel = val.GetlabelValueUint(); - - if (uintLabel) { - labelVal = ::ToString(uintLabel); - } else { - labelVal = val.GetlabelValue(); - } - - return TLabel{labelName, labelVal}; - })); - - return child; - } - - void ParseKeys(TStringBuf keys, i64 repeatedIdx = -1) { - auto parts = StringSplitter(keys) - .Split(' ') - .SkipEmpty(); - - for (auto part : parts) { - auto str = part.Token(); - - TStringBuf lhs, rhs; - - const bool isDynamic = str.TrySplit(':', lhs, rhs); + TString labelVal; + const auto uintLabel = val.GetlabelValueUint(); + + if (uintLabel) { + labelVal = ::ToString(uintLabel); + } else { + labelVal = val.GetlabelValue(); + } + + return TLabel{labelName, labelVal}; + })); + + return child; + } + + void ParseKeys(TStringBuf keys, i64 repeatedIdx = -1) { + auto parts = StringSplitter(keys) + .Split(' ') + .SkipEmpty(); + + for (auto part : parts) { + auto str = part.Token(); + + TStringBuf lhs, rhs; + + const bool isDynamic = str.TrySplit(':', lhs, rhs); const bool isIndexing = isDynamic && rhs == TStringBuf("#"); - - if (isIndexing) { - TRACE("parsed index labels"); - - // <label_name>:# means that we should use index of the repeated - // field as label value - Y_ENSURE(repeatedIdx != -1); - Labels_.push_back(::MakeIntrusive<TLazyLabel>([=](const auto&) { - return TLabel{lhs, ::ToString(repeatedIdx)}; - })); - } else if (isDynamic) { - TRACE("parsed dynamic labels"); - - // <label_name>:<field_name> means that we need to take label value - // later from message's field - Labels_.push_back(::MakeIntrusive<TLazyLabel>([=](const auto& msg) { - return TLabel{lhs, LabelFromField(msg, TString{rhs})}; - })); - } else if (str.TrySplit('=', lhs, rhs)) { - TRACE("parsed static labels"); - - // <label_name>=<label_value> stands for constant label - Labels_.push_back(::MakeIntrusive<TFixedLabel>(TLabel{lhs, rhs})); - } else { - ythrow yexception() << "Incorrect Keys format"; - } - } - } - - void AppendPath(TStringBuf fieldName) { - Path_ += '/'; - Path_ += fieldName; - } - - const TString& Path() const { - return Path_; - } - - TLabels Labels() const { - TLabels result; - for (auto&& l : Labels_) { - result.Add(l->Get(Message())); - } - - return result; - } - - const NProtoBuf::Message& Message() const { - Y_VERIFY_DEBUG(Message_); - return *Message_; - } - - const NProtoBuf::Reflection& Reflection() const { - return *Reflection_; - } - - private: - const NProtoBuf::Message* Message_{nullptr}; - const NProtoBuf::Reflection* Reflection_{nullptr}; - - TString Path_; - TVector<TIntrusivePtr<ILabelGetter>> Labels_; - }; - - class TDecoder { - public: + + if (isIndexing) { + TRACE("parsed index labels"); + + // <label_name>:# means that we should use index of the repeated + // field as label value + Y_ENSURE(repeatedIdx != -1); + Labels_.push_back(::MakeIntrusive<TLazyLabel>([=](const auto&) { + return TLabel{lhs, ::ToString(repeatedIdx)}; + })); + } else if (isDynamic) { + TRACE("parsed dynamic labels"); + + // <label_name>:<field_name> means that we need to take label value + // later from message's field + Labels_.push_back(::MakeIntrusive<TLazyLabel>([=](const auto& msg) { + return TLabel{lhs, LabelFromField(msg, TString{rhs})}; + })); + } else if (str.TrySplit('=', lhs, rhs)) { + TRACE("parsed static labels"); + + // <label_name>=<label_value> stands for constant label + Labels_.push_back(::MakeIntrusive<TFixedLabel>(TLabel{lhs, rhs})); + } else { + ythrow yexception() << "Incorrect Keys format"; + } + } + } + + void AppendPath(TStringBuf fieldName) { + Path_ += '/'; + Path_ += fieldName; + } + + const TString& Path() const { + return Path_; + } + + TLabels Labels() const { + TLabels result; + for (auto&& l : Labels_) { + result.Add(l->Get(Message())); + } + + return result; + } + + const NProtoBuf::Message& Message() const { + Y_VERIFY_DEBUG(Message_); + return *Message_; + } + + const NProtoBuf::Reflection& Reflection() const { + return *Reflection_; + } + + private: + const NProtoBuf::Message* Message_{nullptr}; + const NProtoBuf::Reflection* Reflection_{nullptr}; + + TString Path_; + TVector<TIntrusivePtr<ILabelGetter>> Labels_; + }; + + class TDecoder { + public: TDecoder(IMetricConsumer* consumer, const NProtoBuf::Message& message, TInstant timestamp) - : Consumer_{consumer} - , Message_{message} - , Timestamp_{timestamp} - { - } - - void Decode() const { - Consumer_->OnStreamBegin(); - DecodeToStream(); - Consumer_->OnStreamEnd(); - } - - void DecodeToStream() const { - DecodeImpl(Message_, {}); - } - - private: + : Consumer_{consumer} + , Message_{message} + , Timestamp_{timestamp} + { + } + + void Decode() const { + Consumer_->OnStreamBegin(); + DecodeToStream(); + Consumer_->OnStreamEnd(); + } + + void DecodeToStream() const { + DecodeImpl(Message_, {}); + } + + private: static const NMonProto::TExtraLabelMetrics& ExtractExtraMetrics(TDecoderContext& ctx, const NProtoBuf::FieldDescriptor& f) { - const auto& parent = ctx.Message(); - const auto& reflection = ctx.Reflection(); - auto& subMessage = reflection.GetMessage(parent, &f); - + const auto& parent = ctx.Message(); + const auto& reflection = ctx.Reflection(); + auto& subMessage = reflection.GetMessage(parent, &f); + return dynamic_cast<const NMonProto::TExtraLabelMetrics&>(subMessage); - } - - void DecodeImpl(const NProtoBuf::Message& msg, TDecoderContext ctx) const { - std::vector<const NProtoBuf::FieldDescriptor*> fields; - - ctx.Init(&msg); - - ctx.Reflection().ListFields(msg, &fields); - - for (const auto* f : fields) { - Y_ENSURE(f); - - const auto& opts = f->options(); - const auto isMessage = f->type() == NProtoBuf::FieldDescriptor::TYPE_MESSAGE; + } + + void DecodeImpl(const NProtoBuf::Message& msg, TDecoderContext ctx) const { + std::vector<const NProtoBuf::FieldDescriptor*> fields; + + ctx.Init(&msg); + + ctx.Reflection().ListFields(msg, &fields); + + for (const auto* f : fields) { + Y_ENSURE(f); + + const auto& opts = f->options(); + const auto isMessage = f->type() == NProtoBuf::FieldDescriptor::TYPE_MESSAGE; const auto isExtraLabelMetrics = isMessage && f->message_type()->full_name() == "NMonProto.TExtraLabelMetrics"; - const auto maybeMeta = MaybeGetMeta(opts); - + const auto maybeMeta = MaybeGetMeta(opts); + if (!(maybeMeta || isExtraLabelMetrics)) { - continue; - } - + continue; + } + if (isExtraLabelMetrics) { const auto& extra = ExtractExtraMetrics(ctx, *f); RecurseExtraLabelMetrics(ctx, extra, f->name(), maybeMeta); - } else if (isMessage) { - RecurseMessage(ctx, *maybeMeta, *f); - } else if (f->is_repeated()) { - RecurseRepeatedScalar(ctx, *maybeMeta, *f); + } else if (isMessage) { + RecurseMessage(ctx, *maybeMeta, *f); + } else if (f->is_repeated()) { + RecurseRepeatedScalar(ctx, *maybeMeta, *f); } else if (maybeMeta->HasType()) { - const auto val = ReadFieldAsDouble(msg, f, ctx.Reflection()); + const auto val = ReadFieldAsDouble(msg, f, ctx.Reflection()); const bool isRate = maybeMeta->GetType() == NMonProto::EMetricType::RATE; WriteMetric(val, ctx, f->name(), isRate); - } - } - } - + } + } + } + void RecurseRepeatedScalar(TDecoderContext ctx, const NMonProto::TMetricMeta& meta, const NProtoBuf::FieldDescriptor& f) const { - auto&& msg = ctx.Message(); - auto&& reflection = ctx.Reflection(); + auto&& msg = ctx.Message(); + auto&& reflection = ctx.Reflection(); const bool isRate = meta.GetType() == NMonProto::EMetricType::RATE; - + // this is a repeated scalar field, which makes metric only if it's indexing - for (auto i = 0; i < reflection.FieldSize(msg, &f); ++i) { - auto subCtx = ctx.CreateChildFromRepeatedScalar(meta, i); - subCtx.Init(&msg); - auto val = ReadRepeatedAsDouble(msg, &f, reflection, i); + for (auto i = 0; i < reflection.FieldSize(msg, &f); ++i) { + auto subCtx = ctx.CreateChildFromRepeatedScalar(meta, i); + subCtx.Init(&msg); + auto val = ReadRepeatedAsDouble(msg, &f, reflection, i); WriteMetric(val, subCtx, f.name(), isRate); - } - } - + } + } + void RecurseExtraLabelMetrics(TDecoderContext ctx, const NMonProto::TExtraLabelMetrics& msg, const TString& name, const TMaybeMeta& meta) const { - auto i = 0; - for (const auto& val : msg.Getvalues()) { - auto subCtx = ctx.CreateChildFromEls(name, msg, i++, meta); - subCtx.Init(&val); - + auto i = 0; + for (const auto& val : msg.Getvalues()) { + auto subCtx = ctx.CreateChildFromEls(name, msg, i++, meta); + subCtx.Init(&val); + const bool isRate = val.Hastype() ? val.Gettype() == NMonProto::EMetricType::RATE : meta->GetType() == NMonProto::EMetricType::RATE; - + double metricVal{0}; if (isRate) { metricVal = val.GetlongValue(); - } else { + } else { metricVal = val.GetdoubleValue(); - } - + } + WriteMetric(metricVal, subCtx, "", isRate); - - for (const auto& child : val.Getchildren()) { + + for (const auto& child : val.Getchildren()) { RecurseExtraLabelMetrics(subCtx, child, "", meta); - } - } - } - + } + } + } + void RecurseMessage(TDecoderContext ctx, const NMonProto::TMetricMeta& metricMeta, const NProtoBuf::FieldDescriptor& f) const { - const auto& msg = ctx.Message(); - const auto& reflection = ctx.Reflection(); - - if (f.is_repeated()) { - TRACE("recurse into repeated message " << f.name()); - for (auto i = 0; i < reflection.FieldSize(msg, &f); ++i) { - auto& subMessage = reflection.GetRepeatedMessage(msg, &f, i); + const auto& msg = ctx.Message(); + const auto& reflection = ctx.Reflection(); + + if (f.is_repeated()) { + TRACE("recurse into repeated message " << f.name()); + for (auto i = 0; i < reflection.FieldSize(msg, &f); ++i) { + auto& subMessage = reflection.GetRepeatedMessage(msg, &f, i); DecodeImpl(subMessage, ctx.CreateChildFromMeta(metricMeta, f.name(), i)); - } - } else { - TRACE("recurse into message " << f.name()); - auto& subMessage = reflection.GetMessage(msg, &f); + } + } else { + TRACE("recurse into message " << f.name()); + auto& subMessage = reflection.GetMessage(msg, &f); DecodeImpl(subMessage, ctx.CreateChildFromMeta(metricMeta, f.name())); - } - } - - inline void WriteValue(ui64 value) const { - Consumer_->OnUint64(Timestamp_, value); - } - - inline void WriteValue(double value) const { - Consumer_->OnDouble(Timestamp_, value); - } - + } + } + + inline void WriteValue(ui64 value) const { + Consumer_->OnUint64(Timestamp_, value); + } + + inline void WriteValue(double value) const { + Consumer_->OnDouble(Timestamp_, value); + } + void WriteMetric(double value, const TDecoderContext& ctx, const TString& name, bool isRate) const { if (isRate) { Consumer_->OnMetricBegin(EMetricType::RATE); - WriteValue(static_cast<ui64>(value)); - } else { + WriteValue(static_cast<ui64>(value)); + } else { Consumer_->OnMetricBegin(EMetricType::GAUGE); - WriteValue(static_cast<double>(value)); - } - - Consumer_->OnLabelsBegin(); - - for (const auto& label : ctx.Labels()) { - Consumer_->OnLabel(label.Name(), label.Value()); - } - + WriteValue(static_cast<double>(value)); + } + + Consumer_->OnLabelsBegin(); + + for (const auto& label : ctx.Labels()) { + Consumer_->OnLabel(label.Name(), label.Value()); + } + const auto fullPath = name.empty() - ? ctx.Path() - : ctx.Path() + '/' + name; - - if (fullPath) { - Consumer_->OnLabel("path", fullPath); - } - - Consumer_->OnLabelsEnd(); + ? ctx.Path() + : ctx.Path() + '/' + name; + + if (fullPath) { + Consumer_->OnLabel("path", fullPath); + } + + Consumer_->OnLabelsEnd(); Consumer_->OnMetricEnd(); - } - - private: + } + + private: IMetricConsumer* Consumer_{nullptr}; - const NProtoBuf::Message& Message_; - TInstant Timestamp_; - }; - - } - + const NProtoBuf::Message& Message_; + TInstant Timestamp_; + }; + + } + void DecodeLegacyProto(const NProtoBuf::Message& data, IMetricConsumer* consumer, TInstant ts) { - Y_ENSURE(consumer); - TDecoder(consumer, data, ts).Decode(); - } - + Y_ENSURE(consumer); + TDecoder(consumer, data, ts).Decode(); + } + void DecodeLegacyProtoToStream(const NProtoBuf::Message& data, IMetricConsumer* consumer, TInstant ts) { - Y_ENSURE(consumer); - TDecoder(consumer, data, ts).DecodeToStream(); - } -} + Y_ENSURE(consumer); + TDecoder(consumer, data, ts).DecodeToStream(); + } +} diff --git a/library/cpp/monlib/encode/legacy_protobuf/legacy_protobuf.h b/library/cpp/monlib/encode/legacy_protobuf/legacy_protobuf.h index c1bd7cd436..7cf8985d65 100644 --- a/library/cpp/monlib/encode/legacy_protobuf/legacy_protobuf.h +++ b/library/cpp/monlib/encode/legacy_protobuf/legacy_protobuf.h @@ -1,16 +1,16 @@ -#pragma once - +#pragma once + #include <google/protobuf/message.h> -#include <util/datetime/base.h> - -namespace NMonitoring { - // Unsupported features of the original format: - // - histograms; - // - memOnly; - // - dropHost/ignorePath - +#include <util/datetime/base.h> + +namespace NMonitoring { + // Unsupported features of the original format: + // - histograms; + // - memOnly; + // - dropHost/ignorePath + void DecodeLegacyProto(const NProtoBuf::Message& data, class IMetricConsumer* c, TInstant ts = TInstant::Zero()); - - /// Does not open/close consumer stream unlike the above function. + + /// Does not open/close consumer stream unlike the above function. void DecodeLegacyProtoToStream(const NProtoBuf::Message& data, class IMetricConsumer* c, TInstant ts = TInstant::Zero()); -} +} diff --git a/library/cpp/monlib/encode/legacy_protobuf/legacy_protobuf_ut.cpp b/library/cpp/monlib/encode/legacy_protobuf/legacy_protobuf_ut.cpp index 140ef1ef58..53683cb39c 100644 --- a/library/cpp/monlib/encode/legacy_protobuf/legacy_protobuf_ut.cpp +++ b/library/cpp/monlib/encode/legacy_protobuf/legacy_protobuf_ut.cpp @@ -1,422 +1,422 @@ -#include "legacy_protobuf.h" - +#include "legacy_protobuf.h" + #include <library/cpp/testing/unittest/registar.h> - + #include <library/cpp/monlib/encode/legacy_protobuf/ut/test_cases.pb.h> #include <library/cpp/monlib/encode/legacy_protobuf/protos/metric_meta.pb.h> - + #include <library/cpp/monlib/encode/protobuf/protobuf.h> #include <library/cpp/monlib/encode/text/text.h> #include <library/cpp/monlib/metrics/labels.h> - -#include <util/generic/algorithm.h> -#include <util/generic/hash_set.h> - -using namespace NMonitoring; - -TSimple MakeSimpleMessage() { - TSimple msg; - - msg.SetFoo(1); - msg.SetBar(2.); - msg.SetBaz(42.); - - return msg; -} - + +#include <util/generic/algorithm.h> +#include <util/generic/hash_set.h> + +using namespace NMonitoring; + +TSimple MakeSimpleMessage() { + TSimple msg; + + msg.SetFoo(1); + msg.SetBar(2.); + msg.SetBaz(42.); + + return msg; +} + IMetricEncoderPtr debugPrinter = EncoderText(&Cerr); - -namespace NMonitoring { - inline bool operator<(const TLabel& lhs, const TLabel& rhs) { - return lhs.Name() < rhs.Name() || - (lhs.Name() == rhs.Name() && lhs.Value() < rhs.Value()); - } - -} - + +namespace NMonitoring { + inline bool operator<(const TLabel& lhs, const TLabel& rhs) { + return lhs.Name() < rhs.Name() || + (lhs.Name() == rhs.Name() && lhs.Value() < rhs.Value()); + } + +} + void SetLabelValue(NMonProto::TExtraLabelMetrics::TValue& val, TString s) { - val.SetlabelValue(s); -} - + val.SetlabelValue(s); +} + void SetLabelValue(NMonProto::TExtraLabelMetrics::TValue& val, ui64 u) { - val.SetlabelValueUint(u); -} - -template <typename T, typename V> + val.SetlabelValueUint(u); +} + +template <typename T, typename V> NMonProto::TExtraLabelMetrics MakeExtra(TString labelName, V labelValue, T value, bool isDeriv) { NMonProto::TExtraLabelMetrics metric; auto* val = metric.Addvalues(); - + metric.SetlabelName(labelName); - SetLabelValue(*val, labelValue); - - if (isDeriv) { - val->SetlongValue(value); - } else { - val->SetdoubleValue(value); - } - + SetLabelValue(*val, labelValue); + + if (isDeriv) { + val->SetlongValue(value); + } else { + val->SetdoubleValue(value); + } + return metric; -} - -void AssertLabels(const TLabels& expected, const NProto::TMultiSample& actual) { - UNIT_ASSERT_EQUAL(actual.LabelsSize(), expected.Size()); - - TSet<TLabel> actualSet; - TSet<TLabel> expectedSet; - Transform(expected.begin(), expected.end(), std::inserter(expectedSet, expectedSet.end()), [] (auto&& l) { - return TLabel{l.Name(), l.Value()}; - }); - - const auto& l = actual.GetLabels(); - Transform(std::begin(l), std::end(l), std::inserter(actualSet, std::begin(actualSet)), - [](auto&& elem) -> TLabel { - return {elem.GetName(), elem.GetValue()}; - }); - - TVector<TLabel> diff; - SetSymmetricDifference(std::begin(expectedSet), std::end(expectedSet), - std::begin(actualSet), std::end(actualSet), std::back_inserter(diff)); - - if (diff.size() > 0) { - for (auto&& l : diff) { - Cerr << l << Endl; - } - - UNIT_FAIL("Labels don't match"); - } -} - -void AssertSimpleMessage(const NProto::TMultiSamplesList& samples, TString pathPrefix = "/") { - UNIT_ASSERT_EQUAL(samples.SamplesSize(), 3); - - THashSet<TString> expectedValues{pathPrefix + "Foo", pathPrefix + "Bar", pathPrefix + "Baz"}; - - for (const auto& s : samples.GetSamples()) { - UNIT_ASSERT_EQUAL(s.LabelsSize(), 1); - UNIT_ASSERT_EQUAL(s.PointsSize(), 1); - - const auto labelVal = s.GetLabels(0).GetValue(); +} + +void AssertLabels(const TLabels& expected, const NProto::TMultiSample& actual) { + UNIT_ASSERT_EQUAL(actual.LabelsSize(), expected.Size()); + + TSet<TLabel> actualSet; + TSet<TLabel> expectedSet; + Transform(expected.begin(), expected.end(), std::inserter(expectedSet, expectedSet.end()), [] (auto&& l) { + return TLabel{l.Name(), l.Value()}; + }); + + const auto& l = actual.GetLabels(); + Transform(std::begin(l), std::end(l), std::inserter(actualSet, std::begin(actualSet)), + [](auto&& elem) -> TLabel { + return {elem.GetName(), elem.GetValue()}; + }); + + TVector<TLabel> diff; + SetSymmetricDifference(std::begin(expectedSet), std::end(expectedSet), + std::begin(actualSet), std::end(actualSet), std::back_inserter(diff)); + + if (diff.size() > 0) { + for (auto&& l : diff) { + Cerr << l << Endl; + } + + UNIT_FAIL("Labels don't match"); + } +} + +void AssertSimpleMessage(const NProto::TMultiSamplesList& samples, TString pathPrefix = "/") { + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 3); + + THashSet<TString> expectedValues{pathPrefix + "Foo", pathPrefix + "Bar", pathPrefix + "Baz"}; + + for (const auto& s : samples.GetSamples()) { + UNIT_ASSERT_EQUAL(s.LabelsSize(), 1); + UNIT_ASSERT_EQUAL(s.PointsSize(), 1); + + const auto labelVal = s.GetLabels(0).GetValue(); UNIT_ASSERT(expectedValues.contains(labelVal)); - - if (labelVal == pathPrefix + "Foo") { + + if (labelVal == pathPrefix + "Foo") { UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); - UNIT_ASSERT_DOUBLES_EQUAL(s.GetPoints(0).GetFloat64(), 1, 1e-6); - } else if (labelVal == pathPrefix + "Bar") { + UNIT_ASSERT_DOUBLES_EQUAL(s.GetPoints(0).GetFloat64(), 1, 1e-6); + } else if (labelVal == pathPrefix + "Bar") { UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::GAUGE); - UNIT_ASSERT_DOUBLES_EQUAL(s.GetPoints(0).GetFloat64(), 2, 1e-6); - } else if (labelVal == pathPrefix + "Baz") { + UNIT_ASSERT_DOUBLES_EQUAL(s.GetPoints(0).GetFloat64(), 2, 1e-6); + } else if (labelVal == pathPrefix + "Baz") { UNIT_ASSERT_EQUAL(s.GetMetricType(), NProto::RATE); - UNIT_ASSERT_EQUAL(s.GetPoints(0).GetUint64(), 42); - } - } -} - -Y_UNIT_TEST_SUITE(TLegacyProtoDecoderTest) { - Y_UNIT_TEST(SimpleProto) { - NProto::TMultiSamplesList samples; + UNIT_ASSERT_EQUAL(s.GetPoints(0).GetUint64(), 42); + } + } +} + +Y_UNIT_TEST_SUITE(TLegacyProtoDecoderTest) { + Y_UNIT_TEST(SimpleProto) { + NProto::TMultiSamplesList samples; IMetricEncoderPtr e = EncoderProtobuf(&samples); - - auto msg = MakeSimpleMessage(); - DecodeLegacyProto(msg, e.Get()); - - AssertSimpleMessage(samples); - }; - - Y_UNIT_TEST(RepeatedProto) { - NProto::TMultiSamplesList samples; + + auto msg = MakeSimpleMessage(); + DecodeLegacyProto(msg, e.Get()); + + AssertSimpleMessage(samples); + }; + + Y_UNIT_TEST(RepeatedProto) { + NProto::TMultiSamplesList samples; IMetricEncoderPtr e = EncoderProtobuf(&samples); - - auto simple = MakeSimpleMessage(); - TRepeated msg; - msg.AddMessages()->CopyFrom(simple); - - DecodeLegacyProto(msg, e.Get()); - - AssertSimpleMessage(samples); - } - - Y_UNIT_TEST(RepeatedProtoWithPath) { - NProto::TMultiSamplesList samples; + + auto simple = MakeSimpleMessage(); + TRepeated msg; + msg.AddMessages()->CopyFrom(simple); + + DecodeLegacyProto(msg, e.Get()); + + AssertSimpleMessage(samples); + } + + Y_UNIT_TEST(RepeatedProtoWithPath) { + NProto::TMultiSamplesList samples; IMetricEncoderPtr e = EncoderProtobuf(&samples); - - auto simple = MakeSimpleMessage(); - TRepeatedWithPath msg; - msg.AddNamespace()->CopyFrom(simple); - - DecodeLegacyProto(msg, e.Get()); - - AssertSimpleMessage(samples, "/Namespace/"); - } - - Y_UNIT_TEST(DeepNesting) { - NProto::TMultiSamplesList samples; + + auto simple = MakeSimpleMessage(); + TRepeatedWithPath msg; + msg.AddNamespace()->CopyFrom(simple); + + DecodeLegacyProto(msg, e.Get()); + + AssertSimpleMessage(samples, "/Namespace/"); + } + + Y_UNIT_TEST(DeepNesting) { + NProto::TMultiSamplesList samples; IMetricEncoderPtr e = EncoderProtobuf(&samples); - - auto simple = MakeSimpleMessage(); - TRepeatedWithPath internal; - internal.AddNamespace()->CopyFrom(simple); - - TDeepNesting msg; - msg.MutableNested()->CopyFrom(internal); - - DecodeLegacyProto(msg, e.Get()); - - AssertSimpleMessage(samples, "/Namespace/"); - } - - Y_UNIT_TEST(Keys) { - NProto::TMultiSamplesList samples; + + auto simple = MakeSimpleMessage(); + TRepeatedWithPath internal; + internal.AddNamespace()->CopyFrom(simple); + + TDeepNesting msg; + msg.MutableNested()->CopyFrom(internal); + + DecodeLegacyProto(msg, e.Get()); + + AssertSimpleMessage(samples, "/Namespace/"); + } + + Y_UNIT_TEST(Keys) { + NProto::TMultiSamplesList samples; IMetricEncoderPtr e = EncoderProtobuf(&samples); - - auto simple = MakeSimpleMessage(); - simple.SetLabel("my_label_value"); - - TNestedWithKeys msg; - msg.AddNamespace()->CopyFrom(simple); - - DecodeLegacyProto(msg, e.Get()); - - auto i = 0; - for (const auto& s : samples.GetSamples()) { - UNIT_ASSERT_EQUAL(s.LabelsSize(), 4); - - bool foundLabel = false; - bool foundFixed = false; - bool foundNumbered = false; - - for (const auto& label : s.GetLabels()) { - if (label.GetName() == "my_label") { - foundLabel = true; - UNIT_ASSERT_STRINGS_EQUAL(label.GetValue(), "my_label_value"); - } else if (label.GetName() == "fixed_label") { - foundFixed = true; - UNIT_ASSERT_STRINGS_EQUAL(label.GetValue(), "fixed_value"); - } else if (label.GetName() == "numbered") { - foundNumbered = true; - UNIT_ASSERT_STRINGS_EQUAL(label.GetValue(), ::ToString(i)); - } - } - - UNIT_ASSERT(foundLabel); - UNIT_ASSERT(foundFixed); - UNIT_ASSERT(foundNumbered); - } - } - - Y_UNIT_TEST(NonStringKeys) { - NProto::TMultiSamplesList samples; + + auto simple = MakeSimpleMessage(); + simple.SetLabel("my_label_value"); + + TNestedWithKeys msg; + msg.AddNamespace()->CopyFrom(simple); + + DecodeLegacyProto(msg, e.Get()); + + auto i = 0; + for (const auto& s : samples.GetSamples()) { + UNIT_ASSERT_EQUAL(s.LabelsSize(), 4); + + bool foundLabel = false; + bool foundFixed = false; + bool foundNumbered = false; + + for (const auto& label : s.GetLabels()) { + if (label.GetName() == "my_label") { + foundLabel = true; + UNIT_ASSERT_STRINGS_EQUAL(label.GetValue(), "my_label_value"); + } else if (label.GetName() == "fixed_label") { + foundFixed = true; + UNIT_ASSERT_STRINGS_EQUAL(label.GetValue(), "fixed_value"); + } else if (label.GetName() == "numbered") { + foundNumbered = true; + UNIT_ASSERT_STRINGS_EQUAL(label.GetValue(), ::ToString(i)); + } + } + + UNIT_ASSERT(foundLabel); + UNIT_ASSERT(foundFixed); + UNIT_ASSERT(foundNumbered); + } + } + + Y_UNIT_TEST(NonStringKeys) { + NProto::TMultiSamplesList samples; IMetricEncoderPtr e = EncoderProtobuf(&samples); - - TNonStringKeys msg; - msg.SetFoo(42); - msg.SetEnum(ENUM); - msg.SetInt(43); - - TRepeatedNonStringKeys msgs; - msgs.AddNested()->CopyFrom(msg); - - DecodeLegacyProto(msgs, e.Get()); - - for (const auto& s : samples.GetSamples()) { - bool foundEnum = false; - bool foundInt = false; - - for (const auto& label : s.GetLabels()) { - if (label.GetName() == "enum") { - foundEnum = true; - UNIT_ASSERT_STRINGS_EQUAL(label.GetValue(), "ENUM"); - } else if (label.GetName() == "int") { - foundInt = true; - UNIT_ASSERT_STRINGS_EQUAL(label.GetValue(), "43"); - } - } - - UNIT_ASSERT(foundEnum); - UNIT_ASSERT(foundInt); - - UNIT_ASSERT_DOUBLES_EQUAL(s.GetPoints(0).GetFloat64(), 42, 1e-6); - } - } - - Y_UNIT_TEST(KeysFromNonLeafNodes) { - NProto::TMultiSamplesList samples; + + TNonStringKeys msg; + msg.SetFoo(42); + msg.SetEnum(ENUM); + msg.SetInt(43); + + TRepeatedNonStringKeys msgs; + msgs.AddNested()->CopyFrom(msg); + + DecodeLegacyProto(msgs, e.Get()); + + for (const auto& s : samples.GetSamples()) { + bool foundEnum = false; + bool foundInt = false; + + for (const auto& label : s.GetLabels()) { + if (label.GetName() == "enum") { + foundEnum = true; + UNIT_ASSERT_STRINGS_EQUAL(label.GetValue(), "ENUM"); + } else if (label.GetName() == "int") { + foundInt = true; + UNIT_ASSERT_STRINGS_EQUAL(label.GetValue(), "43"); + } + } + + UNIT_ASSERT(foundEnum); + UNIT_ASSERT(foundInt); + + UNIT_ASSERT_DOUBLES_EQUAL(s.GetPoints(0).GetFloat64(), 42, 1e-6); + } + } + + Y_UNIT_TEST(KeysFromNonLeafNodes) { + NProto::TMultiSamplesList samples; IMetricEncoderPtr e = EncoderProtobuf(&samples); - - auto simple = MakeSimpleMessage(); - simple.SetLabel("label_value"); - - TRepeatedWithName nested; - nested.SetName("my_name"); - nested.AddNested()->CopyFrom(simple); - - TKeysFromNonLeaf msg; - msg.AddNested()->CopyFrom(nested); - - DecodeLegacyProto(msg, e.Get()); - - AssertLabels({{"my_label", "label_value"}, {"path", "/Nested/Nested/Foo"}, {"name", "my_name"}}, samples.GetSamples(0)); - } - - Y_UNIT_TEST(SpacesAreGetReplaced) { - NProto::TMultiSamplesList samples; + + auto simple = MakeSimpleMessage(); + simple.SetLabel("label_value"); + + TRepeatedWithName nested; + nested.SetName("my_name"); + nested.AddNested()->CopyFrom(simple); + + TKeysFromNonLeaf msg; + msg.AddNested()->CopyFrom(nested); + + DecodeLegacyProto(msg, e.Get()); + + AssertLabels({{"my_label", "label_value"}, {"path", "/Nested/Nested/Foo"}, {"name", "my_name"}}, samples.GetSamples(0)); + } + + Y_UNIT_TEST(SpacesAreGetReplaced) { + NProto::TMultiSamplesList samples; IMetricEncoderPtr e = EncoderProtobuf(&samples); - - auto simple = MakeSimpleMessage(); - simple.SetLabel("my label_value"); - - TNestedWithKeys msg; - msg.AddNamespace()->CopyFrom(simple); - - DecodeLegacyProto(msg, e.Get()); - - for (const auto& s : samples.GetSamples()) { - UNIT_ASSERT_EQUAL(s.LabelsSize(), 4); - - bool foundLabel = false; - - for (const auto& label : s.GetLabels()) { - if (label.GetName() == "my_label") { - foundLabel = true; - UNIT_ASSERT_STRINGS_EQUAL(label.GetValue(), "my_label_value"); - } - } - - UNIT_ASSERT(foundLabel); - } - } - - Y_UNIT_TEST(ExtraLabels) { - NProto::TMultiSamplesList samples; + + auto simple = MakeSimpleMessage(); + simple.SetLabel("my label_value"); + + TNestedWithKeys msg; + msg.AddNamespace()->CopyFrom(simple); + + DecodeLegacyProto(msg, e.Get()); + + for (const auto& s : samples.GetSamples()) { + UNIT_ASSERT_EQUAL(s.LabelsSize(), 4); + + bool foundLabel = false; + + for (const auto& label : s.GetLabels()) { + if (label.GetName() == "my_label") { + foundLabel = true; + UNIT_ASSERT_STRINGS_EQUAL(label.GetValue(), "my_label_value"); + } + } + + UNIT_ASSERT(foundLabel); + } + } + + Y_UNIT_TEST(ExtraLabels) { + NProto::TMultiSamplesList samples; IMetricEncoderPtr e = EncoderProtobuf(&samples); - - TExtraLabels msg; - msg.MutableExtraAsIs()->CopyFrom(MakeExtra("label", "foo", 42, false)); - msg.MutableExtraDeriv()->CopyFrom(MakeExtra("deriv_label", "deriv_foo", 43, true)); - - DecodeLegacyProto(msg, e.Get()); - - UNIT_ASSERT_EQUAL(samples.SamplesSize(), 2); - { - auto s = samples.GetSamples(0); - AssertLabels({{"label", "foo"}, {"path", "/ExtraAsIs"}}, s); - - UNIT_ASSERT_EQUAL(s.PointsSize(), 1); - auto point = s.GetPoints(0); - UNIT_ASSERT_DOUBLES_EQUAL(point.GetFloat64(), 42, 1e-6); - } - - { - auto s = samples.GetSamples(1); - AssertLabels({{"deriv_label", "deriv_foo"}, {"path", "/ExtraDeriv"}}, s); - - UNIT_ASSERT_EQUAL(s.PointsSize(), 1); - auto point = s.GetPoints(0); - UNIT_ASSERT_EQUAL(point.GetUint64(), 43); - } - } - - Y_UNIT_TEST(NestedExtraLabels) { - NProto::TMultiSamplesList samples; + + TExtraLabels msg; + msg.MutableExtraAsIs()->CopyFrom(MakeExtra("label", "foo", 42, false)); + msg.MutableExtraDeriv()->CopyFrom(MakeExtra("deriv_label", "deriv_foo", 43, true)); + + DecodeLegacyProto(msg, e.Get()); + + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 2); + { + auto s = samples.GetSamples(0); + AssertLabels({{"label", "foo"}, {"path", "/ExtraAsIs"}}, s); + + UNIT_ASSERT_EQUAL(s.PointsSize(), 1); + auto point = s.GetPoints(0); + UNIT_ASSERT_DOUBLES_EQUAL(point.GetFloat64(), 42, 1e-6); + } + + { + auto s = samples.GetSamples(1); + AssertLabels({{"deriv_label", "deriv_foo"}, {"path", "/ExtraDeriv"}}, s); + + UNIT_ASSERT_EQUAL(s.PointsSize(), 1); + auto point = s.GetPoints(0); + UNIT_ASSERT_EQUAL(point.GetUint64(), 43); + } + } + + Y_UNIT_TEST(NestedExtraLabels) { + NProto::TMultiSamplesList samples; IMetricEncoderPtr e = EncoderProtobuf(&samples); - - TExtraLabels msg; - auto extra = MakeExtra("label", "foo", 42, false); - auto* val = extra.Mutablevalues(0); - { - auto child = MakeExtra("child1", "label1", 24, true); + + TExtraLabels msg; + auto extra = MakeExtra("label", "foo", 42, false); + auto* val = extra.Mutablevalues(0); + { + auto child = MakeExtra("child1", "label1", 24, true); child.Mutablevalues(0)->Settype(NMonProto::EMetricType::RATE); - val->Addchildren()->CopyFrom(child); - } - - { - auto child = MakeExtra("child2", 34, 23, false); - val->Addchildren()->CopyFrom(child); - } - msg.MutableExtraAsIs()->CopyFrom(extra); - - DecodeLegacyProto(msg, e.Get()); - - UNIT_ASSERT_EQUAL(samples.SamplesSize(), 3); - { - auto s = samples.GetSamples(0); - AssertLabels({{"label", "foo"}, {"path", "/ExtraAsIs"}}, s); - - UNIT_ASSERT_EQUAL(s.PointsSize(), 1); - auto point = s.GetPoints(0); - UNIT_ASSERT_DOUBLES_EQUAL(point.GetFloat64(), 42, 1e-6); - } - - { - auto s = samples.GetSamples(1); - AssertLabels({{"label", "foo"}, {"child1", "label1"}, {"path", "/ExtraAsIs"}}, s); - - UNIT_ASSERT_EQUAL(s.PointsSize(), 1); - auto point = s.GetPoints(0); - UNIT_ASSERT_EQUAL(point.GetUint64(), 24); - } - - { - auto s = samples.GetSamples(2); - AssertLabels({{"label", "foo"}, {"child2", "34"}, {"path", "/ExtraAsIs"}}, s); - - UNIT_ASSERT_EQUAL(s.PointsSize(), 1); - auto point = s.GetPoints(0); - UNIT_ASSERT_EQUAL(point.GetFloat64(), 23); - } - } - - Y_UNIT_TEST(RobotLabels) { - NProto::TMultiSamplesList samples; + val->Addchildren()->CopyFrom(child); + } + + { + auto child = MakeExtra("child2", 34, 23, false); + val->Addchildren()->CopyFrom(child); + } + msg.MutableExtraAsIs()->CopyFrom(extra); + + DecodeLegacyProto(msg, e.Get()); + + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 3); + { + auto s = samples.GetSamples(0); + AssertLabels({{"label", "foo"}, {"path", "/ExtraAsIs"}}, s); + + UNIT_ASSERT_EQUAL(s.PointsSize(), 1); + auto point = s.GetPoints(0); + UNIT_ASSERT_DOUBLES_EQUAL(point.GetFloat64(), 42, 1e-6); + } + + { + auto s = samples.GetSamples(1); + AssertLabels({{"label", "foo"}, {"child1", "label1"}, {"path", "/ExtraAsIs"}}, s); + + UNIT_ASSERT_EQUAL(s.PointsSize(), 1); + auto point = s.GetPoints(0); + UNIT_ASSERT_EQUAL(point.GetUint64(), 24); + } + + { + auto s = samples.GetSamples(2); + AssertLabels({{"label", "foo"}, {"child2", "34"}, {"path", "/ExtraAsIs"}}, s); + + UNIT_ASSERT_EQUAL(s.PointsSize(), 1); + auto point = s.GetPoints(0); + UNIT_ASSERT_EQUAL(point.GetFloat64(), 23); + } + } + + Y_UNIT_TEST(RobotLabels) { + NProto::TMultiSamplesList samples; IMetricEncoderPtr e = EncoderProtobuf(&samples); - - TNamedCounter responses; - responses.SetName("responses"); - responses.SetCount(42); - - TCrawlerCounters::TStatusCounters statusCounters; - statusCounters.AddZoraResponses()->CopyFrom(responses); - - TCrawlerCounters::TPolicyCounters policyCounters; - policyCounters.SetSubComponent("mySubComponent"); - policyCounters.SetName("myComponentName"); - policyCounters.SetZone("myComponentZone"); - policyCounters.MutableStatusCounters()->CopyFrom(statusCounters); - - TCrawlerCounters counters; - counters.SetComponent("myComponent"); - counters.AddPoliciesCounters()->CopyFrom(policyCounters); - - DecodeLegacyProto(counters, e.Get()); - UNIT_ASSERT_EQUAL(samples.SamplesSize(), 1); - auto s = samples.GetSamples(0); - AssertLabels({ - {"SubComponent", "mySubComponent"}, {"Policy", "myComponentName"}, {"Zone", "myComponentZone"}, - {"ZoraResponse", "responses"}, {"path", "/PoliciesCounters/StatusCounters/ZoraResponses/Count"}}, s); - } - - Y_UNIT_TEST(ZoraLabels) { - NProto::TMultiSamplesList samples; + + TNamedCounter responses; + responses.SetName("responses"); + responses.SetCount(42); + + TCrawlerCounters::TStatusCounters statusCounters; + statusCounters.AddZoraResponses()->CopyFrom(responses); + + TCrawlerCounters::TPolicyCounters policyCounters; + policyCounters.SetSubComponent("mySubComponent"); + policyCounters.SetName("myComponentName"); + policyCounters.SetZone("myComponentZone"); + policyCounters.MutableStatusCounters()->CopyFrom(statusCounters); + + TCrawlerCounters counters; + counters.SetComponent("myComponent"); + counters.AddPoliciesCounters()->CopyFrom(policyCounters); + + DecodeLegacyProto(counters, e.Get()); + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 1); + auto s = samples.GetSamples(0); + AssertLabels({ + {"SubComponent", "mySubComponent"}, {"Policy", "myComponentName"}, {"Zone", "myComponentZone"}, + {"ZoraResponse", "responses"}, {"path", "/PoliciesCounters/StatusCounters/ZoraResponses/Count"}}, s); + } + + Y_UNIT_TEST(ZoraLabels) { + NProto::TMultiSamplesList samples; IMetricEncoderPtr e = EncoderProtobuf(&samples); - - TTimeLogHist hist; - hist.AddBuckets(42); - hist.AddBuckets(0); - - TKiwiCounters counters; - counters.MutableTimes()->CopyFrom(hist); - - DecodeLegacyProto(counters, e.Get()); - - UNIT_ASSERT_EQUAL(samples.SamplesSize(), 2); - - auto s = samples.GetSamples(0); - AssertLabels({{"slot", "0"}, {"path", "/Times/Buckets"}}, s); - UNIT_ASSERT_EQUAL(s.PointsSize(), 1); - UNIT_ASSERT_EQUAL(s.GetPoints(0).GetUint64(), 42); - - s = samples.GetSamples(1); - AssertLabels({{"slot", "1"}, {"path", "/Times/Buckets"}}, s); - UNIT_ASSERT_EQUAL(s.GetPoints(0).GetUint64(), 0); - } -} + + TTimeLogHist hist; + hist.AddBuckets(42); + hist.AddBuckets(0); + + TKiwiCounters counters; + counters.MutableTimes()->CopyFrom(hist); + + DecodeLegacyProto(counters, e.Get()); + + UNIT_ASSERT_EQUAL(samples.SamplesSize(), 2); + + auto s = samples.GetSamples(0); + AssertLabels({{"slot", "0"}, {"path", "/Times/Buckets"}}, s); + UNIT_ASSERT_EQUAL(s.PointsSize(), 1); + UNIT_ASSERT_EQUAL(s.GetPoints(0).GetUint64(), 42); + + s = samples.GetSamples(1); + AssertLabels({{"slot", "1"}, {"path", "/Times/Buckets"}}, s); + UNIT_ASSERT_EQUAL(s.GetPoints(0).GetUint64(), 0); + } +} diff --git a/library/cpp/monlib/encode/legacy_protobuf/protos/ya.make b/library/cpp/monlib/encode/legacy_protobuf/protos/ya.make index 01505f61ca..489f361ab1 100644 --- a/library/cpp/monlib/encode/legacy_protobuf/protos/ya.make +++ b/library/cpp/monlib/encode/legacy_protobuf/protos/ya.make @@ -1,13 +1,13 @@ -PROTO_LIBRARY() - -OWNER(g:solomon) - -SRCS( +PROTO_LIBRARY() + +OWNER(g:solomon) + +SRCS( metric_meta.proto -) - +) + IF (NOT PY_PROTOS_FOR) EXCLUDE_TAGS(GO_PROTO) ENDIF() -END() +END() diff --git a/library/cpp/monlib/encode/legacy_protobuf/ut/test_cases.proto b/library/cpp/monlib/encode/legacy_protobuf/ut/test_cases.proto index de870cb9e2..37e901de48 100644 --- a/library/cpp/monlib/encode/legacy_protobuf/ut/test_cases.proto +++ b/library/cpp/monlib/encode/legacy_protobuf/ut/test_cases.proto @@ -1,90 +1,90 @@ import "library/cpp/monlib/encode/legacy_protobuf/protos/metric_meta.proto"; - -message TSimple { + +message TSimple { optional uint64 Foo = 1 [ (NMonProto.Metric).Type = GAUGE ]; optional double Bar = 2 [ (NMonProto.Metric).Type = GAUGE ]; optional double Baz = 3 [ (NMonProto.Metric).Type = RATE ]; - optional string Label = 4; -} - -message TRepeated { + optional string Label = 4; +} + +message TRepeated { repeated TSimple Messages = 1 [ (NMonProto.Metric).Path = false ]; -}; - -message TRepeatedWithPath { +}; + +message TRepeatedWithPath { repeated TSimple Namespace = 1 [ (NMonProto.Metric).Path = true ]; -}; - -message TNestedWithKeys { +}; + +message TNestedWithKeys { repeated TSimple Namespace = 1 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "my_label:Label fixed_label=fixed_value numbered:#" ]; -}; - -message TDeepNesting { +}; + +message TDeepNesting { optional TRepeatedWithPath Nested = 1 [ (NMonProto.Metric).Path = false ]; -}; - -enum EEnum { - MY = 1; - ENUM = 2; -}; - -message TNonStringKeys { +}; + +enum EEnum { + MY = 1; + ENUM = 2; +}; + +message TNonStringKeys { optional uint32 Foo = 1 [ (NMonProto.Metric).Type = GAUGE ]; - optional EEnum Enum = 2; - optional uint32 Int = 3; -}; - -message TRepeatedNonStringKeys { + optional EEnum Enum = 2; + optional uint32 Int = 3; +}; + +message TRepeatedNonStringKeys { repeated TNonStringKeys Nested = 1 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "enum:Enum int:Int" ]; -}; - -message TExtraLabels { +}; + +message TExtraLabels { optional NMonProto.TExtraLabelMetrics ExtraAsIs = 1 [ (NMonProto.Metric).Type = GAUGE ]; optional NMonProto.TExtraLabelMetrics ExtraDeriv = 2 [ (NMonProto.Metric).Type = RATE ]; -}; - -message TRepeatedWithName { - optional string Name = 1; +}; + +message TRepeatedWithName { + optional string Name = 1; repeated TSimple Nested = 2 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "my_label:Label" ]; -}; - -message TKeysFromNonLeaf { +}; + +message TKeysFromNonLeaf { repeated TRepeatedWithName Nested = 1 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "name:Name" ]; -}; - - -message TNamedCounter { - optional string Name = 1; +}; + + +message TNamedCounter { + optional string Name = 1; optional uint64 Count = 2 [ (NMonProto.Metric).Type = RATE ]; -} - -message TCrawlerCounters { - message TStatusCounters { +} + +message TCrawlerCounters { + message TStatusCounters { repeated TNamedCounter ZoraResponses = 3 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "ZoraResponse:Name" ]; repeated TNamedCounter FetcherResponses = 4 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "SpiderResponse:Name" ]; repeated TNamedCounter RotorResponses = 5 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "SpiderResponse:Name" ]; repeated TNamedCounter PDFetchResponses = 6 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "PDFetchResponse:Name" ]; repeated TNamedCounter CalcResponses = 7 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "CalcResponse:Name" ]; repeated TNamedCounter Responses = 8 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "AggregatedResponse:Name" ]; - } - - message TPolicyCounters { - optional string SubComponent = 1; - optional string Name = 2; - optional string Zone = 3; - + } + + message TPolicyCounters { + optional string SubComponent = 1; + optional string Name = 2; + optional string Zone = 3; + optional TStatusCounters StatusCounters = 4 [ (NMonProto.Metric).Path = true ]; - } - - optional string Component = 1; + } + + optional string Component = 1; repeated TPolicyCounters PoliciesCounters = 3 [ (NMonProto.Metric).Path = true, (NMonProto.Metric).Keys = "SubComponent:SubComponent Policy:Name Zone:Zone" ]; -} - -message TTimeLogHist { - optional uint32 MinBucketMillisec = 1; +} + +message TTimeLogHist { + optional uint32 MinBucketMillisec = 1; repeated uint64 Buckets = 2 [ (NMonProto.Metric).Type = RATE, (NMonProto.Metric).Keys = "slot:#" ]; -} - -message TKiwiCounters { +} + +message TKiwiCounters { optional TTimeLogHist Times = 22 [ (NMonProto.Metric).Path = true ]; -} +} diff --git a/library/cpp/monlib/encode/legacy_protobuf/ut/ya.make b/library/cpp/monlib/encode/legacy_protobuf/ut/ya.make index f066b6c327..479a0c46c9 100644 --- a/library/cpp/monlib/encode/legacy_protobuf/ut/ya.make +++ b/library/cpp/monlib/encode/legacy_protobuf/ut/ya.make @@ -1,18 +1,18 @@ UNITTEST_FOR(library/cpp/monlib/encode/legacy_protobuf) - + OWNER( g:solomon msherbakov ) - -SRCS( - legacy_protobuf_ut.cpp - test_cases.proto -) - + +SRCS( + legacy_protobuf_ut.cpp + test_cases.proto +) + PEERDIR( library/cpp/monlib/encode/protobuf library/cpp/monlib/encode/text ) - -END() + +END() diff --git a/library/cpp/monlib/encode/legacy_protobuf/ya.make b/library/cpp/monlib/encode/legacy_protobuf/ya.make index 16a8ebb1f3..74c82aac93 100644 --- a/library/cpp/monlib/encode/legacy_protobuf/ya.make +++ b/library/cpp/monlib/encode/legacy_protobuf/ya.make @@ -1,16 +1,16 @@ -LIBRARY() - +LIBRARY() + OWNER( g:solomon msherbakov ) - -SRCS( - legacy_proto_decoder.cpp -) - -PEERDIR( + +SRCS( + legacy_proto_decoder.cpp +) + +PEERDIR( library/cpp/monlib/encode/legacy_protobuf/protos -) - -END() +) + +END() diff --git a/library/cpp/monlib/encode/prometheus/prometheus_decoder.cpp b/library/cpp/monlib/encode/prometheus/prometheus_decoder.cpp index 2195d9e125..7e81357dbd 100644 --- a/library/cpp/monlib/encode/prometheus/prometheus_decoder.cpp +++ b/library/cpp/monlib/encode/prometheus/prometheus_decoder.cpp @@ -46,24 +46,24 @@ namespace NMonitoring { return sb; } - template <typename T, typename U> - bool TryStaticCast(U val, T& out) { - static_assert(std::is_arithmetic_v<U>); + template <typename T, typename U> + bool TryStaticCast(U val, T& out) { + static_assert(std::is_arithmetic_v<U>); if constexpr (std::is_floating_point_v<T> || std::is_floating_point_v<U>) { if (val > MaxFloor<T>() || val < -MaxFloor<T>()) { - return false; - } - - } else { - if (val > Max<T>() || val < Min<T>()) { - return false; - } - } - - out = static_cast<T>(val); - return true; - } - + return false; + } + + } else { + if (val > Max<T>() || val < Min<T>()) { + return false; + } + } + + out = static_cast<T>(val); + return true; + } + /////////////////////////////////////////////////////////////////////// // THistogramBuilder /////////////////////////////////////////////////////////////////////// @@ -298,9 +298,9 @@ namespace NMonitoring { HistogramBuilder_.SetName(baseName); } - TBucketValue bucketVal; - Y_PARSER_ENSURE(TryStaticCast(value, bucketVal), "Cannot convert " << value << " to bucket value type"); - HistogramBuilder_.AddBucket(bound, bucketVal); + TBucketValue bucketVal; + Y_PARSER_ENSURE(TryStaticCast(value, bucketVal), "Cannot convert " << value << " to bucket value type"); + HistogramBuilder_.AddBucket(bound, bucketVal); HistogramBuilder_.SetTime(time); HistogramBuilder_.SetLabels(std::move(labels)); } else if (NPrometheus::IsCount(name)) { @@ -398,7 +398,7 @@ namespace NMonitoring { CurrentByte_ = Data_[CurrentPos_++]; } - Y_FORCE_INLINE bool IsSpace(char ch) { + Y_FORCE_INLINE bool IsSpace(char ch) { return ch == ' ' || ch == '\t'; } @@ -414,7 +414,7 @@ namespace NMonitoring { } void SkipSpaces() { - while (HasRemaining() && IsSpace(CurrentByte_)) { + while (HasRemaining() && IsSpace(CurrentByte_)) { ReadNextByteUnsafe(); } } @@ -428,7 +428,7 @@ namespace NMonitoring { TStringBuf ReadToken() { Y_VERIFY_DEBUG(CurrentPos_ > 0); size_t begin = CurrentPos_ - 1; // read first byte again - while (HasRemaining() && !IsSpace(CurrentByte_) && CurrentByte_ != '\n') { + while (HasRemaining() && !IsSpace(CurrentByte_) && CurrentByte_ != '\n') { ReadNextByteUnsafe(); } return TokenFromPos(begin); @@ -529,11 +529,11 @@ namespace NMonitoring { } void ConsumeCounter(TStringBuf name, const TLabelsMap& labels, TInstant time, double value) { - i64 intValue{0}; - // not nan - if (value == value) { - Y_PARSER_ENSURE(TryStaticCast(value, intValue), "value " << value << " is out of range"); - } + i64 intValue{0}; + // not nan + if (value == value) { + Y_PARSER_ENSURE(TryStaticCast(value, intValue), "value " << value << " is out of range"); + } // see https://st.yandex-team.ru/SOLOMON-4142 for more details // why we convert Prometheus COUNTER into Solomon RATE diff --git a/library/cpp/monlib/encode/prometheus/prometheus_encoder.cpp b/library/cpp/monlib/encode/prometheus/prometheus_encoder.cpp index db7f76978c..15efeb8c03 100644 --- a/library/cpp/monlib/encode/prometheus/prometheus_encoder.cpp +++ b/library/cpp/monlib/encode/prometheus/prometheus_encoder.cpp @@ -126,7 +126,7 @@ namespace NMonitoring { void WriteLabels(const TLabels& labels, TStringBuf addLabelKey, TStringBuf addLabelValue) { Out_->Write('{'); - for (auto&& l: labels) { + for (auto&& l: labels) { Out_->Write(l.Name()); Out_->Write('='); WriteLabelValue(l.Value()); @@ -354,7 +354,7 @@ namespace NMonitoring { } // XXX: poor performace - for (auto&& l: CommonLabels_) { + for (auto&& l: CommonLabels_) { MetricState_.Labels.Add(l.Name(), l.Value()); } @@ -363,7 +363,7 @@ namespace NMonitoring { "labels " << MetricState_.Labels << " does not contain label '" << MetricNameLabel_ << '\''); - const TString& metricName = ToString(nameLabel->Value()); + const TString& metricName = ToString(nameLabel->Value()); if (MetricState_.Type != EMetricType::DSUMMARY) { Writer_.WriteType(MetricState_.Type, metricName); } diff --git a/library/cpp/monlib/encode/protobuf/protobuf_encoder.cpp b/library/cpp/monlib/encode/protobuf/protobuf_encoder.cpp index 9ed5136efa..2d11b9d5ba 100644 --- a/library/cpp/monlib/encode/protobuf/protobuf_encoder.cpp +++ b/library/cpp/monlib/encode/protobuf/protobuf_encoder.cpp @@ -15,9 +15,9 @@ namespace NMonitoring { case EMetricType::IGAUGE: return NProto::IGAUGE; case EMetricType::HIST: - return NProto::HISTOGRAM; + return NProto::HISTOGRAM; case EMetricType::HIST_RATE: - return NProto::HIST_RATE; + return NProto::HIST_RATE; case EMetricType::DSUMMARY: return NProto::DSUMMARY; case EMetricType::LOGHIST: diff --git a/library/cpp/monlib/encode/protobuf/protos/samples.proto b/library/cpp/monlib/encode/protobuf/protos/samples.proto index 8653d0a06e..371f4181d2 100644 --- a/library/cpp/monlib/encode/protobuf/protos/samples.proto +++ b/library/cpp/monlib/encode/protobuf/protos/samples.proto @@ -18,7 +18,7 @@ enum EMetricType { COUNTER = 3; RATE = 4; HISTOGRAM = 5; - HIST_RATE = 6; + HIST_RATE = 6; DSUMMARY = 7; LOGHISTOGRAM = 8; } diff --git a/library/cpp/monlib/encode/spack/compression.cpp b/library/cpp/monlib/encode/spack/compression.cpp index 922e11d6ad..0d2152fc85 100644 --- a/library/cpp/monlib/encode/spack/compression.cpp +++ b/library/cpp/monlib/encode/spack/compression.cpp @@ -1,12 +1,12 @@ #include "compression.h" -#include <util/generic/buffer.h> +#include <util/generic/buffer.h> #include <util/generic/cast.h> #include <util/generic/ptr.h> #include <util/generic/scope.h> -#include <util/generic/size_literals.h> -#include <util/stream/format.h> -#include <util/stream/output.h> +#include <util/generic/size_literals.h> +#include <util/stream/format.h> +#include <util/stream/output.h> #include <util/stream/walk.h> #include <contrib/libs/lz4/lz4.h> @@ -24,10 +24,10 @@ namespace NMonitoring { using TUncompressedSize = ui32; using TCheckSum = ui32; - constexpr size_t COMPRESSED_FRAME_SIZE_LIMIT = 512_KB; - constexpr size_t UNCOMPRESSED_FRAME_SIZE_LIMIT = COMPRESSED_FRAME_SIZE_LIMIT; - constexpr size_t FRAME_SIZE_LIMIT = 2_MB; - constexpr size_t DEFAULT_FRAME_LEN = 64_KB; + constexpr size_t COMPRESSED_FRAME_SIZE_LIMIT = 512_KB; + constexpr size_t UNCOMPRESSED_FRAME_SIZE_LIMIT = COMPRESSED_FRAME_SIZE_LIMIT; + constexpr size_t FRAME_SIZE_LIMIT = 2_MB; + constexpr size_t DEFAULT_FRAME_LEN = 64_KB; struct Y_PACKED TFrameHeader { TCompressedSize CompressedSize; @@ -46,7 +46,7 @@ namespace NMonitoring { TBlock(T&& t) : TStringBuf(t.data(), t.size()) { - Y_ENSURE(t.data() != nullptr); + Y_ENSURE(t.data() != nullptr); } char* data() noexcept { @@ -208,14 +208,14 @@ namespace NMonitoring { return 0; } - Y_ENSURE(header.CompressedSize <= COMPRESSED_FRAME_SIZE_LIMIT, "Compressed frame size is limited to " - << HumanReadableSize(COMPRESSED_FRAME_SIZE_LIMIT, SF_BYTES) - << " but is " << HumanReadableSize(header.CompressedSize, SF_BYTES)); - - Y_ENSURE(header.UncompressedSize <= UNCOMPRESSED_FRAME_SIZE_LIMIT, "Uncompressed frame size is limited to " - << HumanReadableSize(UNCOMPRESSED_FRAME_SIZE_LIMIT, SF_BYTES) - << " but is " << HumanReadableSize(header.UncompressedSize, SF_BYTES)); - + Y_ENSURE(header.CompressedSize <= COMPRESSED_FRAME_SIZE_LIMIT, "Compressed frame size is limited to " + << HumanReadableSize(COMPRESSED_FRAME_SIZE_LIMIT, SF_BYTES) + << " but is " << HumanReadableSize(header.CompressedSize, SF_BYTES)); + + Y_ENSURE(header.UncompressedSize <= UNCOMPRESSED_FRAME_SIZE_LIMIT, "Uncompressed frame size is limited to " + << HumanReadableSize(UNCOMPRESSED_FRAME_SIZE_LIMIT, SF_BYTES) + << " but is " << HumanReadableSize(header.UncompressedSize, SF_BYTES)); + Compressed_.Resize(header.CompressedSize); In_->LoadOrFail(Compressed_.Data(), header.CompressedSize); @@ -307,13 +307,13 @@ namespace NMonitoring { void WriteCompressedFrame() { static const auto framePayload = sizeof(TFrameHeader) + sizeof(TFrameFooter); - const auto maxFrameSize = ui64(TCodecAlg::MaxCompressedLength(Uncompressed_.Size())) + framePayload; - Y_ENSURE(maxFrameSize <= FRAME_SIZE_LIMIT, "Frame size in encoder is limited to " - << HumanReadableSize(FRAME_SIZE_LIMIT, SF_BYTES) - << " but is " << HumanReadableSize(maxFrameSize, SF_BYTES)); + const auto maxFrameSize = ui64(TCodecAlg::MaxCompressedLength(Uncompressed_.Size())) + framePayload; + Y_ENSURE(maxFrameSize <= FRAME_SIZE_LIMIT, "Frame size in encoder is limited to " + << HumanReadableSize(FRAME_SIZE_LIMIT, SF_BYTES) + << " but is " << HumanReadableSize(maxFrameSize, SF_BYTES)); + + Frame_.Resize(maxFrameSize); - Frame_.Resize(maxFrameSize); - // compress TBlock compressedBlock = Frame_; compressedBlock.Skip(sizeof(TFrameHeader)); diff --git a/library/cpp/monlib/encode/spack/fuzz/main.cpp b/library/cpp/monlib/encode/spack/fuzz/main.cpp index c546e392cb..6a14afe71c 100644 --- a/library/cpp/monlib/encode/spack/fuzz/main.cpp +++ b/library/cpp/monlib/encode/spack/fuzz/main.cpp @@ -1,20 +1,20 @@ #include <library/cpp/monlib/encode/spack/spack_v1.h> #include <library/cpp/monlib/encode/fake/fake.h> - -#include <util/stream/mem.h> - - -extern "C" int LLVMFuzzerTestOneInput(const ui8* data, size_t size) { - using namespace NMonitoring; - - TMemoryInput min{data, size}; - - auto encoder = EncoderFake(); - - try { - DecodeSpackV1(&min, encoder.Get()); - } catch (...) { - } - - return 0; -} + +#include <util/stream/mem.h> + + +extern "C" int LLVMFuzzerTestOneInput(const ui8* data, size_t size) { + using namespace NMonitoring; + + TMemoryInput min{data, size}; + + auto encoder = EncoderFake(); + + try { + DecodeSpackV1(&min, encoder.Get()); + } catch (...) { + } + + return 0; +} diff --git a/library/cpp/monlib/encode/spack/fuzz/ya.make b/library/cpp/monlib/encode/spack/fuzz/ya.make index fc41465df3..99b63eadd5 100644 --- a/library/cpp/monlib/encode/spack/fuzz/ya.make +++ b/library/cpp/monlib/encode/spack/fuzz/ya.make @@ -1,21 +1,21 @@ -FUZZ() - +FUZZ() + OWNER( g:solomon msherbakov -) - +) + FUZZ_OPTS(-rss_limit_mb=1024) -SIZE(MEDIUM) - -PEERDIR( +SIZE(MEDIUM) + +PEERDIR( library/cpp/monlib/encode/spack library/cpp/monlib/encode/fake -) - +) + SRCS( main.cpp ) - -END() + +END() diff --git a/library/cpp/monlib/encode/spack/spack_v1_decoder.cpp b/library/cpp/monlib/encode/spack/spack_v1_decoder.cpp index b3be13f77c..1f445fc80d 100644 --- a/library/cpp/monlib/encode/spack/spack_v1_decoder.cpp +++ b/library/cpp/monlib/encode/spack/spack_v1_decoder.cpp @@ -9,8 +9,8 @@ #include <util/generic/yexception.h> #include <util/generic/buffer.h> -#include <util/generic/size_literals.h> -#include <util/stream/format.h> +#include <util/generic/size_literals.h> +#include <util/stream/format.h> #ifndef _little_endian_ #error Unsupported platform @@ -20,8 +20,8 @@ namespace NMonitoring { namespace { #define DECODE_ENSURE(COND, ...) MONLIB_ENSURE_EX(COND, TSpackDecodeError() << __VA_ARGS__) - constexpr ui64 LABEL_SIZE_LIMIT = 128_MB; - + constexpr ui64 LABEL_SIZE_LIMIT = 128_MB; + /////////////////////////////////////////////////////////////////////// // TDecoderSpackV1 /////////////////////////////////////////////////////////////////////// @@ -63,20 +63,20 @@ namespace NMonitoring { TimePrecision_ = DecodeTimePrecision(Header_.TimePrecision); const ui64 labelSizeTotal = ui64(Header_.LabelNamesSize) + Header_.LabelValuesSize; - + DECODE_ENSURE(labelSizeTotal <= LABEL_SIZE_LIMIT, "Label names & values size of " << HumanReadableSize(labelSizeTotal, SF_BYTES) - << " exceeds the limit which is " << HumanReadableSize(LABEL_SIZE_LIMIT, SF_BYTES)); - + << " exceeds the limit which is " << HumanReadableSize(LABEL_SIZE_LIMIT, SF_BYTES)); + // (2) read string pools - TVector<char> namesBuf(Header_.LabelNamesSize); - readBytes = In_->Load(namesBuf.data(), namesBuf.size()); + TVector<char> namesBuf(Header_.LabelNamesSize); + readBytes = In_->Load(namesBuf.data(), namesBuf.size()); DECODE_ENSURE(readBytes == Header_.LabelNamesSize, "not enough data to read label names pool"); - TStringPool labelNames(namesBuf.data(), namesBuf.size()); + TStringPool labelNames(namesBuf.data(), namesBuf.size()); - TVector<char> valuesBuf(Header_.LabelValuesSize); - readBytes = In_->Load(valuesBuf.data(), valuesBuf.size()); + TVector<char> valuesBuf(Header_.LabelValuesSize); + readBytes = In_->Load(valuesBuf.data(), valuesBuf.size()); DECODE_ENSURE(readBytes == Header_.LabelValuesSize, "not enough data to read label values pool"); - TStringPool labelValues(valuesBuf.data(), valuesBuf.size()); + TStringPool labelValues(valuesBuf.data(), valuesBuf.size()); // (3) read common time c->OnCommonTime(ReadTime()); diff --git a/library/cpp/monlib/encode/spack/spack_v1_encoder.cpp b/library/cpp/monlib/encode/spack/spack_v1_encoder.cpp index d7c6e1d123..a2b0bb5f50 100644 --- a/library/cpp/monlib/encode/spack/spack_v1_encoder.cpp +++ b/library/cpp/monlib/encode/spack/spack_v1_encoder.cpp @@ -17,7 +17,7 @@ namespace NMonitoring { /////////////////////////////////////////////////////////////////////// // TEncoderSpackV1 /////////////////////////////////////////////////////////////////////// - class TEncoderSpackV1 final: public TBufferedEncoderBase { + class TEncoderSpackV1 final: public TBufferedEncoderBase { public: TEncoderSpackV1( IOutputStream* out, @@ -35,8 +35,8 @@ namespace NMonitoring { { MetricsMergingMode_ = mergingMode; - LabelNamesPool_.SetSorted(true); - LabelValuesPool_.SetSorted(true); + LabelNamesPool_.SetSorted(true); + LabelValuesPool_.SetSorted(true); } ~TEncoderSpackV1() override { @@ -45,7 +45,7 @@ namespace NMonitoring { private: void OnDouble(TInstant time, double value) override { - TBufferedEncoderBase::OnDouble(time, value); + TBufferedEncoderBase::OnDouble(time, value); } void OnInt64(TInstant time, i64 value) override { @@ -53,7 +53,7 @@ namespace NMonitoring { } void OnUint64(TInstant time, ui64 value) override { - TBufferedEncoderBase::OnUint64(time, value); + TBufferedEncoderBase::OnUint64(time, value); } void OnHistogram(TInstant time, IHistogramSnapshotPtr snapshot) override { @@ -74,8 +74,8 @@ namespace NMonitoring { } Closed_ = true; - LabelNamesPool_.Build(); - LabelValuesPool_.Build(); + LabelNamesPool_.Build(); + LabelValuesPool_.Build(); // Sort all points uniquely by ts -- the size can decrease ui64 pointsCount = 0; @@ -192,12 +192,12 @@ namespace NMonitoring { void WriteLabels(const TPooledLabels& labels, const TPooledStr* skipKey) { WriteVarUInt32(Out_, static_cast<ui32>(skipKey ? labels.size() - 1 : labels.size())); - for (auto&& label : labels) { + for (auto&& label : labels) { if (label.Key == skipKey) { continue; } - WriteVarUInt32(Out_, label.Key->Index); - WriteVarUInt32(Out_, label.Value->Index); + WriteVarUInt32(Out_, label.Key->Index); + WriteVarUInt32(Out_, label.Value->Index); } } diff --git a/library/cpp/monlib/encode/text/text_encoder.cpp b/library/cpp/monlib/encode/text/text_encoder.cpp index 8917152f56..10336261f0 100644 --- a/library/cpp/monlib/encode/text/text_encoder.cpp +++ b/library/cpp/monlib/encode/text/text_encoder.cpp @@ -99,9 +99,9 @@ namespace NMonitoring { TimeSeries_.Add(time, snapshot.Get()); } - void OnLogHistogram(TInstant ts, TLogHistogramSnapshotPtr snapshot) override { + void OnLogHistogram(TInstant ts, TLogHistogramSnapshotPtr snapshot) override { State_.Expect(TEncoderState::EState::METRIC); - TimeSeries_.Add(ts, snapshot.Get()); + TimeSeries_.Add(ts, snapshot.Get()); } void Close() override { @@ -135,7 +135,7 @@ namespace NMonitoring { (*Out_) << *value.AsSummaryDouble(); break; case EMetricValueType::LOGHISTOGRAM: - (*Out_) << *value.AsLogHistogram(); + (*Out_) << *value.AsLogHistogram(); break; case EMetricValueType::UNKNOWN: ythrow yexception() << "unknown metric value type"; @@ -143,21 +143,21 @@ namespace NMonitoring { } void WriteLabels() { - auto& out = *Out_; - const auto size = Labels_.Size(); - size_t i = 0; - - out << '{'; - for (auto&& l : Labels_) { + auto& out = *Out_; + const auto size = Labels_.Size(); + size_t i = 0; + + out << '{'; + for (auto&& l : Labels_) { out << l.Name() << TStringBuf("='") << l.Value() << '\''; - - ++i; - if (i < size) { + + ++i; + if (i < size) { out << TStringBuf(", "); } - }; - - out << '}'; + }; + + out << '}'; } void WriteMetric() { diff --git a/library/cpp/monlib/encode/unistat/unistat.h b/library/cpp/monlib/encode/unistat/unistat.h index 460141b5bc..1c43b7fa1b 100644 --- a/library/cpp/monlib/encode/unistat/unistat.h +++ b/library/cpp/monlib/encode/unistat/unistat.h @@ -1,13 +1,13 @@ -#pragma once - -#include <util/generic/fwd.h> -#include <util/datetime/base.h> - -namespace NMonitoring { - /// Decodes unistat-style metrics - /// https://wiki.yandex-team.ru/golovan/stat-handle +#pragma once + +#include <util/generic/fwd.h> +#include <util/datetime/base.h> + +namespace NMonitoring { + /// Decodes unistat-style metrics + /// https://wiki.yandex-team.ru/golovan/stat-handle void DecodeUnistat(TStringBuf data, class IMetricConsumer* c, TInstant ts = TInstant::Zero()); - - /// Assumes consumer's stream is open by the caller + + /// Assumes consumer's stream is open by the caller void DecodeUnistatToStream(TStringBuf data, class IMetricConsumer* c, TInstant = TInstant::Zero()); -} +} diff --git a/library/cpp/monlib/encode/unistat/unistat_decoder.cpp b/library/cpp/monlib/encode/unistat/unistat_decoder.cpp index 8006d34070..b2344b0905 100644 --- a/library/cpp/monlib/encode/unistat/unistat_decoder.cpp +++ b/library/cpp/monlib/encode/unistat/unistat_decoder.cpp @@ -1,253 +1,253 @@ -#include "unistat.h" - +#include "unistat.h" + #include <library/cpp/monlib/metrics/histogram_collector.h> #include <library/cpp/monlib/metrics/labels.h> #include <library/cpp/monlib/metrics/metric_type.h> #include <library/cpp/monlib/metrics/metric_value.h> #include <library/cpp/monlib/metrics/metric_consumer.h> - + #include <library/cpp/json/json_reader.h> - -#include <util/datetime/base.h> + +#include <util/datetime/base.h> #include <util/string/split.h> - -#include <contrib/libs/re2/re2/re2.h> - -using namespace NJson; - + +#include <contrib/libs/re2/re2/re2.h> + +using namespace NJson; + const re2::RE2 NAME_RE{R"((?:[a-zA-Z0-9\.\-/@_]+_)+(?:[ad][vehmntx]{3}|summ|hgram|max))"}; - -namespace NMonitoring { - namespace { - bool IsNumber(const NJson::TJsonValue& j) { - switch (j.GetType()) { - case EJsonValueType::JSON_INTEGER: - case EJsonValueType::JSON_UINTEGER: - case EJsonValueType::JSON_DOUBLE: - return true; - - default: - return false; - } - } - - template <typename T> - T ExtractNumber(const TJsonValue& val) { - switch (val.GetType()) { - case EJsonValueType::JSON_INTEGER: - return static_cast<T>(val.GetInteger()); - case EJsonValueType::JSON_UINTEGER: - return static_cast<T>(val.GetUInteger()); - case EJsonValueType::JSON_DOUBLE: - return static_cast<T>(val.GetDouble()); - - default: - ythrow yexception() << "Expected number, but found " << val.GetType(); - } - } - - auto ExtractDouble = ExtractNumber<double>; - auto ExtractUi64 = ExtractNumber<ui64>; - + +namespace NMonitoring { + namespace { + bool IsNumber(const NJson::TJsonValue& j) { + switch (j.GetType()) { + case EJsonValueType::JSON_INTEGER: + case EJsonValueType::JSON_UINTEGER: + case EJsonValueType::JSON_DOUBLE: + return true; + + default: + return false; + } + } + + template <typename T> + T ExtractNumber(const TJsonValue& val) { + switch (val.GetType()) { + case EJsonValueType::JSON_INTEGER: + return static_cast<T>(val.GetInteger()); + case EJsonValueType::JSON_UINTEGER: + return static_cast<T>(val.GetUInteger()); + case EJsonValueType::JSON_DOUBLE: + return static_cast<T>(val.GetDouble()); + + default: + ythrow yexception() << "Expected number, but found " << val.GetType(); + } + } + + auto ExtractDouble = ExtractNumber<double>; + auto ExtractUi64 = ExtractNumber<ui64>; + class THistogramBuilder { - public: - void Add(TBucketBound bound, TBucketValue value) { - /// XXX: yasm uses left-closed intervals, while in monlib we use right-closed ones, - /// so (-inf; 0) [0, 100) [100; +inf) - /// becomes (-inf; 0] (0, 100] (100; +inf) - /// but since we've already lost some information these no way to avoid this kind of error here - Bounds_.push_back(bound); - - /// this will always be 0 for the first bucket, - /// since there's no way to make (-inf; N) bucket in yasm - Values_.push_back(NextValue_); - - /// we will write this value into the next bucket so that [[0, 10], [100, 20], [200, 50]] - /// becomes (-inf; 0] -> 0; (0; 100] -> 10; (100; 200] -> 20; (200; +inf) -> 50 - NextValue_ = value; - } - + public: + void Add(TBucketBound bound, TBucketValue value) { + /// XXX: yasm uses left-closed intervals, while in monlib we use right-closed ones, + /// so (-inf; 0) [0, 100) [100; +inf) + /// becomes (-inf; 0] (0, 100] (100; +inf) + /// but since we've already lost some information these no way to avoid this kind of error here + Bounds_.push_back(bound); + + /// this will always be 0 for the first bucket, + /// since there's no way to make (-inf; N) bucket in yasm + Values_.push_back(NextValue_); + + /// we will write this value into the next bucket so that [[0, 10], [100, 20], [200, 50]] + /// becomes (-inf; 0] -> 0; (0; 100] -> 10; (100; 200] -> 20; (200; +inf) -> 50 + NextValue_ = value; + } + IHistogramSnapshotPtr Finalize() { - Bounds_.push_back(std::numeric_limits<TBucketBound>::max()); - Values_.push_back(NextValue_); - + Bounds_.push_back(std::numeric_limits<TBucketBound>::max()); + Values_.push_back(NextValue_); + Y_ENSURE(Bounds_.size() <= HISTOGRAM_MAX_BUCKETS_COUNT, "Histogram is only allowed to have " << HISTOGRAM_MAX_BUCKETS_COUNT << " buckets, but has " << Bounds_.size()); - + return ExplicitHistogramSnapshot(Bounds_, Values_); - } - - public: - TBucketValue NextValue_ {0}; - TBucketBounds Bounds_; - TBucketValues Values_; - }; - - class TDecoderUnistat { - private: - public: + } + + public: + TBucketValue NextValue_ {0}; + TBucketBounds Bounds_; + TBucketValues Values_; + }; + + class TDecoderUnistat { + private: + public: explicit TDecoderUnistat(IMetricConsumer* consumer, IInputStream* is, TInstant ts) - : Consumer_{consumer} - , Timestamp_{ts} { - ReadJsonTree(is, &Json_, /* throw */ true); - } - - void Decode() { - Y_ENSURE(Json_.IsArray(), "Expected array at the top level, but found " << Json_.GetType()); - - for (auto&& metric : Json_.GetArray()) { - Y_ENSURE(metric.IsArray(), "Metric must be an array"); - auto&& arr = metric.GetArray(); - Y_ENSURE(arr.size() == 2, "Metric must be an array of 2 elements"); - auto&& name = arr[0]; - auto&& value = arr[1]; - MetricContext_ = {}; - - ParseName(name.GetString()); - - if (value.IsArray()) { - OnHistogram(value); - } else if (IsNumber(value)) { - OnScalar(value); - } else { - ythrow yexception() << "Expected list or number, but found " << value.GetType(); - } - - WriteValue(); - } - } - - private: - void OnScalar(const TJsonValue& jsonValue) { - if (MetricContext_.IsDeriv) { + : Consumer_{consumer} + , Timestamp_{ts} { + ReadJsonTree(is, &Json_, /* throw */ true); + } + + void Decode() { + Y_ENSURE(Json_.IsArray(), "Expected array at the top level, but found " << Json_.GetType()); + + for (auto&& metric : Json_.GetArray()) { + Y_ENSURE(metric.IsArray(), "Metric must be an array"); + auto&& arr = metric.GetArray(); + Y_ENSURE(arr.size() == 2, "Metric must be an array of 2 elements"); + auto&& name = arr[0]; + auto&& value = arr[1]; + MetricContext_ = {}; + + ParseName(name.GetString()); + + if (value.IsArray()) { + OnHistogram(value); + } else if (IsNumber(value)) { + OnScalar(value); + } else { + ythrow yexception() << "Expected list or number, but found " << value.GetType(); + } + + WriteValue(); + } + } + + private: + void OnScalar(const TJsonValue& jsonValue) { + if (MetricContext_.IsDeriv) { MetricContext_.Type = EMetricType::RATE; MetricContext_.Value = TMetricValue{ExtractUi64(jsonValue)}; - } else { + } else { MetricContext_.Type = EMetricType::GAUGE; MetricContext_.Value = TMetricValue{ExtractDouble(jsonValue)}; - } - } - - void OnHistogram(const TJsonValue& jsonHist) { - if (MetricContext_.IsDeriv) { + } + } + + void OnHistogram(const TJsonValue& jsonHist) { + if (MetricContext_.IsDeriv) { MetricContext_.Type = EMetricType::HIST_RATE; - } else { + } else { MetricContext_.Type = EMetricType::HIST; - } - + } + auto histogramBuilder = THistogramBuilder(); - - for (auto&& bucket : jsonHist.GetArray()) { - Y_ENSURE(bucket.IsArray(), "Expected an array, but found " << bucket.GetType()); - auto&& arr = bucket.GetArray(); - Y_ENSURE(arr.size() == 2, "Histogram bucket must be an array of 2 elements"); - const auto bound = ExtractDouble(arr[0]); - const auto weight = ExtractUi64(arr[1]); + + for (auto&& bucket : jsonHist.GetArray()) { + Y_ENSURE(bucket.IsArray(), "Expected an array, but found " << bucket.GetType()); + auto&& arr = bucket.GetArray(); + Y_ENSURE(arr.size() == 2, "Histogram bucket must be an array of 2 elements"); + const auto bound = ExtractDouble(arr[0]); + const auto weight = ExtractUi64(arr[1]); histogramBuilder.Add(bound, weight); - } - + } + MetricContext_.Histogram = histogramBuilder.Finalize(); MetricContext_.Value = TMetricValue{MetricContext_.Histogram.Get()}; - } - - bool IsDeriv(TStringBuf name) { - TStringBuf ignore, suffix; - name.RSplit('_', ignore, suffix); - - Y_ENSURE(suffix.size() >= 3 && suffix.size() <= 5, "Disallowed suffix value: " << suffix); - + } + + bool IsDeriv(TStringBuf name) { + TStringBuf ignore, suffix; + name.RSplit('_', ignore, suffix); + + Y_ENSURE(suffix.size() >= 3 && suffix.size() <= 5, "Disallowed suffix value: " << suffix); + if (suffix == TStringBuf("summ") || suffix == TStringBuf("hgram")) { - return true; + return true; } else if (suffix == TStringBuf("max")) { - return false; - } - - return suffix[0] == 'd'; - } - - void ParseName(TStringBuf value) { - TVector<TStringBuf> parts; - StringSplitter(value).Split(';').SkipEmpty().Collect(&parts); - - Y_ENSURE(parts.size() >= 1 && parts.size() <= 16); - - TStringBuf name = parts.back(); - parts.pop_back(); - + return false; + } + + return suffix[0] == 'd'; + } + + void ParseName(TStringBuf value) { + TVector<TStringBuf> parts; + StringSplitter(value).Split(';').SkipEmpty().Collect(&parts); + + Y_ENSURE(parts.size() >= 1 && parts.size() <= 16); + + TStringBuf name = parts.back(); + parts.pop_back(); + Y_ENSURE(RE2::FullMatch(re2::StringPiece{name.data(), name.size()}, NAME_RE), - "Metric name " << name << " doesn't match regex " << NAME_RE.pattern()); - - MetricContext_.Name = name; - MetricContext_.IsDeriv = IsDeriv(MetricContext_.Name); - - for (auto tag : parts) { - TStringBuf n, v; - tag.Split('=', n, v); - Y_ENSURE(n && v, "Unexpected tag format in " << tag); - MetricContext_.Labels.Add(n, v); - } - } - - private: - void WriteValue() { + "Metric name " << name << " doesn't match regex " << NAME_RE.pattern()); + + MetricContext_.Name = name; + MetricContext_.IsDeriv = IsDeriv(MetricContext_.Name); + + for (auto tag : parts) { + TStringBuf n, v; + tag.Split('=', n, v); + Y_ENSURE(n && v, "Unexpected tag format in " << tag); + MetricContext_.Labels.Add(n, v); + } + } + + private: + void WriteValue() { Consumer_->OnMetricBegin(MetricContext_.Type); - - Consumer_->OnLabelsBegin(); - Consumer_->OnLabel("sensor", TString{MetricContext_.Name}); - for (auto&& l : MetricContext_.Labels) { - Consumer_->OnLabel(l.Name(), l.Value()); - } - - Consumer_->OnLabelsEnd(); - + + Consumer_->OnLabelsBegin(); + Consumer_->OnLabel("sensor", TString{MetricContext_.Name}); + for (auto&& l : MetricContext_.Labels) { + Consumer_->OnLabel(l.Name(), l.Value()); + } + + Consumer_->OnLabelsEnd(); + switch (MetricContext_.Type) { case EMetricType::GAUGE: - Consumer_->OnDouble(Timestamp_, MetricContext_.Value.AsDouble()); - break; + Consumer_->OnDouble(Timestamp_, MetricContext_.Value.AsDouble()); + break; case EMetricType::RATE: - Consumer_->OnUint64(Timestamp_, MetricContext_.Value.AsUint64()); - break; + Consumer_->OnUint64(Timestamp_, MetricContext_.Value.AsUint64()); + break; case EMetricType::HIST: case EMetricType::HIST_RATE: - Consumer_->OnHistogram(Timestamp_, MetricContext_.Value.AsHistogram()); - break; + Consumer_->OnHistogram(Timestamp_, MetricContext_.Value.AsHistogram()); + break; case EMetricType::LOGHIST: case EMetricType::DSUMMARY: case EMetricType::IGAUGE: case EMetricType::COUNTER: case EMetricType::UNKNOWN: ythrow yexception() << "Unexpected metric type: " << MetricContext_.Type; - } - + } + Consumer_->OnMetricEnd(); - } - - private: + } + + private: IMetricConsumer* Consumer_; - NJson::TJsonValue Json_; - TInstant Timestamp_; - - struct { - TStringBuf Name; + NJson::TJsonValue Json_; + TInstant Timestamp_; + + struct { + TStringBuf Name; EMetricType Type{EMetricType::UNKNOWN}; TMetricValue Value; - bool IsDeriv{false}; - TLabels Labels; + bool IsDeriv{false}; + TLabels Labels; IHistogramSnapshotPtr Histogram; - } MetricContext_; - }; - - } - + } MetricContext_; + }; + + } + void DecodeUnistat(TStringBuf data, IMetricConsumer* c, TInstant ts) { - c->OnStreamBegin(); - DecodeUnistatToStream(data, c, ts); - c->OnStreamEnd(); - } - + c->OnStreamBegin(); + DecodeUnistatToStream(data, c, ts); + c->OnStreamEnd(); + } + void DecodeUnistatToStream(TStringBuf data, IMetricConsumer* c, TInstant ts) { TMemoryInput in{data.data(), data.size()}; - TDecoderUnistat decoder(c, &in, ts); - decoder.Decode(); - } -} + TDecoderUnistat decoder(c, &in, ts); + decoder.Decode(); + } +} diff --git a/library/cpp/monlib/encode/unistat/unistat_ut.cpp b/library/cpp/monlib/encode/unistat/unistat_ut.cpp index e55d41eec3..dbbc238bf3 100644 --- a/library/cpp/monlib/encode/unistat/unistat_ut.cpp +++ b/library/cpp/monlib/encode/unistat/unistat_ut.cpp @@ -1,124 +1,124 @@ -#include "unistat.h" - +#include "unistat.h" + #include <library/cpp/monlib/encode/protobuf/protobuf.h> #include <library/cpp/monlib/metrics/labels.h> - + #include <library/cpp/testing/unittest/registar.h> - -using namespace NMonitoring; - -Y_UNIT_TEST_SUITE(TUnistatDecoderTest) { - Y_UNIT_TEST(ScalarMetric) { + +using namespace NMonitoring; + +Y_UNIT_TEST_SUITE(TUnistatDecoderTest) { + Y_UNIT_TEST(ScalarMetric) { constexpr auto input = TStringBuf(R"([["something_axxx", 42]])"); - - NProto::TMultiSamplesList samples; - auto encoder = EncoderProtobuf(&samples); - - DecodeUnistat(input, encoder.Get()); - - UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 1); - auto sample = samples.GetSamples(0); + + NProto::TMultiSamplesList samples; + auto encoder = EncoderProtobuf(&samples); + + DecodeUnistat(input, encoder.Get()); + + UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 1); + auto sample = samples.GetSamples(0); UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::GAUGE); - UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1); - UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1); - - auto label = sample.GetLabels(0); - auto point = sample.GetPoints(0); - UNIT_ASSERT_VALUES_EQUAL(point.GetFloat64(), 42.); - UNIT_ASSERT_VALUES_EQUAL(label.GetName(), "sensor"); - UNIT_ASSERT_VALUES_EQUAL(label.GetValue(), "something_axxx"); - } - - Y_UNIT_TEST(OverriddenTags) { + UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1); + UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1); + + auto label = sample.GetLabels(0); + auto point = sample.GetPoints(0); + UNIT_ASSERT_VALUES_EQUAL(point.GetFloat64(), 42.); + UNIT_ASSERT_VALUES_EQUAL(label.GetName(), "sensor"); + UNIT_ASSERT_VALUES_EQUAL(label.GetValue(), "something_axxx"); + } + + Y_UNIT_TEST(OverriddenTags) { constexpr auto input = TStringBuf(R"([["ctype=foo;prj=bar;custom_tag=qwe;something_axxx", 42]])"); - - NProto::TMultiSamplesList samples; - auto encoder = EncoderProtobuf(&samples); - - DecodeUnistat(input, encoder.Get()); - - UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 1); - auto sample = samples.GetSamples(0); - UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1); + + NProto::TMultiSamplesList samples; + auto encoder = EncoderProtobuf(&samples); + + DecodeUnistat(input, encoder.Get()); + + UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 1); + auto sample = samples.GetSamples(0); + UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1); UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 4); - - const auto& labels = sample.GetLabels(); - TLabels actual; - for (auto&& l : labels) { - actual.Add(l.GetName(), l.GetValue()); - } - + + const auto& labels = sample.GetLabels(); + TLabels actual; + for (auto&& l : labels) { + actual.Add(l.GetName(), l.GetValue()); + } + TLabels expected{{"ctype", "foo"}, {"prj", "bar"}, {"custom_tag", "qwe"}, {"sensor", "something_axxx"}}; - - UNIT_ASSERT_VALUES_EQUAL(actual.size(), expected.size()); - for (auto&& l : actual) { - UNIT_ASSERT(expected.Extract(l.Name())->Value() == l.Value()); - } - } - - Y_UNIT_TEST(ThrowsOnTopLevelObject) { + + UNIT_ASSERT_VALUES_EQUAL(actual.size(), expected.size()); + for (auto&& l : actual) { + UNIT_ASSERT(expected.Extract(l.Name())->Value() == l.Value()); + } + } + + Y_UNIT_TEST(ThrowsOnTopLevelObject) { constexpr auto input = TStringBuf(R"({["something_axxx", 42]})"); - - NProto::TMultiSamplesList samples; - auto encoder = EncoderProtobuf(&samples); - - UNIT_ASSERT_EXCEPTION(DecodeUnistat(input, encoder.Get()), yexception); - } - - Y_UNIT_TEST(ThrowsOnUnwrappedMetric) { + + NProto::TMultiSamplesList samples; + auto encoder = EncoderProtobuf(&samples); + + UNIT_ASSERT_EXCEPTION(DecodeUnistat(input, encoder.Get()), yexception); + } + + Y_UNIT_TEST(ThrowsOnUnwrappedMetric) { constexpr auto input = TStringBuf(R"(["something_axxx", 42])"); - - NProto::TMultiSamplesList samples; - auto encoder = EncoderProtobuf(&samples); - - UNIT_ASSERT_EXCEPTION(DecodeUnistat(input, encoder.Get()), yexception); - } - - Y_UNIT_TEST(HistogramMetric) { + + NProto::TMultiSamplesList samples; + auto encoder = EncoderProtobuf(&samples); + + UNIT_ASSERT_EXCEPTION(DecodeUnistat(input, encoder.Get()), yexception); + } + + Y_UNIT_TEST(HistogramMetric) { constexpr auto input = TStringBuf(R"([["something_hgram", [[0, 1], [200, 2], [500, 3]] ]])"); - - NProto::TMultiSamplesList samples; - auto encoder = EncoderProtobuf(&samples); - - DecodeUnistat(input, encoder.Get()); - - auto sample = samples.GetSamples(0); + + NProto::TMultiSamplesList samples; + auto encoder = EncoderProtobuf(&samples); + + DecodeUnistat(input, encoder.Get()); + + auto sample = samples.GetSamples(0); UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::HIST_RATE); - UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1); - UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1); - - auto label = sample.GetLabels(0); - const auto point = sample.GetPoints(0); - const auto histogram = point.GetHistogram(); - const auto size = histogram.BoundsSize(); - UNIT_ASSERT_VALUES_EQUAL(size, 4); - - const TVector<double> expectedBounds {0, 200, 500, std::numeric_limits<double>::max()}; - const TVector<ui64> expectedValues {0, 1, 2, 3}; - - for (auto i = 0; i < 4; ++i) { - UNIT_ASSERT_VALUES_EQUAL(histogram.GetBounds(i), expectedBounds[i]); - UNIT_ASSERT_VALUES_EQUAL(histogram.GetValues(i), expectedValues[i]); - } - - UNIT_ASSERT_VALUES_EQUAL(label.GetName(), "sensor"); - UNIT_ASSERT_VALUES_EQUAL(label.GetValue(), "something_hgram"); - } - - Y_UNIT_TEST(AbsoluteHistogram) { + UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1); + UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1); + + auto label = sample.GetLabels(0); + const auto point = sample.GetPoints(0); + const auto histogram = point.GetHistogram(); + const auto size = histogram.BoundsSize(); + UNIT_ASSERT_VALUES_EQUAL(size, 4); + + const TVector<double> expectedBounds {0, 200, 500, std::numeric_limits<double>::max()}; + const TVector<ui64> expectedValues {0, 1, 2, 3}; + + for (auto i = 0; i < 4; ++i) { + UNIT_ASSERT_VALUES_EQUAL(histogram.GetBounds(i), expectedBounds[i]); + UNIT_ASSERT_VALUES_EQUAL(histogram.GetValues(i), expectedValues[i]); + } + + UNIT_ASSERT_VALUES_EQUAL(label.GetName(), "sensor"); + UNIT_ASSERT_VALUES_EQUAL(label.GetValue(), "something_hgram"); + } + + Y_UNIT_TEST(AbsoluteHistogram) { constexpr auto input = TStringBuf(R"([["something_ahhh", [[0, 1], [200, 2], [500, 3]] ]])"); - - NProto::TMultiSamplesList samples; - auto encoder = EncoderProtobuf(&samples); - - DecodeUnistat(input, encoder.Get()); - - auto sample = samples.GetSamples(0); + + NProto::TMultiSamplesList samples; + auto encoder = EncoderProtobuf(&samples); + + DecodeUnistat(input, encoder.Get()); + + auto sample = samples.GetSamples(0); UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::HISTOGRAM); - UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1); - UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1); - } - + UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1); + UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1); + } + Y_UNIT_TEST(AllowedMetricNames) { NProto::TMultiSamplesList samples; auto encoder = EncoderProtobuf(&samples); @@ -129,95 +129,95 @@ Y_UNIT_TEST_SUITE(TUnistatDecoderTest) { } } - Y_UNIT_TEST(DisallowedMetricNames) { - NProto::TMultiSamplesList samples; - auto encoder = EncoderProtobuf(&samples); - - { + Y_UNIT_TEST(DisallowedMetricNames) { + NProto::TMultiSamplesList samples; + auto encoder = EncoderProtobuf(&samples); + + { constexpr auto input = TStringBuf(R"([["someth!ng_ahhh", [[0, 1], [200, 2], [500, 3]] ]])"); - UNIT_ASSERT_EXCEPTION(DecodeUnistat(input, encoder.Get()), yexception); - } - - { + UNIT_ASSERT_EXCEPTION(DecodeUnistat(input, encoder.Get()), yexception); + } + + { constexpr auto input = TStringBuf(R"([["foo_a", [[0, 1], [200, 2], [500, 3]] ]])"); - UNIT_ASSERT_EXCEPTION(DecodeUnistat(input, encoder.Get()), yexception); - } - - { + UNIT_ASSERT_EXCEPTION(DecodeUnistat(input, encoder.Get()), yexception); + } + + { constexpr auto input = TStringBuf(R"([["foo_ahhh;tag=value", [[0, 1], [200, 2], [500, 3]] ]])"); - UNIT_ASSERT_EXCEPTION(DecodeUnistat(input, encoder.Get()), yexception); - } - } - - Y_UNIT_TEST(MultipleMetrics) { + UNIT_ASSERT_EXCEPTION(DecodeUnistat(input, encoder.Get()), yexception); + } + } + + Y_UNIT_TEST(MultipleMetrics) { constexpr auto input = TStringBuf(R"([["something_axxx", 42], ["some-other_dhhh", 53]])"); - - NProto::TMultiSamplesList samples; - auto encoder = EncoderProtobuf(&samples); - - DecodeUnistat(input, encoder.Get()); - - UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 2); - auto sample = samples.GetSamples(0); + + NProto::TMultiSamplesList samples; + auto encoder = EncoderProtobuf(&samples); + + DecodeUnistat(input, encoder.Get()); + + UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 2); + auto sample = samples.GetSamples(0); UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::GAUGE); - UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1); - UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1); - - auto label = sample.GetLabels(0); - auto point = sample.GetPoints(0); - UNIT_ASSERT_VALUES_EQUAL(point.GetFloat64(), 42.); - UNIT_ASSERT_VALUES_EQUAL(label.GetName(), "sensor"); - UNIT_ASSERT_VALUES_EQUAL(label.GetValue(), "something_axxx"); - - sample = samples.GetSamples(1); + UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1); + UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1); + + auto label = sample.GetLabels(0); + auto point = sample.GetPoints(0); + UNIT_ASSERT_VALUES_EQUAL(point.GetFloat64(), 42.); + UNIT_ASSERT_VALUES_EQUAL(label.GetName(), "sensor"); + UNIT_ASSERT_VALUES_EQUAL(label.GetValue(), "something_axxx"); + + sample = samples.GetSamples(1); UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::RATE); - UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1); - UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1); - - label = sample.GetLabels(0); - point = sample.GetPoints(0); - UNIT_ASSERT_VALUES_EQUAL(point.GetUint64(), 53); - UNIT_ASSERT_VALUES_EQUAL(label.GetName(), "sensor"); - UNIT_ASSERT_VALUES_EQUAL(label.GetValue(), "some-other_dhhh"); - } - - Y_UNIT_TEST(UnderscoreName) { + UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1); + UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1); + + label = sample.GetLabels(0); + point = sample.GetPoints(0); + UNIT_ASSERT_VALUES_EQUAL(point.GetUint64(), 53); + UNIT_ASSERT_VALUES_EQUAL(label.GetName(), "sensor"); + UNIT_ASSERT_VALUES_EQUAL(label.GetValue(), "some-other_dhhh"); + } + + Y_UNIT_TEST(UnderscoreName) { constexpr auto input = TStringBuf(R"([["something_anything_dmmm", 42]])"); - - NProto::TMultiSamplesList samples; - auto encoder = EncoderProtobuf(&samples); - DecodeUnistat(input, encoder.Get()); - - UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 1); - auto sample = samples.GetSamples(0); + + NProto::TMultiSamplesList samples; + auto encoder = EncoderProtobuf(&samples); + DecodeUnistat(input, encoder.Get()); + + UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 1); + auto sample = samples.GetSamples(0); UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::RATE); - UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1); - UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1); - - auto label = sample.GetLabels(0); - auto point = sample.GetPoints(0); - UNIT_ASSERT_VALUES_EQUAL(point.GetUint64(), 42); - UNIT_ASSERT_VALUES_EQUAL(label.GetName(), "sensor"); - UNIT_ASSERT_VALUES_EQUAL(label.GetValue(), "something_anything_dmmm"); - } - - Y_UNIT_TEST(MaxAggr) { + UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1); + UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1); + + auto label = sample.GetLabels(0); + auto point = sample.GetPoints(0); + UNIT_ASSERT_VALUES_EQUAL(point.GetUint64(), 42); + UNIT_ASSERT_VALUES_EQUAL(label.GetName(), "sensor"); + UNIT_ASSERT_VALUES_EQUAL(label.GetValue(), "something_anything_dmmm"); + } + + Y_UNIT_TEST(MaxAggr) { constexpr auto input = TStringBuf(R"([["something_anything_max", 42]])"); - - NProto::TMultiSamplesList samples; - auto encoder = EncoderProtobuf(&samples); - DecodeUnistat(input, encoder.Get()); - - UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 1); - auto sample = samples.GetSamples(0); + + NProto::TMultiSamplesList samples; + auto encoder = EncoderProtobuf(&samples); + DecodeUnistat(input, encoder.Get()); + + UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 1); + auto sample = samples.GetSamples(0); UNIT_ASSERT_EQUAL(sample.GetMetricType(), NProto::GAUGE); - UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1); - UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1); - - auto label = sample.GetLabels(0); - auto point = sample.GetPoints(0); - UNIT_ASSERT_VALUES_EQUAL(point.GetFloat64(), 42.); - UNIT_ASSERT_VALUES_EQUAL(label.GetName(), "sensor"); - UNIT_ASSERT_VALUES_EQUAL(label.GetValue(), "something_anything_max"); - } -} + UNIT_ASSERT_VALUES_EQUAL(sample.PointsSize(), 1); + UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1); + + auto label = sample.GetLabels(0); + auto point = sample.GetPoints(0); + UNIT_ASSERT_VALUES_EQUAL(point.GetFloat64(), 42.); + UNIT_ASSERT_VALUES_EQUAL(label.GetName(), "sensor"); + UNIT_ASSERT_VALUES_EQUAL(label.GetValue(), "something_anything_max"); + } +} diff --git a/library/cpp/monlib/encode/unistat/ut/ya.make b/library/cpp/monlib/encode/unistat/ut/ya.make index 4beba98863..a652139f45 100644 --- a/library/cpp/monlib/encode/unistat/ut/ya.make +++ b/library/cpp/monlib/encode/unistat/ut/ya.make @@ -1,16 +1,16 @@ UNITTEST_FOR(library/cpp/monlib/encode/unistat) - -OWNER( - msherbakov - g:solomon -) - -SRCS( - unistat_ut.cpp -) - -PEERDIR( + +OWNER( + msherbakov + g:solomon +) + +SRCS( + unistat_ut.cpp +) + +PEERDIR( library/cpp/monlib/encode/protobuf -) - -END() +) + +END() diff --git a/library/cpp/monlib/encode/unistat/ya.make b/library/cpp/monlib/encode/unistat/ya.make index c6ec66ca40..4ac2edadf4 100644 --- a/library/cpp/monlib/encode/unistat/ya.make +++ b/library/cpp/monlib/encode/unistat/ya.make @@ -1,18 +1,18 @@ -LIBRARY() - -OWNER( - msherbakov - g:solomon -) - -PEERDIR( - contrib/libs/re2 +LIBRARY() + +OWNER( + msherbakov + g:solomon +) + +PEERDIR( + contrib/libs/re2 library/cpp/json library/cpp/monlib/metrics -) - -SRCS( - unistat_decoder.cpp -) - -END() +) + +SRCS( + unistat_decoder.cpp +) + +END() diff --git a/library/cpp/monlib/messagebus/mon_messagebus.cpp b/library/cpp/monlib/messagebus/mon_messagebus.cpp index 59476e593f..355b4386cd 100644 --- a/library/cpp/monlib/messagebus/mon_messagebus.cpp +++ b/library/cpp/monlib/messagebus/mon_messagebus.cpp @@ -4,8 +4,8 @@ using namespace NMonitoring; -void TBusNgMonPage::Output(NMonitoring::IMonHttpRequest& request) { +void TBusNgMonPage::Output(NMonitoring::IMonHttpRequest& request) { NBus::TBusWww::TOptionalParams params; params.ParentLinks.push_back(NBus::TBusWww::TLink{"/", request.GetServiceTitle()}); - BusWww->ServeHttp(request.Output(), request.GetParams(), params); + BusWww->ServeHttp(request.Output(), request.GetParams(), params); } diff --git a/library/cpp/monlib/messagebus/mon_messagebus.h b/library/cpp/monlib/messagebus/mon_messagebus.h index 09b5226194..e1fa73c69f 100644 --- a/library/cpp/monlib/messagebus/mon_messagebus.h +++ b/library/cpp/monlib/messagebus/mon_messagebus.h @@ -17,13 +17,13 @@ namespace NMonitoring { , Smth(smth) { } - void Output(NMonitoring::IMonHttpRequest& request) override { + void Output(NMonitoring::IMonHttpRequest& request) override { Y_UNUSED(request); - request.Output() << NMonitoring::HTTPOKHTML; - request.Output() << "<h2>" << Title << "</h2>"; - request.Output() << "<pre>"; - request.Output() << Smth->GetStatus(); - request.Output() << "</pre>"; + request.Output() << NMonitoring::HTTPOKHTML; + request.Output() << "<h2>" << Title << "</h2>"; + request.Output() << "<pre>"; + request.Output() << Smth->GetStatus(); + request.Output() << "</pre>"; } }; @@ -40,7 +40,7 @@ namespace NMonitoring { , BusWww(new NBus::TBusWww) { } - void Output(NMonitoring::IMonHttpRequest& request) override; + void Output(NMonitoring::IMonHttpRequest& request) override; }; } diff --git a/library/cpp/monlib/messagebus/ya.make b/library/cpp/monlib/messagebus/ya.make index 67ada78036..a0b5362296 100644 --- a/library/cpp/monlib/messagebus/ya.make +++ b/library/cpp/monlib/messagebus/ya.make @@ -1,6 +1,6 @@ LIBRARY() -OWNER(g:solomon) +OWNER(g:solomon) SRCS( mon_messagebus.cpp diff --git a/library/cpp/monlib/metrics/atomics_array.h b/library/cpp/monlib/metrics/atomics_array.h index a2e6566c96..f19aebf291 100644 --- a/library/cpp/monlib/metrics/atomics_array.h +++ b/library/cpp/monlib/metrics/atomics_array.h @@ -9,17 +9,17 @@ namespace NMonitoring { class TAtomicsArray { public: explicit TAtomicsArray(size_t size) - : Values_(new std::atomic<ui64>[size]) + : Values_(new std::atomic<ui64>[size]) , Size_(size) { for (size_t i = 0; i < Size_; i++) { - Values_[i].store(0, std::memory_order_relaxed); + Values_[i].store(0, std::memory_order_relaxed); } } ui64 operator[](size_t index) const noexcept { Y_VERIFY_DEBUG(index < Size_); - return Values_[index].load(std::memory_order_relaxed); + return Values_[index].load(std::memory_order_relaxed); } size_t Size() const noexcept { @@ -28,7 +28,7 @@ namespace NMonitoring { void Add(size_t index, ui32 count) noexcept { Y_VERIFY_DEBUG(index < Size_); - Values_[index].fetch_add(count, std::memory_order_relaxed); + Values_[index].fetch_add(count, std::memory_order_relaxed); } void Reset() noexcept { @@ -40,13 +40,13 @@ namespace NMonitoring { TVector<ui64> Copy() const { TVector<ui64> copy(Reserve(Size_)); for (size_t i = 0; i < Size_; i++) { - copy.push_back(Values_[i].load(std::memory_order_relaxed)); + copy.push_back(Values_[i].load(std::memory_order_relaxed)); } return copy; } private: - TArrayHolder<std::atomic<ui64>> Values_; + TArrayHolder<std::atomic<ui64>> Values_; size_t Size_; }; } diff --git a/library/cpp/monlib/metrics/ewma.cpp b/library/cpp/monlib/metrics/ewma.cpp index 4838b40d6e..8a296c3225 100644 --- a/library/cpp/monlib/metrics/ewma.cpp +++ b/library/cpp/monlib/metrics/ewma.cpp @@ -1,150 +1,150 @@ -#include "ewma.h" +#include "ewma.h" #include "metric.h" - -#include <atomic> -#include <cmath> - -namespace NMonitoring { -namespace { - constexpr auto DEFAULT_INTERVAL = TDuration::Seconds(5); - - const double ALPHA1 = 1. - std::exp(-double(DEFAULT_INTERVAL.Seconds())/60./1.); - const double ALPHA5 = 1. - std::exp(-double(DEFAULT_INTERVAL.Seconds())/60./5.); - const double ALPHA15 = 1. - std::exp(-double(DEFAULT_INTERVAL.Seconds())/60./15.); - - class TExpMovingAverage final: public IExpMovingAverage { - public: + +#include <atomic> +#include <cmath> + +namespace NMonitoring { +namespace { + constexpr auto DEFAULT_INTERVAL = TDuration::Seconds(5); + + const double ALPHA1 = 1. - std::exp(-double(DEFAULT_INTERVAL.Seconds())/60./1.); + const double ALPHA5 = 1. - std::exp(-double(DEFAULT_INTERVAL.Seconds())/60./5.); + const double ALPHA15 = 1. - std::exp(-double(DEFAULT_INTERVAL.Seconds())/60./15.); + + class TExpMovingAverage final: public IExpMovingAverage { + public: explicit TExpMovingAverage(IGauge* metric, double alpha, TDuration interval) : Metric_{metric} - , Alpha_{alpha} - , Interval_{interval.Seconds()} - { + , Alpha_{alpha} + , Interval_{interval.Seconds()} + { Y_VERIFY(metric != nullptr, "Passing nullptr metric is not allowed"); - } - + } + ~TExpMovingAverage() override = default; - - // This method NOT thread safe - void Tick() override { - const auto current = Uncounted_.fetch_and(0); - const double instantRate = double(current) / Interval_; - - if (Y_UNLIKELY(!IsInitialized())) { + + // This method NOT thread safe + void Tick() override { + const auto current = Uncounted_.fetch_and(0); + const double instantRate = double(current) / Interval_; + + if (Y_UNLIKELY(!IsInitialized())) { Metric_->Set(instantRate); - Init_ = true; - } else { + Init_ = true; + } else { const double currentRate = Metric_->Get(); Metric_->Set(Alpha_ * (instantRate - currentRate) + currentRate); - } - - } - - void Update(i64 value) override { - Uncounted_ += value; - } - - double Rate() const override { + } + + } + + void Update(i64 value) override { + Uncounted_ += value; + } + + double Rate() const override { return Metric_->Get(); - } - - void Reset() override { - Init_ = false; - Uncounted_ = 0; - } - - private: - bool IsInitialized() const { - return Init_; - } - - private: - std::atomic<i64> Uncounted_{0}; - std::atomic<bool> Init_{false}; - + } + + void Reset() override { + Init_ = false; + Uncounted_ = 0; + } + + private: + bool IsInitialized() const { + return Init_; + } + + private: + std::atomic<i64> Uncounted_{0}; + std::atomic<bool> Init_{false}; + IGauge* Metric_{nullptr}; - double Alpha_; - ui64 Interval_; - }; - - struct TFakeEwma: IExpMovingAverage { - void Tick() override {} - void Update(i64) override {} - double Rate() const override { return 0; } - void Reset() override {} - }; - -} // namespace - - TEwmaMeter::TEwmaMeter() - : Ewma_{MakeHolder<TFakeEwma>()} - { - } - - TEwmaMeter::TEwmaMeter(IExpMovingAveragePtr&& ewma) - : Ewma_{std::move(ewma)} - { - } - - TEwmaMeter::TEwmaMeter(TEwmaMeter&& other) { - if (&other == this) { - return; - } - - *this = std::move(other); - } - - TEwmaMeter& TEwmaMeter::operator=(TEwmaMeter&& other) { - Ewma_ = std::move(other.Ewma_); - LastTick_.store(other.LastTick_); - return *this; - } - - void TEwmaMeter::TickIfNeeded() { - constexpr ui64 INTERVAL_SECONDS = DEFAULT_INTERVAL.Seconds(); - - const auto now = TInstant::Now().Seconds(); - ui64 old = LastTick_.load(); - const auto secondsSinceLastTick = now - old; - - if (secondsSinceLastTick > INTERVAL_SECONDS) { - // round to the interval grid - const ui64 newLast = now - (secondsSinceLastTick % INTERVAL_SECONDS); - if (LastTick_.compare_exchange_strong(old, newLast)) { - for (size_t i = 0; i < secondsSinceLastTick / INTERVAL_SECONDS; ++i) { - Ewma_->Tick(); - } - } - } - } - - void TEwmaMeter::Mark() { - TickIfNeeded(); - Ewma_->Update(1); - } - - void TEwmaMeter::Mark(i64 value) { - TickIfNeeded(); - Ewma_->Update(value); - } - - double TEwmaMeter::Get() { - TickIfNeeded(); - return Ewma_->Rate(); - } - + double Alpha_; + ui64 Interval_; + }; + + struct TFakeEwma: IExpMovingAverage { + void Tick() override {} + void Update(i64) override {} + double Rate() const override { return 0; } + void Reset() override {} + }; + +} // namespace + + TEwmaMeter::TEwmaMeter() + : Ewma_{MakeHolder<TFakeEwma>()} + { + } + + TEwmaMeter::TEwmaMeter(IExpMovingAveragePtr&& ewma) + : Ewma_{std::move(ewma)} + { + } + + TEwmaMeter::TEwmaMeter(TEwmaMeter&& other) { + if (&other == this) { + return; + } + + *this = std::move(other); + } + + TEwmaMeter& TEwmaMeter::operator=(TEwmaMeter&& other) { + Ewma_ = std::move(other.Ewma_); + LastTick_.store(other.LastTick_); + return *this; + } + + void TEwmaMeter::TickIfNeeded() { + constexpr ui64 INTERVAL_SECONDS = DEFAULT_INTERVAL.Seconds(); + + const auto now = TInstant::Now().Seconds(); + ui64 old = LastTick_.load(); + const auto secondsSinceLastTick = now - old; + + if (secondsSinceLastTick > INTERVAL_SECONDS) { + // round to the interval grid + const ui64 newLast = now - (secondsSinceLastTick % INTERVAL_SECONDS); + if (LastTick_.compare_exchange_strong(old, newLast)) { + for (size_t i = 0; i < secondsSinceLastTick / INTERVAL_SECONDS; ++i) { + Ewma_->Tick(); + } + } + } + } + + void TEwmaMeter::Mark() { + TickIfNeeded(); + Ewma_->Update(1); + } + + void TEwmaMeter::Mark(i64 value) { + TickIfNeeded(); + Ewma_->Update(value); + } + + double TEwmaMeter::Get() { + TickIfNeeded(); + return Ewma_->Rate(); + } + IExpMovingAveragePtr OneMinuteEwma(IGauge* metric) { return MakeHolder<TExpMovingAverage>(metric, ALPHA1, DEFAULT_INTERVAL); - } - + } + IExpMovingAveragePtr FiveMinuteEwma(IGauge* metric) { return MakeHolder<TExpMovingAverage>(metric, ALPHA5, DEFAULT_INTERVAL); - } - + } + IExpMovingAveragePtr FiveteenMinuteEwma(IGauge* metric) { return MakeHolder<TExpMovingAverage>(metric, ALPHA15, DEFAULT_INTERVAL); - } - + } + IExpMovingAveragePtr CreateEwma(IGauge* metric, double alpha, TDuration interval) { return MakeHolder<TExpMovingAverage>(metric, alpha, interval); - } -} // namespace NMonitoring + } +} // namespace NMonitoring diff --git a/library/cpp/monlib/metrics/ewma.h b/library/cpp/monlib/metrics/ewma.h index 06a2bc5058..9b2dad7cc5 100644 --- a/library/cpp/monlib/metrics/ewma.h +++ b/library/cpp/monlib/metrics/ewma.h @@ -1,47 +1,47 @@ -#pragma once - -#include <util/datetime/base.h> -#include <util/generic/ptr.h> - -#include <atomic> - -namespace NMonitoring { - class IGauge; - - class IExpMovingAverage { - public: - virtual ~IExpMovingAverage() = default; - virtual void Tick() = 0; - virtual void Update(i64 value) = 0; - virtual double Rate() const = 0; - virtual void Reset() = 0; - }; - - using IExpMovingAveragePtr = THolder<IExpMovingAverage>; - - class TEwmaMeter { - public: - // Creates a fake EWMA that will always return 0. Mostly for usage convenience - TEwmaMeter(); - explicit TEwmaMeter(IExpMovingAveragePtr&& ewma); - - TEwmaMeter(TEwmaMeter&& other); - TEwmaMeter& operator=(TEwmaMeter&& other); - - void Mark(); - void Mark(i64 value); - - double Get(); - - private: - void TickIfNeeded(); - - private: - IExpMovingAveragePtr Ewma_; - std::atomic<ui64> LastTick_{TInstant::Now().Seconds()}; - }; - - IExpMovingAveragePtr OneMinuteEwma(IGauge* gauge); - IExpMovingAveragePtr FiveMinuteEwma(IGauge* gauge); - IExpMovingAveragePtr FiveteenMinuteEwma(IGauge* gauge); -} // namespace NMonitoring +#pragma once + +#include <util/datetime/base.h> +#include <util/generic/ptr.h> + +#include <atomic> + +namespace NMonitoring { + class IGauge; + + class IExpMovingAverage { + public: + virtual ~IExpMovingAverage() = default; + virtual void Tick() = 0; + virtual void Update(i64 value) = 0; + virtual double Rate() const = 0; + virtual void Reset() = 0; + }; + + using IExpMovingAveragePtr = THolder<IExpMovingAverage>; + + class TEwmaMeter { + public: + // Creates a fake EWMA that will always return 0. Mostly for usage convenience + TEwmaMeter(); + explicit TEwmaMeter(IExpMovingAveragePtr&& ewma); + + TEwmaMeter(TEwmaMeter&& other); + TEwmaMeter& operator=(TEwmaMeter&& other); + + void Mark(); + void Mark(i64 value); + + double Get(); + + private: + void TickIfNeeded(); + + private: + IExpMovingAveragePtr Ewma_; + std::atomic<ui64> LastTick_{TInstant::Now().Seconds()}; + }; + + IExpMovingAveragePtr OneMinuteEwma(IGauge* gauge); + IExpMovingAveragePtr FiveMinuteEwma(IGauge* gauge); + IExpMovingAveragePtr FiveteenMinuteEwma(IGauge* gauge); +} // namespace NMonitoring diff --git a/library/cpp/monlib/metrics/ewma_ut.cpp b/library/cpp/monlib/metrics/ewma_ut.cpp index 2765c68eae..01ef2478f7 100644 --- a/library/cpp/monlib/metrics/ewma_ut.cpp +++ b/library/cpp/monlib/metrics/ewma_ut.cpp @@ -1,112 +1,112 @@ -#include "ewma.h" +#include "ewma.h" #include "metric.h" - + #include <library/cpp/testing/unittest/registar.h> - - -using namespace NMonitoring; - -const auto EPS = 1e-6; -void ElapseMinute(IExpMovingAverage& ewma) { - for (auto i = 0; i < 12; ++i) { - ewma.Tick(); - } -} - -Y_UNIT_TEST_SUITE(TEwmaTest) { - Y_UNIT_TEST(OneMinute) { + + +using namespace NMonitoring; + +const auto EPS = 1e-6; +void ElapseMinute(IExpMovingAverage& ewma) { + for (auto i = 0; i < 12; ++i) { + ewma.Tick(); + } +} + +Y_UNIT_TEST_SUITE(TEwmaTest) { + Y_UNIT_TEST(OneMinute) { TGauge gauge; - + auto ewma = OneMinuteEwma(&gauge); - ewma->Update(3); - ewma->Tick(); - - TVector<double> expectedValues { - 0.6, - 0.22072766, - 0.08120117, - 0.02987224, - 0.01098938, - 0.00404277, - 0.00148725, - 0.00054713, - 0.00020128, - 0.00007405, - 0.00002724, - 0.00001002, - 0.00000369, - 0.00000136, - 0.00000050, - 0.00000018, - }; - - for (auto expectedValue : expectedValues) { - UNIT_ASSERT_DOUBLES_EQUAL(ewma->Rate(), expectedValue, EPS); - ElapseMinute(*ewma); - } - } - - Y_UNIT_TEST(FiveMinutes) { + ewma->Update(3); + ewma->Tick(); + + TVector<double> expectedValues { + 0.6, + 0.22072766, + 0.08120117, + 0.02987224, + 0.01098938, + 0.00404277, + 0.00148725, + 0.00054713, + 0.00020128, + 0.00007405, + 0.00002724, + 0.00001002, + 0.00000369, + 0.00000136, + 0.00000050, + 0.00000018, + }; + + for (auto expectedValue : expectedValues) { + UNIT_ASSERT_DOUBLES_EQUAL(ewma->Rate(), expectedValue, EPS); + ElapseMinute(*ewma); + } + } + + Y_UNIT_TEST(FiveMinutes) { TGauge gauge; - + auto ewma = FiveMinuteEwma(&gauge); - ewma->Update(3); - ewma->Tick(); - - TVector<double> expectedValues { - 0.6, - 0.49123845, - 0.40219203, - 0.32928698, - 0.26959738, - 0.22072766, - 0.18071653, - 0.14795818, - 0.12113791, - 0.09917933, - 0.08120117, - 0.06648190, - 0.05443077, - 0.04456415, - 0.03648604, - 0.02987224, - }; - - for (auto expectedValue : expectedValues) { - UNIT_ASSERT_DOUBLES_EQUAL(ewma->Rate(), expectedValue, EPS); - ElapseMinute(*ewma); - } - } - - Y_UNIT_TEST(FiveteenMinutes) { + ewma->Update(3); + ewma->Tick(); + + TVector<double> expectedValues { + 0.6, + 0.49123845, + 0.40219203, + 0.32928698, + 0.26959738, + 0.22072766, + 0.18071653, + 0.14795818, + 0.12113791, + 0.09917933, + 0.08120117, + 0.06648190, + 0.05443077, + 0.04456415, + 0.03648604, + 0.02987224, + }; + + for (auto expectedValue : expectedValues) { + UNIT_ASSERT_DOUBLES_EQUAL(ewma->Rate(), expectedValue, EPS); + ElapseMinute(*ewma); + } + } + + Y_UNIT_TEST(FiveteenMinutes) { TGauge gauge; - + auto ewma = FiveteenMinuteEwma(&gauge); - ewma->Update(3); - ewma->Tick(); - - TVector<double> expectedValues { - 0.6, - 0.56130419, - 0.52510399, - 0.49123845, - 0.45955700, - 0.42991879, - 0.40219203, - 0.37625345, - 0.35198773, - 0.32928698, - 0.30805027, - 0.28818318, - 0.26959738, - 0.25221023, - 0.23594443, - 0.22072766, - }; - - for (auto expectedValue : expectedValues) { - UNIT_ASSERT_DOUBLES_EQUAL(ewma->Rate(), expectedValue, EPS); - ElapseMinute(*ewma); - } - } -}; + ewma->Update(3); + ewma->Tick(); + + TVector<double> expectedValues { + 0.6, + 0.56130419, + 0.52510399, + 0.49123845, + 0.45955700, + 0.42991879, + 0.40219203, + 0.37625345, + 0.35198773, + 0.32928698, + 0.30805027, + 0.28818318, + 0.26959738, + 0.25221023, + 0.23594443, + 0.22072766, + }; + + for (auto expectedValue : expectedValues) { + UNIT_ASSERT_DOUBLES_EQUAL(ewma->Rate(), expectedValue, EPS); + ElapseMinute(*ewma); + } + } +}; diff --git a/library/cpp/monlib/metrics/fake.cpp b/library/cpp/monlib/metrics/fake.cpp index 6890ca5ab0..b6f5e37af8 100644 --- a/library/cpp/monlib/metrics/fake.cpp +++ b/library/cpp/monlib/metrics/fake.cpp @@ -1,11 +1,11 @@ -#include "fake.h" - -namespace NMonitoring { +#include "fake.h" + +namespace NMonitoring { IGauge* TFakeMetricRegistry::Gauge(ILabelsPtr labels) { return Metric<TFakeGauge, EMetricType::GAUGE>(std::move(labels)); - } - + } + ILazyGauge* TFakeMetricRegistry::LazyGauge(ILabelsPtr labels, std::function<double()> supplier) { Y_UNUSED(supplier); return Metric<TFakeLazyGauge, EMetricType::GAUGE>(std::move(labels)); @@ -13,8 +13,8 @@ namespace NMonitoring { IIntGauge* TFakeMetricRegistry::IntGauge(ILabelsPtr labels) { return Metric<TFakeIntGauge, EMetricType::IGAUGE>(std::move(labels)); - } - + } + ILazyIntGauge* TFakeMetricRegistry::LazyIntGauge(ILabelsPtr labels, std::function<i64()> supplier) { Y_UNUSED(supplier); return Metric<TFakeLazyIntGauge, EMetricType::IGAUGE>(std::move(labels)); @@ -22,8 +22,8 @@ namespace NMonitoring { ICounter* TFakeMetricRegistry::Counter(ILabelsPtr labels) { return Metric<TFakeCounter, EMetricType::COUNTER>(std::move(labels)); - } - + } + ILazyCounter* TFakeMetricRegistry::LazyCounter(ILabelsPtr labels, std::function<ui64()> supplier) { Y_UNUSED(supplier); return Metric<TFakeLazyCounter, EMetricType::COUNTER>(std::move(labels)); @@ -31,70 +31,70 @@ namespace NMonitoring { IRate* TFakeMetricRegistry::Rate(ILabelsPtr labels) { return Metric<TFakeRate, EMetricType::RATE>(std::move(labels)); - } - + } + ILazyRate* TFakeMetricRegistry::LazyRate(ILabelsPtr labels, std::function<ui64()> supplier) { Y_UNUSED(supplier); return Metric<TFakeLazyRate, EMetricType::RATE>(std::move(labels)); } IHistogram* TFakeMetricRegistry::HistogramCounter(ILabelsPtr labels, IHistogramCollectorPtr collector) { - Y_UNUSED(collector); + Y_UNUSED(collector); return Metric<TFakeHistogram, EMetricType::HIST>(std::move(labels), false); - } - + } + void TFakeMetricRegistry::RemoveMetric(const ILabels& labels) noexcept { - TWriteGuard g{Lock_}; + TWriteGuard g{Lock_}; Metrics_.erase(labels); - } - + } + void TFakeMetricRegistry::Accept(TInstant time, IMetricConsumer* consumer) const { - Y_UNUSED(time); - consumer->OnStreamBegin(); - consumer->OnStreamEnd(); - } - + Y_UNUSED(time); + consumer->OnStreamBegin(); + consumer->OnStreamEnd(); + } + IHistogram* TFakeMetricRegistry::HistogramRate(ILabelsPtr labels, IHistogramCollectorPtr collector) { - Y_UNUSED(collector); + Y_UNUSED(collector); return Metric<TFakeHistogram, EMetricType::HIST_RATE>(std::move(labels), true); - } - + } + void TFakeMetricRegistry::Append(TInstant time, IMetricConsumer* consumer) const { - Y_UNUSED(time, consumer); - } - + Y_UNUSED(time, consumer); + } + const TLabels& TFakeMetricRegistry::CommonLabels() const noexcept { - return CommonLabels_; - } - + return CommonLabels_; + } + template <typename TMetric, EMetricType type, typename TLabelsType, typename... Args> TMetric* TFakeMetricRegistry::Metric(TLabelsType&& labels, Args&&... args) { - { - TReadGuard g{Lock_}; - + { + TReadGuard g{Lock_}; + auto it = Metrics_.find(labels); if (it != Metrics_.end()) { Y_ENSURE(it->second->Type() == type, "cannot create metric " << labels << " with type " << MetricTypeToStr(type) << ", because registry already has same metric with type " << MetricTypeToStr(it->second->Type())); return static_cast<TMetric*>(it->second.Get()); - } - } - - { - TWriteGuard g{Lock_}; - + } + } + + { + TWriteGuard g{Lock_}; + IMetricPtr metric = MakeHolder<TMetric>(std::forward<Args>(args)...); - + // decltype(Metrics_)::iterator breaks build on windows THashMap<ILabelsPtr, IMetricPtr>::iterator it; - if constexpr (!std::is_convertible_v<TLabelsType, ILabelsPtr>) { + if constexpr (!std::is_convertible_v<TLabelsType, ILabelsPtr>) { it = Metrics_.emplace(new TLabels{std::forward<TLabelsType>(labels)}, std::move(metric)).first; - } else { + } else { it = Metrics_.emplace(std::forward<TLabelsType>(labels), std::move(metric)).first; - } - + } + return static_cast<TMetric*>(it->second.Get()); - } - } -} // namespace NMonitoring + } + } +} // namespace NMonitoring diff --git a/library/cpp/monlib/metrics/fake.h b/library/cpp/monlib/metrics/fake.h index 8787d37cb6..61ba4f2bd4 100644 --- a/library/cpp/monlib/metrics/fake.h +++ b/library/cpp/monlib/metrics/fake.h @@ -1,11 +1,11 @@ -#pragma once - +#pragma once + #include "metric.h" #include "metric_registry.h" - -namespace NMonitoring { + +namespace NMonitoring { class TFakeMetricRegistry: public IMetricRegistry { - public: + public: TFakeMetricRegistry() noexcept : CommonLabels_{0} { @@ -16,150 +16,150 @@ namespace NMonitoring { { } - IGauge* Gauge(ILabelsPtr labels) override; + IGauge* Gauge(ILabelsPtr labels) override; ILazyGauge* LazyGauge(ILabelsPtr labels, std::function<double()> supplier) override; - IIntGauge* IntGauge(ILabelsPtr labels) override; + IIntGauge* IntGauge(ILabelsPtr labels) override; ILazyIntGauge* LazyIntGauge(ILabelsPtr labels, std::function<i64()> supplier) override; - ICounter* Counter(ILabelsPtr labels) override; + ICounter* Counter(ILabelsPtr labels) override; ILazyCounter* LazyCounter(ILabelsPtr labels, std::function<ui64()> supplier) override; - IRate* Rate(ILabelsPtr labels) override; + IRate* Rate(ILabelsPtr labels) override; ILazyRate* LazyRate(ILabelsPtr labels, std::function<ui64()> supplier) override; - - IHistogram* HistogramCounter( - ILabelsPtr labels, - IHistogramCollectorPtr collector) override; - - IHistogram* HistogramRate( - ILabelsPtr labels, - IHistogramCollectorPtr collector) override; + + IHistogram* HistogramCounter( + ILabelsPtr labels, + IHistogramCollectorPtr collector) override; + + IHistogram* HistogramRate( + ILabelsPtr labels, + IHistogramCollectorPtr collector) override; void Accept(TInstant time, IMetricConsumer* consumer) const override; void Append(TInstant time, IMetricConsumer* consumer) const override; - - const TLabels& CommonLabels() const noexcept override; + + const TLabels& CommonLabels() const noexcept override; void RemoveMetric(const ILabels& labels) noexcept override; - - private: - TRWMutex Lock_; + + private: + TRWMutex Lock_; THashMap<ILabelsPtr, IMetricPtr> Metrics_; - + template <typename TMetric, EMetricType type, typename TLabelsType, typename... Args> TMetric* Metric(TLabelsType&& labels, Args&&... args); - - const TLabels CommonLabels_; - }; - - template <typename TBase> - struct TFakeAcceptor: TBase { + + const TLabels CommonLabels_; + }; + + template <typename TBase> + struct TFakeAcceptor: TBase { void Accept(TInstant time, IMetricConsumer* consumer) const override { - Y_UNUSED(time, consumer); - } - }; - - struct TFakeIntGauge final: public TFakeAcceptor<IIntGauge> { - i64 Add(i64 n) noexcept override { - Y_UNUSED(n); - return 0; - } - - void Set(i64 n) noexcept override { - Y_UNUSED(n); - } - - i64 Get() const noexcept override { - return 0; - } - }; - + Y_UNUSED(time, consumer); + } + }; + + struct TFakeIntGauge final: public TFakeAcceptor<IIntGauge> { + i64 Add(i64 n) noexcept override { + Y_UNUSED(n); + return 0; + } + + void Set(i64 n) noexcept override { + Y_UNUSED(n); + } + + i64 Get() const noexcept override { + return 0; + } + }; + struct TFakeLazyIntGauge final: public TFakeAcceptor<ILazyIntGauge> { i64 Get() const noexcept override { return 0; } }; - struct TFakeRate final: public TFakeAcceptor<IRate> { - ui64 Add(ui64 n) noexcept override { - Y_UNUSED(n); - return 0; - } - - ui64 Get() const noexcept override { - return 0; - } + struct TFakeRate final: public TFakeAcceptor<IRate> { + ui64 Add(ui64 n) noexcept override { + Y_UNUSED(n); + return 0; + } + + ui64 Get() const noexcept override { + return 0; + } void Reset() noexcept override { } - }; - + }; + struct TFakeLazyRate final: public TFakeAcceptor<ILazyRate> { ui64 Get() const noexcept override { return 0; } }; - struct TFakeGauge final: public TFakeAcceptor<IGauge> { - double Add(double n) noexcept override { - Y_UNUSED(n); - return 0; - } - - void Set(double n) noexcept override { - Y_UNUSED(n); - } - - double Get() const noexcept override { - return 0; - } - }; - + struct TFakeGauge final: public TFakeAcceptor<IGauge> { + double Add(double n) noexcept override { + Y_UNUSED(n); + return 0; + } + + void Set(double n) noexcept override { + Y_UNUSED(n); + } + + double Get() const noexcept override { + return 0; + } + }; + struct TFakeLazyGauge final: public TFakeAcceptor<ILazyGauge> { double Get() const noexcept override { return 0; } }; - struct TFakeHistogram final: public IHistogram { - TFakeHistogram(bool isRate = false) - : IHistogram{isRate} - { - } - + struct TFakeHistogram final: public IHistogram { + TFakeHistogram(bool isRate = false) + : IHistogram{isRate} + { + } + void Record(double value) override { - Y_UNUSED(value); - } - + Y_UNUSED(value); + } + void Record(double value, ui32 count) override { - Y_UNUSED(value, count); - } - - IHistogramSnapshotPtr TakeSnapshot() const override { - return nullptr; - } - + Y_UNUSED(value, count); + } + + IHistogramSnapshotPtr TakeSnapshot() const override { + return nullptr; + } + void Accept(TInstant time, IMetricConsumer* consumer) const override { - Y_UNUSED(time, consumer); - } - - void Reset() override { - } - }; - - struct TFakeCounter final: public TFakeAcceptor<ICounter> { - ui64 Add(ui64 n) noexcept override { - Y_UNUSED(n); - return 0; - } - - ui64 Get() const noexcept override { - return 0; - } - - void Reset() noexcept override { - } - }; + Y_UNUSED(time, consumer); + } + + void Reset() override { + } + }; + + struct TFakeCounter final: public TFakeAcceptor<ICounter> { + ui64 Add(ui64 n) noexcept override { + Y_UNUSED(n); + return 0; + } + + ui64 Get() const noexcept override { + return 0; + } + + void Reset() noexcept override { + } + }; struct TFakeLazyCounter final: public TFakeAcceptor<ILazyCounter> { ui64 Get() const noexcept override { return 0; } }; -} // namespace NMonitoring +} // namespace NMonitoring diff --git a/library/cpp/monlib/metrics/histogram_collector.h b/library/cpp/monlib/metrics/histogram_collector.h index 6e19e2da63..9f6bbbdfb7 100644 --- a/library/cpp/monlib/metrics/histogram_collector.h +++ b/library/cpp/monlib/metrics/histogram_collector.h @@ -68,7 +68,7 @@ namespace NMonitoring { * * @param bounds array of upper bounds for buckets. Values must be sorted. */ - IHistogramCollectorPtr ExplicitHistogram(TBucketBounds bounds); + IHistogramCollectorPtr ExplicitHistogram(TBucketBounds bounds); /** * <p>Creates histogram collector for a sequence of buckets that have a diff --git a/library/cpp/monlib/metrics/histogram_collector_explicit.cpp b/library/cpp/monlib/metrics/histogram_collector_explicit.cpp index bfeeb2531d..377fc233ef 100644 --- a/library/cpp/monlib/metrics/histogram_collector_explicit.cpp +++ b/library/cpp/monlib/metrics/histogram_collector_explicit.cpp @@ -13,7 +13,7 @@ namespace NMonitoring { /////////////////////////////////////////////////////////////////////////// class TExplicitHistogramCollector: public IHistogramCollector { public: - TExplicitHistogramCollector(TBucketBounds bounds) + TExplicitHistogramCollector(TBucketBounds bounds) : Values_(bounds.size() + 1) , Bounds_(std::move(bounds)) { @@ -32,8 +32,8 @@ namespace NMonitoring { } IHistogramSnapshotPtr Snapshot() const override { - auto values = Values_.Copy(); - return ExplicitHistogramSnapshot(Bounds_, values); + auto values = Values_.Copy(); + return ExplicitHistogramSnapshot(Bounds_, values); } private: diff --git a/library/cpp/monlib/metrics/histogram_snapshot.cpp b/library/cpp/monlib/metrics/histogram_snapshot.cpp index f00b972b78..75b5811546 100644 --- a/library/cpp/monlib/metrics/histogram_snapshot.cpp +++ b/library/cpp/monlib/metrics/histogram_snapshot.cpp @@ -7,7 +7,7 @@ namespace NMonitoring { - IHistogramSnapshotPtr ExplicitHistogramSnapshot(TConstArrayRef<TBucketBound> bounds, TConstArrayRef<TBucketValue> values) { + IHistogramSnapshotPtr ExplicitHistogramSnapshot(TConstArrayRef<TBucketBound> bounds, TConstArrayRef<TBucketValue> values) { Y_ENSURE(bounds.size() == values.size(), "mismatched sizes: bounds(" << bounds.size() << ") != buckets(" << values.size() << ')'); diff --git a/library/cpp/monlib/metrics/histogram_snapshot.h b/library/cpp/monlib/metrics/histogram_snapshot.h index 6fc4e51083..e8acf6ac2b 100644 --- a/library/cpp/monlib/metrics/histogram_snapshot.h +++ b/library/cpp/monlib/metrics/histogram_snapshot.h @@ -1,6 +1,6 @@ #pragma once -#include <util/generic/array_ref.h> +#include <util/generic/array_ref.h> #include <util/generic/ptr.h> #include <util/generic/vector.h> #include <util/generic/yexception.h> @@ -203,7 +203,7 @@ namespace NMonitoring { static_assert(alignof(TExplicitHistogramSnapshot) == alignof(TBucket), "mismatched alingments of THistogramSnapshot and TBucket"); - IHistogramSnapshotPtr ExplicitHistogramSnapshot(TConstArrayRef<TBucketBound> bounds, TConstArrayRef<TBucketValue> values); + IHistogramSnapshotPtr ExplicitHistogramSnapshot(TConstArrayRef<TBucketBound> bounds, TConstArrayRef<TBucketValue> values); } // namespace NMonitoring diff --git a/library/cpp/monlib/metrics/labels.cpp b/library/cpp/monlib/metrics/labels.cpp index 4dc27a11fc..1eaadb7cba 100644 --- a/library/cpp/monlib/metrics/labels.cpp +++ b/library/cpp/monlib/metrics/labels.cpp @@ -1,21 +1,21 @@ #include "labels.h" -#include <util/stream/output.h> -#include <util/string/split.h> +#include <util/stream/output.h> +#include <util/string/split.h> static void OutputLabels(IOutputStream& out, const NMonitoring::ILabels& labels) { size_t i = 0; - out << '{'; + out << '{'; for (const auto& label: labels) { if (i++ > 0) { out << TStringBuf(", "); - } + } out << label; } - out << '}'; -} - -template <> + out << '}'; +} + +template <> void Out<NMonitoring::ILabelsPtr>(IOutputStream& out, const NMonitoring::ILabelsPtr& labels) { OutputLabels(out, *labels); } @@ -26,57 +26,57 @@ void Out<NMonitoring::ILabels>(IOutputStream& out, const NMonitoring::ILabels& l } template <> -void Out<NMonitoring::ILabel>(IOutputStream& out, const NMonitoring::ILabel& labels) { - out << labels.Name() << "=" << labels.Value(); -} - -Y_MONLIB_DEFINE_LABELS_OUT(NMonitoring::TLabels); -Y_MONLIB_DEFINE_LABEL_OUT(NMonitoring::TLabel); - -namespace NMonitoring { - bool TryLoadLabelsFromString(TStringBuf sb, ILabels& labels) { - if (sb.Empty()) { - return false; - } - - if (!sb.StartsWith('{') || !sb.EndsWith('}')) { - return false; - } - - sb.Skip(1); - sb.Chop(1); - - if (sb.Empty()) { - return true; - } - - bool ok = true; - TVector<std::pair<TStringBuf, TStringBuf>> rawLabels; - StringSplitter(sb).SplitBySet(" ,").SkipEmpty().Consume([&] (TStringBuf label) { - TStringBuf key, value; - ok &= label.TrySplit('=', key, value); - - if (!ok) { - return; - } - - rawLabels.emplace_back(key, value); - }); - - if (!ok) { - return false; - } - - for (auto&& [k, v] : rawLabels) { - labels.Add(k, v); - } - - return true; - } - - bool TryLoadLabelsFromString(IInputStream& is, ILabels& labels) { - TString str = is.ReadAll(); - return TryLoadLabelsFromString(str, labels); - } - -} // namespace NMonitoring +void Out<NMonitoring::ILabel>(IOutputStream& out, const NMonitoring::ILabel& labels) { + out << labels.Name() << "=" << labels.Value(); +} + +Y_MONLIB_DEFINE_LABELS_OUT(NMonitoring::TLabels); +Y_MONLIB_DEFINE_LABEL_OUT(NMonitoring::TLabel); + +namespace NMonitoring { + bool TryLoadLabelsFromString(TStringBuf sb, ILabels& labels) { + if (sb.Empty()) { + return false; + } + + if (!sb.StartsWith('{') || !sb.EndsWith('}')) { + return false; + } + + sb.Skip(1); + sb.Chop(1); + + if (sb.Empty()) { + return true; + } + + bool ok = true; + TVector<std::pair<TStringBuf, TStringBuf>> rawLabels; + StringSplitter(sb).SplitBySet(" ,").SkipEmpty().Consume([&] (TStringBuf label) { + TStringBuf key, value; + ok &= label.TrySplit('=', key, value); + + if (!ok) { + return; + } + + rawLabels.emplace_back(key, value); + }); + + if (!ok) { + return false; + } + + for (auto&& [k, v] : rawLabels) { + labels.Add(k, v); + } + + return true; + } + + bool TryLoadLabelsFromString(IInputStream& is, ILabels& labels) { + TString str = is.ReadAll(); + return TryLoadLabelsFromString(str, labels); + } + +} // namespace NMonitoring diff --git a/library/cpp/monlib/metrics/labels.h b/library/cpp/monlib/metrics/labels.h index f374148544..63dc997c28 100644 --- a/library/cpp/monlib/metrics/labels.h +++ b/library/cpp/monlib/metrics/labels.h @@ -6,26 +6,26 @@ #include <util/generic/maybe.h> #include <util/generic/string.h> #include <util/generic/vector.h> -#include <util/stream/output.h> +#include <util/stream/output.h> #include <util/string/builder.h> #include <util/string/strip.h> -#include <optional> -#include <type_traits> - +#include <optional> +#include <type_traits> + namespace NMonitoring { - struct ILabel { - virtual ~ILabel() = default; - - virtual TStringBuf Name() const noexcept = 0; - virtual TStringBuf Value() const noexcept = 0; - }; - + struct ILabel { + virtual ~ILabel() = default; + + virtual TStringBuf Name() const noexcept = 0; + virtual TStringBuf Value() const noexcept = 0; + }; + /////////////////////////////////////////////////////////////////////////// // TLabel /////////////////////////////////////////////////////////////////////////// template <typename TStringBackend> - class TLabelImpl: public ILabel { + class TLabelImpl: public ILabel { public: using TStringType = TStringBackend; @@ -45,22 +45,22 @@ namespace NMonitoring { return !(*this == rhs); } - inline TStringBuf Name() const noexcept { + inline TStringBuf Name() const noexcept { + return Name_; + } + + inline TStringBuf Value() const noexcept { + return Value_; + } + + inline const TStringBackend& NameStr() const { return Name_; } - inline TStringBuf Value() const noexcept { + inline const TStringBackend& ValueStr() const { return Value_; } - inline const TStringBackend& NameStr() const { - return Name_; - } - - inline const TStringBackend& ValueStr() const { - return Value_; - } - inline size_t Hash() const noexcept { return MultiHash(Name_, Value_); } @@ -114,80 +114,80 @@ namespace NMonitoring { using TLabel = TLabelImpl<TString>; - struct ILabels { - struct TIterator { - TIterator() = default; - TIterator(const ILabels* labels, size_t idx = 0) - : Labels_{labels} - , Idx_{idx} - { - } - - TIterator& operator++() noexcept { - Idx_++; - return *this; - } - - void operator+=(size_t i) noexcept { - Idx_ += i; - } - - bool operator==(const TIterator& other) const noexcept { - return Idx_ == other.Idx_; - } - - bool operator!=(const TIterator& other) const noexcept { - return !(*this == other); - } - + struct ILabels { + struct TIterator { + TIterator() = default; + TIterator(const ILabels* labels, size_t idx = 0) + : Labels_{labels} + , Idx_{idx} + { + } + + TIterator& operator++() noexcept { + Idx_++; + return *this; + } + + void operator+=(size_t i) noexcept { + Idx_ += i; + } + + bool operator==(const TIterator& other) const noexcept { + return Idx_ == other.Idx_; + } + + bool operator!=(const TIterator& other) const noexcept { + return !(*this == other); + } + const ILabel* operator->() const noexcept { Y_VERIFY_DEBUG(Labels_); return Labels_->Get(Idx_); } - const ILabel& operator*() const noexcept { - Y_VERIFY_DEBUG(Labels_); - return *Labels_->Get(Idx_); - } - - - private: - const ILabels* Labels_{nullptr}; - size_t Idx_{0}; - }; - - virtual ~ILabels() = default; - + const ILabel& operator*() const noexcept { + Y_VERIFY_DEBUG(Labels_); + return *Labels_->Get(Idx_); + } + + + private: + const ILabels* Labels_{nullptr}; + size_t Idx_{0}; + }; + + virtual ~ILabels() = default; + virtual bool Add(TStringBuf name, TStringBuf value) noexcept = 0; - virtual bool Add(const ILabel& label) noexcept { - return Add(label.Name(), label.Value()); - } - + virtual bool Add(const ILabel& label) noexcept { + return Add(label.Name(), label.Value()); + } + virtual bool Has(TStringBuf name) const noexcept = 0; - - virtual size_t Size() const noexcept = 0; - virtual bool Empty() const noexcept = 0; - virtual void Clear() noexcept = 0; - - virtual size_t Hash() const noexcept = 0; - + + virtual size_t Size() const noexcept = 0; + virtual bool Empty() const noexcept = 0; + virtual void Clear() noexcept = 0; + + virtual size_t Hash() const noexcept = 0; + virtual std::optional<const ILabel*> Get(TStringBuf name) const = 0; - - // NB: there's no guarantee that indices are preserved after any object modification + + // NB: there's no guarantee that indices are preserved after any object modification virtual const ILabel* Get(size_t idx) const = 0; - - TIterator begin() const { - return TIterator{this}; - } - - TIterator end() const { - return TIterator{this, Size()}; - } - }; - - bool TryLoadLabelsFromString(TStringBuf sb, ILabels& labels); - bool TryLoadLabelsFromString(IInputStream& is, ILabels& labels); - + + TIterator begin() const { + return TIterator{this}; + } + + TIterator end() const { + return TIterator{this, Size()}; + } + }; + + bool TryLoadLabelsFromString(TStringBuf sb, ILabels& labels); + bool TryLoadLabelsFromString(IInputStream& is, ILabels& labels); + /////////////////////////////////////////////////////////////////////////// // TLabels /////////////////////////////////////////////////////////////////////////// @@ -201,15 +201,15 @@ namespace NMonitoring { explicit TLabelsImpl(::NDetail::TReserveTag rt) : Labels_(std::move(rt)) {} - + explicit TLabelsImpl(size_t count) : Labels_(count) {} - + explicit TLabelsImpl(size_t count, const value_type& label) : Labels_(count, label) {} - + TLabelsImpl(std::initializer_list<value_type> il) : Labels_(std::move(il)) {} @@ -237,7 +237,7 @@ namespace NMonitoring { return true; } - using ILabels::Add; + using ILabels::Add; bool Has(TStringBuf name) const noexcept override { auto it = FindIf(Labels_, [name](const TLabelImpl<TStringBackend>& label) { @@ -253,7 +253,7 @@ namespace NMonitoring { return it != Labels_.end(); } - // XXX for backward compatibility + // XXX for backward compatibility TMaybe<TLabelImpl<TStringBackend>> Find(TStringBuf name) const { auto it = FindIf(Labels_, [name](const TLabelImpl<TStringBackend>& label) { return name == TStringBuf{label.Name()}; @@ -266,20 +266,20 @@ namespace NMonitoring { std::optional<const ILabel*> Get(TStringBuf name) const override { auto it = FindIf(Labels_, [name] (auto&& l) { - return name == l.Name(); - }); - + return name == l.Name(); + }); + if (it == Labels_.end()) { - return {}; - } - - return &*it; - } - - const ILabel* Get(size_t idx) const noexcept override { - return &(*this)[idx]; - } - + return {}; + } + + return &*it; + } + + const ILabel* Get(size_t idx) const noexcept override { + return &(*this)[idx]; + } + TMaybe<TLabelImpl<TStringBackend>> Extract(TStringBuf name) { auto it = FindIf(Labels_, [name](const TLabelImpl<TStringBackend>& label) { return name == TStringBuf{label.Name()}; @@ -298,7 +298,7 @@ namespace NMonitoring { }); } - inline size_t Hash() const noexcept override { + inline size_t Hash() const noexcept override { return TSimpleRangeHash()(Labels_); } @@ -306,18 +306,18 @@ namespace NMonitoring { return const_cast<TLabel*>(Labels_.data()); } - inline size_t Size() const noexcept override { + inline size_t Size() const noexcept override { return Labels_.size(); } - inline bool Empty() const noexcept override { + inline bool Empty() const noexcept override { return Labels_.empty(); } - inline void Clear() noexcept override { + inline void Clear() noexcept override { Labels_.clear(); }; - + TLabelImpl<TStringBackend>& front() { return Labels_.front(); } @@ -383,7 +383,7 @@ namespace NMonitoring { protected: TVector<TLabelImpl<TStringBackend>>& AsVector() { return Labels_; - } + } const TVector<TLabelImpl<TStringBackend>>& AsVector() const { return Labels_; @@ -394,7 +394,7 @@ namespace NMonitoring { }; using TLabels = TLabelsImpl<TString>; - using ILabelsPtr = THolder<ILabels>; + using ILabelsPtr = THolder<ILabels>; template <typename T> ILabelsPtr MakeLabels() { @@ -411,73 +411,73 @@ namespace NMonitoring { } } -template<> -struct THash<NMonitoring::ILabelsPtr> { - size_t operator()(const NMonitoring::ILabelsPtr& labels) const noexcept { - return labels->Hash(); - } - - size_t operator()(const NMonitoring::ILabels& labels) const noexcept { - return labels.Hash(); - } -}; - -template<typename TStringBackend> -struct THash<NMonitoring::TLabelsImpl<TStringBackend>> { +template<> +struct THash<NMonitoring::ILabelsPtr> { + size_t operator()(const NMonitoring::ILabelsPtr& labels) const noexcept { + return labels->Hash(); + } + + size_t operator()(const NMonitoring::ILabels& labels) const noexcept { + return labels.Hash(); + } +}; + +template<typename TStringBackend> +struct THash<NMonitoring::TLabelsImpl<TStringBackend>> { size_t operator()(const NMonitoring::TLabelsImpl<TStringBackend>& labels) const noexcept { - return labels.Hash(); - } -}; - + return labels.Hash(); + } +}; + template <typename TStringBackend> struct THash<NMonitoring::TLabelImpl<TStringBackend>> { - inline size_t operator()(const NMonitoring::TLabelImpl<TStringBackend>& label) const noexcept { + inline size_t operator()(const NMonitoring::TLabelImpl<TStringBackend>& label) const noexcept { return label.Hash(); } }; -inline bool operator==(const NMonitoring::ILabels& lhs, const NMonitoring::ILabels& rhs) { - if (lhs.Size() != rhs.Size()) { - return false; +inline bool operator==(const NMonitoring::ILabels& lhs, const NMonitoring::ILabels& rhs) { + if (lhs.Size() != rhs.Size()) { + return false; + } + + for (auto&& l : lhs) { + auto rl = rhs.Get(l.Name()); + if (!rl || (*rl)->Value() != l.Value()) { + return false; + } } - - for (auto&& l : lhs) { - auto rl = rhs.Get(l.Name()); - if (!rl || (*rl)->Value() != l.Value()) { - return false; - } - } - - return true; -} - + + return true; +} + bool operator==(const NMonitoring::ILabelsPtr& lhs, const NMonitoring::ILabelsPtr& rhs) = delete; bool operator==(const NMonitoring::ILabels& lhs, const NMonitoring::ILabelsPtr& rhs) = delete; bool operator==(const NMonitoring::ILabelsPtr& lhs, const NMonitoring::ILabels& rhs) = delete; - -template<> -struct TEqualTo<NMonitoring::ILabelsPtr> { - bool operator()(const NMonitoring::ILabelsPtr& lhs, const NMonitoring::ILabelsPtr& rhs) { + +template<> +struct TEqualTo<NMonitoring::ILabelsPtr> { + bool operator()(const NMonitoring::ILabelsPtr& lhs, const NMonitoring::ILabelsPtr& rhs) { return *lhs == *rhs; - } - - bool operator()(const NMonitoring::ILabelsPtr& lhs, const NMonitoring::ILabels& rhs) { + } + + bool operator()(const NMonitoring::ILabelsPtr& lhs, const NMonitoring::ILabels& rhs) { return *lhs == rhs; - } - - bool operator()(const NMonitoring::ILabels& lhs, const NMonitoring::ILabelsPtr& rhs) { + } + + bool operator()(const NMonitoring::ILabels& lhs, const NMonitoring::ILabelsPtr& rhs) { return lhs == *rhs; - } + } }; -#define Y_MONLIB_DEFINE_LABELS_OUT(T) \ +#define Y_MONLIB_DEFINE_LABELS_OUT(T) \ template <> \ -void Out<T>(IOutputStream& out, const T& labels) { \ - Out<NMonitoring::ILabels>(out, labels); \ +void Out<T>(IOutputStream& out, const T& labels) { \ + Out<NMonitoring::ILabels>(out, labels); \ } -#define Y_MONLIB_DEFINE_LABEL_OUT(T) \ +#define Y_MONLIB_DEFINE_LABEL_OUT(T) \ template <> \ -void Out<T>(IOutputStream& out, const T& label) { \ - Out<NMonitoring::ILabel>(out, label); \ +void Out<T>(IOutputStream& out, const T& label) { \ + Out<NMonitoring::ILabel>(out, label); \ } diff --git a/library/cpp/monlib/metrics/labels_ut.cpp b/library/cpp/monlib/metrics/labels_ut.cpp index 5fa4f9132c..f0e4f532ab 100644 --- a/library/cpp/monlib/metrics/labels_ut.cpp +++ b/library/cpp/monlib/metrics/labels_ut.cpp @@ -125,8 +125,8 @@ Y_UNIT_TEST_SUITE(TLabelsTest) { UNIT_ASSERT_EQUAL(labels[1], TLabel("name2", "value2")); TVector<TLabel> labelsCopy; - for (auto&& label : labels) { - labelsCopy.emplace_back(label.Name(), label.Value()); + for (auto&& label : labels) { + labelsCopy.emplace_back(label.Name(), label.Value()); } UNIT_ASSERT_EQUAL(labelsCopy, TVector<TLabel>({ diff --git a/library/cpp/monlib/metrics/log_histogram_snapshot.cpp b/library/cpp/monlib/metrics/log_histogram_snapshot.cpp index 1f444f2157..21cf2ca2bb 100644 --- a/library/cpp/monlib/metrics/log_histogram_snapshot.cpp +++ b/library/cpp/monlib/metrics/log_histogram_snapshot.cpp @@ -1,9 +1,9 @@ -#include "log_histogram_snapshot.h" - -#include <util/stream/output.h> - +#include "log_histogram_snapshot.h" + +#include <util/stream/output.h> + #include <iostream> - + namespace { @@ -14,14 +14,14 @@ auto& Output(TStream& o, const NMonitoring::TLogHistogramSnapshot& hist) { for (auto i = 0u; i < hist.Count(); ++i) { o << hist.UpperBound(i) << TStringBuf(": ") << hist.Bucket(i); o << TStringBuf(", "); - } - + } + o << TStringBuf("zeros: ") << hist.ZerosCount(); - + o << TStringBuf("}"); return o; -} +} } // namespace diff --git a/library/cpp/monlib/metrics/metric.h b/library/cpp/monlib/metrics/metric.h index 95c9332e0a..b8ce12d753 100644 --- a/library/cpp/monlib/metrics/metric.h +++ b/library/cpp/monlib/metrics/metric.h @@ -20,19 +20,19 @@ namespace NMonitoring { using IMetricPtr = THolder<IMetric>; class IGauge: public IMetric { - public: + public: EMetricType Type() const noexcept final { return EMetricType::GAUGE; - } - - virtual double Add(double n) noexcept = 0; - virtual void Set(double n) noexcept = 0; - virtual double Get() const noexcept = 0; - virtual void Reset() noexcept { - Set(0); - } - }; - + } + + virtual double Add(double n) noexcept = 0; + virtual void Set(double n) noexcept = 0; + virtual double Get() const noexcept = 0; + virtual void Reset() noexcept { + Set(0); + } + }; + class ILazyGauge: public IMetric { public: EMetricType Type() const noexcept final { @@ -42,27 +42,27 @@ namespace NMonitoring { }; class IIntGauge: public IMetric { - public: + public: EMetricType Type() const noexcept final { return EMetricType::IGAUGE; - } - - virtual i64 Add(i64 n) noexcept = 0; - virtual i64 Inc() noexcept { - return Add(1); - } - - virtual i64 Dec() noexcept { - return Add(-1); - } - - virtual void Set(i64 value) noexcept = 0; - virtual i64 Get() const noexcept = 0; - virtual void Reset() noexcept { - Set(0); - } - }; - + } + + virtual i64 Add(i64 n) noexcept = 0; + virtual i64 Inc() noexcept { + return Add(1); + } + + virtual i64 Dec() noexcept { + return Add(-1); + } + + virtual void Set(i64 value) noexcept = 0; + virtual i64 Get() const noexcept = 0; + virtual void Reset() noexcept { + Set(0); + } + }; + class ILazyIntGauge: public IMetric { public: EMetricType Type() const noexcept final { @@ -73,20 +73,20 @@ namespace NMonitoring { }; class ICounter: public IMetric { - public: + public: EMetricType Type() const noexcept final { return EMetricType::COUNTER; - } - - virtual ui64 Inc() noexcept { - return Add(1); - } - - virtual ui64 Add(ui64 n) noexcept = 0; - virtual ui64 Get() const noexcept = 0; - virtual void Reset() noexcept = 0; - }; - + } + + virtual ui64 Inc() noexcept { + return Add(1); + } + + virtual ui64 Add(ui64 n) noexcept = 0; + virtual ui64 Get() const noexcept = 0; + virtual void Reset() noexcept = 0; + }; + class ILazyCounter: public IMetric { public: EMetricType Type() const noexcept final { @@ -97,20 +97,20 @@ namespace NMonitoring { }; class IRate: public IMetric { - public: + public: EMetricType Type() const noexcept final { return EMetricType::RATE; - } - - virtual ui64 Inc() noexcept { - return Add(1); - } - - virtual ui64 Add(ui64 n) noexcept = 0; - virtual ui64 Get() const noexcept = 0; + } + + virtual ui64 Inc() noexcept { + return Add(1); + } + + virtual ui64 Add(ui64 n) noexcept = 0; + virtual ui64 Get() const noexcept = 0; virtual void Reset() noexcept = 0; - }; - + }; + class ILazyRate: public IMetric { public: EMetricType Type() const noexcept final { @@ -121,51 +121,51 @@ namespace NMonitoring { }; class IHistogram: public IMetric { - public: - explicit IHistogram(bool isRate) - : IsRate_{isRate} - { - } - + public: + explicit IHistogram(bool isRate) + : IsRate_{isRate} + { + } + EMetricType Type() const noexcept final { return IsRate_ ? EMetricType::HIST_RATE : EMetricType::HIST; - } - + } + virtual void Record(double value) = 0; virtual void Record(double value, ui32 count) = 0; - virtual IHistogramSnapshotPtr TakeSnapshot() const = 0; - virtual void Reset() = 0; - - protected: - const bool IsRate_; - }; - + virtual IHistogramSnapshotPtr TakeSnapshot() const = 0; + virtual void Reset() = 0; + + protected: + const bool IsRate_; + }; + /////////////////////////////////////////////////////////////////////////////// // TGauge /////////////////////////////////////////////////////////////////////////////// - class TGauge final: public IGauge { + class TGauge final: public IGauge { public: explicit TGauge(double value = 0.0) { Set(value); } - double Add(double n) noexcept override { - double newValue; - double oldValue = Get(); + double Add(double n) noexcept override { + double newValue; + double oldValue = Get(); do { - newValue = oldValue + n; - } while (!Value_.compare_exchange_weak(oldValue, newValue, std::memory_order_release, std::memory_order_consume)); + newValue = oldValue + n; + } while (!Value_.compare_exchange_weak(oldValue, newValue, std::memory_order_release, std::memory_order_consume)); - return newValue; + return newValue; } - void Set(double n) noexcept override { - Value_.store(n, std::memory_order_relaxed); + void Set(double n) noexcept override { + Value_.store(n, std::memory_order_relaxed); } - double Get() const noexcept override { - return Value_.load(std::memory_order_relaxed); + double Get() const noexcept override { + return Value_.load(std::memory_order_relaxed); } void Accept(TInstant time, IMetricConsumer* consumer) const override { @@ -173,7 +173,7 @@ namespace NMonitoring { } private: - std::atomic<double> Value_; + std::atomic<double> Value_; }; /////////////////////////////////////////////////////////////////////////////// @@ -201,22 +201,22 @@ namespace NMonitoring { /////////////////////////////////////////////////////////////////////////////// // TIntGauge /////////////////////////////////////////////////////////////////////////////// - class TIntGauge final: public IIntGauge { + class TIntGauge final: public IIntGauge { public: explicit TIntGauge(i64 value = 0) { Set(value); } - i64 Add(i64 n) noexcept override { - return Value_.fetch_add(n, std::memory_order_relaxed) + n; + i64 Add(i64 n) noexcept override { + return Value_.fetch_add(n, std::memory_order_relaxed) + n; } - void Set(i64 value) noexcept override { - Value_.store(value, std::memory_order_relaxed); + void Set(i64 value) noexcept override { + Value_.store(value, std::memory_order_relaxed); } - i64 Get() const noexcept override { - return Value_.load(std::memory_order_relaxed); + i64 Get() const noexcept override { + return Value_.load(std::memory_order_relaxed); } void Accept(TInstant time, IMetricConsumer* consumer) const override { @@ -224,7 +224,7 @@ namespace NMonitoring { } private: - std::atomic_int64_t Value_; + std::atomic_int64_t Value_; }; /////////////////////////////////////////////////////////////////////////////// @@ -255,19 +255,19 @@ namespace NMonitoring { class TCounter final: public ICounter { public: explicit TCounter(ui64 value = 0) { - Value_.store(value, std::memory_order_relaxed); + Value_.store(value, std::memory_order_relaxed); } - ui64 Add(ui64 n) noexcept override { - return Value_.fetch_add(n, std::memory_order_relaxed) + n; + ui64 Add(ui64 n) noexcept override { + return Value_.fetch_add(n, std::memory_order_relaxed) + n; } - ui64 Get() const noexcept override { - return Value_.load(std::memory_order_relaxed); + ui64 Get() const noexcept override { + return Value_.load(std::memory_order_relaxed); } - void Reset() noexcept override { - Value_.store(0, std::memory_order_relaxed); + void Reset() noexcept override { + Value_.store(0, std::memory_order_relaxed); } void Accept(TInstant time, IMetricConsumer* consumer) const override { @@ -275,7 +275,7 @@ namespace NMonitoring { } private: - std::atomic_uint64_t Value_; + std::atomic_uint64_t Value_; }; /////////////////////////////////////////////////////////////////////////////// @@ -303,18 +303,18 @@ namespace NMonitoring { /////////////////////////////////////////////////////////////////////////////// // TRate /////////////////////////////////////////////////////////////////////////////// - class TRate final: public IRate { + class TRate final: public IRate { public: explicit TRate(ui64 value = 0) { - Value_.store(value, std::memory_order_relaxed); + Value_.store(value, std::memory_order_relaxed); } - ui64 Add(ui64 n) noexcept override { - return Value_.fetch_add(n, std::memory_order_relaxed) + n; + ui64 Add(ui64 n) noexcept override { + return Value_.fetch_add(n, std::memory_order_relaxed) + n; } - ui64 Get() const noexcept override { - return Value_.load(std::memory_order_relaxed); + ui64 Get() const noexcept override { + return Value_.load(std::memory_order_relaxed); } void Reset() noexcept override { @@ -326,7 +326,7 @@ namespace NMonitoring { } private: - std::atomic_uint64_t Value_; + std::atomic_uint64_t Value_; }; /////////////////////////////////////////////////////////////////////////////// @@ -354,11 +354,11 @@ namespace NMonitoring { /////////////////////////////////////////////////////////////////////////////// // THistogram /////////////////////////////////////////////////////////////////////////////// - class THistogram final: public IHistogram { + class THistogram final: public IHistogram { public: THistogram(IHistogramCollectorPtr collector, bool isRate) - : IHistogram(isRate) - , Collector_(std::move(collector)) + : IHistogram(isRate) + , Collector_(std::move(collector)) { } @@ -374,14 +374,14 @@ namespace NMonitoring { consumer->OnHistogram(time, TakeSnapshot()); } - IHistogramSnapshotPtr TakeSnapshot() const override { + IHistogramSnapshotPtr TakeSnapshot() const override { return Collector_->Snapshot(); } - void Reset() override { + void Reset() override { Collector_->Reset(); } - + private: IHistogramCollectorPtr Collector_; }; diff --git a/library/cpp/monlib/metrics/metric_registry.cpp b/library/cpp/monlib/metrics/metric_registry.cpp index 6701bc61a0..b083163a7b 100644 --- a/library/cpp/monlib/metrics/metric_registry.cpp +++ b/library/cpp/monlib/metrics/metric_registry.cpp @@ -1,7 +1,7 @@ #include "metric_registry.h" -#include <memory> - +#include <memory> + namespace NMonitoring { namespace { void ConsumeLabels(IMetricConsumer* consumer, const ILabels& labels) { @@ -25,21 +25,21 @@ namespace NMonitoring { } } - void WriteLabels(IMetricConsumer* consumer, const ILabels& labels) { - consumer->OnLabelsBegin(); + void WriteLabels(IMetricConsumer* consumer, const ILabels& labels) { + consumer->OnLabelsBegin(); ConsumeLabels(consumer, labels); - consumer->OnLabelsEnd(); - } - + consumer->OnLabelsEnd(); + } + TMetricRegistry::TMetricRegistry() = default; TMetricRegistry::~TMetricRegistry() = default; - + TMetricRegistry::TMetricRegistry(const TLabels& commonLabels) : TMetricRegistry{} - { - CommonLabels_ = commonLabels; - } - + { + CommonLabels_ = commonLabels; + } + TMetricRegistry* TMetricRegistry::Instance() { return Singleton<TMetricRegistry>(); } @@ -78,12 +78,12 @@ namespace NMonitoring { TCounter* TMetricRegistry::Counter(TLabels labels) { return Metric<TCounter, EMetricType::COUNTER>(std::move(labels)); - } - + } + TCounter* TMetricRegistry::Counter(ILabelsPtr labels) { return Metric<TCounter, EMetricType::COUNTER>(std::move(labels)); - } - + } + TLazyCounter* TMetricRegistry::LazyCounter(TLabels labels, std::function<ui64()> supplier) { return Metric<TLazyCounter, EMetricType::COUNTER>(std::move(labels), std::move(supplier)); } @@ -94,12 +94,12 @@ namespace NMonitoring { TRate* TMetricRegistry::Rate(TLabels labels) { return Metric<TRate, EMetricType::RATE>(std::move(labels)); - } - + } + TRate* TMetricRegistry::Rate(ILabelsPtr labels) { return Metric<TRate, EMetricType::RATE>(std::move(labels)); - } - + } + TLazyRate* TMetricRegistry::LazyRate(TLabels labels, std::function<ui64()> supplier) { return Metric<TLazyRate, EMetricType::RATE>(std::move(labels), std::move(supplier)); } @@ -114,16 +114,16 @@ namespace NMonitoring { THistogram* TMetricRegistry::HistogramCounter(ILabelsPtr labels, IHistogramCollectorPtr collector) { return Metric<THistogram, EMetricType::HIST>(std::move(labels), std::move(collector), false); - } - + } + THistogram* TMetricRegistry::HistogramRate(TLabels labels, IHistogramCollectorPtr collector) { return Metric<THistogram, EMetricType::HIST_RATE>(std::move(labels), std::move(collector), true); } THistogram* TMetricRegistry::HistogramRate(ILabelsPtr labels, IHistogramCollectorPtr collector) { return Metric<THistogram, EMetricType::HIST_RATE>(std::move(labels), std::move(collector), true); - } - + } + void TMetricRegistry::Reset() { TWriteGuard g{Lock_}; for (auto& [label, metric] : Metrics_) { @@ -177,12 +177,12 @@ namespace NMonitoring { TWriteGuard g{Lock_}; // decltype(Metrics_)::iterator breaks build on windows THashMap<ILabelsPtr, IMetricPtr>::iterator it; - if constexpr (!std::is_convertible_v<TLabelsType, ILabelsPtr>) { + if constexpr (!std::is_convertible_v<TLabelsType, ILabelsPtr>) { it = Metrics_.emplace(new TLabels{std::forward<TLabelsType>(labels)}, std::move(metric)).first; - } else { + } else { it = Metrics_.emplace(std::forward<TLabelsType>(labels), std::move(metric)).first; - } - + } + return static_cast<TMetric*>(it->second.Get()); } } @@ -193,14 +193,14 @@ namespace NMonitoring { } void TMetricRegistry::Accept(TInstant time, IMetricConsumer* consumer) const { - consumer->OnStreamBegin(); - - if (!CommonLabels_.Empty()) { + consumer->OnStreamBegin(); + + if (!CommonLabels_.Empty()) { consumer->OnLabelsBegin(); ConsumeLabels(consumer, CommonLabels_); consumer->OnLabelsEnd(); - } - + } + { TReadGuard g{Lock_}; for (const auto& it: Metrics_) { @@ -212,12 +212,12 @@ namespace NMonitoring { } } - consumer->OnStreamEnd(); - } + consumer->OnStreamEnd(); + } void TMetricRegistry::Append(TInstant time, IMetricConsumer* consumer) const { TReadGuard g{Lock_}; - + for (const auto& it: Metrics_) { ILabels* labels = it.first.Get(); IMetric* metric = it.second.Get(); diff --git a/library/cpp/monlib/metrics/metric_registry.h b/library/cpp/monlib/metrics/metric_registry.h index 8c92c1500d..670cf8651e 100644 --- a/library/cpp/monlib/metrics/metric_registry.h +++ b/library/cpp/monlib/metrics/metric_registry.h @@ -6,47 +6,47 @@ #include <util/system/rwlock.h> #include <library/cpp/threading/light_rw_lock/lightrwlock.h> - - + + namespace NMonitoring { class IMetricFactory { - public: + public: virtual ~IMetricFactory() = default; - - virtual IGauge* Gauge(ILabelsPtr labels) = 0; + + virtual IGauge* Gauge(ILabelsPtr labels) = 0; virtual ILazyGauge* LazyGauge(ILabelsPtr labels, std::function<double()> supplier) = 0; - virtual IIntGauge* IntGauge(ILabelsPtr labels) = 0; + virtual IIntGauge* IntGauge(ILabelsPtr labels) = 0; virtual ILazyIntGauge* LazyIntGauge(ILabelsPtr labels, std::function<i64()> supplier) = 0; - virtual ICounter* Counter(ILabelsPtr labels) = 0; + virtual ICounter* Counter(ILabelsPtr labels) = 0; virtual ILazyCounter* LazyCounter(ILabelsPtr labels, std::function<ui64()> supplier) = 0; - virtual IRate* Rate(ILabelsPtr labels) = 0; + virtual IRate* Rate(ILabelsPtr labels) = 0; virtual ILazyRate* LazyRate(ILabelsPtr labels, std::function<ui64()> supplier) = 0; - - virtual IHistogram* HistogramCounter( - ILabelsPtr labels, - IHistogramCollectorPtr collector) = 0; - - virtual IHistogram* HistogramRate( - ILabelsPtr labels, - IHistogramCollectorPtr collector) = 0; - }; - - class IMetricSupplier { - public: - virtual ~IMetricSupplier() = default; - + + virtual IHistogram* HistogramCounter( + ILabelsPtr labels, + IHistogramCollectorPtr collector) = 0; + + virtual IHistogram* HistogramRate( + ILabelsPtr labels, + IHistogramCollectorPtr collector) = 0; + }; + + class IMetricSupplier { + public: + virtual ~IMetricSupplier() = default; + virtual void Accept(TInstant time, IMetricConsumer* consumer) const = 0; virtual void Append(TInstant time, IMetricConsumer* consumer) const = 0; - }; - - class IMetricRegistry: public IMetricSupplier, public IMetricFactory { - public: - virtual const TLabels& CommonLabels() const noexcept = 0; + }; + + class IMetricRegistry: public IMetricSupplier, public IMetricFactory { + public: + virtual const TLabels& CommonLabels() const noexcept = 0; virtual void RemoveMetric(const ILabels& labels) noexcept = 0; - }; - - + }; + + /////////////////////////////////////////////////////////////////////////////// // TMetricRegistry /////////////////////////////////////////////////////////////////////////////// @@ -62,21 +62,21 @@ namespace NMonitoring { */ static TMetricRegistry* Instance(); - TGauge* Gauge(TLabels labels); + TGauge* Gauge(TLabels labels); TLazyGauge* LazyGauge(TLabels labels, std::function<double()> supplier); - TIntGauge* IntGauge(TLabels labels); + TIntGauge* IntGauge(TLabels labels); TLazyIntGauge* LazyIntGauge(TLabels labels, std::function<i64()> supplier); TCounter* Counter(TLabels labels); TLazyCounter* LazyCounter(TLabels labels, std::function<ui64()> supplier); - TRate* Rate(TLabels labels); + TRate* Rate(TLabels labels); TLazyRate* LazyRate(TLabels labels, std::function<ui64()> supplier); THistogram* HistogramCounter( - TLabels labels, + TLabels labels, IHistogramCollectorPtr collector); THistogram* HistogramRate( - TLabels labels, + TLabels labels, IHistogramCollectorPtr collector); /** @@ -91,39 +91,39 @@ namespace NMonitoring { void Accept(TInstant time, IMetricConsumer* consumer) const override; void Append(TInstant time, IMetricConsumer* consumer) const override; - const TLabels& CommonLabels() const noexcept override { + const TLabels& CommonLabels() const noexcept override { return CommonLabels_; } void RemoveMetric(const ILabels& labels) noexcept override; private: - TGauge* Gauge(ILabelsPtr labels) override; + TGauge* Gauge(ILabelsPtr labels) override; TLazyGauge* LazyGauge(ILabelsPtr labels, std::function<double()> supplier) override; - TIntGauge* IntGauge(ILabelsPtr labels) override; + TIntGauge* IntGauge(ILabelsPtr labels) override; TLazyIntGauge* LazyIntGauge(ILabelsPtr labels, std::function<i64()> supplier) override; TCounter* Counter(ILabelsPtr labels) override; TLazyCounter* LazyCounter(ILabelsPtr labels, std::function<ui64()> supplier) override; - TRate* Rate(ILabelsPtr labels) override; + TRate* Rate(ILabelsPtr labels) override; TLazyRate* LazyRate(ILabelsPtr labels, std::function<ui64()> supplier) override; - - THistogram* HistogramCounter( - ILabelsPtr labels, - IHistogramCollectorPtr collector) override; - - THistogram* HistogramRate( - ILabelsPtr labels, - IHistogramCollectorPtr collector) override; - - private: + + THistogram* HistogramCounter( + ILabelsPtr labels, + IHistogramCollectorPtr collector) override; + + THistogram* HistogramRate( + ILabelsPtr labels, + IHistogramCollectorPtr collector) override; + + private: TRWMutex Lock_; THashMap<ILabelsPtr, IMetricPtr> Metrics_; - + template <typename TMetric, EMetricType type, typename TLabelsType, typename... Args> TMetric* Metric(TLabelsType&& labels, Args&&... args); TLabels CommonLabels_; }; - void WriteLabels(IMetricConsumer* consumer, const ILabels& labels); + void WriteLabels(IMetricConsumer* consumer, const ILabels& labels); } diff --git a/library/cpp/monlib/metrics/metric_registry_ut.cpp b/library/cpp/monlib/metrics/metric_registry_ut.cpp index 6eb3e14100..86d9a52ec0 100644 --- a/library/cpp/monlib/metrics/metric_registry_ut.cpp +++ b/library/cpp/monlib/metrics/metric_registry_ut.cpp @@ -10,33 +10,33 @@ using namespace NMonitoring; -template<> -void Out<NMonitoring::NProto::TSingleSample::ValueCase>(IOutputStream& os, NMonitoring::NProto::TSingleSample::ValueCase val) { - switch (val) { - case NMonitoring::NProto::TSingleSample::ValueCase::kInt64: - os << "Int64"; - break; - case NMonitoring::NProto::TSingleSample::ValueCase::kUint64: - os << "Uint64"; - break; - case NMonitoring::NProto::TSingleSample::ValueCase::kHistogram: - os << "Histogram"; - break; - case NMonitoring::NProto::TSingleSample::ValueCase::kFloat64: - os << "Float64"; - break; +template<> +void Out<NMonitoring::NProto::TSingleSample::ValueCase>(IOutputStream& os, NMonitoring::NProto::TSingleSample::ValueCase val) { + switch (val) { + case NMonitoring::NProto::TSingleSample::ValueCase::kInt64: + os << "Int64"; + break; + case NMonitoring::NProto::TSingleSample::ValueCase::kUint64: + os << "Uint64"; + break; + case NMonitoring::NProto::TSingleSample::ValueCase::kHistogram: + os << "Histogram"; + break; + case NMonitoring::NProto::TSingleSample::ValueCase::kFloat64: + os << "Float64"; + break; case NMonitoring::NProto::TSingleSample::ValueCase::kSummaryDouble: os << "DSummary"; break; case NMonitoring::NProto::TSingleSample::ValueCase::kLogHistogram: os << "LogHistogram"; break; - case NMonitoring::NProto::TSingleSample::ValueCase::VALUE_NOT_SET: - os << "NOT SET"; - break; - } -} - + case NMonitoring::NProto::TSingleSample::ValueCase::VALUE_NOT_SET: + os << "NOT SET"; + break; + } +} + Y_UNIT_TEST_SUITE(TMetricRegistryTest) { Y_UNIT_TEST(Gauge) { TMetricRegistry registry(TLabels{{"common", "label"}}); @@ -79,29 +79,29 @@ Y_UNIT_TEST_SUITE(TMetricRegistryTest) { TMetricRegistry registry(TLabels{{"common", "label"}}); TIntGauge* g = registry.IntGauge({{"my", "gauge"}}); - UNIT_ASSERT_VALUES_EQUAL(g->Get(), 0); + UNIT_ASSERT_VALUES_EQUAL(g->Get(), 0); i64 val; val = g->Inc(); - UNIT_ASSERT_VALUES_EQUAL(g->Get(), 1); - UNIT_ASSERT_VALUES_EQUAL(g->Get(), val); + UNIT_ASSERT_VALUES_EQUAL(g->Get(), 1); + UNIT_ASSERT_VALUES_EQUAL(g->Get(), val); val = g->Dec(); - UNIT_ASSERT_VALUES_EQUAL(g->Get(), 0); - UNIT_ASSERT_VALUES_EQUAL(g->Get(), val); + UNIT_ASSERT_VALUES_EQUAL(g->Get(), 0); + UNIT_ASSERT_VALUES_EQUAL(g->Get(), val); val = g->Add(1); - UNIT_ASSERT_VALUES_EQUAL(g->Get(), 1); - UNIT_ASSERT_VALUES_EQUAL(g->Get(), val); + UNIT_ASSERT_VALUES_EQUAL(g->Get(), 1); + UNIT_ASSERT_VALUES_EQUAL(g->Get(), val); val = g->Add(2); - UNIT_ASSERT_VALUES_EQUAL(g->Get(), 3); - UNIT_ASSERT_VALUES_EQUAL(g->Get(), val); + UNIT_ASSERT_VALUES_EQUAL(g->Get(), 3); + UNIT_ASSERT_VALUES_EQUAL(g->Get(), val); val = g->Add(-5); - UNIT_ASSERT_VALUES_EQUAL(g->Get(), -2); - UNIT_ASSERT_VALUES_EQUAL(g->Get(), val); + UNIT_ASSERT_VALUES_EQUAL(g->Get(), -2); + UNIT_ASSERT_VALUES_EQUAL(g->Get(), val); } Y_UNIT_TEST(LazyIntGauge) { @@ -126,11 +126,11 @@ Y_UNIT_TEST_SUITE(TMetricRegistryTest) { TMetricRegistry registry(TLabels{{"common", "label"}}); TCounter* c = registry.Counter({{"my", "counter"}}); - UNIT_ASSERT_VALUES_EQUAL(c->Get(), 0); - UNIT_ASSERT_VALUES_EQUAL(c->Inc(), 1); - UNIT_ASSERT_VALUES_EQUAL(c->Get(), 1); - UNIT_ASSERT_VALUES_EQUAL(c->Add(10), 11); - UNIT_ASSERT_VALUES_EQUAL(c->Get(), 11); + UNIT_ASSERT_VALUES_EQUAL(c->Get(), 0); + UNIT_ASSERT_VALUES_EQUAL(c->Inc(), 1); + UNIT_ASSERT_VALUES_EQUAL(c->Get(), 1); + UNIT_ASSERT_VALUES_EQUAL(c->Add(10), 11); + UNIT_ASSERT_VALUES_EQUAL(c->Get(), 11); } Y_UNIT_TEST(LazyCounter) { @@ -159,11 +159,11 @@ Y_UNIT_TEST_SUITE(TMetricRegistryTest) { TMetricRegistry registry(TLabels{{"common", "label"}}); TCounter* c = registry.Counter({{"my", "counter"}}); - UNIT_ASSERT_VALUES_EQUAL(c->Get(), 0); + UNIT_ASSERT_VALUES_EQUAL(c->Get(), 0); c->Add(10); c = registry.Counter({{"my", "counter"}}); - UNIT_ASSERT_VALUES_EQUAL(c->Get(), 10); + UNIT_ASSERT_VALUES_EQUAL(c->Get(), 10); } Y_UNIT_TEST(Sample) { @@ -180,31 +180,31 @@ Y_UNIT_TEST_SUITE(TMetricRegistryTest) { auto now = TInstant::Now(); registry.Accept(now, encoder.Get()); - UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 2); - UNIT_ASSERT_VALUES_EQUAL(samples.CommonLabelsSize(), 1); - { - const NProto::TLabel& label = samples.GetCommonLabels(0); - UNIT_ASSERT_STRINGS_EQUAL(label.GetName(), "common"); - UNIT_ASSERT_STRINGS_EQUAL(label.GetValue(), "label"); - } + UNIT_ASSERT_VALUES_EQUAL(samples.SamplesSize(), 2); + UNIT_ASSERT_VALUES_EQUAL(samples.CommonLabelsSize(), 1); + { + const NProto::TLabel& label = samples.GetCommonLabels(0); + UNIT_ASSERT_STRINGS_EQUAL(label.GetName(), "common"); + UNIT_ASSERT_STRINGS_EQUAL(label.GetValue(), "label"); + } + - for (const NProto::TSingleSample& sample : samples.GetSamples()) { - UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1); - UNIT_ASSERT_VALUES_EQUAL(sample.GetTime(), now.MilliSeconds()); + UNIT_ASSERT_VALUES_EQUAL(sample.LabelsSize(), 1); + UNIT_ASSERT_VALUES_EQUAL(sample.GetTime(), now.MilliSeconds()); if (sample.GetMetricType() == NProto::GAUGE) { - UNIT_ASSERT_VALUES_EQUAL(sample.GetValueCase(), NProto::TSingleSample::kFloat64); + UNIT_ASSERT_VALUES_EQUAL(sample.GetValueCase(), NProto::TSingleSample::kFloat64); UNIT_ASSERT_DOUBLES_EQUAL(sample.GetFloat64(), 12.34, 1E-6); - const NProto::TLabel& label = sample.GetLabels(0); + const NProto::TLabel& label = sample.GetLabels(0); UNIT_ASSERT_STRINGS_EQUAL(label.GetName(), "my"); UNIT_ASSERT_STRINGS_EQUAL(label.GetValue(), "gauge"); } else if (sample.GetMetricType() == NProto::COUNTER) { - UNIT_ASSERT_VALUES_EQUAL(sample.GetValueCase(), NProto::TSingleSample::kUint64); - UNIT_ASSERT_VALUES_EQUAL(sample.GetUint64(), 10); + UNIT_ASSERT_VALUES_EQUAL(sample.GetValueCase(), NProto::TSingleSample::kUint64); + UNIT_ASSERT_VALUES_EQUAL(sample.GetUint64(), 10); - const NProto::TLabel& label = sample.GetLabels(0); + const NProto::TLabel& label = sample.GetLabels(0); UNIT_ASSERT_STRINGS_EQUAL(label.GetName(), "my"); UNIT_ASSERT_STRINGS_EQUAL(label.GetValue(), "counter"); } else { @@ -212,7 +212,7 @@ Y_UNIT_TEST_SUITE(TMetricRegistryTest) { } } } - + Y_UNIT_TEST(Histograms) { TMetricRegistry registry(TLabels{{"common", "label"}}); @@ -239,32 +239,32 @@ Y_UNIT_TEST_SUITE(TMetricRegistryTest) { UNIT_ASSERT_NO_DIFF(ss.Str(), NResource::Find("/histograms.json")); } - Y_UNIT_TEST(StreamingEncoderTest) { - const TString expected { - "{\"commonLabels\":{\"common\":\"label\"}," - "\"sensors\":[{\"kind\":\"GAUGE\",\"labels\":{\"my\":\"gauge\"},\"value\":12.34}]}" - }; - + Y_UNIT_TEST(StreamingEncoderTest) { + const TString expected { + "{\"commonLabels\":{\"common\":\"label\"}," + "\"sensors\":[{\"kind\":\"GAUGE\",\"labels\":{\"my\":\"gauge\"},\"value\":12.34}]}" + }; + TMetricRegistry registry(TLabels{{"common", "label"}}); - - TGauge* g = registry.Gauge({{"my", "gauge"}}); - g->Set(12.34); - - TStringStream os; - auto encoder = EncoderJson(&os); - registry.Accept(TInstant::Zero(), encoder.Get()); - - UNIT_ASSERT_STRINGS_EQUAL(os.Str(), expected); - } - + + TGauge* g = registry.Gauge({{"my", "gauge"}}); + g->Set(12.34); + + TStringStream os; + auto encoder = EncoderJson(&os); + registry.Accept(TInstant::Zero(), encoder.Get()); + + UNIT_ASSERT_STRINGS_EQUAL(os.Str(), expected); + } + Y_UNIT_TEST(CreatingSameMetricWithDifferentTypesShouldThrow) { TMetricRegistry registry; - registry.Gauge({{"foo", "bar"}}); - UNIT_ASSERT_EXCEPTION(registry.Counter({{"foo", "bar"}}), yexception); + registry.Gauge({{"foo", "bar"}}); + UNIT_ASSERT_EXCEPTION(registry.Counter({{"foo", "bar"}}), yexception); - registry.HistogramCounter({{"bar", "baz"}}, nullptr); - UNIT_ASSERT_EXCEPTION(registry.HistogramRate({{"bar", "baz"}}, nullptr), yexception); + registry.HistogramCounter({{"bar", "baz"}}, nullptr); + UNIT_ASSERT_EXCEPTION(registry.HistogramRate({{"bar", "baz"}}, nullptr), yexception); } Y_UNIT_TEST(EncodeRegistryWithCommonLabels) { diff --git a/library/cpp/monlib/metrics/metric_type.cpp b/library/cpp/monlib/metrics/metric_type.cpp index 647d7b2020..a8a546e843 100644 --- a/library/cpp/monlib/metrics/metric_type.cpp +++ b/library/cpp/monlib/metrics/metric_type.cpp @@ -21,7 +21,7 @@ namespace NMonitoring { return TStringBuf("HIST_RATE"); case EMetricType::DSUMMARY: return TStringBuf("DSUMMARY"); - case EMetricType::LOGHIST: + case EMetricType::LOGHIST: return TStringBuf("LOGHIST"); default: return TStringBuf("UNKNOWN"); @@ -44,7 +44,7 @@ namespace NMonitoring { } else if (str == TStringBuf("DSUMMARY")) { return EMetricType::DSUMMARY; } else if (str == TStringBuf("LOGHIST")) { - return EMetricType::LOGHIST; + return EMetricType::LOGHIST; } else { ythrow yexception() << "unknown metric type: " << str; } diff --git a/library/cpp/monlib/metrics/metric_value.h b/library/cpp/monlib/metrics/metric_value.h index 18f8dc6fdb..607fcc8602 100644 --- a/library/cpp/monlib/metrics/metric_value.h +++ b/library/cpp/monlib/metrics/metric_value.h @@ -8,23 +8,23 @@ #include <util/datetime/base.h> #include <util/generic/algorithm.h> #include <util/generic/vector.h> -#include <util/generic/cast.h> -#include <util/generic/ymath.h> +#include <util/generic/cast.h> +#include <util/generic/ymath.h> namespace NMonitoring { - namespace NPrivate { - template <typename T> - T FromFloatSafe(double d) { - static_assert(std::is_integral<T>::value, "this function only converts floats to integers"); + namespace NPrivate { + template <typename T> + T FromFloatSafe(double d) { + static_assert(std::is_integral<T>::value, "this function only converts floats to integers"); Y_ENSURE(::IsValidFloat(d) && d >= Min<T>() && d <= MaxFloor<T>(), "Cannot convert " << d << " to an integer value"); - return static_cast<T>(d); - } + return static_cast<T>(d); + } inline auto POINT_KEY_FN = [](auto& p) { return p.GetTime(); }; - } // namespace NPrivate - + } // namespace NPrivate + template <typename T, typename Enable = void> struct TValueType; @@ -132,9 +132,9 @@ namespace NMonitoring { ui64 AsUint64(EMetricValueType type) const { switch (type) { case EMetricValueType::DOUBLE: - return NPrivate::FromFloatSafe<ui64>(Value_.Double); + return NPrivate::FromFloatSafe<ui64>(Value_.Double); case EMetricValueType::INT64: - return SafeIntegerCast<ui64>(Value_.Int64); + return SafeIntegerCast<ui64>(Value_.Int64); case EMetricValueType::UINT64: return Value_.Uint64; case EMetricValueType::HISTOGRAM: @@ -158,11 +158,11 @@ namespace NMonitoring { i64 AsInt64(EMetricValueType type) const { switch (type) { case EMetricValueType::DOUBLE: - return NPrivate::FromFloatSafe<i64>(Value_.Double); + return NPrivate::FromFloatSafe<i64>(Value_.Double); case EMetricValueType::INT64: return Value_.Int64; case EMetricValueType::UINT64: - return SafeIntegerCast<i64>(Value_.Uint64); + return SafeIntegerCast<i64>(Value_.Uint64); case EMetricValueType::HISTOGRAM: ythrow yexception() << "histogram cannot be casted to Int64"; case EMetricValueType::SUMMARY: @@ -434,8 +434,8 @@ namespace NMonitoring { point.GetValue().AsLogHistogram()->Ref(); } } - } - + } + template <typename TConsumer> void ForEach(TConsumer c) const { for (const auto& point : Points_) { @@ -470,8 +470,8 @@ namespace NMonitoring { private: static void CheckTypes(EMetricValueType t1, EMetricValueType t2) { Y_ENSURE(t1 == t2, - "Series type mismatch: expected " << t1 << - ", but got " << t2); + "Series type mismatch: expected " << t1 << + ", but got " << t2); } private: diff --git a/library/cpp/monlib/metrics/metric_value_ut.cpp b/library/cpp/monlib/metrics/metric_value_ut.cpp index d37492d5da..49b47c4057 100644 --- a/library/cpp/monlib/metrics/metric_value_ut.cpp +++ b/library/cpp/monlib/metrics/metric_value_ut.cpp @@ -169,7 +169,7 @@ Y_UNIT_TEST_SUITE(TMetricValueTest) { UNIT_ASSERT_VALUES_EQUAL(1, h2->RefCount()); UNIT_ASSERT_VALUES_EQUAL(1, h3->RefCount()); } - + Y_UNIT_TEST(LogHistogramsUnique) { auto ts1 = TInstant::Now(); auto ts2 = ts1 + TDuration::Seconds(1); @@ -220,7 +220,7 @@ Y_UNIT_TEST_SUITE(TMetricValueTest) { auto ts1 = TInstant::Now(); auto ts2 = ts1 + TDuration::Seconds(1); auto ts3 = ts2 + TDuration::Seconds(1); - + auto h1 = MakeSummarySnapshot(); auto h2 = MakeSummarySnapshot(); auto h3 = MakeSummarySnapshot(); @@ -262,13 +262,13 @@ Y_UNIT_TEST_SUITE(TMetricValueTest) { UNIT_ASSERT_VALUES_EQUAL(1, h3->RefCount()); } - Y_UNIT_TEST(HistogramsUnique2) { - auto ts1 = TInstant::Now(); - auto ts2 = ts1 + TDuration::Seconds(1); - auto ts3 = ts2 + TDuration::Seconds(1); - auto ts4 = ts3 + TDuration::Seconds(1); - auto ts5 = ts4 + TDuration::Seconds(1); - + Y_UNIT_TEST(HistogramsUnique2) { + auto ts1 = TInstant::Now(); + auto ts2 = ts1 + TDuration::Seconds(1); + auto ts3 = ts2 + TDuration::Seconds(1); + auto ts4 = ts3 + TDuration::Seconds(1); + auto ts5 = ts4 + TDuration::Seconds(1); + auto h1 = MakeIntrusive<TTestHistogram>(1u); auto h2 = MakeIntrusive<TTestHistogram>(2u); auto h3 = MakeIntrusive<TTestHistogram>(3u); @@ -276,30 +276,30 @@ Y_UNIT_TEST_SUITE(TMetricValueTest) { auto h5 = MakeIntrusive<TTestHistogram>(5u); auto h6 = MakeIntrusive<TTestHistogram>(6u); auto h7 = MakeIntrusive<TTestHistogram>(7u); - - { + + { TMetricTimeSeries timeSeries; - timeSeries.Add(ts1, h1.Get()); - timeSeries.Add(ts1, h2.Get()); + timeSeries.Add(ts1, h1.Get()); + timeSeries.Add(ts1, h2.Get()); timeSeries.Add(ts2, h3.Get()); timeSeries.Add(ts3, h4.Get()); - timeSeries.Add(ts3, h5.Get()); - - timeSeries.Add(ts4, h6.Get()); - timeSeries.Add(ts5, h7.Get()); - - timeSeries.SortByTs(); - - UNIT_ASSERT_EQUAL(timeSeries.Size(), 5); + timeSeries.Add(ts3, h5.Get()); + + timeSeries.Add(ts4, h6.Get()); + timeSeries.Add(ts5, h7.Get()); + + timeSeries.SortByTs(); + + UNIT_ASSERT_EQUAL(timeSeries.Size(), 5); UNIT_ASSERT_EQUAL(timeSeries[0].GetValue().AsHistogram()->Count(), 2); UNIT_ASSERT_EQUAL(timeSeries[1].GetValue().AsHistogram()->Count(), 3); UNIT_ASSERT_EQUAL(timeSeries[2].GetValue().AsHistogram()->Count(), 5); UNIT_ASSERT_EQUAL(timeSeries[3].GetValue().AsHistogram()->Count(), 6); UNIT_ASSERT_EQUAL(timeSeries[4].GetValue().AsHistogram()->Count(), 7); - } - } + } + } Y_UNIT_TEST(LogHistogramsUnique2) { auto ts1 = TInstant::Now(); diff --git a/library/cpp/monlib/metrics/timer.h b/library/cpp/monlib/metrics/timer.h index 78f82dd448..5c4e26e37b 100644 --- a/library/cpp/monlib/metrics/timer.h +++ b/library/cpp/monlib/metrics/timer.h @@ -26,25 +26,25 @@ namespace NMonitoring { TMetricTimerScope(TMetricTimerScope&) = delete; TMetricTimerScope& operator=(const TMetricTimerScope&) = delete; - + TMetricTimerScope(TMetricTimerScope&& other) { - *this = std::move(other); - } - + *this = std::move(other); + } + TMetricTimerScope& operator=(TMetricTimerScope&& other) { Metric_ = other.Metric_; other.Metric_ = nullptr; - StartTime_ = std::move(other.StartTime_); - - return *this; - } - - void Record() { + StartTime_ = std::move(other.StartTime_); + + return *this; + } + + void Record() { Y_VERIFY_DEBUG(Metric_); if (Metric_ == nullptr) { - return; - } - + return; + } + auto duration = std::chrono::duration_cast<Resolution>(Clock::now() - StartTime_).count(); if constexpr (std::is_same<TMetric, TGauge>::value) { Metric_->Set(duration); @@ -59,69 +59,69 @@ namespace NMonitoring { } else { static_assert(TDependentFalse<TMetric>, "Not supported metric type"); } - + Metric_ = nullptr; } ~TMetricTimerScope() { if (Metric_ == nullptr) { - return; - } - - Record(); - } - + return; + } + + Record(); + } + private: TMetric* Metric_{nullptr}; typename Clock::time_point StartTime_; }; - /** - * @brief A class that is supposed to use to measure execution time of an asynchronuous operation. - * - * In order to be able to capture an object into a lambda which is then passed to TFuture::Subscribe/Apply, + /** + * @brief A class that is supposed to use to measure execution time of an asynchronuous operation. + * + * In order to be able to capture an object into a lambda which is then passed to TFuture::Subscribe/Apply, * the object must be copy constructible (limitation of the std::function class). So, we cannot use the TMetricTimerScope - * with the abovementioned functions without storing it in a shared pointer or somewhere else. This class works around this - * issue with wrapping the timer with a auto_ptr-like hack Also, Record is const so that one doesn't need to make every lambda mutable - * just to record time measurement. - */ + * with the abovementioned functions without storing it in a shared pointer or somewhere else. This class works around this + * issue with wrapping the timer with a auto_ptr-like hack Also, Record is const so that one doesn't need to make every lambda mutable + * just to record time measurement. + */ template <typename TMetric, - typename Resolution = std::chrono::milliseconds, - typename Clock = std::chrono::high_resolution_clock> - class TFutureFriendlyTimer { - public: + typename Resolution = std::chrono::milliseconds, + typename Clock = std::chrono::high_resolution_clock> + class TFutureFriendlyTimer { + public: explicit TFutureFriendlyTimer(TMetric* metric) : Impl_{metric} - { - } - - TFutureFriendlyTimer(const TFutureFriendlyTimer& other) - : Impl_{std::move(other.Impl_)} - { - } - - TFutureFriendlyTimer& operator=(const TFutureFriendlyTimer& other) { - Impl_ = std::move(other.Impl_); - } - - TFutureFriendlyTimer(TFutureFriendlyTimer&&) = default; - TFutureFriendlyTimer& operator=(TFutureFriendlyTimer&& other) = default; - - void Record() const { - Impl_.Record(); - } - - private: + { + } + + TFutureFriendlyTimer(const TFutureFriendlyTimer& other) + : Impl_{std::move(other.Impl_)} + { + } + + TFutureFriendlyTimer& operator=(const TFutureFriendlyTimer& other) { + Impl_ = std::move(other.Impl_); + } + + TFutureFriendlyTimer(TFutureFriendlyTimer&&) = default; + TFutureFriendlyTimer& operator=(TFutureFriendlyTimer&& other) = default; + + void Record() const { + Impl_.Record(); + } + + private: mutable TMetricTimerScope<TMetric, Resolution, Clock> Impl_; - }; - + }; + template <typename TMetric> TMetricTimerScope<TMetric> ScopeTimer(TMetric* metric) { return TMetricTimerScope<TMetric>{metric}; - } - + } + template <typename TMetric> TFutureFriendlyTimer<TMetric> FutureTimer(TMetric* metric) { return TFutureFriendlyTimer<TMetric>{metric}; - } + } } diff --git a/library/cpp/monlib/metrics/timer_ut.cpp b/library/cpp/monlib/metrics/timer_ut.cpp index 1786f12d17..c244a8c9e1 100644 --- a/library/cpp/monlib/metrics/timer_ut.cpp +++ b/library/cpp/monlib/metrics/timer_ut.cpp @@ -5,7 +5,7 @@ #include <library/cpp/threading/future/future.h> using namespace NMonitoring; -using namespace NThreading; +using namespace NThreading; Y_UNIT_TEST_SUITE(TTimerTest) { @@ -115,43 +115,43 @@ Y_UNIT_TEST_SUITE(TTimerTest) { } assertHistogram({1, 1, 0, 0}, histogram.TakeSnapshot()); } - - Y_UNIT_TEST(Moving) { - TTestClock::TimePoint = TTestClock::time_point::min(); - + + Y_UNIT_TEST(Moving) { + TTestClock::TimePoint = TTestClock::time_point::min(); + TCounter counter(0); - { + { TMetricTimerScope<TCounter, milliseconds, TTestClock> t{&counter}; - [tt = std::move(t)] { - TTestClock::TimePoint += milliseconds(5); - Y_UNUSED(tt); - }(); - - TTestClock::TimePoint += milliseconds(10); - } - - UNIT_ASSERT_EQUAL(counter.Get(), 5); - } - - Y_UNIT_TEST(MovingIntoApply) { - TTestClock::TimePoint = TTestClock::time_point::min(); - auto pool = CreateThreadPool(1); - + [tt = std::move(t)] { + TTestClock::TimePoint += milliseconds(5); + Y_UNUSED(tt); + }(); + + TTestClock::TimePoint += milliseconds(10); + } + + UNIT_ASSERT_EQUAL(counter.Get(), 5); + } + + Y_UNIT_TEST(MovingIntoApply) { + TTestClock::TimePoint = TTestClock::time_point::min(); + auto pool = CreateThreadPool(1); + TCounter counter(0); - { + { TFutureFriendlyTimer<TCounter, milliseconds, TTestClock> t{&counter}; - - auto f = Async([=] { - return; - }, *pool).Apply([tt = t] (auto) { - TTestClock::TimePoint += milliseconds(5); - tt.Record(); - }); - - f.Wait(); - TTestClock::TimePoint += milliseconds(10); - } - - UNIT_ASSERT_EQUAL(counter.Get(), 5); - } + + auto f = Async([=] { + return; + }, *pool).Apply([tt = t] (auto) { + TTestClock::TimePoint += milliseconds(5); + tt.Record(); + }); + + f.Wait(); + TTestClock::TimePoint += milliseconds(10); + } + + UNIT_ASSERT_EQUAL(counter.Get(), 5); + } } diff --git a/library/cpp/monlib/metrics/ya.make b/library/cpp/monlib/metrics/ya.make index 9f95309125..0e1fa143f9 100644 --- a/library/cpp/monlib/metrics/ya.make +++ b/library/cpp/monlib/metrics/ya.make @@ -14,7 +14,7 @@ SRCS( histogram_collector_exponential.cpp histogram_collector_linear.cpp histogram_snapshot.cpp - log_histogram_snapshot.cpp + log_histogram_snapshot.cpp labels.cpp metric_registry.cpp metric_consumer.cpp diff --git a/library/cpp/monlib/service/auth.cpp b/library/cpp/monlib/service/auth.cpp index 1b5e48e261..ddabcfbbf7 100644 --- a/library/cpp/monlib/service/auth.cpp +++ b/library/cpp/monlib/service/auth.cpp @@ -1,22 +1,22 @@ -#include "auth.h" - -#include <util/generic/hash_set.h> - - -namespace NMonitoring { -namespace { - class TFakeAuthProvider final: public IAuthProvider { - public: - TAuthResult Check(const IHttpRequest&) override { - return TAuthResult::Ok(); - } - }; - -} // namespace - -THolder<IAuthProvider> CreateFakeAuth() { +#include "auth.h" + +#include <util/generic/hash_set.h> + + +namespace NMonitoring { +namespace { + class TFakeAuthProvider final: public IAuthProvider { + public: + TAuthResult Check(const IHttpRequest&) override { + return TAuthResult::Ok(); + } + }; + +} // namespace + +THolder<IAuthProvider> CreateFakeAuth() { return MakeHolder<TFakeAuthProvider>(); -} - - -} // namespace NMonitoring +} + + +} // namespace NMonitoring diff --git a/library/cpp/monlib/service/auth.h b/library/cpp/monlib/service/auth.h index 884fe6cf5b..ae53b8bd8e 100644 --- a/library/cpp/monlib/service/auth.h +++ b/library/cpp/monlib/service/auth.h @@ -1,48 +1,48 @@ -#pragma once - -#include "mon_service_http_request.h" - -namespace NMonitoring { - enum class EAuthType { - None = 0, - Tvm = 1, - }; - - struct TAuthResult { - enum class EStatus { - NoCredentials = 0, - Denied, - Ok, - }; - - TAuthResult(EStatus status) - : Status{status} - { - } - - static TAuthResult Denied() { - return TAuthResult(EStatus::Denied); - } - - static TAuthResult NoCredentials() { - return TAuthResult(EStatus::NoCredentials); - } - - static TAuthResult Ok() { - return TAuthResult(EStatus::Ok); - } - - explicit operator bool() const { - return Status == EStatus::Ok; - } - - EStatus Status{EStatus::NoCredentials}; - }; - - struct IAuthProvider { - virtual ~IAuthProvider() = default; - virtual TAuthResult Check(const IHttpRequest& req) = 0; - }; - - THolder<IAuthProvider> CreateFakeAuth(); -} // namespace NMonitoring +#pragma once + +#include "mon_service_http_request.h" + +namespace NMonitoring { + enum class EAuthType { + None = 0, + Tvm = 1, + }; + + struct TAuthResult { + enum class EStatus { + NoCredentials = 0, + Denied, + Ok, + }; + + TAuthResult(EStatus status) + : Status{status} + { + } + + static TAuthResult Denied() { + return TAuthResult(EStatus::Denied); + } + + static TAuthResult NoCredentials() { + return TAuthResult(EStatus::NoCredentials); + } + + static TAuthResult Ok() { + return TAuthResult(EStatus::Ok); + } + + explicit operator bool() const { + return Status == EStatus::Ok; + } + + EStatus Status{EStatus::NoCredentials}; + }; + + struct IAuthProvider { + virtual ~IAuthProvider() = default; + virtual TAuthResult Check(const IHttpRequest& req) = 0; + }; + + THolder<IAuthProvider> CreateFakeAuth(); +} // namespace NMonitoring diff --git a/library/cpp/monlib/service/format.cpp b/library/cpp/monlib/service/format.cpp index 198ab7a42a..b0d6a10246 100644 --- a/library/cpp/monlib/service/format.cpp +++ b/library/cpp/monlib/service/format.cpp @@ -1 +1 @@ -#include "format.h" +#include "format.h" diff --git a/library/cpp/monlib/service/format.h b/library/cpp/monlib/service/format.h index d004f73bf1..0044b586b1 100644 --- a/library/cpp/monlib/service/format.h +++ b/library/cpp/monlib/service/format.h @@ -1,86 +1,86 @@ -#pragma once - +#pragma once + #include <library/cpp/monlib/encode/format.h> - -#include <util/string/ascii.h> -#include <util/generic/yexception.h> -#include <util/generic/typetraits.h> - -namespace NMonitoring { - namespace NPrivate { - Y_HAS_MEMBER(Name, Name); - Y_HAS_MEMBER(second, Second); - } // namespace NPrivate - - template <typename TRequest> - ECompression ParseCompression(const TRequest& req) { - auto&& headers = req.GetHeaders(); - - constexpr auto isPlainPair = NPrivate::THasSecond<std::decay_t<decltype(*headers.begin())>>::value; - - auto it = FindIf(std::begin(headers), std::end(headers), + +#include <util/string/ascii.h> +#include <util/generic/yexception.h> +#include <util/generic/typetraits.h> + +namespace NMonitoring { + namespace NPrivate { + Y_HAS_MEMBER(Name, Name); + Y_HAS_MEMBER(second, Second); + } // namespace NPrivate + + template <typename TRequest> + ECompression ParseCompression(const TRequest& req) { + auto&& headers = req.GetHeaders(); + + constexpr auto isPlainPair = NPrivate::THasSecond<std::decay_t<decltype(*headers.begin())>>::value; + + auto it = FindIf(std::begin(headers), std::end(headers), [=] (const auto& h) { - if constexpr (NPrivate::THasName<std::decay_t<decltype(h)>>::value) { + if constexpr (NPrivate::THasName<std::decay_t<decltype(h)>>::value) { return AsciiCompareIgnoreCase(h.Name(), TStringBuf("accept-encoding")) == 0; - } else if (isPlainPair) { + } else if (isPlainPair) { return AsciiCompareIgnoreCase(h.first, TStringBuf("accept-encoding")) == 0; - } - }); - - if (it == std::end(headers)) { - return NMonitoring::ECompression::IDENTITY; - } - - NMonitoring::ECompression val{}; - if constexpr (isPlainPair) { - val = CompressionFromAcceptEncodingHeader(it->second); - } else { - val = CompressionFromAcceptEncodingHeader(it->Value()); - } - - return val != NMonitoring::ECompression::UNKNOWN - ? val - : NMonitoring::ECompression::IDENTITY; - } - - template <typename TRequest> - NMonitoring::EFormat ParseFormat(const TRequest& req) { - auto&& formatStr = req.GetParams() + } + }); + + if (it == std::end(headers)) { + return NMonitoring::ECompression::IDENTITY; + } + + NMonitoring::ECompression val{}; + if constexpr (isPlainPair) { + val = CompressionFromAcceptEncodingHeader(it->second); + } else { + val = CompressionFromAcceptEncodingHeader(it->Value()); + } + + return val != NMonitoring::ECompression::UNKNOWN + ? val + : NMonitoring::ECompression::IDENTITY; + } + + template <typename TRequest> + NMonitoring::EFormat ParseFormat(const TRequest& req) { + auto&& formatStr = req.GetParams() .Get(TStringBuf("format")); - + if (!formatStr.empty()) { if (formatStr == TStringBuf("SPACK")) { - return EFormat::SPACK; + return EFormat::SPACK; } else if (formatStr == TStringBuf("TEXT")) { - return EFormat::TEXT; + return EFormat::TEXT; } else if (formatStr == TStringBuf("JSON")) { - return EFormat::JSON; - } else { - ythrow yexception() << "unknown format: " << formatStr << ". Only spack is supported here"; - } - } - - auto&& headers = req.GetHeaders(); - constexpr auto isPlainPair = NPrivate::THasSecond<std::decay_t<decltype(*headers.begin())>>::value; - - auto it = FindIf(std::begin(headers), std::end(headers), + return EFormat::JSON; + } else { + ythrow yexception() << "unknown format: " << formatStr << ". Only spack is supported here"; + } + } + + auto&& headers = req.GetHeaders(); + constexpr auto isPlainPair = NPrivate::THasSecond<std::decay_t<decltype(*headers.begin())>>::value; + + auto it = FindIf(std::begin(headers), std::end(headers), [=] (const auto& h) { - if constexpr (NPrivate::THasName<std::decay_t<decltype(h)>>::value) { + if constexpr (NPrivate::THasName<std::decay_t<decltype(h)>>::value) { return AsciiCompareIgnoreCase(h.Name(), TStringBuf("accept")) == 0; - } else if (isPlainPair) { + } else if (isPlainPair) { return AsciiCompareIgnoreCase(h.first, TStringBuf("accept")) == 0; - } - }); - - if (it != std::end(headers)) { - if constexpr (isPlainPair) { - return FormatFromAcceptHeader(it->second); - } else { - return FormatFromAcceptHeader(it->Value()); - } - } - - return EFormat::UNKNOWN; - } - -} // namespace NMonitoring + } + }); + + if (it != std::end(headers)) { + if constexpr (isPlainPair) { + return FormatFromAcceptHeader(it->second); + } else { + return FormatFromAcceptHeader(it->Value()); + } + } + + return EFormat::UNKNOWN; + } + +} // namespace NMonitoring diff --git a/library/cpp/monlib/service/mon_service_http_request.cpp b/library/cpp/monlib/service/mon_service_http_request.cpp index f2ab7b4553..5d805631d9 100644 --- a/library/cpp/monlib/service/mon_service_http_request.cpp +++ b/library/cpp/monlib/service/mon_service_http_request.cpp @@ -3,45 +3,45 @@ using namespace NMonitoring; -IMonHttpRequest::~IMonHttpRequest() { -} - -TMonService2HttpRequest::~TMonService2HttpRequest() { -} - +IMonHttpRequest::~IMonHttpRequest() { +} + +TMonService2HttpRequest::~TMonService2HttpRequest() { +} + TString TMonService2HttpRequest::GetServiceTitle() const { return MonService->GetTitle(); } -IOutputStream& TMonService2HttpRequest::Output() { - return *Out; -} - -HTTP_METHOD TMonService2HttpRequest::GetMethod() const { - return HttpRequest->GetMethod(); -} - -TStringBuf TMonService2HttpRequest::GetPathInfo() const { - return PathInfo; -} - -TStringBuf TMonService2HttpRequest::GetPath() const { - return HttpRequest->GetPath(); -} - -TStringBuf TMonService2HttpRequest::GetUri() const { - return HttpRequest->GetURI(); -} - -const TCgiParameters& TMonService2HttpRequest::GetParams() const { - return HttpRequest->GetParams(); -} - -const TCgiParameters& TMonService2HttpRequest::GetPostParams() const { - return HttpRequest->GetPostParams(); -} - -TStringBuf TMonService2HttpRequest::GetHeader(TStringBuf name) const { +IOutputStream& TMonService2HttpRequest::Output() { + return *Out; +} + +HTTP_METHOD TMonService2HttpRequest::GetMethod() const { + return HttpRequest->GetMethod(); +} + +TStringBuf TMonService2HttpRequest::GetPathInfo() const { + return PathInfo; +} + +TStringBuf TMonService2HttpRequest::GetPath() const { + return HttpRequest->GetPath(); +} + +TStringBuf TMonService2HttpRequest::GetUri() const { + return HttpRequest->GetURI(); +} + +const TCgiParameters& TMonService2HttpRequest::GetParams() const { + return HttpRequest->GetParams(); +} + +const TCgiParameters& TMonService2HttpRequest::GetPostParams() const { + return HttpRequest->GetPostParams(); +} + +TStringBuf TMonService2HttpRequest::GetHeader(TStringBuf name) const { const THttpHeaders& headers = HttpRequest->GetHeaders(); const THttpInputHeader* header = headers.FindHeader(name); if (header != nullptr) { @@ -50,15 +50,15 @@ TStringBuf TMonService2HttpRequest::GetHeader(TStringBuf name) const { return TStringBuf(); } -const THttpHeaders& TMonService2HttpRequest::GetHeaders() const { - return HttpRequest->GetHeaders(); -} +const THttpHeaders& TMonService2HttpRequest::GetHeaders() const { + return HttpRequest->GetHeaders(); +} TString TMonService2HttpRequest::GetRemoteAddr() const { return HttpRequest->GetRemoteAddr(); } -TStringBuf TMonService2HttpRequest::GetCookie(TStringBuf name) const { +TStringBuf TMonService2HttpRequest::GetCookie(TStringBuf name) const { TStringBuf cookie = GetHeader("Cookie"); size_t size = cookie.size(); size_t start = 0; diff --git a/library/cpp/monlib/service/mon_service_http_request.h b/library/cpp/monlib/service/mon_service_http_request.h index 8409205732..b4f2f8f0c5 100644 --- a/library/cpp/monlib/service/mon_service_http_request.h +++ b/library/cpp/monlib/service/mon_service_http_request.h @@ -8,32 +8,32 @@ namespace NMonitoring { class TMonService2; class IMonPage; - // XXX: IHttpRequest is already taken - struct IMonHttpRequest { - virtual ~IMonHttpRequest(); - - virtual IOutputStream& Output() = 0; - - virtual HTTP_METHOD GetMethod() const = 0; - virtual TStringBuf GetPath() const = 0; - virtual TStringBuf GetPathInfo() const = 0; - virtual TStringBuf GetUri() const = 0; - virtual const TCgiParameters& GetParams() const = 0; - virtual const TCgiParameters& GetPostParams() const = 0; - virtual TStringBuf GetPostContent() const = 0; - virtual const THttpHeaders& GetHeaders() const = 0; - virtual TStringBuf GetHeader(TStringBuf name) const = 0; - virtual TStringBuf GetCookie(TStringBuf name) const = 0; + // XXX: IHttpRequest is already taken + struct IMonHttpRequest { + virtual ~IMonHttpRequest(); + + virtual IOutputStream& Output() = 0; + + virtual HTTP_METHOD GetMethod() const = 0; + virtual TStringBuf GetPath() const = 0; + virtual TStringBuf GetPathInfo() const = 0; + virtual TStringBuf GetUri() const = 0; + virtual const TCgiParameters& GetParams() const = 0; + virtual const TCgiParameters& GetPostParams() const = 0; + virtual TStringBuf GetPostContent() const = 0; + virtual const THttpHeaders& GetHeaders() const = 0; + virtual TStringBuf GetHeader(TStringBuf name) const = 0; + virtual TStringBuf GetCookie(TStringBuf name) const = 0; virtual TString GetRemoteAddr() const = 0; - - virtual TString GetServiceTitle() const = 0; - - virtual IMonPage* GetPage() const = 0; - - virtual IMonHttpRequest* MakeChild(IMonPage* page, const TString& pathInfo) const = 0; - }; - - struct TMonService2HttpRequest: IMonHttpRequest { + + virtual TString GetServiceTitle() const = 0; + + virtual IMonPage* GetPage() const = 0; + + virtual IMonHttpRequest* MakeChild(IMonPage* page, const TString& pathInfo) const = 0; + }; + + struct TMonService2HttpRequest: IMonHttpRequest { IOutputStream* const Out; const IHttpRequest* const HttpRequest; TMonService2* const MonService; @@ -55,36 +55,36 @@ namespace NMonitoring { { } - ~TMonService2HttpRequest() override; - - IOutputStream& Output() override; - HTTP_METHOD GetMethod() const override; - TStringBuf GetPath() const override; - TStringBuf GetPathInfo() const override; - TStringBuf GetUri() const override; - const TCgiParameters& GetParams() const override; - const TCgiParameters& GetPostParams() const override; - TStringBuf GetPostContent() const override { - return HttpRequest->GetPostContent(); - } - - TStringBuf GetHeader(TStringBuf name) const override; - TStringBuf GetCookie(TStringBuf name) const override; - const THttpHeaders& GetHeaders() const override; + ~TMonService2HttpRequest() override; + + IOutputStream& Output() override; + HTTP_METHOD GetMethod() const override; + TStringBuf GetPath() const override; + TStringBuf GetPathInfo() const override; + TStringBuf GetUri() const override; + const TCgiParameters& GetParams() const override; + const TCgiParameters& GetPostParams() const override; + TStringBuf GetPostContent() const override { + return HttpRequest->GetPostContent(); + } + + TStringBuf GetHeader(TStringBuf name) const override; + TStringBuf GetCookie(TStringBuf name) const override; + const THttpHeaders& GetHeaders() const override; TString GetRemoteAddr() const override; - IMonPage* GetPage() const override { - return MonPage; - } - - TMonService2HttpRequest* MakeChild(IMonPage* page, const TString& pathInfo) const override { - return new TMonService2HttpRequest{ - Out, HttpRequest, MonService, page, - pathInfo, const_cast<TMonService2HttpRequest*>(this) - }; - } - - TString GetServiceTitle() const override; + IMonPage* GetPage() const override { + return MonPage; + } + + TMonService2HttpRequest* MakeChild(IMonPage* page, const TString& pathInfo) const override { + return new TMonService2HttpRequest{ + Out, HttpRequest, MonService, page, + pathInfo, const_cast<TMonService2HttpRequest*>(this) + }; + } + + TString GetServiceTitle() const override; }; } diff --git a/library/cpp/monlib/service/monservice.cpp b/library/cpp/monlib/service/monservice.cpp index a91df09517..d1b9cda1d2 100644 --- a/library/cpp/monlib/service/monservice.cpp +++ b/library/cpp/monlib/service/monservice.cpp @@ -12,40 +12,40 @@ using namespace NMonitoring; -TMonService2::TMonService2(ui16 port, const TString& host, ui32 threads, const TString& title, THolder<IAuthProvider> auth) - : TMonService2(HttpServerOptions(port, host, threads), title, std::move(auth)) +TMonService2::TMonService2(ui16 port, const TString& host, ui32 threads, const TString& title, THolder<IAuthProvider> auth) + : TMonService2(HttpServerOptions(port, host, threads), title, std::move(auth)) { } -TMonService2::TMonService2(const THttpServerOptions& options, const TString& title, THolder<IAuthProvider> auth) +TMonService2::TMonService2(const THttpServerOptions& options, const TString& title, THolder<IAuthProvider> auth) : NMonitoring::TMtHttpServer(options, std::bind(&TMonService2::ServeRequest, this, std::placeholders::_1, std::placeholders::_2)) , Title(title) , IndexMonPage(new TIndexMonPage("", Title)) - , AuthProvider_{std::move(auth)} + , AuthProvider_{std::move(auth)} { Y_VERIFY(!!title); time_t t = time(nullptr); ctime_r(&t, StartTime); } -TMonService2::TMonService2(const THttpServerOptions& options, TSimpleSharedPtr<IThreadPool> pool, const TString& title, THolder<IAuthProvider> auth) +TMonService2::TMonService2(const THttpServerOptions& options, TSimpleSharedPtr<IThreadPool> pool, const TString& title, THolder<IAuthProvider> auth) : NMonitoring::TMtHttpServer(options, std::bind(&TMonService2::ServeRequest, this, std::placeholders::_1, std::placeholders::_2), std::move(pool)) , Title(title) , IndexMonPage(new TIndexMonPage("", Title)) - , AuthProvider_{std::move(auth)} + , AuthProvider_{std::move(auth)} { Y_VERIFY(!!title); time_t t = time(nullptr); ctime_r(&t, StartTime); } -TMonService2::TMonService2(ui16 port, ui32 threads, const TString& title, THolder<IAuthProvider> auth) - : TMonService2(port, TString(), threads, title, std::move(auth)) +TMonService2::TMonService2(ui16 port, ui32 threads, const TString& title, THolder<IAuthProvider> auth) + : TMonService2(port, TString(), threads, title, std::move(auth)) { } -TMonService2::TMonService2(ui16 port, const TString& title, THolder<IAuthProvider> auth) - : TMonService2(port, TString(), 0, title, std::move(auth)) +TMonService2::TMonService2(ui16 port, const TString& title, THolder<IAuthProvider> auth) + : TMonService2(port, TString(), 0, title, std::move(auth)) { } @@ -80,21 +80,21 @@ void TMonService2::OutputIndexBody(IOutputStream& out) { void TMonService2::ServeRequest(IOutputStream& out, const NMonitoring::IHttpRequest& request) { TString path = request.GetPath(); Y_VERIFY(path.StartsWith('/')); - - if (AuthProvider_) { - const auto authResult = AuthProvider_->Check(request); - switch (authResult.Status) { - case TAuthResult::EStatus::NoCredentials: - out << HTTPUNAUTHORIZED; - return; - case TAuthResult::EStatus::Denied: - out << HTTPFORBIDDEN; - return; - case TAuthResult::EStatus::Ok: - break; - } - } - + + if (AuthProvider_) { + const auto authResult = AuthProvider_->Check(request); + switch (authResult.Status) { + case TAuthResult::EStatus::NoCredentials: + out << HTTPUNAUTHORIZED; + return; + case TAuthResult::EStatus::Denied: + out << HTTPFORBIDDEN; + return; + case TAuthResult::EStatus::Ok: + break; + } + } + if (path == "/") { OutputIndexPage(out); } else { @@ -108,10 +108,10 @@ void TMonService2::Register(IMonPage* page) { IndexMonPage->Register(page); } -void TMonService2::Register(TMonPagePtr page) { +void TMonService2::Register(TMonPagePtr page) { IndexMonPage->Register(std::move(page)); -} - +} + TIndexMonPage* TMonService2::RegisterIndexPage(const TString& path, const TString& title) { return IndexMonPage->RegisterIndexPage(path, title); } diff --git a/library/cpp/monlib/service/monservice.h b/library/cpp/monlib/service/monservice.h index d2bd11d568..8f5e52fcdb 100644 --- a/library/cpp/monlib/service/monservice.h +++ b/library/cpp/monlib/service/monservice.h @@ -1,7 +1,7 @@ #pragma once #include "service.h" -#include "auth.h" +#include "auth.h" #include "mon_service_http_request.h" #include <library/cpp/monlib/service/pages/index_mon_page.h> @@ -17,7 +17,7 @@ namespace NMonitoring { const TString Title; char StartTime[26]; TIntrusivePtr<TIndexMonPage> IndexMonPage; - THolder<IAuthProvider> AuthProvider_; + THolder<IAuthProvider> AuthProvider_; public: static THttpServerOptions HttpServerOptions(ui16 port, const TString& host, ui32 threads) { @@ -38,11 +38,11 @@ namespace NMonitoring { } public: - explicit TMonService2(ui16 port, const TString& title = GetProgramName(), THolder<IAuthProvider> auth = nullptr); - explicit TMonService2(ui16 port, ui32 threads, const TString& title = GetProgramName(), THolder<IAuthProvider> auth = nullptr); - explicit TMonService2(ui16 port, const TString& host, ui32 threads, const TString& title = GetProgramName(), THolder<IAuthProvider> auth = nullptr); - explicit TMonService2(const THttpServerOptions& options, const TString& title = GetProgramName(), THolder<IAuthProvider> auth = nullptr); - explicit TMonService2(const THttpServerOptions& options, TSimpleSharedPtr<IThreadPool> pool, const TString& title = GetProgramName(), THolder<IAuthProvider> auth = nullptr); + explicit TMonService2(ui16 port, const TString& title = GetProgramName(), THolder<IAuthProvider> auth = nullptr); + explicit TMonService2(ui16 port, ui32 threads, const TString& title = GetProgramName(), THolder<IAuthProvider> auth = nullptr); + explicit TMonService2(ui16 port, const TString& host, ui32 threads, const TString& title = GetProgramName(), THolder<IAuthProvider> auth = nullptr); + explicit TMonService2(const THttpServerOptions& options, const TString& title = GetProgramName(), THolder<IAuthProvider> auth = nullptr); + explicit TMonService2(const THttpServerOptions& options, TSimpleSharedPtr<IThreadPool> pool, const TString& title = GetProgramName(), THolder<IAuthProvider> auth = nullptr); ~TMonService2() override { } @@ -61,8 +61,8 @@ namespace NMonitoring { virtual void OutputIndexBody(IOutputStream& out); void Register(IMonPage* page); - void Register(TMonPagePtr page); - + void Register(TMonPagePtr page); + TIndexMonPage* RegisterIndexPage(const TString& path, const TString& title); IMonPage* FindPage(const TString& relativePath); diff --git a/library/cpp/monlib/service/pages/diag_mon_page.cpp b/library/cpp/monlib/service/pages/diag_mon_page.cpp index a3b5292e56..2493ff4fba 100644 --- a/library/cpp/monlib/service/pages/diag_mon_page.cpp +++ b/library/cpp/monlib/service/pages/diag_mon_page.cpp @@ -2,8 +2,8 @@ using namespace NMonitoring; -void TDiagMonPage::OutputText(IOutputStream& out, NMonitoring::IMonHttpRequest& request) { - out << "uri: " << request.GetUri() << "\n"; - out << "path: " << request.GetPath() << "\n"; - out << "path info: " << request.GetPathInfo() << "\n"; +void TDiagMonPage::OutputText(IOutputStream& out, NMonitoring::IMonHttpRequest& request) { + out << "uri: " << request.GetUri() << "\n"; + out << "path: " << request.GetPath() << "\n"; + out << "path info: " << request.GetPathInfo() << "\n"; } diff --git a/library/cpp/monlib/service/pages/diag_mon_page.h b/library/cpp/monlib/service/pages/diag_mon_page.h index a0eb154ba5..761194d4ec 100644 --- a/library/cpp/monlib/service/pages/diag_mon_page.h +++ b/library/cpp/monlib/service/pages/diag_mon_page.h @@ -10,7 +10,7 @@ namespace NMonitoring { { } - void OutputText(IOutputStream& out, NMonitoring::IMonHttpRequest&) override; + void OutputText(IOutputStream& out, NMonitoring::IMonHttpRequest&) override; }; } diff --git a/library/cpp/monlib/service/pages/html_mon_page.cpp b/library/cpp/monlib/service/pages/html_mon_page.cpp index f37d0a961b..eb4eb3b66c 100644 --- a/library/cpp/monlib/service/pages/html_mon_page.cpp +++ b/library/cpp/monlib/service/pages/html_mon_page.cpp @@ -4,8 +4,8 @@ using namespace NMonitoring; -void THtmlMonPage::Output(NMonitoring::IMonHttpRequest& request) { - IOutputStream& out = request.Output(); +void THtmlMonPage::Output(NMonitoring::IMonHttpRequest& request) { + IOutputStream& out = request.Output(); out << HTTPOKHTML; HTML(out) { @@ -47,14 +47,14 @@ void THtmlMonPage::Output(NMonitoring::IMonHttpRequest& request) { } } -void THtmlMonPage::NotFound(NMonitoring::IMonHttpRequest& request) const { - IOutputStream& out = request.Output(); +void THtmlMonPage::NotFound(NMonitoring::IMonHttpRequest& request) const { + IOutputStream& out = request.Output(); out << HTTPNOTFOUND; out.Flush(); } - -void THtmlMonPage::NoContent(NMonitoring::IMonHttpRequest& request) const { - IOutputStream& out = request.Output(); - out << HTTPNOCONTENT; - out.Flush(); -} + +void THtmlMonPage::NoContent(NMonitoring::IMonHttpRequest& request) const { + IOutputStream& out = request.Output(); + out << HTTPNOCONTENT; + out.Flush(); +} diff --git a/library/cpp/monlib/service/pages/html_mon_page.h b/library/cpp/monlib/service/pages/html_mon_page.h index a7d45f50c9..e87c53b62b 100644 --- a/library/cpp/monlib/service/pages/html_mon_page.h +++ b/library/cpp/monlib/service/pages/html_mon_page.h @@ -12,12 +12,12 @@ namespace NMonitoring { { } - void Output(NMonitoring::IMonHttpRequest& request) override; + void Output(NMonitoring::IMonHttpRequest& request) override; - void NotFound(NMonitoring::IMonHttpRequest& request) const; - void NoContent(NMonitoring::IMonHttpRequest& request) const; + void NotFound(NMonitoring::IMonHttpRequest& request) const; + void NoContent(NMonitoring::IMonHttpRequest& request) const; - virtual void OutputContent(NMonitoring::IMonHttpRequest& request) = 0; + virtual void OutputContent(NMonitoring::IMonHttpRequest& request) = 0; bool OutputTableSorterJsCss; }; diff --git a/library/cpp/monlib/service/pages/index_mon_page.cpp b/library/cpp/monlib/service/pages/index_mon_page.cpp index f91591ebad..83ff8b529a 100644 --- a/library/cpp/monlib/service/pages/index_mon_page.cpp +++ b/library/cpp/monlib/service/pages/index_mon_page.cpp @@ -5,33 +5,33 @@ using namespace NMonitoring; -void TIndexMonPage::OutputIndexPage(IMonHttpRequest& request) { - request.Output() << HTTPOKHTML; - request.Output() << "<html>\n"; - OutputHead(request.Output()); +void TIndexMonPage::OutputIndexPage(IMonHttpRequest& request) { + request.Output() << HTTPOKHTML; + request.Output() << "<html>\n"; + OutputHead(request.Output()); OutputBody(request); - request.Output() << "</html>\n"; + request.Output() << "</html>\n"; } -void TIndexMonPage::Output(IMonHttpRequest& request) { - TStringBuf pathInfo = request.GetPathInfo(); +void TIndexMonPage::Output(IMonHttpRequest& request) { + TStringBuf pathInfo = request.GetPathInfo(); if (pathInfo.empty() || pathInfo == TStringBuf("/")) { OutputIndexPage(request); return; } - Y_VERIFY(pathInfo.StartsWith('/')); + Y_VERIFY(pathInfo.StartsWith('/')); TMonPagePtr found; // analogous to CGI PATH_INFO { TGuard<TMutex> g(Mtx); - TStringBuf pathTmp = request.GetPathInfo(); + TStringBuf pathTmp = request.GetPathInfo(); for (;;) { TPagesByPath::iterator i = PagesByPath.find(pathTmp); if (i != PagesByPath.end()) { found = i->second; - pathInfo = request.GetPathInfo().substr(pathTmp.size()); + pathInfo = request.GetPathInfo().substr(pathTmp.size()); Y_VERIFY(pathInfo.empty() || pathInfo.StartsWith('/')); break; } @@ -43,11 +43,11 @@ void TIndexMonPage::Output(IMonHttpRequest& request) { } } } - if (found) { + if (found) { THolder<IMonHttpRequest> child(request.MakeChild(found.Get(), TString{pathInfo})); - found->Output(*child); + found->Output(*child); } else { - request.Output() << HTTPNOTFOUND; + request.Output() << HTTPNOTFOUND; } } @@ -123,17 +123,17 @@ void TIndexMonPage::OutputHead(IOutputStream& out) { out << "</head>\n"; } -void TIndexMonPage::OutputBody(IMonHttpRequest& req) { - auto& out = req.Output(); - out << "<body>\n"; +void TIndexMonPage::OutputBody(IMonHttpRequest& req) { + auto& out = req.Output(); + out << "<body>\n"; // part of common navbar OutputNavBar(out); - out << "<div class='container'>\n" + out << "<div class='container'>\n" << "<h2>" << Title << "</h2>\n"; - OutputIndex(out, req.GetPathInfo().EndsWith('/')); - out << "<div>\n" + OutputIndex(out, req.GetPathInfo().EndsWith('/')); + out << "<div>\n" << "</body>\n"; } diff --git a/library/cpp/monlib/service/pages/index_mon_page.h b/library/cpp/monlib/service/pages/index_mon_page.h index c491b76557..bf514a3105 100644 --- a/library/cpp/monlib/service/pages/index_mon_page.h +++ b/library/cpp/monlib/service/pages/index_mon_page.h @@ -18,12 +18,12 @@ namespace NMonitoring { ~TIndexMonPage() override { } - void Output(IMonHttpRequest& request) override; - void OutputIndexPage(IMonHttpRequest& request); + void Output(IMonHttpRequest& request) override; + void OutputIndexPage(IMonHttpRequest& request); virtual void OutputIndex(IOutputStream& out, bool pathEndsWithSlash); virtual void OutputCommonJsCss(IOutputStream& out); void OutputHead(IOutputStream& out); - void OutputBody(IMonHttpRequest& out); + void OutputBody(IMonHttpRequest& out); void Register(TMonPagePtr page); TIndexMonPage* RegisterIndexPage(const TString& path, const TString& title); diff --git a/library/cpp/monlib/service/pages/mon_page.h b/library/cpp/monlib/service/pages/mon_page.h index b3d2635997..e396612bb0 100644 --- a/library/cpp/monlib/service/pages/mon_page.h +++ b/library/cpp/monlib/service/pages/mon_page.h @@ -17,8 +17,8 @@ namespace NMonitoring { static const char HTTPOKCSS[] = "HTTP/1.1 200 Ok\r\nContent-Type: text/css\r\nConnection: Close\r\n\r\n"; static const char HTTPNOCONTENT[] = "HTTP/1.1 204 No content\r\nConnection: Close\r\n\r\n"; static const char HTTPNOTFOUND[] = "HTTP/1.1 404 Invalid URI\r\nConnection: Close\r\n\r\nInvalid URL\r\n"; - static const char HTTPUNAUTHORIZED[] = "HTTP/1.1 401 Unauthorized\r\nConnection: Close\r\n\r\nUnauthorized\r\n"; - static const char HTTPFORBIDDEN[] = "HTTP/1.1 403 Forbidden\r\nConnection: Close\r\n\r\nForbidden\r\n"; + static const char HTTPUNAUTHORIZED[] = "HTTP/1.1 401 Unauthorized\r\nConnection: Close\r\n\r\nUnauthorized\r\n"; + static const char HTTPFORBIDDEN[] = "HTTP/1.1 403 Forbidden\r\nConnection: Close\r\n\r\nForbidden\r\n"; // Fonts static const char HTTPOKFONTEOT[] = "HTTP/1.1 200 Ok\r\nContent-Type: application/vnd.ms-fontobject\r\nConnection: Close\r\n\r\n"; @@ -60,7 +60,7 @@ namespace NMonitoring { return !Title.empty(); } - virtual void Output(IMonHttpRequest& request) = 0; + virtual void Output(IMonHttpRequest& request) = 0; }; } diff --git a/library/cpp/monlib/service/pages/pre_mon_page.cpp b/library/cpp/monlib/service/pages/pre_mon_page.cpp index fcec142b09..fc03a19b80 100644 --- a/library/cpp/monlib/service/pages/pre_mon_page.cpp +++ b/library/cpp/monlib/service/pages/pre_mon_page.cpp @@ -2,17 +2,17 @@ using namespace NMonitoring; -void TPreMonPage::OutputContent(NMonitoring::IMonHttpRequest& request) { - auto& out = request.Output(); +void TPreMonPage::OutputContent(NMonitoring::IMonHttpRequest& request) { + auto& out = request.Output(); if (PreTag) { BeforePre(request); - out << "<pre>\n"; - OutputText(out, request); - out << "</pre>\n"; + out << "<pre>\n"; + OutputText(out, request); + out << "</pre>\n"; } else { - OutputText(out, request); + OutputText(out, request); } } -void TPreMonPage::BeforePre(NMonitoring::IMonHttpRequest&) { +void TPreMonPage::BeforePre(NMonitoring::IMonHttpRequest&) { } diff --git a/library/cpp/monlib/service/pages/pre_mon_page.h b/library/cpp/monlib/service/pages/pre_mon_page.h index 66a9fe494d..c9a923d39a 100644 --- a/library/cpp/monlib/service/pages/pre_mon_page.h +++ b/library/cpp/monlib/service/pages/pre_mon_page.h @@ -13,13 +13,13 @@ namespace NMonitoring { { } - void OutputContent(NMonitoring::IMonHttpRequest& request) override; + void OutputContent(NMonitoring::IMonHttpRequest& request) override; // hook to customize output - virtual void BeforePre(NMonitoring::IMonHttpRequest& request); + virtual void BeforePre(NMonitoring::IMonHttpRequest& request); // put your text here - virtual void OutputText(IOutputStream& out, NMonitoring::IMonHttpRequest&) = 0; + virtual void OutputText(IOutputStream& out, NMonitoring::IMonHttpRequest&) = 0; const bool PreTag; }; diff --git a/library/cpp/monlib/service/pages/registry_mon_page.cpp b/library/cpp/monlib/service/pages/registry_mon_page.cpp index c47668391f..c59e50f622 100644 --- a/library/cpp/monlib/service/pages/registry_mon_page.cpp +++ b/library/cpp/monlib/service/pages/registry_mon_page.cpp @@ -1,46 +1,46 @@ -#include "registry_mon_page.h" - +#include "registry_mon_page.h" + #include <library/cpp/monlib/encode/text/text.h> #include <library/cpp/monlib/encode/json/json.h> #include <library/cpp/monlib/encode/prometheus/prometheus.h> #include <library/cpp/monlib/encode/spack/spack_v1.h> #include <library/cpp/monlib/service/format.h> - -namespace NMonitoring { + +namespace NMonitoring { void TMetricRegistryPage::Output(NMonitoring::IMonHttpRequest& request) { - const auto formatStr = TStringBuf{request.GetPathInfo()}.RNextTok('/'); - auto& out = request.Output(); - + const auto formatStr = TStringBuf{request.GetPathInfo()}.RNextTok('/'); + auto& out = request.Output(); + if (!formatStr.empty()) { IMetricEncoderPtr encoder; - TString resp; - + TString resp; + if (formatStr == TStringBuf("json")) { - resp = HTTPOKJSON; - encoder = NMonitoring::EncoderJson(&out); + resp = HTTPOKJSON; + encoder = NMonitoring::EncoderJson(&out); } else if (formatStr == TStringBuf("spack")) { - resp = HTTPOKSPACK; + resp = HTTPOKSPACK; const auto compression = ParseCompression(request); encoder = NMonitoring::EncoderSpackV1(&out, ETimePrecision::SECONDS, compression); } else if (formatStr == TStringBuf("prometheus")) { resp = HTTPOKPROMETHEUS; encoder = NMonitoring::EncoderPrometheus(&out); - } else { + } else { ythrow yexception() << "unsupported metric encoding format: " << formatStr; - } - - out.Write(resp); + } + + out.Write(resp); RegistryRawPtr_->Accept(TInstant::Zero(), encoder.Get()); encoder->Close(); - } else { - THtmlMonPage::Output(request); - } - } - + } else { + THtmlMonPage::Output(request); + } + } + void TMetricRegistryPage::OutputText(IOutputStream& out, NMonitoring::IMonHttpRequest&) { IMetricEncoderPtr encoder = NMonitoring::EncoderText(&out); RegistryRawPtr_->Accept(TInstant::Zero(), encoder.Get()); - } - -} + } + +} diff --git a/library/cpp/monlib/service/pages/registry_mon_page.h b/library/cpp/monlib/service/pages/registry_mon_page.h index f23ab69232..2d26d3319c 100644 --- a/library/cpp/monlib/service/pages/registry_mon_page.h +++ b/library/cpp/monlib/service/pages/registry_mon_page.h @@ -1,32 +1,32 @@ -#pragma once - -#include "pre_mon_page.h" - +#pragma once + +#include "pre_mon_page.h" + #include <library/cpp/monlib/metrics/metric_registry.h> - -namespace NMonitoring { + +namespace NMonitoring { // For now this class can only enumerate all metrics without any grouping or serve JSON/Spack/Prometheus class TMetricRegistryPage: public TPreMonPage { - public: + public: TMetricRegistryPage(const TString& path, const TString& title, TAtomicSharedPtr<IMetricSupplier> registry) - : TPreMonPage(path, title) + : TPreMonPage(path, title) , Registry_(registry) , RegistryRawPtr_(Registry_.Get()) - { - } - + { + } + TMetricRegistryPage(const TString& path, const TString& title, IMetricSupplier* registry) : TPreMonPage(path, title) , RegistryRawPtr_(registry) { } - void Output(NMonitoring::IMonHttpRequest& request) override; - void OutputText(IOutputStream& out, NMonitoring::IMonHttpRequest&) override; - - private: + void Output(NMonitoring::IMonHttpRequest& request) override; + void OutputText(IOutputStream& out, NMonitoring::IMonHttpRequest&) override; + + private: TAtomicSharedPtr<IMetricSupplier> Registry_; IMetricSupplier* RegistryRawPtr_; - }; - -} + }; + +} diff --git a/library/cpp/monlib/service/pages/resource_mon_page.cpp b/library/cpp/monlib/service/pages/resource_mon_page.cpp index 16b1eb1aa0..ec4ac6a1a7 100644 --- a/library/cpp/monlib/service/pages/resource_mon_page.cpp +++ b/library/cpp/monlib/service/pages/resource_mon_page.cpp @@ -2,8 +2,8 @@ using namespace NMonitoring; -void TResourceMonPage::Output(NMonitoring::IMonHttpRequest& request) { - IOutputStream& out = request.Output(); +void TResourceMonPage::Output(NMonitoring::IMonHttpRequest& request) { + IOutputStream& out = request.Output(); switch (ResourceType) { case TEXT: out << HTTPOKTEXT; @@ -42,8 +42,8 @@ void TResourceMonPage::Output(NMonitoring::IMonHttpRequest& request) { out << NResource::Find(ResourceName); } -void TResourceMonPage::NotFound(NMonitoring::IMonHttpRequest& request) const { - IOutputStream& out = request.Output(); +void TResourceMonPage::NotFound(NMonitoring::IMonHttpRequest& request) const { + IOutputStream& out = request.Output(); out << HTTPNOTFOUND; out.Flush(); } diff --git a/library/cpp/monlib/service/pages/resource_mon_page.h b/library/cpp/monlib/service/pages/resource_mon_page.h index ec00e8ccc4..f6ab67200e 100644 --- a/library/cpp/monlib/service/pages/resource_mon_page.h +++ b/library/cpp/monlib/service/pages/resource_mon_page.h @@ -31,9 +31,9 @@ namespace NMonitoring { { } - void Output(NMonitoring::IMonHttpRequest& request) override; + void Output(NMonitoring::IMonHttpRequest& request) override; - void NotFound(NMonitoring::IMonHttpRequest& request) const; + void NotFound(NMonitoring::IMonHttpRequest& request) const; private: TString ResourceName; diff --git a/library/cpp/monlib/service/pages/templates.h b/library/cpp/monlib/service/pages/templates.h index eac97955a2..b4656f059f 100644 --- a/library/cpp/monlib/service/pages/templates.h +++ b/library/cpp/monlib/service/pages/templates.h @@ -94,28 +94,28 @@ #define COLLAPSED_BUTTON_CONTENT(targetId, buttonText) \ WITH_SCOPED(tmp, NMonitoring::TCollapsedButton(__stream, targetId, buttonText)) -#define HREF(path) \ - WITH_SCOPED(tmp, NMonitoring::THref(__stream, path)) - +#define HREF(path) \ + WITH_SCOPED(tmp, NMonitoring::THref(__stream, path)) + namespace NMonitoring { - struct THref { - THref(IOutputStream& str, TStringBuf path) - : Str(str) - { - Str << "<a href="<< path << '>'; - } - - ~THref() { - Str << "</a>"; - } - - explicit inline operator bool() const noexcept { - return true; // just to work with WITH_SCOPED - } - - IOutputStream& Str; - }; - + struct THref { + THref(IOutputStream& str, TStringBuf path) + : Str(str) + { + Str << "<a href="<< path << '>'; + } + + ~THref() { + Str << "</a>"; + } + + explicit inline operator bool() const noexcept { + return true; // just to work with WITH_SCOPED + } + + IOutputStream& Str; + }; + template <const char* tag> struct TTag { TTag(IOutputStream& str, TStringBuf cls = "", TStringBuf for0 = "", TStringBuf id = "") diff --git a/library/cpp/monlib/service/pages/version_mon_page.cpp b/library/cpp/monlib/service/pages/version_mon_page.cpp index 0031031815..41e29417da 100644 --- a/library/cpp/monlib/service/pages/version_mon_page.cpp +++ b/library/cpp/monlib/service/pages/version_mon_page.cpp @@ -6,7 +6,7 @@ using namespace NMonitoring; -void TVersionMonPage::OutputText(IOutputStream& out, NMonitoring::IMonHttpRequest&) { +void TVersionMonPage::OutputText(IOutputStream& out, NMonitoring::IMonHttpRequest&) { const char* version = GetProgramSvnVersion(); out << version; if (!TString(version).EndsWith("\n")) diff --git a/library/cpp/monlib/service/pages/version_mon_page.h b/library/cpp/monlib/service/pages/version_mon_page.h index c559e5940d..f7649947e4 100644 --- a/library/cpp/monlib/service/pages/version_mon_page.h +++ b/library/cpp/monlib/service/pages/version_mon_page.h @@ -9,7 +9,7 @@ namespace NMonitoring { { } - void OutputText(IOutputStream& out, NMonitoring::IMonHttpRequest&) override; + void OutputText(IOutputStream& out, NMonitoring::IMonHttpRequest&) override; }; } diff --git a/library/cpp/monlib/service/pages/ya.make b/library/cpp/monlib/service/pages/ya.make index 6c656748ce..48d44a0838 100644 --- a/library/cpp/monlib/service/pages/ya.make +++ b/library/cpp/monlib/service/pages/ya.make @@ -1,6 +1,6 @@ LIBRARY() -OWNER(g:solomon) +OWNER(g:solomon) NO_WSHADOW() @@ -13,7 +13,7 @@ SRCS( resource_mon_page.cpp templates.cpp version_mon_page.cpp - registry_mon_page.cpp + registry_mon_page.cpp ) PEERDIR( diff --git a/library/cpp/monlib/service/ya.make b/library/cpp/monlib/service/ya.make index fab3ff1c42..ad088fc2c6 100644 --- a/library/cpp/monlib/service/ya.make +++ b/library/cpp/monlib/service/ya.make @@ -1,13 +1,13 @@ LIBRARY() -OWNER(g:solomon) +OWNER(g:solomon) SRCS( monservice.cpp mon_service_http_request.cpp service.cpp - format.cpp - auth.cpp + format.cpp + auth.cpp ) PEERDIR( diff --git a/library/cpp/monlib/ya.make b/library/cpp/monlib/ya.make index bcf29d859f..9bd236d6fd 100644 --- a/library/cpp/monlib/ya.make +++ b/library/cpp/monlib/ya.make @@ -9,27 +9,27 @@ RECURSE( counters/ut deprecated dynamic_counters - dynamic_counters/percentile - dynamic_counters/percentile/ut + dynamic_counters/percentile + dynamic_counters/percentile/ut dynamic_counters/ut encode encode/buffered encode/buffered/ut - encode/fake - encode/fuzz + encode/fake + encode/fuzz encode/json encode/json/ut - encode/legacy_protobuf - encode/legacy_protobuf/ut + encode/legacy_protobuf + encode/legacy_protobuf/ut encode/prometheus encode/prometheus/ut - encode/protobuf + encode/protobuf encode/spack encode/spack/ut encode/text encode/text/ut - encode/unistat - encode/unistat/ut + encode/unistat + encode/unistat/ut encode/ut example exception diff --git a/library/python/ya.make b/library/python/ya.make index c17bf477e1..2e1eb6e0e1 100644 --- a/library/python/ya.make +++ b/library/python/ya.make @@ -121,7 +121,7 @@ RECURSE( monitoring monlib monlib/examples - monlib/so + monlib/so murmurhash nirvana nirvana_api diff --git a/util/datetime/parser.rl6 b/util/datetime/parser.rl6 index 6dbb646a6c..931f09eae1 100644 --- a/util/datetime/parser.rl6 +++ b/util/datetime/parser.rl6 @@ -199,7 +199,7 @@ day = int2 @set_day; hour = int2 @set_hour; minute = int2 @set_minute; second = int2 @set_second; -secondFrac = digit {1,6} >clear_int $update_int @{ +secondFrac = digit {1,6} >clear_int $update_int @{ ui32 us = I; for (int k = Dc; k < 6; ++k) { us *= 10; diff --git a/ydb/core/actorlib_impl/test_interconnect_ut.cpp b/ydb/core/actorlib_impl/test_interconnect_ut.cpp index 50f6f5ff3b..0d9d3535ef 100644 --- a/ydb/core/actorlib_impl/test_interconnect_ut.cpp +++ b/ydb/core/actorlib_impl/test_interconnect_ut.cpp @@ -275,7 +275,7 @@ Y_UNIT_TEST_SUITE(TInterconnectTest) { runtime.SetLogPriority(NActorsServices::INTERCONNECT, NActors::NLog::PRI_DEBUG); runtime.SetLogPriority(NActorsServices::INTERCONNECT_SESSION, NActors::NLog::PRI_DEBUG); SOCKET s = INVALID_SOCKET; - runtime.SetObserverFunc([&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>& ev) { + runtime.SetObserverFunc([&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>& ev) { if (auto *p = ev->CastAsLocal<TEvHandshakeDone>()) { s = *p->Socket; } diff --git a/ydb/core/blobstorage/dsproxy/dsproxy_mon.h b/ydb/core/blobstorage/dsproxy/dsproxy_mon.h index c2ccaca3b1..941bea6664 100644 --- a/ydb/core/blobstorage/dsproxy/dsproxy_mon.h +++ b/ydb/core/blobstorage/dsproxy/dsproxy_mon.h @@ -154,21 +154,21 @@ protected: // log response time - NMonitoring::TPercentileTrackerLg<3, 4, 3> PutResponseTime; // Used by whiteboard + NMonitoring::TPercentileTrackerLg<3, 4, 3> PutResponseTime; // Used by whiteboard - NMonitoring::TPercentileTrackerLg<3, 4, 3> PutTabletLogResponseTime; - NMonitoring::TPercentileTrackerLg<3, 4, 3> PutTabletLogResponseTime256; - NMonitoring::TPercentileTrackerLg<3, 4, 3> PutTabletLogResponseTime512; + NMonitoring::TPercentileTrackerLg<3, 4, 3> PutTabletLogResponseTime; + NMonitoring::TPercentileTrackerLg<3, 4, 3> PutTabletLogResponseTime256; + NMonitoring::TPercentileTrackerLg<3, 4, 3> PutTabletLogResponseTime512; - NMonitoring::TPercentileTrackerLg<3, 4, 3> PutAsyncBlobResponseTime; - NMonitoring::TPercentileTrackerLg<3, 4, 3> PutUserDataResponseTime; + NMonitoring::TPercentileTrackerLg<3, 4, 3> PutAsyncBlobResponseTime; + NMonitoring::TPercentileTrackerLg<3, 4, 3> PutUserDataResponseTime; - NMonitoring::TPercentileTrackerLg<3, 4, 3> GetResponseTime; // Used by witheboard + NMonitoring::TPercentileTrackerLg<3, 4, 3> GetResponseTime; // Used by witheboard - NMonitoring::TPercentileTrackerLg<3, 4, 3> BlockResponseTime; - NMonitoring::TPercentileTrackerLg<3, 4, 3> DiscoverResponseTime; - NMonitoring::TPercentileTrackerLg<3, 4, 3> IndexRestoreGetResponseTime; - NMonitoring::TPercentileTrackerLg<3, 4, 3> RangeResponseTime; + NMonitoring::TPercentileTrackerLg<3, 4, 3> BlockResponseTime; + NMonitoring::TPercentileTrackerLg<3, 4, 3> DiscoverResponseTime; + NMonitoring::TPercentileTrackerLg<3, 4, 3> IndexRestoreGetResponseTime; + NMonitoring::TPercentileTrackerLg<3, 4, 3> RangeResponseTime; NMonitoring::TPercentileTrackerLg<3, 4, 3> PatchResponseTime; // event counters diff --git a/ydb/core/blobstorage/dsproxy/dsproxy_nodemon.h b/ydb/core/blobstorage/dsproxy/dsproxy_nodemon.h index 0ec2c9288d..150da172fb 100644 --- a/ydb/core/blobstorage/dsproxy/dsproxy_nodemon.h +++ b/ydb/core/blobstorage/dsproxy/dsproxy_nodemon.h @@ -27,23 +27,23 @@ struct TDsProxyNodeMon : public TThrRefBase { TIntrusivePtr<NMonitoring::TDynamicCounters> Group; TIntrusivePtr<NMonitoring::TDynamicCounters> LatencyOverview; - NMonitoring::TPercentileTracker<4, 512, 15> PutResponseTime; + NMonitoring::TPercentileTracker<4, 512, 15> PutResponseTime; - NMonitoring::TPercentileTracker<4, 512, 15> PutTabletLogResponseTime; - NMonitoring::TPercentileTracker<4, 512, 15> PutTabletLogResponseTime256; - NMonitoring::TPercentileTracker<4, 512, 15> PutTabletLogResponseTime512; + NMonitoring::TPercentileTracker<4, 512, 15> PutTabletLogResponseTime; + NMonitoring::TPercentileTracker<4, 512, 15> PutTabletLogResponseTime256; + NMonitoring::TPercentileTracker<4, 512, 15> PutTabletLogResponseTime512; static constexpr ui32 KnownDeviceTypesCount = 4; using THistoPtrForDeviceType = std::array<NMonitoring::THistogramPtr, KnownDeviceTypesCount>; THistoPtrForDeviceType PutTabletLogResponseTimeHist256Ki; THistoPtrForDeviceType PutTabletLogResponseTimeHistInf; - NMonitoring::TPercentileTracker<4, 512, 15> PutAsyncBlobResponseTime; + NMonitoring::TPercentileTracker<4, 512, 15> PutAsyncBlobResponseTime; THistoPtrForDeviceType PutAsyncBlobResponseTimeHist; - NMonitoring::TPercentileTracker<4, 512, 15> PutUserDataResponseTime; + NMonitoring::TPercentileTracker<4, 512, 15> PutUserDataResponseTime; THistoPtrForDeviceType PutUserDataResponseTimeHist; - NMonitoring::TPercentileTracker<16, 512, 15> GetResponseTime; + NMonitoring::TPercentileTracker<16, 512, 15> GetResponseTime; NMonitoring::TPercentileTracker<16, 512, 15> GetAsyncReadResponseTime; THistoPtrForDeviceType GetAsyncReadResponseTimeHist; NMonitoring::TPercentileTracker<16, 512, 15> GetFastReadResponseTime256Ki; @@ -58,10 +58,10 @@ struct TDsProxyNodeMon : public TThrRefBase { NMonitoring::TPercentileTracker<16, 512, 15> PatchResponseTime; THistoPtrForDeviceType PatchResponseTimeHist; - NMonitoring::TPercentileTracker<16, 512, 15> BlockResponseTime; - NMonitoring::TPercentileTracker<16, 512, 15> DiscoverResponseTime; - NMonitoring::TPercentileTracker<16, 512, 15> IndexRestoreGetResponseTime; - NMonitoring::TPercentileTracker<16, 512, 15> RangeResponseTime; + NMonitoring::TPercentileTracker<16, 512, 15> BlockResponseTime; + NMonitoring::TPercentileTracker<16, 512, 15> DiscoverResponseTime; + NMonitoring::TPercentileTracker<16, 512, 15> IndexRestoreGetResponseTime; + NMonitoring::TPercentileTracker<16, 512, 15> RangeResponseTime; std::array<bool, KnownDeviceTypesCount> IsCountersPresentedForIdx; diff --git a/ydb/core/blobstorage/nodewarden/blobstorage_node_warden_ut.cpp b/ydb/core/blobstorage/nodewarden/blobstorage_node_warden_ut.cpp index 70d3ea8e67..e3c74cce7b 100644 --- a/ydb/core/blobstorage/nodewarden/blobstorage_node_warden_ut.cpp +++ b/ydb/core/blobstorage/nodewarden/blobstorage_node_warden_ut.cpp @@ -414,7 +414,7 @@ Y_UNIT_TEST_SUITE(TBlobStorageWardenTest) { TBlockUpdates(TTestBasicRuntime& runtime) : Runtime(&runtime) { - TTestActorRuntime::TEventObserver observer = [=] (TTestActorRuntimeBase& /*runtime*/, TAutoPtr<IEventHandle>& event) -> TTestActorRuntime::EEventAction { + TTestActorRuntime::TEventObserver observer = [=] (TTestActorRuntimeBase& /*runtime*/, TAutoPtr<IEventHandle>& event) -> TTestActorRuntime::EEventAction { if (event->GetTypeRewrite() == TEvBlobStorage::EvControllerNodeServiceSetUpdate) { return TTestActorRuntime::EEventAction::DROP; } diff --git a/ydb/core/blobstorage/other/mon_blob_range_page.cpp b/ydb/core/blobstorage/other/mon_blob_range_page.cpp index 4975939fea..f4797d62e2 100644 --- a/ydb/core/blobstorage/other/mon_blob_range_page.cpp +++ b/ydb/core/blobstorage/other/mon_blob_range_page.cpp @@ -55,11 +55,11 @@ namespace { , ActorSystem(actorSystem) {} - void Output(NMonitoring::IMonHttpRequest& request) override { - IOutputStream& out = request.Output(); + void Output(NMonitoring::IMonHttpRequest& request) override { + IOutputStream& out = request.Output(); // parse HTTP request - const TCgiParameters& params = request.GetParams(); + const TCgiParameters& params = request.GetParams(); auto generateError = [&](const TString& msg) { out << "HTTP/1.1 400 Bad Request\r\n" diff --git a/ydb/core/blobstorage/other/mon_get_blob_page.cpp b/ydb/core/blobstorage/other/mon_get_blob_page.cpp index 13525b9a06..95308879a3 100644 --- a/ydb/core/blobstorage/other/mon_get_blob_page.cpp +++ b/ydb/core/blobstorage/other/mon_get_blob_page.cpp @@ -82,11 +82,11 @@ namespace { , ActorSystem(actorSystem) {} - void Output(NMonitoring::IMonHttpRequest& request) override { - IOutputStream& out = request.Output(); + void Output(NMonitoring::IMonHttpRequest& request) override { + IOutputStream& out = request.Output(); // parse HTTP request - const TCgiParameters& params = request.GetParams(); + const TCgiParameters& params = request.GetParams(); auto generateError = [&](const TString& msg) { out << "HTTP/1.1 400 Bad Request\r\n" diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_actor.cpp b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_actor.cpp index 0482aafc40..d92b19c6cc 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_actor.cpp +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_actor.cpp @@ -886,7 +886,7 @@ public: } void Handle(NMon::TEvHttpInfo::TPtr &ev) { - const TCgiParameters &cgi = ev->Get()->Request.GetPostParams(); + const TCgiParameters &cgi = ev->Get()->Request.GetPostParams(); if (cgi.Has("chunksLockByRange")) { ui32 begin = strtoul(cgi.Get("chunksLockBegin").c_str(), nullptr, 10); ui32 end = strtoul(cgi.Get("chunksLockEnd").c_str(), nullptr, 10); @@ -925,7 +925,7 @@ public: } bool doGetSchedule = false; - const auto& httpRequest = ev->Get()->Request; + const auto& httpRequest = ev->Get()->Request; if (httpRequest.GetMethod() == HTTP_METHOD_GET) { /* TStringStream out; diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_mon.h b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_mon.h index 6f266c3bdc..38a0f73b7c 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_mon.h +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_mon.h @@ -281,7 +281,7 @@ private: class TBurstmeter { private: TBucketQuoter<i64, TSpinLock, THPTimerUs> Bucket; - NMonitoring::TPercentileTrackerLg<5, 4, 15> Tracker; + NMonitoring::TPercentileTrackerLg<5, 4, 15> Tracker; public: TBurstmeter() : Bucket(1000ull * 1000ull * 1000ull, 0) @@ -417,7 +417,7 @@ struct TPDiskMon { NMonitoring::TDynamicCounters::TCounterPtr PDiskThreadBusyTimeNs; public: - NMonitoring::TPercentileTrackerLg<5, 4, 15> UpdateCycleTime; + NMonitoring::TPercentileTrackerLg<5, 4, 15> UpdateCycleTime; public: TUpdateDurationTracker() @@ -583,16 +583,16 @@ struct TPDiskMon { TBurstmeter BestEffortBurst; // queue length seen by arriving request in front of it (QLA = Queue Length at Arrival) - using TQLATracker = NMonitoring::TPercentileTrackerLg<5, 4, 15>; + using TQLATracker = NMonitoring::TPercentileTrackerLg<5, 4, 15>; TQLATracker InputQLA; // for PDisk.InputQueue // queue cost seen by arriving request in front of it (QCA = Queue Cost at Arrival) - using TQCATracker = NMonitoring::TPercentileTrackerLg<5, 4, 15>; + using TQCATracker = NMonitoring::TPercentileTrackerLg<5, 4, 15>; TQCATracker InputQCA; // for PDisk.InputQueue // log cumulative size bytes // <BASE_BITS, EXP_BITS, FRAME_COUNT> - using TSizeTracker = NMonitoring::TPercentileTrackerLg<5, 4, 15>; + using TSizeTracker = NMonitoring::TPercentileTrackerLg<5, 4, 15>; TSizeTracker LogOperationSizeBytes; TSizeTracker GetSyncLogSizeBytes; diff --git a/ydb/core/blobstorage/testload/test_load_actor.cpp b/ydb/core/blobstorage/testload/test_load_actor.cpp index d0688be1b2..c8788b1310 100644 --- a/ydb/core/blobstorage/testload/test_load_actor.cpp +++ b/ydb/core/blobstorage/testload/test_load_actor.cpp @@ -271,7 +271,7 @@ public: info.ErrorMessage.clear(); - const auto& params = ev->Get()->Request.GetParams(); + const auto& params = ev->Get()->Request.GetParams(); if (params.Has("protobuf")) { NKikimrBlobStorage::TEvTestLoadRequest record; bool status = google::protobuf::TextFormat::ParseFromString(params.Get("protobuf"), &record); diff --git a/ydb/core/blobstorage/vdisk/skeleton/blobstorage_monactors.cpp b/ydb/core/blobstorage/vdisk/skeleton/blobstorage_monactors.cpp index e136208de6..44b3c58c9c 100644 --- a/ydb/core/blobstorage/vdisk/skeleton/blobstorage_monactors.cpp +++ b/ydb/core/blobstorage/vdisk/skeleton/blobstorage_monactors.cpp @@ -319,7 +319,7 @@ namespace NKikimr { } void Bootstrap(const TActorContext &ctx) { - const TCgiParameters& cgi = Ev->Get()->Request.GetParams(); + const TCgiParameters& cgi = Ev->Get()->Request.GetParams(); TString dbnameParam = cgi.Get("dbname"); TString formParam = cgi.Get("form"); @@ -523,7 +523,7 @@ namespace NKikimr { } void Bootstrap(const TActorContext &ctx) { - const TCgiParameters& cgi = Ev->Get()->Request.GetParams(); + const TCgiParameters& cgi = Ev->Get()->Request.GetParams(); TString dbnameParam = cgi.Get("dbname"); TString formParam = cgi.Get("form"); @@ -704,14 +704,14 @@ namespace NKikimr { friend class TActorBootstrapped<TSkeletonFrontMonDbStatActor>; bool PrettyPrint() const { - const TCgiParameters& cgi = Ev->Get()->Request.GetParams(); + const TCgiParameters& cgi = Ev->Get()->Request.GetParams(); // format numbers pretty print const TString &numbers = cgi.Get("numbers"); return (numbers == "pretty" || numbers == ""); // pretty by default } NMonUtil::TParseResult<ui64> ParseNumber(const char *name) const { - const TCgiParameters& cgi = Ev->Get()->Request.GetParams(); + const TCgiParameters& cgi = Ev->Get()->Request.GetParams(); // format numbers pretty print const TString &str = cgi.Get(name); if (str.empty()) @@ -983,7 +983,7 @@ namespace NKikimr { const std::shared_ptr<TBlobStorageGroupInfo::TTopology> &top, NMon::TEvHttpInfo::TPtr &ev, const TString &frontHtml) { - const TCgiParameters& cgi = ev->Get()->Request.GetParams(); + const TCgiParameters& cgi = ev->Get()->Request.GetParams(); const TString &type = cgi.Get("type"); const TString &dbname = cgi.Get("dbname"); diff --git a/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeletonfront.cpp b/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeletonfront.cpp index 10eab522d0..08c32a8921 100644 --- a/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeletonfront.cpp +++ b/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeletonfront.cpp @@ -1352,7 +1352,7 @@ namespace NKikimr { //////////////////////////////////////////////////////////////////////////// void Handle(NMon::TEvHttpInfo::TPtr &ev, const TActorContext &ctx) { // we process mon requests out of order - const TCgiParameters& cgi = ev->Get()->Request.GetParams(); + const TCgiParameters& cgi = ev->Get()->Request.GetParams(); const TString& type = cgi.Get("type"); TString html = (type == TString()) ? GenerateHtmlState(ctx) : TString(); auto aid = ctx.Register(CreateFrontSkeletonMonRequestHandler(SelfVDiskId, ctx.SelfID, SkeletonId, diff --git a/ydb/core/client/server/http_ping.cpp b/ydb/core/client/server/http_ping.cpp index 1bbd3e62ad..64e04c88d4 100644 --- a/ydb/core/client/server/http_ping.cpp +++ b/ydb/core/client/server/http_ping.cpp @@ -7,8 +7,8 @@ TPing::TPing() : NMonitoring::IMonPage("ping") {} -void TPing::Output(NMonitoring::IMonHttpRequest& request) { - IOutputStream& out(request.Output()); +void TPing::Output(NMonitoring::IMonHttpRequest& request) { + IOutputStream& out(request.Output()); out << NMonitoring::HTTPOKTEXT; out << "ok /ping"; } diff --git a/ydb/core/client/server/http_ping.h b/ydb/core/client/server/http_ping.h index a451460a12..e07aba1eea 100644 --- a/ydb/core/client/server/http_ping.h +++ b/ydb/core/client/server/http_ping.h @@ -10,7 +10,7 @@ namespace NHttp { class TPing : public NMonitoring::IMonPage { public: TPing(); - virtual void Output(NMonitoring::IMonHttpRequest& request) override; + virtual void Output(NMonitoring::IMonHttpRequest& request) override; }; TPing* CreatePing(); diff --git a/ydb/core/client/server/msgbus_http_server.cpp b/ydb/core/client/server/msgbus_http_server.cpp index 9d2e04b589..e535ebe335 100644 --- a/ydb/core/client/server/msgbus_http_server.cpp +++ b/ydb/core/client/server/msgbus_http_server.cpp @@ -24,7 +24,7 @@ public: class TBusHttpServerSession : public NBus::TBusServerSession { public: - TBusHttpServerSession(IMessageBusHttpServer* httpServer, NMonitoring::IMonHttpRequest& request, + TBusHttpServerSession(IMessageBusHttpServer* httpServer, NMonitoring::IMonHttpRequest& request, const TProtocol& protocol, const NBus::TBusServerSessionConfig& config) : HttpServer(httpServer) , Request(request) @@ -52,7 +52,7 @@ public: // the only methods used virtual NBus::EMessageStatus SendReply(const NBus::TBusIdentity&, NBus::TBusMessage *pRep) override { TAutoPtr<NBus::TBusBufferBase> response(static_cast<NBus::TBusBufferBase*>(pRep)); - IOutputStream& out(Request.Output()); + IOutputStream& out(Request.Output()); if (response->GetHeader()->Type == MTYPE_CLIENT_RESPONSE) { NKikimrClient::TResponse* pbResponse(static_cast<NKikimrClient::TResponse*>(response->GetRecord())); NMsgBusProxy::EResponseStatus status = static_cast<NMsgBusProxy::EResponseStatus>(pbResponse->GetStatus()); @@ -101,7 +101,7 @@ public: virtual NBus::EMessageStatus ForgetRequest(const NBus::TBusIdentity& ident) override { TBusHttpIdentity& httpIdent(const_cast<TBusHttpIdentity&>(static_cast<const TBusHttpIdentity&>(ident))); if (!ReplySent) { - IOutputStream& out(Request.Output()); + IOutputStream& out(Request.Output()); out << NMonitoring::HTTPNOCONTENT; } httpIdent.FinishWork(); @@ -115,7 +115,7 @@ public: protected: IMessageBusHttpServer* HttpServer; - NMonitoring::IMonHttpRequest& Request; + NMonitoring::IMonHttpRequest& Request; const TProtocol& Protocol; const NBus::TBusServerSessionConfig& Config; TSystemEvent DoneEvent; @@ -126,7 +126,7 @@ class TMessageBusHttpServer : public IMessageBusHttpServer { public: TMessageBusHttpServer(TActorSystem* actorSystem, NBus::IBusServerHandler* handler, const TProtocol& protocol, const NBus::TBusServerSessionConfig& config); ~TMessageBusHttpServer(); - virtual void Output(NMonitoring::IMonHttpRequest& request) override; + virtual void Output(NMonitoring::IMonHttpRequest& request) override; virtual void Shutdown() override; protected: @@ -168,15 +168,15 @@ void TMessageBusHttpServer::Shutdown() { Handler = nullptr; } -void TMessageBusHttpServer::Output(NMonitoring::IMonHttpRequest& request) { +void TMessageBusHttpServer::Output(NMonitoring::IMonHttpRequest& request) { if (Handler != nullptr) { THPTimer startTime; - TString pathInfo{request.GetPathInfo().substr(1)}; - TAutoPtr<NBus::TBusBufferBase> message = Protocol.NewMessage(pathInfo); + TString pathInfo{request.GetPathInfo().substr(1)}; + TAutoPtr<NBus::TBusBufferBase> message = Protocol.NewMessage(pathInfo); if (message != nullptr) { RequestsCount->Inc(); RequestsActive->Inc(); - TStringBuf postContent(request.GetPostContent()); + TStringBuf postContent(request.GetPostContent()); *InboundSize += postContent.size(); const ::google::protobuf::Descriptor* msgDescriptor = message->GetRecord()->GetDescriptor(); if (msgDescriptor->name() == "TJSON") { @@ -204,9 +204,9 @@ void TMessageBusHttpServer::Output(NMonitoring::IMonHttpRequest& request) { } LOG_DEBUG_S(*ActorSystem, NKikimrServices::HTTP, "HttpRequest " - << request.GetMethod() + << request.GetMethod() << " " - << request.GetUri() + << request.GetUri() << " " << message->GetRecord()->AsJSON()); @@ -218,7 +218,7 @@ void TMessageBusHttpServer::Output(NMonitoring::IMonHttpRequest& request) { session.Wait(); RequestsActive->Dec(); } else { - IOutputStream& out(request.Output()); + IOutputStream& out(request.Output()); out << NMonitoring::HTTPNOTFOUND; } RequestTotalTimeHistogram.Add(startTime.Passed() * 1000/*ms*/); diff --git a/ydb/core/client/server/msgbus_server_pq_metarequest_ut.cpp b/ydb/core/client/server/msgbus_server_pq_metarequest_ut.cpp index e412337eb6..237bba147b 100644 --- a/ydb/core/client/server/msgbus_server_pq_metarequest_ut.cpp +++ b/ydb/core/client/server/msgbus_server_pq_metarequest_ut.cpp @@ -42,7 +42,7 @@ protected: // Initialize runtime Runtime = MakeHolder<TTestBasicRuntime>(); - Runtime->SetObserverFunc([this](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>& event) { + Runtime->SetObserverFunc([this](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>& event) { return EventsObserver(event); }); Runtime->SetRegistrationObserverFunc([this](TTestActorRuntimeBase& runtime, const TActorId& parentId, const TActorId& actorId) { diff --git a/ydb/core/cms/cms_ut_common.cpp b/ydb/core/cms/cms_ut_common.cpp index b760a4c66b..b458fe672c 100644 --- a/ydb/core/cms/cms_ut_common.cpp +++ b/ydb/core/cms/cms_ut_common.cpp @@ -452,7 +452,7 @@ TCmsTestEnv::TCmsTestEnv(ui32 nodeCount, GenerateExtendedInfo(*this, config, pdisks, 4, tenants); // Set observer to pass fake base blobstorage config. - auto redirectConfigRequest = [](TTestActorRuntimeBase&, + auto redirectConfigRequest = [](TTestActorRuntimeBase&, TAutoPtr<IEventHandle> &event) -> auto { if (event->GetTypeRewrite() == TEvBlobStorage::EvControllerConfigRequest) { auto fakeId = NNodeWhiteboard::MakeNodeWhiteboardServiceId(event->Recipient.NodeId()); diff --git a/ydb/core/cms/console/configs_dispatcher_ut.cpp b/ydb/core/cms/console/configs_dispatcher_ut.cpp index 5c47cc0d4c..997d1ba451 100644 --- a/ydb/core/cms/console/configs_dispatcher_ut.cpp +++ b/ydb/core/cms/console/configs_dispatcher_ut.cpp @@ -458,7 +458,7 @@ Y_UNIT_TEST_SUITE(TConfigsDispatcherTests) { InitConfigsDispatcher(runtime); ui64 nodeConfigRequests = 0; - auto observer = [&nodeConfigRequests](TTestActorRuntimeBase&, TAutoPtr<IEventHandle> &ev) -> TTenantTestRuntime::EEventAction { + auto observer = [&nodeConfigRequests](TTestActorRuntimeBase&, TAutoPtr<IEventHandle> &ev) -> TTenantTestRuntime::EEventAction { switch (ev->GetTypeRewrite()) { case TEvConsole::EvGetNodeConfigRequest: ++nodeConfigRequests; @@ -518,7 +518,7 @@ Y_UNIT_TEST_SUITE(TConfigsDispatcherTests) { ui64 notifications = 0; TActorId subscriber; - auto observer = [¬ifications,&subscriber,recipient=runtime.Sender](TTestActorRuntimeBase&, TAutoPtr<IEventHandle> &ev) -> TTenantTestRuntime::EEventAction { + auto observer = [¬ifications,&subscriber,recipient=runtime.Sender](TTestActorRuntimeBase&, TAutoPtr<IEventHandle> &ev) -> TTenantTestRuntime::EEventAction { if (ev->Recipient == recipient && ev->Sender == subscriber) { switch (ev->GetTypeRewrite()) { case TEvPrivate::EvGotNotification: diff --git a/ydb/core/cms/console/console_ut_configs.cpp b/ydb/core/cms/console/console_ut_configs.cpp index e74e4d032e..0cb1655281 100644 --- a/ydb/core/cms/console/console_ut_configs.cpp +++ b/ydb/core/cms/console/console_ut_configs.cpp @@ -3437,7 +3437,7 @@ Y_UNIT_TEST_SUITE(TConsoleConfigSubscriptionTests) { ui32 undelivered = 0; bool attemptFinished = false; - auto countRetries = [&](TTestActorRuntimeBase&, + auto countRetries = [&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle> &event) -> auto { if (event->GetTypeRewrite() == TEvents::TSystem::Undelivered) { if (!attemptFinished) diff --git a/ydb/core/cms/console/console_ut_tenants.cpp b/ydb/core/cms/console/console_ut_tenants.cpp index 232f2c2e00..57e4820af7 100644 --- a/ydb/core/cms/console/console_ut_tenants.cpp +++ b/ydb/core/cms/console/console_ut_tenants.cpp @@ -418,7 +418,7 @@ struct CatchPoolEvent { { } - TTenantTestRuntime::EEventAction operator()(TTestActorRuntimeBase&, TAutoPtr<IEventHandle> &ev) + TTenantTestRuntime::EEventAction operator()(TTestActorRuntimeBase&, TAutoPtr<IEventHandle> &ev) { if (ev->HasEvent() && (dynamic_cast<TTenantsManager::TEvPrivate::TEvPoolAllocated*>(ev->GetBase()) diff --git a/ydb/core/cms/http.cpp b/ydb/core/cms/http.cpp index a71cbb86a4..648d864b6e 100644 --- a/ydb/core/cms/http.cpp +++ b/ydb/core/cms/http.cpp @@ -173,14 +173,14 @@ private: << ": dump# " << DumpRequest(msg->Request)); // Check for API call. - if (msg->Request.GetPathInfo().StartsWith("/api/")) { + if (msg->Request.GetPathInfo().StartsWith("/api/")) { // Check for Wall-E call. - if (msg->Request.GetPathInfo().StartsWith(WALLE_API_URL_PREFIX)) { + if (msg->Request.GetPathInfo().StartsWith(WALLE_API_URL_PREFIX)) { ctx.ExecutorThread.RegisterActor(ApiHandlers.find(WALLE_API_URL_PREFIX)->second->CreateHandlerActor(ev)); return; } - auto it = ApiHandlers.find(msg->Request.GetPathInfo()); + auto it = ApiHandlers.find(msg->Request.GetPathInfo()); if (it != ApiHandlers.end()) { ctx.ExecutorThread.RegisterActor(it->second->CreateHandlerActor(ev)); return; @@ -192,7 +192,7 @@ private: return; } - ReplyWithFile(ev, ctx, TString{msg->Request.GetPathInfo()}); + ReplyWithFile(ev, ctx, TString{msg->Request.GetPathInfo()}); } THashMap<TString, TAutoPtr<TApiMethodHandlerBase>> ApiHandlers; diff --git a/ydb/core/cms/json_proxy.h b/ydb/core/cms/json_proxy.h index 0d87bca2b7..185d0ce01b 100644 --- a/ydb/core/cms/json_proxy.h +++ b/ydb/core/cms/json_proxy.h @@ -46,7 +46,7 @@ public: void Bootstrap(const TActorContext &ctx) { LOG_DEBUG_S(ctx, NKikimrServices::CMS, - "TJsonProxyBase::Bootstrap url=" << RequestEvent->Get()->Request.GetPathInfo()); + "TJsonProxyBase::Bootstrap url=" << RequestEvent->Get()->Request.GetPathInfo()); auto dinfo = AppData(ctx)->DomainsInfo; if (dinfo->Domains.size() != 1) { @@ -202,7 +202,7 @@ public: NMon::TEvHttpInfo *msg = TBase::RequestEvent->Get(); try { - const auto &json = msg->Request.GetPostContent(); + const auto &json = msg->Request.GetPostContent(); if (json) request->Record = NProtobufJson::Json2Proto<typename TRequestEvent::ProtoRecordType>(json); } catch (yexception e) { diff --git a/ydb/core/cms/json_proxy_config_items.h b/ydb/core/cms/json_proxy_config_items.h index 45c4e6db2e..53fa0a2d9b 100644 --- a/ydb/core/cms/json_proxy_config_items.h +++ b/ydb/core/cms/json_proxy_config_items.h @@ -21,7 +21,7 @@ public: TAutoPtr<TRequest> PrepareRequest(const TActorContext &) override { TAutoPtr<TRequest> request = new TRequest; - const TCgiParameters& cgi = RequestEvent->Get()->Request.GetParams(); + const TCgiParameters& cgi = RequestEvent->Get()->Request.GetParams(); if (cgi.Has("ids")) { TVector<TString> ids = StringSplitter(cgi.Get("ids")).Split(',').ToList<TString>(); diff --git a/ydb/core/cms/json_proxy_config_updates.h b/ydb/core/cms/json_proxy_config_updates.h index c7f2beaede..1cf9b62e92 100644 --- a/ydb/core/cms/json_proxy_config_updates.h +++ b/ydb/core/cms/json_proxy_config_updates.h @@ -21,7 +21,7 @@ public: TAutoPtr<TRequest> PrepareRequest(const TActorContext &) override { TAutoPtr<TRequest> request = new TRequest; - const TCgiParameters& cgi = RequestEvent->Get()->Request.GetParams(); + const TCgiParameters& cgi = RequestEvent->Get()->Request.GetParams(); if (cgi.Has("base")) { TVector<TString> ids = StringSplitter(cgi.Get("base")).Split(',').ToList<TString>(); diff --git a/ydb/core/cms/json_proxy_log.h b/ydb/core/cms/json_proxy_log.h index 9f219077ea..7017782469 100644 --- a/ydb/core/cms/json_proxy_log.h +++ b/ydb/core/cms/json_proxy_log.h @@ -17,7 +17,7 @@ public: TAutoPtr<TRequest> PrepareRequest(const TActorContext &) override { TAutoPtr<TRequest> request = new TRequest; - const TCgiParameters& cgi = RequestEvent->Get()->Request.GetParams(); + const TCgiParameters& cgi = RequestEvent->Get()->Request.GetParams(); if (cgi.Has("limit")) { ui32 limit = 0; diff --git a/ydb/core/cms/json_proxy_proto.h b/ydb/core/cms/json_proxy_proto.h index c33553fafc..b1b87f30cd 100644 --- a/ydb/core/cms/json_proxy_proto.h +++ b/ydb/core/cms/json_proxy_proto.h @@ -38,7 +38,7 @@ public: void Bootstrap(const TActorContext &ctx) { LOG_DEBUG_S(ctx, NKikimrServices::CMS, - "TJsonProxyProto::Bootstrap url=" << RequestEvent->Get()->Request.GetPathInfo()); + "TJsonProxyProto::Bootstrap url=" << RequestEvent->Get()->Request.GetPathInfo()); ProcessRequest(ctx); Die(ctx); } @@ -47,7 +47,7 @@ protected: void ProcessRequest(const TActorContext &ctx) { - const TCgiParameters &cgi = RequestEvent->Get()->Request.GetParams(); + const TCgiParameters &cgi = RequestEvent->Get()->Request.GetParams(); if (cgi.Has("enum")) { TString name = cgi.Get("enum"); diff --git a/ydb/core/cms/json_proxy_toggle_config_validator.h b/ydb/core/cms/json_proxy_toggle_config_validator.h index 8f251aae08..fdf80b9025 100644 --- a/ydb/core/cms/json_proxy_toggle_config_validator.h +++ b/ydb/core/cms/json_proxy_toggle_config_validator.h @@ -21,7 +21,7 @@ public: TAutoPtr<TRequest> PrepareRequest(const TActorContext &) override { TAutoPtr<TRequest> request = new TRequest; - const TCgiParameters& cgi = RequestEvent->Get()->Request.GetParams(); + const TCgiParameters& cgi = RequestEvent->Get()->Request.GetParams(); if (!cgi.Has("name")) return nullptr; diff --git a/ydb/core/cms/walle_api_handler.cpp b/ydb/core/cms/walle_api_handler.cpp index 2c31f4f985..7b62f3275a 100644 --- a/ydb/core/cms/walle_api_handler.cpp +++ b/ydb/core/cms/walle_api_handler.cpp @@ -35,21 +35,21 @@ public: { NMon::TEvHttpInfo *msg = RequestEvent->Get(); - TString path {msg->Request.GetPathInfo().substr(strlen(WALLE_API_URL_PREFIX))}; + TString path {msg->Request.GetPathInfo().substr(strlen(WALLE_API_URL_PREFIX))}; if (path == "tasks") { - if (msg->Request.GetMethod() == HTTP_METHOD_GET) + if (msg->Request.GetMethod() == HTTP_METHOD_GET) ProcessListTasksRequest(ctx); - else if (msg->Request.GetMethod() == HTTP_METHOD_POST) - ProcessCreateTasksRequest(msg->Request, ctx); + else if (msg->Request.GetMethod() == HTTP_METHOD_POST) + ProcessCreateTasksRequest(msg->Request, ctx); else ReplyWithError("HTTP/1.1 405 Method Not Allowed\r\n\r\n", ctx); } else if (path.StartsWith("tasks/")) { TString id = path.substr(strlen("tasks/")); - if (msg->Request.GetMethod() == HTTP_METHOD_GET) + if (msg->Request.GetMethod() == HTTP_METHOD_GET) ProcessCheckTaskRequest(id, ctx); - else if (msg->Request.GetMethod() == HTTP_METHOD_DELETE) + else if (msg->Request.GetMethod() == HTTP_METHOD_DELETE) ProcessRemoveTaskRequest(id, ctx); else ReplyWithError("HTTP/1.1 405 Method Not Allowed\r\n\r\n", ctx); @@ -62,7 +62,7 @@ private: ////////////////////////////////////////////// // Create task ////////////////////////////////////////////// - void ProcessCreateTasksRequest(const NMonitoring::IMonHttpRequest &http, const TActorContext &ctx) + void ProcessCreateTasksRequest(const NMonitoring::IMonHttpRequest &http, const TActorContext &ctx) { NJson::TJsonValue json; diff --git a/ydb/core/control/immediate_control_board_actor.cpp b/ydb/core/control/immediate_control_board_actor.cpp index 10a6845574..bcb4a3344c 100644 --- a/ydb/core/control/immediate_control_board_actor.cpp +++ b/ydb/core/control/immediate_control_board_actor.cpp @@ -89,9 +89,9 @@ private: } void Handle(NMon::TEvHttpInfo::TPtr &ev, const TActorContext &ctx) { - HTTP_METHOD method = ev->Get()->Request.GetMethod(); + HTTP_METHOD method = ev->Get()->Request.GetMethod(); if (method == HTTP_METHOD_POST) { - HandlePostParams(ev->Get()->Request.GetPostParams()); + HandlePostParams(ev->Get()->Request.GetPostParams()); } TStringStream str; str << Board->RenderAsHtml(); diff --git a/ydb/core/driver_lib/cli_base/cli_cmds_db.cpp b/ydb/core/driver_lib/cli_base/cli_cmds_db.cpp index 7c72fa4aa4..450d4135c6 100644 --- a/ydb/core/driver_lib/cli_base/cli_cmds_db.cpp +++ b/ydb/core/driver_lib/cli_base/cli_cmds_db.cpp @@ -853,14 +853,14 @@ public: return -2; } - NGrpc::TCallMeta meta; + NGrpc::TCallMeta meta; if (config.SecurityToken) { meta.Aux.push_back({NYdb::YDB_AUTH_TICKET_HEADER, config.SecurityToken}); } Ydb::Operations::Operation response; - NGrpc::TResponseCallback<Ydb::Table::DescribeTableOptionsResponse> responseCb = - [&res, &response](NGrpc::TGrpcStatus &&grpcStatus, Ydb::Table::DescribeTableOptionsResponse &&resp) -> void { + NGrpc::TResponseCallback<Ydb::Table::DescribeTableOptionsResponse> responseCb = + [&res, &response](NGrpc::TGrpcStatus &&grpcStatus, Ydb::Table::DescribeTableOptionsResponse &&resp) -> void { res = (int)grpcStatus.GRpcStatusCode; if (!res) { response.CopyFrom(resp.operation()); @@ -870,7 +870,7 @@ public: }; { - NGrpc::TGRpcClientLow clientLow; + NGrpc::TGRpcClientLow clientLow; Ydb::Table::DescribeTableOptionsRequest request; auto connection = clientLow.CreateGRpcServiceConnection<Ydb::Table::V1::TableService>(ClientConfig); connection->DoRequest(request, std::move(responseCb), &Ydb::Table::V1::TableService::Stub::AsyncDescribeTableOptions, meta); diff --git a/ydb/core/driver_lib/cli_base/cli_grpc.h b/ydb/core/driver_lib/cli_base/cli_grpc.h index c0b84fb511..61d09c9f70 100644 --- a/ydb/core/driver_lib/cli_base/cli_grpc.h +++ b/ydb/core/driver_lib/cli_base/cli_grpc.h @@ -27,13 +27,13 @@ int DoGRpcRequest(const NGRpcProxy::TGRpcClientConfig &clientConfig, return -2; } - NGrpc::TCallMeta meta; + NGrpc::TCallMeta meta; if (securityToken) { meta.Aux.push_back({NYdb::YDB_AUTH_TICKET_HEADER, securityToken}); } - NGrpc::TResponseCallback<TResponse> responseCb = - [&res, &response](NGrpc::TGrpcStatus &&grpcStatus, TResponse &&resp) -> void { + NGrpc::TResponseCallback<TResponse> responseCb = + [&res, &response](NGrpc::TGrpcStatus &&grpcStatus, TResponse &&resp) -> void { res = (int)grpcStatus.GRpcStatusCode; if (!res) { response.CopyFrom(resp.operation()); @@ -43,13 +43,13 @@ int DoGRpcRequest(const NGRpcProxy::TGRpcClientConfig &clientConfig, }; { - NGrpc::TGRpcClientLow clientLow; + NGrpc::TGRpcClientLow clientLow; auto connection = clientLow.CreateGRpcServiceConnection<TService>(clientConfig); connection->DoRequest(request, responseCb, function, meta); } - NGrpc::TResponseCallback<Ydb::Operations::GetOperationResponse> operationCb = - [&res, &response](NGrpc::TGrpcStatus &&grpcStatus, Ydb::Operations::GetOperationResponse &&resp) -> void { + NGrpc::TResponseCallback<Ydb::Operations::GetOperationResponse> operationCb = + [&res, &response](NGrpc::TGrpcStatus &&grpcStatus, Ydb::Operations::GetOperationResponse &&resp) -> void { res = (int)grpcStatus.GRpcStatusCode; if (!res) { response.CopyFrom(resp.operation()); @@ -60,7 +60,7 @@ int DoGRpcRequest(const NGRpcProxy::TGRpcClientConfig &clientConfig, while (!res && !response.ready()) { Sleep(TDuration::MilliSeconds(100)); - NGrpc::TGRpcClientLow clientLow; + NGrpc::TGRpcClientLow clientLow; auto connection = clientLow.CreateGRpcServiceConnection<Ydb::Operation::V1::OperationService>(clientConfig); Ydb::Operations::GetOperationRequest request; request.set_id(response.id()); diff --git a/ydb/core/keyvalue/keyvalue_ut.cpp b/ydb/core/keyvalue/keyvalue_ut.cpp index 1cea231e79..4ae1fcf56a 100644 --- a/ydb/core/keyvalue/keyvalue_ut.cpp +++ b/ydb/core/keyvalue/keyvalue_ut.cpp @@ -39,12 +39,12 @@ public: TTestActorRuntime::TEventFilter Prepare() { IsDone = false; - return [&](TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event) { + return [&](TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event) { return (*this)(runtime, event); }; } - bool operator()(TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event) { + bool operator()(TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event) { Y_UNUSED(runtime); Y_UNUSED(event); return false; diff --git a/ydb/core/mind/bscontroller/ut_bscontroller/main.cpp b/ydb/core/mind/bscontroller/ut_bscontroller/main.cpp index 8d0e41c799..87f7b18fb3 100644 --- a/ydb/core/mind/bscontroller/ut_bscontroller/main.cpp +++ b/ydb/core/mind/bscontroller/ut_bscontroller/main.cpp @@ -27,7 +27,7 @@ using namespace NKikimr::NBsController; class TInitialEventsFilter : TNonCopyable { public: TTestActorRuntime::TEventFilter Prepare() { - return [](TTestActorRuntimeBase& /*runtime*/, TAutoPtr<IEventHandle>& /*event*/) { + return [](TTestActorRuntimeBase& /*runtime*/, TAutoPtr<IEventHandle>& /*event*/) { return false; }; } diff --git a/ydb/core/mind/hive/hive_ut.cpp b/ydb/core/mind/hive/hive_ut.cpp index 53594a5b07..0383d49a12 100644 --- a/ydb/core/mind/hive/hive_ut.cpp +++ b/ydb/core/mind/hive/hive_ut.cpp @@ -368,12 +368,12 @@ namespace { TTestActorRuntime::TEventFilter Prepare() { IsDone = false; - return [&](TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event) { + return [&](TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event) { return (*this)(runtime, event); }; } - bool operator()(TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event) { + bool operator()(TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event) { Y_UNUSED(runtime); if (event->GetTypeRewrite() == TEvHive::EvCreateTablet) { IsDone = true; @@ -535,7 +535,7 @@ Y_UNIT_TEST_SUITE(THiveTest) { NKikimrProto::EReplyStatus expectedStatus = NKikimrProto::OK) { bool seenEvDeleteTabletResult = false; TTestActorRuntime::TEventObserver prevObserverFunc; - prevObserverFunc = runtime.SetObserverFunc([&](TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event) { + prevObserverFunc = runtime.SetObserverFunc([&](TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event) { if (event->GetTypeRewrite() == TEvTabletBase::EvDeleteTabletResult) { seenEvDeleteTabletResult = true; } @@ -3909,7 +3909,7 @@ Y_UNIT_TEST_SUITE(THiveTest) { CreateTestBootstrapper(runtime, CreateTestTabletInfo(hiveTablet, TTabletTypes::FLAT_HIVE), &CreateDefaultHive); // Block group assignment - runtime.SetObserverFunc([](TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event) { + runtime.SetObserverFunc([](TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event) { if (event->GetTypeRewrite() == TEvBlobStorage::EvControllerSelectGroups) { return TTestActorRuntime::EEventAction::DROP; } @@ -4135,7 +4135,7 @@ Y_UNIT_TEST_SUITE(THiveTest) { // Setup observer that would drop EvDeleteTabletResult messages TTestActorRuntime::TEventObserver prevObserverFunc; - prevObserverFunc = runtime.SetObserverFunc([&](TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event) { + prevObserverFunc = runtime.SetObserverFunc([&](TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event) { if (event->GetTypeRewrite() == TEvTabletBase::EvDeleteTabletResult) { return TTestActorRuntime::EEventAction::DROP; } diff --git a/ydb/core/mind/node_broker_ut.cpp b/ydb/core/mind/node_broker_ut.cpp index 5a833d0d93..0d9498ddcd 100644 --- a/ydb/core/mind/node_broker_ut.cpp +++ b/ydb/core/mind/node_broker_ut.cpp @@ -203,11 +203,11 @@ void Setup(TTestActorRuntime& runtime, TMallocInfo mallocInfo = MallocInfo(); mallocInfo.SetParam("FillMemoryOnAllocation", "false"); - auto scheduledFilter = [](TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event, TDuration delay, TInstant& deadline) { + auto scheduledFilter = [](TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event, TDuration delay, TInstant& deadline) { if (event->HasEvent() && (dynamic_cast<TDynamicNameserver::TEvPrivate::TEvUpdateEpoch*>(event->GetBase()))) return false; - return TTestActorRuntime::DefaultScheduledFilterFunc(runtime, event, delay, deadline); + return TTestActorRuntime::DefaultScheduledFilterFunc(runtime, event, delay, deadline); }; runtime.SetScheduledEventFilter(scheduledFilter); @@ -1130,7 +1130,7 @@ Y_UNIT_TEST_SUITE(TDynamicNameserverTest) { TVector<NKikimrNodeBroker::TListNodes> listRequests; TVector<NKikimrNodeBroker::TResolveNode> resolveRequests; - auto logRequests = [&](TTestActorRuntimeBase &, TAutoPtr<IEventHandle> &event) -> auto { + auto logRequests = [&](TTestActorRuntimeBase &, TAutoPtr<IEventHandle> &event) -> auto { if (event->GetTypeRewrite() == TEvNodeBroker::EvListNodes) listRequests.push_back(event->Get<TEvNodeBroker::TEvListNodes>()->Record); else if (event->GetTypeRewrite() == TEvNodeBroker::EvResolveNode) diff --git a/ydb/core/mind/tenant_pool.cpp b/ydb/core/mind/tenant_pool.cpp index 5ab98d9e0d..80236570b9 100644 --- a/ydb/core/mind/tenant_pool.cpp +++ b/ydb/core/mind/tenant_pool.cpp @@ -817,7 +817,7 @@ public: { NMon::TEvHttpInfo *msg = ev->Get(); // Check for index page. - if (msg->Request.GetPathInfo() == "" || msg->Request.GetPathInfo() == "/") { + if (msg->Request.GetPathInfo() == "" || msg->Request.GetPathInfo() == "/") { TStringStream str; if (!DomainTenantPools.size()) { str << "No domains are handled"; @@ -832,8 +832,8 @@ public: } ctx.Send(ev->Sender, new NMon::TEvHttpInfoRes(str.Str(), 0, NMon::IEvHttpInfoRes::EContentType::Html)); - } else if (msg->Request.GetPathInfo().StartsWith(DomainPrefix)) { - TString domain{msg->Request.GetPathInfo().substr(DomainPrefix.size())}; + } else if (msg->Request.GetPathInfo().StartsWith(DomainPrefix)) { + TString domain{msg->Request.GetPathInfo().substr(DomainPrefix.size())}; if (!DomainTenantPools.contains(domain)) { ctx.Send(ev->Sender, new NMon::TEvHttpInfoRes("Unknown domain " + domain, 0, NMon::IEvHttpInfoRes::EContentType::Html)); diff --git a/ydb/core/mind/tenant_ut_broker.cpp b/ydb/core/mind/tenant_ut_broker.cpp index 26a327c0cb..4c124b3d83 100644 --- a/ydb/core/mind/tenant_ut_broker.cpp +++ b/ydb/core/mind/tenant_ut_broker.cpp @@ -1017,7 +1017,7 @@ Y_UNIT_TEST_SUITE(TTenantSlotBrokerTests) { runtime.CreateTenantPool(0, AssignedTenantPoolConfig()); TAutoPtr<IEventHandle> captured = nullptr; - auto captureRegister = [&captured](TTestActorRuntimeBase&, TAutoPtr<IEventHandle> &event) -> auto { + auto captureRegister = [&captured](TTestActorRuntimeBase&, TAutoPtr<IEventHandle> &event) -> auto { if (event->GetTypeRewrite() == TEvTenantSlotBroker::EvRegisterPool && !captured) { captured = event; diff --git a/ydb/core/mon/mon.cpp b/ydb/core/mon/mon.cpp index 5266cba5a1..d4722266f3 100644 --- a/ydb/core/mon/mon.cpp +++ b/ydb/core/mon/mon.cpp @@ -217,7 +217,7 @@ namespace NActors { protected: TActorId TargetActorId; - IMonHttpRequest& Request; + IMonHttpRequest& Request; NThreading::TPromise<THttpResponsePtr> Result; const TVector<TString> &AllowedSIDs; TMon::TRequestAuthorizer Authorizer; @@ -286,13 +286,13 @@ namespace NActors { out << "</body>"; } - void OutputContent(NMonitoring::IMonHttpRequest &request) override { + void OutputContent(NMonitoring::IMonHttpRequest &request) override { if (PreTag) { - request.Output() << "<pre>\n"; + request.Output() << "<pre>\n"; } - Result.Output(request.Output()); + Result.Output(request.Output()); if (PreTag) { - request.Output() << "</pre>\n"; + request.Output() << "</pre>\n"; } } @@ -342,7 +342,7 @@ namespace NActors { { } - void Output(IMonHttpRequest &request) override { + void Output(IMonHttpRequest &request) override { auto promise = NThreading::NewPromise<THttpResponsePtr>(); auto future = promise.GetFuture(); @@ -361,13 +361,13 @@ namespace NActors { } private: - void Output(IMonHttpRequest &request, const NMon::IEvHttpInfoRes &result) const { + void Output(IMonHttpRequest &request, const NMon::IEvHttpInfoRes &result) const { if (result.GetContentType() == NMon::IEvHttpInfoRes::Html) { THtmlResultMonPage resultPage(Path, Title, Host, PreTag, result); resultPage.Parent = this->Parent; resultPage.Output(request); } else { - result.Output(request.Output()); + result.Output(request.Output()); } } diff --git a/ydb/core/persqueue/pq_l2_cache.cpp b/ydb/core/persqueue/pq_l2_cache.cpp index de256225e3..e56642c39a 100644 --- a/ydb/core/persqueue/pq_l2_cache.cpp +++ b/ydb/core/persqueue/pq_l2_cache.cpp @@ -198,7 +198,7 @@ void TPersQueueCacheL2::RegretBlobs(const TActorContext& ctx, TString topic, con void TPersQueueCacheL2::Handle(NMon::TEvHttpInfo::TPtr& ev, const TActorContext& ctx) { - const auto& params = ev->Get()->Request.GetParams(); + const auto& params = ev->Get()->Request.GetParams(); if (params.Has("submit")) { TString strParam = params.Get("newCacheLimit"); if (strParam.size()) { diff --git a/ydb/core/security/ticket_parser.cpp b/ydb/core/security/ticket_parser.cpp index 87569953e3..c7d024d03d 100644 --- a/ydb/core/security/ticket_parser.cpp +++ b/ydb/core/security/ticket_parser.cpp @@ -426,7 +426,7 @@ class TTicketParser : public TActorBootstrapped<TTicketParser> { } void Handle(NMon::TEvHttpInfo::TPtr& ev, const TActorContext& ctx) { - const auto& params = ev->Get()->Request.GetParams(); + const auto& params = ev->Get()->Request.GetParams(); TStringBuilder html; if (params.Has("token")) { TString token = params.Get("token"); diff --git a/ydb/core/tablet/node_tablet_monitor.cpp b/ydb/core/tablet/node_tablet_monitor.cpp index 2876bd5fd2..8bb34e7ea0 100644 --- a/ydb/core/tablet/node_tablet_monitor.cpp +++ b/ydb/core/tablet/node_tablet_monitor.cpp @@ -445,7 +445,7 @@ private: void Handle(NMon::TEvHttpInfo::TPtr &ev, const TActorContext &ctx) { NMon::TEvHttpInfo* msg = ev->Get(); - const TCgiParameters& cgi = msg->Request.GetParams(); + const TCgiParameters& cgi = msg->Request.GetParams(); if (cgi.Has("action")) { const TString &actionParam = cgi.Get("action"); if (actionParam == "browse_nodes") { diff --git a/ydb/core/tablet/tablet_counters_aggregator.cpp b/ydb/core/tablet/tablet_counters_aggregator.cpp index 4cafc4f7a7..1885b26e09 100644 --- a/ydb/core/tablet/tablet_counters_aggregator.cpp +++ b/ydb/core/tablet/tablet_counters_aggregator.cpp @@ -1684,7 +1684,7 @@ TTabletCountersAggregatorActor::HandleWork(TEvTabletCounters::TEvTabletLabeledCo TString html; TStringOutput oss(html); NMonitoring::TDynamicCounters counters; - const auto& params = it->second.second->Request.GetParams(); + const auto& params = it->second.second->Request.GetParams(); TString reqTabletType = params.Get("type"); auto mainGroup = counters.GetSubgroup("user_counters", reqTabletType); @@ -1795,9 +1795,9 @@ void TTabletCountersAggregatorActor::HandleWork(TEvTabletCounters::TEvRemoveData void TTabletCountersAggregatorActor::HandleWork(NMon::TEvHttpInfo::TPtr &ev, const TActorContext &ctx) { - TString reqTabletType = ev->Get()->Request.GetParams().Get("type"); + TString reqTabletType = ev->Get()->Request.GetParams().Get("type"); ui32 workers = 0; - TryFromString(ev->Get()->Request.GetParams().Get("workers"), workers); + TryFromString(ev->Get()->Request.GetParams().Get("workers"), workers); for (ui32 tabletType = 0; tabletType < TTabletTypes::USER_TYPE_START; ++tabletType) { if (!NKikimrTabletBase::TTabletTypes::EType_IsValid(tabletType)) continue; diff --git a/ydb/core/tablet/tablet_monitoring_proxy.cpp b/ydb/core/tablet/tablet_monitoring_proxy.cpp index ac92abc30a..a61a43b7e5 100644 --- a/ydb/core/tablet/tablet_monitoring_proxy.cpp +++ b/ydb/core/tablet/tablet_monitoring_proxy.cpp @@ -11,7 +11,7 @@ #include <ydb/core/base/tablet.h> #include <ydb/core/tx/tx.h> #include <library/cpp/monlib/service/pages/templates.h> -#include <util/string/builder.h> +#include <util/string/builder.h> //////////////////////////////////////////// namespace NKikimr { namespace NTabletMonitoringProxy { @@ -207,7 +207,7 @@ TTabletMonitoringProxyActor::Handle(NMon::TEvHttpInfo::TPtr &ev, const TActorCon const ui64 tabletId = TryParseTabletId(tabletIdParam); if (tabletId) { TString url = TStringBuilder() << msg->Request.GetPathInfo() << "?" << cgi->Print(); - ctx.ExecutorThread.RegisterActor(new TForwardingActor(Config, tabletId, true, ev->Sender, std::move(url))); + ctx.ExecutorThread.RegisterActor(new TForwardingActor(Config, tabletId, true, ev->Sender, std::move(url))); return; } } @@ -228,7 +228,7 @@ TTabletMonitoringProxyActor::Handle(NMon::TEvHttpInfo::TPtr &ev, const TActorCon const ui64 tabletId = TryParseTabletId(ssIdParam); if (tabletId) { TString url = TStringBuilder() << msg->Request.GetPathInfo() << "?" << cgi->Print(); - ctx.ExecutorThread.RegisterActor(CreateStateStorageMonitoringActor(tabletId, ev->Sender, std::move(url))); + ctx.ExecutorThread.RegisterActor(CreateStateStorageMonitoringActor(tabletId, ev->Sender, std::move(url))); return; } } diff --git a/ydb/core/tablet/tablet_pipe_ut.cpp b/ydb/core/tablet/tablet_pipe_ut.cpp index e3d02b2542..2a528070fb 100644 --- a/ydb/core/tablet/tablet_pipe_ut.cpp +++ b/ydb/core/tablet/tablet_pipe_ut.cpp @@ -494,7 +494,7 @@ Y_UNIT_TEST_SUITE(TTabletPipeTest) { // We want to close the client right after it has sent EvConnect to the target tablet but before // the client received the EvConnectResult - runtime.SetObserverFunc([clientId, sender](TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event) { + runtime.SetObserverFunc([clientId, sender](TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event) { if (event->Type == TEvTabletPipe::EvConnect) { runtime.Send(new IEventHandle(clientId, sender, new TEvents::TEvPoisonPill()), 0); } diff --git a/ydb/core/testlib/actors/test_runtime.cpp b/ydb/core/testlib/actors/test_runtime.cpp index acc4d36e1e..63f0436bff 100644 --- a/ydb/core/testlib/actors/test_runtime.cpp +++ b/ydb/core/testlib/actors/test_runtime.cpp @@ -1,19 +1,19 @@ -#include "test_runtime.h" - +#include "test_runtime.h" + #include <ydb/core/base/appdata.h> #include <ydb/core/base/blobstorage.h> #include <ydb/core/base/counters.h> #include <ydb/core/mon/mon.h> #include <ydb/core/mon_alloc/profiler.h> #include <ydb/core/tablet/tablet_impl.h> - + #include <library/cpp/actors/core/executor_pool_basic.h> #include <library/cpp/actors/core/executor_pool_io.h> - - -/**** ACHTUNG: Do not make here any new dependecies on kikimr ****/ - -namespace NActors { + + +/**** ACHTUNG: Do not make here any new dependecies on kikimr ****/ + +namespace NActors { void TTestActorRuntime::TNodeData::Stop() { if (Mon) { @@ -22,112 +22,112 @@ namespace NActors { TNodeDataBase::Stop(); } - TTestActorRuntime::TNodeData::~TNodeData() { + TTestActorRuntime::TNodeData::~TNodeData() { Stop(); - } - - ui64 TTestActorRuntime::TNodeData::GetLoggerPoolId() const { - return GetAppData<NKikimr::TAppData>()->IOPoolId; - } - - void TTestActorRuntime::Initialize() { - SetScheduledEventFilter(&TTestActorRuntime::DefaultScheduledFilterFunc); + } + + ui64 TTestActorRuntime::TNodeData::GetLoggerPoolId() const { + return GetAppData<NKikimr::TAppData>()->IOPoolId; + } + + void TTestActorRuntime::Initialize() { + SetScheduledEventFilter(&TTestActorRuntime::DefaultScheduledFilterFunc); NodeFactory = MakeHolder<TNodeFactory>(); - InitNodes(); - } - - TTestActorRuntime::TTestActorRuntime(THeSingleSystemEnv d) + InitNodes(); + } + + TTestActorRuntime::TTestActorRuntime(THeSingleSystemEnv d) : TPortManager(false) , TTestActorRuntimeBase{d} - { - /* How it is possible to do initilization without these components? */ - NKikimr::TAppData::RandomProvider = RandomProvider; - NKikimr::TAppData::TimeProvider = TimeProvider; - - Initialize(); - } - - TTestActorRuntime::TTestActorRuntime(ui32 nodeCount, ui32 dataCenterCount, bool useRealThreads) + { + /* How it is possible to do initilization without these components? */ + NKikimr::TAppData::RandomProvider = RandomProvider; + NKikimr::TAppData::TimeProvider = TimeProvider; + + Initialize(); + } + + TTestActorRuntime::TTestActorRuntime(ui32 nodeCount, ui32 dataCenterCount, bool useRealThreads) : TPortManager(false) , TTestActorRuntimeBase{nodeCount, dataCenterCount, useRealThreads} - { - Initialize(); - } - - TTestActorRuntime::TTestActorRuntime(ui32 nodeCount, ui32 dataCenterCount) + { + Initialize(); + } + + TTestActorRuntime::TTestActorRuntime(ui32 nodeCount, ui32 dataCenterCount) : TPortManager(false) , TTestActorRuntimeBase{nodeCount, dataCenterCount} - { - Initialize(); - } - - TTestActorRuntime::TTestActorRuntime(ui32 nodeCount, bool useRealThreads) + { + Initialize(); + } + + TTestActorRuntime::TTestActorRuntime(ui32 nodeCount, bool useRealThreads) : TPortManager(false) , TTestActorRuntimeBase{nodeCount, useRealThreads} - { - Initialize(); - } - - TTestActorRuntime::~TTestActorRuntime() { - if (!UseRealThreads) { - NKikimr::TAppData::RandomProvider = CreateDefaultRandomProvider(); - NKikimr::TAppData::TimeProvider = CreateDefaultTimeProvider(); - } - - SetObserverFunc(&TTestActorRuntimeBase::DefaultObserverFunc); - SetScheduledEventsSelectorFunc(&CollapsedTimeScheduledEventsSelector); - SetEventFilter(&TTestActorRuntimeBase::DefaultFilterFunc); - SetScheduledEventFilter(&TTestActorRuntimeBase::NopFilterFunc); - SetRegistrationObserverFunc(&TTestActorRuntimeBase::DefaultRegistrationObserver); - - CleanupNodes(); - } - - void TTestActorRuntime::Initialize(TEgg egg) { - IsInitialized = true; - - Opaque = std::move(egg.Opaque); - App0.Reset(egg.App0); - KeyConfigGenerator = std::move(egg.KeyConfigGenerator); - - if (!UseRealThreads) { - NKikimr::TAppData::RandomProvider = RandomProvider; - NKikimr::TAppData::TimeProvider = TimeProvider; - } - + { + Initialize(); + } + + TTestActorRuntime::~TTestActorRuntime() { + if (!UseRealThreads) { + NKikimr::TAppData::RandomProvider = CreateDefaultRandomProvider(); + NKikimr::TAppData::TimeProvider = CreateDefaultTimeProvider(); + } + + SetObserverFunc(&TTestActorRuntimeBase::DefaultObserverFunc); + SetScheduledEventsSelectorFunc(&CollapsedTimeScheduledEventsSelector); + SetEventFilter(&TTestActorRuntimeBase::DefaultFilterFunc); + SetScheduledEventFilter(&TTestActorRuntimeBase::NopFilterFunc); + SetRegistrationObserverFunc(&TTestActorRuntimeBase::DefaultRegistrationObserver); + + CleanupNodes(); + } + + void TTestActorRuntime::Initialize(TEgg egg) { + IsInitialized = true; + + Opaque = std::move(egg.Opaque); + App0.Reset(egg.App0); + KeyConfigGenerator = std::move(egg.KeyConfigGenerator); + + if (!UseRealThreads) { + NKikimr::TAppData::RandomProvider = RandomProvider; + NKikimr::TAppData::TimeProvider = TimeProvider; + } + MonPorts.clear(); - for (ui32 nodeIndex = 0; nodeIndex < NodeCount; ++nodeIndex) { + for (ui32 nodeIndex = 0; nodeIndex < NodeCount; ++nodeIndex) { ui32 nodeId = FirstNodeId + nodeIndex; auto* node = GetNodeById(nodeId); - const auto* app0 = App0.Get(); - if (!SingleSysEnv) { - const TIntrusivePtr<NMonitoring::TDynamicCounters> profilerCounters = NKikimr::GetServiceCounters(node->DynamicCounters, "utils"); + const auto* app0 = App0.Get(); + if (!SingleSysEnv) { + const TIntrusivePtr<NMonitoring::TDynamicCounters> profilerCounters = NKikimr::GetServiceCounters(node->DynamicCounters, "utils"); TActorSetupCmd profilerSetup(CreateProfilerActor(profilerCounters, "."), TMailboxType::Simple, 0); node->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>(MakeProfilerID(FirstNodeId + nodeIndex), profilerSetup)); - } - - if (!UseRealThreads) { - node->AppData0.reset(new NKikimr::TAppData(0, 0, 0, 0, { }, App0->TypeRegistry, App0->FunctionRegistry, App0->FormatFactory, nullptr)); - node->SchedulerPool.Reset(CreateExecutorPoolStub(this, nodeIndex, node, 0)); - node->MailboxTable.Reset(new TMailboxTable()); - node->ActorSystem = MakeActorSystem(nodeIndex, node); + } + + if (!UseRealThreads) { + node->AppData0.reset(new NKikimr::TAppData(0, 0, 0, 0, { }, App0->TypeRegistry, App0->FunctionRegistry, App0->FormatFactory, nullptr)); + node->SchedulerPool.Reset(CreateExecutorPoolStub(this, nodeIndex, node, 0)); + node->MailboxTable.Reset(new TMailboxTable()); + node->ActorSystem = MakeActorSystem(nodeIndex, node); node->ExecutorThread.Reset(new TExecutorThread(0, 0, node->ActorSystem.Get(), node->SchedulerPool.Get(), node->MailboxTable.Get(), "TestExecutor")); - } else { - node->AppData0.reset(new NKikimr::TAppData(0, 1, 2, 3, { }, app0->TypeRegistry, app0->FunctionRegistry, app0->FormatFactory, nullptr)); - node->ActorSystem = MakeActorSystem(nodeIndex, node); - } + } else { + node->AppData0.reset(new NKikimr::TAppData(0, 1, 2, 3, { }, app0->TypeRegistry, app0->FunctionRegistry, app0->FormatFactory, nullptr)); + node->ActorSystem = MakeActorSystem(nodeIndex, node); + } node->LogSettings->MessagePrefix = " node " + ToString(nodeId); - - auto* nodeAppData = node->GetAppData<NKikimr::TAppData>(); + + auto* nodeAppData = node->GetAppData<NKikimr::TAppData>(); nodeAppData->DataShardExportFactory = app0->DataShardExportFactory; - nodeAppData->DomainsInfo = app0->DomainsInfo; - nodeAppData->ChannelProfiles = app0->ChannelProfiles; - nodeAppData->Counters = node->DynamicCounters; - nodeAppData->PollerThreads = node->Poller; - nodeAppData->StreamingConfig.SetEnableOutputStreams(true); + nodeAppData->DomainsInfo = app0->DomainsInfo; + nodeAppData->ChannelProfiles = app0->ChannelProfiles; + nodeAppData->Counters = node->DynamicCounters; + nodeAppData->PollerThreads = node->Poller; + nodeAppData->StreamingConfig.SetEnableOutputStreams(true); nodeAppData->PQConfig = app0->PQConfig; nodeAppData->NetClassifierConfig.CopyFrom(app0->NetClassifierConfig); - nodeAppData->StaticBlobStorageConfig->CopyFrom(*app0->StaticBlobStorageConfig); + nodeAppData->StaticBlobStorageConfig->CopyFrom(*app0->StaticBlobStorageConfig); nodeAppData->EnableKqpSpilling = app0->EnableKqpSpilling; nodeAppData->FeatureFlags = app0->FeatureFlags; nodeAppData->CompactionConfig = app0->CompactionConfig; @@ -136,12 +136,12 @@ namespace NActors { nodeAppData->MeteringConfig = app0->MeteringConfig; nodeAppData->EnableMvccSnapshotWithLegacyDomainRoot = app0->EnableMvccSnapshotWithLegacyDomainRoot; nodeAppData->IoContextFactory = app0->IoContextFactory; - if (KeyConfigGenerator) { - nodeAppData->KeyConfig = KeyConfigGenerator(nodeIndex); - } else { - nodeAppData->KeyConfig.CopyFrom(app0->KeyConfig); - } - + if (KeyConfigGenerator) { + nodeAppData->KeyConfig = KeyConfigGenerator(nodeIndex); + } else { + nodeAppData->KeyConfig.CopyFrom(app0->KeyConfig); + } + if (NeedMonitoring && !SingleSysEnv) { ui16 port = GetPortManager().GetPort(); node->Mon.Reset(new NActors::TMon({ @@ -149,101 +149,101 @@ namespace NActors { .Threads = 10, .Title = "KIKIMR monitoring" })); - nodeAppData->Mon = node->Mon.Get(); - node->Mon->RegisterCountersPage("counters", "Counters", node->DynamicCounters); - auto actorsMonPage = node->Mon->RegisterIndexPage("actors", "Actors"); - node->Mon->RegisterActorPage(actorsMonPage, "profiler", "Profiler", false, node->ActorSystem.Get(), MakeProfilerID(FirstNodeId + nodeIndex)); + nodeAppData->Mon = node->Mon.Get(); + node->Mon->RegisterCountersPage("counters", "Counters", node->DynamicCounters); + auto actorsMonPage = node->Mon->RegisterIndexPage("actors", "Actors"); + node->Mon->RegisterActorPage(actorsMonPage, "profiler", "Profiler", false, node->ActorSystem.Get(), MakeProfilerID(FirstNodeId + nodeIndex)); const NActors::TActorId loggerActorId = NActors::TActorId(FirstNodeId + nodeIndex, "logger"); - node->Mon->RegisterActorPage(actorsMonPage, "logger", "Logger", false, node->ActorSystem.Get(), loggerActorId); + node->Mon->RegisterActorPage(actorsMonPage, "logger", "Logger", false, node->ActorSystem.Get(), loggerActorId); MonPorts.push_back(port); - } - - node->ActorSystem->Start(); - if (nodeAppData->Mon) { - nodeAppData->Mon->Start(); - } - } - } - + } + + node->ActorSystem->Start(); + if (nodeAppData->Mon) { + nodeAppData->Mon->Start(); + } + } + } + ui16 TTestActorRuntime::GetMonPort(ui32 nodeIndex) const { Y_VERIFY(nodeIndex < MonPorts.size(), "Unknown MonPort for nodeIndex = %" PRIu32, nodeIndex); return MonPorts[nodeIndex]; } - void TTestActorRuntime::InitActorSystemSetup(TActorSystemSetup& setup) { + void TTestActorRuntime::InitActorSystemSetup(TActorSystemSetup& setup) { setup.MaxActivityType = GetActivityTypeCount(); - } - - NKikimr::TAppData& TTestActorRuntime::GetAppData(ui32 nodeIndex) { - TGuard<TMutex> guard(Mutex); - Y_VERIFY(nodeIndex < NodeCount); - ui32 nodeId = FirstNodeId + nodeIndex; - auto* node = GetNodeById(nodeId); - return *node->GetAppData<NKikimr::TAppData>(); - } - - bool TTestActorRuntime::DefaultScheduledFilterFunc(TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event, TDuration delay, TInstant& deadline) { - Y_UNUSED(delay); - Y_UNUSED(deadline); - - switch (event->GetTypeRewrite()) { - case NKikimr::TEvBlobStorage::EvConfigureQueryTimeout: - case NKikimr::TEvBlobStorage::EvEstablishingSessionTimeout: - return true; - case NKikimr::TEvBlobStorage::EvNotReadyRetryTimeout: - case NKikimr::TEvTabletPipe::EvClientRetry: + } + + NKikimr::TAppData& TTestActorRuntime::GetAppData(ui32 nodeIndex) { + TGuard<TMutex> guard(Mutex); + Y_VERIFY(nodeIndex < NodeCount); + ui32 nodeId = FirstNodeId + nodeIndex; + auto* node = GetNodeById(nodeId); + return *node->GetAppData<NKikimr::TAppData>(); + } + + bool TTestActorRuntime::DefaultScheduledFilterFunc(TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event, TDuration delay, TInstant& deadline) { + Y_UNUSED(delay); + Y_UNUSED(deadline); + + switch (event->GetTypeRewrite()) { + case NKikimr::TEvBlobStorage::EvConfigureQueryTimeout: + case NKikimr::TEvBlobStorage::EvEstablishingSessionTimeout: + return true; + case NKikimr::TEvBlobStorage::EvNotReadyRetryTimeout: + case NKikimr::TEvTabletPipe::EvClientRetry: case NKikimr::TEvTabletBase::EvFollowerRetry: case NKikimr::TEvTabletBase::EvTryBuildFollowerGraph: case NKikimr::TEvTabletBase::EvTrySyncFollower: - return false; - case NKikimr::TEvents::TEvFlushLog::EventType: - return false; - default: - break; - } - - if (runtime.IsScheduleForActorEnabled(event->GetRecipientRewrite())) { - return false; - } - - return true; - } - + return false; + case NKikimr::TEvents::TEvFlushLog::EventType: + return false; + default: + break; + } + + if (runtime.IsScheduleForActorEnabled(event->GetRecipientRewrite())) { + return false; + } + + return true; + } + void TTestActorRuntime::SendToPipe(ui64 tabletId, const TActorId& sender, IEventBase* payload, ui32 nodeIndex, const NKikimr::NTabletPipe::TClientConfig& pipeConfig, TActorId clientId, ui64 cookie) { bool newPipe = (clientId == TActorId()); - if (newPipe) - clientId = Register(NKikimr::NTabletPipe::CreateClient(sender, tabletId, pipeConfig), nodeIndex); - if (!UseRealThreads) { - EnableScheduleForActor(clientId, true); - } - - auto pipeEv = new IEventHandle(clientId, sender, payload, 0, cookie); - pipeEv->Rewrite(NKikimr::TEvTabletPipe::EvSend, clientId); - Send(pipeEv, nodeIndex, true); - if (newPipe) - Send(new IEventHandle(clientId, sender, new NKikimr::TEvTabletPipe::TEvShutdown()), nodeIndex, true); - } - + if (newPipe) + clientId = Register(NKikimr::NTabletPipe::CreateClient(sender, tabletId, pipeConfig), nodeIndex); + if (!UseRealThreads) { + EnableScheduleForActor(clientId, true); + } + + auto pipeEv = new IEventHandle(clientId, sender, payload, 0, cookie); + pipeEv->Rewrite(NKikimr::TEvTabletPipe::EvSend, clientId); + Send(pipeEv, nodeIndex, true); + if (newPipe) + Send(new IEventHandle(clientId, sender, new NKikimr::TEvTabletPipe::TEvShutdown()), nodeIndex, true); + } + void TTestActorRuntime::SendToPipe(TActorId clientId, const TActorId& sender, IEventBase* payload, - ui32 nodeIndex, ui64 cookie) { - auto pipeEv = new IEventHandle(clientId, sender, payload, 0, cookie); - pipeEv->Rewrite(NKikimr::TEvTabletPipe::EvSend, clientId); - Send(pipeEv, nodeIndex, true); - } - + ui32 nodeIndex, ui64 cookie) { + auto pipeEv = new IEventHandle(clientId, sender, payload, 0, cookie); + pipeEv->Rewrite(NKikimr::TEvTabletPipe::EvSend, clientId); + Send(pipeEv, nodeIndex, true); + } + TActorId TTestActorRuntime::ConnectToPipe(ui64 tabletId, const TActorId& sender, ui32 nodeIndex, const NKikimr::NTabletPipe::TClientConfig& pipeConfig) { TActorId clientId = Register(NKikimr::NTabletPipe::CreateClient(sender, tabletId, pipeConfig), nodeIndex); - if (!UseRealThreads) { - EnableScheduleForActor(clientId, true); - } - return clientId; - } - - TIntrusivePtr<NMonitoring::TDynamicCounters> TTestActorRuntime::GetCountersForComponent(TIntrusivePtr<NMonitoring::TDynamicCounters> counters, const char* component) { - return NKikimr::GetServiceCounters(counters, component); - } - - void TTestActorRuntime::InitNodeImpl(TNodeDataBase* node, size_t) { + if (!UseRealThreads) { + EnableScheduleForActor(clientId, true); + } + return clientId; + } + + TIntrusivePtr<NMonitoring::TDynamicCounters> TTestActorRuntime::GetCountersForComponent(TIntrusivePtr<NMonitoring::TDynamicCounters> counters, const char* component) { + return NKikimr::GetServiceCounters(counters, component); + } + + void TTestActorRuntime::InitNodeImpl(TNodeDataBase* node, size_t) { node->LogSettings->Append( NActorsServices::EServiceCommon_MIN, NActorsServices::EServiceCommon_MAX, @@ -254,12 +254,12 @@ namespace NActors { NKikimrServices::EServiceKikimr_MAX, NKikimrServices::EServiceKikimr_Name ); - // turn off some noisy components - TString explanation; - node->LogSettings->SetLevel(NLog::PRI_CRIT, NKikimrServices::BS_PROXY_DISCOVER, explanation); - node->LogSettings->SetLevel(NLog::PRI_ERROR, NKikimrServices::TABLET_EXECUTOR, explanation); - node->LogSettings->SetLevel(NLog::PRI_ERROR, NKikimrServices::BS_PROXY, explanation); - node->LogSettings->SetLevel(NLog::PRI_CRIT, NKikimrServices::TABLET_MAIN, explanation); - - } -} // namespace NActors + // turn off some noisy components + TString explanation; + node->LogSettings->SetLevel(NLog::PRI_CRIT, NKikimrServices::BS_PROXY_DISCOVER, explanation); + node->LogSettings->SetLevel(NLog::PRI_ERROR, NKikimrServices::TABLET_EXECUTOR, explanation); + node->LogSettings->SetLevel(NLog::PRI_ERROR, NKikimrServices::BS_PROXY, explanation); + node->LogSettings->SetLevel(NLog::PRI_CRIT, NKikimrServices::TABLET_MAIN, explanation); + + } +} // namespace NActors diff --git a/ydb/core/testlib/actors/test_runtime.h b/ydb/core/testlib/actors/test_runtime.h index 6cec4a960c..deb156876b 100644 --- a/ydb/core/testlib/actors/test_runtime.h +++ b/ydb/core/testlib/actors/test_runtime.h @@ -1,91 +1,91 @@ -#pragma once - +#pragma once + #include <ydb/core/base/tablet_pipe.h> #include <ydb/core/mon/mon.h> - + #include <library/cpp/actors/testlib/test_runtime.h> #include <library/cpp/testing/unittest/tests_data.h> - -namespace NKikimr { - struct TAppData; -} - -namespace NKikimrProto { - class TKeyConfig; -} - -namespace NActors { + +namespace NKikimr { + struct TAppData; +} + +namespace NKikimrProto { + class TKeyConfig; +} + +namespace NActors { struct IDestructable { virtual ~IDestructable() = default; }; - - using TKeyConfigGenerator = std::function<NKikimrProto::TKeyConfig (ui32)>; - - + + using TKeyConfigGenerator = std::function<NKikimrProto::TKeyConfig (ui32)>; + + class TTestActorRuntime : private TPortManager , public TTestActorRuntimeBase { - private: - struct TNodeData: public TNodeDataBase { + private: + struct TNodeData: public TNodeDataBase { void Stop(); - ~TNodeData(); - ui64 GetLoggerPoolId() const override; - THolder<NActors::TMon> Mon; - }; - - struct TNodeFactory: public INodeFactory { - TIntrusivePtr<TNodeDataBase> CreateNode() override { - return MakeIntrusive<TNodeData>(); - } - }; - - public: - struct TEgg { - TAutoPtr<NKikimr::TAppData> App0; + ~TNodeData(); + ui64 GetLoggerPoolId() const override; + THolder<NActors::TMon> Mon; + }; + + struct TNodeFactory: public INodeFactory { + TIntrusivePtr<TNodeDataBase> CreateNode() override { + return MakeIntrusive<TNodeData>(); + } + }; + + public: + struct TEgg { + TAutoPtr<NKikimr::TAppData> App0; TAutoPtr<NActors::IDestructable> Opaque; - TKeyConfigGenerator KeyConfigGenerator; - }; - - TTestActorRuntime(THeSingleSystemEnv d); - TTestActorRuntime(ui32 nodeCount, ui32 dataCenterCount, bool UseRealThreads); - TTestActorRuntime(ui32 nodeCount, ui32 dataCenterCount); - TTestActorRuntime(ui32 nodeCount = 1, bool useRealThreads = false); - - ~TTestActorRuntime(); - - virtual void Initialize(TEgg); - + TKeyConfigGenerator KeyConfigGenerator; + }; + + TTestActorRuntime(THeSingleSystemEnv d); + TTestActorRuntime(ui32 nodeCount, ui32 dataCenterCount, bool UseRealThreads); + TTestActorRuntime(ui32 nodeCount, ui32 dataCenterCount); + TTestActorRuntime(ui32 nodeCount = 1, bool useRealThreads = false); + + ~TTestActorRuntime(); + + virtual void Initialize(TEgg); + ui16 GetMonPort(ui32 nodeIndex = 0) const; void SendToPipe(ui64 tabletId, const TActorId& sender, IEventBase* payload, ui32 nodeIndex = 0, const NKikimr::NTabletPipe::TClientConfig& pipeConfig = NKikimr::NTabletPipe::TClientConfig(), TActorId clientId = TActorId(), ui64 cookie = 0); void SendToPipe(TActorId clientId, const TActorId& sender, IEventBase* payload, - ui32 nodeIndex = 0, ui64 cookie = 0); + ui32 nodeIndex = 0, ui64 cookie = 0); TActorId ConnectToPipe(ui64 tabletId, const TActorId& sender, ui32 nodeIndex, const NKikimr::NTabletPipe::TClientConfig& pipeConfig); - NKikimr::TAppData& GetAppData(ui32 nodeIndex = 0); - + NKikimr::TAppData& GetAppData(ui32 nodeIndex = 0); + TPortManager& GetPortManager() { return *this; } - static bool DefaultScheduledFilterFunc(TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event, TDuration delay, TInstant& deadline); - private: - void Initialize() override; - TIntrusivePtr<NMonitoring::TDynamicCounters> GetCountersForComponent(TIntrusivePtr<NMonitoring::TDynamicCounters> counters, const char* component) override; - void InitActorSystemSetup(TActorSystemSetup& setup) override; - - TNodeData* GetNodeById(size_t idx) override { - return static_cast<TNodeData*>(TTestActorRuntimeBase::GetNodeById(idx)); - } - - void InitNodeImpl(TNodeDataBase*, size_t) override; - - private: - using TTestActorRuntimeBase::Initialize; - - private: - THolder<NKikimr::TAppData> App0; - TKeyConfigGenerator KeyConfigGenerator; + static bool DefaultScheduledFilterFunc(TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event, TDuration delay, TInstant& deadline); + private: + void Initialize() override; + TIntrusivePtr<NMonitoring::TDynamicCounters> GetCountersForComponent(TIntrusivePtr<NMonitoring::TDynamicCounters> counters, const char* component) override; + void InitActorSystemSetup(TActorSystemSetup& setup) override; + + TNodeData* GetNodeById(size_t idx) override { + return static_cast<TNodeData*>(TTestActorRuntimeBase::GetNodeById(idx)); + } + + void InitNodeImpl(TNodeDataBase*, size_t) override; + + private: + using TTestActorRuntimeBase::Initialize; + + private: + THolder<NKikimr::TAppData> App0; + TKeyConfigGenerator KeyConfigGenerator; THolder<IDestructable> Opaque; TVector<ui16> MonPorts; - }; -} // namespace NActors + }; +} // namespace NActors diff --git a/ydb/core/testlib/actors/test_runtime_ut.cpp b/ydb/core/testlib/actors/test_runtime_ut.cpp index 7ca783a4d0..fce7f19293 100644 --- a/ydb/core/testlib/actors/test_runtime_ut.cpp +++ b/ydb/core/testlib/actors/test_runtime_ut.cpp @@ -155,7 +155,7 @@ Y_UNIT_TEST_SUITE(TActorTest) { TTestActorRuntime runtime; runtime.Initialize(MakeEgg()); - runtime.SetScheduledEventFilter([](TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event, TDuration delay, TInstant& deadline) { + runtime.SetScheduledEventFilter([](TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event, TDuration delay, TInstant& deadline) { Y_UNUSED(event); deadline = runtime.GetCurrentTime() + delay; return false; @@ -324,7 +324,7 @@ Y_UNIT_TEST_SUITE(TActorTest) { TTestActorRuntime runtime; runtime.Initialize(MakeEgg()); - runtime.SetScheduledEventFilter([](TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event, TDuration delay, TInstant& deadline) { + runtime.SetScheduledEventFilter([](TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event, TDuration delay, TInstant& deadline) { if (event->GetTypeRewrite() != TEvents::TSystem::Wakeup) return true; @@ -332,7 +332,7 @@ Y_UNIT_TEST_SUITE(TActorTest) { return false; }); - runtime.SetScheduledEventsSelectorFunc(&TTestActorRuntimeBase::CollapsedTimeScheduledEventsSelector); + runtime.SetScheduledEventsSelectorFunc(&TTestActorRuntimeBase::CollapsedTimeScheduledEventsSelector); TActorId sender = runtime.AllocateEdgeActor(); auto myActor = new TMyActor; TActorId actorId = runtime.Register(myActor); @@ -453,7 +453,7 @@ Y_UNIT_TEST_SUITE(TActorTest) { auto producerActor = new TProducerActor(count, consumerIds); TActorId producerId = runtime.Register(producerActor); runtime.Send(new IEventHandle(producerId, sender, new TEvents::TEvPing)); - runtime.SetObserverFunc([](TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event) { + runtime.SetObserverFunc([](TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event) { Y_UNUSED(runtime); Y_UNUSED(event); return TTestActorRuntime::EEventAction::PROCESS; diff --git a/ydb/core/testlib/actors/ya.make b/ydb/core/testlib/actors/ya.make index e7c6496a19..984cf7e177 100644 --- a/ydb/core/testlib/actors/ya.make +++ b/ydb/core/testlib/actors/ya.make @@ -1,12 +1,12 @@ -LIBRARY() - +LIBRARY() + OWNER(g:kikimr) - -SRCS( - test_runtime.cpp -) - -PEERDIR( + +SRCS( + test_runtime.cpp +) + +PEERDIR( library/cpp/actors/testlib library/cpp/testing/unittest ydb/core/base @@ -14,15 +14,15 @@ PEERDIR( ydb/core/mon_alloc ydb/core/scheme ydb/core/tablet -) - -IF (GCC) +) + +IF (GCC) CFLAGS( -fno-devirtualize-speculatively ) -ENDIF() - -END() +ENDIF() + +END() RECURSE_FOR_TESTS( ut diff --git a/ydb/core/testlib/tablet_helpers.cpp b/ydb/core/testlib/tablet_helpers.cpp index 2a4371796b..a853040b93 100644 --- a/ydb/core/testlib/tablet_helpers.cpp +++ b/ydb/core/testlib/tablet_helpers.cpp @@ -55,15 +55,15 @@ const bool VARIATE_RANDOM_SEED = false; const ui64 PQ_CACHE_MAX_SIZE_MB = 32; const TDuration PQ_CACHE_KEEP_TIMEOUT = TDuration::Seconds(10); -static NActors::TTestActorRuntime& AsKikimrRuntime(NActors::TTestActorRuntimeBase& r) { - try { - return dynamic_cast<NActors::TTestBasicRuntime&>(r); - } catch (const std::bad_cast& e) { - Cerr << e.what() << Endl; - Y_FAIL("Failed to cast to TTestActorRuntime: %s", e.what()); - } -} - +static NActors::TTestActorRuntime& AsKikimrRuntime(NActors::TTestActorRuntimeBase& r) { + try { + return dynamic_cast<NActors::TTestBasicRuntime&>(r); + } catch (const std::bad_cast& e) { + Cerr << e.what() << Endl; + Y_FAIL("Failed to cast to TTestActorRuntime: %s", e.what()); + } +} + namespace NKikimr { class TFakeMediatorTimecastProxy : public TActor<TFakeMediatorTimecastProxy> { @@ -164,7 +164,7 @@ namespace NKikimr { , TabletIds(tabletIds) {} - void OnEvent(TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event) { + void OnEvent(TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event) { Y_UNUSED(runtime); if (event->GetTypeRewrite() == TEvStateStorage::EvInfo) { auto info = static_cast<TEvStateStorage::TEvInfo*>(event->GetBase()); @@ -548,7 +548,7 @@ namespace NKikimr { : Tracer(tracer) {} - bool operator()(TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event, TDuration delay, TInstant& deadline) { + bool operator()(TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event, TDuration delay, TInstant& deadline) { if (runtime.IsScheduleForActorEnabled(event->GetRecipientRewrite()) || Tracer.IsTabletEvent(event) || Tracer.IsTabletRelatedEvent(event)) { deadline = runtime.GetTimeProvider()->Now() + delay; @@ -759,19 +759,19 @@ namespace NKikimr { TTabletTracer tabletTracer(activeZone, tabletIds); TTabletScheduledFilter scheduledFilter(tabletTracer); try { - testFunc(INITIAL_TEST_DISPATCH_NAME, [&](TTestActorRuntimeBase& runtime) { - runtime.SetObserverFunc([&](TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event) { - tabletTracer.OnEvent(AsKikimrRuntime(runtime), event); + testFunc(INITIAL_TEST_DISPATCH_NAME, [&](TTestActorRuntimeBase& runtime) { + runtime.SetObserverFunc([&](TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event) { + tabletTracer.OnEvent(AsKikimrRuntime(runtime), event); return TTestActorRuntime::EEventAction::PROCESS; }); runtime.SetRegistrationObserverFunc([&](TTestActorRuntimeBase& runtime, const TActorId& parentId, const TActorId& actorId) { - tabletTracer.OnRegistration(AsKikimrRuntime(runtime), parentId, actorId); + tabletTracer.OnRegistration(AsKikimrRuntime(runtime), parentId, actorId); }); - runtime.SetScheduledEventFilter([&](TTestActorRuntimeBase& r, TAutoPtr<IEventHandle>& event, + runtime.SetScheduledEventFilter([&](TTestActorRuntimeBase& r, TAutoPtr<IEventHandle>& event, TDuration delay, TInstant& deadline) { - auto& runtime = AsKikimrRuntime(r); + auto& runtime = AsKikimrRuntime(r); return !(!scheduledFilter(runtime, event, delay, deadline) || !TTestActorRuntime::DefaultScheduledFilterFunc(runtime, event, delay, deadline)); }); @@ -860,18 +860,18 @@ namespace NKikimr { TTabletScheduledFilter scheduledFilter(tabletTracer); testFunc(INITIAL_TEST_DISPATCH_NAME, [&](TTestActorRuntime& runtime) { - runtime.SetObserverFunc([&](TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event) { - tabletTracer.OnEvent(AsKikimrRuntime(runtime), event); + runtime.SetObserverFunc([&](TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event) { + tabletTracer.OnEvent(AsKikimrRuntime(runtime), event); return TTestActorRuntime::EEventAction::PROCESS; }); runtime.SetRegistrationObserverFunc([&](TTestActorRuntimeBase& runtime, const TActorId& parentId, const TActorId& actorId) { - tabletTracer.OnRegistration(AsKikimrRuntime(runtime), parentId, actorId); + tabletTracer.OnRegistration(AsKikimrRuntime(runtime), parentId, actorId); }); - runtime.SetScheduledEventFilter([&](TTestActorRuntimeBase& r, TAutoPtr<IEventHandle>& event, + runtime.SetScheduledEventFilter([&](TTestActorRuntimeBase& r, TAutoPtr<IEventHandle>& event, TDuration delay, TInstant& deadline) { - auto& runtime = AsKikimrRuntime(r); + auto& runtime = AsKikimrRuntime(r); return scheduledFilter(runtime, event, delay, deadline) && TTestActorRuntime::DefaultScheduledFilterFunc(runtime, event, delay, deadline); }); @@ -908,17 +908,17 @@ namespace NKikimr { testFunc(dispatchName, [&](TTestActorRuntime& runtime) { - runtime.SetObserverFunc([&](TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event) { - return pipeResetingObserver.OnEvent(AsKikimrRuntime(runtime), event); + runtime.SetObserverFunc([&](TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event) { + return pipeResetingObserver.OnEvent(AsKikimrRuntime(runtime), event); }); runtime.SetRegistrationObserverFunc([&](TTestActorRuntimeBase& runtime, const TActorId& parentId, const TActorId& actorId) { - pipeResetingObserver.OnRegistration(AsKikimrRuntime(runtime), parentId, actorId); + pipeResetingObserver.OnRegistration(AsKikimrRuntime(runtime), parentId, actorId); }); - runtime.SetScheduledEventFilter([&](TTestActorRuntimeBase& r, TAutoPtr<IEventHandle>& event, + runtime.SetScheduledEventFilter([&](TTestActorRuntimeBase& r, TAutoPtr<IEventHandle>& event, TDuration delay, TInstant& deadline) { - auto& runtime = AsKikimrRuntime(r); + auto& runtime = AsKikimrRuntime(r); return scheduledFilter(runtime, event, delay, deadline) && TTestActorRuntime::DefaultScheduledFilterFunc(runtime, event, delay, deadline); }); @@ -957,17 +957,17 @@ namespace NKikimr { Cout << dispatchName << "\n"; testFunc(dispatchName, [&](TTestActorRuntime& runtime) { - runtime.SetObserverFunc([&](TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event) { - return delayingObserver.OnEvent(AsKikimrRuntime(runtime), event); + runtime.SetObserverFunc([&](TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event) { + return delayingObserver.OnEvent(AsKikimrRuntime(runtime), event); }); runtime.SetRegistrationObserverFunc([&](TTestActorRuntimeBase& runtime, const TActorId& parentId, const TActorId& actorId) { - delayingObserver.OnRegistration(AsKikimrRuntime(runtime), parentId, actorId); + delayingObserver.OnRegistration(AsKikimrRuntime(runtime), parentId, actorId); }); - runtime.SetScheduledEventFilter([&](TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event, + runtime.SetScheduledEventFilter([&](TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event, TDuration delay, TInstant& deadline) { - return scheduledFilter(AsKikimrRuntime(runtime), event, delay, deadline); + return scheduledFilter(AsKikimrRuntime(runtime), event, delay, deadline); }); runtime.SetScheduledEventsSelectorFunc(&TTestActorRuntime::CollapsedTimeScheduledEventsSelector); @@ -995,24 +995,24 @@ namespace NKikimr { , TabletTracer(TracingActive, tabletIds) , ScheduledFilter(TabletTracer) { - PrevObserverFunc = Runtime.SetObserverFunc([&](TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event) { - TabletTracer.OnEvent(AsKikimrRuntime(runtime), event); + PrevObserverFunc = Runtime.SetObserverFunc([&](TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event) { + TabletTracer.OnEvent(AsKikimrRuntime(runtime), event); return TTestActorRuntime::EEventAction::PROCESS; }); PrevRegistrationObserverFunc = Runtime.SetRegistrationObserverFunc( [&](TTestActorRuntimeBase& runtime, const TActorId& parentId, const TActorId& actorId) { - TabletTracer.OnRegistration(AsKikimrRuntime(runtime), parentId, actorId); + TabletTracer.OnRegistration(AsKikimrRuntime(runtime), parentId, actorId); }); - PrevScheduledFilterFunc = Runtime.SetScheduledEventFilter([&](TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event, + PrevScheduledFilterFunc = Runtime.SetScheduledEventFilter([&](TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event, TDuration delay, TInstant& deadline) { if (event->GetRecipientRewrite() == Sender) { deadline = runtime.GetTimeProvider()->Now() + delay; return false; } - return ScheduledFilter(AsKikimrRuntime(runtime), event, delay, deadline); + return ScheduledFilter(AsKikimrRuntime(runtime), event, delay, deadline); }); PrevScheduledEventsSelector = Runtime.SetScheduledEventsSelectorFunc(&TTestActorRuntime::CollapsedTimeScheduledEventsSelector); diff --git a/ydb/core/testlib/tablet_helpers.h b/ydb/core/testlib/tablet_helpers.h index 8c7e1f803c..81a7c614aa 100644 --- a/ydb/core/testlib/tablet_helpers.h +++ b/ydb/core/testlib/tablet_helpers.h @@ -25,7 +25,7 @@ namespace NKikimr { void InvalidateTabletResolverCache(TTestActorRuntime& runtime, ui64 tabletId, ui32 nodeIndex = 0); void RebootTablet(TTestActorRuntime& runtime, ui64 tabletId, const TActorId& sender, ui32 nodeIndex = 0, bool sysTablet = false); void GracefulRestartTablet(TTestActorRuntime& runtime, ui64 tabletId, const TActorId& sender, ui32 nodeIndex = 0); - void SetupTabletServices(TTestActorRuntime& runtime, TAppPrepare* app = nullptr, bool mockDisk = false, + void SetupTabletServices(TTestActorRuntime& runtime, TAppPrepare* app = nullptr, bool mockDisk = false, NFake::TStorage storage = {}, NFake::TCaches caches = {}); const TString DEFAULT_STORAGE_POOL = "Storage Pool with id: 1"; diff --git a/ydb/core/tracing/tablet_info.cpp b/ydb/core/tracing/tablet_info.cpp index 1b0d961b7c..6eb99ffc60 100644 --- a/ydb/core/tracing/tablet_info.cpp +++ b/ydb/core/tracing/tablet_info.cpp @@ -685,7 +685,7 @@ void TTabletInfoActor::Handle(NMon::TEvHttpInfo::TPtr &ev, const TActorContext & return; } NMon::TEvHttpInfo* msg = ev->Get(); - const TCgiParameters& cgi = msg->Request.GetParams(); + const TCgiParameters& cgi = msg->Request.GetParams(); ui32 timeout = FromStringWithDefault<ui32>(cgi.Get("timeout"), 10000); bool hasNodeIdParam = HasProperty(cgi, "NodeID"); diff --git a/ydb/core/tx/datashard/datashard_ut_common.cpp b/ydb/core/tx/datashard/datashard_ut_common.cpp index 9ebfb90027..d754f7ad0e 100644 --- a/ydb/core/tx/datashard/datashard_ut_common.cpp +++ b/ydb/core/tx/datashard/datashard_ut_common.cpp @@ -752,7 +752,7 @@ ui64 TFakeMiniKQLProxy::Plan(ui64 stepId, const TMap<ui64, TFakeProxyTx::TPtr>& UNIT_ASSERT(DelayedReadSets.size() <= 1); UNIT_ASSERT(DelayedData.size() <= 1); - auto observer = [&](TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& ev) { + auto observer = [&](TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& ev) { // send immediate in the right moment if (ev->Type == TEvProposeTransactionResult::EventType) { auto event = ev->Get<TEvProposeTransactionResult>(); @@ -1010,12 +1010,12 @@ TDatashardInitialEventsFilter::TDatashardInitialEventsFilter(const TVector<ui64> TTestActorRuntime::TEventFilter TDatashardInitialEventsFilter::Prepare() { RemainTablets = TabletIds; - return [&](TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event) { + return [&](TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event) { return (*this)(runtime, event); }; } -bool TDatashardInitialEventsFilter::operator()(TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event) { +bool TDatashardInitialEventsFilter::operator()(TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event) { Y_UNUSED(runtime); if (event->GetTypeRewrite() == TEvTxProcessing::EvPlanStepAck) { ui64 tabletId = reinterpret_cast<TEvTxProcessing::TEvPlanStepAck::TPtr&>(event)->Get()->Record.GetTabletId(); diff --git a/ydb/core/tx/datashard/datashard_ut_common.h b/ydb/core/tx/datashard/datashard_ut_common.h index 5e478e54e4..4b5d7fe4ea 100644 --- a/ydb/core/tx/datashard/datashard_ut_common.h +++ b/ydb/core/tx/datashard/datashard_ut_common.h @@ -326,7 +326,7 @@ class TDatashardInitialEventsFilter { public: TDatashardInitialEventsFilter(const TVector<ui64>& tabletIds); TTestActorRuntime::TEventFilter Prepare(); - bool operator()(TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event); + bool operator()(TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event); TDatashardInitialEventsFilter(const TDatashardInitialEventsFilter&) = delete; TDatashardInitialEventsFilter(TDatashardInitialEventsFilter&&) = default; diff --git a/ydb/core/tx/datashard/datashard_ut_init.cpp b/ydb/core/tx/datashard/datashard_ut_init.cpp index 5d4699feef..818e30ab0c 100644 --- a/ydb/core/tx/datashard/datashard_ut_init.cpp +++ b/ydb/core/tx/datashard/datashard_ut_init.cpp @@ -134,7 +134,7 @@ Y_UNIT_TEST_SUITE(TTxDataShardTestInit) { bool sawResolve = false; bool dropResolve = restart; // Remove table path from propose. - auto captureTableId = [&](TTestActorRuntimeBase&, + auto captureTableId = [&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle> &event) -> auto { if (event->GetTypeRewrite() == TEvDataShard::EvProposeTransaction) { auto &rec = event->Get<TEvDataShard::TEvProposeTransaction>()->Record; diff --git a/ydb/core/tx/datashard/datashard_ut_locks.cpp b/ydb/core/tx/datashard/datashard_ut_locks.cpp index 252113670b..1d25c39f99 100644 --- a/ydb/core/tx/datashard/datashard_ut_locks.cpp +++ b/ydb/core/tx/datashard/datashard_ut_locks.cpp @@ -664,7 +664,7 @@ void CheckLocksCacheUsage(bool waitForLocksStore) { // should use RS. Hold one RS to 'pause' one transaction // and then restart its shard. This will cause locks // cache to be used. - auto captureRS = [shards](TTestActorRuntimeBase&, + auto captureRS = [shards](TTestActorRuntimeBase&, TAutoPtr<IEventHandle> &event) -> auto { if (event->GetTypeRewrite() == TEvTxProcessing::EvReadSet) { auto &rec = event->Get<TEvTxProcessing::TEvReadSet>()->Record; diff --git a/ydb/core/tx/datashard/datashard_ut_minstep.cpp b/ydb/core/tx/datashard/datashard_ut_minstep.cpp index 01498e349e..99d72b2e45 100644 --- a/ydb/core/tx/datashard/datashard_ut_minstep.cpp +++ b/ydb/core/tx/datashard/datashard_ut_minstep.cpp @@ -22,7 +22,7 @@ TAutoPtr<IEventHandle> EjectDataPropose(TServer::TPtr server, ui64 dataShard) bool hasFound = false; TAutoPtr<IEventHandle> proposeEvent = nullptr; - auto captureProposes = [dataShard, schemeShard, &hasFound, &proposeEvent] (TTestActorRuntimeBase&, TAutoPtr<IEventHandle> &event) -> auto + auto captureProposes = [dataShard, schemeShard, &hasFound, &proposeEvent] (TTestActorRuntimeBase&, TAutoPtr<IEventHandle> &event) -> auto { if (event->GetTypeRewrite() == TEvTxProxy::EvProposeTransaction) { auto &rec = event->Get<TEvTxProxy::TEvProposeTransaction>()->Record; diff --git a/ydb/core/tx/datashard/datashard_ut_order.cpp b/ydb/core/tx/datashard/datashard_ut_order.cpp index fa9ac9ab70..32472dce59 100644 --- a/ydb/core/tx/datashard/datashard_ut_order.cpp +++ b/ydb/core/tx/datashard/datashard_ut_order.cpp @@ -1262,7 +1262,7 @@ Y_UNIT_TEST_QUAD(TestDelayedTxWaitsForWriteActiveTxOnly, UseMvcc, UseNewEngine) TVector<TAutoPtr<IEventHandle>> rss; // We want to intercept all RS to table-2. - auto captureRS = [shard2,&rss](TTestActorRuntimeBase &, + auto captureRS = [shard2,&rss](TTestActorRuntimeBase &, TAutoPtr<IEventHandle> &event) -> auto { if (event->GetTypeRewrite() == TEvTxProcessing::EvReadSet) { auto &rec = event->Get<TEvTxProcessing::TEvReadSet>()->Record; @@ -1370,7 +1370,7 @@ Y_UNIT_TEST_QUAD(TestOnlyDataTxLagCausesRejects, UseMvcc, UseNewEngine) { ExecSQL(server, sender, Q_("SELECT COUNT(*) FROM `/Root/table-1`")); // Send SQL request which should hang due to lost RS. - auto captureRS = [](TTestActorRuntimeBase&, + auto captureRS = [](TTestActorRuntimeBase&, TAutoPtr<IEventHandle> &event) -> auto { if (event->GetTypeRewrite() == TEvTxProcessing::EvReadSet) return TTestActorRuntime::EEventAction::DROP; diff --git a/ydb/core/tx/datashard/datashard_ut_rs.cpp b/ydb/core/tx/datashard/datashard_ut_rs.cpp index 6fc0c7c37c..edbe2968da 100644 --- a/ydb/core/tx/datashard/datashard_ut_rs.cpp +++ b/ydb/core/tx/datashard/datashard_ut_rs.cpp @@ -107,7 +107,7 @@ Y_UNIT_TEST_SUITE(TDataShardRSTest) { // Run multishard tx but drop RS to pause it on the first shard. { - auto captureRS = [shard=shards[1]](TTestActorRuntimeBase&, + auto captureRS = [shard=shards[1]](TTestActorRuntimeBase&, TAutoPtr<IEventHandle> &event) -> auto { if (event->GetTypeRewrite() == TEvTxProcessing::EvReadSet) { auto &rec = event->Get<TEvTxProcessing::TEvReadSet>()->Record; @@ -219,7 +219,7 @@ Y_UNIT_TEST_SUITE(TDataShardRSTest) { // Run multishard tx but drop all RS acks to the table-1. // Tx should still finish successfully. { - auto captureRS = [shard=shards1[0]](TTestActorRuntimeBase&, + auto captureRS = [shard=shards1[0]](TTestActorRuntimeBase&, TAutoPtr<IEventHandle> &event) -> auto { if (event->GetTypeRewrite() == TEvTxProcessing::EvReadSetAck) { auto &rec = event->Get<TEvTxProcessing::TEvReadSetAck>()->Record; @@ -275,7 +275,7 @@ Y_UNIT_TEST_SUITE(TDataShardRSTest) { // We want to intercept all RS from table-1 and all RS acks // from table-3. - auto captureRS = [shard1,shard3](TTestActorRuntimeBase&, + auto captureRS = [shard1,shard3](TTestActorRuntimeBase&, TAutoPtr<IEventHandle> &event) -> auto { if (event->GetTypeRewrite() == TEvTxProcessing::EvReadSet) { auto &rec = event->Get<TEvTxProcessing::TEvReadSet>()->Record; diff --git a/ydb/core/tx/schemeshard/ut_base.cpp b/ydb/core/tx/schemeshard/ut_base.cpp index 9d96a5d611..a76dcf9d5b 100644 --- a/ydb/core/tx/schemeshard/ut_base.cpp +++ b/ydb/core/tx/schemeshard/ut_base.cpp @@ -7102,12 +7102,12 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { } void SetUp(TTestActorRuntime& runtime) { - runtime.SetObserverFunc([&](TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event) { - return this->OnEvent(static_cast<TTestActorRuntime&>(runtime), event); + runtime.SetObserverFunc([&](TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event) { + return this->OnEvent(static_cast<TTestActorRuntime&>(runtime), event); }); } - bool IsPlanStepMessage(TTestActorRuntimeBase& /*runtime*/, TAutoPtr<IEventHandle>& event) { + bool IsPlanStepMessage(TTestActorRuntimeBase& /*runtime*/, TAutoPtr<IEventHandle>& event) { if (event->GetTypeRewrite() != TEvTxProcessing::EvPlanStep) { return false; } diff --git a/ydb/core/util/address_classifier.h b/ydb/core/util/address_classifier.h index d0fcef3f38..f5e25cd3e4 100644 --- a/ydb/core/util/address_classifier.h +++ b/ydb/core/util/address_classifier.h @@ -1,6 +1,6 @@ #pragma once -#include <library/cpp/ipmath/range_set.h> +#include <library/cpp/ipmath/range_set.h> #include <util/generic/hash.h> #include <util/generic/ptr.h> diff --git a/ydb/core/util/failure_injection.cpp b/ydb/core/util/failure_injection.cpp index e71ee123f1..9eeaaa968b 100644 --- a/ydb/core/util/failure_injection.cpp +++ b/ydb/core/util/failure_injection.cpp @@ -182,7 +182,7 @@ namespace NKikimr { void Handle(NMon::TEvHttpInfo::TPtr& ev, const TActorContext& ctx) { TStringStream str; - const auto& params = ev->Get()->Request.GetParams(); + const auto& params = ev->Get()->Request.GetParams(); if (params.Has("queue")) { TString queue = params.Get("queue"); if (queue) { diff --git a/ydb/core/util/ya.make b/ydb/core/util/ya.make index 343d854b35..4f325d8fcb 100644 --- a/ydb/core/util/ya.make +++ b/ydb/core/util/ya.make @@ -66,7 +66,7 @@ PEERDIR( library/cpp/actors/util library/cpp/containers/stack_vector library/cpp/html/escape - library/cpp/ipmath + library/cpp/ipmath library/cpp/lwtrace library/cpp/monlib/dynamic_counters library/cpp/random_provider diff --git a/ydb/core/viewer/json_browse.h b/ydb/core/viewer/json_browse.h index e059fee9b1..cba672826e 100644 --- a/ydb/core/viewer/json_browse.h +++ b/ydb/core/viewer/json_browse.h @@ -101,7 +101,7 @@ public: } void Bootstrap(const TActorContext& ctx) { - const auto& params(Event->Get()->Request.GetParams()); + const auto& params(Event->Get()->Request.GetParams()); JsonSettings.EnumAsNumbers = !FromStringWithDefault<bool>(params.Get("enums"), true); JsonSettings.UI64AsString = !FromStringWithDefault<bool>(params.Get("ui64"), false); Timeout = FromStringWithDefault<ui32>(params.Get("timeout"), 10000); diff --git a/ydb/core/viewer/json_bscontrollerinfo.h b/ydb/core/viewer/json_bscontrollerinfo.h index d5cb659d4c..5df47ed553 100644 --- a/ydb/core/viewer/json_bscontrollerinfo.h +++ b/ydb/core/viewer/json_bscontrollerinfo.h @@ -33,7 +33,7 @@ public: {} void Bootstrap(const TActorContext& ctx) { - const auto& params(Event->Get()->Request.GetParams()); + const auto& params(Event->Get()->Request.GetParams()); JsonSettings.EnumAsNumbers = !FromStringWithDefault<bool>(params.Get("enums"), false); JsonSettings.UI64AsString = !FromStringWithDefault<bool>(params.Get("ui64"), false); Timeout = FromStringWithDefault<ui32>(params.Get("timeout"), 10000); diff --git a/ydb/core/viewer/json_content.h b/ydb/core/viewer/json_content.h index 179fd02d54..79b230f128 100644 --- a/ydb/core/viewer/json_content.h +++ b/ydb/core/viewer/json_content.h @@ -44,7 +44,7 @@ public: public: void Bootstrap(const TActorContext& ctx) { - BuildRequestContext(&Event->Get()->Request, ContentRequestContext); + BuildRequestContext(&Event->Get()->Request, ContentRequestContext); if (!Event->Get()->UserToken.empty()) { ContentRequestContext.UserToken = Event->Get()->UserToken; } @@ -60,7 +60,7 @@ public: private: static void BuildRequestContext( - const NMonitoring::IMonHttpRequest* httpRequest, + const NMonitoring::IMonHttpRequest* httpRequest, IViewer::TContentRequestContext& reqCtx) { if (!httpRequest) { return; diff --git a/ydb/core/viewer/json_hiveinfo.h b/ydb/core/viewer/json_hiveinfo.h index 3269bad603..f4d04ddfaf 100644 --- a/ydb/core/viewer/json_hiveinfo.h +++ b/ydb/core/viewer/json_hiveinfo.h @@ -34,7 +34,7 @@ public: {} void Bootstrap() { - const auto& params(Event->Get()->Request.GetParams()); + const auto& params(Event->Get()->Request.GetParams()); ui64 hiveId = FromStringWithDefault<ui64>(params.Get("hive_id"), 0); JsonSettings.EnumAsNumbers = !FromStringWithDefault<bool>(params.Get("enums"), false); JsonSettings.UI64AsString = !FromStringWithDefault<bool>(params.Get("ui64"), false); diff --git a/ydb/core/viewer/json_hivestats.h b/ydb/core/viewer/json_hivestats.h index 8e521062cd..7f24693d63 100644 --- a/ydb/core/viewer/json_hivestats.h +++ b/ydb/core/viewer/json_hivestats.h @@ -33,7 +33,7 @@ public: {} void Bootstrap() { - const auto& params(Event->Get()->Request.GetParams()); + const auto& params(Event->Get()->Request.GetParams()); ui64 hiveId = FromStringWithDefault<ui64>(params.Get("hive_id"), 0); JsonSettings.EnumAsNumbers = !FromStringWithDefault<bool>(params.Get("enums"), true); JsonSettings.UI64AsString = !FromStringWithDefault<bool>(params.Get("ui64"), false); diff --git a/ydb/core/viewer/json_labeledcounters.h b/ydb/core/viewer/json_labeledcounters.h index 3af05ef8de..6fce18b32b 100644 --- a/ydb/core/viewer/json_labeledcounters.h +++ b/ydb/core/viewer/json_labeledcounters.h @@ -39,7 +39,7 @@ public: {} void Bootstrap(const TActorContext& ctx) { - const auto& params(Event->Get()->Request.GetParams()); + const auto& params(Event->Get()->Request.GetParams()); JsonSettings.EnumAsNumbers = !FromStringWithDefault<bool>(params.Get("enums"), false); JsonSettings.UI64AsString = !FromStringWithDefault<bool>(params.Get("ui64"), false); Timeout = FromStringWithDefault<ui32>(params.Get("timeout"), 10000); diff --git a/ydb/core/viewer/json_metainfo.h b/ydb/core/viewer/json_metainfo.h index 52bad3dfdc..54064d7403 100644 --- a/ydb/core/viewer/json_metainfo.h +++ b/ydb/core/viewer/json_metainfo.h @@ -47,7 +47,7 @@ public: {} void Bootstrap(const TActorContext& ctx) { - const auto& params(Event->Get()->Request.GetParams()); + const auto& params(Event->Get()->Request.GetParams()); JsonSettings.EnumAsNumbers = !FromStringWithDefault<bool>(params.Get("enums"), true); JsonSettings.UI64AsString = !FromStringWithDefault<bool>(params.Get("ui64"), false); Timeout = FromStringWithDefault<ui32>(params.Get("timeout"), 10000); diff --git a/ydb/core/viewer/json_pqconsumerinfo.h b/ydb/core/viewer/json_pqconsumerinfo.h index 1b5cd386b1..b98dfc2632 100644 --- a/ydb/core/viewer/json_pqconsumerinfo.h +++ b/ydb/core/viewer/json_pqconsumerinfo.h @@ -39,7 +39,7 @@ public: {} void Bootstrap(const TActorContext& ctx) { - const auto& params(Event->Get()->Request.GetParams()); + const auto& params(Event->Get()->Request.GetParams()); JsonSettings.EnumAsNumbers = !FromStringWithDefault<bool>(params.Get("enums"), false); JsonSettings.UI64AsString = !FromStringWithDefault<bool>(params.Get("ui64"), false); Timeout = FromStringWithDefault<ui32>(params.Get("timeout"), 10000); diff --git a/ydb/core/viewer/json_query.h b/ydb/core/viewer/json_query.h index 2e0527aae0..600d8088e1 100644 --- a/ydb/core/viewer/json_query.h +++ b/ydb/core/viewer/json_query.h @@ -60,7 +60,7 @@ public: } void Bootstrap(const TActorContext& ctx) { - const auto& params(Event->Get()->Request.GetParams()); + const auto& params(Event->Get()->Request.GetParams()); auto event = MakeHolder<NKqp::TEvKqp::TEvQueryRequest>(); JsonSettings.EnumAsNumbers = !FromStringWithDefault<bool>(params.Get("enums"), false); JsonSettings.UI64AsString = !FromStringWithDefault<bool>(params.Get("ui64"), false); @@ -69,9 +69,9 @@ public: TString database = params.Get("database"); Stats = params.Get("stats"); Action = params.Get("action"); - if (query.empty() && Event->Get()->Request.GetMethod() == HTTP_METHOD_POST) { - TStringBuf content = Event->Get()->Request.GetPostContent(); - const THttpHeaders& headers = Event->Get()->Request.GetHeaders(); + if (query.empty() && Event->Get()->Request.GetMethod() == HTTP_METHOD_POST) { + TStringBuf content = Event->Get()->Request.GetPostContent(); + const THttpHeaders& headers = Event->Get()->Request.GetHeaders(); auto itContentType = FindIf(headers, [](const auto& header) { return header.Name() == "Content-Type"; }); if (itContentType != headers.end()) { TStringBuf contentTypeHeader = itContentType->Value(); diff --git a/ydb/core/viewer/json_tabletcounters.h b/ydb/core/viewer/json_tabletcounters.h index 5560039af7..7f39910441 100644 --- a/ydb/core/viewer/json_tabletcounters.h +++ b/ydb/core/viewer/json_tabletcounters.h @@ -52,7 +52,7 @@ public: } void Bootstrap(const TActorContext& ctx) { - const auto& params(Event->Get()->Request.GetParams()); + const auto& params(Event->Get()->Request.GetParams()); JsonSettings.EnumAsNumbers = !FromStringWithDefault<bool>(params.Get("enums"), false); JsonSettings.UI64AsString = !FromStringWithDefault<bool>(params.Get("ui64"), false); Timeout = FromStringWithDefault<ui32>(params.Get("timeout"), 10000); diff --git a/ydb/core/viewer/json_tabletinfo.h b/ydb/core/viewer/json_tabletinfo.h index 7fc0ad605f..32546865a1 100644 --- a/ydb/core/viewer/json_tabletinfo.h +++ b/ydb/core/viewer/json_tabletinfo.h @@ -73,7 +73,7 @@ public: } void Bootstrap() override { - const auto& params(Event->Get()->Request.GetParams()); + const auto& params(Event->Get()->Request.GetParams()); Timeout = FromStringWithDefault<ui32>(params.Get("timeout"), 10000); if (params.Has("path")) { THolder<TEvTxUserProxy::TEvNavigate> request(new TEvTxUserProxy::TEvNavigate()); diff --git a/ydb/core/viewer/json_tenantinfo.h b/ydb/core/viewer/json_tenantinfo.h index f172d322fb..749422ba39 100644 --- a/ydb/core/viewer/json_tenantinfo.h +++ b/ydb/core/viewer/json_tenantinfo.h @@ -59,7 +59,7 @@ public: } void Bootstrap() { - const auto& params(Event->Get()->Request.GetParams()); + const auto& params(Event->Get()->Request.GetParams()); JsonSettings.EnumAsNumbers = !FromStringWithDefault<bool>(params.Get("enums"), true); JsonSettings.UI64AsString = !FromStringWithDefault<bool>(params.Get("ui64"), false); Followers = false; diff --git a/ydb/core/viewer/json_tenants.h b/ydb/core/viewer/json_tenants.h index 262a22cb25..762d5e3b43 100644 --- a/ydb/core/viewer/json_tenants.h +++ b/ydb/core/viewer/json_tenants.h @@ -36,7 +36,7 @@ public: {} void Bootstrap() { - const auto& params(Event->Get()->Request.GetParams()); + const auto& params(Event->Get()->Request.GetParams()); JsonSettings.EnumAsNumbers = !FromStringWithDefault<bool>(params.Get("enums"), true); JsonSettings.UI64AsString = !FromStringWithDefault<bool>(params.Get("ui64"), false); InitConfig(params); diff --git a/ydb/core/viewer/json_topicinfo.h b/ydb/core/viewer/json_topicinfo.h index 3767aaf099..cbf42ee727 100644 --- a/ydb/core/viewer/json_topicinfo.h +++ b/ydb/core/viewer/json_topicinfo.h @@ -35,7 +35,7 @@ public: {} void Bootstrap(const TActorContext& ctx) { - const auto& params(Event->Get()->Request.GetParams()); + const auto& params(Event->Get()->Request.GetParams()); JsonSettings.EnumAsNumbers = !FromStringWithDefault<bool>(params.Get("enums"), false); JsonSettings.UI64AsString = !FromStringWithDefault<bool>(params.Get("ui64"), false); Timeout = FromStringWithDefault<ui32>(params.Get("timeout"), 10000); diff --git a/ydb/core/viewer/json_wb_req.h b/ydb/core/viewer/json_wb_req.h index 15eeb4e3e6..f04c85688f 100644 --- a/ydb/core/viewer/json_wb_req.h +++ b/ydb/core/viewer/json_wb_req.h @@ -101,7 +101,7 @@ public: } virtual void Bootstrap() { - const auto& params(Event->Get()->Request.GetParams()); + const auto& params(Event->Get()->Request.GetParams()); TBase::InitConfig(params); SplitIds(params.Get("node_id"), ',', FilterNodeIds); std::replace(FilterNodeIds.begin(), FilterNodeIds.end(), (ui32)0, TlsActivationContext->ActorSystem()->NodeId); diff --git a/ydb/core/viewer/json_whoami.h b/ydb/core/viewer/json_whoami.h index ae030ccb8a..b490a6930f 100644 --- a/ydb/core/viewer/json_whoami.h +++ b/ydb/core/viewer/json_whoami.h @@ -28,7 +28,7 @@ public: {} void Bootstrap(const TActorContext& ctx) { - const auto& params(Event->Get()->Request.GetParams()); + const auto& params(Event->Get()->Request.GetParams()); JsonSettings.EnumAsNumbers = !FromStringWithDefault<bool>(params.Get("enums"), false); JsonSettings.UI64AsString = !FromStringWithDefault<bool>(params.Get("ui64"), false); ReplyAndDie(ctx); diff --git a/ydb/core/viewer/viewer.cpp b/ydb/core/viewer/viewer.cpp index db780e2ea9..f1124115bf 100644 --- a/ydb/core/viewer/viewer.cpp +++ b/ydb/core/viewer/viewer.cpp @@ -255,7 +255,7 @@ private: TString GetSwaggerJson(NMon::TEvHttpInfo::TPtr &ev) { TStringStream json; - TString basepath = ev->Get()->Request.GetParams().Get("basepath"); + TString basepath = ev->Get()->Request.GetParams().Get("basepath"); if (basepath.empty()) { basepath = "/viewer"; } else { @@ -263,7 +263,7 @@ private: basepath = basepath.substr(0, basepath.size() - 5); } } - TString protocol = ev->Get()->Request.GetParams().Get("protocol"); + TString protocol = ev->Get()->Request.GetParams().Get("protocol"); if (protocol.empty()) { protocol = "http"; } @@ -402,8 +402,8 @@ private: } return; } - if (msg->Request.GetPathInfo().StartsWith("/json/")) { - auto itJson = JsonHandlers.find(msg->Request.GetPathInfo()); + if (msg->Request.GetPathInfo().StartsWith("/json/")) { + auto itJson = JsonHandlers.find(msg->Request.GetPathInfo()); if (itJson != JsonHandlers.end()) { try { ctx.ExecutorThread.RegisterActor(itJson->second->CreateRequestActor(this, ev)); @@ -424,7 +424,7 @@ private: } // TODO: check path validity // TODO: cache - if (msg->Request.GetPathInfo().StartsWith('/')) { + if (msg->Request.GetPathInfo().StartsWith('/')) { if (filename.StartsWith("viewer")) { filename.erase(0, 6); } @@ -446,13 +446,13 @@ private: return; } } - if (msg->Request.GetPathInfo() == "/tablet") { - ui64 id = FromStringWithDefault<ui64>(ev->Get()->Request.GetParams().Get("id"), 0); + if (msg->Request.GetPathInfo() == "/tablet") { + ui64 id = FromStringWithDefault<ui64>(ev->Get()->Request.GetParams().Get("id"), 0); ctx.Send(ev->Sender, new NMon::TEvHttpInfoRes("HTTP/1.1 302 Found\r\nLocation: ../tablets?TabletID=" + ToString(id) + "\r\n\r\n", 0, NMon::IEvHttpInfoRes::EContentType::Custom)); return; } - if (msg->Request.GetPathInfo().empty()) { - ctx.Send(ev->Sender, new NMon::TEvHttpInfoRes("HTTP/1.1 302 Found\r\nLocation: " + SplitPath(msg->Request.GetPage()->Path).back() + "/\r\n\r\n", 0, NMon::IEvHttpInfoRes::EContentType::Custom)); + if (msg->Request.GetPathInfo().empty()) { + ctx.Send(ev->Sender, new NMon::TEvHttpInfoRes("HTTP/1.1 302 Found\r\nLocation: " + SplitPath(msg->Request.GetPage()->Path).back() + "/\r\n\r\n", 0, NMon::IEvHttpInfoRes::EContentType::Custom)); return; } ctx.Send(ev->Sender, new NMon::TEvHttpInfoRes(NMonitoring::HTTPNOTFOUND)); diff --git a/ydb/library/schlab/mon/mon.cpp b/ydb/library/schlab/mon/mon.cpp index 6b8671edf4..fc06b412ae 100644 --- a/ydb/library/schlab/mon/mon.cpp +++ b/ydb/library/schlab/mon/mon.cpp @@ -25,7 +25,7 @@ public: , SchVizUrl(schVizUrl) {} - virtual void Output(IMonHttpRequest& request) { + virtual void Output(IMonHttpRequest& request) { TGuard<TMutex> guard(EmuMutex); TStringStream out; @@ -53,11 +53,11 @@ public: out << "exception: " << CurrentExceptionMessage(); } - request.Output() << out.Str(); + request.Output() << out.Str(); } private: - void OutputIndex(const IMonHttpRequest& request, IOutputStream& out) { + void OutputIndex(const IMonHttpRequest& request, IOutputStream& out) { TGuard<TMutex> guard(EmuMutex); Y_UNUSED(request); out << HTTPOKHTML; @@ -70,14 +70,14 @@ private: } } - void InputConfig(const IMonHttpRequest& request, IOutputStream& out) { + void InputConfig(const IMonHttpRequest& request, IOutputStream& out) { TGuard<TMutex> guard(EmuMutex); JsonConfig = TString(request.GetPostContent()); IsRunning = false; out << HTTPOKHTML; } - void GetJson(const IMonHttpRequest& request, IOutputStream& out) { + void GetJson(const IMonHttpRequest& request, IOutputStream& out) { TGuard<TMutex> guard(EmuMutex); Y_UNUSED(request); out << HTTPOKHTML; @@ -101,7 +101,7 @@ public: , DataUrl(dataUrl) {} - virtual void Output(IMonHttpRequest& request) { + virtual void Output(IMonHttpRequest& request) { TStringStream out; try { if (request.GetParams().Get("mode") == "") { @@ -117,11 +117,11 @@ public: out << "exception: " << CurrentExceptionMessage(); } - request.Output() << out.Str(); + request.Output() << out.Str(); } private: - void OutputIndex(const IMonHttpRequest& request, IOutputStream& out) + void OutputIndex(const IMonHttpRequest& request, IOutputStream& out) { Y_UNUSED(request); out << HTTPOKHTML; @@ -143,7 +143,7 @@ private: } } - void OutputJavaScript(const IMonHttpRequest& request, IOutputStream& out) + void OutputJavaScript(const IMonHttpRequest& request, IOutputStream& out) { Y_UNUSED(request); out << HTTPOKJAVASCRIPT; diff --git a/ydb/public/lib/deprecated/client/grpc_client.h b/ydb/public/lib/deprecated/client/grpc_client.h index d002e98aa3..b0483a2646 100644 --- a/ydb/public/lib/deprecated/client/grpc_client.h +++ b/ydb/public/lib/deprecated/client/grpc_client.h @@ -8,7 +8,7 @@ namespace NKikimr { namespace NGRpcProxy { - using TGRpcClientConfig = NGrpc::TGRpcClientConfig; + using TGRpcClientConfig = NGrpc::TGRpcClientConfig; using TGrpcError = std::pair<TString, int>; diff --git a/ydb/public/sdk/cpp/client/ydb_common_client/impl/client.h b/ydb/public/sdk/cpp/client/ydb_common_client/impl/client.h index 46cd0a9ede..c43e9aab69 100644 --- a/ydb/public/sdk/cpp/client/ydb_common_client/impl/client.h +++ b/ydb/public/sdk/cpp/client/ydb_common_client/impl/client.h @@ -45,7 +45,7 @@ public: protected: template<typename TService, typename TRequest, typename TResponse> - using TAsyncRequest = typename NGrpc::TSimpleRequestProcessor< + using TAsyncRequest = typename NGrpc::TSimpleRequestProcessor< typename TService::Stub, TRequest, TResponse>::TAsyncRequest; diff --git a/ydb/public/sdk/cpp/client/ydb_coordination/coordination.cpp b/ydb/public/sdk/cpp/client/ydb_coordination/coordination.cpp index d07d12990b..7a99c95214 100644 --- a/ydb/public/sdk/cpp/client/ydb_coordination/coordination.cpp +++ b/ydb/public/sdk/cpp/client/ydb_coordination/coordination.cpp @@ -17,7 +17,7 @@ namespace NCoordination { using NThreading::TFuture; using NThreading::TPromise; using NThreading::NewPromise; -using NGrpc::TQueueClientFixedEvent; +using NGrpc::TQueueClientFixedEvent; namespace { @@ -177,8 +177,8 @@ class TSessionContext : public TThrRefBase { using TService = Ydb::Coordination::V1::CoordinationService; using TRequest = Ydb::Coordination::SessionRequest; using TResponse = Ydb::Coordination::SessionResponse; - using TGrpcStatus = NGrpc::TGrpcStatus; - using IProcessor = NGrpc::IStreamRequestReadWriteProcessor<TRequest, TResponse>; + using TGrpcStatus = NGrpc::TGrpcStatus; + using IProcessor = NGrpc::IStreamRequestReadWriteProcessor<TRequest, TResponse>; friend class TSession::TImpl; diff --git a/ydb/public/sdk/cpp/client/ydb_driver/driver.cpp b/ydb/public/sdk/cpp/client/ydb_driver/driver.cpp index f420baad15..de17c6e679 100644 --- a/ydb/public/sdk/cpp/client/ydb_driver/driver.cpp +++ b/ydb/public/sdk/cpp/client/ydb_driver/driver.cpp @@ -14,13 +14,13 @@ namespace NYdb { -using NGrpc::TGRpcClientLow; -using NGrpc::TServiceConnection; -using NGrpc::TSimpleRequestProcessor; -using NGrpc::TGRpcClientConfig; -using NGrpc::TResponseCallback; -using NGrpc::TGrpcStatus; -using NGrpc::TTcpKeepAliveSettings; +using NGrpc::TGRpcClientLow; +using NGrpc::TServiceConnection; +using NGrpc::TSimpleRequestProcessor; +using NGrpc::TGRpcClientConfig; +using NGrpc::TResponseCallback; +using NGrpc::TGrpcStatus; +using NGrpc::TTcpKeepAliveSettings; using Ydb::StatusIds; @@ -57,7 +57,7 @@ public: std::shared_ptr<ICredentialsProviderFactory> CredentialsProviderFactory = CreateInsecureCredentialsProviderFactory(); EDiscoveryMode DiscoveryMode = EDiscoveryMode::Sync; size_t MaxQueuedRequests = 100; - NGrpc::TTcpKeepAliveSettings TcpKeepAliveSettings = + NGrpc::TTcpKeepAliveSettings TcpKeepAliveSettings = { true, TCP_KEEPALIVE_IDLE, diff --git a/ydb/public/sdk/cpp/client/ydb_table/table.cpp b/ydb/public/sdk/cpp/client/ydb_table/table.cpp index a044b8e6fa..610fec3884 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/table.cpp +++ b/ydb/public/sdk/cpp/client/ydb_table/table.cpp @@ -1177,9 +1177,9 @@ class TTablePartIterator::TReaderImpl { public: using TSelf = TTablePartIterator::TReaderImpl; using TResponse = Ydb::Table::ReadTableResponse; - using TStreamProcessorPtr = NGrpc::IStreamRequestReadProcessor<TResponse>::TPtr; - using TReadCallback = NGrpc::IStreamRequestReadProcessor<TResponse>::TReadCallback; - using TGRpcStatus = NGrpc::TGrpcStatus; + using TStreamProcessorPtr = NGrpc::IStreamRequestReadProcessor<TResponse>::TPtr; + using TReadCallback = NGrpc::IStreamRequestReadProcessor<TResponse>::TReadCallback; + using TGRpcStatus = NGrpc::TGrpcStatus; using TBatchReadResult = std::pair<TResponse, TGRpcStatus>; TReaderImpl(TStreamProcessorPtr streamProcessor, const TString& endpoint) diff --git a/ydb/services/ydb/ydb_stats_ut.cpp b/ydb/services/ydb/ydb_stats_ut.cpp index b0b65ce127..b40e22fc42 100644 --- a/ydb/services/ydb/ydb_stats_ut.cpp +++ b/ydb/services/ydb/ydb_stats_ut.cpp @@ -72,7 +72,7 @@ public: void OnLabelsBegin() override { } void OnLabelsEnd() override { } - void OnLabel(const TStringBuf name, const TStringBuf value) override { + void OnLabel(const TStringBuf name, const TStringBuf value) override { if (name != "sensor") { return; } diff --git a/ydb/services/ydb/ydb_ut.cpp b/ydb/services/ydb/ydb_ut.cpp index aa6692423e..2d37063079 100644 --- a/ydb/services/ydb/ydb_ut.cpp +++ b/ydb/services/ydb/ydb_ut.cpp @@ -124,13 +124,13 @@ Y_UNIT_TEST_SUITE(TGRpcClientLowTest) { bool allDoneOk = false; { - NGrpc::TGRpcClientLow clientLow; + NGrpc::TGRpcClientLow clientLow; auto connection = clientLow.CreateGRpcServiceConnection<Ydb::Table::V1::TableService>(clientConfig); Ydb::Table::CreateSessionRequest request; - NGrpc::TResponseCallback<Ydb::Table::CreateSessionResponse> responseCb = - [&allDoneOk](NGrpc::TGrpcStatus&& grpcStatus, Ydb::Table::CreateSessionResponse&& response) -> void { + NGrpc::TResponseCallback<Ydb::Table::CreateSessionResponse> responseCb = + [&allDoneOk](NGrpc::TGrpcStatus&& grpcStatus, Ydb::Table::CreateSessionResponse&& response) -> void { UNIT_ASSERT(!grpcStatus.InternalError); UNIT_ASSERT(grpcStatus.GRpcStatusCode == 0); auto deferred = response.operation(); @@ -151,15 +151,15 @@ Y_UNIT_TEST_SUITE(TGRpcClientLowTest) { bool allDoneOk = false; { - NGrpc::TGRpcClientLow clientLow; + NGrpc::TGRpcClientLow clientLow; auto connection = clientLow.CreateGRpcServiceConnection<Draft::Dummy::DummyService>(clientConfig); Draft::Dummy::PingRequest request; request.set_copy(true); request.set_payload("abc"); - NGrpc::TResponseCallback<Draft::Dummy::PingResponse> responseCb = - [&allDoneOk](NGrpc::TGrpcStatus&& grpcStatus, Draft::Dummy::PingResponse&& response) -> void { + NGrpc::TResponseCallback<Draft::Dummy::PingResponse> responseCb = + [&allDoneOk](NGrpc::TGrpcStatus&& grpcStatus, Draft::Dummy::PingResponse&& response) -> void { UNIT_ASSERT(!grpcStatus.InternalError); UNIT_ASSERT(grpcStatus.GRpcStatusCode == 0); UNIT_ASSERT(response.payload() == "abc"); @@ -361,13 +361,13 @@ Y_UNIT_TEST_SUITE(TGRpcClientLowTest) { const size_t numRequests = 1000; { - NGrpc::TGRpcClientLow clientLow; + NGrpc::TGRpcClientLow clientLow; auto connection = clientLow.CreateGRpcServiceConnection<Ydb::Table::V1::TableService>(clientConfig); Ydb::Table::CreateSessionRequest request; for (size_t i = 0; i < numRequests; i++) { - NGrpc::TResponseCallback<Ydb::Table::CreateSessionResponse> responseCb = - [&okResponseCounter](NGrpc::TGrpcStatus&& grpcStatus, Ydb::Table::CreateSessionResponse&& response) -> void { + NGrpc::TResponseCallback<Ydb::Table::CreateSessionResponse> responseCb = + [&okResponseCounter](NGrpc::TGrpcStatus&& grpcStatus, Ydb::Table::CreateSessionResponse&& response) -> void { UNIT_ASSERT(!grpcStatus.InternalError); UNIT_ASSERT(grpcStatus.GRpcStatusCode == 0); auto deferred = response.operation(); @@ -388,7 +388,7 @@ Y_UNIT_TEST_SUITE(TGRpcClientLowTest) { auto clientConfig = NGRpcProxy::TGRpcClientConfig(location); { - NGrpc::TGRpcClientLow clientLow; + NGrpc::TGRpcClientLow clientLow; auto connection = clientLow.CreateGRpcServiceConnection<Ydb::Scheme::V1::SchemeService>(clientConfig); Ydb::Scheme::MakeDirectoryRequest request; @@ -396,8 +396,8 @@ Y_UNIT_TEST_SUITE(TGRpcClientLowTest) { "path: \"/Root/TheDirectory\""); ::google::protobuf::TextFormat::ParseFromString(scheme, &request); - NGrpc::TResponseCallback<Ydb::Scheme::MakeDirectoryResponse> responseCb = - [](NGrpc::TGrpcStatus&& grpcStatus, Ydb::Scheme::MakeDirectoryResponse&& response) -> void { + NGrpc::TResponseCallback<Ydb::Scheme::MakeDirectoryResponse> responseCb = + [](NGrpc::TGrpcStatus&& grpcStatus, Ydb::Scheme::MakeDirectoryResponse&& response) -> void { UNIT_ASSERT(!grpcStatus.InternalError); UNIT_ASSERT(grpcStatus.GRpcStatusCode == 0); auto deferred = response.operation(); @@ -407,7 +407,7 @@ Y_UNIT_TEST_SUITE(TGRpcClientLowTest) { connection->DoRequest(request, std::move(responseCb), &Ydb::Scheme::V1::SchemeService::Stub::AsyncMakeDirectory); } { - NGrpc::TGRpcClientLow clientLow; + NGrpc::TGRpcClientLow clientLow; auto connection = clientLow.CreateGRpcServiceConnection<Ydb::Scheme::V1::SchemeService>(clientConfig); Ydb::Scheme::ModifyPermissionsRequest request; @@ -419,8 +419,8 @@ Y_UNIT_TEST_SUITE(TGRpcClientLowTest) { perm->add_permission_names("ydb.generic.read"); - NGrpc::TResponseCallback<Ydb::Scheme::ModifyPermissionsResponse> responseCb = - [](NGrpc::TGrpcStatus&& grpcStatus, Ydb::Scheme::ModifyPermissionsResponse&& response) -> void { + NGrpc::TResponseCallback<Ydb::Scheme::ModifyPermissionsResponse> responseCb = + [](NGrpc::TGrpcStatus&& grpcStatus, Ydb::Scheme::ModifyPermissionsResponse&& response) -> void { UNIT_ASSERT(!grpcStatus.InternalError); UNIT_ASSERT(grpcStatus.GRpcStatusCode == 0); auto deferred = response.operation(); @@ -431,7 +431,7 @@ Y_UNIT_TEST_SUITE(TGRpcClientLowTest) { connection->DoRequest(request, std::move(responseCb), &Ydb::Scheme::V1::SchemeService::Stub::AsyncModifyPermissions); } { - NGrpc::TGRpcClientLow clientLow; + NGrpc::TGRpcClientLow clientLow; auto connection = clientLow.CreateGRpcServiceConnection<Ydb::Scheme::V1::SchemeService>(clientConfig); Ydb::Scheme::DescribePathRequest request; @@ -439,8 +439,8 @@ Y_UNIT_TEST_SUITE(TGRpcClientLowTest) { "path: \"/Root/TheDirectory\""); ::google::protobuf::TextFormat::ParseFromString(scheme, &request); - NGrpc::TResponseCallback<Ydb::Scheme::DescribePathResponse> responseCb = - [](NGrpc::TGrpcStatus&& grpcStatus, Ydb::Scheme::DescribePathResponse&& response) -> void { + NGrpc::TResponseCallback<Ydb::Scheme::DescribePathResponse> responseCb = + [](NGrpc::TGrpcStatus&& grpcStatus, Ydb::Scheme::DescribePathResponse&& response) -> void { UNIT_ASSERT(!grpcStatus.InternalError); UNIT_ASSERT(grpcStatus.GRpcStatusCode == 0); auto deferred = response.operation(); |