diff options
author | monster <monster@yandex-team.ru> | 2022-02-10 16:47:19 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:47:19 +0300 |
commit | b23c1d7a8015c2006a148fd93b84cdeb0aee17a3 (patch) | |
tree | 9814fbd1c3effac9b8377c5d604b367b14e2db55 /library/cpp | |
parent | dd76ae1f6213d065375ab296699f764faafbe5bd (diff) | |
download | ydb-b23c1d7a8015c2006a148fd93b84cdeb0aee17a3.tar.gz |
Restoring authorship annotation for <monster@yandex-team.ru>. Commit 2 of 2.
Diffstat (limited to 'library/cpp')
74 files changed, 2345 insertions, 2345 deletions
diff --git a/library/cpp/actors/core/README.md b/library/cpp/actors/core/README.md index 3cd4354c6d..439a8dd459 100644 --- a/library/cpp/actors/core/README.md +++ b/library/cpp/actors/core/README.md @@ -1,99 +1,99 @@ -## Memory tracker - -https://a.yandex-team.ru/arc/trunk/arcadia/library/cpp/actors/core/memory_track.h - -Использование: - -* отслеживание аллокаций экземпляров конкретного класса через new/delete и new[]/delete[] -* отслеживание аллокаций в контейнерах -* ручное отслеживание моментов аллокации/деаллокации - ----- - -### Отслеживание аллокаций класса через new/delete - -Использование с автоматически генерируемой меткой: - -```cpp -#include <library/cpp/actors/core/memory_track.h> - -struct TTypeLabeled - : public NActors::NMemory::TTrack<TTypeLabeled> -{ - char payload[16]; -}; -``` - -Использование с пользовательским именем метки: - -```cpp -#include <library/cpp/actors/core/memory_track.h> - -static const char NamedLabel[] = "NamedLabel"; - -struct TNameLabeled - : public NActors::NMemory::TTrack<TNameLabeled, NamedLabel> -{ - char payload[32]; -}; -``` - ----- - -### Отслеживание аллокаций в контейнерах - -```cpp -#include <library/cpp/actors/core/memory_track.h> - -static const char InContainerLabel[] = "InContainerLabel"; - -struct TInContainer { - char payload[16]; -}; - -std::vector<TInContainer, NActors::NMemory::TAlloc<TInContainer>> vecT; - -std::vector<TInContainer, NActors::NMemory::TAlloc<TInContainer, InContainerLabel>> vecN; - -using TKey = int; - -std::map<TKey, TInContainer, std::less<TKey>, - NActors::NMemory::TAlloc<std::pair<const TKey, TInContainer>>> mapT; - -std::map<TKey, TInContainer, std::less<TKey>, - NActors::NMemory::TAlloc<std::pair<const TKey, TInContainer>, InContainerLabel>> mapN; - -std::unordered_map<TKey, TInContainer, std::hash<TKey>, std::equal_to<TKey>, - NActors::NMemory::TAlloc<std::pair<const TKey, TInContainer>>> umapT; - -std::unordered_map<TKey, TInContainer, std::hash<TKey>, std::equal_to<TKey>, - NActors::NMemory::TAlloc<std::pair<const TKey, TInContainer>, InContainerLabel>> umapN; -``` - ----- - -### Ручное отслеживание аллокаций/деаллокаций - -```cpp -#include <library/cpp/actors/core/memory_track.h> - -static const char ManualLabel[] = "ManualLabel"; - -... -NActors::NMemory::TLabel<ManualLabel>::Add(size); - -... -NActors::NMemory::TLabel<ManualLabel>::Sub(size); -``` - ----- - -### Собираемые метрики - -Сервис **utils**, пользовательская метка **label**, сенсоры: - -- MT/Count: количество аллокаций в моменте -- MT/Memory: аллоцированная память в моменте -- MT/PeakCount: пиковое значение количества аллокаций (сэмплится с фиксированной частотой) -- MT/PeakMemory: пиковое значение аллоцированной памяти - +## Memory tracker + +https://a.yandex-team.ru/arc/trunk/arcadia/library/cpp/actors/core/memory_track.h + +Использование: + +* отслеживание аллокаций экземпляров конкретного класса через new/delete и new[]/delete[] +* отслеживание аллокаций в контейнерах +* ручное отслеживание моментов аллокации/деаллокации + +---- + +### Отслеживание аллокаций класса через new/delete + +Использование с автоматически генерируемой меткой: + +```cpp +#include <library/cpp/actors/core/memory_track.h> + +struct TTypeLabeled + : public NActors::NMemory::TTrack<TTypeLabeled> +{ + char payload[16]; +}; +``` + +Использование с пользовательским именем метки: + +```cpp +#include <library/cpp/actors/core/memory_track.h> + +static const char NamedLabel[] = "NamedLabel"; + +struct TNameLabeled + : public NActors::NMemory::TTrack<TNameLabeled, NamedLabel> +{ + char payload[32]; +}; +``` + +---- + +### Отслеживание аллокаций в контейнерах + +```cpp +#include <library/cpp/actors/core/memory_track.h> + +static const char InContainerLabel[] = "InContainerLabel"; + +struct TInContainer { + char payload[16]; +}; + +std::vector<TInContainer, NActors::NMemory::TAlloc<TInContainer>> vecT; + +std::vector<TInContainer, NActors::NMemory::TAlloc<TInContainer, InContainerLabel>> vecN; + +using TKey = int; + +std::map<TKey, TInContainer, std::less<TKey>, + NActors::NMemory::TAlloc<std::pair<const TKey, TInContainer>>> mapT; + +std::map<TKey, TInContainer, std::less<TKey>, + NActors::NMemory::TAlloc<std::pair<const TKey, TInContainer>, InContainerLabel>> mapN; + +std::unordered_map<TKey, TInContainer, std::hash<TKey>, std::equal_to<TKey>, + NActors::NMemory::TAlloc<std::pair<const TKey, TInContainer>>> umapT; + +std::unordered_map<TKey, TInContainer, std::hash<TKey>, std::equal_to<TKey>, + NActors::NMemory::TAlloc<std::pair<const TKey, TInContainer>, InContainerLabel>> umapN; +``` + +---- + +### Ручное отслеживание аллокаций/деаллокаций + +```cpp +#include <library/cpp/actors/core/memory_track.h> + +static const char ManualLabel[] = "ManualLabel"; + +... +NActors::NMemory::TLabel<ManualLabel>::Add(size); + +... +NActors::NMemory::TLabel<ManualLabel>::Sub(size); +``` + +---- + +### Собираемые метрики + +Сервис **utils**, пользовательская метка **label**, сенсоры: + +- MT/Count: количество аллокаций в моменте +- MT/Memory: аллоцированная память в моменте +- MT/PeakCount: пиковое значение количества аллокаций (сэмплится с фиксированной частотой) +- MT/PeakMemory: пиковое значение аллоцированной памяти + diff --git a/library/cpp/actors/core/actorsystem.h b/library/cpp/actors/core/actorsystem.h index 7df0a9f633..40499d7586 100644 --- a/library/cpp/actors/core/actorsystem.h +++ b/library/cpp/actors/core/actorsystem.h @@ -332,8 +332,8 @@ namespace NActors { TActorId LookupLocalService(const TActorId& x) const; TActorId RegisterLocalService(const TActorId& serviceId, const TActorId& actorId); - ui32 GetMaxActivityType() const { - return SystemSetup ? SystemSetup->MaxActivityType : 1; + ui32 GetMaxActivityType() const { + return SystemSetup ? SystemSetup->MaxActivityType : 1; } TInstant Timestamp() const { diff --git a/library/cpp/actors/core/event.h b/library/cpp/actors/core/event.h index dbdb45daac..6ff02aaf94 100644 --- a/library/cpp/actors/core/event.h +++ b/library/cpp/actors/core/event.h @@ -46,8 +46,8 @@ namespace NActors { }; // fat handle - class IEventHandle : TNonCopyable { - struct TOnNondelivery { + class IEventHandle : TNonCopyable { + struct TOnNondelivery { TActorId Recipient; TOnNondelivery(const TActorId& recipient) diff --git a/library/cpp/actors/core/executor_thread.cpp b/library/cpp/actors/core/executor_thread.cpp index ae5f111dff..446b651efd 100644 --- a/library/cpp/actors/core/executor_thread.cpp +++ b/library/cpp/actors/core/executor_thread.cpp @@ -137,7 +137,7 @@ namespace NActors { NHPTimer::STime hpprev = hpstart; IActor* actor = nullptr; - ui32 prevActivityType = std::numeric_limits<ui32>::max(); + ui32 prevActivityType = std::numeric_limits<ui32>::max(); TActorId recipient; for (ui32 executed = 0; executed < Ctx.EventsPerMailbox; ++executed) { TAutoPtr<IEventHandle> ev(mailbox->Pop()); @@ -170,14 +170,14 @@ namespace NActors { ui32 evTypeForTracing = ev->Type; - ui32 activityType = actor->GetActivityType(); - if (activityType != prevActivityType) { - prevActivityType = activityType; - NProfiling::TMemoryTagScope::Reset(ActorSystem->MemProfActivityBase + activityType); + ui32 activityType = actor->GetActivityType(); + if (activityType != prevActivityType) { + prevActivityType = activityType; + NProfiling::TMemoryTagScope::Reset(ActorSystem->MemProfActivityBase + activityType); } - actor->Receive(ev, ctx); - + actor->Receive(ev, ctx); + size_t dyingActorsCnt = DyingActors.size(); Ctx.UpdateActorsStats(dyingActorsCnt); if (dyingActorsCnt) { @@ -273,7 +273,7 @@ namespace NActors { } } - NProfiling::TMemoryTagScope::Reset(0); + NProfiling::TMemoryTagScope::Reset(0); TlsActivationContext = nullptr; UnlockFromExecution(mailbox, Ctx.Executor, reclaimAsFree, hint, Ctx.WorkerId, RevolvingWriteCounter); } diff --git a/library/cpp/actors/core/mailbox.h b/library/cpp/actors/core/mailbox.h index 277a259932..0bd9c4d314 100644 --- a/library/cpp/actors/core/mailbox.h +++ b/library/cpp/actors/core/mailbox.h @@ -397,7 +397,7 @@ namespace NActors { static_assert(sizeof(TRevolvingMailbox) == 128, "expect sizeof(TRevolvingMailbox) == 128"); struct THTSwapMailbox: public TMailboxHeader { - using TQueueType = NThreading::THTSwapQueue<IEventHandle*>; + using TQueueType = NThreading::THTSwapQueue<IEventHandle*>; TQueueType Queue; NHPTimer::STime ScheduleMoment; @@ -447,7 +447,7 @@ namespace NActors { "expect sizeof(THTSwapMailbox) == 64"); struct TReadAsFilledMailbox: public TMailboxHeader { - using TQueueType = NThreading::TReadAsFilledQueue<IEventHandle>; + using TQueueType = NThreading::TReadAsFilledQueue<IEventHandle>; TQueueType Queue; NHPTimer::STime ScheduleMoment; diff --git a/library/cpp/actors/core/memory_track.cpp b/library/cpp/actors/core/memory_track.cpp index 79392ef686..5f422116be 100644 --- a/library/cpp/actors/core/memory_track.cpp +++ b/library/cpp/actors/core/memory_track.cpp @@ -1,38 +1,38 @@ -#include "memory_track.h" -#include "memory_tracker.h" - -namespace NActors { -namespace NMemory { - -namespace NPrivate { - -TThreadLocalInfo::TThreadLocalInfo() - : Metrics(TMemoryTracker::Instance()->GetCount()) -{ - TMemoryTracker::Instance()->OnCreateThread(this); -} - -TThreadLocalInfo::~TThreadLocalInfo() { - TMemoryTracker::Instance()->OnDestroyThread(this); -} - -TMetric* TThreadLocalInfo::GetMetric(size_t index) { - if (Y_UNLIKELY(index >= Metrics.size())) { - return &Null; - } - return &Metrics[index]; -} - -const std::vector<TMetric>& TThreadLocalInfo::GetMetrics() const { - return Metrics; -} - -size_t TBaseLabel::RegisterStaticMemoryLabel(const char* name, bool hasSensor) { - return TMemoryTracker::Instance()->RegisterStaticMemoryLabel(name, hasSensor); -} - -} - -} -} - +#include "memory_track.h" +#include "memory_tracker.h" + +namespace NActors { +namespace NMemory { + +namespace NPrivate { + +TThreadLocalInfo::TThreadLocalInfo() + : Metrics(TMemoryTracker::Instance()->GetCount()) +{ + TMemoryTracker::Instance()->OnCreateThread(this); +} + +TThreadLocalInfo::~TThreadLocalInfo() { + TMemoryTracker::Instance()->OnDestroyThread(this); +} + +TMetric* TThreadLocalInfo::GetMetric(size_t index) { + if (Y_UNLIKELY(index >= Metrics.size())) { + return &Null; + } + return &Metrics[index]; +} + +const std::vector<TMetric>& TThreadLocalInfo::GetMetrics() const { + return Metrics; +} + +size_t TBaseLabel::RegisterStaticMemoryLabel(const char* name, bool hasSensor) { + return TMemoryTracker::Instance()->RegisterStaticMemoryLabel(name, hasSensor); +} + +} + +} +} + diff --git a/library/cpp/actors/core/memory_track.h b/library/cpp/actors/core/memory_track.h index 97a7c5dfae..6035333eeb 100644 --- a/library/cpp/actors/core/memory_track.h +++ b/library/cpp/actors/core/memory_track.h @@ -1,293 +1,293 @@ -#pragma once - -#include <vector> - +#pragma once + +#include <vector> + #include <util/system/type_name.h> -#include <util/thread/singleton.h> - -#define ENABLE_MEMORY_TRACKING - -namespace NActors { -namespace NMemory { - -namespace NPrivate { - -class TMetric { - std::atomic<ssize_t> Memory; - std::atomic<ssize_t> Count; - - void Copy(const TMetric& other) { - Memory.store(other.GetMemory(), std::memory_order_relaxed); - Count.store(other.GetCount(), std::memory_order_relaxed); - } - -public: - TMetric() - : Memory(0) - , Count(0) - {} - - inline TMetric(const TMetric& other) { - Copy(other); - } - - inline TMetric(TMetric&& other) { - Copy(other); - } - - inline TMetric& operator=(const TMetric& other) { - Copy(other); - return *this; - } - - inline TMetric& operator=(TMetric&& other) { - Copy(other); - return *this; - } - - inline ssize_t GetMemory() const { - return Memory.load(std::memory_order_relaxed); - } - inline void SetMemory(ssize_t value) { - Memory.store(value, std::memory_order_relaxed); - } - - inline ssize_t GetCount() const { - return Count.load(std::memory_order_relaxed); - } - inline void SetCount(ssize_t value) { - Count.store(value, std::memory_order_relaxed); - } - - inline void operator+=(const TMetric& other) { - SetMemory(GetMemory() + other.GetMemory()); - SetCount(GetCount() + other.GetCount()); - } - - inline void CalculatePeak(const TMetric& other) { - SetMemory(Max(GetMemory(), other.GetMemory())); - SetCount(Max(GetCount(), other.GetCount())); - } - - inline void Add(size_t size) { - SetMemory(GetMemory() + size); - SetCount(GetCount() + 1); - } - - inline void Sub(size_t size) { - SetMemory(GetMemory() - size); - SetCount(GetCount() - 1); - } -}; - - -class TThreadLocalInfo { -public: - TThreadLocalInfo(); - ~TThreadLocalInfo(); - - TMetric* GetMetric(size_t index); - const std::vector<TMetric>& GetMetrics() const; - -private: - std::vector<TMetric> Metrics; - - inline static TMetric Null = {}; -}; - - -class TBaseLabel { -protected: - static size_t RegisterStaticMemoryLabel(const char* name, bool hasSensor); - - inline static TMetric* GetLocalMetric(size_t index) { - return FastTlsSingleton<TThreadLocalInfo>()->GetMetric(index); - } -}; - - -template <const char* Name> -class TNameLabel - : TBaseLabel -{ -public: - static void Add(size_t size) { -#if defined(ENABLE_MEMORY_TRACKING) - Y_UNUSED(MetricInit); - - if (Y_UNLIKELY(!Metric)) { - Metric = GetLocalMetric(Index); - } - - Metric->Add(size); -#else - Y_UNUSED(size); -#endif - } - - static void Sub(size_t size) { -#if defined(ENABLE_MEMORY_TRACKING) - Y_UNUSED(MetricInit); - - if (Y_UNLIKELY(!Metric)) { - Metric = GetLocalMetric(Index); - } - - Metric->Sub(size); -#else - Y_UNUSED(size); -#endif - } - -private: -#if defined(ENABLE_MEMORY_TRACKING) - inline static size_t Index = Max<size_t>(); - inline static struct TMetricInit { - TMetricInit() { - Index = RegisterStaticMemoryLabel(Name, true); - } - } MetricInit; - - inline static thread_local TMetric* Metric = nullptr; -#endif -}; - - -template <typename TType> -class TTypeLabel - : TBaseLabel -{ -public: - static void Add(size_t size) { -#if defined(ENABLE_MEMORY_TRACKING) - Y_UNUSED(MetricInit); - - if (Y_UNLIKELY(!Metric)) { - Metric = GetLocalMetric(Index); - } - - Metric->Add(size); -#else - Y_UNUSED(size); -#endif - } - - static void Sub(size_t size) { -#if defined(ENABLE_MEMORY_TRACKING) - Y_UNUSED(MetricInit); - - if (Y_UNLIKELY(!Metric)) { - Metric = GetLocalMetric(Index); - } - - Metric->Sub(size); -#else - Y_UNUSED(size); -#endif - } - -private: -#if defined(ENABLE_MEMORY_TRACKING) - inline static size_t Index = Max<size_t>(); - inline static struct TMetricInit { - TMetricInit() { - Index = RegisterStaticMemoryLabel(TypeName<TType>().c_str(), false); - } - } MetricInit; - - inline static thread_local TMetric* Metric = nullptr; -#endif -}; - - -template <typename T> -struct TTrackHelper { -#if defined(ENABLE_MEMORY_TRACKING) - void* operator new(size_t size) { - T::Add(size); - return malloc(size); - } - - void* operator new[](size_t size) { - T::Add(size); - return malloc(size); - } - - void operator delete(void* ptr, size_t size) { - T::Sub(size); - free(ptr); - } - - void operator delete[](void* ptr, size_t size) { - T::Sub(size); - free(ptr); - } -#endif -}; - -template <typename TType, typename T> -struct TAllocHelper { - typedef size_t size_type; - typedef TType value_type; - typedef TType* pointer; - typedef const TType* const_pointer; - - struct propagate_on_container_copy_assignment : public std::false_type {}; - struct propagate_on_container_move_assignment : public std::false_type {}; - struct propagate_on_container_swap : public std::false_type {}; - - pointer allocate(size_type n, const void* hint = nullptr) { - Y_UNUSED(hint); - auto size = n * sizeof(TType); - T::Add(size); - return (pointer)malloc(size); - } - - void deallocate(pointer ptr, size_t n) { - auto size = n * sizeof(TType); - T::Sub(size); - free((void*)ptr); - } -}; - -} // NPrivate - - -template <const char* Name> -using TLabel = NPrivate::TNameLabel<Name>; - -template <typename TType, const char* Name = nullptr> -struct TTrack - : public NPrivate::TTrackHelper<NPrivate::TNameLabel<Name>> -{ -}; - -template <typename TType> -struct TTrack<TType, nullptr> - : public NPrivate::TTrackHelper<NPrivate::TTypeLabel<TType>> -{ -}; - -template <typename TType, const char* Name = nullptr> -struct TAlloc - : public NPrivate::TAllocHelper<TType, NPrivate::TNameLabel<Name>> -{ - template<typename U> - struct rebind { - typedef TAlloc<U, Name> other; - }; -}; - -template <typename TType> -struct TAlloc<TType, nullptr> - : public NPrivate::TAllocHelper<TType, NPrivate::TTypeLabel<TType>> -{ - template<typename U> - struct rebind { - typedef TAlloc<U> other; - }; -}; - -} -} - +#include <util/thread/singleton.h> + +#define ENABLE_MEMORY_TRACKING + +namespace NActors { +namespace NMemory { + +namespace NPrivate { + +class TMetric { + std::atomic<ssize_t> Memory; + std::atomic<ssize_t> Count; + + void Copy(const TMetric& other) { + Memory.store(other.GetMemory(), std::memory_order_relaxed); + Count.store(other.GetCount(), std::memory_order_relaxed); + } + +public: + TMetric() + : Memory(0) + , Count(0) + {} + + inline TMetric(const TMetric& other) { + Copy(other); + } + + inline TMetric(TMetric&& other) { + Copy(other); + } + + inline TMetric& operator=(const TMetric& other) { + Copy(other); + return *this; + } + + inline TMetric& operator=(TMetric&& other) { + Copy(other); + return *this; + } + + inline ssize_t GetMemory() const { + return Memory.load(std::memory_order_relaxed); + } + inline void SetMemory(ssize_t value) { + Memory.store(value, std::memory_order_relaxed); + } + + inline ssize_t GetCount() const { + return Count.load(std::memory_order_relaxed); + } + inline void SetCount(ssize_t value) { + Count.store(value, std::memory_order_relaxed); + } + + inline void operator+=(const TMetric& other) { + SetMemory(GetMemory() + other.GetMemory()); + SetCount(GetCount() + other.GetCount()); + } + + inline void CalculatePeak(const TMetric& other) { + SetMemory(Max(GetMemory(), other.GetMemory())); + SetCount(Max(GetCount(), other.GetCount())); + } + + inline void Add(size_t size) { + SetMemory(GetMemory() + size); + SetCount(GetCount() + 1); + } + + inline void Sub(size_t size) { + SetMemory(GetMemory() - size); + SetCount(GetCount() - 1); + } +}; + + +class TThreadLocalInfo { +public: + TThreadLocalInfo(); + ~TThreadLocalInfo(); + + TMetric* GetMetric(size_t index); + const std::vector<TMetric>& GetMetrics() const; + +private: + std::vector<TMetric> Metrics; + + inline static TMetric Null = {}; +}; + + +class TBaseLabel { +protected: + static size_t RegisterStaticMemoryLabel(const char* name, bool hasSensor); + + inline static TMetric* GetLocalMetric(size_t index) { + return FastTlsSingleton<TThreadLocalInfo>()->GetMetric(index); + } +}; + + +template <const char* Name> +class TNameLabel + : TBaseLabel +{ +public: + static void Add(size_t size) { +#if defined(ENABLE_MEMORY_TRACKING) + Y_UNUSED(MetricInit); + + if (Y_UNLIKELY(!Metric)) { + Metric = GetLocalMetric(Index); + } + + Metric->Add(size); +#else + Y_UNUSED(size); +#endif + } + + static void Sub(size_t size) { +#if defined(ENABLE_MEMORY_TRACKING) + Y_UNUSED(MetricInit); + + if (Y_UNLIKELY(!Metric)) { + Metric = GetLocalMetric(Index); + } + + Metric->Sub(size); +#else + Y_UNUSED(size); +#endif + } + +private: +#if defined(ENABLE_MEMORY_TRACKING) + inline static size_t Index = Max<size_t>(); + inline static struct TMetricInit { + TMetricInit() { + Index = RegisterStaticMemoryLabel(Name, true); + } + } MetricInit; + + inline static thread_local TMetric* Metric = nullptr; +#endif +}; + + +template <typename TType> +class TTypeLabel + : TBaseLabel +{ +public: + static void Add(size_t size) { +#if defined(ENABLE_MEMORY_TRACKING) + Y_UNUSED(MetricInit); + + if (Y_UNLIKELY(!Metric)) { + Metric = GetLocalMetric(Index); + } + + Metric->Add(size); +#else + Y_UNUSED(size); +#endif + } + + static void Sub(size_t size) { +#if defined(ENABLE_MEMORY_TRACKING) + Y_UNUSED(MetricInit); + + if (Y_UNLIKELY(!Metric)) { + Metric = GetLocalMetric(Index); + } + + Metric->Sub(size); +#else + Y_UNUSED(size); +#endif + } + +private: +#if defined(ENABLE_MEMORY_TRACKING) + inline static size_t Index = Max<size_t>(); + inline static struct TMetricInit { + TMetricInit() { + Index = RegisterStaticMemoryLabel(TypeName<TType>().c_str(), false); + } + } MetricInit; + + inline static thread_local TMetric* Metric = nullptr; +#endif +}; + + +template <typename T> +struct TTrackHelper { +#if defined(ENABLE_MEMORY_TRACKING) + void* operator new(size_t size) { + T::Add(size); + return malloc(size); + } + + void* operator new[](size_t size) { + T::Add(size); + return malloc(size); + } + + void operator delete(void* ptr, size_t size) { + T::Sub(size); + free(ptr); + } + + void operator delete[](void* ptr, size_t size) { + T::Sub(size); + free(ptr); + } +#endif +}; + +template <typename TType, typename T> +struct TAllocHelper { + typedef size_t size_type; + typedef TType value_type; + typedef TType* pointer; + typedef const TType* const_pointer; + + struct propagate_on_container_copy_assignment : public std::false_type {}; + struct propagate_on_container_move_assignment : public std::false_type {}; + struct propagate_on_container_swap : public std::false_type {}; + + pointer allocate(size_type n, const void* hint = nullptr) { + Y_UNUSED(hint); + auto size = n * sizeof(TType); + T::Add(size); + return (pointer)malloc(size); + } + + void deallocate(pointer ptr, size_t n) { + auto size = n * sizeof(TType); + T::Sub(size); + free((void*)ptr); + } +}; + +} // NPrivate + + +template <const char* Name> +using TLabel = NPrivate::TNameLabel<Name>; + +template <typename TType, const char* Name = nullptr> +struct TTrack + : public NPrivate::TTrackHelper<NPrivate::TNameLabel<Name>> +{ +}; + +template <typename TType> +struct TTrack<TType, nullptr> + : public NPrivate::TTrackHelper<NPrivate::TTypeLabel<TType>> +{ +}; + +template <typename TType, const char* Name = nullptr> +struct TAlloc + : public NPrivate::TAllocHelper<TType, NPrivate::TNameLabel<Name>> +{ + template<typename U> + struct rebind { + typedef TAlloc<U, Name> other; + }; +}; + +template <typename TType> +struct TAlloc<TType, nullptr> + : public NPrivate::TAllocHelper<TType, NPrivate::TTypeLabel<TType>> +{ + template<typename U> + struct rebind { + typedef TAlloc<U> other; + }; +}; + +} +} + diff --git a/library/cpp/actors/core/memory_tracker.cpp b/library/cpp/actors/core/memory_tracker.cpp index 42b93a2868..8a12452c71 100644 --- a/library/cpp/actors/core/memory_tracker.cpp +++ b/library/cpp/actors/core/memory_tracker.cpp @@ -1,103 +1,103 @@ -#include "memory_tracker.h" - -#include <util/generic/xrange.h> - -namespace NActors { -namespace NMemory { - -namespace NPrivate { - -TMemoryTracker* TMemoryTracker::Instance() { - return SingletonWithPriority<TMemoryTracker, 0>(); -} - -void TMemoryTracker::Initialize() { - GlobalMetrics.resize(Indices.size()); -} - -const std::map<TString, size_t>& TMemoryTracker::GetMetricIndices() const { - return Indices; -} - -const std::unordered_set<size_t>& TMemoryTracker::GetSensors() const { - return Sensors; -} - -TString TMemoryTracker::GetName(size_t index) const { - return Names[index]; -} - -size_t TMemoryTracker::GetCount() const { - return Indices.size(); -} - -void TMemoryTracker::GatherMetrics(std::vector<TMetric>& metrics) const { - metrics.resize(0); - auto count = GetCount(); - - if (!count || GlobalMetrics.size() != count) { - return; - } - - TReadGuard guard(LockThreadInfo); - - metrics.resize(count); - for (size_t i : xrange(count)) { - metrics[i] += GlobalMetrics[i]; - } - - for (auto info : ThreadInfo) { - auto& localMetrics = info->GetMetrics(); - if (localMetrics.size() == count) { - for (size_t i : xrange(count)) { - metrics[i] += localMetrics[i]; - } - } - } -} - -size_t TMemoryTracker::RegisterStaticMemoryLabel(const char* name, bool hasSensor) { - size_t index = 0; - auto found = Indices.find(name); - if (found == Indices.end()) { - TString str(name); - auto next = Names.size(); - Indices.emplace(str, next); - Names.push_back(str); - index = next; - } else { - index = found->second; - } - - if (hasSensor) { - Sensors.emplace(index); - } - return index; -} - -void TMemoryTracker::OnCreateThread(TThreadLocalInfo* info) { - TWriteGuard guard(LockThreadInfo); - ThreadInfo.insert(info); -} - -void TMemoryTracker::OnDestroyThread(TThreadLocalInfo* info) { - TWriteGuard guard(LockThreadInfo); - - auto count = GetCount(); - if (count && GlobalMetrics.size() == count) { - const auto& localMetrics = info->GetMetrics(); - if (localMetrics.size() == count) { - for (size_t i : xrange(count)) { - GlobalMetrics[i] += localMetrics[i]; - } - } - } - - ThreadInfo.erase(info); -} - -} - -} -} - +#include "memory_tracker.h" + +#include <util/generic/xrange.h> + +namespace NActors { +namespace NMemory { + +namespace NPrivate { + +TMemoryTracker* TMemoryTracker::Instance() { + return SingletonWithPriority<TMemoryTracker, 0>(); +} + +void TMemoryTracker::Initialize() { + GlobalMetrics.resize(Indices.size()); +} + +const std::map<TString, size_t>& TMemoryTracker::GetMetricIndices() const { + return Indices; +} + +const std::unordered_set<size_t>& TMemoryTracker::GetSensors() const { + return Sensors; +} + +TString TMemoryTracker::GetName(size_t index) const { + return Names[index]; +} + +size_t TMemoryTracker::GetCount() const { + return Indices.size(); +} + +void TMemoryTracker::GatherMetrics(std::vector<TMetric>& metrics) const { + metrics.resize(0); + auto count = GetCount(); + + if (!count || GlobalMetrics.size() != count) { + return; + } + + TReadGuard guard(LockThreadInfo); + + metrics.resize(count); + for (size_t i : xrange(count)) { + metrics[i] += GlobalMetrics[i]; + } + + for (auto info : ThreadInfo) { + auto& localMetrics = info->GetMetrics(); + if (localMetrics.size() == count) { + for (size_t i : xrange(count)) { + metrics[i] += localMetrics[i]; + } + } + } +} + +size_t TMemoryTracker::RegisterStaticMemoryLabel(const char* name, bool hasSensor) { + size_t index = 0; + auto found = Indices.find(name); + if (found == Indices.end()) { + TString str(name); + auto next = Names.size(); + Indices.emplace(str, next); + Names.push_back(str); + index = next; + } else { + index = found->second; + } + + if (hasSensor) { + Sensors.emplace(index); + } + return index; +} + +void TMemoryTracker::OnCreateThread(TThreadLocalInfo* info) { + TWriteGuard guard(LockThreadInfo); + ThreadInfo.insert(info); +} + +void TMemoryTracker::OnDestroyThread(TThreadLocalInfo* info) { + TWriteGuard guard(LockThreadInfo); + + auto count = GetCount(); + if (count && GlobalMetrics.size() == count) { + const auto& localMetrics = info->GetMetrics(); + if (localMetrics.size() == count) { + for (size_t i : xrange(count)) { + GlobalMetrics[i] += localMetrics[i]; + } + } + } + + ThreadInfo.erase(info); +} + +} + +} +} + diff --git a/library/cpp/actors/core/memory_tracker.h b/library/cpp/actors/core/memory_tracker.h index 7b29c673cf..e74508191b 100644 --- a/library/cpp/actors/core/memory_tracker.h +++ b/library/cpp/actors/core/memory_tracker.h @@ -1,53 +1,53 @@ -#pragma once - -#include "memory_track.h" - -#include <map> -#include <unordered_map> -#include <unordered_set> - -#include <util/system/rwlock.h> - -namespace NActors { -namespace NMemory { - -namespace NPrivate { - -class TMemoryTracker { -public: - static TMemoryTracker* Instance(); - - void Initialize(); - - const std::map<TString, size_t>& GetMetricIndices() const; - const std::unordered_set<size_t>& GetSensors() const; - TString GetName(size_t index) const; - size_t GetCount() const; - - void GatherMetrics(std::vector<TMetric>& metrics) const; - -private: - size_t RegisterStaticMemoryLabel(const char* name, bool hasSensor); - - void OnCreateThread(TThreadLocalInfo* info); - void OnDestroyThread(TThreadLocalInfo* info); - -private: - std::map<TString, size_t> Indices; - std::vector<TString> Names; - - std::vector<TMetric> GlobalMetrics; - - std::unordered_set<size_t> Sensors; - - std::unordered_set<TThreadLocalInfo*> ThreadInfo; - TRWMutex LockThreadInfo; - - friend class TThreadLocalInfo; - friend class TBaseLabel; -}; - -} - -} -} +#pragma once + +#include "memory_track.h" + +#include <map> +#include <unordered_map> +#include <unordered_set> + +#include <util/system/rwlock.h> + +namespace NActors { +namespace NMemory { + +namespace NPrivate { + +class TMemoryTracker { +public: + static TMemoryTracker* Instance(); + + void Initialize(); + + const std::map<TString, size_t>& GetMetricIndices() const; + const std::unordered_set<size_t>& GetSensors() const; + TString GetName(size_t index) const; + size_t GetCount() const; + + void GatherMetrics(std::vector<TMetric>& metrics) const; + +private: + size_t RegisterStaticMemoryLabel(const char* name, bool hasSensor); + + void OnCreateThread(TThreadLocalInfo* info); + void OnDestroyThread(TThreadLocalInfo* info); + +private: + std::map<TString, size_t> Indices; + std::vector<TString> Names; + + std::vector<TMetric> GlobalMetrics; + + std::unordered_set<size_t> Sensors; + + std::unordered_set<TThreadLocalInfo*> ThreadInfo; + TRWMutex LockThreadInfo; + + friend class TThreadLocalInfo; + friend class TBaseLabel; +}; + +} + +} +} diff --git a/library/cpp/actors/core/memory_tracker_ut.cpp b/library/cpp/actors/core/memory_tracker_ut.cpp index a9d3f502f1..d168214da6 100644 --- a/library/cpp/actors/core/memory_tracker_ut.cpp +++ b/library/cpp/actors/core/memory_tracker_ut.cpp @@ -1,262 +1,262 @@ -#include "memory_tracker.h" - -#include <library/cpp/testing/unittest/registar.h> - -#include <util/system/hp_timer.h> -#include <util/system/thread.h> - -namespace NActors { -namespace NMemory { - -Y_UNIT_TEST_SUITE(TMemoryTrackerTest) { - -#if defined(ENABLE_MEMORY_TRACKING) - -using namespace NPrivate; - -size_t FindLabelIndex(const char* label) { - auto indices = TMemoryTracker::Instance()->GetMetricIndices(); - auto it = indices.find(label); - UNIT_ASSERT(it != indices.end()); - return it->second; -} - - -struct TTypeLabeled - : public NActors::NMemory::TTrack<TTypeLabeled> -{ - char payload[16]; -}; - -static constexpr char NamedLabel[] = "NamedLabel"; - -struct TNameLabeled - : public NActors::NMemory::TTrack<TNameLabeled, NamedLabel> -{ - char payload[32]; -}; - -Y_UNIT_TEST(Gathering) -{ - TMemoryTracker::Instance()->Initialize(); - - auto* typed = new TTypeLabeled; - auto* typedArray = new TTypeLabeled[3]; - - auto* named = new TNameLabeled; - auto* namedArray = new TNameLabeled[5]; - NActors::NMemory::TLabel<NamedLabel>::Add(100); - - std::vector<TMetric> metrics; - TMemoryTracker::Instance()->GatherMetrics(metrics); - - auto typeIndex = FindLabelIndex(TypeName<TTypeLabeled>().c_str()); - UNIT_ASSERT(typeIndex < metrics.size()); - UNIT_ASSERT(metrics[typeIndex].GetMemory() == sizeof(TTypeLabeled) * 4 + sizeof(size_t)); - UNIT_ASSERT(metrics[typeIndex].GetCount() == 2); - - auto nameIndex = FindLabelIndex(NamedLabel); - UNIT_ASSERT(nameIndex < metrics.size()); - UNIT_ASSERT(metrics[nameIndex].GetMemory() == sizeof(TNameLabeled) * 6 + sizeof(size_t) + 100); - UNIT_ASSERT(metrics[nameIndex].GetCount() == 3); - - NActors::NMemory::TLabel<NamedLabel>::Sub(100); - delete [] namedArray; - delete named; - - delete [] typedArray; - delete typed; - - TMemoryTracker::Instance()->GatherMetrics(metrics); - - UNIT_ASSERT(metrics[typeIndex].GetMemory() == 0); - UNIT_ASSERT(metrics[typeIndex].GetCount() == 0); - - UNIT_ASSERT(metrics[nameIndex].GetMemory() == 0); - UNIT_ASSERT(metrics[nameIndex].GetCount() == 0); -} - - -static constexpr char InContainerLabel[] = "InContainerLabel"; - -struct TInContainer { - char payload[16]; -}; - -Y_UNIT_TEST(Containers) { - TMemoryTracker::Instance()->Initialize(); - - std::vector<TInContainer, NActors::NMemory::TAlloc<TInContainer>> vecT; - vecT.resize(5); - - std::vector<TInContainer, NActors::NMemory::TAlloc<TInContainer, InContainerLabel>> vecN; - vecN.resize(7); - - using TKey = int; - - std::map<TKey, TInContainer, std::less<TKey>, - NActors::NMemory::TAlloc<std::pair<const TKey, TInContainer>>> mapT; - mapT.emplace(0, TInContainer()); - mapT.emplace(1, TInContainer()); - - std::map<TKey, TInContainer, std::less<TKey>, - NActors::NMemory::TAlloc<std::pair<const TKey, TInContainer>, InContainerLabel>> mapN; - mapN.emplace(0, TInContainer()); - - std::unordered_map<TKey, TInContainer, std::hash<TKey>, std::equal_to<TKey>, - NActors::NMemory::TAlloc<std::pair<const TKey, TInContainer>>> umapT; - umapT.emplace(0, TInContainer()); - - std::unordered_map<TKey, TInContainer, std::hash<TKey>, std::equal_to<TKey>, - NActors::NMemory::TAlloc<std::pair<const TKey, TInContainer>, InContainerLabel>> umapN; - umapN.emplace(0, TInContainer()); - umapN.emplace(1, TInContainer()); - - std::vector<TMetric> metrics; - TMemoryTracker::Instance()->GatherMetrics(metrics); - - auto indices = TMemoryTracker::Instance()->GetMetricIndices(); - for (auto& [name, index] : indices) { - Cerr << "---- " << name - << ": memory = " << metrics[index].GetMemory() - << ", count = " << metrics[index].GetCount() << Endl; - } - - auto vecTIndex = FindLabelIndex(TypeName<TInContainer>().c_str()); - UNIT_ASSERT(metrics[vecTIndex].GetMemory() >= ssize_t(sizeof(TInContainer) * 5)); - UNIT_ASSERT(metrics[vecTIndex].GetCount() == 1); - - auto labelIndex = FindLabelIndex(InContainerLabel); - UNIT_ASSERT(metrics[labelIndex].GetCount() == 5); - UNIT_ASSERT(metrics[labelIndex].GetMemory() >= ssize_t( - sizeof(TInContainer) * 7 + - sizeof(decltype(mapN)::value_type) + - sizeof(decltype(umapN)::value_type) * 2)); -} - - -static constexpr char InThreadLabel[] = "InThreadLabel"; - -struct TInThread - : public NActors::NMemory::TTrack<TInThread, InThreadLabel> -{ - char payload[16]; -}; - -void* ThreadProc(void*) { - return new TInThread; -} - -Y_UNIT_TEST(Threads) { - TMemoryTracker::Instance()->Initialize(); - - auto index = FindLabelIndex(InThreadLabel); - - auto* object1 = new TInThread; - - std::vector<TMetric> metrics; - TMemoryTracker::Instance()->GatherMetrics(metrics); - UNIT_ASSERT(metrics[index].GetMemory() == sizeof(TInThread)); - UNIT_ASSERT(metrics[index].GetCount() == 1); - - TThread thread(&ThreadProc, nullptr); - thread.Start(); - auto* object2 = static_cast<TInThread*>(thread.Join()); - - TMemoryTracker::Instance()->GatherMetrics(metrics); - UNIT_ASSERT(metrics[index].GetMemory() == sizeof(TInThread) * 2); - UNIT_ASSERT(metrics[index].GetCount() == 2); - - delete object2; - - TMemoryTracker::Instance()->GatherMetrics(metrics); - UNIT_ASSERT(metrics[index].GetMemory() == sizeof(TInThread)); - UNIT_ASSERT(metrics[index].GetCount() == 1); - - delete object1; -} - - -struct TNotTracked { - char payload[16]; -}; - -struct TTracked - : public NActors::NMemory::TTrack<TTracked> -{ - char payload[16]; -}; - -template <typename T> -double MeasureAllocations() { - constexpr size_t objectsCount = 4 << 20; - - std::vector<T*> objects; - objects.resize(objectsCount); - - THPTimer timer; - - for (size_t i = 0; i < objectsCount; ++i) { - objects[i] = new T; - } - - for (size_t i = 0; i < objectsCount; ++i) { - delete objects[i]; - } - - auto seconds = timer.Passed(); - Cerr << "---- objects: " << objectsCount << ", time: " << seconds << Endl; - return seconds; -} - -Y_UNIT_TEST(Performance) { - TMemoryTracker::Instance()->Initialize(); - - constexpr size_t Runs = 16; - - Cerr << "---- warmup" << Endl; - MeasureAllocations<TNotTracked>(); - MeasureAllocations<TTracked>(); - - std::vector<double> noTrack; - std::vector<double> track; - - for (size_t run = 0; run < Runs; ++run) { - Cerr << "---- no track" << Endl; - auto time = MeasureAllocations<TNotTracked>(); - noTrack.push_back(time); - - Cerr << "---- track" << Endl; - time = MeasureAllocations<TTracked>(); - track.push_back(time); - } - - double meanNoTrack = 0, stddevNoTrack = 0; - double meanTrack = 0, stddevTrack = 0; - for (size_t i = 0; i < Runs; ++i) { - meanNoTrack += noTrack[i]; - meanTrack += track[i]; - } - meanNoTrack /= Runs; - meanTrack /= Runs; - - auto sqr = [](double val) { return val * val; }; - - for (size_t i = 0; i < Runs; ++i) { - stddevNoTrack += sqr(noTrack[i] - meanNoTrack); - stddevTrack += sqr(track[i] - meanTrack); - } - stddevNoTrack = sqrt(stddevNoTrack / (Runs - 1)); - stddevTrack = sqrt(stddevTrack / (Runs - 1)); - - Cerr << "---- no track - mean: " << meanNoTrack << ", stddev: " << stddevNoTrack << Endl; - Cerr << "---- track - mean: " << meanTrack << ", stddev: " << stddevTrack << Endl; - Cerr << "---- tracking is slower by " << int((meanTrack / meanNoTrack - 1.0) * 100) << "%" << Endl; -} - -#endif - -} - -} -} +#include "memory_tracker.h" + +#include <library/cpp/testing/unittest/registar.h> + +#include <util/system/hp_timer.h> +#include <util/system/thread.h> + +namespace NActors { +namespace NMemory { + +Y_UNIT_TEST_SUITE(TMemoryTrackerTest) { + +#if defined(ENABLE_MEMORY_TRACKING) + +using namespace NPrivate; + +size_t FindLabelIndex(const char* label) { + auto indices = TMemoryTracker::Instance()->GetMetricIndices(); + auto it = indices.find(label); + UNIT_ASSERT(it != indices.end()); + return it->second; +} + + +struct TTypeLabeled + : public NActors::NMemory::TTrack<TTypeLabeled> +{ + char payload[16]; +}; + +static constexpr char NamedLabel[] = "NamedLabel"; + +struct TNameLabeled + : public NActors::NMemory::TTrack<TNameLabeled, NamedLabel> +{ + char payload[32]; +}; + +Y_UNIT_TEST(Gathering) +{ + TMemoryTracker::Instance()->Initialize(); + + auto* typed = new TTypeLabeled; + auto* typedArray = new TTypeLabeled[3]; + + auto* named = new TNameLabeled; + auto* namedArray = new TNameLabeled[5]; + NActors::NMemory::TLabel<NamedLabel>::Add(100); + + std::vector<TMetric> metrics; + TMemoryTracker::Instance()->GatherMetrics(metrics); + + auto typeIndex = FindLabelIndex(TypeName<TTypeLabeled>().c_str()); + UNIT_ASSERT(typeIndex < metrics.size()); + UNIT_ASSERT(metrics[typeIndex].GetMemory() == sizeof(TTypeLabeled) * 4 + sizeof(size_t)); + UNIT_ASSERT(metrics[typeIndex].GetCount() == 2); + + auto nameIndex = FindLabelIndex(NamedLabel); + UNIT_ASSERT(nameIndex < metrics.size()); + UNIT_ASSERT(metrics[nameIndex].GetMemory() == sizeof(TNameLabeled) * 6 + sizeof(size_t) + 100); + UNIT_ASSERT(metrics[nameIndex].GetCount() == 3); + + NActors::NMemory::TLabel<NamedLabel>::Sub(100); + delete [] namedArray; + delete named; + + delete [] typedArray; + delete typed; + + TMemoryTracker::Instance()->GatherMetrics(metrics); + + UNIT_ASSERT(metrics[typeIndex].GetMemory() == 0); + UNIT_ASSERT(metrics[typeIndex].GetCount() == 0); + + UNIT_ASSERT(metrics[nameIndex].GetMemory() == 0); + UNIT_ASSERT(metrics[nameIndex].GetCount() == 0); +} + + +static constexpr char InContainerLabel[] = "InContainerLabel"; + +struct TInContainer { + char payload[16]; +}; + +Y_UNIT_TEST(Containers) { + TMemoryTracker::Instance()->Initialize(); + + std::vector<TInContainer, NActors::NMemory::TAlloc<TInContainer>> vecT; + vecT.resize(5); + + std::vector<TInContainer, NActors::NMemory::TAlloc<TInContainer, InContainerLabel>> vecN; + vecN.resize(7); + + using TKey = int; + + std::map<TKey, TInContainer, std::less<TKey>, + NActors::NMemory::TAlloc<std::pair<const TKey, TInContainer>>> mapT; + mapT.emplace(0, TInContainer()); + mapT.emplace(1, TInContainer()); + + std::map<TKey, TInContainer, std::less<TKey>, + NActors::NMemory::TAlloc<std::pair<const TKey, TInContainer>, InContainerLabel>> mapN; + mapN.emplace(0, TInContainer()); + + std::unordered_map<TKey, TInContainer, std::hash<TKey>, std::equal_to<TKey>, + NActors::NMemory::TAlloc<std::pair<const TKey, TInContainer>>> umapT; + umapT.emplace(0, TInContainer()); + + std::unordered_map<TKey, TInContainer, std::hash<TKey>, std::equal_to<TKey>, + NActors::NMemory::TAlloc<std::pair<const TKey, TInContainer>, InContainerLabel>> umapN; + umapN.emplace(0, TInContainer()); + umapN.emplace(1, TInContainer()); + + std::vector<TMetric> metrics; + TMemoryTracker::Instance()->GatherMetrics(metrics); + + auto indices = TMemoryTracker::Instance()->GetMetricIndices(); + for (auto& [name, index] : indices) { + Cerr << "---- " << name + << ": memory = " << metrics[index].GetMemory() + << ", count = " << metrics[index].GetCount() << Endl; + } + + auto vecTIndex = FindLabelIndex(TypeName<TInContainer>().c_str()); + UNIT_ASSERT(metrics[vecTIndex].GetMemory() >= ssize_t(sizeof(TInContainer) * 5)); + UNIT_ASSERT(metrics[vecTIndex].GetCount() == 1); + + auto labelIndex = FindLabelIndex(InContainerLabel); + UNIT_ASSERT(metrics[labelIndex].GetCount() == 5); + UNIT_ASSERT(metrics[labelIndex].GetMemory() >= ssize_t( + sizeof(TInContainer) * 7 + + sizeof(decltype(mapN)::value_type) + + sizeof(decltype(umapN)::value_type) * 2)); +} + + +static constexpr char InThreadLabel[] = "InThreadLabel"; + +struct TInThread + : public NActors::NMemory::TTrack<TInThread, InThreadLabel> +{ + char payload[16]; +}; + +void* ThreadProc(void*) { + return new TInThread; +} + +Y_UNIT_TEST(Threads) { + TMemoryTracker::Instance()->Initialize(); + + auto index = FindLabelIndex(InThreadLabel); + + auto* object1 = new TInThread; + + std::vector<TMetric> metrics; + TMemoryTracker::Instance()->GatherMetrics(metrics); + UNIT_ASSERT(metrics[index].GetMemory() == sizeof(TInThread)); + UNIT_ASSERT(metrics[index].GetCount() == 1); + + TThread thread(&ThreadProc, nullptr); + thread.Start(); + auto* object2 = static_cast<TInThread*>(thread.Join()); + + TMemoryTracker::Instance()->GatherMetrics(metrics); + UNIT_ASSERT(metrics[index].GetMemory() == sizeof(TInThread) * 2); + UNIT_ASSERT(metrics[index].GetCount() == 2); + + delete object2; + + TMemoryTracker::Instance()->GatherMetrics(metrics); + UNIT_ASSERT(metrics[index].GetMemory() == sizeof(TInThread)); + UNIT_ASSERT(metrics[index].GetCount() == 1); + + delete object1; +} + + +struct TNotTracked { + char payload[16]; +}; + +struct TTracked + : public NActors::NMemory::TTrack<TTracked> +{ + char payload[16]; +}; + +template <typename T> +double MeasureAllocations() { + constexpr size_t objectsCount = 4 << 20; + + std::vector<T*> objects; + objects.resize(objectsCount); + + THPTimer timer; + + for (size_t i = 0; i < objectsCount; ++i) { + objects[i] = new T; + } + + for (size_t i = 0; i < objectsCount; ++i) { + delete objects[i]; + } + + auto seconds = timer.Passed(); + Cerr << "---- objects: " << objectsCount << ", time: " << seconds << Endl; + return seconds; +} + +Y_UNIT_TEST(Performance) { + TMemoryTracker::Instance()->Initialize(); + + constexpr size_t Runs = 16; + + Cerr << "---- warmup" << Endl; + MeasureAllocations<TNotTracked>(); + MeasureAllocations<TTracked>(); + + std::vector<double> noTrack; + std::vector<double> track; + + for (size_t run = 0; run < Runs; ++run) { + Cerr << "---- no track" << Endl; + auto time = MeasureAllocations<TNotTracked>(); + noTrack.push_back(time); + + Cerr << "---- track" << Endl; + time = MeasureAllocations<TTracked>(); + track.push_back(time); + } + + double meanNoTrack = 0, stddevNoTrack = 0; + double meanTrack = 0, stddevTrack = 0; + for (size_t i = 0; i < Runs; ++i) { + meanNoTrack += noTrack[i]; + meanTrack += track[i]; + } + meanNoTrack /= Runs; + meanTrack /= Runs; + + auto sqr = [](double val) { return val * val; }; + + for (size_t i = 0; i < Runs; ++i) { + stddevNoTrack += sqr(noTrack[i] - meanNoTrack); + stddevTrack += sqr(track[i] - meanTrack); + } + stddevNoTrack = sqrt(stddevNoTrack / (Runs - 1)); + stddevTrack = sqrt(stddevTrack / (Runs - 1)); + + Cerr << "---- no track - mean: " << meanNoTrack << ", stddev: " << stddevNoTrack << Endl; + Cerr << "---- track - mean: " << meanTrack << ", stddev: " << stddevTrack << Endl; + Cerr << "---- tracking is slower by " << int((meanTrack / meanNoTrack - 1.0) * 100) << "%" << Endl; +} + +#endif + +} + +} +} diff --git a/library/cpp/actors/core/mon_stats.h b/library/cpp/actors/core/mon_stats.h index e58429ad0a..d55552af0c 100644 --- a/library/cpp/actors/core/mon_stats.h +++ b/library/cpp/actors/core/mon_stats.h @@ -2,13 +2,13 @@ #include "defs.h" #include "actor.h" -#include <library/cpp/monlib/metrics/histogram_snapshot.h> +#include <library/cpp/monlib/metrics/histogram_snapshot.h> #include <util/system/hp_timer.h> namespace NActors { - struct TLogHistogram : public NMonitoring::IHistogramSnapshot { + struct TLogHistogram : public NMonitoring::IHistogramSnapshot { TLogHistogram() { - memset(Buckets, 0, sizeof(Buckets)); + memset(Buckets, 0, sizeof(Buckets)); } inline void Add(ui64 val, ui64 inc = 1) { @@ -17,8 +17,8 @@ namespace NActors { asm volatile("" :: : "memory"); #endif - if (val > 1) { - ind = GetValueBitCount(val - 1); + if (val > 1) { + ind = GetValueBitCount(val - 1); } #if defined(__clang__) && __clang_major__ == 3 && __clang_minor__ == 7 asm volatile("" :: @@ -36,25 +36,25 @@ namespace NActors { } } - // IHistogramSnapshot - ui32 Count() const override { - return Y_ARRAY_SIZE(Buckets); - } - - NMonitoring::TBucketBound UpperBound(ui32 index) const override { - Y_ASSERT(index < Y_ARRAY_SIZE(Buckets)); - if (index == 0) { - return 1; - } - return NMonitoring::TBucketBound(1ull << (index - 1)) * 2.0; - } - - NMonitoring::TBucketValue Value(ui32 index) const override { - Y_ASSERT(index < Y_ARRAY_SIZE(Buckets)); - return Buckets[index]; - } - - ui64 TotalSamples = 0; + // IHistogramSnapshot + ui32 Count() const override { + return Y_ARRAY_SIZE(Buckets); + } + + NMonitoring::TBucketBound UpperBound(ui32 index) const override { + Y_ASSERT(index < Y_ARRAY_SIZE(Buckets)); + if (index == 0) { + return 1; + } + return NMonitoring::TBucketBound(1ull << (index - 1)) * 2.0; + } + + NMonitoring::TBucketValue Value(ui32 index) const override { + Y_ASSERT(index < Y_ARRAY_SIZE(Buckets)); + return Buckets[index]; + } + + ui64 TotalSamples = 0; ui64 Buckets[65]; }; diff --git a/library/cpp/actors/core/process_stats.cpp b/library/cpp/actors/core/process_stats.cpp index 2cc73216b4..0e1dbd0031 100644 --- a/library/cpp/actors/core/process_stats.cpp +++ b/library/cpp/actors/core/process_stats.cpp @@ -181,9 +181,9 @@ namespace { class TDynamicCounterCollector: public TProcStatCollectingActor<TDynamicCounterCollector> { using TBase = TProcStatCollectingActor<TDynamicCounterCollector>; public: - TDynamicCounterCollector( - ui32 intervalSeconds, - NMonitoring::TDynamicCounterPtr counters) + TDynamicCounterCollector( + ui32 intervalSeconds, + NMonitoring::TDynamicCounterPtr counters) : TBase{TDuration::Seconds(intervalSeconds)} { ProcStatGroup = counters->GetSubgroup("counters", "utils"); @@ -218,7 +218,7 @@ namespace { } private: - NMonitoring::TDynamicCounterPtr ProcStatGroup; + NMonitoring::TDynamicCounterPtr ProcStatGroup; NMonitoring::TDynamicCounters::TCounterPtr VmSize; NMonitoring::TDynamicCounters::TCounterPtr AnonRssSize; NMonitoring::TDynamicCounters::TCounterPtr FileRssSize; @@ -293,8 +293,8 @@ namespace { }; } // namespace - IActor* CreateProcStatCollector(ui32 intervalSec, NMonitoring::TDynamicCounterPtr counters) { - return new TDynamicCounterCollector(intervalSec, counters); + IActor* CreateProcStatCollector(ui32 intervalSec, NMonitoring::TDynamicCounterPtr counters) { + return new TDynamicCounterCollector(intervalSec, counters); } IActor* CreateProcStatCollector(TDuration interval, NMonitoring::TMetricRegistry& registry) { diff --git a/library/cpp/actors/core/process_stats.h b/library/cpp/actors/core/process_stats.h index 236ea829ee..66346d0b5a 100644 --- a/library/cpp/actors/core/process_stats.h +++ b/library/cpp/actors/core/process_stats.h @@ -61,6 +61,6 @@ namespace NActors { long ObtainPageSize(); }; - IActor* CreateProcStatCollector(ui32 intervalSec, NMonitoring::TDynamicCounterPtr counters); + IActor* CreateProcStatCollector(ui32 intervalSec, NMonitoring::TDynamicCounterPtr counters); IActor* CreateProcStatCollector(TDuration interval, NMonitoring::TMetricRegistry& registry); } diff --git a/library/cpp/actors/core/scheduler_queue.h b/library/cpp/actors/core/scheduler_queue.h index 000321e7d3..3b8fac28f0 100644 --- a/library/cpp/actors/core/scheduler_queue.h +++ b/library/cpp/actors/core/scheduler_queue.h @@ -13,7 +13,7 @@ namespace NActors { ISchedulerCookie* Cookie; }; - struct TChunk : TQueueChunkDerived<TEntry, 512, TChunk> {}; + struct TChunk : TQueueChunkDerived<TEntry, 512, TChunk> {}; class TReader; class TWriter; diff --git a/library/cpp/actors/core/ut/ya.make b/library/cpp/actors/core/ut/ya.make index fc3195905b..3ee28d5850 100644 --- a/library/cpp/actors/core/ut/ya.make +++ b/library/cpp/actors/core/ut/ya.make @@ -39,7 +39,7 @@ SRCS( executor_pool_basic_ut.cpp executor_pool_united_ut.cpp log_ut.cpp - memory_tracker_ut.cpp + memory_tracker_ut.cpp scheduler_actor_ut.cpp ) diff --git a/library/cpp/actors/core/ya.make b/library/cpp/actors/core/ya.make index 633de9edc1..880a9d00db 100644 --- a/library/cpp/actors/core/ya.make +++ b/library/cpp/actors/core/ya.make @@ -75,10 +75,10 @@ SRCS( mailbox.h mailbox_queue_revolving.h mailbox_queue_simple.h - memory_track.cpp - memory_track.h - memory_tracker.cpp - memory_tracker.h + memory_track.cpp + memory_track.h + memory_tracker.cpp + memory_tracker.h mon.h mon_stats.h monotonic.cpp diff --git a/library/cpp/actors/helpers/mon_histogram_helper.h b/library/cpp/actors/helpers/mon_histogram_helper.h index 1da0eec1db..a9a57e3823 100644 --- a/library/cpp/actors/helpers/mon_histogram_helper.h +++ b/library/cpp/actors/helpers/mon_histogram_helper.h @@ -16,9 +16,9 @@ namespace NActors { THistogramCounterHelper(const THistogramCounterHelper&) = default; - void Init(NMonitoring::TDynamicCounters* group, const TString& baseName, const TString& unit, - ui64 firstBucket, ui64 bucketCnt, bool useSensorLabelName = true) - { + void Init(NMonitoring::TDynamicCounters* group, const TString& baseName, const TString& unit, + ui64 firstBucket, ui64 bucketCnt, bool useSensorLabelName = true) + { Y_ASSERT(FirstBucketVal == 0); Y_ASSERT(BucketCount == 0); @@ -28,8 +28,8 @@ namespace NActors { Buckets.reserve(BucketCount); for (size_t i = 0; i < BucketCount; ++i) { TString bucketName = GetBucketName(i) + " " + unit; - auto labelName = useSensorLabelName ? "sensor" : "name"; - BucketsHolder.push_back(group->GetSubgroup(labelName, baseName)->GetNamedCounter("range", bucketName, true)); + auto labelName = useSensorLabelName ? "sensor" : "name"; + BucketsHolder.push_back(group->GetSubgroup(labelName, baseName)->GetNamedCounter("range", bucketName, true)); Buckets.push_back(BucketsHolder.back().Get()); } } @@ -48,20 +48,20 @@ namespace NActors { Buckets[ind]->Inc(); } - ui64 GetBucketCount() const { - return BucketCount; - } - - ui64 GetBucketValue(size_t index) const { - Y_ASSERT(index < BucketCount); - return Buckets[index]->Val(); - } - - void SetBucketValue(ui64 index, ui64 value) { - Y_ASSERT(index < BucketCount); - *Buckets[index] = value; - } - + ui64 GetBucketCount() const { + return BucketCount; + } + + ui64 GetBucketValue(size_t index) const { + Y_ASSERT(index < BucketCount); + return Buckets[index]->Val(); + } + + void SetBucketValue(ui64 index, ui64 value) { + Y_ASSERT(index < BucketCount); + *Buckets[index] = value; + } + private: TString GetBucketName(size_t ind) const { Y_ASSERT(FirstBucketVal != 0); diff --git a/library/cpp/actors/interconnect/interconnect_counters.cpp b/library/cpp/actors/interconnect/interconnect_counters.cpp index e4b1412f62..224160d4b4 100644 --- a/library/cpp/actors/interconnect/interconnect_counters.cpp +++ b/library/cpp/actors/interconnect/interconnect_counters.cpp @@ -278,11 +278,11 @@ namespace { InflyLimitReach = AdaptiveCounters->GetCounter("InflyLimitReach", true); InflightDataAmount = AdaptiveCounters->GetCounter("Inflight_Data"); - LegacyPingTimeHist = {}; - LegacyPingTimeHist.Init(AdaptiveCounters.Get(), "PingTimeHist", "mks", 125, 18); - - PingTimeHistogram = AdaptiveCounters->GetHistogram( - "PingTimeUs", NMonitoring::ExponentialHistogram(18, 2, 125)); + LegacyPingTimeHist = {}; + LegacyPingTimeHist.Init(AdaptiveCounters.Get(), "PingTimeHist", "mks", 125, 18); + + PingTimeHistogram = AdaptiveCounters->GetHistogram( + "PingTimeUs", NMonitoring::ExponentialHistogram(18, 2, 125)); } if (updateGlobal) { @@ -299,7 +299,7 @@ namespace { auto disconnectReasonGroup = Counters->GetSubgroup("subsystem", "disconnectReason"); for (const char *reason : TDisconnectReason::Reasons) { - DisconnectByReason[reason] = disconnectReasonGroup->GetCounter(reason, true); + DisconnectByReason[reason] = disconnectReasonGroup->GetCounter(reason, true); } } @@ -329,9 +329,9 @@ namespace { NMonitoring::TDynamicCounters::TCounterPtr SpuriousReadWakeups; NMonitoring::TDynamicCounters::TCounterPtr UsefulWriteWakeups; NMonitoring::TDynamicCounters::TCounterPtr SpuriousWriteWakeups; - NMon::THistogramCounterHelper LegacyPingTimeHist; - NMonitoring::THistogramPtr PingTimeHistogram; - + NMon::THistogramCounterHelper LegacyPingTimeHist; + NMonitoring::THistogramPtr PingTimeHistogram; + std::unordered_map<ui16, TOutputChannel> OutputChannels; TOutputChannel OtherOutputChannel; TInputChannels InputChannels; diff --git a/library/cpp/actors/interconnect/interconnect_tcp_input_session.cpp b/library/cpp/actors/interconnect/interconnect_tcp_input_session.cpp index 16f0223502..0abe9fe659 100644 --- a/library/cpp/actors/interconnect/interconnect_tcp_input_session.cpp +++ b/library/cpp/actors/interconnect/interconnect_tcp_input_session.cpp @@ -196,7 +196,7 @@ namespace NActors { if (AtomicGet(Context->ControlPacketId) <= HeaderConfirm && !NewPingProtocol) { ui64 sendTime = AtomicGet(Context->ControlPacketSendTimer); TDuration duration = CyclesToDuration(GetCycleCountFast() - sendTime); - const auto durationUs = duration.MicroSeconds(); + const auto durationUs = duration.MicroSeconds(); Metrics->UpdateLegacyPingTimeHist(durationUs); PingQ.push_back(duration); if (PingQ.size() > 16) { @@ -448,8 +448,8 @@ namespace NActors { PingQ.pop_front(); } const TDuration ping = *std::min_element(PingQ.begin(), PingQ.end()); - const auto pingUs = ping.MicroSeconds(); - Context->PingRTT_us = pingUs; + const auto pingUs = ping.MicroSeconds(); + Context->PingRTT_us = pingUs; NewPingProtocol = true; Metrics->UpdateLegacyPingTimeHist(pingUs); } diff --git a/library/cpp/actors/prof/tag.cpp b/library/cpp/actors/prof/tag.cpp index caf516812b..9ccf03e1a9 100644 --- a/library/cpp/actors/prof/tag.cpp +++ b/library/cpp/actors/prof/tag.cpp @@ -1,9 +1,9 @@ #include "tag.h" -#include "tcmalloc.h" +#include "tcmalloc.h" #include <library/cpp/charset/ci_string.h> #include <library/cpp/containers/atomizer/atomizer.h> -#include <library/cpp/malloc/api/malloc.h> +#include <library/cpp/malloc/api/malloc.h> #if defined(PROFILE_MEMORY_ALLOCATIONS) #include <library/cpp/lfalloc/dbg_info/dbg_info.h> @@ -76,11 +76,11 @@ namespace NProfiling { return TStringAtoms::Instance().GetTagsCount(); } - static ui32 SetThreadAllocTag_Default(ui32 tag) { - Y_UNUSED(tag); - return 0; - } - + static ui32 SetThreadAllocTag_Default(ui32 tag) { + Y_UNUSED(tag); + return 0; + } + #if defined(PROFILE_MEMORY_ALLOCATIONS) static ui32 SetThreadAllocTag_YT(ui32 tag) { auto prev = NYT::NYTAlloc::GetCurrentMemoryTag(); @@ -96,24 +96,24 @@ namespace NProfiling { return (TSetThreadAllocTag*)NAllocDbg::SetThreadAllocTag; } else if (name.StartsWith("yt")) { return SetThreadAllocTag_YT; - } else if (name.StartsWith("tc")) { - return SetTCMallocThreadAllocTag; + } else if (name.StartsWith("tc")) { + return SetTCMallocThreadAllocTag; + } else { + return SetThreadAllocTag_Default; + } + } +#else + static TSetThreadAllocTag* SetThreadAllocTagFn() { + const auto& info = NMalloc::MallocInfo(); + + TStringBuf name(info.Name); + if (name.StartsWith("tc")) { + return SetTCMallocThreadAllocTag; } else { return SetThreadAllocTag_Default; } } -#else - static TSetThreadAllocTag* SetThreadAllocTagFn() { - const auto& info = NMalloc::MallocInfo(); - - TStringBuf name(info.Name); - if (name.StartsWith("tc")) { - return SetTCMallocThreadAllocTag; - } else { - return SetThreadAllocTag_Default; - } - } -#endif - +#endif + TSetThreadAllocTag* SetThreadAllocTag = SetThreadAllocTagFn(); } diff --git a/library/cpp/actors/prof/tag.h b/library/cpp/actors/prof/tag.h index 81815b7289..357e264a22 100644 --- a/library/cpp/actors/prof/tag.h +++ b/library/cpp/actors/prof/tag.h @@ -5,7 +5,7 @@ /* Common registry for tagging memory profiler. Register a new tag with MakeTag using a unique string. - Use registered tags with SetThreadAllocTag function in allocator API. + Use registered tags with SetThreadAllocTag function in allocator API. */ namespace NProfiling { @@ -22,43 +22,43 @@ namespace NProfiling { class TMemoryTagScope { public: - explicit TMemoryTagScope(ui32 tag) + explicit TMemoryTagScope(ui32 tag) : RestoreTag(SetThreadAllocTag(tag)) { } - explicit TMemoryTagScope(const char* tagName) { + explicit TMemoryTagScope(const char* tagName) { ui32 newTag = MakeTag(tagName); RestoreTag = SetThreadAllocTag(newTag); } TMemoryTagScope(TMemoryTagScope&& move) : RestoreTag(move.RestoreTag) - , Released(move.Released) + , Released(move.Released) { - move.Released = true; + move.Released = true; } TMemoryTagScope& operator=(TMemoryTagScope&& move) { RestoreTag = move.RestoreTag; - Released = move.Released; - move.Released = true; + Released = move.Released; + move.Released = true; return *this; } - static void Reset(ui32 tag) { + static void Reset(ui32 tag) { SetThreadAllocTag(tag); } void Release() { - if (!Released) { + if (!Released) { SetThreadAllocTag(RestoreTag); - Released = true; + Released = true; } } ~TMemoryTagScope() { - if (!Released) { + if (!Released) { SetThreadAllocTag(RestoreTag); } } @@ -67,7 +67,7 @@ namespace NProfiling { TMemoryTagScope(const TMemoryTagScope&) = delete; void operator=(const TMemoryTagScope&) = delete; - ui32 RestoreTag = 0; - bool Released = false; + ui32 RestoreTag = 0; + bool Released = false; }; } diff --git a/library/cpp/actors/prof/tcmalloc.cpp b/library/cpp/actors/prof/tcmalloc.cpp index 331181e80b..3d4f203dbb 100644 --- a/library/cpp/actors/prof/tcmalloc.cpp +++ b/library/cpp/actors/prof/tcmalloc.cpp @@ -1,32 +1,32 @@ -#include "tcmalloc.h" - -#include <contrib/libs/tcmalloc/tcmalloc/malloc_extension.h> - -namespace NProfiling { - -static thread_local ui32 AllocationTag = 0; - -static struct TInitTCMallocCallbacks { - static void* CreateTag() { - return reinterpret_cast<void*>(AllocationTag); - } - static void* CopyTag(void* tag) { - return tag; - } - static void DestroyTag(void* tag) { - Y_UNUSED(tag); - } - - TInitTCMallocCallbacks() { - tcmalloc::MallocExtension::SetSampleUserDataCallbacks( - CreateTag, CopyTag, DestroyTag); - } -} InitTCMallocCallbacks; - -ui32 SetTCMallocThreadAllocTag(ui32 tag) { - ui32 prev = AllocationTag; - AllocationTag = tag; - return prev; -} - -} +#include "tcmalloc.h" + +#include <contrib/libs/tcmalloc/tcmalloc/malloc_extension.h> + +namespace NProfiling { + +static thread_local ui32 AllocationTag = 0; + +static struct TInitTCMallocCallbacks { + static void* CreateTag() { + return reinterpret_cast<void*>(AllocationTag); + } + static void* CopyTag(void* tag) { + return tag; + } + static void DestroyTag(void* tag) { + Y_UNUSED(tag); + } + + TInitTCMallocCallbacks() { + tcmalloc::MallocExtension::SetSampleUserDataCallbacks( + CreateTag, CopyTag, DestroyTag); + } +} InitTCMallocCallbacks; + +ui32 SetTCMallocThreadAllocTag(ui32 tag) { + ui32 prev = AllocationTag; + AllocationTag = tag; + return prev; +} + +} diff --git a/library/cpp/actors/prof/tcmalloc.h b/library/cpp/actors/prof/tcmalloc.h index 0da31a2024..659fb4eaf3 100644 --- a/library/cpp/actors/prof/tcmalloc.h +++ b/library/cpp/actors/prof/tcmalloc.h @@ -1,9 +1,9 @@ -#pragma once - -#include <util/generic/fwd.h> - -namespace NProfiling { - -ui32 SetTCMallocThreadAllocTag(ui32 tag); - -} +#pragma once + +#include <util/generic/fwd.h> + +namespace NProfiling { + +ui32 SetTCMallocThreadAllocTag(ui32 tag); + +} diff --git a/library/cpp/actors/prof/tcmalloc_null.cpp b/library/cpp/actors/prof/tcmalloc_null.cpp index 26e6f61bb9..75c0013154 100644 --- a/library/cpp/actors/prof/tcmalloc_null.cpp +++ b/library/cpp/actors/prof/tcmalloc_null.cpp @@ -1,10 +1,10 @@ -#include "tcmalloc.h" - -namespace NProfiling { - -ui32 SetTCMallocThreadAllocTag(ui32 tag) { - Y_UNUSED(tag); - return 0; -} - -} +#include "tcmalloc.h" + +namespace NProfiling { + +ui32 SetTCMallocThreadAllocTag(ui32 tag) { + Y_UNUSED(tag); + return 0; +} + +} diff --git a/library/cpp/actors/prof/ya.make b/library/cpp/actors/prof/ya.make index d20ee798c4..b5e2497563 100644 --- a/library/cpp/actors/prof/ya.make +++ b/library/cpp/actors/prof/ya.make @@ -23,11 +23,11 @@ IF (PROFILE_MEMORY_ALLOCATIONS) ) ENDIF() -IF(ALLOCATOR == "TCMALLOC_256K") - SRCS(tcmalloc.cpp) - PEERDIR(contrib/libs/tcmalloc) -ELSE() - SRCS(tcmalloc_null.cpp) -ENDIF() - +IF(ALLOCATOR == "TCMALLOC_256K") + SRCS(tcmalloc.cpp) + PEERDIR(contrib/libs/tcmalloc) +ELSE() + SRCS(tcmalloc_null.cpp) +ENDIF() + END() diff --git a/library/cpp/binsaver/bin_saver.cpp b/library/cpp/binsaver/bin_saver.cpp index ca215fa809..fe0775af9f 100644 --- a/library/cpp/binsaver/bin_saver.cpp +++ b/library/cpp/binsaver/bin_saver.cpp @@ -42,10 +42,10 @@ void IBinSaver::StoreObject(IObjectBase* pObject) { ObjectQueue.push_back(pObject); (*Objects)[ptrId]; int typeId = pSaverClasses->GetObjectTypeID(pObject); - if (typeId == -1) { - fprintf(stderr, "IBinSaver: trying to save unregistered object\n"); - abort(); - } + if (typeId == -1) { + fprintf(stderr, "IBinSaver: trying to save unregistered object\n"); + abort(); + } DataChunk(&typeId, sizeof(typeId)); } } @@ -64,9 +64,9 @@ IObjectBase* IBinSaver::LoadObject() { IObjectBase* pObj = pSaverClasses->CreateObject(typeId); Y_ASSERT(pObj != nullptr); if (pObj == nullptr) { - fprintf(stderr, "IBinSaver: trying to load unregistered object\n"); - abort(); - } + fprintf(stderr, "IBinSaver: trying to load unregistered object\n"); + abort(); + } (*Objects)[ptrId] = pObj; ObjectQueue.push_back(pObj); return pObj; diff --git a/library/cpp/binsaver/bin_saver.h b/library/cpp/binsaver/bin_saver.h index 0e9e46a2e5..412424889f 100644 --- a/library/cpp/binsaver/bin_saver.h +++ b/library/cpp/binsaver/bin_saver.h @@ -156,13 +156,13 @@ private: } } - // hash_multimap + // hash_multimap template <class AMM> void DoAnyMultiMap(AMM& data) { - if (IsReading()) { - data.clear(); + if (IsReading()) { + data.clear(); TStoredSize nSize; - Add(3, &nSize); + Add(3, &nSize); TVector<typename AMM::key_type, typename std::allocator_traits<typename AMM::allocator_type>::template rebind_alloc<typename AMM::key_type>> indices; indices.resize(nSize); for (TStoredSize i = 0; i < nSize; ++i) @@ -170,19 +170,19 @@ private: for (TStoredSize i = 0; i < nSize; ++i) { std::pair<typename AMM::key_type, typename AMM::mapped_type> valToInsert; valToInsert.first = indices[i]; - Add(2, &valToInsert.second); - data.insert(valToInsert); - } - } else { + Add(2, &valToInsert.second); + data.insert(valToInsert); + } + } else { TStoredSize nSize = data.size(); CheckOverflow(nSize, data.size()); - Add(3, &nSize); + Add(3, &nSize); for (auto pos = data.begin(); pos != data.end(); ++pos) Add(1, (typename AMM::key_type*)(&pos->first)); for (auto pos = data.begin(); pos != data.end(); ++pos) - Add(2, &pos->second); - } - } + Add(2, &pos->second); + } + } template <class T> void DoAnySet(T& data) { @@ -270,7 +270,7 @@ private: IObjectBase* LoadObject(); bool bRead; - TBufferedStream<> File; + TBufferedStream<> File; // maps objects addresses during save(first) to addresses during load(second) - during loading // or serves as a sign that some object has been already stored - during storing bool StableOutput; @@ -357,7 +357,7 @@ public: int Add(const chunk_id, THashMultiMap<T1, T2, T3, T4, T5>* pHash) { DoAnyMultiMap(*pHash); return 0; - } + } template <class K, class L, class A> int Add(const chunk_id, TSet<K, L, A>* pSet) { DoAnySet(*pSet); diff --git a/library/cpp/binsaver/buffered_io.h b/library/cpp/binsaver/buffered_io.h index 659bc9f62b..75465c9c5c 100644 --- a/library/cpp/binsaver/buffered_io.h +++ b/library/cpp/binsaver/buffered_io.h @@ -26,7 +26,7 @@ struct IBinaryStream { } virtual bool IsValid() const = 0; - virtual bool IsFailed() const = 0; + virtual bool IsFailed() const = 0; private: virtual int WriteImpl(const void* userBuffer, int size) = 0; @@ -36,49 +36,49 @@ private: i64 LongWrite(const void* userBuffer, i64 size); }; -template <int N_SIZE = 16384> +template <int N_SIZE = 16384> class TBufferedStream { char Buf[N_SIZE]; i64 Pos, BufSize; IBinaryStream& Stream; - bool bIsReading, bIsEof, bFailed; + bool bIsReading, bIsEof, bFailed; void ReadComplex(void* userBuffer, i64 size) { - if (bIsEof) { - memset(userBuffer, 0, size); - return; - } + if (bIsEof) { + memset(userBuffer, 0, size); + return; + } char* dst = (char*)userBuffer; i64 leftBytes = BufSize - Pos; - memcpy(dst, Buf + Pos, leftBytes); - dst += leftBytes; - size -= leftBytes; - Pos = BufSize = 0; - if (size > N_SIZE) { + memcpy(dst, Buf + Pos, leftBytes); + dst += leftBytes; + size -= leftBytes; + Pos = BufSize = 0; + if (size > N_SIZE) { i64 n = Stream.Read(dst, size); - bFailed = Stream.IsFailed(); - if (n != size) { - bIsEof = true; - memset(dst + n, 0, size - n); - } - } else { - BufSize = Stream.Read(Buf, N_SIZE); - bFailed = Stream.IsFailed(); - if (BufSize == 0) - bIsEof = true; - Read(dst, size); - } - } - + bFailed = Stream.IsFailed(); + if (n != size) { + bIsEof = true; + memset(dst + n, 0, size - n); + } + } else { + BufSize = Stream.Read(Buf, N_SIZE); + bFailed = Stream.IsFailed(); + if (BufSize == 0) + bIsEof = true; + Read(dst, size); + } + } + void WriteComplex(const void* userBuffer, i64 size) { - Flush(); - if (size >= N_SIZE) { - Stream.Write(userBuffer, size); - bFailed = Stream.IsFailed(); + Flush(); + if (size >= N_SIZE) { + Stream.Write(userBuffer, size); + bFailed = Stream.IsFailed(); } else - Write(userBuffer, size); - } - + Write(userBuffer, size); + } + void operator=(const TBufferedStream&) { } @@ -94,16 +94,16 @@ public: } ~TBufferedStream() { if (!bIsReading) - Flush(); + Flush(); } - void Flush() { + void Flush() { Y_ASSERT(!bIsReading); - if (bIsReading) - return; - Stream.Write(Buf, Pos); - bFailed = Stream.IsFailed(); - Pos = 0; - } + if (bIsReading) + return; + Stream.Write(Buf, Pos); + bFailed = Stream.IsFailed(); + Pos = 0; + } bool IsEof() const { return bIsEof; } diff --git a/library/cpp/binsaver/class_factory.h b/library/cpp/binsaver/class_factory.h index 4bfb225b28..e83512331b 100644 --- a/library/cpp/binsaver/class_factory.h +++ b/library/cpp/binsaver/class_factory.h @@ -15,12 +15,12 @@ public: typedef const std::type_info* VFT; private: - typedef T* (*newFunc)(); + typedef T* (*newFunc)(); typedef THashMap<int, newFunc> CTypeNewHash; // typeID->newFunc() typedef THashMap<VFT, int> CTypeIndexHash; // vftable->typeID - CTypeIndexHash typeIndex; - CTypeNewHash typeInfo; + CTypeIndexHash typeIndex; + CTypeNewHash typeInfo; void RegisterTypeBase(int nTypeID, newFunc func, VFT vft); static VFT GetObjectType(T* pObject) { @@ -29,15 +29,15 @@ private: int VFT2TypeID(VFT t) { CTypeIndexHash::iterator i = typeIndex.find(t); if (i != typeIndex.end()) - return i->second; + return i->second; for (i = typeIndex.begin(); i != typeIndex.end(); ++i) { if (*i->first == *t) { - typeIndex[t] = i->second; - return i->second; - } - } - return -1; - } + typeIndex[t] = i->second; + return i->second; + } + } + return -1; + } public: template <class TT> @@ -45,10 +45,10 @@ public: RegisterTypeBase(nTypeID, func, &typeid(TT)); } void RegisterTypeSafe(int nTypeID, newFunc func) { - TPtr<T> pObj = func(); + TPtr<T> pObj = func(); VFT vft = GetObjectType(pObj); RegisterTypeBase(nTypeID, func, vft); - } + } T* CreateObject(int nTypeID) { newFunc f = typeInfo[nTypeID]; if (f) diff --git a/library/cpp/binsaver/mem_io.h b/library/cpp/binsaver/mem_io.h index ca5cf0598a..2a9e36fe68 100644 --- a/library/cpp/binsaver/mem_io.h +++ b/library/cpp/binsaver/mem_io.h @@ -6,7 +6,7 @@ namespace NMemIoInternals { class TMemoryStream: public IBinaryStream { TVector<char>& Data; ui64 Pos; - + public: TMemoryStream(TVector<char>* data, ui64 pos = 0) : Data(*data) @@ -15,7 +15,7 @@ namespace NMemIoInternals { } ~TMemoryStream() override { } // keep gcc happy - + bool IsValid() const override { return true; } diff --git a/library/cpp/digest/sfh/sfh.h b/library/cpp/digest/sfh/sfh.h index 6535013201..372938654c 100644 --- a/library/cpp/digest/sfh/sfh.h +++ b/library/cpp/digest/sfh/sfh.h @@ -1,51 +1,51 @@ #pragma once - + #include <util/system/defaults.h> #include <util/system/unaligned_mem.h> - + inline ui32 SuperFastHash(const void* d, size_t l) noexcept { ui32 hash = (ui32)l; ui32 tmp; if (!l || !d) return 0; - + TUnalignedMemoryIterator<ui16, 4> iter(d, l); - + while (!iter.AtEnd()) { hash += (ui32)iter.Next(); tmp = ((ui32)iter.Next() << 11) ^ hash; hash = (hash << 16) ^ tmp; hash += hash >> 11; - } - + } + switch (iter.Left()) { case 3: hash += (ui32)iter.Next(); - hash ^= hash << 16; + hash ^= hash << 16; hash ^= ((ui32)(i32) * (const i8*)iter.Last()) << 18; - hash += hash >> 11; - break; + hash += hash >> 11; + break; case 2: hash += (ui32)iter.Cur(); - hash ^= hash << 11; - hash += hash >> 17; - break; + hash ^= hash << 11; + hash += hash >> 17; + break; case 1: hash += *((const i8*)iter.Last()); - hash ^= hash << 10; - hash += hash >> 1; - } - - /* Force "avalanching" of final 127 bits */ - hash ^= hash << 3; - hash += hash >> 5; - hash ^= hash << 4; - hash += hash >> 17; - hash ^= hash << 25; - hash += hash >> 6; - - return hash; -} + hash ^= hash << 10; + hash += hash >> 1; + } + + /* Force "avalanching" of final 127 bits */ + hash ^= hash << 3; + hash += hash >> 5; + hash ^= hash << 4; + hash += hash >> 17; + hash ^= hash << 25; + hash += hash >> 6; + + return hash; +} diff --git a/library/cpp/grpc/server/grpc_counters.h b/library/cpp/grpc/server/grpc_counters.h index 29600c7d96..0b6c36c84c 100644 --- a/library/cpp/grpc/server/grpc_counters.h +++ b/library/cpp/grpc/server/grpc_counters.h @@ -18,13 +18,13 @@ struct ICounterBlock : public TThrRefBase { virtual void CountRequestsWithoutDatabase() {} virtual void CountRequestsWithoutToken() {} virtual void CountRequestWithoutTls() {} - - virtual TIntrusivePtr<ICounterBlock> Clone() { return this; } - virtual void UseDatabase(const TString& database) { Y_UNUSED(database); } + + virtual TIntrusivePtr<ICounterBlock> Clone() { return this; } + virtual void UseDatabase(const TString& database) { Y_UNUSED(database); } }; -using ICounterBlockPtr = TIntrusivePtr<ICounterBlock>; - +using ICounterBlockPtr = TIntrusivePtr<ICounterBlock>; + class TCounterBlock final : public ICounterBlock { NMonitoring::TDynamicCounters::TCounterPtr TotalCounter; NMonitoring::TDynamicCounters::TCounterPtr InflyCounter; @@ -113,10 +113,10 @@ public: } } - ICounterBlockPtr Clone() override { - return this; - } - + ICounterBlockPtr Clone() override { + return this; + } + void Update() { if (Percentile) { RequestHistMs.Update(); diff --git a/library/cpp/grpc/server/grpc_request.h b/library/cpp/grpc/server/grpc_request.h index aeb8d31f29..5bd8d3902b 100644 --- a/library/cpp/grpc/server/grpc_request.h +++ b/library/cpp/grpc/server/grpc_request.h @@ -224,10 +224,10 @@ public: return &Arena_; } - void UseDatabase(const TString& database) override { - Counters_->UseDatabase(database); - } - + void UseDatabase(const TString& database) override { + Counters_->UseDatabase(database); + } + private: void Clone() { if (!Server_->IsShuttingDown()) { diff --git a/library/cpp/grpc/server/grpc_request_base.h b/library/cpp/grpc/server/grpc_request_base.h index c6d64ed648..fcfce1c181 100644 --- a/library/cpp/grpc/server/grpc_request_base.h +++ b/library/cpp/grpc/server/grpc_request_base.h @@ -90,9 +90,9 @@ public: //! The metadata will be send at the time of rpc finish virtual void AddTrailingMetadata(const TString& key, const TString& value) = 0; - //! Use validated database name for counters - virtual void UseDatabase(const TString& database) = 0; - + //! Use validated database name for counters + virtual void UseDatabase(const TString& database) = 0; + // Streaming part //! Set callback. The callback will be called when response deliverid to the client diff --git a/library/cpp/grpc/server/grpc_server.cpp b/library/cpp/grpc/server/grpc_server.cpp index 579ee24f06..7437b7a8f5 100644 --- a/library/cpp/grpc/server/grpc_server.cpp +++ b/library/cpp/grpc/server/grpc_server.cpp @@ -147,7 +147,7 @@ void TGRpcServer::Start() { Server_ = builder.BuildAndStart(); if (!Server_) { - ythrow yexception() << "can't start grpc server on " << server_address; + ythrow yexception() << "can't start grpc server on " << server_address; } size_t index = 0; diff --git a/library/cpp/monlib/dynamic_counters/contention_ut.cpp b/library/cpp/monlib/dynamic_counters/contention_ut.cpp index d142397e1a..8798044ee3 100644 --- a/library/cpp/monlib/dynamic_counters/contention_ut.cpp +++ b/library/cpp/monlib/dynamic_counters/contention_ut.cpp @@ -33,7 +33,7 @@ Y_UNIT_TEST_SUITE(TDynamicCountersContentionTest) { Response.Wait(); } - void OnHistogram(const TString& /*labelName*/, const TString& /*labelValue*/, IHistogramSnapshotPtr /*snapshot*/, bool /*derivative*/) override { + void OnHistogram(const TString& /*labelName*/, const TString& /*labelValue*/, IHistogramSnapshotPtr /*snapshot*/, bool /*derivative*/) override { } void OnGroupBegin(const TString& /*labelName*/, const TString& /*labelValue*/, const TDynamicCounters* /*group*/) override { diff --git a/library/cpp/monlib/dynamic_counters/counters.cpp b/library/cpp/monlib/dynamic_counters/counters.cpp index 977c8ea7b0..3635d87d0d 100644 --- a/library/cpp/monlib/dynamic_counters/counters.cpp +++ b/library/cpp/monlib/dynamic_counters/counters.cpp @@ -70,38 +70,38 @@ TDynamicCounters::TCounterPtr TDynamicCounters::GetNamedCounter(const TString& n return AsCounterRef(GetNamedCounterImpl<false, TCounterForPtr>(name, value, derivative, vis)); } -THistogramPtr TDynamicCounters::GetHistogram(const TString& value, IHistogramCollectorPtr collector, bool derivative, EVisibility vis) { - return GetNamedHistogram("sensor", value, std::move(collector), derivative, vis); +THistogramPtr TDynamicCounters::GetHistogram(const TString& value, IHistogramCollectorPtr collector, bool derivative, EVisibility vis) { + return GetNamedHistogram("sensor", value, std::move(collector), derivative, vis); } -THistogramPtr TDynamicCounters::GetNamedHistogram(const TString& name, const TString& value, IHistogramCollectorPtr collector, bool derivative, EVisibility vis) { - return AsHistogramRef(GetNamedCounterImpl<false, THistogramCounter>(name, value, std::move(collector), derivative, vis)); +THistogramPtr TDynamicCounters::GetNamedHistogram(const TString& name, const TString& value, IHistogramCollectorPtr collector, bool derivative, EVisibility vis) { + return AsHistogramRef(GetNamedCounterImpl<false, THistogramCounter>(name, value, std::move(collector), derivative, vis)); } -THistogramPtr TDynamicCounters::GetExpiringHistogram(const TString& value, IHistogramCollectorPtr collector, bool derivative, EVisibility vis) { - return GetExpiringNamedHistogram("sensor", value, std::move(collector), derivative, vis); +THistogramPtr TDynamicCounters::GetExpiringHistogram(const TString& value, IHistogramCollectorPtr collector, bool derivative, EVisibility vis) { + return GetExpiringNamedHistogram("sensor", value, std::move(collector), derivative, vis); } -THistogramPtr TDynamicCounters::GetExpiringNamedHistogram(const TString& name, const TString& value, IHistogramCollectorPtr collector, bool derivative, EVisibility vis) { - return AsHistogramRef(GetNamedCounterImpl<true, TExpiringHistogramCounter>(name, value, std::move(collector), derivative, vis)); +THistogramPtr TDynamicCounters::GetExpiringNamedHistogram(const TString& name, const TString& value, IHistogramCollectorPtr collector, bool derivative, EVisibility vis) { + return AsHistogramRef(GetNamedCounterImpl<true, TExpiringHistogramCounter>(name, value, std::move(collector), derivative, vis)); +} + +TDynamicCounters::TCounterPtr TDynamicCounters::FindCounter(const TString& value) const { + return FindNamedCounter("sensor", value); +} + +TDynamicCounters::TCounterPtr TDynamicCounters::FindNamedCounter(const TString& name, const TString& value) const { + return AsCounterRef(FindNamedCounterImpl<TCounterForPtr>(name, value)); +} + +THistogramPtr TDynamicCounters::FindHistogram(const TString& value) const { + return FindNamedHistogram("sensor", value); +} + +THistogramPtr TDynamicCounters::FindNamedHistogram(const TString& name,const TString& value) const { + return AsHistogramRef(FindNamedCounterImpl<THistogramCounter>(name, value)); } -TDynamicCounters::TCounterPtr TDynamicCounters::FindCounter(const TString& value) const { - return FindNamedCounter("sensor", value); -} - -TDynamicCounters::TCounterPtr TDynamicCounters::FindNamedCounter(const TString& name, const TString& value) const { - return AsCounterRef(FindNamedCounterImpl<TCounterForPtr>(name, value)); -} - -THistogramPtr TDynamicCounters::FindHistogram(const TString& value) const { - return FindNamedHistogram("sensor", value); -} - -THistogramPtr TDynamicCounters::FindNamedHistogram(const TString& name,const TString& value) const { - return AsHistogramRef(FindNamedCounterImpl<THistogramCounter>(name, value)); -} - void TDynamicCounters::RemoveCounter(const TString &value) { RemoveNamedCounter("sensor", value); } @@ -298,11 +298,11 @@ TDynamicCounters::TCountablePtr TDynamicCounters::GetNamedCounterImpl(const TStr } return it->second; } - -template <class TCounterType> -TDynamicCounters::TCountablePtr TDynamicCounters::FindNamedCounterImpl(const TString& name, const TString& value) const { - TReadGuard g(Lock); - auto it = Counters.find({name, value}); - return it != Counters.end() ? it->second : nullptr; -} - + +template <class TCounterType> +TDynamicCounters::TCountablePtr TDynamicCounters::FindNamedCounterImpl(const TString& name, const TString& value) const { + TReadGuard g(Lock); + auto it = Counters.find({name, value}); + return it != Counters.end() ? it->second : nullptr; +} + diff --git a/library/cpp/monlib/dynamic_counters/counters.h b/library/cpp/monlib/dynamic_counters/counters.h index 9882001de5..dc178cfbe0 100644 --- a/library/cpp/monlib/dynamic_counters/counters.h +++ b/library/cpp/monlib/dynamic_counters/counters.h @@ -66,7 +66,7 @@ namespace NMonitoring { virtual void OnHistogram( const TString& labelName, const TString& labelValue, - IHistogramSnapshotPtr snapshot, bool derivative) = 0; + IHistogramSnapshotPtr snapshot, bool derivative) = 0; virtual void OnGroupBegin( const TString& labelName, const TString& labelValue, @@ -93,8 +93,8 @@ namespace NMonitoring { Visibility_ = vis; } - TCounterForPtr(const TCounterForPtr&) = delete; - TCounterForPtr& operator=(const TCounterForPtr& other) = delete; + TCounterForPtr(const TCounterForPtr&) = delete; + TCounterForPtr& operator=(const TCounterForPtr& other) = delete; void Accept( const TString& labelName, const TString& labelValue, @@ -129,10 +129,10 @@ namespace NMonitoring { }; struct THistogramCounter: public TCountableBase { - explicit THistogramCounter( - IHistogramCollectorPtr collector, bool derivative = true, EVisibility vis = EVisibility::Public) + explicit THistogramCounter( + IHistogramCollectorPtr collector, bool derivative = true, EVisibility vis = EVisibility::Public) : Collector_(std::move(collector)) - , Derivative_(derivative) + , Derivative_(derivative) { Visibility_ = vis; } @@ -145,34 +145,34 @@ namespace NMonitoring { Collector_->Collect(value, count); } - void Collect(double value, ui32 count) { - Collector_->Collect(value, count); - } - - void Collect(const IHistogramSnapshot& snapshot) { - Collector_->Collect(snapshot); - } - + void Collect(double value, ui32 count) { + Collector_->Collect(value, count); + } + + void Collect(const IHistogramSnapshot& snapshot) { + Collector_->Collect(snapshot); + } + void Accept( const TString& labelName, const TString& labelValue, ICountableConsumer& consumer) const override { if (IsVisible(Visibility(), consumer.Visibility())) { - consumer.OnHistogram(labelName, labelValue, Collector_->Snapshot(), Derivative_); + consumer.OnHistogram(labelName, labelValue, Collector_->Snapshot(), Derivative_); } } - void Reset() { - Collector_->Reset(); - } - + void Reset() { + Collector_->Reset(); + } + IHistogramSnapshotPtr Snapshot() const { return Collector_->Snapshot(); } private: IHistogramCollectorPtr Collector_; - bool Derivative_; + bool Derivative_; }; struct TExpiringHistogramCounter: public THistogramCounter { @@ -284,14 +284,14 @@ namespace NMonitoring { THistogramPtr GetHistogram( const TString& value, IHistogramCollectorPtr collector, - bool derivative = true, + bool derivative = true, TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public); THistogramPtr GetNamedHistogram( const TString& name, const TString& value, IHistogramCollectorPtr collector, - bool derivative = true, + bool derivative = true, TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public); // These counters will be automatically removed from the registry @@ -310,22 +310,22 @@ namespace NMonitoring { THistogramPtr GetExpiringHistogram( const TString& value, IHistogramCollectorPtr collector, - bool derivative = true, + bool derivative = true, TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public); THistogramPtr GetExpiringNamedHistogram( const TString& name, const TString& value, IHistogramCollectorPtr collector, - bool derivative = true, + bool derivative = true, TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public); - TCounterPtr FindCounter(const TString& value) const; - TCounterPtr FindNamedCounter(const TString& name, const TString& value) const; - - THistogramPtr FindHistogram(const TString& value) const; - THistogramPtr FindNamedHistogram(const TString& name,const TString& value) const; - + TCounterPtr FindCounter(const TString& value) const; + TCounterPtr FindNamedCounter(const TString& name, const TString& value) const; + + THistogramPtr FindHistogram(const TString& value) const; + THistogramPtr FindNamedHistogram(const TString& name,const TString& value) const; + void RemoveCounter(const TString &value); void RemoveNamedCounter(const TString& name, const TString &value); @@ -366,9 +366,9 @@ namespace NMonitoring { template <bool expiring, class TCounterType, class... TArgs> TCountablePtr GetNamedCounterImpl(const TString& name, const TString& value, TArgs&&... args); - - template <class TCounterType> - TCountablePtr FindNamedCounterImpl(const TString& name, const TString& value) const; + + template <class TCounterType> + TCountablePtr FindNamedCounterImpl(const TString& name, const TString& value) const; }; } diff --git a/library/cpp/monlib/dynamic_counters/counters_ut.cpp b/library/cpp/monlib/dynamic_counters/counters_ut.cpp index 04285ae5e3..3591037e0a 100644 --- a/library/cpp/monlib/dynamic_counters/counters_ut.cpp +++ b/library/cpp/monlib/dynamic_counters/counters_ut.cpp @@ -23,7 +23,7 @@ private: void OnHistogram( const TString& labelName, const TString& labelValue, - IHistogramSnapshotPtr snapshot, bool /*derivative*/) override { + IHistogramSnapshotPtr snapshot, bool /*derivative*/) override { Indent(Out_, Level_) << labelName << ':' << labelValue << " = " << *snapshot << '\n'; @@ -311,32 +311,32 @@ Y_UNIT_TEST_SUITE(TDynamicCountersTest) { UNIT_ASSERT_VALUES_EQUAL(subGroupLookups->Val(), 1); UNIT_ASSERT_VALUES_EQUAL(lookups->Val(), 5); } - - Y_UNIT_TEST(FindCounters) { - TDynamicCounterPtr rootGroup(new TDynamicCounters()); - - auto counter = rootGroup->FindCounter("counter1"); - UNIT_ASSERT(!counter); - rootGroup->GetCounter("counter1"); - counter = rootGroup->FindCounter("counter1"); - UNIT_ASSERT(counter); - - counter = rootGroup->FindNamedCounter("name", "counter2"); - UNIT_ASSERT(!counter); - rootGroup->GetNamedCounter("name", "counter2"); - counter = rootGroup->FindNamedCounter("name", "counter2"); - UNIT_ASSERT(counter); - - auto histogram = rootGroup->FindHistogram("histogram1"); - UNIT_ASSERT(!histogram); - rootGroup->GetHistogram("histogram1", ExponentialHistogram(4, 2)); - histogram = rootGroup->FindHistogram("histogram1"); - UNIT_ASSERT(histogram); - - histogram = rootGroup->FindNamedHistogram("name", "histogram2"); - UNIT_ASSERT(!histogram); - rootGroup->GetNamedHistogram("name", "histogram2", ExponentialHistogram(4, 2)); - histogram = rootGroup->FindNamedHistogram("name", "histogram2"); - UNIT_ASSERT(histogram); - } + + Y_UNIT_TEST(FindCounters) { + TDynamicCounterPtr rootGroup(new TDynamicCounters()); + + auto counter = rootGroup->FindCounter("counter1"); + UNIT_ASSERT(!counter); + rootGroup->GetCounter("counter1"); + counter = rootGroup->FindCounter("counter1"); + UNIT_ASSERT(counter); + + counter = rootGroup->FindNamedCounter("name", "counter2"); + UNIT_ASSERT(!counter); + rootGroup->GetNamedCounter("name", "counter2"); + counter = rootGroup->FindNamedCounter("name", "counter2"); + UNIT_ASSERT(counter); + + auto histogram = rootGroup->FindHistogram("histogram1"); + UNIT_ASSERT(!histogram); + rootGroup->GetHistogram("histogram1", ExponentialHistogram(4, 2)); + histogram = rootGroup->FindHistogram("histogram1"); + UNIT_ASSERT(histogram); + + histogram = rootGroup->FindNamedHistogram("name", "histogram2"); + UNIT_ASSERT(!histogram); + rootGroup->GetNamedHistogram("name", "histogram2", ExponentialHistogram(4, 2)); + histogram = rootGroup->FindNamedHistogram("name", "histogram2"); + UNIT_ASSERT(histogram); + } } diff --git a/library/cpp/monlib/dynamic_counters/encode.cpp b/library/cpp/monlib/dynamic_counters/encode.cpp index 6f5005f71a..ffa48d276e 100644 --- a/library/cpp/monlib/dynamic_counters/encode.cpp +++ b/library/cpp/monlib/dynamic_counters/encode.cpp @@ -3,7 +3,7 @@ #include <library/cpp/monlib/encode/encoder.h> #include <library/cpp/monlib/encode/json/json.h> #include <library/cpp/monlib/encode/spack/spack_v1.h> -#include <library/cpp/monlib/encode/prometheus/prometheus.h> +#include <library/cpp/monlib/encode/prometheus/prometheus.h> #include <util/stream/str.h> @@ -41,10 +41,10 @@ namespace NMonitoring { void OnHistogram( const TString& labelName, const TString& labelValue, - IHistogramSnapshotPtr snapshot, bool derivative) override { - NMonitoring::EMetricType metricType = derivative ? EMetricType::HIST_RATE : EMetricType::HIST; - - EncoderImpl_->OnMetricBegin(metricType); + IHistogramSnapshotPtr snapshot, bool derivative) override { + NMonitoring::EMetricType metricType = derivative ? EMetricType::HIST_RATE : EMetricType::HIST; + + EncoderImpl_->OnMetricBegin(metricType); EncodeLabels(labelName, labelValue); EncoderImpl_->OnHistogram(ZERO_TIME, snapshot); EncoderImpl_->OnMetricEnd(); @@ -104,9 +104,9 @@ namespace NMonitoring { out, NMonitoring::ETimePrecision::SECONDS, NMonitoring::ECompression::ZSTD), vis); - case EFormat::PROMETHEUS: - return MakeHolder<TConsumer>(NMonitoring::EncoderPrometheus( - out), vis); + case EFormat::PROMETHEUS: + return MakeHolder<TConsumer>(NMonitoring::EncoderPrometheus( + out), vis); default: ythrow yexception() << "unsupported metric encoding format: " << format; break; diff --git a/library/cpp/monlib/dynamic_counters/golovan_page.cpp b/library/cpp/monlib/dynamic_counters/golovan_page.cpp index 6d76956234..49cf2d39bb 100644 --- a/library/cpp/monlib/dynamic_counters/golovan_page.cpp +++ b/library/cpp/monlib/dynamic_counters/golovan_page.cpp @@ -39,7 +39,7 @@ public: out << "\"," << counter->Val() << "]"; } - void OnHistogram(const TString&, const TString&, IHistogramSnapshotPtr, bool) override { + void OnHistogram(const TString&, const TString&, IHistogramSnapshotPtr, bool) override { } void OnGroupBegin(const TString&, const TString& value, const TDynamicCounters*) override { diff --git a/library/cpp/monlib/dynamic_counters/page.cpp b/library/cpp/monlib/dynamic_counters/page.cpp index 812ee6b4ca..5124a47bb3 100644 --- a/library/cpp/monlib/dynamic_counters/page.cpp +++ b/library/cpp/monlib/dynamic_counters/page.cpp @@ -19,8 +19,8 @@ TMaybe<EFormat> ParseFormat(TStringBuf str) { return EFormat::JSON; } else if (str == TStringBuf("spack")) { return EFormat::SPACK; - } else if (str == TStringBuf("prometheus")) { - return EFormat::PROMETHEUS; + } else if (str == TStringBuf("prometheus")) { + return EFormat::PROMETHEUS; } else { return Nothing(); } @@ -84,8 +84,8 @@ void TDynamicCountersPage::Output(NMonitoring::IMonHttpRequest& request) { out << HTTPOKJSON; } else if (*format == EFormat::SPACK) { out << HTTPOKSPACK; - } else if (*format == EFormat::PROMETHEUS) { - out << HTTPOKPROMETHEUS; + } else if (*format == EFormat::PROMETHEUS) { + out << HTTPOKPROMETHEUS; } else { ythrow yexception() << "unsupported metric encoding format: " << *format; } diff --git a/library/cpp/monlib/dynamic_counters/ya.make b/library/cpp/monlib/dynamic_counters/ya.make index 0b1e97fa4e..aafe1c34be 100644 --- a/library/cpp/monlib/dynamic_counters/ya.make +++ b/library/cpp/monlib/dynamic_counters/ya.make @@ -18,7 +18,7 @@ PEERDIR( library/cpp/containers/stack_vector library/cpp/monlib/encode/json library/cpp/monlib/encode/spack - library/cpp/monlib/encode/prometheus + library/cpp/monlib/encode/prometheus library/cpp/monlib/service/pages library/cpp/string_utils/quote library/cpp/threading/light_rw_lock diff --git a/library/cpp/monlib/service/pages/mon_page.h b/library/cpp/monlib/service/pages/mon_page.h index ad7afb867d..e396612bb0 100644 --- a/library/cpp/monlib/service/pages/mon_page.h +++ b/library/cpp/monlib/service/pages/mon_page.h @@ -12,7 +12,7 @@ namespace NMonitoring { static const char HTTPOKHTML[] = "HTTP/1.1 200 Ok\r\nContent-Type: text/html\r\nConnection: Close\r\n\r\n"; static const char HTTPOKJSON[] = "HTTP/1.1 200 Ok\r\nContent-Type: application/json\r\nConnection: Close\r\n\r\n"; static const char HTTPOKSPACK[] = "HTTP/1.1 200 Ok\r\nContent-Type: application/x-solomon-spack\r\nConnection: Close\r\n\r\n"; - static const char HTTPOKPROMETHEUS[] = "HTTP/1.1 200 Ok\r\nContent-Type: text/plain\r\nConnection: Close\r\n\r\n"; + static const char HTTPOKPROMETHEUS[] = "HTTP/1.1 200 Ok\r\nContent-Type: text/plain\r\nConnection: Close\r\n\r\n"; static const char HTTPOKJAVASCRIPT[] = "HTTP/1.1 200 Ok\r\nContent-Type: text/javascript\r\nConnection: Close\r\n\r\n"; static const char HTTPOKCSS[] = "HTTP/1.1 200 Ok\r\nContent-Type: text/css\r\nConnection: Close\r\n\r\n"; static const char HTTPNOCONTENT[] = "HTTP/1.1 204 No content\r\nConnection: Close\r\n\r\n"; diff --git a/library/cpp/yson/consumer.cpp b/library/cpp/yson/consumer.cpp index 33e9ee6327..40ae452978 100644 --- a/library/cpp/yson/consumer.cpp +++ b/library/cpp/yson/consumer.cpp @@ -1,15 +1,15 @@ -#include "consumer.h" +#include "consumer.h" #include "string.h" -#include "parser.h" - +#include "parser.h" + namespace NYson { //////////////////////////////////////////////////////////////////////////////// - + void TYsonConsumerBase::OnRaw(TStringBuf str, NYT::NYson::EYsonType type) { ParseYsonStringBuffer(str, this, type); } - + //////////////////////////////////////////////////////////////////////////////// } // namespace NYson diff --git a/library/cpp/yson/consumer.h b/library/cpp/yson/consumer.h index fe1902c8f0..d5a9d66335 100644 --- a/library/cpp/yson/consumer.h +++ b/library/cpp/yson/consumer.h @@ -1,10 +1,10 @@ -#pragma once - +#pragma once + #include <library/cpp/yt/yson/consumer.h> - -#include <util/generic/strbuf.h> -#include <util/system/defaults.h> - + +#include <util/generic/strbuf.h> +#include <util/system/defaults.h> + namespace NYson { struct TYsonConsumerBase : public virtual NYT::NYson::IYsonConsumer { diff --git a/library/cpp/yson/detail.h b/library/cpp/yson/detail.h index d2a4a6308e..27f5e8ffff 100644 --- a/library/cpp/yson/detail.h +++ b/library/cpp/yson/detail.h @@ -1,47 +1,47 @@ -#pragma once - -#include "public.h" -#include "zigzag.h" - -#include <util/generic/vector.h> -#include <util/generic/maybe.h> -#include <util/generic/buffer.h> -#include <util/string/escape.h> -#include <util/string/cast.h> -#include <util/stream/input.h> - +#pragma once + +#include "public.h" +#include "zigzag.h" + +#include <util/generic/vector.h> +#include <util/generic/maybe.h> +#include <util/generic/buffer.h> +#include <util/string/escape.h> +#include <util/string/cast.h> +#include <util/stream/input.h> + namespace NYson { namespace NDetail { //////////////////////////////////////////////////////////////////////////////// - + //! Indicates the beginning of a list. const char BeginListSymbol = '['; //! Indicates the end of a list. const char EndListSymbol = ']'; - + //! Indicates the beginning of a map. const char BeginMapSymbol = '{'; //! Indicates the end of a map. const char EndMapSymbol = '}'; - + //! Indicates the beginning of an attribute map. const char BeginAttributesSymbol = '<'; //! Indicates the end of an attribute map. const char EndAttributesSymbol = '>'; - + //! Separates items in lists. const char ListItemSeparatorSymbol = ';'; //! Separates items in maps, attributes. const char KeyedItemSeparatorSymbol = ';'; //! Separates keys from values in maps. const char KeyValueSeparatorSymbol = '='; - + //! Indicates an entity. const char EntitySymbol = '#'; - + //! Indicates end of stream. const char EndSymbol = '\0'; - + //! Marks the beginning of a binary string literal. const char StringMarker = '\x01'; //! Marks the beginning of a binary i64 literal. @@ -53,19 +53,19 @@ namespace NYson { const char TrueMarker = '\x05'; //! Marks the beginning of a binary ui64 literal. const char Uint64Marker = '\x06'; - + //////////////////////////////////////////////////////////////////////////////// - + template <bool EnableLinePositionInfo> class TPositionInfo; - + template <> class TPositionInfo<true> { private: int Offset; int Line; int Column; - + public: TPositionInfo() : Offset(0) @@ -73,7 +73,7 @@ namespace NYson { , Column(1) { } - + void OnRangeConsumed(const char* begin, const char* end) { Offset += end - begin; for (auto current = begin; current != end; ++current) { @@ -85,7 +85,7 @@ namespace NYson { } } }; - + template <> class TPositionInfo<false> { private: @@ -95,13 +95,13 @@ namespace NYson { TPositionInfo() : Offset(0) { - } - + } + void OnRangeConsumed(const char* begin, const char* end) { Offset += end - begin; } }; - + template <class TBlockStream, class TPositionBase> class TCharStream : public TBlockStream, @@ -111,11 +111,11 @@ namespace NYson { : TBlockStream(blockStream) { } - + bool IsEmpty() const { return TBlockStream::Begin() == TBlockStream::End(); } - + template <bool AllowFinish> void Refresh() { while (IsEmpty() && !TBlockStream::IsFinished()) { @@ -125,46 +125,46 @@ namespace NYson { ythrow TYsonException() << "Premature end of yson stream"; } } - + void Refresh() { return Refresh<false>(); } - + template <bool AllowFinish> char GetChar() { Refresh<AllowFinish>(); return !IsEmpty() ? *TBlockStream::Begin() : '\0'; } - + char GetChar() { return GetChar<false>(); } - + void Advance(size_t bytes) { TPositionBase::OnRangeConsumed(TBlockStream::Begin(), TBlockStream::Begin() + bytes); TBlockStream::Advance(bytes); } - + size_t Length() const { return TBlockStream::End() - TBlockStream::Begin(); } }; - + template <class TBaseStream> class TCodedStream : public TBaseStream { private: static const int MaxVarintBytes = 10; static const int MaxVarint32Bytes = 5; - + const ui8* BeginByte() const { return reinterpret_cast<const ui8*>(TBaseStream::Begin()); } - + const ui8* EndByte() const { return reinterpret_cast<const ui8*>(TBaseStream::End()); } - + // Following functions is an adaptation Protobuf code from coded_stream.cc bool ReadVarint32FromArray(ui32* value) { // Fast path: We have enough bytes left in the buffer to guarantee that @@ -172,7 +172,7 @@ namespace NYson { const ui8* ptr = BeginByte(); ui32 b; ui32 result; - + b = *(ptr++); result = (b & 0x7F); if (!(b & 0x80)) @@ -193,26 +193,26 @@ namespace NYson { result |= b << 28; if (!(b & 0x80)) goto done; - + // If the input is larger than 32 bits, we still need to read it all // and discard the high-order bits. - + for (int i = 0; i < MaxVarintBytes - MaxVarint32Bytes; i++) { b = *(ptr++); if (!(b & 0x80)) goto done; } - + // We have overrun the maximum size of a Varint (10 bytes). Assume // the data is corrupt. return false; - + done: TBaseStream::Advance(ptr - BeginByte()); *value = result; return true; } - + bool ReadVarint32Fallback(ui32* value) { if (BeginByte() + MaxVarint32Bytes <= EndByte() || // Optimization: If the Varint ends at exactly the end of the buffer, @@ -227,7 +227,7 @@ namespace NYson { return ReadVarint32Slow(value); } } - + bool ReadVarint32Slow(ui32* value) { ui64 result; // Directly invoke ReadVarint64Fallback, since we already tried to optimize @@ -239,15 +239,15 @@ namespace NYson { return false; } } - + bool ReadVarint64Slow(ui64* value) { // Slow path: This read might cross the end of the buffer, so we // need to check and refresh the buffer if and when it does. - + ui64 result = 0; int count = 0; ui32 b; - + do { if (count == MaxVarintBytes) { return false; @@ -260,11 +260,11 @@ namespace NYson { TBaseStream::Advance(1); ++count; } while (b & 0x80); - + *value = result; return true; - } - + } + bool ReadVarint64Fallback(ui64* value) { if (BeginByte() + MaxVarintBytes <= EndByte() || // Optimization: If the Varint ends at exactly the end of the buffer, @@ -273,14 +273,14 @@ namespace NYson { { // Fast path: We have enough bytes left in the buffer to guarantee that // this read won't cross the end, so we can skip the checks. - + const ui8* ptr = BeginByte(); ui32 b; - + // Splitting into 32-bit pieces gives better performance on 32-bit // processors. ui32 part0 = 0, part1 = 0, part2 = 0; - + b = *(ptr++); part0 = (b & 0x7F); if (!(b & 0x80)) @@ -321,11 +321,11 @@ namespace NYson { part2 |= (b & 0x7F) << 7; if (!(b & 0x80)) goto done; - + // We have overrun the maximum size of a Varint (10 bytes). The data // must be corrupt. return false; - + done: TBaseStream::Advance(ptr - BeginByte()); *value = (static_cast<ui64>(part0)) | @@ -336,13 +336,13 @@ namespace NYson { return ReadVarint64Slow(value); } } - + public: TCodedStream(const TBaseStream& baseStream) : TBaseStream(baseStream) { } - + bool ReadVarint64(ui64* value) { if (BeginByte() < EndByte() && *BeginByte() < 0x80) { *value = *BeginByte(); @@ -352,7 +352,7 @@ namespace NYson { return ReadVarint64Fallback(value); } } - + bool ReadVarint32(ui32* value) { if (BeginByte() < EndByte() && *BeginByte() < 0x80) { *value = *BeginByte(); @@ -363,13 +363,13 @@ namespace NYson { } } }; - + enum ENumericResult { Int64 = 0, Uint64 = 1, Double = 2 }; - + template <class TBlockStream, bool EnableLinePositionInfo> class TLexerBase : public TCodedStream<TCharStream<TBlockStream, TPositionInfo<EnableLinePositionInfo>>> { @@ -377,7 +377,7 @@ namespace NYson { using TBaseStream = TCodedStream<TCharStream<TBlockStream, TPositionInfo<EnableLinePositionInfo>>>; TVector<char> Buffer_; TMaybe<ui64> MemoryLimit_; - + void CheckMemoryLimit() { if (MemoryLimit_ && Buffer_.capacity() > *MemoryLimit_) { ythrow TYsonException() @@ -385,17 +385,17 @@ namespace NYson { << Buffer_.capacity() << ", limit " << (*MemoryLimit_); } } - + public: TLexerBase(const TBlockStream& blockStream, TMaybe<ui64> memoryLimit) : TBaseStream(blockStream) , MemoryLimit_(memoryLimit) { } - + protected: /// Lexer routines - + template <bool AllowFinish> ENumericResult ReadNumeric(TStringBuf* value) { Buffer_.clear(); @@ -418,18 +418,18 @@ namespace NYson { CheckMemoryLimit(); TBaseStream::Advance(1); } - + *value = TStringBuf(Buffer_.data(), Buffer_.size()); return result; - } - + } + template <bool AllowFinish> double ReadNanOrInf() { static const TStringBuf nanString = "nan"; static const TStringBuf infString = "inf"; static const TStringBuf plusInfString = "+inf"; static const TStringBuf minusInfString = "-inf"; - + TStringBuf expectedString; double expectedValue; char ch = TBaseStream::template GetChar<AllowFinish>(); @@ -515,25 +515,25 @@ namespace NYson { } CheckMemoryLimit(); TBaseStream::Advance(1); - } + } *value = TStringBuf(Buffer_.data(), Buffer_.size()); - } - + } + void ReadUnquotedString(TStringBuf* value) { return ReadUnquotedString<false>(value); - } - + } + void ReadBinaryString(TStringBuf* value) { ui32 ulength = 0; if (!TBaseStream::ReadVarint32(&ulength)) { ythrow TYsonException() << "Error parsing varint value"; } - + i32 length = ZigZagDecode32(ulength); if (length < 0) { ythrow TYsonException() << "Negative binary string literal length " << length; } - + if (TBaseStream::Begin() + length <= TBaseStream::End()) { *value = TStringBuf(TBaseStream::Begin(), length); TBaseStream::Advance(length); @@ -546,29 +546,29 @@ namespace NYson { continue; } size_t readingBytes = Min(needToRead, TBaseStream::Length()); - + Buffer_.insert(Buffer_.end(), TBaseStream::Begin(), TBaseStream::Begin() + readingBytes); CheckMemoryLimit(); needToRead -= readingBytes; TBaseStream::Advance(readingBytes); } *value = TStringBuf(Buffer_.data(), Buffer_.size()); - } - } - + } + } + template <bool AllowFinish> bool ReadBoolean() { Buffer_.clear(); - + static TStringBuf trueString = "true"; static TStringBuf falseString = "false"; - + auto throwIncorrectBoolean = [&]() { ythrow TYsonException() << "Incorrect boolean string " << TString(Buffer_.data(), Buffer_.size()); }; - - Buffer_.push_back(TBaseStream::template GetChar<AllowFinish>()); - TBaseStream::Advance(1); + + Buffer_.push_back(TBaseStream::template GetChar<AllowFinish>()); + TBaseStream::Advance(1); if (Buffer_[0] == trueString[0]) { for (size_t i = 1; i < trueString.size(); ++i) { Buffer_.push_back(TBaseStream::template GetChar<AllowFinish>()); @@ -588,21 +588,21 @@ namespace NYson { } return false; } else { - throwIncorrectBoolean(); - } + throwIncorrectBoolean(); + } Y_FAIL("unreachable"); ; - } + } void ReadBinaryInt64(i64* result) { ui64 uvalue; if (!TBaseStream::ReadVarint64(&uvalue)) { ythrow TYsonException() << "Error parsing varint value"; - } + } *result = ZigZagDecode64(uvalue); - } - + } + void ReadBinaryUint64(ui64* result) { ui64 uvalue; if (!TBaseStream::ReadVarint64(&uvalue)) { @@ -610,16 +610,16 @@ namespace NYson { } *result = uvalue; } - + void ReadBinaryDouble(double* value) { size_t needToRead = sizeof(double); - + while (needToRead != 0) { if (TBaseStream::IsEmpty()) { TBaseStream::Refresh(); continue; } - + size_t chunkSize = Min(needToRead, TBaseStream::Length()); if (chunkSize == 0) { ythrow TYsonException() << "Error parsing binary double literal"; @@ -631,8 +631,8 @@ namespace NYson { needToRead -= chunkSize; TBaseStream::Advance(chunkSize); } - } - + } + /// Helpers void SkipCharToken(char symbol) { char ch = SkipSpaceAndGetChar(); @@ -641,8 +641,8 @@ namespace NYson { } TBaseStream::Advance(1); - } - + } + static bool IsSpaceFast(char ch) { static const ui8 lookupTable[] = { @@ -650,24 +650,24 @@ namespace NYson { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; return lookupTable[static_cast<ui8>(ch)]; } - + template <bool AllowFinish> char SkipSpaceAndGetChar() { if (!TBaseStream::IsEmpty()) { @@ -678,11 +678,11 @@ namespace NYson { } return SkipSpaceAndGetCharFallback<AllowFinish>(); } - + char SkipSpaceAndGetChar() { return SkipSpaceAndGetChar<false>(); - } - + } + template <bool AllowFinish> char SkipSpaceAndGetCharFallback() { while (true) { @@ -697,63 +697,63 @@ namespace NYson { break; } TBaseStream::Advance(1); - } + } return TBaseStream::template GetChar<AllowFinish>(); - } + } }; - + //////////////////////////////////////////////////////////////////////////////// - + } - + //////////////////////////////////////////////////////////////////////////////// - + class TStringReader { private: const char* BeginPtr; const char* EndPtr; - + public: TStringReader() : BeginPtr(nullptr) , EndPtr(nullptr) { } - + TStringReader(const char* begin, const char* end) : BeginPtr(begin) , EndPtr(end) { } - + const char* Begin() const { return BeginPtr; } - + const char* End() const { return EndPtr; } - + void RefreshBlock() { Y_FAIL("unreachable"); } - + void Advance(size_t bytes) { BeginPtr += bytes; } - + bool IsFinished() const { return true; } - + void SetBuffer(const char* begin, const char* end) { BeginPtr = begin; EndPtr = end; } }; - + //////////////////////////////////////////////////////////////////////////////// - + class TStreamReader { public: TStreamReader( @@ -767,40 +767,40 @@ namespace NYson { BeginPtr = EndPtr = Buffer; FinishFlag = false; } - + const char* Begin() const { return BeginPtr; } - + const char* End() const { return EndPtr; } - + void RefreshBlock() { size_t bytes = Stream->Read(Buffer, BufferSize); BeginPtr = Buffer; EndPtr = Buffer + bytes; FinishFlag = (bytes == 0); } - + void Advance(size_t bytes) { BeginPtr += bytes; } - + bool IsFinished() const { return FinishFlag; } - + private: IInputStream* Stream; char* Buffer; size_t BufferSize; - + const char* BeginPtr; const char* EndPtr; bool FinishFlag; }; - + //////////////////////////////////////////////////////////////////////////////// - + } // namespace NYson diff --git a/library/cpp/yson/format.h b/library/cpp/yson/format.h index c2bad91a6a..2ff6dc9f6e 100644 --- a/library/cpp/yson/format.h +++ b/library/cpp/yson/format.h @@ -1,25 +1,25 @@ -#pragma once - -#include "token.h" - +#pragma once + +#include "token.h" + namespace NYson { //////////////////////////////////////////////////////////////////////////////// - + const ETokenType BeginListToken = LeftBracket; const ETokenType EndListToken = RightBracket; - + const ETokenType BeginMapToken = LeftBrace; const ETokenType EndMapToken = RightBrace; - + const ETokenType BeginAttributesToken = LeftAngle; const ETokenType EndAttributesToken = RightAngle; - + const ETokenType ListItemSeparatorToken = Semicolon; const ETokenType KeyedItemSeparatorToken = Semicolon; const ETokenType KeyValueSeparatorToken = Equals; - + const ETokenType EntityToken = Hash; - + //////////////////////////////////////////////////////////////////////////////// - + } // namespace NYson diff --git a/library/cpp/yson/json/json_writer.cpp b/library/cpp/yson/json/json_writer.cpp index 2c961a02fa..87481256ec 100644 --- a/library/cpp/yson/json/json_writer.cpp +++ b/library/cpp/yson/json/json_writer.cpp @@ -1,16 +1,16 @@ -#include "json_writer.h" - +#include "json_writer.h" + #include <library/cpp/json/json_writer.h> - -namespace NYT { + +namespace NYT { //////////////////////////////////////////////////////////////////////////////// - + static bool IsSpecialJsonKey(const TStringBuf& key) { return key.size() > 0 && key[0] == '$'; } - + //////////////////////////////////////////////////////////////////////////////// - + TJsonWriter::TJsonWriter( IOutputStream* output, ::NYson::EYsonType type, @@ -41,15 +41,15 @@ namespace NYT { if (Type == ::NYson::EYsonType::MapFragment) { ythrow ::NYson::TYsonException() << ("Map fragments are not supported by Json"); } - + UnderlyingJsonWriter.Reset(new NJson::TJsonWriter( output, config)); JsonWriter = UnderlyingJsonWriter.Get(); HasAttributes = false; InAttributesBalance = 0; - } - + } + void TJsonWriter::EnterNode() { if (AttributesMode == JAM_NEVER) { HasAttributes = false; @@ -65,15 +65,15 @@ namespace NYT { HasAttributes = true; } HasUnfoldedStructureStack.push_back(HasAttributes); - + if (HasAttributes) { JsonWriter->Write("$value"); HasAttributes = false; - } - + } + Depth += 1; - } - + } + void TJsonWriter::LeaveNode() { Y_ASSERT(!HasUnfoldedStructureStack.empty()); if (HasUnfoldedStructureStack.back()) { @@ -81,54 +81,54 @@ namespace NYT { JsonWriter->CloseMap(); } HasUnfoldedStructureStack.pop_back(); - + Depth -= 1; if (Depth == 0 && Type == ::NYson::EYsonType::ListFragment && InAttributesBalance == 0) { JsonWriter->Flush(); Output->Write("\n"); } - } - + } + bool TJsonWriter::IsWriteAllowed() { if (AttributesMode == JAM_NEVER) { return InAttributesBalance == 0; } return true; - } - + } + void TJsonWriter::OnStringScalar(TStringBuf value) { if (IsWriteAllowed()) { EnterNode(); WriteStringScalar(value); LeaveNode(); } - } - + } + void TJsonWriter::OnInt64Scalar(i64 value) { if (IsWriteAllowed()) { EnterNode(); JsonWriter->Write(value); LeaveNode(); } - } - + } + void TJsonWriter::OnUint64Scalar(ui64 value) { if (IsWriteAllowed()) { EnterNode(); JsonWriter->Write(value); LeaveNode(); } - } - + } + void TJsonWriter::OnDoubleScalar(double value) { if (IsWriteAllowed()) { EnterNode(); JsonWriter->Write(value); LeaveNode(); } - } - + } + void TJsonWriter::OnBooleanScalar(bool value) { if (IsWriteAllowed()) { if (BooleanFormat == SBF_STRING) { @@ -139,40 +139,40 @@ namespace NYT { LeaveNode(); } } - } - + } + void TJsonWriter::OnEntity() { if (IsWriteAllowed()) { EnterNode(); JsonWriter->WriteNull(); LeaveNode(); } - } - + } + void TJsonWriter::OnBeginList() { if (IsWriteAllowed()) { EnterNode(); JsonWriter->OpenArray(); } - } - + } + void TJsonWriter::OnListItem() { - } - + } + void TJsonWriter::OnEndList() { if (IsWriteAllowed()) { JsonWriter->CloseArray(); LeaveNode(); } } - + void TJsonWriter::OnBeginMap() { if (IsWriteAllowed()) { EnterNode(); JsonWriter->OpenMap(); } - } - + } + void TJsonWriter::OnKeyedItem(TStringBuf name) { if (IsWriteAllowed()) { if (IsSpecialJsonKey(name)) { @@ -181,15 +181,15 @@ namespace NYT { WriteStringScalar(name); } } - } - + } + void TJsonWriter::OnEndMap() { if (IsWriteAllowed()) { JsonWriter->CloseMap(); LeaveNode(); - } - } - + } + } + void TJsonWriter::OnBeginAttributes() { InAttributesBalance += 1; if (AttributesMode != JAM_NEVER) { @@ -197,24 +197,24 @@ namespace NYT { JsonWriter->Write("$attributes"); JsonWriter->OpenMap(); } - } - + } + void TJsonWriter::OnEndAttributes() { InAttributesBalance -= 1; if (AttributesMode != JAM_NEVER) { HasAttributes = true; JsonWriter->CloseMap(); } - } - + } + void TJsonWriter::WriteStringScalar(const TStringBuf& value) { JsonWriter->Write(value); - } - + } + void TJsonWriter::Flush() { JsonWriter->Flush(); } - + //////////////////////////////////////////////////////////////////////////////// -} +} diff --git a/library/cpp/yson/json/json_writer.h b/library/cpp/yson/json/json_writer.h index 43e5293b0d..d84ac0de53 100644 --- a/library/cpp/yson/json/json_writer.h +++ b/library/cpp/yson/json/json_writer.h @@ -1,31 +1,31 @@ -#pragma once - +#pragma once + #include <library/cpp/yson/public.h> #include <library/cpp/yson/consumer.h> - + #include <library/cpp/json/json_writer.h> - -#include <util/generic/vector.h> - -namespace NYT { + +#include <util/generic/vector.h> + +namespace NYT { //////////////////////////////////////////////////////////////////////////////// - + enum EJsonFormat { JF_TEXT, JF_PRETTY }; - + enum EJsonAttributesMode { JAM_NEVER, JAM_ON_DEMAND, JAM_ALWAYS }; - + enum ESerializedBoolFormat { SBF_BOOLEAN, SBF_STRING }; - + class TJsonWriter : public ::NYson::TYsonConsumerBase { public: @@ -44,26 +44,26 @@ namespace NYT { ESerializedBoolFormat booleanFormat = SBF_STRING); void Flush(); - + void OnStringScalar(TStringBuf value) override; void OnInt64Scalar(i64 value) override; void OnUint64Scalar(ui64 value) override; void OnDoubleScalar(double value) override; void OnBooleanScalar(bool value) override; - + void OnEntity() override; - + void OnBeginList() override; void OnListItem() override; void OnEndList() override; - + void OnBeginMap() override; void OnKeyedItem(TStringBuf key) override; void OnEndMap() override; - + void OnBeginAttributes() override; void OnEndAttributes() override; - + private: THolder<NJson::TJsonWriter> UnderlyingJsonWriter; NJson::TJsonWriter* JsonWriter; @@ -71,19 +71,19 @@ namespace NYT { ::NYson::EYsonType Type; EJsonAttributesMode AttributesMode; ESerializedBoolFormat BooleanFormat; - + void WriteStringScalar(const TStringBuf& value); - + void EnterNode(); void LeaveNode(); bool IsWriteAllowed(); - + TVector<bool> HasUnfoldedStructureStack; int InAttributesBalance; bool HasAttributes; int Depth; }; - + //////////////////////////////////////////////////////////////////////////////// - + } diff --git a/library/cpp/yson/lexer.cpp b/library/cpp/yson/lexer.cpp index 6864991c24..5eae94273b 100644 --- a/library/cpp/yson/lexer.cpp +++ b/library/cpp/yson/lexer.cpp @@ -1,16 +1,16 @@ -#include "lexer.h" -#include "lexer_detail.h" -#include "token.h" - -#include <util/generic/ptr.h> - +#include "lexer.h" +#include "lexer_detail.h" +#include "token.h" + +#include <util/generic/ptr.h> + namespace NYson { //////////////////////////////////////////////////////////////////////////////// - + class TStatelessLexer::TImpl { private: THolder<TStatelessYsonLexerImplBase> Impl; - + public: TImpl(bool enableLinePositionInfo = false) : Impl(enableLinePositionInfo @@ -18,26 +18,26 @@ namespace NYson { : static_cast<TStatelessYsonLexerImplBase*>(new TStatelesYsonLexerImpl<false>())) { } - + size_t GetToken(const TStringBuf& data, TToken* token) { return Impl->GetToken(data, token); } }; - + //////////////////////////////////////////////////////////////////////////////// TStatelessLexer::TStatelessLexer() : Impl(new TImpl()) - { - } - + { + } + TStatelessLexer::~TStatelessLexer() { } - + size_t TStatelessLexer::GetToken(const TStringBuf& data, TToken* token) { return Impl->GetToken(data, token); } - + //////////////////////////////////////////////////////////////////////////////// - + } // namespace NYson diff --git a/library/cpp/yson/lexer.h b/library/cpp/yson/lexer.h index 17cb87ec6d..d9d701874d 100644 --- a/library/cpp/yson/lexer.h +++ b/library/cpp/yson/lexer.h @@ -1,26 +1,26 @@ -#pragma once - -#include "public.h" -#include "token.h" - -#include <util/generic/ptr.h> - +#pragma once + +#include "public.h" +#include "token.h" + +#include <util/generic/ptr.h> + namespace NYson { //////////////////////////////////////////////////////////////////////////////// - + class TStatelessLexer { public: TStatelessLexer(); - + ~TStatelessLexer(); - + size_t GetToken(const TStringBuf& data, TToken* token); - + private: class TImpl; THolder<TImpl> Impl; }; - + //////////////////////////////////////////////////////////////////////////////// - + } // namespace NYson diff --git a/library/cpp/yson/lexer_detail.h b/library/cpp/yson/lexer_detail.h index 5bb5dccc06..0bba30acdd 100644 --- a/library/cpp/yson/lexer_detail.h +++ b/library/cpp/yson/lexer_detail.h @@ -1,15 +1,15 @@ -#pragma once - -#include "detail.h" -#include "token.h" - +#pragma once + +#include "detail.h" +#include "token.h" + namespace NYson { //////////////////////////////////////////////////////////////////////////////// - + namespace NDetail { /*! \internal */ //////////////////////////////////////////////////////////////////////////////// - + // EReadStartCase tree representation: // Root = xb // BinaryStringOrOtherSpecialToken = x0b @@ -29,17 +29,17 @@ namespace NYson { // Plus = 10011b // None = 10111b // Percent = 11011b - + enum EReadStartCase : unsigned { BinaryString = 0, // = 00b OtherSpecialToken = 2, // = 10b - + BinaryInt64 = 1, // = 001b BinaryDouble = 5, // = 101b BinaryFalse = 9, // = 1001b BinaryTrue = 13, // = 1101b BinaryUint64 = 17, // = 10001b - + Quote = 3, // = 00011b DigitOrMinus = 7, // = 00111b String = 11, // = 01011b @@ -48,34 +48,34 @@ namespace NYson { None = 23, // = 10111b Percent = 27 // = 11011b }; - + template <class TBlockStream, bool EnableLinePositionInfo> class TLexer : public TLexerBase<TBlockStream, EnableLinePositionInfo> { private: using TBase = TLexerBase<TBlockStream, EnableLinePositionInfo>; - + static EReadStartCase GetStartState(char ch) { -#define NN EReadStartCase::None -#define BS EReadStartCase::BinaryString -#define BI EReadStartCase::BinaryInt64 -#define BD EReadStartCase::BinaryDouble -#define BF EReadStartCase::BinaryFalse -#define BT EReadStartCase::BinaryTrue -#define BU EReadStartCase::BinaryUint64 -#define SP NN // EReadStartCase::Space -#define DM EReadStartCase::DigitOrMinus -#define ST EReadStartCase::String -#define PL EReadStartCase::Plus -#define QU EReadStartCase::Quote -#define PC EReadStartCase::Percent -#define TT(name) (EReadStartCase(static_cast<ui8>(ETokenType::name) << 2) | EReadStartCase::OtherSpecialToken) - +#define NN EReadStartCase::None +#define BS EReadStartCase::BinaryString +#define BI EReadStartCase::BinaryInt64 +#define BD EReadStartCase::BinaryDouble +#define BF EReadStartCase::BinaryFalse +#define BT EReadStartCase::BinaryTrue +#define BU EReadStartCase::BinaryUint64 +#define SP NN // EReadStartCase::Space +#define DM EReadStartCase::DigitOrMinus +#define ST EReadStartCase::String +#define PL EReadStartCase::Plus +#define QU EReadStartCase::Quote +#define PC EReadStartCase::Percent +#define TT(name) (EReadStartCase(static_cast<ui8>(ETokenType::name) << 2) | EReadStartCase::OtherSpecialToken) + static const ui8 lookupTable[] = { NN, BS, BI, BD, BF, BT, BU, NN, NN, SP, SP, SP, SP, SP, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, - + // 32 SP, // ' ' NN, // '!' @@ -93,7 +93,7 @@ namespace NYson { DM, // '-' NN, // '.' NN, // '/' - + // 48 DM, DM, DM, DM, DM, DM, DM, DM, DM, DM, // '0' - '9' TT(Colon), // ':' @@ -102,7 +102,7 @@ namespace NYson { TT(Equals), // '=' TT(RightAngle), // '>' NN, // '?' - + // 64 NN, // '@' ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, // 'A' - 'M' @@ -112,10 +112,10 @@ namespace NYson { TT(RightBracket), // ']' NN, // '^' ST, // '_' - + // 96 NN, // '`' - + ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, // 'a' - 'm' ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, // 'n' - 'z' TT(LeftBrace), // '{' @@ -128,41 +128,41 @@ namespace NYson { NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, - + NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN}; - -#undef NN -#undef BS -#undef BI -#undef BD -#undef SP -#undef DM -#undef ST -#undef PL -#undef QU -#undef TT + +#undef NN +#undef BS +#undef BI +#undef BD +#undef SP +#undef DM +#undef ST +#undef PL +#undef QU +#undef TT return static_cast<EReadStartCase>(lookupTable[static_cast<ui8>(ch)]); } - + public: TLexer(const TBlockStream& blockStream, TMaybe<ui64> memoryLimit) : TBase(blockStream, memoryLimit) { } - + void GetToken(TToken* token) { char ch1 = TBase::SkipSpaceAndGetChar(); auto state = GetStartState(ch1); auto stateBits = static_cast<unsigned>(state); - + if (ch1 == '\0') { *token = TToken::EndOfStream; return; } - + if (stateBits & 1) { // Other = x1b if (stateBits & 1 << 1) { // Other = xxx11b if (state == EReadStartCase::Quote) { @@ -174,9 +174,9 @@ namespace NYson { ReadNumeric<true>(token); } else if (state == EReadStartCase::Plus) { TBase::Advance(1); - + char ch2 = TBase::template GetChar<true>(); - + if (!isdigit(ch2)) { *token = TToken(ETokenType::Plus); } else { @@ -219,9 +219,9 @@ namespace NYson { } else { Y_FAIL("unreachable"); } - } + } } else { // BinaryStringOrOtherSpecialToken = x0b - TBase::Advance(1); + TBase::Advance(1); if (stateBits & 1 << 1) { // OtherSpecialToken = 10b Y_ASSERT((stateBits & 3) == static_cast<unsigned>(EReadStartCase::OtherSpecialToken)); *token = TToken(ETokenType(stateBits >> 2)); @@ -231,14 +231,14 @@ namespace NYson { TBase::ReadBinaryString(&value); *token = TToken(value); } - } - } - + } + } + template <bool AllowFinish> void ReadNumeric(TToken* token) { TStringBuf valueBuffer; ENumericResult numericResult = TBase::template ReadNumeric<AllowFinish>(&valueBuffer); - + if (numericResult == ENumericResult::Double) { try { *token = TToken(FromString<double>(valueBuffer)); @@ -258,39 +258,39 @@ namespace NYson { ythrow TYsonException() << "Error parsing uint64 literal " << valueBuffer; } } - } + } }; //////////////////////////////////////////////////////////////////////////////// /*! \endinternal */ - } - + } + class TStatelessYsonLexerImplBase { public: virtual size_t GetToken(const TStringBuf& data, TToken* token) = 0; - + virtual ~TStatelessYsonLexerImplBase() { } }; - + template <bool EnableLinePositionInfo> class TStatelesYsonLexerImpl: public TStatelessYsonLexerImplBase { private: using TLexer = NDetail::TLexer<TStringReader, EnableLinePositionInfo>; TLexer Lexer; - + public: TStatelesYsonLexerImpl() : Lexer(TStringReader(), Nothing()) { } - + size_t GetToken(const TStringBuf& data, TToken* token) override { Lexer.SetBuffer(data.begin(), data.end()); Lexer.GetToken(token); return Lexer.Begin() - data.begin(); } }; - + //////////////////////////////////////////////////////////////////////////////// - + } // namespace NYson diff --git a/library/cpp/yson/node/node.cpp b/library/cpp/yson/node/node.cpp index a533e34fa6..b39e070718 100644 --- a/library/cpp/yson/node/node.cpp +++ b/library/cpp/yson/node/node.cpp @@ -1,5 +1,5 @@ #include "node.h" - + #include "node_io.h" #include <library/cpp/yson/writer.h> @@ -13,15 +13,15 @@ namespace NYT { bool TNode::TNull::operator==(const TNull&) const { return true; } - -//////////////////////////////////////////////////////////////////////////////// - + +//////////////////////////////////////////////////////////////////////////////// + bool TNode::TUndefined::operator==(const TUndefined&) const { return true; } - + //////////////////////////////////////////////////////////////////////////////// - + namespace NNodeCmp { bool IsComparableType(const TNode::EType type) { @@ -94,15 +94,15 @@ bool operator>=(const TNode& lhs, const TNode& rhs) TNode::TNode() : Value_(TUndefined{}) { } - + TNode::TNode(const char* s) : Value_(TString(s)) { } - + TNode::TNode(TStringBuf s) : Value_(TString(s)) { } - + TNode::TNode(std::string_view s) : Value_(TString(s)) { } @@ -114,12 +114,12 @@ TNode::TNode(const std::string& s) TNode::TNode(TString s) : Value_(std::move(s)) { } - + TNode::TNode(int i) : Value_(static_cast<i64>(i)) { } - - + + TNode::TNode(unsigned int ui) : Value_(static_cast<ui64>(ui)) { } @@ -127,27 +127,27 @@ TNode::TNode(unsigned int ui) TNode::TNode(long i) : Value_(static_cast<i64>(i)) { } - + TNode::TNode(unsigned long ui) : Value_(static_cast<ui64>(ui)) { } - + TNode::TNode(long long i) : Value_(static_cast<i64>(i)) { } - + TNode::TNode(unsigned long long ui) : Value_(static_cast<ui64>(ui)) { } - + TNode::TNode(double d) : Value_(d) { } - + TNode::TNode(bool b) : Value_(b) { } - + TNode::TNode(TMapType map) : Value_(std::move(map)) { } @@ -161,39 +161,39 @@ TNode::TNode(const TNode& rhs) } Value_ = rhs.Value_; } - + TNode& TNode::operator=(const TNode& rhs) { if (this != &rhs) { TNode tmp = rhs; Move(std::move(tmp)); - } + } return *this; } - + TNode::TNode(TNode&& rhs) noexcept : TNode() { Move(std::move(rhs)); } - + TNode& TNode::operator=(TNode&& rhs) noexcept { if (this != &rhs) { TNode tmp = std::move(rhs); Move(std::move(tmp)); - } + } return *this; } - + TNode::~TNode() = default; - + void TNode::Clear() { ClearAttributes(); Value_ = TUndefined(); } - + bool TNode::IsString() const { return std::holds_alternative<TString>(Value_); @@ -203,37 +203,37 @@ bool TNode::IsInt64() const { return std::holds_alternative<i64>(Value_); } - + bool TNode::IsUint64() const { return std::holds_alternative<ui64>(Value_); } - + bool TNode::IsDouble() const { return std::holds_alternative<double>(Value_); } - + bool TNode::IsBool() const { return std::holds_alternative<bool>(Value_); } - + bool TNode::IsList() const { return std::holds_alternative<TListType>(Value_); } - + bool TNode::IsMap() const { return std::holds_alternative<TMapType>(Value_); } - + bool TNode::IsEntity() const { return IsNull(); } - + bool TNode::IsNull() const { return std::holds_alternative<TNull>(Value_); @@ -260,9 +260,9 @@ bool TNode::Empty() const return std::get<TMapType>(Value_).empty(); default: ythrow TTypeError() << "Empty() called for type " << GetType(); - } + } } - + size_t TNode::Size() const { switch (GetType()) { @@ -274,9 +274,9 @@ size_t TNode::Size() const return std::get<TMapType>(Value_).size(); default: ythrow TTypeError() << "Size() called for type " << GetType(); - } + } } - + TNode::EType TNode::GetType() const { return std::visit(TOverloaded{ @@ -291,61 +291,61 @@ TNode::EType TNode::GetType() const [](const TNull&) { return Null; } }, Value_); } - + const TString& TNode::AsString() const { CheckType(String); return std::get<TString>(Value_); } - + i64 TNode::AsInt64() const { CheckType(Int64); return std::get<i64>(Value_); } - + ui64 TNode::AsUint64() const { CheckType(Uint64); return std::get<ui64>(Value_); } - + double TNode::AsDouble() const { CheckType(Double); return std::get<double>(Value_); } - + bool TNode::AsBool() const { CheckType(Bool); return std::get<bool>(Value_); } - + const TNode::TListType& TNode::AsList() const { CheckType(List); return std::get<TListType>(Value_); } - + const TNode::TMapType& TNode::AsMap() const { CheckType(Map); return std::get<TMapType>(Value_); } - + TNode::TListType& TNode::AsList() { CheckType(List); return std::get<TListType>(Value_); } - + TNode::TMapType& TNode::AsMap() { CheckType(Map); return std::get<TMapType>(Value_); } - + const TString& TNode::UncheckedAsString() const noexcept { return std::get<TString>(Value_); @@ -397,7 +397,7 @@ TNode TNode::CreateList() node.Value_ = TListType{}; return node; } - + TNode TNode::CreateList(TListType list) { TNode node; @@ -411,7 +411,7 @@ TNode TNode::CreateMap() node.Value_ = TMapType{}; return node; } - + TNode TNode::CreateMap(TMapType map) { TNode node; @@ -425,19 +425,19 @@ TNode TNode::CreateEntity() node.Value_ = TNull{}; return node; } - + const TNode& TNode::operator[](size_t index) const { CheckType(List); return std::get<TListType>(Value_)[index]; } - + TNode& TNode::operator[](size_t index) { CheckType(List); return std::get<TListType>(Value_)[index]; } - + const TNode& TNode::At(size_t index) const { CheckType(List); const auto& list = std::get<TListType>(Value_); @@ -461,7 +461,7 @@ TNode& TNode::Add() & AssureList(); return std::get<TListType>(Value_).emplace_back(); } - + TNode TNode::Add() && { return std::move(Add()); @@ -473,7 +473,7 @@ TNode& TNode::Add(const TNode& node) & std::get<TListType>(Value_).emplace_back(node); return *this; } - + TNode TNode::Add(const TNode& node) && { return std::move(Add(node)); @@ -485,7 +485,7 @@ TNode& TNode::Add(TNode&& node) & std::get<TListType>(Value_).emplace_back(std::move(node)); return *this; } - + TNode TNode::Add(TNode&& node) && { return std::move(Add(std::move(node))); @@ -496,14 +496,14 @@ bool TNode::HasKey(const TStringBuf key) const CheckType(Map); return std::get<TMapType>(Value_).contains(key); } - + TNode& TNode::operator()(const TString& key, const TNode& value) & { AssureMap(); std::get<TMapType>(Value_)[key] = value; return *this; } - + TNode TNode::operator()(const TString& key, const TNode& value) && { return std::move(operator()(key, value)); @@ -515,7 +515,7 @@ TNode& TNode::operator()(const TString& key, TNode&& value) & std::get<TMapType>(Value_)[key] = std::move(value); return *this; } - + TNode TNode::operator()(const TString& key, TNode&& value) && { return std::move(operator()(key, std::move(value))); @@ -531,15 +531,15 @@ const TNode& TNode::operator[](const TStringBuf key) const return notFound; } else { return i->second; - } + } } - + TNode& TNode::operator[](const TStringBuf key) { AssureMap(); return std::get<TMapType>(Value_)[key]; } - + const TNode& TNode::At(const TStringBuf key) const { CheckType(Map); const auto& map = std::get<TMapType>(Value_); @@ -782,68 +782,68 @@ bool TNode::HasAttributes() const { return Attributes_ && !Attributes_->Empty(); } - + void TNode::ClearAttributes() { if (Attributes_) { Attributes_.Destroy(); - } + } } - + const TNode& TNode::GetAttributes() const { static TNode notFound = TNode::CreateMap(); if (!Attributes_) { return notFound; - } + } return *Attributes_; } - + TNode& TNode::Attributes() { if (!Attributes_) { CreateAttributes(); - } + } return *Attributes_; } - + void TNode::MoveWithoutAttributes(TNode&& rhs) { Value_ = std::move(rhs.Value_); rhs.Clear(); } - + void TNode::Move(TNode&& rhs) { Value_ = std::move(rhs.Value_); Attributes_ = std::move(rhs.Attributes_); } - + void TNode::CheckType(EType type) const { Y_ENSURE_EX(GetType() == type, TTypeError() << "TNode type " << type << " expected, actual type " << GetType(); ); } - + void TNode::AssureMap() { if (std::holds_alternative<TUndefined>(Value_)) { Value_ = TMapType(); } else { CheckType(Map); - } + } } - + void TNode::AssureList() { if (std::holds_alternative<TUndefined>(Value_)) { Value_ = TListType(); } else { CheckType(List); - } + } } - + void TNode::CreateAttributes() { Attributes_ = MakeHolder<TNode>(); @@ -862,54 +862,54 @@ void TNode::Load(IInputStream* in) } //////////////////////////////////////////////////////////////////////////////// - + bool operator==(const TNode& lhs, const TNode& rhs) { if (std::holds_alternative<TNode::TUndefined>(lhs.Value_) || std::holds_alternative<TNode::TUndefined>(rhs.Value_)) - { + { // TODO: should try to remove this behaviour if nobody uses it. return false; - } - + } + if (lhs.GetType() != rhs.GetType()) { - return false; - } - - if (lhs.Attributes_) { - if (rhs.Attributes_) { + return false; + } + + if (lhs.Attributes_) { + if (rhs.Attributes_) { if (*lhs.Attributes_ != *rhs.Attributes_) { return false; } - } else { - return false; - } - } else { - if (rhs.Attributes_) { - return false; - } - } - + } else { + return false; + } + } else { + if (rhs.Attributes_) { + return false; + } + } + return rhs.Value_ == lhs.Value_; -} - +} + bool operator!=(const TNode& lhs, const TNode& rhs) -{ - return !(lhs == rhs); -} - +{ + return !(lhs == rhs); +} + bool GetBool(const TNode& node) -{ - if (node.IsBool()) { - return node.AsBool(); - } else if (node.IsString()) { - return node.AsString() == "true"; - } else { +{ + if (node.IsBool()) { + return node.AsBool(); + } else if (node.IsString()) { + return node.AsString() == "true"; + } else { ythrow TNode::TTypeError() << "GetBool(): not a boolean or string type"; - } -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT + } +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT diff --git a/library/cpp/yson/node/node.h b/library/cpp/yson/node/node.h index bc7609e6da..5f90f95df0 100644 --- a/library/cpp/yson/node/node.h +++ b/library/cpp/yson/node/node.h @@ -1,11 +1,11 @@ -#pragma once - +#pragma once + #include <util/generic/bt_exception.h> #include <util/generic/cast.h> -#include <util/generic/hash.h> +#include <util/generic/hash.h> #include <util/generic/variant.h> -#include <util/generic/vector.h> -#include <util/generic/yexception.h> +#include <util/generic/vector.h> +#include <util/generic/yexception.h> #include <util/generic/ylimits.h> #include <util/string/cast.h> @@ -14,23 +14,23 @@ class IInputStream; class IOutputStream; - -namespace NYT { - -//////////////////////////////////////////////////////////////////////////////// - -class TNode -{ -public: + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +class TNode +{ +public: class TLookupError : public TWithBackTrace<yexception> { }; - class TTypeError + class TTypeError : public TWithBackTrace<yexception> - { }; - - enum EType { + { }; + + enum EType { Undefined = 0 /*"undefined"*/, // NOTE: string representation of all node types @@ -43,20 +43,20 @@ public: List = 6 /*"list_node"*/, Map = 7 /*"map_node"*/, Null = 8 /*"null"*/, - }; - + }; + using TListType = TVector<TNode>; using TMapType = THashMap<TString, TNode>; - + private: struct TNull { bool operator==(const TNull&) const; }; - + struct TUndefined { bool operator==(const TUndefined&) const; }; - + using TValue = std::variant< bool, i64, @@ -68,9 +68,9 @@ private: TNull, TUndefined >; - + public: - + TNode(); TNode(const char* s); TNode(TStringBuf s); @@ -78,7 +78,7 @@ public: explicit TNode(const std::string& s); TNode(TString s); TNode(int i); - + //this case made speccially for prevent mess cast of EType into TNode through TNode(int) constructor //usual case of error SomeNode == TNode::Undefined <-- SomeNode indeed will be compared with TNode(0) without this method //correct way is SomeNode.GetType() == TNode::Undefined @@ -102,17 +102,17 @@ public: TNode(double d); TNode(bool b); TNode(TMapType map); - + TNode(const TNode& rhs); TNode& operator=(const TNode& rhs); - + TNode(TNode&& rhs) noexcept; TNode& operator=(TNode&& rhs) noexcept; - + ~TNode(); - + void Clear(); - + bool IsString() const; bool IsInt64() const; bool IsUint64() const; @@ -127,7 +127,7 @@ public: bool IsUndefined() const; // Returns true if TNode is neither Null, nor Undefined bool HasValue() const; - + template<typename T> bool IsOfType() const noexcept; @@ -136,9 +136,9 @@ public: bool Empty() const; size_t Size() const; - + EType GetType() const; - + const TString& AsString() const; i64 AsInt64() const; ui64 AsUint64() const; @@ -148,7 +148,7 @@ public: const TMapType& AsMap() const; TListType& AsList(); TMapType& AsMap(); - + const TString& UncheckedAsString() const noexcept; i64 UncheckedAsInt64() const noexcept; ui64 UncheckedAsUint64() const noexcept; @@ -180,31 +180,31 @@ public: static TNode CreateMap(); static TNode CreateMap(TMapType map); static TNode CreateEntity(); - + const TNode& operator[](size_t index) const; TNode& operator[](size_t index); const TNode& At(size_t index) const; TNode& At(size_t index); - + TNode& Add() &; TNode Add() &&; TNode& Add(const TNode& node) &; TNode Add(const TNode& node) &&; TNode& Add(TNode&& node) &; TNode Add(TNode&& node) &&; - + bool HasKey(const TStringBuf key) const; TNode& operator()(const TString& key, const TNode& value) &; TNode operator()(const TString& key, const TNode& value) &&; TNode& operator()(const TString& key, TNode&& value) &; TNode operator()(const TString& key, TNode&& value) &&; - + const TNode& operator[](const TStringBuf key) const; TNode& operator[](const TStringBuf key); const TNode& At(const TStringBuf key) const; TNode& At(const TStringBuf key); - + // map getters // works the same way like simple getters const TString& ChildAsString(const TStringBuf key) const; @@ -254,42 +254,42 @@ public: T& ChildAs(size_t index); - // attributes + // attributes bool HasAttributes() const; void ClearAttributes(); const TNode& GetAttributes() const; TNode& Attributes(); - + void MoveWithoutAttributes(TNode&& rhs); - + // Serialize TNode using binary yson format. // Methods for ysaveload. void Save(IOutputStream* output) const; void Load(IInputStream* input); -private: +private: void Move(TNode&& rhs); - + void CheckType(EType type) const; - + void AssureMap(); void AssureList(); - + void CreateAttributes(); - -private: + +private: TValue Value_; THolder<TNode> Attributes_; - - friend bool operator==(const TNode& lhs, const TNode& rhs); - friend bool operator!=(const TNode& lhs, const TNode& rhs); -}; - + + friend bool operator==(const TNode& lhs, const TNode& rhs); + friend bool operator!=(const TNode& lhs, const TNode& rhs); +}; + bool operator==(const TNode& lhs, const TNode& rhs); bool operator!=(const TNode& lhs, const TNode& rhs); - + bool GetBool(const TNode& node); - + inline bool TNode::IsArithmetic() const { return IsInt64() || IsUint64() || IsDouble() || IsBool(); } @@ -508,8 +508,8 @@ inline const T& TNode::As() const { return std::get<T>(Value_); } -//////////////////////////////////////////////////////////////////////////////// - +//////////////////////////////////////////////////////////////////////////////// + namespace NNodeCmp { bool operator<(const TNode& lhs, const TNode& rhs); bool operator<=(const TNode& lhs, const TNode& rhs); @@ -520,4 +520,4 @@ namespace NNodeCmp { //////////////////////////////////////////////////////////////////////////////// -} // namespace NYT +} // namespace NYT diff --git a/library/cpp/yson/node/node_builder.cpp b/library/cpp/yson/node/node_builder.cpp index b21bc4502e..b4431bc77a 100644 --- a/library/cpp/yson/node/node_builder.cpp +++ b/library/cpp/yson/node/node_builder.cpp @@ -1,96 +1,96 @@ -#include "node_builder.h" - -namespace NYT { - -//////////////////////////////////////////////////////////////////////////////// - -TNodeBuilder::TNodeBuilder(TNode* node) -{ - Stack_.push(node); -} - +#include "node_builder.h" + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +TNodeBuilder::TNodeBuilder(TNode* node) +{ + Stack_.push(node); +} + void TNodeBuilder::OnStringScalar(TStringBuf value) -{ - AddNode(value, true); -} - -void TNodeBuilder::OnInt64Scalar(i64 value) -{ - AddNode(value, true); -} - -void TNodeBuilder::OnUint64Scalar(ui64 value) -{ - AddNode(value, true); -} - -void TNodeBuilder::OnDoubleScalar(double value) -{ - AddNode(value, true); -} - -void TNodeBuilder::OnBooleanScalar(bool value) -{ - AddNode(value, true); -} - -void TNodeBuilder::OnEntity() -{ - AddNode(TNode::CreateEntity(), true); -} - -void TNodeBuilder::OnBeginList() -{ - AddNode(TNode::CreateList(), false); -} - -void TNodeBuilder::OnListItem() -{ - Stack_.push(&Stack_.top()->Add()); -} - -void TNodeBuilder::OnEndList() -{ - Stack_.pop(); -} - -void TNodeBuilder::OnBeginMap() -{ - AddNode(TNode::CreateMap(), false); -} - +{ + AddNode(value, true); +} + +void TNodeBuilder::OnInt64Scalar(i64 value) +{ + AddNode(value, true); +} + +void TNodeBuilder::OnUint64Scalar(ui64 value) +{ + AddNode(value, true); +} + +void TNodeBuilder::OnDoubleScalar(double value) +{ + AddNode(value, true); +} + +void TNodeBuilder::OnBooleanScalar(bool value) +{ + AddNode(value, true); +} + +void TNodeBuilder::OnEntity() +{ + AddNode(TNode::CreateEntity(), true); +} + +void TNodeBuilder::OnBeginList() +{ + AddNode(TNode::CreateList(), false); +} + +void TNodeBuilder::OnListItem() +{ + Stack_.push(&Stack_.top()->Add()); +} + +void TNodeBuilder::OnEndList() +{ + Stack_.pop(); +} + +void TNodeBuilder::OnBeginMap() +{ + AddNode(TNode::CreateMap(), false); +} + void TNodeBuilder::OnKeyedItem(TStringBuf key) -{ +{ Stack_.push(&(*Stack_.top())[TString(key)]); -} - -void TNodeBuilder::OnEndMap() -{ - Stack_.pop(); -} - -void TNodeBuilder::OnBeginAttributes() -{ - Stack_.push(&Stack_.top()->Attributes()); -} - -void TNodeBuilder::OnEndAttributes() -{ - Stack_.pop(); -} - +} + +void TNodeBuilder::OnEndMap() +{ + Stack_.pop(); +} + +void TNodeBuilder::OnBeginAttributes() +{ + Stack_.push(&Stack_.top()->Attributes()); +} + +void TNodeBuilder::OnEndAttributes() +{ + Stack_.pop(); +} + void TNodeBuilder::OnNode(TNode node) { AddNode(std::move(node), true); } -void TNodeBuilder::AddNode(TNode value, bool pop) -{ +void TNodeBuilder::AddNode(TNode value, bool pop) +{ Stack_.top()->MoveWithoutAttributes(std::move(value)); - if (pop) - Stack_.pop(); -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT + if (pop) + Stack_.pop(); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT diff --git a/library/cpp/yson/node/node_builder.h b/library/cpp/yson/node/node_builder.h index 35bb256a5e..69800016e0 100644 --- a/library/cpp/yson/node/node_builder.h +++ b/library/cpp/yson/node/node_builder.h @@ -1,23 +1,23 @@ -#pragma once - +#pragma once + #include "node.h" - + #include <library/cpp/json/json_reader.h> #include <library/cpp/yson/consumer.h> -#include <util/generic/stack.h> - -namespace NYT { - -//////////////////////////////////////////////////////////////////////////////// - -class TNodeBuilder +#include <util/generic/stack.h> + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +class TNodeBuilder : public ::NYson::TYsonConsumerBase -{ -public: - TNodeBuilder(TNode* node); - +{ +public: + TNodeBuilder(TNode* node); + void OnStringScalar(TStringBuf) override; void OnInt64Scalar(i64) override; void OnUint64Scalar(ui64) override; @@ -33,14 +33,14 @@ public: void OnBeginAttributes() override; void OnEndAttributes() override; void OnNode(TNode node); - -private: + +private: TStack<TNode*> Stack_; - -private: - inline void AddNode(TNode node, bool pop); -}; - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT + +private: + inline void AddNode(TNode node, bool pop); +}; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT diff --git a/library/cpp/yson/node/node_io.cpp b/library/cpp/yson/node/node_io.cpp index ea53e4d8b7..294a7f7217 100644 --- a/library/cpp/yson/node/node_io.cpp +++ b/library/cpp/yson/node/node_io.cpp @@ -1,8 +1,8 @@ #include "node_io.h" - -#include "node_builder.h" -#include "node_visitor.h" - + +#include "node_builder.h" +#include "node_visitor.h" + #include <library/cpp/yson/json/json_writer.h> #include <library/cpp/yson/parser.h> #include <library/cpp/yson/writer.h> @@ -11,13 +11,13 @@ #include <library/cpp/json/json_reader.h> #include <library/cpp/json/json_value.h> -#include <util/stream/input.h> -#include <util/stream/output.h> -#include <util/stream/str.h> +#include <util/stream/input.h> +#include <util/stream/output.h> +#include <util/stream/str.h> #include <util/stream/mem.h> - -namespace NYT { - + +namespace NYT { + static void WalkJsonTree(const NJson::TJsonValue& jsonValue, NJson::TJsonCallbacks* callbacks) { using namespace NJson; @@ -68,20 +68,20 @@ static void WalkJsonTree(const NJson::TJsonValue& jsonValue, NJson::TJsonCallbac static TNode CreateEmptyNodeByType(::NYson::EYsonType type) { - TNode result; - switch (type) { + TNode result; + switch (type) { case ::NYson::EYsonType::ListFragment: - result = TNode::CreateList(); - break; + result = TNode::CreateList(); + break; case ::NYson::EYsonType::MapFragment: - result = TNode::CreateMap(); - break; - default: - break; - } + result = TNode::CreateMap(); + break; + default: + break; + } return result; } - + TNode NodeFromYsonString(const TStringBuf input, ::NYson::EYsonType type) { TMemoryInput stream(input); @@ -106,19 +106,19 @@ TNode NodeFromYsonStream(IInputStream* input, ::NYson::EYsonType type) { TNode result = CreateEmptyNodeByType(type); - TNodeBuilder builder(&result); + TNodeBuilder builder(&result); ::NYson::TYsonParser parser(&builder, input, type); - parser.Parse(); - return result; -} - + parser.Parse(); + return result; +} + void NodeToYsonStream(const TNode& node, IOutputStream* output, NYson::EYsonFormat format) -{ +{ ::NYson::TYsonWriter writer(output, format); - TNodeVisitor visitor(&writer); - visitor.Visit(node); -} - + TNodeVisitor visitor(&writer); + visitor.Visit(node); +} + void NodeToCanonicalYsonStream(const TNode& node, IOutputStream* output, NYson::EYsonFormat format) { ::NYson::TYsonWriter writer(output, format); @@ -149,6 +149,6 @@ TNode NodeFromJsonValue(const NJson::TJsonValue& input) return result; } -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT diff --git a/library/cpp/yson/node/node_io.h b/library/cpp/yson/node/node_io.h index 5b63243317..2ad23b658f 100644 --- a/library/cpp/yson/node/node_io.h +++ b/library/cpp/yson/node/node_io.h @@ -1,22 +1,22 @@ -#pragma once - +#pragma once + #include "node.h" #include <library/cpp/yson/public.h> - + namespace NJson { class TJsonValue; } // namespace NJson -namespace NYT { - -//////////////////////////////////////////////////////////////////////////////// - +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + // Parse TNode from string in YSON format TNode NodeFromYsonString(const TStringBuf input, ::NYson::EYsonType type = ::NYson::EYsonType::Node); // Serialize TNode to string in one of YSON formats with random order of maps' keys (don't use in tests) TString NodeToYsonString(const TNode& node, ::NYson::EYsonFormat format = ::NYson::EYsonFormat::Text); - + // Same as the latter, but maps' keys are sorted lexicographically (to be used in tests) TString NodeToCanonicalYsonString(const TNode& node, ::NYson::EYsonFormat format = ::NYson::EYsonFormat::Text); @@ -35,6 +35,6 @@ TNode NodeFromJsonString(const TStringBuf input); // Convert TJsonValue to TNode TNode NodeFromJsonValue(const NJson::TJsonValue& input); -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT diff --git a/library/cpp/yson/node/node_visitor.cpp b/library/cpp/yson/node/node_visitor.cpp index 824d934867..899fbfa02a 100644 --- a/library/cpp/yson/node/node_visitor.cpp +++ b/library/cpp/yson/node/node_visitor.cpp @@ -1,12 +1,12 @@ -#include "node_visitor.h" - +#include "node_visitor.h" + #include <util/generic/algorithm.h> #include <util/string/printf.h> -namespace NYT { - -//////////////////////////////////////////////////////////////////////////////// - +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + namespace { template <typename Fun> @@ -31,122 +31,122 @@ void Iterate(const TNode::TMapType& nodeMap, bool sortByKey, Fun action) //////////////////////////////////////////////////////////////////////////////// TNodeVisitor::TNodeVisitor(NYson::IYsonConsumer* consumer, bool sortMapKeys) - : Consumer_(consumer) + : Consumer_(consumer) , SortMapKeys_(sortMapKeys) -{ } - -void TNodeVisitor::Visit(const TNode& node) -{ - VisitAny(node); -} - -void TNodeVisitor::VisitAny(const TNode& node) -{ - if (node.HasAttributes()) { - Consumer_->OnBeginAttributes(); +{ } + +void TNodeVisitor::Visit(const TNode& node) +{ + VisitAny(node); +} + +void TNodeVisitor::VisitAny(const TNode& node) +{ + if (node.HasAttributes()) { + Consumer_->OnBeginAttributes(); Iterate(node.GetAttributes().AsMap(), SortMapKeys_, [&](const std::pair<TString, TNode>& item) { - Consumer_->OnKeyedItem(item.first); + Consumer_->OnKeyedItem(item.first); if (item.second.IsUndefined()) { ythrow TNode::TTypeError() << "unable to visit attribute value of type " << TNode::EType::Undefined << "; attribute name: `" << item.first << '\'' ; } - VisitAny(item.second); + VisitAny(item.second); }); - Consumer_->OnEndAttributes(); - } - - switch (node.GetType()) { + Consumer_->OnEndAttributes(); + } + + switch (node.GetType()) { case TNode::String: - VisitString(node); - break; + VisitString(node); + break; case TNode::Int64: - VisitInt64(node); - break; + VisitInt64(node); + break; case TNode::Uint64: - VisitUint64(node); - break; + VisitUint64(node); + break; case TNode::Double: - VisitDouble(node); - break; + VisitDouble(node); + break; case TNode::Bool: - VisitBool(node); - break; + VisitBool(node); + break; case TNode::List: VisitList(node.AsList()); - break; + break; case TNode::Map: VisitMap(node.AsMap()); - break; + break; case TNode::Null: - VisitEntity(); - break; + VisitEntity(); + break; case TNode::Undefined: ythrow TNode::TTypeError() << "unable to visit TNode of type " << node.GetType(); default: Y_FAIL("Unexpected type: %d", node.GetType()); - } -} - -void TNodeVisitor::VisitString(const TNode& node) -{ - Consumer_->OnStringScalar(node.AsString()); -} - -void TNodeVisitor::VisitInt64(const TNode& node) -{ - Consumer_->OnInt64Scalar(node.AsInt64()); -} - -void TNodeVisitor::VisitUint64(const TNode& node) -{ - Consumer_->OnUint64Scalar(node.AsUint64()); -} - -void TNodeVisitor::VisitDouble(const TNode& node) -{ - Consumer_->OnDoubleScalar(node.AsDouble()); -} - -void TNodeVisitor::VisitBool(const TNode& node) -{ - Consumer_->OnBooleanScalar(node.AsBool()); -} - + } +} + +void TNodeVisitor::VisitString(const TNode& node) +{ + Consumer_->OnStringScalar(node.AsString()); +} + +void TNodeVisitor::VisitInt64(const TNode& node) +{ + Consumer_->OnInt64Scalar(node.AsInt64()); +} + +void TNodeVisitor::VisitUint64(const TNode& node) +{ + Consumer_->OnUint64Scalar(node.AsUint64()); +} + +void TNodeVisitor::VisitDouble(const TNode& node) +{ + Consumer_->OnDoubleScalar(node.AsDouble()); +} + +void TNodeVisitor::VisitBool(const TNode& node) +{ + Consumer_->OnBooleanScalar(node.AsBool()); +} + void TNodeVisitor::VisitList(const TNode::TListType& nodeList) -{ - Consumer_->OnBeginList(); +{ + Consumer_->OnBeginList(); size_t index = 0; for (const auto& item : nodeList) { - Consumer_->OnListItem(); + Consumer_->OnListItem(); if (item.IsUndefined()) { ythrow TNode::TTypeError() << "unable to visit list node child of type " << TNode::EType::Undefined << "; list index: " << index; } - VisitAny(item); + VisitAny(item); ++index; - } - Consumer_->OnEndList(); -} - + } + Consumer_->OnEndList(); +} + void TNodeVisitor::VisitMap(const TNode::TMapType& nodeMap) -{ - Consumer_->OnBeginMap(); +{ + Consumer_->OnBeginMap(); Iterate(nodeMap, SortMapKeys_, [&](const std::pair<TString, TNode>& item) { - Consumer_->OnKeyedItem(item.first); + Consumer_->OnKeyedItem(item.first); if (item.second.IsUndefined()) { ythrow TNode::TTypeError() << "unable to visit map node child of type " << TNode::EType::Undefined << "; map key: `" << item.first << '\'' ; } - VisitAny(item.second); + VisitAny(item.second); }); - Consumer_->OnEndMap(); -} - -void TNodeVisitor::VisitEntity() -{ - Consumer_->OnEntity(); -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT + Consumer_->OnEndMap(); +} + +void TNodeVisitor::VisitEntity() +{ + Consumer_->OnEntity(); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT diff --git a/library/cpp/yson/node/node_visitor.h b/library/cpp/yson/node/node_visitor.h index d4eb8e5667..db25832309 100644 --- a/library/cpp/yson/node/node_visitor.h +++ b/library/cpp/yson/node/node_visitor.h @@ -1,37 +1,37 @@ -#pragma once - +#pragma once + #include "node.h" #include <library/cpp/yson/consumer.h> - -namespace NYT { - -//////////////////////////////////////////////////////////////////////////////// - -class TNodeVisitor -{ -public: + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +class TNodeVisitor +{ +public: TNodeVisitor(NYson::IYsonConsumer* consumer, bool sortMapKeys = false); - - void Visit(const TNode& node); + + void Visit(const TNode& node); void VisitMap(const TNode::TMapType& nodeMap); void VisitList(const TNode::TListType& nodeMap); - -private: + +private: NYson::IYsonConsumer* Consumer_; bool SortMapKeys_; - -private: - void VisitAny(const TNode& node); - - void VisitString(const TNode& node); - void VisitInt64(const TNode& node); - void VisitUint64(const TNode& node); - void VisitDouble(const TNode& node); - void VisitBool(const TNode& node); - void VisitEntity(); -}; - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT + +private: + void VisitAny(const TNode& node); + + void VisitString(const TNode& node); + void VisitInt64(const TNode& node); + void VisitUint64(const TNode& node); + void VisitDouble(const TNode& node); + void VisitBool(const TNode& node); + void VisitEntity(); +}; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT diff --git a/library/cpp/yson/parser.cpp b/library/cpp/yson/parser.cpp index 3af222ed4e..783f9b9047 100644 --- a/library/cpp/yson/parser.cpp +++ b/library/cpp/yson/parser.cpp @@ -1,14 +1,14 @@ -#include "parser.h" -#include "consumer.h" -#include "format.h" -#include "parser_detail.h" - -#include <util/stream/input.h> -#include <util/generic/buffer.h> - +#include "parser.h" +#include "consumer.h" +#include "format.h" +#include "parser_detail.h" + +#include <util/stream/input.h> +#include <util/generic/buffer.h> + namespace NYson { //////////////////////////////////////////////////////////////////////////////// - + class TYsonParser::TImpl { public: TImpl( @@ -24,7 +24,7 @@ namespace NYson { , MemoryLimit_(memoryLimit) { } - + void Parse() { TBuffer buffer(64 << 10); ParseYsonStreamImpl<NYT::NYson::IYsonConsumer, TStreamReader>( @@ -48,26 +48,26 @@ namespace NYson { TYsonParser::TYsonParser( NYT::NYson::IYsonConsumer* consumer, IInputStream* stream, - EYsonType type, - bool enableLinePositionInfo, + EYsonType type, + bool enableLinePositionInfo, TMaybe<ui64> memoryLimit) : Impl(new TImpl(consumer, stream, type, enableLinePositionInfo, memoryLimit)) - { - } - + { + } + TYsonParser::~TYsonParser() { } - + void TYsonParser::Parse() { Impl->Parse(); } - + //////////////////////////////////////////////////////////////////////////////// - + class TStatelessYsonParser::TImpl { private: THolder<TStatelessYsonParserImplBase> Impl; - + public: TImpl( NYT::NYson::IYsonConsumer* consumer, @@ -79,31 +79,31 @@ namespace NYson { : static_cast<TStatelessYsonParserImplBase*>(new TStatelessYsonParserImpl<NYT::NYson::IYsonConsumer, false>(consumer, memoryLimit))) { } - + void Parse(const TStringBuf& data, EYsonType type = ::NYson::EYsonType::Node) { Impl->Parse(data, type); } }; - + //////////////////////////////////////////////////////////////////////////////// - + TStatelessYsonParser::TStatelessYsonParser( NYT::NYson::IYsonConsumer* consumer, - bool enableLinePositionInfo, - TMaybe<ui64> memoryLimit) + bool enableLinePositionInfo, + TMaybe<ui64> memoryLimit) : Impl(new TImpl(consumer, enableLinePositionInfo, memoryLimit)) { } - + TStatelessYsonParser::~TStatelessYsonParser() { } void TStatelessYsonParser::Parse(const TStringBuf& data, EYsonType type) { - Impl->Parse(data, type); - } - + Impl->Parse(data, type); + } + //////////////////////////////////////////////////////////////////////////////// - + void ParseYsonStringBuffer( const TStringBuf& buffer, NYT::NYson::IYsonConsumer* consumer, @@ -117,9 +117,9 @@ namespace NYson { enableLinePositionInfo, memoryLimit); } - + //////////////////////////////////////////////////////////////////////////////// - + class TYsonListParser::TImpl { public: TImpl( @@ -135,7 +135,7 @@ namespace NYson { , Reader_(Stream_, Buffer_.Data(), Buffer_.Capacity()) { } - + bool Parse() { if (!Impl_) { Impl_.Reset( @@ -145,7 +145,7 @@ namespace NYson { } return Impl_->Parse(); } - + private: NYT::NYson::IYsonConsumer* Consumer_; IInputStream* Stream_; @@ -155,9 +155,9 @@ namespace NYson { TStreamReader Reader_; THolder<TYsonListParserImplBase> Impl_; }; - + //////////////////////////////////////////////////////////////////////////////// - + TYsonListParser::TYsonListParser( NYT::NYson::IYsonConsumer* consumer, IInputStream* stream, diff --git a/library/cpp/yson/parser.h b/library/cpp/yson/parser.h index 715341d5b5..dce35a8cd4 100644 --- a/library/cpp/yson/parser.h +++ b/library/cpp/yson/parser.h @@ -1,19 +1,19 @@ -#pragma once - -#include "public.h" - -#include <util/generic/maybe.h> -#include <util/generic/ptr.h> - +#pragma once + +#include "public.h" + +#include <util/generic/maybe.h> +#include <util/generic/ptr.h> + class IInputStream; - + namespace NYT::NYson { struct IYsonConsumer; } // namespace NYT::NYson namespace NYson { //////////////////////////////////////////////////////////////////////////////// - + class TYsonParser { public: TYsonParser( @@ -22,36 +22,36 @@ namespace NYson { EYsonType type = ::NYson::EYsonType::Node, bool enableLinePositionInfo = false, TMaybe<ui64> memoryLimit = Nothing()); - + ~TYsonParser(); - + void Parse(); - + private: class TImpl; THolder<TImpl> Impl; }; - + //////////////////////////////////////////////////////////////////////////////// - + class TStatelessYsonParser { public: TStatelessYsonParser( NYT::NYson::IYsonConsumer* consumer, bool enableLinePositionInfo = false, TMaybe<ui64> memoryLimit = Nothing()); - + ~TStatelessYsonParser(); - + void Parse(const TStringBuf& data, EYsonType type = ::NYson::EYsonType::Node); - + private: class TImpl; THolder<TImpl> Impl; }; - + //////////////////////////////////////////////////////////////////////////////// - + class TYsonListParser { public: TYsonListParser( @@ -59,7 +59,7 @@ namespace NYson { IInputStream* stream, bool enableLinePositionInfo = false, TMaybe<ui64> memoryLimit = Nothing()); - + ~TYsonListParser(); bool Parse(); // Returns false, if there is no more list items @@ -79,5 +79,5 @@ namespace NYson { TMaybe<ui64> memoryLimit = Nothing()); //////////////////////////////////////////////////////////////////////////////// - + } // namespace NYson diff --git a/library/cpp/yson/parser_detail.h b/library/cpp/yson/parser_detail.h index b6eee05664..44223caf12 100644 --- a/library/cpp/yson/parser_detail.h +++ b/library/cpp/yson/parser_detail.h @@ -1,43 +1,43 @@ -#pragma once - -#include "detail.h" - +#pragma once + +#include "detail.h" + namespace NYson { namespace NDetail { //////////////////////////////////////////////////////////////////////////////// - + template <class TConsumer, class TBlockStream, bool EnableLinePositionInfo> class TParser : public TLexerBase<TBlockStream, EnableLinePositionInfo> { private: using TBase = TLexerBase<TBlockStream, EnableLinePositionInfo>; TConsumer* Consumer; - + public: TParser(const TBlockStream& blockStream, TConsumer* consumer, TMaybe<ui64> memoryLimit) : TBase(blockStream, memoryLimit) , Consumer(consumer) { } - + void DoParse(EYsonType ysonType) { switch (ysonType) { case ::NYson::EYsonType::Node: ParseNode<true>(); break; - + case ::NYson::EYsonType::ListFragment: ParseListFragment<true>(EndSymbol); break; - + case ::NYson::EYsonType::MapFragment: ParseMapFragment<true>(EndSymbol); break; - + default: Y_FAIL("unreachable"); } - + while (!(TBase::IsFinished() && TBase::IsEmpty())) { if (TBase::template SkipSpaceAndGetChar<true>() != EndSymbol) { ythrow TYsonException() << "Stray '" << (*TBase::Begin()) << "' found"; @@ -46,12 +46,12 @@ namespace NYson { } } } - + bool DoParseListFragment(bool first) { bool ret = first ? first : ParseListSeparator<true>(EndSymbol); return ret && ParseListItem<true>(EndSymbol); - } - + } + void ParseAttributes() { Consumer->OnBeginAttributes(); ParseMapFragment(EndAttributesSymbol); @@ -65,19 +65,19 @@ namespace NYson { TBase::SkipCharToken(EndMapSymbol); Consumer->OnEndMap(); } - + void ParseList() { Consumer->OnBeginList(); ParseListFragment(EndListSymbol); TBase::SkipCharToken(EndListSymbol); Consumer->OnEndList(); } - + template <bool AllowFinish> void ParseNode() { return ParseNode<AllowFinish>(TBase::SkipSpaceAndGetChar()); } - + template <bool AllowFinish> void ParseNode(char ch) { if (ch == BeginAttributesSymbol) { @@ -85,18 +85,18 @@ namespace NYson { ParseAttributes(); ch = TBase::SkipSpaceAndGetChar(); } - + switch (ch) { case BeginMapSymbol: TBase::Advance(1); ParseMap(); break; - + case BeginListSymbol: TBase::Advance(1); ParseList(); break; - + case '"': { TBase::Advance(1); TStringBuf value; @@ -146,7 +146,7 @@ namespace NYson { TBase::Advance(1); Consumer->OnEntity(); break; - + default: { if (isdigit((unsigned char)ch) || ch == '-' || ch == '+') { // case of '+' is handled in AfterPlus state ReadNumeric<AllowFinish>(); @@ -167,11 +167,11 @@ namespace NYson { } } } - } + } void ParseKey() { return ParseKey(TBase::SkipSpaceAndGetChar()); - } + } void ParseKey(char ch) { switch (ch) { @@ -199,8 +199,8 @@ namespace NYson { } } } - } - + } + template <bool AllowFinish> void ParseMapFragment(char endSymbol) { char ch = TBase::template SkipSpaceAndGetChar<AllowFinish>(); @@ -221,13 +221,13 @@ namespace NYson { ythrow TYsonException() << "Expected '" << KeyedItemSeparatorSymbol << "' or '" << endSymbol << "' but '" << ch << "' found"; } - } - } - + } + } + void ParseMapFragment(char endSymbol) { ParseMapFragment<false>(endSymbol); } - + template <bool AllowFinish> bool ParseListItem(char endSymbol) { char ch = TBase::template SkipSpaceAndGetChar<AllowFinish>(); @@ -237,7 +237,7 @@ namespace NYson { return true; } return false; - } + } template <bool AllowFinish> bool ParseListSeparator(char endSymbol) { @@ -250,17 +250,17 @@ namespace NYson { << "' or '" << endSymbol << "' but '" << ch << "' found"; } return false; - } + } template <bool AllowFinish> void ParseListFragment(char endSymbol) { while (ParseListItem<AllowFinish>(endSymbol) && ParseListSeparator<AllowFinish>(endSymbol)) { - } - } - + } + } + void ParseListFragment(char endSymbol) { ParseListFragment<false>(endSymbol); - } + } template <bool AllowFinish> void ReadNumeric() { @@ -295,13 +295,13 @@ namespace NYson { } Consumer->OnUint64Scalar(value); } - } + } }; - + //////////////////////////////////////////////////////////////////////////////// - - } - + + } + template <class TConsumer, class TBlockStream> void ParseYsonStreamImpl( const TBlockStream& blockStream, @@ -317,9 +317,9 @@ namespace NYson { using TImpl = NDetail::TParser<TConsumer, TBlockStream, false>; TImpl impl(blockStream, consumer, memoryLimit); impl.DoParse(parsingMode); - } - } - + } + } + class TStatelessYsonParserImplBase { public: virtual void Parse(const TStringBuf& data, EYsonType type = ::NYson::EYsonType::Node) = 0; @@ -334,27 +334,27 @@ namespace NYson { private: using TParser = NDetail::TParser<TConsumer, TStringReader, EnableLinePositionInfo>; TParser Parser; - + public: TStatelessYsonParserImpl(TConsumer* consumer, TMaybe<ui64> memoryLimit) : Parser(TStringReader(), consumer, memoryLimit) { } - + void Parse(const TStringBuf& data, EYsonType type = ::NYson::EYsonType::Node) override { Parser.SetBuffer(data.begin(), data.end()); Parser.DoParse(type); - } + } }; - + class TYsonListParserImplBase { public: virtual bool Parse() = 0; - + virtual ~TYsonListParserImplBase() { } }; - + template <class TConsumer, class TBlockStream, bool EnableLinePositionInfo> class TYsonListParserImpl : public TYsonListParserImplBase { @@ -362,20 +362,20 @@ namespace NYson { using TParser = NDetail::TParser<TConsumer, TBlockStream, EnableLinePositionInfo>; TParser Parser; bool First = true; - + public: TYsonListParserImpl(const TBlockStream& blockStream, TConsumer* consumer, TMaybe<ui64> memoryLimit) : Parser(blockStream, consumer, memoryLimit) { } - + bool Parse() override { bool ret = Parser.DoParseListFragment(First); First = false; return ret; } }; - + //////////////////////////////////////////////////////////////////////////////// - + } // namespace NYson diff --git a/library/cpp/yson/public.h b/library/cpp/yson/public.h index 217fff9c8b..1ed793592b 100644 --- a/library/cpp/yson/public.h +++ b/library/cpp/yson/public.h @@ -1,22 +1,22 @@ -#pragma once - +#pragma once + #include <library/cpp/yt/misc/enum.h> -#include <util/generic/yexception.h> - +#include <util/generic/yexception.h> + #include <library/cpp/yt/yson_string/public.h> #include <library/cpp/yt/yson/public.h> namespace NYson { //////////////////////////////////////////////////////////////////////////////// - + using NYT::NYson::EYsonFormat; using NYT::NYson::EYsonType; - + class TYsonStringBuf; struct TYsonConsumerBase; - + class TYsonWriter; class TYsonParser; class TStatelessYsonParser; @@ -24,7 +24,7 @@ namespace NYson { class TYsonException : public yexception {}; - + //////////////////////////////////////////////////////////////////////////////// - + } // namespace NYson diff --git a/library/cpp/yson/token.cpp b/library/cpp/yson/token.cpp index ae9fd741b8..c8584c8c2e 100644 --- a/library/cpp/yson/token.cpp +++ b/library/cpp/yson/token.cpp @@ -1,11 +1,11 @@ -#include "token.h" - -#include <util/string/vector.h> -#include <util/string/printf.h> - +#include "token.h" + +#include <util/string/vector.h> +#include <util/string/printf.h> + namespace NYson { //////////////////////////////////////////////////////////////////////////////// - + ETokenType CharToTokenType(char ch) { switch (ch) { case ';': @@ -40,7 +40,7 @@ namespace NYson { return ETokenType::EndOfStream; } } - + char TokenTypeToChar(ETokenType type) { switch (type) { case ETokenType::Semicolon: @@ -74,16 +74,16 @@ namespace NYson { default: Y_FAIL("unreachable"); } - } - + } + TString TokenTypeToString(ETokenType type) { return TString(1, TokenTypeToChar(type)); - } - + } + //////////////////////////////////////////////////////////////////////////////// - + const TToken TToken::EndOfStream; - + TToken::TToken() : Type_(ETokenType::EndOfStream) , Int64Value(0) @@ -92,7 +92,7 @@ namespace NYson { , BooleanValue(false) { } - + TToken::TToken(ETokenType type) : Type_(type) , Int64Value(0) @@ -111,7 +111,7 @@ namespace NYson { break; } } - + TToken::TToken(const TStringBuf& stringValue) : Type_(ETokenType::String) , StringValue(stringValue) @@ -120,8 +120,8 @@ namespace NYson { , DoubleValue(0.0) , BooleanValue(false) { - } - + } + TToken::TToken(i64 int64Value) : Type_(ETokenType::Int64) , Int64Value(int64Value) @@ -129,7 +129,7 @@ namespace NYson { , DoubleValue(0.0) { } - + TToken::TToken(ui64 uint64Value) : Type_(ETokenType::Uint64) , Int64Value(0) @@ -138,7 +138,7 @@ namespace NYson { , BooleanValue(false) { } - + TToken::TToken(double doubleValue) : Type_(ETokenType::Double) , Int64Value(0) @@ -147,7 +147,7 @@ namespace NYson { , BooleanValue(false) { } - + TToken::TToken(bool booleanValue) : Type_(ETokenType::Boolean) , Int64Value(0) @@ -155,36 +155,36 @@ namespace NYson { , BooleanValue(booleanValue) { } - + bool TToken::IsEmpty() const { return Type_ == ETokenType::EndOfStream; } - + const TStringBuf& TToken::GetStringValue() const { CheckType(ETokenType::String); return StringValue; } - + i64 TToken::GetInt64Value() const { CheckType(ETokenType::Int64); return Int64Value; } - + ui64 TToken::GetUint64Value() const { CheckType(ETokenType::Uint64); return Uint64Value; } - + double TToken::GetDoubleValue() const { CheckType(ETokenType::Double); return DoubleValue; } - + bool TToken::GetBooleanValue() const { CheckType(ETokenType::Boolean); return BooleanValue; } - + void TToken::CheckType(ETokenType expectedType) const { if (Type_ != expectedType) { if (Type_ == ETokenType::EndOfStream) { @@ -194,9 +194,9 @@ namespace NYson { << "', Type: " << TokenTypeToString(Type_) << ", ExpectedType: " << TokenTypeToString(expectedType) << ")"; } - } - } - + } + } + void TToken::Reset() { Type_ = ETokenType::EndOfStream; Int64Value = 0; @@ -205,32 +205,32 @@ namespace NYson { StringValue = TStringBuf(); BooleanValue = false; } - + TString ToString(const TToken& token) { switch (token.GetType()) { case ETokenType::EndOfStream: return TString(); - + case ETokenType::String: return TString(token.GetStringValue()); - + case ETokenType::Int64: return ::ToString(token.GetInt64Value()); - + case ETokenType::Uint64: return ::ToString(token.GetUint64Value()); - + case ETokenType::Double: return ::ToString(token.GetDoubleValue()); - + case ETokenType::Boolean: return token.GetBooleanValue() ? "true" : "false"; - + default: return TokenTypeToString(token.GetType()); } - } - + } + //////////////////////////////////////////////////////////////////////////////// - + } // namespace NYson diff --git a/library/cpp/yson/token.h b/library/cpp/yson/token.h index ee455a29ca..7283e56950 100644 --- a/library/cpp/yson/token.h +++ b/library/cpp/yson/token.h @@ -1,21 +1,21 @@ -#pragma once - -#include "public.h" - -#include <util/generic/strbuf.h> - +#pragma once + +#include "public.h" + +#include <util/generic/strbuf.h> + namespace NYson { //////////////////////////////////////////////////////////////////////////////// - + enum ETokenType { EndOfStream, - + String, Int64, Uint64, Double, Boolean, - + // Special values: // YSON Semicolon, // ; @@ -27,7 +27,7 @@ namespace NYson { RightBrace, // } LeftAngle, // < RightAngle, // > - + // Table ranges LeftParenthesis, // ( RightParenthesis, // ) @@ -35,23 +35,23 @@ namespace NYson { Colon, // : Comma, // , }; - + //////////////////////////////////////////////////////////////////////////////// - + ETokenType CharToTokenType(char ch); char TokenTypeToChar(ETokenType type); TString TokenTypeToString(ETokenType type); - + //////////////////////////////////////////////////////////////////////////////// - + class TLexerImpl; - + //////////////////////////////////////////////////////////////////////////////// - + class TToken { public: static const TToken EndOfStream; - + TToken(); TToken(ETokenType type); explicit TToken(const TStringBuf& stringValue); @@ -59,35 +59,35 @@ namespace NYson { explicit TToken(ui64 int64Value); explicit TToken(double doubleValue); explicit TToken(bool booleanValue); - + ETokenType GetType() const { return Type_; } - + bool IsEmpty() const; const TStringBuf& GetStringValue() const; i64 GetInt64Value() const; ui64 GetUint64Value() const; double GetDoubleValue() const; bool GetBooleanValue() const; - + void CheckType(ETokenType expectedType) const; void Reset(); - + private: friend class TLexerImpl; - + ETokenType Type_; - + TStringBuf StringValue; i64 Int64Value; ui64 Uint64Value; double DoubleValue; bool BooleanValue; }; - + TString ToString(const TToken& token); - + //////////////////////////////////////////////////////////////////////////////// - + } // namespace NYson diff --git a/library/cpp/yson/tokenizer.cpp b/library/cpp/yson/tokenizer.cpp index 595700e62f..06760170d4 100644 --- a/library/cpp/yson/tokenizer.cpp +++ b/library/cpp/yson/tokenizer.cpp @@ -1,37 +1,37 @@ -#include "tokenizer.h" - +#include "tokenizer.h" + namespace NYson { //////////////////////////////////////////////////////////////////////////////// - + TTokenizer::TTokenizer(const TStringBuf& input) : Input(input) , Parsed(0) { } - + bool TTokenizer::ParseNext() { Input = Input.Tail(Parsed); Token.Reset(); Parsed = Lexer.GetToken(Input, &Token); return !CurrentToken().IsEmpty(); } - + const TToken& TTokenizer::CurrentToken() const { return Token; } - + ETokenType TTokenizer::GetCurrentType() const { return CurrentToken().GetType(); } - + TStringBuf TTokenizer::GetCurrentSuffix() const { return Input.Tail(Parsed); } - + const TStringBuf& TTokenizer::CurrentInput() const { return Input; } - + //////////////////////////////////////////////////////////////////////////////// } // namespace NYson diff --git a/library/cpp/yson/tokenizer.h b/library/cpp/yson/tokenizer.h index 25e42ab9fc..0576aace95 100644 --- a/library/cpp/yson/tokenizer.h +++ b/library/cpp/yson/tokenizer.h @@ -1,28 +1,28 @@ -#pragma once - -#include "public.h" -#include "lexer.h" - +#pragma once + +#include "public.h" +#include "lexer.h" + namespace NYson { //////////////////////////////////////////////////////////////////////////////// - + class TTokenizer { public: explicit TTokenizer(const TStringBuf& input); - + bool ParseNext(); const TToken& CurrentToken() const; ETokenType GetCurrentType() const; TStringBuf GetCurrentSuffix() const; const TStringBuf& CurrentInput() const; - + private: TStringBuf Input; TToken Token; TStatelessLexer Lexer; size_t Parsed; }; - + //////////////////////////////////////////////////////////////////////////////// - + } // namespace NYson diff --git a/library/cpp/yson/varint.cpp b/library/cpp/yson/varint.cpp index b962dcede7..d538ee3cff 100644 --- a/library/cpp/yson/varint.cpp +++ b/library/cpp/yson/varint.cpp @@ -1,12 +1,12 @@ #include "varint.h" -#include "zigzag.h" - +#include "zigzag.h" + #include <util/generic/yexception.h> - + namespace NYson { //////////////////////////////////////////////////////////////////////////////// - + int WriteVarUInt64(IOutputStream* output, ui64 value) { bool stop = false; int bytesWritten = 0; @@ -19,22 +19,22 @@ namespace NYson { byte &= 0x7F; } output->Write(byte); - } + } return bytesWritten; - } - + } + int WriteVarInt32(IOutputStream* output, i32 value) { return WriteVarUInt64(output, static_cast<ui64>(ZigZagEncode32(value))); } - + int WriteVarInt64(IOutputStream* output, i64 value) { return WriteVarUInt64(output, static_cast<ui64>(ZigZagEncode64(value))); } - + int ReadVarUInt64(IInputStream* input, ui64* value) { size_t count = 0; ui64 result = 0; - + ui8 byte = 0; do { if (7 * count > 8 * sizeof(ui64)) { @@ -56,16 +56,16 @@ namespace NYson { int bytesRead = ReadVarUInt64(input, &varInt); if (varInt > Max<ui32>()) { ythrow yexception() << "The data is too long to read ui64"; - } + } *value = ZigZagDecode32(static_cast<ui32>(varInt)); return bytesRead; } - + int ReadVarInt64(IInputStream* input, i64* value) { ui64 varInt; int bytesRead = ReadVarUInt64(input, &varInt); *value = ZigZagDecode64(varInt); return bytesRead; - } - + } + } // namespace NYson diff --git a/library/cpp/yson/writer.cpp b/library/cpp/yson/writer.cpp index 9f0386e213..054459f9f5 100644 --- a/library/cpp/yson/writer.cpp +++ b/library/cpp/yson/writer.cpp @@ -1,18 +1,18 @@ -#include "writer.h" +#include "writer.h" #include "detail.h" -#include "format.h" +#include "format.h" #include "parser.h" #include "varint.h" -#include "zigzag.h" - -#include <util/string/cast.h> - +#include "zigzag.h" + +#include <util/string/cast.h> + #include <cmath> namespace NYson { //////////////////////////////////////////////////////////////////////////////// - + // Copied from <util/string/escape.cpp> namespace { inline char HexDigit(char value) { @@ -22,26 +22,26 @@ namespace NYson { else return 'A' + value - 10; } - + inline char OctDigit(char value) { Y_ASSERT(value < 8); return '0' + value; } - + inline bool IsPrintable(char c) { return c >= 32 && c <= 126; } - + inline bool IsHexDigit(char c) { return (c >= '0' && c <= '9') || (c >= 'A' && c <= 'F') || (c >= 'a' && c <= 'f'); } - + inline bool IsOctDigit(char c) { return c >= '0' && c <= '7'; } - + const size_t ESCAPE_C_BUFFER_SIZE = 4; - + inline size_t EscapeC(unsigned char c, char next, char r[ESCAPE_C_BUFFER_SIZE]) { // (1) Printable characters go as-is, except backslash and double quote. // (2) Characters \r, \n, \t and \0 ... \7 replaced by their simple escape characters (if possible). @@ -87,33 +87,33 @@ namespace NYson { return 4; } } - + void EscapeC(const char* str, size_t len, IOutputStream& output) { char buffer[ESCAPE_C_BUFFER_SIZE]; - + size_t i, j; for (i = 0, j = 0; i < len; ++i) { size_t rlen = EscapeC(str[i], (i + 1 < len ? str[i + 1] : 0), buffer); - + if (rlen > 1) { output.Write(str + j, i - j); j = i + 1; output.Write(buffer, rlen); } } - + if (j > 0) { output.Write(str + j, len - j); } else { output.Write(str, len); } - } - + } + TString FloatToStringWithNanInf(double value) { if (std::isfinite(value)) { return ::ToString(value); } - + static const TStringBuf nanLiteral = "%nan"; static const TStringBuf infLiteral = "%inf"; static const TStringBuf negativeInfLiteral = "%-inf"; @@ -130,7 +130,7 @@ namespace NYson { } } - + //////////////////////////////////////////////////////////////////////////////// TYsonWriter::TYsonWriter( @@ -147,17 +147,17 @@ namespace NYson { { Y_ASSERT(stream); } - + void TYsonWriter::WriteIndent() { for (int i = 0; i < IndentSize * Depth; ++i) { Stream->Write(' '); } } - + bool TYsonWriter::IsTopLevelFragmentContext() const { return Depth == 0 && (Type == ::NYson::EYsonType::ListFragment || Type == ::NYson::EYsonType::MapFragment); - } - + } + void TYsonWriter::EndNode() { if (IsTopLevelFragmentContext()) { ETokenType separatorToken = @@ -168,15 +168,15 @@ namespace NYson { if (Format == EYsonFormat::Text || Format == EYsonFormat::Pretty) { Stream->Write('\n'); } - } - } - + } + } + void TYsonWriter::BeginCollection(ETokenType beginToken) { Stream->Write(TokenTypeToChar(beginToken)); ++Depth; BeforeFirstItem = true; } - + void TYsonWriter::CollectionItem(ETokenType separatorToken) { if (!IsTopLevelFragmentContext()) { if (!BeforeFirstItem) { @@ -187,21 +187,21 @@ namespace NYson { Stream->Write('\n'); WriteIndent(); } - } - + } + BeforeFirstItem = false; } void TYsonWriter::EndCollection(ETokenType endToken) { --Depth; if (Format == EYsonFormat::Pretty && !BeforeFirstItem) { - Stream->Write('\n'); - WriteIndent(); - } + Stream->Write('\n'); + WriteIndent(); + } Stream->Write(TokenTypeToChar(endToken)); BeforeFirstItem = false; - } - + } + void TYsonWriter::WriteStringScalar(const TStringBuf& value) { if (Format == EYsonFormat::Binary) { Stream->Write(NDetail::StringMarker); @@ -213,12 +213,12 @@ namespace NYson { Stream->Write('"'); } } - + void TYsonWriter::OnStringScalar(TStringBuf value) { WriteStringScalar(value); EndNode(); - } - + } + void TYsonWriter::OnInt64Scalar(i64 value) { if (Format == EYsonFormat::Binary) { Stream->Write(NDetail::Int64Marker); @@ -227,8 +227,8 @@ namespace NYson { Stream->Write(::ToString(value)); } EndNode(); - } - + } + void TYsonWriter::OnUint64Scalar(ui64 value) { if (Format == EYsonFormat::Binary) { Stream->Write(NDetail::Uint64Marker); @@ -238,8 +238,8 @@ namespace NYson { Stream->Write("u"); } EndNode(); - } - + } + void TYsonWriter::OnDoubleScalar(double value) { if (Format == EYsonFormat::Binary) { Stream->Write(NDetail::DoubleMarker); @@ -252,44 +252,44 @@ namespace NYson { } } EndNode(); - } - + } + void TYsonWriter::OnBooleanScalar(bool value) { if (Format == EYsonFormat::Binary) { Stream->Write(value ? NDetail::TrueMarker : NDetail::FalseMarker); } else { Stream->Write(value ? "%true" : "%false"); - } + } EndNode(); - } - + } + void TYsonWriter::OnEntity() { Stream->Write(TokenTypeToChar(EntityToken)); EndNode(); - } - + } + void TYsonWriter::OnBeginList() { BeginCollection(BeginListToken); } - + void TYsonWriter::OnListItem() { CollectionItem(ListItemSeparatorToken); } - + void TYsonWriter::OnEndList() { EndCollection(EndListToken); EndNode(); } - + void TYsonWriter::OnBeginMap() { BeginCollection(BeginMapToken); } - + void TYsonWriter::OnKeyedItem(TStringBuf key) { CollectionItem(KeyedItemSeparatorToken); - + WriteStringScalar(key); - + if (Format == NYson::EYsonFormat::Pretty) { Stream->Write(' '); } @@ -297,26 +297,26 @@ namespace NYson { if (Format == NYson::EYsonFormat::Pretty) { Stream->Write(' '); } - + BeforeFirstItem = false; - } + } void TYsonWriter::OnEndMap() { EndCollection(EndMapToken); EndNode(); - } - + } + void TYsonWriter::OnBeginAttributes() { BeginCollection(BeginAttributesToken); } - + void TYsonWriter::OnEndAttributes() { EndCollection(EndAttributesToken); if (Format == NYson::EYsonFormat::Pretty) { Stream->Write(' '); } } - + void TYsonWriter::OnRaw(TStringBuf yson, EYsonType type) { if (EnableRaw) { Stream->Write(yson); @@ -325,19 +325,19 @@ namespace NYson { TYsonConsumerBase::OnRaw(yson, type); } } - + TYsonWriter::TState TYsonWriter::State() const { TState state; state.Depth = Depth; state.BeforeFirstItem = BeforeFirstItem; return state; - } - + } + void TYsonWriter::Reset(const TState& state) { Depth = state.Depth; BeforeFirstItem = state.BeforeFirstItem; - } - + } + //////////////////////////////////////////////////////////////////////////////// void ReformatYsonStream( @@ -351,5 +351,5 @@ namespace NYson { } //////////////////////////////////////////////////////////////////////////////// - + } // namespace NYson diff --git a/library/cpp/yson/writer.h b/library/cpp/yson/writer.h index 0ae04be6e8..40f5d7d501 100644 --- a/library/cpp/yson/writer.h +++ b/library/cpp/yson/writer.h @@ -1,17 +1,17 @@ -#pragma once - -#include "public.h" -#include "token.h" -#include "consumer.h" - -#include <util/generic/noncopyable.h> - +#pragma once + +#include "public.h" +#include "token.h" +#include "consumer.h" + +#include <util/generic/noncopyable.h> + class IOutputStream; class IZeroCopyInput; - + namespace NYson { //////////////////////////////////////////////////////////////////////////////// - + class TYsonWriter : public TYsonConsumerBase, private TNonCopyable { @@ -20,7 +20,7 @@ namespace NYson { private: int Depth; bool BeforeFirstItem; - + friend class TYsonWriter; }; @@ -37,23 +37,23 @@ namespace NYson { void OnDoubleScalar(double value) override; void OnBooleanScalar(bool value) override; void OnEntity() override; - + void OnBeginList() override; void OnListItem() override; void OnEndList() override; - + void OnBeginMap() override; void OnKeyedItem(TStringBuf key) override; void OnEndMap() override; - + void OnBeginAttributes() override; void OnEndAttributes() override; - + void OnRaw(TStringBuf yson, EYsonType type = ::NYson::EYsonType::Node) override; - + TState State() const; void Reset(const TState& state); - + protected: IOutputStream* Stream; EYsonFormat Format; @@ -62,28 +62,28 @@ namespace NYson { int Depth; bool BeforeFirstItem; - + static const int IndentSize = 4; - + void WriteIndent(); void WriteStringScalar(const TStringBuf& value); - + void BeginCollection(ETokenType beginToken); void CollectionItem(ETokenType separatorToken); void EndCollection(ETokenType endToken); - + bool IsTopLevelFragmentContext() const; void EndNode(); }; - + //////////////////////////////////////////////////////////////////////////////// - + void ReformatYsonStream( IInputStream* input, IOutputStream* output, EYsonFormat format = EYsonFormat::Binary, EYsonType type = ::NYson::EYsonType::Node); - + //////////////////////////////////////////////////////////////////////////////// - + } // namespace NYson diff --git a/library/cpp/yson/ya.make b/library/cpp/yson/ya.make index b30109a31f..c55a189b10 100644 --- a/library/cpp/yson/ya.make +++ b/library/cpp/yson/ya.make @@ -1,23 +1,23 @@ -LIBRARY() - +LIBRARY() + OWNER( ermolovd g:yt ) - + PEERDIR( library/cpp/yt/misc library/cpp/yt/yson ) -SRCS( +SRCS( consumer.cpp - lexer.cpp + lexer.cpp parser.cpp - token.cpp - tokenizer.cpp + token.cpp + tokenizer.cpp varint.cpp - writer.cpp -) - -END() + writer.cpp +) + +END() diff --git a/library/cpp/yson/zigzag.h b/library/cpp/yson/zigzag.h index dc130a7410..2f1190508f 100644 --- a/library/cpp/yson/zigzag.h +++ b/library/cpp/yson/zigzag.h @@ -1,31 +1,31 @@ -#pragma once - -#include <util/system/defaults.h> - +#pragma once + +#include <util/system/defaults.h> + namespace NYson { //////////////////////////////////////////////////////////////////////////////// - + //! Functions that provide coding of integers with property: 0 <= f(x) <= 2 * |x| //! Actually taken 'as is' from protobuf/wire_format_lite.h - + inline ui32 ZigZagEncode32(i32 n) { // Note: the right-shift must be arithmetic return (ui32(n) << 1) ^ (n >> 31); } - + inline i32 ZigZagDecode32(ui32 n) { return (n >> 1) ^ -static_cast<i32>(n & 1); } - + inline ui64 ZigZagEncode64(i64 n) { // Note: the right-shift must be arithmetic return (ui64(n) << 1) ^ (n >> 63); } - + inline i64 ZigZagDecode64(ui64 n) { return (n >> 1) ^ -static_cast<i64>(n & 1); } - + //////////////////////////////////////////////////////////////////////////////// } // namespace NYson |